From f0c153969ccf2e1e7489bc4766bed1ec39ce21f1 Mon Sep 17 00:00:00 2001 From: Anbraten Date: Tue, 31 Oct 2023 19:21:02 +0100 Subject: [PATCH 1/5] Stop steps after they are done --- pipeline/backend/docker/docker.go | 16 ++++++++++++++++ pipeline/backend/kubernetes/kubernetes.go | 19 +++++++++++++++++++ pipeline/backend/local/local.go | 5 +++++ pipeline/backend/types/engine.go | 3 +++ pipeline/pipeline.go | 4 ++++ 5 files changed, 47 insertions(+) diff --git a/pipeline/backend/docker/docker.go b/pipeline/backend/docker/docker.go index ba255d805cb..ef2ab40068d 100644 --- a/pipeline/backend/docker/docker.go +++ b/pipeline/backend/docker/docker.go @@ -291,6 +291,22 @@ func (e *docker) TailStep(ctx context.Context, step *backend.Step, taskUUID stri return rc, nil } +func (e *docker) StopStep(ctx context.Context, step *backend.Step, taskUUID string) error { + log.Trace().Str("taskUUID", taskUUID).Msgf("stop step %s", step.Name) + + containerName := toContainerName(step) + + if err := e.client.ContainerKill(ctx, containerName, "9"); err != nil && !isErrContainerNotFoundOrNotRunning(err) { + return err + } + + if err := e.client.ContainerRemove(ctx, containerName, removeOpts); err != nil && !isErrContainerNotFoundOrNotRunning(err) { + return err + } + + return nil +} + func (e *docker) DestroyWorkflow(_ context.Context, conf *backend.Config, taskUUID string) error { log.Trace().Str("taskUUID", taskUUID).Msgf("delete workflow environment") diff --git a/pipeline/backend/kubernetes/kubernetes.go b/pipeline/backend/kubernetes/kubernetes.go index 458a3f08c05..0c4865ed444 100644 --- a/pipeline/backend/kubernetes/kubernetes.go +++ b/pipeline/backend/kubernetes/kubernetes.go @@ -325,6 +325,25 @@ func (e *kube) TailStep(ctx context.Context, step *types.Step, taskUUID string) // return rc, nil } +func (e *kube) StopStep(ctx context.Context, step *types.Step, taskUUID string) error { + podName, err := dnsName(step.Name) + if err != nil { + return err + } + + log.Trace().Str("taskUUID", taskUUID).Msgf("Stopping pod: %s", podName) + + gracePeriodSeconds := int64(0) // immediately + dpb := metav1.DeletePropagationBackground + + deleteOpts := metav1.DeleteOptions{ + GracePeriodSeconds: &gracePeriodSeconds, + PropagationPolicy: &dpb, + } + + return e.client.CoreV1().Pods(e.config.Namespace).Delete(ctx, podName, deleteOpts) +} + // Destroy the pipeline environment. func (e *kube) DestroyWorkflow(_ context.Context, conf *types.Config, taskUUID string) error { log.Trace().Str("taskUUID", taskUUID).Msg("Deleting Kubernetes primitives") diff --git a/pipeline/backend/local/local.go b/pipeline/backend/local/local.go index f2a68eb7d09..3614eb7016e 100644 --- a/pipeline/backend/local/local.go +++ b/pipeline/backend/local/local.go @@ -215,6 +215,11 @@ func (e *local) TailStep(_ context.Context, step *types.Step, taskUUID string) ( return e.output, nil } +func (e *local) StopStep(_ context.Context, _ *types.Step, _ string) error { + // WaitStep already waits for the command to finish, so there is nothing to do here. + return nil +} + // DestroyWorkflow the pipeline environment. func (e *local) DestroyWorkflow(_ context.Context, _ *types.Config, taskUUID string) error { log.Trace().Str("taskUUID", taskUUID).Msgf("delete workflow environment") diff --git a/pipeline/backend/types/engine.go b/pipeline/backend/types/engine.go index 0c1a39dd5f4..95491843280 100644 --- a/pipeline/backend/types/engine.go +++ b/pipeline/backend/types/engine.go @@ -44,6 +44,9 @@ type Engine interface { // TailStep the workflow step logs. TailStep(ctx context.Context, step *Step, taskUUID string) (io.ReadCloser, error) + // StopStep stop the workflow step. + StopStep(ctx context.Context, step *Step, taskUUID string) error + // DestroyWorkflow the workflow environment. DestroyWorkflow(ctx context.Context, conf *Config, taskUUID string) error } diff --git a/pipeline/pipeline.go b/pipeline/pipeline.go index d107b1e4c5c..b02a45f66b6 100644 --- a/pipeline/pipeline.go +++ b/pipeline/pipeline.go @@ -274,6 +274,10 @@ func (r *Runtime) exec(step *backend.Step) (*backend.State, error) { return nil, err } + if err := r.engine.StopStep(r.ctx, step, r.taskUUID); err != nil { + return nil, err + } + if waitState.OOMKilled { return waitState, &OomError{ Name: step.Name, From fc769b577d7b62854e2e226f34bf9c6f87445ce8 Mon Sep 17 00:00:00 2001 From: Anbraten Date: Tue, 31 Oct 2023 19:35:35 +0100 Subject: [PATCH 2/5] ignore not found --- pipeline/backend/kubernetes/kubernetes.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pipeline/backend/kubernetes/kubernetes.go b/pipeline/backend/kubernetes/kubernetes.go index 0c4865ed444..1cf5db58d6b 100644 --- a/pipeline/backend/kubernetes/kubernetes.go +++ b/pipeline/backend/kubernetes/kubernetes.go @@ -341,7 +341,11 @@ func (e *kube) StopStep(ctx context.Context, step *types.Step, taskUUID string) PropagationPolicy: &dpb, } - return e.client.CoreV1().Pods(e.config.Namespace).Delete(ctx, podName, deleteOpts) + if err := e.client.CoreV1().Pods(e.config.Namespace).Delete(ctx, podName, deleteOpts); err != nil && !errors.IsNotFound(err) { + return err + } + + return nil } // Destroy the pipeline environment. From d5b5ac893dea497d1c5b7a2973674fae5ef9df76 Mon Sep 17 00:00:00 2001 From: Anbraten Date: Tue, 31 Oct 2023 19:37:38 +0100 Subject: [PATCH 3/5] rename func --- pipeline/backend/docker/docker.go | 2 +- pipeline/backend/kubernetes/kubernetes.go | 2 +- pipeline/backend/local/local.go | 2 +- pipeline/backend/types/engine.go | 16 ++++++++-------- pipeline/pipeline.go | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pipeline/backend/docker/docker.go b/pipeline/backend/docker/docker.go index ef2ab40068d..7f19f5bc8fe 100644 --- a/pipeline/backend/docker/docker.go +++ b/pipeline/backend/docker/docker.go @@ -291,7 +291,7 @@ func (e *docker) TailStep(ctx context.Context, step *backend.Step, taskUUID stri return rc, nil } -func (e *docker) StopStep(ctx context.Context, step *backend.Step, taskUUID string) error { +func (e *docker) DestroyStep(ctx context.Context, step *backend.Step, taskUUID string) error { log.Trace().Str("taskUUID", taskUUID).Msgf("stop step %s", step.Name) containerName := toContainerName(step) diff --git a/pipeline/backend/kubernetes/kubernetes.go b/pipeline/backend/kubernetes/kubernetes.go index 1cf5db58d6b..dd81bec907a 100644 --- a/pipeline/backend/kubernetes/kubernetes.go +++ b/pipeline/backend/kubernetes/kubernetes.go @@ -325,7 +325,7 @@ func (e *kube) TailStep(ctx context.Context, step *types.Step, taskUUID string) // return rc, nil } -func (e *kube) StopStep(ctx context.Context, step *types.Step, taskUUID string) error { +func (e *kube) DestroyStep(ctx context.Context, step *types.Step, taskUUID string) error { podName, err := dnsName(step.Name) if err != nil { return err diff --git a/pipeline/backend/local/local.go b/pipeline/backend/local/local.go index 3614eb7016e..f90a44b0af1 100644 --- a/pipeline/backend/local/local.go +++ b/pipeline/backend/local/local.go @@ -215,7 +215,7 @@ func (e *local) TailStep(_ context.Context, step *types.Step, taskUUID string) ( return e.output, nil } -func (e *local) StopStep(_ context.Context, _ *types.Step, _ string) error { +func (e *local) DestroyStep(_ context.Context, _ *types.Step, _ string) error { // WaitStep already waits for the command to finish, so there is nothing to do here. return nil } diff --git a/pipeline/backend/types/engine.go b/pipeline/backend/types/engine.go index 95491843280..b06e9a981d0 100644 --- a/pipeline/backend/types/engine.go +++ b/pipeline/backend/types/engine.go @@ -28,25 +28,25 @@ type Engine interface { // IsAvailable check if the backend is available. IsAvailable(ctx context.Context) bool - // Load the backend engine. + // Load loads the backend engine. Load(ctx context.Context) error - // SetupWorkflow the workflow environment. + // SetupWorkflow sets up the workflow environment. SetupWorkflow(ctx context.Context, conf *Config, taskUUID string) error - // StartStep start the workflow step. + // StartStep starts the workflow step. StartStep(ctx context.Context, step *Step, taskUUID string) error - // WaitStep for the workflow step to complete and returns + // WaitStep waits for the workflow step to complete and returns // the completion results. WaitStep(ctx context.Context, step *Step, taskUUID string) (*State, error) - // TailStep the workflow step logs. + // TailStep tails the workflow step logs. TailStep(ctx context.Context, step *Step, taskUUID string) (io.ReadCloser, error) - // StopStep stop the workflow step. - StopStep(ctx context.Context, step *Step, taskUUID string) error + // DestroyStep destroys the workflow step. + DestroyStep(ctx context.Context, step *Step, taskUUID string) error - // DestroyWorkflow the workflow environment. + // DestroyWorkflow destroys the workflow environment. DestroyWorkflow(ctx context.Context, conf *Config, taskUUID string) error } diff --git a/pipeline/pipeline.go b/pipeline/pipeline.go index b02a45f66b6..f5d9d5fd838 100644 --- a/pipeline/pipeline.go +++ b/pipeline/pipeline.go @@ -274,7 +274,7 @@ func (r *Runtime) exec(step *backend.Step) (*backend.State, error) { return nil, err } - if err := r.engine.StopStep(r.ctx, step, r.taskUUID); err != nil { + if err := r.engine.DestroyStep(r.ctx, step, r.taskUUID); err != nil { return nil, err } From ddda4352a6c11a3daea4ab14a04c7a06fbc05ba1 Mon Sep 17 00:00:00 2001 From: Anbraten Date: Tue, 31 Oct 2023 19:42:55 +0100 Subject: [PATCH 4/5] removed log --- pipeline/backend/kubernetes/kubernetes.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pipeline/backend/kubernetes/kubernetes.go b/pipeline/backend/kubernetes/kubernetes.go index dd81bec907a..5b9709392da 100644 --- a/pipeline/backend/kubernetes/kubernetes.go +++ b/pipeline/backend/kubernetes/kubernetes.go @@ -372,9 +372,7 @@ func (e *kube) DestroyWorkflow(_ context.Context, conf *types.Config, taskUUID s } log.Trace().Msgf("Deleting pod: %s", stepName) if err := e.client.CoreV1().Pods(e.config.Namespace).Delete(noContext, stepName, deleteOpts); err != nil { - if errors.IsNotFound(err) { - log.Trace().Err(err).Msgf("Unable to delete pod %s", stepName) - } else { + if !errors.IsNotFound(err) { return err } } From 781c24bedc5f80e7f7eb47a8edaa4e0d958e66b2 Mon Sep 17 00:00:00 2001 From: Anbraten Date: Wed, 1 Nov 2023 09:20:15 +0100 Subject: [PATCH 5/5] remove unused login --- .woodpecker/docker.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.woodpecker/docker.yml b/.woodpecker/docker.yml index 8def35137e8..b64406f0a1f 100644 --- a/.woodpecker/docker.yml +++ b/.woodpecker/docker.yml @@ -135,7 +135,6 @@ steps: repo: woodpeckerci/woodpecker-server dockerfile: docker/Dockerfile.server.multiarch platforms: *platforms_preview - logins: *publish_logins tag: pull_${CI_COMMIT_PULL_REQUEST} when: evaluate: 'not (CI_COMMIT_PULL_REQUEST_LABELS contains "build_pr_images")'