Skip to content

Commit

Permalink
fix delete gc-artifact workflow stuck due to finalizers can not be …
Browse files Browse the repository at this point in the history
…removed

Signed-off-by: joey <zchengjoey@gmail.com>
  • Loading branch information
chengjoey committed Aug 26, 2024
1 parent dcd9436 commit 7185fc7
Show file tree
Hide file tree
Showing 3 changed files with 37 additions and 5 deletions.
8 changes: 8 additions & 0 deletions test/e2e/artifacts_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -388,6 +388,14 @@ func (s *ArtifactsSuite) TestArtifactGC() {
artifactState{s3Location{bucketName: "my-bucket", derivedKey: &artifactDerivedKey{templateName: "artifact-written", artifactName: "present"}}, false, true},
},
},
// Workflow defined output artifact but execution failed, no artifacts to be gced
{
workflowFile: "@testdata/artifactgc/artgc-artifact-not-written-failed.yaml",
hasGC: true,
workflowShouldSucceed: false,
expectedGCPodsOnWFCompletion: 0,
expectedArtifacts: []artifactState{},
},
} {
// for each test make sure that:
// 1. the finalizer gets added
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
generateName: artgc-artifact-not-written-failed-
spec:
entrypoint: main
artifactGC:
strategy: OnWorkflowDeletion
templates:
- name: main
container:
image: argoproj/argosay:v2
command:
- sh
- -c
args:
- |
echo "intentionally not writing anything to disk and intentional failure"
exit 1
outputs:
artifacts:
- name: on-completion
path: /tmp/on-completion.txt
s3:
key: on-completion.txt
artifactGC:
strategy: OnWorkflowCompletion
7 changes: 2 additions & 5 deletions workflow/controller/artifact_gc.go
Original file line number Diff line number Diff line change
Expand Up @@ -508,7 +508,6 @@ func (woc *wfOperationCtx) processArtifactGCCompletion(ctx context.Context) erro
return fmt.Errorf("failed to get pods from informer: %w", err)
}

anyPodSuccess := false
for _, obj := range pods {
pod := obj.(*corev1.Pod)
if pod.Labels[common.LabelKeyComponent] != artifactGCComponent { // make sure it's an Artifact GC Pod
Expand All @@ -534,10 +533,8 @@ func (woc *wfOperationCtx) processArtifactGCCompletion(ctx context.Context) erro
if err != nil {
return err
}

woc.wf.Status.ArtifactGCStatus.SetArtifactGCPodRecouped(pod.Name, true)
if phase == corev1.PodSucceeded {
anyPodSuccess = true
}
woc.updated = true
}
}
Expand All @@ -548,7 +545,7 @@ func (woc *wfOperationCtx) processArtifactGCCompletion(ctx context.Context) erro
removeFinalizer = woc.wf.Status.ArtifactGCStatus.AllArtifactGCPodsRecouped()
} else {
// check if all artifacts have been deleted and if so remove Finalizer
removeFinalizer = anyPodSuccess && woc.allArtifactsDeleted()
removeFinalizer = woc.allArtifactsDeleted()
}
if removeFinalizer {
woc.log.Infof("no remaining artifacts to GC, removing artifact GC finalizer (forceFinalizerRemoval=%v)", forceFinalizerRemoval)
Expand Down

0 comments on commit 7185fc7

Please sign in to comment.