fix: analysis runs to wait for all metrics to complete

analysis/analysis.go

@@ -396,6 +396,14 @@ func (c *Controller) assessRunStatus(run *v1alpha1.AnalysisRun) (v1alpha1.Analys
 					}
 				}
 			}
+		} else {
+			// metric hasn't started running. possible cases where some of the metrics starts with delay
+			everythingCompleted = false
+			if terminating {
+				// we have yet to take a single measurement, but have already been instructed to stop
+				log.Infof("metric assessed %s: run terminated", v1alpha1.AnalysisPhaseSuccessful)
+				return v1alpha1.AnalysisPhaseSuccessful, worstMessage
+			}
 		}
 	}
 	if !everythingCompleted {

test/e2e/analysis_test.go

@@ -25,6 +25,7 @@ func (s *AnalysisSuite) SetupSuite() {
 	// shared analysis templates for suite
 	s.ApplyManifests("@functional/analysistemplate-web-background.yaml")
 	s.ApplyManifests("@functional/analysistemplate-sleep-job.yaml")
+	s.ApplyManifests("@functional/analysistemplate-multiple-job.yaml")
 }
 
 // convenience to generate a new service with a given name
@@ -84,6 +85,28 @@ func (s *AnalysisSuite) TestCanaryInlineAnalysis() {
 		ExpectAnalysisRunCount(3)
 }
 
+func (s *AnalysisSuite) TestCanaryInlineMultipleAnalysis() {
+	s.Given().
+		RolloutObjects("@functional/rollout-inline-multiple-analysis.yaml").
+		When().
+		ApplyManifests().
+		WaitForRolloutStatus("Healthy").
+		Then().
+		ExpectAnalysisRunCount(0).
+		When().
+		UpdateSpec().
+		WaitForRolloutStatus("Paused").
+		PromoteRollout().
+		Sleep(5 * time.Second).
+		Then().
+		ExpectAnalysisRunCount(1).
+		ExpectInlineAnalysisRunPhase("Running").
+		When().
+		WaitForInlineAnalysisRunPhase("Successful").
+		WaitForRolloutStatus("Healthy").
+		Then().
+		ExpectAnalysisRunCount(1)
+}
 // TestBlueGreenAnalysis tests blue-green with pre/post analysis and then fast-tracked rollback
 func (s *AnalysisSuite) TestBlueGreenAnalysis() {
 	original := `

test/e2e/functional/analysistemplate-multiple-job.yaml

@@ -0,0 +1,43 @@
+# AnalysisTemplate which sleeps for a specified duration and exits with a specified exit-code
+kind: AnalysisTemplate
+apiVersion: argoproj.io/v1alpha1
+metadata:
+  name: multiple-job
+spec:
+  args:
+  - name: duration
+    value: 0s
+  - name: exit-code
+    value: "0"
+  - name: count
+    value: "1"
+  metrics:
+  - name: sleep-job
+    initialDelay: 10s
+    count: 1
+    provider:
+      job:
+        spec:
+          template:
+            spec:
+              containers:
+              - name: sleep-job
+                image: nginx:1.19-alpine
+                command: [sh, -c, -x]
+                args: ["sleep {{args.duration}} && exit {{args.exit-code}}"]
+              restartPolicy: Never
+          backoffLimit: 0
+  - name: sleep-job-rep
+    count: 1
+    provider:
+      job:
+        spec:
+          template:
+            spec:
+              containers:
+              - name: sleep-job
+                image: nginx:1.19-alpine
+                command: [sh, -c, -x]
+                args: ["sleep {{args.duration}} && exit {{args.exit-code}}"]
+              restartPolicy: Never
+          backoffLimit: 0

test/e2e/functional/analysistemplate-sleep-job.yaml

@@ -25,4 +25,4 @@ spec:
                 command: [sh, -c, -x]
                 args: ["sleep {{args.duration}} && exit {{args.exit-code}}"]
               restartPolicy: Never
-          backoffLimit: 0
+          backoffLimit: 0

test/e2e/functional/rollout-inline-multiple-analysis.yaml

@@ -0,0 +1,28 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Rollout
+metadata:
+  name: rollout-inline-analysis
+spec:
+  strategy:
+    canary:
+      steps:
+      - setWeight: 10
+      - pause: {}
+      - analysis:
+          templates:
+          - templateName: multiple-job
+  selector:
+    matchLabels:
+      app: rollout-inline-analysis
+  template:
+    metadata:
+      labels:
+        app: rollout-inline-analysis
+    spec:
+      containers:
+      - name: rollouts-demo
+        image: nginx:1.19-alpine
+        resources:
+          requests:
+            memory: 16Mi
+            cpu: 5m

test/fixtures/then.go

@@ -257,6 +257,26 @@ func (t *Then) ExpectBackgroundAnalysisRunPhase(phase string) *Then {
 	)
 }
 
+func (t *Then) ExpectInlineAnalysisRun(expectation string, expectFunc AnalysisRunExpectation) *Then {
+	t.t.Helper()
+	bgArun := t.GetInlineAnalysisRun()
+	if !expectFunc(bgArun) {
+		t.log.Errorf("Inline AnalysisRun expectation '%s' failed", expectation)
+		t.t.FailNow()
+	}
+	t.log.Infof("Inline AnalysisRun expectation '%s' met", expectation)
+	return t
+}
+
+func (t *Then) ExpectInlineAnalysisRunPhase(phase string) *Then {
+	t.t.Helper()
+	return t.ExpectInlineAnalysisRun(fmt.Sprintf("inline analysis phase == %s", phase),
+		func(run *rov1.AnalysisRun) bool {
+			return string(run.Status.Phase) == phase
+		},
+	)
+}
+
 // ExpectStableRevision verifies the ReplicaSet with the specified revision is marked stable
 func (t *Then) ExpectStableRevision(revision string) *Then {
 	t.t.Helper()