Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Create RolloutPaused condition #1054

Merged
merged 6 commits into from
Apr 8, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions pkg/apis/rollouts/v1alpha1/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -639,6 +639,8 @@ const (
// RolloutReplicaFailure ReplicaFailure is added in a deployment when one of its pods
// fails to be created or deleted.
RolloutReplicaFailure RolloutConditionType = "ReplicaFailure"
// RolloutPaused means that rollout is in a paused state. It is still progressing at this point.
RolloutPaused RolloutConditionType = "Paused"
)

// RolloutCondition describes the state of a rollout at a certain point.
Expand Down
53 changes: 44 additions & 9 deletions rollout/analysis_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1443,7 +1443,11 @@ func TestDoNotCreateBackgroundAnalysisRunAfterInconclusiveRun(t *testing.T) {
Reason: v1alpha1.PauseReasonInconclusiveAnalysis,
StartTime: metav1.Now(),
}}
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")

progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

f.rolloutLister = append(f.rolloutLister, r2)
Expand All @@ -1454,6 +1458,13 @@ func TestDoNotCreateBackgroundAnalysisRunAfterInconclusiveRun(t *testing.T) {
f.run(getKey(r2, t))

patch := f.getPatchedRollout(patchIndex)

//expectedPatch := fmt.Sprintf(`{
// "status":{
// "conditions":[%s, %s],
// "observedGeneration": ""
// }
//}`, progressingConditionString, pausedConditionString)
assert.Equal(t, calculatePatch(r2, OnlyObservedGenerationPatch), patch)
}

Expand Down Expand Up @@ -1543,7 +1554,10 @@ func TestCreatePrePromotionAnalysisRun(t *testing.T) {
rs2PodHash := rs2.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]

r2 = updateBlueGreenRolloutStatus(r2, rs2PodHash, rs1PodHash, rs1PodHash, 1, 1, 2, 1, true, true)
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

previewSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs2PodHash}
Expand Down Expand Up @@ -1723,7 +1737,10 @@ func TestRolloutPrePromotionAnalysisBecomesInconclusive(t *testing.T) {
Name: ar.Name,
Status: v1alpha1.AnalysisPhaseRunning,
}
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

activeSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs1PodHash}
Expand Down Expand Up @@ -1788,7 +1805,10 @@ func TestRolloutPrePromotionAnalysisSwitchServiceAfterSuccess(t *testing.T) {
Name: ar.Name,
Status: v1alpha1.AnalysisPhaseRunning,
}
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

activeSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs1PodHash}
Expand Down Expand Up @@ -1851,7 +1871,10 @@ func TestRolloutPrePromotionAnalysisHonorAutoPromotionSeconds(t *testing.T) {
r2 = updateBlueGreenRolloutStatus(r2, "", rs1PodHash, rs1PodHash, 1, 1, 2, 1, true, true)
now := metav1.NewTime(metav1.Now().Add(-10 * time.Second))
r2.Status.PauseConditions[0].StartTime = now
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

activeSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs1PodHash}
Expand Down Expand Up @@ -1914,7 +1937,10 @@ func TestRolloutPrePromotionAnalysisDoNothingOnInconclusiveAnalysis(t *testing.T
}
r2.Status.PauseConditions = append(r2.Status.PauseConditions, inconclusivePauseCondition)
r2.Status.ObservedGeneration = strconv.Itoa(int(r2.Generation))
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

activeSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs1PodHash}
Expand Down Expand Up @@ -1957,7 +1983,10 @@ func TestAbortRolloutOnErrorPrePromotionAnalysis(t *testing.T) {
rs1PodHash := rs1.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]

r2 = updateBlueGreenRolloutStatus(r2, "", rs1PodHash, rs1PodHash, 1, 1, 2, 1, true, true)
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

activeSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs1PodHash}
Expand Down Expand Up @@ -2122,7 +2151,10 @@ func TestPostPromotionAnalysisRunHandleInconclusive(t *testing.T) {
Reason: v1alpha1.PauseReasonInconclusiveAnalysis,
StartTime: metav1.Now(),
}}
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

activeSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs2PodHash}
Expand Down Expand Up @@ -2175,7 +2207,10 @@ func TestAbortRolloutOnErrorPostPromotionAnalysis(t *testing.T) {
rs2PodHash := rs2.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]

r2 = updateBlueGreenRolloutStatus(r2, "", rs2PodHash, rs1PodHash, 1, 1, 2, 1, true, true)
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

activeSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs2PodHash}
Expand Down
32 changes: 25 additions & 7 deletions rollout/bluegreen_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ func TestBlueGreenHandlePause(t *testing.T) {
"conditions": %s
}
}`
addedConditions := generateConditionsPatch(true, conditions.PausedRolloutReason, rs2, true, "")
addedConditions := generateConditionsPatchWithPause(true, conditions.PausedRolloutReason, rs2, true, "", true)
assert.Equal(t, calculatePatch(r2, fmt.Sprintf(expectedPatch, addedConditions)), patch)
})

Expand All @@ -266,7 +266,10 @@ func TestBlueGreenHandlePause(t *testing.T) {
rs2PodHash := rs2.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]

r2 = updateBlueGreenRolloutStatus(r2, rs2PodHash, rs1PodHash, rs1PodHash, 1, 1, 2, 1, true, true)
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

previewSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs2PodHash}
Expand Down Expand Up @@ -310,7 +313,10 @@ func TestBlueGreenHandlePause(t *testing.T) {
Reason: v1alpha1.PauseReasonInconclusiveAnalysis,
StartTime: now,
})
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

previewSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs2PodHash}
Expand Down Expand Up @@ -347,7 +353,10 @@ func TestBlueGreenHandlePause(t *testing.T) {
rs2PodHash := rs2.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]

r2 = updateBlueGreenRolloutStatus(r2, rs2PodHash, rs1PodHash, rs1PodHash, 1, 1, 2, 1, true, true)
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

previewSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs2PodHash}
Expand Down Expand Up @@ -385,7 +394,10 @@ func TestBlueGreenHandlePause(t *testing.T) {
before := metav1.NewTime(now.Add(-1 * time.Minute))
r2.Status.PauseConditions[0].StartTime = before
r2.Status.ControllerPause = true
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

activeSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs1PodHash}
Expand Down Expand Up @@ -439,7 +451,10 @@ func TestBlueGreenHandlePause(t *testing.T) {
r2.Status.PauseConditions[0].StartTime = before
r2.Status.ControllerPause = true
r2.Spec.Paused = true
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

activeSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs1PodHash}
Expand Down Expand Up @@ -731,7 +746,10 @@ func TestBlueGreenRolloutStatusHPAStatusFieldsActiveSelectorSet(t *testing.T) {

r2 = updateBlueGreenRolloutStatus(r2, rs2PodHash, rs1PodHash, rs1PodHash, 0, 0, 0, 0, true, false)
r2.Status.Selector = ""
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

f.rolloutLister = append(f.rolloutLister, r2)
Expand Down
45 changes: 32 additions & 13 deletions rollout/canary_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,9 @@ func TestCanaryRolloutNoProgressWhilePaused(t *testing.T) {
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

rs1 := newReplicaSetWithStatus(r1, 10, 10)
rs2 := newReplicaSetWithStatus(r2, 0, 0)

Expand Down Expand Up @@ -216,9 +219,12 @@ func TestCanaryRolloutUpdatePauseConditionWhilePaused(t *testing.T) {
r1 := newCanaryRollout("foo", 10, nil, steps, pointer.Int32Ptr(0), intstr.FromInt(1), intstr.FromInt(0))
r2 := bumpVersion(r1)

progressingCondition, _ := newProgressingCondition(conditions.ReplicaSetUpdatedReason, r2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

rs1 := newReplicaSetWithStatus(r1, 10, 10)
rs2 := newReplicaSetWithStatus(r2, 0, 0)

Expand All @@ -231,17 +237,10 @@ func TestCanaryRolloutUpdatePauseConditionWhilePaused(t *testing.T) {
f.objects = append(f.objects, r2)

addPausedConditionPatch := f.expectPatchRolloutAction(r2)
f.expectPatchRolloutAction(r2)
f.run(getKey(r2, t))

patch := f.getPatchedRollout(addPausedConditionPatch)
_, pausedCondition := newProgressingCondition(conditions.PausedRolloutReason, rs2, "")
expectedPatch := fmt.Sprintf(`{
"status": {
"conditions": [%s]
}
}`, pausedCondition)
assert.Equal(t, calculatePatch(r2, expectedPatch), patch)
assert.Equal(t, calculatePatch(r2, OnlyObservedGenerationPatch), patch)
}

func TestCanaryRolloutResetProgressDeadlineOnRetry(t *testing.T) {
Expand Down Expand Up @@ -885,6 +884,9 @@ func TestSyncRolloutWaitAddToQueue(t *testing.T) {
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

r2.Status.ObservedGeneration = strconv.Itoa(int(r2.Generation))
f.rolloutLister = append(f.rolloutLister, r2)
f.objects = append(f.objects, r2)
Expand Down Expand Up @@ -927,6 +929,9 @@ func TestSyncRolloutIgnoreWaitOutsideOfReconciliationPeriod(t *testing.T) {
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

f.rolloutLister = append(f.rolloutLister, r2)
f.objects = append(f.objects, r2)

Expand Down Expand Up @@ -964,7 +969,10 @@ func TestSyncRolloutWaitIncrementStepIndex(t *testing.T) {
f.replicaSetLister = append(f.replicaSetLister, rs1, rs2)

r2 = updateCanaryRolloutStatus(r2, rs1PodHash, 10, 1, 10, false)
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

earlier := metav1.Now()
Expand Down Expand Up @@ -1008,6 +1016,9 @@ func TestCanaryRolloutStatusHPAStatusFields(t *testing.T) {
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

rs1 := newReplicaSetWithStatus(r1, 4, 4)
rs1PodHash := rs1.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]
rs2 := newReplicaSetWithStatus(r2, 1, 1)
Expand Down Expand Up @@ -1249,7 +1260,10 @@ func TestCanaryRolloutScaleWhilePaused(t *testing.T) {

r2 = updateCanaryRolloutStatus(r2, rs1PodHash, 5, 0, 5, true)
r2.Spec.Replicas = pointer.Int32Ptr(10)
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

f.rolloutLister = append(f.rolloutLister, r2)
Expand Down Expand Up @@ -1343,7 +1357,9 @@ func TestNoResumeAfterPauseDurationIfUserPaused(t *testing.T) {
Reason: v1alpha1.PauseReasonCanaryPauseStep,
StartTime: overAMinuteAgo,
}}
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs1, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs1, "")
conditions.SetRolloutCondition(&r1.Status, progressingCondition)
pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r1.Status, pausedCondition)
r1.Spec.Paused = true
f.kubeobjects = append(f.kubeobjects, rs1)
Expand Down Expand Up @@ -1380,7 +1396,10 @@ func TestHandleNilNewRSOnScaleAndImageChange(t *testing.T) {
r2 := bumpVersion(r1)
r2.Spec.Replicas = pointer.Int32Ptr(3)
r2 = updateCanaryRolloutStatus(r2, rs1PodHash, 3, 0, 3, true)
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs1, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs1, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

f.kubeobjects = append(f.kubeobjects, rs1)
Expand Down
30 changes: 30 additions & 0 deletions rollout/controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,26 @@ func newReplicaSetWithStatus(r *v1alpha1.Rollout, replicas int, availableReplica
return rs
}

func newPausedCondition(isPaused bool) (v1alpha1.RolloutCondition, string) {
status := corev1.ConditionTrue
if !isPaused {
status = corev1.ConditionFalse
}
condition := v1alpha1.RolloutCondition{
LastTransitionTime: metav1.Now(),
LastUpdateTime: metav1.Now(),
Message: conditions.PausedRolloutMessage,
Reason: conditions.PausedRolloutReason,
Status: status,
Type: v1alpha1.RolloutPaused,
}
conditionBytes, err := json.Marshal(condition)
if err != nil {
panic(err)
}
return condition, string(conditionBytes)
}

func newProgressingCondition(reason string, resourceObj runtime.Object, optionalMessage string) (v1alpha1.RolloutCondition, string) {
status := corev1.ConditionTrue
msg := ""
Expand Down Expand Up @@ -277,6 +297,16 @@ func generateConditionsPatch(available bool, progressingReason string, progressi
return fmt.Sprintf("[%s, %s]", progressingCondition, availableCondition)
}

func generateConditionsPatchWithPause(available bool, progressingReason string, progressingResource runtime.Object, availableConditionFirst bool, progressingMessage string, isPaused bool) string {
_, availableCondition := newAvailableCondition(available)
_, progressingCondition := newProgressingCondition(progressingReason, progressingResource, progressingMessage)
_, pauseCondition := newPausedCondition(isPaused)
if availableConditionFirst {
return fmt.Sprintf("[%s, %s, %s]", availableCondition, progressingCondition, pauseCondition)
}
return fmt.Sprintf("[%s, %s, %s]", progressingCondition, pauseCondition, availableCondition)
}

// func updateBlueGreenRolloutStatus(r *v1alpha1.Rollout, preview, active string, availableReplicas, updatedReplicas, hpaReplicas int32, pause bool, available bool, progressingStatus string) *v1alpha1.Rollout {
func updateBlueGreenRolloutStatus(r *v1alpha1.Rollout, preview, active, stable string, availableReplicas, updatedReplicas, totalReplicas, hpaReplicas int32, pause bool, available bool) *v1alpha1.Rollout {
newRollout := updateBaseRolloutStatus(r, availableReplicas, updatedReplicas, totalReplicas, hpaReplicas)
Expand Down
Loading