diff --git a/docs/features/bluegreen.md b/docs/features/bluegreen.md index 95e2ce4241..afb2d2f158 100644 --- a/docs/features/bluegreen.md +++ b/docs/features/bluegreen.md @@ -68,6 +68,25 @@ spec: scaleDownDelayRevisionLimit: *int32 ``` +## Sequence of Events + +The following describes the sequence of events that happen during a blue-green update. + +1. Beginning at a fully promoted, steady-state, a revision 1 ReplicaSet is pointed to by both the `activeService` and `previewService`. +1. A user initiates an update by modifying the pod template (`spec.template.spec`). +1. The revision 2 ReplicaSet is created with size 0. +1. The preview service is modified to point to the revision 2 ReplicaSet. The `activeService` remains pointing to revision 1. +1. The revision 2 ReplicaSet is scaled to either `spec.replicas` or `previewReplicaCount` if set. +1. Once revision 2 ReplicaSet Pods are fully available, `prePromotionAnalysis` begins. +1. Upon success of `prePromotionAnalysis`, the blue/green pauses if `autoPromotionEnabled` is false, or `autoPromotionSeconds` is non-zero. +1. The rollout is resumed either manually by a user, or automatically by surpassing `autoPromotionSeconds`. +1. The revision 2 ReplicaSet is scaled to the `spec.replicas`, if the `previewReplicaCount` feature was used. +1. The rollout "promotes" the revision 2 ReplicaSet by updating the `activeService` to point to it. At this point, there are no services pointing to revision 1 +1. `postPromotionAnalysis` analysis begins +1. Once `postPromotionAnalysis` completes successfully, the update is successful and the revision 2 ReplicaSet is marked as stable. The rollout is considered fully-promoted. +1. After waiting `scaleDownDelaySeconds` (default 30 seconds), the revision 1 ReplicaSet is scaled down + + ### autoPromotionEnabled The AutoPromotionEnabled will make the rollout automatically promote the new ReplicaSet to the active service once the new ReplicaSet is healthy. This field is defaulted to true if it is not specified. @@ -111,15 +130,6 @@ This feature is used to provide an endpoint that can be used to test a new versi Defaults to an empty string -Here is a timeline of how the active and preview services work (if you use a preview service): - -1. During the Initial deployment there is only one ReplicaSet. Both active and preview services point to it. This is the **old** version of the application. -1. A change happens in the Rollout resource. A new ReplicaSet is created. This is the **new** version of the application. The preview service is modified to point to the new ReplicaSet. The active service still points to the old version. -1. The blue/green deployment is "promoted". Both active and preview services are pointing to the new version. The old version is still there but no service is pointing at it. -1. Once the the blue/green deployment is scaled down (see the `scaleDownDelaySeconds` field) the old ReplicaSet is has 0 replicas and we are back to the initial state. Both active and preview services point to the new version (which is the only one present anyway) - - - ### previewReplicaCount The PreviewReplicaCount field will indicate the number of replicas that the new version of an application should run. Once the application is ready to promote to the active service, the controller will scale the new ReplicaSet to the value of the `spec.replicas`. The rollout will not switch over the active service to the new ReplicaSet until it matches the `spec.replicas` count. @@ -136,3 +146,4 @@ Defaults to 30 The ScaleDownDelayRevisionLimit limits the number of old active ReplicaSets to keep scaled up while they wait for the scaleDownDelay to pass after being removed from the active service. If omitted, all ReplicaSets will be retained for the specified scaleDownDelay + diff --git a/rollout/analysis_test.go b/rollout/analysis_test.go index 8e665ed2f7..e32aa5e982 100644 --- a/rollout/analysis_test.go +++ b/rollout/analysis_test.go @@ -1517,6 +1517,7 @@ func TestDoNotCreateBackgroundAnalysisRunOnNewCanaryRollout(t *testing.T) { f.expectCreateReplicaSetAction(rs1) f.expectUpdateRolloutStatusAction(r1) // update conditions + f.expectUpdateReplicaSetAction(rs1) // scale replica set f.expectPatchRolloutAction(r1) f.run(getKey(r1, t)) } @@ -1551,6 +1552,7 @@ func TestDoNotCreateBackgroundAnalysisRunOnNewCanaryRolloutStableRSEmpty(t *test f.expectCreateReplicaSetAction(rs1) f.expectUpdateRolloutStatusAction(r1) // update conditions + f.expectUpdateReplicaSetAction(rs1) // scale replica set f.expectPatchRolloutAction(r1) f.run(getKey(r1, t)) } @@ -1686,6 +1688,7 @@ func TestDoNotCreatePrePromotionAnalysisRunOnNewRollout(t *testing.T) { f.expectCreateReplicaSetAction(rs) f.expectUpdateRolloutStatusAction(r) + f.expectUpdateReplicaSetAction(rs) // scale RS f.expectPatchRolloutAction(r) f.run(getKey(r, t)) } diff --git a/rollout/bluegreen.go b/rollout/bluegreen.go index fd8ab46848..8789666b41 100644 --- a/rollout/bluegreen.go +++ b/rollout/bluegreen.go @@ -25,6 +25,12 @@ func (c *rolloutContext) rolloutBlueGreen() error { return err } + // This must happen right after the new replicaset is created + err = c.reconcilePreviewService(previewSvc) + if err != nil { + return err + } + if replicasetutil.CheckPodSpecChange(c.rollout, c.newRS) { return c.syncRolloutStatusBlueGreen(previewSvc, activeSvc) } @@ -39,11 +45,6 @@ func (c *rolloutContext) rolloutBlueGreen() error { return err } - err = c.reconcilePreviewService(previewSvc) - if err != nil { - return err - } - c.reconcileBlueGreenPause(activeSvc, previewSvc) err = c.reconcileActiveService(activeSvc) diff --git a/rollout/bluegreen_test.go b/rollout/bluegreen_test.go index 500343a521..25aa70d46c 100644 --- a/rollout/bluegreen_test.go +++ b/rollout/bluegreen_test.go @@ -56,6 +56,7 @@ func TestBlueGreenCreatesReplicaSet(t *testing.T) { f.expectCreateReplicaSetAction(rs) servicePatchIndex := f.expectPatchServiceAction(previewSvc, rsPodHash) + f.expectUpdateReplicaSetAction(rs) // scale up RS updatedRolloutIndex := f.expectUpdateRolloutStatusAction(r) expectedPatchWithoutSubs := `{ "status":{ diff --git a/rollout/canary_test.go b/rollout/canary_test.go index ace03f0ea9..bb7d4cae23 100644 --- a/rollout/canary_test.go +++ b/rollout/canary_test.go @@ -77,15 +77,19 @@ func TestCanaryRolloutBumpVersion(t *testing.T) { f.replicaSetLister = append(f.replicaSetLister, rs1) createdRSIndex := f.expectCreateReplicaSetAction(rs2) + updatedRSIndex := f.expectUpdateReplicaSetAction(rs2) // scale up RS updatedRolloutRevisionIndex := f.expectUpdateRolloutAction(r2) // update rollout revision updatedRolloutConditionsIndex := f.expectUpdateRolloutStatusAction(r2) // update rollout conditions f.expectPatchRolloutAction(r2) f.run(getKey(r2, t)) createdRS := f.getCreatedReplicaSet(createdRSIndex) - assert.Equal(t, int32(1), *createdRS.Spec.Replicas) + assert.Equal(t, int32(0), *createdRS.Spec.Replicas) assert.Equal(t, "2", createdRS.Annotations[annotations.RevisionAnnotation]) + updatedRS := f.getUpdatedReplicaSet(updatedRSIndex) + assert.Equal(t, int32(1), *updatedRS.Spec.Replicas) + updatedRollout := f.getUpdatedRollout(updatedRolloutRevisionIndex) assert.Equal(t, "2", updatedRollout.Annotations[annotations.RevisionAnnotation]) @@ -475,6 +479,7 @@ func TestCanaryRolloutCreateFirstReplicasetNoSteps(t *testing.T) { rs := newReplicaSet(r, 1) f.expectCreateReplicaSetAction(rs) + f.expectUpdateReplicaSetAction(rs) // scale up rs updatedRolloutIndex := f.expectUpdateRolloutStatusAction(r) patchIndex := f.expectPatchRolloutAction(r) f.run(getKey(r, t)) @@ -514,6 +519,7 @@ func TestCanaryRolloutCreateFirstReplicasetWithSteps(t *testing.T) { rs := newReplicaSet(r, 1) f.expectCreateReplicaSetAction(rs) + f.expectUpdateReplicaSetAction(rs) // scale up rs updatedRolloutIndex := f.expectUpdateRolloutStatusAction(r) patchIndex := f.expectPatchRolloutAction(r) f.run(getKey(r, t)) @@ -559,12 +565,15 @@ func TestCanaryRolloutCreateNewReplicaWithCorrectWeight(t *testing.T) { f.replicaSetLister = append(f.replicaSetLister, rs1) createdRSIndex := f.expectCreateReplicaSetAction(rs2) + updatedRSIndex := f.expectUpdateReplicaSetAction(rs2) updatedRolloutIndex := f.expectUpdateRolloutStatusAction(r2) f.expectPatchRolloutAction(r2) f.run(getKey(r2, t)) createdRS := f.getCreatedReplicaSet(createdRSIndex) - assert.Equal(t, int32(1), *createdRS.Spec.Replicas) + assert.Equal(t, int32(0), *createdRS.Spec.Replicas) + updatedRS := f.getUpdatedReplicaSet(updatedRSIndex) + assert.Equal(t, int32(1), *updatedRS.Spec.Replicas) updatedRollout := f.getUpdatedRollout(updatedRolloutIndex) progressingCondition := conditions.GetRolloutCondition(updatedRollout.Status, v1alpha1.RolloutProgressing) diff --git a/rollout/ephemeralmetadata_test.go b/rollout/ephemeralmetadata_test.go index 7e9d4b850d..a68e4f39d2 100644 --- a/rollout/ephemeralmetadata_test.go +++ b/rollout/ephemeralmetadata_test.go @@ -37,6 +37,7 @@ func TestSyncCanaryEphemeralMetadataInitialRevision(t *testing.T) { f.expectUpdateRolloutStatusAction(r1) idx := f.expectCreateReplicaSetAction(rs1) + f.expectUpdateReplicaSetAction(rs1) _ = f.expectPatchRolloutAction(r1) f.run(getKey(r1, t)) createdRS1 := f.getCreatedReplicaSet(idx) @@ -75,8 +76,9 @@ func TestSyncBlueGreenEphemeralMetadataInitialRevision(t *testing.T) { f.expectUpdateRolloutStatusAction(r1) idx := f.expectCreateReplicaSetAction(rs1) - _ = f.expectPatchRolloutAction(r1) + f.expectPatchRolloutAction(r1) f.expectPatchServiceAction(previewSvc, rs1.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]) + f.expectUpdateReplicaSetAction(rs1) // scale replicaset f.run(getKey(r1, t)) createdRS1 := f.getCreatedReplicaSet(idx) expectedLabels := map[string]string{ @@ -209,6 +211,7 @@ func TestSyncBlueGreenEphemeralMetadataSecondRevision(t *testing.T) { f.expectUpdateRolloutStatusAction(r2) // Update Rollout conditions rs2idx := f.expectCreateReplicaSetAction(rs2) // Create revision 2 ReplicaSet f.expectPatchServiceAction(previewSvc, rs2PodHash) // Update preview service to point at revision 2 replicaset + f.expectUpdateReplicaSetAction(rs2) // scale revision 2 ReplicaSet up f.expectListPodAction(r1.Namespace) // list pods to patch ephemeral data on revision 1 ReplicaSets pods` podIdx := f.expectUpdatePodAction(&pod) // Update pod with ephemeral data rs1idx := f.expectUpdateReplicaSetAction(rs1) // update stable replicaset with stable metadata diff --git a/rollout/experiment_test.go b/rollout/experiment_test.go index 32159734fb..18277d3640 100644 --- a/rollout/experiment_test.go +++ b/rollout/experiment_test.go @@ -519,6 +519,7 @@ func TestRolloutDoNotCreateExperimentWithoutStableRS(t *testing.T) { f.expectCreateReplicaSetAction(rs2) f.expectUpdateRolloutAction(r2) // update revision f.expectUpdateRolloutStatusAction(r2) // update progressing condition + f.expectUpdateReplicaSetAction(rs2) // scale replicaset f.expectPatchRolloutAction(r1) f.run(getKey(r2, t)) } diff --git a/rollout/sync.go b/rollout/sync.go index d541ece498..27f9872195 100644 --- a/rollout/sync.go +++ b/rollout/sync.go @@ -159,13 +159,7 @@ func (c *rolloutContext) createDesiredReplicaSet() (*appsv1.ReplicaSet, error) { Template: newRSTemplate, }, } - allRSs := append(c.allRSs, newRS) - newReplicasCount, err := replicasetutil.NewRSNewReplicas(c.rollout, allRSs, newRS) - if err != nil { - return nil, err - } - - newRS.Spec.Replicas = pointer.Int32Ptr(newReplicasCount) + newRS.Spec.Replicas = pointer.Int32Ptr(0) // Set new replica set's annotation annotations.SetNewReplicaSetAnnotations(c.rollout, newRS, newRevision, false) @@ -250,12 +244,10 @@ func (c *rolloutContext) createDesiredReplicaSet() (*appsv1.ReplicaSet, error) { return nil, err } - if !alreadyExists && newReplicasCount > 0 { + if !alreadyExists { revision, _ := replicasetutil.Revision(createdRS) - c.recorder.Eventf(c.rollout, record.EventOptions{EventReason: conditions.NewReplicaSetReason}, conditions.NewReplicaSetDetailedMessage, createdRS.Name, revision, newReplicasCount) - } + c.recorder.Eventf(c.rollout, record.EventOptions{EventReason: conditions.NewReplicaSetReason}, conditions.NewReplicaSetDetailedMessage, createdRS.Name, revision) - if !alreadyExists { msg := fmt.Sprintf(conditions.NewReplicaSetMessage, createdRS.Name) condition := conditions.NewRolloutCondition(v1alpha1.RolloutProgressing, corev1.ConditionTrue, conditions.NewReplicaSetReason, msg) conditions.SetRolloutCondition(&c.rollout.Status, *condition) diff --git a/rollout/sync_test.go b/rollout/sync_test.go index 61902f713b..ba3eb6f2c5 100644 --- a/rollout/sync_test.go +++ b/rollout/sync_test.go @@ -304,14 +304,17 @@ func TestCanaryPromoteFull(t *testing.T) { f.kubeobjects = append(f.kubeobjects, rs1) f.replicaSetLister = append(f.replicaSetLister, rs1) - createdRS2Index := f.expectCreateReplicaSetAction(rs2) // create new ReplicaSet (surge to 10) + createdRS2Index := f.expectCreateReplicaSetAction(rs2) // create new ReplicaSet (size 0) f.expectUpdateRolloutAction(r2) // update rollout revision f.expectUpdateRolloutStatusAction(r2) // update rollout conditions + updatedRS2Index := f.expectUpdateReplicaSetAction(rs2) // scale new ReplicaSet to 10 patchedRolloutIndex := f.expectPatchRolloutAction(r2) f.run(getKey(r2, t)) createdRS2 := f.getCreatedReplicaSet(createdRS2Index) - assert.Equal(t, int32(10), *createdRS2.Spec.Replicas) // verify we ignored steps + assert.Equal(t, int32(0), *createdRS2.Spec.Replicas) + updatedRS2 := f.getUpdatedReplicaSet(updatedRS2Index) + assert.Equal(t, int32(10), *updatedRS2.Spec.Replicas) // verify we ignored steps and fully scaled it patchedRollout := f.getPatchedRolloutAsObject(patchedRolloutIndex) assert.Equal(t, int32(2), *patchedRollout.Status.CurrentStepIndex) // verify we updated to last step diff --git a/test/e2e/functional_test.go b/test/e2e/functional_test.go index fc381773f3..1c07a73620 100644 --- a/test/e2e/functional_test.go +++ b/test/e2e/functional_test.go @@ -90,10 +90,12 @@ spec: ExpectRevisionPodCount("2", 1). ExpectRolloutEvents([]string{ "RolloutUpdated", // Rollout updated to revision 1 - "NewReplicaSetCreated", // Created ReplicaSet abort-retry-promote-698fbfb9dc (revision 1) with size 1 + "NewReplicaSetCreated", // Created ReplicaSet abort-retry-promote-698fbfb9dc (revision 1) + "ScalingReplicaSet", // Scaled up ReplicaSet abort-retry-promote-698fbfb9dc (revision 1) from 0 to 1 "RolloutCompleted", // Rollout completed update to revision 1 (698fbfb9dc): Initial deploy "RolloutUpdated", // Rollout updated to revision 2 - "NewReplicaSetCreated", // Created ReplicaSet abort-retry-promote-75dcb5ddd6 (revision 2) with size 1 + "NewReplicaSetCreated", // Created ReplicaSet abort-retry-promote-75dcb5ddd6 (revision 2) + "ScalingReplicaSet", // Scaled up ReplicaSet abort-retry-promote-75dcb5ddd6 (revision 2) from 0 to 1 "RolloutStepCompleted", // Rollout step 1/2 completed (setWeight: 50) "RolloutPaused", // Rollout is paused (CanaryPauseStep) "ScalingReplicaSet", // Scaled down ReplicaSet abort-retry-promote-75dcb5ddd6 (revision 2) from 1 to 0 @@ -696,11 +698,13 @@ func (s *FunctionalSuite) TestBlueGreenUpdate() { ExpectReplicaCounts(3, 6, 3, 3, 3). ExpectRolloutEvents([]string{ "RolloutUpdated", // Rollout updated to revision 1 - "NewReplicaSetCreated", // Created ReplicaSet bluegreen-7dcd8f8869 (revision 1) with size 3 + "NewReplicaSetCreated", // Created ReplicaSet bluegreen-7dcd8f8869 (revision 1) + "ScalingReplicaSet", // Scaled up ReplicaSet bluegreen-7dcd8f8869 (revision 1) from 0 to 3 "RolloutCompleted", // Rollout completed update to revision 1 (7dcd8f8869): Initial deploy "SwitchService", // Switched selector for service 'bluegreen' from '' to '7dcd8f8869' "RolloutUpdated", // Rollout updated to revision 2 - "NewReplicaSetCreated", // Created ReplicaSet bluegreen-5498785cd6 (revision 2) with size 3 + "NewReplicaSetCreated", // Created ReplicaSet bluegreen-5498785cd6 (revision 2) + "ScalingReplicaSet", // Scaled up ReplicaSet bluegreen-5498785cd6 (revision 2) from 0 to 3 "SwitchService", // Switched selector for service 'bluegreen' from '7dcd8f8869' to '6c779b88b6' "RolloutCompleted", // Rollout completed update to revision 2 (6c779b88b6): Completed blue-green update }) diff --git a/utils/conditions/conditions.go b/utils/conditions/conditions.go index c65d8f4734..5826db3f89 100644 --- a/utils/conditions/conditions.go +++ b/utils/conditions/conditions.go @@ -52,7 +52,7 @@ const ( //NewReplicaSetMessage is added in a rollout when it creates a new replicas set. NewReplicaSetMessage = "Created new replica set %q" // NewReplicaSetDetailedMessage is a more detailed format message - NewReplicaSetDetailedMessage = "Created ReplicaSet %s (revision %d) with size %d" + NewReplicaSetDetailedMessage = "Created ReplicaSet %s (revision %d)" // FoundNewRSReason is added in a rollout when it adopts an existing replica set. FoundNewRSReason = "FoundNewReplicaSet"