⚡️ Improve scale-up/down performance by processing pod-template chang…

…es before replicas (#1310)

go.mod

@@ -32,7 +32,7 @@ require (
 	github.com/nyaruka/phonenumbers v1.1.7
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.70.0
-	github.com/rigdev/rig-go-api v0.0.0-20241023135935-eac99f7a930f
+	github.com/rigdev/rig-go-api v0.0.0-20241029083126-a6814266e716
 	github.com/rigdev/rig-go-sdk v0.0.0-20241021090503-b515c1ca035f
 	github.com/rivo/tview v0.0.0-20240524063012-037df494fb76
 	github.com/robfig/cron v1.2.0

go.sum

@@ -309,8 +309,8 @@ github.com/prometheus/common v0.59.1 h1:LXb1quJHWm1P6wq/U824uxYi4Sg0oGvNeUm1z5dJ
 github.com/prometheus/common v0.59.1/go.mod h1:GpWM7dewqmVYcd7SmRaiWVe9SSqjf0UrwnYnpEZNuT0=
 github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
 github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
-github.com/rigdev/rig-go-api v0.0.0-20241023135935-eac99f7a930f h1:D87qhahpJla10fB/Ptd9/Ocg5ktGFJ3HosQDBrnn9vk=
-github.com/rigdev/rig-go-api v0.0.0-20241023135935-eac99f7a930f/go.mod h1:0yAwJnPID3gHfB3ETQAFYGM1tkNWkJKbVzauZiHmHKw=
+github.com/rigdev/rig-go-api v0.0.0-20241029083126-a6814266e716 h1:QpieTeTr+uVcjBcQJuGpLz7sFuHl87EKf0lST7gK9qc=
+github.com/rigdev/rig-go-api v0.0.0-20241029083126-a6814266e716/go.mod h1:0yAwJnPID3gHfB3ETQAFYGM1tkNWkJKbVzauZiHmHKw=
 github.com/rigdev/rig-go-sdk v0.0.0-20241021090503-b515c1ca035f h1:SScDJVwFt/QzURstwUdi/ejnRoqB1ZsBLlICPD6Lt0c=
 github.com/rigdev/rig-go-sdk v0.0.0-20241021090503-b515c1ca035f/go.mod h1:DJAwoPmFI6Jo71n/2VfFk/Nyzlyd24ARWCLC4Hsm1gg=
 github.com/rivo/tview v0.0.0-20240524063012-037df494fb76 h1:iqvDlgyjmqleATtFbA7c14djmPh2n4mCYUv7JlD/ruA=

pkg/pipeline/request.go

@@ -505,7 +505,7 @@ func (r *RequestBase) applyChange(ctx context.Context, key ObjectKey, state Reso
 	switch state {
 	case ResourceStateUpdated:
 		r.logger.Info("update object", "object", key)
-		obj := r.newObjects[key]
+		object := r.newObjects[key]
 
 		// Edge case: Deployments, when they are scaled, will apply changes in the following order:
 		//   1) Change the number of replicas (pods) in the current ReplicaSet
@@ -518,18 +518,33 @@ func (r *RequestBase) applyChange(ctx context.Context, key ObjectKey, state Reso
 		// 	 1) First, we apply the template changes (use current number of replicas)
 		//	 2) Then apply the scale changes.
 		// That way the scale is only being applied to the new ReplicaSet.
-		if key.GroupVersionKind == AppsDeploymentGVK && obj.Materialized != nil {
-			currentObj := obj.Current.(*v1.Deployment)
-			newObj := obj.New.(*v1.Deployment)
-			materializedObj := obj.Materialized.(*v1.Deployment)
+		if key.GroupVersionKind == AppsDeploymentGVK && object.Materialized != nil {
+			currentObj := object.Current.(*v1.Deployment)
+			newObj := object.New.(*v1.Deployment)
+			materializedObj := object.Materialized.(*v1.Deployment)
+			for _, c := range currentObj.Status.Conditions {
+				if c.Type == v1.DeploymentProgressing && c.Reason == "ReplicaSetUpdated" {
+					// We have an ongoing rollout, wait a little with scale-up.
+					if newObj.Spec.Replicas != nil &&
+						currentObj.Spec.Replicas != nil &&
+						*newObj.Spec.Replicas > *currentObj.Spec.Replicas {
+						newObj.Spec.Replicas = currentObj.Spec.Replicas
+					}
+				}
+			}
+
 			if !equality.Semantic.DeepEqual(currentObj.Spec.Template, materializedObj.Spec.Template) {
 				// We have changes to the template - don't apply potential replica changes yet.
-				newObj.Spec.Replicas = currentObj.Spec.Replicas
+				if newObj.Spec.Replicas != nil &&
+					currentObj.Spec.Replicas != nil &&
+					*newObj.Spec.Replicas > *currentObj.Spec.Replicas {
+					newObj.Spec.Replicas = currentObj.Spec.Replicas
+				}
 			}
 		}
 
-		obj.New.SetResourceVersion(obj.Current.GetResourceVersion())
-		if err := r.client.Update(ctx, obj.New); err != nil {
+		object.New.SetResourceVersion(object.Current.GetResourceVersion())
+		if err := r.client.Update(ctx, object.New); err != nil {
 			return fmt.Errorf("could not update %s: %w", key.GroupVersionKind, err)
 		}