Prepare for removal of cluster-autoscaler scaling in generic `Worke…

…r` actuator (gardener#8010)

* Prepare for removal of CA scaling in generic `Worker` actuator

This is actually `gardenlet` responsibility (even today w/o `gardenlet` taking over management of `machine-controller-manager`).

Co-Authored-By: Jens Schneider <schneider@23technologies.cloud>
Co-Authored-By: mreiger <michael@rauschpfeife.net>

* Address PR review feedback

---------

Co-authored-by: Jens Schneider <schneider@23technologies.cloud>
Co-authored-by: mreiger <michael@rauschpfeife.net>

extensions/pkg/controller/worker/genericactuator/actuator_reconcile.go

@@ -96,6 +96,7 @@ func (a *genericActuator) Reconcile(ctx context.Context, log logr.Logger, worker
 
 	var clusterAutoscalerUsed = extensionsv1alpha1helper.ClusterAutoscalerRequired(worker.Spec.Pools)
 
+	// TODO(rfranzke): Remove this code after v1.77 was released.
 	// When the Shoot is hibernated we want to remove the cluster autoscaler so that it does not interfer
 	// with Gardeners modifications on the machine deployment's replicas fields.
 	isHibernationEnabled := controller.IsHibernationEnabled(cluster)
@@ -198,6 +199,7 @@ func (a *genericActuator) Reconcile(ctx context.Context, log logr.Logger, worker
 		}
 	}
 
+	// TODO(rfranzke): Remove this code after v1.77 was released.
 	if clusterAutoscalerUsed && !isHibernationEnabled {
 		if err = a.scaleClusterAutoscaler(ctx, log, worker, 1); err != nil {
 			return err

pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go

@@ -632,21 +632,26 @@ func (r *Reconciler) runReconcileShootFlow(ctx context.Context, o *operation.Ope
 			deployNginxIngressAddon,
 		)
 
+		scaleClusterAutoscalerToZero = g.Add(flow.Task{
+			Name:         "Scaling down cluster autoscaler",
+			Fn:           flow.TaskFn(botanist.ScaleClusterAutoscalerToZero).RetryUntilTimeout(defaultInterval, defaultTimeout).SkipIf(o.Shoot.IsWorkerless).DoIf(o.Shoot.HibernationEnabled),
+			Dependencies: flow.NewTaskIDs(deployManagedResourcesForAddons, deployManagedResourceForCloudConfigExecutor),
+		})
 		deployWorker = g.Add(flow.Task{
 			Name:         "Configuring shoot worker pools",
 			Fn:           flow.TaskFn(botanist.DeployWorker).RetryUntilTimeout(defaultInterval, defaultTimeout).SkipIf(o.Shoot.IsWorkerless),
-			Dependencies: flow.NewTaskIDs(deployCloudProviderSecret, deployReferencedResources, waitUntilInfrastructureReady, initializeShootClients, waitUntilOperatingSystemConfigReady, waitUntilNetworkIsReady, createNewServiceAccountSecrets),
+			Dependencies: flow.NewTaskIDs(deployCloudProviderSecret, deployReferencedResources, waitUntilInfrastructureReady, initializeShootClients, waitUntilOperatingSystemConfigReady, waitUntilNetworkIsReady, createNewServiceAccountSecrets, scaleClusterAutoscalerToZero),
 		})
 		waitUntilWorkerStatusUpdate = g.Add(flow.Task{
 			Name: "Waiting until worker resource status is updated with latest machine deployments",
 			Fn: flow.TaskFn(func(ctx context.Context) error {
 				return botanist.Shoot.Components.Extensions.Worker.WaitUntilWorkerStatusMachineDeploymentsUpdated(ctx)
-			}).SkipIf(o.Shoot.IsWorkerless),
+			}).SkipIf(o.Shoot.IsWorkerless || o.Shoot.HibernationEnabled),
 			Dependencies: flow.NewTaskIDs(deployWorker),
 		})
 		deployClusterAutoscaler = g.Add(flow.Task{
 			Name:         "Deploying cluster autoscaler",
-			Fn:           flow.TaskFn(botanist.DeployClusterAutoscaler).RetryUntilTimeout(defaultInterval, defaultTimeout).SkipIf(o.Shoot.IsWorkerless),
+			Fn:           flow.TaskFn(botanist.DeployClusterAutoscaler).RetryUntilTimeout(defaultInterval, defaultTimeout).SkipIf(o.Shoot.IsWorkerless || o.Shoot.HibernationEnabled),
 			Dependencies: flow.NewTaskIDs(waitUntilWorkerStatusUpdate, deployManagedResourcesForAddons, deployManagedResourceForCloudConfigExecutor),
 		})
 		_ = g.Add(flow.Task{

pkg/operation/botanist/clusterautoscaler.go

@@ -17,9 +17,14 @@ package botanist
 import (
 	"context"
 
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
+	"github.com/gardener/gardener/pkg/client/kubernetes"
 	"github.com/gardener/gardener/pkg/component/clusterautoscaler"
 	"github.com/gardener/gardener/pkg/utils/images"
 	"github.com/gardener/gardener/pkg/utils/imagevector"
+	kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes"
 )
 
 // DefaultClusterAutoscaler returns a deployer for the cluster-autoscaler.
@@ -51,3 +56,8 @@ func (b *Botanist) DeployClusterAutoscaler(ctx context.Context) error {
 
 	return b.Shoot.Components.ControlPlane.ClusterAutoscaler.Destroy(ctx)
 }
+
+// ScaleClusterAutoscalerToZero scales cluster-autoscaler replicas to zero.
+func (b *Botanist) ScaleClusterAutoscalerToZero(ctx context.Context) error {
+	return client.IgnoreNotFound(kubernetes.ScaleDeployment(ctx, b.SeedClientSet.Client(), kubernetesutils.Key(b.Shoot.SeedNamespace, v1beta1constants.DeploymentNameClusterAutoscaler), 0))
+}

pkg/operation/botanist/clusterautoscaler_test.go

@@ -22,15 +22,18 @@ import (
 	"github.com/golang/mock/gomock"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
 	extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1"
 	kubernetesmock "github.com/gardener/gardener/pkg/client/kubernetes/mock"
 	mockclusterautoscaler "github.com/gardener/gardener/pkg/component/clusterautoscaler/mock"
 	mockworker "github.com/gardener/gardener/pkg/component/extensions/worker/mock"
+	mockclient "github.com/gardener/gardener/pkg/mock/controller-runtime/client"
 	"github.com/gardener/gardener/pkg/operation"
 	. "github.com/gardener/gardener/pkg/operation/botanist"
 	seedpkg "github.com/gardener/gardener/pkg/operation/seed"
@@ -40,30 +43,32 @@ import (
 
 var _ = Describe("ClusterAutoscaler", func() {
 	var (
-		ctrl     *gomock.Controller
-		botanist *Botanist
+		ctx     = context.TODO()
+		fakeErr = fmt.Errorf("fake err")
+
+		ctrl             *gomock.Controller
+		botanist         *Botanist
+		kubernetesClient *kubernetesmock.MockInterface
 	)
 
 	BeforeEach(func() {
 		ctrl = gomock.NewController(GinkgoT())
+		kubernetesClient = kubernetesmock.NewMockInterface(ctrl)
 		botanist = &Botanist{Operation: &operation.Operation{}}
 		botanist.Seed = &seedpkg.Seed{
 			KubernetesVersion: semver.MustParse("1.25.0"),
 		}
+		botanist.SeedClientSet = kubernetesClient
 	})
 
 	AfterEach(func() {
 		ctrl.Finish()
 	})
 
 	Describe("#DefaultClusterAutoscaler", func() {
-		var kubernetesClient *kubernetesmock.MockInterface
-
 		BeforeEach(func() {
-			kubernetesClient = kubernetesmock.NewMockInterface(ctrl)
 			kubernetesClient.EXPECT().Version()
 
-			botanist.SeedClientSet = kubernetesClient
 			botanist.Shoot = &shootpkg.Shoot{}
 			botanist.Shoot.SetInfo(&gardencorev1beta1.Shoot{})
 		})
@@ -91,8 +96,6 @@ var _ = Describe("ClusterAutoscaler", func() {
 			clusterAutoscaler *mockclusterautoscaler.MockInterface
 			worker            *mockworker.MockInterface
 
-			ctx                = context.TODO()
-			fakeErr            = fmt.Errorf("fake err")
 			namespaceUID       = types.UID("5678")
 			machineDeployments = []extensionsv1alpha1.MachineDeployment{{}}
 		)
@@ -154,4 +157,36 @@ var _ = Describe("ClusterAutoscaler", func() {
 			})
 		})
 	})
+
+	Describe("#ScaleClusterAutoscalerToZero", func() {
+		var (
+			c         *mockclient.MockClient
+			sw        *mockclient.MockSubResourceClient
+			patch     = client.RawPatch(types.MergePatchType, []byte(`{"spec":{"replicas":0}}`))
+			namespace = "shoot--foo--bar"
+		)
+
+		BeforeEach(func() {
+			botanist.SeedClientSet = kubernetesClient
+			botanist.Shoot = &shootpkg.Shoot{
+				SeedNamespace: namespace,
+			}
+
+			c = mockclient.NewMockClient(ctrl)
+			kubernetesClient.EXPECT().Client().Return(c)
+
+			sw = mockclient.NewMockSubResourceClient(ctrl)
+			c.EXPECT().SubResource("scale").Return(sw)
+		})
+
+		It("should scale the CA deployment", func() {
+			sw.EXPECT().Patch(ctx, gomock.AssignableToTypeOf(&appsv1.Deployment{}), patch)
+			Expect(botanist.ScaleClusterAutoscalerToZero(ctx)).To(Succeed())
+		})
+
+		It("should fail when the scale call fails", func() {
+			sw.EXPECT().Patch(ctx, gomock.AssignableToTypeOf(&appsv1.Deployment{}), patch).Return(fakeErr)
+			Expect(botanist.ScaleClusterAutoscalerToZero(ctx)).To(MatchError(fakeErr))
+		})
+	})
 })