[GEP-20] Remove zone pinning (gardener#6934)

* Remove zone pinning feature

* Remove zone-enforcement label from namespaces

charts/gardener/resource-manager/charts/application/templates/mutatingwebhookconfiguration.yaml

@@ -166,41 +166,6 @@ webhooks:
   sideEffects: None
   timeoutSeconds: 10
 {{- end }}
-{{- if .Values.global.config.webhooks.podZoneAffinity.enabled }}
-- admissionReviewVersions:
-  - v1beta1
-  - v1
-  clientConfig:
-    {{- if .Values.global.config.server.webhooks.ca }}
-    caBundle: {{ b64enc .Values.global.config.server.webhooks.ca }}
-    {{- end }}
-    service:
-      name: gardener-resource-manager
-      namespace: {{ .Release.Namespace }}
-      path: /webhooks/pod-zone-affinity
-      port: 443
-  failurePolicy: Fail
-  matchPolicy: Exact
-  name: pod-zone-affinity.resources.gardener.cloud
-  namespaceSelector:
-    matchExpressions:
-    - key: control-plane.shoot.gardener.cloud/enforce-zone
-      operator: Exists
-  objectSelector: {}
-  reinvocationPolicy: Never
-  rules:
-    - apiGroups:
-      - ""
-      apiVersions:
-      - v1
-      operations:
-      - CREATE
-      resources:
-      - pods
-      scope: '*'
-  sideEffects: None
-  timeoutSeconds: 10
-{{- end }}
 {{- if .Values.global.config.webhooks.podSchedulerName.enabled }}
 - admissionReviewVersions:
     - v1beta1

charts/gardener/resource-manager/charts/runtime/templates/configmap.yaml

@@ -136,8 +136,6 @@ data:
         {{- end }}
       podTopologySpreadConstraints:
         enabled: {{ .Values.global.config.webhooks.podTopologySpreadConstraints.enabled }}
-      podZoneAffinity:
-        enabled: {{ .Values.global.config.webhooks.podZoneAffinity.enabled }}
       projectedTokenMount:
         enabled: {{ .Values.global.config.webhooks.projectedTokenMount.enabled }}
         {{- if .Values.global.config.webhooks.projectedTokenMount.expirationSeconds }}

charts/gardener/resource-manager/values.yaml

@@ -105,8 +105,6 @@ global:
       # schedulerName: foo-scheduler
       podTopologySpreadConstraints:
         enabled: false
-      podZoneAffinity:
-        enabled: false
       projectedTokenMount:
         enabled: false
       # expirationSeconds: 43200

docs/concepts/resource-manager.md

@@ -564,39 +564,6 @@ Please find an overview below for pods deployed in the Shoot cluster:
 
 ![image](images/resource-manager-projected-token-shoot-to-shoot-apiserver.jpg)
 
-### Pod Zone Affinity
-
-When this webhook is activated and namespaces are annotated with `control-plane.shoot.gardener.cloud/enforce-zone` then it automatically adds a [pod affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity) to all `Pod`s created in these namespaces:
-
-```
-spec:
-  affinity:
-    podAffinity:
-      requiredDuringSchedulingIgnoredDuringExecution:
-      - labelSelector: {}
-        topologyKey: topology.kubernetes.io/zone
-```
-
-In addition, if the annotation key `control-plane.shoot.gardener.cloud/enforce-zone` has a value `<zone-value>`, i.e. zone assigned, this information is added as part of a node affinity.
-
-```
-spec:
-  affinity:
-    nodeAffinity:
-      requiredDuringSchedulingIgnoredDuringExecution:
-          nodeSelectorTerms:
-          - matchExpressions:
-            - key: topology.kubernetes.io/zone
-              operator: In
-              values:
-              - <zone-value>
-```
-
-Those terms let pods within a namespace being scheduled to nodes residing in the very same zone which is either randomly picked, or to a very specific zone.
-In addition, the webhook removes any (anti-)affinities with `topology.kubernetes.io/zone` because they potentially contradict the above shown configuration.
-Gardener uses this webhook to schedule control-plane pods within a single zone on a multi-zonal seed (seed with worker nodes across zones).
-The goal is to reduce cross zonal network traffic within the seed with this approach.
-
 ### Pod Topology Spread Constraints
 
 When this webhook is enabled then it mimics the [topologyKey feature](https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/#spread-constraint-definition) for [Topology Spread Constraints (TSC)](https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints) on the label `pod-template-hash`.

example/resource-manager/10-componentconfig.yaml

@@ -72,8 +72,6 @@ webhooks:
     schedulerName: foo-scheduler
   podTopologySpreadConstraints:
     enabled: true
-  podZoneAffinity:
-    enabled: true
   projectedTokenMount:
     enabled: true
     expirationSeconds: 43200

example/resource-manager/10-mutatingwebhookconfiguration.yaml

@@ -121,27 +121,4 @@ webhooks:
       operator: DoesNotExist
   namespaceSelector: {}
   reinvocationPolicy: Never
-  timeoutSeconds: 10
-- name: pod-zone-affinity.resources.gardener.cloud
-  admissionReviewVersions:
-  - v1beta1
-  - v1
-  clientConfig:
-    url: https://host.docker.internal:9449/webhooks/pod-zone-affinity
-    caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM5akNDQWQ2Z0F3SUJBZ0lVRW9FT0NKWGszV1loL1I4NlFVNnRsMjFJbmM0d0RRWUpLb1pJaHZjTkFRRUwKQlFBd0V6RVJNQThHQTFVRUF4TUlaMkZ5WkdWdVpYSXdIaGNOTWpFeE1ERTRNVE13TnpBd1doY05Nall4TURFMwpNVE13TnpBd1dqQVRNUkV3RHdZRFZRUURFd2huWVhKa1pXNWxjakNDQVNJd0RRWUpLb1pJaHZjTkFRRUJCUUFECmdnRVBBRENDQVFvQ2dnRUJBTi9wMm91Wi9JRU4xQWNTZVpjU3RWcW92bzVaMjdaTUZZQlZUYmNUZDNMMW9LaGgKS3VPN1o4VU9SWFpKS1VYMzEzQ3FNUkU5RGVCMEZvRE9laU9VRnhuNDFDeitlZkk3Rnh4WUtpMUNxWk9QV2w0ZApJaHJzaDIrd3NieGdJVUVWRElNTy8vSEJFOXB1Z3FuOURUdXdwZzJFalQ2N2dYdHRKZ0x5T2NOa2VRUEU2VjJ5CmpsQW9ZeEp1T0xXdC9OcDgycXFBT25pb1pTMnlEdnp3WHJBRkNGSzF0cVNBdDlBOFcrcC9YZ1dUVHFkV1BGOVcKSE9qekZnK1V4K1Z5YXNTQ00yQ2Nob3QxNC9Id0gzU0JWWHZpMFNuUGpwTmV1UXFEbFFBRXFucnVNT0lPT2JWdwpzd09jWG41ZEZjays3dnJDQ2RKbk9LSVorV08rRHJuZ01jT3ZrYzhDQXdFQUFhTkNNRUF3RGdZRFZSMFBBUUgvCkJBUURBZ0VHTUE4R0ExVWRFd0VCL3dRRk1BTUJBZjh3SFFZRFZSME9CQllFRkdwZk01L2VQTVhwc2xVdi9BL0sKNnlqUVVqTWpNQTBHQ1NxR1NJYjNEUUVCQ3dVQUE0SUJBUUNBVUErZTRKdWFFRHoyZ3ZJQWxlZW11US85eWxGcQovWG16ZXZBanFYU25kS21rTVR3T3h2UFdBQTN4TEpqdm93WnkwaTV0UEEvaEZoNWdJa25SZUw0a0dJTkduQ2FMCjUwWmJjQ2x2bG0xQ2xxVXc5N1MwcUJOWlFQNHhHYTNDaGtwS0VSczhsaVh1NjEzeE5xSWdpYjErMEZ0ZlR2R20KcVNId3dBSTZwTWJ4eHpZQkZmdHAwWUxCanB3REhaT2hCQVFjVlB0bGQ4Q2NlL01kNmJOTU5ncXRUa0RWV1JYVgpUclpVUmNmaXpCNWlwUVJnT0NpWGFYL1UwcXhZV2JHMFhycnZ0ODY5d05ubDhES2Z4NVl0ZENlSGhUNzRHbzBBCkZza2NLczA4OGgza1poOHNjOHBHMjVTQ3dLZEVYWGg3dWZPM2FZdEViVmlTQVFicUlpeE5WZFJPCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
-  rules:
-  - apiGroups:
-    - ""
-    apiVersions:
-    - v1
-    resources:
-    - pods
-    operations:
-    - CREATE
-  sideEffects: None
-  failurePolicy: Fail
-  matchPolicy: Exact
-  objectSelector: {}
-  namespaceSelector: {}
-  reinvocationPolicy: Never
-  timeoutSeconds: 10
+  timeoutseconds: 10

pkg/apis/core/v1beta1/constants/types_constants.go

@@ -258,6 +258,7 @@ const (
 
 	// ShootControlPlaneEnforceZone is an annotation key which is used to pin or schedule all control-plane pods
 	// to the very same availability zone.
+	// Deprecated: Only kept for removal of the label.
 	ShootControlPlaneEnforceZone = "control-plane.shoot.gardener.cloud/enforce-zone"
 	// ShootUID is an annotation key for the shoot namespace in the seed cluster,
 	// which value will be the value of `shoot.status.uid`

pkg/gardenlet/controller/seed/seed/components.go

@@ -201,7 +201,6 @@ func defaultGardenerResourceManager(
 		DefaultSeccompProfileEnabled: gardenletfeatures.FeatureGate.Enabled(features.DefaultSeccompProfile),
 		// TODO(timuthy): Remove PodTopologySpreadConstraints webhook once for all seeds the MatchLabelKeysInPodTopologySpread feature gate is beta and enabled by default (probably 1.26+).
 		PodTopologySpreadConstraintsEnabled: true,
-		PodZoneAffinityEnabled:              true,
 		LogLevel:                            conf.LogLevel,
 		LogFormat:                           conf.LogFormat,
 	}), nil

pkg/gardenlet/controller/shoot/shoot_control_delete.go

@@ -238,11 +238,6 @@ func (r *shootReconciler) runDeleteShootFlow(ctx context.Context, o *operation.O
 			Fn:           flow.TaskFn(botanist.WaitUntilEtcdsReady).DoIf(cleanupShootResources),
 			Dependencies: flow.NewTaskIDs(scaleETCD),
 		})
-		_ = g.Add(flow.Task{
-			Name:         "Adding zone information to Shoot namespace",
-			Fn:           flow.TaskFn(botanist.AddZoneInformationToSeedNamespace).DoIf(cleanupShootResources),
-			Dependencies: flow.NewTaskIDs(waitUntilEtcdReady),
-		})
 		// Redeploy the control plane to make sure all components that depend on the cloud provider secret are restarted
 		// in case it has changed. Also, it's needed for other control plane components like the kube-apiserver or kube-
 		// controller-manager to be updateable due to provider config injection.

pkg/gardenlet/controller/shoot/shoot_control_reconcile.go

@@ -274,11 +274,6 @@ func (r *shootReconciler) runReconcileShootFlow(ctx context.Context, o *operatio
 			Fn:           flow.TaskFn(botanist.WaitUntilEtcdsReady).SkipIf(o.Shoot.HibernationEnabled),
 			Dependencies: flow.NewTaskIDs(deployETCD),
 		})
-		_ = g.Add(flow.Task{
-			Name:         "Adding zone information to Shoot namespace",
-			Fn:           flow.TaskFn(botanist.AddZoneInformationToSeedNamespace).SkipIf(o.Shoot.HibernationEnabled),
-			Dependencies: flow.NewTaskIDs(waitUntilEtcdReady),
-		})
 		deployControlPlane = g.Add(flow.Task{
 			Name:         "Deploying shoot control plane components",
 			Fn:           flow.TaskFn(botanist.DeployControlPlane).RetryUntilTimeout(defaultInterval, defaultTimeout),

pkg/operation/botanist/component/resourcemanager/resource_manager.go

@@ -53,7 +53,6 @@ import (
 	resourcemanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/resourcemanager/apis/config/v1alpha1"
 	"github.com/gardener/gardener/pkg/resourcemanager/webhook/podschedulername"
 	"github.com/gardener/gardener/pkg/resourcemanager/webhook/podtopologyspreadconstraints"
-	"github.com/gardener/gardener/pkg/resourcemanager/webhook/podzoneaffinity"
 	"github.com/gardener/gardener/pkg/resourcemanager/webhook/projectedtokenmount"
 	"github.com/gardener/gardener/pkg/resourcemanager/webhook/seccompprofile"
 	"github.com/gardener/gardener/pkg/resourcemanager/webhook/tokeninvalidator"
@@ -266,8 +265,6 @@ type Values struct {
 	DefaultSeccompProfileEnabled bool
 	// PodTopologySpreadConstraintsEnabled specifies if the pod's TSC should be mutated to support rolling updates.
 	PodTopologySpreadConstraintsEnabled bool
-	// PodZoneAffinityEnabled specifies if the pod affinity should for zones be set.
-	PodZoneAffinityEnabled bool
 	// FailureToleranceType determines the failure tolerance type for the resource manager deployment.
 	FailureToleranceType *gardencorev1beta1.FailureToleranceType
 }
@@ -471,9 +468,6 @@ func (r *resourceManager) ensureConfigMap(ctx context.Context, configMap *corev1
 			PodTopologySpreadConstraints: resourcemanagerconfigv1alpha1.PodTopologySpreadConstraintsWebhookConfig{
 				Enabled: r.values.PodTopologySpreadConstraintsEnabled,
 			},
-			PodZoneAffinity: resourcemanagerconfigv1alpha1.PodZoneAffinityWebhookConfig{
-				Enabled: r.values.PodZoneAffinityEnabled,
-			},
 			ProjectedTokenMount: resourcemanagerconfigv1alpha1.ProjectedTokenMountWebhookConfig{
 				Enabled: true,
 			},
@@ -994,7 +988,6 @@ func (r *resourceManager) ensureMutatingWebhookConfiguration(ctx context.Context
 			nil,
 			r.values.DefaultSeccompProfileEnabled,
 			r.values.PodTopologySpreadConstraintsEnabled,
-			r.values.PodZoneAffinityEnabled,
 		)
 		return nil
 	})
@@ -1046,7 +1039,6 @@ func (r *resourceManager) ensureShootResources(ctx context.Context) error {
 		r.values.SchedulingProfile,
 		false,
 		false,
-		false,
 	)
 
 	data, err := registry.AddAllAndSerialize(
@@ -1113,7 +1105,6 @@ func getMutatingWebhookConfigurationWebhooks(
 	schedulingProfile *gardencorev1beta1.SchedulingProfile,
 	seccompWebhookEnabled bool,
 	podTopologySpreadConstraintsWebhookEnabled bool,
-	podZoneAffinityEnabled bool,
 ) []admissionregistrationv1.MutatingWebhook {
 	webhooks := []admissionregistrationv1.MutatingWebhook{
 		GetTokenInvalidatorMutatingWebhook(namespaceSelector, secretServerCA, buildClientConfigFn),
@@ -1133,10 +1124,6 @@ func getMutatingWebhookConfigurationWebhooks(
 		webhooks = append(webhooks, GetPodTopologySpreadConstraintsMutatingWebhook(namespaceSelector, secretServerCA, buildClientConfigFn))
 	}
 
-	if podZoneAffinityEnabled {
-		webhooks = append(webhooks, GetPodZoneAffinityMutatingWebhook(secretServerCA, buildClientConfigFn))
-	}
-
 	return webhooks
 }
 
@@ -1293,43 +1280,6 @@ func GetPodTopologySpreadConstraintsMutatingWebhook(
 	}
 }
 
-// GetPodZoneAffinityMutatingWebhook returns the pod-zone-affinity mutating webhook for the resourcemanager component for reuse
-// between the component and integration tests.
-func GetPodZoneAffinityMutatingWebhook(secretServerCA *corev1.Secret, buildClientConfigFn func(*corev1.Secret, string) admissionregistrationv1.WebhookClientConfig) admissionregistrationv1.MutatingWebhook {
-	var (
-		failurePolicy = admissionregistrationv1.Fail
-		matchPolicy   = admissionregistrationv1.Exact
-		sideEffect    = admissionregistrationv1.SideEffectClassNone
-	)
-
-	return admissionregistrationv1.MutatingWebhook{
-		Name: "pod-zone-affinity.resources.gardener.cloud",
-		Rules: []admissionregistrationv1.RuleWithOperations{{
-			Rule: admissionregistrationv1.Rule{
-				APIGroups:   []string{corev1.GroupName},
-				APIVersions: []string{corev1.SchemeGroupVersion.Version},
-				Resources:   []string{"pods"},
-			},
-			Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create},
-		}},
-		NamespaceSelector: &metav1.LabelSelector{
-			MatchExpressions: []metav1.LabelSelectorRequirement{
-				{
-					Key:      v1beta1constants.ShootControlPlaneEnforceZone,
-					Operator: metav1.LabelSelectorOpExists,
-				},
-			},
-		},
-		ObjectSelector:          &metav1.LabelSelector{},
-		ClientConfig:            buildClientConfigFn(secretServerCA, podzoneaffinity.WebhookPath),
-		AdmissionReviewVersions: []string{admissionv1beta1.SchemeGroupVersion.Version, admissionv1.SchemeGroupVersion.Version},
-		FailurePolicy:           &failurePolicy,
-		MatchPolicy:             &matchPolicy,
-		SideEffects:             &sideEffect,
-		TimeoutSeconds:          pointer.Int32(10),
-	}
-}
-
 // GetSeccompProfileMutatingWebhook returns the seccomp-profile mutating webhook for the resourcemanager component for reuse
 // between the component and integration tests.
 func GetSeccompProfileMutatingWebhook(

pkg/operation/botanist/component/resourcemanager/resource_manager_test.go

@@ -319,7 +319,6 @@ var _ = Describe("ResourceManager", func() {
 			SchedulingProfile:                   &binPackingSchedulingProfile,
 			DefaultSeccompProfileEnabled:        true,
 			PodTopologySpreadConstraintsEnabled: true,
-			PodZoneAffinityEnabled:              true,
 			LogLevel:                            "info",
 			LogFormat:                           "json",
 		}
@@ -410,9 +409,6 @@ var _ = Describe("ResourceManager", func() {
 					PodTopologySpreadConstraints: resourcemanagerconfigv1alpha1.PodTopologySpreadConstraintsWebhookConfig{
 						Enabled: true,
 					},
-					PodZoneAffinity: resourcemanagerconfigv1alpha1.PodZoneAffinityWebhookConfig{
-						Enabled: true,
-					},
 					ProjectedTokenMount: resourcemanagerconfigv1alpha1.ProjectedTokenMountWebhookConfig{
 						Enabled: true,
 					},
@@ -933,36 +929,6 @@ var _ = Describe("ResourceManager", func() {
 					SideEffects:             &sideEffect,
 					TimeoutSeconds:          pointer.Int32(10),
 				},
-				{
-					Name: "pod-zone-affinity.resources.gardener.cloud",
-					Rules: []admissionregistrationv1.RuleWithOperations{{
-						Rule: admissionregistrationv1.Rule{
-							APIGroups:   []string{""},
-							APIVersions: []string{"v1"},
-							Resources:   []string{"pods"},
-						},
-						Operations: []admissionregistrationv1.OperationType{"CREATE"},
-					}},
-					NamespaceSelector: &metav1.LabelSelector{
-						MatchExpressions: []metav1.LabelSelectorRequirement{{
-							Key:      "control-plane.shoot.gardener.cloud/enforce-zone",
-							Operator: metav1.LabelSelectorOpExists,
-						}},
-					},
-					ObjectSelector: &metav1.LabelSelector{},
-					ClientConfig: admissionregistrationv1.WebhookClientConfig{
-						Service: &admissionregistrationv1.ServiceReference{
-							Name:      "gardener-resource-manager",
-							Namespace: deployNamespace,
-							Path:      pointer.String("/webhooks/pod-zone-affinity"),
-						},
-					},
-					AdmissionReviewVersions: []string{"v1beta1", "v1"},
-					FailurePolicy:           &failurePolicy,
-					MatchPolicy:             &matchPolicy,
-					SideEffects:             &sideEffect,
-					TimeoutSeconds:          pointer.Int32(10),
-				},
 			},
 		}
 		mutatingWebhookConfigurationYAML := `apiVersion: admissionregistration.k8s.io/v1