Move apiserver-proxy-pod-webhook logic into `gardener-resource-mana…

…ger` (gardener#7980)

* Drop option to disable SNI injector via `Shoot` annotation

The `alpha.featuregates.shoot.gardener.cloud/apiserver-sni-pod-injector` annotation will no longer be supported.
The injector will now always be enabled when SNI is enabled.

* Documentation

* Extend `gardener-resource-manager`'s component config for `KubernetesServiceHost` webhook

* Add new `kubernetesservicehost` webhook to `gardener-resource-manager`

Logic heavily inspired from https://github.com/gardener/apiserver-proxy/blob/master/internal/admission/admission.go

* Integration test

* Enable new `kubernetesservicehost` webhook for shoots

* Drop old `apiserver-proxy-pod-mutator` now that GRM enables the new webhook

charts/gardener/resource-manager/charts/runtime/templates/configmap.yaml

@@ -156,6 +156,11 @@ data:
         defaultUnreachableTolerationSeconds: {{ .Values.global.nodeToleration.defaultUnreachableTolerationSeconds }}
         {{- end }}
         {{- end }}
+      kubernetesServiceHost:
+        enabled: {{ .Values.global.config.webhooks.kubernetesServiceHost.enabled }}
+        {{- if .Values.global.config.webhooks.kubernetesServiceHost.host }}
+        host: {{ .Values.global.config.webhooks.kubernetesServiceHost.host }}
+        {{- end }}
       podSchedulerName:
         enabled: {{ .Values.global.config.webhooks.podSchedulerName.enabled }}
         {{- if .Values.global.config.webhooks.podSchedulerName.schedulerName }}

charts/gardener/resource-manager/values.yaml

@@ -119,6 +119,9 @@ global:
         enabled: false
       highAvailabilityConfig:
         enabled: false
+      kubernetesServiceHost:
+        enabled: false
+      # host: api.example.com
       podSchedulerName:
         enabled: false
       # schedulerName: foo-scheduler

charts/images.yaml

@@ -677,7 +677,3 @@ images:
   sourceRepository: github.com/gardener/apiserver-proxy
   repository: eu.gcr.io/gardener-project/gardener/apiserver-proxy
   tag: "v0.12.0"
-- name: apiserver-proxy-pod-webhook
-  sourceRepository: github.com/gardener/apiserver-proxy
-  repository: eu.gcr.io/gardener-project/gardener/apiserver-proxy-pod-webhook
-  tag: "v0.12.0"

docs/concepts/resource-manager.md

@@ -941,6 +941,18 @@ The webhook performs the following actions:
    We consider fine-tuned values for those tolerations a matter of high-availability because they often help to reduce recovery times in case of node or zone outages, also see [High-Availability Best Practices](../../docs/usage/shoot_high_availability_best_practices.md).
    In addition, this webhook handling helps to set defaults for many but not all workload components in a cluster. For instance, Gardener can use this webhook to set defaults for nearly every component in seed clusters but only for the system components in shoot clusters. Any customer workload remains unchanged.
 
+#### Kubernetes Service Host Injection
+
+By default, when `Pod`s are created, Kubernetes implicitly injects the `KUBERNETES_SERVICE_HOST` environment variable into all containers.
+The value of this variable points it to the default Kubernetes service (i.e., `kubernetes.default.svc.cluster.local`).
+This allows pods to conveniently talk to the API server of their cluster.
+
+In shoot clusters, this network path involves the `apiserver-proxy` `DaemonSet` which eventually forwards the traffic to the API server.
+Hence, it results in additional network hop.
+
+The purpose of this webhook is to explicitly inject the `KUBERNETES_SERVICE_HOST` environment variable into all containers and setting its value to the FQDN of the API server.
+This way, the additional network hop is avoided.
+
 #### Auto-Mounting Projected `ServiceAccount` Tokens
 
 When this webhook is activated, then it automatically injects projected `ServiceAccount` token volumes into `Pod`s and all its containers if all of the following preconditions are fulfilled:

example/resource-manager/10-componentconfig.yaml

@@ -85,6 +85,9 @@ webhooks:
     enabled: true
     defaultNotReadyTolerationSeconds: 60
     defaultUnreachableTolerationSeconds: 60
+  kubernetesServiceHost:
+    enabled: true
+    host: api.example.com
   podSchedulerName:
     enabled: true
     schedulerName: foo-scheduler

hack/.ci/set_dependency_version

@@ -127,7 +127,7 @@ elif name == 'etcd-custom-image':
 elif name == 'egress-filter-refresher':
     names = ['egress-filter-blackholer', 'egress-filter-firewaller']
 elif name == 'apiserver-proxy':
-    names = ['apiserver-proxy-sidecar', 'apiserver-proxy-pod-webhook']
+    names = ['apiserver-proxy-sidecar']
 else:
     names = [name]
 

pkg/apis/core/v1beta1/constants/types_constants.go

@@ -578,15 +578,6 @@ const (
 	// AnnotationCoreDNSRewritingDisabled disables core dns query rewriting even if the corresponding feature gate is enabled.
 	AnnotationCoreDNSRewritingDisabled = "alpha.featuregates.shoot.gardener.cloud/core-dns-rewriting-disabled"
 
-	// AnnotationShootAPIServerSNIPodInjector is the key for an annotation of a Shoot cluster whose value indicates
-	// if pod injection of 'KUBERNETES_SERVICE_HOST' environment variable should happen for clusters where APIServerSNI
-	// featuregate is enabled.
-	// Any value than 'disable' enables this feature.
-	AnnotationShootAPIServerSNIPodInjector = "alpha.featuregates.shoot.gardener.cloud/apiserver-sni-pod-injector"
-	// AnnotationShootAPIServerSNIPodInjectorDisableValue is the value of the
-	// `alpha.featuregates.shoot.gardener.cloud/apiserver-sni-pod-injector` annotation that disables the pod injection.
-	AnnotationShootAPIServerSNIPodInjectorDisableValue = "disable"
-
 	// AnnotationSeccompDefaultProfile is the key for an annotation applied to a PodSecurityPolicy which specifies
 	// which is the default seccomp profile to apply to containers.
 	AnnotationSeccompDefaultProfile = "seccomp.security.alpha.kubernetes.io/defaultProfileName"

pkg/apis/resources/v1alpha1/types.go

@@ -138,6 +138,11 @@ const (
 	// defaulting of its seccomp profile.
 	SeccompProfileSkip = "seccompprofile.resources.gardener.cloud/skip"
 
+	// KubernetesServiceHostInject is a constant for a label on a Pod or a Namespace which indicates that all pods in
+	// this namespace (or the specific pod) should not be considered for injection of the KUBERNETES_SERVICE_HOST
+	// environment variable.
+	KubernetesServiceHostInject = "apiserver-proxy.networking.gardener.cloud/inject"
+
 	// SystemComponentsConfigSkip is a constant for a label on a Pod which indicates that this Pod should not be considered for
 	// adding default node selector and tolerations.
 	SystemComponentsConfigSkip = "system-components-config.resources.gardener.cloud/skip"

pkg/component/apiserverproxy/apiserver_proxy.go

@@ -23,7 +23,6 @@ import (
 	"time"
 
 	"github.com/Masterminds/sprig"
-	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	policyv1beta1 "k8s.io/api/policy/v1beta1"
@@ -42,7 +41,6 @@ import (
 	"github.com/gardener/gardener/pkg/utils"
 	kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes"
 	"github.com/gardener/gardener/pkg/utils/managedresources"
-	"github.com/gardener/gardener/pkg/utils/secrets"
 	secretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager"
 )
 
@@ -86,7 +84,6 @@ func init() {
 // Values is a set of configuration values for the apiserver-proxy component.
 type Values struct {
 	ProxySeedServerHost string
-	PodMutatorEnabled   bool
 	PSPDisabled         bool
 	Image               string
 	SidecarImage        string
@@ -202,7 +199,7 @@ func (a *apiserverProxy) computeResourcesData() (map[string][]byte, error) {
 						Name:       "metrics",
 						Port:       adminPort,
 						Protocol:   corev1.ProtocolTCP,
-						TargetPort: intstr.FromInt(int(adminPort)),
+						TargetPort: intstr.FromInt(adminPort),
 					},
 				},
 				Selector: getSelector(),
@@ -341,7 +338,7 @@ func (a *apiserverProxy) computeResourcesData() (map[string][]byte, error) {
 									ProbeHandler: corev1.ProbeHandler{
 										HTTPGet: &corev1.HTTPGetAction{
 											Path: "/ready",
-											Port: intstr.FromInt(int(adminPort)),
+											Port: intstr.FromInt(adminPort),
 										},
 									},
 									InitialDelaySeconds: 1,
@@ -353,7 +350,7 @@ func (a *apiserverProxy) computeResourcesData() (map[string][]byte, error) {
 									ProbeHandler: corev1.ProbeHandler{
 										HTTPGet: &corev1.HTTPGetAction{
 											Path: "/ready",
-											Port: intstr.FromInt(int(adminPort)),
+											Port: intstr.FromInt(adminPort),
 										},
 									},
 									InitialDelaySeconds: 1,
@@ -405,80 +402,6 @@ func (a *apiserverProxy) computeResourcesData() (map[string][]byte, error) {
 		}
 	)
 
-	if a.values.PodMutatorEnabled {
-		clusterCASecret, found := a.secretsManager.Get(v1beta1constants.SecretNameCACluster)
-		if !found {
-			return nil, fmt.Errorf("secret %q not found", v1beta1constants.SecretNameCACluster)
-		}
-		var (
-			failurePolicy      = admissionregistrationv1.Ignore
-			matchPolicy        = admissionregistrationv1.Exact
-			reinvocationPolicy = admissionregistrationv1.NeverReinvocationPolicy
-			scope              = admissionregistrationv1.AllScopes
-			sideEffects        = admissionregistrationv1.SideEffectClassNone
-
-			mutatingWebhook = &admissionregistrationv1.MutatingWebhookConfiguration{
-				ObjectMeta: metav1.ObjectMeta{
-					Name: mutatingWebhookName,
-					Annotations: map[string]string{
-						"networking.gardener.cloud/description": `This webhook adds "KUBERNETES_SERVICE_HOST"
-environment variable to all containers and init containers matched by it.`,
-					},
-					Labels: utils.MergeStringMaps(
-						configMap.GetLabels(),
-						map[string]string{v1beta1constants.LabelExcludeWebhookFromRemediation: "true"}),
-				},
-				Webhooks: []admissionregistrationv1.MutatingWebhook{
-					{
-						AdmissionReviewVersions: []string{"v1"},
-						ClientConfig: admissionregistrationv1.WebhookClientConfig{
-							CABundle: clusterCASecret.Data[secrets.DataKeyCertificateBundle],
-							URL:      pointer.String("https://127.0.0.1:9443/webhook/pod-apiserver-env"),
-						},
-						FailurePolicy: &failurePolicy,
-						MatchPolicy:   &matchPolicy,
-						Name:          mutatingWebhookName,
-						NamespaceSelector: &metav1.LabelSelector{
-							MatchExpressions: []metav1.LabelSelectorRequirement{
-								{
-									Key:      webhookExpressionsKey,
-									Operator: metav1.LabelSelectorOpNotIn,
-									Values:   []string{"disable"},
-								},
-							},
-						},
-						ObjectSelector: &metav1.LabelSelector{
-							MatchExpressions: []metav1.LabelSelectorRequirement{
-								{
-									Key:      webhookExpressionsKey,
-									Operator: metav1.LabelSelectorOpNotIn,
-									Values:   []string{"disable"},
-								},
-							},
-						},
-						ReinvocationPolicy: &reinvocationPolicy,
-						Rules: []admissionregistrationv1.RuleWithOperations{
-							{
-								Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create},
-								Rule: admissionregistrationv1.Rule{
-									APIGroups:   []string{""},
-									APIVersions: []string{"v1"},
-									Resources:   []string{"pods"},
-									Scope:       &scope,
-								},
-							},
-						},
-						SideEffects:    &sideEffects,
-						TimeoutSeconds: pointer.Int32(2),
-					},
-				},
-			}
-		)
-		if err := registry.Add(mutatingWebhook); err != nil {
-			return nil, err
-		}
-	}
-
 	if !a.values.PSPDisabled {
 		var (
 			clusterRole = &rbacv1.ClusterRole{

pkg/component/apiserverproxy/apiserver_proxy_test.go

@@ -412,58 +412,6 @@ metadata:
   namespace: kube-system
 `
 
-		webhokkConfigYAML = `apiVersion: admissionregistration.k8s.io/v1
-kind: MutatingWebhookConfiguration
-metadata:
-  annotations:
-    networking.gardener.cloud/description: |-
-      This webhook adds "KUBERNETES_SERVICE_HOST"
-      environment variable to all containers and init containers matched by it.
-  creationTimestamp: null
-  labels:
-    app: kubernetes
-    gardener.cloud/role: system-component
-    origin: gardener
-    remediation.webhook.shoot.gardener.cloud/exclude: "true"
-    resources.gardener.cloud/garbage-collectable-reference: "true"
-    role: apiserver-proxy
-  name: apiserver-proxy.networking.gardener.cloud
-webhooks:
-- admissionReviewVersions:
-  - v1
-  clientConfig:
-    caBundle: Rk9PQkFS
-    url: https://127.0.0.1:9443/webhook/pod-apiserver-env
-  failurePolicy: Ignore
-  matchPolicy: Exact
-  name: apiserver-proxy.networking.gardener.cloud
-  namespaceSelector:
-    matchExpressions:
-    - key: apiserver-proxy.networking.gardener.cloud/inject
-      operator: NotIn
-      values:
-      - disable
-  objectSelector:
-    matchExpressions:
-    - key: apiserver-proxy.networking.gardener.cloud/inject
-      operator: NotIn
-      values:
-      - disable
-  reinvocationPolicy: Never
-  rules:
-  - apiGroups:
-    - ""
-    apiVersions:
-    - v1
-    operations:
-    - CREATE
-    resources:
-    - pods
-    scope: '*'
-  sideEffects: None
-  timeoutSeconds: 2
-`
-
 		clusterRoleYAML = `apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -578,13 +526,12 @@ subjects:
 	})
 
 	Describe("#Deploy", func() {
-		test := func(podMutatorEnabled, pspDisabled bool) {
+		test := func(pspDisabled bool) {
 			By("Verify that managed resource does not exist yet")
 			Expect(c.Get(ctx, client.ObjectKeyFromObject(managedResource), managedResource)).To(MatchError(apierrors.NewNotFound(schema.GroupResource{Group: resourcesv1alpha1.SchemeGroupVersion.Group, Resource: "managedresources"}, managedResource.Name)))
 			Expect(c.Get(ctx, client.ObjectKeyFromObject(managedResourceSecret), managedResourceSecret)).To(MatchError(apierrors.NewNotFound(schema.GroupResource{Group: corev1.SchemeGroupVersion.Group, Resource: "secrets"}, managedResourceSecret.Name)))
 
 			By("Deploy the managed resource successfully")
-			values.PodMutatorEnabled = podMutatorEnabled
 			values.PSPDisabled = pspDisabled
 			component = New(c, namespace, sm, values)
 			component.SetAdvertiseIPAddress(advertiseIPAddress)
@@ -622,32 +569,22 @@ subjects:
 				Expect(string(managedResourceSecret.Data["podsecuritypolicy____gardener.kube-system.apiserver-proxy.yaml"])).To(Equal(pspYAML))
 				Expect(string(managedResourceSecret.Data["rolebinding__kube-system__gardener.cloud_psp_apiserver-proxy.yaml"])).To(Equal(roleBindingYAML))
 			}
-			if podMutatorEnabled {
-				expectedLen++
-				Expect(string(managedResourceSecret.Data["mutatingwebhookconfiguration____apiserver-proxy.networking.gardener.cloud.yaml"])).To(Equal(webhokkConfigYAML))
-			}
 			Expect(managedResourceSecret.Data).To(HaveLen(expectedLen))
 			Expect(string(managedResourceSecret.Data["configmap__kube-system__apiserver-proxy-config-4baf1826.yaml"])).To(Equal(configMapYAML))
 			Expect(string(managedResourceSecret.Data["daemonset__kube-system__apiserver-proxy.yaml"])).To(Equal(daemonSetYAML))
 			Expect(string(managedResourceSecret.Data["service__kube-system__apiserver-proxy.yaml"])).To(Equal(serviceYAML))
 			Expect(string(managedResourceSecret.Data["serviceaccount__kube-system__apiserver-proxy.yaml"])).To(Equal(serviceAccountYAML))
 		}
 
-		Context("Pod mutator disabled, PSP disabled", func() {
-			It("should deploy the managed resource successfully", func() {
-				test(false, true)
-			})
-		})
-
-		Context("Pod mutator enabled, PSP disabled", func() {
+		Context("PSP disabled", func() {
 			It("should deploy the managed resource successfully", func() {
-				test(true, true)
+				test(true)
 			})
 		})
 
-		Context("Pod mutator enabled, PSP enabled", func() {
+		Context("PSP enabled", func() {
 			It("should deploy the managed resource successfully", func() {
-				test(true, false)
+				test(false)
 			})
 		})
 	})

pkg/component/kubeapiserver/deployment.go

@@ -55,10 +55,9 @@ const (
 	secretNameHAVPNSeedClient        = "vpn-seed-client"
 
 	// ContainerNameKubeAPIServer is the name of the kube-apiserver container.
-	ContainerNameKubeAPIServer            = "kube-apiserver"
-	containerNameVPNSeedClient            = "vpn-client"
-	containerNameAPIServerProxyPodMutator = "apiserver-proxy-pod-mutator"
-	containerNameWatchdog                 = "watchdog"
+	ContainerNameKubeAPIServer = "kube-apiserver"
+	containerNameVPNSeedClient = "vpn-client"
+	containerNameWatchdog      = "watchdog"
 
 	volumeNameAdmissionConfiguration          = "admission-config"
 	volumeNameAdmissionKubeconfigSecrets      = "admission-kubeconfigs"
@@ -465,7 +464,6 @@ func (k *kubeAPIServer) reconcileDeployment(
 		k.handleHostCertVolumes(deployment)
 		k.handleSNISettings(deployment)
 		k.handleTLSSNISettings(deployment, tlsSNISecrets)
-		k.handlePodMutatorSettings(deployment)
 		k.handleOIDCSettings(deployment, secretOIDCCABundle)
 		k.handleServiceAccountSigningKeySettings(deployment)
 		k.handleAuditSettings(deployment, configMapAuditPolicy, secretAuditWebhookKubeconfig)
@@ -1210,36 +1208,6 @@ func (k *kubeAPIServer) handleServiceAccountSigningKeySettings(deployment *appsv
 	deployment.Spec.Template.Spec.Containers[0].Command = append(deployment.Spec.Template.Spec.Containers[0].Command, fmt.Sprintf("--service-account-key-file=%s/%s", volumeMountPathServiceAccountKeyBundle, secrets.DataKeyPrivateKeyBundle))
 }
 
-func (k *kubeAPIServer) handlePodMutatorSettings(deployment *appsv1.Deployment) {
-	if k.values.SNI.PodMutatorEnabled {
-		deployment.Spec.Template.Spec.Containers = append(deployment.Spec.Template.Spec.Containers, corev1.Container{
-			Name:  containerNameAPIServerProxyPodMutator,
-			Image: k.values.Images.APIServerProxyPodWebhook,
-			Args: []string{
-				"--apiserver-fqdn=" + k.values.SNI.APIServerFQDN,
-				"--host=localhost",
-				"--port=9443",
-				"--cert-dir=" + volumeMountPathServer,
-				"--cert-name=" + secrets.DataKeyCertificate,
-				"--key-name=" + secrets.DataKeyPrivateKey,
-			},
-			Resources: corev1.ResourceRequirements{
-				Requests: corev1.ResourceList{
-					corev1.ResourceCPU:    resource.MustParse("50m"),
-					corev1.ResourceMemory: resource.MustParse("128M"),
-				},
-				Limits: corev1.ResourceList{
-					corev1.ResourceMemory: resource.MustParse("500M"),
-				},
-			},
-			VolumeMounts: []corev1.VolumeMount{{
-				Name:      volumeNameServer,
-				MountPath: volumeMountPathServer,
-			}},
-		})
-	}
-}
-
 func (k *kubeAPIServer) handleKubeletSettings(deployment *appsv1.Deployment, secretKubeletClient *corev1.Secret) error {
 	if k.values.IsWorkerless {
 		return nil