[operator] Introduce gardenercontrollermanager Golang component (ga…

…rdener#8282)

* Add `gardenercontrollermanager` component boilerplate

* Generate access secret for virtual garden cluster

Ref https://github.com/gardener/gardener/blob/master/charts/gardener/controlplane/charts/runtime/templates/controller-manager/secret-kubeconfig.yaml

* Generate controller manager config `ConfigMap`

Ref https://github.com/gardener/gardener/blob/master/charts/gardener/controlplane/charts/runtime/templates/controller-manager/configmap-componentconfig.yaml

* Generate `PodDisruptionBudget`

Ref https://github.com/gardener/gardener/blob/master/charts/gardener/controlplane/charts/runtime/templates/controller-manager/poddisruptionbudget.yaml

* Generate `Service`

Ref https://github.com/gardener/gardener/blob/master/charts/gardener/controlplane/charts/runtime/templates/controller-manager/service.yaml

* Generate `VPA`

Ref https://github.com/gardener/gardener/blob/master/charts/gardener/controlplane/charts/runtime/templates/controller-manager/vpa.yaml

* Generate `Deployment`

Ref https://github.com/gardener/gardener/blob/master/charts/gardener/controlplane/charts/runtime/templates/controller-manager/deployment.yaml

* Generate `RBAC`

Ref https://github.com/gardener/gardener/blob/master/charts/gardener/controlplane/charts/application/templates/clusterrole-controller-manager.yaml
Ref https://github.com/gardener/gardener/blob/master/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-controller-manager.yaml

* Addressed review feedback.

Most notably adapted to the new changed with regards to managed resources and added defaulting to `EnableShootControlPlaneRestarter`.

* Address review feedback (part 2)

* Fixed import constraint by not using pointer module

* Address review feedback (part 3)

docs/development/priority-classes.md

@@ -25,7 +25,7 @@ When using the `gardener-operator` for managing the garden runtime and virtual c
 | `gardener-garden-system-500`      | 999999500 | `virtual-garden-etcd-events`, `virtual-garden-etcd-main`, `virtual-garden-kube-apiserver`, `gardener-apiserver` |
 | `gardener-garden-system-400`      | 999999400 | `virtual-garden-gardener-resource-manager`, `gardener-admission-controller`                                     |
 | `gardener-garden-system-300`      | 999999300 | `virtual-garden-kube-controller-manager`, `vpa-admission-controller`, `etcd-druid`, `nginx-ingress-controller`  |
-| `gardener-garden-system-200`      | 999999200 | `vpa-recommender`, `vpa-updater`, `hvpa-controller`,                                                            |
+| `gardener-garden-system-200`      | 999999200 | `vpa-recommender`, `vpa-updater`, `hvpa-controller`, `gardener-scheduler`, `gardener-controller-manager`        |
 | `gardener-garden-system-100`      | 999999100 | `kube-state-metrics`, `fluent-operator`, `fluent-bit`, `vali`                                                   |
 
 ## Seed Clusters

pkg/component/gardenercontrollermanager/configmaps.go

@@ -0,0 +1,138 @@
+// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gardenercontrollermanager
+
+import (
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/runtime/serializer/json"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	componentbaseconfigv1alpha1 "k8s.io/component-base/config/v1alpha1"
+	"k8s.io/utils/pointer"
+
+	controllermanagerv1alpha1 "github.com/gardener/gardener/pkg/controllermanager/apis/config/v1alpha1"
+	"github.com/gardener/gardener/pkg/logger"
+	gardenerutils "github.com/gardener/gardener/pkg/utils/gardener"
+	kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes"
+)
+
+const (
+	configMapControllerManagerPrefix = "gardener-controller-manager-config"
+	dataConfigKey                    = "config.yaml"
+)
+
+var controllerManagerCodec runtime.Codec
+
+func init() {
+	controllerManagerScheme := runtime.NewScheme()
+	utilruntime.Must(controllermanagerv1alpha1.AddToScheme(controllerManagerScheme))
+
+	var (
+		ser = json.NewSerializerWithOptions(json.DefaultMetaFactory, controllerManagerScheme, controllerManagerScheme, json.SerializerOptions{
+			Yaml:   true,
+			Pretty: false,
+			Strict: false,
+		})
+		versions = schema.GroupVersions([]schema.GroupVersion{
+			controllermanagerv1alpha1.SchemeGroupVersion,
+		})
+	)
+
+	controllerManagerCodec = serializer.NewCodecFactory(controllerManagerScheme).CodecForVersions(ser, ser, versions, versions)
+}
+
+func (g *gardenerControllerManager) configMapControllerManagerConfig() (*corev1.ConfigMap, error) {
+	controllerManagerConfig := &controllermanagerv1alpha1.ControllerManagerConfiguration{
+		GardenClientConnection: componentbaseconfigv1alpha1.ClientConnectionConfiguration{
+			QPS:        100,
+			Burst:      130,
+			Kubeconfig: gardenerutils.PathGenericKubeconfig,
+		},
+		Controllers: controllermanagerv1alpha1.ControllerManagerControllerConfiguration{
+			ControllerRegistration: &controllermanagerv1alpha1.ControllerRegistrationControllerConfiguration{
+				ConcurrentSyncs: pointer.Int(20),
+			},
+			Project: &controllermanagerv1alpha1.ProjectControllerConfiguration{
+				ConcurrentSyncs: pointer.Int(20),
+				Quotas:          g.values.Quotas,
+			},
+			SecretBinding: &controllermanagerv1alpha1.SecretBindingControllerConfiguration{
+				ConcurrentSyncs: pointer.Int(20),
+			},
+			Seed: &controllermanagerv1alpha1.SeedControllerConfiguration{
+				ConcurrentSyncs:    pointer.Int(20),
+				ShootMonitorPeriod: &metav1.Duration{Duration: 300 * time.Second},
+			},
+			SeedExtensionsCheck: &controllermanagerv1alpha1.SeedExtensionsCheckControllerConfiguration{
+				ConditionThresholds: []controllermanagerv1alpha1.ConditionThreshold{{
+					Duration: metav1.Duration{Duration: 1 * time.Minute},
+					Type:     "ExtensionsReady",
+				}},
+			},
+			SeedBackupBucketsCheck: &controllermanagerv1alpha1.SeedBackupBucketsCheckControllerConfiguration{
+				ConditionThresholds: []controllermanagerv1alpha1.ConditionThreshold{{
+					Duration: metav1.Duration{Duration: 1 * time.Minute},
+					Type:     "BackupBucketsReady",
+				}},
+			},
+			Event: &controllermanagerv1alpha1.EventControllerConfiguration{
+				ConcurrentSyncs:   pointer.Int(10),
+				TTLNonShootEvents: &metav1.Duration{Duration: 2 * time.Hour},
+			},
+			ShootMaintenance: controllermanagerv1alpha1.ShootMaintenanceControllerConfiguration{
+				ConcurrentSyncs: pointer.Int(20),
+			},
+			ShootReference: &controllermanagerv1alpha1.ShootReferenceControllerConfiguration{
+				ConcurrentSyncs: pointer.Int(20),
+			},
+		},
+		LeaderElection: &componentbaseconfigv1alpha1.LeaderElectionConfiguration{
+			LeaderElect:       pointer.Bool(true),
+			ResourceName:      controllermanagerv1alpha1.ControllerManagerDefaultLockObjectName,
+			ResourceNamespace: metav1.NamespaceSystem,
+		},
+		LogLevel:  g.values.LogLevel,
+		LogFormat: logger.FormatJSON,
+		Server: controllermanagerv1alpha1.ServerConfiguration{
+			HealthProbes: &controllermanagerv1alpha1.Server{Port: probePort},
+			Metrics:      &controllermanagerv1alpha1.Server{Port: metricsPort},
+		},
+		FeatureGates: g.values.FeatureGates,
+	}
+
+	data, err := runtime.Encode(controllerManagerCodec, controllerManagerConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	configMap := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      configMapControllerManagerPrefix,
+			Namespace: g.namespace,
+			Labels:    GetLabels(),
+		},
+		Data: map[string]string{
+			dataConfigKey: string(data),
+		},
+	}
+
+	utilruntime.Must(kubernetesutils.MakeUnique(configMap))
+	return configMap, nil
+}

pkg/component/gardenercontrollermanager/deployment.go

@@ -0,0 +1,128 @@
+// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gardenercontrollermanager
+
+import (
+	"fmt"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/utils/pointer"
+
+	v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
+	resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1"
+	kubeapiserverconstants "github.com/gardener/gardener/pkg/component/kubeapiserver/constants"
+	"github.com/gardener/gardener/pkg/resourcemanager/controller/garbagecollector/references"
+	"github.com/gardener/gardener/pkg/utils"
+	gardenerutils "github.com/gardener/gardener/pkg/utils/gardener"
+)
+
+const (
+	volumeMountConfig = "/etc/gardener-controller-manager/config"
+	volumeNameConfig  = "gardener-controller-manager-config"
+)
+
+func (g *gardenerControllerManager) deployment(secretGenericTokenKubeconfig, secretVirtualGardenAccess, configMapControllerManagerConfig string) *appsv1.Deployment {
+	deployment := &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      DeploymentName,
+			Namespace: g.namespace,
+			Labels: utils.MergeStringMaps(GetLabels(), map[string]string{
+				resourcesv1alpha1.HighAvailabilityConfigType: resourcesv1alpha1.HighAvailabilityConfigTypeController,
+			}),
+		},
+		Spec: appsv1.DeploymentSpec{
+			Replicas: pointer.Int32(1),
+			Selector: &metav1.LabelSelector{
+				MatchLabels: GetLabels(),
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: utils.MergeStringMaps(GetLabels(), map[string]string{
+						v1beta1constants.LabelNetworkPolicyToDNS: v1beta1constants.LabelNetworkPolicyAllowed,
+						gardenerutils.NetworkPolicyLabel("virtual-garden-"+v1beta1constants.DeploymentNameKubeAPIServer, kubeapiserverconstants.Port): v1beta1constants.LabelNetworkPolicyAllowed,
+					}),
+				},
+				Spec: corev1.PodSpec{
+					PriorityClassName:            v1beta1constants.PriorityClassNameGardenSystem200,
+					AutomountServiceAccountToken: pointer.Bool(false),
+					Containers: []corev1.Container{
+						{
+							Name:            DeploymentName,
+							Image:           g.values.Image,
+							ImagePullPolicy: corev1.PullIfNotPresent,
+							Args: []string{
+								fmt.Sprintf("--config=%s/%s", volumeMountConfig, dataConfigKey),
+							},
+							Resources: corev1.ResourceRequirements{
+								Requests: map[corev1.ResourceName]resource.Quantity{
+									corev1.ResourceCPU:    resource.MustParse("1"),
+									corev1.ResourceMemory: resource.MustParse("1Gi"),
+								},
+							},
+							LivenessProbe: &corev1.Probe{
+								ProbeHandler: corev1.ProbeHandler{
+									HTTPGet: &corev1.HTTPGetAction{
+										Path:   "/healthz",
+										Port:   intstr.FromInt(probePort),
+										Scheme: corev1.URISchemeHTTP,
+									},
+								},
+								InitialDelaySeconds: 30,
+								TimeoutSeconds:      5,
+							},
+							ReadinessProbe: &corev1.Probe{
+								ProbeHandler: corev1.ProbeHandler{
+									HTTPGet: &corev1.HTTPGetAction{
+										Path:   "/readyz",
+										Port:   intstr.FromInt(probePort),
+										Scheme: corev1.URISchemeHTTP,
+									},
+								},
+								InitialDelaySeconds: 10,
+								TimeoutSeconds:      5,
+							},
+							VolumeMounts: []corev1.VolumeMount{
+								{
+									Name:      volumeNameConfig,
+									MountPath: volumeMountConfig,
+								},
+							},
+						},
+					},
+					Volumes: []corev1.Volume{
+						{
+							Name: volumeNameConfig,
+							VolumeSource: corev1.VolumeSource{
+								ConfigMap: &corev1.ConfigMapVolumeSource{
+									LocalObjectReference: corev1.LocalObjectReference{Name: configMapControllerManagerConfig},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	utilruntime.Must(gardenerutils.InjectGenericKubeconfig(deployment, secretGenericTokenKubeconfig, secretVirtualGardenAccess))
+	utilruntime.Must(references.InjectAnnotations(deployment))
+
+	return deployment
+}