From b51dac49da00f0cb5443b03fff7408f06dd3743b Mon Sep 17 00:00:00 2001 From: Johannes Scheerer Date: Fri, 4 Aug 2023 14:49:15 +0200 Subject: [PATCH] [operator] Introduce `gardenercontrollermanager` Golang component (#8282) * Add `gardenercontrollermanager` component boilerplate * Generate access secret for virtual garden cluster Ref https://github.com/gardener/gardener/blob/master/charts/gardener/controlplane/charts/runtime/templates/controller-manager/secret-kubeconfig.yaml * Generate controller manager config `ConfigMap` Ref https://github.com/gardener/gardener/blob/master/charts/gardener/controlplane/charts/runtime/templates/controller-manager/configmap-componentconfig.yaml * Generate `PodDisruptionBudget` Ref https://github.com/gardener/gardener/blob/master/charts/gardener/controlplane/charts/runtime/templates/controller-manager/poddisruptionbudget.yaml * Generate `Service` Ref https://github.com/gardener/gardener/blob/master/charts/gardener/controlplane/charts/runtime/templates/controller-manager/service.yaml * Generate `VPA` Ref https://github.com/gardener/gardener/blob/master/charts/gardener/controlplane/charts/runtime/templates/controller-manager/vpa.yaml * Generate `Deployment` Ref https://github.com/gardener/gardener/blob/master/charts/gardener/controlplane/charts/runtime/templates/controller-manager/deployment.yaml * Generate `RBAC` Ref https://github.com/gardener/gardener/blob/master/charts/gardener/controlplane/charts/application/templates/clusterrole-controller-manager.yaml Ref https://github.com/gardener/gardener/blob/master/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-controller-manager.yaml * Addressed review feedback. Most notably adapted to the new changed with regards to managed resources and added defaulting to `EnableShootControlPlaneRestarter`. * Address review feedback (part 2) * Fixed import constraint by not using pointer module * Address review feedback (part 3) --- docs/development/priority-classes.md | 2 +- .../gardenercontrollermanager/configmaps.go | 138 +++ .../gardenercontrollermanager/deployment.go | 128 +++ .../gardener_controller_manager.go | 176 ++++ .../gardener_controller_manager_suite_test.go | 27 + .../gardener_controller_manager_test.go | 879 ++++++++++++++++++ .../poddisruptionbudget.go | 36 + .../gardenercontrollermanager/rbac.go | 60 ++ .../gardenercontrollermanager/secrets.go | 23 + .../gardenercontrollermanager/service.go | 45 + .../gardenercontrollermanager/vpa.go | 55 ++ .../apis/.import-restrictions | 1 + .../apis/config/v1alpha1/defaults.go | 4 + .../apis/config/v1alpha1/defaults_test.go | 2 + 14 files changed, 1575 insertions(+), 1 deletion(-) create mode 100644 pkg/component/gardenercontrollermanager/configmaps.go create mode 100644 pkg/component/gardenercontrollermanager/deployment.go create mode 100644 pkg/component/gardenercontrollermanager/gardener_controller_manager.go create mode 100644 pkg/component/gardenercontrollermanager/gardener_controller_manager_suite_test.go create mode 100644 pkg/component/gardenercontrollermanager/gardener_controller_manager_test.go create mode 100644 pkg/component/gardenercontrollermanager/poddisruptionbudget.go create mode 100644 pkg/component/gardenercontrollermanager/rbac.go create mode 100644 pkg/component/gardenercontrollermanager/secrets.go create mode 100644 pkg/component/gardenercontrollermanager/service.go create mode 100644 pkg/component/gardenercontrollermanager/vpa.go diff --git a/docs/development/priority-classes.md b/docs/development/priority-classes.md index 84f1ad38288..fa8d6f873fd 100644 --- a/docs/development/priority-classes.md +++ b/docs/development/priority-classes.md @@ -25,7 +25,7 @@ When using the `gardener-operator` for managing the garden runtime and virtual c | `gardener-garden-system-500` | 999999500 | `virtual-garden-etcd-events`, `virtual-garden-etcd-main`, `virtual-garden-kube-apiserver`, `gardener-apiserver` | | `gardener-garden-system-400` | 999999400 | `virtual-garden-gardener-resource-manager`, `gardener-admission-controller` | | `gardener-garden-system-300` | 999999300 | `virtual-garden-kube-controller-manager`, `vpa-admission-controller`, `etcd-druid`, `nginx-ingress-controller` | -| `gardener-garden-system-200` | 999999200 | `vpa-recommender`, `vpa-updater`, `hvpa-controller`, | +| `gardener-garden-system-200` | 999999200 | `vpa-recommender`, `vpa-updater`, `hvpa-controller`, `gardener-scheduler`, `gardener-controller-manager` | | `gardener-garden-system-100` | 999999100 | `kube-state-metrics`, `fluent-operator`, `fluent-bit`, `vali` | ## Seed Clusters diff --git a/pkg/component/gardenercontrollermanager/configmaps.go b/pkg/component/gardenercontrollermanager/configmaps.go new file mode 100644 index 00000000000..f49666216bb --- /dev/null +++ b/pkg/component/gardenercontrollermanager/configmaps.go @@ -0,0 +1,138 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gardenercontrollermanager + +import ( + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/runtime/serializer/json" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + componentbaseconfigv1alpha1 "k8s.io/component-base/config/v1alpha1" + "k8s.io/utils/pointer" + + controllermanagerv1alpha1 "github.com/gardener/gardener/pkg/controllermanager/apis/config/v1alpha1" + "github.com/gardener/gardener/pkg/logger" + gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" + kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" +) + +const ( + configMapControllerManagerPrefix = "gardener-controller-manager-config" + dataConfigKey = "config.yaml" +) + +var controllerManagerCodec runtime.Codec + +func init() { + controllerManagerScheme := runtime.NewScheme() + utilruntime.Must(controllermanagerv1alpha1.AddToScheme(controllerManagerScheme)) + + var ( + ser = json.NewSerializerWithOptions(json.DefaultMetaFactory, controllerManagerScheme, controllerManagerScheme, json.SerializerOptions{ + Yaml: true, + Pretty: false, + Strict: false, + }) + versions = schema.GroupVersions([]schema.GroupVersion{ + controllermanagerv1alpha1.SchemeGroupVersion, + }) + ) + + controllerManagerCodec = serializer.NewCodecFactory(controllerManagerScheme).CodecForVersions(ser, ser, versions, versions) +} + +func (g *gardenerControllerManager) configMapControllerManagerConfig() (*corev1.ConfigMap, error) { + controllerManagerConfig := &controllermanagerv1alpha1.ControllerManagerConfiguration{ + GardenClientConnection: componentbaseconfigv1alpha1.ClientConnectionConfiguration{ + QPS: 100, + Burst: 130, + Kubeconfig: gardenerutils.PathGenericKubeconfig, + }, + Controllers: controllermanagerv1alpha1.ControllerManagerControllerConfiguration{ + ControllerRegistration: &controllermanagerv1alpha1.ControllerRegistrationControllerConfiguration{ + ConcurrentSyncs: pointer.Int(20), + }, + Project: &controllermanagerv1alpha1.ProjectControllerConfiguration{ + ConcurrentSyncs: pointer.Int(20), + Quotas: g.values.Quotas, + }, + SecretBinding: &controllermanagerv1alpha1.SecretBindingControllerConfiguration{ + ConcurrentSyncs: pointer.Int(20), + }, + Seed: &controllermanagerv1alpha1.SeedControllerConfiguration{ + ConcurrentSyncs: pointer.Int(20), + ShootMonitorPeriod: &metav1.Duration{Duration: 300 * time.Second}, + }, + SeedExtensionsCheck: &controllermanagerv1alpha1.SeedExtensionsCheckControllerConfiguration{ + ConditionThresholds: []controllermanagerv1alpha1.ConditionThreshold{{ + Duration: metav1.Duration{Duration: 1 * time.Minute}, + Type: "ExtensionsReady", + }}, + }, + SeedBackupBucketsCheck: &controllermanagerv1alpha1.SeedBackupBucketsCheckControllerConfiguration{ + ConditionThresholds: []controllermanagerv1alpha1.ConditionThreshold{{ + Duration: metav1.Duration{Duration: 1 * time.Minute}, + Type: "BackupBucketsReady", + }}, + }, + Event: &controllermanagerv1alpha1.EventControllerConfiguration{ + ConcurrentSyncs: pointer.Int(10), + TTLNonShootEvents: &metav1.Duration{Duration: 2 * time.Hour}, + }, + ShootMaintenance: controllermanagerv1alpha1.ShootMaintenanceControllerConfiguration{ + ConcurrentSyncs: pointer.Int(20), + }, + ShootReference: &controllermanagerv1alpha1.ShootReferenceControllerConfiguration{ + ConcurrentSyncs: pointer.Int(20), + }, + }, + LeaderElection: &componentbaseconfigv1alpha1.LeaderElectionConfiguration{ + LeaderElect: pointer.Bool(true), + ResourceName: controllermanagerv1alpha1.ControllerManagerDefaultLockObjectName, + ResourceNamespace: metav1.NamespaceSystem, + }, + LogLevel: g.values.LogLevel, + LogFormat: logger.FormatJSON, + Server: controllermanagerv1alpha1.ServerConfiguration{ + HealthProbes: &controllermanagerv1alpha1.Server{Port: probePort}, + Metrics: &controllermanagerv1alpha1.Server{Port: metricsPort}, + }, + FeatureGates: g.values.FeatureGates, + } + + data, err := runtime.Encode(controllerManagerCodec, controllerManagerConfig) + if err != nil { + return nil, err + } + + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapControllerManagerPrefix, + Namespace: g.namespace, + Labels: GetLabels(), + }, + Data: map[string]string{ + dataConfigKey: string(data), + }, + } + + utilruntime.Must(kubernetesutils.MakeUnique(configMap)) + return configMap, nil +} diff --git a/pkg/component/gardenercontrollermanager/deployment.go b/pkg/component/gardenercontrollermanager/deployment.go new file mode 100644 index 00000000000..a05a7c62195 --- /dev/null +++ b/pkg/component/gardenercontrollermanager/deployment.go @@ -0,0 +1,128 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gardenercontrollermanager + +import ( + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/utils/pointer" + + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" + kubeapiserverconstants "github.com/gardener/gardener/pkg/component/kubeapiserver/constants" + "github.com/gardener/gardener/pkg/resourcemanager/controller/garbagecollector/references" + "github.com/gardener/gardener/pkg/utils" + gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" +) + +const ( + volumeMountConfig = "/etc/gardener-controller-manager/config" + volumeNameConfig = "gardener-controller-manager-config" +) + +func (g *gardenerControllerManager) deployment(secretGenericTokenKubeconfig, secretVirtualGardenAccess, configMapControllerManagerConfig string) *appsv1.Deployment { + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: DeploymentName, + Namespace: g.namespace, + Labels: utils.MergeStringMaps(GetLabels(), map[string]string{ + resourcesv1alpha1.HighAvailabilityConfigType: resourcesv1alpha1.HighAvailabilityConfigTypeController, + }), + }, + Spec: appsv1.DeploymentSpec{ + Replicas: pointer.Int32(1), + Selector: &metav1.LabelSelector{ + MatchLabels: GetLabels(), + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: utils.MergeStringMaps(GetLabels(), map[string]string{ + v1beta1constants.LabelNetworkPolicyToDNS: v1beta1constants.LabelNetworkPolicyAllowed, + gardenerutils.NetworkPolicyLabel("virtual-garden-"+v1beta1constants.DeploymentNameKubeAPIServer, kubeapiserverconstants.Port): v1beta1constants.LabelNetworkPolicyAllowed, + }), + }, + Spec: corev1.PodSpec{ + PriorityClassName: v1beta1constants.PriorityClassNameGardenSystem200, + AutomountServiceAccountToken: pointer.Bool(false), + Containers: []corev1.Container{ + { + Name: DeploymentName, + Image: g.values.Image, + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{ + fmt.Sprintf("--config=%s/%s", volumeMountConfig, dataConfigKey), + }, + Resources: corev1.ResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(probePort), + Scheme: corev1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 30, + TimeoutSeconds: 5, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/readyz", + Port: intstr.FromInt(probePort), + Scheme: corev1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 10, + TimeoutSeconds: 5, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: volumeNameConfig, + MountPath: volumeMountConfig, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: volumeNameConfig, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: configMapControllerManagerConfig}, + }, + }, + }, + }, + }, + }, + }, + } + + utilruntime.Must(gardenerutils.InjectGenericKubeconfig(deployment, secretGenericTokenKubeconfig, secretVirtualGardenAccess)) + utilruntime.Must(references.InjectAnnotations(deployment)) + + return deployment +} diff --git a/pkg/component/gardenercontrollermanager/gardener_controller_manager.go b/pkg/component/gardenercontrollermanager/gardener_controller_manager.go new file mode 100644 index 00000000000..6a48bb8a7ab --- /dev/null +++ b/pkg/component/gardenercontrollermanager/gardener_controller_manager.go @@ -0,0 +1,176 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gardenercontrollermanager + +import ( + "context" + "fmt" + "time" + + "sigs.k8s.io/controller-runtime/pkg/client" + + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/gardener/pkg/component" + controllermanagerv1alpha1 "github.com/gardener/gardener/pkg/controllermanager/apis/config/v1alpha1" + operatorclient "github.com/gardener/gardener/pkg/operator/client" + "github.com/gardener/gardener/pkg/utils/flow" + kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" + "github.com/gardener/gardener/pkg/utils/managedresources" + secretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager" +) + +const ( + // DeploymentName is the name of the deployment. + DeploymentName = "gardener-controller-manager" + + probePort = 2718 + metricsPort = 2719 + + managedResourceNameRuntime = "gardener-controller-manager-runtime" + managedResourceNameVirtual = "gardener-controller-manager-virtual" + + roleName = "controller-manager" +) + +// TimeoutWaitForManagedResource is the timeout used while waiting for the ManagedResources to become healthy or +// deleted. +var TimeoutWaitForManagedResource = 5 * time.Minute + +// Values contains configuration values for the gardener-controller-manager resources. +type Values struct { + // Image defines the container image of gardener-controller-manager. + Image string + // LogLevel is the level/severity for the logs. Must be one of [info,debug,error]. + LogLevel string + // Quotas is the default configuration matching projects are set up with if a quota is not already specified. + Quotas []controllermanagerv1alpha1.QuotaConfiguration + // FeatureGates is the set of feature gates. + FeatureGates map[string]bool +} + +// New creates a new instance of DeployWaiter for the gardener-controller-manager. +func New(client client.Client, namespace string, secretsManager secretsmanager.Interface, values Values) component.DeployWaiter { + return &gardenerControllerManager{ + client: client, + namespace: namespace, + secretsManager: secretsManager, + values: values, + } +} + +type gardenerControllerManager struct { + client client.Client + namespace string + secretsManager secretsmanager.Interface + values Values +} + +func (g *gardenerControllerManager) Deploy(ctx context.Context) error { + var ( + runtimeRegistry = managedresources.NewRegistry(operatorclient.RuntimeScheme, operatorclient.RuntimeCodec, operatorclient.RuntimeSerializer) + virtualGardenAccessSecret = g.newVirtualGardenAccessSecret() + ) + + if err := virtualGardenAccessSecret.Reconcile(ctx, g.client); err != nil { + return err + } + + controllerManagerConfigConfigMap, err := g.configMapControllerManagerConfig() + if err != nil { + return err + } + + secretGenericTokenKubeconfig, found := g.secretsManager.Get(v1beta1constants.SecretNameGenericTokenKubeconfig) + if !found { + return fmt.Errorf("secret %q not found", v1beta1constants.SecretNameGenericTokenKubeconfig) + } + + runtimeResources, err := runtimeRegistry.AddAllAndSerialize( + controllerManagerConfigConfigMap, + g.podDisruptionBudget(), + g.service(), + g.verticalPodAutoscaler(), + g.deployment(secretGenericTokenKubeconfig.Name, virtualGardenAccessSecret.Secret.Name, controllerManagerConfigConfigMap.Name), + ) + if err != nil { + return err + } + + if err := managedresources.CreateForSeed(ctx, g.client, g.namespace, managedResourceNameRuntime, false, runtimeResources); err != nil { + return err + } + + var ( + virtualRegistry = managedresources.NewRegistry(operatorclient.VirtualScheme, operatorclient.VirtualCodec, operatorclient.VirtualSerializer) + ) + + virtualResources, err := virtualRegistry.AddAllAndSerialize( + g.clusterRole(), + g.clusterRoleBinding(virtualGardenAccessSecret.ServiceAccountName), + ) + if err != nil { + return err + } + + return managedresources.CreateForShoot(ctx, g.client, g.namespace, managedResourceNameVirtual, managedresources.LabelValueGardener, false, virtualResources) +} + +func (g *gardenerControllerManager) Wait(ctx context.Context) error { + timeoutCtx, cancel := context.WithTimeout(ctx, TimeoutWaitForManagedResource) + defer cancel() + + return flow.Parallel( + func(ctx context.Context) error { + return managedresources.WaitUntilHealthy(ctx, g.client, g.namespace, managedResourceNameRuntime) + }, + func(ctx context.Context) error { + return managedresources.WaitUntilHealthy(ctx, g.client, g.namespace, managedResourceNameVirtual) + }, + )(timeoutCtx) +} + +func (g *gardenerControllerManager) Destroy(ctx context.Context) error { + if err := managedresources.DeleteForShoot(ctx, g.client, g.namespace, managedResourceNameVirtual); err != nil { + return err + } + + if err := managedresources.DeleteForSeed(ctx, g.client, g.namespace, managedResourceNameRuntime); err != nil { + return err + } + + return kubernetesutils.DeleteObjects(ctx, g.client, g.newVirtualGardenAccessSecret().Secret) +} + +func (g *gardenerControllerManager) WaitCleanup(ctx context.Context) error { + timeoutCtx, cancel := context.WithTimeout(ctx, TimeoutWaitForManagedResource) + defer cancel() + + return flow.Parallel( + func(ctx context.Context) error { + return managedresources.WaitUntilDeleted(ctx, g.client, g.namespace, managedResourceNameRuntime) + }, + func(ctx context.Context) error { + return managedresources.WaitUntilDeleted(ctx, g.client, g.namespace, managedResourceNameVirtual) + }, + )(timeoutCtx) +} + +// GetLabels returns the labels for the gardener-controller-manager. +func GetLabels() map[string]string { + return map[string]string{ + v1beta1constants.LabelApp: v1beta1constants.LabelGardener, + v1beta1constants.LabelRole: roleName, + } +} diff --git a/pkg/component/gardenercontrollermanager/gardener_controller_manager_suite_test.go b/pkg/component/gardenercontrollermanager/gardener_controller_manager_suite_test.go new file mode 100644 index 00000000000..d62054925e1 --- /dev/null +++ b/pkg/component/gardenercontrollermanager/gardener_controller_manager_suite_test.go @@ -0,0 +1,27 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gardenercontrollermanager_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestGardenerControllerManager(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Component GardenerControllerManager Suite") +} diff --git a/pkg/component/gardenercontrollermanager/gardener_controller_manager_test.go b/pkg/component/gardenercontrollermanager/gardener_controller_manager_test.go new file mode 100644 index 00000000000..f2ed4cf5aeb --- /dev/null +++ b/pkg/component/gardenercontrollermanager/gardener_controller_manager_test.go @@ -0,0 +1,879 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gardenercontrollermanager_test + +import ( + "context" + "encoding/json" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + autoscalingv1 "k8s.io/api/autoscaling/v1" + corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + vpaautoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + componentbaseconfigv1alpha1 "k8s.io/component-base/config/v1alpha1" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/yaml" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" + "github.com/gardener/gardener/pkg/component" + . "github.com/gardener/gardener/pkg/component/gardenercontrollermanager" + componenttest "github.com/gardener/gardener/pkg/component/test" + controllermanagerv1alpha1 "github.com/gardener/gardener/pkg/controllermanager/apis/config/v1alpha1" + "github.com/gardener/gardener/pkg/logger" + operatorclient "github.com/gardener/gardener/pkg/operator/client" + "github.com/gardener/gardener/pkg/resourcemanager/controller/garbagecollector/references" + "github.com/gardener/gardener/pkg/utils" + gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" + kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" + "github.com/gardener/gardener/pkg/utils/retry" + retryfake "github.com/gardener/gardener/pkg/utils/retry/fake" + secretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager" + fakesecretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager/fake" + "github.com/gardener/gardener/pkg/utils/test" + . "github.com/gardener/gardener/pkg/utils/test/matchers" +) + +var _ = Describe("GardenerControllerManager", func() { + var ( + ctx context.Context + + managedResourceNameRuntime = "gardener-controller-manager-runtime" + managedResourceNameVirtual = "gardener-controller-manager-virtual" + namespace = "some-namespace" + + fakeClient client.Client + fakeSecretManager secretsmanager.Interface + deployer component.DeployWaiter + values Values + + fakeOps *retryfake.Ops + + managedResourceRuntime *resourcesv1alpha1.ManagedResource + managedResourceVirtual *resourcesv1alpha1.ManagedResource + managedResourceSecretRuntime *corev1.Secret + managedResourceSecretVirtual *corev1.Secret + + podDisruptionBudget *policyv1.PodDisruptionBudget + serviceRuntime *corev1.Service + vpa *vpaautoscalingv1.VerticalPodAutoscaler + + clusterRole *rbacv1.ClusterRole + clusterRoleBinding *rbacv1.ClusterRoleBinding + ) + + BeforeEach(func() { + ctx = context.TODO() + + fakeClient = fakeclient.NewClientBuilder().WithScheme(operatorclient.RuntimeScheme).Build() + fakeSecretManager = fakesecretsmanager.New(fakeClient, namespace) + values = Values{} + + fakeOps = &retryfake.Ops{MaxAttempts: 2} + DeferCleanup(test.WithVars( + &retry.Until, fakeOps.Until, + &retry.UntilTimeout, fakeOps.UntilTimeout, + )) + + managedResourceRuntime = &resourcesv1alpha1.ManagedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: managedResourceNameRuntime, + Namespace: namespace, + }, + } + managedResourceVirtual = &resourcesv1alpha1.ManagedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: managedResourceNameVirtual, + Namespace: namespace, + }, + } + managedResourceSecretRuntime = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "managedresource-" + managedResourceRuntime.Name, + Namespace: namespace, + }, + } + managedResourceSecretVirtual = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "managedresource-" + managedResourceVirtual.Name, + Namespace: namespace, + }, + } + podDisruptionBudget = &policyv1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gardener-controller-manager", + Namespace: namespace, + Labels: map[string]string{ + "app": "gardener", + "role": "controller-manager", + }, + }, + Spec: policyv1.PodDisruptionBudgetSpec{ + MaxUnavailable: utils.IntStrPtrFromInt(1), + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{ + "app": "gardener", + "role": "controller-manager", + }}, + }, + } + serviceRuntime = &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gardener-controller-manager", + Namespace: namespace, + Labels: map[string]string{ + "app": "gardener", + "role": "controller-manager", + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Selector: map[string]string{ + "app": "gardener", + "role": "controller-manager", + }, + Ports: []corev1.ServicePort{{ + Name: "metrics", + Port: 2719, + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromInt(2719), + }}, + }, + } + vpaUpdateMode := vpaautoscalingv1.UpdateModeAuto + vpa = &vpaautoscalingv1.VerticalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gardener-controller-manager-vpa", + Namespace: namespace, + Labels: map[string]string{ + "app": "gardener", + "role": "controller-manager", + }, + }, + Spec: vpaautoscalingv1.VerticalPodAutoscalerSpec{ + TargetRef: &autoscalingv1.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "gardener-controller-manager", + }, + UpdatePolicy: &vpaautoscalingv1.PodUpdatePolicy{ + UpdateMode: &vpaUpdateMode, + }, + ResourcePolicy: &vpaautoscalingv1.PodResourcePolicy{ + ContainerPolicies: []vpaautoscalingv1.ContainerResourcePolicy{ + { + ContainerName: "*", + MinAllowed: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("300Mi"), + }, + }, + }, + }, + }, + } + clusterRole = &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gardener.cloud:system:controller-manager", + Labels: map[string]string{ + "app": "gardener", + "role": "controller-manager", + }, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"*"}, + Resources: []string{"*"}, + Verbs: []string{"*"}, + }, + }, + } + clusterRoleBinding = &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gardener.cloud:system:controller-manager", + Labels: map[string]string{ + "app": "gardener", + "role": "controller-manager", + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "ClusterRole", + Name: "gardener.cloud:system:controller-manager", + }, + Subjects: []rbacv1.Subject{{ + Kind: "ServiceAccount", + Name: "gardener-controller-manager", + Namespace: "kube-system", + }}, + } + + Expect(fakeClient.Create(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "generic-token-kubeconfig", Namespace: namespace}})).To(Succeed()) + }) + + JustBeforeEach(func() { + deployer = New(fakeClient, namespace, fakeSecretManager, values) + }) + + Describe("#Deploy", func() { + Context("resources generation", func() { + BeforeEach(func() { + // test with typical values + values = Values{ + LogLevel: "info", + } + + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResourceRuntime), managedResourceRuntime)).To(BeNotFoundError()) + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResourceVirtual), managedResourceVirtual)).To(BeNotFoundError()) + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResourceSecretRuntime), managedResourceSecretRuntime)).To(BeNotFoundError()) + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResourceSecretVirtual), managedResourceSecretVirtual)).To(BeNotFoundError()) + + Expect(fakeClient.Create(ctx, &resourcesv1alpha1.ManagedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: managedResourceNameRuntime, + Namespace: namespace, + Generation: 1, + }, + Status: healthyManagedResourceStatus, + })).To(Succeed()) + + Expect(fakeClient.Create(ctx, &resourcesv1alpha1.ManagedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: managedResourceNameVirtual, + Namespace: namespace, + Generation: 1, + }, + Status: healthyManagedResourceStatus, + })).To(Succeed()) + }) + + It("should successfully deploy all resources", func() { + Expect(deployer.Deploy(ctx)).To(Succeed()) + + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResourceRuntime), managedResourceRuntime)).To(Succeed()) + expectedRuntimeMr := &resourcesv1alpha1.ManagedResource{ + TypeMeta: metav1.TypeMeta{ + APIVersion: resourcesv1alpha1.SchemeGroupVersion.String(), + Kind: "ManagedResource", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: managedResourceRuntime.Name, + Namespace: managedResourceRuntime.Namespace, + ResourceVersion: "2", + Generation: 1, + Labels: map[string]string{"gardener.cloud/role": "seed-system-component"}, + }, + Spec: resourcesv1alpha1.ManagedResourceSpec{ + Class: pointer.String("seed"), + SecretRefs: []corev1.LocalObjectReference{{Name: managedResourceRuntime.Spec.SecretRefs[0].Name}}, + KeepObjects: pointer.Bool(false), + }, + Status: healthyManagedResourceStatus, + } + utilruntime.Must(references.InjectAnnotations(expectedRuntimeMr)) + Expect(managedResourceRuntime).To(Equal(expectedRuntimeMr)) + + managedResourceSecretRuntime.Name = managedResourceRuntime.Spec.SecretRefs[0].Name + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResourceSecretRuntime), managedResourceSecretRuntime)).To(Succeed()) + + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResourceVirtual), managedResourceVirtual)).To(Succeed()) + expectedVirtualMr := &resourcesv1alpha1.ManagedResource{ + TypeMeta: metav1.TypeMeta{ + APIVersion: resourcesv1alpha1.SchemeGroupVersion.String(), + Kind: "ManagedResource", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: managedResourceVirtual.Name, + Namespace: managedResourceVirtual.Namespace, + ResourceVersion: "2", + Generation: 1, + Labels: map[string]string{"origin": "gardener"}, + }, + Spec: resourcesv1alpha1.ManagedResourceSpec{ + InjectLabels: map[string]string{"shoot.gardener.cloud/no-cleanup": "true"}, + SecretRefs: []corev1.LocalObjectReference{{Name: managedResourceVirtual.Spec.SecretRefs[0].Name}}, + KeepObjects: pointer.Bool(false), + }, + Status: healthyManagedResourceStatus, + } + utilruntime.Must(references.InjectAnnotations(expectedVirtualMr)) + Expect(managedResourceVirtual).To(Equal(expectedVirtualMr)) + + managedResourceSecretVirtual.Name = expectedVirtualMr.Spec.SecretRefs[0].Name + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResourceSecretVirtual), managedResourceSecretVirtual)).To(Succeed()) + + Expect(managedResourceSecretRuntime.Type).To(Equal(corev1.SecretTypeOpaque)) + Expect(managedResourceSecretRuntime.Data).To(HaveLen(5)) + Expect(string(managedResourceSecretRuntime.Data["configmap__some-namespace__gardener-controller-manager-config-cff08f20.yaml"])).To(Equal(configMap(namespace, values))) + Expect(string(managedResourceSecretRuntime.Data["poddisruptionbudget__some-namespace__gardener-controller-manager.yaml"])).To(Equal(componenttest.Serialize(podDisruptionBudget))) + Expect(string(managedResourceSecretRuntime.Data["service__some-namespace__gardener-controller-manager.yaml"])).To(Equal(componenttest.Serialize(serviceRuntime))) + Expect(string(managedResourceSecretRuntime.Data["verticalpodautoscaler__some-namespace__gardener-controller-manager-vpa.yaml"])).To(Equal(componenttest.Serialize(vpa))) + Expect(string(managedResourceSecretRuntime.Data["deployment__some-namespace__gardener-controller-manager.yaml"])).To(Equal(deployment(namespace, "gardener-controller-manager-config-cff08f20", values))) + Expect(managedResourceSecretRuntime.Immutable).To(Equal(pointer.Bool(true))) + Expect(managedResourceSecretRuntime.Labels["resources.gardener.cloud/garbage-collectable-reference"]).To(Equal("true")) + + Expect(managedResourceSecretVirtual.Type).To(Equal(corev1.SecretTypeOpaque)) + Expect(managedResourceSecretVirtual.Data).To(HaveLen(2)) + Expect(string(managedResourceSecretVirtual.Data["clusterrole____gardener.cloud_system_controller-manager.yaml"])).To(Equal(componenttest.Serialize(clusterRole))) + Expect(string(managedResourceSecretVirtual.Data["clusterrolebinding____gardener.cloud_system_controller-manager.yaml"])).To(Equal(componenttest.Serialize(clusterRoleBinding))) + Expect(managedResourceSecretVirtual.Immutable).To(Equal(pointer.Bool(true))) + Expect(managedResourceSecretVirtual.Labels["resources.gardener.cloud/garbage-collectable-reference"]).To(Equal("true")) + }) + }) + + Context("secrets", func() { + It("should successfully deploy the access secret for the virtual garden", func() { + accessSecret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "shoot-access-gardener-controller-manager", + Namespace: namespace, + Labels: map[string]string{ + "resources.gardener.cloud/purpose": "token-requestor", + "resources.gardener.cloud/class": "shoot", + }, + Annotations: map[string]string{ + "serviceaccount.resources.gardener.cloud/name": "gardener-controller-manager", + "serviceaccount.resources.gardener.cloud/namespace": "kube-system", + }, + }, + Type: corev1.SecretTypeOpaque, + } + + Expect(deployer.Deploy(ctx)).To(Succeed()) + + actualShootAccessSecret := &corev1.Secret{} + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(accessSecret), actualShootAccessSecret)).To(Succeed()) + accessSecret.ResourceVersion = "1" + Expect(actualShootAccessSecret).To(Equal(accessSecret)) + }) + }) + }) + + Describe("#Destroy", func() { + It("should successfully destroy all resources", func() { + Expect(fakeClient.Create(ctx, managedResourceRuntime)).To(Succeed()) + Expect(fakeClient.Create(ctx, managedResourceVirtual)).To(Succeed()) + Expect(fakeClient.Create(ctx, managedResourceSecretRuntime)).To(Succeed()) + Expect(fakeClient.Create(ctx, managedResourceSecretVirtual)).To(Succeed()) + + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResourceRuntime), managedResourceRuntime)).To(Succeed()) + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResourceVirtual), managedResourceVirtual)).To(Succeed()) + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResourceSecretRuntime), managedResourceSecretRuntime)).To(Succeed()) + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResourceSecretVirtual), managedResourceSecretVirtual)).To(Succeed()) + + Expect(deployer.Destroy(ctx)).To(Succeed()) + + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResourceRuntime), managedResourceRuntime)).To(BeNotFoundError()) + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResourceVirtual), managedResourceVirtual)).To(BeNotFoundError()) + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResourceSecretRuntime), managedResourceSecretRuntime)).To(BeNotFoundError()) + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResourceSecretVirtual), managedResourceSecretVirtual)).To(BeNotFoundError()) + }) + }) + + Context("waiting functions", func() { + Describe("#Wait", func() { + It("should fail because reading the runtime ManagedResource fails", func() { + Expect(deployer.Wait(ctx)).To(MatchError(ContainSubstring("not found"))) + }) + + It("should fail because the runtime and virtual ManagedResources are unhealthy", func() { + Expect(fakeClient.Create(ctx, &resourcesv1alpha1.ManagedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: managedResourceNameRuntime, + Namespace: namespace, + Generation: 1, + }, + Status: unhealthyManagedResourceStatus, + })).To(Succeed()) + + Expect(fakeClient.Create(ctx, &resourcesv1alpha1.ManagedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: managedResourceNameVirtual, + Namespace: namespace, + Generation: 1, + }, + Status: unhealthyManagedResourceStatus, + })).To(Succeed()) + + Expect(deployer.Wait(ctx)).To(MatchError(ContainSubstring("is not healthy"))) + }) + + It("should fail because the runtime ManagedResource is unhealthy", func() { + Expect(fakeClient.Create(ctx, &resourcesv1alpha1.ManagedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: managedResourceNameRuntime, + Namespace: namespace, + Generation: 1, + }, + Status: unhealthyManagedResourceStatus, + })).To(Succeed()) + + Expect(fakeClient.Create(ctx, &resourcesv1alpha1.ManagedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: managedResourceNameVirtual, + Namespace: namespace, + Generation: 1, + }, + Status: resourcesv1alpha1.ManagedResourceStatus{ + ObservedGeneration: 1, + Conditions: []gardencorev1beta1.Condition{ + { + Type: resourcesv1alpha1.ResourcesApplied, + Status: gardencorev1beta1.ConditionTrue, + }, + { + Type: resourcesv1alpha1.ResourcesHealthy, + Status: gardencorev1beta1.ConditionTrue, + }, + { + Type: resourcesv1alpha1.ResourcesProgressing, + Status: gardencorev1beta1.ConditionFalse, + }, + }, + }, + })).To(Succeed()) + + Expect(deployer.Wait(ctx)).To(MatchError(ContainSubstring("is not healthy"))) + }) + + It("should fail because the virtual ManagedResource is unhealthy", func() { + Expect(fakeClient.Create(ctx, &resourcesv1alpha1.ManagedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: managedResourceNameRuntime, + Namespace: namespace, + Generation: 1, + }, + Status: resourcesv1alpha1.ManagedResourceStatus{ + ObservedGeneration: 1, + Conditions: []gardencorev1beta1.Condition{ + { + Type: resourcesv1alpha1.ResourcesApplied, + Status: gardencorev1beta1.ConditionTrue, + }, + { + Type: resourcesv1alpha1.ResourcesHealthy, + Status: gardencorev1beta1.ConditionTrue, + }, + { + Type: resourcesv1alpha1.ResourcesProgressing, + Status: gardencorev1beta1.ConditionFalse, + }, + }, + }, + })).To(Succeed()) + + Expect(fakeClient.Create(ctx, &resourcesv1alpha1.ManagedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: managedResourceNameVirtual, + Namespace: namespace, + Generation: 1, + }, + Status: unhealthyManagedResourceStatus, + })).To(Succeed()) + + Expect(deployer.Wait(ctx)).To(MatchError(ContainSubstring("is not healthy"))) + }) + + It("should succeed because the runtime and virtual ManagedResource are healthy and progressing", func() { + Expect(fakeClient.Create(ctx, &resourcesv1alpha1.ManagedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: managedResourceNameRuntime, + Namespace: namespace, + Generation: 1, + }, + Status: resourcesv1alpha1.ManagedResourceStatus{ + ObservedGeneration: 1, + Conditions: []gardencorev1beta1.Condition{ + { + Type: resourcesv1alpha1.ResourcesApplied, + Status: gardencorev1beta1.ConditionTrue, + }, + { + Type: resourcesv1alpha1.ResourcesHealthy, + Status: gardencorev1beta1.ConditionTrue, + }, + { + Type: resourcesv1alpha1.ResourcesProgressing, + Status: gardencorev1beta1.ConditionTrue, + }, + }, + }, + })).To(Succeed()) + + Expect(fakeClient.Create(ctx, &resourcesv1alpha1.ManagedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: managedResourceNameVirtual, + Namespace: namespace, + Generation: 1, + }, + Status: resourcesv1alpha1.ManagedResourceStatus{ + ObservedGeneration: 1, + Conditions: []gardencorev1beta1.Condition{ + { + Type: resourcesv1alpha1.ResourcesApplied, + Status: gardencorev1beta1.ConditionTrue, + }, + { + Type: resourcesv1alpha1.ResourcesHealthy, + Status: gardencorev1beta1.ConditionTrue, + }, + { + Type: resourcesv1alpha1.ResourcesProgressing, + Status: gardencorev1beta1.ConditionTrue, + }, + }, + }, + })).To(Succeed()) + + Expect(deployer.Wait(ctx)).To(Succeed()) + }) + + It("should succeed because the both ManagedResource are healthy and progressed", func() { + Expect(fakeClient.Create(ctx, &resourcesv1alpha1.ManagedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: managedResourceNameRuntime, + Namespace: namespace, + Generation: 1, + }, + Status: resourcesv1alpha1.ManagedResourceStatus{ + ObservedGeneration: 1, + Conditions: []gardencorev1beta1.Condition{ + { + Type: resourcesv1alpha1.ResourcesApplied, + Status: gardencorev1beta1.ConditionTrue, + }, + { + Type: resourcesv1alpha1.ResourcesHealthy, + Status: gardencorev1beta1.ConditionTrue, + }, + { + Type: resourcesv1alpha1.ResourcesProgressing, + Status: gardencorev1beta1.ConditionFalse, + }, + }, + }, + })).To(Succeed()) + + Expect(fakeClient.Create(ctx, &resourcesv1alpha1.ManagedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: managedResourceNameVirtual, + Namespace: namespace, + Generation: 1, + }, + Status: resourcesv1alpha1.ManagedResourceStatus{ + ObservedGeneration: 1, + Conditions: []gardencorev1beta1.Condition{ + { + Type: resourcesv1alpha1.ResourcesApplied, + Status: gardencorev1beta1.ConditionTrue, + }, + { + Type: resourcesv1alpha1.ResourcesHealthy, + Status: gardencorev1beta1.ConditionTrue, + }, + { + Type: resourcesv1alpha1.ResourcesProgressing, + Status: gardencorev1beta1.ConditionFalse, + }, + }, + }, + })).To(Succeed()) + + Expect(deployer.Wait(ctx)).To(Succeed()) + }) + }) + + Describe("#WaitCleanup", func() { + It("should fail when the wait for the runtime managed resource deletion times out", func() { + Expect(fakeClient.Create(ctx, managedResourceRuntime)).To(Succeed()) + + Expect(deployer.WaitCleanup(ctx)).To(MatchError(ContainSubstring("still exists"))) + }) + + It("should fail when the wait for the virtual managed resource deletion times out", func() { + Expect(fakeClient.Create(ctx, managedResourceVirtual)).To(Succeed()) + + Expect(deployer.WaitCleanup(ctx)).To(MatchError(ContainSubstring("still exists"))) + }) + + It("should not return an error when they are already removed", func() { + Expect(deployer.WaitCleanup(ctx)).To(Succeed()) + }) + }) + }) +}) + +var ( + healthyManagedResourceStatus = resourcesv1alpha1.ManagedResourceStatus{ + ObservedGeneration: 1, + Conditions: []gardencorev1beta1.Condition{ + { + Type: resourcesv1alpha1.ResourcesApplied, + Status: gardencorev1beta1.ConditionTrue, + }, + { + Type: resourcesv1alpha1.ResourcesHealthy, + Status: gardencorev1beta1.ConditionTrue, + }, + }, + } + unhealthyManagedResourceStatus = resourcesv1alpha1.ManagedResourceStatus{ + ObservedGeneration: 1, + Conditions: []gardencorev1beta1.Condition{ + { + Type: resourcesv1alpha1.ResourcesApplied, + Status: gardencorev1beta1.ConditionFalse, + }, + { + Type: resourcesv1alpha1.ResourcesHealthy, + Status: gardencorev1beta1.ConditionFalse, + }, + }, + } +) + +func configMap(namespace string, testValues Values) string { + controllerManagerConfig := &controllermanagerv1alpha1.ControllerManagerConfiguration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "controllermanager.config.gardener.cloud/v1alpha1", + Kind: "ControllerManagerConfiguration", + }, + GardenClientConnection: componentbaseconfigv1alpha1.ClientConnectionConfiguration{ + QPS: 100, + Burst: 130, + Kubeconfig: gardenerutils.PathGenericKubeconfig, + }, + Controllers: controllermanagerv1alpha1.ControllerManagerControllerConfiguration{ + ControllerRegistration: &controllermanagerv1alpha1.ControllerRegistrationControllerConfiguration{ + ConcurrentSyncs: pointer.Int(20), + }, + Project: &controllermanagerv1alpha1.ProjectControllerConfiguration{ + ConcurrentSyncs: pointer.Int(20), + Quotas: testValues.Quotas, + }, + SecretBinding: &controllermanagerv1alpha1.SecretBindingControllerConfiguration{ + ConcurrentSyncs: pointer.Int(20), + }, + Seed: &controllermanagerv1alpha1.SeedControllerConfiguration{ + ConcurrentSyncs: pointer.Int(20), + ShootMonitorPeriod: &metav1.Duration{Duration: 300 * time.Second}, + }, + SeedExtensionsCheck: &controllermanagerv1alpha1.SeedExtensionsCheckControllerConfiguration{ + ConditionThresholds: []controllermanagerv1alpha1.ConditionThreshold{{ + Duration: metav1.Duration{Duration: 1 * time.Minute}, + Type: "ExtensionsReady", + }}, + }, + SeedBackupBucketsCheck: &controllermanagerv1alpha1.SeedBackupBucketsCheckControllerConfiguration{ + ConditionThresholds: []controllermanagerv1alpha1.ConditionThreshold{{ + Duration: metav1.Duration{Duration: 1 * time.Minute}, + Type: "BackupBucketsReady", + }}, + }, + Event: &controllermanagerv1alpha1.EventControllerConfiguration{ + ConcurrentSyncs: pointer.Int(10), + TTLNonShootEvents: &metav1.Duration{Duration: 2 * time.Hour}, + }, + ShootMaintenance: controllermanagerv1alpha1.ShootMaintenanceControllerConfiguration{ + ConcurrentSyncs: pointer.Int(20), + }, + ShootReference: &controllermanagerv1alpha1.ShootReferenceControllerConfiguration{ + ConcurrentSyncs: pointer.Int(20), + }, + }, + LeaderElection: &componentbaseconfigv1alpha1.LeaderElectionConfiguration{ + LeaderElect: pointer.Bool(true), + ResourceName: controllermanagerv1alpha1.ControllerManagerDefaultLockObjectName, + ResourceNamespace: metav1.NamespaceSystem, + }, + LogLevel: testValues.LogLevel, + LogFormat: logger.FormatJSON, + Server: controllermanagerv1alpha1.ServerConfiguration{ + HealthProbes: &controllermanagerv1alpha1.Server{Port: 2718}, + Metrics: &controllermanagerv1alpha1.Server{Port: 2719}, + }, + FeatureGates: testValues.FeatureGates, + } + + data, err := json.Marshal(controllerManagerConfig) + utilruntime.Must(err) + data, err = yaml.JSONToYAML(data) + utilruntime.Must(err) + + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{}, + Labels: map[string]string{ + "app": "gardener", + "role": "controller-manager", + }, + Name: "gardener-controller-manager-config", + Namespace: namespace, + }, + Data: map[string]string{ + "config.yaml": string(data), + }, + } + utilruntime.Must(kubernetesutils.MakeUnique(configMap)) + + return componenttest.Serialize(configMap) +} + +func deployment(namespace, configSecretName string, testValues Values) string { + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gardener-controller-manager", + Namespace: namespace, + Labels: map[string]string{ + "app": "gardener", + "role": "controller-manager", + "high-availability-config.resources.gardener.cloud/type": "controller", + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: pointer.Int32(1), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "gardener", + "role": "controller-manager", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: utils.MergeStringMaps(GetLabels(), map[string]string{ + "app": "gardener", + "role": "controller-manager", + "networking.gardener.cloud/to-dns": "allowed", + "networking.resources.gardener.cloud/to-virtual-garden-kube-apiserver-tcp-443": "allowed", + }), + }, + Spec: corev1.PodSpec{ + PriorityClassName: "gardener-garden-system-200", + AutomountServiceAccountToken: pointer.Bool(false), + Containers: []corev1.Container{ + { + Name: "gardener-controller-manager", + Image: testValues.Image, + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{ + "--config=/etc/gardener-controller-manager/config/config.yaml", + }, + Resources: corev1.ResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(2718), + Scheme: corev1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 30, + TimeoutSeconds: 5, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/readyz", + Port: intstr.FromInt(2718), + Scheme: corev1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 10, + TimeoutSeconds: 5, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "gardener-controller-manager-config", + MountPath: "/etc/gardener-controller-manager/config", + }, + { + Name: "kubeconfig", + MountPath: "/var/run/secrets/gardener.cloud/shoot/generic-kubeconfig", + ReadOnly: true, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "gardener-controller-manager-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: configSecretName}, + }, + }, + }, + { + Name: "kubeconfig", + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + DefaultMode: pointer.Int32(420), + Sources: []corev1.VolumeProjection{ + { + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "generic-token-kubeconfig", + }, + Items: []corev1.KeyToPath{{ + Key: "kubeconfig", + Path: "kubeconfig", + }}, + Optional: pointer.Bool(false), + }, + }, + { + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "shoot-access-gardener-controller-manager", + }, + Items: []corev1.KeyToPath{{ + Key: "token", + Path: "token", + }}, + Optional: pointer.Bool(false), + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + utilruntime.Must(references.InjectAnnotations(deployment)) + + return componenttest.Serialize(deployment) +} diff --git a/pkg/component/gardenercontrollermanager/poddisruptionbudget.go b/pkg/component/gardenercontrollermanager/poddisruptionbudget.go new file mode 100644 index 00000000000..24e70c43055 --- /dev/null +++ b/pkg/component/gardenercontrollermanager/poddisruptionbudget.go @@ -0,0 +1,36 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gardenercontrollermanager + +import ( + policyv1 "k8s.io/api/policy/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + gardenerutils "github.com/gardener/gardener/pkg/utils" +) + +func (g *gardenerControllerManager) podDisruptionBudget() *policyv1.PodDisruptionBudget { + return &policyv1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: DeploymentName, + Namespace: g.namespace, + Labels: GetLabels(), + }, + Spec: policyv1.PodDisruptionBudgetSpec{ + MaxUnavailable: gardenerutils.IntStrPtrFromInt(1), + Selector: &metav1.LabelSelector{MatchLabels: GetLabels()}, + }, + } +} diff --git a/pkg/component/gardenercontrollermanager/rbac.go b/pkg/component/gardenercontrollermanager/rbac.go new file mode 100644 index 00000000000..60b89f775bc --- /dev/null +++ b/pkg/component/gardenercontrollermanager/rbac.go @@ -0,0 +1,60 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gardenercontrollermanager + +import ( + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + clusterRoleName = "gardener.cloud:system:controller-manager" + clusterRoleBindingName = "gardener.cloud:system:controller-manager" +) + +func (g *gardenerControllerManager) clusterRole() *rbacv1.ClusterRole { + return &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterRoleName, + Labels: GetLabels(), + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"*"}, + Resources: []string{"*"}, + Verbs: []string{"*"}, + }, + }, + } +} + +func (g *gardenerControllerManager) clusterRoleBinding(serviceAccountName string) *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterRoleBindingName, + Labels: GetLabels(), + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "ClusterRole", + Name: clusterRoleName, + }, + Subjects: []rbacv1.Subject{{ + Kind: "ServiceAccount", + Name: serviceAccountName, + Namespace: metav1.NamespaceSystem, + }}, + } +} diff --git a/pkg/component/gardenercontrollermanager/secrets.go b/pkg/component/gardenercontrollermanager/secrets.go new file mode 100644 index 00000000000..8d57a6887cb --- /dev/null +++ b/pkg/component/gardenercontrollermanager/secrets.go @@ -0,0 +1,23 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gardenercontrollermanager + +import ( + gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" +) + +func (g *gardenerControllerManager) newVirtualGardenAccessSecret() *gardenerutils.AccessSecret { + return gardenerutils.NewShootAccessSecret(DeploymentName, g.namespace) +} diff --git a/pkg/component/gardenercontrollermanager/service.go b/pkg/component/gardenercontrollermanager/service.go new file mode 100644 index 00000000000..18bcdc0e7a3 --- /dev/null +++ b/pkg/component/gardenercontrollermanager/service.go @@ -0,0 +1,45 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gardenercontrollermanager + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const serviceName = DeploymentName + +func (g *gardenerControllerManager) service() *corev1.Service { + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName, + Namespace: g.namespace, + Labels: GetLabels(), + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Selector: GetLabels(), + Ports: []corev1.ServicePort{{ + Name: "metrics", + Port: int32(metricsPort), + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromInt(metricsPort), + }}, + }, + } + + return service +} diff --git a/pkg/component/gardenercontrollermanager/vpa.go b/pkg/component/gardenercontrollermanager/vpa.go new file mode 100644 index 00000000000..24e572f1cc4 --- /dev/null +++ b/pkg/component/gardenercontrollermanager/vpa.go @@ -0,0 +1,55 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gardenercontrollermanager + +import ( + appsv1 "k8s.io/api/apps/v1" + autoscalingv1 "k8s.io/api/autoscaling/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + vpaautoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" +) + +func (g *gardenerControllerManager) verticalPodAutoscaler() *vpaautoscalingv1.VerticalPodAutoscaler { + vpaUpdateMode := vpaautoscalingv1.UpdateModeAuto + return &vpaautoscalingv1.VerticalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: DeploymentName + "-vpa", + Namespace: g.namespace, + Labels: GetLabels(), + }, + Spec: vpaautoscalingv1.VerticalPodAutoscalerSpec{ + TargetRef: &autoscalingv1.CrossVersionObjectReference{ + APIVersion: appsv1.SchemeGroupVersion.String(), + Kind: "Deployment", + Name: DeploymentName, + }, + UpdatePolicy: &vpaautoscalingv1.PodUpdatePolicy{ + UpdateMode: &vpaUpdateMode, + }, + ResourcePolicy: &vpaautoscalingv1.PodResourcePolicy{ + ContainerPolicies: []vpaautoscalingv1.ContainerResourcePolicy{ + { + ContainerName: vpaautoscalingv1.DefaultContainerResourcePolicy, + MinAllowed: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("300Mi"), + }, + }, + }, + }, + }, + } +} diff --git a/pkg/controllermanager/apis/.import-restrictions b/pkg/controllermanager/apis/.import-restrictions index 4d933fb5798..2ca3755f39c 100644 --- a/pkg/controllermanager/apis/.import-restrictions +++ b/pkg/controllermanager/apis/.import-restrictions @@ -5,6 +5,7 @@ rules: - k8s.io/api - k8s.io/component-base/config - k8s.io/klog + - k8s.io/utils/pointer - selectorRegexp: github[.]com/gardener allowedPrefixes: - github.com/gardener/gardener/pkg/controllermanager/apis diff --git a/pkg/controllermanager/apis/config/v1alpha1/defaults.go b/pkg/controllermanager/apis/config/v1alpha1/defaults.go index 2063fd7fa82..69b13767784 100644 --- a/pkg/controllermanager/apis/config/v1alpha1/defaults.go +++ b/pkg/controllermanager/apis/config/v1alpha1/defaults.go @@ -20,6 +20,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" componentbaseconfigv1alpha1 "k8s.io/component-base/config/v1alpha1" + "k8s.io/utils/pointer" ) func addDefaultingFuncs(scheme *runtime.Scheme) error { @@ -137,6 +138,9 @@ func SetDefaults_ControllerManagerConfiguration(obj *ControllerManagerConfigurat v := DefaultControllerConcurrentSyncs obj.Controllers.ShootMaintenance.ConcurrentSyncs = &v } + if obj.Controllers.ShootMaintenance.EnableShootControlPlaneRestarter == nil { + obj.Controllers.ShootMaintenance.EnableShootControlPlaneRestarter = pointer.Bool(true) + } if obj.Controllers.ShootQuota == nil { obj.Controllers.ShootQuota = &ShootQuotaControllerConfiguration{} diff --git a/pkg/controllermanager/apis/config/v1alpha1/defaults_test.go b/pkg/controllermanager/apis/config/v1alpha1/defaults_test.go index 3e558298d05..12c47c15b2d 100644 --- a/pkg/controllermanager/apis/config/v1alpha1/defaults_test.go +++ b/pkg/controllermanager/apis/config/v1alpha1/defaults_test.go @@ -85,6 +85,8 @@ var _ = Describe("Defaults", func() { Expect(obj.Controllers.ShootMaintenance.ConcurrentSyncs).NotTo(BeNil()) Expect(obj.Controllers.ShootMaintenance.ConcurrentSyncs).To(PointTo(Equal(5))) + Expect(obj.Controllers.ShootMaintenance.EnableShootControlPlaneRestarter).NotTo(BeNil()) + Expect(obj.Controllers.ShootMaintenance.EnableShootControlPlaneRestarter).To(PointTo(Equal(true))) Expect(obj.Controllers.ShootQuota).NotTo(BeNil()) Expect(obj.Controllers.ShootQuota.ConcurrentSyncs).NotTo(BeNil())