From a57427e0d8c80c51ed021db01fdaf35d1e15e31d Mon Sep 17 00:00:00 2001 From: Rafael Franzke Date: Tue, 16 May 2023 16:35:25 +0200 Subject: [PATCH] [operator] Manage Virtual Garden `kube-controller-manager` Deployment (#7931) * Extend `operatorv1alpha1.VirtualCluster` with API for kube-controller-manager config * Defaulting * [make generate] Without the `allowDangerousTypes=true` option, `controller-gen` yields the following error: ``` /go/src/github.com/gardener/gardener/pkg/apis/core/v1beta1/types_shoot.go:928:13: found float, the usage of which is highly discouraged, as support for them varies across languages. Please consider serializing your float as string instead. If you are really sure you want to use them, re-run with crd:allowDangerousTypes=true ``` * Adapt documentation * Make component instantiation reusable * Deploy/destroy virtual cluster `kubecontrollermanager` component * Implement `Destroy` of `kubecontrollermanager` component Otherwise, sane deletion of `Garden` will not work :) * Adapt integration/e2e tests * Adapt garden credentials rotation e2e test * Address PR review feedback --- .../templates/customresouredefintion.yaml | 72 ++++++++ charts/gardener/operator/templates/role.yaml | 1 + docs/api-reference/operator.md | 63 +++++++ docs/concepts/operator.md | 25 ++- docs/development/priority-classes.md | 4 +- ...0-crd-operator.gardener.cloud_gardens.yaml | 72 ++++++++ example/operator/20-garden.yaml | 4 + example/operator/doc.go | 2 +- hack/generate-crds.sh | 37 ++-- pkg/apis/core/v1beta1/defaults.go | 16 +- .../core/v1beta1/zz_generated.defaults.go | 3 + pkg/apis/core/validation/shoot.go | 5 +- pkg/apis/operator/v1alpha1/types.go | 17 ++ .../v1alpha1/validation/validation.go | 11 ++ .../v1alpha1/validation/validation_test.go | 102 +++++------ .../v1alpha1/zz_generated.deepcopy.go | 31 ++++ .../kube_controller_manager.go | 48 +++-- .../kube_controller_manager_test.go | 65 ++++--- .../component/shared/kubecontrollermanager.go | 81 +++++++++ .../botanist/kubecontrollermanager.go | 41 ++--- .../botanist/kubecontrollermanager_test.go | 5 +- pkg/operation/botanist/resource_manager.go | 1 - pkg/operator/controller/garden/components.go | 58 +++++- .../controller/garden/reconciler_delete.go | 11 +- .../controller/garden/reconciler_reconcile.go | 14 +- pkg/operator/webhook/defaulting/handler.go | 16 +- skaffold-operator.yaml | 1 + .../managedseed/create_rotate_delete.go | 2 +- .../gardener/shoot/create_rotate_delete.go | 2 +- .../gardener/shoot/create_update_delete.go | 2 +- test/e2e/gardener/shoot/internal/node/node.go | 2 +- .../shoot/internal/rotation/shoot_access.go | 18 +- test/e2e/operator/garden/create_delete.go | 1 + .../operator/garden/create_rotate_delete.go | 1 + .../rotation/virtual_garden_access.go | 172 ++++++++++++++++++ test/framework/k8s_utils.go | 2 +- test/framework/shootframework.go | 2 +- test/framework/shootmigrationtest.go | 2 +- .../operator/garden/garden_test.go | 42 +++++ .../shoots/vpntunnel/vpntunnel.go | 2 +- .../{shoots => }/access/adminkubeconfig.go | 0 test/utils/{shoots => }/access/csr.go | 26 +-- .../{shoots => }/access/serviceaccount.go | 32 ++-- .../access/statictokenkubeconfig.go | 0 test/utils/shoots/update/update.go | 2 +- 45 files changed, 913 insertions(+), 203 deletions(-) create mode 100644 pkg/operation/botanist/component/shared/kubecontrollermanager.go create mode 100644 test/e2e/operator/garden/internal/rotation/virtual_garden_access.go rename test/utils/{shoots => }/access/adminkubeconfig.go (100%) rename test/utils/{shoots => }/access/csr.go (76%) rename test/utils/{shoots => }/access/serviceaccount.go (79%) rename test/utils/{shoots => }/access/statictokenkubeconfig.go (100%) diff --git a/charts/gardener/operator/templates/customresouredefintion.yaml b/charts/gardener/operator/templates/customresouredefintion.yaml index 56a02de2144..58e35d03450 100644 --- a/charts/gardener/operator/templates/customresouredefintion.yaml +++ b/charts/gardener/operator/templates/customresouredefintion.yaml @@ -765,6 +765,78 @@ spec: type: array type: object type: object + kubeControllerManager: + description: KubeControllerManager contains configuration + settings for the kube-controller-manager. + properties: + certificateSigningDuration: + default: 48h + description: CertificateSigningDuration is the maximum + length of duration signed certificates will be given. + Individual CSRs may request shorter certs by setting + `spec.expirationSeconds`. + pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$ + type: string + featureGates: + additionalProperties: + type: boolean + description: FeatureGates contains information about enabled + feature gates. + type: object + horizontalPodAutoscaler: + description: HorizontalPodAutoscalerConfig contains horizontal + pod autoscaler configuration settings for the kube-controller-manager. + properties: + cpuInitializationPeriod: + description: The period after which a ready pod transition + is considered to be the first. + type: string + downscaleStabilization: + description: The configurable window at which the + controller will choose the highest recommendation + for autoscaling. + type: string + initialReadinessDelay: + description: The configurable period at which the + horizontal pod autoscaler considers a Pod “not yet + ready” given that it’s unready and it has transitioned + to unready during that time. + type: string + syncPeriod: + description: The period for syncing the number of + pods in horizontal pod autoscaler. + type: string + tolerance: + description: The minimum change (from 1.0) in the + desired-to-actual metrics ratio for the horizontal + pod autoscaler to consider scaling. + type: number + type: object + nodeCIDRMaskSize: + description: NodeCIDRMaskSize defines the mask size for + node cidr in cluster (default is 24). This field is + immutable. + format: int32 + type: integer + nodeMonitorGracePeriod: + description: NodeMonitorGracePeriod defines the grace + period before an unresponsive node is marked unhealthy. + type: string + podEvictionTimeout: + description: "PodEvictionTimeout defines the grace period + for deleting pods on failed nodes. Defaults to 2m. \n + Deprecated: The corresponding kube-controller-manager + flag `--pod-eviction-timeout` is deprecated in favor + of the kube-apiserver flags `--default-not-ready-toleration-seconds` + and `--default-unreachable-toleration-seconds`. The + `--pod-eviction-timeout` flag does not have effect when + the taint besed eviction is enabled. The taint based + eviction is beta (enabled by default) since Kubernetes + 1.13 and GA since Kubernetes 1.18. Hence, instead of + setting this field, set the `spec.kubernetes.kubeAPIServer.defaultNotReadyTolerationSeconds` + and `spec.kubernetes.kubeAPIServer.defaultUnreachableTolerationSeconds`." + type: string + type: object version: description: Version is the semantic Kubernetes version to use for the virtual garden cluster. diff --git a/charts/gardener/operator/templates/role.yaml b/charts/gardener/operator/templates/role.yaml index 3557b8ea461..921e9bc2d67 100644 --- a/charts/gardener/operator/templates/role.yaml +++ b/charts/gardener/operator/templates/role.yaml @@ -128,6 +128,7 @@ rules: - virtual-garden-etcd-events - virtual-garden-etcd-main - virtual-garden-kube-apiserver + - virtual-garden-kube-controller-manager verbs: - delete - patch diff --git a/docs/api-reference/operator.md b/docs/api-reference/operator.md index e3951721bac..2075e1202d5 100644 --- a/docs/api-reference/operator.md +++ b/docs/api-reference/operator.md @@ -943,6 +943,55 @@ SNI +

KubeControllerManagerConfig +

+

+(Appears on: +Kubernetes) +

+

+

KubeControllerManagerConfig contains configuration settings for the kube-controller-manager.

+

+ + + + + + + + + + + + + + + + + +
FieldDescription
+KubeControllerManagerConfig
+ +github.com/gardener/gardener/pkg/apis/core/v1beta1.KubeControllerManagerConfig + +		 +

+(Members of KubeControllerManagerConfig are embedded into this type.) +

+(Optional) +

KubeControllerManagerConfig contains all configuration values not specific to the virtual garden cluster.

+
+certificateSigningDuration
+ + +Kubernetes meta/v1.Duration + + +		 +(Optional) +

CertificateSigningDuration is the maximum length of duration signed certificates will be given. Individual CSRs +may request shorter certs by setting spec.expirationSeconds.

+

Kubernetes

@@ -977,6 +1026,20 @@ KubeAPIServerConfig +kubeControllerManager
+ + +KubeControllerManagerConfig + + + + +(Optional) +

KubeControllerManager contains configuration settings for the kube-controller-manager.

+ + + + version
string diff --git a/docs/concepts/operator.md b/docs/concepts/operator.md index 809f3361918..831d39a2a7f 100644 --- a/docs/concepts/operator.md +++ b/docs/concepts/operator.md @@ -89,11 +89,7 @@ Please refer to [this document](../usage/shoot_credentials_rotation.md#gardener- - ETCD encryption key - `ServiceAccount` token signing key -⚠️ Since `kube-controller-manager` is not yet deployed by `gardener-operator`, rotation of static `ServiceAccount` secrets is not supported and must be performed manually after the `Garden` has reached `Prepared` phase before completing the rotation. - -⚠️ Rotation of the static kubeconfig (which is enabled unconditionally) is not support for now. -The reason is that it such static kubeconfig will be disabled without configuration option in the near future. -Instead, we'll implement an approach similar to the [`adminkubeconfig` subresource on `Shoot`s](../usage/shoot_access.md#shootsadminkubeconfig-subresource) which can be used to retrieve a temporary kubeconfig for the virtual garden cluster. +⚠️ Rotation of static `ServiceAccount` secrets is not supported since the `kube-controller-manager` does not enable the `serviceaccount-token` controller. ## Local Development @@ -110,8 +106,7 @@ This command sets up a new KinD cluster named `gardener-local` and stores the ku > It might be helpful to copy this file to `$HOME/.kube/config`, since you will need to target this KinD cluster multiple times. Alternatively, make sure to set your `KUBECONFIG` environment variable to `./example/gardener-local/kind/operator/kubeconfig` for all future steps via `export KUBECONFIG=example/gardener-local/kind/operator/kubeconfig`. -All of the following steps assume that you are using this kubeconfig. - +All the following steps assume that you are using this kubeconfig. ### Setting Up Gardener Operator @@ -177,10 +172,12 @@ EOF To access the virtual garden, you can acquire a `kubeconfig` by ```shell -kubectl -n garden get secret -l name=user-kubeconfig -o jsonpath={..data.kubeconfig} | base64 -d > /tmp/virtual-garden-kubeconfig +kubectl -n garden get secret gardener -o jsonpath={.data.kubeconfig} | base64 -d > /tmp/virtual-garden-kubeconfig kubectl --kubeconfig /tmp/virtual-garden-kubeconfig get namespaces ``` +Note that this kubeconfig uses a token that has validity of `12h` only, hence it might expire and causing you to re-download the kubeconfig. + ### Deleting the `Garden` ```shell @@ -237,6 +234,8 @@ The virtual garden control plane components are: - `virtual-garden-etcd-main` - `virtual-garden-etcd-events` - `virtual-garden-kube-apiserver` +- `virtual-garden-kube-controller-manager` +- `virtual-garden-gardener-resource-manager` If the `.spec.virtualCluster.controlPlane.highAvailability={}` is set then these components will be deployed in a "highly available" mode. For ETCD, this means that there will be 3 replicas each. @@ -246,10 +245,16 @@ The `gardener-resource-manager`'s [HighAvailabilityConfig webhook](resource-mana > If once set, removing `.spec.virtualCluster.controlPlane.highAvailability` again is not supported. The `virtual-garden-kube-apiserver` `Deployment` is exposed via a `Service` of type `LoadBalancer` with the same name. -In the future, we might switch to exposing it via Istio, similar to how the `kube-apiservers` of shoot clusters are exposed. +In the future, we will switch to exposing it via Istio, similar to how the `kube-apiservers` of shoot clusters are exposed. Similar to the `Shoot` API, the version of the virtual garden cluster is controlled via `.spec.virtualCluster.kubernetes.version`. -Likewise, specific configuration for the control plane components can be provided in the same section, e.g. via `.spec.virtualCluster.kubernetes.kubeAPIServer` for the `kube-apiserver`. +Likewise, specific configuration for the control plane components can be provided in the same section, e.g. via `.spec.virtualCluster.kubernetes.kubeAPIServer` for the `kube-apiserver` or `.spec.virtualCluster.kubernetes.kubeControllerManager` for the `kube-controller-manager`. + +The `kube-controller-manager` only runs a very few controllers that are necessary in the scenario of the virtual garden. +Most prominently, **the `serviceaccount-token` controller is unconditionally disabled**. +Hence, the usage of static `ServiceAccount` secrets is not supported generally. +Instead, the [`TokenRequest` API](https://kubernetes.io/docs/reference/kubernetes-api/authentication-resources/token-request-v1/) should be used. +Third-party components that need to communicate with the virtual cluster can leverage the [`gardener-resource-manager`'s `TokenRequestor` controller](resource-manager.md#tokenrequestor-controller) and the generic kubeconfig, just like it works for `Shoot`s. For the virtual cluster, it is essential to provide a DNS domain via `.spec.virtualCluster.dns.domain`. **The respective DNS record is not managed by `gardener-operator` and should be manually created and pointed to the load balancer IP of the `virtual-garden-kube-apiserver` `Service`.** diff --git a/docs/development/priority-classes.md b/docs/development/priority-classes.md index d59b2bb5329..fbccb2dfd24 100644 --- a/docs/development/priority-classes.md +++ b/docs/development/priority-classes.md @@ -23,8 +23,8 @@ When using the `gardener-operator` for managing the garden runtime and virtual c |---------------------------------- |-----------|-------------------------------------------------------------------------------------------| | `gardener-garden-system-critical` | 999999550 | `gardener-operator`, `gardener-resource-manager`, `istio` | | `gardener-garden-system-500` | 999999500 | `virtual-garden-etcd-events`, `virtual-garden-etcd-main`, `virtual-garden-kube-apiserver` | -| `gardener-garden-system-400` | 999999400 | | -| `gardener-garden-system-300` | 999999300 | `vpa-admission-controller`, `etcd-druid` | +| `gardener-garden-system-400` | 999999400 | `virtual-garden-gardener-resource-manager` | +| `gardener-garden-system-300` | 999999300 | `virtual-garden-kube-controller-manager`, `vpa-admission-controller`, `etcd-druid` | | `gardener-garden-system-200` | 999999200 | `vpa-recommender`, `vpa-updater`, `hvpa-controller` | | `gardener-garden-system-100` | 999999100 | `kube-state-metrics` | diff --git a/example/operator/10-crd-operator.gardener.cloud_gardens.yaml b/example/operator/10-crd-operator.gardener.cloud_gardens.yaml index 56a02de2144..58e35d03450 100644 --- a/example/operator/10-crd-operator.gardener.cloud_gardens.yaml +++ b/example/operator/10-crd-operator.gardener.cloud_gardens.yaml @@ -765,6 +765,78 @@ spec: type: array type: object type: object + kubeControllerManager: + description: KubeControllerManager contains configuration + settings for the kube-controller-manager. + properties: + certificateSigningDuration: + default: 48h + description: CertificateSigningDuration is the maximum + length of duration signed certificates will be given. + Individual CSRs may request shorter certs by setting + `spec.expirationSeconds`. + pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$ + type: string + featureGates: + additionalProperties: + type: boolean + description: FeatureGates contains information about enabled + feature gates. + type: object + horizontalPodAutoscaler: + description: HorizontalPodAutoscalerConfig contains horizontal + pod autoscaler configuration settings for the kube-controller-manager. + properties: + cpuInitializationPeriod: + description: The period after which a ready pod transition + is considered to be the first. + type: string + downscaleStabilization: + description: The configurable window at which the + controller will choose the highest recommendation + for autoscaling. + type: string + initialReadinessDelay: + description: The configurable period at which the + horizontal pod autoscaler considers a Pod “not yet + ready” given that it’s unready and it has transitioned + to unready during that time. + type: string + syncPeriod: + description: The period for syncing the number of + pods in horizontal pod autoscaler. + type: string + tolerance: + description: The minimum change (from 1.0) in the + desired-to-actual metrics ratio for the horizontal + pod autoscaler to consider scaling. + type: number + type: object + nodeCIDRMaskSize: + description: NodeCIDRMaskSize defines the mask size for + node cidr in cluster (default is 24). This field is + immutable. + format: int32 + type: integer + nodeMonitorGracePeriod: + description: NodeMonitorGracePeriod defines the grace + period before an unresponsive node is marked unhealthy. + type: string + podEvictionTimeout: + description: "PodEvictionTimeout defines the grace period + for deleting pods on failed nodes. Defaults to 2m. \n + Deprecated: The corresponding kube-controller-manager + flag `--pod-eviction-timeout` is deprecated in favor + of the kube-apiserver flags `--default-not-ready-toleration-seconds` + and `--default-unreachable-toleration-seconds`. The + `--pod-eviction-timeout` flag does not have effect when + the taint besed eviction is enabled. The taint based + eviction is beta (enabled by default) since Kubernetes + 1.13 and GA since Kubernetes 1.18. Hence, instead of + setting this field, set the `spec.kubernetes.kubeAPIServer.defaultNotReadyTolerationSeconds` + and `spec.kubernetes.kubeAPIServer.defaultUnreachableTolerationSeconds`." + type: string + type: object version: description: Version is the semantic Kubernetes version to use for the virtual garden cluster. diff --git a/example/operator/20-garden.yaml b/example/operator/20-garden.yaml index c65cc3f8234..3e15b95b21e 100644 --- a/example/operator/20-garden.yaml +++ b/example/operator/20-garden.yaml @@ -135,6 +135,10 @@ spec: # resourcesToStoreInETCDEvents: # - group: networking.k8s.io # resources: networkpolicies + # kubeControllerManager: + # featureGates: + # SomeKubernetesFeature: true + # certificateSigningDuration: 48h maintenance: timeWindow: begin: 220000+0100 diff --git a/example/operator/doc.go b/example/operator/doc.go index 454c073cdd9..327eff107df 100644 --- a/example/operator/doc.go +++ b/example/operator/doc.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:generate ../../hack/generate-crds.sh 10-crd- operator.gardener.cloud +//go:generate ../../hack/generate-crds.sh 10-crd- -allow-dangerous-types operator.gardener.cloud //go:generate cp 10-crd-operator.gardener.cloud_gardens.yaml ../../charts/gardener/operator/templates/customresouredefintion.yaml // Package operator contains example manifests for working on operator. diff --git a/hack/generate-crds.sh b/hack/generate-crds.sh index 46a6454ddf2..b119d9a4c29 100755 --- a/hack/generate-crds.sh +++ b/hack/generate-crds.sh @@ -35,6 +35,7 @@ fi output_dir="$(pwd)" file_name_prefix="$1" add_deletion_protection_label=false +crd_options="" get_group_package () { case "$1" in @@ -62,6 +63,15 @@ get_group_package () { esac } +generate_all_groups () { + generate_group extensions.gardener.cloud + generate_group resources.gardener.cloud + generate_group operator.gardener.cloud + generate_group druid.gardener.cloud + generate_group autoscaling.k8s.io + generate_group fluentbit.fluent.io +} + generate_group () { local group="$1" echo "Generating CRDs for $group group" @@ -80,7 +90,7 @@ generate_group () { rm "$output_dir"/*${group}_*.yaml fi - controller-gen crd paths="$package_path" output:crd:dir="$output_dir" output:stdout + controller-gen crd"$crd_options" paths="$package_path" output:crd:dir="$output_dir" output:stdout while IFS= read -r crd; do crd_out="$output_dir/$file_name_prefix$(basename $crd)" @@ -113,12 +123,17 @@ if [ -n "${2:-}" ]; then shift done else - generate_group extensions.gardener.cloud - generate_group resources.gardener.cloud - generate_group operator.gardener.cloud - generate_group druid.gardener.cloud - generate_group autoscaling.k8s.io - generate_group fluentbit.fluent.io + generate_all_groups + fi + elif [ "${2}" == "-allow-dangerous-types" ]; then + crd_options=":allowDangerousTypes=true" + if [ -n "${3:-}" ]; then + while [ -n "${3:-}" ] ; do + generate_group "$3" + shift + done + else + generate_all_groups fi else while [ -n "${2:-}" ] ; do @@ -127,10 +142,6 @@ if [ -n "${2:-}" ]; then done fi else - generate_group extensions.gardener.cloud - generate_group resources.gardener.cloud - generate_group operator.gardener.cloud - generate_group druid.gardener.cloud - generate_group autoscaling.k8s.io - generate_group fluentbit.fluent.io + generate_all_groups fi + diff --git a/pkg/apis/core/v1beta1/defaults.go b/pkg/apis/core/v1beta1/defaults.go index 074661e4254..edeb46a1857 100644 --- a/pkg/apis/core/v1beta1/defaults.go +++ b/pkg/apis/core/v1beta1/defaults.go @@ -160,6 +160,13 @@ func SetDefaults_Shoot(obj *Shoot) { obj.Spec.Kubernetes.AllowPrivilegedContainers = pointer.Bool(true) } + if obj.Spec.Kubernetes.KubeAPIServer.DefaultNotReadyTolerationSeconds == nil { + obj.Spec.Kubernetes.KubeAPIServer.DefaultNotReadyTolerationSeconds = pointer.Int64(300) + } + if obj.Spec.Kubernetes.KubeAPIServer.DefaultUnreachableTolerationSeconds == nil { + obj.Spec.Kubernetes.KubeAPIServer.DefaultUnreachableTolerationSeconds = pointer.Int64(300) + } + if obj.Spec.Kubernetes.KubeControllerManager.NodeCIDRMaskSize == nil { obj.Spec.Kubernetes.KubeControllerManager.NodeCIDRMaskSize = calculateDefaultNodeCIDRMaskSize(&obj.Spec) } @@ -288,14 +295,11 @@ func SetDefaults_KubeAPIServerConfig(obj *KubeAPIServerConfig) { if obj.Logging.Verbosity == nil { obj.Logging.Verbosity = pointer.Int32(2) } - if obj.DefaultNotReadyTolerationSeconds == nil { - obj.DefaultNotReadyTolerationSeconds = pointer.Int64(300) - } - if obj.DefaultUnreachableTolerationSeconds == nil { - obj.DefaultUnreachableTolerationSeconds = pointer.Int64(300) - } } +// SetDefaults_KubeControllerManagerConfig sets default values for KubeControllerManagerConfig objects. +func SetDefaults_KubeControllerManagerConfig(obj *KubeControllerManagerConfig) {} + // SetDefaults_Networking sets default values for Networking objects. func SetDefaults_Networking(obj *Networking) { if len(obj.IPFamilies) == 0 { diff --git a/pkg/apis/core/v1beta1/zz_generated.defaults.go b/pkg/apis/core/v1beta1/zz_generated.defaults.go index 5e124eeb120..752ae794d33 100644 --- a/pkg/apis/core/v1beta1/zz_generated.defaults.go +++ b/pkg/apis/core/v1beta1/zz_generated.defaults.go @@ -145,6 +145,9 @@ func SetObjectDefaults_Shoot(in *Shoot) { if in.Spec.Kubernetes.KubeAPIServer != nil { SetDefaults_KubeAPIServerConfig(in.Spec.Kubernetes.KubeAPIServer) } + if in.Spec.Kubernetes.KubeControllerManager != nil { + SetDefaults_KubeControllerManagerConfig(in.Spec.Kubernetes.KubeControllerManager) + } if in.Spec.Kubernetes.VerticalPodAutoscaler != nil { SetDefaults_VerticalPodAutoscaler(in.Spec.Kubernetes.VerticalPodAutoscaler) } diff --git a/pkg/apis/core/validation/shoot.go b/pkg/apis/core/validation/shoot.go index ee7b65a996f..81563fd080c 100644 --- a/pkg/apis/core/validation/shoot.go +++ b/pkg/apis/core/validation/shoot.go @@ -779,7 +779,7 @@ func validateKubernetes(kubernetes core.Kubernetes, networking *core.Networking, } allErrs = append(allErrs, ValidateKubeAPIServer(kubernetes.KubeAPIServer, kubernetes.Version, false, fldPath.Child("kubeAPIServer"))...) - allErrs = append(allErrs, validateKubeControllerManager(kubernetes.KubeControllerManager, networking, kubernetes.Version, workerless, fldPath.Child("kubeControllerManager"))...) + allErrs = append(allErrs, ValidateKubeControllerManager(kubernetes.KubeControllerManager, networking, kubernetes.Version, workerless, fldPath.Child("kubeControllerManager"))...) if workerless { allErrs = append(allErrs, validateKubernetesForWorkerlessShoot(kubernetes, fldPath)...) @@ -1164,7 +1164,8 @@ func ValidateKubeAPIServer(kubeAPIServer *core.KubeAPIServerConfig, version stri return allErrs } -func validateKubeControllerManager(kcm *core.KubeControllerManagerConfig, networking *core.Networking, version string, workerless bool, fldPath *field.Path) field.ErrorList { +// ValidateKubeControllerManager validates KubeControllerManagerConfig. +func ValidateKubeControllerManager(kcm *core.KubeControllerManagerConfig, networking *core.Networking, version string, workerless bool, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} if kcm == nil { diff --git a/pkg/apis/operator/v1alpha1/types.go b/pkg/apis/operator/v1alpha1/types.go index ffa0016e3a8..16a07c8c996 100644 --- a/pkg/apis/operator/v1alpha1/types.go +++ b/pkg/apis/operator/v1alpha1/types.go @@ -245,6 +245,9 @@ type Kubernetes struct { // KubeAPIServer contains configuration settings for the kube-apiserver. // +optional KubeAPIServer *KubeAPIServerConfig `json:"kubeAPIServer,omitempty"` + // KubeControllerManager contains configuration settings for the kube-controller-manager. + // +optional + KubeControllerManager *KubeControllerManagerConfig `json:"kubeControllerManager,omitempty"` // Version is the semantic Kubernetes version to use for the virtual garden cluster. // +kubebuilder:validation:MinLength=1 Version string `json:"version"` @@ -375,6 +378,20 @@ type Networking struct { Services string `json:"services"` } +// KubeControllerManagerConfig contains configuration settings for the kube-controller-manager. +type KubeControllerManagerConfig struct { + // KubeControllerManagerConfig contains all configuration values not specific to the virtual garden cluster. + // +optional + *gardencorev1beta1.KubeControllerManagerConfig `json:",inline"` + // CertificateSigningDuration is the maximum length of duration signed certificates will be given. Individual CSRs + // may request shorter certs by setting `spec.expirationSeconds`. + // +kubebuilder:validation:Type=string + // +kubebuilder:validation:Pattern="^([0-9]+(\\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$" + // +kubebuilder:default=`48h` + // +optional + CertificateSigningDuration *metav1.Duration `json:"certificateSigningDuration,omitempty"` +} + // GardenStatus is the status of a garden environment. type GardenStatus struct { // Gardener holds information about the Gardener which last acted on the Garden. diff --git a/pkg/apis/operator/v1alpha1/validation/validation.go b/pkg/apis/operator/v1alpha1/validation/validation.go index 12f7304086d..4fee199aaff 100644 --- a/pkg/apis/operator/v1alpha1/validation/validation.go +++ b/pkg/apis/operator/v1alpha1/validation/validation.go @@ -117,6 +117,17 @@ func validateVirtualCluster(virtualCluster operatorv1alpha1.VirtualCluster, runt allErrs = append(allErrs, gardencorevalidation.ValidateKubeAPIServer(coreKubeAPIServerConfig, virtualCluster.Kubernetes.Version, true, path)...) } + if kubeControllerManager := virtualCluster.Kubernetes.KubeControllerManager; kubeControllerManager != nil && kubeControllerManager.KubeControllerManagerConfig != nil { + path := fldPath.Child("kubernetes", "kubeControllerManager") + + coreKubeControllerManagerConfig := &gardencore.KubeControllerManagerConfig{} + if err := gardenCoreScheme.Convert(kubeControllerManager.KubeControllerManagerConfig, coreKubeControllerManagerConfig, nil); err != nil { + allErrs = append(allErrs, field.InternalError(path, err)) + } + + allErrs = append(allErrs, gardencorevalidation.ValidateKubeControllerManager(coreKubeControllerManagerConfig, nil, virtualCluster.Kubernetes.Version, true, path)...) + } + if _, _, err := net.ParseCIDR(virtualCluster.Networking.Services); err != nil { allErrs = append(allErrs, field.Invalid(fldPath.Child("networking", "services"), virtualCluster.Networking.Services, fmt.Sprintf("cannot parse service network cidr: %s", err.Error()))) } diff --git a/pkg/apis/operator/v1alpha1/validation/validation_test.go b/pkg/apis/operator/v1alpha1/validation/validation_test.go index 78f685c0bde..bc4c15e081c 100644 --- a/pkg/apis/operator/v1alpha1/validation/validation_test.go +++ b/pkg/apis/operator/v1alpha1/validation/validation_test.go @@ -730,6 +730,56 @@ var _ = Describe("Validation Tests", func() { })))) }) }) + + Context("topology-aware routing field", func() { + It("should prevent enabling topology-aware routing on single-zone cluster", func() { + garden.Spec.RuntimeCluster.Provider.Zones = []string{"a"} + garden.Spec.RuntimeCluster.Settings = &operatorv1alpha1.Settings{ + TopologyAwareRouting: &operatorv1alpha1.SettingTopologyAwareRouting{ + Enabled: true, + }, + } + garden.Spec.VirtualCluster.ControlPlane = &operatorv1alpha1.ControlPlane{HighAvailability: &operatorv1alpha1.HighAvailability{}} + + Expect(ValidateGarden(garden)).To(ConsistOf( + PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeForbidden), + "Field": Equal("spec.runtimeCluster.settings.topologyAwareRouting.enabled"), + "Detail": Equal("topology-aware routing can only be enabled on multi-zone garden runtime cluster (with at least two zones in spec.provider.zones)"), + })), + )) + }) + + It("should prevent enabling topology-aware routing when control-plane is not HA", func() { + garden.Spec.RuntimeCluster.Provider.Zones = []string{"a", "b", "c"} + garden.Spec.RuntimeCluster.Settings = &operatorv1alpha1.Settings{ + TopologyAwareRouting: &operatorv1alpha1.SettingTopologyAwareRouting{ + Enabled: true, + }, + } + garden.Spec.VirtualCluster.ControlPlane = nil + + Expect(ValidateGarden(garden)).To(ConsistOf( + PointTo(MatchFields(IgnoreExtras, Fields{ + "Type": Equal(field.ErrorTypeForbidden), + "Field": Equal("spec.runtimeCluster.settings.topologyAwareRouting.enabled"), + "Detail": Equal("topology-aware routing can only be enabled when virtual cluster's high-availability is enabled"), + })), + )) + }) + + It("should allow enabling topology-aware routing on multi-zone cluster with HA control-plane", func() { + garden.Spec.RuntimeCluster.Provider.Zones = []string{"a", "b", "c"} + garden.Spec.RuntimeCluster.Settings = &operatorv1alpha1.Settings{ + TopologyAwareRouting: &operatorv1alpha1.SettingTopologyAwareRouting{ + Enabled: true, + }, + } + garden.Spec.VirtualCluster.ControlPlane = &operatorv1alpha1.ControlPlane{HighAvailability: &operatorv1alpha1.HighAvailability{}} + + Expect(ValidateGarden(garden)).To(BeEmpty()) + }) + }) }) Context("virtual cluster", func() { @@ -782,58 +832,6 @@ var _ = Describe("Validation Tests", func() { }) }) }) - - Context("runtime cluster", func() { - Context("topology-aware routing field", func() { - It("should prevent enabling topology-aware routing on single-zone cluster", func() { - garden.Spec.RuntimeCluster.Provider.Zones = []string{"a"} - garden.Spec.RuntimeCluster.Settings = &operatorv1alpha1.Settings{ - TopologyAwareRouting: &operatorv1alpha1.SettingTopologyAwareRouting{ - Enabled: true, - }, - } - garden.Spec.VirtualCluster.ControlPlane = &operatorv1alpha1.ControlPlane{HighAvailability: &operatorv1alpha1.HighAvailability{}} - - Expect(ValidateGarden(garden)).To(ConsistOf( - PointTo(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(field.ErrorTypeForbidden), - "Field": Equal("spec.runtimeCluster.settings.topologyAwareRouting.enabled"), - "Detail": Equal("topology-aware routing can only be enabled on multi-zone garden runtime cluster (with at least two zones in spec.provider.zones)"), - })), - )) - }) - - It("should prevent enabling topology-aware routing when control-plane is not HA", func() { - garden.Spec.RuntimeCluster.Provider.Zones = []string{"a", "b", "c"} - garden.Spec.RuntimeCluster.Settings = &operatorv1alpha1.Settings{ - TopologyAwareRouting: &operatorv1alpha1.SettingTopologyAwareRouting{ - Enabled: true, - }, - } - garden.Spec.VirtualCluster.ControlPlane = nil - - Expect(ValidateGarden(garden)).To(ConsistOf( - PointTo(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(field.ErrorTypeForbidden), - "Field": Equal("spec.runtimeCluster.settings.topologyAwareRouting.enabled"), - "Detail": Equal("topology-aware routing can only be enabled when virtual cluster's high-availability is enabled"), - })), - )) - }) - - It("should allow enabling topology-aware routing on multi-zone cluster with HA control-plane", func() { - garden.Spec.RuntimeCluster.Provider.Zones = []string{"a", "b", "c"} - garden.Spec.RuntimeCluster.Settings = &operatorv1alpha1.Settings{ - TopologyAwareRouting: &operatorv1alpha1.SettingTopologyAwareRouting{ - Enabled: true, - }, - } - garden.Spec.VirtualCluster.ControlPlane = &operatorv1alpha1.ControlPlane{HighAvailability: &operatorv1alpha1.HighAvailability{}} - - Expect(ValidateGarden(garden)).To(BeEmpty()) - }) - }) - }) }) Describe("#ValidateGardenUpdate", func() { diff --git a/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go index 5060d2d67ef..da5cd75d184 100644 --- a/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go @@ -521,6 +521,32 @@ func (in *KubeAPIServerConfig) DeepCopy() *KubeAPIServerConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeControllerManagerConfig) DeepCopyInto(out *KubeControllerManagerConfig) { + *out = *in + if in.KubeControllerManagerConfig != nil { + in, out := &in.KubeControllerManagerConfig, &out.KubeControllerManagerConfig + *out = new(v1beta1.KubeControllerManagerConfig) + (*in).DeepCopyInto(*out) + } + if in.CertificateSigningDuration != nil { + in, out := &in.CertificateSigningDuration, &out.CertificateSigningDuration + *out = new(v1.Duration) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeControllerManagerConfig. +func (in *KubeControllerManagerConfig) DeepCopy() *KubeControllerManagerConfig { + if in == nil { + return nil + } + out := new(KubeControllerManagerConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Kubernetes) DeepCopyInto(out *Kubernetes) { *out = *in @@ -529,6 +555,11 @@ func (in *Kubernetes) DeepCopyInto(out *Kubernetes) { *out = new(KubeAPIServerConfig) (*in).DeepCopyInto(*out) } + if in.KubeControllerManager != nil { + in, out := &in.KubeControllerManager, &out.KubeControllerManager + *out = new(KubeControllerManagerConfig) + (*in).DeepCopyInto(*out) + } return } diff --git a/pkg/operation/botanist/component/kubecontrollermanager/kube_controller_manager.go b/pkg/operation/botanist/component/kubecontrollermanager/kube_controller_manager.go index 93aa7f51563..249c8996f72 100644 --- a/pkg/operation/botanist/component/kubecontrollermanager/kube_controller_manager.go +++ b/pkg/operation/botanist/component/kubecontrollermanager/kube_controller_manager.go @@ -136,8 +136,12 @@ type Values struct { Image string // Replicas is the number of replicas for the kube-controller-manager deployment. Replicas int32 + // PriorityClassName is the name of the priority class. + PriorityClassName string // Config is the configuration of the kube-controller-manager. Config *gardencorev1beta1.KubeControllerManagerConfig + // NamePrefix is the prefix for the resource names. + NamePrefix string // HVPAConfig is the configuration for HVPA. HVPAConfig *HVPAConfig // IsWorkerless specifies whether the cluster has worker nodes. @@ -200,8 +204,8 @@ const ( func (k *kubeControllerManager) Deploy(ctx context.Context) error { serverSecret, err := k.secretsManager.Generate(ctx, &secrets.CertificateSecretConfig{ Name: secretNameServer, - CommonName: v1beta1constants.DeploymentNameKubeControllerManager, - DNSNames: kubernetesutils.DNSNamesForService(serviceName, k.namespace), + CommonName: k.values.NamePrefix + v1beta1constants.DeploymentNameKubeControllerManager, + DNSNames: kubernetesutils.DNSNamesForService(k.values.NamePrefix+serviceName, k.namespace), CertType: secrets.ServerCert, SkipPublishingCACertificate: true, }, secretsmanager.SignedByCA(v1beta1constants.SecretNameCACluster), secretsmanager.Rotate(secretsmanager.InPlace)) @@ -325,12 +329,12 @@ func (k *kubeControllerManager) Deploy(ctx context.Context) error { v1beta1constants.GardenRole: v1beta1constants.GardenRoleControlPlane, v1beta1constants.LabelPodMaintenanceRestart: "true", v1beta1constants.LabelNetworkPolicyToDNS: v1beta1constants.LabelNetworkPolicyAllowed, - gardenerutils.NetworkPolicyLabel(v1beta1constants.DeploymentNameKubeAPIServer, kubeapiserverconstants.Port): v1beta1constants.LabelNetworkPolicyAllowed, + gardenerutils.NetworkPolicyLabel(k.values.NamePrefix+v1beta1constants.DeploymentNameKubeAPIServer, kubeapiserverconstants.Port): v1beta1constants.LabelNetworkPolicyAllowed, }), }, Spec: corev1.PodSpec{ AutomountServiceAccountToken: pointer.Bool(false), - PriorityClassName: v1beta1constants.PriorityClassNameShootControlPlane300, + PriorityClassName: k.values.PriorityClassName, Containers: []corev1.Container{ { Name: containerName, @@ -527,7 +531,7 @@ func (k *kubeControllerManager) Deploy(ctx context.Context) error { hvpa.Spec.TargetRef = &autoscalingv2beta1.CrossVersionObjectReference{ APIVersion: appsv1.SchemeGroupVersion.String(), Kind: "Deployment", - Name: v1beta1constants.DeploymentNameKubeControllerManager, + Name: k.values.NamePrefix + v1beta1constants.DeploymentNameKubeControllerManager, } return nil }); err != nil { @@ -544,7 +548,7 @@ func (k *kubeControllerManager) Deploy(ctx context.Context) error { vpa.Spec.TargetRef = &autoscalingv1.CrossVersionObjectReference{ APIVersion: appsv1.SchemeGroupVersion.String(), Kind: "Deployment", - Name: v1beta1constants.DeploymentNameKubeControllerManager, + Name: k.values.NamePrefix + v1beta1constants.DeploymentNameKubeControllerManager, } vpa.Spec.UpdatePolicy = &vpaautoscalingv1.PodUpdatePolicy{ UpdateMode: &vpaUpdateMode, @@ -559,28 +563,40 @@ func (k *kubeControllerManager) Deploy(ctx context.Context) error { return k.reconcileShootResources(ctx, shootAccessSecret.ServiceAccountName) } -func (k *kubeControllerManager) SetShootClient(c client.Client) { k.shootClient = c } -func (k *kubeControllerManager) SetReplicaCount(replicas int32) { k.values.Replicas = replicas } -func (k *kubeControllerManager) Destroy(_ context.Context) error { return nil } +func (k *kubeControllerManager) Destroy(ctx context.Context) error { + return kubernetesutils.DeleteObjects(ctx, k.seedClient.Client(), + k.emptyManagedResource(), + k.emptyManagedResourceSecret(), + k.emptyVPA(), + k.emptyHVPA(), + k.emptyService(), + k.emptyPodDisruptionBudget(), + k.emptyDeployment(), + k.newShootAccessSecret().Secret, + ) +} + +func (k *kubeControllerManager) SetShootClient(c client.Client) { k.shootClient = c } +func (k *kubeControllerManager) SetReplicaCount(replicas int32) { k.values.Replicas = replicas } func (k *kubeControllerManager) emptyVPA() *vpaautoscalingv1.VerticalPodAutoscaler { - return &vpaautoscalingv1.VerticalPodAutoscaler{ObjectMeta: metav1.ObjectMeta{Name: "kube-controller-manager-vpa", Namespace: k.namespace}} + return &vpaautoscalingv1.VerticalPodAutoscaler{ObjectMeta: metav1.ObjectMeta{Name: k.values.NamePrefix + "kube-controller-manager-vpa", Namespace: k.namespace}} } func (k *kubeControllerManager) emptyHVPA() *hvpav1alpha1.Hvpa { - return &hvpav1alpha1.Hvpa{ObjectMeta: metav1.ObjectMeta{Name: v1beta1constants.DeploymentNameKubeControllerManager, Namespace: k.namespace}} + return &hvpav1alpha1.Hvpa{ObjectMeta: metav1.ObjectMeta{Name: k.values.NamePrefix + v1beta1constants.DeploymentNameKubeControllerManager, Namespace: k.namespace}} } func (k *kubeControllerManager) emptyService() *corev1.Service { - return &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: k.namespace}} + return &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: k.values.NamePrefix + serviceName, Namespace: k.namespace}} } func (k *kubeControllerManager) emptyDeployment() *appsv1.Deployment { - return &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: v1beta1constants.DeploymentNameKubeControllerManager, Namespace: k.namespace}} + return &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: k.values.NamePrefix + v1beta1constants.DeploymentNameKubeControllerManager, Namespace: k.namespace}} } func (k *kubeControllerManager) emptyPodDisruptionBudget() client.Object { - objectMeta := metav1.ObjectMeta{Name: v1beta1constants.DeploymentNameKubeControllerManager, Namespace: k.namespace} + objectMeta := metav1.ObjectMeta{Name: k.values.NamePrefix + v1beta1constants.DeploymentNameKubeControllerManager, Namespace: k.namespace} if versionutils.ConstraintK8sGreaterEqual121.Check(k.values.RuntimeVersion) { return &policyv1.PodDisruptionBudget{ObjectMeta: objectMeta} @@ -720,7 +736,7 @@ func (k *kubeControllerManager) computeCommand(port int32) []string { command = append(command, fmt.Sprintf("--concurrent-serviceaccount-token-syncs=%d", v)) } - if len(k.values.Config.FeatureGates) > 0 { + if k.values.Config != nil && len(k.values.Config.FeatureGates) > 0 { command = append(command, kubernetesutils.FeatureGatesToCommandLineParameter(k.values.Config.FeatureGates)) } @@ -762,7 +778,7 @@ func (k *kubeControllerManager) getHorizontalPodAutoscalerConfig() gardencorev1b Tolerance: &defaultHPATolerance, } - if k.values.Config.HorizontalPodAutoscalerConfig != nil { + if k.values.Config != nil && k.values.Config.HorizontalPodAutoscalerConfig != nil { if v := k.values.Config.HorizontalPodAutoscalerConfig.CPUInitializationPeriod; v != nil { horizontalPodAutoscalerConfig.CPUInitializationPeriod = v } diff --git a/pkg/operation/botanist/component/kubecontrollermanager/kube_controller_manager_test.go b/pkg/operation/botanist/component/kubecontrollermanager/kube_controller_manager_test.go index 0c6ef2b0a32..320a58f8618 100644 --- a/pkg/operation/botanist/component/kubecontrollermanager/kube_controller_manager_test.go +++ b/pkg/operation/botanist/component/kubecontrollermanager/kube_controller_manager_test.go @@ -83,6 +83,7 @@ var _ = Describe("KubeControllerManager", func() { hvpaConfigEnabled = &HVPAConfig{Enabled: true} hvpaConfigEnabledScaleDownOff = &HVPAConfig{Enabled: true, ScaleDownUpdateMode: pointer.String(hvpav1alpha1.UpdateModeOff)} isWorkerless = false + priorityClassName = v1beta1constants.PriorityClassNameShootControlPlane300 hpaConfig = gardencorev1beta1.HorizontalPodAutoscalerConfig{ CPUInitializationPeriod: &metav1.Duration{Duration: 5 * time.Minute}, @@ -146,6 +147,25 @@ var _ = Describe("KubeControllerManager", func() { fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetesscheme.Scheme).Build() sm = fakesecretsmanager.New(fakeClient, namespace) + values = Values{ + RuntimeVersion: runtimeKubernetesVersion, + TargetVersion: semverVersion, + Image: image, + Config: &kcmConfig, + PriorityClassName: priorityClassName, + HVPAConfig: hvpaConfigDisabled, + IsWorkerless: isWorkerless, + PodNetwork: podCIDR, + ServiceNetwork: serviceCIDR, + } + kubeControllerManager = New( + testLogger, + fakeInterface, + namespace, + sm, + values, + ) + By("Create secrets managed outside of this package for whose secretsmanager.Get() will be called") Expect(fakeClient.Create(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "ca", Namespace: namespace}})).To(Succeed()) Expect(fakeClient.Create(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "generic-token-kubeconfig", Namespace: namespace}})).To(Succeed()) @@ -160,26 +180,6 @@ var _ = Describe("KubeControllerManager", func() { Describe("#Deploy", func() { Context("Tests expecting a failure", func() { - BeforeEach(func() { - values = Values{ - RuntimeVersion: runtimeKubernetesVersion, - TargetVersion: semverVersion, - Image: image, - Config: &kcmConfig, - HVPAConfig: hvpaConfigDisabled, - IsWorkerless: isWorkerless, - PodNetwork: podCIDR, - ServiceNetwork: serviceCIDR, - } - kubeControllerManager = New( - testLogger, - fakeInterface, - namespace, - sm, - values, - ) - }) - It("should fail when the service cannot be created", func() { gomock.InOrder( c.EXPECT().Get(ctx, kubernetesutils.Key(namespace, serviceName), gomock.AssignableToTypeOf(&corev1.Service{})), @@ -532,7 +532,7 @@ var _ = Describe("KubeControllerManager", func() { }, Spec: corev1.PodSpec{ AutomountServiceAccountToken: pointer.Bool(false), - PriorityClassName: v1beta1constants.PriorityClassNameShootControlPlane300, + PriorityClassName: priorityClassName, Containers: []corev1.Container{ { Name: "kube-controller-manager", @@ -713,6 +713,7 @@ subjects: TargetVersion: semverVersion, Image: image, Config: config, + PriorityClassName: priorityClassName, HVPAConfig: hvpaConfig, IsWorkerless: isWorkerless, PodNetwork: podCIDR, @@ -826,6 +827,7 @@ subjects: TargetVersion: semverVersion, Image: image, Config: config, + PriorityClassName: priorityClassName, HVPAConfig: hvpaConfig, IsWorkerless: isWorkerless, PodNetwork: podCIDR, @@ -934,7 +936,26 @@ subjects: }) Describe("#Destroy", func() { - It("should return nil as it's not implemented as of now", func() { + It("should successfully destroy all resources", func() { + kubeControllerManager = New( + testLogger, + fakeInterface, + namespace, + sm, + values, + ) + + gomock.InOrder( + c.EXPECT().Delete(ctx, &resourcesv1alpha1.ManagedResource{ObjectMeta: metav1.ObjectMeta{Name: managedResourceName, Namespace: namespace}}), + c.EXPECT().Delete(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: managedResourceSecretName, Namespace: namespace}}), + c.EXPECT().Delete(ctx, &vpaautoscalingv1.VerticalPodAutoscaler{ObjectMeta: metav1.ObjectMeta{Name: vpaName, Namespace: namespace}}), + c.EXPECT().Delete(ctx, &hvpav1alpha1.Hvpa{ObjectMeta: metav1.ObjectMeta{Name: hvpaName, Namespace: namespace}}), + c.EXPECT().Delete(ctx, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespace}}), + c.EXPECT().Delete(ctx, &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: pdbName, Namespace: namespace}}), + c.EXPECT().Delete(ctx, &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "kube-controller-manager", Namespace: namespace}}), + c.EXPECT().Delete(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: secretName, Namespace: namespace}}), + ) + Expect(kubeControllerManager.Destroy(ctx)).To(Succeed()) }) }) diff --git a/pkg/operation/botanist/component/shared/kubecontrollermanager.go b/pkg/operation/botanist/component/shared/kubecontrollermanager.go new file mode 100644 index 00000000000..04b62665b02 --- /dev/null +++ b/pkg/operation/botanist/component/shared/kubecontrollermanager.go @@ -0,0 +1,81 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shared + +import ( + "net" + "time" + + "github.com/Masterminds/semver" + "github.com/go-logr/logr" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + "github.com/gardener/gardener/pkg/client/kubernetes" + "github.com/gardener/gardener/pkg/operation/botanist/component/kubecontrollermanager" + "github.com/gardener/gardener/pkg/utils/images" + "github.com/gardener/gardener/pkg/utils/imagevector" + secretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager" +) + +// NewKubeControllerManager returns a deployer for the kube-controller-manager. +func NewKubeControllerManager( + log logr.Logger, + runtimeClientSet kubernetes.Interface, + runtimeNamespace string, + runtimeVersion *semver.Version, + targetVersion *semver.Version, + imageVector imagevector.ImageVector, + secretsManager secretsmanager.Interface, + namePrefix string, + config *gardencorev1beta1.KubeControllerManagerConfig, + priorityClassName string, + isWorkerless bool, + hvpaConfig *kubecontrollermanager.HVPAConfig, + podNetwork *net.IPNet, + serviceNetwork *net.IPNet, + clusterSigningDuration *time.Duration, + controllerWorkers kubecontrollermanager.ControllerWorkers, + controllerSyncPeriods kubecontrollermanager.ControllerSyncPeriods, +) ( + kubecontrollermanager.Interface, + error, +) { + image, err := imageVector.FindImage(images.ImageNameKubeControllerManager, imagevector.RuntimeVersion(runtimeVersion.String()), imagevector.TargetVersion(targetVersion.String())) + if err != nil { + return nil, err + } + + return kubecontrollermanager.New( + log.WithValues("component", "kube-controller-manager"), + runtimeClientSet, + runtimeNamespace, + secretsManager, + kubecontrollermanager.Values{ + RuntimeVersion: runtimeVersion, + TargetVersion: targetVersion, + Image: image.String(), + Config: config, + PriorityClassName: priorityClassName, + NamePrefix: namePrefix, + HVPAConfig: hvpaConfig, + IsWorkerless: isWorkerless, + PodNetwork: podNetwork, + ServiceNetwork: serviceNetwork, + ClusterSigningDuration: clusterSigningDuration, + ControllerWorkers: controllerWorkers, + ControllerSyncPeriods: controllerSyncPeriods, + }, + ), nil +} diff --git a/pkg/operation/botanist/kubecontrollermanager.go b/pkg/operation/botanist/kubecontrollermanager.go index a58c8517d7f..16aa838387b 100644 --- a/pkg/operation/botanist/kubecontrollermanager.go +++ b/pkg/operation/botanist/kubecontrollermanager.go @@ -25,18 +25,12 @@ import ( "github.com/gardener/gardener/pkg/client/kubernetes" "github.com/gardener/gardener/pkg/features" "github.com/gardener/gardener/pkg/operation/botanist/component/kubecontrollermanager" - "github.com/gardener/gardener/pkg/utils/images" - "github.com/gardener/gardener/pkg/utils/imagevector" + "github.com/gardener/gardener/pkg/operation/botanist/component/shared" kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" ) // DefaultKubeControllerManager returns a deployer for the kube-controller-manager. func (b *Botanist) DefaultKubeControllerManager() (kubecontrollermanager.Interface, error) { - image, err := b.ImageVector.FindImage(images.ImageNameKubeControllerManager, imagevector.RuntimeVersion(b.SeedVersion()), imagevector.TargetVersion(b.ShootVersion())) - if err != nil { - return nil, err - } - hvpaEnabled := features.DefaultFeatureGate.Enabled(features.HVPA) if b.ManagedSeed != nil { hvpaEnabled = features.DefaultFeatureGate.Enabled(features.HVPAForShootedSeed) @@ -53,25 +47,28 @@ func (b *Botanist) DefaultKubeControllerManager() (kubecontrollermanager.Interfa pods = b.Shoot.Networks.Pods } - return kubecontrollermanager.New( - b.Logger.WithValues("component", "kube-controller-manager"), + return shared.NewKubeControllerManager( + b.Logger, b.SeedClientSet, b.Shoot.SeedNamespace, + b.Seed.KubernetesVersion, + b.Shoot.KubernetesVersion, + b.ImageVector, b.SecretsManager, - kubecontrollermanager.Values{ - RuntimeVersion: b.Seed.KubernetesVersion, - TargetVersion: b.Shoot.KubernetesVersion, - Image: image.String(), - Config: b.Shoot.GetInfo().Spec.Kubernetes.KubeControllerManager, - HVPAConfig: &kubecontrollermanager.HVPAConfig{ - Enabled: hvpaEnabled, - ScaleDownUpdateMode: &scaleDownUpdateMode, - }, - IsWorkerless: b.Shoot.IsWorkerless, - PodNetwork: pods, - ServiceNetwork: services, + "", + b.Shoot.GetInfo().Spec.Kubernetes.KubeControllerManager, + v1beta1constants.PriorityClassNameShootControlPlane300, + b.Shoot.IsWorkerless, + &kubecontrollermanager.HVPAConfig{ + Enabled: hvpaEnabled, + ScaleDownUpdateMode: &scaleDownUpdateMode, }, - ), nil + pods, + services, + nil, + kubecontrollermanager.ControllerWorkers{}, + kubecontrollermanager.ControllerSyncPeriods{}, + ) } // DeployKubeControllerManager deploys the Kubernetes Controller Manager. diff --git a/pkg/operation/botanist/kubecontrollermanager_test.go b/pkg/operation/botanist/kubecontrollermanager_test.go index a8fdf29d929..5de73dd5573 100644 --- a/pkg/operation/botanist/kubecontrollermanager_test.go +++ b/pkg/operation/botanist/kubecontrollermanager_test.go @@ -67,15 +67,14 @@ var _ = Describe("KubeControllerManager", func() { Describe("#DefaultKubeControllerManager", func() { BeforeEach(func() { - kubernetesClient.EXPECT().Version() - botanist.Logger = logr.Discard() botanist.SeedClientSet = kubernetesClient botanist.Seed = &seedpkg.Seed{ KubernetesVersion: semver.MustParse("1.25.0"), } botanist.Shoot = &shootpkg.Shoot{ - Networks: &shootpkg.Networks{}, + KubernetesVersion: semver.MustParse("1.25.0"), + Networks: &shootpkg.Networks{}, } botanist.Shoot.SetInfo(&gardencorev1beta1.Shoot{}) }) diff --git a/pkg/operation/botanist/resource_manager.go b/pkg/operation/botanist/resource_manager.go index 68f16fa382d..41f9473b9c3 100644 --- a/pkg/operation/botanist/resource_manager.go +++ b/pkg/operation/botanist/resource_manager.go @@ -65,7 +65,6 @@ func (b *Botanist) DefaultResourceManager() (resourcemanager.Interface, error) { // DeployGardenerResourceManager deploys the gardener-resource-manager func (b *Botanist) DeployGardenerResourceManager(ctx context.Context) error { - return shared.DeployGardenerResourceManager( ctx, b.SeedClientSet.Client(), diff --git a/pkg/operator/controller/garden/components.go b/pkg/operator/controller/garden/components.go index 279c8841cf7..66ee728127d 100644 --- a/pkg/operator/controller/garden/components.go +++ b/pkg/operator/controller/garden/components.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + "net" "os" "time" @@ -44,6 +45,7 @@ import ( "github.com/gardener/gardener/pkg/operation/botanist/component/istio" "github.com/gardener/gardener/pkg/operation/botanist/component/kubeapiserver" "github.com/gardener/gardener/pkg/operation/botanist/component/kubeapiserverexposure" + "github.com/gardener/gardener/pkg/operation/botanist/component/kubecontrollermanager" "github.com/gardener/gardener/pkg/operation/botanist/component/resourcemanager" sharedcomponent "github.com/gardener/gardener/pkg/operation/botanist/component/shared" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" @@ -78,7 +80,7 @@ func (r *Reconciler) newGardenerResourceManager(garden *operatorv1alpha1.Garden, ) } -func (r *Reconciler) newVirtualGardenGardenerResourceManager(garden *operatorv1alpha1.Garden, secretsManager secretsmanager.Interface) (resourcemanager.Interface, error) { +func (r *Reconciler) newVirtualGardenGardenerResourceManager(secretsManager secretsmanager.Interface) (resourcemanager.Interface, error) { return sharedcomponent.NewTargetGardenerResourceManager( r.RuntimeClientSet.Client(), r.GardenNamespace, @@ -91,7 +93,7 @@ func (r *Reconciler) newVirtualGardenGardenerResourceManager(garden *operatorv1a r.Config.LogLevel, r.Config.LogFormat, namePrefix, false, - v1beta1constants.PriorityClassNameGardenSystem500, + v1beta1constants.PriorityClassNameGardenSystem400, nil, operatorv1alpha1.SecretNameCARuntime, nil, @@ -428,6 +430,58 @@ func fetchKubeconfigFromSecret(ctx context.Context, c client.Client, key client. return kubeconfig, nil } +func (r *Reconciler) newKubeControllerManager( + log logr.Logger, + garden *operatorv1alpha1.Garden, + secretsManager secretsmanager.Interface, + targetVersion *semver.Version, +) ( + kubecontrollermanager.Interface, + error, +) { + var ( + config *gardencorev1beta1.KubeControllerManagerConfig + certificateSigningDuration *time.Duration + ) + + if controllerManager := garden.Spec.VirtualCluster.Kubernetes.KubeControllerManager; controllerManager != nil { + config = controllerManager.KubeControllerManagerConfig + certificateSigningDuration = pointer.Duration(controllerManager.CertificateSigningDuration.Duration) + } + + _, services, err := net.ParseCIDR(garden.Spec.VirtualCluster.Networking.Services) + if err != nil { + return nil, fmt.Errorf("cannot parse service network CIDR: %w", err) + } + + return sharedcomponent.NewKubeControllerManager( + log, + r.RuntimeClientSet, + r.GardenNamespace, + r.RuntimeVersion, + targetVersion, + r.ImageVector, + secretsManager, + namePrefix, + config, + v1beta1constants.PriorityClassNameGardenSystem300, + true, + &kubecontrollermanager.HVPAConfig{Enabled: hvpaEnabled()}, + nil, + services, + certificateSigningDuration, + kubecontrollermanager.ControllerWorkers{ + GarbageCollector: pointer.Int(250), + Namespace: pointer.Int(100), + ResourceQuota: pointer.Int(100), + ServiceAccountToken: pointer.Int(0), + }, + kubecontrollermanager.ControllerSyncPeriods{ + ResourceQuota: pointer.Duration(time.Minute), + }, + ) +} + func (r *Reconciler) newKubeStateMetrics() (component.DeployWaiter, error) { return sharedcomponent.NewKubeStateMetrics( r.RuntimeClientSet.Client(), diff --git a/pkg/operator/controller/garden/reconciler_delete.go b/pkg/operator/controller/garden/reconciler_delete.go index 063917c56c5..46771e47710 100644 --- a/pkg/operator/controller/garden/reconciler_delete.go +++ b/pkg/operator/controller/garden/reconciler_delete.go @@ -98,7 +98,11 @@ func (r *Reconciler) delete( if err != nil { return reconcile.Result{}, err } - virtualGardenGardenerResourceManager, err := r.newVirtualGardenGardenerResourceManager(garden, secretsManager) + kubeControllerManager, err := r.newKubeControllerManager(log, garden, secretsManager, targetVersion) + if err != nil { + return reconcile.Result{}, err + } + virtualGardenGardenerResourceManager, err := r.newVirtualGardenGardenerResourceManager(secretsManager) if err != nil { return reconcile.Result{}, err } @@ -126,6 +130,10 @@ func (r *Reconciler) delete( Fn: component.OpDestroyAndWait(virtualGardenGardenerResourceManager).Destroy, Dependencies: flow.NewTaskIDs(destroyVirtualGardenGardenerAccess), }) + destroyKubeControllerManager = g.Add(flow.Task{ + Name: "Destroying Kubernetes Controller Manager Server", + Fn: component.OpDestroyAndWait(kubeControllerManager).Destroy, + }) destroyKubeAPIServerService = g.Add(flow.Task{ Name: "Destroying Kubernetes API Server service", Fn: component.OpDestroyAndWait(kubeAPIServerService).Destroy, @@ -151,6 +159,7 @@ func (r *Reconciler) delete( cleanupGenericTokenKubeconfig, destroyVirtualGardenGardenerAccess, destroyVirtualGardenGardenerResourceManager, + destroyKubeControllerManager, destroyKubeAPIServerService, destroyKubeAPIServer, destroyEtcd, diff --git a/pkg/operator/controller/garden/reconciler_reconcile.go b/pkg/operator/controller/garden/reconciler_reconcile.go index 90ee29dfd16..2c4ade70002 100644 --- a/pkg/operator/controller/garden/reconciler_reconcile.go +++ b/pkg/operator/controller/garden/reconciler_reconcile.go @@ -145,7 +145,11 @@ func (r *Reconciler) reconcile( if err != nil { return reconcile.Result{}, err } - virtualGardenGardenerResourceManager, err := r.newVirtualGardenGardenerResourceManager(garden, secretsManager) + kubeControllerManager, err := r.newKubeControllerManager(log, garden, secretsManager, targetVersion) + if err != nil { + return reconcile.Result{}, err + } + virtualGardenGardenerResourceManager, err := r.newVirtualGardenGardenerResourceManager(secretsManager) if err != nil { return reconcile.Result{}, err } @@ -244,6 +248,14 @@ func (r *Reconciler) reconcile( Fn: kubeAPIServer.Wait, Dependencies: flow.NewTaskIDs(deployKubeAPIServer), }) + _ = g.Add(flow.Task{ + Name: "Deploying Kubernetes Controller Manager", + Fn: func(ctx context.Context) error { + kubeControllerManager.SetReplicaCount(1) + return component.OpWait(kubeControllerManager).Deploy(ctx) + }, + Dependencies: flow.NewTaskIDs(waitUntilKubeAPIServerIsReady), + }) deployVirtualGardenGardenerResourceManager = g.Add(flow.Task{ Name: "Deploying gardener-resource-manager for virtual garden", Fn: func(ctx context.Context) error { diff --git a/pkg/operator/webhook/defaulting/handler.go b/pkg/operator/webhook/defaulting/handler.go index ee9e3eb3fe7..19c25f07517 100644 --- a/pkg/operator/webhook/defaulting/handler.go +++ b/pkg/operator/webhook/defaulting/handler.go @@ -37,9 +37,21 @@ func (h *Handler) Default(_ context.Context, obj runtime.Object) error { return fmt.Errorf("expected *operatorv1alpha1.Garden but got %T", obj) } - if kubeAPIServer := garden.Spec.VirtualCluster.Kubernetes.KubeAPIServer; kubeAPIServer != nil { - gardencorev1beta1.SetDefaults_KubeAPIServerConfig(kubeAPIServer.KubeAPIServerConfig) + if garden.Spec.VirtualCluster.Kubernetes.KubeAPIServer == nil { + garden.Spec.VirtualCluster.Kubernetes.KubeAPIServer = &operatorv1alpha1.KubeAPIServerConfig{} } + if garden.Spec.VirtualCluster.Kubernetes.KubeAPIServer.KubeAPIServerConfig == nil { + garden.Spec.VirtualCluster.Kubernetes.KubeAPIServer.KubeAPIServerConfig = &gardencorev1beta1.KubeAPIServerConfig{} + } + gardencorev1beta1.SetDefaults_KubeAPIServerConfig(garden.Spec.VirtualCluster.Kubernetes.KubeAPIServer.KubeAPIServerConfig) + + if garden.Spec.VirtualCluster.Kubernetes.KubeControllerManager == nil { + garden.Spec.VirtualCluster.Kubernetes.KubeControllerManager = &operatorv1alpha1.KubeControllerManagerConfig{} + } + if garden.Spec.VirtualCluster.Kubernetes.KubeControllerManager.KubeControllerManagerConfig == nil { + garden.Spec.VirtualCluster.Kubernetes.KubeControllerManager.KubeControllerManagerConfig = &gardencorev1beta1.KubeControllerManagerConfig{} + } + gardencorev1beta1.SetDefaults_KubeControllerManagerConfig(garden.Spec.VirtualCluster.Kubernetes.KubeControllerManager.KubeControllerManagerConfig) return nil } diff --git a/skaffold-operator.yaml b/skaffold-operator.yaml index f327063059a..872c56b79fb 100644 --- a/skaffold-operator.yaml +++ b/skaffold-operator.yaml @@ -75,6 +75,7 @@ build: - pkg/operation/botanist/component/kubeapiserver - pkg/operation/botanist/component/kubeapiserver/constants - pkg/operation/botanist/component/kubeapiserverexposure + - pkg/operation/botanist/component/kubecontrollermanager - pkg/operation/botanist/component/kubescheduler - pkg/operation/botanist/component/kubestatemetrics - pkg/operation/botanist/component/nginxingress diff --git a/test/e2e/gardener/managedseed/create_rotate_delete.go b/test/e2e/gardener/managedseed/create_rotate_delete.go index e6347d94d57..dfea48f493b 100644 --- a/test/e2e/gardener/managedseed/create_rotate_delete.go +++ b/test/e2e/gardener/managedseed/create_rotate_delete.go @@ -42,7 +42,7 @@ import ( . "github.com/gardener/gardener/pkg/utils/test" e2e "github.com/gardener/gardener/test/e2e/gardener" "github.com/gardener/gardener/test/framework" - "github.com/gardener/gardener/test/utils/shoots/access" + "github.com/gardener/gardener/test/utils/access" ) var parentCtx context.Context diff --git a/test/e2e/gardener/shoot/create_rotate_delete.go b/test/e2e/gardener/shoot/create_rotate_delete.go index 22b9f20518d..1278307df7a 100644 --- a/test/e2e/gardener/shoot/create_rotate_delete.go +++ b/test/e2e/gardener/shoot/create_rotate_delete.go @@ -29,8 +29,8 @@ import ( "github.com/gardener/gardener/pkg/client/kubernetes" e2e "github.com/gardener/gardener/test/e2e/gardener" "github.com/gardener/gardener/test/e2e/gardener/shoot/internal/rotation" + "github.com/gardener/gardener/test/utils/access" rotationutils "github.com/gardener/gardener/test/utils/rotation" - "github.com/gardener/gardener/test/utils/shoots/access" ) var _ = Describe("Shoot Tests", Label("Shoot", "default"), func() { diff --git a/test/e2e/gardener/shoot/create_update_delete.go b/test/e2e/gardener/shoot/create_update_delete.go index bee6e656c6d..78a066fad75 100644 --- a/test/e2e/gardener/shoot/create_update_delete.go +++ b/test/e2e/gardener/shoot/create_update_delete.go @@ -31,7 +31,7 @@ import ( "github.com/gardener/gardener/pkg/utils" e2e "github.com/gardener/gardener/test/e2e/gardener" "github.com/gardener/gardener/test/framework" - "github.com/gardener/gardener/test/utils/shoots/access" + "github.com/gardener/gardener/test/utils/access" shootupdatesuite "github.com/gardener/gardener/test/utils/shoots/update" ) diff --git a/test/e2e/gardener/shoot/internal/node/node.go b/test/e2e/gardener/shoot/internal/node/node.go index 78f30ffcd07..19ed25a5a59 100644 --- a/test/e2e/gardener/shoot/internal/node/node.go +++ b/test/e2e/gardener/shoot/internal/node/node.go @@ -34,7 +34,7 @@ import ( "github.com/gardener/gardener/pkg/utils" "github.com/gardener/gardener/pkg/utils/managedresources" "github.com/gardener/gardener/test/framework" - "github.com/gardener/gardener/test/utils/shoots/access" + "github.com/gardener/gardener/test/utils/access" ) const nodeCriticalDaemonSetName = "e2e-test-node-critical" diff --git a/test/e2e/gardener/shoot/internal/rotation/shoot_access.go b/test/e2e/gardener/shoot/internal/rotation/shoot_access.go index ea68c62efbe..a359980a913 100644 --- a/test/e2e/gardener/shoot/internal/rotation/shoot_access.go +++ b/test/e2e/gardener/shoot/internal/rotation/shoot_access.go @@ -23,14 +23,14 @@ import ( "github.com/gardener/gardener/pkg/client/kubernetes" "github.com/gardener/gardener/test/framework" - "github.com/gardener/gardener/test/utils/shoots/access" + "github.com/gardener/gardener/test/utils/access" ) type clients struct { staticToken, adminKubeconfig, clientCert, serviceAccountDynamic, serviceAccountStatic kubernetes.Interface } -// ShootAccessVerifier uses the static token and admin kubeconfig to access the Shoot. +// ShootAccessVerifier uses the various access methods to access the Shoot. type ShootAccessVerifier struct { *framework.ShootCreationFramework @@ -61,7 +61,7 @@ func (v *ShootAccessVerifier) Before(ctx context.Context) { By("Request new client certificate and using it to access shoot") Eventually(func(g Gomega) { - shootClient, err := access.CreateShootClientFromCSR(ctx, v.clientsBefore.adminKubeconfig, "e2e-rotate-csr-before") + shootClient, err := access.CreateTargetClientFromCSR(ctx, v.clientsBefore.adminKubeconfig, "e2e-rotate-csr-before") g.Expect(err).NotTo(HaveOccurred()) g.Expect(shootClient.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed()) @@ -71,7 +71,7 @@ func (v *ShootAccessVerifier) Before(ctx context.Context) { By("Request new dynamic token for a ServiceAccount and using it to access shoot") Eventually(func(g Gomega) { - shootClient, err := access.CreateShootClientFromDynamicServiceAccountToken(ctx, v.clientsBefore.adminKubeconfig, "e2e-rotate-sa-dynamic-before") + shootClient, err := access.CreateTargetClientFromDynamicServiceAccountToken(ctx, v.clientsBefore.adminKubeconfig, "e2e-rotate-sa-dynamic-before") g.Expect(err).NotTo(HaveOccurred()) g.Expect(shootClient.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed()) @@ -142,7 +142,7 @@ func (v *ShootAccessVerifier) AfterPrepared(ctx context.Context) { By("Request new client certificate and using it to access shoot") Eventually(func(g Gomega) { - shootClient, err := access.CreateShootClientFromCSR(ctx, v.clientsPrepared.adminKubeconfig, "e2e-rotate-csr-prepared") + shootClient, err := access.CreateTargetClientFromCSR(ctx, v.clientsPrepared.adminKubeconfig, "e2e-rotate-csr-prepared") g.Expect(err).NotTo(HaveOccurred()) g.Expect(shootClient.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed()) @@ -152,7 +152,7 @@ func (v *ShootAccessVerifier) AfterPrepared(ctx context.Context) { By("Request new dynamic token for a ServiceAccount and using it to access shoot") Eventually(func(g Gomega) { - shootClient, err := access.CreateShootClientFromDynamicServiceAccountToken(ctx, v.clientsPrepared.adminKubeconfig, "e2e-rotate-sa-dynamic-prepared") + shootClient, err := access.CreateTargetClientFromDynamicServiceAccountToken(ctx, v.clientsPrepared.adminKubeconfig, "e2e-rotate-sa-dynamic-prepared") g.Expect(err).NotTo(HaveOccurred()) g.Expect(shootClient.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed()) @@ -248,7 +248,7 @@ func (v *ShootAccessVerifier) AfterCompleted(ctx context.Context) { By("Request new client certificate and using it to access shoot") Eventually(func(g Gomega) { - shootClient, err := access.CreateShootClientFromCSR(ctx, v.clientsAfter.adminKubeconfig, "e2e-rotate-csr-after") + shootClient, err := access.CreateTargetClientFromCSR(ctx, v.clientsAfter.adminKubeconfig, "e2e-rotate-csr-after") g.Expect(err).NotTo(HaveOccurred()) g.Expect(shootClient.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed()) @@ -258,7 +258,7 @@ func (v *ShootAccessVerifier) AfterCompleted(ctx context.Context) { By("Request new dynamic token for a ServiceAccount and using it to access shoot") Eventually(func(g Gomega) { - shootClient, err := access.CreateShootClientFromDynamicServiceAccountToken(ctx, v.clientsAfter.adminKubeconfig, "e2e-rotate-sa-dynamic-after") + shootClient, err := access.CreateTargetClientFromDynamicServiceAccountToken(ctx, v.clientsAfter.adminKubeconfig, "e2e-rotate-sa-dynamic-after") g.Expect(err).NotTo(HaveOccurred()) g.Expect(shootClient.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed()) @@ -307,7 +307,7 @@ func (v *ShootAccessVerifier) Cleanup(ctx context.Context) { g.Expect(access.CleanupObjectsFromDynamicServiceAccountTokenAccess(ctx, shootClient)).To(Succeed()) }).Should(Succeed()) - By("Clean up objects in shoot from dynamic ServiceAccount token access") + By("Clean up objects in shoot from static ServiceAccount token access") Eventually(func(g Gomega) { g.Expect(access.CleanupObjectsFromStaticServiceAccountTokenAccess(ctx, shootClient)).To(Succeed()) }).Should(Succeed()) diff --git a/test/e2e/operator/garden/create_delete.go b/test/e2e/operator/garden/create_delete.go index b379d8574c6..da8e94031b0 100644 --- a/test/e2e/operator/garden/create_delete.go +++ b/test/e2e/operator/garden/create_delete.go @@ -90,6 +90,7 @@ var _ = Describe("Garden Tests", Label("Garden", "default"), func() { healthyManagedResource("vpa"), healthyManagedResource("etcd-druid"), healthyManagedResource("kube-state-metrics"), + healthyManagedResource("shoot-core-kube-controller-manager"), healthyManagedResource("shoot-core-gardener-resource-manager"), healthyManagedResource("shoot-core-gardeneraccess"), )) diff --git a/test/e2e/operator/garden/create_rotate_delete.go b/test/e2e/operator/garden/create_rotate_delete.go index 21fcb39101c..7bffa6c2c13 100644 --- a/test/e2e/operator/garden/create_rotate_delete.go +++ b/test/e2e/operator/garden/create_rotate_delete.go @@ -84,6 +84,7 @@ var _ = Describe("Garden Tests", Label("Garden", "default"), func() { &rotationutils.SecretEncryptionVerifier{NewTargetClientFunc: func() (kubernetes.Interface, error) { return kubernetes.NewClientFromSecret(ctx, runtimeClient, namespace, "gardener", kubernetes.WithDisabledCachedClient()) }}, + &rotation.VirtualGardenAccessVerifier{RuntimeClient: runtimeClient, Namespace: namespace}, } DeferCleanup(func() { diff --git a/test/e2e/operator/garden/internal/rotation/virtual_garden_access.go b/test/e2e/operator/garden/internal/rotation/virtual_garden_access.go new file mode 100644 index 00000000000..f2cf42ef4ff --- /dev/null +++ b/test/e2e/operator/garden/internal/rotation/virtual_garden_access.go @@ -0,0 +1,172 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rotation + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/gardener/gardener/pkg/client/kubernetes" + "github.com/gardener/gardener/test/utils/access" +) + +type clients struct { + accessSecret, clientCert, serviceAccountDynamic kubernetes.Interface +} + +// VirtualGardenAccessVerifier uses the various access methods to access the virtual garden. +type VirtualGardenAccessVerifier struct { + RuntimeClient client.Client + Namespace string + + clientsBefore, clientsPrepared, clientsAfter clients +} + +// Before is called before the rotation is started. +func (v *VirtualGardenAccessVerifier) Before(ctx context.Context) { + var err error + v.clientsBefore.accessSecret, err = kubernetes.NewClientFromSecret(ctx, v.RuntimeClient, v.Namespace, "gardener", kubernetes.WithDisabledCachedClient()) + Expect(err).NotTo(HaveOccurred()) + + By("Request new client certificate and using it to access virtual garden") + Eventually(func(g Gomega) { + virtualGardenClient, err := access.CreateTargetClientFromCSR(ctx, v.clientsBefore.accessSecret, "e2e-rotate-csr-before") + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(virtualGardenClient.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed()) + + v.clientsBefore.clientCert = virtualGardenClient + }).Should(Succeed()) + + By("Request new dynamic token for a ServiceAccount and using it to access target cluster") + Eventually(func(g Gomega) { + virtualGardenClient, err := access.CreateTargetClientFromDynamicServiceAccountToken(ctx, v.clientsBefore.accessSecret, "e2e-rotate-sa-dynamic-before") + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(virtualGardenClient.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed()) + + v.clientsBefore.serviceAccountDynamic = virtualGardenClient + }).Should(Succeed()) +} + +// ExpectPreparingStatus is called while waiting for the Preparing status. +func (v *VirtualGardenAccessVerifier) ExpectPreparingStatus(g Gomega) {} + +// AfterPrepared is called when the Shoot is in Prepared status. +func (v *VirtualGardenAccessVerifier) AfterPrepared(ctx context.Context) { + By("Use client certificate from before rotation to access target cluster") + Eventually(func(g Gomega) { + g.Expect(v.clientsBefore.clientCert.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed()) + }).Should(Succeed()) + + By("Use dynamic ServiceAccount token from before rotation to access target cluster") + Eventually(func(g Gomega) { + g.Expect(v.clientsBefore.serviceAccountDynamic.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed()) + }).Should(Succeed()) + + var err error + v.clientsPrepared.accessSecret, err = kubernetes.NewClientFromSecret(ctx, v.RuntimeClient, v.Namespace, "gardener", kubernetes.WithDisabledCachedClient()) + Expect(err).NotTo(HaveOccurred()) + + By("Request new client certificate and using it to access target cluster") + Eventually(func(g Gomega) { + virtualGardenClient, err := access.CreateTargetClientFromCSR(ctx, v.clientsPrepared.accessSecret, "e2e-rotate-csr-prepared") + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(virtualGardenClient.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed()) + + v.clientsPrepared.clientCert = virtualGardenClient + }).Should(Succeed()) + + By("Request new dynamic token for a ServiceAccount and using it to access target cluster") + Eventually(func(g Gomega) { + virtualGardenClient, err := access.CreateTargetClientFromDynamicServiceAccountToken(ctx, v.clientsPrepared.accessSecret, "e2e-rotate-sa-dynamic-prepared") + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(virtualGardenClient.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed()) + + v.clientsPrepared.serviceAccountDynamic = virtualGardenClient + }).Should(Succeed()) +} + +// ExpectCompletingStatus is called while waiting for the Completing status. +func (v *VirtualGardenAccessVerifier) ExpectCompletingStatus(g Gomega) {} + +// AfterCompleted is called when the Shoot is in Completed status. +func (v *VirtualGardenAccessVerifier) AfterCompleted(ctx context.Context) { + By("Use client certificate from before rotation to access target cluster") + Consistently(func(g Gomega) { + g.Expect(v.clientsBefore.clientCert.Client().List(ctx, &corev1.NamespaceList{})).NotTo(Succeed()) + }).Should(Succeed()) + + By("Use dynamic ServiceAccount token from before rotation to access target cluster") + Consistently(func(g Gomega) { + g.Expect(v.clientsBefore.serviceAccountDynamic.Client().List(ctx, &corev1.NamespaceList{})).NotTo(Succeed()) + }).Should(Succeed()) + + By("Use client certificate from after preparation to access target cluster") + Eventually(func(g Gomega) { + g.Expect(v.clientsPrepared.clientCert.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed()) + }).Should(Succeed()) + + By("Use dynamic ServiceAccount token from after preparation to access target cluster") + Eventually(func(g Gomega) { + g.Expect(v.clientsPrepared.serviceAccountDynamic.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed()) + }).Should(Succeed()) + + var err error + v.clientsAfter.accessSecret, err = kubernetes.NewClientFromSecret(ctx, v.RuntimeClient, v.Namespace, "gardener", kubernetes.WithDisabledCachedClient()) + Expect(err).NotTo(HaveOccurred()) + + By("Request new client certificate and using it to access target cluster") + Eventually(func(g Gomega) { + virtualGardenClient, err := access.CreateTargetClientFromCSR(ctx, v.clientsAfter.accessSecret, "e2e-rotate-csr-after") + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(virtualGardenClient.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed()) + + v.clientsAfter.clientCert = virtualGardenClient + }).Should(Succeed()) + + By("Request new dynamic token for a ServiceAccount and using it to access target cluster") + Eventually(func(g Gomega) { + virtualGardenClient, err := access.CreateTargetClientFromDynamicServiceAccountToken(ctx, v.clientsAfter.accessSecret, "e2e-rotate-sa-dynamic-after") + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(virtualGardenClient.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed()) + + v.clientsAfter.serviceAccountDynamic = virtualGardenClient + }).Should(Succeed()) +} + +// Cleanup is passed to ginkgo.DeferCleanup. +func (v *VirtualGardenAccessVerifier) Cleanup(ctx context.Context) { + virtualGardenClient, err := kubernetes.NewClientFromSecret(ctx, v.RuntimeClient, v.Namespace, "gardener", kubernetes.WithDisabledCachedClient()) + Expect(err).NotTo(HaveOccurred()) + + By("Clean up objects in virtual garden from client certificate access") + Eventually(func(g Gomega) { + g.Expect(access.CleanupObjectsFromCSRAccess(ctx, virtualGardenClient)).To(Succeed()) + }).Should(Succeed()) + + By("Clean up objects in shoot from dynamic ServiceAccount token access") + Eventually(func(g Gomega) { + g.Expect(access.CleanupObjectsFromDynamicServiceAccountTokenAccess(ctx, virtualGardenClient)).To(Succeed()) + }).Should(Succeed()) +} diff --git a/test/framework/k8s_utils.go b/test/framework/k8s_utils.go index 9742a7c7b3b..934012c865a 100644 --- a/test/framework/k8s_utils.go +++ b/test/framework/k8s_utils.go @@ -39,7 +39,7 @@ import ( kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/gardener/gardener/pkg/utils/kubernetes/health" "github.com/gardener/gardener/pkg/utils/retry" - "github.com/gardener/gardener/test/utils/shoots/access" + "github.com/gardener/gardener/test/utils/access" ) // WaitUntilDaemonSetIsRunning waits until the daemon set with is running diff --git a/test/framework/shootframework.go b/test/framework/shootframework.go index 27e8318bf48..e360f62bc60 100644 --- a/test/framework/shootframework.go +++ b/test/framework/shootframework.go @@ -35,7 +35,7 @@ import ( "github.com/gardener/gardener/pkg/client/kubernetes" kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/gardener/gardener/pkg/utils/retry" - "github.com/gardener/gardener/test/utils/shoots/access" + "github.com/gardener/gardener/test/utils/access" ) var shootCfg *ShootConfig diff --git a/test/framework/shootmigrationtest.go b/test/framework/shootmigrationtest.go index 8d909944460..b80a0006661 100644 --- a/test/framework/shootmigrationtest.go +++ b/test/framework/shootmigrationtest.go @@ -40,7 +40,7 @@ import ( "github.com/gardener/gardener/pkg/client/kubernetes" "github.com/gardener/gardener/pkg/utils" secretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager" - "github.com/gardener/gardener/test/utils/shoots/access" + "github.com/gardener/gardener/test/utils/access" ) // ShootMigrationTest represents a shoot migration test. diff --git a/test/integration/operator/garden/garden_test.go b/test/integration/operator/garden/garden_test.go index c0e28881c23..187f060402d 100644 --- a/test/integration/operator/garden/garden_test.go +++ b/test/integration/operator/garden/garden_test.go @@ -542,6 +542,47 @@ var _ = Describe("Garden controller tests", func() { g.Expect(testClient.Patch(ctx, secret, patch)).To(Succeed()) }).Should(Succeed()) + By("Ensure virtual-garden-kube-controller-manager was deployed") + Eventually(func(g Gomega) []appsv1.Deployment { + deploymentList := &appsv1.DeploymentList{} + g.Expect(testClient.List(ctx, deploymentList, client.InNamespace(testNamespace.Name))).To(Succeed()) + return deploymentList.Items + }).Should(ContainElements( + MatchFields(IgnoreExtras, Fields{"ObjectMeta": MatchFields(IgnoreExtras, Fields{"Name": Equal("virtual-garden-kube-controller-manager")})}), + )) + + // The garden controller waits for the virtual-garden-kube-controller-manager Deployment to be healthy, so let's fake this here. + By("Patch virtual-garden-kube-controller-manager deployment to report healthiness") + Eventually(func(g Gomega) { + deployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "virtual-garden-kube-controller-manager", Namespace: testNamespace.Name}} + g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(deployment), deployment)).To(Succeed()) + + podList := &corev1.PodList{} + g.Expect(testClient.List(ctx, podList, client.InNamespace(testNamespace.Name), client.MatchingLabels(map[string]string{"app": "kubernetes", "role": "controller-manager"}))).To(Succeed()) + + if desiredReplicas := int(pointer.Int32Deref(deployment.Spec.Replicas, 1)); len(podList.Items) != desiredReplicas { + g.Expect(testClient.DeleteAllOf(ctx, &corev1.Pod{}, client.InNamespace(testNamespace.Name), client.MatchingLabels(map[string]string{"app": "kubernetes", "role": "controller-manager"}))).To(Succeed()) + for i := 0; i < desiredReplicas; i++ { + g.Expect(testClient.Create(ctx, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("virtual-garden-kube-controller-manager-%d", i), + Namespace: testNamespace.Name, + Labels: map[string]string{"app": "kubernetes", "role": "controller-manager"}, + }, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "app", Image: "app"}}}, + })).To(Succeed(), fmt.Sprintf("create virtual-garden-kube-apiserver pod number %d", i)) + } + } + + patch := client.MergeFrom(deployment.DeepCopy()) + deployment.Status.ObservedGeneration = deployment.Generation + deployment.Status.Conditions = []appsv1.DeploymentCondition{ + {Type: appsv1.DeploymentAvailable, Status: corev1.ConditionTrue}, + {Type: appsv1.DeploymentProgressing, Status: corev1.ConditionTrue, Reason: "NewReplicaSetAvailable"}, + } + g.Expect(testClient.Status().Patch(ctx, deployment, patch)).To(Succeed()) + }).Should(Succeed()) + By("Wait for Reconciled condition to be set to True") Eventually(func(g Gomega) []gardencorev1beta1.Condition { g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(garden), garden)).To(Succeed()) @@ -558,6 +599,7 @@ var _ = Describe("Garden controller tests", func() { return deploymentList.Items }).ShouldNot(ContainElements( MatchFields(IgnoreExtras, Fields{"ObjectMeta": MatchFields(IgnoreExtras, Fields{"Name": Equal("virtual-garden-kube-apiserver")})}), + MatchFields(IgnoreExtras, Fields{"ObjectMeta": MatchFields(IgnoreExtras, Fields{"Name": Equal("virtual-garden-kube-controller-manager")})}), MatchFields(IgnoreExtras, Fields{"ObjectMeta": MatchFields(IgnoreExtras, Fields{"Name": Equal("virtual-garden-gardener-resource-manager")})}), )) diff --git a/test/testmachinery/shoots/vpntunnel/vpntunnel.go b/test/testmachinery/shoots/vpntunnel/vpntunnel.go index 7d3def90939..f7ddfe44856 100644 --- a/test/testmachinery/shoots/vpntunnel/vpntunnel.go +++ b/test/testmachinery/shoots/vpntunnel/vpntunnel.go @@ -36,7 +36,7 @@ import ( kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/gardener/gardener/test/framework" "github.com/gardener/gardener/test/framework/resources/templates" - "github.com/gardener/gardener/test/utils/shoots/access" + "github.com/gardener/gardener/test/utils/access" ) const ( diff --git a/test/utils/shoots/access/adminkubeconfig.go b/test/utils/access/adminkubeconfig.go similarity index 100% rename from test/utils/shoots/access/adminkubeconfig.go rename to test/utils/access/adminkubeconfig.go diff --git a/test/utils/shoots/access/csr.go b/test/utils/access/csr.go similarity index 76% rename from test/utils/shoots/access/csr.go rename to test/utils/access/csr.go index 400633876df..b588c41074d 100644 --- a/test/utils/shoots/access/csr.go +++ b/test/utils/access/csr.go @@ -40,9 +40,9 @@ import ( // labelsE2ETestCSRAccess is the set of labels added to all CSRs and ClusterRoleBindings for easy cleanup. var labelsE2ETestCSRAccess = map[string]string{"e2e-test": "csr-access"} -// CreateShootClientFromCSR creates and approves a CSR in the shoot and creates a new shoot client from it. +// CreateTargetClientFromCSR creates and approves a CSR in the shoot and creates a new target client from it. // You should call CleanupObjectsFromCSRAccess to clean up the objects created by this function. -func CreateShootClientFromCSR(ctx context.Context, shootClient kubernetes.Interface, commonName string) (kubernetes.Interface, error) { +func CreateTargetClientFromCSR(ctx context.Context, targetClient kubernetes.Interface, commonName string) (kubernetes.Interface, error) { // use fake key to avoid building complex retry/update logic privateKey, err := secretsutils.FakeGenerateKey(rand.Reader, 4096) if err != nil { @@ -55,7 +55,7 @@ func CreateShootClientFromCSR(ctx context.Context, shootClient kubernetes.Interf } reqName, reqUID, err := csrutil.RequestCertificate( - shootClient.Kubernetes(), + targetClient.Kubernetes(), csrData, commonName, certificatesv1.KubeAPIServerClientSignerName, @@ -72,18 +72,18 @@ func CreateShootClientFromCSR(ctx context.Context, shootClient kubernetes.Interf } csr := &certificatesv1.CertificateSigningRequest{} - if err = shootClient.Client().Get(ctx, client.ObjectKey{Name: reqName}, csr); err != nil { + if err = targetClient.Client().Get(ctx, client.ObjectKey{Name: reqName}, csr); err != nil { return nil, err } patch := client.MergeFrom(csr.DeepCopy()) csr.Labels = utils.MergeStringMaps(csr.Labels, labelsE2ETestCSRAccess) - if err = shootClient.Client().Patch(ctx, csr, patch); err != nil { + if err = targetClient.Client().Patch(ctx, csr, patch); err != nil { return nil, err } clusterRoleBinding := &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: commonName}} - if _, err = controllerutils.GetAndCreateOrMergePatch(ctx, shootClient.Client(), clusterRoleBinding, func() error { + if _, err = controllerutils.GetAndCreateOrMergePatch(ctx, targetClient.Client(), clusterRoleBinding, func() error { clusterRoleBinding.Labels = utils.MergeStringMaps(clusterRoleBinding.Labels, labelsE2ETestCSRAccess) clusterRoleBinding.RoleRef = rbacv1.RoleRef{ APIGroup: rbacv1.GroupName, @@ -114,17 +114,17 @@ func CreateShootClientFromCSR(ctx context.Context, shootClient kubernetes.Interf Message: "Auto approving test CertificateSigningRequest", Status: corev1.ConditionTrue, }) - if err := shootClient.Client().SubResource("approval").Update(ctx, csr); err != nil { + if err := targetClient.Client().SubResource("approval").Update(ctx, csr); err != nil { return nil, err } } - certData, err := csrutil.WaitForCertificate(ctx, shootClient.Kubernetes(), reqName, reqUID) + certData, err := csrutil.WaitForCertificate(ctx, targetClient.Kubernetes(), reqName, reqUID) if err != nil { return nil, err } - r := shootClient.RESTConfig() + r := targetClient.RESTConfig() restConfig := &rest.Config{ Host: r.Host, TLSClientConfig: rest.TLSClientConfig{ @@ -137,11 +137,11 @@ func CreateShootClientFromCSR(ctx context.Context, shootClient kubernetes.Interf return kubernetes.NewWithConfig(kubernetes.WithRESTConfig(restConfig), kubernetes.WithDisabledCachedClient()) } -// CleanupObjectsFromCSRAccess cleans up all objects in the shoot created by all calls to CreateShootClientFromCSR. -func CleanupObjectsFromCSRAccess(ctx context.Context, shootClient kubernetes.Interface) error { +// CleanupObjectsFromCSRAccess cleans up all objects in the target created by all calls to CreateTargetClientFromCSR. +func CleanupObjectsFromCSRAccess(ctx context.Context, targetClient kubernetes.Interface) error { return flow.Parallel(func(ctx context.Context) error { - return shootClient.Client().DeleteAllOf(ctx, &certificatesv1.CertificateSigningRequest{}, client.MatchingLabels(labelsE2ETestCSRAccess)) + return targetClient.Client().DeleteAllOf(ctx, &certificatesv1.CertificateSigningRequest{}, client.MatchingLabels(labelsE2ETestCSRAccess)) }, func(ctx context.Context) error { - return shootClient.Client().DeleteAllOf(ctx, &rbacv1.ClusterRoleBinding{}, client.MatchingLabels(labelsE2ETestCSRAccess)) + return targetClient.Client().DeleteAllOf(ctx, &rbacv1.ClusterRoleBinding{}, client.MatchingLabels(labelsE2ETestCSRAccess)) })(ctx) } diff --git a/test/utils/shoots/access/serviceaccount.go b/test/utils/access/serviceaccount.go similarity index 79% rename from test/utils/shoots/access/serviceaccount.go rename to test/utils/access/serviceaccount.go index 4da832005d0..9fab2e917ed 100644 --- a/test/utils/shoots/access/serviceaccount.go +++ b/test/utils/access/serviceaccount.go @@ -41,11 +41,11 @@ const namespaceE2ETestServiceAccountTokenAccess = metav1.NamespaceDefault // ClusterRoleBindings for easy cleanup. var labelsE2ETestDynamicServiceAccountTokenAccess = map[string]string{"e2e-test": "serviceaccount-dynamic-access"} -// CreateShootClientFromDynamicServiceAccountToken creates a ServiceAccount, uses the kube-apiserver's TokenRequest API -// to request a token for it, and then creates a new shoot client from it. +// CreateTargetClientFromDynamicServiceAccountToken creates a ServiceAccount, uses the kube-apiserver's TokenRequest API +// to request a token for it, and then creates a new target client from it. // You should call CleanupObjectsFromDynamicServiceAccountTokenAccess to clean up the objects created by this function. -func CreateShootClientFromDynamicServiceAccountToken(ctx context.Context, shootClient kubernetes.Interface, name string) (kubernetes.Interface, error) { - return createShootClientFromServiceAccount(ctx, shootClient, name, labelsE2ETestDynamicServiceAccountTokenAccess, func(serviceAccount *corev1.ServiceAccount) (string, error) { +func CreateTargetClientFromDynamicServiceAccountToken(ctx context.Context, targetClient kubernetes.Interface, name string) (kubernetes.Interface, error) { + return createTargetClientFromServiceAccount(ctx, targetClient, name, labelsE2ETestDynamicServiceAccountTokenAccess, func(serviceAccount *corev1.ServiceAccount) (string, error) { tokenRequest := &authenticationv1.TokenRequest{ Spec: authenticationv1.TokenRequestSpec{ Audiences: []string{v1beta1constants.GardenerAudience}, @@ -53,7 +53,7 @@ func CreateShootClientFromDynamicServiceAccountToken(ctx context.Context, shootC }, } - if err := shootClient.Client().SubResource("token").Create(ctx, serviceAccount, tokenRequest); err != nil { + if err := targetClient.Client().SubResource("token").Create(ctx, serviceAccount, tokenRequest); err != nil { return "", err } @@ -61,13 +61,13 @@ func CreateShootClientFromDynamicServiceAccountToken(ctx context.Context, shootC }) } -// CleanupObjectsFromDynamicServiceAccountTokenAccess cleans up all objects in the shoot created by all calls to -// CreateShootClientFromDynamicServiceAccountToken. -func CleanupObjectsFromDynamicServiceAccountTokenAccess(ctx context.Context, shootClient kubernetes.Interface) error { +// CleanupObjectsFromDynamicServiceAccountTokenAccess cleans up all objects in the target created by all calls to +// CreateTargetClientFromDynamicServiceAccountToken. +func CleanupObjectsFromDynamicServiceAccountTokenAccess(ctx context.Context, targetClient kubernetes.Interface) error { return flow.Parallel(func(ctx context.Context) error { - return shootClient.Client().DeleteAllOf(ctx, &corev1.ServiceAccount{}, client.InNamespace(namespaceE2ETestServiceAccountTokenAccess), client.MatchingLabels(labelsE2ETestDynamicServiceAccountTokenAccess)) + return targetClient.Client().DeleteAllOf(ctx, &corev1.ServiceAccount{}, client.InNamespace(namespaceE2ETestServiceAccountTokenAccess), client.MatchingLabels(labelsE2ETestDynamicServiceAccountTokenAccess)) }, func(ctx context.Context) error { - return shootClient.Client().DeleteAllOf(ctx, &rbacv1.ClusterRoleBinding{}, client.MatchingLabels(labelsE2ETestDynamicServiceAccountTokenAccess)) + return targetClient.Client().DeleteAllOf(ctx, &rbacv1.ClusterRoleBinding{}, client.MatchingLabels(labelsE2ETestDynamicServiceAccountTokenAccess)) })(ctx) } @@ -79,7 +79,7 @@ var labelsE2ETestStaticServiceAccountToken = map[string]string{"e2e-test": "serv // by kube-controller-manager), and then creates a new shoot client from it. // You should call CleanupObjectsFromStaticServiceAccountTokenAccess to clean up the objects created by this function. func CreateShootClientFromStaticServiceAccountToken(ctx context.Context, shootClient kubernetes.Interface, name string) (kubernetes.Interface, error) { - return createShootClientFromServiceAccount(ctx, shootClient, name, labelsE2ETestStaticServiceAccountToken, func(serviceAccount *corev1.ServiceAccount) (string, error) { + return createTargetClientFromServiceAccount(ctx, shootClient, name, labelsE2ETestStaticServiceAccountToken, func(serviceAccount *corev1.ServiceAccount) (string, error) { secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: serviceAccount.Namespace}} if _, err := controllerutils.GetAndCreateOrMergePatch(ctx, shootClient.Client(), secret, func() error { secret.Labels = utils.MergeStringMaps(secret.Labels, labelsE2ETestStaticServiceAccountToken) @@ -123,9 +123,9 @@ func CleanupObjectsFromStaticServiceAccountTokenAccess(ctx context.Context, shoo })(ctx) } -func createShootClientFromServiceAccount( +func createTargetClientFromServiceAccount( ctx context.Context, - shootClient kubernetes.Interface, + targetClient kubernetes.Interface, name string, labels map[string]string, getTokenForServiceAccount func(*corev1.ServiceAccount) (string, error), @@ -134,7 +134,7 @@ func createShootClientFromServiceAccount( error, ) { serviceAccount := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespaceE2ETestServiceAccountTokenAccess}} - if _, err := controllerutils.GetAndCreateOrMergePatch(ctx, shootClient.Client(), serviceAccount, func() error { + if _, err := controllerutils.GetAndCreateOrMergePatch(ctx, targetClient.Client(), serviceAccount, func() error { serviceAccount.Labels = utils.MergeStringMaps(serviceAccount.Labels, labels) return nil }); err != nil { @@ -142,7 +142,7 @@ func createShootClientFromServiceAccount( } clusterRoleBinding := &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: name}} - if _, err := controllerutils.GetAndCreateOrMergePatch(ctx, shootClient.Client(), clusterRoleBinding, func() error { + if _, err := controllerutils.GetAndCreateOrMergePatch(ctx, targetClient.Client(), clusterRoleBinding, func() error { clusterRoleBinding.Labels = utils.MergeStringMaps(serviceAccount.Labels, labels) clusterRoleBinding.RoleRef = rbacv1.RoleRef{ APIGroup: rbacv1.GroupName, @@ -164,7 +164,7 @@ func createShootClientFromServiceAccount( return nil, err } - r := shootClient.RESTConfig() + r := targetClient.RESTConfig() restConfig := &rest.Config{ Host: r.Host, TLSClientConfig: rest.TLSClientConfig{CAData: r.CAData}, diff --git a/test/utils/shoots/access/statictokenkubeconfig.go b/test/utils/access/statictokenkubeconfig.go similarity index 100% rename from test/utils/shoots/access/statictokenkubeconfig.go rename to test/utils/access/statictokenkubeconfig.go diff --git a/test/utils/shoots/update/update.go b/test/utils/shoots/update/update.go index 1f1fd4c6e44..3eb0aa7f6a3 100644 --- a/test/utils/shoots/update/update.go +++ b/test/utils/shoots/update/update.go @@ -38,7 +38,7 @@ import ( "github.com/gardener/gardener/pkg/client/kubernetes" versionutils "github.com/gardener/gardener/pkg/utils/version" "github.com/gardener/gardener/test/framework" - "github.com/gardener/gardener/test/utils/shoots/access" + "github.com/gardener/gardener/test/utils/access" "github.com/gardener/gardener/test/utils/shoots/update/highavailability" )