From a57427e0d8c80c51ed021db01fdaf35d1e15e31d Mon Sep 17 00:00:00 2001
From: Rafael Franzke
+(Appears on:
+Kubernetes)
+
+ KubeControllerManagerConfig contains configuration settings for the kube-controller-manager.KubeControllerManagerConfig
+
+
Field | +Description | +
---|---|
+KubeControllerManagerConfig
+
+github.com/gardener/gardener/pkg/apis/core/v1beta1.KubeControllerManagerConfig
+
+ |
+
+
+(Members of KubeControllerManagerConfig contains all configuration values not specific to the virtual garden cluster. + |
+
+certificateSigningDuration
+
+
+Kubernetes meta/v1.Duration
+
+
+ |
+
+(Optional)
+ CertificateSigningDuration is the maximum length of duration signed certificates will be given. Individual CSRs
+may request shorter certs by setting |
+
@@ -977,6 +1026,20 @@ KubeAPIServerConfig
kubeControllerManager
+
+
+KubeControllerManagerConfig
+
+
+KubeControllerManager contains configuration settings for the kube-controller-manager.
+version
string
diff --git a/docs/concepts/operator.md b/docs/concepts/operator.md
index 809f3361918..831d39a2a7f 100644
--- a/docs/concepts/operator.md
+++ b/docs/concepts/operator.md
@@ -89,11 +89,7 @@ Please refer to [this document](../usage/shoot_credentials_rotation.md#gardener-
- ETCD encryption key
- `ServiceAccount` token signing key
-⚠️ Since `kube-controller-manager` is not yet deployed by `gardener-operator`, rotation of static `ServiceAccount` secrets is not supported and must be performed manually after the `Garden` has reached `Prepared` phase before completing the rotation.
-
-⚠️ Rotation of the static kubeconfig (which is enabled unconditionally) is not support for now.
-The reason is that it such static kubeconfig will be disabled without configuration option in the near future.
-Instead, we'll implement an approach similar to the [`adminkubeconfig` subresource on `Shoot`s](../usage/shoot_access.md#shootsadminkubeconfig-subresource) which can be used to retrieve a temporary kubeconfig for the virtual garden cluster.
+⚠️ Rotation of static `ServiceAccount` secrets is not supported since the `kube-controller-manager` does not enable the `serviceaccount-token` controller.
## Local Development
@@ -110,8 +106,7 @@ This command sets up a new KinD cluster named `gardener-local` and stores the ku
> It might be helpful to copy this file to `$HOME/.kube/config`, since you will need to target this KinD cluster multiple times.
Alternatively, make sure to set your `KUBECONFIG` environment variable to `./example/gardener-local/kind/operator/kubeconfig` for all future steps via `export KUBECONFIG=example/gardener-local/kind/operator/kubeconfig`.
-All of the following steps assume that you are using this kubeconfig.
-
+All the following steps assume that you are using this kubeconfig.
### Setting Up Gardener Operator
@@ -177,10 +172,12 @@ EOF
To access the virtual garden, you can acquire a `kubeconfig` by
```shell
-kubectl -n garden get secret -l name=user-kubeconfig -o jsonpath={..data.kubeconfig} | base64 -d > /tmp/virtual-garden-kubeconfig
+kubectl -n garden get secret gardener -o jsonpath={.data.kubeconfig} | base64 -d > /tmp/virtual-garden-kubeconfig
kubectl --kubeconfig /tmp/virtual-garden-kubeconfig get namespaces
```
+Note that this kubeconfig uses a token that has validity of `12h` only, hence it might expire and causing you to re-download the kubeconfig.
+
### Deleting the `Garden`
```shell
@@ -237,6 +234,8 @@ The virtual garden control plane components are:
- `virtual-garden-etcd-main`
- `virtual-garden-etcd-events`
- `virtual-garden-kube-apiserver`
+- `virtual-garden-kube-controller-manager`
+- `virtual-garden-gardener-resource-manager`
If the `.spec.virtualCluster.controlPlane.highAvailability={}` is set then these components will be deployed in a "highly available" mode.
For ETCD, this means that there will be 3 replicas each.
@@ -246,10 +245,16 @@ The `gardener-resource-manager`'s [HighAvailabilityConfig webhook](resource-mana
> If once set, removing `.spec.virtualCluster.controlPlane.highAvailability` again is not supported.
The `virtual-garden-kube-apiserver` `Deployment` is exposed via a `Service` of type `LoadBalancer` with the same name.
-In the future, we might switch to exposing it via Istio, similar to how the `kube-apiservers` of shoot clusters are exposed.
+In the future, we will switch to exposing it via Istio, similar to how the `kube-apiservers` of shoot clusters are exposed.
Similar to the `Shoot` API, the version of the virtual garden cluster is controlled via `.spec.virtualCluster.kubernetes.version`.
-Likewise, specific configuration for the control plane components can be provided in the same section, e.g. via `.spec.virtualCluster.kubernetes.kubeAPIServer` for the `kube-apiserver`.
+Likewise, specific configuration for the control plane components can be provided in the same section, e.g. via `.spec.virtualCluster.kubernetes.kubeAPIServer` for the `kube-apiserver` or `.spec.virtualCluster.kubernetes.kubeControllerManager` for the `kube-controller-manager`.
+
+The `kube-controller-manager` only runs a very few controllers that are necessary in the scenario of the virtual garden.
+Most prominently, **the `serviceaccount-token` controller is unconditionally disabled**.
+Hence, the usage of static `ServiceAccount` secrets is not supported generally.
+Instead, the [`TokenRequest` API](https://kubernetes.io/docs/reference/kubernetes-api/authentication-resources/token-request-v1/) should be used.
+Third-party components that need to communicate with the virtual cluster can leverage the [`gardener-resource-manager`'s `TokenRequestor` controller](resource-manager.md#tokenrequestor-controller) and the generic kubeconfig, just like it works for `Shoot`s.
For the virtual cluster, it is essential to provide a DNS domain via `.spec.virtualCluster.dns.domain`.
**The respective DNS record is not managed by `gardener-operator` and should be manually created and pointed to the load balancer IP of the `virtual-garden-kube-apiserver` `Service`.**
diff --git a/docs/development/priority-classes.md b/docs/development/priority-classes.md
index d59b2bb5329..fbccb2dfd24 100644
--- a/docs/development/priority-classes.md
+++ b/docs/development/priority-classes.md
@@ -23,8 +23,8 @@ When using the `gardener-operator` for managing the garden runtime and virtual c
|---------------------------------- |-----------|-------------------------------------------------------------------------------------------|
| `gardener-garden-system-critical` | 999999550 | `gardener-operator`, `gardener-resource-manager`, `istio` |
| `gardener-garden-system-500` | 999999500 | `virtual-garden-etcd-events`, `virtual-garden-etcd-main`, `virtual-garden-kube-apiserver` |
-| `gardener-garden-system-400` | 999999400 | |
-| `gardener-garden-system-300` | 999999300 | `vpa-admission-controller`, `etcd-druid` |
+| `gardener-garden-system-400` | 999999400 | `virtual-garden-gardener-resource-manager` |
+| `gardener-garden-system-300` | 999999300 | `virtual-garden-kube-controller-manager`, `vpa-admission-controller`, `etcd-druid` |
| `gardener-garden-system-200` | 999999200 | `vpa-recommender`, `vpa-updater`, `hvpa-controller` |
| `gardener-garden-system-100` | 999999100 | `kube-state-metrics` |
diff --git a/example/operator/10-crd-operator.gardener.cloud_gardens.yaml b/example/operator/10-crd-operator.gardener.cloud_gardens.yaml
index 56a02de2144..58e35d03450 100644
--- a/example/operator/10-crd-operator.gardener.cloud_gardens.yaml
+++ b/example/operator/10-crd-operator.gardener.cloud_gardens.yaml
@@ -765,6 +765,78 @@ spec:
type: array
type: object
type: object
+ kubeControllerManager:
+ description: KubeControllerManager contains configuration
+ settings for the kube-controller-manager.
+ properties:
+ certificateSigningDuration:
+ default: 48h
+ description: CertificateSigningDuration is the maximum
+ length of duration signed certificates will be given.
+ Individual CSRs may request shorter certs by setting
+ `spec.expirationSeconds`.
+ pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$
+ type: string
+ featureGates:
+ additionalProperties:
+ type: boolean
+ description: FeatureGates contains information about enabled
+ feature gates.
+ type: object
+ horizontalPodAutoscaler:
+ description: HorizontalPodAutoscalerConfig contains horizontal
+ pod autoscaler configuration settings for the kube-controller-manager.
+ properties:
+ cpuInitializationPeriod:
+ description: The period after which a ready pod transition
+ is considered to be the first.
+ type: string
+ downscaleStabilization:
+ description: The configurable window at which the
+ controller will choose the highest recommendation
+ for autoscaling.
+ type: string
+ initialReadinessDelay:
+ description: The configurable period at which the
+ horizontal pod autoscaler considers a Pod “not yet
+ ready” given that it’s unready and it has transitioned
+ to unready during that time.
+ type: string
+ syncPeriod:
+ description: The period for syncing the number of
+ pods in horizontal pod autoscaler.
+ type: string
+ tolerance:
+ description: The minimum change (from 1.0) in the
+ desired-to-actual metrics ratio for the horizontal
+ pod autoscaler to consider scaling.
+ type: number
+ type: object
+ nodeCIDRMaskSize:
+ description: NodeCIDRMaskSize defines the mask size for
+ node cidr in cluster (default is 24). This field is
+ immutable.
+ format: int32
+ type: integer
+ nodeMonitorGracePeriod:
+ description: NodeMonitorGracePeriod defines the grace
+ period before an unresponsive node is marked unhealthy.
+ type: string
+ podEvictionTimeout:
+ description: "PodEvictionTimeout defines the grace period
+ for deleting pods on failed nodes. Defaults to 2m. \n
+ Deprecated: The corresponding kube-controller-manager
+ flag `--pod-eviction-timeout` is deprecated in favor
+ of the kube-apiserver flags `--default-not-ready-toleration-seconds`
+ and `--default-unreachable-toleration-seconds`. The
+ `--pod-eviction-timeout` flag does not have effect when
+ the taint besed eviction is enabled. The taint based
+ eviction is beta (enabled by default) since Kubernetes
+ 1.13 and GA since Kubernetes 1.18. Hence, instead of
+ setting this field, set the `spec.kubernetes.kubeAPIServer.defaultNotReadyTolerationSeconds`
+ and `spec.kubernetes.kubeAPIServer.defaultUnreachableTolerationSeconds`."
+ type: string
+ type: object
version:
description: Version is the semantic Kubernetes version to
use for the virtual garden cluster.
diff --git a/example/operator/20-garden.yaml b/example/operator/20-garden.yaml
index c65cc3f8234..3e15b95b21e 100644
--- a/example/operator/20-garden.yaml
+++ b/example/operator/20-garden.yaml
@@ -135,6 +135,10 @@ spec:
# resourcesToStoreInETCDEvents:
# - group: networking.k8s.io
# resources: networkpolicies
+ # kubeControllerManager:
+ # featureGates:
+ # SomeKubernetesFeature: true
+ # certificateSigningDuration: 48h
maintenance:
timeWindow:
begin: 220000+0100
diff --git a/example/operator/doc.go b/example/operator/doc.go
index 454c073cdd9..327eff107df 100644
--- a/example/operator/doc.go
+++ b/example/operator/doc.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-//go:generate ../../hack/generate-crds.sh 10-crd- operator.gardener.cloud
+//go:generate ../../hack/generate-crds.sh 10-crd- -allow-dangerous-types operator.gardener.cloud
//go:generate cp 10-crd-operator.gardener.cloud_gardens.yaml ../../charts/gardener/operator/templates/customresouredefintion.yaml
// Package operator contains example manifests for working on operator.
diff --git a/hack/generate-crds.sh b/hack/generate-crds.sh
index 46a6454ddf2..b119d9a4c29 100755
--- a/hack/generate-crds.sh
+++ b/hack/generate-crds.sh
@@ -35,6 +35,7 @@ fi
output_dir="$(pwd)"
file_name_prefix="$1"
add_deletion_protection_label=false
+crd_options=""
get_group_package () {
case "$1" in
@@ -62,6 +63,15 @@ get_group_package () {
esac
}
+generate_all_groups () {
+ generate_group extensions.gardener.cloud
+ generate_group resources.gardener.cloud
+ generate_group operator.gardener.cloud
+ generate_group druid.gardener.cloud
+ generate_group autoscaling.k8s.io
+ generate_group fluentbit.fluent.io
+}
+
generate_group () {
local group="$1"
echo "Generating CRDs for $group group"
@@ -80,7 +90,7 @@ generate_group () {
rm "$output_dir"/*${group}_*.yaml
fi
- controller-gen crd paths="$package_path" output:crd:dir="$output_dir" output:stdout
+ controller-gen crd"$crd_options" paths="$package_path" output:crd:dir="$output_dir" output:stdout
while IFS= read -r crd; do
crd_out="$output_dir/$file_name_prefix$(basename $crd)"
@@ -113,12 +123,17 @@ if [ -n "${2:-}" ]; then
shift
done
else
- generate_group extensions.gardener.cloud
- generate_group resources.gardener.cloud
- generate_group operator.gardener.cloud
- generate_group druid.gardener.cloud
- generate_group autoscaling.k8s.io
- generate_group fluentbit.fluent.io
+ generate_all_groups
+ fi
+ elif [ "${2}" == "-allow-dangerous-types" ]; then
+ crd_options=":allowDangerousTypes=true"
+ if [ -n "${3:-}" ]; then
+ while [ -n "${3:-}" ] ; do
+ generate_group "$3"
+ shift
+ done
+ else
+ generate_all_groups
fi
else
while [ -n "${2:-}" ] ; do
@@ -127,10 +142,6 @@ if [ -n "${2:-}" ]; then
done
fi
else
- generate_group extensions.gardener.cloud
- generate_group resources.gardener.cloud
- generate_group operator.gardener.cloud
- generate_group druid.gardener.cloud
- generate_group autoscaling.k8s.io
- generate_group fluentbit.fluent.io
+ generate_all_groups
fi
+
diff --git a/pkg/apis/core/v1beta1/defaults.go b/pkg/apis/core/v1beta1/defaults.go
index 074661e4254..edeb46a1857 100644
--- a/pkg/apis/core/v1beta1/defaults.go
+++ b/pkg/apis/core/v1beta1/defaults.go
@@ -160,6 +160,13 @@ func SetDefaults_Shoot(obj *Shoot) {
obj.Spec.Kubernetes.AllowPrivilegedContainers = pointer.Bool(true)
}
+ if obj.Spec.Kubernetes.KubeAPIServer.DefaultNotReadyTolerationSeconds == nil {
+ obj.Spec.Kubernetes.KubeAPIServer.DefaultNotReadyTolerationSeconds = pointer.Int64(300)
+ }
+ if obj.Spec.Kubernetes.KubeAPIServer.DefaultUnreachableTolerationSeconds == nil {
+ obj.Spec.Kubernetes.KubeAPIServer.DefaultUnreachableTolerationSeconds = pointer.Int64(300)
+ }
+
if obj.Spec.Kubernetes.KubeControllerManager.NodeCIDRMaskSize == nil {
obj.Spec.Kubernetes.KubeControllerManager.NodeCIDRMaskSize = calculateDefaultNodeCIDRMaskSize(&obj.Spec)
}
@@ -288,14 +295,11 @@ func SetDefaults_KubeAPIServerConfig(obj *KubeAPIServerConfig) {
if obj.Logging.Verbosity == nil {
obj.Logging.Verbosity = pointer.Int32(2)
}
- if obj.DefaultNotReadyTolerationSeconds == nil {
- obj.DefaultNotReadyTolerationSeconds = pointer.Int64(300)
- }
- if obj.DefaultUnreachableTolerationSeconds == nil {
- obj.DefaultUnreachableTolerationSeconds = pointer.Int64(300)
- }
}
+// SetDefaults_KubeControllerManagerConfig sets default values for KubeControllerManagerConfig objects.
+func SetDefaults_KubeControllerManagerConfig(obj *KubeControllerManagerConfig) {}
+
// SetDefaults_Networking sets default values for Networking objects.
func SetDefaults_Networking(obj *Networking) {
if len(obj.IPFamilies) == 0 {
diff --git a/pkg/apis/core/v1beta1/zz_generated.defaults.go b/pkg/apis/core/v1beta1/zz_generated.defaults.go
index 5e124eeb120..752ae794d33 100644
--- a/pkg/apis/core/v1beta1/zz_generated.defaults.go
+++ b/pkg/apis/core/v1beta1/zz_generated.defaults.go
@@ -145,6 +145,9 @@ func SetObjectDefaults_Shoot(in *Shoot) {
if in.Spec.Kubernetes.KubeAPIServer != nil {
SetDefaults_KubeAPIServerConfig(in.Spec.Kubernetes.KubeAPIServer)
}
+ if in.Spec.Kubernetes.KubeControllerManager != nil {
+ SetDefaults_KubeControllerManagerConfig(in.Spec.Kubernetes.KubeControllerManager)
+ }
if in.Spec.Kubernetes.VerticalPodAutoscaler != nil {
SetDefaults_VerticalPodAutoscaler(in.Spec.Kubernetes.VerticalPodAutoscaler)
}
diff --git a/pkg/apis/core/validation/shoot.go b/pkg/apis/core/validation/shoot.go
index ee7b65a996f..81563fd080c 100644
--- a/pkg/apis/core/validation/shoot.go
+++ b/pkg/apis/core/validation/shoot.go
@@ -779,7 +779,7 @@ func validateKubernetes(kubernetes core.Kubernetes, networking *core.Networking,
}
allErrs = append(allErrs, ValidateKubeAPIServer(kubernetes.KubeAPIServer, kubernetes.Version, false, fldPath.Child("kubeAPIServer"))...)
- allErrs = append(allErrs, validateKubeControllerManager(kubernetes.KubeControllerManager, networking, kubernetes.Version, workerless, fldPath.Child("kubeControllerManager"))...)
+ allErrs = append(allErrs, ValidateKubeControllerManager(kubernetes.KubeControllerManager, networking, kubernetes.Version, workerless, fldPath.Child("kubeControllerManager"))...)
if workerless {
allErrs = append(allErrs, validateKubernetesForWorkerlessShoot(kubernetes, fldPath)...)
@@ -1164,7 +1164,8 @@ func ValidateKubeAPIServer(kubeAPIServer *core.KubeAPIServerConfig, version stri
return allErrs
}
-func validateKubeControllerManager(kcm *core.KubeControllerManagerConfig, networking *core.Networking, version string, workerless bool, fldPath *field.Path) field.ErrorList {
+// ValidateKubeControllerManager validates KubeControllerManagerConfig.
+func ValidateKubeControllerManager(kcm *core.KubeControllerManagerConfig, networking *core.Networking, version string, workerless bool, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
if kcm == nil {
diff --git a/pkg/apis/operator/v1alpha1/types.go b/pkg/apis/operator/v1alpha1/types.go
index ffa0016e3a8..16a07c8c996 100644
--- a/pkg/apis/operator/v1alpha1/types.go
+++ b/pkg/apis/operator/v1alpha1/types.go
@@ -245,6 +245,9 @@ type Kubernetes struct {
// KubeAPIServer contains configuration settings for the kube-apiserver.
// +optional
KubeAPIServer *KubeAPIServerConfig `json:"kubeAPIServer,omitempty"`
+ // KubeControllerManager contains configuration settings for the kube-controller-manager.
+ // +optional
+ KubeControllerManager *KubeControllerManagerConfig `json:"kubeControllerManager,omitempty"`
// Version is the semantic Kubernetes version to use for the virtual garden cluster.
// +kubebuilder:validation:MinLength=1
Version string `json:"version"`
@@ -375,6 +378,20 @@ type Networking struct {
Services string `json:"services"`
}
+// KubeControllerManagerConfig contains configuration settings for the kube-controller-manager.
+type KubeControllerManagerConfig struct {
+ // KubeControllerManagerConfig contains all configuration values not specific to the virtual garden cluster.
+ // +optional
+ *gardencorev1beta1.KubeControllerManagerConfig `json:",inline"`
+ // CertificateSigningDuration is the maximum length of duration signed certificates will be given. Individual CSRs
+ // may request shorter certs by setting `spec.expirationSeconds`.
+ // +kubebuilder:validation:Type=string
+ // +kubebuilder:validation:Pattern="^([0-9]+(\\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$"
+ // +kubebuilder:default=`48h`
+ // +optional
+ CertificateSigningDuration *metav1.Duration `json:"certificateSigningDuration,omitempty"`
+}
+
// GardenStatus is the status of a garden environment.
type GardenStatus struct {
// Gardener holds information about the Gardener which last acted on the Garden.
diff --git a/pkg/apis/operator/v1alpha1/validation/validation.go b/pkg/apis/operator/v1alpha1/validation/validation.go
index 12f7304086d..4fee199aaff 100644
--- a/pkg/apis/operator/v1alpha1/validation/validation.go
+++ b/pkg/apis/operator/v1alpha1/validation/validation.go
@@ -117,6 +117,17 @@ func validateVirtualCluster(virtualCluster operatorv1alpha1.VirtualCluster, runt
allErrs = append(allErrs, gardencorevalidation.ValidateKubeAPIServer(coreKubeAPIServerConfig, virtualCluster.Kubernetes.Version, true, path)...)
}
+ if kubeControllerManager := virtualCluster.Kubernetes.KubeControllerManager; kubeControllerManager != nil && kubeControllerManager.KubeControllerManagerConfig != nil {
+ path := fldPath.Child("kubernetes", "kubeControllerManager")
+
+ coreKubeControllerManagerConfig := &gardencore.KubeControllerManagerConfig{}
+ if err := gardenCoreScheme.Convert(kubeControllerManager.KubeControllerManagerConfig, coreKubeControllerManagerConfig, nil); err != nil {
+ allErrs = append(allErrs, field.InternalError(path, err))
+ }
+
+ allErrs = append(allErrs, gardencorevalidation.ValidateKubeControllerManager(coreKubeControllerManagerConfig, nil, virtualCluster.Kubernetes.Version, true, path)...)
+ }
+
if _, _, err := net.ParseCIDR(virtualCluster.Networking.Services); err != nil {
allErrs = append(allErrs, field.Invalid(fldPath.Child("networking", "services"), virtualCluster.Networking.Services, fmt.Sprintf("cannot parse service network cidr: %s", err.Error())))
}
diff --git a/pkg/apis/operator/v1alpha1/validation/validation_test.go b/pkg/apis/operator/v1alpha1/validation/validation_test.go
index 78f685c0bde..bc4c15e081c 100644
--- a/pkg/apis/operator/v1alpha1/validation/validation_test.go
+++ b/pkg/apis/operator/v1alpha1/validation/validation_test.go
@@ -730,6 +730,56 @@ var _ = Describe("Validation Tests", func() {
}))))
})
})
+
+ Context("topology-aware routing field", func() {
+ It("should prevent enabling topology-aware routing on single-zone cluster", func() {
+ garden.Spec.RuntimeCluster.Provider.Zones = []string{"a"}
+ garden.Spec.RuntimeCluster.Settings = &operatorv1alpha1.Settings{
+ TopologyAwareRouting: &operatorv1alpha1.SettingTopologyAwareRouting{
+ Enabled: true,
+ },
+ }
+ garden.Spec.VirtualCluster.ControlPlane = &operatorv1alpha1.ControlPlane{HighAvailability: &operatorv1alpha1.HighAvailability{}}
+
+ Expect(ValidateGarden(garden)).To(ConsistOf(
+ PointTo(MatchFields(IgnoreExtras, Fields{
+ "Type": Equal(field.ErrorTypeForbidden),
+ "Field": Equal("spec.runtimeCluster.settings.topologyAwareRouting.enabled"),
+ "Detail": Equal("topology-aware routing can only be enabled on multi-zone garden runtime cluster (with at least two zones in spec.provider.zones)"),
+ })),
+ ))
+ })
+
+ It("should prevent enabling topology-aware routing when control-plane is not HA", func() {
+ garden.Spec.RuntimeCluster.Provider.Zones = []string{"a", "b", "c"}
+ garden.Spec.RuntimeCluster.Settings = &operatorv1alpha1.Settings{
+ TopologyAwareRouting: &operatorv1alpha1.SettingTopologyAwareRouting{
+ Enabled: true,
+ },
+ }
+ garden.Spec.VirtualCluster.ControlPlane = nil
+
+ Expect(ValidateGarden(garden)).To(ConsistOf(
+ PointTo(MatchFields(IgnoreExtras, Fields{
+ "Type": Equal(field.ErrorTypeForbidden),
+ "Field": Equal("spec.runtimeCluster.settings.topologyAwareRouting.enabled"),
+ "Detail": Equal("topology-aware routing can only be enabled when virtual cluster's high-availability is enabled"),
+ })),
+ ))
+ })
+
+ It("should allow enabling topology-aware routing on multi-zone cluster with HA control-plane", func() {
+ garden.Spec.RuntimeCluster.Provider.Zones = []string{"a", "b", "c"}
+ garden.Spec.RuntimeCluster.Settings = &operatorv1alpha1.Settings{
+ TopologyAwareRouting: &operatorv1alpha1.SettingTopologyAwareRouting{
+ Enabled: true,
+ },
+ }
+ garden.Spec.VirtualCluster.ControlPlane = &operatorv1alpha1.ControlPlane{HighAvailability: &operatorv1alpha1.HighAvailability{}}
+
+ Expect(ValidateGarden(garden)).To(BeEmpty())
+ })
+ })
})
Context("virtual cluster", func() {
@@ -782,58 +832,6 @@ var _ = Describe("Validation Tests", func() {
})
})
})
-
- Context("runtime cluster", func() {
- Context("topology-aware routing field", func() {
- It("should prevent enabling topology-aware routing on single-zone cluster", func() {
- garden.Spec.RuntimeCluster.Provider.Zones = []string{"a"}
- garden.Spec.RuntimeCluster.Settings = &operatorv1alpha1.Settings{
- TopologyAwareRouting: &operatorv1alpha1.SettingTopologyAwareRouting{
- Enabled: true,
- },
- }
- garden.Spec.VirtualCluster.ControlPlane = &operatorv1alpha1.ControlPlane{HighAvailability: &operatorv1alpha1.HighAvailability{}}
-
- Expect(ValidateGarden(garden)).To(ConsistOf(
- PointTo(MatchFields(IgnoreExtras, Fields{
- "Type": Equal(field.ErrorTypeForbidden),
- "Field": Equal("spec.runtimeCluster.settings.topologyAwareRouting.enabled"),
- "Detail": Equal("topology-aware routing can only be enabled on multi-zone garden runtime cluster (with at least two zones in spec.provider.zones)"),
- })),
- ))
- })
-
- It("should prevent enabling topology-aware routing when control-plane is not HA", func() {
- garden.Spec.RuntimeCluster.Provider.Zones = []string{"a", "b", "c"}
- garden.Spec.RuntimeCluster.Settings = &operatorv1alpha1.Settings{
- TopologyAwareRouting: &operatorv1alpha1.SettingTopologyAwareRouting{
- Enabled: true,
- },
- }
- garden.Spec.VirtualCluster.ControlPlane = nil
-
- Expect(ValidateGarden(garden)).To(ConsistOf(
- PointTo(MatchFields(IgnoreExtras, Fields{
- "Type": Equal(field.ErrorTypeForbidden),
- "Field": Equal("spec.runtimeCluster.settings.topologyAwareRouting.enabled"),
- "Detail": Equal("topology-aware routing can only be enabled when virtual cluster's high-availability is enabled"),
- })),
- ))
- })
-
- It("should allow enabling topology-aware routing on multi-zone cluster with HA control-plane", func() {
- garden.Spec.RuntimeCluster.Provider.Zones = []string{"a", "b", "c"}
- garden.Spec.RuntimeCluster.Settings = &operatorv1alpha1.Settings{
- TopologyAwareRouting: &operatorv1alpha1.SettingTopologyAwareRouting{
- Enabled: true,
- },
- }
- garden.Spec.VirtualCluster.ControlPlane = &operatorv1alpha1.ControlPlane{HighAvailability: &operatorv1alpha1.HighAvailability{}}
-
- Expect(ValidateGarden(garden)).To(BeEmpty())
- })
- })
- })
})
Describe("#ValidateGardenUpdate", func() {
diff --git a/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go
index 5060d2d67ef..da5cd75d184 100644
--- a/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go
+++ b/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go
@@ -521,6 +521,32 @@ func (in *KubeAPIServerConfig) DeepCopy() *KubeAPIServerConfig {
return out
}
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KubeControllerManagerConfig) DeepCopyInto(out *KubeControllerManagerConfig) {
+ *out = *in
+ if in.KubeControllerManagerConfig != nil {
+ in, out := &in.KubeControllerManagerConfig, &out.KubeControllerManagerConfig
+ *out = new(v1beta1.KubeControllerManagerConfig)
+ (*in).DeepCopyInto(*out)
+ }
+ if in.CertificateSigningDuration != nil {
+ in, out := &in.CertificateSigningDuration, &out.CertificateSigningDuration
+ *out = new(v1.Duration)
+ **out = **in
+ }
+ return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeControllerManagerConfig.
+func (in *KubeControllerManagerConfig) DeepCopy() *KubeControllerManagerConfig {
+ if in == nil {
+ return nil
+ }
+ out := new(KubeControllerManagerConfig)
+ in.DeepCopyInto(out)
+ return out
+}
+
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Kubernetes) DeepCopyInto(out *Kubernetes) {
*out = *in
@@ -529,6 +555,11 @@ func (in *Kubernetes) DeepCopyInto(out *Kubernetes) {
*out = new(KubeAPIServerConfig)
(*in).DeepCopyInto(*out)
}
+ if in.KubeControllerManager != nil {
+ in, out := &in.KubeControllerManager, &out.KubeControllerManager
+ *out = new(KubeControllerManagerConfig)
+ (*in).DeepCopyInto(*out)
+ }
return
}
diff --git a/pkg/operation/botanist/component/kubecontrollermanager/kube_controller_manager.go b/pkg/operation/botanist/component/kubecontrollermanager/kube_controller_manager.go
index 93aa7f51563..249c8996f72 100644
--- a/pkg/operation/botanist/component/kubecontrollermanager/kube_controller_manager.go
+++ b/pkg/operation/botanist/component/kubecontrollermanager/kube_controller_manager.go
@@ -136,8 +136,12 @@ type Values struct {
Image string
// Replicas is the number of replicas for the kube-controller-manager deployment.
Replicas int32
+ // PriorityClassName is the name of the priority class.
+ PriorityClassName string
// Config is the configuration of the kube-controller-manager.
Config *gardencorev1beta1.KubeControllerManagerConfig
+ // NamePrefix is the prefix for the resource names.
+ NamePrefix string
// HVPAConfig is the configuration for HVPA.
HVPAConfig *HVPAConfig
// IsWorkerless specifies whether the cluster has worker nodes.
@@ -200,8 +204,8 @@ const (
func (k *kubeControllerManager) Deploy(ctx context.Context) error {
serverSecret, err := k.secretsManager.Generate(ctx, &secrets.CertificateSecretConfig{
Name: secretNameServer,
- CommonName: v1beta1constants.DeploymentNameKubeControllerManager,
- DNSNames: kubernetesutils.DNSNamesForService(serviceName, k.namespace),
+ CommonName: k.values.NamePrefix + v1beta1constants.DeploymentNameKubeControllerManager,
+ DNSNames: kubernetesutils.DNSNamesForService(k.values.NamePrefix+serviceName, k.namespace),
CertType: secrets.ServerCert,
SkipPublishingCACertificate: true,
}, secretsmanager.SignedByCA(v1beta1constants.SecretNameCACluster), secretsmanager.Rotate(secretsmanager.InPlace))
@@ -325,12 +329,12 @@ func (k *kubeControllerManager) Deploy(ctx context.Context) error {
v1beta1constants.GardenRole: v1beta1constants.GardenRoleControlPlane,
v1beta1constants.LabelPodMaintenanceRestart: "true",
v1beta1constants.LabelNetworkPolicyToDNS: v1beta1constants.LabelNetworkPolicyAllowed,
- gardenerutils.NetworkPolicyLabel(v1beta1constants.DeploymentNameKubeAPIServer, kubeapiserverconstants.Port): v1beta1constants.LabelNetworkPolicyAllowed,
+ gardenerutils.NetworkPolicyLabel(k.values.NamePrefix+v1beta1constants.DeploymentNameKubeAPIServer, kubeapiserverconstants.Port): v1beta1constants.LabelNetworkPolicyAllowed,
}),
},
Spec: corev1.PodSpec{
AutomountServiceAccountToken: pointer.Bool(false),
- PriorityClassName: v1beta1constants.PriorityClassNameShootControlPlane300,
+ PriorityClassName: k.values.PriorityClassName,
Containers: []corev1.Container{
{
Name: containerName,
@@ -527,7 +531,7 @@ func (k *kubeControllerManager) Deploy(ctx context.Context) error {
hvpa.Spec.TargetRef = &autoscalingv2beta1.CrossVersionObjectReference{
APIVersion: appsv1.SchemeGroupVersion.String(),
Kind: "Deployment",
- Name: v1beta1constants.DeploymentNameKubeControllerManager,
+ Name: k.values.NamePrefix + v1beta1constants.DeploymentNameKubeControllerManager,
}
return nil
}); err != nil {
@@ -544,7 +548,7 @@ func (k *kubeControllerManager) Deploy(ctx context.Context) error {
vpa.Spec.TargetRef = &autoscalingv1.CrossVersionObjectReference{
APIVersion: appsv1.SchemeGroupVersion.String(),
Kind: "Deployment",
- Name: v1beta1constants.DeploymentNameKubeControllerManager,
+ Name: k.values.NamePrefix + v1beta1constants.DeploymentNameKubeControllerManager,
}
vpa.Spec.UpdatePolicy = &vpaautoscalingv1.PodUpdatePolicy{
UpdateMode: &vpaUpdateMode,
@@ -559,28 +563,40 @@ func (k *kubeControllerManager) Deploy(ctx context.Context) error {
return k.reconcileShootResources(ctx, shootAccessSecret.ServiceAccountName)
}
-func (k *kubeControllerManager) SetShootClient(c client.Client) { k.shootClient = c }
-func (k *kubeControllerManager) SetReplicaCount(replicas int32) { k.values.Replicas = replicas }
-func (k *kubeControllerManager) Destroy(_ context.Context) error { return nil }
+func (k *kubeControllerManager) Destroy(ctx context.Context) error {
+ return kubernetesutils.DeleteObjects(ctx, k.seedClient.Client(),
+ k.emptyManagedResource(),
+ k.emptyManagedResourceSecret(),
+ k.emptyVPA(),
+ k.emptyHVPA(),
+ k.emptyService(),
+ k.emptyPodDisruptionBudget(),
+ k.emptyDeployment(),
+ k.newShootAccessSecret().Secret,
+ )
+}
+
+func (k *kubeControllerManager) SetShootClient(c client.Client) { k.shootClient = c }
+func (k *kubeControllerManager) SetReplicaCount(replicas int32) { k.values.Replicas = replicas }
func (k *kubeControllerManager) emptyVPA() *vpaautoscalingv1.VerticalPodAutoscaler {
- return &vpaautoscalingv1.VerticalPodAutoscaler{ObjectMeta: metav1.ObjectMeta{Name: "kube-controller-manager-vpa", Namespace: k.namespace}}
+ return &vpaautoscalingv1.VerticalPodAutoscaler{ObjectMeta: metav1.ObjectMeta{Name: k.values.NamePrefix + "kube-controller-manager-vpa", Namespace: k.namespace}}
}
func (k *kubeControllerManager) emptyHVPA() *hvpav1alpha1.Hvpa {
- return &hvpav1alpha1.Hvpa{ObjectMeta: metav1.ObjectMeta{Name: v1beta1constants.DeploymentNameKubeControllerManager, Namespace: k.namespace}}
+ return &hvpav1alpha1.Hvpa{ObjectMeta: metav1.ObjectMeta{Name: k.values.NamePrefix + v1beta1constants.DeploymentNameKubeControllerManager, Namespace: k.namespace}}
}
func (k *kubeControllerManager) emptyService() *corev1.Service {
- return &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: k.namespace}}
+ return &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: k.values.NamePrefix + serviceName, Namespace: k.namespace}}
}
func (k *kubeControllerManager) emptyDeployment() *appsv1.Deployment {
- return &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: v1beta1constants.DeploymentNameKubeControllerManager, Namespace: k.namespace}}
+ return &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: k.values.NamePrefix + v1beta1constants.DeploymentNameKubeControllerManager, Namespace: k.namespace}}
}
func (k *kubeControllerManager) emptyPodDisruptionBudget() client.Object {
- objectMeta := metav1.ObjectMeta{Name: v1beta1constants.DeploymentNameKubeControllerManager, Namespace: k.namespace}
+ objectMeta := metav1.ObjectMeta{Name: k.values.NamePrefix + v1beta1constants.DeploymentNameKubeControllerManager, Namespace: k.namespace}
if versionutils.ConstraintK8sGreaterEqual121.Check(k.values.RuntimeVersion) {
return &policyv1.PodDisruptionBudget{ObjectMeta: objectMeta}
@@ -720,7 +736,7 @@ func (k *kubeControllerManager) computeCommand(port int32) []string {
command = append(command, fmt.Sprintf("--concurrent-serviceaccount-token-syncs=%d", v))
}
- if len(k.values.Config.FeatureGates) > 0 {
+ if k.values.Config != nil && len(k.values.Config.FeatureGates) > 0 {
command = append(command, kubernetesutils.FeatureGatesToCommandLineParameter(k.values.Config.FeatureGates))
}
@@ -762,7 +778,7 @@ func (k *kubeControllerManager) getHorizontalPodAutoscalerConfig() gardencorev1b
Tolerance: &defaultHPATolerance,
}
- if k.values.Config.HorizontalPodAutoscalerConfig != nil {
+ if k.values.Config != nil && k.values.Config.HorizontalPodAutoscalerConfig != nil {
if v := k.values.Config.HorizontalPodAutoscalerConfig.CPUInitializationPeriod; v != nil {
horizontalPodAutoscalerConfig.CPUInitializationPeriod = v
}
diff --git a/pkg/operation/botanist/component/kubecontrollermanager/kube_controller_manager_test.go b/pkg/operation/botanist/component/kubecontrollermanager/kube_controller_manager_test.go
index 0c6ef2b0a32..320a58f8618 100644
--- a/pkg/operation/botanist/component/kubecontrollermanager/kube_controller_manager_test.go
+++ b/pkg/operation/botanist/component/kubecontrollermanager/kube_controller_manager_test.go
@@ -83,6 +83,7 @@ var _ = Describe("KubeControllerManager", func() {
hvpaConfigEnabled = &HVPAConfig{Enabled: true}
hvpaConfigEnabledScaleDownOff = &HVPAConfig{Enabled: true, ScaleDownUpdateMode: pointer.String(hvpav1alpha1.UpdateModeOff)}
isWorkerless = false
+ priorityClassName = v1beta1constants.PriorityClassNameShootControlPlane300
hpaConfig = gardencorev1beta1.HorizontalPodAutoscalerConfig{
CPUInitializationPeriod: &metav1.Duration{Duration: 5 * time.Minute},
@@ -146,6 +147,25 @@ var _ = Describe("KubeControllerManager", func() {
fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetesscheme.Scheme).Build()
sm = fakesecretsmanager.New(fakeClient, namespace)
+ values = Values{
+ RuntimeVersion: runtimeKubernetesVersion,
+ TargetVersion: semverVersion,
+ Image: image,
+ Config: &kcmConfig,
+ PriorityClassName: priorityClassName,
+ HVPAConfig: hvpaConfigDisabled,
+ IsWorkerless: isWorkerless,
+ PodNetwork: podCIDR,
+ ServiceNetwork: serviceCIDR,
+ }
+ kubeControllerManager = New(
+ testLogger,
+ fakeInterface,
+ namespace,
+ sm,
+ values,
+ )
+
By("Create secrets managed outside of this package for whose secretsmanager.Get() will be called")
Expect(fakeClient.Create(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "ca", Namespace: namespace}})).To(Succeed())
Expect(fakeClient.Create(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "generic-token-kubeconfig", Namespace: namespace}})).To(Succeed())
@@ -160,26 +180,6 @@ var _ = Describe("KubeControllerManager", func() {
Describe("#Deploy", func() {
Context("Tests expecting a failure", func() {
- BeforeEach(func() {
- values = Values{
- RuntimeVersion: runtimeKubernetesVersion,
- TargetVersion: semverVersion,
- Image: image,
- Config: &kcmConfig,
- HVPAConfig: hvpaConfigDisabled,
- IsWorkerless: isWorkerless,
- PodNetwork: podCIDR,
- ServiceNetwork: serviceCIDR,
- }
- kubeControllerManager = New(
- testLogger,
- fakeInterface,
- namespace,
- sm,
- values,
- )
- })
-
It("should fail when the service cannot be created", func() {
gomock.InOrder(
c.EXPECT().Get(ctx, kubernetesutils.Key(namespace, serviceName), gomock.AssignableToTypeOf(&corev1.Service{})),
@@ -532,7 +532,7 @@ var _ = Describe("KubeControllerManager", func() {
},
Spec: corev1.PodSpec{
AutomountServiceAccountToken: pointer.Bool(false),
- PriorityClassName: v1beta1constants.PriorityClassNameShootControlPlane300,
+ PriorityClassName: priorityClassName,
Containers: []corev1.Container{
{
Name: "kube-controller-manager",
@@ -713,6 +713,7 @@ subjects:
TargetVersion: semverVersion,
Image: image,
Config: config,
+ PriorityClassName: priorityClassName,
HVPAConfig: hvpaConfig,
IsWorkerless: isWorkerless,
PodNetwork: podCIDR,
@@ -826,6 +827,7 @@ subjects:
TargetVersion: semverVersion,
Image: image,
Config: config,
+ PriorityClassName: priorityClassName,
HVPAConfig: hvpaConfig,
IsWorkerless: isWorkerless,
PodNetwork: podCIDR,
@@ -934,7 +936,26 @@ subjects:
})
Describe("#Destroy", func() {
- It("should return nil as it's not implemented as of now", func() {
+ It("should successfully destroy all resources", func() {
+ kubeControllerManager = New(
+ testLogger,
+ fakeInterface,
+ namespace,
+ sm,
+ values,
+ )
+
+ gomock.InOrder(
+ c.EXPECT().Delete(ctx, &resourcesv1alpha1.ManagedResource{ObjectMeta: metav1.ObjectMeta{Name: managedResourceName, Namespace: namespace}}),
+ c.EXPECT().Delete(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: managedResourceSecretName, Namespace: namespace}}),
+ c.EXPECT().Delete(ctx, &vpaautoscalingv1.VerticalPodAutoscaler{ObjectMeta: metav1.ObjectMeta{Name: vpaName, Namespace: namespace}}),
+ c.EXPECT().Delete(ctx, &hvpav1alpha1.Hvpa{ObjectMeta: metav1.ObjectMeta{Name: hvpaName, Namespace: namespace}}),
+ c.EXPECT().Delete(ctx, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespace}}),
+ c.EXPECT().Delete(ctx, &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: pdbName, Namespace: namespace}}),
+ c.EXPECT().Delete(ctx, &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "kube-controller-manager", Namespace: namespace}}),
+ c.EXPECT().Delete(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: secretName, Namespace: namespace}}),
+ )
+
Expect(kubeControllerManager.Destroy(ctx)).To(Succeed())
})
})
diff --git a/pkg/operation/botanist/component/shared/kubecontrollermanager.go b/pkg/operation/botanist/component/shared/kubecontrollermanager.go
new file mode 100644
index 00000000000..04b62665b02
--- /dev/null
+++ b/pkg/operation/botanist/component/shared/kubecontrollermanager.go
@@ -0,0 +1,81 @@
+// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shared
+
+import (
+ "net"
+ "time"
+
+ "github.com/Masterminds/semver"
+ "github.com/go-logr/logr"
+
+ gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
+ "github.com/gardener/gardener/pkg/client/kubernetes"
+ "github.com/gardener/gardener/pkg/operation/botanist/component/kubecontrollermanager"
+ "github.com/gardener/gardener/pkg/utils/images"
+ "github.com/gardener/gardener/pkg/utils/imagevector"
+ secretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager"
+)
+
+// NewKubeControllerManager returns a deployer for the kube-controller-manager.
+func NewKubeControllerManager(
+ log logr.Logger,
+ runtimeClientSet kubernetes.Interface,
+ runtimeNamespace string,
+ runtimeVersion *semver.Version,
+ targetVersion *semver.Version,
+ imageVector imagevector.ImageVector,
+ secretsManager secretsmanager.Interface,
+ namePrefix string,
+ config *gardencorev1beta1.KubeControllerManagerConfig,
+ priorityClassName string,
+ isWorkerless bool,
+ hvpaConfig *kubecontrollermanager.HVPAConfig,
+ podNetwork *net.IPNet,
+ serviceNetwork *net.IPNet,
+ clusterSigningDuration *time.Duration,
+ controllerWorkers kubecontrollermanager.ControllerWorkers,
+ controllerSyncPeriods kubecontrollermanager.ControllerSyncPeriods,
+) (
+ kubecontrollermanager.Interface,
+ error,
+) {
+ image, err := imageVector.FindImage(images.ImageNameKubeControllerManager, imagevector.RuntimeVersion(runtimeVersion.String()), imagevector.TargetVersion(targetVersion.String()))
+ if err != nil {
+ return nil, err
+ }
+
+ return kubecontrollermanager.New(
+ log.WithValues("component", "kube-controller-manager"),
+ runtimeClientSet,
+ runtimeNamespace,
+ secretsManager,
+ kubecontrollermanager.Values{
+ RuntimeVersion: runtimeVersion,
+ TargetVersion: targetVersion,
+ Image: image.String(),
+ Config: config,
+ PriorityClassName: priorityClassName,
+ NamePrefix: namePrefix,
+ HVPAConfig: hvpaConfig,
+ IsWorkerless: isWorkerless,
+ PodNetwork: podNetwork,
+ ServiceNetwork: serviceNetwork,
+ ClusterSigningDuration: clusterSigningDuration,
+ ControllerWorkers: controllerWorkers,
+ ControllerSyncPeriods: controllerSyncPeriods,
+ },
+ ), nil
+}
diff --git a/pkg/operation/botanist/kubecontrollermanager.go b/pkg/operation/botanist/kubecontrollermanager.go
index a58c8517d7f..16aa838387b 100644
--- a/pkg/operation/botanist/kubecontrollermanager.go
+++ b/pkg/operation/botanist/kubecontrollermanager.go
@@ -25,18 +25,12 @@ import (
"github.com/gardener/gardener/pkg/client/kubernetes"
"github.com/gardener/gardener/pkg/features"
"github.com/gardener/gardener/pkg/operation/botanist/component/kubecontrollermanager"
- "github.com/gardener/gardener/pkg/utils/images"
- "github.com/gardener/gardener/pkg/utils/imagevector"
+ "github.com/gardener/gardener/pkg/operation/botanist/component/shared"
kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes"
)
// DefaultKubeControllerManager returns a deployer for the kube-controller-manager.
func (b *Botanist) DefaultKubeControllerManager() (kubecontrollermanager.Interface, error) {
- image, err := b.ImageVector.FindImage(images.ImageNameKubeControllerManager, imagevector.RuntimeVersion(b.SeedVersion()), imagevector.TargetVersion(b.ShootVersion()))
- if err != nil {
- return nil, err
- }
-
hvpaEnabled := features.DefaultFeatureGate.Enabled(features.HVPA)
if b.ManagedSeed != nil {
hvpaEnabled = features.DefaultFeatureGate.Enabled(features.HVPAForShootedSeed)
@@ -53,25 +47,28 @@ func (b *Botanist) DefaultKubeControllerManager() (kubecontrollermanager.Interfa
pods = b.Shoot.Networks.Pods
}
- return kubecontrollermanager.New(
- b.Logger.WithValues("component", "kube-controller-manager"),
+ return shared.NewKubeControllerManager(
+ b.Logger,
b.SeedClientSet,
b.Shoot.SeedNamespace,
+ b.Seed.KubernetesVersion,
+ b.Shoot.KubernetesVersion,
+ b.ImageVector,
b.SecretsManager,
- kubecontrollermanager.Values{
- RuntimeVersion: b.Seed.KubernetesVersion,
- TargetVersion: b.Shoot.KubernetesVersion,
- Image: image.String(),
- Config: b.Shoot.GetInfo().Spec.Kubernetes.KubeControllerManager,
- HVPAConfig: &kubecontrollermanager.HVPAConfig{
- Enabled: hvpaEnabled,
- ScaleDownUpdateMode: &scaleDownUpdateMode,
- },
- IsWorkerless: b.Shoot.IsWorkerless,
- PodNetwork: pods,
- ServiceNetwork: services,
+ "",
+ b.Shoot.GetInfo().Spec.Kubernetes.KubeControllerManager,
+ v1beta1constants.PriorityClassNameShootControlPlane300,
+ b.Shoot.IsWorkerless,
+ &kubecontrollermanager.HVPAConfig{
+ Enabled: hvpaEnabled,
+ ScaleDownUpdateMode: &scaleDownUpdateMode,
},
- ), nil
+ pods,
+ services,
+ nil,
+ kubecontrollermanager.ControllerWorkers{},
+ kubecontrollermanager.ControllerSyncPeriods{},
+ )
}
// DeployKubeControllerManager deploys the Kubernetes Controller Manager.
diff --git a/pkg/operation/botanist/kubecontrollermanager_test.go b/pkg/operation/botanist/kubecontrollermanager_test.go
index a8fdf29d929..5de73dd5573 100644
--- a/pkg/operation/botanist/kubecontrollermanager_test.go
+++ b/pkg/operation/botanist/kubecontrollermanager_test.go
@@ -67,15 +67,14 @@ var _ = Describe("KubeControllerManager", func() {
Describe("#DefaultKubeControllerManager", func() {
BeforeEach(func() {
- kubernetesClient.EXPECT().Version()
-
botanist.Logger = logr.Discard()
botanist.SeedClientSet = kubernetesClient
botanist.Seed = &seedpkg.Seed{
KubernetesVersion: semver.MustParse("1.25.0"),
}
botanist.Shoot = &shootpkg.Shoot{
- Networks: &shootpkg.Networks{},
+ KubernetesVersion: semver.MustParse("1.25.0"),
+ Networks: &shootpkg.Networks{},
}
botanist.Shoot.SetInfo(&gardencorev1beta1.Shoot{})
})
diff --git a/pkg/operation/botanist/resource_manager.go b/pkg/operation/botanist/resource_manager.go
index 68f16fa382d..41f9473b9c3 100644
--- a/pkg/operation/botanist/resource_manager.go
+++ b/pkg/operation/botanist/resource_manager.go
@@ -65,7 +65,6 @@ func (b *Botanist) DefaultResourceManager() (resourcemanager.Interface, error) {
// DeployGardenerResourceManager deploys the gardener-resource-manager
func (b *Botanist) DeployGardenerResourceManager(ctx context.Context) error {
-
return shared.DeployGardenerResourceManager(
ctx,
b.SeedClientSet.Client(),
diff --git a/pkg/operator/controller/garden/components.go b/pkg/operator/controller/garden/components.go
index 279c8841cf7..66ee728127d 100644
--- a/pkg/operator/controller/garden/components.go
+++ b/pkg/operator/controller/garden/components.go
@@ -18,6 +18,7 @@ import (
"context"
"errors"
"fmt"
+ "net"
"os"
"time"
@@ -44,6 +45,7 @@ import (
"github.com/gardener/gardener/pkg/operation/botanist/component/istio"
"github.com/gardener/gardener/pkg/operation/botanist/component/kubeapiserver"
"github.com/gardener/gardener/pkg/operation/botanist/component/kubeapiserverexposure"
+ "github.com/gardener/gardener/pkg/operation/botanist/component/kubecontrollermanager"
"github.com/gardener/gardener/pkg/operation/botanist/component/resourcemanager"
sharedcomponent "github.com/gardener/gardener/pkg/operation/botanist/component/shared"
gardenerutils "github.com/gardener/gardener/pkg/utils/gardener"
@@ -78,7 +80,7 @@ func (r *Reconciler) newGardenerResourceManager(garden *operatorv1alpha1.Garden,
)
}
-func (r *Reconciler) newVirtualGardenGardenerResourceManager(garden *operatorv1alpha1.Garden, secretsManager secretsmanager.Interface) (resourcemanager.Interface, error) {
+func (r *Reconciler) newVirtualGardenGardenerResourceManager(secretsManager secretsmanager.Interface) (resourcemanager.Interface, error) {
return sharedcomponent.NewTargetGardenerResourceManager(
r.RuntimeClientSet.Client(),
r.GardenNamespace,
@@ -91,7 +93,7 @@ func (r *Reconciler) newVirtualGardenGardenerResourceManager(garden *operatorv1a
r.Config.LogLevel, r.Config.LogFormat,
namePrefix,
false,
- v1beta1constants.PriorityClassNameGardenSystem500,
+ v1beta1constants.PriorityClassNameGardenSystem400,
nil,
operatorv1alpha1.SecretNameCARuntime,
nil,
@@ -428,6 +430,58 @@ func fetchKubeconfigFromSecret(ctx context.Context, c client.Client, key client.
return kubeconfig, nil
}
+func (r *Reconciler) newKubeControllerManager(
+ log logr.Logger,
+ garden *operatorv1alpha1.Garden,
+ secretsManager secretsmanager.Interface,
+ targetVersion *semver.Version,
+) (
+ kubecontrollermanager.Interface,
+ error,
+) {
+ var (
+ config *gardencorev1beta1.KubeControllerManagerConfig
+ certificateSigningDuration *time.Duration
+ )
+
+ if controllerManager := garden.Spec.VirtualCluster.Kubernetes.KubeControllerManager; controllerManager != nil {
+ config = controllerManager.KubeControllerManagerConfig
+ certificateSigningDuration = pointer.Duration(controllerManager.CertificateSigningDuration.Duration)
+ }
+
+ _, services, err := net.ParseCIDR(garden.Spec.VirtualCluster.Networking.Services)
+ if err != nil {
+ return nil, fmt.Errorf("cannot parse service network CIDR: %w", err)
+ }
+
+ return sharedcomponent.NewKubeControllerManager(
+ log,
+ r.RuntimeClientSet,
+ r.GardenNamespace,
+ r.RuntimeVersion,
+ targetVersion,
+ r.ImageVector,
+ secretsManager,
+ namePrefix,
+ config,
+ v1beta1constants.PriorityClassNameGardenSystem300,
+ true,
+ &kubecontrollermanager.HVPAConfig{Enabled: hvpaEnabled()},
+ nil,
+ services,
+ certificateSigningDuration,
+ kubecontrollermanager.ControllerWorkers{
+ GarbageCollector: pointer.Int(250),
+ Namespace: pointer.Int(100),
+ ResourceQuota: pointer.Int(100),
+ ServiceAccountToken: pointer.Int(0),
+ },
+ kubecontrollermanager.ControllerSyncPeriods{
+ ResourceQuota: pointer.Duration(time.Minute),
+ },
+ )
+}
+
func (r *Reconciler) newKubeStateMetrics() (component.DeployWaiter, error) {
return sharedcomponent.NewKubeStateMetrics(
r.RuntimeClientSet.Client(),
diff --git a/pkg/operator/controller/garden/reconciler_delete.go b/pkg/operator/controller/garden/reconciler_delete.go
index 063917c56c5..46771e47710 100644
--- a/pkg/operator/controller/garden/reconciler_delete.go
+++ b/pkg/operator/controller/garden/reconciler_delete.go
@@ -98,7 +98,11 @@ func (r *Reconciler) delete(
if err != nil {
return reconcile.Result{}, err
}
- virtualGardenGardenerResourceManager, err := r.newVirtualGardenGardenerResourceManager(garden, secretsManager)
+ kubeControllerManager, err := r.newKubeControllerManager(log, garden, secretsManager, targetVersion)
+ if err != nil {
+ return reconcile.Result{}, err
+ }
+ virtualGardenGardenerResourceManager, err := r.newVirtualGardenGardenerResourceManager(secretsManager)
if err != nil {
return reconcile.Result{}, err
}
@@ -126,6 +130,10 @@ func (r *Reconciler) delete(
Fn: component.OpDestroyAndWait(virtualGardenGardenerResourceManager).Destroy,
Dependencies: flow.NewTaskIDs(destroyVirtualGardenGardenerAccess),
})
+ destroyKubeControllerManager = g.Add(flow.Task{
+ Name: "Destroying Kubernetes Controller Manager Server",
+ Fn: component.OpDestroyAndWait(kubeControllerManager).Destroy,
+ })
destroyKubeAPIServerService = g.Add(flow.Task{
Name: "Destroying Kubernetes API Server service",
Fn: component.OpDestroyAndWait(kubeAPIServerService).Destroy,
@@ -151,6 +159,7 @@ func (r *Reconciler) delete(
cleanupGenericTokenKubeconfig,
destroyVirtualGardenGardenerAccess,
destroyVirtualGardenGardenerResourceManager,
+ destroyKubeControllerManager,
destroyKubeAPIServerService,
destroyKubeAPIServer,
destroyEtcd,
diff --git a/pkg/operator/controller/garden/reconciler_reconcile.go b/pkg/operator/controller/garden/reconciler_reconcile.go
index 90ee29dfd16..2c4ade70002 100644
--- a/pkg/operator/controller/garden/reconciler_reconcile.go
+++ b/pkg/operator/controller/garden/reconciler_reconcile.go
@@ -145,7 +145,11 @@ func (r *Reconciler) reconcile(
if err != nil {
return reconcile.Result{}, err
}
- virtualGardenGardenerResourceManager, err := r.newVirtualGardenGardenerResourceManager(garden, secretsManager)
+ kubeControllerManager, err := r.newKubeControllerManager(log, garden, secretsManager, targetVersion)
+ if err != nil {
+ return reconcile.Result{}, err
+ }
+ virtualGardenGardenerResourceManager, err := r.newVirtualGardenGardenerResourceManager(secretsManager)
if err != nil {
return reconcile.Result{}, err
}
@@ -244,6 +248,14 @@ func (r *Reconciler) reconcile(
Fn: kubeAPIServer.Wait,
Dependencies: flow.NewTaskIDs(deployKubeAPIServer),
})
+ _ = g.Add(flow.Task{
+ Name: "Deploying Kubernetes Controller Manager",
+ Fn: func(ctx context.Context) error {
+ kubeControllerManager.SetReplicaCount(1)
+ return component.OpWait(kubeControllerManager).Deploy(ctx)
+ },
+ Dependencies: flow.NewTaskIDs(waitUntilKubeAPIServerIsReady),
+ })
deployVirtualGardenGardenerResourceManager = g.Add(flow.Task{
Name: "Deploying gardener-resource-manager for virtual garden",
Fn: func(ctx context.Context) error {
diff --git a/pkg/operator/webhook/defaulting/handler.go b/pkg/operator/webhook/defaulting/handler.go
index ee9e3eb3fe7..19c25f07517 100644
--- a/pkg/operator/webhook/defaulting/handler.go
+++ b/pkg/operator/webhook/defaulting/handler.go
@@ -37,9 +37,21 @@ func (h *Handler) Default(_ context.Context, obj runtime.Object) error {
return fmt.Errorf("expected *operatorv1alpha1.Garden but got %T", obj)
}
- if kubeAPIServer := garden.Spec.VirtualCluster.Kubernetes.KubeAPIServer; kubeAPIServer != nil {
- gardencorev1beta1.SetDefaults_KubeAPIServerConfig(kubeAPIServer.KubeAPIServerConfig)
+ if garden.Spec.VirtualCluster.Kubernetes.KubeAPIServer == nil {
+ garden.Spec.VirtualCluster.Kubernetes.KubeAPIServer = &operatorv1alpha1.KubeAPIServerConfig{}
}
+ if garden.Spec.VirtualCluster.Kubernetes.KubeAPIServer.KubeAPIServerConfig == nil {
+ garden.Spec.VirtualCluster.Kubernetes.KubeAPIServer.KubeAPIServerConfig = &gardencorev1beta1.KubeAPIServerConfig{}
+ }
+ gardencorev1beta1.SetDefaults_KubeAPIServerConfig(garden.Spec.VirtualCluster.Kubernetes.KubeAPIServer.KubeAPIServerConfig)
+
+ if garden.Spec.VirtualCluster.Kubernetes.KubeControllerManager == nil {
+ garden.Spec.VirtualCluster.Kubernetes.KubeControllerManager = &operatorv1alpha1.KubeControllerManagerConfig{}
+ }
+ if garden.Spec.VirtualCluster.Kubernetes.KubeControllerManager.KubeControllerManagerConfig == nil {
+ garden.Spec.VirtualCluster.Kubernetes.KubeControllerManager.KubeControllerManagerConfig = &gardencorev1beta1.KubeControllerManagerConfig{}
+ }
+ gardencorev1beta1.SetDefaults_KubeControllerManagerConfig(garden.Spec.VirtualCluster.Kubernetes.KubeControllerManager.KubeControllerManagerConfig)
return nil
}
diff --git a/skaffold-operator.yaml b/skaffold-operator.yaml
index f327063059a..872c56b79fb 100644
--- a/skaffold-operator.yaml
+++ b/skaffold-operator.yaml
@@ -75,6 +75,7 @@ build:
- pkg/operation/botanist/component/kubeapiserver
- pkg/operation/botanist/component/kubeapiserver/constants
- pkg/operation/botanist/component/kubeapiserverexposure
+ - pkg/operation/botanist/component/kubecontrollermanager
- pkg/operation/botanist/component/kubescheduler
- pkg/operation/botanist/component/kubestatemetrics
- pkg/operation/botanist/component/nginxingress
diff --git a/test/e2e/gardener/managedseed/create_rotate_delete.go b/test/e2e/gardener/managedseed/create_rotate_delete.go
index e6347d94d57..dfea48f493b 100644
--- a/test/e2e/gardener/managedseed/create_rotate_delete.go
+++ b/test/e2e/gardener/managedseed/create_rotate_delete.go
@@ -42,7 +42,7 @@ import (
. "github.com/gardener/gardener/pkg/utils/test"
e2e "github.com/gardener/gardener/test/e2e/gardener"
"github.com/gardener/gardener/test/framework"
- "github.com/gardener/gardener/test/utils/shoots/access"
+ "github.com/gardener/gardener/test/utils/access"
)
var parentCtx context.Context
diff --git a/test/e2e/gardener/shoot/create_rotate_delete.go b/test/e2e/gardener/shoot/create_rotate_delete.go
index 22b9f20518d..1278307df7a 100644
--- a/test/e2e/gardener/shoot/create_rotate_delete.go
+++ b/test/e2e/gardener/shoot/create_rotate_delete.go
@@ -29,8 +29,8 @@ import (
"github.com/gardener/gardener/pkg/client/kubernetes"
e2e "github.com/gardener/gardener/test/e2e/gardener"
"github.com/gardener/gardener/test/e2e/gardener/shoot/internal/rotation"
+ "github.com/gardener/gardener/test/utils/access"
rotationutils "github.com/gardener/gardener/test/utils/rotation"
- "github.com/gardener/gardener/test/utils/shoots/access"
)
var _ = Describe("Shoot Tests", Label("Shoot", "default"), func() {
diff --git a/test/e2e/gardener/shoot/create_update_delete.go b/test/e2e/gardener/shoot/create_update_delete.go
index bee6e656c6d..78a066fad75 100644
--- a/test/e2e/gardener/shoot/create_update_delete.go
+++ b/test/e2e/gardener/shoot/create_update_delete.go
@@ -31,7 +31,7 @@ import (
"github.com/gardener/gardener/pkg/utils"
e2e "github.com/gardener/gardener/test/e2e/gardener"
"github.com/gardener/gardener/test/framework"
- "github.com/gardener/gardener/test/utils/shoots/access"
+ "github.com/gardener/gardener/test/utils/access"
shootupdatesuite "github.com/gardener/gardener/test/utils/shoots/update"
)
diff --git a/test/e2e/gardener/shoot/internal/node/node.go b/test/e2e/gardener/shoot/internal/node/node.go
index 78f30ffcd07..19ed25a5a59 100644
--- a/test/e2e/gardener/shoot/internal/node/node.go
+++ b/test/e2e/gardener/shoot/internal/node/node.go
@@ -34,7 +34,7 @@ import (
"github.com/gardener/gardener/pkg/utils"
"github.com/gardener/gardener/pkg/utils/managedresources"
"github.com/gardener/gardener/test/framework"
- "github.com/gardener/gardener/test/utils/shoots/access"
+ "github.com/gardener/gardener/test/utils/access"
)
const nodeCriticalDaemonSetName = "e2e-test-node-critical"
diff --git a/test/e2e/gardener/shoot/internal/rotation/shoot_access.go b/test/e2e/gardener/shoot/internal/rotation/shoot_access.go
index ea68c62efbe..a359980a913 100644
--- a/test/e2e/gardener/shoot/internal/rotation/shoot_access.go
+++ b/test/e2e/gardener/shoot/internal/rotation/shoot_access.go
@@ -23,14 +23,14 @@ import (
"github.com/gardener/gardener/pkg/client/kubernetes"
"github.com/gardener/gardener/test/framework"
- "github.com/gardener/gardener/test/utils/shoots/access"
+ "github.com/gardener/gardener/test/utils/access"
)
type clients struct {
staticToken, adminKubeconfig, clientCert, serviceAccountDynamic, serviceAccountStatic kubernetes.Interface
}
-// ShootAccessVerifier uses the static token and admin kubeconfig to access the Shoot.
+// ShootAccessVerifier uses the various access methods to access the Shoot.
type ShootAccessVerifier struct {
*framework.ShootCreationFramework
@@ -61,7 +61,7 @@ func (v *ShootAccessVerifier) Before(ctx context.Context) {
By("Request new client certificate and using it to access shoot")
Eventually(func(g Gomega) {
- shootClient, err := access.CreateShootClientFromCSR(ctx, v.clientsBefore.adminKubeconfig, "e2e-rotate-csr-before")
+ shootClient, err := access.CreateTargetClientFromCSR(ctx, v.clientsBefore.adminKubeconfig, "e2e-rotate-csr-before")
g.Expect(err).NotTo(HaveOccurred())
g.Expect(shootClient.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed())
@@ -71,7 +71,7 @@ func (v *ShootAccessVerifier) Before(ctx context.Context) {
By("Request new dynamic token for a ServiceAccount and using it to access shoot")
Eventually(func(g Gomega) {
- shootClient, err := access.CreateShootClientFromDynamicServiceAccountToken(ctx, v.clientsBefore.adminKubeconfig, "e2e-rotate-sa-dynamic-before")
+ shootClient, err := access.CreateTargetClientFromDynamicServiceAccountToken(ctx, v.clientsBefore.adminKubeconfig, "e2e-rotate-sa-dynamic-before")
g.Expect(err).NotTo(HaveOccurred())
g.Expect(shootClient.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed())
@@ -142,7 +142,7 @@ func (v *ShootAccessVerifier) AfterPrepared(ctx context.Context) {
By("Request new client certificate and using it to access shoot")
Eventually(func(g Gomega) {
- shootClient, err := access.CreateShootClientFromCSR(ctx, v.clientsPrepared.adminKubeconfig, "e2e-rotate-csr-prepared")
+ shootClient, err := access.CreateTargetClientFromCSR(ctx, v.clientsPrepared.adminKubeconfig, "e2e-rotate-csr-prepared")
g.Expect(err).NotTo(HaveOccurred())
g.Expect(shootClient.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed())
@@ -152,7 +152,7 @@ func (v *ShootAccessVerifier) AfterPrepared(ctx context.Context) {
By("Request new dynamic token for a ServiceAccount and using it to access shoot")
Eventually(func(g Gomega) {
- shootClient, err := access.CreateShootClientFromDynamicServiceAccountToken(ctx, v.clientsPrepared.adminKubeconfig, "e2e-rotate-sa-dynamic-prepared")
+ shootClient, err := access.CreateTargetClientFromDynamicServiceAccountToken(ctx, v.clientsPrepared.adminKubeconfig, "e2e-rotate-sa-dynamic-prepared")
g.Expect(err).NotTo(HaveOccurred())
g.Expect(shootClient.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed())
@@ -248,7 +248,7 @@ func (v *ShootAccessVerifier) AfterCompleted(ctx context.Context) {
By("Request new client certificate and using it to access shoot")
Eventually(func(g Gomega) {
- shootClient, err := access.CreateShootClientFromCSR(ctx, v.clientsAfter.adminKubeconfig, "e2e-rotate-csr-after")
+ shootClient, err := access.CreateTargetClientFromCSR(ctx, v.clientsAfter.adminKubeconfig, "e2e-rotate-csr-after")
g.Expect(err).NotTo(HaveOccurred())
g.Expect(shootClient.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed())
@@ -258,7 +258,7 @@ func (v *ShootAccessVerifier) AfterCompleted(ctx context.Context) {
By("Request new dynamic token for a ServiceAccount and using it to access shoot")
Eventually(func(g Gomega) {
- shootClient, err := access.CreateShootClientFromDynamicServiceAccountToken(ctx, v.clientsAfter.adminKubeconfig, "e2e-rotate-sa-dynamic-after")
+ shootClient, err := access.CreateTargetClientFromDynamicServiceAccountToken(ctx, v.clientsAfter.adminKubeconfig, "e2e-rotate-sa-dynamic-after")
g.Expect(err).NotTo(HaveOccurred())
g.Expect(shootClient.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed())
@@ -307,7 +307,7 @@ func (v *ShootAccessVerifier) Cleanup(ctx context.Context) {
g.Expect(access.CleanupObjectsFromDynamicServiceAccountTokenAccess(ctx, shootClient)).To(Succeed())
}).Should(Succeed())
- By("Clean up objects in shoot from dynamic ServiceAccount token access")
+ By("Clean up objects in shoot from static ServiceAccount token access")
Eventually(func(g Gomega) {
g.Expect(access.CleanupObjectsFromStaticServiceAccountTokenAccess(ctx, shootClient)).To(Succeed())
}).Should(Succeed())
diff --git a/test/e2e/operator/garden/create_delete.go b/test/e2e/operator/garden/create_delete.go
index b379d8574c6..da8e94031b0 100644
--- a/test/e2e/operator/garden/create_delete.go
+++ b/test/e2e/operator/garden/create_delete.go
@@ -90,6 +90,7 @@ var _ = Describe("Garden Tests", Label("Garden", "default"), func() {
healthyManagedResource("vpa"),
healthyManagedResource("etcd-druid"),
healthyManagedResource("kube-state-metrics"),
+ healthyManagedResource("shoot-core-kube-controller-manager"),
healthyManagedResource("shoot-core-gardener-resource-manager"),
healthyManagedResource("shoot-core-gardeneraccess"),
))
diff --git a/test/e2e/operator/garden/create_rotate_delete.go b/test/e2e/operator/garden/create_rotate_delete.go
index 21fcb39101c..7bffa6c2c13 100644
--- a/test/e2e/operator/garden/create_rotate_delete.go
+++ b/test/e2e/operator/garden/create_rotate_delete.go
@@ -84,6 +84,7 @@ var _ = Describe("Garden Tests", Label("Garden", "default"), func() {
&rotationutils.SecretEncryptionVerifier{NewTargetClientFunc: func() (kubernetes.Interface, error) {
return kubernetes.NewClientFromSecret(ctx, runtimeClient, namespace, "gardener", kubernetes.WithDisabledCachedClient())
}},
+ &rotation.VirtualGardenAccessVerifier{RuntimeClient: runtimeClient, Namespace: namespace},
}
DeferCleanup(func() {
diff --git a/test/e2e/operator/garden/internal/rotation/virtual_garden_access.go b/test/e2e/operator/garden/internal/rotation/virtual_garden_access.go
new file mode 100644
index 00000000000..f2cf42ef4ff
--- /dev/null
+++ b/test/e2e/operator/garden/internal/rotation/virtual_garden_access.go
@@ -0,0 +1,172 @@
+// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rotation
+
+import (
+ "context"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+ corev1 "k8s.io/api/core/v1"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+
+ "github.com/gardener/gardener/pkg/client/kubernetes"
+ "github.com/gardener/gardener/test/utils/access"
+)
+
+type clients struct {
+ accessSecret, clientCert, serviceAccountDynamic kubernetes.Interface
+}
+
+// VirtualGardenAccessVerifier uses the various access methods to access the virtual garden.
+type VirtualGardenAccessVerifier struct {
+ RuntimeClient client.Client
+ Namespace string
+
+ clientsBefore, clientsPrepared, clientsAfter clients
+}
+
+// Before is called before the rotation is started.
+func (v *VirtualGardenAccessVerifier) Before(ctx context.Context) {
+ var err error
+ v.clientsBefore.accessSecret, err = kubernetes.NewClientFromSecret(ctx, v.RuntimeClient, v.Namespace, "gardener", kubernetes.WithDisabledCachedClient())
+ Expect(err).NotTo(HaveOccurred())
+
+ By("Request new client certificate and using it to access virtual garden")
+ Eventually(func(g Gomega) {
+ virtualGardenClient, err := access.CreateTargetClientFromCSR(ctx, v.clientsBefore.accessSecret, "e2e-rotate-csr-before")
+ g.Expect(err).NotTo(HaveOccurred())
+
+ g.Expect(virtualGardenClient.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed())
+
+ v.clientsBefore.clientCert = virtualGardenClient
+ }).Should(Succeed())
+
+ By("Request new dynamic token for a ServiceAccount and using it to access target cluster")
+ Eventually(func(g Gomega) {
+ virtualGardenClient, err := access.CreateTargetClientFromDynamicServiceAccountToken(ctx, v.clientsBefore.accessSecret, "e2e-rotate-sa-dynamic-before")
+ g.Expect(err).NotTo(HaveOccurred())
+
+ g.Expect(virtualGardenClient.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed())
+
+ v.clientsBefore.serviceAccountDynamic = virtualGardenClient
+ }).Should(Succeed())
+}
+
+// ExpectPreparingStatus is called while waiting for the Preparing status.
+func (v *VirtualGardenAccessVerifier) ExpectPreparingStatus(g Gomega) {}
+
+// AfterPrepared is called when the Shoot is in Prepared status.
+func (v *VirtualGardenAccessVerifier) AfterPrepared(ctx context.Context) {
+ By("Use client certificate from before rotation to access target cluster")
+ Eventually(func(g Gomega) {
+ g.Expect(v.clientsBefore.clientCert.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed())
+ }).Should(Succeed())
+
+ By("Use dynamic ServiceAccount token from before rotation to access target cluster")
+ Eventually(func(g Gomega) {
+ g.Expect(v.clientsBefore.serviceAccountDynamic.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed())
+ }).Should(Succeed())
+
+ var err error
+ v.clientsPrepared.accessSecret, err = kubernetes.NewClientFromSecret(ctx, v.RuntimeClient, v.Namespace, "gardener", kubernetes.WithDisabledCachedClient())
+ Expect(err).NotTo(HaveOccurred())
+
+ By("Request new client certificate and using it to access target cluster")
+ Eventually(func(g Gomega) {
+ virtualGardenClient, err := access.CreateTargetClientFromCSR(ctx, v.clientsPrepared.accessSecret, "e2e-rotate-csr-prepared")
+ g.Expect(err).NotTo(HaveOccurred())
+
+ g.Expect(virtualGardenClient.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed())
+
+ v.clientsPrepared.clientCert = virtualGardenClient
+ }).Should(Succeed())
+
+ By("Request new dynamic token for a ServiceAccount and using it to access target cluster")
+ Eventually(func(g Gomega) {
+ virtualGardenClient, err := access.CreateTargetClientFromDynamicServiceAccountToken(ctx, v.clientsPrepared.accessSecret, "e2e-rotate-sa-dynamic-prepared")
+ g.Expect(err).NotTo(HaveOccurred())
+
+ g.Expect(virtualGardenClient.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed())
+
+ v.clientsPrepared.serviceAccountDynamic = virtualGardenClient
+ }).Should(Succeed())
+}
+
+// ExpectCompletingStatus is called while waiting for the Completing status.
+func (v *VirtualGardenAccessVerifier) ExpectCompletingStatus(g Gomega) {}
+
+// AfterCompleted is called when the Shoot is in Completed status.
+func (v *VirtualGardenAccessVerifier) AfterCompleted(ctx context.Context) {
+ By("Use client certificate from before rotation to access target cluster")
+ Consistently(func(g Gomega) {
+ g.Expect(v.clientsBefore.clientCert.Client().List(ctx, &corev1.NamespaceList{})).NotTo(Succeed())
+ }).Should(Succeed())
+
+ By("Use dynamic ServiceAccount token from before rotation to access target cluster")
+ Consistently(func(g Gomega) {
+ g.Expect(v.clientsBefore.serviceAccountDynamic.Client().List(ctx, &corev1.NamespaceList{})).NotTo(Succeed())
+ }).Should(Succeed())
+
+ By("Use client certificate from after preparation to access target cluster")
+ Eventually(func(g Gomega) {
+ g.Expect(v.clientsPrepared.clientCert.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed())
+ }).Should(Succeed())
+
+ By("Use dynamic ServiceAccount token from after preparation to access target cluster")
+ Eventually(func(g Gomega) {
+ g.Expect(v.clientsPrepared.serviceAccountDynamic.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed())
+ }).Should(Succeed())
+
+ var err error
+ v.clientsAfter.accessSecret, err = kubernetes.NewClientFromSecret(ctx, v.RuntimeClient, v.Namespace, "gardener", kubernetes.WithDisabledCachedClient())
+ Expect(err).NotTo(HaveOccurred())
+
+ By("Request new client certificate and using it to access target cluster")
+ Eventually(func(g Gomega) {
+ virtualGardenClient, err := access.CreateTargetClientFromCSR(ctx, v.clientsAfter.accessSecret, "e2e-rotate-csr-after")
+ g.Expect(err).NotTo(HaveOccurred())
+
+ g.Expect(virtualGardenClient.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed())
+
+ v.clientsAfter.clientCert = virtualGardenClient
+ }).Should(Succeed())
+
+ By("Request new dynamic token for a ServiceAccount and using it to access target cluster")
+ Eventually(func(g Gomega) {
+ virtualGardenClient, err := access.CreateTargetClientFromDynamicServiceAccountToken(ctx, v.clientsAfter.accessSecret, "e2e-rotate-sa-dynamic-after")
+ g.Expect(err).NotTo(HaveOccurred())
+
+ g.Expect(virtualGardenClient.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed())
+
+ v.clientsAfter.serviceAccountDynamic = virtualGardenClient
+ }).Should(Succeed())
+}
+
+// Cleanup is passed to ginkgo.DeferCleanup.
+func (v *VirtualGardenAccessVerifier) Cleanup(ctx context.Context) {
+ virtualGardenClient, err := kubernetes.NewClientFromSecret(ctx, v.RuntimeClient, v.Namespace, "gardener", kubernetes.WithDisabledCachedClient())
+ Expect(err).NotTo(HaveOccurred())
+
+ By("Clean up objects in virtual garden from client certificate access")
+ Eventually(func(g Gomega) {
+ g.Expect(access.CleanupObjectsFromCSRAccess(ctx, virtualGardenClient)).To(Succeed())
+ }).Should(Succeed())
+
+ By("Clean up objects in shoot from dynamic ServiceAccount token access")
+ Eventually(func(g Gomega) {
+ g.Expect(access.CleanupObjectsFromDynamicServiceAccountTokenAccess(ctx, virtualGardenClient)).To(Succeed())
+ }).Should(Succeed())
+}
diff --git a/test/framework/k8s_utils.go b/test/framework/k8s_utils.go
index 9742a7c7b3b..934012c865a 100644
--- a/test/framework/k8s_utils.go
+++ b/test/framework/k8s_utils.go
@@ -39,7 +39,7 @@ import (
kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes"
"github.com/gardener/gardener/pkg/utils/kubernetes/health"
"github.com/gardener/gardener/pkg/utils/retry"
- "github.com/gardener/gardener/test/utils/shoots/access"
+ "github.com/gardener/gardener/test/utils/access"
)
// WaitUntilDaemonSetIsRunning waits until the daemon set with