diff --git a/charts/images.yaml b/charts/images.yaml index 354fe396647..976df8ec863 100644 --- a/charts/images.yaml +++ b/charts/images.yaml @@ -94,10 +94,6 @@ images: repository: eu.gcr.io/gardener-project/gardener/autoscaler/cluster-autoscaler tag: "v1.20.3" targetVersion: "< 1.21" -- name: vpn-seed - sourceRepository: github.com/gardener/vpn - repository: eu.gcr.io/gardener-project/gardener/vpn-seed - tag: "0.20.0" - name: vpn-seed-server sourceRepository: github.com/gardener/vpn2 repository: eu.gcr.io/gardener-project/gardener/vpn-seed-server @@ -138,10 +134,6 @@ images: tag: v0.6.2 # Shoot core addons -- name: vpn-shoot - sourceRepository: github.com/gardener/vpn - repository: eu.gcr.io/gardener-project/gardener/vpn-shoot - tag: "0.20.0" - name: vpn-shoot-client sourceRepository: github.com/gardener/vpn2 repository: eu.gcr.io/gardener-project/gardener/vpn-shoot-client @@ -206,10 +198,6 @@ images: - name: alpine repository: alpine tag: "3.15.4" -- name: alpine-iptables - sourceRepository: github.com/gardener/alpine-iptables - repository: eu.gcr.io/gardener-project/gardener/alpine-iptables - tag: "3.16.3" # Logging - name: fluent-bit diff --git a/charts/seed-monitoring/charts/core/charts/prometheus/templates/prometheus.yaml b/charts/seed-monitoring/charts/core/charts/prometheus/templates/prometheus.yaml index b525eaeb83c..22b18062693 100644 --- a/charts/seed-monitoring/charts/core/charts/prometheus/templates/prometheus.yaml +++ b/charts/seed-monitoring/charts/core/charts/prometheus/templates/prometheus.yaml @@ -59,9 +59,6 @@ spec: role: monitoring networking.gardener.cloud/to-dns: allowed networking.gardener.cloud/to-public-networks: allowed -{{- if not .Values.reversedVPN.enabled }} - networking.gardener.cloud/to-shoot-networks: allowed -{{- end }} networking.gardener.cloud/to-shoot-apiserver: allowed networking.gardener.cloud/to-seed-apiserver: allowed spec: diff --git a/charts/seed-monitoring/charts/core/charts/prometheus/values.yaml b/charts/seed-monitoring/charts/core/charts/prometheus/values.yaml index 1e44a9a0efc..d177d0f302a 100644 --- a/charts/seed-monitoring/charts/core/charts/prometheus/values.yaml +++ b/charts/seed-monitoring/charts/core/charts/prometheus/values.yaml @@ -24,9 +24,6 @@ secretNameClusterCA: ca secretNameEtcdCA: ca-etcd secretNameEtcdClientCert: etcd-client-tls -reversedVPN: - enabled: false - namespace: uid: 100c3bb5-48b9-4f88-96ef-48ed557d4212 diff --git a/cmd/gardenlet/app/app.go b/cmd/gardenlet/app/app.go index bcb9170a279..7008c8a8a73 100644 --- a/cmd/gardenlet/app/app.go +++ b/cmd/gardenlet/app/app.go @@ -57,7 +57,6 @@ import ( clientmapbuilder "github.com/gardener/gardener/pkg/client/kubernetes/clientmap/builder" "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/controllerutils/routes" - "github.com/gardener/gardener/pkg/features" "github.com/gardener/gardener/pkg/gardenlet/apis/config" confighelper "github.com/gardener/gardener/pkg/gardenlet/apis/config/helper" "github.com/gardener/gardener/pkg/gardenlet/bootstrap" @@ -131,11 +130,6 @@ func run(ctx context.Context, cancel context.CancelFunc, log logr.Logger, cfg *c } log.Info("Feature Gates", "featureGates", gardenletfeatures.FeatureGate.String()) - if gardenletfeatures.FeatureGate.Enabled(features.ReversedVPN) && !gardenletfeatures.FeatureGate.Enabled(features.APIServerSNI) { - return fmt.Errorf("inconsistent feature gate: APIServerSNI is required for ReversedVPN (APIServerSNI: %t, ReversedVPN: %t)", - gardenletfeatures.FeatureGate.Enabled(features.APIServerSNI), gardenletfeatures.FeatureGate.Enabled(features.ReversedVPN)) - } - if kubeconfig := os.Getenv("GARDEN_KUBECONFIG"); kubeconfig != "" { cfg.GardenClientConnection.Kubeconfig = kubeconfig } diff --git a/docs/deployment/feature_gates.md b/docs/deployment/feature_gates.md index 79ef2cc3866..7a05e0d0b17 100644 --- a/docs/deployment/feature_gates.md +++ b/docs/deployment/feature_gates.md @@ -30,8 +30,6 @@ The following tables are a summary of the feature gates that you can set on diff | APIServerSNI (deprecated) | `true` | `Beta` | `1.48` | | | SeedChange | `false` | `Alpha` | `1.12` | `1.52` | | SeedChange | `true` | `Beta` | `1.53` | | -| ReversedVPN | `false` | `Alpha` | `1.22` | `1.41` | -| ReversedVPN | `true` | `Beta` | `1.42` | | | CopyEtcdBackupsDuringControlPlaneMigration | `false` | `Alpha` | `1.37` | `1.52` | | CopyEtcdBackupsDuringControlPlaneMigration | `true` | `Beta` | `1.53` | | | ForceRestore | `false` | `Alpha` | `1.39` | | @@ -108,6 +106,9 @@ The following tables are a summary of the feature gates that you can set on diff | ShootSARotation | `true` | `Beta` | `1.51` | `1.56` | | ShootSARotation | `true` | `GA` | `1.57` | `1.59` | | ShootSARotation | | `Removed` | `1.60` | | +| ReversedVPN | `false` | `Alpha` | `1.22` | `1.41` | +| ReversedVPN | `true` | `Beta` | `1.42` | `1.62` | +| ReversedVPN | `true` | `GA` | `1.63` | | ## Using a feature diff --git a/docs/development/local_setup.md b/docs/development/local_setup.md index d75bf7e187f..f661a0f5b46 100644 --- a/docs/development/local_setup.md +++ b/docs/development/local_setup.md @@ -260,7 +260,7 @@ If your remote Garden cluster is a Gardener shoot, and you can access the seed o hack/local-development/remote-garden/enable-seed-authorizer ``` -> Note: The configuration changes introduced by this script result in a working `SeedAuthorization` feature only on shoots for which the `ReversedVPN` feature is not enabled. If the corresponding feature gate is enabled in `gardenlet`, add the annotation `alpha.featuregates.shoot.gardener.cloud/reversed-vpn: 'false'` to the remote Garden shoot to disable it for that particular shoot. +> Note: This script is not working anymore, as the `ReversedVPN` feature can't be disabled. The annotation `alpha.featuregates.shoot.gardener.cloud/reversed-vpn` on `Shoot`s is no longer respected. To prevent Gardener from reconciling the shoot and overwriting your changes, add the annotation `shoot.gardener.cloud/ignore: 'true'` to the remote Garden shoot. Note that this annotation takes effect only if it is enabled via the `constollers.shoot.respectSyncPeriodOverwrite: true` option in the `gardenlet` configuration. diff --git a/docs/extensions/provider-local.md b/docs/extensions/provider-local.md index dbd5ee4d0af..d3dc9f701e7 100644 --- a/docs/extensions/provider-local.md +++ b/docs/extensions/provider-local.md @@ -28,12 +28,7 @@ Please note that all of them are no technical limitations/blockers but simply ad 3. No load balancers for Shoot clusters. - _We have not yet developed a `cloud-controller-manager` which could reconcile load balancer `Service`s in the shoot cluster. Hence, when the gardenlet's `ReversedVPN` feature gate is disabled then the `kube-system/vpn-shoot` `Service` must be manually patched (with `{"status": {"loadBalancer": {"ingress": [{"hostname": "vpn-shoot"}]}}}`) to make the reconciliation work._ - -4. Only one shoot cluster possible when gardenlet's `APIServerSNI` feature gate is disabled. - - _When [`APIServerSNI`](../proposals/08-shoot-apiserver-via-sni.md) is disabled then gardenlet uses load balancer `Service`s in order to expose the shoot clusters' `kube-apiserver`s. Typically, local Kubernetes clusters don't support this. In this case, the local extension uses the host IP to expose the `kube-apiserver`, however, this can only be done once._\ - _However, given that the `APIServerSNI` feature gate is deprecated and will be removed in the future (see [gardener/gardener#6007](https://github.com/gardener/gardener/pull/6007)), we will probably not invest into this._ + _We have not yet developed a `cloud-controller-manager` which could reconcile load balancer `Service`s in the shoot cluster. 5. In case a seed cluster with multiple availability zones, i.e. multiple entries in `.spec.provider.zones`, is used in conjunction with a single-zone shoot control plane, i.e. a shoot cluster without `.spec.controlPlane.highAvailability` or with `.spec.controlPlane.highAvailability.failureTolerance.type` set to `node`, the local address of the API server endpoint needs to be determined manually or via the in cluster `coredns`. @@ -114,9 +109,6 @@ data: This controller generates a `NetworkPolicy` which allows the control plane pods (like `kube-apiserver`) to communicate with the worker machine pods (see [`Worker` section](#worker))). -In addition, it creates a `Service` named `vpn-shoot` which is only used in case the gardenlet's `ReversedVPN` feature gate is disabled. -This `Service` enables the `vpn-seed` containers in the `kube-apiserver` pods in the seed cluster to communicate with the `vpn-shoot` pod running in the shoot cluster. - #### `Network` This controller is not implemented anymore. In the initial version of `provider-local`, there was a `Network` controller deploying [kindnetd](https://github.com/kubernetes-sigs/kind/blob/main/images/kindnetd/README.md) (see https://github.com/gardener/gardener/tree/v1.44.1/pkg/provider-local/controller/network). diff --git a/docs/usage/reversed-vpn-tunnel.md b/docs/usage/reversed-vpn-tunnel.md index 54e9184b12c..812734599a4 100644 --- a/docs/usage/reversed-vpn-tunnel.md +++ b/docs/usage/reversed-vpn-tunnel.md @@ -4,52 +4,23 @@ title: Reversed VPN Tunnel # Reversed VPN Tunnel Setup and Configuration -This is a short guide describing how to enable tunneling traffic from shoot cluster to seed cluster instead of the default "seed to shoot" direction. +The Reversed VPN Tunnel is enabled by default. +A highly available VPN connection is automatically deployed in all shoots that configure an HA control-plane. -## The OpenVPN Default +## Reversed VPN Tunnel -By default, Gardener makes use of OpenVPN to connect the shoot controlplane running on the seed cluster to the dataplane -running on the shoot worker nodes, usually in isolated networks. This is achieved by having a sidecar to certain control plane components such as the `kube-apiserver` and `prometheus`. +In the first VPN solution, connection establishment was initiated by a VPN client in the seed cluster. +Due to several issues with this solution, the tunnel establishment direction has been reverted. +The client is deployed in the shoot and initiates the connection from there. This way, there is no need to deploy a special purpose +loadbalancer for the sake of addressing the data-plane, in addition to saving costs, this is considered the more secure alternative. +For more information on how this is achieved, please have a look at the following [GEP](../proposals/14-reversed-cluster-vpn.md). -With a sidecar, all traffic directed to the cluster is intercepted by iptables rules and redirected -to the tunnel endpoint in the shoot cluster deployed behind a cloud loadbalancer. This has the following disadvantages: - -- Every shoot would require an additional loadbalancer, this accounts for additional overhead in terms of both costs and troubleshooting efforts. -- Private access use-cases would not be possible without having a seed residing in the same private domain as a hard requirement. For example, have a look at [this issue](https://github.com/gardener/gardener-extension-provider-gcp/issues/56) -- Providing a public endpoint to access components in the shoot poses a security risk. - -This is how it looks like today with the OpenVPN solution: - -`APIServer | VPN-seed ---> internet ---> LB --> VPN-Shoot (4314) --> Pods | Nodes | Services` - - -## Reversing the Tunnel - -To address the above issues, the tunnel can establishment direction can be reverted, i.e. instead of having the client reside in the seed, -we deploy the client in the shoot and initiate the connection from there. This way, there is no need to deploy a special purpose -loadbalancer for the sake of addressing the dataplane, in addition to saving costs, this is considered the more secure alternative. -For more information on how this is achieved, please have a look at the following [GEP](../proposals/14-reversed-cluster-vpn.md). - -How it should look like at the end: +Connection establishment with a reversed tunnel: `APIServer --> Envoy-Proxy | VPN-Seed-Server <-- Istio/Envoy-Proxy <-- SNI API Server Endpoint <-- LB (one for all clusters of a seed) <--- internet <--- VPN-Shoot-Client --> Pods | Nodes | Services` -### How to Configure - -To enable the usage of the reversed vpn tunnel feature, either the Gardenlet `ReversedVPN` feature-gate must be set to `true` as shown below or the shoot must be annotated with `"alpha.featuregates.shoot.gardener.cloud/reversed-vpn: true"`. - -```yaml -featureGates: - ReversedVPN: true -``` -Please refer to the examples [here](https://github.com/gardener/gardener/blob/master/example/20-componentconfig-gardenlet.yaml) for more information. - -To disable the feature-gate the shoot must be annotated with `"alpha.featuregates.shoot.gardener.cloud/reversed-vpn: false"` - -Once the feature-gate is enabled, a `vpn-seed-server` deployment will be added to the controlplane. The `kube-apiserver` will be configured to connect to resources in the dataplane such as pods, services and nodes though the `vpn-seed-service` via http proxy/connect protocol. -In the dataplane of the cluster, the `vpn-shoot` will establish the connection to the `vpn-seed-server` indirectly using the SNI API Server endpoint as a http proxy. After the connection has been established requests from the `kube-apiserver` will be handled by the tunnel. - -> Please note this feature is still in Beta, so you might see instabilities every now and then. +The reversed VPN tunnel is always deployed. +The feature gate `ReversedVPN` is GA and will be removed in a future release. ## High Availability for Reversed VPN Tunnel diff --git a/example/20-componentconfig-gardenlet.yaml b/example/20-componentconfig-gardenlet.yaml index 7d1414c0f01..ced4b3cf30c 100644 --- a/example/20-componentconfig-gardenlet.yaml +++ b/example/20-componentconfig-gardenlet.yaml @@ -118,7 +118,6 @@ featureGates: HVPAForShootedSeed: true ManagedIstio: true APIServerSNI: true - ReversedVPN: true CopyEtcdBackupsDuringControlPlaneMigration: true ForceRestore: false DefaultSeccompProfile: true diff --git a/example/gardener-local/gardenlet/values.yaml b/example/gardener-local/gardenlet/values.yaml index ce6445c7dc8..694954f2e91 100644 --- a/example/gardener-local/gardenlet/values.yaml +++ b/example/gardener-local/gardenlet/values.yaml @@ -33,7 +33,6 @@ config: HVPAForShootedSeed: true ManagedIstio: true APIServerSNI: true - ReversedVPN: true CopyEtcdBackupsDuringControlPlaneMigration: true DefaultSeccompProfile: true CoreDNSQueryRewriting: true diff --git a/extensions/pkg/webhook/controlplane/genericmutator/mutator.go b/extensions/pkg/webhook/controlplane/genericmutator/mutator.go index 4e9d46d8f97..8c5a6da165b 100644 --- a/extensions/pkg/webhook/controlplane/genericmutator/mutator.go +++ b/extensions/pkg/webhook/controlplane/genericmutator/mutator.go @@ -63,8 +63,7 @@ type Ensurer interface { // "old" might be "nil" and must always be checked. EnsureETCD(ctx context.Context, gctx gcontext.GardenContext, new, old *druidv1alpha1.Etcd) error // EnsureVPNSeedServerDeployment ensures that the vpn-seed-server deployment conforms to the provider requirements. - // "old" might be "nil" and must always be checked. Please note that the vpn-seed-server deployment will only exist - // if the gardenlet's ReversedVPN feature gate is enabeld. + // "old" might be "nil" and must always be checked. EnsureVPNSeedServerDeployment(ctx context.Context, gctx gcontext.GardenContext, new, old *appsv1.Deployment) error // EnsureKubeletServiceUnitOptions ensures that the kubelet.service unit options conform to the provider requirements. EnsureKubeletServiceUnitOptions(ctx context.Context, gctx gcontext.GardenContext, kubeletVersion *semver.Version, new, old []*unit.UnitOption) ([]*unit.UnitOption, error) diff --git a/hack/.ci/set_dependency_version b/hack/.ci/set_dependency_version index d7e17569e36..bcb1508974d 100755 --- a/hack/.ci/set_dependency_version +++ b/hack/.ci/set_dependency_version @@ -116,8 +116,6 @@ if name in injectedSpecialCases: names = injectedSpecialCases[name] elif name == 'autoscaler': names = ['cluster-autoscaler'] -elif name == 'vpn': - names = ['vpn-seed', 'vpn-shoot'] elif name == 'vpn2': names = ['vpn-seed-server', 'vpn-shoot-client'] elif name == 'external-dns-management': diff --git a/pkg/apis/core/v1beta1/constants/types_constants.go b/pkg/apis/core/v1beta1/constants/types_constants.go index 0d64f5cfa45..8bc3abdc39f 100644 --- a/pkg/apis/core/v1beta1/constants/types_constants.go +++ b/pkg/apis/core/v1beta1/constants/types_constants.go @@ -557,8 +557,6 @@ const ( AnnotationShootCloudConfigExecutionMaxDelaySeconds = "shoot.gardener.cloud/cloud-config-execution-max-delay-seconds" // AnnotationShootForceRestore is a key for an annotation on a Shoot or BackupEntry resource to trigger a forceful restoration to a different seed. AnnotationShootForceRestore = "shoot.gardener.cloud/force-restore" - // AnnotationReversedVPN moves the vpn-server to the seed. - AnnotationReversedVPN = "alpha.featuregates.shoot.gardener.cloud/reversed-vpn" // AnnotationNodeLocalDNS enables a per node dns cache on the shoot cluster. AnnotationNodeLocalDNS = "alpha.featuregates.shoot.gardener.cloud/node-local-dns" // AnnotationNodeLocalDNSForceTcpToClusterDns enforces upgrade to tcp connections for communication between node local and cluster dns. diff --git a/pkg/features/features.go b/pkg/features/features.go index 04de34ea9ea..14b06d12662 100644 --- a/pkg/features/features.go +++ b/pkg/features/features.go @@ -67,6 +67,7 @@ const ( // owner: @ScheererJ @DockToFuture // alpha: v1.22.0 // beta: v1.42.0 + // GA: v1.63.0 ReversedVPN featuregate.Feature = "ReversedVPN" // CopyEtcdBackupsDuringControlPlaneMigration enables the copy of etcd backups from the object store of the source seed @@ -103,10 +104,10 @@ const ( var allFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ HVPA: {Default: false, PreRelease: featuregate.Alpha}, HVPAForShootedSeed: {Default: false, PreRelease: featuregate.Alpha}, - ManagedIstio: {Default: true, PreRelease: featuregate.Deprecated}, - APIServerSNI: {Default: true, PreRelease: featuregate.Deprecated}, + ManagedIstio: {Default: true, PreRelease: featuregate.Deprecated, LockToDefault: true}, + APIServerSNI: {Default: true, PreRelease: featuregate.Deprecated, LockToDefault: true}, SeedChange: {Default: true, PreRelease: featuregate.Beta}, - ReversedVPN: {Default: true, PreRelease: featuregate.Beta}, + ReversedVPN: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, CopyEtcdBackupsDuringControlPlaneMigration: {Default: true, PreRelease: featuregate.Beta}, ForceRestore: {Default: false, PreRelease: featuregate.Alpha}, HAControlPlanes: {Default: false, PreRelease: featuregate.Alpha}, diff --git a/pkg/gardenlet/controller/managedseed/valueshelper_test.go b/pkg/gardenlet/controller/managedseed/valueshelper_test.go index ef1df31e376..0ca40f217e7 100644 --- a/pkg/gardenlet/controller/managedseed/valueshelper_test.go +++ b/pkg/gardenlet/controller/managedseed/valueshelper_test.go @@ -105,8 +105,8 @@ var _ = Describe("ValuesHelper", func() { }, }, FeatureGates: map[string]bool{ - string(features.ReversedVPN): true, - string(features.HVPA): true, + string("FooFeature"): true, + string("BarFeature"): true, }, Logging: &config.Logging{ Enabled: pointer.Bool(true), @@ -146,7 +146,7 @@ var _ = Describe("ValuesHelper", func() { Kind: "GardenletConfiguration", }, FeatureGates: map[string]bool{ - string(features.ReversedVPN): false, + "FooFeature": false, }, } shoot = &gardencorev1beta1.Shoot{ @@ -224,8 +224,8 @@ var _ = Describe("ValuesHelper", func() { }, }, FeatureGates: map[string]bool{ - string(features.ReversedVPN): false, - string(features.HVPA): true, + string("FooFeature"): false, + string("BarFeature"): true, }, Logging: &configv1alpha1.Logging{ Enabled: pointer.Bool(true), @@ -281,8 +281,8 @@ var _ = Describe("ValuesHelper", func() { }, }, "featureGates": map[string]interface{}{ - "ReversedVPN": false, - "HVPA": true, + "FooFeature": false, + "BarFeature": true, }, "logging": map[string]interface{}{ "enabled": true, diff --git a/pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go b/pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go index 2b9e2aeb4cc..f5cd34e2699 100644 --- a/pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go +++ b/pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go @@ -511,7 +511,7 @@ func (r *Reconciler) runReconcileShootFlow(ctx context.Context, o *operation.Ope }) deployVPNShoot = g.Add(flow.Task{ Name: "Deploying vpn-shoot system component", - Fn: flow.TaskFn(botanist.DeployVPNShoot).RetryUntilTimeout(defaultInterval, defaultTimeout).SkipIf(o.Shoot.HibernationEnabled), + Fn: flow.TaskFn(botanist.Shoot.Components.SystemComponents.VPNShoot.Deploy).RetryUntilTimeout(defaultInterval, defaultTimeout).SkipIf(o.Shoot.HibernationEnabled), Dependencies: flow.NewTaskIDs(deployGardenerResourceManager, deployKubeScheduler, deployVPNSeedServer, waitUntilShootNamespacesReady), }) deployNodeProblemDetector = g.Add(flow.Task{ @@ -592,15 +592,10 @@ func (r *Reconciler) runReconcileShootFlow(ctx context.Context, o *operation.Ope Fn: flow.TaskFn(botanist.CleanupOrphanedDNSRecordSecrets).DoIf(!o.Shoot.HibernationEnabled), Dependencies: flow.NewTaskIDs(deployInternalDomainDNSRecord, deployExternalDomainDNSRecord, deployOwnerDomainDNSRecord, deployIngressDomainDNSRecord), }) - vpnLBReady = g.Add(flow.Task{ - Name: "Waiting until vpn-shoot LoadBalancer is ready", - Fn: flow.TaskFn(botanist.WaitUntilVpnShootServiceIsReady).SkipIf(o.Shoot.HibernationEnabled || o.Shoot.ReversedVPNEnabled), - Dependencies: flow.NewTaskIDs(syncPointAllSystemComponentsDeployed, waitUntilNetworkIsReady, waitUntilWorkerReady), - }) waitUntilTunnelConnectionExists = g.Add(flow.Task{ Name: "Waiting until the Kubernetes API server can connect to the Shoot workers", Fn: flow.TaskFn(botanist.WaitUntilTunnelConnectionExists).SkipIf(o.Shoot.HibernationEnabled), - Dependencies: flow.NewTaskIDs(syncPointAllSystemComponentsDeployed, waitUntilNetworkIsReady, waitUntilWorkerReady, vpnLBReady), + Dependencies: flow.NewTaskIDs(syncPointAllSystemComponentsDeployed, waitUntilNetworkIsReady, waitUntilWorkerReady), }) _ = g.Add(flow.Task{ Name: "Waiting until all shoot worker nodes have updated the cloud config user data", diff --git a/pkg/operation/botanist/component/kubeapiserver/configmaps.go b/pkg/operation/botanist/component/kubeapiserver/configmaps.go index 12d926adf9d..b07127284fe 100644 --- a/pkg/operation/botanist/component/kubeapiserver/configmaps.go +++ b/pkg/operation/botanist/component/kubeapiserver/configmaps.go @@ -101,7 +101,7 @@ func (k *kubeAPIServer) reconcileConfigMapAuditPolicy(ctx context.Context, confi } func (k *kubeAPIServer) reconcileConfigMapEgressSelector(ctx context.Context, configMap *corev1.ConfigMap) error { - if !k.values.VPN.ReversedVPNEnabled || k.values.VPN.HighAvailabilityEnabled { + if k.values.VPN.HighAvailabilityEnabled { // We don't delete the confimap here as we don't know its name (as it's unique). Instead, we rely on the usual // garbage collection for unique secrets/configmaps. return nil diff --git a/pkg/operation/botanist/component/kubeapiserver/deployment.go b/pkg/operation/botanist/component/kubeapiserver/deployment.go index b8d16ddfb8a..3161838f80c 100644 --- a/pkg/operation/botanist/component/kubeapiserver/deployment.go +++ b/pkg/operation/botanist/component/kubeapiserver/deployment.go @@ -42,9 +42,6 @@ import ( ) const ( - // SecretNameVPNSeedTLSAuth is the name of the secret containing the TLS auth for the vpn-seed. - SecretNameVPNSeedTLSAuth = "vpn-seed-tlsauth" - secretNameLegacyVPNSeed = "vpn-seed" secretNameHAVPNSeedClient = "vpn-seed-client" secretNameServer = "kube-apiserver" @@ -54,7 +51,6 @@ const ( // ContainerNameKubeAPIServer is the name of the kube-apiserver container. ContainerNameKubeAPIServer = "kube-apiserver" - containerNameVPNSeed = "vpn-seed" containerNameVPNSeedClient = "vpn-client" containerNameAPIServerProxyPodMutator = "apiserver-proxy-pod-mutator" @@ -72,14 +68,12 @@ const ( volumeNameHTTPProxy = "http-proxy" volumeNameKubeAPIServerToKubelet = "kubelet-client" volumeNameKubeAggregator = "kube-aggregator" - volumeNameLibModules = "modules" volumeNameOIDCCABundle = "oidc-cabundle" volumeNameServer = "kube-apiserver-server" volumeNameServiceAccountKey = "service-account-key" volumeNameServiceAccountKeyBundle = "service-account-key-bundle" volumeNameUserProvidedServiceAccountSigningKey = "service-account-signing-key" volumeNameStaticToken = "static-token" - volumeNameVPNSeed = "vpn-seed" volumeNameVPNSeedClient = "vpn-seed-client" volumeNameAPIServerAccess = "kube-api-access-gardener" volumeNameVPNSeedTLSAuth = "vpn-seed-tlsauth" @@ -110,7 +104,6 @@ const ( volumeMountPathServiceAccountKeyBundle = "/srv/kubernetes/service-account-key-bundle" volumeMountPathUserProvidedServiceAccountSigningKey = "/srv/kubernetes/service-account-signing-key" volumeMountPathStaticToken = "/srv/kubernetes/token" - volumeMountPathVPNSeed = "/srv/secrets/vpn-seed" volumeMountPathVPNSeedClient = "/srv/secrets/vpn-client" volumeMountPathAPIServerAccess = "/var/run/secrets/kubernetes.io/serviceaccount" volumeMountPathVPNSeedTLSAuth = "/srv/secrets/tlsauth" @@ -140,8 +133,6 @@ func (k *kubeAPIServer) reconcileDeployment( secretKubeletClient *corev1.Secret, secretKubeAggregator *corev1.Secret, secretHTTPProxy *corev1.Secret, - secretLegacyVPNSeed *corev1.Secret, - secretLegacyVPNSeedTLSAuth *corev1.Secret, secretHAVPNSeedClient *corev1.Secret, secretHAVPNSeedClientSeedTLSAuth *corev1.Secret, ) error { @@ -484,7 +475,7 @@ func (k *kubeAPIServer) reconcileDeployment( k.handleHostCertVolumes(deployment) k.handleSNISettings(deployment) k.handlePodMutatorSettings(deployment) - k.handleVPNSettings(deployment, configMapEgressSelector, secretCAVPN, secretHTTPProxy, secretCAClient, secretLegacyVPNSeed, secretLegacyVPNSeedTLSAuth, secretHAVPNSeedClient, secretHAVPNSeedClientSeedTLSAuth) + k.handleVPNSettings(deployment, configMapEgressSelector, secretCAVPN, secretHTTPProxy, secretHAVPNSeedClient, secretHAVPNSeedClientSeedTLSAuth) k.handleOIDCSettings(deployment, secretOIDCCABundle) k.handleServiceAccountSigningKeySettings(deployment, secretUserProvidedServiceAccountSigningKey) @@ -733,182 +724,17 @@ func (k *kubeAPIServer) handleVPNSettings( configMapEgressSelector *corev1.ConfigMap, secretCAVPN *corev1.Secret, secretHTTPProxy *corev1.Secret, - secretLegacyVPNCAClient *corev1.Secret, - secretLegacyVPNSeed *corev1.Secret, - secretLegacyVPNSeedTLSAuth *corev1.Secret, secretHAVPNSeedClient *corev1.Secret, secretHAVPNSeedClientSeedTLSAuth *corev1.Secret, ) { - if !k.values.VPN.ReversedVPNEnabled { - k.handleVPNSettingsLegacy(deployment, secretLegacyVPNCAClient, secretLegacyVPNSeed, secretLegacyVPNSeedTLSAuth) - } else if k.values.VPN.HighAvailabilityEnabled { + + if k.values.VPN.HighAvailabilityEnabled { k.handleVPNSettingsHAReversedVPN(deployment, secretCAVPN, secretHAVPNSeedClient, secretHAVPNSeedClientSeedTLSAuth) } else { k.handleVPNSettingsReversedVPN(deployment, configMapEgressSelector, secretCAVPN, secretHTTPProxy) } } -func (k *kubeAPIServer) handleVPNSettingsLegacy( - deployment *appsv1.Deployment, - secretLegacyVPNCAClient *corev1.Secret, - secretLegacyVPNSeed *corev1.Secret, - secretLegacyVPNSeedTLSAuth *corev1.Secret, -) { - deployment.Spec.Template.Labels[v1beta1constants.LabelNetworkPolicyToShootNetworks] = v1beta1constants.LabelNetworkPolicyAllowed - deployment.Spec.Template.Spec.InitContainers = []corev1.Container{{ - Name: "set-iptable-rules", - Image: k.values.Images.AlpineIPTables, - Command: []string{ - "/bin/sh", - "-c", - "iptables -A INPUT -i tun0 -p icmp -j ACCEPT && iptables -A INPUT -i tun0 -m state --state NEW -j DROP", - }, - SecurityContext: &corev1.SecurityContext{ - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{"NET_ADMIN"}, - }, - Privileged: pointer.Bool(true), - }, - VolumeMounts: []corev1.VolumeMount{{ - Name: volumeNameLibModules, - MountPath: volumeMountPathLibModules, - }}, - }} - deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, corev1.Volume{ - Name: volumeNameLibModules, - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}, - }, - }) - - vpnSeedContainer := corev1.Container{ - Name: containerNameVPNSeed, - Image: k.values.Images.VPNSeed, - ImagePullPolicy: corev1.PullIfNotPresent, - Env: []corev1.EnvVar{ - { - Name: "MAIN_VPN_SEED", - Value: "true", - }, - { - Name: "OPENVPN_PORT", - Value: "4314", - }, - { - Name: "APISERVER_AUTH_MODE", - Value: "client-cert", - }, - { - Name: "APISERVER_AUTH_MODE_CLIENT_CERT_CA", - Value: volumeMountPathCA + "/" + secrets.DataKeyCertificateBundle, - }, - { - Name: "APISERVER_AUTH_MODE_CLIENT_CERT_CRT", - Value: volumeMountPathVPNSeed + "/" + secrets.DataKeyCertificate, - }, - { - Name: "APISERVER_AUTH_MODE_CLIENT_CERT_KEY", - Value: volumeMountPathVPNSeed + "/" + secrets.DataKeyPrivateKey, - }, - { - Name: "SERVICE_NETWORK", - Value: k.values.VPN.ServiceNetworkCIDR, - }, - { - Name: "POD_NETWORK", - Value: k.values.VPN.PodNetworkCIDR, - }, - }, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("100m"), - corev1.ResourceMemory: resource.MustParse("128Mi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("128Mi"), - }, - }, - SecurityContext: &corev1.SecurityContext{ - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{"NET_ADMIN"}, - }, - Privileged: pointer.Bool(true), - }, - TerminationMessagePath: corev1.TerminationMessagePathDefault, - TerminationMessagePolicy: corev1.TerminationMessageReadFile, - VolumeMounts: []corev1.VolumeMount{ - { - Name: volumeNameVPNSeed, - MountPath: volumeMountPathVPNSeed, - }, - { - Name: volumeNameVPNSeedTLSAuth, - MountPath: volumeMountPathVPNSeedTLSAuth, - }, - { - Name: volumeNameCA, - MountPath: volumeMountPathCA, - }, - }, - } - - if k.values.VPN.NodeNetworkCIDR != nil { - vpnSeedContainer.Env = append(vpnSeedContainer.Env, corev1.EnvVar{ - Name: "NODE_NETWORK", - Value: *k.values.VPN.NodeNetworkCIDR, - }) - } - - deployment.Spec.Template.Spec.Containers = append(deployment.Spec.Template.Spec.Containers, vpnSeedContainer) - deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, []corev1.Volume{ - { - Name: volumeNameVPNSeed, - VolumeSource: corev1.VolumeSource{ - Projected: &corev1.ProjectedVolumeSource{ - DefaultMode: pointer.Int32(400), - Sources: []corev1.VolumeProjection{ - { - Secret: &corev1.SecretProjection{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: secretLegacyVPNCAClient.Name, - }, - Items: []corev1.KeyToPath{{ - Key: secrets.DataKeyCertificateBundle, - Path: secrets.DataKeyCertificateCA, - }}, - }, - }, - { - Secret: &corev1.SecretProjection{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: secretLegacyVPNSeed.Name, - }, - Items: []corev1.KeyToPath{ - { - Key: secrets.DataKeyCertificate, - Path: secrets.DataKeyCertificate, - }, - { - Key: secrets.DataKeyPrivateKey, - Path: secrets.DataKeyPrivateKey, - }, - }, - }, - }, - }, - }, - }, - }, - { - Name: volumeNameVPNSeedTLSAuth, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{SecretName: secretLegacyVPNSeedTLSAuth.Name}, - }, - }, - }...) - -} - func (k *kubeAPIServer) handleVPNSettingsReversedVPN( deployment *appsv1.Deployment, configMapEgressSelector *corev1.ConfigMap, diff --git a/pkg/operation/botanist/component/kubeapiserver/hvpa.go b/pkg/operation/botanist/component/kubeapiserver/hvpa.go index eeb1f334c05..65d2d394f55 100644 --- a/pkg/operation/botanist/component/kubeapiserver/hvpa.go +++ b/pkg/operation/botanist/component/kubeapiserver/hvpa.go @@ -72,11 +72,6 @@ func (k *kubeAPIServer) reconcileHVPA(ctx context.Context, hvpa *hvpav1alpha1.Hv }, ControlledValues: &controlledValues, }, - { - ContainerName: containerNameVPNSeed, - Mode: &containerPolicyOff, - ControlledValues: &controlledValues, - }, } weightBasedScalingIntervals = []hvpav1alpha1.WeightBasedScalingInterval{ { diff --git a/pkg/operation/botanist/component/kubeapiserver/kube_apiserver.go b/pkg/operation/botanist/component/kubeapiserver/kube_apiserver.go index 44deb94d075..bd8e0d3e115 100644 --- a/pkg/operation/botanist/component/kubeapiserver/kube_apiserver.go +++ b/pkg/operation/botanist/component/kubeapiserver/kube_apiserver.go @@ -82,8 +82,6 @@ const ( SecretNameUserKubeconfig = "user-kubeconfig" // ServicePortName is the name of the port in the service. ServicePortName = "kube-apiserver" - // UserNameVPNSeed is the user name for the vpn-seed components (used as common name in its client certificate) - UserNameVPNSeed = "vpn-seed" // UserNameVPNSeedClient is the user name for the HA vpn-seed-client components (used as common name in its client certificate) UserNameVPNSeedClient = "vpn-seed-client" @@ -208,29 +206,23 @@ type ETCDEncryptionConfig struct { // Images is a set of container images used for the containers of the kube-apiserver pods. type Images struct { - // AlpineIPTables is the container image for alpine-iptables. - AlpineIPTables string // APIServerProxyPodWebhook is the container image for the apiserver-proxy-pod-webhook. APIServerProxyPodWebhook string // KubeAPIServer is the container image for the kube-apiserver. KubeAPIServer string - // VPNSeed is the container image for the vpn-seed. - VPNSeed string // VPNClient is the container image for the vpn-seed-client. VPNClient string } // VPNConfig contains information for configuring the VPN settings for the kube-apiserver. type VPNConfig struct { - // ReversedVPNEnabled states whether the 'ReversedVPN' feature gate is enabled. - ReversedVPNEnabled bool // PodNetworkCIDR is the CIDR of the pod network. PodNetworkCIDR string // ServiceNetworkCIDR is the CIDR of the service network. ServiceNetworkCIDR string // NodeNetworkCIDR is the CIDR of the node network. NodeNetworkCIDR *string - // HighAvailabilityEnabled states if VPN uses HA configuration (only works together with ReversedVPNEnabled=true) + // HighAvailabilityEnabled states if VPN uses HA configuration. HighAvailabilityEnabled bool // HighAvailabilityNumberOfSeedServers is the number of VPN seed servers used for HA HighAvailabilityNumberOfSeedServers int @@ -394,16 +386,6 @@ func (k *kubeAPIServer) Deploy(ctx context.Context) error { return err } - secretLegacyVPNSeed, err := k.reconcileSecretLegacyVPNSeed(ctx) - if err != nil { - return err - } - - secretLegacyVPNSeedTLSAuth, err := k.reconcileSecretLegacyVPNSeedTLSAuth(ctx) - if err != nil { - return err - } - secretHAVPNSeedClient, err := k.reconcileSecretHAVPNSeedClient(ctx) if err != nil { return err @@ -450,8 +432,6 @@ func (k *kubeAPIServer) Deploy(ctx context.Context) error { secretKubeletClient, secretKubeAggregator, secretHTTPProxy, - secretLegacyVPNSeed, - secretLegacyVPNSeedTLSAuth, secretHAVPNSeedClient, secretHAVPNClientSeedTLSAuth, ); err != nil { diff --git a/pkg/operation/botanist/component/kubeapiserver/kube_apiserver_test.go b/pkg/operation/botanist/component/kubeapiserver/kube_apiserver_test.go index 1bcd0091469..25ec111ea7e 100644 --- a/pkg/operation/botanist/component/kubeapiserver/kube_apiserver_test.go +++ b/pkg/operation/botanist/component/kubeapiserver/kube_apiserver_test.go @@ -102,8 +102,6 @@ var _ = Describe("KubeAPIServer", func() { secretNameServer = "kube-apiserver" secretNameServiceAccountKey = "service-account-key-c37a87f6" secretNameServiceAccountKeyBundle = "service-account-key-bundle" - secretNameVPNSeed = "vpn-seed" - secretNameVPNSeedTLSAuth = "vpn-seed-tlsauth-de1d12a3" secretNameVPNSeedClient = "vpn-seed-client" secretNameAdmissionConfig = "kube-apiserver-admission-config-e38ff146" @@ -460,11 +458,6 @@ var _ = Describe("KubeAPIServer", func() { }, ControlledValues: &controlledValues, }, - { - ContainerName: "vpn-seed", - Mode: &containerPolicyOff, - ControlledValues: &controlledValues, - }, } defaultExpectedWeightBasedScalingIntervals = []hvpav1alpha1.WeightBasedScalingInterval{ { @@ -681,11 +674,6 @@ var _ = Describe("KubeAPIServer", func() { }, ControlledValues: &controlledValues, }, - { - ContainerName: "vpn-seed", - Mode: &containerPolicyOff, - ControlledValues: &controlledValues, - }, { ContainerName: "apiserver-proxy-pod-mutator", Mode: &containerPolicyOff, @@ -891,87 +879,8 @@ var _ = Describe("KubeAPIServer", func() { portVPNSeedServer = intstr.FromInt(9443) ) - It("w/o ReversedVPN", func() { - Expect(c.Get(ctx, client.ObjectKeyFromObject(networkPolicyAllowKubeAPIServer), networkPolicyAllowKubeAPIServer)).To(MatchError(apierrors.NewNotFound(schema.GroupResource{Group: networkingv1.SchemeGroupVersion.Group, Resource: "networkpolicies"}, networkPolicyAllowKubeAPIServer.Name))) - Expect(kapi.Deploy(ctx)).To(Succeed()) - Expect(c.Get(ctx, client.ObjectKeyFromObject(networkPolicyAllowKubeAPIServer), networkPolicyAllowKubeAPIServer)).To(Succeed()) - Expect(networkPolicyAllowKubeAPIServer).To(DeepEqual(&networkingv1.NetworkPolicy{ - TypeMeta: metav1.TypeMeta{ - APIVersion: networkingv1.SchemeGroupVersion.String(), - Kind: "NetworkPolicy", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: networkPolicyAllowKubeAPIServer.Name, - Namespace: networkPolicyAllowKubeAPIServer.Namespace, - ResourceVersion: "1", - Annotations: map[string]string{ - "gardener.cloud/description": "Allows Ingress to the Shoot's Kubernetes API Server from " + - "pods labeled with 'networking.gardener.cloud/to-shoot-apiserver=allowed' and " + - "Prometheus, and Egress to etcd pods.", - }, - }, - Spec: networkingv1.NetworkPolicySpec{ - PodSelector: metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "kubernetes", - "gardener.cloud/role": "controlplane", - "role": "apiserver", - }, - }, - Egress: []networkingv1.NetworkPolicyEgressRule{{ - To: []networkingv1.NetworkPolicyPeer{{ - PodSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "etcd-statefulset", - "gardener.cloud/role": "controlplane", - }, - }, - }}, - Ports: []networkingv1.NetworkPolicyPort{{ - Protocol: &protocol, - Port: &portEtcd, - }}, - }}, - Ingress: []networkingv1.NetworkPolicyIngressRule{ - { - From: []networkingv1.NetworkPolicyPeer{ - {PodSelector: &metav1.LabelSelector{}, NamespaceSelector: &metav1.LabelSelector{}}, - {IPBlock: &networkingv1.IPBlock{CIDR: "0.0.0.0/0"}}, - }, - Ports: []networkingv1.NetworkPolicyPort{{ - Protocol: &protocol, - Port: &portAPIServer, - }}, - }, - { - From: []networkingv1.NetworkPolicyPeer{{ - PodSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "gardener.cloud/role": "monitoring", - "app": "prometheus", - "role": "monitoring", - }, - }, - }}, - Ports: []networkingv1.NetworkPolicyPort{ - { - Protocol: &protocol, - Port: &portBlackboxExporter, - }, - { - Protocol: &protocol, - Port: &portAPIServer, - }, - }, - }, - }, - PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress, networkingv1.PolicyTypeEgress}, - }, - })) - }) - It("w/ ReversedVPN", func() { - kapi = New(kubernetesInterface, namespace, sm, Values{VPN: VPNConfig{ReversedVPNEnabled: true}, RuntimeVersion: runtimeVersion, Version: version}) + kapi = New(kubernetesInterface, namespace, sm, Values{RuntimeVersion: runtimeVersion, Version: version}) Expect(c.Get(ctx, client.ObjectKeyFromObject(networkPolicyAllowKubeAPIServer), networkPolicyAllowKubeAPIServer)).To(MatchError(apierrors.NewNotFound(schema.GroupResource{Group: networkingv1.SchemeGroupVersion.Group, Resource: "networkpolicies"}, networkPolicyAllowKubeAPIServer.Name))) Expect(kapi.Deploy(ctx)).To(Succeed()) @@ -1525,7 +1434,7 @@ rules: Context("egress selector", func() { It("should successfully deploy the configmap resource for K8s >= 1.20", func() { - kapi = New(kubernetesInterface, namespace, sm, Values{VPN: VPNConfig{ReversedVPNEnabled: true}, RuntimeVersion: runtimeVersion, Version: version}) + kapi = New(kubernetesInterface, namespace, sm, Values{RuntimeVersion: runtimeVersion, Version: version}) configMapEgressSelector = &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Name: "kube-apiserver-egress-selector-config", Namespace: namespace}, @@ -1589,19 +1498,20 @@ rules: deployAndRead() Expect(deployment.Annotations).To(Equal(map[string]string{ + "reference.resources.gardener.cloud/secret-0acc967c": secretNameHTTPProxy, + "reference.resources.gardener.cloud/secret-8ddd8e24": secretNameCAVPN, + "reference.resources.gardener.cloud/secret-a92da147": secretNameCAFrontProxy, "reference.resources.gardener.cloud/secret-a709ce3a": secretNameServiceAccountKey, "reference.resources.gardener.cloud/secret-ad29e1cc": secretNameServiceAccountKeyBundle, + "reference.resources.gardener.cloud/configmap-f79954be": configMapNameEgressPolicy, "reference.resources.gardener.cloud/secret-69590970": secretNameCA, "reference.resources.gardener.cloud/secret-17c26aa4": secretNameCAClient, "reference.resources.gardener.cloud/secret-e01f5645": secretNameCAEtcd, - "reference.resources.gardener.cloud/secret-a92da147": secretNameCAFrontProxy, "reference.resources.gardener.cloud/secret-77bc5458": secretNameCAKubelet, "reference.resources.gardener.cloud/secret-389fbba5": secretNameEtcd, "reference.resources.gardener.cloud/secret-c1267cc2": secretNameKubeAPIServerToKubelet, "reference.resources.gardener.cloud/secret-998b2966": secretNameKubeAggregator, "reference.resources.gardener.cloud/secret-3ddd1800": secretNameServer, - "reference.resources.gardener.cloud/secret-1c730dbc": secretNameVPNSeed, - "reference.resources.gardener.cloud/secret-2fb65543": secretNameVPNSeedTLSAuth, "reference.resources.gardener.cloud/secret-430944e0": secretNameStaticToken, "reference.resources.gardener.cloud/secret-b1b53288": secretNameETCDEncryptionConfig, "reference.resources.gardener.cloud/configmap-130aa219": secretNameAdmissionConfig, @@ -1636,41 +1546,7 @@ rules: }) It("should have the expected pod template metadata", func() { - deployAndRead() - - Expect(deployment.Spec.Template.Annotations).To(Equal(map[string]string{ - "reference.resources.gardener.cloud/secret-a709ce3a": secretNameServiceAccountKey, - "reference.resources.gardener.cloud/secret-ad29e1cc": secretNameServiceAccountKeyBundle, - "reference.resources.gardener.cloud/secret-69590970": secretNameCA, - "reference.resources.gardener.cloud/secret-17c26aa4": secretNameCAClient, - "reference.resources.gardener.cloud/secret-e01f5645": secretNameCAEtcd, - "reference.resources.gardener.cloud/secret-a92da147": secretNameCAFrontProxy, - "reference.resources.gardener.cloud/secret-77bc5458": secretNameCAKubelet, - "reference.resources.gardener.cloud/secret-389fbba5": secretNameEtcd, - "reference.resources.gardener.cloud/secret-c1267cc2": secretNameKubeAPIServerToKubelet, - "reference.resources.gardener.cloud/secret-998b2966": secretNameKubeAggregator, - "reference.resources.gardener.cloud/secret-3ddd1800": secretNameServer, - "reference.resources.gardener.cloud/secret-1c730dbc": secretNameVPNSeed, - "reference.resources.gardener.cloud/secret-2fb65543": secretNameVPNSeedTLSAuth, - "reference.resources.gardener.cloud/secret-430944e0": secretNameStaticToken, - "reference.resources.gardener.cloud/secret-b1b53288": secretNameETCDEncryptionConfig, - "reference.resources.gardener.cloud/configmap-130aa219": secretNameAdmissionConfig, - "reference.resources.gardener.cloud/configmap-d4419cd4": configMapNameAuditPolicy, - })) - Expect(deployment.Spec.Template.Labels).To(Equal(map[string]string{ - "gardener.cloud/role": "controlplane", - "app": "kubernetes", - "role": "apiserver", - "networking.gardener.cloud/to-dns": "allowed", - "networking.gardener.cloud/to-private-networks": "allowed", - "networking.gardener.cloud/to-public-networks": "allowed", - "networking.gardener.cloud/to-shoot-networks": "allowed", - "networking.gardener.cloud/from-prometheus": "allowed", - })) - }) - - It("should have the expected pod template metadata with reversed vpn enabled", func() { - kapi = New(kubernetesInterface, namespace, sm, Values{VPN: VPNConfig{ReversedVPNEnabled: true}, RuntimeVersion: runtimeVersion, Version: version}) + kapi = New(kubernetesInterface, namespace, sm, Values{RuntimeVersion: runtimeVersion, Version: version}) deployAndRead() Expect(deployment.Spec.Template.Annotations).To(Equal(map[string]string{ @@ -1715,180 +1591,18 @@ rules: Expect(deployment.Spec.Template.Spec.TerminationGracePeriodSeconds).To(PointTo(Equal(int64(30)))) }) - It("should have no init containers when reversed vpn is enabled", func() { - kapi = New(kubernetesInterface, namespace, sm, Values{VPN: VPNConfig{ReversedVPNEnabled: true}, RuntimeVersion: runtimeVersion, Version: version}) - deployAndRead() - - Expect(deployment.Spec.Template.Spec.InitContainers).To(BeEmpty()) - }) - - It("should have one init container and the vpn-seed sidecar container when reversed vpn is disabled", func() { - var ( - images = Images{AlpineIPTables: "some-image:latest", VPNSeed: "some-other-image:really-latest"} - vpnConfig = VPNConfig{ - ReversedVPNEnabled: false, - PodNetworkCIDR: "1.2.3.4/5", - ServiceNetworkCIDR: "6.7.8.9/10", - NodeNetworkCIDR: pointer.String("11.12.13.14/15"), - } - ) + It("should have no init containers", func() { + kapi = New(kubernetesInterface, namespace, sm, Values{RuntimeVersion: runtimeVersion, Version: version}) - kapi = New(kubernetesInterface, namespace, sm, Values{VPN: vpnConfig, Images: images, RuntimeVersion: runtimeVersion, Version: version}) deployAndRead() - Expect(deployment.Spec.Template.Spec.InitContainers).To(ConsistOf(corev1.Container{ - Name: "set-iptable-rules", - Image: images.AlpineIPTables, - Command: []string{ - "/bin/sh", - "-c", - "iptables -A INPUT -i tun0 -p icmp -j ACCEPT && iptables -A INPUT -i tun0 -m state --state NEW -j DROP", - }, - SecurityContext: &corev1.SecurityContext{ - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{"NET_ADMIN"}, - }, - Privileged: pointer.Bool(true), - }, - VolumeMounts: []corev1.VolumeMount{{ - Name: "modules", - MountPath: "/lib/modules", - }}, - })) - Expect(deployment.Spec.Template.Spec.Volumes).To(ContainElement(corev1.Volume{ - Name: "modules", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}, - }, - })) - - Expect(deployment.Spec.Template.Spec.Containers).To(ContainElement(corev1.Container{ - Name: "vpn-seed", - Image: images.VPNSeed, - ImagePullPolicy: corev1.PullIfNotPresent, - Env: []corev1.EnvVar{ - { - Name: "MAIN_VPN_SEED", - Value: "true", - }, - { - Name: "OPENVPN_PORT", - Value: "4314", - }, - { - Name: "APISERVER_AUTH_MODE", - Value: "client-cert", - }, - { - Name: "APISERVER_AUTH_MODE_CLIENT_CERT_CA", - Value: "/srv/kubernetes/ca/bundle.crt", - }, - { - Name: "APISERVER_AUTH_MODE_CLIENT_CERT_CRT", - Value: "/srv/secrets/vpn-seed/tls.crt", - }, - { - Name: "APISERVER_AUTH_MODE_CLIENT_CERT_KEY", - Value: "/srv/secrets/vpn-seed/tls.key", - }, - { - Name: "SERVICE_NETWORK", - Value: vpnConfig.ServiceNetworkCIDR, - }, - { - Name: "POD_NETWORK", - Value: vpnConfig.PodNetworkCIDR, - }, - { - Name: "NODE_NETWORK", - Value: *vpnConfig.NodeNetworkCIDR, - }, - }, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("100m"), - corev1.ResourceMemory: resource.MustParse("128Mi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("128Mi"), - }, - }, - SecurityContext: &corev1.SecurityContext{ - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{"NET_ADMIN"}, - }, - Privileged: pointer.Bool(true), - }, - TerminationMessagePath: "/dev/termination-log", - TerminationMessagePolicy: corev1.TerminationMessageReadFile, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "vpn-seed", - MountPath: "/srv/secrets/vpn-seed", - }, - { - Name: "vpn-seed-tlsauth", - MountPath: "/srv/secrets/tlsauth", - }, - { - Name: "ca", - MountPath: "/srv/kubernetes/ca", - }, - }, - })) - Expect(deployment.Spec.Template.Spec.Volumes).To(ContainElements( - corev1.Volume{ - Name: "vpn-seed", - VolumeSource: corev1.VolumeSource{ - Projected: &corev1.ProjectedVolumeSource{ - DefaultMode: pointer.Int32(400), - Sources: []corev1.VolumeProjection{ - { - Secret: &corev1.SecretProjection{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: secretNameCAClient, - }, - Items: []corev1.KeyToPath{{ - Key: "bundle.crt", - Path: "ca.crt", - }}, - }, - }, - { - Secret: &corev1.SecretProjection{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: secretNameVPNSeed, - }, - Items: []corev1.KeyToPath{ - { - Key: "tls.crt", - Path: "tls.crt", - }, - { - Key: "tls.key", - Path: "tls.key", - }, - }, - }, - }, - }, - }, - }, - }, - corev1.Volume{ - Name: "vpn-seed-tlsauth", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{SecretName: secretNameVPNSeedTLSAuth}, - }, - }, - )) + Expect(deployment.Spec.Template.Spec.InitContainers).To(BeEmpty()) }) - It("should have no init container and three vpn-seed-client sidecar containers when reversed vpn and vpn high availability are enabled", func() { + It("should have one init container and three vpn-seed-client sidecar containers when vpn high availability are enabled", func() { values := Values{ Images: Images{VPNClient: "vpn-client-image:really-latest"}, VPN: VPNConfig{ - ReversedVPNEnabled: true, HighAvailabilityEnabled: true, HighAvailabilityNumberOfSeedServers: 2, HighAvailabilityNumberOfShootClients: 3, @@ -2274,6 +1988,7 @@ rules: "--tls-cipher-suites="+strings.Join(tlscipherSuites, ","), "--vmodule=httplog=3", "--v=3", + "--egress-selector-config-file=/etc/kubernetes/egress/egress-selector-configuration.yaml", )) Expect(issuerIdx).To(BeNumerically(">=", 0)) Expect(issuerIdx).To(BeNumerically("<", issuerIdx1)) @@ -2368,6 +2083,21 @@ rules: MountPath: "/usr/share/ca-certificates", ReadOnly: true, }, + corev1.VolumeMount{ + Name: "ca-vpn", + MountPath: "/srv/kubernetes/ca-vpn", + ReadOnly: false, + }, + corev1.VolumeMount{ + Name: "http-proxy", + MountPath: "/etc/srv/kubernetes/envoy", + ReadOnly: false, + }, + corev1.VolumeMount{ + Name: "egress-selection-config", + MountPath: "/etc/kubernetes/egress", + ReadOnly: false, + }, )) Expect(deployment.Spec.Template.Spec.Volumes).To(ConsistOf( corev1.Volume{ @@ -2530,10 +2260,11 @@ rules: }, }, }, + // VPN-related secrets (will be asserted in detail later) - MatchFields(IgnoreExtras, Fields{"Name": Equal("modules")}), - MatchFields(IgnoreExtras, Fields{"Name": Equal("vpn-seed")}), - MatchFields(IgnoreExtras, Fields{"Name": Equal("vpn-seed-tlsauth")}), + MatchFields(IgnoreExtras, Fields{"Name": Equal("ca-vpn")}), + MatchFields(IgnoreExtras, Fields{"Name": Equal("http-proxy")}), + MatchFields(IgnoreExtras, Fields{"Name": Equal("egress-selection-config")}), )) secret := &corev1.Secret{} @@ -2853,7 +2584,7 @@ rules: }) It("should properly configure the settings related to reversed vpn if enabled", func() { - kapi = New(kubernetesInterface, namespace, sm, Values{Images: images, RuntimeVersion: runtimeVersion, Version: version, VPN: VPNConfig{ReversedVPNEnabled: true}}) + kapi = New(kubernetesInterface, namespace, sm, Values{Images: images, RuntimeVersion: runtimeVersion, Version: version}) deployAndRead() Expect(deployment.Spec.Template.Spec.Containers[0].Command).To(ContainElement( @@ -2905,59 +2636,6 @@ rules: )) }) - It("should not configure the settings related to reversed vpn if disabled", func() { - kapi = New(kubernetesInterface, namespace, sm, Values{Images: images, RuntimeVersion: runtimeVersion, Version: version, VPN: VPNConfig{ReversedVPNEnabled: false}}) - deployAndRead() - - Expect(deployment.Spec.Template.Spec.Containers[0].Command).NotTo(ContainElement(ContainSubstring("--egress-selector-config-file="))) - Expect(deployment.Spec.Template.Spec.Containers[0].VolumeMounts).NotTo(ContainElement(MatchFields(IgnoreExtras, Fields{"Name": Equal("http-proxy")}))) - Expect(deployment.Spec.Template.Spec.Volumes).NotTo(ContainElement(MatchFields(IgnoreExtras, Fields{"Name": Equal("http-proxy")}))) - }) - - It("should properly configure the settings related to oidc if enabled", func() { - oidc := &gardencorev1beta1.OIDCConfig{ - IssuerURL: pointer.String("someurl"), - ClientID: pointer.String("clientid"), - CABundle: pointer.String(""), - UsernameClaim: pointer.String("usernameclaim"), - GroupsClaim: pointer.String("groupsclaim"), - UsernamePrefix: pointer.String("usernameprefix"), - GroupsPrefix: pointer.String("groupsprefix"), - SigningAlgs: []string{"foo", "bar"}, - RequiredClaims: map[string]string{"one": "two", "three": "four"}, - } - - kapi = New(kubernetesInterface, namespace, sm, Values{Images: images, RuntimeVersion: runtimeVersion, Version: version, OIDC: oidc}) - deployAndRead() - - Expect(deployment.Spec.Template.Spec.Containers[0].Command).To(ContainElements( - "--oidc-issuer-url="+*oidc.IssuerURL, - "--oidc-client-id="+*oidc.ClientID, - "--oidc-ca-file=/srv/kubernetes/oidc/ca.crt", - "--oidc-username-claim="+*oidc.UsernameClaim, - "--oidc-groups-claim="+*oidc.GroupsClaim, - "--oidc-username-prefix="+*oidc.UsernamePrefix, - "--oidc-groups-prefix="+*oidc.GroupsPrefix, - "--oidc-signing-algs=foo,bar", - "--oidc-required-claim=one=two", - "--oidc-required-claim=three=four", - )) - - Expect(deployment.Spec.Template.Spec.Containers[0].VolumeMounts).To(ContainElement(corev1.VolumeMount{ - Name: "oidc-cabundle", - MountPath: "/srv/kubernetes/oidc", - })) - - Expect(deployment.Spec.Template.Spec.Volumes).To(ContainElement(corev1.Volume{ - Name: "oidc-cabundle", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "kube-apiserver-oidc-cabundle-cd372fb8", - }, - }, - })) - }) - It("should not configure the settings related to oidc if disabled", func() { deployAndRead() @@ -3075,7 +2753,6 @@ rules: values := Values{ Images: Images{VPNClient: "vpn-client-image:really-latest"}, VPN: VPNConfig{ - ReversedVPNEnabled: true, HighAvailabilityEnabled: false, HighAvailabilityNumberOfSeedServers: 2, HighAvailabilityNumberOfShootClients: 3, @@ -3099,7 +2776,6 @@ rules: values := Values{ Images: Images{VPNClient: "vpn-client-image:really-latest"}, VPN: VPNConfig{ - ReversedVPNEnabled: true, HighAvailabilityEnabled: true, HighAvailabilityNumberOfSeedServers: 2, HighAvailabilityNumberOfShootClients: 3, diff --git a/pkg/operation/botanist/component/kubeapiserver/logging.go b/pkg/operation/botanist/component/kubeapiserver/logging.go index 0d4e3b00d6b..fafd32b5ca6 100644 --- a/pkg/operation/botanist/component/kubeapiserver/logging.go +++ b/pkg/operation/botanist/component/kubeapiserver/logging.go @@ -38,22 +38,6 @@ const ( Reserve_Data True ` - loggingParserNameVPNSeed = "vpnSeedParser" - loggingParserVPNSeed = `[PARSER] - Name ` + loggingParserNameVPNSeed + ` - Format regex - Regex ^(?