[provider-local] Harmonize local VPN setup with real-world scenario (#…

…9752)

* Revert "Partially revert gardener-attic/machine-controller-manager-provider-local#42"

This reverts commit 7ec12fe.

* Drop `provider-local` MCM `ClusterRole` webhook

no longer needed now that MCM-provider-local no longer deploys a `Service`

* Drop `allow-to-shoot-networks` `NetworkPolicy`

This network policy is not needed since packets to the shoot networks are always encapsulated in the VPN tunnel and never handled by the seed network policies.

Co-authored-by: Tim Ebert <timebertt@gmail.com>

* Drop `NetworkPolicy`s allowing ingress to machine pods

The shoot networks are always "contacted" via the VPN tunnel (which is established FROM the machine pods TO the `vpn-seed-server`).

Co-authored-by: Tim Ebert <timebertt@gmail.com>

* Define dedicated IP pool and node network for local `Shoot`s

Co-authored-by: Tim Ebert <timebertt@gmail.com>

* Distinguish between `IPPool` for IPv4 and IPv6

* Workaround for upgrade tests

Co-Authored-By: Rafael Franzke <rafael.franzke@sap.com>
Co-Authored-By: Marcel Boehm <marcel.boehm@inovex.de>

* Ignore pods using non-default IPPools during startup

* Drop default values from `IPPool` spec

* Simplify IP families handling

* Set `nodeSelector` to `!all()` for local `IPPool`s

Co-Authored-By: Johannes Scheerer <johannes.scheerer@sap.com>

---------

Co-authored-by: Tim Ebert <timebertt@gmail.com>
Co-authored-by: Marcel Boehm <marcel.boehm@inovex.de>
Co-authored-by: Johannes Scheerer <johannes.scheerer@sap.com>

charts/gardener/provider-local/templates/rbac.yaml

@@ -83,6 +83,7 @@ rules:
   - admissionregistration.k8s.io
   - apiextensions.k8s.io
   - networking.k8s.io
+  - crd.projectcalico.org
   resources:
   - namespaces
   - namespaces/finalizers
@@ -109,6 +110,7 @@ rules:
   - networkpolicies
   - ingresses
   - poddisruptionbudgets
+  - ippools
   verbs:
   - "*"
 - apiGroups:

cmd/gardener-extension-provider-local/app/options.go

@@ -32,7 +32,6 @@ import (
 	workercontroller "github.com/gardener/gardener/pkg/provider-local/controller/worker"
 	controlplanewebhook "github.com/gardener/gardener/pkg/provider-local/webhook/controlplane"
 	dnsconfigwebhook "github.com/gardener/gardener/pkg/provider-local/webhook/dnsconfig"
-	"github.com/gardener/gardener/pkg/provider-local/webhook/machinecontrollermanager"
 	networkpolicywebhook "github.com/gardener/gardener/pkg/provider-local/webhook/networkpolicy"
 	nodewebhook "github.com/gardener/gardener/pkg/provider-local/webhook/node"
 	"github.com/gardener/gardener/pkg/provider-local/webhook/nodeagentosc"
@@ -69,7 +68,6 @@ func WebhookSwitchOptions() *extensionscmdwebhook.SwitchOptions {
 		extensionscmdwebhook.Switch(networkpolicywebhook.WebhookName, networkpolicywebhook.AddToManager),
 		extensionscmdwebhook.Switch(nodewebhook.WebhookName, nodewebhook.AddToManager),
 		extensionscmdwebhook.Switch(nodewebhook.WebhookNameShoot, nodewebhook.AddShootWebhookToManager),
-		extensionscmdwebhook.Switch(machinecontrollermanager.WebhookName, machinecontrollermanager.AddToManager),
 		extensionscmdwebhook.Switch(nodeagentosc.WebhookName, nodeagentosc.AddToManager),
 	)
 }

cmd/gardenlet/app/bootstrappers/seed_config.go

@@ -100,6 +100,13 @@ func checkSeedConfigHeuristically(ctx context.Context, seedClient client.Client,
 	}
 
 	for _, pod := range podList.Items {
+		if pod.Annotations["cni.projectcalico.org/ipv4pools"] != "" || pod.Annotations["cni.projectcalico.org/ipv6pools"] != "" {
+			// machine-controller-manager-provider-local configures machine pods to use IPs from dedicated IPPools that
+			// correlate with the configured shoot node CIDR. I.e., such pods will use IPs outside the configured seed pod
+			// CIDR. Skip pods configuring non-default IPPools in this heuristic check accordingly.
+			continue
+		}
+
 		if !pod.Spec.HostNetwork && pod.Status.PodIP != "" {
 			if ip := net.ParseIP(pod.Status.PodIP); ip != nil && !seedConfigPods.Contains(ip) {
 				return fmt.Errorf("incorrect pod network specified in seed configuration (cluster pod=%q vs. config=%q)", ip, seedConfig.Spec.Networks.Pods)

docs/operations/network_policies.md

@@ -24,7 +24,6 @@ It deploys the following so-called "general `NetworkPolicy`s":
 | `allow-to-blocked-cidrs`     | Allows egress traffic from pods labeled with `networking.gardener.cloud/to-blocked-cidrs=allowed` to explicitly blocked addresses configured by human operators (configured via `.spec.networking.blockedCIDRs` in the `Seed`). For instance, this can be used to block the cloud provider's metadata service.                                                                                                                                                                            |
 | `allow-to-public-networks`   | Allows egress traffic from pods labeled with `networking.gardener.cloud/to-public-networks=allowed` to all public network IPs, except for private networks (RFC1918), carrier-grade NAT (RFC6598), and explicitly blocked addresses configured by human operators for all pods labeled with `networking.gardener.cloud/to-public-networks=allowed`. In practice, this blocks egress traffic to all networks in the cluster and only allows egress traffic to public IPv4 addresses.       |
 | `allow-to-private-networks`  | Allows egress traffic from pods labeled with `networking.gardener.cloud/to-private-networks=allowed` to the private networks (RFC1918) and carrier-grade NAT (RFC6598) except for cluster-specific networks (configured via `.spec.networks` in the `Seed`).                                                                                                                                                                                                                              |
-| `allow-to-shoot-networks`    | Allows egress traffic from pods labeled with `networking.gardener.cloud/to-shoot-networks=allowed` to IPv4 blocks belonging to the shoot networks (configured via `.spec.networking` in the `Shoot`). In practice, this should be used by components which use VPN tunnel to communicate to pods in the shoot cluster. Note that this policy only exists in `shoot-*` namespaces.                                                                                                         |
 
 Apart from those, the `gardener-operator` also enables the [`NetworkPolicy` controller of `gardener-resource-manager`](../concepts/resource-manager.md#networkpolicy-controller).
 Please find more information in the linked document.

example/provider-local/managedseeds/shoot-managedseed.yaml

@@ -11,6 +11,7 @@ spec:
   region: local
   networking:
     type: calico
+    nodes: 10.10.0.0/16
   provider:
     type: local
     workers:

example/provider-local/shoot-ipv6.yaml

@@ -13,8 +13,7 @@ spec:
     type: calico
     ipFamilies:
     - IPv6
-    pods: 2001:db8:1::/48
-    services: 2001:db8:3::/108
+    nodes: fd00:10:a::/64
   provider:
     type: local
     workers:

example/provider-local/shoot.yaml

@@ -12,6 +12,7 @@ spec:
   region: local
   networking:
     type: calico
+    nodes: 10.10.0.0/16
   provider:
     type: local
     workers:

pkg/apis/core/v1beta1/constants/types_constants.go

@@ -476,8 +476,6 @@ const (
 	// LabelNetworkPolicyToRuntimeAPIServer allows Egress from pods labeled with 'networking.gardener.cloud/to-runtime-apiserver=allowed' to runtime Kubernetes
 	// API Server.
 	LabelNetworkPolicyToRuntimeAPIServer = "networking.gardener.cloud/to-runtime-apiserver"
-	// LabelNetworkPolicyToShootNetworks allows Egress from pods labeled with 'networking.gardener.cloud/to-shoot-networks=allowed' to IPv4 blocks belonging to the Shoot network.
-	LabelNetworkPolicyToShootNetworks = "networking.gardener.cloud/to-shoot-networks"
 	// LabelNetworkPolicyFromPrometheus allows Ingress from Prometheus to pods labeled with 'networking.gardener.cloud/from-prometheus=allowed' and ports
 	// named 'metrics' in the PodSpecification.
 	// Deprecated: This label is deprecated and will be removed in a future version. Components in shoot namespaces

pkg/component/kubernetes/apiserver/apiserver_test.go

@@ -2313,7 +2313,6 @@ rules:
 					deployAndRead()
 
 					Expect(deployment.Spec.Template.Labels).To(Equal(utils.MergeStringMaps(defaultLabels, map[string]string{
-						"networking.gardener.cloud/to-shoot-networks":                       "allowed",
 						"networking.gardener.cloud/to-runtime-apiserver":                    "allowed",
 						"networking.resources.gardener.cloud/to-vpn-seed-server-0-tcp-1194": "allowed",
 						"networking.resources.gardener.cloud/to-vpn-seed-server-1-tcp-1194": "allowed",

pkg/component/kubernetes/apiserver/deployment.go

@@ -609,7 +609,6 @@ func (k *kubeAPIServer) handleVPNSettingsHA(
 	}
 
 	deployment.Spec.Template.Spec.ServiceAccountName = serviceAccount.Name
-	deployment.Spec.Template.Labels[v1beta1constants.LabelNetworkPolicyToShootNetworks] = v1beta1constants.LabelNetworkPolicyAllowed
 	deployment.Spec.Template.Labels[v1beta1constants.LabelNetworkPolicyToRuntimeAPIServer] = v1beta1constants.LabelNetworkPolicyAllowed
 
 	for i := 0; i < k.values.VPN.HighAvailabilityNumberOfSeedServers; i++ {

pkg/component/networking/vpn/seedserver/seedserver.go

@@ -262,7 +262,6 @@ func (v *vpnSeedServer) podTemplate(configMap *corev1.ConfigMap, secretCAVPN, se
 	template := &corev1.PodTemplateSpec{
 		ObjectMeta: metav1.ObjectMeta{
 			Labels: utils.MergeStringMaps(getLabels(), map[string]string{
-				v1beta1constants.LabelNetworkPolicyToShootNetworks:                                                          v1beta1constants.LabelNetworkPolicyAllowed,
 				v1beta1constants.LabelNetworkPolicyToDNS:                                                                    v1beta1constants.LabelNetworkPolicyAllowed,
 				v1beta1constants.LabelNetworkPolicyToPrivateNetworks:                                                        v1beta1constants.LabelNetworkPolicyAllowed,
 				gardenerutils.NetworkPolicyLabel(v1beta1constants.DeploymentNameKubeAPIServer, kubeapiserverconstants.Port): v1beta1constants.LabelNetworkPolicyAllowed,

pkg/component/networking/vpn/seedserver/seedserver_test.go

@@ -91,7 +91,6 @@ var _ = Describe("VpnSeedServer", func() {
 					Labels: map[string]string{
 						v1beta1constants.GardenRole:                                     v1beta1constants.GardenRoleControlPlane,
 						v1beta1constants.LabelApp:                                       "vpn-seed-server",
-						v1beta1constants.LabelNetworkPolicyToShootNetworks:              v1beta1constants.LabelNetworkPolicyAllowed,
 						v1beta1constants.LabelNetworkPolicyToDNS:                        v1beta1constants.LabelNetworkPolicyAllowed,
 						v1beta1constants.LabelNetworkPolicyToPrivateNetworks:            v1beta1constants.LabelNetworkPolicyAllowed,
 						"networking.resources.gardener.cloud/to-kube-apiserver-tcp-443": "allowed",

pkg/controller/networkpolicy/reconciler.go

@@ -232,13 +232,8 @@ func (r *Reconciler) networkPolicyConfigs() []networkPolicyConfig {
 				labels.NewSelector().Add(utils.MustNewRequirement(v1beta1constants.LabelExposureClassHandlerName, selection.Exists)),
 			}, r.additionalNamespaceLabelSelectors...),
 		},
-		{
-			name:          "allow-to-shoot-networks",
-			reconcileFunc: r.reconcileNetworkPolicyAllowToShootNetworks,
-			namespaceSelectors: []labels.Selector{
-				labels.SelectorFromSet(labels.Set{v1beta1constants.GardenRole: v1beta1constants.GardenRoleShoot}),
-			},
-		},
+		// TODO(rfranzke): Remove this after v1.98 has been released.
+		{name: "allow-to-shoot-networks"},
 	}
 
 	return configs
@@ -425,50 +420,6 @@ func (r *Reconciler) reconcileNetworkPolicyAllowToPrivateNetworks(ctx context.Co
 	})
 }
 
-func (r *Reconciler) reconcileNetworkPolicyAllowToShootNetworks(ctx context.Context, log logr.Logger, networkPolicy *networkingv1.NetworkPolicy) error {
-	cluster := &extensionsv1alpha1.Cluster{}
-	if err := r.RuntimeClient.Get(ctx, client.ObjectKey{Name: networkPolicy.Namespace}, cluster); err != nil {
-		return err
-	}
-
-	shoot, err := extensions.ShootFromCluster(cluster)
-	if err != nil {
-		return err
-	}
-
-	var shootNetworks []string
-	if shoot.Spec.Networking != nil {
-		if v := shoot.Spec.Networking.Nodes; v != nil {
-			shootNetworks = append(shootNetworks, *v)
-		}
-		if v := shoot.Spec.Networking.Pods; v != nil {
-			shootNetworks = append(shootNetworks, *v)
-		}
-		if v := shoot.Spec.Networking.Services; v != nil {
-			shootNetworks = append(shootNetworks, *v)
-		}
-	}
-
-	shootNetworkPeers, err := networkPolicyPeersWithExceptions(shootNetworks, r.RuntimeNetworks.BlockCIDRs...)
-	if err != nil {
-		return err
-	}
-
-	return r.reconcileNetworkPolicy(ctx, log, networkPolicy, func(policy *networkingv1.NetworkPolicy) {
-		metav1.SetMetaDataAnnotation(&policy.ObjectMeta, v1beta1constants.GardenerDescription, fmt.Sprintf("Allows "+
-			"egress from pods labeled with '%s=%s' to IPv4 blocks belonging to the shoot networks. In practice, this "+
-			"should be used by components which use VPN tunnel to communicate to pods in the shoot cluster.",
-			v1beta1constants.LabelNetworkPolicyToShootNetworks, v1beta1constants.LabelNetworkPolicyAllowed))
-
-		policy.Spec = networkingv1.NetworkPolicySpec{
-			PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{v1beta1constants.LabelNetworkPolicyToShootNetworks: v1beta1constants.LabelNetworkPolicyAllowed}},
-			Egress:      []networkingv1.NetworkPolicyEgressRule{{To: shootNetworkPeers}},
-			Ingress:     []networkingv1.NetworkPolicyIngressRule{},
-			PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeEgress},
-		}
-	})
-}
-
 func (r *Reconciler) reconcileNetworkPolicyAllowToDNS(ctx context.Context, log logr.Logger, networkPolicy *networkingv1.NetworkPolicy) error {
 	_, runtimeServiceCIDR, err := net.ParseCIDR(r.RuntimeNetworks.Services)
 	if err != nil {

pkg/provider-local/controller/infrastructure/actuator.go

@@ -6,6 +6,8 @@ package infrastructure
 
 import (
 	"context"
+	"fmt"
+	"strings"
 
 	"github.com/go-logr/logr"
 	networkingv1 "k8s.io/api/networking/v1"
@@ -15,8 +17,9 @@ import (
 
 	extensionscontroller "github.com/gardener/gardener/extensions/pkg/controller"
 	"github.com/gardener/gardener/extensions/pkg/controller/infrastructure"
-	v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
+	gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
 	extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1"
+	"github.com/gardener/gardener/pkg/client/kubernetes"
 	"github.com/gardener/gardener/pkg/provider-local/local"
 	kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes"
 )
@@ -32,27 +35,16 @@ func NewActuator(mgr manager.Manager) infrastructure.Actuator {
 	}
 }
 
-func (a *actuator) Reconcile(ctx context.Context, _ logr.Logger, infrastructure *extensionsv1alpha1.Infrastructure, _ *extensionscontroller.Cluster) error {
-	networkPolicyAllowToMachinePods := emptyNetworkPolicy("allow-to-machine-pods", infrastructure.Namespace)
-	networkPolicyAllowToMachinePods.Spec = networkingv1.NetworkPolicySpec{
-		Egress: []networkingv1.NetworkPolicyEgressRule{{
-			To: []networkingv1.NetworkPolicyPeer{{
-				PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "machine"}},
-			}},
-		}},
-		PodSelector: metav1.LabelSelector{
-			MatchLabels: map[string]string{v1beta1constants.LabelNetworkPolicyToShootNetworks: v1beta1constants.LabelNetworkPolicyAllowed},
-		},
-		PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeEgress},
-	}
-
+func (a *actuator) Reconcile(ctx context.Context, _ logr.Logger, infrastructure *extensionsv1alpha1.Infrastructure, cluster *extensionscontroller.Cluster) error {
 	networkPolicyAllowMachinePods := emptyNetworkPolicy("allow-machine-pods", infrastructure.Namespace)
 	networkPolicyAllowMachinePods.Spec = networkingv1.NetworkPolicySpec{
 		Ingress: []networkingv1.NetworkPolicyIngressRule{{
 			From: []networkingv1.NetworkPolicyPeer{
-				{PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{v1beta1constants.LabelNetworkPolicyToShootNetworks: v1beta1constants.LabelNetworkPolicyAllowed}}},
-			},
-		}},
+				{
+					PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "machine"}},
+				},
+			}},
+		},
 		Egress: []networkingv1.NetworkPolicyEgressRule{{
 			To: []networkingv1.NetworkPolicyPeer{
 				{
@@ -80,10 +72,23 @@ func (a *actuator) Reconcile(ctx context.Context, _ logr.Logger, infrastructure
 		},
 	}
 
-	for _, obj := range []client.Object{
-		networkPolicyAllowToMachinePods,
+	if cluster.Shoot.Spec.Networking == nil || cluster.Shoot.Spec.Networking.Nodes == nil {
+		return fmt.Errorf("shoot specification does not contain node network CIDR required for VPN tunnel")
+	}
+
+	objects := []client.Object{
 		networkPolicyAllowMachinePods,
-	} {
+	}
+
+	for _, ipFamily := range cluster.Shoot.Spec.Networking.IPFamilies {
+		ipPoolObj, err := ipPool(infrastructure.Namespace, string(ipFamily), *cluster.Shoot.Spec.Networking.Nodes)
+		if err != nil {
+			return err
+		}
+		objects = append(objects, ipPoolObj)
+	}
+
+	for _, obj := range objects {
 		if err := a.client.Patch(ctx, obj, client.Apply, local.FieldOwner, client.ForceOwnership); err != nil {
 			return err
 		}
@@ -97,7 +102,8 @@ func (a *actuator) Delete(ctx context.Context, _ logr.Logger, infrastructure *ex
 		emptyNetworkPolicy("allow-machine-pods", infrastructure.Namespace),
 		emptyNetworkPolicy("allow-to-istio-ingress-gateway", infrastructure.Namespace),
 		emptyNetworkPolicy("allow-to-provider-local-coredns", infrastructure.Namespace),
-		emptyNetworkPolicy("allow-to-machine-pods", infrastructure.Namespace),
+		&metav1.PartialObjectMetadata{TypeMeta: metav1.TypeMeta{APIVersion: "crd.projectcalico.org/v1", Kind: "IPPool"}, ObjectMeta: metav1.ObjectMeta{Name: IPPoolName(infrastructure.Namespace, string(gardencorev1beta1.IPFamilyIPv4))}},
+		&metav1.PartialObjectMetadata{TypeMeta: metav1.TypeMeta{APIVersion: "crd.projectcalico.org/v1", Kind: "IPPool"}, ObjectMeta: metav1.ObjectMeta{Name: IPPoolName(infrastructure.Namespace, string(gardencorev1beta1.IPFamilyIPv6))}},
 	)
 }
 
@@ -125,3 +131,24 @@ func emptyNetworkPolicy(name, namespace string) *networkingv1.NetworkPolicy {
 		},
 	}
 }
+
+// IPPoolName returns the name of the crd.projectcalico.org/v1.IPPool resource for the given shoot namespace.
+func IPPoolName(shootNamespace, ipFamily string) string {
+	return "shoot-machine-pods-" + shootNamespace + "-" + strings.ToLower(ipFamily)
+}
+
+func ipPool(shootNamespace, ipFamily, nodeCIDR string) (client.Object, error) {
+	return kubernetes.NewManifestReader([]byte(`apiVersion: crd.projectcalico.org/v1
+kind: IPPool
+metadata:
+  name: ` + IPPoolName(shootNamespace, ipFamily) + `
+spec:
+  cidr: ` + nodeCIDR + `
+  ipipMode: Always
+  natOutgoing: true
+  nodeSelector: "!all()" # Without this, calico defaults nodeSelector to "all()" and can randomly pick this pool for
+                         # IPAM for pods even if the pod does not explicitly request an IP from this pool via the
+                         # cni.projectcalico.org/IPv{4,6}Pools annotation.
+                         # See https://github.com/projectcalico/calico/issues/7299#issuecomment-1446834103
+`)).Read()
+}

pkg/provider-local/controller/worker/machines.go

@@ -6,6 +6,7 @@ package worker
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 
 	machinev1alpha1 "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1"
@@ -16,9 +17,11 @@ import (
 
 	"github.com/gardener/gardener/extensions/pkg/controller/worker"
 	genericworkeractuator "github.com/gardener/gardener/extensions/pkg/controller/worker/genericactuator"
+	gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
 	v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
 	extensionsv1alpha1helper "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1/helper"
 	api "github.com/gardener/gardener/pkg/provider-local/apis/local"
+	"github.com/gardener/gardener/pkg/provider-local/controller/infrastructure"
 	"github.com/gardener/gardener/pkg/provider-local/local"
 )
 
@@ -103,6 +106,24 @@ func (w *workerDelegate) generateMachineConfig(ctx context.Context) error {
 			Data: map[string][]byte{"userData": userData},
 		})
 
+		providerConfig := map[string]interface{}{
+			"image": image,
+		}
+
+		for _, ipFamily := range w.cluster.Shoot.Spec.Networking.IPFamilies {
+			key := "ipPoolNameV4"
+			if ipFamily == gardencorev1beta1.IPFamilyIPv6 {
+				key = "ipPoolNameV6"
+			}
+
+			providerConfig[key] = infrastructure.IPPoolName(w.worker.Namespace, string(ipFamily))
+		}
+
+		providerConfigBytes, err := json.Marshal(providerConfig)
+		if err != nil {
+			return err
+		}
+
 		machineClasses = append(machineClasses, &machinev1alpha1.MachineClass{
 			TypeMeta: metav1.TypeMeta{
 				APIVersion: machinev1alpha1.SchemeGroupVersion.String(),
@@ -121,7 +142,7 @@ func (w *workerDelegate) generateMachineConfig(ctx context.Context) error {
 				Namespace: w.worker.Spec.SecretRef.Namespace,
 			},
 			Provider:     local.Type,
-			ProviderSpec: runtime.RawExtension{Raw: []byte(`{"image":"` + image + `"}`)},
+			ProviderSpec: runtime.RawExtension{Raw: providerConfigBytes},
 		})
 
 		machineDeployments = append(machineDeployments, worker.MachineDeployment{