Skip to content

Commit

Permalink
Drop GA-ed HAControlPlanes and `FullNetworkPoliciesInRuntimeCluster…
Browse files Browse the repository at this point in the history 
…` feature gates (gardener#8083)

* Drop GA-ed `HAControlPlanes` feature gate

* Drop GA-ed `FullNetworkPoliciesInRuntimeCluster` feature gate

* Address PR review feedback
  • Loading branch information
@rfranzke
rfranzke authored Jun 19, 2023
1 parent 54f9bf8 commit e9bcb79
Show file tree
Hide file tree
Showing 13 changed files with 18 additions and 306 deletions.
182 changes: 0 additions & 182 deletions cmd/gardenlet/app/app.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
"os"
goruntime "runtime"
"strconv"
"strings"
"time"

"github.com/go-logr/logr"
Expand All @@ -31,12 +30,9 @@ import (
coordinationv1 "k8s.io/api/coordination/v1"
corev1 "k8s.io/api/core/v1"
eventsv1 "k8s.io/api/events/v1"
networkingv1 "k8s.io/api/networking/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/util/intstr"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/rest"
"k8s.io/component-base/version"
Expand All @@ -59,11 +55,8 @@ import (
v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
"github.com/gardener/gardener/pkg/apis/operations"
operationsv1alpha1 "github.com/gardener/gardener/pkg/apis/operations/v1alpha1"
resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1"
"github.com/gardener/gardener/pkg/client/kubernetes"
clientmapbuilder "github.com/gardener/gardener/pkg/client/kubernetes/clientmap/builder"
kubeapiserverconstants "github.com/gardener/gardener/pkg/component/kubeapiserver/constants"
"github.com/gardener/gardener/pkg/component/vpnseedserver"
"github.com/gardener/gardener/pkg/controllerutils"
"github.com/gardener/gardener/pkg/controllerutils/routes"
"github.com/gardener/gardener/pkg/features"
Expand Down Expand Up @@ -354,14 +347,6 @@ func (g *garden) Start(ctx context.Context) error {
return err
}

// Migrate all relevant services in shoot control planes once, so that we don't have to wait for their reconciliation
// and can ensure the required policies are created.
// TODO(timuthy, rfranzke): To be removed in a future release.
log.Info("Migrating all relevant shoot control plane services to create required network policies")
if err := g.migrateAllShootServicesForNetworkPolicies(ctx); err != nil {
return err
}

log.Info("Setting up shoot client map")
shootClientMap, err := clientmapbuilder.
NewShootClientMapBuilder().
Expand Down Expand Up @@ -453,173 +438,6 @@ func (g *garden) registerSeed(ctx context.Context, gardenClient client.Client) e
})
}

func (g *garden) migrateAllShootServicesForNetworkPolicies(ctx context.Context) error {
var taskFns []flow.TaskFn

// kube-apiserver services
kubeAPIServerServiceList := &corev1.ServiceList{}
if err := g.mgr.GetClient().List(ctx, kubeAPIServerServiceList, client.MatchingLabels{v1beta1constants.LabelApp: v1beta1constants.LabelKubernetes, v1beta1constants.LabelRole: v1beta1constants.LabelAPIServer}); err != nil {
return err
}

taskFns = append(taskFns, migrationTasksForServices(g.mgr.GetClient(), kubeAPIServerServiceList.Items, kubeapiserverconstants.Port, true)...)

// vpn-seed-server services
for _, serviceName := range []string{vpnseedserver.ServiceName, vpnseedserver.ServiceName + "-0", vpnseedserver.ServiceName + "-1"} {
serviceList := &corev1.ServiceList{}
// Use APIReader here because an index on `metadata.name` is not available in the runtime client.
if err := g.mgr.GetAPIReader().List(ctx, serviceList, client.MatchingFieldsSelector{
Selector: fields.OneTermEqualSelector(metav1.ObjectNameField, serviceName),
}); err != nil {
return err
}

taskFns = append(taskFns, migrationTasksForServices(g.mgr.GetClient(), serviceList.Items, vpnseedserver.MetricsPort, false)...)
}

// vali services
serviceList := &corev1.ServiceList{}
if err := g.mgr.GetClient().List(ctx, serviceList, client.MatchingLabels{"app": "vali", "role": "logging"}); err != nil {
return err
}

// drop vali services of non-shoot namespaces since they should not be mutated
for i := len(serviceList.Items) - 1; i >= 0; i-- {
if !strings.HasPrefix(serviceList.Items[i].Namespace, v1beta1constants.TechnicalIDPrefix) {
serviceList.Items = append(serviceList.Items[:i], serviceList.Items[i+1:]...)
}
}

taskFns = append(taskFns, migrationTasksForValiServices(g.mgr.GetClient(), serviceList.Items)...)

// prometheus namespaces
serviceList = &corev1.ServiceList{}
if err := g.mgr.GetClient().List(ctx, serviceList, client.MatchingLabels{"app": "prometheus", "role": "monitoring"}); err != nil {
return err
}

// drop prometheus services of non-shoot namespaces since they one should not be mutated
for i := len(serviceList.Items) - 1; i >= 0; i-- {
if !strings.HasPrefix(serviceList.Items[i].Namespace, v1beta1constants.TechnicalIDPrefix) {
serviceList.Items = append(serviceList.Items[:i], serviceList.Items[i+1:]...)
}
}

taskFns = append(taskFns, migrationTasksForPrometheusServices(g.mgr.GetClient(), serviceList.Items)...)

// vpa-recommender services for shoot namespaces
namespaceList := &corev1.NamespaceList{}
if err := g.mgr.GetClient().List(ctx, namespaceList, client.MatchingLabels{v1beta1constants.GardenRole: v1beta1constants.GardenRoleShoot}); err != nil {
return err
}

taskFns = append(taskFns, migrationTasksForShootVPARecommenders(g.mgr.GetClient(), namespaceList.Items)...)

return flow.Parallel(taskFns...)(ctx)
}

func migrationTasksForServices(cl client.Client, services []corev1.Service, port int, withGardenNamespaceSelector bool) []flow.TaskFn {
var taskFns []flow.TaskFn

for _, svc := range services {
service := svc

taskFns = append(taskFns, func(ctx context.Context) error {
selectors := []metav1.LabelSelector{}
if withGardenNamespaceSelector {
selectors = append(selectors, metav1.LabelSelector{MatchLabels: map[string]string{corev1.LabelMetadataName: v1beta1constants.GardenNamespace}})
}

selectors = append(selectors,
metav1.LabelSelector{MatchLabels: map[string]string{v1beta1constants.GardenRole: v1beta1constants.GardenRoleIstioIngress}},
metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{{Key: v1beta1constants.LabelExposureClassHandlerName, Operator: metav1.LabelSelectorOpExists}}},
)

if withGardenNamespaceSelector {
selectors = append(selectors, metav1.LabelSelector{MatchLabels: map[string]string{v1beta1constants.GardenRole: v1beta1constants.GardenRoleExtension}})
}

patch := client.MergeFrom(service.DeepCopy())
metav1.SetMetaDataAnnotation(&service.ObjectMeta, resourcesv1alpha1.NetworkingPodLabelSelectorNamespaceAlias, v1beta1constants.LabelNetworkPolicyShootNamespaceAlias)
utilruntime.Must(gardenerutils.InjectNetworkPolicyNamespaceSelectors(&service, selectors...))
utilruntime.Must(gardenerutils.InjectNetworkPolicyAnnotationsForScrapeTargets(&service, networkingv1.NetworkPolicyPort{Port: utils.IntStrPtrFromInt(port), Protocol: utils.ProtocolPtr(corev1.ProtocolTCP)}))
return cl.Patch(ctx, &service, patch)
})
}

return taskFns
}

func migrationTasksForValiServices(cl client.Client, services []corev1.Service) []flow.TaskFn {
var taskFns []flow.TaskFn

for _, svc := range services {
service := svc

taskFns = append(taskFns, func(ctx context.Context) error {
patch := client.MergeFrom(service.DeepCopy())
metav1.SetMetaDataAnnotation(&service.ObjectMeta, resourcesv1alpha1.NetworkingPodLabelSelectorNamespaceAlias, v1beta1constants.LabelNetworkPolicyShootNamespaceAlias)
utilruntime.Must(gardenerutils.InjectNetworkPolicyNamespaceSelectors(&service, metav1.LabelSelector{MatchLabels: map[string]string{corev1.LabelMetadataName: v1beta1constants.GardenNamespace}}))
return cl.Patch(ctx, &service, patch)
})
}

return taskFns
}

func migrationTasksForPrometheusServices(cl client.Client, services []corev1.Service) []flow.TaskFn {
var taskFns []flow.TaskFn

for _, svc := range services {
service := svc

taskFns = append(taskFns, func(ctx context.Context) error {
patch := client.MergeFrom(service.DeepCopy())
metav1.SetMetaDataAnnotation(&service.ObjectMeta, resourcesv1alpha1.NetworkingPodLabelSelectorNamespaceAlias, v1beta1constants.LabelNetworkPolicyShootNamespaceAlias)
utilruntime.Must(gardenerutils.InjectNetworkPolicyNamespaceSelectors(&service, metav1.LabelSelector{MatchLabels: map[string]string{corev1.LabelMetadataName: v1beta1constants.GardenNamespace}}))
return cl.Patch(ctx, &service, patch)
})
}

return taskFns
}

func migrationTasksForShootVPARecommenders(cl client.Client, shootNamespaces []corev1.Namespace) []flow.TaskFn {
var taskFns []flow.TaskFn

for _, ns := range shootNamespaces {
namespace := ns

// It is forbidden to create a new resource in already terminating Namespace.
if namespace.DeletionTimestamp != nil {
continue
}

taskFns = append(taskFns, func(ctx context.Context) error {
service := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "vpa-recommender",
Namespace: namespace.Name,
},
Spec: corev1.ServiceSpec{
Selector: map[string]string{v1beta1constants.LabelApp: "vpa-recommender"},
Ports: []corev1.ServicePort{{
Port: 8942,
TargetPort: intstr.FromInt(8942),
}},
},
}

metav1.SetMetaDataAnnotation(&service.ObjectMeta, resourcesv1alpha1.NetworkingPodLabelSelectorNamespaceAlias, v1beta1constants.LabelNetworkPolicyShootNamespaceAlias)
utilruntime.Must(gardenerutils.InjectNetworkPolicyNamespaceSelectors(service, metav1.LabelSelector{MatchLabels: map[string]string{corev1.LabelMetadataName: v1beta1constants.GardenNamespace}}))

return client.IgnoreAlreadyExists(cl.Create(ctx, service))
})
}

return taskFns
}

func (g *garden) updateProcessingShootStatusToAborted(ctx context.Context, gardenClient client.Client) error {
shootList := &gardencorev1beta1.ShootList{}
if err := gardenClient.List(ctx, shootList); err != nil {
Expand Down
16 changes: 8 additions & 8 deletions docs/deployment/feature_gates.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,16 +22,10 @@ The following tables are a summary of the feature gates that you can set on diff
|-------------------------------------|---------|---------|--------|--------|
| HVPA | `false` | `Alpha` | `0.31` | |
| HVPAForShootedSeed | `false` | `Alpha` | `0.32` | |
| HAControlPlanes | `false` | `Alpha` | `1.49` | `1.70` |
| HAControlPlanes | `true` | `Beta` | `1.71` | `1.72` |
| HAControlPlanes | `true` | `GA` | `1.73` | |
| DefaultSeccompProfile | `false` | `Alpha` | `1.54` | |
| CoreDNSQueryRewriting | `false` | `Alpha` | `1.55` | |
| IPv6SingleStack | `false` | `Alpha` | `1.63` | |
| MutableShootSpecNetworkingNodes | `false` | `Alpha` | `1.64` | |
| FullNetworkPoliciesInRuntimeCluster | `false` | `Alpha` | `1.66` | `1.70` |
| FullNetworkPoliciesInRuntimeCluster | `true` | `Beta` | `1.71` | `1.72` |
| FullNetworkPoliciesInRuntimeCluster | `true` | `GA` | `1.73` | |
| WorkerlessShoots | `false` | `Alpha` | `1.70` | |
| MachineControllerManagerDeployment | `false` | `Alpha` | `1.73` | |
| DisableScalingClassesForShoots | `false` | `Alpha` | `1.73` | |
Expand Down Expand Up @@ -126,6 +120,14 @@ The following tables are a summary of the feature gates that you can set on diff
| APIServerSNI | `true` | `Beta` | `1.19` | |
| APIServerSNI | `true` | `Deprecated` | `1.48` | `1.72` |
| APIServerSNI | `true` | `Removed` | `1.73` | |
| HAControlPlanes | `false` | `Alpha` | `1.49` | `1.70` |
| HAControlPlanes | `true` | `Beta` | `1.71` | `1.72` |
| HAControlPlanes | `true` | `GA` | `1.73` | `1.73` |
| HAControlPlanes | `true` | `Removed` | `1.74` | |
| FullNetworkPoliciesInRuntimeCluster | `false` | `Alpha` | `1.66` | `1.70` |
| FullNetworkPoliciesInRuntimeCluster | `true` | `Beta` | `1.71` | `1.72` |
| FullNetworkPoliciesInRuntimeCluster | `true` | `GA` | `1.73` | `1.73` |
| FullNetworkPoliciesInRuntimeCluster | `true` | `Removed` | `1.74` | |

## Using a Feature

Expand Down Expand Up @@ -168,12 +170,10 @@ A *General Availability* (GA) feature is also referred to as a *stable* feature.
| HVPA | `gardenlet`, `gardener-operator` | Enables simultaneous horizontal and vertical scaling in garden or seed clusters. |
| HVPAForShootedSeed | `gardenlet` | Enables simultaneous horizontal and vertical scaling in managed seed (aka "shooted seed") clusters. |
| SecretBindingProviderValidation | `gardener-apiserver` | Enables validations on Gardener API server that:<br>- requires the provider type of a SecretBinding to be set (on SecretBinding creation)<br>- requires the SecretBinding provider type to match the Shoot provider type (on Shoot creation)<br>- enforces immutability on the provider type of a SecretBinding |
| HAControlPlanes | `gardener-apiserver` | HAControlPlanes allows shoot control planes to be run in high availability mode. |
| DefaultSeccompProfile | `gardenlet`, `gardener-operator` | Enables the defaulting of the seccomp profile for Gardener managed workload in the garden or seed to `RuntimeDefault`. |
| CoreDNSQueryRewriting | `gardenlet` | Enables automatic DNS query rewriting in shoot cluster's CoreDNS to shortcut name resolution of fully qualified in-cluster and out-of-cluster names, which follow a user-defined pattern. Details can be found in [DNS Search Path Optimization](../usage/dns-search-path-optimization.md). |
| IPv6SingleStack | `gardener-apiserver`, `gardenlet` | Allows creating seed and shoot clusters with [IPv6 single-stack networking](../usage/ipv6.md) enabled in their spec ([GEP-21](../proposals/21-ipv6-singlestack-local.md)). If enabled in gardenlet, the default behavior is unchanged, but setting `ipFamilies=[IPv6]` in the `seedConfig` is allowed. Only if the `ipFamilies` setting is changed, gardenlet behaves differently. |
| MutableShootSpecNetworkingNodes | `gardener-apiserver` | Allows updating the field `spec.networking.nodes`. The validity of the values has to be checked in the provider extensions. Only enable this feature gate when your system runs provider extensions which have implemented the validation. |
| FullNetworkPoliciesInRuntimeCluster | `gardenlet`, `gardener-operator` | Enables the `NetworkPolicy` controller to place 'deny-all' network policies in all relevant namespaces in the runtime cluster. |
| WorkerlessShoots | `gardener-apiserver` | WorkerlessShoots allows creation of Shoot clusters with no worker pools. |
| MachineControllerManagerDeployment | `gardenlet` | Enables Gardener to take over the deployment of the machine-controller-manager. If enabled, all registered provider extensions must support injecting the provider-specific MCM sidecar container into the deployment via the `controlplane` webhook. |
| DisableScalingClassesForShoots | `gardenlet` | Disables assigning a ScalingClass to Shoots based on their maximum Node count. All Shoot kube-apiservers will get the same initial resource requests for CPU and memory instead of making this depend on the ScalingClass. |
1 change: 0 additions & 1 deletion pkg/apis/core/v1beta1/constants/types_constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,6 @@ const (
// automatic scale-down shall be disabled for the etcd, kube-apiserver, kube-controller-manager.
// Note that this annotation is alpha and can be removed anytime without further notice. Only use it if you know
// what you do.
// TODO(shreyas-s-rao): Deprecate HA annotation with the stable release of zonal clusters feature.
ShootAlphaControlPlaneScaleDownDisabled = "alpha.control-plane.scaling.shoot.gardener.cloud/scale-down-disabled"

// ShootAlphaControlPlaneHAVPN is a constant for an annotation on the Shoot resource to enforce
Expand Down
1 change: 0 additions & 1 deletion pkg/apiserver/features/features.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import (
// RegisterFeatureGates registers the feature gates of gardener-apiserver.
func RegisterFeatureGates() {
utilruntime.Must(features.DefaultFeatureGate.Add(features.GetFeatures(
features.HAControlPlanes,
features.IPv6SingleStack,
features.MutableShootSpecNetworkingNodes,
features.WorkerlessShoots,
Expand Down
8 changes: 0 additions & 8 deletions pkg/controller/networkpolicy/add_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,14 +53,6 @@ var _ = Describe("Add", func() {
networkPolicy = &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "kubernetes"}}
})

It("should return true because the NetworkPolicy has name 'allow-to-seed-apiserver'", func() {
networkPolicy.Name = "allow-to-seed-apiserver"
Expect(p.Create(event.CreateEvent{Object: networkPolicy})).To(BeTrue())
Expect(p.Update(event.UpdateEvent{ObjectNew: networkPolicy})).To(BeTrue())
Expect(p.Delete(event.DeleteEvent{Object: networkPolicy})).To(BeTrue())
Expect(p.Generic(event.GenericEvent{Object: networkPolicy})).To(BeTrue())
})

It("should return true because the NetworkPolicy has name 'allow-to-runtime-apiserver'", func() {
networkPolicy.Name = "allow-to-runtime-apiserver"
Expect(p.Create(event.CreateEvent{Object: networkPolicy})).To(BeTrue())
Expand Down
Loading

0 comments on commit e9bcb79

Please sign in to comment.