Kubernetes Version Configurability per Worker Pool

docs/usage-as-end-user.md

@@ -452,3 +452,8 @@ Shoot clusters with Kubernetes v1.17 or less will use the in-tree `kubernetes.io
 The Kubernetes scheduler allows configurable limit for the number of volumes that can be attached to a node. See https://k8s.io/docs/concepts/storage/storage-limits/#custom-limits.
 
 CSI drivers usually have a different procedure for configuring this custom limit. By default, the EBS CSI driver parses the machine type name and then decides the volume limit. However, this is only a rough approximation and not good enough in most cases. Specifying the volume attach limit via command line flag (`--volume-attach-limit`) is currently the alternative until a more sophisticated solution presents itself (dynamically discovering the maximum number of attachable volume per EC2 machine type, see also https://github.com/kubernetes-sigs/aws-ebs-csi-driver/issues/347). The AWS extension allows the `--volume-attach-limit` flag of the EBS CSI driver to be configurable via `aws.provider.extensions.gardener.cloud/volume-attach-limit` annotation on the `Shoot` resource. If the annotation is added to an existing `Shoot`, then reconciliation needs to be triggered manually (see [Immediate reconciliation](https://github.com/gardener/gardener/blob/master/docs/usage/shoot_operations.md#immediate-reconciliation)), as in general adding annotation to resource is not a change that leads to `.metadata.generation` increase in general.
+
+## Kubernetes Versions per Worker Pool
+
+This extension supports `gardener/gardener`'s `WorkerPoolKubernetesVersion` feature gate, i.e., having [worker pools with overridden Kubernetes versions](https://github.com/gardener/gardener/blob/8a9c88866ec5fce59b5acf57d4227eeeb73669d7/example/90-shoot.yaml#L69-L70) since `gardener-extension-provider-aws@v1.34`.
+Note that this feature is only usable for `Shoot`s whose `.spec.kubernetes.version` is greater or equal than the CSI migration version (`1.18`).

pkg/admission/validator/shoot.go

@@ -20,14 +20,18 @@ import (
 	"fmt"
 	"reflect"
 
-	"github.com/gardener/gardener-extension-provider-aws/pkg/apis/aws"
+	api "github.com/gardener/gardener-extension-provider-aws/pkg/apis/aws"
 	awsvalidation "github.com/gardener/gardener-extension-provider-aws/pkg/apis/aws/validation"
+	"github.com/gardener/gardener-extension-provider-aws/pkg/aws"
 
+	"github.com/Masterminds/semver"
+	extensionscontroller "github.com/gardener/gardener/extensions/pkg/controller"
 	extensionswebhook "github.com/gardener/gardener/extensions/pkg/webhook"
 	"github.com/gardener/gardener/pkg/apis/core"
 	gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
 	kutil "github.com/gardener/gardener/pkg/utils/kubernetes"
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/apimachinery/pkg/util/validation/field"
@@ -44,10 +48,12 @@ type shoot struct {
 	apiReader      client.Reader
 	decoder        runtime.Decoder
 	lenientDecoder runtime.Decoder
+	scheme         *runtime.Scheme
 }
 
 // InjectScheme injects the given scheme into the validator.
 func (s *shoot) InjectScheme(scheme *runtime.Scheme) error {
+	s.scheme = scheme
 	s.decoder = serializer.NewCodecFactory(scheme, serializer.EnableStrict).UniversalDecoder()
 	s.lenientDecoder = serializer.NewCodecFactory(scheme).UniversalDecoder()
 	return nil
@@ -119,9 +125,24 @@ func (s *shoot) validateShoot(_ context.Context, shoot *core.Shoot) error {
 	}
 
 	// WorkerConfig and Shoot workers
+	shootV1beta1 := &gardencorev1beta1.Shoot{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: gardencorev1beta1.SchemeGroupVersion.String(),
+			Kind:       "Shoot",
+		},
+	}
+	if err := s.scheme.Convert(shoot, shootV1beta1, nil); err != nil {
+		return err
+	}
+
+	csiMigrationVersion, err := semver.NewVersion(aws.GetCSIMigrationKubernetesVersion(&extensionscontroller.Cluster{Shoot: shootV1beta1}))
+	if err != nil {
+		return err
+	}
+
 	fldPath = fldPath.Child("workers")
 	for i, worker := range shoot.Spec.Provider.Workers {
-		var workerConfig *aws.WorkerConfig
+		var workerConfig *api.WorkerConfig
 		if worker.ProviderConfig != nil {
 			wc, err := decodeWorkerConfig(s.decoder, worker.ProviderConfig, fldPath.Index(i).Child("providerConfig"))
 			if err != nil {
@@ -130,7 +151,7 @@ func (s *shoot) validateShoot(_ context.Context, shoot *core.Shoot) error {
 			workerConfig = wc
 		}
 
-		if errList := awsvalidation.ValidateWorker(worker, infraConfig.Networks.Zones, workerConfig, fldPath.Index(i)); len(errList) != 0 {
+		if errList := awsvalidation.ValidateWorker(worker, csiMigrationVersion, infraConfig.Networks.Zones, workerConfig, fldPath.Index(i)); len(errList) != 0 {
 			return errList.ToAggregate()
 		}
 	}
@@ -198,7 +219,7 @@ func (s *shoot) validateShootCreation(ctx context.Context, shoot *core.Shoot) er
 	return s.validateShootSecret(ctx, shoot)
 }
 
-func (s *shoot) validateAgainstCloudProfile(ctx context.Context, shoot *core.Shoot, oldInfraConfig, infraConfig *aws.InfrastructureConfig, fldPath *field.Path) error {
+func (s *shoot) validateAgainstCloudProfile(ctx context.Context, shoot *core.Shoot, oldInfraConfig, infraConfig *api.InfrastructureConfig, fldPath *field.Path) error {
 	cloudProfile := &gardencorev1beta1.CloudProfile{}
 	if err := s.client.Get(ctx, kutil.Key(shoot.Spec.CloudProfileName), cloudProfile); err != nil {
 		return err

pkg/apis/aws/validation/shoot.go

@@ -19,6 +19,7 @@ import (
 
 	apisaws "github.com/gardener/gardener-extension-provider-aws/pkg/apis/aws"
 
+	"github.com/Masterminds/semver"
 	"github.com/gardener/gardener/pkg/apis/core"
 	"github.com/gardener/gardener/pkg/apis/core/validation"
 	apivalidation "k8s.io/apimachinery/pkg/api/validation"
@@ -38,14 +39,29 @@ func ValidateNetworking(networking core.Networking, fldPath *field.Path) field.E
 }
 
 // ValidateWorker validates a worker of a Shoot.
-func ValidateWorker(worker core.Worker, zones []apisaws.Zone, workerConfig *apisaws.WorkerConfig, fldPath *field.Path) field.ErrorList {
+func ValidateWorker(worker core.Worker, csiMigrationVersion *semver.Version, zones []apisaws.Zone, workerConfig *apisaws.WorkerConfig, fldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}
 
 	awsZones := sets.NewString()
 	for _, awsZone := range zones {
 		awsZones.Insert(awsZone.Name)
 	}
 
+	// Ensure the kubelet version is not lower than the version in which the extension performs CSI migration.
+	if worker.Kubernetes != nil && worker.Kubernetes.Version != nil {
+		path := fldPath.Child("kubernetes", "version")
+
+		v, err := semver.NewVersion(*worker.Kubernetes.Version)
+		if err != nil {
+			allErrs = append(allErrs, field.Invalid(path, *worker.Kubernetes.Version, err.Error()))
+			return allErrs
+		}
+
+		if v.LessThan(csiMigrationVersion) {
+			allErrs = append(allErrs, field.Forbidden(path, fmt.Sprintf("cannot use kubelet version (%s) lower than CSI migration version (%s)", v.String(), csiMigrationVersion.String())))
+		}
+	}
+
 	if worker.Volume == nil {
 		allErrs = append(allErrs, field.Required(fldPath.Child("volume"), "must not be nil"))
 	} else {

pkg/apis/aws/validation/shoot_test.go

@@ -20,6 +20,7 @@ import (
 	apisaws "github.com/gardener/gardener-extension-provider-aws/pkg/apis/aws"
 	. "github.com/gardener/gardener-extension-provider-aws/pkg/apis/aws/validation"
 
+	"github.com/Masterminds/semver"
 	"github.com/gardener/gardener/pkg/apis/core"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -35,7 +36,7 @@ var _ = Describe("Shoot validation", func() {
 
 		It("should return no error because nodes CIDR was provided", func() {
 			networking := core.Networking{
-				Nodes: pointer.StringPtr("1.2.3.4/5"),
+				Nodes: pointer.String("1.2.3.4/5"),
 			}
 
 			errorList := ValidateNetworking(networking, networkingPath)
@@ -68,7 +69,7 @@ var _ = Describe("Shoot validation", func() {
 			worker = core.Worker{
 				Name: "worker1",
 				Volume: &core.Volume{
-					Type:       pointer.StringPtr("Volume"),
+					Type:       pointer.String("Volume"),
 					VolumeSize: "30G",
 				},
 				Zones: []string{
@@ -92,21 +93,50 @@ var _ = Describe("Shoot validation", func() {
 
 		Describe("#ValidateWorker", func() {
 			It("should pass when the workerConfig is nil", func() {
-				errorList := ValidateWorker(worker, awsZones, nil, field.NewPath(""))
+				errorList := ValidateWorker(worker, nil, awsZones, nil, field.NewPath(""))
 
 				Expect(errorList).To(BeEmpty())
 			})
 
+			It("should pass when the kubernetes version is equal to the CSI migration version", func() {
+				worker.Kubernetes = &core.WorkerKubernetes{Version: pointer.String("1.18.0")}
+
+				errorList := ValidateWorker(worker, semver.MustParse("1.18"), awsZones, nil, field.NewPath("workers").Index(0))
+
+				Expect(errorList).To(BeEmpty())
+			})
+
+			It("should pass when the kubernetes version is higher to the CSI migration version", func() {
+				worker.Kubernetes = &core.WorkerKubernetes{Version: pointer.String("1.19.0")}
+
+				errorList := ValidateWorker(worker, semver.MustParse("1.18"), awsZones, nil, field.NewPath("workers").Index(0))
+
+				Expect(errorList).To(BeEmpty())
+			})
+
+			It("should not allow when the kubernetes version is lower than the CSI migration version", func() {
+				worker.Kubernetes = &core.WorkerKubernetes{Version: pointer.String("1.17.0")}
+
+				errorList := ValidateWorker(worker, semver.MustParse("1.18.0"), awsZones, nil, field.NewPath("workers").Index(0))
+
+				Expect(errorList).To(ConsistOf(
+					PointTo(MatchFields(IgnoreExtras, Fields{
+						"Type":  Equal(field.ErrorTypeForbidden),
+						"Field": Equal("workers[0].kubernetes.version"),
+					})),
+				))
+			})
+
 			It("should pass because the worker is configured correctly", func() {
-				errorList := ValidateWorker(worker, awsZones, &apisaws.WorkerConfig{}, field.NewPath(""))
+				errorList := ValidateWorker(worker, nil, awsZones, &apisaws.WorkerConfig{}, field.NewPath(""))
 
 				Expect(errorList).To(BeEmpty())
 			})
 
 			It("should forbid because volume is not configured", func() {
 				worker.Volume = nil
 
-				errorList := ValidateWorker(worker, awsZones, &apisaws.WorkerConfig{}, field.NewPath("workers").Index(0))
+				errorList := ValidateWorker(worker, nil, awsZones, &apisaws.WorkerConfig{}, field.NewPath("workers").Index(0))
 
 				Expect(errorList).To(ConsistOf(
 					PointTo(MatchFields(IgnoreExtras, Fields{
@@ -117,9 +147,9 @@ var _ = Describe("Shoot validation", func() {
 			})
 
 			It("should forbid because volume type io1 is used but no worker config provided", func() {
-				worker.Volume.Type = pointer.StringPtr(string(apisaws.VolumeTypeIO1))
+				worker.Volume.Type = pointer.String(string(apisaws.VolumeTypeIO1))
 
-				errorList := ValidateWorker(worker, awsZones, &apisaws.WorkerConfig{}, field.NewPath("workers").Index(0))
+				errorList := ValidateWorker(worker, nil, awsZones, &apisaws.WorkerConfig{}, field.NewPath("workers").Index(0))
 
 				Expect(errorList).To(ConsistOf(
 					PointTo(MatchFields(IgnoreExtras, Fields{
@@ -134,10 +164,10 @@ var _ = Describe("Shoot validation", func() {
 			})
 
 			It("should allow because volume type io1 and worker config provided", func() {
-				worker.Volume.Type = pointer.StringPtr(string(apisaws.VolumeTypeIO1))
+				worker.Volume.Type = pointer.String(string(apisaws.VolumeTypeIO1))
 				worker.ProviderConfig = &runtime.RawExtension{}
 
-				errorList := ValidateWorker(worker, awsZones, &apisaws.WorkerConfig{Volume: &apisaws.Volume{IOPS: &iops}}, field.NewPath("workers").Index(0))
+				errorList := ValidateWorker(worker, nil, awsZones, &apisaws.WorkerConfig{Volume: &apisaws.Volume{IOPS: &iops}}, field.NewPath("workers").Index(0))
 
 				Expect(errorList).To(BeEmpty())
 			})
@@ -147,7 +177,7 @@ var _ = Describe("Shoot validation", func() {
 				worker.Volume.VolumeSize = ""
 				worker.DataVolumes = []core.DataVolume{{}}
 
-				errorList := ValidateWorker(worker, awsZones, &apisaws.WorkerConfig{}, field.NewPath("workers").Index(0))
+				errorList := ValidateWorker(worker, nil, awsZones, &apisaws.WorkerConfig{}, field.NewPath("workers").Index(0))
 
 				Expect(errorList).To(ConsistOf(
 					PointTo(MatchFields(IgnoreExtras, Fields{
@@ -174,11 +204,11 @@ var _ = Describe("Shoot validation", func() {
 					worker.DataVolumes = append(worker.DataVolumes, core.DataVolume{
 						Name:       fmt.Sprintf("foo%d", i),
 						VolumeSize: "20Gi",
-						Type:       pointer.StringPtr("foo"),
+						Type:       pointer.String("foo"),
 					})
 				}
 
-				errorList := ValidateWorker(worker, awsZones, &apisaws.WorkerConfig{}, field.NewPath("workers").Index(0))
+				errorList := ValidateWorker(worker, nil, awsZones, &apisaws.WorkerConfig{}, field.NewPath("workers").Index(0))
 
 				Expect(errorList).To(ConsistOf(
 					PointTo(MatchFields(IgnoreExtras, Fields{
@@ -191,7 +221,7 @@ var _ = Describe("Shoot validation", func() {
 			It("should forbid because worker does not specify a zone", func() {
 				worker.Zones = nil
 
-				errorList := ValidateWorker(worker, awsZones, &apisaws.WorkerConfig{}, field.NewPath("workers").Index(0))
+				errorList := ValidateWorker(worker, nil, awsZones, &apisaws.WorkerConfig{}, field.NewPath("workers").Index(0))
 
 				Expect(errorList).To(ConsistOf(
 					PointTo(MatchFields(IgnoreExtras, Fields{
@@ -205,7 +235,7 @@ var _ = Describe("Shoot validation", func() {
 				worker.Zones[0] = ""
 				worker.Zones[1] = "not-available"
 
-				errorList := ValidateWorker(worker, awsZones, &apisaws.WorkerConfig{}, field.NewPath("workers").Index(0))
+				errorList := ValidateWorker(worker, nil, awsZones, &apisaws.WorkerConfig{}, field.NewPath("workers").Index(0))
 
 				Expect(errorList).To(ConsistOf(
 					PointTo(MatchFields(IgnoreExtras, Fields{

pkg/webhook/controlplane/ensurer.go

@@ -23,7 +23,6 @@ import (
 
 	"github.com/Masterminds/semver"
 	"github.com/coreos/go-systemd/v22/unit"
-	extensionscontroller "github.com/gardener/gardener/extensions/pkg/controller"
 	"github.com/gardener/gardener/extensions/pkg/controller/csimigration"
 	extensionswebhook "github.com/gardener/gardener/extensions/pkg/webhook"
 	gcontext "github.com/gardener/gardener/extensions/pkg/webhook/context"
@@ -60,8 +59,8 @@ func (e *ensurer) InjectClient(client client.Client) error {
 	return nil
 }
 
-func computeCSIMigrationCompleteFeatureGate(cluster *extensionscontroller.Cluster) (string, error) {
-	k8sGreaterEqual121, err := versionutils.CompareVersions(cluster.Shoot.Spec.Kubernetes.Version, ">=", "1.21")
+func computeCSIMigrationCompleteFeatureGate(version string) (string, error) {
+	k8sGreaterEqual121, err := versionutils.CompareVersions(version, ">=", "1.21")
 	if err != nil {
 		return "", err
 	}
@@ -86,7 +85,7 @@ func (e *ensurer) EnsureKubeAPIServerDeployment(ctx context.Context, gctx gconte
 	if err != nil {
 		return err
 	}
-	csiMigrationCompleteFeatureGate, err := computeCSIMigrationCompleteFeatureGate(cluster)
+	csiMigrationCompleteFeatureGate, err := computeCSIMigrationCompleteFeatureGate(cluster.Shoot.Spec.Kubernetes.Version)
 	if err != nil {
 		return err
 	}
@@ -115,7 +114,7 @@ func (e *ensurer) EnsureKubeControllerManagerDeployment(ctx context.Context, gct
 	if err != nil {
 		return err
 	}
-	csiMigrationCompleteFeatureGate, err := computeCSIMigrationCompleteFeatureGate(cluster)
+	csiMigrationCompleteFeatureGate, err := computeCSIMigrationCompleteFeatureGate(cluster.Shoot.Spec.Kubernetes.Version)
 	if err != nil {
 		return err
 	}
@@ -145,7 +144,7 @@ func (e *ensurer) EnsureKubeSchedulerDeployment(ctx context.Context, gctx gconte
 	if err != nil {
 		return err
 	}
-	csiMigrationCompleteFeatureGate, err := computeCSIMigrationCompleteFeatureGate(cluster)
+	csiMigrationCompleteFeatureGate, err := computeCSIMigrationCompleteFeatureGate(cluster.Shoot.Spec.Kubernetes.Version)
 	if err != nil {
 		return err
 	}
@@ -439,7 +438,7 @@ func (e *ensurer) EnsureKubeletConfiguration(ctx context.Context, gctx gcontext.
 	if err != nil {
 		return err
 	}
-	csiMigrationCompleteFeatureGate, err := computeCSIMigrationCompleteFeatureGate(cluster)
+	csiMigrationCompleteFeatureGate, err := computeCSIMigrationCompleteFeatureGate(kubeletVersion.String())
 	if err != nil {
 		return err
 	}