From 0f8b1b3f2873db17952b6041272ad797e3684973 Mon Sep 17 00:00:00 2001 From: Rafael Franzke Date: Fri, 24 Jun 2022 10:34:49 +0200 Subject: [PATCH] Enhance shoot update test and promote `WorkerPoolKubernetesVersion` to GA (#6166) * Make creating shoot client from admin kubeconfig reusable * Make shoot update test work with worker pools with overridden Kubernetes versions * Change label of default e2e shoot test from `fast` to `simple` * Perform shoot update with multiple worker pools in simple e2e test * Promote `WorkerPoolKubernetesVersion` feature gate to GA * Address PR review feedback * Address PR review feedback * Move CreateShootClientFromAdminKubeconfig into access pkg Co-authored-by: Tim Ebert --- .test-defs/ShootKubernetesUpdateTest.yaml | 1 + Makefile | 6 +- .../app/gardener_apiserver.go | 34 --- docs/deployment/feature_gates.md | 5 +- docs/deployment/getting_started_locally.md | 2 +- docs/development/getting_started_locally.md | 2 +- docs/development/testing.md | 2 +- pkg/apis/core/validation/shoot.go | 10 +- pkg/apis/core/validation/shoot_test.go | 27 --- pkg/features/features.go | 3 +- ..._and_delete.go => create_update_delete.go} | 27 ++- .../internal/rotation/secret_encryption.go | 2 +- .../shoot/internal/rotation/shoot_access.go | 2 +- .../system/shoot_update/update_test.go | 43 +--- .../shoots}/access/adminkubeconfig.go | 6 +- .../internal => utils/shoots}/access/csr.go | 0 .../shoots}/access/serviceaccount.go | 0 .../shoots}/access/statictokenkubeconfig.go | 0 test/utils/shoots/update/update.go | 205 ++++++++++++++++++ 19 files changed, 257 insertions(+), 120 deletions(-) rename test/e2e/shoot/{create_and_delete.go => create_update_delete.go} (59%) rename test/{e2e/shoot/internal => utils/shoots}/access/adminkubeconfig.go (100%) rename test/{e2e/shoot/internal => utils/shoots}/access/csr.go (100%) rename test/{e2e/shoot/internal => utils/shoots}/access/serviceaccount.go (100%) rename test/{e2e/shoot/internal => utils/shoots}/access/statictokenkubeconfig.go (100%) create mode 100644 test/utils/shoots/update/update.go diff --git a/.test-defs/ShootKubernetesUpdateTest.yaml b/.test-defs/ShootKubernetesUpdateTest.yaml index 426803521eb..61ed79c15d0 100644 --- a/.test-defs/ShootKubernetesUpdateTest.yaml +++ b/.test-defs/ShootKubernetesUpdateTest.yaml @@ -18,4 +18,5 @@ spec: -shoot-name=$SHOOT_NAME -project-namespace=$PROJECT_NAMESPACE -version=$K8S_VERSION + -version-worker-pools=$K8S_VERSION_WORKER_POOLS image: eu.gcr.io/gardener-project/3rd/golang:1.18.1 diff --git a/Makefile b/Makefile index 2bec3dbfae4..e0e0b3bc1b4 100644 --- a/Makefile +++ b/Makefile @@ -259,7 +259,7 @@ verify-extended: check-generate check format test-cov test-cov-clean test-integr # Rules for local environment # ##################################################################### -kind-up kind-down gardener-up gardener-down register-local-env tear-down-local-env register-kind2-env tear-down-kind2-env test-e2e-local-fast test-e2e-local: export KUBECONFIG = $(GARDENER_LOCAL_KUBECONFIG) +kind-up kind-down gardener-up gardener-down register-local-env tear-down-local-env register-kind2-env tear-down-kind2-env test-e2e-local-simple test-e2e-local: export KUBECONFIG = $(GARDENER_LOCAL_KUBECONFIG) kind2-up kind2-down gardenlet-kind2-up gardenlet-kind2-down: export KUBECONFIG = $(GARDENER_LOCAL2_KUBECONFIG) @@ -325,8 +325,8 @@ register-kind2-env: tear-down-kind2-env: kubectl delete -k $(REPO_ROOT)/example/provider-local/seed-kind2/local -test-e2e-local-fast: $(GINKGO) - ./hack/test-e2e-local.sh --label-filter "Shoot && fast" +test-e2e-local-simple: $(GINKGO) + ./hack/test-e2e-local.sh --label-filter "Shoot && simple" test-e2e-local: $(GINKGO) @# run at maximum 5 tests in parallel for now until we have better experience of how much load a single prow pod can take diff --git a/cmd/gardener-apiserver/app/gardener_apiserver.go b/cmd/gardener-apiserver/app/gardener_apiserver.go index e93fb514948..61476ded8a3 100644 --- a/cmd/gardener-apiserver/app/gardener_apiserver.go +++ b/cmd/gardener-apiserver/app/gardener_apiserver.go @@ -18,8 +18,6 @@ import ( "context" "errors" "flag" - "fmt" - "time" "k8s.io/component-base/logs" @@ -44,7 +42,6 @@ import ( seedmanagementinformer "github.com/gardener/gardener/pkg/client/seedmanagement/informers/externalversions" settingsclientset "github.com/gardener/gardener/pkg/client/settings/clientset/versioned" settingsinformer "github.com/gardener/gardener/pkg/client/settings/informers/externalversions" - "github.com/gardener/gardener/pkg/features" "github.com/gardener/gardener/pkg/logger" "github.com/gardener/gardener/pkg/openapi" @@ -52,7 +49,6 @@ import ( "github.com/spf13/pflag" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" utilerrors "k8s.io/apimachinery/pkg/util/errors" @@ -69,7 +65,6 @@ import ( kubeinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" "k8s.io/component-base/version" "k8s.io/component-base/version/verflag" @@ -348,35 +343,6 @@ func (o *Options) Run(ctx context.Context) error { return err } - if !utilfeature.DefaultFeatureGate.Enabled(features.WorkerPoolKubernetesVersion) { - if err := server.GenericAPIServer.AddPostStartHook("validate-WorkerPoolKubernetesVersion-feature-gate", func(hookContext genericapiserver.PostStartHookContext) error { - timeoutCtx, cancel := context.WithTimeout(ctx, time.Minute) - defer cancel() - - shootInformer := o.CoreInformerFactory.Core().InternalVersion().Shoots() - if !cache.WaitForCacheSync(timeoutCtx.Done(), shootInformer.Informer().HasSynced) { - return fmt.Errorf("error while waiting for shoot informer cache to be synced") - } - - shoots, err := shootInformer.Lister().List(labels.Everything()) - if err != nil { - return err - } - - for _, shoot := range shoots { - for _, worker := range shoot.Spec.Provider.Workers { - if worker.Kubernetes != nil && worker.Kubernetes.Version != nil { - return fmt.Errorf("WorkerPoolKubernetesVersion feature gate cannot be disabled since shoot %q still has worker pool %q which specifies .kubernetes.version", shoot.Name, worker.Name) - } - } - } - - return nil - }); err != nil { - return err - } - } - return server.GenericAPIServer.PrepareRun().Run(ctx.Done()) } diff --git a/docs/deployment/feature_gates.md b/docs/deployment/feature_gates.md index 49a99940bda..14445510df3 100644 --- a/docs/deployment/feature_gates.md +++ b/docs/deployment/feature_gates.md @@ -35,8 +35,6 @@ The following tables are a summary of the feature gates that you can set on diff | RotateSSHKeypairOnMaintenance | `false` | `Alpha` | `1.28` | `1.44` | | RotateSSHKeypairOnMaintenance | `true` | `Beta` | `1.45` | | | RotateSSHKeypairOnMaintenance (deprecated) | `false` | `Beta` | `1.48` | | -| WorkerPoolKubernetesVersion | `false` | `Alpha` | `1.35` | `1.45` | -| WorkerPoolKubernetesVersion | `true` | `Beta` | `1.46` | | | CopyEtcdBackupsDuringControlPlaneMigration | `false` | `Alpha` | `1.37` | | | SecretBindingProviderValidation | `false` | `Alpha` | `1.38` | | | ForceRestore | `false` | `Alpha` | `1.39` | | @@ -82,6 +80,9 @@ The following tables are a summary of the feature gates that you can set on diff | ShootMaxTokenExpirationValidation | `false` | `Alpha` | `1.43` | `1.45` | | ShootMaxTokenExpirationValidation | `true` | `Beta` | `1.46` | `1.47` | | ShootMaxTokenExpirationValidation | `true` | `GA` | `1.48` | | +| WorkerPoolKubernetesVersion | `false` | `Alpha` | `1.35` | `1.45` | +| WorkerPoolKubernetesVersion | `true` | `Beta` | `1.46` | `1.49` | +| WorkerPoolKubernetesVersion | `true` | `GA` | `1.50` | | ## Using a feature diff --git a/docs/deployment/getting_started_locally.md b/docs/deployment/getting_started_locally.md index b6386f6baaf..b68879363b1 100644 --- a/docs/deployment/getting_started_locally.md +++ b/docs/deployment/getting_started_locally.md @@ -78,7 +78,7 @@ local local local local 1.21.0 Awake Create Pr (Optional): You could also execute a simple e2e test (creating and deleting a shoot) by running ```shell -make test-e2e-local-fast KUBECONFIG="$PWD/example/gardener-local/kind/kubeconfig" +make test-e2e-local-simple KUBECONFIG="$PWD/example/gardener-local/kind/kubeconfig" ``` ### Accessing the `Shoot` cluster diff --git a/docs/development/getting_started_locally.md b/docs/development/getting_started_locally.md index c0404683322..8daf90a0a77 100644 --- a/docs/development/getting_started_locally.md +++ b/docs/development/getting_started_locally.md @@ -123,7 +123,7 @@ local local local local 1.21.0 Awake Create Pr (Optional): You could also execute a simple e2e test (creating and deleting a shoot) by running ```shell -make test-e2e-local-fast KUBECONFIG="$PWD/example/gardener-local/kind/kubeconfig" +make test-e2e-local-simple KUBECONFIG="$PWD/example/gardener-local/kind/kubeconfig" ``` When the shoot got successfully created you can access it as follows: diff --git a/docs/development/testing.md b/docs/development/testing.md index 0ae86063da8..14d2d7501de 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -66,7 +66,7 @@ You can also run these tests on your development machine, using the following co make kind-up export KUBECONFIG=$PWD/example/gardener-local/kind/kubeconfig make gardener-up -make test-e2e-local # alternatively: make test-e2e-local-fast +make test-e2e-local # alternatively: make test-e2e-local-simple ``` If you want to run a specific set of e2e test cases, you can also execute them using `./hack/test-e2e-local.sh` directly in combination with [ginkgo label filters](https://onsi.github.io/ginkgo/#spec-labels). For example: diff --git a/pkg/apis/core/validation/shoot.go b/pkg/apis/core/validation/shoot.go index 946da6d7fc1..c70ae855cd1 100644 --- a/pkg/apis/core/validation/shoot.go +++ b/pkg/apis/core/validation/shoot.go @@ -1171,13 +1171,9 @@ func ValidateWorker(worker core.Worker, kubernetesVersion string, fldPath *field } if worker.Kubernetes != nil { if worker.Kubernetes.Version != nil { - if !utilfeature.DefaultFeatureGate.Enabled(features.WorkerPoolKubernetesVersion) { - allErrs = append(allErrs, field.Forbidden(fldPath.Child("kubernetes", "version"), "worker pool kubernetes version may only be set if WorkerPoolKubernetesVersion feature gate is enabled")) - } else { - workerGroupKubernetesVersion := *worker.Kubernetes.Version - allErrs = append(allErrs, validateWorkerGroupAndControlPlaneKubernetesVersion(kubernetesVersion, workerGroupKubernetesVersion, fldPath.Child("kubernetes", "version"))...) - kubernetesVersion = workerGroupKubernetesVersion - } + workerGroupKubernetesVersion := *worker.Kubernetes.Version + allErrs = append(allErrs, validateWorkerGroupAndControlPlaneKubernetesVersion(kubernetesVersion, workerGroupKubernetesVersion, fldPath.Child("kubernetes", "version"))...) + kubernetesVersion = workerGroupKubernetesVersion } if worker.Kubernetes.Kubelet != nil { diff --git a/pkg/apis/core/validation/shoot_test.go b/pkg/apis/core/validation/shoot_test.go index 1e8fa4123cc..c4ae86f1d7b 100644 --- a/pkg/apis/core/validation/shoot_test.go +++ b/pkg/apis/core/validation/shoot_test.go @@ -2271,22 +2271,7 @@ var _ = Describe("Shoot Validation Tests", func() { }) Context("worker pool kubernetes version", func() { - It("should forbid specifying a worker pool kubernetes version since the WorkerPoolKubernetesVersion feature gate is disabled", func() { - defer test.WithFeatureGate(utilfeature.DefaultFeatureGate, features.WorkerPoolKubernetesVersion, false)() - - newShoot := prepareShootForUpdate(shoot) - newShoot.Spec.Provider.Workers[0].Kubernetes = &core.WorkerKubernetes{Version: &shoot.Spec.Kubernetes.Version} - - Expect(ValidateShootUpdate(newShoot, shoot)).To(ConsistOf(PointTo(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(field.ErrorTypeForbidden), - "Field": Equal("spec.provider.workers[0].kubernetes.version"), - "Detail": Equal("worker pool kubernetes version may only be set if WorkerPoolKubernetesVersion feature gate is enabled"), - })))) - }) - It("should forbid worker pool kubernetes version higher than control plane", func() { - defer test.WithFeatureGate(utilfeature.DefaultFeatureGate, features.WorkerPoolKubernetesVersion, true)() - newShoot := prepareShootForUpdate(shoot) newShoot.Spec.Provider.Workers[0].Kubernetes = &core.WorkerKubernetes{Version: pointer.String("1.21.0")} @@ -2298,8 +2283,6 @@ var _ = Describe("Shoot Validation Tests", func() { }) It("should work to set worker pool kubernetes version equal to control plane version", func() { - defer test.WithFeatureGate(utilfeature.DefaultFeatureGate, features.WorkerPoolKubernetesVersion, true)() - newShoot := prepareShootForUpdate(shoot) newShoot.Spec.Provider.Workers[0].Kubernetes = &core.WorkerKubernetes{Version: pointer.String("1.20.2")} @@ -2307,8 +2290,6 @@ var _ = Describe("Shoot Validation Tests", func() { }) It("should work to set worker pool kubernetes version lower one minor than control plane version", func() { - defer test.WithFeatureGate(utilfeature.DefaultFeatureGate, features.WorkerPoolKubernetesVersion, true)() - shoot.Spec.Provider.Workers[0].Kubernetes = &core.WorkerKubernetes{Version: pointer.String("1.20.2")} newShoot := prepareShootForUpdate(shoot) @@ -2318,8 +2299,6 @@ var _ = Describe("Shoot Validation Tests", func() { }) It("should work to set worker pool kubernetes version lower two minor than control plane version", func() { - defer test.WithFeatureGate(utilfeature.DefaultFeatureGate, features.WorkerPoolKubernetesVersion, true)() - shoot.Spec.Kubernetes.Version = "1.21.0" shoot.Spec.Provider.Workers[0].Kubernetes = &core.WorkerKubernetes{Version: pointer.String("1.20.2")} @@ -2330,8 +2309,6 @@ var _ = Describe("Shoot Validation Tests", func() { }) It("forbid to set worker pool kubernetes version lower three minor than control plane version", func() { - defer test.WithFeatureGate(utilfeature.DefaultFeatureGate, features.WorkerPoolKubernetesVersion, true)() - shoot.Spec.Kubernetes.Version = "1.22.0" shoot.Spec.Provider.Workers[0].Kubernetes = &core.WorkerKubernetes{Version: pointer.String("1.20.2")} @@ -2346,8 +2323,6 @@ var _ = Describe("Shoot Validation Tests", func() { }) It("should work to set worker pool kubernetes version to nil with one minor difference", func() { - defer test.WithFeatureGate(utilfeature.DefaultFeatureGate, features.WorkerPoolKubernetesVersion, true)() - shoot.Spec.Kubernetes.Version = "1.21.0" shoot.Spec.Provider.Workers[0].Kubernetes = &core.WorkerKubernetes{Version: pointer.String("1.20.2")} @@ -2358,8 +2333,6 @@ var _ = Describe("Shoot Validation Tests", func() { }) It("forbid to set worker pool kubernetes version to nil with two minor difference", func() { - defer test.WithFeatureGate(utilfeature.DefaultFeatureGate, features.WorkerPoolKubernetesVersion, true)() - shoot.Spec.Kubernetes.Version = "1.21.0" shoot.Spec.Provider.Workers[0].Kubernetes = &core.WorkerKubernetes{Version: pointer.String("1.19.2")} diff --git a/pkg/features/features.go b/pkg/features/features.go index 7924824686f..384315f45e6 100644 --- a/pkg/features/features.go +++ b/pkg/features/features.go @@ -112,6 +112,7 @@ const ( // owner: @rfranzke @majst01 @mwennrich // alpha: v1.35.0 // beta: v1.46.0 + // GA: v1.50.0 WorkerPoolKubernetesVersion featuregate.Feature = "WorkerPoolKubernetesVersion" // CopyEtcdBackupsDuringControlPlaneMigration enables the copy of etcd backups from the object store of the source seed @@ -197,7 +198,7 @@ var allFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ UseDNSRecords: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, RotateSSHKeypairOnMaintenance: {Default: false, PreRelease: featuregate.Beta}, DenyInvalidExtensionResources: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, - WorkerPoolKubernetesVersion: {Default: true, PreRelease: featuregate.Beta}, + WorkerPoolKubernetesVersion: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, CopyEtcdBackupsDuringControlPlaneMigration: {Default: false, PreRelease: featuregate.Alpha}, SecretBindingProviderValidation: {Default: false, PreRelease: featuregate.Alpha}, ForceRestore: {Default: false, PreRelease: featuregate.Alpha}, diff --git a/test/e2e/shoot/create_and_delete.go b/test/e2e/shoot/create_update_delete.go similarity index 59% rename from test/e2e/shoot/create_and_delete.go rename to test/e2e/shoot/create_update_delete.go index 465123a6f23..33f25461a5c 100644 --- a/test/e2e/shoot/create_and_delete.go +++ b/test/e2e/shoot/create_update_delete.go @@ -18,11 +18,15 @@ import ( "context" "time" - "github.com/gardener/gardener/test/e2e/shoot/internal/access" + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + "github.com/gardener/gardener/test/framework" + "github.com/gardener/gardener/test/utils/shoots/access" + shootupdatesuite "github.com/gardener/gardener/test/utils/shoots/update" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" ) var _ = Describe("Shoot Tests", Label("Shoot"), func() { @@ -30,7 +34,20 @@ var _ = Describe("Shoot Tests", Label("Shoot"), func() { f.Shoot = defaultShoot("") f.Shoot.Name = "e2e-default" - It("Create and Delete", Label("fast"), func() { + // explicitly use one version below the latest supported minor version so that Kubernetes version update test can be + // performed + f.Shoot.Spec.Kubernetes.Version = "1.23.6" + + // create two additional worker pools which explicitly specify the kubernetes version + pool1 := f.Shoot.Spec.Provider.Workers[0] + pool2, pool3 := pool1.DeepCopy(), pool1.DeepCopy() + pool2.Name += "2" + pool2.Kubernetes = &gardencorev1beta1.WorkerKubernetes{Version: &f.Shoot.Spec.Kubernetes.Version} + pool3.Name += "3" + pool3.Kubernetes = &gardencorev1beta1.WorkerKubernetes{Version: pointer.String("1.22.0")} + f.Shoot.Spec.Provider.Workers = append(f.Shoot.Spec.Provider.Workers, *pool2, *pool3) + + It("Create, Update, Delete", Label("simple"), func() { By("Create Shoot") ctx, cancel := context.WithTimeout(parentCtx, 15*time.Minute) defer cancel() @@ -45,6 +62,12 @@ var _ = Describe("Shoot Tests", Label("Shoot"), func() { g.Expect(shootClient.Client().List(ctx, &corev1.NamespaceList{})).To(Succeed()) }).Should(Succeed()) + By("Update Shoot") + shootupdatesuite.RunTest(ctx, &framework.ShootFramework{ + GardenerFramework: f.GardenerFramework, + Shoot: f.Shoot, + }, nil, nil) + By("Delete Shoot") ctx, cancel = context.WithTimeout(parentCtx, 15*time.Minute) defer cancel() diff --git a/test/e2e/shoot/internal/rotation/secret_encryption.go b/test/e2e/shoot/internal/rotation/secret_encryption.go index c7dd6fe0717..7d6ad787575 100644 --- a/test/e2e/shoot/internal/rotation/secret_encryption.go +++ b/test/e2e/shoot/internal/rotation/secret_encryption.go @@ -23,8 +23,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/gardener/gardener/pkg/client/kubernetes" - "github.com/gardener/gardener/test/e2e/shoot/internal/access" "github.com/gardener/gardener/test/framework" + "github.com/gardener/gardener/test/utils/shoots/access" ) // SecretEncryptionVerifier creates and reads secrets in the shoot to verify correct configuration of etcd encryption. diff --git a/test/e2e/shoot/internal/rotation/shoot_access.go b/test/e2e/shoot/internal/rotation/shoot_access.go index bf7cbd8cd4e..00aabbc1328 100644 --- a/test/e2e/shoot/internal/rotation/shoot_access.go +++ b/test/e2e/shoot/internal/rotation/shoot_access.go @@ -22,8 +22,8 @@ import ( corev1 "k8s.io/api/core/v1" "github.com/gardener/gardener/pkg/client/kubernetes" - "github.com/gardener/gardener/test/e2e/shoot/internal/access" "github.com/gardener/gardener/test/framework" + "github.com/gardener/gardener/test/utils/shoots/access" ) type clients struct { diff --git a/test/testmachinery/system/shoot_update/update_test.go b/test/testmachinery/system/shoot_update/update_test.go index ce860429971..bf7c1ab5787 100644 --- a/test/testmachinery/system/shoot_update/update_test.go +++ b/test/testmachinery/system/shoot_update/update_test.go @@ -29,18 +29,18 @@ package shootupdate import ( "context" "flag" - "fmt" "time" - gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" - gardencorev1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" "github.com/gardener/gardener/test/framework" + shootupdatesuite "github.com/gardener/gardener/test/utils/shoots/update" . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" ) -var kubernetesVersion = flag.String("version", "", "the version to update the shoot") +var ( + newControlPlaneKubernetesVersion = flag.String("version", "", "the version to use for .spec.kubernetes.version and .spec.provider.workers[].kubernetes.version (only when nil or equal to .spec.kubernetes.version)") + newWorkerPoolKubernetesVersion = flag.String("version-worker-pools", "", "the version to use for .spec.provider.workers[].kubernetes.version (only when not equal to .spec.kubernetes.version)") +) const UpdateKubernetesVersionTimeout = 45 * time.Minute @@ -49,38 +49,9 @@ func init() { } var _ = Describe("Shoot update testing", func() { - f := framework.NewShootFramework(nil) - framework.CIt("should update the kubernetes version of the shoot to the next version", func(ctx context.Context) { - currentVersion := f.Shoot.Spec.Kubernetes.Version - newVersion := *kubernetesVersion - if currentVersion == newVersion { - Skip("shoot already has the desired kubernetes version") - } - if newVersion == "" { - var ( - err error - ok bool - consecutiveMinorAvailable bool - ) - cloudprofile, err := f.GetCloudProfile(ctx) - Expect(err).ToNot(HaveOccurred()) - consecutiveMinorAvailable, newVersion, err = gardencorev1beta1helper.GetKubernetesVersionForMinorUpdate(cloudprofile, currentVersion) - Expect(err).ToNot(HaveOccurred()) - Expect(consecutiveMinorAvailable).To(BeTrue()) - if !ok { - Skip("no new version found") - } - } - - By(fmt.Sprintf("updating shoot %s to version %s", f.Shoot.GetName(), newVersion)) - err := f.UpdateShoot(ctx, func(shoot *gardencorev1beta1.Shoot) error { - shoot.Spec.Kubernetes.Version = newVersion - return nil - }) - Expect(err).ToNot(HaveOccurred()) - + framework.CIt("should update the kubernetes version of the shoot and its worker pools to the respective next versions", func(ctx context.Context) { + shootupdatesuite.RunTest(ctx, f, newControlPlaneKubernetesVersion, newWorkerPoolKubernetesVersion) }, UpdateKubernetesVersionTimeout) - }) diff --git a/test/e2e/shoot/internal/access/adminkubeconfig.go b/test/utils/shoots/access/adminkubeconfig.go similarity index 100% rename from test/e2e/shoot/internal/access/adminkubeconfig.go rename to test/utils/shoots/access/adminkubeconfig.go index 8576765daa7..655ad494193 100644 --- a/test/e2e/shoot/internal/access/adminkubeconfig.go +++ b/test/utils/shoots/access/adminkubeconfig.go @@ -17,13 +17,13 @@ package access import ( "context" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + authenticationv1alpha1 "github.com/gardener/gardener/pkg/apis/authentication/v1alpha1" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" gardenversionedcoreclientset "github.com/gardener/gardener/pkg/client/core/clientset/versioned" "github.com/gardener/gardener/pkg/client/kubernetes" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" ) // CreateShootClientFromAdminKubeconfig requests an admin kubeconfig and creates a shoot client. diff --git a/test/e2e/shoot/internal/access/csr.go b/test/utils/shoots/access/csr.go similarity index 100% rename from test/e2e/shoot/internal/access/csr.go rename to test/utils/shoots/access/csr.go diff --git a/test/e2e/shoot/internal/access/serviceaccount.go b/test/utils/shoots/access/serviceaccount.go similarity index 100% rename from test/e2e/shoot/internal/access/serviceaccount.go rename to test/utils/shoots/access/serviceaccount.go diff --git a/test/e2e/shoot/internal/access/statictokenkubeconfig.go b/test/utils/shoots/access/statictokenkubeconfig.go similarity index 100% rename from test/e2e/shoot/internal/access/statictokenkubeconfig.go rename to test/utils/shoots/access/statictokenkubeconfig.go diff --git a/test/utils/shoots/update/update.go b/test/utils/shoots/update/update.go new file mode 100644 index 00000000000..49bdfc02cb6 --- /dev/null +++ b/test/utils/shoots/update/update.go @@ -0,0 +1,205 @@ +// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package update + +import ( + "context" + "fmt" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + gardencorev1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" + "github.com/gardener/gardener/pkg/client/kubernetes" + "github.com/gardener/gardener/test/framework" + "github.com/gardener/gardener/test/utils/shoots/access" + + "github.com/Masterminds/semver" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" +) + +// RunTest runs the update test for an existing shoot cluster. If provided, it updates .spec.kubernetes.version with the +// value of and the .kubernetes.version fields of all worker pools which currently +// have the same value as .spec.kubernetes.version. For all worker pools specifying a different version, +// will be used. +// If or are nil or empty then the next consecutive +// minor versions will be fetched from the CloudProfile referenced by the shoot. +func RunTest( + ctx context.Context, + f *framework.ShootFramework, + newControlPlaneKubernetesVersion *string, + newWorkerPoolKubernetesVersion *string, +) { + By("creating shoot client") + shootClient, err := access.CreateShootClientFromAdminKubeconfig(ctx, f.GardenClient, f.Shoot) + Expect(err).NotTo(HaveOccurred()) + + By("verifying the Kubernetes version for all existing nodes matches with the versions defined in the Shoot spec [before update]") + Expect(verifyKubernetesVersions(ctx, shootClient, f.Shoot)).To(Succeed()) + + By("reading CloudProfile") + cloudProfile, err := f.GetCloudProfile(ctx) + Expect(err).NotTo(HaveOccurred()) + + By("computing new Kubernetes version for control plane and worker pools") + controlPlaneVersion, poolNameToKubernetesVersion, err := computeNewKubernetesVersions(cloudProfile, f.Shoot, newControlPlaneKubernetesVersion, newWorkerPoolKubernetesVersion) + Expect(err).NotTo(HaveOccurred()) + + if len(controlPlaneVersion) == 0 && len(poolNameToKubernetesVersion) == 0 { + Skip("shoot already has the desired kubernetes versions") + } + + By("updating shoot") + if controlPlaneVersion != "" { + By("updating .spec.kubernetes.version to " + controlPlaneVersion) + } + for poolName, kubernetesVersion := range poolNameToKubernetesVersion { + By("updating .kubernetes.version to " + kubernetesVersion + " for pool " + poolName) + } + + Expect(f.UpdateShoot(ctx, func(shoot *gardencorev1beta1.Shoot) error { + if controlPlaneVersion != "" { + shoot.Spec.Kubernetes.Version = controlPlaneVersion + } + + for i, worker := range shoot.Spec.Provider.Workers { + if workerPoolVersion, ok := poolNameToKubernetesVersion[worker.Name]; ok { + shoot.Spec.Provider.Workers[i].Kubernetes.Version = &workerPoolVersion + } + } + + return nil + })).To(Succeed()) + + By("re-creating shoot client") + shootClient, err = access.CreateShootClientFromAdminKubeconfig(ctx, f.GardenClient, f.Shoot) + Expect(err).NotTo(HaveOccurred()) + + By("verifying the Kubernetes version for all existing nodes matches with the versions defined in the Shoot spec [after update]") + Expect(verifyKubernetesVersions(ctx, shootClient, f.Shoot)).To(Succeed()) +} + +func verifyKubernetesVersions(ctx context.Context, shootClient kubernetes.Interface, shoot *gardencorev1beta1.Shoot) error { + controlPlaneKubernetesVersion, err := semver.NewVersion(shoot.Spec.Kubernetes.Version) + if err != nil { + return err + } + + expectedControlPlaneKubernetesVersion := "v" + controlPlaneKubernetesVersion.String() + if shootClient.Version() != expectedControlPlaneKubernetesVersion { + return fmt.Errorf("control plane version is %q but expected %q", shootClient.Version(), expectedControlPlaneKubernetesVersion) + } + + poolNameToKubernetesVersion := map[string]string{} + for _, worker := range shoot.Spec.Provider.Workers { + poolKubernetesVersion, err := gardencorev1beta1helper.CalculateEffectiveKubernetesVersion(controlPlaneKubernetesVersion, worker.Kubernetes) + if err != nil { + return fmt.Errorf("error when calculating effective Kubernetes version of pool %q: %w", worker.Name, err) + } + poolNameToKubernetesVersion[worker.Name] = "v" + poolKubernetesVersion.String() + } + + nodeList := &corev1.NodeList{} + if err := shootClient.Client().List(ctx, nodeList); err != nil { + return err + } + + for _, node := range nodeList.Items { + var ( + poolName = node.Labels[v1beta1constants.LabelWorkerPool] + kubernetesVersion = poolNameToKubernetesVersion[poolName] + ) + + if kubeletVersion := node.Status.NodeInfo.KubeletVersion; kubeletVersion != kubernetesVersion { + return fmt.Errorf("kubelet version of pool %q is %q but expected %q", poolName, kubeletVersion, kubernetesVersion) + } + + if kubeProxyVersion := node.Status.NodeInfo.KubeProxyVersion; kubeProxyVersion != kubernetesVersion { + return fmt.Errorf("kube-proxy version of pool %q is %q but expected %q", poolName, kubeProxyVersion, kubernetesVersion) + } + } + + return nil +} + +func computeNewKubernetesVersions( + cloudProfile *gardencorev1beta1.CloudProfile, + shoot *gardencorev1beta1.Shoot, + newControlPlaneKubernetesVersion *string, + newWorkerPoolKubernetesVersion *string, +) ( + controlPlaneKubernetesVersion string, + poolNameToKubernetesVersion map[string]string, + err error, +) { + if newControlPlaneKubernetesVersion != nil && *newControlPlaneKubernetesVersion != "" { + controlPlaneKubernetesVersion = *newControlPlaneKubernetesVersion + } else { + controlPlaneKubernetesVersion, err = getNextConsecutiveMinorVersion(cloudProfile, shoot.Spec.Kubernetes.Version) + if err != nil { + return "", nil, err + } + } + + // if current version is already the same as the new version then reset it + if shoot.Spec.Kubernetes.Version == controlPlaneKubernetesVersion { + controlPlaneKubernetesVersion = "" + } + + poolNameToKubernetesVersion = make(map[string]string, len(shoot.Spec.Provider.Workers)) + for _, worker := range shoot.Spec.Provider.Workers { + // worker does not override version + if worker.Kubernetes == nil || worker.Kubernetes.Version == nil { + continue + } + + if *worker.Kubernetes.Version == shoot.Spec.Kubernetes.Version { + // worker overrides version and it's equal to the control plane version + poolNameToKubernetesVersion[worker.Name] = controlPlaneKubernetesVersion + continue + } + + // worker overrides version and it's not equal to the control plane version + if newWorkerPoolKubernetesVersion != nil && *newWorkerPoolKubernetesVersion != "" { + poolNameToKubernetesVersion[worker.Name] = *newWorkerPoolKubernetesVersion + } else { + poolNameToKubernetesVersion[worker.Name], err = getNextConsecutiveMinorVersion(cloudProfile, *worker.Kubernetes.Version) + if err != nil { + return "", nil, err + } + } + + // if current version is already the same as the new version then reset it + if *worker.Kubernetes.Version == poolNameToKubernetesVersion[worker.Name] { + delete(poolNameToKubernetesVersion, worker.Name) + } + } + + return +} + +func getNextConsecutiveMinorVersion(cloudProfile *gardencorev1beta1.CloudProfile, kubernetesVersion string) (string, error) { + consecutiveMinorAvailable, newVersion, err := gardencorev1beta1helper.GetKubernetesVersionForMinorUpdate(cloudProfile, kubernetesVersion) + if err != nil { + return "", err + } + + if !consecutiveMinorAvailable { + return "", fmt.Errorf("no consecutive minor version available for %q", kubernetesVersion) + } + + return newVersion, nil +}