From d0b2a83f6ffd28327717b71d899b896a6e1e168d Mon Sep 17 00:00:00 2001 From: Dimitar Mirchev Date: Wed, 15 Nov 2023 14:18:56 +0200 Subject: [PATCH] Recreate managed resource secrets that have deletion timestamp (#8812) * Recreate deleted MR secrets * Fix label copying * Add test * Format * Change remove version, tmp name, nil finalizers * Do not recreate secrets without del timestamp * Remove some obsolete code * Run tasks in parallel --- cmd/gardenlet/app/app.go | 170 +++++++++++++++++++++ cmd/gardenlet/app/app_test.go | 269 ++++++++++++++++++++++++++++++++++ 2 files changed, 439 insertions(+) create mode 100644 cmd/gardenlet/app/app_test.go diff --git a/cmd/gardenlet/app/app.go b/cmd/gardenlet/app/app.go index 521b37985af..16ef29efd1f 100644 --- a/cmd/gardenlet/app/app.go +++ b/cmd/gardenlet/app/app.go @@ -21,12 +21,14 @@ import ( "net/http" "os" goruntime "runtime" + "slices" "strconv" "strings" "time" "github.com/go-logr/logr" "github.com/spf13/cobra" + "golang.org/x/time/rate" certificatesv1 "k8s.io/api/certificates/v1" coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" @@ -35,6 +37,8 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/rest" @@ -69,6 +73,7 @@ import ( "github.com/gardener/gardener/pkg/gardenlet/bootstrap/certificate" "github.com/gardener/gardener/pkg/gardenlet/controller" gardenerhealthz "github.com/gardener/gardener/pkg/healthz" + "github.com/gardener/gardener/pkg/resourcemanager/controller/garbagecollector/references" "github.com/gardener/gardener/pkg/utils" "github.com/gardener/gardener/pkg/utils/flow" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" @@ -382,6 +387,11 @@ func (g *garden) Start(ctx context.Context) error { return err } + log.Info("Recreating wrongly deleted managed resource secrets") + if err := recreateDeletedManagedResourceSecrets(ctx, g.mgr.GetClient()); err != nil { + return err + } + log.Info("Setting up shoot client map") shootClientMap, err := clientmapbuilder. NewShootClientMapBuilder(). @@ -480,6 +490,166 @@ func (g *garden) cleanupOrphanedExtensionsServiceAccounts(ctx context.Context, g return flow.Parallel(taskFns...)(ctx) } +const ( + grmFinalizer = "resources.gardener.cloud/gardener-resource-manager" + tempSecretLabel = "resources.gardener.cloud/temp-secret" + tempSecretOldNameLabel = "resources.gardener.cloud/temp-secret-old-name" +) + +// TODO(dimityrmirchev): Remove this code after v1.87 has been released. +func recreateDeletedManagedResourceSecrets(ctx context.Context, c client.Client) error { + // check for already existing temp secrets + // these can occur in case the process is killed during cleanup phase + tempSecretList := &corev1.SecretList{} + if err := c.List(ctx, tempSecretList, client.MatchingLabels{tempSecretLabel: "true"}); err != nil { + return err + } + + var ( + tasks []flow.TaskFn + limiter = rate.NewLimiter(rate.Limit(20), 20) + ) + for _, temp := range tempSecretList.Items { + temp := temp + tasks = append(tasks, func(ctx context.Context) error { + originalName := temp.Labels[tempSecretOldNameLabel] + original := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: originalName, Namespace: temp.Namespace}} + + if err := limiter.Wait(ctx); err != nil { + return err + } + + if err := c.Get(ctx, client.ObjectKeyFromObject(original), original); err != nil { + if apierrors.IsNotFound(err) { + // original secret is not found so we recreate it + original := temp.DeepCopy() + delete(original.Labels, tempSecretLabel) + delete(original.Labels, tempSecretOldNameLabel) + original.ResourceVersion = "" + original.Name = originalName + + if err := c.Create(ctx, original); err != nil { + return fmt.Errorf("failed to recreate the original secret %w", err) + } + if err := c.Delete(ctx, &temp); client.IgnoreNotFound(err) != nil { + return err + } + return nil + } + + return err + } + + // the original secret exists. check if the finalizer and deletion timestamp are there + if original.DeletionTimestamp != nil && slices.Contains(original.Finalizers, grmFinalizer) { + if err := removeFinalizersAndWait(ctx, c, original.DeepCopy()); err != nil { + return err + } + + // zero meta info + original.DeletionTimestamp = nil + original.ResourceVersion = "" + original.Finalizers = nil + + if err := c.Create(ctx, original); err != nil { + return fmt.Errorf("failed to recreate the original secret %w", err) + } + } + + // secret was already recreated. just delete the temporary one + if err := c.Delete(ctx, &temp); client.IgnoreNotFound(err) != nil { + return err + } + return nil + }) + } + + if err := flow.Parallel(tasks...)(ctx); err != nil { + return err + } + + secretsToRecreate, err := getSecretsToRecreate(ctx, c) + if err != nil { + return fmt.Errorf("failed listing secrets for recreation %w", err) + } + + tasks = []flow.TaskFn{} + for _, original := range secretsToRecreate { + original := original + tasks = append(tasks, func(ctx context.Context) error { + tempSecret := original.DeepCopy() + tempSecret.Name = "tmp-" + original.Name + metav1.SetMetaDataLabel(&tempSecret.ObjectMeta, tempSecretLabel, "true") + metav1.SetMetaDataLabel(&tempSecret.ObjectMeta, tempSecretOldNameLabel, original.Name) + tempSecret.DeletionTimestamp = nil + tempSecret.ResourceVersion = "" + tempSecret.Finalizers = nil + + if err := limiter.Wait(ctx); err != nil { + return err + } + + if err := c.Create(ctx, tempSecret); err != nil { + return fmt.Errorf("failed to create a temporary secret %w", err) + } + + if err := removeFinalizersAndWait(ctx, c, original.DeepCopy()); err != nil { + return err + } + + // zero meta info + original.DeletionTimestamp = nil + original.ResourceVersion = "" + original.Finalizers = nil + + // recreate the original and delete the temporary one + if err := c.Create(ctx, &original); err != nil { + return fmt.Errorf("failed to recreate the original secret %w", err) + } + if err := c.Delete(ctx, tempSecret); client.IgnoreNotFound(err) != nil { + return err + } + return nil + }) + } + return flow.Parallel(tasks...)(ctx) +} + +// TODO(dimityrmirchev): Remove this code after v1.87 has been released. +func removeFinalizersAndWait(ctx context.Context, c client.Client, secret *corev1.Secret) error { + patch := client.StrategicMergeFrom(secret.DeepCopy()) + secret.Finalizers = []string{} + if err := c.Patch(ctx, secret, patch); err != nil { + return fmt.Errorf("failed to patch the original secret %w", err) + } + + cancelCtx, cancel := context.WithTimeout(ctx, time.Second*10) + defer cancel() + return kubernetesutils.WaitUntilResourceDeleted(cancelCtx, c, secret, 1*time.Second) +} + +// TODO(dimityrmirchev): Remove this code after v1.87 has been released. +func getSecretsToRecreate(ctx context.Context, c client.Client) ([]corev1.Secret, error) { + selector := labels.NewSelector() + isGC, err := labels.NewRequirement(references.LabelKeyGarbageCollectable, selection.Equals, []string{"true"}) + if err != nil { + return nil, err + } + notTemp, err := labels.NewRequirement(tempSecretLabel, selection.DoesNotExist, nil) + if err != nil { + return nil, err + } + selector.Add(*isGC, *notTemp) + secretList := &corev1.SecretList{} + if err := c.List(ctx, secretList, client.MatchingLabelsSelector{Selector: selector}); err != nil { + return nil, err + } + secretsToRecreate := slices.DeleteFunc(secretList.Items, func(s corev1.Secret) bool { + return !slices.Contains(s.Finalizers, grmFinalizer) || s.DeletionTimestamp == nil + }) + return secretsToRecreate, nil +} + func (g *garden) registerSeed(ctx context.Context, gardenClient client.Client) error { seed := &gardencorev1beta1.Seed{ ObjectMeta: metav1.ObjectMeta{ diff --git a/cmd/gardenlet/app/app_test.go b/cmd/gardenlet/app/app_test.go new file mode 100644 index 00000000000..90e8805322f --- /dev/null +++ b/cmd/gardenlet/app/app_test.go @@ -0,0 +1,269 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package app + +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/gardener/gardener/pkg/client/kubernetes" +) + +func TestApp(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Gardenlet App Test Suite") +} + +var _ = Describe("Recreate Managed Resource Secrets", func() { + var ( + ctx = context.TODO() + fakeClient client.Client + + secret1 *corev1.Secret + secret2 *corev1.Secret + expectedSecret1 *corev1.Secret + expectedSecret2 *corev1.Secret + tempSecret3 *corev1.Secret + expectedSecret3 *corev1.Secret + secret4 *corev1.Secret + tempSecret4 *corev1.Secret + expectedSecret4 *corev1.Secret + + secret5 *corev1.Secret + ) + + BeforeEach(func() { + fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).Build() + secret1 = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret1", + Namespace: "shoot-ns", + Labels: map[string]string{ + "resources.gardener.cloud/garbage-collectable-reference": "true", + }, + Finalizers: []string{"resources.gardener.cloud/gardener-resource-manager"}, + }, + Immutable: pointer.Bool(true), + Data: map[string][]byte{ + "test": []byte("foo"), + }, + } + secret2 = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret2", + Namespace: "garden", + Labels: map[string]string{ + "resources.gardener.cloud/garbage-collectable-reference": "true", + }, + Finalizers: []string{"resources.gardener.cloud/gardener-resource-manager"}, + }, + Immutable: pointer.Bool(true), + Data: map[string][]byte{ + "test": []byte("bar"), + }, + } + + tempSecret3 = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret3-temp", + Namespace: "garden", + Labels: map[string]string{ + "resources.gardener.cloud/garbage-collectable-reference": "true", + "resources.gardener.cloud/temp-secret": "true", + "resources.gardener.cloud/temp-secret-old-name": "secret3", + }, + }, + Immutable: pointer.Bool(true), + Data: map[string][]byte{ + "test": []byte("bar1"), + }, + } + + secret4 = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret4", + Namespace: "garden", + Labels: map[string]string{ + "resources.gardener.cloud/garbage-collectable-reference": "true", + }, + Finalizers: []string{"resources.gardener.cloud/gardener-resource-manager"}, + }, + Immutable: pointer.Bool(true), + Data: map[string][]byte{ + "test": []byte("bar1"), + }, + } + + tempSecret4 = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret4-temp", + Namespace: "garden", + Labels: map[string]string{ + "resources.gardener.cloud/garbage-collectable-reference": "true", + "resources.gardener.cloud/temp-secret": "true", + "resources.gardener.cloud/temp-secret-old-name": "secret4", + }, + }, + Immutable: pointer.Bool(true), + Data: map[string][]byte{ + "test": []byte("bar1"), + }, + } + + expectedSecret1 = &corev1.Secret{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "secret1", + Namespace: "shoot-ns", + Labels: map[string]string{ + "resources.gardener.cloud/garbage-collectable-reference": "true", + }, + ResourceVersion: "1", + }, + Immutable: pointer.Bool(true), + Data: map[string][]byte{ + "test": []byte("foo"), + }, + } + + expectedSecret2 = &corev1.Secret{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "secret2", + Namespace: "garden", + Labels: map[string]string{ + "resources.gardener.cloud/garbage-collectable-reference": "true", + }, + ResourceVersion: "1", + }, + Immutable: pointer.Bool(true), + Data: map[string][]byte{ + "test": []byte("bar"), + }, + } + + expectedSecret3 = &corev1.Secret{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "secret3", + Namespace: "garden", + Labels: map[string]string{ + "resources.gardener.cloud/garbage-collectable-reference": "true", + }, + ResourceVersion: "1", + }, + Immutable: pointer.Bool(true), + Data: map[string][]byte{ + "test": []byte("bar1"), + }, + } + + expectedSecret4 = &corev1.Secret{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "secret4", + Namespace: "garden", + Labels: map[string]string{ + "resources.gardener.cloud/garbage-collectable-reference": "true", + }, + ResourceVersion: "1", + }, + Immutable: pointer.Bool(true), + Data: map[string][]byte{ + "test": []byte("bar1"), + }, + } + + secret5 = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret5", + Namespace: "garden", + Labels: map[string]string{ + "resources.gardener.cloud/garbage-collectable-reference": "true", + }, + Finalizers: []string{"resources.gardener.cloud/gardener-resource-manager"}, + }, + Immutable: pointer.Bool(true), + Data: map[string][]byte{ + "test": []byte("foo"), + }, + } + }) + + It("should recreate the managed resource secrets", func() { + Expect(fakeClient.Create(ctx, secret1)).To(Succeed()) + Expect(fakeClient.Create(ctx, secret2)).To(Succeed()) + Expect(fakeClient.Create(ctx, tempSecret3)).To(Succeed()) + Expect(fakeClient.Create(ctx, secret4)).To(Succeed()) + Expect(fakeClient.Create(ctx, tempSecret4)).To(Succeed()) + + Expect(fakeClient.Delete(ctx, secret1)).To(Succeed()) + Expect(fakeClient.Delete(ctx, secret2)).To(Succeed()) + Expect(fakeClient.Delete(ctx, secret4)).To(Succeed()) + + s1 := &corev1.Secret{} + s2 := &corev1.Secret{} + s3 := &corev1.Secret{} + s4 := &corev1.Secret{} + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(secret1), s1)).To(Succeed()) + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(secret2), s2)).To(Succeed()) + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(secret2), s4)).To(Succeed()) + + Expect(s1.DeletionTimestamp).ToNot(BeNil()) + Expect(s2.DeletionTimestamp).ToNot(BeNil()) + Expect(s4.DeletionTimestamp).ToNot(BeNil()) + + Expect(recreateDeletedManagedResourceSecrets(ctx, fakeClient)).To(Succeed()) + + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(expectedSecret1), s1)).To(Succeed()) + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(expectedSecret2), s2)).To(Succeed()) + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(expectedSecret3), s3)).To(Succeed()) + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(expectedSecret4), s4)).To(Succeed()) + + Expect(s1).To(Equal(expectedSecret1)) + Expect(s2).To(Equal(expectedSecret2)) + Expect(s3).To(Equal(expectedSecret3)) + Expect(s4).To(Equal(expectedSecret4)) + + secretList := &corev1.SecretList{} + Expect(fakeClient.List(ctx, secretList)).To(Succeed()) + Expect(secretList.Items).To(HaveLen(4)) + }) + + It("should not recreate the managed resource secret", func() { + Expect(fakeClient.Create(ctx, secret5)).To(Succeed()) + + expected := &corev1.Secret{} + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(secret5), expected)).To(Succeed()) + + Expect(recreateDeletedManagedResourceSecrets(ctx, fakeClient)).To(Succeed()) + + got := &corev1.Secret{} + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(secret5), got)).To(Succeed()) + Expect(expected).To(Equal(got)) + + secretList := &corev1.SecretList{} + Expect(fakeClient.List(ctx, secretList)).To(Succeed()) + Expect(secretList.Items).To(HaveLen(1)) + }) +})