Revert cleanup of legacy ETCD encryption and ServiceAccount token s…

…igning key secret (gardener#6929)

* Revert "Cleanup legacy etcd-encryption-secret secret deletion"

This reverts commit 127819d part of gardener#6891

* Revert "Cleanup legacy `service-account-key` secret deletion"

This reverts commit fb9c4a3 part of gardener#6891.

pkg/operation/botanist/component/kubeapiserver/secrets.go

@@ -105,7 +105,8 @@ func (k *kubeAPIServer) reconcileSecretServiceAccountKey(ctx context.Context) (*
 		return nil, err
 	}
 
-	return secret, nil
+	// TODO(rfranzke): Remove this in a future release.
+	return secret, kutil.DeleteObject(ctx, k.client.Client(), &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "service-account-key", Namespace: k.namespace}})
 }
 
 func (k *kubeAPIServer) reconcileSecretBasicAuth(ctx context.Context) (*corev1.Secret, error) {

pkg/operation/botanist/kubeapiserver.go

@@ -19,6 +19,8 @@ import (
 	"fmt"
 	"net"
 
+	gardencorev1alpha1 "github.com/gardener/gardener/pkg/apis/core/v1alpha1"
+	gardencorev1alpha1helper "github.com/gardener/gardener/pkg/apis/core/v1alpha1/helper"
 	gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
 	v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
 	gardencorev1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper"
@@ -676,7 +678,20 @@ func (b *Botanist) DeployKubeAPIServer(ctx context.Context) error {
 		}
 	}
 
-	return nil
+	// TODO(rfranzke): Remove in a future release.
+	if err := b.SaveGardenerResourceDataInShootState(ctx, func(gardenerResourceData *[]gardencorev1alpha1.GardenerResourceData) error {
+		gardenerResourceDataList := gardencorev1alpha1helper.GardenerResourceDataList(*gardenerResourceData)
+		gardenerResourceDataList.Delete("etcdEncryptionConfiguration")
+		gardenerResourceDataList.Delete("service-account-key")
+		*gardenerResourceData = gardenerResourceDataList
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	return kutil.DeleteObjects(ctx, b.SeedClientSet.Client(),
+		&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: b.Shoot.SeedNamespace, Name: "etcd-encryption-secret"}},
+	)
 }
 
 // DeleteKubeAPIServer deletes the kube-apiserver deployment in the Seed cluster which holds the Shoot's control plane.

pkg/operation/botanist/kubeapiserver_test.go

@@ -1995,6 +1995,25 @@ usernames: ["admin"]
 			))
 		})
 
+		It("should delete the old etcd encryption config secret", func() {
+			kubeAPIServer.EXPECT().GetValues()
+			kubeAPIServer.EXPECT().SetAutoscalingReplicas(gomock.Any())
+			kubeAPIServer.EXPECT().SetSNIConfig(gomock.Any())
+			kubeAPIServer.EXPECT().SetETCDEncryptionConfig(gomock.Any())
+			kubeAPIServer.EXPECT().SetExternalHostname(gomock.Any())
+			kubeAPIServer.EXPECT().SetExternalServer(gomock.Any())
+			kubeAPIServer.EXPECT().SetServerCertificateConfig(gomock.Any())
+			kubeAPIServer.EXPECT().SetServiceAccountConfig(gomock.Any())
+			kubeAPIServer.EXPECT().Deploy(ctx)
+
+			secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: seedNamespace, Name: "etcd-encryption-secret"}}
+			Expect(seedClient.Create(ctx, secret)).To(Succeed())
+
+			Expect(botanist.DeployKubeAPIServer(ctx)).To(Succeed())
+
+			Expect(seedClient.Get(ctx, client.ObjectKeyFromObject(secret), &corev1.Secret{})).To(BeNotFoundError())
+		})
+
 		It("should not sync the kubeconfig to garden project namespace when enableStaticTokenKubeconfig is set to false", func() {
 			secret := &corev1.Secret{
 				ObjectMeta: metav1.ObjectMeta{

pkg/utils/secrets/manager/generate.go

@@ -28,6 +28,11 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/runtime/serializer/json"
+	apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
 	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -110,7 +115,15 @@ func (m *manager) generateAndCreate(ctx context.Context, config secretutils.Conf
 		return nil, err
 	}
 
-	secret := Secret(objectMeta, data.SecretData())
+	// For backwards-compatibility, we need to keep some of the existing secrets (cluster-admin token, basic auth
+	// password, etc.).
+	// TODO(rfranzke): Remove this code in the future
+	dataMap, err := m.keepExistingSecretsIfNeeded(ctx, config.GetName(), data.SecretData())
+	if err != nil {
+		return nil, err
+	}
+
+	secret := Secret(objectMeta, dataMap)
 	if err := m.client.Create(ctx, secret); err != nil {
 		if !apierrors.IsAlreadyExists(err) {
 			return nil, err
@@ -125,6 +138,67 @@ func (m *manager) generateAndCreate(ctx context.Context, config secretutils.Conf
 	return secret, nil
 }
 
+func (m *manager) keepExistingSecretsIfNeeded(ctx context.Context, configName string, newData map[string][]byte) (map[string][]byte, error) {
+	existingSecret := &corev1.Secret{}
+
+	switch configName {
+	case "kube-apiserver-etcd-encryption-key":
+		if err := m.client.Get(ctx, kutil.Key(m.namespace, "etcd-encryption-secret"), existingSecret); err != nil {
+			if !apierrors.IsNotFound(err) {
+				return nil, err
+			}
+			return newData, nil
+		}
+
+		scheme := runtime.NewScheme()
+		if err := apiserverconfigv1.AddToScheme(scheme); err != nil {
+			return nil, err
+		}
+
+		ser := json.NewSerializerWithOptions(json.DefaultMetaFactory, scheme, scheme, json.SerializerOptions{Yaml: true, Pretty: false, Strict: false})
+		versions := schema.GroupVersions([]schema.GroupVersion{apiserverconfigv1.SchemeGroupVersion})
+		codec := serializer.NewCodecFactory(scheme).CodecForVersions(ser, ser, versions, versions)
+
+		encryptionConfiguration := &apiserverconfigv1.EncryptionConfiguration{}
+		if _, _, err := codec.Decode(existingSecret.Data["encryption-configuration.yaml"], nil, encryptionConfiguration); err != nil {
+			return nil, err
+		}
+
+		var existingEncryptionKey, existingEncryptionSecret []byte
+
+		if len(encryptionConfiguration.Resources) != 0 {
+			for _, provider := range encryptionConfiguration.Resources[0].Providers {
+				if provider.AESCBC != nil && len(provider.AESCBC.Keys) != 0 {
+					existingEncryptionKey = []byte(provider.AESCBC.Keys[0].Name)
+					existingEncryptionSecret = []byte(provider.AESCBC.Keys[0].Secret)
+					break
+				}
+			}
+		}
+
+		if existingEncryptionKey == nil || existingEncryptionSecret == nil {
+			return nil, fmt.Errorf("old etcd encryption key or secret was not found")
+		}
+
+		return map[string][]byte{
+			secretutils.DataKeyEncryptionKeyName: existingEncryptionKey,
+			secretutils.DataKeyEncryptionSecret:  existingEncryptionSecret,
+		}, nil
+
+	case "service-account-key":
+		if err := m.client.Get(ctx, kutil.Key(m.namespace, "service-account-key"), existingSecret); err != nil {
+			if !apierrors.IsNotFound(err) {
+				return nil, err
+			}
+			return newData, nil
+		}
+
+		return existingSecret.Data, nil
+	}
+
+	return newData, nil
+}
+
 func (m *manager) shouldIgnoreOldSecrets(issuedAt string, options *GenerateOptions) (bool, error) {
 	// unconditionally ignore old secrets
 	if options.RotationStrategy != KeepOld || options.IgnoreOldSecrets {

pkg/utils/secrets/manager/generate_test.go

@@ -765,6 +765,110 @@ var _ = Describe("Generate", func() {
 				Expect(secretInfos.bundle).To(BeNil())
 			})
 		})
+
+		Context("backwards compatibility", func() {
+			Context("etcd encryption key", func() {
+				var (
+					oldKey    = []byte("old-key")
+					oldSecret = []byte("old-secret")
+					config    *secretutils.ETCDEncryptionKeySecretConfig
+				)
+
+				BeforeEach(func() {
+					config = &secretutils.ETCDEncryptionKeySecretConfig{
+						Name:         "kube-apiserver-etcd-encryption-key",
+						SecretLength: 32,
+					}
+				})
+
+				It("should generate a new encryption key secret if old secret does not exist", func() {
+					By("generating secret")
+					secret, err := m.Generate(ctx, config)
+					Expect(err).NotTo(HaveOccurred())
+
+					By("verifying new key and secret were generated")
+					Expect(secret.Data["key"]).NotTo(Equal(oldKey))
+					Expect(secret.Data["secret"]).NotTo(Equal(oldSecret))
+				})
+
+				It("should keep the existing encryption key and secret if old secret still exists", func() {
+					oldEncryptionConfiguration := `apiVersion: apiserver.config.k8s.io/v1
+kind: EncryptionConfiguration
+resources:
+- providers:
+  - aescbc:
+      keys:
+      - name: ` + string(oldKey) + `
+        secret: ` + string(oldSecret) + `
+  - identity: {}
+  resources:
+  - secrets
+`
+
+					By("creating existing secret with old encryption configuration")
+					existingSecret := &corev1.Secret{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      "etcd-encryption-secret",
+							Namespace: namespace,
+						},
+						Type: corev1.SecretTypeOpaque,
+						Data: map[string][]byte{"encryption-configuration.yaml": []byte(oldEncryptionConfiguration)},
+					}
+					Expect(fakeClient.Create(ctx, existingSecret)).To(Succeed())
+
+					By("generating secret")
+					secret, err := m.Generate(ctx, config)
+					Expect(err).NotTo(HaveOccurred())
+
+					By("verifying old key and secret were kept")
+					Expect(secret.Data["key"]).To(Equal(oldKey))
+					Expect(secret.Data["secret"]).To(Equal(oldSecret))
+				})
+			})
+
+			Context("service account key", func() {
+				var (
+					oldData = map[string][]byte{"id_rsa": []byte("some-old-key")}
+					config  *secretutils.RSASecretConfig
+				)
+
+				BeforeEach(func() {
+					config = &secretutils.RSASecretConfig{
+						Name: "service-account-key",
+						Bits: 4096,
+					}
+				})
+
+				It("should generate a new key if old secret does not exist", func() {
+					By("generating secret")
+					secret, err := m.Generate(ctx, config)
+					Expect(err).NotTo(HaveOccurred())
+
+					By("verifying new key was generated")
+					Expect(secret.Data).NotTo(Equal(oldData))
+				})
+
+				It("should keep the existing key if old secret still exists", func() {
+					By("creating existing secret with old key")
+					existingSecret := &corev1.Secret{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      "service-account-key",
+							Namespace: namespace,
+						},
+						Type: corev1.SecretTypeOpaque,
+						Data: oldData,
+					}
+					Expect(fakeClient.Create(ctx, existingSecret)).To(Succeed())
+
+					By("generating secret")
+					secret, err := m.Generate(ctx, config)
+					Expect(err).NotTo(HaveOccurred())
+
+					By("verifying old password was kept")
+					Expect(secret.Data).To(Equal(oldData))
+				})
+			})
+		})
 	})
 })