diff --git a/pkg/operation/botanist/component/kubeapiserver/secrets.go b/pkg/operation/botanist/component/kubeapiserver/secrets.go index 38bbee4a422..327aa654157 100644 --- a/pkg/operation/botanist/component/kubeapiserver/secrets.go +++ b/pkg/operation/botanist/component/kubeapiserver/secrets.go @@ -105,7 +105,8 @@ func (k *kubeAPIServer) reconcileSecretServiceAccountKey(ctx context.Context) (* return nil, err } - return secret, nil + // TODO(rfranzke): Remove this in a future release. + return secret, kutil.DeleteObject(ctx, k.client.Client(), &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "service-account-key", Namespace: k.namespace}}) } func (k *kubeAPIServer) reconcileSecretBasicAuth(ctx context.Context) (*corev1.Secret, error) { diff --git a/pkg/operation/botanist/kubeapiserver.go b/pkg/operation/botanist/kubeapiserver.go index b315db381f9..480b62d0997 100644 --- a/pkg/operation/botanist/kubeapiserver.go +++ b/pkg/operation/botanist/kubeapiserver.go @@ -19,6 +19,8 @@ import ( "fmt" "net" + gardencorev1alpha1 "github.com/gardener/gardener/pkg/apis/core/v1alpha1" + gardencorev1alpha1helper "github.com/gardener/gardener/pkg/apis/core/v1alpha1/helper" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" gardencorev1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" @@ -676,7 +678,20 @@ func (b *Botanist) DeployKubeAPIServer(ctx context.Context) error { } } - return nil + // TODO(rfranzke): Remove in a future release. + if err := b.SaveGardenerResourceDataInShootState(ctx, func(gardenerResourceData *[]gardencorev1alpha1.GardenerResourceData) error { + gardenerResourceDataList := gardencorev1alpha1helper.GardenerResourceDataList(*gardenerResourceData) + gardenerResourceDataList.Delete("etcdEncryptionConfiguration") + gardenerResourceDataList.Delete("service-account-key") + *gardenerResourceData = gardenerResourceDataList + return nil + }); err != nil { + return err + } + + return kutil.DeleteObjects(ctx, b.SeedClientSet.Client(), + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: b.Shoot.SeedNamespace, Name: "etcd-encryption-secret"}}, + ) } // DeleteKubeAPIServer deletes the kube-apiserver deployment in the Seed cluster which holds the Shoot's control plane. diff --git a/pkg/operation/botanist/kubeapiserver_test.go b/pkg/operation/botanist/kubeapiserver_test.go index 82560319264..46c0fd48651 100644 --- a/pkg/operation/botanist/kubeapiserver_test.go +++ b/pkg/operation/botanist/kubeapiserver_test.go @@ -1995,6 +1995,25 @@ usernames: ["admin"] )) }) + It("should delete the old etcd encryption config secret", func() { + kubeAPIServer.EXPECT().GetValues() + kubeAPIServer.EXPECT().SetAutoscalingReplicas(gomock.Any()) + kubeAPIServer.EXPECT().SetSNIConfig(gomock.Any()) + kubeAPIServer.EXPECT().SetETCDEncryptionConfig(gomock.Any()) + kubeAPIServer.EXPECT().SetExternalHostname(gomock.Any()) + kubeAPIServer.EXPECT().SetExternalServer(gomock.Any()) + kubeAPIServer.EXPECT().SetServerCertificateConfig(gomock.Any()) + kubeAPIServer.EXPECT().SetServiceAccountConfig(gomock.Any()) + kubeAPIServer.EXPECT().Deploy(ctx) + + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: seedNamespace, Name: "etcd-encryption-secret"}} + Expect(seedClient.Create(ctx, secret)).To(Succeed()) + + Expect(botanist.DeployKubeAPIServer(ctx)).To(Succeed()) + + Expect(seedClient.Get(ctx, client.ObjectKeyFromObject(secret), &corev1.Secret{})).To(BeNotFoundError()) + }) + It("should not sync the kubeconfig to garden project namespace when enableStaticTokenKubeconfig is set to false", func() { secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/utils/secrets/manager/generate.go b/pkg/utils/secrets/manager/generate.go index a5a073de4ae..a627a8d1a1a 100644 --- a/pkg/utils/secrets/manager/generate.go +++ b/pkg/utils/secrets/manager/generate.go @@ -28,6 +28,11 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/runtime/serializer/json" + apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -110,7 +115,15 @@ func (m *manager) generateAndCreate(ctx context.Context, config secretutils.Conf return nil, err } - secret := Secret(objectMeta, data.SecretData()) + // For backwards-compatibility, we need to keep some of the existing secrets (cluster-admin token, basic auth + // password, etc.). + // TODO(rfranzke): Remove this code in the future + dataMap, err := m.keepExistingSecretsIfNeeded(ctx, config.GetName(), data.SecretData()) + if err != nil { + return nil, err + } + + secret := Secret(objectMeta, dataMap) if err := m.client.Create(ctx, secret); err != nil { if !apierrors.IsAlreadyExists(err) { return nil, err @@ -125,6 +138,67 @@ func (m *manager) generateAndCreate(ctx context.Context, config secretutils.Conf return secret, nil } +func (m *manager) keepExistingSecretsIfNeeded(ctx context.Context, configName string, newData map[string][]byte) (map[string][]byte, error) { + existingSecret := &corev1.Secret{} + + switch configName { + case "kube-apiserver-etcd-encryption-key": + if err := m.client.Get(ctx, kutil.Key(m.namespace, "etcd-encryption-secret"), existingSecret); err != nil { + if !apierrors.IsNotFound(err) { + return nil, err + } + return newData, nil + } + + scheme := runtime.NewScheme() + if err := apiserverconfigv1.AddToScheme(scheme); err != nil { + return nil, err + } + + ser := json.NewSerializerWithOptions(json.DefaultMetaFactory, scheme, scheme, json.SerializerOptions{Yaml: true, Pretty: false, Strict: false}) + versions := schema.GroupVersions([]schema.GroupVersion{apiserverconfigv1.SchemeGroupVersion}) + codec := serializer.NewCodecFactory(scheme).CodecForVersions(ser, ser, versions, versions) + + encryptionConfiguration := &apiserverconfigv1.EncryptionConfiguration{} + if _, _, err := codec.Decode(existingSecret.Data["encryption-configuration.yaml"], nil, encryptionConfiguration); err != nil { + return nil, err + } + + var existingEncryptionKey, existingEncryptionSecret []byte + + if len(encryptionConfiguration.Resources) != 0 { + for _, provider := range encryptionConfiguration.Resources[0].Providers { + if provider.AESCBC != nil && len(provider.AESCBC.Keys) != 0 { + existingEncryptionKey = []byte(provider.AESCBC.Keys[0].Name) + existingEncryptionSecret = []byte(provider.AESCBC.Keys[0].Secret) + break + } + } + } + + if existingEncryptionKey == nil || existingEncryptionSecret == nil { + return nil, fmt.Errorf("old etcd encryption key or secret was not found") + } + + return map[string][]byte{ + secretutils.DataKeyEncryptionKeyName: existingEncryptionKey, + secretutils.DataKeyEncryptionSecret: existingEncryptionSecret, + }, nil + + case "service-account-key": + if err := m.client.Get(ctx, kutil.Key(m.namespace, "service-account-key"), existingSecret); err != nil { + if !apierrors.IsNotFound(err) { + return nil, err + } + return newData, nil + } + + return existingSecret.Data, nil + } + + return newData, nil +} + func (m *manager) shouldIgnoreOldSecrets(issuedAt string, options *GenerateOptions) (bool, error) { // unconditionally ignore old secrets if options.RotationStrategy != KeepOld || options.IgnoreOldSecrets { diff --git a/pkg/utils/secrets/manager/generate_test.go b/pkg/utils/secrets/manager/generate_test.go index c76b59990f6..db9d28ef0c3 100644 --- a/pkg/utils/secrets/manager/generate_test.go +++ b/pkg/utils/secrets/manager/generate_test.go @@ -765,6 +765,110 @@ var _ = Describe("Generate", func() { Expect(secretInfos.bundle).To(BeNil()) }) }) + + Context("backwards compatibility", func() { + Context("etcd encryption key", func() { + var ( + oldKey = []byte("old-key") + oldSecret = []byte("old-secret") + config *secretutils.ETCDEncryptionKeySecretConfig + ) + + BeforeEach(func() { + config = &secretutils.ETCDEncryptionKeySecretConfig{ + Name: "kube-apiserver-etcd-encryption-key", + SecretLength: 32, + } + }) + + It("should generate a new encryption key secret if old secret does not exist", func() { + By("generating secret") + secret, err := m.Generate(ctx, config) + Expect(err).NotTo(HaveOccurred()) + + By("verifying new key and secret were generated") + Expect(secret.Data["key"]).NotTo(Equal(oldKey)) + Expect(secret.Data["secret"]).NotTo(Equal(oldSecret)) + }) + + It("should keep the existing encryption key and secret if old secret still exists", func() { + oldEncryptionConfiguration := `apiVersion: apiserver.config.k8s.io/v1 +kind: EncryptionConfiguration +resources: +- providers: + - aescbc: + keys: + - name: ` + string(oldKey) + ` + secret: ` + string(oldSecret) + ` + - identity: {} + resources: + - secrets +` + + By("creating existing secret with old encryption configuration") + existingSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "etcd-encryption-secret", + Namespace: namespace, + }, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{"encryption-configuration.yaml": []byte(oldEncryptionConfiguration)}, + } + Expect(fakeClient.Create(ctx, existingSecret)).To(Succeed()) + + By("generating secret") + secret, err := m.Generate(ctx, config) + Expect(err).NotTo(HaveOccurred()) + + By("verifying old key and secret were kept") + Expect(secret.Data["key"]).To(Equal(oldKey)) + Expect(secret.Data["secret"]).To(Equal(oldSecret)) + }) + }) + + Context("service account key", func() { + var ( + oldData = map[string][]byte{"id_rsa": []byte("some-old-key")} + config *secretutils.RSASecretConfig + ) + + BeforeEach(func() { + config = &secretutils.RSASecretConfig{ + Name: "service-account-key", + Bits: 4096, + } + }) + + It("should generate a new key if old secret does not exist", func() { + By("generating secret") + secret, err := m.Generate(ctx, config) + Expect(err).NotTo(HaveOccurred()) + + By("verifying new key was generated") + Expect(secret.Data).NotTo(Equal(oldData)) + }) + + It("should keep the existing key if old secret still exists", func() { + By("creating existing secret with old key") + existingSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "service-account-key", + Namespace: namespace, + }, + Type: corev1.SecretTypeOpaque, + Data: oldData, + } + Expect(fakeClient.Create(ctx, existingSecret)).To(Succeed()) + + By("generating secret") + secret, err := m.Generate(ctx, config) + Expect(err).NotTo(HaveOccurred()) + + By("verifying old password was kept") + Expect(secret.Data).To(Equal(oldData)) + }) + }) + }) }) })