Skip to content

Commit

Permalink
Move observability-ingress{-users} secret generation out of `pluton…
Browse files Browse the repository at this point in the history 
…o` component (gardener#9285)

* Move `observability-ingress-users` secret generation out of `plutono` component to central place in `Shoot` controller

* Move `observability-ingress` secret generation out of `plutono` component to central place in `Garden` controller
  • Loading branch information
@rfranzke
rfranzke authored Feb 29, 2024
1 parent 8e3152a commit 1a8dd4a
Show file tree
Hide file tree
Showing 9 changed files with 130 additions and 94 deletions.
26 changes: 7 additions & 19 deletions pkg/component/observability/plutono/plutono.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -706,32 +706,20 @@ func (p *plutono) getIngress(ctx context.Context) (*networkingv1.Ingress, error)

if p.values.IsGardenCluster {
pathType = networkingv1.PathTypeImplementationSpecific
credentialsSecret, err := p.secretsManager.Generate(ctx, &secrets.BasicAuthSecretConfig{
Name: v1beta1constants.SecretNameObservabilityIngress,
Format: secrets.BasicAuthFormatNormal,
Username: "admin",
PasswordLength: 32,
}, secretsmanager.Persist(), secretsmanager.Rotate(secretsmanager.InPlace),
)
if err != nil {
return nil, err

credentialsSecret, found := p.secretsManager.Get(v1beta1constants.SecretNameObservabilityIngress)
if !found {
return nil, fmt.Errorf("secret %q not found", v1beta1constants.SecretNameObservabilityIngress)
}

credentialsSecretName = credentialsSecret.Name
caName = operatorv1alpha1.SecretNameCARuntime
}

if p.values.ClusterType == component.ClusterTypeShoot {
credentialsSecret, err := p.secretsManager.Generate(ctx, &secrets.BasicAuthSecretConfig{
Name: v1beta1constants.SecretNameObservabilityIngressUsers,
Format: secrets.BasicAuthFormatNormal,
Username: "admin",
PasswordLength: 32,
}, secretsmanager.Persist(),
secretsmanager.Rotate(secretsmanager.InPlace),
)
if err != nil {
return nil, err
credentialsSecret, found := p.secretsManager.Get(v1beta1constants.SecretNameObservabilityIngressUsers)
if !found {
return nil, fmt.Errorf("secret %q not found", v1beta1constants.SecretNameObservabilityIngressUsers)
}

credentialsSecretName = credentialsSecret.Name
Expand Down
8 changes: 6 additions & 2 deletions pkg/component/observability/plutono/plutono_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,10 @@ var _ = Describe("Plutono", func() {
},
}

By("Create secrets managed outside of this function for whose secretsmanager.Get() will be called")
Expect(c.Create(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "observability-ingress", Namespace: namespace}})).To(Succeed())
Expect(c.Create(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "observability-ingress-users", Namespace: namespace}})).To(Succeed())

// extensions dashboard
filePath := filepath.Join("testdata", "configmap.yaml")
cm, err := os.ReadFile(filePath)
Expand Down Expand Up @@ -469,7 +473,7 @@ metadata:
`
if !values.IsGardenCluster {
if values.ClusterType == comp.ClusterTypeShoot {
out += ` nginx.ingress.kubernetes.io/auth-secret: observability-ingress-users-f27eb0bf
out += ` nginx.ingress.kubernetes.io/auth-secret: observability-ingress-users
nginx.ingress.kubernetes.io/auth-type: basic
`
} else {
Expand All @@ -478,7 +482,7 @@ metadata:
`
}
} else {
out += ` nginx.ingress.kubernetes.io/auth-secret: observability-ingress-0da36eb1
out += ` nginx.ingress.kubernetes.io/auth-secret: observability-ingress
nginx.ingress.kubernetes.io/auth-type: basic
`
}
Expand Down
36 changes: 6 additions & 30 deletions pkg/gardenlet/operation/botanist/plutono.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,13 @@ package botanist

import (
"context"
"fmt"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/ptr"

v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
"github.com/gardener/gardener/pkg/component"
"github.com/gardener/gardener/pkg/component/observability/plutono"
"github.com/gardener/gardener/pkg/component/shared"
gardenerutils "github.com/gardener/gardener/pkg/utils/gardener"
kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes"
)

// DefaultPlutono returns a deployer for Plutono.
Expand All @@ -53,33 +48,14 @@ func (b *Botanist) DefaultPlutono() (plutono.Interface, error) {

// DeployPlutono deploys the plutono in the Seed cluster.
func (b *Botanist) DeployPlutono(ctx context.Context) error {
if b.ControlPlaneWildcardCert != nil {
b.Operation.Shoot.Components.ControlPlane.Plutono.SetWildcardCertName(ptr.To(b.ControlPlaneWildcardCert.GetName()))
// disable plutono if no observability components are needed
if !b.Operation.WantsObservabilityComponents() {
return b.Shoot.Components.ControlPlane.Plutono.Destroy(ctx)
}
// disable monitoring if shoot has purpose testing or monitoring and vali is disabled
if !b.Operation.WantsPlutono() {
if err := b.Shoot.Components.ControlPlane.Plutono.Destroy(ctx); err != nil {
return err
}

secretName := gardenerutils.ComputeShootProjectResourceName(b.Shoot.GetInfo().Name, gardenerutils.ShootProjectSecretSuffixMonitoring)
return kubernetesutils.DeleteObject(ctx, b.GardenClient, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: secretName, Namespace: b.Shoot.GetInfo().Namespace}})
}

if err := b.Shoot.Components.ControlPlane.Plutono.Deploy(ctx); err != nil {
return err
}

credentialsSecret, found := b.SecretsManager.Get(v1beta1constants.SecretNameObservabilityIngressUsers)
if !found {
return fmt.Errorf("secret %q not found", v1beta1constants.SecretNameObservabilityIngressUsers)
if b.ControlPlaneWildcardCert != nil {
b.Operation.Shoot.Components.ControlPlane.Plutono.SetWildcardCertName(ptr.To(b.ControlPlaneWildcardCert.GetName()))
}

return b.syncShootCredentialToGarden(
ctx,
gardenerutils.ShootProjectSecretSuffixMonitoring,
map[string]string{v1beta1constants.GardenRole: v1beta1constants.GardenRoleMonitoring},
map[string]string{"url": "https://" + b.ComputePlutonoHost()},
credentialsSecret.Data,
)
return b.Shoot.Components.ControlPlane.Plutono.Deploy(ctx)
}
47 changes: 6 additions & 41 deletions pkg/gardenlet/operation/botanist/plutono_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,6 @@ import (
"github.com/gardener/gardener/pkg/gardenlet/operation/garden"
seedpkg "github.com/gardener/gardener/pkg/gardenlet/operation/seed"
shootpkg "github.com/gardener/gardener/pkg/gardenlet/operation/shoot"
kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes"
secretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager"
fakesecretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager/fake"
. "github.com/gardener/gardener/pkg/utils/test/matchers"
)

var _ = Describe("Plutono", func() {
Expand All @@ -52,8 +48,6 @@ var _ = Describe("Plutono", func() {
seedClient client.Client
seedClientSet kubernetes.Interface

sm secretsmanager.Interface

mockPlutono *mockplutono.MockInterface

botanist *Botanist
Expand All @@ -64,7 +58,6 @@ var _ = Describe("Plutono", func() {
shootName = "bar"

shootPurposeEvaluation = gardencorev1beta1.ShootPurposeEvaluation
shootPurposeTesting = gardencorev1beta1.ShootPurposeTesting
)

BeforeEach(func() {
Expand All @@ -77,27 +70,14 @@ var _ = Describe("Plutono", func() {
WithClient(seedClient).
WithRESTConfig(&rest.Config{}).
Build()
sm = fakesecretsmanager.New(seedClient, seedNamespace)

By("Create secrets managed outside of this function for whose secretsmanager.Get() will be called")
Expect(seedClient.Create(ctx, &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "observability-ingress-users",
Namespace: seedNamespace,
Labels: map[string]string{"gardener.cloud/role": "monitoring"},
Annotations: map[string]string{"url": "https://gu-foo--bar.example.com"},
},
Data: map[string][]byte{"username": {}, "password": {}, "auth": {}},
})).To(Succeed())

mockPlutono = mockplutono.NewMockInterface(ctrl)

botanist = &Botanist{
Operation: &operation.Operation{
GardenClient: gardenClient,
SeedClientSet: seedClientSet,
SecretsManager: sm,
Config: &config.GardenletConfiguration{},
GardenClient: gardenClient,
SeedClientSet: seedClientSet,
Config: &config.GardenletConfiguration{},
Garden: &garden.Garden{
Project: &gardencorev1beta1.Project{},
},
Expand Down Expand Up @@ -143,30 +123,15 @@ var _ = Describe("Plutono", func() {
})

Describe("#DeployPlutono", func() {
It("should successfully deploy plutono sync the ingress credentials for the users observability to the garden project namespace", func() {
Expect(gardenClient.Get(ctx, kubernetesutils.Key(projectNamespace, shootName+".monitoring"), &corev1.Secret{})).To(BeNotFoundError())
It("should successfully deploy plutono", func() {
mockPlutono.EXPECT().Deploy(ctx)
Expect(botanist.DeployPlutono(ctx)).To(Succeed())

secret := &corev1.Secret{}
Expect(gardenClient.Get(ctx, kubernetesutils.Key(projectNamespace, shootName+".monitoring"), secret)).To(Succeed())
Expect(secret.Annotations).To(HaveKeyWithValue("url", "https://gu-foo--bar."))
Expect(secret.Labels).To(HaveKeyWithValue("gardener.cloud/role", "monitoring"))
Expect(secret.Data).To(And(HaveKey("username"), HaveKey("password"), HaveKey("auth")))
})

It("should cleanup the secrets when shoot purpose is changed", func() {
Expect(gardenClient.Get(ctx, kubernetesutils.Key(projectNamespace, shootName+".monitoring"), &corev1.Secret{})).To(BeNotFoundError())
mockPlutono.EXPECT().Deploy(ctx)
Expect(botanist.DeployPlutono(ctx)).To(Succeed())
Expect(gardenClient.Get(ctx, kubernetesutils.Key(projectNamespace, shootName+".monitoring"), &corev1.Secret{})).To(Succeed())
Expect(*botanist.Shoot.GetInfo().Spec.Purpose).To(Equal(shootPurposeEvaluation))

botanist.Shoot.Purpose = shootPurposeTesting
It("should successfully destroy plutono", func() {
botanist.Shoot.Purpose = gardencorev1beta1.ShootPurposeTesting
mockPlutono.EXPECT().Destroy(ctx)
Expect(botanist.DeployPlutono(ctx)).To(Succeed())

Expect(gardenClient.Get(ctx, kubernetesutils.Key(projectNamespace, shootName+".monitoring"), &corev1.Secret{})).To(BeNotFoundError())
})
})
})
30 changes: 30 additions & 0 deletions pkg/gardenlet/operation/botanist/secrets.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,12 @@ func (b *Botanist) InitializeSecretsManagement(ctx context.Context) error {
taskFns = append(taskFns, b.deleteSSHKeypair)
}

if b.WantsObservabilityComponents() {
taskFns = append(taskFns, b.generateObservabilityIngressPassword)
} else {
taskFns = append(taskFns, b.deleteObservabilityIngressPassword)
}

return flow.Sequential(taskFns...)(ctx)
}

Expand Down Expand Up @@ -297,6 +303,30 @@ func (b *Botanist) generateSSHKeypair(ctx context.Context) error {
return nil
}

func (b *Botanist) generateObservabilityIngressPassword(ctx context.Context) error {
secret, err := b.SecretsManager.Generate(ctx, &secretsutils.BasicAuthSecretConfig{
Name: v1beta1constants.SecretNameObservabilityIngressUsers,
Format: secretsutils.BasicAuthFormatNormal,
Username: "admin",
PasswordLength: 32,
}, secretsmanager.Persist(), secretsmanager.Rotate(secretsmanager.InPlace))
if err != nil {
return err
}

return b.syncShootCredentialToGarden(
ctx,
gardenerutils.ShootProjectSecretSuffixMonitoring,
map[string]string{v1beta1constants.GardenRole: v1beta1constants.GardenRoleMonitoring},
map[string]string{"url": "https://" + b.ComputePlutonoHost()},
secret.Data,
)
}

func (b *Botanist) deleteObservabilityIngressPassword(ctx context.Context) error {
return b.deleteShootCredentialFromGarden(ctx, gardenerutils.ShootProjectSecretSuffixMonitoring)
}

// quotaExceededRegex is used to check if an error occurred due to infrastructure quota limits.
var quotaExceededRegex = regexp.MustCompile(`(?i)((?:^|[^t]|(?:[^s]|^)t|(?:[^e]|^)st|(?:[^u]|^)est|(?:[^q]|^)uest|(?:[^e]|^)quest|(?:[^r]|^)equest)LimitExceeded|Quotas|Quota.*exceeded|exceeded quota|Quota has been met|QUOTA_EXCEEDED)`)

Expand Down
56 changes: 56 additions & 0 deletions pkg/gardenlet/operation/botanist/secrets_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import (
kubernetesfake "github.com/gardener/gardener/pkg/client/kubernetes/fake"
"github.com/gardener/gardener/pkg/gardenlet/operation"
. "github.com/gardener/gardener/pkg/gardenlet/operation/botanist"
seedpkg "github.com/gardener/gardener/pkg/gardenlet/operation/seed"
shootpkg "github.com/gardener/gardener/pkg/gardenlet/operation/shoot"
"github.com/gardener/gardener/pkg/utils"
kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes"
Expand Down Expand Up @@ -87,11 +88,19 @@ var _ = Describe("Secrets", func() {
SeedClientSet: seedClientSet,
ShootClientSet: shootClientSet,
SecretsManager: fakeSecretsManager,
Seed: &seedpkg.Seed{},
Shoot: &shootpkg.Shoot{
SeedNamespace: seedNamespace,
},
},
}
botanist.Seed.SetInfo(&gardencorev1beta1.Seed{
Spec: gardencorev1beta1.SeedSpec{
Ingress: &gardencorev1beta1.Ingress{
Domain: "example.com",
},
},
})
botanist.Shoot.SetInfo(&gardencorev1beta1.Shoot{
ObjectMeta: metav1.ObjectMeta{
Name: shootName,
Expand All @@ -104,6 +113,9 @@ var _ = Describe("Secrets", func() {
},
},
},
Status: gardencorev1beta1.ShootStatus{
TechnicalID: seedNamespace,
},
})
botanist.Shoot.SetShootState(&gardencorev1beta1.ShootState{})
})
Expand Down Expand Up @@ -215,6 +227,50 @@ var _ = Describe("Secrets", func() {
Expect(gardenClient.Get(ctx, kubernetesutils.Key(gardenNamespace, shootName+".ssh-keypair"), gardenSecret)).To(BeNotFoundError())
Expect(gardenClient.Get(ctx, kubernetesutils.Key(gardenNamespace, shootName+".ssh-keypair.old"), gardenSecret)).To(BeNotFoundError())
})

Context("observability credentials", func() {
It("should generate the password and sync it to the garden", func() {
Expect(botanist.InitializeSecretsManagement(ctx)).To(Succeed())

secretList := &corev1.SecretList{}
Expect(seedClient.List(ctx, secretList, client.InNamespace(seedNamespace), client.MatchingLabels{
"name": "observability-ingress-users",
"managed-by": "secrets-manager",
})).To(Succeed())
Expect(secretList.Items).To(HaveLen(1))
Expect(secretList.Items[0].Immutable).To(PointTo(BeTrue()))
Expect(secretList.Items[0].Labels).To(And(
HaveKeyWithValue("name", "observability-ingress-users"),
HaveKeyWithValue("managed-by", "secrets-manager"),
HaveKeyWithValue("persist", "true"),
HaveKeyWithValue("rotation-strategy", "inplace"),
HaveKey("checksum-of-config"),
HaveKey("last-rotation-initiation-time"),
))

gardenSecret := &corev1.Secret{}
Expect(gardenClient.Get(ctx, kubernetesutils.Key(gardenNamespace, shootName+".monitoring"), gardenSecret)).To(Succeed())
Expect(gardenSecret.Annotations).To(HaveKeyWithValue("url", "https://gu-foo--bar.example.com"))
Expect(gardenSecret.Labels).To(HaveKeyWithValue("gardener.cloud/role", "monitoring"))
Expect(gardenSecret.Data).To(And(HaveKey("username"), HaveKey("password"), HaveKey("auth")))
})

It("should not generate the password in case no observability components are needed", func() {
botanist.Shoot.Purpose = gardencorev1beta1.ShootPurposeTesting

Expect(botanist.InitializeSecretsManagement(ctx)).To(Succeed())

secretList := &corev1.SecretList{}
Expect(seedClient.List(ctx, secretList, client.InNamespace(seedNamespace), client.MatchingLabels{
"name": "observability-ingress-users",
"managed-by": "secrets-manager",
})).To(Succeed())
Expect(secretList.Items).To(BeEmpty())

gardenSecret := &corev1.Secret{}
Expect(gardenClient.Get(ctx, kubernetesutils.Key(gardenNamespace, shootName+".monitoring"), gardenSecret)).To(BeNotFoundError())
})
})
})

Context("when shoot is in restoration phase", func() {
Expand Down
4 changes: 2 additions & 2 deletions pkg/gardenlet/operation/operation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -411,8 +411,8 @@ func (o *Operation) IsShootMonitoringEnabled() bool {
return helper.IsMonitoringEnabled(o.Config) && o.Shoot.Purpose != gardencorev1beta1.ShootPurposeTesting
}

// WantsPlutono returns true if shoot is not of purpose testing and either shoot monitoring or vali is enabled.
func (o *Operation) WantsPlutono() bool {
// WantsObservabilityComponents returns true if shoot is not of purpose testing and either shoot monitoring or vali is enabled.
func (o *Operation) WantsObservabilityComponents() bool {
return o.Shoot.Purpose != gardencorev1beta1.ShootPurposeTesting && (helper.IsMonitoringEnabled(o.Config) || helper.IsValiEnabled(o.Config))
}

Expand Down
10 changes: 10 additions & 0 deletions pkg/operator/controller/garden/garden/reconciler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -352,6 +352,16 @@ func (r *Reconciler) cleanupGenericTokenKubeconfig(ctx context.Context, secretsM
return client.IgnoreNotFound(r.RuntimeClientSet.Client().Delete(ctx, secret))
}

func (r *Reconciler) generateObservabilityIngressPassword(ctx context.Context, secretsManager secretsmanager.Interface) error {
_, err := secretsManager.Generate(ctx, &secretsutils.BasicAuthSecretConfig{
Name: v1beta1constants.SecretNameObservabilityIngress,
Format: secretsutils.BasicAuthFormatNormal,
Username: "admin",
PasswordLength: 32,
}, secretsmanager.Persist(), secretsmanager.Rotate(secretsmanager.InPlace))
return err
}

func startRotationCA(garden *operatorv1alpha1.Garden, now *metav1.Time) {
helper.MutateCARotation(garden, func(rotation *gardencorev1beta1.CARotation) {
rotation.Phase = gardencorev1beta1.RotationPreparing
Expand Down
Loading

0 comments on commit 1a8dd4a

Please sign in to comment.