diff --git a/cmd/gardenlet/app/app.go b/cmd/gardenlet/app/app.go index 049b5a148a4..1cca579d9cf 100644 --- a/cmd/gardenlet/app/app.go +++ b/cmd/gardenlet/app/app.go @@ -21,6 +21,7 @@ import ( "net/http" "os" goruntime "runtime" + "slices" "strconv" "strings" "time" @@ -32,12 +33,19 @@ import ( coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" eventsv1 "k8s.io/api/events/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/runtime/serializer/json" "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/rest" @@ -63,6 +71,7 @@ import ( resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" "github.com/gardener/gardener/pkg/client/kubernetes" clientmapbuilder "github.com/gardener/gardener/pkg/client/kubernetes/clientmap/builder" + dwd "github.com/gardener/gardener/pkg/component/dependencywatchdog" "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/controllerutils/routes" "github.com/gardener/gardener/pkg/features" @@ -72,6 +81,7 @@ import ( "github.com/gardener/gardener/pkg/gardenlet/bootstrap/certificate" "github.com/gardener/gardener/pkg/gardenlet/controller" gardenerhealthz "github.com/gardener/gardener/pkg/healthz" + resourcemanagerv1alpha1 "github.com/gardener/gardener/pkg/resourcemanager/apis/config/v1alpha1" "github.com/gardener/gardener/pkg/utils" "github.com/gardener/gardener/pkg/utils/flow" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" @@ -80,8 +90,10 @@ import ( thirdpartyapiutil "github.com/gardener/gardener/third_party/controller-runtime/pkg/apiutil" ) -// Name is a const for the name of this component. -const Name = "gardenlet" +const ( + // Name is a const for the name of this component. + Name = "gardenlet" +) // NewCommand creates a new cobra.Command for running gardenlet. func NewCommand() *cobra.Command { @@ -389,6 +401,11 @@ func (g *garden) Start(ctx context.Context) error { return err } + log.Info("Creating new secret and managed resource required by dependency-watchdog") + if err := g.createNewDWDResources(ctx, g.mgr.GetClient()); err != nil { + return err + } + log.Info("Setting up shoot client map") shootClientMap, err := clientmapbuilder. NewShootClientMapBuilder(). @@ -438,6 +455,132 @@ func (g *garden) Start(ctx context.Context) error { return nil } +// TODO(aaronfern): Remove this code after v1.93 has been released. +func (g *garden) createNewDWDResources(ctx context.Context, seedClient client.Client) error { + namespaceList := &corev1.NamespaceList{} + if err := seedClient.List(ctx, namespaceList, client.MatchingLabels(map[string]string{v1beta1constants.GardenRole: v1beta1constants.GardenRoleShoot})); err != nil { + return err + } + + var tasks []flow.TaskFn + for _, ns := range namespaceList.Items { + if ns.DeletionTimestamp != nil || ns.Status.Phase == corev1.NamespaceTerminating { + continue + } + namespace := ns + tasks = append(tasks, func(ctx context.Context) error { + dwdOldSecret := &corev1.Secret{} + if err := seedClient.Get(ctx, types.NamespacedName{Namespace: namespace.Name, Name: dwd.InternalProbeSecretName}, dwdOldSecret); err != nil { + // If ns does not contain old DWD secret, do not procees. + if apierrors.IsNotFound(err) { + return nil + } + return err + } + + // Fetch GRM deployment + grmDeploy := &appsv1.Deployment{} + if err := seedClient.Get(ctx, types.NamespacedName{Namespace: namespace.Name, Name: "gardener-resource-manager"}, grmDeploy); err != nil { + if apierrors.IsNotFound(err) { + // Do not proceed if GRM deployment is not present + return nil + } + return err + } + + // Create a DWDAccess object + inClusterServerURL := fmt.Sprintf("%s.%s.svc", v1beta1constants.DeploymentNameKubeAPIServer, namespace.Name) + dwdAccess := dwd.NewAccess(seedClient, namespace.Name, nil, dwd.AccessValues{ServerInCluster: inClusterServerURL}) + + if err := dwdAccess.DeployMigrate(ctx); err != nil { + return err + } + + //Delete old DWD secrets + if err := kubernetesutils.DeleteObjects(ctx, seedClient, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: dwd.InternalProbeSecretName, Namespace: namespace.Name}}); err != nil { + return err + } + + if err := kubernetesutils.DeleteObjects(ctx, seedClient, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: dwd.ExternalProbeSecretName, Namespace: namespace.Name}}); err != nil { + return err + } + + // Fetch and update the GRM configmap + grmConfigMap := &corev1.ConfigMap{} + var grmCMName string + var grmCMVolumeIndex int + for n, vol := range grmDeploy.Spec.Template.Spec.Volumes { + if vol.Name == "config" { + grmCMName = vol.ConfigMap.Name + grmCMVolumeIndex = n + break + } + } + if len(grmCMName) == 0 { + return nil + } + if err := seedClient.Get(ctx, types.NamespacedName{Namespace: namespace.Name, Name: grmCMName}, grmConfigMap); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return err + } + + cmData := grmConfigMap.Data["config.yaml"] + rmConfig := resourcemanagerv1alpha1.ResourceManagerConfiguration{} + + // create codec + var codec runtime.Codec + configScheme := runtime.NewScheme() + utilruntime.Must(resourcemanagerv1alpha1.AddToScheme(configScheme)) + utilruntime.Must(apiextensionsv1.AddToScheme(configScheme)) + ser := json.NewSerializerWithOptions(json.DefaultMetaFactory, configScheme, configScheme, json.SerializerOptions{ + Yaml: true, + Pretty: false, + Strict: false, + }) + versions := schema.GroupVersions([]schema.GroupVersion{ + resourcemanagerv1alpha1.SchemeGroupVersion, + apiextensionsv1.SchemeGroupVersion, + }) + codec = serializer.NewCodecFactory(configScheme).CodecForVersions(ser, ser, versions, versions) + + obj, err := runtime.Decode(codec, []byte(cmData)) + if err != nil { + return err + } + rmConfig = *(obj.(*resourcemanagerv1alpha1.ResourceManagerConfiguration)) + + if rmConfig.TargetClientConnection == nil || slices.Contains(rmConfig.TargetClientConnection.Namespaces, corev1.NamespaceNodeLease) { + return nil + } + + rmConfig.TargetClientConnection.Namespaces = append(rmConfig.TargetClientConnection.Namespaces, corev1.NamespaceNodeLease) + + data, err := runtime.Encode(codec, &rmConfig) + if err != nil { + return err + } + + newGRMConfigMap := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "gardener-resource-manager-dwd", Namespace: namespace.Name}} + newGRMConfigMap.Data = map[string]string{"config.yaml": string(data)} + utilruntime.Must(kubernetesutils.MakeUnique(newGRMConfigMap)) + + if err = seedClient.Create(ctx, newGRMConfigMap); err != nil { + if !apierrors.IsAlreadyExists(err) { + return err + } + } + + patch := client.MergeFrom(grmDeploy.DeepCopy()) + grmDeploy.Spec.Template.Spec.Volumes[grmCMVolumeIndex].ConfigMap.Name = newGRMConfigMap.Name + + return seedClient.Patch(ctx, grmDeploy, patch) + }) + } + return flow.Parallel(tasks...)(ctx) +} + // TODO(Kostov6): Remove this code after v1.91 has been released. func cleanupGRMSecretFinalizers(ctx context.Context, seedClient client.Client, log logr.Logger) error { var ( diff --git a/go.mod b/go.mod index 4d116bc2b07..fd51851777d 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/containerd/containerd v1.7.13 github.com/coreos/go-systemd/v22 v22.5.0 github.com/fluent/fluent-operator/v2 v2.7.0 - github.com/gardener/dependency-watchdog v1.1.2 + github.com/gardener/dependency-watchdog v1.2.1 github.com/gardener/etcd-druid v0.22.0 github.com/gardener/hvpa-controller/api v0.5.0 github.com/gardener/machine-controller-manager v0.50.0 diff --git a/go.sum b/go.sum index e522d31b7bb..09b51f44d03 100644 --- a/go.sum +++ b/go.sum @@ -146,8 +146,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/gardener/dependency-watchdog v1.1.2 h1:id9FAnjL9kiZec+QNVacw4BlxLrI3deMtN4KyxpCcKk= -github.com/gardener/dependency-watchdog v1.1.2/go.mod h1:giWCBTBkZiY00dv06/DpASzPgc0U+XVF+ZOGkTUewjk= +github.com/gardener/dependency-watchdog v1.2.1 h1:Q0zqinZNImBuNYfNQGAXkUh5qrfJyrynO5QjUTzO/7w= +github.com/gardener/dependency-watchdog v1.2.1/go.mod h1:RgU0VmsdBHxRU8IO9VsLxEinz58xEJdEz5hxvMqLKHQ= github.com/gardener/etcd-druid v0.22.0 h1:DVe+Zjrb93r9vI1uUiCTMHBffIUoMAKhNzFZNC6hsQ8= github.com/gardener/etcd-druid v0.22.0/go.mod h1:FROhfVKyWBo4krlPe3R6FIhJRmOmijEWBdEeUP0CJjE= github.com/gardener/hvpa-controller/api v0.5.0 h1:f4F3O7YUrenwh4S3TgPREPiB287JjjUiUL18OqPLyAA= diff --git a/imagevector/images.yaml b/imagevector/images.yaml index 9a39c1656ff..ea0d64eeccf 100644 --- a/imagevector/images.yaml +++ b/imagevector/images.yaml @@ -58,7 +58,7 @@ images: - name: dependency-watchdog sourceRepository: github.com/gardener/dependency-watchdog repository: europe-docker.pkg.dev/gardener-project/releases/gardener/dependency-watchdog - tag: "v1.1.2" + tag: "v1.2.1" - name: nginx-ingress-controller sourceRepository: github.com/kubernetes/ingress-nginx repository: registry.k8s.io/ingress-nginx/controller-chroot diff --git a/pkg/component/dependencywatchdog/access.go b/pkg/component/dependencywatchdog/access.go index 5bffbf77fde..2a4ce9a2ba1 100644 --- a/pkg/component/dependencywatchdog/access.go +++ b/pkg/component/dependencywatchdog/access.go @@ -17,17 +17,22 @@ package dependencywatchdog import ( "context" "fmt" + "strings" "time" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" clientcmdv1 "k8s.io/client-go/tools/clientcmd/api/v1" "sigs.k8s.io/controller-runtime/pkg/client" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/gardener/pkg/client/kubernetes" "github.com/gardener/gardener/pkg/component" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" + "github.com/gardener/gardener/pkg/utils/managedresources" secretsutils "github.com/gardener/gardener/pkg/utils/secrets" secretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager" ) @@ -37,6 +42,11 @@ const ( DefaultProbeInterval = 30 * time.Second // DefaultWatchDuration is the default value of the total duration for which a DWD Weeder watches for any dependant Pod to transition to CrashLoopBackoff after the target service has recovered. DefaultWatchDuration = 5 * time.Minute + // KubeConfigSecretName is the name of the kubecfg secret with internal DNS for external access. + KubeConfigSecretName = gardenerutils.SecretNamePrefixShootAccess + "dependency-watchdog-probe" + // managedResourceName is the name of the managed resource created for DWD. + managedResourceName = "shoot-core-dependency-watchdog" + // ExternalProbeSecretName is the name of the kubecfg secret with internal DNS for external access. ExternalProbeSecretName = gardenerutils.SecretNamePrefixShootAccess + "dependency-watchdog-external-probe" // InternalProbeSecretName is the name of the kubecfg secret with cluster IP access. @@ -49,7 +59,7 @@ func NewAccess( namespace string, secretsManager secretsmanager.Interface, values AccessValues, -) component.Deployer { +) Interface { return &dependencyWatchdogAccess{ client: client, namespace: namespace, @@ -67,42 +77,107 @@ type dependencyWatchdogAccess struct { // AccessValues contains configurations for the component. type AccessValues struct { - // ServerOutOfCluster is the out-of-cluster address of a kube-apiserver. - ServerOutOfCluster string // ServerInCluster is the in-cluster address of a kube-apiserver. ServerInCluster string } +// Interface exposes methods for deploying dependency-watchdog. +type Interface interface { + component.Deployer + DeployMigrate(ctx context.Context) error +} + func (d *dependencyWatchdogAccess) Deploy(ctx context.Context) error { caSecret, found := d.secretsManager.Get(v1beta1constants.SecretNameCACluster) if !found { return fmt.Errorf("secret %q not found", v1beta1constants.SecretNameCACluster) } - for name, server := range map[string]string{ - InternalProbeSecretName: d.values.ServerInCluster, - ExternalProbeSecretName: d.values.ServerOutOfCluster, - } { - var ( - shootAccessSecret = gardenerutils.NewShootAccessSecret(name, d.namespace).WithNameOverride(name) - kubeconfig = kubernetesutils.NewKubeconfig( - d.namespace, - clientcmdv1.Cluster{Server: server, CertificateAuthorityData: caSecret.Data[secretsutils.DataKeyCertificateBundle]}, - clientcmdv1.AuthInfo{Token: ""}, - ) + if err := d.createShootAccessSecret(ctx, caSecret); err != nil { + return err + } + + return d.createManagedResource(ctx) +} + +// TODO(aaronfern): Remove this function after v1.93 got released. +func (d *dependencyWatchdogAccess) DeployMigrate(ctx context.Context) error { + caSecret := &corev1.Secret{} + if err := d.client.Get(ctx, types.NamespacedName{Namespace: d.namespace, Name: v1beta1constants.SecretNameCACluster}, caSecret); err != nil { + return fmt.Errorf("error in fetching secret %s: %w", v1beta1constants.SecretNameCACluster, err) + } + + if err := d.createShootAccessSecret(ctx, caSecret); err != nil { + return err + } + + return d.createManagedResource(ctx) +} + +func (d *dependencyWatchdogAccess) createShootAccessSecret(ctx context.Context, caSecret *corev1.Secret) error { + var ( + shootAccessSecret = gardenerutils.NewShootAccessSecret(KubeConfigSecretName, d.namespace).WithNameOverride(KubeConfigSecretName) + kubeconfig = kubernetesutils.NewKubeconfig( + d.namespace, + clientcmdv1.Cluster{Server: d.values.ServerInCluster, CertificateAuthorityData: caSecret.Data[secretsutils.DataKeyCertificateBundle]}, + clientcmdv1.AuthInfo{Token: ""}, ) + ) + + return shootAccessSecret.WithKubeconfig(kubeconfig).Reconcile(ctx, d.client) +} + +func (d *dependencyWatchdogAccess) createManagedResource(ctx context.Context) error { + var ( + registry = managedresources.NewRegistry(kubernetes.ShootScheme, kubernetes.ShootCodec, kubernetes.ShootSerializer) - if err := shootAccessSecret.WithKubeconfig(kubeconfig).Reconcile(ctx, d.client); err != nil { - return err + role = &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gardener.cloud:target:" + prefixDependencyWatchdog, + Namespace: corev1.NamespaceNodeLease, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"coordination.k8s.io"}, + Resources: []string{"leases"}, + Verbs: []string{"get", "list"}, + }, + }, } + roleBinding = &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gardener.cloud:target:" + prefixDependencyWatchdog, + Namespace: corev1.NamespaceNodeLease, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "Role", + Name: role.Name, + }, + Subjects: []rbacv1.Subject{{ + Kind: rbacv1.ServiceAccountKind, + Name: strings.TrimPrefix(KubeConfigSecretName, gardenerutils.SecretNamePrefixShootAccess), + Namespace: metav1.NamespaceSystem, + }}, + } + ) + + resources, err := registry.AddAllAndSerialize( + role, + roleBinding, + ) + if err != nil { + return err } - return nil + return managedresources.CreateForShoot(ctx, d.client, d.namespace, managedResourceName, managedresources.LabelValueGardener, false, resources) } func (d *dependencyWatchdogAccess) Destroy(ctx context.Context) error { + if err := managedresources.DeleteForShoot(ctx, d.client, d.namespace, managedResourceName); err != nil { + return err + } return kubernetesutils.DeleteObjects(ctx, d.client, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: InternalProbeSecretName, Namespace: d.namespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: ExternalProbeSecretName, Namespace: d.namespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: KubeConfigSecretName, Namespace: d.namespace}}, ) } diff --git a/pkg/component/dependencywatchdog/access_test.go b/pkg/component/dependencywatchdog/access_test.go index 2e573ce6a79..871d238741a 100644 --- a/pkg/component/dependencywatchdog/access_test.go +++ b/pkg/component/dependencywatchdog/access_test.go @@ -21,9 +21,11 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" "github.com/gardener/gardener/pkg/client/kubernetes" "github.com/gardener/gardener/pkg/component" . "github.com/gardener/gardener/pkg/component/dependencywatchdog" @@ -41,14 +43,45 @@ var _ = Describe("Access", func() { ctx = context.TODO() namespace = "shoot--foo--bar" - externalProbeSecretName = "shoot-access-dependency-watchdog-external-probe" - internalProbeSecretName = "shoot-access-dependency-watchdog-internal-probe" - - serverOutOfCluster = "out-of-cluster" - serverInCluster = "in-cluster" - - expectedExternalProbeSecret *corev1.Secret - expectedInternalProbeSecret *corev1.Secret + probeSecretName = "shoot-access-dependency-watchdog-probe" + + serverInCluster = "in-cluster" + + expectedProbeSecret *corev1.Secret + expectedManagedResource *resourcesv1alpha1.ManagedResource + expectedManagedResourceSecret *corev1.Secret + + roleYAML = `apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + name: gardener.cloud:target:dependency-watchdog + namespace: kube-node-lease +rules: +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list +` + + roleBindingYAML = `apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + name: gardener.cloud:target:dependency-watchdog + namespace: kube-node-lease +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: gardener.cloud:target:dependency-watchdog +subjects: +- kind: ServiceAccount + name: dependency-watchdog-probe + namespace: kube-system +` ) BeforeEach(func() { @@ -59,21 +92,20 @@ var _ = Describe("Access", func() { Expect(fakeClient.Create(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "ca", Namespace: namespace}})).To(Succeed()) access = NewAccess(fakeClient, namespace, sm, AccessValues{ - ServerOutOfCluster: serverOutOfCluster, - ServerInCluster: serverInCluster, + ServerInCluster: serverInCluster, }) - expectedExternalProbeSecret = &corev1.Secret{ + expectedProbeSecret = &corev1.Secret{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", Kind: "Secret", }, ObjectMeta: metav1.ObjectMeta{ - Name: externalProbeSecretName, + Name: probeSecretName, Namespace: namespace, ResourceVersion: "1", Annotations: map[string]string{ - "serviceaccount.resources.gardener.cloud/name": "dependency-watchdog-external-probe", + "serviceaccount.resources.gardener.cloud/name": "dependency-watchdog-probe", "serviceaccount.resources.gardener.cloud/namespace": "kube-system", }, Labels: map[string]string{ @@ -85,7 +117,7 @@ var _ = Describe("Access", func() { Data: map[string][]byte{"kubeconfig": []byte(`apiVersion: v1 clusters: - cluster: - server: https://` + serverOutOfCluster + ` + server: https://` + serverInCluster + ` name: ` + namespace + ` contexts: - context: @@ -101,75 +133,88 @@ users: `)}, } - expectedInternalProbeSecret = &corev1.Secret{ + expectedManagedResource = &resourcesv1alpha1.ManagedResource{ TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Secret", + APIVersion: resourcesv1alpha1.SchemeGroupVersion.String(), + Kind: "ManagedResource", }, ObjectMeta: metav1.ObjectMeta{ - Name: internalProbeSecretName, + Name: "shoot-core-dependency-watchdog", Namespace: namespace, + Labels: map[string]string{"origin": "gardener"}, + Annotations: map[string]string{"reference.resources.gardener.cloud/secret-2c6a3fae": "managedresource-shoot-core-dependency-watchdog-967328e8"}, ResourceVersion: "1", - Annotations: map[string]string{ - "serviceaccount.resources.gardener.cloud/name": "dependency-watchdog-internal-probe", - "serviceaccount.resources.gardener.cloud/namespace": "kube-system", + }, + Spec: resourcesv1alpha1.ManagedResourceSpec{ + SecretRefs: []corev1.LocalObjectReference{ + { + Name: "managedresource-shoot-core-dependency-watchdog-967328e8", + }, }, + InjectLabels: map[string]string{"shoot.gardener.cloud/no-cleanup": "true"}, + KeepObjects: pointer.Bool(false), + }, + } + expectedManagedResourceSecret = &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "managedresource-shoot-core-dependency-watchdog-967328e8", + Namespace: namespace, + ResourceVersion: "1", Labels: map[string]string{ - "resources.gardener.cloud/purpose": "token-requestor", - "resources.gardener.cloud/class": "shoot", + "resources.gardener.cloud/garbage-collectable-reference": "true", }, }, Type: corev1.SecretTypeOpaque, - Data: map[string][]byte{"kubeconfig": []byte(`apiVersion: v1 -clusters: -- cluster: - server: https://` + serverInCluster + ` - name: ` + namespace + ` -contexts: -- context: - cluster: ` + namespace + ` - user: ` + namespace + ` - name: ` + namespace + ` -current-context: ` + namespace + ` -kind: Config -preferences: {} -users: -- name: ` + namespace + ` - user: {} -`)}, + Data: map[string][]byte{ + "role__kube-node-lease__gardener.cloud_target_dependency-watchdog.yaml": []byte(roleYAML), + "rolebinding__kube-node-lease__gardener.cloud_target_dependency-watchdog.yaml": []byte(roleBindingYAML), + }, + Immutable: pointer.Bool(true), } }) AfterEach(func() { - Expect(fakeClient.Delete(ctx, expectedExternalProbeSecret)).To(Or(Succeed(), BeNotFoundError())) - Expect(fakeClient.Delete(ctx, expectedInternalProbeSecret)).To(Or(Succeed(), BeNotFoundError())) + Expect(fakeClient.Delete(ctx, expectedProbeSecret)).To(Or(Succeed(), BeNotFoundError())) + Expect(fakeClient.Delete(ctx, expectedManagedResourceSecret)).To(Or(Succeed(), BeNotFoundError())) + Expect(fakeClient.Delete(ctx, expectedManagedResource)).To(Or(Succeed(), BeNotFoundError())) }) Describe("#Deploy", func() { It("should successfully deploy all resources", func() { Expect(access.Deploy(ctx)).To(Succeed()) - reconciledExternalProbeSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: externalProbeSecretName, Namespace: namespace}} - Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(reconciledExternalProbeSecret), reconciledExternalProbeSecret)).To(Succeed()) - Expect(reconciledExternalProbeSecret).To(DeepEqual(expectedExternalProbeSecret)) - - reconciledInternalProbeSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: internalProbeSecretName, Namespace: namespace}} + reconciledInternalProbeSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: probeSecretName, Namespace: namespace}} Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(reconciledInternalProbeSecret), reconciledInternalProbeSecret)).To(Succeed()) - Expect(reconciledInternalProbeSecret).To(DeepEqual(expectedInternalProbeSecret)) + Expect(reconciledInternalProbeSecret).To(DeepEqual(expectedProbeSecret)) + + reconciledManagedResource := &resourcesv1alpha1.ManagedResource{ObjectMeta: metav1.ObjectMeta{Name: "shoot-core-dependency-watchdog", Namespace: namespace}} + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(reconciledManagedResource), reconciledManagedResource)).To(Succeed()) + Expect(reconciledManagedResource).To(DeepEqual(expectedManagedResource)) + + reconciledManagedResourceSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "managedresource-shoot-core-dependency-watchdog-967328e8", Namespace: namespace}} + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(reconciledManagedResourceSecret), reconciledManagedResourceSecret)).To(Succeed()) + Expect(reconciledManagedResourceSecret).To(DeepEqual(expectedManagedResourceSecret)) }) }) Describe("#Destroy", func() { It("should delete the secrets", func() { - reconciledExternalProbeSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: externalProbeSecretName, Namespace: namespace}} - Expect(fakeClient.Create(ctx, reconciledExternalProbeSecret)).To(Succeed()) - reconciledInternalProbeSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: internalProbeSecretName, Namespace: namespace}} + reconciledInternalProbeSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: probeSecretName, Namespace: namespace}} Expect(fakeClient.Create(ctx, reconciledInternalProbeSecret)).To(Succeed()) + reconciledManagedResource := &resourcesv1alpha1.ManagedResource{ObjectMeta: metav1.ObjectMeta{Name: "shoot-core-dependency-watchdog", Namespace: namespace}} + Expect(fakeClient.Create(ctx, reconciledManagedResource)).To(Succeed()) + reconciledManagedResourceSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "managedresource-shoot-core-dependency-watchdog-967328e8", Namespace: namespace}} + Expect(fakeClient.Create(ctx, reconciledManagedResourceSecret)).To(Succeed()) Expect(access.Destroy(ctx)).To(Succeed()) - Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(reconciledExternalProbeSecret), &corev1.Secret{})).To(BeNotFoundError()) Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(reconciledInternalProbeSecret), &corev1.Secret{})).To(BeNotFoundError()) + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(reconciledManagedResource), &resourcesv1alpha1.ManagedResource{})).To(BeNotFoundError()) + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(reconciledManagedResourceSecret), &resourcesv1alpha1.ManagedResource{})).To(BeNotFoundError()) }) }) }) diff --git a/pkg/component/dependencywatchdog/bootstrap_test.go b/pkg/component/dependencywatchdog/bootstrap_test.go index e322efd89ac..e8431ef6ea2 100644 --- a/pkg/component/dependencywatchdog/bootstrap_test.go +++ b/pkg/component/dependencywatchdog/bootstrap_test.go @@ -259,8 +259,7 @@ data: if role == RoleProber { out += ` dependentResourceInfos: null - externalKubeConfigSecretName: "" - internalKubeConfigSecretName: "" + kubeConfigSecretName: "" ` } @@ -540,7 +539,7 @@ spec: }) Describe("RoleProber", func() { - testSuite(BootstrapperValues{Role: RoleProber, Image: image}, "bad3c18c") + testSuite(BootstrapperValues{Role: RoleProber, Image: image}, "3c10a163") }) }) diff --git a/pkg/component/kubecontrollermanager/kube_controller_manager.go b/pkg/component/kubecontrollermanager/kube_controller_manager.go index b3659872188..b06b497caca 100644 --- a/pkg/component/kubecontrollermanager/kube_controller_manager.go +++ b/pkg/component/kubecontrollermanager/kube_controller_manager.go @@ -77,6 +77,10 @@ const ( volumeMountPathCAKubelet = "/srv/kubernetes/ca-kubelet" volumeMountPathServiceAccountKey = "/srv/kubernetes/service-account-key" volumeMountPathServer = "/var/lib/kube-controller-manager-server" + + nodeMonitorGraceDuration = 2 * time.Minute + // NodeMonitorGraceDurationK8sGreaterEqual127 is the default node monitoring grace duration used with k8s versions >= 1.27 + NodeMonitorGraceDurationK8sGreaterEqual127 = 40 * time.Second ) // Interface contains functions for a kube-controller-manager deployer. @@ -635,7 +639,7 @@ func (k *kubeControllerManager) computeCommand(port int32) []string { var ( defaultHorizontalPodAutoscalerConfig = k.getHorizontalPodAutoscalerConfig() podEvictionTimeout = metav1.Duration{Duration: 2 * time.Minute} - nodeMonitorGracePeriod = metav1.Duration{Duration: 2 * time.Minute} + nodeMonitorGracePeriod = metav1.Duration{Duration: nodeMonitorGraceDuration} command = []string{ "/usr/local/bin/kube-controller-manager", "--authentication-kubeconfig=" + gardenerutils.PathGenericKubeconfig, @@ -648,7 +652,7 @@ func (k *kubeControllerManager) computeCommand(port int32) []string { ) if versionutils.ConstraintK8sGreaterEqual127.Check(k.values.TargetVersion) { - nodeMonitorGracePeriod = metav1.Duration{Duration: 40 * time.Second} + nodeMonitorGracePeriod = metav1.Duration{Duration: NodeMonitorGraceDurationK8sGreaterEqual127} } if !k.values.IsWorkerless { diff --git a/pkg/gardenlet/controller/seed/seed/components.go b/pkg/gardenlet/controller/seed/seed/components.go index cca6e221a31..7611dd54f48 100644 --- a/pkg/gardenlet/controller/seed/seed/components.go +++ b/pkg/gardenlet/controller/seed/seed/components.go @@ -421,10 +421,9 @@ func (r *Reconciler) newDependencyWatchdogs(seedSettings *gardencorev1beta1.Seed kubeapiserver.NewDependencyWatchdogProberConfiguration, } dependencyWatchdogProberConfiguration = proberapi.Config{ - InternalKubeConfigSecretName: dependencywatchdog.InternalProbeSecretName, - ExternalKubeConfigSecretName: dependencywatchdog.ExternalProbeSecretName, - ProbeInterval: &metav1.Duration{Duration: dependencywatchdog.DefaultProbeInterval}, - DependentResourceInfos: make([]proberapi.DependentResourceInfo, 0, len(dependencyWatchdogProberConfigurationFuncs)), + KubeConfigSecretName: dependencywatchdog.KubeConfigSecretName, + ProbeInterval: &metav1.Duration{Duration: dependencywatchdog.DefaultProbeInterval}, + DependentResourceInfos: make([]proberapi.DependentResourceInfo, 0, len(dependencyWatchdogProberConfigurationFuncs)), } ) diff --git a/pkg/operation/botanist/dependency_watchdog.go b/pkg/operation/botanist/dependency_watchdog.go index 8c3c66c7c10..4ae72bd4b07 100644 --- a/pkg/operation/botanist/dependency_watchdog.go +++ b/pkg/operation/botanist/dependency_watchdog.go @@ -30,8 +30,7 @@ func (b *Botanist) DefaultDependencyWatchdogAccess() component.Deployer { b.Shoot.SeedNamespace, b.SecretsManager, dependencywatchdog.AccessValues{ - ServerInCluster: b.Shoot.ComputeInClusterAPIServerAddress(false), - ServerOutOfCluster: b.Shoot.ComputeOutOfClusterAPIServerAddress(true), + ServerInCluster: b.Shoot.ComputeInClusterAPIServerAddress(false), }, ) } diff --git a/pkg/operation/botanist/resource_manager.go b/pkg/operation/botanist/resource_manager.go index d160d601694..baf6f7b5381 100644 --- a/pkg/operation/botanist/resource_manager.go +++ b/pkg/operation/botanist/resource_manager.go @@ -18,6 +18,7 @@ import ( "context" "github.com/Masterminds/semver/v3" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" @@ -63,7 +64,7 @@ func (b *Botanist) DefaultResourceManager() (resourcemanager.Interface, error) { b.Shoot.TopologyAwareRoutingEnabled, ptr.To(b.Shoot.ComputeOutOfClusterAPIServerAddress(true)), b.Shoot.IsWorkerless, - []string{metav1.NamespaceSystem, v1beta1constants.KubernetesDashboardNamespace}, + []string{metav1.NamespaceSystem, v1beta1constants.KubernetesDashboardNamespace, corev1.NamespaceNodeLease}, ) }