From 46d8d6b8a90dd7031c880fdbeb3cc009c5f4a50c Mon Sep 17 00:00:00 2001 From: Rafael Franzke Date: Fri, 13 Oct 2023 12:46:33 +0200 Subject: [PATCH] [node-agent] Introduce `Node` controller (#8632) * Documentation * Controller + Reconciler incl. business logic * Integration test * Address PR review feedback --- cmd/gardener-node-agent/app/app.go | 38 ++++++ docs/concepts/node-agent.md | 10 ++ pkg/nodeagent/controller/add.go | 5 + pkg/nodeagent/controller/node/add.go | 69 +++++++++++ pkg/nodeagent/controller/node/add_test.go | 88 ++++++++++++++ .../controller/node/node_suite_test.go | 27 +++++ pkg/nodeagent/controller/node/reconciler.go | 100 +++++++++++++++ pkg/nodeagent/dbus/dbus.go | 15 +-- pkg/nodeagent/dbus/fake/fake_dbus.go | 114 ++++++++++++++++++ .../nodeagent/node/node_suite_test.go | 73 +++++++++++ test/integration/nodeagent/node/node_test.go | 114 ++++++++++++++++++ 11 files changed, 646 insertions(+), 7 deletions(-) create mode 100644 pkg/nodeagent/controller/node/add.go create mode 100644 pkg/nodeagent/controller/node/add_test.go create mode 100644 pkg/nodeagent/controller/node/node_suite_test.go create mode 100644 pkg/nodeagent/controller/node/reconciler.go create mode 100644 pkg/nodeagent/dbus/fake/fake_dbus.go create mode 100644 test/integration/nodeagent/node/node_suite_test.go create mode 100644 test/integration/nodeagent/node/node_test.go diff --git a/cmd/gardener-node-agent/app/app.go b/cmd/gardener-node-agent/app/app.go index b0de270cd8a..f106de8fa71 100644 --- a/cmd/gardener-node-agent/app/app.go +++ b/cmd/gardener-node-agent/app/app.go @@ -29,6 +29,7 @@ import ( "github.com/spf13/pflag" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" clientcmdlatest "k8s.io/client-go/tools/clientcmd/api/latest" @@ -138,6 +139,12 @@ func run(ctx context.Context, log logr.Logger, cfg *config.NodeAgentConfiguratio } } + log.Info("Fetching node name based on hostname") + nodeName, err := getNodeName(ctx, log, restConfig) + if err != nil { + return err + } + log.Info("Setting up manager") mgr, err := manager.New(restConfig, manager.Options{ Logger: log, @@ -155,6 +162,9 @@ func run(ctx context.Context, log logr.Logger, cfg *config.NodeAgentConfiguratio &corev1.Secret{}: { Namespaces: map[string]cache.Config{metav1.NamespaceSystem: {}}, }, + &corev1.Node{}: { + Field: fields.SelectorFromSet(fields.Set{metav1.ObjectNameField: nodeName}), + }, }, }, LeaderElection: false, @@ -224,3 +234,31 @@ func fetchAccessTokenViaBootstrapToken(ctx context.Context, log logr.Logger, res return nil } + +func getNodeName(ctx context.Context, log logr.Logger, restConfig *rest.Config) (string, error) { + hostname, err := os.Hostname() + if err != nil { + return "", fmt.Errorf("failed fetching hostname: %w", err) + } + + cl, err := client.New(restConfig, client.Options{}) + if err != nil { + return "", fmt.Errorf("unable to create client: %w", err) + } + + nodeList := &metav1.PartialObjectMetadataList{} + nodeList.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("NodeList")) + if err := cl.List(ctx, nodeList, client.MatchingLabels{corev1.LabelHostname: hostname}); err != nil { + return "", err + } + + switch len(nodeList.Items) { + case 0: + return "", fmt.Errorf("could not find any node with label %s=%s", corev1.LabelHostname, hostname) + case 1: + log.Info("Found node name based on hostname", "hostname", hostname, "nodeName", nodeList.Items[0].Name) + return nodeList.Items[0].Name, nil + default: + return "", fmt.Errorf("found more than one node with label %s=%s", corev1.LabelHostname, hostname) + } +} diff --git a/docs/concepts/node-agent.md b/docs/concepts/node-agent.md index a9164fb9a7f..994d441e864 100644 --- a/docs/concepts/node-agent.md +++ b/docs/concepts/node-agent.md @@ -28,6 +28,16 @@ In a bootstrapping phase, the `gardener-node-agent` sets itself up as a systemd This section describes the controllers in more details. +### [`Node` Controller](../../pkg/nodeagent/controller/node) + +This controller watches the `Node` object for the machine it runs on. +The correct `Node` is identified based on the hostname of the machine (`Node`s have the `kubernetes.io/hostname` label). +Whenever the `worker.gardener.cloud/restart-systemd-services` annotation changes, the controller performs the desired changes by restarting the specified systemd unit files. +See also [this document](../usage/shoot_operations.md#restart-systemd-services-on-particular-worker-nodes) for more information. +After restarting all units, the annotation is removed. + +> ℹ️ When the `gardener-node-agent` systemd service itself is requested to be restarted, the annotation is removed first to ensure it does not restart itself indefinitely. + ### [Token Controller](../../pkg/nodeagent/controller/token) This controller watches the access token `Secret` in the `kube-system` namespace whose name is provided via the `gardener-node-agent`'s component configuration (`.accessTokenSecret` field). diff --git a/pkg/nodeagent/controller/add.go b/pkg/nodeagent/controller/add.go index 19827a9285b..98579daf6d5 100644 --- a/pkg/nodeagent/controller/add.go +++ b/pkg/nodeagent/controller/add.go @@ -20,11 +20,16 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/gardener/gardener/pkg/nodeagent/apis/config" + "github.com/gardener/gardener/pkg/nodeagent/controller/node" "github.com/gardener/gardener/pkg/nodeagent/controller/token" ) // AddToManager adds all controllers to the given manager. func AddToManager(mgr manager.Manager, cfg *config.NodeAgentConfiguration) error { + if err := (&node.Reconciler{}).AddToManager(mgr); err != nil { + return fmt.Errorf("failed adding node controller: %w", err) + } + if err := (&token.Reconciler{ AccessTokenSecretName: cfg.AccessTokenSecretName, }).AddToManager(mgr); err != nil { diff --git a/pkg/nodeagent/controller/node/add.go b/pkg/nodeagent/controller/node/add.go new file mode 100644 index 00000000000..2fa22efb237 --- /dev/null +++ b/pkg/nodeagent/controller/node/add.go @@ -0,0 +1,69 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package node + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + "github.com/gardener/gardener/pkg/nodeagent/dbus" +) + +// ControllerName is the name of this controller. +const ControllerName = "node" + +// AddToManager adds Reconciler to the given manager. +func (r *Reconciler) AddToManager(mgr manager.Manager) error { + if r.Client == nil { + r.Client = mgr.GetClient() + } + if r.Recorder == nil { + r.Recorder = mgr.GetEventRecorderFor(ControllerName) + } + if r.DBus == nil { + r.DBus = dbus.New() + } + + node := &metav1.PartialObjectMetadata{} + node.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Node")) + + return builder. + ControllerManagedBy(mgr). + Named(ControllerName). + For(node, builder.WithPredicates(r.NodePredicate())). + WithOptions(controller.Options{MaxConcurrentReconciles: 1}). + Complete(r) +} + +// NodePredicate returns 'true' when the annotation describing which systemd services should be restarted gets set or +// changed. When it's removed, 'false' is returned. +func (r *Reconciler) NodePredicate() predicate.Predicate { + return predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + return e.Object.GetAnnotations()[annotationRestartSystemdServices] != "" + }, + UpdateFunc: func(e event.UpdateEvent) bool { + return e.ObjectOld.GetAnnotations()[annotationRestartSystemdServices] != e.ObjectNew.GetAnnotations()[annotationRestartSystemdServices] && + e.ObjectNew.GetAnnotations()[annotationRestartSystemdServices] != "" + }, + DeleteFunc: func(_ event.DeleteEvent) bool { return false }, + GenericFunc: func(_ event.GenericEvent) bool { return false }, + } +} diff --git a/pkg/nodeagent/controller/node/add_test.go b/pkg/nodeagent/controller/node/add_test.go new file mode 100644 index 00000000000..0eecab8ec5f --- /dev/null +++ b/pkg/nodeagent/controller/node/add_test.go @@ -0,0 +1,88 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package node_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + . "github.com/gardener/gardener/pkg/nodeagent/controller/node" +) + +var _ = Describe("Add", func() { + Describe("#NodePredicate", func() { + var ( + p predicate.Predicate + node *corev1.Node + ) + + BeforeEach(func() { + p = (&Reconciler{}).NodePredicate() + node = &corev1.Node{} + }) + + Describe("#Create", func() { + It("should return false because annotation is not present", func() { + Expect(p.Create(event.CreateEvent{Object: node})).To(BeFalse()) + }) + + It("should return true because annotation is present", func() { + metav1.SetMetaDataAnnotation(&node.ObjectMeta, "worker.gardener.cloud/restart-systemd-services", "foo") + Expect(p.Create(event.CreateEvent{Object: node})).To(BeTrue()) + }) + }) + + Describe("#Update", func() { + It("should return false because annotation is not present", func() { + Expect(p.Update(event.UpdateEvent{ObjectOld: node, ObjectNew: node})).To(BeFalse()) + }) + + It("should return true because annotation got set", func() { + oldNode := node.DeepCopy() + metav1.SetMetaDataAnnotation(&node.ObjectMeta, "worker.gardener.cloud/restart-systemd-services", "foo") + Expect(p.Update(event.UpdateEvent{ObjectOld: oldNode, ObjectNew: node})).To(BeTrue()) + }) + + It("should return true because annotation got changed", func() { + metav1.SetMetaDataAnnotation(&node.ObjectMeta, "worker.gardener.cloud/restart-systemd-services", "foo") + oldNode := node.DeepCopy() + metav1.SetMetaDataAnnotation(&node.ObjectMeta, "worker.gardener.cloud/restart-systemd-services", "bar") + Expect(p.Update(event.UpdateEvent{ObjectOld: oldNode, ObjectNew: node})).To(BeTrue()) + }) + + It("should return false because annotation got removed", func() { + oldNode := node.DeepCopy() + metav1.SetMetaDataAnnotation(&oldNode.ObjectMeta, "worker.gardener.cloud/restart-systemd-services", "foo") + Expect(p.Update(event.UpdateEvent{ObjectOld: oldNode, ObjectNew: node})).To(BeFalse()) + }) + }) + + Describe("#Delete", func() { + It("should return false", func() { + Expect(p.Delete(event.DeleteEvent{})).To(BeFalse()) + }) + }) + + Describe("#Generic", func() { + It("should return false", func() { + Expect(p.Generic(event.GenericEvent{})).To(BeFalse()) + }) + }) + }) +}) diff --git a/pkg/nodeagent/controller/node/node_suite_test.go b/pkg/nodeagent/controller/node/node_suite_test.go new file mode 100644 index 00000000000..773143f5ef6 --- /dev/null +++ b/pkg/nodeagent/controller/node/node_suite_test.go @@ -0,0 +1,27 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package node_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestNode(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "NodeAgent Controller Node Suite") +} diff --git a/pkg/nodeagent/controller/node/reconciler.go b/pkg/nodeagent/controller/node/reconciler.go new file mode 100644 index 00000000000..a5912c102fb --- /dev/null +++ b/pkg/nodeagent/controller/node/reconciler.go @@ -0,0 +1,100 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package node + +import ( + "context" + "fmt" + "strings" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/gardener/gardener/pkg/controllerutils" + nodeagentv1alpha1 "github.com/gardener/gardener/pkg/nodeagent/apis/config/v1alpha1" + "github.com/gardener/gardener/pkg/nodeagent/dbus" +) + +const annotationRestartSystemdServices = "worker.gardener.cloud/restart-systemd-services" + +// Reconciler checks for node annotation changes and restarts the specified systemd services. +type Reconciler struct { + Client client.Client + Recorder record.EventRecorder + DBus dbus.DBus +} + +// Reconcile checks for node annotation changes and restarts the specified systemd services. +func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + log := logf.FromContext(ctx) + + ctx, cancel := controllerutils.GetMainReconciliationContext(ctx, controllerutils.DefaultReconciliationTimeout) + defer cancel() + + node := &metav1.PartialObjectMetadata{} + node.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Node")) + if err := r.Client.Get(ctx, request.NamespacedName, node); err != nil { + if apierrors.IsNotFound(err) { + log.V(1).Info("Object is gone, stop reconciling") + return reconcile.Result{}, nil + } + return reconcile.Result{}, fmt.Errorf("error retrieving object from store: %w", err) + } + + services, ok := node.Annotations[annotationRestartSystemdServices] + if !ok { + return reconcile.Result{}, nil + } + + var restartGardenerNodeAgent bool + for _, serviceName := range strings.Split(services, ",") { + // If the gardener-node-agent itself should be restarted, we have to first remove the annotation from the node. + // Otherwise, the annotation is never removed and it restarts itself indefinitely. + if strings.HasPrefix(serviceName, "gardener-node-agent") { + restartGardenerNodeAgent = true + continue + } + r.restartService(ctx, log, node, serviceName) + } + + log.Info("Removing annotation from node", "annotation", annotationRestartSystemdServices) + patch := client.MergeFrom(node.DeepCopy()) + delete(node.Annotations, annotationRestartSystemdServices) + if err := r.Client.Patch(ctx, node, patch); err != nil { + return reconcile.Result{}, err + } + + if restartGardenerNodeAgent { + r.restartService(ctx, log, node, nodeagentv1alpha1.UnitName) + } + + return reconcile.Result{}, nil +} + +func (r *Reconciler) restartService(ctx context.Context, log logr.Logger, node client.Object, serviceName string) { + log.Info("Restarting systemd service", "serviceName", serviceName) + if err := r.DBus.Restart(ctx, r.Recorder, node, serviceName); err != nil { + // We don't return the error here since we don't want to repeatedly try to restart services again and again. + // In both cases (success or failure), an event will be recorded on the Node so that users can check whether + // the restart worked. + log.Error(err, "Failed restarting systemd service", "serviceName", serviceName) + } +} diff --git a/pkg/nodeagent/dbus/dbus.go b/pkg/nodeagent/dbus/dbus.go index b1e90e8a8e8..420e30153e8 100644 --- a/pkg/nodeagent/dbus/dbus.go +++ b/pkg/nodeagent/dbus/dbus.go @@ -20,6 +20,7 @@ import ( "github.com/coreos/go-systemd/v22/dbus" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/record" ) @@ -34,11 +35,11 @@ type DBus interface { // Disable the given units, same as executing "systemctl disable unit". Disable(ctx context.Context, unitNames ...string) error // Start the given unit and record an event to the node object, same as executing "systemctl start unit". - Start(ctx context.Context, recorder record.EventRecorder, node *corev1.Node, unitName string) error + Start(ctx context.Context, recorder record.EventRecorder, node runtime.Object, unitName string) error // Stop the given unit and record an event to the node object, same as executing "systemctl stop unit". - Stop(ctx context.Context, recorder record.EventRecorder, node *corev1.Node, unitName string) error + Stop(ctx context.Context, recorder record.EventRecorder, node runtime.Object, unitName string) error // Restart the given unit and record an event to the node object, same as executing "systemctl restart unit". - Restart(ctx context.Context, recorder record.EventRecorder, node *corev1.Node, unitName string) error + Restart(ctx context.Context, recorder record.EventRecorder, node runtime.Object, unitName string) error } type db struct{} @@ -70,7 +71,7 @@ func (_ *db) Disable(ctx context.Context, unitNames ...string) error { return err } -func (_ *db) Stop(ctx context.Context, recorder record.EventRecorder, node *corev1.Node, unitName string) error { +func (_ *db) Stop(ctx context.Context, recorder record.EventRecorder, node runtime.Object, unitName string) error { dbc, err := dbus.NewWithContext(ctx) if err != nil { return fmt.Errorf("unable to connect to dbus: %w", err) @@ -91,7 +92,7 @@ func (_ *db) Stop(ctx context.Context, recorder record.EventRecorder, node *core return err } -func (_ *db) Start(ctx context.Context, recorder record.EventRecorder, node *corev1.Node, unitName string) error { +func (_ *db) Start(ctx context.Context, recorder record.EventRecorder, node runtime.Object, unitName string) error { dbc, err := dbus.NewWithContext(ctx) if err != nil { return fmt.Errorf("unable to connect to dbus: %w", err) @@ -113,7 +114,7 @@ func (_ *db) Start(ctx context.Context, recorder record.EventRecorder, node *cor return err } -func (_ *db) Restart(ctx context.Context, recorder record.EventRecorder, node *corev1.Node, unitName string) error { +func (_ *db) Restart(ctx context.Context, recorder record.EventRecorder, node runtime.Object, unitName string) error { dbc, err := dbus.NewWithContext(ctx) if err != nil { return fmt.Errorf("unable to connect to dbus: %w", err) @@ -149,7 +150,7 @@ func (_ *db) DaemonReload(ctx context.Context) error { return nil } -func recordEvent(recorder record.EventRecorder, node *corev1.Node, err error, unitName, reason, operation string) { +func recordEvent(recorder record.EventRecorder, node runtime.Object, err error, unitName, reason, operation string) { if recorder != nil && node != nil { var ( eventType = corev1.EventTypeNormal diff --git a/pkg/nodeagent/dbus/fake/fake_dbus.go b/pkg/nodeagent/dbus/fake/fake_dbus.go new file mode 100644 index 00000000000..98e4b47a5f0 --- /dev/null +++ b/pkg/nodeagent/dbus/fake/fake_dbus.go @@ -0,0 +1,114 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fake + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + + "github.com/gardener/gardener/pkg/nodeagent/dbus" +) + +// Action is an int type alias. +type Action int + +const ( + // ActionDaemonReload is constant for the 'DaemonReload' action. + ActionDaemonReload Action = iota + // ActionDisable is constant for the 'Disable' action. + ActionDisable + // ActionEnable is constant for the 'Enable' action. + ActionEnable + // ActionRestart is constant for the 'Restart' action. + ActionRestart + // ActionStart is constant for the 'Start' action. + ActionStart + // ActionStop is constant for the 'Stop' action. + ActionStop +) + +// SystemdAction is used for the implementation of the fake dbus. +type SystemdAction struct { + Action Action + UnitNames []string +} + +// DBus is a fake implementation for the dbus.DBus interface. +type DBus struct { + Actions []SystemdAction +} + +var _ dbus.DBus = &DBus{} + +// New returns a simple implementation of dbus.DBus which can be used to fake the dbus actions in unit tests. +func New() *DBus { + return &DBus{} +} + +// DaemonReload implements dbus.DBus. +func (d *DBus) DaemonReload(_ context.Context) error { + d.Actions = append(d.Actions, SystemdAction{ + Action: ActionDaemonReload, + }) + return nil +} + +// Disable implements dbus.DBus. +func (d *DBus) Disable(_ context.Context, unitNames ...string) error { + d.Actions = append(d.Actions, SystemdAction{ + Action: ActionDisable, + UnitNames: unitNames, + }) + return nil +} + +// Enable implements dbus.DBus. +func (d *DBus) Enable(_ context.Context, unitNames ...string) error { + d.Actions = append(d.Actions, SystemdAction{ + Action: ActionEnable, + UnitNames: unitNames, + }) + + return nil +} + +// Restart implements dbus.DBus. +func (d *DBus) Restart(_ context.Context, _ record.EventRecorder, _ runtime.Object, unitName string) error { + d.Actions = append(d.Actions, SystemdAction{ + Action: ActionRestart, + UnitNames: []string{unitName}, + }) + return nil +} + +// Start implements dbus.DBus. +func (d *DBus) Start(_ context.Context, _ record.EventRecorder, _ runtime.Object, unitName string) error { + d.Actions = append(d.Actions, SystemdAction{ + Action: ActionStart, + UnitNames: []string{unitName}, + }) + return nil +} + +// Stop implements dbus.DBus. +func (d *DBus) Stop(_ context.Context, _ record.EventRecorder, _ runtime.Object, unitName string) error { + d.Actions = append(d.Actions, SystemdAction{ + Action: ActionStop, + UnitNames: []string{unitName}, + }) + return nil +} diff --git a/test/integration/nodeagent/node/node_suite_test.go b/test/integration/nodeagent/node/node_suite_test.go new file mode 100644 index 00000000000..5cb9db4f7a4 --- /dev/null +++ b/test/integration/nodeagent/node/node_suite_test.go @@ -0,0 +1,73 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package node_test + +import ( + "context" + "testing" + + "github.com/go-logr/logr" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "github.com/gardener/gardener/pkg/logger" + gardenerutils "github.com/gardener/gardener/pkg/utils" +) + +func TestNode(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Test Integration NodeAgent Node Suite") +} + +const testID = "node-controller-test" + +var ( + ctx = context.Background() + log logr.Logger + + restConfig *rest.Config + testEnv *envtest.Environment + testClient client.Client + + testRunID = "test-" + gardenerutils.ComputeSHA256Hex([]byte(uuid.NewUUID()))[:8] +) + +var _ = BeforeSuite(func() { + logf.SetLogger(logger.MustNewZapLogger(logger.DebugLevel, logger.FormatJSON, zap.WriteTo(GinkgoWriter))) + log = logf.Log.WithName(testID) + + By("Start test environment") + testEnv = &envtest.Environment{} + + var err error + restConfig, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(restConfig).NotTo(BeNil()) + + DeferCleanup(func() { + By("Stop test environment") + Expect(testEnv.Stop()).To(Succeed()) + }) + + By("Create test client") + testClient, err = client.New(restConfig, client.Options{}) + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/test/integration/nodeagent/node/node_test.go b/test/integration/nodeagent/node/node_test.go new file mode 100644 index 00000000000..b13c85b81ae --- /dev/null +++ b/test/integration/nodeagent/node/node_test.go @@ -0,0 +1,114 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package node_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + nodecontroller "github.com/gardener/gardener/pkg/nodeagent/controller/node" + "github.com/gardener/gardener/pkg/nodeagent/dbus/fake" +) + +var _ = Describe("Node controller tests", func() { + var ( + fakeDBus *fake.DBus + nodeName = testRunID + node *corev1.Node + ) + + BeforeEach(func() { + By("Setup manager") + mgr, err := manager.New(restConfig, manager.Options{ + Metrics: metricsserver.Options{BindAddress: "0"}, + Cache: cache.Options{ + DefaultLabelSelector: labels.SelectorFromSet(labels.Set{testID: testRunID}), + }, + }) + Expect(err).NotTo(HaveOccurred()) + + By("Register controller") + fakeDBus = fake.New() + Expect((&nodecontroller.Reconciler{ + DBus: fakeDBus, + }).AddToManager(mgr)).To(Succeed()) + + By("Start manager") + mgrContext, mgrCancel := context.WithCancel(ctx) + + go func() { + defer GinkgoRecover() + Expect(mgr.Start(mgrContext)).To(Succeed()) + }() + + DeferCleanup(func() { + By("Stop manager") + mgrCancel() + }) + + node = &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Labels: map[string]string{testID: testRunID}, + }, + } + + By("Create Node") + Expect(testClient.Create(ctx, node)).To(Succeed()) + DeferCleanup(func() { + By("Delete Node") + Expect(testClient.Delete(ctx, node)).To(Succeed()) + }) + }) + + It("should do nothing because node has no restart annotation", func() { + Consistently(func() []fake.SystemdAction { + return fakeDBus.Actions + }).Should(BeEmpty()) + }) + + It("should restart the systemd services specified in the restart annotation", func() { + By("Adding restart annotation to node") + svc1, svc2, svc3 := "gardener-node-agent", "foo", "bar" + metav1.SetMetaDataAnnotation(&node.ObjectMeta, "worker.gardener.cloud/restart-systemd-services", svc1+","+svc2+","+svc3) + Expect(testClient.Update(ctx, node)).To(Succeed()) + + By("Wait for restart annotation to disappear") + Eventually(func(g Gomega) map[string]string { + g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(node), node)).To(Succeed()) + return node.Annotations + }).ShouldNot(HaveKey("worker.gardener.cloud/restart-systemd-services")) + + By("Assert that the systemd services were restarted") + Eventually(func(g Gomega) { + g.Expect(fakeDBus.Actions).To(HaveLen(3)) + g.Expect(fakeDBus.Actions[0].Action).To(Equal(fake.ActionRestart)) + g.Expect(fakeDBus.Actions[0].UnitNames).To(ConsistOf(svc2)) + g.Expect(fakeDBus.Actions[1].Action).To(Equal(fake.ActionRestart)) + g.Expect(fakeDBus.Actions[1].UnitNames).To(ConsistOf(svc3)) + g.Expect(fakeDBus.Actions[2].Action).To(Equal(fake.ActionRestart)) + g.Expect(fakeDBus.Actions[2].UnitNames).To(ConsistOf(svc1 + ".service")) + }).Should(Succeed()) + }) +})