Add NRT garbage collector

Signed-off-by: PiotrProkop <pprokop@nvidia.com>

cmd/nfd-topology-updater/main.go

@@ -125,6 +125,10 @@ func initFlags(flagset *flag.FlagSet) (*topology.Args, *resourcemonitor.Args) {
 	args := &topology.Args{}
 	resourcemonitorArgs := &resourcemonitor.Args{}
 
+	flagset.BoolVar(&args.GCEnabled, "gc-enabled", false,
+		"Run NodeResourceTopology Garbage Collector")
+	flagset.DurationVar(&args.GCPeriod, "gc-interval", time.Duration(10)*time.Minute,
+		"Interval between which Garbage Collector will try to cleanup any missed but already obsolete NodeResourceTopology. [Default: 10m]")
 	flagset.BoolVar(&args.Oneshot, "oneshot", false,
 		"Update once and exit")
 	flagset.BoolVar(&args.NoPublish, "no-publish", false,

deployment/base/rbac-topologyupdater-gc/kustomization.yaml

@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: node-feature-discovery
+
+resources:
+- topologyupdater-gc-clusterrole.yaml
+- topologyupdater-gc-clusterrolebinding.yaml
+- topologyupdater-gc-serviceaccount.yaml

deployment/base/rbac-topologyupdater-gc/topologyupdater-gc-clusterrole.yaml

@@ -0,0 +1,25 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: nfd-topology-gc
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes/proxy
+  verbs:
+  - get
+- apiGroups:
+  - topology.node.k8s.io
+  resources:
+  - noderesourcetopologies
+  verbs:
+  - delete
+  - list

deployment/base/rbac-topologyupdater-gc/topologyupdater-gc-clusterrolebinding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: nfd-topology-gc
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: nfd-topology-gc
+subjects:
+- kind: ServiceAccount
+  name: nfd-topology-gc
+  namespace: default

deployment/base/rbac-topologyupdater-gc/topologyupdater-gc-serviceaccount.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: nfd-topology-gc

deployment/base/topologyupdater-gc/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: node-feature-discovery
+
+resources:
+- topologyupdater-gc.yaml

deployment/base/topologyupdater-gc/topologyupdater-gc.yaml

@@ -0,0 +1,25 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nfd
+  name: nfd-topology-gc
+spec:
+  selector:
+    matchLabels:
+      app: nfd-topology-gc
+  template:
+    metadata:
+      labels:
+        app: nfd-topology-gc
+    spec:
+      dnsPolicy: ClusterFirstWithHostNet
+      serviceAccount: nfd-topology-gc
+      containers:
+        - name: nfd-topology-gc
+          image: gcr.io/k8s-staging-nfd/node-feature-discovery:master
+          imagePullPolicy: Always
+          command:
+            - "nfd-topology-updater"
+            - "-gc-enabled"
+            - "-gc-interval=10s"

deployment/overlays/master-worker-topologyupdater/kustomization.yaml

@@ -6,11 +6,13 @@ namespace: node-feature-discovery
 bases:
 - ../../base/rbac
 - ../../base/rbac-topologyupdater
+- ../../base/rbac-topologyupdater-gc
 - ../../base/nfd-crds
 - ../../base/master
 - ../../base/worker-daemonset
 - ../../base/noderesourcetopologies-crd
 - ../../base/topologyupdater-daemonset
+- ../../base/topologyupdater-gc
 
 resources:
 - namespace.yaml

pkg/nfd-topology-updater/nfd-nrt-gc.go

@@ -0,0 +1,177 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nfdtopologyupdater
+
+import (
+	"context"
+	"time"
+
+	topologyclientset "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/generated/clientset/versioned"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/klog/v2"
+
+	"sigs.k8s.io/node-feature-discovery/pkg/apihelper"
+)
+
+type nrtGarbageCollector struct {
+	stopChan   chan struct{}
+	k8sClient  kubernetes.Interface
+	topoClient topologyclientset.Interface
+	gcPeriod   time.Duration
+}
+
+func newNRTGarbageCollector(config *restclient.Config, stop chan struct{}, gcPeriod time.Duration) (*nrtGarbageCollector, error) {
+	helper := apihelper.K8sHelpers{Kubeconfig: config}
+	cli, err := helper.GetTopologyClient()
+	if err != nil {
+		return nil, err
+	}
+
+	clientset := kubernetes.NewForConfigOrDie(config)
+
+	return &nrtGarbageCollector{
+		k8sClient:  clientset,
+		topoClient: cli,
+		stopChan:   stop,
+		gcPeriod:   gcPeriod,
+	}, nil
+}
+
+func (n *nrtGarbageCollector) deleteNodeHandler(object interface{}) {
+	// handle a case when we are starting up and need to clear stale NRT resources
+	obj := object
+	if deletedFinalStateUnknown, ok := object.(cache.DeletedFinalStateUnknown); ok {
+		klog.V(2).Infof("found stale NodeResourceTopology for node: %s ", object)
+		obj = deletedFinalStateUnknown.Obj
+	}
+
+	node, ok := obj.(*corev1.Node)
+	if !ok {
+		klog.Errorf("cannot convert %v to v1.Node", object)
+		return
+	}
+	if err := n.topoClient.TopologyV1alpha1().NodeResourceTopologies().Delete(context.TODO(), node.GetName(), metav1.DeleteOptions{}); err != nil {
+		if errors.IsNotFound(err) {
+			klog.V(2).Infof("NodeResourceTopology for node %s not found, omitting deletion", node.GetName())
+			return
+		} else {
+			klog.Warningf("failed to delete NodeResourceTopology for node %s: %s", node.GetName(), err.Error())
+			return
+		}
+	}
+	klog.Infof("NodeResourceTopology for node %s has been deleted", node.GetName())
+}
+
+// populateNodeIndexer populates cache with NRTs so we know which one to delete on first synchronization
+func (n *nrtGarbageCollector) populateNodeIndexer(indexer cache.Indexer) error {
+	nrts, err := n.topoClient.TopologyV1alpha1().NodeResourceTopologies().List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+
+	for _, nrt := range nrts.Items {
+		node := &corev1.Node{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: nrt.GetName(),
+			},
+		}
+		klog.V(2).Infof("adding node %s to indexer", node.GetName())
+		err := indexer.Add(node)
+		if err != nil {
+			return err
+		}
+
+	}
+
+	return nil
+}
+
+// periodicGC runs garbage collector at every gcPeriod to make sure we haven't missed any node
+func (n *nrtGarbageCollector) periodicGC(gcPeriod time.Duration, indexer cache.Indexer) {
+	gcTrigger := time.NewTicker(gcPeriod)
+	for {
+		select {
+		case <-gcTrigger.C:
+			klog.Infof("Running periodic GC")
+			objects := indexer.List()
+			nodes := map[string]struct{}{}
+			for _, object := range objects {
+				key, err := cache.MetaNamespaceKeyFunc(object)
+				if err != nil {
+					klog.Warningf("cannot create key for %v: %s", object, err.Error())
+					continue
+				}
+				nodes[key] = struct{}{}
+			}
+			nrts, err := n.topoClient.TopologyV1alpha1().NodeResourceTopologies().List(context.TODO(), metav1.ListOptions{})
+			if err != nil {
+				klog.Warningf("cannot list NRTs %s", err.Error())
+				break
+			}
+			for _, nrt := range nrts.Items {
+				key, err := cache.MetaNamespaceKeyFunc(&nrt)
+				if err != nil {
+					klog.Warningf("cannot create key for %v: %s", nrt, err.Error())
+					continue
+				}
+				if _, ok := nodes[key]; !ok {
+					if err := n.topoClient.TopologyV1alpha1().NodeResourceTopologies().Delete(context.TODO(), nrt.GetName(), metav1.DeleteOptions{}); err != nil {
+						if errors.IsNotFound(err) {
+							klog.V(2).Infof("NodeResourceTopology %s not found, omitting deletion", nrt.GetName())
+							continue
+						} else {
+							klog.Warningf("failed to delete NodeResourceTopology for node %s: %s", nrt.GetName(), err.Error())
+							continue
+						}
+					}
+				}
+			}
+		case <-n.stopChan:
+			klog.Infof("shutting down periodic Garbage Collector")
+			return
+		}
+	}
+}
+
+func (n *nrtGarbageCollector) start() error {
+	factory := informers.NewSharedInformerFactory(n.k8sClient, 5*time.Minute)
+	nodeInformer := factory.Core().V1().Nodes().Informer()
+
+	if _, err := nodeInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+		DeleteFunc: n.deleteNodeHandler,
+	}); err != nil {
+		return err
+	}
+
+	if err := n.populateNodeIndexer(nodeInformer.GetIndexer()); err != nil {
+		return err
+	}
+
+	factory.Start(n.stopChan)
+	// do initial cleanup
+	factory.WaitForCacheSync(n.stopChan)
+
+	go n.periodicGC(n.gcPeriod, nodeInformer.GetIndexer())
+
+	return nil
+}