From 089e759f31f2a4459a642e326d03cc236313efa4 Mon Sep 17 00:00:00 2001
From: vportella <vportella@atlassian.com>
Date: Wed, 20 Sep 2023 11:20:27 +1000
Subject: [PATCH] Add retries when managing the finalizer on the node

---
 pkg/controller/node.go | 51 ++++++++++++++++++++++--------------------
 1 file changed, 27 insertions(+), 24 deletions(-)

diff --git a/pkg/controller/node.go b/pkg/controller/node.go
index 5a62b5d..ababb3b 100644
--- a/pkg/controller/node.go
+++ b/pkg/controller/node.go
@@ -10,6 +10,7 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/util/retry"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -104,30 +105,32 @@ func (rm *ResourceManager) RemoveFinalizerFromNode(nodeName string) error {
 }
 
 func (rm *ResourceManager) manageFinalizerOnNode(nodeName string, fn func(*v1.Node, string, kubernetes.Interface) error) error {
-	// Get the node
-	node, err := rm.GetNode(nodeName)
-
-	// If the node is not found then skip the finalizer operation
-	if err != nil && apierrors.IsNotFound(err) {
-		rm.Logger.Info("node deleted, skip adding finalizer", "node", nodeName)
-		return nil
-	}
+	return retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		// Get the node
+		node, err := rm.GetNode(nodeName)
+
+		// If the node is not found then skip the finalizer operation
+		if err != nil && apierrors.IsNotFound(err) {
+			rm.Logger.Info("node deleted, skip adding finalizer", "node", nodeName)
+			return nil
+		}
+
+		// Account for any other possible errors
+		if err != nil {
+			return err
+		}
+
+		// The node exists as of the previous step, try managing the finalizer for
+		// it now
+		err = fn(node, nodeFinalizerName, rm.RawClient)
+
+		// Account for possible race conditions with other controllers managing
+		// nodes in the cluster
+		if apierrors.IsNotFound(err) {
+			rm.Logger.Info("updating finalizer failed, node already deleted", "node", nodeName)
+			return nil
+		}
 
-	// Account for any other possible errors
-	if err != nil {
 		return err
-	}
-
-	// The node exists as of the previous step, try managing the finalizer for
-	// it now
-	err = fn(node, nodeFinalizerName, rm.RawClient)
-
-	// Account for possible race conditions with other controllers managing
-	// nodes in the cluster
-	if apierrors.IsNotFound(err) {
-		rm.Logger.Info("adding finalizer failed, node already deleted", "node", nodeName)
-		return nil
-	}
-
-	return err
+	})
 }