Merge pull request rancher#41064 from rmweir/add-crb-wait

Add crb wait

tests/framework/extensions/kubeapi/namespaces/namespaces.go

@@ -1,8 +1,10 @@
 package namespaces
 
 import (
+	"context"
 	"fmt"
 
+	"github.com/rancher/rancher/pkg/api/scheme"
 	"github.com/rancher/rancher/tests/framework/clients/rancher"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -28,18 +30,21 @@ func ContainerDefaultResourceLimit(limitsCPU, limitsMemory, requestsCPU, request
 
 // GetNamespaceByName is a helper function that returns the namespace by name in a specific cluster, uses ListNamespaces to get the namespace.
 func GetNamespaceByName(client *rancher.Client, clusterID, namespaceName string) (*corev1.Namespace, error) {
-	var namespace *corev1.Namespace
+	namespace := new(corev1.Namespace)
 
-	namespaceList, err := ListNamespaces(client, clusterID, metav1.ListOptions{})
+	dynamicClient, err := client.GetDownStreamClusterClient(clusterID)
 	if err != nil {
 		return nil, err
 	}
 
-	for i, ns := range namespaceList.Items {
-		if namespaceName == ns.Name {
-			namespace = &namespaceList.Items[i]
-			break
-		}
+	namespaceResource := dynamicClient.Resource(NamespaceGroupVersionResource).Namespace("")
+	unstructuredNamespace, err := namespaceResource.Get(context.TODO(), namespaceName, metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	if err = scheme.Scheme.Convert(unstructuredNamespace, namespace, unstructuredNamespace.GroupVersionKind()); err != nil {
+		return nil, err
 	}
 
 	return namespace, nil

tests/framework/extensions/kubeapi/rbac/list.go

@@ -35,3 +35,30 @@ func ListRoleBindings(client *rancher.Client, clusterName, namespace string, lis
 
 	return rbList, nil
 }
+
+// ListClusterRoleBindings is a helper function that uses the dynamic client to list clusterrolebindings for a specific cluster.
+// ListClusterRoleBindings accepts ListOptions for specifying desired parameters for listed objects.
+func ListClusterRoleBindings(client *rancher.Client, clusterName string, listOpt metav1.ListOptions) (*rbacv1.ClusterRoleBindingList, error) {
+	dynamicClient, err := client.GetDownStreamClusterClient(clusterName)
+	if err != nil {
+		return nil, err
+	}
+
+	unstructuredList, err := dynamicClient.Resource(ClusterRoleBindingGroupVersionResource).Namespace("").List(context.Background(), listOpt)
+	if err != nil {
+		return nil, err
+	}
+
+	crbList := new(rbacv1.ClusterRoleBindingList)
+	for _, unstructuredCRB := range unstructuredList.Items {
+		crb := &rbacv1.ClusterRoleBinding{}
+		err := scheme.Scheme.Convert(&unstructuredCRB, crb, unstructuredCRB.GroupVersionKind())
+		if err != nil {
+			return nil, err
+		}
+
+		crbList.Items = append(crbList.Items, *crb)
+	}
+
+	return crbList, nil
+}

tests/framework/extensions/kubeapi/rbac/rbac.go

@@ -20,3 +20,11 @@ var RoleBindingGroupVersionResource = schema.GroupVersionResource{
 	Version:  rbacv1.SchemeGroupVersion.Version,
 	Resource: "rolebindings",
 }
+
+// ClusterRoleBindingGroupVersionResource is the required Group Version Resource for accessing clusterrolebindings in a cluster,
+// using the dynamic client.
+var ClusterRoleBindingGroupVersionResource = schema.GroupVersionResource{
+	Group:    rbacv1.SchemeGroupVersion.Group,
+	Version:  rbacv1.SchemeGroupVersion.Version,
+	Resource: "clusterrolebindings",
+}

tests/framework/extensions/users/users.go

@@ -232,6 +232,10 @@ func AddClusterRoleToUser(rancherClient *rancher.Client, cluster *management.Clu
 		if err != nil {
 			return false, err
 		}
+		if cluster.Annotations == nil || cluster.Annotations["field.cattle.io/creatorId"] == "" {
+			// no cluster creator, no roles to populate. This will be the case for the "local" cluster.
+			return true, nil
+		}
 		if v3.ClusterConditionInitialRolesPopulated.IsTrue(cluster) {
 			return true, nil
 		}
@@ -281,5 +285,33 @@ func RemoveClusterRoleFromUser(rancherClient *rancher.Client, user *management.U
 		}
 	}
 
-	return rancherClient.Management.ClusterRoleTemplateBinding.Delete(&roleToDelete)
+	if err = rancherClient.Management.ClusterRoleTemplateBinding.Delete(&roleToDelete); err != nil {
+		return err
+	}
+
+	var backoff = kwait.Backoff{
+		Duration: 100 * time.Millisecond,
+		Factor:   1,
+		Jitter:   0,
+		Steps:    5,
+	}
+
+	err = kwait.ExponentialBackoff(backoff, func() (done bool, err error) {
+		req, err := labels.NewRequirement(rtbOwnerLabel, selection.Equals, []string{fmt.Sprintf("%s_%s", roleToDelete.ClusterID, roleToDelete.Name)})
+		if err != nil {
+			return false, err
+		}
+
+		downstreamCRBs, err := rbac.ListClusterRoleBindings(rancherClient, roleToDelete.ClusterID, metav1.ListOptions{
+			LabelSelector: labels.NewSelector().Add(*req).String(),
+		})
+		if err != nil {
+			return false, err
+		}
+		if len(downstreamCRBs.Items) != 0 {
+			return false, nil
+		}
+		return true, nil
+	})
+	return err
 }