From cec4ec824ee795b100a21eba801c37560ba8e057 Mon Sep 17 00:00:00 2001 From: zhujian Date: Fri, 17 Dec 2021 09:37:50 +0000 Subject: [PATCH] separate the manifests and operations for klusterlet operator Signed-off-by: zhujian --- hack/copy-crds.sh | 2 +- hack/verify-crds.sh | 2 +- manifests/fs.go | 6 +- ...usterlet-registration-role-management.yaml | 13 - ...t-registration-rolebinding-management.yaml | 13 - ...anagement.io_appliedmanifestworks.crd.yaml | 0 ...uster-management.io_clusterclaims.crd.yaml | 0 ...anagement.io_appliedmanifestworks.crd.yaml | 0 ...uster-management.io_clusterclaims.crd.yaml | 0 .../klusterlet-registration-clusterrole.yaml | 8 +- ...erlet-registration-clusterrolebinding.yaml | 0 ...lusterlet-registration-serviceaccount.yaml | 0 .../klusterlet-work-clusterrole.yaml | 6 +- ...rlet-work-clusterrolebinding-addition.yaml | 0 .../klusterlet-work-clusterrolebinding.yaml | 0 .../klusterlet-work-serviceaccount.yaml | 0 .../klusterlet-registration-deployment.yaml | 0 .../klusterlet-registration-role.yaml | 3 + ...tion-rolebinding-extension-apiserver.yaml} | 0 .../klusterlet-registration-rolebinding.yaml | 0 ...usterlet-registration-serviceaccount.yaml} | 0 .../klusterlet-work-deployment.yaml | 0 .../klusterlet-work-role.yaml} | 9 +- ...work-rolebinding-extension-apiserver.yaml} | 0 .../klusterlet-work-rolebinding.yaml} | 0 .../klusterlet-work-serviceaccount.yaml} | 0 .../klusterlet_controller.go | 286 ++++++++---------- .../klusterlet_controller_test.go | 58 ++-- test/integration/integration_suite_test.go | 13 +- test/integration/klusterlet_test.go | 46 ++- 30 files changed, 241 insertions(+), 224 deletions(-) delete mode 100644 manifests/klusterlet/detached/klusterlet-registration-role-management.yaml delete mode 100644 manifests/klusterlet/detached/klusterlet-registration-rolebinding-management.yaml rename manifests/klusterlet/{ => managed}/0000_01_work.open-cluster-management.io_appliedmanifestworks.crd.yaml (100%) rename manifests/klusterlet/{ => managed}/0000_02_clusters.open-cluster-management.io_clusterclaims.crd.yaml (100%) rename manifests/klusterlet/{ => managed}/0001_01_work.open-cluster-management.io_appliedmanifestworks.crd.yaml (100%) rename manifests/klusterlet/{ => managed}/0001_02_clusters.open-cluster-management.io_clusterclaims.crd.yaml (100%) rename manifests/klusterlet/{ => managed}/klusterlet-registration-clusterrole.yaml (81%) rename manifests/klusterlet/{ => managed}/klusterlet-registration-clusterrolebinding.yaml (100%) rename manifests/klusterlet/{ => managed}/klusterlet-registration-serviceaccount.yaml (100%) rename manifests/klusterlet/{ => managed}/klusterlet-work-clusterrole.yaml (93%) rename manifests/klusterlet/{ => managed}/klusterlet-work-clusterrolebinding-addition.yaml (100%) rename manifests/klusterlet/{ => managed}/klusterlet-work-clusterrolebinding.yaml (100%) rename manifests/klusterlet/{ => managed}/klusterlet-work-serviceaccount.yaml (100%) rename manifests/klusterlet/{ => management}/klusterlet-registration-deployment.yaml (100%) rename manifests/klusterlet/{ => management}/klusterlet-registration-role.yaml (61%) rename manifests/klusterlet/{detached/klusterlet-registration-rolebinding-extension-apiserver-management.yaml => management/klusterlet-registration-rolebinding-extension-apiserver.yaml} (100%) rename manifests/klusterlet/{ => management}/klusterlet-registration-rolebinding.yaml (100%) rename manifests/klusterlet/{detached/klusterlet-registration-serviceaccount-management.yaml => management/klusterlet-registration-serviceaccount.yaml} (100%) rename manifests/klusterlet/{ => management}/klusterlet-work-deployment.yaml (100%) rename manifests/klusterlet/{detached/klusterlet-work-role-management.yaml => management/klusterlet-work-role.yaml} (62%) rename manifests/klusterlet/{detached/klusterlet-work-rolebinding-extension-apiserver-management.yaml => management/klusterlet-work-rolebinding-extension-apiserver.yaml} (100%) rename manifests/klusterlet/{detached/klusterlet-work-rolebinding-management.yaml => management/klusterlet-work-rolebinding.yaml} (100%) rename manifests/klusterlet/{detached/klusterlet-work-serviceaccount-management.yaml => management/klusterlet-work-serviceaccount.yaml} (100%) diff --git a/hack/copy-crds.sh b/hack/copy-crds.sh index 93c996ef32f..333c72768e6 100755 --- a/hack/copy-crds.sh +++ b/hack/copy-crds.sh @@ -9,7 +9,7 @@ done for f in $SPOKE_CRD_FILES do - cp $f ./manifests/klusterlet/ + cp $f ./manifests/klusterlet/managed/ done cp $CLUSTER_MANAGER_CRD_FILE ./deploy/cluster-manager/config/crds/ diff --git a/hack/verify-crds.sh b/hack/verify-crds.sh index 43445b17d9a..2a989538abc 100755 --- a/hack/verify-crds.sh +++ b/hack/verify-crds.sh @@ -9,7 +9,7 @@ done for f in $SPOKE_CRD_FILES do - diff -N $f ./manifests/klusterlet/$(basename $f) || ( echo 'crd content is incorrect' && false ) + diff -N $f ./manifests/klusterlet/managed/$(basename $f) || ( echo 'crd content is incorrect' && false ) done diff -N $CLUSTER_MANAGER_CRD_FILE ./deploy/cluster-manager/config/crds/$(basename $CLUSTER_MANAGER_CRD_FILE) || ( echo 'crd content is incorrect' && false ) diff --git a/manifests/fs.go b/manifests/fs.go index b2fb68ac1fb..4497b345eaa 100644 --- a/manifests/fs.go +++ b/manifests/fs.go @@ -5,9 +5,7 @@ import "embed" //go:embed cluster-manager var ClusterManagerManifestFiles embed.FS -//go:embed klusterlet -//go:embed klusterlet/detached +//go:embed klusterlet/management +//go:embed klusterlet/managed //go:embed klusterletkube111 var KlusterletManifestFiles embed.FS - -// var Klusterlet111ManifestFiles embed.FS diff --git a/manifests/klusterlet/detached/klusterlet-registration-role-management.yaml b/manifests/klusterlet/detached/klusterlet-registration-role-management.yaml deleted file mode 100644 index de759a721d6..00000000000 --- a/manifests/klusterlet/detached/klusterlet-registration-role-management.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# Role for registration agent. -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: open-cluster-management:{{ .KlusterletName }}-registration:agent - namespace: {{ .KlusterletNamespace }} -rules: -- apiGroups: [""] - resources: ["configmaps", "secrets"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] -- apiGroups: ["", "events.k8s.io"] - resources: ["events"] - verbs: ["create", "patch", "update"] diff --git a/manifests/klusterlet/detached/klusterlet-registration-rolebinding-management.yaml b/manifests/klusterlet/detached/klusterlet-registration-rolebinding-management.yaml deleted file mode 100644 index e9db672f709..00000000000 --- a/manifests/klusterlet/detached/klusterlet-registration-rolebinding-management.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: open-cluster-management:{{ .KlusterletName }}-registration:agent - namespace: {{ .KlusterletNamespace }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: open-cluster-management:{{ .KlusterletName }}-registration:agent -subjects: - - kind: ServiceAccount - name: {{ .KlusterletName }}-registration-sa - namespace: {{ .KlusterletNamespace }} diff --git a/manifests/klusterlet/0000_01_work.open-cluster-management.io_appliedmanifestworks.crd.yaml b/manifests/klusterlet/managed/0000_01_work.open-cluster-management.io_appliedmanifestworks.crd.yaml similarity index 100% rename from manifests/klusterlet/0000_01_work.open-cluster-management.io_appliedmanifestworks.crd.yaml rename to manifests/klusterlet/managed/0000_01_work.open-cluster-management.io_appliedmanifestworks.crd.yaml diff --git a/manifests/klusterlet/0000_02_clusters.open-cluster-management.io_clusterclaims.crd.yaml b/manifests/klusterlet/managed/0000_02_clusters.open-cluster-management.io_clusterclaims.crd.yaml similarity index 100% rename from manifests/klusterlet/0000_02_clusters.open-cluster-management.io_clusterclaims.crd.yaml rename to manifests/klusterlet/managed/0000_02_clusters.open-cluster-management.io_clusterclaims.crd.yaml diff --git a/manifests/klusterlet/0001_01_work.open-cluster-management.io_appliedmanifestworks.crd.yaml b/manifests/klusterlet/managed/0001_01_work.open-cluster-management.io_appliedmanifestworks.crd.yaml similarity index 100% rename from manifests/klusterlet/0001_01_work.open-cluster-management.io_appliedmanifestworks.crd.yaml rename to manifests/klusterlet/managed/0001_01_work.open-cluster-management.io_appliedmanifestworks.crd.yaml diff --git a/manifests/klusterlet/0001_02_clusters.open-cluster-management.io_clusterclaims.crd.yaml b/manifests/klusterlet/managed/0001_02_clusters.open-cluster-management.io_clusterclaims.crd.yaml similarity index 100% rename from manifests/klusterlet/0001_02_clusters.open-cluster-management.io_clusterclaims.crd.yaml rename to manifests/klusterlet/managed/0001_02_clusters.open-cluster-management.io_clusterclaims.crd.yaml diff --git a/manifests/klusterlet/klusterlet-registration-clusterrole.yaml b/manifests/klusterlet/managed/klusterlet-registration-clusterrole.yaml similarity index 81% rename from manifests/klusterlet/klusterlet-registration-clusterrole.yaml rename to manifests/klusterlet/managed/klusterlet-registration-clusterrole.yaml index 3b449a990de..1ac037143f2 100644 --- a/manifests/klusterlet/klusterlet-registration-clusterrole.yaml +++ b/manifests/klusterlet/managed/klusterlet-registration-clusterrole.yaml @@ -5,10 +5,12 @@ metadata: name: open-cluster-management:{{ .KlusterletName }}-registration:agent rules: # Allow agent to get/list/watch nodes and configmaps. -- apiGroups: [""] - resources: ["nodes", "configmaps"] - verbs: ["get", "list", "watch"] +# - apiGroups: [""] +# resources: ["nodes", "configmaps"] +# verbs: ["get", "list", "watch"] + # Allow agent to get/list/watch/create/delete/update/patch secrets. +# TODO(zhujian7): consider to delete when addon agents can deployed on the management cluster. - apiGroups: [""] resources: ["secrets"] verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] diff --git a/manifests/klusterlet/klusterlet-registration-clusterrolebinding.yaml b/manifests/klusterlet/managed/klusterlet-registration-clusterrolebinding.yaml similarity index 100% rename from manifests/klusterlet/klusterlet-registration-clusterrolebinding.yaml rename to manifests/klusterlet/managed/klusterlet-registration-clusterrolebinding.yaml diff --git a/manifests/klusterlet/klusterlet-registration-serviceaccount.yaml b/manifests/klusterlet/managed/klusterlet-registration-serviceaccount.yaml similarity index 100% rename from manifests/klusterlet/klusterlet-registration-serviceaccount.yaml rename to manifests/klusterlet/managed/klusterlet-registration-serviceaccount.yaml diff --git a/manifests/klusterlet/klusterlet-work-clusterrole.yaml b/manifests/klusterlet/managed/klusterlet-work-clusterrole.yaml similarity index 93% rename from manifests/klusterlet/klusterlet-work-clusterrole.yaml rename to manifests/klusterlet/managed/klusterlet-work-clusterrole.yaml index 2f759fbd055..966b69f8958 100644 --- a/manifests/klusterlet/klusterlet-work-clusterrole.yaml +++ b/manifests/klusterlet/managed/klusterlet-work-clusterrole.yaml @@ -24,9 +24,9 @@ rules: resources: ["subjectaccessreviews"] verbs: ["create"] # Allow agent to create events -- apiGroups: ["", "events.k8s.io"] - resources: ["events"] - verbs: ["create", "patch", "update"] +# - apiGroups: ["", "events.k8s.io"] +# resources: ["events"] +# verbs: ["create", "patch", "update"] # Allow agent to managed appliedmanifestworks - apiGroups: ["work.open-cluster-management.io"] resources: ["appliedmanifestworks"] diff --git a/manifests/klusterlet/klusterlet-work-clusterrolebinding-addition.yaml b/manifests/klusterlet/managed/klusterlet-work-clusterrolebinding-addition.yaml similarity index 100% rename from manifests/klusterlet/klusterlet-work-clusterrolebinding-addition.yaml rename to manifests/klusterlet/managed/klusterlet-work-clusterrolebinding-addition.yaml diff --git a/manifests/klusterlet/klusterlet-work-clusterrolebinding.yaml b/manifests/klusterlet/managed/klusterlet-work-clusterrolebinding.yaml similarity index 100% rename from manifests/klusterlet/klusterlet-work-clusterrolebinding.yaml rename to manifests/klusterlet/managed/klusterlet-work-clusterrolebinding.yaml diff --git a/manifests/klusterlet/klusterlet-work-serviceaccount.yaml b/manifests/klusterlet/managed/klusterlet-work-serviceaccount.yaml similarity index 100% rename from manifests/klusterlet/klusterlet-work-serviceaccount.yaml rename to manifests/klusterlet/managed/klusterlet-work-serviceaccount.yaml diff --git a/manifests/klusterlet/klusterlet-registration-deployment.yaml b/manifests/klusterlet/management/klusterlet-registration-deployment.yaml similarity index 100% rename from manifests/klusterlet/klusterlet-registration-deployment.yaml rename to manifests/klusterlet/management/klusterlet-registration-deployment.yaml diff --git a/manifests/klusterlet/klusterlet-registration-role.yaml b/manifests/klusterlet/management/klusterlet-registration-role.yaml similarity index 61% rename from manifests/klusterlet/klusterlet-registration-role.yaml rename to manifests/klusterlet/management/klusterlet-registration-role.yaml index de759a721d6..1dd0b54f303 100644 --- a/manifests/klusterlet/klusterlet-registration-role.yaml +++ b/manifests/klusterlet/management/klusterlet-registration-role.yaml @@ -5,6 +5,9 @@ metadata: name: open-cluster-management:{{ .KlusterletName }}-registration:agent namespace: {{ .KlusterletNamespace }} rules: +# leader election needs to operate configmaps, create hub-kubeconfig external-managed-registration/work secrets +# TODO(zhujian7): may be replaced by a clusterrole to grant secret operation for others namespaces when addon +# agents are supported running on the management cluster - apiGroups: [""] resources: ["configmaps", "secrets"] verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] diff --git a/manifests/klusterlet/detached/klusterlet-registration-rolebinding-extension-apiserver-management.yaml b/manifests/klusterlet/management/klusterlet-registration-rolebinding-extension-apiserver.yaml similarity index 100% rename from manifests/klusterlet/detached/klusterlet-registration-rolebinding-extension-apiserver-management.yaml rename to manifests/klusterlet/management/klusterlet-registration-rolebinding-extension-apiserver.yaml diff --git a/manifests/klusterlet/klusterlet-registration-rolebinding.yaml b/manifests/klusterlet/management/klusterlet-registration-rolebinding.yaml similarity index 100% rename from manifests/klusterlet/klusterlet-registration-rolebinding.yaml rename to manifests/klusterlet/management/klusterlet-registration-rolebinding.yaml diff --git a/manifests/klusterlet/detached/klusterlet-registration-serviceaccount-management.yaml b/manifests/klusterlet/management/klusterlet-registration-serviceaccount.yaml similarity index 100% rename from manifests/klusterlet/detached/klusterlet-registration-serviceaccount-management.yaml rename to manifests/klusterlet/management/klusterlet-registration-serviceaccount.yaml diff --git a/manifests/klusterlet/klusterlet-work-deployment.yaml b/manifests/klusterlet/management/klusterlet-work-deployment.yaml similarity index 100% rename from manifests/klusterlet/klusterlet-work-deployment.yaml rename to manifests/klusterlet/management/klusterlet-work-deployment.yaml diff --git a/manifests/klusterlet/detached/klusterlet-work-role-management.yaml b/manifests/klusterlet/management/klusterlet-work-role.yaml similarity index 62% rename from manifests/klusterlet/detached/klusterlet-work-role-management.yaml rename to manifests/klusterlet/management/klusterlet-work-role.yaml index fcab6422764..3459627047e 100644 --- a/manifests/klusterlet/detached/klusterlet-work-role-management.yaml +++ b/manifests/klusterlet/management/klusterlet-work-role.yaml @@ -5,11 +5,12 @@ metadata: name: open-cluster-management:{{ .KlusterletName }}-work:agent namespace: {{ .KlusterletNamespace }} rules: +# leader election needs to operate configmaps +# - apiGroups: [""] +# resources: ["configmaps"] +# verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] -- apiGroups: [""] - resources: ["secrets"] + resources: ["configmaps", "secrets"] verbs: ["get", "list", "watch"] - apiGroups: ["", "events.k8s.io"] resources: ["events"] diff --git a/manifests/klusterlet/detached/klusterlet-work-rolebinding-extension-apiserver-management.yaml b/manifests/klusterlet/management/klusterlet-work-rolebinding-extension-apiserver.yaml similarity index 100% rename from manifests/klusterlet/detached/klusterlet-work-rolebinding-extension-apiserver-management.yaml rename to manifests/klusterlet/management/klusterlet-work-rolebinding-extension-apiserver.yaml diff --git a/manifests/klusterlet/detached/klusterlet-work-rolebinding-management.yaml b/manifests/klusterlet/management/klusterlet-work-rolebinding.yaml similarity index 100% rename from manifests/klusterlet/detached/klusterlet-work-rolebinding-management.yaml rename to manifests/klusterlet/management/klusterlet-work-rolebinding.yaml diff --git a/manifests/klusterlet/detached/klusterlet-work-serviceaccount-management.yaml b/manifests/klusterlet/management/klusterlet-work-serviceaccount.yaml similarity index 100% rename from manifests/klusterlet/detached/klusterlet-work-serviceaccount-management.yaml rename to manifests/klusterlet/management/klusterlet-work-serviceaccount.yaml diff --git a/pkg/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller.go b/pkg/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller.go index c980f847c38..c8f2374720c 100644 --- a/pkg/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller.go +++ b/pkg/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller.go @@ -49,36 +49,34 @@ const ( var ( crdV1StaticFiles = []string{ - "klusterlet/0000_01_work.open-cluster-management.io_appliedmanifestworks.crd.yaml", - "klusterlet/0000_02_clusters.open-cluster-management.io_clusterclaims.crd.yaml", + "klusterlet/managed/0000_01_work.open-cluster-management.io_appliedmanifestworks.crd.yaml", + "klusterlet/managed/0000_02_clusters.open-cluster-management.io_clusterclaims.crd.yaml", } crdV1beta1StaticFiles = []string{ - "klusterlet/0001_01_work.open-cluster-management.io_appliedmanifestworks.crd.yaml", - "klusterlet/0001_02_clusters.open-cluster-management.io_clusterclaims.crd.yaml", + "klusterlet/managed/0001_01_work.open-cluster-management.io_appliedmanifestworks.crd.yaml", + "klusterlet/managed/0001_02_clusters.open-cluster-management.io_clusterclaims.crd.yaml", } - staticResourceFiles = []string{ - "klusterlet/klusterlet-registration-serviceaccount.yaml", - "klusterlet/klusterlet-registration-clusterrole.yaml", - "klusterlet/klusterlet-registration-clusterrolebinding.yaml", - "klusterlet/klusterlet-registration-role.yaml", - "klusterlet/klusterlet-registration-rolebinding.yaml", - "klusterlet/klusterlet-work-serviceaccount.yaml", - "klusterlet/klusterlet-work-clusterrole.yaml", - "klusterlet/klusterlet-work-clusterrolebinding.yaml", - "klusterlet/klusterlet-work-clusterrolebinding-addition.yaml", + managedStaticResourceFiles = []string{ + "klusterlet/managed/klusterlet-registration-serviceaccount.yaml", + "klusterlet/managed/klusterlet-registration-clusterrole.yaml", + "klusterlet/managed/klusterlet-registration-clusterrolebinding.yaml", + "klusterlet/managed/klusterlet-work-serviceaccount.yaml", + "klusterlet/managed/klusterlet-work-clusterrole.yaml", + "klusterlet/managed/klusterlet-work-clusterrolebinding.yaml", + "klusterlet/managed/klusterlet-work-clusterrolebinding-addition.yaml", } - detachedStaticResourceFiles = []string{ - "klusterlet/detached/klusterlet-registration-serviceaccount-management.yaml", - "klusterlet/detached/klusterlet-registration-role-management.yaml", - "klusterlet/detached/klusterlet-registration-rolebinding-management.yaml", - "klusterlet/detached/klusterlet-registration-rolebinding-extension-apiserver-management.yaml", - "klusterlet/detached/klusterlet-work-serviceaccount-management.yaml", - "klusterlet/detached/klusterlet-work-role-management.yaml", - "klusterlet/detached/klusterlet-work-rolebinding-management.yaml", - "klusterlet/detached/klusterlet-work-rolebinding-extension-apiserver-management.yaml", + managementStaticResourceFiles = []string{ + "klusterlet/management/klusterlet-registration-serviceaccount.yaml", + "klusterlet/management/klusterlet-registration-role.yaml", + "klusterlet/management/klusterlet-registration-rolebinding.yaml", + "klusterlet/management/klusterlet-registration-rolebinding-extension-apiserver.yaml", + "klusterlet/management/klusterlet-work-serviceaccount.yaml", + "klusterlet/management/klusterlet-work-role.yaml", + "klusterlet/management/klusterlet-work-rolebinding.yaml", + "klusterlet/management/klusterlet-work-rolebinding-extension-apiserver.yaml", } kube111StaticResourceFiles = []string{ @@ -167,7 +165,7 @@ func (n *klusterletController) sync(ctx context.Context, controllerContext facto klog.V(4).Infof("Reconciling Klusterlet %q", klusterletName) klusterlet, err := n.klusterletLister.Get(klusterletName) if errors.IsNotFound(err) { - // AgentCore not found, could have been deleted, do nothing. + // Klusterlet not found, could have been deleted, do nothing. return nil } if err != nil { @@ -249,7 +247,7 @@ func (n *klusterletController) sync(ctx context.Context, controllerContext facto if detachedMode { // In detached mode, we should ensure the namespace on the managed cluster since - // some resources(eg:role, rolebinding) are still deployed on managed cluster. + // some resources(eg:service account) are still deployed on managed cluster. err := n.ensureNamespace(ctx, managedClusterClients.kubeClient, klusterletName, config.KlusterletNamespace) if err != nil { return err @@ -260,122 +258,63 @@ func (n *klusterletController) sync(ctx context.Context, controllerContext facto // If kube version is less than 1.12, deploy static resource for kube 1.11 at first // TODO remove this when we do not support kube 1.11 any longer if cnt, err := n.kubeVersion.Compare("v1.12.0"); err == nil && cnt < 0 { - errs := []error{} - resourceResult := resourceapply.ApplyDirectly( - resourceapply.NewKubeClientHolder(managedClusterClients.kubeClient), - controllerContext.Recorder(), - func(name string) ([]byte, error) { - template, err := manifests.KlusterletManifestFiles.ReadFile(name) - if err != nil { - return nil, err - } - objData := assets.MustCreateAssetFromTemplate(name, template, config).Data - helpers.SetRelatedResourcesStatusesWithObj(&relatedResources, objData) - return objData, nil - }, - kube111StaticResourceFiles..., - ) - for _, result := range resourceResult { - if result.Error != nil { - errs = append(errs, fmt.Errorf("%q (%T): %v", result.File, result.Type, result.Error)) - } - } - - if len(errs) > 0 { - applyErrors := operatorhelpers.NewMultiLineAggregate(errs) - _, _, _ = helpers.UpdateKlusterletStatus(ctx, n.klusterletClient, klusterletName, helpers.UpdateKlusterletConditionFn(metav1.Condition{ - Type: klusterletApplied, Status: metav1.ConditionFalse, Reason: "KlusterletApplyFailed", - Message: applyErrors.Error(), - })) - return applyErrors + statuses, err := n.applyStaticFiles(ctx, klusterletName, kube111StaticResourceFiles, &config, managedClusterClients.kubeClient, + managedClusterClients.apiExtensionClient, controllerContext.Recorder()) + if err != nil { + return err } + relatedResources = append(relatedResources, statuses...) } - // Apply static files + // Apply static files on managed cluster var appliedStaticFiles []string // CRD v1beta1 was deprecated from k8s 1.16.0 and will be removed in k8s 1.22 if cnt, err := n.kubeVersion.Compare("v1.16.0"); err == nil && cnt < 0 { - appliedStaticFiles = append(crdV1beta1StaticFiles, staticResourceFiles...) + appliedStaticFiles = append(crdV1beta1StaticFiles, managedStaticResourceFiles...) } else { - appliedStaticFiles = append(crdV1StaticFiles, staticResourceFiles...) + appliedStaticFiles = append(crdV1StaticFiles, managedStaticResourceFiles...) } - err = n.applyStaticFiles(ctx, klusterletName, appliedStaticFiles, &relatedResources, &config, managedClusterClients.kubeClient, + statuses, err := n.applyStaticFiles(ctx, klusterletName, appliedStaticFiles, &config, managedClusterClients.kubeClient, managedClusterClients.apiExtensionClient, controllerContext.Recorder()) if err != nil { return err } + relatedResources = append(relatedResources, statuses...) // Create hub config secret - hubSecret, err := n.kubeClient.CoreV1().Secrets(config.KlusterletNamespace).Get(ctx, helpers.HubKubeConfig, metav1.GetOptions{}) - switch { - case errors.IsNotFound(err): - // Create an empty secret with placeholder - hubSecret = &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: helpers.HubKubeConfig, - Namespace: config.KlusterletNamespace, - }, - Data: map[string][]byte{"placeholder": []byte("placeholder")}, - } - hubSecret, err = n.kubeClient.CoreV1().Secrets(config.KlusterletNamespace).Create(ctx, hubSecret, metav1.CreateOptions{}) - if err != nil { - _, _, _ = helpers.UpdateKlusterletStatus(ctx, n.klusterletClient, klusterletName, helpers.UpdateKlusterletConditionFn(metav1.Condition{ - Type: klusterletApplied, Status: metav1.ConditionFalse, Reason: "KlusterletApplyFailed", - Message: fmt.Sprintf("Failed to create hub kubeconfig secret -n %q %q: %v", hubSecret.Namespace, hubSecret.Name, err), - })) - return err - } - case err != nil: - _, _, _ = helpers.UpdateKlusterletStatus(ctx, n.klusterletClient, klusterletName, helpers.UpdateKlusterletConditionFn(metav1.Condition{ - Type: klusterletApplied, Status: metav1.ConditionFalse, Reason: "KlusterletApplyFailed", - Message: fmt.Sprintf("Failed to get hub kubeconfig secret with error %v", err), - })) + hubSecret, err := n.ensureHubKubeconfigSecret(ctx, &config) + if err != nil { return err } - if detachedMode { - // In Deatched mode, apply role, rolebinding, service account for registration and work to the management cluster. - err = n.applyStaticFiles(ctx, klusterletName, detachedStaticResourceFiles, &relatedResources, &config, n.kubeClient, - n.apiExtensionClient, controllerContext.Recorder()) - if err != nil { - return err - } + // Apply static files on management cluster + // Apply role, rolebinding, service account for registration and work to the management cluster. + statuses, err = n.applyStaticFiles(ctx, klusterletName, managementStaticResourceFiles, &config, n.kubeClient, + n.apiExtensionClient, controllerContext.Recorder()) + if err != nil { + return err + } + relatedResources = append(relatedResources, statuses...) + if detachedMode { // create managed config secret for registration and work. err = n.createManagedClusterKubeconfig(ctx, klusterletName, config.KlusterletNamespace, registrationServiceAccountName(klusterletName), config.ExternalManagedKubeConfigRegistrationSecret, managedClusterClients.kubeconfig, managedClusterClients.kubeClient, n.kubeClient.CoreV1(), controllerContext.Recorder()) if err != nil { return err } - - if err := n.createManagedClusterKubeconfig(ctx, klusterletName, config.KlusterletNamespace, workServiceAccountName(klusterletName), config.ExternalManagedKubeConfigWorkSecret, - managedClusterClients.kubeconfig, managedClusterClients.kubeClient, n.kubeClient.CoreV1(), controllerContext.Recorder()); err != nil { + err := n.createManagedClusterKubeconfig(ctx, klusterletName, config.KlusterletNamespace, workServiceAccountName(klusterletName), config.ExternalManagedKubeConfigWorkSecret, + managedClusterClients.kubeconfig, managedClusterClients.kubeClient, n.kubeClient.CoreV1(), controllerContext.Recorder()) + if err != nil { return err } } // Deploy registration agent - registrationGeneration, err := helpers.ApplyDeployment( - n.kubeClient, - klusterlet.Status.Generations, - klusterlet.Spec.NodePlacement, - func(name string) ([]byte, error) { - template, err := manifests.KlusterletManifestFiles.ReadFile(name) - if err != nil { - return nil, err - } - objData := assets.MustCreateAssetFromTemplate(name, template, config).Data - helpers.SetRelatedResourcesStatusesWithObj(&relatedResources, objData) - return objData, nil - }, - controllerContext.Recorder(), - "klusterlet/klusterlet-registration-deployment.yaml") + statuses, registrationGeneration, err := n.applyDeployment(ctx, klusterlet, &config, "klusterlet/management/klusterlet-registration-deployment.yaml", controllerContext.Recorder()) if err != nil { - _, _, _ = helpers.UpdateKlusterletStatus(ctx, n.klusterletClient, klusterletName, helpers.UpdateKlusterletConditionFn(metav1.Condition{ - Type: klusterletApplied, Status: metav1.ConditionFalse, Reason: "KlusterletApplyFailed", - Message: fmt.Sprintf("Failed to deploy registration deployment with error %v", err), - })) return err } + relatedResources = append(relatedResources, statuses...) // If cluster name is empty, read cluster name from hub config secret if config.ClusterName == "" { @@ -384,9 +323,64 @@ func (n *klusterletController) sync(ctx context.Context, controllerContext facto config.ClusterName = string(clusterName) } } - // Deploy work agent - workGeneration, err := helpers.ApplyDeployment( + statuses, workGeneration, err := n.applyDeployment(ctx, klusterlet, &config, "klusterlet/management/klusterlet-work-deployment.yaml", controllerContext.Recorder()) + if err != nil { + return err + } + relatedResources = append(relatedResources, statuses...) + + // if we get here, we have successfully applied everything and should indicate that + _, _, _ = helpers.UpdateKlusterletStatus(ctx, n.klusterletClient, klusterletName, + helpers.UpdateKlusterletConditionFn(metav1.Condition{ + Type: klusterletApplied, Status: metav1.ConditionTrue, Reason: "KlusterletApplied", + Message: "Klusterlet Component Applied"}), + helpers.UpdateKlusterletGenerationsFn(registrationGeneration, workGeneration), + helpers.UpdateKlusterletRelatedResourcesFn(relatedResources...), + func(oldStatus *operatorapiv1.KlusterletStatus) error { + oldStatus.ObservedGeneration = klusterlet.Generation + return nil + }, + ) + return nil +} + +// ensureHubKubeconfigSecret will create a placeholder hub kubeconfig if it is not exist. +func (n *klusterletController) ensureHubKubeconfigSecret(ctx context.Context, config *klusterletConfig) (*corev1.Secret, error) { + hubSecret, err := n.kubeClient.CoreV1().Secrets(config.KlusterletNamespace).Get(ctx, helpers.HubKubeConfig, metav1.GetOptions{}) + switch { + case errors.IsNotFound(err): + // Create an empty secret with placeholder + hubSecret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: helpers.HubKubeConfig, + Namespace: config.KlusterletNamespace, + }, + Data: map[string][]byte{"placeholder": []byte("placeholder")}, + } + hubSecret, err = n.kubeClient.CoreV1().Secrets(config.KlusterletNamespace).Create(ctx, hubSecret, metav1.CreateOptions{}) + if err != nil { + _, _, _ = helpers.UpdateKlusterletStatus(ctx, n.klusterletClient, config.KlusterletName, helpers.UpdateKlusterletConditionFn(metav1.Condition{ + Type: klusterletApplied, Status: metav1.ConditionFalse, Reason: "KlusterletApplyFailed", + Message: fmt.Sprintf("Failed to create hub kubeconfig secret -n %q %q: %v", hubSecret.Namespace, hubSecret.Name, err), + })) + return hubSecret, err + } + case err != nil: + _, _, _ = helpers.UpdateKlusterletStatus(ctx, n.klusterletClient, config.KlusterletName, helpers.UpdateKlusterletConditionFn(metav1.Condition{ + Type: klusterletApplied, Status: metav1.ConditionFalse, Reason: "KlusterletApplyFailed", + Message: fmt.Sprintf("Failed to get hub kubeconfig secret with error %v", err), + })) + return hubSecret, err + } + return hubSecret, nil +} + +// applyDeployment applies deployment on the management cluster +func (n *klusterletController) applyDeployment(ctx context.Context, klusterlet *operatorapiv1.Klusterlet, config *klusterletConfig, deploymentFile string, recorder events.Recorder) ( + []operatorapiv1.RelatedResourceMeta, operatorapiv1.GenerationStatus, error) { + var relatedResources []operatorapiv1.RelatedResourceMeta + generationStatus, err := helpers.ApplyDeployment( n.kubeClient, klusterlet.Status.Generations, klusterlet.Spec.NodePlacement, @@ -399,41 +393,30 @@ func (n *klusterletController) sync(ctx context.Context, controllerContext facto helpers.SetRelatedResourcesStatusesWithObj(&relatedResources, objData) return objData, nil }, - controllerContext.Recorder(), - "klusterlet/klusterlet-work-deployment.yaml") + recorder, + deploymentFile) if err != nil { - _, _, _ = helpers.UpdateKlusterletStatus(ctx, n.klusterletClient, klusterletName, helpers.UpdateKlusterletConditionFn(metav1.Condition{ + _, _, _ = helpers.UpdateKlusterletStatus(ctx, n.klusterletClient, klusterlet.Name, helpers.UpdateKlusterletConditionFn(metav1.Condition{ Type: klusterletApplied, Status: metav1.ConditionFalse, Reason: "KlusterletApplyFailed", - Message: fmt.Sprintf("Failed to deploy work deployment with error %v", err), + Message: fmt.Sprintf("Failed to deploy %s deployment with error %v", deploymentFile, err), })) - return err + return relatedResources, generationStatus, err } - observedKlusterletGeneration := klusterlet.Generation - // if we get here, we have successfully applied everything and should indicate that - _, _, _ = helpers.UpdateKlusterletStatus(ctx, n.klusterletClient, klusterletName, - helpers.UpdateKlusterletConditionFn(metav1.Condition{ - Type: klusterletApplied, Status: metav1.ConditionTrue, Reason: "KlusterletApplied", - Message: "Klusterlet Component Applied"}), - helpers.UpdateKlusterletGenerationsFn(registrationGeneration, workGeneration), - helpers.UpdateKlusterletRelatedResourcesFn(relatedResources...), - func(oldStatus *operatorapiv1.KlusterletStatus) error { - oldStatus.ObservedGeneration = observedKlusterletGeneration - return nil - }, - ) - return nil + return relatedResources, generationStatus, nil } // applyStaticFiles applies files using destKubeclient nad destApiExtensionClient. // resource status will be saved in the relatedResourcesStatuses, and will save the // result to the klusterlet status if there is any error. func (n *klusterletController) applyStaticFiles(ctx context.Context, klusterletName string, - staticFiles []string, relatedResourcesStatuses *[]operatorapiv1.RelatedResourceMeta, + staticFiles []string, config *klusterletConfig, destKubeclient kubernetes.Interface, destApiExtensionClient apiextensionsclient.Interface, - recorder events.Recorder) error { + recorder events.Recorder) ([]operatorapiv1.RelatedResourceMeta, error) { errs := []error{} + var relatedResources []operatorapiv1.RelatedResourceMeta + resourceResults := resourceapply.ApplyDirectly( resourceapply.NewKubeClientHolder(destKubeclient).WithAPIExtensionsClient(destApiExtensionClient), recorder, @@ -443,7 +426,7 @@ func (n *klusterletController) applyStaticFiles(ctx context.Context, klusterletN return nil, err } objData := assets.MustCreateAssetFromTemplate(name, template, config).Data - helpers.SetRelatedResourcesStatusesWithObj(relatedResourcesStatuses, objData) + helpers.SetRelatedResourcesStatusesWithObj(&relatedResources, objData) return objData, nil }, staticFiles..., @@ -461,10 +444,10 @@ func (n *klusterletController) applyStaticFiles(ctx context.Context, klusterletN Type: klusterletApplied, Status: metav1.ConditionFalse, Reason: "KlusterletApplyFailed", Message: applyErrors.Error(), })) - return applyErrors + return relatedResources, applyErrors } - return nil + return relatedResources, nil } // registrationServiceAccountName splices the name of registration service account @@ -616,16 +599,16 @@ func (n *klusterletController) cleanUp( managedClients *managedClusterClients, config klusterletConfig) error { // Remove deployment - registrationDeployment := fmt.Sprintf("%s-registration-agent", config.KlusterletName) - err := managedClients.kubeClient.AppsV1().Deployments(config.KlusterletNamespace).Delete(ctx, registrationDeployment, metav1.DeleteOptions{}) - if err != nil && !errors.IsNotFound(err) { - return err + deployments := []string{ + fmt.Sprintf("%s-registration-agent", config.KlusterletName), + fmt.Sprintf("%s-work-agent", config.KlusterletName), } - controllerContext.Recorder().Eventf("DeploymentDeleted", "deployment %s is deleted", registrationDeployment) - workDeployment := fmt.Sprintf("%s-work-agent", config.KlusterletName) - err = managedClients.kubeClient.AppsV1().Deployments(config.KlusterletNamespace).Delete(ctx, workDeployment, metav1.DeleteOptions{}) - if err != nil && !errors.IsNotFound(err) { - return err + for _, deployment := range deployments { + err := managedClients.kubeClient.AppsV1().Deployments(config.KlusterletNamespace).Delete(ctx, deployment, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + return err + } + controllerContext.Recorder().Eventf("DeploymentDeleted", "deployment %s is deleted", deployment) } // get hub host from bootstrap kubeconfig @@ -658,16 +641,15 @@ func (n *klusterletController) cleanUp( // remove static file on the managed cluster err = n.removeStaticResources(ctx, managedClients.kubeClient, managedClients.apiExtensionClient, - staticResourceFiles, config) + managedStaticResourceFiles, config) if err != nil { return err } - if config.DetachedMode { - // remove static file on the management cluster - err = n.removeStaticResources(ctx, n.kubeClient, n.apiExtensionClient, detachedStaticResourceFiles, config) - if err != nil { - return err - } + + // remove static file on the management cluster + err = n.removeStaticResources(ctx, n.kubeClient, n.apiExtensionClient, managementStaticResourceFiles, config) + if err != nil { + return err } // TODO remove this when we do not support kube 1.11 any longer diff --git a/pkg/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller_test.go b/pkg/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller_test.go index 1b657f0375d..26321c06e75 100644 --- a/pkg/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller_test.go +++ b/pkg/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller_test.go @@ -411,8 +411,9 @@ func TestSyncDeploy(t *testing.T) { } // Check if resources are created as expected - if len(createObjects) != 12 { - t.Errorf("Expect 12 objects created in the sync loop, actual %d", len(createObjects)) + // 7 managed static manifests + 8 management static manifests - 2 duplicated service account manifests + 1 addon namespace + 2 deployments + if len(createObjects) != 16 { + t.Errorf("Expect 17 objects created in the sync loop, actual %d", len(createObjects)) } for _, object := range createObjects { ensureObject(t, object, klusterlet) @@ -460,21 +461,21 @@ func TestSyncDeployDetached(t *testing.T) { t.Errorf("Expected non error when sync, %v", err) } - createObjects := []runtime.Object{} + createObjectsManagement := []runtime.Object{} kubeActions := controller.kubeClient.Actions() for _, action := range kubeActions { if action.GetVerb() == "create" { object := action.(clienttesting.CreateActionImpl).Object - klog.Infof("kube create: %v\t resource:%v \t subresource:%v \t namespace:%v verb:%v ", object.GetObjectKind(), action.GetResource(), action.GetSubresource(), action.GetNamespace(), action.GetVerb()) - createObjects = append(createObjects, object) + klog.Infof("kube create: %v\t resource:%v \t namespace:%v", object.GetObjectKind(), action.GetResource(), action.GetNamespace()) + createObjectsManagement = append(createObjectsManagement, object) } } // Check if resources are created as expected on the management cluster // 8 static manifests + 2 secrets(external-managed-kubeconfig-registration,external-managed-kubeconfig-work) + 2 deployments(registration-agent,work-agent) - if len(createObjects) != 12 { - t.Errorf("Expect 12 objects created in the sync loop, actual %d", len(createObjects)) + if len(createObjectsManagement) != 12 { + t.Errorf("Expect 12 objects created in the sync loop, actual %d", len(createObjectsManagement)) } - for _, object := range createObjects { + for _, object := range createObjectsManagement { ensureObject(t, object, klusterlet) } @@ -483,15 +484,14 @@ func TestSyncDeployDetached(t *testing.T) { if action.GetVerb() == "create" { object := action.(clienttesting.CreateActionImpl).Object - klog.Infof("managed kube create: %v\t resource:%v \t subresource:%v \t namespace:%v verb:%v ", object.GetObjectKind().GroupVersionKind(), action.GetResource(), action.GetSubresource(), action.GetNamespace(), action.GetVerb()) - + klog.Infof("managed kube create: %v\t resource:%v \t namespace:%v", object.GetObjectKind().GroupVersionKind(), action.GetResource(), action.GetNamespace()) createObjectsManaged = append(createObjectsManaged, object) } } // Check if resources are created as expected on the managed cluster - // 11 static manifests - if len(createObjectsManaged) != 11 { - t.Errorf("Expect 11 objects created in the sync loop, actual %d", len(createObjectsManaged)) + // 7 static manifests + 2 namespaces + if len(createObjectsManaged) != 9 { + t.Errorf("Expect 9 objects created in the sync loop, actual %d", len(createObjectsManaged)) } for _, object := range createObjectsManaged { ensureObject(t, object, klusterlet) @@ -548,7 +548,7 @@ func TestSyncDelete(t *testing.T) { controller := newTestController(klusterlet, appliedManifestWorks, namespace, bootstrapKubeConfigSecret) syncContext := testinghelper.NewFakeSyncContext(t, "klusterlet") - err := controller.controller.sync(nil, syncContext) + err := controller.controller.sync(context.TODO(), syncContext) if err != nil { t.Errorf("Expected non error when sync, %v", err) } @@ -558,12 +558,14 @@ func TestSyncDelete(t *testing.T) { for _, action := range kubeActions { if action.GetVerb() == "delete" { deleteAction := action.(clienttesting.DeleteActionImpl) + klog.Infof("kube delete name: %v\t resource:%v \t namespace:%v", deleteAction.Name, deleteAction.GetResource(), deleteAction.GetNamespace()) deleteActions = append(deleteActions, deleteAction) } } - if len(deleteActions) != 14 { - t.Errorf("Expected 14 delete actions, but got %d", len(deleteActions)) + // 7 managed static manifests + 8 management static manifests + 1 hub kubeconfig + 2 namespaces + 2 deployments + if len(deleteActions) != 20 { + t.Errorf("Expected 20 delete actions, but got %d", len(deleteActions)) } deleteCRDActions := []clienttesting.DeleteActionImpl{} @@ -589,6 +591,7 @@ func TestSyncDelete(t *testing.T) { } } + // update 1 appliedminifestwork to remove appliedManifestWorkFinalizer if len(updateWorkActions) != 1 { t.Errorf("Expected 1 update action, but got %d", len(updateWorkActions)) } @@ -647,7 +650,9 @@ func TestClusterNameChange(t *testing.T) { hubSecret.Data["cluster-name"] = []byte("cluster1") controller := newTestController(klusterlet, nil, bootStrapSecret, hubSecret, namespace) syncContext := testinghelper.NewFakeSyncContext(t, "klusterlet") - err := controller.controller.sync(nil, syncContext) + + ctx := context.TODO() + err := controller.controller.sync(ctx, syncContext) if err != nil { t.Errorf("Expected non error when sync, %v", err) } @@ -676,7 +681,7 @@ func TestClusterNameChange(t *testing.T) { klusterlet.Generation = 1 controller.operatorStore.Update(klusterlet) - err = controller.controller.sync(nil, syncContext) + err = controller.controller.sync(ctx, syncContext) if err != nil { t.Errorf("Expected non error when sync, %v", err) } @@ -729,7 +734,7 @@ func TestSyncWithPullSecret(t *testing.T) { controller := newTestController(klusterlet, nil, bootStrapSecret, hubKubeConfigSecret, namespace, pullSecret) syncContext := testinghelper.NewFakeSyncContext(t, "klusterlet") - err := controller.controller.sync(nil, syncContext) + err := controller.controller.sync(context.TODO(), syncContext) if err != nil { t.Errorf("Expected non error when sync, %v", err) } @@ -760,7 +765,8 @@ func TestDeployOnKube111(t *testing.T) { controller.controller.kubeVersion = kubeVersion syncContext := testinghelper.NewFakeSyncContext(t, "klusterlet") - err := controller.controller.sync(nil, syncContext) + ctx := context.TODO() + err := controller.controller.sync(ctx, syncContext) if err != nil { t.Errorf("Expected non error when sync, %v", err) } @@ -775,8 +781,9 @@ func TestDeployOnKube111(t *testing.T) { } // Check if resources are created as expected - if len(createObjects) != 14 { - t.Errorf("Expect 14 objects created in the sync loop, actual %d", len(createObjects)) + // 7 managed static manifests + 8 management static manifests - 2 duplicated service account manifests + 1 addon namespace + 2 deployments + 2 kube111 clusterrolebindings + if len(createObjects) != 18 { + t.Errorf("Expect 18 objects created in the sync loop, actual %d", len(createObjects)) } for _, object := range createObjects { ensureObject(t, object, klusterlet) @@ -798,7 +805,7 @@ func TestDeployOnKube111(t *testing.T) { klusterlet.ObjectMeta.SetDeletionTimestamp(&now) controller.operatorStore.Update(klusterlet) controller.kubeClient.ClearActions() - err = controller.controller.sync(nil, syncContext) + err = controller.controller.sync(ctx, syncContext) if err != nil { t.Errorf("Expected non error when sync, %v", err) } @@ -812,8 +819,9 @@ func TestDeployOnKube111(t *testing.T) { } } - if len(deleteActions) != 16 { - t.Errorf("Expected 16 delete actions, but got %d", len(kubeActions)) + // 7 managed static manifests + 8 management static manifests + 1 hub kubeconfig + 2 namespaces + 2 deployments + 2 kube111 clusterrolebindings + if len(deleteActions) != 22 { + t.Errorf("Expected 22 delete actions, but got %d", len(kubeActions)) } } diff --git a/test/integration/integration_suite_test.go b/test/integration/integration_suite_test.go index ecea15305f0..86899478f7b 100644 --- a/test/integration/integration_suite_test.go +++ b/test/integration/integration_suite_test.go @@ -8,15 +8,15 @@ import ( "github.com/onsi/ginkgo" "github.com/onsi/gomega" + + apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/envtest/printer" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" operatorclient "open-cluster-management.io/api/client/operator/clientset/versioned" operatorapiv1 "open-cluster-management.io/api/operator/v1" @@ -41,6 +41,8 @@ var testEnv *envtest.Environment var kubeClient kubernetes.Interface +var apiExtensionClient apiextensionsclient.Interface + var restConfig *rest.Config var operatorClient operatorclient.Interface @@ -73,6 +75,11 @@ var _ = ginkgo.BeforeSuite(func(done ginkgo.Done) { kubeClient, err = kubernetes.NewForConfig(cfg) gomega.Expect(err).ToNot(gomega.HaveOccurred()) gomega.Expect(kubeClient).ToNot(gomega.BeNil()) + + apiExtensionClient, err = apiextensionsclient.NewForConfig(cfg) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gomega.Expect(apiExtensionClient).ToNot(gomega.BeNil()) + operatorClient, err = operatorclient.NewForConfig(cfg) gomega.Expect(err).ToNot(gomega.HaveOccurred()) gomega.Expect(kubeClient).ToNot(gomega.BeNil()) diff --git a/test/integration/klusterlet_test.go b/test/integration/klusterlet_test.go index ff71a2231ca..f5136d88328 100644 --- a/test/integration/klusterlet_test.go +++ b/test/integration/klusterlet_test.go @@ -90,6 +90,7 @@ var _ = ginkgo.Describe("Klusterlet", func() { workRoleName = fmt.Sprintf("open-cluster-management:%s-work:agent", klusterlet.Name) registrationSAName = fmt.Sprintf("%s-registration-sa", klusterlet.Name) workSAName = fmt.Sprintf("%s-work-sa", klusterlet.Name) + }) ginkgo.AfterEach(func() { @@ -106,12 +107,28 @@ var _ = ginkgo.Describe("Klusterlet", func() { if err != nil { return err } - if len(actual.Status.RelatedResources) != 13 { - return fmt.Errorf("should get 13 relatedResources, actual got %v", len(actual.Status.RelatedResources)) + + // 7 managed static manifests + 8 management static manifests + 2CRDs + 2 deployments(2 duplicated CRDs, but status also recorded in the klusterlet's status) + if len(actual.Status.RelatedResources) != 19 { + return fmt.Errorf("should get 19 relatedResources, actual got %v", len(actual.Status.RelatedResources)) } return nil }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + // Check CRDs + gomega.Eventually(func() bool { + if _, err := apiExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.Background(), "appliedmanifestworks.work.open-cluster-management.io", metav1.GetOptions{}); err != nil { + return false + } + return true + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue()) + gomega.Eventually(func() bool { + if _, err := apiExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.Background(), "clusterclaims.cluster.open-cluster-management.io", metav1.GetOptions{}); err != nil { + return false + } + return true + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue()) + // Check clusterrole/clusterrolebinding gomega.Eventually(func() bool { if _, err := kubeClient.RbacV1().ClusterRoles().Get(context.Background(), registrationRoleName, metav1.GetOptions{}); err != nil { @@ -145,12 +162,37 @@ var _ = ginkgo.Describe("Klusterlet", func() { } return true }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue()) + gomega.Eventually(func() bool { + if _, err := kubeClient.RbacV1().Roles(klusterletNamespace).Get(context.Background(), workRoleName, metav1.GetOptions{}); err != nil { + return false + } + return true + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue()) gomega.Eventually(func() bool { if _, err := kubeClient.RbacV1().RoleBindings(klusterletNamespace).Get(context.Background(), registrationRoleName, metav1.GetOptions{}); err != nil { return false } return true }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue()) + gomega.Eventually(func() bool { + if _, err := kubeClient.RbacV1().RoleBindings(klusterletNamespace).Get(context.Background(), workRoleName, metav1.GetOptions{}); err != nil { + return false + } + return true + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue()) + // Check extension apiserver rolebinding + gomega.Eventually(func() bool { + if _, err := kubeClient.RbacV1().RoleBindings("kube-system").Get(context.Background(), registrationRoleName, metav1.GetOptions{}); err != nil { + return false + } + return true + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue()) + gomega.Eventually(func() bool { + if _, err := kubeClient.RbacV1().RoleBindings("kube-system").Get(context.Background(), workRoleName, metav1.GetOptions{}); err != nil { + return false + } + return true + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue()) // Check service account gomega.Eventually(func() bool {