From 2c83a0fb83965a6d2fefba4f0b04b82820d56591 Mon Sep 17 00:00:00 2001 From: wulemao <1194736083@qq.com> Date: Sun, 29 Sep 2024 20:01:19 +0800 Subject: [PATCH] support unregister cluster in karmada Signed-off-by: wulemao <1194736083@qq.com> --- pkg/karmadactl/karmadactl.go | 2 + pkg/karmadactl/unjoin/unjoin.go | 42 +--- pkg/karmadactl/unregister/unregister.go | 279 ++++++++++++++++++++++++ pkg/karmadactl/util/cluster.go | 67 ++++++ pkg/util/secret.go | 9 + 5 files changed, 358 insertions(+), 41 deletions(-) create mode 100644 pkg/karmadactl/unregister/unregister.go create mode 100644 pkg/karmadactl/util/cluster.go diff --git a/pkg/karmadactl/karmadactl.go b/pkg/karmadactl/karmadactl.go index 069c7c1ca58f..c0dff7ba96e7 100644 --- a/pkg/karmadactl/karmadactl.go +++ b/pkg/karmadactl/karmadactl.go @@ -56,6 +56,7 @@ import ( "github.com/karmada-io/karmada/pkg/karmadactl/token" "github.com/karmada-io/karmada/pkg/karmadactl/top" "github.com/karmada-io/karmada/pkg/karmadactl/unjoin" + "github.com/karmada-io/karmada/pkg/karmadactl/unregister" "github.com/karmada-io/karmada/pkg/karmadactl/util" utilcomp "github.com/karmada-io/karmada/pkg/karmadactl/util/completion" "github.com/karmada-io/karmada/pkg/version/sharedcommand" @@ -123,6 +124,7 @@ func NewKarmadaCtlCommand(cmdUse, parentCommand string) *cobra.Command { unjoin.NewCmdUnjoin(f, parentCommand), token.NewCmdToken(f, parentCommand, ioStreams), register.NewCmdRegister(parentCommand), + unregister.NewCmdUnregister(parentCommand), }, }, { diff --git a/pkg/karmadactl/unjoin/unjoin.go b/pkg/karmadactl/unjoin/unjoin.go index 4e0fa4450b0e..92faf95644c4 100644 --- a/pkg/karmadactl/unjoin/unjoin.go +++ b/pkg/karmadactl/unjoin/unjoin.go @@ -17,15 +17,11 @@ limitations under the License. package unjoin import ( - "context" "fmt" "time" "github.com/spf13/cobra" "github.com/spf13/pflag" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" kubeclient "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/klog/v2" @@ -187,7 +183,7 @@ func (j *CommandUnjoinOption) RunUnJoinCluster(controlPlaneRestConfig, clusterCo controlPlaneKarmadaClient := karmadaclientset.NewForConfigOrDie(controlPlaneRestConfig) // delete the cluster object in host cluster that associates the unjoining cluster - err := j.deleteClusterObject(controlPlaneKarmadaClient) + err := cmdutil.DeleteClusterObject(controlPlaneKarmadaClient, j.ClusterName, j.Wait, j.DryRun) if err != nil { klog.Errorf("Failed to delete cluster object. cluster name: %s, error: %v", j.ClusterName, err) return err @@ -225,42 +221,6 @@ func (j *CommandUnjoinOption) RunUnJoinCluster(controlPlaneRestConfig, clusterCo return nil } -// deleteClusterObject delete the cluster object in host cluster that associates the unjoining cluster -func (j *CommandUnjoinOption) deleteClusterObject(controlPlaneKarmadaClient *karmadaclientset.Clientset) error { - if j.DryRun { - return nil - } - - err := controlPlaneKarmadaClient.ClusterV1alpha1().Clusters().Delete(context.TODO(), j.ClusterName, metav1.DeleteOptions{}) - if apierrors.IsNotFound(err) { - return fmt.Errorf("no cluster object %s found in karmada control Plane", j.ClusterName) - } - if err != nil { - klog.Errorf("Failed to delete cluster object. cluster name: %s, error: %v", j.ClusterName, err) - return err - } - - // make sure the given cluster object has been deleted - err = wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, j.Wait, false, func(context.Context) (done bool, err error) { - _, err = controlPlaneKarmadaClient.ClusterV1alpha1().Clusters().Get(context.TODO(), j.ClusterName, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - return true, nil - } - if err != nil { - klog.Errorf("Failed to get cluster %s. err: %v", j.ClusterName, err) - return false, err - } - klog.Infof("Waiting for the cluster object %s to be deleted", j.ClusterName) - return false, nil - }) - if err != nil { - klog.Errorf("Failed to delete cluster object. cluster name: %s, error: %v", j.ClusterName, err) - return err - } - - return nil -} - // deleteRBACResources deletes the cluster role, cluster rolebindings from the unjoining cluster. func deleteRBACResources(clusterKubeClient kubeclient.Interface, unjoiningClusterName string, forceDeletion, dryRun bool) error { if dryRun { diff --git a/pkg/karmadactl/unregister/unregister.go b/pkg/karmadactl/unregister/unregister.go new file mode 100644 index 000000000000..0b03da61d884 --- /dev/null +++ b/pkg/karmadactl/unregister/unregister.go @@ -0,0 +1,279 @@ +/* +Copyright 2024 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package unregister + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeclient "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" + "k8s.io/kubectl/pkg/util/templates" + + karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" + "github.com/karmada-io/karmada/pkg/karmadactl/options" + "github.com/karmada-io/karmada/pkg/karmadactl/register" + cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util" + "github.com/karmada-io/karmada/pkg/karmadactl/util/apiclient" + "github.com/karmada-io/karmada/pkg/util" + "github.com/karmada-io/karmada/pkg/util/names" +) + +var ( + unregisterLong = templates.LongDesc(` + Remove a Pull mode cluster from Karmada control plane.`) + + unregisterExample = templates.Examples(` + # Unregister cluster from karmada control plane + %[1]s unregister CLUSTER_NAME + + # Unregister cluster from karmada control plane with timeout + %[1]s unregister CLUSTER_NAME --wait 2m + + # Unregister cluster from karmada control plane, explicitly specifying the kubeconfig and context of the member cluster + %[1]s unregister CLUSTER_NAME --kubeconfig= [--context=]`) +) + +// NewCmdUnregister defines the `unregister` command that removes registration of a pull mode cluster from control plane. +func NewCmdUnregister(parentCommand string) *cobra.Command { + opts := CommandUnregisterOption{} + + cmd := &cobra.Command{ + Use: "unregister CLUSTER_NAME", + Short: "Remove a pull mode cluster from Karmada control plane", + Long: unregisterLong, + Example: fmt.Sprintf(unregisterExample, parentCommand), + SilenceUsage: true, + DisableFlagsInUseLine: true, + RunE: func(_ *cobra.Command, args []string) error { + if err := opts.Complete(args); err != nil { + return err + } + if err := opts.Validate(args); err != nil { + return err + } + if err := opts.Run(); err != nil { + return err + } + return nil + }, + Annotations: map[string]string{ + cmdutil.TagCommandGroup: cmdutil.GroupClusterRegistration, + }, + } + + flags := cmd.Flags() + opts.AddFlags(flags) + + return cmd +} + +// CommandUnregisterOption holds all command options. +type CommandUnregisterOption struct { + // ClusterName is the cluster's name that we are going to unregister. + ClusterName string + + // KubeConfig holds the KUBECONFIG file path for the unregistering cluster. + KubeConfig string + + // Context is the name of the cluster context in KUBECONFIG file. + // Default value is the current-context. + Context string + + // Namespace is the namespace that karmada-agent component deployed. + Namespace string + + // ClusterNamespace holds namespace where the member cluster secrets are stored + ClusterNamespace string + + // Wait tells maximum command execution time + Wait time.Duration + + // DryRun tells if run the command in dry-run mode, without making any server requests. + DryRun bool + + controlPlaneClient *karmadaclientset.Clientset + memberClusterClient *kubeclient.Clientset +} + +// AddFlags adds flags to the specified FlagSet. +func (j *CommandUnregisterOption) AddFlags(flags *pflag.FlagSet) { + flags.StringVar(&j.KubeConfig, "kubeconfig", "", "Path to the kubeconfig file of member cluster.") + flags.StringVar(&j.Context, "context", "", "Name of the cluster context in kubeconfig file.") + flags.StringVarP(&j.Namespace, "namespace", "n", "karmada-system", "Namespace the karmada-agent component deployed.") + flags.StringVar(&j.ClusterNamespace, "cluster-namespace", options.DefaultKarmadaClusterNamespace, "Namespace in the control plane where member cluster secrets are stored.") + flags.DurationVar(&j.Wait, "wait", 60*time.Second, "wait for the unjoin command execution process(default 60s), if there is no success after this time, timeout will be returned.") + flags.BoolVar(&j.DryRun, "dry-run", false, "Run the command in dry-run mode, without making any server requests.") +} + +// Complete ensures that options are valid and marshals them if necessary. +func (j *CommandUnregisterOption) Complete(args []string) error { + // Get cluster name from the command args. + if len(args) > 0 { + j.ClusterName = args[0] + } + return nil +} + +// Validate ensures that command unjoin options are valid. +func (j *CommandUnregisterOption) Validate(args []string) error { + if len(args) > 1 { + return fmt.Errorf("only the cluster name is allowed as an argument") + } + if len(j.ClusterName) == 0 { + return fmt.Errorf("cluster name is required") + } + if j.Wait <= 0 { + return fmt.Errorf(" --wait %v must be a positive duration, e.g. 1m0s ", j.Wait) + } + return nil +} + +// Run is the implementation of the 'unregister' command. +func (j *CommandUnregisterOption) Run() error { + klog.V(1).Infof("Unregistering cluster. cluster name: %s", j.ClusterName) + klog.V(1).Infof("Unregistering cluster. karmada-agent deployed in namespace: %s", j.Namespace) + klog.V(1).Infof("Unregistering cluster. member cluster secrets stored in namespace: %s", j.ClusterNamespace) + + // 1. build member cluster client + restConfig, err := apiclient.RestConfig(j.Context, j.KubeConfig) + if err != nil { + return fmt.Errorf("failed to read member cluster rest config: %w", err) + } + j.memberClusterClient, err = apiclient.NewClientSet(restConfig) + if err != nil { + return fmt.Errorf("failed to build member cluster clientset: %w", err) + } + + // 2. build karmada control plane client (read kubeconfig from member cluster's secret which stores the karmada kubeconfig for karmada agent) + karmadaConfigSecret, err := j.memberClusterClient.CoreV1().Secrets(j.Namespace).Get(context.TODO(), register.KarmadaKubeconfigName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get the secret which stores the karmada kubeconfig") + } + karmadaCfg, err := clientcmd.Load(karmadaConfigSecret.Data[register.KarmadaKubeconfigName]) + if err != nil { + return err + } + if specifiedKarmadaContext := j.getSpecifiedKarmadaContext(j.memberClusterClient); specifiedKarmadaContext != "" { + karmadaCfg.CurrentContext = specifiedKarmadaContext + } + j.controlPlaneClient, err = register.ToKarmadaClient(karmadaCfg) + if err != nil { + return err + } + + return j.RunUnregisterCluster() +} + +type obj struct{ Kind, Name, Namespace string } + +func (o *obj) ToString() string { + if o.Namespace == "" { + return fmt.Sprintf("%s/%s", o.Kind, o.Name) + } else { + return fmt.Sprintf("%s/%s/%s", o.Kind, o.Namespace, o.Name) + } +} + +// RunUnregisterCluster unregister the pull mode cluster from karmada. +func (j *CommandUnregisterOption) RunUnregisterCluster() error { + if j.DryRun { + return nil + } + + // 1. delete the cluster object in host cluster that associates the unregistering cluster + if err := cmdutil.DeleteClusterObject(j.controlPlaneClient, j.ClusterName, j.Wait, j.DryRun); err != nil { + klog.Errorf("Failed to delete cluster object. cluster name: %s, error: %v", j.ClusterName, err) + return err + } + klog.Infof("Successfully delete cluster object (%s) from control plane.", j.ClusterName) + + // 2. delete resource created by karmada in member cluster + clusterResources := []obj{ + // 2.1 delete rbac resource which should upload to control plane to serve the controller-manager and aggregated-apiserver + {Kind: "ClusterRole", Name: names.GenerateRoleName(names.GenerateServiceAccountName(j.ClusterName))}, + {Kind: "ClusterRoleBinding", Name: names.GenerateRoleName(names.GenerateServiceAccountName(j.ClusterName))}, + {Kind: "ServiceAccount", Namespace: j.ClusterNamespace, Name: names.GenerateServiceAccountName(j.ClusterName)}, + {Kind: "Secret", Namespace: j.ClusterNamespace, Name: names.GenerateServiceAccountName(j.ClusterName)}, + {Kind: "ServiceAccount", Namespace: j.ClusterNamespace, Name: names.GenerateServiceAccountName("impersonator")}, + {Kind: "Secret", Namespace: j.ClusterNamespace, Name: names.GenerateServiceAccountName("impersonator")}, + {Kind: "Namespace", Name: j.ClusterNamespace}, + + // 2.2 delete karmada-agent deployment, its own rbac resource and secret/karmada-kubeconfig + {Kind: "ClusterRole", Name: register.KarmadaAgentName}, + {Kind: "ClusterRoleBinding", Name: register.KarmadaAgentName}, + {Kind: "ServiceAccount", Namespace: j.Namespace, Name: register.KarmadaAgentServiceAccountName}, + {Kind: "Secret", Namespace: j.Namespace, Name: register.KarmadaKubeconfigName}, + {Kind: "Deployment", Namespace: j.Namespace, Name: register.KarmadaAgentName}, + } + + var err error + for _, resource := range clusterResources { + switch resource.Kind { + case "ClusterRole": + err = util.DeleteClusterRole(j.memberClusterClient, resource.Name) + case "ClusterRoleBinding": + err = util.DeleteClusterRoleBinding(j.memberClusterClient, resource.Name) + case "ServiceAccount": + err = util.DeleteServiceAccount(j.memberClusterClient, resource.Namespace, resource.Name) + case "Secret": + err = util.DeleteSecret(j.memberClusterClient, resource.Namespace, resource.Name) + case "Deployment": + err = deleteDeployment(j.memberClusterClient, resource.Namespace, resource.Name) + case "Namespace": + err = util.DeleteNamespace(j.memberClusterClient, resource.Name) + } + + if err != nil { + klog.Errorf("Failed to delete (%v) in unregistering cluster %s: %+v.", resource, j.ClusterName, err) + return err + } + klog.Infof("Successfully delete resource (%v) from member cluster %s.", resource, j.ClusterName) + } + + return nil +} + +func (j *CommandUnregisterOption) getSpecifiedKarmadaContext(client kubeclient.Interface) string { + agent, err := client.AppsV1().Deployments(j.Namespace).Get(context.TODO(), register.KarmadaAgentName, metav1.GetOptions{}) + if err != nil { + return "" + } + const karmadaContextPrefix = "--karmada-context=" + for _, cmd := range agent.Spec.Template.Spec.Containers[0].Command { + if strings.HasPrefix(cmd, karmadaContextPrefix) { + return cmd[len(karmadaContextPrefix):] + } + } + return "" +} + +// deleteDeployment just try to delete the Deployment. +func deleteDeployment(client kubeclient.Interface, namespace, name string) error { + err := client.AppsV1().Deployments(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + return nil +} diff --git a/pkg/karmadactl/util/cluster.go b/pkg/karmadactl/util/cluster.go new file mode 100644 index 000000000000..74f3c633dda7 --- /dev/null +++ b/pkg/karmadactl/util/cluster.go @@ -0,0 +1,67 @@ +/* +Copyright 2024 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "context" + "fmt" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog/v2" + + karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" +) + +// DeleteClusterObject delete the cluster object in host cluster +func DeleteClusterObject(controlPlaneKarmadaClient *karmadaclientset.Clientset, clusterName string, + timeout time.Duration, dryRun bool) error { + if dryRun { + return nil + } + + err := controlPlaneKarmadaClient.ClusterV1alpha1().Clusters().Delete(context.TODO(), clusterName, metav1.DeleteOptions{}) + if apierrors.IsNotFound(err) { + return fmt.Errorf("no cluster object %s found in karmada control Plane", clusterName) + } + if err != nil { + klog.Errorf("Failed to delete cluster object. cluster name: %s, error: %v", clusterName, err) + return err + } + + // make sure the given cluster object has been deleted + err = wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, timeout, false, func(context.Context) (done bool, err error) { + _, err = controlPlaneKarmadaClient.ClusterV1alpha1().Clusters().Get(context.TODO(), clusterName, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return true, nil + } + if err != nil { + klog.Errorf("Failed to get cluster %s. err: %v", clusterName, err) + return false, err + } + klog.Infof("Waiting for the cluster object %s to be deleted", clusterName) + return false, nil + }) + if err != nil { + klog.Errorf("Failed to delete cluster object. cluster name: %s, error: %v", clusterName, err) + return err + } + + return nil +} diff --git a/pkg/util/secret.go b/pkg/util/secret.go index ab05bccfeff8..c4b93dc417e1 100644 --- a/pkg/util/secret.go +++ b/pkg/util/secret.go @@ -60,3 +60,12 @@ func PatchSecret(client kubeclient.Interface, namespace, name string, pt types.P } return nil } + +// DeleteSecret just try to delete the Secret. +func DeleteSecret(client kubeclient.Interface, namespace, name string) error { + err := client.CoreV1().Secrets(namespace).Delete(context.Background(), name, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + return nil +}