From 174110689d2b28dd921452f6c2576f3357518b00 Mon Sep 17 00:00:00 2001 From: adamzhoul <770822772@qq.com> Date: Mon, 11 Oct 2021 19:31:29 +0800 Subject: [PATCH] add node-servant --- cmd/yurt-node-servant/convert/convert.go | 64 ++++++++ cmd/yurt-node-servant/node-servant.go | 49 ++++++ cmd/yurt-node-servant/revert/revert.go | 53 +++++++ pkg/node-servant/components/kubelet.go | 182 +++++++++++++++++++++++ pkg/node-servant/components/node.go | 96 ++++++++++++ pkg/node-servant/components/yurthub.go | 166 +++++++++++++++++++++ pkg/node-servant/constant.go | 112 ++++++++++++++ pkg/node-servant/convert/convert.go | 129 ++++++++++++++++ pkg/node-servant/convert/options.go | 107 +++++++++++++ pkg/node-servant/job.go | 71 +++++++++ pkg/node-servant/revert/options.go | 75 ++++++++++ pkg/node-servant/revert/revert.go | 103 +++++++++++++ 12 files changed, 1207 insertions(+) create mode 100644 cmd/yurt-node-servant/convert/convert.go create mode 100644 cmd/yurt-node-servant/node-servant.go create mode 100644 cmd/yurt-node-servant/revert/revert.go create mode 100644 pkg/node-servant/components/kubelet.go create mode 100644 pkg/node-servant/components/node.go create mode 100644 pkg/node-servant/components/yurthub.go create mode 100644 pkg/node-servant/constant.go create mode 100644 pkg/node-servant/convert/convert.go create mode 100644 pkg/node-servant/convert/options.go create mode 100644 pkg/node-servant/job.go create mode 100644 pkg/node-servant/revert/options.go create mode 100644 pkg/node-servant/revert/revert.go diff --git a/cmd/yurt-node-servant/convert/convert.go b/cmd/yurt-node-servant/convert/convert.go new file mode 100644 index 00000000000..2d97176801f --- /dev/null +++ b/cmd/yurt-node-servant/convert/convert.go @@ -0,0 +1,64 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package convert + +import ( + "time" + + "github.com/spf13/cobra" + "k8s.io/klog" + + nodeConverter "github.com/openyurtio/openyurt/pkg/node-servant/convert" +) + +const ( + // defaultYurthubHealthCheckTimeout defines the default timeout for yurthub health check phase + defaultYurthubHealthCheckTimeout = 2 * time.Minute +) + +// NewConvertCmd generates a new convert command +func NewConvertCmd() *cobra.Command { + o := nodeConverter.NewConvertOptions() + cmd := &cobra.Command{ + Use: "convert --working-mode", + Short: "", + Run: func(cmd *cobra.Command, args []string) { + if err := o.Complete(cmd.Flags()); err != nil { + klog.Fatalf("fail to complete the convert option: %s", err) + } + + converter := nodeConverter.NewConverterWithOptions(o) + if err := converter.Do(); err != nil { + klog.Fatalf("fail to convert the kubernetes node to a yurt node: %s", err) + } + }, + } + setFlags(cmd) + + return cmd +} + +// setFlags sets flags. +func setFlags(cmd *cobra.Command) { + cmd.Flags().String("yurthub-image", "openyurt/yurthub:latest", + "The yurthub image.") + cmd.Flags().Duration("yurthub-healthcheck-timeout", defaultYurthubHealthCheckTimeout, + "The timeout for yurthub health check.") + cmd.Flags().String("kubeadm-conf-path", "", + "The path to kubelet service conf that is used by kubelet component to join the cluster on the edge node.") + cmd.Flags().String("join-token", "", "The token used by yurthub for joining the cluster.") +} diff --git a/cmd/yurt-node-servant/node-servant.go b/cmd/yurt-node-servant/node-servant.go new file mode 100644 index 00000000000..316131cc8d5 --- /dev/null +++ b/cmd/yurt-node-servant/node-servant.go @@ -0,0 +1,49 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "math/rand" + "os" + "time" + + "github.com/openyurtio/openyurt/cmd/yurt-node-servant/convert" + "github.com/openyurtio/openyurt/cmd/yurt-node-servant/revert" + "github.com/openyurtio/openyurt/pkg/projectinfo" + "github.com/spf13/cobra" +) + +// node-servant +// running on specific node, do convert/revert job +// yurtctl convert/revert join/reset, yurtcluster operator shall start a k8s job to run this. +func main() { + rand.Seed(time.Now().UnixNano()) + + version := fmt.Sprintf("%#v", projectinfo.Get()) + rootCmd := &cobra.Command{ + Use: "node-servant", + Short: "node-servant do convert/revert specific node", + Version: version, + } + rootCmd.AddCommand(convert.NewConvertCmd()) + rootCmd.AddCommand(revert.NewRevertCmd()) + + if err := rootCmd.Execute(); err != nil { // run command + os.Exit(1) + } +} diff --git a/cmd/yurt-node-servant/revert/revert.go b/cmd/yurt-node-servant/revert/revert.go new file mode 100644 index 00000000000..c5f85a61ebc --- /dev/null +++ b/cmd/yurt-node-servant/revert/revert.go @@ -0,0 +1,53 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package revert + +import ( + nodeReverter "github.com/openyurtio/openyurt/pkg/node-servant/revert" + + "github.com/openyurtio/openyurt/pkg/node-servant/revert" + "github.com/spf13/cobra" + "k8s.io/klog" +) + +// NewRevertCmd generates a new revert command +func NewRevertCmd() *cobra.Command { + o := nodeReverter.NewRevertOptions() + cmd := &cobra.Command{ + Use: "revert", + Short: "", + Run: func(cmd *cobra.Command, args []string) { + if err := o.Complete(cmd.Flags()); err != nil { + klog.Fatalf("fail to complete the revert option: %s", err) + } + + r := revert.NewReverterWithOptions(o) + if err := r.Do(); err != nil { + klog.Fatalf("fail to revert the yurt node to a kubernetes node: %s", err) + } + }, + } + setFlags(cmd) + + return cmd +} + +// setFlags sets flags. +func setFlags(cmd *cobra.Command) { + cmd.Flags().String("kubeadm-conf-path", "", + "The path to kubelet service conf that is used by kubelet component to join the cluster on the edge node.") +} diff --git a/pkg/node-servant/components/kubelet.go b/pkg/node-servant/components/kubelet.go new file mode 100644 index 00000000000..ecc7516463e --- /dev/null +++ b/pkg/node-servant/components/kubelet.go @@ -0,0 +1,182 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package components + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + + "k8s.io/klog" + + enutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" +) + +const ( + kubeletConfigRegularExpression = "\\-\\-kubeconfig=.*kubelet.conf" + apiserverAddrRegularExpression = "server: (http(s)?:\\/\\/)?[\\w][-\\w]{0,62}(\\.[\\w][-\\w]{0,62})*(:[\\d]{1,5})?" + + dirMode = 0755 +) + +type kubeletOperator struct { + openyurtDir string +} + +func NewKubeletOperator(openyurtDir string) *kubeletOperator { + return &kubeletOperator{ + openyurtDir: openyurtDir, + } +} + +// RedirectTrafficToYurtHub +func (op *kubeletOperator) RedirectTrafficToYurtHub() error { + // 1. create a working dir to store revised kubelet.conf + configPath, err := op.writeYurthubKubeletConfig() + if err != nil { + return err + } + + // 2. append /var/lib/kubelet/kubeadm-flags.env + if err := op.appendConfig(configPath); err != nil { + return err + } + + // 3. restart + return restartKubeletService() +} + +// UndoRedirectTrafficToYurtHub +func (op *kubeletOperator) UndoRedirectTrafficToYurtHub() error { + if err := op.undoAppendConfig(); err != nil { + return err + } + + if err := restartKubeletService(); err != nil { + return err + } + + if err := op.undoWriteYurthubKubeletConfig(); err != nil { + return err + } + + return nil +} + +func (op *kubeletOperator) writeYurthubKubeletConfig() (string, error) { + err := os.MkdirAll(op.openyurtDir, dirMode) + if err != nil { + return "", err + } + fullPath := op.getYurthubKubeletConf() + err = ioutil.WriteFile(fullPath, []byte(enutil.OpenyurtKubeletConf), fileMode) + if err != nil { + return "", err + } + klog.Infof("revised kubeconfig %s is generated", fullPath) + return fullPath, nil +} + +func (op *kubeletOperator) undoWriteYurthubKubeletConfig() error { + yurtKubeletConf := op.getYurthubKubeletConf() + return os.Remove(yurtKubeletConf) +} + +func (op *kubeletOperator) appendConfig(configPath string) error { + // set env KUBELET_KUBEADM_ARGS, args set later will override before + // ExecStart: kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS + file, err := os.OpenFile("/var/lib/kubelet/kubeadm-flags.env", os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer func() { + _ = file.Close() + }() + + kubeConfigSetup := fmt.Sprintf("KUBELET_KUBEADM_ARGS=$KUBELET_KUBEADM_ARGS\""+ + "--kubeconfig=%s -bootstrap-kubeconfig=\"", configPath) + + // if wrote, return + contentbyte, err := ioutil.ReadFile("/var/lib/kubelet/kubeadm-flags.env") + if err != nil { + return err + } + if strings.Contains(string(contentbyte), kubeConfigSetup) { + klog.Info("kubeConfigSetup has wrote before") + return nil + } + + if _, err := file.WriteString(kubeConfigSetup); err != nil { + return err + } + + return nil +} + +func (op *kubeletOperator) undoAppendConfig() error { + configPath := op.getYurthubKubeletConf() + kubeConfigSetup := fmt.Sprintf("KUBELET_KUBEADM_ARGS=$KUBELET_KUBEADM_ARGS\""+ + "--kubeconfig=%s -bootstrap-kubeconfig=\"", configPath) + contentbyte, err := ioutil.ReadFile("/var/lib/kubelet/kubeadm-flags.env") + if err != nil { + return err + } + + content := strings.Replace(string(contentbyte), kubeConfigSetup, "", -1) + err = ioutil.WriteFile("/var/lib/kubelet/kubeadm-flags.env", []byte(content), 0644) + if err != nil { + return err + } + + return nil +} + +func (op *kubeletOperator) getYurthubKubeletConf() string { + return filepath.Join(op.openyurtDir, enutil.KubeletConfName) +} + +func restartKubeletService() error { + klog.Info(enutil.DaemonReload) + cmd := exec.Command("bash", "-c", enutil.DaemonReload) + if err := enutil.Exec(cmd); err != nil { + return err + } + klog.Info(enutil.RestartKubeletSvc) + cmd = exec.Command("bash", "-c", enutil.RestartKubeletSvc) + if err := enutil.Exec(cmd); err != nil { + return err + } + klog.Infof("kubelet has been restarted") + return nil +} + +func GetApiServerAddress(kubeadmConfPath string) (string, error) { + kubeletConfPath, err := enutil.GetSingleContentFromFile(kubeadmConfPath, kubeletConfigRegularExpression) + if err != nil { + return "", err + } + kubeletConfPath = strings.Split(kubeletConfPath, "=")[1] + apiserverAddr, err := enutil.GetSingleContentFromFile(kubeletConfPath, apiserverAddrRegularExpression) + if err != nil { + return "", err + } + apiserverAddr = strings.Split(apiserverAddr, " ")[1] + return apiserverAddr, nil +} diff --git a/pkg/node-servant/components/node.go b/pkg/node-servant/components/node.go new file mode 100644 index 00000000000..338729af403 --- /dev/null +++ b/pkg/node-servant/components/node.go @@ -0,0 +1,96 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package components + +import ( + "context" + + nodeutil "github.com/openyurtio/openyurt/pkg/controller/util/node" + "github.com/openyurtio/openyurt/pkg/projectinfo" + "github.com/openyurtio/openyurt/pkg/yurtctl/constants" + kubeutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/kubernetes" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/klog" +) + +type nodeOperator struct { + clientSet *kubernetes.Clientset +} + +func NewNodeOperator() *nodeOperator { + return &nodeOperator{} +} + +func (op *nodeOperator) LabelEdgeAndOpenAutonomous(nodeName string) error { + klog.Infof("mark %s as the edge-node", nodeName) + + node, err := op.clientSet.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + node, err = kubeutil.LabelNode(op.clientSet, node, projectinfo.GetEdgeWorkerLabelKey(), "true") + if err != nil { + return err + } + + klog.Infof("open the %s autonomous", nodeName) + _, err = kubeutil.AnnotateNode(op.clientSet, node, constants.AnnotationAutonomy, "true") + if err != nil { + return err + } + + return nil +} + +func (op *nodeOperator) UndoLabelEdgeAndCloseAutonomous(nodeName string) error { + klog.Infof("clear %s label&annotation ", nodeName) + + node, err := op.clientSet.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + //var foundLabel bool + var foundLabel, foundAutonomy bool + + if _, foundLabel = node.Labels[projectinfo.GetEdgeWorkerLabelKey()]; foundLabel { + delete(node.Labels, projectinfo.GetEdgeWorkerLabelKey()) + } + + if _, foundAutonomy = node.Annotations[constants.AnnotationAutonomy]; foundAutonomy { + delete(node.Annotations, constants.AnnotationAutonomy) + } + + if !foundLabel && !foundAutonomy { + klog.Infof("node %s is not labeled or annotationAutonomy", nodeName) + return nil + } + + if _, err = op.clientSet.CoreV1().Nodes().Update(context.Background(), node, metav1.UpdateOptions{}); err != nil { + return err + } + + klog.Infof("clear %s label&annotation done", nodeName) + return nil +} + +func IsNodeReady(status *v1.NodeStatus) bool { + _, condition := nodeutil.GetNodeCondition(status, v1.NodeReady) + return condition != nil && condition.Status == v1.ConditionTrue +} diff --git a/pkg/node-servant/components/yurthub.go b/pkg/node-servant/components/yurthub.go new file mode 100644 index 00000000000..2b16dc0a044 --- /dev/null +++ b/pkg/node-servant/components/yurthub.go @@ -0,0 +1,166 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package components + +import ( + "fmt" + + "io/ioutil" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + enutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/hubself" + "github.com/openyurtio/openyurt/pkg/yurthub/util" + + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" +) + +const ( + hubHealthzCheckFrequency = 10 * time.Second + fileMode = 0666 +) + +type yurtHubOperator struct { + apiServerAddr string + yurthubImage string + joinToken string + workingMode util.WorkingMode + yurthubHealthCheckTimeout time.Duration +} + +func NewYurthubOperator(apiServerAddr string, yurthubImage string, joinToken string, + workingMode util.WorkingMode, yurthubHealthCheckTimeout time.Duration) *yurtHubOperator { + return &yurtHubOperator{ + apiServerAddr: apiServerAddr, + yurthubImage: yurthubImage, + joinToken: joinToken, + workingMode: workingMode, + yurthubHealthCheckTimeout: yurthubHealthCheckTimeout, + } +} + +// Install +func (op *yurtHubOperator) Install() error { + + // 1. put yurt-hub yaml into /etc/kubernetes/manifests + klog.Infof("setting up yurthub on node") + + // 1-1. replace variables in yaml file + klog.Infof("setting up yurthub apiServer addr") + yurthubTemplate := enutil.ReplaceRegularExpression(enutil.YurthubTemplate, + map[string]string{ + "__kubernetes_service_addr__": op.apiServerAddr, + "__yurthub_image__": op.yurthubImage, + "__join_token__": op.joinToken, + "__working_mode__": string(op.workingMode), + }) + + // 1-2. create yurthub.yaml + podManifestPath := enutil.GetPodManifestPath() + if err := enutil.EnsureDir(podManifestPath); err != nil { + return err + } + err := ioutil.WriteFile(getYurthubYaml(podManifestPath), []byte(yurthubTemplate), fileMode) + if err != nil { + return err + } + klog.Infof("create the %s/yurt-hub.yaml", podManifestPath) + + // 2. wait yurthub pod to be ready + return hubHealthcheck(op.yurthubHealthCheckTimeout) +} + +// UnInstall +func (op *yurtHubOperator) UnInstall() error { + // 1. remove the yurt-hub.yaml to delete the yurt-hub + podManifestPath := enutil.GetPodManifestPath() + yurthubYamlPath := getYurthubYaml(podManifestPath) + err := os.Remove(yurthubYamlPath) + if err != nil { + return err + } + + // 2. remove yurt-hub config directory and certificates in it + yurthubConf := getYurthubConf() + err = os.RemoveAll(yurthubConf) + if err != nil { + return err + } + klog.Infof("yurt-hub has been removed") + return nil +} + +func getYurthubYaml(podManifestPath string) string { + return filepath.Join(podManifestPath, enutil.YurthubYamlName) +} + +func getYurthubConf() string { + return filepath.Join(hubself.HubRootDir, hubself.HubName) +} + +// hubHealthcheck will check the status of yurthub pod +func hubHealthcheck(timeout time.Duration) error { + serverHealthzURL, err := url.Parse(fmt.Sprintf("http://%s", enutil.ServerHealthzServer)) + if err != nil { + return err + } + serverHealthzURL.Path = enutil.ServerHealthzURLPath + + start := time.Now() + return wait.PollImmediate(hubHealthzCheckFrequency, timeout, func() (bool, error) { + _, err := pingClusterHealthz(http.DefaultClient, serverHealthzURL.String()) + if err != nil { + klog.Infof("yurt-hub is not ready, ping cluster healthz with result: %v", err) + return false, nil + } + klog.Infof("yurt-hub healthz is OK after %f seconds", time.Since(start).Seconds()) + return true, nil + }) +} + +func pingClusterHealthz(client *http.Client, addr string) (bool, error) { + if client == nil { + return false, fmt.Errorf("http client is invalid") + } + + resp, err := client.Get(addr) + if err != nil { + return false, err + } + + b, err := ioutil.ReadAll(resp.Body) + defer resp.Body.Close() + if err != nil { + return false, fmt.Errorf("failed to read response of cluster healthz, %v", err) + } + + if resp.StatusCode != http.StatusOK { + return false, fmt.Errorf("response status code is %d", resp.StatusCode) + } + + if strings.ToLower(string(b)) != "ok" { + return false, fmt.Errorf("cluster healthz is %s", string(b)) + } + + return true, nil +} diff --git a/pkg/node-servant/constant.go b/pkg/node-servant/constant.go new file mode 100644 index 00000000000..bbf3c8c6cf7 --- /dev/null +++ b/pkg/node-servant/constant.go @@ -0,0 +1,112 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node_servant + +const ( + + // ConvertJobNameBase is the prefix of the convert ServantJob name + ConvertJobNameBase = "yurtctl-servant-convert" + // RevertJobNameBase is the prefix of the revert ServantJob name + RevertJobNameBase = "yurtctl-servant-revert" + + // ConvertServantJobTemplate defines the yurtctl convert servant job in yaml format + ConvertServantJobTemplate = ` +apiVersion: batch/v1 +kind: Job +metadata: + name: {{.jobName}} + namespace: kube-system +spec: + template: + spec: + hostPID: true + hostNetwork: true + restartPolicy: OnFailure + nodeName: {{.nodeName}} + volumes: + - name: host-var-tmp + hostPath: + path: /var/tmp + type: Directory + containers: + - name: yurtctl-servant + image: {{.yurtctl_servant_image}} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + args: + - "cp /usr/local/bin/yurtctl /tmp && nsenter -t 1 -m -u -n -i -- /var/tmp/yurtctl convert {{.sub_command}} --yurthub-image {{.yurthub_image}} {{if .yurthub_healthcheck_timeout}}--yurthub-healthcheck-timeout {{.yurthub_healthcheck_timeout}} {{end}}--join-token {{.joinToken}} && rm /tmp/yurtctl" + securityContext: + privileged: true + volumeMounts: + - mountPath: /tmp + name: host-var-tmp + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{if .kubeadm_conf_path }} + - name: KUBELET_SVC + value: {{.kubeadm_conf_path}} + {{end}} +` + // RevertServantJobTemplate defines the yurtctl revert servant job in yaml format + RevertServantJobTemplate = ` +apiVersion: batch/v1 +kind: Job +metadata: + name: {{.jobName}} + namespace: kube-system +spec: + template: + spec: + hostPID: true + hostNetwork: true + restartPolicy: OnFailure + nodeName: {{.nodeName}} + volumes: + - name: host-var-tmp + hostPath: + path: /var/tmp + type: Directory + containers: + - name: yurtctl-servant + image: {{.yurtctl_servant_image}} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + args: + - "cp /usr/local/bin/yurtctl /tmp && nsenter -t 1 -m -u -n -i -- /var/tmp/yurtctl revert {{.sub_command}} && rm /tmp/yurtctl" + securityContext: + privileged: true + volumeMounts: + - mountPath: /tmp + name: host-var-tmp + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{if .kubeadm_conf_path }} + - name: KUBELET_SVC + value: {{.kubeadm_conf_path}} + {{end}} +` +) diff --git a/pkg/node-servant/convert/convert.go b/pkg/node-servant/convert/convert.go new file mode 100644 index 00000000000..bdbc70f3e8d --- /dev/null +++ b/pkg/node-servant/convert/convert.go @@ -0,0 +1,129 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package convert + +import ( + "context" + "fmt" + + "github.com/openyurtio/openyurt/pkg/node-servant/components" + "github.com/openyurtio/openyurt/pkg/projectinfo" + enutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" + kubeutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/kubernetes" + "github.com/openyurtio/openyurt/pkg/yurthub/util" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// NodeConverter +type nodeConverter struct { + Options +} + +// NewConverter +func NewConverterWithOptions(o *Options) *nodeConverter { + return &nodeConverter{ + *o, + } +} + +// Do, do the convert job. +// shall be implemented as idempotent, can execute multiple times with no side-affect. +func (n *nodeConverter) Do() error { + if err := n.validateOptions(); err != nil { + return err + } + if err := n.preflightCheck(); err != nil { + return err + } + + if err := n.installYurtHub(); err != nil { + return err + } + if err := n.convertKubelet(); err != nil { + return err + } + + if err := n.labelEdgeAndOpenAutonomous(); err != nil { + return err + } + + return nil +} + +func (n *nodeConverter) validateOptions() error { + if n.workingMode != util.WorkingModeEdge && n.workingMode != util.WorkingModeCloud { + return fmt.Errorf("workingMode must be pointed out as cloud or edge. got %s", n.workingMode) + } + + return nil +} + +func (n *nodeConverter) preflightCheck() error { + // 1. check the server version + if err := kubeutil.ValidateServerVersion(n.clientSet); err != nil { + return err + } + + // 2. check if critical files exist + if _, err := enutil.FileExists(n.kubeadmConfPath); err != nil { + return err + } + + // 3. check the state of node + node, err := n.clientSet.CoreV1().Nodes().Get(context.Background(), n.nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + if !components.IsNodeReady(&node.Status) { + return fmt.Errorf("cannot do the convert, the status of node: %s is not 'Ready'", node.Name) + } + + // 4. check the label of node + labelVal, ok := node.Labels[projectinfo.GetEdgeWorkerLabelKey()] + if n.workingMode == util.WorkingModeEdge && ok { + return fmt.Errorf("cannot do the convert, the edge node: %s is not a Kubernetes node", node.Name) + } + if n.workingMode == util.WorkingModeCloud && labelVal != "false" { + return fmt.Errorf("cannot do the convert, the cloud node: %s is not a Kubernetes node", node.Name) + } + return nil +} + +func (n *nodeConverter) installYurtHub() error { + apiServerAddress, err := components.GetApiServerAddress(n.kubeadmConfPath) + if err != nil { + return err + } + op := components.NewYurthubOperator(apiServerAddress, n.yurthubImage, n.joinToken, + n.workingMode, n.yurthubHealthCheckTimeout) + return op.Install() +} + +func (n *nodeConverter) convertKubelet() error { + op := components.NewKubeletOperator(n.openyurtDir) + return op.RedirectTrafficToYurtHub() +} + +func (n *nodeConverter) labelEdgeAndOpenAutonomous() error { + if n.workingMode != util.WorkingModeEdge { + return nil + } + + op := components.NewNodeOperator() + return op.LabelEdgeAndOpenAutonomous(n.nodeName) +} diff --git a/pkg/node-servant/convert/options.go b/pkg/node-servant/convert/options.go new file mode 100644 index 00000000000..370f0fb7779 --- /dev/null +++ b/pkg/node-servant/convert/options.go @@ -0,0 +1,107 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package convert + +import ( + "os" + "time" + + // todo: move util out of yurtctl + enutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" + kubeutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/kubernetes" + "github.com/openyurtio/openyurt/pkg/yurthub/util" + + "github.com/spf13/pflag" + "k8s.io/client-go/kubernetes" +) + +// Options has the information that required by convert operation +type Options struct { + clientSet *kubernetes.Clientset + + yurthubImage string + yurthubHealthCheckTimeout time.Duration + workingMode util.WorkingMode + + joinToken string + kubeadmConfPath string + openyurtDir string + nodeName string +} + +// NewConvertOptions creates a new Options +func NewConvertOptions() *Options { + return &Options{} +} + +// Complete completes all the required options. +func (o *Options) Complete(flags *pflag.FlagSet) error { + yurthubImage, err := flags.GetString("yurthub-image") + if err != nil { + return err + } + o.yurthubImage = yurthubImage + + yurthubHealthCheckTimeout, err := flags.GetDuration("yurthub-healthcheck-timeout") + if err != nil { + return err + } + o.yurthubHealthCheckTimeout = yurthubHealthCheckTimeout + + kubeadmConfPath, err := flags.GetString("kubeadm-conf-path") + if err != nil { + return err + } + if kubeadmConfPath == "" { + kubeadmConfPath = os.Getenv("KUBELET_SVC") + } + if kubeadmConfPath == "" { + kubeadmConfPath = enutil.KubeletSvcPath + } + o.kubeadmConfPath = kubeadmConfPath + + nodeName, err := enutil.GetNodeName(kubeadmConfPath) + if err != nil { + return err + } + o.nodeName = nodeName + + o.clientSet, err = enutil.GenClientSet(flags) + if err != nil { + return err + } + + joinToken, err := flags.GetString("join-token") + if err != nil { + return err + } + if joinToken == "" { + joinToken, err = kubeutil.GetOrCreateJoinTokenString(o.clientSet) + if err != nil { + return err + } + } + o.joinToken = joinToken + + openyurtDir := os.Getenv("OPENYURT_DIR") + if openyurtDir == "" { + openyurtDir = enutil.OpenyurtDir + } + o.openyurtDir = openyurtDir + + return nil +} diff --git a/pkg/node-servant/job.go b/pkg/node-servant/job.go new file mode 100644 index 00000000000..06d7a70c631 --- /dev/null +++ b/pkg/node-servant/job.go @@ -0,0 +1,71 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node_servant + +import ( + "fmt" + + tmplutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/templates" + batchv1 "k8s.io/api/batch/v1" + k8sruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/client-go/kubernetes/scheme" +) + +// RenderNodeServantJob return k8s job +// to start k8s job to run convert/revert on specific node +func RenderNodeServantJob(action string, tmplCtx map[string]string, nodeName string) (*batchv1.Job, error) { + var servantJobTemplate, jobBaseName string + switch action { + case "convert": + servantJobTemplate = ConvertServantJobTemplate + jobBaseName = ConvertJobNameBase + case "revert": + servantJobTemplate = RevertServantJobTemplate + jobBaseName = RevertJobNameBase + default: + return nil, fmt.Errorf("action invalied: %s ", action) + } + + tmplCtx["jobName"] = jobBaseName + "-" + nodeName + tmplCtx["nodeName"] = nodeName + jobYaml, err := tmplutil.SubsituteTemplate(servantJobTemplate, tmplCtx) + if err != nil { + return nil, err + } + + srvJobObj, err := YamlToObject([]byte(jobYaml)) + if err != nil { + return nil, err + } + srvJob, ok := srvJobObj.(*batchv1.Job) + if !ok { + return nil, fmt.Errorf("fail to assert yurtctl-servant job") + } + + return srvJob, nil +} + +// YamlToObject deserializes object in yaml format to a runtime.Object +func YamlToObject(yamlContent []byte) (k8sruntime.Object, error) { + decode := serializer.NewCodecFactory(scheme.Scheme).UniversalDeserializer().Decode + obj, _, err := decode(yamlContent, nil, nil) + if err != nil { + return nil, err + } + return obj, nil +} diff --git a/pkg/node-servant/revert/options.go b/pkg/node-servant/revert/options.go new file mode 100644 index 00000000000..529813dfeb2 --- /dev/null +++ b/pkg/node-servant/revert/options.go @@ -0,0 +1,75 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package revert + +import ( + "os" + + enutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" + + "github.com/spf13/pflag" + "k8s.io/client-go/kubernetes" +) + +// Options has the information that required by revert operation +type Options struct { + clientSet *kubernetes.Clientset + + kubeadmConfPath string + openyurtDir string + nodeName string +} + +// NewRevertOptions creates a new Options +func NewRevertOptions() *Options { + return &Options{} +} + +// Complete completes all the required options. +func (o *Options) Complete(flags *pflag.FlagSet) error { + + kubeadmConfPath, err := flags.GetString("kubeadm-conf-path") + if err != nil { + return err + } + if kubeadmConfPath == "" { + kubeadmConfPath = os.Getenv("KUBELET_SVC") + } + if kubeadmConfPath == "" { + kubeadmConfPath = enutil.KubeletSvcPath + } + o.kubeadmConfPath = kubeadmConfPath + + nodeName, err := enutil.GetNodeName(kubeadmConfPath) + if err != nil { + return err + } + o.nodeName = nodeName + + o.clientSet, err = enutil.GenClientSet(flags) + if err != nil { + return err + } + + openyurtDir := os.Getenv("OPENYURT_DIR") + if openyurtDir == "" { + openyurtDir = enutil.OpenyurtDir + } + o.openyurtDir = openyurtDir + + return nil +} diff --git a/pkg/node-servant/revert/revert.go b/pkg/node-servant/revert/revert.go new file mode 100644 index 00000000000..bfaef6f8f87 --- /dev/null +++ b/pkg/node-servant/revert/revert.go @@ -0,0 +1,103 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package revert + +import ( + "context" + "fmt" + "time" + + "github.com/openyurtio/openyurt/pkg/node-servant/components" + "github.com/openyurtio/openyurt/pkg/yurthub/util" + + kubeutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/kubernetes" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// NodeReverter +type nodeReverter struct { + Options +} + +// NewReverterWithOptions +func NewReverterWithOptions(o *Options) *nodeReverter { + return &nodeReverter{ + *o, + } +} + +// Do, do the convert job +// shall be implemented as idempotent, can execute multiple times with no side-affect. +func (n *nodeReverter) Do() error { + if err := n.validateOptions(); err != nil { + return err + } + if err := n.preflightCheck(); err != nil { + return err + } + + if err := n.revertKubelet(); err != nil { + return err + } + if err := n.unInstallYurtHub(); err != nil { + return err + } + + if err := n.unLabelEdgeAndOpenAutonomous(); err != nil { + return err + } + + return nil +} + +func (n *nodeReverter) validateOptions() error { + return nil +} + +func (n *nodeReverter) preflightCheck() error { + // 1. check the server version + if err := kubeutil.ValidateServerVersion(n.clientSet); err != nil { + return err + } + + // 2. check the state of Nodes + node, err := n.clientSet.CoreV1().Nodes().Get(context.Background(), n.nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + if !components.IsNodeReady(&node.Status) { + return fmt.Errorf("cannot do the convert, the status of node: %s is not 'Ready'", node.Name) + } + + return nil +} + +func (n *nodeReverter) revertKubelet() error { + op := components.NewKubeletOperator(n.openyurtDir) + return op.UndoRedirectTrafficToYurtHub() +} + +func (n *nodeReverter) unInstallYurtHub() error { + op := components.NewYurthubOperator("", "", "", + util.WorkingModeCloud, time.Duration(1)) // params is not important here + return op.UnInstall() +} + +func (n *nodeReverter) unLabelEdgeAndOpenAutonomous() error { + op := components.NewNodeOperator() + return op.UndoLabelEdgeAndCloseAutonomous(n.nodeName) +}