diff --git a/README.md b/README.md index 8cfc62ceb83..0642c3a11de 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,12 @@ To uninstall OpenYurt and revert back to the original Kubernetes cluster setting _output/bin/yurtctl revert ``` +To create OpenYurt cluster, you can run the following command: + +```bash +_output/bin/yurtctl init +``` + To join nodes to OpenYurt, you can run the following command: ```bash _output/bin/yurtctl join diff --git a/docs/tutorial/yurtctl.md b/docs/tutorial/yurtctl.md index addcb194cb9..2eae74024b2 100644 --- a/docs/tutorial/yurtctl.md +++ b/docs/tutorial/yurtctl.md @@ -204,6 +204,16 @@ Note that before performing the uninstall, please make sure all edge nodes are r In addition, the path of the kubelet service configuration can be set by the option `--kubeadm-conf-path`, and the path of the directory on edge node containing static pod files can be set by the option `--pod-manifest-path`. +## Create OpenYurt cluster +`yurtctl init` will create an OpenYurt cluster, but the user needs to install the runtime in advance and ensure that the swap partition of the node has been closed. + +Using `yurtctl` to create an OpenYurt cluster can be done by doing the following: +``` +$ _output/bin/yurtctl init --image-repository=registry.cn-hangzhou.aliyuncs.com/google_containers --kubernetes-version=v1.18.8 --pod-network-cidr=10.244.0.0/16 +``` +In addition, the OpenYurt components version can be set by the option `--yurt-version`, +and the OpenYurt components image registry can be set by the option `--yurt-image-registry`. + ## Join Edge-Node/Cloud-Node to OpenYurt `yurtctl join` will automatically install the corresponding kubelet according to the cluster version, but the user needs to install the runtime in advance and ensure that the swap partition of the node has been closed. diff --git a/go.mod b/go.mod index 5190620c18b..f0a52292e3f 100644 --- a/go.mod +++ b/go.mod @@ -31,11 +31,13 @@ require ( k8s.io/api v0.19.7 k8s.io/apimachinery v0.19.7 k8s.io/apiserver v0.18.8 + k8s.io/cli-runtime v0.18.8 k8s.io/client-go v0.19.2 k8s.io/cluster-bootstrap v0.0.0 k8s.io/component-base v0.18.8 k8s.io/klog v1.0.0 k8s.io/klog/v2 v2.0.0 + k8s.io/kubectl v0.0.0 k8s.io/kubelet v0.0.0 k8s.io/kubernetes v1.18.8 k8s.io/utils v0.0.0-20200603063816-c1c6865ac451 diff --git a/go.sum b/go.sum index 8fd8aa67d11..5e1d63e9994 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced34534/go.mod h1:iroGtC8B3tQiqtds1l+mgk/BBOrxbqjH+eUfFQYRc14= github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA= +github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd h1:sjQovDkwrZp8u+gxLtPgKGjk5hCxuy2hrRejBTA9xFU= github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= @@ -77,6 +78,7 @@ github.com/caddyserver/caddy v1.0.3/go.mod h1:G+ouvOY32gENkJC+jhgl62TyhvqEsFaDiZ github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/prettybench v0.0.0-20150116022406-03b8cfe5406c/go.mod h1:Xe6ZsFhtM8HrDku0pxJ3/Lr51rwykrzgFwpmTzleatY= +github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 h1:7aWHqerlJ41y6FOsEUvknqgXnGmJyJSbjhAWq5pO4F8= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= github.com/checkpoint-restore/go-criu v0.0.0-20181120144056-17b0214f6c48/go.mod h1:TrMrLQfeENAPYPRsJuq3jsqdlRh3lvi6trTZJG8+tho= github.com/cheekybits/genny v0.0.0-20170328200008-9127e812e1e9/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= @@ -149,6 +151,7 @@ github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdR github.com/evanphx/json-patch v0.0.0-20200808040245-162e5629780b/go.mod h1:NAJj0yf/KaRKURN6nyi7A9IZydMivZEm9oQLWNjfKDc= github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -450,6 +453,7 @@ github.com/mistifyio/go-zfs v2.1.1+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfv github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= +github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -547,6 +551,7 @@ github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= github.com/russross/blackfriday v0.0.0-20170610170232-067529f716f4/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= diff --git a/pkg/yurtctl/cmd/cmd.go b/pkg/yurtctl/cmd/cmd.go index 7efac05402e..acddae478c4 100644 --- a/pkg/yurtctl/cmd/cmd.go +++ b/pkg/yurtctl/cmd/cmd.go @@ -21,8 +21,6 @@ import ( "fmt" "os" - "github.com/spf13/cobra" - flag "github.com/spf13/pflag" "k8s.io/klog" "github.com/openyurtio/openyurt/pkg/projectinfo" @@ -32,6 +30,9 @@ import ( "github.com/openyurtio/openyurt/pkg/yurtctl/cmd/markautonomous" "github.com/openyurtio/openyurt/pkg/yurtctl/cmd/reset" "github.com/openyurtio/openyurt/pkg/yurtctl/cmd/revert" + "github.com/openyurtio/openyurt/pkg/yurtctl/cmd/yurtinit" + "github.com/spf13/cobra" + flag "github.com/spf13/pflag" ) // NewYurtctlCommand creates a new yurtctl command @@ -52,6 +53,7 @@ func NewYurtctlCommand() *cobra.Command { cmds.AddCommand(clusterinfo.NewClusterInfoCmd()) cmds.AddCommand(join.NewCmdJoin(os.Stdout, nil)) cmds.AddCommand(reset.NewCmdReset(os.Stdin, os.Stdout, nil)) + cmds.AddCommand(yurtinit.NewCmdInit(os.Stdout, nil)) klog.InitFlags(nil) // goflag.Parse() diff --git a/pkg/yurtctl/cmd/convert/convert.go b/pkg/yurtctl/cmd/convert/convert.go index 39f21590761..48a44a3d07d 100644 --- a/pkg/yurtctl/cmd/convert/convert.go +++ b/pkg/yurtctl/cmd/convert/convert.go @@ -26,7 +26,6 @@ import ( "github.com/spf13/pflag" "github.com/openyurtio/openyurt/pkg/projectinfo" - "github.com/openyurtio/openyurt/pkg/yurtctl/constants" "github.com/openyurtio/openyurt/pkg/yurtctl/lock" kubeutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/kubernetes" strutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/strings" @@ -327,35 +326,10 @@ func (co *ConvertOptions) RunConvert() (err error) { } // 3. deploy yurt controller manager - // create a service account for yurt-controller-manager - err = kubeutil.CreateServiceAccountFromYaml(co.clientSet, - "kube-system", constants.YurtControllerManagerServiceAccount) - if err != nil { - return - } - // create the clusterrole - err = kubeutil.CreateClusterRoleFromYaml(co.clientSet, - constants.YurtControllerManagerClusterRole) - if err != nil { - return err - } - // bind the clusterrole - err = kubeutil.CreateClusterRoleBindingFromYaml(co.clientSet, - constants.YurtControllerManagerClusterRoleBinding) - if err != nil { - return - } - // create the yurt-controller-manager deployment - err = kubeutil.CreateDeployFromYaml(co.clientSet, - "kube-system", - constants.YurtControllerManagerDeployment, - map[string]string{ - "image": co.YurtControllerManagerImage, - "edgeNodeLabel": projectinfo.GetEdgeWorkerLabelKey()}) - if err != nil { + if err = kubeutil.DeployYurtControllerManager(co.clientSet, co.YurtControllerManagerImage); err != nil { + klog.Errorf("fail to deploy yurtcontrollermanager: %s", err) return } - // 4. disable node-controller ctx := map[string]string{ "action": "disable", @@ -370,7 +344,7 @@ func (co *ConvertOptions) RunConvert() (err error) { // 5. deploy the yurttunnel if required if co.DeployTunnel { - if err = deployYurttunnelServer(co.clientSet, + if err = kubeutil.DeployYurttunnelServer(co.clientSet, co.CloudNodes, co.YurttunnelServerImage, co.SystemArchitecture); err != nil { @@ -379,7 +353,7 @@ func (co *ConvertOptions) RunConvert() (err error) { } klog.Info("yurt-tunnel-server is deployed") // we will deploy yurt-tunnel-agent on every edge node - if err = deployYurttunnelAgent(co.clientSet, + if err = kubeutil.DeployYurttunnelAgent(co.clientSet, edgeNodeNames, co.YurttunnelAgentImage); err != nil { err = fmt.Errorf("fail to deploy the yurt-tunnel-agent: %s", err) @@ -397,7 +371,7 @@ func (co *ConvertOptions) RunConvert() (err error) { //7. deploy the yurtappmanager if required if co.EnableAppManager { - if err = deployYurtAppManager(co.clientSet, + if err = kubeutil.DeployYurtAppManager(co.clientSet, co.YurtAppManagerImage, co.yurtAppManagerClientSet, co.SystemArchitecture); err != nil { @@ -437,156 +411,6 @@ func (co *ConvertOptions) RunConvert() (err error) { return } -func deployYurtAppManager( - client *kubernetes.Clientset, - yurtappmanagerImage string, - yurtAppManagerClient dynamic.Interface, - systemArchitecture string) error { - - // 1.create the YurtAppManagerCustomResourceDefinition - // 1.1 nodepool - if err := kubeutil.CreateCRDFromYaml(client, yurtAppManagerClient, "", []byte(constants.YurtAppManagerNodePool)); err != nil { - return err - } - - // 1.2 uniteddeployment - if err := kubeutil.CreateCRDFromYaml(client, yurtAppManagerClient, "", []byte(constants.YurtAppManagerUnitedDeployment)); err != nil { - return err - } - - // 2. create the YurtAppManagerRole - if err := kubeutil.CreateRoleFromYaml(client, "kube-system", - constants.YurtAppManagerRole); err != nil { - return err - } - - // 3. create the ClusterRole - if err := kubeutil.CreateClusterRoleFromYaml(client, - constants.YurtAppManagerClusterRole); err != nil { - return err - } - - // 4. create the RoleBinding - if err := kubeutil.CreateRoleBindingFromYaml(client, "kube-system", - constants.YurtAppManagerRolebinding); err != nil { - return err - } - - // 5. create the ClusterRoleBinding - if err := kubeutil.CreateClusterRoleBindingFromYaml(client, - constants.YurtAppManagerClusterRolebinding); err != nil { - return err - } - - // 6. create the Secret - if err := kubeutil.CreateSecretFromYaml(client, "kube-system", - constants.YurtAppManagerSecret); err != nil { - return err - } - - // 7. create the Service - if err := kubeutil.CreateServiceFromYaml(client, - constants.YurtAppManagerService); err != nil { - return err - } - - // 8. create the Deployment - if err := kubeutil.CreateDeployFromYaml(client, - "kube-system", - constants.YurtAppManagerDeployment, - map[string]string{ - "image": yurtappmanagerImage, - "arch": systemArchitecture, - "edgeWorkerLabel": projectinfo.GetEdgeWorkerLabelKey()}); err != nil { - return err - } - - // 9. create the YurtAppManagerMutatingWebhookConfiguration - if err := kubeutil.CreateMutatingWebhookConfigurationFromYaml(client, - constants.YurtAppManagerMutatingWebhookConfiguration); err != nil { - return err - } - - // 10. create the YurtAppManagerValidatingWebhookConfiguration - if err := kubeutil.CreateValidatingWebhookConfigurationFromYaml(client, - constants.YurtAppManagerValidatingWebhookConfiguration); err != nil { - return err - } - - return nil -} - -func deployYurttunnelServer( - client *kubernetes.Clientset, - cloudNodes []string, - yurttunnelServerImage string, - systemArchitecture string) error { - // 1. create the ClusterRole - if err := kubeutil.CreateClusterRoleFromYaml(client, - constants.YurttunnelServerClusterRole); err != nil { - return err - } - - // 2. create the ServiceAccount - if err := kubeutil.CreateServiceAccountFromYaml(client, "kube-system", - constants.YurttunnelServerServiceAccount); err != nil { - return err - } - - // 3. create the ClusterRoleBinding - if err := kubeutil.CreateClusterRoleBindingFromYaml(client, - constants.YurttunnelServerClusterRolebinding); err != nil { - return err - } - - // 4. create the Service - if err := kubeutil.CreateServiceFromYaml(client, - constants.YurttunnelServerService); err != nil { - return err - } - - // 5. create the internal Service(type=ClusterIP) - if err := kubeutil.CreateServiceFromYaml(client, - constants.YurttunnelServerInternalService); err != nil { - return err - } - - // 6. create the Configmap - if err := kubeutil.CreateConfigMapFromYaml(client, - "kube-system", - constants.YurttunnelServerConfigMap); err != nil { - return err - } - - // 7. create the Deployment - if err := kubeutil.CreateDeployFromYaml(client, - "kube-system", - constants.YurttunnelServerDeployment, - map[string]string{ - "image": yurttunnelServerImage, - "arch": systemArchitecture, - "edgeWorkerLabel": projectinfo.GetEdgeWorkerLabelKey()}); err != nil { - return err - } - - return nil -} - -func deployYurttunnelAgent( - client *kubernetes.Clientset, - tunnelAgentNodes []string, - yurttunnelAgentImage string) error { - // 1. Deploy the yurt-tunnel-agent DaemonSet - if err := kubeutil.CreateDaemonSetFromYaml(client, - constants.YurttunnelAgentDaemonSet, - map[string]string{ - "image": yurttunnelAgentImage, - "edgeWorkerLabel": projectinfo.GetEdgeWorkerLabelKey()}); err != nil { - return err - } - return nil -} - // prepareClusterInfoConfigMap will create cluster-info configmap in kube-public namespace if it does not exist func prepareClusterInfoConfigMap(client *kubernetes.Clientset, file string) error { info, err := client.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(context.Background(), bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) diff --git a/pkg/yurtctl/cmd/join/phases/constants.go b/pkg/yurtctl/cmd/join/phases/constants.go index d9a90c12801..158e25c1568 100644 --- a/pkg/yurtctl/cmd/join/phases/constants.go +++ b/pkg/yurtctl/cmd/join/phases/constants.go @@ -1,5 +1,3 @@ -package phases - /* Copyright 2021 The OpenYurt Authors. @@ -16,58 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -const ( - ip_forward = "/proc/sys/net/ipv4/ip_forward" +package phases - bridgenf = "/proc/sys/net/bridge/bridge-nf-call-iptables" - bridgenf6 = "/proc/sys/net/bridge/bridge-nf-call-ip6tables" - kubernetsBridgeSetting = `net.bridge.bridge-nf-call-ip6tables = 1 -net.bridge.bridge-nf-call-iptables = 1` - tmpDownloadDir = "/tmp" +const ( defaultYurthubStaticPodFileName = "yurthub.yaml" - defaultYurthubImage = "registry.cn-hangzhou.aliyuncs.com/openyurt/yurthub:latest" - - cniUrlFormat = "https://aliacs-edge-k8s-cn-hangzhou.oss-cn-hangzhou.aliyuncs.com/public/pkg/openyurt/cni/%s/cni-plugins-linux-%s-%s.tgz" - kubeUrlFormat = "https://dl.k8s.io/%s/kubernetes-node-linux-%s.tar.gz" - - EdgeNode = "edge-node" - CloudNode = "cloud-node" ) const ( - kubeletServiceContent = `[Unit] -Description=kubelet: The Kubernetes Node Agent -Documentation=http://kubernetes.io/docs/ - -[Service] -ExecStartPre=/sbin/swapoff -a -ExecStart=/usr/bin/kubelet -Restart=always -StartLimitInterval=0 -RestartSec=10 - -[Install] -WantedBy=multi-user.target` - - edgeKubeletUnitConfig = ` -[Service] -Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf" -Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" -EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env -EnvironmentFile=-/etc/default/kubelet -ExecStart= -ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS -` - cloudKubeletUnitConfig = ` -[Service] -Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" -Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" -EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env -EnvironmentFile=-/etc/default/kubelet -ExecStart= -ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS -` - kubeletConfForEdgeNode = ` apiVersion: v1 clusters: diff --git a/pkg/yurtctl/cmd/join/phases/join-cloud-node.go b/pkg/yurtctl/cmd/join/phases/join-cloud-node.go index 9d8729effae..b9cdd5653cc 100644 --- a/pkg/yurtctl/cmd/join/phases/join-cloud-node.go +++ b/pkg/yurtctl/cmd/join/phases/join-cloud-node.go @@ -40,6 +40,7 @@ import ( kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" "github.com/lithammer/dedent" + "github.com/openyurtio/openyurt/pkg/yurtctl/constants" "github.com/pkg/errors" ) @@ -105,7 +106,7 @@ func runKubeletStartJoinPhase(c workflow.RunData) (returnErr error) { if !ok { return errors.New("kubelet-start phase invoked with an invalid data struct") } - if data.NodeType() != CloudNode { + if data.NodeType() != constants.CloudNode { return } cfg, initCfg, tlsBootstrapCfg, err := getCloudNodeJoinData(c) diff --git a/pkg/yurtctl/cmd/join/phases/join-edge-node.go b/pkg/yurtctl/cmd/join/phases/join-edge-node.go index 9e409fff41e..bf64f7b840c 100644 --- a/pkg/yurtctl/cmd/join/phases/join-edge-node.go +++ b/pkg/yurtctl/cmd/join/phases/join-edge-node.go @@ -1,5 +1,3 @@ -package phases - /* Copyright 2021 The OpenYurt Authors. Copyright 2019 The Kubernetes Authors. @@ -17,6 +15,8 @@ See the License for the specific language governing permissions and limitations under the License. */ +package phases + import ( "fmt" "io/ioutil" @@ -42,7 +42,7 @@ import ( kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/config/scheme" utilcodec "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/codec" - yurtconstants "github.com/openyurtio/openyurt/pkg/yurtctl/constants" + "github.com/openyurtio/openyurt/pkg/yurtctl/constants" "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" "github.com/pkg/errors" ) @@ -62,7 +62,7 @@ func runJoinEdgeNode(c workflow.RunData) error { if !ok { return fmt.Errorf("Join edge-node phase invoked with an invalid data struct. ") } - if data.NodeType() != EdgeNode { + if data.NodeType() != constants.EdgeNode { return nil } cfg, initCfg, tlsBootstrapCfg, err := getEdgeNodeJoinData(data) @@ -118,7 +118,7 @@ func setKubeletConfigForEdgeNode() error { func addYurthubStaticYaml(cfg *kubeadmapi.JoinConfiguration, podManifestPath string, yurthubImage string) error { klog.Info("[join-node] Adding edge hub static yaml") if len(yurthubImage) == 0 { - yurthubImage = defaultYurthubImage + yurthubImage = fmt.Sprintf("%s/%s:%s", constants.DefaultOpenYurtImageRegistry, constants.Yurthub, constants.DefaultOpenYurtVersion) } if _, err := os.Stat(podManifestPath); err != nil { if os.IsNotExist(err) { @@ -192,7 +192,7 @@ func downloadConfig(client clientset.Interface, kubeletVersion *version.Version, return nil, err } if kc.StaticPodPath == "" { - kc.StaticPodPath = yurtconstants.StaticPodPath + kc.StaticPodPath = constants.StaticPodPath } encoder, err := utilcodec.NewKubeletconfigYAMLEncoder(kubeletconfigv1beta1.SchemeGroupVersion) if err != nil { diff --git a/pkg/yurtctl/cmd/join/phases/postcheck.go b/pkg/yurtctl/cmd/join/phases/postcheck.go index 091d4d3d754..af8ed085832 100644 --- a/pkg/yurtctl/cmd/join/phases/postcheck.go +++ b/pkg/yurtctl/cmd/join/phases/postcheck.go @@ -1,5 +1,3 @@ -package phases - /* Copyright 2021 The OpenYurt Authors. @@ -16,6 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ +package phases + import ( "fmt" "io/ioutil" @@ -34,6 +34,7 @@ import ( kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" "github.com/openyurtio/openyurt/pkg/projectinfo" + "github.com/openyurtio/openyurt/pkg/yurtctl/constants" "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" ) @@ -59,7 +60,7 @@ func runPostcheck(c workflow.RunData) error { } cfg := j.Cfg() - if j.NodeType() == EdgeNode { + if j.NodeType() == constants.EdgeNode { klog.V(1).Infof("waiting yurt hub ready.") if err := checkYurthubHealthz(); err != nil { return err diff --git a/pkg/yurtctl/cmd/join/phases/prepare.go b/pkg/yurtctl/cmd/join/phases/prepare.go index 25dbbe5a349..bd4bcb148db 100644 --- a/pkg/yurtctl/cmd/join/phases/prepare.go +++ b/pkg/yurtctl/cmd/join/phases/prepare.go @@ -1,5 +1,3 @@ -package phases - /* Copyright 2021 The OpenYurt Authors. @@ -16,23 +14,15 @@ See the License for the specific language governing permissions and limitations under the License. */ +package phases + import ( "fmt" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "runtime" - "strings" - "k8s.io/apimachinery/pkg/util/version" - "k8s.io/klog" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - selinux "github.com/opencontainers/selinux/go-selinux" - "github.com/openyurtio/openyurt/pkg/yurtctl/constants" - "github.com/openyurtio/openyurt/pkg/yurtctl/util" - "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" + "github.com/openyurtio/openyurt/pkg/yurtctl/util/kubernetes" + "github.com/openyurtio/openyurt/pkg/yurtctl/util/system" ) // NewEdgeNodePhase creates a yurtctl workflow phase that initialize the node environment. @@ -56,174 +46,23 @@ func runPrepare(c workflow.RunData) error { return err } - if err := setIpv4Forward(); err != nil { + if err := system.SetIpv4Forward(); err != nil { return err } - if err := setBridgeSetting(); err != nil { + if err := system.SetBridgeSetting(); err != nil { return err } - if err := setSELinux(); err != nil { + if err := system.SetSELinux(); err != nil { return err } - if err := checkAndInstallKubelet(initCfg.ClusterConfiguration.KubernetesVersion); err != nil { + if err := kubernetes.CheckAndInstallKubelet(initCfg.ClusterConfiguration.KubernetesVersion); err != nil { return err } - if err := setKubeletService(); err != nil { + if err := kubernetes.SetKubeletService(); err != nil { return err } - if err := setKubeletUnitConfig(data.NodeType()); err != nil { + if err := kubernetes.SetKubeletUnitConfig(data.NodeType()); err != nil { return err } return nil } - -//setIpv4Forward turn on the node ipv4 forward. -func setIpv4Forward() error { - klog.Infof("Setting ipv4 forward") - if err := ioutil.WriteFile(ip_forward, []byte("1"), 0644); err != nil { - return fmt.Errorf("Write content 1 to file %s fail: %v ", ip_forward, err) - } - return nil -} - -//setBridgeSetting turn on the node bridge-nf-call-iptables. -func setBridgeSetting() error { - klog.Info("Setting bridge settings for kubernetes.") - if err := ioutil.WriteFile(constants.Sysctl_k8s_config, []byte(kubernetsBridgeSetting), 0644); err != nil { - return fmt.Errorf("Write file %s fail: %v ", constants.Sysctl_k8s_config, err) - } - if err := ioutil.WriteFile(bridgenf, []byte("1"), 0644); err != nil { - return fmt.Errorf("Write file %s fail: %v ", bridgenf, err) - } - if err := ioutil.WriteFile(bridgenf6, []byte("1"), 0644); err != nil { - return fmt.Errorf("Write file %s fail: %v ", bridgenf, err) - } - return nil -} - -// setSELinux turn off the node selinux. -func setSELinux() error { - klog.Info("Disabling SELinux.") - selinux.SetDisabled() - return nil -} - -//checkAndInstallKubelet install kubelet and kubernetes-cni, skip install if they exist. -func checkAndInstallKubelet(clusterVersionStr string) error { - klog.Info("Check and install kubelet.") - kubeletExist := false - clusterVersion, err := version.ParseSemantic(clusterVersionStr) - if err != nil { - return fmt.Errorf("Parse cluster version %s, fail: %v ", clusterVersionStr, err) - } - if _, err := exec.LookPath("kubelet"); err == nil { - if b, err := exec.Command("kubelet", "--version").CombinedOutput(); err == nil { - kubeletVersionStr := strings.Split(string(b), " ")[1] - kubeletVersionStr = strings.TrimSpace(kubeletVersionStr) - kubeletVersion, err := version.ParseSemantic(kubeletVersionStr) - if err != nil { - return fmt.Errorf("Parse kubelet version %s, fail: %v ", kubeletVersionStr, err) - } - klog.Infof("kubelet --version: %s", kubeletVersionStr) - if kubeletVersion.Major() == clusterVersion.Major() && kubeletVersion.Minor() == clusterVersion.Minor() { - klog.Infof("Kubelet %s already exist, skip install.", kubeletVersionStr) - kubeletExist = true - } else { - return fmt.Errorf("The existing kubelet version %s of the node is inconsistent with cluster version %s, please clean it. ", kubeletVersionStr, clusterVersion) - } - } - } - - if !kubeletExist { - //download and install kubernetes-node - packageUrl := fmt.Sprintf(kubeUrlFormat, clusterVersionStr, runtime.GOARCH) - savePath := fmt.Sprintf("%s/kubernetes-node-linux-%s.tar.gz", tmpDownloadDir, runtime.GOARCH) - klog.V(1).Infof("Download kubelet from: %s", packageUrl) - if err := util.DownloadFile(packageUrl, savePath, 3); err != nil { - return fmt.Errorf("Download kuelet fail: %v", err) - } - if err := util.Untar(savePath, tmpDownloadDir); err != nil { - return err - } - for _, comp := range []string{"kubectl", "kubeadm", "kubelet"} { - target := fmt.Sprintf("/usr/bin/%s", comp) - if err := edgenode.CopyFile(tmpDownloadDir+"/kubernetes/node/bin/"+comp, target, 0755); err != nil { - return err - } - } - } - if _, err := os.Stat(constants.StaticPodPath); os.IsNotExist(err) { - if err := os.MkdirAll(constants.StaticPodPath, 0755); err != nil { - return err - } - } - - if _, err := os.Stat(constants.KubeCniDir); err == nil { - klog.Infof("Cni dir %s already exist, skip install.", constants.KubeCniDir) - return nil - } - //download and install kubernetes-cni - cniUrl := fmt.Sprintf(cniUrlFormat, constants.KubeCniVersion, runtime.GOARCH, constants.KubeCniVersion) - savePath := fmt.Sprintf("%s/cni-plugins-linux-%s-%s.tgz", tmpDownloadDir, runtime.GOARCH, constants.KubeCniVersion) - klog.V(1).Infof("Download cni from: %s", cniUrl) - if err := util.DownloadFile(cniUrl, savePath, 3); err != nil { - return err - } - - if err := os.MkdirAll(constants.KubeCniDir, 0600); err != nil { - return err - } - if err := util.Untar(savePath, constants.KubeCniDir); err != nil { - return err - } - return nil -} - -// setKubeletService configure kubelet service. -func setKubeletService() error { - klog.Info("Setting kubelet service.") - kubeletServiceDir := filepath.Dir(constants.KubeletServiceFilepath) - if _, err := os.Stat(kubeletServiceDir); err != nil { - if os.IsNotExist(err) { - if err := os.MkdirAll(kubeletServiceDir, os.ModePerm); err != nil { - klog.Errorf("Create dir %s fail: %v", kubeletServiceDir, err) - return err - } - } else { - klog.Errorf("Describe dir %s fail: %v", kubeletServiceDir, err) - return err - } - } - if err := ioutil.WriteFile(constants.KubeletServiceFilepath, []byte(kubeletServiceContent), 0644); err != nil { - klog.Errorf("Write file %s fail: %v", constants.KubeletServiceFilepath, err) - return err - } - return nil -} - -//setKubeletUnitConfig configure kubelet startup parameters. -func setKubeletUnitConfig(nodeType string) error { - kubeletUnitDir := filepath.Dir(edgenode.KubeletSvcPath) - if _, err := os.Stat(kubeletUnitDir); err != nil { - if os.IsNotExist(err) { - if err := os.MkdirAll(kubeletUnitDir, os.ModePerm); err != nil { - klog.Errorf("Create dir %s fail: %v", kubeletUnitDir, err) - return err - } - } else { - klog.Errorf("Describe dir %s fail: %v", kubeletUnitDir, err) - return err - } - } - if nodeType == EdgeNode { - if err := ioutil.WriteFile(edgenode.KubeletSvcPath, []byte(edgeKubeletUnitConfig), 0600); err != nil { - return err - } - } else { - if err := ioutil.WriteFile(edgenode.KubeletSvcPath, []byte(cloudKubeletUnitConfig), 0600); err != nil { - return err - } - } - - return nil -} diff --git a/pkg/yurtctl/cmd/join/phases/type.go b/pkg/yurtctl/cmd/join/phases/type.go index 851c6a65621..d9ad6c5dcdb 100644 --- a/pkg/yurtctl/cmd/join/phases/type.go +++ b/pkg/yurtctl/cmd/join/phases/type.go @@ -1,5 +1,3 @@ -package phases - /* Copyright 2021 The OpenYurt Authors. @@ -16,6 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ +package phases + import ( joinphases "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/join" ) diff --git a/pkg/yurtctl/cmd/yurtinit/init.go b/pkg/yurtctl/cmd/yurtinit/init.go new file mode 100644 index 00000000000..de5105a97e5 --- /dev/null +++ b/pkg/yurtctl/cmd/yurtinit/init.go @@ -0,0 +1,620 @@ +/* +Copyright 2021 The OpenYurt Authors. +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package yurtinit + +import ( + "bytes" + "crypto/x509" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "text/template" + + "k8s.io/apimachinery/pkg/util/sets" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + clientcertutil "k8s.io/client-go/util/cert" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" + kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" + "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" + kubephases "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/init" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/features" + certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" + kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" + kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" + "k8s.io/kubernetes/cmd/kubeadm/app/util/pubkeypin" + + "github.com/lithammer/dedent" + yurtphase "github.com/openyurtio/openyurt/pkg/yurtctl/cmd/yurtinit/phases" + "github.com/pkg/errors" + "github.com/spf13/cobra" + flag "github.com/spf13/pflag" +) + +var ( + initDoneTempl = template.Must(template.New("init").Parse(dedent.Dedent(` + Your OpenYurt cluster control-plane has initialized successfully! + + To start using your cluster, you need to run the following as a regular user: + + mkdir -p $HOME/.kube + sudo cp -i {{.KubeConfigPath}} $HOME/.kube/config + sudo chown $(id -u):$(id -g) $HOME/.kube/config + + Then you can join any number of edge-nodes by running the following on each as root: + + {{.joinEdgeNodeCommand}} + + And you can join any number of cloud-nodes by running the following on each as root: + + {{.joinCloudNodeCommand}} + + `))) + + joinCommandTemplate = template.Must(template.New("join-edge-node").Parse(`` + + `yurtctl join {{.ControlPlaneHostPort}} --token {{.Token}} \ + {{range $h := .CAPubKeyPins}}--discovery-token-ca-cert-hash {{$h}} {{end}} --node-type={{.NodeType}}`, + )) +) + +// initOptions defines all the init options exposed via flags by kubeadm init. +// Please note that this structure includes the public kubeadm config API, but only a subset of the options +// supported by this api will be exposed as a flag. +type initOptions struct { + cfgPath string + kubeconfigDir string + kubeconfigPath string + featureGatesString string + ignorePreflightErrors []string + bto *options.BootstrapTokenOptions + externalInitCfg *kubeadmapiv1beta2.InitConfiguration + externalClusterCfg *kubeadmapiv1beta2.ClusterConfiguration + kustomizeDir string + installCNIFile string + isConvertOpenYurtCluster bool + openyurtImageRegistry string + openyurtVersion string +} + +// compile-time assert that the local data object satisfies the phases data interface. +var _ yurtphase.YurtInitData = &initData{} + +// initData defines all the runtime information used when running the kubeadm init workflow; +// this data is shared across all the phases that are included in the workflow. +type initData struct { + cfg *kubeadmapi.InitConfiguration + skipTokenPrint bool + dryRun bool + kubeconfigDir string + kubeconfigPath string + ignorePreflightErrors sets.String + certificatesDir string + dryRunDir string + externalCA bool + client clientset.Interface + outputWriter io.Writer + uploadCerts bool + skipCertificateKeyPrint bool + kustomizeDir string + cniFileName string + isConvertOpenYurtCluster bool + openyurtImageRegistry string + openyurtVersion string +} + +// NewCmdInit returns "kubeadm init" command. +// NB. initOptions is exposed as parameter for allowing unit testing of +// the newInitOptions method, that implements all the command options validation logic +func NewCmdInit(out io.Writer, initOptions *initOptions) *cobra.Command { + if initOptions == nil { + initOptions = newInitOptions() + } + initRunner := workflow.NewRunner() + + cmd := &cobra.Command{ + Use: "init", + Short: "Run this command in order to set up the Kubernetes control plane", + RunE: func(cmd *cobra.Command, args []string) error { + c, err := initRunner.InitData(args) + if err != nil { + return err + } + + data := c.(*initData) + fmt.Printf("[init] Using Kubernetes version: %s\n", data.cfg.KubernetesVersion) + + if err := initRunner.Run(args); err != nil { + return err + } + + return showJoinCommand(data, out) + }, + Args: cobra.NoArgs, + } + + // adds flags to the init command + // init command local flags could be eventually inherited by the sub-commands automatically generated for phases + AddInitConfigFlags(cmd.Flags(), initOptions.externalInitCfg) + AddClusterConfigFlags(cmd.Flags(), initOptions.externalClusterCfg, &initOptions.featureGatesString) + AddInitOtherFlags(cmd.Flags(), initOptions) + initOptions.bto.AddTokenFlag(cmd.Flags()) + initOptions.bto.AddTTLFlag(cmd.Flags()) + options.AddImageMetaFlags(cmd.Flags(), &initOptions.externalClusterCfg.ImageRepository) + + // defines additional flag that are not used by the init command but that could be eventually used + // by the sub-commands automatically generated for phases + initRunner.SetAdditionalFlags(func(flags *flag.FlagSet) { + options.AddKubeConfigFlag(flags, &initOptions.kubeconfigPath) + options.AddKubeConfigDirFlag(flags, &initOptions.kubeconfigDir) + options.AddControlPlanExtraArgsFlags(flags, &initOptions.externalClusterCfg.APIServer.ExtraArgs, &initOptions.externalClusterCfg.ControllerManager.ExtraArgs, &initOptions.externalClusterCfg.Scheduler.ExtraArgs) + }) + + // initialize the workflow runner with the list of phases + initRunner.AppendPhase(yurtphase.NewPreparePhase()) + initRunner.AppendPhase(kubephases.NewPreflightPhase()) + initRunner.AppendPhase(kubephases.NewKubeletStartPhase()) + initRunner.AppendPhase(kubephases.NewCertsPhase()) + initRunner.AppendPhase(kubephases.NewKubeConfigPhase()) + initRunner.AppendPhase(yurtphase.NewAllInAoneControlPlanePhase()) + initRunner.AppendPhase(kubephases.NewWaitControlPlanePhase()) + initRunner.AppendPhase(kubephases.NewUploadConfigPhase()) + initRunner.AppendPhase(yurtphase.NewMarkCloudNode()) + initRunner.AppendPhase(kubephases.NewBootstrapTokenPhase()) + initRunner.AppendPhase(kubephases.NewKubeletFinalizePhase()) + initRunner.AppendPhase(kubephases.NewAddonPhase()) + initRunner.AppendPhase(yurtphase.NewInstallCNIPhase()) + initRunner.AppendPhase(yurtphase.NewInstallYurtAddonsPhase()) + + // sets the data builder function, that will be used by the runner + // both when running the entire workflow or single phases + initRunner.SetDataInitializer(func(cmd *cobra.Command, args []string) (workflow.RunData, error) { + return newInitData(cmd, args, initOptions, out) + }) + + // binds the Runner to kubeadm init command by altering + // command help, adding --skip-phases flag and by adding phases subcommands + initRunner.BindToCommand(cmd) + + return cmd +} + +// AddInitConfigFlags adds init flags bound to the config to the specified flagset +func AddInitConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiv1beta2.InitConfiguration) { + flagSet.StringVar( + &cfg.LocalAPIEndpoint.AdvertiseAddress, options.APIServerAdvertiseAddress, cfg.LocalAPIEndpoint.AdvertiseAddress, + "The IP address the API Server will advertise it's listening on. If not set the default network interface will be used.", + ) + flagSet.Int32Var( + &cfg.LocalAPIEndpoint.BindPort, options.APIServerBindPort, cfg.LocalAPIEndpoint.BindPort, + "Port for the API Server to bind to.", + ) + cmdutil.AddCRISocketFlag(flagSet, &cfg.NodeRegistration.CRISocket) +} + +// AddClusterConfigFlags adds cluster flags bound to the config to the specified flagset +func AddClusterConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiv1beta2.ClusterConfiguration, featureGatesString *string) { + flagSet.StringVar( + &cfg.Networking.ServiceSubnet, options.NetworkingServiceSubnet, cfg.Networking.ServiceSubnet, + "Use alternative range of IP address for service VIPs.", + ) + flagSet.StringVar( + &cfg.Networking.PodSubnet, options.NetworkingPodSubnet, cfg.Networking.PodSubnet, + "Specify range of IP addresses for the pod network. If set, the control plane will automatically allocate CIDRs for every node.", + ) + flagSet.StringVar( + &cfg.Networking.DNSDomain, options.NetworkingDNSDomain, cfg.Networking.DNSDomain, + `Use alternative domain for services, e.g. "myorg.internal".`, + ) + flagSet.StringVar( + &cfg.ControlPlaneEndpoint, options.ControlPlaneEndpoint, cfg.ControlPlaneEndpoint, + `Specify a stable IP address or DNS name for the control plane.`, + ) + options.AddKubernetesVersionFlag(flagSet, &cfg.KubernetesVersion) + flagSet.StringSliceVar( + &cfg.APIServer.CertSANs, options.APIServerCertSANs, cfg.APIServer.CertSANs, + `Optional extra Subject Alternative Names (SANs) to use for the API Server serving certificate. Can be both IP addresses and DNS names.`, + ) + options.AddFeatureGatesStringFlag(flagSet, featureGatesString) +} + +// AddInitOtherFlags adds init flags that are not bound to a configuration file to the given flagset +// Note: All flags that are not bound to the cfg object should be allowed in cmd/kubeadm/app/apis/kubeadm/validation/validation.go +func AddInitOtherFlags(flagSet *flag.FlagSet, initOptions *initOptions) { + options.AddConfigFlag(flagSet, &initOptions.cfgPath) + flagSet.StringSliceVar( + &initOptions.ignorePreflightErrors, options.IgnorePreflightErrors, initOptions.ignorePreflightErrors, + "A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.", + ) + flagSet.StringVar(&initOptions.openyurtImageRegistry, "yurt-image-registry", "", + "Choose a container registry to pull OpenYurt component images from") + flagSet.StringVar(&initOptions.openyurtVersion, "yurt-version", "", + "Choose a specific OpenYurt version") + flagSet.StringVar(&initOptions.installCNIFile, "install-cni-file", "", + "Configure install cni yaml file.") + flagSet.BoolVar( + &initOptions.isConvertOpenYurtCluster, "is-convert-openyurt", true, "Convert kubernetes cluster to OpenYurt cluster.") + options.AddKustomizePodsFlag(flagSet, &initOptions.kustomizeDir) +} + +// newInitOptions returns a struct ready for being used for creating cmd init flags. +func newInitOptions() *initOptions { + // initialize the public kubeadm config API by applying defaults + externalInitCfg := &kubeadmapiv1beta2.InitConfiguration{} + kubeadmscheme.Scheme.Default(externalInitCfg) + + externalClusterCfg := &kubeadmapiv1beta2.ClusterConfiguration{} + kubeadmscheme.Scheme.Default(externalClusterCfg) + + // Create the options object for the bootstrap token-related flags, and override the default value for .Description + bto := options.NewBootstrapTokenOptions() + bto.Description = "The default bootstrap token generated by 'kubeadm init'." + + return &initOptions{ + externalInitCfg: externalInitCfg, + externalClusterCfg: externalClusterCfg, + bto: bto, + kubeconfigDir: kubeadmconstants.KubernetesDir, + kubeconfigPath: kubeadmconstants.GetAdminKubeConfigPath(), + } +} + +// newInitData returns a new initData struct to be used for the execution of the kubeadm init workflow. +// This func takes care of validating initOptions passed to the command, and then it converts +// options into the internal InitConfiguration type that is used as input all the phases in the kubeadm init workflow +func newInitData(cmd *cobra.Command, args []string, options *initOptions, out io.Writer) (*initData, error) { + // Re-apply defaults to the public kubeadm API (this will set only values not exposed/not set as a flags) + kubeadmscheme.Scheme.Default(options.externalInitCfg) + kubeadmscheme.Scheme.Default(options.externalClusterCfg) + + // Validate standalone flags values and/or combination of flags and then assigns + // validated values to the public kubeadm config API when applicable + var err error + if options.externalClusterCfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, options.featureGatesString); err != nil { + return nil, err + } + + if err = validation.ValidateMixedArguments(cmd.Flags()); err != nil { + return nil, err + } + + if err = options.bto.ApplyTo(options.externalInitCfg); err != nil { + return nil, err + } + + // Either use the config file if specified, or convert public kubeadm API to the internal InitConfiguration + // and validates InitConfiguration + cfg, err := configutil.LoadOrDefaultInitConfiguration(options.cfgPath, options.externalInitCfg, options.externalClusterCfg) + if err != nil { + return nil, err + } + + ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(options.ignorePreflightErrors, cfg.NodeRegistration.IgnorePreflightErrors) + if err != nil { + return nil, err + } + // Also set the union of pre-flight errors to InitConfiguration, to provide a consistent view of the runtime configuration: + cfg.NodeRegistration.IgnorePreflightErrors = ignorePreflightErrorsSet.List() + + // override node name and CRI socket from the command line options + if options.externalInitCfg.NodeRegistration.Name != "" { + cfg.NodeRegistration.Name = options.externalInitCfg.NodeRegistration.Name + } + if options.externalInitCfg.NodeRegistration.CRISocket != "" { + cfg.NodeRegistration.CRISocket = options.externalInitCfg.NodeRegistration.CRISocket + } + + if err := configutil.VerifyAPIServerBindAddress(cfg.LocalAPIEndpoint.AdvertiseAddress); err != nil { + return nil, err + } + if err := features.ValidateVersion(features.InitFeatureGates, cfg.FeatureGates, cfg.KubernetesVersion); err != nil { + return nil, err + } + + // Checks if an external CA is provided by the user (when the CA Cert is present but the CA Key is not) + externalCA, err := certsphase.UsingExternalCA(&cfg.ClusterConfiguration) + if externalCA { + // In case the certificates signed by CA (that should be provided by the user) are missing or invalid, + // returns, because kubeadm can't regenerate them without the CA Key + if err != nil { + return nil, errors.Wrapf(err, "invalid or incomplete external CA") + } + + // Validate that also the required kubeconfig files exists and are invalid, because + // kubeadm can't regenerate them without the CA Key + kubeconfigDir := options.kubeconfigDir + if err := kubeconfigphase.ValidateKubeconfigsForExternalCA(kubeconfigDir, cfg); err != nil { + return nil, err + } + } + + // Checks if an external Front-Proxy CA is provided by the user (when the Front-Proxy CA Cert is present but the Front-Proxy CA Key is not) + externalFrontProxyCA, err := certsphase.UsingExternalFrontProxyCA(&cfg.ClusterConfiguration) + if externalFrontProxyCA { + // In case the certificates signed by Front-Proxy CA (that should be provided by the user) are missing or invalid, + // returns, because kubeadm can't regenerate them without the Front-Proxy CA Key + if err != nil { + return nil, errors.Wrapf(err, "invalid or incomplete external front-proxy CA") + } + } + + return &initData{ + cfg: cfg, + certificatesDir: cfg.CertificatesDir, + skipTokenPrint: false, + kubeconfigDir: options.kubeconfigDir, + kubeconfigPath: options.kubeconfigPath, + ignorePreflightErrors: ignorePreflightErrorsSet, + externalCA: externalCA, + outputWriter: out, + uploadCerts: false, + skipCertificateKeyPrint: false, + kustomizeDir: options.kustomizeDir, + isConvertOpenYurtCluster: options.isConvertOpenYurtCluster, + openyurtVersion: options.openyurtVersion, + cniFileName: options.installCNIFile, + openyurtImageRegistry: options.openyurtImageRegistry, + }, nil +} + +// UploadCerts returns Uploadcerts flag. +func (d *initData) UploadCerts() bool { + return d.uploadCerts +} + +// CertificateKey returns the key used to encrypt the certs. +func (d *initData) CertificateKey() string { + return d.cfg.CertificateKey +} + +// SetCertificateKey set the key used to encrypt the certs. +func (d *initData) SetCertificateKey(key string) { + d.cfg.CertificateKey = key +} + +// SkipCertificateKeyPrint returns the skipCertificateKeyPrint flag. +func (d *initData) SkipCertificateKeyPrint() bool { + return d.skipCertificateKeyPrint +} + +// Cfg returns initConfiguration. +func (d *initData) Cfg() *kubeadmapi.InitConfiguration { + return d.cfg +} + +// DryRun returns the DryRun flag. +func (d *initData) DryRun() bool { + return d.dryRun +} + +// SkipTokenPrint returns the SkipTokenPrint flag. +func (d *initData) SkipTokenPrint() bool { + return d.skipTokenPrint +} + +// IgnorePreflightErrors returns the IgnorePreflightErrors flag. +func (d *initData) IgnorePreflightErrors() sets.String { + return d.ignorePreflightErrors +} + +// CertificateWriteDir returns the path to the certificate folder or the temporary folder path in case of DryRun. +func (d *initData) CertificateWriteDir() string { + if d.dryRun { + return d.dryRunDir + } + return d.certificatesDir +} + +// CertificateDir returns the CertificateDir as originally specified by the user. +func (d *initData) CertificateDir() string { + return d.certificatesDir +} + +// KubeConfigDir returns the path of the Kubernetes configuration folder or the temporary folder path in case of DryRun. +func (d *initData) KubeConfigDir() string { + if d.dryRun { + return d.dryRunDir + } + return d.kubeconfigDir +} + +// KubeConfigPath returns the path to the kubeconfig file to use for connecting to Kubernetes +func (d *initData) KubeConfigPath() string { + if d.dryRun { + d.kubeconfigPath = filepath.Join(d.dryRunDir, kubeadmconstants.AdminKubeConfigFileName) + } + return d.kubeconfigPath +} + +// ManifestDir returns the path where manifest should be stored or the temporary folder path in case of DryRun. +func (d *initData) ManifestDir() string { + if d.dryRun { + return d.dryRunDir + } + return kubeadmconstants.GetStaticPodDirectory() +} + +// KubeletDir returns path of the kubelet configuration folder or the temporary folder in case of DryRun. +func (d *initData) KubeletDir() string { + if d.dryRun { + return d.dryRunDir + } + return kubeadmconstants.KubeletRunDirectory +} + +// ExternalCA returns true if an external CA is provided by the user. +func (d *initData) ExternalCA() bool { + return d.externalCA +} + +// OutputWriter returns the io.Writer used to write output to by this command. +func (d *initData) OutputWriter() io.Writer { + return d.outputWriter +} + +//IsConvertYurtCluster return whether to convert to OpenYurt cluster. +func (d *initData) IsConvertYurtCluster() bool { + return d.isConvertOpenYurtCluster +} + +//OpenYurtImageRegistry return the image registry to install OpenYurt component. +func (d *initData) OpenYurtImageRegistry() string { + return d.openyurtImageRegistry +} + +//OpenYurtVersion return the OpenYurt version. +func (d *initData) OpenYurtVersion() string { + return d.openyurtVersion +} + +//CNIFileName return the cni install yaml. +func (d *initData) CNIFileName() string { + return d.cniFileName +} + +// Client returns a Kubernetes client to be used by kubeadm. +// This function is implemented as a singleton, thus avoiding to recreate the client when it is used by different phases. +// Important. This function must be called after the admin.conf kubeconfig file is created. +func (d *initData) Client() (clientset.Interface, error) { + if d.client == nil { + if d.dryRun { + svcSubnetCIDR, err := kubeadmconstants.GetKubernetesServiceCIDR(d.cfg.Networking.ServiceSubnet, features.Enabled(d.cfg.FeatureGates, features.IPv6DualStack)) + if err != nil { + return nil, errors.Wrapf(err, "unable to get internal Kubernetes Service IP from the given service CIDR (%s)", d.cfg.Networking.ServiceSubnet) + } + // If we're dry-running, we should create a faked client that answers some GETs in order to be able to do the full init flow and just logs the rest of requests + dryRunGetter := apiclient.NewInitDryRunGetter(d.cfg.NodeRegistration.Name, svcSubnetCIDR.String()) + d.client = apiclient.NewDryRunClient(dryRunGetter, os.Stdout) + } else { + // If we're acting for real, we should create a connection to the API server and wait for it to come up + var err error + d.client, err = kubeconfigutil.ClientSetFromFile(d.KubeConfigPath()) + if err != nil { + return nil, err + } + } + } + return d.client, nil +} + +// Tokens returns an array of token strings. +func (d *initData) Tokens() []string { + tokens := []string{} + for _, bt := range d.cfg.BootstrapTokens { + tokens = append(tokens, bt.Token.String()) + } + return tokens +} + +// KustomizeDir returns the folder where kustomize patches for static pod manifest are stored +func (d *initData) KustomizeDir() string { + return d.kustomizeDir +} + +func printJoinCommand(out io.Writer, adminKubeConfigPath, token string, i *initData) error { + joinEdgeNodeCommand, err := getJoinCommand(adminKubeConfigPath, token, "edge-node") + if err != nil { + return err + } + joinCloudNodeCommand, err := getJoinCommand(adminKubeConfigPath, token, "cloud-node") + if err != nil { + return err + } + + ctx := map[string]interface{}{ + "KubeConfigPath": adminKubeConfigPath, + "ControlPlaneEndpoint": i.Cfg().ControlPlaneEndpoint, + "joinEdgeNodeCommand": joinEdgeNodeCommand, + "joinCloudNodeCommand": joinCloudNodeCommand, + } + + return initDoneTempl.Execute(out, ctx) +} + +// showJoinCommand prints the join command after all the phases in init have finished +func showJoinCommand(i *initData, out io.Writer) error { + adminKubeConfigPath := i.KubeConfigPath() + + // Prints the join command, multiple times in case the user has multiple tokens + for _, token := range i.Tokens() { + if err := printJoinCommand(out, adminKubeConfigPath, token, i); err != nil { + return errors.Wrap(err, "failed to print join command") + } + } + + return nil +} + +//getJoinCommand returns the yurtctl join command for a given token and nodeType. +func getJoinCommand(kubeConfigFile, token, nodeType string) (string, error) { + // load the kubeconfig file to get the CA certificate and endpoint + config, err := clientcmd.LoadFromFile(kubeConfigFile) + if err != nil { + return "", errors.Wrap(err, "failed to load kubeconfig") + } + + // load the default cluster config + clusterConfig := kubeconfigutil.GetClusterFromKubeConfig(config) + if clusterConfig == nil { + return "", errors.New("failed to get default cluster config") + } + + // load CA certificates from the kubeconfig (either from PEM data or by file path) + var caCerts []*x509.Certificate + if clusterConfig.CertificateAuthorityData != nil { + caCerts, err = clientcertutil.ParseCertsPEM(clusterConfig.CertificateAuthorityData) + if err != nil { + return "", errors.Wrap(err, "failed to parse CA certificate from kubeconfig") + } + } else if clusterConfig.CertificateAuthority != "" { + caCerts, err = clientcertutil.CertsFromFile(clusterConfig.CertificateAuthority) + if err != nil { + return "", errors.Wrap(err, "failed to load CA certificate referenced by kubeconfig") + } + } else { + return "", errors.New("no CA certificates found in kubeconfig") + } + + // hash all the CA certs and include their public key pins as trusted values + publicKeyPins := make([]string, 0, len(caCerts)) + for _, caCert := range caCerts { + publicKeyPins = append(publicKeyPins, pubkeypin.Hash(caCert)) + } + + ctx := map[string]interface{}{ + "Token": token, + "CAPubKeyPins": publicKeyPins, + "ControlPlaneHostPort": strings.Replace(clusterConfig.Server, "https://", "", -1), + "NodeType": nodeType, + } + + var out bytes.Buffer + err = joinCommandTemplate.Execute(&out, ctx) + if err != nil { + return "", errors.Wrap(err, "failed to render join command template") + } + return out.String(), nil +} diff --git a/pkg/yurtctl/cmd/yurtinit/phases/controlplane.go b/pkg/yurtctl/cmd/yurtinit/phases/controlplane.go new file mode 100644 index 00000000000..d196d61d30c --- /dev/null +++ b/pkg/yurtctl/cmd/yurtinit/phases/controlplane.go @@ -0,0 +1,130 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package phases + +import ( + "fmt" + "strings" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" + etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd" + "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" +) + +func NewAllInAoneControlPlanePhase() workflow.Phase { + return workflow.Phase{ + Name: "Generate all-in-one controlplane static pod yaml.", + Short: "Generate all-in-one controlplane static pod yaml.", + Run: runAllInAoneControlPlane, + } +} + +func runAllInAoneControlPlane(c workflow.RunData) error { + data, ok := c.(YurtInitData) + if !ok { + return fmt.Errorf("Install controlplane phase invoked with an invalid data struct. ") + } + cfg := data.Cfg() + return CreateStaticPodFile(data.ManifestDir(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, data.IsConvertYurtCluster()) +} + +func CreateStaticPodFile(manifestDir, nodeName string, + cfg *kubeadmapi.ClusterConfiguration, + endpoint *kubeadmapi.APIEndpoint, + convertYurtCluster bool) error { + masterSpecs := controlplane.GetStaticPodSpecs(cfg, endpoint) + if convertYurtCluster { + if err := closeNodeLifeCycleController(masterSpecs); err != nil { + return err + } + } + var containers []v1.Container + var volumes []v1.Volume + for _, spec := range masterSpecs { + componentName := spec.Name + for _, v := range spec.Spec.Volumes { + v.Name = componentName + "-" + v.Name + volumes = append(volumes, v) + } + for _, c := range spec.Spec.Containers { + for i := 0; i < len(c.VolumeMounts); i++ { + c.VolumeMounts[i].Name = componentName + "-" + c.VolumeMounts[i].Name + } + containers = append(containers, c) + } + } + + etcdSpec := etcd.GetEtcdPodSpec(cfg, endpoint, nodeName, []etcdutil.Member{}) + componentName := etcdSpec.Name + for _, v := range etcdSpec.Spec.Volumes { + v.Name = componentName + "-" + v.Name + volumes = append(volumes, v) + } + for _, c := range etcdSpec.Spec.Containers { + for i := 0; i < len(c.VolumeMounts); i++ { + c.VolumeMounts[i].Name = componentName + "-" + c.VolumeMounts[i].Name + } + containers = append(containers, c) + } + pod := componentPod("controlplane", containers, volumes, map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: endpoint.String()}) + return staticpod.WriteStaticPodToDisk("controlplane", manifestDir, pod) +} + +func componentPod(name string, containers []v1.Container, volumes []v1.Volume, annotations map[string]string) v1.Pod { + return v1.Pod{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Pod", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: metav1.NamespaceSystem, + Labels: map[string]string{"component": name, "tier": kubeadmconstants.ControlPlaneTier}, + Annotations: annotations, + }, + Spec: v1.PodSpec{ + Containers: containers, + PriorityClassName: "system-cluster-critical", + HostNetwork: true, + Volumes: volumes, + }, + } +} + +func closeNodeLifeCycleController(masterSpecs map[string]v1.Pod) error { + cmPod, ok := masterSpecs[kubeadmconstants.KubeControllerManager] + if !ok { + return fmt.Errorf("Spec %s is not exists.", kubeadmconstants.KubeControllerManager) + } + cmd := cmPod.Spec.Containers[0].Command + var newCmd []string + for _, p := range cmd { + if strings.Contains(p, "--controllers=") { + p = p + ",-nodelifecycle" + } + newCmd = append(newCmd, p) + } + cmPod.Spec.Containers[0].Command = newCmd + masterSpecs[kubeadmconstants.KubeControllerManager] = cmPod + return nil +} diff --git a/pkg/yurtctl/cmd/yurtinit/phases/install_cni.go b/pkg/yurtctl/cmd/yurtinit/phases/install_cni.go new file mode 100644 index 00000000000..fc9c774672c --- /dev/null +++ b/pkg/yurtctl/cmd/yurtinit/phases/install_cni.go @@ -0,0 +1,88 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package phases + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/resource" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + kubeContants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + + "github.com/openyurtio/openyurt/pkg/yurtctl/constants" +) + +func NewInstallCNIPhase() workflow.Phase { + return workflow.Phase{ + Name: "Install flannel.", + Short: "Install flannel.", + Run: runInstallCNI, + } +} + +func runInstallCNI(c workflow.RunData) error { + data, ok := c.(YurtInitData) + if !ok { + return fmt.Errorf("Install cni phase invoked with an invalid data struct. ") + } + cniInstallFile := constants.FlannelIntallFile + if data.CNIFileName() != "" { + cniInstallFile = data.CNIFileName() + } + fileNameOption := &resource.FilenameOptions{ + Filenames: []string{cniInstallFile}, + } + adminKubeConfig := kubeContants.GetAdminKubeConfigPath() + configFlag := &genericclioptions.ConfigFlags{KubeConfig: &adminKubeConfig} + builder := resource.NewBuilder(configFlag) + r := builder.Unstructured(). + Schema(nil). + ContinueOnError(). + NamespaceParam("").DefaultNamespace(). + FilenameParam(false, fileNameOption). + LabelSelectorParam(""). + Flatten(). + Do() + infos, err := r.Infos() + if err != nil { + return err + } + for _, info := range infos { + if err := applyOneObject(info); err != nil { + return err + } + } + return nil +} + +func applyOneObject(info *resource.Info) error { + if err := util.CreateApplyAnnotation(info.Object, unstructured.UnstructuredJSONScheme); err != nil { + return cmdutil.AddSourceToErr("creating", info.Source, err) + } + + helper := resource.NewHelper(info.Client, info.Mapping) + obj, err := helper.Create(info.Namespace, true, info.Object) + if err != nil { + return cmdutil.AddSourceToErr("creating", info.Source, err) + } + info.Refresh(obj, true) + return nil +} diff --git a/pkg/yurtctl/cmd/yurtinit/phases/install_yurt_addons.go b/pkg/yurtctl/cmd/yurtinit/phases/install_yurt_addons.go new file mode 100644 index 00000000000..b1cd3a47ef0 --- /dev/null +++ b/pkg/yurtctl/cmd/yurtinit/phases/install_yurt_addons.go @@ -0,0 +1,84 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package phases + +import ( + "fmt" + "runtime" + + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + kubeContants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + + YurtContants "github.com/openyurtio/openyurt/pkg/yurtctl/constants" + kubeutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/kubernetes" +) + +func NewInstallYurtAddonsPhase() workflow.Phase { + return workflow.Phase{ + Name: "Install OpenYurt component.", + Short: "Install OpenYurt component.", + Run: runInstallYurtAddons, + } +} + +func runInstallYurtAddons(c workflow.RunData) error { + data, ok := c.(YurtInitData) + if !ok { + return fmt.Errorf("Install yurt addons phase invoked with an invalid data struct. ") + } + if !data.IsConvertYurtCluster() { + return nil + } + + restCfg, err := clientcmd.BuildConfigFromFlags("", kubeContants.GetAdminKubeConfigPath()) + if err != nil { + return err + } + client, err := kubernetes.NewForConfig(restCfg) + if err != nil { + return err + } + dynamicClient, err := dynamic.NewForConfig(restCfg) + if err != nil { + return err + } + imageRegistry := data.OpenYurtImageRegistry() + version := data.OpenYurtVersion() + + if len(imageRegistry) == 0 { + imageRegistry = YurtContants.DefaultOpenYurtImageRegistry + } + if len(version) == 0 { + version = YurtContants.DefaultOpenYurtVersion + } + if err := kubeutil.DeployYurtControllerManager(client, fmt.Sprintf("%s/%s:%s", imageRegistry, YurtContants.YurtControllerManager, version)); err != nil { + return err + } + if err := kubeutil.DeployYurtAppManager(client, fmt.Sprintf("%s/%s:%s", imageRegistry, YurtContants.YurtAppManager, version), dynamicClient, runtime.GOARCH); err != nil { + return err + } + if err := kubeutil.DeployYurttunnelServer(client, nil, fmt.Sprintf("%s/%s:%s", imageRegistry, YurtContants.YurtTunnelServer, version), runtime.GOARCH); err != nil { + return err + } + if err := kubeutil.DeployYurttunnelAgent(client, nil, fmt.Sprintf("%s/%s:%s", imageRegistry, YurtContants.YurtTunnelAgent, version)); err != nil { + return err + } + return nil +} diff --git a/pkg/yurtctl/cmd/yurtinit/phases/labelCloudNode.go b/pkg/yurtctl/cmd/yurtinit/phases/labelCloudNode.go new file mode 100644 index 00000000000..62ee71087c9 --- /dev/null +++ b/pkg/yurtctl/cmd/yurtinit/phases/labelCloudNode.go @@ -0,0 +1,63 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package phases + +import ( + "fmt" + + v1 "k8s.io/api/core/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + + "github.com/openyurtio/openyurt/pkg/projectinfo" +) + +func NewMarkCloudNode() workflow.Phase { + return workflow.Phase{ + Name: "mark cloud node", + Short: "Mark a node as a cloud-node", + Run: runMarkCloudNode, + } +} + +func runMarkCloudNode(c workflow.RunData) error { + data, ok := c.(YurtInitData) + if !ok { + return fmt.Errorf("Label cloud node phase invoked with an invalid data struct. ") + } + client, err := data.Client() + if err != nil { + return err + } + nodeRegistration := data.Cfg().NodeRegistration + return LabelCloudNode(client, nodeRegistration.Name, + map[string]string{projectinfo.GetEdgeWorkerLabelKey(): "false"}) +} + +// LabelCloudNode set cloud-node label +func LabelCloudNode(client clientset.Interface, nodeName string, label map[string]string) error { + return apiclient.PatchNode(client, nodeName, func(n *v1.Node) { + labelCloudNode(n, label) + }) +} + +func labelCloudNode(n *v1.Node, labels map[string]string) { + for k, v := range labels { + n.ObjectMeta.Labels[k] = v + } +} diff --git a/pkg/yurtctl/cmd/yurtinit/phases/prepare.go b/pkg/yurtctl/cmd/yurtinit/phases/prepare.go new file mode 100644 index 00000000000..dbe09dea2e7 --- /dev/null +++ b/pkg/yurtctl/cmd/yurtinit/phases/prepare.go @@ -0,0 +1,65 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package phases + +import ( + "fmt" + + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + + "github.com/openyurtio/openyurt/pkg/yurtctl/constants" + "github.com/openyurtio/openyurt/pkg/yurtctl/util/kubernetes" + "github.com/openyurtio/openyurt/pkg/yurtctl/util/system" +) + +// NewEdgeNodePhase creates a yurtctl workflow phase that initialize the node environment. +func NewPreparePhase() workflow.Phase { + return workflow.Phase{ + Name: "Initialize system environment.", + Short: "Initialize system environment.", + Run: runPrepare, + } +} + +//runPrepare executes the node initialization process. +func runPrepare(c workflow.RunData) error { + data, ok := c.(YurtInitData) + if !ok { + return fmt.Errorf("Prepare phase invoked with an invalid data struct. ") + } + initCfg := data.Cfg() + + if err := system.SetIpv4Forward(); err != nil { + return err + } + if err := system.SetBridgeSetting(); err != nil { + return err + } + if err := system.SetSELinux(); err != nil { + return err + } + if err := kubernetes.CheckAndInstallKubelet(initCfg.ClusterConfiguration.KubernetesVersion); err != nil { + return err + } + if err := kubernetes.SetKubeletService(); err != nil { + return err + } + if err := kubernetes.SetKubeletUnitConfig(constants.CloudNode); err != nil { + return err + } + return nil +} diff --git a/pkg/yurtctl/cmd/yurtinit/phases/types.go b/pkg/yurtctl/cmd/yurtinit/phases/types.go new file mode 100644 index 00000000000..f45a919a923 --- /dev/null +++ b/pkg/yurtctl/cmd/yurtinit/phases/types.go @@ -0,0 +1,29 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package phases + +import ( + phases "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/init" +) + +type YurtInitData interface { + phases.InitData + IsConvertYurtCluster() bool + OpenYurtVersion() string + OpenYurtImageRegistry() string + CNIFileName() string +} diff --git a/pkg/yurtctl/constants/constants.go b/pkg/yurtctl/constants/constants.go index c68bb4b4164..a1e0842112e 100644 --- a/pkg/yurtctl/constants/constants.go +++ b/pkg/yurtctl/constants/constants.go @@ -40,6 +40,55 @@ const ( KubeCniVersion = "v0.8.0" KubeletServiceFilepath string = "/etc/systemd/system/kubelet.service" + CniUrlFormat = "https://aliacs-edge-k8s-cn-hangzhou.oss-cn-hangzhou.aliyuncs.com/public/pkg/openyurt/cni/%s/cni-plugins-linux-%s-%s.tgz" + KubeUrlFormat = "https://dl.k8s.io/%s/kubernetes-node-linux-%s.tar.gz" + TmpDownloadDir = "/tmp" + FlannelIntallFile = "https://aliacs-edge-k8s-cn-hangzhou.oss-cn-hangzhou.aliyuncs.com/public/pkg/openyurt/flannel.yaml" + + EdgeNode = "edge-node" + CloudNode = "cloud-node" + + DefaultOpenYurtImageRegistry = "registry.cn-hangzhou.aliyuncs.com/openyurt" + DefaultOpenYurtVersion = "latest" + YurtControllerManager = "yurt-controller-manager" + YurtTunnelServer = "yurt-tunnel-server" + YurtTunnelAgent = "yurt-tunnel-agent" + Yurthub = "yurthub" + YurtAppManager = "yurt-app-manager" + KubeletServiceContent = ` +[Unit] +Description=kubelet: The Kubernetes Node Agent +Documentation=http://kubernetes.io/docs/ + +[Service] +ExecStartPre=/sbin/swapoff -a +ExecStart=/usr/bin/kubelet +Restart=always +StartLimitInterval=0 +RestartSec=10 + +[Install] +WantedBy=multi-user.target` + + EdgeKubeletUnitConfig = ` +[Service] +Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf" +Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" +EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env +EnvironmentFile=-/etc/default/kubelet +ExecStart= +ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS +` + CloudKubeletUnitConfig = ` +[Service] +Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" +Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" +EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env +EnvironmentFile=-/etc/default/kubelet +ExecStart= +ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS +` + YurtControllerManagerServiceAccount = ` apiVersion: v1 kind: ServiceAccount diff --git a/pkg/yurtctl/util/kubernetes/apply_addons.go b/pkg/yurtctl/util/kubernetes/apply_addons.go new file mode 100644 index 00000000000..fb56ae33dd9 --- /dev/null +++ b/pkg/yurtctl/util/kubernetes/apply_addons.go @@ -0,0 +1,202 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubernetes + +import ( + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + + "github.com/openyurtio/openyurt/pkg/projectinfo" + "github.com/openyurtio/openyurt/pkg/yurtctl/constants" +) + +func DeployYurtControllerManager(client *kubernetes.Clientset, yurtControllerManagerImage string) error { + if err := CreateServiceAccountFromYaml(client, + "kube-system", constants.YurtControllerManagerServiceAccount); err != nil { + return err + } + // create the clusterrole + if err := CreateClusterRoleFromYaml(client, + constants.YurtControllerManagerClusterRole); err != nil { + return err + } + // bind the clusterrole + if err := CreateClusterRoleBindingFromYaml(client, + constants.YurtControllerManagerClusterRoleBinding); err != nil { + return err + } + // create the yurt-controller-manager deployment + if err := CreateDeployFromYaml(client, + "kube-system", + constants.YurtControllerManagerDeployment, + map[string]string{ + "image": yurtControllerManagerImage, + "edgeNodeLabel": projectinfo.GetEdgeWorkerLabelKey()}); err != nil { + return err + } + return nil +} + +func DeployYurtAppManager( + client *kubernetes.Clientset, + yurtappmanagerImage string, + yurtAppManagerClient dynamic.Interface, + systemArchitecture string) error { + + // 1.create the YurtAppManagerCustomResourceDefinition + // 1.1 nodepool + if err := CreateCRDFromYaml(client, yurtAppManagerClient, "", []byte(constants.YurtAppManagerNodePool)); err != nil { + return err + } + + // 1.2 uniteddeployment + if err := CreateCRDFromYaml(client, yurtAppManagerClient, "", []byte(constants.YurtAppManagerUnitedDeployment)); err != nil { + return err + } + + // 2. create the YurtAppManagerRole + if err := CreateRoleFromYaml(client, "kube-system", + constants.YurtAppManagerRole); err != nil { + return err + } + + // 3. create the ClusterRole + if err := CreateClusterRoleFromYaml(client, + constants.YurtAppManagerClusterRole); err != nil { + return err + } + + // 4. create the RoleBinding + if err := CreateRoleBindingFromYaml(client, "kube-system", + constants.YurtAppManagerRolebinding); err != nil { + return err + } + + // 5. create the ClusterRoleBinding + if err := CreateClusterRoleBindingFromYaml(client, + constants.YurtAppManagerClusterRolebinding); err != nil { + return err + } + + // 6. create the Secret + if err := CreateSecretFromYaml(client, "kube-system", + constants.YurtAppManagerSecret); err != nil { + return err + } + + // 7. create the Service + if err := CreateServiceFromYaml(client, + constants.YurtAppManagerService); err != nil { + return err + } + + // 8. create the Deployment + if err := CreateDeployFromYaml(client, + "kube-system", + constants.YurtAppManagerDeployment, + map[string]string{ + "image": yurtappmanagerImage, + "arch": systemArchitecture, + "edgeWorkerLabel": projectinfo.GetEdgeWorkerLabelKey()}); err != nil { + return err + } + + // 9. create the YurtAppManagerMutatingWebhookConfiguration + if err := CreateMutatingWebhookConfigurationFromYaml(client, + constants.YurtAppManagerMutatingWebhookConfiguration); err != nil { + return err + } + + // 10. create the YurtAppManagerValidatingWebhookConfiguration + if err := CreateValidatingWebhookConfigurationFromYaml(client, + constants.YurtAppManagerValidatingWebhookConfiguration); err != nil { + return err + } + + return nil +} + +func DeployYurttunnelServer( + client *kubernetes.Clientset, + cloudNodes []string, + yurttunnelServerImage string, + systemArchitecture string) error { + // 1. create the ClusterRole + if err := CreateClusterRoleFromYaml(client, + constants.YurttunnelServerClusterRole); err != nil { + return err + } + + // 2. create the ServiceAccount + if err := CreateServiceAccountFromYaml(client, "kube-system", + constants.YurttunnelServerServiceAccount); err != nil { + return err + } + + // 3. create the ClusterRoleBinding + if err := CreateClusterRoleBindingFromYaml(client, + constants.YurttunnelServerClusterRolebinding); err != nil { + return err + } + + // 4. create the Service + if err := CreateServiceFromYaml(client, + constants.YurttunnelServerService); err != nil { + return err + } + + // 5. create the internal Service(type=ClusterIP) + if err := CreateServiceFromYaml(client, + constants.YurttunnelServerInternalService); err != nil { + return err + } + + // 6. create the Configmap + if err := CreateConfigMapFromYaml(client, + "kube-system", + constants.YurttunnelServerConfigMap); err != nil { + return err + } + + // 7. create the Deployment + if err := CreateDeployFromYaml(client, + "kube-system", + constants.YurttunnelServerDeployment, + map[string]string{ + "image": yurttunnelServerImage, + "arch": systemArchitecture, + "edgeWorkerLabel": projectinfo.GetEdgeWorkerLabelKey()}); err != nil { + return err + } + + return nil +} + +func DeployYurttunnelAgent( + client *kubernetes.Clientset, + tunnelAgentNodes []string, + yurttunnelAgentImage string) error { + // 1. Deploy the yurt-tunnel-agent DaemonSet + if err := CreateDaemonSetFromYaml(client, + constants.YurttunnelAgentDaemonSet, + map[string]string{ + "image": yurttunnelAgentImage, + "edgeWorkerLabel": projectinfo.GetEdgeWorkerLabelKey()}); err != nil { + return err + } + return nil +} diff --git a/pkg/yurtctl/util/kubernetes/util.go b/pkg/yurtctl/util/kubernetes/util.go index 55cfb40f220..1e703dd20c6 100644 --- a/pkg/yurtctl/util/kubernetes/util.go +++ b/pkg/yurtctl/util/kubernetes/util.go @@ -21,16 +21,15 @@ import ( "context" "errors" "fmt" + "io/ioutil" "os" + "os/exec" "path/filepath" + "runtime" + "strings" "sync" "time" - "github.com/openyurtio/openyurt/pkg/yurtctl/constants" - strutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/strings" - tmplutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/templates" - "github.com/spf13/pflag" - v1beta1 "k8s.io/api/admissionregistration/v1beta1" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" @@ -41,7 +40,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/runtime" + k8sruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/runtime/serializer/yaml" yamlutil "k8s.io/apimachinery/pkg/util/yaml" @@ -58,6 +57,13 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmcontants "k8s.io/kubernetes/cmd/kubeadm/app/constants" tokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node" + + "github.com/openyurtio/openyurt/pkg/yurtctl/constants" + "github.com/openyurtio/openyurt/pkg/yurtctl/util" + "github.com/openyurtio/openyurt/pkg/yurtctl/util/edgenode" + strutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/strings" + tmplutil "github.com/openyurtio/openyurt/pkg/yurtctl/util/templates" + "github.com/spf13/pflag" ) const ( @@ -184,10 +190,17 @@ func CreateDeployFromYaml(cliSet *kubernetes.Clientset, ns, dplyTmpl string, ctx // CreateDaemonSetFromYaml creates the DaemonSet from the yaml template. func CreateDaemonSetFromYaml(cliSet *kubernetes.Clientset, dsTmpl string, ctx interface{}) error { - ytadstmp, err := tmplutil.SubsituteTemplate(dsTmpl, ctx) - if err != nil { - return err + var ytadstmp string + var err error + if ctx != nil { + ytadstmp, err = tmplutil.SubsituteTemplate(dsTmpl, ctx) + if err != nil { + return err + } + } else { + ytadstmp = dsTmpl } + obj, err := YamlToObject([]byte(ytadstmp)) if err != nil { return err @@ -319,7 +332,7 @@ func CreateValidatingWebhookConfigurationFromYaml(cliSet *kubernetes.Clientset, func CreateCRDFromYaml(clientset *kubernetes.Clientset, yurtAppManagerClient dynamic.Interface, nameSpace string, filebytes []byte) error { var err error decoder := yamlutil.NewYAMLOrJSONDecoder(bytes.NewReader(filebytes), 10000) - var rawObj runtime.RawExtension + var rawObj k8sruntime.RawExtension err = decoder.Decode(&rawObj) if err != nil { return err @@ -328,7 +341,7 @@ func CreateCRDFromYaml(clientset *kubernetes.Clientset, yurtAppManagerClient dyn if err != nil { return err } - unstructuredMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + unstructuredMap, err := k8sruntime.DefaultUnstructuredConverter.ToUnstructured(obj) if err != nil { return err } @@ -364,7 +377,7 @@ func CreateCRDFromYaml(clientset *kubernetes.Clientset, yurtAppManagerClient dyn } // YamlToObject deserializes object in yaml format to a runtime.Object -func YamlToObject(yamlContent []byte) (runtime.Object, error) { +func YamlToObject(yamlContent []byte) (k8sruntime.Object, error) { decode := serializer.NewCodecFactory(scheme.Scheme).UniversalDeserializer().Decode obj, _, err := decode(yamlContent, nil, nil) if err != nil { @@ -616,3 +629,115 @@ func GetKubeControllerManagerHANodes(cliSet *kubernetes.Clientset) ([]string, er } return kcmNodeNames, nil } + +//CheckAndInstallKubelet install kubelet and kubernetes-cni, skip install if they exist. +func CheckAndInstallKubelet(clusterVersion string) error { + klog.Info("Check and install kubelet.") + kubeletExist := false + if _, err := exec.LookPath("kubelet"); err == nil { + if b, err := exec.Command("kubelet", "--version").CombinedOutput(); err == nil { + kubeletVersion := strings.Split(string(b), " ")[1] + kubeletVersion = strings.TrimSpace(kubeletVersion) + klog.Infof("kubelet --version: %s", kubeletVersion) + if strings.Contains(string(b), clusterVersion) { + klog.Infof("Kubelet %s already exist, skip install.", clusterVersion) + kubeletExist = true + } else { + return fmt.Errorf("The existing kubelet version %s of the node is inconsistent with cluster version %s, please clean it. ", kubeletVersion, clusterVersion) + } + } + } + + if !kubeletExist { + //download and install kubernetes-node + packageUrl := fmt.Sprintf(constants.KubeUrlFormat, clusterVersion, runtime.GOARCH) + savePath := fmt.Sprintf("%s/kubernetes-node-linux-%s.tar.gz", constants.TmpDownloadDir, runtime.GOARCH) + klog.V(1).Infof("Download kubelet from: %s", packageUrl) + if err := util.DownloadFile(packageUrl, savePath, 3); err != nil { + return fmt.Errorf("Download kuelet fail: %v", err) + } + if err := util.Untar(savePath, constants.TmpDownloadDir); err != nil { + return err + } + for _, comp := range []string{"kubectl", "kubeadm", "kubelet"} { + target := fmt.Sprintf("/usr/bin/%s", comp) + if err := edgenode.CopyFile(constants.TmpDownloadDir+"/kubernetes/node/bin/"+comp, target, 0755); err != nil { + return err + } + } + } + if _, err := os.Stat(constants.StaticPodPath); os.IsNotExist(err) { + if err := os.MkdirAll(constants.StaticPodPath, 0755); err != nil { + return err + } + } + + if _, err := os.Stat(constants.KubeCniDir); err == nil { + klog.Infof("Cni dir %s already exist, skip install.", constants.KubeCniDir) + return nil + } + //download and install kubernetes-cni + cniUrl := fmt.Sprintf(constants.CniUrlFormat, constants.KubeCniVersion, runtime.GOARCH, constants.KubeCniVersion) + savePath := fmt.Sprintf("%s/cni-plugins-linux-%s-%s.tgz", constants.TmpDownloadDir, runtime.GOARCH, constants.KubeCniVersion) + klog.V(1).Infof("Download cni from: %s", cniUrl) + if err := util.DownloadFile(cniUrl, savePath, 3); err != nil { + return err + } + + if err := os.MkdirAll(constants.KubeCniDir, 0600); err != nil { + return err + } + if err := util.Untar(savePath, constants.KubeCniDir); err != nil { + return err + } + return nil +} + +// SetKubeletService configure kubelet service. +func SetKubeletService() error { + klog.Info("Setting kubelet service.") + kubeletServiceDir := filepath.Dir(constants.KubeletServiceFilepath) + if _, err := os.Stat(kubeletServiceDir); err != nil { + if os.IsNotExist(err) { + if err := os.MkdirAll(kubeletServiceDir, os.ModePerm); err != nil { + klog.Errorf("Create dir %s fail: %v", kubeletServiceDir, err) + return err + } + } else { + klog.Errorf("Describe dir %s fail: %v", kubeletServiceDir, err) + return err + } + } + if err := ioutil.WriteFile(constants.KubeletServiceFilepath, []byte(constants.KubeletServiceContent), 0644); err != nil { + klog.Errorf("Write file %s fail: %v", constants.KubeletServiceFilepath, err) + return err + } + return nil +} + +//SetKubeletUnitConfig configure kubelet startup parameters. +func SetKubeletUnitConfig(nodeType string) error { + kubeletUnitDir := filepath.Dir(edgenode.KubeletSvcPath) + if _, err := os.Stat(kubeletUnitDir); err != nil { + if os.IsNotExist(err) { + if err := os.MkdirAll(kubeletUnitDir, os.ModePerm); err != nil { + klog.Errorf("Create dir %s fail: %v", kubeletUnitDir, err) + return err + } + } else { + klog.Errorf("Describe dir %s fail: %v", kubeletUnitDir, err) + return err + } + } + if nodeType == constants.EdgeNode { + if err := ioutil.WriteFile(edgenode.KubeletSvcPath, []byte(constants.EdgeKubeletUnitConfig), 0600); err != nil { + return err + } + } else { + if err := ioutil.WriteFile(edgenode.KubeletSvcPath, []byte(constants.CloudKubeletUnitConfig), 0600); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/yurtctl/util/system/util.go b/pkg/yurtctl/util/system/util.go new file mode 100644 index 00000000000..c30f2d2b995 --- /dev/null +++ b/pkg/yurtctl/util/system/util.go @@ -0,0 +1,68 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package system + +import ( + "fmt" + "io/ioutil" + + "k8s.io/klog" + + "github.com/opencontainers/selinux/go-selinux" + "github.com/openyurtio/openyurt/pkg/yurtctl/constants" +) + +const ( + ip_forward = "/proc/sys/net/ipv4/ip_forward" + bridgenf = "/proc/sys/net/bridge/bridge-nf-call-iptables" + bridgenf6 = "/proc/sys/net/bridge/bridge-nf-call-ip6tables" + + kubernetsBridgeSetting = ` +net.bridge.bridge-nf-call-ip6tables = 1 +net.bridge.bridge-nf-call-iptables = 1` +) + +//setIpv4Forward turn on the node ipv4 forward. +func SetIpv4Forward() error { + klog.Infof("Setting ipv4 forward") + if err := ioutil.WriteFile(ip_forward, []byte("1"), 0644); err != nil { + return fmt.Errorf("Write content 1 to file %s fail: %v ", ip_forward, err) + } + return nil +} + +//setBridgeSetting turn on the node bridge-nf-call-iptables. +func SetBridgeSetting() error { + klog.Info("Setting bridge settings for kubernetes.") + if err := ioutil.WriteFile(constants.Sysctl_k8s_config, []byte(kubernetsBridgeSetting), 0644); err != nil { + return fmt.Errorf("Write file %s fail: %v ", constants.Sysctl_k8s_config, err) + } + if err := ioutil.WriteFile(bridgenf, []byte("1"), 0644); err != nil { + return fmt.Errorf("Write file %s fail: %v ", bridgenf, err) + } + if err := ioutil.WriteFile(bridgenf6, []byte("1"), 0644); err != nil { + return fmt.Errorf("Write file %s fail: %v ", bridgenf, err) + } + return nil +} + +// setSELinux turn off the node selinux. +func SetSELinux() error { + klog.Info("Disabling SELinux.") + selinux.SetDisabled() + return nil +}