Skip to content

Commit

Permalink
implement the markautonomous subcommand (openyurtio#39)
Browse files Browse the repository at this point in the history
  • Loading branch information
charleszheng44 authored Jun 2, 2020
1 parent 785cd39 commit d8d1cd0
Show file tree
Hide file tree
Showing 6 changed files with 204 additions and 83 deletions.
28 changes: 19 additions & 9 deletions docs/tutorial/yurtctl.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,10 @@ $ _output/bin/yurtctl convert --provider minikube

2. `yurtctl` will install all required components and reset the kubelet in the edge node. The output looks like:
```bash
convert.go:148] mark minikube as the edge-node
convert.go:159] mark minikube as autonomous node
convert.go:178] deploy the yurt controller manager
convert.go:190] deploying the yurt-hub and resetting the kubelet service...
util.go:137] servant job(yurtctl-servant-convert-minikube) has succeeded
convert.go:148] mark minikube as the edge-node
convert.go:178] deploy the yurt controller manager
convert.go:190] deploying the yurt-hub and resetting the kubelet service...
util.go:137] servant job(yurtctl-servant-convert-minikube) has succeeded
```

3. yurt controller manager and yurthub Pods will be up and running in one minute. Let us verify them:
Expand All @@ -32,7 +31,13 @@ NAME READY STATUS RESTARTS AGE
yurt-hub-minikube 1/1 Running 0 23h
```

4. As the minikube cluster only contains one node, the node will be marked as an autonomous edge node. Let us verify this by inspecting the node's labels and annotations:
4. Next, we mark desired edge nodes as autonomous (only pods running on the autonomous edge nodes will be prevented from being evicted during disconnection):
```bash
$ _output/bin/yurtctl markautonomous
I0602 14:11:08.610222 89160 markautonomous.go:149] mark minikube-m02 as autonomous
```

5. As the minikube cluster only contains one node, the node will be marked as an autonomous edge node. Let us verify this by inspecting the node's labels and annotations:
```
$ kubectl describe node | grep Labels -A 3
Labels: alibabacloud.com/is-edge-worker=true
Expand Down Expand Up @@ -119,19 +124,24 @@ us-west-1.192.168.0.88 Ready <none> 19h v1.14.8-aliyun.1
$ _output/bin/yurtctl convert --provider ack --cloud-nodes us-west-1.192.168.0.87
I0529 11:21:05.835781 9231 convert.go:145] mark us-west-1.192.168.0.87 as the cloud-node
I0529 11:21:05.861064 9231 convert.go:153] mark us-west-1.192.168.0.88 as the edge-node
I0529 11:21:05.888271 9231 convert.go:164] mark us-west-1.192.168.0.88 as autonomous node
I0529 11:21:05.951483 9231 convert.go:183] deploy the yurt controller manager
I0529 11:21:05.974443 9231 convert.go:195] deploying the yurt-hub and resetting the kubelet service...
I0529 11:21:26.075075 9231 util.go:147] servant job(yurtctl-servant-convert-us-west-1.192.168.0.88) has succeeded
```

3. Node `minikube` will be marked as a non-edge node. You can verify this by inspecting its labels:
3. Node `us-west-1.192.168.0.87` will be marked as a non-edge node. You can verify this by inspecting its labels:
```bash
$ kubectl describe node us-west-1.192.168.0.87 | grep Labels
Labels: alibabacloud.com/is-edge-worker=false
```

4. When the OpenYurt cluster contains cloud nodes, yurt controller manager will be deployed on the cloud node (in this case, the node `us-west-1.192.168.0.87`):
4. Same as before, we make desired edge nodes autonomous:
```bash
$ _output/bin/yurtctl markautonomous
I0602 11:22:05.610222 89160 markautonomous.go:149] mark us-west-1.192.168.0.88 as autonomous
```

5. When the OpenYurt cluster contains cloud nodes, yurt controller manager will be deployed on the cloud node (in this case, the node `us-west-1.192.168.0.87`):
```bash
$ kubectl get pods -A -o=custom-columns='NAME:.metadata.name,NODE:.spec.nodeName'
NAME NODE
Expand Down
2 changes: 2 additions & 0 deletions pkg/yurtctl/cmd/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/spf13/cobra"

"github.com/alibaba/openyurt/pkg/yurtctl/cmd/convert"
"github.com/alibaba/openyurt/pkg/yurtctl/cmd/markautonomous"
"github.com/alibaba/openyurt/pkg/yurtctl/cmd/revert"
)

Expand All @@ -35,6 +36,7 @@ func NewYurtctlCommand() *cobra.Command {
cmds.PersistentFlags().String("kubeconfig", "", "The path to the kubeconfig file")
cmds.AddCommand(convert.NewConvertCmd())
cmds.AddCommand(revert.NewRevertCmd())
cmds.AddCommand(markautonomous.NewMarkAutonomousCmd())

return cmds
}
Expand Down
47 changes: 5 additions & 42 deletions pkg/yurtctl/cmd/convert/convert.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,6 @@ package convert
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"

"github.com/spf13/cobra"
Expand All @@ -29,8 +27,6 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/homedir"
"k8s.io/klog"

"github.com/alibaba/openyurt/pkg/yurtctl/constants"
Expand Down Expand Up @@ -133,31 +129,7 @@ func (co *ConvertOptions) Complete(flags *pflag.FlagSet) error {
co.YurctlServantImage = ycsi

// parse kubeconfig and generate the clientset
kbCfgPath, err := flags.GetString("kubeconfig")
if err != nil {
return err
}

if kbCfgPath == "" {
kbCfgPath = os.Getenv("KUBECONFIG")
}

if kbCfgPath == "" {
if home := homedir.HomeDir(); home != "" {
kbCfgPath = filepath.Join(home, ".kube", "config")
}
}

if kbCfgPath == "" {
return errors.New("either '--kubeconfig', '$HOME/.kube/config' or '$KUBECONFIG' need to be set")
}

restCfg, err := clientcmd.BuildConfigFromFlags("", kbCfgPath)
if err != nil {
return err
}

co.clientSet, err = kubernetes.NewForConfig(restCfg)
co.clientSet, err = kubeutil.GenClientSet(flags)
if err != nil {
return err
}
Expand Down Expand Up @@ -199,23 +171,14 @@ func (co *ConvertOptions) RunConvert() error {
// label node as edge node
klog.Infof("mark %s as the edge-node", node.GetName())
edgeNodeNames = append(edgeNodeNames, node.GetName())
edgeNode, err := kubeutil.LabelNode(co.clientSet,
_, err := kubeutil.LabelNode(co.clientSet,
&node, constants.LabelEdgeWorker, "true")
if err != nil {
return err
}
// 3. annotate all edge nodes as autonomous
// NOTE pods running on an non-autonomous will be evicted, even though
// the node is marked as an edge node.
// TODO should we allow user to decide if a node is autonomous or not ?
klog.Infof("mark %s as autonomous node", node.GetName())
if _, err := kubeutil.AnnotateNode(co.clientSet,
edgeNode, constants.AnnotationAutonomy, "true"); err != nil {
return err
}
}

// 4. deploy yurt controller manager
// 3. deploy yurt controller manager
ycmdp, err := tmpltil.SubsituteTemplate(constants.YurtControllerManagerDeployment,
map[string]string{"image": co.YurtControllerManagerImage})
if err != nil {
Expand All @@ -234,7 +197,7 @@ func (co *ConvertOptions) RunConvert() error {
}
klog.Info("deploy the yurt controller manager")

// 5. delete the node-controller service account to disable node-controller
// 4. delete the node-controller service account to disable node-controller
if err := co.clientSet.CoreV1().ServiceAccounts("kube-system").
Delete("node-controller", &metav1.DeleteOptions{
PropagationPolicy: &kubeutil.PropagationPolicy,
Expand All @@ -243,7 +206,7 @@ func (co *ConvertOptions) RunConvert() error {
return err
}

// 6. deploy yurt-hub and reset the kubelet service
// 5. deploy yurt-hub and reset the kubelet service
klog.Infof("deploying the yurt-hub and resetting the kubelet service...")
if err := kubeutil.RunServantJobs(co.clientSet, map[string]string{
"provider": string(co.Provider),
Expand Down
136 changes: 136 additions & 0 deletions pkg/yurtctl/cmd/markautonomous/markautonomous.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
/*
Copyright 2020 The OpenYurt Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package markautonomous

import (
"fmt"
"strings"

"github.com/spf13/cobra"
"github.com/spf13/pflag"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/klog"

"github.com/alibaba/openyurt/pkg/yurtctl/constants"
kubeutil "github.com/alibaba/openyurt/pkg/yurtctl/util/kubernetes"
)

// MarkAutonomousOptions has the information that required by convert operation
type MarkAutonomousOptions struct {
*kubernetes.Clientset
AutonomousNodes []string
MarkAllEdgeNodes bool
}

// NewMarkAutonomousOptions creates a new MarkAutonomousOptions
func NewMarkAutonomousOptions() *MarkAutonomousOptions {
return &MarkAutonomousOptions{}
}

// NewMarkAutonomousCmd generates a new markautonomous command
func NewMarkAutonomousCmd() *cobra.Command {
co := NewMarkAutonomousOptions()
cmd := &cobra.Command{
Use: "markautonomous -a AUTONOMOUSNODES",
Short: "mark the nodes as autonomous",
Run: func(cmd *cobra.Command, _ []string) {
if err := co.Complete(cmd.Flags()); err != nil {
klog.Fatalf("fail to complete the markautonomous option: %s", err)
}
if err := co.RunMarkAutonomous(); err != nil {
klog.Fatalf("fail to make nodes autonomous: %s", err)
}
},
}

cmd.Flags().StringP("autonomous-nodes", "a", "",
"The list of nodes that will be marked as autonomous."+
"(e.g. -a autonomousnode1,autonomousnode2)")

return cmd
}

// Complete completes all the required options
func (mao *MarkAutonomousOptions) Complete(flags *pflag.FlagSet) error {
anStr, err := flags.GetString("autonomous-nodes")
if err != nil {
return err
}
if anStr == "" {
mao.AutonomousNodes = []string{}
} else {
mao.AutonomousNodes = strings.Split(anStr, ",")
}

// set mark-all-edge-node to false, as user has specifed autonomous nodes
if len(mao.AutonomousNodes) == 0 {
mao.MarkAllEdgeNodes = true
}

mao.Clientset, err = kubeutil.GenClientSet(flags)
if err != nil {
return err
}

return nil
}

// RunMarkAutonomous annotates specified edge nodes as autonomous
func (mao *MarkAutonomousOptions) RunMarkAutonomous() error {
var autonomousNodes []*v1.Node
if mao.MarkAllEdgeNodes {
// make all edge nodes autonomous
labelSelector := fmt.Sprintf("%s=true", constants.LabelEdgeWorker)
edgeNodeList, err := mao.CoreV1().Nodes().
List(metav1.ListOptions{LabelSelector: labelSelector})
if err != nil {
return err
}
if len(edgeNodeList.Items) == 0 {
klog.Warning("there is no edge nodes, please label the edge node first")
return nil
}
for _, node := range edgeNodeList.Items {
autonomousNodes = append(autonomousNodes, &node)
}
} else {
// make only the specified edge nodes autonomous
for _, nodeName := range mao.AutonomousNodes {
node, err := mao.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
if err != nil {
return err
}
if node.Labels[constants.LabelEdgeWorker] == "false" {
return fmt.Errorf("can't make cloud node(%s) autonomous",
node.GetName())
}
autonomousNodes = append(autonomousNodes, node)
}
}

for _, anode := range autonomousNodes {
klog.Infof("mark %s as autonomous", anode.GetName())
if _, err := kubeutil.AnnotateNode(mao.Clientset,
anode, constants.AnnotationAutonomy, "true"); err != nil {
return err
}
}

return nil
}
37 changes: 6 additions & 31 deletions pkg/yurtctl/cmd/revert/revert.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,33 +17,30 @@ limitations under the License.
package revert

import (
"errors"
"os"
"path/filepath"

"github.com/spf13/cobra"
"github.com/spf13/pflag"
"k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/homedir"
"k8s.io/klog"

"github.com/alibaba/openyurt/pkg/yurtctl/constants"
kubeutil "github.com/alibaba/openyurt/pkg/yurtctl/util/kubernetes"
)

// ConvertOptions has the information required by the revert operation
type RevertOptions struct {
clientSet *kubernetes.Clientset
YurtctlServantImage string
}

// NewConvertOptions creates a new RevertOptions
func NewRevertOptions() *RevertOptions {
return &RevertOptions{}
}

// NewRevertCmd generates a new revert command
func NewRevertCmd() *cobra.Command {
ro := NewRevertOptions()
cmd := &cobra.Command{
Expand All @@ -66,44 +63,22 @@ func NewRevertCmd() *cobra.Command {
return cmd
}

// Complete completes all the required options
func (ro *RevertOptions) Complete(flags *pflag.FlagSet) error {
ycsi, err := flags.GetString("yurtctl-servant-image")
if err != nil {
return err
}
ro.YurtctlServantImage = ycsi
// parse kubeconfig and generate the clientset
kbCfgPath, err := flags.GetString("kubeconfig")
if err != nil {
return err
}

if kbCfgPath == "" {
kbCfgPath = os.Getenv("KUBECONFIG")
}

if kbCfgPath == "" {
if home := homedir.HomeDir(); home != "" {
kbCfgPath = filepath.Join(home, ".kube", "config")
}
}

if kbCfgPath == "" {
return errors.New("either '--kubeconfig', '$HOME/.kube/config' or '$KUBECONFIG' need to be set")
}

restCfg, err := clientcmd.BuildConfigFromFlags("", kbCfgPath)
if err != nil {
return err
}

ro.clientSet, err = kubernetes.NewForConfig(restCfg)
ro.clientSet, err = kubeutil.GenClientSet(flags)
if err != nil {
return err
}
return nil
}

// RunRevert reverts the target Yurt cluster back to a standard Kubernetes cluster
func (ro *RevertOptions) RunRevert() error {
// 1. check the server version
if err := kubeutil.ValidateServerVersion(ro.clientSet); err != nil {
Expand Down
Loading

0 comments on commit d8d1cd0

Please sign in to comment.