Skip to content

Commit

Permalink
1. implement revert subcommand, which allows user to revert a yurt (#7)
Browse files Browse the repository at this point in the history
cluster back to a kubernetes cluster
2. extract common functions and put them in pkg/yurtctl/util/kubernetes/util.go
3. fix golint issues
4. add yurtctl tutorial draft docs/tutorial/yurtctl.md
5. add docs/todo.md
  • Loading branch information
charleszheng44 authored May 27, 2020
1 parent 89b41a7 commit cc38966
Show file tree
Hide file tree
Showing 8 changed files with 298 additions and 62 deletions.
73 changes: 65 additions & 8 deletions config/yurtctl-servant/setup_edgenode
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,18 @@

set -o errexit
set -o pipefail
set -o nounset

KUBELET_CONF=${KUBELET_CONF:-/etc/kubernetes/kubelet.conf}
KUBELET_SVC=${KUBELET_SVC:-/etc/systemd/system/kubelet.service.d/10-kubeadm.conf}
BOOTSTRAP_KUBELET_CONF=${BOOTSTRAP_KUBELET_CONF:-/etc/kubernetes/bootstrap-kubelet.conf}
OPENYURT_DIR=${OPENYURT_DIR:-/var/lib/openyurt}
STATIC_POD_PATH=${STATIC_POD_PATH:-/etc/kubernetes/manifests}
MINIKUBE_PKI_DIR=${MINIKUBE_PKI_DIR:-/var/lib/minikube/certs}
ACTION=$1
PROVIDER=$2

# PROVIDER can be nounset
set -o nounset

declare -r YURTHUB_TEMPLATE='
apiVersion: v1
Expand Down Expand Up @@ -82,7 +87,12 @@ spec:

# log outputs the log message with date and program prefix
log() {
echo "$(date +"%m/%d/%Y-%T-%Z") [YURT_SERVANT] $@"
echo "$(date +"%m/%d/%Y-%T-%Z") [YURT_SERVANT] [LOG] $@"
}

# error outputs the error message with data program prefix
error() {
echo "$(date +"%m/%d/%Y-%T-%Z") [YURT_SERVANT] [ERROR] $@"
}

# setup_yurthub sets up the yurthub pod and wait for the its status to be Running
Expand Down Expand Up @@ -127,8 +137,12 @@ setup_yurthub() {
return
else
retry=$((retry-1))
[ $retry -ge 0 ] &&
if [ $retry -ge 0 ]; then
log "yurt-hub-$NODE_NAME is $podPhase, will retry $retry times"
else
error "yurt-hub-$NODE_NAME failed, after retry 5 times"
exit 1
fi
continue
fi
done
Expand All @@ -146,16 +160,59 @@ reset_kubelet() {
/user:/d;
s/ https.*/ http:\/\/127.0.0.1:10261/g' $OPENYURT_DIR/kubelet.conf
log "generated the revised kubeconfig $OPENYURT_DIR/kubelet.conf"
# revise the kubelet.service drop-in
sed -i "s/--bootstrap.*bootstrap-kubelet.conf//g;
s|--kubeconfig=.*|--kubeconfig=$OPENYURT_DIR\/kubelet.conf|g" $KUBELET_SVC
# revise the kubelet.service drop-in
if [ -f $BOOTSTRAP_KUBELET_CONF ]; then
# /etc/kubernetes/bootstrap-kubelet.config exist, keep the
# --bootstrap-kubeconfig option
sed -i "s|--kubeconfig=.*|--kubeconfig=$OPENYURT_DIR\/kubelet.conf|g" $KUBELET_SVC
else
sed -i "s/--bootstrap.*bootstrap-kubelet.conf//g;
s|--kubeconfig=.*|--kubeconfig=$OPENYURT_DIR\/kubelet.conf|g" $KUBELET_SVC
fi
log "revised the kubelet.service drop-in file"
# reset the kubelete.service
systemctl daemon-reload
systemctl restart kubelet.service
log "kubelet has been restarted"
}

setup_yurthub $1
reset_kubelet
# remove_yurthub deletes the yurt-hub pod
remove_yurthub() {
# remove the yurt-hub.yaml to delete the yurt-hub
[ -f $STATIC_POD_PATH/yurt-hub.yaml ] &&
rm $STATIC_POD_PATH/yurt-hub.yaml
log "yurt-hub has been removed"
}

# revert_kubelet resets the kubelet service and makes it connect to the
# apiserver directly
revert_kubelet() {
# remove openyurt's kubelet.conf if exist
[ -f $OPENYURT_DIR/kubelet.conf ] && rm $OPENYURT_DIR/kubelet.conf
# revise the kubelet.service drop-in
sed -i "s|--kubeconfig=.*|--kubeconfig=$KUBELET_CONF|g;" $KUBELET_SVC
log "revised the kubelet.service drop-in file back to the default"
# reset the kubelete.service
systemctl daemon-reload
systemctl restart kubelet.service
log "kubelet has been reset back to default"
}

case $ACTION in
convert)
setup_yurthub $PROVIDER
reset_kubelet
;;
revert)
revert_kubelet
remove_yurthub
;;
*)
error "unknwon action $ACTION"
exit 1
;;
esac



log "done"
8 changes: 8 additions & 0 deletions docs/todo.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# yurtctl

- [ ] fail `set_edgehub`, after retring 5 times
- [ ] create an independent subcommand to add annotation 'autonomous' to desired edge nodes
- [ ] save yurtctl-servant's log to node
- [ ] revert subcommand that revert a yurt cluster back to kubernetes
- [ ] specify edge nodes for upgrading yurthub
- [ ] cluster-info subcommand that list edge/cloud nodes
2 changes: 2 additions & 0 deletions pkg/yurtctl/cmd/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"github.com/spf13/cobra"

"github.com/alibaba/openyurt/pkg/yurtctl/cmd/convert"
"github.com/alibaba/openyurt/pkg/yurtctl/cmd/revert"
)

// NewYurtctlCommand creates a new yurtctl command
Expand All @@ -17,6 +18,7 @@ func NewYurtctlCommand() *cobra.Command {
// add kubeconfig to persistent flags
cmds.PersistentFlags().String("kubeconfig", "", "The path to the kubeconfig file")
cmds.AddCommand(convert.NewConvertCmd())
cmds.AddCommand(revert.NewRevertCmd())

return cmds
}
Expand Down
60 changes: 14 additions & 46 deletions pkg/yurtctl/cmd/convert/convert.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,10 @@ import (
"os"
"path/filepath"
"strings"
"sync"
"time"

"github.com/spf13/cobra"
"github.com/spf13/pflag"
appsv1 "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
Expand All @@ -22,23 +19,16 @@ import (
"github.com/alibaba/openyurt/pkg/yurtctl/constants"
kubeutil "github.com/alibaba/openyurt/pkg/yurtctl/util/kubernetes"
strutil "github.com/alibaba/openyurt/pkg/yurtctl/util/strings"
tmplutil "github.com/alibaba/openyurt/pkg/yurtctl/util/templates"
)

const (
JobNameBase = "yurtctl-servant"
)

var (
WaitServantJobTimeout = time.Minute * 2
CheckServantJobPeriod = time.Second * 10
)

// Provider signifies the provider type
type Provider string

const (
// ProviderMinikube is used if the target kubernetes is run on minikube
ProviderMinikube Provider = "minikube"
ProviderACK Provider = "ack"
// ProviderACK is used if the target kubernetes is run on ack
ProviderACK Provider = "ack"
)

// ConvertOptions has the information that required by convert operation
Expand Down Expand Up @@ -189,44 +179,22 @@ func (co *ConvertOptions) RunConvert() error {

// 4. delete the node-controller service account to disable node-controller
if err := co.clientSet.CoreV1().ServiceAccounts("kube-system").
Delete("node-controller", &metav1.DeleteOptions{}); err != nil {
Delete("node-controller", &metav1.DeleteOptions{
PropagationPolicy: &kubeutil.PropagationPolicy,
}); err != nil {
klog.Errorf("fail to delete ServiceAccount(node-controller): %s", err)
return err
}
klog.Info("delete the node-controller service account")

// 5. deploy yurt-hub and reset the kubelet service
var wg sync.WaitGroup
for _, nodeName := range edgeNodeNames {
tmplCtx := map[string]string{
"jobName": JobNameBase + "-" + nodeName,
"nodeName": nodeName,
"provider": string(co.Provider),
}
jobYaml, err := tmplutil.SubsituteTemplate(constants.ServantJobTemplate, tmplCtx)
if err != nil {
return err
}
srvJobObj, err := kubeutil.YamlToObject([]byte(jobYaml))
if err != nil {
return err
}
srvJob, ok := srvJobObj.(*batchv1.Job)
if !ok {
return errors.New("fail to assert yurtctl-servant job")
}
wg.Add(1)
go func() {
defer wg.Done()
if err := kubeutil.RunJobAndCleanup(co.clientSet, srvJob,
WaitServantJobTimeout, CheckServantJobPeriod); err != nil {
klog.Errorf("fail to run servant job(%s): %s",
srvJob.GetName(), err)
}
klog.Infof("servant job(%s) has succeeded", srvJob.GetName())
}()
klog.Infof("deploying the yurt-hub and resetting the kubelet service...")
if err := kubeutil.RunServantJobs(co.clientSet, map[string]string{
"provider": string(co.Provider),
"action": "convert",
}, edgeNodeNames); err != nil {
klog.Errorf("fail to run ServantJobs: %s", err)
return err
}
wg.Wait()

return nil
}
136 changes: 136 additions & 0 deletions pkg/yurtctl/cmd/revert/revert.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
package revert

import (
"errors"
"os"
"path/filepath"

"github.com/spf13/cobra"
"github.com/spf13/pflag"
"k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/homedir"
"k8s.io/klog"

"github.com/alibaba/openyurt/pkg/yurtctl/constants"
kubeutil "github.com/alibaba/openyurt/pkg/yurtctl/util/kubernetes"
)

type RevertOptions struct {
clientSet *kubernetes.Clientset
}

func NewRevertOptions() *RevertOptions {
return &RevertOptions{}
}

func NewRevertCmd() *cobra.Command {
ro := NewRevertOptions()
cmd := &cobra.Command{
Use: "revert",
Short: "Reverts the yurt cluster back to a Kubernetes cluster",
Run: func(cmd *cobra.Command, _ []string) {
if err := ro.Complete(cmd.Flags()); err != nil {
klog.Fatalf("fail to complete the convert option: %s", err)
}
if err := ro.RunRevert(); err != nil {
klog.Fatalf("fail to convert kubernetes to yurt: %s", err)
}
},
}
return cmd
}

func (ro *RevertOptions) Complete(flags *pflag.FlagSet) error {
// parse kubeconfig and generate the clientset
kbCfgPath, err := flags.GetString("kubeconfig")
if err != nil {
return err
}

if kbCfgPath == "" {
if home := homedir.HomeDir(); home != "" {
kbCfgPath = filepath.Join(home, ".kube", "config")
}
}

if kbCfgPath == "" {
kbCfgPath = os.Getenv("KUBECONFIG")
}

if kbCfgPath == "" {
return errors.New("either '--kubeconfig', '$HOME/.kube/config' or '$KUBECONFIG' need to be set")
}

restCfg, err := clientcmd.BuildConfigFromFlags("", kbCfgPath)
if err != nil {
return err
}

ro.clientSet, err = kubernetes.NewForConfig(restCfg)
if err != nil {
return err
}
return nil
}

func (ro *RevertOptions) RunRevert() error {
// 1. remove labels from nodes
nodeLst, err := ro.clientSet.CoreV1().Nodes().List(metav1.ListOptions{})
if err != nil {
return err
}

var edgeNodeNames []string
for _, node := range nodeLst.Items {
isEdgeNode, ok := node.Labels[constants.LabelEdgeWorker]
if ok && isEdgeNode == "true" {
edgeNodeNames = append(edgeNodeNames, node.GetName())
}
if ok {
delete(node.Labels, constants.LabelEdgeWorker)
if _, err := ro.clientSet.CoreV1().Nodes().Update(&node); err != nil {
return err
}
}
}
klog.Info("label alibabacloud.com/is-edge-worker is removed")

// 2. remove the yurt controller manager
if err := ro.clientSet.AppsV1().Deployments("kube-system").
Delete("yurt-ctrl-mgr", &metav1.DeleteOptions{
PropagationPolicy: &kubeutil.PropagationPolicy,
}); err != nil && !apierrors.IsNotFound(err) {
klog.Errorf("fail to remove yurt controller manager: %s", err)
return err
}
klog.Info("yurt controller manager is removed")

// 3. recreate the node-controller service account
ncSa := &v1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: "node-controller",
Namespace: "kube-system",
},
}
if _, err := ro.clientSet.CoreV1().
ServiceAccounts(ncSa.GetNamespace()).Create(ncSa); err != nil {
klog.Errorf("fail to create node-controller service account: %s", err)
return err
}
klog.Info("ServiceAccount node-controller is created")

// 4. remove yurt-hub and revert kubelet service
if err := kubeutil.RunServantJobs(ro.clientSet,
map[string]string{"action": "revert"},
edgeNodeNames); err != nil {
klog.Errorf("fail to revert edge node: %s", err)
return err
}
klog.Info("yurt-hub is removed, kubelet service is reset")

return nil
}
Loading

0 comments on commit cc38966

Please sign in to comment.