From 6df9b4832a82a717b14f629b50624cfa25400799 Mon Sep 17 00:00:00 2001
From: Kiefer Chang <kiefer.chang@suse.com>
Date: Tue, 14 Sep 2021 16:12:26 +0800
Subject: [PATCH] Promote nodes by setting machine labels

Switch to the preferred way to promote agent to server in Rancher
by setting the following labels on corresponding machines:

```
rke.cattle.io/control-plane-role: "true"
rke.cattle.io/etcd-role: "true"
```

Signed-off-by: Kiefer Chang <kiefer.chang@suse.com>
---
 .../charts/harvester/templates/configmap.yaml | 43 +++++++++++++++++++
 .../master/node/promote_controller.go         | 27 ++++++------
 2 files changed, 58 insertions(+), 12 deletions(-)

diff --git a/deploy/charts/harvester/templates/configmap.yaml b/deploy/charts/harvester/templates/configmap.yaml
index 7633ba856b..fd1a21c09a 100644
--- a/deploy/charts/harvester/templates/configmap.yaml
+++ b/deploy/charts/harvester/templates/configmap.yaml
@@ -21,3 +21,46 @@ data:
   mode: {{ .Values.service.vip.mode }}
   hwAddress: {{ .Values.service.vip.hwAddress | quote }}
   loadBalancerIP: {{ .Values.service.vip.loadBalancerIP | quote }}
+
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: harvester-helpers
+  labels:
+{{ include "harvester.labels" . | indent 4 }}
+data:
+  promote.sh: |-
+    {{`KUBECTL="/host/$(readlink /host/var/lib/rancher/rke2/bin)/kubectl"
+    CUSTOM_MACHINE=$($KUBECTL get node $HOSTNAME -o go-template=$'{{index .metadata.annotations "cluster.x-k8s.io/machine"}}\n')
+
+    until $KUBECTL get machines.cluster.x-k8s.io $CUSTOM_MACHINE -n fleet-local &> /dev/null
+    do
+      echo Waiting for custom machine $CUSTOM_MACHINE...
+      sleep 2
+    done
+
+    VIP=$($KUBECTL get configmap vip -n harvester-system -o=jsonpath='{.data.ip}')
+    cat > /host/etc/rancher/rke2/config.yaml.d/90-harvester-server.yaml <<EOF
+    cni: multus,canal
+    disable: rke2-ingress-nginx
+    cluster-cidr: 10.52.0.0/16
+    service-cidr: 10.53.0.0/16
+    cluster-dns: 10.53.0.10
+    tls-san:
+      - $VIP
+    EOF
+
+    $KUBECTL label -n fleet-local machines.cluster.x-k8s.io $CUSTOM_MACHINE rke.cattle.io/control-plane-role=true rke.cattle.io/etcd-role=true
+
+    while true
+    do
+      CONTROL_PLANE=$($KUBECTL get node $HOSTNAME -o go-template=$'{{index .metadata.labels "node-role.kubernetes.io/control-plane"}}\n' || true)
+
+      if [ "$CONTROL_PLANE" = "true" ]; then
+        break
+      fi
+      echo Waiting for promotion...
+      sleep 2
+    done
+    `}}
diff --git a/pkg/controller/master/node/promote_controller.go b/pkg/controller/master/node/promote_controller.go
index 2089a27802..3db612934c 100644
--- a/pkg/controller/master/node/promote_controller.go
+++ b/pkg/controller/master/node/promote_controller.go
@@ -44,17 +44,9 @@ const (
 	promoteImage         = "busybox:1.32.0"
 	promoteRootMountPath = "/host"
 
-	promoteCommand = `echo start promote && \
-cat > /etc/rancher/rke2/config.yaml.d/90-harvester-server.yaml <<EOF
-cni: multus,canal
-disable: rke2-ingress-nginx
-cluster-cidr: 10.52.0.0/16
-service-cidr: 10.53.0.0/16
-cluster-dns: 10.53.0.10
-EOF
-systemctl disable --now rke2-agent && \
-systemctl enable --now rke2-server && \
-echo finish promote`
+	promoteScriptsMountPath = "/harvester-helpers"
+	promoteScript           = "/harvester-helpers/promote.sh"
+	helperConfigMapName     = "harvester-helpers"
 )
 
 var (
@@ -449,7 +441,17 @@ func buildPromoteJob(namespace string, node *corev1.Node) *batchv1.Job {
 								Path: "/", Type: &hostPathDirectory,
 							},
 						},
+					}, {
+						Name: "helpers",
+						VolumeSource: corev1.VolumeSource{
+							ConfigMap: &corev1.ConfigMapVolumeSource{
+								LocalObjectReference: corev1.LocalObjectReference{
+									Name: helperConfigMapName,
+								},
+							},
+						},
 					}},
+					ServiceAccountName: "harvester",
 				},
 			},
 		},
@@ -461,10 +463,11 @@ func buildPromoteJob(namespace string, node *corev1.Node) *batchv1.Job {
 			Name:      "promote",
 			Image:     promoteImage,
 			Command:   []string{"sh"},
-			Args:      []string{"-c", fmt.Sprintf(`chroot %s bash -c '%s'`, promoteRootMountPath, promoteCommand)},
+			Args:      []string{"-e", promoteScript},
 			Resources: corev1.ResourceRequirements{},
 			VolumeMounts: []corev1.VolumeMount{
 				{Name: "host-root", MountPath: promoteRootMountPath},
+				{Name: "helpers", MountPath: promoteScriptsMountPath},
 			},
 			ImagePullPolicy: corev1.PullIfNotPresent,
 			SecurityContext: &corev1.SecurityContext{