Switch bash to cloud config (#510)

* Switch machine acuator to use cloud config instead of bash while creating new node * fix yaml * yaml is hard * Change meta-data reference in cloud config and update its header * fix yaml indents * go templates are hard * Use kubeadm module in cloud-init configuration * Remove copyright header * Change permissions for kubeadm configuration * Remove unused variable * Add controlplane cloud-config template * Switch controlplane to use cloud-init * yaml doesn't like tabs * fix indents in yaml template * Add helper functions for template generation * encode certificates in base64 * Use Base64Encode template function when initializing cluster * Remove unused variable * Make controlplane to use cloud-init when joining cluster * remove trailing tab * Remove unused variable * yaml is hell * Join the controlplane
kubernetes-sigs · Feb 8, 2019 · d9e9820 · d9e9820
1 parent aacc639
commit d9e9820
Show file tree

Hide file tree

Showing 3 changed files with 244 additions and 129 deletions.
diff --git a/pkg/cloud/aws/services/userdata/controlplane.go b/pkg/cloud/aws/services/userdata/controlplane.go
@@ -16,111 +16,187 @@ limitations under the License.
 
 package userdata
 
-import "github.com/pkg/errors"
+import (
+	"encoding/base64"
+
+	"github.com/pkg/errors"
+)
 
 const (
-	controlPlaneBashScript = `{{.Header}}
-
-set -eox
-
-mkdir -p /etc/kubernetes/pki/etcd
-
-echo -n '{{.CACert}}' > /etc/kubernetes/pki/ca.crt
-echo -n '{{.CAKey}}' > /etc/kubernetes/pki/ca.key
-chmod 600 /etc/kubernetes/pki/ca.key
-
-echo -n '{{.EtcdCACert}}' > /etc/kubernetes/pki/etcd/ca.crt
-echo -n '{{.EtcdCAKey}}' > /etc/kubernetes/pki/etcd/ca.key
-chmod 600 /etc/kubernetes/pki/etcd/ca.key
-
-echo -n '{{.FrontProxyCACert}}' > /etc/kubernetes/pki/front-proxy-ca.crt
-echo -n '{{.FrontProxyCAKey}}' > /etc/kubernetes/pki/front-proxy-ca.key
-chmod 600 /etc/kubernetes/pki/front-proxy-ca.key
-
-echo -n '{{.SaCert}}' > /etc/kubernetes/pki/sa.pub
-echo -n '{{.SaKey}}' > /etc/kubernetes/pki/sa.key
-chmod 600 /etc/kubernetes/pki/sa.key
-
-PRIVATE_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4)
-HOSTNAME="$(curl http://169.254.169.254/latest/meta-data/local-hostname)"
-
-cat >/tmp/kubeadm.yaml <<EOF
----
-apiVersion: kubeadm.k8s.io/v1beta1
-kind: ClusterConfiguration
-apiServer:
-  certSANs:
-    - "$PRIVATE_IP"
-    - "{{.ELBAddress}}"
-  extraArgs:
-    cloud-provider: aws
-controlPlaneEndpoint: "{{.ELBAddress}}:6443"
-clusterName: "{{.ClusterName}}"
-networking:
-  dnsDomain: "{{.ServiceDomain}}"
-  podSubnet: "{{.PodSubnet}}"
-  serviceSubnet: "{{.ServiceSubnet}}"
-kubernetesVersion: "{{.KubernetesVersion}}"
----
-apiVersion: kubeadm.k8s.io/v1beta1
-kind: InitConfiguration
-nodeRegistration:
-  name: ${HOSTNAME}
-  criSocket: /var/run/containerd/containerd.sock
-  kubeletExtraArgs:
-    cloud-provider: aws
-EOF
-
-kubeadm init --config /tmp/kubeadm.yaml --v 10
+	controlPlaneCloudInit = `{{.Header}}
+write_files:
+-   path: /etc/kubernetes/pki/ca.crt
+    encoding: "base64"
+    owner: root:root
+    permissions: '0640'
+    content: |
+      {{.CACert | Base64Encode}}
+
+-   path: /etc/kubernetes/pki/ca.key
+    encoding: "base64"
+    owner: root:root
+    permissions: '0600'
+    content: |
+      {{.CAKey | Base64Encode}}
+
+-   path: /etc/kubernetes/pki/etcd/ca.crt
+    encoding: "base64"
+    owner: root:root
+    permissions: '0640'
+    content: |
+      {{.EtcdCACert | Base64Encode}}
+
+-   path: /etc/kubernetes/pki/etcd/ca.key
+    encoding: "base64"
+    owner: root:root
+    permissions: '0600'
+    content: |
+      {{.EtcdCAKey | Base64Encode}}
+
+-   path: /etc/kubernetes/pki/front-proxy-ca.crt
+    encoding: "base64"
+    owner: root:root
+    permissions: '0640'
+    content: |
+      {{.FrontProxyCACert | Base64Encode}}
+
+-   path: /etc/kubernetes/pki/front-proxy-ca.key
+    encoding: "base64"
+    owner: root:root
+    permissions: '0600'
+    content: |
+      {{.FrontProxyCAKey | Base64Encode}}
+
+-   path: /etc/kubernetes/pki/sa.pub
+    encoding: "base64"
+    owner: root:root
+    permissions: '0640'
+    content: |
+      {{.SaCert | Base64Encode}}
+
+-   path: /etc/kubernetes/pki/sa.key
+    encoding: "base64"
+    owner: root:root
+    permissions: '0600'
+    content: |
+      {{.SaKey | Base64Encode}}
+
+-   path: /tmp/kubeadm.yaml
+    owner: root:root
+    permissions: '0640'
+    content: |
+      ---
+      apiVersion: kubeadm.k8s.io/v1beta1
+      kind: ClusterConfiguration
+      apiServer:
+        certSANs:
+          - {{ "{{ ds.meta_data.local_ipv4 }}" }}
+          - "{{.ELBAddress}}"
+        extraArgs:
+          cloud-provider: aws
+      controlPlaneEndpoint: "{{.ELBAddress}}:6443"
+      clusterName: "{{.ClusterName}}"
+      networking:
+        dnsDomain: "{{.ServiceDomain}}"
+        podSubnet: "{{.PodSubnet}}"
+        serviceSubnet: "{{.ServiceSubnet}}"
+      kubernetesVersion: "{{.KubernetesVersion}}"
+      ---
+      apiVersion: kubeadm.k8s.io/v1beta1
+      kind: InitConfiguration
+      nodeRegistration:
+        name: {{ "{{ ds.meta_data.hostname }}" }}
+        criSocket: /var/run/containerd/containerd.sock
+        kubeletExtraArgs:
+          cloud-provider: aws
+kubeadm:
+  operation: init
+  config: /tmp/kubeadm.yaml
+
 `
 
-	controlPlaneJoinBashScript = `{{.Header}}
-
-set -eox
-
-mkdir -p /etc/kubernetes/pki/etcd
-
-echo -n '{{.CACert}}' > /etc/kubernetes/pki/ca.crt
-echo -n '{{.CAKey}}' > /etc/kubernetes/pki/ca.key
-chmod 600 /etc/kubernetes/pki/ca.key
-
-echo -n '{{.EtcdCACert}}' > /etc/kubernetes/pki/etcd/ca.crt
-echo -n '{{.EtcdCAKey}}' > /etc/kubernetes/pki/etcd/ca.key
-chmod 600 /etc/kubernetes/pki/etcd/ca.key
-
-echo -n '{{.FrontProxyCACert}}' > /etc/kubernetes/pki/front-proxy-ca.crt
-echo -n '{{.FrontProxyCAKey}}' > /etc/kubernetes/pki/front-proxy-ca.key
-chmod 600 /etc/kubernetes/pki/front-proxy-ca.key
-
-echo -n '{{.SaCert}}' > /etc/kubernetes/pki/sa.pub
-echo -n '{{.SaKey}}' > /etc/kubernetes/pki/sa.key
-chmod 600 /etc/kubernetes/pki/sa.key
-
-PRIVATE_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4)
-HOSTNAME="$(curl http://169.254.169.254/latest/meta-data/local-hostname)"
-
-cat >/tmp/kubeadm-controlplane-join-config.yaml <<EOF
----
-apiVersion: kubeadm.k8s.io/v1beta1
-kind: JoinConfiguration
-discovery:
-  bootstrapToken:
-    token: "{{.BootstrapToken}}"
-    apiServerEndpoint: "{{.ELBAddress}}:6443"
-    caCertHashes:
-      - "{{.CACertHash}}"
-nodeRegistration:
-  name: "${HOSTNAME}"
-  criSocket: /var/run/containerd/containerd.sock
-  kubeletExtraArgs:
-    cloud-provider: aws
-controlPlane:
-  localAPIEndpoint:
-    advertiseAddress: "${PRIVATE_IP}"
-    bindPort: 6443
-EOF
-
-kubeadm join --config /tmp/kubeadm-controlplane-join-config.yaml --v 10
+	controlPlaneJoinCloudInit = `{{.Header}}
+write_files:
+-   path: /etc/kubernetes/pki/ca.crt
+    encoding: "base64"
+    owner: root:root
+    permissions: '0640'
+    content: |
+      {{.CACert | Base64Encode}}
+
+-   path: /etc/kubernetes/pki/ca.key
+    encoding: "base64"
+    owner: root:root
+    permissions: '0600'
+    content: |
+      {{.CAKey | Base64Encode}}
+
+-   path: /etc/kubernetes/pki/etcd/ca.crt
+    encoding: "base64"
+    owner: root:root
+    permissions: '0640'
+    content: |
+      {{.EtcdCACert | Base64Encode}}
+
+-   path: /etc/kubernetes/pki/etcd/ca.key
+    encoding: "base64"
+    owner: root:root
+    permissions: '0600'
+    content: |
+      {{.EtcdCAKey | Base64Encode}}
+
+-   path: /etc/kubernetes/pki/front-proxy-ca.crt
+    encoding: "base64"
+    owner: root:root
+    permissions: '0640'
+    content: |
+      {{.FrontProxyCACert | Base64Encode}}
+
+-   path: /etc/kubernetes/pki/front-proxy-ca.key
+    encoding: "base64"
+    owner: root:root
+    permissions: '0600'
+    content: |
+      {{.FrontProxyCAKey | Base64Encode}}
+
+-   path: /etc/kubernetes/pki/sa.pub
+    encoding: "base64"
+    owner: root:root
+    permissions: '0640'
+    content: |
+      {{.SaCert | Base64Encode}}
+
+-   path: /etc/kubernetes/pki/sa.key
+    encoding: "base64"
+    owner: root:root
+    permissions: '0600'
+    content: |
+      {{.SaKey | Base64Encode}}
+
+-   path: /tmp/kubeadm-controlplane-join-config.yaml
+    owner: root:root
+    permissions: '0640'
+    content: |
+      apiVersion: kubeadm.k8s.io/v1beta1
+      kind: JoinConfiguration
+      discovery:
+        bootstrapToken:
+          token: "{{.BootstrapToken}}"
+          apiServerEndpoint: "{{.ELBAddress}}:6443"
+          caCertHashes:
+            - "{{.CACertHash}}"
+      nodeRegistration:
+        name: {{ "{{ ds.meta_data.hostname }}" }}
+        criSocket: /var/run/containerd/containerd.sock
+        kubeletExtraArgs:
+          cloud-provider: aws
+      controlPlane:
+        localAPIEndpoint:
+          advertiseAddress: {{ "{{ ds.meta_data.local_ipv4 }}" }}
+          bindPort: 6443
+kubeadm:
+  operation: join
+  config: /tmp/kubeadm-controlplane-join-config.yaml
 `
 )
 
@@ -207,12 +283,16 @@ func (cpi *ContolPlaneJoinInput) validateCertificates() error {
 
 // NewControlPlane returns the user data string to be used on a controlplane instance.
 func NewControlPlane(input *ControlPlaneInput) (string, error) {
-	input.Header = defaultHeader
+	input.Header = cloudConfigHeader
 	if err := input.validateCertificates(); err != nil {
 		return "", errors.Wrapf(err, "ControlPlaneInput is invalid")
 	}
 
-	userData, err := generate("controlplane", controlPlaneBashScript, input)
+	fMap := map[string]interface{}{
+		"Base64Encode": templateBase64Encode,
+	}
+
+	userData, err := generateWithFuncs("controlplane", controlPlaneCloudInit, funcMap(fMap), input)
 	if err != nil {
 		return "", errors.Wrapf(err, "failed to generate user data for new control plane machine")
 	}
@@ -222,15 +302,23 @@ func NewControlPlane(input *ControlPlaneInput) (string, error) {
 
 // JoinControlPlane returns the user data string to be used on a new contrplplane instance.
 func JoinControlPlane(input *ContolPlaneJoinInput) (string, error) {
-	input.Header = defaultHeader
+	input.Header = cloudConfigHeader
 
 	if err := input.validateCertificates(); err != nil {
 		return "", errors.Wrapf(err, "ControlPlaneInput is invalid")
 	}
 
-	userData, err := generate("controlplane", controlPlaneJoinBashScript, input)
+	fMap := map[string]interface{}{
+		"Base64Encode": templateBase64Encode,
+	}
+
+	userData, err := generateWithFuncs("controlplane", controlPlaneJoinCloudInit, funcMap(fMap), input)
 	if err != nil {
 		return "", errors.Wrapf(err, "failed to generate user data for machine joining control plane")
 	}
 	return userData, err
 }
+
+func templateBase64Encode(s string) string {
+	return base64.StdEncoding.EncodeToString([]byte(s))
+}
diff --git a/pkg/cloud/aws/services/userdata/node.go b/pkg/cloud/aws/services/userdata/node.go
@@ -17,28 +17,29 @@ limitations under the License.
 package userdata
 
 const (
-	nodeBashScript = `{{.Header}}
-
-HOSTNAME="$(curl http://169.254.169.254/latest/meta-data/local-hostname)"
-
-cat >/tmp/kubeadm-node.yaml <<EOF
----
-apiVersion: kubeadm.k8s.io/v1beta1
-kind: JoinConfiguration
-discovery:
-  bootstrapToken:
-    token: "{{.BootstrapToken}}"
-    apiServerEndpoint: "{{.ELBAddress}}:6443"
-    caCertHashes:
-      - "{{.CACertHash}}"
-nodeRegistration:
-  name: "${HOSTNAME}"
-  criSocket: /var/run/containerd/containerd.sock
-  kubeletExtraArgs:
-    cloud-provider: aws
-EOF
-
-kubeadm join --config /tmp/kubeadm-node.yaml
+	nodeCloudInit = `{{.Header}}
+write_files:
+-   path: /tmp/kubeadm-node.yaml
+    owner: root:root
+    permissions: '0640'
+    content: |
+      ---
+      apiVersion: kubeadm.k8s.io/v1beta1
+      kind: JoinConfiguration
+      discovery:
+        bootstrapToken:
+          token: "{{.BootstrapToken}}"
+          apiServerEndpoint: "{{.ELBAddress}}:6443"
+          caCertHashes:
+            - "{{.CACertHash}}"
+      nodeRegistration:
+        name: {{ "{{ ds.meta_data.hostname }}" }}
+        criSocket: /var/run/containerd/containerd.sock
+        kubeletExtraArgs:
+          cloud-provider: aws
+kubeadm:
+  operation: join
+  config: /tmp/kubeadm-node.yaml
 `
 )
 
@@ -53,6 +54,6 @@ type NodeInput struct {
 
 // NewNode returns the user data string to be used on a node instance.
 func NewNode(input *NodeInput) (string, error) {
-	input.Header = defaultHeader
-	return generate("node", nodeBashScript, input)
+	input.Header = cloudConfigHeader
+	return generate("node", nodeCloudInit, input)
 }