kubelet runs on master

controllers/packetmachine_controller.go

@@ -175,13 +175,13 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
 
 	if !machineScope.Cluster.Status.InfrastructureReady {
 		machineScope.Info("Cluster infrastructure is not ready yet")
-		return ctrl.Result{Requeue: true, RequeueAfter: 30 * time.Second}, nil
+		return ctrl.Result{}, nil
 	}
 
 	// Make sure bootstrap data secret is available and populated.
 	if machineScope.Machine.Spec.Bootstrap.DataSecretName == nil {
 		machineScope.Info("Bootstrap data secret is not yet available")
-		return ctrl.Result{Requeue: true, RequeueAfter: 30 * time.Second}, nil
+		return ctrl.Result{}, nil
 	}
 
 	providerID := machineScope.GetInstanceID()
@@ -203,14 +203,9 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
 			packet.GenerateMachineTag(mUID),
 			packet.GenerateClusterTag(clusterScope.Name()),
 		}
-		if machineScope.IsControlPlane() {
-			tags = append(tags, infrastructurev1alpha3.MasterTag)
-		} else {
-			tags = append(tags, infrastructurev1alpha3.WorkerTag)
-		}
 
 		name := machineScope.Name()
-		dev, err = r.PacketClient.NewDevice(name, clusterScope.PacketCluster.Spec.ProjectID, machineScope.PacketMachine.Spec, tags)
+		dev, err = r.PacketClient.NewDevice(name, clusterScope.PacketCluster.Spec.ProjectID, machineScope, tags)
 		if err != nil {
 			errs := fmt.Errorf("failed to create machine %s: %v", name, err)
 			machineScope.SetErrorReason(capierrors.CreateMachineError)

pkg/cloud/packet/client.go

@@ -5,8 +5,10 @@ import (
 	"os"
 	"strings"
 
-	infrav1 "github.com/packethost/cluster-api-provider-packet/api/v1alpha3"
+	infrastructurev1alpha3 "github.com/packethost/cluster-api-provider-packet/api/v1alpha3"
+	"github.com/packethost/cluster-api-provider-packet/pkg/cloud/packet/scope"
 	"github.com/packethost/packngo"
+	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 )
 
@@ -43,16 +45,26 @@ func (p *PacketClient) GetDevice(deviceID string) (*packngo.Device, error) {
 	return dev, err
 }
 
-func (p *PacketClient) NewDevice(hostname, project string, spec infrav1.PacketMachineSpec, extraTags []string) (*packngo.Device, error) {
-	tags := append(spec.Tags, extraTags...)
+func (p *PacketClient) NewDevice(hostname, project string, machineScope *scope.MachineScope, extraTags []string) (*packngo.Device, error) {
+	userData, err := machineScope.GetRawBootstrapData()
+	if err != nil {
+		return nil, errors.Wrap(err, "impossible to retrieve bootstrap data from secret")
+	}
+	tags := append(machineScope.PacketMachine.Spec.Tags, extraTags...)
+	if machineScope.IsControlPlane() {
+		tags = append(tags, infrastructurev1alpha3.MasterTag)
+	} else {
+		tags = append(tags, infrastructurev1alpha3.WorkerTag)
+	}
 	serverCreateOpts := &packngo.DeviceCreateRequest{
 		Hostname:     hostname,
 		ProjectID:    project,
-		Facility:     spec.Facility,
-		BillingCycle: spec.BillingCycle,
-		Plan:         spec.MachineType,
-		OS:           spec.OS,
+		Facility:     machineScope.PacketMachine.Spec.Facility,
+		BillingCycle: machineScope.PacketMachine.Spec.BillingCycle,
+		Plan:         machineScope.PacketMachine.Spec.MachineType,
+		OS:           machineScope.PacketMachine.Spec.OS,
 		Tags:         tags,
+		UserData:     string(userData),
 	}
 
 	dev, _, err := p.Client.Devices.Create(serverCreateOpts)

pkg/cloud/packet/scope/cluster.go

@@ -26,16 +26,13 @@ import (
 
 	"k8s.io/klog/klogr"
 
-	packet "github.com/packethost/cluster-api-provider-packet/pkg/cloud/packet"
-
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
 	"sigs.k8s.io/cluster-api/util/patch"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 // ClusterScopeParams defines the input parameters used to create a new Scope.
 type ClusterScopeParams struct {
-	PacketClient  packet.PacketClient
 	Client        client.Client
 	Logger        logr.Logger
 	Cluster       *clusterv1.Cluster
@@ -63,7 +60,6 @@ func NewClusterScope(params ClusterScopeParams) (*ClusterScope, error) {
 	return &ClusterScope{
 		Logger:        params.Logger,
 		client:        params.Client,
-		PacketClient:  params.PacketClient,
 		Cluster:       params.Cluster,
 		PacketCluster: params.PacketCluster,
 		patchHelper:   helper,
@@ -76,7 +72,6 @@ type ClusterScope struct {
 	client      client.Client
 	patchHelper *patch.Helper
 
-	PacketClient  packet.PacketClient
 	Cluster       *clusterv1.Cluster
 	PacketCluster *infrav1.PacketCluster
 }

pkg/cloud/packet/scope/machine.go

@@ -23,9 +23,9 @@ import (
 	"github.com/pkg/errors"
 
 	infrav1 "github.com/packethost/cluster-api-provider-packet/api/v1alpha3"
-	packet "github.com/packethost/cluster-api-provider-packet/pkg/cloud/packet"
 
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/klog/klogr"
 	"k8s.io/utils/pointer"
 
@@ -39,7 +39,6 @@ import (
 
 // MachineScopeParams defines the input parameters used to create a new MachineScope.
 type MachineScopeParams struct {
-	PacketClient  packet.PacketClient
 	Client        client.Client
 	Logger        logr.Logger
 	Cluster       *clusterv1.Cluster
@@ -104,12 +103,12 @@ func (m *MachineScope) Close() error {
 	return m.patchHelper.Patch(context.TODO(), m.PacketMachine)
 }
 
-// Name returns the DOMachine name.
+// Name returns the PacketMachine name
 func (m *MachineScope) Name() string {
 	return m.PacketMachine.Name
 }
 
-// Namespace returns the namespace name.
+// Namespace returns the PacketMachine namespace
 func (m *MachineScope) Namespace() string {
 	return m.PacketMachine.Namespace
 }
@@ -188,3 +187,23 @@ func (m *MachineScope) Tags() infrav1.Tags {
 
 	return m.PacketMachine.Spec.Tags.DeepCopy()
 }
+
+// GetRawBootstrapData returns the bootstrap data from the secret in the Machine's bootstrap.dataSecretName.
+func (m *MachineScope) GetRawBootstrapData() ([]byte, error) {
+	if m.Machine.Spec.Bootstrap.DataSecretName == nil {
+		return nil, errors.New("error retrieving bootstrap data: linked Machine's bootstrap.dataSecretName is nil")
+	}
+
+	secret := &corev1.Secret{}
+	key := types.NamespacedName{Namespace: m.Namespace(), Name: *m.Machine.Spec.Bootstrap.DataSecretName}
+	if err := m.client.Get(context.TODO(), key, secret); err != nil {
+		return nil, errors.Wrapf(err, "failed to retrieve bootstrap data secret for PacketMachine %s/%s", m.Namespace(), m.Name())
+	}
+
+	value, ok := secret.Data["value"]
+	if !ok {
+		return nil, errors.New("error retrieving bootstrap data: secret value key is missing")
+	}
+
+	return value, nil
+}

templates/cluster-template.yaml

@@ -4,10 +4,28 @@ apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
 metadata:
   name: "${CLUSTER_NAME}-control-plane1-config"
 spec:
+  preKubeadmCommands:
+    - swapoff -a
+    - apt-get -y update
+    - DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
+    - curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
+    - echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list
+    - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+    - apt-key fingerprint 0EBFCD88
+    - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+    - apt-get update -y
+    - apt-get install -y ca-certificates socat jq ebtables apt-transport-https cloud-utils prips docker-ce docker-ce-cli containerd.io kubelet kubeadm kubectl
+    - systemctl daemon-reload
+    - systemctl enable docker
+    - systemctl start docker
+  postKubeadmCommands: 
+    - "kubectl apply --kubeconfig /etc/kubernetes/admin.conf -f https://docs.projectcalico.org/v3.11/manifests/calico.yaml"
+    - "kubectl --kubeconfig /etc/kubernetes/admin.conf create secret generic -n kube-system packet-cloud-config --from-literal=cloud-sa.json='{\"apiKey\": \"${PACKET_API_KEY}\",\"projectID\": \"${PROJECT_ID}\"}'"
+    - "kubectl apply --kubeconfig /etc/kubernetes/admin.conf -f https://raw.githubusercontent.com/packethost/packet-ccm/master/deploy/releases/v1.0.0/deployment.yaml"
   initConfiguration:
     nodeRegistration:
       kubeletExtraArgs:
-        eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%
+        cloud-provider: external
   clusterConfiguration:
     controllerManager:
       extraArgs:
@@ -135,17 +153,45 @@ apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
 metadata:
   name: "${CLUSTER_NAME}-worker1-config"
 spec:
+  preKubeadmCommands:
+    - swapoff -a
+    - apt-get -y update
+    - DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
+    - curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
+    - echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list
+    - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+    - apt-key fingerprint 0EBFCD88
+    - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+    - apt-get update -y
+    - apt-get install -y ca-certificates socat jq ebtables apt-transport-https cloud-utils prips docker-ce docker-ce-cli containerd.io kubelet kubeadm kubectl
+    - systemctl daemon-reload
+    - systemctl enable docker
+    - systemctl start docker
   joinConfiguration:
     nodeRegistration:
       kubeletExtraArgs:
-        eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%
+        cloud-provider: external 
 ---
 kind: KubeadmConfig
 apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
 metadata:
   name: "${CLUSTER_NAME}-worker0-config"
 spec:
+  preKubeadmCommands:
+    - swapoff -a
+    - apt-get -y update
+    - DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
+    - curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
+    - echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list
+    - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+    - apt-key fingerprint 0EBFCD88
+    - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+    - apt-get update -y
+    - apt-get install -y ca-certificates socat jq ebtables apt-transport-https cloud-utils prips docker-ce docker-ce-cli containerd.io kubelet kubeadm kubectl
+    - systemctl daemon-reload
+    - systemctl enable docker
+    - systemctl start docker
   joinConfiguration:
     nodeRegistration:
       kubeletExtraArgs:
-        eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%
+        cloud-provider: external