|  | 
|  | 1 | +apiVersion: v1 | 
|  | 2 | +data: ${CNI_RESOURCES} | 
|  | 3 | +kind: ConfigMap | 
|  | 4 | +metadata: | 
|  | 5 | +  name: ${CLUSTER_NAME}-crs-cni | 
|  | 6 | +--- | 
|  | 7 | +apiVersion: addons.cluster.x-k8s.io/v1beta1 | 
|  | 8 | +kind: ClusterResourceSet | 
|  | 9 | +metadata: | 
|  | 10 | +  name: ${CLUSTER_NAME}-crs-cni | 
|  | 11 | +spec: | 
|  | 12 | +  clusterSelector: | 
|  | 13 | +    matchLabels: | 
|  | 14 | +      cni: ${CLUSTER_NAME}-crs-cni | 
|  | 15 | +  resources: | 
|  | 16 | +  - kind: ConfigMap | 
|  | 17 | +    name: ${CLUSTER_NAME}-crs-cni | 
|  | 18 | +  strategy: ApplyOnce | 
|  | 19 | +--- | 
|  | 20 | +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 | 
|  | 21 | +kind: KubeadmConfigTemplate | 
|  | 22 | +metadata: | 
|  | 23 | +  name: ${CLUSTER_NAME}-worker-a | 
|  | 24 | +spec: | 
|  | 25 | +  template: | 
|  | 26 | +    spec: | 
|  | 27 | +      joinConfiguration: | 
|  | 28 | +        nodeRegistration: | 
|  | 29 | +          kubeletExtraArgs: | 
|  | 30 | +            cloud-provider: external | 
|  | 31 | +            provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }} | 
|  | 32 | +      preKubeadmCommands: | 
|  | 33 | +      - |- | 
|  | 34 | +        sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab | 
|  | 35 | +        swapoff -a | 
|  | 36 | +        mount -a | 
|  | 37 | +        cat <<EOF > /etc/modules-load.d/containerd.conf | 
|  | 38 | +        overlay | 
|  | 39 | +        br_netfilter | 
|  | 40 | +        EOF | 
|  | 41 | +        modprobe overlay | 
|  | 42 | +        modprobe br_netfilter | 
|  | 43 | +        cat <<EOF > /etc/sysctl.d/99-kubernetes-cri.conf | 
|  | 44 | +        net.bridge.bridge-nf-call-iptables  = 1 | 
|  | 45 | +        net.ipv4.ip_forward                 = 1 | 
|  | 46 | +        net.bridge.bridge-nf-call-ip6tables = 1 | 
|  | 47 | +        EOF | 
|  | 48 | +        sysctl --system | 
|  | 49 | +        apt-get -y update | 
|  | 50 | +        DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq | 
|  | 51 | +        curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - | 
|  | 52 | +        echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list | 
|  | 53 | +        apt-get update -y | 
|  | 54 | +        TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//') | 
|  | 55 | +        RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1) | 
|  | 56 | +        DEBIAN_FRONTEND=noninteractive apt-get install -y containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION} | 
|  | 57 | +--- | 
|  | 58 | +apiVersion: cluster.x-k8s.io/v1beta1 | 
|  | 59 | +kind: Cluster | 
|  | 60 | +metadata: | 
|  | 61 | +  labels: | 
|  | 62 | +    cni: ${CLUSTER_NAME}-crs-cni | 
|  | 63 | +  name: ${CLUSTER_NAME} | 
|  | 64 | +spec: | 
|  | 65 | +  clusterNetwork: | 
|  | 66 | +    pods: | 
|  | 67 | +      cidrBlocks: | 
|  | 68 | +      - ${POD_CIDR:=192.168.0.0/16} | 
|  | 69 | +    services: | 
|  | 70 | +      cidrBlocks: | 
|  | 71 | +      - ${SERVICE_CIDR:=172.26.0.0/16} | 
|  | 72 | +  controlPlaneRef: | 
|  | 73 | +    apiVersion: controlplane.cluster.x-k8s.io/v1beta1 | 
|  | 74 | +    kind: KubeadmControlPlane | 
|  | 75 | +    name: ${CLUSTER_NAME}-control-plane | 
|  | 76 | +  infrastructureRef: | 
|  | 77 | +    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 | 
|  | 78 | +    kind: PacketCluster | 
|  | 79 | +    name: ${CLUSTER_NAME} | 
|  | 80 | +--- | 
|  | 81 | +apiVersion: cluster.x-k8s.io/v1beta1 | 
|  | 82 | +kind: MachineDeployment | 
|  | 83 | +metadata: | 
|  | 84 | +  labels: | 
|  | 85 | +    cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} | 
|  | 86 | +    pool: worker-a | 
|  | 87 | +  name: ${CLUSTER_NAME}-worker-a | 
|  | 88 | +spec: | 
|  | 89 | +  clusterName: ${CLUSTER_NAME} | 
|  | 90 | +  replicas: ${WORKER_MACHINE_COUNT} | 
|  | 91 | +  selector: | 
|  | 92 | +    matchLabels: | 
|  | 93 | +      cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} | 
|  | 94 | +      pool: worker-a | 
|  | 95 | +  template: | 
|  | 96 | +    metadata: | 
|  | 97 | +      labels: | 
|  | 98 | +        cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} | 
|  | 99 | +        pool: worker-a | 
|  | 100 | +    spec: | 
|  | 101 | +      bootstrap: | 
|  | 102 | +        configRef: | 
|  | 103 | +          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 | 
|  | 104 | +          kind: KubeadmConfigTemplate | 
|  | 105 | +          name: ${CLUSTER_NAME}-worker-a | 
|  | 106 | +      clusterName: ${CLUSTER_NAME} | 
|  | 107 | +      infrastructureRef: | 
|  | 108 | +        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 | 
|  | 109 | +        kind: PacketMachineTemplate | 
|  | 110 | +        name: ${CLUSTER_NAME}-worker-a | 
|  | 111 | +      version: ${KUBERNETES_VERSION} | 
|  | 112 | +--- | 
|  | 113 | +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 | 
|  | 114 | +kind: KubeadmControlPlane | 
|  | 115 | +metadata: | 
|  | 116 | +  name: ${CLUSTER_NAME}-control-plane | 
|  | 117 | +spec: | 
|  | 118 | +  kubeadmConfigSpec: | 
|  | 119 | +    clusterConfiguration: | 
|  | 120 | +      apiServer: | 
|  | 121 | +        extraArgs: | 
|  | 122 | +          cloud-provider: external | 
|  | 123 | +      controllerManager: | 
|  | 124 | +        extraArgs: | 
|  | 125 | +          cloud-provider: external | 
|  | 126 | +    initConfiguration: | 
|  | 127 | +      nodeRegistration: | 
|  | 128 | +        kubeletExtraArgs: | 
|  | 129 | +          cloud-provider: external | 
|  | 130 | +          provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }} | 
|  | 131 | +    joinConfiguration: | 
|  | 132 | +      nodeRegistration: | 
|  | 133 | +        ignorePreflightErrors: | 
|  | 134 | +        - DirAvailable--etc-kubernetes-manifests | 
|  | 135 | +        kubeletExtraArgs: | 
|  | 136 | +          cloud-provider: external | 
|  | 137 | +          provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }} | 
|  | 138 | +    postKubeadmCommands: | 
|  | 139 | +    - |- | 
|  | 140 | +      if [ -f "/run/kubeadm/kubeadm.yaml" ]; then | 
|  | 141 | +        export KUBECONFIG=/etc/kubernetes/admin.conf | 
|  | 142 | +        export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/v3.5.0/deployment.yaml | 
|  | 143 | +        export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}", "loadbalancer": "kube-vip://", "facility": "${FACILITY}"}''' | 
|  | 144 | +        kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") | 
|  | 145 | +        kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML}) | 
|  | 146 | +      fi | 
|  | 147 | +    preKubeadmCommands: | 
|  | 148 | +    - | | 
|  | 149 | +      sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab | 
|  | 150 | +      swapoff -a | 
|  | 151 | +      mount -a | 
|  | 152 | +      cat <<EOF > /etc/modules-load.d/containerd.conf | 
|  | 153 | +      overlay | 
|  | 154 | +      br_netfilter | 
|  | 155 | +      EOF | 
|  | 156 | +      modprobe overlay | 
|  | 157 | +      modprobe br_netfilter | 
|  | 158 | +      cat <<EOF > /etc/sysctl.d/99-kubernetes-cri.conf | 
|  | 159 | +      net.bridge.bridge-nf-call-iptables  = 1 | 
|  | 160 | +      net.ipv4.ip_forward                 = 1 | 
|  | 161 | +      net.bridge.bridge-nf-call-ip6tables = 1 | 
|  | 162 | +      EOF | 
|  | 163 | +      sysctl --system | 
|  | 164 | +      apt-get -y update | 
|  | 165 | +      DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq | 
|  | 166 | +      curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - | 
|  | 167 | +      echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list | 
|  | 168 | +      apt-get update -y | 
|  | 169 | +      TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//') | 
|  | 170 | +      RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1) | 
|  | 171 | +      DEBIAN_FRONTEND=noninteractive apt-get install -y containerd kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION} | 
|  | 172 | +      curl -o /run/metadata.json -fsSL https://metadata.platformequinix.com/metadata | 
|  | 173 | +      for i in $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[]'); do | 
|  | 174 | +        ip route add $i via $(cat /run/metadata.json | jq -r '.network.addresses[] | select(.public == false and .address_family == 4) | .gateway') | 
|  | 175 | +      done | 
|  | 176 | +      KVVERSION="${KUBE_VIP_VERSION:=v0.5.0}" | 
|  | 177 | +      ctr image pull ghcr.io/kube-vip/kube-vip:$${KVVERSION} | 
|  | 178 | +      ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip /kube-vip manifest pod \ | 
|  | 179 | +      --interface "lo" \ | 
|  | 180 | +      --vip "{{ .controlPlaneEndpoint }}" \ | 
|  | 181 | +      --controlplane \ | 
|  | 182 | +      --bgp \ | 
|  | 183 | +      --peerAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_as') \ | 
|  | 184 | +      --peerAddress $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].peer_ips[0]') \ | 
|  | 185 | +      --localAS $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_as') \ | 
|  | 186 | +      --bgpRouterID $(cat /run/metadata.json | jq -r '.bgp_neighbors[0].customer_ip') > /etc/kubernetes/manifests/vip.yaml | 
|  | 187 | +      rm /run/metadata.json | 
|  | 188 | +  machineTemplate: | 
|  | 189 | +    infrastructureRef: | 
|  | 190 | +      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 | 
|  | 191 | +      kind: PacketMachineTemplate | 
|  | 192 | +      name: ${CLUSTER_NAME}-control-plane | 
|  | 193 | +  replicas: ${CONTROL_PLANE_MACHINE_COUNT} | 
|  | 194 | +  version: ${KUBERNETES_VERSION} | 
|  | 195 | +--- | 
|  | 196 | +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 | 
|  | 197 | +kind: PacketCluster | 
|  | 198 | +metadata: | 
|  | 199 | +  name: ${CLUSTER_NAME} | 
|  | 200 | +spec: | 
|  | 201 | +  facility: ${FACILITY} | 
|  | 202 | +  projectID: ${PROJECT_ID} | 
|  | 203 | +  vipManager: KUBE_VIP | 
|  | 204 | +--- | 
|  | 205 | +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 | 
|  | 206 | +kind: PacketMachineTemplate | 
|  | 207 | +metadata: | 
|  | 208 | +  name: ${CLUSTER_NAME}-control-plane | 
|  | 209 | +spec: | 
|  | 210 | +  template: | 
|  | 211 | +    spec: | 
|  | 212 | +      billingCycle: hourly | 
|  | 213 | +      machineType: ${CONTROLPLANE_NODE_TYPE} | 
|  | 214 | +      os: ${NODE_OS:=ubuntu_18_04} | 
|  | 215 | +      sshKeys: | 
|  | 216 | +      - ${SSH_KEY} | 
|  | 217 | +      tags: [] | 
|  | 218 | +--- | 
|  | 219 | +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 | 
|  | 220 | +kind: PacketMachineTemplate | 
|  | 221 | +metadata: | 
|  | 222 | +  name: ${CLUSTER_NAME}-worker-a | 
|  | 223 | +spec: | 
|  | 224 | +  template: | 
|  | 225 | +    spec: | 
|  | 226 | +      billingCycle: hourly | 
|  | 227 | +      machineType: ${WORKER_NODE_TYPE} | 
|  | 228 | +      os: ${NODE_OS:=ubuntu_18_04} | 
|  | 229 | +      sshKeys: | 
|  | 230 | +      - ${SSH_KEY} | 
|  | 231 | +      tags: [] | 
0 commit comments