use manifest install for containerd, add some manifest schema validat…

…ion (Azure#1467)

.github/workflows/validate-components.yml

@@ -0,0 +1,34 @@
+name: Validate component manifest
+on:
+  pull_request:
+    branches:
+      - master
+  workflow_dispatch: {}
+
+jobs: 
+  build-and-deploy:
+    runs-on: ubuntu-latest
+    environment: test
+    steps:
+      - uses: actions/checkout@v2
+      - name: 'install cue'
+        run: |
+          go version
+          GOPATH="$(go env | grep GOPATH | cut -d= -f2 | tr -d '"')"
+          export PATH="$PATH:$GOPATH/bin"
+          pushd /tmp
+          GO111MODULE=on go get cuelang.org/go/cmd/cue
+          popd
+      - name: 'validate components.json'
+        run: |
+          GOPATH="$(go env | grep GOPATH | cut -d= -f2 | tr -d '"')"
+          export PATH="$PATH:$GOPATH/bin"
+          cue vet ./schemas/manifest.cue ./vhdbuilder/packer/manifest.json
+          cue eval ./schemas/manifest.cue ./vhdbuilder/packer/manifest.json
+      - name: 'validate components.json'
+        run: |
+          GOPATH="$(go env | grep GOPATH | cut -d= -f2 | tr -d '"')"
+          export PATH="$PATH:$GOPATH/bin"
+          cue vet ./schemas/components.cue ./vhdbuilder/packer/components.json
+          cue eval ./schemas/components.cue ./vhdbuilder/packer/components.json
+

.pipelines/.vsts-vhd-builder.yaml

@@ -49,6 +49,14 @@ stages:
             vstsFeedPackage: asc-baseline
             vstsPackageVersion: '1.0.0-35'
             downloadDirectory: vhdbuilder/packer
+        - bash: |
+            GOPATH="$(go env | grep GOPATH | cut -d= -f2 | tr -d '"')"
+            export PATH="$PATH:$GOPATH/bin"
+            pushd /tmp
+            GO111MODULE=on go get cuelang.org/go/cmd/cue
+            popd
+            cue vet ./schemas/manifest.cue ./vhdbuilder/packer/manifest.json
+          displayName: setup cue
         - bash: |
             echo MODE=$(MODE) && \
             docker run --rm \

.pipelines/templates/.builder-release-template.yaml

@@ -4,6 +4,14 @@ parameters:
       default: 1604
 
 steps:
+  - bash: |
+      GOPATH="$(go env | grep GOPATH | cut -d= -f2 | tr -d '"')"
+      export PATH="$PATH:$GOPATH/bin"
+      pushd /tmp
+      GO111MODULE=on go get cuelang.org/go/cmd/cue
+      popd
+      cue vet ./schemas/manifest.cue ./vhdbuilder/packer/manifest.json
+    displayName: setup cue
   - bash: |
         if [[ $(HYPERV_GENERATION) == "V2" ]]; then m="gen2Mode"; elif [[ -n ${SIG_GALLERY_NAME} && -n ${SIG_IMAGE_NAME} && -n ${SIG_IMAGE_VERSION} ]]; then m="sigMode"; else m="default"; fi && \
         echo "Set build mode to $m" && \

e2e/e2e-script.sh

@@ -152,12 +152,11 @@ KUBECONFIG=$(pwd)/kubeconfig; export KUBECONFIG
 
 # Sleep to let the automatic upgrade of the VM finish
 waitForNodeStartTime=$(date +%s)
-for i in $(seq 1 6); do
+for i in $(seq 1 10); do
     set +e
-    kubectl get nodes | grep -q $vmInstanceName
-    retval=$?
-    set -e
-    if [ "$retval" -ne 0 ]; then
+    # pipefail interferes with conditional.
+    # shellcheck disable=SC2143
+    if [ -z "$(kubectl get nodes | grep $vmInstanceName)" ]; then
         log "retrying attempt $i"
         sleep 10s
         continue
@@ -167,7 +166,7 @@ done
 waitForNodeEndTime=$(date +%s)
 log "Waited $((waitForNodeEndTime-waitForNodeStartTime)) seconds for node to join"
 
-# trap 'az vmss delete -g $MC_RESOURCE_GROUP_NAME -n $VMSS_NAME --no-wait' EXIT
+trap 'az vmss delete -g $MC_RESOURCE_GROUP_NAME -n $VMSS_NAME --no-wait' EXIT
 
 # Check if the node joined the cluster
 if [[ "$retval" -eq 0 ]]; then
@@ -178,9 +177,14 @@ else
     PRIVATE_IP="$(az vmss nic list-vm-nics --vmss-name $VMSS_NAME -g $MC_RESOURCE_GROUP_NAME --instance-id $INSTANCE_ID | jq -r .[0].ipConfigurations[0].privateIpAddress)"
     SSH_KEY=$(cat ~/.ssh/id_rsa)
     SSH_OPTS="-o PasswordAuthentication=no -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=5"
-    CMD="echo '$SSH_KEY' > sshkey && chmod 0600 sshkey && ssh -i sshkey $SSH_OPTS azureuser@$PRIVATE_IP cat /var/log/azure/cluster-provision.log"
-    exec_on_host "$CMD" cluster-provision.log
+    SSH_CMD="echo '$SSH_KEY' > sshkey && chmod 0600 sshkey && ssh -i sshkey $SSH_OPTS azureuser@$PRIVATE_IP"
+    exec_on_host "$SSH_CMD cat /var/log/azure/cluster-provision.log" cluster-provision.log
+    exec_on_host "$SSH_CMD systemctl status kubelet" kubelet-status
+    exec_on_host "$SSH_CMD journalctl -u kubelet -r | head -n 500" kubelet.log
+
     cat cluster-provision.log
+    cat kubelet.log
+    cat kubelet-status
     exit 1
 fi
 

schemas/components.cue

@@ -0,0 +1,25 @@
+package components
+
+#ContainerImage: {
+	downloadURL: string
+	amd64OnlyVersions: [...string]
+	multiArchVersions: [...]
+}
+
+#DownloadFile: {
+	fileName:         string
+	downloadLocation: string
+	downloadURL:      string
+	versions: [...string]
+	targetContainerRuntime?: "containerd" | _|_
+}
+
+#Images: [...#ContainerImage]
+#Files: [...#DownloadFile]
+
+#Components: {
+	ContainerImages: #Images
+	DownloadFiles:   #Files
+}
+
+#Components

schemas/manifest.cue

@@ -0,0 +1,38 @@
+// this manifest drives component versions installed during vhd build + cse
+// this file is similar in nature to components.json, but allows broader customization per component
+// it also inverts the key order to make specific components more easily patchable via automation (kubelet, containerd)
+// it's effectively json, but written using cuelang for schema validation
+// export it to json with cue export manifest.cue
+
+// some basic json constraints for validation
+#dep: {
+	fileName:         string
+	downloadLocation: string
+	downloadURL:      string
+	versions: [...string]
+	installedVersion: string
+}
+
+// semver with a revision e.g. 1.4.12-2
+#containerd_ver: =~"[0-9]+.[0-9]+.[0-9]+-[0-9]+"
+
+// containerd includes constraints from #dep and tighter bounds on version
+#containerd: #dep & {
+	versions: [...#containerd_ver]
+}
+
+#runc_ver: =~"[0-9]+.[0-9]+.[0-9]+-(rc)?[0-9]+" // rc92,rc95 previously used.
+
+#runc: #dep & {
+	versions: [...#runc_ver]
+}
+
+// root object schema enforced against manifest.json
+#root: {
+	runc:                       #runc
+	containerd:                 #containerd
+	"nvidia-container-runtime": #runc // demo failure case
+	[string]:                   #dep
+}
+
+#root

vhdbuilder/packer/components.json

@@ -478,4 +478,4 @@
       "targetContainerRuntime": "containerd"
     }
   ]
-}
+}

vhdbuilder/packer/install-dependencies.sh

@@ -20,6 +20,7 @@ source /home/packer/packer_source.sh
 CPU_ARCH=$(getCPUArch)  #amd64 or arm64
 VHD_LOGS_FILEPATH=/opt/azure/vhd-install.complete
 COMPONENTS_FILEPATH=/opt/azure/components.json
+MANIFEST_FILEPATH=/opt/azure/manifest.json
 KUBE_PROXY_IMAGES_FILEPATH=/opt/azure/kube-proxy-images.json
 #this is used by post build test to check whether the compoenents do indeed exist
 cat components.json > ${COMPONENTS_FILEPATH}
@@ -117,22 +118,22 @@ echo "  - krustlet ${KRUSTLET_VERSION}" >> ${VHD_LOGS_FILEPATH}
 
 if [[ ${CONTAINER_RUNTIME:-""} == "containerd" ]]; then
   echo "VHD will be built with containerd as the container runtime"
-  containerd_version="1.4.12"
-  containerd_patch_version="2"
-  downloadContainerdFromVersion ${containerd_version} ${containerd_patch_version}
+  containerd_manifest="$(jq .containerd manifest.json)" || exit $?
+  containerd_versions="$(echo ${containerd_manifest} | jq -r '.versions[]')" || exit $?
+
+  for version in $containerd_versions; do
+    containerd_version="$(echo "$version" | cut -d- -f1)"
+    containerd_patch_version="$(echo "$version" | cut -d- -f2)"
+    downloadContainerd ${containerd_version} ${containerd_patch_version}
+    echo "  - [cached] containerd v${containerd_version}-${containerd_patch_version}" >> ${VHD_LOGS_FILEPATH}
+  done
+
+  installed_version="$(echo ${containerd_manifest} | jq -r '.installedVersion')"
+  containerd_version="$(echo "$installed_version" | cut -d- -f1)"
+  containerd_patch_version="$(echo "$installed_version" | cut -d- -f2)"
   installStandaloneContainerd ${containerd_version} ${containerd_patch_version}
   echo "  - [installed] containerd v${containerd_version}-${containerd_patch_version}" >> ${VHD_LOGS_FILEPATH}
-  if [[ $OS == $UBUNTU_OS_NAME ]]; then
-    # also pre-cache containerd 1.4.4 (last used version)
-    containerd_version="1.4.9"
-    containerd_patch_version="3"
-    downloadContainerdFromVersion ${containerd_version} ${containerd_patch_version}
-    echo "  - [cached] containerd v${containerd_version}-${containerd_patch_version}" >> ${VHD_LOGS_FILEPATH}
-    updated_containerd_version="1.5.9" # also 1.5.9 revision
-    containerd_patch_version="2"
-    downloadContainerdFromVersion ${updated_containerd_version} ${containerd_patch_version}
-    echo "  - [cached] updated containerd v${updated_containerd_version}-${containerd_patch_version}" >> ${VHD_LOGS_FILEPATH}
-  fi
+
   CRICTL_VERSIONS="
   1.19.0
   1.20.0

vhdbuilder/packer/manifest.json

@@ -0,0 +1,43 @@
+{
+    "containerd": {
+        "fileName": "moby-containerd_${CONTAINERD_VERSION}+azure-${CONTAINERD_PATCH_VERSION}.deb",
+        "downloadLocation": "/opt/containerd/downloads",
+        "downloadURL": "https://moby.blob.core.windows.net/moby/moby-containerd/${CONTAINERD_VERSION}+azure/bionic/linux_${CPU_ARCH}/moby-containerd_${CONTAINERD_VERSION}+azure-${CONTAINERD_PATCH_VERSION}_${CPU_ARCH}.deb",
+        "versions": [
+            "1.4.9-3",
+            "1.4.12-2"
+        ],
+        "installedVersion": "1.5.9-2"
+    },
+    "runc": {
+        "fileName": "moby-runc_${RUNC_VERSION}+azure-${RUNC_PATCH_VERSION}.deb",
+        "downloadLocation": "/opt/runc/downloads",
+        "downloadURL": "https://moby.blob.core.windows.net/moby/moby-runc/${RUNC_VERSION}+azure/bionic/linux_${CPU_ARCH}/moby-runc_${RUNC_VERSION}+azure-${RUNC_PATCH_VERSION}_${CPU_ARCH}.deb",
+        "versions": [
+            "1.0.0-rc92",
+            "1.0.0-rc95"
+        ],
+        "installedVersion": "1.0.3"
+    },
+    "nvidia-container-runtime": {
+        "fileName":         "",
+        "downloadLocation": "",
+        "downloadURL":      "",
+        "versions": [],
+        "installedVersion": ""
+    },
+    "nvidia-drivers": {
+        "fileName":         "",
+        "downloadLocation": "",
+        "downloadURL":      "",
+        "versions": [],
+        "installedVersion": ""
+    },
+    "_template": {
+        "fileName":         "",
+        "downloadLocation": "",
+        "downloadURL":      "",
+        "versions": [],
+        "installedVersion": ""
+    }
+}

vhdbuilder/packer/vhd-image-builder-arm64-gen2.json

@@ -125,6 +125,11 @@
       "source": "vhdbuilder/packer/components.json",
       "destination": "/home/packer/components.json"
     },
+    {
+      "type": "file",
+      "source": "vhdbuilder/packer/manifest.json",
+      "destination": "/home/packer/manifest.json"
+    },
     {
       "type": "file",
       "source": "vhdbuilder/packer/kube-proxy-images.json",

vhdbuilder/packer/vhd-image-builder-gen2.json

@@ -117,6 +117,11 @@
       "source": "vhdbuilder/packer/components.json",
       "destination": "/home/packer/components.json"
     },
+    {
+      "type": "file",
+      "source": "vhdbuilder/packer/manifest.json",
+      "destination": "/home/packer/manifest.json"
+    },
     {
       "type": "file",
       "source": "vhdbuilder/packer/kube-proxy-images.json",

vhdbuilder/packer/vhd-image-builder-sig.json

@@ -121,6 +121,11 @@
       "source": "vhdbuilder/packer/components.json",
       "destination": "/home/packer/components.json"
     },
+    {
+      "type": "file",
+      "source": "vhdbuilder/packer/manifest.json",
+      "destination": "/home/packer/manifest.json"
+    },
     {
       "type": "file",
       "source": "vhdbuilder/packer/kube-proxy-images.json",

vhdbuilder/packer/vhd-image-builder-sigminimal.json

@@ -121,6 +121,11 @@
       "source": "vhdbuilder/packer/components.json",
       "destination": "/home/packer/components.json"
     },
+    {
+      "type": "file",
+      "source": "vhdbuilder/packer/manifest.json",
+      "destination": "/home/packer/manifest.json"
+    },
     {
       "type": "file",
       "source": "vhdbuilder/packer/kube-proxy-images.json",

vhdbuilder/packer/vhd-image-builder.json

@@ -110,6 +110,11 @@
       "source": "vhdbuilder/packer/components.json",
       "destination": "/home/packer/components.json"
     },
+    {
+      "type": "file",
+      "source": "vhdbuilder/packer/manifest.json",
+      "destination": "/home/packer/manifest.json"
+    },
     {
       "type": "file",
       "source": "vhdbuilder/packer/kube-proxy-images.json",