chore: install containerd from manifest in cse (Azure#1491)

.github/workflows/check-generated.yml

@@ -6,11 +6,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
+    - uses: actions/setup-go@v2
+      with:
+        go-version: '^1.16'
     - run: |
-        make -C hack/tools/
-        pushd parts
-        ../hack/tools/bin/go-bindata --nometadata --nocompress -pkg templates -o ../pkg/templates/templates_generated_test.go ./...
-        popd
-        pushd pkg/templates/
-        diff templates_generated_test.go templates_generated.go
+        make
+        git diff --exit-code
       name: Make generate and diff

.github/workflows/check-shell.yml

@@ -6,6 +6,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
+    - uses: actions/setup-go@v2
+      with:
+        go-version: '^1.16'
     - run: |
         make validate-shell
       name: Verify Shell scripts

.github/workflows/check-tests.yml

@@ -6,6 +6,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
+    - uses: actions/setup-go@v2
+      with:
+        go-version: '^1.16'
     - run: |
         make test
       name: Unit Test

.github/workflows/validate-components.yml

@@ -23,12 +23,12 @@ jobs:
         run: |
           GOPATH="$(go env | grep GOPATH | cut -d= -f2 | tr -d '"')"
           export PATH="$PATH:$GOPATH/bin"
-          cue vet ./schemas/manifest.cue ./vhdbuilder/packer/manifest.json
-          cue eval ./schemas/manifest.cue ./vhdbuilder/packer/manifest.json
+          cue vet -c ./schemas/manifest.cue
+          cue eval ./schemas/manifest.cue
       - name: 'validate components.json'
         run: |
           GOPATH="$(go env | grep GOPATH | cut -d= -f2 | tr -d '"')"
           export PATH="$PATH:$GOPATH/bin"
-          cue vet ./schemas/components.cue ./vhdbuilder/packer/components.json
+          cue vet -c ./schemas/components.cue ./vhdbuilder/packer/components.json
           cue eval ./schemas/components.cue ./vhdbuilder/packer/components.json
 

.gitignore

@@ -31,3 +31,5 @@ cmd/_test_output
 !/test/e2e/vendor/**/*
 
 translations/
+
+hack/tools/bin

.pipelines/.vsts-vhd-builder.yaml

@@ -55,7 +55,7 @@ stages:
             pushd /tmp
             GO111MODULE=on go get cuelang.org/go/cmd/cue
             popd
-            cue vet ./schemas/manifest.cue ./vhdbuilder/packer/manifest.json
+            cue export ./schemas/manifest.cue > ./parts/linux/cloud-init/artifacts/manifest.json
           displayName: setup cue
         - bash: |
             echo MODE=$(MODE) && \

.pipelines/e2e.yaml

@@ -13,9 +13,17 @@ jobs:
       az login --service-principal -u $(AZURE_CLIENT_ID) -p $(AZURE_CLIENT_SECRET) --tenant $(AZURE_TENANT_ID)
       az account set -s $(AZURE_SUBSCRIPTION_ID)
     displayName: az login
+  - bash: bash .pipelines/scripts/setup_go.sh
+    displayName: setup go
   - bash: |
+      go version
       cd e2e
       bash ./e2e-script.sh
     displayName: run e2e
   - publish: $(System.DefaultWorkingDirectory)/e2e/logs
     artifact: logs
+    condition: always()
+  - bash: |
+      az vmss delete -g $(jq -r .group e2e/vmss.json) -n $(jq -r .vmss e2e/vmss.json)
+    displayName: delete vmss
+    condition: always()

.pipelines/scripts/setup_go.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+GOLANG_VERSION="go1.17.6"
+echo "Downloading ${GOLANG_VERSION}"
+curl -O "https://dl.google.com/go/${GOLANG_VERSION}.linux-amd64.tar.gz"
+
+echo "unpacking go"
+sudo mkdir -p /usr/local/go
+sudo chown -R "$(whoami):$(whoami)" /usr/local/go 
+sudo tar -xvf "${GOLANG_VERSION}.linux-amd64.tar.gz" -C /usr/local
+rm "${GOLANG_VERSION}.linux-amd64.tar.gz"
+
+export PATH="/usr/local/go/bin:$PATH"
+GOPATH="/home/$(whoami)/go"
+export GOPATH

.pipelines/templates/.builder-release-template.yaml

@@ -4,14 +4,6 @@ parameters:
       default: 1604
 
 steps:
-  - bash: |
-      GOPATH="$(go env | grep GOPATH | cut -d= -f2 | tr -d '"')"
-      export PATH="$PATH:$GOPATH/bin"
-      pushd /tmp
-      GO111MODULE=on go get cuelang.org/go/cmd/cue
-      popd
-      cue vet ./schemas/manifest.cue ./vhdbuilder/packer/manifest.json
-    displayName: setup cue
   - bash: |
         if [[ $(HYPERV_GENERATION) == "V2" ]]; then m="gen2Mode"; elif [[ -n ${SIG_GALLERY_NAME} && -n ${SIG_IMAGE_NAME} && -n ${SIG_IMAGE_VERSION} ]]; then m="sigMode"; else m="default"; fi && \
         echo "Set build mode to $m" && \
@@ -29,6 +21,14 @@ steps:
       pwd
       ls -R
     displayName: show Directory
+  - bash: |
+      GOPATH="$(go env | grep GOPATH | cut -d= -f2 | tr -d '"')"
+      export PATH="$PATH:$GOPATH/bin"
+      pushd /tmp
+      GO111MODULE=on go get cuelang.org/go/cmd/cue
+      popd
+      cue export ./schemas/manifest.cue > ./parts/linux/cloud-init/artifacts/manifest.json
+    displayName: setup cue
   - bash: |
         echo MODE=$(MODE) && \
         docker run --rm \

Makefile

@@ -85,6 +85,8 @@ validate-image-version:
 generate: bootstrap
 	@echo $(GOFLAGS)
 	@echo "$$(go-bindata --version)"
+	./hack/tools/bin/cue export ./schemas/manifest.cue > ./parts/linux/cloud-init/artifacts/manifest.json
+	@echo "#EOF" >> ./parts/linux/cloud-init/artifacts/manifest.json
 	(pushd parts && \
 	../hack/tools/bin/go-bindata --nometadata --nocompress -pkg templates -o ../pkg/templates/templates_generated.go ./... && \
 	popd \

e2e/e2e-script.sh

@@ -35,21 +35,27 @@ log "Created resource group in $((rgEndTime-rgStartTime)) seconds"
 # Check if there exists a cluster in the RG. If yes, check if the MC_RG associated with it still exists.
 # MC_RG gets deleted due to ACS-Test Garbage Collection but the cluster hangs around
 out=$(az aks list -g $RESOURCE_GROUP_NAME -ojson | jq '.[].name')
+create_cluster="false"
 if [ -n "$out" ]; then
-    MC_RG_NAME=$(az aks show -n $CLUSTER_NAME -g $RESOURCE_GROUP_NAME | jq -r '.nodeResourceGroup')
+    MC_RG_NAME="MC_${RESOURCE_GROUP_NAME}_${CLUSTER_NAME}_$LOCATION"
+    az vmss list -g $MC_RG_NAME -o table
+    MC_VMSS_NAME=$(az vmss list -g $MC_RG_NAME --query "[?contains(name, 'nodepool')]" -ojson | jq -r '.[0].name')
+
     exists=$(az group exists -n $MC_RG_NAME)
-    if [ $exists = "false" ]; then
+    if [ $exists = "false" ] ||  [ "$MC_VMSS_NAME" == "null" ]; then
         log "Deleting cluster"
         clusterDeleteStartTime=$(date +%s)
         az aks delete -n $CLUSTER_NAME -g $RESOURCE_GROUP_NAME --yes
         clusterDeleteEndTime=$(date +%s)
         log "Deleted cluster in $((clusterDeleteEndTime-clusterDeleteStartTime)) seconds"
-        out=""
+        create_cluster="true"
     fi
+else
+    create_cluster="true"
 fi
 
 # Create the AKS cluster and get the kubeconfig
-if [ -z "$out" ]; then
+if [ "$create_cluster" == "true" ]; then
     log "Creating cluster"
     clusterCreateStartTime=$(date +%s)
     az aks create -g $RESOURCE_GROUP_NAME -n $CLUSTER_NAME --node-count 1 --generate-ssh-keys -ojson
@@ -64,9 +70,10 @@ export KUBECONFIG
 # Store the contents of az aks show to a file to reduce API call overhead
 az aks show -n $CLUSTER_NAME -g $RESOURCE_GROUP_NAME -ojson > cluster_info.json
 
-MC_RESOURCE_GROUP_NAME=$(jq -r '.nodeResourceGroup' < cluster_info.json)
-VMSS_NAME=$(az vmss list -g $MC_RESOURCE_GROUP_NAME -ojson | jq -r '.[length -1].name')
-CLUSTER_ID=$(echo $VMSS_NAME | cut -d '-' -f3)
+MC_RESOURCE_GROUP_NAME="MC_${RESOURCE_GROUP_NAME}_${CLUSTER_NAME}_eastus"
+az vmss list -g $MC_RESOURCE_GROUP_NAME --query "[?contains(name, 'nodepool')]" -otable
+MC_VMSS_NAME=$(az vmss list -g $MC_RESOURCE_GROUP_NAME --query "[?contains(name, 'nodepool')]" -ojson | jq -r '.[0].name')
+CLUSTER_ID=$(echo $MC_VMSS_NAME | cut -d '-' -f3)
 
 # privileged ds with nsenter for host file exfiltration
 kubectl apply -f https://gist.githubusercontent.com/alexeldeib/01f2d3efc8fe17cca7625ecb7c1ec707/raw/6b90f4a12888ebb300bfb2f339cf2b43a66e35a2/deploy.yaml
@@ -75,6 +82,33 @@ kubectl rollout status deploy/debug
 exec_on_host() {
     kubectl exec $(kubectl get pod -l app=debug -o jsonpath="{.items[0].metadata.name}") -- bash -c "nsenter -t 1 -m bash -c \"$1\"" > $2
 }
+
+debug() {
+    local retval
+    retval=0
+    mkdir -p logs
+    INSTANCE_ID="$(az vmss list-instances --name $VMSS_NAME -g $MC_RESOURCE_GROUP_NAME | jq -r '.[0].instanceId')"
+    PRIVATE_IP="$(az vmss nic list-vm-nics --vmss-name $VMSS_NAME -g $MC_RESOURCE_GROUP_NAME --instance-id $INSTANCE_ID | jq -r .[0].ipConfigurations[0].privateIpAddress)"
+    set +x
+    SSH_KEY=$(cat ~/.ssh/id_rsa)
+    SSH_OPTS="-o PasswordAuthentication=no -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=5"
+    SSH_CMD="echo '$SSH_KEY' > sshkey && chmod 0600 sshkey && ssh -i sshkey $SSH_OPTS azureuser@$PRIVATE_IP sudo"
+    exec_on_host "$SSH_CMD cat /var/log/azure/cluster-provision.log" logs/cluster-provision.log || retval=$?
+    if [ "$retval" != "0" ]; then
+        echo "failed cat cluster-provision"
+    fi
+    exec_on_host "$SSH_CMD systemctl status kubelet" logs/kubelet-status.txt  || retval=$?
+    if [ "$retval" != "0" ]; then
+        echo "failed systemctl status kubelet"
+    fi
+    exec_on_host "$SSH_CMD journalctl -u kubelet -r | head -n 500" logs/kubelet.log  || retval=$?
+    if [ "$retval" != "0" ]; then
+        echo "failed journalctl -u kubelet"
+    fi
+    set -x
+    echo "debug done"
+}
+
 # Retrieve the etc/kubernetes/azure.json file for cluster related info
 log "Retrieving cluster info"
 clusterInfoStartTime=$(date +%s)
@@ -112,11 +146,20 @@ if [ ! -f ~/.ssh/id_rsa ]; then
     ssh-keygen -t rsa -b 4096 -f ~/.ssh/id_rsa -N ""
 fi
 
+VMSS_NAME="$(mktemp -u abtest-XXXXXXX | tr '[:upper:]' '[:lower:]')"
+tee vmss.json > /dev/null <<EOF
+{
+    "group": "${MC_RESOURCE_GROUP_NAME}",
+    "vmss": "${VMSS_NAME}"
+}
+EOF
+
+cat vmss.json
+
 # Create a test VMSS with 1 instance 
 # TODO 3: Discuss about the --image version, probably go with aks-ubuntu-1804-gen2-2021-q2:latest
 #       However, how to incorporate chaning quarters?
 log "Creating VMSS"
-VMSS_NAME="$(mktemp -u abtest-XXXXXXX | tr '[:upper:]' '[:lower:]')"
 vmssStartTime=$(date +%s)
 az vmss create -n ${VMSS_NAME} \
     -g $MC_RESOURCE_GROUP_NAME \
@@ -163,6 +206,15 @@ set -e
 vmssExtEndTime=$(date +%s)
 log "Applied extensions in $((vmssExtEndTime-vmssExtStartTime)) seconds"
 
+FAILED=0
+# Check if the node joined the cluster
+if [[ "$retval" != "0" ]]; then
+    err "cse failed to apply"
+    debug
+    tail -n 50 logs/cluster-provision.log || true
+    exit 1
+fi
+
 KUBECONFIG=$(pwd)/kubeconfig; export KUBECONFIG
 
 # Sleep to let the automatic upgrade of the VM finish
@@ -191,23 +243,13 @@ else
     FAILED=1
 fi
 
-mkdir -p logs
-INSTANCE_ID="$(az vmss list-instances --name $VMSS_NAME -g $MC_RESOURCE_GROUP_NAME | jq -r '.[0].instanceId')"
-PRIVATE_IP="$(az vmss nic list-vm-nics --vmss-name $VMSS_NAME -g $MC_RESOURCE_GROUP_NAME --instance-id $INSTANCE_ID | jq -r .[0].ipConfigurations[0].privateIpAddress)"
-SSH_KEY=$(cat ~/.ssh/id_rsa)
-SSH_OPTS="-o PasswordAuthentication=no -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=5"
-SSH_CMD="echo '$SSH_KEY' > sshkey && chmod 0600 sshkey && ssh -i sshkey $SSH_OPTS azureuser@$PRIVATE_IP sudo"
-exec_on_host "$SSH_CMD cat /var/log/azure/cluster-provision.log" logs/cluster-provision.log
-exec_on_host "$SSH_CMD systemctl status kubelet" logs/kubelet-status.txt
-exec_on_host "$SSH_CMD journalctl -u kubelet -r | head -n 500" logs/kubelet.log
-
-# useful for validating some stuff even on success
-cat logs/cluster-provision.log
+debug
+tail -n 50 logs/cluster-provision.log || true
 
 if [ "$FAILED" == "1" ]; then
     echo "node join failed, dumping logs for debug"
-    head -n 500 logs/kubelet.log
-    cat logs/kubelet-status.txt
+    head -n 500 logs/kubelet.log || true
+    cat logs/kubelet-status.txt || true
     exit 1
 fi
 
@@ -244,7 +286,6 @@ fi
 
 waitForDeleteStartTime=$(date +%s)
 
-az vmss delete -g $MC_RESOURCE_GROUP_NAME -n $VMSS_NAME
 kubectl delete node $vmInstanceName
 
 waitForDeleteEndTime=$(date +%s)

hack/tools/Makefile

@@ -4,7 +4,7 @@ LOCALBIN		= $(shell pwd)/bin
 all: install
 
 .PHONY: install
-install: $(LOCALBIN)/go-bindata $(LOCALBIN)/gox $(LOCALBIN)/ginkgo $(LOCALBIN)/golangci-lint $(LOCALBIN)/pub 
+install: $(LOCALBIN)/go-bindata $(LOCALBIN)/gox $(LOCALBIN)/ginkgo $(LOCALBIN)/golangci-lint $(LOCALBIN)/pub $(LOCALBIN)/cue
 	@echo > /dev/null
 
 $(LOCALBIN)/go-bindata:
@@ -14,13 +14,16 @@ $(LOCALBIN)/gox:
 	GOBIN=$(LOCALBIN) $(GO) get github.com/mitchellh/gox/...@v1.0.1
 
 $(LOCALBIN)/ginkgo:
-	GOBIN=$(LOCALBIN) $(GO) get github.com/onsi/ginkgo/ginkgo/...@v1.10.1
+	GOBIN=$(LOCALBIN) $(GO) get github.com/onsi/ginkgo/v2/ginkgo@v2.1.1
 
 $(LOCALBIN)/golangci-lint:
 	curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(LOCALBIN) v1.21.0
 
 $(LOCALBIN)/pub:
-	GOBIN=$(LOCALBIN) $(GO) get github.com/devigned/pub/...@v0.2.6
+	GOBIN=$(LOCALBIN) $(GO) get github.com/devigned/pub/...@v0.3.2
+
+$(LOCALBIN)/cue:
+	GOBIN=$(LOCALBIN) $(GO) get cuelang.org/go/cmd/cue@latest
 
 .PHONY: reload
 reload: clean install

hack/tools/go.mod

@@ -3,10 +3,10 @@ module github.com/Azure/aks-engine/hack/tools
 go 1.12
 
 require (
-	github.com/devigned/pub v0.2.6 // indirect
+	cuelang.org/go v0.4.2 // indirect
+	github.com/devigned/pub v0.3.2 // indirect
 	github.com/go-bindata/go-bindata v3.1.2+incompatible
 	github.com/mitchellh/gox v1.0.1
-	github.com/onsi/ginkgo v1.10.1
-	github.com/onsi/gomega v1.7.1 // indirect
-	golang.org/x/sys v0.0.0-20191104094858-e8c54fb511f6 // indirect
+	github.com/onsi/ginkgo v1.16.4
+	github.com/onsi/ginkgo/v2 v2.1.1 // indirect
 )