Refactor Jenkins CI

* Move scripts in macros.yaml to separate shell files in order to
ensure maintainability.
* run-k8s-e2e-test.sh: Move KUBE_CONFORMANCE_IMAGE_VERSION initialization before _usage.
Otherwise, this var is unset.
* k8s-conformance-image-version: v1.18.0-beta.1 -> v1.18.5
* sonobuy version: v0.17.2 -> v0.18.5

ci/jenkins/README.md

@@ -80,6 +80,8 @@ installing the correct version of
 subset of community tests for Antrea:
 * To run conformance tests: `./run-k8s-e2e-tests.sh --e2e-conformance
   [--kubeconfig <Kubeconfig>]`.
+* To run whole conformance tests: `./run-k8s-e2e-tests.sh --e2e-whole-conformance
+  [--kubeconfig <Kubeconfig>]`.
 * To run network policy tests: `./run-k8s-e2e-tests.sh --e2e-network-policy
   [--kubeconfig <Kubeconfig>]`.
 * To run a single test by name: `./run-k8s-e2e-tests.sh --e2e-focus <TestRegex>
@@ -118,4 +120,4 @@ a PR to modify code under that path is merged, Jenkins jobs on cloud should be
 updated with new code.
 
 ### Tips for Developer
-* [macro.yaml](/ci/jenkins/jobs/macros.yaml): Use "{{}}" instead of "{}" in "builder-list-tests" and "builder-conformance".
+* [macro.yaml](/ci/jenkins/jobs/macros.yaml): Use "{{}}" instead of "{}" in "builder-list-tests" and "builder-conformance".

ci/jenkins/jobs/job-templates.yaml

@@ -147,6 +147,15 @@
         fail: true
         timeout: 40
         type: absolute
+    scm:
+    - git:
+        branches: '{branches}'
+        credentials-id: '{git_credentials_id}'
+        name: origin
+        refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/${{ghprbPullId}}/*:refs/remotes/origin/pr/${{ghprbPullId}}/*
+        url: 'https://github.com/{org_repo}'
+        wipe-workspace: true
+        included-regions: '{included_regions}'
 
 - job-template:
      name: 'cloud-{name}-{test_name}'

ci/jenkins/jobs/macros.yaml

@@ -12,145 +12,7 @@
         - shell: |-
             #!/bin/bash
             set -e
-            WORK_HOME="/var/lib/jenkins"
-            VM_NAME="antrea-integration-0"
-            export GOVC_URL=${GOVC_URL}
-            export GOVC_USERNAME=${GOVC_USERNAME}
-            export GOVC_PASSWORD=${GOVC_PASSWORD}
-            VM_IP=$(govc vm.ip ${VM_NAME})
-            govc snapshot.revert -vm.ip ${VM_IP} initial
-            VM_IP=$(govc vm.ip ${VM_NAME}) # wait for VM to be on
-
-            set -x
-            echo "===== Run Integration test ====="
-            ssh -o StrictHostKeyChecking=no -i "${WORK_HOME}/utils/key" -n jenkins@${VM_IP} "git clone ${ghprbAuthorRepoGitUrl} antrea && cd antrea && git checkout ${GIT_BRANCH} && make docker-test-integration"
-- builder:
-    name: builder-workload-cluster-setup
-    builders:
-      - shell: |-
-          #!/bin/bash
-
-          function saveLogs() {
-            echo "=== Truncate old logs ==="
-            export LOG_DIR=/var/lib/jenkins/antrea_logs
-            find ${LOG_DIR}/* -type d -mmin +10080 | xargs -r rm -rf
-
-            CLUSTER_LOG_DIR="${LOG_DIR}/${cluster}"
-            echo "=== Saving capi logs ==="
-            mkdir -p ${CLUSTER_LOG_DIR}/capi
-            kubectl get -n capi-system pods -o name | awk '{print $1}' | while read capi_pod; do
-              capi_pod_name=$(echo ${capi_pod} | cut -d'/' -f 2)
-              kubectl logs ${capi_pod_name} -c manager -n capi-system > ${CLUSTER_LOG_DIR}/capi/${capi_pod_name}
-            done
-
-            echo "=== Saving capv logs ==="
-            mkdir -p ${CLUSTER_LOG_DIR}/capv
-            kubectl get -n capv-system pods -o name | awk '{print $1}' | while read capv_pod; do
-              capv_pod_name=$(echo ${capv_pod} | cut -d'/' -f 2)
-              kubectl logs ${capv_pod_name} -c manager -n capv-system > ${CLUSTER_LOG_DIR}/capv/${capv_pod_name}
-            done
-
-            echo "=== Saving cluster_api.yaml ==="
-            mkdir -p ${CLUSTER_LOG_DIR}/cluster_api
-            kubectl get cluster-api -A -o yaml > ${CLUSTER_LOG_DIR}/cluster_api/cluster_api.yaml || true
-          }
-
-          set -e
-          cluster="${JOB_NAME}-${BUILD_NUMBER}"
-
-          WORK_HOME="/var/lib/jenkins"
-          export KUBECONFIG="${WORK_HOME}/.kube/config"
-          rm -rf jenkins || true
-
-          echo '=== Generate key pair ==='
-          mkdir -p ${WORKSPACE}/jenkins/key
-          ssh-keygen -b 2048 -t rsa -f  "${WORKSPACE}/jenkins/key/antrea-ci-key" -q -N ""
-          publickey="$(cat ${WORKSPACE}/jenkins/key/antrea-ci-key.pub)"
-
-          echo "=== namespace value substitution ==="
-          mkdir -p ${WORKSPACE}/jenkins/out
-          cp ci/cluster-api/vsphere/templates/* ${WORKSPACE}/jenkins/out
-          sed -i "s/CLUSTERNAMESPACE/${cluster}/g" "${WORKSPACE}/jenkins/out/cluster.yaml"
-          sed -i "s/CLUSTERNAME/${cluster}/g" "${WORKSPACE}/jenkins/out/cluster.yaml"
-          sed -i "s|SSHAUTHORIZEDKEYS|${publickey}|g" "${WORKSPACE}/jenkins/out/cluster.yaml"
-          sed -i "s/CLUSTERNAMESPACE/${cluster}/g" "${WORKSPACE}/jenkins/out/namespace.yaml"
-
-          echo "=== network spec value substitution==="
-          cluster_defaults="${WORK_HOME}/utils/CLUSTERDEFAULTS"
-          while IFS= read -r line
-          do
-            IFS='=' read -ra kv <<< "$line"
-            sed -i "s|${kv[0]}|${kv[1]}|g" "${WORKSPACE}/jenkins/out/cluster.yaml"
-          done < "$cluster_defaults"
-
-          echo '=== Create a cluster in management cluster ==='
-          kubectl apply -f "${WORKSPACE}/jenkins/out/namespace.yaml"
-          kubectl apply -f "${WORKSPACE}/jenkins/out/cluster.yaml"
-
-          echo '=== Wait for workload cluster secret for 10 min ==='
-          for t in {1..10}
-          do
-            sleep 1m
-            echo '=== Get kubeconfig (try for 1m) ==='
-            if kubectl get secret/${cluster}-kubeconfig -n${cluster} ; then
-              kubectl get secret/${cluster}-kubeconfig -n${cluster} -o json \
-                | jq -r .data.value \
-                | base64 --decode \
-                > "${WORKSPACE}/jenkins/out/kubeconfig"
-              touch jenkins/SECRET_EXIST
-              break
-            fi
-          done
-
-          if !(test -f jenkins/SECRET_EXIST); then
-            echo "=== Failed to get secret ==="
-            saveLogs
-            kubectl delete ns ${cluster}
-            exit 1
-          else
-            export KUBECONFIG="${WORKSPACE}/jenkins/out/kubeconfig"
-            echo "=== Waiting all nodes up for 10 min ==="
-
-            set +e
-            for t in {1..10}
-            do
-              sleep 1m
-              echo "=== Get node (try for 1m) ==="
-              mdNum="$(kubectl get node | grep -c ${cluster}-md)"
-              if [ "${mdNum}" == "2" ]; then
-                echo "=== Setup workload cluster succeeded ==="
-                exit 0
-              fi
-            done
-            set -e
-
-            echo "=== Failed to make all nodes up ==="
-            saveLogs
-            KUBECONFIG="${WORK_HOME}/.kube/config" kubectl delete ns ${cluster}
-            exit 1
-          fi
-
-- builder:
-    name: builder-workload-cluster-cleanup
-    builders:
-      - shell: |-
-          #!/bin/bash
-          set -ex
-          echo '=== Clean up workload cluster ==='
-          export KUBECONFIG="/var/lib/jenkins/.kube/config"
-
-          cluster="${JOB_NAME}-${BUILD_NUMBER}"
-          kubectl delete ns ${cluster}
-          rm -rf "${WORKSPACE}/jenkins"
-
-          echo "=== Cleanup workload cluster ${cluster} succeeded ==="
-
-          if !(test -f TEST_FAILURE); then
-            echo "=== SUCCESS !!! ==="
-            exit 0
-          fi
-          echo "=== FAILURE !!! ==="
-          exit 1
+            ./ci/jenkins/test-vmc.sh --testcase integration
 
 - builder:
     name: builder-eks-cluster-cleanup
@@ -180,21 +42,12 @@
          ./ci/test-conformance-aks.sh --cluster-name "${CLUSTERNAME}" --cleanup-only
 
 - builder:
-    name: builder-workload-cluster-gc
+    name: builder-workload-cluster-garbage-collection
     builders:
       - shell: |-
           #!/bin/bash
           set -ex
-          echo "=== Auto cleanup starts ==="
-          export KUBECONFIG="/var/lib/jenkins/.kube/config"
-
-          kubectl get namespace -l antrea-ci | awk '$3 ~ "[0-9][hd]" || $3 ~ "[6-9][0-9]m" || $3 ~ "1[0-9][0-9]m" && $2 ~ "Active" {print $1}' | while read cluster; do
-            echo "=== Currently ${cluster} has been live for more than 1h ==="
-            kubectl delete ns ${cluster}
-            echo "=== Old namespace ${cluster} is deleted !!! ==="
-          done
-
-          echo "=== Auto cleanup finished ==="
+          ./ci/jenkins/test-vmc.sh --garbage-collection
 
 - builder:
     name: builder-list-tests
@@ -242,182 +95,35 @@
     builders:
       - shell: 'exit 1 # fail on purpose'
 
-- builder:
-    name: builder-prepare-antrea
-    builders:
-      - shell: |-
-          echo ====== Building Antrea for the Following Commit ======
-
-          git show --numstat
-
-          export GO111MODULE=on
-          export GOPATH=/var/lib/jenkins/go
-          export GOROOT=/usr/local/go
-          export GOCACHE="${WORKSPACE}/../gocache"
-          export PATH=$GOROOT/bin:$PATH
-
-          make clean
-          docker pull antrea/openvswitch --all-tags
-          docker images | grep "${JOB_NAME}" | awk '{print $3}' | xargs -r docker rmi -f || true
-          # Clean up dangling images generated in previous builds. Recent ones must be excluded
-          # because they might be being used in other builds running simultaneously.
-          docker image prune -f --filter "until=1h" || true
-          for i in `seq 2`
-          do
-            VERSION="$JOB_NAME-$BUILD_NUMBER" make && break
-          done
-
-          if [ "$?" -ne "0" ]; then
-            echo "=== Antrea Image build failed ==="
-            exit 1
-          fi
-
-          sed -i "s|#serviceCIDR: 10.96.0.0/12|serviceCIDR: 100.64.0.0/13|g" build/yamls/antrea.yml
-          sed -i 's|#enablePrometheusMetrics: false|enablePrometheusMetrics: true|g' build/yamls/antrea.yml
-      - shell: |-
-          echo ====== Delivering Antrea to all the Nodes ======
-          export KUBECONFIG="${WORKSPACE}/jenkins/out/kubeconfig"
-          DOCKER_IMG_VERSION="$JOB_NAME-$BUILD_NUMBER"
-
-          docker save -o antrea-ubuntu.tar antrea/antrea-ubuntu:${DOCKER_IMG_VERSION}
-
-          kubectl get nodes -o wide --no-headers=true | awk '$3 == "master" {print $6}' | while read master_ip; do
-            scp -o StrictHostKeyChecking=no -i jenkins/key/antrea-ci-key build/yamls/*.yml capv@${master_ip}:~
-          done
-
-          kubectl get nodes -o wide --no-headers=true | awk '{print $6}' | while read IP; do
-            rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no -i jenkins/key/antrea-ci-key" antrea-ubuntu.tar capv@${IP}:/home/capv/antrea-ubuntu.tar
-            ssh -o StrictHostKeyChecking=no -i jenkins/key/antrea-ci-key -n capv@${IP} "sudo crictl images | grep 'antrea-ubuntu' | awk '{print \$3}' | xargs -r crictl rmi ; sudo ctr -n=k8s.io images import /home/capv/antrea-ubuntu.tar ; sudo ctr -n=k8s.io images tag docker.io/antrea/antrea-ubuntu:${DOCKER_IMG_VERSION} docker.io/antrea/antrea-ubuntu:latest ; sudo crictl images | grep '<none>' | awk '{print \$3}' | xargs -r crictl rmi" || true
-          done
-
 - builder:
     name: builder-e2e
     builders:
       - shell: |-
           #!/bin/bash
           set -ex
-          echo ====== Running Antrea E2E Tests ======
-
-          export GO111MODULE=on
-          export WORK_HOME=/var/lib/jenkins
-          export GOPATH=$WORK_HOME/go
-          export GOROOT=/usr/local/go
-          export GOCACHE=$WORK_HOME/.cache/go-build
-          export PATH=$GOROOT/bin:$PATH
-          export KUBECONFIG=${WORKSPACE}/jenkins/out/kubeconfig
-          cluster="${JOB_NAME}-${BUILD_NUMBER}"
-
-          mkdir -p test/e2e/infra/vagrant/playbook/kube
-          cp -f "${WORKSPACE}/jenkins/out/kubeconfig" test/e2e/infra/vagrant/playbook/kube/config
-
-          echo "=== Generate ssh-config ==="
-          cp -f ci/jenkins/ssh-config test/e2e/infra/vagrant/ssh-config
-          master_name="$(kubectl get nodes -o wide --no-headers=true | awk '$3 == "master" {print $1}')"
-          master_ip="""$(kubectl get nodes -o wide --no-headers=true | awk '$3 == "master" {print $6}')"
-          echo "=== Master node ip: ${master_ip} ==="
-          sed -i "s/MASTERNODEIP/${master_ip}/g" test/e2e/infra/vagrant/ssh-config
-          echo "=== Move kubeconfig to master ==="
-          ssh -o StrictHostKeyChecking=no -i jenkins/key/antrea-ci-key -n capv@${master_ip} "mkdir .kube"
-          scp -o StrictHostKeyChecking=no -i jenkins/key/antrea-ci-key jenkins/out/kubeconfig capv@${master_ip}:~/.kube/config
-          sed -i "s/CONTROLPLANENODE/${master_name}/g" test/e2e/infra/vagrant/ssh-config
-          echo "    IdentityFile ${WORKSPACE}/jenkins/key/antrea-ci-key" >> test/e2e/infra/vagrant/ssh-config
-
-          # Run and configure Prometheus
-          kubectl apply -f build/yamls/antrea-prometheus.yml
-
-          set +e
-          mkdir -p `pwd`/antrea-test-logs
-          go test -v -timeout=50m github.com/vmware-tanzu/antrea/test/e2e --logs-export-dir `pwd`/antrea-test-logs --prometheus
-
-          test_rc=$?
-          set -e
-
-          tar -zcf antrea-test-logs.tar.gz antrea-test-logs
-
-          echo ====== Cleanup Antrea Installation ======
-
-          for antrea_yml in build/yamls/*.yml
-          do
-            kubectl delete -f ${antrea_yml} --ignore-not-found=true || true
-          done
-
-          kubectl delete ns antrea-test || true
-
-          if [ "$test_rc" == "1" ]
-          then
-            echo "=== TEST FAILURE !!! ==="
-            touch TEST_FAILURE
-          fi
-          echo "=== TEST SUCCESS !!! ==="
+          ./ci/jenkins/test-vmc.sh --cluster-name "${JOB_NAME}-${BUILD_NUMBER}" --testcase e2e
 
 - builder:
     name: builder-conformance
     builders:
       - shell: |-
           #!/bin/bash
           set -ex
-          echo ====== Running Antrea Conformance Tests ======
-
-          export GO111MODULE=on
-          export WORK_HOME=/var/lib/jenkins
-          export GOPATH=$WORK_HOME/go
-          export GOROOT=/usr/local/go
-          export GOCACHE=$WORK_HOME/.cache/go-build
-          export PATH=$GOROOT/bin:$PATH
-          export KUBECONFIG=$WORKSPACE/jenkins/out/kubeconfig
-
-          kubectl apply -f build/yamls/antrea.yml
-          kubectl rollout restart deployment/coredns -n kube-system
-          kubectl rollout status --timeout=5m deployment/coredns -n kube-system
-          kubectl rollout status --timeout=5m deployment.apps/antrea-controller -n kube-system
-          kubectl rollout status --timeout=5m daemonset/antrea-agent -n kube-system
-
-          kubectl get nodes -o wide --no-headers=true | awk '$3 == "master" {{print $6}}' | while read master_ip; do
-            echo "=== Move kubeconfig to master ==="
-            ssh -o StrictHostKeyChecking=no -i jenkins/key/antrea-ci-key -n capv@${{master_ip}} "mkdir .kube"
-            scp -o StrictHostKeyChecking=no -i jenkins/key/antrea-ci-key jenkins/out/kubeconfig capv@${{master_ip}}:~/.kube/config
-
-            conformance_image_version=$(head -n1 ci/k8s-conformance-image-version)
-            echo  "=== Run sonobuoy with conformance image version ${{conformance_image_version}} ==="
-
-            sonobuoy delete --wait --kubeconfig jenkins/out/kubeconfig
-            if [ '{mode_regex}' == "" ]; then
-              sonobuoy run --wait --timeout 3600 --e2e-focus '{focus_regex}' --e2e-skip '{skip_regex}' --e2e-parallel y --kube-conformance-image-version ${{conformance_image_version}} --kubeconfig jenkins/out/kubeconfig
-            else
-              sonobuoy run --wait --timeout 10000 --mode='{mode_regex}' --kube-conformance-image-version ${{conformance_image_version}} --kubeconfig jenkins/out/kubeconfig
-            fi
-            sonobuoy retrieve --kubeconfig jenkins/out/kubeconfig
-            echo '=== Print all results ==='
-            sonobuoy results *sonobuoy*.tar.gz
-            echo '=== Print failed cases if any ==='
-            sonobuoy results *sonobuoy*.tar.gz >> RESULT
-
-            if grep -Fxq "Failed tests:" RESULT
-            then
-              echo "Failed cases exist."
-              touch TEST_FAILURE
-            else
-              echo "All tests passed."
-            fi
-
-            echo "=== Clean up sonobouy resources ==="
-            sonobuoy delete --wait --kubeconfig jenkins/out/kubeconfig
-            rm RESULT
-          done
-
-          echo ====== Cleanup Antrea Installation ======
-
-          for antrea_yml in /var/lib/jenkins/*.yml
-          do
-            kubectl delete -f ${{antrea_yml}} --ignore-not-found=true || true
-          done
-
-          kubectl delete ns antrea-test || true
+          ./ci/jenkins/test-vmc.sh --cluster-name "${{JOB_NAME}}-${{BUILD_NUMBER}}" --testcase '{conformance_type}'
 
 - builder:
     name: builder-elk-flow-collector
     builders:
       - shell: |-
           #!/bin/bash
           set -ex
+          ./ci/jenkins/test-vmc.sh --cluster-name "${JOB_NAME}-${BUILD_NUMBER}" --setup-only
           ./ci/test-elk-flow-collector.sh --kubeconfig jenkins/out/kubeconfig
+          ./ci/jenkins/test-vmc.sh --cluster-name "${JOB_NAME}-${BUILD_NUMBER}" --cleanup-only
+
+          if !(test -f TEST_FAILURE); then
+            echo "=== SUCCESS !!! ==="
+            exit 0
+          fi
+          echo "=== FAILURE !!! ==="
+          exit 1