Add a conformance CI test with all alpha features enabled (antrea-io#…

…1428)
lzhecheng · Nov 3, 2020 · 3a0b9c5 · 3a0b9c5
1 parent 2efa050
commit 3a0b9c5
Show file tree

Hide file tree

Showing 4 changed files with 113 additions and 6 deletions.
diff --git a/ci/jenkins/jobs/macros.yaml b/ci/jenkins/jobs/macros.yaml
@@ -57,7 +57,7 @@
 
           rm -f COMMENT_EXIST
           rm -f body.json
-          echo "{{\"body\": \"Thanks for your PR.\\\nUnit tests and code linters are run automatically every time the PR is updated.\\\nE2e, conformance and network policy tests can only be triggered by a member of the vmware-tanzu organization. Regular contributors to the project should join the org.\\\n\\\nThe following commands are available:\\\n* \`/test-e2e\`: to trigger e2e tests.\\\n* \`/skip-e2e\`: to skip e2e tests.\\\n* \`/test-conformance\`: to trigger conformance tests.\\\n* \`/skip-conformance\`: to skip conformance tests.\\\n* \`/test-whole-conformance\`: to trigger all conformance tests on linux.\\\n* \`/skip-whole-conformance\`: to skip all conformance tests on linux.\\\n* \`/test-networkpolicy\`: to trigger networkpolicy tests.\\\n* \`/skip-networkpolicy\`: to skip networkpolicy tests.\\\n* \`/test-windows-conformance\`: to trigger windows conformance tests.\\\n* \`/skip-windows-conformance\`: to skip windows conformance tests.\\\n* \`/test-windows-networkpolicy\`: to trigger windows networkpolicy tests.\\\n* \`/skip-windows-networkpolicy\`: to skip windows networkpolicy tests.\\\n* \`/test-hw-offload\`: to trigger ovs hardware offload test.\\\n* \`/skip-hw-offload\`: to skip ovs hardware offload test.\\\n* \`/test-all\`: to trigger all tests (except whole conformance).\\\n* \`/skip-all\`: to skip all tests (except whole conformance).\"}}" > body.json
+          echo "{{\"body\": \"Thanks for your PR.\\\nUnit tests and code linters are run automatically every time the PR is updated.\\\nE2e, conformance and network policy tests can only be triggered by a member of the vmware-tanzu organization. Regular contributors to the project should join the org.\\\n\\\nThe following commands are available:\\\n* \`/test-e2e\`: to trigger e2e tests.\\\n* \`/skip-e2e\`: to skip e2e tests.\\\n* \`/test-conformance\`: to trigger conformance tests.\\\n* \`/skip-conformance\`: to skip conformance tests.\\\n*  \`/test-all-features-conformance\`: to trigger conformance tests with all alpha features enabled.\\\n* \`/skip-all-features-conformance\`: to skip conformance tests with all alpha features enabled.\\\n* \`/test-whole-conformance\`: to trigger all conformance tests on linux.\\\n* \`/skip-whole-conformance\`: to skip all conformance tests on linux.\\\n* \`/test-networkpolicy\`: to trigger networkpolicy tests.\\\n* \`/skip-networkpolicy\`: to skip networkpolicy tests.\\\n* \`/test-windows-conformance\`: to trigger windows conformance tests.\\\n* \`/skip-windows-conformance\`: to skip windows conformance tests.\\\n* \`/test-windows-networkpolicy\`: to trigger windows networkpolicy tests.\\\n* \`/skip-windows-networkpolicy\`: to skip windows networkpolicy tests.\\\n* \`/test-hw-offload\`: to trigger ovs hardware offload test.\\\n* \`/skip-hw-offload\`: to skip ovs hardware offload test.\\\n* \`/test-all\`: to trigger all tests (except whole conformance).\\\n* \`/skip-all\`: to skip all tests (except whole conformance).\"}}" > body.json
 
           B="$(cat body.json | jq .body)"
 

diff --git a/ci/jenkins/jobs/projects.yaml b/ci/jenkins/jobs/projects.yaml
@@ -410,6 +410,92 @@
           started_status: null
           wrappers: []
           publishers: []
+      - '{name}-{test_name}-for-pull-request':
+          test_name: all-features-conformance-pending-label
+          node: null
+          description: 'This is for marking PR as pending for conformance test with all alpha features enabled.'
+          branches:
+          - ${{sha1}}
+          builders:
+            - builder-pending-label
+          trigger_phrase: null
+          white_list_target_branches: []
+          allow_whitelist_orgs_as_admins: false
+          admin_list: []
+          org_list: []
+          white_list: []
+          only_trigger_phrase: false
+          trigger_permit_all: true
+          status_context: jenkins-all-features-conformance
+          status_url: --none--
+          success_status: Pending test. Mark as failure. Add comment /test-all-features-conformance to trigger.
+          failure_status: Pending test. Mark as failure. Add comment /test-all-features-conformance to trigger.
+          error_status: Pending test. Mark as failure. Add comment /test-all-features-conformance to trigger.
+          triggered_status: null
+          started_status: null
+          wrappers: []
+          publishers: []
+      - '{name}-{test_name}-for-pull-request':
+          test_name: all-features-conformance
+          node: 'antrea-test-node'
+          description: 'This is the {test_name} test for {name}.'
+          branches:
+          - ${{sha1}}
+          builders:
+            - builder-conformance:
+                conformance_type: 'all-features-conformance'
+          trigger_phrase: ^(?!Thanks for your PR).*/test-(all-features-conformance|all).*
+          white_list_target_branches: []
+          allow_whitelist_orgs_as_admins: true
+          admin_list: '{antrea_admin_list}'
+          org_list: '{antrea_org_list}'
+          white_list: '{antrea_white_list}'
+          only_trigger_phrase: true
+          trigger_permit_all: false
+          status_context: jenkins-all-features-conformance
+          status_url: null
+          success_status: Build finished.
+          failure_status: Failed. Add comment /test-all-features-conformance to re-trigger.
+          error_status: Failed. Add comment /test-all-features-conformance to re-trigger.
+          triggered_status: null
+          started_status: null
+          wrappers:
+          - timeout:
+              fail: true
+              timeout: 80
+              type: absolute
+          publishers:
+          - archive:
+              allow-empty: true
+              artifacts: 'ci/jenkins/*sonobuoy*.tar.gz'
+              case-sensitive: true
+              default-excludes: true
+              fingerprint: false
+              only-if-success: false
+      - '{name}-{test_name}-for-pull-request':
+          test_name: all-features-conformance-skip
+          node: null
+          description: 'This is for marking PR as passed.'
+          branches:
+          - ${{sha1}}
+          builders: []
+          allow_whitelist_orgs_as_admins: true
+          admin_list: '{antrea_admin_list}'
+          org_list: '{antrea_org_list}'
+          white_list: '{antrea_white_list}'
+          only_trigger_phrase: true
+          trigger_permit_all: false
+          trigger_phrase: ^(?!Thanks for your PR).*/skip-(all-features-conformance|all).*
+          white_list_target_branches: []
+          status_context: jenkins-all-features-conformance
+          status_url: --none--
+          success_status: Skipped test. Mark as succeeded.
+          failure_status: Skipped test. Mark as succeeded.
+          error_status: Skipped test. Mark as succeeded.
+          triggered_status: null
+          started_status: null
+          wrappers: []
+          publishers: []
       - '{name}-{test_name}-for-pull-request':
           test_name: whole-conformance
           node: 'antrea-test-node'

diff --git a/ci/jenkins/test-vmc.sh b/ci/jenkins/test-vmc.sh
@@ -37,16 +37,16 @@ TEST_FAILURE=false
 CLUSTER_READY=false
 
 _usage="Usage: $0 [--cluster-name <VMCClusterNameToUse>] [--kubeconfig <KubeconfigSavePath>] [--workdir <HomePath>]
-                  [--log-mode <SonobuoyResultLogLevel>] [--testcase <e2e|conformance|whole-conformance|networkpolicy>]
+                  [--log-mode <SonobuoyResultLogLevel>] [--testcase <e2e|conformance|all-features-conformance|whole-conformance|networkpolicy>]
                   [--garbage-collection] [--setup-only] [--cleanup-only] [--coverage] [--test-only]
 
-Setup a VMC cluster to run K8s e2e community tests (E2e, Conformance, whole Conformance & Network Policy).
+Setup a VMC cluster to run K8s e2e community tests (E2e, Conformance, all features Conformance, whole Conformance & Network Policy).
 
         --cluster-name           The cluster name to be used for the generated VMC cluster.
         --kubeconfig             Path to save kubeconfig of generated VMC cluster.
         --workdir                Home path for Go, vSphere information and antrea_logs during cluster setup. Default is $WORKDIR.
         --log-mode               Use the flag to set either 'report', 'detail', or 'dump' level data for sonobouy results.
-        --testcase               The testcase to run: e2e, conformance, whole-conformance or networkpolicy.
+        --testcase               The testcase to run: e2e, conformance, all-features-conformance, whole-conformance or networkpolicy.
         --garbage-collection     Do garbage collection to clean up some unused testbeds.
         --setup-only             Only perform setting up the cluster and run test.
         --cleanup-only           Only perform cleaning up the cluster.
@@ -382,7 +382,12 @@ function run_conformance {
     export PATH=$GOROOT/bin:$PATH
     export KUBECONFIG=$GIT_CHECKOUT_DIR/jenkins/out/kubeconfig
 
-    kubectl apply -f $GIT_CHECKOUT_DIR/build/yamls/antrea.yml
+    if [[ "$TESTCASE" == "all-features-conformance" ]]; then
+      $GIT_CHECKOUT_DIR/hack/generate-manifest.sh --mode dev --all-features > $GIT_CHECKOUT_DIR/build/yamls/antrea-all.yml
+      kubectl apply -f $GIT_CHECKOUT_DIR/build/yamls/antrea-all.yml
+    else
+      kubectl apply -f $GIT_CHECKOUT_DIR/build/yamls/antrea.yml
+    fi
     kubectl rollout restart deployment/coredns -n kube-system
     kubectl rollout status --timeout=5m deployment/coredns -n kube-system
     kubectl rollout status --timeout=5m deployment.apps/antrea-controller -n kube-system
@@ -395,6 +400,8 @@ function run_conformance {
 
     if [[ "$TESTCASE" == "conformance" ]]; then
         ${GIT_CHECKOUT_DIR}/ci/run-k8s-e2e-tests.sh --e2e-conformance --log-mode ${MODE} --kubeconfig ${GIT_CHECKOUT_DIR}/jenkins/out/kubeconfig > ${GIT_CHECKOUT_DIR}/vmc-test.log
+    elif [[ "$TESTCASE" == "all-features-conformance" ]]; then
+        ${GIT_CHECKOUT_DIR}/ci/run-k8s-e2e-tests.sh --e2e-conformance --log-mode ${MODE} --kubeconfig ${GIT_CHECKOUT_DIR}/jenkins/out/kubeconfig > ${GIT_CHECKOUT_DIR}/vmc-test.log
     elif [[ "$TESTCASE" == "whole-conformance" ]]; then
         ${GIT_CHECKOUT_DIR}/ci/run-k8s-e2e-tests.sh --e2e-whole-conformance --log-mode ${MODE} --kubeconfig ${GIT_CHECKOUT_DIR}/jenkins/out/kubeconfig > ${GIT_CHECKOUT_DIR}/vmc-test.log
     else
@@ -467,7 +474,7 @@ if [[ "$RUN_CLEANUP_ONLY" == true ]]; then
     exit 0
 fi
 
-if [[ "$TESTCASE" != "e2e" && "$TESTCASE" != "conformance" && "$TESTCASE" != "whole-conformance" && "$TESTCASE" != "networkpolicy" && "$TESTCASE" != "integration" ]]; then
+if [[ "$TESTCASE" != "e2e" && "$TESTCASE" != "conformance" && "$TESTCASE" != "all-features-conformance" && "$TESTCASE" != "whole-conformance" && "$TESTCASE" != "networkpolicy" && "$TESTCASE" != "integration" ]]; then
     echoerr "testcase should be e2e, integration, conformance, whole-conformance or networkpolicy"
     exit 1
 fi

diff --git a/hack/generate-manifest.sh b/hack/generate-manifest.sh
@@ -27,6 +27,7 @@ Generate a YAML manifest for Antrea using Kustomize and print it to stdout.
         --kind                        Generate a manifest appropriate for running Antrea in a Kind cluster
         --cloud                       Generate a manifest appropriate for running Antrea in Public Cloud
         --ipsec                       Generate a manifest with IPSec encryption of tunnel traffic enabled
+        --all-features                Generate a manifest with all alpha features enabled
         --proxy                       Generate a manifest with Antrea proxy enabled
         --np                          Generate a manifest with ClusterNetworkPolicy and Antrea NetworkPolicy features enabled
         --prometheus                  Generate a manifest with Antrea Controller and Agent Prometheus metrics listener enabled
@@ -58,6 +59,7 @@ function print_help {
 MODE="dev"
 KIND=false
 IPSEC=false
+ALLFEATURES=false
 PROXY=false
 NP=false
 KEEP=false
@@ -94,6 +96,10 @@ case $key in
     IPSEC=true
     shift
     ;;
+    --all-features)
+    ALLFEATURES=true
+    shift
+    ;;
     --proxy)
     PROXY=true
     shift
@@ -214,6 +220,14 @@ if $IPSEC; then
     sed -i.bak -E "s/^[[:space:]]*#[[:space:]]*tunnelType[[:space:]]*:[[:space:]]*[a-z]+[[:space:]]*$/tunnelType: gre/" antrea-agent.conf
 fi
 
+if $ALLFEATURES; then
+    sed -i.bak -E "s/^[[:space:]]*#[[:space:]]*AntreaProxy[[:space:]]*:[[:space:]]*[a-z]+[[:space:]]*$/  AntreaProxy: true/" antrea-agent.conf
+    sed -i.bak -E "s/^[[:space:]]*#[[:space:]]*Traceflow[[:space:]]*:[[:space:]]*[a-z]+[[:space:]]*$/  Traceflow: true/" antrea-agent.conf
+    sed -i.bak -E "s/^[[:space:]]*#[[:space:]]*AntreaPolicy[[:space:]]*:[[:space:]]*[a-z]+[[:space:]]*$/  AntreaPolicy: true/" antrea-agent.conf
+    sed -i.bak -E "s/^[[:space:]]*#[[:space:]]*FlowExporter[[:space:]]*:[[:space:]]*[a-z]+[[:space:]]*$/  FlowExporter: true/" antrea-agent.conf
+    sed -i.bak -E "s/^[[:space:]]*#[[:space:]]*NetworkPolicyStats[[:space:]]*:[[:space:]]*[a-z]+[[:space:]]*$/  NetworkPolicyStats: true/" antrea-agent.conf
+fi
+
 if $PROXY; then
     sed -i.bak -E "s/^[[:space:]]*#[[:space:]]*AntreaProxy[[:space:]]*:[[:space:]]*[a-z]+[[:space:]]*$/  AntreaProxy: true/" antrea-agent.conf
 fi