Improve kind e2e tests

There were 3 feature specific jobs running e2e tests with AntreaProxy disabled, AntreaProxy and proxyAll enabled, and AntreaPolicy disabled separately. Having a job for each feature is not extensible and requires maintenance efforts to configure the job along with the feature's lifecycle. There is a lot of redundancy between these jobs as unrelated tests ran repeatedly in them. There were other feature specific test cases enabling the features at runtime, which doesn't cover the scenario when two features are enabled at the same time. This patch removes the feature specific jobs and creates two jobs running tests with all features enabled and disabled separately, which can cover the previous jobs and all features' interaction. For features that are disabled by default, they will be tested in the "all features enabled" job and their impacts on other features will be covered. The patch also makes necessary changes to kind test script and manifest generating script to achieve it. Signed-off-by: Quan Tian <qtian@vmware.com>
antrea-io · Jun 22, 2022 · 67226c1 · 67226c1
1 parent c6fae02
commit 67226c1
Show file tree

Hide file tree

Showing 19 changed files with 168 additions and 264 deletions.
diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml
@@ -121,8 +121,8 @@ jobs:
         path: log.tar.gz
         retention-days: 30
 
-  test-e2e-encap-no-proxy:
-    name: E2e tests on a Kind cluster on Linux with AntreaProxy disabled
+  test-e2e-encap-all-features-enabled:
+    name: E2e tests on a Kind cluster on Linux with all features enabled
     needs: [build-antrea-coverage-image]
     runs-on: [ubuntu-latest]
     steps:
@@ -148,39 +148,40 @@ jobs:
     - name: Run e2e tests
       run: |
         mkdir log
-        mkdir test-e2e-encap-no-proxy-coverage
-        ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-encap-no-proxy-coverage ./ci/kind/test-e2e-kind.sh --encap-mode encap --no-proxy --coverage --skip mode-irrelevant
+        mkdir test-e2e-encap-all-features-enabled-coverage
+        # Currently multicast tests require specific testbeds, exclude it for now.
+        ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-encap-all-features-enabled-coverage ./ci/kind/test-e2e-kind.sh --encap-mode encap --coverage --feature-gates AllAlpha=true,AllBeta=true,Multicast=false --proxy-all
     - name: Tar coverage files
-      run: tar -czf test-e2e-encap-no-proxy-coverage.tar.gz test-e2e-encap-no-proxy-coverage
-    - name: Upload coverage for test-e2e-encap-no-proxy-coverage
+      run: tar -czf test-e2e-encap-all-features-enabled-coverage.tar.gz test-e2e-encap-all-features-enabled-coverage
+    - name: Upload coverage for test-e2e-encap-all-features-enabled-coverage
       uses: actions/upload-artifact@v3
       with:
-        name: test-e2e-encap-no-proxy-coverage
-        path: test-e2e-encap-no-proxy-coverage.tar.gz
+        name: test-e2e-encap-all-features-enabled-coverage
+        path: test-e2e-encap-all-features-enabled-coverage.tar.gz
         retention-days: 30
     - name: Codecov
       uses: codecov/codecov-action@v3
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         file: '*.cov.out*'
         flags: kind-e2e-tests
-        name: codecov-test-e2e-encap-no-proxy
-        directory: test-e2e-encap-no-proxy-coverage
+        name: codecov-test-e2e-encap-all-features-enabled
+        directory: test-e2e-encap-all-features-enabled-coverage
     - name: Tar log files
       if: ${{ failure() }}
       run: tar -czf log.tar.gz log
     - name: Upload test log
       uses: actions/upload-artifact@v3
       if: ${{ failure() }}
       with:
-        name: e2e-kind-encap-no-proxy.tar.gz
+        name: e2e-kind-encap-all-features-enabled.tar.gz
         path: log.tar.gz
         retention-days: 30
 
-  test-e2e-encap-proxy-all:
-    name: E2e tests on a Kind cluster on Linux with AntreaProxy all Service support
-    needs: [ build-antrea-coverage-image ]
-    runs-on: [ ubuntu-latest ]
+  test-e2e-encap-all-features-disabled:
+    name: E2e tests on a Kind cluster on Linux with all features disabled
+    needs: [build-antrea-coverage-image]
+    runs-on: [ubuntu-latest]
     steps:
       - name: Free disk space
         # https://github.com/actions/virtual-environments/issues/709
@@ -204,32 +205,32 @@ jobs:
       - name: Run e2e tests
         run: |
           mkdir log
-          mkdir test-e2e-encap-proxy-all-coverage
-          ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-encap-proxy-all-coverage ./ci/kind/test-e2e-kind.sh --encap-mode encap --proxy-all --coverage --skip mode-irrelevant
+          mkdir test-e2e-encap-all-features-disabled-coverage
+          ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-encap-all-features-disabled-coverage ./ci/kind/test-e2e-kind.sh --encap-mode encap --coverage --feature-gates AllAlpha=false,AllBeta=false
       - name: Tar coverage files
-        run: tar -czf test-e2e-encap-proxy-all-coverage.tar.gz test-e2e-encap-proxy-all-coverage
-      - name: Upload coverage for test-e2e-encap-proxy-all-coverage
+        run: tar -czf test-e2e-encap-all-features-disabled-coverage.tar.gz test-e2e-encap-all-features-disabled-coverage
+      - name: Upload coverage for test-e2e-encap-all-features-disabled-coverage
         uses: actions/upload-artifact@v3
         with:
-          name: test-e2e-encap-proxy-all-coverage
-          path: test-e2e-encap-proxy-all-coverage.tar.gz
+          name: test-e2e-encap-all-features-disabled-coverage
+          path: test-e2e-encap-all-features-disabled-coverage.tar.gz
           retention-days: 30
       - name: Codecov
         uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           file: '*.cov.out*'
           flags: kind-e2e-tests
-          name: codecov-test-e2e-encap-proxy-all
-          directory: test-e2e-encap-proxy-all-coverage
+          name: codecov-test-e2e-encap-all-features-disabled
+          directory: test-e2e-encap-all-features-disabled-coverage
       - name: Tar log files
         if: ${{ failure() }}
         run: tar -czf log.tar.gz log
       - name: Upload test log
         uses: actions/upload-artifact@v3
         if: ${{ failure() }}
         with:
-          name: e2e-kind-encap-proxy-all.tar.gz
+          name: e2e-kind-encap-all-features-disabled.tar.gz
           path: log.tar.gz
           retention-days: 30
 
@@ -345,65 +346,6 @@ jobs:
         path: log.tar.gz
         retention-days: 30
 
-  # TODO: remove when https://github.com/antrea-io/antrea/issues/897 is fixed.
-  # In the mean time, we keep this test around to ensure that at least one Kind
-  # test uses a Geneve overlay.
-  test-e2e-encap-no-np:
-    name: E2e tests on a Kind cluster on Linux with Antrea-native policies disabled
-    needs: [build-antrea-coverage-image]
-    runs-on: [ubuntu-latest]
-    steps:
-    - name: Free disk space
-      # https://github.com/actions/virtual-environments/issues/709
-      run: |
-        sudo apt-get clean
-        df -h
-    - uses: actions/checkout@v3
-    - uses: actions/setup-go@v3
-      with:
-        go-version: 1.17
-    - name: Download Antrea images from previous jobs
-      uses: actions/download-artifact@v3
-    - name: Load Antrea image
-      run: |
-        docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar
-    - name: Install Kind
-      run: |
-        curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
-        chmod +x ./kind
-        sudo mv kind /usr/local/bin
-    - name: Run e2e tests
-      run: |
-        mkdir log
-        mkdir test-e2e-encap-no-np-coverage
-        ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-encap-no-np-coverage ./ci/kind/test-e2e-kind.sh --encap-mode encap --no-np --coverage --skip mode-irrelevant
-    - name: Tar coverage files
-      run: tar -czf test-e2e-encap-no-np-coverage.tar.gz test-e2e-encap-no-np-coverage
-    - name: Upload coverage for test-e2e-encap-no-np-coverage
-      uses: actions/upload-artifact@v3
-      with:
-        name: test-e2e-encap-no-np-coverage
-        path: test-e2e-encap-no-np-coverage.tar.gz
-        retention-days: 30
-    - name: Codecov
-      uses: codecov/codecov-action@v3
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        file: '*.cov.out*'
-        flags: kind-e2e-tests
-        name: codecov-test-e2e-no-np-encap
-        directory: test-e2e-encap-no-np-coverage
-    - name: Tar log files
-      if: ${{ failure() }}
-      run: tar -czf log.tar.gz log
-    - name: Upload test log
-      uses: actions/upload-artifact@v3
-      if: ${{ failure() }}
-      with:
-        name: e2e-kind-encap-no-np.tar.gz
-        path: log.tar.gz
-        retention-days: 30
-
   test-e2e-flow-visibility:
     name: E2e tests on a Kind cluster on Linux for Flow Visibility
     needs: [build-antrea-coverage-image, build-flow-aggregator-coverage-image]
@@ -536,7 +478,7 @@ jobs:
   # yet.
   artifact-cleanup:
     name: Delete uploaded images
-    needs: [build-antrea-coverage-image, build-flow-aggregator-coverage-image, test-e2e-encap, test-e2e-encap-no-proxy, test-e2e-encap-proxy-all, test-e2e-noencap, test-e2e-hybrid, test-e2e-encap-no-np, test-netpol-tmp, validate-prometheus-metrics-doc, test-e2e-flow-visibility]
+    needs: [build-antrea-coverage-image, build-flow-aggregator-coverage-image, test-e2e-encap, test-e2e-encap-all-features-enabled, test-e2e-encap-all-features-disabled, test-e2e-noencap, test-e2e-hybrid, test-netpol-tmp, validate-prometheus-metrics-doc, test-e2e-flow-visibility]
     if: ${{ always() &&  (needs.build-antrea-coverage-image.result == 'success' || needs.build-flow-aggregator-coverage-image.result == 'success') }}
     runs-on: [ubuntu-latest]
     steps:

diff --git a/build/charts/antrea/conf/antrea-agent.conf b/build/charts/antrea/conf/antrea-agent.conf
@@ -1,5 +1,11 @@
 # FeatureGates is a map of feature names to bools that enable or disable experimental features.
 featureGates:
+# AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha.
+{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "AllAlpha" "default" false) }}
+
+# AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta.
+{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "AllBeta" "default" false) }}
+
 # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
 # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
 # Service traffic.

diff --git a/build/charts/antrea/conf/antrea-controller.conf b/build/charts/antrea/conf/antrea-controller.conf
@@ -1,5 +1,11 @@
 # FeatureGates is a map of feature names to bools that enable or disable experimental features.
 featureGates:
+# AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha.
+{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "AllAlpha" "default" false) }}
+
+# AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta.
+{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "AllBeta" "default" false) }}
+
 # Enable traceflow which provides packet tracing feature to diagnose network issue.
 {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "Traceflow" "default" true) }}
 

diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -58,6 +58,12 @@ data:
   antrea-agent.conf: |
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.
     featureGates:
+    # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha.
+    #  AllAlpha: false
+
+    # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta.
+    #  AllBeta: false
+
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
     # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
     # Service traffic.
@@ -366,6 +372,12 @@ data:
   antrea-controller.conf: |
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.
     featureGates:
+    # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha.
+    #  AllAlpha: false
+
+    # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta.
+    #  AllBeta: false
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -3664,7 +3676,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: a4e7182f54a10234001d8baeb7d950a5053ffb97b2f6951db516244ac3620cd8
+        checksum/config: 890f1364c9b89811375830c94fab2fa9f1957518351cc52c623e22b6964e5e75
       labels:
         app: antrea
         component: antrea-agent
@@ -3904,7 +3916,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: a4e7182f54a10234001d8baeb7d950a5053ffb97b2f6951db516244ac3620cd8
+        checksum/config: 890f1364c9b89811375830c94fab2fa9f1957518351cc52c623e22b6964e5e75
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -58,6 +58,12 @@ data:
   antrea-agent.conf: |
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.
     featureGates:
+    # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha.
+    #  AllAlpha: false
+
+    # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta.
+    #  AllBeta: false
+
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
     # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
     # Service traffic.
@@ -366,6 +372,12 @@ data:
   antrea-controller.conf: |
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.
     featureGates:
+    # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha.
+    #  AllAlpha: false
+
+    # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta.
+    #  AllBeta: false
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -3664,7 +3676,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: a4e7182f54a10234001d8baeb7d950a5053ffb97b2f6951db516244ac3620cd8
+        checksum/config: 890f1364c9b89811375830c94fab2fa9f1957518351cc52c623e22b6964e5e75
       labels:
         app: antrea
         component: antrea-agent
@@ -3906,7 +3918,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: a4e7182f54a10234001d8baeb7d950a5053ffb97b2f6951db516244ac3620cd8
+        checksum/config: 890f1364c9b89811375830c94fab2fa9f1957518351cc52c623e22b6964e5e75
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -58,6 +58,12 @@ data:
   antrea-agent.conf: |
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.
     featureGates:
+    # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha.
+    #  AllAlpha: false
+
+    # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta.
+    #  AllBeta: false
+
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
     # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
     # Service traffic.
@@ -366,6 +372,12 @@ data:
   antrea-controller.conf: |
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.
     featureGates:
+    # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha.
+    #  AllAlpha: false
+
+    # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta.
+    #  AllBeta: false
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -3664,7 +3676,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: afbe6d81852e79d710d0350ab5173f7d0e05b79c75744cffb94ce2294afc328c
+        checksum/config: cc8af4219d403a137ab87500ae0ab15b681fc635e41057b5623df6154443fddf
       labels:
         app: antrea
         component: antrea-agent
@@ -3904,7 +3916,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: afbe6d81852e79d710d0350ab5173f7d0e05b79c75744cffb94ce2294afc328c
+        checksum/config: cc8af4219d403a137ab87500ae0ab15b681fc635e41057b5623df6154443fddf
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -71,6 +71,12 @@ data:
   antrea-agent.conf: |
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.
     featureGates:
+    # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha.
+    #  AllAlpha: false
+
+    # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta.
+    #  AllBeta: false
+
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
     # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
     # Service traffic.
@@ -379,6 +385,12 @@ data:
   antrea-controller.conf: |
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.
     featureGates:
+    # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha.
+    #  AllAlpha: false
+
+    # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta.
+    #  AllBeta: false
+
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
@@ -3677,7 +3689,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 626cd437985469e9b0e55e2faacbab203aee641d8a99ff9831c2bb3319f02e95
+        checksum/config: df5271a5c42a550d3f8e73fbe8e5fad8d178884fb74d81c7322128187546db86
         checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4
       labels:
         app: antrea
@@ -3963,7 +3975,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 626cd437985469e9b0e55e2faacbab203aee641d8a99ff9831c2bb3319f02e95
+        checksum/config: df5271a5c42a550d3f8e73fbe8e5fad8d178884fb74d81c7322128187546db86
       labels:
         app: antrea
         component: antrea-controller