From 67226c1a583029d205cea7f9266911687e3548be Mon Sep 17 00:00:00 2001 From: Quan Tian Date: Tue, 21 Jun 2022 22:40:13 +0800 Subject: [PATCH] Improve kind e2e tests There were 3 feature specific jobs running e2e tests with AntreaProxy disabled, AntreaProxy and proxyAll enabled, and AntreaPolicy disabled separately. Having a job for each feature is not extensible and requires maintenance efforts to configure the job along with the feature's lifecycle. There is a lot of redundancy between these jobs as unrelated tests ran repeatedly in them. There were other feature specific test cases enabling the features at runtime, which doesn't cover the scenario when two features are enabled at the same time. This patch removes the feature specific jobs and creates two jobs running tests with all features enabled and disabled separately, which can cover the previous jobs and all features' interaction. For features that are disabled by default, they will be tested in the "all features enabled" job and their impacts on other features will be covered. The patch also makes necessary changes to kind test script and manifest generating script to achieve it. Signed-off-by: Quan Tian --- .github/workflows/kind.yml | 110 +++++------------- build/charts/antrea/conf/antrea-agent.conf | 6 + .../charts/antrea/conf/antrea-controller.conf | 6 + build/yamls/antrea-aks.yml | 16 ++- build/yamls/antrea-eks.yml | 16 ++- build/yamls/antrea-gke.yml | 16 ++- build/yamls/antrea-ipsec.yml | 16 ++- build/yamls/antrea.yml | 16 ++- ci/jenkins/test-vmc.sh | 4 +- ci/kind/test-e2e-kind.sh | 54 ++++----- hack/generate-manifest.sh | 82 ++----------- test/e2e/antreapolicy_test.go | 3 +- test/e2e/framework.go | 32 +++-- test/e2e/infra/vagrant/push_antrea.sh | 2 +- test/e2e/ipsec_test.go | 10 +- test/e2e/nodeportlocal_test.go | 3 +- test/e2e/secondary_network_ipam_test.go | 14 +-- test/e2e/service_externalip_test.go | 15 +-- test/e2e/trafficcontrol_test.go | 11 +- 19 files changed, 168 insertions(+), 264 deletions(-) diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml index 3ba9f45a4bb..cb950857bdd 100644 --- a/.github/workflows/kind.yml +++ b/.github/workflows/kind.yml @@ -121,8 +121,8 @@ jobs: path: log.tar.gz retention-days: 30 - test-e2e-encap-no-proxy: - name: E2e tests on a Kind cluster on Linux with AntreaProxy disabled + test-e2e-encap-all-features-enabled: + name: E2e tests on a Kind cluster on Linux with all features enabled needs: [build-antrea-coverage-image] runs-on: [ubuntu-latest] steps: @@ -148,15 +148,16 @@ jobs: - name: Run e2e tests run: | mkdir log - mkdir test-e2e-encap-no-proxy-coverage - ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-encap-no-proxy-coverage ./ci/kind/test-e2e-kind.sh --encap-mode encap --no-proxy --coverage --skip mode-irrelevant + mkdir test-e2e-encap-all-features-enabled-coverage + # Currently multicast tests require specific testbeds, exclude it for now. + ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-encap-all-features-enabled-coverage ./ci/kind/test-e2e-kind.sh --encap-mode encap --coverage --feature-gates AllAlpha=true,AllBeta=true,Multicast=false --proxy-all - name: Tar coverage files - run: tar -czf test-e2e-encap-no-proxy-coverage.tar.gz test-e2e-encap-no-proxy-coverage - - name: Upload coverage for test-e2e-encap-no-proxy-coverage + run: tar -czf test-e2e-encap-all-features-enabled-coverage.tar.gz test-e2e-encap-all-features-enabled-coverage + - name: Upload coverage for test-e2e-encap-all-features-enabled-coverage uses: actions/upload-artifact@v3 with: - name: test-e2e-encap-no-proxy-coverage - path: test-e2e-encap-no-proxy-coverage.tar.gz + name: test-e2e-encap-all-features-enabled-coverage + path: test-e2e-encap-all-features-enabled-coverage.tar.gz retention-days: 30 - name: Codecov uses: codecov/codecov-action@v3 @@ -164,8 +165,8 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} file: '*.cov.out*' flags: kind-e2e-tests - name: codecov-test-e2e-encap-no-proxy - directory: test-e2e-encap-no-proxy-coverage + name: codecov-test-e2e-encap-all-features-enabled + directory: test-e2e-encap-all-features-enabled-coverage - name: Tar log files if: ${{ failure() }} run: tar -czf log.tar.gz log @@ -173,14 +174,14 @@ jobs: uses: actions/upload-artifact@v3 if: ${{ failure() }} with: - name: e2e-kind-encap-no-proxy.tar.gz + name: e2e-kind-encap-all-features-enabled.tar.gz path: log.tar.gz retention-days: 30 - test-e2e-encap-proxy-all: - name: E2e tests on a Kind cluster on Linux with AntreaProxy all Service support - needs: [ build-antrea-coverage-image ] - runs-on: [ ubuntu-latest ] + test-e2e-encap-all-features-disabled: + name: E2e tests on a Kind cluster on Linux with all features disabled + needs: [build-antrea-coverage-image] + runs-on: [ubuntu-latest] steps: - name: Free disk space # https://github.com/actions/virtual-environments/issues/709 @@ -204,15 +205,15 @@ jobs: - name: Run e2e tests run: | mkdir log - mkdir test-e2e-encap-proxy-all-coverage - ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-encap-proxy-all-coverage ./ci/kind/test-e2e-kind.sh --encap-mode encap --proxy-all --coverage --skip mode-irrelevant + mkdir test-e2e-encap-all-features-disabled-coverage + ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-encap-all-features-disabled-coverage ./ci/kind/test-e2e-kind.sh --encap-mode encap --coverage --feature-gates AllAlpha=false,AllBeta=false - name: Tar coverage files - run: tar -czf test-e2e-encap-proxy-all-coverage.tar.gz test-e2e-encap-proxy-all-coverage - - name: Upload coverage for test-e2e-encap-proxy-all-coverage + run: tar -czf test-e2e-encap-all-features-disabled-coverage.tar.gz test-e2e-encap-all-features-disabled-coverage + - name: Upload coverage for test-e2e-encap-all-features-disabled-coverage uses: actions/upload-artifact@v3 with: - name: test-e2e-encap-proxy-all-coverage - path: test-e2e-encap-proxy-all-coverage.tar.gz + name: test-e2e-encap-all-features-disabled-coverage + path: test-e2e-encap-all-features-disabled-coverage.tar.gz retention-days: 30 - name: Codecov uses: codecov/codecov-action@v3 @@ -220,8 +221,8 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} file: '*.cov.out*' flags: kind-e2e-tests - name: codecov-test-e2e-encap-proxy-all - directory: test-e2e-encap-proxy-all-coverage + name: codecov-test-e2e-encap-all-features-disabled + directory: test-e2e-encap-all-features-disabled-coverage - name: Tar log files if: ${{ failure() }} run: tar -czf log.tar.gz log @@ -229,7 +230,7 @@ jobs: uses: actions/upload-artifact@v3 if: ${{ failure() }} with: - name: e2e-kind-encap-proxy-all.tar.gz + name: e2e-kind-encap-all-features-disabled.tar.gz path: log.tar.gz retention-days: 30 @@ -345,65 +346,6 @@ jobs: path: log.tar.gz retention-days: 30 - # TODO: remove when https://github.com/antrea-io/antrea/issues/897 is fixed. - # In the mean time, we keep this test around to ensure that at least one Kind - # test uses a Geneve overlay. - test-e2e-encap-no-np: - name: E2e tests on a Kind cluster on Linux with Antrea-native policies disabled - needs: [build-antrea-coverage-image] - runs-on: [ubuntu-latest] - steps: - - name: Free disk space - # https://github.com/actions/virtual-environments/issues/709 - run: | - sudo apt-get clean - df -h - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version: 1.17 - - name: Download Antrea images from previous jobs - uses: actions/download-artifact@v3 - - name: Load Antrea image - run: | - docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar - - name: Install Kind - run: | - curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64 - chmod +x ./kind - sudo mv kind /usr/local/bin - - name: Run e2e tests - run: | - mkdir log - mkdir test-e2e-encap-no-np-coverage - ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-encap-no-np-coverage ./ci/kind/test-e2e-kind.sh --encap-mode encap --no-np --coverage --skip mode-irrelevant - - name: Tar coverage files - run: tar -czf test-e2e-encap-no-np-coverage.tar.gz test-e2e-encap-no-np-coverage - - name: Upload coverage for test-e2e-encap-no-np-coverage - uses: actions/upload-artifact@v3 - with: - name: test-e2e-encap-no-np-coverage - path: test-e2e-encap-no-np-coverage.tar.gz - retention-days: 30 - - name: Codecov - uses: codecov/codecov-action@v3 - with: - token: ${{ secrets.CODECOV_TOKEN }} - file: '*.cov.out*' - flags: kind-e2e-tests - name: codecov-test-e2e-no-np-encap - directory: test-e2e-encap-no-np-coverage - - name: Tar log files - if: ${{ failure() }} - run: tar -czf log.tar.gz log - - name: Upload test log - uses: actions/upload-artifact@v3 - if: ${{ failure() }} - with: - name: e2e-kind-encap-no-np.tar.gz - path: log.tar.gz - retention-days: 30 - test-e2e-flow-visibility: name: E2e tests on a Kind cluster on Linux for Flow Visibility needs: [build-antrea-coverage-image, build-flow-aggregator-coverage-image] @@ -536,7 +478,7 @@ jobs: # yet. artifact-cleanup: name: Delete uploaded images - needs: [build-antrea-coverage-image, build-flow-aggregator-coverage-image, test-e2e-encap, test-e2e-encap-no-proxy, test-e2e-encap-proxy-all, test-e2e-noencap, test-e2e-hybrid, test-e2e-encap-no-np, test-netpol-tmp, validate-prometheus-metrics-doc, test-e2e-flow-visibility] + needs: [build-antrea-coverage-image, build-flow-aggregator-coverage-image, test-e2e-encap, test-e2e-encap-all-features-enabled, test-e2e-encap-all-features-disabled, test-e2e-noencap, test-e2e-hybrid, test-netpol-tmp, validate-prometheus-metrics-doc, test-e2e-flow-visibility] if: ${{ always() && (needs.build-antrea-coverage-image.result == 'success' || needs.build-flow-aggregator-coverage-image.result == 'success') }} runs-on: [ubuntu-latest] steps: diff --git a/build/charts/antrea/conf/antrea-agent.conf b/build/charts/antrea/conf/antrea-agent.conf index e41025688f9..6b3d5f198cd 100644 --- a/build/charts/antrea/conf/antrea-agent.conf +++ b/build/charts/antrea/conf/antrea-agent.conf @@ -1,5 +1,11 @@ # FeatureGates is a map of feature names to bools that enable or disable experimental features. featureGates: +# AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha. +{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "AllAlpha" "default" false) }} + +# AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta. +{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "AllBeta" "default" false) }} + # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent. # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on # Service traffic. diff --git a/build/charts/antrea/conf/antrea-controller.conf b/build/charts/antrea/conf/antrea-controller.conf index f3d89e13638..440952a8757 100644 --- a/build/charts/antrea/conf/antrea-controller.conf +++ b/build/charts/antrea/conf/antrea-controller.conf @@ -1,5 +1,11 @@ # FeatureGates is a map of feature names to bools that enable or disable experimental features. featureGates: +# AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha. +{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "AllAlpha" "default" false) }} + +# AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta. +{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "AllBeta" "default" false) }} + # Enable traceflow which provides packet tracing feature to diagnose network issue. {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "Traceflow" "default" true) }} diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index 0dac8be1865..01490b120d6 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -58,6 +58,12 @@ data: antrea-agent.conf: | # FeatureGates is a map of feature names to bools that enable or disable experimental features. featureGates: + # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha. + # AllAlpha: false + + # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta. + # AllBeta: false + # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent. # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on # Service traffic. @@ -366,6 +372,12 @@ data: antrea-controller.conf: | # FeatureGates is a map of feature names to bools that enable or disable experimental features. featureGates: + # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha. + # AllAlpha: false + + # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta. + # AllBeta: false + # Enable traceflow which provides packet tracing feature to diagnose network issue. # Traceflow: true @@ -3664,7 +3676,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: a4e7182f54a10234001d8baeb7d950a5053ffb97b2f6951db516244ac3620cd8 + checksum/config: 890f1364c9b89811375830c94fab2fa9f1957518351cc52c623e22b6964e5e75 labels: app: antrea component: antrea-agent @@ -3904,7 +3916,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: a4e7182f54a10234001d8baeb7d950a5053ffb97b2f6951db516244ac3620cd8 + checksum/config: 890f1364c9b89811375830c94fab2fa9f1957518351cc52c623e22b6964e5e75 labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 9adf78a6d50..796c2da168c 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -58,6 +58,12 @@ data: antrea-agent.conf: | # FeatureGates is a map of feature names to bools that enable or disable experimental features. featureGates: + # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha. + # AllAlpha: false + + # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta. + # AllBeta: false + # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent. # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on # Service traffic. @@ -366,6 +372,12 @@ data: antrea-controller.conf: | # FeatureGates is a map of feature names to bools that enable or disable experimental features. featureGates: + # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha. + # AllAlpha: false + + # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta. + # AllBeta: false + # Enable traceflow which provides packet tracing feature to diagnose network issue. # Traceflow: true @@ -3664,7 +3676,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: a4e7182f54a10234001d8baeb7d950a5053ffb97b2f6951db516244ac3620cd8 + checksum/config: 890f1364c9b89811375830c94fab2fa9f1957518351cc52c623e22b6964e5e75 labels: app: antrea component: antrea-agent @@ -3906,7 +3918,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: a4e7182f54a10234001d8baeb7d950a5053ffb97b2f6951db516244ac3620cd8 + checksum/config: 890f1364c9b89811375830c94fab2fa9f1957518351cc52c623e22b6964e5e75 labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index a4afa9678a3..9701aadd165 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -58,6 +58,12 @@ data: antrea-agent.conf: | # FeatureGates is a map of feature names to bools that enable or disable experimental features. featureGates: + # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha. + # AllAlpha: false + + # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta. + # AllBeta: false + # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent. # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on # Service traffic. @@ -366,6 +372,12 @@ data: antrea-controller.conf: | # FeatureGates is a map of feature names to bools that enable or disable experimental features. featureGates: + # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha. + # AllAlpha: false + + # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta. + # AllBeta: false + # Enable traceflow which provides packet tracing feature to diagnose network issue. # Traceflow: true @@ -3664,7 +3676,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: afbe6d81852e79d710d0350ab5173f7d0e05b79c75744cffb94ce2294afc328c + checksum/config: cc8af4219d403a137ab87500ae0ab15b681fc635e41057b5623df6154443fddf labels: app: antrea component: antrea-agent @@ -3904,7 +3916,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: afbe6d81852e79d710d0350ab5173f7d0e05b79c75744cffb94ce2294afc328c + checksum/config: cc8af4219d403a137ab87500ae0ab15b681fc635e41057b5623df6154443fddf labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index f3bda059167..46003c4a30d 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -71,6 +71,12 @@ data: antrea-agent.conf: | # FeatureGates is a map of feature names to bools that enable or disable experimental features. featureGates: + # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha. + # AllAlpha: false + + # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta. + # AllBeta: false + # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent. # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on # Service traffic. @@ -379,6 +385,12 @@ data: antrea-controller.conf: | # FeatureGates is a map of feature names to bools that enable or disable experimental features. featureGates: + # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha. + # AllAlpha: false + + # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta. + # AllBeta: false + # Enable traceflow which provides packet tracing feature to diagnose network issue. # Traceflow: true @@ -3677,7 +3689,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 626cd437985469e9b0e55e2faacbab203aee641d8a99ff9831c2bb3319f02e95 + checksum/config: df5271a5c42a550d3f8e73fbe8e5fad8d178884fb74d81c7322128187546db86 checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4 labels: app: antrea @@ -3963,7 +3975,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 626cd437985469e9b0e55e2faacbab203aee641d8a99ff9831c2bb3319f02e95 + checksum/config: df5271a5c42a550d3f8e73fbe8e5fad8d178884fb74d81c7322128187546db86 labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index 78717183955..be903bbf9d3 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -58,6 +58,12 @@ data: antrea-agent.conf: | # FeatureGates is a map of feature names to bools that enable or disable experimental features. featureGates: + # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha. + # AllAlpha: false + + # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta. + # AllBeta: false + # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent. # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on # Service traffic. @@ -366,6 +372,12 @@ data: antrea-controller.conf: | # FeatureGates is a map of feature names to bools that enable or disable experimental features. featureGates: + # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha. + # AllAlpha: false + + # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta. + # AllBeta: false + # Enable traceflow which provides packet tracing feature to diagnose network issue. # Traceflow: true @@ -3664,7 +3676,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 750097fa4ed60fda68fedf2f31b06a1a62dbbe5dc9a38e9c1f9ddc861a389401 + checksum/config: 033b7f8c7b77a918ad1a90c3db034bbfc1df67de264a77f8aee6a035836b6812 labels: app: antrea component: antrea-agent @@ -3904,7 +3916,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 750097fa4ed60fda68fedf2f31b06a1a62dbbe5dc9a38e9c1f9ddc861a389401 + checksum/config: 033b7f8c7b77a918ad1a90c3db034bbfc1df67de264a77f8aee6a035836b6812 labels: app: antrea component: antrea-controller diff --git a/ci/jenkins/test-vmc.sh b/ci/jenkins/test-vmc.sh index 768f14f5dec..310da1b06dc 100755 --- a/ci/jenkins/test-vmc.sh +++ b/ci/jenkins/test-vmc.sh @@ -574,10 +574,10 @@ function run_conformance { if [[ "$TESTCASE" == "all-features-conformance" ]]; then if [[ "$COVERAGE" == true ]]; then - $GIT_CHECKOUT_DIR/hack/generate-manifest.sh --mode dev --all-features --coverage > $GIT_CHECKOUT_DIR/build/yamls/antrea-all-coverage.yml + $GIT_CHECKOUT_DIR/hack/generate-manifest.sh --mode dev --feature-gates AllAlpha=true,AllBeta=true --proxy-all --coverage > $GIT_CHECKOUT_DIR/build/yamls/antrea-all-coverage.yml antrea_yml="antrea-all-coverage.yml" else - $GIT_CHECKOUT_DIR/hack/generate-manifest.sh --mode dev --all-features --verbose-log > $GIT_CHECKOUT_DIR/build/yamls/antrea-all.yml + $GIT_CHECKOUT_DIR/hack/generate-manifest.sh --mode dev --feature-gates AllAlpha=true,AllBeta=true --proxy-all --verbose-log > $GIT_CHECKOUT_DIR/build/yamls/antrea-all.yml antrea_yml="antrea-all.yml" fi fi diff --git a/ci/kind/test-e2e-kind.sh b/ci/kind/test-e2e-kind.sh index f4ad6e2178a..a849969a852 100755 --- a/ci/kind/test-e2e-kind.sh +++ b/ci/kind/test-e2e-kind.sh @@ -22,13 +22,12 @@ function echoerr { >&2 echo "$@" } -_usage="Usage: $0 [--encap-mode ] [--ip-family ] [--no-proxy] [--np] [--coverage] [--help|-h] +_usage="Usage: $0 [--encap-mode ] [--ip-family ] [--coverage] [--help|-h] --encap-mode Traffic encapsulation mode. (default is 'encap'). --ip-family Configures the ipFamily for the KinD cluster. - --no-proxy Disables Antrea proxy. + --feature-gates A comma-separated list of key=value pairs that describe feature gates, e.g. AntreaProxy=true,Egress=false. + --run Run only tests matching the regexp. --proxy-all Enables Antrea proxy with all Service support. - --endpointslice Enables Antrea proxy and EndpointSlice support. - --no-np Disables Antrea-native policies. --flow-visibility Only run flow visibility related e2e tests. --skip A comma-separated list of keywords, with which tests should be skipped. --coverage Enables measure Antrea code coverage when run e2e tests on kind. @@ -61,24 +60,27 @@ function quit { mode="" ipfamily="v4" -proxy=true +feature_gates="" proxy_all=false -endpointslice=false -np=true flow_visibility=false coverage=false skiplist="" setup_only=false cleanup_only=false test_only=false +run="" while [[ $# -gt 0 ]] do key="$1" case $key in - --no-proxy) - proxy=false - shift + --run) + run="$2" + shift 2 + ;; + --feature-gates) + feature_gates="$2" + shift 2 ;; --proxy-all) proxy_all=true @@ -88,14 +90,6 @@ case $key in ipfamily="$2" shift 2 ;; - --endpointslice) - endpointslice=true - shift - ;; - --no-np) - np=false - shift - ;; --flow-visibility) flow_visibility=true shift @@ -142,25 +136,15 @@ fi trap "quit" INT EXIT -manifest_args="" -if ! $proxy; then - manifest_args="$manifest_args --no-proxy" +manifest_args="$manifest_args --verbose-log" +if [ -n "$feature_gates" ]; then + manifest_args="$manifest_args --feature-gates $feature_gates" fi if $proxy_all; then - if ! $proxy; then - echoerr "--proxy-all requires AntreaProxy, so it cannot be used with --no-proxy" - exit 1 - fi manifest_args="$manifest_args --proxy-all" fi -if $endpointslice; then - manifest_args="$manifest_args --endpointslice" -fi -if ! $np; then - manifest_args="$manifest_args --no-np" -fi if $flow_visibility; then - manifest_args="$manifest_args --flow-exporter --extra-helm-values-file $FLOW_VISIBILITY_HELM_VALUES" + manifest_args="$manifest_args --feature-gates FlowExporter=true --extra-helm-values-file $FLOW_VISIBILITY_HELM_VALUES" fi COMMON_IMAGES_LIST=("k8s.gcr.io/e2e-test-images/agnhost:2.29" \ @@ -254,7 +238,11 @@ function run_test { fi sleep 1 - go test -v -timeout=$timeout antrea.io/antrea/test/e2e $flow_visibility_args -provider=kind --logs-export-dir=$ANTREA_LOG_DIR --skip=$skiplist $coverage_args + RUN_OPT="" + if [ -n "$run" ]; then + RUN_OPT="-run $run" + fi + go test -v -timeout=$timeout $RUN_OPT antrea.io/antrea/test/e2e $flow_visibility_args -provider=kind --logs-export-dir=$ANTREA_LOG_DIR --skip=$skiplist $coverage_args } if [[ "$mode" == "" ]] || [[ "$mode" == "encap" ]]; then diff --git a/hack/generate-manifest.sh b/hack/generate-manifest.sh index e42924f3056..542091aa169 100755 --- a/hack/generate-manifest.sh +++ b/hack/generate-manifest.sh @@ -20,18 +20,14 @@ function echoerr { >&2 echo "$@" } -_usage="Usage: $0 [--mode (dev|release)] [--encap-mode] [--ipsec] [--no-proxy] [--no-np] [--keep] [--tun (geneve|vxlan|gre|stt)] [--verbose-log] [--help|-h] +_usage="Usage: $0 [--mode (dev|release)] [--encap-mode] [--ipsec] [--tun (geneve|vxlan|gre|stt)] [--verbose-log] [--help|-h] Generate a YAML manifest for Antrea using Helm and print it to stdout. --mode (dev|release) Choose the configuration variant that you need (default is 'dev') --encap-mode Traffic encapsulation mode. (default is 'encap') --cloud Generate a manifest appropriate for running Antrea in Public Cloud --ipsec Generate a manifest with IPsec encryption of tunnel traffic enabled - --all-features Generate a manifest with all alpha features enabled - --no-proxy Generate a manifest with Antrea proxy disabled + --feature-gates A comma-separated list of key=value pairs that describe feature gates, e.g. AntreaProxy=true,Egress=false. --proxy-all Generate a manifest with Antrea proxy with all Service support enabled - --endpointslice Generate a manifest with EndpointSlice support enabled - --flow-exporter Generate a manifest with FlowExporter support enabled - --no-np Generate a manifest with Antrea-native policies disabled --tun (geneve|vxlan|gre|stt) Choose encap tunnel type from geneve, gre, stt and vxlan (default is geneve) --verbose-log Generate a manifest with increased log-level (level 4) for Antrea agent and controller. This option will work only in 'dev' mode. @@ -67,15 +63,10 @@ function print_help { echoerr "Try '$0 --help' for more information." } +FEATURE_GATES="" MODE="dev" IPSEC=false -ALLFEATURES=false -PROXY=true PROXY_ALL=false -ENDPOINTSLICE=false -FLOW_EXPORTER=false -NP=true -KEEP=false ENCAP_MODE="" CLOUD="" TUN_TYPE="geneve" @@ -118,42 +109,21 @@ case $key in IPSEC=true shift ;; - --all-features) - ALLFEATURES=true - shift - ;; - --no-proxy) - PROXY=false - shift + --feature-gates) + FEATURE_GATES="$2" + shift 2 ;; --proxy-all) PROXY=true PROXY_ALL=true shift ;; - --endpointslice) - PROXY=true - ENDPOINTSLICE=true - shift - ;; - --flow-exporter) - FLOW_EXPORTER=true - shift - ;; - --no-np) - NP=false - shift - ;; --k8s-1.15) echoerr "The --k8s-1.15 flag is no longer supported" exit 1 K8S_115=true shift ;; - --keep) - KEEP=true - shift - ;; --tun) TUN_TYPE="$2" shift 2 @@ -225,18 +195,6 @@ case $key in esac done -if [ "$PROXY" == false ] && [ "$ENDPOINTSLICE" == true ]; then - echoerr "--endpointslice requires AntreaProxy, so it cannot be used with --no-proxy" - print_help - exit 1 -fi - -if [ "$PROXY" == false ] && [ "$PROXY_ALL" == true ]; then - echoerr "--proxy-all requires AntreaProxy, so it cannot be used with --no-proxy" - print_help - exit 1 -fi - if [ "$MODE" != "dev" ] && [ "$MODE" != "release" ]; then echoerr "--mode must be one of 'dev' or 'release'" print_help @@ -278,11 +236,6 @@ if $COVERAGE && $VERBOSE_LOG; then VERBOSE_LOG=false fi -if [[ "$ENCAP_MODE" != "" ]] && [[ "$ENCAP_MODE" != "encap" ]] && ! $PROXY; then - echoerr "Cannot use '--no-proxy' when '--encap-mode' is not 'encap'" - exit 1 -fi - if [[ "$ENCAP_MODE" != "" ]] && [[ "$ENCAP_MODE" != "encap" ]] && $IPSEC; then echoerr "Encap mode '$ENCAP_MODE' does not make sense with IPsec" exit 1 @@ -314,30 +267,15 @@ if $MULTICAST; then HELM_VALUES+=("trafficEncapMode=noEncap" "featureGates.Multicast=true" "multicast.multicastInterfaces={$MULTICAST_INTERFACES}") fi -if $ALLFEATURES; then - HELM_VALUES+=("featureGates.FlowExporter=true" "featureGates.EndpointSlice=true" "antreaProxy.proxyAll=true") -fi - -if ! $PROXY; then - HELM_VALUES+=("featureGates.AntreaProxy=false") -fi +IFS=',' read -r -a feature_gates <<< "$FEATURE_GATES" +for feature_gate in "${feature_gates[@]}"; do + HELM_VALUES+=("featureGates.${feature_gate}") +done if $PROXY_ALL; then HELM_VALUES+=("antreaProxy.proxyAll=true") fi -if $ENDPOINTSLICE; then - HELM_VALUES+=("featureGates.EndpointSlice=true") -fi - -if $FLOW_EXPORTER; then - HELM_VALUES+=("featureGates.FlowExporter=true") -fi - -if ! $NP; then - HELM_VALUES+=("featureGates.AntreaPolicy=false") -fi - if [[ $ENCAP_MODE != "" ]]; then HELM_VALUES+=("trafficEncapMode=$ENCAP_MODE") fi diff --git a/test/e2e/antreapolicy_test.go b/test/e2e/antreapolicy_test.go index 09af4823b69..98f45b124c5 100644 --- a/test/e2e/antreapolicy_test.go +++ b/test/e2e/antreapolicy_test.go @@ -75,6 +75,7 @@ const ( func TestAntreaPolicyStats(t *testing.T) { skipIfHasWindowsNodes(t) skipIfAntreaPolicyDisabled(t) + skipIfNetworkPolicyStatsDisabled(t) data, err := setupTest(t) if err != nil { @@ -83,11 +84,9 @@ func TestAntreaPolicyStats(t *testing.T) { defer teardownTest(t, data) t.Run("testANPNetworkPolicyStatsWithDropAction", func(t *testing.T) { - skipIfNetworkPolicyStatsDisabled(t) testANPNetworkPolicyStatsWithDropAction(t, data) }) t.Run("testAntreaClusterNetworkPolicyStats", func(t *testing.T) { - skipIfNetworkPolicyStatsDisabled(t) testAntreaClusterNetworkPolicyStats(t, data) }) } diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 0c626a767cc..f0a934b3629 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -1981,32 +1981,30 @@ func (data *TestData) isProxyAll() (bool, error) { return agentConf.AntreaProxy.ProxyAll, nil } -func getFeatures(confName string) (featuregate.FeatureGate, error) { +func GetAgentFeatures() (featuregate.FeatureGate, error) { featureGate := features.DefaultMutableFeatureGate.DeepCopy() - var cfg interface{} - if err := yaml.Unmarshal([]byte(AntreaConfigMap.Data[confName]), &cfg); err != nil { + var cfg agentconfig.AgentConfig + if err := yaml.Unmarshal([]byte(AntreaConfigMap.Data[antreaAgentConfName]), &cfg); err != nil { return nil, err } - rawFeatureGateMap, ok := cfg.(map[interface{}]interface{})["featureGates"] - if !ok || rawFeatureGateMap == nil { - return featureGate, nil - } - featureGateMap := make(map[string]bool) - for k, v := range rawFeatureGateMap.(map[interface{}]interface{}) { - featureGateMap[k.(string)] = v.(bool) - } - if err := featureGate.SetFromMap(featureGateMap); err != nil { + err := featureGate.SetFromMap(cfg.FeatureGates) + if err != nil { return nil, err } return featureGate, nil } -func GetAgentFeatures() (featuregate.FeatureGate, error) { - return getFeatures(antreaAgentConfName) -} - func GetControllerFeatures() (featuregate.FeatureGate, error) { - return getFeatures(antreaControllerConfName) + featureGate := features.DefaultMutableFeatureGate.DeepCopy() + var cfg controllerconfig.ControllerConfig + if err := yaml.Unmarshal([]byte(AntreaConfigMap.Data[antreaControllerConfName]), &cfg); err != nil { + return nil, err + } + err := featureGate.SetFromMap(cfg.FeatureGates) + if err != nil { + return nil, err + } + return featureGate, nil } func (data *TestData) GetAntreaWindowsConfigMap(antreaNamespace string) (*corev1.ConfigMap, error) { diff --git a/test/e2e/infra/vagrant/push_antrea.sh b/test/e2e/infra/vagrant/push_antrea.sh index 80fe5b5b2b7..275df127007 100755 --- a/test/e2e/infra/vagrant/push_antrea.sh +++ b/test/e2e/infra/vagrant/push_antrea.sh @@ -170,7 +170,7 @@ FLOW_VIS_YML="/tmp/flow-visibility.yml" # manifest to enable FlowExporter. if [[ $FLOW_COLLECTOR != "" ]]; then echo "Generating manifest with FlowExporter enabled" - $THIS_DIR/../../../../hack/generate-manifest.sh --mode dev --flow-exporter > "${ANTREA_YML}" + $THIS_DIR/../../../../hack/generate-manifest.sh --mode dev --feature-gates FlowExporter=true > "${ANTREA_YML}" fi # Push Antrea image and related manifest. diff --git a/test/e2e/ipsec_test.go b/test/e2e/ipsec_test.go index 054331fe409..3e58bfc6b5b 100644 --- a/test/e2e/ipsec_test.go +++ b/test/e2e/ipsec_test.go @@ -25,6 +25,7 @@ import ( "antrea.io/antrea/pkg/agent/util" agentconfig "antrea.io/antrea/pkg/config/agent" controllerconfig "antrea.io/antrea/pkg/config/controller" + "antrea.io/antrea/pkg/features" ) // TestIPSec is the top-level test which contains all subtests for @@ -60,15 +61,8 @@ func TestIPSec(t *testing.T) { }) t.Run("testIPSecCertificateAuth", func(t *testing.T) { - // restart the Controller first as Agent needs to get the CSR signed. - cc := func(config *controllerconfig.ControllerConfig) { - config.FeatureGates["IPsecCertAuth"] = true - } - if err := data.mutateAntreaConfigMap(cc, nil, true, false); err != nil { - t.Fatalf("Failed to enable IPsecCertAuth feature: %v", err) - } + skipIfFeatureDisabled(t, features.IPsecCertAuth, true, true) ac := func(config *agentconfig.AgentConfig) { - config.FeatureGates["IPsecCertAuth"] = true config.IPsec.AuthenticationMode = "cert" } if err := data.mutateAntreaConfigMap(nil, ac, false, true); err != nil { diff --git a/test/e2e/nodeportlocal_test.go b/test/e2e/nodeportlocal_test.go index 725036cd6f8..344ee661a66 100644 --- a/test/e2e/nodeportlocal_test.go +++ b/test/e2e/nodeportlocal_test.go @@ -78,13 +78,14 @@ func configureNPLForAgent(t *testing.T, data *TestData, startPort, endPort int) func TestNodePortLocal(t *testing.T) { skipIfNotIPv4Cluster(t) skipIfHasWindowsNodes(t) + skipIfNodePortLocalDisabled(t) data, err := setupTest(t) if err != nil { t.Fatalf("Error when setting up test: %v", err) } defer teardownTest(t, data) - skipIfNodePortLocalDisabled(t) + configureNPLForAgent(t, data, defaultStartPort, defaultEndPort) t.Run("testNPLAddPod", func(t *testing.T) { testNPLAddPod(t, data) }) t.Run("testNPLMultiplePodsAgentRestart", func(t *testing.T) { testNPLMultiplePodsAgentRestart(t, data) }) diff --git a/test/e2e/secondary_network_ipam_test.go b/test/e2e/secondary_network_ipam_test.go index 6bb634a3688..86bc47f755b 100644 --- a/test/e2e/secondary_network_ipam_test.go +++ b/test/e2e/secondary_network_ipam_test.go @@ -22,8 +22,7 @@ import ( "antrea.io/antrea/pkg/agent/config" crdv1alpha2 "antrea.io/antrea/pkg/apis/crd/v1alpha2" - agentconfig "antrea.io/antrea/pkg/config/agent" - controllerconfig "antrea.io/antrea/pkg/config/controller" + "antrea.io/antrea/pkg/features" ) var ( @@ -242,6 +241,7 @@ func TestSecondaryNetworkIPAM(t *testing.T) { skipIfProxyDisabled(t) skipIfNotIPv4Cluster(t) skipIfAntreaIPAMTest(t) + skipIfFeatureDisabled(t, features.AntreaIPAM, true, true) data, err := setupTest(t) if err != nil { @@ -250,16 +250,6 @@ func TestSecondaryNetworkIPAM(t *testing.T) { defer teardownTest(t, data) skipIfEncapModeIsNot(t, data, config.TrafficEncapModeEncap) - cc := func(config *controllerconfig.ControllerConfig) { - config.FeatureGates["AntreaIPAM"] = true - } - ac := func(config *agentconfig.AgentConfig) { - config.FeatureGates["AntreaIPAM"] = true - } - if err = data.mutateAntreaConfigMap(cc, ac, true, true); err != nil { - t.Fatalf("Failed to enable AntreaIPAM feature: %v", err) - } - _, err = data.crdClient.CrdV1alpha2().IPPools().Create(context.TODO(), testIPPoolv4, metav1.CreateOptions{}) defer deleteIPPoolWrapper(t, data, testIPPoolv4.Name) if err != nil { diff --git a/test/e2e/service_externalip_test.go b/test/e2e/service_externalip_test.go index 9d1bc4f0391..7ae7e7b44dd 100644 --- a/test/e2e/service_externalip_test.go +++ b/test/e2e/service_externalip_test.go @@ -37,8 +37,7 @@ import ( antreaagenttypes "antrea.io/antrea/pkg/agent/types" "antrea.io/antrea/pkg/apis/crd/v1alpha2" - agentconfig "antrea.io/antrea/pkg/config/agent" - controllerconfig "antrea.io/antrea/pkg/config/controller" + "antrea.io/antrea/pkg/features" "antrea.io/antrea/pkg/querier" ) @@ -46,6 +45,7 @@ func TestServiceExternalIP(t *testing.T) { skipIfHasWindowsNodes(t) skipIfNumNodesLessThan(t, 2) skipIfAntreaIPAMTest(t) + skipIfFeatureDisabled(t, features.ServiceExternalIP, true, true) data, err := setupTest(t) if err != nil { @@ -53,17 +53,6 @@ func TestServiceExternalIP(t *testing.T) { } defer teardownTest(t, data) - cc := func(config *controllerconfig.ControllerConfig) { - config.FeatureGates["ServiceExternalIP"] = true - } - ac := func(config *agentconfig.AgentConfig) { - config.FeatureGates["ServiceExternalIP"] = true - } - - if err := data.mutateAntreaConfigMap(cc, ac, true, true); err != nil { - t.Fatalf("Failed to enable ServiceExternalIP feature: %v", err) - } - t.Run("testServiceWithExternalIPCRUD", func(t *testing.T) { testServiceWithExternalIPCRUD(t, data) }) t.Run("testServiceUpdateExternalIP", func(t *testing.T) { testServiceUpdateExternalIP(t, data) }) t.Run("testServiceExternalTrafficPolicyLocal", func(t *testing.T) { testServiceExternalTrafficPolicyLocal(t, data) }) diff --git a/test/e2e/trafficcontrol_test.go b/test/e2e/trafficcontrol_test.go index fffd0b60124..135a38322be 100644 --- a/test/e2e/trafficcontrol_test.go +++ b/test/e2e/trafficcontrol_test.go @@ -28,7 +28,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "antrea.io/antrea/pkg/apis/crd/v1alpha2" - agentconfig "antrea.io/antrea/pkg/config/agent" + "antrea.io/antrea/pkg/features" ) type trafficControlTestConfig struct { @@ -54,6 +54,7 @@ var ( func TestTrafficControl(t *testing.T) { skipIfHasWindowsNodes(t) + skipIfFeatureDisabled(t, features.TrafficControl, true, false) data, err := setupTest(t) if err != nil { @@ -61,14 +62,6 @@ func TestTrafficControl(t *testing.T) { } defer teardownTest(t, data) - ac := func(config *agentconfig.AgentConfig) { - config.FeatureGates["TrafficControl"] = true - } - - if err = data.mutateAntreaConfigMap(nil, ac, true, true); err != nil { - t.Fatalf("Failed to enable TrafficControl feature: %v", err) - } - tcTestConfig.nodeName = controlPlaneNodeName() createTrafficControlTestPod(t, data, tcTestConfig.podName)