test: add cronjob runs e2e in multiple k8s versions (#1203)

.github/workflows/k8s-timer-ci.yml

@@ -0,0 +1,177 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: k8s-timer-ci
+
+on:
+  schedule:
+    - cron: '0 1 * * 1'
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    outputs:
+      docs: ${{ steps.filter.outputs.docs }}
+      go: ${{ steps.filter.outputs.go }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
+
+      - uses: ./.github/actions/paths-filter
+        id: filter
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          filters: |
+            docs:
+              - 'docs/**/*'
+              - '*.md'
+            go:
+              - '*.go'
+              - '**/*.go'
+              - 'go.mod'
+              - 'go.sum'
+              - 'Makefile'
+              - 'Dockerfile'
+              - 'test/e2e/**/*'
+              - 'conf/**'
+              - 'utils/**'
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    needs: changes
+
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v3 # v3.0.2
+
+      - name: Set up Go 1.18
+        id: go
+        uses: actions/setup-go@v3 # v3.2.0
+        with:
+          go-version: '1.18'
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2 #v2.0.0
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2 # v2.0.0
+        with:
+          version: latest
+
+      - name: Prepare Host
+        run: |
+          curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64
+          chmod +x ./kind
+          sudo mv kind /usr/local/bin
+
+      - name: Build images
+        env:
+          TAG: dev
+          ARCH: amd64
+          REGISTRY: localhost:5000
+          ENABLE_PROXY: "false"
+        run: |
+          echo "building images..."
+          make clean-image build-images
+
+          echo "creating images cache..."
+          docker save \
+            localhost:5000/apache/apisix:dev \
+            localhost:5000/bitnami/etcd:dev \
+            localhost:5000/apache/apisix-ingress-controller:dev \
+            localhost:5000/kennethreitz/httpbin:dev \
+            localhost:5000/test-backend:dev \
+            localhost:5000/jmalloc/echo-server:dev \
+            localhost:5000/busybox:dev \
+            | pigz > docker.tar.gz
+
+      - name: cache
+        uses: actions/upload-artifact@v3 # v3.1.0
+        with:
+          name: docker.tar.gz
+          path: docker.tar.gz
+
+  prepare:
+    needs: changes
+    if: needs.changes.outputs.go == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+
+      - name: Check e2e test cases' naming
+        run: make e2e-names-check
+
+  Kubernetes-e2e-test:
+    needs: 
+      - changes
+      - prepare
+      - build
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false # If false, GitHub will not cancels all in-progress jobs in the matrix if any matrix job fails.
+      matrix:
+        k8s:
+          - v1.19.11@sha256:07db187ae84b4b7de440a73886f008cf903fcf5764ba8106a9fd5243d6f32729
+          - v1.20.7@sha256:cbeaf907fc78ac97ce7b625e4bf0de16e3ea725daf6b04f930bd14c67c671ff9
+          - v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
+
+      - name: Install ginkgo
+        run: |
+          go install github.com/onsi/ginkgo/v2/ginkgo@v2.1.4
+          sudo cp ~/go/bin/ginkgo /usr/local/bin
+
+      - name: cache
+        uses: actions/download-artifact@v2 # v2
+        with:
+          name: docker.tar.gz
+
+      - name: Create Kubernetes ${{ matrix.k8s }} cluster
+        shell: bash
+        env:
+          K8S_VERSION: ${{ matrix.k8s }}
+        run: |
+          make kind-up
+          kubectl wait --for=condition=Ready nodes --all
+
+      - name: Load images from cache
+        run: |
+          echo "loading docker images..."
+          pigz -dc docker.tar.gz | docker load
+          make push-images
+
+      - name: Run E2E test suite
+        shell: bash
+        env:
+          ENABLE_PROXY: "false"
+          E2E_SKIP_BUILD: "1"
+          E2E_FLAKE_ATTEMPTS: "2"
+          E2E_ENV: "ci"
+          E2E_NODES: "8"
+        run: |
+          make e2e-test

.gitignore

@@ -26,6 +26,9 @@
 # Test binary, build with `go test -c`
 *.test
 
+# Related log files
+*.log
+
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
 release

Makefile

@@ -60,13 +60,14 @@ clean-image: ## Removes local image
 ### build-image:          Build apisix-ingress-controller image
 .PHONY: build-image
 build-image:
-	docker build -t apache/apisix-ingress-controller:$(IMAGE_TAG) --build-arg ENABLE_PROXY=true .
-
-### pack-ingress-image:   Build and push Ingress image used in e2e test suites to kind or custom registry.
-.PHONY: pack-ingress-image
-pack-ingress-image:
+ifeq ($(E2E_SKIP_BUILD), 0)
 	docker build -t apache/apisix-ingress-controller:$(IMAGE_TAG) --build-arg ENABLE_PROXY=$(ENABLE_PROXY) .
 	docker tag apache/apisix-ingress-controller:$(IMAGE_TAG) $(REGISTRY)/apache/apisix-ingress-controller:$(IMAGE_TAG)
+endif
+
+### pack-image:   Build and push Ingress image used in e2e test suites to kind or custom registry.
+.PHONY: pack-image
+pack-image: build-image
 	docker push $(REGISTRY)/apache/apisix-ingress-controller:$(IMAGE_TAG)
 
 ### pack-images:          Build and push images used in e2e test suites to kind or custom registry.
@@ -250,10 +251,10 @@ endif
 .PHONY: kind-load-images
 kind-load-images:
 	kind load docker-image --name=apisix \
-			localhost:5000/apache/apisix:dev \
-            localhost:5000/bitnami/etcd:dev \
-            localhost:5000/apache/apisix-ingress-controller:dev \
-            localhost:5000/kennethreitz/httpbin:dev \
-            localhost:5000/test-backend:dev \
-            localhost:5000/jmalloc/echo-server:dev \
-            localhost:5000/busybox:dev
+			$(REGISTRY)/apache/apisix:dev \
+            $(REGISTRY)/bitnami/etcd:dev \
+            $(REGISTRY)/apache/apisix-ingress-controller:dev \
+            $(REGISTRY)/kennethreitz/httpbin:dev \
+            $(REGISTRY)/test-backend:dev \
+            $(REGISTRY)/jmalloc/echo-server:dev \
+            $(REGISTRY)/busybox:dev

test/e2e/scaffold/ingress.go

@@ -390,7 +390,6 @@ spec:
             - %s
             - --ingress-status-address
             - "%s"
-            - --watch-endpointslices
             - --enable-gateway-api
             - "true"
           %s

test/e2e/suite-features/retries.go

@@ -61,7 +61,7 @@ spec:
 `, backendSvc)
 			err = s.CreateVersionedApisixResource(au)
 			assert.Nil(ginkgo.GinkgoT(), err, "create ApisixUpstream")
-			time.Sleep(2 * time.Second)
+			time.Sleep(6 * time.Second)
 
 			ups, err := s.ListApisixUpstreams()
 			assert.Nil(ginkgo.GinkgoT(), err)

utils/kind-with-registry.sh

@@ -22,6 +22,7 @@ set -o pipefail
 
 # desired cluster name; default is "apisix"
 KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-apisix}"
+K8S_VERSION=${K8S_VERSION:-v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6}
 
 if kind get clusters | grep -q ^apisix$ ; then
   echo "cluster already exists, moving on"
@@ -54,7 +55,8 @@ fi
 echo "Registry Host: ${reg_host}"
 
 # create a cluster with the local registry enabled in containerd
-kind_node_image='kindest/node:v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6'
+kind_node_image="kindest/node:${K8S_VERSION}"
+echo "Kubernetes version: ${kind_node_image}"
 cat <<EOF | kind create cluster --name "${KIND_CLUSTER_NAME}" --image ${kind_node_image} --config=-
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4