From 4623bb49a83e5aa16b025b8ffbdc529293a32436 Mon Sep 17 00:00:00 2001 From: Zlatko Bratkovic Date: Thu, 25 Aug 2022 15:13:16 +0200 Subject: [PATCH] BUILD: ci: speed up ci with parallel start e2e stages --- .gitlab-ci.yml | 62 ++++++++++++++++++++++++++++++++++++------ Makefile | 1 + deploy/tests/create.sh | 13 +++++++-- 3 files changed, 65 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a9c1d7b0..c26d1097 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,6 +2,7 @@ stages: - diff - lint - unit-tests + - build - e2e-tests variables: KUBECTL: v1.24.3 @@ -25,6 +26,7 @@ diff: - branches tidy: stage: lint + needs: [] image: name: $CI_REGISTRY_GO/golang:1.19 entrypoint: [""] @@ -38,6 +40,7 @@ tidy: - branches golangci_lint: stage: lint + needs: [] image: name: $CI_REGISTRY_GO/lint:1.33 entrypoint: [""] @@ -50,6 +53,7 @@ golangci_lint: - branches lint-commit-msg: stage: lint + needs: [] image: name: $CI_REGISTRY_GO/check-commit:v2.1.0 entrypoint: [""] @@ -60,6 +64,7 @@ lint-commit-msg: only: - merge_requests unit-tests: + needs: ["diff", "tidy"] stage: unit-tests image: name: $CI_REGISTRY_GO/haproxy-alpine:2.5-go1.19 @@ -72,8 +77,9 @@ unit-tests: only: - merge_requests - branches -.kind_deployment: - stage: e2e-tests +docker-build: + stage: build + needs: [] image: $CI_REGISTRY_GO/docker:stable-go1.19 services: - name: $CI_REGISTRY_GO/docker:18.09.7-dind @@ -82,12 +88,46 @@ unit-tests: - go before_script: - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY_GO - script: - go version - - docker pull $CI_REGISTRY_GO/alpine:3 + - docker pull -q $CI_REGISTRY_GO/alpine:3 - docker image tag $CI_REGISTRY_GO/alpine:3 alpine:3 - - docker pull $CI_REGISTRY_GO/golang:1.19-alpine + - docker pull -q $CI_REGISTRY_GO/golang:1.19-alpine - docker image tag $CI_REGISTRY_GO/golang:1.19-alpine golang:1.19-alpine + - sed -i "s~FROM golang~FROM $CI_REGISTRY_GO/golang~g" "build/Dockerfile" + - sed -i "s~FROM haproxytech/haproxy-alpine~FROM $CI_REGISTRY_GO/haproxy-alpine~g" "build/Dockerfile" + script: + - mkdir -p tar + - make build + - docker save -o tar/k8sIC.tar haproxytech/kubernetes-ingress:latest + artifacts: + when: on_success + paths: + - tar + expire_in: 15 minutes +.kind_deployment: + stage: e2e-tests + needs: ["diff", "tidy", "docker-build"] + image: $CI_REGISTRY_GO/docker:stable-go1.19 + services: + - name: $CI_REGISTRY_GO/docker:18.09.7-dind + alias: docker + tags: + - go + artifacts: + paths: + - tar + expire_in: 30 minutes + before_script: + - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY_GO + - docker load -i tar/k8sIC.tar + - go version + - go mod download & + #- docker pull -q $CI_REGISTRY_GO/alpine:3 + #- docker image tag $CI_REGISTRY_GO/alpine:3 alpine:3 + #- docker pull -q $CI_REGISTRY_GO/golang:1.19-alpine + #- docker image tag $CI_REGISTRY_GO/golang:1.19-alpine golang:1.19-alpine + - docker pull -q $CI_REGISTRY_GO/haproxytech/http-echo:latest + - docker image tag $CI_REGISTRY_GO/haproxytech/http-echo:latest haproxytech/http-echo:latest - wget -nv -O /usr/local/bin/kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND}/kind-linux-amd64 - chmod +x /usr/local/bin/kind - wget -nv -O /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL}/bin/linux/amd64/kubectl @@ -102,16 +142,20 @@ unit-tests: - sed -i -E -e "s/localhost|0\.0\.0\.0|127\.0\.0\.1/docker/g" "deploy/tests/e2e/client.go" - until kubectl get nodes -o wide;do sleep 2; done - kubectl get pods -A - - CI_ENV=gitlab deploy/tests/create.sh - - kubectl get pods -A - - KIND_URL=docker CGO_ENABLED=0 go test -v ./deploy/tests/e2e/... --tags=e2e_parallel - - KIND_URL=docker CGO_ENABLED=0 go test -v -p 1 ./deploy/tests/e2e/... --tags=e2e_sequential after_script: - kubectl get pods -A - kind delete cluster --name=dev only: - merge_requests - branches + parallel: + matrix: + - TEST_PART: [ "18#e2e_parallel", "1#e2e_sequential" ] + VAR_DUMMY: "YES" + script: + - CI_ENV=gitlab deploy/tests/create.sh + - kubectl get pods -A + - IFS='#' read -r -a array <<< $TEST_PART; KIND_URL=docker CGO_ENABLED=0 go test -v -p ${array[0]} ./deploy/tests/e2e/... --tags=${array[1]} .kind_deployment_schedules: only: - schedules diff --git a/Makefile b/Makefile index 6d368f3b..0a82edee 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ test: .PHONY: e2e e2e: + go clean -testcache go test ./... --tags=e2e_parallel go test ./... -p 1 --tags=e2e_sequential diff --git a/deploy/tests/create.sh b/deploy/tests/create.sh index fca84920..bf182725 100755 --- a/deploy/tests/create.sh +++ b/deploy/tests/create.sh @@ -9,7 +9,12 @@ if [ -n "${CI_ENV}" ]; then echo "cluster was already created by $CI_ENV CI" echo "building image for ingress controller" - docker build --build-arg TARGETPLATFORM="linux/amd64" -t haproxytech/kubernetes-ingress -f build/Dockerfile . + if [ -n "${GITLAB_CI}" ]; then + echo "haproxytech/kubernetes-ingress image already available from previous stage" + #docker build --build-arg TARGETPLATFORM="linux/amd64" -t haproxytech/kubernetes-ingress -f build/Dockerfile . + else + docker build --build-arg TARGETPLATFORM="linux/amd64" -t haproxytech/kubernetes-ingress -f build/Dockerfile . + fi echo "loading image of ingress controller in kind" kind load docker-image haproxytech/kubernetes-ingress:latest --name=$clustername @@ -31,7 +36,11 @@ else kind load docker-image haproxytech/kubernetes-ingress:latest --name=$clustername fi -docker build --build-arg TARGETPLATFORM="linux/amd64" -t haproxytech/http-echo -f deploy/tests/images/http-echo/Dockerfile deploy/tests/images/http-echo +if [ -n "${GITLAB_CI}" ]; then + echo "haproxytech/http-echo:latest pulled from CI registry" +else + docker build --build-arg TARGETPLATFORM="linux/amd64" -t haproxytech/http-echo -f deploy/tests/images/http-echo/Dockerfile deploy/tests/images/http-echo +fi echo "loading image http-echo in kind" kind load docker-image haproxytech/http-echo:latest --name=$clustername