Merge pull request #60 from damsien/e2e-refactor

🌱 Enhance test framework & Makefile & dynamic webhook lifecycle & merge dynamic webhooks servers

.github/workflows/tests.yml

@@ -7,8 +7,8 @@ on:
 
 jobs:
 
-  tests:
-    name: Controllers tests
+  tests-build-deploy:
+    name: Build & deploy tests
     runs-on: ubuntu-latest
 
     steps:
@@ -39,11 +39,8 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y make
 
-    - name: Setup the tls certs
-      run: ./config/webhook/gen-certs-serv-cli.sh
-
     - name: Run tests
-      run: make test
+      run: make test-build-deploy
 
   tests-e2e:
     name: End-to-end tests
@@ -77,5 +74,10 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y make
 
+    - name: Install kubernetes binaries
+      run: |
+        make envtest
+        ./bin/setup-envtest-latest use --bin-dir ./bin
+
     - name: Run tests
       run: make test-e2e

Makefile

@@ -24,6 +24,12 @@ CONTAINER_TOOL ?= docker
 SHELL = /usr/bin/env bash -o pipefail
 .SHELLFLAGS = -ec
 
+# WEBHOOK_PATH is the path to the webhook directory.
+WEBHOOK_PATH ?= config/webhook
+
+# DYNAMIC_WEBHOOK_NAME is the name of the webhook that handle the interception logic of RemoteSyncers
+DYNAMIC_WEBHOOK_NAME ?= dynamic-remotesyncer-webhook
+
 .PHONY: all
 all: build
 
@@ -44,53 +50,45 @@ all: build
 help: ## Display this help.
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
-##@ e2e Custom deployments
+.PHONY: pre-commit-check
+pre-commit-check: manifests generate test lint ## Run all the tests and linters.
 
-.PHONY: setup-gitea
-setup-gitea:
-	./test/utils/gitea/launch-gitea-setup.sh
+##@ Dev environment
 
-.PHONY: cleanup-gitea
-cleanup-gitea:
-	helm uninstall gitea -n jupyter
-	kubectl delete ns jupyter
-	helm uninstall gitea -n saturn
-	kubectl delete ns saturn
+# DEV_WEBHOOK_HOST is a IP:PORT combination. The static & dynamic webhooks will be served on this host.
+DEV_WEBHOOK_HOST ?= "172.17.0.1:9443" # 172.17.0.1 is the default docker0 bridge IP.
+# DEV_WEBHOOK_CERT is the path to the certificate that will be used by the webhook server.
+DEV_WEBHOOK_CERT ?= "/tmp/k8s-webhook-server/serving-certs/tls.crt"
 
-##@ Development
-
-WEBHOOK_PATH ?= config/webhook
-.PHONY: deploy-all
-deploy-all: # Launch dev env on the cluster
-	make cleanup-e2e || true
-	kind create cluster --name $(DEV_CLUSTER) 2>/dev/null || true
-	make docker-build IMG=$(IMG)
-	kind load docker-image $(IMG) --name $(DEV_CLUSTER)
-	make deploy IMG=$(IMG)
+.PHONY: run-fast
+run-fast: manifests generate fmt vet ## Run a controller from your host. No resources are installed. No resources are deleted when killed (meant to be run often).
+	cd $(WEBHOOK_PATH) && ./gen-certs-serv-cli.sh 1 >/dev/null
+	export MANAGER_NAMESPACE=syngit DYNAMIC_WEBHOOK_NAME=$(DYNAMIC_WEBHOOK_NAME) DEV_MODE="true" DEV_WEBHOOK_HOST=$(DEV_WEBHOOK_HOST) DEV_WEBHOOK_CERT=$(DEV_WEBHOOK_CERT) && go run cmd/main.go
 
-LATEST_CHART ?= $(shell find charts -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sort -V | tail -n 1)
-.PHONY: chart-install
-chart-install:
-	helm install syngit charts/$(LATEST_CHART) -n syngit --create-namespace \
-		--set controller.image.prefix=local \
-		--set controller.image.name=syngit-controller \
-		--set controller.image.tag=dev
+.PHONY: run
+run: manifests generate fmt vet install-crds install-dev-webhooks ## Install CRDs, webhooks & run a controller from your host. All resources are deleted when killed.
+	cd $(WEBHOOK_PATH) && ./gen-certs-serv-cli.sh 1 >/dev/null
+	export MANAGER_NAMESPACE=syngit DYNAMIC_WEBHOOK_NAME=$(DYNAMIC_WEBHOOK_NAME) DEV_MODE="true" DEV_WEBHOOK_HOST=$(DEV_WEBHOOK_HOST) DEV_WEBHOOK_CERT=$(DEV_WEBHOOK_CERT) && \
+	{ \
+		trap 'echo "Cleanup resources"; make uninstall-crds && make uninstall-dev-webhooks; exit' SIGINT; \
+		go run cmd/main.go; \
+	}
+
+.PHONY: run-full
+run-full: manifests generate fmt vet install-crds install-dev-webhooks ## Install CRDs, webhooks & run a controller from your host. No resources are deleted when killed (meant to be run often).
+	cd $(WEBHOOK_PATH) && ./gen-certs-serv-cli.sh 1 >/dev/null
+	export MANAGER_NAMESPACE=syngit DYNAMIC_WEBHOOK_NAME=$(DYNAMIC_WEBHOOK_NAME) DEV_MODE="true" DEV_WEBHOOK_HOST=$(DEV_WEBHOOK_HOST) DEV_WEBHOOK_CERT=$(DEV_WEBHOOK_CERT) && go run cmd/main.go
 
-.PHONY: chart-upgrade
-chart-upgrade:
-	helm upgrade syngit charts/$(LATEST_CHART) -n syngit \
-		--set controller.image.prefix=local \
-		--set controller.image.name=syngit-controller \
-		--set controller.image.tag=dev
+.PHONY: cleanup-run
+cleanup-run: uninstall-crds uninstall-dev-webhooks ## Cleanup the resources created by make run.
 
-.PHONY: chart-uninstall
-chart-uninstall:
-	helm uninstall syngit -n syngit
+.PHONY: reset-certs
+reset-certs: ## Reset the temporary certificates for the webhook (/tmp/k8s-webhook-server/serving-certs).
+	cd $(WEBHOOK_PATH) && ./cleanup-injector.sh . || true
+	rm -rf /tmp/k8s-webhook-server/serving-certs
+	cd $(WEBHOOK_PATH) && ./gen-certs-serv-cli.sh 1 >/dev/null
 
-.PHONY: undeploy-all
-undeploy-all: # Cleanup
-	cd $(WEBHOOK_PATH) && ./cleanup-injector.sh
-	make undeploy
+##@ Development
 
 .PHONY: manifests
 manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
@@ -108,30 +106,36 @@ fmt: ## Run go fmt against code.
 vet: ## Run go vet against code.
 	go vet ./...
 
+##@ Test
+
 .PHONY: test
-test: manifests generate fmt vet envtest ## Run tests.
-	cd $(WEBHOOK_PATH) && ./cert-injector.sh manifests.yaml ../crd/patches 1 >/dev/null
+test: test-controller test-build-deploy test-e2e ## Run all the tests.
+
+.PHONY: test-controller
+test-controller: manifests generate fmt vet envtest ## Run tests embeded in the controller package & webhook package.
+	cd $(WEBHOOK_PATH) && ./cert-injector.sh . 1 >/dev/null
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out
-	cd $(WEBHOOK_PATH) && ./cleanup-injector.sh
+	cd $(WEBHOOK_PATH) && ./cleanup-injector.sh . || true
 
-# Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors.
-.PHONY: test-e2e  # Run the e2e tests against a Kind k8s instance that is spun up.
-test-e2e:
-	kind create cluster --name $(DEV_CLUSTER) 2>/dev/null || true
-	make undeploy-all || true
+.PHONY: test-build-deploy
+test-build-deploy: ## Run tests to build the Docker image and deploy all the manifests.
 	go test ./test/e2e/build -v -ginkgo.v
-	go test ./test/e2e/syngit -v -ginkgo.v
+
+# DEPRECATED_API_VERSIONS is a list of API versions that should not be tested since they are supposed to be converted to the last one.
+DEPREACTED_API_VERSIONS = $(shell go list ./... | grep -oP 'v\d+\w+\d+' | sort -uV | awk 'NR == 1 {latest = $$0} NR > 1 {print prev} {prev = $$0}' | grep -v '^$$' | paste -sd "|" -)
+# COVERPKG is a list of packages to be covered by the tests (internal/, pkg/ & cmd/).
+COVERPKG = $(shell go list ./... | grep -v 'test' | grep -v -E "$(DEPREACTED_API_VERSIONS)" | paste -sd "," -)
+
+.PHONY: test-e2e
+test-e2e: ## Install the test env (gitea). Run the e2e tests against a Kind k8s instance that is spun up. Cleanup when finished.
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./test/e2e/syngit -v -ginkgo.v -cover -coverpkg=$(COVERPKG)
 
 .PHONY: fast-e2e
-fast-e2e:
-	kind create cluster --name $(DEV_CLUSTER) 2>/dev/null || true
-	make undeploy-all || true
-	make docker-build
-	make kind-load-image
-	go test ./test/e2e/syngit -v -ginkgo.v -setup fast
-
-.PHONY: cleanup-e2e
-cleanup-e2e:
+fast-e2e: ## Install the test env if not already installed. Run the e2e tests against a Kind k8s instance that is spun up. Does not cleanup when finished (meant to be run often).
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./test/e2e/syngit -v -ginkgo.v -cover -coverpkg=$(COVERPKG) -setup fast
+
+.PHONY: cleanup-tests
+cleanup-tests: ## Uninstall all the charts needed for the tests.
 	helm uninstall -n syngit syngit || true
 	helm uninstall -n saturn gitea || true
 	helm uninstall -n jupyter gitea || true
@@ -151,11 +155,6 @@ lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes
 build: manifests generate fmt vet ## Build manager binary.
 	go build -o bin/manager cmd/main.go
 
-.PHONY: run
-run: manifests generate fmt vet ## Run a controller from your host.
-	cd $(WEBHOOK_PATH) && ./gen-certs-serv-cli.sh 1 >/dev/null
-	export MANAGER_NAMESPACE=syngit DYNAMIC_WEBHOOK_NAME=remotesyncer.syngit.io DEV=true && go run cmd/main.go
-
 # If you wish to build the manager image targeting other platforms you can use the --platform flag.
 # (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it.
 # More info: https://docs.docker.com/develop/develop-images/build_enhancements/
@@ -167,10 +166,6 @@ docker-build: ## Build docker image with the manager.
 docker-push: ## Push docker image with the manager.
 	$(CONTAINER_TOOL) push ${IMG}
 
-.PHONY: kind-load-image
-kind-load-image: ## Load the image in KIND
-	kind load docker-image ${IMG} --name ${DEV_CLUSTER}
-
 # PLATFORMS defines the target platforms for the manager image be built to provide support to multiple
 # architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to:
 # - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/
@@ -201,27 +196,93 @@ build-installer: manifests generate kustomize ## Generate a consolidated YAML wi
 ##@ Deployment
 
 ifndef ignore-not-found
-  ignore-not-found = false
+  ignore-not-found = true
 endif
 
-.PHONY: install
-install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
+.PHONY: install-crds
+install-crds: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
 	$(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f -
 
-.PHONY: uninstall
-uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
+.PHONY: install-dev-webhooks
+install-dev-webhooks: manifests kustomize ## Deploy dev webhooks using the docker bridge host into the K8s cluster specified in ~/.kube/config.
+	cd $(WEBHOOK_PATH) && ./cert-injector.sh .
+	cat $(WEBHOOK_PATH)/dev-webhook.yaml | $(KUBECTL) apply -f -
+	cat $(WEBHOOK_PATH)/dynamic-webhook.yaml | $(KUBECTL) apply -f -
+
+.PHONY: uninstall-crds
+uninstall-crds: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config.
 	$(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -
 
+.PHONY: uninstall-dev-webhooks
+uninstall-dev-webhooks: manifests kustomize ## Undeploy dev webhooks using the docker bridge host into the K8s cluster specified in ~/.kube/config.
+	cd $(WEBHOOK_PATH) && ./cleanup-injector.sh . || true
+	cat $(WEBHOOK_PATH)/dev-webhook.yaml | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -
+	cat $(WEBHOOK_PATH)/dynamic-webhook.yaml | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -
+
 .PHONY: deploy
-deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
+deploy: manifests kustomize ## Deploy syngit to the K8s cluster specified in ~/.kube/config.
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
-	cd $(WEBHOOK_PATH) && ./cert-injector.sh manifests.yaml ../crd/patches
+	cd $(WEBHOOK_PATH) && ./cert-injector.sh .
 	$(KUSTOMIZE) build config/default | $(KUBECTL) apply -f -
 
 .PHONY: undeploy
-undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
+undeploy: kustomize ## Undeploy syngit from the K8s cluster specified in ~/.kube/config. Can be use after deploy or deploy-all.
+	cd $(WEBHOOK_PATH) && ./cleanup-injector.sh . || true
 	$(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -
 
+.PHONY: deploy-all
+deploy-all: kind-create-cluster docker-build kind-load-image cleanup-tests deploy # Create the dev cluster, build the image, load it in the cluster and deploy syngit.
+
+##@ KinD & Helm
+
+# BEFORE_LATEST_CHART is the chart version before the latest one listed in the charts/ folder.
+BEFORE_LATEST_CHART ?= $(shell ls -v charts | tail -3 | head -1)
+# LATEST_CHART is the latest chart version listed in the charts/ folder.
+LATEST_CHART ?= $(shell find charts -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sort -V | tail -n 1)
+
+.PHONY: chart-install
+chart-install: ## Install the latest chart version listed in the charts/ folder.
+	helm install syngit charts/$(LATEST_CHART) -n syngit --create-namespace \
+		--set controller.image.prefix=local \
+		--set controller.image.name=syngit-controller \
+		--set controller.image.tag=dev
+
+.PHONY: chart-upgrade
+chart-upgrade: ## Upgrade to the latest chart version listed in the charts/ folder.
+	helm upgrade syngit charts/$(LATEST_CHART) -n syngit \
+		--set controller.image.prefix=local \
+		--set controller.image.name=syngit-controller \
+		--set controller.image.tag=dev
+
+.PHONY: chart-uninstall
+chart-uninstall: ## Uninstall the chart.
+	helm uninstall syngit -n syngit
+
+.PHONY: kind-create-cluster
+kind-create-cluster: ## Create the dev KinD cluster.
+	kind create cluster --name ${DEV_CLUSTER} || true
+
+.PHONY: kind-delete-cluster
+kind-delete-cluster: ## Delete the dev KinD cluster.
+	kind delete cluster --name ${DEV_CLUSTER}
+
+.PHONY: kind-load-image
+kind-load-image: ## Load the image in KinD.
+	kind load docker-image ${IMG} --name ${DEV_CLUSTER}
+
+##@ e2e Custom deployments
+
+.PHONY: setup-gitea
+setup-gitea: ## Setup the 2 gitea platforms in the cluster
+	./test/utils/gitea/launch-gitea-setup.sh
+
+.PHONY: cleanup-gitea
+cleanup-gitea: ## Cleanup the 2 gitea platforms from the cluster.
+	helm uninstall gitea -n jupyter
+	kubectl delete ns jupyter
+	helm uninstall gitea -n saturn
+	kubectl delete ns saturn
+
 ##@ Dependencies
 
 ## Location to install dependencies to

cmd/main.go

@@ -20,6 +20,7 @@ import (
 	"crypto/tls"
 	"flag"
 	"os"
+	"sync"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
@@ -227,8 +228,19 @@ func main() {
 	}
 
 	setupLog.Info("starting manager")
-	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
-		setupLog.Error(err, "problem running manager")
-		os.Exit(1)
-	}
+
+	signalCh := ctrl.SetupSignalHandler()
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	go func() {
+		defer wg.Done() // Notify the WaitGroup when the manager shuts down
+		if err := mgr.Start(signalCh); err != nil {
+			setupLog.Error(err, "problem running manager")
+			os.Exit(1)
+		}
+	}()
+
+	// Block the main goroutine until the signal is received and the manager shuts down
+	wg.Wait() // Wait for the manager goroutine to finish
 }

config/crd/kustomization.yaml

@@ -26,4 +26,4 @@ patches:
 # the following config is for teaching kustomize how to do kustomization for CRDs.
 
 configurations:
-- kustomizeconfig.yaml
+- kustomizeconfig.yaml

config/manager/manager.yaml

@@ -77,7 +77,7 @@ spec:
             fieldRef:
               fieldPath: metadata.namespace
         - name: DYNAMIC_WEBHOOK_NAME
-          value: remotesyncer.syngit.io
+          value: syngit-dynamic-remotesyncer-webhook
         name: manager
         securityContext:
           allowPrivilegeEscalation: false