Allow integration tests to run locally

This change adds the ability to run integration tests locally. You will
still need a number of environment variables set, including a github PAT.

Details on how to use this will come in a subsequent commit.

Signed-off-by: Gabriel Adrian Samfira <gsamfira@cloudbasesolutions.com>

Author: gabriel-samfira

Makefile

@@ -1,5 +1,9 @@
-SHELL := bash
+SHELL := /bin/bash
+export SHELLOPTS:=$(if $(SHELLOPTS),$(SHELLOPTS):)pipefail:errexit
 
+.ONESHELL:
+
+GEN_PASSWORD=$(shell (/bin/bash -c 'tr -dc "a-zA-Z0-9" </dev/urandom | head -c 32 ; echo '';'))
 IMAGE_TAG = garm-build
 
 USER_ID=$(shell ((docker --version | grep -q podman) && echo "0" || id -u))
@@ -9,6 +13,12 @@ GOPATH ?= $(shell go env GOPATH)
 VERSION ?= $(shell git describe --tags --match='v[0-9]*' --dirty --always)
 GARM_REF ?= $(shell git rev-parse --abbrev-ref HEAD)
 GO ?= go
+export GARM_PASSWORD ?= ${GEN_PASSWORD}
+export REPO_WEBHOOK_SECRET = ${GEN_PASSWORD}
+export ORG_WEBHOOK_SECRET = ${GEN_PASSWORD}
+export CREDENTIALS_NAME ?= test-garm-creds
+export WORKFLOW_FILE_NAME ?= test.yml
+export GARM_ADMIN_USERNAME ?= admin
 
 .PHONY: help
 help: ## Display this help.
@@ -63,6 +73,18 @@ verify-vendor: ## verify if all the go.mod/go.sum files are up-to-date
 
 verify: verify-vendor lint fmtcheck ## Run all verify-* targets
 
+integration: build ## Run integration tests
+	function cleanup {
+		if [ -e "$$GITHUB_ENV" ];then
+			source $$GITHUB_ENV
+		fi
+		./test/integration/scripts/taredown_garm.sh
+		$(GO) run ./test/integration/gh_cleanup/main.go
+	}
+	trap cleanup EXIT
+	@./test/integration/scripts/setup-garm.sh
+	@$(GO) run ./test/integration/main.go
+
 ##@ Development
 
 go-test: ## Run tests

test/integration/config/config.toml

@@ -1,5 +1,5 @@
 [default]
-callback_url = "${GARM_BASE_URL}/api/v1/callbacks/status"
+callback_url = "${GARM_BASE_URL}/api/v1/callbacks"
 metadata_url = "${GARM_BASE_URL}/api/v1/metadata"
 webhook_url = "${GARM_BASE_URL}/webhooks"
 enable_webhook_management = true
@@ -14,14 +14,14 @@ time_to_live = "8760h"
 
 [apiserver]
 bind = "0.0.0.0"
-port = 9997
+port = ${GARM_PORT}
 use_tls = false
 
 [database]
 backend = "sqlite3"
 passphrase = "${DB_PASSPHRASE}"
 [database.sqlite3]
-  db_file = "/etc/garm/garm.db"
+  db_file = "${GARM_CONFIG_DIR}/garm.db"
 
 [[provider]]
 name = "lxd_local"
@@ -36,8 +36,8 @@ name = "test_external"
 description = "external test provider"
 provider_type = "external"
   [provider.external]
-  config_file = "/etc/garm/test-provider/config"
-  provider_executable = "/etc/garm/test-provider/garm-external-provider"
+  config_file = "${GARM_CONFIG_DIR}/test-provider/config"
+  provider_executable = "${GARM_CONFIG_DIR}/test-provider/garm-external-provider"
 
 [[github]]
 name = "${CREDENTIALS_NAME}"

test/integration/e2e/e2e.go

@@ -51,6 +51,7 @@ func GetControllerInfo() *params.ControllerInfo {
 }
 
 func GracefulCleanup() {
+	slog.Info("Graceful cleanup")
 	// disable all the pools
 	pools, err := listPools(cli, authToken)
 	if err != nil {
@@ -62,7 +63,7 @@ func GracefulCleanup() {
 		if _, err := updatePool(cli, authToken, pool.ID, poolParams); err != nil {
 			panic(err)
 		}
-		slog.Info("Pool disabled", "pool_id", pool.ID)
+		slog.Info("Pool disabled", "pool_id", pool.ID, "stage", "graceful_cleanup")
 	}
 
 	// delete all the instances
@@ -75,7 +76,7 @@ func GracefulCleanup() {
 			if err := deleteInstance(cli, authToken, instance.Name, false); err != nil {
 				panic(err)
 			}
-			slog.Info("Instance deletion initiated", "instance", instance.Name)
+			slog.Info("Instance deletion initiated", "instance", instance.Name, "stage", "graceful_cleanup")
 		}
 	}
 
@@ -91,7 +92,7 @@ func GracefulCleanup() {
 		if err := deletePool(cli, authToken, pool.ID); err != nil {
 			panic(err)
 		}
-		slog.Info("Pool deleted", "pool_id", pool.ID)
+		slog.Info("Pool deleted", "pool_id", pool.ID, "stage", "graceful_cleanup")
 	}
 
 	// delete all the repositories
@@ -103,7 +104,7 @@ func GracefulCleanup() {
 		if err := deleteRepo(cli, authToken, repo.ID); err != nil {
 			panic(err)
 		}
-		slog.Info("Repo deleted", "repo_id", repo.ID)
+		slog.Info("Repo deleted", "repo_id", repo.ID, "stage", "graceful_cleanup")
 	}
 
 	// delete all the organizations
@@ -115,7 +116,7 @@ func GracefulCleanup() {
 		if err := deleteOrg(cli, authToken, org.ID); err != nil {
 			panic(err)
 		}
-		slog.Info("Org deleted", "org_id", org.ID)
+		slog.Info("Org deleted", "org_id", org.ID, "stage", "graceful_cleanup")
 	}
 }
 
@@ -125,12 +126,12 @@ func appendCtrlInfoToGitHubEnv(controllerInfo *params.ControllerInfo) error {
 		slog.Info("GITHUB_ENV not set, skipping appending controller info")
 		return nil
 	}
-	file, err := os.OpenFile(envFile, os.O_WRONLY|os.O_APPEND, os.ModeAppend)
+	file, err := os.OpenFile(envFile, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o644)
 	if err != nil {
 		return err
 	}
 	defer file.Close()
-	if _, err := file.WriteString(fmt.Sprintf("GARM_CONTROLLER_ID=%s\n", controllerInfo.ControllerID)); err != nil {
+	if _, err := file.WriteString(fmt.Sprintf("export GARM_CONTROLLER_ID=%s\n", controllerInfo.ControllerID)); err != nil {
 		return err
 	}
 	return nil

test/integration/e2e/instances.go

@@ -34,7 +34,9 @@ func waitInstanceStatus(name string, status commonParams.InstanceStatus, runnerS
 }
 
 func DeleteInstance(name string, forceRemove bool) {
+	slog.Info("Delete instance", "instance_name", name, "force_remove", forceRemove)
 	if err := deleteInstance(cli, authToken, name, forceRemove); err != nil {
+		slog.Error("Failed to delete instance", "instance_name", name, "error", err)
 		panic(err)
 	}
 	slog.Info("Instance deletion initiated", "instance_name", name)
@@ -103,7 +105,8 @@ func WaitPoolInstances(poolID string, status commonParams.InstanceStatus, runner
 			"runner_status", runnerStatus,
 			"desired_instance_count", instancesCount,
 			"pool_instance_count", len(poolInstances))
-		if int(pool.MinIdleRunners) == len(poolInstances) {
+		// With the addition of JIT runners, we need to make sure that instancesCount is equal to MinIdleRunners.
+		if int(pool.MinIdleRunners) == len(poolInstances) && int(pool.MinIdleRunners) == instancesCount {
 			return nil
 		}
 		time.Sleep(5 * time.Second)

test/integration/e2e/repositories.go

@@ -42,6 +42,7 @@ func InstallRepoWebhook(id string) *params.HookInfo {
 	}
 	_, err := installRepoWebhook(cli, authToken, id, webhookParams)
 	if err != nil {
+		slog.Error("Failed to install repo webhook", "error", err)
 		panic(err)
 	}
 	webhookInfo, err := getRepoWebhook(cli, authToken, id)
@@ -59,9 +60,10 @@ func UninstallRepoWebhook(id string) {
 }
 
 func CreateRepoPool(repoID string, poolParams params.CreatePoolParams) *params.Pool {
-	slog.Info("Create repo pool", "repo_id", repoID)
+	slog.Info("Create repo pool", "repo_id", repoID, "pool_params", poolParams)
 	pool, err := createRepoPool(cli, authToken, repoID, poolParams)
 	if err != nil {
+		slog.Error("Failed to create repo pool", "error", err)
 		panic(err)
 	}
 	return pool

test/integration/main.go

@@ -11,15 +11,13 @@ import (
 	"github.com/cloudbase/garm/test/integration/e2e"
 )
 
-const (
-	adminUsername = "admin"
+var (
+	adminPassword = os.Getenv("GARM_PASSWORD")
+	adminUsername = os.Getenv("GARM_ADMIN_USERNAME")
 	adminFullName = "GARM Admin"
 	adminEmail    = "admin@example.com"
-)
 
-var (
 	baseURL         = os.Getenv("GARM_BASE_URL")
-	adminPassword   = os.Getenv("GARM_PASSWORD")
 	credentialsName = os.Getenv("CREDENTIALS_NAME")
 
 	repoName          = os.Getenv("REPO_NAME")
@@ -116,11 +114,13 @@ func main() {
 	/////////////////////////////
 	// Test external provider ///
 	/////////////////////////////
+	slog.Info("Testing external provider")
 	repoPool2 := e2e.CreateRepoPool(repo.ID, repoPoolParams2)
-	_ = e2e.UpdateRepoPool(repo.ID, repoPool2.ID, repoPoolParams2.MaxRunners, 1)
+	newParams := e2e.UpdateRepoPool(repo.ID, repoPool2.ID, repoPoolParams2.MaxRunners, 1)
+	slog.Info("Updated repo pool", "new_params", newParams)
 	err := e2e.WaitPoolInstances(repoPool2.ID, commonParams.InstanceRunning, params.RunnerPending, 1*time.Minute)
 	if err != nil {
-		slog.With(slog.Any("error", err)).Error("Failed to wait for instance to be running")
+		slog.With(slog.Any("error", err)).Error("Failed to wait for instance to be running", "pool_id", repoPool2.ID, "provider_name", repoPoolParams2.ProviderName)
 	}
 	repoPool2 = e2e.GetRepoPool(repo.ID, repoPool2.ID)
 	e2e.DisableRepoPool(repo.ID, repoPool2.ID)

test/integration/scripts/setup-garm.sh

@@ -6,7 +6,14 @@ BINARIES_DIR="$PWD/bin"
 CONTRIB_DIR="$PWD/contrib"
 CONFIG_DIR="$PWD/test/integration/config"
 CONFIG_DIR_PROV="$PWD/test/integration/provider"
-PROVIDER_BIN_DIR="/opt/garm/providers.d/lxd"
+GARM_CONFIG_DIR=${GARM_CONFIG_DIR:-"/etc/garm"}
+PROVIDER_BIN_DIR="$GARM_CONFIG_DIR/providers.d/lxd"
+IS_GH_WORKFLOW=${IS_GH_WORKFLOW:-"true"}
+export LXD_PROVIDER_LOCATION=${LXD_PROVIDER_LOCATION:-""}
+export RUN_USER=${RUN_USER:-"garm"}
+export GARM_PORT=${GARM_PORT:-"9997"}
+export GARM_SERVICE_NAME=${GARM_SERVICE_NAME:-"garm"}
+export GARM_CONFIG_FILE=${GARM_CONFIG_FILE:-"${GARM_CONFIG_DIR}/config.toml"}
 
 if [[ ! -f $BINARIES_DIR/garm ]] || [[ ! -f $BINARIES_DIR/garm-cli ]]; then
     echo "ERROR: Please build GARM binaries first"
@@ -41,33 +48,54 @@ function wait_open_port() {
 export JWT_AUTH_SECRET="$(generate_secret)"
 export DB_PASSPHRASE="$(generate_secret)"
 
-# Group "adm" is the LXD daemon group as set by the "canonical/setup-lxd" GitHub action.
-sudo useradd --shell /usr/bin/false --system --groups adm --no-create-home garm
-sudo mkdir -p /etc/garm
+if [ $IS_GH_WORKFLOW == "true" ]; then
+    # Group "adm" is the LXD daemon group as set by the "canonical/setup-lxd" GitHub action.
+    sudo useradd --shell /usr/bin/false --system --groups adm --no-create-home garm
+fi
+
+sudo mkdir -p ${GARM_CONFIG_DIR}
 sudo mkdir -p $PROVIDER_BIN_DIR
+sudo chown -R $RUN_USER:$RUN_USER ${PROVIDER_BIN_DIR}
+sudo chown -R $RUN_USER:$RUN_USER ${GARM_CONFIG_DIR}
 
 export LXD_PROVIDER_EXECUTABLE="$PROVIDER_BIN_DIR/garm-provider-lxd"
-export LXD_PROVIDER_CONFIG="/etc/garm/garm-provider-lxd.toml"
+export LXD_PROVIDER_CONFIG="${GARM_CONFIG_DIR}/garm-provider-lxd.toml"
 sudo cp $CONFIG_DIR/garm-provider-lxd.toml $LXD_PROVIDER_CONFIG
 
-git clone https://github.com/cloudbase/garm-provider-lxd ~/garm-provider-lxd
-pushd ~/garm-provider-lxd
-go build -o $LXD_PROVIDER_EXECUTABLE
-popd
+function clone_and_build_lxd_provider() {
+    git clone https://github.com/cloudbase/garm-provider-lxd ~/garm-provider-lxd
+    pushd ~/garm-provider-lxd
+    go build -o $LXD_PROVIDER_EXECUTABLE
+    popd
+}
 
-cat $CONFIG_DIR/config.toml | envsubst | sudo tee /etc/garm/config.toml
-sudo chown -R garm:garm /etc/garm
+if [ $IS_GH_WORKFLOW == "true" ]; then
+    clone_and_build_lxd_provider
+else
+    if [ -z "$LXD_PROVIDER_LOCATION" ];then
+        clone_and_build_lxd_provider
+    else
+        cp $LXD_PROVIDER_LOCATION $LXD_PROVIDER_EXECUTABLE
+    fi
 
-sudo mkdir /etc/garm/test-provider
+fi
+
+cat $CONFIG_DIR/config.toml | envsubst | sudo tee ${GARM_CONFIG_DIR}/config.toml > /dev/null
+sudo chown -R $RUN_USER:$RUN_USER ${GARM_CONFIG_DIR}
+
+sudo mkdir -p ${GARM_CONFIG_DIR}/test-provider
 sudo touch $CONFIG_DIR_PROV/config
-sudo cp $CONFIG_DIR_PROV/* /etc/garm/test-provider  
+sudo cp $CONFIG_DIR_PROV/* ${GARM_CONFIG_DIR}/test-provider
 
 sudo mv $BINARIES_DIR/* /usr/local/bin/
-sudo cp $CONTRIB_DIR/garm.service /etc/systemd/system/garm.service
-
-sudo systemctl daemon-reload
-sudo systemctl start garm
+mkdir -p $HOME/.local/share/systemd/user/
+cat $CONFIG_DIR/garm.service| envsubst | tee $HOME/.local/share/systemd/user/${GARM_SERVICE_NAME}.service > /dev/null
+sudo chown -R $RUN_USER:$RUN_USER ${GARM_CONFIG_DIR}
 
-wait_open_port 127.0.0.1 9997
+systemctl --user daemon-reload
+systemctl --user restart ${GARM_SERVICE_NAME}
+wait_open_port 127.0.0.1 ${GARM_PORT}
 
 echo "GARM is up and running"
+echo "GARM config file is $GARM_CONFIG_FILE"
+echo "GARM service name is $GARM_SERVICE_NAME"