run e2e tests and modify Makefiles

Signed-off-by: Sanskar Jaiswal <jaiswalsanskar078@gmail.com>

.github/workflows/e2e.yaml

@@ -23,4 +23,4 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-go-
       - name: Run tests
-        run: cd git/e2e && ./run.sh
+        run: cd git && make e2e

Makefile

@@ -1,84 +1,51 @@
 VER?=0.0.1
 MODULES=$(shell find . -mindepth 2 -maxdepth 4 -type f -name 'go.mod' | cut -c 3- | sed 's|/[^/]*$$||' | sort -u | tr / :)
 targets=$(addprefix test-, $(MODULES))
-root_dir=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+root_dir=$(shell git rev-parse --show-toplevel)
 
-# Use $GOBIN from the enviornment if set, otherwise use ./bin
+# Use $GOBIN from the environment if set, otherwise use ./bin
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(root_dir)/bin
 else
 GOBIN=$(shell go env GOBIN)
 endif
 
-# Architecture to use envtest with
-ENVTEST_ARCH ?= amd64
+PKG?=$*
 GO_TEST_ARGS ?= -race
 
-# Repository root based on Git metadata
-REPOSITORY_ROOT := $(shell git rev-parse --show-toplevel)
-
-# Other dependency versions
-ENVTEST_BIN_VERSION ?= 1.19.2
+# Architecture to use envtest with
+ENVTEST_ARCH ?= amd64
 
 all:
 	$(MAKE) $(targets)
 
 tidy-%:
-	@if [ "$(DIR)" = "git" ]; then \
-		cd git && make tidy ;\
-	else \
-		cd $(subst :,/,$*) && go mod tidy -compat=1.17 ;\
-	fi
+	cd $(subst :,/,$*); go mod tidy -compat=1.17
 
 fmt-%:
-	@if [ "$(DIR)" = "git" ]; then \
-		cd git && make fmt ;\
-	else \
-		cd $(subst :,/,$*) && go fmt ./... ;\
-	fi
+	cd $(subst :,/,$*); go fmt ./...
 
 vet-%:
-	@if [ "$(DIR)" = "git" ]; then \
-		cd git && make vet ;\
+	@if [ "$(PKG)" = "git" ]; then \
+		cd $(PKG); make vet ;\
 	else \
-		cd $(subst :,/,$*) && go vet ./... ;\
+		cd $(subst :,/,$*); go vet ./... ;\
 	fi
 
 generate-%: controller-gen
-# Run schemapatch to validate all the kubebuilder markers before generation
-# Skip git/libgit2 as this isn't required for that package and increases
-# the complexity unnecessarily
-	@if [ "$(DIR)" = "git" ]; then \
-		echo "skipping target 'generate' for git " ;\
-	else \
-		cd $(subst :,/,$*) ;\
-		$(CONTROLLER_GEN) schemapatch:manifests="./" paths="./..." ;\
-		$(CONTROLLER_GEN) object:headerFile="$(root_dir)/hack/boilerplate.go.txt" paths="./..." ;\
-	fi
+	# Run schemapatch to validate all the kubebuilder markers before generation
+	cd $(subst :,/,$*); CGO_ENABLED=0 $(CONTROLLER_GEN) schemapatch:manifests="./" paths="./..."
+	cd $(subst :,/,$*); CGO_ENABLED=0 $(CONTROLLER_GEN) object:headerFile="$(root_dir)/hack/boilerplate.go.txt" paths="./..."
 
 # Run tests
 KUBEBUILDER_ASSETS?="$(shell $(ENVTEST) --arch=$(ENVTEST_ARCH) use -i $(ENVTEST_KUBERNETES_VERSION) --bin-dir=$(ENVTEST_ASSETS_DIR) -p path)"
-DIR?=$*
 test-%: tidy-% generate-% fmt-% vet-% install-envtest
-	@if [ "$(DIR)" = "git" ]; then \
-		cd git && make test ;\
+	@if [ "$(PKG)" = "git" ]; then \
+		cd $(PKG); make test ;\
 	else \
-		cd $(subst :,/,$*) && KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) go test $(GO_STATIC_FLAGS) ./... $(GO_TEST_ARGS) -coverprofile cover.out ;\
+		cd $(subst :,/,$*); KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) go test ./... $(GO_TEST_ARGS) -coverprofile cover.out ;\
 	fi
 
-libgit2: $(LIBGIT2)  ## Detect or download libgit2 library
-
-COSIGN = $(GOBIN)/cosign
-$(LIBGIT2): $(MUSL-CC)
-	$(call go-install-tool,$(COSIGN),github.com/sigstore/cosign/cmd/cosign@latest)
-
-	cd git/libgit2 && IMG=$(LIBGIT2_IMG) TAG=$(LIBGIT2_TAG) PATH=$(PATH):$(GOBIN) ./hack/install-libraries.sh
-
-$(MUSL-CC):
-ifneq ($(shell uname -s),Darwin)
-	cd git/libgit2 && ./hack/download-musl.sh
-endif
-
 release-%:
 	$(eval REL_PATH=$(subst :,/,$*))
 	@if ! test -f $(REL_PATH)/go.mod; then echo "Missing ./$(REL_PATH)/go.mod, terminating release process"; exit 1; fi

git/Makefile

@@ -12,7 +12,7 @@ LIBGIT2 := $(LIBGIT2_LIB_PATH)/libgit2.a
 GO_TAGS ?=
 GO_TEST_ARGS ?= -race
 
-# Use $GOBIN from the enviornment if set, otherwise use ./bin
+# Use $GOBIN from the environment if set, otherwise use ./bin
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(PROJECT_DIR)/bin
 else
@@ -45,7 +45,7 @@ COSIGN = $(GOBIN)/cosign
 $(LIBGIT2):
 	$(call go-install-tool,$(COSIGN),github.com/sigstore/cosign/cmd/cosign@latest)
 
-	cd git/libgit2 && IMG=$(LIBGIT2_IMG) TAG=$(LIBGIT2_TAG) PATH=$(PATH):$(GOBIN) ./hack/install-libraries.sh
+	cd libgit2 && IMG=$(LIBGIT2_IMG) TAG=$(LIBGIT2_TAG) PATH=$(PATH):$(GOBIN) ./hack/install-libraries.sh
 
 define go-install-tool
 @[ -f $(1) ] || { \

git/e2e/gitkit_test.go

@@ -22,16 +22,28 @@ package e2e
 import (
 	"fmt"
 	"net/url"
+	"os"
 	"path/filepath"
 	"testing"
 
 	. "github.com/onsi/gomega"
 
 	"github.com/fluxcd/pkg/git"
 	"github.com/fluxcd/pkg/git/gogit"
+	"github.com/fluxcd/pkg/git/libgit2"
+	"github.com/fluxcd/pkg/git/libgit2/transport"
 	"github.com/fluxcd/pkg/gittestserver"
 )
 
+func TestMain(m *testing.M) {
+	err := transport.InitManagedTransport()
+	if err != nil {
+		panic("could not init managed transport")
+	}
+	code := m.Run()
+	os.Exit(code)
+}
+
 func TestGitKitE2E(t *testing.T) {
 	g := NewWithT(t)
 	gitServer, err := gittestserver.NewTempGitServer()
@@ -88,21 +100,29 @@ func TestGitKitE2E(t *testing.T) {
 	}
 
 	protocols := []git.TransportType{git.SSH, git.HTTP}
-	clients := []string{gogit.ClientName}
+	gitClients := []string{gogit.ClientName, libgit2.ClientName}
 
-	testFunc := func(t *testing.T, proto git.TransportType, c string) {
-		t.Run(fmt.Sprintf("repo created using Clone/%s", proto), func(t *testing.T) {
+	testFunc := func(t *testing.T, proto git.TransportType, gitClient string) {
+		t.Run(fmt.Sprintf("repo created using Clone/%s/%s", gitClient, proto), func(t *testing.T) {
 			g := NewWithT(t)
 			var client git.RepositoryClient
 			tmp := t.TempDir()
-			repoName := fmt.Sprintf("gitkit-e2e-checkout-%s", string(proto))
+			repoName := fmt.Sprintf("gitkit-e2e-checkout-%s-%s-%s", string(proto), string(gitClient), randStringRunes(5))
 
 			repoURL, authOptions, err := repoInfo(repoName, proto, gitServer)
 			g.Expect(err).ToNot(HaveOccurred())
 
-			if c == gogit.ClientName {
+			switch gitClient {
+			case gogit.ClientName:
 				client, err = gogit.NewClient(tmp, authOptions)
 				g.Expect(err).ToNot(HaveOccurred())
+				defer client.Close()
+			case libgit2.ClientName:
+				client, err = libgit2.NewClient(tmp, authOptions)
+				g.Expect(err).ToNot(HaveOccurred())
+				defer client.Close()
+			default:
+				t.Fatalf("invalid git client name: %s", gitClient)
 			}
 
 			// init repo on server
@@ -115,27 +135,33 @@ func TestGitKitE2E(t *testing.T) {
 			})
 		})
 
-		t.Run(fmt.Sprintf("repo created using Init/%s", proto), func(t *testing.T) {
+		t.Run(fmt.Sprintf("repo created using Init/%s/%s", gitClient, proto), func(t *testing.T) {
 			g := NewWithT(t)
 			var client git.RepositoryClient
 			tmp := t.TempDir()
-			repoName := fmt.Sprintf("gitkit-e2e-init-%s", string(proto))
+			repoName := fmt.Sprintf("gitkit-e2e-init-%s-%s-%s", string(proto), string(gitClient), randStringRunes(5))
 			upstreamRepoPath := filepath.Join(gitServer.Root(), repoName)
 
 			repoURL, authOptions, err := repoInfo(repoName, proto, gitServer)
 			g.Expect(err).ToNot(HaveOccurred())
 
-			if c == gogit.ClientName {
+			switch gitClient {
+			case gogit.ClientName:
 				client, err = gogit.NewClient(tmp, authOptions)
 				g.Expect(err).ToNot(HaveOccurred())
+			case libgit2.ClientName:
+				client, err = libgit2.NewClient(tmp, authOptions)
+				g.Expect(err).ToNot(HaveOccurred())
+			default:
+				t.Fatalf("invalid git client name: %s", gitClient)
 			}
 
 			testUsingInit(g, client, repoURL, upstreamRepoInfo{
 				url: upstreamRepoPath,
 			})
 		})
 	}
-	for _, client := range clients {
+	for _, client := range gitClients {
 		for _, protocol := range protocols {
 			testFunc(t, protocol, client)
 		}

git/e2e/gitlab_test.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/fluxcd/pkg/git"
 	"github.com/fluxcd/pkg/git/gogit"
+	"github.com/fluxcd/pkg/git/libgit2"
 )
 
 const (
@@ -73,6 +74,7 @@ func TestGitLabE2E(t *testing.T) {
 				return nil, nil, err
 			}
 
+			// ref: https://docs.gitlab.com/15.0/ee/api/users.html#add-ssh-key
 			sshKeyApiEndpoint, err := url.Parse(fmt.Sprintf("%s/api/v4/user/keys", gitlabHTTPHost))
 			if err != nil {
 				return nil, nil, err
@@ -100,7 +102,7 @@ func TestGitLabE2E(t *testing.T) {
 				var body []byte
 				_, err = resp.Body.Read(body)
 				if err != nil {
-					return nil, nil, fmt.Errorf("error reading response body")
+					return nil, nil, fmt.Errorf("error reading response body: %w", err)
 				}
 				return nil, nil, fmt.Errorf("could not register ssh key, resp: %s %s", resp.Status, string(body))
 			}
@@ -125,14 +127,14 @@ func TestGitLabE2E(t *testing.T) {
 		return repoURL, authOptions, nil
 	}
 
-	protocols := []git.TransportType{git.SSH, git.HTTP}
-	clients := []string{gogit.ClientName}
+	protocols := []git.TransportType{git.HTTP, git.SSH}
+	clients := []string{gogit.ClientName, libgit2.ClientName}
 
-	testFunc := func(t *testing.T, proto git.TransportType, c string) {
-		t.Run(fmt.Sprintf("repo created using Clone/%s", proto), func(t *testing.T) {
+	testFunc := func(t *testing.T, proto git.TransportType, gitClient string) {
+		t.Run(fmt.Sprintf("repo created using Clone/%s/%s", gitClient, proto), func(t *testing.T) {
 			g := NewWithT(t)
 
-			repoName := fmt.Sprintf("gitlab-e2e-checkout-%s-%s", string(proto), randStringRunes(5))
+			repoName := fmt.Sprintf("gitlab-e2e-checkout-%s-%s-%s", string(proto), string(gitClient), randStringRunes(5))
 			repoURL, authOptions, err := repoInfo(repoName, proto)
 			g.Expect(err).ToNot(HaveOccurred())
 
@@ -143,9 +145,17 @@ func TestGitLabE2E(t *testing.T) {
 			var client git.RepositoryClient
 			tmp := t.TempDir()
 
-			if c == gogit.ClientName {
+			switch gitClient {
+			case gogit.ClientName:
 				client, err = gogit.NewClient(tmp, authOptions)
 				g.Expect(err).ToNot(HaveOccurred())
+				defer client.Close()
+			case libgit2.ClientName:
+				client, err = libgit2.NewClient(tmp, authOptions)
+				g.Expect(err).ToNot(HaveOccurred())
+				defer client.Close()
+			default:
+				t.Fatalf("invalid git client name: %s", gitClient)
 			}
 
 			testUsingClone(g, client, repoURL, upstreamRepoInfo{
@@ -155,20 +165,28 @@ func TestGitLabE2E(t *testing.T) {
 			})
 		})
 
-		t.Run(fmt.Sprintf("repo created using Init/%s", proto), func(t *testing.T) {
+		t.Run(fmt.Sprintf("repo created using Init/%s/%s", gitClient, proto), func(t *testing.T) {
 			g := NewWithT(t)
 
-			repoName := fmt.Sprintf("gitlab-e2e-checkout-%s-%s", string(proto), randStringRunes(5))
+			repoName := fmt.Sprintf("gitlab-e2e-checkout-%s-%s-%s", string(proto), string(gitClient), randStringRunes(5))
 			repoURL, authOptions, err := repoInfo(repoName, proto)
 			g.Expect(err).ToNot(HaveOccurred())
 			upstreamRepoURL := gitlabHTTPHost + "/" + gitlabUsername + "/" + repoName
 
 			var client git.RepositoryClient
 			tmp := t.TempDir()
 
-			if c == gogit.ClientName {
+			switch gitClient {
+			case gogit.ClientName:
 				client, err = gogit.NewClient(tmp, authOptions)
 				g.Expect(err).ToNot(HaveOccurred())
+				defer client.Close()
+			case libgit2.ClientName:
+				client, err = libgit2.NewClient(tmp, authOptions)
+				g.Expect(err).ToNot(HaveOccurred())
+				defer client.Close()
+			default:
+				t.Fatalf("invalid git client name: %s", gitClient)
 			}
 
 			testUsingInit(g, client, repoURL, upstreamRepoInfo{

git/e2e/run.sh

@@ -1,8 +1,14 @@
 #!/bin/bash
 # This script runs e2e tests for pkg/git.
 
+set -o errexit
 DIR="$(cd "$(dirname "$0")" && pwd)"
+
 source "$DIR"/setup_gitlab.sh
-go test -v -tags e2e ./...
+
+cd "$DIR"
+PKG_CONFIG_PATH=$PKG_CONFIG_PATH CGO_LDFLAGS=$CGO_LDFLAGS go test -v -tags 'netgo,osusergo,static_build,e2e' -race ./...
+
 # cleanup
 docker kill gitlab && docker rm gitlab
+

git/go.mod

@@ -30,7 +30,7 @@ require (
 	// maintained by the ProtonMail team to continue to support the openpgp
 	// module, after the Go team decided to no longer maintain it.
 	// When in doubt (and not using openpgp), use /x/crypto.
-	github.com/ProtonMail/go-crypto v0.0.0-20220517143526-88bb52951d5b
+	github.com/ProtonMail/go-crypto v0.0.0-20220714114130-e85cedf506cd
 	github.com/fluxcd/gitkit v0.6.0
 	github.com/fluxcd/pkg/gittestserver v0.5.4
 	github.com/fluxcd/pkg/gitutil v0.1.0
@@ -44,7 +44,7 @@ require (
 	github.com/libgit2/git2go/v33 v33.0.9
 	github.com/onsi/gomega v1.19.0
 	golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f
-	golang.org/x/net v0.0.0-20220607020251-c690dde0001d
+	golang.org/x/net v0.0.0-20220225172249-27dd8689420f
 	sigs.k8s.io/controller-runtime v0.11.2
 )
 
@@ -103,11 +103,11 @@ require (
 	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/api v0.24.2 // indirect
 	k8s.io/apiextensions-apiserver v0.24.2 // indirect
 	k8s.io/apimachinery v0.24.2 // indirect