feat: [AH-309]: Artifact Registry integration (harness#2318)

.gitignore

@@ -8,6 +8,8 @@ _research
 web/node_modules
 web/dist
 web/coverage
+web/.yalc
+web/yalc.lock
 yarn-error*
 release
 .idea
@@ -18,6 +20,12 @@ web/cypress/node_modules
 *.rsa
 *.rsa.pub
 node_modules/
+dist
+.yalc
+yalc.lock
+node_modules
 
 # ignore any executables we build
 /gitness
+/registry/logs/*
+/distribution-spec

.golangci.yml

@@ -287,6 +287,8 @@ issues:
       linters: [ govet ]
     - source: "^//\\s*go:generate\\s"
       linters: [ lll ]
+    - text: 'local replacement are not allowed: github.com/harness/gitness'
+      linters: [ gomoddirectives ]
     - text: 'replacement are not allowed: github.com/docker/docker'
       linters: [ gomoddirectives ]
     - source: "(noinspection|TODO)"
@@ -297,6 +299,192 @@ issues:
       linters: [ errorlint ]
     - path: "^cli/"
       linters: [forbidigo]
+    #Registry Specific
+    - path: "^registry/app/manifest/.*"
+      linters: [ tagliatelle, staticcheck, revive ]
+    - path: "^registry/app/dist_temp/.*"
+      linters: [ errorlint ]
+    - path: "^registry/app/driver/filesystem/.*"
+      linters: [ gocritic ]
+    - path: "^registry/app/driver/s3-aws/.*"
+      linters: [ gocognit, gocyclo, gosec, nestif, cyclop]
+    - path: "^registry/app/remote/clients/registry/interceptor/interceptor.go"
+      linters: [ goheader ]
+    - path: "^registry/app/common/http/modifier/modifier.go"
+      linters: [ goheader ]
+    - path: "^registry/app/driver/fileinfo.go"
+      linters: [ goheader ]
+    - path: "^registry/app/driver/storagedriver.go"
+      linters: [ goheader ]
+    - path: "^registry/app/driver/walk.go"
+      linters: [ goheader ]
+    - path: "^registry/app/dist_temp/challenge/addr.go"
+      linters: [ goheader ]
+    - path: "^registry/app/dist_temp/challenge/authchallenge.go"
+      linters: [ goheader ]
+    - path: "^registry/app/dist_temp/challenge/authchallenge_test.go"
+      linters: [ goheader ]
+    - path: "^registry/app/dist_temp/requestutil/util.go"
+      linters: [ goheader ]
+    - path: "^registry/app/dist_temp/requestutil/util_test.go"
+      linters: [ goheader ]
+    - path: "^registry/app/manifest/descriptor.go"
+      linters: [ goheader ]
+    - path: "^registry/app/manifest/doc.go"
+      linters: [ goheader ]
+    - path: "^registry/app/manifest/errors.go"
+      linters: [ goheader ]
+    - path: "^registry/app/manifest/manifests.go"
+      linters: [ goheader ]
+    - path: "^registry/app/manifest/versioned.go"
+      linters: [ goheader ]
+    - path: "^registry/app/common/lib/authorizer.go"
+      linters: [ goheader ]
+    - path: "^registry/app/common/lib/link.go"
+      linters: [ goheader ]
+    - path: "^registry/app/common/http/tls.go"
+      linters: [ goheader ]
+    - path: "^registry/app/common/http/transport.go"
+      linters: [ goheader ]
+    - path: "^registry/app/common/http/transport_test.go"
+      linters: [ goheader ]
+    - path: "^registry/app/manifest/schema2/manifest.go"
+      linters: [ goheader ]
+    - path: "^registry/app/manifest/schema2/manifest_test.go"
+      linters: [ goheader ]
+    - path: "^registry/app/manifest/ocischema/index.go"
+      linters: [ goheader ]
+    - path: "^registry/app/manifest/ocischema/manifest.go"
+      linters: [ goheader ]
+    - path: "^registry/app/remote/clients/registry/auth/null/authorizer.go"
+      linters: [ goheader ]
+    - path: "^registry/app/remote/clients/registry/auth/basic/authorizer.go"
+      linters: [ goheader ]
+    - path: "^registry/app/remote/clients/registry/auth/basic/authorizer_test.go"
+      linters: [ goheader ]
+    - path: "^registry/app/common/lib/errors/const.go"
+      linters: [ goheader ]
+    - path: "^registry/app/common/lib/errors/errors.go"
+      linters: [ goheader ]
+    - path: "^registry/app/common/lib/errors/stack.go"
+      linters: [ goheader ]
+    - path: "^registry/app/common/lib/errors/stack_test.go"
+      linters: [ goheader ]
+    - path: "^registry/app/remote/clients/registry/auth/bearer/authorizer.go"
+      linters: [ goheader ]
+    - path: "^registry/app/remote/clients/registry/auth/bearer/cache.go"
+      linters: [ goheader ]
+    - path: "^registry/app/remote/clients/registry/auth/bearer/scope.go"
+      linters: [ goheader ]
+    - path: "^registry/app/manifest/manifestlist/manifestlist.go"
+      linters: [ goheader ]
+    - path: "^registry/app/manifest/manifestlist/manifestlist_test.go"
+      linters: [ goheader ]
+    - path: "^registry/app/driver/factory/factory.go"
+      linters: [ goheader ]
+    - path: "^registry/app/dist_temp/dcontext/context.go"
+      linters: [ goheader ]
+    - path: "^registry/app/dist_temp/dcontext/doc.go"
+      linters: [ goheader ]
+    - path: "^registry/app/dist_temp/dcontext/http.go"
+      linters: [ goheader ]
+    - path: "^registry/app/dist_temp/dcontext/logger.go"
+      linters: [ goheader ]
+    - path: "^registry/app/dist_temp/dcontext/trace.go"
+      linters: [ goheader ]
+    - path: "^registry/app/dist_temp/dcontext/util.go"
+      linters: [ goheader ]
+    - path: "^registry/app/dist_temp/dcontext/version.go"
+      linters: [ goheader ]
+    - path: "^registry/app/dist_temp/dcontext/http_test.go"
+      linters: [ goheader ]
+    - path: "^registry/app/dist_temp/dcontext/trace_test.go"
+      linters: [ goheader ]
+    - path: "^registry/app/dist_temp/dcontext/version_test.go"
+      linters: [ goheader ]
+    - path: "^registry/app/driver/base/base.go"
+      linters: [ goheader ]
+    - path: "^registry/app/driver/base/regulator.go"
+      linters: [ goheader ]
+    - path: "^registry/app/driver/base/regulator_test.go"
+      linters: [ goheader ]
+    - path: "^registry/app/storage/blobs.go"
+      linters: [ goheader ]
+    - path: "^registry/app/storage/blobwriter.go"
+      linters: [ goheader ]
+    - path: "^registry/app/storage/blobwriter_resumable.go"
+      linters: [ goheader ]
+    - path: "^registry/app/storage/errors.go"
+      linters: [ goheader ]
+    - path: "^registry/app/storage/filereader.go"
+      linters: [ goheader ]
+    - path: "^registry/app/storage/gcstoragelient.go"
+      linters: [ goheader ]
+    - path: "^registry/app/storage/io.go"
+      linters: [ goheader ]
+    - path: "^registry/app/storage/middleware.go"
+      linters: [ goheader ]
+    - path: "^registry/app/storage/ociblobstore.go"
+      linters: [ goheader ]
+    - path: "^registry/app/storage/paths.go"
+      linters: [ goheader ]
+    - path: "^registry/app/storage/storageservice.go"
+      linters: [ goheader ]
+    - path: "^registry/app/remote/clients/registry/client.go"
+      linters: [ goheader ]
+    - path: "^registry/app/remote/adapter/adapter.go"
+      linters: [ goheader ]
+    - path: "^registry/app/remote/clients/registry/auth/authorizer.go"
+      linters: [ goheader ]
+    - path: "^registry/app/driver/s3-aws/s3.go"
+      linters: [ goheader ]
+    - path: "^registry/app/driver/s3-aws/s3_v2_signer.go"
+      linters: [ goheader ]
+    - path: "^registry/app/driver/filesystem/driver.go"
+      linters: [ goheader ]
+    - path: "^registry/app/pkg/docker/app.go"
+      linters: [ goheader ]
+    - path: "^registry/app/pkg/docker/catalog.go"
+      linters: [ goheader ]
+    - path: "^registry/app/pkg/docker/compat.go"
+      linters: [ goheader ]
+    - path: "^registry/app/pkg/docker/context.go"
+      linters: [ goheader ]
+    - path: "^registry/app/pkg/docker/controller.go"
+      linters: [ goheader ]
+    - path: "^registry/app/pkg/docker/local.go"
+      linters: [ goheader ]
+    - path: "^registry/app/pkg/docker/manifest_service.go"
+      linters: [ goheader ]
+    - path: "^registry/app/pkg/docker/remote.go"
+      linters: [ goheader ]
+    - path: "^registry/app/remote/adapter/dockerhub/adapter.go"
+      linters: [ goheader ]
+    - path: "^registry/app/remote/adapter/dockerhub/client.go"
+      linters: [ goheader ]
+    - path: "^registry/app/remote/adapter/dockerhub/consts.go"
+      linters: [ goheader ]
+    - path: "^registry/app/driver/testsuites/testsuites.go"
+      linters: [ goheader ]
+    - path: "^registry/app/dist_temp/errcode/errors.go"
+      linters: [ goheader ]
+    - path: "^registry/app/dist_temp/errcode/handler.go"
+      linters: [ goheader ]
+    - path: "^registry/app/dist_temp/errcode/register.go"
+      linters: [ goheader ]
+    - path: "^registry/app/remote/controller/proxy/controller.go"
+      linters: [ goheader ]
+    - path: "^registry/app/remote/controller/proxy/inflight.go"
+      linters: [ goheader ]
+    - path: "^registry/app/remote/controller/proxy/local.go"
+      linters: [ goheader ]
+    - path: "^registry/app/remote/controller/proxy/remote.go"
+      linters: [ goheader ]
+    - path: "^registry/app/remote/controller/proxy/inflight_test.go"
+      linters: [ goheader ]
+    - path: "^registry/app/remote/adapter/native/adapter.go"
+      linters: [ goheader ]
+    #Registry Specific ends
     - text: "mnd: Magic number: \\d"
       linters:
         - gomnd

.local.env

@@ -10,3 +10,8 @@ GITNESS_DEBUG=true
 GITNESS_DOCKER_API_VERSION=1.41
 GITNESS_SSH_ENABLE=true
 GITNESS_SSH_HOST=localhost
+GITNESS_SSH_PORT=2222
+
+GITNESS_REGISTRY_STORAGE_TYPE=filesystem
+GITNESS_REGISTRY_ENABLED=false
+GITNESS_REGISTRY_FILESYSTEM_ROOT_DIRECTORY=/tmp

Makefile

@@ -34,7 +34,7 @@ tools: $(tools) ## Install tools required for the build
 
 ###############################################################################
 #
-# Build and testing rules
+# Gitness Build and testing rules
 #
 ###############################################################################
 
@@ -47,6 +47,43 @@ test: generate  ## Run the go tests
 	go test -v -coverprofile=coverage.out ./...
 	go tool cover -html=coverage.out
 
+
+
+###############################################################################
+#
+# Artifact Registry Build and testing rules
+#
+###############################################################################
+
+run: clean build
+	./gitness server .local.env || true
+
+ar-conformance-test: clean build
+	./gitness server .local.env > logfile.log 2>&1 & echo $$! > server.PID
+	@sleep 10
+	./registry/tests/conformance_test.sh localhost:3000 || true
+	kill `cat server.PID`
+	@rm server.PID
+	@rm logfile.log
+
+ar-hot-conformance-test:
+	rm -rf distribution-spec || true
+	./registry/tests/conformance_test.sh localhost:3000 || true
+
+ar-api-update:
+	@set -e; \
+	oapi-codegen --config ./registry/config/openapi/artifact-services.yaml ./registry/app/api/openapi/api.yaml; \
+	oapi-codegen --config ./registry/config/openapi/artifact-types.yaml ./registry/app/api/openapi/api.yaml;
+
+ar-clean:
+	@rm artifact-registry 2> /dev/null || true
+	@docker stop ps_artifacthub 2> /dev/null || true
+	rm -rf distribution-spec
+	@kill -9 $$(lsof -t -i:3000) || true
+	@rm server.PID || true
+	@rm logfile.log || true
+	go clean
+
 ###############################################################################
 #
 # Code Formatting and linting

NOTICE

@@ -0,0 +1,12 @@
+Copyright 2024 Harness, Inc.
+
+This product includes software developed at
+
+https://github.com/goharbor/harbor
+Licensed under the Apache License, Version 2.0
+
+https://github.com/distribution/distribution
+Licensed under the Apache License, Version 2.0
+
+https://gitlab.com/gitlab-org/container-registry
+Licensed under the Apache License, Version 2.0

README.md

@@ -96,6 +96,14 @@ To regenerate the code, please execute the following steps:
 
 The latest API changes should now be reflected in `web/src/services/code/index.tsx`
 
+# Run Registry Conformance Tests
+```
+make conformance-test
+```
+For running conformance tests with existing running service, use:
+```
+make hot-conformance-test
+```
 
 ## User Interface
 
@@ -104,6 +112,7 @@ This project includes a full user interface for interacting with the system. Whe
 ## REST API
 
 This project includes a swagger specification. When you run the application, you can access the swagger specification by navigating to `http://localhost:3000/swagger` in your browser (for raw yaml see `http://localhost:3000/openapi.yaml`).
+For registry endpoints, currently swagger is located on different endpoint `http://localhost:3000/registry/swagger/` (for raw json see `http://localhost:3000/registry/swagger.json`). These will be later moved to the main swagger endpoint. 
 
 
 For testing, it's simplest to just use the cli to create a token (this requires gitness server to run):

app/api/auth/auth.go

@@ -38,15 +38,40 @@ var (
 // Check checks if a resource specific permission is granted for the current auth session in the scope.
 // Returns nil if the permission is granted, otherwise returns an error.
 // NotAuthenticated, NotAuthorized, or any underlying error.
-func Check(ctx context.Context, authorizer authz.Authorizer, session *auth.Session,
+func Check(
+	ctx context.Context, authorizer authz.Authorizer, session *auth.Session,
 	scope *types.Scope, resource *types.Resource, permission enum.Permission,
 ) error {
 	authorized, err := authorizer.Check(
 		ctx,
 		session,
 		scope,
 		resource,
-		permission)
+		permission,
+	)
+	if err != nil {
+		return err
+	}
+
+	if !authorized {
+		return ErrNotAuthorized
+	}
+
+	return nil
+}
+
+// CheckAll checks if multiple resources specific permission is granted for the current auth session in the scope.
+// Returns nil if the permission is granted, otherwise returns an error.
+// NotAuthenticated, NotAuthorized, or any underlying error.
+func CheckAll(
+	ctx context.Context, authorizer authz.Authorizer, session *auth.Session,
+	permissionChecks ...types.PermissionCheck,
+) error {
+	authorized, err := authorizer.CheckAll(
+		ctx,
+		session,
+		permissionChecks...,
+	)
 	if err != nil {
 		return err
 	}
@@ -62,9 +87,11 @@ func Check(ctx context.Context, authorizer authz.Authorizer, session *auth.Sessi
 // in the scope of a parent.
 // Returns nil if the permission is granted, otherwise returns an error.
 // NotAuthenticated, NotAuthorized, or any underlying error.
-func CheckChild(ctx context.Context, authorizer authz.Authorizer, session *auth.Session,
+func CheckChild(
+	ctx context.Context, authorizer authz.Authorizer, session *auth.Session,
 	spaceStore store.SpaceStore, repoStore store.RepoStore, parentType enum.ParentResourceType, parentID int64,
-	resourceType enum.ResourceType, resourceName string, permission enum.Permission) error {
+	resourceType enum.ResourceType, resourceName string, permission enum.Permission,
+) error {
 	scope, err := getScopeForParent(ctx, spaceStore, repoStore, parentType, parentID)
 	if err != nil {
 		return err
@@ -79,8 +106,10 @@ func CheckChild(ctx context.Context, authorizer authz.Authorizer, session *auth.
 }
 
 // getScopeForParent Returns the scope for a given resource parent (space or repo).
-func getScopeForParent(ctx context.Context, spaceStore store.SpaceStore, repoStore store.RepoStore,
-	parentType enum.ParentResourceType, parentID int64) (*types.Scope, error) {
+func getScopeForParent(
+	ctx context.Context, spaceStore store.SpaceStore, repoStore store.RepoStore,
+	parentType enum.ParentResourceType, parentID int64,
+) (*types.Scope, error) {
 	// TODO: Can this be done cleaner?
 	switch parentType {
 	case enum.ParentResourceTypeSpace: