Add api to delete blob and manifest

make/photon/prepare/templates/registryctl/config.yml.jinja

@@ -12,3 +12,4 @@ protocol: "http"
 port: 8080
 {% endif %}
 log_level: "INFO"
+registry_config: "/etc/registry/config.yml"

src/go.mod

@@ -2,9 +2,10 @@ module github.com/goharbor/harbor/src
 
 go 1.13
 
-replace github.com/goharbor/harbor => ../
-
 require (
+	github.com/Azure/azure-sdk-for-go v37.2.0+incompatible // indirect
+	github.com/Azure/go-autorest/autorest v0.9.3 // indirect
+	github.com/Azure/go-autorest/autorest/to v0.3.0 // indirect
 	github.com/Masterminds/semver v1.4.2
 	github.com/Unknwon/goconfig v0.0.0-20160216183935-5f601ca6ef4d // indirect
 	github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect
@@ -19,6 +20,7 @@ require (
 	github.com/cenkalti/backoff v2.1.1+incompatible // indirect
 	github.com/cloudflare/cfssl v0.0.0-20190510060611-9c027c93ba9e // indirect
 	github.com/coreos/go-oidc v2.1.0+incompatible
+	github.com/denverdino/aliyungo v0.0.0-20191227032621-df38c6fa730c // indirect
 	github.com/dghubble/sling v1.1.0
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/docker/distribution v2.7.1+incompatible
@@ -50,6 +52,7 @@ require (
 	github.com/lib/pq v1.3.0
 	github.com/mattn/go-runewidth v0.0.4 // indirect
 	github.com/miekg/pkcs11 v0.0.0-20170220202408-7283ca79f35e // indirect
+	github.com/ncw/swift v1.0.49 // indirect
 	github.com/olekukonko/tablewriter v0.0.1
 	github.com/opencontainers/go-digest v1.0.0-rc1
 	github.com/opencontainers/image-spec v1.0.1
@@ -76,3 +79,8 @@ require (
 	k8s.io/client-go v0.17.3
 	k8s.io/helm v2.16.3+incompatible
 )
+
+replace (
+	github.com/Azure/go-autorest => github.com/Azure/go-autorest v13.3.3+incompatible
+	github.com/goharbor/harbor => ../
+)

src/lib/errors/const.go

@@ -11,6 +11,8 @@ const (
 	BadRequestCode = "BAD_REQUEST"
 	// ForbiddenCode ...
 	ForbiddenCode = "FORBIDDEN"
+	// MethodNotAllowedCode ...
+	MethodNotAllowedCode = "METHOD_NOT_ALLOWED"
 	// PreconditionCode ...
 	PreconditionCode = "PRECONDITION"
 	// GeneralCode ...
@@ -59,6 +61,11 @@ func ForbiddenError(err error) *Error {
 	return New("forbidden").WithCode(ForbiddenCode).WithCause(err)
 }
 
+// MethodNotAllowedError is error for the case of forbidden
+func MethodNotAllowedError(err error) *Error {
+	return New("method not allowed").WithCode(MethodNotAllowedCode).WithCause(err)
+}
+
 // PreconditionFailedError is error for the case of precondition failed
 func PreconditionFailedError(err error) *Error {
 	return New("precondition failed").WithCode(PreconditionCode).WithCause(err)

src/registryctl/api/base.go

@@ -16,24 +16,36 @@ package api
 
 import (
 	"encoding/json"
+	"github.com/goharbor/harbor/src/lib/errors"
+	server_error "github.com/goharbor/harbor/src/server/error"
 	"net/http"
 )
 
-func handleInternalServerError(w http.ResponseWriter) {
-	http.Error(w, http.StatusText(http.StatusInternalServerError),
-		http.StatusInternalServerError)
+// HandleInternalServerError ...
+func HandleInternalServerError(w http.ResponseWriter, err error) {
+	HandleError(w, errors.UnknownError(err))
 }
 
-func handleUnauthorized(w http.ResponseWriter) {
-	http.Error(w, http.StatusText(http.StatusUnauthorized),
-		http.StatusUnauthorized)
+// HandleNotMethodAllowed ...
+func HandleNotMethodAllowed(w http.ResponseWriter) {
+	HandleError(w, errors.MethodNotAllowedError(nil))
 }
 
-// response status code will be written automatically if there is an error
-func writeJSON(w http.ResponseWriter, v interface{}) error {
+// HandleBadRequest ...
+func HandleBadRequest(w http.ResponseWriter, err error) {
+	HandleError(w, errors.BadRequestError(err))
+}
+
+// HandleError ...
+func HandleError(w http.ResponseWriter, err error) {
+	server_error.SendError(w, err)
+}
+
+// WriteJSON response status code will be written automatically if there is an error
+func WriteJSON(w http.ResponseWriter, v interface{}) error {
 	b, err := json.Marshal(v)
 	if err != nil {
-		handleInternalServerError(w)
+		HandleInternalServerError(w, err)
 		return err
 	}
 

src/registryctl/api/base_test.go

@@ -15,17 +15,36 @@
 package api
 
 import (
+	"github.com/goharbor/harbor/src/lib/errors"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 )
 
-func TestHandleInternalServerError(t *testing.T) {
+func TestHandleError(t *testing.T) {
 	w := httptest.NewRecorder()
-	handleInternalServerError(w)
+	HandleInternalServerError(w, errors.New("internal"))
 
 	if w.Code != http.StatusInternalServerError {
 		t.Errorf("unexpected status code: %d != %d", w.Code, http.StatusInternalServerError)
 	}
 
+	w = httptest.NewRecorder()
+	HandleBadRequest(w, errors.New("BadRequest"))
+	if w.Code != http.StatusBadRequest {
+		t.Errorf("unexpected status code: %d != %d", w.Code, http.StatusBadRequest)
+	}
+
+	w = httptest.NewRecorder()
+	HandleNotMethodAllowed(w)
+	if w.Code != http.StatusMethodNotAllowed {
+		t.Errorf("unexpected status code: %d != %d", w.Code, http.StatusMethodNotAllowed)
+	}
+
+	w = httptest.NewRecorder()
+	HandleError(w, errors.New("handle error"))
+	if w.Code != http.StatusInternalServerError {
+		t.Errorf("unexpected status code: %d != %d", w.Code, http.StatusInternalServerError)
+	}
+
 }

src/registryctl/api/health.go

@@ -22,7 +22,7 @@ import (
 
 // Health ...
 func Health(w http.ResponseWriter, r *http.Request) {
-	if err := writeJSON(w, "healthy"); err != nil {
+	if err := WriteJSON(w, "healthy"); err != nil {
 		log.Errorf("Failed to write response: %v", err)
 		return
 	}

src/registryctl/api/registry/blob/blob.go

@@ -0,0 +1,48 @@
+package blob
+
+import (
+	"errors"
+	"github.com/docker/distribution/registry/storage"
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/goharbor/harbor/src/registryctl/api"
+	"github.com/gorilla/mux"
+	"net/http"
+)
+
+// NewHandler returns the handler to handler blob request
+func NewHandler(storageDriver storagedriver.StorageDriver) http.Handler {
+	return &handler{
+		storageDriver: storageDriver,
+	}
+}
+
+type handler struct {
+	storageDriver storagedriver.StorageDriver
+}
+
+// ServeHTTP ...
+func (h *handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	switch req.Method {
+	case http.MethodDelete:
+		h.delete(w, req)
+	default:
+		api.HandleNotMethodAllowed(w)
+	}
+}
+
+// DeleteBlob ...
+func (h *handler) delete(w http.ResponseWriter, r *http.Request) {
+	ref := mux.Vars(r)["reference"]
+	if ref == "" {
+		api.HandleBadRequest(w, errors.New("no reference specified"))
+		return
+	}
+	// don't parse the reference here as RemoveBlob does.
+	cleaner := storage.NewVacuum(r.Context(), h.storageDriver)
+	if err := cleaner.RemoveBlob(ref); err != nil {
+		log.Infof("failed to remove blob: %s, with error:%v", ref, err)
+		api.HandleError(w, err)
+		return
+	}
+}

src/registryctl/api/registry/blob/blob_test.go

@@ -0,0 +1,64 @@
+package blob
+
+import (
+	"github.com/docker/distribution/registry/storage/driver/inmemory"
+	"github.com/docker/distribution/testutil"
+	"github.com/goharbor/harbor/src/registryctl/api/registry/test"
+	"github.com/gorilla/mux"
+	"github.com/stretchr/testify/assert"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestDeletionBlob(t *testing.T) {
+	inmemoryDriver := inmemory.New()
+
+	registry := test.CreateRegistry(t, inmemoryDriver)
+	repo := test.MakeRepository(t, registry, "blobdeletion")
+
+	// Create random layers
+	randomLayers1, err := testutil.CreateRandomLayers(3)
+	if err != nil {
+		t.Fatalf("failed to make layers: %v", err)
+	}
+
+	randomLayers2, err := testutil.CreateRandomLayers(3)
+	if err != nil {
+		t.Fatalf("failed to make layers: %v", err)
+	}
+
+	// Upload all layers
+	err = testutil.UploadBlobs(repo, randomLayers1)
+	if err != nil {
+		t.Fatalf("failed to upload layers: %v", err)
+	}
+
+	err = testutil.UploadBlobs(repo, randomLayers2)
+	if err != nil {
+		t.Fatalf("failed to upload layers: %v", err)
+	}
+
+	req, err := http.NewRequest(http.MethodDelete, "", nil)
+	varMap := make(map[string]string, 1)
+	varMap["reference"] = test.GetKeys(randomLayers1)[0].String()
+	req = mux.SetURLVars(req, varMap)
+
+	blobHandler := NewHandler(inmemoryDriver)
+	rec := httptest.NewRecorder()
+	blobHandler.ServeHTTP(rec, req)
+	assert.True(t, rec.Result().StatusCode == 200)
+
+	// layer1 is deleted and layer2 is still there
+	blobs := test.AllBlobs(t, registry)
+	for dgst := range randomLayers1 {
+		if _, ok := blobs[dgst]; !ok {
+			t.Logf("random layer 1 blob missing is correct as it has been deleted: %v", dgst)
+		}
+	}
+	for dgst := range randomLayers2 {
+		if _, ok := blobs[dgst]; !ok {
+			t.Fatalf("random layer 2 blob missing: %v", dgst)
+		}
+	}
+}

src/registryctl/api/registry/gc/gc.go

@@ -0,0 +1,62 @@
+package gc
+
+import (
+	"bytes"
+	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/goharbor/harbor/src/registryctl/api"
+	"net/http"
+	"os/exec"
+	"time"
+)
+
+// NewHandler returns the handler to handler blob request
+func NewHandler(registryConf string) http.Handler {
+	return &handler{
+		registryConf: registryConf,
+	}
+}
+
+type handler struct {
+	registryConf string
+}
+
+// ServeHTTP ...
+func (h *handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	switch req.Method {
+	case http.MethodPost:
+		h.start(w, req)
+	default:
+		api.HandleNotMethodAllowed(w)
+	}
+}
+
+// Result ...
+type Result struct {
+	Status    bool      `json:"status"`
+	Msg       string    `json:"msg"`
+	StartTime time.Time `json:"starttime"`
+	EndTime   time.Time `json:"endtime"`
+}
+
+// start ...
+func (h *handler) start(w http.ResponseWriter, r *http.Request) {
+	cmd := exec.Command("/bin/bash", "-c", "registry_DO_NOT_USE_GC garbage-collect --delete-untagged=false "+h.registryConf)
+	var outBuf, errBuf bytes.Buffer
+	cmd.Stdout = &outBuf
+	cmd.Stderr = &errBuf
+
+	start := time.Now()
+	log.Debugf("Start to execute garbage collection...")
+	if err := cmd.Run(); err != nil {
+		log.Errorf("Fail to execute GC: %v, command err: %s", err, errBuf.String())
+		api.HandleInternalServerError(w, err)
+		return
+	}
+
+	gcr := Result{true, outBuf.String(), start, time.Now()}
+	if err := api.WriteJSON(w, gcr); err != nil {
+		log.Errorf("failed to write response: %v", err)
+		return
+	}
+	log.Debugf("Successful to execute garbage collection...")
+}