add huawei cloud registry support

go.mod

@@ -5,6 +5,8 @@ go 1.16
 require (
 	github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
 	github.com/andygrunwald/go-gerrit v0.0.0-20171029143327-95b11af228a1
+	github.com/bitly/go-simplejson v0.5.0 // indirect
+	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
 	github.com/bndr/gojenkins v1.1.0
 	github.com/bradleyfalzon/ghinstallation v1.1.1
 	github.com/bshuster-repo/logrus-logstash-hook v1.0.0 // indirect
@@ -33,6 +35,7 @@ require (
 	github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect
 	github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f
 	github.com/hashicorp/go-multierror v1.1.1
+	github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.0.50
 	github.com/jasonlvhit/gocron v0.0.0-20171226191223-3c914c8681c3
 	github.com/jinzhu/now v1.1.2
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect

go.sum

@@ -97,8 +97,12 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y=
+github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
+github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
 github.com/bndr/gojenkins v1.1.0 h1:TWyJI6ST1qDAfH33DQb3G4mD8KkrBfyfSUoZBHQAvPI=
 github.com/bndr/gojenkins v1.1.0/go.mod h1:QeskxN9F/Csz0XV/01IC8y37CapKKWvOHa0UHLLX1fM=
 github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff/go.mod h1:+RTT1BOk5P97fT2CiHkbFQwkK3mjsFAP6zCYV2aXtjw=
@@ -344,6 +348,7 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -480,6 +485,8 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huandu/xstrings v1.3.1 h1:4jgBlKK6tLKFvO8u5pmYjG91cqytmDCDvGh7ECVFfFs=
 github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
+github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.0.50 h1:nQ1QIkLxAoFsYyjvpQH3gVIy3mHfI0WDYEWdmwwxSG0=
+github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.0.50/go.mod h1:Pp3sd2tx3j9qC7Ij6jGh5phZwTrI+/HUBK90f3Cn2CI=
 github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=

pkg/microservice/aslan/config/consts.go

@@ -216,3 +216,7 @@ const (
 	LatestDay = 10
 	Date      = "2006-01-02"
 )
+
+const (
+	SWRProvider = "swr"
+)

pkg/microservice/aslan/core/common/repository/models/registry_namespace.go

@@ -23,19 +23,18 @@ import (
 )
 
 type RegistryNamespace struct {
-	ID               primitive.ObjectID `bson:"_id,omitempty"               json:"id,omitempty"`
-	OrgID            int                `bson:"org_id"                      json:"org_id"`
-	RegAddr          string             `bson:"reg_addr"                    json:"reg_addr"`
-	RegType          string             `bson:"reg_type"                    json:"reg_type"`
-	RegProvider      string             `bson:"reg_provider"                json:"reg_provider"`
-	IsDefault        bool               `bson:"is_default"                  json:"is_default"`
-	Namespace        string             `bson:"namespace"                   json:"namespace"`
-	AccessKey        string             `bson:"access_key"                  json:"access_key"`
-	SecretyKey       string             `bson:"secret_key"                  json:"secret_key"`
-	TencentSecretID  string             `bson:"tencent_secret_id"           json:"tencent_secret_id"`
-	TencentSecretKey string             `bson:"tencent_secret_key"          json:"tencent_secret_key"`
-	UpdateTime       int64              `bson:"update_time"                 json:"update_time"`
-	UpdateBy         string             `bson:"update_by"                   json:"update_by"`
+	ID          primitive.ObjectID `bson:"_id,omitempty"               json:"id,omitempty"`
+	OrgID       int                `bson:"org_id"                      json:"org_id"`
+	RegAddr     string             `bson:"reg_addr"                    json:"reg_addr"`
+	RegType     string             `bson:"reg_type"                    json:"reg_type"`
+	RegProvider string             `bson:"reg_provider"                json:"reg_provider"`
+	IsDefault   bool               `bson:"is_default"                  json:"is_default"`
+	Namespace   string             `bson:"namespace"                   json:"namespace"`
+	AccessKey   string             `bson:"access_key"                  json:"access_key"`
+	SecretKey   string             `bson:"secret_key"                  json:"secret_key"`
+	Region      string             `bson:"region,omitempty"            json:"region,omitempty"`
+	UpdateTime  int64              `bson:"update_time"                 json:"update_time"`
+	UpdateBy    string             `bson:"update_by"                   json:"update_by"`
 }
 
 func (ns *RegistryNamespace) Validate() error {

pkg/microservice/aslan/core/common/service/kube/actions.go

@@ -45,11 +45,12 @@ func CreateNamespace(namespace string, kubeClient client.Client) error {
 
 func CreateOrUpdateRegistrySecret(namespace string, reg *commonmodels.RegistryNamespace, kubeClient client.Client) error {
 	data := make(map[string][]byte)
+
 	dockerConfig := fmt.Sprintf(
 		`{"%s":{"username":"%s","password":"%s","email":"%s"}}`,
 		reg.RegAddr,
 		reg.AccessKey,
-		reg.SecretyKey,
+		reg.SecretKey,
 		"bot@koderover.com",
 	)
 	data[".dockercfg"] = []byte(dockerConfig)

pkg/microservice/aslan/core/common/service/registry.go

@@ -27,6 +27,7 @@ import (
 	"github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb"
 	"github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube"
 	e "github.com/koderover/zadig/pkg/tool/errors"
+	"github.com/koderover/zadig/pkg/util"
 )
 
 func FindDefaultRegistry(log *zap.SugaredLogger) (*models.RegistryNamespace, error) {
@@ -38,13 +39,22 @@ func FindDefaultRegistry(log *zap.SugaredLogger) (*models.RegistryNamespace, err
 	if err != nil {
 		log.Warnf("RegistryNamespace.Find error: %v", err)
 		resp = &models.RegistryNamespace{
-			RegAddr:    config.RegistryAddress(),
-			AccessKey:  config.RegistryAccessKey(),
-			SecretyKey: config.RegistrySecretKey(),
-			Namespace:  config.RegistryNamespace(),
+			RegAddr:   config.RegistryAddress(),
+			AccessKey: config.RegistryAccessKey(),
+			SecretKey: config.RegistrySecretKey(),
+			Namespace: config.RegistryNamespace(),
 		}
 	}
 
+	ak := resp.AccessKey
+	sk := resp.SecretKey
+	if resp.RegProvider == config.SWRProvider {
+		ak = fmt.Sprintf("%s@%s", resp.Region, resp.AccessKey)
+		sk = util.ComputeHmacSha256(resp.AccessKey, resp.SecretKey)
+	}
+	resp.AccessKey = ak
+	resp.SecretKey = sk
+
 	return resp, nil
 }
 

pkg/microservice/aslan/core/common/service/registry/service.go

@@ -19,6 +19,7 @@ package registry
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"net"
 	"net/http"
 	"net/url"
@@ -36,11 +37,15 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/registry"
 	"github.com/docker/go-connections/sockets"
+	"github.com/huaweicloud/huaweicloud-sdk-go-v3/core/auth/basic"
+	swr "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/swr/v2"
+	"github.com/huaweicloud/huaweicloud-sdk-go-v3/services/swr/v2/model"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
 	"go.uber.org/zap"
 	"golang.org/x/net/proxy"
 
+	"github.com/koderover/zadig/pkg/microservice/aslan/config"
 	commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models"
 	"github.com/koderover/zadig/pkg/tool/pool"
 )
@@ -49,6 +54,7 @@ type Endpoint struct {
 	Addr      string
 	Ak        string
 	Sk        string
+	Region    string
 	Namespace string
 }
 
@@ -68,8 +74,13 @@ type Service interface {
 	GetImageInfo(option GetRepoImageDetailOption, log *zap.SugaredLogger) (*commonmodels.DeliveryImage, error)
 }
 
-func NewV2Service() Service {
-	return &v2RegistryService{}
+func NewV2Service(provider string) Service {
+	switch provider {
+	case config.SWRProvider:
+		return &SwrService{}
+	default:
+		return &v2RegistryService{}
+	}
 }
 
 type v2RegistryService struct {
@@ -364,3 +375,115 @@ func (s *v2RegistryService) ListRepoImages(option ListRepoImagesOption, log *zap
 
 	return resp, nil
 }
+
+type SwrService struct {
+}
+
+func (s *SwrService) createClient(ep Endpoint) (cli *swr.SwrClient) {
+	endpoint := fmt.Sprintf("https://swr-api.%s.myhuaweicloud.com", ep.Region)
+	auth := basic.NewCredentialsBuilder().
+		WithAk(ep.Ak).
+		WithSk(ep.Sk).
+		Build()
+
+	client := swr.NewSwrClient(
+		swr.SwrClientBuilder().
+			WithEndpoint(endpoint).
+			WithCredential(auth).
+			Build())
+	return client
+}
+
+func (s *SwrService) ListRepoImages(option ListRepoImagesOption, log *zap.SugaredLogger) (resp *ReposResp, err error) {
+	swrCli := s.createClient(option.Endpoint)
+
+	var args []pool.TaskArg
+	for _, name := range option.Repos {
+		args = append(args, name)
+	}
+
+	resultChan := make(chan *Repo)
+	tasks := pool.MapTask(func(arg pool.TaskArg) func() error {
+		return func() error {
+			var err error
+			name := arg.(string)
+			defer func() {
+				if err != nil {
+					log.Errorf("failed to list tags of %s: %+v", name, err)
+				}
+			}()
+
+			request := &model.ListReposDetailsRequest{Name: &name, Namespace: &option.Namespace, ContentType: model.GetListReposDetailsRequestContentTypeEnum().APPLICATION_JSONCHARSETUTF_8}
+			repoDetails, err := swrCli.ListReposDetails(request)
+			if err != nil {
+				return err
+			}
+
+			koderoverTags := make([]string, 0)
+			customTags := make([]string, 0)
+			sortedTags := make([]string, 0)
+			for _, repoResp := range *repoDetails.Body {
+				for _, tag := range repoResp.Tags {
+					tagArray := strings.Split(tag, "-")
+					if len(tagArray) > 1 && len(tagArray[0]) == 14 {
+						if _, err := time.Parse("20060102150405", tagArray[0]); err == nil {
+							koderoverTags = append(koderoverTags, tag)
+							continue
+						}
+					}
+					customTags = append(customTags, tag)
+				}
+			}
+
+			sort.Sort(sort.Reverse(sort.StringSlice(koderoverTags)))
+			sortedTags = append(sortedTags, koderoverTags...)
+			sortedTags = append(sortedTags, customTags...)
+
+			resultChan <- &Repo{
+				Name:      name,
+				Namespace: option.Namespace,
+				Tags:      sortedTags,
+			}
+			return nil
+		}
+	}, args)
+
+	executor := pool.NewPool(tasks, 20)
+	go func() {
+		executor.Run()
+		close(resultChan)
+	}()
+
+	resp = &ReposResp{}
+
+	for result := range resultChan {
+		resp.Repos = append(resp.Repos, result)
+		resp.Total++
+	}
+
+	return resp, nil
+
+}
+
+func (s *SwrService) GetImageInfo(option GetRepoImageDetailOption, log *zap.SugaredLogger) (di *commonmodels.DeliveryImage, err error) {
+	swrCli := s.createClient(option.Endpoint)
+
+	request := &model.ListRepositoryTagsRequest{Tag: &option.Tag, Namespace: option.Namespace, Repository: option.Image}
+	repoTags, err := swrCli.ListRepositoryTags(request)
+	if err != nil {
+		err = errors.Wrapf(err, "failed to get image info of %s:%s", option.Image, option.Tag)
+		return
+	}
+
+	for _, repoTag := range *repoTags.Body {
+		return &commonmodels.DeliveryImage{
+			RepoName:     option.Image,
+			TagName:      option.Tag,
+			CreationTime: repoTag.Created,
+			ImageDigest:  repoTag.Digest,
+			ImageSize:    repoTag.Size,
+		}, nil
+	}
+
+	return &commonmodels.DeliveryImage{}, nil
+}

pkg/microservice/aslan/core/common/service/registry/service_test.go

@@ -21,6 +21,7 @@ import (
 	"sort"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/zap"
 
@@ -51,6 +52,7 @@ func Test_v2RegistryService_ListRepoImages(t *testing.T) {
 						"https://n7832lxy.mirror.aliyuncs.com",
 						"",
 						"",
+						"",
 						"library",
 					},
 					[]string{"mysql", "alpine"},
@@ -102,6 +104,7 @@ func Test_v2RegistryService_GetImageInfo(t *testing.T) {
 						"https://n7832lxy.mirror.aliyuncs.com",
 						"",
 						"",
+						"",
 						"library",
 					},
 					"mysql",
@@ -137,3 +140,33 @@ func TestReverseStringSlice_Len(t *testing.T) {
 		t.Error("reverse sort not works")
 	}
 }
+
+func TestSwrListRepoImage(t *testing.T) {
+	s := &SwrService{}
+	listRepoImagesOption := ListRepoImagesOption{
+		Endpoint: Endpoint{
+			Namespace: "lilian",
+			Ak:        "",
+			Sk:        "",
+		},
+		Repos: []string{"nginx-test"},
+	}
+	_, err := s.ListRepoImages(listRepoImagesOption, log.SugaredLogger())
+	assert.Nil(t, err)
+}
+
+func TestSwrImageInfo(t *testing.T) {
+	s := &SwrService{}
+	getRepoImageDetailOption := GetRepoImageDetailOption{
+		Endpoint: Endpoint{
+			Region:    "cn-north-4",
+			Namespace: "lilian",
+			Ak:        "",
+			Sk:        "",
+		},
+		Tag:   "20210712210942-34-master",
+		Image: "nginx-test",
+	}
+	_, err := s.GetImageInfo(getRepoImageDetailOption, log.SugaredLogger())
+	assert.Nil(t, err)
+}

pkg/microservice/aslan/core/system/handler/registry.go

@@ -52,12 +52,12 @@ func GetDefaultRegistryNamespace(c *gin.Context) {
 	}
 
 	ctx.Resp = &Registry{
-		ID:         reg.ID.Hex(),
-		RegAddr:    reg.RegAddr,
-		IsDefault:  reg.IsDefault,
-		Namespace:  reg.Namespace,
-		AccessKey:  reg.AccessKey,
-		SecretyKey: reg.SecretyKey,
+		ID:        reg.ID.Hex(),
+		RegAddr:   reg.RegAddr,
+		IsDefault: reg.IsDefault,
+		Namespace: reg.Namespace,
+		AccessKey: reg.AccessKey,
+		SecretKey: reg.SecretKey,
 	}
 }
 
@@ -93,8 +93,6 @@ func CreateRegistryNamespace(c *gin.Context) {
 		return
 	}
 
-	//args.ID = bson.NewObjectId()
-
 	ctx.Err = service.CreateRegistryNamespace(ctx.Username, args, ctx.Logger)
 }