Skip to content

Commit

Permalink
Simplify gcs catalog (gomods#1224)
Browse files Browse the repository at this point in the history
* simplify gcs catalog impl

* invert condition
  • Loading branch information
marpio authored and marwan-at-work committed May 14, 2019
1 parent 5e2b7e6 commit a75776b
Showing 1 changed file with 13 additions and 41 deletions.
54 changes: 13 additions & 41 deletions pkg/storage/gcp/cataloger.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,61 +19,33 @@ func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]pa
const op errors.Op = "gcp.Catalog"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
res := make([]paths.AllPathParams, 0)
var resToken string
count := pageSize

for count > 0 {
var catalog []string
var err error
catalog, resToken, err = nextPage(ctx, s.bucket, token, 3*count)
if err != nil {
return nil, "", errors.E(op, err)
}
pathsAndVers := fetchModsAndVersions(catalog)
res = append(res, pathsAndVers...)
count -= len(pathsAndVers)

if resToken == "" { // meaning we reached the end
break
}
}
return res, resToken, nil
}
res := make([]paths.AllPathParams, 0)

func nextPage(ctx context.Context, bkt *storage.BucketHandle, token string, pageSize int) ([]string, string, error) {
it := bkt.Objects(ctx, nil)
p := iterator.NewPager(it, pageSize, token)
it := s.bucket.Objects(ctx, nil)
// one module@version consists of 3 pieces - info, mod, zip
objCount := 3 * pageSize
p := iterator.NewPager(it, objCount, token)

attrs := make([]*storage.ObjectAttrs, 0)
nextToken, err := p.NextPage(&attrs)
if err != nil {
return nil, "", err
}

res := []string{}
for _, attr := range attrs {
res = append(res, attr.Name)
}
return res, nextToken, nil
}

func fetchModsAndVersions(catalog []string) []paths.AllPathParams {
res := make([]paths.AllPathParams, 0)
for _, p := range catalog {
if !strings.HasSuffix(p, ".info") {
continue
}
p, err := parseGcpKey(p)
if err != nil {
continue
if strings.HasSuffix(attr.Name, ".info") {
p, err := parsModVer(attr.Name)
if err != nil {
continue
}
res = append(res, p)
}
res = append(res, p)
}
return res
return res, nextToken, nil
}

func parseGcpKey(p string) (paths.AllPathParams, error) {
func parsModVer(p string) (paths.AllPathParams, error) {
const op errors.Op = "gcp.parseS3Key"
// github.com/gomods/testCatalogModule/@v/v1.2.0976.info
m, v := config.ModuleVersionFromPath(p)
Expand Down

0 comments on commit a75776b

Please sign in to comment.