gh2gcs: Improve tag retrieval logic

Signed-off-by: Stephen Augustus <saugustus@vmware.com>

BUILD.bazel

@@ -37,6 +37,7 @@ filegroup(
         "//lib:all-srcs",
         "//pkg/command:all-srcs",
         "//pkg/gcp:all-srcs",
+        "//pkg/gh2gcs:all-srcs",
         "//pkg/git:all-srcs",
         "//pkg/github:all-srcs",
         "//pkg/http:all-srcs",

cmd/gh2gcs/cmd/BUILD.bazel

@@ -6,6 +6,7 @@ go_library(
     importpath = "k8s.io/release/cmd/gh2gcs/cmd",
     visibility = ["//visibility:public"],
     deps = [
+        "//pkg/gh2gcs:go_default_library",
         "//pkg/github:go_default_library",
         "//pkg/log:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",

cmd/gh2gcs/cmd/root.go

@@ -22,6 +22,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
+	"k8s.io/release/pkg/gh2gcs"
 	"k8s.io/release/pkg/github"
 	"k8s.io/release/pkg/log"
 )
@@ -40,12 +41,14 @@ var rootCmd = &cobra.Command{
 }
 
 type options struct {
-	org         string
-	repo        string
-	tag         string
-	gcsEndpoint string
-	outputDir   string
-	logLevel    string
+	org                string
+	repo               string
+	tag                string
+	includePrereleases bool
+	bucket             string
+	releaseDir         string
+	outputDir          string
+	logLevel           string
 }
 
 var opts = &options{}
@@ -64,6 +67,7 @@ func init() {
 		"org",
 		// TODO: Remove test value
 		"containernetworking",
+		// TODO: Improve usage text
 		"org",
 	)
 
@@ -72,30 +76,51 @@ func init() {
 		"repo",
 		// TODO: Remove test value
 		"plugins",
+		// TODO: Improve usage text
 		"repo",
 	)
 
+	// TODO: This should be a string array to accept multiple tags
 	rootCmd.PersistentFlags().StringVar(
 		&opts.tag,
 		"tag",
-		// TODO: Remove test value
 		"",
+		// TODO: Improve usage text
 		"tag",
 	)
 
+	rootCmd.PersistentFlags().BoolVar(
+		&opts.includePrereleases,
+		"include-prereleases",
+		false,
+		// TODO: Improve usage text
+		"include-prereleases",
+	)
+
 	rootCmd.PersistentFlags().StringVar(
-		&opts.gcsEndpoint,
-		"gcs-endpoint",
+		&opts.bucket,
+		"bucket",
 		// TODO: Remove test value
-		"k8s-staging-release-test/augustus/cni",
-		"org",
+		"k8s-staging-release-test",
+		// TODO: Improve usage text
+		"bucket",
+	)
+
+	rootCmd.PersistentFlags().StringVar(
+		&opts.releaseDir,
+		"release-dir",
+		// TODO: Remove test value
+		"augustus/release",
+		// TODO: Improve usage text
+		"release-dir",
 	)
 
 	rootCmd.PersistentFlags().StringVar(
 		&opts.outputDir,
 		"output-dir",
 		"",
-		"org",
+		// TODO: Improve usage text
+		"output-dir",
 	)
 
 	rootCmd.PersistentFlags().StringVar(
@@ -124,12 +149,35 @@ func run(opts *options) error {
 	gh := github.New()
 
 	// TODO: Support downloading releases via yaml config
-	// TODO: Support single release or multi-release scenarios
-	if err := gh.DownloadReleaseAssets(opts.org, opts.repo, opts.tag, opts.outputDir); err != nil {
-		return err
+	uploadConfig := &gh2gcs.Config{}
+	releaseConfig := &gh2gcs.ReleaseConfig{
+		Org:  opts.org,
+		Name: opts.repo,
+		Tags: []string{},
 	}
 
-	// TODO: Add GCS upload logic
+	if opts.tag != "" {
+		releaseConfig.Tags = append(releaseConfig.Tags, opts.tag)
+	} else {
+		releaseTags, err := gh.GetReleaseTags(opts.org, opts.repo, opts.includePrereleases)
+		if err != nil {
+			return err
+		}
+
+		releaseConfig.Tags = releaseTags
+	}
+
+	uploadConfig.ReleaseConfigs = append(uploadConfig.ReleaseConfigs, *releaseConfig)
+
+	for _, rc := range uploadConfig.ReleaseConfigs {
+		if err := gh2gcs.DownloadReleases(&rc, gh, opts.outputDir); err != nil {
+			return err
+		}
+
+		if err := gh2gcs.UploadToGCS(&rc, gh, opts.outputDir); err != nil {
+			return err
+		}
+	}
 
 	return nil
 }

pkg/gh2gcs/BUILD.bazel

@@ -0,0 +1,26 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["gh2gcs.go"],
+    importpath = "k8s.io/release/pkg/gh2gcs",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/github:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

pkg/gh2gcs/gh2gcs.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gh2gcs
+
+import (
+	"github.com/sirupsen/logrus"
+
+	"k8s.io/release/pkg/github"
+)
+
+type Config struct {
+	ReleaseConfigs []ReleaseConfig
+}
+
+type ReleaseConfig struct {
+	Org        string
+	Name       string
+	Tags       []string
+	GCSBucket  string
+	ReleaseDir string
+}
+
+func DownloadReleases(releaseCfg *ReleaseConfig, ghClient *github.GitHub, outputDir string) error {
+	tags := releaseCfg.Tags
+	if err := ghClient.DownloadReleaseAssets(releaseCfg.Org, releaseCfg.Name, tags, outputDir); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// TODO: Add GCS upload logic
+func UploadToGCS(releaseCfg *ReleaseConfig, ghClient *github.GitHub, outputDir string) error {
+	logrus.Info("Uploading to GCS...")
+	return nil
+}

pkg/github/github.go

@@ -382,30 +382,44 @@ func (g *GitHub) Releases(owner, repo string, includePrereleases bool) ([]*githu
 	return releases, nil
 }
 
-// TODO: Support multiple tags
-func (g *GitHub) DownloadReleaseAssets(owner, repo, tag, outputDir string) error {
+// GetReleaseTags returns a list of GitHub release tags for the provided
+// `owner` and `repo`. If `includePrereleases` is `true`, then the resulting
+// slice will also contain pre/drafted releases.
+func (g *GitHub) GetReleaseTags(owner, repo string, includePrereleases bool) ([]string, error) {
+	releases, err := g.Releases(owner, repo, includePrereleases)
+	if err != nil {
+		return nil, err
+	}
+
+	releaseTags := []string{}
+	for _, release := range releases {
+		releaseTags = append(releaseTags, *release.TagName)
+	}
+
+	return releaseTags, nil
+}
+
+func (g *GitHub) DownloadReleaseAssets(owner, repo string, releaseTags []string, outputDir string) error {
 	var releases []*github.RepositoryRelease
-	var err error
 
-	if tag != "" {
-		release, _, err := g.client.GetReleaseByTag(context.Background(), owner, repo, tag)
-		if err != nil {
-			return err
-		}
+	if len(releaseTags) > 0 {
+		for _, tag := range releaseTags {
+			release, _, err := g.client.GetReleaseByTag(context.Background(), owner, repo, tag)
+			if err != nil {
+				return err
+			}
 
-		releases = append(releases, release)
-	} else {
-		releases, err = g.Releases(owner, repo, false)
-		if err != nil {
-			return err
+			releases = append(releases, release)
 		}
+	} else {
+		return errors.New("no release tags were populated")
 	}
 
 	for _, release := range releases {
 		releaseTag := release.GetTagName()
 		logrus.Infof("Download assets for %s/%s@%s", owner, repo, releaseTag)
 
-		releaseDir := filepath.Join(outputDir, releaseTag)
+		releaseDir := filepath.Join(outputDir, owner, repo, releaseTag)
 		if err := os.MkdirAll(releaseDir, os.FileMode(0777)); err != nil {
 			return err
 		}