gh2gcs: Enable uploads to GCS

Signed-off-by: Stephen Augustus <saugustus@vmware.com>

cmd/gh2gcs/cmd/BUILD.bazel

@@ -6,9 +6,11 @@ go_library(
     importpath = "k8s.io/release/cmd/gh2gcs/cmd",
     visibility = ["//visibility:public"],
     deps = [
+        "//pkg/gcp:go_default_library",
         "//pkg/gh2gcs:go_default_library",
         "//pkg/github:go_default_library",
         "//pkg/log:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@com_github_spf13_cobra//:go_default_library",
     ],

cmd/gh2gcs/cmd/root.go

@@ -19,9 +19,11 @@ package cmd
 import (
 	"io/ioutil"
 
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
+	"k8s.io/release/pkg/gcp"
 	"k8s.io/release/pkg/gh2gcs"
 	"k8s.io/release/pkg/github"
 	"k8s.io/release/pkg/log"
@@ -136,6 +138,10 @@ func initLogging(*cobra.Command, []string) error {
 }
 
 func run(opts *options) error {
+	if err := gcp.PreCheck(); err != nil {
+		return errors.Wrap(err, "pre-checking for GCP package usage")
+	}
+
 	if opts.outputDir == "" {
 		tmpDir, err := ioutil.TempDir("", "gh2gcs")
 		if err != nil {
@@ -151,9 +157,11 @@ func run(opts *options) error {
 	// TODO: Support downloading releases via yaml config
 	uploadConfig := &gh2gcs.Config{}
 	releaseConfig := &gh2gcs.ReleaseConfig{
-		Org:  opts.org,
-		Name: opts.repo,
-		Tags: []string{},
+		Org:        opts.org,
+		Repo:       opts.repo,
+		Tags:       []string{},
+		GCSBucket:  opts.bucket,
+		ReleaseDir: opts.releaseDir,
 	}
 
 	if opts.tag != "" {
@@ -174,7 +182,7 @@ func run(opts *options) error {
 			return err
 		}
 
-		if err := gh2gcs.UploadToGCS(&rc, gh, opts.outputDir); err != nil {
+		if err := gh2gcs.Upload(&rc, gh, opts.outputDir); err != nil {
 			return err
 		}
 	}

pkg/gcp/BUILD.bazel

@@ -24,6 +24,7 @@ filegroup(
         ":package-srcs",
         "//pkg/gcp/auth:all-srcs",
         "//pkg/gcp/build:all-srcs",
+        "//pkg/gcp/gcs:all-srcs",
     ],
     tags = ["automanaged"],
     visibility = ["//visibility:public"],

pkg/gcp/gcs/BUILD.bazel

@@ -0,0 +1,27 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["gcs.go"],
+    importpath = "k8s.io/release/pkg/gcp/gcs",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/command:go_default_library",
+        "//pkg/gcp:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

pkg/gcp/gcs/gcs.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gcs
+
+import (
+	"strings"
+
+	"github.com/sirupsen/logrus"
+	"k8s.io/release/pkg/command"
+	"k8s.io/release/pkg/gcp"
+)
+
+var (
+	gcsPrefix      = "gs://"
+	concurrentFlag = "-m"
+	recursiveFlag  = "-r"
+)
+
+func CopyToGCS(src, gcsPath string, recursive, concurrent bool) error {
+	gcsPath = normalizeGCSPath(gcsPath)
+
+	logrus.Infof("Copying %s to GCS (%s)", src, gcsPath)
+	return bucketCopy(src, gcsPath, recursive, concurrent)
+}
+
+func CopyToLocal(gcsPath, dst string, recursive, concurrent bool) error {
+	gcsPath = normalizeGCSPath(gcsPath)
+
+	logrus.Infof("Copying GCS (%s) to %s", gcsPath, dst)
+	return bucketCopy(gcsPath, dst, recursive, concurrent)
+}
+
+func bucketCopy(src, dst string, recursive, concurrent bool) error {
+	args := []string{}
+
+	if concurrent {
+		args = append(args, concurrentFlag)
+	}
+
+	args = append(args, "cp")
+	if recursive {
+		args = append(args, recursiveFlag)
+	}
+	args = append(args, src, dst)
+
+	cpErr := command.Execute(gcp.GSUtilExecutable, args...)
+	if cpErr != nil {
+		return cpErr
+	}
+
+	return nil
+}
+
+func normalizeGCSPath(gcsPath string) string {
+	gcsPath = strings.TrimPrefix(gcsPath, gcsPrefix)
+	gcsPath = gcsPrefix + gcsPath
+
+	return gcsPath
+}

pkg/gh2gcs/BUILD.bazel

@@ -6,8 +6,8 @@ go_library(
     importpath = "k8s.io/release/pkg/gh2gcs",
     visibility = ["//visibility:public"],
     deps = [
+        "//pkg/gcp/gcs:go_default_library",
         "//pkg/github:go_default_library",
-        "@com_github_sirupsen_logrus//:go_default_library",
     ],
 )
 

pkg/gh2gcs/gh2gcs.go

@@ -17,8 +17,9 @@ limitations under the License.
 package gh2gcs
 
 import (
-	"github.com/sirupsen/logrus"
+	"path/filepath"
 
+	"k8s.io/release/pkg/gcp/gcs"
 	"k8s.io/release/pkg/github"
 )
 
@@ -28,23 +29,32 @@ type Config struct {
 
 type ReleaseConfig struct {
 	Org        string
-	Name       string
+	Repo       string
 	Tags       []string
 	GCSBucket  string
 	ReleaseDir string
 }
 
 func DownloadReleases(releaseCfg *ReleaseConfig, ghClient *github.GitHub, outputDir string) error {
 	tags := releaseCfg.Tags
-	if err := ghClient.DownloadReleaseAssets(releaseCfg.Org, releaseCfg.Name, tags, outputDir); err != nil {
+	if err := ghClient.DownloadReleaseAssets(releaseCfg.Org, releaseCfg.Repo, tags, outputDir); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-// TODO: Add GCS upload logic
-func UploadToGCS(releaseCfg *ReleaseConfig, ghClient *github.GitHub, outputDir string) error {
-	logrus.Info("Uploading to GCS...")
+func Upload(releaseCfg *ReleaseConfig, ghClient *github.GitHub, outputDir string) error {
+	uploadBase := filepath.Join(outputDir, releaseCfg.Org, releaseCfg.Repo)
+	gcsPath := filepath.Join(releaseCfg.GCSBucket, releaseCfg.ReleaseDir)
+
+	tags := releaseCfg.Tags
+	for _, tag := range tags {
+		srcDir := filepath.Join(uploadBase, tag)
+		if err := gcs.CopyToGCS(srcDir, gcsPath, true, true); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }