.travis, build: autodelete old unstable archives (#13867)

This commit adds a build step to travis to auto-delete unstable archives older than
14 days (our regular release schedule) from Azure via ci.go purge.

The commit also pulls in the latest Azure storage code, also switching over from
the old import path (github.com/Azure/azure-sdk-for-go) to the new split one
(github.com/Azure/azure-storage-go).

.travis.yml

@@ -178,6 +178,16 @@ matrix:
 
         - go run build/ci.go xcode -signer IOS_SIGNING_KEY -deploy trunk -upload gethstore/builds
 
+    # This builder does the Azure archive purges to avoid accumulating junk
+    - os: linux
+      dist: trusty
+      sudo: required
+      go: 1.8
+      env:
+        - azure-purge
+      script:
+        - go run build/ci.go purge -store gethstore/builds -days 14
+
 install:
   - go get golang.org/x/tools/cmd/cover
 script:

build/ci.go

@@ -32,6 +32,7 @@ Available commands are:
    aar        [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ]                      -- creates an Android archive
    xcode      [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ]                      -- creates an iOS XCode framework
    xgo        [ -alltools ] [ options ]                                                        -- cross builds according to options
+   purge      [ -store blobstore ] [ -days threshold ]                                         -- purges old archives from the blobstore
 
 For all commands, -n prevents execution of external programs (dry run mode).
 
@@ -146,6 +147,8 @@ func main() {
 		doXCodeFramework(os.Args[2:])
 	case "xgo":
 		doXgo(os.Args[2:])
+	case "purge":
+		doPurge(os.Args[2:])
 	default:
 		log.Fatal("unknown command ", os.Args[1])
 	}
@@ -427,6 +430,10 @@ func maybeSkipArchive(env build.Environment) {
 		log.Printf("skipping because this is a PR build")
 		os.Exit(0)
 	}
+	if env.IsCronJob {
+		log.Printf("skipping because this is a cron job")
+		os.Exit(0)
+	}
 	if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") {
 		log.Printf("skipping because branch %q, tag %q is not on the whitelist", env.Branch, env.Tag)
 		os.Exit(0)
@@ -952,3 +959,62 @@ func xgoTool(args []string) *exec.Cmd {
 	}
 	return cmd
 }
+
+// Binary distribution cleanups
+
+func doPurge(cmdline []string) {
+	var (
+		store = flag.String("store", "", `Destination from where to purge archives (usually "gethstore/builds")`)
+		limit = flag.Int("days", 30, `Age threshold above which to delete unstalbe archives`)
+	)
+	flag.CommandLine.Parse(cmdline)
+
+	if env := build.Env(); !env.IsCronJob {
+		log.Printf("skipping because not a cron job")
+		os.Exit(0)
+	}
+	// Create the azure authentication and list the current archives
+	auth := build.AzureBlobstoreConfig{
+		Account:   strings.Split(*store, "/")[0],
+		Token:     os.Getenv("AZURE_BLOBSTORE_TOKEN"),
+		Container: strings.SplitN(*store, "/", 2)[1],
+	}
+	blobs, err := build.AzureBlobstoreList(auth)
+	if err != nil {
+		log.Fatal(err)
+	}
+	// Iterate over the blobs, collect and sort all unstable builds
+	for i := 0; i < len(blobs); i++ {
+		if !strings.Contains(blobs[i].Name, "unstable") {
+			blobs = append(blobs[:i], blobs[i+1:]...)
+			i--
+		}
+	}
+	for i := 0; i < len(blobs); i++ {
+		for j := i + 1; j < len(blobs); j++ {
+			iTime, err := time.Parse(time.RFC1123, blobs[i].Properties.LastModified)
+			if err != nil {
+				log.Fatal(err)
+			}
+			jTime, err := time.Parse(time.RFC1123, blobs[j].Properties.LastModified)
+			if err != nil {
+				log.Fatal(err)
+			}
+			if iTime.After(jTime) {
+				blobs[i], blobs[j] = blobs[j], blobs[i]
+			}
+		}
+	}
+	// Filter out all archives more recent that the given threshold
+	for i, blob := range blobs {
+		timestamp, _ := time.Parse(time.RFC1123, blob.Properties.LastModified)
+		if time.Since(timestamp) < time.Duration(*limit)*24*time.Hour {
+			blobs = blobs[:i]
+			break
+		}
+	}
+	// Delete all marked as such and return
+	if err := build.AzureBlobstoreDelete(auth, blobs); err != nil {
+		log.Fatal(err)
+	}
+}

internal/build/azure.go

@@ -20,7 +20,7 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/Azure/azure-sdk-for-go/storage"
+	storage "github.com/Azure/azure-storage-go"
 )
 
 // AzureBlobstoreConfig is an authentication and configuration struct containing
@@ -42,7 +42,6 @@ func AzureBlobstoreUpload(path string, name string, config AzureBlobstoreConfig)
 		fmt.Printf("would upload %q to %s/%s/%s\n", path, config.Account, config.Container, name)
 		return nil
 	}
-
 	// Create an authenticated client against the Azure cloud
 	rawClient, err := storage.NewBasicClient(config.Account, config.Token)
 	if err != nil {
@@ -63,3 +62,50 @@ func AzureBlobstoreUpload(path string, name string, config AzureBlobstoreConfig)
 	}
 	return client.CreateBlockBlobFromReader(config.Container, name, uint64(info.Size()), in, nil)
 }
+
+// AzureBlobstoreList lists all the files contained within an azure blobstore.
+func AzureBlobstoreList(config AzureBlobstoreConfig) ([]storage.Blob, error) {
+	// Create an authenticated client against the Azure cloud
+	rawClient, err := storage.NewBasicClient(config.Account, config.Token)
+	if err != nil {
+		return nil, err
+	}
+	client := rawClient.GetBlobService()
+
+	// List all the blobs from the container and return them
+	container := client.GetContainerReference(config.Container)
+
+	blobs, err := container.ListBlobs(storage.ListBlobsParameters{
+		MaxResults: 1024 * 1024 * 1024, // Yes, fetch all of them
+		Timeout:    3600,               // Yes, wait for all of them
+	})
+	if err != nil {
+		return nil, err
+	}
+	return blobs.Blobs, nil
+}
+
+// AzureBlobstoreDelete iterates over a list of files to delete and removes them
+// from the blobstore.
+func AzureBlobstoreDelete(config AzureBlobstoreConfig, blobs []storage.Blob) error {
+	if *DryRunFlag {
+		for _, blob := range blobs {
+			fmt.Printf("would delete %s (%s) from %s/%s\n", blob.Name, blob.Properties.LastModified, config.Account, config.Container)
+		}
+		return nil
+	}
+	// Create an authenticated client against the Azure cloud
+	rawClient, err := storage.NewBasicClient(config.Account, config.Token)
+	if err != nil {
+		return err
+	}
+	client := rawClient.GetBlobService()
+
+	// Iterate over the blobs and delete them
+	for _, blob := range blobs {
+		if err := client.DeleteBlob(config.Container, blob.Name, nil); err != nil {
+			return err
+		}
+	}
+	return nil
+}

internal/build/env.go

@@ -30,6 +30,7 @@ var (
 	GitTagFlag      = flag.String("git-tag", "", `Overrides git tag being built`)
 	BuildnumFlag    = flag.String("buildnum", "", `Overrides CI build number`)
 	PullRequestFlag = flag.Bool("pull-request", false, `Overrides pull request status of the build`)
+	CronJobFlag     = flag.Bool("cron-job", false, `Overrides cron job status of the build`)
 )
 
 // Environment contains metadata provided by the build environment.
@@ -39,6 +40,7 @@ type Environment struct {
 	Commit, Branch, Tag string // Git info
 	Buildnum            string
 	IsPullRequest       bool
+	IsCronJob           bool
 }
 
 func (env Environment) String() string {
@@ -59,6 +61,7 @@ func Env() Environment {
 			Tag:           os.Getenv("TRAVIS_TAG"),
 			Buildnum:      os.Getenv("TRAVIS_BUILD_NUMBER"),
 			IsPullRequest: os.Getenv("TRAVIS_PULL_REQUEST") != "false",
+			IsCronJob:     os.Getenv("TRAVIS_EVENT_TYPE") == "cron",
 		}
 	case os.Getenv("CI") == "True" && os.Getenv("APPVEYOR") == "True":
 		return Environment{
@@ -69,6 +72,7 @@ func Env() Environment {
 			Tag:           os.Getenv("APPVEYOR_REPO_TAG_NAME"),
 			Buildnum:      os.Getenv("APPVEYOR_BUILD_NUMBER"),
 			IsPullRequest: os.Getenv("APPVEYOR_PULL_REQUEST_NUMBER") != "",
+			IsCronJob:     os.Getenv("APPVEYOR_SCHEDULED_BUILD") == "True",
 		}
 	default:
 		return LocalEnv()
@@ -118,5 +122,8 @@ func applyEnvFlags(env Environment) Environment {
 	if *PullRequestFlag {
 		env.IsPullRequest = true
 	}
+	if *CronJobFlag {
+		env.IsCronJob = true
+	}
 	return env
 }