Azure blob storage support

Adds a Detector and getter for Azure Blob Storage with dependencies on:
github.com/Azure/go-autorest/autorest/azure
github.com/Azure/azure-storage-go

An access key is required for the SDK Client, public blobs should be able to
make use of the http Getter anyway.

Author: pmcatominey

README.md

@@ -72,6 +72,7 @@ can be augmented at runtime by implementing the `Getter` interface.
   * Mercurial
   * HTTP
   * Amazon S3
+  * Azure Blob Storage
 
 In addition to the above protocols, go-getter has what are called "detectors."
 These take a URL and attempt to automatically choose the best protocol for
@@ -251,3 +252,7 @@ Some examples for these addressing schemes:
 - bucket.s3.amazonaws.com/foo
 - bucket.s3-eu-west-1.amazonaws.com/foo/bar
 
+### Azure Blob Storage (`azureblob`)
+
+Azure Blob Storage requires a valid access key for the storage account, this can be provided by the
+ `ARM_ACCESS_KEY` environment variable, or the `access_key` query value which takes priority.

detect.go

@@ -25,6 +25,7 @@ func init() {
 		new(GitHubDetector),
 		new(BitBucketDetector),
 		new(S3Detector),
+		new(AzureBlobDetector),
 		new(FileDetector),
 	}
 }

detect_azure_blob.go

@@ -0,0 +1,51 @@
+package getter
+
+import (
+	"net/url"
+	"strings"
+
+	"fmt"
+
+	"github.com/Azure/go-autorest/autorest/azure"
+)
+
+var azureStorageSuffixes = map[string]azure.Environment{
+	azure.PublicCloud.StorageEndpointSuffix:       azure.PublicCloud,
+	azure.GermanCloud.StorageEndpointSuffix:       azure.GermanCloud,
+	azure.USGovernmentCloud.StorageEndpointSuffix: azure.USGovernmentCloud,
+	azure.ChinaCloud.StorageEndpointSuffix:        azure.GermanCloud,
+}
+
+// AzureBlobDetector implements Detector to detect Azure URLs and turn
+// them into URLs that the Azure getter can understand.
+type AzureBlobDetector struct{}
+
+func (d *AzureBlobDetector) Detect(src, _ string) (string, bool, error) {
+	if len(src) == 0 {
+		return "", false, nil
+	}
+
+	for s := range azureStorageSuffixes {
+		if strings.Contains(src, s) {
+			return d.detectURL(src)
+		}
+	}
+
+	return "", false, nil
+}
+
+func (d *AzureBlobDetector) detectURL(src string) (string, bool, error) {
+	u, err := url.Parse(src)
+	if err != nil {
+		return "", false, err
+	}
+
+	parts := strings.Split(u.Path, "/")
+	if len(parts) < 2 {
+		return "", false, fmt.Errorf("path to blob must not be empty")
+	}
+
+	u.Scheme = "https"
+
+	return fmt.Sprintf("azureblob::%s", u.String()), true, nil
+}

detect_azure_blob_test.go

@@ -0,0 +1,50 @@
+package getter
+
+import (
+	"testing"
+)
+
+func TestAzureBlobDetector(t *testing.T) {
+	cases := []struct {
+		Input  string
+		Output string
+	}{
+		{
+			"account.blob.core.windows.net/foo/bar",
+			"azureblob::https://account.blob.core.windows.net/foo/bar",
+		},
+		{
+			"account.blob.core.usgovcloudapi.net/foo/bar",
+			"azureblob::https://account.blob.core.usgovcloudapi.net/foo/bar",
+		},
+		{
+			"account.blob.core.chinacloudapi.cn/foo/bar",
+			"azureblob::https://account.blob.core.chinacloudapi.cn/foo/bar",
+		},
+		{
+			"account.blob.core.cloudapi.de/foo/bar",
+			"azureblob::https://account.blob.core.cloudapi.de/foo/bar",
+		},
+		// Misc tests
+		{
+			"account.blob.core.windows.net/foo/bar?version=1234",
+			"azureblob::https://account.blob.core.windows.net/foo/bar?version=1234",
+		},
+	}
+
+	pwd := "/pwd"
+	f := new(AzureBlobDetector)
+	for i, tc := range cases {
+		output, ok, err := f.Detect(tc.Input, pwd)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		if !ok {
+			t.Fatal("not ok")
+		}
+
+		if output != tc.Output {
+			t.Fatalf("%d: bad: %#v", i, output)
+		}
+	}
+}

get.go

@@ -53,12 +53,13 @@ func init() {
 	httpGetter := &HttpGetter{Netrc: true}
 
 	Getters = map[string]Getter{
-		"file":  new(FileGetter),
-		"git":   new(GitGetter),
-		"hg":    new(HgGetter),
-		"s3":    new(S3Getter),
-		"http":  httpGetter,
-		"https": httpGetter,
+		"file":      new(FileGetter),
+		"git":       new(GitGetter),
+		"hg":        new(HgGetter),
+		"s3":        new(S3Getter),
+		"azureblob": new(AzureBlobGetter),
+		"http":      httpGetter,
+		"https":     httpGetter,
 	}
 }
 

get_azure_blob.go

@@ -0,0 +1,210 @@
+package getter
+
+import (
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+
+	storage "github.com/Azure/azure-storage-go"
+)
+
+// AzureBlobGetter is a Getter implementation that will download a module from
+// an Azure Blob Storage Account.
+type AzureBlobGetter struct{}
+
+func (g *AzureBlobGetter) ClientMode(u *url.URL) (ClientMode, error) {
+	// Parse URL
+	accountName, baseURL, containerName, blobPath, accessKey, err := g.parseUrl(u)
+	if err != nil {
+		return 0, err
+	}
+
+	client, err := g.getBobClient(accountName, baseURL, accessKey)
+	if err != nil {
+		return 0, err
+	}
+
+	container := client.GetContainerReference(containerName)
+
+	// List the object(s) at the given prefix
+	params := storage.ListBlobsParameters{
+		Prefix: blobPath,
+	}
+	resp, err := container.ListBlobs(params)
+	if err != nil {
+		return 0, err
+	}
+
+	for _, b := range resp.Blobs {
+		// Use file mode on exact match.
+		if b.Name == blobPath {
+			return ClientModeFile, nil
+		}
+
+		// Use dir mode if child keys are found.
+		if strings.HasPrefix(b.Name, blobPath+"/") {
+			return ClientModeDir, nil
+		}
+	}
+
+	// There was no match, so just return file mode. The download is going
+	// to fail but we will let Azure return the proper error later.
+	return ClientModeFile, nil
+}
+
+func (g *AzureBlobGetter) Get(dst string, u *url.URL) error {
+	// Parse URL
+	accountName, baseURL, containerName, blobPath, accessKey, err := g.parseUrl(u)
+	if err != nil {
+		return err
+	}
+
+	// Remove destination if it already exists
+	_, err = os.Stat(dst)
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+
+	if err == nil {
+		// Remove the destination
+		if err := os.RemoveAll(dst); err != nil {
+			return err
+		}
+	}
+
+	// Create all the parent directories
+	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
+		return err
+	}
+
+	client, err := g.getBobClient(accountName, baseURL, accessKey)
+	if err != nil {
+		return err
+	}
+
+	container := client.GetContainerReference(containerName)
+
+	// List files in path, keep listing until no more objects are found
+	lastMarker := ""
+	hasMore := true
+	for hasMore {
+		params := storage.ListBlobsParameters{
+			Prefix: blobPath,
+		}
+		if lastMarker != "" {
+			params.Marker = lastMarker
+		}
+
+		resp, err := container.ListBlobs(params)
+		if err != nil {
+			return err
+		}
+
+		hasMore = resp.NextMarker != ""
+		lastMarker = resp.NextMarker
+
+		// Get each object storing each file relative to the destination path
+		for _, object := range resp.Blobs {
+			objPath := object.Name
+
+			// If the key ends with a backslash assume it is a directory and ignore
+			if strings.HasSuffix(objPath, "/") {
+				continue
+			}
+
+			// Get the object destination path
+			objDst, err := filepath.Rel(blobPath, objPath)
+			if err != nil {
+				return err
+			}
+			objDst = filepath.Join(dst, objDst)
+
+			if err := g.getObject(client, objDst, containerName, objPath); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (g *AzureBlobGetter) GetFile(dst string, u *url.URL) error {
+	accountName, baseURL, containerName, blobPath, accessKey, err := g.parseUrl(u)
+	if err != nil {
+		return err
+	}
+
+	client, err := g.getBobClient(accountName, baseURL, accessKey)
+	if err != nil {
+		return err
+	}
+
+	return g.getObject(client, dst, containerName, blobPath)
+}
+
+func (g *AzureBlobGetter) getObject(client storage.BlobStorageClient, dst, container, blobName string) error {
+	r, err := client.GetBlob(container, blobName)
+	if err != nil {
+		return err
+	}
+
+	// Create all the parent directories
+	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
+		return err
+	}
+
+	f, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	_, err = io.Copy(f, r)
+	return err
+}
+
+func (g *AzureBlobGetter) getBobClient(accountName string, baseURL string, accountKey string) (storage.BlobStorageClient, error) {
+	var b storage.BlobStorageClient
+
+	if accountKey == "" {
+		accountKey = os.Getenv("ARM_ACCESS_KEY")
+	}
+
+	c, err := storage.NewClient(accountName, accountKey, baseURL, storage.DefaultAPIVersion, true)
+	if err != nil {
+		return b, err
+	}
+
+	b = c.GetBlobService()
+
+	return b, nil
+}
+
+func (g *AzureBlobGetter) parseUrl(u *url.URL) (accountName, baseURL, container, blobPath, accessKey string, err error) {
+	// Expected host style: accountname.blob.core.windows.net.
+	// The last 3 parts will be different across environments.
+	hostParts := strings.SplitN(u.Host, ".", 3)
+	if len(hostParts) != 3 {
+		err = fmt.Errorf("URL is not a valid Azure Blob URL")
+		return
+	}
+
+	accountName = hostParts[0]
+	baseURL = hostParts[2]
+
+	pathParts := strings.SplitN(strings.TrimPrefix(u.Path, "/"), "/", 2)
+	if len(pathParts) != 2 {
+		err = fmt.Errorf("URL is not a valid Azure Blob URL")
+		return
+	}
+
+	container = pathParts[0]
+	blobPath = pathParts[1]
+
+	accessKey = u.Query().Get("access_key")
+
+	return
+}