Merge branch 'master' into docs-update

Makefile

@@ -108,6 +108,7 @@ shell: build-dirs
 		--net host                                              \
 		-v "$(PWD)/.go/pkg:/go/pkg"                             \
 		-v "$(PWD)/.go/cache:/go/.cache"                        \
+		-v "${HOME}/.kube:/root/.kube"                          \
 		-v "$(PWD):/go/src/$(PKG)"                              \
 		-v "$(PWD)/bin/$(ARCH):/go/bin"                         \
 		-v "$(PWD)/bin/$(ARCH):/go/bin/$$(go env GOOS)_$(ARCH)" \

go.mod

@@ -63,6 +63,7 @@ require (
 	go.uber.org/multierr v1.1.0 // indirect
 	go.uber.org/zap v1.10.0
 	golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 // indirect
+	golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 // indirect
 	golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421
 	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
 	google.golang.org/api v0.3.1

go.sum

@@ -67,6 +67,7 @@ github.com/aws/aws-sdk-go v1.20.12 h1:xV7xfLSkiqd7JOnLlfER+Jz8kI98rAGJvtXssYkCRs
 github.com/aws/aws-sdk-go v1.20.12/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.20.20 h1:OAR/GtjMOhenkp1NNKr1N1FgIP3mQXHeGbRhvVIAQp0=
 github.com/aws/aws-sdk-go v1.20.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aws/aws-sdk-go v1.23.4 h1:F6f/iQRhuSfrpUdy80q29898H0NYN27pX+95tkJ+BIY=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/census-instrumentation/opencensus-proto v0.2.0 h1:LzQXZOgg4CQfE6bFvXGM30YZL1WW/M337pXml+GrcZ4=
@@ -452,6 +453,8 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA=
+golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI=

pkg/blockstorage/awsebs/awsebs.go

@@ -9,7 +9,6 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -21,6 +20,7 @@ import (
 	"github.com/kanisterio/kanister/pkg/blockstorage"
 	ktags "github.com/kanisterio/kanister/pkg/blockstorage/tags"
 	"github.com/kanisterio/kanister/pkg/blockstorage/zone"
+	awsconfig "github.com/kanisterio/kanister/pkg/config/aws"
 )
 
 var _ blockstorage.Provider = (*ebsStorage)(nil)
@@ -38,14 +38,6 @@ type EC2 struct {
 
 const (
 	maxRetries = 10
-	// ConfigRegion represents region key required in the map "config"
-	ConfigRegion = "region"
-	// AccessKeyID represents AWS Access key ID
-	AccessKeyID = "AWS_ACCESS_KEY_ID"
-	// SecretAccessKey represents AWS Secret Access Key
-	SecretAccessKey = "AWS_SECRET_ACCESS_KEY"
-	// SessionToken represents AWS Session Key
-	SessionToken = "AWS_SESSION_TOKEN"
 )
 
 func (s *ebsStorage) Type() blockstorage.Type {
@@ -54,7 +46,7 @@ func (s *ebsStorage) Type() blockstorage.Type {
 
 // NewProvider returns a provider for the EBS storage type in the specified region
 func NewProvider(config map[string]string) (blockstorage.Provider, error) {
-	awsConfig, region, err := GetConfig(config)
+	awsConfig, region, _, err := awsconfig.GetConfig(config)
 	if err != nil {
 		return nil, err
 	}
@@ -65,24 +57,6 @@ func NewProvider(config map[string]string) (blockstorage.Provider, error) {
 	return &ebsStorage{ec2Cli: ec2Cli}, nil
 }
 
-// GetConfig returns a configuration to establish AWS connection and the connected region name.
-func GetConfig(config map[string]string) (*aws.Config, string, error) {
-	region, ok := config[ConfigRegion]
-	if !ok {
-		return nil, "", errors.New("region required for storage type EBS")
-	}
-	accessKey, ok := config[AccessKeyID]
-	if !ok {
-		return nil, "", errors.New("AWS_ACCESS_KEY_ID required for storage type EBS")
-	}
-	secretAccessKey, ok := config[SecretAccessKey]
-	if !ok {
-		return nil, "", errors.New("AWS_SECRET_ACCESS_KEY required for storage type EBS")
-	}
-	sessionToken := config[SessionToken]
-	return &aws.Config{Credentials: credentials.NewStaticCredentials(accessKey, secretAccessKey, sessionToken)}, region, nil
-}
-
 // newEC2Client returns ec2 client struct.
 func newEC2Client(awsRegion string, config *aws.Config) (*EC2, error) {
 	httpClient := &http.Client{Transport: http.DefaultTransport}

pkg/blockstorage/awsefs/awsefs.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/backup"
 	awsefs "github.com/aws/aws-sdk-go/service/efs"
@@ -15,8 +16,8 @@ import (
 	"k8s.io/apimachinery/pkg/util/rand"
 
 	"github.com/kanisterio/kanister/pkg/blockstorage"
-	"github.com/kanisterio/kanister/pkg/blockstorage/awsebs"
 	kantags "github.com/kanisterio/kanister/pkg/blockstorage/tags"
+	awsconfig "github.com/kanisterio/kanister/pkg/config/aws"
 )
 
 type efs struct {
@@ -44,7 +45,7 @@ const (
 
 // NewEFSProvider retuns a blockstorage provider for AWS EFS.
 func NewEFSProvider(config map[string]string) (blockstorage.Provider, error) {
-	awsConfig, region, err := awsebs.GetConfig(config)
+	awsConfig, region, role, err := awsconfig.GetConfig(config)
 	if err != nil {
 		return nil, errors.Wrap(err, "Failed to get configuration for EFS")
 	}
@@ -61,13 +62,18 @@ func NewEFSProvider(config map[string]string) (blockstorage.Provider, error) {
 		return nil, errors.New("Account ID is empty")
 	}
 	accountID := *user.Account
-	efsCli := awsefs.New(s, aws.NewConfig().WithRegion(region))
-	backupCli := backup.New(s, aws.NewConfig().WithRegion(region))
+	creds := awsConfig.Credentials
+	if role != "" {
+		creds = stscreds.NewCredentials(s, role)
+	}
+	efsCli := awsefs.New(s, aws.NewConfig().WithRegion(region).WithCredentials(creds))
+	backupCli := backup.New(s, aws.NewConfig().WithRegion(region).WithCredentials(creds))
 	return &efs{
 		EFS:       efsCli,
 		Backup:    backupCli,
 		region:    region,
-		accountID: accountID}, nil
+		accountID: accountID,
+	}, nil
 }
 
 func (e *efs) Type() blockstorage.Type {

pkg/blockstorage/blockstorage_test.go

@@ -10,9 +10,9 @@ import (
 	. "gopkg.in/check.v1"
 
 	"github.com/kanisterio/kanister/pkg/blockstorage"
-	"github.com/kanisterio/kanister/pkg/blockstorage/awsebs"
 	"github.com/kanisterio/kanister/pkg/blockstorage/getter"
 	ktags "github.com/kanisterio/kanister/pkg/blockstorage/tags"
+	awsconfig "github.com/kanisterio/kanister/pkg/config/aws"
 )
 
 const (
@@ -246,17 +246,17 @@ func (s *BlockStorageProviderSuite) getConfig(c *C, region string) map[string]st
 	config := make(map[string]string)
 	switch s.storageType {
 	case blockstorage.TypeEBS:
-		config[awsebs.ConfigRegion] = region
-		accessKey, ok := os.LookupEnv(awsebs.AccessKeyID)
+		config[awsconfig.ConfigRegion] = region
+		accessKey, ok := os.LookupEnv(awsconfig.AccessKeyID)
 		if !ok {
 			c.Skip("The necessary env variable AWS_ACCESS_KEY_ID is not set.")
 		}
-		secretAccessKey, ok := os.LookupEnv(awsebs.SecretAccessKey)
+		secretAccessKey, ok := os.LookupEnv(awsconfig.SecretAccessKey)
 		if !ok {
 			c.Skip("The necessary env variable AWS_SECRET_ACCESS_KEY is not set.")
 		}
-		config[awsebs.AccessKeyID] = accessKey
-		config[awsebs.SecretAccessKey] = secretAccessKey
+		config[awsconfig.AccessKeyID] = accessKey
+		config[awsconfig.SecretAccessKey] = secretAccessKey
 	case blockstorage.TypeGPD:
 		creds, ok := os.LookupEnv(blockstorage.GoogleCloudCreds)
 		if !ok {

pkg/chronicle/chronicle_push.go

@@ -22,7 +22,7 @@ import (
 
 type PushParams struct {
 	ProfilePath  string
-	ArtifactPath string
+	ArtifactFile string
 	Frequency    time.Duration
 	EnvDir       string
 	Command      []string
@@ -85,8 +85,9 @@ func push(ctx context.Context, p PushParams, ord int) error {
 			return err
 		}
 	}
+	ap, err := readArtifactPathFile(p.ArtifactFile)
 	log.Debugf("Pushing output from Command %d: %v. Environment: %v", ord, p.Command, env)
-	return pushWithEnv(ctx, p.Command, p.ArtifactPath, ord, prof, env)
+	return pushWithEnv(ctx, p.Command, ap, ord, prof, env)
 }
 
 func pushWithEnv(ctx context.Context, c []string, suffix string, ord int, prof param.Profile, env []string) error {
@@ -121,12 +122,17 @@ func pushWithEnv(ctx context.Context, c []string, suffix string, ord int, prof p
 	return nil
 }
 
+func readArtifactPathFile(path string) (string, error) {
+	buf, err := ioutil.ReadFile(path)
+	t := strings.TrimSuffix(string(buf), "\n")
+	return t, errors.Wrap(err, "Could not read artifact path file")
+}
+
 func readProfile(path string) (p param.Profile, ok bool, err error) {
 	var buf []byte
 	buf, err = ioutil.ReadFile(path)
 	switch {
 	case os.IsNotExist(err):
-		ok = true
 		err = nil
 		return
 	case err != nil:

pkg/chronicle/chronicle_push_test.go

@@ -2,6 +2,8 @@ package chronicle
 
 import (
 	"context"
+	"io/ioutil"
+	"os"
 	"path/filepath"
 
 	. "gopkg.in/check.v1"
@@ -28,9 +30,12 @@ func (s *ChroniclePushSuite) TestPush(c *C) {
 	err := writeProfile(pp, prof)
 	c.Assert(err, IsNil)
 
+	a := filepath.Join(c.MkDir(), "artifact")
+	err = ioutil.WriteFile(a, []byte(rand.String(10)), os.ModePerm)
+	c.Assert(err, IsNil)
 	p := PushParams{
 		ProfilePath:  pp,
-		ArtifactPath: rand.String(10),
+		ArtifactFile: a,
 		Command:      []string{"echo hello"},
 	}
 	ctx := context.Background()

pkg/chronicle/chronicle_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -42,9 +43,12 @@ func (s *ChronicleSuite) TestPushPull(c *C) {
 	err := writeProfile(pp, s.profile)
 	c.Assert(err, IsNil)
 
+	a := filepath.Join(c.MkDir(), "artifact")
+	err = ioutil.WriteFile(a, []byte(rand.String(10)), os.ModePerm)
+	c.Assert(err, IsNil)
 	p := PushParams{
 		ProfilePath:  pp,
-		ArtifactPath: rand.String(10),
+		ArtifactFile: a,
 	}
 	ctx := context.Background()
 
@@ -56,7 +60,7 @@ func (s *ChronicleSuite) TestPushPull(c *C) {
 
 		// Pull and check that we still get i
 		buf := bytes.NewBuffer(nil)
-		err = Pull(ctx, buf, s.profile, p.ArtifactPath)
+		err = Pull(ctx, buf, s.profile, p.ArtifactFile)
 		c.Assert(err, IsNil)
 		str, err := ioutil.ReadAll(buf)
 		c.Assert(err, IsNil)

pkg/config/aws/aws.go

@@ -0,0 +1,41 @@
+package aws
+
+import (
+	"errors"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+)
+
+const (
+	// ConfigRegion represents region key required in the map "config"
+	ConfigRegion = "region"
+	// ConfigRole represents the key for the ARN of the role which can be assumed.
+	// It is optional.
+	ConfigRole = "role"
+	// AccessKeyID represents AWS Access key ID
+	AccessKeyID = "AWS_ACCESS_KEY_ID"
+	// SecretAccessKey represents AWS Secret Access Key
+	SecretAccessKey = "AWS_SECRET_ACCESS_KEY"
+	// SessionToken represents AWS Session Key
+	SessionToken = "AWS_SESSION_TOKEN"
+)
+
+// GetConfig returns a configuration to establish AWS connection, connected region name and the role to assume if it exists.
+func GetConfig(config map[string]string) (awsConfig *aws.Config, region string, role string, err error) {
+	region, ok := config[ConfigRegion]
+	if !ok {
+		return nil, "", "", errors.New("region required for storage type EBS")
+	}
+	accessKey, ok := config[AccessKeyID]
+	if !ok {
+		return nil, "", "", errors.New("AWS_ACCESS_KEY_ID required for storage type EBS")
+	}
+	secretAccessKey, ok := config[SecretAccessKey]
+	if !ok {
+		return nil, "", "", errors.New("AWS_SECRET_ACCESS_KEY required for storage type EBS")
+	}
+	sessionToken := config[SessionToken]
+	role = config[ConfigRole]
+	return &aws.Config{Credentials: credentials.NewStaticCredentials(accessKey, secretAccessKey, sessionToken)}, region, role, nil
+}

pkg/function/create_volume_from_snapshot.go

@@ -11,8 +11,8 @@ import (
 
 	kanister "github.com/kanisterio/kanister/pkg"
 	"github.com/kanisterio/kanister/pkg/blockstorage"
-	"github.com/kanisterio/kanister/pkg/blockstorage/awsebs"
 	"github.com/kanisterio/kanister/pkg/blockstorage/getter"
+	awsconfig "github.com/kanisterio/kanister/pkg/config/aws"
 	"github.com/kanisterio/kanister/pkg/kube"
 	kubevolume "github.com/kanisterio/kanister/pkg/kube/volume"
 	"github.com/kanisterio/kanister/pkg/param"
@@ -60,9 +60,9 @@ func createVolumeFromSnapshot(ctx context.Context, cli kubernetes.Interface, nam
 		}
 		switch pvcInfo.Type {
 		case blockstorage.TypeEBS:
-			config[awsebs.ConfigRegion] = pvcInfo.Region
-			config[awsebs.AccessKeyID] = profile.Credential.KeyPair.ID
-			config[awsebs.SecretAccessKey] = profile.Credential.KeyPair.Secret
+			config[awsconfig.ConfigRegion] = pvcInfo.Region
+			config[awsconfig.AccessKeyID] = profile.Credential.KeyPair.ID
+			config[awsconfig.SecretAccessKey] = profile.Credential.KeyPair.Secret
 		case blockstorage.TypeGPD:
 			config[blockstorage.GoogleProjectID] = profile.Credential.KeyPair.ID
 			config[blockstorage.GoogleServiceKey] = profile.Credential.KeyPair.Secret

pkg/function/create_volume_snapshot.go

@@ -18,6 +18,7 @@ import (
 	"github.com/kanisterio/kanister/pkg/blockstorage"
 	"github.com/kanisterio/kanister/pkg/blockstorage/awsebs"
 	"github.com/kanisterio/kanister/pkg/blockstorage/getter"
+	awsconfig "github.com/kanisterio/kanister/pkg/config/aws"
 	"github.com/kanisterio/kanister/pkg/kube"
 	kubevolume "github.com/kanisterio/kanister/pkg/kube/volume"
 	"github.com/kanisterio/kanister/pkg/param"
@@ -203,9 +204,9 @@ func getPVCInfo(ctx context.Context, kubeCli kubernetes.Interface, namespace str
 			}
 		}
 		if pvZone, ok := pvLabels[kubevolume.PVZoneLabelName]; ok {
-			config[awsebs.ConfigRegion] = region
-			config[awsebs.AccessKeyID] = tp.Profile.Credential.KeyPair.ID
-			config[awsebs.SecretAccessKey] = tp.Profile.Credential.KeyPair.Secret
+			config[awsconfig.ConfigRegion] = region
+			config[awsconfig.AccessKeyID] = tp.Profile.Credential.KeyPair.ID
+			config[awsconfig.SecretAccessKey] = tp.Profile.Credential.KeyPair.Secret
 			provider, err = getter.Get(blockstorage.TypeEBS, config)
 			if err != nil {
 				return nil, errors.Wrap(err, "Could not get storage provider")

pkg/function/delete_volume_snapshot.go

@@ -11,8 +11,8 @@ import (
 
 	kanister "github.com/kanisterio/kanister/pkg"
 	"github.com/kanisterio/kanister/pkg/blockstorage"
-	"github.com/kanisterio/kanister/pkg/blockstorage/awsebs"
 	"github.com/kanisterio/kanister/pkg/blockstorage/getter"
+	awsconfig "github.com/kanisterio/kanister/pkg/config/aws"
 	"github.com/kanisterio/kanister/pkg/kube"
 	"github.com/kanisterio/kanister/pkg/param"
 )
@@ -52,9 +52,9 @@ func deleteVolumeSnapshot(ctx context.Context, cli kubernetes.Interface, namespa
 		}
 		switch pvcInfo.Type {
 		case blockstorage.TypeEBS:
-			config[awsebs.ConfigRegion] = pvcInfo.Region
-			config[awsebs.AccessKeyID] = profile.Credential.KeyPair.ID
-			config[awsebs.SecretAccessKey] = profile.Credential.KeyPair.Secret
+			config[awsconfig.ConfigRegion] = pvcInfo.Region
+			config[awsconfig.AccessKeyID] = profile.Credential.KeyPair.ID
+			config[awsconfig.SecretAccessKey] = profile.Credential.KeyPair.Secret
 		case blockstorage.TypeGPD:
 			config[blockstorage.GoogleProjectID] = profile.Credential.KeyPair.ID
 			config[blockstorage.GoogleServiceKey] = profile.Credential.KeyPair.Secret