Skip to content

Commit

Permalink
Move aws helper functions to seperate pkg/aws (kanisterio#468)
Browse files Browse the repository at this point in the history
* Move aws helper functions to seperate pkg/aws

Signed-off-by: Prasad Ghangal <prasad.ghangal@gmail.com>

* Merge pkg/config/aws into pkg/aws

Signed-off-by: Prasad Ghangal <prasad.ghangal@gmail.com>

* Update aws-sdk dep to the latest version

Signed-off-by: Prasad Ghangal <prasad.ghangal@gmail.com>

* Update objectstore_test.go

* Update objectstore_test.go

* Update aws.go

* Update objectstore_test.go

* Update aws.go

Co-authored-by: Vaibhav Kamra <vkamra@kasten.io>
  • Loading branch information
2 people authored and mergify[bot] committed Dec 27, 2019
1 parent fc5fc65 commit 551a661
Show file tree
Hide file tree
Showing 21 changed files with 249 additions and 206 deletions.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ require (
github.com/Masterminds/sprig v2.15.0+incompatible
github.com/aokoli/goutils v1.1.0 // indirect
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
github.com/aws/aws-sdk-go v1.20.12
github.com/aws/aws-sdk-go v1.26.8
github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 // indirect
github.com/dnaeon/go-vcr v1.0.1 // indirect
github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect
Expand Down
3 changes: 3 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/aws/aws-sdk-go v1.20.12 h1:xV7xfLSkiqd7JOnLlfER+Jz8kI98rAGJvtXssYkCRs4=
github.com/aws/aws-sdk-go v1.20.12/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.26.8 h1:W+MPuCFLSO/itZkZ5GFOui0YC1j3lZ507/m5DFPtzE4=
github.com/aws/aws-sdk-go v1.26.8/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/census-instrumentation/opencensus-proto v0.2.0 h1:LzQXZOgg4CQfE6bFvXGM30YZL1WW/M337pXml+GrcZ4=
github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
Expand Down Expand Up @@ -152,6 +154,7 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kubernetes-csi/external-snapshotter v1.1.0 h1:godlw8BSOac5TMGH2rVPJrmllek3y8wuqd9JsJHgulw=
github.com/kubernetes-csi/external-snapshotter v1.1.0/go.mod h1:oYfxnsuh48V1UDYORl77YQxQbbdokNy7D73phuFpksY=
github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/luci/go-render v0.0.0-20160219211803-9a04cc21af0f/go.mod h1:aS446i8akEg0DAtNKTVYpNpLPMc0SzsZ0RtGhjl0uFM=
github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
Expand Down
63 changes: 42 additions & 21 deletions pkg/app/rds_postgres.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"os"
"time"

awssdk "github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
awsrds "github.com/aws/aws-sdk-go/service/rds"
"github.com/pkg/errors"
Expand All @@ -29,11 +30,12 @@ import (
"k8s.io/client-go/kubernetes"

crv1alpha1 "github.com/kanisterio/kanister/pkg/apis/cr/v1alpha1"
awsconfig "github.com/kanisterio/kanister/pkg/config/aws"
aws "github.com/kanisterio/kanister/pkg/aws"
"github.com/kanisterio/kanister/pkg/aws/ec2"
"github.com/kanisterio/kanister/pkg/aws/rds"
"github.com/kanisterio/kanister/pkg/field"
"github.com/kanisterio/kanister/pkg/kube"
"github.com/kanisterio/kanister/pkg/log"
"github.com/kanisterio/kanister/pkg/testutil"

// Initialize pq driver
_ "github.com/lib/pq"
Expand Down Expand Up @@ -78,24 +80,24 @@ func (pdb *RDSPostgresDB) Init(ctx context.Context) error {
if err != nil {
return err
}
pdb.region, ok = os.LookupEnv(awsconfig.Region)
pdb.region, ok = os.LookupEnv(aws.Region)
if !ok {
return fmt.Errorf("Env var %s is not set", awsconfig.Region)
return fmt.Errorf("Env var %s is not set", aws.Region)
}

// If sessionToken is set, accessID and secretKey not required
pdb.sessionToken, ok = os.LookupEnv(awsconfig.SessionToken)
pdb.sessionToken, ok = os.LookupEnv(aws.SessionToken)
if ok {
return nil
}

pdb.accessID, ok = os.LookupEnv(awsconfig.AccessKeyID)
pdb.accessID, ok = os.LookupEnv(aws.AccessKeyID)
if !ok {
return fmt.Errorf("Env var %s is not set", awsconfig.AccessKeyID)
return fmt.Errorf("Env var %s is not set", aws.AccessKeyID)
}
pdb.secretKey, ok = os.LookupEnv(awsconfig.SecretAccessKey)
pdb.secretKey, ok = os.LookupEnv(aws.SecretAccessKey)
if !ok {
return fmt.Errorf("Env var %s is not set", awsconfig.SecretAccessKey)
return fmt.Errorf("Env var %s is not set", aws.SecretAccessKey)
}
return nil
}
Expand All @@ -104,49 +106,54 @@ func (pdb *RDSPostgresDB) Install(ctx context.Context, ns string) error {
var err error
pdb.namespace = ns

// Create AWS config
awsConfig, _, err := pdb.getAWSConfig(ctx)
if err != nil {
return errors.Wrapf(err, "app=%s", pdb.name)
}
// Create ec2 client
ec2, err := testutil.NewEC2Client(ctx, pdb.accessID, pdb.secretKey, pdb.region, pdb.sessionToken, "")
ec2Cli, err := ec2.NewClient(ctx, awsConfig)
if err != nil {
return err
}

// Create security group
log.Info().Print("Creating security group.", field.M{"app": pdb.name, "name": "pgtest-sg"})
sg, err := ec2.CreateSecurityGroup(ctx, "pgtest-sg", "pgtest-security-group")
sg, err := ec2Cli.CreateSecurityGroup(ctx, "pgtest-sg", "pgtest-security-group")
if err != nil {
return err
}
pdb.securityGroupID = *sg.GroupId

// Add ingress rule
log.Info().Print("Adding ingress rule to security group.", field.M{"app": pdb.name})
_, err = ec2.AuthorizeSecurityGroupIngress(ctx, "pgtest-sg", "0.0.0.0/0", "tcp", 5432)
_, err = ec2Cli.AuthorizeSecurityGroupIngress(ctx, "pgtest-sg", "0.0.0.0/0", "tcp", 5432)
if err != nil {
return err
}

// Create rds client
rds, err := testutil.NewRDSClient(ctx, pdb.accessID, pdb.secretKey, pdb.region, pdb.sessionToken, "")
rdsCli, err := rds.NewClient(ctx, awsConfig)
if err != nil {
return err
}

// Create RDS instance
log.Info().Print("Creating RDS instance.", field.M{"app": pdb.name, "id": pdb.id})
_, err = rds.CreateDBInstance(ctx, 20, "db.t2.micro", pdb.id, "postgres", pdb.username, pdb.password, pdb.securityGroupID)
_, err = rdsCli.CreateDBInstance(ctx, 20, "db.t2.micro", pdb.id, "postgres", pdb.username, pdb.password, pdb.securityGroupID)
if err != nil {
return err
}

// Wait for DB to be ready
log.Info().Print("Waiting for rds to be ready.", field.M{"app": pdb.name})
err = rds.WaitUntilDBInstanceAvailable(ctx, pdb.id)
err = rdsCli.WaitUntilDBInstanceAvailable(ctx, pdb.id)
if err != nil {
return err
}

// Find host of the instance
dbInstance, err := rds.DescribeDBInstances(ctx, pdb.id)
dbInstance, err := rdsCli.DescribeDBInstances(ctx, pdb.id)
if err != nil {
return err
}
Expand Down Expand Up @@ -300,15 +307,20 @@ func (pdb RDSPostgresDB) Secrets() map[string]crv1alpha1.ObjectReference {
}

func (pdb RDSPostgresDB) Uninstall(ctx context.Context) error {
// Create AWS config
awsConfig, _, err := pdb.getAWSConfig(ctx)
if err != nil {
return errors.Wrapf(err, "app=%s", pdb.name)
}
// Create rds client
rds, err := testutil.NewRDSClient(ctx, pdb.accessID, pdb.secretKey, pdb.region, pdb.sessionToken, "")
rdsCli, err := rds.NewClient(ctx, awsConfig)
if err != nil {
return errors.Wrap(err, "Failed to create rds client. You may need to delete RDS resources manually. app=rds-postgresql")
}

// Delete rds instance
log.Info().Print("Deleting rds instance", field.M{"app": pdb.name})
_, err = rds.DeleteDBInstance(ctx, pdb.id)
_, err = rdsCli.DeleteDBInstance(ctx, pdb.id)
if err != nil {
if err, ok := err.(awserr.Error); ok {
switch err.Code() {
Expand All @@ -323,21 +335,21 @@ func (pdb RDSPostgresDB) Uninstall(ctx context.Context) error {
// Waiting for rds to be deleted
if err == nil {
log.Info().Print("Waiting for rds to be deleted", field.M{"app": pdb.name})
err = rds.WaitUntilDBInstanceDeleted(ctx, pdb.id)
err = rdsCli.WaitUntilDBInstanceDeleted(ctx, pdb.id)
if err != nil {
return errors.Wrapf(err, "Failed to wait for rds instance till delete succeeds. app=rds-postgresql id=%s", pdb.id)
}
}

// Create ec2 client
ec2, err := testutil.NewEC2Client(ctx, pdb.accessID, pdb.secretKey, pdb.region, pdb.sessionToken, "")
ec2Cli, err := ec2.NewClient(ctx, awsConfig)
if err != nil {
return errors.Wrap(err, "Failed to ec2 client. You may need to delete EC2 resources manually. app=rds-postgresql")
}

// Delete security group
log.Info().Print("Deleting security group.", field.M{"app": pdb.name})
_, err = ec2.DeleteSecurityGroup(ctx, "pgtest-sg")
_, err = ec2Cli.DeleteSecurityGroup(ctx, "pgtest-sg")
if err != nil {
if err, ok := err.(awserr.Error); ok {
switch err.Code() {
Expand All @@ -350,3 +362,12 @@ func (pdb RDSPostgresDB) Uninstall(ctx context.Context) error {
}
return nil
}

func (pdb RDSPostgresDB) getAWSConfig(ctx context.Context) (*awssdk.Config, string, error) {
config := make(map[string]string)
config[aws.ConfigRegion] = pdb.region
config[aws.AccessKeyID] = pdb.accessID
config[aws.SecretAccessKey] = pdb.secretKey
config[aws.SessionToken] = pdb.sessionToken
return aws.GetConfig(ctx, config)
}
23 changes: 22 additions & 1 deletion pkg/config/aws/aws.go → pkg/aws/aws.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/pkg/errors"

awsrole "github.com/kanisterio/kanister/pkg/aws/role"
"github.com/kanisterio/kanister/pkg/param"
"github.com/kanisterio/kanister/pkg/secrets"
)

const (
Expand All @@ -41,6 +45,23 @@ const (
assumeRoleDuration = 25 * time.Minute
)

// GetConfigFromProfile extracts AWS creds from profile
func GetConfigFromProfile(ctx context.Context, profile *param.Profile) (*aws.Config, string, error) {
config := make(map[string]string)

if profile.Credential.Type == param.CredentialTypeKeyPair {
config[AccessKeyID] = profile.Credential.KeyPair.ID
config[SecretAccessKey] = profile.Credential.KeyPair.Secret
} else if profile.Credential.Type == param.CredentialTypeSecret {
config[AccessKeyID] = string(profile.Credential.Secret.Data[secrets.AWSAccessKeyID])
config[SecretAccessKey] = string(profile.Credential.Secret.Data[secrets.AWSSecretAccessKey])
config[ConfigRole] = string(profile.Credential.Secret.Data[secrets.ConfigRole])
config[SessionToken] = string(profile.Credential.Secret.Data[secrets.AWSSessionToken])
}
config[ConfigRegion] = profile.Location.Region
return GetConfig(ctx, config)
}

// GetConfig returns a configuration to establish AWS connection and connected region name.
func GetConfig(ctx context.Context, config map[string]string) (awsConfig *aws.Config, region string, err error) {
region, ok := config[ConfigRegion]
Expand All @@ -67,7 +88,7 @@ func GetConfig(ctx context.Context, config map[string]string) (awsConfig *aws.Co
}

func assumeRole(ctx context.Context, accessKey, secretAccessKey, role string) (*aws.Config, error) {
creds, err := SwitchRole(ctx, accessKey, secretAccessKey, role, assumeRoleDuration)
creds, err := awsrole.Switch(ctx, accessKey, secretAccessKey, role, assumeRoleDuration)
if err != nil {
return nil, errors.Wrap(err, "Failed to switch roles")
}
Expand Down
76 changes: 76 additions & 0 deletions pkg/aws/ec2/ec2.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
// Copyright 2019 The Kanister Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ec2

import (
"context"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/pkg/errors"
)

// EC2 is a wrapper around ec2.EC2 structs
type EC2 struct {
*ec2.EC2
DryRun bool
}

// NewEC2Client returns ec2 client struct.
func NewClient(ctx context.Context, awsConfig *aws.Config) (*EC2, error) {
s, err := session.NewSession(awsConfig)
if err != nil {
return nil, errors.Wrap(err, "Failed to create session")
}
return &EC2{EC2: ec2.New(s, awsConfig)}, nil
}

func (e EC2) DescribeSecurityGroup(ctx context.Context, groupName string) (*ec2.DescribeSecurityGroupsOutput, error) {
sgi := &ec2.DescribeSecurityGroupsInput{
DryRun: &e.DryRun,
GroupNames: []*string{&groupName},
}
return e.DescribeSecurityGroupsWithContext(ctx, sgi)
}

func (e EC2) CreateSecurityGroup(ctx context.Context, groupName, description string) (*ec2.CreateSecurityGroupOutput, error) {
sgi := &ec2.CreateSecurityGroupInput{
DryRun: &e.DryRun,
Description: &description,
GroupName: &groupName,
}
return e.CreateSecurityGroupWithContext(ctx, sgi)
}

func (e EC2) AuthorizeSecurityGroupIngress(ctx context.Context, groupName, cidr, protocol string, port int64) (*ec2.AuthorizeSecurityGroupIngressOutput, error) {
sgi := &ec2.AuthorizeSecurityGroupIngressInput{
DryRun: &e.DryRun,
GroupName: &groupName,
CidrIp: &cidr,
IpProtocol: &protocol,
ToPort: &port,
FromPort: &port,
}
return e.AuthorizeSecurityGroupIngressWithContext(ctx, sgi)
}

func (e EC2) DeleteSecurityGroup(ctx context.Context, groupName string) (*ec2.DeleteSecurityGroupOutput, error) {
sgi := &ec2.DeleteSecurityGroupInput{
DryRun: &e.DryRun,
GroupName: &groupName,
}
return e.DeleteSecurityGroupWithContext(ctx, sgi)
}
Loading

0 comments on commit 551a661

Please sign in to comment.