Check AWS Admin access for cluster up and down commands (#878)

Author: deliahu

cli/cmd/cluster.go

@@ -237,6 +237,13 @@ var _downCmd = &cobra.Command{
 			exit.Error(err)
 		}
 
+		// Check AWS access
+		awsClient, err := newAWSClient(*accessConfig.Region, awsCreds)
+		if err != nil {
+			exit.Error(err)
+		}
+		warnIfNotAdmin(awsClient)
+
 		prompt.YesOrExit(fmt.Sprintf("your cluster (%s in %s) will be spun down and all apis will be deleted, are you sure you want to continue?", *accessConfig.ClusterName, *accessConfig.Region), "", "")
 
 		out, exitCode, err := runManagerAccessCommand("/root/uninstall.sh", *accessConfig, awsCreds)

cli/cmd/lib_aws_creds.go

@@ -46,6 +46,28 @@ func newAWSClient(region string, awsCreds AWSCredentials) (*aws.Client, error) {
 	return awsClient, nil
 }
 
+func promptIfNotAdmin(awsClient *aws.Client) {
+	accessKeyMsg := ""
+	if accessKey := awsClient.AccessKeyID(); accessKey != nil {
+		accessKeyMsg = fmt.Sprintf(" (with access key %s)", *accessKey)
+	}
+
+	if !awsClient.IsAdmin() {
+		prompt.YesOrExit(fmt.Sprintf("warning: your IAM user%s does not have administrator access. This will likely prevent Cortex from installing correctly, so it is recommended to attach the AdministratorAccess policy to your IAM user. If you'd like, you may provide separate credentials for your cluster to use after it's running (see https://cortex.dev/cluster-management/security for instructions).\nAre you sure you want to continue?", accessKeyMsg), "", "")
+	}
+}
+
+func warnIfNotAdmin(awsClient *aws.Client) {
+	accessKeyMsg := ""
+	if accessKey := awsClient.AccessKeyID(); accessKey != nil {
+		accessKeyMsg = fmt.Sprintf(" (with access key %s)", *accessKey)
+	}
+
+	if !awsClient.IsAdmin() {
+		fmt.Println(fmt.Sprintf("warning: your IAM user%s does not have administrator access. This may prevent this command from executing correctly, so it is recommended to attach the AdministratorAccess policy to your IAM user.", accessKeyMsg), "", "")
+	}
+}
+
 var _awsCredentialsValidation = &cr.StructValidation{
 	AllowExtraFields: true,
 	StructFieldValidations: []*cr.StructFieldValidation{

cli/cmd/lib_cluster_config.go

@@ -140,6 +140,7 @@ func getInstallClusterConfig(awsCreds AWSCredentials) (*clusterconfig.Config, er
 	if err != nil {
 		return nil, err
 	}
+	promptIfNotAdmin(awsClient)
 
 	err = clusterconfig.InstallPrompt(clusterConfig, awsClient)
 	if err != nil {
@@ -182,6 +183,7 @@ func getClusterUpdateConfig(cachedClusterConfig clusterconfig.Config, awsCreds A
 		if err != nil {
 			return nil, err
 		}
+		promptIfNotAdmin(awsClient)
 	} else {
 		err := readUserClusterConfigFile(userClusterConfig)
 		if err != nil {
@@ -194,6 +196,7 @@ func getClusterUpdateConfig(cachedClusterConfig clusterconfig.Config, awsCreds A
 		if err != nil {
 			return nil, err
 		}
+		promptIfNotAdmin(awsClient)
 
 		if userClusterConfig.Bucket != "" && userClusterConfig.Bucket != cachedClusterConfig.Bucket {
 			return nil, clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.BucketKey, cachedClusterConfig.Bucket)

docs/cluster-management/security.md

@@ -4,7 +4,17 @@ _WARNING: you are on the master branch, please refer to the docs on the branch t
 
 ## IAM permissions
 
-If you are not using a sensitive AWS account and do not have a lot of experience with IAM configuration, attaching the existing policy `AdministratorAccess` to your IAM user will make getting started much easier.
+If you are not using a sensitive AWS account and do not have a lot of experience with IAM configuration, attaching the built-in `AdministratorAccess` policy to your IAM user will make getting started much easier. If you would like to limit IAM permissions, continue reading.
+
+### Cluster spin-up
+
+Spinning up Cortex on your AWS account requires more permissions that Cortex needs once it's running. You can specify different credentials for each purpose in two ways:
+
+1. You can export the environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` which will be used to create your cluster, and export `CORTEX_AWS_ACCESS_KEY_ID` and `CORTEX_AWS_SECRET_ACCESS_KEY` which will be used by the cluster.
+
+2. If you are using a cluster configuration file (e.g. `cluster.yaml`), you can set the fields `aws_access_key_id` and `aws_secret_access_key` which will be used to create your cluster, and set `cortex_aws_access_key_id` and `cortex_aws_secret_access_key` which will be used by the cluster.
+
+In either case, the credentials used when spinning up the cluster will not be used by the cluster itself, and can be safely revoked after the cluster is running. You may need credentials with similar access to run other `cortex cluster` commands, such as `cortex cluster update`, `cortex cluster info`, and `cortex cluster down`.
 
 ### Operator
 
@@ -39,14 +49,12 @@ The operator requires read permissions for any S3 bucket containing exported mod
 }
 ```
 
+It is possible to further restrict access by limiting access to particular resources (e.g. allowing access to only the bucket containing your models and the cortex bucket).
+
 ### CLI
 
 In order to connect to the operator via the CLI, you must provide valid AWS credentials for any user with access to the account. No special permissions are required. The CLI can be configured using the `cortex configure` command.
 
-## API access
-
-By default, your Cortex APIs will be accessible to all traffic. You can restrict access using AWS security groups. Specifically, you will need to edit the security group with the description: "Security group for Kubernetes ELB <ELB name> (istio-system/ingressgateway-apis)".
-
 ## HTTPS
 
 All APIs are accessible via HTTPS. The certificate is autogenerated during installation using `localhost` as the Common Name (CN). Therefore, clients will need to skip certificate verification (e.g. `curl -k`) when using HTTPS.

pkg/lib/aws/clients.go

@@ -21,6 +21,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/cloudwatch"
 	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/iam"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/servicequotas"
 	"github.com/aws/aws-sdk-go/service/sts"
@@ -34,6 +35,7 @@ type clients struct {
 	cloudWatchLogs    *cloudwatchlogs.CloudWatchLogs
 	cloudWatchMetrics *cloudwatch.CloudWatch
 	serviceQuotas     *servicequotas.ServiceQuotas
+	iam               *iam.IAM
 }
 
 func (c *Client) S3() *s3.S3 {
@@ -84,3 +86,10 @@ func (c *Client) ServiceQuotas() *servicequotas.ServiceQuotas {
 	}
 	return c.clients.serviceQuotas
 }
+
+func (c *Client) IAM() *iam.IAM {
+	if c.clients.iam == nil {
+		c.clients.iam = iam.New(c.sess)
+	}
+	return c.clients.iam
+}

pkg/lib/aws/credentials.go

@@ -22,6 +22,24 @@ import (
 	"github.com/aws/aws-sdk-go/aws/credentials"
 )
 
+// access key ID may be unavailable depending on how the client was instantiated
+func (c *Client) AccessKeyID() *string {
+	if c.sess.Config.Credentials == nil {
+		return nil
+	}
+
+	sessCreds, err := c.sess.Config.Credentials.Get()
+	if err != nil {
+		return nil
+	}
+
+	if sessCreds.AccessKeyID == "" {
+		return nil
+	}
+
+	return &sessCreds.AccessKeyID
+}
+
 func GetCredentialsFromCLIConfigFile() (string, string, error) {
 	creds := credentials.NewSharedCredentials("", "")
 	if creds == nil {

pkg/lib/aws/ec2.go

@@ -33,7 +33,7 @@ func (c *Client) SpotInstancePrice(region string, instanceType string) (float64,
 		StartTime:           aws.Time(time.Now()),
 	})
 	if err != nil {
-		return 0, errors.WithStack(err)
+		return 0, errors.Wrap(err, "checking spot instance price")
 	}
 
 	min := math.MaxFloat64

pkg/lib/aws/iam.go

@@ -0,0 +1,109 @@
+/*
+Copyright 2020 Cortex Labs, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/cortexlabs/cortex/pkg/lib/errors"
+)
+
+func (c *Client) GetUser() (iam.User, error) {
+	getUserOutput, err := c.IAM().GetUser(nil)
+	if err != nil {
+		return iam.User{}, errors.WithStack(err)
+	}
+	return *getUserOutput.User, nil
+}
+
+func (c *Client) GetGroupsForUser(userName string) ([]iam.Group, error) {
+	input := &iam.ListGroupsForUserInput{
+		UserName: &userName,
+	}
+
+	var groups []iam.Group
+
+	err := c.IAM().ListGroupsForUserPages(input, func(page *iam.ListGroupsForUserOutput, lastPage bool) bool {
+		for _, group := range page.Groups {
+			groups = append(groups, *group)
+		}
+		return true
+	})
+
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	return groups, nil
+}
+
+func (c *Client) GetManagedPoliciesForUser(userName string) ([]iam.AttachedPolicy, error) {
+	user, err := c.GetUser()
+	if err != nil {
+		return nil, err
+	}
+
+	var policies []iam.AttachedPolicy
+
+	userManagedPolicies, err := c.IAM().ListAttachedUserPolicies(&iam.ListAttachedUserPoliciesInput{
+		UserName: &userName,
+	})
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	for _, policy := range userManagedPolicies.AttachedPolicies {
+		policies = append(policies, *policy)
+	}
+
+	groups, err := c.GetGroupsForUser(*user.UserName)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, group := range groups {
+		groupManagedPolicies, err := c.IAM().ListAttachedGroupPolicies(&iam.ListAttachedGroupPoliciesInput{
+			GroupName: group.GroupName,
+		})
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+		for _, policy := range groupManagedPolicies.AttachedPolicies {
+			policies = append(policies, *policy)
+		}
+	}
+
+	return policies, nil
+}
+
+func (c *Client) IsAdmin() bool {
+	user, err := c.GetUser()
+	if err != nil {
+		return false
+	}
+
+	policies, err := c.GetManagedPoliciesForUser(*user.UserName)
+	if err != nil {
+		return false
+	}
+
+	for _, policy := range policies {
+		if *policy.PolicyArn == "arn:aws:iam::aws:policy/AdministratorAccess" {
+			return true
+		}
+	}
+
+	return false
+}

pkg/types/clusterconfig/clusterconfig.go

@@ -510,11 +510,18 @@ func (cc *Config) Validate(awsClient *aws.Client) error {
 			}
 
 			instanceMetadata := aws.InstanceMetadatas[*cc.Region][instanceType]
-			err := CheckSpotInstanceCompatibility(awsClient, chosenInstance, instanceMetadata, cc.SpotConfig.MaxPrice)
+			err := CheckSpotInstanceCompatibility(chosenInstance, instanceMetadata)
 			if err != nil {
 				return errors.Wrap(err, InstanceDistributionKey)
 			}
 
+			spotInstancePrice, awsErr := awsClient.SpotInstancePrice(instanceMetadata.Region, instanceMetadata.Type)
+			if awsErr == nil {
+				if err := CheckSpotInstancePriceCompatibility(chosenInstance, instanceMetadata, cc.SpotConfig.MaxPrice, spotInstancePrice); err != nil {
+					return errors.Wrap(err, InstanceDistributionKey)
+				}
+			}
+
 			compatibleInstanceCount++
 		}
 
@@ -555,7 +562,7 @@ func CheckCortexSupport(instanceMetadata aws.InstanceMetadata) error {
 	return nil
 }
 
-func CheckSpotInstanceCompatibility(awsClient *aws.Client, target aws.InstanceMetadata, suggested aws.InstanceMetadata, maxPrice *float64) error {
+func CheckSpotInstanceCompatibility(target aws.InstanceMetadata, suggested aws.InstanceMetadata) error {
 	if target.GPU > suggested.GPU {
 		return ErrorIncompatibleSpotInstanceTypeGPU(target, suggested)
 	}
@@ -568,17 +575,16 @@ func CheckSpotInstanceCompatibility(awsClient *aws.Client, target aws.InstanceMe
 		return ErrorIncompatibleSpotInstanceTypeCPU(target, suggested)
 	}
 
-	suggestedInstancePrice, err := awsClient.SpotInstancePrice(target.Region, suggested.Type)
-	if err != nil {
-		return err
-	}
+	return nil
+}
 
-	if (maxPrice == nil || *maxPrice == target.Price) && target.Price < suggestedInstancePrice {
-		return ErrorSpotPriceGreaterThanTargetOnDemand(suggestedInstancePrice, target, suggested)
+func CheckSpotInstancePriceCompatibility(target aws.InstanceMetadata, suggested aws.InstanceMetadata, maxPrice *float64, spotInstancePrice float64) error {
+	if (maxPrice == nil || *maxPrice == target.Price) && target.Price < spotInstancePrice {
+		return ErrorSpotPriceGreaterThanTargetOnDemand(spotInstancePrice, target, suggested)
 	}
 
-	if maxPrice != nil && *maxPrice < suggestedInstancePrice {
-		return ErrorSpotPriceGreaterThanMaxPrice(suggestedInstancePrice, *maxPrice, suggested)
+	if maxPrice != nil && *maxPrice < spotInstancePrice {
+		return ErrorSpotPriceGreaterThanMaxPrice(spotInstancePrice, *maxPrice, suggested)
 	}
 	return nil
 }
@@ -604,10 +610,17 @@ func CompatibleSpotInstances(awsClient *aws.Client, targetInstance aws.InstanceM
 			continue
 		}
 
-		if err := CheckSpotInstanceCompatibility(awsClient, targetInstance, instanceMetadata, maxPrice); err != nil {
+		if err := CheckSpotInstanceCompatibility(targetInstance, instanceMetadata); err != nil {
 			continue
 		}
 
+		spotInstancePrice, awsErr := awsClient.SpotInstancePrice(instanceMetadata.Region, instanceMetadata.Type)
+		if awsErr == nil {
+			if err := CheckSpotInstancePriceCompatibility(targetInstance, instanceMetadata, maxPrice, spotInstancePrice); err != nil {
+				continue
+			}
+		}
+
 		compatibleInstances = append(compatibleInstances, instanceMetadata)
 
 		if len(compatibleInstances) == numInstances {