Add support for multi-instance-type clusters to AWS/GCP providers

.circleci/config.yml

@@ -146,15 +146,27 @@ jobs:
             echo 'export AWS_SECRET_ACCESS_KEY=${NIGHTLY_AWS_SECRET_ACCESS_KEY}' >> $BASH_ENV
       - run:
           name: Generate Cluster Config
+           # using a variety of node groups to test the multi-instance-type cluster functionality
           command: |
             cat \<< EOF > ./cluster.yaml
-            cluster_name: cortex
             provider: aws
+            cluster_name: cortex
             region: us-east-1
-            instance_type: g4dn.xlarge
-            min_instances: 1
-            max_instances: 2
             bucket: cortex-dev-nightly
+            node_groups:
+              - name: spot
+                instance_type: t3.medium
+                min_instances: 0
+                max_instances: 1
+                spot: true
+              - name: cpu
+                instance_type: c5.xlarge
+                min_instances: 1
+                max_instances: 2
+              - name: gpu
+                instance_type: g4dn.xlarge
+                min_instances: 1
+                max_instances: 2
             EOF
       - run-e2e-tests:
           provider: aws
@@ -174,16 +186,28 @@ jobs:
             echo 'export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/google_service_account.json' >> $BASH_ENV
       - run:
           name: Generate Cluster Config
+          # using a variety of node pools to test the multi-instance-type cluster functionality
           command: |
             cat \<< EOF > ./cluster.yaml
+            provider: gcp
             cluster_name: cortex
             project: cortexlabs-dev
             zone: us-east1-c
-            provider: gcp
-            instance_type: n1-standard-2
-            accelerator_type: nvidia-tesla-t4
-            min_instances: 1
-            max_instances: 2
+            node_pools:
+              - name: preemptible
+                instance_type: n1-standard-2
+                min_instances: 0
+                max_instances: 1
+                preemptible: true
+              - name: cpu
+                instance_type: n1-standard-2
+                min_instances: 1
+                max_instances: 2
+              - name: gpu
+                instance_type: n1-standard-2
+                accelerator_type: nvidia-tesla-t4
+                min_instances: 1
+                max_instances: 2
             EOF
       - run-e2e-tests:
           provider: gcp

cli/cmd/cluster.go

@@ -302,7 +302,7 @@ var _clusterConfigureCmd = &cobra.Command{
 			exit.Error(err)
 		}
 
-		accessConfig, err := getClusterAccessConfigWithCache()
+		accessConfig, err := getNewClusterAccessConfig(clusterConfigFile)
 		if err != nil {
 			exit.Error(err)
 		}
@@ -317,7 +317,7 @@ var _clusterConfigureCmd = &cobra.Command{
 			exit.Error(err)
 		}
 
-		err = clusterstate.AssertClusterStatus(accessConfig.ClusterName, accessConfig.Region, clusterState.Status, clusterstate.StatusCreateComplete)
+		err = clusterstate.AssertClusterStatus(accessConfig.ClusterName, accessConfig.Region, clusterState.Status, clusterstate.StatusCreateComplete, clusterstate.StatusUpdateComplete, clusterstate.StatusUpdateRollbackComplete)
 		if err != nil {
 			exit.Error(err)
 		}
@@ -527,7 +527,7 @@ var _clusterExportCmd = &cobra.Command{
 			exit.Error(err)
 		}
 
-		err = clusterstate.AssertClusterStatus(accessConfig.ClusterName, accessConfig.Region, clusterState.Status, clusterstate.StatusCreateComplete)
+		err = clusterstate.AssertClusterStatus(accessConfig.ClusterName, accessConfig.Region, clusterState.Status, clusterstate.StatusCreateComplete, clusterstate.StatusUpdateComplete, clusterstate.StatusUpdateRollbackComplete)
 		if err != nil {
 			exit.Error(err)
 		}
@@ -668,7 +668,7 @@ func printInfoClusterState(awsClient *aws.Client, accessConfig *clusterconfig.Ac
 		fmt.Println()
 	}
 
-	err = clusterstate.AssertClusterStatus(accessConfig.ClusterName, accessConfig.Region, clusterState.Status, clusterstate.StatusCreateComplete)
+	err = clusterstate.AssertClusterStatus(accessConfig.ClusterName, accessConfig.Region, clusterState.Status, clusterstate.StatusCreateComplete, clusterstate.StatusUpdateComplete, clusterstate.StatusUpdateRollbackComplete)
 	if err != nil {
 		return err
 	}
@@ -679,6 +679,12 @@ func printInfoClusterState(awsClient *aws.Client, accessConfig *clusterconfig.Ac
 func printInfoOperatorResponse(clusterConfig clusterconfig.Config, operatorEndpoint string) error {
 	fmt.Print("fetching cluster status ...\n\n")
 
+	yamlBytes, err := yaml.Marshal(clusterConfig)
+	if err != nil {
+		return err
+	}
+	yamlString := string(yamlBytes)
+
 	operatorConfig := cluster.OperatorConfig{
 		Telemetry:        isTelemetryEnabled(),
 		ClientID:         clientID(),
@@ -688,42 +694,67 @@ func printInfoOperatorResponse(clusterConfig clusterconfig.Config, operatorEndpo
 
 	infoResponse, err := cluster.Info(operatorConfig)
 	if err != nil {
-		fmt.Println(clusterConfig.UserStr())
+		fmt.Println(yamlString)
 		return err
 	}
 	infoResponse.ClusterConfig.Config = clusterConfig
 
-	printInfoClusterConfig(infoResponse)
+	fmt.Println(console.Bold("metadata:"))
+	fmt.Println(fmt.Sprintf("aws access key id: %s", infoResponse.MaskedAWSAccessKeyID))
+	fmt.Println(fmt.Sprintf("%s: %s", clusterconfig.APIVersionUserKey, infoResponse.ClusterConfig.APIVersion))
+
+	fmt.Println()
+	fmt.Println(console.Bold("cluster config:"))
+	fmt.Print(yamlString)
+
 	printInfoPricing(infoResponse, clusterConfig)
 	printInfoNodes(infoResponse)
 
 	return nil
 }
 
-func printInfoClusterConfig(infoResponse *schema.InfoResponse) {
-	var items table.KeyValuePairs
-	items.Add("aws access key id", infoResponse.MaskedAWSAccessKeyID)
-	items.AddAll(infoResponse.ClusterConfig.UserTable())
-	items.Print()
-}
-
 func printInfoPricing(infoResponse *schema.InfoResponse, clusterConfig clusterconfig.Config) {
-	numAPIInstances := len(infoResponse.NodeInfos)
-
-	var totalAPIInstancePrice float64
-	for _, nodeInfo := range infoResponse.NodeInfos {
-		totalAPIInstancePrice += nodeInfo.Price
-	}
-
 	eksPrice := aws.EKSPrices[clusterConfig.Region]
 	operatorInstancePrice := aws.InstanceMetadatas[clusterConfig.Region]["t3.medium"].Price
 	operatorEBSPrice := aws.EBSMetadatas[clusterConfig.Region]["gp2"].PriceGB * 20 / 30 / 24
 	metricsEBSPrice := aws.EBSMetadatas[clusterConfig.Region]["gp2"].PriceGB * 40 / 30 / 24
 	nlbPrice := aws.NLBMetadatas[clusterConfig.Region].Price
 	natUnitPrice := aws.NATMetadatas[clusterConfig.Region].Price
-	apiEBSPrice := aws.EBSMetadatas[clusterConfig.Region][clusterConfig.InstanceVolumeType.String()].PriceGB * float64(clusterConfig.InstanceVolumeSize) / 30 / 24
-	if clusterConfig.InstanceVolumeType.String() == "io1" && clusterConfig.InstanceVolumeIOPS != nil {
-		apiEBSPrice += aws.EBSMetadatas[clusterConfig.Region][clusterConfig.InstanceVolumeType.String()].PriceIOPS * float64(*clusterConfig.InstanceVolumeIOPS) / 30 / 24
+
+	headers := []table.Header{
+		{Title: "aws resource"},
+		{Title: "cost per hour"},
+	}
+
+	var rows [][]interface{}
+	rows = append(rows, []interface{}{"1 eks cluster", s.DollarsMaxPrecision(eksPrice)})
+
+	var totalNodeGroupsPrice float64
+	for _, ng := range clusterConfig.NodeGroups {
+		var ngNamePrefix string
+		if ng.Spot {
+			ngNamePrefix = "cx-ws-"
+		} else {
+			ngNamePrefix = "cx-wd-"
+		}
+		nodesInfo := infoResponse.GetNodesWithNodeGroupName(ngNamePrefix + ng.Name)
+		numInstances := len(nodesInfo)
+
+		ebsPrice := aws.EBSMetadatas[clusterConfig.Region][ng.InstanceVolumeType.String()].PriceGB * float64(ng.InstanceVolumeSize) / 30 / 24
+		if ng.InstanceVolumeType.String() == "io1" && ng.InstanceVolumeIOPS != nil {
+			ebsPrice += aws.EBSMetadatas[clusterConfig.Region][ng.InstanceVolumeType.String()].PriceIOPS * float64(*ng.InstanceVolumeIOPS) / 30 / 24
+		}
+		totalEBSPrice := ebsPrice * float64(numInstances)
+
+		totalInstancePrice := float64(0)
+		for _, nodeInfo := range nodesInfo {
+			totalInstancePrice += nodeInfo.Price
+		}
+
+		rows = append(rows, []interface{}{fmt.Sprintf("nodegroup %s: %d (out of %d) %s for your apis", ng.Name, numInstances, ng.MaxInstances, s.PluralS("instance", numInstances)), s.DollarsAndTenthsOfCents(totalInstancePrice) + " total"})
+		rows = append(rows, []interface{}{fmt.Sprintf("nodegroup %s: %d (out of %d) %dgb ebs %s for your apis", ng.Name, numInstances, ng.MaxInstances, ng.InstanceVolumeSize, s.PluralS("volume", numInstances)), s.DollarsAndTenthsOfCents(totalEBSPrice) + " total"})
+
+		totalNodeGroupsPrice += totalEBSPrice + totalInstancePrice
 	}
 
 	var natTotalPrice float64
@@ -732,20 +763,9 @@ func printInfoPricing(infoResponse *schema.InfoResponse, clusterConfig clusterco
 	} else if clusterConfig.NATGateway == clusterconfig.HighlyAvailableNATGateway {
 		natTotalPrice = natUnitPrice * float64(len(clusterConfig.AvailabilityZones))
 	}
-
-	totalPrice := eksPrice + totalAPIInstancePrice + apiEBSPrice*float64(numAPIInstances) +
-		operatorInstancePrice*2 + operatorEBSPrice + metricsEBSPrice + nlbPrice*2 + natTotalPrice
+	totalPrice := eksPrice + totalNodeGroupsPrice + operatorInstancePrice*2 + operatorEBSPrice + metricsEBSPrice + nlbPrice*2 + natTotalPrice
 	fmt.Printf(console.Bold("\nyour cluster currently costs %s per hour\n\n"), s.DollarsAndCents(totalPrice))
 
-	headers := []table.Header{
-		{Title: "aws resource"},
-		{Title: "cost per hour"},
-	}
-
-	var rows [][]interface{}
-	rows = append(rows, []interface{}{"1 eks cluster", s.DollarsMaxPrecision(eksPrice)})
-	rows = append(rows, []interface{}{fmt.Sprintf("%d %s for your apis", numAPIInstances, s.PluralS("instance", numAPIInstances)), s.DollarsAndTenthsOfCents(totalAPIInstancePrice) + " total"})
-	rows = append(rows, []interface{}{fmt.Sprintf("%d %dgb ebs %s for your apis", numAPIInstances, clusterConfig.InstanceVolumeSize, s.PluralS("volume", numAPIInstances)), s.DollarsAndTenthsOfCents(apiEBSPrice*float64(numAPIInstances)) + " total"})
 	rows = append(rows, []interface{}{"2 t3.medium instances for cortex", s.DollarsMaxPrecision(operatorInstancePrice * 2)})
 	rows = append(rows, []interface{}{"1 20gb ebs volume for the operator", s.DollarsAndTenthsOfCents(operatorEBSPrice)})
 	rows = append(rows, []interface{}{"1 40gb ebs volume for prometheus", s.DollarsAndTenthsOfCents(metricsEBSPrice)})

cli/cmd/cluster_gcp.go

@@ -35,6 +35,7 @@ import (
 	"github.com/cortexlabs/cortex/pkg/lib/telemetry"
 	"github.com/cortexlabs/cortex/pkg/types"
 	"github.com/cortexlabs/cortex/pkg/types/clusterconfig"
+	"github.com/cortexlabs/yaml"
 	"github.com/spf13/cobra"
 	containerpb "google.golang.org/genproto/googleapis/container/v1"
 )
@@ -373,7 +374,18 @@ func printInfoOperatorResponseGCP(accessConfig *clusterconfig.GCPAccessConfig, o
 		return err
 	}
 
-	infoResponse.ClusterConfig.UserTable().Print()
+	yamlBytes, err := yaml.Marshal(infoResponse.ClusterConfig.GCPConfig)
+	if err != nil {
+		return err
+	}
+	yamlString := string(yamlBytes)
+
+	fmt.Println(console.Bold("metadata:"))
+	fmt.Println(fmt.Sprintf("%s: %s", clusterconfig.APIVersionUserKey, infoResponse.ClusterConfig.APIVersion))
+
+	fmt.Println()
+	fmt.Println(console.Bold("cluster config:"))
+	fmt.Print(yamlString)
 
 	return nil
 }
@@ -448,25 +460,9 @@ func updateGCPCLIEnv(envName string, operatorEndpoint string, disallowPrompt boo
 func createGKECluster(clusterConfig *clusterconfig.GCPConfig, gcpClient *gcp.Client) error {
 	fmt.Print("￮ creating GKE cluster ")
 
-	nodeLabels := map[string]string{"workload": "true"}
-	var accelerators []*containerpb.AcceleratorConfig
-
-	if clusterConfig.AcceleratorType != nil {
-		accelerators = append(accelerators, &containerpb.AcceleratorConfig{
-			AcceleratorCount: *clusterConfig.AcceleratorsPerInstance,
-			AcceleratorType:  *clusterConfig.AcceleratorType,
-		})
-		nodeLabels["nvidia.com/gpu"] = "present"
-	}
-
 	gkeClusterParent := fmt.Sprintf("projects/%s/locations/%s", clusterConfig.Project, clusterConfig.Zone)
 	gkeClusterName := fmt.Sprintf("%s/clusters/%s", gkeClusterParent, clusterConfig.ClusterName)
 
-	initialNodeCount := int64(1)
-	if clusterConfig.MinInstances > 0 {
-		initialNodeCount = clusterConfig.MinInstances
-	}
-
 	gkeClusterConfig := containerpb.Cluster{
 		Name:                  clusterConfig.ClusterName,
 		InitialClusterVersion: "1.18",
@@ -488,52 +484,68 @@ func createGKECluster(clusterConfig *clusterconfig.GCPConfig, gcpClient *gcp.Cli
 		Locations: []string{clusterConfig.Zone},
 	}
 
-	if clusterConfig.Preemptible {
-		gkeClusterConfig.NodePools = append(gkeClusterConfig.NodePools, &containerpb.NodePool{
-			Name: "ng-cortex-wk-preemp",
-			Config: &containerpb.NodeConfig{
-				MachineType: clusterConfig.InstanceType,
-				Labels:      nodeLabels,
-				Taints: []*containerpb.NodeTaint{
-					{
-						Key:    "workload",
-						Value:  "true",
-						Effect: containerpb.NodeTaint_NO_SCHEDULE,
+	for _, nodePool := range clusterConfig.NodePools {
+		nodeLabels := map[string]string{"workload": "true"}
+		initialNodeCount := int64(1)
+		if nodePool.MinInstances > 0 {
+			initialNodeCount = nodePool.MinInstances
+		}
+
+		var accelerators []*containerpb.AcceleratorConfig
+		if nodePool.AcceleratorType != nil {
+			accelerators = append(accelerators, &containerpb.AcceleratorConfig{
+				AcceleratorCount: *nodePool.AcceleratorsPerInstance,
+				AcceleratorType:  *nodePool.AcceleratorType,
+			})
+			nodeLabels["nvidia.com/gpu"] = "present"
+		}
+
+		if nodePool.Preemptible {
+			gkeClusterConfig.NodePools = append(gkeClusterConfig.NodePools, &containerpb.NodePool{
+				Name: "cx-ws-" + nodePool.Name,
+				Config: &containerpb.NodeConfig{
+					MachineType: nodePool.InstanceType,
+					Labels:      nodeLabels,
+					Taints: []*containerpb.NodeTaint{
+						{
+							Key:    "workload",
+							Value:  "true",
+							Effect: containerpb.NodeTaint_NO_SCHEDULE,
+						},
 					},
-				},
-				Accelerators: accelerators,
-				OauthScopes: []string{
-					"https://www.googleapis.com/auth/compute",
-					"https://www.googleapis.com/auth/devstorage.read_only",
-				},
-				ServiceAccount: gcpClient.ClientEmail,
-				Preemptible:    true,
-			},
-			InitialNodeCount: int32(initialNodeCount),
-		})
-	}
-	if clusterConfig.OnDemandBackup || !clusterConfig.Preemptible {
-		gkeClusterConfig.NodePools = append(gkeClusterConfig.NodePools, &containerpb.NodePool{
-			Name: "ng-cortex-wk-on-dmd",
-			Config: &containerpb.NodeConfig{
-				MachineType: clusterConfig.InstanceType,
-				Labels:      nodeLabels,
-				Taints: []*containerpb.NodeTaint{
-					{
-						Key:    "workload",
-						Value:  "true",
-						Effect: containerpb.NodeTaint_NO_SCHEDULE,
+					Accelerators: accelerators,
+					OauthScopes: []string{
+						"https://www.googleapis.com/auth/compute",
+						"https://www.googleapis.com/auth/devstorage.read_only",
 					},
+					ServiceAccount: gcpClient.ClientEmail,
+					Preemptible:    true,
 				},
-				Accelerators: accelerators,
-				OauthScopes: []string{
-					"https://www.googleapis.com/auth/compute",
-					"https://www.googleapis.com/auth/devstorage.read_only",
+				InitialNodeCount: int32(initialNodeCount),
+			})
+		} else {
+			gkeClusterConfig.NodePools = append(gkeClusterConfig.NodePools, &containerpb.NodePool{
+				Name: "cx-wd-" + nodePool.Name,
+				Config: &containerpb.NodeConfig{
+					MachineType: nodePool.InstanceType,
+					Labels:      nodeLabels,
+					Taints: []*containerpb.NodeTaint{
+						{
+							Key:    "workload",
+							Value:  "true",
+							Effect: containerpb.NodeTaint_NO_SCHEDULE,
+						},
+					},
+					Accelerators: accelerators,
+					OauthScopes: []string{
+						"https://www.googleapis.com/auth/compute",
+						"https://www.googleapis.com/auth/devstorage.read_only",
+					},
+					ServiceAccount: gcpClient.ClientEmail,
 				},
-				ServiceAccount: gcpClient.ClientEmail,
-			},
-			InitialNodeCount: int32(initialNodeCount),
-		})
+				InitialNodeCount: int32(initialNodeCount),
+			})
+		}
 	}
 
 	if clusterConfig.Network != nil {