Remove region from external data

Author: deliahu

cli/cmd/get.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/cortexlabs/yaml"
 	"github.com/spf13/cobra"
 
 	"github.com/cortexlabs/cortex/pkg/lib/errors"
@@ -412,8 +413,8 @@ func describeAPI(name string, resourcesRes *schema.GetResourcesResponse) (string
 	out += "Method:   POST\n"
 	out += `Header:   "Content-Type: application/json"` + "\n"
 
-	if api.Model != nil {
-		model := ctx.Models[api.ModelName]
+	if modelName, ok := yaml.ExtractAtSymbolText(api.Model); ok {
+		model := ctx.Models[modelName]
 		resIDs := strset.New()
 		combinedInput := []interface{}{model.Input, model.TrainingInput}
 		for _, res := range ctx.ExtractCortexResources(combinedInput, resource.ConstantType, resource.RawColumnType, resource.AggregateType, resource.TransformedColumnType) {

docs/apis/apis.md

@@ -7,9 +7,7 @@ Serve models at scale and use them to build smarter applications.
 ```yaml
 - kind: api
   name: <string>  # API name (required)
-  external_model:
-    path: <string>  # path to a zipped model dir (e.g. s3://my-bucket/model.zip)
-    region: <string>  # S3 region (default: us-west-2)
+  model: <string>  # path to a zipped model dir (e.g. s3://my-bucket/model.zip)
   compute:
     replicas: <int>  # number of replicas to launch (default: 1)
     cpu: <string>  # CPU request per replica (default: Null)
@@ -24,9 +22,7 @@ See [packaging models](packaging-models.md) for how to create the zipped model.
 ```yaml
 - kind: api
   name: my-api
-  external_model:
-    path: s3://my-bucket/my-model.zip
-    region: us-west-2
+  model: s3://my-bucket/my-model.zip
   compute:
     replicas: 3
     gpu: 2

docs/apis/packaging-models.md

@@ -17,12 +17,10 @@ Upload the zipped file to Amazon S3, e.g.
 $ aws s3 cp model.zip s3://my-bucket/model.zip
 ```
 
-Specify `external_model` in an API, e.g.
+Specify `model` in an API, e.g.
 
 ```yaml
 - kind: api
   name: my-api
-  external_model:
-    path: s3://my-bucket/model.zip
-    region: us-west-2
+  model: s3://my-bucket/model.zip
 ```

docs/apis/tutorial.md

@@ -32,9 +32,7 @@ Add to `cortex.yaml`:
 ```yaml
 - kind: api
   name: iris-type
-  external_model:
-    path: s3://cortex-examples/iris-tensorflow.zip
-    region: us-west-2
+  model: s3://cortex-examples/iris-tensorflow.zip
   compute:
     replicas: 3
 ```

docs/pipelines/apis.md

@@ -7,18 +7,15 @@ Serve models at scale and use them to build smarter applications.
 ```yaml
 - kind: api
   name: <string>  # API name (required)
-  model: <string>  # reference to a model (e.g. @dnn) (this or external_model must be specified)
-  external_model:  # (this or model must be specified)
-    path: <string>  # path to a zipped model dir (e.g. s3://my-bucket/model.zip)
-    region: <string>  # S3 region (default: us-west-2)
+  model: <string>  # reference to a model (e.g. @dnn) or path to a zipped model dir (e.g. s3://my-bucket/model.zip)
   compute:
     replicas: <int>  # number of replicas to launch (default: 1)
     cpu: <string>  # CPU request per replica (default: Null)
     gpu: <string>  # gpu request per replica (default: Null)
     mem: <string>  # memory request per replica (default: Null)
 ```
 
-See [packaging models](packaging-models.md) for how to create the zipped model (for use in `external_model`).
+See [packaging models](packaging-models.md) for how to create a zipped model (only if using an externally-built models).
 
 ## Example
 

docs/pipelines/constants.md

@@ -8,10 +8,8 @@ Constants represent literal values which can be used in other Cortex resources.
 - kind: constant
   name: <string>  # constant name (required)
   type: <output_type>  # the type of the constant (optional, will be inferred from value if not specified)
-  value: <output_value>  # a literal value (this or external_model must be specified)
-  external_model:  # (this or value must be specified)
-    path: <string>  # path to a JSON object (e.g. s3://my-bucket/constant.json)
-    region: <string>  # S3 region (default: us-west-2)
+  value: <output_value>  # a literal value (this or path must be specified)
+  path: <string>  # path to a JSON object (e.g. s3://my-bucket/constant.json) (this or value must be specified)
 
 ```
 

docs/pipelines/environments.md

@@ -26,7 +26,6 @@ Transfer data at scale from data warehouses like S3 into the Cortex environment.
 data:
   type: csv  # file type (required)
   path: s3a://<bucket_name>/<file_name>  # S3 is currently supported (required)
-  region: us-west-2  # S3 region (default: us-west-2)
   drop_null: <bool>  # drop any rows that contain at least 1 null value (default: false)
   csv_config: <csv_config>  # optional configuration that can be provided
   schema:
@@ -65,7 +64,6 @@ csv_config:
 data:
   type: parquet  # file type (required)
   path: s3a://<bucket_name>/<file_name>  # S3 is currently supported (required)
-  region: us-west-2  # S3 region (default: us-west-2)
   drop_null: <bool>  # drop any rows that contain at least 1 null value (default: false)
   schema:
     - parquet_column_name: <string>  # name of the column in the parquet file (required)

docs/pipelines/packaging-models.md

@@ -17,12 +17,10 @@ Upload the zipped file to Amazon S3, e.g.
 $ aws s3 cp model.zip s3://my-bucket/model.zip
 ```
 
-Specify `external_model` in an API, e.g.
+Specify `model` in an API, e.g.
 
 ```yaml
 - kind: api
   name: my-api
-  external_model:
-    path: s3://my-bucket/model.zip
-    region: us-west-2
+  model: s3://my-bucket/model.zip
 ```

examples/iris/cortex.yaml

@@ -3,8 +3,6 @@
 
 - kind: api
   name: iris-type
-  external_model:
-    path: s3://cortex-examples/iris-tensorflow.zip
-    region: us-west-2
+  model: s3://cortex-examples/iris-tensorflow.zip
   compute:
     replicas: 3

go.mod

@@ -5,7 +5,7 @@
 //   replace these lines in go.mod:
 //     github.com/GoogleCloudPlatform/spark-on-k8s-operator v1alpha1-0.5-2.4.0
 //     github.com/argoproj/argo v2.3.0
-//     github.com/cortexlabs/yaml v2.2.3
+//     github.com/cortexlabs/yaml v2.2.4
 //     k8s.io/client-go v10.0.0
 //     k8s.io/api 89a74a8d264df0e993299876a8cde88379b940ee
 //     k8s.io/apimachinery 2b1284ed4c93a43499e781493253e2ac5959c4fd
@@ -20,8 +20,8 @@ go 1.12
 require (
 	github.com/GoogleCloudPlatform/spark-on-k8s-operator v0.0.0-20181208011959-62db1d66dafa
 	github.com/argoproj/argo v2.3.0+incompatible
-	github.com/aws/aws-sdk-go v1.20.7
-	github.com/cortexlabs/yaml v0.0.0-20190624201412-7f31702857b6
+	github.com/aws/aws-sdk-go v1.20.8
+	github.com/cortexlabs/yaml v0.0.0-20190626164117-202ab3a3d475
 	github.com/davecgh/go-spew v1.1.1
 	github.com/emicklei/go-restful v2.9.6+incompatible // indirect
 	github.com/go-openapi/spec v0.19.2 // indirect

go.sum

@@ -13,13 +13,13 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko
 github.com/argoproj/argo v2.3.0+incompatible h1:L1OYZ86Q7NK19ahdl/eJOq78Mlf52wUKGmp7VDNQVz8=
 github.com/argoproj/argo v2.3.0+incompatible/go.mod h1:KJ0MB+tuhtAklR4jkPM10mIZXfRA0peTYJ1sLUnFLVU=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
-github.com/aws/aws-sdk-go v1.20.7 h1:fRjZRYJg0wPCA8yaDdb3DeP4rVjjmEiuqYhYoqOaIJg=
-github.com/aws/aws-sdk-go v1.20.7/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aws/aws-sdk-go v1.20.8 h1:oXnNFH/8PosUvHXqZHYLUn6LCSzHZ/34H52vrB4YhpA=
+github.com/aws/aws-sdk-go v1.20.8/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/cortexlabs/yaml v0.0.0-20190624201412-7f31702857b6 h1:NRxzHeHMTVyPV9yTGoXo3zRPplzVhrLTNzSFLNgpQII=
-github.com/cortexlabs/yaml v0.0.0-20190624201412-7f31702857b6/go.mod h1:nuzR4zMPuiBWg1HyZo9bzSZmtdSVjKfn8+RyO7egs0c=
+github.com/cortexlabs/yaml v0.0.0-20190626164117-202ab3a3d475 h1:N+pms5TCPH2F/DIX6a+2dgJI/CHweh45pEhGW/+stxQ=
+github.com/cortexlabs/yaml v0.0.0-20190626164117-202ab3a3d475/go.mod h1:nuzR4zMPuiBWg1HyZo9bzSZmtdSVjKfn8+RyO7egs0c=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

pkg/lib/aws/s3.go

@@ -25,6 +25,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/endpoints"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 
 	"github.com/cortexlabs/cortex/pkg/lib/errors"
 	"github.com/cortexlabs/cortex/pkg/lib/files"
@@ -285,7 +286,21 @@ func SplitS3Path(s3Path string) (string, string, error) {
 	return bucket, key, nil
 }
 
-func IsS3PrefixExternal(bucket string, prefix string, region string) (bool, error) {
+func GetBucketRegion(bucket string) (string, error) {
+	sess := session.Must(session.NewSession())
+	region, err := s3manager.GetBucketRegion(aws.BackgroundContext(), sess, bucket, endpoints.UsWest2RegionID)
+	if err != nil {
+		return "", errors.Wrap(err, bucket)
+	}
+	return region, nil
+}
+
+func IsS3PrefixExternal(bucket string, prefix string) (bool, error) {
+	region, err := GetBucketRegion(bucket)
+	if err != nil {
+		return false, err
+	}
+
 	sess := session.Must(session.NewSession(&aws.Config{
 		Region: aws.String(region),
 	}))
@@ -296,19 +311,24 @@ func IsS3PrefixExternal(bucket string, prefix string, region string) (bool, erro
 	})
 
 	if err != nil {
-		return false, errors.Wrap(err, prefix)
+		return false, errors.Wrap(err, bucket, prefix)
 	}
 
 	hasPrefix := *out.KeyCount > 0
 	return hasPrefix, nil
 }
 
-func IsS3FileExternal(bucket string, key string, region string) (bool, error) {
+func IsS3FileExternal(bucket string, key string) (bool, error) {
+	region, err := GetBucketRegion(bucket)
+	if err != nil {
+		return false, err
+	}
+
 	sess := session.Must(session.NewSession(&aws.Config{
 		Region: aws.String(region),
 	}))
 
-	_, err := s3.New(sess).HeadObject(&s3.HeadObjectInput{
+	_, err = s3.New(sess).HeadObject(&s3.HeadObjectInput{
 		Bucket: aws.String(bucket),
 		Key:    aws.String(key),
 	})
@@ -318,16 +338,25 @@ func IsS3FileExternal(bucket string, key string, region string) (bool, error) {
 	}
 
 	if err != nil {
-		return false, errors.Wrap(err, key)
+		return false, errors.Wrap(err, bucket, key)
 	}
 
 	return true, nil
 }
 
-func IsS3aPrefixExternal(s3aPath string, region string) (bool, error) {
+func IsS3aPathPrefixExternal(s3aPath string) (bool, error) {
 	bucket, prefix, err := SplitS3aPath(s3aPath)
 	if err != nil {
 		return false, err
 	}
-	return IsS3PrefixExternal(bucket, prefix, region)
+	return IsS3PrefixExternal(bucket, prefix)
+}
+
+func IsS3PathFileExternal(s3Path string) (bool, error) {
+	bucket, key, err := SplitS3Path(s3Path)
+	if err != nil {
+		return false, err
+	}
+
+	return IsS3FileExternal(bucket, key)
 }

pkg/operator/api/context/apis.go

@@ -25,8 +25,7 @@ type APIs map[string]*API
 type API struct {
 	*userconfig.API
 	*ComputedResourceFields
-	Path      string `json:"path"`
-	ModelName string `json:"model_name"` // This removes the @ from userconfig.API.Model, or sets it to userconfig.API.ModelPath if it's external
+	Path string `json:"path"`
 }
 
 func APIPath(apiName string, appName string) string {

pkg/operator/api/context/dependencies.go

@@ -151,10 +151,11 @@ func (ctx *Context) modelDependencies(model *Model) strset.Set {
 }
 
 func (ctx *Context) apiDependencies(api *API) strset.Set {
-	if api.Model == nil {
+	modelName, ok := yaml.ExtractAtSymbolText(api.Model)
+	if !ok {
 		return strset.New()
 	}
-	model := ctx.Models[api.ModelName]
+	model := ctx.Models[modelName]
 	return strset.New(model.ID)
 }
 
@@ -210,7 +211,7 @@ func extractCortexResourcesHelper(
 		return
 	}
 
-	if resourceName, ok := yaml.ExtractAtSymbolText(input); ok {
+	if resourceName, ok := yaml.ExtractAtSymbolTextInter(input); ok {
 		for _, res := range validResourcesMap[resourceName] {
 			foundMatch := false
 			if len(resourceTypeFilter) == 0 || resourceTypeFilter[res.GetResourceType()] == true {

pkg/operator/api/userconfig/aggregates.go

@@ -108,11 +108,11 @@ func (aggregates Aggregates) Validate() error {
 
 func (aggregate *Aggregate) Validate() error {
 	if aggregate.AggregatorPath == nil && aggregate.Aggregator == "" {
-		return errors.Wrap(ErrorSpecifyOnlyOneMissing("aggregator", "aggregator_path"), Identify(aggregate))
+		return errors.Wrap(ErrorSpecifyOnlyOneMissing(AggregatorKey, AggregatorPathKey), Identify(aggregate))
 	}
 
 	if aggregate.AggregatorPath != nil && aggregate.Aggregator != "" {
-		return errors.Wrap(ErrorSpecifyOnlyOne("aggregator", "aggregator_path"), Identify(aggregate))
+		return errors.Wrap(ErrorSpecifyOnlyOne(AggregatorKey, AggregatorPathKey), Identify(aggregate))
 	}
 
 	return nil