Show config path in config errors

pkg/api/context/models.go

@@ -52,6 +52,10 @@ func (trainingDataset *TrainingDataset) GetResourceType() resource.Type {
 	return resource.TrainingDatasetType
 }
 
+func (trainingDataset *TrainingDataset) GetFilePath() string {
+	return ""
+}
+
 func (models Models) OneByID(id string) *Model {
 	for _, model := range models {
 		if model.ID == id {

pkg/api/userconfig/aggregates.go

@@ -30,6 +30,7 @@ type Aggregate struct {
 	Inputs     *Inputs       `json:"inputs" yaml:"inputs"`
 	Compute    *SparkCompute `json:"compute" yaml:"compute"`
 	Tags       Tags          `json:"tags" yaml:"tags"`
+	FilePath   string        `json:"file_path"  yaml:"-"`
 }
 
 var aggregateValidation = &cr.StructValidation{
@@ -56,9 +57,14 @@ var aggregateValidation = &cr.StructValidation{
 }
 
 func (aggregates Aggregates) Validate() error {
-	dups := util.FindDuplicateStrs(aggregates.Names())
+	resources := make([]Resource, len(aggregates))
+	for i, res := range aggregates {
+		resources[i] = res
+	}
+
+	dups := FindDuplicateResourceName(resources...)
 	if len(dups) > 0 {
-		return ErrorDuplicateConfigName(dups[0], resource.AggregateType)
+		return ErrorDuplicateResourceName(dups...)
 	}
 	return nil
 }
@@ -71,6 +77,10 @@ func (aggregate *Aggregate) GetResourceType() resource.Type {
 	return resource.AggregateType
 }
 
+func (aggregate *Aggregate) GetFilePath() string {
+	return aggregate.FilePath
+}
+
 func (aggregates Aggregates) Names() []string {
 	names := make([]string, len(aggregates))
 	for i, aggregate := range aggregates {

pkg/api/userconfig/aggregators.go

@@ -19,7 +19,6 @@ package userconfig
 import (
 	"github.com/cortexlabs/cortex/pkg/api/resource"
 	cr "github.com/cortexlabs/cortex/pkg/utils/configreader"
-	"github.com/cortexlabs/cortex/pkg/utils/util"
 )
 
 type Aggregators []*Aggregator
@@ -29,6 +28,7 @@ type Aggregator struct {
 	Inputs     *Inputs     `json:"inputs"  yaml:"inputs"`
 	OutputType interface{} `json:"output_type"  yaml:"output_type"`
 	Path       string      `json:"path"  yaml:"path"`
+	FilePath   string      `json:"file_path" yaml:"-"`
 }
 
 var aggregatorValidation = &cr.StructValidation{
@@ -63,9 +63,14 @@ var aggregatorValidation = &cr.StructValidation{
 }
 
 func (aggregators Aggregators) Validate() error {
-	dups := util.FindDuplicateStrs(aggregators.Names())
+	resources := make([]Resource, len(aggregators))
+	for i, res := range aggregators {
+		resources[i] = res
+	}
+
+	dups := FindDuplicateResourceName(resources...)
 	if len(dups) > 0 {
-		return ErrorDuplicateConfigName(dups[0], resource.AggregatorType)
+		return ErrorDuplicateResourceName(dups...)
 	}
 	return nil
 }
@@ -87,6 +92,10 @@ func (aggregator *Aggregator) GetResourceType() resource.Type {
 	return resource.AggregatorType
 }
 
+func (aggregator *Aggregator) GetFilePath() string {
+	return aggregator.FilePath
+}
+
 func (aggregators Aggregators) Names() []string {
 	names := make([]string, len(aggregators))
 	for i, aggregator := range aggregators {

pkg/api/userconfig/apis.go

@@ -19,7 +19,6 @@ package userconfig
 import (
 	"github.com/cortexlabs/cortex/pkg/api/resource"
 	cr "github.com/cortexlabs/cortex/pkg/utils/configreader"
-	"github.com/cortexlabs/cortex/pkg/utils/util"
 )
 
 type APIs []*API
@@ -29,6 +28,7 @@ type API struct {
 	ModelName string      `json:"model_name" yaml:"model_name"`
 	Compute   *APICompute `json:"compute" yaml:"compute"`
 	Tags      Tags        `json:"tags" yaml:"tags"`
+	FilePath  string      `json:"file_path"  yaml:"-"`
 }
 
 var apiValidation = &cr.StructValidation{
@@ -55,9 +55,14 @@ var apiValidation = &cr.StructValidation{
 }
 
 func (apis APIs) Validate() error {
-	dups := util.FindDuplicateStrs(apis.Names())
+	resources := make([]Resource, len(apis))
+	for i, res := range apis {
+		resources[i] = res
+	}
+
+	dups := FindDuplicateResourceName(resources...)
 	if len(dups) > 0 {
-		return ErrorDuplicateConfigName(dups[0], resource.APIType)
+		return ErrorDuplicateResourceName(dups...)
 	}
 	return nil
 }
@@ -70,6 +75,10 @@ func (api *API) GetResourceType() resource.Type {
 	return resource.APIType
 }
 
+func (api *API) GetFilePath() string {
+	return api.FilePath
+}
+
 func (apis APIs) Names() []string {
 	names := make([]string, len(apis))
 	for i, api := range apis {

pkg/api/userconfig/app.go

@@ -22,7 +22,8 @@ import (
 )
 
 type App struct {
-	Name string `json:"name" yaml:"name"`
+	Name     string `json:"name" yaml:"name"`
+	FilePath string `json:"file_path"  yaml:"-"`
 }
 
 var appValidation = &cr.StructValidation{

pkg/api/userconfig/columns.go

@@ -30,9 +30,18 @@ type Column interface {
 }
 
 func (config *Config) ValidateColumns() error {
-	dups := util.FindDuplicateStrs(config.ColumnNames())
+	columnResources := make([]Resource, len(config.RawColumns)+len(config.TransformedColumns))
+	for i, res := range config.RawColumns {
+		columnResources[i] = res
+	}
+
+	for i, res := range config.TransformedColumns {
+		columnResources[i+len(config.RawColumns)] = res
+	}
+
+	dups := FindDuplicateResourceName(columnResources...)
 	if len(dups) > 0 {
-		return ErrorDuplicateConfigName(dups[0], resource.RawColumnType, resource.TransformedColumnType)
+		return ErrorDuplicateResourceName(dups...)
 	}
 
 	for _, aggregate := range config.Aggregates {

pkg/api/userconfig/config.go

@@ -45,15 +45,6 @@ type Config struct {
 	Embeds             Embeds             `json:"embeds" yaml:"embeds"`
 }
 
-type Resource interface {
-	GetName() string
-	GetResourceType() resource.Type
-}
-
-func Identify(r Resource) string {
-	return fmt.Sprintf("%s: %s", r.GetResourceType().String(), r.GetName())
-}
-
 var typeFieldValidation = &cr.StructFieldValidation{
 	Key: "kind",
 	Nil: true,
@@ -235,13 +226,13 @@ func (config *Config) Validate(envName string) error {
 	return nil
 }
 
-func (config *Config) MergeBytes(configBytes []byte) (*Config, error) {
+func (config *Config) MergeBytes(configBytes []byte, filePath string) (*Config, error) {
 	sliceData, err := cr.ReadYAMLBytes(configBytes)
 	if err != nil {
 		return nil, err
 	}
 
-	subConfig, err := newPartial(sliceData)
+	subConfig, err := newPartial(sliceData, filePath)
 	if err != nil {
 		return nil, err
 	}
@@ -253,7 +244,7 @@ func (config *Config) MergeBytes(configBytes []byte) (*Config, error) {
 	return config, nil
 }
 
-func newPartial(configData interface{}) (*Config, error) {
+func newPartial(configData interface{}, filePath string) (*Config, error) {
 	configDataSlice, ok := cast.InterfaceToStrInterfaceMapSlice(configData)
 	if !ok {
 		return nil, ErrorMalformedConfig()
@@ -271,52 +262,65 @@ func newPartial(configData interface{}) (*Config, error) {
 		case resource.AppType:
 			app := &App{}
 			errs = cr.Struct(app, data, appValidation)
+			app.FilePath = filePath
 			config.App = app
 		case resource.RawColumnType:
 			var rawColumnInter interface{}
 			rawColumnInter, errs = cr.InterfaceStruct(data, rawColumnValidation)
 			if rawColumnInter != nil {
-				config.RawColumns = append(config.RawColumns, rawColumnInter.(RawColumn))
+				rawColumn := rawColumnInter.(RawColumn)
+				rawColumn.SetFilePath(filePath)
+				config.RawColumns = append(config.RawColumns, rawColumn)
 			}
 		case resource.TransformedColumnType:
 			transformedColumn := &TransformedColumn{}
 			errs = cr.Struct(transformedColumn, data, transformedColumnValidation)
+			transformedColumn.FilePath = filePath
 			config.TransformedColumns = append(config.TransformedColumns, transformedColumn)
 		case resource.AggregateType:
 			aggregate := &Aggregate{}
 			errs = cr.Struct(aggregate, data, aggregateValidation)
+			aggregate.FilePath = filePath
 			config.Aggregates = append(config.Aggregates, aggregate)
 		case resource.ConstantType:
 			constant := &Constant{}
 			errs = cr.Struct(constant, data, constantValidation)
+			constant.FilePath = filePath
 			config.Constants = append(config.Constants, constant)
 		case resource.APIType:
 			api := &API{}
 			errs = cr.Struct(api, data, apiValidation)
+			api.FilePath = filePath
 			config.APIs = append(config.APIs, api)
 		case resource.ModelType:
 			model := &Model{}
 			errs = cr.Struct(model, data, modelValidation)
+			model.FilePath = filePath
 			config.Models = append(config.Models, model)
 		case resource.EnvironmentType:
 			environment := &Environment{}
 			errs = cr.Struct(environment, data, environmentValidation)
+			environment.FilePath = filePath
 			config.Environments = append(config.Environments, environment)
 		case resource.AggregatorType:
 			aggregator := &Aggregator{}
 			errs = cr.Struct(aggregator, data, aggregatorValidation)
+			aggregator.FilePath = filePath
 			config.Aggregators = append(config.Aggregators, aggregator)
 		case resource.TransformerType:
 			transformer := &Transformer{}
 			errs = cr.Struct(transformer, data, transformerValidation)
+			transformer.FilePath = filePath
 			config.Transformers = append(config.Transformers, transformer)
 		case resource.TemplateType:
 			template := &Template{}
 			errs = cr.Struct(template, data, templateValidation)
+			template.FilePath = filePath
 			config.Templates = append(config.Templates, template)
 		case resource.EmbedType:
 			embed := &Embed{}
 			errs = cr.Struct(embed, data, embedValidation)
+			embed.FilePath = filePath
 			config.Embeds = append(config.Embeds, embed)
 		default:
 			return nil, errors.Wrap(resource.ErrorUnknownKind(kindStr), "resource at "+s.Index(i))
@@ -339,29 +343,29 @@ func newPartial(configData interface{}) (*Config, error) {
 	return config, nil
 }
 
-func NewPartialPath(configPath string) (*Config, error) {
-	configBytes, err := ioutil.ReadFile(configPath)
+func NewPartialPath(filePath string) (*Config, error) {
+	configBytes, err := ioutil.ReadFile(filePath)
 	if err != nil {
-		return nil, errors.Wrap(err, configPath, ErrorReadConfig().Error())
+		return nil, errors.Wrap(err, filePath, ErrorReadConfig().Error())
 	}
 
 	configData, err := cr.ReadYAMLBytes(configBytes)
 	if err != nil {
-		return nil, errors.Wrap(err, configPath, ErrorParseConfig().Error())
+		return nil, errors.Wrap(err, filePath, ErrorParseConfig().Error())
 	}
-	return newPartial(configData)
+	return newPartial(configData, filePath)
 }
 
 func New(configs map[string][]byte, envName string) (*Config, error) {
 	var err error
 	config := &Config{}
-	for configPath, configBytes := range configs {
-		if !util.IsFilePathYAML(configPath) {
+	for filePath, configBytes := range configs {
+		if !util.IsFilePathYAML(filePath) {
 			continue
 		}
-		config, err = config.MergeBytes(configBytes)
+		config, err = config.MergeBytes(configBytes, filePath)
 		if err != nil {
-			return nil, errors.Wrap(err, configPath)
+			return nil, err
 		}
 	}
 
@@ -374,10 +378,10 @@ func New(configs map[string][]byte, envName string) (*Config, error) {
 
 		populatedTemplate, err := template.Populate(emb)
 		if err != nil {
-			return nil, errors.Wrap(err, resource.EmbedType.String())
+			return nil, errors.Wrap(err, emb.FilePath, resource.EmbedType.String())
 		}
 
-		config, err = config.MergeBytes([]byte(populatedTemplate))
+		config, err = config.MergeBytes([]byte(populatedTemplate), emb.FilePath)
 		if err != nil {
 			return nil, errors.Wrap(err, resource.EmbedType.String(), fmt.Sprintf("%s %s", resource.TemplateType.String(), s.UserStr(template.Name)))
 		}

pkg/api/userconfig/constants.go

@@ -20,16 +20,16 @@ import (
 	"github.com/cortexlabs/cortex/pkg/api/resource"
 	cr "github.com/cortexlabs/cortex/pkg/utils/configreader"
 	"github.com/cortexlabs/cortex/pkg/utils/errors"
-	"github.com/cortexlabs/cortex/pkg/utils/util"
 )
 
 type Constants []*Constant
 
 type Constant struct {
-	Name  string      `json:"name" yaml:"name"`
-	Type  interface{} `json:"type" yaml:"type"`
-	Value interface{} `json:"value" yaml:"value"`
-	Tags  Tags        `json:"tags" yaml:"tags"`
+	Name     string      `json:"name" yaml:"name"`
+	Type     interface{} `json:"type" yaml:"type"`
+	Value    interface{} `json:"value" yaml:"value"`
+	Tags     Tags        `json:"tags" yaml:"tags"`
+	FilePath string      `json:"file_path"  yaml:"-"`
 }
 
 var constantValidation = &cr.StructValidation{
@@ -71,9 +71,14 @@ func (constants Constants) Validate() error {
 		}
 	}
 
-	dups := util.FindDuplicateStrs(constants.Names())
+	resources := make([]Resource, len(constants))
+	for i, res := range constants {
+		resources[i] = res
+	}
+
+	dups := FindDuplicateResourceName(resources...)
 	if len(dups) > 0 {
-		return ErrorDuplicateConfigName(dups[0], resource.ConstantType)
+		return ErrorDuplicateResourceName(dups...)
 	}
 
 	return nil
@@ -101,6 +106,10 @@ func (constant *Constant) GetResourceType() resource.Type {
 	return resource.ConstantType
 }
 
+func (constant *Constant) GetFilePath() string {
+	return constant.FilePath
+}
+
 func (constants Constants) Names() []string {
 	names := make([]string, len(constants))
 	for i, constant := range constants {

pkg/api/userconfig/embed.go

@@ -25,6 +25,7 @@ type Embeds []*Embed
 type Embed struct {
 	Template string                 `json:"template" yaml:"template"`
 	Args     map[string]interface{} `json:"args" yaml:"args"`
+	FilePath string                 `json:"file_path"  yaml:"-"`
 }
 
 var embedValidation = &cr.StructValidation{