Add index to resource configs, centralize resource config fields

pkg/api/context/models.go

@@ -36,32 +36,18 @@ type Model struct {
 }
 
 type TrainingDataset struct {
+	userconfig.ResourceConfigFields
 	*ComputedResourceFields
-	Name        string            `json:"name"`
-	ModelName   string            `json:"model_name"`
-	TrainKey    string            `json:"train_key"`
-	EvalKey     string            `json:"eval_key"`
-	MetadataKey string            `json:"metadata_key"`
-	FilePath    string            `json:"file_path"`
-	Embed       *userconfig.Embed `json:"embed"`
-}
-
-func (trainingDataset *TrainingDataset) GetName() string {
-	return trainingDataset.Name
+	ModelName   string `json:"model_name"`
+	TrainKey    string `json:"train_key"`
+	EvalKey     string `json:"eval_key"`
+	MetadataKey string `json:"metadata_key"`
 }
 
 func (trainingDataset *TrainingDataset) GetResourceType() resource.Type {
 	return resource.TrainingDatasetType
 }
 
-func (trainingDataset *TrainingDataset) GetFilePath() string {
-	return trainingDataset.FilePath
-}
-
-func (trainingDataset *TrainingDataset) GetEmbed() *userconfig.Embed {
-	return trainingDataset.Embed
-}
-
 func (models Models) OneByID(id string) *Model {
 	for _, model := range models {
 		if model.ID == id {

pkg/api/context/python_packages.go

@@ -24,24 +24,12 @@ import (
 type PythonPackages map[string]*PythonPackage
 
 type PythonPackage struct {
-	Name       string `json:"name"`
+	userconfig.ResourceConfigFields
+	*ComputedResourceFields
 	SrcKey     string `json:"src_key"`
 	PackageKey string `json:"package_key"`
-	*ComputedResourceFields
-}
-
-func (pythonPackage *PythonPackage) GetName() string {
-	return pythonPackage.Name
 }
 
 func (pythonPackage *PythonPackage) GetResourceType() resource.Type {
 	return resource.PythonPackageType
 }
-
-func (pythonPackage *PythonPackage) GetFilePath() string {
-	return ""
-}
-
-func (pythonPackage *PythonPackage) GetEmbed() *userconfig.Embed {
-	return nil
-}

pkg/api/userconfig/aggregates.go

@@ -25,13 +25,11 @@ import (
 type Aggregates []*Aggregate
 
 type Aggregate struct {
-	Name       string        `json:"name" yaml:"name"`
+	ResourceConfigFields
 	Aggregator string        `json:"aggregator" yaml:"aggregator"`
 	Inputs     *Inputs       `json:"inputs" yaml:"inputs"`
 	Compute    *SparkCompute `json:"compute" yaml:"compute"`
 	Tags       Tags          `json:"tags" yaml:"tags"`
-	FilePath   string        `json:"file_path"  yaml:"-"`
-	Embed      *Embed        `json:"embed"  yaml:"-"`
 }
 
 var aggregateValidation = &cr.StructValidation{
@@ -70,22 +68,10 @@ func (aggregates Aggregates) Validate() error {
 	return nil
 }
 
-func (aggregate *Aggregate) GetName() string {
-	return aggregate.Name
-}
-
 func (aggregate *Aggregate) GetResourceType() resource.Type {
 	return resource.AggregateType
 }
 
-func (aggregate *Aggregate) GetFilePath() string {
-	return aggregate.FilePath
-}
-
-func (aggregate *Aggregate) GetEmbed() *Embed {
-	return aggregate.Embed
-}
-
 func (aggregates Aggregates) Names() []string {
 	names := make([]string, len(aggregates))
 	for i, aggregate := range aggregates {

pkg/api/userconfig/aggregators.go

@@ -24,12 +24,10 @@ import (
 type Aggregators []*Aggregator
 
 type Aggregator struct {
-	Name       string      `json:"name"  yaml:"name"`
+	ResourceConfigFields
 	Inputs     *Inputs     `json:"inputs"  yaml:"inputs"`
 	OutputType interface{} `json:"output_type"  yaml:"output_type"`
 	Path       string      `json:"path"  yaml:"path"`
-	FilePath   string      `json:"file_path" yaml:"-"`
-	Embed      *Embed      `json:"embed"  yaml:"-"`
 }
 
 var aggregatorValidation = &cr.StructValidation{
@@ -85,22 +83,10 @@ func (aggregators Aggregators) Get(name string) *Aggregator {
 	return nil
 }
 
-func (aggregator *Aggregator) GetName() string {
-	return aggregator.Name
-}
-
 func (aggregator *Aggregator) GetResourceType() resource.Type {
 	return resource.AggregatorType
 }
 
-func (aggregator *Aggregator) GetFilePath() string {
-	return aggregator.FilePath
-}
-
-func (aggregator *Aggregator) GetEmbed() *Embed {
-	return aggregator.Embed
-}
-
 func (aggregators Aggregators) Names() []string {
 	names := make([]string, len(aggregators))
 	for i, aggregator := range aggregators {

pkg/api/userconfig/apis.go

@@ -24,12 +24,10 @@ import (
 type APIs []*API
 
 type API struct {
-	Name      string      `json:"name" yaml:"name"`
+	ResourceConfigFields
 	ModelName string      `json:"model_name" yaml:"model_name"`
 	Compute   *APICompute `json:"compute" yaml:"compute"`
 	Tags      Tags        `json:"tags" yaml:"tags"`
-	FilePath  string      `json:"file_path"  yaml:"-"`
-	Embed     *Embed      `json:"embed"  yaml:"-"`
 }
 
 var apiValidation = &cr.StructValidation{
@@ -68,22 +66,10 @@ func (apis APIs) Validate() error {
 	return nil
 }
 
-func (api *API) GetName() string {
-	return api.Name
-}
-
 func (api *API) GetResourceType() resource.Type {
 	return resource.APIType
 }
 
-func (api *API) GetFilePath() string {
-	return api.FilePath
-}
-
-func (api *API) GetEmbed() *Embed {
-	return api.Embed
-}
-
 func (apis APIs) Names() []string {
 	names := make([]string, len(apis))
 	for i, api := range apis {

pkg/api/userconfig/app.go

@@ -21,8 +21,7 @@ import (
 )
 
 type App struct {
-	Name     string `json:"name" yaml:"name"`
-	FilePath string `json:"file_path"  yaml:"-"`
+	Name string `json:"name" yaml:"name"`
 }
 
 var appValidation = &cr.StructValidation{

pkg/api/userconfig/columns.go

@@ -25,7 +25,7 @@ import (
 )
 
 type Column interface {
-	GetName() string
+	Resource
 	IsRaw() bool
 }
 

pkg/api/userconfig/config.go

@@ -271,106 +271,85 @@ func newPartial(configData interface{}, filePath string, emb *Embed, template *T
 
 		var errs []error
 		resourceType := resource.TypeFromKindString(kindStr)
+		var newResource Resource
 		switch resourceType {
 		case resource.AppType:
 			app := &App{}
 			errs = cr.Struct(app, data, appValidation)
-			app.FilePath = filePath
 			config.App = app
 		case resource.RawColumnType:
 			var rawColumnInter interface{}
 			rawColumnInter, errs = cr.InterfaceStruct(data, rawColumnValidation)
 			if !errors.HasErrors(errs) && rawColumnInter != nil {
-				rawColumn := rawColumnInter.(RawColumn)
-				rawColumn.SetFilePath(filePath)
-				rawColumn.SetEmbed(emb)
-				config.RawColumns = append(config.RawColumns, rawColumn)
+				newResource = rawColumnInter.(RawColumn)
+				config.RawColumns = append(config.RawColumns, newResource.(RawColumn))
 			}
 		case resource.TransformedColumnType:
-			transformedColumn := &TransformedColumn{}
-			errs = cr.Struct(transformedColumn, data, transformedColumnValidation)
+			newResource = &TransformedColumn{}
+			errs = cr.Struct(newResource, data, transformedColumnValidation)
 			if !errors.HasErrors(errs) {
-				transformedColumn.FilePath = filePath
-				transformedColumn.Embed = emb
-				config.TransformedColumns = append(config.TransformedColumns, transformedColumn)
+				config.TransformedColumns = append(config.TransformedColumns, newResource.(*TransformedColumn))
 			}
 		case resource.AggregateType:
-			aggregate := &Aggregate{}
-			errs = cr.Struct(aggregate, data, aggregateValidation)
+			newResource = &Aggregate{}
+			errs = cr.Struct(newResource, data, aggregateValidation)
 			if !errors.HasErrors(errs) {
-				aggregate.FilePath = filePath
-				aggregate.Embed = emb
-				config.Aggregates = append(config.Aggregates, aggregate)
+				config.Aggregates = append(config.Aggregates, newResource.(*Aggregate))
 			}
 		case resource.ConstantType:
-			constant := &Constant{}
-			errs = cr.Struct(constant, data, constantValidation)
+			newResource = &Constant{}
+			errs = cr.Struct(newResource, data, constantValidation)
 			if !errors.HasErrors(errs) {
-				constant.FilePath = filePath
-				constant.Embed = emb
-				config.Constants = append(config.Constants, constant)
+				config.Constants = append(config.Constants, newResource.(*Constant))
 			}
 		case resource.APIType:
-			api := &API{}
-			errs = cr.Struct(api, data, apiValidation)
+			newResource = &API{}
+			errs = cr.Struct(newResource, data, apiValidation)
 			if !errors.HasErrors(errs) {
-				api.FilePath = filePath
-				api.Embed = emb
-				config.APIs = append(config.APIs, api)
+				config.APIs = append(config.APIs, newResource.(*API))
 			}
 		case resource.ModelType:
-			model := &Model{}
-			errs = cr.Struct(model, data, modelValidation)
+			newResource = &Model{}
+			errs = cr.Struct(newResource, data, modelValidation)
 			if !errors.HasErrors(errs) {
-				model.FilePath = filePath
-				model.Embed = emb
-				config.Models = append(config.Models, model)
+				config.Models = append(config.Models, newResource.(*Model))
 			}
 		case resource.EnvironmentType:
-			environment := &Environment{}
-			errs = cr.Struct(environment, data, environmentValidation)
+			newResource = &Environment{}
+			errs = cr.Struct(newResource, data, environmentValidation)
 			if !errors.HasErrors(errs) {
-				environment.FilePath = filePath
-				environment.Embed = emb
-				config.Environments = append(config.Environments, environment)
+				config.Environments = append(config.Environments, newResource.(*Environment))
 			}
 		case resource.AggregatorType:
-			aggregator := &Aggregator{}
-			errs = cr.Struct(aggregator, data, aggregatorValidation)
+			newResource = &Aggregator{}
+			errs = cr.Struct(newResource, data, aggregatorValidation)
 			if !errors.HasErrors(errs) {
-				aggregator.FilePath = filePath
-				aggregator.Embed = emb
-				config.Aggregators = append(config.Aggregators, aggregator)
+				config.Aggregators = append(config.Aggregators, newResource.(*Aggregator))
 			}
 		case resource.TransformerType:
-			transformer := &Transformer{}
-			errs = cr.Struct(transformer, data, transformerValidation)
+			newResource = &Transformer{}
+			errs = cr.Struct(newResource, data, transformerValidation)
 			if !errors.HasErrors(errs) {
-				transformer.FilePath = filePath
-				transformer.Embed = emb
-				config.Transformers = append(config.Transformers, transformer)
+				config.Transformers = append(config.Transformers, newResource.(*Transformer))
 			}
 		case resource.TemplateType:
 			if emb != nil {
 				errs = []error{resource.ErrorTemplateInTemplate()}
 			} else {
-				template := &Template{}
-				errs = cr.Struct(template, data, templateValidation)
+				newResource = &Template{}
+				errs = cr.Struct(newResource, data, templateValidation)
 				if !errors.HasErrors(errs) {
-					template.FilePath = filePath
-					config.Templates = append(config.Templates, template)
+					config.Templates = append(config.Templates, newResource.(*Template))
 				}
 			}
 		case resource.EmbedType:
 			if emb != nil {
 				errs = []error{resource.ErrorEmbedInTemplate()}
 			} else {
-				embed := &Embed{}
-				errs = cr.Struct(embed, data, embedValidation)
+				newResource = &Embed{}
+				errs = cr.Struct(newResource, data, embedValidation)
 				if !errors.HasErrors(errs) {
-					embed.FilePath = filePath
-					embed.ConfigIndex = i
-					config.Embeds = append(config.Embeds, embed)
+					config.Embeds = append(config.Embeds, newResource.(*Embed))
 				}
 			}
 		default:
@@ -381,6 +360,12 @@ func newPartial(configData interface{}, filePath string, emb *Embed, template *T
 			name, _ := data[NameKey].(string)
 			return nil, errors.Wrap(errors.FirstError(errs...), identifyHelper(filePath, resourceType, name, i, emb))
 		}
+
+		if newResource != nil {
+			newResource.SetIndex(i)
+			newResource.SetFilePath(filePath)
+			newResource.SetEmbed(emb)
+		}
 	}
 
 	err := config.ValidatePartial()
@@ -421,12 +406,12 @@ func New(configs map[string][]byte, envName string) (*Config, error) {
 	for _, emb := range config.Embeds {
 		template, ok := templates[emb.Template]
 		if !ok {
-			return nil, errors.Wrap(ErrorUndefinedResource(emb.Template, resource.TemplateType), emb.Identify())
+			return nil, errors.Wrap(ErrorUndefinedResource(emb.Template, resource.TemplateType), Identify(emb))
 		}
 
 		populatedTemplate, err := template.Populate(emb)
 		if err != nil {
-			return nil, errors.Wrap(err, emb.Identify())
+			return nil, errors.Wrap(err, Identify(emb))
 		}
 
 		config, err = config.MergeBytes([]byte(populatedTemplate), emb.FilePath, emb, template)