Skip to content

Commit 6a1fb24

Browse files
1vn1vn
authored
Show config path in config errors (#14)
1 parent 91e3613 commit 6a1fb24

18 files changed

+258
-64
lines changed

‎pkg/api/context/models.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,10 @@ func (trainingDataset *TrainingDataset) GetResourceType() resource.Type {
5252
return resource.TrainingDatasetType
5353
}
5454

55+
func (trainingDataset *TrainingDataset) GetFilePath() string {
56+
return ""
57+
}
58+
5559
func (models Models) OneByID(id string) *Model {
5660
for _, model := range models {
5761
if model.ID == id {

‎pkg/api/userconfig/aggregates.go

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ type Aggregate struct {
3030
Inputs *Inputs `json:"inputs" yaml:"inputs"`
3131
Compute *SparkCompute `json:"compute" yaml:"compute"`
3232
Tags Tags `json:"tags" yaml:"tags"`
33+
FilePath string `json:"file_path" yaml:"-"`
3334
}
3435

3536
var aggregateValidation = &cr.StructValidation{
@@ -56,9 +57,14 @@ var aggregateValidation = &cr.StructValidation{
5657
}
5758

5859
func (aggregates Aggregates) Validate() error {
59-
dups := util.FindDuplicateStrs(aggregates.Names())
60+
resources := make([]Resource, len(aggregates))
61+
for i, res := range aggregates {
62+
resources[i] = res
63+
}
64+
65+
dups := FindDuplicateResourceName(resources...)
6066
if len(dups) > 0 {
61-
return ErrorDuplicateConfigName(dups[0], resource.AggregateType)
67+
return ErrorDuplicateResourceName(dups...)
6268
}
6369
return nil
6470
}
@@ -71,6 +77,10 @@ func (aggregate *Aggregate) GetResourceType() resource.Type {
7177
return resource.AggregateType
7278
}
7379

80+
func (aggregate *Aggregate) GetFilePath() string {
81+
return aggregate.FilePath
82+
}
83+
7484
func (aggregates Aggregates) Names() []string {
7585
names := make([]string, len(aggregates))
7686
for i, aggregate := range aggregates {

‎pkg/api/userconfig/aggregators.go

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ package userconfig
1919
import (
2020
"github.com/cortexlabs/cortex/pkg/api/resource"
2121
cr "github.com/cortexlabs/cortex/pkg/utils/configreader"
22-
"github.com/cortexlabs/cortex/pkg/utils/util"
2322
)
2423

2524
type Aggregators []*Aggregator
@@ -29,6 +28,7 @@ type Aggregator struct {
2928
Inputs *Inputs `json:"inputs" yaml:"inputs"`
3029
OutputType interface{} `json:"output_type" yaml:"output_type"`
3130
Path string `json:"path" yaml:"path"`
31+
FilePath string `json:"file_path" yaml:"-"`
3232
}
3333

3434
var aggregatorValidation = &cr.StructValidation{
@@ -63,9 +63,14 @@ var aggregatorValidation = &cr.StructValidation{
6363
}
6464

6565
func (aggregators Aggregators) Validate() error {
66-
dups := util.FindDuplicateStrs(aggregators.Names())
66+
resources := make([]Resource, len(aggregators))
67+
for i, res := range aggregators {
68+
resources[i] = res
69+
}
70+
71+
dups := FindDuplicateResourceName(resources...)
6772
if len(dups) > 0 {
68-
return ErrorDuplicateConfigName(dups[0], resource.AggregatorType)
73+
return ErrorDuplicateResourceName(dups...)
6974
}
7075
return nil
7176
}
@@ -87,6 +92,10 @@ func (aggregator *Aggregator) GetResourceType() resource.Type {
8792
return resource.AggregatorType
8893
}
8994

95+
func (aggregator *Aggregator) GetFilePath() string {
96+
return aggregator.FilePath
97+
}
98+
9099
func (aggregators Aggregators) Names() []string {
91100
names := make([]string, len(aggregators))
92101
for i, aggregator := range aggregators {

‎pkg/api/userconfig/apis.go

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ package userconfig
1919
import (
2020
"github.com/cortexlabs/cortex/pkg/api/resource"
2121
cr "github.com/cortexlabs/cortex/pkg/utils/configreader"
22-
"github.com/cortexlabs/cortex/pkg/utils/util"
2322
)
2423

2524
type APIs []*API
@@ -29,6 +28,7 @@ type API struct {
2928
ModelName string `json:"model_name" yaml:"model_name"`
3029
Compute *APICompute `json:"compute" yaml:"compute"`
3130
Tags Tags `json:"tags" yaml:"tags"`
31+
FilePath string `json:"file_path" yaml:"-"`
3232
}
3333

3434
var apiValidation = &cr.StructValidation{
@@ -55,9 +55,14 @@ var apiValidation = &cr.StructValidation{
5555
}
5656

5757
func (apis APIs) Validate() error {
58-
dups := util.FindDuplicateStrs(apis.Names())
58+
resources := make([]Resource, len(apis))
59+
for i, res := range apis {
60+
resources[i] = res
61+
}
62+
63+
dups := FindDuplicateResourceName(resources...)
5964
if len(dups) > 0 {
60-
return ErrorDuplicateConfigName(dups[0], resource.APIType)
65+
return ErrorDuplicateResourceName(dups...)
6166
}
6267
return nil
6368
}
@@ -70,6 +75,10 @@ func (api *API) GetResourceType() resource.Type {
7075
return resource.APIType
7176
}
7277

78+
func (api *API) GetFilePath() string {
79+
return api.FilePath
80+
}
81+
7382
func (apis APIs) Names() []string {
7483
names := make([]string, len(apis))
7584
for i, api := range apis {

‎pkg/api/userconfig/app.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ import (
2222
)
2323

2424
type App struct {
25-
Name string `json:"name" yaml:"name"`
25+
Name string `json:"name" yaml:"name"`
26+
FilePath string `json:"file_path" yaml:"-"`
2627
}
2728

2829
var appValidation = &cr.StructValidation{

‎pkg/api/userconfig/columns.go

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,18 @@ type Column interface {
3030
}
3131

3232
func (config *Config) ValidateColumns() error {
33-
dups := util.FindDuplicateStrs(config.ColumnNames())
33+
columnResources := make([]Resource, len(config.RawColumns)+len(config.TransformedColumns))
34+
for i, res := range config.RawColumns {
35+
columnResources[i] = res
36+
}
37+
38+
for i, res := range config.TransformedColumns {
39+
columnResources[i+len(config.RawColumns)] = res
40+
}
41+
42+
dups := FindDuplicateResourceName(columnResources...)
3443
if len(dups) > 0 {
35-
return ErrorDuplicateConfigName(dups[0], resource.RawColumnType, resource.TransformedColumnType)
44+
return ErrorDuplicateResourceName(dups...)
3645
}
3746

3847
for _, aggregate := range config.Aggregates {

‎pkg/api/userconfig/config.go

Lines changed: 28 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -45,15 +45,6 @@ type Config struct {
4545
Embeds Embeds `json:"embeds" yaml:"embeds"`
4646
}
4747

48-
type Resource interface {
49-
GetName() string
50-
GetResourceType() resource.Type
51-
}
52-
53-
func Identify(r Resource) string {
54-
return fmt.Sprintf("%s: %s", r.GetResourceType().String(), r.GetName())
55-
}
56-
5748
var typeFieldValidation = &cr.StructFieldValidation{
5849
Key: "kind",
5950
Nil: true,
@@ -235,13 +226,13 @@ func (config *Config) Validate(envName string) error {
235226
return nil
236227
}
237228

238-
func (config *Config) MergeBytes(configBytes []byte) (*Config, error) {
229+
func (config *Config) MergeBytes(configBytes []byte, filePath string) (*Config, error) {
239230
sliceData, err := cr.ReadYAMLBytes(configBytes)
240231
if err != nil {
241232
return nil, err
242233
}
243234

244-
subConfig, err := newPartial(sliceData)
235+
subConfig, err := newPartial(sliceData, filePath)
245236
if err != nil {
246237
return nil, err
247238
}
@@ -253,7 +244,7 @@ func (config *Config) MergeBytes(configBytes []byte) (*Config, error) {
253244
return config, nil
254245
}
255246

256-
func newPartial(configData interface{}) (*Config, error) {
247+
func newPartial(configData interface{}, filePath string) (*Config, error) {
257248
configDataSlice, ok := cast.InterfaceToStrInterfaceMapSlice(configData)
258249
if !ok {
259250
return nil, ErrorMalformedConfig()
@@ -271,52 +262,65 @@ func newPartial(configData interface{}) (*Config, error) {
271262
case resource.AppType:
272263
app := &App{}
273264
errs = cr.Struct(app, data, appValidation)
265+
app.FilePath = filePath
274266
config.App = app
275267
case resource.RawColumnType:
276268
var rawColumnInter interface{}
277269
rawColumnInter, errs = cr.InterfaceStruct(data, rawColumnValidation)
278270
if rawColumnInter != nil {
279-
config.RawColumns = append(config.RawColumns, rawColumnInter.(RawColumn))
271+
rawColumn := rawColumnInter.(RawColumn)
272+
rawColumn.SetFilePath(filePath)
273+
config.RawColumns = append(config.RawColumns, rawColumn)
280274
}
281275
case resource.TransformedColumnType:
282276
transformedColumn := &TransformedColumn{}
283277
errs = cr.Struct(transformedColumn, data, transformedColumnValidation)
278+
transformedColumn.FilePath = filePath
284279
config.TransformedColumns = append(config.TransformedColumns, transformedColumn)
285280
case resource.AggregateType:
286281
aggregate := &Aggregate{}
287282
errs = cr.Struct(aggregate, data, aggregateValidation)
283+
aggregate.FilePath = filePath
288284
config.Aggregates = append(config.Aggregates, aggregate)
289285
case resource.ConstantType:
290286
constant := &Constant{}
291287
errs = cr.Struct(constant, data, constantValidation)
288+
constant.FilePath = filePath
292289
config.Constants = append(config.Constants, constant)
293290
case resource.APIType:
294291
api := &API{}
295292
errs = cr.Struct(api, data, apiValidation)
293+
api.FilePath = filePath
296294
config.APIs = append(config.APIs, api)
297295
case resource.ModelType:
298296
model := &Model{}
299297
errs = cr.Struct(model, data, modelValidation)
298+
model.FilePath = filePath
300299
config.Models = append(config.Models, model)
301300
case resource.EnvironmentType:
302301
environment := &Environment{}
303302
errs = cr.Struct(environment, data, environmentValidation)
303+
environment.FilePath = filePath
304304
config.Environments = append(config.Environments, environment)
305305
case resource.AggregatorType:
306306
aggregator := &Aggregator{}
307307
errs = cr.Struct(aggregator, data, aggregatorValidation)
308+
aggregator.FilePath = filePath
308309
config.Aggregators = append(config.Aggregators, aggregator)
309310
case resource.TransformerType:
310311
transformer := &Transformer{}
311312
errs = cr.Struct(transformer, data, transformerValidation)
313+
transformer.FilePath = filePath
312314
config.Transformers = append(config.Transformers, transformer)
313315
case resource.TemplateType:
314316
template := &Template{}
315317
errs = cr.Struct(template, data, templateValidation)
318+
template.FilePath = filePath
316319
config.Templates = append(config.Templates, template)
317320
case resource.EmbedType:
318321
embed := &Embed{}
319322
errs = cr.Struct(embed, data, embedValidation)
323+
embed.FilePath = filePath
320324
config.Embeds = append(config.Embeds, embed)
321325
default:
322326
return nil, errors.Wrap(resource.ErrorUnknownKind(kindStr), "resource at "+s.Index(i))
@@ -339,29 +343,29 @@ func newPartial(configData interface{}) (*Config, error) {
339343
return config, nil
340344
}
341345

342-
func NewPartialPath(configPath string) (*Config, error) {
343-
configBytes, err := ioutil.ReadFile(configPath)
346+
func NewPartialPath(filePath string) (*Config, error) {
347+
configBytes, err := ioutil.ReadFile(filePath)
344348
if err != nil {
345-
return nil, errors.Wrap(err, configPath, ErrorReadConfig().Error())
349+
return nil, errors.Wrap(err, filePath, ErrorReadConfig().Error())
346350
}
347351

348352
configData, err := cr.ReadYAMLBytes(configBytes)
349353
if err != nil {
350-
return nil, errors.Wrap(err, configPath, ErrorParseConfig().Error())
354+
return nil, errors.Wrap(err, filePath, ErrorParseConfig().Error())
351355
}
352-
return newPartial(configData)
356+
return newPartial(configData, filePath)
353357
}
354358

355359
func New(configs map[string][]byte, envName string) (*Config, error) {
356360
var err error
357361
config := &Config{}
358-
for configPath, configBytes := range configs {
359-
if !util.IsFilePathYAML(configPath) {
362+
for filePath, configBytes := range configs {
363+
if !util.IsFilePathYAML(filePath) {
360364
continue
361365
}
362-
config, err = config.MergeBytes(configBytes)
366+
config, err = config.MergeBytes(configBytes, filePath)
363367
if err != nil {
364-
return nil, errors.Wrap(err, configPath)
368+
return nil, err
365369
}
366370
}
367371

@@ -374,10 +378,10 @@ func New(configs map[string][]byte, envName string) (*Config, error) {
374378

375379
populatedTemplate, err := template.Populate(emb)
376380
if err != nil {
377-
return nil, errors.Wrap(err, resource.EmbedType.String())
381+
return nil, errors.Wrap(err, emb.FilePath, resource.EmbedType.String())
378382
}
379383

380-
config, err = config.MergeBytes([]byte(populatedTemplate))
384+
config, err = config.MergeBytes([]byte(populatedTemplate), emb.FilePath)
381385
if err != nil {
382386
return nil, errors.Wrap(err, resource.EmbedType.String(), fmt.Sprintf("%s %s", resource.TemplateType.String(), s.UserStr(template.Name)))
383387
}

‎pkg/api/userconfig/constants.go

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,16 +20,16 @@ import (
2020
"github.com/cortexlabs/cortex/pkg/api/resource"
2121
cr "github.com/cortexlabs/cortex/pkg/utils/configreader"
2222
"github.com/cortexlabs/cortex/pkg/utils/errors"
23-
"github.com/cortexlabs/cortex/pkg/utils/util"
2423
)
2524

2625
type Constants []*Constant
2726

2827
type Constant struct {
29-
Name string `json:"name" yaml:"name"`
30-
Type interface{} `json:"type" yaml:"type"`
31-
Value interface{} `json:"value" yaml:"value"`
32-
Tags Tags `json:"tags" yaml:"tags"`
28+
Name string `json:"name" yaml:"name"`
29+
Type interface{} `json:"type" yaml:"type"`
30+
Value interface{} `json:"value" yaml:"value"`
31+
Tags Tags `json:"tags" yaml:"tags"`
32+
FilePath string `json:"file_path" yaml:"-"`
3333
}
3434

3535
var constantValidation = &cr.StructValidation{
@@ -71,9 +71,14 @@ func (constants Constants) Validate() error {
7171
}
7272
}
7373

74-
dups := util.FindDuplicateStrs(constants.Names())
74+
resources := make([]Resource, len(constants))
75+
for i, res := range constants {
76+
resources[i] = res
77+
}
78+
79+
dups := FindDuplicateResourceName(resources...)
7580
if len(dups) > 0 {
76-
return ErrorDuplicateConfigName(dups[0], resource.ConstantType)
81+
return ErrorDuplicateResourceName(dups...)
7782
}
7883

7984
return nil
@@ -101,6 +106,10 @@ func (constant *Constant) GetResourceType() resource.Type {
101106
return resource.ConstantType
102107
}
103108

109+
func (constant *Constant) GetFilePath() string {
110+
return constant.FilePath
111+
}
112+
104113
func (constants Constants) Names() []string {
105114
names := make([]string, len(constants))
106115
for i, constant := range constants {

‎pkg/api/userconfig/embed.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ type Embeds []*Embed
2525
type Embed struct {
2626
Template string `json:"template" yaml:"template"`
2727
Args map[string]interface{} `json:"args" yaml:"args"`
28+
FilePath string `json:"file_path" yaml:"-"`
2829
}
2930

3031
var embedValidation = &cr.StructValidation{

0 commit comments

Comments
 (0)