Skip to content

Commit 5e301c6

Browse files
deliahudeliahu
committed
Improve get command resource printing
1 parent ff4910a commit 5e301c6

File tree

15 files changed

+361
-35
lines changed

15 files changed

+361
-35
lines changed

‎Makefile

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,9 @@ cli:
8686
@mkdir -p ./bin
8787
@GOARCH=amd64 CGO_ENABLED=0 go build -o ./bin/cortex ./cli
8888

89+
cli-watch:
90+
@rerun -watch ./pkg ./cli -ignore ./vendor ./bin -run sh -c "go build -installsuffix cgo -o ./bin/cortex ./cli && echo 'CLI built.'"
91+
8992
aws-clear-bucket:
9093
@./dev/aws.sh clear-bucket
9194

‎cli/cmd/get.go

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -304,7 +304,7 @@ func describeRawColumn(name string, resourcesRes *schema.GetResourcesResponse) (
304304
}
305305
dataStatus := resourcesRes.DataStatuses[rawColumn.GetID()]
306306
out := dataStatusSummary(dataStatus)
307-
out += resourceStr(context.GetRawColumnUserConfig(rawColumn))
307+
out += titleStr("Configuration") + rawColumn.UserConfigStr()
308308
return out, nil
309309
}
310310

@@ -336,7 +336,7 @@ func describeAggregate(name string, resourcesRes *schema.GetResourcesResponse) (
336336
out += valueStr(obj)
337337
}
338338

339-
out += resourceStr(aggregate.Aggregate)
339+
out += titleStr("Configuration") + aggregate.UserConfigStr()
340340
return out, nil
341341
}
342342

@@ -347,7 +347,7 @@ func describeTransformedColumn(name string, resourcesRes *schema.GetResourcesRes
347347
}
348348
dataStatus := resourcesRes.DataStatuses[transformedColumn.ID]
349349
out := dataStatusSummary(dataStatus)
350-
out += resourceStr(transformedColumn.TransformedColumn)
350+
out += titleStr("Configuration") + transformedColumn.UserConfigStr()
351351
return out, nil
352352
}
353353

@@ -367,7 +367,7 @@ func describeModel(name string, resourcesRes *schema.GetResourcesResponse) (stri
367367
}
368368
dataStatus := resourcesRes.DataStatuses[model.ID]
369369
out := dataStatusSummary(dataStatus)
370-
out += resourceStr(model.Model)
370+
out += titleStr("Configuration") + model.UserConfigStr()
371371
return out, nil
372372
}
373373

@@ -432,7 +432,7 @@ func describeAPI(name string, resourcesRes *schema.GetResourcesResponse) (string
432432
out += "Payload: " + samplesPlaceholderStr + "\n"
433433
}
434434
if api != nil {
435-
out += resourceStr(api.API)
435+
out += titleStr("Configuration") + api.UserConfigStr()
436436
}
437437

438438
return out, nil
@@ -446,10 +446,6 @@ func dataStatusSummary(dataStatus *resource.DataStatus) string {
446446
return out
447447
}
448448

449-
func resourceStr(resource userconfig.Resource) string {
450-
return titleStr("Configuration") + s.Obj(resource) + "\n"
451-
}
452-
453449
func valueStr(value interface{}) string {
454450
return titleStr("Value") + s.Obj(value) + "\n"
455451
}

‎go.mod

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
// replace these lines in go.mod:
66
// github.com/GoogleCloudPlatform/spark-on-k8s-operator v1alpha1-0.5-2.4.0
77
// github.com/argoproj/argo v2.3.0
8-
// github.com/cortexlabs/yaml v2.2.2
8+
// github.com/cortexlabs/yaml v2.2.3
99
// k8s.io/client-go v10.0.0
1010
// k8s.io/api 89a74a8d264df0e993299876a8cde88379b940ee
1111
// k8s.io/apimachinery 2b1284ed4c93a43499e781493253e2ac5959c4fd
@@ -20,8 +20,8 @@ go 1.12
2020
require (
2121
github.com/GoogleCloudPlatform/spark-on-k8s-operator v0.0.0-20181208011959-62db1d66dafa
2222
github.com/argoproj/argo v2.3.0+incompatible
23-
github.com/aws/aws-sdk-go v1.20.5
24-
github.com/cortexlabs/yaml v0.0.0-20190530233410-11baebde6c89
23+
github.com/aws/aws-sdk-go v1.20.7
24+
github.com/cortexlabs/yaml v0.0.0-20190624201412-7f31702857b6
2525
github.com/davecgh/go-spew v1.1.1
2626
github.com/emicklei/go-restful v2.9.6+incompatible // indirect
2727
github.com/go-openapi/spec v0.19.2 // indirect

‎go.sum

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,13 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko
1313
github.com/argoproj/argo v2.3.0+incompatible h1:L1OYZ86Q7NK19ahdl/eJOq78Mlf52wUKGmp7VDNQVz8=
1414
github.com/argoproj/argo v2.3.0+incompatible/go.mod h1:KJ0MB+tuhtAklR4jkPM10mIZXfRA0peTYJ1sLUnFLVU=
1515
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
16-
github.com/aws/aws-sdk-go v1.20.5 h1:Ytq5AxpA2pr4vRJM9onvgAjjVRZKKO63WStbG/jLHw0=
17-
github.com/aws/aws-sdk-go v1.20.5/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
16+
github.com/aws/aws-sdk-go v1.20.7 h1:fRjZRYJg0wPCA8yaDdb3DeP4rVjjmEiuqYhYoqOaIJg=
17+
github.com/aws/aws-sdk-go v1.20.7/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
1818
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
1919
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
2020
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
21-
github.com/cortexlabs/yaml v0.0.0-20190530233410-11baebde6c89 h1:NdtOtp57mz3NX/O8k0X9Qic0syGjOyT0b1+3GSq1uMU=
22-
github.com/cortexlabs/yaml v0.0.0-20190530233410-11baebde6c89/go.mod h1:ZQaiMs8i2UxfSwf0gNtM0kfmXg6hQHArtgaI6aL0+/U=
21+
github.com/cortexlabs/yaml v0.0.0-20190624201412-7f31702857b6 h1:NRxzHeHMTVyPV9yTGoXo3zRPplzVhrLTNzSFLNgpQII=
22+
github.com/cortexlabs/yaml v0.0.0-20190624201412-7f31702857b6/go.mod h1:nuzR4zMPuiBWg1HyZo9bzSZmtdSVjKfn8+RyO7egs0c=
2323
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
2424
github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
2525
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

‎pkg/lib/strings/stringify.go

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,11 @@ func strIndent(val interface{}, indent string, currentIndent string, newlineChar
219219
var t string
220220
casted := value.Convert(reflect.TypeOf(t)).Interface().(string)
221221

222-
casted, _ = yaml.UnescapeAtSymbol(casted)
222+
var ok bool
223+
casted, ok = yaml.UnescapeAtSymbolOk(casted)
224+
if ok {
225+
return casted
226+
}
223227

224228
switch val.(type) {
225229
case json.Number:
@@ -349,3 +353,19 @@ func UserStrs(val interface{}) []string {
349353
func Index(index int) string {
350354
return fmt.Sprintf("index %d", index)
351355
}
356+
357+
func Indent(str string, indent string) string {
358+
if str[len(str)-1:] == "\n" {
359+
out := ""
360+
for _, line := range strings.Split(str[:len(str)-1], "\n") {
361+
out += indent + line + "\n"
362+
}
363+
return out
364+
}
365+
366+
out := ""
367+
for _, line := range strings.Split(strings.TrimRight(str, "\n"), "\n") {
368+
out += indent + line + "\n"
369+
}
370+
return out[:len(out)-1]
371+
}

‎pkg/operator/api/context/columns.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ type Column interface {
2727
ComputedResource
2828
GetColumnType() userconfig.ColumnType
2929
IsRaw() bool
30+
UserConfigStr() string
3031
}
3132

3233
func (ctx *Context) Columns() Columns {

‎pkg/operator/api/context/raw_columns.go

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -55,18 +55,3 @@ func (rawColumns RawColumns) OneByID(id string) RawColumn {
5555
}
5656
return nil
5757
}
58-
59-
func GetRawColumnUserConfig(rawColumn RawColumn) userconfig.Resource {
60-
switch rawColumn.GetColumnType() {
61-
case userconfig.IntegerColumnType:
62-
return rawColumn.(*RawIntColumn).RawIntColumn
63-
case userconfig.FloatColumnType:
64-
return rawColumn.(*RawFloatColumn).RawFloatColumn
65-
case userconfig.StringColumnType:
66-
return rawColumn.(*RawStringColumn).RawStringColumn
67-
case userconfig.InferredColumnType:
68-
return rawColumn.(*RawInferredColumn).RawInferredColumn
69-
}
70-
71-
return nil
72-
}

‎pkg/operator/api/userconfig/aggregates.go

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,12 @@ limitations under the License.
1717
package userconfig
1818

1919
import (
20+
"fmt"
21+
"strings"
22+
2023
"github.com/cortexlabs/cortex/pkg/lib/configreader"
2124
"github.com/cortexlabs/cortex/pkg/lib/errors"
25+
s "github.com/cortexlabs/cortex/pkg/lib/strings"
2226
"github.com/cortexlabs/cortex/pkg/operator/api/resource"
2327
)
2428

@@ -66,6 +70,22 @@ var aggregateValidation = &configreader.StructValidation{
6670
},
6771
}
6872

73+
func (aggregate *Aggregate) UserConfigStr() string {
74+
var sb strings.Builder
75+
sb.WriteString(aggregate.ResourceFields.UserConfigStr())
76+
if aggregate.AggregatorPath != nil {
77+
sb.WriteString(fmt.Sprintf("%s: %s\n", AggregatorPathKey, *aggregate.AggregatorPath))
78+
} else {
79+
sb.WriteString(fmt.Sprintf("%s: %s\n", AggregatorKey, aggregate.Aggregator))
80+
}
81+
sb.WriteString(fmt.Sprintf("%s: %s\n", InputKey, s.Obj(aggregate.Input)))
82+
if aggregate.Compute != nil {
83+
sb.WriteString(fmt.Sprintf("%s:\n", ComputeKey))
84+
sb.WriteString(s.Indent(aggregate.Compute.UserConfigStr(), " "))
85+
}
86+
return sb.String()
87+
}
88+
6989
func (aggregates Aggregates) Validate() error {
7090
for _, aggregate := range aggregates {
7191
if err := aggregate.Validate(); err != nil {

‎pkg/operator/api/userconfig/apis.go

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,15 @@ limitations under the License.
1717
package userconfig
1818

1919
import (
20+
"fmt"
21+
"strings"
22+
23+
"github.com/cortexlabs/yaml"
24+
2025
"github.com/cortexlabs/cortex/pkg/lib/aws"
2126
cr "github.com/cortexlabs/cortex/pkg/lib/configreader"
2227
"github.com/cortexlabs/cortex/pkg/lib/errors"
28+
s "github.com/cortexlabs/cortex/pkg/lib/strings"
2329
"github.com/cortexlabs/cortex/pkg/operator/api/resource"
2430
)
2531

@@ -58,6 +64,23 @@ var apiValidation = &cr.StructValidation{
5864
},
5965
}
6066

67+
func (api *API) UserConfigStr() string {
68+
var sb strings.Builder
69+
sb.WriteString(api.ResourceFields.UserConfigStr())
70+
if api.Model != nil {
71+
sb.WriteString(fmt.Sprintf("%s: %s\n", ModelKey, yaml.UnescapeAtSymbol(*api.Model)))
72+
}
73+
if api.ExternalModel != nil {
74+
sb.WriteString(fmt.Sprintf("%s:\n", ExternalModelKey))
75+
sb.WriteString(s.Indent(api.ExternalModel.UserConfigStr(), " "))
76+
}
77+
if api.Compute != nil {
78+
sb.WriteString(fmt.Sprintf("%s:\n", ComputeKey))
79+
sb.WriteString(s.Indent(api.Compute.UserConfigStr(), " "))
80+
}
81+
return sb.String()
82+
}
83+
6184
type ExternalModel struct {
6285
Path string `json:"path" yaml:"path"`
6386
Region string `json:"region" yaml:"region"`
@@ -83,6 +106,13 @@ var externalModelFieldValidation = &cr.StructValidation{
83106
},
84107
}
85108

109+
func (extModel *ExternalModel) UserConfigStr() string {
110+
var sb strings.Builder
111+
sb.WriteString(fmt.Sprintf("%s: %s\n", PathKey, extModel.Path))
112+
sb.WriteString(fmt.Sprintf("%s: %s\n", RegionKey, extModel.Region))
113+
return sb.String()
114+
}
115+
86116
func (apis APIs) Validate() error {
87117
for _, api := range apis {
88118
if err := api.Validate(); err != nil {

‎pkg/operator/api/userconfig/compute.go

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ package userconfig
2525

2626
import (
2727
"bytes"
28+
"fmt"
29+
"strings"
2830

2931
k8sresource "k8s.io/apimachinery/pkg/api/resource"
3032

@@ -127,6 +129,25 @@ func sparkComputeFieldValidation(fieldName string) *cr.StructFieldValidation {
127129
}
128130
}
129131

132+
func (sparkCompute *SparkCompute) UserConfigStr() string {
133+
var sb strings.Builder
134+
sb.WriteString(fmt.Sprintf("%s: %s\n", ExecutorsKey, s.Int32(sparkCompute.Executors)))
135+
sb.WriteString(fmt.Sprintf("%s: %s\n", DriverCPUKey, sparkCompute.DriverCPU.UserString))
136+
sb.WriteString(fmt.Sprintf("%s: %s\n", DriverMemKey, sparkCompute.DriverMem.UserString))
137+
if sparkCompute.DriverMemOverhead != nil {
138+
sb.WriteString(fmt.Sprintf("%s: %s\n", DriverMemOverheadKey, sparkCompute.DriverMemOverhead.UserString))
139+
}
140+
sb.WriteString(fmt.Sprintf("%s: %s\n", ExecutorCPUKey, sparkCompute.ExecutorCPU.UserString))
141+
sb.WriteString(fmt.Sprintf("%s: %s\n", ExecutorMemKey, sparkCompute.ExecutorMem.UserString))
142+
if sparkCompute.ExecutorMemOverhead != nil {
143+
sb.WriteString(fmt.Sprintf("%s: %s\n", ExecutorMemOverheadKey, sparkCompute.ExecutorMemOverhead.UserString))
144+
}
145+
if sparkCompute.MemOverheadFactor != nil {
146+
sb.WriteString(fmt.Sprintf("%s: %s\n", ExecutorMemOverheadKey, s.Float64(*sparkCompute.MemOverheadFactor)))
147+
}
148+
return sb.String()
149+
}
150+
130151
func (sparkCompute *SparkCompute) ID() string {
131152
var buf bytes.Buffer
132153
buf.WriteString(s.Int32(sparkCompute.Executors))
@@ -183,6 +204,20 @@ var tfComputeFieldValidation = &cr.StructFieldValidation{
183204
},
184205
}
185206

207+
func (tfCompute *TFCompute) UserConfigStr() string {
208+
var sb strings.Builder
209+
if tfCompute.CPU != nil {
210+
sb.WriteString(fmt.Sprintf("%s: %s\n", CPUKey, tfCompute.CPU.UserString))
211+
}
212+
if tfCompute.GPU != nil {
213+
sb.WriteString(fmt.Sprintf("%s: %s\n", GPUKey, s.Int64(*tfCompute.GPU)))
214+
}
215+
if tfCompute.Mem != nil {
216+
sb.WriteString(fmt.Sprintf("%s: %s\n", MemKey, tfCompute.Mem.UserString))
217+
}
218+
return sb.String()
219+
}
220+
186221
func (tfCompute *TFCompute) ID() string {
187222
var buf bytes.Buffer
188223
buf.WriteString(QuantityPtrID(tfCompute.CPU))
@@ -237,6 +272,21 @@ var apiComputeFieldValidation = &cr.StructFieldValidation{
237272
},
238273
}
239274

275+
func (apiCompute *APICompute) UserConfigStr() string {
276+
var sb strings.Builder
277+
sb.WriteString(fmt.Sprintf("%s: %s\n", ReplicasKey, s.Int32(apiCompute.Replicas)))
278+
if apiCompute.CPU != nil {
279+
sb.WriteString(fmt.Sprintf("%s: %s\n", CPUKey, apiCompute.CPU.UserString))
280+
}
281+
if apiCompute.GPU != 0 {
282+
sb.WriteString(fmt.Sprintf("%s: %s\n", GPUKey, s.Int64(apiCompute.GPU)))
283+
}
284+
if apiCompute.Mem != nil {
285+
sb.WriteString(fmt.Sprintf("%s: %s\n", MemKey, apiCompute.Mem.UserString))
286+
}
287+
return sb.String()
288+
}
289+
240290
func (apiCompute *APICompute) ID() string {
241291
var buf bytes.Buffer
242292
buf.WriteString(s.Int32(apiCompute.Replicas))

‎pkg/operator/api/userconfig/config_key.go

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,9 @@ const (
2222
NameKey = "name"
2323
KindKey = "kind"
2424
InputKey = "input"
25-
ComputeKey = "compute"
2625
TypeKey = "type"
2726
PathKey = "path"
27+
RegionKey = "region"
2828
OutputTypeKey = "output_type"
2929
TagsKey = "tags"
3030
ExternalKey = "external"
@@ -91,10 +91,25 @@ const (
9191
KeepCheckpointEveryNHoursKey = "keep_checkpoint_every_n_hours"
9292
StartDelaySecsKey = "start_delay_secs"
9393
ThrottleSecsKey = "throttle_secs"
94-
DatasetComputeKey = "dataset_compute"
9594

9695
// API
9796
ModelKey = "model"
9897
ModelNameKey = "model_name"
9998
ExternalModelKey = "external_model"
99+
100+
// compute
101+
ComputeKey = "compute"
102+
DatasetComputeKey = "dataset_compute"
103+
ReplicasKey = "replicas"
104+
CPUKey = "cpu"
105+
GPUKey = "gpu"
106+
MemKey = "mem"
107+
ExecutorsKey = "executors"
108+
DriverCPUKey = "driver_cpu"
109+
DriverMemKey = "driver_mem"
110+
DriverMemOverheadKey = "driver_mem_overhead"
111+
ExecutorCPUKey = "executor_cpu"
112+
ExecutorMemKey = "executor_mem"
113+
ExecutorMemOverheadKey = "executor_mem_overhead"
114+
MemOverheadFactorKey = "mem_overhead_factor"
100115
)

0 commit comments

Comments
 (0)