Skip to content

Commit

Permalink
chore(backend): clean up pipelinespec.Value usage (#7407)
Browse files Browse the repository at this point in the history
  • Loading branch information
@chensun
chensun authored Mar 14, 2022
1 parent 1163510 commit 5fe4f50
Show file tree
Hide file tree
Showing 11 changed files with 104 additions and 110 deletions.
2 changes: 1 addition & 1 deletion api/v2alpha1/cache_key.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import "pipeline_spec.proto";

message CacheKey {
map<string, ArtifactNameList> inputArtifactNames = 1;
map<string, Value> inputParameters = 2;
map<string, Value> inputParameters = 2 [deprecated = true];
map<string, RuntimeArtifact> outputArtifactsSpec = 3;
map<string, string> outputParametersSpec=4;
ContainerSpec containerSpec=5;
Expand Down
138 changes: 70 additions & 68 deletions api/v2alpha1/go/cachekey/cache_key.pb.go
View file

Large diffs are not rendered by default.

9 changes: 1 addition & 8 deletions backend/src/v2/cacheutils/cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,9 @@ func GenerateCacheKey(

cacheKey := cachekey.CacheKey{
InputArtifactNames: make(map[string]*cachekey.ArtifactNameList),
InputParameters: make(map[string]*pipelinespec.Value),
InputParameterValues: make(map[string]*structpb.Value),
OutputArtifactsSpec: make(map[string]*pipelinespec.RuntimeArtifact),
OutputParametersSpec: make(map[string]string),
InputParameterValues: make(map[string]*structpb.Value),
}

for inputArtifactName, inputArtifactList := range inputs.GetArtifacts() {
Expand All @@ -70,12 +69,6 @@ func GenerateCacheKey(
cacheKey.InputArtifactNames[inputArtifactName] = &inputArtifactNameList
}

for inputParameterName, inputParameterValue := range inputs.GetParameters() {
cacheKey.InputParameters[inputParameterName] = &pipelinespec.Value{
Value: inputParameterValue.Value,
}
}

for inputParameterName, inputParameterValue := range inputs.GetParameterValues() {
cacheKey.InputParameterValues[inputParameterName] = inputParameterValue
}
Expand Down
28 changes: 14 additions & 14 deletions backend/src/v2/cacheutils/cache_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@ func TestGenerateCacheKey(t *testing.T) {
{
name: "Generate CacheKey Correctly",
executorInputInputs: &pipelinespec.ExecutorInput_Inputs{
Parameters: map[string]*pipelinespec.Value{
"message": {Value: &pipelinespec.Value_StringValue{StringValue: "Some string value"}},
"num_steps": {Value: &pipelinespec.Value_IntValue{IntValue: 5}},
ParameterValues: map[string]*structpb.Value{
"message": {Kind: &structpb.Value_StringValue{StringValue: "Some string value"}},
"num_steps": {Kind: &structpb.Value_NumberValue{NumberValue: 5}},
},
Artifacts: map[string]*pipelinespec.ArtifactList{
"dataset_one": {
Expand Down Expand Up @@ -98,9 +98,9 @@ func TestGenerateCacheKey(t *testing.T) {
"dataset_one": {ArtifactNames: []string{"1"}},
"dataset_two": {ArtifactNames: []string{"2"}},
},
InputParameters: map[string]*pipelinespec.Value{
"message": {Value: &pipelinespec.Value_StringValue{StringValue: "Some string value"}},
"num_steps": {Value: &pipelinespec.Value_IntValue{IntValue: 5}},
InputParameterValues: map[string]*structpb.Value{
"message": {Kind: &structpb.Value_StringValue{StringValue: "Some string value"}},
"num_steps": {Kind: &structpb.Value_NumberValue{NumberValue: 5}},
},
OutputArtifactsSpec: map[string]*pipelinespec.RuntimeArtifact{
"model": {
Expand Down Expand Up @@ -158,9 +158,9 @@ func TestGenerateFingerPrint(t *testing.T) {
"dataset_one": {ArtifactNames: []string{"1"}},
"dataset_two": {ArtifactNames: []string{"2"}},
},
InputParameters: map[string]*pipelinespec.Value{
"message": {Value: &pipelinespec.Value_StringValue{StringValue: "Some string value"}},
"num_steps": {Value: &pipelinespec.Value_IntValue{IntValue: 5}},
InputParameterValues: map[string]*structpb.Value{
"message": {Kind: &structpb.Value_StringValue{StringValue: "Some string value"}},
"num_steps": {Kind: &structpb.Value_NumberValue{NumberValue: 5}},
},
OutputArtifactsSpec: map[string]*pipelinespec.RuntimeArtifact{
"model": {
Expand Down Expand Up @@ -202,9 +202,9 @@ func TestGenerateFingerPrint(t *testing.T) {
"dataset_one": {ArtifactNames: []string{"1"}},
"dataset_two": {ArtifactNames: []string{"2"}},
},
InputParameters: map[string]*pipelinespec.Value{
"message": {Value: &pipelinespec.Value_StringValue{StringValue: "Some string value"}},
"num_steps": {Value: &pipelinespec.Value_IntValue{IntValue: 5}},
InputParameterValues: map[string]*structpb.Value{
"message": {Kind: &structpb.Value_StringValue{StringValue: "Some string value"}},
"num_steps": {Kind: &structpb.Value_NumberValue{NumberValue: 5}},
},
OutputArtifactsSpec: map[string]*pipelinespec.RuntimeArtifact{
"model": {
Expand Down Expand Up @@ -234,7 +234,7 @@ func TestGenerateFingerPrint(t *testing.T) {
},
},
wantEqual: true,
fingerPrint: "c5e35810df04b64f63342644ab1041f4dcfec76528dcb98483a40be10d705015",
fingerPrint: "4e8a5d7d70997b0a35429fcd481af8fcd5b9f58ef4391bdb6ad900fd1c63622b",
}, {
name: "Generated Different FingerPrint",
cacheKey: &cachekey.CacheKey{
Expand All @@ -260,7 +260,7 @@ func TestGenerateFingerPrint(t *testing.T) {
testFingerPrint, err := GenerateFingerPrint(test.cacheKey)
assert.Nil(t, err)
assert.Equal(t, fingerPrint == testFingerPrint, test.wantEqual)
assert.Equal(t, testFingerPrint, test.fingerPrint)
assert.Equal(t, test.fingerPrint, testFingerPrint)
})
}
}
3 changes: 0 additions & 3 deletions backend/src/v2/driver/driver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -798,9 +798,6 @@ func resolveInputs(ctx context.Context, dag *metadata.DAG, iterationIndex *int,
switch t := runtimeValue.Value.(type) {
case *pipelinespec.ValueOrRuntimeParameter_Constant:
inputs.ParameterValues[name] = runtimeValue.GetConstant()
// TODO(v2): clean up pipelinespec.Value usages
case *pipelinespec.ValueOrRuntimeParameter_ConstantValue:
inputs.Parameters[name] = runtimeValue.GetConstantValue()
default:
return nil, paramError(fmt.Errorf("param runtime value spec of type %T not implemented", t))
}
Expand Down
6 changes: 3 additions & 3 deletions backend/third_party_licenses/apiserver.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ github.com/josharian/intern, https://github.com/josharian/intern/blob/v1.0.0/lic
github.com/json-iterator/go, https://github.com/json-iterator/go/blob/v1.1.11/LICENSE, MIT
github.com/klauspost/compress/flate, https://github.com/klauspost/compress/blob/v1.13.1/LICENSE, BSD-3-Clause
github.com/klauspost/pgzip, https://github.com/klauspost/pgzip/blob/v1.2.5/LICENSE, MIT
github.com/kubeflow/pipelines/api/v2alpha1/go, https://github.com/kubeflow/pipelines/blob/e78ed557ddcb/api/LICENSE, Apache-2.0
github.com/kubeflow/pipelines/api/v2alpha1/go, https://github.com/kubeflow/pipelines/blob/11635101d944/api/LICENSE, Apache-2.0
github.com/kubeflow/pipelines/backend, https://github.com/kubeflow/pipelines/blob/master/LICENSE, Apache-2.0
github.com/kubeflow/pipelines/third_party/ml-metadata/go/ml_metadata, https://github.com/kubeflow/pipelines/blob/e78ed557ddcb/third_party/ml-metadata/LICENSE, Apache-2.0
github.com/lann/builder, https://github.com/lann/builder/blob/47ae307949d0/LICENSE, MIT
Expand Down Expand Up @@ -113,8 +113,8 @@ golang.org/x/time/rate, https://cs.opensource.google/go/x/time/+/1f47c861:LICENS
golang.org/x/xerrors, https://cs.opensource.google/go/x/xerrors/+/5ec99f83:LICENSE, BSD-3-Clause
google.golang.org/api, https://github.com/googleapis/google-api-go-client/blob/v0.44.0/LICENSE, BSD-3-Clause
google.golang.org/api/internal/third_party/uritemplates, https://github.com/googleapis/google-api-go-client/blob/v0.44.0/internal/third_party/uritemplates/LICENSE, BSD-3-Clause
google.golang.org/genproto, https://github.com/googleapis/go-genproto/blob/d629cc9a93d5/LICENSE, Apache-2.0
google.golang.org/grpc, https://github.com/grpc/grpc-go/blob/v1.43.0/LICENSE, Apache-2.0
google.golang.org/genproto, https://github.com/googleapis/go-genproto/blob/1973136f34c6/LICENSE, Apache-2.0
google.golang.org/grpc, https://github.com/grpc/grpc-go/blob/v1.44.0/LICENSE, Apache-2.0
google.golang.org/protobuf, https://github.com/protocolbuffers/protobuf-go/blob/v1.27.1/LICENSE, BSD-3-Clause
gopkg.in/inf.v0, https://github.com/go-inf/inf/blob/v0.9.1/LICENSE, BSD-3-Clause
gopkg.in/ini.v1, https://github.com/go-ini/ini/blob/v1.62.0/LICENSE, Apache-2.0
Expand Down
4 changes: 2 additions & 2 deletions backend/third_party_licenses/cache_server.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,8 @@ golang.org/x/sys, https://cs.opensource.google/go/x/sys/+/1d35b9e2:LICENSE, BSD-
golang.org/x/term, https://cs.opensource.google/go/x/term/+/6a3ed077:LICENSE, BSD-3-Clause
golang.org/x/text, https://cs.opensource.google/go/x/text/+/v0.3.7:LICENSE, BSD-3-Clause
golang.org/x/time/rate, https://cs.opensource.google/go/x/time/+/1f47c861:LICENSE, BSD-3-Clause
google.golang.org/genproto, https://github.com/googleapis/go-genproto/blob/d629cc9a93d5/LICENSE, Apache-2.0
google.golang.org/grpc, https://github.com/grpc/grpc-go/blob/v1.43.0/LICENSE, Apache-2.0
google.golang.org/genproto, https://github.com/googleapis/go-genproto/blob/1973136f34c6/LICENSE, Apache-2.0
google.golang.org/grpc, https://github.com/grpc/grpc-go/blob/v1.44.0/LICENSE, Apache-2.0
google.golang.org/protobuf, https://github.com/protocolbuffers/protobuf-go/blob/v1.27.1/LICENSE, BSD-3-Clause
gopkg.in/inf.v0, https://github.com/go-inf/inf/blob/v0.9.1/LICENSE, BSD-3-Clause
gopkg.in/yaml.v2, https://github.com/go-yaml/yaml/blob/v2.4.0/LICENSE, Apache-2.0
Expand Down
4 changes: 2 additions & 2 deletions backend/third_party_licenses/persistence_agent.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@ golang.org/x/sys, https://cs.opensource.google/go/x/sys/+/1d35b9e2:LICENSE, BSD-
golang.org/x/term, https://cs.opensource.google/go/x/term/+/6a3ed077:LICENSE, BSD-3-Clause
golang.org/x/text, https://cs.opensource.google/go/x/text/+/v0.3.7:LICENSE, BSD-3-Clause
golang.org/x/time/rate, https://cs.opensource.google/go/x/time/+/1f47c861:LICENSE, BSD-3-Clause
google.golang.org/genproto, https://github.com/googleapis/go-genproto/blob/d629cc9a93d5/LICENSE, Apache-2.0
google.golang.org/grpc, https://github.com/grpc/grpc-go/blob/v1.43.0/LICENSE, Apache-2.0
google.golang.org/genproto, https://github.com/googleapis/go-genproto/blob/1973136f34c6/LICENSE, Apache-2.0
google.golang.org/grpc, https://github.com/grpc/grpc-go/blob/v1.44.0/LICENSE, Apache-2.0
google.golang.org/protobuf, https://github.com/protocolbuffers/protobuf-go/blob/v1.27.1/LICENSE, BSD-3-Clause
gopkg.in/inf.v0, https://github.com/go-inf/inf/blob/v0.9.1/LICENSE, BSD-3-Clause
gopkg.in/yaml.v2, https://github.com/go-yaml/yaml/blob/v2.4.0/LICENSE, Apache-2.0
Expand Down
4 changes: 2 additions & 2 deletions backend/third_party_licenses/swf.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,8 @@ golang.org/x/sys, https://cs.opensource.google/go/x/sys/+/1d35b9e2:LICENSE, BSD-
golang.org/x/term, https://cs.opensource.google/go/x/term/+/6a3ed077:LICENSE, BSD-3-Clause
golang.org/x/text, https://cs.opensource.google/go/x/text/+/v0.3.7:LICENSE, BSD-3-Clause
golang.org/x/time/rate, https://cs.opensource.google/go/x/time/+/1f47c861:LICENSE, BSD-3-Clause
google.golang.org/genproto, https://github.com/googleapis/go-genproto/blob/d629cc9a93d5/LICENSE, Apache-2.0
google.golang.org/grpc, https://github.com/grpc/grpc-go/blob/v1.43.0/LICENSE, Apache-2.0
google.golang.org/genproto, https://github.com/googleapis/go-genproto/blob/1973136f34c6/LICENSE, Apache-2.0
google.golang.org/grpc, https://github.com/grpc/grpc-go/blob/v1.44.0/LICENSE, Apache-2.0
google.golang.org/protobuf, https://github.com/protocolbuffers/protobuf-go/blob/v1.27.1/LICENSE, BSD-3-Clause
gopkg.in/inf.v0, https://github.com/go-inf/inf/blob/v0.9.1/LICENSE, BSD-3-Clause
gopkg.in/ini.v1, https://github.com/go-ini/ini/blob/v1.62.0/LICENSE, Apache-2.0
Expand Down
6 changes: 3 additions & 3 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ require (
github.com/jinzhu/gorm v1.9.1
github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a // indirect
github.com/jinzhu/now v0.0.0-20181116074157-8ec929ed50c3 // indirect
github.com/kubeflow/pipelines/api v0.0.0-20220118175555-e78ed557ddcb
github.com/kubeflow/pipelines/api v0.0.0-20220311022801-11635101d944
github.com/kubeflow/pipelines/third_party/ml-metadata v0.0.0-20220118175555-e78ed557ddcb
github.com/lestrrat-go/strftime v1.0.4
github.com/mattn/go-sqlite3 v1.9.0
Expand All @@ -44,8 +44,8 @@ require (
github.com/stretchr/testify v1.7.0
gocloud.dev v0.22.0
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f
google.golang.org/genproto v0.0.0-20211221231510-d629cc9a93d5
google.golang.org/grpc v1.43.0
google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6
google.golang.org/grpc v1.44.0
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0
google.golang.org/protobuf v1.27.1
gopkg.in/yaml.v2 v2.4.0
Expand Down
10 changes: 6 additions & 4 deletions go.sum
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 5fe4f50

Please sign in to comment.