SDK - Skip attributes with missing values during PipelineMeta seriali…

…zation (kubeflow#1448)

* SDK - Skip attributes with missing values during PipelineMeta serialization

* Fixed the tests

sdk/python/kfp/dsl/_metadata.py

@@ -46,10 +46,16 @@ def __init__(self,
     self.default = default
 
   def to_dict(self):
-    return {'name': self.name,
-            'description': self.description,
-            'type': self.param_type or '',
-            'default': self.default}
+    result = {}
+    if self.name:
+      result['name'] = self.name
+    if self.description:
+      result['description'] = self.description
+    if self.param_type:
+      result['type'] = self.param_type
+    if self.default:
+      result['default'] = self.default
+    return result
 
 
 class ComponentMeta(BaseMeta):
@@ -66,11 +72,16 @@ def __init__(
     self.outputs = [] if outputs is None else outputs
 
   def to_dict(self):
-    return {'name': self.name,
-            'description': self.description,
-            'inputs': [ input.to_dict() for input in self.inputs ],
-            'outputs': [ output.to_dict() for output in self.outputs ]
-            }
+    result = {}
+    if self.name:
+      result['name'] = self.name
+    if self.description:
+      result['description'] = self.description
+    if self.inputs:
+      result['inputs'] = [input.to_dict() for input in self.inputs]
+    if self.outputs:
+      result['outputs'] = [output.to_dict() for output in self.outputs]
+    return result
 
 
 # Add a pipeline level metadata calss here.
@@ -87,10 +98,14 @@ def __init__(
     self.inputs = [] if inputs is None else inputs
 
   def to_dict(self):
-    return {'name': self.name,
-            'description': self.description,
-            'inputs': [ input.to_dict() for input in self.inputs ]
-            }
+    result = {}
+    if self.name:
+      result['name'] = self.name
+    if self.description:
+      result['description'] = self.description
+    if self.inputs:
+      result['inputs'] = [input.to_dict() for input in self.inputs]
+    return result
 
 def _annotation_to_typemeta(annotation):
   '''_annotation_to_type_meta converts an annotation to a type structure

sdk/python/tests/compiler/testdata/add_pod_env.yaml

@@ -2,8 +2,7 @@ apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
   annotations:
-    pipelines.kubeflow.org/pipeline_spec: '{"description": "Test adding pod env",
-      "inputs": [], "name": "Test adding pod env"}'
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "Test adding pod env", "name": "Test adding pod env"}'
   generateName: test-adding-pod-env-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/artifact_location.yaml

@@ -15,10 +15,7 @@ apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
   annotations:
-    pipelines.kubeflow.org/pipeline_spec: '{"description": "hello world", "inputs":
-      [{"default": null, "description": "", "name": "tag", "type": ""}, {"default":
-      "kubeflow", "description": "", "name": "namespace", "type": ""}, {"default":
-      "foobar", "description": "", "name": "bucket", "type": ""}], "name": "foo"}'
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "hello world", "inputs": [{"name": "tag"}, {"default": "kubeflow", "name": "namespace"}, {"default": "foobar", "name": "bucket"}], "name": "foo"}'
   generateName: foo-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/basic.yaml

@@ -15,7 +15,7 @@ apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
   annotations:
-    pipelines.kubeflow.org/pipeline_spec: '{"description": "Get Most Frequent Word and Save to GCS", "inputs": [{"default": null, "description": "", "name": "message", "type": ""}, {"default": null, "description": "", "name": "outputpath", "type": ""}], "name": "Save Most Frequent"}'
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "Get Most Frequent Word and Save to GCS", "inputs": [{"name": "message"}, {"name": "outputpath"}], "name": "Save Most Frequent"}'
   generateName: save-most-frequent-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/coin.yaml

@@ -15,7 +15,7 @@ apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
   annotations:
-    pipelines.kubeflow.org/pipeline_spec: '{"description": "shows how to use dsl.Condition.", "inputs": [], "name": "pipeline flip coin"}'
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "shows how to use dsl.Condition.", "name": "pipeline flip coin"}'
   generateName: pipeline-flip-coin-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/compose.yaml

@@ -15,7 +15,7 @@ apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
   annotations:
-    pipelines.kubeflow.org/pipeline_spec: '{"description": "Download and Get Most Frequent Word and Save to GCS", "inputs": [{"default": null, "description": "", "name": "url", "type": ""}, {"default": null, "description": "", "name": "outputpath", "type": ""}], "name": "Download and Save Most Frequent"}'
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "Download and Get Most Frequent Word and Save to GCS", "inputs": [{"name": "url"}, {"name": "outputpath"}], "name": "Download and Save Most Frequent"}'
   generateName: download-and-save-most-frequent-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/default_value.yaml

@@ -15,7 +15,7 @@ apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
   annotations:
-    pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with parameter and default value.", "inputs": [{"default": "gs://ml-pipeline/shakespeare1.txt", "description": "", "name": "url", "type": ""}], "name": "Default Value"}'
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with parameter and default value.", "inputs": [{"default": "gs://ml-pipeline/shakespeare1.txt", "name": "url"}], "name": "Default Value"}'
   generateName: default-value-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/imagepullsecrets.yaml

@@ -2,7 +2,7 @@ apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
   annotations:
-    pipelines.kubeflow.org/pipeline_spec: '{"description": "Get Most Frequent Word and Save to GCS", "inputs": [{"default": null, "description": "", "name": "message", "type": ""}], "name": "Save Most Frequent"}'
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "Get Most Frequent Word and Save to GCS", "inputs": [{"name": "message"}], "name": "Save Most Frequent"}'
   generateName: save-most-frequent-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/param_op_transform.yaml

@@ -4,8 +4,7 @@ metadata:
   annotations:
     pipelines.kubeflow.org/pipeline_spec: '{"description": "Test that parameters used
       in Op transformation functions as pod labels would be correcly identified and
-      set as arguments in he generated yaml", "inputs": [{"default": null, "description":
-      "", "name": "param", "type": ""}], "name": "Parameters in Op transformation
+      set as arguments in he generated yaml", "inputs": [{"name": "param"}], "name": "Parameters in Op transformation
       functions"}'
   generateName: parameters-in-op-transformation-functions-
 spec:

sdk/python/tests/compiler/testdata/param_substitutions.yaml

@@ -3,7 +3,7 @@ kind: Workflow
 metadata:
   annotations:
     pipelines.kubeflow.org/pipeline_spec: '{"description": "Test the same PipelineParam
-      getting substituted in multiple places", "inputs": [], "name": "Param Substitutions"}'
+      getting substituted in multiple places", "name": "Param Substitutions"}'
   generateName: param-substitutions-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/pipelineparams.yaml

@@ -16,9 +16,8 @@ apiVersion: argoproj.io/v1alpha1
 metadata:
   annotations:
     pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with multiple
-      pipeline params.", "inputs": [{"default": "latest", "description": "", "name":
-      "tag", "type": ""}, {"default": 10, "description": "", "name": "sleep_ms", "type":
-      ""}], "name": "PipelineParams"}'
+      pipeline params.", "inputs": [{"default": "latest", "name":
+      "tag"}, {"default": 10, "name": "sleep_ms"}], "name": "PipelineParams"}'
   generateName: pipelineparams-
 spec:
   entrypoint: pipelineparams

sdk/python/tests/compiler/testdata/preemptible_tpu_gpu.yaml

@@ -2,7 +2,7 @@ apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
   annotations:
-    pipelines.kubeflow.org/pipeline_spec: '{"description": "shows how to use dsl.Condition.", "inputs": [], "name": "pipeline flip coin"}'
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "shows how to use dsl.Condition.", "name": "pipeline flip coin"}'
   generateName: pipeline-flip-coin-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/recursive_do_while.yaml

@@ -3,7 +3,7 @@ kind: Workflow
 metadata:
   annotations:
     pipelines.kubeflow.org/pipeline_spec: '{"description": "shows how to use graph_component.",
-      "inputs": [], "name": "pipeline flip coin"}'
+      "name": "pipeline flip coin"}'
   generateName: pipeline-flip-coin-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/recursive_while.yaml

@@ -3,7 +3,7 @@ kind: Workflow
 metadata:
   annotations:
     pipelines.kubeflow.org/pipeline_spec: '{"description": "shows how to use dsl.Condition.",
-      "inputs": [{"default": 12, "description": "", "name": "maxVal", "type": ""}],
+      "inputs": [{"default": 12, "name": "maxVal"}],
       "name": "pipeline flip coin"}'
   generateName: pipeline-flip-coin-
 spec:

sdk/python/tests/compiler/testdata/resourceop_basic.yaml

@@ -3,9 +3,7 @@ kind: Workflow
 metadata:
   annotations:
     pipelines.kubeflow.org/pipeline_spec: '{"description": "A Basic Example on ResourceOp
-      Usage.", "inputs": [{"default": null, "description": "", "name": "username",
-      "type": ""}, {"default": null, "description": "", "name": "password", "type":
-      ""}], "name": "ResourceOp Basic"}'
+      Usage.", "inputs": [{"name": "username"}, {"name": "password"}], "name": "ResourceOp Basic"}'
   generateName: resourceop-basic-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/sidecar.yaml

@@ -15,7 +15,7 @@ kind: Workflow
 metadata:
   annotations:
     pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with sidecars.",
-      "inputs": [], "name": "Sidecar"}'
+      "name": "Sidecar"}'
   generateName: sidecar-
 apiVersion: argoproj.io/v1alpha1
 spec:

sdk/python/tests/compiler/testdata/timeout.yaml

@@ -3,7 +3,7 @@ kind: Workflow
 metadata:
   annotations:
     pipelines.kubeflow.org/pipeline_spec: '{"description": "shows how to use ContainerOp
-      set_retry().", "inputs": [], "name": "pipeline includes two steps which fail
+      set_retry().", "name": "pipeline includes two steps which fail
       randomly."}'
   generateName: pipeline-includes-two-steps-which-fail-randomly-
 spec:

sdk/python/tests/compiler/testdata/volume.yaml

@@ -16,7 +16,7 @@ kind: Workflow
 metadata:
   annotations:
     pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with volume.",
-      "inputs": [], "name": "Volume"}'
+      "name": "Volume"}'
   generateName: volume-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml

@@ -3,8 +3,7 @@ kind: Workflow
 metadata:
   annotations:
     pipelines.kubeflow.org/pipeline_spec: '{"description": "The fifth example of the
-      design doc.", "inputs": [{"default": null, "description": "", "name": "rok_url",
-      "type": ""}], "name": "VolumeSnapshotOp RokURL"}'
+      design doc.", "inputs": [{"name": "rok_url"}], "name": "VolumeSnapshotOp RokURL"}'
   generateName: volumesnapshotop-rokurl-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.yaml

@@ -3,8 +3,7 @@ kind: Workflow
 metadata:
   annotations:
     pipelines.kubeflow.org/pipeline_spec: '{"description": "The fourth example of
-      the design doc.", "inputs": [{"default": null, "description": "", "name": "url",
-      "type": ""}], "name": "VolumeSnapshotOp Sequential"}'
+      the design doc.", "inputs": [{"name": "url"}], "name": "VolumeSnapshotOp Sequential"}'
   generateName: volumesnapshotop-sequential-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/volumeop_basic.yaml

@@ -3,8 +3,7 @@ kind: Workflow
 metadata:
   annotations:
     pipelines.kubeflow.org/pipeline_spec: '{"description": "A Basic Example on VolumeOp
-      Usage.", "inputs": [{"default": null, "description": "", "name": "size", "type":
-      ""}], "name": "VolumeOp Basic"}'
+      Usage.", "inputs": [{"name": "size"}], "name": "VolumeOp Basic"}'
   generateName: volumeop-basic-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/volumeop_dag.yaml

@@ -3,7 +3,7 @@ kind: Workflow
 metadata:
   annotations:
     pipelines.kubeflow.org/pipeline_spec: '{"description": "The second example of
-      the design doc.", "inputs": [], "name": "Volume Op DAG"}'
+      the design doc.", "name": "Volume Op DAG"}'
   generateName: volume-op-dag-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/volumeop_parallel.yaml

@@ -3,7 +3,7 @@ kind: Workflow
 metadata:
   annotations:
     pipelines.kubeflow.org/pipeline_spec: '{"description": "The first example of the
-      design doc.", "inputs": [], "name": "VolumeOp Parallel"}'
+      design doc.", "name": "VolumeOp Parallel"}'
   generateName: volumeop-parallel-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/volumeop_sequential.yaml

@@ -3,7 +3,7 @@ kind: Workflow
 metadata:
   annotations:
     pipelines.kubeflow.org/pipeline_spec: '{"description": "The third example of the
-      design doc.", "inputs": [], "name": "VolumeOp Sequential"}'
+      design doc.", "name": "VolumeOp Sequential"}'
   generateName: volumeop-sequential-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/withitem_basic.yaml

@@ -2,8 +2,8 @@ apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
   annotations:
-    pipelines.kubeflow.org/pipeline_spec: '{"description": null, "inputs": [{"default":
-      10, "description": "", "name": "my_pipe_param", "type": ""}], "name": "my-pipeline"}'
+    pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default":
+      10, "name": "my_pipe_param"}], "name": "my-pipeline"}'
   generateName: my-pipeline-
 spec:
   arguments:

sdk/python/tests/compiler/testdata/withitem_nested.yaml

@@ -2,8 +2,8 @@ apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
   annotations:
-    pipelines.kubeflow.org/pipeline_spec: '{"description": null, "inputs": [{"default":
-      10, "description": "", "name": "my_pipe_param", "type": ""}], "name": "my-pipeline"}'
+    pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default":
+      10, "name": "my_pipe_param"}], "name": "my-pipeline"}'
   generateName: my-pipeline-
 spec:
   arguments: