optimize(sdk): Move common task labels to pipeline labels to reduce y…

…aml size (kubeflow#1020) * move common task labels to pipeline labels * move common task labels to pipeline labels * update new python lint requirements
red-hat-data-services · Aug 3, 2022 · 1abffb6 · 1abffb6
1 parent d0079ac
commit 1abffb6
Show file tree

Hide file tree

Showing 136 changed files with 727 additions and 1,181 deletions.
diff --git a/sdk/python/kfp_tekton/compiler/_op_to_template.py b/sdk/python/kfp_tekton/compiler/_op_to_template.py
@@ -579,7 +579,7 @@ def _op_to_template(op: BaseOp,
         try:
             result_size_map = json.loads(result_size_map)
         except ValueError:
-            raise("tekton-result-sizes annotation is not a valid JSON")
+            raise ("tekton-result-sizes annotation is not a valid JSON")
         # Normalize estimated result size keys.
         result_size_map = {sanitize_k8s_name(key, allow_capital_underscore=True): value
                            for key, value in result_size_map.items()}
@@ -596,7 +596,7 @@ def _op_to_template(op: BaseOp,
             try:
                 value = int(value)
             except ValueError:
-                raise("Estimated value for result %s is %s, but it needs to be an integer." % (key, value))
+                raise ("Estimated value for result %s is %s, but it needs to be an integer." % (key, value))
             if key in op_result_names:
                 packed_index = -1
                 # Look for bin that can fit the result value

diff --git a/sdk/python/kfp_tekton/compiler/compiler.py b/sdk/python/kfp_tekton/compiler/compiler.py
@@ -150,6 +150,8 @@ def __init__(self, **kwargs):
 
   def _set_pipeline_conf(self, tekton_pipeline_conf: TektonPipelineConf):
     self.pipeline_labels = tekton_pipeline_conf.pipeline_labels
+    self.pipeline_labels['pipelines.kubeflow.org/pipelinename'] = ''
+    self.pipeline_labels['pipelines.kubeflow.org/generation'] = ''
     self.pipeline_annotations = tekton_pipeline_conf.pipeline_annotations
     self.tekton_inline_spec = tekton_pipeline_conf.tekton_inline_spec
     self.resource_in_separate_yaml = tekton_pipeline_conf.resource_in_separate_yaml
@@ -1012,7 +1014,7 @@ def map_cel_vars(a):
 
             # Only one of --taskRef and --taskSpec allowed.
             if custom_task_args.get('taskRef', '') and custom_task_args.get('taskSpec', ''):
-              raise("Custom task invalid configuration %s, Only one of --taskRef and --taskSpec allowed." % custom_task_args)
+              raise ("Custom task invalid configuration %s, Only one of --taskRef and --taskSpec allowed." % custom_task_args)
             if custom_task_args.get('taskRef', ''):
               try:
                 custom_task_cr = {
@@ -1031,7 +1033,7 @@ def map_cel_vars(a):
                 if custom_task_cr:
                   self.custom_task_crs.append(custom_task_cr)
               except ValueError:
-                raise("Custom task ref %s is not a valid Python Dictionary" % custom_task_args['taskRef'])
+                raise ("Custom task ref %s is not a valid Python Dictionary" % custom_task_args['taskRef'])
             # Setting --taskRef flag indicates, that spec be inlined.
             if custom_task_args.get('taskSpec', ''):
               try:
@@ -1046,7 +1048,7 @@ def map_cel_vars(a):
                   }
                 }
               except ValueError:
-                raise("Custom task spec %s is not a valid Python Dictionary" % custom_task_args['taskSpec'])
+                raise ("Custom task spec %s is not a valid Python Dictionary" % custom_task_args['taskSpec'])
             # Pop custom task artifacts since we have no control of how
             # custom task controller is handling the container/task execution.
             self.artifact_items.pop(template['metadata']['name'], None)
@@ -1055,8 +1057,6 @@ def map_cel_vars(a):
         if task_ref.get('taskSpec', ''):
           task_ref['taskSpec']['metadata'] = task_ref['taskSpec'].get('metadata', {})
           task_labels = template['metadata'].get('labels', {})
-          task_labels['pipelines.kubeflow.org/pipelinename'] = task_labels.get('pipelines.kubeflow.org/pipelinename', '')
-          task_labels['pipelines.kubeflow.org/generation'] = task_labels.get('pipelines.kubeflow.org/generation', '')
           cache_default = self.pipeline_labels.get('pipelines.kubeflow.org/cache_enabled', 'true')
           task_labels['pipelines.kubeflow.org/cache_enabled'] = task_labels.get('pipelines.kubeflow.org/cache_enabled', cache_default)
 
@@ -1855,8 +1855,6 @@ def _inline_tasks(self, tasks: List[Dict[Text, Any]], crs: List[Dict[Text, Any]]
       if 'taskSpec' in workflow_tasks[j]:
         workflow_tasks[j]['taskSpec']['metadata'] = workflow_tasks[j]['taskSpec'].get('metadata', {})
         task_labels = workflow_tasks[j]['taskSpec']['metadata'].get('labels', {})
-        task_labels['pipelines.kubeflow.org/pipelinename'] = task_labels.get('pipelines.kubeflow.org/pipelinename', '')
-        task_labels['pipelines.kubeflow.org/generation'] = task_labels.get('pipelines.kubeflow.org/generation', '')
         cache_default = self.pipeline_labels.get('pipelines.kubeflow.org/cache_enabled', 'true')
         task_labels['pipelines.kubeflow.org/cache_enabled'] = task_labels.get('pipelines.kubeflow.org/cache_enabled', cache_default)
         workflow_tasks[j]['taskSpec']['metadata']['labels'] = task_labels

diff --git a/sdk/python/tests/compiler/testdata/affinity.yaml b/sdk/python/tests/compiler/testdata/affinity.yaml
@@ -27,6 +27,9 @@ metadata:
     pipelines.kubeflow.org/big_data_passing_format: $(workspaces.$TASK_NAME.path)/artifacts/$ORIG_PR_NAME/$TASKRUN_NAME/$TASK_PARAM_NAME
     pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with affinity",
       "name": "affinity"}'
+  labels:
+    pipelines.kubeflow.org/pipelinename: ''
+    pipelines.kubeflow.org/generation: ''
 spec:
   pipelineSpec:
     tasks:
@@ -44,8 +47,6 @@ spec:
             runAsUser: 0
         metadata:
           labels:
-            pipelines.kubeflow.org/pipelinename: ''
-            pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
           annotations:
             pipelines.kubeflow.org/component_spec_digest: '{"name": "echo", "outputs":

diff --git a/sdk/python/tests/compiler/testdata/any_sequencer.yaml b/sdk/python/tests/compiler/testdata/any_sequencer.yaml
@@ -32,6 +32,9 @@ metadata:
     pipelines.kubeflow.org/big_data_passing_format: $(workspaces.$TASK_NAME.path)/artifacts/$ORIG_PR_NAME/$TASKRUN_NAME/$TASK_PARAM_NAME
     pipelines.kubeflow.org/pipeline_spec: '{"description": "Any Sequencer Component
       Demo", "name": "any-sequencer"}'
+  labels:
+    pipelines.kubeflow.org/pipelinename: ''
+    pipelines.kubeflow.org/generation: ''
 spec:
   pipelineSpec:
     tasks:
@@ -46,8 +49,6 @@ spec:
           image: alpine:latest
         metadata:
           labels:
-            pipelines.kubeflow.org/pipelinename: ''
-            pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
           annotations:
             pipelines.kubeflow.org/component_spec_digest: '{"name": "sleepComponent",
@@ -65,8 +66,6 @@ spec:
           image: alpine:latest
         metadata:
           labels:
-            pipelines.kubeflow.org/pipelinename: ''
-            pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
           annotations:
             pipelines.kubeflow.org/component_spec_digest: '{"name": "sleepComponent",
@@ -84,8 +83,6 @@ spec:
           image: alpine:latest
         metadata:
           labels:
-            pipelines.kubeflow.org/pipelinename: ''
-            pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
           annotations:
             pipelines.kubeflow.org/component_spec_digest: '{"name": "sleepComponent",
@@ -150,8 +147,6 @@ spec:
           description: /tmp/outputs/Output/data
         metadata:
           labels:
-            pipelines.kubeflow.org/pipelinename: ''
-            pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
           annotations:
             pipelines.kubeflow.org/component_spec_digest: '{"name": "Flip coin", "outputs":
@@ -171,8 +166,6 @@ spec:
           image: alpine:latest
         metadata:
           labels:
-            pipelines.kubeflow.org/pipelinename: ''
-            pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
           annotations:
             pipelines.kubeflow.org/component_spec_digest: '{"name": "sleepComponent",
@@ -205,8 +198,6 @@ spec:
           description: /tmp/outputs/status/data
         metadata:
           labels:
-            pipelines.kubeflow.org/pipelinename: ''
-            pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
           annotations:
             pipelines.kubeflow.org/component_spec_digest: '{"name": "any_test", "outputs":

diff --git a/sdk/python/tests/compiler/testdata/any_sequencer_looped.yaml b/sdk/python/tests/compiler/testdata/any_sequencer_looped.yaml
@@ -40,6 +40,9 @@ metadata:
     pipelines.kubeflow.org/big_data_passing_format: $(workspaces.$TASK_NAME.path)/artifacts/$ORIG_PR_NAME/$TASKRUN_NAME/$TASK_PARAM_NAME
     pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "[\"a\", \"b\",
       \"c\"]", "name": "param", "optional": true, "type": "JsonArray"}], "name": "any-sequencer-looped"}'
+  labels:
+    pipelines.kubeflow.org/pipelinename: ''
+    pipelines.kubeflow.org/generation: ''
 spec:
   params:
   - name: param
@@ -68,8 +71,6 @@ spec:
           description: /tmp/outputs/output_value/data
         metadata:
           labels:
-            pipelines.kubeflow.org/pipelinename: ''
-            pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
           annotations:
             pipelines.kubeflow.org/component_spec_digest: '{"name": "print-00", "outputs":
@@ -96,8 +97,6 @@ spec:
           description: /tmp/outputs/output_value/data
         metadata:
           labels:
-            pipelines.kubeflow.org/pipelinename: ''
-            pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
           annotations:
             pipelines.kubeflow.org/component_spec_digest: '{"name": "print-01", "outputs":
@@ -127,8 +126,6 @@ spec:
           description: /tmp/outputs/status/data
         metadata:
           labels:
-            pipelines.kubeflow.org/pipelinename: ''
-            pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
           annotations:
             pipelines.kubeflow.org/component_spec_digest: '{"name": "any-seq-0", "outputs":
@@ -176,8 +173,6 @@ spec:
                   description: /tmp/outputs/output_value/data
                 metadata:
                   labels:
-                    pipelines.kubeflow.org/pipelinename: ''
-                    pipelines.kubeflow.org/generation: ''
                     pipelines.kubeflow.org/cache_enabled: "true"
                   annotations:
                     pipelines.kubeflow.org/component_spec_digest: '{"name": "print-10",
@@ -204,8 +199,6 @@ spec:
                   description: /tmp/outputs/output_value/data
                 metadata:
                   labels:
-                    pipelines.kubeflow.org/pipelinename: ''
-                    pipelines.kubeflow.org/generation: ''
                     pipelines.kubeflow.org/cache_enabled: "true"
                   annotations:
                     pipelines.kubeflow.org/component_spec_digest: '{"name": "print-11",
@@ -235,8 +228,6 @@ spec:
                   description: /tmp/outputs/status/data
                 metadata:
                   labels:
-                    pipelines.kubeflow.org/pipelinename: ''
-                    pipelines.kubeflow.org/generation: ''
                     pipelines.kubeflow.org/cache_enabled: "true"
                   annotations:
                     pipelines.kubeflow.org/component_spec_digest: '{"name": "any-seq-1",
@@ -257,7 +248,5 @@ spec:
           iterateParam: param-loop-item
         metadata:
           labels:
-            pipelines.kubeflow.org/pipelinename: ''
-            pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
   timeout: 525600m
diff --git a/sdk/python/tests/compiler/testdata/any_sequencer_looped_noninlined.yaml b/sdk/python/tests/compiler/testdata/any_sequencer_looped_noninlined.yaml
@@ -49,17 +49,15 @@ metadata:
       paramter.\", \"name\": \"output_value\", \"type\": \"String\"}], \"version\":
       \"print-10@sha256=ea7fa8edfd92cf64a15f8464292143a4e0d5f34898368bfd63618884decf3be1\"}",
       "tekton.dev/template": ""}, "labels": {"pipelines.kubeflow.org/cache_enabled":
-      "true", "pipelines.kubeflow.org/generation": "", "pipelines.kubeflow.org/pipelinename":
-      ""}}, "results": [{"description": "/tmp/outputs/output_value/data", "name":
+      "true"}}, "results": [{"description": "/tmp/outputs/output_value/data", "name":
       "output-value", "type": "string"}], "steps": [{"command": ["sh", "-c", "set
       -e\necho $0 > $1\n", "print-10", "$(results.output-value.path)"], "image": "alpine:3.6",
       "name": "main"}]}, "timeout": "525600m"}, {"name": "print-11", "taskSpec": {"metadata":
       {"annotations": {"pipelines.kubeflow.org/component_spec_digest": "{\"name\":
       \"print-11\", \"outputs\": [{\"description\": \"Represents an output paramter.\",
       \"name\": \"output_value\", \"type\": \"String\"}], \"version\": \"print-11@sha256=3b423bb0ced0bc21fb9157ce814a38c94d3e72718a33cfa8b4ef4f75abe725bd\"}",
       "tekton.dev/template": ""}, "labels": {"pipelines.kubeflow.org/cache_enabled":
-      "true", "pipelines.kubeflow.org/generation": "", "pipelines.kubeflow.org/pipelinename":
-      ""}}, "results": [{"description": "/tmp/outputs/output_value/data", "name":
+      "true"}}, "results": [{"description": "/tmp/outputs/output_value/data", "name":
       "output-value", "type": "string"}], "steps": [{"command": ["sh", "-c", "set
       -e\necho $0 > $1\n", "print-11", "$(results.output-value.path)"], "image": "alpine:3.6",
       "name": "main"}]}, "timeout": "525600m"}, {"name": "any-seq-1", "params": [{"name":
@@ -69,13 +67,15 @@ metadata:
       \"outputs\": [{\"description\": \"The output file to create the status\", \"name\":
       \"status\"}], \"version\": \"any-seq-1@sha256=75c65b3423dd61b79c35122746a1f16d67a98ab7b570daa36de2d310e1e57c22\"}",
       "tekton.dev/template": ""}, "labels": {"pipelines.kubeflow.org/cache_enabled":
-      "true", "pipelines.kubeflow.org/generation": "", "pipelines.kubeflow.org/pipelinename":
-      ""}}, "params": [{"name": "pipelineRun-name", "type": "string"}, {"name": "pipelineRun-namespace",
-      "type": "string"}], "results": [{"description": "/tmp/outputs/status/data",
+      "true"}}, "params": [{"name": "pipelineRun-name", "type": "string"}, {"name":
+      "pipelineRun-namespace", "type": "string"}], "results": [{"description": "/tmp/outputs/status/data",
       "name": "status", "type": "string"}], "steps": [{"args": ["--namespace", "$(params.pipelineRun-namespace)",
       "--prName", "$(params.pipelineRun-name)", "--taskList", "print-10,print-11",
       "--statusPath", "$(results.status.path)"], "command": ["any-task"], "image":
       "dspipelines/any-sequencer:latest", "name": "main"}]}, "timeout": "525600m"}]}}}]'
+  labels:
+    pipelines.kubeflow.org/pipelinename: ''
+    pipelines.kubeflow.org/generation: ''
 spec:
   params:
   - name: param
@@ -104,8 +104,6 @@ spec:
           description: /tmp/outputs/output_value/data
         metadata:
           labels:
-            pipelines.kubeflow.org/pipelinename: ''
-            pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
           annotations:
             pipelines.kubeflow.org/component_spec_digest: '{"name": "print-00", "outputs":
@@ -132,8 +130,6 @@ spec:
           description: /tmp/outputs/output_value/data
         metadata:
           labels:
-            pipelines.kubeflow.org/pipelinename: ''
-            pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
           annotations:
             pipelines.kubeflow.org/component_spec_digest: '{"name": "print-01", "outputs":
@@ -163,8 +159,6 @@ spec:
           description: /tmp/outputs/status/data
         metadata:
           labels:
-            pipelines.kubeflow.org/pipelinename: ''
-            pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
           annotations:
             pipelines.kubeflow.org/component_spec_digest: '{"name": "any-seq-0", "outputs":

diff --git a/sdk/python/tests/compiler/testdata/artifact_outputs.yaml b/sdk/python/tests/compiler/testdata/artifact_outputs.yaml
@@ -31,6 +31,9 @@ metadata:
     pipelines.kubeflow.org/pipeline_spec: '{"description": "Add labels to identify
       outputs as artifacts.", "inputs": [{"default": "gs://ml-pipeline-playground/shakespeare1.txt",
       "name": "url1", "optional": true, "type": "String"}], "name": "artifact-out-pipeline"}'
+  labels:
+    pipelines.kubeflow.org/pipelinename: ''
+    pipelines.kubeflow.org/generation: ''
 spec:
   params:
   - name: url1
@@ -109,8 +112,6 @@ spec:
           type: string
         metadata:
           labels:
-            pipelines.kubeflow.org/pipelinename: ''
-            pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
           annotations:
             artifact_outputs: '["data"]'

diff --git a/sdk/python/tests/compiler/testdata/artifact_passing_using_volume.yaml b/sdk/python/tests/compiler/testdata/artifact_passing_using_volume.yaml
@@ -41,6 +41,9 @@ metadata:
     sidecar.istio.io/inject: "false"
     pipelines.kubeflow.org/big_data_passing_format: $(workspaces.$TASK_NAME.path)/artifacts/$ORIG_PR_NAME/$TASKRUN_NAME/$TASK_PARAM_NAME
     pipelines.kubeflow.org/pipeline_spec: '{"name": "Artifact passing pipeline"}'
+  labels:
+    pipelines.kubeflow.org/pipelinename: ''
+    pipelines.kubeflow.org/generation: ''
 spec:
   pipelineSpec:
     tasks:
@@ -110,8 +113,6 @@ spec:
           type: string
         metadata:
           labels:
-            pipelines.kubeflow.org/pipelinename: ''
-            pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
           annotations:
             pipelines.kubeflow.org/component_spec_digest: '{"name": "Producer", "outputs":
@@ -199,8 +200,6 @@ spec:
           type: string
         metadata:
           labels:
-            pipelines.kubeflow.org/pipelinename: ''
-            pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
           annotations:
             pipelines.kubeflow.org/component_spec_digest: '{"name": "Processor", "outputs":
@@ -243,8 +242,6 @@ spec:
         - name: processor-trname
         metadata:
           labels:
-            pipelines.kubeflow.org/pipelinename: ''
-            pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
           annotations:
             pipelines.kubeflow.org/component_spec_digest: '{"name": "Consumer", "outputs":
@@ -337,8 +334,6 @@ spec:
           emptyDir: {}
         metadata:
           labels:
-            pipelines.kubeflow.org/pipelinename: ''
-            pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
           annotations:
             pipelines.kubeflow.org/component_spec_digest: '{"name": "Metadata and

diff --git a/sdk/python/tests/compiler/testdata/basic_no_decorator.yaml b/sdk/python/tests/compiler/testdata/basic_no_decorator.yaml
@@ -72,8 +72,6 @@ spec:
           description: /tmp/outputs/word/data
         metadata:
           labels:
-            pipelines.kubeflow.org/pipelinename: ''
-            pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
           annotations:
             pipelines.kubeflow.org/component_spec_digest: '{"name": "frequent-word",
@@ -107,8 +105,6 @@ spec:
           value: default_output
         metadata:
           labels:
-            pipelines.kubeflow.org/pipelinename: ''
-            pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
           annotations:
             pipelines.kubeflow.org/component_spec_digest: '{"name": "save-message",
@@ -127,8 +123,6 @@ spec:
           image: python:3.6-jessie
         metadata:
           labels:
-            pipelines.kubeflow.org/pipelinename: ''
-            pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
           annotations:
             pipelines.kubeflow.org/component_spec_digest: '{"name": "exit-handler",