remove ContainerOp from test cases - part 1 (kubeflow#685)

eliminate the usage of ContainerOp in testcases and
update the testcase accordingly.

Signed-off-by: Yihong Wang <yh.wang@ibm.com>

sdk/python/tests/compiler/testdata/affinity.py

@@ -13,16 +13,22 @@
 # limitations under the License.
 
 from kubernetes.client import V1Affinity, V1NodeSelector, V1NodeSelectorRequirement, V1NodeSelectorTerm, V1NodeAffinity
-from kfp import dsl
+from kfp import dsl, components
 
+ECHO_OP_STR = """
+name: echo
+description: echo component
+implementation:
+  container:
+    image: busybox
+    command:
+    - sh
+    - -c
+    args:
+    - echo "Got scheduled"
+"""
 
-def echo_op():
-    return dsl.ContainerOp(
-        name='echo',
-        image='busybox',
-        command=['sh', '-c'],
-        arguments=['echo "Got scheduled"']
-    )
+echo_op = components.load_component_from_text(ECHO_OP_STR)
 
 
 @dsl.pipeline(

sdk/python/tests/compiler/testdata/affinity.yaml

@@ -45,6 +45,9 @@ spec:
             pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
           annotations:
+            pipelines.kubeflow.org/component_spec: '{"description": "echo component",
+              "implementation": {"container": {"args": ["echo \"Got scheduled\""],
+              "command": ["sh", "-c"], "image": "busybox"}}, "name": "echo"}'
             tekton.dev/template: ''
       timeout: 0s
   taskRunSpecs:

sdk/python/tests/compiler/testdata/affinity_noninlined.yaml

@@ -45,6 +45,9 @@ spec:
             pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
           annotations:
+            pipelines.kubeflow.org/component_spec: '{"description": "echo component",
+              "implementation": {"container": {"args": ["echo \"Got scheduled\""],
+              "command": ["sh", "-c"], "image": "busybox"}}, "name": "echo"}'
             tekton.dev/template: ''
       timeout: 0s
   taskRunSpecs:

sdk/python/tests/compiler/testdata/any_sequencer.py

@@ -18,6 +18,17 @@
 from kfp_tekton.tekton import after_any
 
 
+# Note:
+# any_sequencer needs k8s permission to watch the pipelinerun/taskrun
+# status. Therefore, make sure the service account that is used to
+# run the pipeline has sufficient permission. For example, in
+# multi-user deployment, you need to run the follow command to add
+# `cluster-admin` cluster-role to the `default-editor` service account
+# under your namespace:
+#   # assuming the namespace is: mynamespace
+#   kubectl create clusterrolebinding pipeline-runner-extend \
+#     --clusterrole cluster-admin --serviceaccount=mynamespace:default-editor
+
 def flip_coin() -> str:
     """Flip a coin and output heads or tails randomly."""
     import random
@@ -48,7 +59,7 @@ def flip_coin() -> str:
 
 
 @dsl.pipeline(
-    name="Any Sequencer",
+    name="any-sequencer",
     description="Any Sequencer Component Demo",
 )
 def any_sequence_pipeline(

sdk/python/tests/compiler/testdata/any_sequencer.yaml

@@ -30,7 +30,7 @@ metadata:
       [], "sleepcomponent-3": [], "sleepcomponent-4": []}'
     sidecar.istio.io/inject: "false"
     pipelines.kubeflow.org/pipeline_spec: '{"description": "Any Sequencer Component
-      Demo", "name": "Any Sequencer"}'
+      Demo", "name": "any-sequencer"}'
 spec:
   pipelineSpec:
     tasks:

sdk/python/tests/compiler/testdata/any_sequencer_noninlined.yaml

@@ -30,7 +30,7 @@ metadata:
       [], "sleepcomponent-3": [], "sleepcomponent-4": []}'
     sidecar.istio.io/inject: "false"
     pipelines.kubeflow.org/pipeline_spec: '{"description": "Any Sequencer Component
-      Demo", "name": "Any Sequencer"}'
+      Demo", "name": "any-sequencer"}'
 spec:
   pipelineSpec:
     tasks:

sdk/python/tests/compiler/testdata/artifact_passing_using_volume.py

@@ -53,4 +53,5 @@ def artifact_passing_pipeline():
 if __name__ == '__main__':
     pipeline_conf = kfp.dsl.PipelineConf()
     pipeline_conf.data_passing_method = volume_based_data_passing_method
-    kfp.compiler.Compiler().compile(artifact_passing_pipeline, __file__ + '.yaml', pipeline_conf=pipeline_conf)
+    from kfp_tekton.compiler import TektonCompiler
+    TektonCompiler().compile(artifact_passing_pipeline, __file__.replace('.py', '.yaml'), pipeline_conf=pipeline_conf)

sdk/python/tests/compiler/testdata/basic_no_decorator.py

@@ -12,75 +12,81 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import kfp.dsl as dsl
+from kfp import dsl, components
 # import kfp.gcp as gcp
 
 
 message_param = dsl.PipelineParam(name='message', value='When flies fly behind flies')
 output_path_param = dsl.PipelineParam(name='outputpath', value='default_output')
 
+FREQUENT_WORD_STR = """
+name: frequent-word
+description: Calculate the frequent word from a text
+inputs:
+  - {name: message, type: String, description: 'Required. message'}
+outputs:
+  - {name: word, type: String}
+implementation:
+  container:
+    image: python:3.6-jessie
+    command:
+    - sh
+    - -c
+    - |
+      python -c "from collections import Counter; \
+      words = Counter('$0'.split()); print(max(words, key=words.get))" \
+      | tee $1
+    - {inputValue: message}
+    - {outputPath: word}
+"""
 
-class GetFrequentWordOp(dsl.ContainerOp):
-  """A get frequent word class representing a component in ML Pipelines.
-  The class provides a nice interface to users by hiding details such as container,
-  command, arguments.
-  """
-  def __init__(self, name, message="When flies fly behind flies,"
-                                   " then flies are following flies."):
-    """__init__
-    Args:
-      name: An identifier of the step which needs to be unique within a pipeline.
-      message: a dsl.PipelineParam object representing an input message.
-    """
-    super(GetFrequentWordOp, self).__init__(
-        name=name,
-        image='python:3.6-jessie',
-        command=['sh', '-c'],
-        arguments=['python -c "from collections import Counter; '
-                   'words = Counter(\'%s\'.split()); print(max(words, key=words.get))" '
-                   '| tee /tmp/message.txt' % message],
-        file_outputs={'word': '/tmp/message.txt'})
+frequent_word_op = components.load_component_from_text(FREQUENT_WORD_STR)
 
+SAVE_MESSAGE_STR = """
+name: save-message
+description: |
+  save message to a given output_path
+inputs:
+  - {name: message, type: String, description: 'Required. message'}
+  - {name: output_path, type: String, description: 'Required. output path'}
+implementation:
+  container:
+    image: google/cloud-sdk
+    command:
+    - sh
+    - -c
+    - |
+      set -e
+      echo "$0"| gsutil cp - "$1"
+    - {inputValue: message}
+    - {inputValue: output_path}
+"""
 
-class SaveMessageOp(dsl.ContainerOp):
-  """A class representing a component in ML Pipelines.
-  It saves a message to a given output_path.
-  """
-  def __init__(self, name, message, output_path):
-    """Args:
-         name: An identifier of the step which needs to be unique within a pipeline.
-         message: a dsl.PipelineParam object representing the message to be saved.
-         output_path: a dsl.PipelineParam object representing the GCS path for output file.
-    """
-    super(SaveMessageOp, self).__init__(
-        name=name,
-        image='google/cloud-sdk',
-        command=['sh', '-c'],
-        arguments=['echo "%s" | tee /tmp/results.txt | gsutil cp /tmp/results.txt %s'
-                   % (message, output_path)])
+save_message_op = components.load_component_from_text(SAVE_MESSAGE_STR)
 
+EXIT_STR = """
+name: exit-handler
+description: exit function
+implementation:
+  container:
+    image: python:3.6-jessie
+    command:
+    - sh
+    - -c
+    - echo "exit!"
+"""
 
-class ExitHandlerOp(dsl.ContainerOp):
-  """A class representing a component in ML Pipelines."""
-  def __init__(self, name):
-    super(ExitHandlerOp, self).__init__(
-        name=name,
-        image='python:3.6-jessie',
-        command=['sh', '-c'],
-        arguments=['echo exit!'])
+exit_op = components.load_component_from_text(EXIT_STR)
 
 
 def save_most_frequent_word():
-  exit_op = ExitHandlerOp('exiting')
-  with dsl.ExitHandler(exit_op):
-    counter = GetFrequentWordOp(
-        name='get-Frequent',
-        message=message_param)
+  exit_task = exit_op()
+  with dsl.ExitHandler(exit_task):
+    counter = frequent_word_op(message=message_param)
     counter.container.set_memory_request('200M')
 
-    saver = SaveMessageOp(
-        name='save',
-        message=counter.output,
+    saver = save_message_op(
+        message=counter.outputs['word'],
         output_path=output_path_param)
     saver.container.set_cpu_limit('0.5')
     # saver.container.set_gpu_limit('2')

sdk/python/tests/compiler/testdata/basic_no_decorator.yaml

@@ -17,15 +17,15 @@ kind: PipelineRun
 metadata:
   name: save-most-frequent-word
   annotations:
-    tekton.dev/output_artifacts: '{"get-frequent": [{"key": "artifacts/$PIPELINERUN/get-frequent/word.tgz",
-      "name": "get-frequent-word", "path": "/tmp/message.txt"}]}'
-    tekton.dev/input_artifacts: '{"save": [{"name": "get-frequent-word", "parent_task":
-      "get-frequent"}]}'
+    tekton.dev/output_artifacts: '{"frequent-word": [{"key": "artifacts/$PIPELINERUN/frequent-word/word.tgz",
+      "name": "frequent-word-word", "path": "/tmp/outputs/word/data"}]}'
+    tekton.dev/input_artifacts: '{"save-message": [{"name": "frequent-word-word",
+      "parent_task": "frequent-word"}]}'
     tekton.dev/artifact_bucket: mlpipeline
     tekton.dev/artifact_endpoint: minio-service.kubeflow:9000
     tekton.dev/artifact_endpoint_scheme: http://
-    tekton.dev/artifact_items: '{"exiting": [], "get-frequent": [["word", "$(results.word.path)"]],
-      "save": []}'
+    tekton.dev/artifact_items: '{"exit-handler": [], "frequent-word": [["word", "$(results.word.path)"]],
+      "save-message": []}'
     sidecar.istio.io/inject: "false"
     pipelines.kubeflow.org/pipeline_spec: '{"description": "Get Most Frequent Word
       and Save to GCS", "inputs": [{"default": "When flies fly behind flies", "name":
@@ -44,19 +44,20 @@ spec:
     - name: outputpath
       default: default_output
     tasks:
-    - name: get-frequent
+    - name: frequent-word
       params:
       - name: message
         value: $(params.message)
       taskSpec:
         steps:
         - name: main
-          args:
-          - python -c "from collections import Counter; words = Counter('$(inputs.params.message)'.split());
-            print(max(words, key=words.get))" | tee $(results.word.path)
           command:
           - sh
           - -c
+          - |
+            python -c "from collections import Counter;       words = Counter('$0'.split()); print(max(words, key=words.get))"       | tee $1
+          - $(inputs.params.message)
+          - $(results.word.path)
           image: python:3.6-jessie
           resources:
             requests:
@@ -65,66 +66,85 @@ spec:
         - name: message
         results:
         - name: word
-          description: /tmp/message.txt
+          description: /tmp/outputs/word/data
         metadata:
           labels:
             pipelines.kubeflow.org/pipelinename: ''
             pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
           annotations:
+            pipelines.kubeflow.org/component_spec: '{"description": "Calculate the
+              frequent word from a text", "implementation": {"container": {"command":
+              ["sh", "-c", "python -c \"from collections import Counter;       words
+              = Counter(''$0''.split()); print(max(words, key=words.get))\"       |
+              tee $1\n", {"inputValue": "message"}, {"outputPath": "word"}], "image":
+              "python:3.6-jessie"}}, "inputs": [{"description": "Required. message",
+              "name": "message", "type": "String"}], "name": "frequent-word", "outputs":
+              [{"name": "word", "type": "String"}]}'
             tekton.dev/template: ''
       timeout: 0s
-    - name: save
+    - name: save-message
       params:
-      - name: get-frequent-word
-        value: $(tasks.get-frequent.results.word)
+      - name: frequent-word-word
+        value: $(tasks.frequent-word.results.word)
       - name: outputpath
         value: $(params.outputpath)
       taskSpec:
         steps:
         - name: main
-          args:
-          - echo "$(inputs.params.get-frequent-word)" | tee /tmp/results.txt | gsutil
-            cp /tmp/results.txt $(inputs.params.outputpath)
           command:
           - sh
           - -c
+          - |
+            set -e
+            echo "$0"| gsutil cp - "$1"
+          - $(inputs.params.frequent-word-word)
+          - $(inputs.params.outputpath)
           image: google/cloud-sdk
           resources:
             limits:
               cpu: '0.5'
         params:
-        - name: get-frequent-word
+        - name: frequent-word-word
         - name: outputpath
         metadata:
           labels:
             pipelines.kubeflow.org/pipelinename: ''
             pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
           annotations:
+            pipelines.kubeflow.org/component_spec: '{"description": "save message
+              to a given output_path\n", "implementation": {"container": {"command":
+              ["sh", "-c", "set -e\necho \"$0\"| gsutil cp - \"$1\"\n", {"inputValue":
+              "message"}, {"inputValue": "output_path"}], "image": "google/cloud-sdk"}},
+              "inputs": [{"description": "Required. message", "name": "message", "type":
+              "String"}, {"description": "Required. output path", "name": "output_path",
+              "type": "String"}], "name": "save-message"}'
             tekton.dev/template: ''
       timeout: 0s
     finally:
-    - name: exiting
+    - name: exit-handler
       taskSpec:
         steps:
         - name: main
-          args:
-          - echo exit!
           command:
           - sh
           - -c
+          - echo "exit!"
           image: python:3.6-jessie
         metadata:
           labels:
             pipelines.kubeflow.org/pipelinename: ''
             pipelines.kubeflow.org/generation: ''
             pipelines.kubeflow.org/cache_enabled: "true"
           annotations:
+            pipelines.kubeflow.org/component_spec: '{"description": "exit function",
+              "implementation": {"container": {"command": ["sh", "-c", "echo \"exit!\""],
+              "image": "python:3.6-jessie"}}, "name": "exit-handler"}'
             tekton.dev/template: ''
       timeout: 0s
   taskRunSpecs:
-  - pipelineTaskName: save
+  - pipelineTaskName: save-message
     taskPodTemplate:
       nodeSelector:
         kubernetes.io/os: linux