Enhance error messages for resource op and artifact tracking (kubeflo…

…w#572) * enchance error messages * update logs * update logs
red-hat-data-services · Apr 29, 2021 · 76e249d · 76e249d
1 parent 3179556
commit 76e249d
Show file tree

Hide file tree

Showing 10 changed files with 36 additions and 43 deletions.
diff --git a/manifests/kustomize/base/pipeline/kfp-pipeline-config.yaml b/manifests/kustomize/base/pipeline/kfp-pipeline-config.yaml
@@ -14,8 +14,12 @@ data:
   artifact_script: |-
     #!/usr/bin/env sh
     push_artifact() {
-        tar -cvzf $1.tgz $2
-        mc cp $1.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
+        if [ -f "$2" ]; then
+            tar -cvzf $1.tgz $2
+            mc cp $1.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
+        else
+            echo "$2 file does not exist. Skip artifact tracking for $1"
+        fi
     }
     push_log() {
         cat /var/log/containers/$PODNAME*$NAMESPACE*step-main*.log > step-main.log

diff --git a/sdk/python/kfp_tekton/compiler/_op_to_template.py b/sdk/python/kfp_tekton/compiler/_op_to_template.py
@@ -25,6 +25,10 @@
 from kfp.dsl._container_op import BaseOp
 
 from kfp_tekton.compiler import __tekton_api_version__ as tekton_api_version
+from .. import __version__
+
+
+RESOURCE_OP_IMAGE = ":".join(["aipipeline/kubectl-wrapper", __version__])
 
 
 def _get_base_step(name: str):
@@ -102,7 +106,7 @@ def _get_resourceOp_template(op: BaseOp,
                 {
                     # This image is hosted by the kfp-tekton maintainers
                     # Source code: https://github.com/kubeflow/kfp-tekton/tree/master/tekton-catalog/kubectl-wrapper
-                    "default": "index.docker.io/aipipeline/kubeclient:v0.0.2",
+                    "default": RESOURCE_OP_IMAGE,
                     "description": "Kubectl wrapper image",
                     "name": "image",
                     "type": "string"

diff --git a/sdk/python/tests/compiler/testdata/artifact_passing_using_volume.log b/sdk/python/tests/compiler/testdata/artifact_passing_using_volume.log
@@ -1,26 +1,23 @@
 
 [producer : copy-artifacts] Added `storage` successfully.
-[producer : copy-artifacts] tekton/results/output-1
 [producer : copy-artifacts] tar: removing leading '/' from member names
+[producer : copy-artifacts] tekton/results/output-1
 [producer : copy-artifacts] `Output-1.tgz` -> `storage/mlpipeline/artifacts/artifact-passing-pipeline/producer/Output-1.tgz`
-[producer : copy-artifacts] Total: 0 B, Transferred: 121 B, Speed: 29 B/s
+[producer : copy-artifacts] Total: 0 B, Transferred: 121 B, Speed: 31 B/s
 [producer : copy-artifacts] tar: removing leading '/' from member names
 [producer : copy-artifacts] workspace/producer/producer-Output-2
 [producer : copy-artifacts] `Output-2.tgz` -> `storage/mlpipeline/artifacts/artifact-passing-pipeline/producer/Output-2.tgz`
-[producer : copy-artifacts] Total: 0 B, Transferred: 144 B, Speed: 41 B/s
+[producer : copy-artifacts] Total: 0 B, Transferred: 143 B, Speed: 44 B/s
 
 
 [processor : copy-artifacts] Added `storage` successfully.
-[processor : copy-artifacts] tekton/results/output-1
 [processor : copy-artifacts] tar: removing leading '/' from member names
+[processor : copy-artifacts] tekton/results/output-1
 [processor : copy-artifacts] `Output-1.tgz` -> `storage/mlpipeline/artifacts/artifact-passing-pipeline/processor/Output-1.tgz`
-[processor : copy-artifacts] Total: 0 B, Transferred: 121 B, Speed: 39 B/s
-[processor : copy-artifacts] /tekton/scripts/script-1-b4jcg: line 17: workspaces.producer.path: not found
-[processor : copy-artifacts] tar: /producer-Output-2: No such file or directory
-[processor : copy-artifacts] tar: error exit delayed from previous errors
-[processor : copy-artifacts] `Output-2.tgz` -> `storage/mlpipeline/artifacts/artifact-passing-pipeline/processor/Output-2.tgz`
-[processor : copy-artifacts] Total: 0 B, Transferred: 29 B, Speed: 8 B/s
-[processor : copy-artifacts] /tekton/scripts/script-1-b4jcg: line 19: workspaces.producer.path: not found
+[processor : copy-artifacts] Total: 0 B, Transferred: 121 B, Speed: 42 B/s
+[processor : copy-artifacts] /tekton/scripts/script-1-v92rb: line 21: workspaces.producer.path: not found
+[processor : copy-artifacts] /producer-Output-2 file does not exist. Skip artifact tracking for Output-2
+[processor : copy-artifacts] /tekton/scripts/script-1-v92rb: line 23: workspaces.producer.path: not found
 
 [consumer : main] Input parameter = Data 1
 [consumer : main] Input artifact = 

diff --git a/sdk/python/tests/compiler/testdata/load_from_yaml.log b/sdk/python/tests/compiler/testdata/load_from_yaml.log
@@ -1,8 +1,5 @@
 [busybox : main] start  /tekton/results/dummy-output-path end
 
 [busybox : copy-artifacts] Added `storage` successfully.
-[busybox : copy-artifacts] tar: /tekton/results/dummy-output-path: No such file or directory
-[busybox : copy-artifacts] tar: error exit delayed from previous errors
-[busybox : copy-artifacts] `dummy_output_path.tgz` -> `storage/mlpipeline/artifacts/component-yaml-pipeline/busybox/dummy_output_path.tgz`
-[busybox : copy-artifacts] Total: 0 B, Transferred: 29 B, Speed: 5 B/s
+[busybox : copy-artifacts] /tekton/results/dummy-output-path file does not exist. Skip artifact tracking for dummy_output_path
 
diff --git a/sdk/python/tests/compiler/testdata/resourceop_basic.yaml b/sdk/python/tests/compiler/testdata/resourceop_basic.yaml
@@ -67,7 +67,7 @@ spec:
             is failure.
           name: failure-condition
           type: string
-        - default: index.docker.io/aipipeline/kubeclient:v0.0.2
+        - default: aipipeline/kubectl-wrapper:0.8.0
           description: Kubectl wrapper image
           name: image
           type: string

diff --git a/sdk/python/tests/compiler/testdata/volume_op.log b/sdk/python/tests/compiler/testdata/volume_op.log
@@ -1,12 +1,9 @@
-[create-pvc : main] time="2021-04-23T20:28:56Z" level=error msg="Initialize script failed: exit status 2:"
-[create-pvc : main] time="2021-04-23T20:28:56Z" level=info msg="kubectl create -f /tmp/manifest.yaml -o json"
-[create-pvc : main] time="2021-04-23T20:28:56Z" level=info msg=kubeflow/PersistentVolumeClaim./volumeop-basic-my-pvc
-[create-pvc : main] time="2021-04-23T20:28:56Z" level=info msg="Saving resource output parameters"
-[create-pvc : main] time="2021-04-23T20:28:56Z" level=info msg="[kubectl get PersistentVolumeClaim./volumeop-basic-my-pvc -o jsonpath={} -n kubeflow]"
-[create-pvc : main] time="2021-04-23T20:28:56Z" level=info msg="Saved output parameter: manifest, value: map[apiVersion:v1 kind:PersistentVolumeClaim metadata:map[annotations:map[ibm.io/provisioning-status:{\"status\":\"inprogress\",\"time\":\"2021-04-23T20:28:56Z\",\"attempt\":1,\"retry\":false,\"pluginid\":\"ibm-file-plugin-1619200208\",\"pvcid\":\"8d0a5e2a-7e12-46eb-8c46-dc40e2ceb280\"} volume.beta.kubernetes.io/storage-provisioner:ibm.io/ibmc-file] creationTimestamp:2021-04-23T20:28:56Z finalizers:[kubernetes.io/pvc-protection] labels:map[region:us-south zone:sjc04] managedFields:[map[apiVersion:v1 fieldsType:FieldsV1 fieldsV1:map[f:metadata:map[f:annotations:map[f:ibm.io/provisioning-status:map[]] f:labels:map[.:map[] f:region:map[] f:zone:map[]]]] manager:armada-storage-file-plugin operation:Update time:2021-04-23T20:28:56Z] map[apiVersion:v1 fieldsType:FieldsV1 fieldsV1:map[f:metadata:map[f:annotations:map[.:map[] f:volume.beta.kubernetes.io/storage-provisioner:map[]]]] manager:kube-controller-manager operation:Update time:2021-04-23T20:28:56Z] map[apiVersion:v1 fieldsType:FieldsV1 fieldsV1:map[f:spec:map[f:accessModes:map[] f:resources:map[f:requests:map[.:map[] f:storage:map[]]] f:volumeMode:map[]] f:status:map[f:phase:map[]]] manager:kubectl operation:Update time:2021-04-23T20:28:56Z]] name:volumeop-basic-my-pvc namespace:kubeflow resourceVersion:98730 selfLink:/api/v1/namespaces/kubeflow/persistentvolumeclaims/volumeop-basic-my-pvc uid:8d0a5e2a-7e12-46eb-8c46-dc40e2ceb280] spec:map[accessModes:[ReadWriteOnce] resources:map[requests:map[storage:10M]] storageClassName:ibmc-file-gold-gid volumeMode:Filesystem] status:map[phase:Pending]]"
-[create-pvc : main] time="2021-04-23T20:28:56Z" level=info msg="[kubectl get PersistentVolumeClaim./volumeop-basic-my-pvc -o jsonpath={.metadata.name} -n kubeflow]"
-[create-pvc : main] time="2021-04-23T20:28:57Z" level=info msg="Saved output parameter: name, value: volumeop-basic-my-pvc"
-[create-pvc : main] time="2021-04-23T20:28:57Z" level=info msg="[kubectl get PersistentVolumeClaim./volumeop-basic-my-pvc -o jsonpath={.status.capacity.storage} -n kubeflow]"
-[create-pvc : main] time="2021-04-23T20:28:57Z" level=info msg="Saved output parameter: size, value: "
-
-
+[create-pvc : main] time="2021-04-29T01:02:11Z" level=info msg="kubectl create -f /tmp/manifest.yaml -o json"
+[create-pvc : main] time="2021-04-29T01:02:11Z" level=info msg=kubeflow/PersistentVolumeClaim./volumeop-basic-63777-my-pvc
+[create-pvc : main] time="2021-04-29T01:02:11Z" level=info msg="Saving resource output parameters"
+[create-pvc : main] time="2021-04-29T01:02:11Z" level=info msg="[kubectl get PersistentVolumeClaim./volumeop-basic-63777-my-pvc -o jsonpath={} -n kubeflow]"
+[create-pvc : main] time="2021-04-29T01:02:12Z" level=info msg="Saved output parameter: manifest, value: {\"apiVersion\":\"v1\",\"kind\":\"PersistentVolumeClaim\",\"metadata\":{\"annotations\":{\"ibm.io/provisioning-status\":\"{\\\"status\\\":\\\"inprogress\\\",\\\"time\\\":\\\"2021-04-29T01:02:11Z\\\",\\\"attempt\\\":1,\\\"retry\\\":false,\\\"pluginid\\\":\\\"ibm-file-plugin-1619479127\\\",\\\"pvcid\\\":\\\"2dac3c70-eb65-40c2-9d1f-5632dad10232\\\"}\",\"volume.beta.kubernetes.io/storage-provisioner\":\"ibm.io/ibmc-file\"},\"creationTimestamp\":\"2021-04-29T01:02:11Z\",\"finalizers\":[\"kubernetes.io/pvc-protection\"],\"labels\":{\"region\":\"us-south\",\"zone\":\"sjc04\"},\"managedFields\":[{\"apiVersion\":\"v1\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\".\":{},\"f:volume.beta.kubernetes.io/storage-provisioner\":{}}}},\"manager\":\"kube-controller-manager\",\"operation\":\"Update\",\"time\":\"2021-04-29T01:02:11Z\"},{\"apiVersion\":\"v1\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:spec\":{\"f:accessModes\":{},\"f:resources\":{\"f:requests\":{\".\":{},\"f:storage\":{}}},\"f:volumeMode\":{}},\"f:status\":{\"f:phase\":{}}},\"manager\":\"kubectl-create\",\"operation\":\"Update\",\"time\":\"2021-04-29T01:02:11Z\"},{\"apiVersion\":\"v1\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\"f:ibm.io/provisioning-status\":{}},\"f:labels\":{\".\":{},\"f:region\":{},\"f:zone\":{}}}},\"manager\":\"armada-storage-file-plugin\",\"operation\":\"Update\",\"time\":\"2021-04-29T01:02:12Z\"}],\"name\":\"volumeop-basic-63777-my-pvc\",\"namespace\":\"kubeflow\",\"resourceVersion\":\"1402962\",\"selfLink\":\"/api/v1/namespaces/kubeflow/persistentvolumeclaims/volumeop-basic-63777-my-pvc\",\"uid\":\"2dac3c70-eb65-40c2-9d1f-5632dad10232\"},\"spec\":{\"accessModes\":[\"ReadWriteOnce\"],\"resources\":{\"requests\":{\"storage\":\"10M\"}},\"storageClassName\":\"ibmc-file-gold-gid\",\"volumeMode\":\"Filesystem\"},\"status\":{\"phase\":\"Pending\"}}"
+[create-pvc : main] time="2021-04-29T01:02:12Z" level=info msg="[kubectl get PersistentVolumeClaim./volumeop-basic-63777-my-pvc -o jsonpath={.metadata.name} -n kubeflow]"
+[create-pvc : main] time="2021-04-29T01:02:12Z" level=info msg="Saved output parameter: name, value: volumeop-basic-63777-my-pvc"
+[create-pvc : main] time="2021-04-29T01:02:12Z" level=info msg="[kubectl get PersistentVolumeClaim./volumeop-basic-63777-my-pvc -o jsonpath={.status.capacity.storage} -n kubeflow]"
+[create-pvc : main] time="2021-04-29T01:02:12Z" level=info msg="Saved output parameter: size, value: "
diff --git a/sdk/python/tests/compiler/testdata/volume_op.yaml b/sdk/python/tests/compiler/testdata/volume_op.yaml
@@ -78,7 +78,7 @@ spec:
             is failure.
           name: failure-condition
           type: string
-        - default: index.docker.io/aipipeline/kubeclient:v0.0.2
+        - default: aipipeline/kubectl-wrapper:0.8.0
           description: Kubectl wrapper image
           name: image
           type: string

diff --git a/sdk/python/tests/compiler/testdata/volume_snapshot_op.yaml b/sdk/python/tests/compiler/testdata/volume_snapshot_op.yaml
@@ -84,7 +84,7 @@ spec:
             is failure.
           name: failure-condition
           type: string
-        - default: index.docker.io/aipipeline/kubeclient:v0.0.2
+        - default: aipipeline/kubectl-wrapper:0.8.0
           description: Kubectl wrapper image
           name: image
           type: string
@@ -197,7 +197,7 @@ spec:
             is failure.
           name: failure-condition
           type: string
-        - default: index.docker.io/aipipeline/kubeclient:v0.0.2
+        - default: aipipeline/kubectl-wrapper:0.8.0
           description: Kubectl wrapper image
           name: image
           type: string
@@ -310,7 +310,7 @@ spec:
             is failure.
           name: failure-condition
           type: string
-        - default: index.docker.io/aipipeline/kubeclient:v0.0.2
+        - default: aipipeline/kubectl-wrapper:0.8.0
           description: Kubectl wrapper image
           name: image
           type: string
@@ -423,7 +423,7 @@ spec:
             is failure.
           name: failure-condition
           type: string
-        - default: index.docker.io/aipipeline/kubeclient:v0.0.2
+        - default: aipipeline/kubectl-wrapper:0.8.0
           description: Kubectl wrapper image
           name: image
           type: string

diff --git a/tekton-catalog/kubectl-wrapper/cmd/client/main.go b/tekton-catalog/kubectl-wrapper/cmd/client/main.go
@@ -69,12 +69,6 @@ func main() {
 		}
 	}
 
-	// cmd := exec.Command("/bin/sh", "/builder/kubectl.bash")
-	// _, err = cmd.Output()
-	// if err != nil {
-	// 	log.Errorf("Initialize script failed: %+v:", err)
-	// }
-
 	isDelete := action == "delete"
 	resourceNamespace, resourceName, err := execResource(action, mergeStrategy)
 	if err != nil {

diff --git a/tekton-catalog/kubectl-wrapper/deploy/kubectl-deploy.yaml b/tekton-catalog/kubectl-wrapper/deploy/kubectl-deploy.yaml
@@ -25,7 +25,7 @@ spec:
     default: "false"
     description: Enable set owner reference for created resource.
   - name: image
-    default: index.docker.io/aipipeline/kubeclient:v0.0.2 # it is huge
+    default: aipipeline/kubeclient:0.8.0 # it is huge
     description: Kubectl wrapper image
   steps:
   - name: kubeconfig