From 76e249d4d98c769686d8c0576b4e911701e38e47 Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Thu, 29 Apr 2021 14:53:30 -0700 Subject: [PATCH] Enhance error messages for resource op and artifact tracking (#572) * enchance error messages * update logs * update logs --- .../base/pipeline/kfp-pipeline-config.yaml | 8 +++++-- .../kfp_tekton/compiler/_op_to_template.py | 6 +++++- .../artifact_passing_using_volume.log | 19 +++++++---------- .../compiler/testdata/load_from_yaml.log | 5 +---- .../compiler/testdata/resourceop_basic.yaml | 2 +- .../tests/compiler/testdata/volume_op.log | 21 ++++++++----------- .../tests/compiler/testdata/volume_op.yaml | 2 +- .../compiler/testdata/volume_snapshot_op.yaml | 8 +++---- .../kubectl-wrapper/cmd/client/main.go | 6 ------ .../deploy/kubectl-deploy.yaml | 2 +- 10 files changed, 36 insertions(+), 43 deletions(-) diff --git a/manifests/kustomize/base/pipeline/kfp-pipeline-config.yaml b/manifests/kustomize/base/pipeline/kfp-pipeline-config.yaml index 2cc5013151b..50e4da4f1f3 100644 --- a/manifests/kustomize/base/pipeline/kfp-pipeline-config.yaml +++ b/manifests/kustomize/base/pipeline/kfp-pipeline-config.yaml @@ -14,8 +14,12 @@ data: artifact_script: |- #!/usr/bin/env sh push_artifact() { - tar -cvzf $1.tgz $2 - mc cp $1.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz + if [ -f "$2" ]; then + tar -cvzf $1.tgz $2 + mc cp $1.tgz storage/$ARTIFACT_BUCKET/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz + else + echo "$2 file does not exist. Skip artifact tracking for $1" + fi } push_log() { cat /var/log/containers/$PODNAME*$NAMESPACE*step-main*.log > step-main.log diff --git a/sdk/python/kfp_tekton/compiler/_op_to_template.py b/sdk/python/kfp_tekton/compiler/_op_to_template.py index 7896a040038..293df5de787 100644 --- a/sdk/python/kfp_tekton/compiler/_op_to_template.py +++ b/sdk/python/kfp_tekton/compiler/_op_to_template.py @@ -25,6 +25,10 @@ from kfp.dsl._container_op import BaseOp from kfp_tekton.compiler import __tekton_api_version__ as tekton_api_version +from .. import __version__ + + +RESOURCE_OP_IMAGE = ":".join(["aipipeline/kubectl-wrapper", __version__]) def _get_base_step(name: str): @@ -102,7 +106,7 @@ def _get_resourceOp_template(op: BaseOp, { # This image is hosted by the kfp-tekton maintainers # Source code: https://github.com/kubeflow/kfp-tekton/tree/master/tekton-catalog/kubectl-wrapper - "default": "index.docker.io/aipipeline/kubeclient:v0.0.2", + "default": RESOURCE_OP_IMAGE, "description": "Kubectl wrapper image", "name": "image", "type": "string" diff --git a/sdk/python/tests/compiler/testdata/artifact_passing_using_volume.log b/sdk/python/tests/compiler/testdata/artifact_passing_using_volume.log index bbe05cc250a..58a2f19a761 100644 --- a/sdk/python/tests/compiler/testdata/artifact_passing_using_volume.log +++ b/sdk/python/tests/compiler/testdata/artifact_passing_using_volume.log @@ -1,26 +1,23 @@ [producer : copy-artifacts] Added `storage` successfully. -[producer : copy-artifacts] tekton/results/output-1 [producer : copy-artifacts] tar: removing leading '/' from member names +[producer : copy-artifacts] tekton/results/output-1 [producer : copy-artifacts] `Output-1.tgz` -> `storage/mlpipeline/artifacts/artifact-passing-pipeline/producer/Output-1.tgz` -[producer : copy-artifacts] Total: 0 B, Transferred: 121 B, Speed: 29 B/s +[producer : copy-artifacts] Total: 0 B, Transferred: 121 B, Speed: 31 B/s [producer : copy-artifacts] tar: removing leading '/' from member names [producer : copy-artifacts] workspace/producer/producer-Output-2 [producer : copy-artifacts] `Output-2.tgz` -> `storage/mlpipeline/artifacts/artifact-passing-pipeline/producer/Output-2.tgz` -[producer : copy-artifacts] Total: 0 B, Transferred: 144 B, Speed: 41 B/s +[producer : copy-artifacts] Total: 0 B, Transferred: 143 B, Speed: 44 B/s [processor : copy-artifacts] Added `storage` successfully. -[processor : copy-artifacts] tekton/results/output-1 [processor : copy-artifacts] tar: removing leading '/' from member names +[processor : copy-artifacts] tekton/results/output-1 [processor : copy-artifacts] `Output-1.tgz` -> `storage/mlpipeline/artifacts/artifact-passing-pipeline/processor/Output-1.tgz` -[processor : copy-artifacts] Total: 0 B, Transferred: 121 B, Speed: 39 B/s -[processor : copy-artifacts] /tekton/scripts/script-1-b4jcg: line 17: workspaces.producer.path: not found -[processor : copy-artifacts] tar: /producer-Output-2: No such file or directory -[processor : copy-artifacts] tar: error exit delayed from previous errors -[processor : copy-artifacts] `Output-2.tgz` -> `storage/mlpipeline/artifacts/artifact-passing-pipeline/processor/Output-2.tgz` -[processor : copy-artifacts] Total: 0 B, Transferred: 29 B, Speed: 8 B/s -[processor : copy-artifacts] /tekton/scripts/script-1-b4jcg: line 19: workspaces.producer.path: not found +[processor : copy-artifacts] Total: 0 B, Transferred: 121 B, Speed: 42 B/s +[processor : copy-artifacts] /tekton/scripts/script-1-v92rb: line 21: workspaces.producer.path: not found +[processor : copy-artifacts] /producer-Output-2 file does not exist. Skip artifact tracking for Output-2 +[processor : copy-artifacts] /tekton/scripts/script-1-v92rb: line 23: workspaces.producer.path: not found [consumer : main] Input parameter = Data 1 [consumer : main] Input artifact = diff --git a/sdk/python/tests/compiler/testdata/load_from_yaml.log b/sdk/python/tests/compiler/testdata/load_from_yaml.log index 6dd0fa0f1cc..d13625be3a9 100644 --- a/sdk/python/tests/compiler/testdata/load_from_yaml.log +++ b/sdk/python/tests/compiler/testdata/load_from_yaml.log @@ -1,8 +1,5 @@ [busybox : main] start /tekton/results/dummy-output-path end [busybox : copy-artifacts] Added `storage` successfully. -[busybox : copy-artifacts] tar: /tekton/results/dummy-output-path: No such file or directory -[busybox : copy-artifacts] tar: error exit delayed from previous errors -[busybox : copy-artifacts] `dummy_output_path.tgz` -> `storage/mlpipeline/artifacts/component-yaml-pipeline/busybox/dummy_output_path.tgz` -[busybox : copy-artifacts] Total: 0 B, Transferred: 29 B, Speed: 5 B/s +[busybox : copy-artifacts] /tekton/results/dummy-output-path file does not exist. Skip artifact tracking for dummy_output_path diff --git a/sdk/python/tests/compiler/testdata/resourceop_basic.yaml b/sdk/python/tests/compiler/testdata/resourceop_basic.yaml index 6e4ea1ce93d..ba76a81ded2 100644 --- a/sdk/python/tests/compiler/testdata/resourceop_basic.yaml +++ b/sdk/python/tests/compiler/testdata/resourceop_basic.yaml @@ -67,7 +67,7 @@ spec: is failure. name: failure-condition type: string - - default: index.docker.io/aipipeline/kubeclient:v0.0.2 + - default: aipipeline/kubectl-wrapper:0.8.0 description: Kubectl wrapper image name: image type: string diff --git a/sdk/python/tests/compiler/testdata/volume_op.log b/sdk/python/tests/compiler/testdata/volume_op.log index 641a074d199..281503e3461 100644 --- a/sdk/python/tests/compiler/testdata/volume_op.log +++ b/sdk/python/tests/compiler/testdata/volume_op.log @@ -1,12 +1,9 @@ -[create-pvc : main] time="2021-04-23T20:28:56Z" level=error msg="Initialize script failed: exit status 2:" -[create-pvc : main] time="2021-04-23T20:28:56Z" level=info msg="kubectl create -f /tmp/manifest.yaml -o json" -[create-pvc : main] time="2021-04-23T20:28:56Z" level=info msg=kubeflow/PersistentVolumeClaim./volumeop-basic-my-pvc -[create-pvc : main] time="2021-04-23T20:28:56Z" level=info msg="Saving resource output parameters" -[create-pvc : main] time="2021-04-23T20:28:56Z" level=info msg="[kubectl get PersistentVolumeClaim./volumeop-basic-my-pvc -o jsonpath={} -n kubeflow]" -[create-pvc : main] time="2021-04-23T20:28:56Z" level=info msg="Saved output parameter: manifest, value: map[apiVersion:v1 kind:PersistentVolumeClaim metadata:map[annotations:map[ibm.io/provisioning-status:{\"status\":\"inprogress\",\"time\":\"2021-04-23T20:28:56Z\",\"attempt\":1,\"retry\":false,\"pluginid\":\"ibm-file-plugin-1619200208\",\"pvcid\":\"8d0a5e2a-7e12-46eb-8c46-dc40e2ceb280\"} volume.beta.kubernetes.io/storage-provisioner:ibm.io/ibmc-file] creationTimestamp:2021-04-23T20:28:56Z finalizers:[kubernetes.io/pvc-protection] labels:map[region:us-south zone:sjc04] managedFields:[map[apiVersion:v1 fieldsType:FieldsV1 fieldsV1:map[f:metadata:map[f:annotations:map[f:ibm.io/provisioning-status:map[]] f:labels:map[.:map[] f:region:map[] f:zone:map[]]]] manager:armada-storage-file-plugin operation:Update time:2021-04-23T20:28:56Z] map[apiVersion:v1 fieldsType:FieldsV1 fieldsV1:map[f:metadata:map[f:annotations:map[.:map[] f:volume.beta.kubernetes.io/storage-provisioner:map[]]]] manager:kube-controller-manager operation:Update time:2021-04-23T20:28:56Z] map[apiVersion:v1 fieldsType:FieldsV1 fieldsV1:map[f:spec:map[f:accessModes:map[] f:resources:map[f:requests:map[.:map[] f:storage:map[]]] f:volumeMode:map[]] f:status:map[f:phase:map[]]] manager:kubectl operation:Update time:2021-04-23T20:28:56Z]] name:volumeop-basic-my-pvc namespace:kubeflow resourceVersion:98730 selfLink:/api/v1/namespaces/kubeflow/persistentvolumeclaims/volumeop-basic-my-pvc uid:8d0a5e2a-7e12-46eb-8c46-dc40e2ceb280] spec:map[accessModes:[ReadWriteOnce] resources:map[requests:map[storage:10M]] storageClassName:ibmc-file-gold-gid volumeMode:Filesystem] status:map[phase:Pending]]" -[create-pvc : main] time="2021-04-23T20:28:56Z" level=info msg="[kubectl get PersistentVolumeClaim./volumeop-basic-my-pvc -o jsonpath={.metadata.name} -n kubeflow]" -[create-pvc : main] time="2021-04-23T20:28:57Z" level=info msg="Saved output parameter: name, value: volumeop-basic-my-pvc" -[create-pvc : main] time="2021-04-23T20:28:57Z" level=info msg="[kubectl get PersistentVolumeClaim./volumeop-basic-my-pvc -o jsonpath={.status.capacity.storage} -n kubeflow]" -[create-pvc : main] time="2021-04-23T20:28:57Z" level=info msg="Saved output parameter: size, value: " - - +[create-pvc : main] time="2021-04-29T01:02:11Z" level=info msg="kubectl create -f /tmp/manifest.yaml -o json" +[create-pvc : main] time="2021-04-29T01:02:11Z" level=info msg=kubeflow/PersistentVolumeClaim./volumeop-basic-63777-my-pvc +[create-pvc : main] time="2021-04-29T01:02:11Z" level=info msg="Saving resource output parameters" +[create-pvc : main] time="2021-04-29T01:02:11Z" level=info msg="[kubectl get PersistentVolumeClaim./volumeop-basic-63777-my-pvc -o jsonpath={} -n kubeflow]" +[create-pvc : main] time="2021-04-29T01:02:12Z" level=info msg="Saved output parameter: manifest, value: {\"apiVersion\":\"v1\",\"kind\":\"PersistentVolumeClaim\",\"metadata\":{\"annotations\":{\"ibm.io/provisioning-status\":\"{\\\"status\\\":\\\"inprogress\\\",\\\"time\\\":\\\"2021-04-29T01:02:11Z\\\",\\\"attempt\\\":1,\\\"retry\\\":false,\\\"pluginid\\\":\\\"ibm-file-plugin-1619479127\\\",\\\"pvcid\\\":\\\"2dac3c70-eb65-40c2-9d1f-5632dad10232\\\"}\",\"volume.beta.kubernetes.io/storage-provisioner\":\"ibm.io/ibmc-file\"},\"creationTimestamp\":\"2021-04-29T01:02:11Z\",\"finalizers\":[\"kubernetes.io/pvc-protection\"],\"labels\":{\"region\":\"us-south\",\"zone\":\"sjc04\"},\"managedFields\":[{\"apiVersion\":\"v1\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\".\":{},\"f:volume.beta.kubernetes.io/storage-provisioner\":{}}}},\"manager\":\"kube-controller-manager\",\"operation\":\"Update\",\"time\":\"2021-04-29T01:02:11Z\"},{\"apiVersion\":\"v1\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:spec\":{\"f:accessModes\":{},\"f:resources\":{\"f:requests\":{\".\":{},\"f:storage\":{}}},\"f:volumeMode\":{}},\"f:status\":{\"f:phase\":{}}},\"manager\":\"kubectl-create\",\"operation\":\"Update\",\"time\":\"2021-04-29T01:02:11Z\"},{\"apiVersion\":\"v1\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:metadata\":{\"f:annotations\":{\"f:ibm.io/provisioning-status\":{}},\"f:labels\":{\".\":{},\"f:region\":{},\"f:zone\":{}}}},\"manager\":\"armada-storage-file-plugin\",\"operation\":\"Update\",\"time\":\"2021-04-29T01:02:12Z\"}],\"name\":\"volumeop-basic-63777-my-pvc\",\"namespace\":\"kubeflow\",\"resourceVersion\":\"1402962\",\"selfLink\":\"/api/v1/namespaces/kubeflow/persistentvolumeclaims/volumeop-basic-63777-my-pvc\",\"uid\":\"2dac3c70-eb65-40c2-9d1f-5632dad10232\"},\"spec\":{\"accessModes\":[\"ReadWriteOnce\"],\"resources\":{\"requests\":{\"storage\":\"10M\"}},\"storageClassName\":\"ibmc-file-gold-gid\",\"volumeMode\":\"Filesystem\"},\"status\":{\"phase\":\"Pending\"}}" +[create-pvc : main] time="2021-04-29T01:02:12Z" level=info msg="[kubectl get PersistentVolumeClaim./volumeop-basic-63777-my-pvc -o jsonpath={.metadata.name} -n kubeflow]" +[create-pvc : main] time="2021-04-29T01:02:12Z" level=info msg="Saved output parameter: name, value: volumeop-basic-63777-my-pvc" +[create-pvc : main] time="2021-04-29T01:02:12Z" level=info msg="[kubectl get PersistentVolumeClaim./volumeop-basic-63777-my-pvc -o jsonpath={.status.capacity.storage} -n kubeflow]" +[create-pvc : main] time="2021-04-29T01:02:12Z" level=info msg="Saved output parameter: size, value: " diff --git a/sdk/python/tests/compiler/testdata/volume_op.yaml b/sdk/python/tests/compiler/testdata/volume_op.yaml index 1f2d8f061b7..892b9c47b27 100644 --- a/sdk/python/tests/compiler/testdata/volume_op.yaml +++ b/sdk/python/tests/compiler/testdata/volume_op.yaml @@ -78,7 +78,7 @@ spec: is failure. name: failure-condition type: string - - default: index.docker.io/aipipeline/kubeclient:v0.0.2 + - default: aipipeline/kubectl-wrapper:0.8.0 description: Kubectl wrapper image name: image type: string diff --git a/sdk/python/tests/compiler/testdata/volume_snapshot_op.yaml b/sdk/python/tests/compiler/testdata/volume_snapshot_op.yaml index 9183482d963..3128da9c1b1 100644 --- a/sdk/python/tests/compiler/testdata/volume_snapshot_op.yaml +++ b/sdk/python/tests/compiler/testdata/volume_snapshot_op.yaml @@ -84,7 +84,7 @@ spec: is failure. name: failure-condition type: string - - default: index.docker.io/aipipeline/kubeclient:v0.0.2 + - default: aipipeline/kubectl-wrapper:0.8.0 description: Kubectl wrapper image name: image type: string @@ -197,7 +197,7 @@ spec: is failure. name: failure-condition type: string - - default: index.docker.io/aipipeline/kubeclient:v0.0.2 + - default: aipipeline/kubectl-wrapper:0.8.0 description: Kubectl wrapper image name: image type: string @@ -310,7 +310,7 @@ spec: is failure. name: failure-condition type: string - - default: index.docker.io/aipipeline/kubeclient:v0.0.2 + - default: aipipeline/kubectl-wrapper:0.8.0 description: Kubectl wrapper image name: image type: string @@ -423,7 +423,7 @@ spec: is failure. name: failure-condition type: string - - default: index.docker.io/aipipeline/kubeclient:v0.0.2 + - default: aipipeline/kubectl-wrapper:0.8.0 description: Kubectl wrapper image name: image type: string diff --git a/tekton-catalog/kubectl-wrapper/cmd/client/main.go b/tekton-catalog/kubectl-wrapper/cmd/client/main.go index a046a341bce..f89c73b1318 100644 --- a/tekton-catalog/kubectl-wrapper/cmd/client/main.go +++ b/tekton-catalog/kubectl-wrapper/cmd/client/main.go @@ -69,12 +69,6 @@ func main() { } } - // cmd := exec.Command("/bin/sh", "/builder/kubectl.bash") - // _, err = cmd.Output() - // if err != nil { - // log.Errorf("Initialize script failed: %+v:", err) - // } - isDelete := action == "delete" resourceNamespace, resourceName, err := execResource(action, mergeStrategy) if err != nil { diff --git a/tekton-catalog/kubectl-wrapper/deploy/kubectl-deploy.yaml b/tekton-catalog/kubectl-wrapper/deploy/kubectl-deploy.yaml index 6020739e5e0..7cf5fbe607d 100644 --- a/tekton-catalog/kubectl-wrapper/deploy/kubectl-deploy.yaml +++ b/tekton-catalog/kubectl-wrapper/deploy/kubectl-deploy.yaml @@ -25,7 +25,7 @@ spec: default: "false" description: Enable set owner reference for created resource. - name: image - default: index.docker.io/aipipeline/kubeclient:v0.0.2 # it is huge + default: aipipeline/kubeclient:0.8.0 # it is huge description: Kubectl wrapper image steps: - name: kubeconfig