From da6ff9cdd5f9618650471b732e1ce6f57dbd71fc Mon Sep 17 00:00:00 2001 From: nealgao Date: Wed, 21 Nov 2018 10:55:15 -0800 Subject: [PATCH] restructure dataflow component structure (#338) * restructure dataflow components * add . directory before the script name --- .cloudbuild.yaml | 8 ++++---- components/dataflow/{containers => }/base/Dockerfile | 0 .../base/build.sh => base/build_image.sh} | 12 ++++++------ .../dataflow/{containers => }/predict/Dockerfile | 0 .../predict/build.sh => predict/build_image.sh} | 2 +- components/dataflow/predict/{ => src}/predict.py | 0 components/dataflow/predict/{ => src}/setup.py | 0 components/dataflow/{containers => }/tfdv/Dockerfile | 0 .../tfdv/build.sh => tfdv/build_image.sh} | 2 +- components/dataflow/tfdv/{ => src}/validate.py | 0 .../dataflow/tfdv/{ => src}/validation/setup.py | 0 components/dataflow/{containers => }/tfma/Dockerfile | 0 .../tfma/build.sh => tfma/build_image.sh} | 2 +- components/dataflow/tfma/{ => src}/analysis/setup.py | 0 components/dataflow/tfma/{ => src}/model_analysis.py | 0 components/dataflow/{containers => }/tft/Dockerfile | 0 .../{containers/tft/build.sh => tft/build_image.sh} | 2 +- components/dataflow/tft/{ => src}/transform.py | 0 samples/kubeflow-tf/README.md | 8 ++++---- samples/tfx/README.md | 12 ++++++------ test/sample_test.yaml | 8 ++++---- 21 files changed, 28 insertions(+), 28 deletions(-) rename components/dataflow/{containers => }/base/Dockerfile (100%) rename components/dataflow/{containers/base/build.sh => base/build_image.sh} (75%) rename components/dataflow/{containers => }/predict/Dockerfile (100%) rename components/dataflow/{containers/predict/build.sh => predict/build_image.sh} (99%) rename components/dataflow/predict/{ => src}/predict.py (100%) rename components/dataflow/predict/{ => src}/setup.py (100%) rename components/dataflow/{containers => }/tfdv/Dockerfile (100%) rename components/dataflow/{containers/tfdv/build.sh => tfdv/build_image.sh} (99%) rename components/dataflow/tfdv/{ => src}/validate.py (100%) rename components/dataflow/tfdv/{ => src}/validation/setup.py (100%) rename components/dataflow/{containers => }/tfma/Dockerfile (100%) rename components/dataflow/{containers/tfma/build.sh => tfma/build_image.sh} (99%) rename components/dataflow/tfma/{ => src}/analysis/setup.py (100%) rename components/dataflow/tfma/{ => src}/model_analysis.py (100%) rename components/dataflow/{containers => }/tft/Dockerfile (100%) rename components/dataflow/{containers/tft/build.sh => tft/build_image.sh} (98%) rename components/dataflow/tft/{ => src}/transform.py (100%) diff --git a/.cloudbuild.yaml b/.cloudbuild.yaml index 7ab4d92d1bd..44247aa2c8b 100644 --- a/.cloudbuild.yaml +++ b/.cloudbuild.yaml @@ -97,19 +97,19 @@ steps: # Build the Dataflow-based pipeline component images - name: 'gcr.io/cloud-builders/docker' entrypoint: '/bin/bash' - args: ['-c', 'cd /workspace/components/dataflow/containers/predict && ./build.sh -p $PROJECT_ID -t $COMMIT_SHA'] + args: ['-c', 'cd /workspace/components/dataflow/predict && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA'] id: 'buildPredictComponent' - name: 'gcr.io/cloud-builders/docker' entrypoint: '/bin/bash' - args: ['-c', 'cd /workspace/components/dataflow/containers/tfdv && ./build.sh -p $PROJECT_ID -t $COMMIT_SHA'] + args: ['-c', 'cd /workspace/components/dataflow/tfdv && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA'] id: 'buildTFDVComponent' - name: 'gcr.io/cloud-builders/docker' entrypoint: '/bin/bash' - args: ['-c', 'cd /workspace/components/dataflow/containers/tft && ./build.sh -p $PROJECT_ID -t $COMMIT_SHA'] + args: ['-c', 'cd /workspace/components/dataflow/tft && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA'] id: 'buildTFTComponent' - name: 'gcr.io/cloud-builders/docker' entrypoint: '/bin/bash' - args: ['-c', 'cd /workspace/components/dataflow/containers/tfma && ./build.sh -p $PROJECT_ID -t $COMMIT_SHA'] + args: ['-c', 'cd /workspace/components/dataflow/tfma && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA'] id: 'buildTMAComponent' # Build the Kubeflow-based pipeline component images diff --git a/components/dataflow/containers/base/Dockerfile b/components/dataflow/base/Dockerfile similarity index 100% rename from components/dataflow/containers/base/Dockerfile rename to components/dataflow/base/Dockerfile diff --git a/components/dataflow/containers/base/build.sh b/components/dataflow/base/build_image.sh similarity index 75% rename from components/dataflow/containers/base/build.sh rename to components/dataflow/base/build_image.sh index 7c20f9de63d..38ea8f6ff0a 100755 --- a/components/dataflow/containers/base/build.sh +++ b/components/dataflow/base/build_image.sh @@ -15,12 +15,12 @@ mkdir -p ./build -rsync -arvp "../../tfdv"/ ./build/ -rsync -arvp "../../tft"/ ./build/ -rsync -arvp "../../tfma"/ ./build/ -rsync -arvp "../../predict"/ ./build/ -cp ../../../license.sh ./build -cp ../../../third_party_licenses.csv ./build +rsync -arvp "../tfdv/src"/ ./build/ +rsync -arvp "../tft/src"/ ./build/ +rsync -arvp "../tfma/src"/ ./build/ +rsync -arvp "../predict/src"/ ./build/ +cp ../../license.sh ./build +cp ../../third_party_licenses.csv ./build docker build -t ml-pipeline-dataflow-base . rm -rf ./build diff --git a/components/dataflow/containers/predict/Dockerfile b/components/dataflow/predict/Dockerfile similarity index 100% rename from components/dataflow/containers/predict/Dockerfile rename to components/dataflow/predict/Dockerfile diff --git a/components/dataflow/containers/predict/build.sh b/components/dataflow/predict/build_image.sh similarity index 99% rename from components/dataflow/containers/predict/build.sh rename to components/dataflow/predict/build_image.sh index fc461219beb..128420445a5 100755 --- a/components/dataflow/containers/predict/build.sh +++ b/components/dataflow/predict/build_image.sh @@ -44,7 +44,7 @@ fi # build base image pushd ../base -./build.sh +./build_image.sh popd docker build -t ${LOCAL_IMAGE_NAME} . diff --git a/components/dataflow/predict/predict.py b/components/dataflow/predict/src/predict.py similarity index 100% rename from components/dataflow/predict/predict.py rename to components/dataflow/predict/src/predict.py diff --git a/components/dataflow/predict/setup.py b/components/dataflow/predict/src/setup.py similarity index 100% rename from components/dataflow/predict/setup.py rename to components/dataflow/predict/src/setup.py diff --git a/components/dataflow/containers/tfdv/Dockerfile b/components/dataflow/tfdv/Dockerfile similarity index 100% rename from components/dataflow/containers/tfdv/Dockerfile rename to components/dataflow/tfdv/Dockerfile diff --git a/components/dataflow/containers/tfdv/build.sh b/components/dataflow/tfdv/build_image.sh similarity index 99% rename from components/dataflow/containers/tfdv/build.sh rename to components/dataflow/tfdv/build_image.sh index 58c5ac62785..01c8c47ff97 100755 --- a/components/dataflow/containers/tfdv/build.sh +++ b/components/dataflow/tfdv/build_image.sh @@ -44,7 +44,7 @@ fi # build base image pushd ../base -./build.sh +./build_image.sh popd docker build -t ${LOCAL_IMAGE_NAME} . diff --git a/components/dataflow/tfdv/validate.py b/components/dataflow/tfdv/src/validate.py similarity index 100% rename from components/dataflow/tfdv/validate.py rename to components/dataflow/tfdv/src/validate.py diff --git a/components/dataflow/tfdv/validation/setup.py b/components/dataflow/tfdv/src/validation/setup.py similarity index 100% rename from components/dataflow/tfdv/validation/setup.py rename to components/dataflow/tfdv/src/validation/setup.py diff --git a/components/dataflow/containers/tfma/Dockerfile b/components/dataflow/tfma/Dockerfile similarity index 100% rename from components/dataflow/containers/tfma/Dockerfile rename to components/dataflow/tfma/Dockerfile diff --git a/components/dataflow/containers/tfma/build.sh b/components/dataflow/tfma/build_image.sh similarity index 99% rename from components/dataflow/containers/tfma/build.sh rename to components/dataflow/tfma/build_image.sh index 48f3d320821..097b8ec2fde 100755 --- a/components/dataflow/containers/tfma/build.sh +++ b/components/dataflow/tfma/build_image.sh @@ -44,7 +44,7 @@ fi # build base image pushd ../base -./build.sh +./build_image.sh popd docker build -t ${LOCAL_IMAGE_NAME} . diff --git a/components/dataflow/tfma/analysis/setup.py b/components/dataflow/tfma/src/analysis/setup.py similarity index 100% rename from components/dataflow/tfma/analysis/setup.py rename to components/dataflow/tfma/src/analysis/setup.py diff --git a/components/dataflow/tfma/model_analysis.py b/components/dataflow/tfma/src/model_analysis.py similarity index 100% rename from components/dataflow/tfma/model_analysis.py rename to components/dataflow/tfma/src/model_analysis.py diff --git a/components/dataflow/containers/tft/Dockerfile b/components/dataflow/tft/Dockerfile similarity index 100% rename from components/dataflow/containers/tft/Dockerfile rename to components/dataflow/tft/Dockerfile diff --git a/components/dataflow/containers/tft/build.sh b/components/dataflow/tft/build_image.sh similarity index 98% rename from components/dataflow/containers/tft/build.sh rename to components/dataflow/tft/build_image.sh index 4cc8b3778f4..f562b2ea2ca 100755 --- a/components/dataflow/containers/tft/build.sh +++ b/components/dataflow/tft/build_image.sh @@ -44,7 +44,7 @@ fi # build base image pushd ../base -./build.sh +./build_image.sh popd docker build -t ${LOCAL_IMAGE_NAME} . diff --git a/components/dataflow/tft/transform.py b/components/dataflow/tft/src/transform.py similarity index 100% rename from components/dataflow/tft/transform.py rename to components/dataflow/tft/src/transform.py diff --git a/samples/kubeflow-tf/README.md b/samples/kubeflow-tf/README.md index 77b9f4cf200..b89b8af5b2b 100644 --- a/samples/kubeflow-tf/README.md +++ b/samples/kubeflow-tf/README.md @@ -31,16 +31,16 @@ The pipeline requires one argument: ## Components source Preprocessing: - [source code](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/tft), - [container](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/containers/tft) + [source code](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/tft/src), + [container](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/tft) Training: [source code](https://github.com/kubeflow/pipelines/tree/master/components/kubeflow/launcher/src), [container](https://github.com/kubeflow/pipelines/tree/master/components/kubeflow/launcher) Prediction: - [source code](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/predict), - [container](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/containers/predict) + [source code](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/predict/src), + [container](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/predict) Confusion Matrix: [source code](https://github.com/kubeflow/pipelines/tree/master/components/local/evaluation), diff --git a/samples/tfx/README.md b/samples/tfx/README.md index 85ddbf9cbd0..69e8ab3c2c5 100644 --- a/samples/tfx/README.md +++ b/samples/tfx/README.md @@ -47,17 +47,17 @@ The pipeline requires two arguments: ## Components source Preprocessing: - [source code](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/tft) - [container](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/containers/tft) + [source code](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/tft/src) + [container](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/tft) Training: [source code](https://github.com/kubeflow/pipelines/tree/master/components/kubeflow/launcher/src) [container](https://github.com/kubeflow/pipelines/tree/master/components/kubeflow/launcher) Analysis: - [source code](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/tfma) - [container](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/containers/tfma) + [source code](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/tfma/src) + [container](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/tfma) Prediction: - [source code](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/predict) - [container](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/containers/predict) + [source code](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/predict/src) + [container](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/predict) diff --git a/test/sample_test.yaml b/test/sample_test.yaml index dc1b9f6c784..3182549aa9d 100644 --- a/test/sample_test.yaml +++ b/test/sample_test.yaml @@ -112,7 +112,7 @@ spec: - name: image-suffix value: "{{inputs.parameters.dataflow-tft-image-suffix}}" - name: build-script - value: components/dataflow/containers/tft/build.sh + value: components/dataflow/tft/build_image.sh - name: build-dataflow-predict-image template: build-image-by-script arguments: @@ -124,7 +124,7 @@ spec: - name: image-suffix value: "{{inputs.parameters.dataflow-predict-image-suffix}}" - name: build-script - value: components/dataflow/containers/predict/build.sh + value: components/dataflow/predict/build_image.sh - name: build-dataflow-tfma-image template: build-image-by-script arguments: @@ -136,7 +136,7 @@ spec: - name: image-suffix value: "{{inputs.parameters.dataflow-tfma-image-suffix}}" - name: build-script - value: components/dataflow/containers/tfma/build.sh + value: components/dataflow/tfma/build_image.sh - name: build-dataflow-tfdv-image template: build-image-by-script arguments: @@ -148,7 +148,7 @@ spec: - name: image-suffix value: "{{inputs.parameters.dataflow-tfdv-image-suffix}}" - name: build-script - value: components/dataflow/containers/tfdv/build.sh + value: components/dataflow/tfdv/build_image.sh - name: build-dataproc-create-cluster-image template: build-image-by-script arguments: