restructure dataflow component structure (#338)

* restructure dataflow components * add . directory before the script name
kubeflow · Nov 21, 2018 · da6ff9c · da6ff9c
1 parent 33f191f
commit da6ff9c
Show file tree

Hide file tree

Showing 21 changed files with 28 additions and 28 deletions.
diff --git a/.cloudbuild.yaml b/.cloudbuild.yaml
@@ -97,19 +97,19 @@ steps:
 # Build the Dataflow-based pipeline component images
 - name: 'gcr.io/cloud-builders/docker'
   entrypoint: '/bin/bash'
-  args: ['-c', 'cd /workspace/components/dataflow/containers/predict && ./build.sh -p $PROJECT_ID -t $COMMIT_SHA']
+  args: ['-c', 'cd /workspace/components/dataflow/predict && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
   id: 'buildPredictComponent'
 - name: 'gcr.io/cloud-builders/docker'
   entrypoint: '/bin/bash'
-  args: ['-c', 'cd /workspace/components/dataflow/containers/tfdv && ./build.sh -p $PROJECT_ID -t $COMMIT_SHA']
+  args: ['-c', 'cd /workspace/components/dataflow/tfdv && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
   id: 'buildTFDVComponent'
 - name: 'gcr.io/cloud-builders/docker'
   entrypoint: '/bin/bash'
-  args: ['-c', 'cd /workspace/components/dataflow/containers/tft && ./build.sh -p $PROJECT_ID -t $COMMIT_SHA']
+  args: ['-c', 'cd /workspace/components/dataflow/tft && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
   id: 'buildTFTComponent'
 - name: 'gcr.io/cloud-builders/docker'
   entrypoint: '/bin/bash'
-  args: ['-c', 'cd /workspace/components/dataflow/containers/tfma && ./build.sh -p $PROJECT_ID -t $COMMIT_SHA']
+  args: ['-c', 'cd /workspace/components/dataflow/tfma && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
   id: 'buildTMAComponent'
 
 # Build the Kubeflow-based pipeline component images

diff --git a/...nents/dataflow/containers/base/Dockerfile → components/dataflow/base/Dockerfile b/...nents/dataflow/containers/base/Dockerfile → components/dataflow/base/Dockerfile
diff --git a/components/dataflow/containers/base/build.sh → components/dataflow/base/build_image.sh b/components/dataflow/containers/base/build.sh → components/dataflow/base/build_image.sh
@@ -15,12 +15,12 @@
 
 
 mkdir -p ./build
-rsync -arvp "../../tfdv"/ ./build/
-rsync -arvp "../../tft"/ ./build/
-rsync -arvp "../../tfma"/ ./build/
-rsync -arvp "../../predict"/ ./build/
-cp ../../../license.sh ./build
-cp ../../../third_party_licenses.csv ./build
+rsync -arvp "../tfdv/src"/ ./build/
+rsync -arvp "../tft/src"/ ./build/
+rsync -arvp "../tfma/src"/ ./build/
+rsync -arvp "../predict/src"/ ./build/
+cp ../../license.sh ./build
+cp ../../third_party_licenses.csv ./build
 
 docker build -t ml-pipeline-dataflow-base .
 rm -rf ./build

diff --git a/...ts/dataflow/containers/predict/Dockerfile → components/dataflow/predict/Dockerfile b/...ts/dataflow/containers/predict/Dockerfile → components/dataflow/predict/Dockerfile
diff --git a/...ents/dataflow/containers/predict/build.sh → components/dataflow/predict/build_image.sh b/...ents/dataflow/containers/predict/build.sh → components/dataflow/predict/build_image.sh
@@ -44,7 +44,7 @@ fi
 
 # build base image
 pushd ../base
-./build.sh
+./build_image.sh
 popd
 
 docker build -t ${LOCAL_IMAGE_NAME} .

diff --git a/components/dataflow/predict/predict.py → components/dataflow/predict/src/predict.py b/components/dataflow/predict/predict.py → components/dataflow/predict/src/predict.py
diff --git a/components/dataflow/predict/setup.py → components/dataflow/predict/src/setup.py b/components/dataflow/predict/setup.py → components/dataflow/predict/src/setup.py
diff --git a/...nents/dataflow/containers/tfdv/Dockerfile → components/dataflow/tfdv/Dockerfile b/...nents/dataflow/containers/tfdv/Dockerfile → components/dataflow/tfdv/Dockerfile
diff --git a/components/dataflow/containers/tfdv/build.sh → components/dataflow/tfdv/build_image.sh b/components/dataflow/containers/tfdv/build.sh → components/dataflow/tfdv/build_image.sh
@@ -44,7 +44,7 @@ fi
 
 # build base image
 pushd ../base
-./build.sh
+./build_image.sh
 popd
 
 docker build -t ${LOCAL_IMAGE_NAME} .

diff --git a/components/dataflow/tfdv/validate.py → components/dataflow/tfdv/src/validate.py b/components/dataflow/tfdv/validate.py → components/dataflow/tfdv/src/validate.py
diff --git a/components/dataflow/tfdv/validation/setup.py → ...nts/dataflow/tfdv/src/validation/setup.py b/components/dataflow/tfdv/validation/setup.py → ...nts/dataflow/tfdv/src/validation/setup.py
diff --git a/...nents/dataflow/containers/tfma/Dockerfile → components/dataflow/tfma/Dockerfile b/...nents/dataflow/containers/tfma/Dockerfile → components/dataflow/tfma/Dockerfile
diff --git a/components/dataflow/containers/tfma/build.sh → components/dataflow/tfma/build_image.sh b/components/dataflow/containers/tfma/build.sh → components/dataflow/tfma/build_image.sh
@@ -44,7 +44,7 @@ fi
 
 # build base image
 pushd ../base
-./build.sh
+./build_image.sh
 popd
 
 docker build -t ${LOCAL_IMAGE_NAME} .

diff --git a/components/dataflow/tfma/analysis/setup.py → ...nents/dataflow/tfma/src/analysis/setup.py b/components/dataflow/tfma/analysis/setup.py → ...nents/dataflow/tfma/src/analysis/setup.py
diff --git a/components/dataflow/tfma/model_analysis.py → ...nents/dataflow/tfma/src/model_analysis.py b/components/dataflow/tfma/model_analysis.py → ...nents/dataflow/tfma/src/model_analysis.py
diff --git a/...onents/dataflow/containers/tft/Dockerfile → components/dataflow/tft/Dockerfile b/...onents/dataflow/containers/tft/Dockerfile → components/dataflow/tft/Dockerfile
diff --git a/components/dataflow/containers/tft/build.sh → components/dataflow/tft/build_image.sh b/components/dataflow/containers/tft/build.sh → components/dataflow/tft/build_image.sh
@@ -44,7 +44,7 @@ fi
 
 # build base image
 pushd ../base
-./build.sh
+./build_image.sh
 popd
 
 docker build -t ${LOCAL_IMAGE_NAME} .

diff --git a/components/dataflow/tft/transform.py → components/dataflow/tft/src/transform.py b/components/dataflow/tft/transform.py → components/dataflow/tft/src/transform.py
diff --git a/samples/kubeflow-tf/README.md b/samples/kubeflow-tf/README.md
@@ -31,16 +31,16 @@ The pipeline requires one argument:
 ## Components source
 
 Preprocessing:
-  [source code](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/tft), 
-  [container](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/containers/tft)
+  [source code](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/tft/src), 
+  [container](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/tft)
 
 Training:
   [source code](https://github.com/kubeflow/pipelines/tree/master/components/kubeflow/launcher/src), 
   [container](https://github.com/kubeflow/pipelines/tree/master/components/kubeflow/launcher)
 
 Prediction:
-  [source code](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/predict), 
-  [container](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/containers/predict)
+  [source code](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/predict/src), 
+  [container](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/predict)
 
 Confusion Matrix:
   [source code](https://github.com/kubeflow/pipelines/tree/master/components/local/evaluation), 

diff --git a/samples/tfx/README.md b/samples/tfx/README.md
@@ -47,17 +47,17 @@ The pipeline requires two arguments:
 ## Components source
 
 Preprocessing:
-  [source code](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/tft) 
-  [container](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/containers/tft)
+  [source code](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/tft/src) 
+  [container](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/tft)
 
 Training:
   [source code](https://github.com/kubeflow/pipelines/tree/master/components/kubeflow/launcher/src) 
   [container](https://github.com/kubeflow/pipelines/tree/master/components/kubeflow/launcher)
 
 Analysis:
-  [source code](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/tfma) 
-  [container](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/containers/tfma)
+  [source code](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/tfma/src) 
+  [container](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/tfma)
 
 Prediction:
-  [source code](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/predict) 
-  [container](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/containers/predict)
+  [source code](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/predict/src) 
+  [container](https://github.com/kubeflow/pipelines/tree/master/components/dataflow/predict)
diff --git a/test/sample_test.yaml b/test/sample_test.yaml
@@ -112,7 +112,7 @@ spec:
           - name: image-suffix
             value: "{{inputs.parameters.dataflow-tft-image-suffix}}"
           - name: build-script
-            value: components/dataflow/containers/tft/build.sh
+            value: components/dataflow/tft/build_image.sh
       - name: build-dataflow-predict-image
         template: build-image-by-script
         arguments:
@@ -124,7 +124,7 @@ spec:
           - name: image-suffix
             value: "{{inputs.parameters.dataflow-predict-image-suffix}}"
           - name: build-script
-            value: components/dataflow/containers/predict/build.sh
+            value: components/dataflow/predict/build_image.sh
       - name: build-dataflow-tfma-image
         template: build-image-by-script
         arguments:
@@ -136,7 +136,7 @@ spec:
           - name: image-suffix
             value: "{{inputs.parameters.dataflow-tfma-image-suffix}}"
           - name: build-script
-            value: components/dataflow/containers/tfma/build.sh
+            value: components/dataflow/tfma/build_image.sh
       - name: build-dataflow-tfdv-image
         template: build-image-by-script
         arguments:
@@ -148,7 +148,7 @@ spec:
           - name: image-suffix
             value: "{{inputs.parameters.dataflow-tfdv-image-suffix}}"
           - name: build-script
-            value: components/dataflow/containers/tfdv/build.sh
+            value: components/dataflow/tfdv/build_image.sh
       - name: build-dataproc-create-cluster-image
         template: build-image-by-script
         arguments: