.cloudbuild.yaml

# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

steps:

# Build the Python SDK
- name: 'python:3-alpine'
  entrypoint: '/bin/sh'
  args: ['-c', 'cd /workspace/sdk/python/; python3 setup.py sdist --format=gztar; cp dist/*.tar.gz /workspace/kfp.tar.gz']
  id:   'preparePythonSDK'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/gsutil'
  args: ['cp', '/workspace/kfp.tar.gz', 'gs://$PROJECT_ID/builds/$COMMIT_SHA/kfp.tar.gz']
  id:   'copyPythonSDK'
  waitFor: ['preparePythonSDK']
- name: 'gcr.io/cloud-builders/gsutil'
  args: ['cp', '/workspace/kfp.tar.gz', 'gs://$PROJECT_ID/builds/latest/kfp.tar.gz']
  id:   'copyPythonSDKToLatest'
  waitFor: ['preparePythonSDK']

# Build the Python Component SDK
- name: 'python:2-alpine'
  entrypoint: '/bin/sh'
  args: ['-c', 'cd /workspace/components/gcp/container/component_sdk/python;python setup.py sdist --format=gztar; cp dist/*.tar.gz /workspace/kfp-component.tar.gz']
  id:   'preparePythonComponentSDK'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/gsutil'
  args: ['cp', '/workspace/kfp-component.tar.gz', 'gs://$PROJECT_ID/builds/$COMMIT_SHA/kfp-component.tar.gz']
  id:   'copyPythonComponentSDK'
  waitFor: ['preparePythonComponentSDK']
- name: 'gcr.io/cloud-builders/gsutil'
  args: ['cp', '/workspace/kfp-component.tar.gz', 'gs://$PROJECT_ID/builds/latest/kfp-component.tar.gz']
  id:   'copyPythonComponentSDKToLatest'
  waitFor: ['preparePythonComponentSDK']

# Build the pipeline system images
- name: 'debian'
  entrypoint: '/bin/bash'
  args: ['-c', 'sed -i -e "s/ARG DATE/ENV DATE \"$(date -u)\"/" /workspace/frontend/Dockerfile']
  id:   'prepareFrontend'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA',
         '--build-arg', 'COMMIT_HASH=$COMMIT_SHA', '-f',
         '/workspace/frontend/Dockerfile', '/workspace']
  id:   'buildFrontend'
  waitFor: ['prepareFrontend']
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/api-server:$COMMIT_SHA',
         '--build-arg', 'COMMIT_SHA=$COMMIT_SHA', '-f',
         '/workspace/backend/Dockerfile', '/workspace']
  id:   'buildApiServer'
  waitFor: ['copyPythonSDK']

- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/scheduledworkflow:$COMMIT_SHA', '-f',
         '/workspace/backend/Dockerfile.scheduledworkflow', '/workspace']
  id:   'buildScheduledWorkflow'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA', '-f',
         '/workspace/backend/Dockerfile.viewercontroller', '/workspace']
  id:   'buildViewerCrdController'
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/persistenceagent:$COMMIT_SHA', '-f',
         '/workspace/backend/Dockerfile.persistenceagent', '/workspace']
  id:   'buildPersistenceAgent'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/inverse-proxy-agent:$COMMIT_SHA', '-f',
         '/workspace/proxy/Dockerfile', '/workspace/proxy']
  id:   'buildInverseProxyAgent'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/visualization-server:$COMMIT_SHA',
         '--build-arg', 'COMMIT_HASH=$COMMIT_SHA', '-f',
         '/workspace/backend/Dockerfile.visualization', '/workspace']
  id: 'buildVisualizationServer'
  waitFor: ["-"]

# Build the Dataflow-based pipeline component images
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/deprecated/dataflow/predict && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildPredictComponent'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/deprecated/dataflow/tfdv && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildTFDVComponent'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/deprecated/dataflow/tft && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildTFTComponent'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/deprecated/dataflow/tfma && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildTMAComponent'
  waitFor: ["-"]

# Build the Kubeflow-based pipeline component images
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-deployer:$COMMIT_SHA',
         '/workspace/components/kubeflow/deployer']
  id: 'buildDeployer'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/kubeflow/launcher && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildLauncher'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/kubeflow/dnntrainer && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA -l ml-pipeline-kubeflow-tf-trainer-gpu -b 1.6.0-gpu']
  id: 'buildGpuTrainer'
  waitFor: ["-"]

# Build the Dataproc-based pipeline component images
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/deprecated/dataproc/analyze && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildDataprocAnalyze'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/deprecated/dataproc/create_cluster && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildDataprocCreateCluster'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/deprecated/dataproc/delete_cluster && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildDataprocDeleteCluster'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/deprecated/dataproc/predict && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildDataprocPredict'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/deprecated/dataproc/transform && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildDataprocTransform'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/deprecated/dataproc/train && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildDataprocTrain'
  waitFor: ["-"]

# Build the Generic GCP component image
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/gcp/container/ && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildGcpGenericComponent'
  waitFor: ["-"]

# Build the local pipeline component images
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/local/confusion_matrix && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildConfusionMatrix'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/local/roc && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildROC'
  waitFor: ["-"]

images:
# Images for the pipeline system itself
- 'gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/api-server:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/scheduledworkflow:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/persistenceagent:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/inverse-proxy-agent:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/visualization-server:$COMMIT_SHA'

# Images for the Dataflow-based pipeline components
- 'gcr.io/$PROJECT_ID/ml-pipeline-dataflow-tf-predict:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-dataflow-tfdv:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-dataflow-tft:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-dataflow-tfma:$COMMIT_SHA'

# Images for the Kubeflow-based pipeline components
- 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-deployer:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tf-trainer:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tf-trainer-gpu:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tf:$COMMIT_SHA'

# Images for the Dataproc-based pipeline components
- 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-analyze:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-create-cluster:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-delete-cluster:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-predict:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-transform:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-train:$COMMIT_SHA'

# Images for the GCP generic pipeline components
- 'gcr.io/$PROJECT_ID/ml-pipeline-gcp:$COMMIT_SHA'

# Images for the local components
- 'gcr.io/$PROJECT_ID/ml-pipeline-local-confusion-matrix:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-local-roc:$COMMIT_SHA'

timeout: '3600s'
options:
 diskSizeGb: 300
 machineType: 'N1_HIGHCPU_8'