Add postsubmit component test

test/check-argo-status.sh

@@ -14,6 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+ARTIFACT_DIR=$WORKSPACE/_artifacts
+WORKFLOW_COMPLETE_KEYWORD="completed=true"
+WORKFLOW_FAILED_KEYWORD="phase=Failed"
+PULL_ARGO_WORKFLOW_STATUS_MAX_ATTEMPT=$(expr $TIMEOUT_SECONDS / 20 )
 
 echo "check status of argo workflow $ARGO_WORKFLOW...."
 # probing the argo workflow status until it completed. Timeout after 30 minutes

test/deploy-kubeflow.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+#
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"
+
+# Install ksonnet
+KS_VERSION="0.13.0"
+curl -LO https://github.com/ksonnet/ksonnet/releases/download/v${KS_VERSION}/ks_${KS_VERSION}_linux_amd64.tar.gz
+tar -xzf ks_${KS_VERSION}_linux_amd64.tar.gz
+chmod +x ./ks_${KS_VERSION}_linux_amd64/ks
+mv ./ks_${KS_VERSION}_linux_amd64/ks /usr/local/bin/
+
+# Download kubeflow master
+KUBEFLOW_MASTER=${DIR}/kubeflow_master
+git clone https://github.com/kubeflow/kubeflow.git ${KUBEFLOW_MASTER}
+
+## Download latest kubeflow release source code
+KUBEFLOW_SRC=${DIR}/kubeflow_latest_release
+mkdir ${KUBEFLOW_SRC}
+cd ${KUBEFLOW_SRC}
+export KUBEFLOW_TAG=v0.3.1
+curl https://raw.githubusercontent.com/kubeflow/kubeflow/${KUBEFLOW_TAG}/scripts/download.sh | bash
+
+## Override the pipeline config with code from master
+cp -r ${KUBEFLOW_MASTER}/kubeflow/pipeline ${KUBEFLOW_SRC}/kubeflow/pipeline
+cp -r ${KUBEFLOW_MASTER}/kubeflow/argo ${KUBEFLOW_SRC}/kubeflow/argo
+
+# TODO temporarily set KUBEFLOW_SRC as KUBEFLOW_MASTER. This should be deleted when latest release have the pipeline entry
+KUBEFLOW_SRC=${KUBEFLOW_MASTER}
+
+export CLIENT_ID=${RANDOM}
+export CLIENT_SECRET=${RANDOM}
+KFAPP=${TEST_CLUSTER}
+
+function clean_up {
+  echo "Clean up..."
+  cd ${KFAPP}
+  ${KUBEFLOW_SRC}/scripts/kfctl.sh delete all
+  # delete the storage
+  gcloud deployment-manager --project=${PROJECT} deployments delete ${KFAPP}-storage --quiet
+}
+trap clean_up EXIT
+
+${KUBEFLOW_SRC}/scripts/kfctl.sh init ${KFAPP} --platform ${PLATFORM} --project ${PROJECT} --skipInitProject
+
+cd ${KFAPP}
+${KUBEFLOW_SRC}/scripts/kfctl.sh generate platform
+${KUBEFLOW_SRC}/scripts/kfctl.sh apply platform
+${KUBEFLOW_SRC}/scripts/kfctl.sh generate k8s
+${KUBEFLOW_SRC}/scripts/kfctl.sh apply k8s
+
+gcloud container clusters get-credentials ${TEST_CLUSTER}

test/deploy-pipeline.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+#
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+
+
+usage()
+{
+    echo "usage: deploy.sh
+    [--gcr_image_base_dir   the gcr image base directory including images such as apiImage and persistenceAgentImage]
+    [--gcr_image_tag   		the tags for images such as apiImage and persistenceAgentImage]
+    [-h help]"
+}
+GCR_IMAGE_TAG=latest
+
+while [ "$1" != "" ]; do
+    case $1 in
+             --gcr_image_base_dir )   shift
+                                      GCR_IMAGE_BASE_DIR=$1
+                                      ;;
+             --gcr_image_tag )   	  shift
+                                      GCR_IMAGE_TAG=$1
+                                      ;;                                      
+             -h | --help )            usage
+                                      exit
+                                      ;;
+             * )                      usage
+                                      exit 1
+    esac
+    shift
+done
+
+cd ${KUBEFLOW_SRC}
+cd ${KFAPP}
+
+## Update pipeline component image
+pushd ks_app
+ks param set pipeline apiImage ${GCR_IMAGE_BASE_DIR}/api:${GCR_IMAGE_TAG}
+ks param set pipeline persistenceAgentImage ${GCR_IMAGE_BASE_DIR}/persistenceagent:${GCR_IMAGE_TAG}
+ks param set pipeline scheduledWorkflowImage ${GCR_IMAGE_BASE_DIR}/scheduledworkflow:${GCR_IMAGE_TAG}
+ks param set pipeline uiImage ${GCR_IMAGE_BASE_DIR}/frontend:${GCR_IMAGE_TAG}
+ks apply default -c pipeline
+popd

test/postsubmit-tests-with-pipeline-deployment.sh

@@ -0,0 +1,138 @@
+#!/bin/bash
+#
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+
+usage()
+{
+    echo "usage: deploy.sh
+    [--platform             the deployment platform. Valid values are: [gcp, minikube]. Default is gcp.]
+    [--workflow_file        the file name of the argo workflow to run]
+    [--test_result_bucket   the gcs bucket that argo workflow store the result to. Default is ml-pipeline-test
+    [--test_result_folder   the gcs folder that argo workflow store the result to. Always a relative directory to gs://<gs_bucket>/[PULL_SHA]]
+    [--timeout              timeout of the tests in seconds. Default is 1800 seconds. ]
+    [-h help]"
+}
+
+PLATFORM=gcp
+PROJECT=ml-pipeline-test
+TEST_RESULT_BUCKET=ml-pipeline-test
+GCR_IMAGE_BASE_DIR=gcr.io/ml-pipeline-staging/
+TARGET_IMAGE_BASE_DIR=gcr.io/ml-pipeline-test/${PULL_BASE_SHA}
+TIMEOUT_SECONDS=1800
+NAMESPACE=kubeflow
+
+while [ "$1" != "" ]; do
+    case $1 in
+             --platform )             shift
+                                      PLATFORM=$1
+                                      ;;
+             --workflow_file )        shift
+                                      WORKFLOW_FILE=$1
+                                      ;;
+             --test_result_bucket )   shift
+                                      TEST_RESULT_BUCKET=$1
+                                      ;;
+             --test_result_folder )   shift
+                                      TEST_RESULT_FOLDER=$1
+                                      ;;
+             --timeout )              shift
+                                      TIMEOUT_SECONDS=$1
+                                      ;;
+             -h | --help )            usage
+                                      exit
+                                      ;;
+             * )                      usage
+                                      exit 1
+    esac
+    shift
+done
+
+#Variables
+# Refer to https://github.com/kubernetes/test-infra/blob/e357ffaaeceafe737bd6ab89d2feff132d92ea50/prow/jobs.md for the Prow job environment variables
+TEST_RESULTS_GCS_DIR=gs://${TEST_RESULT_BUCKET}/${PULL_BASE_SHA}/${TEST_RESULT_FOLDER}
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"
+
+echo "postsubmit test starts"
+
+source "${DIR}/test-prep.sh"
+source "${DIR}/deploy-kubeflow.sh"
+
+# Install Argo
+source "${DIR}/install-argo.sh"
+
+## Wait for the cloudbuild job to be started
+CLOUDBUILD_TIMEOUT_SECONDS=3600
+PULL_CLOUDBUILD_STATUS_MAX_ATTEMPT=$(expr ${CLOUDBUILD_TIMEOUT_SECONDS} / 20 )
+CLOUDBUILD_STARTED=TIMEOUT
+
+for i in $(seq 1 ${PULL_CLOUDBUILD_STATUS_MAX_ATTEMPT})
+do
+  output=`gcloud builds list --filter="sourceProvenance.resolvedRepoSource.commitSha:${PULL_BASE_SHA}"`
+  if [[ ${output} != "" ]]; then
+    CLOUDBUILD_STARTED=True
+    break
+  fi
+  sleep 20
+done
+
+if [[ ${CLOUDBUILD_STARTED} == TIMEOUT ]];then
+  echo "Wait for cloudbuild job to start, timeout exiting..."
+  exit 1
+fi
+
+## Wait for the cloudbuild job to complete
+CLOUDBUILD_FINISHED=TIMEOUT
+for i in $(seq 1 ${PULL_CLOUDBUILD_STATUS_MAX_ATTEMPT})
+do
+  output=`gcloud builds list --filter="sourceProvenance.resolvedRepoSource.commitSha:${PULL_BASE_SHA}"`
+  if [[ ${output} == *"SUCCESS"* ]]; then
+    CLOUDBUILD_FINISHED=SUCCESS
+    break
+  elif [[ ${output} == *"FAILURE"* ]]; then
+    CLOUDBUILD_FINISHED=FAILURE
+    break
+  fi
+  sleep 20
+done
+
+if [[ ${CLOUDBUILD_FINISHED} == FAILURE ]];then
+  echo "Cloud build failure, postsubmit tests cannot proceed. exiting..."
+  exit 1
+elif [[ ${CLOUDBUILD_FINISHED} == TIMEOUT ]];then
+  echo "Wait for cloudbuild job to finish, timeout exiting..."
+  exit 1
+fi
+
+# Deploy the pipeline
+source ${DIR}/deploy-pipeline.sh --gcr_image_base_dir ${GCR_IMAGE_BASE_DIR} --gcr_image_tag ${PULL_BASE_SHA}
+
+# Submit the argo job and check the results
+echo "submitting argo workflow for commit ${PULL_BASE_SHA}..."
+ARGO_WORKFLOW=`argo submit ${DIR}/${WORKFLOW_FILE} \
+-p image-build-context-gcs-uri="$remote_code_archive_uri" \
+-p commit-sha="${PULL_BASE_SHA}" \
+-p component-image-prefix="${GCR_IMAGE_BASE_DIR}" \
+-p target-image-prefix="${TARGET_IMAGE_BASE_DIR}/" \
+-p test-results-gcs-dir="${TEST_RESULTS_GCS_DIR}" \
+-p cluster-type="${CLUSTER_TYPE}" \
+-n ${NAMESPACE} \
+--serviceaccount test-runner \
+-o name
+`
+echo "argo workflow submitted successfully"
+source "${DIR}/check-argo-status.sh"
+echo "test workflow completed"

test/presubmit-tests-with-pipeline-deployment.sh

@@ -60,82 +60,18 @@ while [ "$1" != "" ]; do
     shift
 done
 
+# Variables
 TEST_RESULTS_GCS_DIR=gs://${TEST_RESULT_BUCKET}/${PULL_PULL_SHA}/${TEST_RESULT_FOLDER}
-ARTIFACT_DIR=$WORKSPACE/_artifacts
-WORKFLOW_COMPLETE_KEYWORD="completed=true"
-WORKFLOW_FAILED_KEYWORD="phase=Failed"
-PULL_ARGO_WORKFLOW_STATUS_MAX_ATTEMPT=$(expr $TIMEOUT_SECONDS / 20 )
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"
 
 echo "presubmit test starts"
+source "${DIR}/test-prep.sh"
+source "${DIR}/deploy-kubeflow.sh"
 
-# activating the service account
-gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}"
-gcloud config set compute/zone us-east1-b
-gcloud config set core/project ${PROJECT}
-
-#Uploading the source code to GCS:
-local_code_archive_file=$(mktemp)
-date_string=$(TZ=PST8PDT date +%Y-%m-%d_%H-%M-%S_%Z)
-code_archive_prefix="gs://${TEST_RESULT_BUCKET}/${PULL_PULL_SHA}/source_code"
-remote_code_archive_uri="${code_archive_prefix}_${PULL_BASE_SHA}_${date_string}.tar.gz"
-
-tar -czf "$local_code_archive_file" .
-gsutil cp "$local_code_archive_file" "$remote_code_archive_uri"
-
-# Install ksonnet
-KS_VERSION="0.13.0"
-curl -LO https://github.com/ksonnet/ksonnet/releases/download/v${KS_VERSION}/ks_${KS_VERSION}_linux_amd64.tar.gz
-tar -xzf ks_${KS_VERSION}_linux_amd64.tar.gz
-chmod +x ./ks_${KS_VERSION}_linux_amd64/ks
-mv ./ks_${KS_VERSION}_linux_amd64/ks /usr/local/bin/
-
-# Download kubeflow master
-KUBEFLOW_MASTER=${DIR}/kubeflow_master
-git clone https://github.com/kubeflow/kubeflow.git ${KUBEFLOW_MASTER}
-
-## Download latest kubeflow release source code
-KUBEFLOW_SRC=${DIR}/kubeflow_latest_release
-mkdir ${KUBEFLOW_SRC}
-cd ${KUBEFLOW_SRC}
-export KUBEFLOW_TAG=v0.3.1
-curl https://raw.githubusercontent.com/kubeflow/kubeflow/${KUBEFLOW_TAG}/scripts/download.sh | bash
-
-## Override the pipeline config with code from master
-cp -r ${KUBEFLOW_MASTER}/kubeflow/pipeline ${KUBEFLOW_SRC}/kubeflow/pipeline
-cp -r ${KUBEFLOW_MASTER}/kubeflow/argo ${KUBEFLOW_SRC}/kubeflow/argo
-
-# TODO temporarily set KUBEFLOW_SRC as KUBEFLOW_MASTER. This should be deleted when latest release have the pipeline entry
-KUBEFLOW_SRC=${KUBEFLOW_MASTER}
-
-TEST_CLUSTER_PREFIX=${WORKFLOW_FILE%.*}
-TEST_CLUSTER=$(echo $TEST_CLUSTER_PREFIX | cut -d _ -f 1)-${PULL_PULL_SHA:0:7}-${RANDOM}
-
-export CLIENT_ID=${RANDOM}
-export CLIENT_SECRET=${RANDOM}
-KFAPP=${TEST_CLUSTER}
-
-function clean_up {
-  echo "Clean up..."
-  cd ${KFAPP}
-  ${KUBEFLOW_SRC}/scripts/kfctl.sh delete all
-  # delete the storage
-  gcloud deployment-manager --project=${PROJECT} deployments delete ${KFAPP}-storage --quiet
-}
-trap clean_up EXIT
-
-${KUBEFLOW_SRC}/scripts/kfctl.sh init ${KFAPP} --platform ${PLATFORM} --project ${PROJECT} --skipInitProject
-
-cd ${KFAPP}
-${KUBEFLOW_SRC}/scripts/kfctl.sh generate platform
-${KUBEFLOW_SRC}/scripts/kfctl.sh apply platform
-${KUBEFLOW_SRC}/scripts/kfctl.sh generate k8s
-${KUBEFLOW_SRC}/scripts/kfctl.sh apply k8s
-
-gcloud container clusters get-credentials ${TEST_CLUSTER}
-
+# Install Argo
 source "${DIR}/install-argo.sh"
 
+# Build Images
 echo "submitting argo workflow to build docker images for commit ${PULL_PULL_SHA}..."
 ARGO_WORKFLOW=`argo submit ${DIR}/build_image.yaml \
 -p image-build-context-gcs-uri="$remote_code_archive_uri" \
@@ -151,19 +87,11 @@ ARGO_WORKFLOW=`argo submit ${DIR}/build_image.yaml \
 -o name
 `
 echo "build docker images workflow submitted successfully"
-
 source "${DIR}/check-argo-status.sh"
-
 echo "build docker images workflow completed"
 
-## Update pipeline component with the newly built image
-pushd ks_app
-ks param set pipeline apiImage ${GCR_IMAGE_BASE_DIR}/api
-ks param set pipeline persistenceAgentImage ${GCR_IMAGE_BASE_DIR}/persistenceagent
-ks param set pipeline scheduledWorkflowImage ${GCR_IMAGE_BASE_DIR}/scheduledworkflow
-ks param set pipeline uiImage ${GCR_IMAGE_BASE_DIR}/frontend
-ks apply default -c pipeline
-popd
+# Deploy the pipeline
+source ${DIR}/deploy-pipeline.sh --gcr_image_base_dir ${GCR_IMAGE_BASE_DIR}
 
 echo "submitting argo workflow to run tests for commit ${PULL_PULL_SHA}..."
 ARGO_WORKFLOW=`argo submit ${DIR}/${WORKFLOW_FILE} \
@@ -177,7 +105,5 @@ ARGO_WORKFLOW=`argo submit ${DIR}/${WORKFLOW_FILE} \
 `
 
 echo "test workflow submitted successfully"
-
 source "${DIR}/check-argo-status.sh"
-
 echo "test workflow completed"