forked from kubeflow/pipelines
-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add postsubmit component test (kubeflow#613)
* add postsubmit script and yaml * remove old sample tests component file * extract deploy-pipeline.sh, deploy-kubeflow.sh and test-prep.sh from presubmit and postsubmit scripts
- Loading branch information
1 parent
c4e7271
commit 8cb8377
Showing
8 changed files
with
541 additions
and
629 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| #!/bin/bash | ||
| # | ||
| # Copyright 2018 Google LLC | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| set -x | ||
|
|
||
| DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)" | ||
|
|
||
| # Install ksonnet | ||
| KS_VERSION="0.13.0" | ||
| curl -LO https://github.com/ksonnet/ksonnet/releases/download/v${KS_VERSION}/ks_${KS_VERSION}_linux_amd64.tar.gz | ||
| tar -xzf ks_${KS_VERSION}_linux_amd64.tar.gz | ||
| chmod +x ./ks_${KS_VERSION}_linux_amd64/ks | ||
| mv ./ks_${KS_VERSION}_linux_amd64/ks /usr/local/bin/ | ||
|
|
||
| # Download kubeflow master | ||
| KUBEFLOW_MASTER=${DIR}/kubeflow_master | ||
| git clone https://github.com/kubeflow/kubeflow.git ${KUBEFLOW_MASTER} | ||
|
|
||
| ## Download latest kubeflow release source code | ||
| KUBEFLOW_SRC=${DIR}/kubeflow_latest_release | ||
| mkdir ${KUBEFLOW_SRC} | ||
| cd ${KUBEFLOW_SRC} | ||
| export KUBEFLOW_TAG=v0.3.1 | ||
| curl https://raw.githubusercontent.com/kubeflow/kubeflow/${KUBEFLOW_TAG}/scripts/download.sh | bash | ||
|
|
||
| ## Override the pipeline config with code from master | ||
| cp -r ${KUBEFLOW_MASTER}/kubeflow/pipeline ${KUBEFLOW_SRC}/kubeflow/pipeline | ||
| cp -r ${KUBEFLOW_MASTER}/kubeflow/argo ${KUBEFLOW_SRC}/kubeflow/argo | ||
|
|
||
| # TODO temporarily set KUBEFLOW_SRC as KUBEFLOW_MASTER. This should be deleted when latest release have the pipeline entry | ||
| KUBEFLOW_SRC=${KUBEFLOW_MASTER} | ||
|
|
||
| export CLIENT_ID=${RANDOM} | ||
| export CLIENT_SECRET=${RANDOM} | ||
| KFAPP=${TEST_CLUSTER} | ||
|
|
||
| function clean_up { | ||
| echo "Clean up..." | ||
| cd ${KFAPP} | ||
| ${KUBEFLOW_SRC}/scripts/kfctl.sh delete all | ||
| # delete the storage | ||
| gcloud deployment-manager --project=${PROJECT} deployments delete ${KFAPP}-storage --quiet | ||
| } | ||
| trap clean_up EXIT | ||
|
|
||
| ${KUBEFLOW_SRC}/scripts/kfctl.sh init ${KFAPP} --platform ${PLATFORM} --project ${PROJECT} --skipInitProject | ||
|
|
||
| cd ${KFAPP} | ||
| ${KUBEFLOW_SRC}/scripts/kfctl.sh generate platform | ||
| ${KUBEFLOW_SRC}/scripts/kfctl.sh apply platform | ||
| ${KUBEFLOW_SRC}/scripts/kfctl.sh generate k8s | ||
| ${KUBEFLOW_SRC}/scripts/kfctl.sh apply k8s | ||
|
|
||
| gcloud container clusters get-credentials ${TEST_CLUSTER} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,56 @@ | ||
| #!/bin/bash | ||
| # | ||
| # Copyright 2018 Google LLC | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| set -x | ||
|
|
||
|
|
||
| usage() | ||
| { | ||
| echo "usage: deploy.sh | ||
| [--gcr_image_base_dir the gcr image base directory including images such as apiImage and persistenceAgentImage] | ||
| [--gcr_image_tag the tags for images such as apiImage and persistenceAgentImage] | ||
| [-h help]" | ||
| } | ||
| GCR_IMAGE_TAG=latest | ||
|
|
||
| while [ "$1" != "" ]; do | ||
| case $1 in | ||
| --gcr_image_base_dir ) shift | ||
| GCR_IMAGE_BASE_DIR=$1 | ||
| ;; | ||
| --gcr_image_tag ) shift | ||
| GCR_IMAGE_TAG=$1 | ||
| ;; | ||
| -h | --help ) usage | ||
| exit | ||
| ;; | ||
| * ) usage | ||
| exit 1 | ||
| esac | ||
| shift | ||
| done | ||
|
|
||
| cd ${KUBEFLOW_SRC} | ||
| cd ${KFAPP} | ||
|
|
||
| ## Update pipeline component image | ||
| pushd ks_app | ||
| ks param set pipeline apiImage ${GCR_IMAGE_BASE_DIR}/api:${GCR_IMAGE_TAG} | ||
| ks param set pipeline persistenceAgentImage ${GCR_IMAGE_BASE_DIR}/persistenceagent:${GCR_IMAGE_TAG} | ||
| ks param set pipeline scheduledWorkflowImage ${GCR_IMAGE_BASE_DIR}/scheduledworkflow:${GCR_IMAGE_TAG} | ||
| ks param set pipeline uiImage ${GCR_IMAGE_BASE_DIR}/frontend:${GCR_IMAGE_TAG} | ||
| ks apply default -c pipeline | ||
| popd |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,138 @@ | ||
| #!/bin/bash | ||
| # | ||
| # Copyright 2018 Google LLC | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| set -x | ||
|
|
||
| usage() | ||
| { | ||
| echo "usage: deploy.sh | ||
| [--platform the deployment platform. Valid values are: [gcp, minikube]. Default is gcp.] | ||
| [--workflow_file the file name of the argo workflow to run] | ||
| [--test_result_bucket the gcs bucket that argo workflow store the result to. Default is ml-pipeline-test | ||
| [--test_result_folder the gcs folder that argo workflow store the result to. Always a relative directory to gs://<gs_bucket>/[PULL_SHA]] | ||
| [--timeout timeout of the tests in seconds. Default is 1800 seconds. ] | ||
| [-h help]" | ||
| } | ||
|
|
||
| PLATFORM=gcp | ||
| PROJECT=ml-pipeline-test | ||
| TEST_RESULT_BUCKET=ml-pipeline-test | ||
| GCR_IMAGE_BASE_DIR=gcr.io/ml-pipeline-staging/ | ||
| TARGET_IMAGE_BASE_DIR=gcr.io/ml-pipeline-test/${PULL_BASE_SHA} | ||
| TIMEOUT_SECONDS=1800 | ||
| NAMESPACE=kubeflow | ||
|
|
||
| while [ "$1" != "" ]; do | ||
| case $1 in | ||
| --platform ) shift | ||
| PLATFORM=$1 | ||
| ;; | ||
| --workflow_file ) shift | ||
| WORKFLOW_FILE=$1 | ||
| ;; | ||
| --test_result_bucket ) shift | ||
| TEST_RESULT_BUCKET=$1 | ||
| ;; | ||
| --test_result_folder ) shift | ||
| TEST_RESULT_FOLDER=$1 | ||
| ;; | ||
| --timeout ) shift | ||
| TIMEOUT_SECONDS=$1 | ||
| ;; | ||
| -h | --help ) usage | ||
| exit | ||
| ;; | ||
| * ) usage | ||
| exit 1 | ||
| esac | ||
| shift | ||
| done | ||
|
|
||
| #Variables | ||
| # Refer to https://github.com/kubernetes/test-infra/blob/e357ffaaeceafe737bd6ab89d2feff132d92ea50/prow/jobs.md for the Prow job environment variables | ||
| TEST_RESULTS_GCS_DIR=gs://${TEST_RESULT_BUCKET}/${PULL_BASE_SHA}/${TEST_RESULT_FOLDER} | ||
| DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)" | ||
|
|
||
| echo "postsubmit test starts" | ||
|
|
||
| source "${DIR}/test-prep.sh" | ||
| source "${DIR}/deploy-kubeflow.sh" | ||
|
|
||
| # Install Argo | ||
| source "${DIR}/install-argo.sh" | ||
|
|
||
| ## Wait for the cloudbuild job to be started | ||
| CLOUDBUILD_TIMEOUT_SECONDS=3600 | ||
| PULL_CLOUDBUILD_STATUS_MAX_ATTEMPT=$(expr ${CLOUDBUILD_TIMEOUT_SECONDS} / 20 ) | ||
| CLOUDBUILD_STARTED=TIMEOUT | ||
|
|
||
| for i in $(seq 1 ${PULL_CLOUDBUILD_STATUS_MAX_ATTEMPT}) | ||
| do | ||
| output=`gcloud builds list --filter="sourceProvenance.resolvedRepoSource.commitSha:${PULL_BASE_SHA}"` | ||
| if [[ ${output} != "" ]]; then | ||
| CLOUDBUILD_STARTED=True | ||
| break | ||
| fi | ||
| sleep 20 | ||
| done | ||
|
|
||
| if [[ ${CLOUDBUILD_STARTED} == TIMEOUT ]];then | ||
| echo "Wait for cloudbuild job to start, timeout exiting..." | ||
| exit 1 | ||
| fi | ||
|
|
||
| ## Wait for the cloudbuild job to complete | ||
| CLOUDBUILD_FINISHED=TIMEOUT | ||
| for i in $(seq 1 ${PULL_CLOUDBUILD_STATUS_MAX_ATTEMPT}) | ||
| do | ||
| output=`gcloud builds list --filter="sourceProvenance.resolvedRepoSource.commitSha:${PULL_BASE_SHA}"` | ||
| if [[ ${output} == *"SUCCESS"* ]]; then | ||
| CLOUDBUILD_FINISHED=SUCCESS | ||
| break | ||
| elif [[ ${output} == *"FAILURE"* ]]; then | ||
| CLOUDBUILD_FINISHED=FAILURE | ||
| break | ||
| fi | ||
| sleep 20 | ||
| done | ||
|
|
||
| if [[ ${CLOUDBUILD_FINISHED} == FAILURE ]];then | ||
| echo "Cloud build failure, postsubmit tests cannot proceed. exiting..." | ||
| exit 1 | ||
| elif [[ ${CLOUDBUILD_FINISHED} == TIMEOUT ]];then | ||
| echo "Wait for cloudbuild job to finish, timeout exiting..." | ||
| exit 1 | ||
| fi | ||
|
|
||
| # Deploy the pipeline | ||
| source ${DIR}/deploy-pipeline.sh --gcr_image_base_dir ${GCR_IMAGE_BASE_DIR} --gcr_image_tag ${PULL_BASE_SHA} | ||
|
|
||
| # Submit the argo job and check the results | ||
| echo "submitting argo workflow for commit ${PULL_BASE_SHA}..." | ||
| ARGO_WORKFLOW=`argo submit ${DIR}/${WORKFLOW_FILE} \ | ||
| -p image-build-context-gcs-uri="$remote_code_archive_uri" \ | ||
| -p commit-sha="${PULL_BASE_SHA}" \ | ||
| -p component-image-prefix="${GCR_IMAGE_BASE_DIR}" \ | ||
| -p target-image-prefix="${TARGET_IMAGE_BASE_DIR}/" \ | ||
| -p test-results-gcs-dir="${TEST_RESULTS_GCS_DIR}" \ | ||
| -p cluster-type="${CLUSTER_TYPE}" \ | ||
| -n ${NAMESPACE} \ | ||
| --serviceaccount test-runner \ | ||
| -o name | ||
| ` | ||
| echo "argo workflow submitted successfully" | ||
| source "${DIR}/check-argo-status.sh" | ||
| echo "test workflow completed" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.