Tests - test source code archive upload download extract

kubeflow · Nov 21, 2018 · 8a98c62 · 8a98c62
1 parent 18f54d9
commit 8a98c62
Show file tree

Hide file tree

Showing 11 changed files with 238 additions and 129 deletions.
diff --git a/test/api-integration-test/Dockerfile b/test/api-integration-test/Dockerfile
@@ -17,7 +17,6 @@ ENV PATH $PATH:/usr/local/gcloud/google-cloud-sdk/bin
 RUN go get -u github.com/jstemmer/go-junit-report
 RUN go build github.com/jstemmer/go-junit-report
 
-COPY ./run_test.sh /run_test.sh
-RUN chmod +x /run_test.sh
+COPY ../.. /go/src/github.com/kubeflow/pipelines
 
-ENTRYPOINT ["/run_test.sh"]
+ENTRYPOINT ["/go/src/github.com/kubeflow/pipelines/test/api-integration-test/run_test.sh"]
diff --git a/test/api-integration-test/run_test.sh b/test/api-integration-test/run_test.sh
@@ -23,7 +23,7 @@ usage()
 {
     echo "usage: run_test.sh
     --results-gcs-dir GCS directory for the test results. Usually gs://<project-id>/<commit-sha>/api_integration_test
-    [--commit_sha     commit SHA to pull code from]
+    [--image-build-context-gcs-uri GCS URI pointing to a .tar.gz archive of Docker image build context]
     [--namespace      k8s namespace where ml-pipelines is deployed. The tests run against the instance in this namespace]
     [-h help]"
 }
@@ -33,9 +33,6 @@ while [ "$1" != "" ]; do
              --results-gcs-dir )shift
                                 RESULTS_GCS_DIR=$1
                                 ;;
-             --commit_sha )     shift
-                                COMMIT_SHA=$1
-                                ;;
              --namespace )      shift
                                 NAMESPACE=$1
                                 ;;
@@ -58,13 +55,7 @@ BASE_DIR=/go/src/github.com/${GITHUB_REPO}
 JUNIT_TEST_RESULT=junit_ApiIntegrationTestOutput.xml
 TEST_DIR=backend/test
 
-echo "Clone ML pipeline code in COMMIT SHA ${COMMIT_SHA}..."
-git clone https://github.com/${GITHUB_REPO} ${BASE_DIR}
-cd ${BASE_DIR}
-git config --local user.name 'K8S Bootstrap'
-git config --local user.email k8s_bootstrap@localhost
-git merge --no-ff ${COMMIT_SHA} -m "Merged PR ${COMMIT_SHA}"
-cd ${TEST_DIR}
+cd "${BASE_DIR}/${TEST_DIR}"
 
 echo "Run integration test..."
 TEST_RESULT=`go test -v ./... -namespace ${NAMESPACE} 2>&1`

diff --git a/test/backend-unit-test/Dockerfile b/test/backend-unit-test/Dockerfile
@@ -17,7 +17,6 @@ ENV PATH $PATH:/usr/local/gcloud/google-cloud-sdk/bin
 RUN go get -u github.com/jstemmer/go-junit-report
 RUN go build github.com/jstemmer/go-junit-report
 
-COPY ./run_test.sh /run_test.sh
-RUN chmod +x /run_test.sh
+COPY ../.. /go/src/github.com/kubeflow/pipelines
 
-ENTRYPOINT ["/run_test.sh"]
+ENTRYPOINT ["/go/src/github.com/kubeflow/pipelines/test/backend-unit-test/run_test.sh"]
diff --git a/test/backend-unit-test/run_test.sh b/test/backend-unit-test/run_test.sh
@@ -20,7 +20,6 @@ usage()
 {
     echo "usage: run_test.sh
     --results-gcs-dir GCS directory for the test results. Usually gs://<project-id>/<commit-sha>/api_integration_test
-    [--commit_sha     commit SHA to pull code from]
     [-h help]"
 }
 
@@ -29,9 +28,6 @@ while [ "$1" != "" ]; do
              --results-gcs-dir )shift
                                 RESULTS_GCS_DIR=$1
                                 ;;
-             --commit_sha )     shift
-                                COMMIT_SHA=$1
-                                ;;
              -h | --help )      usage
                                 exit
                                 ;;
@@ -51,13 +47,7 @@ BASE_DIR=/go/src/github.com/${GITHUB_REPO}
 JUNIT_TEST_RESULT=junit_BackendUnitTestOutput.xml
 TEST_DIR=backend/src
 
-echo "Clone ML pipeline code in COMMIT SHA ${COMMIT_SHA}..."
-git clone https://github.com/${GITHUB_REPO} ${BASE_DIR}
-cd ${BASE_DIR}
-git config --local user.name 'K8S Bootstrap'
-git config --local user.email k8s_bootstrap@localhost
-git merge --no-ff ${COMMIT_SHA} -m "Merged PR ${COMMIT_SHA}"
-cd ${TEST_DIR}
+cd "${BASE_DIR}/${TEST_DIR}"
 
 # Run test and store the exit code.
 echo "Run unit test..."

diff --git a/test/build_image.yaml b/test/build_image.yaml
@@ -20,8 +20,7 @@ spec:
   entrypoint: build-images
   arguments:
     parameters:
-    - name: commit-sha
-      value: master
+    - name: image-build-context-gcs-uri
     - name: bootstrapper-image
     - name: api-image
     - name: frontend-image
@@ -31,68 +30,67 @@ spec:
   - name: build-images
     inputs:
       parameters:
-      - name: commit-sha
       - name: bootstrapper-image
       - name: api-image
       - name: frontend-image
       - name: scheduledworkflow-image
       - name: persistenceagent-image
     steps:
     - - name: build-bootstrapper-image
-        template: build-image
+        template: build-image-from-context-archive
         arguments:
           parameters:
-          - name: commit-sha
-            value: "{{inputs.parameters.commit-sha}}"
+          - name: image-build-context-gcs-uri
+            value: "{{workflow.parameters.image-build-context-gcs-uri}}"
           - name: docker-path
             value: pipeline
-          - name: image-name
+          - name: target-image
             value: "{{inputs.parameters.bootstrapper-image}}"
       - name: build-api-server-image
-        template: build-image
+        template: build-image-from-context-archive
         arguments:
           parameters:
-          - name: commit-sha
-            value: "{{inputs.parameters.commit-sha}}"
+          - name: image-build-context-gcs-uri
+            value: "{{workflow.parameters.image-build-context-gcs-uri}}"
           - name: docker-path
             value: .
-          - name: image-name
+          - name: target-image
             value: "{{inputs.parameters.api-image}}"
           - name: docker-file
             value: backend/Dockerfile
       - name: build-frontend-image
-        template: build-image
+        template: build-image-from-context-archive
         arguments:
           parameters:
-          - name: commit-sha
-            value: "{{inputs.parameters.commit-sha}}"
+          - name: image-build-context-gcs-uri
+            value: "{{workflow.parameters.image-build-context-gcs-uri}}"
           - name: docker-path
             value: .
           - name: docker-file
             value: frontend/Dockerfile
-          - name: image-name
+          - name: target-image
             value: "{{inputs.parameters.frontend-image}}"
       - name: build-scheduledworkflow-image
-        template: build-image
+        template: build-image-from-context-archive
         arguments:
           parameters:
-          - name: commit-sha
-            value: "{{inputs.parameters.commit-sha}}"
+          - name: image-build-context-gcs-uri
+            value: "{{workflow.parameters.image-build-context-gcs-uri}}"
           - name: docker-path
             value: .
-          - name: image-name
+          - name: target-image
             value: "{{inputs.parameters.scheduledworkflow-image}}"
           - name: docker-file
             value: backend/Dockerfile.scheduledworkflow
       - name: build-persistenceagent-image
-        template: build-image
+        template: build-image-from-context-archive
         arguments:
           parameters:
-          - name: commit-sha
-            value: "{{inputs.parameters.commit-sha}}"
+          - name: image-build-context-gcs-uri
+            value: "{{workflow.parameters.image-build-context-gcs-uri}}"
           - name: docker-path
             value: .
-          - name: image-name
+          - name: target-image
             value: "{{inputs.parameters.persistenceagent-image}}"
           - name: docker-file
             value: backend/Dockerfile.persistenceagent
@@ -127,3 +125,70 @@ spec:
       securityContext:
         privileged: true
       mirrorVolumeMounts: true
+
+  # Build image and upload to GCR
+  - name: build-image-from-context-archive
+    inputs:
+      parameters:
+      # GCS URI prefix pointing to a .tar.gz archive of Docker build context
+      - name: image-build-context-gcs-uri
+      # The relative code path to the Dockerfile
+      - name: docker-path
+      # Name of the Docker file to use. "Dockerfile" by default
+      - name: docker-file
+        value: Dockerfile
+      - name: target-image
+    outputs:
+      parameters:
+        - name: strict-image-name
+          valueFrom:
+            path: /outputs/strict-image-name/file
+    container:
+      image: google/cloud-sdk
+      imagePullPolicy: 'Always'
+      env:
+      - name: DOCKER_HOST
+        value: 127.0.0.1
+      command: [bash]
+      source: |
+          CONTEXT_GCS_URI="{{inputs.parameters.image-build-context-gcs-uri}}" #gs://$TEST_RESULT_BUCKET/$PULL_PULL_SHA/source_code_<base_sha>_<date>.tar.gz
+          DOCKER_PATH="{{inputs.parameters.docker-path}}"
+          DOCKER_FILE="{{inputs.parameters.docker-file}}"
+          IMAGE_NAME="{{inputs.parameters.target-image}}"
+
+          BASE_DIR=/image_builder
+          mkdir $BASE_DIR
+          cd $BASE_DIR
+
+          echo "Downloading Docker build context from $CONTEXT_GCS_URI..."
+          downloaded_code_archive_file=$(mktemp)
+          gsutil cp "$CONTEXT_GCS_URI" "$downloaded_code_archive_file"
+          tar -xzf "$downloaded_code_archive_file" --directory .
+
+          echo "Waiting for Docker-in-Docker daemon to start..."
+          until docker ps; do sleep 3; done;
+
+          gcloud auth configure-docker
+
+          if [ "$BUILD_SCRIPT" == "" ]; then
+            echo "Build image ${IMAGE_NAME} using ${DOCKER_PATH}/${DOCKER_FILE}..."
+            docker build -t ${IMAGE_NAME} -f ${DOCKER_PATH}/${DOCKER_FILE} ${DOCKER_PATH}
+          else
+            echo "Build image ${IMAGE_NAME} using ${BUILD_SCRIPT}..."
+            cd $(dirname ${BUILD_SCRIPT})
+            bash $(basename ${BUILD_SCRIPT}) -i ${IMAGE_NAME}
+          fi
+
+          echo "Pushing image ${IMAGE_NAME}..."
+          docker push ${IMAGE_NAME}
+
+          image_name_with_digest=$(docker inspect --format="{{index .RepoDigests 0}}" "$IMAGE_NAME")
+          strict_image_name_output_file=/outputs/strict-image-name/file
+          mkdir -p "$(dirname "$strict_image_name_output_file")"
+          echo $image_name_with_digest > "$strict_image_name_output_file"
+    sidecars:
+    - name: dind
+      image: docker:17.10-dind
+      securityContext:
+        privileged: true
+      mirrorVolumeMounts: true