Merge branch 'master' into notebook2

README.md

@@ -15,3 +15,7 @@ The Kubeflow pipelines service has the following goals:
 ## Documentation
 
 Get started with your first pipeline and read further information in the [documentation](https://github.com/kubeflow/pipelines/wiki).
+
+## Aknowledgments
+
+Kubeflow pipelines uses [Argo](https://github.com/argoproj/argo) under the hood to orchestrate Kubernetes resources. The Argo community has been very supportive and we are very grateful.

test/api-integration-test/run_test.sh

@@ -58,12 +58,8 @@ BASE_DIR=/go/src/github.com/${GITHUB_REPO}
 JUNIT_TEST_RESULT=junit_ApiIntegrationTestOutput.xml
 TEST_DIR=backend/test
 
-# Add github to SSH known host.
-ssh-keygen -F github.com || ssh-keyscan github.com >>~/.ssh/known_hosts
-cp ~/.ssh/github/* ~/.ssh
-
 echo "Clone ML pipeline code in COMMIT SHA ${COMMIT_SHA}..."
-git clone git@github.com:${GITHUB_REPO}.git ${BASE_DIR}
+git clone https://github.com/${GITHUB_REPO} ${BASE_DIR}
 cd ${BASE_DIR}/${TEST_DIR}
 git checkout ${COMMIT_SHA}
 

test/backend-unit-test/run_test.sh

@@ -51,12 +51,8 @@ BASE_DIR=/go/src/github.com/${GITHUB_REPO}
 JUNIT_TEST_RESULT=junit_BackendUnitTestOutput.xml
 TEST_DIR=backend/src
 
-# Add github to SSH known host.
-ssh-keygen -F github.com || ssh-keyscan github.com >>~/.ssh/known_hosts
-cp ~/.ssh/github/* ~/.ssh
-
 echo "Clone ML pipeline code in COMMIT SHA ${COMMIT_SHA}..."
-git clone git@github.com:${GITHUB_REPO}.git ${BASE_DIR}
+git clone https://github.com/${GITHUB_REPO} ${BASE_DIR}
 cd ${BASE_DIR}/${TEST_DIR}
 git checkout ${COMMIT_SHA}
 

test/build_image.yaml

@@ -110,17 +110,14 @@ spec:
         value: Dockerfile
       - name: image-name
     container:
-      image: gcr.io/ml-pipeline-staging/image-builder:v20181102-0.0.20-195-g2c8e37ab-e3b0c4
+      image: gcr.io/ml-pipeline-staging/image-builder:v20181108-0.1.1-38-g93afa4d-e3b0c4
       imagePullPolicy: 'Always'
       args: [
         "--commit_sha", "{{inputs.parameters.commit-sha}}",
         "--docker_path", "{{inputs.parameters.docker-path}}",
         "--docker_file", "{{inputs.parameters.docker-file}}",
         "--image_name", "{{inputs.parameters.image-name}}",
       ]
-      volumeMounts:
-      - name: secret-volume
-        mountPath: "/root/.ssh/github"
       env:
       - name: DOCKER_HOST
         value: 127.0.0.1
@@ -130,9 +127,3 @@ spec:
       securityContext:
         privileged: true
       mirrorVolumeMounts: true
-
-  volumes:
-  - name: secret-volume
-    secret:
-      defaultMode: 256
-      secretName: ssh-key-secret

test/e2e_test_gke.yaml

@@ -133,17 +133,14 @@ spec:
         value: Dockerfile
       - name: image-suffix
     container:
-      image: gcr.io/ml-pipeline-staging/image-builder:v20181102-0.0.20-195-g2c8e37ab-e3b0c4
+      image: gcr.io/ml-pipeline-staging/image-builder:v20181108-0.1.1-38-g93afa4d-e3b0c4
       imagePullPolicy: 'Always'
       args: [
         "--commit_sha", "{{inputs.parameters.commit-sha}}",
         "--docker_path", "{{inputs.parameters.docker-path}}",
         "--docker_file", "{{inputs.parameters.docker-file}}",
         "--image_name", "gcr.io/{{inputs.parameters.project}}/{{inputs.parameters.commit-sha}}/{{inputs.parameters.image-suffix}}",
       ]
-      volumeMounts:
-      - name: secret-volume
-        mountPath: "/root/.ssh/github"
       env:
       - name: DOCKER_HOST
         value: 127.0.0.1
@@ -185,9 +182,6 @@ spec:
         "--commit_sha", "{{inputs.parameters.commit-sha}}",
         "--results-gcs-dir", "{{inputs.parameters.test-results-gcs-dir}}",
       ]
-      volumeMounts:
-      - name: secret-volume
-        mountPath: "/root/.ssh/github"
 
   - name: run-frontend-integration-tests
     inputs:
@@ -202,8 +196,3 @@ spec:
         "--results-gcs-dir", "{{inputs.parameters.test-results-gcs-dir}}",
       ]
 
-  volumes:
-  - name: secret-volume
-    secret:
-      defaultMode: 256
-      secretName: ssh-key-secret

test/imagebuilder/build.sh

@@ -57,11 +57,8 @@ done
 
 BASE_DIR=/ml
 
-ssh-keygen -F github.com || ssh-keyscan github.com >>~/.ssh/known_hosts
-cp ~/.ssh/github/* ~/.ssh
-
 echo "Clone ML pipeline code in COMMIT SHA ${COMMIT_SHA}..."
-git clone git@github.com:kubeflow/pipelines.git ${BASE_DIR}
+git clone https://github.com/kubeflow/pipelines ${BASE_DIR}
 cd ${BASE_DIR}
 git checkout ${COMMIT_SHA}
 

test/presubmit-tests-gce-minikube.sh

@@ -33,6 +33,9 @@ instance_name=${instance_name:-test-minikube-${PULL_PULL_SHA:0:6}-$(date +%s)-$(
 
 firewall_rule_name=allow-prow-ssh-$instance_name
 
+# activating the service account
+gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}"
+
 #Function to delete VM
 function delete_vm {
     if [ "$keep_created_vm" != true ]; then
@@ -67,11 +70,6 @@ if [ "$(whoami)" == root ]; then
   export USER=not-root
 fi
 
-#Copy the ssh keys
-ssh_key_file=${ssh_key_file:-/etc/ssh-knative/ssh-knative}
-gcloud compute ssh --zone=$ZONE $instance_name -- 'sudo mkdir -m 777 -p /etc/ssh-knative' #Making the directory writable by a non-root user so that we can copy the file
-gcloud compute scp --zone=$ZONE "$ssh_key_file" $instance_name:/etc/ssh-knative/ssh-knative
-
 #Copy repo
 git_root=$(git rev-parse --show-toplevel)
 git_root_parent=$(dirname "$git_root")

test/presubmit-tests.gke.sh

@@ -39,6 +39,9 @@ TEST_CLUSTER=${TEST_CLUSTER_PREFIX//_}-${PULL_PULL_SHA:0:10}-${RANDOM}
 machine_type=n1-standard-2
 num_nodes=3
 
+# activating the service account
+gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}"
+
 function delete_cluster {
     echo "Delete cluster..."
     gcloud container clusters delete ${TEST_CLUSTER} --async

test/presubmit-tests.sh

@@ -67,6 +67,9 @@ PULL_ARGO_WORKFLOW_STATUS_MAX_ATTEMPT=$(expr $TIMEOUT_SECONDS / 20 )
 
 echo "presubmit test starts"
 
+# activating the service account
+gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}"
+
 #Creating a new GKE cluster if needed
 if [ "$CLUSTER_TYPE" == "create-gke" ]; then
   echo "create test cluster"
@@ -100,10 +103,6 @@ ACCOUNT=$(gcloud info --format='value(config.account)')
 kubectl create clusterrolebinding PROW_BINDING --clusterrole=cluster-admin --user=$ACCOUNT
 kubectl create clusterrolebinding DEFAULT_BINDING --clusterrole=cluster-admin --serviceaccount=default:default
 
-echo "Create k8s secret for github SSH credentials"
-cp /etc/ssh-knative/ssh-knative ./id_rsa
-kubectl create secret generic ssh-key-secret --from-file=id_rsa=./id_rsa
-
 echo "install argo"
 ARGO_VERSION=v2.2.0
 mkdir -p ~/bin/

test/sample-test/run_test.sh

@@ -109,12 +109,8 @@ fi
 GITHUB_REPO=kubeflow/pipelines
 BASE_DIR=/python/src/github.com/${GITHUB_REPO}
 
-# Add github to SSH known host.
-ssh-keygen -F github.com || ssh-keyscan github.com >>~/.ssh/known_hosts
-cp ~/.ssh/github/* ~/.ssh
-
 echo "Clone ML pipeline code in COMMIT SHA ${COMMIT_SHA}..."
-git clone git@github.com:${GITHUB_REPO}.git ${BASE_DIR}
+git clone https://github.com/${GITHUB_REPO} ${BASE_DIR}
 cd ${BASE_DIR}
 git checkout ${COMMIT_SHA}
 

test/sample_test.yaml

@@ -622,17 +622,14 @@ spec:
         value: Dockerfile
       - name: image-suffix
     container:
-      image: gcr.io/ml-pipeline-staging/image-builder:v20181102-0.0.20-195-g2c8e37ab-e3b0c4
+      image: gcr.io/ml-pipeline-staging/image-builder:v20181108-0.1.1-38-g93afa4d-e3b0c4
       imagePullPolicy: 'Always'
       args: [
         "--commit_sha", "{{inputs.parameters.commit-sha}}",
         "--docker_path", "{{inputs.parameters.docker-path}}",
         "--docker_file", "{{inputs.parameters.docker-file}}",
         "--image_name", "gcr.io/{{inputs.parameters.project}}/{{inputs.parameters.commit-sha}}/{{inputs.parameters.image-suffix}}",
       ]
-      volumeMounts:
-      - name: secret-volume
-        mountPath: "/root/.ssh/github"
       env:
       - name: DOCKER_HOST
         value: 127.0.0.1
@@ -654,16 +651,13 @@ spec:
       - name: build-script
       - name: image-suffix
     container:
-      image: gcr.io/ml-pipeline-staging/image-builder:v20181102-0.0.20-195-g2c8e37ab-e3b0c4
+      image: gcr.io/ml-pipeline-staging/image-builder:v20181108-0.1.1-38-g93afa4d-e3b0c4
       imagePullPolicy: 'Always'
       args: [
         "--commit_sha", "{{inputs.parameters.commit-sha}}",
         "--build_script", "{{inputs.parameters.build-script}}",
         "--image_name", "gcr.io/{{inputs.parameters.project}}/{{inputs.parameters.commit-sha}}/{{inputs.parameters.image-suffix}}",
       ]
-      volumeMounts:
-      - name: secret-volume
-        mountPath: "/root/.ssh/github"
       env:
       - name: DOCKER_HOST
         value: 127.0.0.1
@@ -735,12 +729,4 @@ spec:
         "--local-roc-image", "{{inputs.parameters.local-roc-image}}",
         "--test-name", "{{inputs.parameters.test-name}}",
       ]
-      volumeMounts:
-      - name: secret-volume
-        mountPath: "/root/.ssh/github"
 
-  volumes:
-  - name: secret-volume
-    secret:
-      defaultMode: 256
-      secretName: ssh-key-secret

test/unit_test_gke.yaml

@@ -45,18 +45,9 @@ spec:
       - name: commit-sha
       - name: test-results-gcs-dir
     container:
-      image: gcr.io/ml-pipeline-staging/backend-unit-test:v20181102-0.0.20-196-g4d66038f-e3b0c4
+      image: gcr.io/ml-pipeline-staging/backend-unit-test:v20181108-0.1.1-37-gbd71789-e3b0c4
       imagePullPolicy: 'Always'
       args: [
         "--results-gcs-dir", "{{inputs.parameters.test-results-gcs-dir}}",
         "--commit_sha", "{{inputs.parameters.commit-sha}}",
       ]
-      volumeMounts:
-      - name: secret-volume
-        mountPath: "/root/.ssh/github"
-
-  volumes:
-  - name: secret-volume
-    secret:
-      defaultMode: 256
-      secretName: ssh-key-secret