Switching test to kubeflow deployment (#351)

* test

* fix

* fix

* fix

* fix

* fix

* update

* cleanup

* fix

* coopy test

* chmod

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* update

* fix

* fix

* fix

* fix

* fix

* fix

* fix sample test

* fix

* fix

* merge

* update image builder image

* update script

* mount permission

backend/test/test_utils.go

@@ -23,6 +23,8 @@ import (
 
 	"testing"
 
+	"net/http"
+
 	"github.com/cenkalti/backoff"
 	experimentparams "github.com/kubeflow/pipelines/backend/api/go_http_client/experiment_client/experiment_service"
 	jobparams "github.com/kubeflow/pipelines/backend/api/go_http_client/job_client/job_service"
@@ -31,59 +33,29 @@ import (
 	"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 )
 
-const (
-	// ML pipeline API server root URL
-	mlPipelineAPIServerBase = "/api/v1/namespaces/%s/services/ml-pipeline:8888/proxy/apis/v1beta1/%s"
-)
-
 var namespace = flag.String("namespace", "kubeflow", "The namespace ml pipeline deployed to")
 var initializeTimeout = flag.Duration("initializeTimeout", 2*time.Minute, "Duration to wait for test initialization")
 
-func getKubernetesClient() (*kubernetes.Clientset, error) {
-	// use the current context in kubeconfig
-	config, err := rest.InClusterConfig()
-	if err != nil {
-		return nil, errors.Wrapf(err, "Failed to get cluster config during K8s client initialization")
-	}
-	// create the clientset
-	clientSet, err := kubernetes.NewForConfig(config)
-	if err != nil {
-		return nil, errors.Wrapf(err, "Failed to create client set during K8s client initialization")
-	}
-
-	return clientSet, nil
-}
-
 func waitForReady(namespace string, initializeTimeout time.Duration) error {
-	clientSet, err := getKubernetesClient()
-	if err != nil {
-		return errors.Wrapf(err, "Failed to get K8s client set when waiting for ML pipeline to be ready")
-	}
-
 	var operation = func() error {
-		response := clientSet.RESTClient().Get().
-			AbsPath(fmt.Sprintf(mlPipelineAPIServerBase, namespace, "healthz")).Do()
-		if response.Error() == nil {
+		response, err := http.Get(fmt.Sprintf("http://ml-pipeline.%s.svc.cluster.local:8888/apis/v1beta1/healthz", namespace))
+		if err == nil {
 			return nil
 		}
-		var code int
-		response.StatusCode(&code)
 		// we wait only on 503 service unavailable. Stop retry otherwise.
-		if code != 503 {
-			return backoff.Permanent(errors.Wrapf(response.Error(), "Waiting for ml pipeline failed with non retriable error."))
+		if response.StatusCode != 503 {
+			return backoff.Permanent(errors.Wrapf(err, "Waiting for ml pipeline failed with non retriable error."))
 		}
-		return response.Error()
+		return err
 	}
 
 	b := backoff.NewExponentialBackOff()
 	b.MaxElapsedTime = initializeTimeout
-	err = backoff.Retry(operation, b)
+	err := backoff.Retry(operation, b)
 	return errors.Wrapf(err, "Waiting for ml pipeline failed after all attempts.")
 }
 

test/api-integration-test/run_test.sh

@@ -49,6 +49,10 @@ if [ -z "$RESULTS_GCS_DIR" ]; then
     exit 1
 fi
 
+if [[ ! -z "${GOOGLE_APPLICATION_CREDENTIALS}" ]]; then
+  gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}"
+fi
+
 GITHUB_REPO=kubeflow/pipelines
 BASE_DIR=/go/src/github.com/${GITHUB_REPO}
 JUNIT_TEST_RESULT=junit_ApiIntegrationTestOutput.xml

test/build_image.yaml

@@ -113,7 +113,7 @@ spec:
           valueFrom:
             path: /outputs/strict-image-name/file
     container:
-      image: gcr.io/ml-pipeline-staging/image-builder:v20181120-0.1.3-rc.1-21-gb7c8af0-dirty-9786df
+      image: gcr.io/ml-pipeline-test/image-builder:v20181128-0.1.3-rc.1-109-ga5a14dc-e3b0c4
       imagePullPolicy: 'Always'
       args: [
         "--image-build-context-gcs-uri", "{{inputs.parameters.image-build-context-gcs-uri}}",

test/check-argo-status.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+#
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+echo "check status of argo workflow $ARGO_WORKFLOW...."
+# probing the argo workflow status until it completed. Timeout after 30 minutes
+for i in $(seq 1 ${PULL_ARGO_WORKFLOW_STATUS_MAX_ATTEMPT})
+do
+  WORKFLOW_STATUS=`kubectl get workflow $ARGO_WORKFLOW -n ${NAMESPACE} --show-labels`
+  echo $WORKFLOW_STATUS | grep ${WORKFLOW_COMPLETE_KEYWORD} && s=0 && break || s=$? && printf "Workflow ${ARGO_WORKFLOW} is not finished.\n${WORKFLOW_STATUS}\nSleep for 20 seconds...\n" && sleep 20
+done
+
+# Check whether the argo workflow finished or not and exit if not.
+if [[ $s != 0 ]]; then
+ echo "Prow job Failed: Argo workflow timeout.."
+ argo logs -w ${ARGO_WORKFLOW} -n ${NAMESPACE}
+ exit $s
+fi
+
+echo "Argo workflow finished."
+
+if [[ ! -z "$TEST_RESULT_FOLDER" ]]
+then
+  echo "Copy test result"
+  mkdir -p $ARTIFACT_DIR
+  gsutil cp -r "${TEST_RESULTS_GCS_DIR}"/* "${ARTIFACT_DIR}" || true
+fi
+
+if [[ $WORKFLOW_STATUS = *"${WORKFLOW_FAILED_KEYWORD}"* ]]; then
+  echo "Test workflow failed."
+  echo "=========Argo Workflow Logs========="
+  argo logs -w ${ARGO_WORKFLOW} -n ${NAMESPACE}
+  echo "===================================="
+  argo get ${ARGO_WORKFLOW} -n ${NAMESPACE}
+  exit 1
+else
+  argo get ${ARGO_WORKFLOW} -n ${NAMESPACE}
+  exit 0
+fi

test/e2e_test_gke.yaml

@@ -195,7 +195,7 @@ spec:
           valueFrom:
             path: /outputs/strict-image-name/file
     container:
-      image: gcr.io/ml-pipeline-staging/image-builder:v20181120-0.1.3-rc.1-21-gb7c8af0-dirty-9786df
+      image: gcr.io/ml-pipeline-test/image-builder:v20181128-0.1.3-rc.1-109-ga5a14dc-e3b0c4
       imagePullPolicy: 'Always'
       args: [
         "--image-build-context-gcs-uri", "{{inputs.parameters.image-build-context-gcs-uri}}",
@@ -267,4 +267,3 @@ spec:
         "--namespace", "{{inputs.parameters.namespace}}",
         "--test-name", "{{inputs.parameters.test-name}}",
       ]
-