Revert accidental changes from previous revert

.cloudbuild.yaml

@@ -165,11 +165,11 @@ steps:
 # Build the local pipeline component images
 - name: 'gcr.io/cloud-builders/docker'
   entrypoint: '/bin/bash'
-  args: ['-c', 'cd /workspace/components/local/containers/confusion_matrix && ./build.sh -p $PROJECT_ID -t $COMMIT_SHA']
+  args: ['-c', 'cd /workspace/components/local/confusion_matrix && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
   id: 'buildConfusionMatrix'
 - name: 'gcr.io/cloud-builders/docker'
   entrypoint: '/bin/bash'
-  args: ['-c', 'cd /workspace/components/local/containers/roc && ./build.sh -p $PROJECT_ID -t $COMMIT_SHA']
+  args: ['-c', 'cd /workspace/components/local/roc && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
   id: 'buildROC'
 
 # Build the tagged samples

backend/Dockerfile

@@ -14,14 +14,33 @@ COPY . .
 RUN apk add --update gcc musl-dev
 RUN go build -o /bin/apiserver backend/src/apiserver/*.go
 
-FROM python:3.5.0-slim as compiler
+FROM python:3.5 as compiler
+
+RUN apt-get update -y && \
+    apt-get install --no-install-recommends -y -q default-jdk wget 
+
+RUN pip3 install setuptools==40.5.0
+
+RUN wget http://central.maven.org/maven2/io/swagger/swagger-codegen-cli/2.3.1/swagger-codegen-cli-2.3.1.jar -O /tmp/swagger-codegen-cli.jar
+
+WORKDIR /go/src/github.com/kubeflow/pipelines
+COPY . .
+WORKDIR /go/src/github.com/kubeflow/pipelines/sdk/python
+RUN ./build.sh /kfp.tar.gz
+RUN pip3 install /kfp.tar.gz
 
-# This is hard coded to 0.0.26. Once kfp DSK release process is automated,
-# we can dynamically refer to the version from same commit SHA.
-RUN pip install https://storage.googleapis.com/ml-pipeline/release/0.0.26/kfp-0.0.26.tar.gz --upgrade
 WORKDIR /samples
 COPY ./samples .
-RUN  find . -maxdepth 2 -name "*.py" -exec dsl-compile --py {} --output {}.tar.gz \;
+
+#We need to check that all samples have been compiled without error.
+#For find program, the -exec argument is a filter predicate just like -name. It only affects whether the file is "found", not the find's exit code.
+#One way to solve this problem is to check whether we have any python pipelines that cannot compile. Here the exit code is the number of such files:
+#RUN  bash -e -c 'exit $(find . -maxdepth 2 -name "*.py" ! -exec dsl-compile --py {} --output {}.tar.gz \; -print | wc -l)'
+#I think it's better to just use a shell loop though.
+#RUN  for pipeline in $(find . -maxdepth 2 -name '*.py' -type f); do dsl-compile --py "$pipeline" --output "$pipeline.tar.gz"; done
+#The "for" loop breaks on all whitespace, so we either need to override IFS or use the "read" command instead.
+RUN  find . -maxdepth 2 -name '*.py' -type f | while read pipeline; do dsl-compile --py "$pipeline" --output "$pipeline.tar.gz"; done
+
 
 FROM alpine
 

backend/src/apiserver/interceptor.go

@@ -26,13 +26,14 @@ import (
 // to be executed before and after all API handler calls, e.g. Logging, error handling.
 // For more details, see https://github.com/grpc/grpc-go/blob/master/interceptor.go
 func apiServerInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
-	glog.Infof("%v called", info.FullMethod)
+	glog.Infof("%v handler starting", info.FullMethod)
 	resp, err = handler(ctx, req)
 	if err != nil {
 		util.LogError(util.Wrapf(err, "%s call failed", info.FullMethod))
 		// Convert error to gRPC errors
 		err = util.ToGRPCError(err)
 		return
 	}
+	glog.Infof("%v handler finished", info.FullMethod)
 	return
 }

backend/test/test_utils.go

@@ -23,6 +23,8 @@ import (
 
 	"testing"
 
+	"net/http"
+
 	"github.com/cenkalti/backoff"
 	experimentparams "github.com/kubeflow/pipelines/backend/api/go_http_client/experiment_client/experiment_service"
 	jobparams "github.com/kubeflow/pipelines/backend/api/go_http_client/job_client/job_service"
@@ -31,59 +33,29 @@ import (
 	"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 )
 
-const (
-	// ML pipeline API server root URL
-	mlPipelineAPIServerBase = "/api/v1/namespaces/%s/services/ml-pipeline:8888/proxy/apis/v1beta1/%s"
-)
-
 var namespace = flag.String("namespace", "kubeflow", "The namespace ml pipeline deployed to")
 var initializeTimeout = flag.Duration("initializeTimeout", 2*time.Minute, "Duration to wait for test initialization")
 
-func getKubernetesClient() (*kubernetes.Clientset, error) {
-	// use the current context in kubeconfig
-	config, err := rest.InClusterConfig()
-	if err != nil {
-		return nil, errors.Wrapf(err, "Failed to get cluster config during K8s client initialization")
-	}
-	// create the clientset
-	clientSet, err := kubernetes.NewForConfig(config)
-	if err != nil {
-		return nil, errors.Wrapf(err, "Failed to create client set during K8s client initialization")
-	}
-
-	return clientSet, nil
-}
-
 func waitForReady(namespace string, initializeTimeout time.Duration) error {
-	clientSet, err := getKubernetesClient()
-	if err != nil {
-		return errors.Wrapf(err, "Failed to get K8s client set when waiting for ML pipeline to be ready")
-	}
-
 	var operation = func() error {
-		response := clientSet.RESTClient().Get().
-			AbsPath(fmt.Sprintf(mlPipelineAPIServerBase, namespace, "healthz")).Do()
-		if response.Error() == nil {
+		response, err := http.Get(fmt.Sprintf("http://ml-pipeline.%s.svc.cluster.local:8888/apis/v1beta1/healthz", namespace))
+		if err == nil {
 			return nil
 		}
-		var code int
-		response.StatusCode(&code)
 		// we wait only on 503 service unavailable. Stop retry otherwise.
-		if code != 503 {
-			return backoff.Permanent(errors.Wrapf(response.Error(), "Waiting for ml pipeline failed with non retriable error."))
+		if response.StatusCode != 503 {
+			return backoff.Permanent(errors.Wrapf(err, "Waiting for ml pipeline failed with non retriable error."))
 		}
-		return response.Error()
+		return err
 	}
 
 	b := backoff.NewExponentialBackOff()
 	b.MaxElapsedTime = initializeTimeout
-	err = backoff.Retry(operation, b)
+	err := backoff.Retry(operation, b)
 	return errors.Wrapf(err, "Waiting for ml pipeline failed after all attempts.")
 }
 

bootstrapper.yaml

@@ -65,7 +65,7 @@ spec:
     spec:
       containers:
       - name: deploy
-        image: gcr.io/ml-pipeline/bootstrapper:0.1.2 #TODO-release: update the release tag for the next release
+        image: gcr.io/ml-pipeline/bootstrapper:0.1.3-rc.2 #TODO-release: update the release tag for the next release
         imagePullPolicy: 'Always'
         # Additional parameter available:
         args: [

components/README.md

@@ -1,9 +1,9 @@
 # Kubeflow pipeline components
 
 Kubeflow pipeline components are implementations of Kubeflow pipeline tasks. Each task takes
-one or more [artifacts](https://github.com/kubeflow/pipelines/wiki/Concepts#step-output-artifacts)
+one or more [artifacts](https://www.kubeflow.org/docs/guides/pipelines/pipelines-concepts/#step-output-artifacts)
 as input and may produce one or more
-[artifacts](https://github.com/kubeflow/pipelines/wiki/Concepts#step-output-artifacts) as output.
+[artifacts](https://www.kubeflow.org/docs/guides/pipelines/pipelines-concepts/#step-output-artifacts) as output.
 
 
 **Example: XGBoost DataProc components**
@@ -31,4 +31,4 @@ Note the naming convention for client code and runtime code—for a task nam
 * The `mytask.py` program contains the client code.
 * The `mytask` directory contains all the runtime code.
 
-See [how to build your own components](https://github.com/kubeflow/pipelines/wiki/Build-Your-Own-Component)
+See [how to build your own components](https://www.kubeflow.org/docs/guides/pipelines/build-component/)

components/kubeflow/launcher/kubeflow_tfjob_launcher_op.py

@@ -17,7 +17,7 @@
 def kubeflow_tfjob_launcher_op(container_image, command, number_of_workers: int, number_of_parameter_servers: int, tfjob_timeout_minutes: int, output_dir=None, step_name='TFJob-launcher'):
     return dsl.ContainerOp(
         name = step_name,
-        image = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf:0.1.0',#TODO-release: update the release tag for the next release
+        image = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf:0.1.3-rc.2',#TODO-release: update the release tag for the next release
         arguments = [
             '--workers', number_of_workers,
             '--pss', number_of_parameter_servers,

components/kubeflow/launcher/src/train.template.yaml

@@ -26,7 +26,7 @@ spec:
         spec:
           containers:
           - name: tensorflow
-            image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:0.1.0 #TODO-release: update the release tag for the next release
+            image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:0.1.3-rc.2 #TODO-release: update the release tag for the next release
             command:
               - python
               - -m
@@ -38,7 +38,7 @@ spec:
         spec:
           containers:
           - name: tensorflow
-            image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:0.1.0 #TODO-release: update the release tag for the next release
+            image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:0.1.3-rc.2 #TODO-release: update the release tag for the next release
             command:
               - python
               - -m
@@ -50,7 +50,7 @@ spec:
             spec:
               containers:
               - name: tensorflow
-                image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:0.1.0 #TODO-release: update the release tag for the next release
+                image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:0.1.3-rc.2 #TODO-release: update the release tag for the next release
                 command:
                   - python
                   - -m

components/local/base/Dockerfile

@@ -0,0 +1,41 @@
+# Copyright 2018 The Kubeflow Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+FROM ubuntu:16.04
+
+RUN apt-get update -y && apt-get install --no-install-recommends -y -q ca-certificates python-dev python-setuptools wget unzip
+
+RUN easy_install pip
+
+RUN pip install google-api-python-client==1.6.2
+RUN pip install pandas==0.18.1
+RUN pip install scikit-learn==0.19.1
+RUN pip install scipy==1.0.0
+RUN pip install tensorflow==1.5
+
+RUN wget -nv https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.zip && \
+    unzip -qq google-cloud-sdk.zip -d tools && \
+    rm google-cloud-sdk.zip && \
+    tools/google-cloud-sdk/install.sh --usage-reporting=false \
+        --path-update=false --bash-completion=false \
+        --disable-installation-options && \
+    tools/google-cloud-sdk/bin/gcloud -q components update \
+        gcloud core gsutil && \
+    tools/google-cloud-sdk/bin/gcloud config set component_manager/disable_update_check true && \
+    touch /tools/google-cloud-sdk/lib/third_party/google.py
+
+ADD build /ml
+
+ENV PATH $PATH:/tools/node/bin:/tools/google-cloud-sdk/bin

components/local/base/build_image.sh

@@ -0,0 +1,25 @@
+#!/bin/bash -e
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+mkdir -p ./build
+rsync -arvp "../confusion_matrix/src"/ ./build/
+rsync -arvp "../roc/src"/ ./build/
+
+cp ../../license.sh ./build
+cp ../../third_party_licenses.csv ./build
+
+docker build -t ml-pipeline-local-base .
+rm -rf ./build

components/local/confusion_matrix/Dockerfile

@@ -0,0 +1,20 @@
+# Copyright 2018 The Kubeflow Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ml-pipeline-local-base
+
+RUN mkdir /usr/licenses && \
+    /ml/license.sh /ml/third_party_licenses.csv /usr/licenses
+
+ENTRYPOINT ["python", "/ml/confusion_matrix.py"]

components/local/confusion_matrix/build_image.sh

@@ -0,0 +1,57 @@
+#!/bin/bash -e
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+while getopts ":hp:t:i:" opt; do
+  case "${opt}" in
+    h) echo "-p: project name"
+        echo "-t: tag name"
+        echo "-i: image name. If provided, project name and tag name are not necessary"
+        exit
+      ;;
+    p) PROJECT_ID=${OPTARG}
+      ;;
+    t) TAG_NAME=${OPTARG}
+      ;;
+    i) IMAGE_NAME=${OPTARG}
+      ;;
+    \? ) echo "Usage: cmd [-p] project [-t] tag [-i] image"
+      exit
+      ;;
+  esac
+done
+
+LOCAL_IMAGE_NAME=ml-pipeline-local-confusion-matrix
+
+if [ -z "${PROJECT_ID}" ]; then
+  PROJECT_ID=$(gcloud config config-helper --format "value(configuration.properties.core.project)")
+fi
+
+if [ -z "${TAG_NAME}" ]; then
+  TAG_NAME=$(date +v%Y%m%d)-$(git describe --tags --always --dirty)-$(git diff | shasum -a256 | cut -c -6)
+fi
+
+# build base image
+pushd ../base
+./build_image.sh
+popd
+
+docker build -t ${LOCAL_IMAGE_NAME} .
+if [ -z "${IMAGE_NAME}" ]; then
+  docker tag ${LOCAL_IMAGE_NAME} gcr.io/${PROJECT_ID}/${LOCAL_IMAGE_NAME}:${TAG_NAME}
+  docker push gcr.io/${PROJECT_ID}/${LOCAL_IMAGE_NAME}:${TAG_NAME}
+else
+  docker tag ${LOCAL_IMAGE_NAME} "${IMAGE_NAME}"
+  docker push "${IMAGE_NAME}"
+fi