Squashed commit of the following:

commit af456bc Author: Eterna2 <eterna2@hotmail.com> Date: Fri Aug 23 02:08:33 2019 +0800 [front-end-server] Allow viewer:tensorboard podTemplateSpec to be customizable (kubeflow#1906) * Allow front-end server to provide custom viewer podTemplateSpec via path/configmap * Fix JSON.parse input to string commit 06b7ad6 Author: IronPan <yangpa@google.com> Date: Thu Aug 22 10:03:28 2019 -0700 Cleanup pipeline-lite deployment (kubeflow#1921) * restructure * working example * working example * move mysql * moving minio and mysql out * add gcp * add files * fix test * extract parameters to single place * update * update readme * update readme * address pr comment commit 56160f1 Author: Kirin Patel <kirinpatel@gmail.com> Date: Thu Aug 22 09:26:33 2019 -0700 Added support for environment specified kernel timeout (kubeflow#1920)
numerology · Aug 22, 2019 · 35ae6f5 · 35ae6f5
1 parent 04546c5
commit 35ae6f5
Show file tree

Hide file tree

Showing 21 changed files with 182 additions and 130 deletions.
diff --git a/backend/src/apiserver/visualization/server.py b/backend/src/apiserver/visualization/server.py
@@ -16,6 +16,7 @@
 from argparse import Namespace
 import importlib
 import json
+import os
 from pathlib import Path
 from typing import Text
 import shlex
@@ -31,7 +32,7 @@
 parser.add_argument(
     "--timeout",
     type=int,
-    default=100,
+    default=os.getenv('KERNEL_TIMEOUT', 100),
     help="Amount of time in seconds that a visualization can run for before " +
          "being stopped."
 )

diff --git a/frontend/server/k8s-helper.ts b/frontend/server/k8s-helper.ts
@@ -30,6 +30,30 @@ const viewerGroup = 'kubeflow.org';
 const viewerVersion = 'v1beta1';
 const viewerPlural = 'viewers';
 
+export const defaultPodTemplateSpec = {
+  spec: {
+    containers: [{
+      env: [{
+        name: "GOOGLE_APPLICATION_CREDENTIALS",
+        value: "/secret/gcp-credentials/user-gcp-sa.json"
+      }],
+      volumeMounts: [{
+        name: "gcp-credentials",
+        mountPath: "/secret/gcp-credentials/user-gcp-sa.json",
+        readOnly: true
+      }]
+    }],
+    volumes: [{
+      name: "gcp-credentials",
+      volumeSource: {
+        secret: {
+          secretName: "user-gcp-sa"
+        }
+      }
+    }]
+  }
+}
+
 export const isInCluster = fs.existsSync(namespaceFilePath);
 
 if (isInCluster) {
@@ -49,7 +73,7 @@ function getNameOfViewerResource(logdir: string): string {
  * Create Tensorboard instance via CRD with the given logdir if there is no
  * existing Tensorboard instance.
  */
-export async function newTensorboardInstance(logdir: string): Promise<void> {
+export async function newTensorboardInstance(logdir: string, podTemplateSpec: Object = defaultPodTemplateSpec): Promise<void> {
   if (!k8sV1CustomObjectClient) {
     throw new Error('Cannot access kubernetes Custom Object API');
   }
@@ -70,29 +94,7 @@ export async function newTensorboardInstance(logdir: string): Promise<void> {
       tensorboardSpec: {
         logDir: logdir,
       },
-      podTemplateSpec: {
-        spec: {
-          containers: [{
-            env: [{
-              name: "GOOGLE_APPLICATION_CREDENTIALS",
-              value: "/secret/gcp-credentials/user-gcp-sa.json"
-            }],
-            volumeMounts: [{
-              name: "gcp-credentials",
-              mountPath: "/secret/gcp-credentials/user-gcp-sa.json",
-              readOnly: true
-            }]
-          }],
-          volumes: [{
-            name: "gcp-credentials",
-            volumeSource: {
-              secret: {
-                secretName: "user-gcp-sa"
-              }
-            }
-          }]
-        }
-      }
+      podTemplateSpec
     }
   };
   await k8sV1CustomObjectClient.createNamespacedCustomObject(viewerGroup,

diff --git a/frontend/server/server.ts b/frontend/server/server.ts
@@ -27,6 +27,8 @@ import proxyMiddleware from './proxy-middleware';
 import { Storage } from '@google-cloud/storage';
 import {Stream} from 'stream';
 
+import {loadJSON} from './utils';
+
 const BASEPATH = '/pipeline';
 
 /** All configurable environment variables can be found here. */
@@ -50,7 +52,9 @@ const {
   /** API service will listen to this host */
   ML_PIPELINE_SERVICE_HOST = 'localhost',
   /** API service will listen to this port */
-  ML_PIPELINE_SERVICE_PORT = '3001'
+  ML_PIPELINE_SERVICE_PORT = '3001',
+  /** path to viewer:tensorboard pod template spec */
+  VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH
 } = process.env;
 
 /** construct minio endpoint from host and namespace (optional) */
@@ -75,6 +79,9 @@ const s3Client = new MinioClient({
   secretKey: AWS_SECRET_ACCESS_KEY,
 } as any);
 
+/** pod template spec to use for viewer crd */
+const podTemplateSpec = loadJSON(VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH, k8sHelper.defaultPodTemplateSpec)
+
 const app = express() as Application;
 
 app.use(function (req, _, next) {
@@ -278,7 +285,7 @@ const createTensorboardHandler = async (req, res) => {
   }
 
   try {
-    await k8sHelper.newTensorboardInstance(logdir);
+    await k8sHelper.newTensorboardInstance(logdir, podTemplateSpec);
     const tensorboardAddress = await k8sHelper.waitForTensorboardInstance(logdir, 60 * 1000);
     res.send(tensorboardAddress);
   } catch (err) {

diff --git a/frontend/server/utils.ts b/frontend/server/utils.ts
@@ -11,6 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+import {readFileSync} from 'fs';
 
 export function equalArrays(a1: any[], a2: any[]): boolean {
   if (!Array.isArray(a1) || !Array.isArray(a2) || a1.length !== a2.length) {
@@ -32,3 +33,12 @@ export function generateRandomString(length: number): string {
   }
   return str;
 }
+
+export function loadJSON(filepath: string, defaultValue: Object = {}): Object {
+  if (!filepath) return defaultValue;
+  try {
+    return JSON.parse(readFileSync(filepath, "utf-8"))
+  } catch (error) {
+    return defaultValue;
+  }
+}
diff --git a/manifests/kustomize/README.md b/manifests/kustomize/README.md
@@ -1,41 +1,40 @@
+# Install Kubeflow Pipelines
 This folder contains Kubeflow Pipelines Kustomize manifests for a light weight deployment. You can follow the instruction and deploy Kubeflow Pipelines in an existing cluster.
 
 
-# TL;DR
+## TL;DR
 
-If you want to skip any customization, you can deploy Kubeflow Pipelines by running
+Deploy latest version of Kubeflow Pipelines
 ```
 export PIPELINE_VERSION=0.1.26
 kubectl apply -f https://raw.githubusercontent.com/kubeflow/pipelines/$PIPELINE_VERSION/manifests/kustomize/namespaced-install.yaml
 ```
 
-You might lack the permission to create role and command might partially fail. If so, bind your account as cluster admin and rerun the same command.
-(Or role creator in your namespace)
+Then get the Pipeline URL
 ```
-kubectl create clusterrolebinding your-binding --clusterrole=cluster-admin --user=[your-user-name]
-```
-
-When deployment is complete, you can access Kubeflow Pipelines UI by an IAM controlled public endpoint, which can be found by
-```
-kubectl describe configmap inverse-proxy-config -n kubeflow
+kubectl describe configmap inverse-proxy-config -n kubeflow | grep googleusercontent.com 
 ```
-and check the Hostname section. The endpoint should have format like **1234567-dot-datalab-vm-us-west1.googleusercontent.com**
 
-# Customization
+## Customization
 Customization can be done through Kustomize [Overlay](https://github.com/kubernetes-sigs/kustomize/blob/master/docs/glossary.md#overlay). 
 
 Note - The instruction below assume you installed kubectl v1.14.0 or later, which has native support of kustomize.
 To get latest kubectl, visit [here](https://kubernetes.io/docs/tasks/tools/install-kubectl/)
 
-## Change deploy namespace
+### Deploy on GCP with CloudSQL and GCS
+See [here](env/gcp/README.md) for more details. 
+
+### Change deploy namespace
 To deploy Kubeflow Pipelines in namespace FOO,
-- Edit [kustomization.yaml](env/dev/kustomization.yaml) namespace section to FOO
+- Edit [dev/kustomization.yaml](env/dev/kustomization.yaml) or [gcp/kustomization.yaml](env/gcp/kustomization.yaml) namespace section to FOO
 - Then run 
 ```
 kubectl kustomize env/dev | kubectl apply -f -
+# or 
+kubectl kustomize env/gcp | kubectl apply -f -
 ```
 
-## Disable the public endpoint
+### Disable the public endpoint
 By default, the deployment install an [invert proxy agent](https://github.com/google/inverting-proxy) that exposes a public URL. If you want to skip installing it,
 - Comment out the proxy component in the [kustomization.yaml](base/kustomization.yaml).
 - Then run 
@@ -51,20 +50,36 @@ and open http://localhost:8080/
 
 
 
-# Uninstall
+## Uninstall
 You can uninstall Kubeflow Pipelines by running
 ```
+export PIPELINE_VERSION=0.1.26
 kubectl delete -f https://raw.githubusercontent.com/kubeflow/pipelines/$PIPELINE_VERSION/manifests/kustomize/namespaced-install.yaml
 ```
 
 Or if you deploy through kustomize
 ```
 kubectl kustomize env/dev | kubectl delete -f -
+# or
+kubectl kustomize env/gcp | kubectl delete -f -
+```
+
+## Troubleshooting
+
+### Permission error installing Kubeflow Pipelines to a cluster
+Run 
+```
+kubectl create clusterrolebinding your-binding --clusterrole=cluster-admin --user=[your-user-name]
 ```
-# FAQ
+
+### Samples requires "user-gcp-sa" secret
 If sample code requires a "user-gcp-sa" secret, you could create one by 
-- First download the GCE VM service account token following this [instruction](https://cloud.google.com/kubernetes-engine/docs/tutorials/authenticating-to-cloud-platform#step_3_create_service_account_credentials)
+- First download the GCE VM service account token [Document](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#creating_service_account_keys)
+```
+gcloud iam service-accounts keys create application_default_credentials.json \
+  --iam-account [SA-NAME]@[PROJECT-ID].iam.gserviceaccount.com
+```
 - Run
 ```
-kubectl create secret -n [your-namespace] generic user-gcp-sa --from-file=user-gcp-sa.json=[your-token-file].json
+kubectl create secret -n [your-namespace] generic user-gcp-sa --from-file=user-gcp-sa.json=application_default_credentials.json
 ```
diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-persistenceagent-deployment.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-persistenceagent-deployment.yaml
@@ -14,7 +14,12 @@ spec:
         app: ml-pipeline-persistenceagent
     spec:
       containers:
-      - image: gcr.io/ml-pipeline/persistenceagent:0.1.15
+      - env:
+          - name: NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+        image: gcr.io/ml-pipeline/persistenceagent:0.1.15
         imagePullPolicy: IfNotPresent
         name: ml-pipeline-persistenceagent
       serviceAccountName: ml-pipeline-persistenceagent
diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-scheduledworkflow-deployment.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-scheduledworkflow-deployment.yaml
@@ -17,4 +17,9 @@ spec:
       - image: gcr.io/ml-pipeline/scheduledworkflow:0.1.15
         imagePullPolicy: IfNotPresent
         name: ml-pipeline-scheduledworkflow
+        env:
+          - name: NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
       serviceAccountName: ml-pipeline-scheduledworkflow
diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-ui-deployment.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-ui-deployment.yaml
@@ -19,4 +19,9 @@ spec:
         name: ml-pipeline-ui
         ports:
         - containerPort: 3000
+        env:
+          - name: MINIO_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
       serviceAccountName: ml-pipeline-ui
diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-viewer-crd-deployment.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-viewer-crd-deployment.yaml
@@ -20,4 +20,8 @@ spec:
         env:
         - name: MAX_NUM_VIEWERS
           value: "50"
+        - name: MINIO_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
       serviceAccountName: ml-pipeline-viewer-crd-service-account
diff --git a/manifests/kustomize/env/gcp/README.md b/manifests/kustomize/env/gcp/README.md
@@ -0,0 +1,13 @@
+# TL;DR
+1. To access the GCP services, the application needs a GCP service account token. Download the token to the current folder manifests/kustomize/env/gcp. [Reference](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#creating_service_account_keys)
+```
+gcloud iam service-accounts keys create application_default_credentials.json \
+  --iam-account [SA-NAME]@[PROJECT-ID].iam.gserviceaccount.com
+```
+2. [Create](https://cloud.google.com/sql/docs/mysql/quickstart) or use an existing CloudSQL instance. The service account should have the access to the CloudSQL instance.
+3. Fill in gcp-configurations-patch.yaml with your CloudSQL and GCS configuration.
+
+# Why Cloud SQL and GCS
+Kubeflow Pipelines keeps its metadata in mysql database and artifacts in S3 compatible object storage. 
+Using CloudSQL and GCS for persisting the data provides better reliability and performance, as well as things like data backups, and usage monitoring.
+This is the recommended setup especially for production environments.   
diff --git a/manifests/kustomize/env/gcp/gcp-configurations-patch.yaml b/manifests/kustomize/env/gcp/gcp-configurations-patch.yaml
@@ -0,0 +1,34 @@
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  name: ml-pipeline
+spec:
+  template:
+    spec:
+      containers:
+        - name: ml-pipeline-api-server
+          env:
+            - name: OBJECTSTORECONFIG_BUCKETNAME
+              # Don't add gs:// prefix.
+              # If bucket doesn't exist, the deployment will create one.
+              value: ''
+            - name: DBCONFIG_PASSWORD
+              value: ''
+---
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  name: cloudsqlproxy
+spec:
+  template:
+    spec:
+      containers:
+        - name: cloudsqlproxy
+          env:
+            - name: GCP_PROJECT
+              value: ''
+            - name: CLOUDSQL_ZONE
+              # E.g. us-central1
+              value: ''
+            - name: CLOUDSQL_INSTANCE_NAME
+              value: ''
diff --git a/manifests/kustomize/env/gcp/kustomization.yaml b/manifests/kustomize/env/gcp/kustomization.yaml
@@ -10,12 +10,12 @@ bases:
 namespace: kubeflow
 
 patchesStrategicMerge:
-  - ml-pipeline-apiserver-deployment-patch.yaml
+  - gcp-configurations-patch.yaml
 
 images:
   - name: gcr.io/cloudsql-docker/gce-proxy
     newTag: "1.14"
-  - name: minio/minio
+  - name: gcr.io/ml-pipeline/minio
     newTag: RELEASE.2019-08-14T20-37-41Z
 
 secretGenerator:

diff --git a/manifests/kustomize/env/gcp/minio/minio-gcs-gateway-deployment.yaml b/manifests/kustomize/env/gcp/minio/minio-gcs-gateway-deployment.yaml
@@ -1,8 +1,11 @@
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1beta2
 kind: Deployment
 metadata:
-  name: minio-deployment
+  name: minio
 spec:
+  selector:
+    matchLabels:
+      app: minio
   strategy:
     type: Recreate
   template:
@@ -12,12 +15,10 @@ spec:
     spec:
       containers:
         - name: minio
-          image: minio/minio:RELEASE.2019-08-14T20-37-41Z
+          image: gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z
           args:
             - gateway
             - gcs
-            # Replace this with your own GCP project
-            - yang-experiment-6
           env:
             - name: MINIO_ACCESS_KEY
               value: "minio"