kustomize kf pipeline

manifests/README.md

@@ -0,0 +1,41 @@
+# TL;DR
+If you want to skip any customization, you can deploy Kubeflow Pipelines by running
+```
+export PIPELINE_VERSION=0.1.20
+kubectl create -f https://raw.githubusercontent.com/kubeflow/pipelines/$PIPELINE_VERSION/manifests/namespaced-install.yaml
+```
+
+You might lack the permission to create role and command might partially fail. If so, bind your account as cluster admin.
+(Or role creator in your namespace)
+```
+kubectl create clusterrolebinding your-binding --clusterrole=cluster-admin --user=[your-user-name]
+```
+
+# Customization
+Customization can be done through Kustomize Overlay, and don't need to modify the base directory. 
+
+## Change deploy namespace
+This directory contains the Kustomize Manifest for deploying Kubeflow Pipelines. 
+Kustomize allows you to easily customize your deployment.
+
+To deploy Kubeflow Pipelines in namespace FOO
+- Edit [kustomization.yaml](namespaced-install/kustomization.yaml) namespace section to FOO
+- Then run 
+```
+kubectl kustomize . | kubectl apply -f -
+```
+
+## Reinstall with existing data
+TODO
+
+
+# Uninstall
+You can uninstall everything by running
+```
+kubectl delete -f https://raw.githubusercontent.com/kubeflow/pipelines/$PIPELINE_VERSION/manifests/namespaced-install.yaml
+```
+
+Or if you deploy using kustomize
+```
+kubectl kustomize . | kubectl delete -f -
+```

manifests/base/argo/kustomization.yaml

@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- workflow-controller-configmap.yaml
+- workflow-controller-deployment.yaml
+- workflow-controller-role.yaml
+- workflow-controller-rolebinding.yaml
+- workflow-controller-sa.yaml

manifests/base/argo/workflow-controller-configmap.yaml

@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: workflow-controller-configmap
+data:
+  config: |
+    {
+    executorImage: argoproj/argoexec:v2.2.0,
+    artifactRepository:
+    {
+        s3: {
+            bucket: mlpipeline,
+            keyPrefix: artifacts,
+            endpoint: minio-service.default:9000,
+            insecure: true,
+            accessKeySecret: {
+                name: mlpipeline-minio-artifact,
+                key: accesskey
+            },
+            secretKeySecret: {
+                name: mlpipeline-minio-artifact,
+                key: secretkey
+            }
+        }
+    }
+    }

manifests/base/argo/workflow-controller-deployment.yaml

@@ -0,0 +1,51 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  labels:
+    app: workflow-controller
+  name: workflow-controller
+spec:
+  progressDeadlineSeconds: 600
+  replicas: 1
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      app: workflow-controller
+  strategy:
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 25%
+    type: RollingUpdate
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app: workflow-controller
+    spec:
+      containers:
+      - args:
+        - --configmap
+        - workflow-controller-configmap
+        - --executor-image
+        - argoproj/argoexec:v2.2.0
+        command:
+        - workflow-controller
+        env:
+        - name: ARGO_NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+        image: argoproj/workflow-controller:v2.2.0
+        imagePullPolicy: IfNotPresent
+        name: workflow-controller
+        resources: {}
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+      dnsPolicy: ClusterFirst
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      securityContext: {}
+      serviceAccount: argo
+      serviceAccountName: argo
+      terminationGracePeriodSeconds: 30

manifests/base/argo/workflow-controller-role.yaml

@@ -0,0 +1,45 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: argo-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - pods/exec
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - watch
+  - list
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - create
+  - delete
+- apiGroups:
+  - argoproj.io
+  resources:
+  - workflows
+  - workflows/finalizers
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+  - delete

manifests/base/argo/workflow-controller-rolebinding.yaml

@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: argo-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: argo-role
+subjects:
+- kind: ServiceAccount
+  name: argo

manifests/base/argo/workflow-controller-sa.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: argo

manifests/base/crds/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- scheduled-workflow-crd.yaml
+- viewer-crd.yaml
+- workflow-crd.yaml

manifests/base/crds/scheduled-workflow-crd.yaml

@@ -0,0 +1,18 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: scheduledworkflows.kubeflow.org
+spec:
+  group: kubeflow.org
+  names:
+    kind: ScheduledWorkflow
+    listKind: ScheduledWorkflowList
+    plural: scheduledworkflows
+    shortNames:
+    - swf
+    singular: scheduledworkflow
+  scope: Namespaced
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true

manifests/base/crds/viewer-crd.yaml

@@ -0,0 +1,20 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  labels:
+    ksonnet.io/component: pipeline
+  name: viewers.kubeflow.org
+spec:
+  group: kubeflow.org
+  names:
+    kind: Viewer
+    listKind: ViewerList
+    plural: viewers
+    shortNames:
+    - vi
+    singular: viewer
+  scope: Namespaced
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true

manifests/base/crds/workflow-crd.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: workflows.argoproj.io
+spec:
+  group: argoproj.io
+  version: v1alpha1
+  scope: Namespaced
+  names:
+    kind: Workflow
+    plural: workflows
+    shortNames:
+    - wf

manifests/base/kustomization.yaml

@@ -0,0 +1,27 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+bases:
+- argo
+- crds
+- minio
+- mysql
+- pipeline
+
+images:
+- name: argoproj/workflow-controller
+  newTag: v2.2.0
+- name: minio/minio
+  newTag: RELEASE.2018-02-09T22-40-05Z
+- name: mysql
+  newTag: "5.6"
+- name: gcr.io/ml-pipeline/api-server
+  newTag: 0.1.20
+- name: gcr.io/ml-pipeline/persistenceagent
+  newTag: 0.1.20
+- name: gcr.io/ml-pipeline/scheduledworkflow
+  newTag: 0.1.20
+- name: gcr.io/ml-pipeline/frontend
+  newTag: 0.1.20
+- name: gcr.io/ml-pipeline/viewer-crd-controller
+  newTag: 0.1.20

manifests/base/minio/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- minio-artifact-secret.yaml
+- minio-deployment.yaml
+- minio-pvc.yaml
+- minio-service.yaml

manifests/base/minio/minio-artifact-secret.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+data:
+  accesskey: bWluaW8=
+  secretkey: bWluaW8xMjM=
+kind: Secret
+metadata:
+  name: mlpipeline-minio-artifact
+  namespace: kubeflow
+type: Opaque

manifests/base/minio/minio-deployment.yaml

@@ -0,0 +1,36 @@
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+  name: minio
+spec:
+  selector:
+    matchLabels:
+      app: minio
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: minio
+    spec:
+      containers:
+      - args:
+        - server
+        - /data
+        env:
+        - name: MINIO_ACCESS_KEY
+          value: minio
+        - name: MINIO_SECRET_KEY
+          value: minio123
+        image: minio/minio:RELEASE.2018-02-09T22-40-05Z
+        name: minio
+        ports:
+        - containerPort: 9000
+        volumeMounts:
+        - mountPath: /data
+          name: data
+          subPath: minio
+      volumes:
+      - name: data
+        persistentVolumeClaim:
+          claimName: minio-pvc

manifests/base/minio/minio-pvc.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: minio-pvc
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 20Gi

manifests/base/minio/minio-service.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: minio-service
+spec:
+  ports:
+  - port: 9000
+    protocol: TCP
+    targetPort: 9000
+  selector:
+    app: minio

manifests/base/mysql/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- mysql-deployment.yaml
+- mysql-pv-claim.yaml
+- mysql-service.yaml