[Manifest]Cache - mkp deployment (#3343)

* Initial execution cache This commit adds initial execution cache service. Including http service and execution key generation. * fix master * Add cache manifests for mkp deployment * revert go.sum * Add helm on delete policy for cache deployer job * Change cache deployer job to statefulset * remove unnecessary cluster role * seperate clusterrole and role * add role and rolebinding to mkp * change secret role to clusterrole * Add cloudsql support to cache
kubeflow · Apr 1, 2020 · b7d71b8 · b7d71b8
1 parent 62269eb
commit b7d71b8
Show file tree

Hide file tree

Showing 9 changed files with 327 additions and 66 deletions.
diff --git a/backend/src/cache/client_manager.go b/backend/src/cache/client_manager.go
@@ -30,7 +30,6 @@ import (
 )
 
 const (
-	DBName                   = "cachedb"
 	DefaultConnectionTimeout = "6m"
 )
 
@@ -131,7 +130,7 @@ func initMysql(params WhSvrDBParameters, initConnectionTimeout time.Duration) st
 	util.TerminateIfError(err)
 
 	// Create database if not exist
-	dbName := DBName
+	dbName := params.dbName
 	operation = func() error {
 		_, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", dbName))
 		if err != nil {

diff --git a/backend/src/cache/main.go b/backend/src/cache/main.go
@@ -47,6 +47,7 @@ type WhSvrDBParameters struct {
 	dbDriver            string
 	dbHost              string
 	dbPort              string
+	dbName              string
 	dbUser              string
 	dbPwd               string
 	dbGroupConcatMaxLen string
@@ -58,6 +59,7 @@ func main() {
 	flag.StringVar(&params.dbDriver, "db_driver", mysqlDBDriverDefault, "Database driver name, mysql is the default value")
 	flag.StringVar(&params.dbHost, "db_host", mysqlDBHostDefault, "Database host name.")
 	flag.StringVar(&params.dbPort, "db_port", mysqlDBPortDefault, "Database port number.")
+	flag.StringVar(&params.dbName, "db_name", "cachedb", "Database name.")
 	flag.StringVar(&params.dbUser, "db_user", "root", "Database user name.")
 	flag.StringVar(&params.dbPwd, "db_password", "", "Database password.")
 	flag.StringVar(&params.dbGroupConcatMaxLen, "db_group_concat_max_len", mysqlDBGroupConcatMaxLenDefault, "Database group concat max length.")

diff --git a/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/cache.yaml b/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/cache.yaml
@@ -0,0 +1,283 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: cache-deployer-statefulset
+  labels:
+    app: cache-deployer
+    app.kubernetes.io/name: {{ .Release.Name }}
+spec:
+  replicas: 1
+  serviceName: cache-deployer
+  selector:
+    matchLabels:
+      app: cache-deployer
+      app.kubernetes.io/name: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app: cache-deployer
+        app.kubernetes.io/name: {{ .Release.Name }}
+    spec:
+      containers:
+      - name: main
+        image: {{ .Values.images.cachedeployer }}
+        imagePullPolicy: Always
+        env:
+        - name: NAMESPACE_TO_WATCH
+          value: {{ .Release.Namespace }}
+      serviceAccountName: kubeflow-pipelines-cache-deployer-sa
+      restartPolicy: Always
+  volumeClaimTemplates: []
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app: kubeflow-pipelines-cache-deployer-clusterrole
+    app.kubernetes.io/name: {{ .Release.Name }}
+  name: kubeflow-pipelines-cache-deployer-clusterrole
+rules:
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - certificatesigningrequests
+  - certificatesigningrequests/approval
+  verbs:
+  - create
+  - delete
+  - get
+  - update
+- apiGroups:
+  - admissionregistration.k8s.io
+  resources:
+  - mutatingwebhookconfigurations
+  verbs:
+  - create
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app: kubeflow-pipelines-cache-deployer-secret-clusterrole
+    app.kubernetes.io/name: {{ .Release.Name }}
+  name: kubeflow-pipelines-cache-deployer-secret-clusterrole
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - create
+  - get
+  - patch
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kubeflow-pipelines-cache-deployer-sa
+  labels:
+    app.kubernetes.io/name: {{ .Release.Name }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kubeflow-pipelines-cache-deployer-clusterrolebinding
+  labels:
+    app.kubernetes.io/name: {{ .Release.Name }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kubeflow-pipelines-cache-deployer-clusterrole
+subjects:
+- kind: ServiceAccount
+  name: kubeflow-pipelines-cache-deployer-sa
+  namespace: {{ .Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kubeflow-pipelines-cache-deployer-rolebinding
+  labels:
+    app.kubernetes.io/name: {{ .Release.Name }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kubeflow-pipelines-cache-deployer-secret-clusterrole
+subjects:
+- kind: ServiceAccount
+  name: kubeflow-pipelines-cache-deployer-sa
+  namespace: {{ .Release.Namespace }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cache-server
+  labels:
+    app: cache-server
+    app.kubernetes.io/name: {{ .Release.Name }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: cache-server
+      app.kubernetes.io/name: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app: cache-server
+        app.kubernetes.io/name: {{ .Release.Name }}
+    spec:
+      containers:
+      - name: server
+        image: {{ .Values.images.cacheserver }}
+        env:
+          {{ if .Values.managedstorage.enabled }}
+          - name: DBCONFIG_USER
+            valueFrom:
+              secretKeyRef:
+                name: mysql-credential
+                key: username
+          - name: DBCONFIG_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: mysql-credential
+                key: password
+          {{ else }}
+          - name: DBCONFIG_USER
+            value: 'root'
+          - name: DBCONFIG_PASSWORD
+            value: ''
+          {{ end }}
+          - name: DBCONFIG_DRIVER
+            valueFrom:
+              configMapKeyRef:
+                name: cache-configmap
+                key: mysql_driver
+          - name: DBCONFIG_DB_NAME
+            valueFrom:
+              configMapKeyRef:
+                name: cache-configmap
+                key: mysql_database
+          - name: DBCONFIG_HOST_NAME
+            valueFrom:
+              configMapKeyRef:
+                name: cache-configmap
+                key: mysql_host
+          - name: DBCONFIG_PORT
+            valueFrom:
+              configMapKeyRef:
+                name: cache-configmap
+                key: mysql_port
+          - name: NAMESPACE_TO_WATCH
+            value: {{ .Release.Namespace }}
+        args: ["--db_driver=$(DBCONFIG_DRIVER)",
+               "--db_host=$(DBCONFIG_HOST_NAME)",
+               "--db_port=$(DBCONFIG_PORT)",
+               "--db_name=$(DBCONFIG_DB_NAME)"
+               "--db_user=$(DBCONFIG_USER)",
+               "--db_password=$(DBCONFIG_PASSWORD)",
+               "--namespace_to_watch=$(NAMESPACE_TO_WATCH)",
+               ]
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 8443
+          name: webhook-api
+        volumeMounts:
+        - name: webhook-tls-certs
+          mountPath: /etc/webhook/certs
+          readOnly: true
+      volumes:
+      - name: webhook-tls-certs
+        secret:
+          secretName: webhook-server-tls
+      serviceAccountName: kubeflow-pipelines-cache
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: cache-server
+  labels: 
+    app: cache-server
+    app.kubernetes.io/name: {{ .Release.Name }}
+spec:
+  selector:
+    app: cache-server
+    app.kubernetes.io/name: {{ .Release.Name }}
+  ports:
+    - port: 443
+      targetPort: webhook-api
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cache-configmap
+  labels:
+    component: cache-server
+data:
+  {{ if .Values.managedstorage.databaseNamePrefix }}
+  mysql_database: '{{ .Values.managedstorage.databaseNamePrefix }}_cachedb'
+  {{ else }}
+  mysql_database: '{{ .Release.Name | replace "-" "_" | replace "." "_"}}_cachedb'
+  {{ end }}
+  mysql_driver: "mysql"
+  mysql_host: "mysql"
+  mysql_port: "3306"
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app: kubeflow-pipelines-cache-role
+    app.kubernetes.io/name: {{ .Release.Name }}
+  name: kubeflow-pipelines-cache-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+- apiGroups:
+  - argoproj.io
+  resources:
+  - workflows
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kubeflow-pipelines-cache
+  labels:
+    app.kubernetes.io/name: {{ .Release.Name }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kubeflow-pipelines-cache-binding
+  labels:
+    app.kubernetes.io/name: {{ .Release.Name }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kubeflow-pipelines-cache-role
+subjects:
+- kind: ServiceAccount
+  name: kubeflow-pipelines-cache
+  namespace: {{ .Release.Namespace }}
+
diff --git a/manifests/gcp_marketplace/chart/kubeflow-pipelines/values.yaml b/manifests/gcp_marketplace/chart/kubeflow-pipelines/values.yaml
@@ -14,6 +14,8 @@ images:
   visualizationserver: gcr.io/ml-pipeline/google/pipelines/visualizationserver:dummy
   metadataenvoy: gcr.io/ml-pipeline/google/pipelines/metadataenvoy:dummy
   metadatawriter: gcr.io/ml-pipeline/google/pipelines/metadatawriter:dummy
+  cacheserver: gcr.io/ml-pipeline/google/pipelines/cacheserver:dummy
+  cachedeployer: gcr.io/ml-pipeline/google/pipelines/cachedeployer:dummy
 
 gcpSecretName: "user-gcp-sa"
 serviceAccountCredential: ""

diff --git a/manifests/gcp_marketplace/schema.yaml b/manifests/gcp_marketplace/schema.yaml
@@ -77,13 +77,21 @@ x-google-marketplace:
       properties:
         images.metadatawriter:
           type: FULL
+    cacheserver:
+      properties:
+        images.cacheserver:
+          type: FULL
+    cachedeployer:
+      properties:
+        images.cachedeployer:
+          type: FULL
   deployerServiceAccount:
     roles:
       - type: ClusterRole        # This is a cluster-wide ClusterRole
         rulesType: CUSTOM        # We specify our own custom RBAC roles
         rules:
-          - apiGroups: ['apiextensions.k8s.io']
-            resources: ['customresourcedefinitions']
+          - apiGroups: ['apiextensions.k8s.io', 'rbac.authorization.k8s.io']
+            resources: ['customresourcedefinitions', 'clusterroles', 'clusterrolebindings']
             verbs: ['*']
   clusterConstraints:
     resources: