feat(deployment): marketplace - allow specifying gcs bucket directly (#…

…4112)

* feat(deployment): marketplace - allow specifying gcs bucket directly

* Switch tfx default bucket to user specified one

* Update schema description

* Update version to 0.5.1 to match marketplace expectation

* Fix gcsBucketName var

* Remove gcp secret credentials

VERSION

@@ -1 +1 @@
-1.0.0-dev.2
+0.5.1

manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/argo.yaml

@@ -146,7 +146,7 @@ data:
     artifactRepository:
     {
         s3: {
-            bucket: '{{ if .Values.managedstorage.enabled }}{{ tpl .Values.managedstorage.gcsBucketName . }}{{ else }}mlpipeline{{ end }}',
+            bucket: '{{ if .Values.managedstorage.enabled }}{{ .Values.managedstorage.gcsBucketName }}{{ else }}mlpipeline{{ end }}',
             keyPrefix: artifacts,
             endpoint: minio-service.{{ .Release.Namespace }}:9000,
             insecure: true,

manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/minio.yaml

@@ -43,7 +43,7 @@ spec:
             - name: PROJECT_ID
               valueFrom:
                 configMapKeyRef:
-                  name: {{ .Values.gcpDefaultConfigName}}
+                  name: "{{ .Values.gcpDefaultConfigName }}"
                   key: "project_id"
             - name: MINIO_ACCESS_KEY
               value: minio
@@ -58,14 +58,6 @@ spec:
           name: minio
           ports:
             - containerPort: 9000
-          volumeMounts:
-            - name: gcp-sa-token
-              mountPath: "/etc/credentials"
-              readOnly: true
-      volumes:
-        - name: gcp-sa-token
-          secret:
-            secretName: {{ .Values.gcpSecretName }}
 {{ end }}
 ---
 {{ if not .Values.managedstorage.enabled }}

manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/mysql.yaml

@@ -60,14 +60,9 @@ spec:
           volumeMounts:
             - mountPath: /cloudsql
               name: cloudsql
-            - mountPath: /credentials
-              name: gcp-sa-token
       volumes:
         - name: cloudsql
           emptyDir:
-        - name: gcp-sa-token
-          secret:
-            secretName: {{ .Values.gcpSecretName }}
 ---
 apiVersion: v1
 kind: Secret

manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/pipeline.yaml

@@ -705,20 +705,27 @@ spec:
     spec:
       containers:
         - env:
+            {{ if .Values.managedstorage.enabled }}
+            - name: HAS_DEFAULT_BUCKET
+              value: "true"
+            - name: BUCKET_NAME
+              value: "{{ .Values.managedstorage.gcsBucketName }}"
+            {{ else }}
             - name: HAS_DEFAULT_BUCKET
               valueFrom:
                 configMapKeyRef:
-                  name: {{ .Values.gcpDefaultConfigName}}
+                  name: "{{ .Values.gcpDefaultConfigName }}"
                   key: "has_default_bucket"
             - name: BUCKET_NAME
               valueFrom:
                 configMapKeyRef:
-                  name: {{ .Values.gcpDefaultConfigName}}
+                  name: "{{ .Values.gcpDefaultConfigName }}"
                   key: "bucket_name"
+            {{ end }}
             - name: PROJECT_ID
               valueFrom:
                 configMapKeyRef:
-                  name: {{ .Values.gcpDefaultConfigName}}
+                  name: "{{ .Values.gcpDefaultConfigName }}"
                   key: "project_id"
             - name: POD_NAMESPACE
               valueFrom:
@@ -730,7 +737,7 @@ spec:
               value: "false"
             {{ if .Values.managedstorage.enabled }}
             - name: OBJECTSTORECONFIG_BUCKETNAME
-              value: '{{ tpl .Values.managedstorage.gcsBucketName . }}'
+              value: "{{ .Values.managedstorage.gcsBucketName }}"
             - name: DBCONFIG_DBNAME
               {{ if .Values.managedstorage.databaseNamePrefix }}
               value: '{{ .Values.managedstorage.databaseNamePrefix }}_pipeline'

manifests/gcp_marketplace/chart/kubeflow-pipelines/values.yaml

@@ -17,23 +17,13 @@ images:
   cacheserver: gcr.io/ml-pipeline/google/pipelines/cacheserver:dummy
   cachedeployer: gcr.io/ml-pipeline/google/pipelines/cachedeployer:dummy
 
-gcpSecretName: "user-gcp-sa"
 serviceAccountCredential: ""
 gcpDefaultConfigName: "gcp-default-config"
 
 managedstorage:
   enabled: false
   cloudsqlInstanceConnectionName: null
-  # gcsBucketName should be determined by cloudsqlInstanceConnectionName to make
-  # sure user is always using a valid pair of connection name + gcs bucket name.
-  #
-  # gcsBucketName is used in two places, so I wrote a template string here that
-  # can be evaluated in each place.
-  #
-  # Name pattern:
-  # If spedify databaseNamePrefix: %{cloudsqlInstanceConnectionName}-%{truncedDatabaseNamePrefix}
-  # else: %{cloudsqlInstanceConnectionName}-%{releaseName}
-  gcsBucketName: '{{ if .Values.managedstorage.databaseNamePrefix }}{{ printf "%s-%s" .Values.managedstorage.cloudsqlInstanceConnectionName .Values.managedstorage.databaseNamePrefix | replace ":" "-" | lower | trunc 60 }}{{ else }}{{ printf "%s-%s" .Values.managedstorage.cloudsqlInstanceConnectionName .Release.Name | replace ":" "-" | lower | trunc 60 }}{{ end }}'
+  gcsBucketName: null
   databaseNamePrefix: null
   dbUsername: 'root'
   dbPassword: ''

manifests/gcp_marketplace/schema.yaml

@@ -122,8 +122,30 @@ properties:
       Using CloudSQL and GCS provides better reliability and performance,
       as well as features such as data backup and usage monitoring.
       This is the recommended option especially for production scenarios.
-      If false, the data will be stored in GCE Persistent Disk.
+
+      If yes, you must specify GCS bucket and CloudSQL instance below. Note, to
+      reserve your data while reinstalling Kubeflow Pipelines, you must use
+      exactly the same managed storage options as before.
+
+      If upgrading from managed storage Kubeflow Pipelines 0.5.1, GCS bucket
+      name was previously auto generated in the same project, it should have been
+      named like "<cloudsql instance connection name>-<database prefix or instance name>".
+
+      If false, following options will be ignored and the data will be stored in
+      GCE Persistent Disks.
     default: false
+  managedstorage.gcsBucketName:
+    type: string
+    title: Artifact storage GCS bucket (Managed storage only)
+    description: |-
+      This field must be specified if choose to use managed storage.
+      Provide the Google Cloud Storage (GCS) bucket name for an existing bucket.
+      The bucket will be used for storing the output artifacts.
+      For more details on how to create a new bucket, see https://cloud.google.com/storage/docs/creating-buckets.
+
+      Warning, if provided bucket doesn't exist, deployer will automatically
+      create a bucket for you in us-central1, but you won't be able to customize
+      any parameters.
   managedstorage.cloudsqlInstanceConnectionName:
     type: string
     title: Cloud SQL instance connection name (Managed storage only)