[Backend] Patch default bucket name and project ID

backend/src/apiserver/main.go

@@ -40,6 +40,12 @@ import (
 	"google.golang.org/grpc/reflection"
 )
 
+const (
+	HasDefaultBucketEnvVar  = "HASDEFAULTBUCKET"
+	ProjectIDEnvVar         = "PROJECTID"
+	DefaultBucketNameEnvVar = "BUCKETNAME"
+)
+
 var (
 	rpcPortFlag      = flag.String("rpcPortFlag", ":8887", "RPC Port")
 	httpPortFlag     = flag.String("httpPortFlag", ":8888", "Http Proxy Port")
@@ -189,6 +195,24 @@ func loadSamples(resourceManager *resource.ResourceManager) error {
 		if configErr != nil {
 			return fmt.Errorf("Failed to decompress the file %s. Error: %v", config.Name, configErr)
 		}
+		// Patch the default bucket name read from ConfigMap
+		if common.GetBoolConfigWithDefault(HasDefaultBucketEnvVar, false) {
+			defaultBucket := common.GetStringConfig(DefaultBucketNameEnvVar)
+			projectId := common.GetStringConfig(ProjectIDEnvVar)
+			patchMap := map[string]string{
+				"<your-gcs-bucket>": defaultBucket,
+				"<your-project-id>": projectId,
+			}
+			if config.Name == "[Sample] ML - XGBoost - Training with Confusion Matrix" {
+				pipelineFile, err := server.PatchPipelineDefaultParameter(pipelineFile, patchMap)
+			}
+			if config.Name == "[Sample] Unified DSL - Taxi Tip Prediction Model Trainer" {
+				pipelineFile, err := server.PatchPipelineDefaultParameter(pipelineFile, patchMap)
+			}
+			if err != nil {
+				return fmt.Errorf("Failed to patch default value to %s. Error: %v", config.Name, err)
+			}
+		}
 		_, configErr = resourceManager.CreatePipeline(config.Name, config.Description, pipelineFile)
 		if configErr != nil {
 			// Log the error but not fail. The API Server pod can restart and it could potentially cause name collision.

backend/src/apiserver/server/util.go

@@ -182,6 +182,18 @@ func ReadPipelineFile(fileName string, fileReader io.Reader, maxFileLength int)
 	return processedFile, nil
 }
 
+// Mutate default values of specified pipeline file.
+// Args:
+//  file: pipeline file in bytes.
+//  toPatch: mapping from the old value to its new value.
+func PatchPipelineDefaultParameter(file []byte, toPatch map[string]string) ([]byte, error) {
+  pipelineRawString := string(file)
+  for key, value := range toPatch {
+    strings.Replace(pipelineRawString, key, value)
+  }
+  return []byte(pipelineRawString), nil
+}
+
 func printParameters(params []*api.Parameter) string {
 	var s strings.Builder
 	for _, p := range params {

manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/pipeline.yaml

@@ -639,6 +639,21 @@ spec:
     spec:
       containers:
         - env:
+            - name: HASDEFAULTBUCKET
+              valueFrom:
+                configMapKeyRef:
+                  name: {{ .Value.gcpDefaultConfigName}}
+                  key: "has_default_bucket"
+            - name: BUCKETNAME
+              valueFrom:
+                configMapKeyRef:
+                  name: {{ .Value.gcpDefaultConfigName}}
+                  key: "bucket_name"
+            - name: PROJECTID
+              valueFrom:
+                configMapKeyRef:
+                  name: {{ .Value.gcpDefaultConfigName}}
+                  key: "project_id"
             - name: POD_NAMESPACE
               valueFrom:
                 fieldRef:

manifests/gcp_marketplace/chart/kubeflow-pipelines/values.yaml

@@ -17,6 +17,7 @@ images:
 
 gcpSecretName: "user-gcp-sa"
 serviceAccountCredential: ""
+gcpDefaultConfigName: "gcp-default-config"
 
 managedstorage:
   enabled: false

samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py

@@ -49,7 +49,7 @@
 
 # Path of pipeline root, should be a GCS path.
 pipeline_root = os.path.join(
-    'gs://your-bucket', 'tfx_taxi_simple', kfp.dsl.RUN_ID_PLACEHOLDER
+    'gs://<your-gcs-bucket>', 'tfx_taxi_simple', kfp.dsl.RUN_ID_PLACEHOLDER
 )
 
 

samples/core/xgboost_training_cm/xgboost_training_cm.py

@@ -202,8 +202,8 @@ def dataproc_predict_op(
     description='A trainer that does end-to-end distributed training for XGBoost models.'
 )
 def xgb_train_pipeline(
-    output='gs://your-gcs-bucket',
-    project='your-gcp-project',
+    output='gs://<your-gcs-bucket>',
+    project='<your-project-id>',
     cluster_name='xgb-%s' % dsl.RUN_ID_PLACEHOLDER,
     region='us-central1',
     train_data='gs://ml-pipeline-playground/sfpd/train.csv',