Merge pull request kubeflow#122 from gmfrasca/cherrypick-feat-custom-…

…objstore-creds

feat(backend): Source ObjStore Creds from Env in Tekton Template

backend/src/apiserver/common/config.go

@@ -47,8 +47,11 @@ const (
 	TerminateStatus                         string = "TERMINATE_STATUS"
 	MoveResultsImage                        string = "MOVERESULTS_IMAGE"
 	Path4InternalResults                    string = "PATH_FOR_INTERNAL_RESULTS"
+	ObjectStoreCredentialsSecret            string = "OBJECTSTORECONFIG_CREDENTIALSSECRET"
+	ObjectStoreCredentialsAccessKeyKey      string = "OBJECTSTORECONFIG_CREDENTIALSACCESSKEYKEY"
+	ObjectStoreCredentialsSecretKeyKey      string = "OBJECTSTORECONFIG_CREDENTIALSSECRETKEYKEY"
 	ObjectStoreAccessKey                    string = "OBJECTSTORECONFIG_ACCESSKEY"
-	ObjectStoreSecretKey                    string = "OBJECTSTORECONFIG_SECRETKEY"
+	ObjectStoreSecretKey                    string = "OBJECTSTORECONFIG_SECRETACCESSKEY"
 )
 
 func IsPipelineVersionUpdatedByDefault() bool {
@@ -145,6 +148,26 @@ func GetArtifactImage() string {
 	return GetStringConfigWithDefault(ArtifactImage, DefaultArtifactImage)
 }
 
+func GetObjectStoreAccessKey() string {
+	return GetStringConfig(ObjectStoreAccessKey)
+}
+
+func GetObjectStoreSecretKey() string {
+	return GetStringConfig(ObjectStoreSecretKey)
+}
+
+func GetObjectStoreCredentialsSecretName() string {
+	return GetStringConfigWithDefault(ObjectStoreCredentialsSecret, DefaultObjectStoreCredentialsSecret)
+}
+
+func GetObjectStoreCredentialsAccessKeyKey() string {
+	return GetStringConfigWithDefault(ObjectStoreCredentialsAccessKeyKey, DefaultObjectStoreCredentialsAccessKeyKey)
+}
+
+func GetObjectStoreCredentialsSecretKeyKey() string {
+	return GetStringConfigWithDefault(ObjectStoreCredentialsSecretKeyKey, DefaultObjectStoreCredentialsSecretKeyKey)
+}
+
 func GetMoveResultsImage() string {
 	return GetStringConfigWithDefault(MoveResultsImage, DefaultMoveResultImage)
 }

backend/src/apiserver/common/const.go

@@ -76,6 +76,12 @@ const (
 	DefaultMoveResultImage        string = "busybox"
 )
 
+const (
+	DefaultObjectStoreCredentialsSecret       string = "mlpipeline-minio-artifact"
+	DefaultObjectStoreCredentialsAccessKeyKey string = "accesskey"
+	DefaultObjectStoreCredentialsSecretKeyKey string = "secretkey"
+)
+
 const (
 	ArtifactItemsAnnotation          string = "tekton.dev/artifact_items"
 	ArtifactBucketAnnotation         string = "tekton.dev/artifact_bucket"

backend/src/apiserver/template/tekton_template.go

@@ -258,6 +258,9 @@ func (t *Tekton) injectArchivalStep(workflow util.Workflow, artifactItemsJSON ma
 		artifacts, hasArtifacts := artifactItemsJSON[task.Name]
 		archiveLogs := common.IsArchiveLogs()
 		trackArtifacts := common.IsTrackArtifacts()
+		objectStoreCredentialsSecretName := common.GetObjectStoreCredentialsSecretName()
+		objectStoreCredentialsSecretAccessKeyKey := common.GetObjectStoreCredentialsAccessKeyKey()
+		objectStoreCredentialsSecretSecretKeyKey := common.GetObjectStoreCredentialsSecretKeyKey()
 		stripEOF := common.IsStripEOF()
 		injectDefaultScript := common.IsInjectDefaultScript()
 		copyStepTemplate := common.GetCopyStepTemplate()
@@ -340,8 +343,8 @@ func (t *Tekton) injectArchivalStep(workflow util.Workflow, artifactItemsJSON ma
 					t.getObjectFieldSelector("PIPELINERUN", "metadata.labels['tekton.dev/pipelineRun']"),
 					t.getObjectFieldSelector("PODNAME", "metadata.name"),
 					t.getObjectFieldSelector("NAMESPACE", "metadata.namespace"),
-					t.getSecretKeySelector("AWS_ACCESS_KEY_ID", "mlpipeline-minio-artifact", "accesskey"),
-					t.getSecretKeySelector("AWS_SECRET_ACCESS_KEY", "mlpipeline-minio-artifact", "secretkey"),
+					t.getSecretKeySelector("AWS_ACCESS_KEY_ID", objectStoreCredentialsSecretName, objectStoreCredentialsSecretAccessKeyKey),
+					t.getSecretKeySelector("AWS_SECRET_ACCESS_KEY", objectStoreCredentialsSecretName, objectStoreCredentialsSecretSecretKeyKey),
 					t.getEnvVar("ARCHIVE_LOGS", strconv.FormatBool(archiveLogs)),
 					t.getEnvVar("TRACK_ARTIFACTS", strconv.FormatBool(trackArtifacts)),
 					t.getEnvVar("STRIP_EOF", strconv.FormatBool(stripEOF)),

guides/advanced_user_guide.md

@@ -77,7 +77,7 @@ Here, the `apiVersion`, `kind`, and `name` are mandatory fields for all custom t
       - **--taskRef** (optional): Kubernetes Resource Spec for your custom task CRD. One of `--taskSpec` or `--taskRef` can be specified at a time.
         The value should be a Python Dictionary.
       - **--taskSpec** (optional): Kubernetes Resource Spec for your custom task CRD. This gets inlined in the pipeline.  One of `--taskSpec` or `--taskRef` can be specified at a time.
-        Custom task controller should support [embedded spec](https://github.com/tektoncd/pipeline/blob/main/docs/runs.md#2-specifying-the-target-custom-task-by-embedding-its-spec).
+        Custom task controller should support [embedded spec](https://github.com/tektoncd/pipeline/blob/main/docs/customruns.md#2-specifying-the-target-custom-task-by-embedding-its-spec).
         The value should be a Python Dictionary.
       - **Other arguments** (optional): Parameters for your custom task CRD inputs.
 
@@ -216,4 +216,4 @@ impact kfp-tekton backend:
   The default value for kfp-tekton deployment is `full`, which stores all TaskRuns/Runs statuses under PipelineRun's status.
   kfp-tekton backend also supports the `minimal` setting, which only records the list of TaskRuns/Runs under PipelineRun's status.
   In this case, statuses of TaskRuns/Runs only exist in their own CRs. kfp-tekton backend retrieves statuses of TaskRuns/Runs
-  from individual CR, aggregates, and stores them into the backend storage.
+  from individual CR, aggregates, and stores them into the backend storage.

sdk/python/tests/compiler/compiler_tests_e2e.py

@@ -299,9 +299,9 @@ def exit_on_error(cmd, expected_output=None):
     tkn_ver_out = exit_on_error("tkn version")
     tkn_pipeline_ver = re.search(r"^Pipeline version: (.*)$", tkn_ver_out, re.MULTILINE).group(1)
     tkn_client_ver = re.search(r"^Client version: (.*)$", tkn_ver_out, re.MULTILINE).group(1)
-    assert version.parse(TKN_PIPELINE_MIN_VERSION) <= version.parse(tkn_pipeline_ver),\
+    assert version.parse(TKN_PIPELINE_MIN_VERSION) <= version.parse(tkn_pipeline_ver), \
         "Tekton Pipeline version must be >= {}, found '{}'".format(TKN_PIPELINE_MIN_VERSION, tkn_pipeline_ver)
-    assert version.parse(TKN_CLIENT_MIN_VERSION) <= version.parse(tkn_client_ver),\
+    assert version.parse(TKN_CLIENT_MIN_VERSION) <= version.parse(tkn_client_ver), \
         "Tekton CLI version must be >= {}, found '{}'".format(TKN_CLIENT_MIN_VERSION, tkn_client_ver)