From 8d492002883653d4b93c2f2978519a24be1473f6 Mon Sep 17 00:00:00 2001 From: Giulio Frasca Date: Tue, 23 May 2023 17:15:25 -0400 Subject: [PATCH 1/3] feat(backend): Source ObjStore Creds from Env in Tekton Template --- backend/src/apiserver/common/config.go | 25 ++++++++++++++++++- backend/src/apiserver/common/const.go | 6 +++++ .../src/apiserver/template/tekton_template.go | 7 ++++-- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/backend/src/apiserver/common/config.go b/backend/src/apiserver/common/config.go index 95d892aecf3..9ea3e685567 100644 --- a/backend/src/apiserver/common/config.go +++ b/backend/src/apiserver/common/config.go @@ -47,8 +47,11 @@ const ( TerminateStatus string = "TERMINATE_STATUS" MoveResultsImage string = "MOVERESULTS_IMAGE" Path4InternalResults string = "PATH_FOR_INTERNAL_RESULTS" + ObjectStoreCredentialsSecret string = "OBJECTSTORECONFIG_CREDENTIALSSECRET" + ObjectStoreCredentialsAccessKeyKey string = "OBJECTSTORECONFIG_CREDENTIALSACCESSKEYKEY" + ObjectStoreCredentialsSecretKeyKey string = "OBJECTSTORECONFIG_CREDENTIALSSECRETKEYKEY" ObjectStoreAccessKey string = "OBJECTSTORECONFIG_ACCESSKEY" - ObjectStoreSecretKey string = "OBJECTSTORECONFIG_SECRETKEY" + ObjectStoreSecretKey string = "OBJECTSTORECONFIG_SECRETACCESSKEY" ) func IsPipelineVersionUpdatedByDefault() bool { @@ -145,6 +148,26 @@ func GetArtifactImage() string { return GetStringConfigWithDefault(ArtifactImage, DefaultArtifactImage) } +func GetObjectStoreAccessKey() string { + return GetStringConfig(ObjectStoreAccessKey) +} + +func GetObjectStoreSecretKey() string { + return GetStringConfig(ObjectStoreSecretKey) +} + +func GetObjectStoreCredentialsSecretName() string { + return GetStringConfigWithDefault(ObjectStoreCredentialsSecret, DefaultObjectStoreCredentialsSecret) +} + +func GetObjectStoreCredentialsAccessKeyKey() string { + return GetStringConfigWithDefault(ObjectStoreCredentialsAccessKeyKey, DefaultObjectStoreCredentialsAccessKeyKey) +} + +func GetObjectStoreCredentialsSecretKeyKey() string { + return GetStringConfigWithDefault(ObjectStoreCredentialsSecretKeyKey, DefaultObjectStoreCredentialsSecretKeyKey) +} + func GetMoveResultsImage() string { return GetStringConfigWithDefault(MoveResultsImage, DefaultMoveResultImage) } diff --git a/backend/src/apiserver/common/const.go b/backend/src/apiserver/common/const.go index 611c617db58..adc3b043f2e 100644 --- a/backend/src/apiserver/common/const.go +++ b/backend/src/apiserver/common/const.go @@ -76,6 +76,12 @@ const ( DefaultMoveResultImage string = "busybox" ) +const ( + DefaultObjectStoreCredentialsSecret string = "mlpipeline-minio-artifact" + DefaultObjectStoreCredentialsAccessKeyKey string = "accesskey" + DefaultObjectStoreCredentialsSecretKeyKey string = "secretkey" +) + const ( ArtifactItemsAnnotation string = "tekton.dev/artifact_items" ArtifactBucketAnnotation string = "tekton.dev/artifact_bucket" diff --git a/backend/src/apiserver/template/tekton_template.go b/backend/src/apiserver/template/tekton_template.go index 4fab567cef1..514236f39db 100644 --- a/backend/src/apiserver/template/tekton_template.go +++ b/backend/src/apiserver/template/tekton_template.go @@ -258,6 +258,9 @@ func (t *Tekton) injectArchivalStep(workflow util.Workflow, artifactItemsJSON ma artifacts, hasArtifacts := artifactItemsJSON[task.Name] archiveLogs := common.IsArchiveLogs() trackArtifacts := common.IsTrackArtifacts() + objectStoreCredentialsSecretName := common.GetObjectStoreCredentialsSecretName() + objectStoreCredentialsSecretAccessKeyKey := common.GetObjectStoreCredentialsAccessKeyKey() + objectStoreCredentialsSecretSecretKeyKey := common.GetObjectStoreCredentialsSecretKeyKey() stripEOF := common.IsStripEOF() injectDefaultScript := common.IsInjectDefaultScript() copyStepTemplate := common.GetCopyStepTemplate() @@ -340,8 +343,8 @@ func (t *Tekton) injectArchivalStep(workflow util.Workflow, artifactItemsJSON ma t.getObjectFieldSelector("PIPELINERUN", "metadata.labels['tekton.dev/pipelineRun']"), t.getObjectFieldSelector("PODNAME", "metadata.name"), t.getObjectFieldSelector("NAMESPACE", "metadata.namespace"), - t.getSecretKeySelector("AWS_ACCESS_KEY_ID", "mlpipeline-minio-artifact", "accesskey"), - t.getSecretKeySelector("AWS_SECRET_ACCESS_KEY", "mlpipeline-minio-artifact", "secretkey"), + t.getSecretKeySelector("AWS_ACCESS_KEY_ID", objectStoreCredentialsSecretName, objectStoreCredentialsSecretAccessKeyKey), + t.getSecretKeySelector("AWS_SECRET_ACCESS_KEY", objectStoreCredentialsSecretName, objectStoreCredentialsSecretSecretKeyKey), t.getEnvVar("ARCHIVE_LOGS", strconv.FormatBool(archiveLogs)), t.getEnvVar("TRACK_ARTIFACTS", strconv.FormatBool(trackArtifacts)), t.getEnvVar("STRIP_EOF", strconv.FormatBool(stripEOF)), From b4f3d4e72522b65ebb549171520fe4da1db621c1 Mon Sep 17 00:00:00 2001 From: Giulio Frasca Date: Fri, 25 Aug 2023 19:02:28 -0400 Subject: [PATCH 2/3] Fix linting issue in compiler test --- sdk/python/tests/compiler/compiler_tests_e2e.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/tests/compiler/compiler_tests_e2e.py b/sdk/python/tests/compiler/compiler_tests_e2e.py index 2ee282f958a..b736c60d5ca 100644 --- a/sdk/python/tests/compiler/compiler_tests_e2e.py +++ b/sdk/python/tests/compiler/compiler_tests_e2e.py @@ -299,9 +299,9 @@ def exit_on_error(cmd, expected_output=None): tkn_ver_out = exit_on_error("tkn version") tkn_pipeline_ver = re.search(r"^Pipeline version: (.*)$", tkn_ver_out, re.MULTILINE).group(1) tkn_client_ver = re.search(r"^Client version: (.*)$", tkn_ver_out, re.MULTILINE).group(1) - assert version.parse(TKN_PIPELINE_MIN_VERSION) <= version.parse(tkn_pipeline_ver),\ + assert version.parse(TKN_PIPELINE_MIN_VERSION) <= version.parse(tkn_pipeline_ver), \ "Tekton Pipeline version must be >= {}, found '{}'".format(TKN_PIPELINE_MIN_VERSION, tkn_pipeline_ver) - assert version.parse(TKN_CLIENT_MIN_VERSION) <= version.parse(tkn_client_ver),\ + assert version.parse(TKN_CLIENT_MIN_VERSION) <= version.parse(tkn_client_ver), \ "Tekton CLI version must be >= {}, found '{}'".format(TKN_CLIENT_MIN_VERSION, tkn_client_ver) From f075e5e94b98fdd2bdef879d1f49ddc55a85ad0a Mon Sep 17 00:00:00 2001 From: Giulio Frasca Date: Fri, 25 Aug 2023 19:07:51 -0400 Subject: [PATCH 3/3] Update broken link in advanced user guide --- guides/advanced_user_guide.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/guides/advanced_user_guide.md b/guides/advanced_user_guide.md index 34c7cd10ae5..0e70785b0ca 100644 --- a/guides/advanced_user_guide.md +++ b/guides/advanced_user_guide.md @@ -77,7 +77,7 @@ Here, the `apiVersion`, `kind`, and `name` are mandatory fields for all custom t - **--taskRef** (optional): Kubernetes Resource Spec for your custom task CRD. One of `--taskSpec` or `--taskRef` can be specified at a time. The value should be a Python Dictionary. - **--taskSpec** (optional): Kubernetes Resource Spec for your custom task CRD. This gets inlined in the pipeline. One of `--taskSpec` or `--taskRef` can be specified at a time. - Custom task controller should support [embedded spec](https://github.com/tektoncd/pipeline/blob/main/docs/runs.md#2-specifying-the-target-custom-task-by-embedding-its-spec). + Custom task controller should support [embedded spec](https://github.com/tektoncd/pipeline/blob/main/docs/customruns.md#2-specifying-the-target-custom-task-by-embedding-its-spec). The value should be a Python Dictionary. - **Other arguments** (optional): Parameters for your custom task CRD inputs. @@ -216,4 +216,4 @@ impact kfp-tekton backend: The default value for kfp-tekton deployment is `full`, which stores all TaskRuns/Runs statuses under PipelineRun's status. kfp-tekton backend also supports the `minimal` setting, which only records the list of TaskRuns/Runs under PipelineRun's status. In this case, statuses of TaskRuns/Runs only exist in their own CRs. kfp-tekton backend retrieves statuses of TaskRuns/Runs - from individual CR, aggregates, and stores them into the backend storage. \ No newline at end of file + from individual CR, aggregates, and stores them into the backend storage.