Merge pull request kubeflow#178 from HumairAK/stable

Merge v1.4.1 to Stable

backend/src/apiserver/common/config.go

@@ -42,13 +42,19 @@ const (
 	ArtifactScript                          string = "ARTIFACT_SCRIPT"
 	ArtifactImage                           string = "ARTIFACT_IMAGE"
 	ArtifactCopyStepTemplate                string = "ARTIFACT_COPY_STEP_TEMPLATE"
+	ArtifactCopyStepCABundleConfigMapName   string = "ARTIFACT_COPY_STEP_CABUNDLE_CONFIGMAP_NAME"
+	ArtifactCopyStepCABundleConfigMapKey    string = "ARTIFACT_COPY_STEP_CABUNDLE_CONFIGMAP_KEY"
+	ArtifactCopyStepCABundleMountPath       string = "ARTIFACT_COPY_STEP_CABUNDLE_MOUNTPATH"
 	InjectDefaultScript                     string = "INJECT_DEFAULT_SCRIPT"
 	ApplyTektonCustomResource               string = "APPLY_TEKTON_CUSTOM_RESOURCE"
 	TerminateStatus                         string = "TERMINATE_STATUS"
 	MoveResultsImage                        string = "MOVERESULTS_IMAGE"
 	Path4InternalResults                    string = "PATH_FOR_INTERNAL_RESULTS"
+	ObjectStoreCredentialsSecret            string = "OBJECTSTORECONFIG_CREDENTIALSSECRET"
+	ObjectStoreCredentialsAccessKeyKey      string = "OBJECTSTORECONFIG_CREDENTIALSACCESSKEYKEY"
+	ObjectStoreCredentialsSecretKeyKey      string = "OBJECTSTORECONFIG_CREDENTIALSSECRETKEYKEY"
 	ObjectStoreAccessKey                    string = "OBJECTSTORECONFIG_ACCESSKEY"
-	ObjectStoreSecretKey                    string = "OBJECTSTORECONFIG_SECRETKEY"
+	ObjectStoreSecretKey                    string = "OBJECTSTORECONFIG_SECRETACCESSKEY"
 )
 
 func IsPipelineVersionUpdatedByDefault() bool {
@@ -137,6 +143,18 @@ func IsApplyTektonCustomResource() string {
 	return GetStringConfigWithDefault(ApplyTektonCustomResource, "true")
 }
 
+func GetCABundleConfigMapName() string {
+	return GetStringConfigWithDefault(ArtifactCopyStepCABundleConfigMapName, "")
+}
+
+func GetCABundleConfigMapKey() string {
+	return GetStringConfigWithDefault(ArtifactCopyStepCABundleConfigMapKey, "")
+}
+
+func GetCABundleMountPath() string {
+	return GetStringConfigWithDefault(ArtifactCopyStepCABundleMountPath, "/etc/ssl/certs")
+}
+
 func GetPodNamespace() string {
 	return GetStringConfig(PodNamespace)
 }
@@ -145,6 +163,26 @@ func GetArtifactImage() string {
 	return GetStringConfigWithDefault(ArtifactImage, DefaultArtifactImage)
 }
 
+func GetObjectStoreAccessKey() string {
+	return GetStringConfig(ObjectStoreAccessKey)
+}
+
+func GetObjectStoreSecretKey() string {
+	return GetStringConfig(ObjectStoreSecretKey)
+}
+
+func GetObjectStoreCredentialsSecretName() string {
+	return GetStringConfigWithDefault(ObjectStoreCredentialsSecret, DefaultObjectStoreCredentialsSecret)
+}
+
+func GetObjectStoreCredentialsAccessKeyKey() string {
+	return GetStringConfigWithDefault(ObjectStoreCredentialsAccessKeyKey, DefaultObjectStoreCredentialsAccessKeyKey)
+}
+
+func GetObjectStoreCredentialsSecretKeyKey() string {
+	return GetStringConfigWithDefault(ObjectStoreCredentialsSecretKeyKey, DefaultObjectStoreCredentialsSecretKeyKey)
+}
+
 func GetMoveResultsImage() string {
 	return GetStringConfigWithDefault(MoveResultsImage, DefaultMoveResultImage)
 }

backend/src/apiserver/common/const.go

@@ -76,6 +76,12 @@ const (
 	DefaultMoveResultImage        string = "busybox"
 )
 
+const (
+	DefaultObjectStoreCredentialsSecret       string = "mlpipeline-minio-artifact"
+	DefaultObjectStoreCredentialsAccessKeyKey string = "accesskey"
+	DefaultObjectStoreCredentialsSecretKeyKey string = "secretkey"
+)
+
 const (
 	ArtifactItemsAnnotation          string = "tekton.dev/artifact_items"
 	ArtifactBucketAnnotation         string = "tekton.dev/artifact_bucket"

backend/src/apiserver/template/tekton_template.go

@@ -262,6 +262,9 @@ func (t *Tekton) injectArchivalStep(workflow util.Workflow, artifactItemsJSON ma
 		artifacts, hasArtifacts := artifactItemsJSON[task.Name]
 		archiveLogs := common.IsArchiveLogs()
 		trackArtifacts := common.IsTrackArtifacts()
+		objectStoreCredentialsSecretName := common.GetObjectStoreCredentialsSecretName()
+		objectStoreCredentialsSecretAccessKeyKey := common.GetObjectStoreCredentialsAccessKeyKey()
+		objectStoreCredentialsSecretSecretKeyKey := common.GetObjectStoreCredentialsSecretKeyKey()
 		stripEOF := common.IsStripEOF()
 		injectDefaultScript := common.IsInjectDefaultScript()
 		copyStepTemplate := common.GetCopyStepTemplate()
@@ -344,15 +347,31 @@ func (t *Tekton) injectArchivalStep(workflow util.Workflow, artifactItemsJSON ma
 					t.getObjectFieldSelector("PIPELINERUN", "metadata.labels['tekton.dev/pipelineRun']"),
 					t.getObjectFieldSelector("PODNAME", "metadata.name"),
 					t.getObjectFieldSelector("NAMESPACE", "metadata.namespace"),
-					t.getSecretKeySelector("AWS_ACCESS_KEY_ID", "mlpipeline-minio-artifact", "accesskey"),
-					t.getSecretKeySelector("AWS_SECRET_ACCESS_KEY", "mlpipeline-minio-artifact", "secretkey"),
+					t.getSecretKeySelector("AWS_ACCESS_KEY_ID", objectStoreCredentialsSecretName, objectStoreCredentialsSecretAccessKeyKey),
+					t.getSecretKeySelector("AWS_SECRET_ACCESS_KEY", objectStoreCredentialsSecretName, objectStoreCredentialsSecretSecretKeyKey),
 					t.getEnvVar("ARCHIVE_LOGS", strconv.FormatBool(archiveLogs)),
 					t.getEnvVar("TRACK_ARTIFACTS", strconv.FormatBool(trackArtifacts)),
 					t.getEnvVar("STRIP_EOF", strconv.FormatBool(stripEOF)),
 				)
 				step.Command = []string{"sh", "-c"}
 				step.Args = []string{artifactScript}
 
+				caBundleCfgMapName := common.GetCABundleConfigMapName()
+				caBundleCfgMapKey := common.GetCABundleConfigMapKey()
+				if caBundleCfgMapName != "" && caBundleCfgMapKey != "" {
+					if step.VolumeMounts == nil {
+						step.VolumeMounts = []corev1.VolumeMount{}
+					}
+					volName := "custom-ca-bundle"
+					bundleVolume := t.getConfigMapVolumeSource(volName, caBundleCfgMapName)
+					task.TaskSpec.Volumes = append(task.TaskSpec.Volumes, bundleVolume)
+					bundleVolumeMount := corev1.VolumeMount{
+						Name:      volName,
+						MountPath: fmt.Sprintf("%s/%s", common.GetCABundleMountPath(), caBundleCfgMapKey),
+						SubPath:   caBundleCfgMapKey,
+					}
+					step.VolumeMounts = append(step.VolumeMounts, bundleVolumeMount)
+				}
 				task.TaskSpec.Steps = append(task.TaskSpec.Steps, step)
 			}
 		}
@@ -441,6 +460,19 @@ func (t *Tekton) getHostPathVolumeSource(name string, path string) corev1.Volume
 	}
 }
 
+func (t *Tekton) getConfigMapVolumeSource(name string, configMapName string) corev1.Volume {
+	return corev1.Volume{
+		Name: name,
+		VolumeSource: corev1.VolumeSource{
+			ConfigMap: &corev1.ConfigMapVolumeSource{
+				LocalObjectReference: corev1.LocalObjectReference{
+					Name: configMapName,
+				},
+			},
+		},
+	}
+}
+
 func (t *Tekton) applyCustomResources(workflow util.Workflow, tektonTemplates string, namespace string) error {
 	// Create kubeClient to deploy Tekton custom task crd
 	var config *rest.Config

guides/advanced_user_guide.md

@@ -77,7 +77,7 @@ Here, the `apiVersion`, `kind`, and `name` are mandatory fields for all custom t
       - **--taskRef** (optional): Kubernetes Resource Spec for your custom task CRD. One of `--taskSpec` or `--taskRef` can be specified at a time.
         The value should be a Python Dictionary.
       - **--taskSpec** (optional): Kubernetes Resource Spec for your custom task CRD. This gets inlined in the pipeline.  One of `--taskSpec` or `--taskRef` can be specified at a time.
-        Custom task controller should support [embedded spec](https://github.com/tektoncd/pipeline/blob/main/docs/runs.md#2-specifying-the-target-custom-task-by-embedding-its-spec).
+        Custom task controller should support [embedded spec](https://github.com/tektoncd/pipeline/blob/main/docs/customruns.md#2-specifying-the-target-custom-task-by-embedding-its-spec).
         The value should be a Python Dictionary.
       - **Other arguments** (optional): Parameters for your custom task CRD inputs.
 
@@ -216,4 +216,4 @@ impact kfp-tekton backend:
   The default value for kfp-tekton deployment is `full`, which stores all TaskRuns/Runs statuses under PipelineRun's status.
   kfp-tekton backend also supports the `minimal` setting, which only records the list of TaskRuns/Runs under PipelineRun's status.
   In this case, statuses of TaskRuns/Runs only exist in their own CRs. kfp-tekton backend retrieves statuses of TaskRuns/Runs
-  from individual CR, aggregates, and stores them into the backend storage.
+  from individual CR, aggregates, and stores them into the backend storage.

sdk/python/tests/compiler/compiler_tests_e2e.py

@@ -299,9 +299,9 @@ def exit_on_error(cmd, expected_output=None):
     tkn_ver_out = exit_on_error("tkn version")
     tkn_pipeline_ver = re.search(r"^Pipeline version: (.*)$", tkn_ver_out, re.MULTILINE).group(1)
     tkn_client_ver = re.search(r"^Client version: (.*)$", tkn_ver_out, re.MULTILINE).group(1)
-    assert version.parse(TKN_PIPELINE_MIN_VERSION) <= version.parse(tkn_pipeline_ver),\
+    assert version.parse(TKN_PIPELINE_MIN_VERSION) <= version.parse(tkn_pipeline_ver), \
         "Tekton Pipeline version must be >= {}, found '{}'".format(TKN_PIPELINE_MIN_VERSION, tkn_pipeline_ver)
-    assert version.parse(TKN_CLIENT_MIN_VERSION) <= version.parse(tkn_client_ver),\
+    assert version.parse(TKN_CLIENT_MIN_VERSION) <= version.parse(tkn_client_ver), \
         "Tekton CLI version must be >= {}, found '{}'".format(TKN_CLIENT_MIN_VERSION, tkn_client_ver)