UPSTREAM: <carry>: feat: mount EmptyDir volumes for launcher write lo…

…cations Launcher writes input artifacts to root paths /gcs, /minio, and /s3. These paths are not accessible by non-root users by default, which is problematic in locked-down Kubernetes installations and/or OpenShift. /gcs is currently a contract for KFP v2 python component wrappers, so the path cannot be changed. Mount an EmptyDir scratch volume to these paths to work around this. Additionally, /.local and /.cache are written to by pip, so add EmptyDir mounts for those too. Fixes: https://issues.redhat.com/browse/RHOAIENG-1889 Ref: kubeflow#5673 Ref: kubeflow#7345
gregsheremeta · Feb 20, 2024 · dfe6edc · dfe6edc
1 parent d48d383
commit dfe6edc
Showing 1 changed file with 84 additions and 18 deletions.
diff --git a/backend/src/v2/compiler/argocompiler/container.go b/backend/src/v2/compiler/argocompiler/container.go
@@ -23,11 +23,21 @@ import (
 )
 
 const (
-	volumeNameKFPLauncher = "kfp-launcher"
-	DefaultLauncherImage = "gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4"
-	LauncherImageEnvVar   = "V2_LAUNCHER_IMAGE"
-	DefaultDriverImage = "gcr.io/ml-pipeline/kfp-driver@sha256:8e60086b04d92b657898a310ca9757631d58547e76bbbb8bfc376d654bef1707"
-	DriverImageEnvVar   = "V2_DRIVER_IMAGE"
+	volumeNameKFPLauncher   = "kfp-launcher"
+	DefaultLauncherImage    = "gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4"
+	LauncherImageEnvVar     = "V2_LAUNCHER_IMAGE"
+	DefaultDriverImage      = "gcr.io/ml-pipeline/kfp-driver@sha256:8e60086b04d92b657898a310ca9757631d58547e76bbbb8bfc376d654bef1707"
+	DriverImageEnvVar       = "V2_DRIVER_IMAGE"
+	gcsScratchLocation      = "/gcs"
+	gcsScratchName          = "gcs-scratch"
+	s3ScratchLocation       = "/s3"
+	s3ScratchName           = "s3-scratch"
+	minioScratchLocation    = "/minio"
+	minioScratchName        = "minio-scratch"
+	dotLocalScratchLocation = "/.local"
+	dotLocalScratchName     = "dot-local-scratch"
+	dotCacheScratchLocation = "/.cache"
+	dotCacheScratchName     = "dot-cache-scratch"
 )
 
 func (c *workflowCompiler) Container(name string, component *pipelinespec.ComponentSpec, container *pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec) error {
@@ -238,21 +248,55 @@ func (c *workflowCompiler) addContainerExecutorTemplate() string {
 		// args come from. It is treated as a strategic merge patch on
 		// top of the Pod spec.
 		PodSpecPatch: inputValue(paramPodSpecPatch),
-		Volumes: []k8score.Volume{{
-			Name: volumeNameKFPLauncher,
-			VolumeSource: k8score.VolumeSource{
-				EmptyDir: &k8score.EmptyDirVolumeSource{},
+		Volumes: []k8score.Volume{
+			{
+				Name: volumeNameKFPLauncher,
+				VolumeSource: k8score.VolumeSource{
+					EmptyDir: &k8score.EmptyDirVolumeSource{},
+				},
 			},
-		}},
+			{
+				Name: gcsScratchName,
+				VolumeSource: k8score.VolumeSource{
+					EmptyDir: &k8score.EmptyDirVolumeSource{},
+				},
+			},
+			{
+				Name: s3ScratchName,
+				VolumeSource: k8score.VolumeSource{
+					EmptyDir: &k8score.EmptyDirVolumeSource{},
+				},
+			},
+			{
+				Name: minioScratchName,
+				VolumeSource: k8score.VolumeSource{
+					EmptyDir: &k8score.EmptyDirVolumeSource{},
+				},
+			},
+			{
+				Name: dotLocalScratchName,
+				VolumeSource: k8score.VolumeSource{
+					EmptyDir: &k8score.EmptyDirVolumeSource{},
+				},
+			},
+			{
+				Name: dotCacheScratchName,
+				VolumeSource: k8score.VolumeSource{
+					EmptyDir: &k8score.EmptyDirVolumeSource{},
+				},
+			},
+		},
 		InitContainers: []wfapi.UserContainer{{
 			Container: k8score.Container{
 				Name:    "kfp-launcher",
 				Image:   GetLauncherImage(),
 				Command: []string{"launcher-v2", "--copy", component.KFPLauncherPath},
-				VolumeMounts: []k8score.VolumeMount{{
-					Name:      volumeNameKFPLauncher,
-					MountPath: component.VolumePathKFPLauncher,
-				}},
+				VolumeMounts: []k8score.VolumeMount{
+					{
+						Name:      volumeNameKFPLauncher,
+						MountPath: component.VolumePathKFPLauncher,
+					},
+				},
 				Resources: launcherResources,
 			},
 		}},
@@ -265,10 +309,32 @@ func (c *workflowCompiler) addContainerExecutorTemplate() string {
 			// These are added to pass argo workflows linting.
 			Image:   "gcr.io/ml-pipeline/should-be-overridden-during-runtime",
 			Command: []string{"should-be-overridden-during-runtime"},
-			VolumeMounts: []k8score.VolumeMount{{
-				Name:      volumeNameKFPLauncher,
-				MountPath: component.VolumePathKFPLauncher,
-			}},
+			VolumeMounts: []k8score.VolumeMount{
+				{
+					Name:      volumeNameKFPLauncher,
+					MountPath: component.VolumePathKFPLauncher,
+				},
+				{
+					Name:      gcsScratchName,
+					MountPath: gcsScratchLocation,
+				},
+				{
+					Name:      s3ScratchName,
+					MountPath: s3ScratchLocation,
+				},
+				{
+					Name:      minioScratchName,
+					MountPath: minioScratchLocation,
+				},
+				{
+					Name:      dotLocalScratchName,
+					MountPath: dotLocalScratchLocation,
+				},
+				{
+					Name:      dotCacheScratchName,
+					MountPath: dotCacheScratchLocation,
+				},
+			},
 			EnvFrom: []k8score.EnvFromSource{metadataEnvFrom},
 			Env:     commonEnvs,
 		},