diff --git a/backend/src/v2/driver/driver.go b/backend/src/v2/driver/driver.go index a150cb40d87..8203ccab5e2 100644 --- a/backend/src/v2/driver/driver.go +++ b/backend/src/v2/driver/driver.go @@ -534,6 +534,39 @@ func extendPodSpecPatch( } } + // Get config map mount information + for _, configMapAsVolume := range kubernetesExecutorConfig.GetConfigMapAsVolume() { + configMapVolume := k8score.Volume{ + Name: configMapAsVolume.GetConfigMapName(), + VolumeSource: k8score.VolumeSource{ + ConfigMap: &k8score.ConfigMapVolumeSource{ + LocalObjectReference: k8score.LocalObjectReference{Name: configMapAsVolume.GetConfigMapName()}}, + }, + } + configMapVolumeMount := k8score.VolumeMount{ + Name: configMapAsVolume.GetConfigMapName(), + MountPath: configMapAsVolume.GetMountPath(), + } + podSpec.Volumes = append(podSpec.Volumes, configMapVolume) + podSpec.Containers[0].VolumeMounts = append(podSpec.Containers[0].VolumeMounts, configMapVolumeMount) + } + + // Get config map env information + for _, configMapAsEnv := range kubernetesExecutorConfig.GetConfigMapAsEnv() { + for _, keyToEnv := range configMapAsEnv.GetKeyToEnv() { + configMapEnvVar := k8score.EnvVar{ + Name: keyToEnv.GetEnvVar(), + ValueFrom: &k8score.EnvVarSource{ + ConfigMapKeyRef: &k8score.ConfigMapKeySelector{ + Key: keyToEnv.GetConfigMapKey(), + }, + }, + } + configMapEnvVar.ValueFrom.ConfigMapKeyRef.LocalObjectReference.Name = configMapAsEnv.GetConfigMapName() + podSpec.Containers[0].Env = append(podSpec.Containers[0].Env, configMapEnvVar) + } + } + // Get image pull secret information for _, imagePullSecret := range kubernetesExecutorConfig.GetImagePullSecret() { podSpec.ImagePullSecrets = append(podSpec.ImagePullSecrets, k8score.LocalObjectReference{Name: imagePullSecret.GetSecretName()}) diff --git a/backend/src/v2/driver/driver_test.go b/backend/src/v2/driver/driver_test.go index acf8d2ed356..fdad05d24e8 100644 --- a/backend/src/v2/driver/driver_test.go +++ b/backend/src/v2/driver/driver_test.go @@ -606,6 +606,123 @@ func Test_extendPodSpecPatch_Secret(t *testing.T) { } } +func Test_extendPodSpecPatch_ConfigMap(t *testing.T) { + tests := []struct { + name string + k8sExecCfg *kubernetesplatform.KubernetesExecutorConfig + podSpec *k8score.PodSpec + expected *k8score.PodSpec + }{ + { + "Valid - config map as volume", + &kubernetesplatform.KubernetesExecutorConfig{ + ConfigMapAsVolume: []*kubernetesplatform.ConfigMapAsVolume{ + { + ConfigMapName: "cm1", + MountPath: "/data/path", + }, + }, + }, + &k8score.PodSpec{ + Containers: []k8score.Container{ + { + Name: "main", + }, + }, + }, + &k8score.PodSpec{ + Containers: []k8score.Container{ + { + Name: "main", + VolumeMounts: []k8score.VolumeMount{ + { + Name: "cm1", + MountPath: "/data/path", + }, + }, + }, + }, + Volumes: []k8score.Volume{ + { + Name: "cm1", + VolumeSource: k8score.VolumeSource{ + ConfigMap: &k8score.ConfigMapVolumeSource{ + LocalObjectReference: k8score.LocalObjectReference{Name: "cm1"}}, + }, + }, + }, + }, + }, + { + "Valid - config map not specified", + &kubernetesplatform.KubernetesExecutorConfig{}, + &k8score.PodSpec{ + Containers: []k8score.Container{ + { + Name: "main", + }, + }, + }, + &k8score.PodSpec{ + Containers: []k8score.Container{ + { + Name: "main", + }, + }, + }, + }, + { + "Valid - config map as env", + &kubernetesplatform.KubernetesExecutorConfig{ + ConfigMapAsEnv: []*kubernetesplatform.ConfigMapAsEnv{ + { + ConfigMapName: "my-cm", + KeyToEnv: []*kubernetesplatform.ConfigMapAsEnv_ConfigMapKeyToEnvMap{ + { + ConfigMapKey: "foo", + EnvVar: "CONFIG_MAP_VAR", + }, + }, + }, + }, + }, + &k8score.PodSpec{ + Containers: []k8score.Container{ + { + Name: "main", + }, + }, + }, + &k8score.PodSpec{ + Containers: []k8score.Container{ + { + Name: "main", + Env: []k8score.EnvVar{ + { + Name: "CONFIG_MAP_VAR", + ValueFrom: &k8score.EnvVarSource{ + ConfigMapKeyRef: &k8score.ConfigMapKeySelector{ + k8score.LocalObjectReference{Name: "my-cm"}, + "foo", + nil, + }, + }, + }, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := extendPodSpecPatch(tt.podSpec, tt.k8sExecCfg, nil, nil) + assert.Nil(t, err) + assert.Equal(t, tt.expected, tt.podSpec) + }) + } +} + func Test_extendPodSpecPatch_ImagePullSecrets(t *testing.T) { tests := []struct { name string diff --git a/backend/third_party_licenses/apiserver.csv b/backend/third_party_licenses/apiserver.csv index 61f8aa78c4e..17024d98bf3 100644 --- a/backend/third_party_licenses/apiserver.csv +++ b/backend/third_party_licenses/apiserver.csv @@ -61,7 +61,7 @@ github.com/klauspost/cpuid,https://github.com/klauspost/cpuid/blob/v1.3.1/LICENS github.com/klauspost/pgzip,https://github.com/klauspost/pgzip/blob/v1.2.5/LICENSE,MIT github.com/kubeflow/pipelines/api/v2alpha1/go,https://github.com/kubeflow/pipelines/blob/758c91f76784/api/LICENSE,Apache-2.0 github.com/kubeflow/pipelines/backend,https://github.com/kubeflow/pipelines/blob/HEAD/LICENSE,Apache-2.0 -github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform,https://github.com/kubeflow/pipelines/blob/e129b0501379/kubernetes_platform/LICENSE,Apache-2.0 +github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform,https://github.com/kubeflow/pipelines/blob/2983a7d49078/kubernetes_platform/LICENSE,Apache-2.0 github.com/kubeflow/pipelines/third_party/ml-metadata/go/ml_metadata,https://github.com/kubeflow/pipelines/blob/e1f0c010f800/third_party/ml-metadata/LICENSE,Apache-2.0 github.com/lann/builder,https://github.com/lann/builder/blob/47ae307949d0/LICENSE,MIT github.com/lann/ps,https://github.com/lann/ps/blob/62de8c46ede0/LICENSE,MIT diff --git a/backend/third_party_licenses/driver.csv b/backend/third_party_licenses/driver.csv index 0cd11345fff..07ea9be357e 100644 --- a/backend/third_party_licenses/driver.csv +++ b/backend/third_party_licenses/driver.csv @@ -31,7 +31,7 @@ github.com/josharian/intern,https://github.com/josharian/intern/blob/v1.0.0/lice github.com/json-iterator/go,https://github.com/json-iterator/go/blob/v1.1.12/LICENSE,MIT github.com/kubeflow/pipelines/api/v2alpha1/go,https://github.com/kubeflow/pipelines/blob/758c91f76784/api/LICENSE,Apache-2.0 github.com/kubeflow/pipelines/backend,https://github.com/kubeflow/pipelines/blob/HEAD/LICENSE,Apache-2.0 -github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform,https://github.com/kubeflow/pipelines/blob/e129b0501379/kubernetes_platform/LICENSE,Apache-2.0 +github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform,https://github.com/kubeflow/pipelines/blob/2983a7d49078/kubernetes_platform/LICENSE,Apache-2.0 github.com/kubeflow/pipelines/third_party/ml-metadata/go/ml_metadata,https://github.com/kubeflow/pipelines/blob/e1f0c010f800/third_party/ml-metadata/LICENSE,Apache-2.0 github.com/mailru/easyjson,https://github.com/mailru/easyjson/blob/v0.7.7/LICENSE,MIT github.com/modern-go/concurrent,https://github.com/modern-go/concurrent/blob/bacd9c7ef1dd/LICENSE,Apache-2.0 diff --git a/go.mod b/go.mod index 18d0eeeec0a..746d905c10f 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.4 // indirect github.com/kubeflow/pipelines/api v0.0.0-20230331215358-758c91f76784 - github.com/kubeflow/pipelines/kubernetes_platform v0.0.0-20240216222951-e129b0501379 + github.com/kubeflow/pipelines/kubernetes_platform v0.0.0-20240222213131-2983a7d49078 github.com/kubeflow/pipelines/third_party/ml-metadata v0.0.0-20230810215105-e1f0c010f800 github.com/lestrrat-go/strftime v1.0.4 github.com/mattn/go-sqlite3 v1.14.16 diff --git a/go.sum b/go.sum index 84ed4eadd08..4ad6032ef9a 100644 --- a/go.sum +++ b/go.sum @@ -936,8 +936,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/ktrysmt/go-bitbucket v0.9.32/go.mod h1:FWxy2UK7GlK5b0NSJGc5hPqnssVlkNnsChvyuOf/Xno= github.com/kubeflow/pipelines/api v0.0.0-20230331215358-758c91f76784 h1:ZVCoqnKnC2vctD7AqAHbWf05qw15VO5XSxCqkjObwtw= github.com/kubeflow/pipelines/api v0.0.0-20230331215358-758c91f76784/go.mod h1:T7TOQB36gGe97yUdfVAnYK5uuT0+uQbLNHDUHxYkmE4= -github.com/kubeflow/pipelines/kubernetes_platform v0.0.0-20240216222951-e129b0501379 h1:yUdN1NDKYYztsB+JzNXJnvNO2g1vqGFgVwIQHd8P33s= -github.com/kubeflow/pipelines/kubernetes_platform v0.0.0-20240216222951-e129b0501379/go.mod h1:CJkKr356RlpZP/gQRuHf3Myrn1qJtoUVe4EMCmtwarg= +github.com/kubeflow/pipelines/kubernetes_platform v0.0.0-20240222213131-2983a7d49078 h1:+XJ0wE7OFzE80jWHan75Q+gJU0SYxqhfEDfAr+wwZ2M= +github.com/kubeflow/pipelines/kubernetes_platform v0.0.0-20240222213131-2983a7d49078/go.mod h1:CJkKr356RlpZP/gQRuHf3Myrn1qJtoUVe4EMCmtwarg= github.com/kubeflow/pipelines/third_party/ml-metadata v0.0.0-20230810215105-e1f0c010f800 h1:YAW+X9xCW8Yq5tQaBBQaLTNU9CJj8Nr7lx1+k66ZHJ0= github.com/kubeflow/pipelines/third_party/ml-metadata v0.0.0-20230810215105-e1f0c010f800/go.mod h1:chIDffBaVQ/asNl1pTTdbAymYcuBKf8BR3YtSP+3FEU= github.com/labstack/echo v3.2.1+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s= diff --git a/kubernetes_platform/python/README.md b/kubernetes_platform/python/README.md index 652ad93e638..9203b937ddd 100644 --- a/kubernetes_platform/python/README.md +++ b/kubernetes_platform/python/README.md @@ -57,6 +57,44 @@ def pipeline(): mount_path='/mnt/my_vol') ``` +### ConfigMap: As environment variable +```python +from kfp import dsl +from kfp import kubernetes + +@dsl.component +def print_config_map(): + import os + print(os.environ['my-cm']) + +@dsl.pipeline +def pipeline(): + task = print_config_map() + kubernetes.use_config_map_as_env(task, + config_map_name='my-cm', + secret_key_to_env={'foo': 'CM_VAR'}) +``` + +### ConfigMap: As mounted volume +```python +from kfp import dsl +from kfp import kubernetes + +@dsl.component +def print_config_map(): + with open('/mnt/my_vol') as f: + print(f.read()) + +@dsl.pipeline +def pipeline(): + task = print_config_map() + kubernetes.use_secret_as_volume(task, + config_map_name='my-cm', + mount_path='/mnt/my_vol') +``` + + + ### PersistentVolumeClaim: Dynamically create PVC, mount, then delete ```python from kfp import dsl @@ -127,4 +165,4 @@ def my_pipeline(): annotation_key='run_id', annotation_value='123456', ) -``` \ No newline at end of file +``` diff --git a/kubernetes_platform/python/kfp/kubernetes/__init__.py b/kubernetes_platform/python/kfp/kubernetes/__init__.py index b4ac4bc16e6..7499c8fc67e 100644 --- a/kubernetes_platform/python/kfp/kubernetes/__init__.py +++ b/kubernetes_platform/python/kfp/kubernetes/__init__.py @@ -23,11 +23,15 @@ 'DeletePVC', 'mount_pvc', 'set_image_pull_secrets', + 'use_config_map_as_env', + 'use_config_map_as_volume', 'use_secret_as_env', 'use_secret_as_volume', ] from kfp.kubernetes.image import set_image_pull_secrets +from kfp.kubernetes.config_map import use_config_map_as_volume +from kfp.kubernetes.config_map import use_config_map_as_env from kfp.kubernetes.node_selector import add_node_selector from kfp.kubernetes.pod_metadata import add_pod_annotation from kfp.kubernetes.pod_metadata import add_pod_label diff --git a/kubernetes_platform/python/kfp/kubernetes/config_map.py b/kubernetes_platform/python/kfp/kubernetes/config_map.py new file mode 100644 index 00000000000..7b5c3f19356 --- /dev/null +++ b/kubernetes_platform/python/kfp/kubernetes/config_map.py @@ -0,0 +1,87 @@ +# Copyright 2024 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict + +from google.protobuf import json_format +from kfp.dsl import PipelineTask +from kfp.kubernetes import common +from kfp.kubernetes import kubernetes_executor_config_pb2 as pb + + +def use_config_map_as_env( + task: PipelineTask, + config_map_name: str, + config_map_key_to_env: Dict[str, str], +) -> PipelineTask: + """Use a Kubernetes ConfigMap as an environment variable as described by the `Kubernetes documentation + https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#define-container-environment-variables-using-configmap-data` _. + + Args: + task: Pipeline task. + config_map_name: Name of the ConfigMap. + config_map_key_to_env: Dictionary of ConfigMap key to environment variable name. For example, ``{'foo': 'FOO'}`` sets the value of the ConfigMap's foo field to the environment variable ``FOO``. + + Returns: + Task object with updated ConfigMap configuration. + """ + + msg = common.get_existing_kubernetes_config_as_message(task) + + key_to_env = [ + pb.ConfigMapAsEnv.ConfigMapKeyToEnvMap( + config_map_key=config_map_key, + env_var=env_var, + ) for config_map_key, env_var in config_map_key_to_env.items() + ] + config_map_as_env = pb.ConfigMapAsEnv( + config_map_name=config_map_name, + key_to_env=key_to_env, + ) + + msg.config_map_as_env.append(config_map_as_env) + + task.platform_config['kubernetes'] = json_format.MessageToDict(msg) + + return task + + +def use_config_map_as_volume( + task: PipelineTask, + config_map_name: str, + mount_path: str, +) -> PipelineTask: + """Use a Kubernetes ConfigMap by mounting its data to the task's container as + described by the `Kubernetes documentation `_. + + Args: + task: Pipeline task. + config_map_name: Name of the ConfigMap. + mount_path: Path to which to mount the ConfigMap data. + + Returns: + Task object with updated ConfigMap configuration. + """ + + msg = common.get_existing_kubernetes_config_as_message(task) + + config_map_as_vol = pb.ConfigMapAsVolume( + config_map_name=config_map_name, + mount_path=mount_path, + ) + msg.config_map_as_volume.append(config_map_as_vol) + + task.platform_config['kubernetes'] = json_format.MessageToDict(msg) + + return task diff --git a/kubernetes_platform/python/kfp/kubernetes/secret.py b/kubernetes_platform/python/kfp/kubernetes/secret.py index 9472d6d7ff7..dfc678f277f 100644 --- a/kubernetes_platform/python/kfp/kubernetes/secret.py +++ b/kubernetes_platform/python/kfp/kubernetes/secret.py @@ -25,9 +25,8 @@ def use_secret_as_env( secret_name: str, secret_key_to_env: Dict[str, str], ) -> PipelineTask: - """Use a Kubernetes Secret as an environment variable as described in - https://kubernetes.io/docs/concepts/configuration/secret/#using-secrets-as- - environment-variables. + """Use a Kubernetes Secret as an environment variable as described by the `Kubernetes documentation + https://kubernetes.io/docs/concepts/configuration/secret/#using-secrets-as-environment-variables `_. Args: task: Pipeline task. diff --git a/kubernetes_platform/python/test/snapshot/data/config_map_as_env.py b/kubernetes_platform/python/test/snapshot/data/config_map_as_env.py new file mode 100644 index 00000000000..3e03f3101a2 --- /dev/null +++ b/kubernetes_platform/python/test/snapshot/data/config_map_as_env.py @@ -0,0 +1,35 @@ +# Copyright 2024 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from kfp import dsl +from kfp import kubernetes + + +@dsl.component +def comp(): + pass + + +@dsl.pipeline +def my_pipeline(): + task = comp() + kubernetes.use_config_map_as_env( + task, + config_map_name='my-cm', + config_map_key_to_env={'foo': 'CONFIG_MAP_VAR'}) + + +if __name__ == '__main__': + from kfp import compiler + compiler.Compiler().compile(my_pipeline, __file__.replace('.py', '.yaml')) diff --git a/kubernetes_platform/python/test/snapshot/data/config_map_as_env.yaml b/kubernetes_platform/python/test/snapshot/data/config_map_as_env.yaml new file mode 100644 index 00000000000..51a63574a07 --- /dev/null +++ b/kubernetes_platform/python/test/snapshot/data/config_map_as_env.yaml @@ -0,0 +1,60 @@ +# PIPELINE DEFINITION +# Name: my-pipeline +components: + comp-comp: + executorLabel: exec-comp +deploymentSpec: + executors: + exec-comp: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - comp + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.4.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef comp():\n pass\n\n" + image: python:3.7 +pipelineInfo: + name: my-pipeline +root: + dag: + tasks: + comp: + cachingOptions: + enableCache: true + componentRef: + name: comp-comp + taskInfo: + name: comp +schemaVersion: 2.1.0 +sdkVersion: kfp-2.4.0 +--- +platforms: + kubernetes: + deploymentSpec: + executors: + exec-comp: + configMapAsEnv: + - keyToEnv: + - envVar: CONFIG_MAP_VAR + configMapKey: foo + configMapName: my-cm diff --git a/kubernetes_platform/python/test/snapshot/data/config_map_as_vol.py b/kubernetes_platform/python/test/snapshot/data/config_map_as_vol.py new file mode 100644 index 00000000000..76ee922fdc7 --- /dev/null +++ b/kubernetes_platform/python/test/snapshot/data/config_map_as_vol.py @@ -0,0 +1,33 @@ +# Copyright 2024 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from kfp import dsl +from kfp import kubernetes + + +@dsl.component +def comp(): + pass + + +@dsl.pipeline +def my_pipeline(): + task = comp() + kubernetes.use_config_map_as_volume( + task, config_map_name='my-cm', mount_path='/mnt/my_vol') + + +if __name__ == '__main__': + from kfp import compiler + compiler.Compiler().compile(my_pipeline, __file__.replace('.py', '.yaml')) diff --git a/kubernetes_platform/python/test/snapshot/data/config_map_as_vol.yaml b/kubernetes_platform/python/test/snapshot/data/config_map_as_vol.yaml new file mode 100644 index 00000000000..80be94504f1 --- /dev/null +++ b/kubernetes_platform/python/test/snapshot/data/config_map_as_vol.yaml @@ -0,0 +1,58 @@ +# PIPELINE DEFINITION +# Name: my-pipeline +components: + comp-comp: + executorLabel: exec-comp +deploymentSpec: + executors: + exec-comp: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - comp + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.4.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef comp():\n pass\n\n" + image: python:3.7 +pipelineInfo: + name: my-pipeline +root: + dag: + tasks: + comp: + cachingOptions: + enableCache: true + componentRef: + name: comp-comp + taskInfo: + name: comp +schemaVersion: 2.1.0 +sdkVersion: kfp-2.4.0 +--- +platforms: + kubernetes: + deploymentSpec: + executors: + exec-comp: + configMapAsVolume: + - mountPath: /mnt/my_vol + configMapName: my-cm diff --git a/kubernetes_platform/python/test/unit/test_config_map.py b/kubernetes_platform/python/test/unit/test_config_map.py new file mode 100644 index 00000000000..b607d587177 --- /dev/null +++ b/kubernetes_platform/python/test/unit/test_config_map.py @@ -0,0 +1,345 @@ +# Copyright 2023 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.protobuf import json_format +from kfp import dsl +from kfp import kubernetes + + +class TestUseConfigMapAsVolume: + + def test_use_one(self): + + @dsl.pipeline + def my_pipeline(): + task = comp() + kubernetes.use_config_map_as_volume( + task, + config_map_name='cm-name', + mount_path='cmpath', + ) + + assert json_format.MessageToDict(my_pipeline.platform_spec) == { + 'platforms': { + 'kubernetes': { + 'deploymentSpec': { + 'executors': { + 'exec-comp': { + 'configMapAsVolume': [{ + 'configMapName': 'cm-name', + 'mountPath': 'cmpath' + }] + } + } + } + } + } + } + + def test_use_two(self): + + @dsl.pipeline + def my_pipeline(): + task = comp() + kubernetes.use_config_map_as_volume( + task, + config_map_name='cm-name1', + mount_path='cmpath1', + ) + kubernetes.use_config_map_as_volume( + task, + config_map_name='cm-name2', + mount_path='cmpath2', + ) + + assert json_format.MessageToDict(my_pipeline.platform_spec) == { + 'platforms': { + 'kubernetes': { + 'deploymentSpec': { + 'executors': { + 'exec-comp': { + 'configMapAsVolume': [ + { + 'configMapName': 'cm-name1', + 'mountPath': 'cmpath1' + }, + { + 'configMapName': 'cm-name2', + 'mountPath': 'cmpath2' + }, + ] + } + } + } + } + } + } + + def test_preserves_config_map_as_env(self): + # checks that use_config map_as_volume respects previously set config maps as env + + @dsl.pipeline + def my_pipeline(): + task = comp() + kubernetes.use_config_map_as_env( + task, + config_map_name='cm-name1', + config_map_key_to_env={'foo': 'CM_VAR'}, + ) + kubernetes.use_config_map_as_volume( + task, + config_map_name='cm-name2', + mount_path='cmpath2', + ) + + assert json_format.MessageToDict(my_pipeline.platform_spec) == { + 'platforms': { + 'kubernetes': { + 'deploymentSpec': { + 'executors': { + 'exec-comp': { + 'configMapAsEnv': [{ + 'configMapName': + 'cm-name1', + 'keyToEnv': [{ + 'configMapKey': 'foo', + 'envVar': 'CM_VAR' + }] + }], + 'configMapAsVolume': [{ + 'configMapName': 'cm-name2', + 'mountPath': 'cmpath2' + },] + } + } + } + } + } + } + + def test_alongside_pvc_mount(self): + # checks that use_config_map_as_volume respects previously set pvc + @dsl.pipeline + def my_pipeline(): + task = comp() + kubernetes.mount_pvc( + task, + pvc_name='pvc-name', + mount_path='path', + ) + kubernetes.use_config_map_as_volume( + task, + config_map_name='cm-name', + mount_path='cmpath', + ) + + assert json_format.MessageToDict(my_pipeline.platform_spec) == { + 'platforms': { + 'kubernetes': { + 'deploymentSpec': { + 'executors': { + 'exec-comp': { + 'pvcMount': [{ + 'constant': 'pvc-name', + 'mountPath': 'path' + }], + 'configMapAsVolume': [{ + 'configMapName': 'cm-name', + 'mountPath': 'cmpath' + }] + } + } + } + } + } + } + + +class TestUseConfigMapAsEnv: + + def test_use_one(self): + + @dsl.pipeline + def my_pipeline(): + task = comp() + kubernetes.use_config_map_as_env( + task, + config_map_name='cm-name', + config_map_key_to_env={ + 'foo': 'FOO', + 'bar': 'BAR', + }, + ) + + assert json_format.MessageToDict(my_pipeline.platform_spec) == { + 'platforms': { + 'kubernetes': { + 'deploymentSpec': { + 'executors': { + 'exec-comp': { + 'configMapAsEnv': [{ + 'configMapName': + 'cm-name', + 'keyToEnv': [ + { + 'configMapKey': 'foo', + 'envVar': 'FOO' + }, + { + 'configMapKey': 'bar', + 'envVar': 'BAR' + }, + ] + }] + } + } + } + } + } + } + + def test_use_two(self): + + @dsl.pipeline + def my_pipeline(): + task = comp() + kubernetes.use_config_map_as_env( + task, + config_map_name='cm-name1', + config_map_key_to_env={'foo1': 'CM_VAR1'}, + ) + kubernetes.use_config_map_as_env( + task, + config_map_name='cm-name2', + config_map_key_to_env={'foo2': 'CM_VAR2'}, + ) + + assert json_format.MessageToDict(my_pipeline.platform_spec) == { + 'platforms': { + 'kubernetes': { + 'deploymentSpec': { + 'executors': { + 'exec-comp': { + 'configMapAsEnv': [ + { + 'configMapName': + 'cm-name1', + 'keyToEnv': [{ + 'configMapKey': 'foo1', + 'envVar': 'CM_VAR1' + }] + }, + { + 'configMapName': + 'cm-name2', + 'keyToEnv': [{ + 'configMapKey': 'foo2', + 'envVar': 'CM_VAR2' + }] + }, + ] + } + } + } + } + } + } + + def test_preserves_config_map_as_volume(self): + # checks that use_config_map_as_env respects previously set ConfigMaps as vol + + @dsl.pipeline + def my_pipeline(): + task = comp() + kubernetes.use_config_map_as_volume( + task, + config_map_name='cm-name2', + mount_path='cmpath2', + ) + kubernetes.use_config_map_as_env( + task, + config_map_name='cm-name1', + config_map_key_to_env={'foo': 'CM_VAR'}, + ) + + assert json_format.MessageToDict(my_pipeline.platform_spec) == { + 'platforms': { + 'kubernetes': { + 'deploymentSpec': { + 'executors': { + 'exec-comp': { + 'configMapAsEnv': [{ + 'configMapName': + 'cm-name1', + 'keyToEnv': [{ + 'configMapKey': 'foo', + 'envVar': 'CM_VAR' + }] + }], + 'configMapAsVolume': [{ + 'configMapName': 'cm-name2', + 'mountPath': 'cmpath2' + },] + } + } + } + } + } + } + + def test_preserves_pvc_mount(self): + # checks that use_config_map_as_env respects previously set pvc + @dsl.pipeline + def my_pipeline(): + task = comp() + kubernetes.mount_pvc( + task, + pvc_name='pvc-name', + mount_path='path', + ) + kubernetes.use_config_map_as_env( + task, + config_map_name='cm-name', + config_map_key_to_env={'foo': 'CM_VAR'}, + ) + + assert json_format.MessageToDict(my_pipeline.platform_spec) == { + 'platforms': { + 'kubernetes': { + 'deploymentSpec': { + 'executors': { + 'exec-comp': { + 'pvcMount': [{ + 'constant': 'pvc-name', + 'mountPath': 'path' + }], + 'configMapAsEnv': [{ + 'configMapName': + 'cm-name', + 'keyToEnv': [{ + 'configMapKey': 'foo', + 'envVar': 'CM_VAR' + }] + }] + } + } + } + } + } + } + + +@dsl.component +def comp(): + pass