Control Service: Add supported python version configuration

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java

@@ -799,7 +799,7 @@ public Set<String> listCronJobs() throws ApiException {
     } catch (ApiException e) {
       if (e.getCode()
           == 404) { // as soon as the minimum supported k8s version is >=1.21 then we should remove
-                    // this.
+        // this.
         log.debug("Unable to query for v1 batch jobs", e);
       } else {
         throw e;

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/SupportedPythonVersions.java

@@ -0,0 +1,116 @@
+/*
+ * Copyright 2023-2023 VMware, Inc.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.vmware.taurus.service.deploy;
+
+import java.util.*;
+
+import org.springframework.beans.factory.annotation.Value;
+import lombok.RequiredArgsConstructor;
+import org.springframework.stereotype.Component;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Handles operations related to supported python versions for data job deployments. The class
+ * contains utility methods which provide functionality to read the configuration related to the
+ * python versions supported by the Control Service. These utility methods are meant to be used in
+ * other components, as needed.
+ */
+@Component
+@RequiredArgsConstructor
+@Slf4j
+public class SupportedPythonVersions {
+
+  private static String BASE_IMAGE = "baseImage";
+
+  private static String VDK_IMAGE = "vdkImage";
+
+  @Value("#{${datajobs.deployment.supportedPythonVersions:{}}}")
+  private Map<String, Map<String, String>> supportedPythonVersions;
+
+  @Value("${datajobs.deployment.defaultPythonVersion}")
+  private String defaultPythonVersion;
+
+  /**
+   * Check if the pythonVersion passed by the user is supported by the Control Service.
+   *
+   * @param pythonVersion python version passed by the user.
+   * @return true if the version is supported, and false otherwise.
+   */
+  public boolean isPythonVersionSupported(String pythonVersion) {
+    return !supportedPythonVersions.isEmpty() && supportedPythonVersions.containsKey(pythonVersion);
+  }
+
+  /**
+   * Returns a set of the python versions supported by the Control Service, in the format: [3.7,
+   * 3.8, ...]. If the supportedPythonVersions configuration is not set, the method returns an empty
+   * set.
+   *
+   * @return A set of all python versions supported by the Control Service.
+   */
+  public Set<String> getSupportedPythonVersions() {
+    return Optional.ofNullable(supportedPythonVersions)
+        .map(Map::keySet)
+        .orElse(Collections.emptySet());
+  }
+
+  /**
+   * Returns the name of the data job base image as stored in the docker registry. If
+   * supportedPythonVersions is set, and the pythonVersion passed by the user is supported according
+   * to the configuration, the base image corresponding to the pythonVersion is returned. Otherwise,
+   * the default base image is returned.
+   *
+   * @param pythonVersion a string indicating the python version passed by the user
+   * @return a string of the data job base image.
+   */
+  public String getJobBaseImage(String pythonVersion) {
+    if (isPythonVersionSupported(pythonVersion)) {
+      return supportedPythonVersions.get(pythonVersion).get(BASE_IMAGE);
+    } else {
+      log.warn(
+          "An issue with the passed pythonVersion or supportedPythonVersions configuration has"
+              + " occurred. Returning default job base image");
+      return getDefaultJobBaseImage();
+    }
+  }
+
+  public String getDefaultJobBaseImage() {
+    return supportedPythonVersions.get(defaultPythonVersion).get(BASE_IMAGE);
+  }
+
+  /**
+   * Returns the name of the vdk image as stored in the docker registry. If supportedPythonVersions
+   * is set, and the pythonVersion, passed by the user, is supported according to the configuration,
+   * the vdk image corresponding to the pythonVersion is returned. Otherwise, the default vdk image
+   * is returned.
+   *
+   * @param pythonVersion a string indicating the python version passed by the user
+   * @return a string of the vdk image.
+   */
+  public String getVdkImage(String pythonVersion) {
+    if (isPythonVersionSupported(pythonVersion)) {
+      return supportedPythonVersions.get(pythonVersion).get(VDK_IMAGE);
+    } else {
+      log.warn(
+          "An issue with the passed pythonVersion or supportedPythonVersions configuration has"
+              + " occurred. Returning default vdk image");
+      return getDefaultVdkImage();
+    }
+  }
+
+  public String getDefaultVdkImage() {
+    return supportedPythonVersions.get(defaultPythonVersion).get(VDK_IMAGE);
+  }
+
+  /**
+   * Returns the default python version supported by the Control Service. The version number is read
+   * from the datajobs.deployment.defaultPythonVersion application property.
+   *
+   * @return a string indicating the default python version supported by the Control Service.
+   */
+  public String getDefaultPythonVersion() {
+    return defaultPythonVersion;
+  }
+}

projects/control-service/projects/pipelines_control_service/src/main/resources/application-dev.properties

@@ -17,6 +17,8 @@ datajobs.vdk_options_ini=
 
 datajobs.deployment.k8s.kubeconfig=${HOME}/.kube/config
 datajobs.deployment.k8s.namespace=default
+datajobs.deployment.supportedPythonVersions={3.9: {vdkImage: 'registry.hub.docker.com/versatiledatakit/quickstart-vdk:release', baseImage: 'python:3.9-slim'}}
+datajobs.deployment.defaultPythonVersion=3.9
 
 # The owner name and email address that will be used to send all Versatile Data Kit related email notifications.
 datajobs.notification.owner.email=versatiledatakit@groups.vmware.com

projects/control-service/projects/pipelines_control_service/src/main/resources/application.properties

@@ -124,6 +124,13 @@ datajobs.status.watch.initial.delay=10000
 # On top of it the job source and its dependencies are installed for each job
 datajobs.deployment.dataJobBaseImage=python:3.9-slim
 
+# The map of python version and respective data job base and vdk images that would be
+# used for data job deployments
+datajobs.deployment.supportedPythonVersions=${DATAJOBS_DEPLOYMENT_SUPPORTED_PYTHON_VERSIONS:{3.9: {vdkImage: 'registry.hub.docker.com/versatiledatakit/quickstart-vdk:release', baseImage: 'python:3.9-slim'}}}
+# The default python version, which is to be used when selecting the vdk and job base images from
+# the supportedPythonVersions for data job deployments
+datajobs.deployment.defaultPythonVersion=${DATAJOBS_DEPLOYMENT_DEFAULT_PYTHON_VERSION:3.9}
+
 #Configuration variables used for data job execution cleanup
 #This is a spring cron expression, used to schedule the clean up job / default is every 3 hours
 datajobs.executions.cleanupJob.scheduleCron=0 0 */3 * * *

projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/deploy/SupportedPythonVersionsTest.java

@@ -0,0 +1,127 @@
+/*
+ * Copyright 2023-2023 VMware, Inc.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.vmware.taurus.service.deploy;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.InjectMocks;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import java.util.Map;
+import java.util.Set;
+
+@ExtendWith(MockitoExtension.class)
+public class SupportedPythonVersionsTest {
+  private static final String SUPPORTED_PYTHON_VERSIONS = "supportedPythonVersions";
+  private static final String BASE_IMAGE = "baseImage";
+  private static final String VDK_IMAGE = "vdkImage";
+  private static final String DEFAULT_PYTHON_VERSION = "defaultPythonVersion";
+
+  @InjectMocks private SupportedPythonVersions supportedPythonVersions;
+
+  @Test
+  public void isPythonVersionSupported_noSupportedVersions() {
+    ReflectionTestUtils.setField(supportedPythonVersions, SUPPORTED_PYTHON_VERSIONS, Map.of());
+
+    Assertions.assertFalse(supportedPythonVersions.isPythonVersionSupported("3.7"));
+  }
+
+  @Test
+  public void isPythonVersionSupported_versionSupported() {
+    Map<String, Map<String, String>> supportedVersions =
+        Map.of("3.7", Map.of(BASE_IMAGE, "python:3.7-slim", VDK_IMAGE, "test_vdk_image"));
+
+    ReflectionTestUtils.setField(
+        supportedPythonVersions, SUPPORTED_PYTHON_VERSIONS, supportedVersions);
+
+    Assertions.assertTrue(supportedPythonVersions.isPythonVersionSupported("3.7"));
+  }
+
+  @Test
+  public void isPythonVersionSupported_versionNotInSupported() {
+    Map<String, Map<String, String>> supportedVersions =
+        Map.of("3.8", Map.of(BASE_IMAGE, "python:3.8-slim", VDK_IMAGE, "test_vdk_image"));
+
+    ReflectionTestUtils.setField(
+        supportedPythonVersions, SUPPORTED_PYTHON_VERSIONS, supportedVersions);
+
+    Assertions.assertFalse(supportedPythonVersions.isPythonVersionSupported("3.7"));
+  }
+
+  @Test
+  public void getSupportedPythonVersions_multipleSupportedVersions() {
+    var supportedVersions = generateSupportedPythonVersionsConf();
+
+    var res = Set.of("3.7", "3.8", "3.9");
+
+    ReflectionTestUtils.setField(
+        supportedPythonVersions, SUPPORTED_PYTHON_VERSIONS, supportedVersions);
+
+    Assertions.assertEquals(res, supportedPythonVersions.getSupportedPythonVersions());
+  }
+
+  @Test
+  public void getDefaultPythonVersion() {
+    ReflectionTestUtils.setField(supportedPythonVersions, DEFAULT_PYTHON_VERSION, "3.7");
+    var res = "3.7";
+
+    Assertions.assertEquals(res, supportedPythonVersions.getDefaultPythonVersion());
+  }
+
+  @Test
+  public void getJobBaseImage_defaultImage() {
+    var supportedVersions = generateSupportedPythonVersionsConf();
+    ReflectionTestUtils.setField(
+        supportedPythonVersions, SUPPORTED_PYTHON_VERSIONS, supportedVersions);
+    ReflectionTestUtils.setField(supportedPythonVersions, DEFAULT_PYTHON_VERSION, "3.7");
+    final String defaultBaseImage = "python:3.7-slim";
+
+    Assertions.assertEquals(defaultBaseImage, supportedPythonVersions.getJobBaseImage("3.11"));
+  }
+
+  @Test
+  public void getJobBaseImage_multipleSupportedVersions() {
+    var supportedVersions = generateSupportedPythonVersionsConf();
+
+    final String resultBaseImg = "python:3.8-slim";
+    ReflectionTestUtils.setField(
+        supportedPythonVersions, SUPPORTED_PYTHON_VERSIONS, supportedVersions);
+
+    Assertions.assertEquals(resultBaseImg, supportedPythonVersions.getJobBaseImage("3.8"));
+  }
+
+  @Test
+  public void getVdkImage_defaultImage() {
+    var supportedVersions = generateSupportedPythonVersionsConf();
+    ReflectionTestUtils.setField(
+        supportedPythonVersions, SUPPORTED_PYTHON_VERSIONS, supportedVersions);
+    ReflectionTestUtils.setField(supportedPythonVersions, DEFAULT_PYTHON_VERSION, "3.7");
+
+    final String defaultVdkImage = "test_vdk_image_3.7";
+
+    Assertions.assertEquals(defaultVdkImage, supportedPythonVersions.getVdkImage("3.11"));
+  }
+
+  @Test
+  public void getVdkImage_multipleSupportedVersions() {
+    var supportedVersions = generateSupportedPythonVersionsConf();
+
+    final String resultVdkImg = "test_vdk_image_3.8";
+    ReflectionTestUtils.setField(
+        supportedPythonVersions, SUPPORTED_PYTHON_VERSIONS, supportedVersions);
+
+    Assertions.assertEquals(resultVdkImg, supportedPythonVersions.getVdkImage("3.8"));
+  }
+
+  private static Map<String, Map<String, String>> generateSupportedPythonVersionsConf() {
+    return Map.of(
+        "3.7", Map.of(BASE_IMAGE, "python:3.7-slim", VDK_IMAGE, "test_vdk_image_3.7"),
+        "3.8", Map.of(BASE_IMAGE, "python:3.8-slim", VDK_IMAGE, "test_vdk_image_3.8"),
+        "3.9", Map.of(BASE_IMAGE, "python:3.9-slim", VDK_IMAGE, "test_vdk_image_3.9"));
+  }
+}

projects/control-service/projects/pipelines_control_service/src/test/resources/application-unittest.properties

@@ -25,6 +25,8 @@ datajobs.git.password=
 
 datajobs.control.k8s.namespace=${CONTROL_K8S_NAMESPACE:default}
 datajobs.deployment.k8s.namespace=${DEPLOYMENT_K8S_NAMESPACE:default}
+datajobs.deployment.supportedPythonVersions={3.9: {vdkImage: 'registry.hub.docker.com/versatiledatakit/quickstart-vdk:release', baseImage: 'python:3.9-slim'}}
+datajobs.deployment.defaultPythonVersion=3.9
 
 # Docker repository used to store Versatile Data Kit images (ECR)
 datajobs.docker.repositoryUrl=