control-service: add ability to check if docker image exists in ecr

projects/control-service/cicd/.gitlab-ci.yml

@@ -81,6 +81,8 @@ control_service_integration_test:
     GIT_PASSWORD: $CICD_GIT_PASSWORD
     GIT_USERNAME_READ_WRITE: $CICD_GIT_USER
     GIT_PASSWORD_READ_WRITE: $CICD_GIT_PASSWORD
+    DATAJOBS_AWS_SERVICE_ACCOUNT_ACCESS_KEY_ID: $DATAJOBS_AWS_SERVICE_ACCOUNT_ACCESS_KEY_ID
+    DATAJOBS_AWS_SERVICE_ACCOUNT_SECRET_ACCESS_KEY: $DATAJOBS_AWS_SERVICE_ACCOUNT_SECRET_ACCESS_KEY
   script:
     - cd projects/control-service/projects
     - ./gradlew -p ./model build publishToMavenLocal

projects/control-service/projects/base/src/main/java/com/vmware/taurus/security/SecurityConfiguration.java

@@ -217,6 +217,7 @@ private Set<String> parseRoles(String roles) {
     }
     return Collections.emptySet();
   }
+
   /*
      KERBEROS config settings, a lot of these are optional.
   */

projects/control-service/projects/base/src/main/java/com/vmware/taurus/service/diag/telemetry/Telemetry.java

@@ -33,6 +33,7 @@ public Telemetry(@Value("${datajobs.telemetry.webhook.endpoint:}") String teleme
       log.info("Telemetry endpoint is empty and sending telemetry is skipped");
     }
   }
+
   // TODO: add support for buffering, re-tries (with back-off)
   private final String telemetryEndpoint;
   private final HttpClient client;

projects/control-service/projects/pipelines_control_service/build.gradle

@@ -97,7 +97,7 @@ dependencies { // Implementation dependencies are found on compile classpath of
     testImplementation 'com.kerb4j:kerb4j-client:0.1.3'
     testImplementation 'org.apache.kerby:kerb-simplekdc:2.0.3'
     implementation 'org.springdoc:springdoc-openapi-ui:1.7.0'
-    testImplementation versions.'com.amazonaws:aws-java-sdk-ecr'
+    implementation versions.'com.amazonaws:aws-java-sdk-ecr'
     implementation 'org.springdoc:springdoc-openapi-core:1.1.49'
     // transitive dependencies version force
     // on next upgrade, revise if those still need to be set explicitly

projects/control-service/projects/pipelines_control_service/src/integration-test/README.md

@@ -34,7 +34,11 @@ docker pull registry.hub.docker.com/versatiledatakit/job-builder:1.2.3
 docker tag registry.hub.docker.com/versatiledatakit/job-builder:1.2.3 ${DOCKER_REGISTRY_URL}/versatiledatakit/job-builder:1.2.3
 docker push ${DOCKER_REGISTRY_URL}/versatiledatakit/job-builder:1.2.3
 ```
-
+* To run the TestJobDeployTemporaryCredsIT test case the following is needed
+  - Amazon Elastic Container test Registry with IAM credentials that have rights to create/delete images in the test registry.
+  - If you intend to use the Service User model -
+  The corresponding variables in [application.properties](../main/resources/application.properties) -
+  must be filled in. To find the variables with description and documentation on the service user model search for: 'Variables for using the Service Account pattern.' in application.properties.
 # Run
 ## IntelliJ
 For the ```org.unbroken-dome.test-sets``` plugin to work well with IntelliJ you need Intellij Version ```2019.3 +```

projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/TestJobDeployTemporaryCredsIT.java → projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/datajobs/it/TestJobDeployTempCredsIT.java

@@ -6,34 +6,34 @@
 package com.vmware.taurus.datajobs.it;
 
 import static com.vmware.taurus.datajobs.it.common.WebHookServerMockExtension.TEST_TEAM_NAME;
+import static org.awaitility.Awaitility.await;
 import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user;
 import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
 import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
 
-import com.amazonaws.auth.AWSStaticCredentialsProvider;
-import com.amazonaws.auth.BasicSessionCredentials;
 import com.amazonaws.services.ecr.AmazonECR;
 import com.amazonaws.services.ecr.AmazonECRClientBuilder;
 import com.amazonaws.services.ecr.model.DeleteRepositoryRequest;
+import com.amazonaws.services.ecr.model.DescribeImagesRequest;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.vmware.taurus.ControlplaneApplication;
-import com.vmware.taurus.controlplane.model.data.DataJobDeploymentStatus;
-import com.vmware.taurus.controlplane.model.data.DataJobMode;
 import com.vmware.taurus.controlplane.model.data.DataJobVersion;
 import com.vmware.taurus.datajobs.it.common.BaseIT;
 import com.vmware.taurus.service.credentials.AWSCredentialsService;
+import com.vmware.taurus.service.deploy.DockerRegistryService;
+import com.vmware.taurus.service.deploy.EcrRegistryInterface;
 import com.vmware.taurus.service.deploy.JobImageDeployer;
 import com.vmware.taurus.service.model.JobDeploymentStatus;
+import java.time.Duration;
 import java.util.Optional;
 import java.util.UUID;
+import java.util.concurrent.TimeUnit;
 import org.apache.commons.io.IOUtils;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.platform.commons.util.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -50,25 +50,32 @@
     properties = {
       "datajobs.control.k8s.k8sSupportsV1CronJob=true",
       "datajobs.aws.assumeIAMRole=true",
-      "datajobs.aws.RoleArn=arn:aws:iam::850879199482:role/svc.supercollider.user",
+      "datajobs.aws.RoleArn=arn:aws:iam::320807031117:role/svc.ecr-integration-test",
       "datajobs.docker.registryType=ecr",
-      "datajobs.aws.region=us-west-2"
+      "datajobs.aws.region=us-west-2",
+      "DOCKER_REGISTRY_URL = 320807031117.dkr.ecr.us-west-2.amazonaws.com/sc/dp"
     })
 @SpringBootTest(
     webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT,
     classes = ControlplaneApplication.class)
-@Disabled("Disabled until we create an IAM user for testing purposes.")
-public class TestJobDeployTemporaryCredsIT extends BaseIT {
+public class TestJobDeployTempCredsIT extends BaseIT {
 
   private static final String TEST_JOB_NAME =
       "integration-test-" + UUID.randomUUID().toString().substring(0, 8);
-  private static final Object DEPLOYMENT_ID = "testing-temp-creds";
+
+  @Autowired DockerRegistryService dockerRegistryService;
 
   @Autowired AWSCredentialsService awsCredentialsService;
 
+  @Autowired EcrRegistryInterface ecrRegistryInterface;
+
   @Value("${datajobs.docker.repositoryUrl}")
   private String dockerRepositoryUrl;
 
+  private AWSCredentialsService.AWSCredentialsDTO credentialsDTO;
+  private AmazonECR ecrClient;
+  private String repositoryName;
+
   @BeforeEach
   public void setup() throws Exception {
     String dataJobRequestBody = getDataJobRequestBody(TEST_TEAM_NAME, TEST_JOB_NAME);
@@ -91,6 +98,14 @@ public void setup() throws Exception {
                                 String.format(
                                     "/data-jobs/for-team/%s/jobs/%s",
                                     TEST_TEAM_NAME, TEST_JOB_NAME)))));
+
+    this.repositoryName = dockerRepositoryUrl + "/" + TEST_JOB_NAME;
+    this.credentialsDTO = awsCredentialsService.createTemporaryCredentials();
+    this.ecrClient =
+        AmazonECRClientBuilder.standard()
+            .withCredentials(ecrRegistryInterface.createStaticCredentialsProvider(credentialsDTO))
+            .withRegion(credentialsDTO.region())
+            .build();
   }
 
   @Test
@@ -118,6 +133,11 @@ public void testDeployment() throws Exception {
     Assertions.assertNotNull(testDataJobVersion);
 
     String testJobVersionSha = testDataJobVersion.getVersionSha();
+
+    var jobUri = dockerRegistryService.dataJobImage(TEST_JOB_NAME, testJobVersionSha);
+
+    Assertions.assertFalse(dockerRegistryService.dataJobImageExists(jobUri, credentialsDTO));
+
     Assertions.assertFalse(StringUtils.isBlank(testJobVersionSha));
 
     // Setup
@@ -133,33 +153,48 @@ public void testDeployment() throws Exception {
                 .contentType(MediaType.APPLICATION_JSON))
         .andExpect(status().isAccepted());
 
+    await()
+        .atMost(10, TimeUnit.MINUTES)
+        .with()
+        .pollInterval(30, TimeUnit.SECONDS)
+        .untilAsserted(
+            () ->
+                Assertions.assertTrue(
+                    dockerRegistryService.dataJobImageExists(jobUri, credentialsDTO)));
+
     String jobDeploymentName = JobImageDeployer.getCronJobName(TEST_JOB_NAME);
 
     // Verify job deployment created
     Optional<JobDeploymentStatus> cronJobOptional =
         dataJobsKubernetesService.readCronJob(jobDeploymentName);
     Assertions.assertTrue(cronJobOptional.isPresent());
-    JobDeploymentStatus cronJob = cronJobOptional.get();
-    Assertions.assertEquals(testJobVersionSha, cronJob.getGitCommitSha());
-    Assertions.assertEquals(DataJobMode.RELEASE.toString(), cronJob.getMode());
-    Assertions.assertEquals(true, cronJob.getEnabled());
-    Assertions.assertTrue(cronJob.getImageName().endsWith(testJobVersionSha));
 
-    MvcResult result =
-        mockMvc
-            .perform(
-                get(String.format(
-                        "/data-jobs/for-team/%s/jobs/%s/deployments/%s",
-                        TEST_TEAM_NAME, TEST_JOB_NAME, DEPLOYMENT_ID))
-                    .with(user("user"))
-                    .contentType(MediaType.APPLICATION_JSON))
-            .andExpect(status().isOk())
-            .andReturn();
+    // Re-deploy job
+    mockMvc
+        .perform(
+            post(String.format(
+                    "/data-jobs/for-team/%s/jobs/%s/deployments", TEST_TEAM_NAME, TEST_JOB_NAME))
+                .with(user("user"))
+                .content(dataJobDeploymentRequestBody)
+                .contentType(MediaType.APPLICATION_JSON))
+        .andExpect(status().isAccepted());
 
-    // Verify response
-    DataJobDeploymentStatus jobDeployment =
-        mapper.readValue(result.getResponse().getContentAsString(), DataJobDeploymentStatus.class);
-    Assertions.assertEquals(testJobVersionSha, jobDeployment.getJobVersion());
+    // Make sure same image still exists. Wait 1 minute for update method to finish before checking.
+    await()
+        .atMost(3, TimeUnit.MINUTES)
+        .pollDelay(Duration.ofMinutes(1))
+        .pollInterval(30, TimeUnit.SECONDS)
+        .untilAsserted(
+            () ->
+                Assertions.assertTrue(
+                    dockerRegistryService.dataJobImageExists(jobUri, credentialsDTO)));
+
+    // Making sure only one image present, even though job was redeployed.
+    DescribeImagesRequest countImagesRequest =
+        new DescribeImagesRequest()
+            .withRepositoryName(ecrRegistryInterface.extractImageRepositoryTag(repositoryName));
+    var response = ecrClient.describeImages(countImagesRequest).getImageDetails();
+    Assertions.assertEquals(1, response.size(), "Expecting only one image");
   }
 
   @AfterEach
@@ -172,24 +207,10 @@ public void cleanUp() throws Exception {
                 .with(user("user")))
         .andExpect(status().isOk());
 
-    // Delete job repository from ECR
-    var repositoryName = dockerRepositoryUrl + "/" + TEST_JOB_NAME;
-    var credentials = awsCredentialsService.createTemporaryCredentials();
-    BasicSessionCredentials sessionCredentials =
-        new BasicSessionCredentials(
-            credentials.awsAccessKeyId(),
-            credentials.awsSecretAccessKey(),
-            credentials.awsSessionToken());
-
-    AmazonECR ecrClient =
-        AmazonECRClientBuilder.standard()
-            .withCredentials(new AWSStaticCredentialsProvider(sessionCredentials))
-            .withRegion(credentials.region())
-            .build();
-
+    // delete repository and images
     DeleteRepositoryRequest request =
         new DeleteRepositoryRequest()
-            .withRepositoryName(repositoryName)
+            .withRepositoryName(ecrRegistryInterface.extractImageRepositoryTag(repositoryName))
             .withForce(true); // Set force to true to delete the repository even if it's not empty.
 
     ecrClient.deleteRepository(request);

projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-private-builder.properties

@@ -1,4 +1,4 @@
 datajobs.builder.registrySecret=integration-test-docker-pull-secret
 datajobs.builder.registrySecret.content.testOnly=${BUILDER_TEST_REGISTRY_SECRET}
-datajobs.builder.image=${DOCKER_REGISTRY_URL}/versatiledatakit/job-builder:1.2.3
+datajobs.builder.image=${DOCKER_REGISTRY_URL}/versatiledatakit/job-builder:1.3.3
 datajobs.deployment.dataJobBaseImage=ghcr.io/versatile-data-kit-dev/dp/versatiledatakit/data-job-base-python-3.7:latest

projects/control-service/projects/pipelines_control_service/src/integration-test/resources/application-test.properties

@@ -57,7 +57,7 @@ integrationTest.controlNamespace=${CONTROL_K8S_NAMESPACE:}
 # property should have the name of the secret in the env.
 datajobs.builder.registrySecret=${DATAJOBS_BUILDER_REGISTRY_SECRET:}
 
-datajobs.builder.image=registry.hub.docker.com/versatiledatakit/job-builder:1.2.3
+datajobs.builder.image=registry.hub.docker.com/versatiledatakit/job-builder:1.3.3
 datajobs.proxy.repositoryUrl=${DOCKER_REGISTRY_URL}
 datajobs.deployment.dataJobBaseImage=versatiledatakit/data-job-base-python-3.7:latest
 

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DockerRegistryService.java

@@ -5,10 +5,13 @@
 
 package com.vmware.taurus.service.deploy;
 
+import com.vmware.taurus.service.credentials.AWSCredentialsService;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
 @Service
+@Slf4j
 public class DockerRegistryService {
 
   @Value("${datajobs.proxy.repositoryUrl}")
@@ -20,6 +23,15 @@ public class DockerRegistryService {
   @Value("${datajobs.builder.registrySecret:}")
   private String registrySecret;
 
+  @Value("${datajobs.docker.registryType:}")
+  private String registryType;
+
+  private EcrRegistryInterface ecrRegistryInterface;
+
+  public DockerRegistryService(EcrRegistryInterface ecrRegistryInterface) {
+    this.ecrRegistryInterface = ecrRegistryInterface;
+  }
+
   public String dataJobImage(String dataJobName, String gitCommitSha) {
     return String.format("%s/%s:%s", proxyRepositoryURL, dataJobName, gitCommitSha);
   }
@@ -33,7 +45,9 @@ public String builderImage() {
   }
 
   // TODO: Implement
-  public boolean dataJobImageExists(String imageName) {
-    return false;
+  public boolean dataJobImageExists(
+      String imageName, AWSCredentialsService.AWSCredentialsDTO awsCredentialsDTO) {
+    return registryType.equalsIgnoreCase("ecr")
+        && ecrRegistryInterface.checkEcrImageExists(imageName, awsCredentialsDTO);
   }
 }