Add ability to set uploader additional files at deploy level

SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeploy.java

@@ -4,6 +4,7 @@
 import static com.hubspot.singularity.JsonHelpers.copyOfMap;
 import static com.hubspot.singularity.JsonHelpers.copyOfSet;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -107,6 +108,7 @@ public class SingularityDeploy {
   private final Optional<Integer> maxTaskRetries;
   private final Optional<Boolean> shell;
   private final Optional<String> user;
+  private final List<SingularityS3UploaderFile> s3UploaderAdditionalFiles;
 
   public static SingularityDeployBuilder newBuilder(String requestId, String id) {
     return new SingularityDeployBuilder(requestId, id);
@@ -160,7 +162,8 @@ public SingularityDeploy(@JsonProperty("requestId") String requestId,
                            @JsonProperty("autoAdvanceDeploySteps") Optional<Boolean> autoAdvanceDeploySteps,
                            @JsonProperty("maxTaskRetries") Optional<Integer> maxTaskRetries,
                            @JsonProperty("shell") Optional<Boolean> shell,
-                           @JsonProperty("user") Optional<String> user) {
+                           @JsonProperty("user") Optional<String> user,
+                           @JsonProperty("s3UploaderAdditionalFiles") List<SingularityS3UploaderFile> s3UploaderAdditionalFiles) {
     this.requestId = requestId;
     this.command = command;
     this.arguments = arguments;
@@ -230,6 +233,7 @@ public SingularityDeploy(@JsonProperty("requestId") String requestId,
     this.maxTaskRetries = maxTaskRetries;
     this.shell = shell;
     this.user = user;
+    this.s3UploaderAdditionalFiles = s3UploaderAdditionalFiles == null ? Collections.emptyList() : s3UploaderAdditionalFiles;
   }
 
   private static Map<Integer, List<SingularityMesosTaskLabel>> parseMesosTaskLabelsFromMap(Map<Integer, Map<String, String>> taskLabels) {
@@ -286,7 +290,8 @@ public SingularityDeployBuilder toBuilder() {
     .setAutoAdvanceDeploySteps(autoAdvanceDeploySteps)
     .setMaxTaskRetries(maxTaskRetries)
     .setShell(shell)
-    .setUser(user);
+    .setUser(user)
+    .setS3UploaderAdditionalFiles(s3UploaderAdditionalFiles);
   }
 
   @Schema(nullable = true, description = "Number of seconds that Singularity waits for this service to become healthy (for it to download artifacts, start running, and optionally pass healthchecks)")
@@ -535,6 +540,11 @@ public Optional<String> getUser() {
     return user;
   }
 
+  @Schema(description = "Specify additional sandbox files to upload to S3 for this deploy")
+  public List<SingularityS3UploaderFile> getS3UploaderAdditionalFiles() {
+    return s3UploaderAdditionalFiles;
+  }
+
   @Override
   public String toString() {
     return "SingularityDeploy{" +
@@ -586,6 +596,7 @@ public String toString() {
         ", maxTaskRetries=" + maxTaskRetries +
         ", shell=" + shell +
         ", user=" + user +
+        ", s3UploaderAdditionalFiles=" + s3UploaderAdditionalFiles +
         '}';
   }
 }

SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeployBuilder.java

@@ -1,5 +1,6 @@
 package com.hubspot.singularity;
 
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -104,6 +105,7 @@ public class SingularityDeployBuilder {
   private Optional<Integer> maxTaskRetries;
   private Optional<Boolean> shell;
   private Optional<String> user;
+  private List<SingularityS3UploaderFile> s3UploaderAdditionalFiles;
 
   public SingularityDeployBuilder(String requestId, String id) {
     this.requestId = requestId;
@@ -154,13 +156,14 @@ public SingularityDeployBuilder(String requestId, String id) {
     this.maxTaskRetries = Optional.absent();
     this.shell = Optional.absent();
     this.user = Optional.absent();
+    this.s3UploaderAdditionalFiles = Collections.emptyList();
   }
 
   public SingularityDeploy build() {
     return new SingularityDeploy(requestId, id, command, arguments, containerInfo, customExecutorCmd, customExecutorId, customExecutorSource, customExecutorResources, resources,
       env, taskEnv, runImmediately, uris, metadata, executorData, version, timestamp, labels, mesosLabels, taskLabels, mesosTaskLabels, deployHealthTimeoutSeconds, healthcheckUri, healthcheckIntervalSeconds, healthcheckTimeoutSeconds, healthcheckPortIndex, healthcheckMaxRetries,
       healthcheckMaxTotalTimeoutSeconds, healthcheck, serviceBasePath, loadBalancerGroups, loadBalancerPortIndex, considerHealthyAfterRunningForSeconds, loadBalancerOptions, loadBalancerDomains, loadBalancerAdditionalRoutes,
-      loadBalancerTemplate, loadBalancerServiceIdOverride, loadBalancerUpstreamGroup, skipHealthchecksOnDeploy, healthcheckProtocol, deployInstanceCountPerStep, deployStepWaitTimeMs, autoAdvanceDeploySteps, maxTaskRetries, shell, user);
+      loadBalancerTemplate, loadBalancerServiceIdOverride, loadBalancerUpstreamGroup, skipHealthchecksOnDeploy, healthcheckProtocol, deployInstanceCountPerStep, deployStepWaitTimeMs, autoAdvanceDeploySteps, maxTaskRetries, shell, user, s3UploaderAdditionalFiles);
   }
 
   public String getRequestId() {
@@ -594,6 +597,15 @@ public SingularityDeployBuilder setLoadBalancerUpstreamGroup(Optional<String> lo
     return this;
   }
 
+  public List<SingularityS3UploaderFile> getS3UploaderAdditionalFiles() {
+    return s3UploaderAdditionalFiles;
+  }
+
+  public SingularityDeployBuilder setS3UploaderAdditionalFiles(List<SingularityS3UploaderFile> s3UploaderAdditionalFiles) {
+    this.s3UploaderAdditionalFiles = s3UploaderAdditionalFiles;
+    return this;
+  }
+
   @Override
   public String toString() {
     return "SingularityDeployBuilder{" +
@@ -644,6 +656,7 @@ public String toString() {
         ", maxTaskRetries=" + maxTaskRetries +
         ", shell=" + shell +
         ", user=" + user +
+        ", s3UploaderAdditionalFiles=" + s3UploaderAdditionalFiles +
         '}';
   }
 }

SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilder.java

@@ -536,6 +536,7 @@ private void prepareCustomExecutor(final TaskInfo.Builder bldr, final Singularit
         uploaderAdditionalFiles.addAll(configuration.getS3ConfigurationOptional().get().getS3UploaderAdditionalFiles());
       }
       uploaderAdditionalFiles.addAll(task.getPendingTask().getS3UploaderAdditionalFiles());
+      uploaderAdditionalFiles.addAll(task.getDeploy().getS3UploaderAdditionalFiles());
 
       Optional<String> maybeS3StorageClass = configuration.getS3ConfigurationOptional().isPresent() ? configuration.getS3ConfigurationOptional().get().getS3StorageClass() : Optional.<String>absent();
       Optional<Long> maybeApplyAfterBytes = configuration.getS3ConfigurationOptional().isPresent() ? configuration.getS3ConfigurationOptional().get().getApplyS3StorageClassAfterBytes() : Optional.<Long>absent();