Configurably skip the file open check in S3Uploader.

SingularityBase/src/main/java/com/hubspot/singularity/SingularityS3UploaderFile.java

@@ -18,6 +18,7 @@ public class SingularityS3UploaderFile {
   private final Optional<Long> applyS3StorageClassAfterBytes;
   private final boolean checkSubdirectories;
   private final boolean compressBeforeUpload;
+  private final boolean checkIfOpen;
 
   @JsonCreator
   public static SingularityS3UploaderFile fromString(String value) {
@@ -30,6 +31,7 @@ public static SingularityS3UploaderFile fromString(String value) {
       Optional.empty(),
       Optional.empty(),
       Optional.empty(),
+      Optional.empty(),
       Optional.empty()
     );
   }
@@ -46,7 +48,8 @@ public SingularityS3UploaderFile(
       "applyS3StorageClassAfterBytes"
     ) Optional<Long> applyS3StorageClassAfterBytes,
     @JsonProperty("checkSubdirectories") Optional<Boolean> checkSubdirectories,
-    @JsonProperty("compressBeforeUpload") Optional<Boolean> compressBeforeUpload
+    @JsonProperty("compressBeforeUpload") Optional<Boolean> compressBeforeUpload,
+    @JsonProperty("checkIfOpen") Optional<Boolean> checkIfOpen
   ) {
     this.filename = filename;
     this.s3UploaderBucket = s3UploaderBucket;
@@ -57,6 +60,7 @@ public SingularityS3UploaderFile(
     this.applyS3StorageClassAfterBytes = applyS3StorageClassAfterBytes;
     this.checkSubdirectories = checkSubdirectories.orElse(false);
     this.compressBeforeUpload = compressBeforeUpload.orElse(false);
+    this.checkIfOpen = checkIfOpen.orElse(true);
   }
 
   @Schema(description = "The name of the file")
@@ -130,6 +134,14 @@ public boolean isCompressBeforeUpload() {
     return compressBeforeUpload;
   }
 
+  @Schema(
+    title = "Check if matched files are open before processing them for upload",
+    defaultValue = "true"
+  )
+  public boolean isCheckIfOpen() {
+    return checkIfOpen;
+  }
+
   @Override
   public String toString() {
     return (
@@ -153,6 +165,8 @@ public String toString() {
       checkSubdirectories +
       ", compressBeforeUpload=" +
       compressBeforeUpload +
+      ", checkIfOpen=" +
+      checkIfOpen +
       '}'
     );
   }

SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java

@@ -204,7 +204,8 @@ private boolean writeS3MetadataFileForRotatedFiles(boolean finished) {
             ? additionalFile.getApplyS3StorageClassAfterBytes()
             : taskDefinition.getExecutorData().getApplyS3StorageClassAfterBytes(),
           additionalFile.isCheckSubdirectories(),
-          additionalFile.isCompressBeforeUpload()
+          additionalFile.isCompressBeforeUpload(),
+          additionalFile.isCheckIfOpen()
         );
       index++;
       handledLogs.add(additionalFile.getFilename());
@@ -227,7 +228,8 @@ private boolean writeS3MetadataFileForRotatedFiles(boolean finished) {
           taskDefinition.getExecutorData().getS3StorageClass(),
           taskDefinition.getExecutorData().getApplyS3StorageClassAfterBytes(),
           false,
-          false
+          false,
+          true
         );
     }
 
@@ -399,7 +401,8 @@ public boolean teardown() {
           taskDefinition.getExecutorData().getS3StorageClass(),
           taskDefinition.getExecutorData().getApplyS3StorageClassAfterBytes(),
           false,
-          false
+          false,
+          true
         );
     }
 
@@ -681,7 +684,8 @@ private boolean writeS3MetadataFile(
     Optional<String> s3StorageClass,
     Optional<Long> applyS3StorageClassAfterBytes,
     boolean checkSubdirectories,
-    boolean compressBeforeUpload
+    boolean compressBeforeUpload,
+    boolean checkIfOpen
   ) {
     final String s3UploaderBucket = s3Bucket.orElse(
       taskDefinition.getExecutorData().getDefaultS3Bucket()
@@ -713,6 +717,7 @@ private boolean writeS3MetadataFile(
       Optional.of(finished),
       Optional.of(checkSubdirectories),
       Optional.of(compressBeforeUpload),
+      Optional.of(checkIfOpen),
       Optional.empty(),
       Collections.emptyMap(),
       Optional.empty(),

SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java

@@ -55,6 +55,7 @@ public class S3UploadMetadata {
   private final Optional<Boolean> uploadImmediately;
   private final boolean checkSubdirectories;
   private final boolean compressBeforeUpload;
+  private final boolean checkIfOpen;
   private final SingularityUploaderType uploaderType;
   private final Map<String, Object> gcsCredentials;
   private final Optional<String> gcsStorageClass;
@@ -82,6 +83,7 @@ public S3UploadMetadata(
     @JsonProperty("uploadImmediately") Optional<Boolean> uploadImmediately,
     @JsonProperty("checkSubdirectories") Optional<Boolean> checkSubdirectories,
     @JsonProperty("compressBeforeUpload") Optional<Boolean> compressBeforeUpload,
+    @JsonProperty("checkIfOpen") Optional<Boolean> checkIfOpen,
     @JsonProperty("uploaderType") Optional<SingularityUploaderType> uploaderType,
     @JsonProperty("gcsCredentials") Map<String, Object> gcsCredentials,
     @JsonProperty("gcsStorageClass") Optional<String> gcsStorageClass,
@@ -108,6 +110,7 @@ public S3UploadMetadata(
     this.uploadImmediately = uploadImmediately;
     this.checkSubdirectories = checkSubdirectories.orElse(false);
     this.compressBeforeUpload = compressBeforeUpload.orElse(false);
+    this.checkIfOpen = checkIfOpen.orElse(true);
     this.uploaderType = uploaderType.orElse(SingularityUploaderType.S3);
     this.gcsCredentials =
       gcsCredentials != null ? gcsCredentials : Collections.emptyMap();
@@ -218,6 +221,10 @@ public boolean isCompressBeforeUpload() {
     return compressBeforeUpload;
   }
 
+  public boolean isCheckIfOpen() {
+    return checkIfOpen;
+  }
+
   public SingularityUploaderType getUploaderType() {
     return uploaderType;
   }
@@ -277,6 +284,8 @@ public String toString() {
       checkSubdirectories +
       ", compressBeforeUpload=" +
       compressBeforeUpload +
+      ", checkIfOpen=" +
+      checkIfOpen +
       ", uploaderType=" +
       uploaderType +
       ", gcsStorageClass=" +

SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityUploader.java

@@ -105,6 +105,7 @@ int uploadBatch(List<Path> toUpload) {
       final Path file = toUpload.get(i);
       if (
         !configuration.isCheckForOpenFiles() ||
+        !uploadMetadata.isCheckIfOpen() ||
         uploadMetadata.isImmediate() ||
         !isFileOpen(file, configuration.isCheckOpenFilesViaFuser())
       ) {