Ability to rotate at a max service log size

SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java

@@ -232,6 +232,8 @@ public class SingularityExecutorConfiguration extends BaseRunnerConfiguration {
 
   private String extraDockerScriptContent = "";
 
+  private Optional<Long> maxServiceLogSizeMb = Optional.absent();
+
   public SingularityExecutorConfiguration() {
     super(Optional.of("singularity-executor.log"));
   }
@@ -686,6 +688,14 @@ public void setExtraDockerScriptContent(String extraDockerScriptContent) {
     this.extraDockerScriptContent = extraDockerScriptContent;
   }
 
+  public Optional<Long> getMaxServiceLogSizeMb() {
+    return maxServiceLogSizeMb;
+  }
+
+  public void setMaxServiceLogSizeMb(Optional<Long> maxServiceLogSizeMb) {
+    this.maxServiceLogSizeMb = maxServiceLogSizeMb;
+  }
+
   @Override
   public String toString() {
     return "SingularityExecutorConfiguration{" +

SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTask.java

@@ -61,7 +61,7 @@ public SingularityExecutorTask(ExecutorDriver driver, ExecutorUtils executorUtil
 
     this.taskDefinition = taskDefinition;
 
-    this.taskLogManager = new SingularityExecutorTaskLogManager(taskDefinition, templateManager, baseConfiguration, executorConfiguration, log, jsonObjectFileHelper);
+    this.taskLogManager = new SingularityExecutorTaskLogManager(taskDefinition, templateManager, baseConfiguration, executorConfiguration, log, jsonObjectFileHelper, executorConfiguration.getMaxServiceLogSizeMb().isPresent());
     this.taskCleanup = new SingularityExecutorTaskCleanup(taskLogManager, executorConfiguration, taskDefinition, log, dockerUtils);
     this.processBuilder = new SingularityExecutorTaskProcessBuilder(this, executorUtils, artifactFetcher, templateManager, executorConfiguration, taskDefinition.getExecutorData(), executorPid, dockerUtils, objectMapper);
     this.artifactVerifier = new SingularityExecutorArtifactVerifier(taskDefinition, log, executorConfiguration, s3Configuration);

SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java

@@ -9,12 +9,17 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 
 import org.slf4j.Logger;
 
 import com.google.common.base.Optional;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import com.hubspot.singularity.SingularityS3FormatHelper;
 import com.hubspot.singularity.SingularityS3UploaderFile;
 import com.hubspot.singularity.SingularityTaskId;
@@ -41,22 +46,68 @@ public class SingularityExecutorTaskLogManager {
   private final Logger log;
   private final JsonObjectFileHelper jsonObjectFileHelper;
   private final SingularityExecutorLogrotateFrequency logrotateFrequency;
+  private final ScheduledExecutorService logCheckExecutor;
 
-  public SingularityExecutorTaskLogManager(SingularityExecutorTaskDefinition taskDefinition, TemplateManager templateManager, SingularityRunnerBaseConfiguration baseConfiguration, SingularityExecutorConfiguration configuration, Logger log, JsonObjectFileHelper jsonObjectFileHelper) {
+  private Future<?> logCheckFuture = null;
+
+  public SingularityExecutorTaskLogManager(SingularityExecutorTaskDefinition taskDefinition, TemplateManager templateManager, SingularityRunnerBaseConfiguration baseConfiguration, SingularityExecutorConfiguration configuration, Logger log, JsonObjectFileHelper jsonObjectFileHelper, boolean startServiceLogChecker) {
     this.log = log;
     this.taskDefinition = taskDefinition;
     this.templateManager = templateManager;
     this.configuration = configuration;
     this.baseConfiguration = baseConfiguration;
     this.jsonObjectFileHelper = jsonObjectFileHelper;
     this.logrotateFrequency = taskDefinition.getExecutorData().getLogrotateFrequency().or(configuration.getLogrotateFrequency());
+    if (startServiceLogChecker) {
+      this.logCheckExecutor = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder().setNameFormat("service-log-checker-%d").build());
+    } else {
+      logCheckExecutor = null;
+    }
   }
 
   public void setup() {
     ensureServiceOutExists();
     writeLogrotateFile();
     writeTailMetadata(false);
     writeS3MetadataFileForRotatedFiles(false);
+    startLogChecker();
+  }
+
+  private void startLogChecker() {
+    try {
+      if (logCheckExecutor != null) {
+        log.info("Starting service log checker to rotate logs over {} MB", configuration.getMaxServiceLogSizeMb());
+        logCheckFuture = logCheckExecutor.scheduleAtFixedRate(this::checkServiceLogSize, 5, 5, TimeUnit.MINUTES);
+      }
+    } catch (Throwable t) {
+      log.warn("Could not start service log checker", t);
+    }
+  }
+
+  private void stopLogChecker() {
+    try {
+      if (logCheckFuture != null) {
+        logCheckFuture.cancel(true);
+      }
+      if (logCheckExecutor != null) {
+        logCheckExecutor.shutdown();
+      }
+    } catch (Throwable t) {
+      log.warn("Coud not properly shut down log checker", t);
+    }
+  }
+
+  private void checkServiceLogSize() {
+    try {
+      long fileBytes = taskDefinition.getServiceLogOutPath().toFile().length();
+      long fileMb = fileBytes / 1024 / 1024;
+      log.debug("service log is currently {} MB (limit before logrotate: {} MB)", fileMb, configuration.getMaxServiceLogSizeMb());
+      if (configuration.getMaxServiceLogSizeMb().isPresent() && fileMb > configuration.getMaxServiceLogSizeMb().get()) {
+        manualLogrotate();
+      }
+    } catch (Throwable t) {
+      log.warn("Could not run file size check on service log", t);
+    }
   }
 
   @SuppressFBWarnings
@@ -141,6 +192,7 @@ private Optional<SingularityExecutorLogrotateFrequency> getAdditionalHourlyFileF
 
   @SuppressFBWarnings
   public boolean teardown() {
+    stopLogChecker();
     boolean writeTailMetadataSuccess = writeTailMetadata(true);
 
     ensureServiceOutExists();

SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/SingularityExecutorCleanup.java

@@ -265,7 +265,7 @@ private boolean executorStillRunning(SingularityExecutorTaskDefinition taskDefin
   }
 
   private TaskCleanupResult cleanTask(SingularityExecutorTaskDefinition taskDefinition, Optional<SingularityTaskHistory> taskHistory) {
-    SingularityExecutorTaskLogManager logManager = new SingularityExecutorTaskLogManager(taskDefinition, templateManager, baseConfiguration, executorConfiguration, LOG, jsonObjectFileHelper);
+    SingularityExecutorTaskLogManager logManager = new SingularityExecutorTaskLogManager(taskDefinition, templateManager, baseConfiguration, executorConfiguration, LOG, jsonObjectFileHelper, false);
 
     SingularityExecutorTaskCleanup taskCleanup = new SingularityExecutorTaskCleanup(logManager, executorConfiguration, taskDefinition, LOG, dockerUtils);