Granular forced logrotate crons

SingularityBase/src/main/java/com/hubspot/singularity/executor/SingularityExecutorLogrotateFrequency.java

@@ -1,8 +1,12 @@
 package com.hubspot.singularity.executor;
 
+import com.google.common.collect.ImmutableSet;
 import java.util.Optional;
+import java.util.Set;
 
 public enum SingularityExecutorLogrotateFrequency {
+  EVERY_MINUTE("daily", Optional.of("* * * * *")),
+  EVERY_FIVE_MINUTES("daily", Optional.of("*/5 * * * *")),
   HOURLY("daily", Optional.of("0 * * * *")), // we have to use the "daily" frequency because not all versions of logrotate support "hourly"
   DAILY("daily", Optional.empty()),
   WEEKLY("weekly", Optional.empty()),
@@ -11,6 +15,12 @@ public enum SingularityExecutorLogrotateFrequency {
   private final String logrotateValue;
   private final Optional<String> cronSchedule;
 
+  public static final Set<SingularityExecutorLogrotateFrequency> HOURLY_OR_MORE_FREQUENT_LOGROTATE_VALUES = ImmutableSet.of(
+    EVERY_MINUTE,
+    EVERY_FIVE_MINUTES,
+    HOURLY
+  );
+
   SingularityExecutorLogrotateFrequency(
     String logrotateValue,
     Optional<String> cronSchedule

SingularityExecutor/src/main/java/com/hubspot/singularity/executor/TemplateManager.java

@@ -33,7 +33,7 @@ public TemplateManager(
     @Named(SingularityExecutorModule.ENVIRONMENT_TEMPLATE) Template environmentTemplate,
     @Named(SingularityExecutorModule.LOGROTATE_TEMPLATE) Template logrotateTemplate,
     @Named(
-      SingularityExecutorModule.LOGROTATE_HOURLY_TEMPLATE
+      SingularityExecutorModule.LOGROTATE_HOURLY_OR_MORE_FREQUENT_TEMPLATE
     ) Template logrotateHourlyTemplate,
     @Named(
       SingularityExecutorModule.LOGROTATE_SIZE_BASED_TEMPLATE

SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorModule.java

@@ -34,7 +34,8 @@ public class SingularityExecutorModule extends AbstractModule {
   public static final String RUNNER_TEMPLATE = "runner.sh";
   public static final String ENVIRONMENT_TEMPLATE = "deploy.env";
   public static final String LOGROTATE_TEMPLATE = "logrotate.conf";
-  public static final String LOGROTATE_HOURLY_TEMPLATE = "logrotate.hourly.conf";
+  public static final String LOGROTATE_HOURLY_OR_MORE_FREQUENT_TEMPLATE =
+    "logrotate.hourlyormorefrequent.conf";
   public static final String LOGROTATE_SIZE_BASED_TEMPLATE = "logrotate.sizebased.conf";
   public static final String LOGROTATE_CRON_TEMPLATE = "logrotate.cron";
   public static final String DOCKER_TEMPLATE = "docker.sh";
@@ -108,10 +109,10 @@ public Template providesLogrotateTemplate(Handlebars handlebars) throws IOExcept
 
   @Provides
   @Singleton
-  @Named(LOGROTATE_HOURLY_TEMPLATE)
+  @Named(LOGROTATE_HOURLY_OR_MORE_FREQUENT_TEMPLATE)
   public Template providesLogrotateHourlyTemplate(Handlebars handlebars)
     throws IOException {
-    return handlebars.compile(LOGROTATE_HOURLY_TEMPLATE);
+    return handlebars.compile(LOGROTATE_HOURLY_OR_MORE_FREQUENT_TEMPLATE);
   }
 
   @Provides

SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateCronTemplateContext.java

@@ -3,53 +3,88 @@
 import com.hubspot.singularity.executor.SingularityExecutorLogrotateFrequency;
 import com.hubspot.singularity.executor.config.SingularityExecutorConfiguration;
 import com.hubspot.singularity.executor.task.SingularityExecutorTaskDefinition;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 public class LogrotateCronTemplateContext {
-  private final String cronSchedule;
-  private final String logrotateCommand;
   private final String logrotateStateFile;
-  private final String logrotateForceHourlyConfig;
   private final String logrotateSizeBasedConfig;
-  private final String outputRedirect;
+  Map<SingularityExecutorLogrotateFrequency, String> logrotateConfPathsByLogrotateFrequency;
+  private final SingularityExecutorConfiguration configuration;
 
   public LogrotateCronTemplateContext(
     SingularityExecutorConfiguration configuration,
     SingularityExecutorTaskDefinition taskDefinition,
-    SingularityExecutorLogrotateFrequency logrotateFrequency,
-    String logrotateForceHourlyConfig,
+    Map<SingularityExecutorLogrotateFrequency, String> logrotateConfPathsByLogrotateFrequency,
     String logrotateSizeBasedConfig
   ) {
-    this.logrotateCommand = configuration.getLogrotateCommand();
+    this.configuration = configuration;
     this.logrotateStateFile = taskDefinition.getLogrotateStateFilePath().toString();
-    this.logrotateForceHourlyConfig = logrotateForceHourlyConfig;
+    this.logrotateConfPathsByLogrotateFrequency = logrotateConfPathsByLogrotateFrequency;
     this.logrotateSizeBasedConfig = logrotateSizeBasedConfig;
+  }
+
+  public List<LogrotateForceConfig> getLogrotateForceConfigs() {
+    return logrotateConfPathsByLogrotateFrequency
+      .entrySet()
+      .stream()
+      .map(
+        frequencyWithLogrotateConfPath -> {
+          SingularityExecutorLogrotateFrequency frequency = frequencyWithLogrotateConfPath.getKey();
+          String frequencySpecificLogrotateConfPath = frequencyWithLogrotateConfPath.getValue();
 
-    this.cronSchedule = logrotateFrequency.getCronSchedule().get();
-    this.outputRedirect =
-      configuration.isIgnoreLogrotateOutput() ? "> /dev/null 2>&1" : "";
+          return new LogrotateForceConfig(
+            configuration.getLogrotateCommand(),
+            frequencySpecificLogrotateConfPath,
+            frequency.getCronSchedule().get(),
+            configuration.isIgnoreLogrotateOutput() ? "> /dev/null 2>&1" : ""
+          );
+        }
+      )
+      .collect(Collectors.toList());
   }
 
-  public String getLogrotateCommand() {
-    return logrotateCommand;
+  public static class LogrotateForceConfig {
+    private final String logrotateCommand;
+    private final String logrotateForceConfigPath;
+    private final String cronSchedule;
+    private final String outputRedirect;
+
+    public LogrotateForceConfig(
+      String logrotateCommand,
+      String logrotateForceConfigPath,
+      String cronSchedule,
+      String outputRedirect
+    ) {
+      this.logrotateCommand = logrotateCommand;
+      this.logrotateForceConfigPath = logrotateForceConfigPath;
+      this.cronSchedule = cronSchedule;
+      this.outputRedirect = outputRedirect;
+    }
+
+    public String getLogrotateCommand() {
+      return logrotateCommand;
+    }
+
+    public String getLogrotateForceConfigPath() {
+      return logrotateForceConfigPath;
+    }
+
+    public String getCronSchedule() {
+      return cronSchedule;
+    }
+
+    public String getOutputRedirect() {
+      return outputRedirect;
+    }
   }
 
   public String getLogrotateStateFile() {
     return logrotateStateFile;
   }
 
-  public String getLogrotateForceHourlyConfig() {
-    return logrotateForceHourlyConfig;
-  }
-
   public String getLogrotateSizeBasedConfig() {
     return logrotateSizeBasedConfig;
   }
-
-  public String getCronSchedule() {
-    return cronSchedule;
-  }
-
-  public String getOutputRedirect() {
-    return outputRedirect;
-  }
 }

SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java

@@ -11,23 +11,28 @@
 import java.util.Optional;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * Handlebars context for generating logrotate.conf files.
  * Check `man logrotate` for more information.
  */
 public class LogrotateTemplateContext {
-  private static final Predicate<LogrotateAdditionalFile> BELONGS_IN_HOURLY_CRON_FORCED_LOGROTATE_CONF = p ->
-    p
-      .getLogrotateFrequencyOverride()
-      .equals(SingularityExecutorLogrotateFrequency.HOURLY.getLogrotateValue());
+  private static final Predicate<LogrotateAdditionalFile> BELONGS_IN_HOURLY_OR_MORE_FREQUENT_CRON_FORCED_LOGROTATE_CONF = p ->
+    SingularityExecutorLogrotateFrequency
+      .HOURLY_OR_MORE_FREQUENT_LOGROTATE_VALUES.stream()
+      .map(SingularityExecutorLogrotateFrequency::getLogrotateValue)
+      .collect(Collectors.toSet())
+      .contains(p.getLogrotateFrequencyOverride());
 
   private static final Predicate<LogrotateAdditionalFile> BELONGS_IN_SIZE_BASED_LOGROTATE_CONF = p ->
     p.getLogrotateSizeOverride() != null && !p.getLogrotateSizeOverride().isEmpty();
 
   private final SingularityExecutorTaskDefinition taskDefinition;
   private final SingularityExecutorConfiguration configuration;
 
+  private Optional<SingularityExecutorLogrotateFrequency> extrasFilesFrequencyFilter = Optional.empty();
+
   public LogrotateTemplateContext(
     SingularityExecutorConfiguration configuration,
     SingularityExecutorTaskDefinition taskDefinition
@@ -93,31 +98,46 @@ public String getCompressExt() {
   }
 
   /**
-   * Extra files for logrotate to rotate (non-hourly). If these do not exist logrotate will continue without error.
+   * Extra files for logrotate to rotate (less frequent than hourly). If these do not exist logrotate will continue without error.
    * @return filenames to rotate.
    */
   public List<LogrotateAdditionalFile> getExtrasFiles() {
     return getAllExtraFiles()
       .stream()
       .filter(
-        BELONGS_IN_HOURLY_CRON_FORCED_LOGROTATE_CONF
+        BELONGS_IN_HOURLY_OR_MORE_FREQUENT_CRON_FORCED_LOGROTATE_CONF
           .negate()
           .and(BELONGS_IN_SIZE_BASED_LOGROTATE_CONF.negate())
       )
       .collect(Collectors.toList());
   }
 
   /**
-   * Extra files for logrotate to rotate hourly.
-   * Since we don't want to rely on native `hourly` support in logrotate(8), we fake it by running an hourly cron with a force `-f` flag.
+   * Extra files for logrotate to rotate hourly or .
+   * Since we don't want to rely on native `hourly` (or more frequent) support in logrotate(8), we fake it by running an hourly cron with a force `-f` flag.
    * If these do not exist logrotate will continue without error.
    * @return filenames to rotate.
    */
-  public List<LogrotateAdditionalFile> getExtrasFilesHourly() {
-    return getAllExtraFiles()
+  public List<LogrotateAdditionalFile> getExtrasFilesHourlyOrMoreFrequent() {
+    Stream<LogrotateAdditionalFile> hourlyOrMoreFrequentLogrotateAdditionalFiles = getAllExtraFiles()
       .stream()
-      .filter(BELONGS_IN_HOURLY_CRON_FORCED_LOGROTATE_CONF)
-      .collect(Collectors.toList());
+      .filter(BELONGS_IN_HOURLY_OR_MORE_FREQUENT_CRON_FORCED_LOGROTATE_CONF);
+
+    return extrasFilesFrequencyFilter
+      .map(
+        singularityExecutorLogrotateFrequency ->
+          hourlyOrMoreFrequentLogrotateAdditionalFiles
+            .filter(
+              file ->
+                file
+                  .getLogrotateFrequencyOverride()
+                  .equals(singularityExecutorLogrotateFrequency.getLogrotateValue())
+            )
+            .collect(Collectors.toList())
+      )
+      .orElseGet(
+        () -> hourlyOrMoreFrequentLogrotateAdditionalFiles.collect(Collectors.toList())
+      );
   }
 
   /**
@@ -213,6 +233,12 @@ public boolean isUseFileAttributes() {
     return configuration.isUseFileAttributes();
   }
 
+  public void setExtrasFilesFrequencyFilter(
+    SingularityExecutorLogrotateFrequency frequencyFilter
+  ) {
+    this.extrasFilesFrequencyFilter = Optional.of(frequencyFilter);
+  }
+
   @Override
   public String toString() {
     return (