Merge pull request #36 from takipi/feature/OO-1024/hybrid-cleanup

work in progress
takipi · Apr 12, 2018 · 7397ea3 · 7397ea3
2 parents 28c4236 + 822734e
commit 7397ea3
Show file tree

Hide file tree

Showing 13 changed files with 543 additions and 219 deletions.
diff --git a/docker/settings.yml b/docker/settings.yml
@@ -2,6 +2,7 @@ folderPath: /opt/takipi-storage/storage
 maxUsedStoragePercentage: 0.95
 enableCors: true
 corsOrigins: "*"
+retentionPeriodDays: 92
 
 server:
   applicationConnectors:
@@ -16,6 +17,10 @@ server:
         currentLogFilename: ./storage/log/access.log
         archivedLogFilenamePattern: ./storage/log/access.%d.log.gz
         archivedFileCount: 14
+
+jobs:
+  cleanup: 6h
+
 logging:
   level: INFO
   loggers:

diff --git a/pom.xml b/pom.xml
@@ -11,7 +11,7 @@
 		<!-- use UTF-8 for everything -->
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-		<dropwizard.version>0.7.1</dropwizard.version>
+		<dropwizard.version>1.3.1</dropwizard.version>
 		<dropwizard-debpkg-maven-plugin.version>0.7</dropwizard-debpkg-maven-plugin.version>
 	</properties>
 	<dependencies>
@@ -20,6 +20,11 @@
 			<artifactId>dropwizard-core</artifactId>
 			<version>${dropwizard.version}</version>
 		</dependency>
+		<dependency>
+			<groupId>de.spinscale.dropwizard</groupId>
+			<artifactId>dropwizard-jobs-core</artifactId>
+			<version>3.0.0</version>
+		</dependency>
 		<dependency>
 			<groupId>commons-io</groupId>
 			<artifactId>commons-io</artifactId>

diff --git a/settings.yml b/settings.yml
@@ -1,7 +1,8 @@
 folderPath: /opt/takipi-storage/storage
-maxUsedStoragePercentage: 0.95
+maxUsedStoragePercentage: 0.90
 enableCors: true
 corsOrigins: "*"
+retentionPeriodDays: 92
 
 server:
 #  softNofileLimit: 1000
@@ -18,6 +19,9 @@ server:
   adminConnectors:
     - type: http
       port: 8081
+
+jobs:
+  cleanup: 6h
 
 # Logging settings.
 logging:

diff --git a/src/main/java/com/takipi/oss/storage/TakipiStorageConfiguration.java b/src/main/java/com/takipi/oss/storage/TakipiStorageConfiguration.java
@@ -1,5 +1,7 @@
 package com.takipi.oss.storage;
 
+import java.util.Map;
+
 import javax.validation.constraints.Max;
 import javax.validation.constraints.Min;
 
@@ -8,57 +10,83 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 
 import io.dropwizard.Configuration;
-
-public class TakipiStorageConfiguration extends Configuration {
-    @NotEmpty
-    private String folderPath;
-
-    @Min(0)
-    @Max(1)
-    private double maxUsedStoragePercentage = 0.9;
-
-    private boolean enableCors;
-
-    @NotEmpty
-    private String corsOrigins;
-
-    @JsonProperty
-    public boolean isEnableCors() {
-        return enableCors;
-    }
-
-    @JsonProperty
-    public void setEnableCors(boolean enableCors) {
-        this.enableCors = enableCors;
-    }
-
-    @JsonProperty
-    public double getMaxUsedStoragePercentage() {
-        return maxUsedStoragePercentage;
-    }
-
-    @JsonProperty
-    public void setMaxUsedStoragePercentage(double maxUsedStoragePercentage) {
-        this.maxUsedStoragePercentage = maxUsedStoragePercentage;
-    }
-
-    @JsonProperty
-    public String getCorsOrigins() {
-        return corsOrigins;
-    }
-
-    @JsonProperty
-    public void setCorsOrigins(String corsOrigins) {
-        this.corsOrigins = corsOrigins;
-    }
-
-    @JsonProperty
-    public String getFolderPath() {
-        return folderPath;
-    }
-
-    @JsonProperty
-    public void setFolderPath(String folderPath) {
-        this.folderPath = folderPath;
-    }
+import de.spinscale.dropwizard.jobs.JobConfiguration;
+
+public class TakipiStorageConfiguration extends Configuration implements JobConfiguration {
+	@NotEmpty
+	private String folderPath;
+
+	@Min(0)
+	@Max(1)
+	private double maxUsedStoragePercentage = 0.9;
+
+	private boolean enableCors;
+
+	@NotEmpty
+	private String corsOrigins;
+
+	private int retentionPeriodDays;
+
+	@JsonProperty("jobs")
+	private Map<String, String> jobs;
+
+	@JsonProperty
+	public boolean isEnableCors() {
+		return enableCors;
+	}
+
+	@JsonProperty
+	public void setEnableCors(boolean enableCors) {
+		this.enableCors = enableCors;
+	}
+
+	@JsonProperty
+	public double getMaxUsedStoragePercentage() {
+		return maxUsedStoragePercentage;
+	}
+
+	@JsonProperty
+	public void setMaxUsedStoragePercentage(double maxUsedStoragePercentage) {
+		this.maxUsedStoragePercentage = maxUsedStoragePercentage;
+	}
+
+	@JsonProperty
+	public String getCorsOrigins() {
+		return corsOrigins;
+	}
+
+	@JsonProperty
+	public void setCorsOrigins(String corsOrigins) {
+		this.corsOrigins = corsOrigins;
+	}
+
+	@JsonProperty
+	public String getFolderPath() {
+		return folderPath;
+	}
+
+	@JsonProperty
+	public void setFolderPath(String folderPath) {
+		this.folderPath = folderPath;
+	}
+
+	@JsonProperty
+	public Map<String, String> getJobs() {
+		return jobs;
+	}
+
+	@JsonProperty
+	public void setJobs(Map<String, String> jobs) {
+		this.jobs = jobs;
+	}
+
+	@JsonProperty
+	public int getRetentionPeriodDays() {
+		return retentionPeriodDays;
+	}
+
+	@JsonProperty
+	public void setRetentionPeriodDays(int retentionPeriodDays) {
+		this.retentionPeriodDays = retentionPeriodDays;
+	}
 }
diff --git a/src/main/java/com/takipi/oss/storage/TakipiStorageMain.java b/src/main/java/com/takipi/oss/storage/TakipiStorageMain.java
@@ -3,6 +3,7 @@
 import io.dropwizard.Application;
 import io.dropwizard.setup.Bootstrap;
 import io.dropwizard.setup.Environment;
+import de.spinscale.dropwizard.jobs.JobsBundle;
 
 import java.util.EnumSet;
 
@@ -12,6 +13,7 @@
 import org.eclipse.jetty.servlets.CrossOriginFilter;
 
 import com.takipi.oss.storage.health.FilesystemHealthCheck;
+import com.takipi.oss.storage.resources.admin.CleanupResource;
 import com.takipi.oss.storage.resources.diag.PingStorageResource;
 import com.takipi.oss.storage.resources.diag.StatusStorageResource;
 import com.takipi.oss.storage.resources.diag.TreeStorageResource;
@@ -21,50 +23,56 @@
 import com.takipi.oss.storage.resources.fs.JsonMultiFetchStorageResource;
 import com.takipi.oss.storage.resources.fs.JsonSimpleFetchStorageResource;
 import com.takipi.oss.storage.resources.fs.JsonSimpleSearchStorageResource;
+import com.takipi.oss.storage.jobs.PeriodicCleanupJob;
 
 public class TakipiStorageMain extends Application<TakipiStorageConfiguration> {
-    public static void main(String[] args) throws Exception {
-        new TakipiStorageMain().run(args);
-    }
+	public static void main(String[] args) throws Exception {
+		new TakipiStorageMain().run(args);
+	}
+
+	private final PeriodicCleanupJob cleanupJob = new PeriodicCleanupJob();
+
+	@Override
+	public String getName() {
+		return "takipi-storage";
+	}
 
-    @Override
-    public String getName() {
-        return "takipi-storage";
-    }
+	@Override
+	public void initialize(Bootstrap<TakipiStorageConfiguration> bootstrap) {
+		bootstrap.addBundle(new JobsBundle(cleanupJob));
+	}
 
-    @Override
-    public void initialize(Bootstrap<TakipiStorageConfiguration> bootstrap) {
+	@Override
+	public void run(TakipiStorageConfiguration configuration, Environment environment) {
+		if (configuration.isEnableCors()) {
+			enableCors(configuration, environment);
+		}
+
+		cleanupJob.configure(configuration);
+
+		environment.jersey().register(new BinaryStorageResource(configuration));
+		environment.jersey().register(new JsonMultiFetchStorageResource(configuration));
+		environment.jersey().register(new JsonMultiDeleteStorageResource(configuration));
+
+		environment.jersey().register(new JsonSimpleFetchStorageResource(configuration));
+		environment.jersey().register(new JsonSimpleSearchStorageResource(configuration));
+
+		environment.jersey().register(new PingStorageResource());
+		environment.jersey().register(new VersionStorageResource());
+		environment.jersey().register(new TreeStorageResource(configuration));
+		environment.jersey().register(new StatusStorageResource(configuration));
+		environment.jersey().register(new CleanupResource(configuration));
+
+		environment.healthChecks().register("filesystem", new FilesystemHealthCheck(configuration));
+	}
 
-    }
+	private void enableCors(TakipiStorageConfiguration configuration, Environment environment) {
+		FilterRegistration.Dynamic cors = environment.servlets().addFilter("CORS", CrossOriginFilter.class);
 
-    @Override
-    public void run(TakipiStorageConfiguration configuration, Environment environment) {
-        if (configuration.isEnableCors()) {
-            enableCors(configuration, environment);
-        }
+		cors.setInitParameter("allowedOrigins", configuration.getCorsOrigins());
+		cors.setInitParameter("allowedHeaders", "X-Requested-With,Content-Type,Accept,Origin");
+		cors.setInitParameter("allowedMethods", "OPTIONS,GET,PUT,POST,DELETE,HEAD");
 
-        environment.jersey().register(new BinaryStorageResource(configuration));
-        environment.jersey().register(new JsonMultiFetchStorageResource(configuration));
-        environment.jersey().register(new JsonMultiDeleteStorageResource(configuration));
-
-        environment.jersey().register(new JsonSimpleFetchStorageResource(configuration));
-        environment.jersey().register(new JsonSimpleSearchStorageResource(configuration));
-
-        environment.jersey().register(new PingStorageResource());
-        environment.jersey().register(new VersionStorageResource());
-        environment.jersey().register(new TreeStorageResource(configuration));
-        environment.jersey().register(new StatusStorageResource(configuration));
-
-        environment.healthChecks().register("filesystem", new FilesystemHealthCheck(configuration));
-    }
-
-    private void enableCors(TakipiStorageConfiguration configuration, Environment environment) {
-        FilterRegistration.Dynamic cors = environment.servlets().addFilter("CORS", CrossOriginFilter.class);
-
-        cors.setInitParameter("allowedOrigins", configuration.getCorsOrigins());
-        cors.setInitParameter("allowedHeaders", "X-Requested-With,Content-Type,Accept,Origin");
-        cors.setInitParameter("allowedMethods", "OPTIONS,GET,PUT,POST,DELETE,HEAD");
-
-        cors.addMappingForUrlPatterns(EnumSet.allOf(DispatcherType.class), true, "/*");
-    }
+		cors.addMappingForUrlPatterns(EnumSet.allOf(DispatcherType.class), true, "/*");
+	}
 }
diff --git a/src/main/java/com/takipi/oss/storage/data/status/MachineStatus.java b/src/main/java/com/takipi/oss/storage/data/status/MachineStatus.java
@@ -1,5 +1,8 @@
 package com.takipi.oss.storage.data.status;
 
+import java.io.File;
+import java.util.List;
+
 public class MachineStatus
 {
 	private long hitsCount;
@@ -24,6 +27,9 @@ public class MachineStatus
 	private long heapSizeBytes;
 	private long permGenSizeBytes;
 	private String version;
+	private String lastCleanupStartTime;
+	private long lastCleanupDurationMillis;
+	private List<File> lastCleanupRemovedFiles;
 
 	public void setHitsCount(long hitsCount) {
 		this.hitsCount = hitsCount;
@@ -200,4 +206,28 @@ public String getVersion() {
 	public void setVersion(String version) {
 		this.version = version;
 	}
+
+	public String getLastCleanupStartTime() {
+		return lastCleanupStartTime;
+	}
+
+	public void setLastCleanupStartTime(String lastCleanupStartTime) {
+		this.lastCleanupStartTime = lastCleanupStartTime;
+	}
+
+	public void setLastCleanupDurationMillis(long lastCleanupDurationMillis) {
+		this.lastCleanupDurationMillis = lastCleanupDurationMillis;
+	}
+
+	public long getLastCleanupDurationMillis() {
+		return lastCleanupDurationMillis;
+	}
+
+	public void setLastCleanupRemovedFiles(List<File> lastCleanupRemovedFiles) {
+		this.lastCleanupRemovedFiles = lastCleanupRemovedFiles;
+	}
+
+	public List<File> getLastCleanupRemovedFiles() {
+		return lastCleanupRemovedFiles;
+	}
 }
diff --git a/src/main/java/com/takipi/oss/storage/fs/Record.java b/src/main/java/com/takipi/oss/storage/fs/Record.java
@@ -1,6 +1,6 @@
 package com.takipi.oss.storage.fs;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 
 public class Record {
     private String serviceId;