work in progress

takipi · Apr 10, 2018 · 5d1f17c · 5d1f17c
1 parent 28c4236
commit 5d1f17c
Show file tree

Hide file tree

Showing 8 changed files with 358 additions and 179 deletions.
diff --git a/pom.xml b/pom.xml
@@ -11,7 +11,7 @@
 		<!-- use UTF-8 for everything -->
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-		<dropwizard.version>0.7.1</dropwizard.version>
+		<dropwizard.version>1.3.1</dropwizard.version>
 		<dropwizard-debpkg-maven-plugin.version>0.7</dropwizard-debpkg-maven-plugin.version>
 	</properties>
 	<dependencies>
@@ -20,6 +20,11 @@
 			<artifactId>dropwizard-core</artifactId>
 			<version>${dropwizard.version}</version>
 		</dependency>
+		<dependency>
+			<groupId>de.spinscale.dropwizard</groupId>
+			<artifactId>dropwizard-jobs-core</artifactId>
+			<version>3.0.0</version>
+		</dependency>
 		<dependency>
 			<groupId>commons-io</groupId>
 			<artifactId>commons-io</artifactId>

diff --git a/settings.yml b/settings.yml
@@ -2,6 +2,7 @@ folderPath: /opt/takipi-storage/storage
 maxUsedStoragePercentage: 0.95
 enableCors: true
 corsOrigins: "*"
+retentionPeriodDays: 15
 
 server:
 #  softNofileLimit: 1000
@@ -18,6 +19,9 @@ server:
   adminConnectors:
     - type: http
       port: 8081
+
+jobs:
+  cleanup: 6h
 
 # Logging settings.
 logging:

diff --git a/src/main/java/com/takipi/oss/storage/TakipiStorageConfiguration.java b/src/main/java/com/takipi/oss/storage/TakipiStorageConfiguration.java
@@ -1,5 +1,7 @@
 package com.takipi.oss.storage;
 
+import java.util.Map;
+
 import javax.validation.constraints.Max;
 import javax.validation.constraints.Min;
 
@@ -8,57 +10,83 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 
 import io.dropwizard.Configuration;
-
-public class TakipiStorageConfiguration extends Configuration {
-    @NotEmpty
-    private String folderPath;
-
-    @Min(0)
-    @Max(1)
-    private double maxUsedStoragePercentage = 0.9;
-
-    private boolean enableCors;
-
-    @NotEmpty
-    private String corsOrigins;
-
-    @JsonProperty
-    public boolean isEnableCors() {
-        return enableCors;
-    }
-
-    @JsonProperty
-    public void setEnableCors(boolean enableCors) {
-        this.enableCors = enableCors;
-    }
-
-    @JsonProperty
-    public double getMaxUsedStoragePercentage() {
-        return maxUsedStoragePercentage;
-    }
-
-    @JsonProperty
-    public void setMaxUsedStoragePercentage(double maxUsedStoragePercentage) {
-        this.maxUsedStoragePercentage = maxUsedStoragePercentage;
-    }
-
-    @JsonProperty
-    public String getCorsOrigins() {
-        return corsOrigins;
-    }
-
-    @JsonProperty
-    public void setCorsOrigins(String corsOrigins) {
-        this.corsOrigins = corsOrigins;
-    }
-
-    @JsonProperty
-    public String getFolderPath() {
-        return folderPath;
-    }
-
-    @JsonProperty
-    public void setFolderPath(String folderPath) {
-        this.folderPath = folderPath;
-    }
+import de.spinscale.dropwizard.jobs.JobConfiguration;
+
+public class TakipiStorageConfiguration extends Configuration implements JobConfiguration {
+	@NotEmpty
+	private String folderPath;
+
+	@Min(0)
+	@Max(1)
+	private double maxUsedStoragePercentage = 0.9;
+
+	private boolean enableCors;
+
+	@NotEmpty
+	private String corsOrigins;
+
+	private int retentionPeriodDays;
+
+	@JsonProperty("jobs")
+	private Map<String, String> jobs;
+
+	@JsonProperty
+	public boolean isEnableCors() {
+		return enableCors;
+	}
+
+	@JsonProperty
+	public void setEnableCors(boolean enableCors) {
+		this.enableCors = enableCors;
+	}
+
+	@JsonProperty
+	public double getMaxUsedStoragePercentage() {
+		return maxUsedStoragePercentage;
+	}
+
+	@JsonProperty
+	public void setMaxUsedStoragePercentage(double maxUsedStoragePercentage) {
+		this.maxUsedStoragePercentage = maxUsedStoragePercentage;
+	}
+
+	@JsonProperty
+	public String getCorsOrigins() {
+		return corsOrigins;
+	}
+
+	@JsonProperty
+	public void setCorsOrigins(String corsOrigins) {
+		this.corsOrigins = corsOrigins;
+	}
+
+	@JsonProperty
+	public String getFolderPath() {
+		return folderPath;
+	}
+
+	@JsonProperty
+	public void setFolderPath(String folderPath) {
+		this.folderPath = folderPath;
+	}
+
+	@JsonProperty
+	public Map<String, String> getJobs() {
+		return jobs;
+	}
+
+	@JsonProperty
+	public void setJobs(Map<String, String> jobs) {
+		this.jobs = jobs;
+	}
+
+	@JsonProperty
+	public int getRetentionPeriodDays() {
+		return retentionPeriodDays;
+	}
+
+	@JsonProperty
+	public void setRetentionPeriodDays(int retentionPeriodDays) {
+		this.retentionPeriodDays = retentionPeriodDays;
+	}
 }
diff --git a/src/main/java/com/takipi/oss/storage/TakipiStorageMain.java b/src/main/java/com/takipi/oss/storage/TakipiStorageMain.java
@@ -3,6 +3,7 @@
 import io.dropwizard.Application;
 import io.dropwizard.setup.Bootstrap;
 import io.dropwizard.setup.Environment;
+import de.spinscale.dropwizard.jobs.JobsBundle;
 
 import java.util.EnumSet;
 
@@ -12,6 +13,7 @@
 import org.eclipse.jetty.servlets.CrossOriginFilter;
 
 import com.takipi.oss.storage.health.FilesystemHealthCheck;
+import com.takipi.oss.storage.resources.admin.CleanupResource;
 import com.takipi.oss.storage.resources.diag.PingStorageResource;
 import com.takipi.oss.storage.resources.diag.StatusStorageResource;
 import com.takipi.oss.storage.resources.diag.TreeStorageResource;
@@ -21,50 +23,56 @@
 import com.takipi.oss.storage.resources.fs.JsonMultiFetchStorageResource;
 import com.takipi.oss.storage.resources.fs.JsonSimpleFetchStorageResource;
 import com.takipi.oss.storage.resources.fs.JsonSimpleSearchStorageResource;
+import com.takipi.oss.storage.jobs.PeriodicCleanupJob;
 
 public class TakipiStorageMain extends Application<TakipiStorageConfiguration> {
-    public static void main(String[] args) throws Exception {
-        new TakipiStorageMain().run(args);
-    }
+	public static void main(String[] args) throws Exception {
+		new TakipiStorageMain().run(args);
+	}
+
+	private final PeriodicCleanupJob cleanupJob = new PeriodicCleanupJob();
+
+	@Override
+	public String getName() {
+		return "takipi-storage";
+	}
 
-    @Override
-    public String getName() {
-        return "takipi-storage";
-    }
+	@Override
+	public void initialize(Bootstrap<TakipiStorageConfiguration> bootstrap) {
+		bootstrap.addBundle(new JobsBundle(cleanupJob));
+	}
 
-    @Override
-    public void initialize(Bootstrap<TakipiStorageConfiguration> bootstrap) {
+	@Override
+	public void run(TakipiStorageConfiguration configuration, Environment environment) {
+		if (configuration.isEnableCors()) {
+			enableCors(configuration, environment);
+		}
+
+		cleanupJob.configure(configuration);
+
+		environment.jersey().register(new BinaryStorageResource(configuration));
+		environment.jersey().register(new JsonMultiFetchStorageResource(configuration));
+		environment.jersey().register(new JsonMultiDeleteStorageResource(configuration));
+
+		environment.jersey().register(new JsonSimpleFetchStorageResource(configuration));
+		environment.jersey().register(new JsonSimpleSearchStorageResource(configuration));
+
+		environment.jersey().register(new PingStorageResource());
+		environment.jersey().register(new VersionStorageResource());
+		environment.jersey().register(new TreeStorageResource(configuration));
+		environment.jersey().register(new StatusStorageResource(configuration));
+		environment.jersey().register(new CleanupResource(configuration));
+
+		environment.healthChecks().register("filesystem", new FilesystemHealthCheck(configuration));
+	}
 
-    }
+	private void enableCors(TakipiStorageConfiguration configuration, Environment environment) {
+		FilterRegistration.Dynamic cors = environment.servlets().addFilter("CORS", CrossOriginFilter.class);
 
-    @Override
-    public void run(TakipiStorageConfiguration configuration, Environment environment) {
-        if (configuration.isEnableCors()) {
-            enableCors(configuration, environment);
-        }
+		cors.setInitParameter("allowedOrigins", configuration.getCorsOrigins());
+		cors.setInitParameter("allowedHeaders", "X-Requested-With,Content-Type,Accept,Origin");
+		cors.setInitParameter("allowedMethods", "OPTIONS,GET,PUT,POST,DELETE,HEAD");
 
-        environment.jersey().register(new BinaryStorageResource(configuration));
-        environment.jersey().register(new JsonMultiFetchStorageResource(configuration));
-        environment.jersey().register(new JsonMultiDeleteStorageResource(configuration));
-
-        environment.jersey().register(new JsonSimpleFetchStorageResource(configuration));
-        environment.jersey().register(new JsonSimpleSearchStorageResource(configuration));
-
-        environment.jersey().register(new PingStorageResource());
-        environment.jersey().register(new VersionStorageResource());
-        environment.jersey().register(new TreeStorageResource(configuration));
-        environment.jersey().register(new StatusStorageResource(configuration));
-
-        environment.healthChecks().register("filesystem", new FilesystemHealthCheck(configuration));
-    }
-
-    private void enableCors(TakipiStorageConfiguration configuration, Environment environment) {
-        FilterRegistration.Dynamic cors = environment.servlets().addFilter("CORS", CrossOriginFilter.class);
-
-        cors.setInitParameter("allowedOrigins", configuration.getCorsOrigins());
-        cors.setInitParameter("allowedHeaders", "X-Requested-With,Content-Type,Accept,Origin");
-        cors.setInitParameter("allowedMethods", "OPTIONS,GET,PUT,POST,DELETE,HEAD");
-
-        cors.addMappingForUrlPatterns(EnumSet.allOf(DispatcherType.class), true, "/*");
-    }
+		cors.addMappingForUrlPatterns(EnumSet.allOf(DispatcherType.class), true, "/*");
+	}
 }
diff --git a/src/main/java/com/takipi/oss/storage/fs/Record.java b/src/main/java/com/takipi/oss/storage/fs/Record.java
@@ -1,6 +1,6 @@
 package com.takipi.oss.storage.fs;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 
 public class Record {
     private String serviceId;

diff --git a/src/main/java/com/takipi/oss/storage/jobs/PeriodicCleanupJob.java b/src/main/java/com/takipi/oss/storage/jobs/PeriodicCleanupJob.java
@@ -0,0 +1,91 @@
+package com.takipi.oss.storage.jobs;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.FileVisitor;
+import java.nio.file.FileVisitResult;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.lang3.StringUtils;
+
+import org.quartz.JobExecutionContext;
+import org.quartz.JobExecutionException;
+
+import de.spinscale.dropwizard.jobs.Job;
+import de.spinscale.dropwizard.jobs.annotations.Every;
+
+import com.takipi.oss.storage.health.FilesystemHealthCheck;
+import com.takipi.oss.storage.TakipiStorageConfiguration;
+
+@Every("${cleanup}")
+public class PeriodicCleanupJob extends Job {
+	private final String[] PREFIXES_SAFE_TO_REMOVE = new String[] {
+		"HYB_HIT_", 
+		"HYB_CER_", 
+		"HYB_OM_", 
+		"HYB_WTGR_", 
+		"HYB_SAFE_"
+	};
+
+	private Path rootFolder;
+	private FilesystemHealthCheck fileSystemHealthCheck;
+	private int retentionPeriodDays;
+
+	public void configure(TakipiStorageConfiguration configuration) {
+		String rootFolderPath = configuration.getFolderPath();
+
+		if (rootFolderPath == null || rootFolderPath.isEmpty()) {
+			return;
+		}
+
+		rootFolder = Paths.get(rootFolderPath);
+		fileSystemHealthCheck = new FilesystemHealthCheck(configuration);
+		retentionPeriodDays = configuration.getRetentionPeriodDays();
+	}
+
+	@Override
+	public void doJob(JobExecutionContext context) throws JobExecutionException {
+		run();
+	}
+
+	public void run() {
+		if (rootFolder == null) {
+			return;
+		}
+
+		long retentionPeriodDaysInMillis = TimeUnit.MILLISECONDS.convert(retentionPeriodDays, TimeUnit.DAYS);
+		long minimumTimeMillis = System.currentTimeMillis() - retentionPeriodDaysInMillis;
+
+		System.out.println("RUN EVERY folder: " + rootFolder);
+		System.out.println("RUN EVERY period: " + retentionPeriodDays);
+		System.out.println("RUN EVERY retentionPeriodDaysInMillis: " + retentionPeriodDaysInMillis);
+		System.out.println("RUN EVERY minimumTimeMillis: " + minimumTimeMillis);
+		System.out.println("RUN EVERY healthy: " + fileSystemHealthCheck.execute().isHealthy());
+
+		try {
+			Files.walkFileTree(rootFolder, new SimpleFileVisitor<Path>() {
+				@Override
+				public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException {
+					if (!StringUtils.startsWithAny(path.getFileName().toString(), PREFIXES_SAFE_TO_REMOVE)) {
+						return FileVisitResult.CONTINUE;
+					}
+
+					File file = path.toFile();
+
+					System.out.println(file + " - " + file.lastModified());
+
+					return FileVisitResult.CONTINUE;
+				}
+			});
+		} catch (Exception e) {
+			e.printStackTrace();
+		} finally {
+
+		}
+	}
+}