From 5d1f17c28a3e656b90802a3d0cc985a2db0caba5 Mon Sep 17 00:00:00 2001 From: David Levanon Date: Tue, 10 Apr 2018 20:37:04 +0300 Subject: [PATCH 1/4] work in progress --- pom.xml | 7 +- settings.yml | 4 + .../storage/TakipiStorageConfiguration.java | 134 +++++++------ .../takipi/oss/storage/TakipiStorageMain.java | 86 +++++---- .../com/takipi/oss/storage/fs/Record.java | 2 +- .../oss/storage/jobs/PeriodicCleanupJob.java | 91 +++++++++ .../resources/admin/CleanupResource.java | 37 ++++ .../fs/JsonSimpleSearchStorageResource.java | 176 +++++++++--------- 8 files changed, 358 insertions(+), 179 deletions(-) create mode 100644 src/main/java/com/takipi/oss/storage/jobs/PeriodicCleanupJob.java create mode 100644 src/main/java/com/takipi/oss/storage/resources/admin/CleanupResource.java diff --git a/pom.xml b/pom.xml index 160ab5f..2c245f2 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ UTF-8 UTF-8 - 0.7.1 + 1.3.1 0.7 @@ -20,6 +20,11 @@ dropwizard-core ${dropwizard.version} + + de.spinscale.dropwizard + dropwizard-jobs-core + 3.0.0 + commons-io commons-io diff --git a/settings.yml b/settings.yml index 09361ef..58cbaed 100644 --- a/settings.yml +++ b/settings.yml @@ -2,6 +2,7 @@ folderPath: /opt/takipi-storage/storage maxUsedStoragePercentage: 0.95 enableCors: true corsOrigins: "*" +retentionPeriodDays: 15 server: # softNofileLimit: 1000 @@ -18,6 +19,9 @@ server: adminConnectors: - type: http port: 8081 + +jobs: + cleanup: 6h # Logging settings. logging: diff --git a/src/main/java/com/takipi/oss/storage/TakipiStorageConfiguration.java b/src/main/java/com/takipi/oss/storage/TakipiStorageConfiguration.java index 8693630..89ba8e5 100644 --- a/src/main/java/com/takipi/oss/storage/TakipiStorageConfiguration.java +++ b/src/main/java/com/takipi/oss/storage/TakipiStorageConfiguration.java @@ -1,5 +1,7 @@ package com.takipi.oss.storage; +import java.util.Map; + import javax.validation.constraints.Max; import javax.validation.constraints.Min; @@ -8,57 +10,83 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.dropwizard.Configuration; - -public class TakipiStorageConfiguration extends Configuration { - @NotEmpty - private String folderPath; - - @Min(0) - @Max(1) - private double maxUsedStoragePercentage = 0.9; - - private boolean enableCors; - - @NotEmpty - private String corsOrigins; - - @JsonProperty - public boolean isEnableCors() { - return enableCors; - } - - @JsonProperty - public void setEnableCors(boolean enableCors) { - this.enableCors = enableCors; - } - - @JsonProperty - public double getMaxUsedStoragePercentage() { - return maxUsedStoragePercentage; - } - - @JsonProperty - public void setMaxUsedStoragePercentage(double maxUsedStoragePercentage) { - this.maxUsedStoragePercentage = maxUsedStoragePercentage; - } - - @JsonProperty - public String getCorsOrigins() { - return corsOrigins; - } - - @JsonProperty - public void setCorsOrigins(String corsOrigins) { - this.corsOrigins = corsOrigins; - } - - @JsonProperty - public String getFolderPath() { - return folderPath; - } - - @JsonProperty - public void setFolderPath(String folderPath) { - this.folderPath = folderPath; - } +import de.spinscale.dropwizard.jobs.JobConfiguration; + +public class TakipiStorageConfiguration extends Configuration implements JobConfiguration { + @NotEmpty + private String folderPath; + + @Min(0) + @Max(1) + private double maxUsedStoragePercentage = 0.9; + + private boolean enableCors; + + @NotEmpty + private String corsOrigins; + + private int retentionPeriodDays; + + @JsonProperty("jobs") + private Map jobs; + + @JsonProperty + public boolean isEnableCors() { + return enableCors; + } + + @JsonProperty + public void setEnableCors(boolean enableCors) { + this.enableCors = enableCors; + } + + @JsonProperty + public double getMaxUsedStoragePercentage() { + return maxUsedStoragePercentage; + } + + @JsonProperty + public void setMaxUsedStoragePercentage(double maxUsedStoragePercentage) { + this.maxUsedStoragePercentage = maxUsedStoragePercentage; + } + + @JsonProperty + public String getCorsOrigins() { + return corsOrigins; + } + + @JsonProperty + public void setCorsOrigins(String corsOrigins) { + this.corsOrigins = corsOrigins; + } + + @JsonProperty + public String getFolderPath() { + return folderPath; + } + + @JsonProperty + public void setFolderPath(String folderPath) { + this.folderPath = folderPath; + } + + @JsonProperty + public Map getJobs() { + return jobs; + } + + @JsonProperty + public void setJobs(Map jobs) { + this.jobs = jobs; + } + + @JsonProperty + public int getRetentionPeriodDays() { + return retentionPeriodDays; + } + + @JsonProperty + public void setRetentionPeriodDays(int retentionPeriodDays) { + this.retentionPeriodDays = retentionPeriodDays; + } } diff --git a/src/main/java/com/takipi/oss/storage/TakipiStorageMain.java b/src/main/java/com/takipi/oss/storage/TakipiStorageMain.java index c998dae..87692ea 100644 --- a/src/main/java/com/takipi/oss/storage/TakipiStorageMain.java +++ b/src/main/java/com/takipi/oss/storage/TakipiStorageMain.java @@ -3,6 +3,7 @@ import io.dropwizard.Application; import io.dropwizard.setup.Bootstrap; import io.dropwizard.setup.Environment; +import de.spinscale.dropwizard.jobs.JobsBundle; import java.util.EnumSet; @@ -12,6 +13,7 @@ import org.eclipse.jetty.servlets.CrossOriginFilter; import com.takipi.oss.storage.health.FilesystemHealthCheck; +import com.takipi.oss.storage.resources.admin.CleanupResource; import com.takipi.oss.storage.resources.diag.PingStorageResource; import com.takipi.oss.storage.resources.diag.StatusStorageResource; import com.takipi.oss.storage.resources.diag.TreeStorageResource; @@ -21,50 +23,56 @@ import com.takipi.oss.storage.resources.fs.JsonMultiFetchStorageResource; import com.takipi.oss.storage.resources.fs.JsonSimpleFetchStorageResource; import com.takipi.oss.storage.resources.fs.JsonSimpleSearchStorageResource; +import com.takipi.oss.storage.jobs.PeriodicCleanupJob; public class TakipiStorageMain extends Application { - public static void main(String[] args) throws Exception { - new TakipiStorageMain().run(args); - } + public static void main(String[] args) throws Exception { + new TakipiStorageMain().run(args); + } + + private final PeriodicCleanupJob cleanupJob = new PeriodicCleanupJob(); + + @Override + public String getName() { + return "takipi-storage"; + } - @Override - public String getName() { - return "takipi-storage"; - } + @Override + public void initialize(Bootstrap bootstrap) { + bootstrap.addBundle(new JobsBundle(cleanupJob)); + } - @Override - public void initialize(Bootstrap bootstrap) { + @Override + public void run(TakipiStorageConfiguration configuration, Environment environment) { + if (configuration.isEnableCors()) { + enableCors(configuration, environment); + } + + cleanupJob.configure(configuration); + + environment.jersey().register(new BinaryStorageResource(configuration)); + environment.jersey().register(new JsonMultiFetchStorageResource(configuration)); + environment.jersey().register(new JsonMultiDeleteStorageResource(configuration)); + + environment.jersey().register(new JsonSimpleFetchStorageResource(configuration)); + environment.jersey().register(new JsonSimpleSearchStorageResource(configuration)); + + environment.jersey().register(new PingStorageResource()); + environment.jersey().register(new VersionStorageResource()); + environment.jersey().register(new TreeStorageResource(configuration)); + environment.jersey().register(new StatusStorageResource(configuration)); + environment.jersey().register(new CleanupResource(configuration)); + + environment.healthChecks().register("filesystem", new FilesystemHealthCheck(configuration)); + } - } + private void enableCors(TakipiStorageConfiguration configuration, Environment environment) { + FilterRegistration.Dynamic cors = environment.servlets().addFilter("CORS", CrossOriginFilter.class); - @Override - public void run(TakipiStorageConfiguration configuration, Environment environment) { - if (configuration.isEnableCors()) { - enableCors(configuration, environment); - } + cors.setInitParameter("allowedOrigins", configuration.getCorsOrigins()); + cors.setInitParameter("allowedHeaders", "X-Requested-With,Content-Type,Accept,Origin"); + cors.setInitParameter("allowedMethods", "OPTIONS,GET,PUT,POST,DELETE,HEAD"); - environment.jersey().register(new BinaryStorageResource(configuration)); - environment.jersey().register(new JsonMultiFetchStorageResource(configuration)); - environment.jersey().register(new JsonMultiDeleteStorageResource(configuration)); - - environment.jersey().register(new JsonSimpleFetchStorageResource(configuration)); - environment.jersey().register(new JsonSimpleSearchStorageResource(configuration)); - - environment.jersey().register(new PingStorageResource()); - environment.jersey().register(new VersionStorageResource()); - environment.jersey().register(new TreeStorageResource(configuration)); - environment.jersey().register(new StatusStorageResource(configuration)); - - environment.healthChecks().register("filesystem", new FilesystemHealthCheck(configuration)); - } - - private void enableCors(TakipiStorageConfiguration configuration, Environment environment) { - FilterRegistration.Dynamic cors = environment.servlets().addFilter("CORS", CrossOriginFilter.class); - - cors.setInitParameter("allowedOrigins", configuration.getCorsOrigins()); - cors.setInitParameter("allowedHeaders", "X-Requested-With,Content-Type,Accept,Origin"); - cors.setInitParameter("allowedMethods", "OPTIONS,GET,PUT,POST,DELETE,HEAD"); - - cors.addMappingForUrlPatterns(EnumSet.allOf(DispatcherType.class), true, "/*"); - } + cors.addMappingForUrlPatterns(EnumSet.allOf(DispatcherType.class), true, "/*"); + } } diff --git a/src/main/java/com/takipi/oss/storage/fs/Record.java b/src/main/java/com/takipi/oss/storage/fs/Record.java index f396dab..4b8dcc7 100644 --- a/src/main/java/com/takipi/oss/storage/fs/Record.java +++ b/src/main/java/com/takipi/oss/storage/fs/Record.java @@ -1,6 +1,6 @@ package com.takipi.oss.storage.fs; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; public class Record { private String serviceId; diff --git a/src/main/java/com/takipi/oss/storage/jobs/PeriodicCleanupJob.java b/src/main/java/com/takipi/oss/storage/jobs/PeriodicCleanupJob.java new file mode 100644 index 0000000..a4663f4 --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/jobs/PeriodicCleanupJob.java @@ -0,0 +1,91 @@ +package com.takipi.oss.storage.jobs; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.FileVisitor; +import java.nio.file.FileVisitResult; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.lang3.StringUtils; + +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; + +import de.spinscale.dropwizard.jobs.Job; +import de.spinscale.dropwizard.jobs.annotations.Every; + +import com.takipi.oss.storage.health.FilesystemHealthCheck; +import com.takipi.oss.storage.TakipiStorageConfiguration; + +@Every("${cleanup}") +public class PeriodicCleanupJob extends Job { + private final String[] PREFIXES_SAFE_TO_REMOVE = new String[] { + "HYB_HIT_", + "HYB_CER_", + "HYB_OM_", + "HYB_WTGR_", + "HYB_SAFE_" + }; + + private Path rootFolder; + private FilesystemHealthCheck fileSystemHealthCheck; + private int retentionPeriodDays; + + public void configure(TakipiStorageConfiguration configuration) { + String rootFolderPath = configuration.getFolderPath(); + + if (rootFolderPath == null || rootFolderPath.isEmpty()) { + return; + } + + rootFolder = Paths.get(rootFolderPath); + fileSystemHealthCheck = new FilesystemHealthCheck(configuration); + retentionPeriodDays = configuration.getRetentionPeriodDays(); + } + + @Override + public void doJob(JobExecutionContext context) throws JobExecutionException { + run(); + } + + public void run() { + if (rootFolder == null) { + return; + } + + long retentionPeriodDaysInMillis = TimeUnit.MILLISECONDS.convert(retentionPeriodDays, TimeUnit.DAYS); + long minimumTimeMillis = System.currentTimeMillis() - retentionPeriodDaysInMillis; + + System.out.println("RUN EVERY folder: " + rootFolder); + System.out.println("RUN EVERY period: " + retentionPeriodDays); + System.out.println("RUN EVERY retentionPeriodDaysInMillis: " + retentionPeriodDaysInMillis); + System.out.println("RUN EVERY minimumTimeMillis: " + minimumTimeMillis); + System.out.println("RUN EVERY healthy: " + fileSystemHealthCheck.execute().isHealthy()); + + try { + Files.walkFileTree(rootFolder, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException { + if (!StringUtils.startsWithAny(path.getFileName().toString(), PREFIXES_SAFE_TO_REMOVE)) { + return FileVisitResult.CONTINUE; + } + + File file = path.toFile(); + + System.out.println(file + " - " + file.lastModified()); + + return FileVisitResult.CONTINUE; + } + }); + } catch (Exception e) { + e.printStackTrace(); + } finally { + + } + } +} diff --git a/src/main/java/com/takipi/oss/storage/resources/admin/CleanupResource.java b/src/main/java/com/takipi/oss/storage/resources/admin/CleanupResource.java new file mode 100644 index 0000000..06cccb9 --- /dev/null +++ b/src/main/java/com/takipi/oss/storage/resources/admin/CleanupResource.java @@ -0,0 +1,37 @@ +package com.takipi.oss.storage.resources.admin; + +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.codahale.metrics.annotation.Timed; +import com.takipi.oss.storage.TakipiStorageConfiguration; +import com.takipi.oss.storage.jobs.PeriodicCleanupJob; + +@Path("/storage/v1/admin/clean") +@Consumes(MediaType.TEXT_PLAIN) +@Produces(MediaType.TEXT_PLAIN) +public class CleanupResource { + private static final Logger logger = LoggerFactory.getLogger(CleanupResource.class); + + private final TakipiStorageConfiguration configuration; + + public CleanupResource(TakipiStorageConfiguration configuration) { + this.configuration = configuration; + } + + @GET + @Timed + public Response get() { + PeriodicCleanupJob periodicCleanupJob = new PeriodicCleanupJob(); + periodicCleanupJob.configure(configuration); + periodicCleanupJob.run(); + return Response.ok("cleaned").build(); + } +} diff --git a/src/main/java/com/takipi/oss/storage/resources/fs/JsonSimpleSearchStorageResource.java b/src/main/java/com/takipi/oss/storage/resources/fs/JsonSimpleSearchStorageResource.java index 3268658..058f68a 100644 --- a/src/main/java/com/takipi/oss/storage/resources/fs/JsonSimpleSearchStorageResource.java +++ b/src/main/java/com/takipi/oss/storage/resources/fs/JsonSimpleSearchStorageResource.java @@ -24,92 +24,98 @@ @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) public class JsonSimpleSearchStorageResource extends SimpleFileSystemStorageResource { - private static final Logger logger = LoggerFactory.getLogger(JsonSimpleSearchStorageResource.class); + private static final Logger logger = LoggerFactory.getLogger(JsonSimpleSearchStorageResource.class); - public JsonSimpleSearchStorageResource(TakipiStorageConfiguration configuration) { - super(configuration); - } + public JsonSimpleSearchStorageResource(TakipiStorageConfiguration configuration) { + super(configuration); + } - @POST - @Timed - public Response post(SimpleSearchRequest request) { - try { - return handleResponse(request); - } catch (Exception e) { - return Response.serverError().entity("Problem simple searching").build(); - } - } + @POST + @Timed + public Response post(SimpleSearchRequest request) { + try { + return handleResponse(request); + } catch (Exception e) { + return Response.serverError().entity("Problem simple searching").build(); + } + } - private Response handleResponse(SimpleSearchRequest request) { - try { - File searchRoot = new File(fs.getRoot(), FilesystemUtil.fixPath(request.baseSearchPath)); - - ResourceFileCallback fileCallback = new ResourceFileCallback(request.name, request.preventDuplicates); - FilesystemUtil.listFilesRecursively(searchRoot, fileCallback); - File result = fileCallback.getFoundFile(); - - if (result == null) { - return searchFailed(request.name); - } - - String relFSPath = result.getAbsolutePath().replace(fs.getRoot().getAbsolutePath(), ""); - String data = FilesystemUtil.read(fs, relFSPath, request.encodingType); - - if (data == null) { - return searchFailed(request.name); - } - - return Response.ok(new SimpleSearchResponse(data, relFSPath.replace(request.name, ""))).build(); - - } catch (Exception e) { - logger.error("Problem getting: " + request.name, e); - return Response.serverError().entity("Problem getting " + request.name).build(); - } - } - - private Response searchFailed(String name) { - logger.warn("File not found: {}", name); - return Response.status(404).entity("File not found" + name).build(); - } - - private static class ResourceFileCallback implements Predicate - { - private final String resourceName; - private final boolean preventDuplicates; - - private File foundFile; - - protected ResourceFileCallback(String resourceName, boolean preventDuplicates) - { - this.resourceName = resourceName; - this.preventDuplicates = preventDuplicates; - - this.foundFile = null; - } - - @Override - public boolean apply(File file) - { - if (!resourceName.equals(file.getName())) - { - return false; - } - - if ((preventDuplicates) && - (foundFile != null)) - { - foundFile = null; // never find more than one result if preventing duplicates - return true; - } - - foundFile = file; - - return !preventDuplicates; // if we don't prevent duplicates, we stop right now - } - - public File getFoundFile() - { - return foundFile; - } - } + private Response handleResponse(SimpleSearchRequest request) { + try { + File searchRoot = new File(fs.getRoot(), FilesystemUtil.fixPath(request.baseSearchPath)); + + ResourceFileCallback fileCallback = new ResourceFileCallback(request.name, request.preventDuplicates); + FilesystemUtil.listFilesRecursively(searchRoot, fileCallback); + File result = fileCallback.getFoundFile(); + + if (result == null) { + return searchFailed(request.name); + } + + String relFSPath = result.getAbsolutePath().replace(fs.getRoot().getAbsolutePath(), ""); + String data = FilesystemUtil.read(fs, relFSPath, request.encodingType); + + if (data == null) { + return searchFailed(request.name); + } + + return Response.ok(new SimpleSearchResponse(data, relFSPath.replace(request.name, ""))).build(); + + } catch (Exception e) { + logger.error("Problem getting: " + request.name, e); + return Response.serverError().entity("Problem getting " + request.name).build(); + } + } + + private Response searchFailed(String name) { + logger.warn("File not found: {}", name); + return Response.status(404).entity("File not found" + name).build(); + } + + private static class ResourceFileCallback implements Predicate + { + private final String resourceName; + private final boolean preventDuplicates; + + private File foundFile; + + protected ResourceFileCallback(String resourceName, boolean preventDuplicates) + { + this.resourceName = resourceName; + this.preventDuplicates = preventDuplicates; + + this.foundFile = null; + } + + @Override + public boolean apply(File file) + { + return test(file); + } + + @Override + public boolean test(File file) + { + if (!resourceName.equals(file.getName())) + { + return false; + } + + if ((preventDuplicates) && + (foundFile != null)) + { + foundFile = null; // never find more than one result if preventing duplicates + return true; + } + + foundFile = file; + + return !preventDuplicates; // if we don't prevent duplicates, we stop right now + } + + public File getFoundFile() + { + return foundFile; + } + } } From a01dade5bd66d23c6c47bd7005f49ab5f004cbe1 Mon Sep 17 00:00:00 2001 From: David Levanon Date: Wed, 11 Apr 2018 20:50:29 +0300 Subject: [PATCH 2/4] almost finished --- settings.yml | 6 +- .../storage/data/status/MachineStatus.java | 30 +++++ .../fs/folder/FolderFilesystemHealth.java | 67 ++++++---- .../storage/health/FilesystemHealthCheck.java | 31 +++-- .../oss/storage/jobs/PeriodicCleanupJob.java | 124 ++++++++++++++---- .../resources/diag/StatusStorageResource.java | 10 ++ 6 files changed, 204 insertions(+), 64 deletions(-) diff --git a/settings.yml b/settings.yml index 58cbaed..28f191c 100644 --- a/settings.yml +++ b/settings.yml @@ -1,8 +1,8 @@ -folderPath: /opt/takipi-storage/storage -maxUsedStoragePercentage: 0.95 +folderPath: /home/david/temp/temp/1523379396 +maxUsedStoragePercentage: 0.90 enableCors: true corsOrigins: "*" -retentionPeriodDays: 15 +retentionPeriodDays: 92 server: # softNofileLimit: 1000 diff --git a/src/main/java/com/takipi/oss/storage/data/status/MachineStatus.java b/src/main/java/com/takipi/oss/storage/data/status/MachineStatus.java index 4c8ccf1..24f6569 100644 --- a/src/main/java/com/takipi/oss/storage/data/status/MachineStatus.java +++ b/src/main/java/com/takipi/oss/storage/data/status/MachineStatus.java @@ -1,5 +1,8 @@ package com.takipi.oss.storage.data.status; +import java.io.File; +import java.util.List; + public class MachineStatus { private long hitsCount; @@ -24,6 +27,9 @@ public class MachineStatus private long heapSizeBytes; private long permGenSizeBytes; private String version; + private String lastCleanupStartTime; + private long lastCleanupDurationMillis; + private List lastCleanupRemovedFiles; public void setHitsCount(long hitsCount) { this.hitsCount = hitsCount; @@ -200,4 +206,28 @@ public String getVersion() { public void setVersion(String version) { this.version = version; } + + public String getLastCleanupStartTime() { + return lastCleanupStartTime; + } + + public void setLastCleanupStartTime(String lastCleanupStartTime) { + this.lastCleanupStartTime = lastCleanupStartTime; + } + + public void setLastCleanupDurationMillis(long lastCleanupDurationMillis) { + this.lastCleanupDurationMillis = lastCleanupDurationMillis; + } + + public long getLastCleanupDurationMillis() { + return lastCleanupDurationMillis; + } + + public void setLastCleanupRemovedFiles(List lastCleanupRemovedFiles) { + this.lastCleanupRemovedFiles = lastCleanupRemovedFiles; + } + + public List getLastCleanupRemovedFiles() { + return lastCleanupRemovedFiles; + } } diff --git a/src/main/java/com/takipi/oss/storage/fs/folder/FolderFilesystemHealth.java b/src/main/java/com/takipi/oss/storage/fs/folder/FolderFilesystemHealth.java index 742b10a..9f58495 100644 --- a/src/main/java/com/takipi/oss/storage/fs/folder/FolderFilesystemHealth.java +++ b/src/main/java/com/takipi/oss/storage/fs/folder/FolderFilesystemHealth.java @@ -5,29 +5,46 @@ import com.takipi.oss.storage.fs.api.FilesystemHealth; public class FolderFilesystemHealth implements FilesystemHealth { - protected final File root; - private final double maxUsedStoragePercentage; - - public FolderFilesystemHealth(String rootFolder, double maxUsedStoragePercentage) { - this.root = new File(rootFolder); - this.maxUsedStoragePercentage = maxUsedStoragePercentage; - - if (!healthy()) { - throw new IllegalStateException("Problem with path " + rootFolder); - } - } - - @Override - public boolean healthy() { - return (folderCheck() && maxUsedStorageCheck()); - } - - private boolean folderCheck() { - return ((this.root.canRead()) && (this.root.canWrite())); - } - - private boolean maxUsedStorageCheck() { - return ((maxUsedStoragePercentage >= 0) && (maxUsedStoragePercentage < 1) && ((this.root.getUsableSpace() / this.root - .getTotalSpace()) <= maxUsedStoragePercentage)); - } + protected final File root; + private final double maxUsedStoragePercentage; + + public FolderFilesystemHealth(String rootFolder, double maxUsedStoragePercentage) { + this.root = new File(rootFolder); + this.maxUsedStoragePercentage = maxUsedStoragePercentage; + + if (!folderCheck()) { + throw new IllegalStateException("Problem with path " + rootFolder + " can't read or write"); + } + + if (!maxUsedStorageCheck()) { + throw new IllegalStateException("Problem with path " + rootFolder + " max limit reached"); + } + } + + public double getMaxUsedStoragePercentage() { + return maxUsedStoragePercentage; + } + + @Override + public boolean healthy() { + return (folderCheck() && maxUsedStorageCheck()); + } + + private boolean folderCheck() { + return ((this.root.canRead()) && (this.root.canWrite())); + } + + private boolean maxUsedStorageCheck() { + if ((maxUsedStoragePercentage < 0) && + (maxUsedStoragePercentage > 1)) + { + return false; + } + + long totalSpace = this.root.getTotalSpace(); + long usedSpace = totalSpace - this.root.getUsableSpace(); + double freeSpacePercentage = (double)usedSpace / totalSpace; + + return freeSpacePercentage <= maxUsedStoragePercentage; + } } diff --git a/src/main/java/com/takipi/oss/storage/health/FilesystemHealthCheck.java b/src/main/java/com/takipi/oss/storage/health/FilesystemHealthCheck.java index 53d4735..81bb54e 100644 --- a/src/main/java/com/takipi/oss/storage/health/FilesystemHealthCheck.java +++ b/src/main/java/com/takipi/oss/storage/health/FilesystemHealthCheck.java @@ -2,23 +2,26 @@ import com.codahale.metrics.health.HealthCheck; import com.takipi.oss.storage.TakipiStorageConfiguration; -import com.takipi.oss.storage.fs.api.FilesystemHealth; import com.takipi.oss.storage.fs.folder.FolderFilesystemHealth; public class FilesystemHealthCheck extends HealthCheck { - private final FilesystemHealth fsh; + private final FolderFilesystemHealth fsh; - public FilesystemHealthCheck(TakipiStorageConfiguration configuration) { - this.fsh = new FolderFilesystemHealth(configuration.getFolderPath(), - configuration.getMaxUsedStoragePercentage()); - } + public FilesystemHealthCheck(TakipiStorageConfiguration configuration) { + this.fsh = new FolderFilesystemHealth(configuration.getFolderPath(), + configuration.getMaxUsedStoragePercentage()); + } - @Override - protected Result check() throws Exception { - if (fsh.healthy()) { - return Result.healthy(); - } else { - return Result.unhealthy("Problem with filesystem"); - } - } + @Override + protected Result check() throws Exception { + if (fsh.healthy()) { + return Result.healthy(); + } else { + return Result.unhealthy("Problem with filesystem"); + } + } + + public double getMaxUsedStoragePercentage() { + return fsh.getMaxUsedStoragePercentage(); + } } diff --git a/src/main/java/com/takipi/oss/storage/jobs/PeriodicCleanupJob.java b/src/main/java/com/takipi/oss/storage/jobs/PeriodicCleanupJob.java index a4663f4..bf13f75 100644 --- a/src/main/java/com/takipi/oss/storage/jobs/PeriodicCleanupJob.java +++ b/src/main/java/com/takipi/oss/storage/jobs/PeriodicCleanupJob.java @@ -1,5 +1,9 @@ package com.takipi.oss.storage.jobs; +import java.util.Date; +import java.util.List; +import java.util.LinkedList; +import java.text.SimpleDateFormat; import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -11,7 +15,11 @@ import java.nio.file.Paths; import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.commons.lang3.StringUtils; +import org.apache.commons.io.FileUtils; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; @@ -24,6 +32,8 @@ @Every("${cleanup}") public class PeriodicCleanupJob extends Job { + private static final Logger logger = LoggerFactory.getLogger(PeriodicCleanupJob.class); + private final String[] PREFIXES_SAFE_TO_REMOVE = new String[] { "HYB_HIT_", "HYB_CER_", @@ -32,6 +42,10 @@ public class PeriodicCleanupJob extends Job { "HYB_SAFE_" }; + private static final SimpleDateFormat timeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + public static volatile CleanupStats lastCleanupStats = null; + private Path rootFolder; private FilesystemHealthCheck fileSystemHealthCheck; private int retentionPeriodDays; @@ -58,34 +72,100 @@ public void run() { return; } + long startMillis = System.currentTimeMillis(); long retentionPeriodDaysInMillis = TimeUnit.MILLISECONDS.convert(retentionPeriodDays, TimeUnit.DAYS); long minimumTimeMillis = System.currentTimeMillis() - retentionPeriodDaysInMillis; + long currentRetentionPeriodDays = retentionPeriodDays; + List removedFiles = new LinkedList(); + + while (true) { + logger.info("About to clean storage files. (folder: {}) (retentionPeriodDays: {}) " + + "(maxUsedPercentage: {}) (minimumTimeMillis: {})", + rootFolder, currentRetentionPeriodDays, + fileSystemHealthCheck.getMaxUsedStoragePercentage(), minimumTimeMillis); + + try { + removedFiles.addAll(cleanFiles(minimumTimeMillis)); + } catch (Exception e) { + logger.error("An error occured during file cleanup", e); + break; + } + + if (fileSystemHealthCheck.execute().isHealthy()) { + break; + } + + if (currentRetentionPeriodDays < 1) { + logger.warn("Clean stopped, no more files to clean"); + break; + } + + // an optimization to save resources, if the initial retentionPeriodDays was big + // + if (currentRetentionPeriodDays > 20) { + currentRetentionPeriodDays = currentRetentionPeriodDays / 2; + } else { + currentRetentionPeriodDays--; + } + + retentionPeriodDaysInMillis = TimeUnit.MILLISECONDS.convert(currentRetentionPeriodDays, TimeUnit.DAYS); + minimumTimeMillis = System.currentTimeMillis() - retentionPeriodDaysInMillis; + } - System.out.println("RUN EVERY folder: " + rootFolder); - System.out.println("RUN EVERY period: " + retentionPeriodDays); - System.out.println("RUN EVERY retentionPeriodDaysInMillis: " + retentionPeriodDaysInMillis); - System.out.println("RUN EVERY minimumTimeMillis: " + minimumTimeMillis); - System.out.println("RUN EVERY healthy: " + fileSystemHealthCheck.execute().isHealthy()); + long durationMillis = System.currentTimeMillis() - startMillis; + + logger.info("Cleanup finished, (removed files: {})", removedFiles.size()); + PeriodicCleanupJob.lastCleanupStats = new CleanupStats(startMillis, durationMillis, removedFiles); + } + + private List cleanFiles(final long minimumTimeMillis) throws Exception { + final List removedFiles = new LinkedList(); - try { - Files.walkFileTree(rootFolder, new SimpleFileVisitor() { - @Override - public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException { - if (!StringUtils.startsWithAny(path.getFileName().toString(), PREFIXES_SAFE_TO_REMOVE)) { - return FileVisitResult.CONTINUE; - } - - File file = path.toFile(); - - System.out.println(file + " - " + file.lastModified()); - + Files.walkFileTree(rootFolder, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException { + if (!StringUtils.startsWithAny(path.getFileName().toString(), PREFIXES_SAFE_TO_REMOVE)) { return FileVisitResult.CONTINUE; } - }); - } catch (Exception e) { - e.printStackTrace(); - } finally { - + + File file = path.toFile(); + + if (file.lastModified() > minimumTimeMillis) { + return FileVisitResult.CONTINUE; + } + + if (FileUtils.deleteQuietly(file)) { + removedFiles.add(file); + } + + return FileVisitResult.CONTINUE; + } + }); + + return removedFiles; + } + + public class CleanupStats { + private final long startEpochTime; + private final long durationMillis; + private final List removedFiles; + + public CleanupStats(long startEpochTime, long durationMillis, List removedFiles) { + this.startEpochTime = startEpochTime; + this.durationMillis = durationMillis; + this.removedFiles = removedFiles; + } + + public String getFormattedStartTime() { + return timeFormat.format(new Date(startEpochTime)); + } + + public long getDurationMillis() { + return durationMillis; + } + + public List getRemovedFiles() { + return removedFiles; } } } diff --git a/src/main/java/com/takipi/oss/storage/resources/diag/StatusStorageResource.java b/src/main/java/com/takipi/oss/storage/resources/diag/StatusStorageResource.java index 67c1534..740b2e2 100644 --- a/src/main/java/com/takipi/oss/storage/resources/diag/StatusStorageResource.java +++ b/src/main/java/com/takipi/oss/storage/resources/diag/StatusStorageResource.java @@ -19,6 +19,7 @@ import com.takipi.oss.storage.TakipiStorageConfiguration; import com.takipi.oss.storage.data.status.MachineStatus; import com.takipi.oss.storage.helper.StatusUtil; +import com.takipi.oss.storage.jobs.PeriodicCleanupJob; @Path("/storage/v1/diag/status") @Consumes(MediaType.TEXT_PLAIN) @@ -55,6 +56,7 @@ public Response post() { collectMachineInfo(machineStatus); collectDataInfo(machineStatus); + collectLastCleanupInfo(machineStatus); return Response.ok(machineStatus).build(); } catch (Exception e) { @@ -80,6 +82,14 @@ private void collectDataInfo(MachineStatus machineStatus) { machineStatus.setFreeSpaceLeftBytes(directory.getFreeSpace()); } + private void collectLastCleanupInfo(MachineStatus machineStatus) { + PeriodicCleanupJob.CleanupStats cleanupStats = PeriodicCleanupJob.lastCleanupStats; + + machineStatus.setLastCleanupStartTime(cleanupStats.getFormattedStartTime()); + machineStatus.setLastCleanupDurationMillis(cleanupStats.getDurationMillis()); + machineStatus.setLastCleanupRemovedFiles(cleanupStats.getRemovedFiles()); + } + private Map traverseTreeForData(File directory) { Map map = initializeMapForData(); From 521f7f46516a9224aa302f5884e2fd4d9fef2fad Mon Sep 17 00:00:00 2001 From: David Levanon Date: Thu, 12 Apr 2018 11:51:50 +0300 Subject: [PATCH 3/4] revert folder path location --- settings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.yml b/settings.yml index 28f191c..c07dbae 100644 --- a/settings.yml +++ b/settings.yml @@ -1,4 +1,4 @@ -folderPath: /home/david/temp/temp/1523379396 +folderPath: /opt/takipi-storage/storage maxUsedStoragePercentage: 0.90 enableCors: true corsOrigins: "*" From 822734ee959df99e79336714277d15e959e531af Mon Sep 17 00:00:00 2001 From: David Levanon Date: Thu, 12 Apr 2018 19:18:48 +0300 Subject: [PATCH 4/4] fix docker settings --- docker/settings.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/settings.yml b/docker/settings.yml index 4dd3045..15ae6cb 100644 --- a/docker/settings.yml +++ b/docker/settings.yml @@ -2,6 +2,7 @@ folderPath: /opt/takipi-storage/storage maxUsedStoragePercentage: 0.95 enableCors: true corsOrigins: "*" +retentionPeriodDays: 92 server: applicationConnectors: @@ -16,6 +17,10 @@ server: currentLogFilename: ./storage/log/access.log archivedLogFilenamePattern: ./storage/log/access.%d.log.gz archivedFileCount: 14 + +jobs: + cleanup: 6h + logging: level: INFO loggers: