diff --git a/api/src/main/java/ca/bc/gov/educ/api/distribution/config/EducDistributionApiConfig.java b/api/src/main/java/ca/bc/gov/educ/api/distribution/config/EducDistributionApiConfig.java index 62698958..b4dcf7e3 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/distribution/config/EducDistributionApiConfig.java +++ b/api/src/main/java/ca/bc/gov/educ/api/distribution/config/EducDistributionApiConfig.java @@ -1,8 +1,6 @@ package ca.bc.gov.educ.api.distribution.config; -import ca.bc.gov.educ.api.distribution.util.DeleteExpiredFilesFileVisitorImpl; import org.modelmapper.ModelMapper; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -13,8 +11,7 @@ import org.springframework.web.reactive.function.client.WebClient; import reactor.netty.http.client.HttpClient; -import java.nio.file.*; -import java.time.LocalDateTime; +import java.io.FileFilter; @Configuration @IntegrationComponentScan @@ -42,8 +39,16 @@ public RestTemplate restTemplate(RestTemplateBuilder builder) { return builder.build(); } - @Bean("TmpCacheFileVisitor") - public FileVisitor createCleanTmpCacheFilesFileVisitor(@Value("${scheduler.clean-tmp-cache-filter}") String filter, @Value("${scheduler.clean-tmp-cache-interval-in-days}") int days) { - return new DeleteExpiredFilesFileVisitorImpl(filter, LocalDateTime.now().minusDays(days)); + + @Bean + public FileFilter createFileFilter(){ + return pathname -> { + String name = pathname.getName(); + return !name.startsWith(".") && + !name.contains("undertow") && + !name.contains("hsperfdata"); + }; } + + } diff --git a/api/src/main/java/ca/bc/gov/educ/api/distribution/config/ScheduledTasksConfig.java b/api/src/main/java/ca/bc/gov/educ/api/distribution/config/ScheduledTasksConfig.java index a77a30a5..fd0ad43c 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/distribution/config/ScheduledTasksConfig.java +++ b/api/src/main/java/ca/bc/gov/educ/api/distribution/config/ScheduledTasksConfig.java @@ -1,48 +1,45 @@ package ca.bc.gov.educ.api.distribution.config; import ca.bc.gov.educ.api.distribution.util.EducDistributionApiConstants; +import ca.bc.gov.educ.api.distribution.util.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Configuration; import org.springframework.scheduling.annotation.Scheduled; -import java.io.IOException; -import java.nio.file.FileVisitor; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; +import java.io.File; +import java.io.FileFilter; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Arrays; +import java.util.List; @Configuration public class ScheduledTasksConfig { private static final Logger logger = LoggerFactory.getLogger(ScheduledTasksConfig.class); - EducDistributionApiConstants educDistributionApiConstants; - FileVisitor fileVisitor; + FileFilter fileFilter; @Autowired - public ScheduledTasksConfig(@Qualifier("TmpCacheFileVisitor") FileVisitor fileVisitor, EducDistributionApiConstants educDistributionApiConstants) { - this.fileVisitor = fileVisitor; - this.educDistributionApiConstants = educDistributionApiConstants; + public ScheduledTasksConfig(FileFilter fileFilter) { + this.fileFilter = fileFilter; } - /** - * Removes artifacts from the /tmp directory that are expired - */ @Scheduled(cron = "${scheduler.clean-tmp-cache-cron}") public void cleanTmpCacheFiles() { - logger.debug("Running clean cache..."); - Path startingDir = Paths.get(educDistributionApiConstants.TMP_DIR); - if(Files.exists(startingDir)){ - try { - Files.walkFileTree(startingDir, fileVisitor); - } catch (IOException e) { - logger.error("ScheduledTasksConfig: There was an error removing file cache: {}", e.getLocalizedMessage()); + File dir = new File(EducDistributionApiConstants.TMP_DIR); + List files = Arrays.asList(dir.listFiles(this.fileFilter)); + LocalDateTime fileExpiry = LocalDateTime.now().minusHours(3); + files.forEach(file -> { + LocalDateTime lastModified = LocalDateTime.ofInstant(Instant.ofEpochMilli(file.lastModified()), ZoneId.systemDefault()); + if(lastModified.isBefore(fileExpiry)){ + logger.debug("Removing file or directory: {}", file.getName()); + IOUtils.removeFileOrDirectory(file); } - } + }); } - } diff --git a/api/src/main/java/ca/bc/gov/educ/api/distribution/model/dto/Student.java b/api/src/main/java/ca/bc/gov/educ/api/distribution/model/dto/Student.java index 38af766d..520760e6 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/distribution/model/dto/Student.java +++ b/api/src/main/java/ca/bc/gov/educ/api/distribution/model/dto/Student.java @@ -5,8 +5,8 @@ import java.io.Serializable; import java.time.LocalDate; -import java.time.LocalDateTime; import java.util.ArrayList; +import java.util.Date; import java.util.List; import java.util.Objects; @@ -19,7 +19,7 @@ public class Student implements Serializable { private String gender; private String citizenship; private LocalDate birthdate; - private LocalDateTime lastUpdateDate; + private Date lastUpdateDate; private Address address; private String grade; private String gradProgram; @@ -102,11 +102,11 @@ public void setBirthdate(LocalDate value) { } @JsonFormat(pattern="yyyy-MM-dd HH:mm:ss") - public LocalDateTime getLastUpdateDate() { + public Date getLastUpdateDate() { return lastUpdateDate; } - public void setLastUpdateDate(LocalDateTime lastUpdateDate) { + public void setLastUpdateDate(Date lastUpdateDate) { this.lastUpdateDate = lastUpdateDate; } diff --git a/api/src/main/java/ca/bc/gov/educ/api/distribution/process/MergeProcess.java b/api/src/main/java/ca/bc/gov/educ/api/distribution/process/MergeProcess.java index fbe7657d..b5bda1c5 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/distribution/process/MergeProcess.java +++ b/api/src/main/java/ca/bc/gov/educ/api/distribution/process/MergeProcess.java @@ -18,7 +18,9 @@ import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.sql.Date; import java.time.LocalDate; +import java.time.ZoneId; import java.util.ArrayList; import java.util.Comparator; import java.util.List; @@ -218,7 +220,7 @@ private Student prepareStudentObj(StudentCredentialDistribution scd, List { - - private static final Logger logger = LoggerFactory.getLogger(DeleteExpiredFilesFileVisitorImpl.class); - private final PathMatcher pathMatcher; - private final LocalDateTime fileExpiry; - - public DeleteExpiredFilesFileVisitorImpl(String filter, LocalDateTime fileExpiry){ - this.pathMatcher = FileSystems.getDefault().getPathMatcher("glob:" + filter); - this.fileExpiry = fileExpiry; - } - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - if(fileOrDirectoryIsExpired(file)){ - logger.debug("Deleting: {}", file.getFileName()); - Files.delete(file); - } - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFileFailed(Path file, IOException exc) { - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { - if(fileOrDirectoryIsExpired(dir)){ - logger.debug("Deleting: {}", dir.getFileName()); - FileUtils.deleteDirectory(dir.toFile()); - } - return FileVisitResult.CONTINUE; - } - - /** - * Checks whether file or directory is expired - * @param file the file to check - * @return whether or not to delete - */ - private boolean fileOrDirectoryIsExpired(Path file){ - if(!pathMatcher.matches(file.getFileName())){ - File theFile = file.toFile(); - LocalDateTime lastModified = LocalDateTime.ofInstant(Instant.ofEpochMilli(theFile.lastModified()), ZoneId.systemDefault()); - return lastModified.isBefore(fileExpiry); - } - return false; - } -} diff --git a/api/src/main/java/ca/bc/gov/educ/api/distribution/util/EducDistributionApiConstants.java b/api/src/main/java/ca/bc/gov/educ/api/distribution/util/EducDistributionApiConstants.java index d272a51d..343a28b3 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/distribution/util/EducDistributionApiConstants.java +++ b/api/src/main/java/ca/bc/gov/educ/api/distribution/util/EducDistributionApiConstants.java @@ -28,8 +28,6 @@ public class EducDistributionApiConstants { //Grad2-1931 - mchintha public static final String DATE_FORMAT_YYYYMMDD = "yyyyMMdd"; public static final String TMP_DIR = "/tmp"; - - public static final String BATCH_DIR = TMP_DIR + "/Batch"; public static final String DEL = "/"; public static final String FILES_FOLDER_STRUCTURE = "/Batch/PSI/"; public static final String TRANSMISSION_MODE_FTP = "FTP"; @@ -165,13 +163,4 @@ public class EducDistributionApiConstants { private int threadPoolMaxSize = 15; - @Value("${scheduler.clean-tmp-cache-cron}") - private String cleanTmpCacheCron; - - @Value("${scheduler.clean-tmp-cache-interval-in-days}") - private int cleanTmpCacheIntervalInDays; - - @Value("${scheduler.clean-tmp-cache-filter}") - private String cleanTmpCacheFilter; - } diff --git a/api/src/main/java/ca/bc/gov/educ/api/distribution/util/IOUtils.java b/api/src/main/java/ca/bc/gov/educ/api/distribution/util/IOUtils.java index 44ce5c0a..8bd85874 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/distribution/util/IOUtils.java +++ b/api/src/main/java/ca/bc/gov/educ/api/distribution/util/IOUtils.java @@ -7,7 +7,9 @@ import java.io.File; import java.io.IOException; -import java.nio.file.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermissions; diff --git a/api/src/main/resources/application.yaml b/api/src/main/resources/application.yaml index 282d62ae..ccdef19e 100644 --- a/api/src/main/resources/application.yaml +++ b/api/src/main/resources/application.yaml @@ -170,10 +170,5 @@ splunk: enabled: ${ENABLE_SPLUNK_LOG_HELPER} scheduler: - # how often to check for expired files clean-tmp-cache-cron: ${CLEAN_TMP_CACHE_CRON} - # the age in days of files that are considered 'expired' - clean-tmp-cache-interval-in-days: ${CLEAN_TMP_CACHE_INTERVAL} - # a glob filter to ignore certain files or directories - clean-tmp-cache-filter: ${CLEAN_TEMP_CACHE_FILTER} diff --git a/api/src/test/java/ca/bc/gov/educ/api/distribution/util/CacheClearingTest.java b/api/src/test/java/ca/bc/gov/educ/api/distribution/util/CacheClearingTest.java deleted file mode 100644 index a0dc95fd..00000000 --- a/api/src/test/java/ca/bc/gov/educ/api/distribution/util/CacheClearingTest.java +++ /dev/null @@ -1,167 +0,0 @@ -package ca.bc.gov.educ.api.distribution.util; - -import ca.bc.gov.educ.api.distribution.config.EducDistributionApiConfig; -import org.apache.commons.io.FileUtils; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.nio.file.*; -import java.nio.file.attribute.FileTime; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.util.HashMap; -import java.util.Map; - -public class CacheClearingTest { - - private enum FILES { - HSPERF, - UNDERT, - NFS, - JAV, - BATCH_DIR, - FTP_DIR, - FTP_SUB, - FTP_SUB_FILE, - FTP_ZIP, - PAPER_DIR, - PAPER_SUB, - PAPER_SUB_FILE, - PAPER_ZIP - } - - private String tmpDir; - private final String FILE_SEP = File.separator; - private final Map files = new HashMap<>(); - - @Before - public void createCache() throws IOException { - this.tmpDir = Files.createTempDirectory("cacheTestDir").toFile().getAbsolutePath(); - createTmpCacheFiles(); - EducDistributionApiConfig config = new EducDistributionApiConfig(); - FileVisitor fileVisitor = config.createCleanTmpCacheFilesFileVisitor("{hsperf*,undertow*,.nfs*,.java*,.nfs*,Batch,FTP,PAPER}", 1); - Path startingDir = Paths.get(this.tmpDir); - if (Files.exists(startingDir)) { - Files.walkFileTree(startingDir, fileVisitor); - } - } - - @After - public void removeCache() throws IOException { - Path tmp = Paths.get(tmpDir); - FileUtils.deleteDirectory(tmp.toFile()); - } - - - @Test - public void testFilesAreExpiredButShouldNotBeDeleted() { - for (Map.Entry entry : this.files.entrySet()) { - FILES f = entry.getKey(); - if (f != FILES.FTP_SUB && - f != FILES.FTP_SUB_FILE && - f != FILES.FTP_ZIP && - f != FILES.PAPER_SUB && - f != FILES.PAPER_SUB_FILE && - f != FILES.PAPER_ZIP) { - Assert.assertTrue(Files.exists(entry.getValue())); - } - } - } - - @Test - public void testFilesAreExpiredAndShouldBeDeleted() { - for (Map.Entry entry : this.files.entrySet()) { - FILES f = entry.getKey(); - if (f == FILES.FTP_SUB_FILE || - f == FILES.PAPER_ZIP) { - Assert.assertFalse(Files.exists(entry.getValue())); - } - } - } - - @Test - public void testFilesAreNotExpiredThatShouldNotBeDeleted() { - for (Map.Entry entry : this.files.entrySet()) { - FILES f = entry.getKey(); - if (f == FILES.FTP_ZIP || f == FILES.PAPER_SUB || f == FILES.PAPER_SUB_FILE) { - Assert.assertTrue(Files.exists(entry.getValue())); - } - } - } - - private void createTmpCacheFiles() throws IOException { - // create an expiry time of one day for testing - FileTime expiry = FileTime.from(LocalDateTime.now().minusDays(2).toInstant(ZoneOffset.UTC)); - - // create directory structure - // - // hsperfdata_101279000 (HSPERF expired, should not be deleted) - // undertow-docbase.8080.11764866489534620758 (UNDERT expired, should not be deleted) - // .nfs000000002abc01e400010418 (NFS expired, should not be deleted) - // .java_pid1 (JAV expired, should not be deleted) - // Batch (BATCH_DIR expired, should not be deleted) - // |_FTP (FTP_DIR expired, should not be deleted) - // |_18873 (FTP_SUB expired, should be deleted) - // | FileA.pdf (FTP_SUB_FILE expired, should be deleted) - // | EDGRAD.BATCH.18873.zip (FTP_ZIP not expired, should not be deleted) - // |_PAPER (PAPER_DIR expired should not be deleted) - // |_18874 (PAPER_SUB not expired, should not be deleted) - // | FileB.pdf (PAPER_SUB_FILE not expired, should not be deleted) - // | EDGRAD.BATCH.18874.zip (PAPER_ZIP expired, should be deleted) - // - // add files and update expiry times on some - files.put(FILES.HSPERF, Paths.get(tmpDir + FILE_SEP + "hsperfdata_101279000")); - files.put(FILES.UNDERT, Paths.get(tmpDir + FILE_SEP + "undertow-docbase.8080.11764866489534620758")); - files.put(FILES.NFS, Paths.get(tmpDir + FILE_SEP + ".nfs000000002abc01e400010418")); - files.put(FILES.JAV, Paths.get(tmpDir + FILE_SEP + ".java_pid1")); - - Files.createFile(files.get(FILES.HSPERF)); - Files.createFile(files.get(FILES.UNDERT)); - Files.createFile(files.get(FILES.NFS)); - Files.createFile(files.get(FILES.JAV)); - - Files.setLastModifiedTime(files.get(FILES.HSPERF), expiry); - Files.setLastModifiedTime(files.get(FILES.UNDERT), expiry); - Files.setLastModifiedTime(files.get(FILES.NFS), expiry); - Files.setLastModifiedTime(files.get(FILES.JAV), expiry); - - files.put(FILES.BATCH_DIR, Paths.get(tmpDir + FILE_SEP + "Batch")); - files.put(FILES.FTP_DIR, Paths.get(files.get(FILES.BATCH_DIR) + FILE_SEP + "FTP")); - files.put(FILES.PAPER_DIR, Paths.get(files.get(FILES.BATCH_DIR) + FILE_SEP + "PAPER")); - Files.createDirectory(files.get(FILES.BATCH_DIR)); - Files.createDirectory(files.get(FILES.FTP_DIR)); - Files.createDirectory(files.get(FILES.PAPER_DIR)); - - files.put(FILES.FTP_SUB, Path.of(files.get(FILES.FTP_DIR) + FILE_SEP + "18873")); - Files.createDirectory(files.get(FILES.FTP_SUB)); - - - files.put(FILES.FTP_SUB_FILE, Path.of(files.get(FILES.FTP_SUB) + FILE_SEP + "FileA.pdf")); - Files.createFile(files.get(FILES.FTP_SUB_FILE)); - - - files.put(FILES.FTP_ZIP, Path.of(files.get(FILES.FTP_DIR) + FILE_SEP + "EDGRAD.BATCH.18873.zip")); - Files.createFile(files.get(FILES.FTP_ZIP)); - - files.put(FILES.PAPER_SUB, Path.of(files.get(FILES.PAPER_DIR) + FILE_SEP + "18874")); - Files.createDirectory(files.get(FILES.PAPER_SUB)); - - files.put(FILES.PAPER_SUB_FILE, Path.of(files.get(FILES.PAPER_SUB) + FILE_SEP + "FileB.pdf")); - Files.createFile(files.get(FILES.PAPER_SUB_FILE)); - - files.put(FILES.PAPER_ZIP, Path.of(files.get(FILES.PAPER_DIR) + FILE_SEP + "EDGRAD.BATCH.18874.zip")); - Files.createFile(files.get(FILES.PAPER_ZIP)); - - Files.setLastModifiedTime(files.get(FILES.FTP_SUB_FILE), expiry); - Files.setLastModifiedTime(files.get(FILES.FTP_SUB), expiry); - Files.setLastModifiedTime(files.get(FILES.FTP_DIR), expiry); - Files.setLastModifiedTime(files.get(FILES.PAPER_ZIP), expiry); - Files.setLastModifiedTime(files.get(FILES.PAPER_DIR), expiry); - Files.setLastModifiedTime(files.get(FILES.BATCH_DIR), expiry); - } - -} diff --git a/api/src/test/resources/application.yaml b/api/src/test/resources/application.yaml index 7f9ad092..cd725935 100644 --- a/api/src/test/resources/application.yaml +++ b/api/src/test/resources/application.yaml @@ -173,6 +173,4 @@ splunk: enabled: false scheduler: - clean-tmp-cache-cron: 0 0 */4 * * * - clean-tmp-cache-interval-in-days: 1 - clean-tmp-cache-filter: '{hsperf*,undertow*,.nfs*,.java*,.nfs*,Batch,FTP,PAPER}' \ No newline at end of file + clean-tmp-cache-cron: 0 0 */4 * * * \ No newline at end of file