diff --git a/api/src/main/java/ca/bc/gov/educ/api/distribution/config/EducDistributionApiConfig.java b/api/src/main/java/ca/bc/gov/educ/api/distribution/config/EducDistributionApiConfig.java index 9077655a..71e3cedc 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/distribution/config/EducDistributionApiConfig.java +++ b/api/src/main/java/ca/bc/gov/educ/api/distribution/config/EducDistributionApiConfig.java @@ -1,9 +1,6 @@ package ca.bc.gov.educ.api.distribution.config; -import ca.bc.gov.educ.api.distribution.util.DeleteExpiredFilesFileVisitorImpl; -import ca.bc.gov.educ.api.distribution.util.IOUtils; import org.modelmapper.ModelMapper; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -12,17 +9,7 @@ import org.springframework.web.reactive.function.client.WebClient; import reactor.netty.http.client.HttpClient; -import java.io.File; import java.io.FileFilter; -import java.io.IOException; -import java.nio.file.*; -import java.nio.file.attribute.BasicFileAttributes; -import java.time.Instant; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.util.Arrays; -import java.util.List; @Configuration public class EducDistributionApiConfig { @@ -48,8 +35,16 @@ public RestTemplate restTemplate(RestTemplateBuilder builder) { return builder.build(); } - @Bean("TmpCacheFileVisitor") - public FileVisitor createCleanTmpCacheFilesFileVisitor(@Value("${scheduler.clean-tmp-cache-ignore}") List ignore, @Value("${scheduler.clean-tmp-cache-interval-in-days}") int days) { - return new DeleteExpiredFilesFileVisitorImpl(ignore, LocalDateTime.now().minusDays(days)); + + @Bean + public FileFilter createFileFilter(){ + return pathname -> { + String name = pathname.getName(); + return !name.startsWith(".") && + !name.contains("undertow") && + !name.contains("hsperfdata"); + }; } + + } diff --git a/api/src/main/java/ca/bc/gov/educ/api/distribution/config/ScheduledTasksConfig.java b/api/src/main/java/ca/bc/gov/educ/api/distribution/config/ScheduledTasksConfig.java index fd040909..fd0ad43c 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/distribution/config/ScheduledTasksConfig.java +++ b/api/src/main/java/ca/bc/gov/educ/api/distribution/config/ScheduledTasksConfig.java @@ -1,45 +1,45 @@ package ca.bc.gov.educ.api.distribution.config; import ca.bc.gov.educ.api.distribution.util.EducDistributionApiConstants; +import ca.bc.gov.educ.api.distribution.util.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Configuration; import org.springframework.scheduling.annotation.Scheduled; -import java.io.IOException; -import java.nio.file.FileVisitor; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; +import java.io.File; +import java.io.FileFilter; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Arrays; +import java.util.List; @Configuration public class ScheduledTasksConfig { private static final Logger logger = LoggerFactory.getLogger(ScheduledTasksConfig.class); - EducDistributionApiConstants educDistributionApiConstants; - FileVisitor fileVisitor; + FileFilter fileFilter; @Autowired - public ScheduledTasksConfig(@Qualifier("TmpCacheFileVisitor") FileVisitor fileVisitor, EducDistributionApiConstants educDistributionApiConstants) { - this.fileVisitor = fileVisitor; - this.educDistributionApiConstants = educDistributionApiConstants; + public ScheduledTasksConfig(FileFilter fileFilter) { + this.fileFilter = fileFilter; } @Scheduled(cron = "${scheduler.clean-tmp-cache-cron}") public void cleanTmpCacheFiles() { - Path startingDir = Paths.get(educDistributionApiConstants.getCleanTmpCacheBaseDir()); - if(Files.exists(startingDir)){ - try { - Files.walkFileTree(startingDir, fileVisitor); - } catch (IOException e) { - logger.error("ScheduledTasksConfig: There was an error removing file cache: {}", e.getLocalizedMessage()); + File dir = new File(EducDistributionApiConstants.TMP_DIR); + List files = Arrays.asList(dir.listFiles(this.fileFilter)); + LocalDateTime fileExpiry = LocalDateTime.now().minusHours(3); + files.forEach(file -> { + LocalDateTime lastModified = LocalDateTime.ofInstant(Instant.ofEpochMilli(file.lastModified()), ZoneId.systemDefault()); + if(lastModified.isBefore(fileExpiry)){ + logger.debug("Removing file or directory: {}", file.getName()); + IOUtils.removeFileOrDirectory(file); } - } - + }); } - } diff --git a/api/src/main/java/ca/bc/gov/educ/api/distribution/util/DeleteExpiredFilesFileVisitorImpl.java b/api/src/main/java/ca/bc/gov/educ/api/distribution/util/DeleteExpiredFilesFileVisitorImpl.java deleted file mode 100644 index b732ca77..00000000 --- a/api/src/main/java/ca/bc/gov/educ/api/distribution/util/DeleteExpiredFilesFileVisitorImpl.java +++ /dev/null @@ -1,74 +0,0 @@ -package ca.bc.gov.educ.api.distribution.util; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.FileVisitor; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.BasicFileAttributes; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.util.List; - -public class DeleteExpiredFilesFileVisitorImpl implements FileVisitor { - - private static final Logger logger = LoggerFactory.getLogger(DeleteExpiredFilesFileVisitorImpl.class); - private final List skipList; - private final LocalDateTime fileExpiry; - - /** - * Constructor - * @param skipList - a list of filenames or directories NOT to delete - * @param fileExpiry - file expiry - */ - public DeleteExpiredFilesFileVisitorImpl(List skipList, LocalDateTime fileExpiry){ - this.skipList = skipList; - this.fileExpiry = fileExpiry; - } - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - if(fileOrDirectoryIsExpired(file)){ - logger.info("Deleting: {}", file.getFileName()); - Files.delete(file); - } - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFileFailed(Path file, IOException exc) { - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { - if(fileOrDirectoryIsExpired(dir)){ - logger.info("Deleting: {}", dir.getFileName()); - Files.delete(dir); - } - return FileVisitResult.CONTINUE; - } - - /** - * Checks whether file or directory is expired - * @param file the file to check - * @return whether or not to delete - */ - private boolean fileOrDirectoryIsExpired(Path file){ - if(!skipList.contains(file.getFileName().toString().toLowerCase())){ - File theFile = file.toFile(); - LocalDateTime lastModified = LocalDateTime.ofInstant(Instant.ofEpochMilli(theFile.lastModified()), ZoneId.systemDefault()); - return lastModified.isBefore(fileExpiry); - } - return false; - } -} diff --git a/api/src/main/java/ca/bc/gov/educ/api/distribution/util/EducDistributionApiConstants.java b/api/src/main/java/ca/bc/gov/educ/api/distribution/util/EducDistributionApiConstants.java index 141c9e99..72cae334 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/distribution/util/EducDistributionApiConstants.java +++ b/api/src/main/java/ca/bc/gov/educ/api/distribution/util/EducDistributionApiConstants.java @@ -5,8 +5,6 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import java.util.List; - @Component @Getter @Setter @@ -27,8 +25,6 @@ public class EducDistributionApiConstants { //Grad2-1931 - mchintha public static final String DATE_FORMAT_YYYYMMDD = "yyyyMMdd"; public static final String TMP_DIR = "/tmp"; - - public static final String BATCH_DIR = TMP_DIR + "/Batch"; public static final String DEL = "/"; public static final String FILES_FOLDER_STRUCTURE = "/Batch/PSI/"; public static final String TRANSMISSION_MODE_FTP = "FTP"; @@ -158,16 +154,4 @@ public class EducDistributionApiConstants { private int threadPoolMaxSize = 15; - @Value("${scheduler.clean-tmp-cache-cron}") - private String cleanTmpCacheCron; - - @Value("${scheduler.clean-tmp-cache-interval-in-days}") - private int cleanTmpCacheIntervalInDays; - - @Value("${scheduler.clean-tmp-cache-base-dir}") - private String cleanTmpCacheBaseDir; - - @Value("${scheduler.clean-tmp-cache-ignore}") - private List cleanTmpCacheIgnore; - } diff --git a/api/src/main/java/ca/bc/gov/educ/api/distribution/util/IOUtils.java b/api/src/main/java/ca/bc/gov/educ/api/distribution/util/IOUtils.java index 44ce5c0a..8bd85874 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/distribution/util/IOUtils.java +++ b/api/src/main/java/ca/bc/gov/educ/api/distribution/util/IOUtils.java @@ -7,7 +7,9 @@ import java.io.File; import java.io.IOException; -import java.nio.file.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermissions; diff --git a/api/src/main/resources/application.yaml b/api/src/main/resources/application.yaml index c9112369..38a2fb78 100644 --- a/api/src/main/resources/application.yaml +++ b/api/src/main/resources/application.yaml @@ -168,11 +168,4 @@ splunk: scheduler: clean-tmp-cache-cron: ${CLEAN_TMP_CACHE_CRON} - clean-tmp-cache-interval-in-days: 5 - clean-tmp-cache-base-dir: /tmp/Batch - clean-tmp-cache-ignore: - - batch - - psi - - ftp - - paper diff --git a/api/src/test/resources/application.yaml b/api/src/test/resources/application.yaml index 7e07a454..3da28341 100644 --- a/api/src/test/resources/application.yaml +++ b/api/src/test/resources/application.yaml @@ -170,11 +170,4 @@ splunk: enabled: false scheduler: - clean-tmp-cache-cron: 0 0 */4 * * * - clean-tmp-cache-interval-in-days: 5 - clean-tmp-cache-base-dir: /tmp/Batch - clean-tmp-cache-ignore: - - batch - - psi - - ftp - - paper \ No newline at end of file + clean-tmp-cache-cron: 0 0 */4 * * * \ No newline at end of file