GRAD2-2218

api/src/main/java/ca/bc/gov/educ/api/distribution/config/EducDistributionApiConfig.java

@@ -1,6 +1,8 @@
 package ca.bc.gov.educ.api.distribution.config;
 
+import ca.bc.gov.educ.api.distribution.util.DeleteExpiredFilesFileVisitorImpl;
 import org.modelmapper.ModelMapper;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.web.client.RestTemplateBuilder;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
@@ -11,7 +13,8 @@
 import org.springframework.web.reactive.function.client.WebClient;
 import reactor.netty.http.client.HttpClient;
 
-import java.io.FileFilter;
+import java.nio.file.*;
+import java.time.LocalDateTime;
 
 @Configuration
 @IntegrationComponentScan
@@ -39,16 +42,8 @@ public RestTemplate restTemplate(RestTemplateBuilder builder) {
         return builder.build();
     }
 
-
-    @Bean
-    public FileFilter createFileFilter(){
-        return pathname -> {
-            String name = pathname.getName();
-            return !name.startsWith(".") &&
-                    !name.contains("undertow") &&
-                    !name.contains("hsperfdata");
-        };
+    @Bean("TmpCacheFileVisitor")
+    public FileVisitor<Path> createCleanTmpCacheFilesFileVisitor(@Value("${scheduler.clean-tmp-cache-filter}") String filter, @Value("${scheduler.clean-tmp-cache-interval-in-days}") int days) {
+        return new DeleteExpiredFilesFileVisitorImpl(filter, LocalDateTime.now().minusDays(days));
     }
-
-
 }

api/src/main/java/ca/bc/gov/educ/api/distribution/config/ScheduledTasksConfig.java

@@ -1,45 +1,48 @@
 package ca.bc.gov.educ.api.distribution.config;
 
 import ca.bc.gov.educ.api.distribution.util.EducDistributionApiConstants;
-import ca.bc.gov.educ.api.distribution.util.IOUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.scheduling.annotation.Scheduled;
 
-import java.io.File;
-import java.io.FileFilter;
-import java.time.Instant;
-import java.time.LocalDateTime;
-import java.time.ZoneId;
-import java.util.Arrays;
-import java.util.List;
+import java.io.IOException;
+import java.nio.file.FileVisitor;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 
 @Configuration
 public class ScheduledTasksConfig {
 
     private static final Logger logger = LoggerFactory.getLogger(ScheduledTasksConfig.class);
 
-    FileFilter fileFilter;
+    EducDistributionApiConstants educDistributionApiConstants;
+    FileVisitor<Path> fileVisitor;
 
     @Autowired
-    public ScheduledTasksConfig(FileFilter fileFilter) {
-        this.fileFilter = fileFilter;
+    public ScheduledTasksConfig(@Qualifier("TmpCacheFileVisitor") FileVisitor<Path> fileVisitor, EducDistributionApiConstants educDistributionApiConstants) {
+        this.fileVisitor = fileVisitor;
+        this.educDistributionApiConstants = educDistributionApiConstants;
     }
 
+    /**
+     * Removes artifacts from the /tmp directory that are expired
+     */
     @Scheduled(cron = "${scheduler.clean-tmp-cache-cron}")
     public void cleanTmpCacheFiles() {
-        File dir = new File(EducDistributionApiConstants.TMP_DIR);
-        List<File> files = Arrays.asList(dir.listFiles(this.fileFilter));
-        LocalDateTime fileExpiry = LocalDateTime.now().minusHours(3);
-        files.forEach(file -> {
-            LocalDateTime lastModified = LocalDateTime.ofInstant(Instant.ofEpochMilli(file.lastModified()), ZoneId.systemDefault());
-            if(lastModified.isBefore(fileExpiry)){
-                logger.debug("Removing file or directory: {}", file.getName());
-                IOUtils.removeFileOrDirectory(file);
+        logger.debug("Running clean cache...");
+        Path startingDir = Paths.get(educDistributionApiConstants.TMP_DIR);
+        if(Files.exists(startingDir)){
+            try {
+                Files.walkFileTree(startingDir, fileVisitor);
+            } catch (IOException e) {
+                logger.error("ScheduledTasksConfig: There was an error removing file cache: {}", e.getLocalizedMessage());
             }
-        });
+        }
     }
 
+
 }

api/src/main/java/ca/bc/gov/educ/api/distribution/util/DeleteExpiredFilesFileVisitorImpl.java

@@ -0,0 +1,66 @@
+package ca.bc.gov.educ.api.distribution.util;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.*;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+
+public class DeleteExpiredFilesFileVisitorImpl implements FileVisitor<Path> {
+
+    private static final Logger logger = LoggerFactory.getLogger(DeleteExpiredFilesFileVisitorImpl.class);
+    private final PathMatcher pathMatcher;
+    private final LocalDateTime fileExpiry;
+
+    public DeleteExpiredFilesFileVisitorImpl(String filter, LocalDateTime fileExpiry){
+        this.pathMatcher = FileSystems.getDefault().getPathMatcher("glob:" + filter);
+        this.fileExpiry = fileExpiry;
+    }
+    @Override
+    public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
+        return FileVisitResult.CONTINUE;
+    }
+
+    @Override
+    public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+        if(fileOrDirectoryIsExpired(file)){
+            logger.debug("Deleting: {}", file.getFileName());
+            Files.delete(file);
+        }
+        return FileVisitResult.CONTINUE;
+    }
+
+    @Override
+    public FileVisitResult visitFileFailed(Path file, IOException exc) {
+        return FileVisitResult.CONTINUE;
+    }
+
+    @Override
+    public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
+        if(fileOrDirectoryIsExpired(dir)){
+            logger.debug("Deleting: {}", dir.getFileName());
+            FileUtils.deleteDirectory(dir.toFile());
+        }
+        return FileVisitResult.CONTINUE;
+    }
+
+    /**
+     * Checks whether file or directory is expired
+     * @param file the file to check
+     * @return whether or not to delete
+     */
+    private boolean fileOrDirectoryIsExpired(Path file){
+        if(!pathMatcher.matches(file.getFileName())){
+            File theFile = file.toFile();
+            LocalDateTime lastModified = LocalDateTime.ofInstant(Instant.ofEpochMilli(theFile.lastModified()), ZoneId.systemDefault());
+            return lastModified.isBefore(fileExpiry);
+        }
+        return false;
+    }
+}

api/src/main/java/ca/bc/gov/educ/api/distribution/util/EducDistributionApiConstants.java

@@ -28,6 +28,8 @@ public class EducDistributionApiConstants {
     //Grad2-1931 - mchintha
     public static final String DATE_FORMAT_YYYYMMDD = "yyyyMMdd";
     public static final String TMP_DIR = "/tmp";
+
+    public static final String BATCH_DIR = TMP_DIR + "/Batch";
     public static final String DEL = "/";
     public static final String FILES_FOLDER_STRUCTURE = "/Batch/PSI/";
     public static final String TRANSMISSION_MODE_FTP = "FTP";
@@ -163,4 +165,13 @@ public class EducDistributionApiConstants {
 
     private int threadPoolMaxSize = 15;
 
+    @Value("${scheduler.clean-tmp-cache-cron}")
+    private String cleanTmpCacheCron;
+
+    @Value("${scheduler.clean-tmp-cache-interval-in-days}")
+    private int cleanTmpCacheIntervalInDays;
+
+    @Value("${scheduler.clean-tmp-cache-filter}")
+    private String cleanTmpCacheFilter;
+
 }

api/src/main/java/ca/bc/gov/educ/api/distribution/util/IOUtils.java

@@ -7,9 +7,7 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
+import java.nio.file.*;
 import java.nio.file.attribute.FileAttribute;
 import java.nio.file.attribute.PosixFilePermission;
 import java.nio.file.attribute.PosixFilePermissions;

api/src/main/resources/application.yaml

@@ -170,5 +170,10 @@ splunk:
     enabled: ${ENABLE_SPLUNK_LOG_HELPER}
 
 scheduler:
+  # how often to check for expired files
   clean-tmp-cache-cron: ${CLEAN_TMP_CACHE_CRON}
+  # the age in days of files that are considered 'expired'
+  clean-tmp-cache-interval-in-days: ${CLEAN_TMP_CACHE_INTERVAL}
+  # a glob filter to ignore certain files or directories
+  clean-tmp-cache-filter: ${CLEAN_TEMP_CACHE_FILTER}
 

api/src/test/java/ca/bc/gov/educ/api/distribution/util/CacheClearingTest.java

@@ -0,0 +1,167 @@
+package ca.bc.gov.educ.api.distribution.util;
+
+import ca.bc.gov.educ.api.distribution.config.EducDistributionApiConfig;
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.*;
+import java.nio.file.attribute.FileTime;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.util.HashMap;
+import java.util.Map;
+
+public class CacheClearingTest {
+
+    private enum FILES {
+        HSPERF,
+        UNDERT,
+        NFS,
+        JAV,
+        BATCH_DIR,
+        FTP_DIR,
+        FTP_SUB,
+        FTP_SUB_FILE,
+        FTP_ZIP,
+        PAPER_DIR,
+        PAPER_SUB,
+        PAPER_SUB_FILE,
+        PAPER_ZIP
+    }
+
+    private String tmpDir;
+    private final String FILE_SEP = File.separator;
+    private final Map<FILES, Path> files = new HashMap<>();
+
+    @Before
+    public void createCache() throws IOException {
+        this.tmpDir = Files.createTempDirectory("cacheTestDir").toFile().getAbsolutePath();
+        createTmpCacheFiles();
+        EducDistributionApiConfig config = new EducDistributionApiConfig();
+        FileVisitor<Path> fileVisitor = config.createCleanTmpCacheFilesFileVisitor("{hsperf*,undertow*,.nfs*,.java*,.nfs*,Batch,FTP,PAPER}", 1);
+        Path startingDir = Paths.get(this.tmpDir);
+        if (Files.exists(startingDir)) {
+            Files.walkFileTree(startingDir, fileVisitor);
+        }
+    }
+
+    @After
+    public void removeCache() throws IOException {
+        Path tmp = Paths.get(tmpDir);
+        FileUtils.deleteDirectory(tmp.toFile());
+    }
+
+
+    @Test
+    public void testFilesAreExpiredButShouldNotBeDeleted() {
+        for (Map.Entry<FILES, Path> entry : this.files.entrySet()) {
+            FILES f = entry.getKey();
+            if (f != FILES.FTP_SUB &&
+                    f != FILES.FTP_SUB_FILE &&
+                    f != FILES.FTP_ZIP &&
+                    f != FILES.PAPER_SUB &&
+                    f != FILES.PAPER_SUB_FILE &&
+                    f != FILES.PAPER_ZIP) {
+                Assert.assertTrue(Files.exists(entry.getValue()));
+            }
+        }
+    }
+
+    @Test
+    public void testFilesAreExpiredAndShouldBeDeleted() {
+        for (Map.Entry<FILES, Path> entry : this.files.entrySet()) {
+            FILES f = entry.getKey();
+            if (f == FILES.FTP_SUB_FILE ||
+                    f == FILES.PAPER_ZIP) {
+                Assert.assertFalse(Files.exists(entry.getValue()));
+            }
+        }
+    }
+
+    @Test
+    public void testFilesAreNotExpiredThatShouldNotBeDeleted() {
+        for (Map.Entry<FILES, Path> entry : this.files.entrySet()) {
+            FILES f = entry.getKey();
+            if (f == FILES.FTP_ZIP || f == FILES.PAPER_SUB || f == FILES.PAPER_SUB_FILE) {
+                Assert.assertTrue(Files.exists(entry.getValue()));
+            }
+        }
+    }
+
+    private void createTmpCacheFiles() throws IOException {
+        // create an expiry time of one day for testing
+        FileTime expiry = FileTime.from(LocalDateTime.now().minusDays(2).toInstant(ZoneOffset.UTC));
+
+        // create directory structure
+        //
+        // hsperfdata_101279000                       (HSPERF         expired, should not be deleted)
+        // undertow-docbase.8080.11764866489534620758 (UNDERT         expired, should not be deleted)
+        // .nfs000000002abc01e400010418               (NFS            expired, should not be deleted)
+        // .java_pid1                                 (JAV            expired, should not be deleted)
+        // Batch                                      (BATCH_DIR      expired, should not be deleted)
+        //   |_FTP                                    (FTP_DIR        expired, should not be deleted)
+        //      |_18873                               (FTP_SUB        expired, should be deleted)
+        //          | FileA.pdf                       (FTP_SUB_FILE   expired, should be deleted)
+        //      | EDGRAD.BATCH.18873.zip              (FTP_ZIP        not expired, should not be deleted)
+        //   |_PAPER                                  (PAPER_DIR      expired should not be deleted)
+        //      |_18874                               (PAPER_SUB      not expired, should not be deleted)
+        //          | FileB.pdf                       (PAPER_SUB_FILE not expired, should not be deleted)
+        //      | EDGRAD.BATCH.18874.zip              (PAPER_ZIP      expired, should be deleted)
+        //
+        // add files and update expiry times on some
+        files.put(FILES.HSPERF, Paths.get(tmpDir + FILE_SEP + "hsperfdata_101279000"));
+        files.put(FILES.UNDERT, Paths.get(tmpDir + FILE_SEP + "undertow-docbase.8080.11764866489534620758"));
+        files.put(FILES.NFS, Paths.get(tmpDir + FILE_SEP + ".nfs000000002abc01e400010418"));
+        files.put(FILES.JAV, Paths.get(tmpDir + FILE_SEP + ".java_pid1"));
+
+        Files.createFile(files.get(FILES.HSPERF));
+        Files.createFile(files.get(FILES.UNDERT));
+        Files.createFile(files.get(FILES.NFS));
+        Files.createFile(files.get(FILES.JAV));
+
+        Files.setLastModifiedTime(files.get(FILES.HSPERF), expiry);
+        Files.setLastModifiedTime(files.get(FILES.UNDERT), expiry);
+        Files.setLastModifiedTime(files.get(FILES.NFS), expiry);
+        Files.setLastModifiedTime(files.get(FILES.JAV), expiry);
+
+        files.put(FILES.BATCH_DIR, Paths.get(tmpDir + FILE_SEP + "Batch"));
+        files.put(FILES.FTP_DIR, Paths.get(files.get(FILES.BATCH_DIR) + FILE_SEP + "FTP"));
+        files.put(FILES.PAPER_DIR, Paths.get(files.get(FILES.BATCH_DIR) + FILE_SEP + "PAPER"));
+        Files.createDirectory(files.get(FILES.BATCH_DIR));
+        Files.createDirectory(files.get(FILES.FTP_DIR));
+        Files.createDirectory(files.get(FILES.PAPER_DIR));
+
+        files.put(FILES.FTP_SUB, Path.of(files.get(FILES.FTP_DIR) + FILE_SEP + "18873"));
+        Files.createDirectory(files.get(FILES.FTP_SUB));
+
+
+        files.put(FILES.FTP_SUB_FILE, Path.of(files.get(FILES.FTP_SUB) + FILE_SEP + "FileA.pdf"));
+        Files.createFile(files.get(FILES.FTP_SUB_FILE));
+
+
+        files.put(FILES.FTP_ZIP, Path.of(files.get(FILES.FTP_DIR) + FILE_SEP + "EDGRAD.BATCH.18873.zip"));
+        Files.createFile(files.get(FILES.FTP_ZIP));
+
+        files.put(FILES.PAPER_SUB, Path.of(files.get(FILES.PAPER_DIR) + FILE_SEP + "18874"));
+        Files.createDirectory(files.get(FILES.PAPER_SUB));
+
+        files.put(FILES.PAPER_SUB_FILE, Path.of(files.get(FILES.PAPER_SUB) + FILE_SEP + "FileB.pdf"));
+        Files.createFile(files.get(FILES.PAPER_SUB_FILE));
+
+        files.put(FILES.PAPER_ZIP, Path.of(files.get(FILES.PAPER_DIR) + FILE_SEP + "EDGRAD.BATCH.18874.zip"));
+        Files.createFile(files.get(FILES.PAPER_ZIP));
+
+        Files.setLastModifiedTime(files.get(FILES.FTP_SUB_FILE), expiry);
+        Files.setLastModifiedTime(files.get(FILES.FTP_SUB), expiry);
+        Files.setLastModifiedTime(files.get(FILES.FTP_DIR), expiry);
+        Files.setLastModifiedTime(files.get(FILES.PAPER_ZIP), expiry);
+        Files.setLastModifiedTime(files.get(FILES.PAPER_DIR), expiry);
+        Files.setLastModifiedTime(files.get(FILES.BATCH_DIR), expiry);
+    }
+
+}

api/src/test/resources/application.yaml

@@ -173,4 +173,6 @@ splunk:
     enabled: false
 
 scheduler:
-  clean-tmp-cache-cron: 0 0 */4 * * *
+  clean-tmp-cache-cron: 0 0 */4 * * *
+  clean-tmp-cache-interval-in-days: 1
+  clean-tmp-cache-filter: '{hsperf*,undertow*,.nfs*,.java*,.nfs*,Batch,FTP,PAPER}'