Merge branch 'main' into grad-release

api/src/main/java/ca/bc/gov/educ/api/distribution/config/EducDistributionApiConfig.java

@@ -1,8 +1,6 @@
 package ca.bc.gov.educ.api.distribution.config;
 
-import ca.bc.gov.educ.api.distribution.util.DeleteExpiredFilesFileVisitorImpl;
 import org.modelmapper.ModelMapper;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.web.client.RestTemplateBuilder;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
@@ -13,8 +11,7 @@
 import org.springframework.web.reactive.function.client.WebClient;
 import reactor.netty.http.client.HttpClient;
 
-import java.nio.file.*;
-import java.time.LocalDateTime;
+import java.io.FileFilter;
 
 @Configuration
 @IntegrationComponentScan
@@ -42,8 +39,16 @@ public RestTemplate restTemplate(RestTemplateBuilder builder) {
         return builder.build();
     }
 
-    @Bean("TmpCacheFileVisitor")
-    public FileVisitor<Path> createCleanTmpCacheFilesFileVisitor(@Value("${scheduler.clean-tmp-cache-filter}") String filter, @Value("${scheduler.clean-tmp-cache-interval-in-days}") int days) {
-        return new DeleteExpiredFilesFileVisitorImpl(filter, LocalDateTime.now().minusDays(days));
+
+    @Bean
+    public FileFilter createFileFilter(){
+        return pathname -> {
+            String name = pathname.getName();
+            return !name.startsWith(".") &&
+                    !name.contains("undertow") &&
+                    !name.contains("hsperfdata");
+        };
     }
+
+
 }

api/src/main/java/ca/bc/gov/educ/api/distribution/config/ScheduledTasksConfig.java

@@ -1,48 +1,45 @@
 package ca.bc.gov.educ.api.distribution.config;
 
 import ca.bc.gov.educ.api.distribution.util.EducDistributionApiConstants;
+import ca.bc.gov.educ.api.distribution.util.IOUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.scheduling.annotation.Scheduled;
 
-import java.io.IOException;
-import java.nio.file.FileVisitor;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
+import java.io.File;
+import java.io.FileFilter;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.util.Arrays;
+import java.util.List;
 
 @Configuration
 public class ScheduledTasksConfig {
 
     private static final Logger logger = LoggerFactory.getLogger(ScheduledTasksConfig.class);
 
-    EducDistributionApiConstants educDistributionApiConstants;
-    FileVisitor<Path> fileVisitor;
+    FileFilter fileFilter;
 
     @Autowired
-    public ScheduledTasksConfig(@Qualifier("TmpCacheFileVisitor") FileVisitor<Path> fileVisitor, EducDistributionApiConstants educDistributionApiConstants) {
-        this.fileVisitor = fileVisitor;
-        this.educDistributionApiConstants = educDistributionApiConstants;
+    public ScheduledTasksConfig(FileFilter fileFilter) {
+        this.fileFilter = fileFilter;
     }
 
-    /**
-     * Removes artifacts from the /tmp directory that are expired
-     */
     @Scheduled(cron = "${scheduler.clean-tmp-cache-cron}")
     public void cleanTmpCacheFiles() {
-        logger.debug("Running clean cache...");
-        Path startingDir = Paths.get(educDistributionApiConstants.TMP_DIR);
-        if(Files.exists(startingDir)){
-            try {
-                Files.walkFileTree(startingDir, fileVisitor);
-            } catch (IOException e) {
-                logger.error("ScheduledTasksConfig: There was an error removing file cache: {}", e.getLocalizedMessage());
+        File dir = new File(EducDistributionApiConstants.TMP_DIR);
+        List<File> files = Arrays.asList(dir.listFiles(this.fileFilter));
+        LocalDateTime fileExpiry = LocalDateTime.now().minusHours(3);
+        files.forEach(file -> {
+            LocalDateTime lastModified = LocalDateTime.ofInstant(Instant.ofEpochMilli(file.lastModified()), ZoneId.systemDefault());
+            if(lastModified.isBefore(fileExpiry)){
+                logger.debug("Removing file or directory: {}", file.getName());
+                IOUtils.removeFileOrDirectory(file);
             }
-        }
+        });
     }
 
-
 }

api/src/main/java/ca/bc/gov/educ/api/distribution/model/dto/Student.java

@@ -5,8 +5,8 @@
 
 import java.io.Serializable;
 import java.time.LocalDate;
-import java.time.LocalDateTime;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.List;
 import java.util.Objects;
 
@@ -19,7 +19,7 @@ public class Student implements Serializable {
     private String gender;
     private String citizenship;
     private LocalDate birthdate;
-    private LocalDateTime lastUpdateDate;
+    private Date lastUpdateDate;
     private Address address;
     private String grade;
     private String gradProgram;
@@ -102,11 +102,11 @@ public void setBirthdate(LocalDate value) {
     }
 
     @JsonFormat(pattern="yyyy-MM-dd HH:mm:ss")
-    public LocalDateTime getLastUpdateDate() {
+    public Date getLastUpdateDate() {
         return lastUpdateDate;
     }
 
-    public void setLastUpdateDate(LocalDateTime lastUpdateDate) {
+    public void setLastUpdateDate(Date lastUpdateDate) {
         this.lastUpdateDate = lastUpdateDate;
     }
 

api/src/main/java/ca/bc/gov/educ/api/distribution/process/MergeProcess.java

@@ -18,7 +18,9 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
+import java.sql.Date;
 import java.time.LocalDate;
+import java.time.ZoneId;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.List;
@@ -218,7 +220,7 @@ private Student prepareStudentObj(StudentCredentialDistribution scd, List<Studen
 			std.setPen(pen);
 			std.setGrade(scd.getStudentGrade());
 			std.setGradProgram(scd.getProgram());
-			std.setLastUpdateDate(scd.getLastUpdateDate());
+			std.setLastUpdateDate(Date.from(scd.getLastUpdateDate().atZone(ZoneId.systemDefault()).toInstant()));
 			std.setGraduationData(new GraduationData());
 			std.setNonGradReasons(getNonGradReasons(scd.getNonGradReasons()));
 

api/src/main/java/ca/bc/gov/educ/api/distribution/util/EducDistributionApiConstants.java

@@ -28,8 +28,6 @@ public class EducDistributionApiConstants {
     //Grad2-1931 - mchintha
     public static final String DATE_FORMAT_YYYYMMDD = "yyyyMMdd";
     public static final String TMP_DIR = "/tmp";
-
-    public static final String BATCH_DIR = TMP_DIR + "/Batch";
     public static final String DEL = "/";
     public static final String FILES_FOLDER_STRUCTURE = "/Batch/PSI/";
     public static final String TRANSMISSION_MODE_FTP = "FTP";
@@ -165,13 +163,4 @@ public class EducDistributionApiConstants {
 
     private int threadPoolMaxSize = 15;
 
-    @Value("${scheduler.clean-tmp-cache-cron}")
-    private String cleanTmpCacheCron;
-
-    @Value("${scheduler.clean-tmp-cache-interval-in-days}")
-    private int cleanTmpCacheIntervalInDays;
-
-    @Value("${scheduler.clean-tmp-cache-filter}")
-    private String cleanTmpCacheFilter;
-
 }

api/src/main/java/ca/bc/gov/educ/api/distribution/util/IOUtils.java

@@ -7,7 +7,9 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.nio.file.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.nio.file.attribute.FileAttribute;
 import java.nio.file.attribute.PosixFilePermission;
 import java.nio.file.attribute.PosixFilePermissions;

api/src/main/resources/application.yaml

@@ -170,10 +170,5 @@ splunk:
     enabled: ${ENABLE_SPLUNK_LOG_HELPER}
 
 scheduler:
-  # how often to check for expired files
   clean-tmp-cache-cron: ${CLEAN_TMP_CACHE_CRON}
-  # the age in days of files that are considered 'expired'
-  clean-tmp-cache-interval-in-days: ${CLEAN_TMP_CACHE_INTERVAL}
-  # a glob filter to ignore certain files or directories
-  clean-tmp-cache-filter: ${CLEAN_TEMP_CACHE_FILTER}