Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
57 commits
Select commit Hold shift + click to select a range
97c474b
Switching from string file path to nio Paths.
staudtMarius Jan 26, 2023
3a5e400
Adapting some tests.
staudtMarius Jan 27, 2023
c561ff4
Adapting some tests.
staudtMarius Feb 3, 2023
5715237
Merge branch 'dev' into ms/#723-use-nio-paths-instead-of-strings
staudtMarius Feb 7, 2023
8ae6fd1
Adapting tests.
staudtMarius Feb 7, 2023
9899c0e
Fixing failing test.
staudtMarius Feb 7, 2023
1531ae0
Adding changes to CHANGELOG.
staudtMarius Feb 7, 2023
756b530
Merge branch 'dev' into ms/#723-use-nio-paths-instead-of-strings
staudtMarius Feb 14, 2023
1cc76d6
Merge branch 'dev' into ms/#723-use-nio-paths-instead-of-strings
staudtMarius Feb 20, 2023
337b187
Changing ``CsvJointGridContainer`` to nio.Paths.
staudtMarius Feb 20, 2023
73f56c0
Merge branch 'dev' into ms/#723-use-nio-paths-instead-of-strings
staudtMarius Mar 3, 2023
2687118
Merge branch 'dev' into ms/#723-use-nio-paths-instead-of-strings
staudtMarius Mar 9, 2023
f87c315
Merge branch 'dev' into ms/#723-use-nio-paths-instead-of-strings
sebastian-peter Apr 3, 2023
636b74e
Fixing typos
sebastian-peter Apr 3, 2023
dc82399
Adding requested changes.
staudtMarius Apr 11, 2023
1c8c0c8
Merge branch 'dev' into ms/#723-use-nio-paths-instead-of-strings
staudtMarius Apr 14, 2023
c522c1b
Implementing requested changes.
staudtMarius Apr 14, 2023
c3e1814
Merge branch 'dev' into ms/#723-use-nio-paths-instead-of-strings
staudtMarius Apr 18, 2023
05aa7b2
Merge branch 'dev' into ms/#723-use-nio-paths-instead-of-strings
staudtMarius Apr 20, 2023
749e01f
Merge branch 'dev' into ms/#723-use-nio-paths-instead-of-strings
staudtMarius Apr 24, 2023
c5c8c57
Merge branch 'dev' into ms/#723-use-nio-paths-instead-of-strings
staudtMarius Apr 24, 2023
85257aa
Merge branch 'dev' into ms/#723-use-nio-paths-instead-of-strings
staudtMarius May 2, 2023
9367aa9
Merge branch 'dev' into ms/#723-use-nio-paths-instead-of-strings
staudtMarius May 9, 2023
e28dc58
fmt
staudtMarius May 9, 2023
f0a5f7a
Merge branch 'dev' into ms/#723-use-nio-paths-instead-of-strings
staudtMarius May 15, 2023
05e593c
Merge branch 'dev' into ms/#723-use-nio-paths-instead-of-strings
staudtMarius May 16, 2023
b8e9645
Merge branch 'dev' into ms/#723-use-nio-paths-instead-of-strings
staudtMarius May 22, 2023
5af6e82
Merge branch 'dev' into ms/#723-use-nio-paths-instead-of-strings
staudtMarius May 25, 2023
b275413
Merge branch 'dev' into ms/#723-use-nio-paths-instead-of-strings
staudtMarius May 30, 2023
0005128
Merge branch 'dev' into ms/#723-use-nio-paths-instead-of-strings
sebastian-peter May 31, 2023
2275d35
Replacing Paths.get
sebastian-peter May 31, 2023
a326483
Solving Groovy 4 issue
sebastian-peter May 31, 2023
b4b8639
Fixing tests that are using CsvTestDataMeta
sebastian-peter Jun 1, 2023
8a76b11
Merge branch 'dev' into ms/#723-use-nio-paths-instead-of-strings
sebastian-peter Jun 1, 2023
a6b1820
Merge branch 'dev' into ms/#723-use-nio-paths-instead-of-strings
staudtMarius Jun 5, 2023
21b3e1f
Implementing ``FileDefinition`` class.
staudtMarius Jun 5, 2023
21a6207
fmt
staudtMarius Jun 5, 2023
eae983a
Fix failing test.
staudtMarius Jun 5, 2023
1e77b48
Merge branch 'dev' into ms/#723-use-nio-paths-instead-of-strings
sebastian-peter Jun 9, 2023
7446df7
Replacing ```FileDefinition`` with ``FileUtils``.
staudtMarius Jun 12, 2023
a7945df
Fixing some ``Codacy`` issues.
staudtMarius Jun 13, 2023
aafa9a1
Merge branch 'dev' into ms/#723-use-nio-paths-instead-of-strings
staudtMarius Jun 13, 2023
e788185
Reverting renaming of tmpDirectory
sebastian-peter Jun 14, 2023
bcfd31b
A try of a fix
sebastian-peter Jun 14, 2023
621579b
Revert "A try of a fix"
sebastian-peter Jun 16, 2023
d5929ad
gradlew needs lf line endings
sebastian-peter Jun 16, 2023
cc7916c
Merge branch 'dev' into ms/#723-use-nio-paths-instead-of-strings
staudtMarius Jun 19, 2023
a6a656c
Fixing failing tests.
staudtMarius Jun 19, 2023
2c9e1de
Fixing failing tests.
staudtMarius Jun 19, 2023
38e0377
Fixing failing tests.
staudtMarius Jun 20, 2023
3e56b97
Merge branch 'dev' into ms/#723-use-nio-paths-instead-of-strings
staudtMarius Jun 20, 2023
b15d119
Fixing failing tests.
staudtMarius Jun 20, 2023
9b6f63e
Fixing failing tests.
staudtMarius Jun 20, 2023
aa10af9
Implementing requested changes.
staudtMarius Jun 21, 2023
e9ab824
fmt
staudtMarius Jun 21, 2023
0578bbc
Fixing some more code smells
sebastian-peter Jun 21, 2023
04463a9
Rearranged order of fields
sebastian-peter Jun 21, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 26 additions & 31 deletions src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,10 @@
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.io.FilenameUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -48,11 +46,11 @@ public class CsvFileConnector implements DataConnector {
individualTimeSeriesMetaInformation;

private final FileNamingStrategy fileNamingStrategy;
private final String baseDirectoryName;
private final Path baseDirectoryName;

private static final String FILE_ENDING = ".csv";

public CsvFileConnector(String baseDirectoryName, FileNamingStrategy fileNamingStrategy) {
public CsvFileConnector(Path baseDirectoryName, FileNamingStrategy fileNamingStrategy) {
this.baseDirectoryName = baseDirectoryName;
this.fileNamingStrategy = fileNamingStrategy;
}
Expand Down Expand Up @@ -106,22 +104,21 @@ BufferedCsvWriter getOrInitWriter(T timeSeries, String[] headerElements, String
* @throws ConnectorException If the base folder is a file
* @throws IOException If the writer cannot be initialized correctly
*/
private BufferedCsvWriter initWriter(String baseDirectory, CsvFileDefinition fileDefinition)
private BufferedCsvWriter initWriter(Path baseDirectory, CsvFileDefinition fileDefinition)
throws ConnectorException, IOException {
/* Join the full DIRECTORY path (excluding file name) */
String baseDirectoryHarmonized = IoUtil.harmonizeFileSeparator(baseDirectory);
String fullDirectoryPath =
FilenameUtils.concat(baseDirectoryHarmonized, fileDefinition.directoryPath());
String fullPath = FilenameUtils.concat(baseDirectoryHarmonized, fileDefinition.getFilePath());
Path baseDirectoryHarmonized = Path.of(IoUtil.harmonizeFileSeparator(baseDirectory.toString()));
Path fullDirectoryPath = baseDirectoryHarmonized.resolve(fileDefinition.directoryPath());
Path fullPath = baseDirectoryHarmonized.resolve(fileDefinition.getFilePath());

/* Create missing directories */
File directories = new File(fullDirectoryPath);
File directories = fullDirectoryPath.toFile();
if (directories.isFile())
throw new ConnectorException("Directory '" + directories + "' already exists and is a file!");
if (!directories.exists() && !directories.mkdirs())
throw new IOException("Unable to create directory tree '" + directories + "'");

File pathFile = new File(fullPath);
File pathFile = fullPath.toFile();
boolean append = pathFile.exists();
BufferedCsvWriter writer =
new BufferedCsvWriter(
Expand Down Expand Up @@ -182,7 +179,7 @@ public synchronized <C extends UniqueEntity> void closeEntityWriter(Class<C> clz
* @throws FileNotFoundException If the matching file cannot be found
*/
public BufferedReader initReader(Class<? extends UniqueEntity> clz) throws FileNotFoundException {
String filePath = null;
Path filePath = null;
try {
filePath =
fileNamingStrategy
Expand Down Expand Up @@ -210,8 +207,8 @@ public BufferedReader initReader(Class<? extends UniqueEntity> clz) throws FileN
* @return the reader that contains information about the file to be read in
* @throws FileNotFoundException if no file with the provided file name can be found
*/
public BufferedReader initReader(String filePath) throws FileNotFoundException {
File fullPath = new File(baseDirectoryName + File.separator + filePath + FILE_ENDING);
public BufferedReader initReader(Path filePath) throws FileNotFoundException {
File fullPath = baseDirectoryName.resolve(filePath).resolve(FILE_ENDING).toFile();
return new BufferedReader(
new InputStreamReader(new FileInputStream(fullPath), StandardCharsets.UTF_8), 16384);
}
Expand Down Expand Up @@ -252,9 +249,10 @@ public BufferedReader initReader(String filePath) throws FileNotFoundException {
filePath -> {
/* Extract meta information from file path and enhance it with the file path itself */
IndividualTimeSeriesMetaInformation metaInformation =
fileNamingStrategy.individualTimeSeriesMetaInformation(filePath);
fileNamingStrategy.individualTimeSeriesMetaInformation(filePath.toString());
return new edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation(
metaInformation, FileNamingStrategy.removeFileNameEnding(filePath));
metaInformation,
Path.of(FileNamingStrategy.removeFileNameEnding(filePath.toString())));
})
.filter(
metaInformation ->
Expand All @@ -271,23 +269,20 @@ public BufferedReader initReader(String filePath) throws FileNotFoundException {
*
* @return A set of relative paths to time series files, with respect to the base folder path
*/
private Set<String> getIndividualTimeSeriesFilePaths() {
Path baseDirectoryPath =
Paths.get(
FilenameUtils.getFullPath(baseDirectoryName)
+ FilenameUtils.getName(baseDirectoryName));
private Set<Path> getIndividualTimeSeriesFilePaths() {
Path baseDirectoryPath = baseDirectoryName.resolve(baseDirectoryName);
try (Stream<Path> pathStream = Files.walk(baseDirectoryPath)) {
return pathStream
.map(baseDirectoryPath::relativize)
.filter(
path -> {
String withoutEnding = FileNamingStrategy.removeFileNameEnding(path.toString());
Path withoutEnding =
Path.of(FileNamingStrategy.removeFileNameEnding(path.toString()));
return fileNamingStrategy
.getIndividualTimeSeriesPattern()
.matcher(withoutEnding)
.matcher(withoutEnding.toString())
.matches();
})
.map(Path::toString)
.collect(Collectors.toSet());
} catch (IOException e) {
log.error("Unable to determine time series files readers for time series.", e);
Expand All @@ -303,7 +298,7 @@ private Set<String> getIndividualTimeSeriesFilePaths() {
* @throws FileNotFoundException If the file is not present
*/
public BufferedReader initIdCoordinateReader() throws FileNotFoundException {
String filePath = fileNamingStrategy.getIdCoordinateEntityName();
Path filePath = Path.of(fileNamingStrategy.getIdCoordinateEntityName());
return initReader(filePath);
}

Expand All @@ -319,7 +314,7 @@ public BufferedReader initIdCoordinateReader() throws FileNotFoundException {
private <T extends TimeSeries<E, V>, E extends TimeSeriesEntry<V>, V extends Value>
CsvFileDefinition buildFileDefinition(T timeSeries, String[] headLineElements, String csvSep)
throws ConnectorException {
String directoryPath = fileNamingStrategy.getDirectoryPath(timeSeries).orElse("");
Path directoryPath = fileNamingStrategy.getDirectoryPath(timeSeries).orElse(Path.of(""));
String fileName =
fileNamingStrategy
.getEntityName(timeSeries)
Expand All @@ -342,7 +337,7 @@ CsvFileDefinition buildFileDefinition(T timeSeries, String[] headLineElements, S
private CsvFileDefinition buildFileDefinition(
Class<? extends UniqueEntity> clz, String[] headLineElements, String csvSep)
throws ConnectorException {
String directoryPath = fileNamingStrategy.getDirectoryPath(clz).orElse("");
Path directoryPath = fileNamingStrategy.getDirectoryPath(clz).orElse(Path.of(""));
String fileName =
fileNamingStrategy
.getEntityName(clz)
Expand Down Expand Up @@ -376,23 +371,23 @@ public void shutdown() {
@Deprecated(since = "3.0", forRemoval = true)
public static class CsvIndividualTimeSeriesMetaInformation
extends edu.ie3.datamodel.io.csv.timeseries.IndividualTimeSeriesMetaInformation {
private final String fullFilePath;
private final Path fullFilePath;

public CsvIndividualTimeSeriesMetaInformation(
UUID uuid,
edu.ie3.datamodel.io.csv.timeseries.ColumnScheme columnScheme,
String fullFilePath) {
Path fullFilePath) {
super(uuid, columnScheme);
this.fullFilePath = fullFilePath;
}

public CsvIndividualTimeSeriesMetaInformation(
edu.ie3.datamodel.io.csv.timeseries.IndividualTimeSeriesMetaInformation metaInformation,
String fullFilePath) {
Path fullFilePath) {
this(metaInformation.getUuid(), metaInformation.getColumnScheme(), fullFilePath);
}

public String getFullFilePath() {
public Path getFullFilePath() {
return fullFilePath;
}

Expand Down
12 changes: 7 additions & 5 deletions src/main/java/edu/ie3/datamodel/io/csv/BufferedCsvWriter.java
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import edu.ie3.util.StringUtils;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Map;
import java.util.Objects;
Expand Down Expand Up @@ -39,10 +40,11 @@ public class BufferedCsvWriter extends BufferedWriter {
* if no file exists, a new one will be created in both cases
* @throws IOException If the FileOutputStream cannot be established.
*/
public BufferedCsvWriter(
String filePath, String[] headLineElements, String csvSep, boolean append)
public BufferedCsvWriter(Path filePath, String[] headLineElements, String csvSep, boolean append)
throws IOException {
super(new OutputStreamWriter(new FileOutputStream(filePath, append), StandardCharsets.UTF_8));
super(
new OutputStreamWriter(
new FileOutputStream(filePath.toFile(), append), StandardCharsets.UTF_8));
this.headLineElements = headLineElements;
this.csvSep = csvSep;
}
Expand All @@ -59,10 +61,10 @@ public BufferedCsvWriter(
* if no file exists, a new one will be created in both cases
* @throws IOException If the FileOutputStream cannot be established.
*/
public BufferedCsvWriter(String baseFolder, CsvFileDefinition fileDefinition, boolean append)
public BufferedCsvWriter(Path baseFolder, CsvFileDefinition fileDefinition, boolean append)
throws IOException {
this(
baseFolder + File.separator + fileDefinition.getFilePath(),
baseFolder.resolve(fileDefinition.getFilePath()),
fileDefinition.headLineElements(),
fileDefinition.csvSep(),
append);
Expand Down
27 changes: 15 additions & 12 deletions src/main/java/edu/ie3/datamodel/io/csv/CsvFileDefinition.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,17 @@
package edu.ie3.datamodel.io.csv;

import edu.ie3.datamodel.io.IoUtil;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.FilenameUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public record CsvFileDefinition(
String fileName, String directoryPath, String[] headLineElements, String csvSep) {
String fileName, Path directoryPath, String[] headLineElements, String csvSep) {
private static final Logger logger = LoggerFactory.getLogger(CsvFileDefinition.class);

private static final Pattern FILE_NAME_PATTERN =
Expand All @@ -25,15 +26,17 @@ public record CsvFileDefinition(
private static final String FILE_EXTENSION = "csv";

public CsvFileDefinition(
String fileName, String directoryPath, String[] headLineElements, String csvSep) {
String fileName, Path directoryPath, String[] headLineElements, String csvSep) {
/* Remove all file separators at the beginning and end of a directory path and ensure harmonized file separator */
this.directoryPath =
Objects.nonNull(directoryPath)
? IoUtil.harmonizeFileSeparator(
directoryPath
.replaceFirst("^" + IoUtil.FILE_SEPARATOR_REGEX, "")
.replaceAll(IoUtil.FILE_SEPARATOR_REGEX + "$", ""))
: "";
Path.of(
Objects.nonNull(directoryPath)
? IoUtil.harmonizeFileSeparator(
directoryPath
.toString()
.replaceFirst("^" + IoUtil.FILE_SEPARATOR_REGEX, "")
.replaceAll(IoUtil.FILE_SEPARATOR_REGEX + "$", ""))
: "");

/* Check the given information of the file name */
Matcher matcher = FILE_NAME_PATTERN.matcher(fileName);
Expand All @@ -58,7 +61,7 @@ public CsvFileDefinition(

/** @deprecated since 3.0. Use {@link #directoryPath()} instead */
@Deprecated(since = "3.0")
public String getDirectoryPath() {
public Path getDirectoryPath() {
return directoryPath;
}

Expand All @@ -75,8 +78,8 @@ public String getFileName() {
* @return The path to the file relative to a not explicitly defined base directory, including the
* file extension
*/
public String getFilePath() {
return !directoryPath.isEmpty() ? FilenameUtils.concat(directoryPath, fileName) : fileName;
public Path getFilePath() {
return Files.exists(directoryPath) ? directoryPath.resolve(fileName) : Path.of(fileName);
}

/** @deprecated since 3.0. Use {@link #headLineElements()} instead */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,25 +7,26 @@

import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme;
import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation;
import java.nio.file.Path;
import java.util.Objects;
import java.util.UUID;

/** Enhancing the {@link IndividualTimeSeriesMetaInformation} with the full path to csv file */
public class CsvIndividualTimeSeriesMetaInformation extends IndividualTimeSeriesMetaInformation {
private final String fullFilePath;
private final Path fullFilePath;

public CsvIndividualTimeSeriesMetaInformation(
UUID uuid, ColumnScheme columnScheme, String fullFilePath) {
UUID uuid, ColumnScheme columnScheme, Path fullFilePath) {
super(uuid, columnScheme);
this.fullFilePath = fullFilePath;
}

public CsvIndividualTimeSeriesMetaInformation(
IndividualTimeSeriesMetaInformation metaInformation, String fullFilePath) {
IndividualTimeSeriesMetaInformation metaInformation, Path fullFilePath) {
this(metaInformation.getUuid(), metaInformation.getColumnScheme(), fullFilePath);
}

public String getFullFilePath() {
public Path getFullFilePath() {
return fullFilePath;
}

Expand Down
Loading