Skip to content
Closed
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
- SQL time series sources (`SqlTimeSeriesSource` and `SqlTimeSeriesMappingSource`) [#467](https://github.com/ie3-institute/PowerSystemDataModel/issues/467)
- Graph with impedance weighted edges including facilities to create it [#440](https://github.com/ie3-institute/PowerSystemDataModel/issues/440)
- Introducing SqlIndividualTimeSeriesMetaInformation which also provides sql table names [#513](https://github.com/ie3-institute/PowerSystemDataModel/issues/513)

### Fixed
- Reduced code smells [#492](https://github.com/ie3-institute/PowerSystemDataModel/issues/492)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,10 @@
import edu.ie3.datamodel.exceptions.ConnectorException;
import edu.ie3.datamodel.io.IoUtil;
import edu.ie3.datamodel.io.csv.*;
import edu.ie3.datamodel.io.csv.timeseries.ColumnScheme;
import edu.ie3.datamodel.io.csv.timeseries.IndividualTimeSeriesMetaInformation;
import edu.ie3.datamodel.io.naming.DataSourceMetaInformation;
import edu.ie3.datamodel.io.naming.FileNamingStrategy;
import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme;
import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation;
import edu.ie3.datamodel.models.UniqueEntity;
import edu.ie3.datamodel.models.timeseries.TimeSeries;
import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry;
Expand Down Expand Up @@ -246,7 +247,7 @@ public Optional<IndividualTimeSeriesMetaInformation> getIndividualTimeSeriesMeta
return new CsvIndividualTimeSeriesMetaInformation(
metaInformation, filePathWithoutEnding);
})
.collect(Collectors.toMap(FileNameMetaInformation::getUuid, v -> v));
.collect(Collectors.toMap(DataSourceMetaInformation::getUuid, v -> v));
}

/**
Expand Down Expand Up @@ -325,7 +326,7 @@ private Set<String> getIndividualTimeSeriesFilePaths() {
private Optional<CsvIndividualTimeSeriesMetaInformation> buildCsvTimeSeriesMetaInformation(
String filePathString, ColumnScheme... columnSchemes) {
try {
FileNameMetaInformation metaInformation =
DataSourceMetaInformation metaInformation =
fileNamingStrategy.extractTimeSeriesMetaInformation(filePathString);
if (!IndividualTimeSeriesMetaInformation.class.isAssignableFrom(metaInformation.getClass())) {
log.error(
Expand Down Expand Up @@ -456,9 +457,8 @@ public String getFullFilePath() {
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof CsvIndividualTimeSeriesMetaInformation)) return false;
if (!(o instanceof CsvIndividualTimeSeriesMetaInformation that)) return false;
if (!super.equals(o)) return false;
CsvIndividualTimeSeriesMetaInformation that = (CsvIndividualTimeSeriesMetaInformation) o;
return fullFilePath.equals(that.fullFilePath);
}

Expand Down
52 changes: 52 additions & 0 deletions src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
*/
package edu.ie3.datamodel.io.connectors;

import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme;
import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation;
import edu.ie3.util.StringUtils;
import edu.ie3.util.TimeUtil;
import java.sql.*;
Expand Down Expand Up @@ -156,4 +158,54 @@ public Map<String, String> extractFieldMap(ResultSet rs) {
}
return insensitiveFieldsToAttributes;
}

/**
* Enhancing the {@link IndividualTimeSeriesMetaInformation} with the name of the table containing
* the time series
*/
public static class SqlIndividualTimeSeriesMetaInformation
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The class nearly has the same outline as CsvIndividualTimeSeriesMetaInformation. Maybe we could also merge both classes? Instead of tableName and fullFilePath something like location?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I like your suggestion, however I also like having a distinction between csv and sql meta information

extends IndividualTimeSeriesMetaInformation {
private final String tableName;

public SqlIndividualTimeSeriesMetaInformation(
UUID uuid, ColumnScheme columnScheme, String tableName) {
super(uuid, columnScheme);
this.tableName = tableName;
}

public SqlIndividualTimeSeriesMetaInformation(
IndividualTimeSeriesMetaInformation metaInformation, String fullFilePath) {
this(metaInformation.getUuid(), metaInformation.getColumnScheme(), fullFilePath);
}

public String getTableName() {
return tableName;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof SqlIndividualTimeSeriesMetaInformation that)) return false;
if (!super.equals(o)) return false;
return tableName.equals(that.tableName);
}

@Override
public int hashCode() {
return Objects.hash(super.hashCode(), tableName);
}

@Override
public String toString() {
return "SqlIndividualTimeSeriesMetaInformation{"
+ "uuid="
+ getUuid()
+ ", columnScheme="
+ getColumnScheme()
+ ", tableName='"
+ tableName
+ '\''
+ '}';
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,16 @@
* Institute of Energy Systems, Energy Efficiency and Energy Economics,
* Research group Distribution grid planning and operation
*/
package edu.ie3.datamodel.io.csv;
package edu.ie3.datamodel.io.naming;

import java.util.Objects;
import java.util.UUID;

/** Meta information, that can be derived from a certain file name */
public abstract class FileNameMetaInformation {
/** Meta information, that can be derived from a data source */
public abstract class DataSourceMetaInformation {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not fully convinced with the name... Yes, it does describe a data source, but it does not describe a data source in general. It's more describing a time series data source. But TimeSeriesDataSourceMetaInformation is a bit long. Do you have any other suggestion?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In that case, how about simply TimeSeriesMetaInformation?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

private final UUID uuid;

protected FileNameMetaInformation(UUID uuid) {
protected DataSourceMetaInformation(UUID uuid) {
this.uuid = uuid;
}

Expand All @@ -23,7 +23,7 @@ public UUID getUuid() {
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof FileNameMetaInformation that)) return false;
if (!(o instanceof DataSourceMetaInformation that)) return false;
return uuid.equals(that.uuid);
}

Expand All @@ -34,6 +34,6 @@ public int hashCode() {

@Override
public String toString() {
return "FileNameMetaInformation{" + "uuid=" + uuid + '}';
return "DataSourceMetaInformation{" + "uuid=" + uuid + '}';
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@
*/
package edu.ie3.datamodel.io.naming;

import edu.ie3.datamodel.io.csv.timeseries.ColumnScheme;
import edu.ie3.datamodel.io.csv.timeseries.IndividualTimeSeriesMetaInformation;
import edu.ie3.datamodel.io.csv.timeseries.LoadProfileTimeSeriesMetaInformation;
import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme;
import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation;
import edu.ie3.datamodel.io.naming.timeseries.LoadProfileTimeSeriesMetaInformation;
import edu.ie3.datamodel.io.source.TimeSeriesMappingSource;
import edu.ie3.datamodel.models.UniqueEntity;
import edu.ie3.datamodel.models.input.*;
Expand Down Expand Up @@ -42,9 +42,9 @@ public class EntityPersistenceNamingStrategy {
private static final String UUID_STRING =
"[a-zA-Z0-9]{8}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{12}";
/**
* Regex to match the naming convention of a file for an individual time series. The column scheme
* is accessible via the named capturing group "columnScheme". The time series' UUID is accessible
* by the named capturing group "uuid"
* Regex to match the naming convention of a source for an individual time series. The column
* scheme is accessible via the named capturing group "columnScheme". The time series' UUID is
* accessible by the named capturing group "uuid"
*/
private static final Pattern INDIVIDUAL_TIME_SERIES_PATTERN =
Pattern.compile("its_(?<columnScheme>[a-zA-Z]{1,11})_(?<uuid>" + UUID_STRING + ")");
Expand Down Expand Up @@ -125,17 +125,18 @@ public Pattern getIndividualTimeSeriesPattern() {
}

/**
* Extracts meta information from a valid file name for an individual time series
* Extracts meta information from a valid source name for an individual time series
*
* @param fileName File name to extract information from
* @return Meta information form individual time series file name
* @param sourceName Name of the source to extract information from, e.g. file name or SQL table
* name
* @return Meta information form individual time series source name
*/
public IndividualTimeSeriesMetaInformation extractIndividualTimesSeriesMetaInformation(
String fileName) {
Matcher matcher = getIndividualTimeSeriesPattern().matcher(fileName);
String sourceName) {
Matcher matcher = getIndividualTimeSeriesPattern().matcher(sourceName);
if (!matcher.matches())
throw new IllegalArgumentException(
"Cannot extract meta information on individual time series from '" + fileName + "'.");
"Cannot extract meta information on individual time series from '" + sourceName + "'.");

String columnSchemeKey = matcher.group("columnScheme");
ColumnScheme columnScheme =
Expand Down Expand Up @@ -189,7 +190,7 @@ private static String prepareSuffix(String suffix) {
/**
* Returns the name of the entity, that should be used for persistence.
*
* @param cls Targeted class of the given file
* @param cls Targeted class of the given entity
* @return The name of the entity
*/
public Optional<String> getEntityName(Class<? extends UniqueEntity> cls) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
package edu.ie3.datamodel.io.naming;

import edu.ie3.datamodel.io.IoUtil;
import edu.ie3.datamodel.io.csv.FileNameMetaInformation;
import edu.ie3.datamodel.models.UniqueEntity;
import edu.ie3.datamodel.models.timeseries.TimeSeries;
import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry;
Expand Down Expand Up @@ -218,7 +217,7 @@ public Pattern getLoadProfileTimeSeriesPattern() {
* @param path Path to the file
* @return The meeting meta information
*/
public FileNameMetaInformation extractTimeSeriesMetaInformation(Path path) {
public DataSourceMetaInformation extractTimeSeriesMetaInformation(Path path) {
/* Extract file name from possibly fully qualified path */
Path fileName = path.getFileName();
if (fileName == null)
Expand All @@ -233,7 +232,7 @@ public FileNameMetaInformation extractTimeSeriesMetaInformation(Path path) {
* @param fileName File name
* @return The meeting meta information
*/
public FileNameMetaInformation extractTimeSeriesMetaInformation(String fileName) {
public DataSourceMetaInformation extractTimeSeriesMetaInformation(String fileName) {
/* Remove the file ending (ending limited to 255 chars, which is the max file name allowed in NTFS and ext4) */
String withoutEnding = fileName.replaceAll("(?:\\.[^\\\\/\\s]{1,255}){1,2}$", "");

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* Institute of Energy Systems, Energy Efficiency and Energy Economics,
* Research group Distribution grid planning and operation
*/
package edu.ie3.datamodel.io.csv.timeseries;
package edu.ie3.datamodel.io.naming.timeseries;

import edu.ie3.datamodel.models.value.*;
import edu.ie3.util.StringUtils;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,14 @@
* Institute of Energy Systems, Energy Efficiency and Energy Economics,
* Research group Distribution grid planning and operation
*/
package edu.ie3.datamodel.io.csv.timeseries;
package edu.ie3.datamodel.io.naming.timeseries;

import edu.ie3.datamodel.io.csv.FileNameMetaInformation;
import edu.ie3.datamodel.io.naming.DataSourceMetaInformation;
import java.util.Objects;
import java.util.UUID;

/** Specific meta information, that can be derived from a individual time series file */
public class IndividualTimeSeriesMetaInformation extends FileNameMetaInformation {
public class IndividualTimeSeriesMetaInformation extends DataSourceMetaInformation {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I know, it's not your fault (I introduced this flaw), but I'm not super happy with the fact, that this is not an abstract class. It would be better, if you could only instantiate SqlIndividualTimeSeriesMetaInformation and CsvIndividualTimeSeriesInformation.

One opportunity, I would like you to check out, is to check, if EntityPersistenceNamingStrategy#extractIndividualTimesSeriesMetaInformation couldn't be moved to both of the implementing classes as static method parse(String, ...). Parts of the logic of the method could obviously be moved here and then the individual time series pattern doesn't have to be publicly accessible.

What do you think?

Copy link
Member Author

@sebastian-peter sebastian-peter Feb 2, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To the second part: A counter-argument would be here, that multiple EntityPersistenceNamingStrategy can exist and thus allow for different naming strategies of files/tables. When integrating the function into the timeseries meta information, only one single naming strategy could exist.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To the first point: IndividualTimeSeriesMetaInformation is currently instantiated in EntityPersistenceNamingStrategy#extractIndividualTimesSeriesMetaInformation and I think it makes some sense that way. If we want to make the class abstract, we have to think of another solution there.

private final ColumnScheme columnScheme;

public IndividualTimeSeriesMetaInformation(UUID uuid, ColumnScheme columnScheme) {
Expand All @@ -25,9 +25,8 @@ public ColumnScheme getColumnScheme() {
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof IndividualTimeSeriesMetaInformation)) return false;
if (!(o instanceof IndividualTimeSeriesMetaInformation that)) return false;
if (!super.equals(o)) return false;
IndividualTimeSeriesMetaInformation that = (IndividualTimeSeriesMetaInformation) o;
return columnScheme == that.columnScheme;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,14 @@
* Institute of Energy Systems, Energy Efficiency and Energy Economics,
* Research group Distribution grid planning and operation
*/
package edu.ie3.datamodel.io.csv.timeseries;
package edu.ie3.datamodel.io.naming.timeseries;

import edu.ie3.datamodel.io.csv.FileNameMetaInformation;
import edu.ie3.datamodel.io.naming.DataSourceMetaInformation;
import java.util.Objects;
import java.util.UUID;

/** Specific meta information, that can be derived from a load profile time series file */
public class LoadProfileTimeSeriesMetaInformation extends FileNameMetaInformation {
public class LoadProfileTimeSeriesMetaInformation extends DataSourceMetaInformation {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same applies as for InvdividualTimeSeriesMetaInformation. ^^

private final String profile;

public LoadProfileTimeSeriesMetaInformation(UUID uuid, String profile) {
Expand All @@ -25,9 +25,8 @@ public String getProfile() {
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof LoadProfileTimeSeriesMetaInformation)) return false;
if (!(o instanceof LoadProfileTimeSeriesMetaInformation that)) return false;
if (!super.equals(o)) return false;
LoadProfileTimeSeriesMetaInformation that = (LoadProfileTimeSeriesMetaInformation) o;
return profile.equals(that.profile);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
*/
package edu.ie3.datamodel.io.source;

import edu.ie3.datamodel.io.csv.timeseries.IndividualTimeSeriesMetaInformation;
import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation;
import edu.ie3.datamodel.models.input.InputEntity;
import java.util.Map;
import java.util.Objects;
Expand Down Expand Up @@ -64,9 +64,8 @@ public UUID getTimeSeries() {
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof MappingEntry)) return false;
if (!(o instanceof MappingEntry that)) return false;
if (!super.equals(o)) return false;
MappingEntry that = (MappingEntry) o;
return participant.equals(that.participant) && timeSeries.equals(that.timeSeries);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@
*/
package edu.ie3.datamodel.io.source;

import static edu.ie3.datamodel.io.csv.timeseries.ColumnScheme.*;
import static edu.ie3.datamodel.io.naming.timeseries.ColumnScheme.*;

import edu.ie3.datamodel.io.csv.timeseries.ColumnScheme;
import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme;
import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries;
import edu.ie3.datamodel.models.value.Value;
import edu.ie3.util.interval.ClosedInterval;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@
*/
package edu.ie3.datamodel.io.source.csv;

import edu.ie3.datamodel.io.csv.timeseries.IndividualTimeSeriesMetaInformation;
import edu.ie3.datamodel.io.factory.SimpleEntityData;
import edu.ie3.datamodel.io.factory.timeseries.TimeSeriesMappingFactory;
import edu.ie3.datamodel.io.naming.FileNamingStrategy;
import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation;
import edu.ie3.datamodel.io.source.TimeSeriesMappingSource;
import java.util.Map;
import java.util.Optional;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@
package edu.ie3.datamodel.io.source.csv;

import edu.ie3.datamodel.io.connectors.CsvFileConnector;
import edu.ie3.datamodel.io.csv.timeseries.ColumnScheme;
import edu.ie3.datamodel.io.factory.timeseries.IdCoordinateFactory;
import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueData;
import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueFactory;
import edu.ie3.datamodel.io.naming.FileNamingStrategy;
import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme;
import edu.ie3.datamodel.io.source.IdCoordinateSource;
import edu.ie3.datamodel.io.source.WeatherSource;
import edu.ie3.datamodel.models.UniqueEntity;
Expand Down
Loading