Improving column name validation

CHANGELOG.md

@@ -11,6 +11,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 
 ### Changed
+- Enhancing the error message for coordinate sources with invalid column names [#670](https://github.com/ie3-institute/PowerSystemDataModel/issues/670) 
+- Allowing for additional unused columns in sources [#839](https://github.com/ie3-institute/PowerSystemDataModel/issues/839)
+- Improving column name validation to only run once per source [#849](https://github.com/ie3-institute/PowerSystemDataModel/issues/849)
 
 ## [4.1.0] - 2023-11-02
 

src/main/java/edu/ie3/datamodel/exceptions/FactoryException.java

@@ -5,7 +5,7 @@
 */
 package edu.ie3.datamodel.exceptions;
 
-/** Is thrown, when an something went wrong during entity creation process in a EntityFactory */
+/** Is thrown, when something went wrong during entity creation process in a EntityFactory */
 public class FactoryException extends RuntimeException {
   public FactoryException(final String message, final Throwable cause) {
     super(message, cause);

src/main/java/edu/ie3/datamodel/io/connectors/CouchbaseConnector.java

@@ -9,10 +9,15 @@
 import com.couchbase.client.java.AsyncCollection;
 import com.couchbase.client.java.Cluster;
 import com.couchbase.client.java.Collection;
+import com.couchbase.client.java.json.JsonObject;
 import com.couchbase.client.java.kv.GetResult;
 import com.couchbase.client.java.kv.MutationResult;
 import com.couchbase.client.java.query.QueryResult;
+import edu.ie3.datamodel.io.source.SourceValidator;
+import java.time.Duration;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 
 /**
@@ -36,6 +41,33 @@ public CouchbaseConnector(String url, String bucketName, String username, String
     cluster = Cluster.connect(url, username, password);
   }
 
+  /**
+   * This method should be used to validate a given couchbaseDb.
+   *
+   * @param entityClass class of the entity
+   * @param validator for validation
+   */
+  @SuppressWarnings("unchecked")
+  public final void validateDb(Class<?> entityClass, SourceValidator validator) {
+    String query =
+        "SELECT ARRAY_DISTINCT(ARRAY_AGG(v)) AS column FROM "
+            + bucketName
+            + " b UNNEST OBJECT_NAMES(b) AS v";
+    cluster.bucket(bucketName).waitUntilReady(Duration.ofSeconds(30));
+
+    QueryResult queryResult = query(query).join();
+    JsonObject jsonObject = queryResult.rowsAsObject().get(0);
+    Object columns = jsonObject.toMap().get("column");
+
+    Set<String> set = new HashSet<>();
+
+    if (columns != null) {
+      set.addAll((List<String>) columns);
+    }
+
+    validator.validate(set, entityClass);
+  }
+
   /**
    * Return the couchbase java sdk equivalent of a session - a collection - to the previously set
    * bucket

src/main/java/edu/ie3/datamodel/io/connectors/InfluxDbConnector.java

@@ -5,6 +5,7 @@
 */
 package edu.ie3.datamodel.io.connectors;
 
+import edu.ie3.datamodel.io.source.SourceValidator;
 import java.util.*;
 import java.util.function.BinaryOperator;
 import java.util.stream.Collectors;
@@ -38,6 +39,7 @@ public class InfluxDbConnector implements DataConnector {
         return maps;
       };
 
+  private final String databaseName;
   private final String scenarioName;
   private final InfluxDB session;
 
@@ -77,6 +79,7 @@ public InfluxDbConnector(
    */
   public InfluxDbConnector(
       InfluxDB session, String scenarioName, String databaseName, boolean createDb) {
+    this.databaseName = databaseName;
     this.scenarioName = scenarioName;
     this.session = session;
 
@@ -107,6 +110,26 @@ public InfluxDbConnector(String url, String databaseName) {
     this(url, databaseName, NO_SCENARIO, true, InfluxDB.LogLevel.NONE, BatchOptions.DEFAULTS);
   }
 
+  /**
+   * This method should be used to validate a given {@link InfluxDB}.
+   *
+   * @param entityClass class of the entity
+   * @param validator for validation
+   */
+  public final void validateDb(Class<?> entityClass, SourceValidator validator) {
+    QueryResult tagKeys = session.query(new Query("SHOW TAG KEYS ON " + databaseName));
+    Map<String, Set<Map<String, String>>> tagResults = parseQueryResult(tagKeys);
+
+    QueryResult fieldKeys = session.query(new Query("SHOW FIELD KEYS ON " + databaseName));
+    Map<String, Set<Map<String, String>>> fieldResults = parseQueryResult(fieldKeys);
+
+    Set<String> set = new HashSet<>();
+    tagResults.values().forEach(v -> v.stream().map(m -> m.get("tagKey")).forEach(set::add));
+    fieldResults.values().forEach(v -> v.stream().map(m -> m.get("fieldKey")).forEach(set::add));
+
+    validator.validate(set, entityClass);
+  }
+
   /**
    * Create the database of this connector if it doesn't exist yet
    *

src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java

@@ -5,6 +5,7 @@
 */
 package edu.ie3.datamodel.io.connectors;
 
+import edu.ie3.datamodel.io.source.SourceValidator;
 import edu.ie3.util.StringUtils;
 import edu.ie3.util.TimeUtil;
 import java.sql.*;
@@ -41,6 +42,28 @@ public SqlConnector(String jdbcUrl, String userName, String password) {
     connectionProps.put("password", password);
   }
 
+  /**
+   * This method should be used to validate a given sql table.
+   *
+   * @param tableName name of the table
+   * @param entityClass class of the entity
+   * @param validator for validation
+   * @throws SQLException – if the connection could not be established
+   */
+  public void validateDBTable(String tableName, Class<?> entityClass, SourceValidator validator)
+      throws SQLException {
+    ResultSet rs = getConnection().getMetaData().getColumns(null, null, tableName, null);
+
+    Set<String> columnNames = new HashSet<>();
+
+    while (rs.next()) {
+      String name = rs.getString("COLUMN_NAME");
+      columnNames.add(StringUtils.snakeCaseToCamelCase(name));
+    }
+
+    validator.validate(columnNames, entityClass);
+  }
+
   /**
    * Executes the given query. For update queries please use {@link
    * SqlConnector#executeUpdate(String)}.

src/main/java/edu/ie3/datamodel/io/factory/Factory.java

@@ -6,10 +6,11 @@
 package edu.ie3.datamodel.io.factory;
 
 import edu.ie3.datamodel.exceptions.FactoryException;
+import edu.ie3.datamodel.io.source.SourceValidator;
 import edu.ie3.datamodel.utils.Try;
 import edu.ie3.datamodel.utils.Try.*;
+import edu.ie3.util.StringUtils;
 import java.util.*;
-import java.util.function.IntFunction;
 import java.util.stream.Collectors;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -23,7 +24,7 @@
  * @param <R> Type of the intended return type (might differ slightly from target class (cf. {@link
  *     edu.ie3.datamodel.io.factory.timeseries.TimeBasedValueFactory})).
  */
-public abstract class Factory<C, D extends FactoryData, R> {
+public abstract class Factory<C, D extends FactoryData, R> implements SourceValidator {
   public static final Logger log = LoggerFactory.getLogger(Factory.class);
 
   private final List<Class<? extends C>> supportedClasses;
@@ -48,15 +49,10 @@ public List<Class<? extends C>> getSupportedClasses() {
   public Try<R, FactoryException> get(D data) {
     isSupportedClass(data.getTargetClass());
 
-    // magic: case-insensitive get/set calls on set strings
-    final List<Set<String>> allFields = getFields(data);
-
     try {
-      validateParameters(data, allFields.toArray((IntFunction<Set<String>[]>) Set[]::new));
-
       // build the model
       return Success.of(buildModel(data));
-    } catch (FactoryException e) {
+    } catch (Exception e) {
       return Failure.of(
           new FactoryException(
               "An error occurred when creating instance of "
@@ -108,76 +104,87 @@ private void isSupportedClass(Class<?> desiredClass) {
    * Returns list of sets of attribute names that the entity requires to be built. At least one of
    * these sets needs to be delivered for entity creation to be successful.
    *
-   * @param data EntityData (or subclass) containing the data
    * @return list of possible attribute sets
    */
-  protected abstract List<Set<String>> getFields(D data);
+  protected abstract List<Set<String>> getFields(Class<?> entityClass);
 
   /**
-   * Validates the factory specific constructor parameters in two ways. 1) the biggest set of the
-   * provided field sets is compared against fields the class implements. If this test passes then
-   * we know for sure that the field names at least in the biggest constructor are equal to the
-   * provided factory strings 2) if 1) passes, the provided entity data (which is equal to the data
-   * e.g. read from the outside) is compared to all available constructor parameters provided by the
-   * fieldSets Array. If we find exactly one constructor, that matches the field names we can
-   * proceed. Otherwise a detailed exception message is thrown.
+   * Method for validating the found fields. The found fields needs to fully contain at least one of
+   * the sets returned by {@link #getFields(Class)}. If the found fields don't contain all necessary
+   * fields, an {@link FactoryException} with a detail message is thrown. If the found fields
+   * contain more fields than necessary, these fields are ignored.
    *
-   * @param data the entity containing at least the entity class as well a mapping of the provided
-   *     field name strings to its value (e.g. a headline of a csv to column values)
-   * @param fieldSets a set containing all available constructor combinations as field names
-   * @return the index of the set in the fieldSets array that fits the provided entity data
+   * @param foundFields that were found
+   * @param entityClass of the build data
    */
-  protected int validateParameters(D data, Set<String>... fieldSets) {
-    Map<String, String> fieldsToValues = data.getFieldsToValues();
-
-    // get all sets that match the fields to attributes
+  public void validate(Set<String> foundFields, Class<?> entityClass) {
+    List<Set<String>> fieldSets =
+        getFields(entityClass).stream().map(Factory::toSnakeCase).toList();
+    Set<String> harmonizedFoundFields = toSnakeCase(foundFields);
+
+    // comparing the found fields to a list of possible fields (allows additional fields)
+    // if not all fields were found in a set, this set is filtered out
+    // all other fields are saved as a list
+    // allows snake, camel and mixed cases
     List<Set<String>> validFieldSets =
-        Arrays.stream(fieldSets).filter(x -> x.equals(fieldsToValues.keySet())).toList();
+        fieldSets.stream()
+            .filter(s -> foundFields.containsAll(s) || harmonizedFoundFields.containsAll(s))
+            .toList();
 
-    if (validFieldSets.size() == 1) {
-      // if we can identify a unique parameter set for a constructor, we take it and return the
-      // index
-      Set<String> validFieldSet = validFieldSets.get(0);
-      return Arrays.asList(fieldSets).indexOf(validFieldSet);
-    } else {
+    if (validFieldSets.isEmpty()) {
       // build the exception string with extensive debug information
-      String providedFieldMapString =
-          fieldsToValues.keySet().stream()
-              .map(key -> key + " -> " + fieldsToValues.get(key))
-              .collect(Collectors.joining(",\n"));
-
-      String providedKeysString = "[" + String.join(", ", fieldsToValues.keySet()) + "]";
+      String providedKeysString = "[" + String.join(", ", foundFields) + "]";
 
       String possibleOptions = getFieldsString(fieldSets).toString();
 
       throw new FactoryException(
           "The provided fields "
               + providedKeysString
-              + " with data \n{"
-              + providedFieldMapString
-              + "}"
-              + " are invalid for instance of "
-              + data.getTargetClass().getSimpleName()
-              + ". \nThe following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of '"
-              + data.getTargetClass().getSimpleName()
+              + " are invalid for instance of '"
+              + entityClass.getSimpleName()
+              + "'. \nThe following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of '"
+              + entityClass.getSimpleName()
               + "' are possible (NOT case-sensitive!):\n"
               + possibleOptions);
+    } else {
+      // checking for additional fields
+      Set<String> additionalFields = new HashSet<>();
+      Set<String> allFields =
+          validFieldSets.stream().flatMap(Collection::stream).collect(Collectors.toSet());
+
+      foundFields.stream().filter(e -> !allFields.contains(e)).forEach(additionalFields::add);
+      harmonizedFoundFields.stream()
+          .filter(e -> !allFields.contains(e))
+          .forEach(additionalFields::add);
+
+      if (!additionalFields.isEmpty()) {
+        log.debug(
+            "The following additional fields were found for instance of '{}': {}",
+            entityClass.getSimpleName(),
+            additionalFields);
+      }
     }
   }
 
-  protected static StringBuilder getFieldsString(Set<String>... fieldSets) {
+  protected static StringBuilder getFieldsString(List<Set<String>> fieldSets) {
     StringBuilder possibleOptions = new StringBuilder();
-    for (int i = 0; i < fieldSets.length; i++) {
-      Set<String> fieldSet = fieldSets[i];
-      String option = i + ": [" + String.join(", ", fieldSet) + "]\n";
+    for (int i = 0; i < fieldSets.size(); i++) {
+      Set<String> fieldSet = fieldSets.get(i);
+      String option =
+          i
+              + ": ["
+              + String.join(", ", fieldSet)
+              + "] or ["
+              + String.join(", ", toCamelCase(fieldSet))
+              + "]\n";
       possibleOptions.append(option);
     }
     return possibleOptions;
   }
 
   /**
    * Creates a new set of attribute names from given list of attributes. This method should always
-   * be used when returning attribute sets, i.e. through {@link #getFields(FactoryData)}.
+   * be used when returning attribute sets, i.e. through {@link #getFields(Class)}.
    *
    * @param attributes attribute names
    * @return new set exactly containing attribute names
@@ -203,4 +210,22 @@ protected TreeSet<String> expandSet(Set<String> attributeSet, String... more) {
     newSet.addAll(Arrays.asList(more));
     return newSet;
   }
+
+  protected static Set<String> toSnakeCase(Set<String> set) {
+    TreeSet<String> newSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+    newSet.addAll(set.stream().map(StringUtils::camelCaseToSnakeCase).toList());
+    return newSet;
+  }
+
+  protected static Set<String> toCamelCase(Set<String> set) {
+    TreeSet<String> newSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+    newSet.addAll(set.stream().map(StringUtils::snakeCaseToCamelCase).toList());
+    return newSet;
+  }
+
+  protected static Set<String> toLowerCase(Set<String> set) {
+    TreeSet<String> newSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+    newSet.addAll(set.stream().map(String::toLowerCase).toList());
+    return newSet;
+  }
 }

src/main/java/edu/ie3/datamodel/io/factory/input/AssetInputEntityFactory.java

@@ -25,8 +25,8 @@ public abstract class AssetInputEntityFactory<T extends AssetInput, D extends As
     extends EntityFactory<T, D> {
 
   private static final String UUID = "uuid";
-  private static final String OPERATES_FROM = "operatesfrom";
-  private static final String OPERATES_UNTIL = "operatesuntil";
+  private static final String OPERATES_FROM = "operatesFrom";
+  private static final String OPERATES_UNTIL = "operatesUntil";
   private static final String ID = "id";
 
   protected AssetInputEntityFactory(Class<? extends T>... allowedClasses) {
@@ -39,11 +39,11 @@ protected AssetInputEntityFactory(Class<? extends T>... allowedClasses) {
    * <p>The mandatory attributes required to create an {@link AssetInput} are enhanced with custom
    * attribute names that each subclass factory determines in {@link #getAdditionalFields()}.
    *
-   * @param data EntityData (or subclass) containing the data
+   * @param entityClass class of the entity
    * @return list of possible attribute sets
    */
   @Override
-  protected List<Set<String>> getFields(D data) {
+  protected List<Set<String>> getFields(Class<?> entityClass) {
     Set<String> constructorParamsMin = newSet(UUID, ID);
     Set<String> constructorParamsFrom = expandSet(constructorParamsMin, OPERATES_FROM);
     Set<String> constructorParamsUntil = expandSet(constructorParamsMin, OPERATES_UNTIL);

src/main/java/edu/ie3/datamodel/io/factory/input/ConnectorInputEntityFactory.java

@@ -29,7 +29,7 @@ abstract class ConnectorInputEntityFactory<
    * ConnectorInput}s. Thus, this attribute name declaration can be used in subclasses of {@link
    * ConnectorInputEntityFactory}
    */
-  protected static final String PARALLEL_DEVICES = "paralleldevices";
+  protected static final String PARALLEL_DEVICES = "parallelDevices";
 
   protected ConnectorInputEntityFactory(Class<? extends T>... allowedClasses) {
     super(allowedClasses);

src/main/java/edu/ie3/datamodel/io/factory/input/CylindricalStorageInputFactory.java

@@ -18,10 +18,10 @@
 
 public class CylindricalStorageInputFactory
     extends AssetInputEntityFactory<CylindricalStorageInput, ThermalUnitInputEntityData> {
-  private static final String STORAGE_VOLUME_LVL = "storagevolumelvl";
-  private static final String STORAGE_VOLUME_LVL_MIN = "storagevolumelvlmin";
-  private static final String INLET_TEMP = "inlettemp";
-  private static final String RETURN_TEMP = "returntemp";
+  private static final String STORAGE_VOLUME_LVL = "storageVolumeLvl";
+  private static final String STORAGE_VOLUME_LVL_MIN = "storageVolumeLvlMin";
+  private static final String INLET_TEMP = "inletTemp";
+  private static final String RETURN_TEMP = "returnTemp";
   private static final String C = "c";
 
   public CylindricalStorageInputFactory() {