Enforcing specific fields. Updating the test grid.

Author: staudtMarius

CHANGELOG.md

@@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased/Snapshot]
 
 ### Added
-- Added check for replaced field names in sources [#1383](https://github.com/ie3-institute/PowerSystemDataModel/issues/1383)
+- Enhanced check for invalid field names in sources [#1383](https://github.com/ie3-institute/PowerSystemDataModel/issues/1383)
 
 ### Fixed
 

src/main/java/edu/ie3/datamodel/io/factory/Factory.java

@@ -111,36 +111,24 @@ private void isSupportedClass(Class<?> desiredClass) {
    */
   protected abstract List<Set<String>> getFields(Class<?> entityClass);
 
-  /**
-   * Returns a map: old field name to new field name.
-   *
-   * @param entityClass class that can be used to specify the fields that are returned
-   * @return map of replaced field names
-   */
-  protected Map<String, String> getReplacedFields(Class<?> entityClass) {
-    return Collections.emptyMap();
-  }
-
   /**
    * Method to find and return additional fields that were found in a source and are not used by the
    * data model. This method will return the minimal unused fields among all field sets, meaning
    * that the set of actual fields is compared to the field set with the least unused fields.
    *
    * @param actualFields found in the source
    * @param validFieldSets that contains at least all fields found in the source
-   * @param replacedFields fields, that are replaced and therefore, reported as unused
    * @return a set of unused fields
    */
   protected Set<String> getUnusedFields(
-      Set<String> actualFields, List<Set<String>> validFieldSets, Set<String> replacedFields) {
+      Set<String> actualFields, List<Set<String>> validFieldSets) {
     // checking for additional fields
     // and returning the set with the least additional fields
     return validFieldSets.stream()
         .map(
             s -> {
               Set<String> set = new HashSet<>(actualFields);
               set.removeAll(s);
-              set.removeAll(replacedFields);
               return set;
             })
         .min(Comparator.comparing(Collection::size))
@@ -186,27 +174,8 @@ public Try<Void, ValidationException> validate(
                   + "' are possible (NOT case-sensitive!):\n"
                   + possibleOptions));
     } else {
-      // find all replaced fields
-      Map<String, String> replacedFields = getReplacedFields(entityClass);
-
-      Set<String> unsupportedFields =
-          actualFields.stream().filter(replacedFields::containsKey).collect(Collectors.toSet());
-
-      if (!unsupportedFields.isEmpty()) {
-        List<String> strings =
-            unsupportedFields.stream()
-                .map(field -> field + " -> " + replacedFields.get(field))
-                .toList();
-
-        log.warn(
-            "The following field were renamed '{}'. Please rename field like: [{}]",
-            unsupportedFields,
-            String.join(", ", strings));
-      }
-
       // find all unused fields
-      Set<String> unused =
-          getUnusedFields(harmonizedActualFields, validFieldSets, replacedFields.keySet());
+      Set<String> unused = getUnusedFields(harmonizedActualFields, validFieldSets);
 
       if (!unused.isEmpty()) {
         log.info(
@@ -248,17 +217,6 @@ protected static TreeSet<String> newSet(String... attributes) {
     return set;
   }
 
-  protected static Map.Entry<String, String> entry(String key, String value) {
-    return new AbstractMap.SimpleEntry<>(key, value);
-  }
-
-  @SafeVarargs
-  protected static Map<String, String> newMap(Map.Entry<String, String>... attributes) {
-    Map<String, String> map = new HashMap<>();
-    Arrays.stream(attributes).forEach(e -> map.put(e.getKey(), e.getValue()));
-    return map;
-  }
-
   /**
    * Expands a set of attributes with further attributes. This method should always be used when
    * returning attribute sets, i.e. through getting the needed fields. The set maintains a

src/main/java/edu/ie3/datamodel/io/factory/input/EmInputFactory.java

@@ -38,9 +38,4 @@ protected EmInput buildModel(
 
     return new EmInput(uuid, id, operator, operationTime, controlStrategy, parentEm);
   }
-
-  @Override
-  protected Map<String, String> getReplacedFields(Class<?> entityClass) {
-    return newMap(entry("parentEm", CONTROLLING_EM));
-  }
 }

src/main/java/edu/ie3/datamodel/io/factory/input/participant/SystemParticipantInputEntityFactory.java

@@ -44,24 +44,16 @@ protected SystemParticipantInputEntityFactory(Class<? extends T>... allowedClass
   @Override
   protected List<Set<String>> getFields(Class<?> entityClass) {
     List<Set<String>> fields = new ArrayList<>(super.getFields(entityClass));
+
     for (Set<String> set : fields) {
       set.add(Q_CHARACTERISTICS);
       set.add(NODE);
+      set.add(CONTROLLING_EM);
     }
 
-    List<Set<String>> withEm =
-        fields.stream().map(f -> (Set<String>) expandSet(f, CONTROLLING_EM)).toList();
-
-    fields.addAll(withEm);
-
     return fields;
   }
 
-  @Override
-  protected Map<String, String> getReplacedFields(Class<?> entityClass) {
-    return newMap(entry("em", CONTROLLING_EM));
-  }
-
   @Override
   protected T buildModel(
       D data, UUID uuid, String id, OperatorInput operator, OperationTime operationTime) {

src/test/groovy/edu/ie3/datamodel/io/factory/FactoryTest.groovy

@@ -18,7 +18,7 @@ class FactoryTest extends Specification {
 
   def "A Factory can return unused fields correctly"() {
     when:
-    def unused = factory.getUnusedFields(actualFields as Set<String>, validFieldSets, [] as Set<String>)
+    def unused = factory.getUnusedFields(actualFields as Set<String>, validFieldSets)
 
     then:
     unused == expected as Set<String>
@@ -98,14 +98,5 @@ class FactoryTest extends Specification {
         ["id", "time", "value1"] as Set<String>
       ]
     }
-
-    @Override
-    protected Map<String, String> getReplacedFields(Class<?> entityClass) {
-      if (entityClass.isAssignableFrom(EmInput)) {
-        return ["parentEm": "controllingEM"]
-      } else {
-        return [:]
-      }
-    }
   }
 }

src/test/groovy/edu/ie3/datamodel/io/factory/input/participant/FixedFeedInInputFactoryTest.groovy

@@ -88,21 +88,13 @@ class FixedFeedInInputFactoryTest extends Specification implements FactoryTestHe
     input.failure
     input.exception.get().message == "The provided fields [cosphi_rated, id, s_rated, uuid] are invalid for instance of 'FixedFeedInInput'. \n" +
         "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'FixedFeedInInput' are possible (NOT case-sensitive!):\n" +
-        "0: [cosPhiRated, id, node, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, node, q_characteristics, s_rated, uuid]\n" +
-        "1: [cosPhiRated, id, node, operatesFrom, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, node, operates_from, q_characteristics, s_rated, uuid]\n" +
-        "2: [cosPhiRated, id, node, operatesUntil, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, node, operates_until, q_characteristics, s_rated, uuid]\n" +
-        "3: [cosPhiRated, id, node, operatesFrom, operatesUntil, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, node, operates_from, operates_until, q_characteristics, s_rated, uuid]\n" +
-        "4: [cosPhiRated, id, node, operator, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, node, operator, q_characteristics, s_rated, uuid]\n" +
-        "5: [cosPhiRated, id, node, operatesFrom, operator, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, node, operates_from, operator, q_characteristics, s_rated, uuid]\n" +
-        "6: [cosPhiRated, id, node, operatesUntil, operator, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, node, operates_until, operator, q_characteristics, s_rated, uuid]\n" +
-        "7: [cosPhiRated, id, node, operatesFrom, operatesUntil, operator, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, node, operates_from, operates_until, operator, q_characteristics, s_rated, uuid]\n" +
-        "8: [controllingEm, cosPhiRated, id, node, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, node, q_characteristics, s_rated, uuid]\n" +
-        "9: [controllingEm, cosPhiRated, id, node, operatesFrom, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, node, operates_from, q_characteristics, s_rated, uuid]\n" +
-        "10: [controllingEm, cosPhiRated, id, node, operatesUntil, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, node, operates_until, q_characteristics, s_rated, uuid]\n" +
-        "11: [controllingEm, cosPhiRated, id, node, operatesFrom, operatesUntil, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, node, operates_from, operates_until, q_characteristics, s_rated, uuid]\n" +
-        "12: [controllingEm, cosPhiRated, id, node, operator, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, node, operator, q_characteristics, s_rated, uuid]\n" +
-        "13: [controllingEm, cosPhiRated, id, node, operatesFrom, operator, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, node, operates_from, operator, q_characteristics, s_rated, uuid]\n" +
-        "14: [controllingEm, cosPhiRated, id, node, operatesUntil, operator, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, node, operates_until, operator, q_characteristics, s_rated, uuid]\n" +
-        "15: [controllingEm, cosPhiRated, id, node, operatesFrom, operatesUntil, operator, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, node, operates_from, operates_until, operator, q_characteristics, s_rated, uuid]\n"
+        "0: [controllingEm, cosPhiRated, id, node, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, node, q_characteristics, s_rated, uuid]\n" +
+        "1: [controllingEm, cosPhiRated, id, node, operatesFrom, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, node, operates_from, q_characteristics, s_rated, uuid]\n" +
+        "2: [controllingEm, cosPhiRated, id, node, operatesUntil, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, node, operates_until, q_characteristics, s_rated, uuid]\n" +
+        "3: [controllingEm, cosPhiRated, id, node, operatesFrom, operatesUntil, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, node, operates_from, operates_until, q_characteristics, s_rated, uuid]\n" +
+        "4: [controllingEm, cosPhiRated, id, node, operator, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, node, operator, q_characteristics, s_rated, uuid]\n" +
+        "5: [controllingEm, cosPhiRated, id, node, operatesFrom, operator, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, node, operates_from, operator, q_characteristics, s_rated, uuid]\n" +
+        "6: [controllingEm, cosPhiRated, id, node, operatesUntil, operator, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, node, operates_until, operator, q_characteristics, s_rated, uuid]\n" +
+        "7: [controllingEm, cosPhiRated, id, node, operatesFrom, operatesUntil, operator, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, node, operates_from, operates_until, operator, q_characteristics, s_rated, uuid]\n"
   }
 }

src/test/resources/edu/ie3/datamodel/io/source/csv/_joint_grid/bm_input.csv

@@ -1,2 +1,2 @@
-uuid,cost_controlled,feed_in_tariff,id,market_reaction,node,operates_from,operates_until,operator,q_characteristics,type
-a3b7576b-cac7-4350-90ff-06316cdca192,true,51.0,BM_Test,true,f5839ade-5968-4879-a824-90b5fb3552cd,,,,"cosPhiFixed:{(0.0,1.0)}",2fdca5f1-c11b-4169-a695-4c98f0e0a84a
+uuid,cost_controlled,feed_in_tariff,id,market_reaction,node,operates_from,operates_until,operator,q_characteristics,type,controlling_em
+a3b7576b-cac7-4350-90ff-06316cdca192,true,51.0,BM_Test,true,f5839ade-5968-4879-a824-90b5fb3552cd,,,,"cosPhiFixed:{(0.0,1.0)}",2fdca5f1-c11b-4169-a695-4c98f0e0a84a,

src/test/resources/edu/ie3/datamodel/io/source/csv/_joint_grid/evcs_input.csv

@@ -1,3 +1,3 @@
-uuid,charging_points,cos_phi_rated,id,location_type,node,operates_from,operates_until,operator,q_characteristics,type,v_2g_support
-06a14909-366e-4e94-a593-1016e1455b30,4,0.9,test_evcs_1,HOME,5f1c776c-6935-40f7-ba9e-60646e08992b,,,,"cosPhiFixed:{(0.0,1.0)}",ChargingStationType1,true
-104acdaa-5dc5-4197-aed2-2fddb3c4f237,4,0.9,test_evcs_2,HOME,ed4697fd-016c-40c2-a66b-e793878dadea,,,,"cosPhiFixed:{(0.0,1.0)}",ChargingStationType1,true
+uuid,charging_points,cos_phi_rated,id,location_type,node,operates_from,operates_until,operator,q_characteristics,type,v_2g_support,controlling_em
+06a14909-366e-4e94-a593-1016e1455b30,4,0.9,test_evcs_1,HOME,5f1c776c-6935-40f7-ba9e-60646e08992b,,,,"cosPhiFixed:{(0.0,1.0)}",ChargingStationType1,true,
+104acdaa-5dc5-4197-aed2-2fddb3c4f237,4,0.9,test_evcs_2,HOME,ed4697fd-016c-40c2-a66b-e793878dadea,,,,"cosPhiFixed:{(0.0,1.0)}",ChargingStationType1,true,

src/test/resources/edu/ie3/datamodel/io/source/csv/_joint_grid/fixed_feed_in_input.csv

@@ -1,2 +1,2 @@
-uuid,cos_phi_rated,id,node,operates_from,operates_until,operator,q_characteristics,s_rated
-9abe950d-362e-4efe-b686-500f84d8f368,0.9,test_feed_in,5f1c776c-6935-40f7-ba9e-60646e08992b,,,,"cosPhiFixed:{(0.0,0.95)}",200.0
+uuid,cos_phi_rated,id,node,operates_from,operates_until,operator,q_characteristics,s_rated,controlling_em
+9abe950d-362e-4efe-b686-500f84d8f368,0.9,test_feed_in,5f1c776c-6935-40f7-ba9e-60646e08992b,,,,"cosPhiFixed:{(0.0,0.95)}",200.0,