Merge pull request #1385 from ie3-institute/ms/#1383-check-for-invalid-or-deprecated-fields-in-source-fields

Enhanced check for invalid field names in sources

Author: danielfeismann

CHANGELOG.md

@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased/Snapshot]
 
 ### Added
+- Enhanced check for invalid field names in sources [#1383](https://github.com/ie3-institute/PowerSystemDataModel/issues/1383)
 
 ### Fixed
 

src/main/java/edu/ie3/datamodel/io/factory/Factory.java

@@ -174,10 +174,11 @@ public Try<Void, ValidationException> validate(
                   + "' are possible (NOT case-sensitive!):\n"
                   + possibleOptions));
     } else {
+      // find all unused fields
       Set<String> unused = getUnusedFields(harmonizedActualFields, validFieldSets);
 
       if (!unused.isEmpty()) {
-        log.debug(
+        log.info(
             "The following additional fields were found for entity class of '{}': {}",
             entityClass.getSimpleName(),
             unused);

src/main/java/edu/ie3/datamodel/io/factory/input/participant/SystemParticipantInputEntityFactory.java

@@ -30,23 +30,26 @@ public abstract class SystemParticipantInputEntityFactory<
         T extends SystemParticipantInput, D extends SystemParticipantEntityData>
     extends AssetInputEntityFactory<T, D> {
 
+  private static final String NODE = "node";
+
   private static final String Q_CHARACTERISTICS = "qCharacteristics";
 
   public static final String CONTROLLING_EM = "controllingEm";
 
+  @SafeVarargs
   protected SystemParticipantInputEntityFactory(Class<? extends T>... allowedClasses) {
     super(allowedClasses);
   }
 
   @Override
   protected List<Set<String>> getFields(Class<?> entityClass) {
     List<Set<String>> fields = new ArrayList<>(super.getFields(entityClass));
-    for (Set<String> set : fields) set.add(Q_CHARACTERISTICS);
-
-    List<Set<String>> withEm =
-        fields.stream().map(f -> (Set<String>) expandSet(f, CONTROLLING_EM)).toList();
 
-    fields.addAll(withEm);
+    for (Set<String> set : fields) {
+      set.add(Q_CHARACTERISTICS);
+      set.add(NODE);
+      set.add(CONTROLLING_EM);
+    }
 
     return fields;
   }

src/test/groovy/edu/ie3/datamodel/io/factory/FactoryTest.groovy

@@ -6,6 +6,7 @@
 package edu.ie3.datamodel.io.factory
 
 import edu.ie3.datamodel.exceptions.FactoryException
+import edu.ie3.datamodel.models.input.EmInput
 import edu.ie3.datamodel.utils.Try
 import spock.lang.Shared
 import spock.lang.Specification

src/test/groovy/edu/ie3/datamodel/io/factory/input/participant/FixedFeedInInputFactoryTest.groovy

@@ -88,21 +88,13 @@ class FixedFeedInInputFactoryTest extends Specification implements FactoryTestHe
     input.failure
     input.exception.get().message == "The provided fields [cosphi_rated, id, s_rated, uuid] are invalid for instance of 'FixedFeedInInput'. \n" +
         "The following fields (without complex objects e.g. nodes, operators, ...) to be passed to a constructor of 'FixedFeedInInput' are possible (NOT case-sensitive!):\n" +
-        "0: [cosPhiRated, id, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, q_characteristics, s_rated, uuid]\n" +
-        "1: [cosPhiRated, id, operatesFrom, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, operates_from, q_characteristics, s_rated, uuid]\n" +
-        "2: [cosPhiRated, id, operatesUntil, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, operates_until, q_characteristics, s_rated, uuid]\n" +
-        "3: [cosPhiRated, id, operatesFrom, operatesUntil, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, operates_from, operates_until, q_characteristics, s_rated, uuid]\n" +
-        "4: [cosPhiRated, id, operator, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, operator, q_characteristics, s_rated, uuid]\n" +
-        "5: [cosPhiRated, id, operatesFrom, operator, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, operates_from, operator, q_characteristics, s_rated, uuid]\n" +
-        "6: [cosPhiRated, id, operatesUntil, operator, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, operates_until, operator, q_characteristics, s_rated, uuid]\n" +
-        "7: [cosPhiRated, id, operatesFrom, operatesUntil, operator, qCharacteristics, sRated, uuid] or [cos_phi_rated, id, operates_from, operates_until, operator, q_characteristics, s_rated, uuid]\n" +
-        "8: [controllingEm, cosPhiRated, id, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, q_characteristics, s_rated, uuid]\n" +
-        "9: [controllingEm, cosPhiRated, id, operatesFrom, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, operates_from, q_characteristics, s_rated, uuid]\n" +
-        "10: [controllingEm, cosPhiRated, id, operatesUntil, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, operates_until, q_characteristics, s_rated, uuid]\n" +
-        "11: [controllingEm, cosPhiRated, id, operatesFrom, operatesUntil, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, operates_from, operates_until, q_characteristics, s_rated, uuid]\n" +
-        "12: [controllingEm, cosPhiRated, id, operator, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, operator, q_characteristics, s_rated, uuid]\n" +
-        "13: [controllingEm, cosPhiRated, id, operatesFrom, operator, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, operates_from, operator, q_characteristics, s_rated, uuid]\n" +
-        "14: [controllingEm, cosPhiRated, id, operatesUntil, operator, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, operates_until, operator, q_characteristics, s_rated, uuid]\n" +
-        "15: [controllingEm, cosPhiRated, id, operatesFrom, operatesUntil, operator, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, operates_from, operates_until, operator, q_characteristics, s_rated, uuid]\n"
+        "0: [controllingEm, cosPhiRated, id, node, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, node, q_characteristics, s_rated, uuid]\n" +
+        "1: [controllingEm, cosPhiRated, id, node, operatesFrom, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, node, operates_from, q_characteristics, s_rated, uuid]\n" +
+        "2: [controllingEm, cosPhiRated, id, node, operatesUntil, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, node, operates_until, q_characteristics, s_rated, uuid]\n" +
+        "3: [controllingEm, cosPhiRated, id, node, operatesFrom, operatesUntil, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, node, operates_from, operates_until, q_characteristics, s_rated, uuid]\n" +
+        "4: [controllingEm, cosPhiRated, id, node, operator, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, node, operator, q_characteristics, s_rated, uuid]\n" +
+        "5: [controllingEm, cosPhiRated, id, node, operatesFrom, operator, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, node, operates_from, operator, q_characteristics, s_rated, uuid]\n" +
+        "6: [controllingEm, cosPhiRated, id, node, operatesUntil, operator, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, node, operates_until, operator, q_characteristics, s_rated, uuid]\n" +
+        "7: [controllingEm, cosPhiRated, id, node, operatesFrom, operatesUntil, operator, qCharacteristics, sRated, uuid] or [controlling_em, cos_phi_rated, id, node, operates_from, operates_until, operator, q_characteristics, s_rated, uuid]\n"
   }
 }

src/test/resources/edu/ie3/datamodel/io/source/csv/_joint_grid/bm_input.csv

@@ -1,2 +1,2 @@
-uuid,cost_controlled,feed_in_tariff,id,market_reaction,node,operates_from,operates_until,operator,q_characteristics,type
-a3b7576b-cac7-4350-90ff-06316cdca192,true,51.0,BM_Test,true,f5839ade-5968-4879-a824-90b5fb3552cd,,,,"cosPhiFixed:{(0.0,1.0)}",2fdca5f1-c11b-4169-a695-4c98f0e0a84a
+uuid,cost_controlled,feed_in_tariff,id,market_reaction,node,operates_from,operates_until,operator,q_characteristics,type,controlling_em
+a3b7576b-cac7-4350-90ff-06316cdca192,true,51.0,BM_Test,true,f5839ade-5968-4879-a824-90b5fb3552cd,,,,"cosPhiFixed:{(0.0,1.0)}",2fdca5f1-c11b-4169-a695-4c98f0e0a84a,

src/test/resources/edu/ie3/datamodel/io/source/csv/_joint_grid/evcs_input.csv

@@ -1,3 +1,3 @@
-uuid,charging_points,cos_phi_rated,id,location_type,node,operates_from,operates_until,operator,q_characteristics,type,v_2g_support
-06a14909-366e-4e94-a593-1016e1455b30,4,0.9,test_evcs_1,HOME,5f1c776c-6935-40f7-ba9e-60646e08992b,,,,"cosPhiFixed:{(0.0,1.0)}",ChargingStationType1,true
-104acdaa-5dc5-4197-aed2-2fddb3c4f237,4,0.9,test_evcs_2,HOME,ed4697fd-016c-40c2-a66b-e793878dadea,,,,"cosPhiFixed:{(0.0,1.0)}",ChargingStationType1,true
+uuid,charging_points,cos_phi_rated,id,location_type,node,operates_from,operates_until,operator,q_characteristics,type,v_2g_support,controlling_em
+06a14909-366e-4e94-a593-1016e1455b30,4,0.9,test_evcs_1,HOME,5f1c776c-6935-40f7-ba9e-60646e08992b,,,,"cosPhiFixed:{(0.0,1.0)}",ChargingStationType1,true,
+104acdaa-5dc5-4197-aed2-2fddb3c4f237,4,0.9,test_evcs_2,HOME,ed4697fd-016c-40c2-a66b-e793878dadea,,,,"cosPhiFixed:{(0.0,1.0)}",ChargingStationType1,true,

src/test/resources/edu/ie3/datamodel/io/source/csv/_joint_grid/fixed_feed_in_input.csv

@@ -1,2 +1,2 @@
-uuid,cos_phi_rated,id,node,operates_from,operates_until,operator,q_characteristics,s_rated
-9abe950d-362e-4efe-b686-500f84d8f368,0.9,test_feed_in,5f1c776c-6935-40f7-ba9e-60646e08992b,,,,"cosPhiFixed:{(0.0,0.95)}",200.0
+uuid,cos_phi_rated,id,node,operates_from,operates_until,operator,q_characteristics,s_rated,controlling_em
+9abe950d-362e-4efe-b686-500f84d8f368,0.9,test_feed_in,5f1c776c-6935-40f7-ba9e-60646e08992b,,,,"cosPhiFixed:{(0.0,0.95)}",200.0,