Removing support for old csv format

CHANGELOG.md

@@ -39,6 +39,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Don't throw exceptions for not yet implemented validations [#879](https://github.com/ie3-institute/PowerSystemDataModel/issues/879)
 - `CsvDataSource` throws exceptions on error [#954](https://github.com/ie3-institute/PowerSystemDataModel/issues/954)
 - Removing `uuid` as required column from input and result time series [#826](https://github.com/ie3-institute/PowerSystemDataModel/issues/826)
+- Removing the support for the old csv format that was marked `deprecated` back in version `1.1.0` [#795](https://github.com/ie3-institute/PowerSystemDataModel/issues/795)
 
 ## [4.1.0] - 2023-11-02
 

docs/readthedocs/gettingstarted.md

@@ -1,5 +1,4 @@
 # Getting started
-
 Welcome, dear fellow of bottom up power system modelling!
 This section is meant to give you some help getting hands on our project.
 If you feel, something is missing, please contact us!
@@ -45,3 +44,32 @@ and add the dependency:
   <version>3.0-SNAPSHOT</version>
 </dependency>
 ```
+
+## Important changes
+
+With the release of `PowerSystemDataModel` version `5.0` the support for the old csv file format will be fully removed.
+It was already marked as `deprecated` back in version `1.1.0`. For those who are still using a model in the old csv format
+the following guide will provide a fast and easy way to convert old format into the new one.
+
+- Since the support is removed in version `5.0`, the `PowerSystemDataModel` version `3.x` or `4.x` must be 
+  used to read the old format. The `PSDM` will automatically write the output model in the new csv format.
+
+
+``` java
+    /* Parameterization */
+    String gridName = "gridWithOldFormat";
+    String csvSep = ",";
+    Path folderPath = Path.of("PATH_TO_THE_FOLDER");
+    boolean isHierarchic = false;
+    Path output = Path.of("PATH_OF_THE_OUTPUT_FOLDER");
+    FileNamingStrategy namingStrategy = new FileNamingStrategy();
+
+    /* Reading the old format */
+    JointGridContainer container = CsvJointGridContainerSource.read(gridName, csvSep, folderPath, isHierarchic);
+
+    /* Writing in the new format */
+    CsvFileSink sink = new CsvFileSink(output, namingStrategy, csvSep);
+    sink.persistJointGrid(container);
+```
+
+

src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java

@@ -11,16 +11,14 @@
 import edu.ie3.datamodel.io.source.DataSource;
 import edu.ie3.datamodel.models.Entity;
 import edu.ie3.datamodel.utils.Try;
-import edu.ie3.datamodel.utils.Try.*;
+import edu.ie3.datamodel.utils.Try.Failure;
+import edu.ie3.datamodel.utils.Try.Success;
 import edu.ie3.util.StringUtils;
 import java.io.BufferedReader;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.*;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
@@ -44,13 +42,6 @@ public class CsvDataSource implements DataSource {
 
   private final FileNamingStrategy fileNamingStrategy;
 
-  /**
-   * @deprecated ensures downward compatibility with old csv data format. Can be removed when
-   *     support for old csv format is removed. *
-   */
-  @Deprecated(since = "1.1.0", forRemoval = true)
-  private boolean notYetLoggedWarning = true;
-
   public CsvDataSource(String csvSep, Path folderPath, FileNamingStrategy fileNamingStrategy) {
     this.csvSep = csvSep;
     this.connector = new CsvFileConnector(folderPath, fileNamingStrategy);
@@ -90,6 +81,11 @@ public Stream<Map<String, String>> getSourceData(Class<? extends Entity> entityC
 
   // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
 
+  /** Returns the set {@link FileNamingStrategy}. */
+  public FileNamingStrategy getNamingStrategy() {
+    return fileNamingStrategy;
+  }
+
   /**
    * Takes a row string of a .csv file and a string array of the csv file headline, tries to split
    * the csv row string based and zip it together with the headline. This method does not contain
@@ -107,35 +103,14 @@ protected Map<String, String> buildFieldsToAttributes(
     TreeMap<String, String> insensitiveFieldsToAttributes =
         new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
 
-    // todo when replacing deprecated workaround code below add final modifier before parseCsvRow as
-    // well as remove
-    //  'finalFieldVals' and notYetLoggedWarning below!
-    String[] fieldVals = parseCsvRow(csvRow, csvSep);
-
-    // start workaround for deprecated data model processing
-    if (fieldVals.length != headline.length) {
-      // try to parse old structure
-      fieldVals = oldFieldVals(csvSep, csvRow);
-      // if this works log a warning to inform the user that this will not work much longer,
-      // otherwise parsing will fail regularly as expected below
-      if (fieldVals.length == headline.length && notYetLoggedWarning) {
-        notYetLoggedWarning = false;
-        log.warn(
-            "You are using an outdated version of the data "
-                + "model with invalid formatted csv rows. This is okay for now, but please updated your files, as the "
-                + "support for the old model will be removed soon.");
-      }
-    }
-    // end workaround for deprecated data model processing
-
     try {
-      String[] finalFieldVals = fieldVals;
+      String[] fieldVals = parseCsvRow(csvRow, csvSep);
       insensitiveFieldsToAttributes.putAll(
           IntStream.range(0, fieldVals.length)
               .boxed()
               .collect(
                   Collectors.toMap(
-                      k -> StringUtils.snakeCaseToCamelCase(headline[k]), v -> finalFieldVals[v])));
+                      k -> StringUtils.snakeCaseToCamelCase(headline[k]), v -> fieldVals[v])));
 
       if (insensitiveFieldsToAttributes.size() != headline.length) {
         Set<String> fieldsToAttributesKeySet = insensitiveFieldsToAttributes.keySet();
@@ -178,78 +153,6 @@ protected String[] parseCsvRow(String csvRow, String csvSep) {
         .toArray(String[]::new);
   }
 
-  /**
-   * Build an array of from the provided csv row string considering special cases where geoJson or
-   * {@link edu.ie3.datamodel.models.input.system.characteristic.CharacteristicInput} are provided
-   * in the csv row string.
-   *
-   * @param csvSep the column separator of the csv row string
-   * @param csvRow the csv row string
-   * @return an array with one entry per column of the provided csv row string
-   * @deprecated only left for downward compatibility. Will be removed in a major release
-   */
-  @Deprecated(since = "1.1.0", forRemoval = true)
-  protected String[] oldFieldVals(String csvSep, String csvRow) {
-
-    /*geo json support*/
-    final String geoJsonRegex = "\\{.+?}}}";
-    final String geoReplacement = "geoJSON";
-
-    /*characteristic input support */
-    final String charInputRegex = "(cP:|olm:|cosPhiFixed:|cosPhiP:|qV:)\\{[^}]++}";
-    final String charReplacement = "charRepl";
-
-    /*removes double double quotes*/
-    List<String> geoList = extractMatchingStrings(geoJsonRegex, csvRow.replace("\"\"", "\""));
-    List<String> charList = extractMatchingStrings(charInputRegex, csvRow.replace("\"\"", "\""));
-
-    AtomicInteger geoCounter = new AtomicInteger(0);
-    AtomicInteger charCounter = new AtomicInteger(0);
-
-    return Arrays.stream(
-            csvRow
-                .replaceAll(charInputRegex, charReplacement)
-                .replaceAll(geoJsonRegex, geoReplacement)
-                .replaceAll("\"*", "") // remove all quotes from
-                .split(csvSep, -1))
-        .map(
-            fieldVal -> {
-              String returningFieldVal = fieldVal;
-              if (fieldVal.equalsIgnoreCase(geoReplacement)) {
-                returningFieldVal = geoList.get(geoCounter.getAndIncrement());
-              }
-              if (fieldVal.equalsIgnoreCase(charReplacement)) {
-                returningFieldVal = charList.get(charCounter.getAndIncrement());
-              }
-              return returningFieldVal.trim();
-            })
-        .toArray(String[]::new);
-  }
-
-  /**
-   * Extracts all strings from the provided csvRow matching the provided regexString and returns a
-   * list of strings in the order of their appearance in the csvRow string
-   *
-   * @param regexString regex string that should be searched for
-   * @param csvRow csv row string that should be searched in for the regex string
-   * @return a list of strings matching the provided regex in the order of their appearance in the
-   *     provided csv row string
-   */
-  private List<String> extractMatchingStrings(String regexString, String csvRow) {
-    Pattern pattern = Pattern.compile(regexString);
-    Matcher matcher = pattern.matcher(csvRow);
-
-    ArrayList<String> matchingList = new ArrayList<>();
-    while (matcher.find()) {
-      matchingList.add(matcher.group());
-    }
-    return matchingList;
-  }
-
-  public FileNamingStrategy getNamingStrategy() {
-    return fileNamingStrategy;
-  }
-
   /**
    * Tries to open a file reader based on the provided entity class and hands it over for further
    * processing.

src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy

@@ -33,11 +33,6 @@ class CsvDataSourceTest extends Specification implements CsvTestDataMeta {
         String csvRow, String csvSep) {
       return super.parseCsvRow(csvRow, csvSep)
     }
-
-    String[] oldFieldVals(
-        String csvSep, String csvRow) {
-      return super.oldFieldVals(csvSep, csvRow)
-    }
   }
 
   @Shared
@@ -102,7 +97,7 @@ class CsvDataSourceTest extends Specification implements CsvTestDataMeta {
       "olmcharacteristic",
       "cosPhiFixed"
     ] as String[]
-    def validCsvRow = "5ebd8f7e-dedb-4017-bb86-6373c4b68eb8,25.0,100.0,0.95,98.0,test_bmTypeInput,50.0,25.0,olm:{(0.0,1.0)},cosPhiFixed:{(0.0,1.0)}"
+    def validCsvRow = "5ebd8f7e-dedb-4017-bb86-6373c4b68eb8,25.0,100.0,0.95,98.0,test_bmTypeInput,50.0,25.0,\"olm:{(0.0,1.0)}\",\"cosPhiFixed:{(0.0,1.0)}\""
 
     expect:
     dummyCsvSource.buildFieldsToAttributes(validCsvRow, validHeadline) == [
@@ -119,97 +114,6 @@ class CsvDataSourceTest extends Specification implements CsvTestDataMeta {
     ]
   }
 
-  def "A CsvDataSource should be able to handle deprecated invalid csvRows correctly"() {
-    expect:
-    dummyCsvSource.oldFieldVals(csvSep, csvRow) as List == resultingArray
-
-    where:
-    csvSep | csvRow                                                                                                                                                                                                                                                                                                                                                                                                              || resultingArray
-    ","    | "4ca90220-74c2-4369-9afa-a18bf068840d,{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}},node_a,2020-03-25T15:11:31Z[UTC],2020-03-24T15:11:31Z[UTC],8f9682df-0744-4b58-a122-f0dc730f6510,true,1,1.0,Höchstspannung,380.0,olm:{(0.00,1.00)},cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}"                                                     || [
-      "4ca90220-74c2-4369-9afa-a18bf068840d",
-      "{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}}",
-      "node_a",
-      "2020-03-25T15:11:31Z[UTC]",
-      "2020-03-24T15:11:31Z[UTC]",
-      "8f9682df-0744-4b58-a122-f0dc730f6510",
-      "true",
-      "1",
-      "1.0",
-      "Höchstspannung",
-      "380.0",
-      "olm:{(0.00,1.00)}",
-      "cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}"
-    ]
-    ","    | "\"4ca90220-74c2-4369-9afa-a18bf068840d\",\"{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}}\",\"node_a\",\"2020-03-25T15:11:31Z[UTC]\",\"2020-03-24T15:11:31Z[UTC]\",\"8f9682df-0744-4b58-a122-f0dc730f6510\",\"true\",\"1\",\"1.0\",\"Höchstspannung\",\"380.0\",\"olm:{(0.00,1.00)}\",\"cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}\"" || [
-      "4ca90220-74c2-4369-9afa-a18bf068840d",
-      "{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}}",
-      "node_a",
-      "2020-03-25T15:11:31Z[UTC]",
-      "2020-03-24T15:11:31Z[UTC]",
-      "8f9682df-0744-4b58-a122-f0dc730f6510",
-      "true",
-      "1",
-      "1.0",
-      "Höchstspannung",
-      "380.0",
-      "olm:{(0.00,1.00)}",
-      "cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}"
-    ]
-    ";"    | "4ca90220-74c2-4369-9afa-a18bf068840d;cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)};{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}};node_a;2020-03-25T15:11:31Z[UTC];2020-03-24T15:11:31Z[UTC];8f9682df-0744-4b58-a122-f0dc730f6510;true;1;1.0;Höchstspannung;380.0;olm:{(0.00,1.00)};cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}"            || [
-      "4ca90220-74c2-4369-9afa-a18bf068840d",
-      "cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}",
-      "{(0.0,1.0),(0.9,1.0),(1.2,-0.3)};{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}}",
-      "node_a",
-      "2020-03-25T15:11:31Z[UTC]",
-      "2020-03-24T15:11:31Z[UTC]",
-      "8f9682df-0744-4b58-a122-f0dc730f6510",
-      "true",
-      "1",
-      "1.0",
-      "Höchstspannung",
-      "380.0",
-      "olm:{(0.00,1.00)}",
-      "cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}"
-    ]
-    ";"    | "\"4ca90220-74c2-4369-9afa-a18bf068840d\";\"{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}}\";\"node_a\";\"2020-03-25T15:11:31Z[UTC]\";\"2020-03-24T15:11:31Z[UTC]\";\"8f9682df-0744-4b58-a122-f0dc730f6510\";\"true\";\"1\";\"1.0\";\"Höchstspannung\";\"380.0\";\"olm:{(0.00,1.00)}\";\"cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}\"" || [
-      "4ca90220-74c2-4369-9afa-a18bf068840d",
-      "{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}}",
-      "node_a",
-      "2020-03-25T15:11:31Z[UTC]",
-      "2020-03-24T15:11:31Z[UTC]",
-      "8f9682df-0744-4b58-a122-f0dc730f6510",
-      "true",
-      "1",
-      "1.0",
-      "Höchstspannung",
-      "380.0",
-      "olm:{(0.00,1.00)}",
-      "cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}"
-    ]
-    ","    | "66275bfd-978b-4974-9f73-f270165a6351,Standard,f18a5a9b-6d45-4843-be12-be6d12de0e6b,{\"type\":\"LineString\",\"coordinates\":[[7.4116482,51.4843281],[7.4116482,51.4843281]],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}},{\"type\":\"Point\",\"coordinates\":[0.25423729,0.75409836],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:0\"}}}\""                                  || [
-      "66275bfd-978b-4974-9f73-f270165a6351",
-      "Standard",
-      "f18a5a9b-6d45-4843-be12-be6d12de0e6b",
-      "{\"type\":\"LineString\",\"coordinates\":[[7.4116482,51.4843281],[7.4116482,51.4843281]],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}}",
-      "{\"type\":\"Point\",\"coordinates\":[0.25423729,0.75409836],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:0\"}}}"
-    ]
-    ","    | "4ca90220-74c2-4369-9afa-a18bf068840d,{\"\"type\"\":\"\"Point\"\",\"\"coordinates\"\":[7.411111,51.492528],\"\"crs\"\":{\"\"type\"\":\"\"name\"\",\"\"properties\"\":{\"\"name\"\":\"\"EPSG:4326\"\"}}},node_a,2020-03-25T15:11:31Z[UTC],2020-03-24T15:11:31Z[UTC],8f9682df-0744-4b58-a122-f0dc730f6510,true,1,1.0,Höchstspannung,380.0,\"olm:{(0.00,1.00)}\",\"cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}\""         || [
-      "4ca90220-74c2-4369-9afa-a18bf068840d",
-      "{\"type\":\"Point\",\"coordinates\":[7.411111,51.492528],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:4326\"}}}",
-      "node_a",
-      "2020-03-25T15:11:31Z[UTC]",
-      "2020-03-24T15:11:31Z[UTC]",
-      "8f9682df-0744-4b58-a122-f0dc730f6510",
-      "true",
-      "1",
-      "1.0",
-      "Höchstspannung",
-      "380.0",
-      "olm:{(0.00,1.00)}",
-      "cosPhiP:{(0.0,1.0),(0.9,1.0),(1.2,-0.3)}"
-    ]
-  }
-
   def "A CsvDataSource should be able to handle a variety of different csvRows correctly"() {
     expect:
     dummyCsvSource.parseCsvRow(csvRow, csvSep) as List == resultingArray

src/test/resources/edu/ie3/datamodel/io/source/csv/_grid/default/line_input.csv

@@ -1,3 +1,3 @@
 uuid,geo_position,id,length,node_a,node_b,olm_characteristic,operates_from,operates_until,operator,parallel_devices,type
-92ec3bcf-1777-4d38-af67-0bf7c9fa73c7,"{""type"":""LineString"",""coordinates"":[[7.411111,51.492528],[7.414116,51.484136]],""crs"":{""type"":""name"",""properties"":{""name"":""EPSG:4326""}}}",test_line_AtoB,0.003,4ca90220-74c2-4369-9afa-a18bf068840d,47d29df0-ba2d-4d23-8e75-c82229c5c758,olm:{(0.00,1.00)},2020-03-24T15:11:31Z[UTC],2020-03-25T15:11:31Z[UTC],f15105c4-a2de-4ab8-a621-4bc98e372d92,2,3bed3eb3-9790-4874-89b5-a5434d408088
-91ec3bcf-1777-4d38-af67-0bf7c9fa73c7,"{""type"":""LineString"",""coordinates"":[[7.411111,51.492528],[7.414116,51.484136]],""crs"":{""type"":""name"",""properties"":{""name"":""EPSG:4326""}}}",test_line_CtoD,0.003,bd837a25-58f3-44ac-aa90-c6b6e3cd91b2,6e0980e0-10f2-4e18-862b-eb2b7c90509b,olm:{(0.00,1.00)},2020-03-24T15:11:31Z[UTC],2020-03-25T15:11:31Z[UTC],f15105c4-a2de-4ab8-a621-4bc98e372d92,2,3bed3eb3-9790-4874-89b5-a5434d408088
+92ec3bcf-1777-4d38-af67-0bf7c9fa73c7,"{""type"":""LineString"",""coordinates"":[[7.411111,51.492528],[7.414116,51.484136]],""crs"":{""type"":""name"",""properties"":{""name"":""EPSG:4326""}}}",test_line_AtoB,0.003,4ca90220-74c2-4369-9afa-a18bf068840d,47d29df0-ba2d-4d23-8e75-c82229c5c758,"olm:{(0.00,1.00)}",2020-03-24T15:11:31Z[UTC],2020-03-25T15:11:31Z[UTC],f15105c4-a2de-4ab8-a621-4bc98e372d92,2,3bed3eb3-9790-4874-89b5-a5434d408088
+91ec3bcf-1777-4d38-af67-0bf7c9fa73c7,"{""type"":""LineString"",""coordinates"":[[7.411111,51.492528],[7.414116,51.484136]],""crs"":{""type"":""name"",""properties"":{""name"":""EPSG:4326""}}}",test_line_CtoD,0.003,bd837a25-58f3-44ac-aa90-c6b6e3cd91b2,6e0980e0-10f2-4e18-862b-eb2b7c90509b,"olm:{(0.00,1.00)}",2020-03-24T15:11:31Z[UTC],2020-03-25T15:11:31Z[UTC],f15105c4-a2de-4ab8-a621-4bc98e372d92,2,3bed3eb3-9790-4874-89b5-a5434d408088