Add DD 3 to 4 conversion logic

imas/ids_convert.py

@@ -15,7 +15,7 @@
 from scipy.interpolate import interp1d
 
 import imas
-from imas.dd_zip import parse_dd_version
+from imas.dd_zip import parse_dd_version, dd_etree
 from imas.ids_base import IDSBase
 from imas.ids_data_type import IDSDataType
 from imas.ids_defs import IDS_TIME_MODE_HETEROGENEOUS
@@ -332,27 +332,59 @@ def add_rename(old_path: str, new_path: str):
             new_version = parse_dd_version(new_version_node.text)
         # Additional conversion rules for DDv3 to DDv4
         if self.version_old.major == 3 and new_version and new_version.major == 4:
-            # Postprocessing for COCOS definition change:
-            for psi_like in ["psi_like", "dodpsi_like"]:
-                xpath_query = f".//field[@cocos_label_transformation='{psi_like}']"
-                for old_item in old.iterfind(xpath_query):
-                    old_path = old_item.get("path")
-                    new_path = self.old_to_new.path.get(old_path, old_path)
-                    self.new_to_old.post_process[new_path] = _cocos_change
-                    self.old_to_new.post_process[old_path] = _cocos_change
-            # Definition change for pf_active circuit/connections
-            if self.ids_name == "pf_active":
-                path = "circuit/connections"
-                self.new_to_old.post_process[path] = _circuit_connections_4to3
-                self.old_to_new.post_process[path] = _circuit_connections_3to4
-            # Migrate ids_properties/source to ids_properties/provenance
-            # Only implement forward conversion (DD3 -> 4):
-            # - Pretend that this is a rename from ids_properties/source -> provenance
-            # - And register type_change handler which will be called with the source
-            #   element and the new provenance structure
-            path = "ids_properties/source"
-            self.old_to_new.path[path] = "ids_properties/provenance"
-            self.old_to_new.type_change[path] = _ids_properties_source
+            self._apply_3to4_conversion(old, new)
+
+    def _apply_3to4_conversion(self, old: Element, new: Element) -> None:
+        # Postprocessing for COCOS definition change:
+        for psi_like in ["psi_like", "dodpsi_like"]:
+            xpath_query = f".//field[@cocos_label_transformation='{psi_like}']"
+            for old_item in old.iterfind(xpath_query):
+                old_path = old_item.get("path")
+                new_path = self.old_to_new.path.get(old_path, old_path)
+                self.new_to_old.post_process[new_path] = _cocos_change
+                self.old_to_new.post_process[old_path] = _cocos_change
+        # Definition change for pf_active circuit/connections
+        if self.ids_name == "pf_active":
+            path = "circuit/connections"
+            self.new_to_old.post_process[path] = _circuit_connections_4to3
+            self.old_to_new.post_process[path] = _circuit_connections_3to4
+
+        # Migrate ids_properties/source to ids_properties/provenance
+        # Only implement forward conversion (DD3 -> 4):
+        # - Pretend that this is a rename from ids_properties/source -> provenance
+        # - And register type_change handler which will be called with the source
+        #   element and the new provenance structure
+        path = "ids_properties/source"
+        self.old_to_new.path[path] = "ids_properties/provenance"
+        self.old_to_new.type_change[path] = _ids_properties_source
+
+        # GH#55: add logic to migrate some obsolete nodes in DD3.42.0 -> 4.0
+        # These nodes (e.g. equilibrium profiles_1d/j_tor) have an NBC rename rule
+        # (to e.g. equilibrium profiles_1d/j_phi) applying to DD 3.41 and older.
+        # In DD 3.42, both the old AND new node names are present.
+        if self.version_old.minor >= 42:  # Only apply for DD 3.42+ -> DD 4
+            # Get a rename map for 3.41 -> new version
+            factory341 = imas.IDSFactory("3.41.0")
+            if self.ids_name in factory341.ids_names():  # Ensure the IDS exists in 3.41
+                dd341_map = _DDVersionMap(
+                    self.ids_name,
+                    dd_etree("3.41.0"),
+                    self.new_version,
+                    Version("3.41.0"),
+                )
+                to_update = {}
+                for path, newpath in self.old_to_new.path.items():
+                    # Find all nodes that have disappeared in DD 4.x, and apply the
+                    # rename rule from DD3.41 -> DD 4.x
+                    if newpath is None and path in dd341_map.old_to_new:
+                        self.old_to_new.path[path] = dd341_map.old_to_new.path[path]
+                        # Note: path could be a structure or AoS, so we also put all
+                        # child paths in our map:
+                        path = path + "/"  # All child nodes will start with this
+                        for p, v in dd341_map.old_to_new.path.items():
+                            if p.startswith(path):
+                                to_update[p] = v
+                self.old_to_new.path.update(to_update)
 
     def _map_missing(self, is_new: bool, missing_paths: Set[str]):
         rename_map = self.new_to_old if is_new else self.old_to_new

imas/test/test_ids_convert.py

@@ -481,3 +481,51 @@ def test_3to4_pulse_schedule_fuzz():
 
     fill_consistent(ps)
     convert_ids(ps, "4.0.0")
+
+
+def test_3to4_migrate_deprecated_fields():  # GH#55
+    # Test j_phi -> j_tor rename
+    eq342 = IDSFactory("3.42.0").equilibrium()
+    eq342.ids_properties.homogeneous_time = IDS_TIME_MODE_HOMOGENEOUS
+    eq342.time = [0.0]
+    eq342.time_slice.resize(1)
+    eq342.time_slice[0].profiles_1d.j_tor = [0.3, 0.2, 0.1]
+    eq342.time_slice[0].profiles_1d.j_tor_error_upper = [1.0]
+    eq342.time_slice[0].profiles_1d.j_tor_error_lower = [2.0]
+    eq342.time_slice[0].profiles_1d.psi = [1.0, 0.5, 0.0]
+
+    # Basic case, check that j_tor (although deprecated) is migrated to j_phi:
+    eq4 = convert_ids(eq342, "4.0.0")
+    assert array_equal(eq4.time_slice[0].profiles_1d.j_phi.value, [0.3, 0.2, 0.1])
+    assert array_equal(eq4.time_slice[0].profiles_1d.j_phi_error_upper.value, [1.0])
+    assert array_equal(eq4.time_slice[0].profiles_1d.j_phi_error_lower.value, [2.0])
+
+    # When both j_tor and j_phi are present in the source IDS, we expect that j_phi
+    # takes precedence. This is a happy accident with how the DD defines both attributes
+    eq342.time_slice[0].profiles_1d.j_phi = [0.6, 0.4, 0.2]
+    eq4 = convert_ids(eq342, "4.0.0")
+    assert array_equal(eq4.time_slice[0].profiles_1d.j_phi.value, [0.6, 0.4, 0.2])
+
+    # Just to be sure, when j_tor has no value, it should also still work
+    del eq342.time_slice[0].profiles_1d.j_tor
+    eq4 = convert_ids(eq342, "4.0.0")
+    assert array_equal(eq4.time_slice[0].profiles_1d.j_phi.value, [0.6, 0.4, 0.2])
+
+    # Same applies to label -> name renames
+    cp342 = IDSFactory("3.42.0").core_profiles()
+    cp342.ids_properties.homogeneous_time = IDS_TIME_MODE_HOMOGENEOUS
+    cp342.time = [0.0]
+    cp342.profiles_1d.resize(1)
+    cp342.profiles_1d[0].ion.resize(1)
+    cp342.profiles_1d[0].ion[0].label = "x"
+
+    cp4 = convert_ids(cp342, "4.0.0")
+    assert cp4.profiles_1d[0].ion[0].name == "x"
+
+    cp342.profiles_1d[0].ion[0].name = "y"
+    cp4 = convert_ids(cp342, "4.0.0")
+    assert cp4.profiles_1d[0].ion[0].name == "y"
+
+    del cp342.profiles_1d[0].ion[0].label
+    cp4 = convert_ids(cp342, "4.0.0")
+    assert cp4.profiles_1d[0].ion[0].name == "y"