Update dumpers to write relative to yaml, make loader/dumper tests independent

Author: rly

poetry.lock

@@ -61,19 +61,17 @@ def _iterate_element(
             else:
                 output_file_name = f"{found_slot.name}"
 
-            # if output_dir is absolute, make it relative to current working directory
-            # and create the directory if it does not exist
-            if output_dir.is_absolute():
-                output_dir = Path(os.path.relpath(output_dir, start=os.getcwd()))
-            output_dir.mkdir(exist_ok=True)
-            output_file_path_no_suffix = output_dir / output_file_name
+            output_file_path_no_suffix = (output_dir / output_file_name)
 
             # save the numpy array to file and write the file path to the dictionary
             output_file_path = write_array(v, output_file_path_no_suffix)
+
+            # write the path to the array file relative to the output directory where the yaml is written
+            relative_output_file_path = os.path.relpath(output_file_path, start=output_dir)
             ret_dict[k] = {
                 "source": [
                     {
-                        "file": f"./{output_file_path}",
+                        "file": f"{relative_output_file_path}",
                         "format": format,
                     }
                 ]
@@ -100,6 +98,21 @@ class YamlArrayFileDumper(Dumper, metaclass=ABCMeta):
 
     # FORMAT is a class attribute that must be set by subclasses
 
+    def dump(
+        self,
+        element: Union[YAMLRoot, BaseModel],
+        to_file: str,
+        schemaview: SchemaView,
+        **kwargs,
+    ):
+        """Dump the element to a YAML file with paths to array files."""
+        output_dir = Path(to_file).parent
+        input = _iterate_element(
+            element, schemaview, Path(output_dir), self.write_array, self.FORMAT
+        )
+        with open(to_file, "w") as f:
+            yaml.dump(input, f)
+
     def dumps(
         self,
         element: Union[YAMLRoot, BaseModel],

src/linkml_arrays/dumpers/yaml_array_file_dumper.py

@@ -3,25 +3,25 @@ latitude_series:
   name: my_latitude
   values:
     source:
-      - file: ./out/my_latitude.values.h5
+      - file: my_latitude.values.h5
         format: hdf5
 longitude_series:
   name: my_longitude
   values:
     source:
-      - file: ./out/my_longitude.values.h5
+      - file: my_longitude.values.h5
         format: hdf5
 temperature_dataset:
   date:
     values:
       source:
-        - file: ./out/my_temperature.date.values.h5
+        - file: my_temperature.date.values.h5
           format: hdf5
   day_in_d:
     reference_date: '2020-01-01'
     values:
       source:
-        - file: ./out/my_temperature.day_in_d.values.h5
+        - file: my_temperature.day_in_d.values.h5
           format: hdf5
   latitude_in_deg: my_latitude
   longitude_in_deg: my_longitude
@@ -30,5 +30,5 @@ temperature_dataset:
     conversion_factor: 1000.0
     values:
       source:
-        - file: ./out/my_temperature.temperatures_in_K.values.h5
+        - file: my_temperature.temperatures_in_K.values.h5
           format: hdf5

tests/input/container_yaml.yaml renamed to tests/test_dumpers/ground_truth/container_yaml.yaml

@@ -3,25 +3,25 @@ latitude_series:
   name: my_latitude
   values:
     source:
-      - file: "./out/my_latitude.values.npy"
+      - file: my_latitude.values.npy
         format: numpy
 longitude_series:
   name: my_longitude
   values:
     source:
-      - file: "./out/my_longitude.values.npy"
+      - file: my_longitude.values.npy
         format: numpy
 temperature_dataset:
   date:
     values:
       source:
-        - file: "./out/my_temperature.date.values.npy"
+        - file: my_temperature.date.values.npy
           format: numpy
   day_in_d:
     reference_date: '2020-01-01'
     values:
       source:
-        - file: "./out/my_temperature.day_in_d.values.npy"
+        - file: my_temperature.day_in_d.values.npy
           format: numpy
   latitude_in_deg: my_latitude
   longitude_in_deg: my_longitude
@@ -30,5 +30,5 @@ temperature_dataset:
     conversion_factor: 1000.0
     values:
       source:
-        - file: "./out/my_temperature.temperatures_in_K.values.npy"
+        - file: my_temperature.temperatures_in_K.values.npy
           format: numpy

tests/input/container_yaml_hdf5.yaml renamed to tests/test_dumpers/ground_truth/container_yaml_hdf5.yaml

@@ -3,25 +3,25 @@ latitude_series:
   name: my_latitude
   values:
     source:
-      - file: "./out/my_latitude.values.nc"
+      - file: my_latitude.values.nc
         format: netcdf
 longitude_series:
   name: my_longitude
   values:
     source:
-      - file: "./out/my_longitude.values.nc"
+      - file: my_longitude.values.nc
         format: netcdf
 temperature_dataset:
   date:
     values:
       source:
-        - file: "./out/my_temperature.date.values.nc"
+        - file: my_temperature.date.values.nc
           format: netcdf
   day_in_d:
     reference_date: '2020-01-01'
     values:
       source:
-        - file: "./out/my_temperature.day_in_d.values.nc"
+        - file: my_temperature.day_in_d.values.nc
           format: netcdf
   latitude_in_deg: my_latitude
   longitude_in_deg: my_longitude
@@ -30,5 +30,5 @@ temperature_dataset:
     conversion_factor: 1000.0
     values:
       source:
-        - file: "./out/my_temperature.temperatures_in_K.values.nc"
+        - file: my_temperature.temperatures_in_K.values.nc
           format: netcdf

tests/input/container_yaml_numpy.yaml renamed to tests/test_dumpers/ground_truth/container_yaml_numpy.yaml

@@ -3,25 +3,25 @@ latitude_series:
   name: my_latitude
   values:
     source:
-      - file: "./out/my_latitude.values.zarr"
+      - file: my_latitude.values.zarr
         format: zarr
 longitude_series:
   name: my_longitude
   values:
     source:
-      - file: "./out/my_longitude.values.zarr"
+      - file: my_longitude.values.zarr
         format: zarr
 temperature_dataset:
   date:
     values:
       source:
-        - file: "./out/my_temperature.date.values.zarr"
+        - file: my_temperature.date.values.zarr
           format: zarr
   day_in_d:
     reference_date: '2020-01-01'
     values:
       source:
-        - file: "./out/my_temperature.day_in_d.values.zarr"
+        - file: my_temperature.day_in_d.values.zarr
           format: zarr
   latitude_in_deg: my_latitude
   longitude_in_deg: my_longitude
@@ -30,5 +30,5 @@ temperature_dataset:
     conversion_factor: 1000.0
     values:
       source:
-        - file: "./out/my_temperature.temperatures_in_K.values.zarr"
+        - file: my_temperature.temperatures_in_K.values.zarr
           format: zarr

tests/input/container_yaml_xarray_netcdf.yaml renamed to tests/test_dumpers/ground_truth/container_yaml_xarray_netcdf.yaml

@@ -0,0 +1 @@
+{}

tests/input/container_yaml_xarray_zarr.yaml renamed to tests/test_dumpers/ground_truth/container_yaml_xarray_zarr.yaml

@@ -0,0 +1,3 @@
+{
+    "zarr_format": 2
+}

tests/input/my_container.h5 renamed to tests/test_dumpers/ground_truth/my_container.h5

@@ -0,0 +1,37 @@
+{
+    "metadata": {
+        ".zattrs": {},
+        ".zgroup": {
+            "zarr_format": 2
+        },
+        "__xarray_dataarray_variable__/.zarray": {
+            "chunks": [
+                2,
+                2
+            ],
+            "compressor": {
+                "blocksize": 0,
+                "clevel": 5,
+                "cname": "lz4",
+                "id": "blosc",
+                "shuffle": 1
+            },
+            "dtype": "<f8",
+            "fill_value": "NaN",
+            "filters": null,
+            "order": "C",
+            "shape": [
+                2,
+                2
+            ],
+            "zarr_format": 2
+        },
+        "__xarray_dataarray_variable__/.zattrs": {
+            "_ARRAY_DIMENSIONS": [
+                "dim_0",
+                "dim_1"
+            ]
+        }
+    },
+    "zarr_consolidated_format": 1
+}

tests/input/my_container.nc renamed to tests/test_dumpers/ground_truth/my_container.nc

@@ -0,0 +1,22 @@
+{
+    "chunks": [
+        2,
+        2
+    ],
+    "compressor": {
+        "blocksize": 0,
+        "clevel": 5,
+        "cname": "lz4",
+        "id": "blosc",
+        "shuffle": 1
+    },
+    "dtype": "<f8",
+    "fill_value": "NaN",
+    "filters": null,
+    "order": "C",
+    "shape": [
+        2,
+        2
+    ],
+    "zarr_format": 2
+}

tests/input/my_container.zarr/.zattrs renamed to tests/test_dumpers/ground_truth/my_container.zarr/.zattrs

@@ -0,0 +1,6 @@
+{
+    "_ARRAY_DIMENSIONS": [
+        "dim_0",
+        "dim_1"
+    ]
+}

tests/input/my_container.zarr/.zgroup renamed to tests/test_dumpers/ground_truth/my_container.zarr/.zgroup

@@ -0,0 +1 @@
+{}

tests/input/my_container.zarr/latitude_series/.zattrs renamed to tests/test_dumpers/ground_truth/my_container.zarr/latitude_series/.zattrs

@@ -0,0 +1,3 @@
+{
+    "zarr_format": 2
+}

tests/input/my_container.zarr/latitude_series/.zgroup renamed to tests/test_dumpers/ground_truth/my_container.zarr/latitude_series/.zgroup

@@ -0,0 +1,37 @@
+{
+    "metadata": {
+        ".zattrs": {},
+        ".zgroup": {
+            "zarr_format": 2
+        },
+        "__xarray_dataarray_variable__/.zarray": {
+            "chunks": [
+                2,
+                2
+            ],
+            "compressor": {
+                "blocksize": 0,
+                "clevel": 5,
+                "cname": "lz4",
+                "id": "blosc",
+                "shuffle": 1
+            },
+            "dtype": "<f8",
+            "fill_value": "NaN",
+            "filters": null,
+            "order": "C",
+            "shape": [
+                2,
+                2
+            ],
+            "zarr_format": 2
+        },
+        "__xarray_dataarray_variable__/.zattrs": {
+            "_ARRAY_DIMENSIONS": [
+                "dim_0",
+                "dim_1"
+            ]
+        }
+    },
+    "zarr_consolidated_format": 1
+}