Fix issue 308 fillvalue (#309)

* added subset test for fillvalue issue 308

* base_operation.Operation._remove_redundant_fill_values - assert _FillValue and missing_value are the same, set both to missing_value if there are differences

* Update HISTORY.rst

---------

Co-authored-by: sol1105 <martin.schupfner@web.de>
Co-authored-by: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com>

HISTORY.rst

@@ -8,6 +8,7 @@ Other Changes
 ^^^^^^^^^^^^^
 * Warnings are now emitted if the user attempts to run the regridding utilities with a version of `xarray` that is not compatible with `cf-xarray`. (#310).
 * Dependency pins now constrain the `xarray` version when installing with `$ pip install ".[extra]"`. (#310).
+* Instead of raising an exception, now aligning _FillValue and missing_value if they deviate from one another. (#309).
 
 v0.12.0 (2023-11-23)
 --------------------

clisops/ops/base_operation.py

@@ -90,9 +90,14 @@ def _remove_redundant_fill_values(self, ds):
                 ds[var].encoding["missing_value"] = mval
                 ds[var].attrs.pop("missing_value", None)
             else:
+                # Issue 308 - Assert missing_value and _FillValue are the same
                 if fval != mval:
-                    raise Exception(
-                        f"The defined _FillValue and missing_value for '{var}' are not the same '{fval}' != '{mval}'."
+                    ds[var].encoding["_FillValue"] = mval
+                    ds[var].encoding["missing_value"] = mval
+                    ds[var].attrs.pop("missing_value", None)
+                    ds[var].attrs.pop("_FillValue", None)
+                    logger.warning(
+                        f"The defined _FillValue and missing_value for '{var}' are not the same '{fval}' != '{mval}'. Setting '{mval}' for both."
                     )
         return ds
 

tests/_common.py

@@ -309,6 +309,12 @@ def cmip6_archive_base():
     "master/test_data/badc/cmip6/data/CMIP6/CMIP/MPI-M/MPI-ESM1-2-LR/historical/r1i1p1f1/Omon/tauvo/gn/v20190710/tauvo_Omon_MPI-ESM1-2-LR_historical_r1i1p1f1_gn_185001.nc ",
 ).as_posix()
 
+# CMIP6 dataset to test fillvalue issue #308
+CMIP6_FILLVALUE = Path(
+    MINI_ESGF_CACHE_DIR,
+    "master/test_data/pool/data/CMIP6/data/CMIP6/CMIP/NCAR/CESM2-WACCM/historical/r1i1p1f1/day/tas/gn/v20190227/tas_day_CESM2-WACCM_historical_r1i1p1f1_gn_20000101-20091231.nc",
+).as_posix()
+
 # CORDEX dataset on regional curvilinear grid
 CORDEX_TAS_ONE_TIMESTEP = Path(
     MINI_ESGF_CACHE_DIR,

tests/test_ops_subset.py

@@ -20,6 +20,7 @@
     CMIP5_RH,
     CMIP5_TAS,
     CMIP5_ZOSTOGA,
+    CMIP6_FILLVALUE,
     CMIP6_MRSOFC,
     CMIP6_RLDS,
     CMIP6_RLDS_ONE_TIME_STEP,
@@ -1682,3 +1683,37 @@ def test_subset_cmip6_nc_consistent_bounds(cmip5_tas_file, tmpdir):
     assert "coordinates" not in res.lat_bnds.encoding
     assert "coordinates" not in res.lon_bnds.encoding
     assert "coordinates" not in res.time_bnds.encoding
+
+
+def test_subset_cmip6_issue_308_fillvalue(tmpdir, load_esgf_test_data, capsys):
+    """Tests clisops subset function with a time subset and check the metadata.
+
+    Notes
+    -----
+    This test is used for fillvalue issues. See: https://github.com/roocs/clisops/issues/308
+    """
+    from _common import ContextLogger
+    from clisops.utils.common import enable_logging
+
+    with ContextLogger():
+        enable_logging()
+        result = subset(
+            ds=CMIP6_FILLVALUE,
+            time=time_interval("2000-01-01T00:00:00", "2000-12-31T00:00:00"),
+            output_dir=tmpdir,
+            output_type="nc",
+            file_namer="simple",
+        )
+        res = _load_ds(result)
+        # check fill value in bounds
+        assert "_FillValue" not in res.lat_bnds.encoding
+        assert "_FillValue" not in res.lon_bnds.encoding
+        assert "_FillValue" not in res.time_bnds.encoding
+        # xarray should set the appropriate dtype
+        assert res.tas.encoding["_FillValue"].dtype == np.float32
+        assert res.tas.encoding["missing_value"].dtype == np.float32
+        captured = capsys.readouterr()
+        assert (
+            "The defined _FillValue and missing_value for 'tas' are not the same '1.0000000200408773e+20' != '1e+20'. Setting '1e+20' for both."
+            in captured.err
+        )