Blended forecast coordinate deprecation (metoppv#1393)

* Added code to deprecate forecast period and forecast reference time on cycle or model blending.

* Regenerated acceptance test data, checked and fixed bug.

* Response to first review.

improver/blending/calculate_weights_and_blend.py

@@ -192,6 +192,10 @@ def _update_metadata_only(self, cube, attributes_dict, cycletime):
 
         (result,) = rebadge_forecasts_as_latest_cycle([result], cycletime)
         if self.blend_coord in ["forecast_reference_time", "model_id"]:
+            for coord in ["forecast_period", "forecast_reference_time"]:
+                msg = f"{coord} will be removed in future and should not be used"
+                result.coord(coord).attributes.update({"deprecation_message": msg})
+
             if cycletime is not None:
                 add_blend_time(result, cycletime)
             else:

improver/blending/weighted_blend.py

@@ -164,9 +164,19 @@ def _remove_blend_time(cube):
             cube.remove_coord("blend_time")
         return cube
 
+    @staticmethod
+    def _remove_deprecation_warnings(cube):
+        """Remove deprecation warnings from forecast period and forecast reference
+        time coordinates so that these can be merged before blending"""
+        for coord in ["forecast_period", "forecast_reference_time"]:
+            cube.coord(coord).attributes.pop("deprecation_message", None)
+        return cube
+
     def process(self, cubes_in, cycletime=None):
         """
-        Prepares merged input cube for cycle and grid blending
+        Prepares merged input cube for cycle and grid blending. Makes updates to
+        metadata (attributes and time-type coordinates) ONLY in so far as these are
+        needed to ensure inputs can be merged into a single cube.
 
         Args:
             cubes_in (iris.cube.CubeList or iris.cube.Cube):
@@ -192,6 +202,7 @@ def process(self, cubes_in, cycletime=None):
 
         if "model" in self.blend_coord:
             cubelist = [self._remove_blend_time(cube) for cube in cubelist]
+            cubelist = [self._remove_deprecation_warnings(cube) for cube in cubelist]
             if (
                 self.weighting_coord is not None
                 and "forecast_period" in self.weighting_coord
@@ -801,10 +812,11 @@ def _get_cycletime_point(self, cube):
     def _set_blended_time_coords(self, blended_cube):
         """
         For cycle and model blending:
-        - Update the forecast reference time and forecast period coordinate points
-        to reflect the current cycle time
-        - Remove any bounds from the forecast reference time
         - Add a "blend_time" coordinate equal to the current cycletime
+        - Update the forecast reference time and forecast period coordinate points
+        to reflect the current cycle time (behaviour is DEPRECATED)
+        - Remove any bounds from the forecast reference time (behaviour is DEPRECATED)
+        - Mark the forecast reference time and forecast period as DEPRECATED
 
         Modifies cube in place.
 
@@ -818,14 +830,17 @@ def _set_blended_time_coords(self, blended_cube):
                 "Current cycle time is required for cycle and model blending"
             )
 
+        add_blend_time(blended_cube, self.cycletime)
         blended_cube.coord("forecast_reference_time").points = [cycletime_point]
         blended_cube.coord("forecast_reference_time").bounds = None
         if blended_cube.coords("forecast_period"):
             blended_cube.remove_coord("forecast_period")
         new_forecast_period = forecast_period_coord(blended_cube)
         time_dim = blended_cube.coord_dims("time")
         blended_cube.add_aux_coord(new_forecast_period, data_dims=time_dim)
-        add_blend_time(blended_cube, self.cycletime)
+        for coord in ["forecast_period", "forecast_reference_time"]:
+            msg = f"{coord} will be removed in future and should not be used"
+            blended_cube.coord(coord).attributes.update({"deprecation_message": msg})
 
     def _update_blended_metadata(self, blended_cube, attributes_dict):
         """

improver/metadata/constants/mo_attributes.py

@@ -35,7 +35,7 @@
 
 MOSG_GRID_ATTRIBUTES = {"mosg__grid_type", "mosg__grid_version", "mosg__grid_domain"}
 
-# Define current StaGE grid metadata
+# Define current StaGE and MONOW grid metadata
 MOSG_GRID_DEFINITION = {
     "uk_ens": {
         "mosg__grid_type": GRID_TYPE,
@@ -61,6 +61,7 @@
         "mosg__grid_domain": "global",
         "mosg__grid_version": STAGE_VERSION,
     },
+    "nc_det": {"mosg__model_configuration": "nc_det",},
 }
 
 # Map correct metadata from StaGE v1.1.0

improver/metadata/forecast_times.py

@@ -35,6 +35,7 @@
 import iris
 import numpy as np
 from cf_units import Unit
+from iris.coords import AuxCoord, DimCoord
 from iris.exceptions import CoordinateNotFoundError
 
 from improver.metadata.check_datatypes import check_mandatory_standards
@@ -67,7 +68,7 @@ def forecast_period_coord(cube, force_lead_time_calculation=False):
     """
     create_dim_coord = False
     if cube.coords("forecast_period"):
-        if isinstance(cube.coord("forecast_period"), iris.coords.DimCoord):
+        if isinstance(cube.coord("forecast_period"), DimCoord):
             create_dim_coord = True
 
     if cube.coords("forecast_period") and not force_lead_time_calculation:
@@ -138,7 +139,7 @@ def _calculate_forecast_period(
     else:
         required_lead_time_bounds = None
 
-    coord_type = iris.coords.DimCoord if dim_coord else iris.coords.AuxCoord
+    coord_type = DimCoord if dim_coord else AuxCoord
     result_coord = coord_type(
         required_lead_times,
         standard_name="forecast_period",
@@ -188,8 +189,10 @@ def _create_frt_type_coord(cube, point, name="forecast_reference_time"):
     coord_type_spec = TIME_COORDS[frt_coord_name]
     coord_units = Unit(coord_type_spec.units)
     new_points = round_close([coord_units.date2num(point)], dtype=coord_type_spec.dtype)
-    new_coord = cube.coord(frt_coord_name).copy(points=new_points)
-    new_coord.rename(name)
+    try:
+        new_coord = DimCoord(new_points, standard_name=name, units=coord_units)
+    except ValueError:
+        new_coord = DimCoord(new_points, long_name=name, units=coord_units)
     return new_coord
 
 

improver_tests/acceptance/SHA256SUMS

@@ -53,7 +53,7 @@ f7ab20f8ae2f8593ab39f354484a01b122905f5e10a8e4aab516262f6a5a4664  ./blend-adjace
 5b3b15c00cc1203a56cd48f18970323ed26b3b55ef3bf82075148c8a86b2655a  ./blend-adjacent-points/time_bounds/kgo.nc
 44c5013aa2712a578d87b687cfe90388c91b18370badc624c8a85099c3ed8989  ./blend-cycles-and-realizations/basic/0900Z_precip_rate.nc
 9a7011137198b5cd8c0b222942ebc3673ab3764829e6979585fccaae67880a89  ./blend-cycles-and-realizations/basic/1000Z_precip_rate.nc
-de1bcff4bbc69264e3d93c170e8f8c926deda40fe363becbb79dfaaa3099abc6  ./blend-cycles-and-realizations/basic/kgo.nc
+f0e677d691c5739c4cd17e8ca0b9ade475416c4ad71fe5f7f23a4bb350fe6dac  ./blend-cycles-and-realizations/basic/kgo.nc
 d5e0180cc33eb4f4b0c86ab5c5ac77fce77eff14734122530395724c1f7b6535  ./combine/accum/20180101T0100Z-PT0001H-rainfall_accumulation.nc
 3d6e33d3ac42b59c7b74e54abe3d7455d57f07f387472f070a0ba493330028b8  ./combine/accum/20180101T0200Z-PT0002H-rainfall_accumulation.nc
 0c848cfbd577f22f9b1cc5f9c6005790d04a720ff32d2ae81e01caa082b3dfb8  ./combine/accum/20180101T0300Z-PT0003H-rainfall_accumulation.nc
@@ -476,49 +476,49 @@ a44625de472458c728dbdec023f3c5b93adb30d09959d125c6772d70d3eba076  ./time-lagged-
 abb612a42deaaee46ad5574fe0da845a4e367ecba42b9cf3bb80b6c0a308a398  ./uv-index/basic/20181210T0600Z-PT0000H00M-radiation_flux_in_uv_downward_at_surface.nc
 07670a4333f994f4704bd931569048416080baf10a424fc8830627bf7259ae45  ./uv-index/basic/20181210T0600Z-PT0000H00M-radiation_flux_in_uv_upward_at_surface.nc
 490367f992aa0fd0b734e2e5550117f44f47cc58bc281337b64764a107ba0b88  ./uv-index/basic/kgo.nc
-3b16f2c35ee2d920c60dc73635e91ffce266cf2e4b9cacf9b3cd24bf211f8162  ./weighted_blending/accum_cycle_blend/kgo.nc
+39f4e26f97d10bdc1ae612cb0a5c81517883e6d8ec54a534d6b4fa8b42301aca  ./weighted_blending/accum_cycle_blend/kgo.nc
 44d59ee66384542e34a20106e76119da1852a57a9a43fdace71a17c7a07d1637  ./weighted_blending/accum_cycle_blend/ukv_prob_accum_PT3H.nc
 2621d0b98ff12ba3c1e6356b2df3964536e8a1769c7c08c62805705a34a9549c  ./weighted_blending/accum_cycle_blend/ukv_prob_accum_PT4H.nc
 c163c7abc3cfca24c9509b9f461da8e6defc2af6a33f26b52dc52d066389c265  ./weighted_blending/attributes.json
-bff22bbef96a4d5007d2f3becf6580e7de5d281abbb90beb9916de2dd1b998b9  ./weighted_blending/basic_lin/kgo.nc
+7a42f552e18b428c1960ab794dcad75d32fa3372ff69fdcd9e697c0922cbeef9  ./weighted_blending/basic_lin/kgo.nc
 abbd593e44562084dde15730f4c49c2b44183665a4c7cf0c9e689f4292a42a27  ./weighted_blending/basic_lin/multiple_probabilities_rain_1H.nc
 bd5aad477f6ae70242cbc5d9f218ca970ec8b00ea4b5c302c57fef9f0b4ff22c  ./weighted_blending/basic_lin/multiple_probabilities_rain_2H.nc
 93b5ff885e6d79efc90d90ec3a512b3908c0145bd9c70d72b71a8a86d6dda36f  ./weighted_blending/basic_lin/multiple_probabilities_rain_3H.nc
-8925b3c7a13ebcea164da09a6f6f2630cd495aeca12700283bdd02402cab11d1  ./weighted_blending/basic_nonlin/kgo.nc
+1a53263de78c0faed5ae83c93226f489d49dc58be4240614f79f232db2b596d2  ./weighted_blending/basic_nonlin/kgo.nc
 ae39e532a031cd1331594a940c18b3bde8df8d909b8d268a30b387c32656d948  ./weighted_blending/cycletime/input_temperature_0.nc
 00e63a136e21c9b5091949a335382ee684cf343bde68c297cf19b7c1eba263df  ./weighted_blending/cycletime/input_temperature_1.nc
 133588df222e60800af74b0a81d98a7c4625ee084537713a287b6214c3e18088  ./weighted_blending/cycletime/input_temperature_2.nc
-0d655f0e601a04ad6e0f9ec5ca273494eb56f5bb7adfb5ce4c5b227eb70535e3  ./weighted_blending/cycletime/kgo_single_input.nc
-5b3a0871e6b6e3a9239f70118aa929c88141415cdf5042fb0cb590219ec91d20  ./weighted_blending/cycletime/kgo_specified_frt.nc
+f81413994a9924cb5f2a2171dc6d7a0fbddbf73a8a019d2d5eefdca9ad7427cc  ./weighted_blending/cycletime/kgo_single_input.nc
+3c6e3c971270272784750f808993b424a817806ac9aab33887968a3342d0a5e0  ./weighted_blending/cycletime/kgo_specified_frt.nc
 12522741f9618cebfe135ddfcfbcae6b3d66bd582499560008dd1f76b53b93db  ./weighted_blending/model/enuk_input.nc
-ecd1b261fd19a65bf76b35f4e35bd4201cf25b2a54d707b3c5ac1497073cc60b  ./weighted_blending/model/kgo.nc
+31f4c5e16603d6e76494fc1e6a716dead2d7ca04c8b8514553f999b004ed0485  ./weighted_blending/model/kgo.nc
 73bab9d2e95fa7cf3af797f533f2194e682460a833ca4a4243641d2ea203b21b  ./weighted_blending/model/ukv_input.nc
 7dd21ad6014ce01f1c77eb58f52d8006c441c8d4d93081f24f0c45d8ff2ba3ed  ./weighted_blending/non_mo_attributes.json
-1b892f00db230754d3e68d0bb2a2ec11012219bb21fb2f56857aa4e136c4c49e  ./weighted_blending/non_mo_model/kgo.nc
+4e6529db034099a327bdf06a5eaec69c14e7ecffdf37e0e0d332d08e9b77b797  ./weighted_blending/non_mo_model/kgo.nc
 5a607718c4d0716759d9cdafbfb2c2f255d37a68247b9dd19772c2e362f714a7  ./weighted_blending/non_mo_model/non_mo_det.nc
 3e1a1dd4ef00a11897394bb567be6ab71ebda76581917a1ed4458d9627f88765  ./weighted_blending/non_mo_model/non_mo_ens.nc
 c5453445db66742c0a746a66431d6f934b6ef9077cccf66995dcfa1dbd98c3ad  ./weighted_blending/percentile_weights_from_dict/enuk_input.nc
-0d8133524a9a52a125ff709adaa049850711f80007bd3daf1dda7f098d64a92c  ./weighted_blending/percentile_weights_from_dict/kgo.nc
+8fd258f4e73489a34975abbb58cb8362d497230cb3ff47141498af93d3887cae  ./weighted_blending/percentile_weights_from_dict/kgo.nc
 53209899ecc2b3bb9e29fbc9b94b16ef875aa10366033940778748002c639800  ./weighted_blending/percentile_weights_from_dict/ukv_input.nc
 39a44d152b84d5eaf1945e5dac355f2468f3dbef6c53346a923a4bd95bed2a0b  ./weighted_blending/percentiles/input.nc
-77feb99f92e6f0de639097ccb7cea2a61d9d9e13b3d7d31d43a13493df84eb75  ./weighted_blending/percentiles/kgo.nc
+f75721b1868517e4f36a270dcb8dbdf71772102fc831970e6475d120d5cce697  ./weighted_blending/percentiles/kgo.nc
 ca8e236046d649cb22aaa04d52862161d1461210677d32ccd47752787f963f18  ./weighted_blending/realizations/input.nc
 f7deb5c422ce57098ce3438f446f242b4b81f0401a1fdccdcb3116c86434f473  ./weighted_blending/realizations/kgo.nc
-010da2b8cdf8cedfcba556620feab7847381a046d07392f015df519ad7f1e12c  ./weighted_blending/spatial_weights/kgo/cycle.nc
-22297767086a02eb2a91a4ae7f43e9c26f4f992ecf48b5393f97b7f989c87720  ./weighted_blending/spatial_weights/kgo/cycle_no_fuzzy.nc
-001403649c81ad683b19b40334050cb87a88d6918e9bf7aaf813a26d62e6caae  ./weighted_blending/spatial_weights/kgo/model.nc
-97064dd5009c07f24f798cf6e0e2987ebae102f3e03fb1c3ae5da6f6b420ba9d  ./weighted_blending/spatial_weights/kgo/model_no_fuzzy.nc
+a5a6c91c36dbab853ff92e6c7a8e4da006485c75ed8f37e7331aab2cb61da19b  ./weighted_blending/spatial_weights/kgo/cycle.nc
+f13078aea2da97af49feecaaab4fe4217dd9783bb09d7ec18e7b7b0c87208427  ./weighted_blending/spatial_weights/kgo/cycle_no_fuzzy.nc
+00306693137c6b4dba8a1bfab667b4455a40e70834d169394b2317b20440a6b4  ./weighted_blending/spatial_weights/kgo/model.nc
+6dc294dc69b11e679cd65f20f71a83a59f3ba139f083f8b13849307a2d3a12f3  ./weighted_blending/spatial_weights/kgo/model_no_fuzzy.nc
 ff5e4d0a6c2cd347da5109b8602968b94e2b5e396608d8dcc2156391a7a10752  ./weighted_blending/spatial_weights/nowcast_data/20181129T1000Z-PT0002H00M-lwe_precipitation_rate.nc
 7a7e1bd7397828a91ec5a80a46468307706fc7bdb291840810d2010029406509  ./weighted_blending/spatial_weights/nowcast_data/20181129T1000Z-PT0003H00M-lwe_precipitation_rate.nc
 ee93c4134fde76ded27b56fae3493db841e2beb330acc845a1f3616b081b390b  ./weighted_blending/spatial_weights/nowcast_data/20181129T1000Z-PT0004H00M-lwe_precipitation_rate.nc
 db8b35b973cfa2f51a8d7269c1599bea747994e1bc19f1c2df75ae9818ce6027  ./weighted_blending/spatial_weights/ukvx_data/20181129T1000Z-PT0002H00M-lwe_precipitation_rate.nc
 efd633380f8404e90af9dd3d4cf8ba34ffaa3258f6a6112fda9e06ad708dc7c2  ./weighted_blending/three_models/blending-weights-preciprate.json
 166f8cd9c254262373fd5b58326b9ea12dd9bdfc89c2ab9d5872ee2fe0788c0d  ./weighted_blending/three_models/enukxhrly/20190101T0400Z-PT0004H00M-precip_rate.nc
-8e86a9f46baf2e9ecdfd52e2adda2df86e4e7541bb3d61f11c2a5a4cceb19761  ./weighted_blending/three_models/kgo.nc
+cb41135cd15369f7df16fe4865ed84643d5bd50bbc16af4acc721011368dd7e8  ./weighted_blending/three_models/kgo.nc
 d4b410e3c9d967c095659f5b0bed89e1234b22d8eae8bcfd089a70f651254e81  ./weighted_blending/three_models/nc/20190101T0400Z-PT0001H00M-precip_rate.nc
 7c95af11ab8aecc7760c7eb4480ff79c12b3316a77d05e2e396f614bb6eedff6  ./weighted_blending/three_models/ukvx/20190101T0400Z-PT0002H00M-precip_rate.nc
 b99322a54e361f2192d39f106f9b63a0db9fb4fbe734d45e9c1942678431f844  ./weighted_blending/weights_from_dict/input_dict.json
-4f172ed07db6416cf49133f1348e603e26b490f8bbaa98990dc7267b1579ef49  ./weighted_blending/weights_from_dict/kgo.nc
+34534abbc00fc5fd27a7291faa3ceae52613c0f8f7b44ef3aa1611512c4a86d2  ./weighted_blending/weights_from_dict/kgo.nc
 124a429618807548ee8890a2e1d4275be0e1e7977d9cfc804e55892bb045d323  ./wet-bulb-temperature-integral/basic/input.nc
 e3bef1754689e3ed8760774bfd293f0fee8ee647c76b308cf7a5a75fac22369e  ./wet-bulb-temperature-integral/basic/kgo.nc
 10bd2c82425fafd8d7cc86fb5262d8d6359302f74b79072775fa15ea43a406d7  ./wet-bulb-temperature-integral/realizations/input.nc

improver_tests/blending/calculate_weights_and_blend/test_WeightAndBlend.py

@@ -388,9 +388,17 @@ def test_cycle_blend_linear(self):
             [self.ukv_cube, self.ukv_cube_latest], cycletime=self.cycletime,
         )
         self.assertArrayAlmostEqual(result.data, expected_data)
-        self.assertEqual(result.coord("time"), self.ukv_cube_latest.coord("time"))
-        self.assertEqual(result.coord("forecast_reference_time"), self.expected_frt)
-        self.assertEqual(result.coord("forecast_period"), self.expected_fp)
+        self.assertArrayEqual(
+            result.coord("time").points, self.ukv_cube_latest.coord("time").points
+        )
+        self.assertArrayEqual(
+            result.coord("forecast_reference_time").points, self.expected_frt.points
+        )
+        self.assertArrayEqual(
+            result.coord("forecast_period").points, self.expected_fp.points
+        )
+        for coord in ["forecast_reference_time", "forecast_period"]:
+            self.assertIn("deprecation_message", result.coord(coord).attributes)
 
     @ManageWarnings(ignored_messages=["Collapsing a non-contiguous coordinate"])
     def test_model_blend(self):
@@ -409,6 +417,11 @@ def test_model_blend(self):
         result_coords = [coord.name() for coord in result.coords()]
         self.assertNotIn("model_id", result_coords)
         self.assertNotIn("model_configuration", result_coords)
+        for coord in ["forecast_reference_time", "forecast_period"]:
+            self.assertIn("deprecation_message", result.coord(coord).attributes)
+            self.assertIn(
+                "will be removed", result.coord(coord).attributes["deprecation_message"]
+            )
 
     @ManageWarnings(
         ignored_messages=[
@@ -460,7 +473,26 @@ def test_blend_three_models(self):
         # make sure output cube has the forecast reference time and period
         # from the most recent contributing model
         for coord in ["time", "forecast_period", "forecast_reference_time"]:
-            self.assertEqual(result.coord(coord), self.nowcast_cube.coord(coord))
+            self.assertArrayEqual(
+                result.coord(coord).points, self.nowcast_cube.coord(coord).points
+            )
+
+    def test_forecast_coord_deprecation(self):
+        """Test model blending works if some (but not all) inputs have previously
+        been cycle blended"""
+        for cube in [self.ukv_cube, self.enukx_cube]:
+            for coord in ["forecast_period", "forecast_reference_time"]:
+                cube.coord(coord).attributes.update({"deprecation_message": "blah"})
+        result = self.plugin_model.process(
+            [self.ukv_cube, self.enukx_cube, self.nowcast_cube],
+            model_id_attr="mosg__model_configuration",
+            cycletime=self.cycletime,
+        )
+        for coord in ["forecast_reference_time", "forecast_period"]:
+            self.assertIn("deprecation_message", result.coord(coord).attributes)
+            self.assertIn(
+                "will be removed", result.coord(coord).attributes["deprecation_message"]
+            )
 
     def test_one_cube(self):
         """Test the plugin returns a single input cube with updated attributes