Make is_iterable changes applicable to percentiles option (#2090)

* MOBT-797: Coordinate retention in percentile generation (#2087)

* Enable retention of scalar coordinates that are produced during percentile generation from coordinate collapse. This change also ensures that any scalar time coordinates that are created in this way adhere to the improver metadata standards.

* style fixes.

* Setting percentiles with as_iterable

---------

Co-authored-by: bayliffe <benjamin.ayliffe@metoffice.gov.uk>

improver/cli/generate_percentiles.py

@@ -15,6 +15,7 @@ def process(
     *,
     coordinates: cli.comma_separated_list = None,
     percentiles: cli.comma_separated_list = None,
+    retained_coordinates: cli.comma_separated_list = None,
     ignore_ecc_bounds_exceedance: bool = False,
     skip_ecc_bounds: bool = False,
     mask_percentiles: bool = False,
@@ -48,6 +49,10 @@ def process(
             coordinate.
         percentiles (list):
             Optional definition of percentiles at which to calculate data.
+        retained_coordinates (list):
+            Optional list of collapsed coordinates that should be retained in
+            their new scalar form. The default behaviour is to remove the
+            scalar coordinates that result from coordinate collapse.
         ignore_ecc_bounds_exceedance (bool):
             If True, where calculated percentiles are outside the ECC bounds
             range, raises a warning rather than an exception.
@@ -145,6 +150,7 @@ def process(
         result = PercentileConverter(
             coordinates,
             percentiles=percentiles,
+            retained_coordinates=retained_coordinates,
             fast_percentile_method=fast_percentile_method,
         )(cube)
     else:

improver/nbhood/nbhood.py

@@ -16,7 +16,7 @@
 from improver.constants import DEFAULT_PERCENTILES
 from improver.metadata.forecast_times import forecast_period_coord
 from improver.nbhood import radius_by_lead_time
-from improver.utilities.common_input_handle import as_cube
+from improver.utilities.common_input_handle import as_cube, as_iterable
 from improver.utilities.complex_conversion import complex_to_deg, deg_to_complex
 from improver.utilities.cube_checker import (
     check_cube_coordinates,
@@ -472,7 +472,7 @@ def __init__(
         self,
         radii: Union[float, List[float]],
         lead_times: Optional[List] = None,
-        percentiles: List = DEFAULT_PERCENTILES,
+        percentiles: Union[float, List[float]] = DEFAULT_PERCENTILES,
     ) -> None:
         """
         Create a neighbourhood processing subclass that generates percentiles
@@ -495,7 +495,7 @@ def __init__(
                 DEFAULT_PERCENTILES.
         """
         super().__init__(radii, lead_times=lead_times)
-        self.percentiles = tuple(percentiles)
+        self.percentiles = tuple(as_iterable(percentiles))
 
     def pad_and_unpad_cube(self, slice_2d: Cube, kernel: ndarray) -> Cube:
         """
@@ -766,13 +766,13 @@ class MetaNeighbourhood(BasePlugin):
     def __init__(
         self,
         neighbourhood_output: str,
-        radii: List[float],
+        radii: Union[float, List[float]],
         lead_times: Optional[List[int]] = None,
         neighbourhood_shape: str = "square",
         degrees_as_complex: bool = False,
         weighted_mode: bool = False,
         area_sum: bool = False,
-        percentiles: List[float] = DEFAULT_PERCENTILES,
+        percentiles: Union[float, List[float]] = DEFAULT_PERCENTILES,
         halo_radius: Optional[float] = None,
     ) -> None:
         """

improver/percentile.py

@@ -13,7 +13,9 @@
 
 from improver import BasePlugin
 from improver.constants import DEFAULT_PERCENTILES
+from improver.metadata.constants.time_types import TIME_COORDS
 from improver.metadata.probabilistic import find_percentile_coordinate
+from improver.metadata.utilities import enforce_time_point_standard
 from improver.utilities.cube_manipulation import collapsed
 
 
@@ -28,6 +30,7 @@ def __init__(
         self,
         collapse_coord: Union[str, List[str]],
         percentiles: Optional[List[float]] = None,
+        retained_coordinates: Optional[Union[str, List[str]]] = None,
         fast_percentile_method: bool = True,
     ) -> None:
         """
@@ -41,6 +44,13 @@ def __init__(
             percentiles:
                 Percentile values at which to calculate; if not provided uses
                 DEFAULT_PERCENTILES. (optional)
+            retained_coordinates:
+                Optional list of collapsed coordinates that should be retained
+                in their new scalar form. The default behaviour is to remove
+                the scalar coordinates that result from coordinate collapse.
+            fast_percentile_method:
+                If True use the numpy percentile method within Iris, which is
+                much faster than scipy, but cannot handle masked data.
 
         Raises:
             TypeError: If collapse_coord is not a string.
@@ -65,6 +75,7 @@ def __init__(
         # percentile coordinate has a consistent name regardless of the order
         # in which the user provides the original coordinate names.
         self.collapse_coord = sorted(collapse_coord)
+        self.retained_coordinates = retained_coordinates
         self.fast_percentile_method = fast_percentile_method
 
     def __repr__(self) -> str:
@@ -114,8 +125,23 @@ def process(self, cube: Cube) -> Cube:
             )
 
             result.data = result.data.astype(data_type)
-            for coord in self.collapse_coord:
+
+            remove_crds = self.collapse_coord
+            if self.retained_coordinates is not None:
+                remove_crds = [
+                    crd
+                    for crd in self.collapse_coord
+                    if crd not in self.retained_coordinates
+                ]
+            for coord in remove_crds:
                 result.remove_coord(coord)
+
+            # If a time related coordinate has been collapsed we need to
+            # enforce the IMPROVER standard of a coordinate point that aligns
+            # with the upper bound of the period.
+            if any([crd in TIME_COORDS for crd in self.collapse_coord]):
+                enforce_time_point_standard(result)
+
             percentile_coord = find_percentile_coordinate(result)
             result.coord(percentile_coord).rename("percentile")
             result.coord(percentile_coord).units = "%"

improver_tests/percentile/test_PercentileConverter.py

@@ -52,6 +52,8 @@ def test_valid_single_coord_string(self):
         self.assertArrayAlmostEqual(
             result.data[:, 0, 0], self.default_percentiles * 0.1
         )
+        # Check collapsed coordinate removed
+        self.assertNotIn(collapse_coord, [crd.name() for crd in result.coords()])
         # Check coordinate name.
         self.assertEqual(result.coords()[0].name(), "percentile")
         # Check coordinate units.
@@ -87,6 +89,8 @@ def test_valid_single_coord_string_for_time(self):
         self.assertArrayAlmostEqual(
             result.data[:, 0, 0, 0], self.default_percentiles * 0.01
         )
+        # Check collapsed coordinate removed
+        self.assertNotIn(collapse_coord, [crd.name() for crd in result.coords()])
         # Check coordinate name.
         self.assertEqual(result.coords()[0].name(), "percentile")
         # Check coordinate units.
@@ -99,6 +103,85 @@ def test_valid_single_coord_string_for_time(self):
         # Check resulting data shape.
         self.assertEqual(result.data.shape, (15, 3, 11, 11))
 
+    def test_retain_time_coordinate(self):
+        """Test that the plugin handles time being the collapse_coord and that
+        coordinate being retained as a scalar coordinate on the resulting
+        cube. In this case the input cubes have no bounds, meaning the
+        constructed scalar time coordinate simply spans the input time
+        points."""
+        data = [[list(range(1, 12, 1))] * 11] * 3
+        data = np.array(data).astype(np.float32)
+        data.resize((3, 11, 11))
+        new_cube = set_up_variable_cube(
+            data,
+            time=datetime(2017, 11, 11, 4, 0),
+            frt=datetime(2017, 11, 11, 0, 0),
+            realizations=[0, 1, 2],
+        )
+        cube = iris.cube.CubeList([self.cube, new_cube]).merge_cube()
+        collapse_coord = "time"
+
+        plugin = PercentileConverter(collapse_coord, retained_coordinates="time")
+        result = plugin.process(cube)
+
+        # Check time coordinate has been retained.
+        self.assertTrue("time" in [crd.name() for crd in result.coords()])
+        # Check time and associated forecast_reference_time scalar coordinates
+        for crd in ["time", "forecast_reference_time"]:
+            self.assertEqual(result.coord(crd).points[0], cube.coord(crd).points[-1])
+            self.assertEqual(result.coord(crd).bounds[0][0], cube.coord(crd).points[0])
+            self.assertEqual(
+                result.coord(crd).bounds[0][-1], cube.coord(crd).points[-1]
+            )
+
+    def test_retain_time_coordinate_bounds(self):
+        """Test that the plugin handles time being the collapse_coord and that
+        coordinate being retained as a scalar coordinate on the resulting
+        cube. In this case the input cubes have bounds, meaning the
+        constructed scalar time coordinate should span the input time
+        bounds."""
+        data = [[list(range(1, 12, 1))] * 11] * 3
+        data = np.array(data).astype(np.float32)
+        data.resize((3, 11, 11))
+        new_cube = set_up_variable_cube(
+            data,
+            time=datetime(2017, 11, 11, 4, 0),
+            frt=datetime(2017, 11, 11, 0, 0),
+            time_bounds=[datetime(2017, 11, 11, 3, 0), datetime(2017, 11, 11, 4, 0)],
+            realizations=[0, 1, 2],
+        )
+        self.cube.coord("time").bounds = [1510282800, 1510286400]
+
+        cube = iris.cube.CubeList([self.cube, new_cube]).merge_cube()
+        collapse_coord = "time"
+
+        plugin = PercentileConverter(collapse_coord, retained_coordinates="time")
+        result = plugin.process(cube)
+
+        # Check time coordinate has been retained.
+        self.assertTrue("time" in [crd.name() for crd in result.coords()])
+        # Check time scalar coordinate
+        self.assertEqual(result.coord("time").points[0], cube.coord("time").points[-1])
+        self.assertEqual(
+            result.coord("time").bounds[0][0], cube.coord("time").bounds[0][0]
+        )
+        self.assertEqual(
+            result.coord("time").bounds[0][-1], cube.coord("time").bounds[-1][-1]
+        )
+        # Check forecast_reference_time scalar coordinate
+        self.assertEqual(
+            result.coord("forecast_reference_time").points[0],
+            cube.coord("forecast_reference_time").points[-1],
+        )
+        self.assertEqual(
+            result.coord("forecast_reference_time").bounds[0][0],
+            cube.coord("forecast_reference_time").points[0],
+        )
+        self.assertEqual(
+            result.coord("forecast_reference_time").bounds[0][-1],
+            cube.coord("forecast_reference_time").points[-1],
+        )
+
     def test_valid_multi_coord_string_list(self):
         """Test that the plugin handles a valid list of collapse_coords passed
         in as a list of strings."""
@@ -129,6 +212,9 @@ def test_valid_multi_coord_string_list(self):
                 10.0,
             ],
         )
+        # Check collapsed coordinate removed
+        for coord in collapse_coord:
+            self.assertNotIn(coord, [crd.name() for crd in result.coords()])
         # Check coordinate name.
         self.assertEqual(result.coords()[0].name(), "percentile")
         # Check coordinate units.
@@ -141,6 +227,20 @@ def test_valid_multi_coord_string_list(self):
         # Check resulting data shape.
         self.assertEqual(result.data.shape, (15, 3))
 
+    def test_retention_of_multiple_coords(self):
+        """Test that multiple coordinates that have been collapsed can be
+        retained as scalars using the retained_coordinates option."""
+
+        collapse_coord = ["longitude", "latitude"]
+
+        plugin = PercentileConverter(
+            collapse_coord, retained_coordinates=collapse_coord
+        )
+        result = plugin.process(self.cube)
+
+        for coord in collapse_coord:
+            self.assertTrue(coord in [crd.name() for crd in result.coords()])
+
     def test_single_percentile(self):
         """Test dimensions of output at median only"""
         collapse_coord = ["realization"]