Support netCDF load+save on dataset-like objects as well as filepaths.

docs/src/whatsnew/latest.rst

@@ -72,7 +72,10 @@ This document explains the changes made to Iris for this release
 💼 Internal
 ===========
 
-#. N/A
+#. `@pp-mo`_ supported loading and saving netcdf :class:`netCDF4.Dataset` compatible
+   objects in place of file-paths, as hooks for a forthcoming
+   `"Xarray bridge" <https://github.com/SciTools/iris/issues/4994>`_ facility.
+   (:pull:`5214`)
 
 
 .. comment

lib/iris/__init__.py

@@ -89,12 +89,12 @@ def callback(cube, field, filename):
 
 """
 
+from collections.abc import Iterable
 import contextlib
 import glob
 import importlib
 import itertools
 import os.path
-import pathlib
 import threading
 
 import iris._constraints
@@ -256,7 +256,8 @@ def context(self, **kwargs):
 
 def _generate_cubes(uris, callback, constraints):
     """Returns a generator of cubes given the URIs and a callback."""
-    if isinstance(uris, (str, pathlib.PurePath)):
+    if isinstance(uris, str) or not isinstance(uris, Iterable):
+        # Make a string, or other single item, into an iterable.
         uris = [uris]
 
     # Group collections of uris by their iris handler
@@ -273,6 +274,10 @@ def _generate_cubes(uris, callback, constraints):
             urls = [":".join(x) for x in groups]
             for cube in iris.io.load_http(urls, callback):
                 yield cube
+        elif scheme == "data":
+            data_objects = [x[1] for x in groups]
+            for cube in iris.io.load_data_objects(data_objects, callback):
+                yield cube
         else:
             raise ValueError("Iris cannot handle the URI scheme: %s" % scheme)
 

lib/iris/fileformats/__init__.py

@@ -9,6 +9,7 @@
 """
 
 from iris.io.format_picker import (
+    DataSourceObjectProtocol,
     FileExtension,
     FormatAgent,
     FormatSpecification,
@@ -125,16 +126,34 @@ def _load_grib(*args, **kwargs):
 )
 
 
-_nc_dap = FormatSpecification(
-    "NetCDF OPeNDAP",
-    UriProtocol(),
-    lambda protocol: protocol in ["http", "https"],
-    netcdf.load_cubes,
-    priority=6,
-    constraint_aware_handler=True,
+FORMAT_AGENT.add_spec(
+    FormatSpecification(
+        "NetCDF OPeNDAP",
+        UriProtocol(),
+        lambda protocol: protocol in ["http", "https"],
+        netcdf.load_cubes,
+        priority=6,
+        constraint_aware_handler=True,
+    )
+)
+
+# NetCDF file presented as an open, readable netCDF4 dataset (or mimic).
+FORMAT_AGENT.add_spec(
+    FormatSpecification(
+        "NetCDF dataset",
+        DataSourceObjectProtocol(),
+        lambda object: all(
+            hasattr(object, x)
+            for x in ("variables", "dimensions", "groups", "ncattrs")
+        ),
+        # Note: this uses the same call as the above "NetCDF_v4" (and "NetCDF OPeNDAP")
+        # The handler itself needs to detect what is passed + handle it appropriately.
+        netcdf.load_cubes,
+        priority=4,
+        constraint_aware_handler=True,
+    )
 )
-FORMAT_AGENT.add_spec(_nc_dap)
-del _nc_dap
+
 
 #
 # UM Fieldsfiles.

lib/iris/fileformats/cf.py

@@ -1043,17 +1043,25 @@ class CFReader:
     # TODO: remove once iris.experimental.ugrid.CFUGridReader is folded in.
     CFGroup = CFGroup
 
-    def __init__(self, filename, warn=False, monotonic=False):
-        self._dataset = None
-        self._filename = os.path.expanduser(filename)
+    def __init__(self, file_source, warn=False, monotonic=False):
+        # Ensure safe operation for destructor, should init fail.
+        self._own_file = False
+        if isinstance(file_source, str):
+            # Create from filepath : open it + own it (=close when we die).
+            self._filename = os.path.expanduser(file_source)
+            self._dataset = _thread_safe_nc.DatasetWrapper(
+                self._filename, mode="r"
+            )
+            self._own_file = True
+        else:
+            # We have been passed an open dataset.
+            # We use it but don't own it (don't close it).
+            self._dataset = file_source
+            self._filename = self._dataset.filepath()
 
         #: Collection of CF-netCDF variables associated with this netCDF file
         self.cf_group = self.CFGroup()
 
-        self._dataset = _thread_safe_nc.DatasetWrapper(
-            self._filename, mode="r"
-        )
-
         # Issue load optimisation warning.
         if warn and self._dataset.file_format in [
             "NETCDF3_CLASSIC",
@@ -1311,7 +1319,7 @@ def _reset(self):
 
     def _close(self):
         # Explicitly close dataset to prevent file remaining open.
-        if self._dataset is not None:
+        if self._own_file and self._dataset is not None:
             self._dataset.close()
             self._dataset = None
 

lib/iris/fileformats/netcdf/_thread_safe_nc.py

@@ -35,22 +35,36 @@ class _ThreadSafeWrapper(ABC):
     the C-layer.
     """
 
+    # Note: this is only used to create a "contained" from passed args.
     CONTAINED_CLASS = NotImplemented
+    # Note: this defines how we identify/check that a contained is of the expected type
+    # (in a duck-type way).
+    _DUCKTYPE_CHECK_PROPERTIES: typing.List[str] = [NotImplemented]
 
     # Allows easy type checking, avoiding difficulties with isinstance and mocking.
     THREAD_SAFE_FLAG = True
 
     @classmethod
-    def _from_existing(cls, instance):
+    def is_contained_type(cls, instance):
+        return all(
+            hasattr(instance, attr) for attr in cls._DUCKTYPE_CHECK_PROPERTIES
+        )
+
+    @classmethod
+    def from_existing(cls, instance):
         """Pass an existing instance to __init__, where it is contained."""
-        assert isinstance(instance, cls.CONTAINED_CLASS)
+        assert cls.is_contained_type(instance)
         return cls(instance)
 
     def __init__(self, *args, **kwargs):
         """Contain an existing instance, or generate a new one from arguments."""
-        if isinstance(args[0], self.CONTAINED_CLASS):
+        if len(args) == 1 and self.is_contained_type(args[0]):
+            # Passed a contained-type object : Wrap ourself around that.
             instance = args[0]
+            # We should never find ourselves "wrapping a wrapper".
+            assert not hasattr(instance, "THREAD_SAFE_FLAG")
         else:
+            # Create a contained object of the intended type from passed args.
             with _GLOBAL_NETCDF4_LOCK:
                 instance = self.CONTAINED_CLASS(*args, **kwargs)
 
@@ -89,6 +103,7 @@ class DimensionWrapper(_ThreadSafeWrapper):
     """
 
     CONTAINED_CLASS = netCDF4.Dimension
+    _DUCKTYPE_CHECK_PROPERTIES = ["isunlimited"]
 
 
 class VariableWrapper(_ThreadSafeWrapper):
@@ -99,6 +114,7 @@ class VariableWrapper(_ThreadSafeWrapper):
     """
 
     CONTAINED_CLASS = netCDF4.Variable
+    _DUCKTYPE_CHECK_PROPERTIES = ["dimensions", "dtype"]
 
     def setncattr(self, *args, **kwargs) -> None:
         """
@@ -136,7 +152,7 @@ def get_dims(self, *args, **kwargs) -> typing.Tuple[DimensionWrapper]:
             dimensions_ = list(
                 self._contained_instance.get_dims(*args, **kwargs)
             )
-        return tuple([DimensionWrapper._from_existing(d) for d in dimensions_])
+        return tuple([DimensionWrapper.from_existing(d) for d in dimensions_])
 
 
 class GroupWrapper(_ThreadSafeWrapper):
@@ -147,6 +163,8 @@ class GroupWrapper(_ThreadSafeWrapper):
     """
 
     CONTAINED_CLASS = netCDF4.Group
+    # Note: will also accept a whole Dataset object, but that is OK.
+    _DUCKTYPE_CHECK_PROPERTIES = ["createVariable"]
 
     # All Group API that returns Dimension(s) is wrapped to instead return
     #  DimensionWrapper(s).
@@ -163,7 +181,7 @@ def dimensions(self) -> typing.Dict[str, DimensionWrapper]:
         with _GLOBAL_NETCDF4_LOCK:
             dimensions_ = self._contained_instance.dimensions
         return {
-            k: DimensionWrapper._from_existing(v)
+            k: DimensionWrapper.from_existing(v)
             for k, v in dimensions_.items()
         }
 
@@ -179,7 +197,7 @@ def createDimension(self, *args, **kwargs) -> DimensionWrapper:
             new_dimension = self._contained_instance.createDimension(
                 *args, **kwargs
             )
-        return DimensionWrapper._from_existing(new_dimension)
+        return DimensionWrapper.from_existing(new_dimension)
 
     # All Group API that returns Variable(s) is wrapped to instead return
     #  VariableWrapper(s).
@@ -196,7 +214,7 @@ def variables(self) -> typing.Dict[str, VariableWrapper]:
         with _GLOBAL_NETCDF4_LOCK:
             variables_ = self._contained_instance.variables
         return {
-            k: VariableWrapper._from_existing(v) for k, v in variables_.items()
+            k: VariableWrapper.from_existing(v) for k, v in variables_.items()
         }
 
     def createVariable(self, *args, **kwargs) -> VariableWrapper:
@@ -211,7 +229,7 @@ def createVariable(self, *args, **kwargs) -> VariableWrapper:
             new_variable = self._contained_instance.createVariable(
                 *args, **kwargs
             )
-        return VariableWrapper._from_existing(new_variable)
+        return VariableWrapper.from_existing(new_variable)
 
     def get_variables_by_attributes(
         self, *args, **kwargs
@@ -229,7 +247,7 @@ def get_variables_by_attributes(
                     *args, **kwargs
                 )
             )
-        return [VariableWrapper._from_existing(v) for v in variables_]
+        return [VariableWrapper.from_existing(v) for v in variables_]
 
     # All Group API that returns Group(s) is wrapped to instead return
     #  GroupWrapper(s).
@@ -245,7 +263,7 @@ def groups(self):
         """
         with _GLOBAL_NETCDF4_LOCK:
             groups_ = self._contained_instance.groups
-        return {k: GroupWrapper._from_existing(v) for k, v in groups_.items()}
+        return {k: GroupWrapper.from_existing(v) for k, v in groups_.items()}
 
     @property
     def parent(self):
@@ -258,7 +276,7 @@ def parent(self):
         """
         with _GLOBAL_NETCDF4_LOCK:
             parent_ = self._contained_instance.parent
-        return GroupWrapper._from_existing(parent_)
+        return GroupWrapper.from_existing(parent_)
 
     def createGroup(self, *args, **kwargs):
         """
@@ -270,7 +288,7 @@ def createGroup(self, *args, **kwargs):
         """
         with _GLOBAL_NETCDF4_LOCK:
             new_group = self._contained_instance.createGroup(*args, **kwargs)
-        return GroupWrapper._from_existing(new_group)
+        return GroupWrapper.from_existing(new_group)
 
 
 class DatasetWrapper(GroupWrapper):
@@ -281,6 +299,8 @@ class DatasetWrapper(GroupWrapper):
     """
 
     CONTAINED_CLASS = netCDF4.Dataset
+    # Note: 'close' exists on Dataset but not Group (though a rather weak distinction).
+    _DUCKTYPE_CHECK_PROPERTIES = ["createVariable", "close"]
 
     @classmethod
     def fromcdl(cls, *args, **kwargs):
@@ -293,7 +313,7 @@ def fromcdl(cls, *args, **kwargs):
         """
         with _GLOBAL_NETCDF4_LOCK:
             instance = cls.CONTAINED_CLASS.fromcdl(*args, **kwargs)
-        return cls._from_existing(instance)
+        return cls.from_existing(instance)
 
 
 class NetCDFDataProxy:

lib/iris/fileformats/netcdf/loader.py

@@ -13,6 +13,7 @@
 Also : `CF Conventions <https://cfconventions.org/>`_.
 
 """
+from collections.abc import Iterable
 import warnings
 
 import numpy as np
@@ -483,14 +484,15 @@ def inner(cf_datavar):
     return result
 
 
-def load_cubes(filenames, callback=None, constraints=None):
+def load_cubes(file_sources, callback=None, constraints=None):
     """
     Loads cubes from a list of NetCDF filenames/OPeNDAP URLs.
 
     Args:
 
-    * filenames (string/list):
+    * file_sources (string/list):
         One or more NetCDF filenames/OPeNDAP URLs to load from.
+        OR open datasets.
 
     Kwargs:
 
@@ -518,18 +520,18 @@ def load_cubes(filenames, callback=None, constraints=None):
     # Create an actions engine.
     engine = _actions_engine()
 
-    if isinstance(filenames, str):
-        filenames = [filenames]
+    if isinstance(file_sources, str) or not isinstance(file_sources, Iterable):
+        file_sources = [file_sources]
 
-    for filename in filenames:
-        # Ingest the netCDF file.
+    for file_source in file_sources:
+        # Ingest the file.  At present may be a filepath or an open netCDF4.Dataset.
         meshes = {}
         if PARSE_UGRID_ON_LOAD:
             cf_reader_class = CFUGridReader
         else:
             cf_reader_class = iris.fileformats.cf.CFReader
 
-        with cf_reader_class(filename) as cf:
+        with cf_reader_class(file_source) as cf:
             if PARSE_UGRID_ON_LOAD:
                 meshes = _meshes_from_cf(cf)
 
@@ -563,7 +565,7 @@ def load_cubes(filenames, callback=None, constraints=None):
                 if mesh is not None:
                     mesh_coords, mesh_dim = _build_mesh_coords(mesh, cf_var)
 
-                cube = _load_cube(engine, cf, cf_var, filename)
+                cube = _load_cube(engine, cf, cf_var, cf.filename)
 
                 # Attach the mesh (if present) to the cube.
                 for mesh_coord in mesh_coords:
@@ -577,7 +579,7 @@ def load_cubes(filenames, callback=None, constraints=None):
                     warnings.warn("{}".format(e))
 
                 # Perform any user registered callback function.
-                cube = run_callback(callback, cube, cf_var, filename)
+                cube = run_callback(callback, cube, cf_var, file_source)
 
                 # Callback mechanism may return None, which must not be yielded
                 if cube is None: