Merge branch 'master' into drop_duplicates

Author: max-sixty

.github/dependabot.yml

@@ -0,0 +1,7 @@
+version: 2
+updates:
+  - package-ecosystem: 'github-actions'
+    directory: '/'
+    schedule:
+      # Check for updates once a week
+      interval: 'weekly'

.github/workflows/cancel-duplicate-runs.yaml

@@ -9,6 +9,6 @@ jobs:
     name: Cancel previous runs
     runs-on: ubuntu-latest
     steps:
-    - uses: styfle/cancel-workflow-action@0.8.0
+    - uses: styfle/cancel-workflow-action@0.9.0
       with:
         workflow_id: ${{ github.event.workflow.id }}

.github/workflows/ci-additional.yaml

@@ -181,7 +181,7 @@ jobs:
           python xarray/util/print_versions.py
       - name: Run mypy
         run: |
-          python -m mypy xarray
+          python -m mypy .
 
   min-version-policy:
     name: Minimum Version Policy

.github/workflows/ci-pre-commit.yml

@@ -13,4 +13,4 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
-      - uses: pre-commit/action@v2.0.0
+      - uses: pre-commit/action@v2.0.2

.pre-commit-config.yaml

@@ -34,6 +34,7 @@ repos:
     rev: v0.812
     hooks:
       - id: mypy
+        # Copied from setup.cfg
         exclude: "properties|asv_bench"
   # run this occasionally, ref discussion https://github.com/pydata/xarray/pull/3194
   # - repo: https://github.com/asottile/pyupgrade

ci/requirements/mypy_only

@@ -1,3 +1,4 @@
 # used for the "Type checking (mypy)" CI run
 # version must correspond to the one in .pre-commit-config.yaml
+# See https://github.com/pydata/xarray/issues/4881 for more details.
 mypy=0.812

doc/howdoi.rst

@@ -23,6 +23,8 @@ How do I ...
      - :py:meth:`Dataset.set_coords`
    * - change the order of dimensions
      - :py:meth:`DataArray.transpose`, :py:meth:`Dataset.transpose`
+   * - reshape dimensions
+     - :py:meth:`DataArray.stack`, :py:meth:`Dataset.stack`
    * - remove a variable from my object
      - :py:meth:`Dataset.drop_vars`, :py:meth:`DataArray.drop_vars`
    * - remove dimensions of length 1 or 0
@@ -34,7 +36,9 @@ How do I ...
    * - rename a variable, dimension or coordinate
      - :py:meth:`Dataset.rename`, :py:meth:`DataArray.rename`, :py:meth:`Dataset.rename_vars`, :py:meth:`Dataset.rename_dims`,
    * - convert a DataArray to Dataset or vice versa
-     - :py:meth:`DataArray.to_dataset`, :py:meth:`Dataset.to_array`
+     - :py:meth:`DataArray.to_dataset`, :py:meth:`Dataset.to_array`, :py:meth:`Dataset.to_stacked_array`, :py:meth:`DataArray.to_unstacked_dataset`
+   * - extract variables that have certain attributes
+     - :py:meth:`Dataset.filter_by_attrs`
    * - extract the underlying array (e.g. numpy or Dask arrays)
      - :py:attr:`DataArray.data`
    * - convert to and extract the underlying numpy array
@@ -43,6 +47,8 @@ How do I ...
      - :py:func:`dask.is_dask_collection`
    * - know how much memory my object requires
      - :py:attr:`DataArray.nbytes`, :py:attr:`Dataset.nbytes`
+   * - Get axis number for a dimension
+     - :py:meth:`DataArray.get_axis_num`
    * - convert a possibly irregularly sampled timeseries to a regularly sampled timeseries
      - :py:meth:`DataArray.resample`, :py:meth:`Dataset.resample` (see :ref:`resampling` for more)
    * - apply a function on all data variables in a Dataset
@@ -51,6 +57,8 @@ How do I ...
      - :py:func:`Dataset.to_netcdf`, :py:func:`DataArray.to_netcdf` specifying ``engine="h5netcdf", invalid_netcdf=True``
    * - make xarray objects look like other xarray objects
      - :py:func:`~xarray.ones_like`, :py:func:`~xarray.zeros_like`, :py:func:`~xarray.full_like`, :py:meth:`Dataset.reindex_like`, :py:meth:`Dataset.interp_like`, :py:meth:`Dataset.broadcast_like`, :py:meth:`DataArray.reindex_like`, :py:meth:`DataArray.interp_like`, :py:meth:`DataArray.broadcast_like`
+   * - Make sure my datasets have values at the same coordinate locations
+     - ``xr.align(dataset_1, dataset_2, join="exact")``
    * - replace NaNs with other values
      - :py:meth:`Dataset.fillna`, :py:meth:`Dataset.ffill`, :py:meth:`Dataset.bfill`, :py:meth:`Dataset.interpolate_na`, :py:meth:`DataArray.fillna`, :py:meth:`DataArray.ffill`, :py:meth:`DataArray.bfill`, :py:meth:`DataArray.interpolate_na`
    * - extract the year, month, day or similar from a DataArray of time values
@@ -59,3 +67,7 @@ How do I ...
      - ``obj.dt.ceil``, ``obj.dt.floor``, ``obj.dt.round``. See :ref:`dt_accessor` for more.
    * - make a mask that is ``True`` where an object contains any of the values in a array
      - :py:meth:`Dataset.isin`, :py:meth:`DataArray.isin`
+   * - Index using a boolean mask
+     - :py:meth:`Dataset.query`, :py:meth:`DataArray.query`, :py:meth:`Dataset.where`, :py:meth:`DataArray.where`
+   * - preserve ``attrs`` during (most) xarray operations
+     - ``xr.set_options(keep_attrs=True)``

doc/internals/how-to-add-new-backend.rst

@@ -32,16 +32,19 @@ This is what a ``BackendEntrypoint`` subclass should look like:
 
 .. code-block:: python
 
+    from xarray.backends import BackendEntrypoint
+
+
     class MyBackendEntrypoint(BackendEntrypoint):
         def open_dataset(
             self,
             filename_or_obj,
             *,
             drop_variables=None,
             # other backend specific keyword arguments
+            # `chunks` and `cache` DO NOT go here, they are handled by xarray
         ):
-            ...
-            return ds
+            return my_open_dataset(filename_or_obj, drop_variables=drop_variables)
 
         open_dataset_parameters = ["filename_or_obj", "drop_variables"]
 
@@ -50,7 +53,7 @@ This is what a ``BackendEntrypoint`` subclass should look like:
                 _, ext = os.path.splitext(filename_or_obj)
             except TypeError:
                 return False
-            return ext in {...}
+            return ext in {".my_format", ".my_fmt"}
 
 ``BackendEntrypoint`` subclass methods and attributes are detailed in the following.
 
@@ -74,20 +77,19 @@ The following is an example of the high level processing steps:
         decode_times=True,
         decode_timedelta=True,
         decode_coords=True,
-        my_backend_param=None,
+        my_backend_option=None,
     ):
         vars, attrs, coords = my_reader(
             filename_or_obj,
             drop_variables=drop_variables,
-            my_backend_param=my_backend_param,
+            my_backend_option=my_backend_option,
         )
         vars, attrs, coords = my_decode_variables(
             vars, attrs, decode_times, decode_timedelta, decode_coords
         )  #  see also conventions.decode_cf_variables
 
-        ds = xr.Dataset(vars, attrs=attrs)
-        ds = ds.set_coords(coords)
-        ds.set_close(store.close)
+        ds = xr.Dataset(vars, attrs=attrs, coords=coords)
+        ds.set_close(my_close_method)
 
         return ds
 
@@ -98,9 +100,9 @@ method shall be set by using :py:meth:`~xarray.Dataset.set_close`.
 
 
 The input of ``open_dataset`` method are one argument
-(``filename``) and one keyword argument (``drop_variables``):
+(``filename_or_obj``) and one keyword argument (``drop_variables``):
 
-- ``filename``: can be a string containing a path or an instance of
+- ``filename_or_obj``: can be any object but usually it is a string containing a path or an instance of
   :py:class:`pathlib.Path`.
 - ``drop_variables``: can be `None` or an iterable containing the variable
   names to be dropped when reading the data.
@@ -117,7 +119,7 @@ should implement in its interface the following boolean keyword arguments, calle
 - ``decode_coords``
 
 Note: all the supported decoders shall be declared explicitly
-in backend ``open_dataset`` signature.
+in backend ``open_dataset`` signature and adding a ``**kargs`` is not allowed.
 
 These keyword arguments are explicitly defined in Xarray
 :py:func:`~xarray.open_dataset` signature. Xarray will pass them to the
@@ -241,7 +243,7 @@ How to register a backend
 
 Define a new entrypoint in your ``setup.py`` (or ``setup.cfg``) with:
 
-- group: ``xarray.backend``
+- group: ``xarray.backends``
 - name: the name to be passed to :py:meth:`~xarray.open_dataset`  as ``engine``
 - object reference: the reference of the class that you have implemented.
 
@@ -251,9 +253,7 @@ You can declare the entrypoint in ``setup.py`` using the following syntax:
 
     setuptools.setup(
         entry_points={
-            "xarray.backends": [
-                "engine_name=your_package.your_module:YourBackendEntryClass"
-            ],
+            "xarray.backends": ["my_engine=my_package.my_module:MyBackendEntryClass"],
         },
     )
 
@@ -263,18 +263,18 @@ in ``setup.cfg``:
 
     [options.entry_points]
     xarray.backends =
-        engine_name = your_package.your_module:YourBackendEntryClass
+        my_engine = my_package.my_module:MyBackendEntryClass
 
 
 See https://packaging.python.org/specifications/entry-points/#data-model
 for more information
 
-If you are using [Poetry](https://python-poetry.org/) for your build system, you can accomplish the same thing using "plugins". In this case you would need to add the following to your ``pyproject.toml`` file:
+If you are using `Poetry <https://python-poetry.org/>`_ for your build system, you can accomplish the same thing using "plugins". In this case you would need to add the following to your ``pyproject.toml`` file:
 
 .. code-block:: toml
 
     [tool.poetry.plugins."xarray_backends"]
-    "engine_name" = "your_package.your_module:YourBackendEntryClass"
+    "my_engine" = "my_package.my_module:MyBackendEntryClass"
 
 See https://python-poetry.org/docs/pyproject/#plugins for more information on Poetry plugins.
 
@@ -328,6 +328,9 @@ This is an example ``BackendArray`` subclass implementation:
 
 .. code-block:: python
 
+    from xarray.backends import BackendArray
+
+
     class MyBackendArray(BackendArray):
         def __init__(
             self,

doc/user-guide/data-structures.rst

@@ -239,7 +239,7 @@ to access any variable in a dataset, datasets have four key properties:
   used in ``data_vars`` (e.g., arrays of numbers, datetime objects or strings)
 - ``attrs``: :py:class:`dict` to hold arbitrary metadata
 
-The distinction between whether a variables falls in data or coordinates
+The distinction between whether a variable falls in data or coordinates
 (borrowed from `CF conventions`_) is mostly semantic, and you can probably get
 away with ignoring it if you like: dictionary like access on a dataset will
 supply variables found in either category. However, xarray does make use of the

doc/whats-new.rst

@@ -67,6 +67,11 @@ New Features
 - Implement :py:meth:`Dataset.drop_duplicate_coords` and :py:meth:`DataArray.drop_duplicate_coords`
   to remove duplicate coordinate values (:pull:`5089`).
   By `Andrew Huang <https://github.com/ahuang11>`_.
+- Add typing information to unary and binary arithmetic operators operating on
+  :py:class:`~core.dataset.Dataset`, :py:class:`~core.dataarray.DataArray`,
+  :py:class:`~core.variable.Variable`, :py:class:`~core.groupby.DatasetGroupBy` or
+  :py:class:`~core.groupby.DataArrayGroupBy` (:pull:`4904`).
+  By `Richard Kleijn <https://github.com/rhkleijn>`_ .
 - Add a ``combine_attrs`` parameter to :py:func:`open_mfdataset` (:pull:`4971`).
   By `Justus Magin <https://github.com/keewis>`_.
 - Disable the `cfgrib` backend if the `eccodes` library is not installed (:pull:`5083`). By `Baudouin Raoult <https://github.com/b8raoult>`_.

setup.cfg

@@ -162,6 +162,8 @@ default_section = THIRDPARTY
 known_first_party = xarray
 
 [mypy]
+exclude = properties|asv_bench|doc
+files = xarray/**/*.py
 show_error_codes = True
 
 # Most of the numerical computing stack doesn't have type annotations yet.
@@ -238,7 +240,6 @@ ignore_missing_imports = True
 [mypy-xarray.core.pycompat]
 ignore_errors = True
 
-
 [aliases]
 test = pytest
 

xarray/backends/__init__.py

@@ -4,7 +4,7 @@
 formats. They should not be used directly, but rather through Dataset objects.
 """
 from .cfgrib_ import CfGribDataStore
-from .common import AbstractDataStore
+from .common import AbstractDataStore, BackendArray, BackendEntrypoint
 from .file_manager import CachingFileManager, DummyFileManager, FileManager
 from .h5netcdf_ import H5NetCDFStore
 from .memory import InMemoryDataStore
@@ -18,6 +18,8 @@
 
 __all__ = [
     "AbstractDataStore",
+    "BackendArray",
+    "BackendEntrypoint",
     "FileManager",
     "CachingFileManager",
     "CfGribDataStore",

xarray/backends/api.py

@@ -28,7 +28,7 @@
 from ..core.dataset import Dataset, _get_chunk, _maybe_chunk
 from ..core.utils import is_remote_uri
 from . import plugins
-from .common import AbstractDataStore, ArrayWriter
+from .common import AbstractDataStore, ArrayWriter, _normalize_path
 from .locks import _get_scheduler
 
 if TYPE_CHECKING:
@@ -109,16 +109,6 @@ def _get_default_engine(path: str, allow_remote: bool = False):
     return engine
 
 
-def _normalize_path(path):
-    if isinstance(path, Path):
-        path = str(path)
-
-    if isinstance(path, str) and not is_remote_uri(path):
-        path = os.path.abspath(os.path.expanduser(path))
-
-    return path
-
-
 def _validate_dataset_names(dataset):
     """DataArray.name and Dataset keys must be a string or None"""
 
@@ -375,10 +365,11 @@ def open_dataset(
         scipy.io.netcdf (only netCDF3 supported). Byte-strings or file-like
         objects are opened by scipy.io.netcdf (netCDF3) or h5py (netCDF4/HDF).
     engine : {"netcdf4", "scipy", "pydap", "h5netcdf", "pynio", "cfgrib", \
-        "pseudonetcdf", "zarr"}, optional
+        "pseudonetcdf", "zarr"} or subclass of xarray.backends.BackendEntrypoint, optional
         Engine to use when reading files. If not provided, the default engine
         is chosen based on available dependencies, with a preference for
-        "netcdf4".
+        "netcdf4". A custom backend class (a subclass of ``BackendEntrypoint``)
+        can also be used.
     chunks : int or dict, optional
         If chunks is provided, it is used to load the new dataset into dask
         arrays. ``chunks=-1`` loads the dataset with dask using a single

xarray/backends/cfgrib_.py

@@ -11,6 +11,7 @@
     AbstractDataStore,
     BackendArray,
     BackendEntrypoint,
+    _normalize_path,
 )
 from .locks import SerializableLock, ensure_lock
 from .store import StoreBackendEntrypoint
@@ -22,9 +23,10 @@
 except ModuleNotFoundError:
     has_cfgrib = False
 # cfgrib throws a RuntimeError if eccodes is not installed
-except RuntimeError:
+except (ImportError, RuntimeError):
     warnings.warn(
-        "Failed to load cfgrib - most likely eccodes is missing. Try `import cfgrib` to get the error message"
+        "Failed to load cfgrib - most likely there is a problem accessing the ecCodes library. "
+        "Try `import cfgrib` to get the full error message"
     )
     has_cfgrib = False
 
@@ -120,6 +122,7 @@ def open_dataset(
         time_dims=("time", "step"),
     ):
 
+        filename_or_obj = _normalize_path(filename_or_obj)
         store = CfGribDataStore(
             filename_or_obj,
             indexpath=indexpath,

xarray/backends/common.py

@@ -1,14 +1,16 @@
 import logging
+import os.path
 import time
 import traceback
+from pathlib import Path
 from typing import Any, Dict, Tuple, Type, Union
 
 import numpy as np
 
 from ..conventions import cf_encoder
 from ..core import indexing
 from ..core.pycompat import is_duck_dask_array
-from ..core.utils import FrozenDict, NdimSizeLenMixin
+from ..core.utils import FrozenDict, NdimSizeLenMixin, is_remote_uri
 
 # Create a logger object, but don't add any handlers. Leave that to user code.
 logger = logging.getLogger(__name__)
@@ -17,6 +19,16 @@
 NONE_VAR_NAME = "__values__"
 
 
+def _normalize_path(path):
+    if isinstance(path, Path):
+        path = str(path)
+
+    if isinstance(path, str) and not is_remote_uri(path):
+        path = os.path.abspath(os.path.expanduser(path))
+
+    return path
+
+
 def _encode_variable_name(name):
     if name is None:
         name = NONE_VAR_NAME

xarray/backends/h5netcdf_.py

@@ -12,6 +12,7 @@
     BACKEND_ENTRYPOINTS,
     BackendEntrypoint,
     WritableCFDataStore,
+    _normalize_path,
     find_root_and_group,
 )
 from .file_manager import CachingFileManager, DummyFileManager
@@ -366,6 +367,7 @@ def open_dataset(
         decode_vlen_strings=True,
     ):
 
+        filename_or_obj = _normalize_path(filename_or_obj)
         store = H5NetCDFStore.open(
             filename_or_obj,
             format=format,