Merge branch 'main' into fix_s3_start_byte

Author: weiji14

.github/workflows/ci-additional.yaml

@@ -134,6 +134,64 @@ jobs:
           name: codecov-umbrella
           fail_ci_if_error: false
 
+  mypy38:
+    name: Mypy 3.8
+    runs-on: "ubuntu-latest"
+    needs: detect-ci-trigger
+    # temporarily skipping due to https://github.com/pydata/xarray/issues/6551
+    if: needs.detect-ci-trigger.outputs.triggered == 'false'
+    defaults:
+      run:
+        shell: bash -l {0}
+    env:
+      CONDA_ENV_FILE: ci/requirements/environment.yml
+      PYTHON_VERSION: "3.8"
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # Fetch all history for all branches and tags.
+
+      - name: set environment variables
+        run: |
+          echo "TODAY=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
+      - name: Setup micromamba
+        uses: mamba-org/provision-with-micromamba@v14
+        with:
+          environment-file: ${{env.CONDA_ENV_FILE}}
+          environment-name: xarray-tests
+          extra-specs: |
+            python=${{env.PYTHON_VERSION}}
+            conda
+          cache-env: true
+          cache-env-key: "${{runner.os}}-${{runner.arch}}-py${{env.PYTHON_VERSION}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}"
+      - name: Install xarray
+        run: |
+          python -m pip install --no-deps -e .
+      - name: Version info
+        run: |
+          conda info -a
+          conda list
+          python xarray/util/print_versions.py
+      - name: Install mypy
+        run: |
+          python -m pip install 'mypy<0.990'
+
+      - name: Run mypy
+        run: |
+          python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report
+
+      - name: Upload mypy coverage to Codecov
+        uses: codecov/codecov-action@v3.1.1
+        with:
+          file: mypy_report/cobertura.xml
+          flags: mypy38
+          env_vars: PYTHON_VERSION
+          name: codecov-umbrella
+          fail_ci_if_error: false
+
+
+
   min-version-policy:
     name: Minimum Version Policy
     runs-on: "ubuntu-latest"

.github/workflows/upstream-dev-ci.yaml

@@ -80,6 +80,7 @@ jobs:
         if: success()
         id: status
         run: |
+          export ZARR_V3_EXPERIMENTAL_API=1
           python -m pytest --timeout=60 -rf \
             --report-log output-${{ matrix.python-version }}-log.jsonl
       - name: Generate and publish the report

.pre-commit-config.yaml

@@ -47,7 +47,7 @@ repos:
   #     - id: velin
   #       args: ["--write", "--compact"]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.990
+    rev: v0.991
     hooks:
       - id: mypy
         # Copied from setup.cfg

ci/requirements/all-but-dask.yml

@@ -30,7 +30,6 @@ dependencies:
   - pip
   - pseudonetcdf
   - pydap
-  # - pynio  # not compatible with netCDF4>1.5.3, see #4491
   - pytest
   - pytest-cov
   - pytest-env

ci/requirements/environment-windows.yml

@@ -30,7 +30,6 @@ dependencies:
   - pre-commit
   - pseudonetcdf
   - pydap
-  # - pynio  # Not available on Windows
   - pytest
   - pytest-cov
   - pytest-env

ci/requirements/environment.yml

@@ -34,7 +34,6 @@ dependencies:
   - pre-commit
   - pseudonetcdf
   - pydap
-  # - pynio  # not compatible with netCDF4>1.5.3, see #4491
   - pytest
   - pytest-cov
   - pytest-env

ci/requirements/min-all-deps.yml

@@ -40,7 +40,6 @@ dependencies:
   - pip
   - pseudonetcdf=3.2
   - pydap=3.2
-  # - pynio=1.5.5  # see: https://github.com/pydata/xarray/issues/4491
   - pytest
   - pytest-cov
   - pytest-env

doc/user-guide/io.rst

@@ -1209,6 +1209,10 @@ We recommend installing cfgrib via conda::
 Formats supported by PyNIO
 --------------------------
 
+.. warning::
+
+    The PyNIO backend is deprecated_. PyNIO is no longer maintained_. See
+
 Xarray can also read GRIB, HDF4 and other file formats supported by PyNIO_,
 if PyNIO is installed. To use PyNIO to read such files, supply
 ``engine='pynio'`` to :py:func:`open_dataset`.
@@ -1217,12 +1221,9 @@ We recommend installing PyNIO via conda::
 
     conda install -c conda-forge pynio
 
-.. warning::
-
-    PyNIO is no longer actively maintained and conflicts with netcdf4 > 1.5.3.
-    The PyNIO backend may be moved outside of xarray in the future.
-
 .. _PyNIO: https://www.pyngl.ucar.edu/Nio.shtml
+.. _deprecated: https://github.com/pydata/xarray/issues/4491
+.. _maintained: https://github.com/NCAR/pynio/issues/53
 
 .. _io.PseudoNetCDF:
 

doc/whats-new.rst

@@ -14,15 +14,18 @@ What's New
 
     np.random.seed(123456)
 
-
 .. _whats-new.2022.11.1:
 
 v2022.11.1 (unreleased)
 -----------------------
 
 New Features
 ~~~~~~~~~~~~
-
+- Enable using `offset` and `origin` arguments in :py:meth:`DataArray.resample`
+  and :py:meth:`Dataset.resample` (:issue:`7266`, :pull:`6538`).  By `Spencer
+  Clark <https://github.com/spencerkclark>`_.
+- Add experimental support for Zarr's in-progress V3 specification. (:pull:`6475`).
+  By `Gregory Lee  <https://github.com/grlee77>`_ and `Joe Hamman <https://github.com/jhamman>`_.
 
 Breaking changes
 ~~~~~~~~~~~~~~~~
@@ -49,13 +52,16 @@ Breaking changes
 
 Deprecations
 ~~~~~~~~~~~~
-
+- The PyNIO backend has been deprecated (:issue:`4491`, :pull:`7301`).
+  By `Joe Hamman <https://github.com/jhamman>`_.
 
 Bug fixes
 ~~~~~~~~~
 
 - Import ``nc_time_axis`` when needed (:issue:`7275`, :pull:`7276`).
   By `Michael Niklas <https://github.com/headtr1ck>`_.
+- Fix static typing of :py:meth:`xr.polyval` (:issue:`7312`, :pull:`7315`).
+  By `Michael Niklas <https://github.com/headtr1ck>`_.
 - Fix multiple reads on fsspec S3 files by resetting file pointer to 0 when reading file streams (:issue:`6813`, :pull:`7304`).
   By `David Hoese <https://github.com/djhoese>`_ and `Wei Ji Leong <https://github.com/weiji14>`_.
 

pyproject.toml

@@ -40,8 +40,6 @@ module = [
   "cfgrib.*",
   "cftime.*",
   "cupy.*",
-  "dask.*",
-  "distributed.*",
   "fsspec.*",
   "h5netcdf.*",
   "h5py.*",
@@ -52,11 +50,9 @@ module = [
   "Nio.*",
   "nc_time_axis.*",
   "numbagg.*",
-  "numpy.*",
   "netCDF4.*",
   "netcdftime.*",
   "pandas.*",
-  "pint.*",
   "pooch.*",
   "PseudoNetCDF.*",
   "pydap.*",
@@ -70,8 +66,6 @@ module = [
   "zarr.*",
 ]
 
-# version spanning code is hard to type annotate (and most of this module will
-# be going away soon anyways)
 [[tool.mypy.overrides]]
 ignore_errors = true
-module = "xarray.core.pycompat"
+module = []

setup.cfg

@@ -147,11 +147,12 @@ markers =
 
 [flake8]
 ignore =
-    E203 # whitespace before ':' - doesn't work well with black
-    E402 # module level import not at top of file
-    E501 # line too long - let black worry about that
-    E731 # do not assign a lambda expression, use a def
-    W503 # line break before binary operator
+    # E203: whitespace before ':' - doesn't work well with black
+    # E402: module level import not at top of file
+    # E501: line too long - let black worry about that
+    # E731: do not assign a lambda expression, use a def
+    # W503: line break before binary operator
+    E203, E402, E501, E731, W503
 exclude =
     .eggs
     doc

xarray/backends/api.py

@@ -9,6 +9,7 @@
     TYPE_CHECKING,
     Any,
     Callable,
+    Dict,
     Final,
     Hashable,
     Iterable,
@@ -62,7 +63,7 @@
         str,  # no nice typing support for custom backends
         None,
     ]
-    T_Chunks = Union[int, dict[Any, Any], Literal["auto"], None]
+    T_Chunks = Union[int, Dict[Any, Any], Literal["auto"], None]
     T_NetcdfTypes = Literal[
         "NETCDF4", "NETCDF4_CLASSIC", "NETCDF3_64BIT", "NETCDF3_CLASSIC"
     ]
@@ -1503,6 +1504,7 @@ def to_zarr(
     region: Mapping[str, slice] | None = None,
     safe_chunks: bool = True,
     storage_options: dict[str, str] | None = None,
+    zarr_version: int | None = None,
 ) -> backends.ZarrStore:
     ...
 
@@ -1524,6 +1526,7 @@ def to_zarr(
     region: Mapping[str, slice] | None = None,
     safe_chunks: bool = True,
     storage_options: dict[str, str] | None = None,
+    zarr_version: int | None = None,
 ) -> Delayed:
     ...
 
@@ -1542,6 +1545,7 @@ def to_zarr(
     region: Mapping[str, slice] | None = None,
     safe_chunks: bool = True,
     storage_options: dict[str, str] | None = None,
+    zarr_version: int | None = None,
 ) -> backends.ZarrStore | Delayed:
     """This function creates an appropriate datastore for writing a dataset to
     a zarr ztore
@@ -1608,6 +1612,13 @@ def to_zarr(
                 f"``region`` with to_zarr(), got {append_dim} in both"
             )
 
+    if zarr_version is None:
+        # default to 2 if store doesn't specify it's version (e.g. a path)
+        zarr_version = int(getattr(store, "_store_version", 2))
+
+    if consolidated is None and zarr_version > 2:
+        consolidated = False
+
     if mode == "r+":
         already_consolidated = consolidated
         consolidate_on_close = False
@@ -1626,6 +1637,7 @@ def to_zarr(
         write_region=region,
         safe_chunks=safe_chunks,
         stacklevel=4,  # for Dataset.to_zarr()
+        zarr_version=zarr_version,
     )
 
     if mode in ["a", "r+"]:

xarray/backends/pynio_.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import warnings
+
 import numpy as np
 
 from ..core import indexing
@@ -55,6 +57,12 @@ class NioDataStore(AbstractDataStore):
     def __init__(self, filename, mode="r", lock=None, **kwargs):
         import Nio
 
+        warnings.warn(
+            "The PyNIO backend is Deprecated and will be removed from Xarray in a future release. "
+            "See https://github.com/pydata/xarray/issues/4491 for more information",
+            DeprecationWarning,
+        )
+
         if lock is None:
             lock = PYNIO_LOCK
         self.lock = ensure_lock(lock)
@@ -94,6 +102,15 @@ def close(self):
 
 
 class PynioBackendEntrypoint(BackendEntrypoint):
+    """
+    PyNIO backend
+
+        .. deprecated:: 0.20.0
+
+        Deprecated as PyNIO is no longer supported. See
+        https://github.com/pydata/xarray/issues/4491 for more information
+    """
+
     available = module_available("Nio")
 
     def open_dataset(