Merge remote-tracking branch 'upstream/main' into groupby-aggs-using-numpy-groupies

* upstream/main: (23 commits)
  Small typing fix (pydata#6159)
  Drop support for python 3.7 (pydata#5892)
  _season_from_months can now handle np.nan (pydata#5876)
  Use base ImportError not MoudleNotFoundError when trying to see if the (pydata#6154)
  Remove numpy from mypy pre-commit (pydata#6151)
  Change concat dims to be Hashable (pydata#6121)
  Bump pypa/gh-action-pypi-publish from 1.4.2 to 1.5.0 (pydata#6147)
  Remove registration of pandas datetime converter in plotting (pydata#6109)
  Remove pd.Panel checks (pydata#6145)
  Remove paren from DataArray.from_dict docstring (pydata#6140)
  Revert "Deprecate bool(ds) (pydata#6126)" (pydata#6141)
  remove paren from data that is fed to 1D DataArray (pydata#6139)
  Check for just `...`, rather than `[...]` in `da.stack` (pydata#6132)
  DOC: Add "auto" to dataarray `chunk` method (pydata#6068)
  TST: check datetime converter is Matplotlibs (pydata#6128)
  New algorithm for forward filling (pydata#6118)
  Limit and format number of displayed dimensions in repr (pydata#5662)
  Add labels to dataset diagram (pydata#6076)
  Deprecate bool(ds) (pydata#6126)
  Revert "disable pytest-xdist (to check CI failure)" (pydata#6127)
  ...

Author: dcherian

.github/workflows/ci-additional.yaml

@@ -40,10 +40,13 @@ jobs:
         os: ["ubuntu-latest"]
         env:
           [
-            "py37-bare-minimum",
-            "py37-min-all-deps",
-            "py38-all-but-dask",
-            "py38-flaky",
+            # Minimum python version:
+            "py38-bare-minimum",
+            "py38-min-all-deps",
+
+            # Latest python version:
+            "py39-all-but-dask",
+            "py39-flaky",
           ]
     steps:
       - uses: actions/checkout@v2
@@ -52,7 +55,7 @@ jobs:
 
       - name: Set environment variables
         run: |
-          if [[ ${{ matrix.env }} == "py38-flaky" ]] ;
+          if [[ ${{ matrix.env }} == "py39-flaky" ]] ;
           then
             echo "CONDA_ENV_FILE=ci/requirements/environment.yml" >> $GITHUB_ENV
             echo "PYTEST_EXTRA_FLAGS=--run-flaky --run-network-tests" >> $GITHUB_ENV
@@ -75,7 +78,7 @@ jobs:
           mamba-version: "*"
           activate-environment: xarray-tests
           auto-update-conda: false
-          python-version: 3.8
+          python-version: 3.9
           use-only-tar-bz2: true
 
       - name: Install conda dependencies
@@ -96,7 +99,7 @@ jobs:
           python -c "import xarray"
       - name: Run tests
         run: |
-          python -m pytest \
+          python -m pytest -n 4 \
              --cov=xarray \
              --cov-report=xml \
              $PYTEST_EXTRA_FLAGS
@@ -128,7 +131,7 @@ jobs:
           mamba-version: "*"
           activate-environment: xarray-tests
           auto-update-conda: false
-          python-version: "3.8"
+          python-version: "3.9"
 
       - name: Install conda dependencies
         run: |
@@ -164,10 +167,10 @@ jobs:
           channel-priority: strict
           mamba-version: "*"
           auto-update-conda: false
-          python-version: "3.8"
+          python-version: "3.9"
 
       - name: minimum versions policy
         run: |
           mamba install -y pyyaml conda python-dateutil
-          python ci/min_deps_check.py ci/requirements/py37-bare-minimum.yml
-          python ci/min_deps_check.py ci/requirements/py37-min-all-deps.yml
+          python ci/min_deps_check.py ci/requirements/py38-bare-minimum.yml
+          python ci/min_deps_check.py ci/requirements/py38-min-all-deps.yml

.github/workflows/ci.yaml

@@ -38,7 +38,7 @@ jobs:
       matrix:
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
         # Bookend python versions
-        python-version: ["3.7", "3.9"]
+        python-version: ["3.8", "3.9"]
     steps:
       - uses: actions/checkout@v2
         with:
@@ -87,7 +87,7 @@ jobs:
         run: |
           python -c "import xarray"
       - name: Run tests
-        run: python -m pytest
+        run: python -m pytest -n 4
           --cov=xarray
           --cov-report=xml
           --junitxml=pytest.xml

.github/workflows/pypi-release.yaml

@@ -64,7 +64,7 @@ jobs:
           ls -ltrh dist
       - name: Publish package to TestPyPI
         if: github.event_name == 'push'
-        uses: pypa/gh-action-pypi-publish@v1.4.2
+        uses: pypa/gh-action-pypi-publish@v1.5.0
         with:
           user: __token__
           password: ${{ secrets.TESTPYPI_TOKEN }}
@@ -89,7 +89,7 @@ jobs:
           name: releases
           path: dist
       - name: Publish package to PyPI
-        uses: pypa/gh-action-pypi-publish@v1.4.2
+        uses: pypa/gh-action-pypi-publish@v1.5.0
         with:
           user: __token__
           password: ${{ secrets.PYPI_TOKEN }}

.pre-commit-config.yaml

@@ -32,7 +32,7 @@ repos:
   #     - id: velin
   #       args: ["--write", "--compact"]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.930
+    rev: v0.931
     hooks:
       - id: mypy
         # `properies` & `asv_bench` are copied from setup.cfg.
@@ -45,8 +45,6 @@ repos:
             types-PyYAML,
             types-pytz,
             typing-extensions==3.10.0.0,
-            # Dependencies that are typed
-            numpy,
           ]
   # run this occasionally, ref discussion https://github.com/pydata/xarray/pull/3194
   # - repo: https://github.com/asottile/pyupgrade

ci/requirements/environment-windows.yml

@@ -37,7 +37,6 @@ dependencies:
   - rasterio
   - scipy
   - seaborn
-  - setuptools
   - sparse
   - toolz
   - typing_extensions

ci/requirements/environment.yml

@@ -41,7 +41,6 @@ dependencies:
   - rasterio
   - scipy
   - seaborn
-  - setuptools
   - sparse
   - toolz
   - typing_extensions

ci/requirements/py37-bare-minimum.yml renamed to ci/requirements/py38-bare-minimum.yml

@@ -3,7 +3,7 @@ channels:
   - conda-forge
   - nodefaults
 dependencies:
-  - python=3.7
+  - python=3.8
   - coveralls
   - pip
   - pytest
@@ -12,5 +12,3 @@ dependencies:
   - pytest-xdist
   - numpy=1.18
   - pandas=1.1
-  - typing_extensions=3.7
-  - importlib-metadata=2.0

ci/requirements/py37-min-all-deps.yml renamed to ci/requirements/py38-min-all-deps.yml

@@ -7,7 +7,7 @@ dependencies:
   # Run ci/min_deps_check.py to verify that this file respects the policy.
   # When upgrading python, numpy, or pandas, must also change
   # doc/installing.rst and setup.py.
-  - python=3.7
+  - python=3.8
   - boto3=1.13
   - bottleneck=1.3
   # cartopy 0.18 conflicts with pynio
@@ -24,7 +24,6 @@ dependencies:
   - hdf5=1.10
   - hypothesis
   - iris=2.4
-  - importlib-metadata=2.0
   - lxml=4.6  # Optional dep of pydap
   - matplotlib-base=3.3
   - nc-time-axis=1.2

ci/requirements/py38-all-but-dask.yml renamed to ci/requirements/py39-all-but-dask.yml

@@ -3,7 +3,7 @@ channels:
   - conda-forge
   - nodefaults
 dependencies:
-  - python=3.8
+  - python=3.9
   - black
   - aiobotocore
   - boto3
@@ -38,7 +38,6 @@ dependencies:
   - rasterio
   - scipy
   - seaborn
-  - setuptools
   - sparse
   - toolz
   - typing_extensions

doc/_static/dataset-diagram.png

@@ -77,6 +77,7 @@
    core.accessor_dt.DatetimeAccessor.floor
    core.accessor_dt.DatetimeAccessor.round
    core.accessor_dt.DatetimeAccessor.strftime
+   core.accessor_dt.DatetimeAccessor.calendar
    core.accessor_dt.DatetimeAccessor.date
    core.accessor_dt.DatetimeAccessor.day
    core.accessor_dt.DatetimeAccessor.dayofweek

doc/api-hidden.rst

@@ -109,6 +109,8 @@ Dataset contents
    Dataset.drop_dims
    Dataset.set_coords
    Dataset.reset_coords
+   Dataset.convert_calendar
+   Dataset.interp_calendar
    Dataset.get_index
 
 Comparisons
@@ -308,6 +310,8 @@ DataArray contents
    DataArray.drop_duplicates
    DataArray.reset_coords
    DataArray.copy
+   DataArray.convert_calendar
+   DataArray.interp_calendar
    DataArray.get_index
    DataArray.astype
    DataArray.item
@@ -526,6 +530,7 @@ Datetimelike properties
    DataArray.dt.season
    DataArray.dt.time
    DataArray.dt.date
+   DataArray.dt.calendar
    DataArray.dt.is_month_start
    DataArray.dt.is_month_end
    DataArray.dt.is_quarter_end
@@ -1064,6 +1069,8 @@ Creating custom indexes
    :toctree: generated/
 
    cftime_range
+   date_range
+   date_range_like
 
 Faceting
 --------

doc/api.rst

@@ -6,11 +6,9 @@ Installation
 Required dependencies
 ---------------------
 
-- Python (3.7 or later)
-- `importlib_metadata <https://importlib_metadata.readthedocs.io/>`__ (1.4 or later, Python 3.7 only)
-- ``typing_extensions`` (3.7 or later, Python 3.7 only)
-- `numpy <http://www.numpy.org/>`__ (1.17 or later)
-- `pandas <http://pandas.pydata.org/>`__ (1.0 or later)
+- Python (3.8 or later)
+- `numpy <https://www.numpy.org/>`__ (1.18 or later)
+- `pandas <https://pandas.pydata.org/>`__ (1.1 or later)
 
 .. _optional-dependencies:
 
@@ -103,7 +101,7 @@ release is guaranteed to work.
 
 You can see the actual minimum tested versions:
 
-`<https://github.com/pydata/xarray/blob/main/ci/requirements/py37-min-all-deps.yml>`_
+`<https://github.com/pydata/xarray/blob/main/ci/requirements/py38-min-all-deps.yml>`_
 
 .. _installation-instructions:
 

doc/getting-started-guide/installing.rst

@@ -127,6 +127,23 @@ using the same formatting as the standard `datetime.strftime`_ convention .
     dates.strftime("%c")
     da["time"].dt.strftime("%Y%m%d")
 
+Conversion between non-standard calendar and to/from pandas DatetimeIndexes is
+facilitated with the :py:meth:`xarray.Dataset.convert_calendar` method (also available as
+:py:meth:`xarray.DataArray.convert_calendar`). Here, like elsewhere in xarray, the ``use_cftime``
+argument controls which datetime backend is used in the output. The default (``None``) is to
+use `pandas` when possible, i.e. when the calendar is standard and dates are within 1678 and 2262.
+
+.. ipython:: python
+
+    dates = xr.cftime_range(start="2001", periods=24, freq="MS", calendar="noleap")
+    da_nl = xr.DataArray(np.arange(24), coords=[dates], dims=["time"], name="foo")
+    da_std = da.convert_calendar("standard", use_cftime=True)
+
+The data is unchanged, only the timestamps are modified. Further options are implemented
+for the special ``"360_day"`` calendar and for handling missing dates. There is also
+:py:meth:`xarray.Dataset.interp_calendar` (and :py:meth:`xarray.DataArray.interp_calendar`)
+for `interpolating` data between calendars.
+
 For data indexed by a :py:class:`~xarray.CFTimeIndex` xarray currently supports:
 
 - `Partial datetime string indexing`_:
@@ -150,7 +167,8 @@ For data indexed by a :py:class:`~xarray.CFTimeIndex` xarray currently supports:
 
 - Access of basic datetime components via the ``dt`` accessor (in this case
   just "year", "month", "day", "hour", "minute", "second", "microsecond",
-  "season", "dayofyear", "dayofweek", and "days_in_month"):
+  "season", "dayofyear", "dayofweek", and "days_in_month") with the addition
+  of "calendar", absent from pandas:
 
 .. ipython:: python
 
@@ -160,6 +178,7 @@ For data indexed by a :py:class:`~xarray.CFTimeIndex` xarray currently supports:
     da.time.dt.dayofyear
     da.time.dt.dayofweek
     da.time.dt.days_in_month
+    da.time.dt.calendar
 
 - Rounding of datetimes to fixed frequencies via the ``dt`` accessor:
 
@@ -214,30 +233,6 @@ For data indexed by a :py:class:`~xarray.CFTimeIndex` xarray currently supports:
 
     da.resample(time="81T", closed="right", label="right", base=3).mean()
 
-.. note::
-
-
-   For some use-cases it may still be useful to convert from
-   a :py:class:`~xarray.CFTimeIndex` to a :py:class:`pandas.DatetimeIndex`,
-   despite the difference in calendar types. The recommended way of doing this
-   is to use the built-in :py:meth:`~xarray.CFTimeIndex.to_datetimeindex`
-   method:
-
-   .. ipython:: python
-       :okwarning:
-
-       modern_times = xr.cftime_range("2000", periods=24, freq="MS", calendar="noleap")
-       da = xr.DataArray(range(24), [("time", modern_times)])
-       da
-       datetimeindex = da.indexes["time"].to_datetimeindex()
-       da["time"] = datetimeindex
-
-   However in this case one should use caution to only perform operations which
-   do not depend on differences between dates (e.g. differentiation,
-   interpolation, or upsampling with resample), as these could introduce subtle
-   and silent errors due to the difference in calendar types between the dates
-   encoded in your data and the dates stored in memory.
-
 .. _Timestamp-valid range: https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#timestamp-limitations
 .. _ISO 8601 standard: https://en.wikipedia.org/wiki/ISO_8601
 .. _partial datetime string indexing: https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#partial-string-indexing

doc/user-guide/weather-climate.rst

@@ -24,19 +24,39 @@ New Features
 - New top-level function :py:func:`cross`. (:issue:`3279`, :pull:`5365`).
   By `Jimmy Westling <https://github.com/illviljan>`_.
 
+- Enable the limit option for dask array in the following methods :py:meth:`DataArray.ffill`, :py:meth:`DataArray.bfill`, :py:meth:`Dataset.ffill` and :py:meth:`Dataset.bfill` (:issue:`6112`)
+  By `Joseph Nowak <https://github.com/josephnowak>`_.
 
 Breaking changes
 ~~~~~~~~~~~~~~~~
+- Rely on matplotlib's default datetime converters instead of pandas' (:issue:`6102`, :pull:`6109`).
+  By `Jimmy Westling <https://github.com/illviljan>`_.
+- Improve repr readability when there are a large number of dimensions in datasets or dataarrays by
+  wrapping the text once the maximum display width has been exceeded. (:issue: `5546`, :pull:`5662`)
+  By `Jimmy Westling <https://github.com/illviljan>`_.
 
 
 Deprecations
 ~~~~~~~~~~~~
 - Removed the lock kwarg from the zarr and pydap backends, completing the deprecation cycle started in :issue:`5256`.
   By `Tom Nicholas <https://github.com/TomNicholas>`_.
+- Support for ``python 3.7`` has been dropped. (:pull:`5892`)
+  By `Jimmy Westling <https://github.com/illviljan>`_.
 
 
 Bug fixes
 ~~~~~~~~~
+- Properly support :py:meth:`DataArray.ffill`, :py:meth:`DataArray.bfill`, :py:meth:`Dataset.ffill` and :py:meth:`Dataset.bfill` along chunked dimensions (:issue:`6112`).
+  By `Joseph Nowak <https://github.com/josephnowak>`_.
+
+- Subclasses of ``byte`` and ``str`` (e.g. ``np.str_`` and ``np.bytes_``) will now serialise to disk rather than raising a ``ValueError: unsupported dtype for netCDF4 variable: object`` as they did previously (:pull:`5264`).
+  By `Zeb Nicholls <https://github.com/znicholls>`_.
+
+- Fix applying function with non-xarray arguments using :py:func:`xr.map_blocks`.
+  By `Cindy Chiao <https://github.com/tcchiao>`_.
+
+- `dt.season <https://xarray.pydata.org/en/stable/generated/xarray.DataArray.dt.season.html>`_  can now handle NaN and NaT.  (:pull:`5876`).
+  By `Pierre Loicq <https://github.com/pierreloicq>`_.
 
 
 Documentation
@@ -49,6 +69,9 @@ Internal Changes
 - Replace ``distutils.version`` with ``packaging.version``  (:issue:`6092`).
   By `Mathias Hauser <https://github.com/mathause>`_.
 
+- Removed internal checks for ``pd.Panel`` (:issue:`6145`).
+  By `Matthew Roeschke <https://github.com/mroeschke>`_.
+
 
 .. _whats-new.0.20.2:
 
@@ -155,6 +178,8 @@ New Features
 - Added ``storage_options`` argument to :py:meth:`to_zarr` (:issue:`5601`, :pull:`5615`).
   By `Ray Bell <https://github.com/raybellwaves>`_, `Zachary Blackwood <https://github.com/blackary>`_ and
   `Nathan Lis <https://github.com/wxman22>`_.
+- Added calendar utilities :py:func:`DataArray.convert_calendar`, :py:func:`DataArray.interp_calendar`, :py:func:`date_range`, :py:func:`date_range_like` and :py:attr:`DataArray.dt.calendar` (:issue:`5155`, :pull:`5233`).
+  By `Pascal Bourgault <https://github.com/aulemahal>`_.
 - Histogram plots are set with a title displaying the scalar coords if any, similarly to the other plots (:issue:`5791`, :pull:`5792`).
   By `Maxime Liquet <https://github.com/maximlt>`_.
 - Slice plots display the coords units in the same way as x/y/colorbar labels (:pull:`5847`).

doc/whats-new.rst

@@ -2,7 +2,5 @@
 # it exists to let GitHub build the repository dependency graph
 # https://help.github.com/en/github/visualizing-repository-data-with-graphs/listing-the-packages-that-a-repository-depends-on
 
-numpy >= 1.17
-pandas >= 1.0
-setuptools >= 40.4
-typing-extensions >= 3.7
+numpy >= 1.18
+pandas >= 1.1