Merge branch 'main' into concat-multi-var-index

* main:
  Fix performance regression in interp from pydata#9881 (pydata#10370)
  html repr: improve style for dropdown sections (pydata#10354)
  Grouper tweaks. (pydata#10362)
  Docs: Add links to getting help mermaid diagram (pydata#10324)
  Enforce ruff/flynt rules (FLY) (pydata#10375)
  Add missing AbstractWritableDataStore base methods and arguments (pydata#10343)
  Improve html repr in dark mode (Jupyterlab + Xarray docs) (pydata#10353)
  Pin Mypy to 1.15 (pydata#10378)
  use numpy dtype exposed by zarr array instead of metadata.data_type (pydata#10348)
  Fix doc typo for caption "Interoperability" (pydata#10374)
  Implement cftime vectorization as discussed in PR pydata#8322 (pydata#8324)
  Enforce ruff/flake8-pyi rules (PYI) (pydata#10359)
  Apply assorted ruff/Pylint rules (PL) / Enforce PLE rules (pydata#10366)
  (fix): pandas extension array repr for int64[pyarrow] (pydata#10317)
  Enforce ruff/flake8-implicit-str-concat rules (ISC) (pydata#10368)
  Enforce ruff/refurb rules (FURB) (pydata#10367)
  Ignore ruff/Pyflakes rule F401 more precisely (pydata#10369)
  Apply assorted ruff/flake8-simplify rules (SIM) (pydata#10364)
  Apply assorted ruff/flake8-pytest-style rules (PT) (pydata#10363)
  Fix "a array" misspelling (pydata#10365)

Author: dcherian

ci/requirements/environment.yml

@@ -20,6 +20,7 @@ dependencies:
   - iris
   - lxml # Optional dep of pydap
   - matplotlib-base
+  - mypy==1.15 # mypy 1.16 breaks CI
   - nc-time-axis
   - netcdf4
   - numba

doc/_static/style.css

@@ -44,3 +44,11 @@ html[data-theme="dark"] .sd-card img[src*=".svg"] {
 .bd-content .sd-card .sd-card-body {
   background-color: unset !important;
 }
+
+/* workaround Pydata Sphinx theme using light colors for widget cell outputs in dark-mode */
+/* works for many widgets but not for Xarray html reprs */
+/* https://github.com/pydata/pydata-sphinx-theme/issues/2189 */
+html[data-theme="dark"] div.cell_output .text_html:has(div.xr-wrap) {
+  background-color: var(--pst-color-on-background) !important;
+  color: var(--pst-color-text-base) !important;
+}

doc/conf.py

@@ -6,7 +6,7 @@
 import sys
 from contextlib import suppress
 from textwrap import dedent, indent
-
+import packaging.version
 import sphinx_autosummary_accessors
 import yaml
 from sphinx.application import Sphinx
@@ -185,7 +185,7 @@
 autodoc_type_aliases = napoleon_type_aliases  # Keep both in sync
 
 # mermaid config
-mermaid_version = "10.9.1"
+mermaid_version = "11.6.0"
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates", sphinx_autosummary_accessors.templates_path]
@@ -201,10 +201,9 @@
 project = "xarray"
 copyright = f"2014-{datetime.datetime.now().year}, xarray Developers"
 
-# The short X.Y version.
-version = xarray.__version__.split("+")[0]
-# The full version, including alpha/beta/rc tags.
-release = xarray.__version__
+# The short Y.M.D version.
+v = packaging.version.parse(xarray.__version__)
+version = ".".join(str(p) for p in v.release)
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
@@ -314,6 +313,7 @@
     "installing.rst": "getting-started-guide/installing.rst",
     "quick-overview.rst": "getting-started-guide/quick-overview.rst",
     "contributing.rst": "contribute/contributing.rst",
+    "developers-meeting.rst": "contribute/developers-meeting.rst",
 }
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,

doc/get-help/help-diagram.rst

@@ -3,53 +3,72 @@ Getting Help
 
 Navigating the wealth of resources available for Xarray can be overwhelming.
 We've created this flow chart to help guide you towards the best way to get help, depending on what you're working towards.
-The links to each resource are provided below the diagram.
 
-Also be sure to check out our "FAQ" and "How do I..." pages in this section for solutions to common questions.
+Also be sure to check out our :ref:`faq`. and :ref:`howdoi` pages for solutions to common questions.
 
-A major strength of Xarray is in the user community. Sometimes you might not have a concrete question by would simply like to connect with other Xarray users. We have a few
+A major strength of Xarray is in the user community. Sometimes you might not yet have a concrete question but would simply like to connect with other Xarray users. We have a few accounts on different social platforms for that! :ref:`socials`.
 
 We look forward to hearing from you!
 
+Help Flowchart
+--------------
+..
+   _comment: mermaid Flowcharg "link" text gets secondary color background, SVG icon fill gets primary color
+
+.. raw:: html
+
+    <style>
+      /* Ensure PST link colors don't override mermaid text colors */
+      .mermaid a {
+        color: white;
+      }
+      .mermaid a:hover {
+        color: magenta;
+        text-decoration-color: magenta;
+      }
+      .mermaid a:visited {
+        color: white;
+        text-decoration-color: white;
+      }
+    </style>
+
 .. mermaid::
+    :config: {"theme":"base","themeVariables":{"fontSize":"20px","primaryColor":"#fff","primaryTextColor":"#fff","primaryBorderColor":"#59c7d6","lineColor":"#e28126","secondaryColor":"#767985"}}
     :alt: Flowchart illustrating the different ways to access help using or contributing to Xarray.
 
     flowchart TD
         intro[Welcome to Xarray! How can we help?]:::quesNodefmt
-        usage(["fa:fa-chalkboard-user Xarray Tutorials
-            fab:fa-readme Xarray Docs
-            fab:fa-google Google/fab:fa-stack-overflow Stack Exchange
-            fa:fa-robot Ask AI/a Language Learning Model (LLM)"]):::ansNodefmt
-        API([fab:fa-readme Xarray Docs
-            fab:fa-readme extension's docs]):::ansNodefmt
-        help([fab:fa-github Xarray Discussions
-            fab:fa-discord Xarray Discord
-            fa:fa-users Xarray Office Hours
-            fa:fa-globe Pangeo Discourse]):::ansNodefmt
-        bug([Report and Propose here:
-            fab:fa-github Xarray Issues]):::ansNodefmt
-        contrib([fa:fa-book-open Xarray Contributor's Guide]):::ansNodefmt
-        pr(["fab:fa-github Pull Request (PR)"]):::ansNodefmt
-        dev([fab:fa-github Comment on your PR
-            fa:fa-users Developer's Meeting]):::ansNodefmt
+        usage([fa:fa-chalkboard-user <a href="https://tutorial.xarray.dev">Xarray Tutorial</a>
+            fab:fa-readme <a href="https://docs.xarray.dev">Xarray Docs</a>
+            fab:fa-stack-overflow <a href="https://stackoverflow.com/questions/tagged/python-xarray">Stack Exchange</a>
+            fab:fa-google <a href="https://www.google.com">Ask Google</a>
+            fa:fa-robot Ask AI ChatBot]):::ansNodefmt
+        extensions([Extension docs:
+            fab:fa-readme <a href="https://docs.dask.org">Dask</a>
+            fab:fa-readme <a href="https://corteva.github.io/rioxarray">Rioxarray</a>]):::ansNodefmt
+        help([fab:fa-github <a href="https://github.com/pydata/xarray/discussions">Xarray Discussions</a>
+            fab:fa-discord <a href="https://discord.com/invite/wEKPCt4PDu">Xarray Discord</a>
+            fa:fa-globe <a href="https://discourse.pangeo.io">Pangeo Discourse</a>]):::ansNodefmt
+        bug([Let us know:
+            fab:fa-github <a href="https://github.com/pydata/xarray/issues">Xarray Issues</a>]):::ansNodefmt
+        contrib([fa:fa-book-open <a href="https://docs.xarray.dev/en/latest/contribute">Xarray Contributor's Guide</a>]):::ansNodefmt
+        pr([fab:fa-github <a href="https://github.com/pydata/xarray/pulls">Pull Request</a>]):::ansNodefmt
+        dev([fab:fa-github Add PR Comment
+            fa:fa-users <a href="https://docs.xarray.dev/en/stable/contribute/developers-meeting.html">Attend Developer's Meeting</a> ]):::ansNodefmt
         report[Thanks for letting us know!]:::quesNodefmt
-        merged[fa:fa-hands-clapping Your PR was merged.
-            Thanks for contributing to Xarray!]:::quesNodefmt
+        merged[fa:fa-hands-clapping Thanks for contributing to Xarray!]:::quesNodefmt
 
 
         intro -->|How do I use Xarray?| usage
-        usage -->|"with extensions (like Dask)"| API
+        usage -->|"With extensions (like Dask, Rioxarray, etc.)"| extensions
 
-        usage -->|I'd like some more help| help
-        intro -->|I found a bug| bug
-        intro -->|I'd like to make a small change| contrib
-        subgraph bugcontrib[Bugs and Contributions]
-            bug
-            contrib
-            bug -->|I just wanted to tell you| report
-            bug<-->|I'd like to fix the bug!| contrib
-            pr -->|my PR was approved| merged
-        end
+        usage -->|I still have questions or could use some guidance | help
+        intro -->|I think I found a bug| bug
+        bug
+        contrib
+        bug -->|I just wanted to tell you| report
+        bug<-->|I'd like to fix the bug!| contrib
+        pr -->|my PR was approved| merged
 
 
         intro -->|I wish Xarray could...| bug
@@ -58,20 +77,16 @@ We look forward to hearing from you!
         pr <-->|my PR is quiet| dev
         contrib -->pr
 
-        classDef quesNodefmt fill:#9DEEF4,stroke:#206C89
-
-        classDef ansNodefmt fill:#FFAA05,stroke:#E37F17
+        classDef quesNodefmt font-size:20pt,fill:#0e4666,stroke:#59c7d6,stroke-width:3
+        classDef ansNodefmt font-size:18pt,fill:#4a4a4a,stroke:#17afb4,stroke-width:3
+        linkStyle default font-size:16pt,stroke-width:4
 
-        classDef boxfmt fill:#FFF5ED,stroke:#E37F17
-        class bugcontrib boxfmt
-
-        linkStyle default font-size:20pt,color:#206C89
 
 Flowchart links
 ---------------
 - `Xarray Tutorials <https://tutorial.xarray.dev/>`__
 - `Xarray Docs <https://docs.xarray.dev>`__
-- `Google/Stack Exchange <https://stackoverflow.com/questions/tagged/python-xarray>`__
+- `Stack Exchange <https://stackoverflow.com/questions/tagged/python-xarray>`__
 - `Xarray Discussions <https://github.com/pydata/xarray/discussions>`__
 - `Xarray Discord <https://discord.com/invite/wEKPCt4PDu>`__
 - `Xarray Office Hours <https://github.com/pydata/xarray/discussions/categories/office-hours>`__
@@ -80,7 +95,6 @@ Flowchart links
 - :ref:`contributing`
 - :ref:`developers-meeting`
 
-
 .. toctree::
    :maxdepth: 1
    :hidden:

doc/get-help/howdoi.rst

@@ -69,7 +69,7 @@ How do I ...
      - ``obj.dt.month`` for example where ``obj`` is a :py:class:`~xarray.DataArray` containing ``datetime64`` or ``cftime`` values. See :ref:`dt_accessor` for more.
    * - round off time values to a specified frequency
      - ``obj.dt.ceil``, ``obj.dt.floor``, ``obj.dt.round``. See :ref:`dt_accessor` for more.
-   * - make a mask that is ``True`` where an object contains any of the values in a array
+   * - make a mask that is ``True`` where an object contains any of the values in an array
      - :py:meth:`Dataset.isin`, :py:meth:`DataArray.isin`
    * - Index using a boolean mask
      - :py:meth:`Dataset.query`, :py:meth:`DataArray.query`, :py:meth:`Dataset.where`, :py:meth:`DataArray.where`

doc/user-guide/index.rst

@@ -43,7 +43,7 @@ examples that describe many common tasks that you can accomplish with Xarray.
 
 .. toctree::
    :maxdepth: 2
-   :caption: Interoperatbility
+   :caption: Interoperability
 
    pandas
    duckarrays

doc/whats-new.rst

@@ -50,6 +50,10 @@ Bug fixes
   calculating mean in rolling for correct operations (preserve float dtypes,
   correct mean of bool arrays) (:issue:`10340`, :pull:`10341`).
   By `Kai Mühlbauer <https://github.com/kmuehlbauer>`_.
+- Improve the html ``repr`` of Xarray objects (dark mode, icons and variable attribute / data
+  dropdown sections).
+  (:pull:`10353`, :pull:`10354`)
+  By `Benoit Bovy <https://github.com/benbovy>`_.
 - Raise an error when attempting to encode :py:class:`numpy.datetime64` values
   prior to the Gregorian calendar reform date of 1582-10-15 with a
   ``"standard"`` or ``"gregorian"`` calendar. Previously we would warn and
@@ -65,7 +69,10 @@ Performance
   in :py:class:`~xarray.indexing.VectorizedIndexer` and :py:class:`~xarray.indexing.OuterIndexer`
   (:issue:`10316`).
   By `Jesse Rusak <https://github.com/jder>`_.
-
+- Fix performance regression in interp where more data was loaded than was necessary. (:issue:`10287`).
+  By `Deepak Cherian <https://github.com/dcherian>`_.
+- Speed up encoding of :py:class:`cftime.datetime` objects by roughly a factor
+  of three (:pull:`8324`). By `Antoine Gibek <https://github.com/antscloud>`_.
 
 Documentation
 ~~~~~~~~~~~~~

pyproject.toml

@@ -249,27 +249,35 @@ extend-exclude = ["doc", "_typed_ops.pyi"]
 
 [tool.ruff.lint]
 extend-select = [
-  "W",    # pycodestyle warnings
-  "I",    # isort
-  "UP",   # pyupgrade
   "B",    # flake8-bugbear
   "C4",   # flake8-comprehensions
+  "ISC",  # flake8-implicit-str-concat
   "PIE",  # flake8-pie
   "TID",  # flake8-tidy-imports (absolute imports)
-  "PGH",  # pygrep-hooks
+  "PYI",  # flake8-pyi
+  "FLY",  # flynt
+  "I",    # isort
   "PERF", # Perflint
+  "W",    # pycodestyle warnings
+  "PGH",  # pygrep-hooks
+  "PLE",  # Pylint Errors
+  "UP",   # pyupgrade
+  "FURB", # refurb
   "RUF",
 ]
 extend-safe-fixes = [
   "TID252", # absolute imports
 ]
 ignore = [
-  "E402",    # module level import not at top of file
-  "E731",    # do not assign a lambda expression, use a def
-  "UP007",   # use X | Y for type annotations
   "C40",     # unnecessary generator, comprehension, or literal
   "PIE790",  # unnecessary pass statement
+  "PYI019",  # use `Self` instead of custom TypeVar
+  "PYI041",  # use `float` instead of `int | float`
   "PERF203", # try-except within a loop incurs performance overhead
+  "E402",    # module level import not at top of file
+  "E731",    # do not assign a lambda expression, use a def
+  "UP007",   # use X | Y for type annotations
+  "FURB105", # unnecessary empty string passed to `print`
   "RUF001",  # string contains ambiguous unicode character
   "RUF002",  # docstring contains ambiguous acute accent unicode character
   "RUF003",  # comment contains ambiguous no-break space unicode character
@@ -280,6 +288,9 @@ ignore = [
 [tool.ruff.lint.per-file-ignores]
 # don't enforce absolute imports
 "asv_bench/**" = ["TID252"]
+# looks like ruff bugs
+"xarray/core/_typed_ops.py" = ["PYI034"]
+"xarray/namedarray/_typing.py" = ["PYI018", "PYI046"]
 
 [tool.ruff.lint.isort]
 known-first-party = ["xarray"]

xarray/backends/api.py

@@ -71,7 +71,7 @@
     T_NetcdfEngine = Literal["netcdf4", "scipy", "h5netcdf"]
     T_Engine = Union[
         T_NetcdfEngine,
-        Literal["pydap", "zarr"],
+        Literal["pydap", "zarr"],  # noqa: PYI051
         type[BackendEntrypoint],
         str,  # no nice typing support for custom backends
         None,
@@ -710,8 +710,8 @@ def open_dataset(
 def open_dataarray(
     filename_or_obj: str | os.PathLike[Any] | ReadBuffer | AbstractDataStore,
     *,
-    engine: T_Engine | None = None,
-    chunks: T_Chunks | None = None,
+    engine: T_Engine = None,
+    chunks: T_Chunks = None,
     cache: bool | None = None,
     decode_cf: bool | None = None,
     mask_and_scale: bool | None = None,
@@ -1394,7 +1394,7 @@ def open_mfdataset(
     | os.PathLike
     | ReadBuffer
     | NestedSequence[str | os.PathLike | ReadBuffer],
-    chunks: T_Chunks | None = None,
+    chunks: T_Chunks = None,
     concat_dim: (
         str
         | DataArray
@@ -1406,7 +1406,7 @@ def open_mfdataset(
     ) = None,
     compat: CompatOptions = "no_conflicts",
     preprocess: Callable[[Dataset], Dataset] | None = None,
-    engine: T_Engine | None = None,
+    engine: T_Engine = None,
     data_vars: Literal["all", "minimal", "different"] | list[str] = "all",
     coords="different",
     combine: Literal["by_coords", "nested"] = "by_coords",

xarray/backends/common.py

@@ -338,11 +338,10 @@ def add(self, source, target, region=None):
             self.sources.append(source)
             self.targets.append(target)
             self.regions.append(region)
+        elif region:
+            target[region] = source
         else:
-            if region:
-                target[region] = source
-            else:
-                target[...] = source
+            target[...] = source
 
     def sync(self, compute=True, chunkmanager_store_kwargs=None):
         if self.sources:
@@ -402,7 +401,10 @@ def encode_attribute(self, a):
         """encode one attribute"""
         return a
 
-    def set_dimension(self, dim, length):  # pragma: no cover
+    def prepare_variable(self, name, variable, check_encoding, unlimited_dims):
+        raise NotImplementedError()
+
+    def set_dimension(self, dim, length, is_unlimited):  # pragma: no cover
         raise NotImplementedError()
 
     def set_attribute(self, k, v):  # pragma: no cover

xarray/backends/h5netcdf_.py

@@ -68,17 +68,16 @@ def _read_attributes(h5netcdf_var):
     # bytes attributes to strings
     attrs = {}
     for k, v in h5netcdf_var.attrs.items():
-        if k not in ["_FillValue", "missing_value"]:
-            if isinstance(v, bytes):
-                try:
-                    v = v.decode("utf-8")
-                except UnicodeDecodeError:
-                    emit_user_level_warning(
-                        f"'utf-8' codec can't decode bytes for attribute "
-                        f"{k!r} of h5netcdf object {h5netcdf_var.name!r}, "
-                        f"returning bytes undecoded.",
-                        UnicodeWarning,
-                    )
+        if k not in ["_FillValue", "missing_value"] and isinstance(v, bytes):
+            try:
+                v = v.decode("utf-8")
+            except UnicodeDecodeError:
+                emit_user_level_warning(
+                    f"'utf-8' codec can't decode bytes for attribute "
+                    f"{k!r} of h5netcdf object {h5netcdf_var.name!r}, "
+                    f"returning bytes undecoded.",
+                    UnicodeWarning,
+                )
         attrs[k] = v
     return attrs
 

xarray/backends/locks.py

@@ -118,9 +118,7 @@ def _get_lock_maker(scheduler=None):
     dask.utils.get_scheduler_lock
     """
 
-    if scheduler is None:
-        return _get_threaded_lock
-    elif scheduler == "threaded":
+    if scheduler is None or scheduler == "threaded":
         return _get_threaded_lock
     elif scheduler == "multiprocessing":
         return _get_multiprocessing_lock