Merge branch 'main' into vlen-string-coding

pydata · Nov 16, 2023 · 1ca494d · 1ca494d
2 parents 3c3dd0c + 1411474
commit 1ca494d
Show file tree

Hide file tree

Showing 108 changed files with 5,714 additions and 1,975 deletions.
diff --git a/.binder/environment.yml b/.binder/environment.yml
@@ -2,11 +2,10 @@ name: xarray-examples
 channels:
   - conda-forge
 dependencies:
-  - python=3.9
+  - python=3.10
   - boto3
   - bottleneck
   - cartopy
-  - cdms2
   - cfgrib
   - cftime
   - coveralls
@@ -25,7 +24,7 @@ dependencies:
   - numpy
   - packaging
   - pandas
-  - pint
+  - pint>=0.22
   - pip
   - pooch
   - pydap
@@ -38,5 +37,4 @@ dependencies:
   - toolz
   - xarray
   - zarr
-  - pip:
-    - numbagg
+  - numbagg
diff --git a/.github/ISSUE_TEMPLATE/bugreport.yml b/.github/ISSUE_TEMPLATE/bugreport.yml
@@ -44,6 +44,7 @@ body:
         - label: Complete example — the example is self-contained, including all data and the text of any traceback.
         - label: Verifiable example — the example copy & pastes into an IPython prompt or [Binder notebook](https://mybinder.org/v2/gh/pydata/xarray/main?urlpath=lab/tree/doc/examples/blank_template.ipynb), returning the result.
         - label: New issue — a search of GitHub Issues suggests this is not a duplicate.
+        - label: Recent environment — the issue occurs with the latest version of xarray and its dependencies.
 
   - type: textarea
     id: log-output

diff --git a/.github/labeler.yml b/.github/labeler.yml
@@ -54,6 +54,9 @@ topic-indexing:
   - xarray/core/indexes.py
   - xarray/core/indexing.py
 
+topic-NamedArray:
+  - xarray/namedarray/*
+
 topic-performance:
   - asv_bench/benchmarks/*
   - asv_bench/benchmarks/**/*

diff --git a/.github/workflows/ci-additional.yaml b/.github/workflows/ci-additional.yaml
@@ -82,8 +82,6 @@ jobs:
     name: Mypy
     runs-on: "ubuntu-latest"
     needs: detect-ci-trigger
-    # temporarily skipping due to https://github.com/pydata/xarray/issues/6551
-    if: needs.detect-ci-trigger.outputs.triggered == 'false'
     defaults:
       run:
         shell: bash -l {0}
@@ -190,6 +188,126 @@ jobs:
 
 
 
+  pyright:
+    name: Pyright
+    runs-on: "ubuntu-latest"
+    needs: detect-ci-trigger
+    if: |
+        always()
+        && (
+            contains( github.event.pull_request.labels.*.name, 'run-pyright')
+        )
+    defaults:
+      run:
+        shell: bash -l {0}
+    env:
+      CONDA_ENV_FILE: ci/requirements/environment.yml
+      PYTHON_VERSION: "3.10"
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # Fetch all history for all branches and tags.
+
+      - name: set environment variables
+        run: |
+          echo "TODAY=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
+      - name: Setup micromamba
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: ${{env.CONDA_ENV_FILE}}
+          environment-name: xarray-tests
+          create-args: >-
+            python=${{env.PYTHON_VERSION}}
+            conda
+          cache-environment: true
+          cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{env.PYTHON_VERSION}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}"
+      - name: Install xarray
+        run: |
+          python -m pip install --no-deps -e .
+      - name: Version info
+        run: |
+          conda info -a
+          conda list
+          python xarray/util/print_versions.py
+      - name: Install pyright
+        run: |
+          python -m pip install pyright --force-reinstall
+
+      - name: Run pyright
+        run: |
+          python -m pyright xarray/
+
+      - name: Upload pyright coverage to Codecov
+        uses: codecov/codecov-action@v3.1.4
+        with:
+          file: pyright_report/cobertura.xml
+          flags: pyright
+          env_vars: PYTHON_VERSION
+          name: codecov-umbrella
+          fail_ci_if_error: false
+
+  pyright39:
+    name: Pyright 3.9
+    runs-on: "ubuntu-latest"
+    needs: detect-ci-trigger
+    if: |
+        always()
+        && (
+            contains( github.event.pull_request.labels.*.name, 'run-pyright')
+        )
+    defaults:
+      run:
+        shell: bash -l {0}
+    env:
+      CONDA_ENV_FILE: ci/requirements/environment.yml
+      PYTHON_VERSION: "3.9"
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # Fetch all history for all branches and tags.
+
+      - name: set environment variables
+        run: |
+          echo "TODAY=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
+      - name: Setup micromamba
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: ${{env.CONDA_ENV_FILE}}
+          environment-name: xarray-tests
+          create-args: >-
+            python=${{env.PYTHON_VERSION}}
+            conda
+          cache-environment: true
+          cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{env.PYTHON_VERSION}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}"
+      - name: Install xarray
+        run: |
+          python -m pip install --no-deps -e .
+      - name: Version info
+        run: |
+          conda info -a
+          conda list
+          python xarray/util/print_versions.py
+      - name: Install pyright
+        run: |
+          python -m pip install pyright --force-reinstall
+
+      - name: Run pyright
+        run: |
+          python -m pyright xarray/
+
+      - name: Upload pyright coverage to Codecov
+        uses: codecov/codecov-action@v3.1.4
+        with:
+          file: pyright_report/cobertura.xml
+          flags: pyright39
+          env_vars: PYTHON_VERSION
+          name: codecov-umbrella
+          fail_ci_if_error: false
+
+
+
   min-version-policy:
     name: Minimum Version Policy
     runs-on: "ubuntu-latest"

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -67,13 +67,7 @@ jobs:
         run: |
           echo "TODAY=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
 
-          if [[ "${{matrix.python-version}}" == "3.11" ]]; then
-            if [[ ${{matrix.os}} == windows* ]]; then
-              echo "CONDA_ENV_FILE=ci/requirements/environment-windows-py311.yml" >> $GITHUB_ENV
-            else
-              echo "CONDA_ENV_FILE=ci/requirements/environment-py311.yml" >> $GITHUB_ENV
-            fi
-          elif [[ ${{ matrix.os }} == windows* ]] ;
+          if [[ ${{ matrix.os }} == windows* ]] ;
           then
             echo "CONDA_ENV_FILE=ci/requirements/environment-windows.yml" >> $GITHUB_ENV
           elif [[ "${{ matrix.env }}" != "" ]] ;

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -3,7 +3,7 @@ ci:
     autoupdate_schedule: monthly
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -18,24 +18,24 @@ repos:
         files: ^xarray/
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: 'v0.0.287'
+    rev: 'v0.1.4'
     hooks:
       - id: ruff
         args: ["--fix"]
   # https://github.com/python/black#version-control-integration
   - repo: https://github.com/psf/black
-    rev: 23.7.0
+    rev: 23.10.1
     hooks:
       - id: black-jupyter
   - repo: https://github.com/keewis/blackdoc
-    rev: v0.3.8
+    rev: v0.3.9
     hooks:
       - id: blackdoc
         exclude: "generate_aggregations.py"
-        additional_dependencies: ["black==23.7.0"]
+        additional_dependencies: ["black==23.10.1"]
       - id: blackdoc-autoupdate-black
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.5.1
+    rev: v1.6.1
     hooks:
       - id: mypy
         # Copied from setup.cfg

diff --git a/asv_bench/benchmarks/dataset.py b/asv_bench/benchmarks/dataset.py
@@ -0,0 +1,32 @@
+import numpy as np
+
+from xarray import Dataset
+
+from . import requires_dask
+
+
+class DatasetBinaryOp:
+    def setup(self):
+        self.ds = Dataset(
+            {
+                "a": (("x", "y"), np.ones((300, 400))),
+                "b": (("x", "y"), np.ones((300, 400))),
+            }
+        )
+        self.mean = self.ds.mean()
+        self.std = self.ds.std()
+
+    def time_normalize(self):
+        (self.ds - self.mean) / self.std
+
+
+class DatasetChunk:
+    def setup(self):
+        requires_dask()
+        self.ds = Dataset()
+        array = np.ones(1000)
+        for i in range(250):
+            self.ds[f"var{i}"] = ("x", array)
+
+    def time_chunk(self):
+        self.ds.chunk(x=(1,) * 1000)
diff --git a/asv_bench/benchmarks/dataset_io.py b/asv_bench/benchmarks/dataset_io.py
@@ -593,7 +593,7 @@ def load(self) -> tuple:
                 n_variables = 2000
 
                 # Important to have a shape and dtype for lazy loading.
-                shape = (1,)
+                shape = (1000,)
                 dtype = np.dtype(int)
                 variables = {
                     f"long_variable_name_{v}": xr.Variable(
@@ -643,7 +643,7 @@ def open_dataset(
 
         self.engine = PerformanceBackend
 
-    @parameterized(["chunks"], ([None, {}]))
+    @parameterized(["chunks"], ([None, {}, {"time": 10}]))
     def time_open_dataset(self, chunks):
         """
         Time how fast xr.open_dataset is without the slow data reading part.

diff --git a/ci/install-upstream-wheels.sh b/ci/install-upstream-wheels.sh
@@ -45,4 +45,5 @@ python -m pip install \
     git+https://github.com/intake/filesystem_spec \
     git+https://github.com/SciTools/nc-time-axis \
     git+https://github.com/xarray-contrib/flox \
-    git+https://github.com/h5netcdf/h5netcdf
+    git+https://github.com/h5netcdf/h5netcdf \
+    git+https://github.com/dgasmith/opt_einsum
diff --git a/ci/requirements/all-but-dask.yml b/ci/requirements/all-but-dask.yml
@@ -3,13 +3,11 @@ channels:
   - conda-forge
   - nodefaults
 dependencies:
-  - python=3.10
   - black
   - aiobotocore
   - boto3
   - bottleneck
   - cartopy
-  - cdms2
   - cftime
   - coveralls
   - flox
@@ -26,9 +24,8 @@ dependencies:
   - numpy
   - packaging
   - pandas
-  - pint<0.21
+  - pint>=0.22
   - pip
-  - pseudonetcdf
   - pydap
   - pytest
   - pytest-cov

diff --git a/ci/requirements/doc.yml b/ci/requirements/doc.yml
@@ -34,6 +34,7 @@ dependencies:
   - sphinx-book-theme >= 0.3.0
   - sphinx-copybutton
   - sphinx-design
+  - sphinx-inline-tabs
   - sphinx>=5.0
   - zarr>=2.10
   - pip:

diff --git a/ci/requirements/environment-py311.yml b/ci/requirements/environment-py311.yml