Streamline fixtures, use pytest-xdist, drop Python3.8

.github/workflows/main.yml

@@ -11,6 +11,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
 
+env:
+  ESGF_TEST_DATA_VERSION: restructuring
+
 permissions:
   contents: read
 
@@ -44,8 +47,6 @@ jobs:
     strategy:
       matrix:
         include:
-          - python-version: "3.8"
-            tox-env: py38
           - python-version: "3.9"
             tox-env: py39
           - python-version: "3.10"
@@ -117,7 +118,7 @@ jobs:
           python -m pip check || true
       - name: Test with conda
         run: |
-          python -m pytest --cov tests
+          python -m pytest --numprocesses=logical --cov tests
 
   finish:
     needs:

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
     rev: v3.17.0
     hooks:
       - id: pyupgrade
-        args: [ '--py38-plus' ]
+        args: [ '--py39-plus' ]
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.6.0
     hooks:
@@ -72,5 +72,5 @@ ci:
   autoupdate_branch: ''
   autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate'
   autoupdate_schedule: quarterly
-  skip: []
+  skip: [ ]
   submodules: false

CONTRIBUTING.rst

@@ -204,7 +204,7 @@ Before you submit a pull request, please follow these guidelines:
     Remember to add the feature or bug fixes explanation to the appropriate section in the HISTORY.rst.
 
 #.
-    The pull request should work for Python 3.8+ as well as raise test coverage.
+    The pull request should work for Python 3.9+ as well as raise test coverage.
     Pull requests are also checked for documentation build status and for `PEP8`_ compliance.
 
     The build statuses and build errors for pull requests can be found at:

clisops/core/average.py

@@ -1,8 +1,9 @@
 """Average module."""
 
 import warnings
+from collections.abc import Sequence
 from pathlib import Path
-from typing import Sequence, Tuple, Union
+from typing import Union
 
 import cf_xarray  # noqa
 import geopandas as gpd

clisops/core/regrid.py

@@ -99,8 +99,7 @@ def weights_cache_init(
         weights_dir = config["clisops:grid_weights"]["local_weights_dir"]
 
     # Create directory tree if required
-    if not os.path.isdir(weights_dir):
-        os.makedirs(weights_dir)
+    os.makedirs(weights_dir, exist_ok=True)
 
 
 # Initialize weights cache as defined in the clisops configuration (roocs.ini)
@@ -638,7 +637,7 @@ def detect_type(self) -> str:
                         )
                         and all(
                             [
-                                self.ds.dims[dim] > 2
+                                self.ds.sizes[dim] > 2
                                 for dim in [
                                     self.ds[self.lon_bnds].dims[-1],
                                     self.ds[self.lat_bnds].dims[-1],

clisops/core/subset.py

@@ -2,9 +2,10 @@
 
 import numbers
 import re
+from collections.abc import Sequence
 from functools import wraps
 from pathlib import Path
-from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union
+from typing import Callable, Optional, Union
 
 import cf_xarray  # noqa
 import geopandas as gpd
@@ -1186,8 +1187,8 @@ def subset_shape(
 @check_lons
 def subset_bbox(
     da: Union[xarray.DataArray, xarray.Dataset],
-    lon_bnds: Union[np.array, Tuple[Optional[float], Optional[float]]] = None,
-    lat_bnds: Union[np.array, Tuple[Optional[float], Optional[float]]] = None,
+    lon_bnds: Union[np.array, tuple[Optional[float], Optional[float]]] = None,
+    lat_bnds: Union[np.array, tuple[Optional[float], Optional[float]]] = None,
     start_date: Optional[str] = None,
     end_date: Optional[str] = None,
     first_level: Optional[Union[float, int]] = None,
@@ -1371,8 +1372,8 @@ def subset_bbox(
 
 
 def assign_bounds(
-    bounds: Tuple[Optional[float], Optional[float]], coord: xarray.DataArray
-) -> Tuple[Optional[float], Optional[float]]:
+    bounds: tuple[Optional[float], Optional[float]], coord: xarray.DataArray
+) -> tuple[Optional[float], Optional[float]]:
     """Replace unset boundaries by the minimum and maximum coordinates.
 
     Parameters
@@ -1396,7 +1397,7 @@ def assign_bounds(
     return bn, bx
 
 
-def in_bounds(bounds: Tuple[float, float], coord: xarray.DataArray) -> xarray.DataArray:
+def in_bounds(bounds: tuple[float, float], coord: xarray.DataArray) -> xarray.DataArray:
     """Check which coordinates are within the boundaries.
 
     Parameters
@@ -1417,9 +1418,9 @@ def in_bounds(bounds: Tuple[float, float], coord: xarray.DataArray) -> xarray.Da
 
 def _check_desc_coords(
     coord: xarray.Dataset,
-    bounds: Union[Tuple[float, float], List[np.ndarray]],
+    bounds: Union[tuple[float, float], list[np.ndarray]],
     dim: str,
-) -> Tuple[float, float]:
+) -> tuple[float, float]:
     """If Dataset coordinates are descending, and bounds are ascending, reverse bounds."""
     if np.all(coord.diff(dim=dim) < 0) and len(coord) > 1 and bounds[1] > bounds[0]:
         bounds = np.flip(bounds)
@@ -1716,7 +1717,7 @@ def subset_time_by_values(
 def subset_time_by_components(
     da: Union[xarray.DataArray, xarray.Dataset],
     *,
-    time_components: Union[Dict, None] = None,
+    time_components: Union[dict, None] = None,
 ) -> xarray.DataArray:
     """Subsets by one or more time components (year, month, day etc).
 

clisops/ops/average.py

@@ -1,5 +1,6 @@
+from collections.abc import Sequence
 from pathlib import Path
-from typing import List, Optional, Sequence, Tuple, Union
+from typing import Optional, Union
 
 import geopandas as gpd
 import xarray as xr
@@ -50,7 +51,7 @@ def average_over_dims(
     output_type: str = "netcdf",
     split_method: str = "time:auto",
     file_namer: str = "standard",
-) -> List[Union[xr.Dataset, str]]:
+) -> list[Union[xr.Dataset, str]]:
     """Calculate an average over given dimensions.
 
     Parameters
@@ -125,7 +126,7 @@ def average_shape(
     output_type: str = "netcdf",
     split_method: str = "time:auto",
     file_namer: str = "standard",
-) -> List[Union[xr.Dataset, str]]:
+) -> list[Union[xr.Dataset, str]]:
     """Calculate a spatial average over a given shape.
 
     Parameters
@@ -201,7 +202,7 @@ def average_time(
     output_type: str = "netcdf",
     split_method: str = "time:auto",
     file_namer: str = "standard",
-) -> List[Union[xr.Dataset, str]]:
+) -> list[Union[xr.Dataset, str]]:
     """
 
     Parameters

clisops/ops/base_operation.py

@@ -1,10 +1,10 @@
 from collections import ChainMap
 from pathlib import Path
-from typing import List, Optional, Union
+from typing import Optional, Union
 
 import xarray as xr
 from loguru import logger
-from roocs_utils.xarray_utils.xarray_utils import get_main_variable, open_xr_dataset
+from roocs_utils.xarray_utils.xarray_utils import open_xr_dataset
 
 from clisops.utils.common import expand_wildcards
 from clisops.utils.file_namers import get_file_namer
@@ -199,7 +199,7 @@ def _remove_redundant_coordinates_attr(ds):
                 ds[var].attrs.pop("coordinates", None)
         return ds
 
-    def process(self) -> List[Union[xr.Dataset, Path]]:
+    def process(self) -> list[Union[xr.Dataset, Path]]:
         """Main processing method used by all subclasses.
 
         Returns

clisops/ops/regrid.py

@@ -1,6 +1,6 @@
 from datetime import datetime as dt
 from pathlib import Path
-from typing import List, Optional, Union
+from typing import Optional, Union
 
 import xarray as xr
 from loguru import logger
@@ -179,7 +179,7 @@ def regrid(
     split_method: Optional[str] = "time:auto",
     file_namer: Optional[str] = "standard",
     keep_attrs: Optional[Union[bool, str]] = True,
-) -> List[Union[xr.Dataset, str]]:
+) -> list[Union[xr.Dataset, str]]:
     """Regrid specified input file or xarray object.
 
     Parameters

clisops/ops/subset.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple, Union
+from typing import List, Optional, Union
 
 import xarray as xr
 from loguru import logger
@@ -162,11 +162,11 @@ def _calculate(self):
 def subset(
     ds: Union[xr.Dataset, str, Path],
     *,
-    time: Optional[Union[str, Tuple[str, str], TimeParameter, Series, Interval]] = None,
+    time: Optional[Union[str, tuple[str, str], TimeParameter, Series, Interval]] = None,
     area: Optional[
         Union[
             str,
-            Tuple[
+            tuple[
                 Union[int, float, str],
                 Union[int, float, str],
                 Union[int, float, str],
@@ -178,19 +178,19 @@ def subset(
     level: Optional[
         Union[
             str,
-            Tuple[Union[int, float, str], Union[int, float, str]],
+            tuple[Union[int, float, str], Union[int, float, str]],
             LevelParameter,
             Interval,
         ]
     ] = None,
     time_components: Optional[
-        Union[str, Dict, TimeComponents, TimeComponentsParameter]
+        Union[str, dict, TimeComponents, TimeComponentsParameter]
     ] = None,
     output_dir: Optional[Union[str, Path]] = None,
     output_type="netcdf",
     split_method="time:auto",
     file_namer="standard",
-) -> List[Union[xr.Dataset, str]]:
+) -> list[Union[xr.Dataset, str]]:
     """Subset operation.
 
     Parameters

clisops/utils/__init__.py

@@ -1,2 +1,2 @@
 from .common import *
-from .tutorial import *
+from .testing import *

clisops/utils/common.py

@@ -4,7 +4,7 @@
 import warnings
 from pathlib import Path
 from types import FunctionType, ModuleType
-from typing import List, Optional, Union
+from typing import Optional, Union
 
 from loguru import logger
 from packaging.version import Version
@@ -74,7 +74,7 @@ def _logging_examples() -> None:
     logger.critical("5")
 
 
-def enable_logging() -> List[int]:
+def enable_logging() -> list[int]:
     logger.enable("clisops")
 
     config = dict(

clisops/utils/dataset_utils.py

@@ -1,5 +1,5 @@
 import warnings
-from typing import Optional, Tuple
+from typing import Optional
 
 import cf_xarray as cfxr  # noqa
 import cftime
@@ -498,14 +498,14 @@ def reformat_SCRIP_to_CF(ds, keep_attrs=False):
             (
                 ds.grid_dims.values[1],
                 ds.grid_dims.values[0],
-                ds.dims["grid_corners"],
+                ds.sizes["grid_corners"],
             )
         ).astype(np.float32)
         lon_b = ds.grid_corner_lon.values.reshape(
             (
                 ds.grid_dims.values[1],
                 ds.grid_dims.values[0],
-                ds.dims["grid_corners"],
+                ds.sizes["grid_corners"],
             )
         ).astype(np.float32)
         lat_bnds = np.zeros((ds.grid_dims.values[1], 2), dtype=np.float32)
@@ -668,7 +668,7 @@ def detect_format(ds):
         raise Exception("The grid format is not supported.")
 
 
-def detect_shape(ds, lat, lon, grid_type) -> Tuple[int, int, int]:
+def detect_shape(ds, lat, lon, grid_type) -> tuple[int, int, int]:
     """Detect the shape of the grid.
 
     Returns a tuple of (nlat, nlon, ncells). For an unstructured grid nlat and nlon are not defined
@@ -996,7 +996,7 @@ def detect_gridtype(ds, lon, lat, lon_bnds=None, lat_bnds=None):
                 and all([ds[bnds].ndim == 2 for bnds in [lon_bnds, lat_bnds]])
                 and all(
                     [
-                        ds.dims[dim] > 2
+                        ds.sizes[dim] > 2
                         for dim in [
                             ds[lon_bnds].dims[-1],
                             ds[lat_bnds].dims[-1],
@@ -1008,7 +1008,7 @@ def detect_gridtype(ds, lon, lat, lon_bnds=None, lat_bnds=None):
             # rectilinear: bounds [nlat/nlon, 2]
             elif (
                 all([ds[bnds].ndim == 2 for bnds in [lon_bnds, lat_bnds]])
-                and ds.dims[ds.cf.get_bounds_dim_name(lon)] == 2
+                and ds.sizes[ds.cf.get_bounds_dim_name(lon)] == 2
             ):
                 return "regular_lat_lon"
             else: