ci: add codspeed benchmarks (#152)

* ci: add codspeed

* ci: add action

* fix bench

* test: use marker

* ci: change name

* add more files

* fix typing

* add dask bench

* put benchmarks in main script

* move benchmarks out again, add more deps

* remove legacy file

* add to readme

* fix legacy

.github/workflows/ci.yml

@@ -69,6 +69,34 @@ jobs:
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
 
+  benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - uses: actions/cache@v3
+        id: cache
+        with:
+          path: tests/data
+          key: ${{ hashFiles('scripts/download_samples.py') }}
+
+      - name: Download Samples
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          pip install requests
+          python scripts/download_samples.py
+
+      - name: install
+        run: python -m pip install .[test]
+
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@v1
+        with:
+          run: pytest --codspeed -v --color=yes
+
   deploy:
     needs: test
     runs-on: ubuntu-latest

README.md

@@ -5,6 +5,7 @@
 [![Python Version](https://img.shields.io/pypi/pyversions/nd2.svg?color=green)](https://python.org)
 [![Tests](https://github.com/tlambert03/nd2/actions/workflows/ci.yml/badge.svg)](https://github.com/tlambert03/nd2/actions/workflows/ci.yml)
 [![codecov](https://codecov.io/gh/tlambert03/nd2/branch/main/graph/badge.svg)](https://codecov.io/gh/tlambert03/nd2)
+[![Benchmarks](https://img.shields.io/badge/⏱-codspeed-%23FF7B53)](https://codspeed.io/tlambert03/nd2)
 
 `.nd2` (Nikon NIS Elements) file reader.
 

pyproject.toml

@@ -26,14 +26,15 @@ dependencies = ["resource-backed-dask-array", "typing-extensions", "numpy"]
 [project.optional-dependencies]
 legacy = ["imagecodecs"]
 test = [
-    "lxml; python_version<'3.11'",
-    "aicsimageio; python_version<'3.11'",
+    "lxml",
+    "aicsimageio",
     "dask[array]",
-    "imagecodecs; platform_system=='Windows' or python_version<'3.11'",
+    "imagecodecs",
     "psutil",
     "pytest-cov",
     "pytest>=6.0",
     "pytest-pretty",
+    "pytest-codspeed",
     "xarray",
 ]
 dev = [
@@ -52,11 +53,7 @@ dev = [
     "xarray",
     "lxml-stubs",
 ]
-docs = [
-    "mkdocs",
-    "mkdocs-material",
-    "mkdocstrings-python",
-]
+docs = ["mkdocs", "mkdocs-material", "mkdocstrings-python"]
 
 [project.urls]
 homepage = "https://github.com/tlambert03/nd2"

src/nd2/nd2file.py

@@ -1268,9 +1268,9 @@ def binary_data(self) -> BinaryLayers | None:
 def imread(
     file: Path | str,
     *,
-    dask: Literal[False],
-    xarray: Literal[False],
-    validate_frames: bool = False,
+    dask: Literal[False] = ...,
+    xarray: Literal[False] = ...,
+    validate_frames: bool = ...,
     read_using_sdk: bool | None = None,
 ) -> np.ndarray:
     ...
@@ -1282,7 +1282,7 @@ def imread(
     *,
     dask: bool = ...,
     xarray: Literal[True],
-    validate_frames: bool = False,
+    validate_frames: bool = ...,
     read_using_sdk: bool | None = None,
 ) -> xr.DataArray:
     ...
@@ -1293,8 +1293,8 @@ def imread(
     file: Path | str,
     *,
     dask: Literal[True],
-    xarray: Literal[False],
-    validate_frames: bool = False,
+    xarray: Literal[False] = ...,
+    validate_frames: bool = ...,
     read_using_sdk: bool | None = None,
 ) -> dask.array.core.Array:
     ...

tests/test_codspeed.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import nd2
+import pytest
+
+if all(x not in {"--codspeed", "tests/test_codspeed.py"} for x in sys.argv):
+    pytest.skip("use --codspeed to run benchmarks", allow_module_level=True)
+
+# fmt: off
+DATA = Path(__file__).parent / "data"
+FILES: list[Path] = [
+    DATA / "jonas_control002.nd2",          # v2.1 -  48.05 MB
+    DATA / "dims_p2z5t3-2c4y32x32.nd2",     # v3.0 -   1.10 MB
+    DATA / "train_TR67_Inj7_fr50.nd2",      # v3.0 -  69.55 MB
+    DATA / "karl_sample_image.nd2",         # v3.0 - 218.65 MB
+]
+# super slow on codspeed...
+LEGACY = DATA / "aryeh_but3_cont200-1.nd2"  # v1.0 -  16.14 MB
+# fmt: on
+
+
+@pytest.mark.benchmark
+@pytest.mark.parametrize("file", FILES, ids=lambda x: x.stem)
+def test_time_imread(file: Path) -> None:
+    """Test time to read a file."""
+    _ = nd2.imread(file)
+
+
+@pytest.mark.benchmark
+@pytest.mark.parametrize("file", FILES, ids=lambda x: x.stem)
+def test_time_imread_dask(file: Path) -> None:
+    """Test time to read a file."""
+    _ = nd2.imread(file, dask=True).compute()
+
+
+@pytest.mark.benchmark
+@pytest.mark.parametrize("file", [*FILES, LEGACY], ids=lambda x: x.stem)
+def test_time_all_metadata(file: Path) -> None:
+    """Test time to read all metadata."""
+    with nd2.ND2File(file) as nd:
+        _ = nd.metadata
+        _ = nd.frame_metadata(0)
+        _ = nd.attributes
+        _ = nd.experiment
+        _ = nd.text_info