Helper microbench

.github/workflows/ci.yml

@@ -412,16 +412,32 @@ jobs:
 
       - name: Install / Upgrade Python requirements
         run: |
-          python -m pip install --upgrade pip wheel 'setuptools>=60.2'
-          python -m pip install pytest cffi
+          pip install --upgrade pip wheel 'setuptools>=60.2' setuptools-scm
+          pip install pytest cffi
 
       - name: Build and install HPy
         run: |
           make
-          python -m pip install .
+          pip install .
+
+      - name: Build and install hpy.microbench
+        run: |
+          pip install ./microbench --no-build-isolation --no-deps
 
       - name: Run microbenchmarks
         run: |
           cd microbench
-          python setup.py build_ext -i
-          python -m pytest -v
+          pytest -v
+
+      - name: Uninstall hpy.microbench
+        run: |
+          pip uninstall hpy.microbench --yes
+
+      - name: Build and install hpy.microbench universal ABI
+        run: |
+          pip install -e ./microbench --no-build-isolation --no-deps --config-settings="--global-option=--hpy-abi=universal"
+
+      - name: Run microbenchmarks universal ABI
+        run: |
+          cd microbench
+          pytest -v -m hpy

.gitignore

@@ -90,3 +90,7 @@ docs/_build/
 
 # vscode
 .vscode
+
+microbench/pixi.lock
+microbench/.venv_*
+microbench/tmp_*

microbench/Makefile

@@ -0,0 +1,57 @@
+ifeq ($(PYTHON),)
+PYTHON := python3
+endif
+
+install:
+	$(PYTHON) -m pip install .
+
+install_universal:
+	$(PYTHON) -m pip install . --config-settings="--global-option=--hpy-abi=universal"
+
+uninstall:
+	$(PYTHON) -m pip uninstall hpy.microbench --yes
+
+test:
+	$(PYTHON) -m pytest -v | tee tmp_results_$(shell $(PYTHON) -c "import sys; print(sys.implementation.name)").txt
+
+bench: test
+
+bench_hpy:
+	$(PYTHON) -m pytest -v -m hpy | tee tmp_results_$(shell $(PYTHON) -c "import sys; print(sys.implementation.name)").txt
+
+clean:
+	rm -f src/*.so src/hpy_simple.py
+
+cleanall: clean
+	rm -rf .venv_* tmp_*.txt
+
+create_venv_cpy:
+	uv python install 3.12
+	$(shell uv python find 3.12) -m venv .venv_cpy --upgrade-deps
+
+create_venv_pypy:
+	uv python install pypy
+	$(shell uv python find pypy) -m venv .venv_pypy --upgrade-deps
+
+create_venv_graalpy:
+	uv python install graalpy
+	# cannot use --upgrade-deps because pip is patched for GraalPy
+	$(shell uv python find graalpy) -m venv .venv_graalpy
+
+print_cpy:
+	@echo =================================== CPython ====================================
+	@tail tmp_results_cpython.txt -n 29
+
+print_pypy:
+	@echo ==================================== PyPy ======================================
+	@tail tmp_results_pypy.txt -n 29
+
+print_graalpy:
+	@echo =================================== GraalPy ====================================
+	@tail tmp_results_graalpy.txt -n 29
+
+print_pypy_vs_cpy:
+	@$(PYTHON) print_other_vs_cpy.py PyPy
+
+print_graalpy_vs_cpy:
+	@$(PYTHON) print_other_vs_cpy.py GraalPy

microbench/README.md

@@ -1,29 +1,117 @@
-To run the microbenchmarks
---------------------------
+# To run the microbenchmarks
 
-1. You need to have `hpy` installed in your virtuanenv. The easiest way
+## Non-Python dependencies
+
+The benchmarks depends on Valgrind, which can be installed with
+
+```sh
+sudo apt update && sudo apt install -y valgrind
+```
+
+Alternatively, you can also use [Pixi] to get it in a new shell with
+
+```sh
+pixi shell
+```
+
+Similarly, building with GraalPy requires `libffi` to build the Python package `ffi`.
+`pixi shell` provides it.
+
+[UV] can be useful since it is used in few Makefile targets to install Python interpreters.
+
+## Python virtual environments
+
+We assume in the following that a virtual environment is activated. One can create
+environments with the Makefile targets `create_venv_...` as
+
+```sh
+cd /path/to/hpy/microbench
+make create_venv_pypy
+. .venv_pypy/bin/activate
+```
+
+## Non-editable install with build isolation
+
+One can build these microbenchmarks with HPy from PyPI (on CPython) or bundled with the Python implementation.
+
+```sh
+pip install .
+```
+
+This builds the HPy extension with the CPython ABI for CPython and with the universal ABI for other implementations.
+To build this extension with the universal ABI with CPython:
+
+```sh
+pip install . --config-settings="--global-option=--hpy-abi=universal"
+```
+
+## Editable install without build isolation
+
+1. On CPython, you need to have `hpy` installed in your virtualenv. The easiest way
    to do it is:
 
-       $ cd /path/to/hpy
-       $ python setup.py develop
+   ```sh
+   cd /path/to/hpy
+   pip install -e .
+   ```
+
+2. Install build and runtime dependencies
+
+   ```sh
+   # cffi needed to build _valgrind
+   pip install cffi pytest
+   ```
+
+3. Build and install the extension modules needed for the microbenchmarks
+
+   ```sh
+   cd /path/to/hpy/microbench
+   pip install . --no-build-isolation
+   # or for the universal ABI (on with CPython)
+   rm -f src/*.so src/hpy_simple.py
+   pip install -e . --no-build-isolation --config-settings="--global-option=--hpy-abi=universal"
+   ```
+
+## Run the benchmarks
+
+```sh
+pytest -v
+```
+
+To run only cpy or hpy tests, use -m (to select markers):
+
+```sh
+pytest -v -m hpy
+pytest -v -m cpy
+```
+
+## Comparing alternative Python implementations to CPython
 
-2. Build the extension modules needed for the microbenchmarks
+One can run things like
 
-       $ cd /path/to/hpy/microbench
-       $ pip install cffi # needed to build _valgrind
-       $ python setup.py build_ext --inplace
+```sh
+make cleanall
+pixi shell
+make create_venv_cpy
+make create_venv_pypy
+make create_venv_graalpy
 
-2. `py.test -v`
+make install PYTHON=.venv_cpy/bin/python
+make install PYTHON=.venv_pypy/bin/python
+make install PYTHON=.venv_graalpy/bin/python
 
-3. To run only cpy or hpy tests, use -m (to select markers):
+make bench PYTHON=.venv_cpy/bin/python
+make bench PYTHON=.venv_pypy/bin/python
+# only HPy for GraalPy since the full benchmarks are a bit too long
+make bench_hpy PYTHON=.venv_graalpy/bin/python
 
-       $ py.test -v -m hpy
-       $ py.test -v -m cpy
+make print_cpy
+make print_pypy
+make print_graalpy
 
-4. Step (2) build `hpy_simple` using the CPython ABI by default. If you want
-   to benchmark the universal mode, you need to build it explicitly:
+make print_pypy_vs_cpy
+make print_graalpy_vs_cpy
+```
 
-       $ cd /path/to/hpy/microbench
-       $ rm *.so  # make sure to delete CPython-ABI versions
-       $ python setup.py --hpy-abi=universal build_ext --inplace
-       $ py.test -v
+[Pixi]: https://pixi.sh
+[UV]: https://docs.astral.sh/uv/

microbench/pixi.toml

@@ -0,0 +1,13 @@
+[workspace]
+authors = ["The HPy team <hpy-dev@python.org>"]
+channels = ["conda-forge"]
+name = "hpy.microbench"
+platforms = ["linux-64"]
+version = "0.1.0"
+
+[tasks]
+
+[dependencies]
+gcc = ">=15.1.0,<15.2"
+valgrind = ">=3.25.0,<4"
+libffi = ">=3.4.6,<4"

microbench/print_other_vs_cpy.py

@@ -0,0 +1,59 @@
+import sys
+
+from pathlib import Path
+
+try:
+    other = sys.argv[1]
+except IndexError:
+    other = "PyPy"
+
+path_result_cpy = Path("tmp_results_cpython.txt")
+path_result_other = Path(f"tmp_results_{other.lower()}.txt")
+
+assert path_result_cpy.exists()
+assert path_result_other.exists()
+
+
+def data_from_path(path):
+    txt = path.read_text()
+    _, txt = txt.split(
+        "================================== BENCHMARKS =================================="
+    )
+    lines = txt.splitlines()[3:-2]
+
+    if "cpy" in path.name:
+        index_time = 1
+    else:
+        parts = lines[0].split()
+        if len(parts) == 3:
+            index_time = 1
+        else:
+            index_time = 3
+
+    names = []
+    times = []
+
+    for line in lines:
+        parts = line.split()
+        names.append(parts[0])
+        times.append(float(parts[index_time]))
+
+    return names, times
+
+
+names, times_cpy = data_from_path(path_result_cpy)
+names, times_other = data_from_path(path_result_other)
+
+max_length_name = 45
+fmt_name = f"{{:{max_length_name}s}}"
+
+out = f" {other} HPy univ / CPy native (time ratio, smaller is better) "
+num_chars = 81
+num_equals = (num_chars - len(out)) // 2
+
+print("\n" + num_equals * "=" + out + num_equals * "=")
+
+for index, t_other in enumerate(times_other):
+    ratio = t_other / times_cpy[index]
+    name = fmt_name.format(names[index])
+    print(f"{name} {ratio:.2f}")

microbench/pyproject.toml

@@ -0,0 +1,22 @@
+[project]
+name = "hpy.microbench"
+authors = [
+    {name = "The HPy team", email = "hpy-dev@python.org"},
+]
+version = "0.1.0"
+description = "HPy microbenchmarks."
+readme = "README.md"
+keywords = ["hpy"]
+requires-python = ">=3.8"
+dependencies = [
+    "hpy>=0.9.0; implementation_name == 'cpython'",
+    "pytest",
+    "cffi; implementation_name != 'pypy'",
+    ]
+
+[build-system]
+requires = [
+    "setuptools>=64.0",
+    "hpy>=0.9.0; implementation_name == 'cpython'",
+    "cffi",
+]

microbench/setup.py

@@ -1,8 +1,10 @@
 from setuptools import setup, Extension
 
 setup(
-    name="hpy.microbench",
-    setup_requires=['cffi', 'hpy'],
+    # Workaround: HPy adds files to the sources list and uses absolute paths.
+    # Newer setuptools complain about that if package data should be included.
+    # Therefore, we explicitly disable this here.
+    include_package_data=False,
     ext_modules = [
         Extension('cpy_simple',
                   ['src/cpy_simple.c'],