Skip to content

Commit 518b440

Browse files
author
Denis Jelovina
committed
pyalp: squash all local changes since github/358-python-api into single commit
Consolidated pybind11 bindings and removed legacy duplicate source files; added a single shared binding implementation under pyalp. Packaging moved to top-level CMake with deterministic prebuilt discovery in setup.py and support for CMAKE_BUILD_DIR so wheels are built from the CMake artifacts (avoids needing pybind11 in isolated PEP517 build environment). Enabled module-local pybind11 registrations by default to allow importing multiple backend extension modules into the same interpreter safely; made it configurable via CMake option for cases that need cross-module sharing. Fixed an iterator instantiation issue in matrix_wrappers.hpp by iterating directly over M.cbegin()/M.cend(). Added an in-process smoke test test_bckds_inprocess.py to validate multiple-backend imports and object creation, and updated CI to install test wheels from TestPyPI with robust retry-and-verify logic. Updated developer docs and CI workflows to reflect the new packaging, testing, and build flows.
1 parent ccba46d commit 518b440

21 files changed

+780
-477
lines changed

‎.github/workflows/publish-to-testpypi.yml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -323,3 +323,65 @@ jobs:
323323
env:
324324
TWINE_USERNAME: __token__
325325
TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
326+
327+
verify-installed-inprocess:
328+
needs: publish
329+
runs-on: ubuntu-latest
330+
name: Verify installed wheel (in-process smoke)
331+
steps:
332+
- name: Checkout repository (for tests)
333+
uses: actions/checkout@v4
334+
with:
335+
fetch-depth: 0
336+
337+
- name: Set up Python
338+
uses: actions/setup-python@v5
339+
with:
340+
python-version: '3.11'
341+
342+
- name: Create venv and install prerequisites
343+
shell: bash
344+
run: |
345+
set -euo pipefail
346+
PY=$(which python3 || which python)
347+
VENV_DIR="./.venv_test_inprocess"
348+
rm -rf "${VENV_DIR}"
349+
${PY} -m venv "${VENV_DIR}"
350+
source "${VENV_DIR}/bin/activate"
351+
python -m pip install --upgrade pip setuptools wheel numpy
352+
353+
# Retry pip install from TestPyPI with exponential backoff (bounded attempts)
354+
PYALP_VERSION=$(grep -E '^version\s*=\s*"' pyalp/pyproject.toml | head -n1 | sed -E 's/^version\s*=\s*"([^"]+)".*/\1/')
355+
echo "Installing alp-graphblas==${PYALP_VERSION} from TestPyPI (with retries)"
356+
357+
MAX_ATTEMPTS=6
358+
SLEEP_BASE=10
359+
SUCCESS=0
360+
361+
for attempt in $(seq 1 ${MAX_ATTEMPTS}); do
362+
echo "--- attempt ${attempt} of ${MAX_ATTEMPTS} ---"
363+
# verbose pip output helps debugging in CI logs
364+
python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple alp-graphblas==${PYALP_VERSION} -v && SUCCESS=1 && break
365+
echo "pip install failed on attempt ${attempt}"
366+
if [ "${attempt}" -lt "${MAX_ATTEMPTS}" ]; then
367+
SLEEP_SECONDS=$((SLEEP_BASE * attempt))
368+
echo "Sleeping ${SLEEP_SECONDS}s before retry..."
369+
sleep "${SLEEP_SECONDS}"
370+
fi
371+
done
372+
373+
if [ "${SUCCESS}" -ne 1 ]; then
374+
echo "ERROR: failed to install alp-graphblas from TestPyPI after ${MAX_ATTEMPTS} attempts" >&2
375+
exit 1
376+
fi
377+
378+
# Print a compact JSON summary of installed backends for easy scanning in CI logs
379+
python -c "import json,importlib,sys; print(json.dumps({'backends': importlib.import_module('pyalp').list_backends()}))"
380+
381+
- name: Run in-process backend import smoke test
382+
shell: bash
383+
run: |
384+
set -euo pipefail
385+
source ./.venv_test_inprocess/bin/activate
386+
echo "Running pyalp/tests/test_bckds_inprocess.py"
387+
python pyalp/tests/test_bckds_inprocess.py

‎.github/workflows/pyalp-ci.yml‎

Lines changed: 25 additions & 134 deletions
Original file line numberDiff line numberDiff line change
@@ -1,169 +1,60 @@
1-
name: pyalp CI
1+
name: pyalp CI (local-build smoke test)
22

3-
# Run only on pushes that create tags starting with 'pyalp'
43
on:
54
push:
65
tags: [ 'pyalp*' ]
6+
workflow_dispatch: {}
77

88
jobs:
9-
build-bindings:
10-
name: Build C++ bindings
9+
build-and-test-local:
10+
name: Build pyalp with LOCAL profile and run smoke tests
1111
runs-on: ubuntu-latest
1212
steps:
1313
- name: Checkout (with submodules)
1414
uses: actions/checkout@v4
1515
with:
16-
submodules: 'recursive'
16+
submodules: recursive
1717
fetch-depth: 0
1818

19-
- name: Verify pinned pybind11 submodule commit
20-
# Fail early if the checked-out pybind11 is not the pinned commit
21-
run: |
22-
set -euo pipefail
23-
# Prefer top-level pinned file so it survives moves; fallback to submodule path
24-
if [ -f pyalp/PINNED_PYBIND11 ];
25-
then
26-
PINNED_SHA=$(cat pyalp/PINNED_PYBIND11 | tr -d '\n')
27-
elif [ -f pyalp/extern/pybind11/PINNED_COMMIT ];
28-
then
29-
PINNED_SHA=$(cat pyalp/extern/pybind11/PINNED_COMMIT | tr -d '\n')
30-
else
31-
echo "No pinned commit file found (tried pyalp/PINNED_PYBIND11 and pyalp/extern/pybind11/PINNED_COMMIT)" >&2
32-
exit 2
33-
fi
34-
echo "Expected pybind11 commit: $PINNED_SHA"
35-
ACTUAL=$(git -C pyalp/extern/pybind11 rev-parse HEAD || true)
36-
echo "Found pybind11 commit: $ACTUAL"
37-
if [ "$ACTUAL" != "$PINNED_SHA" ];
38-
then
39-
echo "ERROR: pybind11 submodule commit does not match pinned commit" >&2
40-
exit 2
41-
fi
19+
- name: Set up Python
20+
uses: actions/setup-python@v5
21+
with:
22+
python-version: '3.11'
4223

43-
- name: Install build dependencies
24+
- name: Install system build deps
4425
run: |
45-
set -euo pipefail
4626
sudo apt-get update
47-
# libnuma-dev provides NUMA headers/libraries needed by FindNuma.cmake
4827
sudo apt-get install -y build-essential cmake ninja-build pkg-config python3-venv python3-dev python3-pip libnuma-dev
4928
50-
- name: Configure and build pyalp bindings
51-
run: |
52-
set -euo pipefail
53-
mkdir -p build_alp
54-
cmake -S . -B build_alp -DENABLE_PYALP=ON
55-
# Only attempt to build pyalp targets if the pyalp CMake directory exists
56-
if [ -f pyalp/CMakeLists.txt ];
57-
then
58-
echo "pyalp CMakeLists found — building pyalp targets"
59-
cmake --build build_alp --target pyalp_ref -- -j || true
60-
cmake --build build_alp --target pyalp_omp -- -j || true
61-
else
62-
echo "pyalp directory or CMakeLists not present — skipping pyalp targets"
63-
fi
64-
65-
- name: Find and list built shared objects
66-
run: |
67-
set -euo pipefail
68-
echo "Searching for shared objects under build_alp and pyalp"
69-
find build_alp -name "*.so" -maxdepth 8 -print || true
70-
find pyalp -name "*.so" -maxdepth 8 -print || true
71-
72-
- name: Collect built shared objects into artifacts/
29+
- name: Configure top-level CMake with LOCAL profile
7330
run: |
7431
set -euo pipefail
75-
mkdir -p artifacts
76-
# copy any discovered .so files into a flat artifacts directory so upload-artifact can find them
77-
find build_alp -name "*.so" -print0 | xargs -0 -I{} bash -lc 'cp -v "{}" artifacts/ || true' || true
78-
find pyalp -name "*.so" -print0 | xargs -0 -I{} bash -lc 'cp -v "{}" artifacts/ || true' || true
79-
echo "Artifacts now contains:" && ls -la artifacts || true
80-
81-
- name: Upload built bindings
82-
uses: actions/upload-artifact@v4
83-
with:
84-
name: pyalp-so
85-
path: |
86-
build_alp/**/*.so
87-
artifacts/**/*.so
88-
pyalp/**/pyalp*.so
89-
pyalp/**/_pyalp*.so
90-
pyalp/**/libpyalp*.so
91-
pyalp/**/*.so
92-
93-
build-wheel-and-test:
94-
name: Build wheel from prebuilt .so and smoke-test
95-
runs-on: ubuntu-latest
96-
needs: build-bindings
97-
steps:
98-
- name: Checkout
99-
uses: actions/checkout@v4
100-
101-
- name: Download built bindings
102-
uses: actions/download-artifact@v4
103-
with:
104-
name: pyalp-so
105-
path: artifacts
106-
107-
- name: Show downloaded artifacts
108-
run: ls -la artifacts || true
32+
# Configure from repository root using the LOCAL profile to enable native optimizations
33+
cmake -S . -B build/ci_local -G Ninja \
34+
-DALP_BUILD_PROFILE=LOCAL \
35+
-DENABLE_PYALP=ON \
36+
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
37+
-DPython3_EXECUTABLE=$(which python3)
10938
110-
- name: Prepare wheel inputs
111-
id: prep
39+
- name: Build pyalp backends
11240
run: |
11341
set -euo pipefail
114-
# List candidate shared-object files for debugging
115-
echo "Candidate .so files in artifacts:" && find artifacts -type f -name "*.so" -print || true
116-
# Find likely candidates (prefer _pyalp, pyalp, libpyalp)
117-
SO_PATH=$(find artifacts \( -name "_pyalp*.so" -o -name "pyalp*.so" -o -name "libpyalp*.so" -o -name "*.so" \) | head -n1)
118-
if [ -z "$SO_PATH" ];
119-
then
120-
echo "ERROR: no built .so artifact found to package" >&2
121-
echo "Artifacts listing:" && ls -la artifacts || true
122-
exit 2
123-
fi
124-
echo "so_path=$SO_PATH" >> "$GITHUB_OUTPUT"
125-
# Prefer helper located inside pyalp/ but fall back to top-level tools/
126-
if [ -f pyalp/tools/make_wheel_from_so.py ]; then
127-
echo "builder=pyalp/tools/make_wheel_from_so.py" >> "$GITHUB_OUTPUT"
128-
else
129-
echo "builder=tools/make_wheel_from_so.py" >> "$GITHUB_OUTPUT"
130-
fi
131-
# Derive Python version from the .so filename (e.g., cpython-311 -> 3.11, cp312 -> 3.12)
132-
PY_VER=""
133-
if [[ "$SO_PATH" =~ cpython-([0-9]{3}) ]];
134-
then
135-
n=${BASH_REMATCH[1]}
136-
PY_VER="${n:0:1}.${n:1}"
137-
elif [[ "$SO_PATH" =~ cp([0-9]{2,3}) ]];
138-
then
139-
n=${BASH_REMATCH[1]}
140-
PY_VER="${n:0:1}.${n:1}"
141-
fi
142-
echo "python_version=$PY_VER" >> "$GITHUB_OUTPUT"
42+
cmake --build build/ci_local --target pyalp_ref pyalp_omp pyalp_nonblocking --parallel
14343
144-
- name: Run wheel builder
44+
- name: Package pyalp wheel from CMake build
14545
run: |
14646
set -euo pipefail
147-
echo "builder=${{ steps.prep.outputs.builder }}"
148-
echo "so=${{ steps.prep.outputs.so_path }}"
149-
python3 "${{ steps.prep.outputs.builder }}" "${{ steps.prep.outputs.so_path }}" --out-dir dist_wheel
150-
151-
- name: Show wheel
152-
run: ls -la dist_wheel || true
153-
154-
- name: Set up Python matching built extension
155-
if: ${{ steps.prep.outputs.python_version != '' }}
156-
uses: actions/setup-python@v5
157-
with:
158-
python-version: ${{ steps.prep.outputs.python_version }}
47+
mkdir -p dist_wheel
48+
export CMAKE_BUILD_DIR=$(pwd)/build/ci_local
49+
( cd pyalp && python -m pip wheel . -w ../dist_wheel )
15950
16051
- name: Smoke test wheel in venv
16152
run: |
16253
set -euo pipefail
16354
python3 -V
164-
which python3
16555
python3 -m venv venv
16656
. venv/bin/activate
16757
pip install --upgrade pip wheel
16858
pip install dist_wheel/*.whl
169-
tools/smoke_test_pyalp.py
59+
# run the smoke test script which should import pyalp and backends e.g. pyalp.pyalp_ref
60+
python tools/smoke_test_pyalp.py

‎CMakeLists.txt‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -382,6 +382,93 @@ if( ENABLE_PYALP )
382382
message(STATUS "pyalp subdirectory not present in source tree; skipping add_subdirectory(pyalp)")
383383
endif()
384384
endif()
385+
386+
# Provide a top-level convenience packaging target for pyalp so callers can run
387+
# cmake --build <build-dir> --target pyalp --parallel
388+
# even if the pyalp CMakeLists placed a packaging target in a subdirectory or
389+
# the generator didn't expose that target at the top-level. This mirrors the
390+
# packaging flow implemented under pyalp/src/CMakeLists.txt and is only added
391+
# when pyalp is enabled and present in source.
392+
if( ENABLE_PYALP AND EXISTS "${PROJECT_SOURCE_DIR}/pyalp/CMakeLists.txt" )
393+
# Attempt to find a Python interpreter (non-fatal if already found elsewhere)
394+
find_package(PythonInterp QUIET)
395+
396+
# Build the list of backend targets that should be packaged. Keep this in
397+
# sync with pyalp/src/CMakeLists.txt.
398+
set(pyalp_package_targets "")
399+
if(WITH_REFERENCE_BACKEND)
400+
list(APPEND pyalp_package_targets pyalp_ref)
401+
endif()
402+
if(WITH_OMP_BACKEND)
403+
list(APPEND pyalp_package_targets pyalp_omp)
404+
endif()
405+
if(WITH_NONBLOCKING_BACKEND)
406+
list(APPEND pyalp_package_targets pyalp_nonblocking)
407+
endif()
408+
string(JOIN " " pyalp_package_targets_str ${pyalp_package_targets})
409+
410+
# Only add the top-level pyalp target if one is not already defined.
411+
if(NOT TARGET pyalp)
412+
add_custom_target(pyalp
413+
COMMENT "Build enabled pyalp backends and package wheel(s) into ${CMAKE_BINARY_DIR}/dist"
414+
)
415+
416+
add_custom_command(TARGET pyalp
417+
# Build each backend target individually (cmake --build --target accepts one target at a time)
418+
VERBATIM
419+
)
420+
# Add per-backend build commands so each target is invoked separately.
421+
foreach(_pyalp_backend IN LISTS pyalp_package_targets)
422+
add_custom_command(TARGET pyalp
423+
COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target ${_pyalp_backend} --parallel
424+
VERBATIM
425+
)
426+
endforeach()
427+
add_custom_command(TARGET pyalp
428+
COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_BINARY_DIR}/dist
429+
COMMAND ${CMAKE_COMMAND} -E env CMAKE_BUILD_DIR=${CMAKE_BINARY_DIR} ${PYTHON_EXECUTABLE} -m pip wheel ${CMAKE_SOURCE_DIR}/pyalp -w ${CMAKE_BINARY_DIR}/dist
430+
COMMAND ${CMAKE_COMMAND} -E echo ""
431+
COMMAND ${CMAKE_COMMAND} -E echo "============================================================"
432+
COMMAND ${CMAKE_COMMAND} -E echo "Packaged wheel(s) into: ${CMAKE_BINARY_DIR}/dist"
433+
COMMAND ${CMAKE_COMMAND} -E echo "To install the wheel(s):"
434+
COMMAND ${CMAKE_COMMAND} -E echo " python -m pip install ${CMAKE_BINARY_DIR}/dist/<wheel-file>.whl"
435+
COMMAND ${CMAKE_COMMAND} -E echo "or install all wheels in dist:"
436+
COMMAND ${CMAKE_COMMAND} -E echo " python -m pip install ${CMAKE_BINARY_DIR}/dist/*.whl"
437+
COMMAND ${CMAKE_COMMAND} -E echo "After installation, import the package in Python, e.g.:"
438+
COMMAND ${CMAKE_COMMAND} -E echo " python -c \"import alp_graphblas; print(alp_graphblas.__version__)\""
439+
VERBATIM
440+
)
441+
endif()
442+
endif()
443+
## Also expose a clearly-named packaging target that avoids name collisions
444+
if( ENABLE_PYALP AND EXISTS "${PROJECT_SOURCE_DIR}/pyalp/CMakeLists.txt" )
445+
if(NOT TARGET pyalp-package)
446+
add_custom_target(pyalp-package
447+
COMMENT "(convenience) Build enabled pyalp backends and package wheel(s) into ${CMAKE_BINARY_DIR}/dist"
448+
)
449+
450+
foreach(_pyalp_backend IN LISTS pyalp_package_targets)
451+
add_custom_command(TARGET pyalp-package
452+
COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target ${_pyalp_backend} --parallel
453+
VERBATIM
454+
)
455+
endforeach()
456+
add_custom_command(TARGET pyalp-package
457+
COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_BINARY_DIR}/dist
458+
COMMAND ${CMAKE_COMMAND} -E env CMAKE_BUILD_DIR=${CMAKE_BINARY_DIR} ${PYTHON_EXECUTABLE} -m pip wheel ${CMAKE_SOURCE_DIR}/pyalp -w ${CMAKE_BINARY_DIR}/dist
459+
COMMAND ${CMAKE_COMMAND} -E echo ""
460+
COMMAND ${CMAKE_COMMAND} -E echo "============================================================"
461+
COMMAND ${CMAKE_COMMAND} -E echo "Packaged wheel(s) into: ${CMAKE_BINARY_DIR}/dist"
462+
COMMAND ${CMAKE_COMMAND} -E echo "To install the wheel(s):"
463+
COMMAND ${CMAKE_COMMAND} -E echo " python -m pip install ${CMAKE_BINARY_DIR}/dist/<wheel-file>.whl"
464+
COMMAND ${CMAKE_COMMAND} -E echo "or install all wheels in dist:"
465+
COMMAND ${CMAKE_COMMAND} -E echo " python -m pip install ${CMAKE_BINARY_DIR}/dist/*.whl"
466+
COMMAND ${CMAKE_COMMAND} -E echo "After installation, import the package in Python, e.g.:"
467+
COMMAND ${CMAKE_COMMAND} -E echo " python -c \"import alp_graphblas; print(alp_graphblas.__version__)\""
468+
VERBATIM
469+
)
470+
endif()
471+
endif()
385472
add_subdirectory( examples )
386473

387474

‎cmake/CompileFlags.cmake‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,25 @@ set( COMMON_PERF_DEFS_Release "NDEBUG" )
9898
# building wheels in CI set -DALP_PORTABLE_BUILD=ON to get portable artifacts.
9999
option( ALP_PORTABLE_BUILD "Build portable binaries (disable host-specific optimizations)" OFF )
100100

101+
# Build profile: controls portability and default LTO/optimization choices.
102+
# Use -DALP_BUILD_PROFILE=LOCAL for developer/local builds (enables native
103+
# host optimizations, enables LTO by default). Use -DALP_BUILD_PROFILE=DEPLOYMENT
104+
# for wheel/deployment builds (portable by default).
105+
set(ALP_BUILD_PROFILE "DEPLOYMENT" CACHE STRING "Build profile: LOCAL or DEPLOYMENT. LOCAL enables native optimizations; DEPLOYMENT favors portability for wheels.")
106+
string(TOUPPER "${ALP_BUILD_PROFILE}" ALP_BUILD_PROFILE_UP)
107+
108+
if(ALP_BUILD_PROFILE_UP STREQUAL "LOCAL")
109+
# Local builds should prefer host-specific optimizations
110+
set(ALP_PORTABLE_BUILD OFF CACHE BOOL "Build portable binaries (disable host-specific optimizations)" FORCE)
111+
# Enable LTO by default for local performance builds; user may override.
112+
set(CMAKE_INTERPROCEDURAL_OPTIMIZATION ON CACHE BOOL "Enable LTO (interprocedural optimization)" FORCE)
113+
else()
114+
# Deployment builds default to portable flags for maximum wheel compatibility
115+
set(ALP_PORTABLE_BUILD ON CACHE BOOL "Build portable binaries (disable host-specific optimizations)" FORCE)
116+
# Disable LTO for portable deployment builds; user may override explicitly
117+
set(CMAKE_INTERPROCEDURAL_OPTIMIZATION OFF CACHE BOOL "Enable LTO (interprocedural optimization)" FORCE)
118+
endif()
119+
101120
# Avoid GCC/GNU-specific microarchitecture flags on Apple/Clang toolchains
102121
if(APPLE)
103122
# On macOS with AppleClang, -march/-mtune and aggressive unrolling can

0 commit comments

Comments
 (0)