Refactor cibuildwheel setup (#276)

.github/workflows/build.yml

@@ -131,34 +131,20 @@ jobs:
   wheels:
     name: Build wheel on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
-    env:
-      CIBW_SKIP: "pp* *-win32 cp312-*"  # remove once numpy supports Python 3.12
-      CIBW_TEST_REQUIRES: pytest numpy
-      CIBW_TEST_COMMAND: "pytest -v {project}/tests"
-      # we are copying the shared libraries ourselves so skip magical copy
-      CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""
-      MACOSX_DEPLOYMENT_TARGET: 10.9
-      CIBW_BUILD_VERBOSITY_MACOS: 3
-      CIBW_TEST_SKIP: "*-macosx_arm64 *-macosx_universal2:arm64 *-musllinux* *i686"
-      CIBW_ARCHS_MACOS: "x86_64 arm64"
-      CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: ""
-      CIBW_BEFORE_ALL_LINUX: "pip install cmake; bash {project}/ci/install_libspatialindex.bash"
     strategy:
       matrix:
         os: [windows-latest, ubuntu-latest, macos-latest]
     steps:
     - uses: actions/checkout@v3
+    - name: Set up QEMU
+      if: runner.os == 'Linux'
+      uses: docker/setup-qemu-action@v2
+      with:
+        platforms: arm64
     - uses: actions/setup-python@v4
       name: Install Python
       with:
         python-version: '3.11'
-    - name: Run MacOS Preinstall Build
-      if: startsWith(matrix.os, 'macos')
-      run: |
-        # provides sha256sum
-        brew install coreutils
-        pip install cmake
-        bash ci/install_libspatialindex.bash
     - uses: ilammy/msvc-dev-cmd@v1
       if: startsWith(matrix.os, 'windows')
     - name: Run Windows Preinstall Build
@@ -173,39 +159,11 @@ jobs:
         name: ${{ matrix.os }}-whl
         path: wheelhouse/*.whl
 
-  wheels_aarch64:
-    name: Build Linux wheels on aarch64
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
-      name: Install Python
-      with:
-        python-version: '3.11'
-    - uses: docker/setup-qemu-action@v2
-      name: Set up QEMU
-      with:
-        platforms: arm64
-    - name: Build wheels
-      uses: pypa/cibuildwheel@v2.15.0
-      env:
-        CIBW_SKIP: pp*
-        CIBW_ARCHS_LINUX: aarch64
-        CIBW_TEST_REQUIRES: pytest numpy
-        CIBW_TEST_COMMAND: "pytest -v {project}/tests"
-        CIBW_TEST_SKIP: "*-musllinux*"
-        CIBW_BEFORE_ALL_LINUX: "pip install cmake; bash {project}/ci/install_libspatialindex.bash"
-    - uses: actions/upload-artifact@v3
-      with:
-        name: aarch64-whl
-        path: wheelhouse/*.whl
-
   collect-artifacts:
     name: Package and push release
 
     #needs: [windows-wheel, linux-wheel, osx-wheel, conda, ubuntu]
-    needs: [conda, ubuntu, wheels, wheels_aarch64]
+    needs: [conda, ubuntu, wheels]
 
     runs-on: 'ubuntu-latest'
     strategy:

.gitignore

@@ -8,3 +8,5 @@ dist/
 include
 lib
 .coverage
+.tox
+wheelhouse

ci/install_libspatialindex.bash

@@ -5,7 +5,6 @@ set -xe
 VERSION=1.9.3
 SHA256=63a03bfb26aa65cf0159f925f6c3491b6ef79bc0e3db5a631d96772d6541187e
 
-
 # where to copy resulting files
 # this has to be run before `cd`-ing anywhere
 gentarget() {
@@ -37,15 +36,21 @@ curl -L -O https://github.com/libspatialindex/libspatialindex/archive/$VERSION.z
 echo "${SHA256}  ${VERSION}.zip" | sha256sum -c -
 
 rm -rf "libspatialindex-${VERSION}" || true
-unzip $VERSION
+unzip -q $VERSION
 cd libspatialindex-${VERSION}
 
 mkdir build
 cd build
 
 cp "${SL}/CMakeLists.txt" ..
 
-cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_OSX_ARCHITECTURES="${CIBW_ARCHS_MACOS/ /;}" ..
+printenv
+
+if [ "$(uname)" == "Darwin" ]; then
+    CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES=${ARCHFLAGS##* }"
+fi
+
+cmake -DCMAKE_BUILD_TYPE=Release ${CMAKE_ARGS} ..
 make -j 4
 
 # copy built libraries relative to path of this script

pyproject.toml

@@ -50,6 +50,40 @@ target-version = ["py38", "py39", "py310", "py311"]
 color = true
 skip_magic_trailing_comma = true
 
+[tool.cibuildwheel]
+build = "cp38-*"
+build-verbosity = "3"
+repair-wheel-command = "python scripts/repair_wheel.py -w {dest_dir} {wheel}"
+test-requires = "tox"
+test-command = "tox --conf {project} --installpkg {wheel}"
+
+[tool.cibuildwheel.linux]
+archs = ["auto", "aarch64"]
+test-skip = "*aarch64"  # slow!
+before-build = [
+    "yum install -y cmake libffi-devel",
+    "bash {project}/ci/install_libspatialindex.bash",
+]
+
+[[tool.cibuildwheel.overrides]]
+select = "*-musllinux*"
+before-build = [
+    "apk add cmake libffi-dev",
+    "bash {project}/ci/install_libspatialindex.bash",
+]
+
+[tool.cibuildwheel.macos]
+archs = ["x86_64", "arm64"]
+test-skip = "*-macosx_arm64"
+environment = { MACOSX_DEPLOYMENT_TARGET="10.9" }
+before-build = [
+    "brew install coreutils cmake",
+    "bash {project}/ci/install_libspatialindex.bash",
+]
+
+[tool.cibuildwheel.windows]
+archs = ["AMD64"]
+
 [tool.coverage.report]
 # Ignore warnings for overloads
 # https://github.com/nedbat/coveragepy/issues/970#issuecomment-612602180

rtree/finder.py

@@ -73,9 +73,25 @@ def load() -> ctypes.CDLL:
             # macos shared libraries are `.dylib`
             lib_name = "libspatialindex_c.dylib"
         else:
+            import importlib.metadata
+
             # linux shared libraries are `.so`
             lib_name = "libspatialindex_c.so"
 
+            # add path for binary wheel prepared with cibuildwheel/auditwheel
+            try:
+                pkg_files = importlib.metadata.files("rtree")
+                for file in pkg_files:  # type: ignore
+                    if (
+                        file.parent.name == "Rtree.libs"
+                        and file.stem.startswith("libspatialindex")
+                        and ".so" in file.suffixes
+                    ):
+                        _candidates.insert(1, os.path.join(str(file.locate())))
+                        break
+            except importlib.metadata.PackageNotFoundError:
+                pass
+
         # get the starting working directory
         cwd = os.getcwd()
         for cand in _candidates:

scripts/repair_wheel.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+
+import argparse
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+
+def main():
+    if sys.platform.startswith("linux"):
+        os_ = "linux"
+    elif sys.platform.startswith("darwin"):
+        os_ = "macos"
+    elif sys.platform.startswith("win32"):
+        os_ = "windows"
+    else:
+        raise NotImplementedError(
+            "sys.platform '{}' is not supported yet.".format(sys.platform)
+        )
+
+    p = argparse.ArgumentParser(
+        description="Convert wheel to be independent of python implementation and ABI"
+    )
+    p.set_defaults(prog=Path(sys.argv[0]).name)
+    p.add_argument("WHEEL_FILE", help="Path to wheel file.")
+    p.add_argument(
+        "-w",
+        "--wheel-dir",
+        dest="WHEEL_DIR",
+        help=('Directory to store delocated wheels (default: "wheelhouse/")'),
+        default="wheelhouse/",
+    )
+
+    args = p.parse_args()
+
+    file = Path(args.WHEEL_FILE).resolve(strict=True)
+    wheelhouse = Path(args.WHEEL_DIR).resolve()
+    wheelhouse.mkdir(parents=True, exist_ok=True)
+
+    with tempfile.TemporaryDirectory() as tmpdir_:
+        tmpdir = Path(tmpdir_)
+        # use the platform specific repair tool first
+        if os_ == "linux":
+            subprocess.run(
+                ["auditwheel", "repair", "-w", str(tmpdir), str(file)], check=True
+            )
+        elif os_ == "macos":
+            subprocess.run(
+                [
+                    "delocate-wheel",
+                    # "--require-archs",
+                    # "arm64,x86_64",
+                    "-w",
+                    str(tmpdir),
+                    str(file),
+                ],
+                check=True,
+            )
+        elif os_ == "windows":
+            # no specific tool, just copy
+            shutil.copyfile(file, tmpdir / file.name)
+        (file,) = tmpdir.glob("*.whl")
+
+        # make this a py3 wheel
+        subprocess.run(
+            [
+                "wheel",
+                "tags",
+                "--python-tag",
+                "py3",
+                "--abi-tag",
+                "none",
+                "--remove",
+                str(file),
+            ],
+            check=True,
+        )
+        (file,) = tmpdir.glob("*.whl")
+        # unpack
+        subprocess.run(["wheel", "unpack", file.name], cwd=tmpdir, check=True)
+        for unpackdir in tmpdir.iterdir():
+            if unpackdir.is_dir():
+                break
+        else:
+            raise RuntimeError("subdirectory not found")
+        if os_ == "linux":
+            # remove duplicated dir
+            assert len(list((unpackdir / "Rtree.libs").glob("*.so*"))) >= 1
+            lib_dir = unpackdir / "rtree" / "lib"
+            shutil.rmtree(lib_dir)
+        # re-pack
+        subprocess.run(["wheel", "pack", str(unpackdir.name)], cwd=tmpdir, check=True)
+        files = list(tmpdir.glob("*.whl"))
+        assert len(files) == 1, files
+        file = files[0]
+        file.rename(wheelhouse / file.name)
+
+
+if __name__ == "__main__":
+    main()

tox.ini

@@ -0,0 +1,16 @@
+[tox]
+requires =
+    tox>=4
+env_list = py{38,39,310,311,312}
+
+[testenv]
+description = run unit tests
+deps =
+    pytest
+    numpy
+install_command =
+    python -I -m pip install --only-binary=:all: {opts} {packages}
+ignore_errors = True
+ignore_outcome = True
+commands =
+    pytest {posargs:tests}