Merge pull request #21 from openscm/cibuildwheel-matplotlib

znicholls · web-flow · commit 92213ce9f862 · 2025-08-27T12:11:07.000+02:00
Add cibuildwheel drawing inspiration from matplotlib and scipy's approach
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -80,6 +80,19 @@ jobs:
         os: [ "ubuntu-latest" ]
         # Test against all security and bugfix versions: https://devguide.python.org/versions/
         python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13" ]
+        include:
+          # Test other OS's for a 'very stable' Python version too
+          - os: "windows-latest"
+            python-version: "3.11"
+          - os: "macos-latest"
+            python-version: "3.11"
+          # Include other OS for latest Python
+          # because these seem to be the flakiest from experience
+          # so are worth the extra testing
+          - os: "windows-latest"
+            python-version: "3.13"
+          - os: "macos-latest"
+            python-version: "3.13"
     runs-on: "${{ matrix.os }}"
     defaults:
       run:
@@ -91,6 +104,15 @@ jobs:
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
+      - name: Install Fortran compiler - MacOS
+        # When building on Mac, we need to have a Fortran compiler
+        if: ${{ startsWith(matrix.os, 'macos') }}
+        uses: fortran-lang/setup-fortran@v1
+        id: setup-fortran-macos
+        with:
+          # TODO: figure out whether we need/want to use other compilers too
+          compiler: "gcc"
+          version: "13"
       - uses: ./.github/actions/setup
         with:
           python-version: ${{ matrix.python-version }}
@@ -100,10 +122,9 @@ jobs:
           # when people try to run without installing optional dependencies,
           # we should add a CI step that runs the tests without optional dependencies too.
           # We don't have that right now, because we're not sure this pain point exists.
-          uv-dependency-install-flags: "--all-extras --group tests"
+          uv-dependency-install-flags: "-v --reinstall-package example-fgen-basic -Ccompile-args='-v' --all-extras --group tests "
       - name: Run tests
         run: |
-
           COV_DIR=`uv run --no-sync python -c 'from pathlib import  Path; import example_fgen_basic; print(Path(example_fgen_basic.__file__).parent)'`
           uv run --no-sync pytest -r a -v tests src --doctest-modules --doctest-report ndiff --cov=${COV_DIR} --cov-report=term-missing --cov-report=xml
           uv run --no-sync coverage report
@@ -222,6 +243,11 @@ jobs:
   check-build:
     strategy:
       matrix:
+        # The tests are also effectively a test of the build (at least the Fortran compilation).
+        # This test is really just about checking what is packaged
+        # and that we have all the required pieces.
+        # Hence, for now, only test on ubuntu
+        # (little value testing on other OS's because this test is not testing compilation directly).
         os: [ "ubuntu-latest" ]
         python-version: [ "3.11" ]
     runs-on: "${{ matrix.os }}"
@@ -248,11 +274,16 @@ jobs:
           cd dist
           ls
           python3 -m venv venv
-          # TODO: alter for windows
           source venv/bin/activate
           pip install example_fgen_basic*
           python -c 'from example_fgen_basic.get_wavelength import get_wavelength_plain; print(get_wavelength_plain(23.4))'
 
+  # check-build-wheel
+  # For now, decide not to do this as odds of breaking the build are low
+  # (and we already implicitly test this when running the tests).
+  # If we find we are breaking the wheel build inadvertently often,
+  # we can always add this step in.
+
   check-dependency-licences:
     strategy:
       matrix:
diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml
@@ -24,19 +24,22 @@ jobs:
       # this permission is mandatory for trusted publishing with PyPI
       id-token: write
     steps:
-      - name: Check out repository
+      - name: Checkout repository
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: ./.github/actions/setup
+      - name: Download artifacts
+        env:
+          GH_TOKEN: "${{ secrets.PERSONAL_ACCESS_TOKEN }}"
+        run: |
+          gh release download --pattern "*.whl" --dir dist
+          gh release download --pattern "*.tar.gz" --dir dist
+          # Check what we have
+          tree dist
+      - name: Setup uv
+        id: setup-uv
+        uses: astral-sh/setup-uv@v4
         with:
-          python-version: ${{ matrix.python-version }}
-          uv-dependency-install-flags: "--all-extras --group dev"
+          version: "0.8.8"
       - name: Publish to PyPI
-        run: |
-          # TODO: move to using cibuildwheel so we have wheels for multiple platforms and python versions
-          # starting docs: https://cibuildwheel.pypa.io/en/stable/
-          uv build --sdist
-          uv publish
-          # Just in case, undo the changes to `pyproject.toml`
-          git restore --staged . && git restore .
+        run: uv publish
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -9,8 +9,104 @@ defaults:
     shell: bash
 
 jobs:
+  build-wheels:
+    name: Build wheels on ${{ matrix.os }} for ${{ matrix.cibw_archs }}
+    runs-on: ${{ matrix.os }}
+    env:
+      # 3.9, 3.10 seem to not be happy on windows arm64
+      # because you can't install numpy from a wheel for the build environment
+      # (and we're not going to try recompiling from source)
+      CIBW_SKIP: "cp39-win_arm64 cp310-win_arm64"
+      # TODO: consider turning this on
+      # CIBW_ENABLE: cpython-freethreading
+      CIBW_BEFORE_BUILD_WINDOWS: >-
+        pip install delvewheel
+      CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: >-
+        delvewheel repair -w {dest_dir} {wheel}
+      CIBW_BEFORE_TEST: >-
+        pip install -r {project}/requirements-only-tests-min-locked.txt
+      CIBW_TEST_COMMAND: >-
+        pytest {project}/tests
+      MACOSX_DEPLOYMENT_TARGET: "13.0"
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            cibw_archs: "x86_64"
+          - os: ubuntu-24.04-arm
+            cibw_archs: "aarch64"
+          - os: windows-2025
+            cibw_archs: "AMD64"
+          - os: windows-11-arm
+            cibw_archs: "ARM64"
+          - os: macos-13
+            cibw_archs: "x86_64"
+          - os: macos-14
+            cibw_archs: "arm64"
+
+    steps:
+      - uses: actions/checkout@v5
+      - name: Install Fortran compiler - MacOS
+        # When building on Mac, we need to have a Fortran compiler
+        if: ${{ startsWith(matrix.os, 'macos') }}
+        uses: fortran-lang/setup-fortran@v1
+        id: setup-fortran-macos
+        with:
+          # TODO: figure out whether we need/want to use other compilers too
+          compiler: "gcc"
+          version: "13"
+      - name: Specify LLVM - windows-arm
+        if: ${{ matrix.os == 'windows-11-arm' }}
+        shell: pwsh
+        run: |
+          Invoke-WebRequest https://github.com/llvm/llvm-project/releases/download/llvmorg-20.1.8/LLVM-20.1.8-woa64.exe -OutFile LLVM-woa64.exe
+          Start-Process -FilePath ".\LLVM-woa64.exe" -ArgumentList "/S" -Wait
+
+          $env:PATH = "C:\Program Files\LLVM\bin;" + $env:PATH
+          echo "C:\Program Files\LLVM\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+          echo "CC=clang-cl"  | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+          echo "CXX=clang-cl" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+          echo "FC=flang-new" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+          echo "AR=llvm-lib.exe" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v3.1.4  # matplotlib had v3.1.3
+        with:
+          package-dir: .
+          output-dir: wheelhouse
+          config-file: "{package}/pyproject.toml"
+        env:
+          CIBW_ARCHS: ${{ matrix.cibw_archs }}
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: cibw-wheels-${{ matrix.os }}-${{ matrix.cibw_archs }}
+          path: ./wheelhouse/*.whl
+          if-no-files-found: error
+
+  make-sdist:
+    name: Make source distribution
+    runs-on: "ubuntu-latest"
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: ./.github/actions/setup
+        with:
+          python-version: ${{ matrix.python-version }}
+          uv-dependency-install-flags: "--all-extras --group dev"
+      - name: Create source distribution
+        run: uv build --sdist
+      - name: Upload the source distribution artefact
+        uses: actions/upload-artifact@v4
+        with:
+          name: cibw-sdist
+          path: dist/*.tar.gz
+
   draft-release:
     name: Create draft release
+    needs: [ build-wheels, make-sdist ]
     strategy:
       matrix:
         os: [ "ubuntu-latest" ]
@@ -25,15 +121,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           uv-dependency-install-flags: "--all-extras --group dev"
-      - name: Add version to environment
-        run: |
-          PROJECT_VERSION=`sed -ne 's/^version = "\([0-9\.]*\)"/\1/p' pyproject.toml`
-          echo "PROJECT_VERSION=$PROJECT_VERSION" >> $GITHUB_ENV
-      - name: Build package for PyPI
-        run: |
-          uv build
-          # Just in case, undo the changes to `pyproject.toml`
-          git restore --staged . && git restore .
       - name: Generate Release Notes
         run: |
           echo "" >> ".github/release_template.md"
@@ -44,12 +131,18 @@ jobs:
           echo "## Changes" >> ".github/release_template.md"
           echo "" >> ".github/release_template.md"
           git log $(git describe --tags --abbrev=0 HEAD^)..HEAD --pretty='format:* %h %s' --no-merges >> ".github/release_template.md"
+      - name: Download artifacts
+        uses: actions/download-artifact@v5
+        with:
+          pattern: cibw-*
+          path: dist
+          merge-multiple: true
       - name: Create Release Draft
         uses: softprops/action-gh-release@v2
         with:
           body_path: ".github/release_template.md"
           token: "${{ secrets.PERSONAL_ACCESS_TOKEN }}"
           draft: true
           files: |
-            dist/example_fgen_basic-${{ env.PROJECT_VERSION }}-py3-none-any.whl
-            dist/example_fgen_basic-${{ env.PROJECT_VERSION }}.tar.gz
+            dist/example_fgen_basic-*.whl
+            dist/example_fgen_basic-*.tar.gz
diff --git a/changelog/21.feature.md b/changelog/21.feature.md
@@ -0,0 +1 @@
+Added compilation and publication to PyPI of wheels for major OS's and supported Python versions
diff --git a/docs/development.md b/docs/development.md
@@ -80,6 +80,12 @@ that will be performed to be manually specified.
 See the `uv version` docs (specifically the `--bump` flag) for the
 [list of available bump rules](https://docs.astral.sh/uv/reference/cli/#uv-version).
 
+Releasing also includes building of wheels for major operating systems and supported Python versions.
+This is done thanks to [cibuildwheel](https://cibuildwheel.pypa.io/), an incredible service.
+The wheel building can be quite tricky and fiddly.
+If there are issues, we recommend starting with the [cibuildwheel](https://cibuildwheel.pypa.io/) docs
+or raising an issue to discuss with the other maintainers.
+
 ### Standard process
 
 The steps required are the following:
@@ -94,7 +100,8 @@ The steps required are the following:
    (see here:
    [project releases](https://github.com/openscm/example-fgen-basic/releases)).
    Once you are happy with the release
-   (removed placeholders, added key announcements etc.)
+   (removed placeholders, added key announcements,
+   checked that all the expected wheels and source distribution is there etc.)
    then hit 'Publish release'.
    This triggers a release to PyPI
    (which you can then add to the release if you want).
diff --git a/meson.build b/meson.build
@@ -68,17 +68,17 @@ if pyprojectwheelbuild_enabled
     'src/example_fgen_basic/runtime_helpers.py',
   )
 
-  # The ancillary library,
-  # i.e. all the stuff for wrapping that isn't directly exposed to Python.
-  ancillary_lib = library(
-    '@0@-ancillary'.format(meson.project_name()),
-    sources: srcs_ancillary_lib,
-    version: meson.project_version(),
-    dependencies: [],
-    # any other dependencies which aren't in source e.g. fgen-core
-    # e.g. dependencies: [fgen_core_dep],
-    install: false,
-  )
+    # The ancillary library,
+    # i.e. all the stuff for wrapping that isn't directly exposed to Python.
+   ancillary_lib = library(
+      '@0@-ancillary'.format(meson.project_name()),
+      sources: srcs_ancillary_lib,
+      version: meson.project_version(),
+      dependencies: [],
+      # any other dependencies which aren't in source e.g. fgen-core
+      # e.g. dependencies: [fgen_core_dep],
+      install: false,
+      )
 
   ancillary_dep = declare_dependency(link_with: ancillary_lib)
 
diff --git a/pyproject.toml b/pyproject.toml
@@ -125,18 +125,20 @@ all-dev = [
 [build-system]
 build-backend = "mesonpy"
 requires = [
-  "meson-python>=0.15.0",
-  "numpy",
+    "meson-python>=0.15.0",
+    # "numpy",
+    "numpy>=1.26.0; python_version < '3.13'",
+    "numpy>=2.1.0; python_version >= '3.13'",
 ]
 
 # https://mesonbuild.com/meson-python/how-to-guides/meson-args.html
 [tool.meson-python.args]
 setup = [
-  '--default-library=static',
-  '-Dpyprojectwheelbuild=enabled',
-  ]
+    '--default-library=static',
+    '-Dpyprojectwheelbuild=enabled',
+]
 install = [
-'--skip-subprojects',
+    '--skip-subprojects',
 ]
 
 [tool.coverage.run]

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+Added compilation and publication to PyPI of wheels for major OS's and supported Python versions`