MAINT, CI: cibuildwheel support (#741)

* pyproject.toml updates to support cibuildwheel

* drop '--build-ext' option in setup.py for env var

This option to build the C extension is only used in the wheel
building process. cibuildwheel does not appear to support an
ability to pass options to setuptools, and even if it did there
appears to be no support for pip installing wheels with flags
like '--build-option' (see
pypa/pip#5771).

This commit also updates our build-wheels script to use this
env var, even though those scripts are more or less deprecated.

* add first cut at build_wheels github action

* tell build_wheels action about pydarshan directory

* checkout autoperf submodule before building wheels

* pin matplotlib<3.5 for cibuildwheel test phase

see gh-479 for more details

* avoid macos builds for now

* move cibuildwheel linux specific bits

* only build cpython, re-enable macos builds

* install automake on macos

* bug fix in specifying cibuildwheel before-all

* debug pypy 3.9 builds

* add more brew dependencies for macos

* skip pypy builds, as well as ppc/s390x

* explicitly specicy repair wheel commands

* restrict when wheels are built

for now, just runs manually via GitHub actions, or by appending
a message to the commit message

* add dummy file to test commit message wheel hooks

[wheel build]

* forgot part of the github workflow config

[wheel build]

* actually add the get commit message job

[wheel build]

* let's try one more time... [wheel build]

* allow manual running of wheels workflow

* update release checklist to reflect wheel changes

.github/workflows/build_wheels.yml

@@ -0,0 +1,60 @@
+name: Build Wheels
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - darshan-util/**
+      - include/**
+  pull_request:
+    branches:
+      - main
+    paths:
+      - darshan-util/**
+      - include/*
+  workflow_dispatch:
+
+jobs:
+  get_commit_message:
+    name: Get commit message
+    runs-on: ubuntu-latest
+    if: github.repository == 'darshan-hpc/darshan'
+    outputs:
+      message: ${{ steps.commit_message.outputs.message }}
+    steps:
+      - name: Checkout darshan
+        uses: actions/checkout@v3
+        # Gets the correct commit message for pull request
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - name: Get commit message
+        id: commit_message
+        run: |
+          set -xe
+          COMMIT_MSG=$(git log --no-merges -1 --oneline)
+          echo "::set-output name=message::$COMMIT_MSG"
+          echo github.ref ${{ github.ref }}
+
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    needs: get_commit_message
+    if: >-
+      contains(needs.get_commit_message.outputs.message, '[wheel build]') ||
+      github.event_name == 'workflow_dispatch'
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v2.5.0
+        with:
+          package-dir: ./darshan-util/pydarshan
+
+      - uses: actions/upload-artifact@v2
+        with:
+          path: ./wheelhouse/*.whl

darshan-util/pydarshan/RELEASE-CHECKLIST-PyDarshan.txt

@@ -25,11 +25,15 @@ Notes on how to release a new version of PyDarshan
         (flake8 syntax warnings can be ignored)
     - TODO: CI?
 
- - Submit to PyPi using twine:
-    - make wheels       # requires docker, creates ./wheelhouse and builds architecture-specific *.whl that include libdarshan-util
-    - make dist         # gathers relevant wheels build earlier, adds non-binary wheel and a source distribution (zip/tgz)
-    - make release      # pushes contents of ./dist/* to PyPi
-        (be prompted for username/password)
+ - Submit to PyPI using cibuildwheel/twine:
+    - pipx run build --sdist
+        (in the pydarshan directory, creates a source distribution)
+    - download wheel artifacts from GitHub, copy to 'dist/' directory created above
+        (click on Actions in GitHub, find most recent "Build Wheels" workflow run with artifacts)
+        (download artifact zip, extract wheels to 'dist/' directory)
+    - pipx run twine upload dist/*
+        (requires PyPI credentials for darshan project)
+        (use '--repository testpypi' to test wheels using https://test.pypi.org/project/darshan/ before uploading to PyPI)
 
  - Add/update spack package: py-darshan
     - add version entry

darshan-util/pydarshan/devel/build-wheels.sh

@@ -41,13 +41,15 @@ cd /
 
 ls /opt/python
 
+# Force setup.py to build the C extension
+export PYDARSHAN_BUILD_EXT=1
 
 # Compile wheels
 for PYBIN in /opt/python/*/bin; do
     # JL: we do not really need any dependencies to build the wheel,
     #     but requirements install needs to be renabled when testing automatically
     #"${PYBIN}/pip" install -r /io/requirements_wheels.txt
-    "${PYBIN}/pip" wheel /io/ --build-option "--with-extension" --no-deps -w /io/wheelhouse/${PLAT}
+    "${PYBIN}/pip" wheel /io/ --no-deps -w /io/wheelhouse/${PLAT}
 done
 
 # Bundle external shared libraries into the wheels

darshan-util/pydarshan/pyproject.toml

@@ -3,3 +3,47 @@ requires = [
     "wheel",
     "setuptools",
 ]
+
+[tool.cibuildwheel]
+environment = "PYDARSHAN_BUILD_EXT=1"
+skip = [
+    "pp*",
+    "*musllinux*",
+    "*i686*",
+    "*_ppc64le",
+    "*_s390x"
+]
+test-requires = [
+    "pytest",
+    "lxml",
+    "matplotlib<3.5",
+    "importlib_resources;python_version<'3.9'"
+]
+test-command = "pytest {package}"
+
+[tool.cibuildwheel.linux]
+before-all = [
+    "yum install -y blas-devel lapack-devel",
+    "yum install -y libxslt-devel libxml2-devel",
+    "yum install -y libjpeg libjpeg-devel",
+    "git submodule update --init",
+    "./prepare.sh",
+    "./configure --disable-darshan-runtime --enable-apxc-mod --enable-apmpi-mod",
+    "make install"
+]
+repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel}"
+
+[tool.cibuildwheel.macos]
+before-all = [
+    "brew install automake",
+    "brew install openblas",
+    "brew install lapack",
+    "git submodule update --init",
+    "./prepare.sh",
+    "./configure --disable-darshan-runtime --enable-apxc-mod --enable-apmpi-mod",
+    "make install"
+]
+repair-wheel-command = [
+  "delocate-listdeps {wheel}",
+  "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}",
+]

darshan-util/pydarshan/setup.py

@@ -28,7 +28,7 @@
 # discoverable in the environment by means of LD_LIBRARY_PATH or
 # pkg-config there is no need to build the extension.
 ext_modules = []
-if "--with-extension" in sys.argv:
+if "PYDARSHAN_BUILD_EXT" in os.environ:
     ext_modules.append(
         Extension(
             "darshan.extension",
@@ -38,7 +38,6 @@
             libraries=["darshan-util"],
         )
     )
-    sys.argv.remove("--with-extension")
 
 #
 # Find backend python files in modules and copy them into lib