ci: add AArch64 wheels via cross-compilation (apache#217)

- Use cibuildwheel for adbc_driver_manager
- Consolidate the different Python version builds into a single job
- Split the job by architecture instead

.github/workflows/packaging-wheels.yml

@@ -123,14 +123,13 @@ jobs:
           GEMFURY_PUSH_TOKEN: ${{ secrets.GEMFURY_PUSH_TOKEN }}
 
   python-manylinux:
-    name: "Python ${{ matrix.python_version }} manylinux${{ matrix.manylinux_version }}"
+    name: "Python ${{ matrix.arch }} manylinux${{ matrix.manylinux_version }}"
     runs-on: ubuntu-latest
     strategy:
       matrix:
+        arch: ["amd64", "arm64v8"]
         manylinux_version: ["2014"]
-        python_version: ["3.9", "3.10", "3.11"]
-      # Limit parallelism; gemfury appears to return 409 CONFLICT on concurrent uploads
-      max-parallel: 1
+      # Let this one run in parallel because the arm64 build is so much slower
     steps:
       - uses: actions/checkout@v3
         with:
@@ -149,17 +148,21 @@ jobs:
           echo "schedule: ${{ github.event.schedule }}"
           echo "ref: ${{ github.ref }}"
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
       - name: Build wheel
         shell: bash
+        env:
+          ARCH: ${{ matrix.arch }}
+          MANYLINUX: ${{ matrix.manylinux_version }}
         run: |
-          export MANYLINUX=${{ matrix.manylinux_version }}
-          export PYTHON=${{ matrix.python_version }}
           docker-compose run python-wheel-manylinux
 
       - name: Archive wheels
         uses: actions/upload-artifact@v3
         with:
-          name: python${{ matrix.python_version }}-manylinux${{ matrix.manylinux_version }}
+          name: python-${{ matrix.arch }}-manylinux${{ matrix.manylinux_version }}
           retention-days: 7
           path: |
             python/adbc_driver_manager/repaired_wheels/*.whl
@@ -168,10 +171,13 @@ jobs:
 
       - name: Test wheel
         shell: bash
+        env:
+          ARCH: ${{ matrix.arch }}
+          MANYLINUX: ${{ matrix.manylinux_version }}
         run: |
-          export MANYLINUX=${{ matrix.manylinux_version }}
-          export PYTHON=${{ matrix.python_version }}
-          docker-compose run python-wheel-manylinux-test
+          env PYTHON=3.9 docker-compose run python-wheel-manylinux-test
+          env PYTHON=3.10 docker-compose run python-wheel-manylinux-test
+          env PYTHON=3.11 docker-compose run python-wheel-manylinux-test
 
       - name: Upload wheels to Gemfury
         shell: bash
@@ -182,19 +188,16 @@ jobs:
           GEMFURY_PUSH_TOKEN: ${{ secrets.GEMFURY_PUSH_TOKEN }}
 
   python-macos:
-    name: "Python ${{ matrix.python_version }} macOS"
+    name: "Python ${{ matrix.arch }} macOS"
     runs-on: macos-latest
     strategy:
       matrix:
-        python_version: ["3.9", "3.10", "3.11"]
+        arch: ["amd64", "arm64v8"]
       # Limit parallelism; gemfury appears to return 409 CONFLICT on concurrent uploads
       max-parallel: 1
     env:
       MACOSX_DEPLOYMENT_TARGET: "10.15"
-      PYTHON: "/Library/Frameworks/Python.framework/Versions/${{ matrix.python_version }}/bin/python${{ matrix.python_version }}"
-      PYTHON_VERSION: "${{ matrix.python_version }}"
-      # Use a custom triplet to only build release packages
-      VCPKG_DEFAULT_TRIPLET: "x64-osx-static-release"
+      PYTHON: "/Library/Frameworks/Python.framework/Versions/3.10/bin/python3.10"
       # Where to install vcpkg
       VCPKG_ROOT: "${{ github.workspace }}/vcpkg"
     steps:
@@ -229,39 +232,59 @@ jobs:
         shell: bash
         run: ci/scripts/install_vcpkg.sh $VCPKG_ROOT $VCPKG_VERSION
 
-      - name: Install Python ${{ matrix.python_version }}
+      - name: Install Python
         shell: bash
-        run: sudo ci/scripts/install_python.sh macos ${{ matrix.python_version }}
+        run: |
+          sudo ci/scripts/install_python.sh macos 3.9
+          sudo ci/scripts/install_python.sh macos 3.10
+          sudo ci/scripts/install_python.sh macos 3.11
 
       - name: Build wheel
         shell: bash
+        env:
+          ARCH: ${{ matrix.arch }}
         run: |
           $PYTHON -m venv build-env
           source build-env/bin/activate
-          ./ci/scripts/python_wheel_unix_build.sh x86_64 $(pwd) $(pwd)/build
+          ./ci/scripts/python_wheel_unix_build.sh $ARCH $(pwd) $(pwd)/build
 
       - name: Archive wheels
         uses: actions/upload-artifact@v3
         with:
-          name: python${{ matrix.python_version }}-macOS
+          name: python-${{ matrix.arch }}-macos
           retention-days: 7
           path: |
             python/adbc_driver_manager/repaired_wheels/*.whl
             python/adbc_driver_postgres/repaired_wheels/*.whl
             python/adbc_driver_sqlite/repaired_wheels/*.whl
 
       - name: Test wheel
+        if: matrix.arch == 'amd64'
         shell: bash
         run: |
-          $PYTHON -m venv test-env
-          source test-env/bin/activate
+          /Library/Frameworks/Python.framework/Versions/3.9/bin/python3.9 -m venv test-env-39
+          source test-env-39/bin/activate
+          export PYTHON_VERSION=3.9
+          ./ci/scripts/python_wheel_unix_test.sh $(pwd)
+          deactivate
+
+          /Library/Frameworks/Python.framework/Versions/3.10/bin/python3.10 -m venv test-env-310
+          source test-env-310/bin/activate
+          export PYTHON_VERSION=3.10
           ./ci/scripts/python_wheel_unix_test.sh $(pwd)
+          deactivate
+
+          /Library/Frameworks/Python.framework/Versions/3.11/bin/python3.11 -m venv test-env-311
+          source test-env-311/bin/activate
+          export PYTHON_VERSION=3.11
+          ./ci/scripts/python_wheel_unix_test.sh $(pwd)
+          deactivate
 
       - name: Upload wheels to Gemfury
         shell: bash
         if: github.ref == 'refs/heads/main' && (github.event.schedule || github.event.inputs.upload_artifacts == true)
         run: |
-          ./ci/scripts/python_wheel_upload.sh python/adbc_driver_{manager,postgres,sqlite}/dist/*.whl
+          ./ci/scripts/python_wheel_upload.sh python/adbc_driver_{manager,postgres,sqlite}/repaired_wheels/*.whl
         env:
           GEMFURY_PUSH_TOKEN: ${{ secrets.GEMFURY_PUSH_TOKEN }}
 

NOTICE.txt

@@ -6,3 +6,6 @@ The Apache Software Foundation (http://www.apache.org/).
 
 This product includes software from the miniver project (CC0).
 https://github.com/jbweston/miniver
+
+This product includes software from the vcpkg project (MIT).
+Copyright (c) Microsoft Corporation

ci/docker/python-wheel-manylinux.dockerfile

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG ARCH
+ARG MANYLINUX
+ARG PYTHON
+ARG REPO
+ARG VCPKG
+
+FROM ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-${MANYLINUX}-vcpkg-${VCPKG}
+
+RUN yum install -y docker

ci/scripts/python_sdist_build.sh

@@ -36,7 +36,7 @@ export _ADBC_IS_SDIST=1
 for component in ${COMPONENTS}; do
     pushd ${source_dir}/python/$component
 
-    echo "=== (${PYTHON_VERSION}) Building $component sdist ==="
+    echo "=== Building $component sdist ==="
     # python -m build copies to a tempdir, so we can't reference other files in the repo
     # https://github.com/pypa/pip/issues/5519
     python setup.py sdist

ci/scripts/python_sdist_test.sh

@@ -21,22 +21,25 @@ set -e
 set -x
 set -o pipefail
 
-if [ "$#" -ne 1 ]; then
-  echo "Usage: $0 <adbc-src-dir>"
+if [ "$#" -ne 2 ]; then
+  echo "Usage: $0 <arch> <adbc-src-dir>"
   exit 1
 fi
 
-source_dir=${1}
+arch=${1}
+source_dir=${2}
 build_dir="${source_dir}/build"
 script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 source "${script_dir}/python_util.sh"
 
-echo "=== (${PYTHON_VERSION}) Building ADBC libpq driver ==="
-# Sets ADBC_POSTGRES_LIBRARY
+echo "=== Set up platform variables ==="
+setup_build_vars "${arch}"
+
+echo "=== Building C/C++ driver components ==="
 build_drivers "${source_dir}" "${build_dir}"
 
-echo "=== (${PYTHON_VERSION}) Installing sdists ==="
+echo "=== Installing sdists ==="
 for component in ${COMPONENTS}; do
     pip install --force-reinstall ${source_dir}/python/${component}/dist/*.tar.gz
 done

ci/scripts/python_util.sh

@@ -21,7 +21,7 @@ COMPONENTS="adbc_driver_manager adbc_driver_postgres adbc_driver_sqlite"
 
 function build_drivers {
     local -r source_dir="$1"
-    local -r build_dir="$2"
+    local -r build_dir="$2/${VCPKG_ARCH}"
 
     : ${CMAKE_BUILD_TYPE:=release}
     : ${CMAKE_UNITY_BUILD:=ON}
@@ -35,13 +35,21 @@ function build_drivers {
     if [[ $(uname) == "Linux" ]]; then
         export ADBC_POSTGRES_LIBRARY=${build_dir}/lib/libadbc_driver_postgres.so
         export ADBC_SQLITE_LIBRARY=${build_dir}/lib/libadbc_driver_sqlite.so
-        export VCPKG_DEFAULT_TRIPLET="x64-linux-static-release"
+        export VCPKG_DEFAULT_TRIPLET="${VCPKG_ARCH}-linux-static-release"
     else # macOS
         export ADBC_POSTGRES_LIBRARY=${build_dir}/lib/libadbc_driver_postgres.dylib
         export ADBC_SQLITE_LIBRARY=${build_dir}/lib/libadbc_driver_sqlite.dylib
-        export VCPKG_DEFAULT_TRIPLET="x64-osx-static-release"
+        export VCPKG_DEFAULT_TRIPLET="${VCPKG_ARCH}-osx-static-release"
     fi
 
+    pushd "${VCPKG_ROOT}"
+    # XXX: patch an odd issue where the path of some file is inconsistent between builds
+    patch -N -p1 < "${source_dir}/ci/vcpkg/0001-Work-around-inconsistent-path.patch" || true
+
+    # XXX: make vcpkg retry downloads https://github.com/microsoft/vcpkg/discussions/20583
+    patch -N -p1 < "${source_dir}/ci/vcpkg/0002-Retry-downloads.patch" || true
+    popd
+
     "${VCPKG_ROOT}/vcpkg" install libpq sqlite3 \
           --overlay-triplets "${VCPKG_OVERLAY_TRIPLETS}" \
           --triplet "${VCPKG_DEFAULT_TRIPLET}"
@@ -81,6 +89,42 @@ function build_drivers {
     popd
 }
 
+function setup_build_vars {
+    local -r arch="${1}"
+    if [[ "$(uname)" = "Darwin" ]]; then
+        if [[ "${arch}" = "amd64" ]]; then
+            export CIBW_ARCHS="x86_64"
+            export PYTHON_ARCH="x86_64"
+            export VCPKG_ARCH="x64"
+        elif [[ "${arch}" = "arm64v8" ]]; then
+            export CIBW_ARCHS="arm64"
+            export PYTHON_ARCH="arm64"
+            export VCPKG_ARCH="arm64"
+        else
+            echo "Unknown architecture: ${arch}"
+            exit 1
+        fi
+        export CIBW_BUILD='*-macosx_*'
+        export CIBW_PLATFORM="macos"
+    else
+        if [[ "${arch}" = "amd64" ]]; then
+            export CIBW_ARCHS="x86_64"
+            export PYTHON_ARCH="x86_64"
+            export VCPKG_ARCH="x64"
+        elif [[ "${arch}" = "arm64v8" ]]; then
+            export CIBW_ARCHS="aarch64"
+            export PYTHON_ARCH="arm64"
+            export VCPKG_ARCH="arm64"
+        else
+            echo "Unknown architecture: ${arch}"
+            exit 1
+        fi
+        export CIBW_BUILD='*-manylinux_*'
+        export CIBW_PLATFORM="linux"
+    fi
+    export CIBW_SKIP="pp* ${CIBW_SKIP}"
+}
+
 function test_packages {
     for component in ${COMPONENTS}; do
         echo "=== Testing $component ==="