ci(c/driver_manager,c/driver/postgres): set up Python sdist build (ap…

…ache#179)

Co-authored-by: Joris Van den Bossche <jorisvandenbossche@gmail.com>

.env

@@ -30,7 +30,7 @@ ARCH_SHORT=amd64
 JDK=8
 MANYLINUX=2014
 MAVEN=3.5.4
-PYTHON=3.8
+PYTHON=3.10
 
 # Used through docker-compose.yml and serves as the default version for the
 # ci/scripts/install_vcpkg.sh script. Prefer to use short SHAs to keep the

.gitattributes

@@ -16,5 +16,5 @@
 # under the License.
 
 c/vendor/* linguist-vendored
-python/adbc_driver_manager/poetry.lock linguist-generated=true
-python/adbc_driver_manager/requirements-dev.txt linguist-generated=true
+python/adbc_driver_manager/adbc_driver_manager/_static_version.py export-subst
+python/adbc_driver_postgres/adbc_driver_postgres/_static_version.py export-subst

.github/workflows/packaging-wheels.yml

@@ -197,3 +197,46 @@ jobs:
           ./ci/scripts/python_wheel_upload.sh python/adbc_driver_{manager,postgres}/dist/*.whl
         env:
           GEMFURY_PUSH_TOKEN: ${{ secrets.GEMFURY_PUSH_TOKEN }}
+
+  python-sdist:
+    name: "Python sdist"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Show inputs
+        shell: bash
+        run: |
+          echo "upload_artifacts: ${{ github.event.inputs.upload_artifacts }}"
+          echo "schedule: ${{ github.event.schedule }}"
+          echo "ref: ${{ github.ref }}"
+
+      - name: Build sdist
+        shell: bash
+        run: |
+          docker-compose run python-sdist
+
+      - name: Archive sdist
+        uses: actions/upload-artifact@v3
+        with:
+          name: python${{ matrix.python_version }}-manylinux${{ matrix.manylinux_version }}
+          retention-days: 7
+          path: |
+            python/adbc_driver_manager/dist/*.tar.gz
+            python/adbc_driver_postgres/dist/*.tar.gz
+
+      - name: Test sdist
+        shell: bash
+        run: |
+          docker-compose run python-sdist-test
+
+      - name: Upload sdist to Gemfury
+        shell: bash
+        if: github.ref == 'refs/heads/main' && (github.event.schedule || github.event.inputs.upload_artifacts == true || github.event.inputs.upload_artifacts == 'true')
+        run: |
+          ./ci/scripts/python_wheel_upload.sh python/adbc_driver_{manager,postgres}/dist/*.tar.gz
+        env:
+          GEMFURY_PUSH_TOKEN: ${{ secrets.GEMFURY_PUSH_TOKEN }}

NOTICE.txt

@@ -3,3 +3,6 @@ Copyright 2022 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
+
+This product includes software from the miniver project (CC0).
+https://github.com/jbweston/miniver

ci/conda_env_python.txt

@@ -18,3 +18,4 @@
 Cython
 pyarrow>=8.0.0
 pytest
+setuptools

ci/scripts/python_sdist_build.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -ex
+
+source_dir=${1}
+
+echo "=== (${PYTHON_VERSION}) Building ADBC sdists ==="
+
+# https://github.com/pypa/pip/issues/7555
+# Get the latest pip so we have in-tree-build by default
+pip install --upgrade pip setuptools
+
+# For drivers, which bundle shared libraries, defer that to install time
+export _ADBC_IS_SDIST=1
+
+for component in adbc_driver_manager adbc_driver_postgres; do
+    pushd ${source_dir}/python/$component
+
+    echo "=== (${PYTHON_VERSION}) Building $component sdist ==="
+    # python -m build copies to a tempdir, so we can't reference other files in the repo
+    # https://github.com/pypa/pip/issues/5519
+    python setup.py sdist
+
+    popd
+done

ci/scripts/python_sdist_test.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+set -x
+set -o pipefail
+
+if [ "$#" -ne 1 ]; then
+  echo "Usage: $0 <adbc-src-dir>"
+  exit 1
+fi
+
+source_dir=${1}
+build_dir="${source_dir}/build"
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+source "${script_dir}/python_util.sh"
+
+echo "=== (${PYTHON_VERSION}) Building ADBC libpq driver ==="
+# Sets ADBC_POSTGRES_LIBRARY
+build_drivers "${source_dir}" "${build_dir}"
+
+echo "=== (${PYTHON_VERSION}) Installing sdists ==="
+for component in ${COMPONENTS}; do
+    pip install --force-reinstall ${source_dir}/python/${component}/dist/*.tar.gz
+done
+pip install pytest pyarrow pandas
+
+echo "=== (${PYTHON_VERSION}) Testing sdists ==="
+test_packages

ci/scripts/python_util.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+COMPONENTS="adbc_driver_manager adbc_driver_postgres"
+
+function build_drivers {
+    local -r source_dir="$1"
+    local -r build_dir="$2"
+
+    : ${CMAKE_BUILD_TYPE:=release}
+    : ${CMAKE_UNITY_BUILD:=ON}
+    : ${CMAKE_GENERATOR:=Ninja}
+    : ${VCPKG_ROOT:=/opt/vcpkg}
+    # Enable manifest mode
+    : ${VCPKG_FEATURE_FLAGS:=manifests}
+    # Add our custom triplets
+    : ${VCPKG_OVERLAY_TRIPLETS:="${source_dir}/ci/vcpkg/triplets/"}
+
+    if [[ $(uname) == "Linux" ]]; then
+        export ADBC_POSTGRES_LIBRARY=${build_dir}/lib/libadbc_driver_postgres.so
+        export VCPKG_DEFAULT_TRIPLET="x64-linux-static-release"
+    else # macOS
+        export ADBC_POSTGRES_LIBRARY=${build_dir}/lib/libadbc_driver_postgres.dylib
+        export VCPKG_DEFAULT_TRIPLET="x64-osx-static-release"
+    fi
+
+    echo ${VCPKG_DEFAULT_TRIPLET}
+
+    mkdir -p ${build_dir}
+    pushd ${build_dir}
+
+    cmake \
+        -G ${CMAKE_GENERATOR} \
+        -DADBC_BUILD_SHARED=ON \
+        -DADBC_BUILD_STATIC=OFF \
+        -DCMAKE_INSTALL_LIBDIR=lib \
+        -DCMAKE_INSTALL_PREFIX=${build_dir} \
+        -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake \
+        -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
+        ${source_dir}/c/driver/postgres
+    cmake --build . --target install -j
+    popd
+}
+
+function test_packages {
+    python -c "
+import adbc_driver_manager
+import adbc_driver_manager.dbapi
+import adbc_driver_postgres
+import adbc_driver_postgres.dbapi
+"
+
+    # Will only run some smoke tests
+    # --import-mode required, else tries to import from the source dir instead of installed package
+    echo "=== Testing adbc_driver_manager ==="
+    python -m pytest -vvx --import-mode append -k "not sqlite" ${source_dir}/python/adbc_driver_manager/tests
+    echo "=== Testing adbc_driver_postgres ==="
+    python -m pytest -vvx --import-mode append ${source_dir}/python/adbc_driver_postgres/tests
+}

ci/scripts/python_wheel_unix_build.sh

@@ -22,6 +22,9 @@ set -ex
 arch=${1}
 source_dir=${2}
 build_dir=${3}
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+source "${script_dir}/python_util.sh"
 
 function check_visibility {
     if [[ $(uname) != "Linux" ]]; then
@@ -60,39 +63,8 @@ function check_wheels {
 }
 
 echo "=== (${PYTHON_VERSION}) Building ADBC libpq driver ==="
-: ${CMAKE_BUILD_TYPE:=release}
-: ${CMAKE_UNITY_BUILD:=ON}
-: ${CMAKE_GENERATOR:=Ninja}
-: ${VCPKG_ROOT:=/opt/vcpkg}
-# Enable manifest mode
-: ${VCPKG_FEATURE_FLAGS:=manifests}
-# Add our custom triplets
-: ${VCPKG_OVERLAY_TRIPLETS:="${source_dir}/ci/vcpkg/triplets/"}
-
-if [[ $(uname) == "Linux" ]]; then
-    export ADBC_POSTGRES_LIBRARY=${build_dir}/lib/libadbc_driver_postgres.so
-    export VCPKG_DEFAULT_TRIPLET="x64-linux-static-release"
-else # macOS
-    export ADBC_POSTGRES_LIBRARY=${build_dir}/lib/libadbc_driver_postgres.dylib
-    export VCPKG_DEFAULT_TRIPLET="x64-osx-static-release"
-fi
-
-echo ${VCPKG_DEFAULT_TRIPLET}
-
-mkdir -p ${build_dir}
-pushd ${build_dir}
-
-cmake \
-    -G ${CMAKE_GENERATOR} \
-    -DADBC_BUILD_SHARED=ON \
-    -DADBC_BUILD_STATIC=OFF \
-    -DCMAKE_INSTALL_LIBDIR=lib \
-    -DCMAKE_INSTALL_PREFIX=${build_dir} \
-    -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake \
-    -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
-    ${source_dir}/c/driver/postgres
-cmake --build . --target install -j
-popd
+# Sets ADBC_POSTGRES_LIBRARY
+build_drivers "${source_dir}" "${build_dir}"
 
 # Check that we don't expose any unwanted symbols
 check_visibility $ADBC_POSTGRES_LIBRARY

ci/scripts/python_wheel_unix_test.sh

@@ -26,11 +26,12 @@ if [ "$#" -ne 1 ]; then
   exit 1
 fi
 
-COMPONENTS="adbc_driver_manager adbc_driver_postgres"
-
 source_dir=${1}
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+source "${script_dir}/python_util.sh"
 
-# Install the built wheels
+echo "=== (${PYTHON_VERSION}) Installing sdists ==="
 for component in ${COMPONENTS}; do
     if [[ -d ${source_dir}/python/${component}/repaired_wheels/ ]]; then
         pip install --force-reinstall \
@@ -42,16 +43,6 @@ for component in ${COMPONENTS}; do
 done
 pip install pytest pyarrow pandas
 
-# Test that the modules are importable
-python -c "
-import adbc_driver_manager
-import adbc_driver_manager.dbapi
-import adbc_driver_postgres
-import adbc_driver_postgres.dbapi
-"
 
-# Will only run some smoke tests
-echo "=== Testing adbc_driver_manager ==="
-python -m pytest -vvx -k "not sqlite" ${source_dir}/python/adbc_driver_manager/tests
-echo "=== Testing adbc_driver_postgres ==="
-python -m pytest -vvx ${source_dir}/python/adbc_driver_postgres/tests
+echo "=== (${PYTHON_VERSION}) Testing sdists ==="
+test_packages

docker-compose.yml

@@ -38,13 +38,28 @@ services:
       - .:/adbc:delegated
     command: "/bin/bash -c '/adbc/ci/scripts/java_build.sh /adbc && /adbc/ci/scripts/java_test.sh /adbc'"
 
+  ############################ Python sdist ##################################
+
+  python-sdist:
+    image: ${ARCH}/python:${PYTHON}
+    volumes:
+      - .:/adbc:delegated
+    command: /adbc/ci/scripts/python_sdist_build.sh /adbc
+
+  python-sdist-test:
+    image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-${MANYLINUX}-vcpkg-${VCPKG}
+    volumes:
+      - .:/adbc:delegated
+    command: "'/adbc/ci/scripts/python_sdist_test.sh /adbc'"
+
   ############################ Python wheels ##################################
 
   python-wheel-manylinux:
     image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-${MANYLINUX}-vcpkg-${VCPKG}
     volumes:
       - .:/adbc:delegated
-    command: "'/adbc/ci/scripts/python_wheel_unix_build.sh x86_64 /adbc /adbc/build'"
+    # Must set safe.directory so miniver won't error when calling git
+    command: "'git config --global --add safe.directory /adbc && git config --global --get safe.directory && /adbc/ci/scripts/python_wheel_unix_build.sh x86_64 /adbc /adbc/build'"
 
   python-wheel-manylinux-test:
     image: ${ARCH}/python:${PYTHON}

python/adbc_driver_manager/.gitignore

@@ -18,5 +18,7 @@
 adbc_driver_manager/*.c
 adbc_driver_manager/*.cc
 adbc_driver_manager/*.cpp
-adbc_driver_manager/_version.py
+adbc_driver_manager/*.h
 build/
+dist/
+repaired_wheels/

python/adbc_driver_manager/MANIFEST.in

@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# setuptools manifest
+
+include adbc_driver_manager/adbc.h
+include adbc_driver_manager/adbc_driver_manager.cc
+include adbc_driver_manager/adbc_driver_manager.h

python/adbc_driver_manager/adbc_driver_manager/__init__.py

@@ -46,7 +46,7 @@
     OperationalError,
     ProgrammingError,
 )
-from ._version import version as __version__  # noqa: F401
+from ._version import __version__
 
 __all__ = [
     "__version__",