merge with feature #1563
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python | |
| on: | |
| workflow_call: | |
| inputs: | |
| override_git_describe: | |
| type: string | |
| git_ref: | |
| type: string | |
| skip_tests: | |
| type: string | |
| workflow_dispatch: | |
| inputs: | |
| override_git_describe: | |
| type: string | |
| git_ref: | |
| type: string | |
| skip_tests: | |
| type: string | |
| repository_dispatch: | |
| push: | |
| branches: | |
| - '**' | |
| - '!main' | |
| - '!feature' | |
| paths-ignore: | |
| - '**.md' | |
| - 'examples/**' | |
| - 'test/**' | |
| - 'tools/**' | |
| - '!tools/pythonpkg/**' | |
| - '.github/patches/duckdb-wasm/**' | |
| - '.github/workflows/**' | |
| - '!.github/workflows/Python.yml' | |
| pull_request: | |
| types: [opened, reopened, ready_for_review] | |
| paths-ignore: | |
| - '**.md' | |
| - 'examples/**' | |
| - 'test/**' | |
| - 'tools/**' | |
| - '!tools/pythonpkg/**' | |
| - '.github/patches/duckdb-wasm/**' | |
| - '.github/workflows/**' | |
| - '!.github/workflows/Python.yml' | |
| concurrency: | |
| group: python-${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}-${{ inputs.override_git_describe }} | |
| cancel-in-progress: true | |
| env: | |
| GH_TOKEN: ${{ secrets.GH_TOKEN }} | |
| OVERRIDE_GIT_DESCRIBE: ${{ inputs.override_git_describe }} | |
| CIBW_TEST_SKIP: ${{ inputs.skip_tests == 'true' && '*-*' || 'cp37-*' }} | |
| jobs: | |
| # This is just a sanity check of Python 3.9 running with Arrow | |
| linux-python3-9: | |
| name: Python 3.9 Linux | |
| runs-on: ubuntu-20.04 | |
| env: | |
| CIBW_BUILD: 'cp39-manylinux_x86_64' | |
| CIBW_TEST_COMMAND: 'python -m pytest {project}/tests --verbose && USE_ACTUAL_SPARK=true JAVA_HOME=$(dirname $(dirname $(readlink -f $(which java)))) SPARK_HOME={project}/spark_installation/spark python -m pytest {project}/tests/fast/spark --verbose' | |
| PYTEST_TIMEOUT: '600' | |
| CIBW_ENVIRONMENT: 'OVERRIDE_GIT_DESCRIBE=${{ inputs.override_git_describe }}' | |
| CIBW_BEFORE_ALL: 'cd {project} && ./scripts/install_spark_in_cibuildwheels_linux_container.sh' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ inputs.git_ref }} | |
| - name: Check/Act on inputs.override_git_describe | |
| shell: bash | |
| run: | | |
| if [[ "${{ inputs.override_git_describe }}" == *-* ]]; then | |
| echo "override_git_describe ${{ inputs.override_git_describe }}: provide either vX.Y.Z or empty string" | |
| exit 1 | |
| elif [[ -z "${{ inputs.override_git_describe }}" ]]; then | |
| echo "No override_git_describe provided" | |
| else | |
| echo "UPLOAD_ASSETS_TO_STAGING_TARGET=$(git log -1 --format=%h)" >> "$GITHUB_ENV" | |
| echo "override_git_describe ${{ inputs.override_git_describe }}: add tag" | |
| git tag ${{ inputs.override_git_describe }} | |
| fi | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Install | |
| shell: bash | |
| run: pip install 'cibuildwheel>=2.16.2' build | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build source dist | |
| shell: bash | |
| working-directory: tools/pythonpkg | |
| run: | | |
| pyproject-build . --sdist | |
| mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball | |
| - name: Build | |
| shell: bash | |
| working-directory: tools/pythonpkg | |
| run: | | |
| export DISTUTILS_C_COMPILER_LAUNCHER=ccache | |
| # TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030 | |
| cibuildwheel --output-dir wheelhouse --config-file pyproject.toml duckdb_tarball | |
| manylinux-extensions-x64: | |
| name: Linux Extensions (linux_amd64_gcc4) | |
| runs-on: ubuntu-latest | |
| container: quay.io/pypa/manylinux2014_x86_64 | |
| needs: linux-python3-9 | |
| env: | |
| GEN: ninja | |
| ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ inputs.git_ref }} | |
| - uses: ./.github/actions/manylinux_2014_setup | |
| with: | |
| aws-cli: 1 | |
| ninja-build: 1 | |
| ccache: 1 | |
| nodejs: 1 | |
| ssh: 1 | |
| python_alias: 1 | |
| openssl: 1 | |
| - uses: ./.github/actions/build_extensions | |
| with: | |
| vcpkg_target_triplet: x64-linux | |
| post_install: rm build/release/src/libduckdb* | |
| deploy_as: linux_amd64_gcc4 | |
| s3_id: ${{ secrets.S3_ID }} | |
| s3_key: ${{ secrets.S3_KEY }} | |
| signing_pk: ${{ secrets.DUCKDB_EXTENSION_SIGNING_PK }} | |
| run_tests: ${{ inputs.skip_tests != 'true' && 1 || 0 }} | |
| run_autoload_tests: ${{ inputs.skip_tests != 'true' && 1 || 0 }} | |
| treat_warn_as_error: 0 | |
| ninja: 1 | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: manylinux-extensions-x64 | |
| path: | | |
| build/release/extension/*/*.duckdb_extension | |
| linux-python3: | |
| name: Python 3 Linux | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| matrix: | |
| arch: [x86_64, aarch64] | |
| python_build: [cp37-*, cp38-*, cp39-*, cp310-*, cp311-*, cp312-*, cp313-*] | |
| manylinux: [manylinux2014, manylinux_2_28] | |
| isRelease: | |
| - ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} | |
| exclude: | |
| # For now we don't distribute the 2_28 wheels on x64 | |
| - arch: x86_64 | |
| manylinux: manylinux_2_28 | |
| # Speed things up a bit for non-releases | |
| - isRelease: false | |
| python_build: 'cp37-*' | |
| - isRelease: false | |
| python_build: 'cp38-*' | |
| - isRelease: false | |
| python_build: 'cp39-*' | |
| - isRelease: false | |
| python_build: 'cp310-*' | |
| - isRelease: false | |
| python_build: 'cp311-*' | |
| - isRelease: false | |
| arch: aarch64 | |
| needs: manylinux-extensions-x64 | |
| env: | |
| CIBW_BUILD: ${{ matrix.python_build}} | |
| CIBW_SKIP: '*-musllinux_aarch64 cp37-musllinux* cp38-musllinux*' | |
| CIBW_ARCHS: ${{ matrix.arch == 'aarch64' && 'aarch64' || 'auto64' }} | |
| CIBW_MANYLINUX_AARCH64_IMAGE: ${{ matrix.manylinux }} | |
| CIBW_MANYLINUX_PYPY_AARCH64_IMAGE: ${{ matrix.manylinux }} | |
| CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux }} | |
| CIBW_MANYLINUX_PYPY_X86_64_IMAGE: ${{ matrix.manylinux }} | |
| PYTEST_TIMEOUT: '600' | |
| DUCKDB_BUILD_UNITY: 1 | |
| CIBW_ENVIRONMENT: 'OVERRIDE_GIT_DESCRIBE=${{ inputs.override_git_describe }}' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ inputs.git_ref }} | |
| - name: Check/Act on inputs.override_git_describe | |
| shell: bash | |
| run: | | |
| if [[ "${{ inputs.override_git_describe }}" == *-* ]]; then | |
| echo "override_git_describe ${{ inputs.override_git_describe }}: provide either vX.Y.Z or empty string" | |
| exit 1 | |
| elif [[ -z "${{ inputs.override_git_describe }}" ]]; then | |
| echo "No override_git_describe provided" | |
| else | |
| echo "UPLOAD_ASSETS_TO_STAGING_TARGET=$(git log -1 --format=%h)" >> "$GITHUB_ENV" | |
| echo "override_git_describe ${{ inputs.override_git_describe }}: add tag" | |
| git tag ${{ inputs.override_git_describe }} | |
| fi | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| if: ${{ matrix.arch == 'aarch64' }} | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Install | |
| shell: bash | |
| run: | | |
| pip install 'cibuildwheel>=2.16.2' build | |
| python -m pip install numpy --config-settings=setup-args="-Dallow-noblas=true" | |
| - uses: actions/download-artifact@v3 | |
| if: ${{ matrix.arch == 'x86_64' }} | |
| with: | |
| name: manylinux-extensions-x64 | |
| path: tools/pythonpkg | |
| - name: List extensions to be tested | |
| shell: bash | |
| run: | | |
| find tools/pythonpkg -maxdepth 2 -type f -name "*.duckdb_extension" | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@v1.2.11 # Note: pinned due to GLIBC incompatibility in later releases | |
| with: | |
| key: ${{ github.job }}-${{ matrix.arch }}-${{ matrix.python_build }} | |
| save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| run: | | |
| cd tools/pythonpkg | |
| pyproject-build . --sdist | |
| mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball | |
| ls duckdb_tarball | |
| export DISTUTILS_C_COMPILER_LAUNCHER=ccache | |
| # TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030 | |
| if [[ "$GITHUB_REF" =~ ^refs/tags/v.+$ ]] ; then | |
| export CIBW_TEST_COMMAND='python -m pytest {project}/tests/fast/api/test_dbapi00.py --verbose' | |
| fi | |
| cibuildwheel --output-dir wheelhouse --config-file pyproject.toml duckdb_tarball | |
| ls wheelhouse | |
| - name: Deploy | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} | |
| shell: bash | |
| run: | | |
| cp tools/pythonpkg/dist/duckdb-*.tar.gz duckdb_python_src.tar.gz | |
| ./scripts/upload-assets-to-staging.sh github_release duckdb_python_src.tar.gz | |
| ./scripts/upload-assets-to-staging.sh twine_upload tools/pythonpkg/dist/duckdb-*.tar.gz tools/pythonpkg/wheelhouse/*.whl | |
| osx-python3: | |
| if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' | |
| name: Python 3 OSX | |
| runs-on: macos-latest | |
| strategy: | |
| matrix: | |
| python_build: [cp38-*, cp39-*, cp310-*, cp311-*, cp312-*, cp313-*] | |
| needs: linux-python3-9 | |
| env: | |
| CIBW_BUILD: ${{ matrix.python_build}} | |
| CIBW_ARCHS: 'x86_64 universal2 arm64' | |
| CIBW_TEST_COMMAND: 'python -m pytest {project}/tests/fast --verbose' | |
| CIBW_ENVIRONMENT: 'MACOSX_DEPLOYMENT_TARGET=12.0' | |
| DUCKDB_BUILD_UNITY: 1 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ inputs.git_ref }} | |
| - name: Check/Act on inputs.override_git_describe | |
| shell: bash | |
| run: | | |
| if [[ "${{ inputs.override_git_describe }}" == *-* ]]; then | |
| echo "override_git_describe ${{ inputs.override_git_describe }}: provide either vX.Y.Z or empty string" | |
| exit 1 | |
| elif [[ -z "${{ inputs.override_git_describe }}" ]]; then | |
| echo "No override_git_describe provided" | |
| else | |
| echo "UPLOAD_ASSETS_TO_STAGING_TARGET=$(git log -1 --format=%h)" >> "$GITHUB_ENV" | |
| echo "override_git_describe ${{ inputs.override_git_describe }}: add tag" | |
| git tag ${{ inputs.override_git_describe }} | |
| fi | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Install | |
| shell: bash | |
| run: pip install 'cibuildwheel>=2.16.2' build | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }}-${{ matrix.python_build }} | |
| save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build | |
| shell: bash | |
| run: | | |
| cd tools/pythonpkg | |
| pyproject-build . --sdist | |
| mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball | |
| export DISTUTILS_C_COMPILER_LAUNCHER=ccache | |
| # TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030 | |
| if [[ "$GITHUB_REF" =~ ^refs/tags/v.+$ ]] ; then | |
| export CIBW_TEST_COMMAND='python -m pytest {project}/tests/fast/api/test_dbapi00.py --verbose' | |
| fi | |
| cibuildwheel --output-dir wheelhouse --config-file pyproject.toml duckdb_tarball | |
| - name: Deploy | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} | |
| shell: bash | |
| run: | | |
| ./scripts/upload-assets-to-staging.sh twine_upload tools/pythonpkg/wheelhouse/*.whl | |
| win-python3: | |
| name: Python 3 Windows | |
| runs-on: windows-2019 | |
| strategy: | |
| matrix: | |
| python_build: [cp37-*, cp38-*, cp39-*, cp310-*, cp311-*, cp312-*, cp313-*] | |
| isRelease: | |
| - ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} | |
| exclude: | |
| - isRelease: false | |
| python_build: 'cp37-*' | |
| - isRelease: false | |
| python_build: 'cp38-*' | |
| - isRelease: false | |
| python_build: 'cp39-*' | |
| - isRelease: false | |
| python_build: 'cp311-*' | |
| needs: linux-python3-9 | |
| env: | |
| CIBW_BUILD: ${{ matrix.python_build}} | |
| DUCKDB_BUILD_UNITY: 1 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ inputs.git_ref }} | |
| - name: Check/Act on inputs.override_git_describe | |
| shell: bash | |
| run: | | |
| if [[ "${{ inputs.override_git_describe }}" == *-* ]]; then | |
| echo "override_git_describe ${{ inputs.override_git_describe }}: provide either vX.Y.Z or empty string" | |
| exit 1 | |
| elif [[ -z "${{ inputs.override_git_describe }}" ]]; then | |
| echo "No override_git_describe provided" | |
| else | |
| echo "UPLOAD_ASSETS_TO_STAGING_TARGET=$(git log -1 --format=%h)" >> "$GITHUB_ENV" | |
| echo "override_git_describe ${{ inputs.override_git_describe }}: add tag" | |
| git tag ${{ inputs.override_git_describe }} | |
| fi | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Install | |
| shell: bash | |
| run: pip install 'cibuildwheel>=2.16.2' build | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }}-${{ matrix.python_build }} | |
| save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} | |
| - name: Build source dist | |
| shell: bash | |
| working-directory: tools/pythonpkg | |
| run: | | |
| pyproject-build . --sdist | |
| mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball | |
| - name: Build wheels | |
| shell: bash | |
| working-directory: tools/pythonpkg | |
| run: | | |
| export DISTUTILS_C_COMPILER_LAUNCHER=ccache | |
| # TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030 | |
| if [[ "$GITHUB_REF" =~ ^refs/tags/v.+$ ]] ; then | |
| export CIBW_TEST_COMMAND='python -m pytest {project}/tests/fast/api/test_dbapi00.py --verbose' | |
| fi | |
| cibuildwheel --output-dir wheelhouse --config-file pyproject.toml duckdb_tarball | |
| - name: Deploy | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} | |
| shell: bash | |
| run: | | |
| ./scripts/upload-assets-to-staging.sh twine_upload tools/pythonpkg/wheelhouse/*.whl | |
| linux-release-cleanup: | |
| if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' | |
| name: PyPi Release Cleanup | |
| runs-on: ubuntu-20.04 | |
| needs: linux-python3-9 | |
| env: | |
| PYPI_CLEANUP_USERNAME: 'mytherin' | |
| PYPI_CLEANUP_PASSWORD: ${{secrets.PYPI_CLEANUP_PASSWORD}} | |
| PYPI_CLEANUP_OTP: ${{secrets.PYPI_CLEANUP_OTP}} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ inputs.git_ref }} | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Install Packages | |
| shell: bash | |
| run: python3 -m pip install pyotp requests | |
| - name: Cleanup Releases | |
| shell: bash | |
| if: ${{ github.ref == 'refs/heads/main' && github.repository == 'duckdb/duckdb' }} | |
| run: python3 scripts/pypi_cleanup.py | |
| twine-upload: | |
| needs: | |
| - osx-python3 | |
| - win-python3 | |
| - linux-python3 | |
| # Note that want to run this by default ONLY if no override_git_describe is provided | |
| # This means we are not staging a release | |
| if: (( startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' )) && (( inputs.override_git_describe == '' )) && (( github.repository == 'duckdb/duckdb' )) | |
| uses: ./.github/workflows/TwineUpload.yml | |
| secrets: inherit | |
| # Why? If present, it means we are staging releases, and we want to do the upload manually |