Python.yml

name: Python
on:
  workflow_call:
    inputs:
      override_git_describe:
        type: string
      git_ref:
        type: string
      skip_tests:
        type: string
  workflow_dispatch:
    inputs:
      override_git_describe:
        type: string
      git_ref:
        type: string
      skip_tests:
        type: string
  repository_dispatch:
  push:
    branches:
      - '**'
      - '!main'
      - '!feature'
    paths-ignore:
      - '**.md'
      - 'examples/**'
      - 'test/**'
      - 'tools/**'
      - '!tools/pythonpkg/**'
      - '.github/patches/duckdb-wasm/**'
      - '.github/workflows/**'
      - '!.github/workflows/Python.yml'

  pull_request:
    types: [opened, reopened, ready_for_review]
    paths-ignore:
      - '**.md'
      - 'examples/**'
      - 'test/**'
      - 'tools/**'
      - '!tools/pythonpkg/**'
      - '.github/patches/duckdb-wasm/**'
      - '.github/workflows/**'
      - '!.github/workflows/Python.yml'


concurrency:
  group: python-${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}-${{ inputs.override_git_describe }}
  cancel-in-progress: true

env:
  GH_TOKEN: ${{ secrets.GH_TOKEN }}
  OVERRIDE_GIT_DESCRIBE: ${{ inputs.override_git_describe }}
  CIBW_TEST_SKIP: ${{ inputs.skip_tests == 'true' && '*-*' || '' }}

jobs:
#  This is just a sanity check of Python 3.9 running with Arrow
   linux-python3-9:
    name: Python 3.9 Linux
    runs-on: ubuntu-20.04

    env:
      CIBW_BUILD: 'cp39-manylinux_x86_64'
      CIBW_TEST_COMMAND: 'python -m pytest {project}/tests --verbose'
      PYTEST_TIMEOUT: '600'
      CIBW_ENVIRONMENT: 'OVERRIDE_GIT_DESCRIBE=${{ inputs.override_git_describe }}'

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
        ref: ${{ inputs.git_ref }}

    - name: Check/Act on inputs.override_git_describe
      shell: bash
      run: |
        if [[ "${{ inputs.override_git_describe }}" == *-* ]]; then
            echo "override_git_describe ${{ inputs.override_git_describe }}: provide either vX.Y.Z or empty string"
            exit 1
        elif [[ -z "${{ inputs.override_git_describe }}" ]]; then
            echo "No override_git_describe provided"
        else
            echo "UPLOAD_ASSETS_TO_STAGING_TARGET=$(git log -1 --format=%h)" >> "$GITHUB_ENV"
            echo "override_git_describe ${{ inputs.override_git_describe }}: add tag"
            git tag ${{ inputs.override_git_describe }}
        fi

    - uses: actions/setup-python@v5
      with:
        python-version: '3.10'

    - name: Install
      shell: bash
      run: pip install 'cibuildwheel>=2.16.2' build

    - name: Setup Ccache
      uses: hendrikmuhs/ccache-action@main
      with:
        key: ${{ github.job }}
        save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}

    - name: Build source dist
      shell: bash
      working-directory: tools/pythonpkg
      run: |
        pyproject-build . --sdist
        mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball

    - name: Build
      shell: bash
      working-directory: tools/pythonpkg
      run: |
        export DISTUTILS_C_COMPILER_LAUNCHER=ccache
        # TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030
        cibuildwheel --output-dir wheelhouse --config-file pyproject.toml duckdb_tarball

   manylinux-extensions-x64:
    name: Linux Extensions (linux_amd64_gcc4)
    runs-on: ubuntu-latest
    container: quay.io/pypa/manylinux2014_x86_64
    needs: linux-python3-9
    env:
      GEN: ninja
      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
        ref: ${{ inputs.git_ref }}

    - uses: ./.github/actions/manylinux_2014_setup
      with:
        aws-cli: 1
        ninja-build: 1
        ccache: 1
        nodejs: 1
        ssh: 1
        python_alias: 1
        openssl: 1

    - uses: ./.github/actions/build_extensions
      with:
        vcpkg_target_triplet: x64-linux
        post_install: rm build/release/src/libduckdb*
        deploy_as: linux_amd64_gcc4
        s3_id: ${{ secrets.S3_ID }}
        s3_key: ${{ secrets.S3_KEY }}
        signing_pk: ${{ secrets.DUCKDB_EXTENSION_SIGNING_PK }}
        run_tests: ${{ inputs.skip_tests != 'true' && 1 || 0 }}
        run_autoload_tests: ${{ inputs.skip_tests != 'true' && 1 || 0 }}
        treat_warn_as_error: 0
        ninja: 1

    - uses: actions/upload-artifact@v3
      with:
        name: manylinux-extensions-x64
        path: |
          build/release/extension/*/*.duckdb_extension

   linux-python3:
    name: Python 3 Linux
    runs-on: ubuntu-20.04
    strategy:
      matrix:
        arch: [x86_64, aarch64]
        python_build: [cp37-*, cp38-*, cp39-*, cp310-*, cp311-*, cp312-*]
        manylinux: [manylinux2014, manylinux_2_28]
        isRelease:
          - ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
        exclude:
          # For now we don't distribute the 2_28 wheels on x64
          - arch: x86_64
            manylinux: manylinux_2_28
          # Speed things up a bit for non-releases
          - isRelease: false
            python_build: 'cp37-*'
          - isRelease: false
            python_build: 'cp38-*'
          - isRelease: false
            python_build: 'cp39-*'
          - isRelease: false
            python_build: 'cp310-*'
          - isRelease: false
            python_build: 'cp311-*'
          - isRelease: false
            arch: aarch64
    needs: manylinux-extensions-x64
    env:
      CIBW_BUILD: ${{ matrix.python_build}}
      CIBW_SKIP: '*-musllinux_aarch64'
      CIBW_ARCHS: ${{ matrix.arch == 'aarch64' && 'aarch64' || 'auto64' }}
      CIBW_MANYLINUX_AARCH64_IMAGE: ${{ matrix.manylinux }}
      CIBW_MANYLINUX_PYPY_AARCH64_IMAGE: ${{ matrix.manylinux }}
      CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux }}
      CIBW_MANYLINUX_PYPY_X86_64_IMAGE: ${{ matrix.manylinux }}
      PYTEST_TIMEOUT: '600'
      DUCKDB_BUILD_UNITY: 1
      CIBW_ENVIRONMENT: 'OVERRIDE_GIT_DESCRIBE=${{ inputs.override_git_describe }}'

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
        ref: ${{ inputs.git_ref }}

    - name: Check/Act on inputs.override_git_describe
      shell: bash
      run: |
        if [[ "${{ inputs.override_git_describe }}" == *-* ]]; then
            echo "override_git_describe ${{ inputs.override_git_describe }}: provide either vX.Y.Z or empty string"
            exit 1
        elif [[ -z "${{ inputs.override_git_describe }}" ]]; then
            echo "No override_git_describe provided"
        else
            echo "UPLOAD_ASSETS_TO_STAGING_TARGET=$(git log -1 --format=%h)" >> "$GITHUB_ENV"
            echo "override_git_describe ${{ inputs.override_git_describe }}: add tag"
            git tag ${{ inputs.override_git_describe }}
        fi

    - name: Set up QEMU
      uses: docker/setup-qemu-action@v2
      if: ${{ matrix.arch == 'aarch64' }}

    - uses: actions/setup-python@v5
      with:
        python-version: '3.10'

    - name: Install
      shell: bash
      run: |
        pip install 'cibuildwheel>=2.16.2' build
        python -m pip install numpy --config-settings=setup-args="-Dallow-noblas=true"

    - uses: actions/download-artifact@v3
      if: ${{ matrix.arch == 'x86_64' }}
      with:
        name: manylinux-extensions-x64
        path: tools/pythonpkg

    - name: List extensions to be tested
      shell: bash
      run: |
        find tools/pythonpkg -maxdepth 2 -type f -name "*.duckdb_extension"

    - name: Setup Ccache
      uses: hendrikmuhs/ccache-action@v1.2.11 # Note: pinned due to GLIBC incompatibility in later releases
      with:
        key: ${{ github.job }}-${{ matrix.arch }}-${{ matrix.python_build }}
        save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}

    - name: Build
      shell: bash
      run: |
        cd tools/pythonpkg
        pyproject-build . --sdist
        mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball
        ls duckdb_tarball
        export DISTUTILS_C_COMPILER_LAUNCHER=ccache
        # TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030
        if [[ "$GITHUB_REF" =~ ^refs/tags/v.+$ ]] ; then
          export CIBW_TEST_COMMAND='python -m pytest {project}/tests/fast/api/test_dbapi00.py  --verbose'
        fi
        cibuildwheel --output-dir wheelhouse --config-file pyproject.toml duckdb_tarball
        ls wheelhouse

    - name: Deploy
      env:
        AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }}
      shell: bash
      run: |
        cp tools/pythonpkg/dist/duckdb-*.tar.gz duckdb_python_src.tar.gz
        ./scripts/upload-assets-to-staging.sh github_release duckdb_python_src.tar.gz
        ./scripts/upload-assets-to-staging.sh twine_upload tools/pythonpkg/dist/duckdb-*.tar.gz tools/pythonpkg/wheelhouse/*.whl

   osx-python3:
      if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main'
      name: Python 3 OSX
      runs-on: macos-latest
      strategy:
       matrix:
        python_build: [cp38-*, cp39-*, cp310-*, cp311-*, cp312-*]
      needs: linux-python3-9
      env:
        CIBW_BUILD: ${{ matrix.python_build}}
        CIBW_ARCHS: 'x86_64 universal2 arm64'
        CIBW_TEST_COMMAND: 'python -m pytest {project}/tests/fast  --verbose'
        CIBW_ENVIRONMENT: 'MACOSX_DEPLOYMENT_TARGET=12.0'
        DUCKDB_BUILD_UNITY: 1

      steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
          ref: ${{ inputs.git_ref }}

      - name: Check/Act on inputs.override_git_describe
        shell: bash
        run: |
          if [[ "${{ inputs.override_git_describe }}" == *-* ]]; then
              echo "override_git_describe ${{ inputs.override_git_describe }}: provide either vX.Y.Z or empty string"
              exit 1
          elif [[ -z "${{ inputs.override_git_describe }}" ]]; then
              echo "No override_git_describe provided"
          else
              echo "UPLOAD_ASSETS_TO_STAGING_TARGET=$(git log -1 --format=%h)" >> "$GITHUB_ENV"
              echo "override_git_describe ${{ inputs.override_git_describe }}: add tag"
              git tag ${{ inputs.override_git_describe }}
          fi

      - uses: actions/setup-python@v5
        with:
          python-version: '3.10'

      - name: Install
        shell: bash
        run: pip install 'cibuildwheel>=2.16.2' build

      - name: Setup Ccache
        uses: hendrikmuhs/ccache-action@main
        with:
          key: ${{ github.job }}-${{ matrix.python_build }}
          save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}

      - name: Build
        shell: bash
        run: |
          cd tools/pythonpkg
          pyproject-build . --sdist
          mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball
          export DISTUTILS_C_COMPILER_LAUNCHER=ccache
          # TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030
          if [[ "$GITHUB_REF" =~ ^refs/tags/v.+$ ]] ; then
            export CIBW_TEST_COMMAND='python -m pytest {project}/tests/fast/api/test_dbapi00.py  --verbose'
          fi
          cibuildwheel --output-dir wheelhouse --config-file pyproject.toml duckdb_tarball

      - name: Deploy
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }}
        shell: bash
        run: |
          ./scripts/upload-assets-to-staging.sh twine_upload tools/pythonpkg/wheelhouse/*.whl

   win-python3:
      name: Python 3 Windows
      runs-on: windows-2019
      strategy:
       matrix:
        python_build: [cp37-*, cp38-*, cp39-*, cp310-*, cp311-*, cp312-*]
        isRelease:
          - ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
        exclude:
          - isRelease: false
            python_build: 'cp37-*'
          - isRelease: false
            python_build: 'cp38-*'
          - isRelease: false
            python_build: 'cp39-*'
          - isRelease: false
            python_build: 'cp311-*'
      needs: linux-python3-9

      env:
        CIBW_BUILD: ${{ matrix.python_build}}
        DUCKDB_BUILD_UNITY: 1

      steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
          ref: ${{ inputs.git_ref }}

      - name: Check/Act on inputs.override_git_describe
        shell: bash
        run: |
          if [[ "${{ inputs.override_git_describe }}" == *-* ]]; then
              echo "override_git_describe ${{ inputs.override_git_describe }}: provide either vX.Y.Z or empty string"
              exit 1
          elif [[ -z "${{ inputs.override_git_describe }}" ]]; then
              echo "No override_git_describe provided"
          else
              echo "UPLOAD_ASSETS_TO_STAGING_TARGET=$(git log -1 --format=%h)" >> "$GITHUB_ENV"
              echo "override_git_describe ${{ inputs.override_git_describe }}: add tag"
              git tag ${{ inputs.override_git_describe }}
          fi

      - uses: actions/setup-python@v5
        with:
          python-version: '3.10'

      - name: Install
        shell: bash
        run: pip install 'cibuildwheel>=2.16.2' build

      - name: Setup Ccache
        uses: hendrikmuhs/ccache-action@main
        with:
          key: ${{ github.job }}-${{ matrix.python_build }}
          save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}

      - name: Build source dist
        shell: bash
        working-directory: tools/pythonpkg
        run: |
          pyproject-build . --sdist
          mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball

      - name: Build wheels
        shell: bash
        working-directory: tools/pythonpkg
        run: |
          export DISTUTILS_C_COMPILER_LAUNCHER=ccache
          # TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030
          if [[ "$GITHUB_REF" =~ ^refs/tags/v.+$ ]] ; then
            export CIBW_TEST_COMMAND='python -m pytest {project}/tests/fast/api/test_dbapi00.py  --verbose'
          fi
          cibuildwheel --output-dir wheelhouse --config-file pyproject.toml duckdb_tarball

      - name: Deploy
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }}
        shell: bash
        run: |
          ./scripts/upload-assets-to-staging.sh twine_upload tools/pythonpkg/wheelhouse/*.whl

   linux-release-cleanup:
     if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main'
     name: PyPi Release Cleanup
     runs-on: ubuntu-20.04
     needs: linux-python3-9
     env:
       PYPI_CLEANUP_USERNAME: 'mytherin'
       PYPI_CLEANUP_PASSWORD: ${{secrets.PYPI_CLEANUP_PASSWORD}}
       PYPI_CLEANUP_OTP: ${{secrets.PYPI_CLEANUP_OTP}}

     steps:
     - uses: actions/checkout@v3
       with:
         fetch-depth: 0
         ref: ${{ inputs.git_ref }}

     - uses: actions/setup-python@v5
       with:
         python-version: '3.10'

     - name: Install Packages
       shell: bash
       run: python3 -m pip install pyotp requests

     - name: Cleanup Releases
       shell: bash
       if: ${{ github.ref == 'refs/heads/main' && github.repository == 'duckdb/duckdb' }}
       run: python3 scripts/pypi_cleanup.py

   twine-upload:
    needs:
      - osx-python3
      - win-python3
      - linux-python3
    # Note that want to run this by default ONLY if no override_git_describe is provided
    # This means we are not staging a release
    if: (( startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' )) && (( inputs.override_git_describe == '' )) && (( github.repository == 'duckdb/duckdb' ))
    uses: ./.github/workflows/TwineUpload.yml
    secrets: inherit
    # Why? If present, it means we are staging releases, and we want to do the upload manually