Skip to content

reading piped json in a recursive CTE works now #955

reading piped json in a recursive CTE works now

reading piped json in a recursive CTE works now #955

Workflow file for this run

name: Python
on:
workflow_dispatch:
repository_dispatch:
push:
branches:
- '**'
- '!main'
- '!feature'
tags:
- '**'
paths-ignore:
- '**.md'
- 'examples/**'
- 'test/**'
- 'tools/**'
- '!tools/pythonpkg/**'
- '.github/patches/duckdb-wasm/**'
- '.github/workflows/**'
- '!.github/workflows/Python.yml'
- '.github/config/uncovered_files.csv'
pull_request:
types: [opened, reopened, ready_for_review]
paths-ignore:
- '**.md'
- 'examples/**'
- 'test/**'
- 'tools/**'
- '!tools/pythonpkg/**'
- '.github/patches/duckdb-wasm/**'
- '.github/workflows/**'
- '!.github/workflows/Python.yml'
- '.github/config/uncovered_files.csv'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
cancel-in-progress: true
env:
GH_TOKEN: ${{ secrets.GH_TOKEN }}
jobs:
# This is just a sanity check of Python 3.9 running with Arrow
linux-python3-9:
name: Python 3.9 Linux
runs-on: ubuntu-20.04
env:
CIBW_BUILD: 'cp39-manylinux_x86_64'
CIBW_TEST_COMMAND: 'python -m pytest {project}/tests'
SETUPTOOLS_SCM_NO_LOCAL: 'yes'
PYTEST_TIMEOUT: '600'
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Install
shell: bash
run: pip install cibuildwheel twine
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}
save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}
- name: Build source dist
shell: bash
working-directory: tools/pythonpkg
run: |
python setup.py sdist
mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball
- name: Build
shell: bash
working-directory: tools/pythonpkg
run: |
export DISTUTILS_C_COMPILER_LAUNCHER=ccache
# TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030
cibuildwheel --output-dir wheelhouse --config-file cibw.toml duckdb_tarball
manylinux-extensions-x64:
# Builds extensions for linux_md64_gcc4
name: Linux Extensions (ggc4)
runs-on: ubuntu-latest
container: quay.io/pypa/manylinux2014_x86_64
needs: linux-python3-9
env:
GEN: ninja
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: ./.github/actions/manylinux_2014_setup
with:
aws-cli: 1
ninja-build: 1
ccache: 1
ssh: 1
python_alias: 1
openssl: 1
- uses: ./.github/actions/build_extensions
with:
vcpkg_target_triplet: x64-linux
post_install: rm build/release/src/libduckdb*
deploy_as: linux_amd64_gcc4
s3_id: ${{ secrets.S3_ID }}
s3_key: ${{ secrets.S3_KEY }}
signing_pk: ${{ secrets.DUCKDB_EXTENSION_SIGNING_PK }}
treat_warn_as_error: 0
ninja: 1
- uses: actions/upload-artifact@v3
with:
name: manylinux-extensions-x64
path: |
build/release/extension/*/*.duckdb_extension
linux-python3:
name: Python 3 Linux
runs-on: ubuntu-20.04
strategy:
matrix:
arch: [i686, x86_64, aarch64]
python_build: [cp37-*, cp38-*, cp39-*, cp310-*, cp311-*]
isRelease:
- ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
exclude:
- isRelease: false
python_build: 'cp37-*'
- isRelease: false
python_build: 'cp38-*'
- isRelease: false
python_build: 'cp39-*'
- isRelease: false
python_build: 'cp311-*'
- isRelease: false
arch: i686
- isRelease: false
arch: aarch64
needs: manylinux-extensions-x64
env:
CIBW_BUILD: ${{ matrix.python_build}}
CIBW_SKIP: '*-musllinux_aarch64'
CIBW_ARCHS: ${{ matrix.arch == 'i686' && 'auto32' || matrix.arch == 'aarch64' && 'aarch64' || 'auto64' }}
SETUPTOOLS_SCM_NO_LOCAL: 'yes'
PYTEST_TIMEOUT: '600'
DUCKDB_BUILD_UNITY: 1
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
if: ${{ matrix.arch == 'aarch64' }}
- uses: actions/setup-python@v4
with:
python-version: '3.7'
- name: Install
shell: bash
run: pip install cibuildwheel twine
- uses: actions/download-artifact@v3
if: ${{ matrix.arch == 'x86_64' }}
with:
name: manylinux-extensions-x64
path: tools/pythonpkg
- name: List extensions to be tested
shell: bash
run: |
find tools/pythonpkg -maxdepth 2 -type f -name "*.duckdb_extension"
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}-${{ matrix.arch }}-${{ matrix.python_build }}
save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}
- name: Build
shell: bash
run: |
cd tools/pythonpkg
python setup.py sdist
mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball
ls duckdb_tarball
export DISTUTILS_C_COMPILER_LAUNCHER=ccache
# TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030
cibuildwheel --output-dir wheelhouse --config-file cibw.toml duckdb_tarball
ls wheelhouse
- name: Deploy
env:
TWINE_USERNAME: '__token__'
TWINE_PASSWORD: ${{ secrets.TWINE_TOKEN }}
shell: bash
run: |
python scripts/asset-upload-gha.py duckdb_python_src.tar.gz=tools/pythonpkg/dist/duckdb-*.tar.gz
if [[ "$GITHUB_REF" =~ ^(refs/heads/main|refs/tags/v.+)$ && "$GITHUB_REPOSITORY" = "duckdb/duckdb" ]] ; then
twine upload --non-interactive --disable-progress-bar --skip-existing tools/pythonpkg/wheelhouse/*.whl tools/pythonpkg/dist/duckdb-*.tar.gz
fi
osx-python3:
if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main'
name: Python 3 OSX
runs-on: macos-latest
strategy:
matrix:
python_build: [cp37-*, cp38-*, cp39-*, cp310-*, cp311-*]
needs: linux-python3-9
env:
CIBW_BUILD: ${{ matrix.python_build}}
CIBW_ARCHS: 'x86_64 universal2 arm64'
CIBW_TEST_COMMAND: 'python -m pytest {project}/tests/fast'
SETUPTOOLS_SCM_NO_LOCAL: 'yes'
TWINE_USERNAME: '__token__'
DUCKDB_BUILD_UNITY: 1
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: actions/setup-python@v4
with:
python-version: '3.7'
- name: Install
shell: bash
run: pip install cibuildwheel twine
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}-${{ matrix.python_build }}
save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}
- name: Build
shell: bash
run: |
cd tools/pythonpkg
python setup.py sdist
mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball
export DISTUTILS_C_COMPILER_LAUNCHER=ccache
# TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030
cibuildwheel --output-dir wheelhouse --config-file cibw.toml duckdb_tarball
- name: Deploy
env:
TWINE_USERNAME: '__token__'
TWINE_PASSWORD: ${{ secrets.TWINE_TOKEN }}
shell: bash
run: |
if [[ "$GITHUB_REF" =~ ^(refs/heads/main|refs/tags/v.+)$ && "$GITHUB_REPOSITORY" = "duckdb/duckdb" ]] ; then
twine upload --non-interactive --disable-progress-bar --skip-existing tools/pythonpkg/wheelhouse/*.whl
fi
win-python3:
name: Python 3 Windows
runs-on: windows-2019
strategy:
matrix:
python_build: [cp37-*, cp38-*, cp39-*, cp310-*, cp311-*]
isRelease:
- ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
exclude:
- isRelease: false
python_build: 'cp37-*'
- isRelease: false
python_build: 'cp38-*'
- isRelease: false
python_build: 'cp39-*'
- isRelease: false
python_build: 'cp311-*'
needs: linux-python3-9
env:
CIBW_BUILD: ${{ matrix.python_build}}
SETUPTOOLS_SCM_NO_LOCAL: 'yes'
SETUPTOOLS_USE_DISTUTILS: 'stdlib'
TWINE_USERNAME: '__token__'
DUCKDB_BUILD_UNITY: 1
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: actions/setup-python@v4
with:
python-version: '3.7'
- name: Install
shell: bash
run: pip install cibuildwheel twine
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}-${{ matrix.python_build }}
save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}
- name: Build
shell: bash
run: |
cd tools/pythonpkg
python setup.py sdist
mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball
export DISTUTILS_C_COMPILER_LAUNCHER=ccache
# TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030
cibuildwheel --output-dir wheelhouse --config-file cibw.toml duckdb_tarball
- name: Deploy
env:
TWINE_USERNAME: '__token__'
TWINE_PASSWORD: ${{ secrets.TWINE_TOKEN }}
shell: bash
run: |
if [[ "$GITHUB_REF" =~ ^(refs/heads/main|refs/tags/v.+)$ && "$GITHUB_REPOSITORY" = "duckdb/duckdb" ]] ; then
twine upload --non-interactive --disable-progress-bar --skip-existing tools/pythonpkg/wheelhouse/*.whl
fi
linux-release-cleanup:
if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main'
name: PyPi Release Cleanup
runs-on: ubuntu-20.04
needs: linux-python3-9
env:
PYPI_CLEANUP_USERNAME: 'mytherin'
PYPI_CLEANUP_PASSWORD: ${{secrets.PYPI_CLEANUP_PASSWORD}}
PYPI_CLEANUP_OTP: ${{secrets.PYPI_CLEANUP_OTP}}
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: actions/setup-python@v4
with:
python-version: '3.7'
- name: Install Packages
shell: bash
run: python3 -m pip install pyotp requests
- name: Cleanup Releases
shell: bash
run: python3 scripts/pypi_cleanup.py