Skip to content
NightlyTests

NightlyTests #513

Sign in to view logs

NightlyTests

NightlyTests #513

Workflow file for this run

.github/workflows/NightlyTests.yml at e791508
name: NightlyTests
on:
workflow_dispatch:
repository_dispatch:
push:
branches:
- '**'
- '!main'
- '!feature'
paths-ignore:
- '**'
- '!.github/workflows/NightlyTests.yml'
- '!.github/patches/duckdb-wasm/**'
pull_request:
types: [opened, reopened, ready_for_review]
paths-ignore:
- '**'
- '!.github/workflows/NightlyTests.yml'
- '!.github/patches/duckdb-wasm/**'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
cancel-in-progress: true
env:
GH_TOKEN: ${{ secrets.GH_TOKEN }}
DUCKDB_WASM_VERSION: "cf2048bd6d669ffa05c56d7d453e09e99de8b87e"
CCACHE_SAVE: ${{ github.repository != 'duckdb/duckdb' }}
BASE_BRANCH: ${{ github.base_ref || (endsWith(github.ref, '_feature') && 'feature' || 'main') }}
jobs:
linux-memory-leaks:
name: Linux Memory Leaks
runs-on: ubuntu-20.04
env:
GEN: ninja
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install Ninja
shell: bash
run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}
save: ${{ env.CCACHE_SAVE }}
- name: Build
shell: bash
run: make
- name: Test
shell: bash
run: |
python3 test/memoryleak/test_memory_leaks.py
release-assert:
name: Release Assertions
runs-on: ubuntu-20.04
needs: linux-memory-leaks
env:
CC: gcc-10
CXX: g++-10
GEN: ninja
BUILD_ICU: 1
BUILD_TPCH: 1
BUILD_TPCDS: 1
BUILD_FTS: 1
BUILD_JSON: 1
BUILD_JEMALLOC: 1
CORE_EXTENSIONS: "inet"
DISABLE_SANITIZER: 1
CRASH_ON_ASSERT: 1
RUN_SLOW_VERIFIERS: 1
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Install
shell: bash
run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}
save: ${{ env.CCACHE_SAVE }}
- name: Build
shell: bash
run: UNSAFE_NUMERIC_CAST=1 make relassert
- name: Test
shell: bash
run: |
python3 scripts/run_tests_one_by_one.py build/relassert/test/unittest "*" --no-exit --timeout 1200
linux-clang:
name: Clang 14
runs-on: ubuntu-20.04
needs: linux-memory-leaks
env:
CC: /home/runner/work/llvm/bin/clang
CPP: /home/runner/work/llvm/bin/clang-cpp
CXX: /home/runner/work/llvm/bin/clang++
LD: /home/runner/work/llvm/bin/ld.lld
EXTENSION_STATIC_BUILD: 1
BUILD_JSON: 1
TREAT_WARNINGS_AS_ERRORS: 1
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Install LLVM and Clang
uses: KyleMayes/install-llvm-action@v1
with:
version: "14.0"
directory: '/home/runner/work/llvm'
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}
save: ${{ env.CCACHE_SAVE }}
- name: Build
shell: bash
run: make release
- name: Test
shell: bash
run: make allunit
old-gcc:
name: GCC 4.8
needs: linux-memory-leaks
runs-on: ubuntu-latest
container: quay.io/pypa/manylinux2014_x86_64
env:
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: ./.github/actions/manylinux_2014_setup
with:
ninja-build: 1
ccache: 1
gcc_4_8: 1
- name: Test
shell: bash
run: make unit
env:
CC: /usr/bin/gcc
CXX: /usr/bin/g++
GEN: ninja
linux-release-32:
name: Linux (32 Bit)
runs-on: ubuntu-latest
container: quay.io/pypa/manylinux2014_x86_64
needs: linux-memory-leaks
env:
CC: /usr/bin/gcc
CXX: /usr/bin/g++
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: ./.github/actions/manylinux_2014_setup
with:
ninja-build: 1
ccache: 1
glibc32: 1
gcc_4_8: 1 # Note: we run this job on the older gcc 4.8 toolchain
- name: Build
shell: bash
run: |
mkdir -p build/release
(cd build/release && cmake -G "Ninja" -DSTATIC_LIBCPP=1 -DBUILD_EXTENSIONS='icu;parquet;fts;json' -DFORCE_32_BIT=1 -DCMAKE_BUILD_TYPE=Release ../.. && cmake --build .)
- name: Test
shell: bash
run: build/release/test/unittest
linux-tarball:
name: Python 3 Tarball
runs-on: ubuntu-20.04
needs: linux-memory-leaks
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install
shell: bash
working-directory: tools/pythonpkg
run: pip install -r requirements-dev.txt
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}
save: ${{ env.CCACHE_SAVE }}
- name: Build
shell: bash
run: |
python --version
git archive --format zip --output test-tarball.zip HEAD
mkdir duckdb-test-tarball
mv test-tarball.zip duckdb-test-tarball
cd duckdb-test-tarball
unzip test-tarball.zip
cd tools/pythonpkg
export SETUPTOOLS_SCM_PRETEND_VERSION=0.2.2
export DISTUTILS_C_COMPILER_LAUNCHER=ccache
pip install . --user
(cd tests/ && python -m pytest)
python-address-sanitizer:
name: Python Address Sanitizer (Linux)
runs-on: ubuntu-latest
needs: linux-memory-leaks
container: ubuntu:18.04
env:
GEN: ninja
DUCKDEBUG: 1
ASAN_OPTIONS: detect_leaks=0
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: ./.github/actions/ubuntu_18_setup
with:
openssl: 0
python: 0
- name: Install OpenSSL
shell: bash
run: |
apt-get install -y -qq libbz2-dev
wget https://www.openssl.org/source/openssl-1.0.2o.tar.gz
tar -xf openssl-1.0.2o.tar.gz
cd openssl-1.0.2o
./config shared --prefix=/usr/local/
make
make install
mkdir lib
cp ./*.{so,so.1.0.0,a,pc} ./lib
- name: Install Python 3 With Sanitizers
shell: bash
run: |
wget https://github.com/python/cpython/archive/refs/tags/v3.9.12.zip
unzip v3.9.12.zip
rm v3.9.12.zip
cd cpython-3.9.12
mkdir debug-build
export LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/openssl-1.0.2o/lib
./configure --with-ensurepip=install --with-address-sanitizer --with-undefined-behavior-sanitizer --with-openssl=${GITHUB_WORKSPACE}/openssl-1.0.2o --prefix=${GITHUB_WORKSPACE}/debug-build
make -j
make install
- name: Install Python Test Dependencies
shell: bash
run: |
cd tools/pythonpkg
${GITHUB_WORKSPACE}/debug-build/bin/python3 -m pip install -r requirements-dev.txt
- name: Install DuckDB Python
shell: bash
run: |
cd tools/pythonpkg
${GITHUB_WORKSPACE}/debug-build/bin/python3 -m pip install .
# explicitly skip the TestMemoryLeaks tests, as the address sanitizer will increase the memory usage, causing the test to fail
- name: Run Python Tests
shell: bash
run: |
${GITHUB_WORKSPACE}/debug-build/bin/python3 -m pytest tools/pythonpkg/tests/fast --skiplist TestMemoryLeaks
python-sqllogic:
name: Python SQLLogicTest Library (Linux)
runs-on: ubuntu-20.04
env:
GEN: ninja
DUCKDEBUG: 1
ASAN_OPTIONS: detect_leaks=0
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: actions/setup-python@v5
with:
python-version: '3.10'
- name: Install Python Test Dependencies
shell: bash
run: |
cd tools/pythonpkg
python3 -m pip install -r requirements-dev.txt
- name: Install DuckDB Python
shell: bash
run: |
cd tools/pythonpkg
python3 -m pip install .
- name: Run Python SQLLogicTester
shell: bash
run: |
python3 tools/pythonpkg/scripts/sqllogictest_python.py
sqllogic:
name: Sqllogic tests
runs-on: ubuntu-latest # Secondary task of this CI job is to test building duckdb on latest ubuntu
needs: linux-memory-leaks
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@v1.2.11 # Note: pinned due to GLIBC incompatibility in later releases
with:
key: ${{ github.job }}
save: ${{ env.CCACHE_SAVE }}
# Build is implied by 'make sqlite' that will invoke implicitly 'make release' (we make it explicit)
- name: Build
shell: bash
run: make release
- name: Test
shell: bash
run: make sqlite
storage-initialization:
name: Storage Initialization Verification
runs-on: ubuntu-20.04
needs: linux-memory-leaks
env:
CC: gcc-10
CXX: g++-10
GEN: ninja
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Install
shell: bash
run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}
save: ${{ env.CCACHE_SAVE }}
- name: Build
shell: bash
run: make debug
- name: Test
shell: bash
run: python3 scripts/test_zero_initialize.py
extension-updating:
name: Extension updating test
runs-on: ubuntu-20.04
needs: linux-memory-leaks
env:
CC: gcc-10
CXX: g++-10
GEN: ninja
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Install
shell: bash
run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build
- uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: Install
shell: bash
run: pip install awscli
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}
save: ${{ env.CCACHE_SAVE }}
- name: Build
shell: bash
run: BUILD_TPCH=1 make
- name: Start Minio
shell: bash
run: |
sudo ./scripts/install_s3_test_server.sh
./scripts/generate_presigned_url.sh
source ./scripts/run_s3_test_server.sh
source ./scripts/set_s3_test_server_variables.sh
sleep 60
- name: Build
shell: bash
run: |
./scripts/run_extension_medata_tests.sh
no-string-inline:
name: No String Inline / Destroy Unpinned Blocks
runs-on: ubuntu-20.04
needs: linux-memory-leaks
env:
CC: gcc-10
CXX: g++-10
GEN: ninja
BUILD_ICU: 1
BUILD_PARQUET: 1
BUILD_TPCH: 1
BUILD_TPCDS: 1
BUILD_FTS: 1
BUILD_JSON: 1
CORE_EXTENSIONS: "inet"
DISABLE_STRING_INLINE: 1
DESTROY_UNPINNED_BLOCKS: 1
ALTERNATIVE_VERIFY: 1
LSAN_OPTIONS: suppressions=${{ github.workspace }}/.sanitizer-leak-suppressions.txt
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Install
shell: bash
run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}
save: ${{ env.CCACHE_SAVE }}
- name: Build
shell: bash
run: make relassert
- name: Test
shell: bash
run: build/relassert/test/unittest
latest-storage:
name: Latest Storage
runs-on: ubuntu-20.04
needs: linux-memory-leaks
env:
CC: gcc-10
CXX: g++-10
GEN: ninja
BUILD_PARQUET: 1
BUILD_TPCH: 1
BUILD_TPCDS: 1
BUILD_JSON: 1
LATEST_STORAGE: 1
BLOCK_VERIFICATION: 1
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Install
shell: bash
run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}
save: ${{ env.CCACHE_SAVE }}
- name: Build
shell: bash
run: make reldebug
- name: Test
shell: bash
run: |
python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest "*" --no-exit --time_execution
vector-verification:
name: Vector Verification Tests (${{ matrix.vector_type }})
runs-on: ubuntu-20.04
needs: linux-memory-leaks
strategy:
fail-fast: false
matrix:
include:
- vector_type: "dictionary_expression"
test_parameter: '"*"'
- vector_type: "dictionary_operator"
test_parameter: '' # skip running all tests here because it is too slow
- vector_type: "constant_operator"
test_parameter: '' # skip running all tests here because it is too slow
- vector_type: "sequence_operator"
test_parameter: '' # skip running all tests here because it is too slow
- vector_type: "nested_shuffle"
test_parameter: '' # skip running all tests here because it is too slow
env:
CC: gcc-10
CXX: g++-10
GEN: ninja
BUILD_ICU: 1
BUILD_PARQUET: 1
BUILD_TPCH: 1
BUILD_TPCDS: 1
BUILD_FTS: 1
BUILD_JSON: 1
CORE_EXTENSIONS: "inet"
VERIFY_VECTOR: ${{ matrix.vector_type }}
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Install
shell: bash
run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}
save: ${{ env.CCACHE_SAVE }}
- name: Build
shell: bash
run: make reldebug
- name: Test
shell: bash
run: python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest ${{ matrix.test_parameter }} --no-exit
force-blocking-sink-source:
name: Forcing async Sinks/Sources
runs-on: ubuntu-20.04
needs: linux-memory-leaks
env:
CC: gcc-10
CXX: g++-10
GEN: ninja
BUILD_ICU: 1
BUILD_PARQUET: 1
BUILD_TPCH: 1
BUILD_TPCDS: 1
BUILD_FTS: 1
BUILD_JSON: 1
CORE_EXTENSIONS: "inet"
FORCE_ASYNC_SINK_SOURCE: 1
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Install
shell: bash
run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}
save: ${{ env.CCACHE_SAVE }}
- name: Build
shell: bash
run: make relassert
- name: Test
shell: bash
run: python3 scripts/run_tests_one_by_one.py build/relassert/test/unittest --no-exit --timeout 600
regression-test-memory-safety:
name: Regression Tests between safe and unsafe builds
runs-on: ubuntu-20.04
needs: linux-memory-leaks
env:
CC: gcc-10
CXX: g++-10
GEN: ninja
BUILD_BENCHMARK: 1
BUILD_TPCH: 1
BUILD_TPCDS: 1
BUILD_HTTPFS: 1
BUILD_JEMALLOC: 1
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Checkout tools repo
uses: actions/checkout@v3
with:
fetch-depth: 0
path: unsafe
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install
shell: bash
run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build && pip install requests
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}
save: ${{ env.CCACHE_SAVE }}
- name: Build
shell: bash
run: |
make
cd unsafe
UNSAFE_NUMERIC_CAST=1 DISABLE_MEMORY_SAFETY=1 make
- name: Set up benchmarks
shell: bash
run: |
cp -r benchmark unsafe/
- name: Regression Test Micro
if: always()
shell: bash
run: |
python scripts/regression_test_runner.py --old=unsafe/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/micro.csv --verbose --threads=2
- name: Regression Test TPCH
if: always()
shell: bash
run: |
python scripts/regression_test_runner.py --old=unsafe/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/tpch.csv --verbose --threads=2
- name: Regression Test TPCDS
if: always()
shell: bash
run: |
python scripts/regression_test_runner.py --old=unsafe/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/tpcds.csv --verbose --threads=2
- name: Regression Test H2OAI
if: always()
shell: bash
run: |
python scripts/regression_test_runner.py --old=unsafe/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/h2oai.csv --verbose --threads=2
- name: Regression Test IMDB
if: always()
shell: bash
run: |
python scripts/regression_test_runner.py --old=unsafe/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/imdb.csv --verbose --threads=2
threadsan:
name: Thread Sanitizer
runs-on: ubuntu-24.04
needs: linux-memory-leaks
env:
CC: clang
CXX: clang++
GEN: ninja
BUILD_ICU: 1
BUILD_TPCH: 1
BUILD_TPCDS: 1
BUILD_FTS: 1
BUILD_JSON: 1
BUILD_JEMALLOC: 1
CORE_EXTENSIONS: "inet"
TSAN_OPTIONS: suppressions=${{ github.workspace }}/.sanitizer-thread-suppressions.txt
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Install
shell: bash
run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build clang
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}
save: ${{ env.CCACHE_SAVE }}
- name: Build
shell: bash
run: THREADSAN=1 make reldebug
- name: Test
shell: bash
run: |
python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest --no-exit --timeout 600
python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest "[intraquery]" --no-exit --timeout 600
python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest "[interquery]" --no-exit --timeout 1800
python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest "[interquery]" --no-exit --timeout 1800 --force-storage
python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest "[interquery]" --no-exit --timeout 1800 --force-storage --force-reload
python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest "[detailed_profiler]" --no-exit --timeout 600
python3 scripts/run_tests_one_by_one.py build/reldebug/test/unittest test/sql/tpch/tpch_sf01.test_slow --no-exit --timeout 600
vector-sizes:
name: Vector Sizes
runs-on: ubuntu-20.04
needs: linux-memory-leaks
env:
CC: gcc-10
CXX: g++-10
GEN: ninja
STANDARD_VECTOR_SIZE: 2
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Install
shell: bash
run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}
save: ${{ env.CCACHE_SAVE }}
- name: Build
shell: bash
run: make relassert
- name: Test
shell: bash
run: |
python3 scripts/run_tests_one_by_one.py build/relassert/test/unittest --no-exit --time_execution
block-sizes:
name: Block Sizes
runs-on: ubuntu-20.04
needs: linux-memory-leaks
env:
CC: gcc-10
CXX: g++-10
GEN: ninja
BLOCK_ALLOC_SIZE: 16384
BUILD_JSON: 1
BUILD_PARQUET: 1
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Install
shell: bash
run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}
save: ${{ env.CCACHE_SAVE }}
- name: Build with standard vector size
shell: bash
run: make relassert
- name: Fast and storage tests
shell: bash
run: |
python3 scripts/run_tests_one_by_one.py ./build/relassert/test/unittest --no-exit --time_execution
python3 scripts/run_tests_one_by_one.py ./build/relassert/test/unittest "test/sql/storage/*" --no-exit --time_execution
- name: Build with vector size of 512
shell: bash
run: rm -rf ./build && rm -rf ./duckdb_unittest_tempdir && make clean && STANDARD_VECTOR_SIZE=512 make relassert
- name: Fast and storage tests
shell: bash
run: |
python3 scripts/run_tests_one_by_one.py ./build/relassert/test/unittest --no-exit --time_execution
python3 scripts/run_tests_one_by_one.py ./build/relassert/test/unittest "test/sql/storage/*" --no-exit --time_execution
linux-wasm-experimental:
name: WebAssembly duckdb-wasm builds
runs-on: ubuntu-22.04
steps:
- uses: mymindstorm/setup-emsdk@v12
with:
version: 'latest'
- name: Setup
shell: bash
run: |
git clone https://github.com/duckdb/duckdb-wasm
cd duckdb-wasm
git checkout ${{ env.DUCKDB_WASM_VERSION }}
shopt -s nullglob
for filename in ../.github/patches/duckdb-wasm/*.patch; do
git apply $filename
done
git submodule init
git submodule update
git rm -r submodules/duckdb
- uses: actions/checkout@v3
with:
fetch-depth: 0
path: duckdb-wasm/submodules/duckdb
- name: Setup
shell: bash
run: |
cd duckdb-wasm
make patch_duckdb || echo "done"
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}
save: ${{ env.CCACHE_SAVE }}
- name: Print version
shell: bash
run: |
emcc --version
- name: Build WebAssembly MVP
shell: bash
run: |
cd duckdb-wasm
bash scripts/wasm_build_lib.sh relsize mvp $(pwd)/submodules/duckdb
- name: Build WebAssembly EH
shell: bash
run: |
cd duckdb-wasm
bash scripts/wasm_build_lib.sh relsize eh $(pwd)/submodules/duckdb
- name: Build WebAssembly COI
shell: bash
run: |
cd duckdb-wasm
bash scripts/wasm_build_lib.sh relsize coi $(pwd)/submodules/duckdb
- name: Package
shell: bash
run: |
zip -r duckdb-wasm32.zip duckdb-wasm/packages/duckdb-wasm/src/bindings
- uses: actions/upload-artifact@v3
with:
name: duckdb-wasm32
path: |
duckdb-wasm32.zip
codecov:
name: Code Coverage
runs-on: ubuntu-20.04
needs: linux-memory-leaks
strategy:
fail-fast: false
env:
GEN: ninja
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Install
shell: bash
run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build lcov curl g++ zip
- name: Set up Python 3.9
uses: actions/setup-python@v5
with:
python-version: '3.9'
- name: Install pytest
run: |
python3 -m pip install pytest
- name: Check Coverage
shell: bash
run: |
make coverage-check
- name: Create Archive
if: always()
shell: bash
run: |
zip -r coverage.zip coverage_html
- uses: actions/upload-artifact@v3
if: always()
with:
name: coverage
path: coverage.zip
if-no-files-found: error
pyodide:
uses: ./.github/workflows/Pyodide.yml
secrets: inherit
linux-httpfs:
name: Linux HTTPFS
runs-on: ubuntu-20.04
env:
BUILD_HTTPFS: 1
BUILD_TPCH: 1
BUILD_TPCDS: 1
BUILD_PARQUET: 1
BUILD_JSON: 1
S3_TEST_SERVER_AVAILABLE: 1
AWS_DEFAULT_REGION: eu-west-1
AWS_ACCESS_KEY_ID: minio_duckdb_user
AWS_SECRET_ACCESS_KEY: minio_duckdb_user_password
DUCKDB_S3_ENDPOINT: duckdb-minio.com:9000
DUCKDB_S3_USE_SSL: false
HTTP_PROXY_PUBLIC: localhost:3128
GEN: ninja
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install Ninja
shell: bash
run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build
# TODO: fix the authenticated proxy here
- name: Install and run http proxy squid
shell: bash
run: |
sudo apt-get install squid
./scripts/run_squid.sh --port 3128 --log_dir squid_logs &
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}
save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}
- name: Build
shell: bash
run: make
- name: Start test server & run tests
shell: bash
run: |
sudo ./scripts/install_s3_test_server.sh
./scripts/generate_presigned_url.sh
source ./scripts/run_s3_test_server.sh
source ./scripts/set_s3_test_server_variables.sh
sleep 60
python3 scripts/get_test_list.py --file-contains 'require httpfs' --list '"*"' > test.list
python3 scripts/run_tests_one_by_one.py ./build/release/test/unittest '-f test.list'
python3 scripts/run_tests_one_by_one.py ./build/release/test/unittest '[secret]'