Main candidate test

.github/actions/build_extensions/action.yml

@@ -81,6 +81,9 @@ inputs:
   unittest_script:
     description: 'Script/program to execute the unittests'
     default: 'python3 scripts/run_tests_one_by_one.py ./build/release/test/unittest'
+  cmake_flags:
+    description: 'Flags to be passed to cmake'
+    default: ''
 
 runs:
   using: "composite"
@@ -144,6 +147,8 @@ runs:
         GEN: ${{ inputs.ninja == 1 && 'ninja' || '' }}
         USE_MERGED_VCPKG_MANIFEST: 1
         DUCKDB_PLATFORM: ${{ inputs.duckdb_arch }}
+        CMAKE_VARS_BUILD: ${{ inputs.cmake_flags }}
+
       run: |
         ls
         mkdir -p ~/.ssh

.github/config/bundled_extensions.cmake

@@ -25,5 +25,4 @@ duckdb_extension_load(autocomplete)
 #
 ## Extensions that are not linked, but we do want to test them as part of the release build
 #
-duckdb_extension_load(sqlsmith DONT_LINK)
 duckdb_extension_load(tpcds DONT_LINK)

.github/config/in_tree_extensions.cmake

@@ -12,6 +12,5 @@ duckdb_extension_load(inet)
 duckdb_extension_load(icu)
 duckdb_extension_load(json)
 duckdb_extension_load(parquet)
-duckdb_extension_load(sqlsmith)
 duckdb_extension_load(tpcds)
 duckdb_extension_load(tpch)

.github/config/out_of_tree_extensions.cmake

@@ -100,12 +100,18 @@ duckdb_extension_load(sqlite_scanner
         GIT_TAG 50b7870be099186f195bc72bac5e9e11247ee2f9
         )
 
+duckdb_extension_load(sqlsmith
+        GIT_URL https://github.com/duckdb/duckdb_sqlsmith
+        GIT_TAG 721460ff1f31ce1dc1e4a9c4a55c0faf0b466dcb
+        )
+
 ################# SUBSTRAIT
 if (NOT WIN32)
     duckdb_extension_load(substrait
             LOAD_TESTS DONT_LINK
             GIT_URL https://github.com/duckdb/substrait
             GIT_TAG 237931391ebc7e6aee7aa81052fa1411f6c4128e
+            APPLY_PATCHES
             )
 endif()
 

.github/patches/extensions/substrait/pushdown_semi_anti.patch

@@ -0,0 +1,40 @@
+diff --git a/src/to_substrait.cpp b/src/to_substrait.cpp
+index de0ffb3..ece0f5d 100644
+--- a/src/to_substrait.cpp
++++ b/src/to_substrait.cpp
+@@ -864,7 +864,11 @@ substrait::Rel *DuckDBToSubstrait::TransformComparisonJoin(LogicalOperator &dop)
+ 	auto left_col_count = dop.children[0]->types.size();
+ 	if (dop.children[0]->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) {
+ 		auto child_join = (LogicalComparisonJoin *)dop.children[0].get();
+-		left_col_count = child_join->left_projection_map.size() + child_join->right_projection_map.size();
++		if (child_join->join_type != JoinType::SEMI && child_join->join_type != JoinType::ANTI) {
++			left_col_count = child_join->left_projection_map.size() + child_join->right_projection_map.size();
++		} else {
++			left_col_count = child_join->left_projection_map.size();
++		}
+ 	}
+ 	sjoin->set_allocated_expression(
+ 	    CreateConjunction(djoin.conditions, [&](JoinCondition &in) { return TransformJoinCond(in, left_col_count); }));
+diff --git a/test/sql/test_substrait_types.test b/test/sql/test_substrait_types.test
+index 2bc47da..f3d2c65 100644
+--- a/test/sql/test_substrait_types.test
++++ b/test/sql/test_substrait_types.test
+@@ -148,6 +148,9 @@ INSERT INTO timestz VALUES ('2021-11-26 10:15:13.123456+00');
+ statement ok
+ CALL get_substrait('SELECT s FROM timestz WHERE s > ''2001-11-26 05:02:23.123456+00'' ');
+
++# not supported because TIMETZ comparisons generate a UBIGINT
++mode skip
++
+ statement ok
+ CREATE table times_tz (s TIMETZ);
+
+@@ -157,6 +160,8 @@ INSERT INTO times_tz VALUES ('05:40:00.000001');
+ statement ok
+ CALL get_substrait('SELECT s FROM times_tz WHERE s = ''05:40:00.000001'' ');
+
++mode unskip
++
+ statement ok
+ CREATE table times (s TIME);
+

.github/regression/micro_extended.csv

@@ -1,3 +1,4 @@
+benchmark/micro/aggregate/any_value_uuid.benchmark
 benchmark/micro/aggregate/bitstring_aggregate.benchmark
 benchmark/micro/aggregate/bitwise_aggregate.benchmark
 benchmark/micro/aggregate/grouped_distinct.benchmark
@@ -98,6 +99,8 @@ benchmark/micro/date/extract_multiple_icu.benchmark
 benchmark/micro/date/extract_struct.benchmark
 benchmark/micro/date/extract_struct_icu.benchmark
 benchmark/micro/date/extract_year.benchmark
+benchmark/micro/distinct/distinct_on_pushdown.benchmark
+benchmark/micro/filter/in_conversion.benchmark
 benchmark/micro/filter/parallel_complex_filter.benchmark
 benchmark/micro/functions/random.benchmark
 benchmark/micro/functions/random_uuid.benchmark
@@ -127,6 +130,7 @@ benchmark/micro/join/asof_join.benchmark
 benchmark/micro/join/asof_join_small_probe.benchmark
 benchmark/micro/join/blockwise_nl_join.benchmark
 benchmark/micro/join/delim_join_no_blowup.benchmark
+benchmark/micro/join/hashjoin_dups_rhs.benchmark
 benchmark/micro/join/hashjoin_highcardinality.benchmark
 benchmark/micro/join/hashjoin_lhsarithmetic.benchmark
 benchmark/micro/join/iejoin_employees.benchmark
@@ -160,7 +164,9 @@ benchmark/micro/list/column_data_collection_copy/small_lists.benchmark
 benchmark/micro/list/column_data_collection_copy/unnest.benchmark
 benchmark/micro/list/list_few_rows.benchmark
 benchmark/micro/list/list_many_rows.benchmark
+benchmark/micro/list/list_min_max.benchmark
 benchmark/micro/list/list_order_by.benchmark
+benchmark/micro/list/list_value.benchmark
 benchmark/micro/list/string_split.benchmark
 benchmark/micro/list/string_split_regexp.benchmark
 benchmark/micro/list/string_split_unicode.benchmark
@@ -170,8 +176,10 @@ benchmark/micro/map/map_extract.benchmark
 benchmark/micro/map/map_lookup.benchmark
 benchmark/micro/nulls/no_nulls_addition.benchmark
 benchmark/micro/nulls/null_addition.benchmark
+benchmark/micro/optimizer/topn_optimization.benchmark
 benchmark/micro/order/orderby.benchmark
 benchmark/micro/pushdown/or_pushdown.benchmark
+benchmark/micro/pushdown/window_partition_pushdown.benchmark
 benchmark/micro/result_collection/batched_stream_query.benchmark
 benchmark/micro/simd/auto-vectorization.benchmark
 benchmark/micro/string/bitstring.benchmark
@@ -196,6 +204,7 @@ benchmark/micro/string/string_agg_order_by.benchmark
 benchmark/micro/string/string_agg_short.benchmark
 benchmark/micro/timestamp/extract_month.benchmark
 benchmark/micro/timestamp/extract_year.benchmark
+benchmark/micro/update/update_with_join.benchmark
 benchmark/micro/view/many_values.benchmark
 benchmark/micro/window/window_constant_aggregates.benchmark
 benchmark/micro/window/window_constant_count.benchmark

.github/workflows/Android.yml

@@ -0,0 +1,111 @@
+name: Android
+on:
+  workflow_call:
+    inputs:
+      override_git_describe:
+        type: string
+      git_ref:
+        type: string
+      skip_tests:
+        type: string
+  workflow_dispatch:
+    inputs:
+      override_git_describe:
+        type: string
+      git_ref:
+        type: string
+      skip_tests:
+        type: string
+  repository_dispatch:
+  push:
+    branches:
+      - '**'
+      - '!main'
+      - '!feature'
+    paths-ignore:
+      - '**.md'
+      - 'tools/**'
+      - '!tools/odbc/**'
+      - '!tools/shell/**'
+      - '.github/patches/duckdb-wasm/**'
+      - '.github/workflows/**'
+      - '!.github/workflows/Android.yml'
+
+  pull_request:
+    types: [opened, reopened, ready_for_review]
+    paths-ignore:
+      - '**.md'
+      - 'tools/**'
+      - '!tools/odbc/**'
+      - '!tools/shell/**'
+      - '.github/patches/duckdb-wasm/**'
+      - '.github/workflows/**'
+      - '!.github/workflows/Android.yml'
+
+
+concurrency:
+  group: android-${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}-${{ inputs.override_git_describe }}
+  cancel-in-progress: true
+
+env:
+  GH_TOKEN: ${{ secrets.GH_TOKEN }}
+  OVERRIDE_GIT_DESCRIBE: ${{ inputs.override_git_describe }}
+
+jobs:
+  android:
+    name: Android
+    runs-on: ubuntu-latest
+    container: ubuntu:18.04
+
+    strategy:
+      matrix:
+        arch: [ armeabi-v7a, arm64-v8a ]
+
+    env:
+      ENABLE_EXTENSION_AUTOLOADING: 1
+      ENABLE_EXTENSION_AUTOINSTALL: 1
+      GEN: ninja
+      EXTENSION_STATIC_BUILD: 1
+      DUCKDB_PLATFORM: android_${{ matrix.arch}}
+      DUCKDB_CUSTOM_PLATFORM: android_${{ matrix.arch}}
+      CMAKE_VARS_BUILD: -DBUILD_UNITTESTS=0 -DBUILD_SHELL=0 -DANDROID_ABI=${{ matrix.arch}} -DCMAKE_TOOLCHAIN_FILE=./android-ndk/build/cmake/android.toolchain.cmake
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        ref: ${{ inputs.git_ref }}
+
+    - uses: ./.github/actions/ubuntu_18_setup
+      with:
+        ccache: 1
+
+    - name: Checkout (again)
+      shell: bash
+      run: git checkout ${{ inputs.git_ref }}
+
+    - name: Install Android NDK
+      shell: bash
+      run: |
+        wget https://dl.google.com/android/repository/android-ndk-r26d-linux.zip
+        unzip android-ndk-r26d-linux.zip
+        mv android-ndk-r26d android-ndk
+
+    - name:  Build
+      shell: bash
+      run: make
+
+    - name: Deploy
+      shell: bash
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }}
+      run: |
+        zip -j libduckdb-android_${{matrix.arch}}.zip build/release/src/libduckdb*.*  src/include/duckdb.h
+        ./scripts/upload-assets-to-staging.sh github_release libduckdb-android_${{matrix.arch}}.zip
+
+    - uses: actions/upload-artifact@v3
+      with:
+        name: duckdb-binaries-android-${{matrix.arch}}
+        path: |
+          libduckdb-android_${{matrix.arch}}.zip

.github/workflows/CodeQuality.yml

@@ -47,7 +47,7 @@ jobs:
 
       - name: Install
         shell: bash
-        run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build clang-format-11 && sudo pip3 install cmake-format black
+        run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build clang-format-11 && sudo pip3 install cmake-format black libclang==16.0.6
 
       - name: Format Check
         shell: bash

.github/workflows/LinuxRelease.yml

@@ -384,63 +384,6 @@ jobs:
       shell: bash
       run: python3 scripts/exported_symbols_check.py build/release/src/libduckdb*.so
 
- linux-httpfs:
-    name: Linux HTTPFS
-    if: ${{ inputs.skip_tests != 'true' }}
-    runs-on: ubuntu-20.04
-    needs: linux-release-64
-    env:
-      BUILD_HTTPFS: 1
-      BUILD_TPCH: 1
-      BUILD_TPCDS: 1
-      BUILD_PARQUET: 1
-      BUILD_JSON: 1
-      S3_TEST_SERVER_AVAILABLE: 1
-      AWS_DEFAULT_REGION: eu-west-1
-      AWS_ACCESS_KEY_ID: minio_duckdb_user
-      AWS_SECRET_ACCESS_KEY: minio_duckdb_user_password
-      DUCKDB_S3_ENDPOINT: duckdb-minio.com:9000
-      DUCKDB_S3_USE_SSL: false
-
-      GEN: ninja
-
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        ref: ${{ inputs.git_ref }}
-
-    - uses: actions/setup-python@v5
-      with:
-        python-version: '3.12'
-
-    - name: Install Ninja
-      shell: bash
-      run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build
-
-    - name: Setup Ccache
-      uses: hendrikmuhs/ccache-action@main
-      with:
-        key: ${{ github.job }}
-        save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}
-
-    - name: Build
-      shell: bash
-      run: make
-
-    - name: Start test server & run tests
-      shell: bash
-      run: |
-        sudo ./scripts/install_s3_test_server.sh
-        ./scripts/generate_presigned_url.sh
-        source ./scripts/run_s3_test_server.sh
-        source ./scripts/set_s3_test_server_variables.sh
-        sleep 60
-
-        python3 scripts/get_test_list.py --file-contains 'require httpfs' --list '"*"' > test.list
-        python3 scripts/run_tests_one_by_one.py ./build/release/test/unittest '-f test.list'
-        python3 scripts/run_tests_one_by_one.py ./build/release/test/unittest '[secret]'  
-
  amalgamation-tests:
     name: Amalgamation Tests
     if: ${{ inputs.skip_tests != 'true' }}

.github/workflows/NightlyTests.yml

@@ -881,3 +881,57 @@ jobs:
   pyodide:
     uses: ./.github/workflows/Pyodide.yml
     secrets: inherit
+
+  linux-httpfs:
+    name: Linux HTTPFS
+    runs-on: ubuntu-20.04
+    env:
+      BUILD_HTTPFS: 1
+      BUILD_TPCH: 1
+      BUILD_TPCDS: 1
+      BUILD_PARQUET: 1
+      BUILD_JSON: 1
+      S3_TEST_SERVER_AVAILABLE: 1
+      AWS_DEFAULT_REGION: eu-west-1
+      AWS_ACCESS_KEY_ID: minio_duckdb_user
+      AWS_SECRET_ACCESS_KEY: minio_duckdb_user_password
+      DUCKDB_S3_ENDPOINT: duckdb-minio.com:9000
+      DUCKDB_S3_USE_SSL: false
+
+      GEN: ninja
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+
+    - name: Install Ninja
+      shell: bash
+      run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build
+
+    - name: Setup Ccache
+      uses: hendrikmuhs/ccache-action@main
+      with:
+        key: ${{ github.job }}
+        save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}
+
+    - name: Build
+      shell: bash
+      run: make
+
+    - name: Start test server & run tests
+      shell: bash
+      run: |
+        sudo ./scripts/install_s3_test_server.sh
+        ./scripts/generate_presigned_url.sh
+        source ./scripts/run_s3_test_server.sh
+        source ./scripts/set_s3_test_server_variables.sh
+        sleep 60
+
+        python3 scripts/get_test_list.py --file-contains 'require httpfs' --list '"*"' > test.list
+        python3 scripts/run_tests_one_by_one.py ./build/release/test/unittest '-f test.list'
+        python3 scripts/run_tests_one_by_one.py ./build/release/test/unittest '[secret]'