From 7a841e066eb4c677a86bef87afb8469558189443 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Wed, 22 May 2024 11:14:21 +0200 Subject: [PATCH 1/7] Mark correct database as modified in CreateIndex --- src/planner/binder/statement/bind_create.cpp | 5 ++--- test/sql/attach/attach_create_index.test | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 test/sql/attach/attach_create_index.test diff --git a/src/planner/binder/statement/bind_create.cpp b/src/planner/binder/statement/bind_create.cpp index eb31c9529518..ece53e6d7b73 100644 --- a/src/planner/binder/statement/bind_create.cpp +++ b/src/planner/binder/statement/bind_create.cpp @@ -613,9 +613,6 @@ BoundStatement Binder::Bind(CreateStatement &stmt) { case CatalogType::INDEX_ENTRY: { auto &base = stmt.info->Cast(); - auto catalog = BindCatalog(base.catalog); - properties.modified_databases.insert(catalog); - // visit the table reference auto table_ref = make_uniq(); table_ref->catalog_name = base.catalog; @@ -631,6 +628,8 @@ BoundStatement Binder::Bind(CreateStatement &stmt) { if (table.temporary) { stmt.info->temporary = true; } + properties.modified_databases.insert(table.catalog.GetName()); + // create a plan over the bound table auto plan = CreatePlan(*bound_table); if (plan->type != LogicalOperatorType::LOGICAL_GET) { diff --git a/test/sql/attach/attach_create_index.test b/test/sql/attach/attach_create_index.test new file mode 100644 index 000000000000..1dbd031ae730 --- /dev/null +++ b/test/sql/attach/attach_create_index.test @@ -0,0 +1,14 @@ +# name: test/sql/attach/attach_create_index.test +# description: Test create index on an attached database with an alias +# group: [attach] + +require skip_reload + +statement ok +ATTACH '' AS tmp; + +statement ok +CREATE TABLE tmp.t1(id int); + +statement ok +CREATE INDEX idx ON tmp.t1(id); From b7c4d955bd93ab9add9d138caf9bea726354b376 Mon Sep 17 00:00:00 2001 From: gitccl Date: Wed, 22 May 2024 22:11:08 +0800 Subject: [PATCH 2/7] Fix move constants optimization --- src/optimizer/rule/move_constants.cpp | 4 ++++ test/optimizer/issue_12181.test | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 test/optimizer/issue_12181.test diff --git a/src/optimizer/rule/move_constants.cpp b/src/optimizer/rule/move_constants.cpp index 636265ff9131..29404164f7d2 100644 --- a/src/optimizer/rule/move_constants.cpp +++ b/src/optimizer/rule/move_constants.cpp @@ -41,6 +41,10 @@ unique_ptr MoveConstantsRule::Apply(LogicalOperator &op, vectorreturn_type.IsIntegral()); if (inner_constant.value.IsNull() || outer_constant.value.IsNull()) { + if (comparison.type == ExpressionType::COMPARE_DISTINCT_FROM || + comparison.type == ExpressionType::COMPARE_NOT_DISTINCT_FROM) { + return nullptr; + } return make_uniq(Value(comparison.return_type)); } auto &constant_type = outer_constant.return_type; diff --git a/test/optimizer/issue_12181.test b/test/optimizer/issue_12181.test new file mode 100644 index 000000000000..873b90cde7e6 --- /dev/null +++ b/test/optimizer/issue_12181.test @@ -0,0 +1,22 @@ +# name: test/optimizer/issue_12181.test +# description: Test move constants optimization involving DISTINCT FROM comparison +# group: [optimizer] + +statement ok +CREATE TABLE t0(c0 INT) + +statement ok +CREATE TABLE t1(c0 INT, c1 INT) + +statement ok +INSERT INTO t1(c0, c1) VALUES (0, 1) + +query I +SELECT NULL IS DISTINCT FROM (1 + t1.c1) FROM t1 NATURAL LEFT JOIN t0 +---- +true + +query II +SELECT * FROM t0 NATURAL RIGHT JOIN t1 WHERE (CASE t0.c0 WHEN t0.c0 THEN 1 ELSE NULL END) IS DISTINCT FROM (1 + (CASE t1.c1 WHEN t1.c1 THEN 2 ELSE NULL END)) +---- +0 1 From 9c4a294ed1b7a973d405295ecbefb92851c82101 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 22 May 2024 16:32:18 +0200 Subject: [PATCH 3/7] Move .ipp files to _ipp.hpp --- ...shared_from_this.ipp => enable_shared_from_this_ipp.hpp} | 0 src/include/duckdb/common/shared_ptr.hpp | 6 +++--- .../duckdb/common/{shared_ptr.ipp => shared_ptr_ipp.hpp} | 0 .../duckdb/common/{weak_ptr.ipp => weak_ptr_ipp.hpp} | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename src/include/duckdb/common/{enable_shared_from_this.ipp => enable_shared_from_this_ipp.hpp} (100%) rename src/include/duckdb/common/{shared_ptr.ipp => shared_ptr_ipp.hpp} (100%) rename src/include/duckdb/common/{weak_ptr.ipp => weak_ptr_ipp.hpp} (100%) diff --git a/src/include/duckdb/common/enable_shared_from_this.ipp b/src/include/duckdb/common/enable_shared_from_this_ipp.hpp similarity index 100% rename from src/include/duckdb/common/enable_shared_from_this.ipp rename to src/include/duckdb/common/enable_shared_from_this_ipp.hpp diff --git a/src/include/duckdb/common/shared_ptr.hpp b/src/include/duckdb/common/shared_ptr.hpp index 6d0910ca6bf7..eff60e3db2ec 100644 --- a/src/include/duckdb/common/shared_ptr.hpp +++ b/src/include/duckdb/common/shared_ptr.hpp @@ -37,9 +37,9 @@ struct compatible_with_t : std::is_convertible {}; // NOLINT: invalid } // namespace duckdb -#include "duckdb/common/shared_ptr.ipp" -#include "duckdb/common/weak_ptr.ipp" -#include "duckdb/common/enable_shared_from_this.ipp" +#include "duckdb/common/shared_ptr_ipp.hpp" +#include "duckdb/common/weak_ptr_ipp.hpp" +#include "duckdb/common/enable_shared_from_this_ipp.hpp" namespace duckdb { diff --git a/src/include/duckdb/common/shared_ptr.ipp b/src/include/duckdb/common/shared_ptr_ipp.hpp similarity index 100% rename from src/include/duckdb/common/shared_ptr.ipp rename to src/include/duckdb/common/shared_ptr_ipp.hpp diff --git a/src/include/duckdb/common/weak_ptr.ipp b/src/include/duckdb/common/weak_ptr_ipp.hpp similarity index 100% rename from src/include/duckdb/common/weak_ptr.ipp rename to src/include/duckdb/common/weak_ptr_ipp.hpp From a4f9ab4fc5dc93b6ddfbf327af4d26b14d47d61b Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Wed, 22 May 2024 18:44:38 +0200 Subject: [PATCH 4/7] FALLOC_FL_PUNCH_HOLE requires glibc 2.18 --- src/common/local_file_system.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/common/local_file_system.cpp b/src/common/local_file_system.cpp index cb15b5d683ed..d5b0fd6faa39 100644 --- a/src/common/local_file_system.cpp +++ b/src/common/local_file_system.cpp @@ -485,9 +485,14 @@ int64_t LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_byte bool LocalFileSystem::Trim(FileHandle &handle, idx_t offset_bytes, idx_t length_bytes) { #if defined(__linux__) + // FALLOC_FL_PUNCH_HOLE requires glibc 2.18 or up +#if __GLIBC__ < 2 || __GLIBC_MINOR__ < 18 + return false; +#else int fd = handle.Cast().fd; int res = fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, offset_bytes, length_bytes); return res == 0; +#endif #else return false; #endif From 8516607a855c0ca02c8a38ca8d165bd24977391d Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Wed, 22 May 2024 18:45:12 +0200 Subject: [PATCH 5/7] Like this --- src/common/local_file_system.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/local_file_system.cpp b/src/common/local_file_system.cpp index d5b0fd6faa39..e650d79bebb0 100644 --- a/src/common/local_file_system.cpp +++ b/src/common/local_file_system.cpp @@ -486,7 +486,7 @@ int64_t LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_byte bool LocalFileSystem::Trim(FileHandle &handle, idx_t offset_bytes, idx_t length_bytes) { #if defined(__linux__) // FALLOC_FL_PUNCH_HOLE requires glibc 2.18 or up -#if __GLIBC__ < 2 || __GLIBC_MINOR__ < 18 +#if __GLIBC__ < 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ < 18) return false; #else int fd = handle.Cast().fd; From e8b0a6b60f3438918964a21a1d571b1f4e24b4c2 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 22 May 2024 17:04:03 +0200 Subject: [PATCH 6/7] Test docker: Add test that cmake --install(ed) headers can be compiled --- scripts/test_docker_images.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/test_docker_images.sh b/scripts/test_docker_images.sh index c805ce897ead..ef8e993dae2a 100755 --- a/scripts/test_docker_images.sh +++ b/scripts/test_docker_images.sh @@ -8,6 +8,7 @@ make clean # Currently not working due to cmake version being too low # docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb amazonlinux:2 <<< "yum install gcc gcc-c++ git make cmake ninja-build -y && GEN=ninja make && $TEST" 2>&1 +docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb alpine:latest <<< "apk add g++ git make cmake ninja python3 && cmake -Bbuild . && cmake --build build && cmake --install build && g++ -std=c++11 examples/embedded-c++/main.cpp" docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb amazonlinux:latest <<< "yum install clang git make cmake ninja-build -y && GEN=ninja make && $TEST" 2>&1 docker run -i --platform linux/arm64 --rm -v $(pwd):/duckdb --workdir /duckdb alpine:latest <<< "apk add g++ git make cmake ninja && GEN=ninja make && $TEST" 2>&1 docker run -i --platform linux/amd64 --rm -v $(pwd):/duckdb --workdir /duckdb alpine:latest <<< "apk add g++ git make cmake ninja && GEN=ninja make && $TEST" 2>&1 From 11cce43f087cd90c0c912e286565bcc09499125d Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 22 May 2024 21:32:19 +0200 Subject: [PATCH 7/7] Move HTTPFS test to nightly given it's noisy --- .github/workflows/LinuxRelease.yml | 57 ------------------------------ .github/workflows/NightlyTests.yml | 54 ++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 57 deletions(-) diff --git a/.github/workflows/LinuxRelease.yml b/.github/workflows/LinuxRelease.yml index a1ff2fc4c169..1ab848d53ffb 100644 --- a/.github/workflows/LinuxRelease.yml +++ b/.github/workflows/LinuxRelease.yml @@ -381,63 +381,6 @@ jobs: shell: bash run: python3 scripts/exported_symbols_check.py build/release/src/libduckdb*.so - linux-httpfs: - name: Linux HTTPFS - if: ${{ inputs.skip_tests != 'true' }} - runs-on: ubuntu-20.04 - needs: linux-release-64 - env: - BUILD_HTTPFS: 1 - BUILD_TPCH: 1 - BUILD_TPCDS: 1 - BUILD_PARQUET: 1 - BUILD_JSON: 1 - S3_TEST_SERVER_AVAILABLE: 1 - AWS_DEFAULT_REGION: eu-west-1 - AWS_ACCESS_KEY_ID: minio_duckdb_user - AWS_SECRET_ACCESS_KEY: minio_duckdb_user_password - DUCKDB_S3_ENDPOINT: duckdb-minio.com:9000 - DUCKDB_S3_USE_SSL: false - - GEN: ninja - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - ref: ${{ inputs.git_ref }} - - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Install Ninja - shell: bash - run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build - - - name: Setup Ccache - uses: hendrikmuhs/ccache-action@main - with: - key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} - - - name: Build - shell: bash - run: make - - - name: Start test server & run tests - shell: bash - run: | - sudo ./scripts/install_s3_test_server.sh - ./scripts/generate_presigned_url.sh - source ./scripts/run_s3_test_server.sh - source ./scripts/set_s3_test_server_variables.sh - sleep 60 - - python3 scripts/get_test_list.py --file-contains 'require httpfs' --list '"*"' > test.list - python3 scripts/run_tests_one_by_one.py ./build/release/test/unittest '-f test.list' - python3 scripts/run_tests_one_by_one.py ./build/release/test/unittest '[secret]' - amalgamation-tests: name: Amalgamation Tests if: ${{ inputs.skip_tests != 'true' }} diff --git a/.github/workflows/NightlyTests.yml b/.github/workflows/NightlyTests.yml index 2393822ab6b2..d0cc77f80d0e 100644 --- a/.github/workflows/NightlyTests.yml +++ b/.github/workflows/NightlyTests.yml @@ -891,3 +891,57 @@ jobs: pyodide: uses: ./.github/workflows/Pyodide.yml secrets: inherit + + linux-httpfs: + name: Linux HTTPFS + runs-on: ubuntu-20.04 + env: + BUILD_HTTPFS: 1 + BUILD_TPCH: 1 + BUILD_TPCDS: 1 + BUILD_PARQUET: 1 + BUILD_JSON: 1 + S3_TEST_SERVER_AVAILABLE: 1 + AWS_DEFAULT_REGION: eu-west-1 + AWS_ACCESS_KEY_ID: minio_duckdb_user + AWS_SECRET_ACCESS_KEY: minio_duckdb_user_password + DUCKDB_S3_ENDPOINT: duckdb-minio.com:9000 + DUCKDB_S3_USE_SSL: false + + GEN: ninja + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install Ninja + shell: bash + run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build + + - name: Setup Ccache + uses: hendrikmuhs/ccache-action@main + with: + key: ${{ github.job }} + save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + + - name: Build + shell: bash + run: make + + - name: Start test server & run tests + shell: bash + run: | + sudo ./scripts/install_s3_test_server.sh + ./scripts/generate_presigned_url.sh + source ./scripts/run_s3_test_server.sh + source ./scripts/set_s3_test_server_variables.sh + sleep 60 + + python3 scripts/get_test_list.py --file-contains 'require httpfs' --list '"*"' > test.list + python3 scripts/run_tests_one_by_one.py ./build/release/test/unittest '-f test.list' + python3 scripts/run_tests_one_by_one.py ./build/release/test/unittest '[secret]'