Skip to content

Commit

Permalink
Merge branch 'main' into join_limit
Browse files Browse the repository at this point in the history
  • Loading branch information
lnkuiper committed Aug 23, 2024
2 parents 4c6d78c + 7abb706 commit ffe4c67
Show file tree
Hide file tree
Showing 294 changed files with 9,657 additions and 4,340 deletions.
8 changes: 3 additions & 5 deletions .github/config/out_of_tree_extensions.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ endif()
################# DELTA
# MinGW build is not available, and our current manylinux ci does not have enough storage space to run the rust build
# for Delta
if (NOT MINGW AND NOT "${OS_NAME}" STREQUAL "linux")
if (NOT MINGW AND NOT "${OS_NAME}" STREQUAL "linux" AND NOT WIN32)
duckdb_extension_load(delta
LOAD_TESTS
GIT_URL https://github.com/duckdb/duckdb_delta
Expand Down Expand Up @@ -111,8 +111,7 @@ endif()
duckdb_extension_load(sqlite_scanner
${STATIC_LINK_SQLITE} LOAD_TESTS
GIT_URL https://github.com/duckdb/sqlite_scanner
GIT_TAG 50b7870be099186f195bc72bac5e9e11247ee2f9
APPLY_PATCHES
GIT_TAG 1ee5404c4b9a62a572c9076ebf4cabb7f4181ea6
)

duckdb_extension_load(sqlsmith
Expand Down Expand Up @@ -147,7 +146,6 @@ if (NOT MINGW)
DONT_LINK
LOAD_TESTS
GIT_URL https://github.com/duckdb/duckdb_mysql
GIT_TAG 4dd5963cc5f7f04f81a0ea308b104e65791d9975
APPLY_PATCHES
GIT_TAG 6519c1953655b86f59884bb1b79f554b9fdf551b
)
endif()
60 changes: 0 additions & 60 deletions .github/patches/extensions/mysql_scanner/params_to_string.patch

This file was deleted.

90 changes: 0 additions & 90 deletions .github/patches/extensions/sqlite_scanner/params_to_string.patch

This file was deleted.

58 changes: 58 additions & 0 deletions .github/patches/extensions/vss/get_storage_info.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
diff --git a/src/hnsw/hnsw_index.cpp b/src/hnsw/hnsw_index.cpp
index 012a8ea..8d627a5 100644
--- a/src/hnsw/hnsw_index.cpp
+++ b/src/hnsw/hnsw_index.cpp
@@ -490,7 +490,7 @@ void HNSWIndex::PersistToDisk() {
is_dirty = false;
}

-IndexStorageInfo HNSWIndex::GetStorageInfo(const bool get_buffers) {
+IndexStorageInfo HNSWIndex::GetStorageInfo(const case_insensitive_map_t<Value> &options, const bool to_wal) {

PersistToDisk();

@@ -498,7 +498,7 @@ IndexStorageInfo HNSWIndex::GetStorageInfo(const bool get_buffers) {
info.name = name;
info.root = root_block_ptr.Get();

- if (!get_buffers) {
+ if (!to_wal) {
// use the partial block manager to serialize all allocator data
auto &block_manager = table_io_manager.GetIndexBlockManager();
PartialBlockManager partial_block_manager(block_manager, PartialBlockType::FULL_CHECKPOINT);
@@ -532,6 +532,10 @@ string HNSWIndex::VerifyAndToString(IndexLock &state, const bool only_verify) {
throw NotImplementedException("HNSWIndex::VerifyAndToString() not implemented");
}

+void HNSWIndex::VerifyAllocations(IndexLock &state) {
+ throw NotImplementedException("HNSWIndex::VerifyAllocations() not implemented");
+}
+
//------------------------------------------------------------------------------
// Register Index Type
//------------------------------------------------------------------------------
diff --git a/src/include/hnsw/hnsw_index.hpp b/src/include/hnsw/hnsw_index.hpp
index bd0a60d..82e527e 100644
--- a/src/include/hnsw/hnsw_index.hpp
+++ b/src/include/hnsw/hnsw_index.hpp
@@ -72,7 +72,7 @@ public:
//! Insert a chunk of entries into the index
ErrorData Insert(IndexLock &lock, DataChunk &data, Vector &row_ids) override;

- IndexStorageInfo GetStorageInfo(const bool get_buffers) override;
+ IndexStorageInfo GetStorageInfo(const case_insensitive_map_t<Value> &options, const bool to_wal) override;
idx_t GetInMemorySize(IndexLock &state) override;

//! Merge another index into this index. The lock obtained from InitializeLock must be held, and the other
@@ -85,8 +85,10 @@ public:
//! Performs constraint checking for a chunk of input data
void CheckConstraintsForChunk(DataChunk &input, ConflictManager &conflict_manager) override;

- //! Returns the string representation of the HNSWIndex, or only traverses and verifies the index
+ //! Returns the string representation of the HNSWIndex, or only traverses and verifies the index.
string VerifyAndToString(IndexLock &state, const bool only_verify) override;
+ //! Ensures that the node allocation counts match the node counts.
+ void VerifyAllocations(IndexLock &state) override;

string GetConstraintViolationMessage(VerifyExistenceType verify_type, idx_t failed_index,
DataChunk &input) override {
36 changes: 36 additions & 0 deletions .github/workflows/NightlyTests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -494,6 +494,42 @@ jobs:
shell: bash
run: build/relassert/test/unittest

latest-storage:
name: Latest Storage
runs-on: ubuntu-20.04
needs: linux-memory-leaks
env:
CC: gcc-10
CXX: g++-10
GEN: ninja
BUILD_PARQUET: 1
BUILD_TPCH: 1
BUILD_TPCDS: 1
LATEST_STORAGE: 1

steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0

- name: Install
shell: bash
run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build

- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}
save: ${{ env.CCACHE_SAVE }}

- name: Build
shell: bash
run: make relassert

- name: Test
shell: bash
run: build/relassert/test/unittest "*"

vector-verification:
name: Vector Verification Tests (${{ matrix.vector_type }})
runs-on: ubuntu-20.04
Expand Down
4 changes: 4 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -455,6 +455,10 @@ if(ALTERNATIVE_VERIFY)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDUCKDB_ALTERNATIVE_VERIFY")
endif()

if(LATEST_STORAGE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDUCKDB_LATEST_STORAGE")
endif()

if(DEBUG_ALLOCATION)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDUCKDB_DEBUG_ALLOCATION")
endif()
Expand Down
3 changes: 3 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,9 @@ endif
ifeq (${ALTERNATIVE_VERIFY}, 1)
CMAKE_VARS:=${CMAKE_VARS} -DALTERNATIVE_VERIFY=1
endif
ifeq (${LATEST_STORAGE}, 1)
CMAKE_VARS:=${CMAKE_VARS} -DLATEST_STORAGE=1
endif
ifneq (${VERIFY_VECTOR}, )
CMAKE_VARS:=${CMAKE_VARS} -DVERIFY_VECTOR=${VERIFY_VECTOR}
endif
Expand Down
31 changes: 31 additions & 0 deletions benchmark/micro/index/insert/insert_pk_fk.benchmark
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
# name: benchmark/micro/index/insert/insert_pk_fk.benchmark
# description: Insert into multiple indexes
# group: [insert]

name Insert into PK and FK
group index

require tpcds

load
BEGIN TRANSACTION;
CALL dsdgen(sf=0.1);
pragma threads=8;
EXPORT DATABASE '${BENCHMARK_DIR}/sf1data';
ROLLBACK;
CREATE TABLE date_dim (d_date_sk Integer Not Null PRIMARY KEY, d_date_id String Not Null, d_date Date Not Null, d_month_seq Integer, d_week_seq Integer, d_quarter_seq Integer, d_year Integer, d_dow Integer, d_moy Integer, d_dom Integer, d_qoy Integer, d_fy_year Integer, d_fy_quarter_seq Integer, d_fy_week_seq Integer, d_day_name String, d_quarter_name String, d_holiday String, d_weekend String, d_following_holiday String, d_first_dom Integer, d_last_dom Integer, d_same_day_ly Integer, d_same_day_lq Integer, d_current_day String, d_current_week String, d_current_month String, d_current_quarter String, d_current_year String);
CREATE TABLE warehouse (w_warehouse_sk Integer Not Null PRIMARY KEY, w_warehouse_id String Not Null, w_warehouse_name String, w_warehouse_sq_ft Integer, w_street_number String, w_street_name String, w_street_type String, w_suite_number String, w_city String, w_county String, w_state String, w_zip String, w_country String, w_gmt_offset Decimal(5,2));
CREATE TABLE item (i_item_sk Integer Not Null PRIMARY KEY, i_item_id String Not Null, i_rec_start_date Date, i_rec_end_date Date, i_item_desc String, i_current_price Decimal(7,2), i_wholesale_cost Decimal(7,2), i_brand_id Integer, i_brand String, i_class_id Integer, i_class String, i_category_id Integer, i_category String, i_manufact_id Integer, i_manufact String, i_size String, i_formulation String, i_color String, i_units String, i_container String, i_manager_id Integer, i_product_name String);
CREATE TABLE inventory (inv_date_sk Integer Not Null REFERENCES date_dim (d_date_sk), inv_item_sk Integer Not Null REFERENCES item (i_item_sk), inv_warehouse_sk Integer Not Null REFERENCES warehouse (w_warehouse_sk), inv_quantity_on_hand Integer);

run
COPY date_dim FROM '${BENCHMARK_DIR}/sf1data/date_dim.csv' (FORMAT 'csv', header 1, delimiter ',', quote '"');
COPY item FROM '${BENCHMARK_DIR}/sf1data/item.csv' (FORMAT 'csv', header 1, delimiter ',', quote '"');
COPY warehouse FROM '${BENCHMARK_DIR}/sf1data/warehouse.csv' (FORMAT 'csv', header 1, delimiter ',', quote '"');
COPY inventory FROM '${BENCHMARK_DIR}/sf1data/inventory.csv' (FORMAT 'csv', header 1, delimiter ',', quote '"');

cleanup
CREATE OR REPLACE TABLE date_dim (d_date_sk Integer Not Null PRIMARY KEY, d_date_id String Not Null, d_date Date Not Null, d_month_seq Integer, d_week_seq Integer, d_quarter_seq Integer, d_year Integer, d_dow Integer, d_moy Integer, d_dom Integer, d_qoy Integer, d_fy_year Integer, d_fy_quarter_seq Integer, d_fy_week_seq Integer, d_day_name String, d_quarter_name String, d_holiday String, d_weekend String, d_following_holiday String, d_first_dom Integer, d_last_dom Integer, d_same_day_ly Integer, d_same_day_lq Integer, d_current_day String, d_current_week String, d_current_month String, d_current_quarter String, d_current_year String);
CREATE OR REPLACE TABLE warehouse (w_warehouse_sk Integer Not Null PRIMARY KEY, w_warehouse_id String Not Null, w_warehouse_name String, w_warehouse_sq_ft Integer, w_street_number String, w_street_name String, w_street_type String, w_suite_number String, w_city String, w_county String, w_state String, w_zip String, w_country String, w_gmt_offset Decimal(5,2));
CREATE OR REPLACE TABLE item (i_item_sk Integer Not Null PRIMARY KEY, i_item_id String Not Null, i_rec_start_date Date, i_rec_end_date Date, i_item_desc String, i_current_price Decimal(7,2), i_wholesale_cost Decimal(7,2), i_brand_id Integer, i_brand String, i_class_id Integer, i_class String, i_category_id Integer, i_category String, i_manufact_id Integer, i_manufact String, i_size String, i_formulation String, i_color String, i_units String, i_container String, i_manager_id Integer, i_product_name String);
CREATE OR REPLACE TABLE inventory (inv_date_sk Integer Not Null REFERENCES date_dim (d_date_sk), inv_item_sk Integer Not Null REFERENCES item (i_item_sk), inv_warehouse_sk Integer Not Null REFERENCES warehouse (w_warehouse_sk), inv_quantity_on_hand Integer);
2 changes: 2 additions & 0 deletions extension/json/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ set(JSON_EXTENSION_FILES
json_functions/copy_json.cpp
json_functions/json_array_length.cpp
json_functions/json_contains.cpp
json_functions/json_exists.cpp
json_functions/json_extract.cpp
json_functions/json_keys.cpp
json_functions/json_merge_patch.cpp
Expand All @@ -27,6 +28,7 @@ set(JSON_EXTENSION_FILES
json_functions/json_create.cpp
json_functions/json_type.cpp
json_functions/json_valid.cpp
json_functions/json_value.cpp
json_functions/json_serialize_sql.cpp
json_functions/json_serialize_plan.cpp
json_functions/read_json.cpp
Expand Down
Loading

0 comments on commit ffe4c67

Please sign in to comment.