Merge branch 'main' into join_limit

.github/config/out_of_tree_extensions.cmake

@@ -46,7 +46,7 @@ endif()
 ################# DELTA
 # MinGW build is not available, and our current manylinux ci does not have enough storage space to run the rust build
 # for Delta
-if (NOT MINGW AND NOT "${OS_NAME}" STREQUAL "linux")
+if (NOT MINGW AND NOT "${OS_NAME}" STREQUAL "linux" AND NOT WIN32)
     duckdb_extension_load(delta
             LOAD_TESTS
             GIT_URL https://github.com/duckdb/duckdb_delta
@@ -111,8 +111,7 @@ endif()
 duckdb_extension_load(sqlite_scanner
         ${STATIC_LINK_SQLITE} LOAD_TESTS
         GIT_URL https://github.com/duckdb/sqlite_scanner
-        GIT_TAG 50b7870be099186f195bc72bac5e9e11247ee2f9
-        APPLY_PATCHES
+        GIT_TAG 1ee5404c4b9a62a572c9076ebf4cabb7f4181ea6
         )
 
 duckdb_extension_load(sqlsmith
@@ -147,7 +146,6 @@ if (NOT MINGW)
             DONT_LINK
             LOAD_TESTS
             GIT_URL https://github.com/duckdb/duckdb_mysql
-            GIT_TAG 4dd5963cc5f7f04f81a0ea308b104e65791d9975
-            APPLY_PATCHES
+            GIT_TAG 6519c1953655b86f59884bb1b79f554b9fdf551b
             )
 endif()

.github/patches/extensions/vss/get_storage_info.patch

@@ -0,0 +1,58 @@
+diff --git a/src/hnsw/hnsw_index.cpp b/src/hnsw/hnsw_index.cpp
+index 012a8ea..8d627a5 100644
+--- a/src/hnsw/hnsw_index.cpp
++++ b/src/hnsw/hnsw_index.cpp
+@@ -490,7 +490,7 @@ void HNSWIndex::PersistToDisk() {
+ 	is_dirty = false;
+ }
+
+-IndexStorageInfo HNSWIndex::GetStorageInfo(const bool get_buffers) {
++IndexStorageInfo HNSWIndex::GetStorageInfo(const case_insensitive_map_t<Value> &options, const bool to_wal) {
+
+ 	PersistToDisk();
+
+@@ -498,7 +498,7 @@ IndexStorageInfo HNSWIndex::GetStorageInfo(const bool get_buffers) {
+ 	info.name = name;
+ 	info.root = root_block_ptr.Get();
+
+-	if (!get_buffers) {
++	if (!to_wal) {
+ 		// use the partial block manager to serialize all allocator data
+ 		auto &block_manager = table_io_manager.GetIndexBlockManager();
+ 		PartialBlockManager partial_block_manager(block_manager, PartialBlockType::FULL_CHECKPOINT);
+@@ -532,6 +532,10 @@ string HNSWIndex::VerifyAndToString(IndexLock &state, const bool only_verify) {
+ 	throw NotImplementedException("HNSWIndex::VerifyAndToString() not implemented");
+ }
+
++void HNSWIndex::VerifyAllocations(IndexLock &state) {
++	throw NotImplementedException("HNSWIndex::VerifyAllocations() not implemented");
++}
++
+ //------------------------------------------------------------------------------
+ // Register Index Type
+ //------------------------------------------------------------------------------
+diff --git a/src/include/hnsw/hnsw_index.hpp b/src/include/hnsw/hnsw_index.hpp
+index bd0a60d..82e527e 100644
+--- a/src/include/hnsw/hnsw_index.hpp
++++ b/src/include/hnsw/hnsw_index.hpp
+@@ -72,7 +72,7 @@ public:
+ 	//! Insert a chunk of entries into the index
+ 	ErrorData Insert(IndexLock &lock, DataChunk &data, Vector &row_ids) override;
+
+-	IndexStorageInfo GetStorageInfo(const bool get_buffers) override;
++	IndexStorageInfo GetStorageInfo(const case_insensitive_map_t<Value> &options, const bool to_wal) override;
+ 	idx_t GetInMemorySize(IndexLock &state) override;
+
+ 	//! Merge another index into this index. The lock obtained from InitializeLock must be held, and the other
+@@ -85,8 +85,10 @@ public:
+ 	//! Performs constraint checking for a chunk of input data
+ 	void CheckConstraintsForChunk(DataChunk &input, ConflictManager &conflict_manager) override;
+
+-	//! Returns the string representation of the HNSWIndex, or only traverses and verifies the index
++	//! Returns the string representation of the HNSWIndex, or only traverses and verifies the index.
+ 	string VerifyAndToString(IndexLock &state, const bool only_verify) override;
++	//! Ensures that the node allocation counts match the node counts.
++	void VerifyAllocations(IndexLock &state) override;
+
+ 	string GetConstraintViolationMessage(VerifyExistenceType verify_type, idx_t failed_index,
+ 	                                     DataChunk &input) override {

.github/workflows/NightlyTests.yml

@@ -494,6 +494,42 @@ jobs:
         shell: bash
         run: build/relassert/test/unittest
 
+  latest-storage:
+    name: Latest Storage
+    runs-on: ubuntu-20.04
+    needs: linux-memory-leaks
+    env:
+      CC: gcc-10
+      CXX: g++-10
+      GEN: ninja
+      BUILD_PARQUET: 1
+      BUILD_TPCH: 1
+      BUILD_TPCDS: 1
+      LATEST_STORAGE: 1
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Install
+        shell: bash
+        run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build
+
+      - name: Setup Ccache
+        uses: hendrikmuhs/ccache-action@main
+        with:
+          key: ${{ github.job }}
+          save: ${{ env.CCACHE_SAVE }}
+
+      - name: Build
+        shell: bash
+        run: make relassert
+
+      - name: Test
+        shell: bash
+        run: build/relassert/test/unittest "*"
+
   vector-verification:
     name: Vector Verification Tests (${{ matrix.vector_type }})
     runs-on: ubuntu-20.04

CMakeLists.txt

@@ -455,6 +455,10 @@ if(ALTERNATIVE_VERIFY)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDUCKDB_ALTERNATIVE_VERIFY")
 endif()
 
+if(LATEST_STORAGE)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDUCKDB_LATEST_STORAGE")
+endif()
+
 if(DEBUG_ALLOCATION)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDUCKDB_DEBUG_ALLOCATION")
 endif()

Makefile

@@ -207,6 +207,9 @@ endif
 ifeq (${ALTERNATIVE_VERIFY}, 1)
 	CMAKE_VARS:=${CMAKE_VARS} -DALTERNATIVE_VERIFY=1
 endif
+ifeq (${LATEST_STORAGE}, 1)
+	CMAKE_VARS:=${CMAKE_VARS} -DLATEST_STORAGE=1
+endif
 ifneq (${VERIFY_VECTOR}, )
 	CMAKE_VARS:=${CMAKE_VARS} -DVERIFY_VECTOR=${VERIFY_VECTOR}
 endif

benchmark/micro/index/insert/insert_pk_fk.benchmark

@@ -0,0 +1,31 @@
+# name: benchmark/micro/index/insert/insert_pk_fk.benchmark
+# description: Insert into multiple indexes
+# group: [insert]
+
+name Insert into PK and FK
+group index
+
+require tpcds
+
+load
+BEGIN TRANSACTION;
+CALL dsdgen(sf=0.1);
+pragma threads=8;
+EXPORT DATABASE '${BENCHMARK_DIR}/sf1data';
+ROLLBACK;
+CREATE TABLE date_dim (d_date_sk Integer Not Null PRIMARY KEY, d_date_id String Not Null, d_date Date Not Null, d_month_seq Integer, d_week_seq Integer, d_quarter_seq Integer, d_year Integer, d_dow Integer, d_moy Integer, d_dom Integer, d_qoy Integer, d_fy_year Integer, d_fy_quarter_seq Integer, d_fy_week_seq Integer, d_day_name String, d_quarter_name String, d_holiday String, d_weekend String, d_following_holiday String, d_first_dom Integer, d_last_dom Integer, d_same_day_ly Integer, d_same_day_lq Integer, d_current_day String, d_current_week String, d_current_month String, d_current_quarter String, d_current_year String);
+CREATE TABLE warehouse (w_warehouse_sk Integer Not Null PRIMARY KEY, w_warehouse_id String Not Null, w_warehouse_name String, w_warehouse_sq_ft Integer, w_street_number String, w_street_name String, w_street_type String, w_suite_number String, w_city String, w_county String, w_state String, w_zip String, w_country String, w_gmt_offset Decimal(5,2));
+CREATE TABLE item (i_item_sk Integer Not Null PRIMARY KEY, i_item_id String Not Null, i_rec_start_date Date, i_rec_end_date Date, i_item_desc String, i_current_price Decimal(7,2), i_wholesale_cost Decimal(7,2), i_brand_id Integer, i_brand String, i_class_id Integer, i_class String, i_category_id Integer, i_category String, i_manufact_id Integer, i_manufact String, i_size String, i_formulation String, i_color String, i_units String, i_container String, i_manager_id Integer, i_product_name String);
+CREATE TABLE inventory (inv_date_sk Integer Not Null REFERENCES date_dim (d_date_sk), inv_item_sk Integer Not Null REFERENCES item (i_item_sk), inv_warehouse_sk Integer Not Null REFERENCES warehouse (w_warehouse_sk), inv_quantity_on_hand Integer);
+
+run
+COPY date_dim FROM '${BENCHMARK_DIR}/sf1data/date_dim.csv' (FORMAT 'csv', header 1, delimiter ',', quote '"');
+COPY item FROM '${BENCHMARK_DIR}/sf1data/item.csv' (FORMAT 'csv', header 1, delimiter ',', quote '"');
+COPY warehouse FROM '${BENCHMARK_DIR}/sf1data/warehouse.csv' (FORMAT 'csv', header 1, delimiter ',', quote '"');
+COPY inventory FROM '${BENCHMARK_DIR}/sf1data/inventory.csv' (FORMAT 'csv', header 1, delimiter ',', quote '"');
+
+cleanup
+CREATE OR REPLACE TABLE date_dim (d_date_sk Integer Not Null PRIMARY KEY, d_date_id String Not Null, d_date Date Not Null, d_month_seq Integer, d_week_seq Integer, d_quarter_seq Integer, d_year Integer, d_dow Integer, d_moy Integer, d_dom Integer, d_qoy Integer, d_fy_year Integer, d_fy_quarter_seq Integer, d_fy_week_seq Integer, d_day_name String, d_quarter_name String, d_holiday String, d_weekend String, d_following_holiday String, d_first_dom Integer, d_last_dom Integer, d_same_day_ly Integer, d_same_day_lq Integer, d_current_day String, d_current_week String, d_current_month String, d_current_quarter String, d_current_year String);
+CREATE OR REPLACE TABLE warehouse (w_warehouse_sk Integer Not Null PRIMARY KEY, w_warehouse_id String Not Null, w_warehouse_name String, w_warehouse_sq_ft Integer, w_street_number String, w_street_name String, w_street_type String, w_suite_number String, w_city String, w_county String, w_state String, w_zip String, w_country String, w_gmt_offset Decimal(5,2));
+CREATE OR REPLACE TABLE item (i_item_sk Integer Not Null PRIMARY KEY, i_item_id String Not Null, i_rec_start_date Date, i_rec_end_date Date, i_item_desc String, i_current_price Decimal(7,2), i_wholesale_cost Decimal(7,2), i_brand_id Integer, i_brand String, i_class_id Integer, i_class String, i_category_id Integer, i_category String, i_manufact_id Integer, i_manufact String, i_size String, i_formulation String, i_color String, i_units String, i_container String, i_manager_id Integer, i_product_name String);
+CREATE OR REPLACE TABLE inventory (inv_date_sk Integer Not Null REFERENCES date_dim (d_date_sk), inv_item_sk Integer Not Null REFERENCES item (i_item_sk), inv_warehouse_sk Integer Not Null REFERENCES warehouse (w_warehouse_sk), inv_quantity_on_hand Integer);

extension/json/CMakeLists.txt

@@ -18,6 +18,7 @@ set(JSON_EXTENSION_FILES
     json_functions/copy_json.cpp
     json_functions/json_array_length.cpp
     json_functions/json_contains.cpp
+    json_functions/json_exists.cpp
     json_functions/json_extract.cpp
     json_functions/json_keys.cpp
     json_functions/json_merge_patch.cpp
@@ -27,6 +28,7 @@ set(JSON_EXTENSION_FILES
     json_functions/json_create.cpp
     json_functions/json_type.cpp
     json_functions/json_valid.cpp
+    json_functions/json_value.cpp
     json_functions/json_serialize_sql.cpp
     json_functions/json_serialize_plan.cpp
     json_functions/read_json.cpp