Merge branch 'feature' into temp_file_compression

.github/patches/extensions/spatial/random_test_fix.patch

@@ -56,10 +56,10 @@ index 465cb87..5aa49dd 100644
 
  	ExtensionUtil::RegisterFunction(db, read);
 diff --git a/spatial/src/spatial/gdal/functions/st_read.cpp b/spatial/src/spatial/gdal/functions/st_read.cpp
-index 177548c..42d2df7 100644
+index b730baa..8d08898 100644
 --- a/spatial/src/spatial/gdal/functions/st_read.cpp
 +++ b/spatial/src/spatial/gdal/functions/st_read.cpp
-@@ -675,7 +675,7 @@ void GdalTableFunction::Register(DatabaseInstance &db) {
+@@ -676,7 +676,7 @@ void GdalTableFunction::Register(DatabaseInstance &db) {
  	                   GdalTableFunction::InitGlobal, GdalTableFunction::InitLocal);
 
  	scan.cardinality = GdalTableFunction::Cardinality;
@@ -68,3 +68,25 @@ index 177548c..42d2df7 100644
 
  	scan.projection_pushdown = true;
  	scan.filter_pushdown = true;
+diff --git a/spatial/src/spatial/geos/functions/aggregate.cpp b/spatial/src/spatial/geos/functions/aggregate.cpp
+index aacc668..c478786 100644
+--- a/spatial/src/spatial/geos/functions/aggregate.cpp
++++ b/spatial/src/spatial/geos/functions/aggregate.cpp
+@@ -197,7 +197,7 @@ void GeosAggregateFunctions::Register(DatabaseInstance &db) {
+
+ 	AggregateFunctionSet st_intersection_agg("ST_Intersection_Agg");
+ 	st_intersection_agg.AddFunction(
+-	    AggregateFunction::UnaryAggregateDestructor<GEOSAggState, geometry_t, geometry_t, IntersectionAggFunction>(
++	    AggregateFunction::UnaryAggregateDestructor<GEOSAggState, geometry_t, geometry_t, IntersectionAggFunction, AggregateDestructorType::LEGACY>(
+ 	        core::GeoTypes::GEOMETRY(), core::GeoTypes::GEOMETRY()));
+
+ 	ExtensionUtil::RegisterFunction(db, st_intersection_agg);
+@@ -206,7 +206,7 @@ void GeosAggregateFunctions::Register(DatabaseInstance &db) {
+
+ 	AggregateFunctionSet st_union_agg("ST_Union_Agg");
+ 	st_union_agg.AddFunction(
+-	    AggregateFunction::UnaryAggregateDestructor<GEOSAggState, geometry_t, geometry_t, UnionAggFunction>(
++	    AggregateFunction::UnaryAggregateDestructor<GEOSAggState, geometry_t, geometry_t, UnionAggFunction, AggregateDestructorType::LEGACY>(
+ 	        core::GeoTypes::GEOMETRY(), core::GeoTypes::GEOMETRY()));
+
+ 	ExtensionUtil::RegisterFunction(db, st_union_agg);

.github/workflows/InternalIssuesCreateMirror.yml

@@ -21,7 +21,25 @@ jobs:
         run: |
           gh issue edit --repo duckdb/duckdb ${{ github.event.issue.number }} --remove-label "needs triage"
 
-  create_or_label_issue:
+  add_needs_reproducible_example_comment:
+    if: github.event.label.name == 'needs reproducible example'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Add comment
+        run: |
+          cat > needs-reproducible-example-comment.md << EOF
+          Thanks for opening this issue in the DuckDB issue tracker! To resolve this issue, our team needs a reproducible example. This includes:
+
+          * A source code snippet which reproduces the issue.
+          * The snippet should be self-contained, i.e., it should contain all imports and should use relative paths instead of hard coded paths (please avoid \`/Users/JohnDoe/...\`).
+          * A lot of issues can be reproduced with plain SQL code executed in the [DuckDB command line client](https://duckdb.org/docs/api/cli/overview). If you can provide such an example, it greatly simplifies the reproduction process and likely results in a faster fix.
+          * If the script needs additional data, please share the data as a CSV, JSON, or Parquet file. Unfortunately, we cannot fix issues that can only be reproduced with a confidential data set. [Support contracts](https://duckdblabs.com/#support) allow sharing confidential data with the core DuckDB team under NDA.
+
+          For more detailed guidelines on how to create reproducible examples, please visit Stack Overflow's [“Minimal, Reproducible Example”](https://stackoverflow.com/help/minimal-reproducible-example) page.
+          EOF
+          gh issue comment ${{ github.event.issue.number }} --body-file needs-reproducible-example-comment.md
+
+  create_or_label_mirror_issue:
     if: github.event.label.name == 'reproduced' || github.event.label.name == 'under review'
     runs-on: ubuntu-latest
     steps:

extension/core_functions/aggregate/distributive/arg_min_max.cpp

@@ -314,21 +314,22 @@ struct VectorArgMinMaxBase : ArgMinMaxBase<COMPARATOR, IGNORE_NULL> {
 template <class OP>
 AggregateFunction GetGenericArgMinMaxFunction() {
 	using STATE = ArgMinMaxState<string_t, string_t>;
-	return AggregateFunction({LogicalType::ANY, LogicalType::ANY}, LogicalType::ANY,
-	                         AggregateFunction::StateSize<STATE>, AggregateFunction::StateInitialize<STATE, OP>,
-	                         OP::template Update<STATE>, AggregateFunction::StateCombine<STATE, OP>,
-	                         AggregateFunction::StateVoidFinalize<STATE, OP>, nullptr, OP::Bind,
-	                         AggregateFunction::StateDestroy<STATE, OP>);
+	return AggregateFunction(
+	    {LogicalType::ANY, LogicalType::ANY}, LogicalType::ANY, AggregateFunction::StateSize<STATE>,
+	    AggregateFunction::StateInitialize<STATE, OP, AggregateDestructorType::LEGACY>, OP::template Update<STATE>,
+	    AggregateFunction::StateCombine<STATE, OP>, AggregateFunction::StateVoidFinalize<STATE, OP>, nullptr, OP::Bind,
+	    AggregateFunction::StateDestroy<STATE, OP>);
 }
 
 template <class OP, class ARG_TYPE, class BY_TYPE>
 AggregateFunction GetVectorArgMinMaxFunctionInternal(const LogicalType &by_type, const LogicalType &type) {
 #ifndef DUCKDB_SMALLER_BINARY
 	using STATE = ArgMinMaxState<ARG_TYPE, BY_TYPE>;
-	return AggregateFunction(
-	    {type, by_type}, type, AggregateFunction::StateSize<STATE>, AggregateFunction::StateInitialize<STATE, OP>,
-	    OP::template Update<STATE>, AggregateFunction::StateCombine<STATE, OP>,
-	    AggregateFunction::StateVoidFinalize<STATE, OP>, nullptr, OP::Bind, AggregateFunction::StateDestroy<STATE, OP>);
+	return AggregateFunction({type, by_type}, type, AggregateFunction::StateSize<STATE>,
+	                         AggregateFunction::StateInitialize<STATE, OP, AggregateDestructorType::LEGACY>,
+	                         OP::template Update<STATE>, AggregateFunction::StateCombine<STATE, OP>,
+	                         AggregateFunction::StateVoidFinalize<STATE, OP>, nullptr, OP::Bind,
+	                         AggregateFunction::StateDestroy<STATE, OP>);
 #else
 	auto function = GetGenericArgMinMaxFunction<OP>();
 	function.arguments = {type, by_type};
@@ -380,7 +381,9 @@ template <class OP, class ARG_TYPE, class BY_TYPE>
 AggregateFunction GetArgMinMaxFunctionInternal(const LogicalType &by_type, const LogicalType &type) {
 #ifndef DUCKDB_SMALLER_BINARY
 	using STATE = ArgMinMaxState<ARG_TYPE, BY_TYPE>;
-	auto function = AggregateFunction::BinaryAggregate<STATE, ARG_TYPE, BY_TYPE, ARG_TYPE, OP>(type, by_type, type);
+	auto function =
+	    AggregateFunction::BinaryAggregate<STATE, ARG_TYPE, BY_TYPE, ARG_TYPE, OP, AggregateDestructorType::LEGACY>(
+	        type, by_type, type);
 	if (type.InternalType() == PhysicalType::VARCHAR || by_type.InternalType() == PhysicalType::VARCHAR) {
 		function.destructor = AggregateFunction::StateDestroy<STATE, OP>;
 	}
@@ -618,7 +621,7 @@ static void SpecializeArgMinMaxNFunction(AggregateFunction &function) {
 	using OP = MinMaxNOperation;
 
 	function.state_size = AggregateFunction::StateSize<STATE>;
-	function.initialize = AggregateFunction::StateInitialize<STATE, OP>;
+	function.initialize = AggregateFunction::StateInitialize<STATE, OP, AggregateDestructorType::LEGACY>;
 	function.combine = AggregateFunction::StateCombine<STATE, OP>;
 	function.destructor = AggregateFunction::StateDestroy<STATE, OP>;
 

extension/core_functions/aggregate/holistic/mad.cpp

@@ -265,7 +265,8 @@ AggregateFunction GetTypedMedianAbsoluteDeviationAggregateFunction(const Logical
                                                                    const LogicalType &target_type) {
 	using STATE = QuantileState<INPUT_TYPE, QuantileStandardType>;
 	using OP = MedianAbsoluteDeviationOperation<MEDIAN_TYPE>;
-	auto fun = AggregateFunction::UnaryAggregateDestructor<STATE, INPUT_TYPE, TARGET_TYPE, OP>(input_type, target_type);
+	auto fun = AggregateFunction::UnaryAggregateDestructor<STATE, INPUT_TYPE, TARGET_TYPE, OP,
+	                                                       AggregateDestructorType::LEGACY>(input_type, target_type);
 	fun.bind = BindMAD;
 	fun.order_dependent = AggregateOrderDependent::NOT_ORDER_DEPENDENT;
 #ifndef DUCKDB_SMALLER_BINARY

extension/core_functions/aggregate/holistic/mode.cpp

@@ -424,7 +424,7 @@ AggregateFunction GetFallbackModeFunction(const LogicalType &type) {
 	using STATE = ModeState<string_t, ModeString>;
 	using OP = ModeFallbackFunction<ModeString>;
 	AggregateFunction aggr({type}, type, AggregateFunction::StateSize<STATE>,
-	                       AggregateFunction::StateInitialize<STATE, OP>,
+	                       AggregateFunction::StateInitialize<STATE, OP, AggregateDestructorType::LEGACY>,
 	                       AggregateSortKeyHelpers::UnaryUpdate<STATE, OP>, AggregateFunction::StateCombine<STATE, OP>,
 	                       AggregateFunction::StateVoidFinalize<STATE, OP>, nullptr);
 	aggr.destructor = AggregateFunction::StateDestroy<STATE, OP>;
@@ -435,7 +435,9 @@ template <typename INPUT_TYPE, typename TYPE_OP = ModeStandard<INPUT_TYPE>>
 AggregateFunction GetTypedModeFunction(const LogicalType &type) {
 	using STATE = ModeState<INPUT_TYPE, TYPE_OP>;
 	using OP = ModeFunction<TYPE_OP>;
-	auto func = AggregateFunction::UnaryAggregateDestructor<STATE, INPUT_TYPE, INPUT_TYPE, OP>(type, type);
+	auto func =
+	    AggregateFunction::UnaryAggregateDestructor<STATE, INPUT_TYPE, INPUT_TYPE, OP, AggregateDestructorType::LEGACY>(
+	        type, type);
 	func.window = OP::template Window<STATE, INPUT_TYPE, INPUT_TYPE>;
 	return func;
 }
@@ -528,7 +530,9 @@ template <typename INPUT_TYPE, typename TYPE_OP = ModeStandard<INPUT_TYPE>>
 AggregateFunction GetTypedEntropyFunction(const LogicalType &type) {
 	using STATE = ModeState<INPUT_TYPE, TYPE_OP>;
 	using OP = EntropyFunction<TYPE_OP>;
-	auto func = AggregateFunction::UnaryAggregateDestructor<STATE, INPUT_TYPE, double, OP>(type, LogicalType::DOUBLE);
+	auto func =
+	    AggregateFunction::UnaryAggregateDestructor<STATE, INPUT_TYPE, double, OP, AggregateDestructorType::LEGACY>(
+	        type, LogicalType::DOUBLE);
 	func.null_handling = FunctionNullHandling::SPECIAL_HANDLING;
 	return func;
 }
@@ -537,7 +541,7 @@ AggregateFunction GetFallbackEntropyFunction(const LogicalType &type) {
 	using STATE = ModeState<string_t, ModeString>;
 	using OP = EntropyFallbackFunction<ModeString>;
 	AggregateFunction func({type}, LogicalType::DOUBLE, AggregateFunction::StateSize<STATE>,
-	                       AggregateFunction::StateInitialize<STATE, OP>,
+	                       AggregateFunction::StateInitialize<STATE, OP, AggregateDestructorType::LEGACY>,
 	                       AggregateSortKeyHelpers::UnaryUpdate<STATE, OP>, AggregateFunction::StateCombine<STATE, OP>,
 	                       AggregateFunction::StateFinalize<STATE, double, OP>, nullptr);
 	func.destructor = AggregateFunction::StateDestroy<STATE, OP>;

extension/core_functions/aggregate/holistic/quantile.cpp

@@ -420,7 +420,8 @@ struct ScalarDiscreteQuantile {
 	static AggregateFunction GetFunction(const LogicalType &type) {
 		using STATE = QuantileState<INPUT_TYPE, TYPE_OP>;
 		using OP = QuantileScalarOperation<true>;
-		auto fun = AggregateFunction::UnaryAggregateDestructor<STATE, INPUT_TYPE, INPUT_TYPE, OP>(type, type);
+		auto fun = AggregateFunction::UnaryAggregateDestructor<STATE, INPUT_TYPE, INPUT_TYPE, OP,
+		                                                       AggregateDestructorType::LEGACY>(type, type);
 #ifndef DUCKDB_SMALLER_BINARY
 		fun.window = OP::Window<STATE, INPUT_TYPE, INPUT_TYPE>;
 		fun.window_init = OP::WindowInit<STATE, INPUT_TYPE>;
@@ -432,11 +433,12 @@ struct ScalarDiscreteQuantile {
 		using STATE = QuantileState<string_t, QuantileStringType>;
 		using OP = QuantileScalarFallback;
 
-		AggregateFunction fun(
-		    {type}, type, AggregateFunction::StateSize<STATE>, AggregateFunction::StateInitialize<STATE, OP>,
-		    AggregateSortKeyHelpers::UnaryUpdate<STATE, OP>, AggregateFunction::StateCombine<STATE, OP>,
-		    AggregateFunction::StateVoidFinalize<STATE, OP>, nullptr, nullptr,
-		    AggregateFunction::StateDestroy<STATE, OP>);
+		AggregateFunction fun({type}, type, AggregateFunction::StateSize<STATE>,
+		                      AggregateFunction::StateInitialize<STATE, OP, AggregateDestructorType::LEGACY>,
+		                      AggregateSortKeyHelpers::UnaryUpdate<STATE, OP>,
+		                      AggregateFunction::StateCombine<STATE, OP>,
+		                      AggregateFunction::StateVoidFinalize<STATE, OP>, nullptr, nullptr,
+		                      AggregateFunction::StateDestroy<STATE, OP>);
 		return fun;
 	}
 };
@@ -445,7 +447,8 @@ template <class STATE, class INPUT_TYPE, class RESULT_TYPE, class OP>
 static AggregateFunction QuantileListAggregate(const LogicalType &input_type, const LogicalType &child_type) { // NOLINT
 	LogicalType result_type = LogicalType::LIST(child_type);
 	return AggregateFunction(
-	    {input_type}, result_type, AggregateFunction::StateSize<STATE>, AggregateFunction::StateInitialize<STATE, OP>,
+	    {input_type}, result_type, AggregateFunction::StateSize<STATE>,
+	    AggregateFunction::StateInitialize<STATE, OP, AggregateDestructorType::LEGACY>,
 	    AggregateFunction::UnaryScatterUpdate<STATE, INPUT_TYPE, OP>, AggregateFunction::StateCombine<STATE, OP>,
 	    AggregateFunction::StateFinalize<STATE, RESULT_TYPE, OP>, AggregateFunction::UnaryUpdate<STATE, INPUT_TYPE, OP>,
 	    nullptr, AggregateFunction::StateDestroy<STATE, OP>);
@@ -469,11 +472,12 @@ struct ListDiscreteQuantile {
 		using STATE = QuantileState<string_t, QuantileStringType>;
 		using OP = QuantileListFallback;
 
-		AggregateFunction fun(
-		    {type}, LogicalType::LIST(type), AggregateFunction::StateSize<STATE>,
-		    AggregateFunction::StateInitialize<STATE, OP>, AggregateSortKeyHelpers::UnaryUpdate<STATE, OP>,
-		    AggregateFunction::StateCombine<STATE, OP>, AggregateFunction::StateFinalize<STATE, list_entry_t, OP>,
-		    nullptr, nullptr, AggregateFunction::StateDestroy<STATE, OP>);
+		AggregateFunction fun({type}, LogicalType::LIST(type), AggregateFunction::StateSize<STATE>,
+		                      AggregateFunction::StateInitialize<STATE, OP, AggregateDestructorType::LEGACY>,
+		                      AggregateSortKeyHelpers::UnaryUpdate<STATE, OP>,
+		                      AggregateFunction::StateCombine<STATE, OP>,
+		                      AggregateFunction::StateFinalize<STATE, list_entry_t, OP>, nullptr, nullptr,
+		                      AggregateFunction::StateDestroy<STATE, OP>);
 		return fun;
 	}
 };
@@ -547,7 +551,8 @@ struct ScalarContinuousQuantile {
 		using STATE = QuantileState<INPUT_TYPE, QuantileStandardType>;
 		using OP = QuantileScalarOperation<false>;
 		auto fun =
-		    AggregateFunction::UnaryAggregateDestructor<STATE, INPUT_TYPE, TARGET_TYPE, OP>(input_type, target_type);
+		    AggregateFunction::UnaryAggregateDestructor<STATE, INPUT_TYPE, TARGET_TYPE, OP,
+		                                                AggregateDestructorType::LEGACY>(input_type, target_type);
 		fun.order_dependent = AggregateOrderDependent::NOT_ORDER_DEPENDENT;
 #ifndef DUCKDB_SMALLER_BINARY
 		fun.window = OP::template Window<STATE, INPUT_TYPE, TARGET_TYPE>;

extension/parquet/serialize_parquet.cpp

@@ -7,8 +7,6 @@
 #include "duckdb/common/serializer/deserializer.hpp"
 #include "parquet_reader.hpp"
 #include "parquet_crypto.hpp"
-#include "parquet_reader.hpp"
-#include "parquet_writer.hpp"
 #include "parquet_writer.hpp"
 
 namespace duckdb {

scripts/generate_enum_util.py

@@ -192,27 +192,31 @@ def remove_prefix(str, prefix):
     f.write("namespace duckdb {\n\n")
 
     for enum_name, enum_type, enum_members in enums:
+        enum_string_array = "Get" + enum_name + "Values()"
         # Write the enum from string
-        f.write(f"template<>\nconst char* EnumUtil::ToChars<{enum_name}>({enum_name} value) {{\n")
-        f.write("\tswitch(value) {\n")
+        f.write(f"const StringUtil::EnumStringLiteral *{enum_string_array} {{\n")
+        f.write(f"\tstatic constexpr StringUtil::EnumStringLiteral values[] {{\n")
+        member_count = 0
         for key, strings in enum_members:
-            # Always use the first string as the enum string
-            f.write(f"\tcase {enum_name}::{key}:\n\t\treturn \"{strings[0]}\";\n")
+            for str_val in strings:
+                if member_count != 0:
+                    f.write(",\n")
+                f.write(f"\t\t{{ static_cast<uint32_t>({enum_name}::{key}), \"{str_val}\" }}")
+                member_count += 1
+        f.write("\n\t};")
+        f.write("\n\treturn values;")
+        f.write("\n}\n\n")
+        f.write(f"template<>\nconst char* EnumUtil::ToChars<{enum_name}>({enum_name} value) {{\n")
         f.write(
-            f"\tdefault:\n\t\tthrow NotImplementedException(StringUtil::Format(\"Enum value: \'%d\' not implemented in ToChars<{enum_name}>\", value));\n"
+            f"\treturn StringUtil::EnumToString({enum_string_array}, {member_count}, \"{enum_name}\", static_cast<uint32_t>(value));\n"
         )
-        f.write("\t}\n")
         f.write("}\n\n")
 
         # Write the string to enum
         f.write(f"template<>\n{enum_name} EnumUtil::FromString<{enum_name}>(const char *value) {{\n")
-        for key, strings in enum_members:
-            cond = " || ".join([f'StringUtil::Equals(value, "{string}")' for string in strings])
-            f.write(f'\tif ({cond}) {{\n\t\treturn {enum_name}::{key};\n\t}}\n')
         f.write(
-            f"\tthrow NotImplementedException(StringUtil::Format(\"Enum value: \'%s\' not implemented in FromString<{enum_name}>\", value));\n"
+            f"\treturn static_cast<{enum_name}>(StringUtil::StringToEnum({enum_string_array}, {member_count}, \"{enum_name}\", value));"
         )
-
-        f.write("}\n\n")
+        f.write("\n}\n\n")
 
     f.write("}\n\n")

scripts/generate_serialization.py

@@ -771,7 +771,12 @@ def check_children_for_duplicate_members(node: SerializableClass, parents: list,
 
     for entry in json_data:
         if 'includes' in entry:
-            include_list += entry['includes']
+            if type(entry['includes']) != type([]):
+                print(f"Include list must be a list, found {type(entry['includes'])} (in {str(entry)})")
+                exit(1)
+            for include_entry in entry['includes']:
+                if include_entry not in include_list:
+                    include_list.append(include_entry)
         new_class = SerializableClass(entry)
         if new_class.is_base_class:
             # this class is a base class itself - construct the base class list