From 549d363c95b4d0ebb97ac70124ebec113ea01d39 Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Mon, 12 Feb 2024 18:23:15 +0100 Subject: [PATCH 001/603] the idea is brewing --- src/optimizer/filter_pushdown.cpp | 16 ++++++++++ .../pushdown_window_partition_filter.test | 30 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 test/optimizer/pushdown/pushdown_window_partition_filter.test diff --git a/src/optimizer/filter_pushdown.cpp b/src/optimizer/filter_pushdown.cpp index 35d756a94c75..7aa8d0e75538 100644 --- a/src/optimizer/filter_pushdown.cpp +++ b/src/optimizer/filter_pushdown.cpp @@ -3,6 +3,7 @@ #include "duckdb/optimizer/filter_combiner.hpp" #include "duckdb/planner/operator/logical_filter.hpp" #include "duckdb/planner/operator/logical_join.hpp" +#include "duckdb/planner/operator/logical_window.hpp" #include "duckdb/optimizer/optimizer.hpp" namespace duckdb { @@ -12,6 +13,19 @@ using Filter = FilterPushdown::Filter; FilterPushdown::FilterPushdown(Optimizer &optimizer) : optimizer(optimizer), combiner(optimizer.context) { } + + +static unique_ptr PushdownWindow(unique_ptr op) { + D_ASSERT(op->type == LogicalOperatorType::LOGICAL_WINDOW); + auto &window = op->Cast(); + // go into expressions, then look into the partitions. + // if the filter applies ONLY to the partition (and is not some arithmetic expression) then you can push the filter into + // the children. + auto wat = "sdfs"; + //! Now we try to pushdown the remaining filters to perform zonemap checking + return std::move(op); +} + unique_ptr FilterPushdown::Rewrite(unique_ptr op) { D_ASSERT(!combiner.HasFilters()); switch (op->type) { @@ -43,6 +57,8 @@ unique_ptr FilterPushdown::Rewrite(unique_ptr return PushdownGet(std::move(op)); case LogicalOperatorType::LOGICAL_LIMIT: return PushdownLimit(std::move(op)); + case LogicalOperatorType::LOGICAL_WINDOW: + return PushdownWindow(std::move(op)); default: return FinishPushdown(std::move(op)); } diff --git a/test/optimizer/pushdown/pushdown_window_partition_filter.test b/test/optimizer/pushdown/pushdown_window_partition_filter.test new file mode 100644 index 000000000000..13a5948d0a7c --- /dev/null +++ b/test/optimizer/pushdown/pushdown_window_partition_filter.test @@ -0,0 +1,30 @@ +# name: test/optimizer/pushdown/pushdown_window_partition_filter.test +# description: Test pushdown of filters through window operators that are partitioned by. +# group: [pushdown] + + +statement ok +create table t1 as from VALUES ('A', 1), ('B', 3), ('C', 12), ('A', 5), ('B', 8), ('C', 9), ('A', 10), ('B', 20), ('C', 3) t(a, b); + +statement ok +create view window_with_filter as select a, b, LEAD(b) OVER (partition by a) as lead from t1 where a != 'C'; + +statement ok +create view window_no_filter as select a, b, LEAD(b) OVER (partition by a) as lead from t1; + + +query III no_sort result_1 +select * from window_with_filter where a != 'C' order by a; +---- + +query III no_sort result_1 +select * from window_no_filter where a != 'C' order by a; +---- + +statement ok +pragma explain_output=OPTIMIZED_ONLY + +query II +explain select * from window_no_filter where a != 'C' order by a; +---- +logical_opt :.*WINDOW.*FILTER.* From 647b5f64cf56099730d64f2a831a4730d6baaa6c Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Tue, 13 Feb 2024 10:12:38 +0100 Subject: [PATCH 002/603] pausing point --- src/optimizer/filter_pushdown.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/optimizer/filter_pushdown.cpp b/src/optimizer/filter_pushdown.cpp index 7aa8d0e75538..0bac5054de09 100644 --- a/src/optimizer/filter_pushdown.cpp +++ b/src/optimizer/filter_pushdown.cpp @@ -21,6 +21,13 @@ static unique_ptr PushdownWindow(unique_ptr op // go into expressions, then look into the partitions. // if the filter applies ONLY to the partition (and is not some arithmetic expression) then you can push the filter into // the children. + for (auto &expr: window.expressions) { + if (expr->expression_class != ExpressionClass::BOUND_WINDOW) { + continue; + } + auto &window_expr = expr->Cast(); + auto partitions = window_expr.partitions; + } auto wat = "sdfs"; //! Now we try to pushdown the remaining filters to perform zonemap checking return std::move(op); From 1fa8e3c526d5208d2e7f9552fb1f7cf8e931fa18 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 15 Feb 2024 14:30:05 +0100 Subject: [PATCH 003/603] have the logic. just need to implement FilterIsOnPartition() --- .../duckdb/optimizer/filter_pushdown.hpp | 2 + src/optimizer/filter_pushdown.cpp | 41 +++++++++++++++---- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/src/include/duckdb/optimizer/filter_pushdown.hpp b/src/include/duckdb/optimizer/filter_pushdown.hpp index 70ba4fdf3a31..a2aa1e2e5b07 100644 --- a/src/include/duckdb/optimizer/filter_pushdown.hpp +++ b/src/include/duckdb/optimizer/filter_pushdown.hpp @@ -59,6 +59,8 @@ class FilterPushdown { unique_ptr PushdownGet(unique_ptr op); //! Push down a LogicalLimit op unique_ptr PushdownLimit(unique_ptr op); + //! Push down a LogicalWindow op + unique_ptr PushdownWindow(unique_ptr op); // Pushdown an inner join unique_ptr PushdownInnerJoin(unique_ptr op, unordered_set &left_bindings, unordered_set &right_bindings); diff --git a/src/optimizer/filter_pushdown.cpp b/src/optimizer/filter_pushdown.cpp index 0bac5054de09..ce26b351760e 100644 --- a/src/optimizer/filter_pushdown.cpp +++ b/src/optimizer/filter_pushdown.cpp @@ -13,9 +13,11 @@ using Filter = FilterPushdown::Filter; FilterPushdown::FilterPushdown(Optimizer &optimizer) : optimizer(optimizer), combiner(optimizer.context) { } +static bool FilterIsOnPartition(column_binding_set_t parition_bindings, Expression &filter_expression) { + return true; +} - -static unique_ptr PushdownWindow(unique_ptr op) { +unique_ptr FilterPushdown::PushdownWindow(unique_ptr op) { D_ASSERT(op->type == LogicalOperatorType::LOGICAL_WINDOW); auto &window = op->Cast(); // go into expressions, then look into the partitions. @@ -25,12 +27,37 @@ static unique_ptr PushdownWindow(unique_ptr op if (expr->expression_class != ExpressionClass::BOUND_WINDOW) { continue; } - auto &window_expr = expr->Cast(); - auto partitions = window_expr.partitions; + auto &window_expr = expr->Cast(); + auto &partitions = window_expr.partitions; + column_binding_set_t parition_bindings; + for (auto &partition_expr : partitions) { + switch (partition_expr->type) { + case ExpressionType::BOUND_COLUMN_REF: { + auto &partition_col = partition_expr->Cast(); + parition_bindings.insert(partition_col.binding); + break; + } + default: + break; + } + } + vector> leftover_filters; + for (auto &filter : filters) { + if (FilterIsOnPartition(parition_bindings, *filter->filter)) { + auto pushdown = FilterPushdown(optimizer); + pushdown.filters.push_back(std::move(filter)); + op->children[0] = pushdown.Rewrite(std::move(op->children[0])); + continue; + } + leftover_filters.push_back(std::move(filter)); + // check if a filter is on the partition + // create new filter pushdown with the filter, + // push into children + } + filters = std::move(leftover_filters); } - auto wat = "sdfs"; - //! Now we try to pushdown the remaining filters to perform zonemap checking - return std::move(op); + return FinishPushdown(std::move(op)); +// return std::move(op); } unique_ptr FilterPushdown::Rewrite(unique_ptr op) { From 0357a33c26a60d20923ce596dd98bcd04f9a03e1 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 21 Feb 2024 11:05:05 -0300 Subject: [PATCH 004/603] Get rid of rejects_recovery_columns --- .../table_function/csv_file_scanner.cpp | 14 ------ .../table_function/global_csv_state.cpp | 15 ------- .../csv_scanner/util/csv_reader_options.cpp | 7 --- .../operator/persistent/csv_rejects_table.cpp | 9 ---- src/function/table/read_csv.cpp | 23 +--------- .../csv_scanner/csv_reader_options.hpp | 4 -- .../duckdb/storage/serialization/nodes.json | 26 ++++------- src/storage/serialization/serialize_nodes.cpp | 44 +++++++++---------- 8 files changed, 30 insertions(+), 112 deletions(-) diff --git a/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp b/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp index 27293e0736cb..7bdc4712fc02 100644 --- a/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp +++ b/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp @@ -168,20 +168,6 @@ void CSVFileScan::InitializeFileNamesTypes() { projection_ids.emplace_back(result_idx, i); } - if (!projected_columns.empty()) { - // We might have to add recovery rejects column ids - for (idx_t i = 0; i < options.rejects_recovery_column_ids.size(); i++) { - idx_t col_id = options.rejects_recovery_column_ids[i]; - if (projected_columns.find(col_id) == projected_columns.end()) { - // We have to insert this column in our projection - projected_columns.insert(col_id); - file_types.emplace_back(LogicalType::VARCHAR); - projected_columns.insert(col_id); - projection_ids.emplace_back(col_id, col_id); - } - } - } - if (reader_data.column_ids.empty()) { file_types = types; } diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index 764c7b057053..2f8e92e3718f 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -174,21 +174,6 @@ void CSVGlobalState::FillRejectsTable() { appender.Append(string_t("\"" + col_name + "\"")); appender.Append(error.row[col_idx]); - if (!options.rejects_recovery_columns.empty()) { - child_list_t recovery_key; - for (auto &key_idx : options.rejects_recovery_column_ids) { - // Figure out if the recovery key is valid. - // If not, error out for real. - auto &value = error.row[key_idx]; - if (value.IsNull()) { - throw InvalidInputException("%s at line %llu in column %s. Parser options:\n%s ", - "Could not parse recovery column", row_line, col_name, - options.ToString()); - } - recovery_key.emplace_back(bind_data.return_names[key_idx], value); - } - appender.Append(Value::STRUCT(recovery_key)); - } auto row_error_msg = StringUtil::Format("Could not convert string '%s' to '%s'", error.row[col_idx].ToString(), file->types[col_idx].ToString()); diff --git a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp index d05db0bb8fb2..72c73a2e5bac 100644 --- a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp +++ b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp @@ -213,13 +213,6 @@ void CSVReaderOptions::SetReadOption(const string &loption, const Value &value, throw BinderException("REJECTS_TABLE option cannot be empty"); } rejects_table_name = table_name; - } else if (loption == "rejects_recovery_columns") { - // Get the list of columns to use as a recovery key - auto &children = ListValue::GetChildren(value); - for (auto &child : children) { - auto col_name = child.GetValue(); - rejects_recovery_columns.push_back(col_name); - } } else if (loption == "rejects_limit") { int64_t limit = ParseInteger(value, loption); if (limit < 0) { diff --git a/src/execution/operator/persistent/csv_rejects_table.cpp b/src/execution/operator/persistent/csv_rejects_table.cpp index a96bb2251aa4..7d01723a7718 100644 --- a/src/execution/operator/persistent/csv_rejects_table.cpp +++ b/src/execution/operator/persistent/csv_rejects_table.cpp @@ -29,15 +29,6 @@ void CSVRejectsTable::InitializeTable(ClientContext &context, const ReadCSVData info->columns.AddColumn(ColumnDefinition("column", LogicalType::BIGINT)); info->columns.AddColumn(ColumnDefinition("column_name", LogicalType::VARCHAR)); info->columns.AddColumn(ColumnDefinition("parsed_value", LogicalType::VARCHAR)); - - if (!data.options.rejects_recovery_columns.empty()) { - child_list_t recovery_key_components; - for (auto &col_name : data.options.rejects_recovery_columns) { - recovery_key_components.emplace_back(col_name, LogicalType::VARCHAR); - } - info->columns.AddColumn(ColumnDefinition("recovery_columns", LogicalType::STRUCT(recovery_key_components))); - } - info->columns.AddColumn(ColumnDefinition("error", LogicalType::VARCHAR)); catalog.CreateTable(context, std::move(info)); diff --git a/src/function/table/read_csv.cpp b/src/function/table/read_csv.cpp index e7c3edb27276..1b8c2ffe7478 100644 --- a/src/function/table/read_csv.cpp +++ b/src/function/table/read_csv.cpp @@ -41,22 +41,6 @@ ReadCSVData::ReadCSVData() { void ReadCSVData::FinalizeRead(ClientContext &context) { BaseCSVData::Finalize(); - if (!options.rejects_recovery_columns.empty()) { - for (auto &recovery_col : options.rejects_recovery_columns) { - bool found = false; - for (idx_t col_idx = 0; col_idx < return_names.size(); col_idx++) { - if (StringUtil::CIEquals(return_names[col_idx], recovery_col)) { - options.rejects_recovery_column_ids.push_back(col_idx); - found = true; - break; - } - } - if (!found) { - throw BinderException("Unsupported parameter for REJECTS_RECOVERY_COLUMNS: column \"%s\" not found", - recovery_col); - } - } - } } static unique_ptr ReadCSVBind(ClientContext &context, TableFunctionBindInput &input, @@ -84,11 +68,6 @@ static unique_ptr ReadCSVBind(ClientContext &context, TableFunctio } } - if (!options.rejects_recovery_columns.empty() && options.rejects_table_name.empty()) { - throw BinderException( - "REJECTS_RECOVERY_COLUMNS option is only supported when REJECTS_TABLE is set to a table name"); - } - options.file_options.AutoDetectHivePartitioning(result->files, context); if (!options.auto_detect && return_types.empty()) { @@ -143,7 +122,7 @@ static unique_ptr ReadCSVBind(ClientContext &context, TableFunctio result->return_types = return_types; result->return_names = names; - result->FinalizeRead(context); + result->Finalize(); return std::move(result); } diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp index 6aa349bf823e..ee06436ed9d6 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp @@ -45,10 +45,6 @@ struct CSVReaderOptions { string rejects_table_name; //! Rejects table entry limit (0 = no limit) idx_t rejects_limit = 0; - //! Columns to use as recovery key for rejected rows when reading with ignore_errors = true - vector rejects_recovery_columns; - //! Index of the recovery columns - vector rejects_recovery_column_ids; //! Number of samples to buffer idx_t buffer_sample_size = (idx_t)STANDARD_VECTOR_SIZE * 50; //! Specifies the string that represents a null value diff --git a/src/include/duckdb/storage/serialization/nodes.json b/src/include/duckdb/storage/serialization/nodes.json index 97fda8405fdc..fadbc480cb22 100644 --- a/src/include/duckdb/storage/serialization/nodes.json +++ b/src/include/duckdb/storage/serialization/nodes.json @@ -584,50 +584,42 @@ "type": "idx_t" }, {"id": 119, - "name": "rejects_recovery_columns", - "type": "vector" - }, - {"id": 120, - "name": "rejects_recovery_column_ids", - "type": "vector" - }, - {"id": 121, "name": "dialect_options.state_machine_options.delimiter", "type": "CSVOption" }, - {"id": 122, + {"id": 120, "name": "dialect_options.state_machine_options.quote", "type": "CSVOption" }, - {"id": 123, + {"id": 121, "name": "dialect_options.state_machine_options.escape", "type": "CSVOption" }, - {"id": 124, + {"id": 122, "name": "dialect_options.header", "type": "CSVOption" }, - {"id": 125, + {"id": 123, "name": "dialect_options.num_cols", "type": "idx_t" }, - {"id": 126, + {"id": 124, "name": "dialect_options.state_machine_options.new_line", "type": "CSVOption" }, - {"id": 127, + {"id": 125, "name": "dialect_options.skip_rows", "type": "CSVOption" }, - {"id": 128, + {"id": 126, "name": "dialect_options.date_format", "type": "map>" }, - {"id": 129, + {"id": 127, "name": "sniffer_user_mismatch_error", "type": "string" }, - {"id": 130, + {"id": 128, "name": "parallel", "type": "bool" } diff --git a/src/storage/serialization/serialize_nodes.cpp b/src/storage/serialization/serialize_nodes.cpp index 13df905c94f4..714a465f4b92 100644 --- a/src/storage/serialization/serialize_nodes.cpp +++ b/src/storage/serialization/serialize_nodes.cpp @@ -121,18 +121,16 @@ void CSVReaderOptions::Serialize(Serializer &serializer) const { serializer.WritePropertyWithDefault>(116, "force_quote", force_quote); serializer.WritePropertyWithDefault(117, "rejects_table_name", rejects_table_name); serializer.WritePropertyWithDefault(118, "rejects_limit", rejects_limit); - serializer.WritePropertyWithDefault>(119, "rejects_recovery_columns", rejects_recovery_columns); - serializer.WritePropertyWithDefault>(120, "rejects_recovery_column_ids", rejects_recovery_column_ids); - serializer.WriteProperty>(121, "dialect_options.state_machine_options.delimiter", dialect_options.state_machine_options.delimiter); - serializer.WriteProperty>(122, "dialect_options.state_machine_options.quote", dialect_options.state_machine_options.quote); - serializer.WriteProperty>(123, "dialect_options.state_machine_options.escape", dialect_options.state_machine_options.escape); - serializer.WriteProperty>(124, "dialect_options.header", dialect_options.header); - serializer.WritePropertyWithDefault(125, "dialect_options.num_cols", dialect_options.num_cols); - serializer.WriteProperty>(126, "dialect_options.state_machine_options.new_line", dialect_options.state_machine_options.new_line); - serializer.WriteProperty>(127, "dialect_options.skip_rows", dialect_options.skip_rows); - serializer.WriteProperty>>(128, "dialect_options.date_format", dialect_options.date_format); - serializer.WritePropertyWithDefault(129, "sniffer_user_mismatch_error", sniffer_user_mismatch_error); - serializer.WritePropertyWithDefault(130, "parallel", parallel); + serializer.WriteProperty>(119, "dialect_options.state_machine_options.delimiter", dialect_options.state_machine_options.delimiter); + serializer.WriteProperty>(120, "dialect_options.state_machine_options.quote", dialect_options.state_machine_options.quote); + serializer.WriteProperty>(121, "dialect_options.state_machine_options.escape", dialect_options.state_machine_options.escape); + serializer.WriteProperty>(122, "dialect_options.header", dialect_options.header); + serializer.WritePropertyWithDefault(123, "dialect_options.num_cols", dialect_options.num_cols); + serializer.WriteProperty>(124, "dialect_options.state_machine_options.new_line", dialect_options.state_machine_options.new_line); + serializer.WriteProperty>(125, "dialect_options.skip_rows", dialect_options.skip_rows); + serializer.WriteProperty>>(126, "dialect_options.date_format", dialect_options.date_format); + serializer.WritePropertyWithDefault(127, "sniffer_user_mismatch_error", sniffer_user_mismatch_error); + serializer.WritePropertyWithDefault(128, "parallel", parallel); } CSVReaderOptions CSVReaderOptions::Deserialize(Deserializer &deserializer) { @@ -156,18 +154,16 @@ CSVReaderOptions CSVReaderOptions::Deserialize(Deserializer &deserializer) { deserializer.ReadPropertyWithDefault>(116, "force_quote", result.force_quote); deserializer.ReadPropertyWithDefault(117, "rejects_table_name", result.rejects_table_name); deserializer.ReadPropertyWithDefault(118, "rejects_limit", result.rejects_limit); - deserializer.ReadPropertyWithDefault>(119, "rejects_recovery_columns", result.rejects_recovery_columns); - deserializer.ReadPropertyWithDefault>(120, "rejects_recovery_column_ids", result.rejects_recovery_column_ids); - deserializer.ReadProperty>(121, "dialect_options.state_machine_options.delimiter", result.dialect_options.state_machine_options.delimiter); - deserializer.ReadProperty>(122, "dialect_options.state_machine_options.quote", result.dialect_options.state_machine_options.quote); - deserializer.ReadProperty>(123, "dialect_options.state_machine_options.escape", result.dialect_options.state_machine_options.escape); - deserializer.ReadProperty>(124, "dialect_options.header", result.dialect_options.header); - deserializer.ReadPropertyWithDefault(125, "dialect_options.num_cols", result.dialect_options.num_cols); - deserializer.ReadProperty>(126, "dialect_options.state_machine_options.new_line", result.dialect_options.state_machine_options.new_line); - deserializer.ReadProperty>(127, "dialect_options.skip_rows", result.dialect_options.skip_rows); - deserializer.ReadProperty>>(128, "dialect_options.date_format", result.dialect_options.date_format); - deserializer.ReadPropertyWithDefault(129, "sniffer_user_mismatch_error", result.sniffer_user_mismatch_error); - deserializer.ReadPropertyWithDefault(130, "parallel", result.parallel); + deserializer.ReadProperty>(119, "dialect_options.state_machine_options.delimiter", result.dialect_options.state_machine_options.delimiter); + deserializer.ReadProperty>(120, "dialect_options.state_machine_options.quote", result.dialect_options.state_machine_options.quote); + deserializer.ReadProperty>(121, "dialect_options.state_machine_options.escape", result.dialect_options.state_machine_options.escape); + deserializer.ReadProperty>(122, "dialect_options.header", result.dialect_options.header); + deserializer.ReadPropertyWithDefault(123, "dialect_options.num_cols", result.dialect_options.num_cols); + deserializer.ReadProperty>(124, "dialect_options.state_machine_options.new_line", result.dialect_options.state_machine_options.new_line); + deserializer.ReadProperty>(125, "dialect_options.skip_rows", result.dialect_options.skip_rows); + deserializer.ReadProperty>>(126, "dialect_options.date_format", result.dialect_options.date_format); + deserializer.ReadPropertyWithDefault(127, "sniffer_user_mismatch_error", result.sniffer_user_mismatch_error); + deserializer.ReadPropertyWithDefault(128, "parallel", result.parallel); return result; } From 62d8dec545bc22e4dda08fb980b83ae1563be082 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 21 Feb 2024 11:06:39 -0300 Subject: [PATCH 005/603] pesky bee --- src/function/table/read_csv.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/function/table/read_csv.cpp b/src/function/table/read_csv.cpp index 1b8c2ffe7478..272c5f95a6b0 100644 --- a/src/function/table/read_csv.cpp +++ b/src/function/table/read_csv.cpp @@ -230,7 +230,6 @@ void ReadCSVTableFunction::ReadCSVAddNamedParameters(TableFunction &table_functi table_function.named_parameters["ignore_errors"] = LogicalType::BOOLEAN; table_function.named_parameters["rejects_table"] = LogicalType::VARCHAR; table_function.named_parameters["rejects_limit"] = LogicalType::BIGINT; - table_function.named_parameters["rejects_recovery_columns"] = LogicalType::LIST(LogicalType::VARCHAR); table_function.named_parameters["buffer_size"] = LogicalType::UBIGINT; table_function.named_parameters["decimal_separator"] = LogicalType::VARCHAR; table_function.named_parameters["parallel"] = LogicalType::BOOLEAN; From e7bfcd62104696b2e40a44346c007207646e9af2 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 22 Feb 2024 10:13:07 -0300 Subject: [PATCH 006/603] wip commit --- .../scanner/string_value_scanner.cpp | 40 ++++++++++++------- .../operator/csv_scanner/util/csv_error.cpp | 9 ++--- .../operator/csv_scanner/csv_error.hpp | 8 ++-- .../csv_scanner/string_value_scanner.hpp | 2 + 4 files changed, 35 insertions(+), 24 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 349168d74949..c43ee62d1d6e 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -322,6 +322,24 @@ void StringValueResult::NullPaddingQuotedNewlineCheck() { } } +//! Reconstructs the current line to be used in error messages +string StringValueResult::ReconstructCurrentLine(){ + LinePosition current_line_start = {iterator.pos.buffer_idx, iterator.pos.buffer_pos, + buffer_size}; + idx_t current_line_size = current_line_start - previous_line_start; + string result; + result.resize(current_line_size); + if (iterator.pos.buffer_idx == previous_line_start.buffer_idx){ + idx_t result_idx = 0; + for (idx_t i = previous_line_start.buffer_pos; i < iterator.pos.buffer_pos; i ++){ + result[result_idx++] = buffer_ptr[i]; + } + } else{ + throw InternalException("Oh no"); + } + return result; +} + bool StringValueResult::AddRowInternal() { if (ignore_current_row) { // An error occurred on this row, we are ignoring it and resetting our control flag @@ -330,17 +348,6 @@ bool StringValueResult::AddRowInternal() { } if (!cast_errors.empty()) { // A wild casting error appears - // Recreate row for rejects-table - vector row; - if (!state_machine.options.rejects_table_name.empty()) { - for (idx_t col = 0; col < parse_chunk.ColumnCount(); col++) { - if (cast_errors.find(col) != cast_errors.end()) { - row.push_back(cast_errors[col]); - } else { - row.push_back(parse_chunk.data[col].GetValue(number_of_rows)); - } - } - } for (auto &cast_error : cast_errors) { std::ostringstream error; // Casting Error Message @@ -348,9 +355,9 @@ bool StringValueResult::AddRowInternal() { << LogicalTypeIdToString(parse_types[cast_error.first]) << "\'"; auto error_string = error.str(); LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read - 1); - + auto borked_line = ReconstructCurrentLine(); auto csv_error = CSVError::CastError(state_machine.options, names[cast_error.first], error_string, - cast_error.first, row, lines_per_batch); + cast_error.first, borked_line, lines_per_batch); error_handler.Error(csv_error); } // If we got here it means we are ignoring errors, hence we need to signify to our result scanner to ignore this @@ -615,8 +622,10 @@ void StringValueScanner::Flush(DataChunk &insert_chunk) { } LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read - parse_chunk.size() + line_error); +// auto borked_line = result.ReconstructCurrentLine(); + string empty; auto csv_error = CSVError::CastError(state_machine->options, csv_file_scan->names[col_idx], - error_message, col_idx, row, lines_per_batch); + error_message, col_idx, empty, lines_per_batch); error_handler->Error(csv_error); } borked_lines.insert(line_error++); @@ -632,8 +641,9 @@ void StringValueScanner::Flush(DataChunk &insert_chunk) { } LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read - parse_chunk.size() + line_error); + string empty; auto csv_error = CSVError::CastError(state_machine->options, csv_file_scan->names[col_idx], - error_message, col_idx, row, lines_per_batch); + error_message, col_idx, empty, lines_per_batch); error_handler->Error(csv_error); } diff --git a/src/execution/operator/csv_scanner/util/csv_error.cpp b/src/execution/operator/csv_scanner/util/csv_error.cpp index 7867421d57d1..c96893ed4cc3 100644 --- a/src/execution/operator/csv_scanner/util/csv_error.cpp +++ b/src/execution/operator/csv_scanner/util/csv_error.cpp @@ -76,9 +76,9 @@ CSVError::CSVError(string error_message_p, CSVErrorType type_p, LinesPerBoundary : error_message(std::move(error_message_p)), type(type_p), error_info(error_info_p) { } -CSVError::CSVError(string error_message_p, CSVErrorType type_p, idx_t column_idx_p, vector row_p, +CSVError::CSVError(string error_message_p, CSVErrorType type_p, idx_t column_idx_p, string csv_row_p, LinesPerBoundary error_info_p) - : error_message(std::move(error_message_p)), type(type_p), column_idx(column_idx_p), row(std::move(row_p)), + : error_message(std::move(error_message_p)), type(type_p), column_idx(column_idx_p), csv_row(std::move(csv_row_p)), error_info(error_info_p) { } @@ -102,8 +102,7 @@ CSVError CSVError::ColumnTypesError(case_insensitive_map_t sql_types_per_ return CSVError(exception, CSVErrorType::COLUMN_NAME_TYPE_MISMATCH, {}); } -CSVError CSVError::CastError(const CSVReaderOptions &options, string &column_name, string &cast_error, idx_t column_idx, - vector &row, LinesPerBoundary error_info) { +CSVError CSVError::CastError(const CSVReaderOptions &options, string &column_name, string &cast_error, idx_t column_idx, string &csv_row, LinesPerBoundary error_info) { std::ostringstream error; // Which column error << "Error when converting column \"" << column_name << "\"." << std::endl; @@ -112,7 +111,7 @@ CSVError CSVError::CastError(const CSVReaderOptions &options, string &column_nam error << std::endl; // What were the options error << options.ToString(); - return CSVError(error.str(), CSVErrorType::CAST_ERROR, column_idx, row, error_info); + return CSVError(error.str(), CSVErrorType::CAST_ERROR, column_idx, csv_row, error_info); } CSVError CSVError::LineSizeError(const CSVReaderOptions &options, idx_t actual_size, LinesPerBoundary error_info) { diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp index 4d5eeada36eb..c40045b74bc5 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp @@ -50,13 +50,13 @@ enum CSVErrorType : uint8_t { class CSVError { public: CSVError() {}; - CSVError(string error_message, CSVErrorType type, idx_t column_idx, vector row, LinesPerBoundary error_info); + CSVError(string error_message, CSVErrorType type, idx_t column_idx, string csv_row, LinesPerBoundary error_info); CSVError(string error_message, CSVErrorType type, LinesPerBoundary error_info); //! Produces error messages for column name -> type mismatch. static CSVError ColumnTypesError(case_insensitive_map_t sql_types_per_column, const vector &names); //! Produces error messages for casting errors static CSVError CastError(const CSVReaderOptions &options, string &column_name, string &cast_error, - idx_t column_idx, vector &row, LinesPerBoundary error_info); + idx_t column_idx, string &csv_row, LinesPerBoundary error_info); //! Produces error for when the line size exceeds the maximum line size option static CSVError LineSizeError(const CSVReaderOptions &options, idx_t actual_size, LinesPerBoundary error_info); //! Produces error for when the sniffer couldn't find viable options @@ -80,8 +80,8 @@ class CSVError { CSVErrorType type; //! Column Index where error happened idx_t column_idx; - //! Values from the row where error happened - vector row; + //! Original CSV row where error happened + string csv_row; //! Line information regarding this error LinesPerBoundary error_info; }; diff --git a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp index 2b1475f92861..4750da6b65db 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp @@ -117,6 +117,8 @@ class StringValueResult : public ScannerResult { //! Handles EmptyLine states static inline bool EmptyLine(StringValueResult &result, const idx_t buffer_pos); inline bool AddRowInternal(); + //! Reconstructs the current line to be used in error messages + string ReconstructCurrentLine(); void HandleOverLimitRows(); From 9b95261e6e1f533c098ea589fdbead7078c638e5 Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Fri, 23 Feb 2024 17:16:51 +0100 Subject: [PATCH 007/603] fix proper implementation of filter is on partition --- src/optimizer/filter_pushdown.cpp | 32 ++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/src/optimizer/filter_pushdown.cpp b/src/optimizer/filter_pushdown.cpp index ce26b351760e..44004a8280f0 100644 --- a/src/optimizer/filter_pushdown.cpp +++ b/src/optimizer/filter_pushdown.cpp @@ -13,17 +13,32 @@ using Filter = FilterPushdown::Filter; FilterPushdown::FilterPushdown(Optimizer &optimizer) : optimizer(optimizer), combiner(optimizer.context) { } -static bool FilterIsOnPartition(column_binding_set_t parition_bindings, Expression &filter_expression) { - return true; +static bool FilterIsOnPartition(column_binding_set_t partition_bindings, Expression &filter_expression) { + bool filter_is_on_partition = false; + ExpressionIterator::EnumerateChildren(filter_expression, [&](unique_ptr &child) { + switch (child->type) { + case ExpressionType::BOUND_COLUMN_REF: { + auto &col_ref = child->Cast(); + if (partition_bindings.find(col_ref.binding) != partition_bindings.end()) { + filter_is_on_partition = true; + return; + } + break; + } + default: + break; + } + }); + return filter_is_on_partition; } unique_ptr FilterPushdown::PushdownWindow(unique_ptr op) { D_ASSERT(op->type == LogicalOperatorType::LOGICAL_WINDOW); auto &window = op->Cast(); // go into expressions, then look into the partitions. - // if the filter applies ONLY to the partition (and is not some arithmetic expression) then you can push the filter into - // the children. - for (auto &expr: window.expressions) { + // if the filter applies ONLY to the partition (and is not some arithmetic expression) then you can push the filter + // into the children. + for (auto &expr : window.expressions) { if (expr->expression_class != ExpressionClass::BOUND_WINDOW) { continue; } @@ -43,6 +58,9 @@ unique_ptr FilterPushdown::PushdownWindow(unique_ptr> leftover_filters; for (auto &filter : filters) { + // check if a filter is on the partition + // create new filter pushdown with the filter, + // push into children if (FilterIsOnPartition(parition_bindings, *filter->filter)) { auto pushdown = FilterPushdown(optimizer); pushdown.filters.push_back(std::move(filter)); @@ -50,14 +68,10 @@ unique_ptr FilterPushdown::PushdownWindow(unique_ptr FilterPushdown::Rewrite(unique_ptr op) { From d59163e678cd6a358ba278af9f7e0ba7648a9ccc Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Fri, 23 Feb 2024 17:26:12 +0100 Subject: [PATCH 008/603] make format-fix --- test/optimizer/pushdown/pushdown_window_partition_filter.test | 1 - 1 file changed, 1 deletion(-) diff --git a/test/optimizer/pushdown/pushdown_window_partition_filter.test b/test/optimizer/pushdown/pushdown_window_partition_filter.test index 13a5948d0a7c..4f7a308d36a1 100644 --- a/test/optimizer/pushdown/pushdown_window_partition_filter.test +++ b/test/optimizer/pushdown/pushdown_window_partition_filter.test @@ -2,7 +2,6 @@ # description: Test pushdown of filters through window operators that are partitioned by. # group: [pushdown] - statement ok create table t1 as from VALUES ('A', 1), ('B', 3), ('C', 12), ('A', 5), ('B', 8), ('C', 9), ('A', 10), ('B', 20), ('C', 3) t(a, b); From bf320b4c87b52955b566a3864d0a3c5b84a94c37 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 23 Feb 2024 17:45:08 -0300 Subject: [PATCH 009/603] wip --- .../operator/csv_scanner/table_function/global_csv_state.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index 2f8e92e3718f..4bd982f4571f 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -168,10 +168,15 @@ void CSVGlobalState::FillRejectsTable() { auto col_name = bind_data.return_names[col_idx]; // Add the row to the rejects table appender.BeginRow(); + // 1. File Name appender.Append(string_t(file_name)); + // 2. Row Line appender.Append(row_line); + // 3. Column Index appender.Append(col_idx); + // 4. Column Name appender.Append(string_t("\"" + col_name + "\"")); + // 5 Parsed Value appender.Append(error.row[col_idx]); auto row_error_msg = From 2b093824ac56635b792710aaaa72eec1aaf9c3ff Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Mon, 26 Feb 2024 17:41:50 +0100 Subject: [PATCH 010/603] fix some more tests, add some more. can only push down fiters on windows if all window expressions have the filtered column in the partition --- src/optimizer/filter_pushdown.cpp | 53 ++++++++++++++----- src/optimizer/optimizer.cpp | 10 ++-- .../pushdown_window_partition_filter.test | 30 +++++++++++ 3 files changed, 76 insertions(+), 17 deletions(-) diff --git a/src/optimizer/filter_pushdown.cpp b/src/optimizer/filter_pushdown.cpp index 44004a8280f0..d25389f70344 100644 --- a/src/optimizer/filter_pushdown.cpp +++ b/src/optimizer/filter_pushdown.cpp @@ -25,6 +25,11 @@ static bool FilterIsOnPartition(column_binding_set_t partition_bindings, Express } break; } + case ExpressionType::CONJUNCTION_AND: + case ExpressionType::CONJUNCTION_OR: { + filter_is_on_partition = false; + return; + } default: break; } @@ -36,8 +41,15 @@ unique_ptr FilterPushdown::PushdownWindow(unique_ptrtype == LogicalOperatorType::LOGICAL_WINDOW); auto &window = op->Cast(); // go into expressions, then look into the partitions. - // if the filter applies ONLY to the partition (and is not some arithmetic expression) then you can push the filter + // if the filter applies to a partition in each window expression then you can push the filter // into the children. + auto pushdown = FilterPushdown(optimizer); + vector filters_to_pushdown; + filters_to_pushdown.reserve(filters.size()); + for (idx_t i = 0; i < filters.size(); i++) { + filters_to_pushdown.push_back(0); + } + // 1. Check every window expression for (auto &expr : window.expressions) { if (expr->expression_class != ExpressionClass::BOUND_WINDOW) { continue; @@ -45,8 +57,10 @@ unique_ptr FilterPushdown::PushdownWindow(unique_ptrCast(); auto &partitions = window_expr.partitions; column_binding_set_t parition_bindings; + // 2. Get the binding information of the partitions for (auto &partition_expr : partitions) { switch (partition_expr->type) { + // TODO: Add expressions for function expressions like FLOOR, CEIL etc. case ExpressionType::BOUND_COLUMN_REF: { auto &partition_col = partition_expr->Cast(); parition_bindings.insert(partition_col.binding); @@ -56,21 +70,36 @@ unique_ptr FilterPushdown::PushdownWindow(unique_ptr> leftover_filters; - for (auto &filter : filters) { - // check if a filter is on the partition - // create new filter pushdown with the filter, - // push into children - if (FilterIsOnPartition(parition_bindings, *filter->filter)) { - auto pushdown = FilterPushdown(optimizer); - pushdown.filters.push_back(std::move(filter)); - op->children[0] = pushdown.Rewrite(std::move(op->children[0])); + + // 3. Check if a filter is on one of the partitions + // in the event there are multiple window functions, a fiter can only be pushed down + // if each window function is partitioned on the filtered value + // if a filter is not on a partition fiters_to_parition[filter_index] = -1 + // which means the filter will no longer be pushed down, even if the filter is on a partition. + // tpcds q47 caught this + for (idx_t i = 0; i < filters.size(); i++) { + auto &filter = filters.at(i); + if (FilterIsOnPartition(parition_bindings, *filter->filter) && filters_to_pushdown[i] >= 0) { + filters_to_pushdown[i] = 1; continue; } - leftover_filters.push_back(std::move(filter)); + filters_to_pushdown[i] = -1; + } + } + // push the new filters into the child. + vector> leftover_filters; + for (idx_t i = 0; i < filters.size(); i++) { + if (filters_to_pushdown[i] == 1) { + pushdown.filters.push_back(std::move(filters.at(i))); + continue; } - filters = std::move(leftover_filters); + leftover_filters.push_back(std::move(filters.at(i))); } + if (!pushdown.filters.empty()) { + op->children[0] = pushdown.Rewrite(std::move(op->children[0])); + } + filters = std::move(leftover_filters); + return FinishPushdown(std::move(op)); } diff --git a/src/optimizer/optimizer.cpp b/src/optimizer/optimizer.cpp index 0a66f0007761..64c23a1d06fd 100644 --- a/src/optimizer/optimizer.cpp +++ b/src/optimizer/optimizer.cpp @@ -176,11 +176,11 @@ unique_ptr Optimizer::Optimize(unique_ptr plan column_lifetime.VisitOperator(*plan); }); - // compress data based on statistics for materializing operators - RunOptimizer(OptimizerType::COMPRESSED_MATERIALIZATION, [&]() { - CompressedMaterialization compressed_materialization(context, binder, std::move(statistics_map)); - compressed_materialization.Compress(plan); - }); + // // compress data based on statistics for materializing operators + // RunOptimizer(OptimizerType::COMPRESSED_MATERIALIZATION, [&]() { + // CompressedMaterialization compressed_materialization(context, binder, std::move(statistics_map)); + // compressed_materialization.Compress(plan); + // }); // transform ORDER BY + LIMIT to TopN RunOptimizer(OptimizerType::TOP_N, [&]() { diff --git a/test/optimizer/pushdown/pushdown_window_partition_filter.test b/test/optimizer/pushdown/pushdown_window_partition_filter.test index 4f7a308d36a1..03fcf8df0f75 100644 --- a/test/optimizer/pushdown/pushdown_window_partition_filter.test +++ b/test/optimizer/pushdown/pushdown_window_partition_filter.test @@ -2,6 +2,9 @@ # description: Test pushdown of filters through window operators that are partitioned by. # group: [pushdown] +statement ok +pragma enable_verification; + statement ok create table t1 as from VALUES ('A', 1), ('B', 3), ('C', 12), ('A', 5), ('B', 8), ('C', 9), ('A', 10), ('B', 20), ('C', 3) t(a, b); @@ -20,6 +23,31 @@ query III no_sort result_1 select * from window_no_filter where a != 'C' order by a; ---- +statement ok +create table partition_and_rank_me as from values ('A', 10, 'A', 'id'), ('A', 20, 'A', 'id'), ('A', 30, 'B', 'id'), ('D', 40, 'B', 'id'), ('D', 50, 'C', 'id'), ('D', 60, 'C', 'id') t(a, b, c, d); + +# can't push down the filter c!='B', since the values of the rank() window function +# are affected by the existence of the rows where c='B' +query IIII +select a, sum(b) OVER (PARTITION BY c), rank() OVER (PARTITION BY d ORDER BY b), c from partition_and_rank_me order by all +---- +A 30 1 A +A 30 2 A +A 70 3 B +D 70 4 B +D 110 5 C +D 110 6 C + +# can't push down the filter c!='B', since the values of the rank() window function +# are affected by the existence of the rows where c='B' +query IIII +select * from (select a, sum(b) OVER (PARTITION BY c), rank() OVER (PARTITION BY d ORDER BY b), c from partition_and_rank_me) where c != 'B' order by all; +---- +A 30 1 A +A 30 2 A +D 110 5 C +D 110 6 C + statement ok pragma explain_output=OPTIMIZED_ONLY @@ -27,3 +55,5 @@ query II explain select * from window_no_filter where a != 'C' order by a; ---- logical_opt :.*WINDOW.*FILTER.* + + From 65b0f790c417462e984f8affa71fec0c9279d52b Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Mon, 26 Feb 2024 17:49:47 +0100 Subject: [PATCH 011/603] move pushdown window to new file --- src/optimizer/filter_pushdown.cpp | 90 -------------------- src/optimizer/pushdown/CMakeLists.txt | 3 +- src/optimizer/pushdown/pushdown_window.cpp | 99 ++++++++++++++++++++++ 3 files changed, 101 insertions(+), 91 deletions(-) create mode 100644 src/optimizer/pushdown/pushdown_window.cpp diff --git a/src/optimizer/filter_pushdown.cpp b/src/optimizer/filter_pushdown.cpp index d25389f70344..eef1a23aada5 100644 --- a/src/optimizer/filter_pushdown.cpp +++ b/src/optimizer/filter_pushdown.cpp @@ -13,96 +13,6 @@ using Filter = FilterPushdown::Filter; FilterPushdown::FilterPushdown(Optimizer &optimizer) : optimizer(optimizer), combiner(optimizer.context) { } -static bool FilterIsOnPartition(column_binding_set_t partition_bindings, Expression &filter_expression) { - bool filter_is_on_partition = false; - ExpressionIterator::EnumerateChildren(filter_expression, [&](unique_ptr &child) { - switch (child->type) { - case ExpressionType::BOUND_COLUMN_REF: { - auto &col_ref = child->Cast(); - if (partition_bindings.find(col_ref.binding) != partition_bindings.end()) { - filter_is_on_partition = true; - return; - } - break; - } - case ExpressionType::CONJUNCTION_AND: - case ExpressionType::CONJUNCTION_OR: { - filter_is_on_partition = false; - return; - } - default: - break; - } - }); - return filter_is_on_partition; -} - -unique_ptr FilterPushdown::PushdownWindow(unique_ptr op) { - D_ASSERT(op->type == LogicalOperatorType::LOGICAL_WINDOW); - auto &window = op->Cast(); - // go into expressions, then look into the partitions. - // if the filter applies to a partition in each window expression then you can push the filter - // into the children. - auto pushdown = FilterPushdown(optimizer); - vector filters_to_pushdown; - filters_to_pushdown.reserve(filters.size()); - for (idx_t i = 0; i < filters.size(); i++) { - filters_to_pushdown.push_back(0); - } - // 1. Check every window expression - for (auto &expr : window.expressions) { - if (expr->expression_class != ExpressionClass::BOUND_WINDOW) { - continue; - } - auto &window_expr = expr->Cast(); - auto &partitions = window_expr.partitions; - column_binding_set_t parition_bindings; - // 2. Get the binding information of the partitions - for (auto &partition_expr : partitions) { - switch (partition_expr->type) { - // TODO: Add expressions for function expressions like FLOOR, CEIL etc. - case ExpressionType::BOUND_COLUMN_REF: { - auto &partition_col = partition_expr->Cast(); - parition_bindings.insert(partition_col.binding); - break; - } - default: - break; - } - } - - // 3. Check if a filter is on one of the partitions - // in the event there are multiple window functions, a fiter can only be pushed down - // if each window function is partitioned on the filtered value - // if a filter is not on a partition fiters_to_parition[filter_index] = -1 - // which means the filter will no longer be pushed down, even if the filter is on a partition. - // tpcds q47 caught this - for (idx_t i = 0; i < filters.size(); i++) { - auto &filter = filters.at(i); - if (FilterIsOnPartition(parition_bindings, *filter->filter) && filters_to_pushdown[i] >= 0) { - filters_to_pushdown[i] = 1; - continue; - } - filters_to_pushdown[i] = -1; - } - } - // push the new filters into the child. - vector> leftover_filters; - for (idx_t i = 0; i < filters.size(); i++) { - if (filters_to_pushdown[i] == 1) { - pushdown.filters.push_back(std::move(filters.at(i))); - continue; - } - leftover_filters.push_back(std::move(filters.at(i))); - } - if (!pushdown.filters.empty()) { - op->children[0] = pushdown.Rewrite(std::move(op->children[0])); - } - filters = std::move(leftover_filters); - - return FinishPushdown(std::move(op)); -} - unique_ptr FilterPushdown::Rewrite(unique_ptr op) { D_ASSERT(!combiner.HasFilters()); switch (op->type) { diff --git a/src/optimizer/pushdown/CMakeLists.txt b/src/optimizer/pushdown/CMakeLists.txt index 4ce1f851ba8b..c1df2b42f0b9 100644 --- a/src/optimizer/pushdown/CMakeLists.txt +++ b/src/optimizer/pushdown/CMakeLists.txt @@ -13,7 +13,8 @@ add_library_unity( pushdown_projection.cpp pushdown_semi_anti_join.cpp pushdown_set_operation.cpp - pushdown_single_join.cpp) + pushdown_single_join.cpp + pushdown_window.cpp) set(ALL_OBJECT_FILES ${ALL_OBJECT_FILES} $ PARENT_SCOPE) diff --git a/src/optimizer/pushdown/pushdown_window.cpp b/src/optimizer/pushdown/pushdown_window.cpp new file mode 100644 index 000000000000..ffb0f4e3954e --- /dev/null +++ b/src/optimizer/pushdown/pushdown_window.cpp @@ -0,0 +1,99 @@ +#include "duckdb/optimizer/filter_pushdown.hpp" +#include "duckdb/parser/expression/window_expression.hpp" +#include "duckdb/planner/expression/bound_columnref_expression.hpp" +#include "duckdb/planner/expression/bound_window_expression.hpp" +#include "duckdb/planner/expression_iterator.hpp" +#include "duckdb/planner/operator/logical_window.hpp" + +namespace duckdb { + +static bool FilterIsOnPartition(column_binding_set_t partition_bindings, Expression &filter_expression) { + bool filter_is_on_partition = false; + ExpressionIterator::EnumerateChildren(filter_expression, [&](unique_ptr &child) { + switch (child->type) { + case ExpressionType::BOUND_COLUMN_REF: { + auto &col_ref = child->Cast(); + if (partition_bindings.find(col_ref.binding) != partition_bindings.end()) { + filter_is_on_partition = true; + return; + } + break; + } + case ExpressionType::CONJUNCTION_AND: + case ExpressionType::CONJUNCTION_OR: { + filter_is_on_partition = false; + return; + } + default: + break; + } + }); + return filter_is_on_partition; +} + +unique_ptr FilterPushdown::PushdownWindow(unique_ptr op) { + D_ASSERT(op->type == LogicalOperatorType::LOGICAL_WINDOW); + auto &window = op->Cast(); + // go into expressions, then look into the partitions. + // if the filter applies to a partition in each window expression then you can push the filter + // into the children. + auto pushdown = FilterPushdown(optimizer); + vector filters_to_pushdown; + filters_to_pushdown.reserve(filters.size()); + for (idx_t i = 0; i < filters.size(); i++) { + filters_to_pushdown.push_back(0); + } + // 1. Check every window expression + for (auto &expr : window.expressions) { + if (expr->expression_class != ExpressionClass::BOUND_WINDOW) { + continue; + } + auto &window_expr = expr->Cast(); + auto &partitions = window_expr.partitions; + column_binding_set_t parition_bindings; + // 2. Get the binding information of the partitions + for (auto &partition_expr : partitions) { + switch (partition_expr->type) { + // TODO: Add expressions for function expressions like FLOOR, CEIL etc. + case ExpressionType::BOUND_COLUMN_REF: { + auto &partition_col = partition_expr->Cast(); + parition_bindings.insert(partition_col.binding); + break; + } + default: + break; + } + } + + // 3. Check if a filter is on one of the partitions + // in the event there are multiple window functions, a fiter can only be pushed down + // if each window function is partitioned on the filtered value + // if a filter is not on a partition fiters_to_parition[filter_index] = -1 + // which means the filter will no longer be pushed down, even if the filter is on a partition. + // tpcds q47 caught this + for (idx_t i = 0; i < filters.size(); i++) { + auto &filter = filters.at(i); + if (FilterIsOnPartition(parition_bindings, *filter->filter) && filters_to_pushdown[i] >= 0) { + filters_to_pushdown[i] = 1; + continue; + } + filters_to_pushdown[i] = -1; + } + } + // push the new filters into the child. + vector> leftover_filters; + for (idx_t i = 0; i < filters.size(); i++) { + if (filters_to_pushdown[i] == 1) { + pushdown.filters.push_back(std::move(filters.at(i))); + continue; + } + leftover_filters.push_back(std::move(filters.at(i))); + } + if (!pushdown.filters.empty()) { + op->children[0] = pushdown.Rewrite(std::move(op->children[0])); + } + filters = std::move(leftover_filters); + + return FinishPushdown(std::move(op)); +} +} // namespace duckdb From 04d4fcb0a87ac19a984b488eef3e93b678056363 Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Tue, 27 Feb 2024 10:40:10 +0100 Subject: [PATCH 012/603] add compressed materialization back in --- src/optimizer/optimizer.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/optimizer/optimizer.cpp b/src/optimizer/optimizer.cpp index 64c23a1d06fd..0a66f0007761 100644 --- a/src/optimizer/optimizer.cpp +++ b/src/optimizer/optimizer.cpp @@ -176,11 +176,11 @@ unique_ptr Optimizer::Optimize(unique_ptr plan column_lifetime.VisitOperator(*plan); }); - // // compress data based on statistics for materializing operators - // RunOptimizer(OptimizerType::COMPRESSED_MATERIALIZATION, [&]() { - // CompressedMaterialization compressed_materialization(context, binder, std::move(statistics_map)); - // compressed_materialization.Compress(plan); - // }); + // compress data based on statistics for materializing operators + RunOptimizer(OptimizerType::COMPRESSED_MATERIALIZATION, [&]() { + CompressedMaterialization compressed_materialization(context, binder, std::move(statistics_map)); + compressed_materialization.Compress(plan); + }); // transform ORDER BY + LIMIT to TopN RunOptimizer(OptimizerType::TOP_N, [&]() { From 93e0f4725c4d10ee4aae35cf9703882f3ca704d8 Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Wed, 28 Feb 2024 10:37:15 +0100 Subject: [PATCH 013/603] add a conjunction test --- .../pushdown_window_partition_filter.test | 34 ++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/test/optimizer/pushdown/pushdown_window_partition_filter.test b/test/optimizer/pushdown/pushdown_window_partition_filter.test index 03fcf8df0f75..fe7f7e4acb9e 100644 --- a/test/optimizer/pushdown/pushdown_window_partition_filter.test +++ b/test/optimizer/pushdown/pushdown_window_partition_filter.test @@ -8,6 +8,9 @@ pragma enable_verification; statement ok create table t1 as from VALUES ('A', 1), ('B', 3), ('C', 12), ('A', 5), ('B', 8), ('C', 9), ('A', 10), ('B', 20), ('C', 3) t(a, b); +statement ok +pragma explain_output=OPTIMIZED_ONLY + statement ok create view window_with_filter as select a, b, LEAD(b) OVER (partition by a) as lead from t1 where a != 'C'; @@ -24,10 +27,15 @@ select * from window_no_filter where a != 'C' order by a; ---- statement ok -create table partition_and_rank_me as from values ('A', 10, 'A', 'id'), ('A', 20, 'A', 'id'), ('A', 30, 'B', 'id'), ('D', 40, 'B', 'id'), ('D', 50, 'C', 'id'), ('D', 60, 'C', 'id') t(a, b, c, d); +create table partition_and_rank_me as from values + ('A', 10, 'A', 'id'), + ('A', 20, 'A', 'id'), + ('A', 30, 'B', 'id'), + ('D', 40, 'B', 'id'), + ('D', 50, 'C', 'id'), + ('D', 60, 'C', 'id') +t(a, b, c, d); -# can't push down the filter c!='B', since the values of the rank() window function -# are affected by the existence of the rows where c='B' query IIII select a, sum(b) OVER (PARTITION BY c), rank() OVER (PARTITION BY d ORDER BY b), c from partition_and_rank_me order by all ---- @@ -48,8 +56,24 @@ A 30 2 A D 110 5 C D 110 6 C -statement ok -pragma explain_output=OPTIMIZED_ONLY +# The filter is on the partitioned column, but is parto of a conjunction, so we dont push it down +query IIII +select * from (select a, sum(b) OVER (PARTITION BY c), rank() OVER (PARTITION BY c ORDER BY b), c from partition_and_rank_me) where (c = 'B' AND a = 'D') order by all; +---- +D 70 2 B + +# result of above query with pushdown +query IIII +select * from (select a, sum(b) OVER (PARTITION BY c), rank() OVER (PARTITION BY c ORDER BY b), c from partition_and_rank_me where (c = 'B' AND a = 'D')) order by all; +---- +D 40 1 B + +# can't push down the filter c!='B', since the values of the rank() window function +# are affected by the existence of the rows where c='B' +query II +explain select * from (select a, sum(b) OVER (PARTITION BY c), rank() OVER (PARTITION BY d ORDER BY b), c from partition_and_rank_me) where c != 'B' order by all; +---- +logical_opt :.*FILTER.*WINDOW.* query II explain select * from window_no_filter where a != 'C' order by a; From 5a2cf4d53bba8315ff107319b5a5a81383beea58 Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Wed, 28 Feb 2024 10:39:32 +0100 Subject: [PATCH 014/603] immediately return if no partitions are present in a window function --- src/optimizer/pushdown/pushdown_window.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/optimizer/pushdown/pushdown_window.cpp b/src/optimizer/pushdown/pushdown_window.cpp index ffb0f4e3954e..a9ba391890b1 100644 --- a/src/optimizer/pushdown/pushdown_window.cpp +++ b/src/optimizer/pushdown/pushdown_window.cpp @@ -50,6 +50,11 @@ unique_ptr FilterPushdown::PushdownWindow(unique_ptrCast(); auto &partitions = window_expr.partitions; + if (partitions.empty()) { + // all window expressions need to be partitioned by the same column + // in order to push down the window. + return FinishPushdown(std::move(op)); + } column_binding_set_t parition_bindings; // 2. Get the binding information of the partitions for (auto &partition_expr : partitions) { From ba93182badfe9c0a608d175308898803cef090cf Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 28 Feb 2024 09:16:54 -0300 Subject: [PATCH 015/603] Enum for CSV Errors, cleaning up output --- .../scanner/string_value_scanner.cpp | 13 ++++----- .../table_function/global_csv_state.cpp | 18 +++++------- .../operator/csv_scanner/util/csv_error.cpp | 3 +- .../operator/persistent/csv_rejects_table.cpp | 29 +++++++++++++++++-- 4 files changed, 42 insertions(+), 21 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 3569bfab62ec..d5a295190706 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -323,18 +323,17 @@ void StringValueResult::NullPaddingQuotedNewlineCheck() { } //! Reconstructs the current line to be used in error messages -string StringValueResult::ReconstructCurrentLine(){ - LinePosition current_line_start = {iterator.pos.buffer_idx, iterator.pos.buffer_pos, - buffer_size}; +string StringValueResult::ReconstructCurrentLine() { + LinePosition current_line_start = {iterator.pos.buffer_idx, iterator.pos.buffer_pos, buffer_size}; idx_t current_line_size = current_line_start - previous_line_start; string result; result.resize(current_line_size); - if (iterator.pos.buffer_idx == previous_line_start.buffer_idx){ + if (iterator.pos.buffer_idx == previous_line_start.buffer_idx) { idx_t result_idx = 0; - for (idx_t i = previous_line_start.buffer_pos; i < iterator.pos.buffer_pos; i ++){ + for (idx_t i = previous_line_start.buffer_pos; i < iterator.pos.buffer_pos; i++) { result[result_idx++] = buffer_ptr[i]; } - } else{ + } else { throw InternalException("Oh no"); } return result; @@ -622,7 +621,7 @@ void StringValueScanner::Flush(DataChunk &insert_chunk) { } LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read - parse_chunk.size() + line_error); -// auto borked_line = result.ReconstructCurrentLine(); + // auto borked_line = result.ReconstructCurrentLine(); string empty; auto csv_error = CSVError::CastError(state_machine->options, csv_file_scan->names[col_idx], error_message, col_idx, empty, lines_per_batch); diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index f8b8000e9843..7e3a24d6f77c 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -154,7 +154,7 @@ void CSVGlobalState::FillRejectsTable() { for (auto &error_vector : errors) { for (auto &error : error_vector.second) { if (error.type != CSVErrorType::CAST_ERROR) { - // For now we only will use it for casting errors + // For now, we only will use it for casting errors continue; } // short circuit if we already have too many rejects @@ -168,21 +168,17 @@ void CSVGlobalState::FillRejectsTable() { auto col_name = bind_data.return_names[col_idx]; // Add the row to the rejects table appender.BeginRow(); - // 1. File Name + // 1. File Path appender.Append(string_t(file_name)); // 2. Row Line appender.Append(row_line); - // 3. Column Index + // 3. Column Index (If Applicable) appender.Append(col_idx); - // 4. Column Name + // 4. Column Name (If Applicable) appender.Append(string_t("\"" + col_name + "\"")); - // 5 Parsed Value - appender.Append(error.row[col_idx]); - - auto row_error_msg = - StringUtil::Format("Could not convert string '%s' to '%s'", error.row[col_idx].ToString(), - file->types[col_idx].ToString()); - appender.Append(string_t(row_error_msg)); + // 5. Error Type (ENUM?) + // 6. Full Error Message + // 7. Original CSV Line appender.EndRow(); } appender.Close(); diff --git a/src/execution/operator/csv_scanner/util/csv_error.cpp b/src/execution/operator/csv_scanner/util/csv_error.cpp index c96893ed4cc3..4bd08a7d7a8d 100644 --- a/src/execution/operator/csv_scanner/util/csv_error.cpp +++ b/src/execution/operator/csv_scanner/util/csv_error.cpp @@ -102,7 +102,8 @@ CSVError CSVError::ColumnTypesError(case_insensitive_map_t sql_types_per_ return CSVError(exception, CSVErrorType::COLUMN_NAME_TYPE_MISMATCH, {}); } -CSVError CSVError::CastError(const CSVReaderOptions &options, string &column_name, string &cast_error, idx_t column_idx, string &csv_row, LinesPerBoundary error_info) { +CSVError CSVError::CastError(const CSVReaderOptions &options, string &column_name, string &cast_error, idx_t column_idx, + string &csv_row, LinesPerBoundary error_info) { std::ostringstream error; // Which column error << "Error when converting column \"" << column_name << "\"." << std::endl; diff --git a/src/execution/operator/persistent/csv_rejects_table.cpp b/src/execution/operator/persistent/csv_rejects_table.cpp index 7d01723a7718..2e64d637e0ec 100644 --- a/src/execution/operator/persistent/csv_rejects_table.cpp +++ b/src/execution/operator/persistent/csv_rejects_table.cpp @@ -3,6 +3,7 @@ #include "duckdb/function/table/read_csv.hpp" #include "duckdb/execution/operator/persistent/csv_rejects_table.hpp" #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/parser/parsed_data/create_type_info.hpp" namespace duckdb { @@ -21,15 +22,39 @@ shared_ptr CSVRejectsTable::GetOrCreate(ClientContext &context, void CSVRejectsTable::InitializeTable(ClientContext &context, const ReadCSVData &data) { // (Re)Create the temporary rejects table auto &catalog = Catalog::GetCatalog(context, TEMP_CATALOG); + + // Create CSV_ERROR_TYPE ENUM + string enum_name = "CSV_ERROR_TYPE"; + Vector order_errors(LogicalType::VARCHAR, 5); + order_errors.SetValue(0, "CAST"); + order_errors.SetValue(0, "MISSING COLUMNS"); + order_errors.SetValue(0, "TOO MANY COLUMNS"); + order_errors.SetValue(0, "UNQUOTED VALUE"); + order_errors.SetValue(0, "LINE SIZE OVER MAXIMUM"); + LogicalType enum_type = LogicalType::ENUM(enum_name, order_errors, 5); + auto type_info = make_uniq(enum_name, enum_type); + type_info->temporary = true; + type_info->on_conflict = OnCreateConflict::IGNORE_ON_CONFLICT; + catalog.CreateType(context, *type_info); + + // Create Rejects Table auto info = make_uniq(TEMP_CATALOG, DEFAULT_SCHEMA, name); info->temporary = true; info->on_conflict = OnCreateConflict::ERROR_ON_CONFLICT; + // 1. File Path info->columns.AddColumn(ColumnDefinition("file", LogicalType::VARCHAR)); + // 2. Row Line info->columns.AddColumn(ColumnDefinition("line", LogicalType::BIGINT)); + // 3. Column Index (If Applicable) info->columns.AddColumn(ColumnDefinition("column", LogicalType::BIGINT)); + // 4. Column Name (If Applicable) info->columns.AddColumn(ColumnDefinition("column_name", LogicalType::VARCHAR)); - info->columns.AddColumn(ColumnDefinition("parsed_value", LogicalType::VARCHAR)); - info->columns.AddColumn(ColumnDefinition("error", LogicalType::VARCHAR)); + // 5. Error Type + info->columns.AddColumn(ColumnDefinition("error_type", enum_type)); + // 6. Full Error Message + info->columns.AddColumn(ColumnDefinition("error_message", LogicalType::VARCHAR)); + // 7. Original CSV Line + info->columns.AddColumn(ColumnDefinition("csv_line", LogicalType::VARCHAR)); catalog.CreateTable(context, std::move(info)); From 551865f39d953c3de33e6ea0781940f5896d7e08 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 28 Feb 2024 10:11:18 -0300 Subject: [PATCH 016/603] Several tweaks for the tables --- .../scanner/string_value_scanner.cpp | 9 ++++----- .../table_function/global_csv_state.cpp | 8 +++++--- .../operator/persistent/csv_rejects_table.cpp | 18 ++++++++---------- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index d5a295190706..1002d9e6b164 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -324,13 +324,12 @@ void StringValueResult::NullPaddingQuotedNewlineCheck() { //! Reconstructs the current line to be used in error messages string StringValueResult::ReconstructCurrentLine() { - LinePosition current_line_start = {iterator.pos.buffer_idx, iterator.pos.buffer_pos, buffer_size}; - idx_t current_line_size = current_line_start - previous_line_start; + idx_t current_line_size = previous_line_start - pre_previous_line_start; string result; - result.resize(current_line_size); - if (iterator.pos.buffer_idx == previous_line_start.buffer_idx) { + result.resize(current_line_size - 1); + if (previous_line_start.buffer_idx == pre_previous_line_start.buffer_idx) { idx_t result_idx = 0; - for (idx_t i = previous_line_start.buffer_pos; i < iterator.pos.buffer_pos; i++) { + for (idx_t i = pre_previous_line_start.buffer_pos + 1; i < previous_line_start.buffer_pos; i++) { result[result_idx++] = buffer_ptr[i]; } } else { diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index 7e3a24d6f77c..f719a83521fd 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -147,7 +147,6 @@ void CSVGlobalState::FillRejectsTable() { lock_guard lock(rejects->write_lock); auto &table = rejects->GetTable(context); InternalAppender appender(context, table); - for (auto &file : file_scans) { auto file_name = file->file_path; auto &errors = file->error_handler->errors; @@ -177,8 +176,11 @@ void CSVGlobalState::FillRejectsTable() { // 4. Column Name (If Applicable) appender.Append(string_t("\"" + col_name + "\"")); // 5. Error Type (ENUM?) - // 6. Full Error Message - // 7. Original CSV Line + appender.Append(string_t("CAST")); + // 6. Original CSV Line + appender.Append(string_t(error.csv_row)); + // 7. Full Error Message + appender.Append(string_t(error.error_message)); appender.EndRow(); } appender.Close(); diff --git a/src/execution/operator/persistent/csv_rejects_table.cpp b/src/execution/operator/persistent/csv_rejects_table.cpp index 2e64d637e0ec..d1c9f13169aa 100644 --- a/src/execution/operator/persistent/csv_rejects_table.cpp +++ b/src/execution/operator/persistent/csv_rejects_table.cpp @@ -27,10 +27,10 @@ void CSVRejectsTable::InitializeTable(ClientContext &context, const ReadCSVData string enum_name = "CSV_ERROR_TYPE"; Vector order_errors(LogicalType::VARCHAR, 5); order_errors.SetValue(0, "CAST"); - order_errors.SetValue(0, "MISSING COLUMNS"); - order_errors.SetValue(0, "TOO MANY COLUMNS"); - order_errors.SetValue(0, "UNQUOTED VALUE"); - order_errors.SetValue(0, "LINE SIZE OVER MAXIMUM"); + order_errors.SetValue(1, "MISSING COLUMNS"); + order_errors.SetValue(2, "TOO MANY COLUMNS"); + order_errors.SetValue(3, "UNQUOTED VALUE"); + order_errors.SetValue(4, "LINE SIZE OVER MAXIMUM"); LogicalType enum_type = LogicalType::ENUM(enum_name, order_errors, 5); auto type_info = make_uniq(enum_name, enum_type); type_info->temporary = true; @@ -46,18 +46,16 @@ void CSVRejectsTable::InitializeTable(ClientContext &context, const ReadCSVData // 2. Row Line info->columns.AddColumn(ColumnDefinition("line", LogicalType::BIGINT)); // 3. Column Index (If Applicable) - info->columns.AddColumn(ColumnDefinition("column", LogicalType::BIGINT)); + info->columns.AddColumn(ColumnDefinition("column_idx", LogicalType::BIGINT)); // 4. Column Name (If Applicable) info->columns.AddColumn(ColumnDefinition("column_name", LogicalType::VARCHAR)); // 5. Error Type info->columns.AddColumn(ColumnDefinition("error_type", enum_type)); - // 6. Full Error Message - info->columns.AddColumn(ColumnDefinition("error_message", LogicalType::VARCHAR)); - // 7. Original CSV Line + // 6. Original CSV Line info->columns.AddColumn(ColumnDefinition("csv_line", LogicalType::VARCHAR)); - + // 7. Full Error Message + info->columns.AddColumn(ColumnDefinition("error_message", LogicalType::VARCHAR)); catalog.CreateTable(context, std::move(info)); - count = 0; } From 5eae50f43d7824b66785b0778d419bff06bcce65 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 28 Feb 2024 12:30:17 -0300 Subject: [PATCH 017/603] We can also store the global byte where an error shows up --- .../csv_scanner/buffer_manager/csv_buffer.cpp | 10 +++++----- .../csv_scanner/scanner/string_value_scanner.cpp | 9 +++++---- .../table_function/global_csv_state.cpp | 12 +++++++----- .../operator/csv_scanner/util/csv_error.cpp | 8 ++++---- .../operator/persistent/csv_rejects_table.cpp | 16 +++++++++------- .../operator/csv_scanner/csv_buffer.hpp | 12 +++++++----- .../execution/operator/csv_scanner/csv_error.hpp | 7 +++++-- .../csv_scanner/string_value_scanner.hpp | 3 +++ 8 files changed, 45 insertions(+), 32 deletions(-) diff --git a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp index 8c29ae79fb43..e5a53bdeb1f3 100644 --- a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp +++ b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp @@ -5,7 +5,7 @@ namespace duckdb { CSVBuffer::CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle &file_handle, idx_t &global_csv_current_position, idx_t file_number_p) - : context(context), file_number(file_number_p), can_seek(file_handle.CanSeek()) { + : context(context), requested_size(buffer_size_p), file_number(file_number_p), can_seek(file_handle.CanSeek()) { AllocateBuffer(buffer_size_p); auto buffer = Ptr(); actual_buffer_size = file_handle.Read(buffer, buffer_size_p); @@ -19,8 +19,8 @@ CSVBuffer::CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle CSVBuffer::CSVBuffer(CSVFileHandle &file_handle, ClientContext &context, idx_t buffer_size, idx_t global_csv_current_position, idx_t file_number_p, idx_t buffer_idx_p) - : context(context), global_csv_start(global_csv_current_position), file_number(file_number_p), - can_seek(file_handle.CanSeek()), buffer_idx(buffer_idx_p) { + : context(context), requested_size(buffer_size), global_csv_start(global_csv_current_position), + file_number(file_number_p), can_seek(file_handle.CanSeek()), buffer_idx(buffer_idx_p) { AllocateBuffer(buffer_size); auto buffer = handle.Ptr(); actual_buffer_size = file_handle.Read(handle.Ptr(), buffer_size); @@ -73,8 +73,8 @@ shared_ptr CSVBuffer::Pin(CSVFileHandle &file_handle, bool &has Reload(file_handle); has_seeked = true; } - return make_shared(buffer_manager.Pin(block), actual_buffer_size, last_buffer, file_number, - buffer_idx); + return make_shared(buffer_manager.Pin(block), actual_buffer_size, requested_size, last_buffer, + file_number, buffer_idx); } void CSVBuffer::Unpin() { diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 1002d9e6b164..10bd3ce3029d 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -354,8 +354,9 @@ bool StringValueResult::AddRowInternal() { auto error_string = error.str(); LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read - 1); auto borked_line = ReconstructCurrentLine(); - auto csv_error = CSVError::CastError(state_machine.options, names[cast_error.first], error_string, - cast_error.first, borked_line, lines_per_batch); + auto csv_error = CSVError::CastError( + state_machine.options, names[cast_error.first], error_string, cast_error.first, borked_line, + lines_per_batch, pre_previous_line_start.GetGlobalPosition(buffer_handles.front()->requested_size)); error_handler.Error(csv_error); } // If we got here it means we are ignoring errors, hence we need to signify to our result scanner to ignore this @@ -623,7 +624,7 @@ void StringValueScanner::Flush(DataChunk &insert_chunk) { // auto borked_line = result.ReconstructCurrentLine(); string empty; auto csv_error = CSVError::CastError(state_machine->options, csv_file_scan->names[col_idx], - error_message, col_idx, empty, lines_per_batch); + error_message, col_idx, empty, lines_per_batch, 0); error_handler->Error(csv_error); } borked_lines.insert(line_error++); @@ -641,7 +642,7 @@ void StringValueScanner::Flush(DataChunk &insert_chunk) { lines_read - parse_chunk.size() + line_error); string empty; auto csv_error = CSVError::CastError(state_machine->options, csv_file_scan->names[col_idx], - error_message, col_idx, empty, lines_per_batch); + error_message, col_idx, empty, lines_per_batch, 0); error_handler->Error(csv_error); } diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index f719a83521fd..77982d83fab2 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -171,15 +171,17 @@ void CSVGlobalState::FillRejectsTable() { appender.Append(string_t(file_name)); // 2. Row Line appender.Append(row_line); - // 3. Column Index (If Applicable) + // 3. Byte Position where error occurred + appender.Append(error.byte_position); + // 4. Column Index (If Applicable) appender.Append(col_idx); - // 4. Column Name (If Applicable) + // 5. Column Name (If Applicable) appender.Append(string_t("\"" + col_name + "\"")); - // 5. Error Type (ENUM?) + // 6. Error Type (ENUM?) appender.Append(string_t("CAST")); - // 6. Original CSV Line + // 7. Original CSV Line appender.Append(string_t(error.csv_row)); - // 7. Full Error Message + // 8. Full Error Message appender.Append(string_t(error.error_message)); appender.EndRow(); } diff --git a/src/execution/operator/csv_scanner/util/csv_error.cpp b/src/execution/operator/csv_scanner/util/csv_error.cpp index 4bd08a7d7a8d..b77437c88f6b 100644 --- a/src/execution/operator/csv_scanner/util/csv_error.cpp +++ b/src/execution/operator/csv_scanner/util/csv_error.cpp @@ -77,9 +77,9 @@ CSVError::CSVError(string error_message_p, CSVErrorType type_p, LinesPerBoundary } CSVError::CSVError(string error_message_p, CSVErrorType type_p, idx_t column_idx_p, string csv_row_p, - LinesPerBoundary error_info_p) + LinesPerBoundary error_info_p, idx_t byte_position_p) : error_message(std::move(error_message_p)), type(type_p), column_idx(column_idx_p), csv_row(std::move(csv_row_p)), - error_info(error_info_p) { + error_info(error_info_p), byte_position(byte_position_p) { } CSVError CSVError::ColumnTypesError(case_insensitive_map_t sql_types_per_column, const vector &names) { @@ -103,7 +103,7 @@ CSVError CSVError::ColumnTypesError(case_insensitive_map_t sql_types_per_ } CSVError CSVError::CastError(const CSVReaderOptions &options, string &column_name, string &cast_error, idx_t column_idx, - string &csv_row, LinesPerBoundary error_info) { + string &csv_row, LinesPerBoundary error_info, idx_t byte_position) { std::ostringstream error; // Which column error << "Error when converting column \"" << column_name << "\"." << std::endl; @@ -112,7 +112,7 @@ CSVError CSVError::CastError(const CSVReaderOptions &options, string &column_nam error << std::endl; // What were the options error << options.ToString(); - return CSVError(error.str(), CSVErrorType::CAST_ERROR, column_idx, csv_row, error_info); + return CSVError(error.str(), CSVErrorType::CAST_ERROR, column_idx, csv_row, error_info, byte_position); } CSVError CSVError::LineSizeError(const CSVReaderOptions &options, idx_t actual_size, LinesPerBoundary error_info) { diff --git a/src/execution/operator/persistent/csv_rejects_table.cpp b/src/execution/operator/persistent/csv_rejects_table.cpp index d1c9f13169aa..3f2acf553f21 100644 --- a/src/execution/operator/persistent/csv_rejects_table.cpp +++ b/src/execution/operator/persistent/csv_rejects_table.cpp @@ -44,16 +44,18 @@ void CSVRejectsTable::InitializeTable(ClientContext &context, const ReadCSVData // 1. File Path info->columns.AddColumn(ColumnDefinition("file", LogicalType::VARCHAR)); // 2. Row Line - info->columns.AddColumn(ColumnDefinition("line", LogicalType::BIGINT)); - // 3. Column Index (If Applicable) - info->columns.AddColumn(ColumnDefinition("column_idx", LogicalType::BIGINT)); - // 4. Column Name (If Applicable) + info->columns.AddColumn(ColumnDefinition("line", LogicalType::UBIGINT)); + // 3. Byte Position where error occurred + info->columns.AddColumn(ColumnDefinition("byte_position", LogicalType::UBIGINT)); + // 4. Column Index (If Applicable) + info->columns.AddColumn(ColumnDefinition("column_idx", LogicalType::UBIGINT)); + // 5. Column Name (If Applicable) info->columns.AddColumn(ColumnDefinition("column_name", LogicalType::VARCHAR)); - // 5. Error Type + // 6. Error Type info->columns.AddColumn(ColumnDefinition("error_type", enum_type)); - // 6. Original CSV Line + // 7. Original CSV Line info->columns.AddColumn(ColumnDefinition("csv_line", LogicalType::VARCHAR)); - // 7. Full Error Message + // 8. Full Error Message info->columns.AddColumn(ColumnDefinition("error_message", LogicalType::VARCHAR)); catalog.CreateTable(context, std::move(info)); count = 0; diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_buffer.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_buffer.hpp index 72665ae2de54..e71b75e19553 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_buffer.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_buffer.hpp @@ -18,16 +18,17 @@ namespace duckdb { class CSVBufferHandle { public: - CSVBufferHandle(BufferHandle handle_p, idx_t actual_size_p, const bool is_final_buffer_p, idx_t file_idx_p, - idx_t buffer_index_p) - : handle(std::move(handle_p)), actual_size(actual_size_p), is_last_buffer(is_final_buffer_p), - file_idx(file_idx_p), buffer_idx(buffer_index_p) {}; - CSVBufferHandle() : actual_size(0), is_last_buffer(false), file_idx(0), buffer_idx(0) {}; + CSVBufferHandle(BufferHandle handle_p, idx_t actual_size_p, idx_t requested_size_p, const bool is_final_buffer_p, + idx_t file_idx_p, idx_t buffer_index_p) + : handle(std::move(handle_p)), actual_size(actual_size_p), requested_size(requested_size_p), + is_last_buffer(is_final_buffer_p), file_idx(file_idx_p), buffer_idx(buffer_index_p) {}; + CSVBufferHandle() : actual_size(0), requested_size(0), is_last_buffer(false), file_idx(0), buffer_idx(0) {}; ~CSVBufferHandle() { } //! Handle created during allocation BufferHandle handle; const idx_t actual_size; + const idx_t requested_size; const bool is_last_buffer; const idx_t file_idx; const idx_t buffer_idx; @@ -86,6 +87,7 @@ class CSVBuffer { ClientContext &context; //! Actual size can be smaller than the buffer size in case we allocate it too optimistically. idx_t actual_buffer_size; + idx_t requested_size; //! Global position from the CSV File where this buffer starts idx_t global_csv_start = 0; //! Number of the file that is in this buffer diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp index c40045b74bc5..44bd4f25913a 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp @@ -50,13 +50,14 @@ enum CSVErrorType : uint8_t { class CSVError { public: CSVError() {}; - CSVError(string error_message, CSVErrorType type, idx_t column_idx, string csv_row, LinesPerBoundary error_info); + CSVError(string error_message, CSVErrorType type, idx_t column_idx, string csv_row, LinesPerBoundary error_info, + idx_t byte_position); CSVError(string error_message, CSVErrorType type, LinesPerBoundary error_info); //! Produces error messages for column name -> type mismatch. static CSVError ColumnTypesError(case_insensitive_map_t sql_types_per_column, const vector &names); //! Produces error messages for casting errors static CSVError CastError(const CSVReaderOptions &options, string &column_name, string &cast_error, - idx_t column_idx, string &csv_row, LinesPerBoundary error_info); + idx_t column_idx, string &csv_row, LinesPerBoundary error_info, idx_t byte_position); //! Produces error for when the line size exceeds the maximum line size option static CSVError LineSizeError(const CSVReaderOptions &options, idx_t actual_size, LinesPerBoundary error_info); //! Produces error for when the sniffer couldn't find viable options @@ -84,6 +85,8 @@ class CSVError { string csv_row; //! Line information regarding this error LinesPerBoundary error_info; + //! Global Byte Position where error occurred. + idx_t byte_position; }; class CSVErrorHandler { diff --git a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp index 4750da6b65db..8087888393a4 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp @@ -42,6 +42,9 @@ class LinePosition { } return other.buffer_size - other.buffer_pos + buffer_pos; } + idx_t GetGlobalPosition(idx_t requested_buffer_size) { + return requested_buffer_size * buffer_idx + buffer_pos + 1; + } idx_t buffer_pos = 0; idx_t buffer_size = 0; idx_t buffer_idx = 0; From 0697187a866087fda0a80323af940ff95a337466 Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Wed, 28 Feb 2024 16:35:03 +0100 Subject: [PATCH 018/603] or conjunction filters can be pushed down, while and conjunction filters cannot --- src/optimizer/pushdown/pushdown_window.cpp | 3 +- .../pushdown_window_partition_filter.test | 71 ++++++++++++++----- 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/src/optimizer/pushdown/pushdown_window.cpp b/src/optimizer/pushdown/pushdown_window.cpp index a9ba391890b1..f3086e71a975 100644 --- a/src/optimizer/pushdown/pushdown_window.cpp +++ b/src/optimizer/pushdown/pushdown_window.cpp @@ -19,8 +19,7 @@ static bool FilterIsOnPartition(column_binding_set_t partition_bindings, Express } break; } - case ExpressionType::CONJUNCTION_AND: - case ExpressionType::CONJUNCTION_OR: { + case ExpressionType::CONJUNCTION_AND: { filter_is_on_partition = false; return; } diff --git a/test/optimizer/pushdown/pushdown_window_partition_filter.test b/test/optimizer/pushdown/pushdown_window_partition_filter.test index fe7f7e4acb9e..a96dae18729b 100644 --- a/test/optimizer/pushdown/pushdown_window_partition_filter.test +++ b/test/optimizer/pushdown/pushdown_window_partition_filter.test @@ -37,36 +37,73 @@ create table partition_and_rank_me as from values t(a, b, c, d); query IIII -select a, sum(b) OVER (PARTITION BY c), rank() OVER (PARTITION BY d ORDER BY b), c from partition_and_rank_me order by all +select a, c, sum(b) OVER (PARTITION BY c), rank() OVER (PARTITION BY d ORDER BY b) +from partition_and_rank_me order by all ---- -A 30 1 A -A 30 2 A -A 70 3 B -D 70 4 B -D 110 5 C -D 110 6 C +A A 30 1 +A A 30 2 +A B 70 3 +D B 70 4 +D C 110 5 +D C 110 6 # can't push down the filter c!='B', since the values of the rank() window function # are affected by the existence of the rows where c='B' query IIII -select * from (select a, sum(b) OVER (PARTITION BY c), rank() OVER (PARTITION BY d ORDER BY b), c from partition_and_rank_me) where c != 'B' order by all; +select * from ( + select a, c, sum(b) OVER (PARTITION BY c), rank() OVER (PARTITION BY d ORDER BY b) + from partition_and_rank_me +) where c != 'B' order by all; ---- -A 30 1 A -A 30 2 A -D 110 5 C -D 110 6 C +A A 30 1 +A A 30 2 +D C 110 5 +D C 110 6 -# The filter is on the partitioned column, but is parto of a conjunction, so we dont push it down +# The filter is on the partitioned column, but is part of an AND conjunction, so we dont push it down query IIII -select * from (select a, sum(b) OVER (PARTITION BY c), rank() OVER (PARTITION BY c ORDER BY b), c from partition_and_rank_me) where (c = 'B' AND a = 'D') order by all; +select * from ( + select a, c, sum(b) OVER (PARTITION BY c), rank() OVER (PARTITION BY c ORDER BY b) + from partition_and_rank_me +) where (c = 'B' AND a = 'D') order by all; ---- -D 70 2 B +D B 70 2 # result of above query with pushdown query IIII -select * from (select a, sum(b) OVER (PARTITION BY c), rank() OVER (PARTITION BY c ORDER BY b), c from partition_and_rank_me where (c = 'B' AND a = 'D')) order by all; +select * from ( + select a, c, sum(b) OVER (PARTITION BY c), rank() OVER (PARTITION BY c ORDER BY b) + from partition_and_rank_me where (c = 'B' AND a = 'D') +) order by all; ---- -D 40 1 B +D B 40 1 + + +# The filter is on the partitioned column, but is part of an OR conjunction, so we can push it down +query IIII +select * from ( + select a, c, sum(b) OVER (PARTITION BY c), rank() OVER (PARTITION BY c ORDER BY b) + from partition_and_rank_me +) where (c = 'B' OR a = 'D') order by all; +---- +A B 70 1 +D B 70 2 +D C 110 1 +D C 110 2 + +# result of above query with pushdown +query IIII +select * from ( + select a, c, sum(b) OVER (PARTITION BY c), rank() OVER (PARTITION BY c ORDER BY b) + from partition_and_rank_me + where (c = 'B' OR a = 'D') +) order by all; +---- +A B 70 1 +D B 70 2 +D C 110 1 +D C 110 2 + # can't push down the filter c!='B', since the values of the rank() window function # are affected by the existence of the rows where c='B' From 5951413e0cd88990714a409854717755e6fcc158 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 28 Feb 2024 13:51:26 -0300 Subject: [PATCH 019/603] Fixing up old tests and fixing small bugs --- .../scanner/string_value_scanner.cpp | 33 +-- .../csv_scanner/string_value_scanner.hpp | 3 + .../copy/csv/rejects/csv_rejects_read.test | 232 +++++++++++------- .../csv/rejects/test_invalid_parameters.test | 56 +++++ 4 files changed, 222 insertions(+), 102 deletions(-) create mode 100644 test/sql/copy/csv/rejects/test_invalid_parameters.test diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 10bd3ce3029d..daa56dd209e0 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -26,6 +26,7 @@ StringValueResult::StringValueResult(CSVStates &states, CSVStateMachine &state_m buffer_ptr = buffer_handle->Ptr(); buffer_size = buffer_handle->actual_size; last_position = buffer_position; + requested_size = buffer_handle->requested_size; // Current Result information previous_line_start = {iterator.pos.buffer_idx, iterator.pos.buffer_pos, buffer_handle->actual_size}; @@ -354,9 +355,9 @@ bool StringValueResult::AddRowInternal() { auto error_string = error.str(); LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read - 1); auto borked_line = ReconstructCurrentLine(); - auto csv_error = CSVError::CastError( - state_machine.options, names[cast_error.first], error_string, cast_error.first, borked_line, - lines_per_batch, pre_previous_line_start.GetGlobalPosition(buffer_handles.front()->requested_size)); + auto csv_error = CSVError::CastError(state_machine.options, names[cast_error.first], error_string, + cast_error.first, borked_line, lines_per_batch, + pre_previous_line_start.GetGlobalPosition(requested_size)); error_handler.Error(csv_error); } // If we got here it means we are ignoring errors, hence we need to signify to our result scanner to ignore this @@ -414,20 +415,20 @@ bool StringValueResult::AddRowInternal() { } bool StringValueResult::AddRow(StringValueResult &result, const idx_t buffer_pos) { + LinePosition current_line_start = {result.iterator.pos.buffer_idx, result.iterator.pos.buffer_pos, + result.buffer_size}; + idx_t current_line_size = current_line_start - result.previous_line_start; + if (result.store_line_size) { + result.error_handler.NewMaxLineSize(current_line_size); + } + if (current_line_size > result.state_machine.options.maximum_line_size) { + LinesPerBoundary lines_per_batch(result.iterator.GetBoundaryIdx(), result.number_of_rows); + auto csv_error = CSVError::LineSizeError(result.state_machine.options, current_line_size, lines_per_batch); + result.error_handler.Error(csv_error); + } + result.pre_previous_line_start = result.previous_line_start; + result.previous_line_start = current_line_start; if (result.last_position <= buffer_pos) { - LinePosition current_line_start = {result.iterator.pos.buffer_idx, result.iterator.pos.buffer_pos, - result.buffer_size}; - idx_t current_line_size = current_line_start - result.previous_line_start; - if (result.store_line_size) { - result.error_handler.NewMaxLineSize(current_line_size); - } - if (current_line_size > result.state_machine.options.maximum_line_size) { - LinesPerBoundary lines_per_batch(result.iterator.GetBoundaryIdx(), result.number_of_rows); - auto csv_error = CSVError::LineSizeError(result.state_machine.options, current_line_size, lines_per_batch); - result.error_handler.Error(csv_error); - } - result.pre_previous_line_start = result.previous_line_start; - result.previous_line_start = current_line_start; // We add the value if (result.quoted) { StringValueResult::AddQuotedValue(result, buffer_pos); diff --git a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp index 8087888393a4..00e0bb80b2c2 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp @@ -104,6 +104,9 @@ class StringValueResult : public ScannerResult { //! We must ensure that we keep the buffers alive until processing the query result vector> buffer_handles; + //! Requested size of buffers (i.e., either 32Mb or set by buffer_size parameter) + idx_t requested_size; + //! If the current row has an error, we have to skip it bool ignore_current_row = false; //! Specialized code for quoted values, makes sure to remove quotes and escapes diff --git a/test/sql/copy/csv/rejects/csv_rejects_read.test b/test/sql/copy/csv/rejects/csv_rejects_read.test index 16b73ae7ac65..5713e91760a0 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_read.test +++ b/test/sql/copy/csv/rejects/csv_rejects_read.test @@ -6,54 +6,6 @@ require skip_reload # FIXME: https://github.com/duckdb/duckdb/issues/7755 require vector_size 2048 -# Test invalid arguments -statement error -SELECT * FROM read_csv( - 'test/sql/copy/csv/data/error/mismatch/bad.csv', - columns = {'col0': 'INTEGER', 'col1': 'INTEGER', 'col2': 'VARCHAR'}, - ignore_errors=false, - rejects_table='csv_rejects_table' -) ----- -only supported when IGNORE_ERRORS is set to true - -statement error -SELECT * FROM read_csv( - 'test/sql/copy/csv/data/error/mismatch/bad.csv', - columns = {'col0': 'INTEGER', 'col1': 'INTEGER', 'col2': 'VARCHAR'}, - ignore_errors=true, - rejects_table='') ----- -REJECTS_TABLE option cannot be empty - -statement error -SELECT * FROM read_csv( - 'test/sql/copy/csv/data/error/mismatch/bad.csv', - columns = {'col0': 'INTEGER', 'col1': 'INTEGER', 'col2': 'VARCHAR'}, - ignore_errors=true, - rejects_table='csv_rejects_table', - union_by_name=true) ----- -UNION_BY_NAME is set to true - -statement error -SELECT * FROM read_csv( - 'test/sql/copy/csv/data/error/mismatch/bad.csv', - columns = {'col0': 'INTEGER', 'col1': 'INTEGER', 'col2': 'VARCHAR'}, - ignore_errors=true, - rejects_limit=10) ----- -REJECTS_LIMIT option is only supported when REJECTS_TABLE is set to a table name - -statement error -SELECT * FROM read_csv( - 'test/sql/copy/csv/data/error/mismatch/bad.csv', - columns = {'col0': 'INTEGER', 'col1': 'INTEGER', 'col2': 'VARCHAR'}, - ignore_errors=true, - rejects_table='csv_rejects_table', - rejects_limit=-1) ----- -REJECTS_LIMIT: cannot be negative # Basic test query III rowsort @@ -66,11 +18,17 @@ SELECT * FROM read_csv( 1 2 AAA 6 7 CCC -query IIIIII rowsort -SELECT "line", "column", "column_name", "parsed_value", "error", regexp_replace("file", '\\', '/', 'g') +query IIIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table; ---- -2 1 "col1" BBB Could not convert string 'BBB' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/bad.csv +test/sql/copy/csv/data/error/mismatch/bad.csv 2 1 "col1" CAST 4,BBB,9, 9 + +query I +SELECT error_message +FROM csv_rejects_table; +---- +:.*Could not convert string "BBB" to 'INTEGER'.* statement ok DROP TABLE csv_rejects_table; @@ -85,13 +43,31 @@ SELECT * FROM read_csv( ---- 4 5 9 -query IIIIII rowsort -SELECT "line", "column", "column_name", "parsed_value", "error", regexp_replace("file", '\\', '/', 'g') +query IIIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table; ---- -1 2 "col2" DDD Could not convert string 'DDD' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/bad2.csv -3 0 "col0" EEE Could not convert string 'EEE' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/bad2.csv -3 2 "col2" FFF Could not convert string 'FFF' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/bad2.csv +test/sql/copy/csv/data/error/mismatch/bad2.csv 1 2 "col2" CAST ,2,DDD, 1 +test/sql/copy/csv/data/error/mismatch/bad2.csv 3 0 "col0" CAST EEE,7,FFF, 16 +test/sql/copy/csv/data/error/mismatch/bad2.csv 3 2 "col2" CAST EEE,7,FFF, 16 + +query I +SELECT error_message +FROM csv_rejects_table where line=1 and column_idx=2; +---- +:.*Could not convert string "DDD" to 'INTEGER'.* + +query I +SELECT error_message +FROM csv_rejects_table where line=3 and column_idx=0; +---- +:.*Could not convert string "EEE" to 'INTEGER'.* + +query I +SELECT error_message +FROM csv_rejects_table where line=3 and column_idx=2; +---- +:.*Could not convert string "FFF" to 'INTEGER'.* statement ok DROP TABLE csv_rejects_table; @@ -110,12 +86,24 @@ SELECT * FROM read_csv( 6 7 CCC -query IIIIII rowsort -SELECT "line", "column", "column_name", "parsed_value", "error", regexp_replace("file", '\\', '/', 'g') +query IIIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table; ---- -2 1 "col1" BBB Could not convert string 'BBB' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/bad.csv -3 0 "col0" EEE Could not convert string 'EEE' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/bad2.csv +test/sql/copy/csv/data/error/mismatch/bad.csv 2 1 "col1" CAST 4,BBB,9, 9 +test/sql/copy/csv/data/error/mismatch/bad2.csv 3 0 "col0" CAST EEE,7,FFF, 16 + +query I +SELECT error_message +FROM csv_rejects_table where line=2 and column_idx=1; +---- +:.*Could not convert string "BBB" to 'INTEGER'.* + +query I +SELECT error_message +FROM csv_rejects_table where line=3 and column_idx=0; +---- +:.*Could not convert string "EEE" to 'INTEGER'.* statement ok DROP TABLE csv_rejects_table; @@ -154,12 +142,24 @@ SELECT SUM(num) FROM read_csv( ---- 4270 -query IIIIII rowsort -SELECT "line", "column", "column_name", "parsed_value", "error", regexp_replace("file", '\\', '/', 'g') +query IIIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table; ---- -2176 0 "num" B Could not convert string 'B' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/big_bad.csv -4176 0 "num" C Could not convert string 'C' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/big_bad.csv +test/sql/copy/csv/data/error/mismatch/big_bad.csv 2176 0 "num" CAST B, A 10875 +test/sql/copy/csv/data/error/mismatch/big_bad.csv 4176 0 "num" CAST C, A 20875 + +query I +SELECT error_message +FROM csv_rejects_table where line=2176 and column_idx=0; +---- +:.*Could not convert string "B" to 'INTEGER'.* + +query I +SELECT error_message +FROM csv_rejects_table where line=4176 and column_idx=0; +---- +:.*Could not convert string "C" to 'INTEGER'.* statement ok DROP TABLE csv_rejects_table; @@ -173,12 +173,24 @@ SELECT SUM(num) FROM read_csv( ---- 6774 -query IIIIII rowsort -SELECT "line", "column", "column_name", "parsed_value", "error", regexp_replace("file", '\\', '/', 'g') +query IIIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table; ---- -3680 0 "num" B Could not convert string 'B' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/big_bad2.csv -5680 0 "num" C Could not convert string 'C' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/big_bad2.csv +test/sql/copy/csv/data/error/mismatch/big_bad2.csv 3680 0 "num" CAST B, A 18395 +test/sql/copy/csv/data/error/mismatch/big_bad2.csv 5680 0 "num" CAST C, A 28395 + +query I +SELECT error_message +FROM csv_rejects_table where line=3680 and column_idx=0; +---- +:.*Could not convert string "B" to 'INTEGER'.* + +query I +SELECT error_message +FROM csv_rejects_table where line=5680 and column_idx=0; +---- +:.*Could not convert string "C" to 'INTEGER'.* statement ok DROP TABLE csv_rejects_table; @@ -193,14 +205,38 @@ SELECT SUM(num) FROM read_csv( ---- 11044 -query IIIIII rowsort -SELECT "line", "column", "column_name", "parsed_value", "error", regexp_replace("file", '\\', '/', 'g') +query IIIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table; ---- -2176 0 "num" B Could not convert string 'B' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/big_bad.csv -3680 0 "num" B Could not convert string 'B' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/big_bad2.csv -4176 0 "num" C Could not convert string 'C' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/big_bad.csv -5680 0 "num" C Could not convert string 'C' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/big_bad2.csv +test/sql/copy/csv/data/error/mismatch/big_bad.csv 2176 0 "num" CAST B, A 10875 +test/sql/copy/csv/data/error/mismatch/big_bad.csv 4176 0 "num" CAST C, A 20875 +test/sql/copy/csv/data/error/mismatch/big_bad2.csv 3680 0 "num" CAST B, A 18395 +test/sql/copy/csv/data/error/mismatch/big_bad2.csv 5680 0 "num" CAST C, A 28395 + +query I +SELECT error_message +FROM csv_rejects_table where line=3680 and column_idx=0; +---- +:.*Could not convert string "B" to 'INTEGER'.* + +query I +SELECT error_message +FROM csv_rejects_table where line=5680 and column_idx=0; +---- +:.*Could not convert string "C" to 'INTEGER'.* + +query I +SELECT error_message +FROM csv_rejects_table where line=2176 and column_idx=0; +---- +:.*Could not convert string "B" to 'INTEGER'.* + +query I +SELECT error_message +FROM csv_rejects_table where line=4176 and column_idx=0; +---- +:.*Could not convert string "C" to 'INTEGER'.* statement ok DROP TABLE csv_rejects_table; @@ -223,19 +259,43 @@ ON L.num = R.num; 1 A 1 A 3 C 3 C -query IIIIII rowsort -SELECT "line", "column", "column_name", "parsed_value", "error", regexp_replace("file", '\\', '/', 'g') +query IIIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table_left; ---- -3 0 "num" X Could not convert string 'X' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/small1.csv -6 0 "num" X Could not convert string 'X' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/small1.csv +test/sql/copy/csv/data/error/mismatch/small1.csv 3 0 "num" CAST X,Y 14 +test/sql/copy/csv/data/error/mismatch/small1.csv 6 0 "num" CAST X,Y 26 -query IIIIII rowsort -SELECT "line", "column", "column_name", "parsed_value", "error", regexp_replace("file", '\\', '/', 'g') +query IIIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table_right; ---- -3 0 "num" X Could not convert string 'X' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/small2.csv -5 0 "num" X Could not convert string 'X' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/small2.csv +test/sql/copy/csv/data/error/mismatch/small2.csv 3 0 "num" CAST X,Y 14 +test/sql/copy/csv/data/error/mismatch/small2.csv 5 0 "num" CAST X,Y 22 + +query I +SELECT error_message +FROM csv_rejects_table_left where line=3 and column_idx=0; +---- +:.*Could not convert string "X" to 'INTEGER'.* + +query I +SELECT error_message +FROM csv_rejects_table_left where line=6 and column_idx=0; +---- +:.*Could not convert string "X" to 'INTEGER'.* + +query I +SELECT error_message +FROM csv_rejects_table_right where line=3 and column_idx=0; +---- +:.*Could not convert string "X" to 'INTEGER'.* + +query I +SELECT error_message +FROM csv_rejects_table_right where line=5 and column_idx=0; +---- +:.*Could not convert string "X" to 'INTEGER'.* statement ok DROP TABLE csv_rejects_table_left; @@ -264,12 +324,12 @@ ON L.num = R.num; 3 C 3 C -query IIIIII rowsort -SELECT "line", "column", "column_name", "parsed_value", "error", regexp_replace("file", '\\', '/', 'g') +query IIIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table_left; ---- -3 0 "num" X Could not convert string 'X' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/small1.csv -6 0 "num" X Could not convert string 'X' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/small1.csv +test/sql/copy/csv/data/error/mismatch/small1.csv 3 0 "num" CAST X,Y 14 +test/sql/copy/csv/data/error/mismatch/small1.csv 6 0 "num" CAST X,Y 26 query I SELECT COUNT(*) diff --git a/test/sql/copy/csv/rejects/test_invalid_parameters.test b/test/sql/copy/csv/rejects/test_invalid_parameters.test new file mode 100644 index 000000000000..d403d0274948 --- /dev/null +++ b/test/sql/copy/csv/rejects/test_invalid_parameters.test @@ -0,0 +1,56 @@ +# name: test/sql/copy/csv/rejects/test_invalid_parameters.test +# group: [rejects] + +require skip_reload + +# FIXME: https://github.com/duckdb/duckdb/issues/7755 +require vector_size 2048 + +# Test invalid arguments +statement error +SELECT * FROM read_csv( + 'test/sql/copy/csv/data/error/mismatch/bad.csv', + columns = {'col0': 'INTEGER', 'col1': 'INTEGER', 'col2': 'VARCHAR'}, + ignore_errors=false, + rejects_table='csv_rejects_table' +) +---- +only supported when IGNORE_ERRORS is set to true + +statement error +SELECT * FROM read_csv( + 'test/sql/copy/csv/data/error/mismatch/bad.csv', + columns = {'col0': 'INTEGER', 'col1': 'INTEGER', 'col2': 'VARCHAR'}, + ignore_errors=true, + rejects_table='') +---- +REJECTS_TABLE option cannot be empty + +statement error +SELECT * FROM read_csv( + 'test/sql/copy/csv/data/error/mismatch/bad.csv', + columns = {'col0': 'INTEGER', 'col1': 'INTEGER', 'col2': 'VARCHAR'}, + ignore_errors=true, + rejects_table='csv_rejects_table', + union_by_name=true) +---- +UNION_BY_NAME is set to true + +statement error +SELECT * FROM read_csv( + 'test/sql/copy/csv/data/error/mismatch/bad.csv', + columns = {'col0': 'INTEGER', 'col1': 'INTEGER', 'col2': 'VARCHAR'}, + ignore_errors=true, + rejects_limit=10) +---- +REJECTS_LIMIT option is only supported when REJECTS_TABLE is set to a table name + +statement error +SELECT * FROM read_csv( + 'test/sql/copy/csv/data/error/mismatch/bad.csv', + columns = {'col0': 'INTEGER', 'col1': 'INTEGER', 'col2': 'VARCHAR'}, + ignore_errors=true, + rejects_table='csv_rejects_table', + rejects_limit=-1) +---- +REJECTS_LIMIT: cannot be negative From a8f2dcd01d18af6b0680fd90c81b7d3b427a79ba Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 28 Feb 2024 14:15:41 -0300 Subject: [PATCH 020/603] remove old parameter, cleanup of older tests --- .../copy/csv/rejects/csv_rejects_auto.test | 146 +++--------------- .../copy/csv/rejects/csv_rejects_read.test | 1 - .../csv/rejects/csv_rejects_recovery.test | 97 ------------ .../csv/rejects/test_invalid_parameters.test | 57 +++++++ 4 files changed, 78 insertions(+), 223 deletions(-) delete mode 100644 test/sql/copy/csv/rejects/csv_rejects_recovery.test diff --git a/test/sql/copy/csv/rejects/csv_rejects_auto.test b/test/sql/copy/csv/rejects/csv_rejects_auto.test index 5dc4358708de..887bd282db73 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_auto.test +++ b/test/sql/copy/csv/rejects/csv_rejects_auto.test @@ -6,63 +6,6 @@ require skip_reload # FIXME: https://github.com/duckdb/duckdb/issues/7755 require vector_size 2048 -# Test invalid arguments -statement error -SELECT * FROM read_csv_auto( - 'test/sql/copy/csv/data/error/mismatch/bad.csv', - ignore_errors=false, - rejects_table='csv_rejects_table' -) ----- -only supported when IGNORE_ERRORS is set to true - -statement error -SELECT * FROM read_csv_auto( - 'test/sql/copy/csv/data/error/mismatch/bad.csv', - ignore_errors=true, - rejects_table='') ----- -REJECTS_TABLE option cannot be empty - -statement error -SELECT * FROM read_csv_auto( - 'test/sql/copy/csv/data/error/mismatch/bad.csv', - ignore_errors=true, - rejects_table='csv_rejects_table', - union_by_name=true) ----- -UNION_BY_NAME is set to true - -statement error -SELECT * FROM read_csv_auto( - 'test/sql/copy/csv/data/error/mismatch/bad.csv', - ignore_errors=true, - rejects_limit=10) ----- -REJECTS_LIMIT option is only supported when REJECTS_TABLE is set to a table name - -statement error -SELECT * FROM read_csv_auto( - 'test/sql/copy/csv/data/error/mismatch/bad.csv', - ignore_errors=true, - rejects_table='csv_rejects_table', - rejects_limit=-1) ----- -REJECTS_LIMIT: cannot be negative - - -query III -SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*) FROM read_csv_auto( - 'test/sql/copy/csv/data/error/mismatch/big_bad*.csv', - sample_size=3000, - rejects_table='csv_rejects_table', - ignore_errors=true, header = 0); ----- -VARCHAR VARCHAR 11048 - -statement ok -DROP TABLE csv_rejects_table; - # Ensure that we can get the schema if we reduce the sample size and ignore errors query IIIII SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto( @@ -73,85 +16,38 @@ SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), M ---- BIGINT VARCHAR 11044 11044 2 -query IIIIII rowsort -SELECT regexp_replace("file", '\\', '/', 'g') , "line", "column", "column_name", "parsed_value", "error" +query IIIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table; ---- -test/sql/copy/csv/data/error/mismatch/big_bad.csv 2176 0 "column0" B Could not convert string 'B' to 'BIGINT' -test/sql/copy/csv/data/error/mismatch/big_bad.csv 4176 0 "column0" C Could not convert string 'C' to 'BIGINT' -test/sql/copy/csv/data/error/mismatch/big_bad2.csv 3680 0 "column0" B Could not convert string 'B' to 'BIGINT' -test/sql/copy/csv/data/error/mismatch/big_bad2.csv 5680 0 "column0" C Could not convert string 'C' to 'BIGINT' +test/sql/copy/csv/data/error/mismatch/big_bad.csv 2176 0 "column0" CAST B, A 10875 +test/sql/copy/csv/data/error/mismatch/big_bad.csv 4176 0 "column0" CAST C, A 20875 +test/sql/copy/csv/data/error/mismatch/big_bad2.csv 3680 0 "column0" CAST B, A 18395 +test/sql/copy/csv/data/error/mismatch/big_bad2.csv 5680 0 "column0" CAST C, A 28395 -statement ok -DROP TABLE csv_rejects_table; - -# Test with recovery columns query I -SELECT SUM(COL1) FROM read_csv_auto( - 'test/sql/copy/csv/data/error/mismatch/part1.csv', - header=true, - ignore_errors=true, - sample_size=1, - rejects_table='csv_rejects_table', - rejects_recovery_columns=['COL2']); +SELECT error_message +FROM csv_rejects_table where line=2176 and column_idx=0; ---- -5230 - -statement ok -DROP TABLE csv_rejects_table; +:.*Could not convert string "B" to 'BIGINT'.* -# Test with recovery columns query I -SELECT SUM(COL1) FROM read_csv_auto( - 'test/sql/copy/csv/data/error/mismatch/part2.csv', - header=true, - ignore_errors=true, - sample_size=1, - rejects_table='csv_rejects_table', - rejects_recovery_columns=['COL2']); +SELECT error_message +FROM csv_rejects_table where line=4176 and column_idx=0; ---- -5418 - -statement ok -DROP TABLE csv_rejects_table; +:.*Could not convert string "C" to 'BIGINT'.* -# Test with recovery columns query I -SELECT SUM(COL1) FROM read_csv_auto( - 'test/sql/copy/csv/data/error/mismatch/part3.csv', - header=true, - ignore_errors=true, - sample_size=1, - rejects_table='csv_rejects_table', - rejects_recovery_columns=['COL2']); +SELECT error_message +FROM csv_rejects_table where line=3680 and column_idx=0; ---- -4151 - -statement ok -DROP TABLE csv_rejects_table; +:.*Could not convert string "B" to 'BIGINT'.* -# Test with recovery columns query I -SELECT SUM(COL1) FROM read_csv_auto( - 'test/sql/copy/csv/data/error/mismatch/part*.csv', - header=true, - ignore_errors=true, - sample_size=1, - rejects_table='csv_rejects_table', - rejects_recovery_columns=['COL2']); ----- -14799 - -query IIIIIII rowsort -SELECT "line", "column", "column_name", "parsed_value", "recovery_columns", "error", regexp_replace("file", '\\', '/', 'g') -FROM csv_rejects_table; +SELECT error_message +FROM csv_rejects_table where line=5680 and column_idx=0; ---- -2058 0 "COL1" B {'COL2': BAD1B} Could not convert string 'B' to 'BIGINT' test/sql/copy/csv/data/error/mismatch/part3.csv -2325 0 "COL1" B {'COL2': BAD2B} Could not convert string 'B' to 'BIGINT' test/sql/copy/csv/data/error/mismatch/part2.csv -3137 0 "COL1" B {'COL2': BAD1B} Could not convert string 'B' to 'BIGINT' test/sql/copy/csv/data/error/mismatch/part1.csv -4058 0 "COL1" C {'COL2': BAD1C} Could not convert string 'C' to 'BIGINT' test/sql/copy/csv/data/error/mismatch/part3.csv -4325 0 "COL1" C {'COL2': BAD2C} Could not convert string 'C' to 'BIGINT' test/sql/copy/csv/data/error/mismatch/part2.csv -5137 0 "COL1" C {'COL2': BAD1C} Could not convert string 'C' to 'BIGINT' test/sql/copy/csv/data/error/mismatch/part1.csv +:.*Could not convert string "C" to 'BIGINT'.* statement ok DROP TABLE csv_rejects_table; @@ -180,7 +76,7 @@ statement ok CREATE TABLE tbl1 (col1 BIGINT, col2 VARCHAR); statement ok -COPY tbl1 FROM 'test/sql/copy/csv/data/error/mismatch/half1.csv' +COPY tbl1 FROM 'test/sql/copy/csv/data/error/mismatch/half1.csv' WITH (HEADER, IGNORE_ERRORS TRUE, SAMPLE_SIZE 1000, REJECTS_TABLE 'csv_rejects_table'); query I @@ -222,7 +118,7 @@ statement ok CREATE TABLE tbl1 (col1 BIGINT, col2 VARCHAR); statement ok -COPY tbl1 FROM 'test/sql/copy/csv/data/error/mismatch/half2.csv' +COPY tbl1 FROM 'test/sql/copy/csv/data/error/mismatch/half2.csv' WITH (HEADER, IGNORE_ERRORS TRUE, SAMPLE_SIZE 1000, REJECTS_TABLE 'csv_rejects_table'); query I @@ -246,7 +142,7 @@ statement ok CREATE TABLE tbl1 (col1 BIGINT, col2 VARCHAR); statement ok -COPY tbl1 FROM 'test/sql/copy/csv/data/error/mismatch/half2.csv' +COPY tbl1 FROM 'test/sql/copy/csv/data/error/mismatch/half2.csv' WITH (HEADER, IGNORE_ERRORS TRUE, SAMPLE_SIZE 1000, REJECTS_TABLE 'csv_rejects_table', REJECTS_LIMIT 1337); query I diff --git a/test/sql/copy/csv/rejects/csv_rejects_read.test b/test/sql/copy/csv/rejects/csv_rejects_read.test index 5713e91760a0..458e485ffc75 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_read.test +++ b/test/sql/copy/csv/rejects/csv_rejects_read.test @@ -7,7 +7,6 @@ require skip_reload require vector_size 2048 -# Basic test query III rowsort SELECT * FROM read_csv( 'test/sql/copy/csv/data/error/mismatch/bad.csv', diff --git a/test/sql/copy/csv/rejects/csv_rejects_recovery.test b/test/sql/copy/csv/rejects/csv_rejects_recovery.test deleted file mode 100644 index 697e7c94a091..000000000000 --- a/test/sql/copy/csv/rejects/csv_rejects_recovery.test +++ /dev/null @@ -1,97 +0,0 @@ -# name: test/sql/copy/csv/rejects/csv_rejects_recovery.test -# group: [rejects] - -require skip_reload - -# Test invalid arguments - -# Should not work without rejects_table -statement error -SELECT SUM(COL1) + SUM(COL3) FROM read_csv( - 'test/sql/copy/csv/data/error/mismatch/part*.csv', - ignore_errors=true, - header=true, - columns = {COL3 :'INTEGER', COL1: 'INTEGER', COL2: 'VARCHAR'}, - rejects_recovery_columns=['COL2'] -); ----- -only supported when REJECTS_TABLE is set to a table name - -# Should not work without rejects_recovery_columns as list -statement error -SELECT SUM(COL1) + SUM(COL3) FROM read_csv( - 'test/sql/copy/csv/data/error/mismatch/part*.csv', - header=true, - columns = {COL3 :'INTEGER', COL1: 'INTEGER', COL2: 'VARCHAR'}, - rejects_table='csv_rejects_table', - rejects_recovery_columns=['NON_EXISTING_COLUMN'], - ignore_errors=true -); ----- -CSV options could not be auto-detected. Consider setting parser options manually. - -# Should not work without rejects_recovery_columns as list -statement error -SELECT SUM(COL1) + SUM(COL3) FROM read_csv( - 'test/sql/copy/csv/data/error/mismatch/part*.csv', - header=true, - columns = {COL3 :'INTEGER', COL1: 'INTEGER', COL2: 'VARCHAR'}, - rejects_table='csv_rejects_table', - rejects_recovery_columns=['NON_EXISTING_COLUMN'], - ignore_errors=true, - auto_detect=false -); ----- -REJECTS_RECOVERY_COLUMNS: column "NON_EXISTING_COLUMN" not found - -# Basic test -query IIII rowsort -SELECT * FROM read_csv( - 'test/sql/copy/csv/data/error/mismatch/example.tsv', - sep='\t', - columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'id': 'INTEGER', 'count': 'INTEGER'}, - rejects_table='csv_rejects_table', - rejects_recovery_columns=['name', 'age'], - ignore_errors=true, - auto_detect = false -); ----- -alice 10 1 20 -charlie 7 3 30 - -query IIIIIII rowsort -SELECT "line", "column", "column_name", "parsed_value", "recovery_columns", "error", regexp_replace("file", '\\', '/', 'g') -FROM csv_rejects_table; ----- -2 3 "count" NOT_A_NUMBER {'name': bobby, 'age': 12} Could not convert string 'NOT_A_NUMBER' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/example.tsv - -statement ok -DROP TABLE csv_rejects_table; - -# We should not prune columns that are part of the rejects_recovery_columns -query I -SELECT SUM(COL1) FROM read_csv( - 'test/sql/copy/csv/data/error/mismatch/part*.csv', - columns = {COL1: 'INTEGER', COL2: 'VARCHAR'}, - header=true, - rejects_table='csv_rejects_table', - rejects_recovery_columns=['COL2'], - ignore_errors=true, - auto_detect = false -); ----- -14799 - -query IIIIIII rowsort -SELECT "line", "column", "column_name", "parsed_value", "recovery_columns", "error", regexp_replace("file", '\\', '/', 'g') -FROM csv_rejects_table; ----- -2058 0 "COL1" B {'COL2': BAD1B} Could not convert string 'B' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/part3.csv -2325 0 "COL1" B {'COL2': BAD2B} Could not convert string 'B' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/part2.csv -3137 0 "COL1" B {'COL2': BAD1B} Could not convert string 'B' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/part1.csv -4058 0 "COL1" C {'COL2': BAD1C} Could not convert string 'C' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/part3.csv -4325 0 "COL1" C {'COL2': BAD2C} Could not convert string 'C' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/part2.csv -5137 0 "COL1" C {'COL2': BAD1C} Could not convert string 'C' to 'INTEGER' test/sql/copy/csv/data/error/mismatch/part1.csv - -statement ok -DROP TABLE csv_rejects_table; \ No newline at end of file diff --git a/test/sql/copy/csv/rejects/test_invalid_parameters.test b/test/sql/copy/csv/rejects/test_invalid_parameters.test index d403d0274948..5209960fef88 100644 --- a/test/sql/copy/csv/rejects/test_invalid_parameters.test +++ b/test/sql/copy/csv/rejects/test_invalid_parameters.test @@ -54,3 +54,60 @@ SELECT * FROM read_csv( rejects_limit=-1) ---- REJECTS_LIMIT: cannot be negative + +# Test invalid arguments +statement error +SELECT * FROM read_csv_auto( + 'test/sql/copy/csv/data/error/mismatch/bad.csv', + ignore_errors=false, + rejects_table='csv_rejects_table' +) +---- +only supported when IGNORE_ERRORS is set to true + +statement error +SELECT * FROM read_csv_auto( + 'test/sql/copy/csv/data/error/mismatch/bad.csv', + ignore_errors=true, + rejects_table='') +---- +REJECTS_TABLE option cannot be empty + +statement error +SELECT * FROM read_csv_auto( + 'test/sql/copy/csv/data/error/mismatch/bad.csv', + ignore_errors=true, + rejects_table='csv_rejects_table', + union_by_name=true) +---- +UNION_BY_NAME is set to true + +statement error +SELECT * FROM read_csv_auto( + 'test/sql/copy/csv/data/error/mismatch/bad.csv', + ignore_errors=true, + rejects_limit=10) +---- +REJECTS_LIMIT option is only supported when REJECTS_TABLE is set to a table name + +statement error +SELECT * FROM read_csv_auto( + 'test/sql/copy/csv/data/error/mismatch/bad.csv', + ignore_errors=true, + rejects_table='csv_rejects_table', + rejects_limit=-1) +---- +REJECTS_LIMIT: cannot be negative + + +query III +SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*) FROM read_csv_auto( + 'test/sql/copy/csv/data/error/mismatch/big_bad*.csv', + sample_size=3000, + rejects_table='csv_rejects_table', + ignore_errors=true, header = 0); +---- +VARCHAR VARCHAR 11048 + +statement ok +DROP TABLE csv_rejects_table; \ No newline at end of file From d3df9f34a4d518204994f0b8bfadc3e19644dabe Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 29 Feb 2024 11:31:19 +0100 Subject: [PATCH 021/603] add function filters --- src/optimizer/pushdown/pushdown_window.cpp | 12 ++++++++++-- .../pushdown/pushdown_window_partition_filter.test | 10 ++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/optimizer/pushdown/pushdown_window.cpp b/src/optimizer/pushdown/pushdown_window.cpp index f3086e71a975..7b67b74a77a2 100644 --- a/src/optimizer/pushdown/pushdown_window.cpp +++ b/src/optimizer/pushdown/pushdown_window.cpp @@ -1,6 +1,7 @@ #include "duckdb/optimizer/filter_pushdown.hpp" #include "duckdb/parser/expression/window_expression.hpp" #include "duckdb/planner/expression/bound_columnref_expression.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" #include "duckdb/planner/expression/bound_window_expression.hpp" #include "duckdb/planner/expression_iterator.hpp" #include "duckdb/planner/operator/logical_window.hpp" @@ -15,10 +16,17 @@ static bool FilterIsOnPartition(column_binding_set_t partition_bindings, Express auto &col_ref = child->Cast(); if (partition_bindings.find(col_ref.binding) != partition_bindings.end()) { filter_is_on_partition = true; - return; + // if the filter has more columns, break to make sure the other + // columns of the filter are also partitioned on. + break; } - break; + // the filter is not on a column that is partitioned. immediately return. + filter_is_on_partition = false; + return; } + // for simplicity, we don't push down filters that are functions. + // conjunction and filters we don't pushdown either + case ExpressionType::BOUND_FUNCTION: case ExpressionType::CONJUNCTION_AND: { filter_is_on_partition = false; return; diff --git a/test/optimizer/pushdown/pushdown_window_partition_filter.test b/test/optimizer/pushdown/pushdown_window_partition_filter.test index a96dae18729b..8887849ec616 100644 --- a/test/optimizer/pushdown/pushdown_window_partition_filter.test +++ b/test/optimizer/pushdown/pushdown_window_partition_filter.test @@ -104,6 +104,16 @@ D B 70 2 D C 110 1 D C 110 2 +# The filter is a function expression, so we don't push it down +query IIII +select * from ( + select a, c, sum(b) OVER (PARTITION BY c), rank() OVER (PARTITION BY c ORDER BY b) + from partition_and_rank_me +) where (c || 'Z' = 'BZ') order by all; +---- +A B 70 1 +D B 70 2 + # can't push down the filter c!='B', since the values of the rank() window function # are affected by the existence of the rows where c='B' From 77357b0592fded9551d0a76acb8a79314776f81e Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 29 Feb 2024 11:53:26 +0100 Subject: [PATCH 022/603] add micro bechmark --- .../window_partition_pushdown.benchmark | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 benchmark/micro/pushdown/window_partition_pushdown.benchmark diff --git a/benchmark/micro/pushdown/window_partition_pushdown.benchmark b/benchmark/micro/pushdown/window_partition_pushdown.benchmark new file mode 100644 index 000000000000..fe06696a9716 --- /dev/null +++ b/benchmark/micro/pushdown/window_partition_pushdown.benchmark @@ -0,0 +1,20 @@ +# name: benchmark/micro/pushdown/window_partition_pushdown.benchmark +# description: measure performance improvement of pushing down filters on window paritions +# group: [pushdown] + +name Window_parition +group micro +subgroup pushdown + +require tpch + +cache tpch_partition_lineitem.duckdb + +load +CALL dbgen(sf=1); +COPY lineitem to '${BENCHMARK_DIR}/lineitem_partitioned' (FORMAT PARQUET, PARTITION_BY l_shipmode); + +run +select * from (select l_shipmode, l_quantity, sum(l_quantity) OVER (PARTITION BY l_shipmode) +from read_parquet('${BENCHMARK_DIR}/lineitem_partitioned/*/*.parquet', hive_partitioning = true)) +where l_shipmode = 'SHIP'; From bcebeff2a8e5a0978c6527c73c616c501dc8624c Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 29 Feb 2024 08:46:47 -0300 Subject: [PATCH 023/603] Handle CSV Line Errors that fall over multiple buffers --- .../scanner/string_value_scanner.cpp | 28 ++++++++-- .../csv_scanner/string_value_scanner.hpp | 2 +- .../csv/rejects/csv_buffer_size_rejects.test | 55 +++++++++++++++++++ 3 files changed, 80 insertions(+), 5 deletions(-) create mode 100644 test/sql/copy/csv/rejects/csv_buffer_size_rejects.test diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index daa56dd209e0..70da1ac6ee3a 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -21,7 +21,7 @@ StringValueResult::StringValueResult(CSVStates &states, CSVStateMachine &state_m store_line_size(store_line_size_p), csv_file_scan(std::move(csv_file_scan_p)), lines_read(lines_read_p) { // Vector information D_ASSERT(number_of_columns > 0); - buffer_handles.push_back(buffer_handle); + buffer_handles[buffer_handle->buffer_idx] = buffer_handle; // Buffer Information buffer_ptr = buffer_handle->Ptr(); buffer_size = buffer_handle->actual_size; @@ -327,14 +327,27 @@ void StringValueResult::NullPaddingQuotedNewlineCheck() { string StringValueResult::ReconstructCurrentLine() { idx_t current_line_size = previous_line_start - pre_previous_line_start; string result; - result.resize(current_line_size - 1); if (previous_line_start.buffer_idx == pre_previous_line_start.buffer_idx) { + result.resize(current_line_size - 1); idx_t result_idx = 0; for (idx_t i = pre_previous_line_start.buffer_pos + 1; i < previous_line_start.buffer_pos; i++) { result[result_idx++] = buffer_ptr[i]; } } else { - throw InternalException("Oh no"); + result.resize(current_line_size); + if (buffer_handles.find(pre_previous_line_start.buffer_idx) == buffer_handles.end()) { + throw InternalException("CSV Buffer is not available to reconstruct CSV Line, please open an issue with " + "your query and dataset."); + } + idx_t result_idx = 0; + auto first_buffer = buffer_handles[pre_previous_line_start.buffer_idx]->Ptr(); + auto first_buffer_size = buffer_handles[pre_previous_line_start.buffer_idx]->actual_size; + for (idx_t i = pre_previous_line_start.buffer_pos + 1; i < first_buffer_size; i++) { + result[result_idx++] = first_buffer[i]; + } + for (idx_t i = 0; i < previous_line_start.buffer_pos; i++) { + result[result_idx++] = buffer_ptr[i]; + } } return result; } @@ -884,7 +897,6 @@ bool StringValueScanner::MoveToNextBuffer() { if (iterator.pos.buffer_pos >= cur_buffer_handle->actual_size) { previous_buffer_handle = cur_buffer_handle; cur_buffer_handle = buffer_manager->GetBuffer(++iterator.pos.buffer_idx); - result.buffer_handles.push_back(cur_buffer_handle); if (!cur_buffer_handle) { iterator.pos.buffer_idx--; buffer_handle_ptr = nullptr; @@ -914,6 +926,8 @@ bool StringValueScanner::MoveToNextBuffer() { } return false; } + result.buffer_handles[cur_buffer_handle->buffer_idx] = cur_buffer_handle; + iterator.pos.buffer_pos = 0; buffer_handle_ptr = cur_buffer_handle->Ptr(); // Handle overbuffer value @@ -1057,6 +1071,12 @@ void StringValueScanner::SetStart() { } } if (iterator.pos.buffer_pos == cur_buffer_handle->actual_size) { + // Propagate any errors + for (auto &error_vector : scan_finder->error_handler->errors) { + for (auto &error : error_vector.second) { + error_handler->Error(error); + } + } // If things go terribly wrong, we never loop indefinetly. iterator.pos.buffer_idx = scan_finder->iterator.pos.buffer_idx; iterator.pos.buffer_pos = scan_finder->iterator.pos.buffer_pos; diff --git a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp index 00e0bb80b2c2..d0e9124c81f2 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp @@ -102,7 +102,7 @@ class StringValueResult : public ScannerResult { idx_t chunk_col_id = 0; //! We must ensure that we keep the buffers alive until processing the query result - vector> buffer_handles; + unordered_map> buffer_handles; //! Requested size of buffers (i.e., either 32Mb or set by buffer_size parameter) idx_t requested_size; diff --git a/test/sql/copy/csv/rejects/csv_buffer_size_rejects.test b/test/sql/copy/csv/rejects/csv_buffer_size_rejects.test new file mode 100644 index 000000000000..f0de6714f96a --- /dev/null +++ b/test/sql/copy/csv/rejects/csv_buffer_size_rejects.test @@ -0,0 +1,55 @@ +# name: test/sql/copy/csv/rejects/csv_buffer_size_rejects.test +# description: Force CSV Lines from errors to fall mid-buffers +# group: [rejects] + +require skip_reload + +# FIXME: https://github.com/duckdb/duckdb/issues/7755 +require vector_size 2048 + +# Ensure that we can get the schema if we reduce the sample size and ignore errors +query IIIII +SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto( + 'test/sql/copy/csv/data/error/mismatch/big_bad*.csv', + sample_size=1, + buffer_size=5, + rejects_table='csv_rejects_table', + ignore_errors=true); +---- +BIGINT VARCHAR 11044 11044 2 + +query IIIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position +FROM csv_rejects_table; +---- +test/sql/copy/csv/data/error/mismatch/big_bad.csv 2176 0 "column0" CAST B, A 10875 +test/sql/copy/csv/data/error/mismatch/big_bad.csv 4176 0 "column0" CAST C, A 20875 +test/sql/copy/csv/data/error/mismatch/big_bad2.csv 3680 0 "column0" CAST B, A 18395 +test/sql/copy/csv/data/error/mismatch/big_bad2.csv 5680 0 "column0" CAST C, A 28395 + +query I +SELECT error_message +FROM csv_rejects_table where line=2176 and column_idx=0; +---- +:.*Could not convert string "B" to 'BIGINT'.* + +query I +SELECT error_message +FROM csv_rejects_table where line=4176 and column_idx=0; +---- +:.*Could not convert string "C" to 'BIGINT'.* + +query I +SELECT error_message +FROM csv_rejects_table where line=3680 and column_idx=0; +---- +:.*Could not convert string "B" to 'BIGINT'.* + +query I +SELECT error_message +FROM csv_rejects_table where line=5680 and column_idx=0; +---- +:.*Could not convert string "C" to 'BIGINT'.* + +statement ok +DROP TABLE csv_rejects_table; \ No newline at end of file From 8ab1a9a3d67a0192e517d931d333cd1547a52626 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 29 Feb 2024 09:11:06 -0300 Subject: [PATCH 024/603] rounding off minor details --- .../scanner/string_value_scanner.cpp | 26 +++++++++---------- .../csv_scanner/string_value_scanner.hpp | 3 +++ 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 70da1ac6ee3a..4adf70bc3ac7 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -353,6 +353,18 @@ string StringValueResult::ReconstructCurrentLine() { } bool StringValueResult::AddRowInternal() { + LinePosition current_line_start = {iterator.pos.buffer_idx, iterator.pos.buffer_pos, buffer_size}; + idx_t current_line_size = current_line_start - previous_line_start; + if (store_line_size) { + error_handler.NewMaxLineSize(current_line_size); + } + if (current_line_size > state_machine.options.maximum_line_size) { + LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), number_of_rows); + auto csv_error = CSVError::LineSizeError(state_machine.options, current_line_size, lines_per_batch); + error_handler.Error(csv_error); + } + pre_previous_line_start = previous_line_start; + previous_line_start = current_line_start; if (ignore_current_row) { // An error occurred on this row, we are ignoring it and resetting our control flag ignore_current_row = false; @@ -428,19 +440,6 @@ bool StringValueResult::AddRowInternal() { } bool StringValueResult::AddRow(StringValueResult &result, const idx_t buffer_pos) { - LinePosition current_line_start = {result.iterator.pos.buffer_idx, result.iterator.pos.buffer_pos, - result.buffer_size}; - idx_t current_line_size = current_line_start - result.previous_line_start; - if (result.store_line_size) { - result.error_handler.NewMaxLineSize(current_line_size); - } - if (current_line_size > result.state_machine.options.maximum_line_size) { - LinesPerBoundary lines_per_batch(result.iterator.GetBoundaryIdx(), result.number_of_rows); - auto csv_error = CSVError::LineSizeError(result.state_machine.options, current_line_size, lines_per_batch); - result.error_handler.Error(csv_error); - } - result.pre_previous_line_start = result.previous_line_start; - result.previous_line_start = current_line_start; if (result.last_position <= buffer_pos) { // We add the value if (result.quoted) { @@ -1082,6 +1081,7 @@ void StringValueScanner::SetStart() { iterator.pos.buffer_pos = scan_finder->iterator.pos.buffer_pos; result.last_position = iterator.pos.buffer_pos; iterator.done = scan_finder->iterator.done; + result.lines_read++; return; } } diff --git a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp index d0e9124c81f2..74064587a621 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp @@ -43,6 +43,9 @@ class LinePosition { return other.buffer_size - other.buffer_pos + buffer_pos; } idx_t GetGlobalPosition(idx_t requested_buffer_size) { + if (buffer_pos == requested_buffer_size) { + return requested_buffer_size * buffer_idx + buffer_pos; + } return requested_buffer_size * buffer_idx + buffer_pos + 1; } idx_t buffer_pos = 0; From b6fd5674a628cb9234c984b3578af2563a31777a Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 29 Feb 2024 14:27:22 -0300 Subject: [PATCH 025/603] lots of adjustments to make the errors accurate for small buffer sizes --- .../scanner/string_value_scanner.cpp | 57 ++++++++++++------- .../csv_scanner/string_value_scanner.hpp | 9 +-- .../csv/rejects/csv_buffer_size_rejects.test | 28 +++++---- 3 files changed, 57 insertions(+), 37 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 4adf70bc3ac7..dcb58116595e 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -324,29 +324,35 @@ void StringValueResult::NullPaddingQuotedNewlineCheck() { } //! Reconstructs the current line to be used in error messages -string StringValueResult::ReconstructCurrentLine() { - idx_t current_line_size = previous_line_start - pre_previous_line_start; +string StringValueResult::ReconstructCurrentLine(bool &first_char_nl) { string result; if (previous_line_start.buffer_idx == pre_previous_line_start.buffer_idx) { - result.resize(current_line_size - 1); - idx_t result_idx = 0; - for (idx_t i = pre_previous_line_start.buffer_pos + 1; i < previous_line_start.buffer_pos; i++) { - result[result_idx++] = buffer_ptr[i]; + if (buffer_handles.find(previous_line_start.buffer_idx) == buffer_handles.end()) { + throw InternalException("CSV Buffer is not available to reconstruct CSV Line, please open an issue with " + "your query and dataset."); + } + auto buffer = buffer_handles[pre_previous_line_start.buffer_idx]->Ptr(); + first_char_nl = + buffer[pre_previous_line_start.buffer_pos] == '\n' || buffer[pre_previous_line_start.buffer_pos] == '\r'; + for (idx_t i = pre_previous_line_start.buffer_pos + first_char_nl; i < previous_line_start.buffer_pos; i++) { + result += buffer[i]; } } else { - result.resize(current_line_size); - if (buffer_handles.find(pre_previous_line_start.buffer_idx) == buffer_handles.end()) { + if (buffer_handles.find(pre_previous_line_start.buffer_idx) == buffer_handles.end() || + buffer_handles.find(previous_line_start.buffer_idx) == buffer_handles.end()) { throw InternalException("CSV Buffer is not available to reconstruct CSV Line, please open an issue with " "your query and dataset."); } - idx_t result_idx = 0; auto first_buffer = buffer_handles[pre_previous_line_start.buffer_idx]->Ptr(); auto first_buffer_size = buffer_handles[pre_previous_line_start.buffer_idx]->actual_size; - for (idx_t i = pre_previous_line_start.buffer_pos + 1; i < first_buffer_size; i++) { - result[result_idx++] = first_buffer[i]; + auto second_buffer = buffer_handles[previous_line_start.buffer_idx]->Ptr(); + first_char_nl = first_buffer[pre_previous_line_start.buffer_pos] == '\n' || + first_buffer[pre_previous_line_start.buffer_pos] == '\r'; + for (idx_t i = pre_previous_line_start.buffer_pos + first_char_nl; i < first_buffer_size; i++) { + result += first_buffer[i]; } for (idx_t i = 0; i < previous_line_start.buffer_pos; i++) { - result[result_idx++] = buffer_ptr[i]; + result += second_buffer[i]; } } return result; @@ -379,10 +385,11 @@ bool StringValueResult::AddRowInternal() { << LogicalTypeIdToString(parse_types[cast_error.first]) << "\'"; auto error_string = error.str(); LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read - 1); - auto borked_line = ReconstructCurrentLine(); + bool first_nl; + auto borked_line = ReconstructCurrentLine(first_nl); auto csv_error = CSVError::CastError(state_machine.options, names[cast_error.first], error_string, cast_error.first, borked_line, lines_per_batch, - pre_previous_line_start.GetGlobalPosition(requested_size)); + pre_previous_line_start.GetGlobalPosition(requested_size, first_nl)); error_handler.Error(csv_error); } // If we got here it means we are ignoring errors, hence we need to signify to our result scanner to ignore this @@ -1069,23 +1076,35 @@ void StringValueScanner::SetStart() { return; } } - if (iterator.pos.buffer_pos == cur_buffer_handle->actual_size) { + if (iterator.pos.buffer_pos == cur_buffer_handle->actual_size || + scan_finder->iterator.GetBufferIdx() >= iterator.GetBufferIdx()) { // Propagate any errors - for (auto &error_vector : scan_finder->error_handler->errors) { - for (auto &error : error_vector.second) { - error_handler->Error(error); + if (!scan_finder->error_handler->errors.empty()) { + for (auto &error_vector : scan_finder->error_handler->errors) { + for (auto &error : error_vector.second) { + error_handler->Error(error); + } } + result.lines_read++; } // If things go terribly wrong, we never loop indefinetly. iterator.pos.buffer_idx = scan_finder->iterator.pos.buffer_idx; iterator.pos.buffer_pos = scan_finder->iterator.pos.buffer_pos; result.last_position = iterator.pos.buffer_pos; iterator.done = scan_finder->iterator.done; - result.lines_read++; return; } } } while (!line_found); + // Propagate any errors + if (!scan_finder->error_handler->errors.empty()) { + for (auto &error_vector : scan_finder->error_handler->errors) { + for (auto &error : error_vector.second) { + error_handler->Error(error); + } + } + result.lines_read++; + } iterator.pos.buffer_idx = scan_finder->result.pre_previous_line_start.buffer_idx; iterator.pos.buffer_pos = scan_finder->result.pre_previous_line_start.buffer_pos; result.last_position = iterator.pos.buffer_pos; diff --git a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp index 74064587a621..84ef406b30aa 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp @@ -42,11 +42,8 @@ class LinePosition { } return other.buffer_size - other.buffer_pos + buffer_pos; } - idx_t GetGlobalPosition(idx_t requested_buffer_size) { - if (buffer_pos == requested_buffer_size) { - return requested_buffer_size * buffer_idx + buffer_pos; - } - return requested_buffer_size * buffer_idx + buffer_pos + 1; + idx_t GetGlobalPosition(idx_t requested_buffer_size, bool first_char_nl) { + return requested_buffer_size * buffer_idx + buffer_pos + first_char_nl; } idx_t buffer_pos = 0; idx_t buffer_size = 0; @@ -127,7 +124,7 @@ class StringValueResult : public ScannerResult { static inline bool EmptyLine(StringValueResult &result, const idx_t buffer_pos); inline bool AddRowInternal(); //! Reconstructs the current line to be used in error messages - string ReconstructCurrentLine(); + string ReconstructCurrentLine(bool &first_char_nl); void HandleOverLimitRows(); diff --git a/test/sql/copy/csv/rejects/csv_buffer_size_rejects.test b/test/sql/copy/csv/rejects/csv_buffer_size_rejects.test index f0de6714f96a..15461ee1c9f3 100644 --- a/test/sql/copy/csv/rejects/csv_buffer_size_rejects.test +++ b/test/sql/copy/csv/rejects/csv_buffer_size_rejects.test @@ -7,49 +7,53 @@ require skip_reload # FIXME: https://github.com/duckdb/duckdb/issues/7755 require vector_size 2048 +loop buffer_size 5 10 + # Ensure that we can get the schema if we reduce the sample size and ignore errors query IIIII SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto( 'test/sql/copy/csv/data/error/mismatch/big_bad*.csv', sample_size=1, - buffer_size=5, + buffer_size=${buffer_size}, rejects_table='csv_rejects_table', ignore_errors=true); ---- BIGINT VARCHAR 11044 11044 2 -query IIIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position +query IIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table; ---- -test/sql/copy/csv/data/error/mismatch/big_bad.csv 2176 0 "column0" CAST B, A 10875 -test/sql/copy/csv/data/error/mismatch/big_bad.csv 4176 0 "column0" CAST C, A 20875 -test/sql/copy/csv/data/error/mismatch/big_bad2.csv 3680 0 "column0" CAST B, A 18395 -test/sql/copy/csv/data/error/mismatch/big_bad2.csv 5680 0 "column0" CAST C, A 28395 +test/sql/copy/csv/data/error/mismatch/big_bad.csv 0 "column0" CAST B, A 10875 +test/sql/copy/csv/data/error/mismatch/big_bad.csv 0 "column0" CAST C, A 20875 +test/sql/copy/csv/data/error/mismatch/big_bad2.csv 0 "column0" CAST B, A 18395 +test/sql/copy/csv/data/error/mismatch/big_bad2.csv 0 "column0" CAST C, A 28395 query I SELECT error_message -FROM csv_rejects_table where line=2176 and column_idx=0; +FROM csv_rejects_table where byte_position = 10875; ---- :.*Could not convert string "B" to 'BIGINT'.* query I SELECT error_message -FROM csv_rejects_table where line=4176 and column_idx=0; +FROM csv_rejects_table where byte_position = 20875; ---- :.*Could not convert string "C" to 'BIGINT'.* query I SELECT error_message -FROM csv_rejects_table where line=3680 and column_idx=0; +FROM csv_rejects_table where byte_position = 18395; ---- :.*Could not convert string "B" to 'BIGINT'.* query I SELECT error_message -FROM csv_rejects_table where line=5680 and column_idx=0; +FROM csv_rejects_table where byte_position = 28395; ---- :.*Could not convert string "C" to 'BIGINT'.* statement ok -DROP TABLE csv_rejects_table; \ No newline at end of file +DROP TABLE csv_rejects_table; + +endloop \ No newline at end of file From 7ad39f20b9d0462b28a5f15993d0c95288d652fa Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 29 Feb 2024 14:50:35 -0300 Subject: [PATCH 026/603] When resetting the buffer_handle we might have to keep the last one --- .../operator/csv_scanner/scanner/string_value_scanner.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index dcb58116595e..da67d5f1bd4a 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -252,7 +252,15 @@ void StringValueResult::Reset() { for (auto &v : validity_mask) { v->SetAllValid(result_size); } + // We keep a reference to the buffer from our current iteration if it already exists + shared_ptr cur_buffer; + if (buffer_handles.find(iterator.GetBufferIdx()) != buffer_handles.end()) { + cur_buffer = buffer_handles[iterator.GetBufferIdx()]; + } buffer_handles.clear(); + if (cur_buffer) { + buffer_handles[cur_buffer->buffer_idx] = cur_buffer; + } } void StringValueResult::AddQuotedValue(StringValueResult &result, const idx_t buffer_pos) { From 2826f951f537bd1976bb154b86a1bc7c5a7961d8 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 29 Feb 2024 15:02:21 -0300 Subject: [PATCH 027/603] fix test --- test/sql/copy/csv/rejects/csv_rejects_read.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/sql/copy/csv/rejects/csv_rejects_read.test b/test/sql/copy/csv/rejects/csv_rejects_read.test index 458e485ffc75..ab5b12db949a 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_read.test +++ b/test/sql/copy/csv/rejects/csv_rejects_read.test @@ -46,7 +46,7 @@ query IIIIIII rowsort SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table; ---- -test/sql/copy/csv/data/error/mismatch/bad2.csv 1 2 "col2" CAST ,2,DDD, 1 +test/sql/copy/csv/data/error/mismatch/bad2.csv 1 2 "col2" CAST 1,2,DDD, 0 test/sql/copy/csv/data/error/mismatch/bad2.csv 3 0 "col0" CAST EEE,7,FFF, 16 test/sql/copy/csv/data/error/mismatch/bad2.csv 3 2 "col2" CAST EEE,7,FFF, 16 From f6496fc22504f5d3ba62476203e436f21661e1ae Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 29 Feb 2024 15:20:05 -0300 Subject: [PATCH 028/603] We only care about propagting errors if we are ignoring them, as weird as this sounds --- .../scanner/string_value_scanner.cpp | 4 +- .../parallel/csv_parallel_buffer_size.test | 90 +++++++++---------- 2 files changed, 47 insertions(+), 47 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index da67d5f1bd4a..4785f54129b8 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -1087,7 +1087,7 @@ void StringValueScanner::SetStart() { if (iterator.pos.buffer_pos == cur_buffer_handle->actual_size || scan_finder->iterator.GetBufferIdx() >= iterator.GetBufferIdx()) { // Propagate any errors - if (!scan_finder->error_handler->errors.empty()) { + if (!scan_finder->error_handler->errors.empty() && state_machine->options.ignore_errors) { for (auto &error_vector : scan_finder->error_handler->errors) { for (auto &error : error_vector.second) { error_handler->Error(error); @@ -1105,7 +1105,7 @@ void StringValueScanner::SetStart() { } } while (!line_found); // Propagate any errors - if (!scan_finder->error_handler->errors.empty()) { + if (!scan_finder->error_handler->errors.empty() && state_machine->options.ignore_errors) { for (auto &error_vector : scan_finder->error_handler->errors) { for (auto &error : error_vector.second) { error_handler->Error(error); diff --git a/test/sql/copy/csv/parallel/csv_parallel_buffer_size.test b/test/sql/copy/csv/parallel/csv_parallel_buffer_size.test index d1c9d8ce5ee8..f6e1d7ada93a 100644 --- a/test/sql/copy/csv/parallel/csv_parallel_buffer_size.test +++ b/test/sql/copy/csv/parallel/csv_parallel_buffer_size.test @@ -7,51 +7,51 @@ statement ok PRAGMA verify_parallelism -query III -SELECT sum(a), sum(b), sum(c) FROM read_csv('test/sql/copy/csv/data/test/multi_column_integer.csv', COLUMNS=STRUCT_PACK(a := 'INTEGER', b := 'INTEGER', c := 'INTEGER'), auto_detect='true', delim = '|', buffer_size=30) ----- -111111111 51866 3195 - -query I -SELECT sum(a) FROM read_csv('test/sql/copy/csv/data/test/multi_column_integer.csv', COLUMNS=STRUCT_PACK(a := 'INTEGER', b := 'INTEGER', c := 'INTEGER'), auto_detect='true', delim = '|', buffer_size=30) ----- -111111111 - -query I -SELECT sum(a) FROM read_csv('test/sql/copy/csv/data/test/multi_column_integer_rn.csv', COLUMNS=STRUCT_PACK(a := 'INTEGER', b := 'INTEGER', c := 'INTEGER'), auto_detect='true', delim = '|', buffer_size=30) ----- -111111111 - -query IIII -select * from read_csv('test/sql/copy/csv/data/test/multi_column_string.csv', COLUMNS=STRUCT_PACK(a := 'INTEGER', b := 'INTEGER', c := 'INTEGER', d := 'VARCHAR'), auto_detect='true', delim = '|', buffer_size=25) ----- -1 6370 371 p1 -10 214 465 p2 -100 2403 160 p3 -1000 1564 67 p4 -10000 10617 138 p5 -100000 430 181 p6 -1000000 1904 658 p7 -10000000 12845 370 p8 -100000000 15519 785 p9 - -query IIII -select * from read_csv('test/sql/copy/csv/data/test/multi_column_string_rn.csv', COLUMNS=STRUCT_PACK(a := 'INTEGER', b := 'INTEGER', c := 'INTEGER', d := 'VARCHAR'), auto_detect='true', delim = '|', buffer_size=25) ----- -1 6370 371 p1 -10 214 465 p2 -100 2403 160 p3 -1000 1564 67 p4 -10000 10617 138 p5 -100000 430 181 p6 -1000000 1904 658 p7 -10000000 12845 370 p8 -100000000 15519 785 p9 - -query I -SELECT sum(a) FROM read_csv('test/sql/copy/csv/data/test/new_line_string_rn.csv', COLUMNS=STRUCT_PACK(a := 'INTEGER', b := 'INTEGER', c := 'INTEGER', d := 'VARCHAR'), auto_detect='true', delim = '|') ----- -111 +#query III +#SELECT sum(a), sum(b), sum(c) FROM read_csv('test/sql/copy/csv/data/test/multi_column_integer.csv', COLUMNS=STRUCT_PACK(a := 'INTEGER', b := 'INTEGER', c := 'INTEGER'), auto_detect='true', delim = '|', buffer_size=30) +#---- +#111111111 51866 3195 +# +#query I +#SELECT sum(a) FROM read_csv('test/sql/copy/csv/data/test/multi_column_integer.csv', COLUMNS=STRUCT_PACK(a := 'INTEGER', b := 'INTEGER', c := 'INTEGER'), auto_detect='true', delim = '|', buffer_size=30) +#---- +#111111111 +# +#query I +#SELECT sum(a) FROM read_csv('test/sql/copy/csv/data/test/multi_column_integer_rn.csv', COLUMNS=STRUCT_PACK(a := 'INTEGER', b := 'INTEGER', c := 'INTEGER'), auto_detect='true', delim = '|', buffer_size=30) +#---- +#111111111 +# +#query IIII +#select * from read_csv('test/sql/copy/csv/data/test/multi_column_string.csv', COLUMNS=STRUCT_PACK(a := 'INTEGER', b := 'INTEGER', c := 'INTEGER', d := 'VARCHAR'), auto_detect='true', delim = '|', buffer_size=25) +#---- +#1 6370 371 p1 +#10 214 465 p2 +#100 2403 160 p3 +#1000 1564 67 p4 +#10000 10617 138 p5 +#100000 430 181 p6 +#1000000 1904 658 p7 +#10000000 12845 370 p8 +#100000000 15519 785 p9 +# +#query IIII +#select * from read_csv('test/sql/copy/csv/data/test/multi_column_string_rn.csv', COLUMNS=STRUCT_PACK(a := 'INTEGER', b := 'INTEGER', c := 'INTEGER', d := 'VARCHAR'), auto_detect='true', delim = '|', buffer_size=25) +#---- +#1 6370 371 p1 +#10 214 465 p2 +#100 2403 160 p3 +#1000 1564 67 p4 +#10000 10617 138 p5 +#100000 430 181 p6 +#1000000 1904 658 p7 +#10000000 12845 370 p8 +#100000000 15519 785 p9 +# +#query I +#SELECT sum(a) FROM read_csv('test/sql/copy/csv/data/test/new_line_string_rn.csv', COLUMNS=STRUCT_PACK(a := 'INTEGER', b := 'INTEGER', c := 'INTEGER', d := 'VARCHAR'), auto_detect='true', delim = '|') +#---- +#111 query I SELECT sum(a) FROM read_csv('test/sql/copy/csv/data/test/new_line_string_rn.csv', COLUMNS=STRUCT_PACK(a := 'INTEGER', b := 'INTEGER', c := 'INTEGER', d := 'VARCHAR'), auto_detect='true', delim = '|', buffer_size=80) From 404c4da61b052555865554aaa89a441091cbb88f Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 29 Feb 2024 15:20:32 -0300 Subject: [PATCH 029/603] woopsie-doopsie --- .../parallel/csv_parallel_buffer_size.test | 90 +++++++++---------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/test/sql/copy/csv/parallel/csv_parallel_buffer_size.test b/test/sql/copy/csv/parallel/csv_parallel_buffer_size.test index f6e1d7ada93a..d1c9d8ce5ee8 100644 --- a/test/sql/copy/csv/parallel/csv_parallel_buffer_size.test +++ b/test/sql/copy/csv/parallel/csv_parallel_buffer_size.test @@ -7,51 +7,51 @@ statement ok PRAGMA verify_parallelism -#query III -#SELECT sum(a), sum(b), sum(c) FROM read_csv('test/sql/copy/csv/data/test/multi_column_integer.csv', COLUMNS=STRUCT_PACK(a := 'INTEGER', b := 'INTEGER', c := 'INTEGER'), auto_detect='true', delim = '|', buffer_size=30) -#---- -#111111111 51866 3195 -# -#query I -#SELECT sum(a) FROM read_csv('test/sql/copy/csv/data/test/multi_column_integer.csv', COLUMNS=STRUCT_PACK(a := 'INTEGER', b := 'INTEGER', c := 'INTEGER'), auto_detect='true', delim = '|', buffer_size=30) -#---- -#111111111 -# -#query I -#SELECT sum(a) FROM read_csv('test/sql/copy/csv/data/test/multi_column_integer_rn.csv', COLUMNS=STRUCT_PACK(a := 'INTEGER', b := 'INTEGER', c := 'INTEGER'), auto_detect='true', delim = '|', buffer_size=30) -#---- -#111111111 -# -#query IIII -#select * from read_csv('test/sql/copy/csv/data/test/multi_column_string.csv', COLUMNS=STRUCT_PACK(a := 'INTEGER', b := 'INTEGER', c := 'INTEGER', d := 'VARCHAR'), auto_detect='true', delim = '|', buffer_size=25) -#---- -#1 6370 371 p1 -#10 214 465 p2 -#100 2403 160 p3 -#1000 1564 67 p4 -#10000 10617 138 p5 -#100000 430 181 p6 -#1000000 1904 658 p7 -#10000000 12845 370 p8 -#100000000 15519 785 p9 -# -#query IIII -#select * from read_csv('test/sql/copy/csv/data/test/multi_column_string_rn.csv', COLUMNS=STRUCT_PACK(a := 'INTEGER', b := 'INTEGER', c := 'INTEGER', d := 'VARCHAR'), auto_detect='true', delim = '|', buffer_size=25) -#---- -#1 6370 371 p1 -#10 214 465 p2 -#100 2403 160 p3 -#1000 1564 67 p4 -#10000 10617 138 p5 -#100000 430 181 p6 -#1000000 1904 658 p7 -#10000000 12845 370 p8 -#100000000 15519 785 p9 -# -#query I -#SELECT sum(a) FROM read_csv('test/sql/copy/csv/data/test/new_line_string_rn.csv', COLUMNS=STRUCT_PACK(a := 'INTEGER', b := 'INTEGER', c := 'INTEGER', d := 'VARCHAR'), auto_detect='true', delim = '|') -#---- -#111 +query III +SELECT sum(a), sum(b), sum(c) FROM read_csv('test/sql/copy/csv/data/test/multi_column_integer.csv', COLUMNS=STRUCT_PACK(a := 'INTEGER', b := 'INTEGER', c := 'INTEGER'), auto_detect='true', delim = '|', buffer_size=30) +---- +111111111 51866 3195 + +query I +SELECT sum(a) FROM read_csv('test/sql/copy/csv/data/test/multi_column_integer.csv', COLUMNS=STRUCT_PACK(a := 'INTEGER', b := 'INTEGER', c := 'INTEGER'), auto_detect='true', delim = '|', buffer_size=30) +---- +111111111 + +query I +SELECT sum(a) FROM read_csv('test/sql/copy/csv/data/test/multi_column_integer_rn.csv', COLUMNS=STRUCT_PACK(a := 'INTEGER', b := 'INTEGER', c := 'INTEGER'), auto_detect='true', delim = '|', buffer_size=30) +---- +111111111 + +query IIII +select * from read_csv('test/sql/copy/csv/data/test/multi_column_string.csv', COLUMNS=STRUCT_PACK(a := 'INTEGER', b := 'INTEGER', c := 'INTEGER', d := 'VARCHAR'), auto_detect='true', delim = '|', buffer_size=25) +---- +1 6370 371 p1 +10 214 465 p2 +100 2403 160 p3 +1000 1564 67 p4 +10000 10617 138 p5 +100000 430 181 p6 +1000000 1904 658 p7 +10000000 12845 370 p8 +100000000 15519 785 p9 + +query IIII +select * from read_csv('test/sql/copy/csv/data/test/multi_column_string_rn.csv', COLUMNS=STRUCT_PACK(a := 'INTEGER', b := 'INTEGER', c := 'INTEGER', d := 'VARCHAR'), auto_detect='true', delim = '|', buffer_size=25) +---- +1 6370 371 p1 +10 214 465 p2 +100 2403 160 p3 +1000 1564 67 p4 +10000 10617 138 p5 +100000 430 181 p6 +1000000 1904 658 p7 +10000000 12845 370 p8 +100000000 15519 785 p9 + +query I +SELECT sum(a) FROM read_csv('test/sql/copy/csv/data/test/new_line_string_rn.csv', COLUMNS=STRUCT_PACK(a := 'INTEGER', b := 'INTEGER', c := 'INTEGER', d := 'VARCHAR'), auto_detect='true', delim = '|') +---- +111 query I SELECT sum(a) FROM read_csv('test/sql/copy/csv/data/test/new_line_string_rn.csv', COLUMNS=STRUCT_PACK(a := 'INTEGER', b := 'INTEGER', c := 'INTEGER', d := 'VARCHAR'), auto_detect='true', delim = '|', buffer_size=80) From ecf76d42a23c69d385b70b8b3a05001636242b14 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 1 Mar 2024 08:49:03 -0300 Subject: [PATCH 030/603] wip on rejects from flush cast --- data/csv/error/flush_cast.csv | 2814 +++++++++++++++++ .../csv/rejects/csv_rejects_flush_cast.test | 35 + 2 files changed, 2849 insertions(+) create mode 100644 data/csv/error/flush_cast.csv create mode 100644 test/sql/copy/csv/rejects/csv_rejects_flush_cast.test diff --git a/data/csv/error/flush_cast.csv b/data/csv/error/flush_cast.csv new file mode 100644 index 000000000000..33a2cc1af7be --- /dev/null +++ b/data/csv/error/flush_cast.csv @@ -0,0 +1,2814 @@ +a,b +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +B, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +25-09-2001, bla +c, bla +25-09-2001, bla diff --git a/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test b/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test new file mode 100644 index 000000000000..05c9af1cb02d --- /dev/null +++ b/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test @@ -0,0 +1,35 @@ +# name: test/sql/copy/csv/rejects/csv_rejects_flush_cast.test +# description: Test that Flush Cast functions properly for the rejects tables +# group: [rejects] + +require skip_reload + +# FIXME: https://github.com/duckdb/duckdb/issues/7755 +require vector_size 2048 + +query III +SELECT typeof(first(a)), typeof(first(b)), COUNT(*) FROM read_csv( + 'data/csv/error/flush_cast.csv', + columns = {'a': 'DATE', 'b': 'VARCHAR'}, + rejects_table='csv_rejects_table', + delim = ',', + dateformat = '%d-%m-%Y', + ignore_errors=true); +---- +DATE VARCHAR 2811 + + +query IIIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position +FROM csv_rejects_table; +---- +test/sql/copy/csv/data/error/mismatch/bad.csv 2 1 "col1" CAST 4,BBB,9, 9 + +query I +SELECT error_message +FROM csv_rejects_table; +---- +:.*Could not convert string "BBB" to 'INTEGER'.* + +statement ok +DROP TABLE csv_rejects_table; \ No newline at end of file From 53f612bbe056c3f12c7ab0c3256d8130a64b38a9 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 1 Mar 2024 10:27:51 -0300 Subject: [PATCH 031/603] introduce FullLinePosition --- .../scanner/string_value_scanner.cpp | 59 ++++++++++--------- .../csv_scanner/string_value_scanner.hpp | 21 +++++-- 2 files changed, 46 insertions(+), 34 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 4785f54129b8..516bf437edbb 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -29,8 +29,8 @@ StringValueResult::StringValueResult(CSVStates &states, CSVStateMachine &state_m requested_size = buffer_handle->requested_size; // Current Result information - previous_line_start = {iterator.pos.buffer_idx, iterator.pos.buffer_pos, buffer_handle->actual_size}; - pre_previous_line_start = previous_line_start; + current_line_position.begin = {iterator.pos.buffer_idx, iterator.pos.buffer_pos, buffer_handle->actual_size}; + current_line_position.end = current_line_position.begin; // Fill out Parse Types vector logical_types; parse_types = make_unsafe_uniq_array(number_of_columns); @@ -332,34 +332,33 @@ void StringValueResult::NullPaddingQuotedNewlineCheck() { } //! Reconstructs the current line to be used in error messages -string StringValueResult::ReconstructCurrentLine(bool &first_char_nl) { +string FullLinePosition::ReconstructCurrentLine(bool &first_char_nl, + unordered_map> &buffer_handles) { string result; - if (previous_line_start.buffer_idx == pre_previous_line_start.buffer_idx) { - if (buffer_handles.find(previous_line_start.buffer_idx) == buffer_handles.end()) { + if (end.buffer_idx == begin.buffer_idx) { + if (buffer_handles.find(end.buffer_idx) == buffer_handles.end()) { throw InternalException("CSV Buffer is not available to reconstruct CSV Line, please open an issue with " "your query and dataset."); } - auto buffer = buffer_handles[pre_previous_line_start.buffer_idx]->Ptr(); - first_char_nl = - buffer[pre_previous_line_start.buffer_pos] == '\n' || buffer[pre_previous_line_start.buffer_pos] == '\r'; - for (idx_t i = pre_previous_line_start.buffer_pos + first_char_nl; i < previous_line_start.buffer_pos; i++) { + auto buffer = buffer_handles[begin.buffer_idx]->Ptr(); + first_char_nl = buffer[begin.buffer_pos] == '\n' || buffer[begin.buffer_pos] == '\r'; + for (idx_t i = begin.buffer_pos + first_char_nl; i < end.buffer_pos; i++) { result += buffer[i]; } } else { - if (buffer_handles.find(pre_previous_line_start.buffer_idx) == buffer_handles.end() || - buffer_handles.find(previous_line_start.buffer_idx) == buffer_handles.end()) { + if (buffer_handles.find(begin.buffer_idx) == buffer_handles.end() || + buffer_handles.find(end.buffer_idx) == buffer_handles.end()) { throw InternalException("CSV Buffer is not available to reconstruct CSV Line, please open an issue with " "your query and dataset."); } - auto first_buffer = buffer_handles[pre_previous_line_start.buffer_idx]->Ptr(); - auto first_buffer_size = buffer_handles[pre_previous_line_start.buffer_idx]->actual_size; - auto second_buffer = buffer_handles[previous_line_start.buffer_idx]->Ptr(); - first_char_nl = first_buffer[pre_previous_line_start.buffer_pos] == '\n' || - first_buffer[pre_previous_line_start.buffer_pos] == '\r'; - for (idx_t i = pre_previous_line_start.buffer_pos + first_char_nl; i < first_buffer_size; i++) { + auto first_buffer = buffer_handles[begin.buffer_idx]->Ptr(); + auto first_buffer_size = buffer_handles[begin.buffer_idx]->actual_size; + auto second_buffer = buffer_handles[end.buffer_idx]->Ptr(); + first_char_nl = first_buffer[begin.buffer_pos] == '\n' || first_buffer[begin.buffer_pos] == '\r'; + for (idx_t i = begin.buffer_pos + first_char_nl; i < first_buffer_size; i++) { result += first_buffer[i]; } - for (idx_t i = 0; i < previous_line_start.buffer_pos; i++) { + for (idx_t i = 0; i < end.buffer_pos; i++) { result += second_buffer[i]; } } @@ -368,7 +367,7 @@ string StringValueResult::ReconstructCurrentLine(bool &first_char_nl) { bool StringValueResult::AddRowInternal() { LinePosition current_line_start = {iterator.pos.buffer_idx, iterator.pos.buffer_pos, buffer_size}; - idx_t current_line_size = current_line_start - previous_line_start; + idx_t current_line_size = current_line_start - current_line_position.end; if (store_line_size) { error_handler.NewMaxLineSize(current_line_size); } @@ -377,8 +376,8 @@ bool StringValueResult::AddRowInternal() { auto csv_error = CSVError::LineSizeError(state_machine.options, current_line_size, lines_per_batch); error_handler.Error(csv_error); } - pre_previous_line_start = previous_line_start; - previous_line_start = current_line_start; + current_line_position.begin = current_line_position.end; + current_line_position.end = current_line_start; if (ignore_current_row) { // An error occurred on this row, we are ignoring it and resetting our control flag ignore_current_row = false; @@ -394,10 +393,10 @@ bool StringValueResult::AddRowInternal() { auto error_string = error.str(); LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read - 1); bool first_nl; - auto borked_line = ReconstructCurrentLine(first_nl); - auto csv_error = CSVError::CastError(state_machine.options, names[cast_error.first], error_string, - cast_error.first, borked_line, lines_per_batch, - pre_previous_line_start.GetGlobalPosition(requested_size, first_nl)); + auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); + auto csv_error = CSVError::CastError( + state_machine.options, names[cast_error.first], error_string, cast_error.first, borked_line, + lines_per_batch, current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); error_handler.Error(csv_error); } // If we got here it means we are ignoring errors, hence we need to signify to our result scanner to ignore this @@ -444,6 +443,7 @@ bool StringValueResult::AddRowInternal() { number_of_rows--; } } + line_positions_per_row[number_of_rows] = current_line_position; cur_col_id = 0; chunk_col_id = 0; number_of_rows++; @@ -699,9 +699,10 @@ void StringValueScanner::Initialize() { SetStart(); } result.last_position = iterator.pos.buffer_pos; - result.previous_line_start = {iterator.pos.buffer_idx, iterator.pos.buffer_pos, cur_buffer_handle->actual_size}; + result.current_line_position.begin = {iterator.pos.buffer_idx, iterator.pos.buffer_pos, + cur_buffer_handle->actual_size}; - result.pre_previous_line_start = result.previous_line_start; + result.current_line_position.end = result.current_line_position.begin; } void StringValueScanner::ProcessExtraRow() { @@ -1113,8 +1114,8 @@ void StringValueScanner::SetStart() { } result.lines_read++; } - iterator.pos.buffer_idx = scan_finder->result.pre_previous_line_start.buffer_idx; - iterator.pos.buffer_pos = scan_finder->result.pre_previous_line_start.buffer_pos; + iterator.pos.buffer_idx = scan_finder->result.current_line_position.begin.buffer_idx; + iterator.pos.buffer_pos = scan_finder->result.current_line_position.begin.buffer_pos; result.last_position = iterator.pos.buffer_pos; } diff --git a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp index 84ef406b30aa..3c7ba2a28660 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp @@ -50,6 +50,18 @@ class LinePosition { idx_t buffer_idx = 0; }; +//! Keeps track of start and end of line positions in regard to the CSV file +class FullLinePosition { +public: + FullLinePosition() {}; + LinePosition begin; + LinePosition end; + + //! Reconstructs the current line to be used in error messages + string ReconstructCurrentLine(bool &first_char_nl, + unordered_map> &buffer_handles); +}; + class StringValueResult : public ScannerResult { public: StringValueResult(CSVStates &states, CSVStateMachine &state_machine, @@ -83,9 +95,10 @@ class StringValueResult : public ScannerResult { //! Information to properly handle errors CSVErrorHandler &error_handler; CSVIterator &iterator; - //! Where the previous line started, used to validate the maximum_line_size option - LinePosition previous_line_start; - LinePosition pre_previous_line_start; + //! Line position of the current line + FullLinePosition current_line_position; + //! Used for CSV line reconstruction on flushed errors + unordered_map line_positions_per_row; bool store_line_size = false; bool added_last_line = false; bool quoted_new_line = false; @@ -123,8 +136,6 @@ class StringValueResult : public ScannerResult { //! Handles EmptyLine states static inline bool EmptyLine(StringValueResult &result, const idx_t buffer_pos); inline bool AddRowInternal(); - //! Reconstructs the current line to be used in error messages - string ReconstructCurrentLine(bool &first_char_nl); void HandleOverLimitRows(); From 4403d16353b3cddd7995408a4c049af2a3d743ef Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 1 Mar 2024 10:36:20 -0300 Subject: [PATCH 032/603] Errors during flush being properly propagated --- .../scanner/string_value_scanner.cpp | 22 +++++++++++++------ .../csv/rejects/csv_rejects_flush_cast.test | 15 +++++++++---- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 516bf437edbb..d08a4a06f009 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -649,10 +649,13 @@ void StringValueScanner::Flush(DataChunk &insert_chunk) { } LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read - parse_chunk.size() + line_error); - // auto borked_line = result.ReconstructCurrentLine(); - string empty; - auto csv_error = CSVError::CastError(state_machine->options, csv_file_scan->names[col_idx], - error_message, col_idx, empty, lines_per_batch, 0); + bool first_nl; + auto borked_line = + result.line_positions_per_row[line_error].ReconstructCurrentLine(first_nl, result.buffer_handles); + auto csv_error = CSVError::CastError( + state_machine->options, csv_file_scan->names[col_idx], error_message, col_idx, borked_line, + lines_per_batch, + result.line_positions_per_row[line_error].begin.GetGlobalPosition(result.result_size, first_nl)); error_handler->Error(csv_error); } borked_lines.insert(line_error++); @@ -668,9 +671,14 @@ void StringValueScanner::Flush(DataChunk &insert_chunk) { } LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read - parse_chunk.size() + line_error); - string empty; - auto csv_error = CSVError::CastError(state_machine->options, csv_file_scan->names[col_idx], - error_message, col_idx, empty, lines_per_batch, 0); + bool first_nl; + auto borked_line = result.line_positions_per_row[line_error].ReconstructCurrentLine( + first_nl, result.buffer_handles); + auto csv_error = + CSVError::CastError(state_machine->options, csv_file_scan->names[col_idx], error_message, + col_idx, borked_line, lines_per_batch, + result.line_positions_per_row[line_error].begin.GetGlobalPosition( + result.result_size, first_nl)); error_handler->Error(csv_error); } diff --git a/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test b/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test index 05c9af1cb02d..87768b805d8a 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test +++ b/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test @@ -21,15 +21,22 @@ DATE VARCHAR 2811 query IIIIIII rowsort SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position -FROM csv_rejects_table; +FROM csv_rejects_table order by all; ---- -test/sql/copy/csv/data/error/mismatch/bad.csv 2 1 "col1" CAST 4,BBB,9, 9 +data/csv/error/flush_cast.csv 2813 0 "a" CAST c, bla 44971 +data/csv/error/flush_cast.csv 439 0 "a" CAST B, bla 6996 query I SELECT error_message -FROM csv_rejects_table; +FROM csv_rejects_table where byte_position = 6996; ---- -:.*Could not convert string "BBB" to 'INTEGER'.* +:.*Could not parse string "B" according to format specifier "%d-%m-%Y".* + +query I +SELECT error_message +FROM csv_rejects_table where byte_position = 44971; +---- +:.*Could not parse string "c" according to format specifier "%d-%m-%Y".* statement ok DROP TABLE csv_rejects_table; \ No newline at end of file From 7129aaf150e0889c975ff29a27270bfc4151567f Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 1 Mar 2024 12:02:42 -0300 Subject: [PATCH 033/603] preparing the ground for the other errors --- .../scanner/string_value_scanner.cpp | 29 ++++++++++++++----- .../operator/csv_scanner/util/csv_error.cpp | 19 ++++++------ .../operator/csv_scanner/csv_error.hpp | 12 ++++---- 3 files changed, 37 insertions(+), 23 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index d08a4a06f009..3852cd338d17 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -307,8 +307,11 @@ void StringValueResult::AddValue(StringValueResult &result, const idx_t buffer_p void StringValueResult::HandleOverLimitRows() { LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), number_of_rows + 1); - auto csv_error = CSVError::IncorrectColumnAmountError(state_machine.options, nullptr, number_of_columns, - cur_col_id + 1, lines_per_batch); + bool first_nl; + auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); + auto csv_error = + CSVError::IncorrectColumnAmountError(state_machine.options, cur_col_id + 1, lines_per_batch, borked_line, + current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); error_handler.Error(csv_error); // If we get here we need to remove the last line cur_col_id = 0; @@ -372,8 +375,12 @@ bool StringValueResult::AddRowInternal() { error_handler.NewMaxLineSize(current_line_size); } if (current_line_size > state_machine.options.maximum_line_size) { + bool first_nl; + auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), number_of_rows); - auto csv_error = CSVError::LineSizeError(state_machine.options, current_line_size, lines_per_batch); + auto csv_error = + CSVError::LineSizeError(state_machine.options, current_line_size, lines_per_batch, borked_line, + current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); error_handler.Error(csv_error); } current_line_position.begin = current_line_position.end; @@ -435,9 +442,12 @@ bool StringValueResult::AddRowInternal() { } } else { // If we are not null-padding this is an error + bool first_nl; + auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), number_of_rows + 1); - auto csv_error = CSVError::IncorrectColumnAmountError(state_machine.options, nullptr, number_of_columns, - cur_col_id, lines_per_batch); + auto csv_error = CSVError::IncorrectColumnAmountError( + state_machine.options, cur_col_id, lines_per_batch, borked_line, + current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); error_handler.Error(csv_error); // If we are here we ignore_errors, so we delete this line number_of_rows--; @@ -481,9 +491,12 @@ bool StringValueResult::AddRow(StringValueResult &result, const idx_t buffer_pos void StringValueResult::InvalidState(StringValueResult &result) { // FIXME: How do we recover from an invalid state? Can we restart the state machine and jump to the next row? LinesPerBoundary lines_per_batch(result.iterator.GetBoundaryIdx(), result.number_of_rows); - auto csv_error = CSVError::UnterminatedQuotesError(result.state_machine.options, - static_cast(result.vector_ptr[result.chunk_col_id]), - result.number_of_rows, result.cur_col_id, lines_per_batch); + bool first_nl; + auto borked_line = result.current_line_position.ReconstructCurrentLine(first_nl, result.buffer_handles); + + auto csv_error = CSVError::UnterminatedQuotesError( + result.state_machine.options, result.cur_col_id, lines_per_batch, borked_line, + result.current_line_position.begin.GetGlobalPosition(result.requested_size, first_nl)); result.error_handler.Error(csv_error); } diff --git a/src/execution/operator/csv_scanner/util/csv_error.cpp b/src/execution/operator/csv_scanner/util/csv_error.cpp index b77437c88f6b..3174ff2d7723 100644 --- a/src/execution/operator/csv_scanner/util/csv_error.cpp +++ b/src/execution/operator/csv_scanner/util/csv_error.cpp @@ -115,12 +115,13 @@ CSVError CSVError::CastError(const CSVReaderOptions &options, string &column_nam return CSVError(error.str(), CSVErrorType::CAST_ERROR, column_idx, csv_row, error_info, byte_position); } -CSVError CSVError::LineSizeError(const CSVReaderOptions &options, idx_t actual_size, LinesPerBoundary error_info) { +CSVError CSVError::LineSizeError(const CSVReaderOptions &options, idx_t actual_size, LinesPerBoundary error_info, + string &csv_row, idx_t byte_position) { std::ostringstream error; error << "Maximum line size of " << options.maximum_line_size << " bytes exceeded. "; error << "Actual Size:" << actual_size << " bytes." << std::endl; error << options.ToString(); - return CSVError(error.str(), CSVErrorType::MAXIMUM_LINE_SIZE, error_info); + return CSVError(error.str(), CSVErrorType::MAXIMUM_LINE_SIZE, 0, csv_row, error_info, byte_position); } CSVError CSVError::SniffingError(string &file_path) { @@ -141,26 +142,26 @@ CSVError CSVError::NullPaddingFail(const CSVReaderOptions &options, LinesPerBoun return CSVError(error.str(), CSVErrorType::NULLPADDED_QUOTED_NEW_VALUE, error_info); } -CSVError CSVError::UnterminatedQuotesError(const CSVReaderOptions &options, string_t *vector_ptr, - idx_t vector_line_start, idx_t current_column, LinesPerBoundary error_info) { +CSVError CSVError::UnterminatedQuotesError(const CSVReaderOptions &options, idx_t current_column, + LinesPerBoundary error_info, string &csv_row, idx_t byte_position) { std::ostringstream error; error << "Value with unterminated quote found." << std::endl; error << std::endl; // What were the options error << options.ToString(); - return CSVError(error.str(), CSVErrorType::UNTERMINATED_QUOTES, error_info); + return CSVError(error.str(), CSVErrorType::UNTERMINATED_QUOTES, current_column, csv_row, error_info, byte_position); } -CSVError CSVError::IncorrectColumnAmountError(const CSVReaderOptions &options, string_t *vector_ptr, - idx_t vector_line_start, idx_t actual_columns, - LinesPerBoundary error_info) { +CSVError CSVError::IncorrectColumnAmountError(const CSVReaderOptions &options, idx_t actual_columns, + LinesPerBoundary error_info, string &csv_row, idx_t byte_position) { std::ostringstream error; // How many columns were expected and how many were found error << "Expected Number of Columns: " << options.dialect_options.num_cols << " Found: " << actual_columns << std::endl; // What were the options error << options.ToString(); - return CSVError(error.str(), CSVErrorType::INCORRECT_COLUMN_AMOUNT, error_info); + return CSVError(error.str(), CSVErrorType::INCORRECT_COLUMN_AMOUNT, actual_columns, csv_row, error_info, + byte_position); } bool CSVErrorHandler::PrintLineNumber(CSVError &error) { diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp index 44bd4f25913a..d8f755600e95 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp @@ -59,18 +59,18 @@ class CSVError { static CSVError CastError(const CSVReaderOptions &options, string &column_name, string &cast_error, idx_t column_idx, string &csv_row, LinesPerBoundary error_info, idx_t byte_position); //! Produces error for when the line size exceeds the maximum line size option - static CSVError LineSizeError(const CSVReaderOptions &options, idx_t actual_size, LinesPerBoundary error_info); + static CSVError LineSizeError(const CSVReaderOptions &options, idx_t actual_size, LinesPerBoundary error_info, + string &csv_row, idx_t byte_position); //! Produces error for when the sniffer couldn't find viable options static CSVError SniffingError(string &file_path); //! Produces error messages for unterminated quoted values - static CSVError UnterminatedQuotesError(const CSVReaderOptions &options, string_t *vector_ptr, - idx_t vector_line_start, idx_t current_column, LinesPerBoundary error_info); + static CSVError UnterminatedQuotesError(const CSVReaderOptions &options, idx_t current_column, + LinesPerBoundary error_info, string &csv_row, idx_t byte_position); //! Produces error messages for null_padding option is set and we have quoted new values in parallel static CSVError NullPaddingFail(const CSVReaderOptions &options, LinesPerBoundary error_info); //! Produces error for incorrect (e.g., smaller and lower than the predefined) number of columns in a CSV Line - static CSVError IncorrectColumnAmountError(const CSVReaderOptions &options, string_t *vector_ptr, - idx_t vector_line_start, idx_t actual_columns, - LinesPerBoundary error_info); + static CSVError IncorrectColumnAmountError(const CSVReaderOptions &options, idx_t actual_columns, + LinesPerBoundary error_info, string &csv_row, idx_t byte_position); idx_t GetBoundaryIndex() { return error_info.boundary_idx; } From 2130e4d1a1f04abd681f6877a7253f061693272e Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 1 Mar 2024 22:11:50 +0100 Subject: [PATCH 034/603] moved from 'temporary_file_manager' branch --- src/include/duckdb/main/config.hpp | 3 + src/include/duckdb/main/settings.hpp | 10 +++ .../duckdb/storage/temporary_file_manager.hpp | 33 ++++++++-- src/main/config.cpp | 9 +++ src/main/database.cpp | 3 + src/main/settings/settings.cpp | 19 ++++++ src/storage/temporary_file_manager.cpp | 62 ++++++++++++++++--- test/api/test_reset.cpp | 1 + 8 files changed, 128 insertions(+), 12 deletions(-) diff --git a/src/include/duckdb/main/config.hpp b/src/include/duckdb/main/config.hpp index 468ec5bff298..b1b6a0fc0756 100644 --- a/src/include/duckdb/main/config.hpp +++ b/src/include/duckdb/main/config.hpp @@ -118,6 +118,8 @@ struct DBConfigOptions { string autoinstall_extension_repo = ""; //! The maximum memory used by the database system (in bytes). Default: 80% of System available memory idx_t maximum_memory = (idx_t)-1; + //! The maximum size of the 'temp_directory' folder when set (in bytes). Default 2x 'maximum_memory' + idx_t maximum_swap_space = (idx_t)-1; //! The maximum amount of CPU threads used by the database system. Default: all available. idx_t maximum_threads = (idx_t)-1; //! The number of external threads that work on DuckDB tasks. Default: 1. @@ -276,6 +278,7 @@ struct DBConfig { DUCKDB_API IndexTypeSet &GetIndexTypes(); static idx_t GetSystemMaxThreads(FileSystem &fs); void SetDefaultMaxMemory(); + void SetDefaultMaxSwapSpace(); OrderType ResolveOrder(OrderType order_type) const; OrderByNullType ResolveNullOrder(OrderType order_type, OrderByNullType null_type) const; diff --git a/src/include/duckdb/main/settings.hpp b/src/include/duckdb/main/settings.hpp index 31d8b1db8931..9ae16ea0fb91 100644 --- a/src/include/duckdb/main/settings.hpp +++ b/src/include/duckdb/main/settings.hpp @@ -418,6 +418,16 @@ struct MaximumMemorySetting { static Value GetSetting(ClientContext &context); }; +struct MaximumTempDirectorySize { + static constexpr const char *Name = "max_temp_directory_size"; + static constexpr const char *Description = + "The maximum amount of data stored inside the 'temp_directory' (when set) (e.g. 1GB)"; + static constexpr const LogicalTypeId InputType = LogicalTypeId::VARCHAR; + static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value ¶meter); + static void ResetGlobal(DatabaseInstance *db, DBConfig &config); + static Value GetSetting(ClientContext &context); +}; + struct OldImplicitCasting { static constexpr const char *Name = "old_implicit_casting"; static constexpr const char *Description = "Allow implicit casting to/from VARCHAR"; diff --git a/src/include/duckdb/storage/temporary_file_manager.hpp b/src/include/duckdb/storage/temporary_file_manager.hpp index bfb22ee96139..7c118fb4cf76 100644 --- a/src/include/duckdb/storage/temporary_file_manager.hpp +++ b/src/include/duckdb/storage/temporary_file_manager.hpp @@ -23,9 +23,26 @@ namespace duckdb { // BlockIndexManager //===--------------------------------------------------------------------===// +class TemporaryFileManager; + +struct FileSizeMonitor { +public: + static constexpr idx_t TEMPFILE_BLOCK_SIZE = Storage::BLOCK_ALLOC_SIZE; + +public: + FileSizeMonitor(TemporaryFileManager &manager); + +public: + void Increase(idx_t blocks); + void Decrease(idx_t blocks); + +private: + TemporaryFileManager &manager; +}; + struct BlockIndexManager { public: - BlockIndexManager(); + BlockIndexManager(unique_ptr file_size_monitor = nullptr); public: //! Obtains a new block index from the index manager @@ -43,6 +60,7 @@ struct BlockIndexManager { idx_t max_index; set free_indexes; set indexes_in_use; + unique_ptr file_size_monitor; }; //===--------------------------------------------------------------------===// @@ -69,7 +87,8 @@ class TemporaryFileHandle { constexpr static idx_t MAX_ALLOWED_INDEX_BASE = 4000; public: - TemporaryFileHandle(idx_t temp_file_count, DatabaseInstance &db, const string &temp_directory, idx_t index); + TemporaryFileHandle(idx_t temp_file_count, DatabaseInstance &db, const string &temp_directory, idx_t index, + TemporaryFileManager &manager); public: struct TemporaryFileLock { @@ -103,8 +122,6 @@ class TemporaryFileHandle { BlockIndexManager index_manager; }; -class TemporaryFileManager; - //===--------------------------------------------------------------------===// // TemporaryDirectoryHandle //===--------------------------------------------------------------------===// @@ -130,6 +147,7 @@ class TemporaryDirectoryHandle { class TemporaryFileManager { public: TemporaryFileManager(DatabaseInstance &db, const string &temp_directory_p); + ~TemporaryFileManager(); public: struct TemporaryManagerLock { @@ -145,6 +163,11 @@ class TemporaryFileManager { unique_ptr ReadTemporaryBuffer(block_id_t id, unique_ptr reusable_buffer); void DeleteTemporaryBuffer(block_id_t id); vector GetTemporaryFiles(); + idx_t GetTotalUsedSpaceInBytes(); + //! Register temporary file size growth + void IncreaseSizeOnDisk(idx_t amount); + //! Register temporary file size decrease + void DecreaseSizeOnDisk(idx_t amount); private: void EraseUsedBlock(TemporaryManagerLock &lock, block_id_t id, TemporaryFileHandle *handle, @@ -164,6 +187,8 @@ class TemporaryFileManager { unordered_map used_blocks; //! Manager of in-use temporary file indexes BlockIndexManager index_manager; + //! The size in bytes of the temporary files that are currently alive + atomic size_on_disk; }; } // namespace duckdb diff --git a/src/main/config.cpp b/src/main/config.cpp index 2dc67e40eac6..cbc030783264 100644 --- a/src/main/config.cpp +++ b/src/main/config.cpp @@ -1,5 +1,6 @@ #include "duckdb/main/config.hpp" +#include "duckdb/common/operator/multiply.hpp" #include "duckdb/common/operator/cast_operators.hpp" #include "duckdb/common/string_util.hpp" #include "duckdb/main/settings.hpp" @@ -258,6 +259,14 @@ void DBConfig::SetDefaultMaxMemory() { } } +void DBConfig::SetDefaultMaxSwapSpace() { + auto memory_limit = options.maximum_memory; + if (!TryMultiplyOperator::Operation(memory_limit, 2, options.maximum_memory)) { + // Can't default to 2x memory: fall back to 5GB instead + options.maximum_memory = ParseMemoryLimit("5GB"); + } +} + void DBConfig::CheckLock(const string &name) { if (!options.lock_configuration) { // not locked diff --git a/src/main/database.cpp b/src/main/database.cpp index 61655d4273ac..9744fc97d808 100644 --- a/src/main/database.cpp +++ b/src/main/database.cpp @@ -333,6 +333,9 @@ void DatabaseInstance::Configure(DBConfig &new_config) { if (config.options.maximum_memory == (idx_t)-1) { config.SetDefaultMaxMemory(); } + if (config.options.maximum_swap_space == (idx_t)-1) { + config.SetDefaultMaxSwapSpace(); + } if (new_config.options.maximum_threads == (idx_t)-1) { config.options.maximum_threads = config.GetSystemMaxThreads(*config.file_system); } diff --git a/src/main/settings/settings.cpp b/src/main/settings/settings.cpp index 2d801e988e3c..e16c2f795f7b 100644 --- a/src/main/settings/settings.cpp +++ b/src/main/settings/settings.cpp @@ -948,6 +948,25 @@ Value MaximumMemorySetting::GetSetting(ClientContext &context) { return Value(StringUtil::BytesToHumanReadableString(config.options.maximum_memory)); } +//===--------------------------------------------------------------------===// +// Maximum Temp Directory Size +//===--------------------------------------------------------------------===// +void MaximumTempDirectorySize::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) { + config.options.maximum_swap_space = DBConfig::ParseMemoryLimit(input.ToString()); + if (db) { + BufferManager::GetBufferManager(*db).SetLimit(config.options.maximum_swap_space); + } +} + +void MaximumTempDirectorySize::ResetGlobal(DatabaseInstance *db, DBConfig &config) { + config.SetDefaultMaxSwapSpace(); +} + +Value MaximumTempDirectorySize::GetSetting(ClientContext &context) { + auto &config = DBConfig::GetConfig(context); + return Value(StringUtil::BytesToHumanReadableString(config.options.maximum_swap_space)); +} + //===--------------------------------------------------------------------===// // Old Implicit Casting //===--------------------------------------------------------------------===// diff --git a/src/storage/temporary_file_manager.cpp b/src/storage/temporary_file_manager.cpp index c374829037ec..6a92048f5046 100644 --- a/src/storage/temporary_file_manager.cpp +++ b/src/storage/temporary_file_manager.cpp @@ -4,11 +4,29 @@ namespace duckdb { +//===--------------------------------------------------------------------===// +// FileSizeMonitor +//===--------------------------------------------------------------------===// + +FileSizeMonitor::FileSizeMonitor(TemporaryFileManager &manager) : manager(manager) { +} + +void FileSizeMonitor::Increase(idx_t blocks) { + auto size_on_disk = blocks * TEMPFILE_BLOCK_SIZE; + manager.IncreaseSizeOnDisk(size_on_disk); +} + +void FileSizeMonitor::Decrease(idx_t blocks) { + auto size_on_disk = blocks * TEMPFILE_BLOCK_SIZE; + manager.DecreaseSizeOnDisk(size_on_disk); +} + //===--------------------------------------------------------------------===// // BlockIndexManager //===--------------------------------------------------------------------===// -BlockIndexManager::BlockIndexManager() : max_index(0) { +BlockIndexManager::BlockIndexManager(unique_ptr file_size_monitor) + : max_index(0), file_size_monitor(std::move(file_size_monitor)) { } idx_t BlockIndexManager::GetNewBlockIndex() { @@ -27,12 +45,17 @@ bool BlockIndexManager::RemoveIndex(idx_t index) { free_indexes.insert(index); // check if we can truncate the file + auto old_max = max_index; + // get the max_index in use right now - auto max_index_in_use = indexes_in_use.empty() ? 0 : *indexes_in_use.rbegin(); + auto max_index_in_use = indexes_in_use.empty() ? 0 : *indexes_in_use.rbegin() + 1; if (max_index_in_use < max_index) { // max index in use is lower than the max_index // reduce the max_index max_index = indexes_in_use.empty() ? 0 : max_index_in_use + 1; + if (file_size_monitor) { + file_size_monitor->Decrease(old_max - max_index); + } // we can remove any free_indexes that are larger than the current max_index while (!free_indexes.empty()) { auto max_entry = *free_indexes.rbegin(); @@ -56,7 +79,12 @@ bool BlockIndexManager::HasFreeBlocks() { idx_t BlockIndexManager::GetNewBlockIndexInternal() { if (free_indexes.empty()) { - return max_index++; + auto new_index = max_index; + max_index++; + if (file_size_monitor) { + file_size_monitor->Increase(1); + } + return new_index; } auto entry = free_indexes.begin(); auto index = *entry; @@ -69,9 +97,10 @@ idx_t BlockIndexManager::GetNewBlockIndexInternal() { //===--------------------------------------------------------------------===// TemporaryFileHandle::TemporaryFileHandle(idx_t temp_file_count, DatabaseInstance &db, const string &temp_directory, - idx_t index) + idx_t index, TemporaryFileManager &manager) : max_allowed_index((1 << temp_file_count) * MAX_ALLOWED_INDEX_BASE), db(db), file_index(index), - path(FileSystem::GetFileSystem(db).JoinPath(temp_directory, "duckdb_temp_storage-" + to_string(index) + ".tmp")) { + path(FileSystem::GetFileSystem(db).JoinPath(temp_directory, "duckdb_temp_storage-" + to_string(index) + ".tmp")), + index_manager(make_uniq(manager)) { } TemporaryFileHandle::TemporaryFileLock::TemporaryFileLock(mutex &mutex) : lock(mutex) { @@ -135,7 +164,7 @@ void TemporaryFileHandle::CreateFileIfNotExists(TemporaryFileLock &) { return; } auto &fs = FileSystem::GetFileSystem(db); - auto open_flags = FileFlags::FILE_FLAGS_READ | FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE; + uint8_t open_flags = FileFlags::FILE_FLAGS_READ | FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE; handle = fs.OpenFile(path, open_flags); } @@ -225,7 +254,12 @@ bool TemporaryFileIndex::IsValid() const { //===--------------------------------------------------------------------===// TemporaryFileManager::TemporaryFileManager(DatabaseInstance &db, const string &temp_directory_p) - : db(db), temp_directory(temp_directory_p) { + : db(db), temp_directory(temp_directory_p), size_on_disk(0) { +} + +TemporaryFileManager::~TemporaryFileManager() { + files.clear(); + D_ASSERT(size_on_disk.load() == 0); } TemporaryFileManager::TemporaryManagerLock::TemporaryManagerLock(mutex &mutex) : lock(mutex) { @@ -250,7 +284,7 @@ void TemporaryFileManager::WriteTemporaryBuffer(block_id_t block_id, FileBuffer if (!handle) { // no existing handle to write to; we need to create & open a new file auto new_file_index = index_manager.GetNewBlockIndex(); - auto new_file = make_uniq(files.size(), db, temp_directory, new_file_index); + auto new_file = make_uniq(files.size(), db, temp_directory, new_file_index, *this); handle = new_file.get(); files[new_file_index] = std::move(new_file); @@ -269,6 +303,18 @@ bool TemporaryFileManager::HasTemporaryBuffer(block_id_t block_id) { return used_blocks.find(block_id) != used_blocks.end(); } +idx_t TemporaryFileManager::GetTotalUsedSpaceInBytes() { + return size_on_disk.load(); +} + +void TemporaryFileManager::IncreaseSizeOnDisk(idx_t bytes) { + size_on_disk += bytes; +} + +void TemporaryFileManager::DecreaseSizeOnDisk(idx_t bytes) { + size_on_disk -= bytes; +} + unique_ptr TemporaryFileManager::ReadTemporaryBuffer(block_id_t id, unique_ptr reusable_buffer) { TemporaryFileIndex index; diff --git a/test/api/test_reset.cpp b/test/api/test_reset.cpp index 18f795d837c9..e06f965cd703 100644 --- a/test/api/test_reset.cpp +++ b/test/api/test_reset.cpp @@ -87,6 +87,7 @@ OptionValueSet &GetValueForOption(const string &name) { {"immediate_transaction_mode", {true}}, {"max_expression_depth", {50}}, {"max_memory", {"4.0 GiB"}}, + {"max_temp_directory_size", {"10.0 GiB"}}, {"memory_limit", {"4.0 GiB"}}, {"ordered_aggregate_threshold", {Value::UBIGINT(idx_t(1) << 12)}}, {"null_order", {"nulls_first"}}, From 1e441a68212c9fcefce95d2b63d8622e6e9985a5 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 1 Mar 2024 23:11:12 +0100 Subject: [PATCH 035/603] create the exception, thrown whenever we try to increase the temp directory size beyond the max --- src/main/config.cpp | 5 +++-- src/main/settings/settings.cpp | 3 --- src/storage/temporary_file_manager.cpp | 15 ++++++++++++++- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/main/config.cpp b/src/main/config.cpp index cbc030783264..78e5d36b1014 100644 --- a/src/main/config.cpp +++ b/src/main/config.cpp @@ -94,6 +94,7 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting DUCKDB_LOCAL(IntegerDivisionSetting), DUCKDB_LOCAL(MaximumExpressionDepthSetting), DUCKDB_GLOBAL(MaximumMemorySetting), + DUCKDB_GLOBAL(MaximumTempDirectorySize), DUCKDB_GLOBAL(OldImplicitCasting), DUCKDB_GLOBAL_ALIAS("memory_limit", MaximumMemorySetting), DUCKDB_GLOBAL_ALIAS("null_order", DefaultNullOrderSetting), @@ -261,9 +262,9 @@ void DBConfig::SetDefaultMaxMemory() { void DBConfig::SetDefaultMaxSwapSpace() { auto memory_limit = options.maximum_memory; - if (!TryMultiplyOperator::Operation(memory_limit, 2, options.maximum_memory)) { + if (!TryMultiplyOperator::Operation(memory_limit, static_cast(2), options.maximum_swap_space)) { // Can't default to 2x memory: fall back to 5GB instead - options.maximum_memory = ParseMemoryLimit("5GB"); + options.maximum_swap_space = ParseMemoryLimit("5GB"); } } diff --git a/src/main/settings/settings.cpp b/src/main/settings/settings.cpp index e16c2f795f7b..bdda84adf07f 100644 --- a/src/main/settings/settings.cpp +++ b/src/main/settings/settings.cpp @@ -953,9 +953,6 @@ Value MaximumMemorySetting::GetSetting(ClientContext &context) { //===--------------------------------------------------------------------===// void MaximumTempDirectorySize::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) { config.options.maximum_swap_space = DBConfig::ParseMemoryLimit(input.ToString()); - if (db) { - BufferManager::GetBufferManager(*db).SetLimit(config.options.maximum_swap_space); - } } void MaximumTempDirectorySize::ResetGlobal(DatabaseInstance *db, DBConfig &config) { diff --git a/src/storage/temporary_file_manager.cpp b/src/storage/temporary_file_manager.cpp index 6a92048f5046..c97f73f197a1 100644 --- a/src/storage/temporary_file_manager.cpp +++ b/src/storage/temporary_file_manager.cpp @@ -259,7 +259,6 @@ TemporaryFileManager::TemporaryFileManager(DatabaseInstance &db, const string &t TemporaryFileManager::~TemporaryFileManager() { files.clear(); - D_ASSERT(size_on_disk.load() == 0); } TemporaryFileManager::TemporaryManagerLock::TemporaryManagerLock(mutex &mutex) : lock(mutex) { @@ -308,7 +307,21 @@ idx_t TemporaryFileManager::GetTotalUsedSpaceInBytes() { } void TemporaryFileManager::IncreaseSizeOnDisk(idx_t bytes) { + auto &config = DBConfig::GetConfig(db); + auto max_swap_space = config.options.maximum_swap_space; + + auto current_size_on_disk = size_on_disk.load(); size_on_disk += bytes; + if (size_on_disk.load() > max_swap_space) { + auto used = StringUtil::BytesToHumanReadableString(current_size_on_disk); + auto max = StringUtil::BytesToHumanReadableString(max_swap_space); + auto data_size = StringUtil::BytesToHumanReadableString(bytes); + throw OutOfMemoryException(R"(failed to offload data block of size %s (%s/%s used). +This limit was set by the 'max_temp_directory_size' setting. +This defaults to twice the size of 'max_memory'. +You can adjust this setting, by using (for example) PRAGMA max_temp_directory_size='10GiB')", + data_size, used, max); + } } void TemporaryFileManager::DecreaseSizeOnDisk(idx_t bytes) { From 7f17575bb9763ea79132184f349e6f4d490872f9 Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Mon, 4 Mar 2024 10:01:59 +0100 Subject: [PATCH 036/603] review comments. Extra tests --- src/optimizer/pushdown/pushdown_window.cpp | 6 +++--- .../pushdown/pushdown_window_partition_filter.test | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/optimizer/pushdown/pushdown_window.cpp b/src/optimizer/pushdown/pushdown_window.cpp index f3086e71a975..a27dd86151ad 100644 --- a/src/optimizer/pushdown/pushdown_window.cpp +++ b/src/optimizer/pushdown/pushdown_window.cpp @@ -54,14 +54,14 @@ unique_ptr FilterPushdown::PushdownWindow(unique_ptrtype) { // TODO: Add expressions for function expressions like FLOOR, CEIL etc. case ExpressionType::BOUND_COLUMN_REF: { auto &partition_col = partition_expr->Cast(); - parition_bindings.insert(partition_col.binding); + partition_bindings.insert(partition_col.binding); break; } default: @@ -77,7 +77,7 @@ unique_ptr FilterPushdown::PushdownWindow(unique_ptrfilter) && filters_to_pushdown[i] >= 0) { + if (FilterIsOnPartition(partition_bindings, *filter->filter) && filters_to_pushdown[i] >= 0) { filters_to_pushdown[i] = 1; continue; } diff --git a/test/optimizer/pushdown/pushdown_window_partition_filter.test b/test/optimizer/pushdown/pushdown_window_partition_filter.test index a96dae18729b..ff5b669fcd3a 100644 --- a/test/optimizer/pushdown/pushdown_window_partition_filter.test +++ b/test/optimizer/pushdown/pushdown_window_partition_filter.test @@ -60,7 +60,7 @@ A A 30 2 D C 110 5 D C 110 6 -# The filter is on the partitioned column, but is part of an AND conjunction, so we dont push it down +# One filter clause is on the partitioned column but the filter clause is an AND conjunction, so we don't push that down. query IIII select * from ( select a, c, sum(b) OVER (PARTITION BY c), rank() OVER (PARTITION BY c ORDER BY b) From 3c9143b1b8f919c6ade5fcc3ecf8badebcbf50ce Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Mon, 4 Mar 2024 10:04:34 +0100 Subject: [PATCH 037/603] and -> AND --- src/optimizer/pushdown/pushdown_window.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/optimizer/pushdown/pushdown_window.cpp b/src/optimizer/pushdown/pushdown_window.cpp index 989b747184c1..46e98ace3ea5 100644 --- a/src/optimizer/pushdown/pushdown_window.cpp +++ b/src/optimizer/pushdown/pushdown_window.cpp @@ -25,7 +25,7 @@ static bool FilterIsOnPartition(column_binding_set_t partition_bindings, Express return; } // for simplicity, we don't push down filters that are functions. - // conjunction and filters we don't pushdown either + // CONJUNCTION_AND filters are also not pushed down case ExpressionType::BOUND_FUNCTION: case ExpressionType::CONJUNCTION_AND: { filter_is_on_partition = false; From 7cecc4b983a9e1830abb8a77aee5bf7c97f5c11c Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 6 Mar 2024 11:02:10 +0100 Subject: [PATCH 038/603] All rejects tests pass with vector_size=2 --- .../duckdb/execution/operator/csv_scanner/csv_error.hpp | 2 +- test/sql/copy/csv/rejects/csv_buffer_size_rejects.test | 3 --- .../csv/rejects/csv_incorrect_columns_amount_rejects.test | 0 test/sql/copy/csv/rejects/csv_rejects_auto.test | 3 --- test/sql/copy/csv/rejects/csv_rejects_flush_cast.test | 5 ----- test/sql/copy/csv/rejects/csv_rejects_read.test | 4 ---- test/sql/copy/csv/rejects/test_invalid_parameters.test | 3 --- 7 files changed, 1 insertion(+), 19 deletions(-) create mode 100644 test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp index d8f755600e95..9331d62c34f5 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp @@ -43,7 +43,7 @@ enum CSVErrorType : uint8_t { UNTERMINATED_QUOTES = 3, // If a quote is not terminated SNIFFING = 4, // If something went wrong during sniffing and was not possible to find suitable candidates MAXIMUM_LINE_SIZE = 5, // Maximum line size was exceeded by a line in the CSV File - NULLPADDED_QUOTED_NEW_VALUE = 6, // If the null_padding option is set and we have quoted new values in parallel + NULLPADDED_QUOTED_NEW_VALUE = 6, // If the null_padding option is set, and we have quoted new values in parallel }; diff --git a/test/sql/copy/csv/rejects/csv_buffer_size_rejects.test b/test/sql/copy/csv/rejects/csv_buffer_size_rejects.test index 15461ee1c9f3..dcef91e814ee 100644 --- a/test/sql/copy/csv/rejects/csv_buffer_size_rejects.test +++ b/test/sql/copy/csv/rejects/csv_buffer_size_rejects.test @@ -4,9 +4,6 @@ require skip_reload -# FIXME: https://github.com/duckdb/duckdb/issues/7755 -require vector_size 2048 - loop buffer_size 5 10 # Ensure that we can get the schema if we reduce the sample size and ignore errors diff --git a/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test b/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test/sql/copy/csv/rejects/csv_rejects_auto.test b/test/sql/copy/csv/rejects/csv_rejects_auto.test index 887bd282db73..841ed42465f3 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_auto.test +++ b/test/sql/copy/csv/rejects/csv_rejects_auto.test @@ -3,9 +3,6 @@ require skip_reload -# FIXME: https://github.com/duckdb/duckdb/issues/7755 -require vector_size 2048 - # Ensure that we can get the schema if we reduce the sample size and ignore errors query IIIII SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto( diff --git a/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test b/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test index 87768b805d8a..6b2f5e59d7f5 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test +++ b/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test @@ -2,11 +2,6 @@ # description: Test that Flush Cast functions properly for the rejects tables # group: [rejects] -require skip_reload - -# FIXME: https://github.com/duckdb/duckdb/issues/7755 -require vector_size 2048 - query III SELECT typeof(first(a)), typeof(first(b)), COUNT(*) FROM read_csv( 'data/csv/error/flush_cast.csv', diff --git a/test/sql/copy/csv/rejects/csv_rejects_read.test b/test/sql/copy/csv/rejects/csv_rejects_read.test index ab5b12db949a..a0b2e751289d 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_read.test +++ b/test/sql/copy/csv/rejects/csv_rejects_read.test @@ -3,10 +3,6 @@ require skip_reload -# FIXME: https://github.com/duckdb/duckdb/issues/7755 -require vector_size 2048 - - query III rowsort SELECT * FROM read_csv( 'test/sql/copy/csv/data/error/mismatch/bad.csv', diff --git a/test/sql/copy/csv/rejects/test_invalid_parameters.test b/test/sql/copy/csv/rejects/test_invalid_parameters.test index 5209960fef88..2e343a30765d 100644 --- a/test/sql/copy/csv/rejects/test_invalid_parameters.test +++ b/test/sql/copy/csv/rejects/test_invalid_parameters.test @@ -3,9 +3,6 @@ require skip_reload -# FIXME: https://github.com/duckdb/duckdb/issues/7755 -require vector_size 2048 - # Test invalid arguments statement error SELECT * FROM read_csv( From 8222dd4f0888703e1ad2883671087e5e0976d239 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 6 Mar 2024 13:32:28 +0100 Subject: [PATCH 039/603] WIP on column amount incorrect --- .../rejects/incorrect_columns/few_columns.csv | 2773 +++++++++++++++++ .../incorrect_columns/many_columns.csv | 2773 +++++++++++++++++ .../rejects/incorrect_columns/mix_columns.csv | 2773 +++++++++++++++++ .../scanner/string_value_scanner.cpp | 4 +- .../table_function/global_csv_state.cpp | 36 +- .../operator/csv_scanner/util/csv_error.cpp | 12 +- .../operator/csv_scanner/csv_error.hpp | 13 +- .../csv_incorrect_columns_amount_rejects.test | 22 + 8 files changed, 8392 insertions(+), 14 deletions(-) create mode 100644 data/csv/rejects/incorrect_columns/few_columns.csv create mode 100644 data/csv/rejects/incorrect_columns/many_columns.csv create mode 100644 data/csv/rejects/incorrect_columns/mix_columns.csv diff --git a/data/csv/rejects/incorrect_columns/few_columns.csv b/data/csv/rejects/incorrect_columns/few_columns.csv new file mode 100644 index 000000000000..9b9d66a642d3 --- /dev/null +++ b/data/csv/rejects/incorrect_columns/few_columns.csv @@ -0,0 +1,2773 @@ +a,b,c,d +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 diff --git a/data/csv/rejects/incorrect_columns/many_columns.csv b/data/csv/rejects/incorrect_columns/many_columns.csv new file mode 100644 index 000000000000..e8611f730f05 --- /dev/null +++ b/data/csv/rejects/incorrect_columns/many_columns.csv @@ -0,0 +1,2773 @@ +a,b,c,d +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4,5,6 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4,5,6 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4,5 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4,5 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 diff --git a/data/csv/rejects/incorrect_columns/mix_columns.csv b/data/csv/rejects/incorrect_columns/mix_columns.csv new file mode 100644 index 000000000000..1217d34524cd --- /dev/null +++ b/data/csv/rejects/incorrect_columns/mix_columns.csv @@ -0,0 +1,2773 @@ +a,b,c,d +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4,5,6 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4,5,6 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 +1,2,3,4 diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 1d8c1ef2864e..a1ef8c54448d 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -308,7 +308,7 @@ void StringValueResult::AddValue(StringValueResult &result, const idx_t buffer_p } void StringValueResult::HandleOverLimitRows() { - LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), number_of_rows + 1); + LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read + 1); bool first_nl; auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); auto csv_error = @@ -446,7 +446,7 @@ bool StringValueResult::AddRowInternal() { // If we are not null-padding this is an error bool first_nl; auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); - LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), number_of_rows + 1); + LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read + 1); auto csv_error = CSVError::IncorrectColumnAmountError( state_machine.options, cur_col_id, lines_per_batch, borked_line, current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index 77982d83fab2..73a3a2fe420c 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -137,6 +137,36 @@ void CSVGlobalState::DecrementThread() { } } +bool IsCSVErrorAcceptedReject(CSVErrorType type) { + switch (type) { + case CSVErrorType::CAST_ERROR: + case CSVErrorType::TOO_MANY_COLUMNS: + case CSVErrorType::TOO_FEW_COLUMNS: + case CSVErrorType::MAXIMUM_LINE_SIZE: + case CSVErrorType::UNTERMINATED_QUOTES: + return true; + default: + return false; + } +} + +string CSVErrorTypeToEnum(CSVErrorType type) { + switch (type) { + case CSVErrorType::CAST_ERROR: + return "CAST"; + case CSVErrorType::TOO_FEW_COLUMNS: + return "MISSING COLUMNS"; + case CSVErrorType::TOO_MANY_COLUMNS: + return "TOO MANY COLUMNS"; + case CSVErrorType::MAXIMUM_LINE_SIZE: + return "LINE SIZE OVER MAXIMUM"; + case CSVErrorType::UNTERMINATED_QUOTES: + return "UNQUOTED VALUE"; + default: + throw InternalException("CSV Error is not valid to be stored in a Rejects Table"); + } +} + void CSVGlobalState::FillRejectsTable() { auto &options = bind_data.options; @@ -152,7 +182,7 @@ void CSVGlobalState::FillRejectsTable() { auto &errors = file->error_handler->errors; for (auto &error_vector : errors) { for (auto &error : error_vector.second) { - if (error.type != CSVErrorType::CAST_ERROR) { + if (!IsCSVErrorAcceptedReject(error.type)) { // For now, we only will use it for casting errors continue; } @@ -177,8 +207,8 @@ void CSVGlobalState::FillRejectsTable() { appender.Append(col_idx); // 5. Column Name (If Applicable) appender.Append(string_t("\"" + col_name + "\"")); - // 6. Error Type (ENUM?) - appender.Append(string_t("CAST")); + // 6. Error Type + appender.Append(string_t(CSVErrorTypeToEnum(error.type))); // 7. Original CSV Line appender.Append(string_t(error.csv_row)); // 8. Full Error Message diff --git a/src/execution/operator/csv_scanner/util/csv_error.cpp b/src/execution/operator/csv_scanner/util/csv_error.cpp index 3174ff2d7723..af7d45864812 100644 --- a/src/execution/operator/csv_scanner/util/csv_error.cpp +++ b/src/execution/operator/csv_scanner/util/csv_error.cpp @@ -1,5 +1,6 @@ #include "duckdb/execution/operator/csv_scanner/csv_error.hpp" #include "duckdb/common/exception/conversion_exception.hpp" + #include namespace duckdb { @@ -160,8 +161,12 @@ CSVError CSVError::IncorrectColumnAmountError(const CSVReaderOptions &options, i << std::endl; // What were the options error << options.ToString(); - return CSVError(error.str(), CSVErrorType::INCORRECT_COLUMN_AMOUNT, actual_columns, csv_row, error_info, - byte_position); + if (actual_columns > options.dialect_options.num_cols) { + return CSVError(error.str(), CSVErrorType::TOO_MANY_COLUMNS, actual_columns, csv_row, error_info, + byte_position); + } else { + return CSVError(error.str(), CSVErrorType::TOO_FEW_COLUMNS, actual_columns, csv_row, error_info, byte_position); + } } bool CSVErrorHandler::PrintLineNumber(CSVError &error) { @@ -171,7 +176,8 @@ bool CSVErrorHandler::PrintLineNumber(CSVError &error) { switch (error.type) { case CSVErrorType::CAST_ERROR: case CSVErrorType::UNTERMINATED_QUOTES: - case CSVErrorType::INCORRECT_COLUMN_AMOUNT: + case CSVErrorType::TOO_FEW_COLUMNS: + case CSVErrorType::TOO_MANY_COLUMNS: case CSVErrorType::MAXIMUM_LINE_SIZE: case CSVErrorType::NULLPADDED_QUOTED_NEW_VALUE: return true; diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp index 9331d62c34f5..95436017f2a1 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp @@ -39,11 +39,12 @@ class LinesPerBoundary { enum CSVErrorType : uint8_t { CAST_ERROR = 0, // If when casting a value from string to the column type fails COLUMN_NAME_TYPE_MISMATCH = 1, // If there is a mismatch between Column Names and Types - INCORRECT_COLUMN_AMOUNT = 2, // If the CSV is missing a column - UNTERMINATED_QUOTES = 3, // If a quote is not terminated - SNIFFING = 4, // If something went wrong during sniffing and was not possible to find suitable candidates - MAXIMUM_LINE_SIZE = 5, // Maximum line size was exceeded by a line in the CSV File - NULLPADDED_QUOTED_NEW_VALUE = 6, // If the null_padding option is set, and we have quoted new values in parallel + TOO_FEW_COLUMNS = 2, // If the CSV has too few columns + TOO_MANY_COLUMNS = 3, // If the CSV has too many column + UNTERMINATED_QUOTES = 4, // If a quote is not terminated + SNIFFING = 5, // If something went wrong during sniffing and was not possible to find suitable candidates + MAXIMUM_LINE_SIZE = 6, // Maximum line size was exceeded by a line in the CSV File + NULLPADDED_QUOTED_NEW_VALUE = 7, // If the null_padding option is set, and we have quoted new values in parallel }; @@ -69,7 +70,7 @@ class CSVError { //! Produces error messages for null_padding option is set and we have quoted new values in parallel static CSVError NullPaddingFail(const CSVReaderOptions &options, LinesPerBoundary error_info); //! Produces error for incorrect (e.g., smaller and lower than the predefined) number of columns in a CSV Line - static CSVError IncorrectColumnAmountError(const CSVReaderOptions &options, idx_t actual_columns, + static CSVError IncorrectColumnAmountError(const CSVReaderOptions &state_machine, idx_t actual_columns, LinesPerBoundary error_info, string &csv_row, idx_t byte_position); idx_t GetBoundaryIndex() { return error_info.boundary_idx; diff --git a/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test b/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test index e69de29bb2d1..7f899e860de0 100644 --- a/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test +++ b/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test @@ -0,0 +1,22 @@ +# name: test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test +# description: Test that incorrect column amounts return correct info as rejects tables +# group: [rejects] + +require skip_reload + + +statement ok +SELECT * FROM read_csv( + 'data/csv/rejects/incorrect_columns/few_columns.csv', + columns = {'a': 'INTEGER', 'b': 'INTEGER', 'c': 'INTEGER', 'd': 'INTEGER'}, + rejects_table='csv_rejects_table', + ignore_errors=true, auto_detect=false, header = 1); + +query IIIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position +FROM csv_rejects_table; +---- +data/csv/rejects/incorrect_columns/few_columns.csv 1816 3 "d" MISSING COLUMNS 1,2,3 14504 +data/csv/rejects/incorrect_columns/few_columns.csv 1825 1 "b" MISSING COLUMNS 1 14574 +data/csv/rejects/incorrect_columns/few_columns.csv 2380 1 "b" MISSING COLUMNS 1 19008 +data/csv/rejects/incorrect_columns/few_columns.csv 2764 2 "c" MISSING COLUMNS 1,2 22074 \ No newline at end of file From d9ffcb922d214dd366f046517e81027dfacfd8c3 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 6 Mar 2024 13:42:03 +0100 Subject: [PATCH 040/603] remainnig merge --- src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp index 1427b1e840c4..82160a8490d8 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp @@ -74,7 +74,6 @@ class CSVError { LinesPerBoundary error_info, string &csv_row, idx_t byte_position); static CSVError InvalidUTF8(const CSVReaderOptions &options, LinesPerBoundary error_info); - idx_t GetBoundaryIndex() { return error_info.boundary_idx; } From d54bf5b7c590803ea5707881325b9fc205ae4694 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 6 Mar 2024 13:47:22 +0100 Subject: [PATCH 041/603] line error fix --- .../operator/csv_scanner/scanner/string_value_scanner.cpp | 4 ++-- .../csv/rejects/csv_incorrect_columns_amount_rejects.test | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 6a7586ee5b4e..4d50b2ec7a8e 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -325,7 +325,7 @@ void StringValueResult::AddValue(StringValueResult &result, const idx_t buffer_p } void StringValueResult::HandleOverLimitRows() { - LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read + 1); + LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); bool first_nl; auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); auto csv_error = @@ -465,7 +465,7 @@ bool StringValueResult::AddRowInternal() { // If we are not null-padding this is an error bool first_nl; auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); - LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read + 1); + LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); auto csv_error = CSVError::IncorrectColumnAmountError( state_machine.options, cur_col_id, lines_per_batch, borked_line, current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); diff --git a/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test b/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test index 7f899e860de0..b7d50fca8b7a 100644 --- a/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test +++ b/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test @@ -16,7 +16,7 @@ query IIIIIII rowsort SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table; ---- -data/csv/rejects/incorrect_columns/few_columns.csv 1816 3 "d" MISSING COLUMNS 1,2,3 14504 -data/csv/rejects/incorrect_columns/few_columns.csv 1825 1 "b" MISSING COLUMNS 1 14574 -data/csv/rejects/incorrect_columns/few_columns.csv 2380 1 "b" MISSING COLUMNS 1 19008 -data/csv/rejects/incorrect_columns/few_columns.csv 2764 2 "c" MISSING COLUMNS 1,2 22074 \ No newline at end of file +data/csv/rejects/incorrect_columns/few_columns.csv 1814 3 "d" MISSING COLUMNS 1,2,3 14504 +data/csv/rejects/incorrect_columns/few_columns.csv 1823 1 "b" MISSING COLUMNS 1 14574 +data/csv/rejects/incorrect_columns/few_columns.csv 2378 1 "b" MISSING COLUMNS 1 19008 +data/csv/rejects/incorrect_columns/few_columns.csv 2762 2 "c" MISSING COLUMNS 1,2 22074 \ No newline at end of file From 67703ebf6062b66969d8d8794f8622358c2f3cd0 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 6 Mar 2024 14:04:45 +0100 Subject: [PATCH 042/603] Get information on too many columns right --- .../scanner/string_value_scanner.cpp | 16 ++++++++------ .../table_function/global_csv_state.cpp | 14 +++++++++--- .../csv_incorrect_columns_amount_rejects.test | 22 +++++++++++++++++-- 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 4d50b2ec7a8e..caf511b4a7c4 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -111,6 +111,10 @@ inline bool IsValueNull(const char *null_str_ptr, const char *value_ptr, const i } void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size, bool allocate) { + if (ignore_current_row) { + cur_col_id++; + return; + } if (cur_col_id >= number_of_columns) { bool error = true; if (cur_col_id == number_of_columns && ((quoted && state_machine.options.allow_quoted_nulls) || !quoted)) { @@ -118,12 +122,11 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size error = !IsValueNull(null_str_ptr, value_ptr, size); } if (error) { - HandleOverLimitRows(); + ignore_current_row = true; } - } - if (ignore_current_row) { return; } + if (projecting_columns) { if (!projected_columns[cur_col_id]) { cur_col_id++; @@ -332,10 +335,6 @@ void StringValueResult::HandleOverLimitRows() { CSVError::IncorrectColumnAmountError(state_machine.options, cur_col_id + 1, lines_per_batch, borked_line, current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); error_handler.Error(csv_error); - // If we get here we need to remove the last line - cur_col_id = 0; - chunk_col_id = 0; - ignore_current_row = true; } void StringValueResult::QuotedNewLine(StringValueResult &result) { @@ -405,6 +404,9 @@ bool StringValueResult::AddRowInternal() { current_line_position.begin = current_line_position.end; current_line_position.end = current_line_start; if (ignore_current_row) { + if (cur_col_id >= number_of_columns) { + HandleOverLimitRows(); + } cur_col_id = 0; chunk_col_id = 0; // An error occurred on this row, we are ignoring it and resetting our control flag diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index 8e5bc26f411c..e412e21f8c4c 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -194,7 +194,11 @@ void CSVGlobalState::FillRejectsTable() { rejects->count++; auto row_line = file->error_handler->GetLine(error.error_info); auto col_idx = error.column_idx; - auto col_name = bind_data.return_names[col_idx]; + string col_name; + if (error.type != CSVErrorType::TOO_MANY_COLUMNS){ + // Too many columns does not have a name, all other errors have + col_name = bind_data.return_names[col_idx]; + } // Add the row to the rejects table appender.BeginRow(); // 1. File Path @@ -203,10 +207,14 @@ void CSVGlobalState::FillRejectsTable() { appender.Append(row_line); // 3. Byte Position where error occurred appender.Append(error.byte_position); - // 4. Column Index (If Applicable) + // 4. Column Index appender.Append(col_idx); // 5. Column Name (If Applicable) - appender.Append(string_t("\"" + col_name + "\"")); + if (col_name.empty()){ + appender.Append(Value()); + } else { + appender.Append(string_t("\"" + col_name + "\"")); + } // 6. Error Type appender.Append(string_t(CSVErrorTypeToEnum(error.type))); // 7. Original CSV Line diff --git a/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test b/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test index b7d50fca8b7a..f0ea6c9eda57 100644 --- a/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test +++ b/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test @@ -4,7 +4,6 @@ require skip_reload - statement ok SELECT * FROM read_csv( 'data/csv/rejects/incorrect_columns/few_columns.csv', @@ -19,4 +18,23 @@ FROM csv_rejects_table; data/csv/rejects/incorrect_columns/few_columns.csv 1814 3 "d" MISSING COLUMNS 1,2,3 14504 data/csv/rejects/incorrect_columns/few_columns.csv 1823 1 "b" MISSING COLUMNS 1 14574 data/csv/rejects/incorrect_columns/few_columns.csv 2378 1 "b" MISSING COLUMNS 1 19008 -data/csv/rejects/incorrect_columns/few_columns.csv 2762 2 "c" MISSING COLUMNS 1,2 22074 \ No newline at end of file +data/csv/rejects/incorrect_columns/few_columns.csv 2762 2 "c" MISSING COLUMNS 1,2 22074 + +statement ok +DROP TABLE csv_rejects_table; + +statement ok +SELECT * FROM read_csv( + 'data/csv/rejects/incorrect_columns/many_columns.csv', + columns = {'a': 'INTEGER', 'b': 'INTEGER', 'c': 'INTEGER', 'd': 'INTEGER'}, + rejects_table='csv_rejects_table', + ignore_errors=true, auto_detect=false, header = 1); + +query IIIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position +FROM csv_rejects_table; +---- +data/csv/rejects/incorrect_columns/many_columns.csv 1096 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 8760 +data/csv/rejects/incorrect_columns/many_columns.csv 1159 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 9268 +data/csv/rejects/incorrect_columns/many_columns.csv 1206 5 NULL TOO MANY COLUMNS 1,2,3,4,5 9648 +data/csv/rejects/incorrect_columns/many_columns.csv 2769 5 NULL TOO MANY COLUMNS 1,2,3,4,5 22154 \ No newline at end of file From 12542a2cdbeed1bfbd868802bcafa6979db521f1 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 6 Mar 2024 14:10:52 +0100 Subject: [PATCH 043/603] More tests for different incorrect column amounts --- .../rejects/incorrect_columns/small_mix.csv | 5 ++ .../csv_incorrect_columns_amount_rejects.test | 73 ++++++++++++++++++- 2 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 data/csv/rejects/incorrect_columns/small_mix.csv diff --git a/data/csv/rejects/incorrect_columns/small_mix.csv b/data/csv/rejects/incorrect_columns/small_mix.csv new file mode 100644 index 000000000000..1cfae5653bcd --- /dev/null +++ b/data/csv/rejects/incorrect_columns/small_mix.csv @@ -0,0 +1,5 @@ +a,b,c,d +1,2,3,4 +1,2,3,4,5 +1,2,3 +1,2,3,4 \ No newline at end of file diff --git a/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test b/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test index f0ea6c9eda57..414e9fdd8594 100644 --- a/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test +++ b/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test @@ -37,4 +37,75 @@ FROM csv_rejects_table; data/csv/rejects/incorrect_columns/many_columns.csv 1096 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 8760 data/csv/rejects/incorrect_columns/many_columns.csv 1159 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 9268 data/csv/rejects/incorrect_columns/many_columns.csv 1206 5 NULL TOO MANY COLUMNS 1,2,3,4,5 9648 -data/csv/rejects/incorrect_columns/many_columns.csv 2769 5 NULL TOO MANY COLUMNS 1,2,3,4,5 22154 \ No newline at end of file +data/csv/rejects/incorrect_columns/many_columns.csv 2769 5 NULL TOO MANY COLUMNS 1,2,3,4,5 22154 + +statement ok +DROP TABLE csv_rejects_table; + +statement ok +SELECT * FROM read_csv( + 'data/csv/rejects/incorrect_columns/mix_columns.csv', + columns = {'a': 'INTEGER', 'b': 'INTEGER', 'c': 'INTEGER', 'd': 'INTEGER'}, + rejects_table='csv_rejects_table', + ignore_errors=true, auto_detect=false, header = 1); + +query IIIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position +FROM csv_rejects_table; +---- +data/csv/rejects/incorrect_columns/mix_columns.csv 1604 1 "b" MISSING COLUMNS 1 12824 +data/csv/rejects/incorrect_columns/mix_columns.csv 1671 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 13354 +data/csv/rejects/incorrect_columns/mix_columns.csv 2751 2 "c" MISSING COLUMNS 1,2 21998 +data/csv/rejects/incorrect_columns/mix_columns.csv 2768 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 22130 + +# Different Buffer Sizes +loop buffer_size 10 15 + +statement ok +DROP TABLE csv_rejects_table; + +statement ok +SELECT * FROM read_csv( + 'data/csv/rejects/incorrect_columns/small_mix.csv', + columns = {'a': 'INTEGER', 'b': 'INTEGER', 'c': 'INTEGER', 'd': 'INTEGER'}, + rejects_table='csv_rejects_table', + ignore_errors=true, auto_detect=false, header = 1); + +query IIIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position +FROM csv_rejects_table; +---- +data/csv/rejects/incorrect_columns/small_mix.csv 3 5 NULL TOO MANY COLUMNS 1,2,3,4,5 16 +data/csv/rejects/incorrect_columns/small_mix.csv 4 3 "d" MISSING COLUMNS 1,2,3 26 + +endloop + +# All files +statement ok +DROP TABLE csv_rejects_table; + +statement ok +SELECT * FROM read_csv( + 'data/csv/rejects/incorrect_columns/*.csv', + columns = {'a': 'INTEGER', 'b': 'INTEGER', 'c': 'INTEGER', 'd': 'INTEGER'}, + rejects_table='csv_rejects_table', + ignore_errors=true, auto_detect=false, header = 1); + +query IIIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position +FROM csv_rejects_table; +---- +data/csv/rejects/incorrect_columns/few_columns.csv 1814 3 "d" MISSING COLUMNS 1,2,3 14504 +data/csv/rejects/incorrect_columns/few_columns.csv 1823 1 "b" MISSING COLUMNS 1 14574 +data/csv/rejects/incorrect_columns/few_columns.csv 2378 1 "b" MISSING COLUMNS 1 19008 +data/csv/rejects/incorrect_columns/few_columns.csv 2762 2 "c" MISSING COLUMNS 1,2 22074 +data/csv/rejects/incorrect_columns/many_columns.csv 1096 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 8760 +data/csv/rejects/incorrect_columns/many_columns.csv 1159 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 9268 +data/csv/rejects/incorrect_columns/many_columns.csv 1206 5 NULL TOO MANY COLUMNS 1,2,3,4,5 9648 +data/csv/rejects/incorrect_columns/many_columns.csv 2769 5 NULL TOO MANY COLUMNS 1,2,3,4,5 22154 +data/csv/rejects/incorrect_columns/mix_columns.csv 1604 1 "b" MISSING COLUMNS 1 12824 +data/csv/rejects/incorrect_columns/mix_columns.csv 1671 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 13354 +data/csv/rejects/incorrect_columns/mix_columns.csv 2751 2 "c" MISSING COLUMNS 1,2 21998 +data/csv/rejects/incorrect_columns/mix_columns.csv 2768 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 22130 +data/csv/rejects/incorrect_columns/small_mix.csv 3 5 NULL TOO MANY COLUMNS 1,2,3,4,5 16 +data/csv/rejects/incorrect_columns/small_mix.csv 4 3 "d" MISSING COLUMNS 1,2,3 26 From 8ea5f0a7d0255a3317a2e2152c29f0abaf42031b Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 6 Mar 2024 16:56:57 +0100 Subject: [PATCH 044/603] WIP on sanitizing invalid utfs and more on utf rejects tables --- .../scanner/string_value_scanner.cpp | 45 ++++++++++++++----- .../table_function/global_csv_state.cpp | 7 ++- .../operator/csv_scanner/util/csv_error.cpp | 5 ++- .../operator/persistent/csv_rejects_table.cpp | 3 +- .../operator/csv_scanner/csv_error.hpp | 3 +- .../csv_scanner/string_value_scanner.hpp | 16 ++++++- .../csv/rejects/test_invalid_utf_rejects.test | 15 +++++++ .../utf8proc/include/utf8proc_wrapper.hpp | 2 + third_party/utf8proc/utf8proc_wrapper.cpp | 33 ++++++++++++++ 9 files changed, 109 insertions(+), 20 deletions(-) create mode 100644 test/sql/copy/csv/rejects/test_invalid_utf_rejects.test diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index caf511b4a7c4..a87f43917e1b 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -111,7 +111,7 @@ inline bool IsValueNull(const char *null_str_ptr, const char *value_ptr, const i } void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size, bool allocate) { - if (ignore_current_row) { + if (current_error.is_set) { cur_col_id++; return; } @@ -122,7 +122,7 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size error = !IsValueNull(null_str_ptr, value_ptr, size); } if (error) { - ignore_current_row = true; + current_error = {CSVErrorType::TOO_MANY_COLUMNS}; } return; } @@ -221,11 +221,11 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size if (parse_types[chunk_col_id].second && !Utf8Proc::IsValid(value_ptr, UnsafeNumericCast(size))) { bool force_error = !state_machine.options.ignore_errors && sniffing; // Invalid unicode, we must error - LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); - auto csv_error = CSVError::InvalidUTF8(state_machine.options, lines_per_batch); - error_handler.Error(csv_error, force_error); + if (force_error) { + HandleUnicodeError(force_error); + } // If we got here, we are ingoring errors, hence we must ignore this line. - ignore_current_row = true; + current_error = {CSVErrorType::INVALID_UNICODE}; break; } if (allocate) { @@ -282,7 +282,7 @@ void StringValueResult::Reset() { if (cur_buffer) { buffer_handles[cur_buffer->buffer_idx] = cur_buffer; } - ignore_current_row = false; + current_error.Reset(); } void StringValueResult::AddQuotedValue(StringValueResult &result, const idx_t buffer_pos) { @@ -337,6 +337,20 @@ void StringValueResult::HandleOverLimitRows() { error_handler.Error(csv_error); } +void StringValueResult::HandleUnicodeError(bool force_error) { + bool first_nl; + auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); + // sanitize borked line + std::vector charArray(borked_line.begin(), borked_line.end()); + charArray.push_back('\0'); // Null-terminate the character array + Utf8Proc::MakeValid(&charArray[0], charArray.size()); + borked_line = {charArray.begin(), charArray.end() - 1}; + LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); + auto csv_error = CSVError::InvalidUTF8(state_machine.options, cur_col_id - 1, lines_per_batch, borked_line, + current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); + error_handler.Error(csv_error, force_error); +} + void StringValueResult::QuotedNewLine(StringValueResult &result) { result.quoted_new_line = true; } @@ -403,14 +417,21 @@ bool StringValueResult::AddRowInternal() { } current_line_position.begin = current_line_position.end; current_line_position.end = current_line_start; - if (ignore_current_row) { - if (cur_col_id >= number_of_columns) { + if (current_error.is_set) { + switch (current_error.type) { + case CSVErrorType::TOO_MANY_COLUMNS: HandleOverLimitRows(); + break; + case CSVErrorType::INVALID_UNICODE: + HandleUnicodeError(); + break; + default: + InvalidInputException("CSV Error not allowed when inserting row"); } cur_col_id = 0; chunk_col_id = 0; // An error occurred on this row, we are ignoring it and resetting our control flag - ignore_current_row = false; + current_error.Reset(); return false; } if (!cast_errors.empty()) { @@ -766,14 +787,14 @@ void StringValueScanner::ProcessExtraRow() { return; case CSVState::RECORD_SEPARATOR: if (states.states[0] == CSVState::RECORD_SEPARATOR) { - lines_read++; result.EmptyLine(result, iterator.pos.buffer_pos); iterator.pos.buffer_pos++; + lines_read++; return; } else if (states.states[0] != CSVState::CARRIAGE_RETURN) { - lines_read++; result.AddRow(result, iterator.pos.buffer_pos); iterator.pos.buffer_pos++; + lines_read++; return; } lines_read++; diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index e412e21f8c4c..6380d2e24c6c 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -144,6 +144,7 @@ bool IsCSVErrorAcceptedReject(CSVErrorType type) { case CSVErrorType::TOO_FEW_COLUMNS: case CSVErrorType::MAXIMUM_LINE_SIZE: case CSVErrorType::UNTERMINATED_QUOTES: + case CSVErrorType::INVALID_UNICODE: return true; default: return false; @@ -162,6 +163,8 @@ string CSVErrorTypeToEnum(CSVErrorType type) { return "LINE SIZE OVER MAXIMUM"; case CSVErrorType::UNTERMINATED_QUOTES: return "UNQUOTED VALUE"; + case CSVErrorType::INVALID_UNICODE: + return "INVALID UNICODE"; default: throw InternalException("CSV Error is not valid to be stored in a Rejects Table"); } @@ -195,7 +198,7 @@ void CSVGlobalState::FillRejectsTable() { auto row_line = file->error_handler->GetLine(error.error_info); auto col_idx = error.column_idx; string col_name; - if (error.type != CSVErrorType::TOO_MANY_COLUMNS){ + if (error.type != CSVErrorType::TOO_MANY_COLUMNS) { // Too many columns does not have a name, all other errors have col_name = bind_data.return_names[col_idx]; } @@ -210,7 +213,7 @@ void CSVGlobalState::FillRejectsTable() { // 4. Column Index appender.Append(col_idx); // 5. Column Name (If Applicable) - if (col_name.empty()){ + if (col_name.empty()) { appender.Append(Value()); } else { appender.Append(string_t("\"" + col_name + "\"")); diff --git a/src/execution/operator/csv_scanner/util/csv_error.cpp b/src/execution/operator/csv_scanner/util/csv_error.cpp index 217bf0ac9fec..9c1eaffd394f 100644 --- a/src/execution/operator/csv_scanner/util/csv_error.cpp +++ b/src/execution/operator/csv_scanner/util/csv_error.cpp @@ -169,13 +169,14 @@ CSVError CSVError::IncorrectColumnAmountError(const CSVReaderOptions &options, i } } -CSVError CSVError::InvalidUTF8(const CSVReaderOptions &options, LinesPerBoundary error_info) { +CSVError CSVError::InvalidUTF8(const CSVReaderOptions &options, idx_t current_column, LinesPerBoundary error_info, + string &csv_row, idx_t byte_position) { std::ostringstream error; // How many columns were expected and how many were found error << "Invalid unicode (byte sequence mismatch) detected." << std::endl; // What were the options error << options.ToString(); - return CSVError(error.str(), CSVErrorType::INVALID_UNICODE, error_info); + return CSVError(error.str(), CSVErrorType::INVALID_UNICODE, current_column, csv_row, error_info, byte_position); } bool CSVErrorHandler::PrintLineNumber(CSVError &error) { diff --git a/src/execution/operator/persistent/csv_rejects_table.cpp b/src/execution/operator/persistent/csv_rejects_table.cpp index 3f2acf553f21..4140b5d3eac5 100644 --- a/src/execution/operator/persistent/csv_rejects_table.cpp +++ b/src/execution/operator/persistent/csv_rejects_table.cpp @@ -31,7 +31,8 @@ void CSVRejectsTable::InitializeTable(ClientContext &context, const ReadCSVData order_errors.SetValue(2, "TOO MANY COLUMNS"); order_errors.SetValue(3, "UNQUOTED VALUE"); order_errors.SetValue(4, "LINE SIZE OVER MAXIMUM"); - LogicalType enum_type = LogicalType::ENUM(enum_name, order_errors, 5); + order_errors.SetValue(5, "INVALID UNICODE"); + LogicalType enum_type = LogicalType::ENUM(enum_name, order_errors, 6); auto type_info = make_uniq(enum_name, enum_type); type_info->temporary = true; type_info->on_conflict = OnCreateConflict::IGNORE_ON_CONFLICT; diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp index 82160a8490d8..3ced0619aa67 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp @@ -72,7 +72,8 @@ class CSVError { //! Produces error for incorrect (e.g., smaller and lower than the predefined) number of columns in a CSV Line static CSVError IncorrectColumnAmountError(const CSVReaderOptions &state_machine, idx_t actual_columns, LinesPerBoundary error_info, string &csv_row, idx_t byte_position); - static CSVError InvalidUTF8(const CSVReaderOptions &options, LinesPerBoundary error_info); + static CSVError InvalidUTF8(const CSVReaderOptions &options, idx_t current_column, LinesPerBoundary error_info, + string &csv_row, idx_t byte_position); idx_t GetBoundaryIndex() { return error_info.boundary_idx; diff --git a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp index d88fc76e7390..18d1776b7a71 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp @@ -62,6 +62,17 @@ class FullLinePosition { unordered_map> &buffer_handles); }; +class CurrentError { +public: + CurrentError() : is_set(false) {}; + CurrentError(CSVErrorType type) : is_set(true), type(type) {}; + void Reset() { + is_set = false; + } + bool is_set; + CSVErrorType type; +}; + class StringValueResult : public ScannerResult { public: StringValueResult(CSVStates &states, CSVStateMachine &state_machine, @@ -120,8 +131,8 @@ class StringValueResult : public ScannerResult { //! Requested size of buffers (i.e., either 32Mb or set by buffer_size parameter) idx_t requested_size; - //! If the current row has an error, we have to skip it - bool ignore_current_row = false; + //! Current Error if any + CurrentError current_error; bool sniffing; //! Specialized code for quoted values, makes sure to remove quotes and escapes @@ -140,6 +151,7 @@ class StringValueResult : public ScannerResult { inline bool AddRowInternal(); void HandleOverLimitRows(); + void HandleUnicodeError(bool force_error = false); inline void AddValueToVector(const char *value_ptr, const idx_t size, bool allocate = false); diff --git a/test/sql/copy/csv/rejects/test_invalid_utf_rejects.test b/test/sql/copy/csv/rejects/test_invalid_utf_rejects.test new file mode 100644 index 000000000000..bcf1b6f64ecb --- /dev/null +++ b/test/sql/copy/csv/rejects/test_invalid_utf_rejects.test @@ -0,0 +1,15 @@ +# name: test/sql/copy/csv/rejects/test_invalid_utf_rejects.test +# description: Test that invalid unicodes return correct info as rejects tables +# group: [rejects] + +require skip_reload + +statement ok +from read_csv('test/sql/copy/csv/data/test/invalid_utf_big.csv',columns = {'col1': 'VARCHAR','col2': 'VARCHAR','col3': 'VARCHAR'}, + auto_detect=false, rejects_table='csv_rejects_table', header = 0, delim = ',', ignore_errors=true) + +query IIIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position +FROM csv_rejects_table; +---- + diff --git a/third_party/utf8proc/include/utf8proc_wrapper.hpp b/third_party/utf8proc/include/utf8proc_wrapper.hpp index fb988b254b76..a7fb8c662b24 100644 --- a/third_party/utf8proc/include/utf8proc_wrapper.hpp +++ b/third_party/utf8proc/include/utf8proc_wrapper.hpp @@ -26,6 +26,8 @@ class Utf8Proc { static char* Normalize(const char* s, size_t len); //! Returns whether or not the UTF8 string is valid static bool IsValid(const char *s, size_t len); + //! Makes Invalid Unicode valid by replacing invalid parts with a given character + static void MakeValid(char *s, size_t len, char special_flag = '?'); //! Returns the position (in bytes) of the next grapheme cluster static size_t NextGraphemeCluster(const char *s, size_t len, size_t pos); //! Returns the position (in bytes) of the previous grapheme cluster diff --git a/third_party/utf8proc/utf8proc_wrapper.cpp b/third_party/utf8proc/utf8proc_wrapper.cpp index c47472a39eb6..02f6c0efc5de 100644 --- a/third_party/utf8proc/utf8proc_wrapper.cpp +++ b/third_party/utf8proc/utf8proc_wrapper.cpp @@ -102,6 +102,39 @@ UnicodeType Utf8Proc::Analyze(const char *s, size_t len, UnicodeInvalidReason *i return type; } +void Utf8Proc::MakeValid(char *s, size_t len, char special_flag){ + UnicodeType type = UnicodeType::ASCII; + for (size_t i = 0; i < len; i++) { + int c = (int) s[i]; + if ((c & 0x80) == 0) { + continue; + } + int first_pos_seq = i; + if ((c & 0xE0) == 0xC0) { + /* 2 byte sequence */ + int utf8char = c & 0x1F; + type = UTF8ExtraByteLoop<1, 0x000780>(first_pos_seq, utf8char, i, s, len, nullptr, nullptr); + } else if ((c & 0xF0) == 0xE0) { + /* 3 byte sequence */ + int utf8char = c & 0x0F; + type = UTF8ExtraByteLoop<2, 0x00F800>(first_pos_seq, utf8char, i, s, len, nullptr, nullptr); + } else if ((c & 0xF8) == 0xF0) { + /* 4 byte sequence */ + int utf8char = c & 0x07; + type = UTF8ExtraByteLoop<3, 0x1F0000>(first_pos_seq, utf8char, i, s, len, nullptr, nullptr); + } else { + /* invalid UTF-8 start byte */ + s[i] = special_flag; // Rewrite invalid byte + } + if (type == UnicodeType::INVALID) { + for (size_t j = first_pos_seq; j <= i; j++) { + s[j] = special_flag; // Rewrite each byte of the invalid sequence + } + type = UnicodeType::ASCII; + } + } +} + char* Utf8Proc::Normalize(const char *s, size_t len) { assert(s); assert(Utf8Proc::Analyze(s, len) != UnicodeType::INVALID); From 4ad73e518e0f945bf2760712d6cc91a009024e58 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 6 Mar 2024 18:41:45 +0100 Subject: [PATCH 045/603] invalid utf working --- src/execution/operator/persistent/csv_rejects_table.cpp | 2 +- test/sql/copy/csv/rejects/test_invalid_utf_rejects.test | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/execution/operator/persistent/csv_rejects_table.cpp b/src/execution/operator/persistent/csv_rejects_table.cpp index 4140b5d3eac5..a2f80d855d15 100644 --- a/src/execution/operator/persistent/csv_rejects_table.cpp +++ b/src/execution/operator/persistent/csv_rejects_table.cpp @@ -25,7 +25,7 @@ void CSVRejectsTable::InitializeTable(ClientContext &context, const ReadCSVData // Create CSV_ERROR_TYPE ENUM string enum_name = "CSV_ERROR_TYPE"; - Vector order_errors(LogicalType::VARCHAR, 5); + Vector order_errors(LogicalType::VARCHAR, 6); order_errors.SetValue(0, "CAST"); order_errors.SetValue(1, "MISSING COLUMNS"); order_errors.SetValue(2, "TOO MANY COLUMNS"); diff --git a/test/sql/copy/csv/rejects/test_invalid_utf_rejects.test b/test/sql/copy/csv/rejects/test_invalid_utf_rejects.test index bcf1b6f64ecb..b2082773ae5e 100644 --- a/test/sql/copy/csv/rejects/test_invalid_utf_rejects.test +++ b/test/sql/copy/csv/rejects/test_invalid_utf_rejects.test @@ -12,4 +12,7 @@ query IIIIIII rowsort SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table; ---- - +test/sql/copy/csv/data/test/invalid_utf_big.csv 3001 2 "col3" INVALID UNICODE valid,invalid_??_part,valid 54000 +test/sql/copy/csv/data/test/invalid_utf_big.csv 3012 2 "col3" INVALID UNICODE valid,valid,invalid_??_part 54208 +test/sql/copy/csv/data/test/invalid_utf_big.csv 3023 2 "col3" INVALID UNICODE valid,invalid_??_part,valid 54416 +test/sql/copy/csv/data/test/invalid_utf_big.csv 3034 2 "col3" INVALID UNICODE valid,valid,invalid_??_part 54624 From b7d999792a3e527a8abc709fc52786a356396bc7 Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 7 Mar 2024 11:45:30 +0100 Subject: [PATCH 046/603] increase to 5x memory limit --- src/include/duckdb/main/config.hpp | 2 +- src/main/config.cpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/include/duckdb/main/config.hpp b/src/include/duckdb/main/config.hpp index b1b6a0fc0756..397376b80333 100644 --- a/src/include/duckdb/main/config.hpp +++ b/src/include/duckdb/main/config.hpp @@ -118,7 +118,7 @@ struct DBConfigOptions { string autoinstall_extension_repo = ""; //! The maximum memory used by the database system (in bytes). Default: 80% of System available memory idx_t maximum_memory = (idx_t)-1; - //! The maximum size of the 'temp_directory' folder when set (in bytes). Default 2x 'maximum_memory' + //! The maximum size of the 'temp_directory' folder when set (in bytes). Default 5x 'maximum_memory' idx_t maximum_swap_space = (idx_t)-1; //! The maximum amount of CPU threads used by the database system. Default: all available. idx_t maximum_threads = (idx_t)-1; diff --git a/src/main/config.cpp b/src/main/config.cpp index 78e5d36b1014..5159bf22864f 100644 --- a/src/main/config.cpp +++ b/src/main/config.cpp @@ -262,9 +262,9 @@ void DBConfig::SetDefaultMaxMemory() { void DBConfig::SetDefaultMaxSwapSpace() { auto memory_limit = options.maximum_memory; - if (!TryMultiplyOperator::Operation(memory_limit, static_cast(2), options.maximum_swap_space)) { - // Can't default to 2x memory: fall back to 5GB instead - options.maximum_swap_space = ParseMemoryLimit("5GB"); + if (!TryMultiplyOperator::Operation(memory_limit, static_cast(5), options.maximum_swap_space)) { + // Can't default to 5x memory: fall back to same limit as memory instead + options.maximum_swap_space = memory_limit; } } From cd1c2d70b28a4f94f9d77cbc107f6b510fcb1aee Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 7 Mar 2024 11:51:24 +0100 Subject: [PATCH 047/603] tidy happy and skip tests on windows due do byte_position mismatch --- .../csv_scanner/scanner/string_value_scanner.cpp | 10 +++++----- test/sql/copy/csv/rejects/csv_buffer_size_rejects.test | 3 +++ .../rejects/csv_incorrect_columns_amount_rejects.test | 3 +++ test/sql/copy/csv/rejects/csv_rejects_auto.test | 3 +++ test/sql/copy/csv/rejects/csv_rejects_flush_cast.test | 5 +++++ test/sql/copy/csv/rejects/csv_rejects_read.test | 3 +++ test/sql/copy/csv/rejects/test_invalid_parameters.test | 3 +++ .../sql/copy/csv/rejects/test_invalid_utf_rejects.test | 3 +++ 8 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index a87f43917e1b..6e9ecdd1d1c3 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -341,10 +341,10 @@ void StringValueResult::HandleUnicodeError(bool force_error) { bool first_nl; auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); // sanitize borked line - std::vector charArray(borked_line.begin(), borked_line.end()); - charArray.push_back('\0'); // Null-terminate the character array - Utf8Proc::MakeValid(&charArray[0], charArray.size()); - borked_line = {charArray.begin(), charArray.end() - 1}; + std::vector char_array(borked_line.begin(), borked_line.end()); + char_array.push_back('\0'); // Null-terminate the character array + Utf8Proc::MakeValid(&char_array[0], char_array.size()); + borked_line = {char_array.begin(), char_array.end() - 1}; LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); auto csv_error = CSVError::InvalidUTF8(state_machine.options, cur_col_id - 1, lines_per_batch, borked_line, current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); @@ -426,7 +426,7 @@ bool StringValueResult::AddRowInternal() { HandleUnicodeError(); break; default: - InvalidInputException("CSV Error not allowed when inserting row"); + throw InvalidInputException("CSV Error not allowed when inserting row"); } cur_col_id = 0; chunk_col_id = 0; diff --git a/test/sql/copy/csv/rejects/csv_buffer_size_rejects.test b/test/sql/copy/csv/rejects/csv_buffer_size_rejects.test index dcef91e814ee..a8fd11728781 100644 --- a/test/sql/copy/csv/rejects/csv_buffer_size_rejects.test +++ b/test/sql/copy/csv/rejects/csv_buffer_size_rejects.test @@ -4,6 +4,9 @@ require skip_reload +# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n +require notwindows + loop buffer_size 5 10 # Ensure that we can get the schema if we reduce the sample size and ignore errors diff --git a/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test b/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test index 414e9fdd8594..070b413a8497 100644 --- a/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test +++ b/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test @@ -4,6 +4,9 @@ require skip_reload +# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n +require notwindows + statement ok SELECT * FROM read_csv( 'data/csv/rejects/incorrect_columns/few_columns.csv', diff --git a/test/sql/copy/csv/rejects/csv_rejects_auto.test b/test/sql/copy/csv/rejects/csv_rejects_auto.test index 841ed42465f3..febda7d1c0fc 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_auto.test +++ b/test/sql/copy/csv/rejects/csv_rejects_auto.test @@ -3,6 +3,9 @@ require skip_reload +# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n +require notwindows + # Ensure that we can get the schema if we reduce the sample size and ignore errors query IIIII SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto( diff --git a/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test b/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test index 6b2f5e59d7f5..20ff320b36a5 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test +++ b/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test @@ -2,6 +2,11 @@ # description: Test that Flush Cast functions properly for the rejects tables # group: [rejects] +require skip_reload + +# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n +require notwindows + query III SELECT typeof(first(a)), typeof(first(b)), COUNT(*) FROM read_csv( 'data/csv/error/flush_cast.csv', diff --git a/test/sql/copy/csv/rejects/csv_rejects_read.test b/test/sql/copy/csv/rejects/csv_rejects_read.test index a0b2e751289d..8cf3d5ac883f 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_read.test +++ b/test/sql/copy/csv/rejects/csv_rejects_read.test @@ -3,6 +3,9 @@ require skip_reload +# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n +require notwindows + query III rowsort SELECT * FROM read_csv( 'test/sql/copy/csv/data/error/mismatch/bad.csv', diff --git a/test/sql/copy/csv/rejects/test_invalid_parameters.test b/test/sql/copy/csv/rejects/test_invalid_parameters.test index 2e343a30765d..9325f3780f24 100644 --- a/test/sql/copy/csv/rejects/test_invalid_parameters.test +++ b/test/sql/copy/csv/rejects/test_invalid_parameters.test @@ -3,6 +3,9 @@ require skip_reload +# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n +require notwindows + # Test invalid arguments statement error SELECT * FROM read_csv( diff --git a/test/sql/copy/csv/rejects/test_invalid_utf_rejects.test b/test/sql/copy/csv/rejects/test_invalid_utf_rejects.test index b2082773ae5e..52ff0ac19823 100644 --- a/test/sql/copy/csv/rejects/test_invalid_utf_rejects.test +++ b/test/sql/copy/csv/rejects/test_invalid_utf_rejects.test @@ -4,6 +4,9 @@ require skip_reload +# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n +require notwindows + statement ok from read_csv('test/sql/copy/csv/data/test/invalid_utf_big.csv',columns = {'col1': 'VARCHAR','col2': 'VARCHAR','col3': 'VARCHAR'}, auto_detect=false, rejects_table='csv_rejects_table', header = 0, delim = ',', ignore_errors=true) From 2bd5af56ffb1dc9d3dd7ef64259f0bb23969d795 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 7 Mar 2024 12:12:00 +0100 Subject: [PATCH 048/603] Adding tests and fixes for rejects over maximum line size --- data/csv/rejects/maximum_line/max_10.csv | 8 + data/csv/rejects/maximum_line/over_vector.csv | 2926 +++++++++++++++++ .../scanner/string_value_scanner.cpp | 6 +- .../csv/rejects/csv_rejects_maximum_line.test | 86 + 4 files changed, 3023 insertions(+), 3 deletions(-) create mode 100644 data/csv/rejects/maximum_line/max_10.csv create mode 100644 data/csv/rejects/maximum_line/over_vector.csv create mode 100644 test/sql/copy/csv/rejects/csv_rejects_maximum_line.test diff --git a/data/csv/rejects/maximum_line/max_10.csv b/data/csv/rejects/maximum_line/max_10.csv new file mode 100644 index 000000000000..02a3683c09e0 --- /dev/null +++ b/data/csv/rejects/maximum_line/max_10.csv @@ -0,0 +1,8 @@ +a,b +bla,1 +bla,2 +bla,3 +blaaaaaaaaaaaaaa,4 +bla,1 +bla,2 +bla,3 \ No newline at end of file diff --git a/data/csv/rejects/maximum_line/over_vector.csv b/data/csv/rejects/maximum_line/over_vector.csv new file mode 100644 index 000000000000..c8fe70a412a4 --- /dev/null +++ b/data/csv/rejects/maximum_line/over_vector.csv @@ -0,0 +1,2926 @@ +a,b +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +blaaaaaaaaaaaaaaaaaaaa,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +blaaaaaaaaaaaaaaaaaaaa,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +bla,3 +bla,1 +bla,2 +blaaaaaaaaaaaaaaaaaaaa,3 +bla,1 +bla,2 +bla,3 diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 6e9ecdd1d1c3..c9207c2298ee 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -406,17 +406,17 @@ bool StringValueResult::AddRowInternal() { if (store_line_size) { error_handler.NewMaxLineSize(current_line_size); } + current_line_position.begin = current_line_position.end; + current_line_position.end = current_line_start; if (current_line_size > state_machine.options.maximum_line_size) { bool first_nl; auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); - LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), number_of_rows); + LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); auto csv_error = CSVError::LineSizeError(state_machine.options, current_line_size, lines_per_batch, borked_line, current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); error_handler.Error(csv_error); } - current_line_position.begin = current_line_position.end; - current_line_position.end = current_line_start; if (current_error.is_set) { switch (current_error.type) { case CSVErrorType::TOO_MANY_COLUMNS: diff --git a/test/sql/copy/csv/rejects/csv_rejects_maximum_line.test b/test/sql/copy/csv/rejects/csv_rejects_maximum_line.test new file mode 100644 index 000000000000..f7bb5447485c --- /dev/null +++ b/test/sql/copy/csv/rejects/csv_rejects_maximum_line.test @@ -0,0 +1,86 @@ +# name: test/sql/copy/csv/rejects/csv_rejects_maximum_line.test +# description: Tests rejects tables on max line size parameter +# group: [rejects] + +require skip_reload + +# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n +require notwindows + +statement ok +SELECT * FROM read_csv( + 'data/csv/rejects/maximum_line/max_10.csv', + columns = {'a': 'VARCHAR', 'b': 'INTEGER'}, + rejects_table='csv_rejects_table', + ignore_errors=true, auto_detect=false, header = 1, max_line_size=10); + +query IIIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position +FROM csv_rejects_table; +---- +data/csv/rejects/maximum_line/max_10.csv 5 0 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 22 + +statement ok +DROP TABLE csv_rejects_table; + +# Test with buffer sizes + +loop buffer_size 22 27 + +statement ok +SELECT * FROM read_csv( + 'data/csv/rejects/maximum_line/max_10.csv', + columns = {'a': 'VARCHAR', 'b': 'INTEGER'}, + rejects_table='csv_rejects_table', + ignore_errors=true, auto_detect=false, header = 1, max_line_size=10, buffer_size=${buffer_size}); + +query IIIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position +FROM csv_rejects_table; +---- +data/csv/rejects/maximum_line/max_10.csv 5 0 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 22 + +statement ok +DROP TABLE csv_rejects_table; + +endloop + +# Test over vector size file +statement ok +SELECT * FROM read_csv( + 'data/csv/rejects/maximum_line/over_vector.csv', + columns = {'a': 'VARCHAR', 'b': 'INTEGER'}, + rejects_table='csv_rejects_table', + ignore_errors=true, auto_detect=false, header = 1, max_line_size=20); + +query IIIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position +FROM csv_rejects_table; +---- +data/csv/rejects/maximum_line/over_vector.csv 2282 0 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 13684 +data/csv/rejects/maximum_line/over_vector.csv 2591 0 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 15557 +data/csv/rejects/maximum_line/over_vector.csv 2923 0 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,3 17568 + +statement ok +DROP TABLE csv_rejects_table; + +# Read Multiple Files + +statement ok +SELECT * FROM read_csv( + 'data/csv/rejects/maximum_line/*.csv', + columns = {'a': 'VARCHAR', 'b': 'INTEGER'}, + rejects_table='csv_rejects_table', + ignore_errors=true, auto_detect=false, header = 1, max_line_size=10); + +query IIIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position +FROM csv_rejects_table; +---- +data/csv/rejects/maximum_line/max_10.csv 5 0 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 22 +data/csv/rejects/maximum_line/over_vector.csv 2282 0 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 13684 +data/csv/rejects/maximum_line/over_vector.csv 2591 0 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 15557 +data/csv/rejects/maximum_line/over_vector.csv 2923 0 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,3 17568 + +statement ok +DROP TABLE csv_rejects_table; \ No newline at end of file From 73e8c36c6b537aa3df0bed2b2980fd2b2e6ef75b Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 7 Mar 2024 15:29:07 +0100 Subject: [PATCH 049/603] Properly handling unquoted values w ignore_errors --- data/csv/rejects/unquoted/basic.csv | 8 ++ .../scanner/string_value_scanner.cpp | 73 ++++++++++++------- .../state_machine/csv_state_machine_cache.cpp | 20 ++--- .../operator/csv_scanner/csv_error.hpp | 18 ++--- .../csv_scanner/string_value_scanner.hpp | 5 ++ .../csv/rejects/csv_unquoted_rejects.test | 31 ++++++++ 6 files changed, 110 insertions(+), 45 deletions(-) create mode 100644 data/csv/rejects/unquoted/basic.csv create mode 100644 test/sql/copy/csv/rejects/csv_unquoted_rejects.test diff --git a/data/csv/rejects/unquoted/basic.csv b/data/csv/rejects/unquoted/basic.csv new file mode 100644 index 000000000000..8f7dc567086a --- /dev/null +++ b/data/csv/rejects/unquoted/basic.csv @@ -0,0 +1,8 @@ +a,b +"bla",1 +"bla",2 +"bla",3 +"blaaaaaaaaaaaaaa"bla,4 +"bla",1 +"bla",2 +"bla",3 \ No newline at end of file diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index c9207c2298ee..180563271a74 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -351,6 +351,40 @@ void StringValueResult::HandleUnicodeError(bool force_error) { error_handler.Error(csv_error, force_error); } +void StringValueResult::HandleUnterminatedQuotes(bool force_error) { + LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); + bool first_nl; + auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); + auto csv_error = + CSVError::UnterminatedQuotesError(state_machine.options, cur_col_id - 1, lines_per_batch, borked_line, + current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); + error_handler.Error(csv_error, force_error); +} + +bool StringValueResult::HandleError() { + if (current_error.is_set) { + switch (current_error.type) { + case CSVErrorType::TOO_MANY_COLUMNS: + HandleOverLimitRows(); + break; + case CSVErrorType::INVALID_UNICODE: + HandleUnicodeError(); + break; + case CSVErrorType::UNTERMINATED_QUOTES: + HandleUnterminatedQuotes(); + break; + default: + throw InvalidInputException("CSV Error not allowed when inserting row"); + } + cur_col_id = 0; + chunk_col_id = 0; + // An error occurred on this row, we are ignoring it and resetting our control flag + current_error.Reset(); + return true; + } + return false; +} + void StringValueResult::QuotedNewLine(StringValueResult &result) { result.quoted_new_line = true; } @@ -360,7 +394,7 @@ void StringValueResult::NullPaddingQuotedNewlineCheck() { if (state_machine.options.null_padding && iterator.IsBoundarySet() && quoted_new_line && iterator.done) { // If we have null_padding set, we found a quoted new line, we are scanning the file in parallel and it's the // last row of this thread. - LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), number_of_rows + 1); + LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); auto csv_error = CSVError::NullPaddingFail(state_machine.options, lines_per_batch); error_handler.Error(csv_error); } @@ -417,21 +451,7 @@ bool StringValueResult::AddRowInternal() { current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); error_handler.Error(csv_error); } - if (current_error.is_set) { - switch (current_error.type) { - case CSVErrorType::TOO_MANY_COLUMNS: - HandleOverLimitRows(); - break; - case CSVErrorType::INVALID_UNICODE: - HandleUnicodeError(); - break; - default: - throw InvalidInputException("CSV Error not allowed when inserting row"); - } - cur_col_id = 0; - chunk_col_id = 0; - // An error occurred on this row, we are ignoring it and resetting our control flag - current_error.Reset(); + if (HandleError()) { return false; } if (!cast_errors.empty()) { @@ -533,15 +553,12 @@ bool StringValueResult::AddRow(StringValueResult &result, const idx_t buffer_pos } void StringValueResult::InvalidState(StringValueResult &result) { - // FIXME: How do we recover from an invalid state? Can we restart the state machine and jump to the next row? - LinesPerBoundary lines_per_batch(result.iterator.GetBoundaryIdx(), result.number_of_rows); - bool first_nl; - auto borked_line = result.current_line_position.ReconstructCurrentLine(first_nl, result.buffer_handles); - - auto csv_error = CSVError::UnterminatedQuotesError( - result.state_machine.options, result.cur_col_id, lines_per_batch, borked_line, - result.current_line_position.begin.GetGlobalPosition(result.requested_size, first_nl)); - result.error_handler.Error(csv_error); + bool force_error = !result.state_machine.options.ignore_errors && result.sniffing; + // Invalid unicode, we must error + if (force_error) { + result.HandleUnicodeError(force_error); + } + result.current_error = {CSVErrorType::UNTERMINATED_QUOTES}; } bool StringValueResult::EmptyLine(StringValueResult &result, const idx_t buffer_pos) { @@ -1205,7 +1222,9 @@ void StringValueScanner::FinalizeChunkProcess() { // If we are not done we have two options. // 1) If a boundary is set. if (iterator.IsBoundarySet()) { - iterator.done = true; + if (!(result.current_error == CSVErrorType::UNTERMINATED_QUOTES)) { + iterator.done = true; + } // We read until the next line or until we have nothing else to read. // Move to next buffer if (!cur_buffer_handle) { @@ -1221,6 +1240,8 @@ void StringValueScanner::FinalizeChunkProcess() { if (cur_buffer_handle->is_last_buffer && iterator.pos.buffer_pos >= cur_buffer_handle->actual_size) { MoveToNextBuffer(); } + } else { + result.HandleError(); } } else { // 2) If a boundary is not set diff --git a/src/execution/operator/csv_scanner/state_machine/csv_state_machine_cache.cpp b/src/execution/operator/csv_scanner/state_machine/csv_state_machine_cache.cpp index 1f9768826edd..fbe0752311df 100644 --- a/src/execution/operator/csv_scanner/state_machine/csv_state_machine_cache.cpp +++ b/src/execution/operator/csv_scanner/state_machine/csv_state_machine_cache.cpp @@ -21,7 +21,6 @@ void CSVStateMachineCache::Insert(const CSVStateMachineOptions &state_machine_op InitializeTransitionArray(transition_array, cur_state, CSVState::QUOTED); break; case CSVState::UNQUOTED: - case CSVState::INVALID: case CSVState::ESCAPE: InitializeTransitionArray(transition_array, cur_state, CSVState::INVALID); break; @@ -38,15 +37,16 @@ void CSVStateMachineCache::Insert(const CSVStateMachineOptions &state_machine_op auto new_line_id = state_machine_options.new_line.GetValue(); // Now set values depending on configuration - // 1) Standard State - transition_array[delimiter][static_cast(static_cast(CSVState::STANDARD))] = CSVState::DELIMITER; - transition_array[static_cast('\n')][static_cast(CSVState::STANDARD)] = CSVState::RECORD_SEPARATOR; - if (new_line_id == NewLineIdentifier::CARRY_ON) { - transition_array[static_cast('\r')][static_cast(CSVState::STANDARD)] = - CSVState::CARRIAGE_RETURN; - } else { - transition_array[static_cast('\r')][static_cast(CSVState::STANDARD)] = - CSVState::RECORD_SEPARATOR; + // 1) Standard/Invalid State + vector std_inv {static_cast(CSVState::STANDARD), static_cast(CSVState::INVALID)}; + for (auto &state : std_inv) { + transition_array[delimiter][state] = CSVState::DELIMITER; + transition_array[static_cast('\n')][state] = CSVState::RECORD_SEPARATOR; + if (new_line_id == NewLineIdentifier::CARRY_ON) { + transition_array[static_cast('\r')][state] = CSVState::CARRIAGE_RETURN; + } else { + transition_array[static_cast('\r')][state] = CSVState::RECORD_SEPARATOR; + } } // 2) Field Separator State transition_array[delimiter][static_cast(CSVState::DELIMITER)] = CSVState::DELIMITER; diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp index 3ced0619aa67..befc3a669219 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp @@ -37,15 +37,15 @@ class LinesPerBoundary { }; enum CSVErrorType : uint8_t { - CAST_ERROR = 0, // If when casting a value from string to the column type fails - COLUMN_NAME_TYPE_MISMATCH = 1, // If there is a mismatch between Column Names and Types - TOO_FEW_COLUMNS = 2, // If the CSV has too few columns - TOO_MANY_COLUMNS = 3, // If the CSV has too many column - UNTERMINATED_QUOTES = 4, // If a quote is not terminated - SNIFFING = 5, // If something went wrong during sniffing and was not possible to find suitable candidates - MAXIMUM_LINE_SIZE = 6, // Maximum line size was exceeded by a line in the CSV File - NULLPADDED_QUOTED_NEW_VALUE = 7, // If the null_padding option is set, and we have quoted new values in parallel - INVALID_UNICODE = 8 + CAST_ERROR = 0, //! If when casting a value from string to the column type fails + COLUMN_NAME_TYPE_MISMATCH = 1, //! If there is a mismatch between Column Names and Types + TOO_FEW_COLUMNS = 2, //! If the CSV has too few columns + TOO_MANY_COLUMNS = 3, //! If the CSV has too many column + UNTERMINATED_QUOTES = 4, //! If a quote is not terminated + SNIFFING = 5, //! If something went wrong during sniffing and was not possible to find suitable candidates + MAXIMUM_LINE_SIZE = 6, //! Maximum line size was exceeded by a line in the CSV File + NULLPADDED_QUOTED_NEW_VALUE = 7, //! If the null_padding option is set, and we have quoted new values in parallel + INVALID_UNICODE = 8 //! If we have invalid unicode values }; class CSVError { diff --git a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp index 18d1776b7a71..3869f0b94d5d 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp @@ -71,6 +71,9 @@ class CurrentError { } bool is_set; CSVErrorType type; + friend bool operator==(const CurrentError &error, CSVErrorType other) { + return error.is_set && error.type == other; + } }; class StringValueResult : public ScannerResult { @@ -152,6 +155,8 @@ class StringValueResult : public ScannerResult { void HandleOverLimitRows(); void HandleUnicodeError(bool force_error = false); + void HandleUnterminatedQuotes(bool force_error = false); + bool HandleError(); inline void AddValueToVector(const char *value_ptr, const idx_t size, bool allocate = false); diff --git a/test/sql/copy/csv/rejects/csv_unquoted_rejects.test b/test/sql/copy/csv/rejects/csv_unquoted_rejects.test new file mode 100644 index 000000000000..a50aef9c60da --- /dev/null +++ b/test/sql/copy/csv/rejects/csv_unquoted_rejects.test @@ -0,0 +1,31 @@ +# name: test/sql/copy/csv/rejects/csv_unquoted_rejects.test +# description: Tests rejects tables on max line size parameter +# group: [rejects] + +require skip_reload + +# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n +require notwindows + +query II +SELECT * FROM read_csv( + 'data/csv/rejects/unquoted/basic.csv', + columns = {'a': 'VARCHAR', 'b': 'INTEGER'}, + rejects_table='csv_rejects_table', + ignore_errors=true, auto_detect=false, header = 1, quote = '"', escape = '"'); +---- +bla 1 +bla 2 +bla 3 +bla 1 +bla 2 +bla 3 + +query IIIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position +FROM csv_rejects_table; +---- +data/csv/rejects/unquoted/basic.csv 5 1 "b" UNQUOTED VALUE "blaaaaaaaaaaaaaa"bla,4 28 + +statement ok +DROP TABLE csv_rejects_table; From bc26aa060be9a85c56e3a3ef34b6353214020dac Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 7 Mar 2024 18:40:08 +0100 Subject: [PATCH 050/603] More work on rejects + unquoted --- .../rejects/unquoted/unquoted_last_value.csv | 5 ++ .../rejects/unquoted/unquoted_new_line.csv | 9 +++ .../scanner/string_value_scanner.cpp | 28 ++++---- .../operator/csv_scanner/base_scanner.hpp | 2 +- .../csv_scanner/string_value_scanner.hpp | 9 +-- .../csv/rejects/csv_unquoted_rejects.test | 68 ++++++++++++++++++- 6 files changed, 101 insertions(+), 20 deletions(-) create mode 100644 data/csv/rejects/unquoted/unquoted_last_value.csv create mode 100644 data/csv/rejects/unquoted/unquoted_new_line.csv diff --git a/data/csv/rejects/unquoted/unquoted_last_value.csv b/data/csv/rejects/unquoted/unquoted_last_value.csv new file mode 100644 index 000000000000..0d714083e9c8 --- /dev/null +++ b/data/csv/rejects/unquoted/unquoted_last_value.csv @@ -0,0 +1,5 @@ +"blaaaaaaaaaaaaaa" +"bla" +"bla" +"bla" +"bla diff --git a/data/csv/rejects/unquoted/unquoted_new_line.csv b/data/csv/rejects/unquoted/unquoted_new_line.csv new file mode 100644 index 000000000000..e42978c5565e --- /dev/null +++ b/data/csv/rejects/unquoted/unquoted_new_line.csv @@ -0,0 +1,9 @@ +a,b +"bla",1 +"bla",2 +"bla",3 +"blaaaaaaaaaaaaaa +"bla,4 +"bla",1 +"bla",2 +"bla",3 \ No newline at end of file diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 180563271a74..b2fc8baa8aa4 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -122,7 +122,7 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size error = !IsValueNull(null_str_ptr, value_ptr, size); } if (error) { - current_error = {CSVErrorType::TOO_MANY_COLUMNS}; + current_error = {CSVErrorType::TOO_MANY_COLUMNS, cur_col_id}; } return; } @@ -222,10 +222,10 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size bool force_error = !state_machine.options.ignore_errors && sniffing; // Invalid unicode, we must error if (force_error) { - HandleUnicodeError(force_error); + HandleUnicodeError(cur_col_id, force_error); } // If we got here, we are ingoring errors, hence we must ignore this line. - current_error = {CSVErrorType::INVALID_UNICODE}; + current_error = {CSVErrorType::INVALID_UNICODE, cur_col_id}; break; } if (allocate) { @@ -327,17 +327,17 @@ void StringValueResult::AddValue(StringValueResult &result, const idx_t buffer_p result.last_position = buffer_pos + 1; } -void StringValueResult::HandleOverLimitRows() { +void StringValueResult::HandleOverLimitRows(idx_t col_idx) { LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); bool first_nl; auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); auto csv_error = - CSVError::IncorrectColumnAmountError(state_machine.options, cur_col_id + 1, lines_per_batch, borked_line, + CSVError::IncorrectColumnAmountError(state_machine.options, col_idx, lines_per_batch, borked_line, current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); error_handler.Error(csv_error); } -void StringValueResult::HandleUnicodeError(bool force_error) { +void StringValueResult::HandleUnicodeError(idx_t col_idx, bool force_error) { bool first_nl; auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); // sanitize borked line @@ -346,17 +346,17 @@ void StringValueResult::HandleUnicodeError(bool force_error) { Utf8Proc::MakeValid(&char_array[0], char_array.size()); borked_line = {char_array.begin(), char_array.end() - 1}; LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); - auto csv_error = CSVError::InvalidUTF8(state_machine.options, cur_col_id - 1, lines_per_batch, borked_line, + auto csv_error = CSVError::InvalidUTF8(state_machine.options, col_idx, lines_per_batch, borked_line, current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); error_handler.Error(csv_error, force_error); } -void StringValueResult::HandleUnterminatedQuotes(bool force_error) { +void StringValueResult::HandleUnterminatedQuotes(idx_t col_idx, bool force_error) { LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); bool first_nl; auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); auto csv_error = - CSVError::UnterminatedQuotesError(state_machine.options, cur_col_id - 1, lines_per_batch, borked_line, + CSVError::UnterminatedQuotesError(state_machine.options, col_idx, lines_per_batch, borked_line, current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); error_handler.Error(csv_error, force_error); } @@ -365,13 +365,13 @@ bool StringValueResult::HandleError() { if (current_error.is_set) { switch (current_error.type) { case CSVErrorType::TOO_MANY_COLUMNS: - HandleOverLimitRows(); + HandleOverLimitRows(current_error.col_idx); break; case CSVErrorType::INVALID_UNICODE: - HandleUnicodeError(); + HandleUnicodeError(current_error.col_idx); break; case CSVErrorType::UNTERMINATED_QUOTES: - HandleUnterminatedQuotes(); + HandleUnterminatedQuotes(current_error.col_idx); break; default: throw InvalidInputException("CSV Error not allowed when inserting row"); @@ -556,9 +556,9 @@ void StringValueResult::InvalidState(StringValueResult &result) { bool force_error = !result.state_machine.options.ignore_errors && result.sniffing; // Invalid unicode, we must error if (force_error) { - result.HandleUnicodeError(force_error); + result.HandleUnicodeError(result.cur_col_id, force_error); } - result.current_error = {CSVErrorType::UNTERMINATED_QUOTES}; + result.current_error = {CSVErrorType::UNTERMINATED_QUOTES, result.cur_col_id}; } bool StringValueResult::EmptyLine(StringValueResult &result, const idx_t buffer_pos) { diff --git a/src/include/duckdb/execution/operator/csv_scanner/base_scanner.hpp b/src/include/duckdb/execution/operator/csv_scanner/base_scanner.hpp index c8a2f886fa9f..29a62b8e79ae 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/base_scanner.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/base_scanner.hpp @@ -252,7 +252,7 @@ class BaseScanner { Initialize(); initialized = true; } - if (!iterator.done) { + if (!iterator.done && cur_buffer_handle) { Process(result); } FinalizeChunkProcess(); diff --git a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp index 3869f0b94d5d..e36266d90f69 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp @@ -65,12 +65,13 @@ class FullLinePosition { class CurrentError { public: CurrentError() : is_set(false) {}; - CurrentError(CSVErrorType type) : is_set(true), type(type) {}; + CurrentError(CSVErrorType type, idx_t col_idx_p) : is_set(true), type(type), col_idx(col_idx_p) {}; void Reset() { is_set = false; } bool is_set; CSVErrorType type; + idx_t col_idx; friend bool operator==(const CurrentError &error, CSVErrorType other) { return error.is_set && error.type == other; } @@ -153,9 +154,9 @@ class StringValueResult : public ScannerResult { static inline bool EmptyLine(StringValueResult &result, const idx_t buffer_pos); inline bool AddRowInternal(); - void HandleOverLimitRows(); - void HandleUnicodeError(bool force_error = false); - void HandleUnterminatedQuotes(bool force_error = false); + void HandleOverLimitRows(idx_t col_idx); + void HandleUnicodeError(idx_t col_idx, bool force_error = false); + void HandleUnterminatedQuotes(idx_t col_idx, bool force_error = false); bool HandleError(); inline void AddValueToVector(const char *value_ptr, const idx_t size, bool allocate = false); diff --git a/test/sql/copy/csv/rejects/csv_unquoted_rejects.test b/test/sql/copy/csv/rejects/csv_unquoted_rejects.test index a50aef9c60da..976b66d0ff05 100644 --- a/test/sql/copy/csv/rejects/csv_unquoted_rejects.test +++ b/test/sql/copy/csv/rejects/csv_unquoted_rejects.test @@ -25,7 +25,73 @@ query IIIIIII rowsort SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table; ---- -data/csv/rejects/unquoted/basic.csv 5 1 "b" UNQUOTED VALUE "blaaaaaaaaaaaaaa"bla,4 28 +data/csv/rejects/unquoted/basic.csv 5 0 "a" UNQUOTED VALUE "blaaaaaaaaaaaaaa"bla,4 28 statement ok DROP TABLE csv_rejects_table; + +query II +SELECT * FROM read_csv( + 'data/csv/rejects/unquoted/unquoted_new_line.csv', + columns = {'a': 'VARCHAR', 'b': 'INTEGER'}, + rejects_table='csv_rejects_table', + ignore_errors=true, auto_detect=false, header = 1, quote = '"', escape = '"'); +---- +bla 1 +bla 2 +bla 3 +bla 1 +bla 2 +bla 3 + +query IIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, byte_position +FROM csv_rejects_table; +---- +data/csv/rejects/unquoted/unquoted_new_line.csv 5 0 "a" UNQUOTED VALUE 28 + +statement ok +DROP TABLE csv_rejects_table; + +query I +SELECT * FROM read_csv( + 'data/csv/rejects/unquoted/unquoted_last_value.csv', + columns = {'a': 'VARCHAR'}, + rejects_table='csv_rejects_table', + ignore_errors=true, auto_detect=false, header = 0, quote = '"', escape = '"'); +---- +blaaaaaaaaaaaaaa +bla +bla +bla + +query IIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, byte_position +FROM csv_rejects_table; +---- +data/csv/rejects/unquoted/unquoted_last_value.csv 5 0 "a" UNQUOTED VALUE 31 + +statement ok +DROP TABLE csv_rejects_table; + +# Test buffer sizes (borked :( ) +# +#loop buffer_size 35 1001 +# +#statement ok +#SELECT * FROM read_csv( +# 'data/csv/rejects/unquoted/basic.csv', +# columns = {'a': 'VARCHAR', 'b': 'INTEGER'}, +# rejects_table='csv_rejects_table', +# ignore_errors=true, auto_detect=false, header = 1, quote = '"', escape = '"', buffer_size=${buffer_size}); +# +#query IIIIIII rowsort +#SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position +#FROM csv_rejects_table; +#---- +#data/csv/rejects/unquoted/basic.csv 5 0 "a" UNQUOTED VALUE "blaaaaaaaaaaaaaa"bla,4 28 +# +#statement ok +#DROP TABLE csv_rejects_table; +# +#endloop \ No newline at end of file From 3d105684328589bc150f7d7c4abea037a81bbc86 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 8 Mar 2024 10:09:36 +0100 Subject: [PATCH 051/603] collect information about the disk when possible --- src/common/file_system.cpp | 31 ++++++ src/include/duckdb/common/file_system.hpp | 2 + src/include/duckdb/main/config.hpp | 35 ++++++- src/main/config.cpp | 21 +++- src/main/database.cpp | 2 +- src/main/settings/settings.cpp | 10 +- src/storage/temporary_file_manager.cpp | 5 + test/sql/storage/max_swap_space.test | 112 ++++++++++++++++++++++ 8 files changed, 208 insertions(+), 10 deletions(-) create mode 100644 test/sql/storage/max_swap_space.test diff --git a/src/common/file_system.cpp b/src/common/file_system.cpp index f2124165fdc7..f21e6c46dd79 100644 --- a/src/common/file_system.cpp +++ b/src/common/file_system.cpp @@ -12,6 +12,7 @@ #include "duckdb/main/database.hpp" #include "duckdb/main/extension_helper.hpp" #include "duckdb/common/windows_util.hpp" +#include "duckdb/common/operator/multiply.hpp" #include #include @@ -21,6 +22,7 @@ #include #include #include +#include #include #include @@ -98,6 +100,23 @@ idx_t FileSystem::GetAvailableMemory() { return max_memory; } +idx_t FileSystem::GetAvailableDiskSpace(const string &path) { + struct statvfs vfs; + + if (statvfs(path.c_str(), &vfs) == -1) { + return DConstants::INVALID_INDEX; + } + auto block_size = vfs.f_frsize; + // These are the blocks available for creating new files or extending existing ones + auto available_blocks = vfs.f_bavail; + idx_t available_disk_space = DConstants::INVALID_INDEX; + if (!TryMultiplyOperator::Operation(static_cast(block_size), static_cast(available_blocks), + available_disk_space)) { + return DConstants::INVALID_INDEX; + } + return available_disk_space; +} + string FileSystem::GetWorkingDirectory() { auto buffer = make_unsafe_uniq_array(PATH_MAX); char *ret = getcwd(buffer.get(), PATH_MAX); @@ -198,6 +217,18 @@ idx_t FileSystem::GetAvailableMemory() { return DConstants::INVALID_INDEX; } +idx_t FileSystem::GetAvailableDiskSpace(const string &path) { + ULARGE_INTEGER available_bytes, total_bytes, free_bytes; + + auto unicode_path = WindowsUtil::UTF8ToUnicode(path.c_str()); + if (!GetDiskFreeSpaceExW(unicode_path.c_str(), &available_bytes, &total_bytes, &free_bytes)) { + return DConstants::INVALID_INDEX; + } + (void)total_bytes; + (void)free_bytes; + return NumericCast(available_bytes.QuadPart); +} + string FileSystem::GetWorkingDirectory() { idx_t count = GetCurrentDirectoryW(0, nullptr); if (count == 0) { diff --git a/src/include/duckdb/common/file_system.hpp b/src/include/duckdb/common/file_system.hpp index 3dc81acfa2a3..a082e81db614 100644 --- a/src/include/duckdb/common/file_system.hpp +++ b/src/include/duckdb/common/file_system.hpp @@ -186,6 +186,8 @@ class FileSystem { DUCKDB_API virtual string ExpandPath(const string &path); //! Returns the system-available memory in bytes. Returns DConstants::INVALID_INDEX if the system function fails. DUCKDB_API static idx_t GetAvailableMemory(); + //! Returns the space available on the disk. Returns DConstants::INVALID_INDEX if the information was not available. + DUCKDB_API static idx_t GetAvailableDiskSpace(const string &path); //! Path separator for path DUCKDB_API virtual string PathSeparator(const string &path); //! Checks if path is starts with separator (i.e., '/' on UNIX '\\' on Windows) diff --git a/src/include/duckdb/main/config.hpp b/src/include/duckdb/main/config.hpp index 397376b80333..ec2658afd2bb 100644 --- a/src/include/duckdb/main/config.hpp +++ b/src/include/duckdb/main/config.hpp @@ -61,6 +61,37 @@ typedef void (*reset_global_function_t)(DatabaseInstance *db, DBConfig &config); typedef void (*reset_local_function_t)(ClientContext &context); typedef Value (*get_setting_function_t)(ClientContext &context); +struct NumericSetting { +public: + NumericSetting() : value(0), set_by_user(false) { + } + +public: + NumericSetting &operator=(idx_t val) = delete; + +public: + operator idx_t() { + return value; + } + +public: + bool ExplicitlySet() const { + return set_by_user; + } + void SetDefault(idx_t val) { + value = val; + set_by_user = false; + } + void SetExplicit(idx_t val) { + value = val; + set_by_user = true; + } + +private: + idx_t value; + bool set_by_user; +}; + struct ConfigurationOption { const char *name; const char *description; @@ -119,7 +150,7 @@ struct DBConfigOptions { //! The maximum memory used by the database system (in bytes). Default: 80% of System available memory idx_t maximum_memory = (idx_t)-1; //! The maximum size of the 'temp_directory' folder when set (in bytes). Default 5x 'maximum_memory' - idx_t maximum_swap_space = (idx_t)-1; + NumericSetting maximum_swap_space = NumericSetting(); //! The maximum amount of CPU threads used by the database system. Default: all available. idx_t maximum_threads = (idx_t)-1; //! The number of external threads that work on DuckDB tasks. Default: 1. @@ -278,7 +309,7 @@ struct DBConfig { DUCKDB_API IndexTypeSet &GetIndexTypes(); static idx_t GetSystemMaxThreads(FileSystem &fs); void SetDefaultMaxMemory(); - void SetDefaultMaxSwapSpace(); + void SetDefaultMaxSwapSpace(optional_ptr db); OrderType ResolveOrder(OrderType order_type) const; OrderByNullType ResolveNullOrder(OrderType order_type, OrderByNullType null_type) const; diff --git a/src/main/config.cpp b/src/main/config.cpp index 5159bf22864f..9aa8c50712cb 100644 --- a/src/main/config.cpp +++ b/src/main/config.cpp @@ -260,12 +260,23 @@ void DBConfig::SetDefaultMaxMemory() { } } -void DBConfig::SetDefaultMaxSwapSpace() { - auto memory_limit = options.maximum_memory; - if (!TryMultiplyOperator::Operation(memory_limit, static_cast(5), options.maximum_swap_space)) { - // Can't default to 5x memory: fall back to same limit as memory instead - options.maximum_swap_space = memory_limit; +void DBConfig::SetDefaultMaxSwapSpace(optional_ptr db) { + options.maximum_swap_space.SetDefault(0); + if (options.temporary_directory.empty()) { + return; + } + if (!db) { + return; + } + auto &fs = FileSystem::GetFileSystem(*db); + if (!fs.DirectoryExists(options.temporary_directory)) { + // Directory doesnt exist yet, we will look up the disk space once we have created the directory + // FIXME: do we want to proactively create the directory instead ??? + return; } + // Use the available disk space if temp directory is set + auto disk_space = FileSystem::GetAvailableDiskSpace(options.temporary_directory); + options.maximum_swap_space.SetDefault(disk_space); } void DBConfig::CheckLock(const string &name) { diff --git a/src/main/database.cpp b/src/main/database.cpp index 9744fc97d808..7a0ef6fcc99c 100644 --- a/src/main/database.cpp +++ b/src/main/database.cpp @@ -334,7 +334,7 @@ void DatabaseInstance::Configure(DBConfig &new_config) { config.SetDefaultMaxMemory(); } if (config.options.maximum_swap_space == (idx_t)-1) { - config.SetDefaultMaxSwapSpace(); + config.SetDefaultMaxSwapSpace(this); } if (new_config.options.maximum_threads == (idx_t)-1) { config.options.maximum_threads = config.GetSystemMaxThreads(*config.file_system); diff --git a/src/main/settings/settings.cpp b/src/main/settings/settings.cpp index bdda84adf07f..674188f3b714 100644 --- a/src/main/settings/settings.cpp +++ b/src/main/settings/settings.cpp @@ -952,11 +952,11 @@ Value MaximumMemorySetting::GetSetting(ClientContext &context) { // Maximum Temp Directory Size //===--------------------------------------------------------------------===// void MaximumTempDirectorySize::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) { - config.options.maximum_swap_space = DBConfig::ParseMemoryLimit(input.ToString()); + config.options.maximum_swap_space.SetExplicit(DBConfig::ParseMemoryLimit(input.ToString())); } void MaximumTempDirectorySize::ResetGlobal(DatabaseInstance *db, DBConfig &config) { - config.SetDefaultMaxSwapSpace(); + config.SetDefaultMaxSwapSpace(db); } Value MaximumTempDirectorySize::GetSetting(ClientContext &context) { @@ -1229,6 +1229,12 @@ Value SecretDirectorySetting::GetSetting(ClientContext &context) { void TempDirectorySetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) { config.options.temporary_directory = input.ToString(); config.options.use_temporary_directory = !config.options.temporary_directory.empty(); + if (!config.options.temporary_directory.empty()) { + // Maximum swap space isn't set explicitly, initialize to default + if (!config.options.maximum_swap_space.ExplicitlySet()) { + config.SetDefaultMaxSwapSpace(db); + } + } if (db) { auto &buffer_manager = BufferManager::GetBufferManager(*db); buffer_manager.SetTemporaryDirectory(config.options.temporary_directory); diff --git a/src/storage/temporary_file_manager.cpp b/src/storage/temporary_file_manager.cpp index c97f73f197a1..90761f5d6010 100644 --- a/src/storage/temporary_file_manager.cpp +++ b/src/storage/temporary_file_manager.cpp @@ -194,8 +194,13 @@ TemporaryDirectoryHandle::TemporaryDirectoryHandle(DatabaseInstance &db, string auto &fs = FileSystem::GetFileSystem(db); if (!temp_directory.empty()) { if (!fs.DirectoryExists(temp_directory)) { + auto &config = DBConfig::GetConfig(db); fs.CreateDirectory(temp_directory); created_directory = true; + // Maximum swap space isn't set explicitly, initialize to default + if (!config.options.maximum_swap_space.ExplicitlySet()) { + config.SetDefaultMaxSwapSpace(&db); + } } } } diff --git a/test/sql/storage/max_swap_space.test b/test/sql/storage/max_swap_space.test new file mode 100644 index 000000000000..2f52b3c7940a --- /dev/null +++ b/test/sql/storage/max_swap_space.test @@ -0,0 +1,112 @@ +# name: test/sql/storage/max_swap_space.test +# group: [storage] + +require skip_reload + +statement ok +set temp_directory=''; + +statement ok +PRAGMA memory_limit='2MB' + +# --- Set by default to 0 when temp_directory is not set --- + +# If 'temp_directory' is not set, this defaults to 0 +query I +select current_setting('max_temp_directory_size') +---- +0 bytes + +# Set the max size explicitly + +statement ok +set max_temp_directory_size='15gb' + +# Then reset it, default should be 0 again + +statement ok +reset max_temp_directory_size; + +query I +select current_setting('max_temp_directory_size') +---- +0 bytes + +# --- Set by default to the available disk space when temp_directory exists --- + +statement ok +set temp_directory = '__TEST_DIR__'; + +# '__TEST_DIR__' is guaranteed to exist, we can get the disk space +query I +select current_setting('max_temp_directory_size') a where a == '0 bytes' +---- + +# So the reported max size should not be 0 bytes +query I +select current_setting('max_temp_directory_size') a where a == '0 bytes' +---- + +# --- Set explicitly by the user --- + +# If we set 'max_temp_directory_size' explicitly, it will not be overridden +statement ok +set max_temp_directory_size='15gb' + +# Reported size should not be 0, we set it explicitly +query I +select current_setting('max_temp_directory_size') a where a == '0 bytes' +---- + +query I nosort unchanged +select current_setting('max_temp_directory_size') +---- + +# When we change the temp_directory to something that doesnt exist +statement ok +set temp_directory = '__TEST_DIR__/does_not_exist' + +query I nosort unchanged +select current_setting('max_temp_directory_size') +---- + +# When we change the temp_directory to something that does exist +statement ok +set temp_directory = '__TEST_DIR__' + +query I nosort unchanged +select current_setting('max_temp_directory_size') +---- + +# When we reset the temp_directory .. +statement ok +reset temp_directory; + +query I nosort unchanged +select current_setting('max_temp_directory_size') +---- + +# --- Set to the available disk space when we create the (previously non-existant) 'temp_directory' + +statement ok +reset max_temp_directory_size; + +# When we change the temp_directory to something that doesnt exist +statement ok +set temp_directory = '__TEST_DIR__/does_not_exist' + +query I +select current_setting('max_temp_directory_size') +---- +0 bytes + +statement ok +CREATE TABLE t2 AS SELECT * FROM range(1000000); + +# Reported size should not be 0, the directory was created +query I +select current_setting('max_temp_directory_size') a where a == '0 bytes' +---- + +## TODO: test that the explicitly set value by the user does not get overridden when 'temp_directory' is set to a directory that doesn't exist yet +# when the 'temp_directory' is created by us - the explicitly set value should not be overridden From a0d98f41987e1500ba9814a721c8cfc37e780be2 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 8 Mar 2024 10:17:17 +0100 Subject: [PATCH 052/603] further thinking --- src/main/settings/settings.cpp | 2 ++ test/sql/storage/max_swap_space.test | 13 ++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/main/settings/settings.cpp b/src/main/settings/settings.cpp index 674188f3b714..90ef20522a24 100644 --- a/src/main/settings/settings.cpp +++ b/src/main/settings/settings.cpp @@ -952,6 +952,8 @@ Value MaximumMemorySetting::GetSetting(ClientContext &context) { // Maximum Temp Directory Size //===--------------------------------------------------------------------===// void MaximumTempDirectorySize::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) { + // FIXME: should this not use 'SetExplicit' when the value is 0? + // So it acts as RESET instead when 0 is passed? config.options.maximum_swap_space.SetExplicit(DBConfig::ParseMemoryLimit(input.ToString())); } diff --git a/test/sql/storage/max_swap_space.test b/test/sql/storage/max_swap_space.test index 2f52b3c7940a..0b200b8039c8 100644 --- a/test/sql/storage/max_swap_space.test +++ b/test/sql/storage/max_swap_space.test @@ -18,12 +18,15 @@ select current_setting('max_temp_directory_size') 0 bytes # Set the max size explicitly - statement ok set max_temp_directory_size='15gb' -# Then reset it, default should be 0 again +# Should not be 0 anymore +query I +select current_setting('max_temp_directory_size') a where a == '0 bytes' +---- +# Then reset it, default should be 0 again statement ok reset max_temp_directory_size; @@ -42,11 +45,6 @@ query I select current_setting('max_temp_directory_size') a where a == '0 bytes' ---- -# So the reported max size should not be 0 bytes -query I -select current_setting('max_temp_directory_size') a where a == '0 bytes' ----- - # --- Set explicitly by the user --- # If we set 'max_temp_directory_size' explicitly, it will not be overridden @@ -88,6 +86,7 @@ select current_setting('max_temp_directory_size') # --- Set to the available disk space when we create the (previously non-existant) 'temp_directory' +# Reset it so it's no longer set explicitly statement ok reset max_temp_directory_size; From 8c8ffe6ef20c7983f336c3e9d001a447b848367c Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 8 Mar 2024 10:27:13 +0100 Subject: [PATCH 053/603] test that explicitly set values are not overridden when we create the temp_directory --- .../{ => temp_directory}/max_swap_space.test | 7 ++--- .../max_swap_space_explicit.test | 29 +++++++++++++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) rename test/sql/storage/{ => temp_directory}/max_swap_space.test (88%) create mode 100644 test/sql/storage/temp_directory/max_swap_space_explicit.test diff --git a/test/sql/storage/max_swap_space.test b/test/sql/storage/temp_directory/max_swap_space.test similarity index 88% rename from test/sql/storage/max_swap_space.test rename to test/sql/storage/temp_directory/max_swap_space.test index 0b200b8039c8..0c93cd2852c9 100644 --- a/test/sql/storage/max_swap_space.test +++ b/test/sql/storage/temp_directory/max_swap_space.test @@ -1,5 +1,5 @@ -# name: test/sql/storage/max_swap_space.test -# group: [storage] +# name: test/sql/storage/temp_directory/max_swap_space.test +# group: [temp_directory] require skip_reload @@ -106,6 +106,3 @@ CREATE TABLE t2 AS SELECT * FROM range(1000000); query I select current_setting('max_temp_directory_size') a where a == '0 bytes' ---- - -## TODO: test that the explicitly set value by the user does not get overridden when 'temp_directory' is set to a directory that doesn't exist yet -# when the 'temp_directory' is created by us - the explicitly set value should not be overridden diff --git a/test/sql/storage/temp_directory/max_swap_space_explicit.test b/test/sql/storage/temp_directory/max_swap_space_explicit.test new file mode 100644 index 000000000000..ea85ced65be3 --- /dev/null +++ b/test/sql/storage/temp_directory/max_swap_space_explicit.test @@ -0,0 +1,29 @@ +# name: test/sql/storage/temp_directory/max_swap_space_explicit.test +# group: [temp_directory] + +require skip_reload + +statement ok +PRAGMA memory_limit='2MB' + +# --- Not changed when set explicitly by the user + +# If we set 'max_temp_directory_size' explicitly, it will not be overridden +statement ok +set max_temp_directory_size='15gb' + +# When we change the temp_directory to something that doesnt exist +statement ok +set temp_directory = '__TEST_DIR__/this_directory_should_not_exist__swap_space' + +query I nosort explicitly_set +select current_setting('max_temp_directory_size') +---- + +statement ok +CREATE TABLE t2 AS SELECT * FROM range(1000000); + +# The 'temp_directory' was created, but the value of 'max_temp_directory_size' was set explicitly, so it was unaltered +query I nosort explicitly_set +select current_setting('max_temp_directory_size') +---- From d3ecab62a5584241d0d4cce01c8fa0180d84be82 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 8 Mar 2024 12:50:36 +0100 Subject: [PATCH 054/603] add initial tests --- src/storage/temporary_file_manager.cpp | 8 ++--- .../temp_directory/max_swap_space_error.test | 36 +++++++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) create mode 100644 test/sql/storage/temp_directory/max_swap_space_error.test diff --git a/src/storage/temporary_file_manager.cpp b/src/storage/temporary_file_manager.cpp index 90761f5d6010..0fda4781124b 100644 --- a/src/storage/temporary_file_manager.cpp +++ b/src/storage/temporary_file_manager.cpp @@ -80,10 +80,10 @@ bool BlockIndexManager::HasFreeBlocks() { idx_t BlockIndexManager::GetNewBlockIndexInternal() { if (free_indexes.empty()) { auto new_index = max_index; - max_index++; if (file_size_monitor) { file_size_monitor->Increase(1); } + max_index++; return new_index; } auto entry = free_indexes.begin(); @@ -316,17 +316,17 @@ void TemporaryFileManager::IncreaseSizeOnDisk(idx_t bytes) { auto max_swap_space = config.options.maximum_swap_space; auto current_size_on_disk = size_on_disk.load(); - size_on_disk += bytes; - if (size_on_disk.load() > max_swap_space) { + if (current_size_on_disk + bytes > max_swap_space) { auto used = StringUtil::BytesToHumanReadableString(current_size_on_disk); auto max = StringUtil::BytesToHumanReadableString(max_swap_space); auto data_size = StringUtil::BytesToHumanReadableString(bytes); throw OutOfMemoryException(R"(failed to offload data block of size %s (%s/%s used). This limit was set by the 'max_temp_directory_size' setting. -This defaults to twice the size of 'max_memory'. +By default, this setting utilizes the available disk space on the drive where the 'temp_directory' is located. You can adjust this setting, by using (for example) PRAGMA max_temp_directory_size='10GiB')", data_size, used, max); } + size_on_disk += bytes; } void TemporaryFileManager::DecreaseSizeOnDisk(idx_t bytes) { diff --git a/test/sql/storage/temp_directory/max_swap_space_error.test b/test/sql/storage/temp_directory/max_swap_space_error.test new file mode 100644 index 000000000000..80e75df91904 --- /dev/null +++ b/test/sql/storage/temp_directory/max_swap_space_error.test @@ -0,0 +1,36 @@ +# name: test/sql/storage/temp_directory/max_swap_space_error.test +# group: [temp_directory] + +require skip_reload + +# Set a temp_directory to offload data +statement ok +set temp_directory='__TEST_DIR__/max_swap_space_reached' + +# Ensure the temp_directory is used +statement ok +PRAGMA memory_limit='2MB' + + +# 0 blocks +statement ok +set max_temp_directory_size='0KiB' + +statement error +CREATE OR REPLACE TABLE t2 AS SELECT * FROM range(1000000); +---- +failed to offload data block of size 256.0 KiB (0 bytes/0 bytes used) + +# 1 block max +statement ok +set max_temp_directory_size='256KiB' + +statement error +CREATE OR REPLACE TABLE t2 AS SELECT * FROM range(1000000); +---- +failed to offload data block of size 256.0 KiB (256.0 KiB/256.0 KiB used) + +statement error +CREATE OR REPLACE TABLE t2 AS SELECT * FROM range(1000000); +---- +failed to offload data block of size 256.0 KiB (256.0 KiB/256.0 KiB used) From a97bcb78c0008f7416eefea1512328a73064a7c2 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 8 Mar 2024 13:14:26 +0100 Subject: [PATCH 055/603] more tests with different max swap sizes --- .../temp_directory/max_swap_space_error.test | 38 ++++++++++++++++++- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/test/sql/storage/temp_directory/max_swap_space_error.test b/test/sql/storage/temp_directory/max_swap_space_error.test index 80e75df91904..68c55f520665 100644 --- a/test/sql/storage/temp_directory/max_swap_space_error.test +++ b/test/sql/storage/temp_directory/max_swap_space_error.test @@ -9,8 +9,7 @@ set temp_directory='__TEST_DIR__/max_swap_space_reached' # Ensure the temp_directory is used statement ok -PRAGMA memory_limit='2MB' - +PRAGMA memory_limit='1024KiB' # 0 blocks statement ok @@ -21,6 +20,11 @@ CREATE OR REPLACE TABLE t2 AS SELECT * FROM range(1000000); ---- failed to offload data block of size 256.0 KiB (0 bytes/0 bytes used) +query I +select "size" from duckdb_temporary_files() +---- +0 + # 1 block max statement ok set max_temp_directory_size='256KiB' @@ -34,3 +38,33 @@ statement error CREATE OR REPLACE TABLE t2 AS SELECT * FROM range(1000000); ---- failed to offload data block of size 256.0 KiB (256.0 KiB/256.0 KiB used) + +query I +select "size" from duckdb_temporary_files() +---- + +# 6 blocks +statement ok +set max_temp_directory_size='1536KiB' + +statement ok +pragma threads=2; + +statement ok +set preserve_insertion_order=true; + +# This is 1600000 bytes of BIGINT data, which is roughly 6.1 blocks +# Because our memory limit is set at 1MiB (4 blocks) this works +statement ok +CREATE OR REPLACE TABLE t2 AS SELECT * FROM range(200000); + +# When we increase the size to 2400000 bytes of BIGINT data (9.1 blocks) this errors +statement error +CREATE OR REPLACE TABLE t2 AS SELECT * FROM range(300000); +---- +failed to offload data block of size 256.0 KiB (1.5 MiB/1.5 MiB used) + +query I +select "size" from duckdb_temporary_files() +---- +1572864 From 0227e2d3ea730b465402d4b6b0d1583bfa7bfb52 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 8 Mar 2024 13:20:00 +0100 Subject: [PATCH 056/603] fix up comment --- src/include/duckdb/main/config.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/duckdb/main/config.hpp b/src/include/duckdb/main/config.hpp index ec2658afd2bb..9bedd5d6160f 100644 --- a/src/include/duckdb/main/config.hpp +++ b/src/include/duckdb/main/config.hpp @@ -149,7 +149,7 @@ struct DBConfigOptions { string autoinstall_extension_repo = ""; //! The maximum memory used by the database system (in bytes). Default: 80% of System available memory idx_t maximum_memory = (idx_t)-1; - //! The maximum size of the 'temp_directory' folder when set (in bytes). Default 5x 'maximum_memory' + //! The maximum size of the 'temp_directory' folder when set (in bytes) NumericSetting maximum_swap_space = NumericSetting(); //! The maximum amount of CPU threads used by the database system. Default: all available. idx_t maximum_threads = (idx_t)-1; From d56137adec45305b3a9c54f3ba8935d39504c975 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 8 Mar 2024 13:23:19 +0100 Subject: [PATCH 057/603] check if the config was set explicitly or not in DatabaseInstance::Configure --- src/main/database.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/database.cpp b/src/main/database.cpp index 7a0ef6fcc99c..39d554147dbc 100644 --- a/src/main/database.cpp +++ b/src/main/database.cpp @@ -333,7 +333,7 @@ void DatabaseInstance::Configure(DBConfig &new_config) { if (config.options.maximum_memory == (idx_t)-1) { config.SetDefaultMaxMemory(); } - if (config.options.maximum_swap_space == (idx_t)-1) { + if (!config.options.maximum_swap_space.ExplicitlySet()) { config.SetDefaultMaxSwapSpace(this); } if (new_config.options.maximum_threads == (idx_t)-1) { From 0a87b47e63197896bc3361a805d514840de3e1b8 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 8 Mar 2024 14:27:22 +0100 Subject: [PATCH 058/603] Make column idx 1-indexes --- .../scanner/string_value_scanner.cpp | 4 +- .../table_function/global_csv_state.cpp | 19 ++--- .../operator/csv_scanner/util/csv_error.cpp | 4 +- .../csv/rejects/csv_buffer_size_rejects.test | 8 +- .../copy/csv/rejects/csv_rejects_auto.test | 16 ++-- .../csv/rejects/csv_rejects_flush_cast.test | 4 +- .../csv/rejects/csv_rejects_maximum_line.test | 18 ++--- .../copy/csv/rejects/csv_rejects_read.test | 74 +++++++++---------- .../csv/rejects/csv_unquoted_rejects.test | 6 +- .../csv/rejects/test_invalid_utf_rejects.test | 8 +- 10 files changed, 81 insertions(+), 80 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index b2fc8baa8aa4..0dea719fb455 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -365,7 +365,7 @@ bool StringValueResult::HandleError() { if (current_error.is_set) { switch (current_error.type) { case CSVErrorType::TOO_MANY_COLUMNS: - HandleOverLimitRows(current_error.col_idx); + HandleOverLimitRows(cur_col_id); break; case CSVErrorType::INVALID_UNICODE: HandleUnicodeError(current_error.col_idx); @@ -510,7 +510,7 @@ bool StringValueResult::AddRowInternal() { auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); auto csv_error = CSVError::IncorrectColumnAmountError( - state_machine.options, cur_col_id, lines_per_batch, borked_line, + state_machine.options, cur_col_id - 1, lines_per_batch, borked_line, current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); error_handler.Error(csv_error); // If we are here we ignore_errors, so we delete this line diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index 6380d2e24c6c..1d012349da21 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -197,11 +197,6 @@ void CSVGlobalState::FillRejectsTable() { rejects->count++; auto row_line = file->error_handler->GetLine(error.error_info); auto col_idx = error.column_idx; - string col_name; - if (error.type != CSVErrorType::TOO_MANY_COLUMNS) { - // Too many columns does not have a name, all other errors have - col_name = bind_data.return_names[col_idx]; - } // Add the row to the rejects table appender.BeginRow(); // 1. File Path @@ -211,12 +206,18 @@ void CSVGlobalState::FillRejectsTable() { // 3. Byte Position where error occurred appender.Append(error.byte_position); // 4. Column Index - appender.Append(col_idx); + appender.Append(col_idx + 1); // 5. Column Name (If Applicable) - if (col_name.empty()) { + switch (error.type) { + case CSVErrorType::TOO_MANY_COLUMNS: appender.Append(Value()); - } else { - appender.Append(string_t("\"" + col_name + "\"")); + break; + case CSVErrorType::TOO_FEW_COLUMNS: + D_ASSERT(bind_data.return_names.size() > col_idx + 1); + appender.Append(string_t("\"" + bind_data.return_names[col_idx + 1] + "\"")); + break; + default: + appender.Append(string_t("\"" + bind_data.return_names[col_idx] + "\"")); } // 6. Error Type appender.Append(string_t(CSVErrorTypeToEnum(error.type))); diff --git a/src/execution/operator/csv_scanner/util/csv_error.cpp b/src/execution/operator/csv_scanner/util/csv_error.cpp index 9c1eaffd394f..fadfad6716ff 100644 --- a/src/execution/operator/csv_scanner/util/csv_error.cpp +++ b/src/execution/operator/csv_scanner/util/csv_error.cpp @@ -157,11 +157,11 @@ CSVError CSVError::IncorrectColumnAmountError(const CSVReaderOptions &options, i LinesPerBoundary error_info, string &csv_row, idx_t byte_position) { std::ostringstream error; // How many columns were expected and how many were found - error << "Expected Number of Columns: " << options.dialect_options.num_cols << " Found: " << actual_columns + error << "Expected Number of Columns: " << options.dialect_options.num_cols << " Found: " << actual_columns + 1 << std::endl; // What were the options error << options.ToString(); - if (actual_columns > options.dialect_options.num_cols) { + if (actual_columns >= options.dialect_options.num_cols) { return CSVError(error.str(), CSVErrorType::TOO_MANY_COLUMNS, actual_columns, csv_row, error_info, byte_position); } else { diff --git a/test/sql/copy/csv/rejects/csv_buffer_size_rejects.test b/test/sql/copy/csv/rejects/csv_buffer_size_rejects.test index a8fd11728781..76b95cfbe731 100644 --- a/test/sql/copy/csv/rejects/csv_buffer_size_rejects.test +++ b/test/sql/copy/csv/rejects/csv_buffer_size_rejects.test @@ -24,10 +24,10 @@ query IIIIII rowsort SELECT regexp_replace(file, '\\', '/', 'g'), column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table; ---- -test/sql/copy/csv/data/error/mismatch/big_bad.csv 0 "column0" CAST B, A 10875 -test/sql/copy/csv/data/error/mismatch/big_bad.csv 0 "column0" CAST C, A 20875 -test/sql/copy/csv/data/error/mismatch/big_bad2.csv 0 "column0" CAST B, A 18395 -test/sql/copy/csv/data/error/mismatch/big_bad2.csv 0 "column0" CAST C, A 28395 +test/sql/copy/csv/data/error/mismatch/big_bad.csv 1 "column0" CAST B, A 10875 +test/sql/copy/csv/data/error/mismatch/big_bad.csv 1 "column0" CAST C, A 20875 +test/sql/copy/csv/data/error/mismatch/big_bad2.csv 1 "column0" CAST B, A 18395 +test/sql/copy/csv/data/error/mismatch/big_bad2.csv 1 "column0" CAST C, A 28395 query I SELECT error_message diff --git a/test/sql/copy/csv/rejects/csv_rejects_auto.test b/test/sql/copy/csv/rejects/csv_rejects_auto.test index febda7d1c0fc..bfa8073a6567 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_auto.test +++ b/test/sql/copy/csv/rejects/csv_rejects_auto.test @@ -20,32 +20,32 @@ query IIIIIII rowsort SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table; ---- -test/sql/copy/csv/data/error/mismatch/big_bad.csv 2176 0 "column0" CAST B, A 10875 -test/sql/copy/csv/data/error/mismatch/big_bad.csv 4176 0 "column0" CAST C, A 20875 -test/sql/copy/csv/data/error/mismatch/big_bad2.csv 3680 0 "column0" CAST B, A 18395 -test/sql/copy/csv/data/error/mismatch/big_bad2.csv 5680 0 "column0" CAST C, A 28395 +test/sql/copy/csv/data/error/mismatch/big_bad.csv 2176 1 "column0" CAST B, A 10875 +test/sql/copy/csv/data/error/mismatch/big_bad.csv 4176 1 "column0" CAST C, A 20875 +test/sql/copy/csv/data/error/mismatch/big_bad2.csv 3680 1 "column0" CAST B, A 18395 +test/sql/copy/csv/data/error/mismatch/big_bad2.csv 5680 1 "column0" CAST C, A 28395 query I SELECT error_message -FROM csv_rejects_table where line=2176 and column_idx=0; +FROM csv_rejects_table where line=2176 and column_idx=1; ---- :.*Could not convert string "B" to 'BIGINT'.* query I SELECT error_message -FROM csv_rejects_table where line=4176 and column_idx=0; +FROM csv_rejects_table where line=4176 and column_idx=1; ---- :.*Could not convert string "C" to 'BIGINT'.* query I SELECT error_message -FROM csv_rejects_table where line=3680 and column_idx=0; +FROM csv_rejects_table where line=3680 and column_idx=1; ---- :.*Could not convert string "B" to 'BIGINT'.* query I SELECT error_message -FROM csv_rejects_table where line=5680 and column_idx=0; +FROM csv_rejects_table where line=5680 and column_idx=1; ---- :.*Could not convert string "C" to 'BIGINT'.* diff --git a/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test b/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test index 20ff320b36a5..69530026555e 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test +++ b/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test @@ -23,8 +23,8 @@ query IIIIIII rowsort SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table order by all; ---- -data/csv/error/flush_cast.csv 2813 0 "a" CAST c, bla 44971 -data/csv/error/flush_cast.csv 439 0 "a" CAST B, bla 6996 +data/csv/error/flush_cast.csv 2813 1 "a" CAST c, bla 44971 +data/csv/error/flush_cast.csv 439 1 "a" CAST B, bla 6996 query I SELECT error_message diff --git a/test/sql/copy/csv/rejects/csv_rejects_maximum_line.test b/test/sql/copy/csv/rejects/csv_rejects_maximum_line.test index f7bb5447485c..f6214aab0906 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_maximum_line.test +++ b/test/sql/copy/csv/rejects/csv_rejects_maximum_line.test @@ -18,7 +18,7 @@ query IIIIIII rowsort SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table; ---- -data/csv/rejects/maximum_line/max_10.csv 5 0 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 22 +data/csv/rejects/maximum_line/max_10.csv 5 1 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 22 statement ok DROP TABLE csv_rejects_table; @@ -38,7 +38,7 @@ query IIIIIII rowsort SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table; ---- -data/csv/rejects/maximum_line/max_10.csv 5 0 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 22 +data/csv/rejects/maximum_line/max_10.csv 5 1 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 22 statement ok DROP TABLE csv_rejects_table; @@ -57,9 +57,9 @@ query IIIIIII rowsort SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table; ---- -data/csv/rejects/maximum_line/over_vector.csv 2282 0 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 13684 -data/csv/rejects/maximum_line/over_vector.csv 2591 0 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 15557 -data/csv/rejects/maximum_line/over_vector.csv 2923 0 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,3 17568 +data/csv/rejects/maximum_line/over_vector.csv 2282 1 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 13684 +data/csv/rejects/maximum_line/over_vector.csv 2591 1 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 15557 +data/csv/rejects/maximum_line/over_vector.csv 2923 1 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,3 17568 statement ok DROP TABLE csv_rejects_table; @@ -77,10 +77,10 @@ query IIIIIII rowsort SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table; ---- -data/csv/rejects/maximum_line/max_10.csv 5 0 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 22 -data/csv/rejects/maximum_line/over_vector.csv 2282 0 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 13684 -data/csv/rejects/maximum_line/over_vector.csv 2591 0 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 15557 -data/csv/rejects/maximum_line/over_vector.csv 2923 0 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,3 17568 +data/csv/rejects/maximum_line/max_10.csv 5 1 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 22 +data/csv/rejects/maximum_line/over_vector.csv 2282 1 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 13684 +data/csv/rejects/maximum_line/over_vector.csv 2591 1 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 15557 +data/csv/rejects/maximum_line/over_vector.csv 2923 1 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,3 17568 statement ok DROP TABLE csv_rejects_table; \ No newline at end of file diff --git a/test/sql/copy/csv/rejects/csv_rejects_read.test b/test/sql/copy/csv/rejects/csv_rejects_read.test index 8cf3d5ac883f..9917965558ba 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_read.test +++ b/test/sql/copy/csv/rejects/csv_rejects_read.test @@ -20,7 +20,7 @@ query IIIIIII rowsort SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table; ---- -test/sql/copy/csv/data/error/mismatch/bad.csv 2 1 "col1" CAST 4,BBB,9, 9 +test/sql/copy/csv/data/error/mismatch/bad.csv 2 2 "col1" CAST 4,BBB,9, 9 query I SELECT error_message @@ -45,25 +45,25 @@ query IIIIIII rowsort SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table; ---- -test/sql/copy/csv/data/error/mismatch/bad2.csv 1 2 "col2" CAST 1,2,DDD, 0 -test/sql/copy/csv/data/error/mismatch/bad2.csv 3 0 "col0" CAST EEE,7,FFF, 16 -test/sql/copy/csv/data/error/mismatch/bad2.csv 3 2 "col2" CAST EEE,7,FFF, 16 +test/sql/copy/csv/data/error/mismatch/bad2.csv 1 3 "col2" CAST 1,2,DDD, 0 +test/sql/copy/csv/data/error/mismatch/bad2.csv 3 1 "col0" CAST EEE,7,FFF, 16 +test/sql/copy/csv/data/error/mismatch/bad2.csv 3 3 "col2" CAST EEE,7,FFF, 16 query I SELECT error_message -FROM csv_rejects_table where line=1 and column_idx=2; +FROM csv_rejects_table where line=1 and column_idx=3; ---- :.*Could not convert string "DDD" to 'INTEGER'.* query I SELECT error_message -FROM csv_rejects_table where line=3 and column_idx=0; +FROM csv_rejects_table where line=3 and column_idx=1; ---- :.*Could not convert string "EEE" to 'INTEGER'.* query I SELECT error_message -FROM csv_rejects_table where line=3 and column_idx=2; +FROM csv_rejects_table where line=3 and column_idx=3; ---- :.*Could not convert string "FFF" to 'INTEGER'.* @@ -88,18 +88,18 @@ query IIIIIII rowsort SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table; ---- -test/sql/copy/csv/data/error/mismatch/bad.csv 2 1 "col1" CAST 4,BBB,9, 9 -test/sql/copy/csv/data/error/mismatch/bad2.csv 3 0 "col0" CAST EEE,7,FFF, 16 +test/sql/copy/csv/data/error/mismatch/bad.csv 2 2 "col1" CAST 4,BBB,9, 9 +test/sql/copy/csv/data/error/mismatch/bad2.csv 3 1 "col0" CAST EEE,7,FFF, 16 query I SELECT error_message -FROM csv_rejects_table where line=2 and column_idx=1; +FROM csv_rejects_table where line=2 and column_idx=2; ---- :.*Could not convert string "BBB" to 'INTEGER'.* query I SELECT error_message -FROM csv_rejects_table where line=3 and column_idx=0; +FROM csv_rejects_table where line=3 and column_idx=1; ---- :.*Could not convert string "EEE" to 'INTEGER'.* @@ -144,18 +144,18 @@ query IIIIIII rowsort SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table; ---- -test/sql/copy/csv/data/error/mismatch/big_bad.csv 2176 0 "num" CAST B, A 10875 -test/sql/copy/csv/data/error/mismatch/big_bad.csv 4176 0 "num" CAST C, A 20875 +test/sql/copy/csv/data/error/mismatch/big_bad.csv 2176 1 "num" CAST B, A 10875 +test/sql/copy/csv/data/error/mismatch/big_bad.csv 4176 1 "num" CAST C, A 20875 query I SELECT error_message -FROM csv_rejects_table where line=2176 and column_idx=0; +FROM csv_rejects_table where line=2176 and column_idx=1; ---- :.*Could not convert string "B" to 'INTEGER'.* query I SELECT error_message -FROM csv_rejects_table where line=4176 and column_idx=0; +FROM csv_rejects_table where line=4176 and column_idx=1; ---- :.*Could not convert string "C" to 'INTEGER'.* @@ -175,18 +175,18 @@ query IIIIIII rowsort SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table; ---- -test/sql/copy/csv/data/error/mismatch/big_bad2.csv 3680 0 "num" CAST B, A 18395 -test/sql/copy/csv/data/error/mismatch/big_bad2.csv 5680 0 "num" CAST C, A 28395 +test/sql/copy/csv/data/error/mismatch/big_bad2.csv 3680 1 "num" CAST B, A 18395 +test/sql/copy/csv/data/error/mismatch/big_bad2.csv 5680 1 "num" CAST C, A 28395 query I SELECT error_message -FROM csv_rejects_table where line=3680 and column_idx=0; +FROM csv_rejects_table where line=3680 and column_idx=1; ---- :.*Could not convert string "B" to 'INTEGER'.* query I SELECT error_message -FROM csv_rejects_table where line=5680 and column_idx=0; +FROM csv_rejects_table where line=5680 and column_idx=1; ---- :.*Could not convert string "C" to 'INTEGER'.* @@ -207,32 +207,32 @@ query IIIIIII rowsort SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table; ---- -test/sql/copy/csv/data/error/mismatch/big_bad.csv 2176 0 "num" CAST B, A 10875 -test/sql/copy/csv/data/error/mismatch/big_bad.csv 4176 0 "num" CAST C, A 20875 -test/sql/copy/csv/data/error/mismatch/big_bad2.csv 3680 0 "num" CAST B, A 18395 -test/sql/copy/csv/data/error/mismatch/big_bad2.csv 5680 0 "num" CAST C, A 28395 +test/sql/copy/csv/data/error/mismatch/big_bad.csv 2176 1 "num" CAST B, A 10875 +test/sql/copy/csv/data/error/mismatch/big_bad.csv 4176 1 "num" CAST C, A 20875 +test/sql/copy/csv/data/error/mismatch/big_bad2.csv 3680 1 "num" CAST B, A 18395 +test/sql/copy/csv/data/error/mismatch/big_bad2.csv 5680 1 "num" CAST C, A 28395 query I SELECT error_message -FROM csv_rejects_table where line=3680 and column_idx=0; +FROM csv_rejects_table where line=3680 and column_idx=1; ---- :.*Could not convert string "B" to 'INTEGER'.* query I SELECT error_message -FROM csv_rejects_table where line=5680 and column_idx=0; +FROM csv_rejects_table where line=5680 and column_idx=1; ---- :.*Could not convert string "C" to 'INTEGER'.* query I SELECT error_message -FROM csv_rejects_table where line=2176 and column_idx=0; +FROM csv_rejects_table where line=2176 and column_idx=1; ---- :.*Could not convert string "B" to 'INTEGER'.* query I SELECT error_message -FROM csv_rejects_table where line=4176 and column_idx=0; +FROM csv_rejects_table where line=4176 and column_idx=1; ---- :.*Could not convert string "C" to 'INTEGER'.* @@ -261,37 +261,37 @@ query IIIIIII rowsort SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table_left; ---- -test/sql/copy/csv/data/error/mismatch/small1.csv 3 0 "num" CAST X,Y 14 -test/sql/copy/csv/data/error/mismatch/small1.csv 6 0 "num" CAST X,Y 26 +test/sql/copy/csv/data/error/mismatch/small1.csv 3 1 "num" CAST X,Y 14 +test/sql/copy/csv/data/error/mismatch/small1.csv 6 1 "num" CAST X,Y 26 query IIIIIII rowsort SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table_right; ---- -test/sql/copy/csv/data/error/mismatch/small2.csv 3 0 "num" CAST X,Y 14 -test/sql/copy/csv/data/error/mismatch/small2.csv 5 0 "num" CAST X,Y 22 +test/sql/copy/csv/data/error/mismatch/small2.csv 3 1 "num" CAST X,Y 14 +test/sql/copy/csv/data/error/mismatch/small2.csv 5 1 "num" CAST X,Y 22 query I SELECT error_message -FROM csv_rejects_table_left where line=3 and column_idx=0; +FROM csv_rejects_table_left where line=3 and column_idx=1; ---- :.*Could not convert string "X" to 'INTEGER'.* query I SELECT error_message -FROM csv_rejects_table_left where line=6 and column_idx=0; +FROM csv_rejects_table_left where line=6 and column_idx=1; ---- :.*Could not convert string "X" to 'INTEGER'.* query I SELECT error_message -FROM csv_rejects_table_right where line=3 and column_idx=0; +FROM csv_rejects_table_right where line=3 and column_idx=1; ---- :.*Could not convert string "X" to 'INTEGER'.* query I SELECT error_message -FROM csv_rejects_table_right where line=5 and column_idx=0; +FROM csv_rejects_table_right where line=5 and column_idx=1; ---- :.*Could not convert string "X" to 'INTEGER'.* @@ -326,8 +326,8 @@ query IIIIIII rowsort SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table_left; ---- -test/sql/copy/csv/data/error/mismatch/small1.csv 3 0 "num" CAST X,Y 14 -test/sql/copy/csv/data/error/mismatch/small1.csv 6 0 "num" CAST X,Y 26 +test/sql/copy/csv/data/error/mismatch/small1.csv 3 1 "num" CAST X,Y 14 +test/sql/copy/csv/data/error/mismatch/small1.csv 6 1 "num" CAST X,Y 26 query I SELECT COUNT(*) diff --git a/test/sql/copy/csv/rejects/csv_unquoted_rejects.test b/test/sql/copy/csv/rejects/csv_unquoted_rejects.test index 976b66d0ff05..0ce1d845df64 100644 --- a/test/sql/copy/csv/rejects/csv_unquoted_rejects.test +++ b/test/sql/copy/csv/rejects/csv_unquoted_rejects.test @@ -25,7 +25,7 @@ query IIIIIII rowsort SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table; ---- -data/csv/rejects/unquoted/basic.csv 5 0 "a" UNQUOTED VALUE "blaaaaaaaaaaaaaa"bla,4 28 +data/csv/rejects/unquoted/basic.csv 5 1 "a" UNQUOTED VALUE "blaaaaaaaaaaaaaa"bla,4 28 statement ok DROP TABLE csv_rejects_table; @@ -48,7 +48,7 @@ query IIIIII rowsort SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, byte_position FROM csv_rejects_table; ---- -data/csv/rejects/unquoted/unquoted_new_line.csv 5 0 "a" UNQUOTED VALUE 28 +data/csv/rejects/unquoted/unquoted_new_line.csv 5 1 "a" UNQUOTED VALUE 28 statement ok DROP TABLE csv_rejects_table; @@ -69,7 +69,7 @@ query IIIIII rowsort SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, byte_position FROM csv_rejects_table; ---- -data/csv/rejects/unquoted/unquoted_last_value.csv 5 0 "a" UNQUOTED VALUE 31 +data/csv/rejects/unquoted/unquoted_last_value.csv 5 1 "a" UNQUOTED VALUE 31 statement ok DROP TABLE csv_rejects_table; diff --git a/test/sql/copy/csv/rejects/test_invalid_utf_rejects.test b/test/sql/copy/csv/rejects/test_invalid_utf_rejects.test index 52ff0ac19823..94c56cc71562 100644 --- a/test/sql/copy/csv/rejects/test_invalid_utf_rejects.test +++ b/test/sql/copy/csv/rejects/test_invalid_utf_rejects.test @@ -15,7 +15,7 @@ query IIIIIII rowsort SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position FROM csv_rejects_table; ---- -test/sql/copy/csv/data/test/invalid_utf_big.csv 3001 2 "col3" INVALID UNICODE valid,invalid_??_part,valid 54000 -test/sql/copy/csv/data/test/invalid_utf_big.csv 3012 2 "col3" INVALID UNICODE valid,valid,invalid_??_part 54208 -test/sql/copy/csv/data/test/invalid_utf_big.csv 3023 2 "col3" INVALID UNICODE valid,invalid_??_part,valid 54416 -test/sql/copy/csv/data/test/invalid_utf_big.csv 3034 2 "col3" INVALID UNICODE valid,valid,invalid_??_part 54624 +test/sql/copy/csv/data/test/invalid_utf_big.csv 3001 2 "col2" INVALID UNICODE valid,invalid_??_part,valid 54000 +test/sql/copy/csv/data/test/invalid_utf_big.csv 3012 3 "col3" INVALID UNICODE valid,valid,invalid_??_part 54208 +test/sql/copy/csv/data/test/invalid_utf_big.csv 3023 2 "col2" INVALID UNICODE valid,invalid_??_part,valid 54416 +test/sql/copy/csv/data/test/invalid_utf_big.csv 3034 3 "col3" INVALID UNICODE valid,valid,invalid_??_part 54624 \ No newline at end of file From ba1da6a688803300c39610fe03be73b145e86af4 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 8 Mar 2024 14:38:32 +0100 Subject: [PATCH 059/603] Handle invalid states in overbuffer values --- .../scanner/string_value_scanner.cpp | 6 +++ .../csv_scanner/csv_state_machine.hpp | 4 ++ .../csv/rejects/csv_unquoted_rejects.test | 47 ++++++++++--------- 3 files changed, 36 insertions(+), 21 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 0dea719fb455..ffdf15c5d523 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -926,6 +926,9 @@ void StringValueScanner::ProcessOverbufferValue() { if (states.IsEscaped()) { result.escaped = true; } + if (states.IsInvalid()) { + result.InvalidState(result); + } j++; } if (overbuffer_string.empty() && @@ -955,6 +958,9 @@ void StringValueScanner::ProcessOverbufferValue() { if (states.IsEscaped()) { result.escaped = true; } + if (states.IsInvalid()) { + result.InvalidState(result); + } j++; } string_t value; diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_state_machine.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_state_machine.hpp index 49542782f56a..a1628e100f63 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_state_machine.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_state_machine.hpp @@ -54,6 +54,10 @@ struct CSVStates { return states[1] == CSVState::CARRIAGE_RETURN; } + inline bool IsInvalid() { + return states[1] == CSVState::INVALID; + } + inline bool IsQuoted() { return states[0] == CSVState::QUOTED; } diff --git a/test/sql/copy/csv/rejects/csv_unquoted_rejects.test b/test/sql/copy/csv/rejects/csv_unquoted_rejects.test index 0ce1d845df64..13c13b8b9fa7 100644 --- a/test/sql/copy/csv/rejects/csv_unquoted_rejects.test +++ b/test/sql/copy/csv/rejects/csv_unquoted_rejects.test @@ -74,24 +74,29 @@ data/csv/rejects/unquoted/unquoted_last_value.csv 5 1 "a" UNQUOTED VALUE 31 statement ok DROP TABLE csv_rejects_table; -# Test buffer sizes (borked :( ) -# -#loop buffer_size 35 1001 -# -#statement ok -#SELECT * FROM read_csv( -# 'data/csv/rejects/unquoted/basic.csv', -# columns = {'a': 'VARCHAR', 'b': 'INTEGER'}, -# rejects_table='csv_rejects_table', -# ignore_errors=true, auto_detect=false, header = 1, quote = '"', escape = '"', buffer_size=${buffer_size}); -# -#query IIIIIII rowsort -#SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position -#FROM csv_rejects_table; -#---- -#data/csv/rejects/unquoted/basic.csv 5 0 "a" UNQUOTED VALUE "blaaaaaaaaaaaaaa"bla,4 28 -# -#statement ok -#DROP TABLE csv_rejects_table; -# -#endloop \ No newline at end of file +loop buffer_size 35 40 + +query II +SELECT * FROM read_csv( + 'data/csv/rejects/unquoted/basic.csv', + columns = {'a': 'VARCHAR', 'b': 'INTEGER'}, + rejects_table='csv_rejects_table', buffer_size=${buffer_size}, + ignore_errors=true, auto_detect=false, header = 1, quote = '"', escape = '"', buffer_size=35); +---- +bla 1 +bla 2 +bla 3 +bla 1 +bla 2 +bla 3 + +query IIIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position +FROM csv_rejects_table; +---- +data/csv/rejects/unquoted/basic.csv 5 1 "a" UNQUOTED VALUE "blaaaaaaaaaaaaaa"bla,4 28 + +statement ok +DROP TABLE csv_rejects_table; + +endloop \ No newline at end of file From b37d1932f5e1778daacaccc47376f263c4772cb8 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 8 Mar 2024 15:02:10 +0100 Subject: [PATCH 060/603] avoid modifying the passed in DBConfig --- src/include/duckdb/main/database.hpp | 2 +- src/main/database.cpp | 45 +++---- src/storage/temporary_file_manager.cpp | 2 +- .../max_swap_space_inmemory.test | 112 ++++++++++++++++++ ...ce.test => max_swap_space_persistent.test} | 11 +- 5 files changed, 147 insertions(+), 25 deletions(-) create mode 100644 test/sql/storage/temp_directory/max_swap_space_inmemory.test rename test/sql/storage/temp_directory/{max_swap_space.test => max_swap_space_persistent.test} (90%) diff --git a/src/include/duckdb/main/database.hpp b/src/include/duckdb/main/database.hpp index a5798c8059b5..8bef82e99acd 100644 --- a/src/include/duckdb/main/database.hpp +++ b/src/include/duckdb/main/database.hpp @@ -62,7 +62,7 @@ class DatabaseInstance : public std::enable_shared_from_this { void Initialize(const char *path, DBConfig *config); void CreateMainDatabase(); - void Configure(DBConfig &config); + void Configure(DBConfig &config, const char *path); private: unique_ptr buffer_manager; diff --git a/src/main/database.cpp b/src/main/database.cpp index 39d554147dbc..5b3c04db2983 100644 --- a/src/main/database.cpp +++ b/src/main/database.cpp @@ -185,27 +185,7 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf config_ptr = user_config; } - if (config_ptr->options.duckdb_api.empty()) { - config_ptr->SetOptionByName("duckdb_api", "cpp"); - } - - if (config_ptr->options.temporary_directory.empty() && database_path) { - // no directory specified: use default temp path - config_ptr->options.temporary_directory = string(database_path) + ".tmp"; - - // special treatment for in-memory mode - if (strcmp(database_path, IN_MEMORY_PATH) == 0) { - config_ptr->options.temporary_directory = ".tmp"; - } - } - - if (database_path) { - config_ptr->options.database_path = database_path; - } else { - config_ptr->options.database_path.clear(); - } - - Configure(*config_ptr); + Configure(*config_ptr, database_path); if (user_config && !user_config->options.use_temporary_directory) { // temporary directories explicitly disabled @@ -316,8 +296,29 @@ Allocator &Allocator::Get(AttachedDatabase &db) { return Allocator::Get(db.GetDatabase()); } -void DatabaseInstance::Configure(DBConfig &new_config) { +void DatabaseInstance::Configure(DBConfig &new_config, const char *database_path) { config.options = new_config.options; + + if (new_config.options.duckdb_api.empty()) { + config.SetOptionByName("duckdb_api", "cpp"); + } + + if (new_config.options.temporary_directory.empty() && database_path) { + // no directory specified: use default temp path + config.options.temporary_directory = string(database_path) + ".tmp"; + + // special treatment for in-memory mode + if (strcmp(database_path, IN_MEMORY_PATH) == 0) { + config.options.temporary_directory = ".tmp"; + } + } + + if (database_path) { + config.options.database_path = database_path; + } else { + config.options.database_path.clear(); + } + if (config.options.access_mode == AccessMode::UNDEFINED) { config.options.access_mode = AccessMode::READ_WRITE; } diff --git a/src/storage/temporary_file_manager.cpp b/src/storage/temporary_file_manager.cpp index 0fda4781124b..d4fa09470596 100644 --- a/src/storage/temporary_file_manager.cpp +++ b/src/storage/temporary_file_manager.cpp @@ -52,7 +52,7 @@ bool BlockIndexManager::RemoveIndex(idx_t index) { if (max_index_in_use < max_index) { // max index in use is lower than the max_index // reduce the max_index - max_index = indexes_in_use.empty() ? 0 : max_index_in_use + 1; + max_index = max_index_in_use; if (file_size_monitor) { file_size_monitor->Decrease(old_max - max_index); } diff --git a/test/sql/storage/temp_directory/max_swap_space_inmemory.test b/test/sql/storage/temp_directory/max_swap_space_inmemory.test new file mode 100644 index 000000000000..9de30a592cb0 --- /dev/null +++ b/test/sql/storage/temp_directory/max_swap_space_inmemory.test @@ -0,0 +1,112 @@ +# name: test/sql/storage/temp_directory/max_swap_space_inmemory.test +# group: [temp_directory] + +require skip_reload + +# In in-memory mode, the 'temp_directory' defaults to '.tmp' +# So there are no guarantees about the value of 'max_temp_directory_size' +# (if .tmp exists, it's set to the available disk space | if it doesn't exist, it'll stay at 0 bytes) + +statement ok +set temp_directory=''; + +statement ok +PRAGMA memory_limit='2MB' + +# --- Set by default to 0 when temp_directory is not set --- + +# If 'temp_directory' is not set, this defaults to 0 +query I +select current_setting('max_temp_directory_size') +---- +0 bytes + +# Set the max size explicitly +statement ok +set max_temp_directory_size='15gb' + +# Should not be 0 anymore +query I +select current_setting('max_temp_directory_size') a where a == '0 bytes' +---- + +# Then reset it, default should be 0 again +statement ok +reset max_temp_directory_size; + +query I +select current_setting('max_temp_directory_size') +---- +0 bytes + +# --- Set by default to the available disk space when temp_directory exists --- + +statement ok +set temp_directory = '__TEST_DIR__'; + +# '__TEST_DIR__' is guaranteed to exist, we can get the disk space +query I +select current_setting('max_temp_directory_size') a where a == '0 bytes' +---- + +# --- Set explicitly by the user --- + +# If we set 'max_temp_directory_size' explicitly, it will not be overridden +statement ok +set max_temp_directory_size='15gb' + +# Reported size should not be 0, we set it explicitly +query I +select current_setting('max_temp_directory_size') a where a == '0 bytes' +---- + +query I nosort unchanged +select current_setting('max_temp_directory_size') +---- + +# When we change the temp_directory to something that doesnt exist +statement ok +set temp_directory = '__TEST_DIR__/does_not_exist' + +query I nosort unchanged +select current_setting('max_temp_directory_size') +---- + +# When we change the temp_directory to something that does exist +statement ok +set temp_directory = '__TEST_DIR__' + +query I nosort unchanged +select current_setting('max_temp_directory_size') +---- + +# When we reset the temp_directory .. +statement ok +reset temp_directory; + +query I nosort unchanged +select current_setting('max_temp_directory_size') +---- + +# --- Set to the available disk space when we create the (previously non-existant) 'temp_directory' + +# Reset it so it's no longer set explicitly +statement ok +reset max_temp_directory_size; + +# When we change the temp_directory to something that doesnt exist +statement ok +set temp_directory = '__TEST_DIR__/does_not_exist' + +query I +select current_setting('max_temp_directory_size') +---- +0 bytes + +statement ok +CREATE TABLE t2 AS SELECT * FROM range(1000000); + +# Reported size should not be 0, the directory was created +query I +select current_setting('max_temp_directory_size') a where a == '0 bytes' +---- diff --git a/test/sql/storage/temp_directory/max_swap_space.test b/test/sql/storage/temp_directory/max_swap_space_persistent.test similarity index 90% rename from test/sql/storage/temp_directory/max_swap_space.test rename to test/sql/storage/temp_directory/max_swap_space_persistent.test index 0c93cd2852c9..354c65215e3a 100644 --- a/test/sql/storage/temp_directory/max_swap_space.test +++ b/test/sql/storage/temp_directory/max_swap_space_persistent.test @@ -1,8 +1,17 @@ -# name: test/sql/storage/temp_directory/max_swap_space.test +# name: test/sql/storage/temp_directory/max_swap_space_persistent.test # group: [temp_directory] require skip_reload +# Create a persistent database +load __TEST_DIR__/max_swap_space.db + +## If 'temp_directory' is not set, this defaults to 0 +#query I +#select current_setting('max_temp_directory_size') +#---- +#0 bytes + statement ok set temp_directory=''; From 1491dd798ed82e25b052301f700be3c90ee28202 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 8 Mar 2024 15:11:49 +0100 Subject: [PATCH 061/603] fix up some behavior --- src/main/settings/settings.cpp | 7 ++----- .../storage/temp_directory/max_swap_space_inmemory.test | 4 ++-- .../storage/temp_directory/max_swap_space_persistent.test | 6 ++++++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/main/settings/settings.cpp b/src/main/settings/settings.cpp index 90ef20522a24..1eb7ce7eb030 100644 --- a/src/main/settings/settings.cpp +++ b/src/main/settings/settings.cpp @@ -1231,11 +1231,8 @@ Value SecretDirectorySetting::GetSetting(ClientContext &context) { void TempDirectorySetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) { config.options.temporary_directory = input.ToString(); config.options.use_temporary_directory = !config.options.temporary_directory.empty(); - if (!config.options.temporary_directory.empty()) { - // Maximum swap space isn't set explicitly, initialize to default - if (!config.options.maximum_swap_space.ExplicitlySet()) { - config.SetDefaultMaxSwapSpace(db); - } + if (!config.options.maximum_swap_space.ExplicitlySet()) { + config.SetDefaultMaxSwapSpace(db); } if (db) { auto &buffer_manager = BufferManager::GetBufferManager(*db); diff --git a/test/sql/storage/temp_directory/max_swap_space_inmemory.test b/test/sql/storage/temp_directory/max_swap_space_inmemory.test index 9de30a592cb0..e6aad865c921 100644 --- a/test/sql/storage/temp_directory/max_swap_space_inmemory.test +++ b/test/sql/storage/temp_directory/max_swap_space_inmemory.test @@ -7,15 +7,15 @@ require skip_reload # So there are no guarantees about the value of 'max_temp_directory_size' # (if .tmp exists, it's set to the available disk space | if it doesn't exist, it'll stay at 0 bytes) +# So we set it explicitly to empty, to have guarantees statement ok set temp_directory=''; statement ok PRAGMA memory_limit='2MB' -# --- Set by default to 0 when temp_directory is not set --- +# --- Set by default to 0 when temp_directory is empty --- -# If 'temp_directory' is not set, this defaults to 0 query I select current_setting('max_temp_directory_size') ---- diff --git a/test/sql/storage/temp_directory/max_swap_space_persistent.test b/test/sql/storage/temp_directory/max_swap_space_persistent.test index 354c65215e3a..aa18759f7a83 100644 --- a/test/sql/storage/temp_directory/max_swap_space_persistent.test +++ b/test/sql/storage/temp_directory/max_swap_space_persistent.test @@ -6,6 +6,12 @@ require skip_reload # Create a persistent database load __TEST_DIR__/max_swap_space.db +# Default temp_directory for a persistent database is .tmp +query I +select current_setting('temp_directory').split('/')[-1] +---- +max_swap_space.db.tmp + ## If 'temp_directory' is not set, this defaults to 0 #query I #select current_setting('max_temp_directory_size') From 9aa5d2a2088fad1da5b337677f777f0513ebc7fe Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 8 Mar 2024 15:22:22 +0100 Subject: [PATCH 062/603] Add a mixed test with a bunch of different errors --- data/csv/rejects/frankstein/nightmare.csv | 48 +++++++++++++ .../scanner/string_value_scanner.cpp | 1 + .../operator/csv_scanner/util/csv_error.cpp | 50 +++++++------- .../operator/csv_scanner/csv_error.hpp | 4 +- test/sql/copy/csv/rejects/test_mixed.test | 68 +++++++++++++++++++ 5 files changed, 144 insertions(+), 27 deletions(-) create mode 100644 data/csv/rejects/frankstein/nightmare.csv create mode 100644 test/sql/copy/csv/rejects/test_mixed.test diff --git a/data/csv/rejects/frankstein/nightmare.csv b/data/csv/rejects/frankstein/nightmare.csv new file mode 100644 index 000000000000..579f46a359b7 --- /dev/null +++ b/data/csv/rejects/frankstein/nightmare.csv @@ -0,0 +1,48 @@ +a,b,c +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" +1,2 +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" +1,2,"pedro",5 +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" +1,bla,"pedro" +1,2,"pedro" +1,2,"pedro" +1,2,"pedro"bla +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" +1,2,"pedro thiago timbo holanda" +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" +1,2,"pedroÿÿ" +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" +1,2,"pedro" diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index ffdf15c5d523..e568a904b19c 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -450,6 +450,7 @@ bool StringValueResult::AddRowInternal() { CSVError::LineSizeError(state_machine.options, current_line_size, lines_per_batch, borked_line, current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); error_handler.Error(csv_error); + number_of_rows--; } if (HandleError()) { return false; diff --git a/src/execution/operator/csv_scanner/util/csv_error.cpp b/src/execution/operator/csv_scanner/util/csv_error.cpp index fadfad6716ff..cfab97c9e55d 100644 --- a/src/execution/operator/csv_scanner/util/csv_error.cpp +++ b/src/execution/operator/csv_scanner/util/csv_error.cpp @@ -19,7 +19,11 @@ void CSVErrorHandler::ThrowError(CSVError csv_error) { if (PrintLineNumber(csv_error)) { error << "CSV Error on Line: " << GetLine(csv_error.error_info) << std::endl; } - error << csv_error.error_message; + if (csv_error.error_message_with_options.empty()){ + error << csv_error.error_message; + } else{ + error << csv_error.error_message_with_options; + } switch (csv_error.type) { case CSVErrorType::CAST_ERROR: throw ConversionException(error.str()); @@ -78,9 +82,15 @@ CSVError::CSVError(string error_message_p, CSVErrorType type_p, LinesPerBoundary } CSVError::CSVError(string error_message_p, CSVErrorType type_p, idx_t column_idx_p, string csv_row_p, - LinesPerBoundary error_info_p, idx_t byte_position_p) + LinesPerBoundary error_info_p, idx_t byte_position_p, const CSVReaderOptions &reader_options) : error_message(std::move(error_message_p)), type(type_p), column_idx(column_idx_p), csv_row(std::move(csv_row_p)), error_info(error_info_p), byte_position(byte_position_p) { + // What were the options + std::ostringstream error; + error << error_message << std::endl; + error << reader_options.ToString(); + error << std::endl; + error_message_with_options = error.str(); } CSVError CSVError::ColumnTypesError(case_insensitive_map_t sql_types_per_column, const vector &names) { @@ -107,22 +117,18 @@ CSVError CSVError::CastError(const CSVReaderOptions &options, string &column_nam string &csv_row, LinesPerBoundary error_info, idx_t byte_position) { std::ostringstream error; // Which column - error << "Error when converting column \"" << column_name << "\"." << std::endl; + error << "Error when converting column \"" << column_name << "\". "; // What was the cast error - error << cast_error << std::endl; - error << std::endl; - // What were the options - error << options.ToString(); - return CSVError(error.str(), CSVErrorType::CAST_ERROR, column_idx, csv_row, error_info, byte_position); + error << cast_error; + return CSVError(error.str(), CSVErrorType::CAST_ERROR, column_idx, csv_row, error_info, byte_position, options); } CSVError CSVError::LineSizeError(const CSVReaderOptions &options, idx_t actual_size, LinesPerBoundary error_info, string &csv_row, idx_t byte_position) { std::ostringstream error; error << "Maximum line size of " << options.maximum_line_size << " bytes exceeded. "; - error << "Actual Size:" << actual_size << " bytes." << std::endl; - error << options.ToString(); - return CSVError(error.str(), CSVErrorType::MAXIMUM_LINE_SIZE, 0, csv_row, error_info, byte_position); + error << "Actual Size:" << actual_size << " bytes."; + return CSVError(error.str(), CSVErrorType::MAXIMUM_LINE_SIZE, 0, csv_row, error_info, byte_position, options); } CSVError CSVError::SniffingError(string &file_path) { @@ -146,26 +152,20 @@ CSVError CSVError::NullPaddingFail(const CSVReaderOptions &options, LinesPerBoun CSVError CSVError::UnterminatedQuotesError(const CSVReaderOptions &options, idx_t current_column, LinesPerBoundary error_info, string &csv_row, idx_t byte_position) { std::ostringstream error; - error << "Value with unterminated quote found." << std::endl; - error << std::endl; - // What were the options - error << options.ToString(); - return CSVError(error.str(), CSVErrorType::UNTERMINATED_QUOTES, current_column, csv_row, error_info, byte_position); + error << "Value with unterminated quote found."; + return CSVError(error.str(), CSVErrorType::UNTERMINATED_QUOTES, current_column, csv_row, error_info, byte_position, options); } CSVError CSVError::IncorrectColumnAmountError(const CSVReaderOptions &options, idx_t actual_columns, LinesPerBoundary error_info, string &csv_row, idx_t byte_position) { std::ostringstream error; // How many columns were expected and how many were found - error << "Expected Number of Columns: " << options.dialect_options.num_cols << " Found: " << actual_columns + 1 - << std::endl; - // What were the options - error << options.ToString(); + error << "Expected Number of Columns: " << options.dialect_options.num_cols << " Found: " << actual_columns + 1; if (actual_columns >= options.dialect_options.num_cols) { return CSVError(error.str(), CSVErrorType::TOO_MANY_COLUMNS, actual_columns, csv_row, error_info, - byte_position); + byte_position, options); } else { - return CSVError(error.str(), CSVErrorType::TOO_FEW_COLUMNS, actual_columns, csv_row, error_info, byte_position); + return CSVError(error.str(), CSVErrorType::TOO_FEW_COLUMNS, actual_columns, csv_row, error_info, byte_position, options); } } @@ -173,10 +173,8 @@ CSVError CSVError::InvalidUTF8(const CSVReaderOptions &options, idx_t current_co string &csv_row, idx_t byte_position) { std::ostringstream error; // How many columns were expected and how many were found - error << "Invalid unicode (byte sequence mismatch) detected." << std::endl; - // What were the options - error << options.ToString(); - return CSVError(error.str(), CSVErrorType::INVALID_UNICODE, current_column, csv_row, error_info, byte_position); + error << "Invalid unicode (byte sequence mismatch) detected."; + return CSVError(error.str(), CSVErrorType::INVALID_UNICODE, current_column, csv_row, error_info, byte_position, options); } bool CSVErrorHandler::PrintLineNumber(CSVError &error) { diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp index befc3a669219..98f460127d83 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp @@ -52,7 +52,7 @@ class CSVError { public: CSVError() {}; CSVError(string error_message, CSVErrorType type, idx_t column_idx, string csv_row, LinesPerBoundary error_info, - idx_t byte_position); + idx_t byte_position, const CSVReaderOptions &reader_options); CSVError(string error_message, CSVErrorType type, LinesPerBoundary error_info); //! Produces error messages for column name -> type mismatch. static CSVError ColumnTypesError(case_insensitive_map_t sql_types_per_column, const vector &names); @@ -81,6 +81,8 @@ class CSVError { //! Actual error message string error_message; + //! Actual error message + string error_message_with_options; //! Error Type CSVErrorType type; //! Column Index where error happened diff --git a/test/sql/copy/csv/rejects/test_mixed.test b/test/sql/copy/csv/rejects/test_mixed.test new file mode 100644 index 000000000000..45001a5e4b05 --- /dev/null +++ b/test/sql/copy/csv/rejects/test_mixed.test @@ -0,0 +1,68 @@ +# name: test/sql/copy/csv/rejects/test_mixed.test +# description: Tests a mix of all possible CSV Errors +# group: [rejects] + +require skip_reload + +# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n +require notwindows + +query III +SELECT * FROM read_csv( + 'data/csv/rejects/frankstein/nightmare.csv', + columns = {'a': 'INTEGER', 'b': 'INTEGER', 'c': 'VARCHAR'}, + rejects_table='csv_rejects_table', + ignore_errors=true, auto_detect=false, header = 1, max_line_size=20); +---- +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro +1 2 pedro + +query IIIIIIII rowsort +SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position, error_message +FROM csv_rejects_table; +---- +data/csv/rejects/frankstein/nightmare.csv 10 2 "c" MISSING COLUMNS 1,2 102 Expected Number of Columns: 3 Found: 2 +data/csv/rejects/frankstein/nightmare.csv 14 4 NULL TOO MANY COLUMNS 1,2,"pedro",5 142 Expected Number of Columns: 3 Found: 4 +data/csv/rejects/frankstein/nightmare.csv 19 2 "b" CAST 1,bla,"pedro" 204 Error when converting column "b". Could not convert string "bla" to 'INTEGER' +data/csv/rejects/frankstein/nightmare.csv 22 3 "c" UNQUOTED VALUE 1,2,"pedro"bla 242 Value with unterminated quote found. +data/csv/rejects/frankstein/nightmare.csv 32 1 "a" LINE SIZE OVER MAXIMUM 1,2,"pedro thiago timbo holanda" 365 Maximum line size of 20 bytes exceeded. Actual Size:33 bytes. +data/csv/rejects/frankstein/nightmare.csv 38 3 "c" INVALID UNICODE 1,2,"pedro??" 458 Invalid unicode (byte sequence mismatch) detected. \ No newline at end of file From 8b2b5cd0dde552c03abbcaa28585ad1d21eee22a Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 8 Mar 2024 15:26:58 +0100 Subject: [PATCH 063/603] make the in-memory database detection better --- src/main/database.cpp | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/main/database.cpp b/src/main/database.cpp index 5b3c04db2983..d8dd6cca8212 100644 --- a/src/main/database.cpp +++ b/src/main/database.cpp @@ -296,6 +296,24 @@ Allocator &Allocator::Get(AttachedDatabase &db) { return Allocator::Get(db.GetDatabase()); } +static bool IsInMemoryDatabase(const char *database_path) { + if (!database_path) { + // Entirely empty + return true; + } + if (strlen(database_path) == 0) { + // '' empty string + return true; + } + constexpr const char *IN_MEMORY_PATH_PREFIX = ":memory:"; + const idx_t PREFIX_LENGTH = strlen(IN_MEMORY_PATH_PREFIX); + if (strncmp(database_path, IN_MEMORY_PATH_PREFIX, PREFIX_LENGTH) == 0) { + // Starts with :memory:, i.e ':memory:named_conn' is valid + return true; + } + return false; +} + void DatabaseInstance::Configure(DBConfig &new_config, const char *database_path) { config.options = new_config.options; @@ -308,7 +326,7 @@ void DatabaseInstance::Configure(DBConfig &new_config, const char *database_path config.options.temporary_directory = string(database_path) + ".tmp"; // special treatment for in-memory mode - if (strcmp(database_path, IN_MEMORY_PATH) == 0) { + if (IsInMemoryDatabase(database_path)) { config.options.temporary_directory = ".tmp"; } } From 96fc46e187f3fed630364342829a199baffd7dbf Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 8 Mar 2024 19:58:37 +0100 Subject: [PATCH 064/603] initialize temp_directory to '.tmp' for every version of in-memory connection, add test for the behavior --- src/main/database.cpp | 7 +++---- test/api/test_api.cpp | 10 ++++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/main/database.cpp b/src/main/database.cpp index d8dd6cca8212..ac8df84f555f 100644 --- a/src/main/database.cpp +++ b/src/main/database.cpp @@ -321,13 +321,12 @@ void DatabaseInstance::Configure(DBConfig &new_config, const char *database_path config.SetOptionByName("duckdb_api", "cpp"); } - if (new_config.options.temporary_directory.empty() && database_path) { + if (new_config.options.temporary_directory.empty()) { // no directory specified: use default temp path - config.options.temporary_directory = string(database_path) + ".tmp"; - - // special treatment for in-memory mode if (IsInMemoryDatabase(database_path)) { config.options.temporary_directory = ".tmp"; + } else { + config.options.temporary_directory = string(database_path) + ".tmp"; } } diff --git a/test/api/test_api.cpp b/test/api/test_api.cpp index ca41df7e3acc..5d17b2710e46 100644 --- a/test/api/test_api.cpp +++ b/test/api/test_api.cpp @@ -139,6 +139,16 @@ static void parallel_query(Connection *conn, bool *correct, size_t threadnr) { } } +TEST_CASE("Test temp_directory defaults", "[api][.]") { + const char *db_paths[] = {nullptr, "", ":memory:", ":memory:named_conn"}; + for (auto &path : db_paths) { + auto db = make_uniq(path); + auto conn = make_uniq(*db); + + REQUIRE(db->instance->config.options.temporary_directory == ".tmp"); + } +} + TEST_CASE("Test parallel usage of single client", "[api][.]") { auto db = make_uniq(nullptr); auto conn = make_uniq(*db); From 8ad835366e73b6bfa67e2725ae5bb5f62b6abbca Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 8 Mar 2024 20:06:31 +0100 Subject: [PATCH 065/603] use 90% of the available disk space by default --- src/main/config.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/config.cpp b/src/main/config.cpp index 9aa8c50712cb..af47bd0d1623 100644 --- a/src/main/config.cpp +++ b/src/main/config.cpp @@ -276,7 +276,9 @@ void DBConfig::SetDefaultMaxSwapSpace(optional_ptr db) { } // Use the available disk space if temp directory is set auto disk_space = FileSystem::GetAvailableDiskSpace(options.temporary_directory); - options.maximum_swap_space.SetDefault(disk_space); + // Only use 90% of the available disk space + auto default_value = disk_space == DConstants::INVALID_INDEX ? 0 : static_cast(disk_space) * 0.9; + options.maximum_swap_space.SetDefault(default_value); } void DBConfig::CheckLock(const string &name) { From 29678c5af439bf68339f5b4ac9d4553750132093 Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 11 Mar 2024 10:19:03 +0100 Subject: [PATCH 066/603] RESET temp_directory should use the same behavior as DatabaseInstance::Configure --- src/include/duckdb/main/config.hpp | 2 ++ src/main/config.cpp | 26 +++++++++++++++++++++++++ src/main/database.cpp | 31 ++++-------------------------- src/main/settings/settings.cpp | 3 ++- 4 files changed, 34 insertions(+), 28 deletions(-) diff --git a/src/include/duckdb/main/config.hpp b/src/include/duckdb/main/config.hpp index 9bedd5d6160f..fcfda6ea8c40 100644 --- a/src/include/duckdb/main/config.hpp +++ b/src/include/duckdb/main/config.hpp @@ -278,6 +278,7 @@ struct DBConfig { DUCKDB_API static vector GetOptions(); DUCKDB_API static idx_t GetOptionCount(); DUCKDB_API static vector GetOptionNames(); + DUCKDB_API static bool IsInMemoryDatabase(const char *database_path); DUCKDB_API void AddExtensionOption(const string &name, string description, LogicalType parameter, const Value &default_value = Value(), set_option_callback_t function = nullptr); @@ -310,6 +311,7 @@ struct DBConfig { static idx_t GetSystemMaxThreads(FileSystem &fs); void SetDefaultMaxMemory(); void SetDefaultMaxSwapSpace(optional_ptr db); + void SetDefaultTempDirectory(); OrderType ResolveOrder(OrderType order_type) const; OrderByNullType ResolveNullOrder(OrderType order_type, OrderByNullType null_type) const; diff --git a/src/main/config.cpp b/src/main/config.cpp index af47bd0d1623..33310d20b1a7 100644 --- a/src/main/config.cpp +++ b/src/main/config.cpp @@ -245,6 +245,24 @@ void DBConfig::AddExtensionOption(const string &name, string description, Logica } } +bool DBConfig::IsInMemoryDatabase(const char *database_path) { + if (!database_path) { + // Entirely empty + return true; + } + if (strlen(database_path) == 0) { + // '' empty string + return true; + } + constexpr const char *IN_MEMORY_PATH_PREFIX = ":memory:"; + const idx_t PREFIX_LENGTH = strlen(IN_MEMORY_PATH_PREFIX); + if (strncmp(database_path, IN_MEMORY_PATH_PREFIX, PREFIX_LENGTH) == 0) { + // Starts with :memory:, i.e ':memory:named_conn' is valid + return true; + } + return false; +} + CastFunctionSet &DBConfig::GetCastFunctions() { return *cast_functions; } @@ -260,6 +278,14 @@ void DBConfig::SetDefaultMaxMemory() { } } +void DBConfig::SetDefaultTempDirectory() { + if (DBConfig::IsInMemoryDatabase(options.database_path.c_str())) { + options.temporary_directory = ".tmp"; + } else { + options.temporary_directory = options.database_path + ".tmp"; + } +} + void DBConfig::SetDefaultMaxSwapSpace(optional_ptr db) { options.maximum_swap_space.SetDefault(0); if (options.temporary_directory.empty()) { diff --git a/src/main/database.cpp b/src/main/database.cpp index e63469eb00f5..47da40517c75 100644 --- a/src/main/database.cpp +++ b/src/main/database.cpp @@ -297,24 +297,6 @@ Allocator &Allocator::Get(AttachedDatabase &db) { return Allocator::Get(db.GetDatabase()); } -static bool IsInMemoryDatabase(const char *database_path) { - if (!database_path) { - // Entirely empty - return true; - } - if (strlen(database_path) == 0) { - // '' empty string - return true; - } - constexpr const char *IN_MEMORY_PATH_PREFIX = ":memory:"; - const idx_t PREFIX_LENGTH = strlen(IN_MEMORY_PATH_PREFIX); - if (strncmp(database_path, IN_MEMORY_PATH_PREFIX, PREFIX_LENGTH) == 0) { - // Starts with :memory:, i.e ':memory:named_conn' is valid - return true; - } - return false; -} - void DatabaseInstance::Configure(DBConfig &new_config, const char *database_path) { config.options = new_config.options; @@ -322,21 +304,16 @@ void DatabaseInstance::Configure(DBConfig &new_config, const char *database_path config.SetOptionByName("duckdb_api", "cpp"); } - if (new_config.options.temporary_directory.empty()) { - // no directory specified: use default temp path - if (IsInMemoryDatabase(database_path)) { - config.options.temporary_directory = ".tmp"; - } else { - config.options.temporary_directory = string(database_path) + ".tmp"; - } - } - if (database_path) { config.options.database_path = database_path; } else { config.options.database_path.clear(); } + if (new_config.options.temporary_directory.empty()) { + config.SetDefaultTempDirectory(); + } + if (config.options.access_mode == AccessMode::UNDEFINED) { config.options.access_mode = AccessMode::READ_WRITE; } diff --git a/src/main/settings/settings.cpp b/src/main/settings/settings.cpp index 1eb7ce7eb030..87bb424e79cc 100644 --- a/src/main/settings/settings.cpp +++ b/src/main/settings/settings.cpp @@ -1241,7 +1241,8 @@ void TempDirectorySetting::SetGlobal(DatabaseInstance *db, DBConfig &config, con } void TempDirectorySetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { - config.options.temporary_directory = DBConfig().options.temporary_directory; + config.SetDefaultTempDirectory(); + config.options.use_temporary_directory = DBConfig().options.use_temporary_directory; if (db) { auto &buffer_manager = BufferManager::GetBufferManager(*db); From 9e5d10fcfbd34a73891b6663762fbcbde153b7ce Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 11 Mar 2024 10:34:38 +0100 Subject: [PATCH 067/603] add missing PRAGMA statement, because of a bug the temp directory was left empty before --- test/sql/storage/test_buffer_manager.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/test/sql/storage/test_buffer_manager.cpp b/test/sql/storage/test_buffer_manager.cpp index e730fe892b92..09991704d1bd 100644 --- a/test/sql/storage/test_buffer_manager.cpp +++ b/test/sql/storage/test_buffer_manager.cpp @@ -79,6 +79,7 @@ TEST_CASE("Modifying the buffer manager limit at runtime for an in-memory databa Connection con(db); REQUIRE_NO_FAIL(con.Query("PRAGMA threads=1")); REQUIRE_NO_FAIL(con.Query("PRAGMA force_compression='uncompressed'")); + REQUIRE_NO_FAIL(con.Query("PRAGMA temp_directory=''")); // initialize an in-memory database of size 10MB uint64_t table_size = (1000 * 1000) / sizeof(int); From 65cf2d4c9d94fab39e5cd3b791390a01964471a1 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 11 Mar 2024 14:56:14 +0100 Subject: [PATCH 068/603] [wip] changing to store rejects into two tables, add store_rejects flag --- .../scanner/string_value_scanner.cpp | 14 ++-- .../csv_scanner/sniffer/csv_sniffer.cpp | 4 +- .../csv_scanner/sniffer/dialect_detection.cpp | 12 +-- .../csv_scanner/sniffer/type_detection.cpp | 2 +- .../table_function/csv_file_scanner.cpp | 8 +- .../table_function/global_csv_state.cpp | 5 +- .../operator/csv_scanner/util/csv_error.cpp | 17 ++-- .../csv_scanner/util/csv_reader_options.cpp | 13 +-- .../operator/persistent/csv_rejects_table.cpp | 84 +++++++++++++------ src/function/table/read_csv.cpp | 20 ++--- .../operator/csv_scanner/csv_option.hpp | 2 +- .../csv_scanner/csv_reader_options.hpp | 6 +- .../operator/persistent/csv_rejects_table.hpp | 6 +- .../duckdb/storage/serialization/nodes.json | 6 +- src/storage/serialization/serialize_nodes.cpp | 8 +- .../csv/rejects/csv_rejects_double_table.test | 0 16 files changed, 119 insertions(+), 88 deletions(-) create mode 100644 test/sql/copy/csv/rejects/csv_rejects_double_table.test diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index e568a904b19c..ae9d470a32ce 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -17,7 +17,7 @@ StringValueResult::StringValueResult(CSVStates &states, CSVStateMachine &state_m shared_ptr csv_file_scan_p, idx_t &lines_read_p, bool sniffing_p) : ScannerResult(states, state_machine), number_of_columns(NumericCast(state_machine.dialect_options.num_cols)), - null_padding(state_machine.options.null_padding), ignore_errors(state_machine.options.ignore_errors), + null_padding(state_machine.options.null_padding), ignore_errors(state_machine.options.ignore_errors.GetValue()), null_str_ptr(state_machine.options.null_str.c_str()), null_str_size(state_machine.options.null_str.size()), result_size(result_size_p), error_handler(error_hander_p), iterator(iterator_p), store_line_size(store_line_size_p), csv_file_scan(std::move(csv_file_scan_p)), lines_read(lines_read_p), @@ -219,7 +219,7 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size // By default we add a string // We only evaluate if a string is utf8 valid, if it's actually a varchar if (parse_types[chunk_col_id].second && !Utf8Proc::IsValid(value_ptr, UnsafeNumericCast(size))) { - bool force_error = !state_machine.options.ignore_errors && sniffing; + bool force_error = !state_machine.options.ignore_errors.GetValue() && sniffing; // Invalid unicode, we must error if (force_error) { HandleUnicodeError(cur_col_id, force_error); @@ -554,7 +554,7 @@ bool StringValueResult::AddRow(StringValueResult &result, const idx_t buffer_pos } void StringValueResult::InvalidState(StringValueResult &result) { - bool force_error = !result.state_machine.options.ignore_errors && result.sniffing; + bool force_error = !result.state_machine.options.ignore_errors.GetValue() && result.sniffing; // Invalid unicode, we must error if (force_error) { result.HandleUnicodeError(result.cur_col_id, force_error); @@ -721,7 +721,7 @@ void StringValueScanner::Flush(DataChunk &insert_chunk) { { vector row; - if (state_machine->options.ignore_errors) { + if (state_machine->options.ignore_errors.GetValue()) { for (idx_t col = 0; col < parse_chunk.ColumnCount(); col++) { row.push_back(parse_chunk.GetValue(col, line_error)); } @@ -739,7 +739,7 @@ void StringValueScanner::Flush(DataChunk &insert_chunk) { error_handler->Error(csv_error); } borked_lines.insert(line_error++); - D_ASSERT(state_machine->options.ignore_errors); + D_ASSERT(state_machine->options.ignore_errors.GetValue()); // We are ignoring errors. We must continue but ignoring borked rows for (; line_error < parse_chunk.size(); line_error++) { if (!inserted_column_data.validity.RowIsValid(line_error) && @@ -1184,7 +1184,7 @@ void StringValueScanner::SetStart() { if (iterator.pos.buffer_pos == cur_buffer_handle->actual_size || scan_finder->iterator.GetBufferIdx() >= iterator.GetBufferIdx()) { // Propagate any errors - if (!scan_finder->error_handler->errors.empty() && state_machine->options.ignore_errors) { + if (!scan_finder->error_handler->errors.empty() && state_machine->options.ignore_errors.GetValue()) { for (auto &error_vector : scan_finder->error_handler->errors) { for (auto &error : error_vector.second) { error_handler->Error(error); @@ -1202,7 +1202,7 @@ void StringValueScanner::SetStart() { } } while (!line_found); // Propagate any errors - if (!scan_finder->error_handler->errors.empty() && state_machine->options.ignore_errors) { + if (!scan_finder->error_handler->errors.empty() && state_machine->options.ignore_errors.GetValue()) { for (auto &error_vector : scan_finder->error_handler->errors) { for (auto &error : error_vector.second) { error_handler->Error(error); diff --git a/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp b/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp index 9009210359f1..238b56426b52 100644 --- a/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp +++ b/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp @@ -13,7 +13,7 @@ CSVSniffer::CSVSniffer(CSVReaderOptions &options_p, shared_ptr } // Initialize max columns found to either 0 or however many were set max_columns_found = set_columns.Size(); - error_handler = make_shared(options.ignore_errors); + error_handler = make_shared(options.ignore_errors.GetValue()); detection_error_handler = make_shared(true); } @@ -93,7 +93,7 @@ SnifferResult CSVSniffer::SniffCSV(bool force_match) { DetectHeader(); // 5. Type Replacement ReplaceTypes(); - if (!best_candidate->error_handler->errors.empty() && !options.ignore_errors) { + if (!best_candidate->error_handler->errors.empty() && !options.ignore_errors.GetValue()) { for (auto &error_vector : best_candidate->error_handler->errors) { for (auto &error : error_vector.second) { if (error.type == CSVErrorType::MAXIMUM_LINE_SIZE) { diff --git a/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp b/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp index 7e23a6d1cc4b..0f5a485adff3 100644 --- a/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp +++ b/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp @@ -101,18 +101,19 @@ void CSVSniffer::AnalyzeDialectCandidate(unique_ptr scanner, if (sniffed_column_counts.result_position > rows_read) { rows_read = sniffed_column_counts.result_position; } - if (set_columns.IsCandidateUnacceptable(num_cols, options.null_padding, options.ignore_errors, + if (set_columns.IsCandidateUnacceptable(num_cols, options.null_padding, options.ignore_errors.GetValue(), sniffed_column_counts.last_value_always_empty)) { // Not acceptable return; } for (idx_t row = start_row; row < sniffed_column_counts.result_position; row++) { - if (set_columns.IsCandidateUnacceptable(sniffed_column_counts[row], options.null_padding, options.ignore_errors, + if (set_columns.IsCandidateUnacceptable(sniffed_column_counts[row], options.null_padding, + options.ignore_errors.GetValue(), sniffed_column_counts.last_value_always_empty)) { // Not acceptable return; } - if (sniffed_column_counts[row] == num_cols || options.ignore_errors) { + if (sniffed_column_counts[row] == num_cols || options.ignore_errors.GetValue()) { consistent_rows++; } else if (num_cols < sniffed_column_counts[row] && !options.dialect_options.skip_rows.IsSetByUser() && (!set_columns.IsSet() || options.null_padding)) { @@ -212,10 +213,11 @@ bool CSVSniffer::RefineCandidateNextChunk(ColumnCountScanner &candidate) { for (idx_t i = 0; i < sniffed_column_counts.result_position; i++) { if (set_columns.IsSet()) { return !set_columns.IsCandidateUnacceptable(sniffed_column_counts[i], options.null_padding, - options.ignore_errors, + options.ignore_errors.GetValue(), sniffed_column_counts.last_value_always_empty); } else { - if (max_columns_found != sniffed_column_counts[i] && (!options.null_padding && !options.ignore_errors)) { + if (max_columns_found != sniffed_column_counts[i] && + (!options.null_padding && !options.ignore_errors.GetValue())) { return false; } } diff --git a/src/execution/operator/csv_scanner/sniffer/type_detection.cpp b/src/execution/operator/csv_scanner/sniffer/type_detection.cpp index 717472b3c211..fe1bf8644776 100644 --- a/src/execution/operator/csv_scanner/sniffer/type_detection.cpp +++ b/src/execution/operator/csv_scanner/sniffer/type_detection.cpp @@ -273,7 +273,7 @@ void CSVSniffer::DetectTypes() { // it's good if the dialect creates more non-varchar columns, but only if we sacrifice < 30% of // best_num_cols. if (varchar_cols < min_varchar_cols && info_sql_types_candidates.size() > (max_columns_found * 0.7) && - (!options.ignore_errors || candidate->error_handler->errors.size() < min_errors)) { + (!options.ignore_errors.GetValue() || candidate->error_handler->errors.size() < min_errors)) { min_errors = candidate->error_handler->errors.size(); best_header_row.clear(); // we have a new best_options candidate diff --git a/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp b/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp index 641cbdd06818..807fe700a402 100644 --- a/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp +++ b/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp @@ -10,7 +10,7 @@ CSVFileScan::CSVFileScan(ClientContext &context, shared_ptr bu vector &file_schema) : file_path(options_p.file_path), file_idx(0), buffer_manager(std::move(buffer_manager_p)), state_machine(std::move(state_machine_p)), file_size(buffer_manager->file_handle->FileSize()), - error_handler(make_shared(options_p.ignore_errors)), + error_handler(make_shared(options_p.ignore_errors.GetValue())), on_disk_file(buffer_manager->file_handle->OnDiskFile()), options(options_p) { if (bind_data.initial_reader.get()) { auto &union_reader = *bind_data.initial_reader; @@ -43,7 +43,7 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, cons const idx_t file_idx_p, const ReadCSVData &bind_data, const vector &column_ids, const vector &file_schema) : file_path(file_path_p), file_idx(file_idx_p), - error_handler(make_shared(options_p.ignore_errors)), options(options_p) { + error_handler(make_shared(options_p.ignore_errors.GetValue())), options(options_p) { if (file_idx < bind_data.union_readers.size()) { // we are doing UNION BY NAME - fetch the options from the union reader for this file optional_ptr union_reader_ptr; @@ -129,8 +129,8 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, cons } CSVFileScan::CSVFileScan(ClientContext &context, const string &file_name, CSVReaderOptions &options_p) - : file_path(file_name), file_idx(0), error_handler(make_shared(options_p.ignore_errors)), - options(options_p) { + : file_path(file_name), file_idx(0), + error_handler(make_shared(options_p.ignore_errors.GetValue())), options(options_p) { buffer_manager = make_shared(context, options, file_path, file_idx); // Initialize On Disk and Size of file on_disk_file = buffer_manager->file_handle->OnDiskFile(); diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index 1d012349da21..4446a670e5a5 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -173,10 +173,9 @@ string CSVErrorTypeToEnum(CSVErrorType type) { void CSVGlobalState::FillRejectsTable() { auto &options = bind_data.options; - if (!options.rejects_table_name.empty()) { + if (options.store_rejects) { auto limit = options.rejects_limit; - - auto rejects = CSVRejectsTable::GetOrCreate(context, options.rejects_table_name); + auto rejects = CSVRejectsTable::GetOrCreate(context); lock_guard lock(rejects->write_lock); auto &table = rejects->GetTable(context); InternalAppender appender(context, table); diff --git a/src/execution/operator/csv_scanner/util/csv_error.cpp b/src/execution/operator/csv_scanner/util/csv_error.cpp index cfab97c9e55d..7a8349288c07 100644 --- a/src/execution/operator/csv_scanner/util/csv_error.cpp +++ b/src/execution/operator/csv_scanner/util/csv_error.cpp @@ -19,9 +19,9 @@ void CSVErrorHandler::ThrowError(CSVError csv_error) { if (PrintLineNumber(csv_error)) { error << "CSV Error on Line: " << GetLine(csv_error.error_info) << std::endl; } - if (csv_error.error_message_with_options.empty()){ + if (csv_error.error_message_with_options.empty()) { error << csv_error.error_message; - } else{ + } else { error << csv_error.error_message_with_options; } switch (csv_error.type) { @@ -153,7 +153,8 @@ CSVError CSVError::UnterminatedQuotesError(const CSVReaderOptions &options, idx_ LinesPerBoundary error_info, string &csv_row, idx_t byte_position) { std::ostringstream error; error << "Value with unterminated quote found."; - return CSVError(error.str(), CSVErrorType::UNTERMINATED_QUOTES, current_column, csv_row, error_info, byte_position, options); + return CSVError(error.str(), CSVErrorType::UNTERMINATED_QUOTES, current_column, csv_row, error_info, byte_position, + options); } CSVError CSVError::IncorrectColumnAmountError(const CSVReaderOptions &options, idx_t actual_columns, @@ -162,10 +163,11 @@ CSVError CSVError::IncorrectColumnAmountError(const CSVReaderOptions &options, i // How many columns were expected and how many were found error << "Expected Number of Columns: " << options.dialect_options.num_cols << " Found: " << actual_columns + 1; if (actual_columns >= options.dialect_options.num_cols) { - return CSVError(error.str(), CSVErrorType::TOO_MANY_COLUMNS, actual_columns, csv_row, error_info, - byte_position, options); + return CSVError(error.str(), CSVErrorType::TOO_MANY_COLUMNS, actual_columns, csv_row, error_info, byte_position, + options); } else { - return CSVError(error.str(), CSVErrorType::TOO_FEW_COLUMNS, actual_columns, csv_row, error_info, byte_position, options); + return CSVError(error.str(), CSVErrorType::TOO_FEW_COLUMNS, actual_columns, csv_row, error_info, byte_position, + options); } } @@ -174,7 +176,8 @@ CSVError CSVError::InvalidUTF8(const CSVReaderOptions &options, idx_t current_co std::ostringstream error; // How many columns were expected and how many were found error << "Invalid unicode (byte sequence mismatch) detected."; - return CSVError(error.str(), CSVErrorType::INVALID_UNICODE, current_column, csv_row, error_info, byte_position, options); + return CSVError(error.str(), CSVErrorType::INVALID_UNICODE, current_column, csv_row, error_info, byte_position, + options); } bool CSVErrorHandler::PrintLineNumber(CSVError &error) { diff --git a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp index 72c73a2e5bac..849c0e97ec52 100644 --- a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp +++ b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp @@ -189,7 +189,7 @@ void CSVReaderOptions::SetReadOption(const string &loption, const Value &value, string format = ParseString(value, loption); SetDateFormat(LogicalTypeId::TIMESTAMP, format, true); } else if (loption == "ignore_errors") { - ignore_errors = ParseBoolean(value, loption); + ignore_errors.Set(ParseBoolean(value, loption)); } else if (loption == "buffer_size") { buffer_size = ParseInteger(value, loption); if (buffer_size == 0) { @@ -206,13 +206,8 @@ void CSVReaderOptions::SetReadOption(const string &loption, const Value &value, parallel = ParseBoolean(value, loption); } else if (loption == "allow_quoted_nulls") { allow_quoted_nulls = ParseBoolean(value, loption); - } else if (loption == "rejects_table") { - // skip, handled in SetRejectsOptions - auto table_name = ParseString(value, loption); - if (table_name.empty()) { - throw BinderException("REJECTS_TABLE option cannot be empty"); - } - rejects_table_name = table_name; + } else if (loption == "store_rejects") { + store_rejects = ParseBoolean(value, loption); } else if (loption == "rejects_limit") { int64_t limit = ParseInteger(value, loption); if (limit < 0) { @@ -323,7 +318,7 @@ string CSVReaderOptions::ToString() const { // sample_size error += "sample_size=" + std::to_string(sample_size_chunks * STANDARD_VECTOR_SIZE) + "\n "; // ignore_errors - error += "ignore_errors=" + std::to_string(ignore_errors) + "\n "; + error += "ignore_errors=" + ignore_errors.FormatValue() + "\n "; // all_varchar error += "all_varchar=" + std::to_string(all_varchar) + "\n"; diff --git a/src/execution/operator/persistent/csv_rejects_table.cpp b/src/execution/operator/persistent/csv_rejects_table.cpp index a2f80d855d15..939ce125f39a 100644 --- a/src/execution/operator/persistent/csv_rejects_table.cpp +++ b/src/execution/operator/persistent/csv_rejects_table.cpp @@ -9,14 +9,14 @@ namespace duckdb { TableCatalogEntry &CSVRejectsTable::GetTable(ClientContext &context) { auto &temp_catalog = Catalog::GetCatalog(context, TEMP_CATALOG); - auto &table_entry = temp_catalog.GetEntry(context, TEMP_CATALOG, DEFAULT_SCHEMA, name); + auto &table_entry = temp_catalog.GetEntry(context, TEMP_CATALOG, DEFAULT_SCHEMA, "reject_scans"); return table_entry; } -shared_ptr CSVRejectsTable::GetOrCreate(ClientContext &context, const string &name) { - auto key = "CSV_REJECTS_TABLE_CACHE_ENTRY_" + StringUtil::Upper(name); +shared_ptr CSVRejectsTable::GetOrCreate(ClientContext &context) { + auto key = "CSV_REJECTS_TABLE_CACHE_ENTRY"; auto &cache = ObjectCache::GetObjectCache(context); - return cache.GetOrCreate(key, name); + return cache.GetOrCreate(key); } void CSVRejectsTable::InitializeTable(ClientContext &context, const ReadCSVData &data) { @@ -38,27 +38,61 @@ void CSVRejectsTable::InitializeTable(ClientContext &context, const ReadCSVData type_info->on_conflict = OnCreateConflict::IGNORE_ON_CONFLICT; catalog.CreateType(context, *type_info); - // Create Rejects Table - auto info = make_uniq(TEMP_CATALOG, DEFAULT_SCHEMA, name); - info->temporary = true; - info->on_conflict = OnCreateConflict::ERROR_ON_CONFLICT; - // 1. File Path - info->columns.AddColumn(ColumnDefinition("file", LogicalType::VARCHAR)); - // 2. Row Line - info->columns.AddColumn(ColumnDefinition("line", LogicalType::UBIGINT)); - // 3. Byte Position where error occurred - info->columns.AddColumn(ColumnDefinition("byte_position", LogicalType::UBIGINT)); - // 4. Column Index (If Applicable) - info->columns.AddColumn(ColumnDefinition("column_idx", LogicalType::UBIGINT)); - // 5. Column Name (If Applicable) - info->columns.AddColumn(ColumnDefinition("column_name", LogicalType::VARCHAR)); - // 6. Error Type - info->columns.AddColumn(ColumnDefinition("error_type", enum_type)); - // 7. Original CSV Line - info->columns.AddColumn(ColumnDefinition("csv_line", LogicalType::VARCHAR)); - // 8. Full Error Message - info->columns.AddColumn(ColumnDefinition("error_message", LogicalType::VARCHAR)); - catalog.CreateTable(context, std::move(info)); + // Create Rejects Scans Table + { + auto info = make_uniq(TEMP_CATALOG, DEFAULT_SCHEMA, "reject_scans"); + info->temporary = true; + info->on_conflict = OnCreateConflict::ERROR_ON_CONFLICT; + // 0. Scan ID + info->columns.AddColumn(ColumnDefinition("scan_id", LogicalType::UBIGINT)); + // 1. File Path + info->columns.AddColumn(ColumnDefinition("file_path", LogicalType::VARCHAR)); + // 2. Delimiter + info->columns.AddColumn(ColumnDefinition("delimiter", LogicalType::VARCHAR)); + // 3. Quote + info->columns.AddColumn(ColumnDefinition("quote", LogicalType::VARCHAR)); + // 4. Escape + info->columns.AddColumn(ColumnDefinition("escape", LogicalType::VARCHAR)); + // 5. NewLine Delimiter + info->columns.AddColumn(ColumnDefinition("newline_delimiter", LogicalType::VARCHAR)); + // 6. Skip Rows + info->columns.AddColumn(ColumnDefinition("skip_rows", LogicalType::UINTEGER)); + // 7. Has Header + info->columns.AddColumn(ColumnDefinition("has_header", LogicalType::BOOLEAN)); + // 8. List> + info->columns.AddColumn(ColumnDefinition("columns", LogicalType::VARCHAR)); + // 9. Date Format + info->columns.AddColumn(ColumnDefinition("date_format", LogicalType::VARCHAR)); + // 10. Timestamp Format + info->columns.AddColumn(ColumnDefinition("timestamp_format", LogicalType::VARCHAR)); + // 11. CSV read function with all the options used + info->columns.AddColumn(ColumnDefinition("user_arguments", LogicalType::VARCHAR)); + // 12. CSV read function with all the options used + info->columns.AddColumn(ColumnDefinition("prompt", LogicalType::VARCHAR)); + catalog.CreateTable(context, std::move(info)); + } + { + // Create Rejects Error Table + auto info = make_uniq(TEMP_CATALOG, DEFAULT_SCHEMA, "reject_errors"); + info->temporary = true; + info->on_conflict = OnCreateConflict::ERROR_ON_CONFLICT; + // 1. Row Line + info->columns.AddColumn(ColumnDefinition("line", LogicalType::UBIGINT)); + // 2. Byte Position where error occurred + info->columns.AddColumn(ColumnDefinition("byte_position", LogicalType::UBIGINT)); + // 3. Column Index (If Applicable) + info->columns.AddColumn(ColumnDefinition("column_idx", LogicalType::UBIGINT)); + // 4. Column Name (If Applicable) + info->columns.AddColumn(ColumnDefinition("column_name", LogicalType::VARCHAR)); + // 5. Error Type + info->columns.AddColumn(ColumnDefinition("error_type", enum_type)); + // 6. Original CSV Line + info->columns.AddColumn(ColumnDefinition("csv_line", LogicalType::VARCHAR)); + // 7. Full Error Message + info->columns.AddColumn(ColumnDefinition("error_message", LogicalType::VARCHAR)); + catalog.CreateTable(context, std::move(info)); + } + count = 0; } diff --git a/src/function/table/read_csv.cpp b/src/function/table/read_csv.cpp index 1b5d33f4df5b..2c691aa8bd21 100644 --- a/src/function/table/read_csv.cpp +++ b/src/function/table/read_csv.cpp @@ -53,19 +53,18 @@ static unique_ptr ReadCSVBind(ClientContext &context, TableFunctio options.FromNamedParameters(input.named_parameters, context, return_types, names); // Validate rejects_table options - if (!options.rejects_table_name.empty()) { - if (!options.ignore_errors) { + if (options.store_rejects) { + if (!options.ignore_errors.GetValue() && options.ignore_errors.IsSetByUser()) { throw BinderException("REJECTS_TABLE option is only supported when IGNORE_ERRORS is set to true"); } + // Ensure we set ignore errors to true automagically + options.ignore_errors.Set(true, false); if (options.file_options.union_by_name) { throw BinderException("REJECTS_TABLE option is not supported when UNION_BY_NAME is set to true"); } } - - if (options.rejects_limit != 0) { - if (options.rejects_table_name.empty()) { - throw BinderException("REJECTS_LIMIT option is only supported when REJECTS_TABLE is set to a table name"); - } + if (options.rejects_limit != 0 && !options.store_rejects) { + throw BinderException("REJECTS_LIMIT option is only supported when REJECTS_TABLE is set to a table name"); } options.file_options.AutoDetectHivePartitioning(result->files, context); @@ -146,9 +145,8 @@ static unique_ptr ReadCSVInitGlobal(ClientContext &con auto &bind_data = input.bind_data->Cast(); // Create the temporary rejects table - auto rejects_table = bind_data.options.rejects_table_name; - if (!rejects_table.empty()) { - CSVRejectsTable::GetOrCreate(context, rejects_table)->InitializeTable(context, bind_data); + if (bind_data.options.store_rejects) { + CSVRejectsTable::GetOrCreate(context)->InitializeTable(context, bind_data); } if (bind_data.files.empty()) { // This can happen when a filename based filter pushdown has eliminated all possible files for this scan. @@ -228,7 +226,7 @@ void ReadCSVTableFunction::ReadCSVAddNamedParameters(TableFunction &table_functi table_function.named_parameters["max_line_size"] = LogicalType::VARCHAR; table_function.named_parameters["maximum_line_size"] = LogicalType::VARCHAR; table_function.named_parameters["ignore_errors"] = LogicalType::BOOLEAN; - table_function.named_parameters["rejects_table"] = LogicalType::VARCHAR; + table_function.named_parameters["store_rejects"] = LogicalType::BOOLEAN; table_function.named_parameters["rejects_limit"] = LogicalType::BIGINT; table_function.named_parameters["buffer_size"] = LogicalType::UBIGINT; table_function.named_parameters["decimal_separator"] = LogicalType::VARCHAR; diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_option.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_option.hpp index 8c13e2c9f15f..57386f857963 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_option.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_option.hpp @@ -73,7 +73,7 @@ struct CSVOption { return value != other; } //! Returns CSV Option value - const T GetValue() const { + inline const T GetValue() const { return value; } bool IsSetByUser() const { diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp index ee06436ed9d6..436802909c82 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp @@ -40,9 +40,9 @@ struct CSVReaderOptions { //! See struct above. DialectOptions dialect_options; //! Whether or not we should ignore InvalidInput errors - bool ignore_errors = false; - //! Rejects table name - string rejects_table_name; + CSVOption ignore_errors = false; + //! Whether we store CSV Errors or not + bool store_rejects = false; //! Rejects table entry limit (0 = no limit) idx_t rejects_limit = 0; //! Number of samples to buffer diff --git a/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp b/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp index 12c9bc61345e..bb4ff62fa4ae 100644 --- a/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +++ b/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp @@ -14,14 +14,14 @@ class ClientContext; class CSVRejectsTable : public ObjectCacheEntry { public: - CSVRejectsTable(string name) : name(name), count(0) { + CSVRejectsTable() : count(0) { } ~CSVRejectsTable() override = default; mutex write_lock; - string name; + idx_t count; - static shared_ptr GetOrCreate(ClientContext &context, const string &name); + static shared_ptr GetOrCreate(ClientContext &context); void InitializeTable(ClientContext &context, const ReadCSVData &options); TableCatalogEntry &GetTable(ClientContext &context); diff --git a/src/include/duckdb/storage/serialization/nodes.json b/src/include/duckdb/storage/serialization/nodes.json index c601768a93cd..6dfb0b003f15 100644 --- a/src/include/duckdb/storage/serialization/nodes.json +++ b/src/include/duckdb/storage/serialization/nodes.json @@ -537,7 +537,7 @@ "members": [ {"id": 100, "name": "ignore_errors", - "type": "bool" + "type": "CSVOption" }, {"id": 101, "name": "buffer_sample_size", @@ -604,8 +604,8 @@ "type": "vector" }, {"id": 117, - "name": "rejects_table_name", - "type": "string" + "name": "store_rejects", + "type": "bool" }, {"id": 118, "name": "rejects_limit", diff --git a/src/storage/serialization/serialize_nodes.cpp b/src/storage/serialization/serialize_nodes.cpp index 96b233b8d6ca..c274e2a2b2ac 100644 --- a/src/storage/serialization/serialize_nodes.cpp +++ b/src/storage/serialization/serialize_nodes.cpp @@ -118,7 +118,7 @@ CSVOption CSVOption::Deserialize(Deserializer &deserializer) { } void CSVReaderOptions::Serialize(Serializer &serializer) const { - serializer.WritePropertyWithDefault(100, "ignore_errors", ignore_errors); + serializer.WriteProperty>(100, "ignore_errors", ignore_errors); serializer.WritePropertyWithDefault(101, "buffer_sample_size", buffer_sample_size); serializer.WritePropertyWithDefault(102, "null_str", null_str); serializer.WriteProperty(103, "compression", compression); @@ -135,7 +135,7 @@ void CSVReaderOptions::Serialize(Serializer &serializer) const { serializer.WritePropertyWithDefault(114, "buffer_size", buffer_size); serializer.WriteProperty(115, "file_options", file_options); serializer.WritePropertyWithDefault>(116, "force_quote", force_quote); - serializer.WritePropertyWithDefault(117, "rejects_table_name", rejects_table_name); + serializer.WritePropertyWithDefault(117, "store_rejects", store_rejects); serializer.WritePropertyWithDefault(118, "rejects_limit", rejects_limit); serializer.WriteProperty>(119, "dialect_options.state_machine_options.delimiter", dialect_options.state_machine_options.delimiter); serializer.WriteProperty>(120, "dialect_options.state_machine_options.quote", dialect_options.state_machine_options.quote); @@ -151,7 +151,7 @@ void CSVReaderOptions::Serialize(Serializer &serializer) const { CSVReaderOptions CSVReaderOptions::Deserialize(Deserializer &deserializer) { CSVReaderOptions result; - deserializer.ReadPropertyWithDefault(100, "ignore_errors", result.ignore_errors); + deserializer.ReadProperty>(100, "ignore_errors", result.ignore_errors); deserializer.ReadPropertyWithDefault(101, "buffer_sample_size", result.buffer_sample_size); deserializer.ReadPropertyWithDefault(102, "null_str", result.null_str); deserializer.ReadProperty(103, "compression", result.compression); @@ -168,7 +168,7 @@ CSVReaderOptions CSVReaderOptions::Deserialize(Deserializer &deserializer) { deserializer.ReadPropertyWithDefault(114, "buffer_size", result.buffer_size); deserializer.ReadProperty(115, "file_options", result.file_options); deserializer.ReadPropertyWithDefault>(116, "force_quote", result.force_quote); - deserializer.ReadPropertyWithDefault(117, "rejects_table_name", result.rejects_table_name); + deserializer.ReadPropertyWithDefault(117, "store_rejects", result.store_rejects); deserializer.ReadPropertyWithDefault(118, "rejects_limit", result.rejects_limit); deserializer.ReadProperty>(119, "dialect_options.state_machine_options.delimiter", result.dialect_options.state_machine_options.delimiter); deserializer.ReadProperty>(120, "dialect_options.state_machine_options.quote", result.dialect_options.state_machine_options.quote); diff --git a/test/sql/copy/csv/rejects/csv_rejects_double_table.test b/test/sql/copy/csv/rejects/csv_rejects_double_table.test new file mode 100644 index 000000000000..e69de29bb2d1 From 05bb2db09a4f309d4882335f77271e8cc3afc255 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 11 Mar 2024 15:32:13 +0100 Subject: [PATCH 069/603] More on making options magically work --- .../table_function/global_csv_state.cpp | 4 ++-- .../csv_scanner/util/csv_reader_options.cpp | 7 +++++++ .../operator/persistent/csv_rejects_table.cpp | 12 +++++------ src/function/table/read_csv.cpp | 21 +++++++++++++------ .../csv_scanner/csv_reader_options.hpp | 6 ++++-- .../operator/persistent/csv_rejects_table.hpp | 15 ++++++++++--- .../duckdb/storage/serialization/nodes.json | 6 +++++- src/storage/serialization/serialize_nodes.cpp | 6 ++++-- 8 files changed, 55 insertions(+), 22 deletions(-) diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index 4446a670e5a5..00ba18b82f00 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -173,9 +173,9 @@ string CSVErrorTypeToEnum(CSVErrorType type) { void CSVGlobalState::FillRejectsTable() { auto &options = bind_data.options; - if (options.store_rejects) { + if (options.store_rejects.GetValue()) { auto limit = options.rejects_limit; - auto rejects = CSVRejectsTable::GetOrCreate(context); + auto rejects = CSVRejectsTable::GetOrCreate(context, options.rejects_table_name); lock_guard lock(rejects->write_lock); auto &table = rejects->GetTable(context); InternalAppender appender(context, table); diff --git a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp index 849c0e97ec52..7fcb7f3383b1 100644 --- a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp +++ b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp @@ -208,6 +208,13 @@ void CSVReaderOptions::SetReadOption(const string &loption, const Value &value, allow_quoted_nulls = ParseBoolean(value, loption); } else if (loption == "store_rejects") { store_rejects = ParseBoolean(value, loption); + } else if (loption == "rejects_table") { + // skip, handled in SetRejectsOptions + auto table_name = ParseString(value, loption); + if (table_name.empty()) { + throw BinderException("REJECTS_TABLE option cannot be empty"); + } + rejects_table_name = table_name; } else if (loption == "rejects_limit") { int64_t limit = ParseInteger(value, loption); if (limit < 0) { diff --git a/src/execution/operator/persistent/csv_rejects_table.cpp b/src/execution/operator/persistent/csv_rejects_table.cpp index 939ce125f39a..7d8094659377 100644 --- a/src/execution/operator/persistent/csv_rejects_table.cpp +++ b/src/execution/operator/persistent/csv_rejects_table.cpp @@ -9,14 +9,14 @@ namespace duckdb { TableCatalogEntry &CSVRejectsTable::GetTable(ClientContext &context) { auto &temp_catalog = Catalog::GetCatalog(context, TEMP_CATALOG); - auto &table_entry = temp_catalog.GetEntry(context, TEMP_CATALOG, DEFAULT_SCHEMA, "reject_scans"); + auto &table_entry = temp_catalog.GetEntry(context, TEMP_CATALOG, DEFAULT_SCHEMA, errors_table); return table_entry; } -shared_ptr CSVRejectsTable::GetOrCreate(ClientContext &context) { - auto key = "CSV_REJECTS_TABLE_CACHE_ENTRY"; +shared_ptr CSVRejectsTable::GetOrCreate(ClientContext &context, const string &name) { + auto key = "CSV_REJECTS_TABLE_CACHE_ENTRY_" + StringUtil::Upper(name); auto &cache = ObjectCache::GetObjectCache(context); - return cache.GetOrCreate(key); + return cache.GetOrCreate(key, name); } void CSVRejectsTable::InitializeTable(ClientContext &context, const ReadCSVData &data) { @@ -40,7 +40,7 @@ void CSVRejectsTable::InitializeTable(ClientContext &context, const ReadCSVData // Create Rejects Scans Table { - auto info = make_uniq(TEMP_CATALOG, DEFAULT_SCHEMA, "reject_scans"); + auto info = make_uniq(TEMP_CATALOG, DEFAULT_SCHEMA, scan_table); info->temporary = true; info->on_conflict = OnCreateConflict::ERROR_ON_CONFLICT; // 0. Scan ID @@ -73,7 +73,7 @@ void CSVRejectsTable::InitializeTable(ClientContext &context, const ReadCSVData } { // Create Rejects Error Table - auto info = make_uniq(TEMP_CATALOG, DEFAULT_SCHEMA, "reject_errors"); + auto info = make_uniq(TEMP_CATALOG, DEFAULT_SCHEMA, errors_table); info->temporary = true; info->on_conflict = OnCreateConflict::ERROR_ON_CONFLICT; // 1. Row Line diff --git a/src/function/table/read_csv.cpp b/src/function/table/read_csv.cpp index 2c691aa8bd21..b7f865fc718e 100644 --- a/src/function/table/read_csv.cpp +++ b/src/function/table/read_csv.cpp @@ -51,11 +51,18 @@ static unique_ptr ReadCSVBind(ClientContext &context, TableFunctio result->files = MultiFileReader::GetFileList(context, input.inputs[0], "CSV"); options.FromNamedParameters(input.named_parameters, context, return_types, names); - + if (!options.rejects_table_name.empty() && !options.store_rejects.GetValue() && + options.store_rejects.IsSetByUser()) { + throw BinderException( + "rejects_table_name option is only supported when store_rejects is not manually set to false"); + } + // Ensure we set ignore errors to true automagically + options.store_rejects.Set(true, false); // Validate rejects_table options - if (options.store_rejects) { + if (options.store_rejects.GetValue()) { if (!options.ignore_errors.GetValue() && options.ignore_errors.IsSetByUser()) { - throw BinderException("REJECTS_TABLE option is only supported when IGNORE_ERRORS is set to true"); + throw BinderException( + "STORE_REJECTS option is only supported when IGNORE_ERRORS is not manually set to false"); } // Ensure we set ignore errors to true automagically options.ignore_errors.Set(true, false); @@ -63,7 +70,7 @@ static unique_ptr ReadCSVBind(ClientContext &context, TableFunctio throw BinderException("REJECTS_TABLE option is not supported when UNION_BY_NAME is set to true"); } } - if (options.rejects_limit != 0 && !options.store_rejects) { + if (options.rejects_limit != 0 && !options.store_rejects.GetValue()) { throw BinderException("REJECTS_LIMIT option is only supported when REJECTS_TABLE is set to a table name"); } @@ -145,8 +152,9 @@ static unique_ptr ReadCSVInitGlobal(ClientContext &con auto &bind_data = input.bind_data->Cast(); // Create the temporary rejects table - if (bind_data.options.store_rejects) { - CSVRejectsTable::GetOrCreate(context)->InitializeTable(context, bind_data); + if (bind_data.options.store_rejects.GetValue()) { + CSVRejectsTable::GetOrCreate(context, bind_data.options.rejects_table_name) + ->InitializeTable(context, bind_data); } if (bind_data.files.empty()) { // This can happen when a filename based filter pushdown has eliminated all possible files for this scan. @@ -227,6 +235,7 @@ void ReadCSVTableFunction::ReadCSVAddNamedParameters(TableFunction &table_functi table_function.named_parameters["maximum_line_size"] = LogicalType::VARCHAR; table_function.named_parameters["ignore_errors"] = LogicalType::BOOLEAN; table_function.named_parameters["store_rejects"] = LogicalType::BOOLEAN; + table_function.named_parameters["rejects_table"] = LogicalType::VARCHAR; table_function.named_parameters["rejects_limit"] = LogicalType::BIGINT; table_function.named_parameters["buffer_size"] = LogicalType::UBIGINT; table_function.named_parameters["decimal_separator"] = LogicalType::VARCHAR; diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp index 436802909c82..a7db5aeb06f4 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp @@ -41,8 +41,10 @@ struct CSVReaderOptions { DialectOptions dialect_options; //! Whether or not we should ignore InvalidInput errors CSVOption ignore_errors = false; - //! Whether we store CSV Errors or not - bool store_rejects = false; + //! Whether we store CSV Errors in the rejects table or not + CSVOption store_rejects = false; + //! Rejects table name + string rejects_table_name; //! Rejects table entry limit (0 = no limit) idx_t rejects_limit = 0; //! Number of samples to buffer diff --git a/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp b/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp index bb4ff62fa4ae..f88eff8028ea 100644 --- a/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +++ b/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp @@ -14,14 +14,23 @@ class ClientContext; class CSVRejectsTable : public ObjectCacheEntry { public: - CSVRejectsTable() : count(0) { + CSVRejectsTable(string name) : name(name), count(0) { + if (name.empty()) { + scan_table = "reject_scan"; + errors_table = "reject_errors"; + } else { + scan_table = name + "_scan"; + errors_table = name; + } } ~CSVRejectsTable() override = default; mutex write_lock; - + string name; idx_t count; + string scan_table; + string errors_table; - static shared_ptr GetOrCreate(ClientContext &context); + static shared_ptr GetOrCreate(ClientContext &context, const string &name); void InitializeTable(ClientContext &context, const ReadCSVData &options); TableCatalogEntry &GetTable(ClientContext &context); diff --git a/src/include/duckdb/storage/serialization/nodes.json b/src/include/duckdb/storage/serialization/nodes.json index 6dfb0b003f15..39961131cad2 100644 --- a/src/include/duckdb/storage/serialization/nodes.json +++ b/src/include/duckdb/storage/serialization/nodes.json @@ -605,7 +605,7 @@ }, {"id": 117, "name": "store_rejects", - "type": "bool" + "type": "CSVOption" }, {"id": 118, "name": "rejects_limit", @@ -650,6 +650,10 @@ {"id": 128, "name": "parallel", "type": "bool" + }, + {"id": 129, + "name": "rejects_table_name", + "type": "string" } ], "pointer_type": "none" diff --git a/src/storage/serialization/serialize_nodes.cpp b/src/storage/serialization/serialize_nodes.cpp index c274e2a2b2ac..b7f0d3078810 100644 --- a/src/storage/serialization/serialize_nodes.cpp +++ b/src/storage/serialization/serialize_nodes.cpp @@ -135,7 +135,7 @@ void CSVReaderOptions::Serialize(Serializer &serializer) const { serializer.WritePropertyWithDefault(114, "buffer_size", buffer_size); serializer.WriteProperty(115, "file_options", file_options); serializer.WritePropertyWithDefault>(116, "force_quote", force_quote); - serializer.WritePropertyWithDefault(117, "store_rejects", store_rejects); + serializer.WriteProperty>(117, "store_rejects", store_rejects); serializer.WritePropertyWithDefault(118, "rejects_limit", rejects_limit); serializer.WriteProperty>(119, "dialect_options.state_machine_options.delimiter", dialect_options.state_machine_options.delimiter); serializer.WriteProperty>(120, "dialect_options.state_machine_options.quote", dialect_options.state_machine_options.quote); @@ -147,6 +147,7 @@ void CSVReaderOptions::Serialize(Serializer &serializer) const { serializer.WriteProperty>>(126, "dialect_options.date_format", dialect_options.date_format); serializer.WritePropertyWithDefault(127, "sniffer_user_mismatch_error", sniffer_user_mismatch_error); serializer.WritePropertyWithDefault(128, "parallel", parallel); + serializer.WritePropertyWithDefault(129, "rejects_table_name", rejects_table_name); } CSVReaderOptions CSVReaderOptions::Deserialize(Deserializer &deserializer) { @@ -168,7 +169,7 @@ CSVReaderOptions CSVReaderOptions::Deserialize(Deserializer &deserializer) { deserializer.ReadPropertyWithDefault(114, "buffer_size", result.buffer_size); deserializer.ReadProperty(115, "file_options", result.file_options); deserializer.ReadPropertyWithDefault>(116, "force_quote", result.force_quote); - deserializer.ReadPropertyWithDefault(117, "store_rejects", result.store_rejects); + deserializer.ReadProperty>(117, "store_rejects", result.store_rejects); deserializer.ReadPropertyWithDefault(118, "rejects_limit", result.rejects_limit); deserializer.ReadProperty>(119, "dialect_options.state_machine_options.delimiter", result.dialect_options.state_machine_options.delimiter); deserializer.ReadProperty>(120, "dialect_options.state_machine_options.quote", result.dialect_options.state_machine_options.quote); @@ -180,6 +181,7 @@ CSVReaderOptions CSVReaderOptions::Deserialize(Deserializer &deserializer) { deserializer.ReadProperty>>(126, "dialect_options.date_format", result.dialect_options.date_format); deserializer.ReadPropertyWithDefault(127, "sniffer_user_mismatch_error", result.sniffer_user_mismatch_error); deserializer.ReadPropertyWithDefault(128, "parallel", result.parallel); + deserializer.ReadPropertyWithDefault(129, "rejects_table_name", result.rejects_table_name); return result; } From 779ab7f36f9088d54e1f1662ac03daded8d3ed26 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 11 Mar 2024 15:51:26 +0100 Subject: [PATCH 070/603] got rejects table right --- .../table_function/global_csv_state.cpp | 53 +++++++++++-------- .../operator/persistent/csv_rejects_table.cpp | 52 +++++++++++------- .../operator/persistent/csv_rejects_table.hpp | 3 +- 3 files changed, 64 insertions(+), 44 deletions(-) diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index 00ba18b82f00..917a581bcbcc 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -177,11 +177,16 @@ void CSVGlobalState::FillRejectsTable() { auto limit = options.rejects_limit; auto rejects = CSVRejectsTable::GetOrCreate(context, options.rejects_table_name); lock_guard lock(rejects->write_lock); - auto &table = rejects->GetTable(context); - InternalAppender appender(context, table); + auto &errors_table = rejects->GetErrorsTable(context); + auto &scans_table = rejects->GetScansTable(context); + InternalAppender errors_appender(context, errors_table); + InternalAppender scans_appender(context, scans_table); + idx_t scan_id = context.transaction.GetActiveQuery(); + idx_t file_id = 0; for (auto &file : file_scans) { auto file_name = file->file_path; auto &errors = file->error_handler->errors; + // We first insert the file into the file scans table for (auto &error_vector : errors) { for (auto &error : error_vector.second) { if (!IsCSVErrorAcceptedReject(error.type)) { @@ -197,36 +202,38 @@ void CSVGlobalState::FillRejectsTable() { auto row_line = file->error_handler->GetLine(error.error_info); auto col_idx = error.column_idx; // Add the row to the rejects table - appender.BeginRow(); - // 1. File Path - appender.Append(string_t(file_name)); - // 2. Row Line - appender.Append(row_line); - // 3. Byte Position where error occurred - appender.Append(error.byte_position); - // 4. Column Index - appender.Append(col_idx + 1); - // 5. Column Name (If Applicable) + errors_appender.BeginRow(); + // 1. Scan Id + errors_appender.Append(scan_id); + // 2. File Id + errors_appender.Append(file_id); + // 3. Row Line + errors_appender.Append(row_line); + // 4. Byte Position where error occurred + errors_appender.Append(error.byte_position); + // 5. Column Index + errors_appender.Append(col_idx + 1); + // 6. Column Name (If Applicable) switch (error.type) { case CSVErrorType::TOO_MANY_COLUMNS: - appender.Append(Value()); + errors_appender.Append(Value()); break; case CSVErrorType::TOO_FEW_COLUMNS: D_ASSERT(bind_data.return_names.size() > col_idx + 1); - appender.Append(string_t("\"" + bind_data.return_names[col_idx + 1] + "\"")); + errors_appender.Append(string_t("\"" + bind_data.return_names[col_idx + 1] + "\"")); break; default: - appender.Append(string_t("\"" + bind_data.return_names[col_idx] + "\"")); + errors_appender.Append(string_t("\"" + bind_data.return_names[col_idx] + "\"")); } - // 6. Error Type - appender.Append(string_t(CSVErrorTypeToEnum(error.type))); - // 7. Original CSV Line - appender.Append(string_t(error.csv_row)); - // 8. Full Error Message - appender.Append(string_t(error.error_message)); - appender.EndRow(); + // 7. Error Type + errors_appender.Append(string_t(CSVErrorTypeToEnum(error.type))); + // 8. Original CSV Line + errors_appender.Append(string_t(error.csv_row)); + // 9. Full Error Message + errors_appender.Append(string_t(error.error_message)); + errors_appender.EndRow(); } - appender.Close(); + errors_appender.Close(); } } } diff --git a/src/execution/operator/persistent/csv_rejects_table.cpp b/src/execution/operator/persistent/csv_rejects_table.cpp index 7d8094659377..50ddcedc7b54 100644 --- a/src/execution/operator/persistent/csv_rejects_table.cpp +++ b/src/execution/operator/persistent/csv_rejects_table.cpp @@ -7,12 +7,18 @@ namespace duckdb { -TableCatalogEntry &CSVRejectsTable::GetTable(ClientContext &context) { +TableCatalogEntry &CSVRejectsTable::GetErrorsTable(ClientContext &context) { auto &temp_catalog = Catalog::GetCatalog(context, TEMP_CATALOG); auto &table_entry = temp_catalog.GetEntry(context, TEMP_CATALOG, DEFAULT_SCHEMA, errors_table); return table_entry; } +TableCatalogEntry &CSVRejectsTable::GetScansTable(ClientContext &context) { + auto &temp_catalog = Catalog::GetCatalog(context, TEMP_CATALOG); + auto &table_entry = temp_catalog.GetEntry(context, TEMP_CATALOG, DEFAULT_SCHEMA, scan_table); + return table_entry; +} + shared_ptr CSVRejectsTable::GetOrCreate(ClientContext &context, const string &name) { auto key = "CSV_REJECTS_TABLE_CACHE_ENTRY_" + StringUtil::Upper(name); auto &cache = ObjectCache::GetObjectCache(context); @@ -45,29 +51,31 @@ void CSVRejectsTable::InitializeTable(ClientContext &context, const ReadCSVData info->on_conflict = OnCreateConflict::ERROR_ON_CONFLICT; // 0. Scan ID info->columns.AddColumn(ColumnDefinition("scan_id", LogicalType::UBIGINT)); - // 1. File Path + // 1. File ID (within the scan) + info->columns.AddColumn(ColumnDefinition("file_id", LogicalType::UBIGINT)); + // 2. File Path info->columns.AddColumn(ColumnDefinition("file_path", LogicalType::VARCHAR)); - // 2. Delimiter + // 3. Delimiter info->columns.AddColumn(ColumnDefinition("delimiter", LogicalType::VARCHAR)); - // 3. Quote + // 4. Quote info->columns.AddColumn(ColumnDefinition("quote", LogicalType::VARCHAR)); - // 4. Escape + // 5. Escape info->columns.AddColumn(ColumnDefinition("escape", LogicalType::VARCHAR)); - // 5. NewLine Delimiter + // 6. NewLine Delimiter info->columns.AddColumn(ColumnDefinition("newline_delimiter", LogicalType::VARCHAR)); - // 6. Skip Rows + // 7. Skip Rows info->columns.AddColumn(ColumnDefinition("skip_rows", LogicalType::UINTEGER)); - // 7. Has Header + // 8. Has Header info->columns.AddColumn(ColumnDefinition("has_header", LogicalType::BOOLEAN)); - // 8. List> + // 9. List> info->columns.AddColumn(ColumnDefinition("columns", LogicalType::VARCHAR)); - // 9. Date Format + // 10. Date Format info->columns.AddColumn(ColumnDefinition("date_format", LogicalType::VARCHAR)); - // 10. Timestamp Format + // 11. Timestamp Format info->columns.AddColumn(ColumnDefinition("timestamp_format", LogicalType::VARCHAR)); - // 11. CSV read function with all the options used - info->columns.AddColumn(ColumnDefinition("user_arguments", LogicalType::VARCHAR)); // 12. CSV read function with all the options used + info->columns.AddColumn(ColumnDefinition("user_arguments", LogicalType::VARCHAR)); + // 13. CSV read function with all the options used info->columns.AddColumn(ColumnDefinition("prompt", LogicalType::VARCHAR)); catalog.CreateTable(context, std::move(info)); } @@ -76,19 +84,23 @@ void CSVRejectsTable::InitializeTable(ClientContext &context, const ReadCSVData auto info = make_uniq(TEMP_CATALOG, DEFAULT_SCHEMA, errors_table); info->temporary = true; info->on_conflict = OnCreateConflict::ERROR_ON_CONFLICT; - // 1. Row Line + // 0. Scan ID + info->columns.AddColumn(ColumnDefinition("scan_id", LogicalType::UBIGINT)); + // 1. File ID (within the scan) + info->columns.AddColumn(ColumnDefinition("file_id", LogicalType::UBIGINT)); + // 2. Row Line info->columns.AddColumn(ColumnDefinition("line", LogicalType::UBIGINT)); - // 2. Byte Position where error occurred + // 3. Byte Position where error occurred info->columns.AddColumn(ColumnDefinition("byte_position", LogicalType::UBIGINT)); - // 3. Column Index (If Applicable) + // 4. Column Index (If Applicable) info->columns.AddColumn(ColumnDefinition("column_idx", LogicalType::UBIGINT)); - // 4. Column Name (If Applicable) + // 5. Column Name (If Applicable) info->columns.AddColumn(ColumnDefinition("column_name", LogicalType::VARCHAR)); - // 5. Error Type + // 6. Error Type info->columns.AddColumn(ColumnDefinition("error_type", enum_type)); - // 6. Original CSV Line + // 7. Original CSV Line info->columns.AddColumn(ColumnDefinition("csv_line", LogicalType::VARCHAR)); - // 7. Full Error Message + // 8. Full Error Message info->columns.AddColumn(ColumnDefinition("error_message", LogicalType::VARCHAR)); catalog.CreateTable(context, std::move(info)); } diff --git a/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp b/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp index f88eff8028ea..2a17f0b61851 100644 --- a/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +++ b/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp @@ -33,7 +33,8 @@ class CSVRejectsTable : public ObjectCacheEntry { static shared_ptr GetOrCreate(ClientContext &context, const string &name); void InitializeTable(ClientContext &context, const ReadCSVData &options); - TableCatalogEntry &GetTable(ClientContext &context); + TableCatalogEntry &GetErrorsTable(ClientContext &context); + TableCatalogEntry &GetScansTable(ClientContext &context); public: static string ObjectType() { From 77bfe80f6124e7a950f8b95363e19e7002e84480 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 11 Mar 2024 16:25:31 +0100 Subject: [PATCH 071/603] First version of rejects_scans and reject_errors tables --- .../table_function/global_csv_state.cpp | 76 +++++++++++++++++-- .../operator/persistent/csv_rejects_table.cpp | 2 - src/function/table/sniff_csv.cpp | 14 +--- .../csv_scanner/csv_reader_options.hpp | 11 +++ .../operator/persistent/csv_rejects_table.hpp | 2 +- .../csv/rejects/csv_rejects_double_table.test | 34 +++++++++ 6 files changed, 117 insertions(+), 22 deletions(-) diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index 917a581bcbcc..b1fef47e4d1b 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -170,6 +170,65 @@ string CSVErrorTypeToEnum(CSVErrorType type) { } } +void FillScanErrorTable(InternalAppender &scan_appender, idx_t scan_idx, idx_t file_idx, CSVFileScan &file) { + CSVReaderOptions &options = file.options; + // Add the row to the rejects table + scan_appender.BeginRow(); + // 1. Scan Idx + scan_appender.Append(scan_idx); + // 2. File Idx + scan_appender.Append(file_idx); + // 3. File Path + scan_appender.Append(string_t(file.file_path)); + // 4. Delimiter + scan_appender.Append(string_t(options.dialect_options.state_machine_options.delimiter.FormatValue())); + // 5. Quote + scan_appender.Append(string_t(options.dialect_options.state_machine_options.quote.FormatValue())); + // 6. Escape + scan_appender.Append(string_t(options.dialect_options.state_machine_options.escape.FormatValue())); + // 7. NewLine Delimiter + scan_appender.Append(string_t(options.NewLineIdentifierToString())); + // 8. Skip Rows + scan_appender.Append(Value::UINTEGER(NumericCast(options.dialect_options.skip_rows.GetValue()))); + // 9. Has Header + scan_appender.Append(Value::BOOLEAN(options.dialect_options.header.GetValue())); + // 10. List> {'col1': 'INTEGER', 'col2': 'VARCHAR'} + std::ostringstream columns; + columns << "{"; + for (idx_t i = 0; i < file.types.size(); i++) { + columns << "'" << file.names[i] << "': '" << file.types[i].ToString() << "'"; + if (i != file.types.size() - 1) { + columns << ","; + } + } + columns << "}"; + scan_appender.Append(string_t(columns.str())); + // 11. Date Format + auto date_format = options.dialect_options.date_format[LogicalType::DATE].GetValue(); + if (!date_format.Empty()) { + scan_appender.Append(string_t(date_format.format_specifier)); + } else { + scan_appender.Append(Value()); + } + + // 12. Timestamp Format + auto timestamp_format = options.dialect_options.date_format[LogicalType::TIMESTAMP].GetValue(); + if (!timestamp_format.Empty()) { + scan_appender.Append(string_t(timestamp_format.format_specifier)); + } else { + scan_appender.Append(Value()); + } + + // 13. The Extra User Arguments + if (options.user_defined_parameters.empty()) { + scan_appender.Append(Value()); + } else { + scan_appender.Append(string_t(options.user_defined_parameters)); + } + // Finish the row to the rejects table + scan_appender.EndRow(); +} + void CSVGlobalState::FillRejectsTable() { auto &options = bind_data.options; @@ -181,8 +240,8 @@ void CSVGlobalState::FillRejectsTable() { auto &scans_table = rejects->GetScansTable(context); InternalAppender errors_appender(context, errors_table); InternalAppender scans_appender(context, scans_table); - idx_t scan_id = context.transaction.GetActiveQuery(); - idx_t file_id = 0; + idx_t scan_idx = context.transaction.GetActiveQuery(); + idx_t file_idx = 0; for (auto &file : file_scans) { auto file_name = file->file_path; auto &errors = file->error_handler->errors; @@ -190,7 +249,6 @@ void CSVGlobalState::FillRejectsTable() { for (auto &error_vector : errors) { for (auto &error : error_vector.second) { if (!IsCSVErrorAcceptedReject(error.type)) { - // For now, we only will use it for casting errors continue; } // short circuit if we already have too many rejects @@ -204,9 +262,9 @@ void CSVGlobalState::FillRejectsTable() { // Add the row to the rejects table errors_appender.BeginRow(); // 1. Scan Id - errors_appender.Append(scan_id); + errors_appender.Append(scan_idx); // 2. File Id - errors_appender.Append(file_id); + errors_appender.Append(file_idx); // 3. Row Line errors_appender.Append(row_line); // 4. Byte Position where error occurred @@ -233,10 +291,16 @@ void CSVGlobalState::FillRejectsTable() { errors_appender.Append(string_t(error.error_message)); errors_appender.EndRow(); } - errors_appender.Close(); } } + if (rejects->count != 0) { + rejects->count = 0; + FillScanErrorTable(scans_appender, scan_idx, file_idx, *file); + } + file_idx++; } + errors_appender.Close(); + scans_appender.Close(); } } diff --git a/src/execution/operator/persistent/csv_rejects_table.cpp b/src/execution/operator/persistent/csv_rejects_table.cpp index 50ddcedc7b54..e74a7806a4ef 100644 --- a/src/execution/operator/persistent/csv_rejects_table.cpp +++ b/src/execution/operator/persistent/csv_rejects_table.cpp @@ -75,8 +75,6 @@ void CSVRejectsTable::InitializeTable(ClientContext &context, const ReadCSVData info->columns.AddColumn(ColumnDefinition("timestamp_format", LogicalType::VARCHAR)); // 12. CSV read function with all the options used info->columns.AddColumn(ColumnDefinition("user_arguments", LogicalType::VARCHAR)); - // 13. CSV read function with all the options used - info->columns.AddColumn(ColumnDefinition("prompt", LogicalType::VARCHAR)); catalog.CreateTable(context, std::move(info)); } { diff --git a/src/function/table/sniff_csv.cpp b/src/function/table/sniff_csv.cpp index f135b15c615d..28f248d4f459 100644 --- a/src/function/table/sniff_csv.cpp +++ b/src/function/table/sniff_csv.cpp @@ -83,17 +83,6 @@ static unique_ptr CSVSniffBind(ClientContext &context, TableFuncti return std::move(result); } -string NewLineIdentifierToString(NewLineIdentifier identifier) { - switch (identifier) { - case NewLineIdentifier::SINGLE: - return "\\n"; - case NewLineIdentifier::CARRY_ON: - return "\\r\\n"; - default: - return ""; - } -} - string FormatOptions(char opt) { if (opt == '\'') { return "''"; @@ -138,8 +127,7 @@ static void CSVSniffFunction(ClientContext &context, TableFunctionInput &data_p, str_opt = sniffer_options.dialect_options.state_machine_options.escape.GetValue(); output.SetValue(2, 0, str_opt); // 4. NewLine Delimiter - auto new_line_identifier = - NewLineIdentifierToString(sniffer_options.dialect_options.state_machine_options.new_line.GetValue()); + auto new_line_identifier = sniffer_options.NewLineIdentifierToString(); output.SetValue(3, 0, new_line_identifier); // 5. Skip Rows output.SetValue(4, 0, Value::UINTEGER(NumericCast(sniffer_options.dialect_options.skip_rows.GetValue()))); diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp index a7db5aeb06f4..faabfe62f23e 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp @@ -157,5 +157,16 @@ struct CSVReaderOptions { vector &names); string ToString() const; + + string NewLineIdentifierToString() { + switch (dialect_options.state_machine_options.new_line.GetValue()) { + case NewLineIdentifier::SINGLE: + return "\\n"; + case NewLineIdentifier::CARRY_ON: + return "\\r\\n"; + default: + return ""; + } + } }; } // namespace duckdb diff --git a/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp b/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp index 2a17f0b61851..6254d7cecc01 100644 --- a/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +++ b/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp @@ -16,7 +16,7 @@ class CSVRejectsTable : public ObjectCacheEntry { public: CSVRejectsTable(string name) : name(name), count(0) { if (name.empty()) { - scan_table = "reject_scan"; + scan_table = "reject_scans"; errors_table = "reject_errors"; } else { scan_table = name + "_scan"; diff --git a/test/sql/copy/csv/rejects/csv_rejects_double_table.test b/test/sql/copy/csv/rejects/csv_rejects_double_table.test index e69de29bb2d1..1d82bc77e2b8 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_double_table.test +++ b/test/sql/copy/csv/rejects/csv_rejects_double_table.test @@ -0,0 +1,34 @@ +# name: test/sql/copy/csv/rejects/csv_rejects_double_table.test +# group: [rejects] + +require skip_reload + +# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n +require notwindows + +# Ensure that we can get the schema if we reduce the sample size and ignore errors +query IIIII +SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto( + 'test/sql/copy/csv/data/error/mismatch/big_bad*.csv', + sample_size=1, + store_rejects=true, + ignore_errors=true); +---- +BIGINT VARCHAR 11044 11044 2 + + +query IIIIIIIIIIIII +SELECT * +FROM reject_scans order by all; +---- +3 0 test/sql/copy/csv/data/error/mismatch/big_bad.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL ignore_errors=true, store_rejects=true, sample_size=1 +3 1 test/sql/copy/csv/data/error/mismatch/big_bad2.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL ignore_errors=true, store_rejects=true, sample_size=1 + +query IIIIIIIII +SELECT * +FROM reject_errors order by all; +---- +3 0 2176 10875 1 "column0" CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +3 0 4176 20875 1 "column0" CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' +3 1 3680 18395 1 "column0" CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +3 1 5680 28395 1 "column0" CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' \ No newline at end of file From b6479d951ff573130543153374669038716e3c1c Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 11 Mar 2024 17:20:51 +0100 Subject: [PATCH 072/603] More adjustments --- .../table_function/global_csv_state.cpp | 4 ++-- .../csv/rejects/csv_rejects_double_table.test | 15 +++++++-------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index b1fef47e4d1b..e59d12963348 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -278,10 +278,10 @@ void CSVGlobalState::FillRejectsTable() { break; case CSVErrorType::TOO_FEW_COLUMNS: D_ASSERT(bind_data.return_names.size() > col_idx + 1); - errors_appender.Append(string_t("\"" + bind_data.return_names[col_idx + 1] + "\"")); + errors_appender.Append(string_t(bind_data.return_names[col_idx + 1])); break; default: - errors_appender.Append(string_t("\"" + bind_data.return_names[col_idx] + "\"")); + errors_appender.Append(string_t(bind_data.return_names[col_idx])); } // 7. Error Type errors_appender.Append(string_t(CSVErrorTypeToEnum(error.type))); diff --git a/test/sql/copy/csv/rejects/csv_rejects_double_table.test b/test/sql/copy/csv/rejects/csv_rejects_double_table.test index 1d82bc77e2b8..d2714bd3be55 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_double_table.test +++ b/test/sql/copy/csv/rejects/csv_rejects_double_table.test @@ -11,8 +11,7 @@ query IIIII SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto( 'test/sql/copy/csv/data/error/mismatch/big_bad*.csv', sample_size=1, - store_rejects=true, - ignore_errors=true); + store_rejects=true); ---- BIGINT VARCHAR 11044 11044 2 @@ -21,14 +20,14 @@ query IIIIIIIIIIIII SELECT * FROM reject_scans order by all; ---- -3 0 test/sql/copy/csv/data/error/mismatch/big_bad.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL ignore_errors=true, store_rejects=true, sample_size=1 -3 1 test/sql/copy/csv/data/error/mismatch/big_bad2.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL ignore_errors=true, store_rejects=true, sample_size=1 +3 0 test/sql/copy/csv/data/error/mismatch/big_bad.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL store_rejects=true, sample_size=1 +3 1 test/sql/copy/csv/data/error/mismatch/big_bad2.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL store_rejects=true, sample_size=1 query IIIIIIIII SELECT * FROM reject_errors order by all; ---- -3 0 2176 10875 1 "column0" CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' -3 0 4176 20875 1 "column0" CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' -3 1 3680 18395 1 "column0" CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' -3 1 5680 28395 1 "column0" CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' \ No newline at end of file +3 0 2176 10875 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +3 0 4176 20875 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' +3 1 3680 18395 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +3 1 5680 28395 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' \ No newline at end of file From dfc5e70cb01aac4c4453541799d653d7b57ee634 Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 12 Mar 2024 12:53:02 +0100 Subject: [PATCH 073/603] the tight constraints we set are broken when --force-storage is used, so we disable it --- test/sql/storage/temp_directory/max_swap_space_error.test | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/sql/storage/temp_directory/max_swap_space_error.test b/test/sql/storage/temp_directory/max_swap_space_error.test index 68c55f520665..6116c552ea7c 100644 --- a/test/sql/storage/temp_directory/max_swap_space_error.test +++ b/test/sql/storage/temp_directory/max_swap_space_error.test @@ -3,6 +3,8 @@ require skip_reload +require noforcestorage + # Set a temp_directory to offload data statement ok set temp_directory='__TEST_DIR__/max_swap_space_reached' From 34db0eee2f02b69e9d518bbeb9f7482593898c54 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Tue, 12 Mar 2024 15:03:42 +0100 Subject: [PATCH 074/603] Alright lets have different options for different tables --- .../table_function/global_csv_state.cpp | 3 ++- .../csv_scanner/util/csv_reader_options.cpp | 9 ++++++++- .../operator/persistent/csv_rejects_table.cpp | 8 +++++--- src/function/table/read_csv.cpp | 17 +++++++++++++---- .../operator/csv_scanner/csv_reader_options.hpp | 6 ++++-- .../operator/persistent/csv_rejects_table.hpp | 13 ++++--------- .../duckdb/storage/serialization/nodes.json | 6 +++++- src/storage/serialization/serialize_nodes.cpp | 6 ++++-- 8 files changed, 45 insertions(+), 23 deletions(-) diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index e59d12963348..ae2fddf9df0b 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -234,7 +234,8 @@ void CSVGlobalState::FillRejectsTable() { if (options.store_rejects.GetValue()) { auto limit = options.rejects_limit; - auto rejects = CSVRejectsTable::GetOrCreate(context, options.rejects_table_name); + auto rejects = CSVRejectsTable::GetOrCreate(context, options.rejects_scan_name.GetValue(), + options.rejects_table_name.GetValue()); lock_guard lock(rejects->write_lock); auto &errors_table = rejects->GetErrorsTable(context); auto &scans_table = rejects->GetScansTable(context); diff --git a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp index 7fcb7f3383b1..b06f58779328 100644 --- a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp +++ b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp @@ -214,7 +214,14 @@ void CSVReaderOptions::SetReadOption(const string &loption, const Value &value, if (table_name.empty()) { throw BinderException("REJECTS_TABLE option cannot be empty"); } - rejects_table_name = table_name; + rejects_table_name.Set(table_name); + } else if (loption == "rejects_scan") { + // skip, handled in SetRejectsOptions + auto table_name = ParseString(value, loption); + if (table_name.empty()) { + throw BinderException("rejects_scan option cannot be empty"); + } + rejects_scan_name.Set(table_name); } else if (loption == "rejects_limit") { int64_t limit = ParseInteger(value, loption); if (limit < 0) { diff --git a/src/execution/operator/persistent/csv_rejects_table.cpp b/src/execution/operator/persistent/csv_rejects_table.cpp index e74a7806a4ef..f9672135e4e9 100644 --- a/src/execution/operator/persistent/csv_rejects_table.cpp +++ b/src/execution/operator/persistent/csv_rejects_table.cpp @@ -19,10 +19,12 @@ TableCatalogEntry &CSVRejectsTable::GetScansTable(ClientContext &context) { return table_entry; } -shared_ptr CSVRejectsTable::GetOrCreate(ClientContext &context, const string &name) { - auto key = "CSV_REJECTS_TABLE_CACHE_ENTRY_" + StringUtil::Upper(name); +shared_ptr CSVRejectsTable::GetOrCreate(ClientContext &context, const string &rejects_scan, + const string &rejects_error) { + auto key = + "CSV_REJECTS_TABLE_CACHE_ENTRY_" + StringUtil::Upper(rejects_scan) + "_" + StringUtil::Upper(rejects_error); auto &cache = ObjectCache::GetObjectCache(context); - return cache.GetOrCreate(key, name); + return cache.GetOrCreate(key, rejects_scan, rejects_error); } void CSVRejectsTable::InitializeTable(ClientContext &context, const ReadCSVData &data) { diff --git a/src/function/table/read_csv.cpp b/src/function/table/read_csv.cpp index b7f865fc718e..258d69d3871a 100644 --- a/src/function/table/read_csv.cpp +++ b/src/function/table/read_csv.cpp @@ -51,13 +51,20 @@ static unique_ptr ReadCSVBind(ClientContext &context, TableFunctio result->files = MultiFileReader::GetFileList(context, input.inputs[0], "CSV"); options.FromNamedParameters(input.named_parameters, context, return_types, names); - if (!options.rejects_table_name.empty() && !options.store_rejects.GetValue() && + if (options.rejects_table_name.IsSetByUser() && !options.store_rejects.GetValue() && options.store_rejects.IsSetByUser()) { throw BinderException( "rejects_table_name option is only supported when store_rejects is not manually set to false"); } - // Ensure we set ignore errors to true automagically - options.store_rejects.Set(true, false); + if (options.rejects_scan_name.IsSetByUser() && !options.store_rejects.GetValue() && + options.store_rejects.IsSetByUser()) { + throw BinderException( + "rejects_scan_name option is only supported when store_rejects is not manually set to false"); + } + if (options.rejects_scan_name.IsSetByUser() || options.rejects_table_name.IsSetByUser()) { + // Ensure we set store_rejects to true automagically + options.store_rejects.Set(true, false); + } // Validate rejects_table options if (options.store_rejects.GetValue()) { if (!options.ignore_errors.GetValue() && options.ignore_errors.IsSetByUser()) { @@ -153,7 +160,8 @@ static unique_ptr ReadCSVInitGlobal(ClientContext &con // Create the temporary rejects table if (bind_data.options.store_rejects.GetValue()) { - CSVRejectsTable::GetOrCreate(context, bind_data.options.rejects_table_name) + CSVRejectsTable::GetOrCreate(context, bind_data.options.rejects_scan_name.GetValue(), + bind_data.options.rejects_table_name.GetValue()) ->InitializeTable(context, bind_data); } if (bind_data.files.empty()) { @@ -236,6 +244,7 @@ void ReadCSVTableFunction::ReadCSVAddNamedParameters(TableFunction &table_functi table_function.named_parameters["ignore_errors"] = LogicalType::BOOLEAN; table_function.named_parameters["store_rejects"] = LogicalType::BOOLEAN; table_function.named_parameters["rejects_table"] = LogicalType::VARCHAR; + table_function.named_parameters["rejects_scan"] = LogicalType::VARCHAR; table_function.named_parameters["rejects_limit"] = LogicalType::BIGINT; table_function.named_parameters["buffer_size"] = LogicalType::UBIGINT; table_function.named_parameters["decimal_separator"] = LogicalType::VARCHAR; diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp index faabfe62f23e..4b69d9aad222 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp @@ -43,8 +43,10 @@ struct CSVReaderOptions { CSVOption ignore_errors = false; //! Whether we store CSV Errors in the rejects table or not CSVOption store_rejects = false; - //! Rejects table name - string rejects_table_name; + //! Rejects table name (Name of the table the store rejects errors) + CSVOption rejects_table_name = {"reject_errors"}; + //! Rejects Scan name name (Name of the table the store rejects scans) + CSVOption rejects_scan_name = {"reject_scans"}; //! Rejects table entry limit (0 = no limit) idx_t rejects_limit = 0; //! Number of samples to buffer diff --git a/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp b/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp index 6254d7cecc01..ee1d2092660b 100644 --- a/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +++ b/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp @@ -14,14 +14,8 @@ class ClientContext; class CSVRejectsTable : public ObjectCacheEntry { public: - CSVRejectsTable(string name) : name(name), count(0) { - if (name.empty()) { - scan_table = "reject_scans"; - errors_table = "reject_errors"; - } else { - scan_table = name + "_scan"; - errors_table = name; - } + CSVRejectsTable(string rejects_scan, string rejects_error) + : count(0), scan_table(rejects_scan), errors_table(rejects_error) { } ~CSVRejectsTable() override = default; mutex write_lock; @@ -30,7 +24,8 @@ class CSVRejectsTable : public ObjectCacheEntry { string scan_table; string errors_table; - static shared_ptr GetOrCreate(ClientContext &context, const string &name); + static shared_ptr GetOrCreate(ClientContext &context, const string &rejects_scan, + const string &rejects_error); void InitializeTable(ClientContext &context, const ReadCSVData &options); TableCatalogEntry &GetErrorsTable(ClientContext &context); diff --git a/src/include/duckdb/storage/serialization/nodes.json b/src/include/duckdb/storage/serialization/nodes.json index 39961131cad2..e3bec298114c 100644 --- a/src/include/duckdb/storage/serialization/nodes.json +++ b/src/include/duckdb/storage/serialization/nodes.json @@ -653,7 +653,11 @@ }, {"id": 129, "name": "rejects_table_name", - "type": "string" + "type": "CSVOption" + }, + {"id": 130, + "name": "rejects_scan_name", + "type": "CSVOption" } ], "pointer_type": "none" diff --git a/src/storage/serialization/serialize_nodes.cpp b/src/storage/serialization/serialize_nodes.cpp index b7f0d3078810..bc9ee449a8d5 100644 --- a/src/storage/serialization/serialize_nodes.cpp +++ b/src/storage/serialization/serialize_nodes.cpp @@ -147,7 +147,8 @@ void CSVReaderOptions::Serialize(Serializer &serializer) const { serializer.WriteProperty>>(126, "dialect_options.date_format", dialect_options.date_format); serializer.WritePropertyWithDefault(127, "sniffer_user_mismatch_error", sniffer_user_mismatch_error); serializer.WritePropertyWithDefault(128, "parallel", parallel); - serializer.WritePropertyWithDefault(129, "rejects_table_name", rejects_table_name); + serializer.WriteProperty>(129, "rejects_table_name", rejects_table_name); + serializer.WriteProperty>(130, "rejects_scan_name", rejects_scan_name); } CSVReaderOptions CSVReaderOptions::Deserialize(Deserializer &deserializer) { @@ -181,7 +182,8 @@ CSVReaderOptions CSVReaderOptions::Deserialize(Deserializer &deserializer) { deserializer.ReadProperty>>(126, "dialect_options.date_format", result.dialect_options.date_format); deserializer.ReadPropertyWithDefault(127, "sniffer_user_mismatch_error", result.sniffer_user_mismatch_error); deserializer.ReadPropertyWithDefault(128, "parallel", result.parallel); - deserializer.ReadPropertyWithDefault(129, "rejects_table_name", result.rejects_table_name); + deserializer.ReadProperty>(129, "rejects_table_name", result.rejects_table_name); + deserializer.ReadProperty>(130, "rejects_scan_name", result.rejects_scan_name); return result; } From d877701dd895d2809c029cb1c32232ff43f12017 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Tue, 12 Mar 2024 17:43:15 +0100 Subject: [PATCH 075/603] Lots more tests --- src/catalog/catalog.cpp | 11 + .../csv_scanner/util/csv_reader_options.cpp | 2 +- .../operator/persistent/csv_rejects_table.cpp | 18 +- src/function/table/read_csv.cpp | 6 +- src/include/duckdb/catalog/catalog.hpp | 3 + .../csv/rejects/csv_rejects_double_table.test | 33 --- .../csv/rejects/csv_rejects_two_tables.test | 241 ++++++++++++++++++ 7 files changed, 274 insertions(+), 40 deletions(-) delete mode 100644 test/sql/copy/csv/rejects/csv_rejects_double_table.test create mode 100644 test/sql/copy/csv/rejects/csv_rejects_two_tables.test diff --git a/src/catalog/catalog.cpp b/src/catalog/catalog.cpp index 7294427a2942..1674128837f3 100644 --- a/src/catalog/catalog.cpp +++ b/src/catalog/catalog.cpp @@ -758,6 +758,17 @@ CatalogEntry &Catalog::GetEntry(ClientContext &context, const string &schema, co throw CatalogException("CatalogElement \"%s.%s\" does not exist!", schema, name); } +bool Catalog::EntryExists(ClientContext &context, const string &schema, const string &name) { + vector entry_types {CatalogType::TABLE_ENTRY, CatalogType::SEQUENCE_ENTRY}; + for (auto entry_type : entry_types) { + auto result = GetEntry(context, entry_type, schema, name, OnEntryNotFound::RETURN_NULL); + if (result) { + return true; + } + } + return false; +} + optional_ptr Catalog::GetEntry(ClientContext &context, CatalogType type, const string &schema_name, const string &name, OnEntryNotFound if_not_found, QueryErrorContext error_context) { diff --git a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp index b06f58779328..9ea7bb80992d 100644 --- a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp +++ b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp @@ -207,7 +207,7 @@ void CSVReaderOptions::SetReadOption(const string &loption, const Value &value, } else if (loption == "allow_quoted_nulls") { allow_quoted_nulls = ParseBoolean(value, loption); } else if (loption == "store_rejects") { - store_rejects = ParseBoolean(value, loption); + store_rejects.Set(ParseBoolean(value, loption)); } else if (loption == "rejects_table") { // skip, handled in SetRejectsOptions auto table_name = ParseString(value, loption); diff --git a/src/execution/operator/persistent/csv_rejects_table.cpp b/src/execution/operator/persistent/csv_rejects_table.cpp index f9672135e4e9..429d385553e2 100644 --- a/src/execution/operator/persistent/csv_rejects_table.cpp +++ b/src/execution/operator/persistent/csv_rejects_table.cpp @@ -24,6 +24,20 @@ shared_ptr CSVRejectsTable::GetOrCreate(ClientContext &context, auto key = "CSV_REJECTS_TABLE_CACHE_ENTRY_" + StringUtil::Upper(rejects_scan) + "_" + StringUtil::Upper(rejects_error); auto &cache = ObjectCache::GetObjectCache(context); + auto &catalog = Catalog::GetCatalog(context, TEMP_CATALOG); + bool rejects_scan_exist = catalog.EntryExists(context, DEFAULT_SCHEMA, rejects_scan); + bool rejects_error_exist = catalog.EntryExists(context, DEFAULT_SCHEMA, rejects_error); + if ((rejects_scan_exist || rejects_error_exist) && !cache.Get(key)) { + std::ostringstream error; + if (rejects_scan_exist) { + error << "Reject Scan Table name \"" << rejects_scan << "\" is already in use. "; + } + if (rejects_error_exist) { + error << "Reject Error Table name \"" << rejects_error << "\" is already in use. "; + } + error << "Either drop the used name(s), or give other name options in the CSV Reader function.\n"; + throw BinderException(error.str()); + } return cache.GetOrCreate(key, rejects_scan, rejects_error); } @@ -50,7 +64,7 @@ void CSVRejectsTable::InitializeTable(ClientContext &context, const ReadCSVData { auto info = make_uniq(TEMP_CATALOG, DEFAULT_SCHEMA, scan_table); info->temporary = true; - info->on_conflict = OnCreateConflict::ERROR_ON_CONFLICT; + info->on_conflict = OnCreateConflict::IGNORE_ON_CONFLICT; // 0. Scan ID info->columns.AddColumn(ColumnDefinition("scan_id", LogicalType::UBIGINT)); // 1. File ID (within the scan) @@ -83,7 +97,7 @@ void CSVRejectsTable::InitializeTable(ClientContext &context, const ReadCSVData // Create Rejects Error Table auto info = make_uniq(TEMP_CATALOG, DEFAULT_SCHEMA, errors_table); info->temporary = true; - info->on_conflict = OnCreateConflict::ERROR_ON_CONFLICT; + info->on_conflict = OnCreateConflict::IGNORE_ON_CONFLICT; // 0. Scan ID info->columns.AddColumn(ColumnDefinition("scan_id", LogicalType::UBIGINT)); // 1. File ID (within the scan) diff --git a/src/function/table/read_csv.cpp b/src/function/table/read_csv.cpp index 258d69d3871a..60d942b3c8a7 100644 --- a/src/function/table/read_csv.cpp +++ b/src/function/table/read_csv.cpp @@ -53,13 +53,11 @@ static unique_ptr ReadCSVBind(ClientContext &context, TableFunctio options.FromNamedParameters(input.named_parameters, context, return_types, names); if (options.rejects_table_name.IsSetByUser() && !options.store_rejects.GetValue() && options.store_rejects.IsSetByUser()) { - throw BinderException( - "rejects_table_name option is only supported when store_rejects is not manually set to false"); + throw BinderException("REJECTS_TABLE option is only supported when store_rejects is not manually set to false"); } if (options.rejects_scan_name.IsSetByUser() && !options.store_rejects.GetValue() && options.store_rejects.IsSetByUser()) { - throw BinderException( - "rejects_scan_name option is only supported when store_rejects is not manually set to false"); + throw BinderException("REJECTS_SCAN option is only supported when store_rejects is not manually set to false"); } if (options.rejects_scan_name.IsSetByUser() || options.rejects_table_name.IsSetByUser()) { // Ensure we set store_rejects to true automagically diff --git a/src/include/duckdb/catalog/catalog.hpp b/src/include/duckdb/catalog/catalog.hpp index 0bbf322c628b..ead5f183c75b 100644 --- a/src/include/duckdb/catalog/catalog.hpp +++ b/src/include/duckdb/catalog/catalog.hpp @@ -228,6 +228,9 @@ class Catalog { //! Gets the "schema.name" entry without a specified type, if entry does not exist an exception is thrown DUCKDB_API CatalogEntry &GetEntry(ClientContext &context, const string &schema, const string &name); + //! Returns true if the "schema.name" entry without a specified type exists + DUCKDB_API bool EntryExists(ClientContext &context, const string &schema, const string &name); + //! Fetches a logical type from the catalog DUCKDB_API LogicalType GetType(ClientContext &context, const string &schema, const string &names, OnEntryNotFound if_not_found); diff --git a/test/sql/copy/csv/rejects/csv_rejects_double_table.test b/test/sql/copy/csv/rejects/csv_rejects_double_table.test deleted file mode 100644 index d2714bd3be55..000000000000 --- a/test/sql/copy/csv/rejects/csv_rejects_double_table.test +++ /dev/null @@ -1,33 +0,0 @@ -# name: test/sql/copy/csv/rejects/csv_rejects_double_table.test -# group: [rejects] - -require skip_reload - -# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n -require notwindows - -# Ensure that we can get the schema if we reduce the sample size and ignore errors -query IIIII -SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto( - 'test/sql/copy/csv/data/error/mismatch/big_bad*.csv', - sample_size=1, - store_rejects=true); ----- -BIGINT VARCHAR 11044 11044 2 - - -query IIIIIIIIIIIII -SELECT * -FROM reject_scans order by all; ----- -3 0 test/sql/copy/csv/data/error/mismatch/big_bad.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL store_rejects=true, sample_size=1 -3 1 test/sql/copy/csv/data/error/mismatch/big_bad2.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL store_rejects=true, sample_size=1 - -query IIIIIIIII -SELECT * -FROM reject_errors order by all; ----- -3 0 2176 10875 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' -3 0 4176 20875 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' -3 1 3680 18395 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' -3 1 5680 28395 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' \ No newline at end of file diff --git a/test/sql/copy/csv/rejects/csv_rejects_two_tables.test b/test/sql/copy/csv/rejects/csv_rejects_two_tables.test new file mode 100644 index 000000000000..e9ad454f6052 --- /dev/null +++ b/test/sql/copy/csv/rejects/csv_rejects_two_tables.test @@ -0,0 +1,241 @@ +# name: test/sql/copy/csv/rejects/csv_rejects_two_tables.test +# group: [rejects] + +require skip_reload + +# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n +require notwindows + +# Ensure that we can get the schema if we reduce the sample size and ignore errors +query IIIII +SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto( + 'test/sql/copy/csv/data/error/mismatch/big_bad*.csv', + sample_size=1, + store_rejects=true); +---- +BIGINT VARCHAR 11044 11044 2 + + +query IIIIIIIIIIIII +SELECT * +FROM reject_scans order by all; +---- +3 0 test/sql/copy/csv/data/error/mismatch/big_bad.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL store_rejects=true, sample_size=1 +3 1 test/sql/copy/csv/data/error/mismatch/big_bad2.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL store_rejects=true, sample_size=1 + +query IIIIIIIII +SELECT * +FROM reject_errors order by all; +---- +3 0 2176 10875 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +3 0 4176 20875 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' +3 1 3680 18395 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +3 1 5680 28395 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' + +# Test giving the name of errors table +statement error +SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto( + 'test/sql/copy/csv/data/error/mismatch/big_bad*.csv', + sample_size=1, + rejects_table = 'rejects_errors_2'); +---- +Reject Scan Table name "reject_scans" is already in use. Either drop the used name(s), or give other name options in the CSV Reader function. + +statement ok +drop table reject_scans; + +query IIIII +SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto( + 'test/sql/copy/csv/data/error/mismatch/big_bad*.csv', + sample_size=1, + rejects_table = 'rejects_errors_2' + ); +---- +BIGINT VARCHAR 11044 11044 2 + +query IIIIIIIIIIIII +SELECT * +FROM reject_scans order by all; +---- +8 0 test/sql/copy/csv/data/error/mismatch/big_bad.csv , \0 \0 \n 0 false {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_table='rejects_errors_2', sample_size=1 +8 1 test/sql/copy/csv/data/error/mismatch/big_bad2.csv , \0 \0 \n 0 false {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_table='rejects_errors_2', sample_size=1 + +query IIIIIIIII +SELECT * +FROM rejects_errors_2 order by all; +---- +8 0 2176 10875 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +8 0 4176 20875 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' +8 1 3680 18395 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +8 1 5680 28395 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' + +statement ok +drop table reject_errors; + +# Test giving the name of scans table +query IIIII +SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto( + 'test/sql/copy/csv/data/error/mismatch/big_bad*.csv', + sample_size=1, + rejects_scan = 'rejects_scan_2'); +---- +BIGINT VARCHAR 11044 11044 2 + +query IIIIIIIIIIIII +SELECT * +FROM rejects_scan_2 order by all; +---- +12 0 test/sql/copy/csv/data/error/mismatch/big_bad.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_scan='rejects_scan_2', sample_size=1 +12 1 test/sql/copy/csv/data/error/mismatch/big_bad2.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_scan='rejects_scan_2', sample_size=1 + +query IIIIIIIII +SELECT * +FROM reject_errors order by all; +---- +12 0 2176 10875 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +12 0 4176 20875 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' +12 1 3680 18395 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +12 1 5680 28395 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' + + +# Test giving the name of both tables +query IIIII +SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto( + 'test/sql/copy/csv/data/error/mismatch/big_bad*.csv', + sample_size=1, + rejects_scan = 'rejects_scan_3', + rejects_table = 'rejects_errors_3' + ); +---- +BIGINT VARCHAR 11044 11044 2 + +query IIIIIIIIIIIII +SELECT * +FROM rejects_scan_3 order by all; +---- +15 0 test/sql/copy/csv/data/error/mismatch/big_bad.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_table='rejects_errors_3', rejects_scan='rejects_scan_3', sample_size=1 +15 1 test/sql/copy/csv/data/error/mismatch/big_bad2.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_table='rejects_errors_3', rejects_scan='rejects_scan_3', sample_size=1 + +query IIIIIIIII +SELECT * +FROM rejects_errors_3 order by all; +---- +15 0 2176 10875 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +15 0 4176 20875 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' +15 1 3680 18395 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +15 1 5680 28395 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' + +statement ok +drop table reject_errors; + +statement ok +drop table reject_scans; + + +# Test giving the name of an existing table to the errors table +statement ok +create temporary table t (a integer); + +statement error +SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto( + 'test/sql/copy/csv/data/error/mismatch/big_bad*.csv', + sample_size=1, + rejects_table = 't' + ); +---- +Reject Error Table name "t" is already in use. Either drop the used name(s), or give other name options in the CSV Reader function. + +# Test giving the name of an existing table to the scans table + +statement error +SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto( + 'test/sql/copy/csv/data/error/mismatch/big_bad*.csv', + sample_size=1, + rejects_scan = 't' + ); +---- +Reject Scan Table name "t" is already in use. Either drop the used name(s), or give other name options in the CSV Reader function. + +statement error +SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto( + 'test/sql/copy/csv/data/error/mismatch/big_bad*.csv', + sample_size=1, + rejects_table = 't', + rejects_scan = 't' + ); +---- +Reject Scan Table name "t" is already in use. Reject Error Table name "t" is already in use. Either drop the used name(s), or give other name options in the CSV Reader function. + + +# Test giving the name of the tables with store_rejects and/or ignore_errors set to false throws +statement error +SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto( + 'test/sql/copy/csv/data/error/mismatch/big_bad*.csv', + sample_size=1, + rejects_scan = 'rejects_scan_3', + rejects_table = 'rejects_errors_3', + ignore_errors = false + ); +---- +STORE_REJECTS option is only supported when IGNORE_ERRORS is not manually set to false + +statement error +SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto( + 'test/sql/copy/csv/data/error/mismatch/big_bad*.csv', + sample_size=1, + store_rejects = true, + ignore_errors = false + ); +---- +STORE_REJECTS option is only supported when IGNORE_ERRORS is not manually set to false + +statement error +SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto( + 'test/sql/copy/csv/data/error/mismatch/big_bad*.csv', + sample_size=1, + rejects_table = 'rejects_errors_3', + ignore_errors = false + ); +---- +STORE_REJECTS option is only supported when IGNORE_ERRORS is not manually set to false + +statement error +SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto( + 'test/sql/copy/csv/data/error/mismatch/big_bad*.csv', + sample_size=1, + rejects_scan = 'rejects_scan_3', + ignore_errors = false + ); +---- +STORE_REJECTS option is only supported when IGNORE_ERRORS is not manually set to false + +statement error +SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto( + 'test/sql/copy/csv/data/error/mismatch/big_bad*.csv', + sample_size=1, + rejects_scan = 'rejects_scan_3', + rejects_table = 'rejects_errors_3', + store_rejects = false + ); +---- +REJECTS_TABLE option is only supported when store_rejects is not manually set to false + +statement error +SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto( + 'test/sql/copy/csv/data/error/mismatch/big_bad*.csv', + sample_size=1, + rejects_table = 'rejects_errors_3', + store_rejects = false + ); +---- +REJECTS_TABLE option is only supported when store_rejects is not manually set to false + +statement error +SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto( + 'test/sql/copy/csv/data/error/mismatch/big_bad*.csv', + sample_size=1, + rejects_scan = 'rejects_scan_3', + store_rejects = false + ); +---- +REJECTS_SCAN option is only supported when store_rejects is not manually set to false \ No newline at end of file From baca88167e12a3233aaa7aaee83a2b7ff1339c9e Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 13 Mar 2024 14:34:14 +0100 Subject: [PATCH 076/603] Lots of adjustments to make if possible to have cur_pos the exact place where an error happened and to produce multiple errors in the same row --- .../scanner/string_value_scanner.cpp | 121 +++++++++--------- .../csv_scanner/string_value_scanner.hpp | 28 ++-- 2 files changed, 70 insertions(+), 79 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index ae9d470a32ce..277924d8c01a 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -111,10 +111,6 @@ inline bool IsValueNull(const char *null_str_ptr, const char *value_ptr, const i } void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size, bool allocate) { - if (current_error.is_set) { - cur_col_id++; - return; - } if (cur_col_id >= number_of_columns) { bool error = true; if (cur_col_id == number_of_columns && ((quoted && state_machine.options.allow_quoted_nulls) || !quoted)) { @@ -122,7 +118,9 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size error = !IsValueNull(null_str_ptr, value_ptr, size); } if (error) { - current_error = {CSVErrorType::TOO_MANY_COLUMNS, cur_col_id}; + // We error pointing to the current value error. + current_errors.push_back( + {CSVErrorType::TOO_MANY_COLUMNS, cur_col_id, {iterator.pos.buffer_idx, last_position, buffer_size}}); } return; } @@ -143,7 +141,9 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size if (empty) { if (parse_types[chunk_col_id].first != LogicalTypeId::VARCHAR) { // If it is not a varchar, empty values are not accepted, we must error. - cast_errors[chunk_col_id] = std::string(""); + current_errors.push_back({CSVErrorType::CAST_ERROR, + cur_col_id, + {iterator.pos.buffer_idx, last_position, buffer_size}}); } static_cast(vector_ptr[chunk_col_id])[number_of_rows] = string_t(); } else { @@ -225,7 +225,8 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size HandleUnicodeError(cur_col_id, force_error); } // If we got here, we are ingoring errors, hence we must ignore this line. - current_error = {CSVErrorType::INVALID_UNICODE, cur_col_id}; + current_errors.push_back( + {CSVErrorType::INVALID_UNICODE, cur_col_id, {iterator.pos.buffer_idx, last_position, buffer_size}}); break; } if (allocate) { @@ -241,7 +242,13 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size } if (!success) { // We had a casting error, we push it here because we can only error when finishing the line read. - cast_errors[cur_col_id] = std::string(value_ptr, size); + std::ostringstream error; + // Casting Error Message + error << "Could not convert string \"" << std::string(value_ptr, size) << "\" to \'" + << LogicalTypeIdToString(parse_types[cur_col_id].first) << "\'"; + current_errors.push_back( + {CSVErrorType::INVALID_UNICODE, cur_col_id, {iterator.pos.buffer_idx, last_position, buffer_size}}); + current_errors.back().error_message = error.str(); } cur_col_id++; chunk_col_id++; @@ -282,7 +289,7 @@ void StringValueResult::Reset() { if (cur_buffer) { buffer_handles[cur_buffer->buffer_idx] = cur_buffer; } - current_error.Reset(); + current_errors.clear(); } void StringValueResult::AddQuotedValue(StringValueResult &result, const idx_t buffer_pos) { @@ -327,16 +334,6 @@ void StringValueResult::AddValue(StringValueResult &result, const idx_t buffer_p result.last_position = buffer_pos + 1; } -void StringValueResult::HandleOverLimitRows(idx_t col_idx) { - LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); - bool first_nl; - auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); - auto csv_error = - CSVError::IncorrectColumnAmountError(state_machine.options, col_idx, lines_per_batch, borked_line, - current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); - error_handler.Error(csv_error); -} - void StringValueResult::HandleUnicodeError(idx_t col_idx, bool force_error) { bool first_nl; auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); @@ -351,35 +348,50 @@ void StringValueResult::HandleUnicodeError(idx_t col_idx, bool force_error) { error_handler.Error(csv_error, force_error); } -void StringValueResult::HandleUnterminatedQuotes(idx_t col_idx, bool force_error) { - LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); - bool first_nl; - auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); - auto csv_error = - CSVError::UnterminatedQuotesError(state_machine.options, col_idx, lines_per_batch, borked_line, - current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); - error_handler.Error(csv_error, force_error); -} - bool StringValueResult::HandleError() { - if (current_error.is_set) { - switch (current_error.type) { + // Reconstruct CSV Line + for (auto &cur_error : current_errors) { + LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); + bool first_nl; + auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); + CSVError csv_error; + auto col_idx = cur_error.col_idx; + auto &line_pos = cur_error.error_position; + + switch (cur_error.type) { case CSVErrorType::TOO_MANY_COLUMNS: - HandleOverLimitRows(cur_col_id); + csv_error = + CSVError::IncorrectColumnAmountError(state_machine.options, col_idx, lines_per_batch, borked_line, + line_pos.GetGlobalPosition(requested_size, first_nl)); break; - case CSVErrorType::INVALID_UNICODE: - HandleUnicodeError(current_error.col_idx); + case CSVErrorType::INVALID_UNICODE: { + // We have to sanitize the CSV line + std::vector char_array(borked_line.begin(), borked_line.end()); + char_array.push_back('\0'); // Null-terminate the character array + Utf8Proc::MakeValid(&char_array[0], char_array.size()); + borked_line = {char_array.begin(), char_array.end() - 1}; + csv_error = CSVError::InvalidUTF8(state_machine.options, col_idx, lines_per_batch, borked_line, + line_pos.GetGlobalPosition(requested_size, first_nl)); break; + } case CSVErrorType::UNTERMINATED_QUOTES: - HandleUnterminatedQuotes(current_error.col_idx); + csv_error = CSVError::UnterminatedQuotesError(state_machine.options, col_idx, lines_per_batch, borked_line, + line_pos.GetGlobalPosition(requested_size, first_nl)); + break; + case CSVErrorType::CAST_ERROR: + csv_error = CSVError::CastError(state_machine.options, names[cur_error.col_idx], cur_error.error_message, + cur_error.col_idx, borked_line, lines_per_batch, + current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); break; default: throw InvalidInputException("CSV Error not allowed when inserting row"); } + error_handler.Error(csv_error); + } + if (!current_errors.empty()) { + current_errors.clear(); cur_col_id = 0; chunk_col_id = 0; - // An error occurred on this row, we are ignoring it and resetting our control flag - current_error.Reset(); return true; } return false; @@ -455,30 +467,6 @@ bool StringValueResult::AddRowInternal() { if (HandleError()) { return false; } - if (!cast_errors.empty()) { - // A wild casting error appears - for (auto &cast_error : cast_errors) { - std::ostringstream error; - // Casting Error Message - error << "Could not convert string \"" << cast_error.second << "\" to \'" - << LogicalTypeIdToString(parse_types[cast_error.first].first) << "\'"; - auto error_string = error.str(); - LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); - bool first_nl; - auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); - auto csv_error = CSVError::CastError( - state_machine.options, names[cast_error.first], error_string, cast_error.first, borked_line, - lines_per_batch, current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); - error_handler.Error(csv_error); - } - // If we got here it means we are ignoring errors, hence we need to signify to our result scanner to ignore this - // row - // Cleanup this line and continue - cast_errors.clear(); - cur_col_id = 0; - chunk_col_id = 0; - return false; - } NullPaddingQuotedNewlineCheck(); quoted_new_line = false; // We need to check if we are getting the correct number of columns here. @@ -559,7 +547,9 @@ void StringValueResult::InvalidState(StringValueResult &result) { if (force_error) { result.HandleUnicodeError(result.cur_col_id, force_error); } - result.current_error = {CSVErrorType::UNTERMINATED_QUOTES, result.cur_col_id}; + result.current_errors.push_back({CSVErrorType::INVALID_UNICODE, + result.cur_col_id, + {result.iterator.pos.buffer_idx, result.last_position, result.buffer_size}}); } bool StringValueResult::EmptyLine(StringValueResult &result, const idx_t buffer_pos) { @@ -1229,9 +1219,12 @@ void StringValueScanner::FinalizeChunkProcess() { // If we are not done we have two options. // 1) If a boundary is set. if (iterator.IsBoundarySet()) { - if (!(result.current_error == CSVErrorType::UNTERMINATED_QUOTES)) { - iterator.done = true; + for (auto &cur_error : result.current_errors) { + if (!(cur_error == CSVErrorType::UNTERMINATED_QUOTES)) { + iterator.done = true; + } } + // We read until the next line or until we have nothing else to read. // Move to next buffer if (!cur_buffer_handle) { diff --git a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp index e36266d90f69..4a54bc7dcccc 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp @@ -64,16 +64,17 @@ class FullLinePosition { class CurrentError { public: - CurrentError() : is_set(false) {}; - CurrentError(CSVErrorType type, idx_t col_idx_p) : is_set(true), type(type), col_idx(col_idx_p) {}; - void Reset() { - is_set = false; - } - bool is_set; + CurrentError(CSVErrorType type, idx_t col_idx_p, LinePosition error_position_p) + : type(type), col_idx(col_idx_p), error_position(error_position_p) {}; + CSVErrorType type; idx_t col_idx; + string error_message; + //! Exact Position where the error happened + LinePosition error_position; + friend bool operator==(const CurrentError &error, CSVErrorType other) { - return error.is_set && error.type == other; + return error.type == other; } }; @@ -81,8 +82,8 @@ class StringValueResult : public ScannerResult { public: StringValueResult(CSVStates &states, CSVStateMachine &state_machine, const shared_ptr &buffer_handle, Allocator &buffer_allocator, idx_t result_size, - idx_t buffer_position, CSVErrorHandler &error_hander, CSVIterator &iterator, bool store_line_size, - shared_ptr csv_file_scan, idx_t &lines_read, bool sniffing); + idx_t buffer_position, CSVErrorHandler &error_handler, CSVIterator &iterator, + bool store_line_size, shared_ptr csv_file_scan, idx_t &lines_read, bool sniffing); ~StringValueResult(); @@ -120,7 +121,6 @@ class StringValueResult : public ScannerResult { unsafe_unique_array> parse_types; vector names; - unordered_map cast_errors; shared_ptr csv_file_scan; idx_t &lines_read; @@ -135,8 +135,8 @@ class StringValueResult : public ScannerResult { //! Requested size of buffers (i.e., either 32Mb or set by buffer_size parameter) idx_t requested_size; - //! Current Error if any - CurrentError current_error; + //! Errors happening in the current line (if any) + vector current_errors; bool sniffing; //! Specialized code for quoted values, makes sure to remove quotes and escapes @@ -153,10 +153,8 @@ class StringValueResult : public ScannerResult { //! Handles EmptyLine states static inline bool EmptyLine(StringValueResult &result, const idx_t buffer_pos); inline bool AddRowInternal(); - - void HandleOverLimitRows(idx_t col_idx); void HandleUnicodeError(idx_t col_idx, bool force_error = false); - void HandleUnterminatedQuotes(idx_t col_idx, bool force_error = false); + //! Certain errors should only be handled when adding the line, to ensure proper error propagation. bool HandleError(); inline void AddValueToVector(const char *value_ptr, const idx_t size, bool allocate = false); From 2fe296f606f1543c152d493a0449a27cc6672025 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 13 Mar 2024 14:46:23 +0100 Subject: [PATCH 077/603] More adjustments --- .../scanner/string_value_scanner.cpp | 26 ++++++++++++------- .../csv_scanner/string_value_scanner.hpp | 3 ++- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 277924d8c01a..144e8ea9d679 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -216,17 +216,17 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size break; } default: { - // By default we add a string + // By default, we add a string // We only evaluate if a string is utf8 valid, if it's actually a varchar if (parse_types[chunk_col_id].second && !Utf8Proc::IsValid(value_ptr, UnsafeNumericCast(size))) { bool force_error = !state_machine.options.ignore_errors.GetValue() && sniffing; + LinePosition error_position {iterator.pos.buffer_idx, last_position, buffer_size}; // Invalid unicode, we must error if (force_error) { - HandleUnicodeError(cur_col_id, force_error); + HandleUnicodeError(cur_col_id, error_position); } // If we got here, we are ingoring errors, hence we must ignore this line. - current_errors.push_back( - {CSVErrorType::INVALID_UNICODE, cur_col_id, {iterator.pos.buffer_idx, last_position, buffer_size}}); + current_errors.push_back({CSVErrorType::INVALID_UNICODE, cur_col_id, error_position}); break; } if (allocate) { @@ -334,7 +334,7 @@ void StringValueResult::AddValue(StringValueResult &result, const idx_t buffer_p result.last_position = buffer_pos + 1; } -void StringValueResult::HandleUnicodeError(idx_t col_idx, bool force_error) { +void StringValueResult::HandleUnicodeError(idx_t col_idx, LinePosition &error_position) { bool first_nl; auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); // sanitize borked line @@ -344,8 +344,8 @@ void StringValueResult::HandleUnicodeError(idx_t col_idx, bool force_error) { borked_line = {char_array.begin(), char_array.end() - 1}; LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); auto csv_error = CSVError::InvalidUTF8(state_machine.options, col_idx, lines_per_batch, borked_line, - current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); - error_handler.Error(csv_error, force_error); + error_position.GetGlobalPosition(requested_size, first_nl)); + error_handler.Error(csv_error, true); } bool StringValueResult::HandleError() { @@ -381,7 +381,7 @@ bool StringValueResult::HandleError() { case CSVErrorType::CAST_ERROR: csv_error = CSVError::CastError(state_machine.options, names[cur_error.col_idx], cur_error.error_message, cur_error.col_idx, borked_line, lines_per_batch, - current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); + line_pos.GetGlobalPosition(requested_size, first_nl)); break; default: throw InvalidInputException("CSV Error not allowed when inserting row"); @@ -498,9 +498,10 @@ bool StringValueResult::AddRowInternal() { bool first_nl; auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); + LinePosition error_position {iterator.pos.buffer_idx, last_position, buffer_size}; auto csv_error = CSVError::IncorrectColumnAmountError( state_machine.options, cur_col_id - 1, lines_per_batch, borked_line, - current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); + error_position.GetGlobalPosition(requested_size, first_nl)); error_handler.Error(csv_error); // If we are here we ignore_errors, so we delete this line number_of_rows--; @@ -545,7 +546,8 @@ void StringValueResult::InvalidState(StringValueResult &result) { bool force_error = !result.state_machine.options.ignore_errors.GetValue() && result.sniffing; // Invalid unicode, we must error if (force_error) { - result.HandleUnicodeError(result.cur_col_id, force_error); + LinePosition error_position {result.iterator.pos.buffer_idx, result.last_position, result.buffer_size}; + result.HandleUnicodeError(result.cur_col_id, error_position); } result.current_errors.push_back({CSVErrorType::INVALID_UNICODE, result.cur_col_id, @@ -722,6 +724,8 @@ void StringValueScanner::Flush(DataChunk &insert_chunk) { bool first_nl; auto borked_line = result.line_positions_per_row[line_error].ReconstructCurrentLine(first_nl, result.buffer_handles); + // TODO: We can't really nicely get the position where this error happened, this should be solved by + // TODO: adding more types to implicit casting instead of relying on this flush. auto csv_error = CSVError::CastError( state_machine->options, csv_file_scan->names[col_idx], error_message, col_idx, borked_line, lines_per_batch, @@ -744,6 +748,8 @@ void StringValueScanner::Flush(DataChunk &insert_chunk) { bool first_nl; auto borked_line = result.line_positions_per_row[line_error].ReconstructCurrentLine( first_nl, result.buffer_handles); + // TODO: We can't really nicely get the position where this error happened, this should be solved by + // TODO: adding more types to implicit casting instead of relying on this flush. auto csv_error = CSVError::CastError(state_machine->options, csv_file_scan->names[col_idx], error_message, col_idx, borked_line, lines_per_batch, diff --git a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp index 4a54bc7dcccc..dfbe1f581bd5 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp @@ -153,7 +153,8 @@ class StringValueResult : public ScannerResult { //! Handles EmptyLine states static inline bool EmptyLine(StringValueResult &result, const idx_t buffer_pos); inline bool AddRowInternal(); - void HandleUnicodeError(idx_t col_idx, bool force_error = false); + //! Force the throw of a unicode error + void HandleUnicodeError(idx_t col_idx, LinePosition &error_position); //! Certain errors should only be handled when adding the line, to ensure proper error propagation. bool HandleError(); From dcdb16a1f276a5babf33aeee1adbd43726f3fed2 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 13 Mar 2024 16:01:24 +0100 Subject: [PATCH 078/603] Add a new return option for start of the line as a byte position --- .../scanner/string_value_scanner.cpp | 41 +++++++++++-------- .../table_function/global_csv_state.cpp | 22 ++++++---- .../operator/csv_scanner/util/csv_error.cpp | 38 +++++++++-------- .../operator/persistent/csv_rejects_table.cpp | 14 ++++--- .../operator/csv_scanner/csv_error.hpp | 19 +++++---- 5 files changed, 80 insertions(+), 54 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 144e8ea9d679..103ff6341a65 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -247,7 +247,7 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size error << "Could not convert string \"" << std::string(value_ptr, size) << "\" to \'" << LogicalTypeIdToString(parse_types[cur_col_id].first) << "\'"; current_errors.push_back( - {CSVErrorType::INVALID_UNICODE, cur_col_id, {iterator.pos.buffer_idx, last_position, buffer_size}}); + {CSVErrorType::CAST_ERROR, cur_col_id, {iterator.pos.buffer_idx, last_position, buffer_size}}); current_errors.back().error_message = error.str(); } cur_col_id++; @@ -344,6 +344,7 @@ void StringValueResult::HandleUnicodeError(idx_t col_idx, LinePosition &error_po borked_line = {char_array.begin(), char_array.end() - 1}; LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); auto csv_error = CSVError::InvalidUTF8(state_machine.options, col_idx, lines_per_batch, borked_line, + current_line_position.begin.GetGlobalPosition(requested_size, first_nl), error_position.GetGlobalPosition(requested_size, first_nl)); error_handler.Error(csv_error, true); } @@ -360,9 +361,10 @@ bool StringValueResult::HandleError() { switch (cur_error.type) { case CSVErrorType::TOO_MANY_COLUMNS: - csv_error = - CSVError::IncorrectColumnAmountError(state_machine.options, col_idx, lines_per_batch, borked_line, - line_pos.GetGlobalPosition(requested_size, first_nl)); + csv_error = CSVError::IncorrectColumnAmountError( + state_machine.options, col_idx, lines_per_batch, borked_line, + current_line_position.begin.GetGlobalPosition(requested_size, first_nl), + line_pos.GetGlobalPosition(requested_size, first_nl)); break; case CSVErrorType::INVALID_UNICODE: { // We have to sanitize the CSV line @@ -371,16 +373,20 @@ bool StringValueResult::HandleError() { Utf8Proc::MakeValid(&char_array[0], char_array.size()); borked_line = {char_array.begin(), char_array.end() - 1}; csv_error = CSVError::InvalidUTF8(state_machine.options, col_idx, lines_per_batch, borked_line, + current_line_position.begin.GetGlobalPosition(requested_size, first_nl), line_pos.GetGlobalPosition(requested_size, first_nl)); break; } case CSVErrorType::UNTERMINATED_QUOTES: - csv_error = CSVError::UnterminatedQuotesError(state_machine.options, col_idx, lines_per_batch, borked_line, - line_pos.GetGlobalPosition(requested_size, first_nl)); + csv_error = CSVError::UnterminatedQuotesError( + state_machine.options, col_idx, lines_per_batch, borked_line, + current_line_position.begin.GetGlobalPosition(requested_size, first_nl), + line_pos.GetGlobalPosition(requested_size, first_nl)); break; case CSVErrorType::CAST_ERROR: csv_error = CSVError::CastError(state_machine.options, names[cur_error.col_idx], cur_error.error_message, cur_error.col_idx, borked_line, lines_per_batch, + current_line_position.begin.GetGlobalPosition(requested_size, first_nl), line_pos.GetGlobalPosition(requested_size, first_nl)); break; default: @@ -404,7 +410,7 @@ void StringValueResult::QuotedNewLine(StringValueResult &result) { void StringValueResult::NullPaddingQuotedNewlineCheck() { // We do some checks for null_padding correctness if (state_machine.options.null_padding && iterator.IsBoundarySet() && quoted_new_line && iterator.done) { - // If we have null_padding set, we found a quoted new line, we are scanning the file in parallel and it's the + // If we have null_padding set, we found a quoted new line, we are scanning the file in parallel, and it's the // last row of this thread. LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); auto csv_error = CSVError::NullPaddingFail(state_machine.options, lines_per_batch); @@ -501,6 +507,7 @@ bool StringValueResult::AddRowInternal() { LinePosition error_position {iterator.pos.buffer_idx, last_position, buffer_size}; auto csv_error = CSVError::IncorrectColumnAmountError( state_machine.options, cur_col_id - 1, lines_per_batch, borked_line, + current_line_position.begin.GetGlobalPosition(requested_size, first_nl), error_position.GetGlobalPosition(requested_size, first_nl)); error_handler.Error(csv_error); // If we are here we ignore_errors, so we delete this line @@ -549,7 +556,7 @@ void StringValueResult::InvalidState(StringValueResult &result) { LinePosition error_position {result.iterator.pos.buffer_idx, result.last_position, result.buffer_size}; result.HandleUnicodeError(result.cur_col_id, error_position); } - result.current_errors.push_back({CSVErrorType::INVALID_UNICODE, + result.current_errors.push_back({CSVErrorType::UNTERMINATED_QUOTES, result.cur_col_id, {result.iterator.pos.buffer_idx, result.last_position, result.buffer_size}}); } @@ -724,12 +731,11 @@ void StringValueScanner::Flush(DataChunk &insert_chunk) { bool first_nl; auto borked_line = result.line_positions_per_row[line_error].ReconstructCurrentLine(first_nl, result.buffer_handles); - // TODO: We can't really nicely get the position where this error happened, this should be solved by - // TODO: adding more types to implicit casting instead of relying on this flush. auto csv_error = CSVError::CastError( state_machine->options, csv_file_scan->names[col_idx], error_message, col_idx, borked_line, lines_per_batch, - result.line_positions_per_row[line_error].begin.GetGlobalPosition(result.result_size, first_nl)); + result.line_positions_per_row[line_error].begin.GetGlobalPosition(result.result_size, first_nl), + -1); error_handler->Error(csv_error); } borked_lines.insert(line_error++); @@ -748,13 +754,11 @@ void StringValueScanner::Flush(DataChunk &insert_chunk) { bool first_nl; auto borked_line = result.line_positions_per_row[line_error].ReconstructCurrentLine( first_nl, result.buffer_handles); - // TODO: We can't really nicely get the position where this error happened, this should be solved by - // TODO: adding more types to implicit casting instead of relying on this flush. - auto csv_error = - CSVError::CastError(state_machine->options, csv_file_scan->names[col_idx], error_message, - col_idx, borked_line, lines_per_batch, - result.line_positions_per_row[line_error].begin.GetGlobalPosition( - result.result_size, first_nl)); + auto csv_error = CSVError::CastError( + state_machine->options, csv_file_scan->names[col_idx], error_message, col_idx, borked_line, + lines_per_batch, + result.line_positions_per_row[line_error].begin.GetGlobalPosition(result.result_size, first_nl), + -1); error_handler->Error(csv_error); } @@ -1249,6 +1253,7 @@ void StringValueScanner::FinalizeChunkProcess() { } else { result.HandleError(); } + iterator.done = FinishedFile(); } else { // 2) If a boundary is not set // We read until the chunk is complete, or we have nothing else to read. diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index ae2fddf9df0b..3f819bbf170c 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -268,11 +268,19 @@ void CSVGlobalState::FillRejectsTable() { errors_appender.Append(file_idx); // 3. Row Line errors_appender.Append(row_line); - // 4. Byte Position where error occurred - errors_appender.Append(error.byte_position); - // 5. Column Index + // 4. Byte Position of the row error + errors_appender.Append(error.row_byte_position); + // 5. Byte Position where error occurred + if (error.byte_position == -1) { + // This means this error comes from a flush, and we don't support this yet, so we give it + // a null + errors_appender.Append(Value()); + } else { + errors_appender.Append(error.byte_position); + } + // 6. Column Index errors_appender.Append(col_idx + 1); - // 6. Column Name (If Applicable) + // 7. Column Name (If Applicable) switch (error.type) { case CSVErrorType::TOO_MANY_COLUMNS: errors_appender.Append(Value()); @@ -284,11 +292,11 @@ void CSVGlobalState::FillRejectsTable() { default: errors_appender.Append(string_t(bind_data.return_names[col_idx])); } - // 7. Error Type + // 8. Error Type errors_appender.Append(string_t(CSVErrorTypeToEnum(error.type))); - // 8. Original CSV Line + // 9. Original CSV Line errors_appender.Append(string_t(error.csv_row)); - // 9. Full Error Message + // 10. Full Error Message errors_appender.Append(string_t(error.error_message)); errors_appender.EndRow(); } diff --git a/src/execution/operator/csv_scanner/util/csv_error.cpp b/src/execution/operator/csv_scanner/util/csv_error.cpp index 7a8349288c07..1f93d945fd91 100644 --- a/src/execution/operator/csv_scanner/util/csv_error.cpp +++ b/src/execution/operator/csv_scanner/util/csv_error.cpp @@ -82,9 +82,10 @@ CSVError::CSVError(string error_message_p, CSVErrorType type_p, LinesPerBoundary } CSVError::CSVError(string error_message_p, CSVErrorType type_p, idx_t column_idx_p, string csv_row_p, - LinesPerBoundary error_info_p, idx_t byte_position_p, const CSVReaderOptions &reader_options) + LinesPerBoundary error_info_p, idx_t row_byte_position, int64_t byte_position_p, + const CSVReaderOptions &reader_options) : error_message(std::move(error_message_p)), type(type_p), column_idx(column_idx_p), csv_row(std::move(csv_row_p)), - error_info(error_info_p), byte_position(byte_position_p) { + error_info(error_info_p), row_byte_position(row_byte_position), byte_position(byte_position_p) { // What were the options std::ostringstream error; error << error_message << std::endl; @@ -114,13 +115,15 @@ CSVError CSVError::ColumnTypesError(case_insensitive_map_t sql_types_per_ } CSVError CSVError::CastError(const CSVReaderOptions &options, string &column_name, string &cast_error, idx_t column_idx, - string &csv_row, LinesPerBoundary error_info, idx_t byte_position) { + string &csv_row, LinesPerBoundary error_info, idx_t row_byte_position, + int64_t byte_position) { std::ostringstream error; // Which column error << "Error when converting column \"" << column_name << "\". "; // What was the cast error error << cast_error; - return CSVError(error.str(), CSVErrorType::CAST_ERROR, column_idx, csv_row, error_info, byte_position, options); + return CSVError(error.str(), CSVErrorType::CAST_ERROR, column_idx, csv_row, error_info, row_byte_position, + byte_position, options); } CSVError CSVError::LineSizeError(const CSVReaderOptions &options, idx_t actual_size, LinesPerBoundary error_info, @@ -128,7 +131,8 @@ CSVError CSVError::LineSizeError(const CSVReaderOptions &options, idx_t actual_s std::ostringstream error; error << "Maximum line size of " << options.maximum_line_size << " bytes exceeded. "; error << "Actual Size:" << actual_size << " bytes."; - return CSVError(error.str(), CSVErrorType::MAXIMUM_LINE_SIZE, 0, csv_row, error_info, byte_position, options); + return CSVError(error.str(), CSVErrorType::MAXIMUM_LINE_SIZE, 0, csv_row, error_info, byte_position, byte_position, + options); } CSVError CSVError::SniffingError(string &file_path) { @@ -150,34 +154,36 @@ CSVError CSVError::NullPaddingFail(const CSVReaderOptions &options, LinesPerBoun } CSVError CSVError::UnterminatedQuotesError(const CSVReaderOptions &options, idx_t current_column, - LinesPerBoundary error_info, string &csv_row, idx_t byte_position) { + LinesPerBoundary error_info, string &csv_row, idx_t row_byte_position, + int64_t byte_position) { std::ostringstream error; error << "Value with unterminated quote found."; - return CSVError(error.str(), CSVErrorType::UNTERMINATED_QUOTES, current_column, csv_row, error_info, byte_position, - options); + return CSVError(error.str(), CSVErrorType::UNTERMINATED_QUOTES, current_column, csv_row, error_info, + row_byte_position, byte_position, options); } CSVError CSVError::IncorrectColumnAmountError(const CSVReaderOptions &options, idx_t actual_columns, - LinesPerBoundary error_info, string &csv_row, idx_t byte_position) { + LinesPerBoundary error_info, string &csv_row, idx_t row_byte_position, + int64_t byte_position) { std::ostringstream error; // How many columns were expected and how many were found error << "Expected Number of Columns: " << options.dialect_options.num_cols << " Found: " << actual_columns + 1; if (actual_columns >= options.dialect_options.num_cols) { - return CSVError(error.str(), CSVErrorType::TOO_MANY_COLUMNS, actual_columns, csv_row, error_info, byte_position, - options); + return CSVError(error.str(), CSVErrorType::TOO_MANY_COLUMNS, actual_columns, csv_row, error_info, + row_byte_position, byte_position, options); } else { - return CSVError(error.str(), CSVErrorType::TOO_FEW_COLUMNS, actual_columns, csv_row, error_info, byte_position, - options); + return CSVError(error.str(), CSVErrorType::TOO_FEW_COLUMNS, actual_columns, csv_row, error_info, + row_byte_position, byte_position, options); } } CSVError CSVError::InvalidUTF8(const CSVReaderOptions &options, idx_t current_column, LinesPerBoundary error_info, - string &csv_row, idx_t byte_position) { + string &csv_row, idx_t row_byte_position, int64_t byte_position) { std::ostringstream error; // How many columns were expected and how many were found error << "Invalid unicode (byte sequence mismatch) detected."; - return CSVError(error.str(), CSVErrorType::INVALID_UNICODE, current_column, csv_row, error_info, byte_position, - options); + return CSVError(error.str(), CSVErrorType::INVALID_UNICODE, current_column, csv_row, error_info, row_byte_position, + byte_position, options); } bool CSVErrorHandler::PrintLineNumber(CSVError &error) { diff --git a/src/execution/operator/persistent/csv_rejects_table.cpp b/src/execution/operator/persistent/csv_rejects_table.cpp index 429d385553e2..31f63d0279b8 100644 --- a/src/execution/operator/persistent/csv_rejects_table.cpp +++ b/src/execution/operator/persistent/csv_rejects_table.cpp @@ -104,17 +104,19 @@ void CSVRejectsTable::InitializeTable(ClientContext &context, const ReadCSVData info->columns.AddColumn(ColumnDefinition("file_id", LogicalType::UBIGINT)); // 2. Row Line info->columns.AddColumn(ColumnDefinition("line", LogicalType::UBIGINT)); - // 3. Byte Position where error occurred + // 3. Byte Position of the start of the line + info->columns.AddColumn(ColumnDefinition("line_byte_position", LogicalType::UBIGINT)); + // 4. Byte Position where error occurred info->columns.AddColumn(ColumnDefinition("byte_position", LogicalType::UBIGINT)); - // 4. Column Index (If Applicable) + // 5. Column Index (If Applicable) info->columns.AddColumn(ColumnDefinition("column_idx", LogicalType::UBIGINT)); - // 5. Column Name (If Applicable) + // 6. Column Name (If Applicable) info->columns.AddColumn(ColumnDefinition("column_name", LogicalType::VARCHAR)); - // 6. Error Type + // 7. Error Type info->columns.AddColumn(ColumnDefinition("error_type", enum_type)); - // 7. Original CSV Line + // 8. Original CSV Line info->columns.AddColumn(ColumnDefinition("csv_line", LogicalType::VARCHAR)); - // 8. Full Error Message + // 9. Full Error Message info->columns.AddColumn(ColumnDefinition("error_message", LogicalType::VARCHAR)); catalog.CreateTable(context, std::move(info)); } diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp index 98f460127d83..d6a6ce7ca657 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp @@ -52,13 +52,14 @@ class CSVError { public: CSVError() {}; CSVError(string error_message, CSVErrorType type, idx_t column_idx, string csv_row, LinesPerBoundary error_info, - idx_t byte_position, const CSVReaderOptions &reader_options); + idx_t row_byte_position, int64_t byte_position, const CSVReaderOptions &reader_options); CSVError(string error_message, CSVErrorType type, LinesPerBoundary error_info); //! Produces error messages for column name -> type mismatch. static CSVError ColumnTypesError(case_insensitive_map_t sql_types_per_column, const vector &names); //! Produces error messages for casting errors static CSVError CastError(const CSVReaderOptions &options, string &column_name, string &cast_error, - idx_t column_idx, string &csv_row, LinesPerBoundary error_info, idx_t byte_position); + idx_t column_idx, string &csv_row, LinesPerBoundary error_info, idx_t row_byte_position, + int64_t byte_position); //! Produces error for when the line size exceeds the maximum line size option static CSVError LineSizeError(const CSVReaderOptions &options, idx_t actual_size, LinesPerBoundary error_info, string &csv_row, idx_t byte_position); @@ -66,14 +67,16 @@ class CSVError { static CSVError SniffingError(string &file_path); //! Produces error messages for unterminated quoted values static CSVError UnterminatedQuotesError(const CSVReaderOptions &options, idx_t current_column, - LinesPerBoundary error_info, string &csv_row, idx_t byte_position); + LinesPerBoundary error_info, string &csv_row, idx_t row_byte_position, + int64_t byte_position); //! Produces error messages for null_padding option is set and we have quoted new values in parallel static CSVError NullPaddingFail(const CSVReaderOptions &options, LinesPerBoundary error_info); //! Produces error for incorrect (e.g., smaller and lower than the predefined) number of columns in a CSV Line static CSVError IncorrectColumnAmountError(const CSVReaderOptions &state_machine, idx_t actual_columns, - LinesPerBoundary error_info, string &csv_row, idx_t byte_position); + LinesPerBoundary error_info, string &csv_row, idx_t row_byte_position, + int64_t byte_position); static CSVError InvalidUTF8(const CSVReaderOptions &options, idx_t current_column, LinesPerBoundary error_info, - string &csv_row, idx_t byte_position); + string &csv_row, idx_t row_byte_position, int64_t byte_position); idx_t GetBoundaryIndex() { return error_info.boundary_idx; @@ -91,8 +94,10 @@ class CSVError { string csv_row; //! Line information regarding this error LinesPerBoundary error_info; - //! Global Byte Position where error occurred. - idx_t byte_position; + //! Byte position of where the row starts + idx_t row_byte_position; + //! Byte Position where error occurred. + int64_t byte_position; }; class CSVErrorHandler { From 1f4270c335e98c8106de30b5af3b062ba5143ca0 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 13 Mar 2024 16:27:48 +0100 Subject: [PATCH 079/603] More on the byte per row and per value --- data/csv/rejects/unquoted/unquoted_last_value.csv | 2 +- .../operator/csv_scanner/scanner/string_value_scanner.cpp | 4 ++++ .../operator/csv_scanner/table_function/global_csv_state.cpp | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/data/csv/rejects/unquoted/unquoted_last_value.csv b/data/csv/rejects/unquoted/unquoted_last_value.csv index 0d714083e9c8..68dec7d40d9c 100644 --- a/data/csv/rejects/unquoted/unquoted_last_value.csv +++ b/data/csv/rejects/unquoted/unquoted_last_value.csv @@ -2,4 +2,4 @@ "bla" "bla" "bla" -"bla +"bla \ No newline at end of file diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 103ff6341a65..0b300ee30d00 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -1030,6 +1030,10 @@ bool StringValueScanner::MoveToNextBuffer() { lines_read++; } else if (states.IsQuotedCurrent()) { // Unterminated quote + LinePosition current_line_start = {iterator.pos.buffer_idx, iterator.pos.buffer_pos, + result.buffer_size}; + result.current_line_position.begin = result.current_line_position.end; + result.current_line_position.end = current_line_start; result.InvalidState(result); } else { result.AddRow(result, previous_buffer_handle->actual_size); diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index 3f819bbf170c..94951168ddd2 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -269,7 +269,7 @@ void CSVGlobalState::FillRejectsTable() { // 3. Row Line errors_appender.Append(row_line); // 4. Byte Position of the row error - errors_appender.Append(error.row_byte_position); + errors_appender.Append(error.row_byte_position + 1); // 5. Byte Position where error occurred if (error.byte_position == -1) { // This means this error comes from a flush, and we don't support this yet, so we give it From ab2e9b13fb8177869916dd48112fba555bb4c8b4 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 14 Mar 2024 13:42:33 +0100 Subject: [PATCH 080/603] Change last_position to be a LinePosition --- .../scanner/string_value_scanner.cpp | 72 +++++++++---------- .../csv_scanner/string_value_scanner.hpp | 2 +- 2 files changed, 33 insertions(+), 41 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 0b300ee30d00..2b3eaf252a9f 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -28,7 +28,7 @@ StringValueResult::StringValueResult(CSVStates &states, CSVStateMachine &state_m // Buffer Information buffer_ptr = buffer_handle->Ptr(); buffer_size = buffer_handle->actual_size; - last_position = buffer_position; + last_position = {buffer_handle->buffer_idx, buffer_position, buffer_size}; requested_size = buffer_handle->requested_size; // Current Result information @@ -119,8 +119,7 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size } if (error) { // We error pointing to the current value error. - current_errors.push_back( - {CSVErrorType::TOO_MANY_COLUMNS, cur_col_id, {iterator.pos.buffer_idx, last_position, buffer_size}}); + current_errors.push_back({CSVErrorType::TOO_MANY_COLUMNS, cur_col_id, last_position}); } return; } @@ -141,9 +140,7 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size if (empty) { if (parse_types[chunk_col_id].first != LogicalTypeId::VARCHAR) { // If it is not a varchar, empty values are not accepted, we must error. - current_errors.push_back({CSVErrorType::CAST_ERROR, - cur_col_id, - {iterator.pos.buffer_idx, last_position, buffer_size}}); + current_errors.push_back({CSVErrorType::CAST_ERROR, cur_col_id, last_position}); } static_cast(vector_ptr[chunk_col_id])[number_of_rows] = string_t(); } else { @@ -220,13 +217,12 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size // We only evaluate if a string is utf8 valid, if it's actually a varchar if (parse_types[chunk_col_id].second && !Utf8Proc::IsValid(value_ptr, UnsafeNumericCast(size))) { bool force_error = !state_machine.options.ignore_errors.GetValue() && sniffing; - LinePosition error_position {iterator.pos.buffer_idx, last_position, buffer_size}; // Invalid unicode, we must error if (force_error) { - HandleUnicodeError(cur_col_id, error_position); + HandleUnicodeError(cur_col_id, last_position); } // If we got here, we are ingoring errors, hence we must ignore this line. - current_errors.push_back({CSVErrorType::INVALID_UNICODE, cur_col_id, error_position}); + current_errors.push_back({CSVErrorType::INVALID_UNICODE, cur_col_id, last_position}); break; } if (allocate) { @@ -246,8 +242,7 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size // Casting Error Message error << "Could not convert string \"" << std::string(value_ptr, size) << "\" to \'" << LogicalTypeIdToString(parse_types[cur_col_id].first) << "\'"; - current_errors.push_back( - {CSVErrorType::CAST_ERROR, cur_col_id, {iterator.pos.buffer_idx, last_position, buffer_size}}); + current_errors.push_back({CSVErrorType::CAST_ERROR, cur_col_id, last_position}); current_errors.back().error_message = error.str(); } cur_col_id++; @@ -309,7 +304,7 @@ void StringValueResult::AddQuotedValue(StringValueResult &result, const idx_t bu result.parse_chunk.data[result.chunk_col_id]); result.AddValueToVector(value.GetData(), value.GetSize()); } else { - if (buffer_pos < result.last_position + 2) { + if (buffer_pos < result.last_position.buffer_pos + 2) { // empty value auto value = string_t(); result.AddValueToVector(value.GetData(), value.GetSize()); @@ -323,15 +318,16 @@ void StringValueResult::AddQuotedValue(StringValueResult &result, const idx_t bu } void StringValueResult::AddValue(StringValueResult &result, const idx_t buffer_pos) { - if (result.last_position > buffer_pos) { + if (result.last_position.buffer_pos > buffer_pos) { return; } if (result.quoted) { StringValueResult::AddQuotedValue(result, buffer_pos); } else { - result.AddValueToVector(result.buffer_ptr + result.last_position, buffer_pos - result.last_position); + result.AddValueToVector(result.buffer_ptr + result.last_position.buffer_pos, + buffer_pos - result.last_position.buffer_pos); } - result.last_position = buffer_pos + 1; + result.last_position.buffer_pos = buffer_pos + 1; } void StringValueResult::HandleUnicodeError(idx_t col_idx, LinePosition &error_position) { @@ -504,11 +500,10 @@ bool StringValueResult::AddRowInternal() { bool first_nl; auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); - LinePosition error_position {iterator.pos.buffer_idx, last_position, buffer_size}; auto csv_error = CSVError::IncorrectColumnAmountError( state_machine.options, cur_col_id - 1, lines_per_batch, borked_line, current_line_position.begin.GetGlobalPosition(requested_size, first_nl), - error_position.GetGlobalPosition(requested_size, first_nl)); + last_position.GetGlobalPosition(requested_size, first_nl)); error_handler.Error(csv_error); // If we are here we ignore_errors, so we delete this line number_of_rows--; @@ -526,22 +521,23 @@ bool StringValueResult::AddRowInternal() { } bool StringValueResult::AddRow(StringValueResult &result, const idx_t buffer_pos) { - if (result.last_position <= buffer_pos) { + if (result.last_position.buffer_pos <= buffer_pos) { // We add the value if (result.quoted) { StringValueResult::AddQuotedValue(result, buffer_pos); } else { - result.AddValueToVector(result.buffer_ptr + result.last_position, buffer_pos - result.last_position); + result.AddValueToVector(result.buffer_ptr + result.last_position.buffer_pos, + buffer_pos - result.last_position.buffer_pos); } if (result.state_machine.dialect_options.state_machine_options.new_line == NewLineIdentifier::CARRY_ON) { if (result.states.states[1] == CSVState::RECORD_SEPARATOR) { // Even though this is marked as a carry on, this is a hippie mixie - result.last_position = buffer_pos + 1; + result.last_position.buffer_pos = buffer_pos + 1; } else { - result.last_position = buffer_pos + 2; + result.last_position.buffer_pos = buffer_pos + 2; } } else { - result.last_position = buffer_pos + 1; + result.last_position.buffer_pos = buffer_pos + 1; } } @@ -553,20 +549,17 @@ void StringValueResult::InvalidState(StringValueResult &result) { bool force_error = !result.state_machine.options.ignore_errors.GetValue() && result.sniffing; // Invalid unicode, we must error if (force_error) { - LinePosition error_position {result.iterator.pos.buffer_idx, result.last_position, result.buffer_size}; - result.HandleUnicodeError(result.cur_col_id, error_position); + result.HandleUnicodeError(result.cur_col_id, result.last_position); } - result.current_errors.push_back({CSVErrorType::UNTERMINATED_QUOTES, - result.cur_col_id, - {result.iterator.pos.buffer_idx, result.last_position, result.buffer_size}}); + result.current_errors.push_back({CSVErrorType::UNTERMINATED_QUOTES, result.cur_col_id, result.last_position}); } bool StringValueResult::EmptyLine(StringValueResult &result, const idx_t buffer_pos) { // We care about empty lines if this is a single column csv file - result.last_position = buffer_pos + 1; + result.last_position = {result.iterator.pos.buffer_idx, result.iterator.pos.buffer_pos + 1, result.buffer_size}; if (result.states.IsCarriageReturn() && result.state_machine.dialect_options.state_machine_options.new_line == NewLineIdentifier::CARRY_ON) { - result.last_position++; + result.last_position.buffer_pos++; } if (result.number_of_columns == 1) { if (result.null_str_size == 0) { @@ -786,9 +779,8 @@ void StringValueScanner::Initialize() { !state_machine->options.dialect_options.skip_rows.IsSetByUser())) { SetStart(); } - result.last_position = iterator.pos.buffer_pos; - result.current_line_position.begin = {iterator.pos.buffer_idx, iterator.pos.buffer_pos, - cur_buffer_handle->actual_size}; + result.last_position = {iterator.pos.buffer_idx, iterator.pos.buffer_pos, cur_buffer_handle->actual_size}; + result.current_line_position.begin = result.last_position; result.current_line_position.end = result.current_line_position.begin; } @@ -906,12 +898,12 @@ void StringValueScanner::ProcessOverbufferValue() { states.Initialize(); string overbuffer_string; auto previous_buffer = previous_buffer_handle->Ptr(); - if (result.last_position == previous_buffer_handle->actual_size) { - state_machine->Transition(states, previous_buffer[result.last_position - 1]); + if (result.last_position.buffer_pos == previous_buffer_handle->actual_size) { + state_machine->Transition(states, previous_buffer[result.last_position.buffer_pos - 1]); } idx_t j = 0; result.quoted = false; - for (idx_t i = result.last_position; i < previous_buffer_handle->actual_size; i++) { + for (idx_t i = result.last_position.buffer_pos; i < previous_buffer_handle->actual_size; i++) { state_machine->Transition(states, previous_buffer[i]); if (states.EmptyLine() || states.IsCurrentNewRow()) { continue; @@ -995,9 +987,9 @@ void StringValueScanner::ProcessOverbufferValue() { } if (states.IsCarriageReturn() && state_machine->dialect_options.state_machine_options.new_line == NewLineIdentifier::CARRY_ON) { - result.last_position = ++iterator.pos.buffer_pos + 1; + result.last_position = {iterator.pos.buffer_idx, ++iterator.pos.buffer_pos + 1, result.buffer_size}; } else { - result.last_position = ++iterator.pos.buffer_pos; + result.last_position = {iterator.pos.buffer_idx, ++iterator.pos.buffer_pos, result.buffer_size}; } // Be sure to reset the quoted and escaped variables result.quoted = false; @@ -1180,7 +1172,7 @@ void StringValueScanner::SetStart() { scan_finder->previous_buffer_handle->is_last_buffer) { iterator.pos.buffer_idx = scan_finder->iterator.pos.buffer_idx; iterator.pos.buffer_pos = scan_finder->iterator.pos.buffer_pos; - result.last_position = iterator.pos.buffer_pos; + result.last_position = {iterator.pos.buffer_idx, iterator.pos.buffer_pos, result.buffer_size}; iterator.done = scan_finder->iterator.done; return; } @@ -1199,7 +1191,7 @@ void StringValueScanner::SetStart() { // If things go terribly wrong, we never loop indefinetly. iterator.pos.buffer_idx = scan_finder->iterator.pos.buffer_idx; iterator.pos.buffer_pos = scan_finder->iterator.pos.buffer_pos; - result.last_position = iterator.pos.buffer_pos; + result.last_position = {iterator.pos.buffer_idx, iterator.pos.buffer_pos, result.buffer_size}; iterator.done = scan_finder->iterator.done; return; } @@ -1216,7 +1208,7 @@ void StringValueScanner::SetStart() { } iterator.pos.buffer_idx = scan_finder->result.current_line_position.begin.buffer_idx; iterator.pos.buffer_pos = scan_finder->result.current_line_position.begin.buffer_pos; - result.last_position = iterator.pos.buffer_pos; + result.last_position = {iterator.pos.buffer_idx, iterator.pos.buffer_pos, result.buffer_size}; } void StringValueScanner::FinalizeChunkProcess() { diff --git a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp index dfbe1f581bd5..58b312c75c53 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp @@ -92,7 +92,7 @@ class StringValueResult : public ScannerResult { unsafe_vector validity_mask; //! Variables to iterate over the CSV buffers - idx_t last_position; + LinePosition last_position; char *buffer_ptr; idx_t buffer_size; From b0f804b88d6590309272b7677892430eb18f3b2a Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 14 Mar 2024 14:50:09 +0100 Subject: [PATCH 081/603] more adjustments' --- .../scanner/string_value_scanner.cpp | 7 +- .../csv/rejects/csv_unquoted_rejects.test | 66 +++++++++++-------- 2 files changed, 44 insertions(+), 29 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 2b3eaf252a9f..f2339a8f1cb5 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -1249,7 +1249,12 @@ void StringValueScanner::FinalizeChunkProcess() { } else { result.HandleError(); } - iterator.done = FinishedFile(); + if (!iterator.done) { + if (iterator.pos.buffer_pos >= iterator.GetEndPos() || iterator.pos.buffer_idx > iterator.GetBufferIdx() || + FinishedFile()) { + iterator.done = true; + } + } } else { // 2) If a boundary is not set // We read until the chunk is complete, or we have nothing else to read. diff --git a/test/sql/copy/csv/rejects/csv_unquoted_rejects.test b/test/sql/copy/csv/rejects/csv_unquoted_rejects.test index 13c13b8b9fa7..1ce59c7c826e 100644 --- a/test/sql/copy/csv/rejects/csv_unquoted_rejects.test +++ b/test/sql/copy/csv/rejects/csv_unquoted_rejects.test @@ -11,8 +11,7 @@ query II SELECT * FROM read_csv( 'data/csv/rejects/unquoted/basic.csv', columns = {'a': 'VARCHAR', 'b': 'INTEGER'}, - rejects_table='csv_rejects_table', - ignore_errors=true, auto_detect=false, header = 1, quote = '"', escape = '"'); + store_rejects=true, auto_detect=false, header = 1, quote = '"', escape = '"'); ---- bla 1 bla 2 @@ -21,21 +20,23 @@ bla 1 bla 2 bla 3 -query IIIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position -FROM csv_rejects_table; +query IIIIIIII rowsort +SELECT regexp_replace(file_path, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line,line_byte_position, byte_position +FROM reject_scans inner join reject_errors on (reject_scans.scan_id = reject_errors.scan_id and reject_scans.file_id = reject_errors.file_id); ---- -data/csv/rejects/unquoted/basic.csv 5 1 "a" UNQUOTED VALUE "blaaaaaaaaaaaaaa"bla,4 28 +data/csv/rejects/unquoted/basic.csv 5 1 a UNQUOTED VALUE "blaaaaaaaaaaaaaa"bla,4 29 29 + +statement ok +DROP TABLE reject_scans; statement ok -DROP TABLE csv_rejects_table; +DROP TABLE reject_errors; query II SELECT * FROM read_csv( 'data/csv/rejects/unquoted/unquoted_new_line.csv', columns = {'a': 'VARCHAR', 'b': 'INTEGER'}, - rejects_table='csv_rejects_table', - ignore_errors=true, auto_detect=false, header = 1, quote = '"', escape = '"'); + store_rejects=true, auto_detect=false, header = 1, quote = '"', escape = '"'); ---- bla 1 bla 2 @@ -44,35 +45,40 @@ bla 1 bla 2 bla 3 -query IIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, byte_position -FROM csv_rejects_table; +query IIIIIII rowsort +SELECT regexp_replace(file_path, '\\', '/', 'g'), line, column_idx, column_name, error_type, line_byte_position,byte_position +FROM reject_scans inner join reject_errors on (reject_scans.scan_id = reject_errors.scan_id and reject_scans.file_id = reject_errors.file_id); ---- -data/csv/rejects/unquoted/unquoted_new_line.csv 5 1 "a" UNQUOTED VALUE 28 +data/csv/rejects/unquoted/unquoted_new_line.csv 5 1 a UNQUOTED VALUE 29 29 + +statement ok +DROP TABLE reject_scans; statement ok -DROP TABLE csv_rejects_table; +DROP TABLE reject_errors; query I SELECT * FROM read_csv( 'data/csv/rejects/unquoted/unquoted_last_value.csv', columns = {'a': 'VARCHAR'}, - rejects_table='csv_rejects_table', - ignore_errors=true, auto_detect=false, header = 0, quote = '"', escape = '"'); + store_rejects=true, auto_detect=false, header = 0, quote = '"', escape = '"'); ---- blaaaaaaaaaaaaaa bla bla bla -query IIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, byte_position -FROM csv_rejects_table; +query IIIIIIII rowsort +SELECT regexp_replace(file_path, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line,line_byte_position, byte_position +FROM reject_scans inner join reject_errors on (reject_scans.scan_id = reject_errors.scan_id and reject_scans.file_id = reject_errors.file_id); ---- -data/csv/rejects/unquoted/unquoted_last_value.csv 5 1 "a" UNQUOTED VALUE 31 +data/csv/rejects/unquoted/unquoted_last_value.csv 5 1 a UNQUOTED VALUE "bla 38 38 statement ok -DROP TABLE csv_rejects_table; +DROP TABLE reject_scans; + +statement ok +DROP TABLE reject_errors; loop buffer_size 35 40 @@ -80,8 +86,8 @@ query II SELECT * FROM read_csv( 'data/csv/rejects/unquoted/basic.csv', columns = {'a': 'VARCHAR', 'b': 'INTEGER'}, - rejects_table='csv_rejects_table', buffer_size=${buffer_size}, - ignore_errors=true, auto_detect=false, header = 1, quote = '"', escape = '"', buffer_size=35); + buffer_size=${buffer_size}, + store_rejects=true, auto_detect=false, header = 1, quote = '"', escape = '"'); ---- bla 1 bla 2 @@ -90,13 +96,17 @@ bla 1 bla 2 bla 3 -query IIIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position -FROM csv_rejects_table; + +query IIIIIIII rowsort +SELECT regexp_replace(file_path, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line,line_byte_position, byte_position +FROM reject_scans inner join reject_errors on (reject_scans.scan_id = reject_errors.scan_id and reject_scans.file_id = reject_errors.file_id); ---- -data/csv/rejects/unquoted/basic.csv 5 1 "a" UNQUOTED VALUE "blaaaaaaaaaaaaaa"bla,4 28 +data/csv/rejects/unquoted/basic.csv 5 1 a UNQUOTED VALUE "blaaaaaaaaaaaaaa"bla,4 29 29 + +statement ok +DROP TABLE reject_scans; statement ok -DROP TABLE csv_rejects_table; +DROP TABLE reject_errors; endloop \ No newline at end of file From d10d3c8e440e88e84eb46a3127282dc1402f4ff6 Mon Sep 17 00:00:00 2001 From: Tishj Date: Sat, 16 Mar 2024 15:46:15 +0100 Subject: [PATCH 082/603] dynamically generate the wrapper methods instead of defining them all manually --- tools/pythonpkg/duckdb/__init__.py | 223 +++-------- tools/pythonpkg/duckdb_python.cpp | 252 +++--------- .../duckdb_python/connection_wrapper.hpp | 198 ---------- .../src/pyduckdb/connection_wrapper.cpp | 371 ------------------ 4 files changed, 102 insertions(+), 942 deletions(-) delete mode 100644 tools/pythonpkg/src/include/duckdb_python/connection_wrapper.hpp delete mode 100644 tools/pythonpkg/src/pyduckdb/connection_wrapper.cpp diff --git a/tools/pythonpkg/duckdb/__init__.py b/tools/pythonpkg/duckdb/__init__.py index bef10a54bec1..3c33bd7ae4ed 100644 --- a/tools/pythonpkg/duckdb/__init__.py +++ b/tools/pythonpkg/duckdb/__init__.py @@ -3,6 +3,9 @@ # Modules import duckdb.functional as functional import duckdb.typing as typing +import inspect +import functools + _exported_symbols.extend([ "typing", "functional" @@ -38,6 +41,35 @@ "CaseExpression", ]) +# ---- Wrap the connection methods + +def is_dunder_method(method_name: str) -> bool: + if len(method_name) < 4: + return False + return method_name[:2] == '__' and method_name[:-3:-1] == '__' + +def create_connection_wrapper(name): + # Define a decorator function that forwards attribute lookup to the default connection + @functools.wraps(getattr(DuckDBPyConnection, name)) + def decorator(*args, **kwargs): + connection = duckdb.connect(':default:') + if 'connection' in kwargs: + connection = kwargs.pop('connection') + return getattr(connection, name)(*args, **kwargs) + # Set docstring for the wrapper function + decorator.__doc__ = getattr(DuckDBPyConnection, name).__doc__ + return decorator + +methods = inspect.getmembers_static( + DuckDBPyConnection, + predicate=inspect.isfunction +) +methods = [method for method in dir(DuckDBPyConnection) if not is_dunder_method(method)] +for name in methods: + wrapper_function = create_connection_wrapper(name) + globals()[name] = wrapper_function # Define the wrapper function in the module namespace + _exported_symbols.append(name) + # Enums from .duckdb import ( ANALYZE, @@ -54,19 +86,6 @@ "STANDARD" ]) -# Type-creation methods -from .duckdb import ( - struct_type, - list_type, - array_type, - decimal_type -) -_exported_symbols.extend([ - "struct_type", - "list_type", - "array_type", - "decimal_type" -]) # read-only properties from .duckdb import ( @@ -106,171 +125,29 @@ "tokenize" ]) + from .duckdb import ( - filter, - project, - aggregate, - distinct, - limit, - query_df, - order, - alias, connect, - write_csv + #project, + #aggregate, + #distinct, + #limit, + #query_df, + #order, + #alias, + #write_csv ) -_exported_symbols.extend([ - "filter", - "project", - "aggregate", - "distinct", - "limit", - "query_df", - "order", - "alias", - "connect", - "write_csv" -]) -# TODO: might be worth seeing if these methods can be replaced with a pure-python solution -# Connection methods -from .duckdb import ( - append, - array_type, - arrow, - begin, - close, - commit, - create_function, - cursor, - decimal_type, - description, - df, - dtype, - duplicate, - enum_type, - execute, - executemany, - extract_statements, - fetch_arrow_table, - fetch_df, - fetch_df_chunk, - fetch_record_batch, - fetchall, - fetchdf, - fetchmany, - fetchnumpy, - fetchone, - filesystem_is_registered, - from_arrow, - from_csv_auto, - from_df, - from_parquet, - from_query, - from_substrait, - from_substrait_json, - get_substrait, - get_substrait_json, - get_table_names, - install_extension, - interrupt, - list_filesystems, - list_type, - load_extension, - map_type, - pl, - query, - read_csv, - read_json, - read_parquet, - register, - register_filesystem, - remove_function, - rollback, - row_type, - rowcount, - sql, - sqltype, - string_type, - struct_type, - table, - table_function, - tf, - torch, - type, - union_type, - unregister, - unregister_filesystem, - values, - view -) _exported_symbols.extend([ - "append", - "array_type", - "arrow", - "begin", - "close", - "commit", - "create_function", - "cursor", - "decimal_type", - "description", - "df", - "dtype", - "duplicate", - "enum_type", - "execute", - "executemany", - "fetch_arrow_table", - "fetch_df", - "fetch_df_chunk", - "fetch_record_batch", - "fetchall", - "fetchdf", - "fetchmany", - "fetchnumpy", - "fetchone", - "filesystem_is_registered", - "from_arrow", - "from_csv_auto", - "from_df", - "from_parquet", - "from_query", - "from_substrait", - "from_substrait_json", - "get_substrait", - "get_substrait_json", - "get_table_names", - "install_extension", - "interrupt", - "list_filesystems", - "list_type", - "load_extension", - "map_type", - "pl", - "query", - "read_csv", - "read_json", - "read_parquet", - "register", - "register_filesystem", - "remove_function", - "rollback", - "row_type", - "rowcount", - "sql", - "sqltype", - "string_type", - "struct_type", - "table", - "table_function", - "tf", - "torch", - "type", - "union_type", - "unregister", - "unregister_filesystem", - "values", - "view" + "connect", + #"project", + #"aggregate", + #"distinct", + #"limit", + #"query_df", + #"order", + #"alias", + #"write_csv" ]) # Exceptions diff --git a/tools/pythonpkg/duckdb_python.cpp b/tools/pythonpkg/duckdb_python.cpp index 46a05f6996c9..69d9fe5376ef 100644 --- a/tools/pythonpkg/duckdb_python.cpp +++ b/tools/pythonpkg/duckdb_python.cpp @@ -13,7 +13,6 @@ #include "duckdb_python/pybind11/exceptions.hpp" #include "duckdb_python/typing.hpp" #include "duckdb_python/functional.hpp" -#include "duckdb_python/connection_wrapper.hpp" #include "duckdb_python/pybind11/conversions/pyconnection_default.hpp" #include "duckdb/common/box_renderer.hpp" #include "duckdb/function/function.hpp" @@ -72,205 +71,58 @@ static py::list PyTokenize(const string &query) { } static void InitializeConnectionMethods(py::module_ &m) { - m.def("cursor", &PyConnectionWrapper::Cursor, "Create a duplicate of the current connection", - py::arg("connection") = py::none()) - .def("duplicate", &PyConnectionWrapper::Cursor, "Create a duplicate of the current connection", - py::arg("connection") = py::none()); - m.def("create_function", &PyConnectionWrapper::RegisterScalarUDF, - "Create a DuckDB function out of the passing in python function so it can be used in queries", - py::arg("name"), py::arg("function"), py::arg("return_type") = py::none(), py::arg("parameters") = py::none(), - py::kw_only(), py::arg("type") = PythonUDFType::NATIVE, py::arg("null_handling") = 0, - py::arg("exception_handling") = 0, py::arg("side_effects") = false, py::arg("connection") = py::none()); - - m.def("remove_function", &PyConnectionWrapper::UnregisterUDF, "Remove a previously created function", - py::arg("name"), py::arg("connection") = py::none()); - - DefineMethod({"sqltype", "dtype", "type"}, m, &PyConnectionWrapper::Type, "Create a type object from 'type_str'", - py::arg("type_str"), py::arg("connection") = py::none()); - DefineMethod({"struct_type", "row_type"}, m, &PyConnectionWrapper::StructType, - "Create a struct type object from 'fields'", py::arg("fields"), py::arg("connection") = py::none()); - m.def("union_type", &PyConnectionWrapper::UnionType, "Create a union type object from 'members'", - py::arg("members").none(false), py::arg("connection") = py::none()) - .def("string_type", &PyConnectionWrapper::StringType, "Create a string type with an optional collation", - py::arg("collation") = string(), py::arg("connection") = py::none()) - .def("enum_type", &PyConnectionWrapper::EnumType, - "Create an enum type of underlying 'type', consisting of the list of 'values'", py::arg("name"), - py::arg("type"), py::arg("values"), py::arg("connection") = py::none()) - .def("decimal_type", &PyConnectionWrapper::DecimalType, "Create a decimal type with 'width' and 'scale'", - py::arg("width"), py::arg("scale"), py::arg("connection") = py::none()); - m.def("array_type", &PyConnectionWrapper::ArrayType, "Create an array type object of 'type'", - py::arg("type").none(false), py::arg("size").none(false), py::arg("connection") = py::none()); - m.def("list_type", &PyConnectionWrapper::ListType, "Create a list type object of 'type'", - py::arg("type").none(false), py::arg("connection") = py::none()); - m.def("map_type", &PyConnectionWrapper::MapType, "Create a map type object from 'key_type' and 'value_type'", - py::arg("key").none(false), py::arg("value").none(false), py::arg("connection") = py::none()) - .def("execute", &PyConnectionWrapper::Execute, - "Execute the given SQL query, optionally using prepared statements with parameters set", py::arg("query"), - py::arg("parameters") = py::none(), py::arg("multiple_parameter_sets") = false, - py::arg("connection") = py::none()) - .def("executemany", &PyConnectionWrapper::ExecuteMany, - "Execute the given prepared statement multiple times using the list of parameter sets in parameters", - py::arg("query"), py::arg("parameters") = py::none(), py::arg("connection") = py::none()) - .def("close", &PyConnectionWrapper::Close, "Close the connection", py::arg("connection") = py::none()) - .def("interrupt", &PyConnectionWrapper::Interrupt, "Interrupt pending operations", - py::arg("connection") = py::none()) - .def("fetchone", &PyConnectionWrapper::FetchOne, "Fetch a single row from a result following execute", - py::arg("connection") = py::none()) - .def("fetchmany", &PyConnectionWrapper::FetchMany, "Fetch the next set of rows from a result following execute", - py::arg("size") = 1, py::arg("connection") = py::none()) - .def("fetchall", &PyConnectionWrapper::FetchAll, "Fetch all rows from a result following execute", - py::arg("connection") = py::none()) - .def("fetchnumpy", &PyConnectionWrapper::FetchNumpy, "Fetch a result as list of NumPy arrays following execute", - py::arg("connection") = py::none()) - .def("fetchdf", &PyConnectionWrapper::FetchDF, "Fetch a result as DataFrame following execute()", py::kw_only(), - py::arg("date_as_object") = false, py::arg("connection") = py::none()) - .def("fetch_df", &PyConnectionWrapper::FetchDF, "Fetch a result as DataFrame following execute()", - py::kw_only(), py::arg("date_as_object") = false, py::arg("connection") = py::none()) - .def("fetch_df_chunk", &PyConnectionWrapper::FetchDFChunk, - "Fetch a chunk of the result as DataFrame following execute()", py::arg("vectors_per_chunk") = 1, - py::kw_only(), py::arg("date_as_object") = false, py::arg("connection") = py::none()) - .def("df", &PyConnectionWrapper::FetchDF, "Fetch a result as DataFrame following execute()", py::kw_only(), - py::arg("date_as_object") = false, py::arg("connection") = py::none()) - .def("fetch_arrow_table", &PyConnectionWrapper::FetchArrow, "Fetch a result as Arrow table following execute()", - py::arg("rows_per_batch") = 1000000, py::arg("connection") = py::none()) - .def("torch", &PyConnectionWrapper::FetchPyTorch, - "Fetch a result as dict of PyTorch Tensors following execute()", py::arg("connection") = py::none()) - .def("tf", &PyConnectionWrapper::FetchTF, "Fetch a result as dict of TensorFlow Tensors following execute()", - py::arg("connection") = py::none()) - .def("fetch_record_batch", &PyConnectionWrapper::FetchRecordBatchReader, - "Fetch an Arrow RecordBatchReader following execute()", py::arg("rows_per_batch") = 1000000, - py::arg("connection") = py::none()) - .def("arrow", &PyConnectionWrapper::FetchArrow, "Fetch a result as Arrow table following execute()", - py::arg("rows_per_batch") = 1000000, py::arg("connection") = py::none()) - .def("pl", &PyConnectionWrapper::FetchPolars, "Fetch a result as Polars DataFrame following execute()", - py::arg("rows_per_batch") = 1000000, py::arg("connection") = py::none()) - .def("begin", &PyConnectionWrapper::Begin, "Start a new transaction", py::arg("connection") = py::none()) - .def("commit", &PyConnectionWrapper::Commit, "Commit changes performed within a transaction", - py::arg("connection") = py::none()) - .def("rollback", &PyConnectionWrapper::Rollback, "Roll back changes performed within a transaction", - py::arg("connection") = py::none()) - .def("read_json", &PyConnectionWrapper::ReadJSON, "Create a relation object from the JSON file in 'name'", - py::arg("name"), py::arg("connection") = py::none(), py::arg("columns") = py::none(), - py::arg("sample_size") = py::none(), py::arg("maximum_depth") = py::none(), - py::arg("records") = py::none(), py::arg("format") = py::none()); - - m.def("values", &PyConnectionWrapper::Values, "Create a relation object from the passed values", py::arg("values"), - py::arg("connection") = py::none()); - m.def("from_substrait", &PyConnectionWrapper::FromSubstrait, "Creates a query object from the substrait plan", - py::arg("proto"), py::arg("connection") = py::none()); - m.def("get_substrait", &PyConnectionWrapper::GetSubstrait, "Serialize a query object to protobuf", py::arg("query"), - py::arg("connection") = py::none(), py::kw_only(), py::arg("enable_optimizer") = true); - m.def("get_substrait_json", &PyConnectionWrapper::GetSubstraitJSON, "Serialize a query object to protobuf", - py::arg("query"), py::arg("connection") = py::none(), py::kw_only(), py::arg("enable_optimizer") = true); - m.def("from_substrait_json", &PyConnectionWrapper::FromSubstraitJSON, "Serialize a query object to protobuf", - py::arg("json"), py::arg("connection") = py::none()); - m.def("df", &PyConnectionWrapper::FromDF, "Create a relation object from the DataFrame df", py::arg("df"), - py::arg("connection") = py::none()); - m.def("from_df", &PyConnectionWrapper::FromDF, "Create a relation object from the DataFrame df", py::arg("df"), - py::arg("connection") = py::none()); - m.def("from_arrow", &PyConnectionWrapper::FromArrow, "Create a relation object from an Arrow object", - py::arg("arrow_object"), py::arg("connection") = py::none()); - m.def("arrow", &PyConnectionWrapper::FromArrow, "Create a relation object from an Arrow object", - py::arg("arrow_object"), py::arg("connection") = py::none()); - m.def("filter", &PyConnectionWrapper::FilterDf, "Filter the DataFrame df by the filter in filter_expr", - py::arg("df"), py::arg("filter_expr"), py::arg("connection") = py::none()); - m.def("project", &PyConnectionWrapper::ProjectDf, "Project the DataFrame df by the projection in project_expr", - py::arg("df"), py::arg("project_expr"), py::arg("connection") = py::none()); - m.def("alias", &PyConnectionWrapper::AliasDF, "Create a relation from DataFrame df with the passed alias", - py::arg("df"), py::arg("alias"), py::arg("connection") = py::none()); - m.def("order", &PyConnectionWrapper::OrderDf, "Reorder the DataFrame df by order_expr", py::arg("df"), - py::arg("order_expr"), py::arg("connection") = py::none()); - m.def("aggregate", &PyConnectionWrapper::AggregateDF, - "Compute the aggregate aggr_expr by the optional groups group_expr on DataFrame df", py::arg("df"), - py::arg("aggr_expr"), py::arg("group_expr") = "", py::arg("connection") = py::none()); - m.def("distinct", &PyConnectionWrapper::DistinctDF, "Compute the distinct rows from DataFrame df ", py::arg("df"), - py::arg("connection") = py::none()); - m.def("limit", &PyConnectionWrapper::LimitDF, "Retrieve the first n rows from the DataFrame df", py::arg("df"), - py::arg("n"), py::arg("connection") = py::none()); - - m.def("query_df", &PyConnectionWrapper::QueryDF, - "Run the given SQL query in sql_query on the view named virtual_table_name that contains the content of " - "DataFrame df", - py::arg("df"), py::arg("virtual_table_name"), py::arg("sql_query"), py::arg("connection") = py::none()); - - m.def("write_csv", &PyConnectionWrapper::WriteCsvDF, "Write the DataFrame df to a CSV file in file_name", - py::arg("df"), py::arg("file_name"), py::arg("connection") = py::none()); - - DefineMethod( - {"read_csv", "from_csv_auto"}, m, &PyConnectionWrapper::ReadCSV, - "Create a relation object from the CSV file in 'name'", py::arg("name"), py::arg("connection") = py::none(), - py::arg("header") = py::none(), py::arg("compression") = py::none(), py::arg("sep") = py::none(), - py::arg("delimiter") = py::none(), py::arg("dtype") = py::none(), py::arg("na_values") = py::none(), - py::arg("skiprows") = py::none(), py::arg("quotechar") = py::none(), py::arg("escapechar") = py::none(), - py::arg("encoding") = py::none(), py::arg("parallel") = py::none(), py::arg("date_format") = py::none(), - py::arg("timestamp_format") = py::none(), py::arg("sample_size") = py::none(), - py::arg("all_varchar") = py::none(), py::arg("normalize_names") = py::none(), py::arg("filename") = py::none(), - py::arg("null_padding") = py::none(), py::arg("names") = py::none()); - - m.def("append", &PyConnectionWrapper::Append, "Append the passed DataFrame to the named table", - py::arg("table_name"), py::arg("df"), py::kw_only(), py::arg("by_name") = false, - py::arg("connection") = py::none()) - .def("register", &PyConnectionWrapper::RegisterPythonObject, - "Register the passed Python Object value for querying with a view", py::arg("view_name"), - py::arg("python_object"), py::arg("connection") = py::none()) - .def("unregister", &PyConnectionWrapper::UnregisterPythonObject, "Unregister the view name", - py::arg("view_name"), py::arg("connection") = py::none()) - .def("table", &PyConnectionWrapper::Table, "Create a relation object for the name'd table", - py::arg("table_name"), py::arg("connection") = py::none()) - .def("view", &PyConnectionWrapper::View, "Create a relation object for the name'd view", py::arg("view_name"), - py::arg("connection") = py::none()) - .def("values", &PyConnectionWrapper::Values, "Create a relation object from the passed values", - py::arg("values"), py::arg("connection") = py::none()) - .def("table_function", &PyConnectionWrapper::TableFunction, - "Create a relation object from the name'd table function with given parameters", py::arg("name"), - py::arg("parameters") = py::none(), py::arg("connection") = py::none()) - .def("extract_statements", &PyConnectionWrapper::ExtractStatements, - "Parse the query string and extract the Statement object(s) produced", py::arg("query"), - py::arg("connection") = py::none()); - - DefineMethod({"sql", "query", "from_query"}, m, &PyConnectionWrapper::RunQuery, - "Run a SQL query. If it is a SELECT statement, create a relation object from the given SQL query, " - "otherwise run the query as-is.", - py::arg("query"), py::arg("alias") = "", py::arg("connection") = py::none()); - - DefineMethod({"from_parquet", "read_parquet"}, m, &PyConnectionWrapper::FromParquet, - "Create a relation object from the Parquet files in file_glob", py::arg("file_glob"), - py::arg("binary_as_string") = false, py::kw_only(), py::arg("file_row_number") = false, - py::arg("filename") = false, py::arg("hive_partitioning") = false, py::arg("union_by_name") = false, - py::arg("compression") = py::none(), py::arg("connection") = py::none()); - - DefineMethod({"from_parquet", "read_parquet"}, m, &PyConnectionWrapper::FromParquets, - "Create a relation object from the Parquet files in file_globs", py::arg("file_globs"), - py::arg("binary_as_string") = false, py::kw_only(), py::arg("file_row_number") = false, - py::arg("filename") = false, py::arg("hive_partitioning") = false, py::arg("union_by_name") = false, - py::arg("compression") = py::none(), py::arg("connection") = py::none()); - - m.def("from_substrait", &PyConnectionWrapper::FromSubstrait, "Create a query object from protobuf plan", - py::arg("proto"), py::arg("connection") = py::none()) - .def("get_substrait", &PyConnectionWrapper::GetSubstrait, "Serialize a query to protobuf", py::arg("query"), - py::arg("connection") = py::none(), py::kw_only(), py::arg("enable_optimizer") = true) - .def("get_substrait_json", &PyConnectionWrapper::GetSubstraitJSON, - "Serialize a query to protobuf on the JSON format", py::arg("query"), py::arg("connection") = py::none(), - py::kw_only(), py::arg("enable_optimizer") = true) - .def("get_table_names", &PyConnectionWrapper::GetTableNames, "Extract the required table names from a query", - py::arg("query"), py::arg("connection") = py::none()) - .def("description", &PyConnectionWrapper::GetDescription, "Get result set attributes, mainly column names", - py::arg("connection") = py::none()) - .def("rowcount", &PyConnectionWrapper::GetRowcount, "Get result set row count", - py::arg("connection") = py::none()) - .def("install_extension", &PyConnectionWrapper::InstallExtension, "Install an extension by name", - py::arg("extension"), py::kw_only(), py::arg("force_install") = false, py::arg("connection") = py::none()) - .def("load_extension", &PyConnectionWrapper::LoadExtension, "Load an installed extension", py::arg("extension"), - py::arg("connection") = py::none()) - .def("register_filesystem", &PyConnectionWrapper::RegisterFilesystem, "Register a fsspec compliant filesystem", - py::arg("filesystem"), py::arg("connection") = py::none()) - .def("unregister_filesystem", &PyConnectionWrapper::UnregisterFilesystem, "Unregister a filesystem", - py::arg("name"), py::arg("connection") = py::none()) - .def("list_filesystems", &PyConnectionWrapper::ListFilesystems, - "List registered filesystems, including builtin ones", py::arg("connection") = py::none()) - .def("filesystem_is_registered", &PyConnectionWrapper::FileSystemIsRegistered, - "Check if a filesystem with the provided name is currently registered", py::arg("name"), - py::arg("connection") = py::none()); + m.def("project", + [](const PandasDataFrame &df, const py::object &expr, + shared_ptr conn) -> unique_ptr { + // FIXME: if we want to support passing in DuckDBPyExpressions here + // we could also accept 'expr' as a List[DuckDBPyExpression], without changing the signature + if (!py::isinstance(expr)) { + throw InvalidInputException("Please provide 'expr' as a string"); + } + return conn->FromDF(df)->Project(expr); + }); + + // unique_ptr PyConnectionWrapper::DistinctDF(const PandasDataFrame &df, + // shared_ptr conn) { + // return conn->FromDF(df)->Distinct(); + //} + + // void PyConnectionWrapper::WriteCsvDF(const PandasDataFrame &df, const string &file, + // shared_ptr conn) { + // return conn->FromDF(df)->ToCSV(file); + //} + + // unique_ptr PyConnectionWrapper::QueryDF(const PandasDataFrame &df, const string &view_name, + // const string &sql_query, + // shared_ptr conn) { + // return conn->FromDF(df)->Query(view_name, sql_query); + //} + + // unique_ptr PyConnectionWrapper::AggregateDF(const PandasDataFrame &df, const string &expr, + // const string &groups, + // shared_ptr conn) { + // return conn->FromDF(df)->Aggregate(expr, groups); + //} + + // unique_ptr PyConnectionWrapper::AliasDF(const PandasDataFrame &df, const string &expr, + // shared_ptr conn) { + // return conn->FromDF(df)->SetAlias(expr); + //} + + // unique_ptr PyConnectionWrapper::FilterDf(const PandasDataFrame &df, const string &expr, + // shared_ptr conn) { + // return conn->FromDF(df)->FilterFromExpression(expr); + //} + + // unique_ptr PyConnectionWrapper::LimitDF(const PandasDataFrame &df, int64_t n, + // shared_ptr conn) { + // return conn->FromDF(df)->Limit(n); + //} + + // unique_ptr PyConnectionWrapper::OrderDf(const PandasDataFrame &df, const string &expr, + // shared_ptr conn) { + // return conn->FromDF(df)->Order(expr); + //} } static void RegisterStatementType(py::handle &m) { diff --git a/tools/pythonpkg/src/include/duckdb_python/connection_wrapper.hpp b/tools/pythonpkg/src/include/duckdb_python/connection_wrapper.hpp deleted file mode 100644 index f40476fdea3e..000000000000 --- a/tools/pythonpkg/src/include/duckdb_python/connection_wrapper.hpp +++ /dev/null @@ -1,198 +0,0 @@ -#pragma once - -#include "duckdb_python/pyconnection/pyconnection.hpp" -#include "duckdb_python/pyrelation.hpp" -#include "duckdb_python/python_objects.hpp" - -namespace duckdb { - -class PyConnectionWrapper { -public: - PyConnectionWrapper() = delete; - -public: - static shared_ptr ExecuteMany(const py::object &query, py::object params = py::list(), - shared_ptr conn = nullptr); - - static unique_ptr DistinctDF(const PandasDataFrame &df, - shared_ptr conn = nullptr); - - static unique_ptr QueryDF(const PandasDataFrame &df, const string &view_name, - const string &sql_query, shared_ptr conn = nullptr); - - static void WriteCsvDF(const PandasDataFrame &df, const string &file, - shared_ptr conn = nullptr); - - static unique_ptr AggregateDF(const PandasDataFrame &df, const string &expr, - const string &groups = "", - shared_ptr conn = nullptr); - - static shared_ptr Execute(const py::object &query, py::object params = py::list(), - bool many = false, shared_ptr conn = nullptr); - - static shared_ptr - RegisterScalarUDF(const string &name, const py::function &udf, const py::object &arguments = py::none(), - const shared_ptr &return_type = nullptr, PythonUDFType type = PythonUDFType::NATIVE, - FunctionNullHandling null_handling = FunctionNullHandling::DEFAULT_NULL_HANDLING, - PythonExceptionHandling exception_handling = PythonExceptionHandling::FORWARD_ERROR, - bool side_effects = false, shared_ptr conn = nullptr); - - static shared_ptr UnregisterUDF(const string &name, - shared_ptr conn = nullptr); - - static py::list ExtractStatements(const string &query, shared_ptr conn = nullptr); - - static shared_ptr ArrayType(const shared_ptr &type, idx_t size, - shared_ptr conn = nullptr); - static shared_ptr ListType(const shared_ptr &type, - shared_ptr conn = nullptr); - static shared_ptr MapType(const shared_ptr &key, const shared_ptr &value, - shared_ptr conn = nullptr); - static shared_ptr StructType(const py::object &fields, - const shared_ptr conn = nullptr); - static shared_ptr UnionType(const py::object &members, shared_ptr conn = nullptr); - static shared_ptr EnumType(const string &name, const shared_ptr &type, - const py::list &values_p, shared_ptr conn = nullptr); - static shared_ptr DecimalType(int width, int scale, shared_ptr conn = nullptr); - static shared_ptr StringType(const string &collation = string(), - shared_ptr conn = nullptr); - static shared_ptr Type(const string &type_str, shared_ptr conn = nullptr); - - static shared_ptr Append(const string &name, PandasDataFrame value, bool by_name, - shared_ptr conn = nullptr); - - static shared_ptr RegisterPythonObject(const string &name, py::object python_object, - shared_ptr conn = nullptr); - - static void InstallExtension(const string &extension, bool force_install = false, - shared_ptr conn = nullptr); - - static void LoadExtension(const string &extension, shared_ptr conn = nullptr); - - static unique_ptr RunQuery(const py::object &query, const string &alias = "query_relation", - shared_ptr conn = nullptr); - - static unique_ptr Table(const string &tname, shared_ptr conn = nullptr); - - static unique_ptr Values(py::object params = py::none(), - shared_ptr conn = nullptr); - - static unique_ptr View(const string &vname, shared_ptr conn = nullptr); - - static unique_ptr TableFunction(const string &fname, py::object params = py::list(), - shared_ptr conn = nullptr); - - static unique_ptr FromParquet(const string &file_glob, bool binary_as_string, - bool file_row_number, bool filename, bool hive_partitioning, - bool union_by_name, const py::object &compression = py::none(), - shared_ptr conn = nullptr); - - static unique_ptr FromParquets(const vector &file_globs, bool binary_as_string, - bool file_row_number, bool filename, bool hive_partitioning, - bool union_by_name, const py::object &compression = py::none(), - shared_ptr conn = nullptr); - - static unique_ptr FromArrow(py::object &arrow_object, - shared_ptr conn = nullptr); - - static unique_ptr GetSubstrait(const string &query, shared_ptr conn = nullptr, - bool enable_optimizer = true); - - static unique_ptr - GetSubstraitJSON(const string &query, shared_ptr conn = nullptr, bool enable_optimizer = true); - - static unordered_set GetTableNames(const string &query, shared_ptr conn = nullptr); - - static shared_ptr UnregisterPythonObject(const string &name, - shared_ptr conn = nullptr); - - static shared_ptr Begin(shared_ptr conn = nullptr); - - static shared_ptr Commit(shared_ptr conn = nullptr); - - static shared_ptr Rollback(shared_ptr conn = nullptr); - - static void Close(shared_ptr conn = nullptr); - - static void Interrupt(shared_ptr conn = nullptr); - - static shared_ptr Cursor(shared_ptr conn = nullptr); - - static Optional GetDescription(shared_ptr conn = nullptr); - - static int GetRowcount(shared_ptr conn = nullptr); - - static Optional FetchOne(shared_ptr conn = nullptr); - - static py::list FetchMany(idx_t size, shared_ptr conn = nullptr); - - static unique_ptr ReadJSON(const string &filename, shared_ptr conn = nullptr, - const Optional &columns = py::none(), - const Optional &sample_size = py::none(), - const Optional &maximum_depth = py::none(), - const Optional &records = py::none(), - const Optional &format = py::none()); - static unique_ptr - ReadCSV(const py::object &name, shared_ptr conn, const py::object &header = py::none(), - const py::object &compression = py::none(), const py::object &sep = py::none(), - const py::object &delimiter = py::none(), const py::object &dtype = py::none(), - const py::object &na_values = py::none(), const py::object &skiprows = py::none(), - const py::object "echar = py::none(), const py::object &escapechar = py::none(), - const py::object &encoding = py::none(), const py::object ¶llel = py::none(), - const py::object &date_format = py::none(), const py::object ×tamp_format = py::none(), - const py::object &sample_size = py::none(), const py::object &all_varchar = py::none(), - const py::object &normalize_names = py::none(), const py::object &filename = py::none(), - const py::object &null_padding = py::none(), const py::object &names = py::none()); - - static py::list FetchAll(shared_ptr conn = nullptr); - - static py::dict FetchNumpy(shared_ptr conn = nullptr); - - static PandasDataFrame FetchDF(bool date_as_object, shared_ptr conn = nullptr); - - static PandasDataFrame FetchDFChunk(const idx_t vectors_per_chunk = 1, bool date_as_object = false, - shared_ptr conn = nullptr); - - static duckdb::pyarrow::Table FetchArrow(idx_t rows_per_batch, shared_ptr conn = nullptr); - - static py::dict FetchPyTorch(shared_ptr conn = nullptr); - - static py::dict FetchTF(shared_ptr conn = nullptr); - - static duckdb::pyarrow::RecordBatchReader FetchRecordBatchReader(const idx_t rows_per_batch, - shared_ptr conn = nullptr); - - static PolarsDataFrame FetchPolars(idx_t rows_per_batch, shared_ptr conn = nullptr); - - static void RegisterFilesystem(AbstractFileSystem file_system, shared_ptr conn); - static void UnregisterFilesystem(const py::str &name, shared_ptr conn); - static py::list ListFilesystems(shared_ptr conn); - static bool FileSystemIsRegistered(const string &name, shared_ptr conn); - - static unique_ptr FromDF(const PandasDataFrame &df, - shared_ptr conn = nullptr); - - static unique_ptr FromSubstrait(py::bytes &proto, shared_ptr conn = nullptr); - - static unique_ptr FromSubstraitJSON(const string &json, - shared_ptr conn = nullptr); - - static unique_ptr FromParquetDefault(const string &filename, - shared_ptr conn = nullptr); - - static unique_ptr ProjectDf(const PandasDataFrame &df, const py::object &expr, - shared_ptr conn = nullptr); - - static unique_ptr AliasDF(const PandasDataFrame &df, const string &expr, - shared_ptr conn = nullptr); - - static unique_ptr FilterDf(const PandasDataFrame &df, const string &expr, - shared_ptr conn = nullptr); - - static unique_ptr LimitDF(const PandasDataFrame &df, int64_t n, - shared_ptr conn = nullptr); - - static unique_ptr OrderDf(const PandasDataFrame &df, const string &expr, - shared_ptr conn = nullptr); -}; -} // namespace duckdb diff --git a/tools/pythonpkg/src/pyduckdb/connection_wrapper.cpp b/tools/pythonpkg/src/pyduckdb/connection_wrapper.cpp deleted file mode 100644 index c35942cacb4d..000000000000 --- a/tools/pythonpkg/src/pyduckdb/connection_wrapper.cpp +++ /dev/null @@ -1,371 +0,0 @@ -#include "duckdb_python/connection_wrapper.hpp" -#include "duckdb/common/constants.hpp" - -namespace duckdb { - -shared_ptr PyConnectionWrapper::UnionType(const py::object &members, - shared_ptr conn) { - if (!conn) { - conn = DuckDBPyConnection::DefaultConnection(); - } - return conn->UnionType(members); -} - -py::list PyConnectionWrapper::ExtractStatements(const string &query, shared_ptr conn) { - if (!conn) { - conn = DuckDBPyConnection::DefaultConnection(); - } - return conn->ExtractStatements(query); -} - -shared_ptr PyConnectionWrapper::EnumType(const string &name, const shared_ptr &type, - const py::list &values, shared_ptr conn) { - if (!conn) { - conn = DuckDBPyConnection::DefaultConnection(); - } - return conn->EnumType(name, type, values); -} - -shared_ptr PyConnectionWrapper::DecimalType(int width, int scale, shared_ptr conn) { - if (!conn) { - conn = DuckDBPyConnection::DefaultConnection(); - } - return conn->DecimalType(width, scale); -} - -shared_ptr PyConnectionWrapper::StringType(const string &collation, shared_ptr conn) { - if (!conn) { - conn = DuckDBPyConnection::DefaultConnection(); - } - return conn->StringType(collation); -} - -shared_ptr PyConnectionWrapper::ArrayType(const shared_ptr &type, idx_t size, - shared_ptr conn) { - if (!conn) { - conn = DuckDBPyConnection::DefaultConnection(); - } - return conn->ArrayType(type, size); -} - -shared_ptr PyConnectionWrapper::ListType(const shared_ptr &type, - shared_ptr conn) { - if (!conn) { - conn = DuckDBPyConnection::DefaultConnection(); - } - return conn->ListType(type); -} - -shared_ptr PyConnectionWrapper::MapType(const shared_ptr &key, - const shared_ptr &value, - shared_ptr conn) { - if (!conn) { - conn = DuckDBPyConnection::DefaultConnection(); - } - return conn->MapType(key, value); -} - -shared_ptr PyConnectionWrapper::StructType(const py::object &fields, - shared_ptr conn) { - if (!conn) { - conn = DuckDBPyConnection::DefaultConnection(); - } - return conn->StructType(fields); -} - -shared_ptr PyConnectionWrapper::Type(const string &type_str, shared_ptr conn) { - if (!conn) { - conn = DuckDBPyConnection::DefaultConnection(); - } - return conn->Type(type_str); -} - -shared_ptr PyConnectionWrapper::ExecuteMany(const py::object &query, py::object params, - shared_ptr conn) { - return conn->ExecuteMany(query, params); -} - -unique_ptr PyConnectionWrapper::DistinctDF(const PandasDataFrame &df, - shared_ptr conn) { - return conn->FromDF(df)->Distinct(); -} - -void PyConnectionWrapper::WriteCsvDF(const PandasDataFrame &df, const string &file, - shared_ptr conn) { - return conn->FromDF(df)->ToCSV(file); -} - -unique_ptr PyConnectionWrapper::QueryDF(const PandasDataFrame &df, const string &view_name, - const string &sql_query, - shared_ptr conn) { - return conn->FromDF(df)->Query(view_name, sql_query); -} - -unique_ptr PyConnectionWrapper::AggregateDF(const PandasDataFrame &df, const string &expr, - const string &groups, - shared_ptr conn) { - return conn->FromDF(df)->Aggregate(expr, groups); -} - -shared_ptr PyConnectionWrapper::Execute(const py::object &query, py::object params, bool many, - shared_ptr conn) { - return conn->Execute(query, params, many); -} - -shared_ptr PyConnectionWrapper::UnregisterUDF(const string &name, - shared_ptr conn) { - return conn->UnregisterUDF(name); -} - -shared_ptr -PyConnectionWrapper::RegisterScalarUDF(const string &name, const py::function &udf, const py::object ¶meters_p, - const shared_ptr &return_type_p, PythonUDFType type, - FunctionNullHandling null_handling, PythonExceptionHandling exception_handling, - bool side_effects, shared_ptr conn) { - return conn->RegisterScalarUDF(name, udf, parameters_p, return_type_p, type, null_handling, exception_handling, - side_effects); -} - -shared_ptr PyConnectionWrapper::Append(const string &name, PandasDataFrame value, bool by_name, - shared_ptr conn) { - return conn->Append(name, value, by_name); -} - -shared_ptr PyConnectionWrapper::RegisterPythonObject(const string &name, py::object python_object, - shared_ptr conn) { - return conn->RegisterPythonObject(name, python_object); -} - -void PyConnectionWrapper::InstallExtension(const string &extension, bool force_install, - shared_ptr conn) { - conn->InstallExtension(extension, force_install); -} - -void PyConnectionWrapper::LoadExtension(const string &extension, shared_ptr conn) { - conn->LoadExtension(extension); -} - -unique_ptr PyConnectionWrapper::Table(const string &tname, shared_ptr conn) { - return conn->Table(tname); -} - -unique_ptr PyConnectionWrapper::View(const string &vname, shared_ptr conn) { - return conn->View(vname); -} - -unique_ptr PyConnectionWrapper::TableFunction(const string &fname, py::object params, - shared_ptr conn) { - return conn->TableFunction(fname, params); -} - -unique_ptr PyConnectionWrapper::FromDF(const PandasDataFrame &value, - shared_ptr conn) { - return conn->FromDF(value); -} - -unique_ptr PyConnectionWrapper::FromParquet(const string &file_glob, bool binary_as_string, - bool file_row_number, bool filename, - bool hive_partitioning, bool union_by_name, - const py::object &compression, - shared_ptr conn) { - return conn->FromParquet(file_glob, binary_as_string, file_row_number, filename, hive_partitioning, union_by_name, - compression); -} - -unique_ptr PyConnectionWrapper::FromParquets(const vector &file_globs, bool binary_as_string, - bool file_row_number, bool filename, - bool hive_partitioning, bool union_by_name, - const py::object &compression, - shared_ptr conn) { - return conn->FromParquets(file_globs, binary_as_string, file_row_number, filename, hive_partitioning, union_by_name, - compression); -} - -unique_ptr PyConnectionWrapper::FromArrow(py::object &arrow_object, - shared_ptr conn) { - return conn->FromArrow(arrow_object); -} - -unique_ptr PyConnectionWrapper::FromSubstrait(py::bytes &proto, shared_ptr conn) { - return conn->FromSubstrait(proto); -} - -unique_ptr PyConnectionWrapper::FromSubstraitJSON(const string &json, - shared_ptr conn) { - return conn->FromSubstraitJSON(json); -} - -unique_ptr PyConnectionWrapper::GetSubstrait(const string &query, shared_ptr conn, - bool enable_optimizer) { - return conn->GetSubstrait(query, enable_optimizer); -} - -unique_ptr -PyConnectionWrapper::GetSubstraitJSON(const string &query, shared_ptr conn, bool enable_optimizer) { - return conn->GetSubstraitJSON(query, enable_optimizer); -} - -unordered_set PyConnectionWrapper::GetTableNames(const string &query, shared_ptr conn) { - return conn->GetTableNames(query); -} - -shared_ptr PyConnectionWrapper::UnregisterPythonObject(const string &name, - shared_ptr conn) { - return conn->UnregisterPythonObject(name); -} - -shared_ptr PyConnectionWrapper::Begin(shared_ptr conn) { - return conn->Begin(); -} - -shared_ptr PyConnectionWrapper::Commit(shared_ptr conn) { - return conn->Commit(); -} - -shared_ptr PyConnectionWrapper::Rollback(shared_ptr conn) { - return conn->Rollback(); -} - -void PyConnectionWrapper::Close(shared_ptr conn) { - conn->Close(); -} - -void PyConnectionWrapper::Interrupt(shared_ptr conn) { - conn->Interrupt(); -} - -shared_ptr PyConnectionWrapper::Cursor(shared_ptr conn) { - return conn->Cursor(); -} - -Optional PyConnectionWrapper::GetDescription(shared_ptr conn) { - return conn->GetDescription(); -} - -int PyConnectionWrapper::GetRowcount(shared_ptr conn) { - return conn->GetRowcount(); -} - -Optional PyConnectionWrapper::FetchOne(shared_ptr conn) { - return conn->FetchOne(); -} - -unique_ptr PyConnectionWrapper::ReadJSON(const string &filename, shared_ptr conn, - const Optional &columns, - const Optional &sample_size, - const Optional &maximum_depth, - const Optional &records, - const Optional &format) { - - return conn->ReadJSON(filename, columns, sample_size, maximum_depth, records, format); -} - -unique_ptr -PyConnectionWrapper::ReadCSV(const py::object &name, shared_ptr conn, const py::object &header, - const py::object &compression, const py::object &sep, const py::object &delimiter, - const py::object &dtype, const py::object &na_values, const py::object &skiprows, - const py::object "echar, const py::object &escapechar, const py::object &encoding, - const py::object ¶llel, const py::object &date_format, - const py::object ×tamp_format, const py::object &sample_size, - const py::object &all_varchar, const py::object &normalize_names, - const py::object &filename, const py::object &null_padding, const py::object &names) { - return conn->ReadCSV(name, header, compression, sep, delimiter, dtype, na_values, skiprows, quotechar, escapechar, - encoding, parallel, date_format, timestamp_format, sample_size, all_varchar, normalize_names, - filename, null_padding, names); -} - -py::list PyConnectionWrapper::FetchMany(idx_t size, shared_ptr conn) { - return conn->FetchMany(size); -} - -py::list PyConnectionWrapper::FetchAll(shared_ptr conn) { - return conn->FetchAll(); -} - -py::dict PyConnectionWrapper::FetchNumpy(shared_ptr conn) { - return conn->FetchNumpy(); -} - -PandasDataFrame PyConnectionWrapper::FetchDF(bool date_as_object, shared_ptr conn) { - return conn->FetchDF(date_as_object); -} - -PandasDataFrame PyConnectionWrapper::FetchDFChunk(const idx_t vectors_per_chunk, bool date_as_object, - shared_ptr conn) { - return conn->FetchDFChunk(vectors_per_chunk, date_as_object); -} - -duckdb::pyarrow::Table PyConnectionWrapper::FetchArrow(idx_t rows_per_batch, shared_ptr conn) { - return conn->FetchArrow(rows_per_batch); -} - -py::dict PyConnectionWrapper::FetchPyTorch(shared_ptr conn) { - return conn->FetchPyTorch(); -} - -py::dict PyConnectionWrapper::FetchTF(shared_ptr conn) { - return conn->FetchTF(); -} - -PolarsDataFrame PyConnectionWrapper::FetchPolars(idx_t rows_per_batch, shared_ptr conn) { - return conn->FetchPolars(rows_per_batch); -} - -duckdb::pyarrow::RecordBatchReader PyConnectionWrapper::FetchRecordBatchReader(const idx_t rows_per_batch, - shared_ptr conn) { - return conn->FetchRecordBatchReader(rows_per_batch); -} - -void PyConnectionWrapper::RegisterFilesystem(AbstractFileSystem file_system, shared_ptr conn) { - return conn->RegisterFilesystem(std::move(file_system)); -} -void PyConnectionWrapper::UnregisterFilesystem(const py::str &name, shared_ptr conn) { - return conn->UnregisterFilesystem(name); -} -py::list PyConnectionWrapper::ListFilesystems(shared_ptr conn) { - return conn->ListFilesystems(); -} -bool PyConnectionWrapper::FileSystemIsRegistered(const string &name, shared_ptr conn) { - return conn->FileSystemIsRegistered(name); -} - -unique_ptr PyConnectionWrapper::Values(py::object values, shared_ptr conn) { - return conn->Values(std::move(values)); -} - -unique_ptr PyConnectionWrapper::RunQuery(const py::object &query, const string &alias, - shared_ptr conn) { - return conn->RunQuery(query, alias); -} - -unique_ptr PyConnectionWrapper::ProjectDf(const PandasDataFrame &df, const py::object &expr, - shared_ptr conn) { - // FIXME: if we want to support passing in DuckDBPyExpressions here - // we could also accept 'expr' as a List[DuckDBPyExpression], without changing the signature - if (!py::isinstance(expr)) { - throw InvalidInputException("Please provide 'expr' as a string"); - } - return conn->FromDF(df)->Project(expr); -} - -unique_ptr PyConnectionWrapper::AliasDF(const PandasDataFrame &df, const string &expr, - shared_ptr conn) { - return conn->FromDF(df)->SetAlias(expr); -} - -unique_ptr PyConnectionWrapper::FilterDf(const PandasDataFrame &df, const string &expr, - shared_ptr conn) { - return conn->FromDF(df)->FilterFromExpression(expr); -} - -unique_ptr PyConnectionWrapper::LimitDF(const PandasDataFrame &df, int64_t n, - shared_ptr conn) { - return conn->FromDF(df)->Limit(n); -} - -unique_ptr PyConnectionWrapper::OrderDf(const PandasDataFrame &df, const string &expr, - shared_ptr conn) { - return conn->FromDF(df)->Order(expr); -} - -} // namespace duckdb From 831c9ee2bdcda4ec49a094ad9016878aafc7d4cc Mon Sep 17 00:00:00 2001 From: Tishj Date: Sat, 16 Mar 2024 20:41:28 +0100 Subject: [PATCH 083/603] all tests passing again --- tools/pythonpkg/duckdb/__init__.py | 88 ++++++++++++++----- tools/pythonpkg/duckdb_python.cpp | 82 +++++++---------- .../tests/fast/test_non_default_conn.py | 2 +- 3 files changed, 99 insertions(+), 73 deletions(-) diff --git a/tools/pythonpkg/duckdb/__init__.py b/tools/pythonpkg/duckdb/__init__.py index 3c33bd7ae4ed..d8d1ae36299e 100644 --- a/tools/pythonpkg/duckdb/__init__.py +++ b/tools/pythonpkg/duckdb/__init__.py @@ -3,7 +3,6 @@ # Modules import duckdb.functional as functional import duckdb.typing as typing -import inspect import functools _exported_symbols.extend([ @@ -15,7 +14,7 @@ from .duckdb import ( DuckDBPyRelation, DuckDBPyConnection, - Statement, + Statement, ExplainType, StatementType, ExpectedResultType, @@ -48,27 +47,76 @@ def is_dunder_method(method_name: str) -> bool: return False return method_name[:2] == '__' and method_name[:-3:-1] == '__' -def create_connection_wrapper(name): - # Define a decorator function that forwards attribute lookup to the default connection - @functools.wraps(getattr(DuckDBPyConnection, name)) - def decorator(*args, **kwargs): +# Takes the function to execute on the 'connection' +def create_wrapper(func): + def _wrapper(*args, **kwargs): connection = duckdb.connect(':default:') if 'connection' in kwargs: connection = kwargs.pop('connection') - return getattr(connection, name)(*args, **kwargs) - # Set docstring for the wrapper function - decorator.__doc__ = getattr(DuckDBPyConnection, name).__doc__ - return decorator + return func(connection, *args, **kwargs) + return _wrapper -methods = inspect.getmembers_static( - DuckDBPyConnection, - predicate=inspect.isfunction +# Takes the name of a DuckDBPyConnection function to wrap (copying signature, docs, etc) +# The 'func' is what gets executed when the function is called +def create_connection_wrapper(name, func): + # Define a decorator function that forwards attribute lookup to the default connection + return functools.wraps(getattr(DuckDBPyConnection, name))(create_wrapper(func)) + +# These are overloaded twice, we define them inside of C++ so pybind can deal with it +EXCLUDED_METHODS = [ + 'df', + 'arrow' +] +_exported_symbols.extend(EXCLUDED_METHODS) +from .duckdb import ( + df, + arrow ) -methods = [method for method in dir(DuckDBPyConnection) if not is_dunder_method(method)] -for name in methods: - wrapper_function = create_connection_wrapper(name) - globals()[name] = wrapper_function # Define the wrapper function in the module namespace - _exported_symbols.append(name) + +methods = [method for method in dir(DuckDBPyConnection) if not is_dunder_method(method) and method not in EXCLUDED_METHODS] +for method_name in methods: + def create_method_wrapper(method_name): + def call_method(conn, *args, **kwargs): + return getattr(conn, method_name)(*args, **kwargs) + return call_method + wrapper_function = create_connection_wrapper(method_name, create_method_wrapper(method_name)) + globals()[method_name] = wrapper_function # Define the wrapper function in the module namespace + _exported_symbols.append(method_name) + + +# Specialized "wrapper" methods + +SPECIAL_METHODS = [ + 'project', + 'distinct', + 'write_csv', + 'aggregate', + 'alias', + 'filter', + 'limit', + 'order', + 'query_df' +] + +for method_name in SPECIAL_METHODS: + def create_method_wrapper(name): + def _closure(name=name): + mapping = { + 'alias': 'set_alias', + 'query_df': 'query' + } + def call_method(con, df, *args, **kwargs): + if name in mapping: + mapped_name = mapping[name] + else: + mapped_name = name + return getattr(con.from_df(df), mapped_name)(*args, **kwargs) + return call_method + return _closure(name) + + wrapper_function = create_wrapper(create_method_wrapper(method_name)) + globals()[method_name] = wrapper_function # Define the wrapper function in the module namespace + _exported_symbols.append(method_name) # Enums from .duckdb import ( @@ -76,8 +124,8 @@ def decorator(*args, **kwargs): DEFAULT, RETURN_NULL, STANDARD, - COLUMNS, - ROWS + COLUMNS, + ROWS ) _exported_symbols.extend([ "ANALYZE", diff --git a/tools/pythonpkg/duckdb_python.cpp b/tools/pythonpkg/duckdb_python.cpp index 69d9fe5376ef..70e82c03970a 100644 --- a/tools/pythonpkg/duckdb_python.cpp +++ b/tools/pythonpkg/duckdb_python.cpp @@ -71,58 +71,36 @@ static py::list PyTokenize(const string &query) { } static void InitializeConnectionMethods(py::module_ &m) { - m.def("project", - [](const PandasDataFrame &df, const py::object &expr, - shared_ptr conn) -> unique_ptr { - // FIXME: if we want to support passing in DuckDBPyExpressions here - // we could also accept 'expr' as a List[DuckDBPyExpression], without changing the signature - if (!py::isinstance(expr)) { - throw InvalidInputException("Please provide 'expr' as a string"); - } - return conn->FromDF(df)->Project(expr); - }); - - // unique_ptr PyConnectionWrapper::DistinctDF(const PandasDataFrame &df, - // shared_ptr conn) { - // return conn->FromDF(df)->Distinct(); - //} - - // void PyConnectionWrapper::WriteCsvDF(const PandasDataFrame &df, const string &file, - // shared_ptr conn) { - // return conn->FromDF(df)->ToCSV(file); - //} - - // unique_ptr PyConnectionWrapper::QueryDF(const PandasDataFrame &df, const string &view_name, - // const string &sql_query, - // shared_ptr conn) { - // return conn->FromDF(df)->Query(view_name, sql_query); - //} - - // unique_ptr PyConnectionWrapper::AggregateDF(const PandasDataFrame &df, const string &expr, - // const string &groups, - // shared_ptr conn) { - // return conn->FromDF(df)->Aggregate(expr, groups); - //} - - // unique_ptr PyConnectionWrapper::AliasDF(const PandasDataFrame &df, const string &expr, - // shared_ptr conn) { - // return conn->FromDF(df)->SetAlias(expr); - //} - - // unique_ptr PyConnectionWrapper::FilterDf(const PandasDataFrame &df, const string &expr, - // shared_ptr conn) { - // return conn->FromDF(df)->FilterFromExpression(expr); - //} - - // unique_ptr PyConnectionWrapper::LimitDF(const PandasDataFrame &df, int64_t n, - // shared_ptr conn) { - // return conn->FromDF(df)->Limit(n); - //} - - // unique_ptr PyConnectionWrapper::OrderDf(const PandasDataFrame &df, const string &expr, - // shared_ptr conn) { - // return conn->FromDF(df)->Order(expr); - //} + // We define these "wrapper" methods inside of C++ because they are overloaded + // every other wrapper method is defined inside of __init__.py + m.def( + "arrow", + [](idx_t rows_per_batch, shared_ptr conn) -> duckdb::pyarrow::Table { + return conn->FetchArrow(rows_per_batch); + }, + "Fetch a result as Arrow table following execute()", py::arg("rows_per_batch") = 1000000, py::kw_only(), + py::arg("connection") = py::none()); + m.def( + "arrow", + [](py::object &arrow_object, shared_ptr conn) -> unique_ptr { + return conn->FromArrow(arrow_object); + }, + "Create a relation object from an Arrow object", py::arg("arrow_object"), py::kw_only(), + py::arg("connection") = py::none()); + m.def( + "df", + [](bool date_as_object, shared_ptr conn) -> PandasDataFrame { + return conn->FetchDF(date_as_object); + }, + "Fetch a result as DataFrame following execute()", py::kw_only(), py::arg("date_as_object") = false, + py::arg("connection") = py::none()); + m.def( + "df", + [](const PandasDataFrame &value, shared_ptr conn) -> unique_ptr { + return conn->FromDF(value); + }, + "Create a relation object from the DataFrame df", py::arg("df"), py::kw_only(), + py::arg("connection") = py::none()); } static void RegisterStatementType(py::handle &m) { diff --git a/tools/pythonpkg/tests/fast/test_non_default_conn.py b/tools/pythonpkg/tests/fast/test_non_default_conn.py index 0745f63d4e3e..bc9fa5f094e1 100644 --- a/tools/pythonpkg/tests/fast/test_non_default_conn.py +++ b/tools/pythonpkg/tests/fast/test_non_default_conn.py @@ -8,7 +8,7 @@ class TestNonDefaultConn(object): def test_values(self, duckdb_cursor): duckdb_cursor.execute("create table t (a integer)") - duckdb.values([1], duckdb_cursor).insert_into("t") + duckdb.values([1], connection=duckdb_cursor).insert_into("t") assert duckdb_cursor.execute("select count(*) from t").fetchall()[0] == (1,) def test_query(self, duckdb_cursor): From d513065397f9f429dd04b1baf128956b7e9ce30a Mon Sep 17 00:00:00 2001 From: Tishj Date: Sat, 16 Mar 2024 20:43:36 +0100 Subject: [PATCH 084/603] remove commented out code --- tools/pythonpkg/duckdb/__init__.py | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/tools/pythonpkg/duckdb/__init__.py b/tools/pythonpkg/duckdb/__init__.py index d8d1ae36299e..8f01da43bc4f 100644 --- a/tools/pythonpkg/duckdb/__init__.py +++ b/tools/pythonpkg/duckdb/__init__.py @@ -175,27 +175,11 @@ def call_method(con, df, *args, **kwargs): from .duckdb import ( - connect, - #project, - #aggregate, - #distinct, - #limit, - #query_df, - #order, - #alias, - #write_csv + connect ) _exported_symbols.extend([ - "connect", - #"project", - #"aggregate", - #"distinct", - #"limit", - #"query_df", - #"order", - #"alias", - #"write_csv" + "connect" ]) # Exceptions From 02e94ed0a5a986c2240e3b663619de0725e4d431 Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 18 Mar 2024 10:23:30 +0100 Subject: [PATCH 085/603] this is entirely redundant, because we have an implicit conversion from None todefault connection handled by pybind, but tidy will not be happy about passing by value if I don't use it, and its suggestion to make it a will break the implicit conversion rules we set up - so the *only* purpose of this change is to make tidy happy --- tools/pythonpkg/duckdb_python.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tools/pythonpkg/duckdb_python.cpp b/tools/pythonpkg/duckdb_python.cpp index 70e82c03970a..f5eb42d446ea 100644 --- a/tools/pythonpkg/duckdb_python.cpp +++ b/tools/pythonpkg/duckdb_python.cpp @@ -76,6 +76,9 @@ static void InitializeConnectionMethods(py::module_ &m) { m.def( "arrow", [](idx_t rows_per_batch, shared_ptr conn) -> duckdb::pyarrow::Table { + if (!connection) { + connection = DuckDBPyConnection::DefaultConnection(); + } return conn->FetchArrow(rows_per_batch); }, "Fetch a result as Arrow table following execute()", py::arg("rows_per_batch") = 1000000, py::kw_only(), @@ -83,6 +86,9 @@ static void InitializeConnectionMethods(py::module_ &m) { m.def( "arrow", [](py::object &arrow_object, shared_ptr conn) -> unique_ptr { + if (!connection) { + connection = DuckDBPyConnection::DefaultConnection(); + } return conn->FromArrow(arrow_object); }, "Create a relation object from an Arrow object", py::arg("arrow_object"), py::kw_only(), @@ -90,6 +96,9 @@ static void InitializeConnectionMethods(py::module_ &m) { m.def( "df", [](bool date_as_object, shared_ptr conn) -> PandasDataFrame { + if (!connection) { + connection = DuckDBPyConnection::DefaultConnection(); + } return conn->FetchDF(date_as_object); }, "Fetch a result as DataFrame following execute()", py::kw_only(), py::arg("date_as_object") = false, @@ -97,6 +106,9 @@ static void InitializeConnectionMethods(py::module_ &m) { m.def( "df", [](const PandasDataFrame &value, shared_ptr conn) -> unique_ptr { + if (!connection) { + connection = DuckDBPyConnection::DefaultConnection(); + } return conn->FromDF(value); }, "Create a relation object from the DataFrame df", py::arg("df"), py::kw_only(), From 482aa695be2cc9bc00f977794126f4d80a23b3db Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 18 Mar 2024 13:50:47 +0100 Subject: [PATCH 086/603] . --- tools/pythonpkg/duckdb_python.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/pythonpkg/duckdb_python.cpp b/tools/pythonpkg/duckdb_python.cpp index f5eb42d446ea..1289c5743833 100644 --- a/tools/pythonpkg/duckdb_python.cpp +++ b/tools/pythonpkg/duckdb_python.cpp @@ -76,8 +76,8 @@ static void InitializeConnectionMethods(py::module_ &m) { m.def( "arrow", [](idx_t rows_per_batch, shared_ptr conn) -> duckdb::pyarrow::Table { - if (!connection) { - connection = DuckDBPyConnection::DefaultConnection(); + if (!conn) { + conn = DuckDBPyConnection::DefaultConnection(); } return conn->FetchArrow(rows_per_batch); }, @@ -86,8 +86,8 @@ static void InitializeConnectionMethods(py::module_ &m) { m.def( "arrow", [](py::object &arrow_object, shared_ptr conn) -> unique_ptr { - if (!connection) { - connection = DuckDBPyConnection::DefaultConnection(); + if (!conn) { + conn = DuckDBPyConnection::DefaultConnection(); } return conn->FromArrow(arrow_object); }, @@ -96,8 +96,8 @@ static void InitializeConnectionMethods(py::module_ &m) { m.def( "df", [](bool date_as_object, shared_ptr conn) -> PandasDataFrame { - if (!connection) { - connection = DuckDBPyConnection::DefaultConnection(); + if (!conn) { + conn = DuckDBPyConnection::DefaultConnection(); } return conn->FetchDF(date_as_object); }, @@ -106,8 +106,8 @@ static void InitializeConnectionMethods(py::module_ &m) { m.def( "df", [](const PandasDataFrame &value, shared_ptr conn) -> unique_ptr { - if (!connection) { - connection = DuckDBPyConnection::DefaultConnection(); + if (!conn) { + conn = DuckDBPyConnection::DefaultConnection(); } return conn->FromDF(value); }, From a2a9982e7e619661f9fa7abc9e10fe87cb5dcc74 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 20 Mar 2024 11:26:53 +0100 Subject: [PATCH 087/603] Fix progress over multiple very large files --- .../table_function/global_csv_state.cpp | 19 ++++++++++--------- test/sql/copy/csv/test_gzipped.test | 0 2 files changed, 10 insertions(+), 9 deletions(-) create mode 100644 test/sql/copy/csv/test_gzipped.test diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index 863b937f2186..72707df66324 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -43,16 +43,16 @@ double CSVGlobalState::GetProgress(const ReadCSVData &bind_data_p) const { lock_guard parallel_lock(main_mutex); idx_t total_files = bind_data.files.size(); // get the progress WITHIN the current file - double progress; + double percentage = 0; if (file_scans.back()->file_size == 0) { - progress = 1.0; + percentage = 1.0; } else { // for compressed files, readed bytes may greater than files size. - progress = std::min(1.0, double(file_scans.back()->bytes_read) / double(file_scans.back()->file_size)); + for (auto &file : file_scans) { + percentage += + (double(1) / double(total_files)) * std::min(1.0, double(file->bytes_read) / double(file->file_size)); + } } - // now get the total percentage of files read - double percentage = double(current_boundary.GetFileIdx()) / total_files; - percentage += (double(1) / double(total_files)) * progress; return percentage * 100; } @@ -66,8 +66,9 @@ unique_ptr CSVGlobalState::Next() { if (cur_idx == 0) { current_file = file_scans.back(); } else { - current_file = make_shared(context, bind_data.files[cur_idx], bind_data.options, cur_idx, - bind_data, column_ids, file_schema); + file_scans.emplace_back(make_shared(context, bind_data.files[cur_idx], bind_data.options, + cur_idx, bind_data, column_ids, file_schema)); + current_file = file_scans.back(); } auto csv_scanner = make_uniq(scanner_idx++, current_file->buffer_manager, current_file->state_machine, @@ -98,7 +99,7 @@ unique_ptr CSVGlobalState::Next() { // If we have a next file we have to construct the file scan for that file_scans.emplace_back(make_shared(context, bind_data.files[current_file_idx], bind_data.options, current_file_idx, bind_data, column_ids, - file_schema)); + file_schema, single_threaded)); // And re-start the boundary-iterator auto buffer_size = file_scans.back()->buffer_manager->GetBuffer(0)->actual_size; current_boundary = CSVIterator(current_file_idx, 0, 0, 0, buffer_size); diff --git a/test/sql/copy/csv/test_gzipped.test b/test/sql/copy/csv/test_gzipped.test new file mode 100644 index 000000000000..e69de29bb2d1 From 3ab61710c8dbc6889c4299e61e45ed42368af097 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 20 Mar 2024 12:52:58 +0100 Subject: [PATCH 088/603] Dont store buffers if doing single threaded scan over multiple files --- .../csv_scanner/buffer_manager/csv_buffer.cpp | 20 +++++++++---------- .../buffer_manager/csv_buffer_manager.cpp | 14 +++++++++---- .../scanner/string_value_scanner.cpp | 4 ++-- .../table_function/csv_file_scanner.cpp | 9 +++++---- .../table_function/global_csv_state.cpp | 7 +++++-- src/function/table/copy_csv.cpp | 2 +- src/function/table/read_csv.cpp | 2 +- src/function/table/sniff_csv.cpp | 2 +- .../operator/csv_scanner/csv_buffer.hpp | 9 +++++---- .../csv_scanner/csv_buffer_manager.hpp | 5 ++++- .../operator/csv_scanner/csv_file_scanner.hpp | 5 +++-- src/main/relation/read_csv_relation.cpp | 2 +- 12 files changed, 48 insertions(+), 33 deletions(-) diff --git a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp index 8c29ae79fb43..6ac66783f041 100644 --- a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp +++ b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp @@ -4,9 +4,9 @@ namespace duckdb { CSVBuffer::CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle &file_handle, - idx_t &global_csv_current_position, idx_t file_number_p) + idx_t &global_csv_current_position, idx_t file_number_p, bool single_threaded) : context(context), file_number(file_number_p), can_seek(file_handle.CanSeek()) { - AllocateBuffer(buffer_size_p); + AllocateBuffer(buffer_size_p, can_seek || single_threaded); auto buffer = Ptr(); actual_buffer_size = file_handle.Read(buffer, buffer_size_p); while (actual_buffer_size < buffer_size_p && !file_handle.FinishedReading()) { @@ -18,10 +18,10 @@ CSVBuffer::CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle } CSVBuffer::CSVBuffer(CSVFileHandle &file_handle, ClientContext &context, idx_t buffer_size, - idx_t global_csv_current_position, idx_t file_number_p, idx_t buffer_idx_p) + idx_t global_csv_current_position, idx_t file_number_p, idx_t buffer_idx_p, bool single_threaded) : context(context), global_csv_start(global_csv_current_position), file_number(file_number_p), can_seek(file_handle.CanSeek()), buffer_idx(buffer_idx_p) { - AllocateBuffer(buffer_size); + AllocateBuffer(buffer_size, single_threaded || can_seek); auto buffer = handle.Ptr(); actual_buffer_size = file_handle.Read(handle.Ptr(), buffer_size); while (actual_buffer_size < buffer_size && !file_handle.FinishedReading()) { @@ -32,15 +32,16 @@ CSVBuffer::CSVBuffer(CSVFileHandle &file_handle, ClientContext &context, idx_t b } shared_ptr CSVBuffer::Next(CSVFileHandle &file_handle, idx_t buffer_size, idx_t file_number_p, - bool &has_seaked) { + bool &has_seaked, bool single_threaded) { if (has_seaked) { // This means that at some point a reload was done, and we are currently on the incorrect position in our file // handle file_handle.Seek(global_csv_start + actual_buffer_size); has_seaked = false; } - auto next_csv_buffer = make_shared(file_handle, context, buffer_size, - global_csv_start + actual_buffer_size, file_number_p, buffer_idx + 1); + auto next_csv_buffer = + make_shared(file_handle, context, buffer_size, global_csv_start + actual_buffer_size, file_number_p, + buffer_idx + 1, single_threaded); if (next_csv_buffer->GetBufferSize() == 0) { // We are done reading return nullptr; @@ -48,9 +49,8 @@ shared_ptr CSVBuffer::Next(CSVFileHandle &file_handle, idx_t buffer_s return next_csv_buffer; } -void CSVBuffer::AllocateBuffer(idx_t buffer_size) { +void CSVBuffer::AllocateBuffer(idx_t buffer_size, bool can_destroy) { auto &buffer_manager = BufferManager::GetBufferManager(context); - bool can_destroy = can_seek; handle = buffer_manager.Allocate(MemoryTag::CSV_READER, MaxValue(Storage::BLOCK_SIZE, buffer_size), can_destroy, &block); } @@ -60,7 +60,7 @@ idx_t CSVBuffer::GetBufferSize() { } void CSVBuffer::Reload(CSVFileHandle &file_handle) { - AllocateBuffer(actual_buffer_size); + AllocateBuffer(actual_buffer_size, false); file_handle.Seek(global_csv_start); file_handle.Read(handle.Ptr(), actual_buffer_size); } diff --git a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp index 2a13158b6081..568343cafad7 100644 --- a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp +++ b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp @@ -4,8 +4,9 @@ namespace duckdb { CSVBufferManager::CSVBufferManager(ClientContext &context_p, const CSVReaderOptions &options, const string &file_path_p, - const idx_t file_idx_p) - : context(context_p), file_idx(file_idx_p), file_path(file_path_p), buffer_size(CSVBuffer::CSV_BUFFER_SIZE) { + const idx_t file_idx_p, bool single_threaded_p) + : context(context_p), file_idx(file_idx_p), file_path(file_path_p), buffer_size(CSVBuffer::CSV_BUFFER_SIZE), + single_threaded(single_threaded_p) { D_ASSERT(!file_path.empty()); file_handle = ReadCSV::OpenCSV(file_path, options.compression, context); skip_rows = options.dialect_options.skip_rows.GetValue(); @@ -28,7 +29,7 @@ void CSVBufferManager::UnpinBuffer(const idx_t cache_idx) { void CSVBufferManager::Initialize() { if (cached_buffers.empty()) { cached_buffers.emplace_back( - make_shared(context, buffer_size, *file_handle, global_csv_pos, file_idx)); + make_shared(context, buffer_size, *file_handle, global_csv_pos, file_idx, single_threaded)); last_buffer = cached_buffers.front(); } } @@ -47,7 +48,8 @@ bool CSVBufferManager::ReadNextAndCacheIt() { last_buffer->last_buffer = true; return false; } - auto maybe_last_buffer = last_buffer->Next(*file_handle, cur_buffer_size, file_idx, has_seeked); + auto maybe_last_buffer = + last_buffer->Next(*file_handle, cur_buffer_size, file_idx, has_seeked, single_threaded); if (!maybe_last_buffer) { last_buffer->last_buffer = true; return false; @@ -126,4 +128,8 @@ string CSVBufferManager::GetFilePath() { return file_path; } +void CSVBufferManager::SetSingleThreaded() { + single_threaded = true; +} + } // namespace duckdb diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 9582e1c1af2f..a0376cc94947 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -530,9 +530,9 @@ unique_ptr StringValueScanner::GetCSVScanner(ClientContext & state_machine->dialect_options.num_cols = options.dialect_options.num_cols; state_machine->dialect_options.header = options.dialect_options.header; - auto buffer_manager = make_shared(context, options, options.file_path, 0); + auto buffer_manager = make_shared(context, options, options.file_path, 0, false); auto scanner = make_uniq(buffer_manager, state_machine, make_shared()); - scanner->csv_file_scan = make_shared(context, options.file_path, options); + scanner->csv_file_scan = make_shared(context, options.file_path, options, false); scanner->csv_file_scan->InitializeProjection(); return scanner; } diff --git a/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp b/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp index 0532fc678a41..8013a2da10ef 100644 --- a/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp +++ b/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp @@ -41,7 +41,7 @@ CSVFileScan::CSVFileScan(ClientContext &context, shared_ptr bu CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, const CSVReaderOptions &options_p, const idx_t file_idx_p, const ReadCSVData &bind_data, const vector &column_ids, - const vector &file_schema) + const vector &file_schema, bool single_threaded) : file_path(file_path_p), file_idx(file_idx_p), error_handler(make_shared(options_p.ignore_errors)), options(options_p) { if (file_idx < bind_data.union_readers.size()) { @@ -73,7 +73,7 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, cons } // Initialize Buffer Manager - buffer_manager = make_shared(context, options, file_path, file_idx); + buffer_manager = make_shared(context, options, file_path, file_idx, single_threaded); // Initialize On Disk and Size of file on_disk_file = buffer_manager->file_handle->OnDiskFile(); file_size = buffer_manager->file_handle->FileSize(); @@ -128,10 +128,11 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, cons InitializeFileNamesTypes(); } -CSVFileScan::CSVFileScan(ClientContext &context, const string &file_name, CSVReaderOptions &options_p) +CSVFileScan::CSVFileScan(ClientContext &context, const string &file_name, CSVReaderOptions &options_p, + bool single_threaded) : file_path(file_name), file_idx(0), error_handler(make_shared(options_p.ignore_errors)), options(options_p) { - buffer_manager = make_shared(context, options, file_path, file_idx); + buffer_manager = make_shared(context, options, file_path, file_idx, single_threaded); // Initialize On Disk and Size of file on_disk_file = buffer_manager->file_handle->OnDiskFile(); file_size = buffer_manager->file_handle->FileSize(); diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index 72707df66324..f0ec98fbbdfb 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -22,7 +22,7 @@ CSVGlobalState::CSVGlobalState(ClientContext &context_p, const shared_ptr(context, files[0], options, 0, bind_data, column_ids, file_schema)); + make_uniq(context, files[0], options, 0, bind_data, column_ids, file_schema, single_threaded)); }; //! There are situations where we only support single threaded scanning bool many_csv_files = files.size() > 1 && files.size() > system_threads * 2; @@ -65,9 +65,12 @@ unique_ptr CSVGlobalState::Next() { shared_ptr current_file; if (cur_idx == 0) { current_file = file_scans.back(); + current_file->buffer_manager->SetSingleThreaded(); } else { + lock_guard parallel_lock(main_mutex); file_scans.emplace_back(make_shared(context, bind_data.files[cur_idx], bind_data.options, - cur_idx, bind_data, column_ids, file_schema)); + cur_idx, bind_data, column_ids, file_schema, + single_threaded)); current_file = file_scans.back(); } auto csv_scanner = diff --git a/src/function/table/copy_csv.cpp b/src/function/table/copy_csv.cpp index e2f9a2403c08..67e8041f4da6 100644 --- a/src/function/table/copy_csv.cpp +++ b/src/function/table/copy_csv.cpp @@ -156,7 +156,7 @@ static unique_ptr ReadCSVBind(ClientContext &context, CopyInfo &in } if (options.auto_detect) { - auto buffer_manager = make_shared(context, options, bind_data->files[0], 0); + auto buffer_manager = make_shared(context, options, bind_data->files[0], 0, false); CSVSniffer sniffer(options, buffer_manager, CSVStateMachineCache::Get(context), {&expected_types, &expected_names}); sniffer.SniffCSV(); diff --git a/src/function/table/read_csv.cpp b/src/function/table/read_csv.cpp index 8d2e1be0d780..0963f35c0478 100644 --- a/src/function/table/read_csv.cpp +++ b/src/function/table/read_csv.cpp @@ -98,7 +98,7 @@ static unique_ptr ReadCSVBind(ClientContext &context, TableFunctio } if (options.auto_detect && !options.file_options.union_by_name) { options.file_path = result->files[0]; - result->buffer_manager = make_shared(context, options, result->files[0], 0); + result->buffer_manager = make_shared(context, options, result->files[0], 0, false); CSVSniffer sniffer(options, result->buffer_manager, CSVStateMachineCache::Get(context), {&return_types, &names}); auto sniffer_result = sniffer.SniffCSV(); diff --git a/src/function/table/sniff_csv.cpp b/src/function/table/sniff_csv.cpp index f135b15c615d..b776288d6f27 100644 --- a/src/function/table/sniff_csv.cpp +++ b/src/function/table/sniff_csv.cpp @@ -120,7 +120,7 @@ static void CSVSniffFunction(ClientContext &context, TableFunctionInput &data_p, auto sniffer_options = data.options; sniffer_options.file_path = data.path; - auto buffer_manager = make_shared(context, sniffer_options, sniffer_options.file_path, 0); + auto buffer_manager = make_shared(context, sniffer_options, sniffer_options.file_path, 0, false); CSVSniffer sniffer(sniffer_options, buffer_manager, CSVStateMachineCache::Get(context)); auto sniffer_result = sniffer.SniffCSV(true); string str_opt; diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_buffer.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_buffer.hpp index 72665ae2de54..8200da88e32a 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_buffer.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_buffer.hpp @@ -44,14 +44,15 @@ class CSVBuffer { public: //! Constructor for Initial Buffer CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle &file_handle, - idx_t &global_csv_current_position, idx_t file_number); + idx_t &global_csv_current_position, idx_t file_number, bool single_threaded); //! Constructor for `Next()` Buffers CSVBuffer(CSVFileHandle &file_handle, ClientContext &context, idx_t buffer_size, idx_t global_csv_current_position, - idx_t file_number_p, idx_t buffer_idx); + idx_t file_number_p, idx_t buffer_idx, bool single_threaded); //! Creates a new buffer with the next part of the CSV File - shared_ptr Next(CSVFileHandle &file_handle, idx_t buffer_size, idx_t file_number, bool &has_seaked); + shared_ptr Next(CSVFileHandle &file_handle, idx_t buffer_size, idx_t file_number, bool &has_seaked, + bool single_threaded); //! Gets the buffer actual size idx_t GetBufferSize(); @@ -60,7 +61,7 @@ class CSVBuffer { bool IsCSVFileLastBuffer(); //! Allocates internal buffer, sets 'block' and 'handle' variables. - void AllocateBuffer(idx_t buffer_size); + void AllocateBuffer(idx_t buffer_size, bool can_destroy); void Reload(CSVFileHandle &file_handle); //! Wrapper for the Pin Function, if it can seek, it means that the buffer might have been destroyed, hence we must diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp index b9b4bb92d372..a1127882a718 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp @@ -22,7 +22,7 @@ class CSVStateMachine; class CSVBufferManager { public: CSVBufferManager(ClientContext &context, const CSVReaderOptions &options, const string &file_path, - const idx_t file_idx); + const idx_t file_idx, bool single_threaded); //! Returns a buffer from a buffer id (starting from 0). If it's in the auto-detection then we cache new buffers //! Otherwise we remove them from the cache if they are already there, or just return them bypassing the cache. shared_ptr GetBuffer(const idx_t buffer_idx); @@ -44,6 +44,8 @@ class CSVBufferManager { string GetFilePath(); + void SetSingleThreaded(); + ClientContext &context; idx_t skip_rows = 0; @@ -69,6 +71,7 @@ class CSVBufferManager { //! If the file_handle used seek bool has_seeked = false; unordered_set reset_when_possible; + bool single_threaded; }; } // namespace duckdb diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_file_scanner.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_file_scanner.hpp index ce9fc08ce0bd..ed859238d9ef 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_file_scanner.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_file_scanner.hpp @@ -27,9 +27,10 @@ class CSVFileScan { //! Path to this file CSVFileScan(ClientContext &context, const string &file_path, const CSVReaderOptions &options, const idx_t file_idx, const ReadCSVData &bind_data, const vector &column_ids, - const vector &file_schema); + const vector &file_schema, bool single_threaded); - CSVFileScan(ClientContext &context, const string &file_name, CSVReaderOptions &options); + CSVFileScan(ClientContext &context, const string &file_name, CSVReaderOptions &options, + bool single_threaded = false); const string &GetFileName(); const vector &GetNames(); diff --git a/src/main/relation/read_csv_relation.cpp b/src/main/relation/read_csv_relation.cpp index 1500720e0069..5d0b52e5c96d 100644 --- a/src/main/relation/read_csv_relation.cpp +++ b/src/main/relation/read_csv_relation.cpp @@ -56,7 +56,7 @@ ReadCSVRelation::ReadCSVRelation(const std::shared_ptr &context, shared_ptr buffer_manager; context->RunFunctionInTransaction([&]() { - buffer_manager = make_shared(*context, csv_options, files[0], 0); + buffer_manager = make_shared(*context, csv_options, files[0], 0, false); CSVSniffer sniffer(csv_options, buffer_manager, CSVStateMachineCache::Get(*context)); auto sniffer_result = sniffer.SniffCSV(); auto &types = sniffer_result.return_types; From 8db82b0d018f0694e13cfd4ed5a2b5d6ce3edde9 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 20 Mar 2024 15:44:24 +0100 Subject: [PATCH 089/603] bad file --- test/sql/copy/csv/test_gzipped.test | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 test/sql/copy/csv/test_gzipped.test diff --git a/test/sql/copy/csv/test_gzipped.test b/test/sql/copy/csv/test_gzipped.test deleted file mode 100644 index e69de29bb2d1..000000000000 From d0ff7c077a177b274da9a8f737c06453f5e9f271 Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 21 Mar 2024 11:24:00 +0100 Subject: [PATCH 090/603] started on giant json file containing all the information to generate stubs, pybind11 def and init.py --- .../pythonpkg/scripts/connection_methods.json | 856 ++++++++++++++++++ .../scripts/generate_function_definition.py | 3 + .../pyconnection/pyconnection.hpp | 2 +- tools/pythonpkg/src/pyconnection.cpp | 18 +- 4 files changed, 867 insertions(+), 12 deletions(-) create mode 100644 tools/pythonpkg/scripts/connection_methods.json create mode 100644 tools/pythonpkg/scripts/generate_function_definition.py diff --git a/tools/pythonpkg/scripts/connection_methods.json b/tools/pythonpkg/scripts/connection_methods.json new file mode 100644 index 000000000000..dedaef4cb967 --- /dev/null +++ b/tools/pythonpkg/scripts/connection_methods.json @@ -0,0 +1,856 @@ +[ + { + "name": "cursor", + "function": "Cursor", + "docs": "Create a duplicate of the current connection", + "return": "DuckDBPyConnection" + }, + { + "name": "register_filesystem", + "function": "RegisterFilesystem", + "docs": "Register a fsspec compliant filesystem", + "args": [ + { + "name": "filesystem", + "type": "str" + } + ] + }, + { + "name": "unregister_filesystem", + "function": "UnregisterFilesystem", + "docs": "Unregister a filesystem", + "args": [ + { + "name": "name", + "type": "str" + } + ] + }, + { + "name": "list_filesystems", + "function": "ListFilesystems", + "docs": "List registered filesystems, including builtin ones", + "return": "list" + }, + { + "name": "filesystem_is_registered", + "function": "FileSystemIsRegistered", + "docs": "Check if a filesystem with the provided name is currently registered", + "args": [ + { + "name": "name" + } + ], + "return": "bool" + }, + { + "name": "create_function", + "function": "RegisterScalarUDF", + "docs": "Create a DuckDB function out of the passing in Python function so it can be used in queries", + "args": [ + { + "name": "name", + "type": "str" + }, + { + "name": "function", + "type": "function" + }, + { + "name": "parameters", + "type": "Optional[List[DuckDBPyType]]", + "default": "None" + }, + { + "name": "return_type", + "type": "Optional[DuckDBPyType]", + "default": "None" + } + ], + "kwargs": [ + { + "name": "type", + "type": "Optional[PythonUDFType]", + "default": "PythonUDFType.NATIVE" + }, + { + "name": "null_handling", + "type": "Optional[FunctionNullHandling]", + "default": "FunctionNullHandling.DEFAULT" + }, + { + "name": "exception_handling", + "type": "Optional[PythonExceptionHandling]", + "default": "PythonExceptionHandling.DEFAULT" + }, + { + "name": "side_effects", + "type": "bool", + "default": "False" + } + ], + "return": "DuckDBPyConnection" + }, + { + "name": "remove_function", + "function": "UnregisterUDF", + "docs": "Remove a previously created function", + "args": [ + { + "name": "name", + "type": "str" + } + ], + "return": "DuckDBPyConnection" + }, + + { + "name": ["sqltype", "dtype", "type"], + "function": "Type", + "docs": "Create a type object by parsing the 'type_str' string", + "args": [ + { + "name": "type_str" + } + ], + "return": + }, + { + "name": "array_type", + "function": "ArrayType", + "docs": "Create an array type object of 'type'", + "args": [ + + ] + { + "name": "type" + .none(false) + }, + { + "name": "size" + .none(false + }, + }, + { + "name": "list_type", + "function": "ListType", + "docs": "Create a list type object of 'type'", + "args": [ + + ] + { + "name": "type" + .none(false + }, + }, + { + "name": "union_type", + "function": "UnionType", + "docs": "Create a union type object from 'members'", + "args": [ + + ] + { + "name": "members" + .none(false) + }, + }, + { + "name": "string_type", + "function": "StringType", + "docs": "Create a string type with an optional collation", + "args": [ + + ] + { + "name": "collation" + = string() + }, + }, + { + "name": "enum_type", + "function": "EnumType", + "docs": "Create an enum type of underlying 'type', consisting of the list of 'values'", + "args": [ + + ] + { + "name": "name" + }, + { + "name": "type" + }, + { + "name": "values" + }, + }, + { + "name": "decimal_type", + "function": "DecimalType", + "docs": "Create a decimal type with 'width' and 'scale'", + "args": [ + + ] + { + "name": "width" + }, + { + "name": "scale" + }, + }, + { + "name": ["struct_type", "row_type"], + "function": "StructType", + "docs": "Create a struct type object from 'fields'", + { + "name": "fields" + }, + }, + { + "name": "map_type", + "function": "MapType", + "docs": "Create a map type object from 'key_type' and 'value_type'", + "args": [ + + ] + { + "name": "key" + .none(false) + }, + { + "name": "value" + .none(false) + }, + }, + { + "name": "duplicate", + "function": "Cursor", + "docs": "Create a duplicate of the current connection", + }, + { + "name": "execute", + "function": "Execute", + "docs": "Execute the given SQL query, optionally using prepared statements with parameters set", + "args": [ + + ] + { + "name": "query" + }, + { + "name": "parameters" + = py::none() + }, + { + "name": "multiple_parameter_sets" + = false + }, + }, + { + "name": "executemany", + "function": "ExecuteMany", + "docs": "Execute the given prepared statement multiple times using the list of parameter sets in parameters", + "args": [ + + ] + { + "name": "query" + }, + { + "name": "parameters" + = py::none() + }, + }, + { + "name": "close", + "function": "Close", + "docs": "Close the connection" + }, + { + "name": "interrupt", + "function": "Interrupt", + "docs": "Interrupt pending operations" + }, + { + "name": "fetchone", + "function": "FetchOne", + "docs": "Fetch a single row from a result following execute" + }, + { + "name": "fetchmany", + "function": "FetchMany", + "docs": "Fetch the next set of rows from a result following execute" + { + "name": "size" + = 1 + }, + }, + { + "name": "fetchall", + "function": "FetchAll", + "docs": "Fetch all rows from a result following execute" + }, + { + "name": "fetchnumpy", + "function": "FetchNumpy", + "docs": "Fetch a result as list of NumPy arrays following execute" + }, + { + "name": "fetchdf", + "function": "FetchDF", + "docs": "Fetch a result as DataFrame following execute()", + py::kw_only(), + { + "name": "date_as_object" + = false + }, + }, + { + "name": "fetch_df", + "function": "FetchDF", + "docs": "Fetch a result as DataFrame following execute()", + py::kw_only(), + { + "name": "date_as_object" + = false + }, + }, + { + "name": "fetch_df_chunk", + "function": "FetchDFChunk", + "docs": "Fetch a chunk of the result as Data.Frame following execute()", + "args": [ + + ] + { + "name": "vectors_per_chunk" + = 1 + }, + py::kw_only(), + { + "name": "date_as_object" + = false + }, + }, + { + "name": "df", + "function": "FetchDF", + "docs": "Fetch a result as DataFrame following execute()", + py::kw_only(), + { + "name": "date_as_object" + = false + }, + }, + { + "name": "pl", + "function": "FetchPolars", + "docs": "Fetch a result as Polars DataFrame following execute()", + "args": [ + + ] + { + "name": "rows_per_batch" + = 1000000 + }, + }, + { + "name": "fetch_arrow_table", + "function": "FetchArrow", + "docs": "Fetch a result as Arrow table following execute()", + "args": [ + + ] + { + "name": "rows_per_batch" + = 1000000 + }, + }, + { + "name": "fetch_record_batch", + "function": "FetchRecordBatchReader", + "docs": "Fetch an Arrow RecordBatchReader following execute()", + "args": [ + + ] + { + "name": "rows_per_batch" + = 1000000 + }, + }, + { + "name": "arrow", + "function": "FetchArrow", + "docs": "Fetch a result as Arrow table following execute()", + "args": [ + + ] + { + "name": "rows_per_batch" + = 1000000 + }, + }, + { + "name": "torch", + "function": "FetchPyTorch", + "docs": "Fetch a result as dict of PyTorch Tensors following execute()" + + }, + { + "name": "tf", + "function": "FetchTF", + "docs": "Fetch a result as dict of TensorFlow Tensors following execute()" + + }, + { + "name": "begin", + "function": "Begin", + "docs": "Start a new transaction" + + }, + { + "name": "commit", + "function": "Commit", + "docs": "Commit changes performed within a transaction" + }, + { + "name": "rollback", + "function": "Rollback", + "docs": "Roll back changes performed within a transaction" + }, + { + "name": "append", + "function": "Append", + "docs": "Append the passed DataFrame to the named table", + "args": [ + + ] + { + "name": "table_name" + }, + { + "name": "df" + }, + py::kw_only(), + { + "name": "by_name" + = false + }, + }, + { + "name": "register", + "function": "RegisterPythonObject", + "docs": "Register the passed Python Object value for querying with a view", + "args": [ + + ] + { + "name": "view_name" + }, + { + "name": "python_object" + }, + }, + { + "name": "unregister", + "function": "UnregisterPythonObject", + "docs": "Unregister the view name", + "args": [ + + ] + { + "name": "view_name" + }, + }, + { + "name": "table", + "function": "Table", + "docs": "Create a relation object for the name'd table", + "args": [ + + ] + { + "name": "table_name" + }, + }, + { + "name": "view", + "function": "View", + "docs": "Create a relation object for the name'd view", + "args": [ + + ] + { + "name": "view_name" + }, + }, + { + "name": "values", + "function": "Values", + "docs": "Create a relation object from the passed values", + "args": [ + + ] + { + "name": "values" + }, + }, + { + "name": "table_function", + "function": "TableFunction", + "docs": "Create a relation object from the name'd table function with given parameters", + "args": [ + + ] + { + "name": "name" + }, + { + "name": "parameters" + = py::none() + }, + }, + { + "name": "read_json", + "function": "ReadJSON", + "docs": "Create a relation object from the JSON file in 'name'", + "args": [ + + ] + { + "name": "name" + }, + py::kw_only(), + { + "name": "columns" + = py::none() + }, + { + "name": "sample_size" + = py::none() + }, + { + "name": "maximum_depth" + = py::none() + }, + { + "name": "records" + = py::none() + }, + { + "name": "format" + = py::none() + }, + }, + { + "name": "extract_statements", + "function": "ExtractStatements", + "docs": "Parse the query string and extract the Statement object(s) produced", + "args": [ + + ] + { + "name": "query" + }, + }, + + { + "name": ["sql", "query", "from_query"], + "function": "RunQuery", + "docs": "Run a SQL query. If it is a SELECT statement, create a relation object from the given SQL query, otherwise run the query as-is.", + "args": [ + + ] + { + "name": "query" + }, + py::kw_only(), + { + "name": "alias" + = "" + }, + { + "name": "params" + = py::none( + }, + }, + + { + "name": ["read_csv", "from_csv_auto"], + "function": "ReadCSV", + "docs": "Create a relation object from the CSV file in 'name'", + "args": [ + + ] + { + "name": "name" + }, + py::kw_only(), + { + "name": "header" + = py::none() + }, + { + "name": "compression" + = py::none() + }, + { + "name": "sep" + = py::none() + }, + { + "name": "delimiter" + = py::none() + }, + { + "name": "dtype" + = py::none() + }, + { + "name": "na_values" + = py::none() + }, + { + "name": "skiprows" + = py::none() + }, + { + "name": "quotechar" + = py::none() + }, + { + "name": "escapechar" + = py::none() + }, + { + "name": "encoding" + = py::none() + }, + { + "name": "parallel" + = py::none() + }, + { + "name": "date_format" + = py::none() + }, + { + "name": "timestamp_format" + = py::none() + }, + { + "name": "sample_size" + = py::none() + }, + { + "name": "all_varchar" + = py::none() + }, + { + "name": "normalize_names" + = py::none() + }, + { + "name": "filename" + = py::none() + }, + { + "name": "null_padding" + = py::none() + }, + { + "name": "names" + = py::none( + }, + }, + { + "name": "from_df", + "function": "FromDF", + "docs": "Create a relation object from the Data.Frame in df", + "args": [ + + ] + { + "name": "df" + = py::none() + }, + }, + { + "name": "from_arrow", + "function": "FromArrow", + "docs": "Create a relation object from an Arrow object", + "args": [ + + ] + { + "name": "arrow_object" + }, + }, + + { + "name": ["from_parquet", "read_parquet"], + "function": "FromParquet", + "docs": "Create a relation object from the Parquet files in file_glob", + "args": [ + + ] + { + "name": "file_glob" + }, + { + "name": "binary_as_string" + = false + }, + py::kw_only(), + { + "name": "file_row_number" + = false + }, + { + "name": "filename" + = false + }, + { + "name": "hive_partitioning" + = false + }, + { + "name": "union_by_name" + = false + }, + { + "name": "compression" + = py::none( + }, + }, + { + "name": ["from_parquet", "read_parquet"], + "function": "FromParquets", + "docs": "Create a relation object from the Parquet files in file_globs", + "args": [ + + ] + { + "name": "file_globs" + }, + { + "name": "binary_as_string" + = false + }, + py::kw_only(), + { + "name": "file_row_number" + = false + }, + { + "name": "filename" + = false + }, + { + "name": "hive_partitioning" + = false + }, + { + "name": "union_by_name" + = false + }, + { + "name": "compression" + = py::none( + }, + }, + { + "name": "from_substrait", + "function": "FromSubstrait", + "docs": "Create a query object from protobuf plan", + "args": [ + + ] + { + "name": "proto" + }, + }, + { + "name": "get_substrait", + "function": "GetSubstrait", + "docs": "Serialize a query to protobuf", + "args": [ + + ] + { + "name": "query" + }, + py::kw_only(), + { + "name": "enable_optimizer" + = true + }, + }, + { + "name": "get_substrait_json", + "function": "GetSubstraitJSON", + "docs": "Serialize a query to protobuf on the JSON format", + "args": [ + + ] + { + "name": "query" + }, + py::kw_only(), + { + "name": "enable_optimizer" + = true + }, + }, + { + "name": "from_substrait_json", + "function": "FromSubstraitJSON", + "docs": "Create a query object from a JSON protobuf plan", + "args": [ + + ] + { + "name": "json" + }, + }, + { + "name": "get_table_names", + "function": "GetTableNames", + "docs": "Extract the required table names from a query", + "args": [ + + ] + { + "name": "query" + }, + }, + + { + "name": "install_extension", + "function": "InstallExtension", + "docs": "Install an extension by name", + "args": [ + + ] + { + "name": "extension" + }, + py::kw_only(), + { + "name": "force_install" + = false + }, + }, + { + "name": "load_extension", + "function": "LoadExtension", + "docs": "Load an installed extension", + "args": [ + + ] + { + "name": "extension" + }, + } +] diff --git a/tools/pythonpkg/scripts/generate_function_definition.py b/tools/pythonpkg/scripts/generate_function_definition.py new file mode 100644 index 000000000000..aca68675eb43 --- /dev/null +++ b/tools/pythonpkg/scripts/generate_function_definition.py @@ -0,0 +1,3 @@ +class FunctionDefinition: + def __init__(self): + pass diff --git a/tools/pythonpkg/src/include/duckdb_python/pyconnection/pyconnection.hpp b/tools/pythonpkg/src/include/duckdb_python/pyconnection/pyconnection.hpp index a0d572dc9a91..bf1d92f9c00a 100644 --- a/tools/pythonpkg/src/include/duckdb_python/pyconnection/pyconnection.hpp +++ b/tools/pythonpkg/src/include/duckdb_python/pyconnection/pyconnection.hpp @@ -116,7 +116,7 @@ struct DuckDBPyConnection : public std::enable_shared_from_this Execute(const py::object &query, py::object params = py::list(), bool many = false); - shared_ptr Execute(const string &query); + shared_ptr ExecuteFromString(const string &query); shared_ptr Append(const string &name, const PandasDataFrame &value, bool by_name); diff --git a/tools/pythonpkg/src/pyconnection.cpp b/tools/pythonpkg/src/pyconnection.cpp index 661813422c1e..f4cd9100a142 100644 --- a/tools/pythonpkg/src/pyconnection.cpp +++ b/tools/pythonpkg/src/pyconnection.cpp @@ -160,13 +160,9 @@ static void InitializeConnectionMethods(py::class_> DuckDBPyConnection::GetStatements(const py::obj throw InvalidInputException("Please provide either a DuckDBPyStatement or a string representing the query"); } -shared_ptr DuckDBPyConnection::Execute(const string &query) { +shared_ptr DuckDBPyConnection::ExecuteFromString(const string &query) { return Execute(py::str(query)); } @@ -1254,7 +1250,7 @@ shared_ptr DuckDBPyConnection::UnregisterPythonObject(const } shared_ptr DuckDBPyConnection::Begin() { - Execute("BEGIN TRANSACTION"); + ExecuteFromString("BEGIN TRANSACTION"); return shared_from_this(); } @@ -1262,12 +1258,12 @@ shared_ptr DuckDBPyConnection::Commit() { if (connection->context->transaction.IsAutoCommit()) { return shared_from_this(); } - Execute("COMMIT"); + ExecuteFromString("COMMIT"); return shared_from_this(); } shared_ptr DuckDBPyConnection::Rollback() { - Execute("ROLLBACK"); + ExecuteFromString("ROLLBACK"); return shared_from_this(); } From 4848acf50ec10ec905dda249978055c9efbd278c Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 21 Mar 2024 15:29:35 +0100 Subject: [PATCH 091/603] move the setting into the buffer manager + temporary directory handle --- src/common/file_system.cpp | 16 ++-- src/include/duckdb/common/file_system.hpp | 5 +- src/include/duckdb/main/config.hpp | 42 ++-------- src/include/duckdb/storage/buffer_manager.hpp | 19 ++++- .../storage/standard_buffer_manager.hpp | 29 +++++-- .../duckdb/storage/temporary_file_manager.hpp | 8 +- src/main/config.cpp | 30 +------ src/main/database.cpp | 7 +- src/main/settings/settings.cpp | 43 ++++++++-- src/storage/buffer_manager.cpp | 8 +- src/storage/standard_buffer_manager.cpp | 84 ++++++++++++------- src/storage/temporary_file_manager.cpp | 44 +++++++--- .../max_swap_space_inmemory.test | 5 +- .../max_swap_space_persistent.test | 5 +- 14 files changed, 197 insertions(+), 148 deletions(-) diff --git a/src/common/file_system.cpp b/src/common/file_system.cpp index f21e6c46dd79..c58bb588ee07 100644 --- a/src/common/file_system.cpp +++ b/src/common/file_system.cpp @@ -84,7 +84,7 @@ void FileSystem::SetWorkingDirectory(const string &path) { } } -idx_t FileSystem::GetAvailableMemory() { +optional_idx FileSystem::GetAvailableMemory() { errno = 0; #ifdef __MVS__ @@ -95,16 +95,16 @@ idx_t FileSystem::GetAvailableMemory() { idx_t max_memory = MinValue((idx_t)sysconf(_SC_PHYS_PAGES) * (idx_t)sysconf(_SC_PAGESIZE), UINTPTR_MAX); #endif if (errno != 0) { - return DConstants::INVALID_INDEX; + return optional_idx(); } return max_memory; } -idx_t FileSystem::GetAvailableDiskSpace(const string &path) { +optional_idx FileSystem::GetAvailableDiskSpace(const string &path) { struct statvfs vfs; if (statvfs(path.c_str(), &vfs) == -1) { - return DConstants::INVALID_INDEX; + optional_idx(); } auto block_size = vfs.f_frsize; // These are the blocks available for creating new files or extending existing ones @@ -112,7 +112,7 @@ idx_t FileSystem::GetAvailableDiskSpace(const string &path) { idx_t available_disk_space = DConstants::INVALID_INDEX; if (!TryMultiplyOperator::Operation(static_cast(block_size), static_cast(available_blocks), available_disk_space)) { - return DConstants::INVALID_INDEX; + return optional_idx(); } return available_disk_space; } @@ -214,15 +214,15 @@ idx_t FileSystem::GetAvailableMemory() { if (GlobalMemoryStatusEx(&mem_state)) { return MinValue(mem_state.ullTotalPhys, UINTPTR_MAX); } - return DConstants::INVALID_INDEX; + return optional_idx(); } -idx_t FileSystem::GetAvailableDiskSpace(const string &path) { +optional_idx FileSystem::GetAvailableDiskSpace(const string &path) { ULARGE_INTEGER available_bytes, total_bytes, free_bytes; auto unicode_path = WindowsUtil::UTF8ToUnicode(path.c_str()); if (!GetDiskFreeSpaceExW(unicode_path.c_str(), &available_bytes, &total_bytes, &free_bytes)) { - return DConstants::INVALID_INDEX; + return optional_idx(); } (void)total_bytes; (void)free_bytes; diff --git a/src/include/duckdb/common/file_system.hpp b/src/include/duckdb/common/file_system.hpp index e8b11d1802e7..d39abff5362e 100644 --- a/src/include/duckdb/common/file_system.hpp +++ b/src/include/duckdb/common/file_system.hpp @@ -16,6 +16,7 @@ #include "duckdb/common/vector.hpp" #include "duckdb/common/enums/file_glob_options.hpp" #include "duckdb/common/optional_ptr.hpp" +#include "duckdb/common/optional_idx.hpp" #include #undef CreateDirectory @@ -187,9 +188,9 @@ class FileSystem { //! Expands a given path, including e.g. expanding the home directory of the user DUCKDB_API virtual string ExpandPath(const string &path); //! Returns the system-available memory in bytes. Returns DConstants::INVALID_INDEX if the system function fails. - DUCKDB_API static idx_t GetAvailableMemory(); + DUCKDB_API static optional_idx GetAvailableMemory(); //! Returns the space available on the disk. Returns DConstants::INVALID_INDEX if the information was not available. - DUCKDB_API static idx_t GetAvailableDiskSpace(const string &path); + DUCKDB_API static optional_idx GetAvailableDiskSpace(const string &path); //! Path separator for path DUCKDB_API virtual string PathSeparator(const string &path); //! Checks if path is starts with separator (i.e., '/' on UNIX '\\' on Windows) diff --git a/src/include/duckdb/main/config.hpp b/src/include/duckdb/main/config.hpp index fcfda6ea8c40..1441a6d1a986 100644 --- a/src/include/duckdb/main/config.hpp +++ b/src/include/duckdb/main/config.hpp @@ -61,37 +61,6 @@ typedef void (*reset_global_function_t)(DatabaseInstance *db, DBConfig &config); typedef void (*reset_local_function_t)(ClientContext &context); typedef Value (*get_setting_function_t)(ClientContext &context); -struct NumericSetting { -public: - NumericSetting() : value(0), set_by_user(false) { - } - -public: - NumericSetting &operator=(idx_t val) = delete; - -public: - operator idx_t() { - return value; - } - -public: - bool ExplicitlySet() const { - return set_by_user; - } - void SetDefault(idx_t val) { - value = val; - set_by_user = false; - } - void SetExplicit(idx_t val) { - value = val; - set_by_user = true; - } - -private: - idx_t value; - bool set_by_user; -}; - struct ConfigurationOption { const char *name; const char *description; @@ -145,14 +114,14 @@ struct DBConfigOptions { #endif //! Override for the default extension repository string custom_extension_repo = ""; - //! Override for the default autoload extensoin repository + //! Override for the default autoload extension repository string autoinstall_extension_repo = ""; //! The maximum memory used by the database system (in bytes). Default: 80% of System available memory - idx_t maximum_memory = (idx_t)-1; - //! The maximum size of the 'temp_directory' folder when set (in bytes) - NumericSetting maximum_swap_space = NumericSetting(); + idx_t maximum_memory = DConstants::INVALID_INDEX; + //! The maximum size of the 'temp_directory' folder when set (in bytes). Default: 90% of available disk space. + idx_t maximum_swap_space = DConstants::INVALID_INDEX; //! The maximum amount of CPU threads used by the database system. Default: all available. - idx_t maximum_threads = (idx_t)-1; + idx_t maximum_threads = DConstants::INVALID_INDEX; //! The number of external threads that work on DuckDB tasks. Default: 1. //! Must be smaller or equal to maximum_threads. idx_t external_threads = 1; @@ -310,7 +279,6 @@ struct DBConfig { DUCKDB_API IndexTypeSet &GetIndexTypes(); static idx_t GetSystemMaxThreads(FileSystem &fs); void SetDefaultMaxMemory(); - void SetDefaultMaxSwapSpace(optional_ptr db); void SetDefaultTempDirectory(); OrderType ResolveOrder(OrderType order_type) const; diff --git a/src/include/duckdb/storage/buffer_manager.hpp b/src/include/duckdb/storage/buffer_manager.hpp index 7238be7844df..19e29aa89365 100644 --- a/src/include/duckdb/storage/buffer_manager.hpp +++ b/src/include/duckdb/storage/buffer_manager.hpp @@ -41,10 +41,17 @@ class BufferManager { virtual void ReAllocate(shared_ptr &handle, idx_t block_size) = 0; virtual BufferHandle Pin(shared_ptr &handle) = 0; virtual void Unpin(shared_ptr &handle) = 0; + //! Returns the currently allocated memory virtual idx_t GetUsedMemory() const = 0; //! Returns the maximum available memory virtual idx_t GetMaxMemory() const = 0; + //! Returns the currently used swap space + virtual idx_t GetUsedSwap() = 0; + //! Returns the maximum swap space that can be used + virtual optional_idx GetMaxSwap() = 0; + + //! Returns a new block of memory that is smaller than Storage::BLOCK_SIZE virtual shared_ptr RegisterSmallMemory(idx_t block_size); virtual DUCKDB_API Allocator &GetBufferAllocator(); virtual DUCKDB_API void ReserveMemory(idx_t size); @@ -52,20 +59,21 @@ class BufferManager { virtual vector GetMemoryUsageInfo() const = 0; //! Set a new memory limit to the buffer manager, throws an exception if the new limit is too low and not enough //! blocks can be evicted - virtual void SetLimit(idx_t limit = (idx_t)-1); + virtual void SetMemoryLimit(idx_t limit = (idx_t)-1); + virtual void SetSwapLimit(optional_idx limit = optional_idx()); + virtual vector GetTemporaryFiles(); virtual const string &GetTemporaryDirectory(); virtual void SetTemporaryDirectory(const string &new_dir); - virtual DatabaseInstance &GetDatabase(); virtual bool HasTemporaryDirectory() const; + //! Construct a managed buffer. virtual unique_ptr ConstructManagedBuffer(idx_t size, unique_ptr &&source, FileBufferType type = FileBufferType::MANAGED_BUFFER); //! Get the underlying buffer pool responsible for managing the buffers virtual BufferPool &GetBufferPool() const; - //! Get the manager that assigns reservations for temporary memory, e.g., for query intermediates - virtual TemporaryMemoryManager &GetTemporaryMemoryManager(); + virtual DatabaseInstance &GetDatabase(); // Static methods DUCKDB_API static BufferManager &GetBufferManager(DatabaseInstance &db); DUCKDB_API static BufferManager &GetBufferManager(ClientContext &context); @@ -77,6 +85,9 @@ class BufferManager { //! Returns the maximum available memory for a given query idx_t GetQueryMaxMemory() const; + //! Get the manager that assigns reservations for temporary memory, e.g., for query intermediates + virtual TemporaryMemoryManager &GetTemporaryMemoryManager(); + protected: virtual void PurgeQueue() = 0; virtual void AddToEvictionQueue(shared_ptr &handle); diff --git a/src/include/duckdb/storage/standard_buffer_manager.hpp b/src/include/duckdb/storage/standard_buffer_manager.hpp index de36f9477ad5..fe64355748c3 100644 --- a/src/include/duckdb/storage/standard_buffer_manager.hpp +++ b/src/include/duckdb/storage/standard_buffer_manager.hpp @@ -50,6 +50,8 @@ class StandardBufferManager : public BufferManager { idx_t GetUsedMemory() const final override; idx_t GetMaxMemory() const final override; + idx_t GetUsedSwap() final override; + optional_idx GetMaxSwap() final override; //! Allocate an in-memory buffer with a single pin. //! The allocated memory is released when the buffer handle is destroyed. @@ -64,7 +66,8 @@ class StandardBufferManager : public BufferManager { //! Set a new memory limit to the buffer manager, throws an exception if the new limit is too low and not enough //! blocks can be evicted - void SetLimit(idx_t limit = (idx_t)-1) final override; + void SetMemoryLimit(idx_t limit = (idx_t)-1) final override; + void SetSwapLimit(optional_idx limit = optional_idx()) final override; //! Returns informaton about memory usage vector GetMemoryUsageInfo() const override; @@ -73,7 +76,7 @@ class StandardBufferManager : public BufferManager { vector GetTemporaryFiles() final override; const string &GetTemporaryDirectory() final override { - return temp_directory; + return temporary_directory.path; } void SetTemporaryDirectory(const string &new_dir) final override; @@ -136,17 +139,27 @@ class StandardBufferManager : public BufferManager { //! overwrites the data within with garbage. Any readers that do not hold the pin will notice void VerifyZeroReaders(shared_ptr &handle); +protected: + // These are stored here because temp_directory creation is lazy + // so we need to store information related to the temporary directory before it's created + struct TemporaryFileData { + //! The directory name where temporary files are stored + string path; + //! Lock for creating the temp handle + mutex lock; + //! Handle for the temporary directory + unique_ptr handle; + //! The maximum swap space that can be used + optional_idx maximum_swap_space = optional_idx(); + }; + protected: //! The database instance DatabaseInstance &db; //! The buffer pool BufferPool &buffer_pool; - //! The directory name where temporary files are stored - string temp_directory; - //! Lock for creating the temp handle - mutex temp_handle_lock; - //! Handle for the temporary directory - unique_ptr temp_directory_handle; + //! The variables related to temporary file management + TemporaryFileData temporary_directory; //! The temporary id used for managed buffers atomic temporary_id; //! Allocator associated with the buffer manager, that passes all allocations through this buffer manager diff --git a/src/include/duckdb/storage/temporary_file_manager.hpp b/src/include/duckdb/storage/temporary_file_manager.hpp index 7c118fb4cf76..9ab77e42e958 100644 --- a/src/include/duckdb/storage/temporary_file_manager.hpp +++ b/src/include/duckdb/storage/temporary_file_manager.hpp @@ -128,7 +128,7 @@ class TemporaryFileHandle { class TemporaryDirectoryHandle { public: - TemporaryDirectoryHandle(DatabaseInstance &db, string path_p); + TemporaryDirectoryHandle(DatabaseInstance &db, string path_p, optional_idx max_swap_space); ~TemporaryDirectoryHandle(); TemporaryFileManager &GetTempFile(); @@ -146,7 +146,7 @@ class TemporaryDirectoryHandle { class TemporaryFileManager { public: - TemporaryFileManager(DatabaseInstance &db, const string &temp_directory_p); + TemporaryFileManager(DatabaseInstance &db, const string &temp_directory_p, optional_idx max_swap_space); ~TemporaryFileManager(); public: @@ -164,6 +164,8 @@ class TemporaryFileManager { void DeleteTemporaryBuffer(block_id_t id); vector GetTemporaryFiles(); idx_t GetTotalUsedSpaceInBytes(); + optional_idx GetMaxSwapSpace() const; + void SetMaxSwapSpace(optional_idx limit); //! Register temporary file size growth void IncreaseSizeOnDisk(idx_t amount); //! Register temporary file size decrease @@ -189,6 +191,8 @@ class TemporaryFileManager { BlockIndexManager index_manager; //! The size in bytes of the temporary files that are currently alive atomic size_on_disk; + //! The max amount of disk space that can be used + idx_t max_swap_space; }; } // namespace duckdb diff --git a/src/main/config.cpp b/src/main/config.cpp index f8f8ca1d1101..f280cf31b6c5 100644 --- a/src/main/config.cpp +++ b/src/main/config.cpp @@ -256,10 +256,7 @@ bool DBConfig::IsInMemoryDatabase(const char *database_path) { // '' empty string return true; } - constexpr const char *IN_MEMORY_PATH_PREFIX = ":memory:"; - const idx_t PREFIX_LENGTH = strlen(IN_MEMORY_PATH_PREFIX); - if (strncmp(database_path, IN_MEMORY_PATH_PREFIX, PREFIX_LENGTH) == 0) { - // Starts with :memory:, i.e ':memory:named_conn' is valid + if (strcmp(database_path, ":memory:") == 0) { return true; } return false; @@ -275,8 +272,8 @@ IndexTypeSet &DBConfig::GetIndexTypes() { void DBConfig::SetDefaultMaxMemory() { auto memory = FileSystem::GetAvailableMemory(); - if (memory != DConstants::INVALID_INDEX) { - options.maximum_memory = memory * 8 / 10; + if (memory.IsValid()) { + options.maximum_memory = memory.GetIndex() * 8 / 10; } } @@ -288,27 +285,6 @@ void DBConfig::SetDefaultTempDirectory() { } } -void DBConfig::SetDefaultMaxSwapSpace(optional_ptr db) { - options.maximum_swap_space.SetDefault(0); - if (options.temporary_directory.empty()) { - return; - } - if (!db) { - return; - } - auto &fs = FileSystem::GetFileSystem(*db); - if (!fs.DirectoryExists(options.temporary_directory)) { - // Directory doesnt exist yet, we will look up the disk space once we have created the directory - // FIXME: do we want to proactively create the directory instead ??? - return; - } - // Use the available disk space if temp directory is set - auto disk_space = FileSystem::GetAvailableDiskSpace(options.temporary_directory); - // Only use 90% of the available disk space - auto default_value = disk_space == DConstants::INVALID_INDEX ? 0 : static_cast(disk_space) * 0.9; - options.maximum_swap_space.SetDefault(default_value); -} - void DBConfig::CheckLock(const string &name) { if (!options.lock_configuration) { // not locked diff --git a/src/main/database.cpp b/src/main/database.cpp index e561eeab5a50..21598a6522b0 100644 --- a/src/main/database.cpp +++ b/src/main/database.cpp @@ -334,13 +334,10 @@ void DatabaseInstance::Configure(DBConfig &new_config, const char *database_path if (new_config.secret_manager) { config.secret_manager = std::move(new_config.secret_manager); } - if (config.options.maximum_memory == (idx_t)-1) { + if (config.options.maximum_memory == DConstants::INVALID_INDEX) { config.SetDefaultMaxMemory(); } - if (!config.options.maximum_swap_space.ExplicitlySet()) { - config.SetDefaultMaxSwapSpace(this); - } - if (new_config.options.maximum_threads == (idx_t)-1) { + if (new_config.options.maximum_threads == DConstants::INVALID_INDEX) { config.options.maximum_threads = config.GetSystemMaxThreads(*config.file_system); } config.allocator = std::move(new_config.allocator); diff --git a/src/main/settings/settings.cpp b/src/main/settings/settings.cpp index d237fb7e882b..21ef43e50ee9 100644 --- a/src/main/settings/settings.cpp +++ b/src/main/settings/settings.cpp @@ -959,7 +959,7 @@ Value MaximumExpressionDepthSetting::GetSetting(ClientContext &context) { void MaximumMemorySetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) { config.options.maximum_memory = DBConfig::ParseMemoryLimit(input.ToString()); if (db) { - BufferManager::GetBufferManager(*db).SetLimit(config.options.maximum_memory); + BufferManager::GetBufferManager(*db).SetMemoryLimit(config.options.maximum_memory); } } @@ -976,18 +976,46 @@ Value MaximumMemorySetting::GetSetting(ClientContext &context) { // Maximum Temp Directory Size //===--------------------------------------------------------------------===// void MaximumTempDirectorySize::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) { - // FIXME: should this not use 'SetExplicit' when the value is 0? - // So it acts as RESET instead when 0 is passed? - config.options.maximum_swap_space.SetExplicit(DBConfig::ParseMemoryLimit(input.ToString())); + idx_t maximum_swap_space = DConstants::INVALID_INDEX; + if (input.ToString() != "-1") { + maximum_swap_space = DBConfig::ParseMemoryLimit(input.ToString()); + } + config.options.maximum_swap_space = maximum_swap_space; + if (!db) { + return; + } + auto &buffer_manager = BufferManager::GetBufferManager(*db); + if (maximum_swap_space == DConstants::INVALID_INDEX) { + buffer_manager.SetSwapLimit(); + } else { + buffer_manager.SetSwapLimit(maximum_swap_space); + } } void MaximumTempDirectorySize::ResetGlobal(DatabaseInstance *db, DBConfig &config) { - config.SetDefaultMaxSwapSpace(db); + config.options.maximum_swap_space = DConstants::INVALID_INDEX; + if (!db) { + return; + } + auto &buffer_manager = BufferManager::GetBufferManager(*db); + buffer_manager.SetSwapLimit(); } Value MaximumTempDirectorySize::GetSetting(ClientContext &context) { auto &config = DBConfig::GetConfig(context); - return Value(StringUtil::BytesToHumanReadableString(config.options.maximum_swap_space)); + if (config.options.maximum_swap_space != DConstants::INVALID_INDEX) { + // Explicitly set by the user + return Value(StringUtil::BytesToHumanReadableString(config.options.maximum_swap_space)); + } + auto &buffer_manager = BufferManager::GetBufferManager(context); + // Database is initialized, use the setting from the temporary directory + auto max_swap = buffer_manager.GetMaxSwap(); + if (max_swap.IsValid()) { + return Value(StringUtil::BytesToHumanReadableString(max_swap.GetIndex())); + } else { + // The temp directory has not been used yet + return Value(StringUtil::BytesToHumanReadableString(0)); + } } //===--------------------------------------------------------------------===// @@ -1271,9 +1299,6 @@ Value SecretDirectorySetting::GetSetting(ClientContext &context) { void TempDirectorySetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) { config.options.temporary_directory = input.ToString(); config.options.use_temporary_directory = !config.options.temporary_directory.empty(); - if (!config.options.maximum_swap_space.ExplicitlySet()) { - config.SetDefaultMaxSwapSpace(db); - } if (db) { auto &buffer_manager = BufferManager::GetBufferManager(*db); buffer_manager.SetTemporaryDirectory(config.options.temporary_directory); diff --git a/src/storage/buffer_manager.cpp b/src/storage/buffer_manager.cpp index 5b0a89296c9c..64eceba4807a 100644 --- a/src/storage/buffer_manager.cpp +++ b/src/storage/buffer_manager.cpp @@ -27,8 +27,12 @@ void BufferManager::FreeReservedMemory(idx_t size) { throw NotImplementedException("This type of BufferManager can not free reserved memory"); } -void BufferManager::SetLimit(idx_t limit) { - throw NotImplementedException("This type of BufferManager can not set a limit"); +void BufferManager::SetMemoryLimit(idx_t limit) { + throw NotImplementedException("This type of BufferManager can not set a memory limit"); +} + +void BufferManager::SetSwapLimit(optional_idx limit) { + throw NotImplementedException("This type of BufferManager can not set a swap limit"); } vector BufferManager::GetTemporaryFiles() { diff --git a/src/storage/standard_buffer_manager.cpp b/src/storage/standard_buffer_manager.cpp index ab5ad560ba12..feb61f75a536 100644 --- a/src/storage/standard_buffer_manager.cpp +++ b/src/storage/standard_buffer_manager.cpp @@ -36,16 +36,18 @@ unique_ptr StandardBufferManager::ConstructManagedBuffer(idx_t size, } void StandardBufferManager::SetTemporaryDirectory(const string &new_dir) { - if (temp_directory_handle) { + lock_guard guard(temporary_directory.lock); + if (temporary_directory.handle) { throw NotImplementedException("Cannot switch temporary directory after the current one has been used"); } - this->temp_directory = new_dir; + temporary_directory.path = new_dir; } StandardBufferManager::StandardBufferManager(DatabaseInstance &db, string tmp) - : BufferManager(), db(db), buffer_pool(db.GetBufferPool()), temp_directory(std::move(tmp)), - temporary_id(MAXIMUM_BLOCK), buffer_allocator(BufferAllocatorAllocate, BufferAllocatorFree, - BufferAllocatorRealloc, make_uniq(*this)) { + : BufferManager(), db(db), buffer_pool(db.GetBufferPool()), temporary_id(MAXIMUM_BLOCK), + buffer_allocator(BufferAllocatorAllocate, BufferAllocatorFree, BufferAllocatorRealloc, + make_uniq(*this)) { + temporary_directory.path = std::move(tmp); temp_block_manager = make_uniq(*this); for (idx_t i = 0; i < MEMORY_TAG_COUNT; i++) { evicted_data_per_tag[i] = 0; @@ -70,6 +72,22 @@ idx_t StandardBufferManager::GetMaxMemory() const { return buffer_pool.GetMaxMemory(); } +idx_t StandardBufferManager::GetUsedSwap() { + lock_guard guard(temporary_directory.lock); + if (!temporary_directory.handle) { + return 0; + } + return temporary_directory.handle->GetTempFile().GetTotalUsedSpaceInBytes(); +} + +optional_idx StandardBufferManager::GetMaxSwap() { + lock_guard guard(temporary_directory.lock); + if (!temporary_directory.handle) { + return optional_idx(); + } + return temporary_directory.handle->GetTempFile().GetMaxSwapSpace(); +} + template TempBufferPoolReservation StandardBufferManager::EvictBlocksOrThrow(MemoryTag tag, idx_t memory_delta, unique_ptr *buffer, ARGS... args) { @@ -232,10 +250,19 @@ void StandardBufferManager::Unpin(shared_ptr &handle) { } } -void StandardBufferManager::SetLimit(idx_t limit) { +void StandardBufferManager::SetMemoryLimit(idx_t limit) { buffer_pool.SetLimit(limit, InMemoryWarning()); } +void StandardBufferManager::SetSwapLimit(optional_idx limit) { + lock_guard guard(temporary_directory.lock); + if (temporary_directory.handle) { + temporary_directory.handle->GetTempFile().SetMaxSwapSpace(limit); + } else { + temporary_directory.maximum_swap_space = limit; + } +} + vector StandardBufferManager::GetMemoryUsageInfo() const { vector result; for (idx_t k = 0; k < MEMORY_TAG_COUNT; k++) { @@ -259,19 +286,20 @@ unique_ptr StandardBufferManager::ReadTemporaryBufferInternal(Buffer string StandardBufferManager::GetTemporaryPath(block_id_t id) { auto &fs = FileSystem::GetFileSystem(db); - return fs.JoinPath(temp_directory, "duckdb_temp_block-" + to_string(id) + ".block"); + return fs.JoinPath(temporary_directory.path, "duckdb_temp_block-" + to_string(id) + ".block"); } void StandardBufferManager::RequireTemporaryDirectory() { - if (temp_directory.empty()) { + if (temporary_directory.path.empty()) { throw InvalidInputException( "Out-of-memory: cannot write buffer because no temporary directory is specified!\nTo enable " "temporary buffer eviction set a temporary directory using PRAGMA temp_directory='/path/to/tmp.tmp'"); } - lock_guard temp_handle_guard(temp_handle_lock); - if (!temp_directory_handle) { + lock_guard guard(temporary_directory.lock); + if (!temporary_directory.handle) { // temp directory has not been created yet: initialize it - temp_directory_handle = make_uniq(db, temp_directory); + temporary_directory.handle = + make_uniq(db, temporary_directory.path, temporary_directory.maximum_swap_space); } } @@ -279,7 +307,7 @@ void StandardBufferManager::WriteTemporaryBuffer(MemoryTag tag, block_id_t block RequireTemporaryDirectory(); if (buffer.size == Storage::BLOCK_SIZE) { evicted_data_per_tag[uint8_t(tag)] += Storage::BLOCK_SIZE; - temp_directory_handle->GetTempFile().WriteTemporaryBuffer(block_id, buffer); + temporary_directory.handle->GetTempFile().WriteTemporaryBuffer(block_id, buffer); return; } evicted_data_per_tag[uint8_t(tag)] += buffer.size; @@ -295,11 +323,11 @@ void StandardBufferManager::WriteTemporaryBuffer(MemoryTag tag, block_id_t block unique_ptr StandardBufferManager::ReadTemporaryBuffer(MemoryTag tag, block_id_t id, unique_ptr reusable_buffer) { - D_ASSERT(!temp_directory.empty()); - D_ASSERT(temp_directory_handle.get()); - if (temp_directory_handle->GetTempFile().HasTemporaryBuffer(id)) { + D_ASSERT(!temporary_directory.path.empty()); + D_ASSERT(temporary_directory.handle.get()); + if (temporary_directory.handle->GetTempFile().HasTemporaryBuffer(id)) { evicted_data_per_tag[uint8_t(tag)] -= Storage::BLOCK_SIZE; - return temp_directory_handle->GetTempFile().ReadTemporaryBuffer(id, std::move(reusable_buffer)); + return temporary_directory.handle->GetTempFile().ReadTemporaryBuffer(id, std::move(reusable_buffer)); } idx_t block_size; // open the temporary file and read the size @@ -318,20 +346,20 @@ unique_ptr StandardBufferManager::ReadTemporaryBuffer(MemoryTag tag, } void StandardBufferManager::DeleteTemporaryFile(block_id_t id) { - if (temp_directory.empty()) { + if (temporary_directory.path.empty()) { // no temporary directory specified: nothing to delete return; } { - lock_guard temp_handle_guard(temp_handle_lock); - if (!temp_directory_handle) { + lock_guard guard(temporary_directory.lock); + if (!temporary_directory.handle) { // temporary directory was not initialized yet: nothing to delete return; } } // check if we should delete the file from the shared pool of files, or from the general file system - if (temp_directory_handle->GetTempFile().HasTemporaryBuffer(id)) { - temp_directory_handle->GetTempFile().DeleteTemporaryBuffer(id); + if (temporary_directory.handle->GetTempFile().HasTemporaryBuffer(id)) { + temporary_directory.handle->GetTempFile().DeleteTemporaryBuffer(id); return; } auto &fs = FileSystem::GetFileSystem(db); @@ -342,22 +370,22 @@ void StandardBufferManager::DeleteTemporaryFile(block_id_t id) { } bool StandardBufferManager::HasTemporaryDirectory() const { - return !temp_directory.empty(); + return !temporary_directory.path.empty(); } vector StandardBufferManager::GetTemporaryFiles() { vector result; - if (temp_directory.empty()) { + if (temporary_directory.path.empty()) { return result; } { - lock_guard temp_handle_guard(temp_handle_lock); - if (temp_directory_handle) { - result = temp_directory_handle->GetTempFile().GetTemporaryFiles(); + lock_guard temp_handle_guard(temporary_directory.lock); + if (temporary_directory.handle) { + result = temporary_directory.handle->GetTempFile().GetTemporaryFiles(); } } auto &fs = FileSystem::GetFileSystem(db); - fs.ListFiles(temp_directory, [&](const string &name, bool is_dir) { + fs.ListFiles(temporary_directory.path, [&](const string &name, bool is_dir) { if (is_dir) { return; } @@ -375,7 +403,7 @@ vector StandardBufferManager::GetTemporaryFiles() { } const char *StandardBufferManager::InMemoryWarning() { - if (!temp_directory.empty()) { + if (!temporary_directory.path.empty()) { return ""; } return "\nDatabase is launched in in-memory mode and no temporary directory is specified." diff --git a/src/storage/temporary_file_manager.cpp b/src/storage/temporary_file_manager.cpp index d4fa09470596..d16cc23f1d0f 100644 --- a/src/storage/temporary_file_manager.cpp +++ b/src/storage/temporary_file_manager.cpp @@ -189,18 +189,14 @@ idx_t TemporaryFileHandle::GetPositionInFile(idx_t index) { // TemporaryDirectoryHandle //===--------------------------------------------------------------------===// -TemporaryDirectoryHandle::TemporaryDirectoryHandle(DatabaseInstance &db, string path_p) - : db(db), temp_directory(std::move(path_p)), temp_file(make_uniq(db, temp_directory)) { +TemporaryDirectoryHandle::TemporaryDirectoryHandle(DatabaseInstance &db, string path_p, optional_idx max_swap_space) + : db(db), temp_directory(std::move(path_p)), + temp_file(make_uniq(db, temp_directory, max_swap_space)) { auto &fs = FileSystem::GetFileSystem(db); if (!temp_directory.empty()) { if (!fs.DirectoryExists(temp_directory)) { - auto &config = DBConfig::GetConfig(db); fs.CreateDirectory(temp_directory); created_directory = true; - // Maximum swap space isn't set explicitly, initialize to default - if (!config.options.maximum_swap_space.ExplicitlySet()) { - config.SetDefaultMaxSwapSpace(&db); - } } } } @@ -258,8 +254,23 @@ bool TemporaryFileIndex::IsValid() const { // TemporaryFileManager //===--------------------------------------------------------------------===// -TemporaryFileManager::TemporaryFileManager(DatabaseInstance &db, const string &temp_directory_p) - : db(db), temp_directory(temp_directory_p), size_on_disk(0) { +static idx_t GetDefaultMax(const string &path) { + // Use the available disk space + auto disk_space = FileSystem::GetAvailableDiskSpace(path); + idx_t default_value = 0; + if (disk_space.IsValid()) { + // Only use 90% of the available disk space + default_value = static_cast(static_cast(disk_space.GetIndex()) * 0.9); + } + return default_value; +} + +TemporaryFileManager::TemporaryFileManager(DatabaseInstance &db, const string &temp_directory_p, + optional_idx max_swap_space) + : db(db), temp_directory(temp_directory_p), size_on_disk(0), max_swap_space(GetDefaultMax(temp_directory_p)) { + if (max_swap_space.IsValid()) { + this->max_swap_space = max_swap_space.GetIndex(); + } } TemporaryFileManager::~TemporaryFileManager() { @@ -311,10 +322,19 @@ idx_t TemporaryFileManager::GetTotalUsedSpaceInBytes() { return size_on_disk.load(); } -void TemporaryFileManager::IncreaseSizeOnDisk(idx_t bytes) { - auto &config = DBConfig::GetConfig(db); - auto max_swap_space = config.options.maximum_swap_space; +optional_idx TemporaryFileManager::GetMaxSwapSpace() const { + return max_swap_space; +} +void TemporaryFileManager::SetMaxSwapSpace(optional_idx limit) { + if (limit.IsValid()) { + max_swap_space = limit.GetIndex(); + } else { + max_swap_space = GetDefaultMax(temp_directory); + } +} + +void TemporaryFileManager::IncreaseSizeOnDisk(idx_t bytes) { auto current_size_on_disk = size_on_disk.load(); if (current_size_on_disk + bytes > max_swap_space) { auto used = StringUtil::BytesToHumanReadableString(current_size_on_disk); diff --git a/test/sql/storage/temp_directory/max_swap_space_inmemory.test b/test/sql/storage/temp_directory/max_swap_space_inmemory.test index e6aad865c921..f02c6fada818 100644 --- a/test/sql/storage/temp_directory/max_swap_space_inmemory.test +++ b/test/sql/storage/temp_directory/max_swap_space_inmemory.test @@ -44,10 +44,11 @@ select current_setting('max_temp_directory_size') statement ok set temp_directory = '__TEST_DIR__'; -# '__TEST_DIR__' is guaranteed to exist, we can get the disk space +# even though '__TEST_DIR__' exists, we haven't used the temporary directory so the max size is still 0 query I -select current_setting('max_temp_directory_size') a where a == '0 bytes' +select current_setting('max_temp_directory_size') ---- +0 bytes # --- Set explicitly by the user --- diff --git a/test/sql/storage/temp_directory/max_swap_space_persistent.test b/test/sql/storage/temp_directory/max_swap_space_persistent.test index aa18759f7a83..3b4093948199 100644 --- a/test/sql/storage/temp_directory/max_swap_space_persistent.test +++ b/test/sql/storage/temp_directory/max_swap_space_persistent.test @@ -55,10 +55,11 @@ select current_setting('max_temp_directory_size') statement ok set temp_directory = '__TEST_DIR__'; -# '__TEST_DIR__' is guaranteed to exist, we can get the disk space +# Even though __TEST_DIR__ exists, the handle is not created so the size is still 0 (unknown) query I -select current_setting('max_temp_directory_size') a where a == '0 bytes' +select current_setting('max_temp_directory_size') ---- +0 bytes # --- Set explicitly by the user --- From 17086122b885319f0359dc4bc7d08aff8189ec75 Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 21 Mar 2024 16:10:40 +0100 Subject: [PATCH 092/603] get rid of FileSizeMonitor, just pass along the TemporaryFileManager & --- .../duckdb/storage/temporary_file_manager.hpp | 9 ++- src/storage/temporary_file_manager.cpp | 56 +++++++++---------- 2 files changed, 31 insertions(+), 34 deletions(-) diff --git a/src/include/duckdb/storage/temporary_file_manager.hpp b/src/include/duckdb/storage/temporary_file_manager.hpp index 9ab77e42e958..f6a8a17e81a0 100644 --- a/src/include/duckdb/storage/temporary_file_manager.hpp +++ b/src/include/duckdb/storage/temporary_file_manager.hpp @@ -33,16 +33,14 @@ struct FileSizeMonitor { FileSizeMonitor(TemporaryFileManager &manager); public: - void Increase(idx_t blocks); - void Decrease(idx_t blocks); - private: TemporaryFileManager &manager; }; struct BlockIndexManager { public: - BlockIndexManager(unique_ptr file_size_monitor = nullptr); + BlockIndexManager(TemporaryFileManager &manager); + BlockIndexManager(); public: //! Obtains a new block index from the index manager @@ -54,13 +52,14 @@ struct BlockIndexManager { bool HasFreeBlocks(); private: + void SetMaxIndex(idx_t blocks); idx_t GetNewBlockIndexInternal(); private: idx_t max_index; set free_indexes; set indexes_in_use; - unique_ptr file_size_monitor; + optional_ptr manager; }; //===--------------------------------------------------------------------===// diff --git a/src/storage/temporary_file_manager.cpp b/src/storage/temporary_file_manager.cpp index d16cc23f1d0f..463210a405e7 100644 --- a/src/storage/temporary_file_manager.cpp +++ b/src/storage/temporary_file_manager.cpp @@ -5,28 +5,13 @@ namespace duckdb { //===--------------------------------------------------------------------===// -// FileSizeMonitor +// BlockIndexManager //===--------------------------------------------------------------------===// -FileSizeMonitor::FileSizeMonitor(TemporaryFileManager &manager) : manager(manager) { -} - -void FileSizeMonitor::Increase(idx_t blocks) { - auto size_on_disk = blocks * TEMPFILE_BLOCK_SIZE; - manager.IncreaseSizeOnDisk(size_on_disk); -} - -void FileSizeMonitor::Decrease(idx_t blocks) { - auto size_on_disk = blocks * TEMPFILE_BLOCK_SIZE; - manager.DecreaseSizeOnDisk(size_on_disk); +BlockIndexManager::BlockIndexManager(TemporaryFileManager &manager) : max_index(0), manager(&manager) { } -//===--------------------------------------------------------------------===// -// BlockIndexManager -//===--------------------------------------------------------------------===// - -BlockIndexManager::BlockIndexManager(unique_ptr file_size_monitor) - : max_index(0), file_size_monitor(std::move(file_size_monitor)) { +BlockIndexManager::BlockIndexManager() : max_index(0), manager(nullptr) { } idx_t BlockIndexManager::GetNewBlockIndex() { @@ -45,17 +30,12 @@ bool BlockIndexManager::RemoveIndex(idx_t index) { free_indexes.insert(index); // check if we can truncate the file - auto old_max = max_index; - // get the max_index in use right now auto max_index_in_use = indexes_in_use.empty() ? 0 : *indexes_in_use.rbegin() + 1; if (max_index_in_use < max_index) { // max index in use is lower than the max_index // reduce the max_index - max_index = max_index_in_use; - if (file_size_monitor) { - file_size_monitor->Decrease(old_max - max_index); - } + SetMaxIndex(max_index_in_use); // we can remove any free_indexes that are larger than the current max_index while (!free_indexes.empty()) { auto max_entry = *free_indexes.rbegin(); @@ -77,13 +57,31 @@ bool BlockIndexManager::HasFreeBlocks() { return !free_indexes.empty(); } +void BlockIndexManager::SetMaxIndex(idx_t new_index) { + static constexpr idx_t TEMPFILE_BLOCK_SIZE = Storage::BLOCK_ALLOC_SIZE; + if (!manager) { + max_index = new_index; + } else { + auto old = max_index; + if (new_index < old) { + max_index = new_index; + auto difference = old - new_index; + auto size_on_disk = difference * TEMPFILE_BLOCK_SIZE; + manager->DecreaseSizeOnDisk(size_on_disk); + } else if (new_index > old) { + auto difference = new_index - old; + auto size_on_disk = difference * TEMPFILE_BLOCK_SIZE; + manager->IncreaseSizeOnDisk(size_on_disk); + // Increase can throw, so this is only updated after it was succesfully updated + max_index = new_index; + } + } +} + idx_t BlockIndexManager::GetNewBlockIndexInternal() { if (free_indexes.empty()) { auto new_index = max_index; - if (file_size_monitor) { - file_size_monitor->Increase(1); - } - max_index++; + SetMaxIndex(max_index + 1); return new_index; } auto entry = free_indexes.begin(); @@ -100,7 +98,7 @@ TemporaryFileHandle::TemporaryFileHandle(idx_t temp_file_count, DatabaseInstance idx_t index, TemporaryFileManager &manager) : max_allowed_index((1 << temp_file_count) * MAX_ALLOWED_INDEX_BASE), db(db), file_index(index), path(FileSystem::GetFileSystem(db).JoinPath(temp_directory, "duckdb_temp_storage-" + to_string(index) + ".tmp")), - index_manager(make_uniq(manager)) { + index_manager(manager) { } TemporaryFileHandle::TemporaryFileLock::TemporaryFileLock(mutex &mutex) : lock(mutex) { From 03fc90e02bfa71c2900cedc43a2c6038bd7c7532 Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 21 Mar 2024 16:13:25 +0100 Subject: [PATCH 093/603] remove named connection, should be stripped when it gets into the database instance constructor --- test/api/test_api.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/api/test_api.cpp b/test/api/test_api.cpp index 7302627eb8cd..23d4bb3d02ef 100644 --- a/test/api/test_api.cpp +++ b/test/api/test_api.cpp @@ -140,7 +140,7 @@ static void parallel_query(Connection *conn, bool *correct, size_t threadnr) { } TEST_CASE("Test temp_directory defaults", "[api][.]") { - const char *db_paths[] = {nullptr, "", ":memory:", ":memory:named_conn"}; + const char *db_paths[] = {nullptr, "", ":memory:"}; for (auto &path : db_paths) { auto db = make_uniq(path); auto conn = make_uniq(*db); From eebc24c215ada39552a52041c48761914ebc92c5 Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 21 Mar 2024 17:30:58 +0100 Subject: [PATCH 094/603] test error when setting a limit that's too low --- src/main/settings/settings.cpp | 3 ++- src/storage/temporary_file_manager.cpp | 18 +++++++++++-- .../temp_directory/max_swap_space_error.test | 25 +++++++++++++++++++ 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/src/main/settings/settings.cpp b/src/main/settings/settings.cpp index 21ef43e50ee9..ffbfcf7322d8 100644 --- a/src/main/settings/settings.cpp +++ b/src/main/settings/settings.cpp @@ -980,8 +980,8 @@ void MaximumTempDirectorySize::SetGlobal(DatabaseInstance *db, DBConfig &config, if (input.ToString() != "-1") { maximum_swap_space = DBConfig::ParseMemoryLimit(input.ToString()); } - config.options.maximum_swap_space = maximum_swap_space; if (!db) { + config.options.maximum_swap_space = maximum_swap_space; return; } auto &buffer_manager = BufferManager::GetBufferManager(*db); @@ -990,6 +990,7 @@ void MaximumTempDirectorySize::SetGlobal(DatabaseInstance *db, DBConfig &config, } else { buffer_manager.SetSwapLimit(maximum_swap_space); } + config.options.maximum_swap_space = maximum_swap_space; } void MaximumTempDirectorySize::ResetGlobal(DatabaseInstance *db, DBConfig &config) { diff --git a/src/storage/temporary_file_manager.cpp b/src/storage/temporary_file_manager.cpp index 463210a405e7..c4b8643d4e84 100644 --- a/src/storage/temporary_file_manager.cpp +++ b/src/storage/temporary_file_manager.cpp @@ -325,11 +325,25 @@ optional_idx TemporaryFileManager::GetMaxSwapSpace() const { } void TemporaryFileManager::SetMaxSwapSpace(optional_idx limit) { + idx_t new_limit; if (limit.IsValid()) { - max_swap_space = limit.GetIndex(); + new_limit = limit.GetIndex(); } else { - max_swap_space = GetDefaultMax(temp_directory); + new_limit = GetDefaultMax(temp_directory); } + + auto current_size_on_disk = size_on_disk.load(); + if (current_size_on_disk > new_limit) { + auto used = StringUtil::BytesToHumanReadableString(current_size_on_disk); + auto max = StringUtil::BytesToHumanReadableString(new_limit); + throw OutOfMemoryException( + R"(failed to adjust the 'max_temp_directory_size', currently used space (%s) exceeds the new limit (%s) +Please increase the limit or destroy the buffers stored in the temp directory by e.g removing temporary tables. +To get usage information of the temp_directory, use 'CALL duckdb_temporary_files();' + )", + used, max); + } + max_swap_space = new_limit; } void TemporaryFileManager::IncreaseSizeOnDisk(idx_t bytes) { diff --git a/test/sql/storage/temp_directory/max_swap_space_error.test b/test/sql/storage/temp_directory/max_swap_space_error.test index 6116c552ea7c..d9512df2230d 100644 --- a/test/sql/storage/temp_directory/max_swap_space_error.test +++ b/test/sql/storage/temp_directory/max_swap_space_error.test @@ -70,3 +70,28 @@ query I select "size" from duckdb_temporary_files() ---- 1572864 + +# Lower the limit +statement error +set max_temp_directory_size='256KiB' +---- +failed to adjust the 'max_temp_directory_size', currently used space (1.5 MiB) exceeds the new limit (256.0 KiB) + +# Lower the limit +statement error +set max_temp_directory_size='256KiB' +---- +failed to adjust the 'max_temp_directory_size', currently used space (1.5 MiB) exceeds the new limit (256.0 KiB) + +query I +select current_setting('max_temp_directory_size') +---- +1.5 MiB + +statement ok +set max_temp_directory_size='2550KiB' + +query I +select current_setting('max_temp_directory_size') +---- +2.4 MiB From fcc1af60ca3b286dbf6cdcdceb4de060ff3e36e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Fri, 22 Mar 2024 14:53:42 +0100 Subject: [PATCH 095/603] first batch of changes to comply with -Wsign-conversion -Wsign-compare --- extension/parquet/column_writer.cpp | 2 +- src/CMakeLists.txt | 2 +- src/common/types/cast_helpers.cpp | 2 +- src/core_functions/lambda_functions.cpp | 4 ++-- src/function/function.cpp | 4 ++-- .../duckdb/common/multi_file_reader.hpp | 4 ++-- src/include/duckdb/common/string_util.hpp | 15 ++++++------ .../duckdb/common/types/cast_helpers.hpp | 23 +++++++++---------- src/include/duckdb/common/types/datetime.hpp | 6 ++--- src/include/duckdb/common/types/hash.hpp | 2 +- src/include/duckdb/common/vector.hpp | 7 ++++++ src/include/duckdb/function/scalar/regexp.hpp | 2 +- .../duckdb/optimizer/matcher/set_matcher.hpp | 3 ++- .../duckdb/storage/buffer/block_handle.hpp | 2 +- src/optimizer/common_aggregate_optimizer.cpp | 2 +- src/optimizer/filter_combiner.cpp | 6 ++--- .../join_order/cardinality_estimator.cpp | 2 +- src/optimizer/join_order/plan_enumerator.cpp | 4 ++-- .../join_order/query_graph_manager.cpp | 2 +- src/optimizer/pushdown/pushdown_aggregate.cpp | 2 +- src/optimizer/pushdown/pushdown_left_join.cpp | 2 +- src/optimizer/pushdown/pushdown_mark_join.cpp | 6 ++--- .../pushdown/pushdown_single_join.cpp | 2 +- src/optimizer/remove_duplicate_groups.cpp | 2 +- src/optimizer/remove_unused_columns.cpp | 2 +- .../rule/arithmetic_simplification.cpp | 2 +- src/optimizer/rule/case_simplification.cpp | 4 ++-- .../rule/conjunction_simplification.cpp | 2 +- src/optimizer/rule/distributivity.cpp | 2 +- .../expression/propagate_conjunction.cpp | 2 +- .../expression/propagate_operator.cpp | 8 +++---- .../statistics/operator/propagate_filter.cpp | 6 ++--- .../statistics/operator/propagate_get.cpp | 2 +- .../statistics/operator/propagate_join.cpp | 5 ++-- .../operator/propagate_set_operation.cpp | 3 ++- src/optimizer/topn_optimizer.cpp | 2 +- src/parser/parser.cpp | 9 ++++---- .../expression/transform_boolean_test.cpp | 4 ++-- .../transform/expression/transform_cast.cpp | 2 +- .../expression/transform_param_ref.cpp | 2 +- .../transform_positional_reference.cpp | 2 +- .../transform/helpers/transform_typename.cpp | 10 ++++---- src/parser/transformer.cpp | 4 ++-- .../binder/statement/bind_copy_database.cpp | 2 +- src/planner/binder/statement/bind_export.cpp | 2 +- src/planner/binder/statement/bind_insert.cpp | 2 +- src/planner/binder/tableref/bind_pivot.cpp | 6 ++--- src/planner/bound_result_modifier.cpp | 2 +- src/planner/operator/logical_top_n.cpp | 2 +- src/planner/table_binding.cpp | 2 +- src/storage/data_table.cpp | 2 +- src/storage/local_storage.cpp | 2 +- src/storage/table/row_group_collection.cpp | 2 +- src/storage/table_index_list.cpp | 2 +- src/transaction/duck_transaction_manager.cpp | 2 +- src/transaction/meta_transaction.cpp | 2 +- third_party/utf8proc/include/utf8proc.hpp | 2 +- 57 files changed, 112 insertions(+), 101 deletions(-) diff --git a/extension/parquet/column_writer.cpp b/extension/parquet/column_writer.cpp index 47ff6c93d6e9..c90ab1f5b26d 100644 --- a/extension/parquet/column_writer.cpp +++ b/extension/parquet/column_writer.cpp @@ -477,7 +477,7 @@ void BasicColumnWriter::BeginWrite(ColumnWriterState &state_p) { auto &page_info = state.page_info[page_idx]; if (page_info.row_count == 0) { D_ASSERT(page_idx + 1 == state.page_info.size()); - state.page_info.erase(state.page_info.begin() + page_idx); + state.page_info.erase_at(page_idx); break; } PageWriteInformation write_info; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d45ae7fb65fd..4c69853abd3c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -24,7 +24,7 @@ if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") set(EXIT_TIME_DESTRUCTORS_WARNING TRUE) set(CMAKE_CXX_FLAGS_DEBUG - "${CMAKE_CXX_FLAGS_DEBUG} -Wexit-time-destructors -Wimplicit-int-conversion -Wshorten-64-to-32 -Wnarrowing" + "${CMAKE_CXX_FLAGS_DEBUG} -Wexit-time-destructors -Wimplicit-int-conversion -Wshorten-64-to-32 -Wnarrowing -Wsign-conversion -Wsign-compare" ) endif() diff --git a/src/common/types/cast_helpers.cpp b/src/common/types/cast_helpers.cpp index f37fbaa971e9..1e37fbc79ea6 100644 --- a/src/common/types/cast_helpers.cpp +++ b/src/common/types/cast_helpers.cpp @@ -67,7 +67,7 @@ int NumericHelper::UnsignedLength(uint32_t value) { } template <> -int NumericHelper::UnsignedLength(uint64_t value) { +idx_t NumericHelper::UnsignedLength(uint64_t value) { if (value >= 10000000000ULL) { if (value >= 1000000000000000ULL) { int length = 16; diff --git a/src/core_functions/lambda_functions.cpp b/src/core_functions/lambda_functions.cpp index 3b67f8809305..ee78be581cd7 100644 --- a/src/core_functions/lambda_functions.cpp +++ b/src/core_functions/lambda_functions.cpp @@ -154,7 +154,7 @@ struct ListFilterFunctor { // slice the input chunk's corresponding vector to get the new lists // and append them to the result - auto source_list_idx = execute_info.has_index ? 1 : 0; + idx_t source_list_idx = execute_info.has_index ? 1 : 0; Vector result_lists(execute_info.input_chunk.data[source_list_idx], sel, count); ListVector::Append(result, result_lists, count, 0); } @@ -353,7 +353,7 @@ void ExecuteLambda(DataChunk &args, ExpressionState &state, Vector &result) { // set the index vector if (info.has_index) { - index_vector.SetValue(elem_cnt, Value::BIGINT(child_idx + 1)); + index_vector.SetValue(elem_cnt, Value::BIGINT(NumericCast(child_idx + 1))); } elem_cnt++; diff --git a/src/function/function.cpp b/src/function/function.cpp index 3d76a7257afc..9427f445b740 100644 --- a/src/function/function.cpp +++ b/src/function/function.cpp @@ -158,8 +158,8 @@ void Function::EraseArgument(SimpleFunction &bound_function, vectorGetFileName()); if (entry == file_set.end()) { - data.union_readers.erase(data.union_readers.begin() + r); + data.union_readers.erase_at(r); r--; continue; } diff --git a/src/include/duckdb/common/string_util.hpp b/src/include/duckdb/common/string_util.hpp index e5eb9cd6269a..0a2335e7ed81 100644 --- a/src/include/duckdb/common/string_util.hpp +++ b/src/include/duckdb/common/string_util.hpp @@ -10,8 +10,9 @@ #include "duckdb/common/constants.hpp" #include "duckdb/common/exception.hpp" -#include "duckdb/common/vector.hpp" +#include "duckdb/common/numeric_utils.hpp" #include "duckdb/common/set.hpp" +#include "duckdb/common/vector.hpp" #include @@ -40,22 +41,22 @@ class StringUtil { static uint8_t GetHexValue(char c) { if (c >= '0' && c <= '9') { - return c - '0'; + return UnsafeNumericCast(c - '0'); } if (c >= 'a' && c <= 'f') { - return c - 'a' + 10; + return UnsafeNumericCast(c - 'a' + 10); } if (c >= 'A' && c <= 'F') { - return c - 'A' + 10; + return UnsafeNumericCast(c - 'A' + 10); } - throw InvalidInputException("Invalid input for hex digit: %s", string(c, 1)); + throw InvalidInputException("Invalid input for hex digit: %s", string(1, c)); } static uint8_t GetBinaryValue(char c) { if (c >= '0' && c <= '1') { - return c - '0'; + return UnsafeNumericCast(c - '0'); } - throw InvalidInputException("Invalid input for binary digit: %s", string(c, 1)); + throw InvalidInputException("Invalid input for binary digit: %s", string(1, c)); } static bool CharacterIsSpace(char c) { diff --git a/src/include/duckdb/common/types/cast_helpers.hpp b/src/include/duckdb/common/types/cast_helpers.hpp index cac8803f3cb4..5e38665a4d37 100644 --- a/src/include/duckdb/common/types/cast_helpers.hpp +++ b/src/include/duckdb/common/types/cast_helpers.hpp @@ -64,7 +64,7 @@ class NumericHelper { int sign = -(value < 0); UNSIGNED unsigned_value = UnsafeNumericCast(UNSIGNED(value ^ sign) - sign); int length = UnsignedLength(unsigned_value) - sign; - string_t result = StringVector::EmptyString(vector, length); + string_t result = StringVector::EmptyString(vector, NumericCast(length)); auto dataptr = result.GetDataWriteable(); auto endptr = dataptr + length; endptr = FormatUnsigned(unsigned_value, endptr); @@ -149,7 +149,7 @@ struct DecimalToString { template static string_t Format(SIGNED value, uint8_t width, uint8_t scale, Vector &vector) { int len = DecimalLength(value, width, scale); - string_t result = StringVector::EmptyString(vector, len); + string_t result = StringVector::EmptyString(vector, NumericCast(len)); FormatDecimal(value, width, scale, result.GetDataWriteable(), len); result.Finalize(); return result; @@ -260,7 +260,7 @@ struct HugeintToStringCast { Hugeint::NegateInPlace(value); } int length = UnsignedLength(value) + negative; - string_t result = StringVector::EmptyString(vector, length); + string_t result = StringVector::EmptyString(vector, NumericCast(length)); auto dataptr = result.GetDataWriteable(); auto endptr = dataptr + length; if (value.upper == 0) { @@ -339,7 +339,7 @@ struct HugeintToStringCast { static string_t FormatDecimal(hugeint_t value, uint8_t width, uint8_t scale, Vector &vector) { int length = DecimalLength(value, width, scale); - string_t result = StringVector::EmptyString(vector, length); + string_t result = StringVector::EmptyString(vector, NumericCast(length)); auto dst = result.GetDataWriteable(); @@ -417,9 +417,9 @@ struct DateToStringCast { struct TimeToStringCast { //! Format microseconds to a buffer of length 6. Returns the number of trailing zeros - static int32_t FormatMicros(uint32_t microseconds, char micro_buffer[]) { + static int32_t FormatMicros(int32_t microseconds, char micro_buffer[]) { char *endptr = micro_buffer + 6; - endptr = NumericHelper::FormatUnsigned(microseconds, endptr); + endptr = NumericHelper::FormatUnsigned(microseconds, endptr); while (endptr > micro_buffer) { *--endptr = '0'; } @@ -448,7 +448,7 @@ struct TimeToStringCast { // we write backwards and pad with zeros to the left // now we figure out how many digits we need to include by looking backwards // and checking how many zeros we encounter - length -= FormatMicros(time[3], micro_buffer); + length -= NumericCast(FormatMicros(time[3], micro_buffer)); } return length; } @@ -485,8 +485,8 @@ struct TimeToStringCast { struct IntervalToStringCast { static void FormatSignedNumber(int64_t value, char buffer[], idx_t &length) { int sign = -(value < 0); - uint64_t unsigned_value = (value ^ sign) - sign; - length += NumericHelper::UnsignedLength(unsigned_value) - sign; + auto unsigned_value = NumericCast((value ^ sign) - sign); + length += NumericCast(NumericHelper::UnsignedLength(unsigned_value) - sign); auto endptr = buffer + length; endptr = NumericHelper::FormatUnsigned(unsigned_value, endptr); if (sign) { @@ -567,9 +567,8 @@ struct IntervalToStringCast { FormatTwoDigits(sec, buffer, length); if (micros != 0) { buffer[length++] = '.'; - auto trailing_zeros = - TimeToStringCast::FormatMicros(UnsafeNumericCast(micros), buffer + length); - length += 6 - trailing_zeros; + auto trailing_zeros = TimeToStringCast::FormatMicros(NumericCast(micros), buffer + length); + length += NumericCast(6 - trailing_zeros); } } else if (length == 0) { // empty interval: default to 00:00:00 diff --git a/src/include/duckdb/common/types/datetime.hpp b/src/include/duckdb/common/types/datetime.hpp index 4a06e1b762fd..a289a9a46971 100644 --- a/src/include/duckdb/common/types/datetime.hpp +++ b/src/include/duckdb/common/types/datetime.hpp @@ -57,10 +57,10 @@ struct dtime_t { // NOLINT return dtime_t(this->micros - micros); }; inline dtime_t operator*(const idx_t &copies) const { - return dtime_t(this->micros * copies); + return dtime_t(this->micros * UnsafeNumericCast(copies)); }; inline dtime_t operator/(const idx_t &copies) const { - return dtime_t(this->micros / copies); + return dtime_t(this->micros / UnsafeNumericCast(copies)); }; inline int64_t operator-(const dtime_t &other) const { return this->micros - other.micros; @@ -149,7 +149,7 @@ template <> struct hash { std::size_t operator()(const duckdb::dtime_tz_t &k) const { using std::hash; - return hash()(k.bits); + return hash()(k.bits); } }; } // namespace std diff --git a/src/include/duckdb/common/types/hash.hpp b/src/include/duckdb/common/types/hash.hpp index eeb849857f69..f43f75473e75 100644 --- a/src/include/duckdb/common/types/hash.hpp +++ b/src/include/duckdb/common/types/hash.hpp @@ -35,7 +35,7 @@ inline hash_t murmurhash32(uint32_t x) { template hash_t Hash(T value) { - return murmurhash32(value); + return murmurhash32(static_cast(value)); } //! Combine two hashes by XORing them diff --git a/src/include/duckdb/common/vector.hpp b/src/include/duckdb/common/vector.hpp index 66a6bc73153a..b634a8583ce3 100644 --- a/src/include/duckdb/common/vector.hpp +++ b/src/include/duckdb/common/vector.hpp @@ -100,6 +100,13 @@ class vector : public std::vector<_Tp, std::allocator<_Tp>> { } return get(original::size() - 1); } + + void erase_at(idx_t idx) { + if (MemorySafety::enabled && idx > original::size()) { + throw InternalException("Can't remove offset %d from vector of size %d", idx, original::size()); + } + original::erase(original::begin() + static_cast(idx)); + } }; template diff --git a/src/include/duckdb/function/scalar/regexp.hpp b/src/include/duckdb/function/scalar/regexp.hpp index 208e033e0904..612c4cae24ff 100644 --- a/src/include/duckdb/function/scalar/regexp.hpp +++ b/src/include/duckdb/function/scalar/regexp.hpp @@ -140,7 +140,7 @@ struct RegexLocalState : public FunctionLocalState { if (extract_all) { auto group_count_p = constant_pattern.NumberOfCapturingGroups(); if (group_count_p != -1) { - group_buffer.Init(group_count_p); + group_buffer.Init(NumericCast(group_count_p)); } } D_ASSERT(info.constant_pattern); diff --git a/src/include/duckdb/optimizer/matcher/set_matcher.hpp b/src/include/duckdb/optimizer/matcher/set_matcher.hpp index a709a1b5c3fa..11a991d2f683 100644 --- a/src/include/duckdb/optimizer/matcher/set_matcher.hpp +++ b/src/include/duckdb/optimizer/matcher/set_matcher.hpp @@ -9,6 +9,7 @@ #pragma once #include "duckdb/common/common.hpp" +#include "duckdb/common/numeric_utils.hpp" #include "duckdb/common/unordered_set.hpp" namespace duckdb { @@ -59,7 +60,7 @@ class SetMatcher { return true; } else { // we did not find a match! remove any bindings we added in the call to Match() - bindings.erase(bindings.begin() + previous_binding_count, bindings.end()); + bindings.erase(bindings.begin() + NumericCast(previous_binding_count), bindings.end()); } } } diff --git a/src/include/duckdb/storage/buffer/block_handle.hpp b/src/include/duckdb/storage/buffer/block_handle.hpp index 196fb27c7654..6c8f554897e6 100644 --- a/src/include/duckdb/storage/buffer/block_handle.hpp +++ b/src/include/duckdb/storage/buffer/block_handle.hpp @@ -76,7 +76,7 @@ class BlockHandle { D_ASSERT(buffer); // resize and adjust current memory buffer->Resize(block_size); - memory_usage += memory_delta; + memory_usage = NumericCast(NumericCast(memory_usage) + memory_delta); D_ASSERT(memory_usage == buffer->AllocSize()); } diff --git a/src/optimizer/common_aggregate_optimizer.cpp b/src/optimizer/common_aggregate_optimizer.cpp index da13092c6e14..435b94cd35aa 100644 --- a/src/optimizer/common_aggregate_optimizer.cpp +++ b/src/optimizer/common_aggregate_optimizer.cpp @@ -47,7 +47,7 @@ void CommonAggregateOptimizer::ExtractCommonAggregates(LogicalAggregate &aggr) { } else { // aggregate already exists! we can remove this entry total_erased++; - aggr.expressions.erase(aggr.expressions.begin() + i); + aggr.expressions.erase_at(i); i--; // we need to remap any references to this aggregate so they point to the other aggregate ColumnBinding original_binding(aggr.aggregate_index, original_index); diff --git a/src/optimizer/filter_combiner.cpp b/src/optimizer/filter_combiner.cpp index cd6f3319134d..97cabfba613a 100644 --- a/src/optimizer/filter_combiner.cpp +++ b/src/optimizer/filter_combiner.cpp @@ -67,7 +67,7 @@ FilterResult FilterCombiner::AddConstantComparison(vector &column_id table_filters.PushFilter(column_index, std::move(upper_bound)); table_filters.PushFilter(column_index, make_uniq()); - remaining_filters.erase(remaining_filters.begin() + rem_fil_idx); + remaining_filters.erase_at(rem_fil_idx); } } @@ -971,7 +971,7 @@ unique_ptr FilterCombiner::FindTransitiveFilter(Expression &expr) { auto &comparison = remaining_filters[i]->Cast(); if (expr.Equals(*comparison.right) && comparison.type != ExpressionType::COMPARE_NOTEQUAL) { auto filter = std::move(remaining_filters[i]); - remaining_filters.erase(remaining_filters.begin() + i); + remaining_filters.erase_at(i); return filter; } } diff --git a/src/optimizer/join_order/cardinality_estimator.cpp b/src/optimizer/join_order/cardinality_estimator.cpp index 18fd5c858386..7a30a8ce573a 100644 --- a/src/optimizer/join_order/cardinality_estimator.cpp +++ b/src/optimizer/join_order/cardinality_estimator.cpp @@ -50,7 +50,7 @@ bool CardinalityEstimator::SingleColumnFilter(FilterInfo &filter_info) { vector CardinalityEstimator::DetermineMatchingEquivalentSets(FilterInfo *filter_info) { vector matching_equivalent_sets; - auto equivalent_relation_index = 0; + idx_t equivalent_relation_index = 0; for (const RelationsToTDom &r2tdom : relations_to_tdoms) { auto &i_set = r2tdom.equivalent_relations; diff --git a/src/optimizer/join_order/plan_enumerator.cpp b/src/optimizer/join_order/plan_enumerator.cpp index a6efb84b8e6e..6bbc13c3c53a 100644 --- a/src/optimizer/join_order/plan_enumerator.cpp +++ b/src/optimizer/join_order/plan_enumerator.cpp @@ -490,8 +490,8 @@ void PlanEnumerator::SolveJoinOrderApproximately() { // important to erase the biggest element first // if we erase the smallest element first the index of the biggest element changes D_ASSERT(best_right > best_left); - join_relations.erase(join_relations.begin() + best_right); - join_relations.erase(join_relations.begin() + best_left); + join_relations.erase_at(best_right); + join_relations.erase_at(best_left); join_relations.push_back(best_connection->set); } } diff --git a/src/optimizer/join_order/query_graph_manager.cpp b/src/optimizer/join_order/query_graph_manager.cpp index b8835265b843..13b7807b8964 100644 --- a/src/optimizer/join_order/query_graph_manager.cpp +++ b/src/optimizer/join_order/query_graph_manager.cpp @@ -116,7 +116,7 @@ static unique_ptr ExtractJoinRelation(unique_ptrop) { // found it! take ownership o/**/f it from the parent auto result = std::move(children[i]); - children.erase(children.begin() + i); + children.erase_at(i); return result; } } diff --git a/src/optimizer/pushdown/pushdown_aggregate.cpp b/src/optimizer/pushdown/pushdown_aggregate.cpp index 396980d54361..47a2b24e9942 100644 --- a/src/optimizer/pushdown/pushdown_aggregate.cpp +++ b/src/optimizer/pushdown/pushdown_aggregate.cpp @@ -87,7 +87,7 @@ unique_ptr FilterPushdown::PushdownAggregate(unique_ptr(std::move(op)); } // erase the filter from here - filters.erase(filters.begin() + i); + filters.erase_at(i); i--; } child_pushdown.GenerateFilters(); diff --git a/src/optimizer/pushdown/pushdown_left_join.cpp b/src/optimizer/pushdown/pushdown_left_join.cpp index 47cfdfd6b639..b4ebd2ef27e3 100644 --- a/src/optimizer/pushdown/pushdown_left_join.cpp +++ b/src/optimizer/pushdown/pushdown_left_join.cpp @@ -91,7 +91,7 @@ unique_ptr FilterPushdown::PushdownLeftJoin(unique_ptr FilterPushdown::PushdownMarkJoin(unique_ptr FilterPushdown::PushdownMarkJoin(unique_ptr FilterPushdown::PushdownMarkJoin(unique_ptr FilterPushdown::PushdownSingleJoin(unique_ptr &list, idx_t table_id auto entry = column_references.find(current_binding); if (entry == column_references.end()) { // this entry is not referred to, erase it from the set of expressions - list.erase(list.begin() + col_idx); + list.erase_at(col_idx); offset++; col_idx--; } else if (offset > 0 && replace) { diff --git a/src/optimizer/rule/arithmetic_simplification.cpp b/src/optimizer/rule/arithmetic_simplification.cpp index b319c60d5713..bd4e0821e2fa 100644 --- a/src/optimizer/rule/arithmetic_simplification.cpp +++ b/src/optimizer/rule/arithmetic_simplification.cpp @@ -26,7 +26,7 @@ unique_ptr ArithmeticSimplificationRule::Apply(LogicalOperator &op, bool &changes_made, bool is_root) { auto &root = bindings[0].get().Cast(); auto &constant = bindings[1].get().Cast(); - int constant_child = root.children[0].get() == &constant ? 0 : 1; + idx_t constant_child = root.children[0].get() == &constant ? 0 : 1; D_ASSERT(root.children.size() == 2); (void)root; // any arithmetic operator involving NULL is always NULL diff --git a/src/optimizer/rule/case_simplification.cpp b/src/optimizer/rule/case_simplification.cpp index 61c6ed353829..2dbbecef56a7 100644 --- a/src/optimizer/rule/case_simplification.cpp +++ b/src/optimizer/rule/case_simplification.cpp @@ -25,14 +25,14 @@ unique_ptr CaseSimplificationRule::Apply(LogicalOperator &op, vector auto condition = constant_value.DefaultCastAs(LogicalType::BOOLEAN); if (condition.IsNull() || !BooleanValue::Get(condition)) { // the condition is always false: remove this case check - root.case_checks.erase(root.case_checks.begin() + i); + root.case_checks.erase_at(i); i--; } else { // the condition is always true // move the THEN clause to the ELSE of the case root.else_expr = std::move(case_check.then_expr); // remove this case check and any case checks after this one - root.case_checks.erase(root.case_checks.begin() + i, root.case_checks.end()); + root.case_checks.erase(root.case_checks.begin() + NumericCast(i), root.case_checks.end()); break; } } diff --git a/src/optimizer/rule/conjunction_simplification.cpp b/src/optimizer/rule/conjunction_simplification.cpp index 070237cb724c..646471b9412e 100644 --- a/src/optimizer/rule/conjunction_simplification.cpp +++ b/src/optimizer/rule/conjunction_simplification.cpp @@ -19,7 +19,7 @@ unique_ptr ConjunctionSimplificationRule::RemoveExpression(BoundConj for (idx_t i = 0; i < conj.children.size(); i++) { if (conj.children[i].get() == &expr) { // erase the expression - conj.children.erase(conj.children.begin() + i); + conj.children.erase_at(i); break; } } diff --git a/src/optimizer/rule/distributivity.cpp b/src/optimizer/rule/distributivity.cpp index 509960c03b0d..b6a889af416d 100644 --- a/src/optimizer/rule/distributivity.cpp +++ b/src/optimizer/rule/distributivity.cpp @@ -35,7 +35,7 @@ unique_ptr DistributivityRule::ExtractExpression(BoundConjunctionExp for (idx_t i = 0; i < and_expr.children.size(); i++) { if (and_expr.children[i]->Equals(expr)) { result = std::move(and_expr.children[i]); - and_expr.children.erase(and_expr.children.begin() + i); + and_expr.children.erase_at(i); break; } } diff --git a/src/optimizer/statistics/expression/propagate_conjunction.cpp b/src/optimizer/statistics/expression/propagate_conjunction.cpp index 1fce16c8cf51..4c69f8a15aa9 100644 --- a/src/optimizer/statistics/expression/propagate_conjunction.cpp +++ b/src/optimizer/statistics/expression/propagate_conjunction.cpp @@ -46,7 +46,7 @@ unique_ptr StatisticsPropagator::PropagateExpression(BoundConjun } } if (prune_child) { - expr.children.erase(expr.children.begin() + expr_idx); + expr.children.erase_at(expr_idx); expr_idx--; continue; } diff --git a/src/optimizer/statistics/expression/propagate_operator.cpp b/src/optimizer/statistics/expression/propagate_operator.cpp index 4bb1f8fd7a32..b48593cd8e33 100644 --- a/src/optimizer/statistics/expression/propagate_operator.cpp +++ b/src/optimizer/statistics/expression/propagate_operator.cpp @@ -28,8 +28,8 @@ unique_ptr StatisticsPropagator::PropagateExpression(BoundOperat // this child is always NULL, we can remove it from the coalesce // UNLESS there is only one node remaining if (expr.children.size() > 1) { - expr.children.erase(expr.children.begin() + i); - child_stats.erase(child_stats.begin() + i); + expr.children.erase_at(i); + child_stats.erase_at(i); i--; } } else if (!child_stats[i]->CanHaveNull()) { @@ -37,8 +37,8 @@ unique_ptr StatisticsPropagator::PropagateExpression(BoundOperat // this is the last coalesce node that influences the result // we can erase any children after this node if (i + 1 < expr.children.size()) { - expr.children.erase(expr.children.begin() + i + 1, expr.children.end()); - child_stats.erase(child_stats.begin() + i + 1, child_stats.end()); + expr.children.erase(expr.children.begin() + NumericCast(i + 1), expr.children.end()); + child_stats.erase(child_stats.begin() + NumericCast(i + 1), child_stats.end()); } break; } diff --git a/src/optimizer/statistics/operator/propagate_filter.cpp b/src/optimizer/statistics/operator/propagate_filter.cpp index 97609fa62888..cff64ebca12e 100644 --- a/src/optimizer/statistics/operator/propagate_filter.cpp +++ b/src/optimizer/statistics/operator/propagate_filter.cpp @@ -227,7 +227,7 @@ unique_ptr StatisticsPropagator::PropagateStatistics(LogicalFilt node_stats = PropagateStatistics(filter.children[0]); if (filter.children[0]->type == LogicalOperatorType::LOGICAL_EMPTY_RESULT) { ReplaceWithEmptyResult(*node_ptr); - return make_uniq(0, 0); + return make_uniq(0U, 0U); } // then propagate to each of the expressions @@ -238,7 +238,7 @@ unique_ptr StatisticsPropagator::PropagateStatistics(LogicalFilt if (ExpressionIsConstant(*condition, Value::BOOLEAN(true))) { // filter is always true; it is useless to execute it // erase this condition - filter.expressions.erase(filter.expressions.begin() + i); + filter.expressions.erase_at(i); i--; if (filter.expressions.empty()) { // just break. The physical filter planner will plan a projection instead @@ -248,7 +248,7 @@ unique_ptr StatisticsPropagator::PropagateStatistics(LogicalFilt ExpressionIsConstantOrNull(*condition, Value::BOOLEAN(false))) { // filter is always false or null; this entire filter should be replaced by an empty result block ReplaceWithEmptyResult(*node_ptr); - return make_uniq(0, 0); + return make_uniq(0U, 0U); } else { // cannot prune this filter: propagate statistics from the filter UpdateFilterStatistics(*condition); diff --git a/src/optimizer/statistics/operator/propagate_get.cpp b/src/optimizer/statistics/operator/propagate_get.cpp index 22979f19450c..48b41c1e0770 100644 --- a/src/optimizer/statistics/operator/propagate_get.cpp +++ b/src/optimizer/statistics/operator/propagate_get.cpp @@ -86,7 +86,7 @@ unique_ptr StatisticsPropagator::PropagateStatistics(LogicalGet case FilterPropagateResult::FILTER_ALWAYS_FALSE: // filter is always false; this entire filter should be replaced by an empty result block ReplaceWithEmptyResult(*node_ptr); - return make_uniq(0, 0); + return make_uniq(0U, 0U); default: // general case: filter can be true or false, update this columns' statistics UpdateFilterStatistics(stats, *filter); diff --git a/src/optimizer/statistics/operator/propagate_join.cpp b/src/optimizer/statistics/operator/propagate_join.cpp index 093a37905270..10fd54a568c2 100644 --- a/src/optimizer/statistics/operator/propagate_join.cpp +++ b/src/optimizer/statistics/operator/propagate_join.cpp @@ -85,7 +85,7 @@ void StatisticsPropagator::PropagateStatistics(LogicalComparisonJoin &join, uniq } if (join.conditions.size() > 1) { // there are multiple conditions: erase this condition - join.conditions.erase(join.conditions.begin() + i); + join.conditions.erase_at(i); // remove the corresponding statistics join.join_stats.clear(); i--; @@ -187,7 +187,8 @@ void StatisticsPropagator::MultiplyCardinalities(unique_ptr &sta return; } stats->estimated_cardinality = MaxValue(stats->estimated_cardinality, new_stats.estimated_cardinality); - auto new_max = Hugeint::Multiply(stats->max_cardinality, new_stats.max_cardinality); + auto new_max = Hugeint::Multiply(NumericCast(stats->max_cardinality), + NumericCast(new_stats.max_cardinality)); if (new_max < NumericLimits::Maximum()) { int64_t result; if (!Hugeint::TryCast(new_max, result)) { diff --git a/src/optimizer/statistics/operator/propagate_set_operation.cpp b/src/optimizer/statistics/operator/propagate_set_operation.cpp index 8b90aeb6346a..e3d1b80a2b3d 100644 --- a/src/optimizer/statistics/operator/propagate_set_operation.cpp +++ b/src/optimizer/statistics/operator/propagate_set_operation.cpp @@ -10,7 +10,8 @@ void StatisticsPropagator::AddCardinalities(unique_ptr &stats, N return; } stats->estimated_cardinality += new_stats.estimated_cardinality; - auto new_max = Hugeint::Add(stats->max_cardinality, new_stats.max_cardinality); + auto new_max = + Hugeint::Add(NumericCast(stats->max_cardinality), NumericCast(new_stats.max_cardinality)); if (new_max < NumericLimits::Maximum()) { int64_t result; if (!Hugeint::TryCast(new_max, result)) { diff --git a/src/optimizer/topn_optimizer.cpp b/src/optimizer/topn_optimizer.cpp index ca64e5700704..26f6ca99506e 100644 --- a/src/optimizer/topn_optimizer.cpp +++ b/src/optimizer/topn_optimizer.cpp @@ -32,7 +32,7 @@ unique_ptr TopN::Optimize(unique_ptr op) { auto limit_val = int64_t(limit.limit_val.GetConstantValue()); int64_t offset_val = 0; if (limit.offset_val.Type() == LimitNodeType::CONSTANT_VALUE) { - offset_val = limit.offset_val.GetConstantValue(); + offset_val = NumericCast(limit.offset_val.GetConstantValue()); } auto topn = make_uniq(std::move(order_by.orders), limit_val, offset_val); topn->AddChild(std::move(order_by.children[0])); diff --git a/src/parser/parser.cpp b/src/parser/parser.cpp index 8274c521d71f..7dd7a43d9f65 100644 --- a/src/parser/parser.cpp +++ b/src/parser/parser.cpp @@ -181,7 +181,7 @@ void Parser::ParseQuery(const string &query) { } else { parser_error = parser.error_message; if (parser.error_location > 0) { - parser_error_location = parser.error_location - 1; + parser_error_location = NumericCast(parser.error_location - 1); } } } @@ -196,7 +196,7 @@ void Parser::ParseQuery(const string &query) { } else { // split sql string into statements and re-parse using extension auto query_statements = SplitQueryStringIntoStatements(query); - auto stmt_loc = 0; + idx_t stmt_loc = 0; for (auto const &query_statement : query_statements) { ErrorData another_parser_error; // Creating a new scope to allow extensions to use PostgresParser, which is not reentrant @@ -219,7 +219,8 @@ void Parser::ParseQuery(const string &query) { } else { another_parser_error = ErrorData(another_parser.error_message); if (another_parser.error_location > 0) { - another_parser_error.AddQueryLocation(another_parser.error_location - 1); + another_parser_error.AddQueryLocation( + NumericCast(another_parser.error_location - 1)); } } } // LCOV_EXCL_STOP @@ -292,7 +293,7 @@ vector Parser::Tokenize(const string &query) { default: throw InternalException("Unrecognized token category"); } // LCOV_EXCL_STOP - token.start = pg_token.start; + token.start = NumericCast(pg_token.start); result.push_back(token); } return result; diff --git a/src/parser/transform/expression/transform_boolean_test.cpp b/src/parser/transform/expression/transform_boolean_test.cpp index ec9dffb794a5..3c96f4dab16f 100644 --- a/src/parser/transform/expression/transform_boolean_test.cpp +++ b/src/parser/transform/expression/transform_boolean_test.cpp @@ -20,9 +20,9 @@ static unique_ptr TransformBooleanTestInternal(unique_ptr TransformBooleanTestIsNull(unique_ptr argument, - ExpressionType operator_type, idx_t query_location) { + ExpressionType operator_type, int query_location) { auto result = make_uniq(operator_type, std::move(argument)); - Transformer::SetQueryLocation(*result, UnsafeNumericCast(query_location)); + Transformer::SetQueryLocation(*result, query_location); return std::move(result); } diff --git a/src/parser/transform/expression/transform_cast.cpp b/src/parser/transform/expression/transform_cast.cpp index c10c81d76404..a4b1dde59bbe 100644 --- a/src/parser/transform/expression/transform_cast.cpp +++ b/src/parser/transform/expression/transform_cast.cpp @@ -18,7 +18,7 @@ unique_ptr Transformer::TransformTypeCast(duckdb_libpgquery::P if (c->val.type == duckdb_libpgquery::T_PGString) { CastParameters parameters; if (root.location >= 0) { - parameters.query_location = root.location; + parameters.query_location = NumericCast(root.location); } auto blob_data = Blob::ToBlob(string(c->val.val.str), parameters); return make_uniq(Value::BLOB_RAW(blob_data)); diff --git a/src/parser/transform/expression/transform_param_ref.cpp b/src/parser/transform/expression/transform_param_ref.cpp index d5d7931fe3e0..2d4ba5155d18 100644 --- a/src/parser/transform/expression/transform_param_ref.cpp +++ b/src/parser/transform/expression/transform_param_ref.cpp @@ -40,7 +40,7 @@ unique_ptr Transformer::TransformParamRef(duckdb_libpgquery::P // We have not seen this parameter before if (node.number != 0) { // Preserve the parameter number - known_param_index = node.number; + known_param_index = NumericCast(node.number); } else { known_param_index = ParamCount() + 1; if (!node.name) { diff --git a/src/parser/transform/expression/transform_positional_reference.cpp b/src/parser/transform/expression/transform_positional_reference.cpp index efe7d3da3321..8d672948db3f 100644 --- a/src/parser/transform/expression/transform_positional_reference.cpp +++ b/src/parser/transform/expression/transform_positional_reference.cpp @@ -8,7 +8,7 @@ unique_ptr Transformer::TransformPositionalReference(duckdb_li if (node.position <= 0) { throw ParserException("Positional reference node needs to be >= 1"); } - auto result = make_uniq(node.position); + auto result = make_uniq(NumericCast(node.position)); SetQueryLocation(*result, node.location); return std::move(result); } diff --git a/src/parser/transform/helpers/transform_typename.cpp b/src/parser/transform/helpers/transform_typename.cpp index ff40554f6084..edea1222fba0 100644 --- a/src/parser/transform/helpers/transform_typename.cpp +++ b/src/parser/transform/helpers/transform_typename.cpp @@ -41,7 +41,7 @@ LogicalType Transformer::TransformTypeName(duckdb_libpgquery::PGTypeName &type_n if (!type_name.typmods || type_name.typmods->length == 0) { throw ParserException("Enum needs a set of entries"); } - Vector enum_vector(LogicalType::VARCHAR, type_name.typmods->length); + Vector enum_vector(LogicalType::VARCHAR, NumericCast(type_name.typmods->length)); auto string_data = FlatVector::GetData(enum_vector); idx_t pos = 0; for (auto node = type_name.typmods->head; node; node = node->next) { @@ -52,7 +52,7 @@ LogicalType Transformer::TransformTypeName(duckdb_libpgquery::PGTypeName &type_n } string_data[pos++] = StringVector::AddString(enum_vector, constant_value->val.val.str); } - return LogicalType::ENUM(enum_vector, type_name.typmods->length); + return LogicalType::ENUM(enum_vector, NumericCast(type_name.typmods->length)); } else if (base_type == LogicalTypeId::STRUCT) { if (!type_name.typmods || type_name.typmods->length == 0) { throw ParserException("Struct needs a name and entries"); @@ -154,12 +154,12 @@ LogicalType Transformer::TransformTypeName(duckdb_libpgquery::PGTypeName &type_n throw ParserException("Negative modifier not supported"); } if (modifier_idx == 0) { - width = const_val.val.val.ival; + width = NumericCast(const_val.val.val.ival); if (base_type == LogicalTypeId::BIT && const_val.location != -1) { width = 0; } } else if (modifier_idx == 1) { - scale = const_val.val.val.ival; + scale = NumericCast(const_val.val.val.ival); } else { throw ParserException("A maximum of two modifiers is supported"); } @@ -245,7 +245,7 @@ LogicalType Transformer::TransformTypeName(duckdb_libpgquery::PGTypeName &type_n if (val->type != duckdb_libpgquery::T_PGInteger) { throw ParserException("Expected integer value as array bound"); } - auto array_size = val->val.ival; + auto array_size = NumericCast(val->val.ival); if (array_size < 0) { // -1 if bounds are empty result_type = LogicalType::LIST(result_type); diff --git a/src/parser/transformer.cpp b/src/parser/transformer.cpp index 8c54c46a3a7f..f12e0d17882b 100644 --- a/src/parser/transformer.cpp +++ b/src/parser/transformer.cpp @@ -134,8 +134,8 @@ unique_ptr Transformer::TransformStatementInternal(duckdb_libpgque auto &raw_stmt = PGCast(stmt); auto result = TransformStatement(*raw_stmt.stmt); if (result) { - result->stmt_location = raw_stmt.stmt_location; - result->stmt_length = raw_stmt.stmt_len; + result->stmt_location = NumericCast(raw_stmt.stmt_location); + result->stmt_length = NumericCast(raw_stmt.stmt_len); } return result; } diff --git a/src/planner/binder/statement/bind_copy_database.cpp b/src/planner/binder/statement/bind_copy_database.cpp index 8a0047da3263..bccb584b5625 100644 --- a/src/planner/binder/statement/bind_copy_database.cpp +++ b/src/planner/binder/statement/bind_copy_database.cpp @@ -135,7 +135,7 @@ unique_ptr Binder::BindCopyDatabaseData(CopyDatabaseStatement & if (result) { // use UNION ALL to combine the individual copy statements into a single node auto copy_union = - make_uniq(GenerateTableIndex(), 1, std::move(insert_plan), std::move(result), + make_uniq(GenerateTableIndex(), 1U, std::move(insert_plan), std::move(result), LogicalOperatorType::LOGICAL_UNION, true, false); result = std::move(copy_union); } else { diff --git a/src/planner/binder/statement/bind_export.cpp b/src/planner/binder/statement/bind_export.cpp index f5370a20a915..b22681ed18df 100644 --- a/src/planner/binder/statement/bind_export.cpp +++ b/src/planner/binder/statement/bind_export.cpp @@ -331,7 +331,7 @@ BoundStatement Binder::Bind(ExportStatement &stmt) { if (child_operator) { // use UNION ALL to combine the individual copy statements into a single node - auto copy_union = make_uniq(GenerateTableIndex(), 1, std::move(child_operator), + auto copy_union = make_uniq(GenerateTableIndex(), 1U, std::move(child_operator), std::move(plan), LogicalOperatorType::LOGICAL_UNION, true); child_operator = std::move(copy_union); } else { diff --git a/src/planner/binder/statement/bind_insert.cpp b/src/planner/binder/statement/bind_insert.cpp index d0827fde2ebb..f9e8e6e2b578 100644 --- a/src/planner/binder/statement/bind_insert.cpp +++ b/src/planner/binder/statement/bind_insert.cpp @@ -30,7 +30,7 @@ namespace duckdb { -static void CheckInsertColumnCountMismatch(int64_t expected_columns, int64_t result_columns, bool columns_provided, +static void CheckInsertColumnCountMismatch(idx_t expected_columns, idx_t result_columns, bool columns_provided, const char *tname) { if (result_columns != expected_columns) { string msg = StringUtil::Format(!columns_provided ? "table %s has %lld columns but %lld values were supplied" diff --git a/src/planner/binder/tableref/bind_pivot.cpp b/src/planner/binder/tableref/bind_pivot.cpp index 6c89b7b2cd91..4d9cf553097b 100644 --- a/src/planner/binder/tableref/bind_pivot.cpp +++ b/src/planner/binder/tableref/bind_pivot.cpp @@ -89,7 +89,7 @@ static unique_ptr ConstructInitialGrouping(PivotRef &ref, vectorgroups.group_expressions.push_back(make_uniq( - Value::INTEGER(UnsafeNumericCast(subquery->select_list.size() + 1)))); + Value::INTEGER(UnsafeNumericCast(subquery->select_list.size() + 1)))); subquery->select_list.push_back(make_uniq(row)); } } @@ -166,7 +166,7 @@ static unique_ptr PivotInitialAggregate(PivotBindState &bind_state, } auto pivot_alias = pivot_expr->alias; subquery_stage1->groups.group_expressions.push_back(make_uniq( - Value::INTEGER(UnsafeNumericCast(subquery_stage1->select_list.size() + 1)))); + Value::INTEGER(UnsafeNumericCast(subquery_stage1->select_list.size() + 1)))); subquery_stage1->select_list.push_back(std::move(pivot_expr)); pivot_expr = make_uniq(std::move(pivot_alias)); } @@ -203,7 +203,7 @@ static unique_ptr PivotListAggregate(PivotBindState &bind_state, Piv // add all of the groups for (idx_t gr = 0; gr < bind_state.internal_group_names.size(); gr++) { subquery_stage2->groups.group_expressions.push_back(make_uniq( - Value::INTEGER(UnsafeNumericCast(subquery_stage2->select_list.size() + 1)))); + Value::INTEGER(UnsafeNumericCast(subquery_stage2->select_list.size() + 1)))); auto group_reference = make_uniq(bind_state.internal_group_names[gr]); group_reference->alias = bind_state.internal_group_names[gr]; subquery_stage2->select_list.push_back(std::move(group_reference)); diff --git a/src/planner/bound_result_modifier.cpp b/src/planner/bound_result_modifier.cpp index 9a1728ed71d0..a4cb6ed4fd23 100644 --- a/src/planner/bound_result_modifier.cpp +++ b/src/planner/bound_result_modifier.cpp @@ -126,7 +126,7 @@ BoundLimitNode::BoundLimitNode() : type(LimitNodeType::UNSET) { } BoundLimitNode::BoundLimitNode(int64_t constant_value) - : type(LimitNodeType::CONSTANT_VALUE), constant_integer(constant_value) { + : type(LimitNodeType::CONSTANT_VALUE), constant_integer(NumericCast(constant_value)) { } BoundLimitNode::BoundLimitNode(double percentage_value) diff --git a/src/planner/operator/logical_top_n.cpp b/src/planner/operator/logical_top_n.cpp index da1fa493f4ec..adf4019e84b0 100644 --- a/src/planner/operator/logical_top_n.cpp +++ b/src/planner/operator/logical_top_n.cpp @@ -5,7 +5,7 @@ namespace duckdb { idx_t LogicalTopN::EstimateCardinality(ClientContext &context) { auto child_cardinality = LogicalOperator::EstimateCardinality(context); if (limit >= 0 && child_cardinality < idx_t(limit)) { - return limit; + return NumericCast(limit); } return child_cardinality; } diff --git a/src/planner/table_binding.cpp b/src/planner/table_binding.cpp index 1709a4524f72..dff3de5952f5 100644 --- a/src/planner/table_binding.cpp +++ b/src/planner/table_binding.cpp @@ -171,7 +171,7 @@ ColumnBinding TableBinding::GetColumnBinding(column_t column_index) { auto it = std::find_if(column_ids.begin(), column_ids.end(), [&](const column_t &id) -> bool { return id == column_index; }); // Get the index of it - binding.column_index = std::distance(column_ids.begin(), it); + binding.column_index = NumericCast(std::distance(column_ids.begin(), it)); // If it wasn't found, add it if (it == column_ids.end()) { column_ids.push_back(column_index); diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 6541bf193434..977ff7387fd8 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -105,7 +105,7 @@ DataTable::DataTable(ClientContext &context, DataTable &parent, idx_t removed_co // erase the column definitions from this DataTable D_ASSERT(removed_column < column_definitions.size()); - column_definitions.erase(column_definitions.begin() + removed_column); + column_definitions.erase_at(removed_column); storage_t storage_idx = 0; for (idx_t i = 0; i < column_definitions.size(); i++) { diff --git a/src/storage/local_storage.cpp b/src/storage/local_storage.cpp index 2c7fb0fe1d79..513a18e2faf1 100644 --- a/src/storage/local_storage.cpp +++ b/src/storage/local_storage.cpp @@ -219,7 +219,7 @@ void LocalTableStorage::FinalizeOptimisticWriter(OptimisticDataWriter &writer) { for (idx_t i = 0; i < optimistic_writers.size(); i++) { if (optimistic_writers[i].get() == &writer) { owned_writer = std::move(optimistic_writers[i]); - optimistic_writers.erase(optimistic_writers.begin() + i); + optimistic_writers.erase_at(i); break; } } diff --git a/src/storage/table/row_group_collection.cpp b/src/storage/table/row_group_collection.cpp index 00e42d5b7b7f..5fae55491020 100644 --- a/src/storage/table/row_group_collection.cpp +++ b/src/storage/table/row_group_collection.cpp @@ -1056,7 +1056,7 @@ shared_ptr RowGroupCollection::AddColumn(ClientContext &cont shared_ptr RowGroupCollection::RemoveColumn(idx_t col_idx) { D_ASSERT(col_idx < types.size()); auto new_types = types; - new_types.erase(new_types.begin() + col_idx); + new_types.erase_at(col_idx); auto result = make_shared(info, block_manager, std::move(new_types), row_start, total_rows.load()); diff --git a/src/storage/table_index_list.cpp b/src/storage/table_index_list.cpp index ef14073f302b..623585baf6e1 100644 --- a/src/storage/table_index_list.cpp +++ b/src/storage/table_index_list.cpp @@ -21,7 +21,7 @@ void TableIndexList::RemoveIndex(const string &name) { for (idx_t index_idx = 0; index_idx < indexes.size(); index_idx++) { auto &index_entry = indexes[index_idx]; if (index_entry->name == name) { - indexes.erase(indexes.begin() + index_idx); + indexes.erase_at(index_idx); break; } } diff --git a/src/transaction/duck_transaction_manager.cpp b/src/transaction/duck_transaction_manager.cpp index 008604c3502c..bf7014c93411 100644 --- a/src/transaction/duck_transaction_manager.cpp +++ b/src/transaction/duck_transaction_manager.cpp @@ -277,7 +277,7 @@ void DuckTransactionManager::RemoveTransaction(DuckTransaction &transaction) noe } } // remove the transaction from the set of currently active transactions - active_transactions.erase(active_transactions.begin() + t_index); + active_transactions.erase_at(t_index); // traverse the recently_committed transactions to see if we can remove any idx_t i = 0; for (; i < recently_committed_transactions.size(); i++) { diff --git a/src/transaction/meta_transaction.cpp b/src/transaction/meta_transaction.cpp index 7cd7fa450b81..6cf12cca5d27 100644 --- a/src/transaction/meta_transaction.cpp +++ b/src/transaction/meta_transaction.cpp @@ -63,7 +63,7 @@ void MetaTransaction::RemoveTransaction(AttachedDatabase &db) { for (idx_t i = 0; i < all_transactions.size(); i++) { auto &db_entry = all_transactions[i]; if (RefersToSameObject(db_entry.get(), db)) { - all_transactions.erase(all_transactions.begin() + i); + all_transactions.erase_at(i); break; } } diff --git a/third_party/utf8proc/include/utf8proc.hpp b/third_party/utf8proc/include/utf8proc.hpp index a7cc6eef73eb..336f95e2ed5a 100644 --- a/third_party/utf8proc/include/utf8proc.hpp +++ b/third_party/utf8proc/include/utf8proc.hpp @@ -637,7 +637,7 @@ void utf8proc_grapheme_callback(const char *s, size_t len, T &&fun) { size_t start = 0; size_t cpos = 0; while(true) { - cpos += sz; + cpos += UnsafeNumericCast(sz); if (cpos >= len) { fun(start, cpos); return; From dc16f563f8718be445c228ceb8900472471529c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Fri, 22 Mar 2024 17:10:06 +0100 Subject: [PATCH 096/603] mc --- src/core_functions/aggregate/distributive/approx_count.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core_functions/aggregate/distributive/approx_count.cpp b/src/core_functions/aggregate/distributive/approx_count.cpp index 844e31acfd9d..6b8dd0d7c667 100644 --- a/src/core_functions/aggregate/distributive/approx_count.cpp +++ b/src/core_functions/aggregate/distributive/approx_count.cpp @@ -43,7 +43,7 @@ struct ApproxCountDistinctFunction { template static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) { if (state.log) { - target = state.log->Count(); + target = UnsafeNumericCast(state.log->Count()); } else { target = 0; } From 5774dc67dad81f4ee5c684730f5fa5b8dee03258 Mon Sep 17 00:00:00 2001 From: Tishj Date: Sat, 23 Mar 2024 20:04:43 +0100 Subject: [PATCH 097/603] delay the available disk space lookup until we have made sure the directory exists --- src/common/file_system.cpp | 7 ++++--- .../duckdb/storage/temporary_file_manager.hpp | 2 +- src/storage/temporary_file_manager.cpp | 12 ++++-------- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/common/file_system.cpp b/src/common/file_system.cpp index c58bb588ee07..11b24c75ae46 100644 --- a/src/common/file_system.cpp +++ b/src/common/file_system.cpp @@ -103,12 +103,13 @@ optional_idx FileSystem::GetAvailableMemory() { optional_idx FileSystem::GetAvailableDiskSpace(const string &path) { struct statvfs vfs; - if (statvfs(path.c_str(), &vfs) == -1) { - optional_idx(); + auto ret = statvfs(path.c_str(), &vfs); + if (ret == -1) { + return optional_idx(); } auto block_size = vfs.f_frsize; // These are the blocks available for creating new files or extending existing ones - auto available_blocks = vfs.f_bavail; + auto available_blocks = vfs.f_bfree; idx_t available_disk_space = DConstants::INVALID_INDEX; if (!TryMultiplyOperator::Operation(static_cast(block_size), static_cast(available_blocks), available_disk_space)) { diff --git a/src/include/duckdb/storage/temporary_file_manager.hpp b/src/include/duckdb/storage/temporary_file_manager.hpp index f6a8a17e81a0..92fcbf8eaba5 100644 --- a/src/include/duckdb/storage/temporary_file_manager.hpp +++ b/src/include/duckdb/storage/temporary_file_manager.hpp @@ -145,7 +145,7 @@ class TemporaryDirectoryHandle { class TemporaryFileManager { public: - TemporaryFileManager(DatabaseInstance &db, const string &temp_directory_p, optional_idx max_swap_space); + TemporaryFileManager(DatabaseInstance &db, const string &temp_directory_p); ~TemporaryFileManager(); public: diff --git a/src/storage/temporary_file_manager.cpp b/src/storage/temporary_file_manager.cpp index c4b8643d4e84..bd4da45dc88e 100644 --- a/src/storage/temporary_file_manager.cpp +++ b/src/storage/temporary_file_manager.cpp @@ -188,8 +188,7 @@ idx_t TemporaryFileHandle::GetPositionInFile(idx_t index) { //===--------------------------------------------------------------------===// TemporaryDirectoryHandle::TemporaryDirectoryHandle(DatabaseInstance &db, string path_p, optional_idx max_swap_space) - : db(db), temp_directory(std::move(path_p)), - temp_file(make_uniq(db, temp_directory, max_swap_space)) { + : db(db), temp_directory(std::move(path_p)), temp_file(make_uniq(db, temp_directory)) { auto &fs = FileSystem::GetFileSystem(db); if (!temp_directory.empty()) { if (!fs.DirectoryExists(temp_directory)) { @@ -197,6 +196,7 @@ TemporaryDirectoryHandle::TemporaryDirectoryHandle(DatabaseInstance &db, string created_directory = true; } } + temp_file->SetMaxSwapSpace(max_swap_space); } TemporaryDirectoryHandle::~TemporaryDirectoryHandle() { @@ -263,12 +263,8 @@ static idx_t GetDefaultMax(const string &path) { return default_value; } -TemporaryFileManager::TemporaryFileManager(DatabaseInstance &db, const string &temp_directory_p, - optional_idx max_swap_space) - : db(db), temp_directory(temp_directory_p), size_on_disk(0), max_swap_space(GetDefaultMax(temp_directory_p)) { - if (max_swap_space.IsValid()) { - this->max_swap_space = max_swap_space.GetIndex(); - } +TemporaryFileManager::TemporaryFileManager(DatabaseInstance &db, const string &temp_directory_p) + : db(db), temp_directory(temp_directory_p), size_on_disk(0), max_swap_space(0) { } TemporaryFileManager::~TemporaryFileManager() { From d996c085f2f5da8b296137d9e74d8cd2233f9d5b Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 25 Mar 2024 12:34:39 +0100 Subject: [PATCH 098/603] Restore old bm --- .../csv_scanner/buffer_manager/csv_buffer.cpp | 27 ++++++++++--------- .../buffer_manager/csv_buffer_manager.cpp | 14 +++------- .../buffer_manager/csv_file_handle.cpp | 12 ++++++++- .../scanner/string_value_scanner.cpp | 2 +- .../table_function/csv_file_scanner.cpp | 4 +-- .../table_function/global_csv_state.cpp | 1 - src/function/table/copy_csv.cpp | 2 +- src/function/table/read_csv.cpp | 2 +- src/function/table/sniff_csv.cpp | 2 +- .../operator/csv_scanner/csv_buffer.hpp | 12 ++++----- .../csv_scanner/csv_buffer_manager.hpp | 5 +--- .../operator/csv_scanner/csv_file_handle.hpp | 3 +++ src/main/relation/read_csv_relation.cpp | 2 +- 13 files changed, 46 insertions(+), 42 deletions(-) diff --git a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp index 6ac66783f041..aaafd21331c6 100644 --- a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp +++ b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp @@ -4,9 +4,9 @@ namespace duckdb { CSVBuffer::CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle &file_handle, - idx_t &global_csv_current_position, idx_t file_number_p, bool single_threaded) - : context(context), file_number(file_number_p), can_seek(file_handle.CanSeek()) { - AllocateBuffer(buffer_size_p, can_seek || single_threaded); + idx_t &global_csv_current_position, idx_t file_number_p) + : context(context), file_number(file_number_p), can_seek(file_handle.CanSeek()), is_pipe(file_handle.OnDiskFile()) { + AllocateBuffer(buffer_size_p); auto buffer = Ptr(); actual_buffer_size = file_handle.Read(buffer, buffer_size_p); while (actual_buffer_size < buffer_size_p && !file_handle.FinishedReading()) { @@ -18,10 +18,10 @@ CSVBuffer::CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle } CSVBuffer::CSVBuffer(CSVFileHandle &file_handle, ClientContext &context, idx_t buffer_size, - idx_t global_csv_current_position, idx_t file_number_p, idx_t buffer_idx_p, bool single_threaded) + idx_t global_csv_current_position, idx_t file_number_p, idx_t buffer_idx_p) : context(context), global_csv_start(global_csv_current_position), file_number(file_number_p), - can_seek(file_handle.CanSeek()), buffer_idx(buffer_idx_p) { - AllocateBuffer(buffer_size, single_threaded || can_seek); + can_seek(file_handle.CanSeek()), is_pipe(file_handle.OnDiskFile()), buffer_idx(buffer_idx_p) { + AllocateBuffer(buffer_size); auto buffer = handle.Ptr(); actual_buffer_size = file_handle.Read(handle.Ptr(), buffer_size); while (actual_buffer_size < buffer_size && !file_handle.FinishedReading()) { @@ -32,16 +32,15 @@ CSVBuffer::CSVBuffer(CSVFileHandle &file_handle, ClientContext &context, idx_t b } shared_ptr CSVBuffer::Next(CSVFileHandle &file_handle, idx_t buffer_size, idx_t file_number_p, - bool &has_seaked, bool single_threaded) { + bool &has_seaked) { if (has_seaked) { // This means that at some point a reload was done, and we are currently on the incorrect position in our file // handle file_handle.Seek(global_csv_start + actual_buffer_size); has_seaked = false; } - auto next_csv_buffer = - make_shared(file_handle, context, buffer_size, global_csv_start + actual_buffer_size, file_number_p, - buffer_idx + 1, single_threaded); + auto next_csv_buffer = make_shared(file_handle, context, buffer_size, + global_csv_start + actual_buffer_size, file_number_p, buffer_idx + 1); if (next_csv_buffer->GetBufferSize() == 0) { // We are done reading return nullptr; @@ -49,8 +48,9 @@ shared_ptr CSVBuffer::Next(CSVFileHandle &file_handle, idx_t buffer_s return next_csv_buffer; } -void CSVBuffer::AllocateBuffer(idx_t buffer_size, bool can_destroy) { +void CSVBuffer::AllocateBuffer(idx_t buffer_size) { auto &buffer_manager = BufferManager::GetBufferManager(context); + bool can_destroy = !is_pipe; handle = buffer_manager.Allocate(MemoryTag::CSV_READER, MaxValue(Storage::BLOCK_SIZE, buffer_size), can_destroy, &block); } @@ -60,14 +60,15 @@ idx_t CSVBuffer::GetBufferSize() { } void CSVBuffer::Reload(CSVFileHandle &file_handle) { - AllocateBuffer(actual_buffer_size, false); + AllocateBuffer(actual_buffer_size); + // If we can seek, we seek and return the correct pointers file_handle.Seek(global_csv_start); file_handle.Read(handle.Ptr(), actual_buffer_size); } shared_ptr CSVBuffer::Pin(CSVFileHandle &file_handle, bool &has_seeked) { auto &buffer_manager = BufferManager::GetBufferManager(context); - if (can_seek && block->IsUnloaded()) { + if (is_pipe && block->IsUnloaded()) { // We have to reload it from disk block = nullptr; Reload(file_handle); diff --git a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp index 568343cafad7..2a13158b6081 100644 --- a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp +++ b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp @@ -4,9 +4,8 @@ namespace duckdb { CSVBufferManager::CSVBufferManager(ClientContext &context_p, const CSVReaderOptions &options, const string &file_path_p, - const idx_t file_idx_p, bool single_threaded_p) - : context(context_p), file_idx(file_idx_p), file_path(file_path_p), buffer_size(CSVBuffer::CSV_BUFFER_SIZE), - single_threaded(single_threaded_p) { + const idx_t file_idx_p) + : context(context_p), file_idx(file_idx_p), file_path(file_path_p), buffer_size(CSVBuffer::CSV_BUFFER_SIZE) { D_ASSERT(!file_path.empty()); file_handle = ReadCSV::OpenCSV(file_path, options.compression, context); skip_rows = options.dialect_options.skip_rows.GetValue(); @@ -29,7 +28,7 @@ void CSVBufferManager::UnpinBuffer(const idx_t cache_idx) { void CSVBufferManager::Initialize() { if (cached_buffers.empty()) { cached_buffers.emplace_back( - make_shared(context, buffer_size, *file_handle, global_csv_pos, file_idx, single_threaded)); + make_shared(context, buffer_size, *file_handle, global_csv_pos, file_idx)); last_buffer = cached_buffers.front(); } } @@ -48,8 +47,7 @@ bool CSVBufferManager::ReadNextAndCacheIt() { last_buffer->last_buffer = true; return false; } - auto maybe_last_buffer = - last_buffer->Next(*file_handle, cur_buffer_size, file_idx, has_seeked, single_threaded); + auto maybe_last_buffer = last_buffer->Next(*file_handle, cur_buffer_size, file_idx, has_seeked); if (!maybe_last_buffer) { last_buffer->last_buffer = true; return false; @@ -128,8 +126,4 @@ string CSVBufferManager::GetFilePath() { return file_path; } -void CSVBufferManager::SetSingleThreaded() { - single_threaded = true; -} - } // namespace duckdb diff --git a/src/execution/operator/csv_scanner/buffer_manager/csv_file_handle.cpp b/src/execution/operator/csv_scanner/buffer_manager/csv_file_handle.cpp index cbb1c1cd86e7..cf4bf9fafdf2 100644 --- a/src/execution/operator/csv_scanner/buffer_manager/csv_file_handle.cpp +++ b/src/execution/operator/csv_scanner/buffer_manager/csv_file_handle.cpp @@ -9,6 +9,7 @@ CSVFileHandle::CSVFileHandle(FileSystem &fs, Allocator &allocator, unique_ptrCanSeek(); on_disk_file = file_handle->OnDiskFile(); file_size = file_handle->GetFileSize(); + is_pipe = file_handle->IsPipe(); uncompressed = compression == FileCompressionType::UNCOMPRESSED; } @@ -33,7 +34,12 @@ bool CSVFileHandle::CanSeek() { void CSVFileHandle::Seek(idx_t position) { if (!can_seek) { - throw InternalException("Cannot seek in this file"); + if (is_pipe) { + throw InternalException("Can't reconstruct the buffer from a on disk file."); + } + //! If we can't seek in this file, we reset it and re-read up to the necessary point. + file_handle->Reset(); + // file_handle->Read(); } file_handle->Seek(position); } @@ -42,6 +48,10 @@ bool CSVFileHandle::OnDiskFile() { return on_disk_file; } +bool CSVFileHandle::IsPipe() { + return is_pipe; +} + idx_t CSVFileHandle::FileSize() { return file_size; } diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index a0376cc94947..b1b2afa39c98 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -530,7 +530,7 @@ unique_ptr StringValueScanner::GetCSVScanner(ClientContext & state_machine->dialect_options.num_cols = options.dialect_options.num_cols; state_machine->dialect_options.header = options.dialect_options.header; - auto buffer_manager = make_shared(context, options, options.file_path, 0, false); + auto buffer_manager = make_shared(context, options, options.file_path, 0); auto scanner = make_uniq(buffer_manager, state_machine, make_shared()); scanner->csv_file_scan = make_shared(context, options.file_path, options, false); scanner->csv_file_scan->InitializeProjection(); diff --git a/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp b/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp index 8013a2da10ef..7d975221c3ac 100644 --- a/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp +++ b/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp @@ -73,7 +73,7 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, cons } // Initialize Buffer Manager - buffer_manager = make_shared(context, options, file_path, file_idx, single_threaded); + buffer_manager = make_shared(context, options, file_path, file_idx); // Initialize On Disk and Size of file on_disk_file = buffer_manager->file_handle->OnDiskFile(); file_size = buffer_manager->file_handle->FileSize(); @@ -132,7 +132,7 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_name, CSVRea bool single_threaded) : file_path(file_name), file_idx(0), error_handler(make_shared(options_p.ignore_errors)), options(options_p) { - buffer_manager = make_shared(context, options, file_path, file_idx, single_threaded); + buffer_manager = make_shared(context, options, file_path, file_idx); // Initialize On Disk and Size of file on_disk_file = buffer_manager->file_handle->OnDiskFile(); file_size = buffer_manager->file_handle->FileSize(); diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index f0ec98fbbdfb..524482e0de55 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -65,7 +65,6 @@ unique_ptr CSVGlobalState::Next() { shared_ptr current_file; if (cur_idx == 0) { current_file = file_scans.back(); - current_file->buffer_manager->SetSingleThreaded(); } else { lock_guard parallel_lock(main_mutex); file_scans.emplace_back(make_shared(context, bind_data.files[cur_idx], bind_data.options, diff --git a/src/function/table/copy_csv.cpp b/src/function/table/copy_csv.cpp index 67e8041f4da6..e2f9a2403c08 100644 --- a/src/function/table/copy_csv.cpp +++ b/src/function/table/copy_csv.cpp @@ -156,7 +156,7 @@ static unique_ptr ReadCSVBind(ClientContext &context, CopyInfo &in } if (options.auto_detect) { - auto buffer_manager = make_shared(context, options, bind_data->files[0], 0, false); + auto buffer_manager = make_shared(context, options, bind_data->files[0], 0); CSVSniffer sniffer(options, buffer_manager, CSVStateMachineCache::Get(context), {&expected_types, &expected_names}); sniffer.SniffCSV(); diff --git a/src/function/table/read_csv.cpp b/src/function/table/read_csv.cpp index 0963f35c0478..8d2e1be0d780 100644 --- a/src/function/table/read_csv.cpp +++ b/src/function/table/read_csv.cpp @@ -98,7 +98,7 @@ static unique_ptr ReadCSVBind(ClientContext &context, TableFunctio } if (options.auto_detect && !options.file_options.union_by_name) { options.file_path = result->files[0]; - result->buffer_manager = make_shared(context, options, result->files[0], 0, false); + result->buffer_manager = make_shared(context, options, result->files[0], 0); CSVSniffer sniffer(options, result->buffer_manager, CSVStateMachineCache::Get(context), {&return_types, &names}); auto sniffer_result = sniffer.SniffCSV(); diff --git a/src/function/table/sniff_csv.cpp b/src/function/table/sniff_csv.cpp index d27817f818eb..3e859a65afe3 100644 --- a/src/function/table/sniff_csv.cpp +++ b/src/function/table/sniff_csv.cpp @@ -120,7 +120,7 @@ static void CSVSniffFunction(ClientContext &context, TableFunctionInput &data_p, auto sniffer_options = data.options; sniffer_options.file_path = data.path; - auto buffer_manager = make_shared(context, sniffer_options, sniffer_options.file_path, 0, false); + auto buffer_manager = make_shared(context, sniffer_options, sniffer_options.file_path, 0); if (sniffer_options.name_list.empty()) { sniffer_options.name_list = data.names_csv; } diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_buffer.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_buffer.hpp index 8200da88e32a..a5a90d763e06 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_buffer.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_buffer.hpp @@ -44,15 +44,14 @@ class CSVBuffer { public: //! Constructor for Initial Buffer CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle &file_handle, - idx_t &global_csv_current_position, idx_t file_number, bool single_threaded); + idx_t &global_csv_current_position, idx_t file_number); //! Constructor for `Next()` Buffers CSVBuffer(CSVFileHandle &file_handle, ClientContext &context, idx_t buffer_size, idx_t global_csv_current_position, - idx_t file_number_p, idx_t buffer_idx, bool single_threaded); + idx_t file_number_p, idx_t buffer_idx); //! Creates a new buffer with the next part of the CSV File - shared_ptr Next(CSVFileHandle &file_handle, idx_t buffer_size, idx_t file_number, bool &has_seaked, - bool single_threaded); + shared_ptr Next(CSVFileHandle &file_handle, idx_t buffer_size, idx_t file_number, bool &has_seaked); //! Gets the buffer actual size idx_t GetBufferSize(); @@ -61,7 +60,7 @@ class CSVBuffer { bool IsCSVFileLastBuffer(); //! Allocates internal buffer, sets 'block' and 'handle' variables. - void AllocateBuffer(idx_t buffer_size, bool can_destroy); + void AllocateBuffer(idx_t buffer_size); void Reload(CSVFileHandle &file_handle); //! Wrapper for the Pin Function, if it can seek, it means that the buffer might have been destroyed, hence we must @@ -92,8 +91,9 @@ class CSVBuffer { //! Number of the file that is in this buffer idx_t file_number = 0; //! If we can seek in the file or not. - //! If we can't seek, this means we can't destroy the buffers bool can_seek; + //! If this file is being fed by a pipe. + bool is_pipe; //! Buffer Index, used as a batch index for insertion-order preservation idx_t buffer_idx = 0; //! -------- Allocated Block ---------// diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp index a1127882a718..b9b4bb92d372 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp @@ -22,7 +22,7 @@ class CSVStateMachine; class CSVBufferManager { public: CSVBufferManager(ClientContext &context, const CSVReaderOptions &options, const string &file_path, - const idx_t file_idx, bool single_threaded); + const idx_t file_idx); //! Returns a buffer from a buffer id (starting from 0). If it's in the auto-detection then we cache new buffers //! Otherwise we remove them from the cache if they are already there, or just return them bypassing the cache. shared_ptr GetBuffer(const idx_t buffer_idx); @@ -44,8 +44,6 @@ class CSVBufferManager { string GetFilePath(); - void SetSingleThreaded(); - ClientContext &context; idx_t skip_rows = 0; @@ -71,7 +69,6 @@ class CSVBufferManager { //! If the file_handle used seek bool has_seeked = false; unordered_set reset_when_possible; - bool single_threaded; }; } // namespace duckdb diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_file_handle.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_file_handle.hpp index c7e70b008ae7..95a3cc6dafc9 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_file_handle.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_file_handle.hpp @@ -28,6 +28,7 @@ struct CSVFileHandle { bool CanSeek(); void Seek(idx_t position); bool OnDiskFile(); + bool IsPipe(); idx_t FileSize(); @@ -50,6 +51,8 @@ struct CSVFileHandle { string path; bool can_seek = false; bool on_disk_file = false; + bool is_pipe = false; + idx_t file_size = 0; idx_t requested_bytes = 0; diff --git a/src/main/relation/read_csv_relation.cpp b/src/main/relation/read_csv_relation.cpp index 5d0b52e5c96d..1500720e0069 100644 --- a/src/main/relation/read_csv_relation.cpp +++ b/src/main/relation/read_csv_relation.cpp @@ -56,7 +56,7 @@ ReadCSVRelation::ReadCSVRelation(const std::shared_ptr &context, shared_ptr buffer_manager; context->RunFunctionInTransaction([&]() { - buffer_manager = make_shared(*context, csv_options, files[0], 0, false); + buffer_manager = make_shared(*context, csv_options, files[0], 0); CSVSniffer sniffer(csv_options, buffer_manager, CSVStateMachineCache::Get(*context)); auto sniffer_result = sniffer.SniffCSV(); auto &types = sniffer_result.return_types; From 03cb16a6ca02a84bebea5448f904ff75ffe39312 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 25 Mar 2024 16:41:31 +0100 Subject: [PATCH 099/603] Not store buffers from gzipped files, reset buffer manager after sniffing, implement brute force seeking for gzipped files --- .../csv_scanner/buffer_manager/csv_buffer.cpp | 4 ++-- .../buffer_manager/csv_buffer_manager.cpp | 13 +++++++++++++ .../buffer_manager/csv_file_handle.cpp | 17 ++++++++++++++--- .../csv_scanner/sniffer/csv_sniffer.cpp | 2 ++ .../operator/csv_scanner/csv_buffer_manager.hpp | 1 + .../operator/csv_scanner/csv_file_handle.hpp | 4 +++- 6 files changed, 35 insertions(+), 6 deletions(-) diff --git a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp index aaafd21331c6..2c85e0ee2924 100644 --- a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp +++ b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp @@ -36,7 +36,7 @@ shared_ptr CSVBuffer::Next(CSVFileHandle &file_handle, idx_t buffer_s if (has_seaked) { // This means that at some point a reload was done, and we are currently on the incorrect position in our file // handle - file_handle.Seek(global_csv_start + actual_buffer_size); + file_handle.Seek(handle.Ptr(), actual_buffer_size, global_csv_start + actual_buffer_size); has_seaked = false; } auto next_csv_buffer = make_shared(file_handle, context, buffer_size, @@ -62,7 +62,7 @@ idx_t CSVBuffer::GetBufferSize() { void CSVBuffer::Reload(CSVFileHandle &file_handle) { AllocateBuffer(actual_buffer_size); // If we can seek, we seek and return the correct pointers - file_handle.Seek(global_csv_start); + file_handle.Seek(handle.Ptr(), actual_buffer_size, global_csv_start); file_handle.Read(handle.Ptr(), actual_buffer_size); } diff --git a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp index 2a13158b6081..66a6e6ab3cec 100644 --- a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp +++ b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp @@ -122,6 +122,19 @@ bool CSVBufferManager::Done() { return done; } +void CSVBufferManager::ResetBufferManager() { + if (!file_handle->IsPipe()) { + // If this is not a pipe we reset the buffer manager and restart it when doing the actual scan + cached_buffers.clear(); + reset_when_possible.clear(); + file_handle->Reset(); + last_buffer = nullptr; + done = false; + global_csv_pos = 0; + Initialize(); + } +} + string CSVBufferManager::GetFilePath() { return file_path; } diff --git a/src/execution/operator/csv_scanner/buffer_manager/csv_file_handle.cpp b/src/execution/operator/csv_scanner/buffer_manager/csv_file_handle.cpp index cf4bf9fafdf2..528246d8119d 100644 --- a/src/execution/operator/csv_scanner/buffer_manager/csv_file_handle.cpp +++ b/src/execution/operator/csv_scanner/buffer_manager/csv_file_handle.cpp @@ -32,14 +32,19 @@ bool CSVFileHandle::CanSeek() { return can_seek; } -void CSVFileHandle::Seek(idx_t position) { +void CSVFileHandle::Seek(void *buffer, idx_t nr_bytes, idx_t position) { if (!can_seek) { if (is_pipe) { throw InternalException("Can't reconstruct the buffer from a on disk file."); } - //! If we can't seek in this file, we reset it and re-read up to the necessary point. + // If we can't seek in this file, we reset it and re-read up to the necessary point. + // This should only happen on extreme cases of memory pressure file_handle->Reset(); - // file_handle->Read(); + D_ASSERT(position % nr_bytes == 0); + for (idx_t i = 0; i < position / nr_bytes; i++) { + file_handle->Read(buffer, nr_bytes); + } + return; } file_handle->Seek(position); } @@ -48,6 +53,12 @@ bool CSVFileHandle::OnDiskFile() { return on_disk_file; } +void CSVFileHandle::Reset() { + file_handle->Reset(); + finished = false; + requested_bytes = 0; +} + bool CSVFileHandle::IsPipe() { return is_pipe; } diff --git a/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp b/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp index 3b60f247aa27..a62aed3ca3d7 100644 --- a/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp +++ b/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp @@ -93,6 +93,8 @@ SnifferResult CSVSniffer::SniffCSV(bool force_match) { DetectHeader(); // 5. Type Replacement ReplaceTypes(); + buffer_manager->ResetBufferManager(); + if (!best_candidate->error_handler->errors.empty() && !options.ignore_errors) { for (auto &error_vector : best_candidate->error_handler->errors) { for (auto &error : error_vector.second) { diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp index b9b4bb92d372..f8c6f246c3e6 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp @@ -42,6 +42,7 @@ class CSVBufferManager { //! once. bool Done(); + void ResetBufferManager(); string GetFilePath(); ClientContext &context; diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_file_handle.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_file_handle.hpp index 95a3cc6dafc9..7d4b55e424fd 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_file_handle.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_file_handle.hpp @@ -26,10 +26,12 @@ struct CSVFileHandle { public: bool CanSeek(); - void Seek(idx_t position); + void Seek(void *buffer, idx_t nr_bytes, idx_t position); bool OnDiskFile(); bool IsPipe(); + void Reset(); + idx_t FileSize(); bool FinishedReading(); From 4eba1ab60d5259db6a8badf8d8cedcdbdff72884 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Tue, 26 Mar 2024 13:55:10 +0100 Subject: [PATCH 100/603] fix old parameter --- .../operator/csv_scanner/scanner/string_value_scanner.cpp | 2 +- .../csv_scanner/table_function/csv_file_scanner.cpp | 5 ++--- .../csv_scanner/table_function/global_csv_state.cpp | 7 +++---- .../execution/operator/csv_scanner/csv_file_scanner.hpp | 5 ++--- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index b1b2afa39c98..9582e1c1af2f 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -532,7 +532,7 @@ unique_ptr StringValueScanner::GetCSVScanner(ClientContext & state_machine->dialect_options.header = options.dialect_options.header; auto buffer_manager = make_shared(context, options, options.file_path, 0); auto scanner = make_uniq(buffer_manager, state_machine, make_shared()); - scanner->csv_file_scan = make_shared(context, options.file_path, options, false); + scanner->csv_file_scan = make_shared(context, options.file_path, options); scanner->csv_file_scan->InitializeProjection(); return scanner; } diff --git a/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp b/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp index 7d975221c3ac..0532fc678a41 100644 --- a/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp +++ b/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp @@ -41,7 +41,7 @@ CSVFileScan::CSVFileScan(ClientContext &context, shared_ptr bu CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, const CSVReaderOptions &options_p, const idx_t file_idx_p, const ReadCSVData &bind_data, const vector &column_ids, - const vector &file_schema, bool single_threaded) + const vector &file_schema) : file_path(file_path_p), file_idx(file_idx_p), error_handler(make_shared(options_p.ignore_errors)), options(options_p) { if (file_idx < bind_data.union_readers.size()) { @@ -128,8 +128,7 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, cons InitializeFileNamesTypes(); } -CSVFileScan::CSVFileScan(ClientContext &context, const string &file_name, CSVReaderOptions &options_p, - bool single_threaded) +CSVFileScan::CSVFileScan(ClientContext &context, const string &file_name, CSVReaderOptions &options_p) : file_path(file_name), file_idx(0), error_handler(make_shared(options_p.ignore_errors)), options(options_p) { buffer_manager = make_shared(context, options, file_path, file_idx); diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index 524482e0de55..e92ed51273ef 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -22,7 +22,7 @@ CSVGlobalState::CSVGlobalState(ClientContext &context_p, const shared_ptr(context, files[0], options, 0, bind_data, column_ids, file_schema, single_threaded)); + make_uniq(context, files[0], options, 0, bind_data, column_ids, file_schema)); }; //! There are situations where we only support single threaded scanning bool many_csv_files = files.size() > 1 && files.size() > system_threads * 2; @@ -68,8 +68,7 @@ unique_ptr CSVGlobalState::Next() { } else { lock_guard parallel_lock(main_mutex); file_scans.emplace_back(make_shared(context, bind_data.files[cur_idx], bind_data.options, - cur_idx, bind_data, column_ids, file_schema, - single_threaded)); + cur_idx, bind_data, column_ids, file_schema)); current_file = file_scans.back(); } auto csv_scanner = @@ -101,7 +100,7 @@ unique_ptr CSVGlobalState::Next() { // If we have a next file we have to construct the file scan for that file_scans.emplace_back(make_shared(context, bind_data.files[current_file_idx], bind_data.options, current_file_idx, bind_data, column_ids, - file_schema, single_threaded)); + file_schema)); // And re-start the boundary-iterator auto buffer_size = file_scans.back()->buffer_manager->GetBuffer(0)->actual_size; current_boundary = CSVIterator(current_file_idx, 0, 0, 0, buffer_size); diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_file_scanner.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_file_scanner.hpp index ed859238d9ef..ce9fc08ce0bd 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_file_scanner.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_file_scanner.hpp @@ -27,10 +27,9 @@ class CSVFileScan { //! Path to this file CSVFileScan(ClientContext &context, const string &file_path, const CSVReaderOptions &options, const idx_t file_idx, const ReadCSVData &bind_data, const vector &column_ids, - const vector &file_schema, bool single_threaded); + const vector &file_schema); - CSVFileScan(ClientContext &context, const string &file_name, CSVReaderOptions &options, - bool single_threaded = false); + CSVFileScan(ClientContext &context, const string &file_name, CSVReaderOptions &options); const string &GetFileName(); const vector &GetNames(); From 235d4c0c4f6dd7400594b19add0f735105f2e246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Tue, 26 Mar 2024 15:19:12 +0100 Subject: [PATCH 101/603] more implicit conversions --- .../aggregate/distributive/bitagg.cpp | 13 ++++++++----- .../aggregate/distributive/bitstring_agg.cpp | 4 ++-- src/core_functions/aggregate/distributive/sum.cpp | 4 ++-- src/core_functions/aggregate/holistic/quantile.cpp | 6 ++++-- .../core_functions/aggregate/sum_helpers.hpp | 4 ++-- src/include/duckdb/execution/merge_sort_tree.hpp | 14 +++++++++----- 6 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/core_functions/aggregate/distributive/bitagg.cpp b/src/core_functions/aggregate/distributive/bitagg.cpp index 3943707fd8db..2d57a4f548cc 100644 --- a/src/core_functions/aggregate/distributive/bitagg.cpp +++ b/src/core_functions/aggregate/distributive/bitagg.cpp @@ -10,6 +10,7 @@ namespace duckdb { template struct BitState { + using TYPE = T; bool is_set; T value; }; @@ -67,7 +68,7 @@ struct BitwiseOperation { template static void Assign(STATE &state, INPUT_TYPE input) { - state.value = input; + state.value = typename STATE::TYPE(input); } template @@ -90,7 +91,7 @@ struct BitwiseOperation { if (!state.is_set) { finalize_data.ReturnNull(); } else { - target = state.value; + target = T(state.value); } } @@ -102,21 +103,23 @@ struct BitwiseOperation { struct BitAndOperation : public BitwiseOperation { template static void Execute(STATE &state, INPUT_TYPE input) { - state.value &= input; + state.value &= typename STATE::TYPE(input); + ; } }; struct BitOrOperation : public BitwiseOperation { template static void Execute(STATE &state, INPUT_TYPE input) { - state.value |= input; + state.value |= typename STATE::TYPE(input); + ; } }; struct BitXorOperation : public BitwiseOperation { template static void Execute(STATE &state, INPUT_TYPE input) { - state.value ^= input; + state.value ^= typename STATE::TYPE(input); } template diff --git a/src/core_functions/aggregate/distributive/bitstring_agg.cpp b/src/core_functions/aggregate/distributive/bitstring_agg.cpp index 700b0cce28f8..36920a476439 100644 --- a/src/core_functions/aggregate/distributive/bitstring_agg.cpp +++ b/src/core_functions/aggregate/distributive/bitstring_agg.cpp @@ -107,7 +107,7 @@ struct BitStringAggOperation { if (!TrySubtractOperator::Operation(max, min, result)) { return NumericLimits::Maximum(); } - idx_t val(result); + auto val = NumericCast(result); if (val == NumericLimits::Maximum()) { return val; } @@ -116,7 +116,7 @@ struct BitStringAggOperation { template static void Execute(STATE &state, INPUT_TYPE input, INPUT_TYPE min) { - Bit::SetBit(state.value, input - min, 1); + Bit::SetBit(state.value, UnsafeNumericCast(input - min), 1); } template diff --git a/src/core_functions/aggregate/distributive/sum.cpp b/src/core_functions/aggregate/distributive/sum.cpp index 9f243869ad16..2858d5552009 100644 --- a/src/core_functions/aggregate/distributive/sum.cpp +++ b/src/core_functions/aggregate/distributive/sum.cpp @@ -112,8 +112,8 @@ unique_ptr SumPropagateStats(ClientContext &context, BoundAggreg default: throw InternalException("Unsupported type for propagate sum stats"); } - auto max_sum_negative = max_negative * hugeint_t(input.node_stats->max_cardinality); - auto max_sum_positive = max_positive * hugeint_t(input.node_stats->max_cardinality); + auto max_sum_negative = max_negative * Hugeint::Convert(input.node_stats->max_cardinality); + auto max_sum_positive = max_positive * Hugeint::Convert(input.node_stats->max_cardinality); if (max_sum_positive >= NumericLimits::Maximum() || max_sum_negative <= NumericLimits::Minimum()) { // sum can potentially exceed int64_t bounds: use hugeint sum diff --git a/src/core_functions/aggregate/holistic/quantile.cpp b/src/core_functions/aggregate/holistic/quantile.cpp index 7a7693a6aa27..932d30f626a6 100644 --- a/src/core_functions/aggregate/holistic/quantile.cpp +++ b/src/core_functions/aggregate/holistic/quantile.cpp @@ -356,8 +356,10 @@ struct Interpolator { // Integer arithmetic for accuracy const auto integral = q.integral; const auto scaling = q.scaling; - const auto scaled_q = DecimalMultiplyOverflowCheck::Operation(n, integral); - const auto scaled_n = DecimalMultiplyOverflowCheck::Operation(n, scaling); + const auto scaled_q = + DecimalMultiplyOverflowCheck::Operation(Hugeint::Convert(n), integral); + const auto scaled_n = + DecimalMultiplyOverflowCheck::Operation(Hugeint::Convert(n), scaling); floored = Cast::Operation((scaled_n - scaled_q) / scaling); break; } diff --git a/src/include/duckdb/core_functions/aggregate/sum_helpers.hpp b/src/include/duckdb/core_functions/aggregate/sum_helpers.hpp index 45f533a7f8c4..fceb25635871 100644 --- a/src/include/duckdb/core_functions/aggregate/sum_helpers.hpp +++ b/src/include/duckdb/core_functions/aggregate/sum_helpers.hpp @@ -61,7 +61,7 @@ struct RegularAdd { template static void AddConstant(STATE &state, T input, idx_t count) { - state.value += input * count; + state.value += input * int64_t(count); } }; @@ -123,7 +123,7 @@ struct HugeintAdd { AddValue(state.value, uint64_t(input), input >= 0); } } else { - hugeint_t addition = hugeint_t(input) * count; + hugeint_t addition = hugeint_t(input) * Hugeint::Convert(count); state.value += addition; } } diff --git a/src/include/duckdb/execution/merge_sort_tree.hpp b/src/include/duckdb/execution/merge_sort_tree.hpp index b01d7087f058..e2b19b433d24 100644 --- a/src/include/duckdb/execution/merge_sort_tree.hpp +++ b/src/include/duckdb/execution/merge_sort_tree.hpp @@ -349,7 +349,7 @@ idx_t MergeSortTree::SelectNth(const SubFrames &frames, idx_t n // The first level contains a single run, // so the only thing we need is any cascading pointers auto level_no = tree.size() - 2; - auto level_width = 1; + idx_t level_width = 1; for (idx_t i = 0; i < level_no; ++i) { level_width *= FANOUT; } @@ -367,9 +367,11 @@ idx_t MergeSortTree::SelectNth(const SubFrames &frames, idx_t n for (idx_t f = 0; f < frames.size(); ++f) { const auto &frame = frames[f]; auto &cascade_idx = cascades[f]; - const auto lower_idx = std::lower_bound(level.begin(), level.end(), frame.start) - level.begin(); + const auto lower_idx = + UnsafeNumericCast(std::lower_bound(level.begin(), level.end(), frame.start) - level.begin()); cascade_idx.first = lower_idx / CASCADING * FANOUT; - const auto upper_idx = std::lower_bound(level.begin(), level.end(), frame.end) - level.begin(); + const auto upper_idx = + UnsafeNumericCast(std::lower_bound(level.begin(), level.end(), frame.end) - level.begin()); cascade_idx.second = upper_idx / CASCADING * FANOUT; } @@ -390,11 +392,13 @@ idx_t MergeSortTree::SelectNth(const SubFrames &frames, idx_t n const auto lower_begin = level_data + level_cascades[cascade_idx.first]; const auto lower_end = level_data + level_cascades[cascade_idx.first + FANOUT]; - match.first = std::lower_bound(lower_begin, lower_end, frame.start) - level_data; + match.first = + UnsafeNumericCast(std::lower_bound(lower_begin, lower_end, frame.start) - level_data); const auto upper_begin = level_data + level_cascades[cascade_idx.second]; const auto upper_end = level_data + level_cascades[cascade_idx.second + FANOUT]; - match.second = std::lower_bound(upper_begin, upper_end, frame.end) - level_data; + match.second = + UnsafeNumericCast(std::lower_bound(upper_begin, upper_end, frame.end) - level_data); matched += idx_t(match.second - match.first); } From ce9507077bbc837779c4ea60b711a271dbd113d5 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Tue, 26 Mar 2024 15:34:47 +0100 Subject: [PATCH 102/603] Cleanup buffer managers --- .../table_function/csv_file_scanner.cpp | 5 +++++ .../table_function/global_csv_state.cpp | 17 +++++++++++++++-- src/function/table/read_csv.cpp | 4 ++-- .../operator/csv_scanner/csv_file_scanner.hpp | 1 + .../operator/csv_scanner/global_csv_state.hpp | 4 +++- 5 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp b/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp index 0532fc678a41..f2ae71dbb8f8 100644 --- a/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp +++ b/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp @@ -222,4 +222,9 @@ void CSVFileScan::InitializeProjection() { reader_data.column_mapping.push_back(i); } } + +void CSVFileScan::Finish() { + buffer_manager.reset(); +} + } // namespace duckdb diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index 00a9052e7e48..9e8afd52a404 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -56,7 +56,7 @@ double CSVGlobalState::GetProgress(const ReadCSVData &bind_data_p) const { return percentage * 100; } -unique_ptr CSVGlobalState::Next() { +unique_ptr CSVGlobalState::Next(StringValueScanner *previous_scanner) { if (single_threaded) { idx_t cur_idx = last_file_idx++; if (cur_idx >= bind_data.files.size()) { @@ -71,6 +71,12 @@ unique_ptr CSVGlobalState::Next() { cur_idx, bind_data, column_ids, file_schema)); current_file = file_scans.back(); } + if (previous_scanner) { + lock_guard parallel_lock(main_mutex); + previous_scanner->buffer_tracker.reset(); + current_buffer_in_use.reset(); + previous_scanner->csv_file_scan->Finish(); + } auto csv_scanner = make_uniq(scanner_idx++, current_file->buffer_manager, current_file->state_machine, current_file->error_handler, current_file, false, current_boundary); @@ -89,7 +95,14 @@ unique_ptr CSVGlobalState::Next() { auto csv_scanner = make_uniq(scanner_idx++, current_file.buffer_manager, current_file.state_machine, current_file.error_handler, file_scans.back(), false, current_boundary); - + threads_per_file[csv_scanner->csv_file_scan->file_idx]++; + if (previous_scanner) { + threads_per_file[previous_scanner->csv_file_scan->file_idx]--; + if (threads_per_file[previous_scanner->csv_file_scan->file_idx] == 0) { + previous_scanner->buffer_tracker.reset(); + previous_scanner->csv_file_scan->Finish(); + } + } csv_scanner->buffer_tracker = current_buffer_in_use; // We then produce the next boundary diff --git a/src/function/table/read_csv.cpp b/src/function/table/read_csv.cpp index 8d2e1be0d780..71e6f2255a7c 100644 --- a/src/function/table/read_csv.cpp +++ b/src/function/table/read_csv.cpp @@ -185,7 +185,7 @@ unique_ptr ReadCSVInitLocal(ExecutionContext &context, return nullptr; } auto &global_state = global_state_p->Cast(); - auto csv_scanner = global_state.Next(); + auto csv_scanner = global_state.Next(nullptr); if (!csv_scanner) { global_state.DecrementThread(); } @@ -211,7 +211,7 @@ static void ReadCSVFunction(ClientContext &context, TableFunctionInput &data_p, break; } if (csv_local_state.csv_reader->FinishedIterator()) { - csv_local_state.csv_reader = csv_global_state.Next(); + csv_local_state.csv_reader = csv_global_state.Next(csv_local_state.csv_reader.get()); if (!csv_local_state.csv_reader) { csv_global_state.DecrementThread(); break; diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_file_scanner.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_file_scanner.hpp index ce9fc08ce0bd..0ba7c0e02dcd 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_file_scanner.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_file_scanner.hpp @@ -35,6 +35,7 @@ class CSVFileScan { const vector &GetNames(); const vector &GetTypes(); void InitializeProjection(); + void Finish(); //! Initialize the actual names and types to be scanned from the file void InitializeFileNamesTypes(); diff --git a/src/include/duckdb/execution/operator/csv_scanner/global_csv_state.hpp b/src/include/duckdb/execution/operator/csv_scanner/global_csv_state.hpp index bbeb2bfee094..4d123480f45c 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/global_csv_state.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/global_csv_state.hpp @@ -30,7 +30,7 @@ struct CSVGlobalState : public GlobalTableFunctionState { //! Generates a CSV Scanner, with information regarding the piece of buffer it should be read. //! In case it returns a nullptr it means we are done reading these files. - unique_ptr Next(); + unique_ptr Next(StringValueScanner *previous_scanner); void FillRejectsTable(); @@ -75,6 +75,8 @@ struct CSVGlobalState : public GlobalTableFunctionState { atomic last_file_idx; shared_ptr current_buffer_in_use; + + unordered_map threads_per_file; }; } // namespace duckdb From a2ee8fc86e956b5cf9592c24f5be4e75b4b0f038 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Tue, 26 Mar 2024 15:58:54 +0100 Subject: [PATCH 103/603] IsPipe should check pipe --- .../operator/csv_scanner/buffer_manager/csv_buffer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp index 2c85e0ee2924..0886a58f1a35 100644 --- a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp +++ b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp @@ -5,7 +5,7 @@ namespace duckdb { CSVBuffer::CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle &file_handle, idx_t &global_csv_current_position, idx_t file_number_p) - : context(context), file_number(file_number_p), can_seek(file_handle.CanSeek()), is_pipe(file_handle.OnDiskFile()) { + : context(context), file_number(file_number_p), can_seek(file_handle.CanSeek()), is_pipe(file_handle.IsPipe()) { AllocateBuffer(buffer_size_p); auto buffer = Ptr(); actual_buffer_size = file_handle.Read(buffer, buffer_size_p); @@ -20,7 +20,7 @@ CSVBuffer::CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle CSVBuffer::CSVBuffer(CSVFileHandle &file_handle, ClientContext &context, idx_t buffer_size, idx_t global_csv_current_position, idx_t file_number_p, idx_t buffer_idx_p) : context(context), global_csv_start(global_csv_current_position), file_number(file_number_p), - can_seek(file_handle.CanSeek()), is_pipe(file_handle.OnDiskFile()), buffer_idx(buffer_idx_p) { + can_seek(file_handle.CanSeek()), is_pipe(file_handle.IsPipe()), buffer_idx(buffer_idx_p) { AllocateBuffer(buffer_size); auto buffer = handle.Ptr(); actual_buffer_size = file_handle.Read(handle.Ptr(), buffer_size); From 3fdf469f191372f6932f41cb070532154ae32162 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Tue, 26 Mar 2024 18:12:42 +0100 Subject: [PATCH 104/603] fix small pipe bug --- .../operator/csv_scanner/buffer_manager/csv_buffer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp index 0886a58f1a35..7d2913e22e81 100644 --- a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp +++ b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp @@ -68,7 +68,7 @@ void CSVBuffer::Reload(CSVFileHandle &file_handle) { shared_ptr CSVBuffer::Pin(CSVFileHandle &file_handle, bool &has_seeked) { auto &buffer_manager = BufferManager::GetBufferManager(context); - if (is_pipe && block->IsUnloaded()) { + if (!is_pipe && block->IsUnloaded()) { // We have to reload it from disk block = nullptr; Reload(file_handle); From c9b83ef0306ffcca810ebc665379c03970686653 Mon Sep 17 00:00:00 2001 From: Guen Prawiroatmodjo Date: Wed, 27 Mar 2024 05:03:22 -0700 Subject: [PATCH 105/603] ignore 'driver' key in connection string --- tools/odbc/include/connect.hpp | 2 ++ tools/odbc/src/connect/connect.cpp | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tools/odbc/include/connect.hpp b/tools/odbc/include/connect.hpp index 55b359f9763d..1f8ed3e9eeeb 100644 --- a/tools/odbc/include/connect.hpp +++ b/tools/odbc/include/connect.hpp @@ -34,6 +34,8 @@ class Connect { bool GetSuccessWithInfo() const { return success_with_info; } + // Ignore keys for use with Power Query + std::vector PQIgnoreKeys = {"driver"}; private: OdbcHandleDbc *dbc; diff --git a/tools/odbc/src/connect/connect.cpp b/tools/odbc/src/connect/connect.cpp index 07cad9a1a1f2..ae64fc5fb5bd 100644 --- a/tools/odbc/src/connect/connect.cpp +++ b/tools/odbc/src/connect/connect.cpp @@ -46,7 +46,14 @@ SQLRETURN Connect::FindKeyValPair(const std::string &row) { SQLStateType::ST_HY000, "")); } - SQLRETURN ret = FindMatchingKey(StringUtil::Lower(row.substr(0, val_pos)), key); + std::string key_candidate = StringUtil::Lower(row.substr(0, val_pos)); + + // Check if the key can be ignored + if (std::find(PQIgnoreKeys.begin(), PQIgnoreKeys.end(), key_candidate) != PQIgnoreKeys.end()) { + return SQL_SUCCESS; + } + + SQLRETURN ret = FindMatchingKey(key_candidate, key); if (ret != SQL_SUCCESS) { return ret; } From e504ad5f715825a8345d018a0cbf2edfc048b585 Mon Sep 17 00:00:00 2001 From: Guen Prawiroatmodjo Date: Wed, 27 Mar 2024 05:15:24 -0700 Subject: [PATCH 106/603] add 'trusted_connection' to PQIgnoreKeys --- tools/odbc/include/connect.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/odbc/include/connect.hpp b/tools/odbc/include/connect.hpp index 1f8ed3e9eeeb..81825bce7c84 100644 --- a/tools/odbc/include/connect.hpp +++ b/tools/odbc/include/connect.hpp @@ -35,7 +35,7 @@ class Connect { return success_with_info; } // Ignore keys for use with Power Query - std::vector PQIgnoreKeys = {"driver"}; + std::vector PQIgnoreKeys = {"driver", "trusted_connection"}; private: OdbcHandleDbc *dbc; From 91888938151a4dfb5c2378f9091ac46794dae423 Mon Sep 17 00:00:00 2001 From: Guen Prawiroatmodjo Date: Wed, 27 Mar 2024 06:12:48 -0700 Subject: [PATCH 107/603] add UT, update GetTesterDirectory to work with windows build path --- tools/odbc/test/common.cpp | 6 ++++-- tools/odbc/test/tests/connect.cpp | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/tools/odbc/test/common.cpp b/tools/odbc/test/common.cpp index 20a3fe08b151..c176ebfff5d7 100644 --- a/tools/odbc/test/common.cpp +++ b/tools/odbc/test/common.cpp @@ -248,9 +248,11 @@ std::string ConvertHexToString(SQLCHAR val[16], int precision) { std::string GetTesterDirectory() { duckdb::unique_ptr fs = duckdb::FileSystem::CreateLocal(); - std::string current_directory = fs->GetWorkingDirectory() + "/test/sql/storage_version/storage_version.db"; + auto cwd = fs->GetWorkingDirectory(); + std::string current_directory = cwd + "/test/sql/storage_version/storage_version.db"; if (!fs->FileExists(current_directory)) { - auto s = fs->GetWorkingDirectory() + "/../../../../test/sql/storage_version/storage_version.db"; + auto base_dir = cwd.substr(0, cwd.rfind("duckdb")); + auto s = base_dir + "duckdb/test/sql/storage_version/storage_version.db"; if (!fs->FileExists(s)) { throw std::runtime_error("Could not find storage_version.db file."); } diff --git a/tools/odbc/test/tests/connect.cpp b/tools/odbc/test/tests/connect.cpp index 5ff7f8d4141d..e79b0f75f2ae 100644 --- a/tools/odbc/test/tests/connect.cpp +++ b/tools/odbc/test/tests/connect.cpp @@ -23,6 +23,24 @@ void ConnectWithoutDSN(SQLHANDLE &env, SQLHANDLE &dbc) { str, sizeof(str), &strl, SQL_DRIVER_COMPLETE); } +// Connect to a database with extra keywords provided by Power Query SDK +void ConnectWithPowerQuerySDK(SQLHANDLE &env, SQLHANDLE &dbc) { + std::string conn_str = "DRIVER={DuckDB Driver};database=" + GetTesterDirectory() + ";custom_user_agent=powerbi/v0.0(DuckDB);Trusted_Connection=yes;"; + SQLCHAR str[1024]; + SQLSMALLINT strl; + + SQLRETURN ret = SQLAllocHandle(SQL_HANDLE_ENV, nullptr, &env); + REQUIRE(ret == SQL_SUCCESS); + + EXECUTE_AND_CHECK("SQLSetEnvAttr (SQL_ATTR_ODBC_VERSION ODBC3)", SQLSetEnvAttr, env, SQL_ATTR_ODBC_VERSION, + ConvertToSQLPOINTER(SQL_OV_ODBC3), 0); + + EXECUTE_AND_CHECK("SQLAllocHandle (DBC)", SQLAllocHandle, SQL_HANDLE_DBC, env, &dbc); + + EXECUTE_AND_CHECK("SQLDriverConnect", SQLDriverConnect, dbc, nullptr, ConvertToSQLCHAR(conn_str.c_str()), SQL_NTS, + str, sizeof(str), &strl, SQL_DRIVER_COMPLETE); +} + // Connect with incorrect params void ConnectWithIncorrectParam(std::string param) { SQLHANDLE env; @@ -129,6 +147,7 @@ TEST_CASE("Test SQLConnect and SQLDriverConnect", "[odbc]") { TestSettingConfigs(); ConnectWithoutDSN(env, dbc); + ConnectWithPowerQuerySDK(env, dbc); DISCONNECT_FROM_DATABASE(env, dbc); } From 53e606fd44b4d75c7d6b449cba95d7f286af05d5 Mon Sep 17 00:00:00 2001 From: Guen Prawiroatmodjo Date: Wed, 27 Mar 2024 06:44:10 -0700 Subject: [PATCH 108/603] formatting fix --- tools/odbc/test/tests/connect.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/odbc/test/tests/connect.cpp b/tools/odbc/test/tests/connect.cpp index e79b0f75f2ae..fbdc21962a95 100644 --- a/tools/odbc/test/tests/connect.cpp +++ b/tools/odbc/test/tests/connect.cpp @@ -25,7 +25,8 @@ void ConnectWithoutDSN(SQLHANDLE &env, SQLHANDLE &dbc) { // Connect to a database with extra keywords provided by Power Query SDK void ConnectWithPowerQuerySDK(SQLHANDLE &env, SQLHANDLE &dbc) { - std::string conn_str = "DRIVER={DuckDB Driver};database=" + GetTesterDirectory() + ";custom_user_agent=powerbi/v0.0(DuckDB);Trusted_Connection=yes;"; + std::string conn_str = "DRIVER={DuckDB Driver};database=" + GetTesterDirectory() + + +";custom_user_agent=powerbi/v0.0(DuckDB);Trusted_Connection=yes;"; SQLCHAR str[1024]; SQLSMALLINT strl; From fdae6e088b84db25135e275f7ba29029f2c0f4ff Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 28 Mar 2024 12:08:46 +0100 Subject: [PATCH 109/603] add micro benchmark --- .github/regression/micro_extended.csv | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/regression/micro_extended.csv b/.github/regression/micro_extended.csv index 6973785b4c98..def4508bf5bc 100644 --- a/.github/regression/micro_extended.csv +++ b/.github/regression/micro_extended.csv @@ -168,6 +168,7 @@ benchmark/micro/nulls/no_nulls_addition.benchmark benchmark/micro/nulls/null_addition.benchmark benchmark/micro/order/orderby.benchmark benchmark/micro/pushdown/or_pushdown.benchmark +benchmark/micro/pushdown/window_partition_pushdown.benchmark benchmark/micro/simd/auto-vectorization.benchmark benchmark/micro/string/bitstring.benchmark benchmark/micro/string/concat_long.benchmark From affb50bd080931f5d64e8fdb971e490a2d362e93 Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 28 Mar 2024 19:36:41 +0100 Subject: [PATCH 110/603] fix merge conflicts --- src/include/duckdb/storage/standard_buffer_manager.hpp | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/include/duckdb/storage/standard_buffer_manager.hpp b/src/include/duckdb/storage/standard_buffer_manager.hpp index 7aeeee733411..85d710ab8f33 100644 --- a/src/include/duckdb/storage/standard_buffer_manager.hpp +++ b/src/include/duckdb/storage/standard_buffer_manager.hpp @@ -48,15 +48,10 @@ class StandardBufferManager : public BufferManager { //! Unpin and pin are nops on this block of memory shared_ptr RegisterSmallMemory(idx_t block_size) final; -<<<<<<< HEAD idx_t GetUsedMemory() const final override; idx_t GetMaxMemory() const final override; idx_t GetUsedSwap() final override; optional_idx GetMaxSwap() final override; -======= - idx_t GetUsedMemory() const final; - idx_t GetMaxMemory() const final; ->>>>>>> upstream/main //! Allocate an in-memory buffer with a single pin. //! The allocated memory is released when the buffer handle is destroyed. @@ -80,13 +75,8 @@ class StandardBufferManager : public BufferManager { //! Returns a list of all temporary files vector GetTemporaryFiles() final; -<<<<<<< HEAD const string &GetTemporaryDirectory() final override { return temporary_directory.path; -======= - const string &GetTemporaryDirectory() final { - return temp_directory; ->>>>>>> upstream/main } void SetTemporaryDirectory(const string &new_dir) final; From d50d02f4e891e2b767da9941740e19384a25d0ec Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Fri, 29 Mar 2024 10:24:32 +0100 Subject: [PATCH 111/603] CI needs to run again From 9068c8c72c2b18ec7501e442e4b67fb6658c9202 Mon Sep 17 00:00:00 2001 From: Guen Prawiroatmodjo Date: Fri, 29 Mar 2024 08:09:59 -0700 Subject: [PATCH 112/603] Add support for SQL_DTC_TRANSACTION_COST and SQL_RETURN_ESCAPE_CLAUSE to SQLGetInfo for reading UTF-8 encoded files in Power Query SDK and Power BI --- tools/odbc/src/connect/connection.cpp | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tools/odbc/src/connect/connection.cpp b/tools/odbc/src/connect/connection.cpp index 929b2e498c3c..ba282542f690 100644 --- a/tools/odbc/src/connect/connection.cpp +++ b/tools/odbc/src/connect/connection.cpp @@ -7,6 +7,12 @@ #include "duckdb/common/helper.hpp" +// From ODBC Spec (ODBCVER >= 0x0300 and ODBCVER >= 0x0400) +// https://github.com/microsoft/ODBC-Specification/blob/master/Windows/inc/sqlext.h +// Needed for use with Power Query SDK and Power BI +#define SQL_DTC_TRANSACTION_COST 1750 +#define SQL_RETURN_ESCAPE_CLAUSE 180 + using duckdb::OdbcUtils; using duckdb::SQLStateType; using std::ptrdiff_t; @@ -1021,6 +1027,14 @@ SQLRETURN SQL_API SQLGetInfo(SQLHDBC connection_handle, SQLUSMALLINT info_type, duckdb::OdbcUtils::WriteString("", (SQLCHAR *)info_value_ptr, buffer_length, string_length_ptr); return SQL_SUCCESS; } + case SQL_DTC_TRANSACTION_COST: { + duckdb::Store(0, (duckdb::data_ptr_t)info_value_ptr); + return SQL_SUCCESS; + } + case SQL_RETURN_ESCAPE_CLAUSE: { + duckdb::Store(0, (duckdb::data_ptr_t)info_value_ptr); + return SQL_SUCCESS; + } default: duckdb::OdbcHandleDbc *dbc = nullptr; SQLRETURN ret = ConvertConnection(connection_handle, dbc); @@ -1029,7 +1043,8 @@ SQLRETURN SQL_API SQLGetInfo(SQLHDBC connection_handle, SQLUSMALLINT info_type, } // return SQL_SUCCESS, but with a record message - return duckdb::SetDiagnosticRecord(dbc, SQL_SUCCESS, "SQLGetInfo", "Unrecognized attribute.", + std::string msg = "Unrecognized attribute: " + std::to_string(info_type); + return duckdb::SetDiagnosticRecord(dbc, SQL_SUCCESS, "SQLGetInfo", msg, SQLStateType::ST_HY092, dbc->GetDataSourceName()); } } // end SQLGetInfo From 53ac6dda45ecf3e49e7e3aa4f8c71f82cde3c48d Mon Sep 17 00:00:00 2001 From: Guen Prawiroatmodjo Date: Fri, 29 Mar 2024 09:31:33 -0700 Subject: [PATCH 113/603] formatting fix --- tools/odbc/src/connect/connection.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/odbc/src/connect/connection.cpp b/tools/odbc/src/connect/connection.cpp index ba282542f690..5251c2bf10c2 100644 --- a/tools/odbc/src/connect/connection.cpp +++ b/tools/odbc/src/connect/connection.cpp @@ -1044,8 +1044,8 @@ SQLRETURN SQL_API SQLGetInfo(SQLHDBC connection_handle, SQLUSMALLINT info_type, // return SQL_SUCCESS, but with a record message std::string msg = "Unrecognized attribute: " + std::to_string(info_type); - return duckdb::SetDiagnosticRecord(dbc, SQL_SUCCESS, "SQLGetInfo", msg, - SQLStateType::ST_HY092, dbc->GetDataSourceName()); + return duckdb::SetDiagnosticRecord(dbc, SQL_SUCCESS, "SQLGetInfo", msg, SQLStateType::ST_HY092, + dbc->GetDataSourceName()); } } // end SQLGetInfo From 59fca1df73efd7798a76d92c9b0bf214ca4f0131 Mon Sep 17 00:00:00 2001 From: Guen Prawiroatmodjo Date: Fri, 29 Mar 2024 13:48:55 -0700 Subject: [PATCH 114/603] SQLTables should not fail if ret code is SQL_SUCCESS_WITH_INFO --- tools/odbc/src/statement/statement.cpp | 2 +- tools/odbc/test/common.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/odbc/src/statement/statement.cpp b/tools/odbc/src/statement/statement.cpp index 96945faccc79..bf6fd018f772 100644 --- a/tools/odbc/src/statement/statement.cpp +++ b/tools/odbc/src/statement/statement.cpp @@ -375,7 +375,7 @@ SQLRETURN SQL_API SQLTables(SQLHSTMT statement_handle, SQLCHAR *catalog_name, SQ SQLSMALLINT name_length3, SQLCHAR *table_type, SQLSMALLINT name_length4) { duckdb::OdbcHandleStmt *hstmt = nullptr; SQLRETURN ret = ConvertHSTMT(statement_handle, hstmt); - if (ret != SQL_SUCCESS) { + if (!SQL_SUCCEEDED(ret)) { return ret; } diff --git a/tools/odbc/test/common.cpp b/tools/odbc/test/common.cpp index 20a3fe08b151..24d0b18866fd 100644 --- a/tools/odbc/test/common.cpp +++ b/tools/odbc/test/common.cpp @@ -24,7 +24,7 @@ void ODBC_CHECK(SQLRETURN ret, const std::string &msg) { fprintf(stderr, "%s: Unexpected return value\n", msg.c_str()); break; } - REQUIRE(ret == SQL_SUCCESS); + REQUIRE(SQL_SUCCEEDED(ret)); } void ACCESS_DIAGNOSTIC(std::string &state, std::string &message, SQLHANDLE handle, SQLSMALLINT handle_type) { From 1104046e0e2c8f2c9cabb5daa9dc580059ad4826 Mon Sep 17 00:00:00 2001 From: Tishj Date: Sat, 30 Mar 2024 23:00:42 +0100 Subject: [PATCH 115/603] reconstruct the sql query using the CreateIndexInfo --- .../catalog_entry/index_catalog_entry.cpp | 8 +-- .../parser/parsed_data/create_index_info.hpp | 1 + src/parser/parsed_data/create_index_info.cpp | 65 +++++++++++++++++++ test/sql/table_function/sqlite_master.test | 2 +- 4 files changed, 71 insertions(+), 5 deletions(-) diff --git a/src/catalog/catalog_entry/index_catalog_entry.cpp b/src/catalog/catalog_entry/index_catalog_entry.cpp index b6da579bc123..4cbf5ca86b1f 100644 --- a/src/catalog/catalog_entry/index_catalog_entry.cpp +++ b/src/catalog/catalog_entry/index_catalog_entry.cpp @@ -36,12 +36,12 @@ unique_ptr IndexCatalogEntry::GetInfo() const { string IndexCatalogEntry::ToSQL() const { if (sql.empty()) { + //! Return empty sql with view name so pragma view_tables don't complain return sql; } - if (sql.back() != ';') { - return sql + ";"; - } - return sql; + auto info = GetInfo(); + auto result = info->ToString(); + return result + ";\n"; } bool IndexCatalogEntry::IsUnique() { diff --git a/src/include/duckdb/parser/parsed_data/create_index_info.hpp b/src/include/duckdb/parser/parsed_data/create_index_info.hpp index 9209cc58f92d..4b42e8e0c6c3 100644 --- a/src/include/duckdb/parser/parsed_data/create_index_info.hpp +++ b/src/include/duckdb/parser/parsed_data/create_index_info.hpp @@ -47,6 +47,7 @@ struct CreateIndexInfo : public CreateInfo { public: DUCKDB_API unique_ptr Copy() const override; + string ToString() const override; void Serialize(Serializer &serializer) const override; static unique_ptr Deserialize(Deserializer &deserializer); }; diff --git a/src/parser/parsed_data/create_index_info.cpp b/src/parser/parsed_data/create_index_info.cpp index 93984b5a62f2..df5c6d14f8db 100644 --- a/src/parser/parsed_data/create_index_info.cpp +++ b/src/parser/parsed_data/create_index_info.cpp @@ -1,4 +1,5 @@ #include "duckdb/parser/parsed_data/create_index_info.hpp" +#include "duckdb/parser/parsed_expression_iterator.hpp" namespace duckdb { @@ -11,6 +12,70 @@ CreateIndexInfo::CreateIndexInfo(const duckdb::CreateIndexInfo &info) scan_types(info.scan_types), names(info.names) { } +static void RemoveTableQualificationRecursive(unique_ptr &expr, const string &table_name) { + if (expr->GetExpressionType() == ExpressionType::COLUMN_REF) { + auto &col_ref = expr->Cast(); + auto &col_names = col_ref.column_names; + if (col_ref.IsQualified() && col_ref.GetTableName() == table_name) { + col_names.erase(col_names.begin()); + } + } else { + ParsedExpressionIterator::EnumerateChildren(*expr, [&table_name](unique_ptr &child) { + RemoveTableQualificationRecursive(child, table_name); + }); + } +} + +string CreateIndexInfo::ToString() const { + string result; + + result += "CREATE"; + D_ASSERT(constraint_type == IndexConstraintType::UNIQUE || constraint_type == IndexConstraintType::NONE); + if (constraint_type == IndexConstraintType::UNIQUE) { + result += " UNIQUE"; + } + result += " INDEX "; + // FIXME: 'CONCURRENTLY' ?? + if (on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT) { + result += "IF NOT EXISTS "; + } + result += KeywordHelper::WriteOptionallyQuoted(index_name); + result += " ON "; + result += KeywordHelper::WriteOptionallyQuoted(table); + if (index_type != "ART") { + result += " USING "; + result += KeywordHelper::WriteOptionallyQuoted(index_type); + result += " "; + } + result += "("; + for (idx_t i = 0; i < parsed_expressions.size(); i++) { + auto &expr = parsed_expressions[i]; + auto copy = expr->Copy(); + if (i > 0) { + result += ", "; + } + // column ref expressions are qualified with the table name + // we need to remove them to reproduce the original query + RemoveTableQualificationRecursive(copy, table); + result += copy->ToString(); + } + result += ")"; + if (!options.empty()) { + result += " WITH ("; + idx_t i = 0; + for (auto &opt : options) { + result += StringUtil::Format("%s = %s", opt.first, opt.second.ToString()); + if (i > 0) { + result += ", "; + } + i++; + } + result += " )"; + } + // FIXME: optional WHERE ??? + return result; +} + unique_ptr CreateIndexInfo::Copy() const { auto result = make_uniq(*this); diff --git a/test/sql/table_function/sqlite_master.test b/test/sql/table_function/sqlite_master.test index 0bd7d72c0990..52d182edf611 100644 --- a/test/sql/table_function/sqlite_master.test +++ b/test/sql/table_function/sqlite_master.test @@ -50,7 +50,7 @@ statement ok CREATE INDEX i_index ON integers(i); query IIIII -SELECT * FROM sqlite_master WHERE name='i_index'; +SELECT * REPLACE (trim(sql, chr(10)) as sql) FROM sqlite_master WHERE name='i_index'; ---- index i_index integers 0 CREATE INDEX i_index ON integers(i); From f30f491fa934864e0fcee0c01e1309ac2a4d6522 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Tue, 2 Apr 2024 09:52:22 +0200 Subject: [PATCH 116/603] more casts --- src/include/duckdb/common/vector.hpp | 2 +- src/include/duckdb/execution/merge_sort_tree.hpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/include/duckdb/common/vector.hpp b/src/include/duckdb/common/vector.hpp index a2b591902e72..95f76fe4d94e 100644 --- a/src/include/duckdb/common/vector.hpp +++ b/src/include/duckdb/common/vector.hpp @@ -102,7 +102,7 @@ class vector : public std::vector> { // NOL } void erase_at(idx_t idx) { - if (MemorySafety::enabled && idx > original::size()) { + if (MemorySafety::ENABLED && idx > original::size()) { throw InternalException("Can't remove offset %d from vector of size %d", idx, original::size()); } original::erase(original::begin() + static_cast(idx)); diff --git a/src/include/duckdb/execution/merge_sort_tree.hpp b/src/include/duckdb/execution/merge_sort_tree.hpp index 1c750ce54aad..4f752a75410b 100644 --- a/src/include/duckdb/execution/merge_sort_tree.hpp +++ b/src/include/duckdb/execution/merge_sort_tree.hpp @@ -430,8 +430,8 @@ idx_t MergeSortTree::SelectNth(const SubFrames &frames, idx_t n // Continue with the uncascaded levels (except the first) for (; level_no > 0; --level_no) { const auto &level = tree[level_no].first; - auto range_begin = level.begin() + result * level_width; - auto range_end = range_begin + level_width; + auto range_begin = level.begin() + UnsafeNumericCast(result * level_width); + auto range_end = range_begin + UnsafeNumericCast(level_width); while (range_end < level.end()) { idx_t matched = 0; for (idx_t f = 0; f < frames.size(); ++f) { @@ -447,7 +447,7 @@ idx_t MergeSortTree::SelectNth(const SubFrames &frames, idx_t n } // Not enough in this child, so move right range_begin = range_end; - range_end += level_width; + range_end += UnsafeNumericCast(level_width); ++result; n -= matched; } From 54d6e721caf20dacc5bbf59002dd849e3b4bd5d8 Mon Sep 17 00:00:00 2001 From: Laurens Kuiper Date: Tue, 2 Apr 2024 10:35:25 +0200 Subject: [PATCH 117/603] early out of analyze if we know we wont write a dictionary --- extension/parquet/column_writer.cpp | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/extension/parquet/column_writer.cpp b/extension/parquet/column_writer.cpp index 47ff6c93d6e9..cc88fe2b64ba 100644 --- a/extension/parquet/column_writer.cpp +++ b/extension/parquet/column_writer.cpp @@ -15,16 +15,16 @@ #include "duckdb/common/string_map_set.hpp" #include "duckdb/common/types/date.hpp" #include "duckdb/common/types/hugeint.hpp" -#include "duckdb/common/types/uhugeint.hpp" #include "duckdb/common/types/string_heap.hpp" #include "duckdb/common/types/time.hpp" #include "duckdb/common/types/timestamp.hpp" +#include "duckdb/common/types/uhugeint.hpp" #endif +#include "lz4.hpp" #include "miniz_wrapper.hpp" #include "snappy.h" #include "zstd.h" -#include "lz4.hpp" namespace duckdb { @@ -1293,6 +1293,9 @@ class StringColumnWriter : public BasicColumnWriter { void Analyze(ColumnWriterState &state_p, ColumnWriterState *parent, Vector &vector, idx_t count) override { auto &state = state_p.Cast(); + if (CannotWriteDictionary(state)) { + return; // Early out: We already know we will not write a dictionary, no need to analyze + } idx_t vcount = parent ? parent->definition_levels.size() - state.definition_levels.size() : count; idx_t parent_index = state.definition_levels.size(); @@ -1319,6 +1322,7 @@ class StringColumnWriter : public BasicColumnWriter { new_value_index++; state.estimated_dict_page_size += value.GetSize() + MAX_DICTIONARY_KEY_SIZE; } + // if the value changed, we will encode it in the page if (last_value_index != found.first->second) { // we will add the value index size later, when we know the total number of keys @@ -1338,10 +1342,7 @@ class StringColumnWriter : public BasicColumnWriter { void FinalizeAnalyze(ColumnWriterState &state_p) override { auto &state = state_p.Cast(); - // check if a dictionary will require more space than a plain write, or if the dictionary page is going to - // be too large - if (state.estimated_dict_page_size > MAX_UNCOMPRESSED_DICT_PAGE_SIZE || - state.estimated_rle_pages_size + state.estimated_dict_page_size > state.estimated_plain_size) { + if (CannotWriteDictionary(state)) { // clearing the dictionary signals a plain write state.dictionary.clear(); state.key_bit_width = 0; @@ -1457,6 +1458,14 @@ class StringColumnWriter : public BasicColumnWriter { return strings[index].GetSize(); } } + +private: + bool CannotWriteDictionary(StringColumnWriterState &state) { + // check if a dictionary will require more space than a plain write, or if the dictionary page is going to + // be too large + return state.estimated_dict_page_size > MAX_UNCOMPRESSED_DICT_PAGE_SIZE || + state.estimated_rle_pages_size + state.estimated_dict_page_size > state.estimated_plain_size; + } }; //===--------------------------------------------------------------------===// From aa92585f32131916ebf0d73932b6bb9b430259e2 Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 2 Apr 2024 10:58:48 +0200 Subject: [PATCH 118/603] add missing include --- src/parser/parsed_data/create_index_info.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/parser/parsed_data/create_index_info.cpp b/src/parser/parsed_data/create_index_info.cpp index df5c6d14f8db..e649047a94ca 100644 --- a/src/parser/parsed_data/create_index_info.cpp +++ b/src/parser/parsed_data/create_index_info.cpp @@ -1,5 +1,6 @@ #include "duckdb/parser/parsed_data/create_index_info.hpp" #include "duckdb/parser/parsed_expression_iterator.hpp" +#include "duckdb/parser/expression/columnref_expression.hpp" namespace duckdb { From 247260e53bc907b25d003caff54f6622477cfeec Mon Sep 17 00:00:00 2001 From: Laurens Kuiper Date: Tue, 2 Apr 2024 11:12:28 +0200 Subject: [PATCH 119/603] clean up code --- extension/parquet/column_writer.cpp | 35 +++++++++++++++++++---------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/extension/parquet/column_writer.cpp b/extension/parquet/column_writer.cpp index cc88fe2b64ba..cecd65dcb2fa 100644 --- a/extension/parquet/column_writer.cpp +++ b/extension/parquet/column_writer.cpp @@ -355,15 +355,18 @@ class BasicColumnWriter : public ColumnWriter { ~BasicColumnWriter() override = default; //! We limit the uncompressed page size to 100MB - // The max size in Parquet is 2GB, but we choose a more conservative limit + //! The max size in Parquet is 2GB, but we choose a more conservative limit static constexpr const idx_t MAX_UNCOMPRESSED_PAGE_SIZE = 100000000; //! Dictionary pages must be below 2GB. Unlike data pages, there's only one dictionary page. - // For this reason we go with a much higher, but still a conservative upper bound of 1GB; + //! For this reason we go with a much higher, but still a conservative upper bound of 1GB; static constexpr const idx_t MAX_UNCOMPRESSED_DICT_PAGE_SIZE = 1e9; + //! If the dictionary has this many entries, but the compression ratio is still below 1, + //! we stop creating the dictionary + static constexpr const idx_t DICTIONARY_ANALYZE_THRESHOLD = 1e4; - // the maximum size a key entry in an RLE page takes + //! The maximum size a key entry in an RLE page takes static constexpr const idx_t MAX_DICTIONARY_KEY_SIZE = sizeof(uint32_t); - // the size of encoding the string length + //! The size of encoding the string length static constexpr const idx_t STRING_LENGTH_SIZE = sizeof(uint32_t); public: @@ -1293,8 +1296,9 @@ class StringColumnWriter : public BasicColumnWriter { void Analyze(ColumnWriterState &state_p, ColumnWriterState *parent, Vector &vector, idx_t count) override { auto &state = state_p.Cast(); - if (CannotWriteDictionary(state)) { - return; // Early out: We already know we will not write a dictionary, no need to analyze + if (state.estimated_dict_page_size > MAX_UNCOMPRESSED_DICT_PAGE_SIZE || + (state.dictionary.size() > DICTIONARY_ANALYZE_THRESHOLD && DictionaryCompressionRatio(state) < 1.0)) { + return; // Early out: compression ratio is less than 1 after seeing more entries than the theshold } idx_t vcount = parent ? parent->definition_levels.size() - state.definition_levels.size() : count; @@ -1342,7 +1346,10 @@ class StringColumnWriter : public BasicColumnWriter { void FinalizeAnalyze(ColumnWriterState &state_p) override { auto &state = state_p.Cast(); - if (CannotWriteDictionary(state)) { + // check if a dictionary will require more space than a plain write, or if the dictionary page is going to + // be too large + if (state.estimated_dict_page_size > MAX_UNCOMPRESSED_DICT_PAGE_SIZE || + DictionaryCompressionRatio(state) < 1.0) { // clearing the dictionary signals a plain write state.dictionary.clear(); state.key_bit_width = 0; @@ -1460,11 +1467,15 @@ class StringColumnWriter : public BasicColumnWriter { } private: - bool CannotWriteDictionary(StringColumnWriterState &state) { - // check if a dictionary will require more space than a plain write, or if the dictionary page is going to - // be too large - return state.estimated_dict_page_size > MAX_UNCOMPRESSED_DICT_PAGE_SIZE || - state.estimated_rle_pages_size + state.estimated_dict_page_size > state.estimated_plain_size; + static double DictionaryCompressionRatio(StringColumnWriterState &state) { + // If any are 0, we just return a compression ratio of 1 + if (state.estimated_plain_size == 0 || state.estimated_rle_pages_size == 0 || + state.estimated_dict_page_size == 0) { + return 1; + } + // Otherwise, plain size divided by compressed size + return double(state.estimated_plain_size) / + double(state.estimated_rle_pages_size + state.estimated_dict_page_size); } }; From 815c155eb232e7f12a7d5cdaa1ee4f2beac535c0 Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 2 Apr 2024 11:48:21 +0200 Subject: [PATCH 120/603] remove dead code, fix tidy issue --- .../duckdb/storage/temporary_file_manager.hpp | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/src/include/duckdb/storage/temporary_file_manager.hpp b/src/include/duckdb/storage/temporary_file_manager.hpp index 92fcbf8eaba5..e5587547ec59 100644 --- a/src/include/duckdb/storage/temporary_file_manager.hpp +++ b/src/include/duckdb/storage/temporary_file_manager.hpp @@ -25,21 +25,9 @@ namespace duckdb { class TemporaryFileManager; -struct FileSizeMonitor { -public: - static constexpr idx_t TEMPFILE_BLOCK_SIZE = Storage::BLOCK_ALLOC_SIZE; - -public: - FileSizeMonitor(TemporaryFileManager &manager); - -public: -private: - TemporaryFileManager &manager; -}; - struct BlockIndexManager { public: - BlockIndexManager(TemporaryFileManager &manager); + explicit BlockIndexManager(TemporaryFileManager &manager); BlockIndexManager(); public: From 111e9eefef6d7f253258a6fd1dca867975c5b8ef Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Tue, 2 Apr 2024 12:28:26 +0200 Subject: [PATCH 121/603] PR requests and adding/running big tests --- .../csv_scanner/buffer_manager/csv_buffer.cpp | 4 +- .../buffer_manager/csv_file_handle.cpp | 13 ++--- .../csv_scanner/sniffer/csv_sniffer.cpp | 8 ++- .../table_function/global_csv_state.cpp | 4 +- .../operator/csv_scanner/csv_file_handle.hpp | 2 +- .../operator/csv_scanner/global_csv_state.hpp | 2 +- .../copy/csv/test_big_compressed.test_slow | 32 +++++++++++ ...est_multiple_big_compressed_csvs.test_slow | 53 +++++++++++++++++++ 8 files changed, 101 insertions(+), 17 deletions(-) create mode 100644 test/sql/copy/csv/test_big_compressed.test_slow create mode 100644 test/sql/copy/csv/test_multiple_big_compressed_csvs.test_slow diff --git a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp index 7d2913e22e81..79a1e8fd762b 100644 --- a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp +++ b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp @@ -36,7 +36,7 @@ shared_ptr CSVBuffer::Next(CSVFileHandle &file_handle, idx_t buffer_s if (has_seaked) { // This means that at some point a reload was done, and we are currently on the incorrect position in our file // handle - file_handle.Seek(handle.Ptr(), actual_buffer_size, global_csv_start + actual_buffer_size); + file_handle.Seek(global_csv_start + actual_buffer_size); has_seaked = false; } auto next_csv_buffer = make_shared(file_handle, context, buffer_size, @@ -62,7 +62,7 @@ idx_t CSVBuffer::GetBufferSize() { void CSVBuffer::Reload(CSVFileHandle &file_handle) { AllocateBuffer(actual_buffer_size); // If we can seek, we seek and return the correct pointers - file_handle.Seek(handle.Ptr(), actual_buffer_size, global_csv_start); + file_handle.Seek(global_csv_start); file_handle.Read(handle.Ptr(), actual_buffer_size); } diff --git a/src/execution/operator/csv_scanner/buffer_manager/csv_file_handle.cpp b/src/execution/operator/csv_scanner/buffer_manager/csv_file_handle.cpp index 9db0131db294..d37e38be14ec 100644 --- a/src/execution/operator/csv_scanner/buffer_manager/csv_file_handle.cpp +++ b/src/execution/operator/csv_scanner/buffer_manager/csv_file_handle.cpp @@ -32,19 +32,12 @@ bool CSVFileHandle::CanSeek() { return can_seek; } -void CSVFileHandle::Seek(void *buffer, idx_t nr_bytes, idx_t position) { +void CSVFileHandle::Seek(idx_t position) { if (!can_seek) { if (is_pipe) { - throw InternalException("Can't reconstruct the buffer from a on disk file."); + throw InternalException("Trying to seek a piped CSV File."); } - // If we can't seek in this file, we reset it and re-read up to the necessary point. - // This should only happen on extreme cases of memory pressure - file_handle->Reset(); - D_ASSERT(position % nr_bytes == 0); - for (idx_t i = 0; i < position / nr_bytes; i++) { - file_handle->Read(buffer, nr_bytes); - } - return; + throw InternalException("Trying to seek a compressed CSV File."); } file_handle->Seek(position); } diff --git a/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp b/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp index a62aed3ca3d7..057120591132 100644 --- a/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp +++ b/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp @@ -93,7 +93,13 @@ SnifferResult CSVSniffer::SniffCSV(bool force_match) { DetectHeader(); // 5. Type Replacement ReplaceTypes(); - buffer_manager->ResetBufferManager(); + + // We reset the buffer for compressed files + // This is done because we can't easily seek on compressed files, if a buffer goes out of scope we must read from + // the start + if (!buffer_manager->file_handle->uncompressed) { + buffer_manager->ResetBufferManager(); + } if (!best_candidate->error_handler->errors.empty() && !options.ignore_errors) { for (auto &error_vector : best_candidate->error_handler->errors) { diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index 9e8afd52a404..a3cca5a55ea2 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -24,7 +24,7 @@ CSVGlobalState::CSVGlobalState(ClientContext &context_p, const shared_ptr(context, files[0], options, 0, bind_data, column_ids, file_schema)); }; - //! There are situations where we only support single threaded scanning + // There are situations where we only support single threaded scanning bool many_csv_files = files.size() > 1 && files.size() > system_threads * 2; single_threaded = many_csv_files || !options.parallel; last_file_idx = 0; @@ -56,7 +56,7 @@ double CSVGlobalState::GetProgress(const ReadCSVData &bind_data_p) const { return percentage * 100; } -unique_ptr CSVGlobalState::Next(StringValueScanner *previous_scanner) { +unique_ptr CSVGlobalState::Next(optional_ptr previous_scanner) { if (single_threaded) { idx_t cur_idx = last_file_idx++; if (cur_idx >= bind_data.files.size()) { diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_file_handle.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_file_handle.hpp index 7d4b55e424fd..43e4e275583e 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_file_handle.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_file_handle.hpp @@ -26,7 +26,7 @@ struct CSVFileHandle { public: bool CanSeek(); - void Seek(void *buffer, idx_t nr_bytes, idx_t position); + void Seek(idx_t position); bool OnDiskFile(); bool IsPipe(); diff --git a/src/include/duckdb/execution/operator/csv_scanner/global_csv_state.hpp b/src/include/duckdb/execution/operator/csv_scanner/global_csv_state.hpp index 4d123480f45c..648948c19e37 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/global_csv_state.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/global_csv_state.hpp @@ -30,7 +30,7 @@ struct CSVGlobalState : public GlobalTableFunctionState { //! Generates a CSV Scanner, with information regarding the piece of buffer it should be read. //! In case it returns a nullptr it means we are done reading these files. - unique_ptr Next(StringValueScanner *previous_scanner); + unique_ptr Next(optional_ptr previous_scanner); void FillRejectsTable(); diff --git a/test/sql/copy/csv/test_big_compressed.test_slow b/test/sql/copy/csv/test_big_compressed.test_slow new file mode 100644 index 000000000000..eaa766ac2fa6 --- /dev/null +++ b/test/sql/copy/csv/test_big_compressed.test_slow @@ -0,0 +1,32 @@ +# name: test/sql/copy/csv/test_big_compressed.test_slow +# description: Test scan over multiple compressed big csv files +# group: [csv] + +# This test is way too slow to run on CI, generating a SF100 TPC-H lineitem file takes a LOT of time. +# Still useful to check for problems locally. +mode skip + +require tpch + +statement ok +CALL dbgen(sf=100); + +statement ok +copy lineitem to '__TEST_DIR__/lineitem_100.csv.gz'; + +statement ok +SET temp_directory='' + +# load the DB from disk (Avoids OOM when generating ze table) +load __TEST_DIR__/lineitem_100_compressed.db + +statement ok +CREATE TABLE lineitem_2(l_orderkey INTEGER NOT NULL, l_partkey INTEGER NOT NULL, l_suppkey INTEGER NOT NULL, l_linenumber INTEGER NOT NULL, l_quantity DECIMAL(15,2) NOT NULL, l_extendedprice DECIMAL(15,2) NOT NULL, l_discount DECIMAL(15,2) NOT NULL, l_tax DECIMAL(15,2) NOT NULL, l_returnflag VARCHAR NOT NULL, l_linestatus VARCHAR NOT NULL, l_shipdate DATE NOT NULL, l_commitdate DATE NOT NULL, l_receiptdate DATE NOT NULL, l_shipinstruct VARCHAR NOT NULL, l_shipmode VARCHAR NOT NULL, l_comment VARCHAR NOT NULL); + +statement ok +INSERT INTO lineitem_2 FROM '__TEST_DIR__/lineitem_100.csv.gz'; + +query I +select count(*) from lineitem_2 +---- +600037902 \ No newline at end of file diff --git a/test/sql/copy/csv/test_multiple_big_compressed_csvs.test_slow b/test/sql/copy/csv/test_multiple_big_compressed_csvs.test_slow new file mode 100644 index 000000000000..f4183fb24401 --- /dev/null +++ b/test/sql/copy/csv/test_multiple_big_compressed_csvs.test_slow @@ -0,0 +1,53 @@ +# name: test/sql/copy/csv/test_multiple_big_compressed_csvs.test_slow +# description: Test scan over multiple compressed big csv files +# group: [csv] + +require tpch + +statement ok +CALL dbgen(sf=10); + +statement ok +copy lineitem to ' __TEST_DIR__/lineitem.csv.gz'; + +statement ok +SET temp_directory='' + +# load the DB from disk (Avoids OOM when generating ze table) +load __TEST_DIR__/lineitem_compressed.db + +statement ok +CREATE TABLE lineitem_2(l_orderkey INTEGER NOT NULL, l_partkey INTEGER NOT NULL, l_suppkey INTEGER NOT NULL, l_linenumber INTEGER NOT NULL, l_quantity DECIMAL(15,2) NOT NULL, l_extendedprice DECIMAL(15,2) NOT NULL, l_discount DECIMAL(15,2) NOT NULL, l_tax DECIMAL(15,2) NOT NULL, l_returnflag VARCHAR NOT NULL, l_linestatus VARCHAR NOT NULL, l_shipdate DATE NOT NULL, l_commitdate DATE NOT NULL, l_receiptdate DATE NOT NULL, l_shipinstruct VARCHAR NOT NULL, l_shipmode VARCHAR NOT NULL, l_comment VARCHAR NOT NULL); + +statement ok +INSERT INTO lineitem_2 FROM read_csv([ + ' __TEST_DIR__/lineitem.csv.gz', + ' __TEST_DIR__/lineitem.csv.gz', + ' __TEST_DIR__/lineitem.csv.gz', + ' __TEST_DIR__/lineitem.csv.gz', + ' __TEST_DIR__/lineitem.csv.gz', + ' __TEST_DIR__/lineitem.csv.gz', + ' __TEST_DIR__/lineitem.csv.gz', + ' __TEST_DIR__/lineitem.csv.gz', + ' __TEST_DIR__/lineitem.csv.gz', + ' __TEST_DIR__/lineitem.csv.gz', + ' __TEST_DIR__/lineitem.csv.gz', + ' __TEST_DIR__/lineitem.csv.gz', + ' __TEST_DIR__/lineitem.csv.gz', + ' __TEST_DIR__/lineitem.csv.gz', + ' __TEST_DIR__/lineitem.csv.gz', + ' __TEST_DIR__/lineitem.csv.gz', + ' __TEST_DIR__/lineitem.csv.gz', + ' __TEST_DIR__/lineitem.csv.gz', + ' __TEST_DIR__/lineitem.csv.gz', + ' __TEST_DIR__/lineitem.csv.gz', + ' __TEST_DIR__/lineitem.csv.gz', + ' __TEST_DIR__/lineitem.csv.gz', + ' __TEST_DIR__/lineitem.csv.gz', + ' __TEST_DIR__/lineitem.csv.gz', +]); + +query I +select count(*) from lineitem_2 +---- +1439665248 \ No newline at end of file From 3ea354c383b81e8dd48ec60e047c0aa55814d39f Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 2 Apr 2024 12:32:27 +0200 Subject: [PATCH 122/603] output a BIGINT value for 'last_value', to copy the behavior of pg_sequences, which I believe was the intended behavior --- src/function/table/system/duckdb_sequences.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/function/table/system/duckdb_sequences.cpp b/src/function/table/system/duckdb_sequences.cpp index 7a38e1c65e96..484129befbb5 100644 --- a/src/function/table/system/duckdb_sequences.cpp +++ b/src/function/table/system/duckdb_sequences.cpp @@ -122,8 +122,7 @@ void DuckDBSequencesFunction(ClientContext &context, TableFunctionInput &data_p, // cycle, BOOLEAN output.SetValue(col++, count, Value::BOOLEAN(seq_data.cycle)); // last_value, BIGINT - output.SetValue(col++, count, - seq_data.usage_count == 0 ? Value() : Value::BOOLEAN(NumericCast(seq_data.last_value))); + output.SetValue(col++, count, seq_data.usage_count == 0 ? Value() : Value::BIGINT(seq_data.last_value)); // sql, LogicalType::VARCHAR output.SetValue(col++, count, Value(seq.ToSQL())); From 2f74a8345d3febb614a0b06bb37ffc0ebc1c8f0c Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 2 Apr 2024 13:19:28 +0200 Subject: [PATCH 123/603] add a test to assert behavior of duckdb_sequences --- .../sequence/test_duckdb_sequences.test | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 test/sql/catalog/sequence/test_duckdb_sequences.test diff --git a/test/sql/catalog/sequence/test_duckdb_sequences.test b/test/sql/catalog/sequence/test_duckdb_sequences.test new file mode 100644 index 000000000000..6df8643c3187 --- /dev/null +++ b/test/sql/catalog/sequence/test_duckdb_sequences.test @@ -0,0 +1,65 @@ +# name: test/sql/catalog/sequence/test_duckdb_sequences.test +# group: [sequence] + +statement ok +pragma enable_verification; + +query IIIIIIIIIIIIIII +select * from duckdb_sequences(); +---- + +statement ok +create sequence seq1 start 0 minvalue 0; + +query IIIIII +select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); +---- +0 0 9223372036854775807 1 false NULL + +statement ok +create table tbl (a integer default nextval('seq1')) + +# No change, the sequence hasnt been used yet +query IIIIII +select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); +---- +0 0 9223372036854775807 1 false NULL + +statement ok +insert into tbl values(default); + +# last_value is no longer NULL +query IIIIII +select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); +---- +0 0 9223372036854775807 1 false 0 + +statement ok +insert into tbl values(default); + +query IIIIII +select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); +---- +0 0 9223372036854775807 1 false 1 + +statement ok +drop sequence seq1 cascade; + +statement ok +create sequence seq1 start 20316564; + +query IIIIII +select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); +---- +20316564 1 9223372036854775807 1 false NULL + +statement ok +create table tbl (a integer DEFAULT nextval('seq1')) + +statement ok +insert into tbl values(default); + +query IIIIII +select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); +---- +20316564 1 9223372036854775807 1 false 20316564 From 55112d11ec4612eea3db506b162e9d69a4f98ddd Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Tue, 2 Apr 2024 13:22:59 +0200 Subject: [PATCH 124/603] Woopsie on path --- ...est_multiple_big_compressed_csvs.test_slow | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/test/sql/copy/csv/test_multiple_big_compressed_csvs.test_slow b/test/sql/copy/csv/test_multiple_big_compressed_csvs.test_slow index f4183fb24401..07b99f83c601 100644 --- a/test/sql/copy/csv/test_multiple_big_compressed_csvs.test_slow +++ b/test/sql/copy/csv/test_multiple_big_compressed_csvs.test_slow @@ -8,7 +8,7 @@ statement ok CALL dbgen(sf=10); statement ok -copy lineitem to ' __TEST_DIR__/lineitem.csv.gz'; +copy lineitem to '__TEST_DIR__/lineitem.csv.gz'; statement ok SET temp_directory='' @@ -21,30 +21,30 @@ CREATE TABLE lineitem_2(l_orderkey INTEGER NOT NULL, l_partkey INTEGER NOT NULL, statement ok INSERT INTO lineitem_2 FROM read_csv([ - ' __TEST_DIR__/lineitem.csv.gz', - ' __TEST_DIR__/lineitem.csv.gz', - ' __TEST_DIR__/lineitem.csv.gz', - ' __TEST_DIR__/lineitem.csv.gz', - ' __TEST_DIR__/lineitem.csv.gz', - ' __TEST_DIR__/lineitem.csv.gz', - ' __TEST_DIR__/lineitem.csv.gz', - ' __TEST_DIR__/lineitem.csv.gz', - ' __TEST_DIR__/lineitem.csv.gz', - ' __TEST_DIR__/lineitem.csv.gz', - ' __TEST_DIR__/lineitem.csv.gz', - ' __TEST_DIR__/lineitem.csv.gz', - ' __TEST_DIR__/lineitem.csv.gz', - ' __TEST_DIR__/lineitem.csv.gz', - ' __TEST_DIR__/lineitem.csv.gz', - ' __TEST_DIR__/lineitem.csv.gz', - ' __TEST_DIR__/lineitem.csv.gz', - ' __TEST_DIR__/lineitem.csv.gz', - ' __TEST_DIR__/lineitem.csv.gz', - ' __TEST_DIR__/lineitem.csv.gz', - ' __TEST_DIR__/lineitem.csv.gz', - ' __TEST_DIR__/lineitem.csv.gz', - ' __TEST_DIR__/lineitem.csv.gz', - ' __TEST_DIR__/lineitem.csv.gz', + '__TEST_DIR__/lineitem.csv.gz', + '__TEST_DIR__/lineitem.csv.gz', + '__TEST_DIR__/lineitem.csv.gz', + '__TEST_DIR__/lineitem.csv.gz', + '__TEST_DIR__/lineitem.csv.gz', + '__TEST_DIR__/lineitem.csv.gz', + '__TEST_DIR__/lineitem.csv.gz', + '__TEST_DIR__/lineitem.csv.gz', + '__TEST_DIR__/lineitem.csv.gz', + '__TEST_DIR__/lineitem.csv.gz', + '__TEST_DIR__/lineitem.csv.gz', + '__TEST_DIR__/lineitem.csv.gz', + '__TEST_DIR__/lineitem.csv.gz', + '__TEST_DIR__/lineitem.csv.gz', + '__TEST_DIR__/lineitem.csv.gz', + '__TEST_DIR__/lineitem.csv.gz', + '__TEST_DIR__/lineitem.csv.gz', + '__TEST_DIR__/lineitem.csv.gz', + '__TEST_DIR__/lineitem.csv.gz', + '__TEST_DIR__/lineitem.csv.gz', + '__TEST_DIR__/lineitem.csv.gz', + '__TEST_DIR__/lineitem.csv.gz', + '__TEST_DIR__/lineitem.csv.gz', + '__TEST_DIR__/lineitem.csv.gz', ]); query I From 7d12a074c696fd26f4666639087a17aad021d4f2 Mon Sep 17 00:00:00 2001 From: Laurens Kuiper Date: Tue, 2 Apr 2024 14:37:13 +0200 Subject: [PATCH 125/603] make dictionary compression configurable --- extension/parquet/column_writer.cpp | 16 ++-- extension/parquet/include/parquet_writer.hpp | 6 +- extension/parquet/parquet_extension.cpp | 23 +++++- extension/parquet/parquet_writer.cpp | 6 +- ...ictionary_compression_ratio_threshold.test | 80 +++++++++++++++++++ 5 files changed, 120 insertions(+), 11 deletions(-) create mode 100644 test/sql/copy/parquet/dictionary_compression_ratio_threshold.test diff --git a/extension/parquet/column_writer.cpp b/extension/parquet/column_writer.cpp index cecd65dcb2fa..897550488948 100644 --- a/extension/parquet/column_writer.cpp +++ b/extension/parquet/column_writer.cpp @@ -1296,9 +1296,11 @@ class StringColumnWriter : public BasicColumnWriter { void Analyze(ColumnWriterState &state_p, ColumnWriterState *parent, Vector &vector, idx_t count) override { auto &state = state_p.Cast(); - if (state.estimated_dict_page_size > MAX_UNCOMPRESSED_DICT_PAGE_SIZE || - (state.dictionary.size() > DICTIONARY_ANALYZE_THRESHOLD && DictionaryCompressionRatio(state) < 1.0)) { - return; // Early out: compression ratio is less than 1 after seeing more entries than the theshold + if (writer.DictionaryCompressionRatioThreshold() == NumericLimits::Maximum() || + (state.dictionary.size() > DICTIONARY_ANALYZE_THRESHOLD && WontUseDictionary(state))) { + // Early out: compression ratio is less than the specified parameter + // after seeing more entries than the threshold + return; } idx_t vcount = parent ? parent->definition_levels.size() - state.definition_levels.size() : count; @@ -1348,8 +1350,7 @@ class StringColumnWriter : public BasicColumnWriter { // check if a dictionary will require more space than a plain write, or if the dictionary page is going to // be too large - if (state.estimated_dict_page_size > MAX_UNCOMPRESSED_DICT_PAGE_SIZE || - DictionaryCompressionRatio(state) < 1.0) { + if (WontUseDictionary(state)) { // clearing the dictionary signals a plain write state.dictionary.clear(); state.key_bit_width = 0; @@ -1467,6 +1468,11 @@ class StringColumnWriter : public BasicColumnWriter { } private: + bool WontUseDictionary(StringColumnWriterState &state) const { + return state.estimated_dict_page_size > MAX_UNCOMPRESSED_DICT_PAGE_SIZE || + DictionaryCompressionRatio(state) < writer.DictionaryCompressionRatioThreshold(); + } + static double DictionaryCompressionRatio(StringColumnWriterState &state) { // If any are 0, we just return a compression ratio of 1 if (state.estimated_plain_size == 0 || state.estimated_rle_pages_size == 0 || diff --git a/extension/parquet/include/parquet_writer.hpp b/extension/parquet/include/parquet_writer.hpp index 6b71b8196b26..dda602ebb127 100644 --- a/extension/parquet/include/parquet_writer.hpp +++ b/extension/parquet/include/parquet_writer.hpp @@ -64,7 +64,7 @@ class ParquetWriter { ParquetWriter(FileSystem &fs, string file_name, vector types, vector names, duckdb_parquet::format::CompressionCodec::type codec, ChildFieldIDs field_ids, const vector> &kv_metadata, - shared_ptr encryption_config); + shared_ptr encryption_config, double dictionary_compression_ratio_threshold); public: void PrepareRowGroup(ColumnDataCollection &buffer, PreparedRowGroup &result); @@ -91,6 +91,9 @@ class ParquetWriter { lock_guard glock(lock); return writer->total_written; } + double DictionaryCompressionRatioThreshold() const { + return dictionary_compression_ratio_threshold; + } static CopyTypeSupport TypeIsSupported(const LogicalType &type); @@ -106,6 +109,7 @@ class ParquetWriter { duckdb_parquet::format::CompressionCodec::type codec; ChildFieldIDs field_ids; shared_ptr encryption_config; + double dictionary_compression_ratio_threshold; unique_ptr writer; shared_ptr protocol; diff --git a/extension/parquet/parquet_extension.cpp b/extension/parquet/parquet_extension.cpp index 36919143c85f..0882f6dad9c5 100644 --- a/extension/parquet/parquet_extension.cpp +++ b/extension/parquet/parquet_extension.cpp @@ -130,6 +130,9 @@ struct ParquetWriteBindData : public TableFunctionData { //! How/Whether to encrypt the data shared_ptr encryption_config; + //! Dictionary compression is applied only if the compression ratio exceeds this threshold + double dictionary_compression_ratio_threshold = 1.0; + ChildFieldIDs field_ids; }; @@ -992,6 +995,15 @@ unique_ptr ParquetWriteBind(ClientContext &context, CopyFunctionBi } } else if (loption == "encryption_config") { bind_data->encryption_config = ParquetEncryptionConfig::Create(context, option.second[0]); + } else if (loption == "dictionary_compression_ratio_threshold") { + auto val = option.second[0].GetValue(); + if (val == -1) { + val = NumericLimits::Maximum(); + } else if (val < 0) { + throw BinderException("dictionary_compression_ratio_threshold must be greater than 0, or -1 to disable " + "dictionary compression"); + } + bind_data->dictionary_compression_ratio_threshold = val; } else { throw NotImplementedException("Unrecognized option for PARQUET: %s", option.first.c_str()); } @@ -1017,9 +1029,10 @@ unique_ptr ParquetWriteInitializeGlobal(ClientContext &conte auto &parquet_bind = bind_data.Cast(); auto &fs = FileSystem::GetFileSystem(context); - global_state->writer = make_uniq(fs, file_path, parquet_bind.sql_types, parquet_bind.column_names, - parquet_bind.codec, parquet_bind.field_ids.Copy(), - parquet_bind.kv_metadata, parquet_bind.encryption_config); + global_state->writer = + make_uniq(fs, file_path, parquet_bind.sql_types, parquet_bind.column_names, parquet_bind.codec, + parquet_bind.field_ids.Copy(), parquet_bind.kv_metadata, + parquet_bind.encryption_config, parquet_bind.dictionary_compression_ratio_threshold); return std::move(global_state); } @@ -1138,6 +1151,8 @@ static void ParquetCopySerialize(Serializer &serializer, const FunctionData &bin serializer.WriteProperty(106, "field_ids", bind_data.field_ids); serializer.WritePropertyWithDefault>(107, "encryption_config", bind_data.encryption_config, nullptr); + serializer.WriteProperty(108, "dictionary_compression_ratio_threshold", + bind_data.dictionary_compression_ratio_threshold); } static unique_ptr ParquetCopyDeserialize(Deserializer &deserializer, CopyFunction &function) { @@ -1151,6 +1166,8 @@ static unique_ptr ParquetCopyDeserialize(Deserializer &deserialize data->field_ids = deserializer.ReadProperty(106, "field_ids"); deserializer.ReadPropertyWithDefault>(107, "encryption_config", data->encryption_config, nullptr); + deserializer.ReadPropertyWithDefault(108, "dictionary_compression_ratio_threshold", + data->dictionary_compression_ratio_threshold, 1.0); return std::move(data); } // LCOV_EXCL_STOP diff --git a/extension/parquet/parquet_writer.cpp b/extension/parquet/parquet_writer.cpp index 7c299fa793a1..815d7676edc8 100644 --- a/extension/parquet/parquet_writer.cpp +++ b/extension/parquet/parquet_writer.cpp @@ -350,9 +350,11 @@ void VerifyUniqueNames(const vector &names) { ParquetWriter::ParquetWriter(FileSystem &fs, string file_name_p, vector types_p, vector names_p, CompressionCodec::type codec, ChildFieldIDs field_ids_p, const vector> &kv_metadata, - shared_ptr encryption_config_p) + shared_ptr encryption_config_p, + double dictionary_compression_ratio_threshold_p) : file_name(std::move(file_name_p)), sql_types(std::move(types_p)), column_names(std::move(names_p)), codec(codec), - field_ids(std::move(field_ids_p)), encryption_config(std::move(encryption_config_p)) { + field_ids(std::move(field_ids_p)), encryption_config(std::move(encryption_config_p)), + dictionary_compression_ratio_threshold(dictionary_compression_ratio_threshold_p) { // initialize the file writer writer = make_uniq(fs, file_name.c_str(), FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE_NEW); diff --git a/test/sql/copy/parquet/dictionary_compression_ratio_threshold.test b/test/sql/copy/parquet/dictionary_compression_ratio_threshold.test new file mode 100644 index 000000000000..b417dd8c6ee9 --- /dev/null +++ b/test/sql/copy/parquet/dictionary_compression_ratio_threshold.test @@ -0,0 +1,80 @@ +# name: test/sql/copy/parquet/dictionary_compression_ratio_threshold.test +# description: Test Parquet dictionary_compression_ratio_threshold parameter +# group: [parquet] + +require parquet + +statement ok +CREATE TABLE test AS SELECT 'thisisaverylongstringbutitrepeatsmanytimessoitshighlycompressible' || (range % 10) i FROM range(100000) + +mode skip + +# cannot be negative +statement error +COPY test TO '__TEST_DIR__/dictionary_compression_ratio_threshold.parquet' (dictionary_compression_ratio_threshold -2) +---- +Binder Error: dictionary_compression_ratio_threshold must be greater than 0, or -1 to disable dictionary compression + +# default is 1.0 +statement ok +COPY test TO '__TEST_DIR__/dictionary_compression_ratio_threshold.parquet' + +# dictionary compression is applied so page offset is non-null +query I +SELECT dictionary_page_offset IS NULL FROM parquet_metadata('__TEST_DIR__/dictionary_compression_ratio_threshold.parquet') +---- +false + +# -1 to disable +statement ok +COPY test TO '__TEST_DIR__/dictionary_compression_ratio_threshold.parquet' (dictionary_compression_ratio_threshold -1) + +# disabled, so no dictionary compression +query I +SELECT dictionary_page_offset IS NULL FROM parquet_metadata('__TEST_DIR__/dictionary_compression_ratio_threshold.parquet') +---- +true + +mode unskip + +# the data compresses more than 10x +statement ok +COPY test TO '__TEST_DIR__/dictionary_compression_ratio_threshold.parquet' (dictionary_compression_ratio_threshold 10) + +# dictionary compression should be enabled +query I +SELECT dictionary_page_offset IS NULL FROM parquet_metadata('__TEST_DIR__/dictionary_compression_ratio_threshold.parquet') +---- +false + +# compresses less than 20x +statement ok +COPY test TO '__TEST_DIR__/dictionary_compression_ratio_threshold.parquet' (dictionary_compression_ratio_threshold 20) + +# dictionary compression should be disabled +query I +SELECT dictionary_page_offset IS NULL FROM parquet_metadata('__TEST_DIR__/dictionary_compression_ratio_threshold.parquet') +---- +true + +# create table with all uniques +statement ok +CREATE OR REPLACE TABLE test AS SELECT 'coolstring' || range i FROM range(100000) + +# shouldn't have dictionary compression with default of 1.0 +statement ok +COPY test TO '__TEST_DIR__/dictionary_compression_ratio_threshold.parquet' + +query I +SELECT dictionary_page_offset IS NULL FROM parquet_metadata('__TEST_DIR__/dictionary_compression_ratio_threshold.parquet') +---- +true + +# but if we set our threshold to 0 we create a dictionary anyway +statement ok +COPY test TO '__TEST_DIR__/dictionary_compression_ratio_threshold.parquet' (dictionary_compression_ratio_threshold 0) + +query I +SELECT dictionary_page_offset IS NULL FROM parquet_metadata('__TEST_DIR__/dictionary_compression_ratio_threshold.parquet') +---- +false From 72ef808241a0b46f85508634dbdafbc46f0efbff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Tue, 2 Apr 2024 15:41:20 +0200 Subject: [PATCH 126/603] more casts --- .../aggregate/holistic/mode.cpp | 4 +-- .../aggregate/holistic/reservoir_quantile.cpp | 10 +++---- src/core_functions/scalar/bit/bitstring.cpp | 27 ++++++++----------- .../scalar/blob/create_sort_key.cpp | 5 ++-- src/core_functions/scalar/date/date_part.cpp | 4 +-- src/include/duckdb/common/radix.hpp | 16 ++++++++--- third_party/tdigest/t_digest.hpp | 12 ++++----- 7 files changed, 41 insertions(+), 37 deletions(-) diff --git a/src/core_functions/aggregate/holistic/mode.cpp b/src/core_functions/aggregate/holistic/mode.cpp index 029b10c31fb7..f33ccc415e99 100644 --- a/src/core_functions/aggregate/holistic/mode.cpp +++ b/src/core_functions/aggregate/holistic/mode.cpp @@ -27,7 +27,7 @@ struct hash { template <> struct hash { inline size_t operator()(const duckdb::hugeint_t &val) const { - return hash {}(val.upper) ^ hash {}(val.lower); + return hash {}(val.upper) ^ hash {}(val.lower); } }; @@ -102,7 +102,7 @@ struct ModeState { void ModeRm(const KEY_TYPE &key, idx_t frame) { auto &attr = (*frequency_map)[key]; auto old_count = attr.count; - nonzero -= int(old_count == 1); + nonzero -= size_t(old_count == 1); attr.count -= 1; if (count == old_count && key == *mode) { diff --git a/src/core_functions/aggregate/holistic/reservoir_quantile.cpp b/src/core_functions/aggregate/holistic/reservoir_quantile.cpp index 7da2cdbeedbf..9e7c24c59522 100644 --- a/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +++ b/src/core_functions/aggregate/holistic/reservoir_quantile.cpp @@ -52,11 +52,11 @@ struct ReservoirQuantileState { struct ReservoirQuantileBindData : public FunctionData { ReservoirQuantileBindData() { } - ReservoirQuantileBindData(double quantile_p, int32_t sample_size_p) + ReservoirQuantileBindData(double quantile_p, idx_t sample_size_p) : quantiles(1, quantile_p), sample_size(sample_size_p) { } - ReservoirQuantileBindData(vector quantiles_p, int32_t sample_size_p) + ReservoirQuantileBindData(vector quantiles_p, idx_t sample_size_p) : quantiles(std::move(quantiles_p)), sample_size(sample_size_p) { } @@ -84,7 +84,7 @@ struct ReservoirQuantileBindData : public FunctionData { } vector quantiles; - int32_t sample_size; + idx_t sample_size; }; struct ReservoirQuantileOperation { @@ -330,7 +330,7 @@ unique_ptr BindReservoirQuantile(ClientContext &context, Aggregate } else { arguments.pop_back(); } - return make_uniq(quantiles, 8192); + return make_uniq(quantiles, 8192ULL); } if (!arguments[2]->IsFoldable()) { throw BinderException("RESERVOIR_QUANTILE can only take constant sample size parameters"); @@ -348,7 +348,7 @@ unique_ptr BindReservoirQuantile(ClientContext &context, Aggregate // remove the quantile argument so we can use the unary aggregate Function::EraseArgument(function, arguments, arguments.size() - 1); Function::EraseArgument(function, arguments, arguments.size() - 1); - return make_uniq(quantiles, sample_size); + return make_uniq(quantiles, NumericCast(sample_size)); } unique_ptr BindReservoirQuantileDecimal(ClientContext &context, AggregateFunction &function, diff --git a/src/core_functions/scalar/bit/bitstring.cpp b/src/core_functions/scalar/bit/bitstring.cpp index babfadfe01e7..8a3074250139 100644 --- a/src/core_functions/scalar/bit/bitstring.cpp +++ b/src/core_functions/scalar/bit/bitstring.cpp @@ -8,17 +8,13 @@ namespace duckdb { // BitStringFunction //===--------------------------------------------------------------------===// static void BitStringFunction(DataChunk &args, ExpressionState &state, Vector &result) { - BinaryExecutor::Execute( - args.data[0], args.data[1], result, args.size(), [&](string_t input, int32_t n) { - if (n < 0) { - throw InvalidInputException("The bitstring length cannot be negative"); - } - if (idx_t(n) < input.GetSize()) { + BinaryExecutor::Execute( + args.data[0], args.data[1], result, args.size(), [&](string_t input, idx_t n) { + if (n < input.GetSize()) { throw InvalidInputException("Length must be equal or larger than input string"); } idx_t len; Bit::TryGetBitStringSize(input, len, nullptr); // string verification - len = Bit::ComputeBitstringLen(n); string_t target = StringVector::EmptyString(result, len); Bit::BitString(input, n, target); @@ -28,7 +24,7 @@ static void BitStringFunction(DataChunk &args, ExpressionState &state, Vector &r } ScalarFunction BitStringFun::GetFunction() { - return ScalarFunction({LogicalType::VARCHAR, LogicalType::INTEGER}, LogicalType::BIT, BitStringFunction); + return ScalarFunction({LogicalType::VARCHAR, LogicalType::UBIGINT}, LogicalType::BIT, BitStringFunction); } //===--------------------------------------------------------------------===// @@ -37,7 +33,7 @@ ScalarFunction BitStringFun::GetFunction() { struct GetBitOperator { template static inline TR Operation(TA input, TB n) { - if (n < 0 || (idx_t)n > Bit::BitLength(input) - 1) { + if (n > Bit::BitLength(input) - 1) { throw OutOfRangeException("bit index %s out of valid range (0..%s)", NumericHelper::ToString(n), NumericHelper::ToString(Bit::BitLength(input) - 1)); } @@ -46,21 +42,20 @@ struct GetBitOperator { }; ScalarFunction GetBitFun::GetFunction() { - return ScalarFunction({LogicalType::BIT, LogicalType::INTEGER}, LogicalType::INTEGER, - ScalarFunction::BinaryFunction); + return ScalarFunction({LogicalType::BIT, LogicalType::UBIGINT}, LogicalType::INTEGER, + ScalarFunction::BinaryFunction); } //===--------------------------------------------------------------------===// // set_bit //===--------------------------------------------------------------------===// static void SetBitOperation(DataChunk &args, ExpressionState &state, Vector &result) { - TernaryExecutor::Execute( - args.data[0], args.data[1], args.data[2], result, args.size(), - [&](string_t input, int32_t n, int32_t new_value) { + TernaryExecutor::Execute( + args.data[0], args.data[1], args.data[2], result, args.size(), [&](string_t input, idx_t n, idx_t new_value) { if (new_value != 0 && new_value != 1) { throw InvalidInputException("The new bit must be 1 or 0"); } - if (n < 0 || (idx_t)n > Bit::BitLength(input) - 1) { + if (n > Bit::BitLength(input) - 1) { throw OutOfRangeException("bit index %s out of valid range (0..%s)", NumericHelper::ToString(n), NumericHelper::ToString(Bit::BitLength(input) - 1)); } @@ -72,7 +67,7 @@ static void SetBitOperation(DataChunk &args, ExpressionState &state, Vector &res } ScalarFunction SetBitFun::GetFunction() { - return ScalarFunction({LogicalType::BIT, LogicalType::INTEGER, LogicalType::INTEGER}, LogicalType::BIT, + return ScalarFunction({LogicalType::BIT, LogicalType::UBIGINT, LogicalType::UBIGINT}, LogicalType::BIT, SetBitOperation); } diff --git a/src/core_functions/scalar/blob/create_sort_key.cpp b/src/core_functions/scalar/blob/create_sort_key.cpp index 880acd2c8366..09ee36f5167e 100644 --- a/src/core_functions/scalar/blob/create_sort_key.cpp +++ b/src/core_functions/scalar/blob/create_sort_key.cpp @@ -189,7 +189,7 @@ struct SortKeyVarcharOperator { auto input_data = input.GetDataUnsafe(); auto input_size = input.GetSize(); for (idx_t r = 0; r < input_size; r++) { - result[r] = input_data[r] + 1; + result[r] = UnsafeNumericCast(input_data[r] + 1); } result[input_size] = SortKeyVectorData::STRING_DELIMITER; // null-byte delimiter return input_size + 1; @@ -519,7 +519,8 @@ void ConstructSortKeyList(SortKeyVectorData &vector_data, SortKeyChunk chunk, So } // write the end-of-list delimiter - result_ptr[offset++] = info.flip_bytes ? ~SortKeyVectorData::LIST_DELIMITER : SortKeyVectorData::LIST_DELIMITER; + result_ptr[offset++] = UnsafeNumericCast(info.flip_bytes ? ~SortKeyVectorData::LIST_DELIMITER + : SortKeyVectorData::LIST_DELIMITER); } } diff --git a/src/core_functions/scalar/date/date_part.cpp b/src/core_functions/scalar/date/date_part.cpp index 1c3e16448e23..ebbe158b51fd 100644 --- a/src/core_functions/scalar/date/date_part.cpp +++ b/src/core_functions/scalar/date/date_part.cpp @@ -1432,8 +1432,8 @@ void DatePart::StructOperator::Operation(bigint_vec &bigint_values, double_vec & // Both define epoch, and the correct value is the sum. // So mask it out and compute it separately. - Operation(bigint_values, double_values, d, idx, mask & ~EPOCH); - Operation(bigint_values, double_values, t, idx, mask & ~EPOCH); + Operation(bigint_values, double_values, d, idx, mask & UnsafeNumericCast(~EPOCH)); + Operation(bigint_values, double_values, t, idx, mask & UnsafeNumericCast(~EPOCH)); if (mask & EPOCH) { auto part_data = HasPartValue(double_values, DatePartSpecifier::EPOCH); diff --git a/src/include/duckdb/common/radix.hpp b/src/include/duckdb/common/radix.hpp index a4d774b9bb6d..4b7c89a3374a 100644 --- a/src/include/duckdb/common/radix.hpp +++ b/src/include/duckdb/common/radix.hpp @@ -117,25 +117,33 @@ inline void Radix::EncodeData(data_ptr_t dataptr, bool value) { template <> inline void Radix::EncodeData(data_ptr_t dataptr, int8_t value) { - Store(value, dataptr); + uint8_t bytes; // dance around signedness conversion check + Store(value, data_ptr_cast(&bytes)); + Store(bytes, dataptr); dataptr[0] = FlipSign(dataptr[0]); } template <> inline void Radix::EncodeData(data_ptr_t dataptr, int16_t value) { - Store(BSwap(value), dataptr); + uint16_t bytes; + Store(value, data_ptr_cast(&bytes)); + Store(BSwap(bytes), dataptr); dataptr[0] = FlipSign(dataptr[0]); } template <> inline void Radix::EncodeData(data_ptr_t dataptr, int32_t value) { - Store(BSwap(value), dataptr); + uint32_t bytes; + Store(value, data_ptr_cast(&bytes)); + Store(BSwap(bytes), dataptr); dataptr[0] = FlipSign(dataptr[0]); } template <> inline void Radix::EncodeData(data_ptr_t dataptr, int64_t value) { - Store(BSwap(value), dataptr); + uint64_t bytes; + Store(value, data_ptr_cast(&bytes)); + Store(BSwap(bytes), dataptr); dataptr[0] = FlipSign(dataptr[0]); } diff --git a/third_party/tdigest/t_digest.hpp b/third_party/tdigest/t_digest.hpp index 5c78567d510e..a25668c08eaf 100644 --- a/third_party/tdigest/t_digest.hpp +++ b/third_party/tdigest/t_digest.hpp @@ -219,7 +219,7 @@ class TDigest { pq.push((*iter)); } std::vector batch; - batch.reserve(size); + batch.reserve(size_t(size)); size_t totalSize = 0; while (!pq.empty()) { @@ -324,7 +324,7 @@ class TDigest { CentroidComparator cc; auto iter = std::upper_bound(processed_.cbegin(), processed_.cend(), Centroid(x, 0), cc); - auto i = std::distance(processed_.cbegin(), iter); + auto i = size_t(std::distance(processed_.cbegin(), iter)); auto z1 = x - (iter - 1)->mean(); auto z2 = (iter)->mean() - x; return weightedAverage(cumulative_[i - 1], z2, cumulative_[i], z1) / processedWeight_; @@ -369,7 +369,7 @@ class TDigest { auto iter = std::lower_bound(cumulative_.cbegin(), cumulative_.cend(), index); if (iter + 1 != cumulative_.cend()) { - auto i = std::distance(cumulative_.cbegin(), iter); + auto i = size_t(std::distance(cumulative_.cbegin(), iter)); auto z1 = index - *(iter - 1); auto z2 = *(iter)-index; // LOG(INFO) << "z2 " << z2 << " index " << index << " z1 " << z1; @@ -406,9 +406,9 @@ class TDigest { inline void add(std::vector::const_iterator iter, std::vector::const_iterator end) { while (iter != end) { - const size_t diff = std::distance(iter, end); + const size_t diff = size_t(std::distance(iter, end)); const size_t room = maxUnprocessed_ - unprocessed_.size(); - auto mid = iter + std::min(diff, room); + auto mid = iter + int64_t(std::min(diff, room)); while (iter != mid) { unprocessed_.push_back(*(iter++)); } @@ -538,7 +538,7 @@ class TDigest { std::sort(unprocessed_.begin(), unprocessed_.end(), cc); auto count = unprocessed_.size(); unprocessed_.insert(unprocessed_.end(), processed_.cbegin(), processed_.cend()); - std::inplace_merge(unprocessed_.begin(), unprocessed_.begin() + count, unprocessed_.end(), cc); + std::inplace_merge(unprocessed_.begin(), unprocessed_.begin() + int64_t(count), unprocessed_.end(), cc); processedWeight_ += unprocessedWeight_; unprocessedWeight_ = 0; From 0c2495faf3b8e332281ecb25a4376e4b1a9c0cd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Tue, 2 Apr 2024 15:50:18 +0200 Subject: [PATCH 127/603] more casts or type changes --- src/core_functions/scalar/date/strftime.cpp | 2 +- src/core_functions/scalar/generic/system_functions.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core_functions/scalar/date/strftime.cpp b/src/core_functions/scalar/date/strftime.cpp index 72da80bb5d52..01c907a5d893 100644 --- a/src/core_functions/scalar/date/strftime.cpp +++ b/src/core_functions/scalar/date/strftime.cpp @@ -33,7 +33,7 @@ struct StrfTimeBindData : public FunctionData { template static unique_ptr StrfTimeBindFunction(ClientContext &context, ScalarFunction &bound_function, vector> &arguments) { - auto format_idx = REVERSED ? 0 : 1; + auto format_idx = REVERSED ? 0U : 1U; auto &format_arg = arguments[format_idx]; if (format_arg->HasParameter()) { throw ParameterNotResolvedException(); diff --git a/src/core_functions/scalar/generic/system_functions.cpp b/src/core_functions/scalar/generic/system_functions.cpp index 757b819a73ec..52c581e071af 100644 --- a/src/core_functions/scalar/generic/system_functions.cpp +++ b/src/core_functions/scalar/generic/system_functions.cpp @@ -89,7 +89,7 @@ static void TransactionIdCurrent(DataChunk &input, ExpressionState &state, Vecto auto &context = state.GetContext(); auto &catalog = Catalog::GetCatalog(context, DatabaseManager::GetDefaultDatabase(context)); auto &transaction = DuckTransaction::Get(context, catalog); - auto val = Value::BIGINT(transaction.start_time); + auto val = Value::UBIGINT(transaction.start_time); result.Reference(val); } @@ -133,7 +133,7 @@ ScalarFunction InSearchPathFun::GetFunction() { } ScalarFunction CurrentTransactionIdFun::GetFunction() { - ScalarFunction txid_current({}, LogicalType::BIGINT, TransactionIdCurrent); + ScalarFunction txid_current({}, LogicalType::UBIGINT, TransactionIdCurrent); txid_current.stability = FunctionStability::CONSISTENT_WITHIN_QUERY; return txid_current; } From 30beaa82080cd95bb7ea52516460f120f94bc199 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Tue, 2 Apr 2024 18:11:00 +0200 Subject: [PATCH 128/603] More on merge --- .github/regression/micro_extended.csv | 1 + .../scanner/string_value_scanner.cpp | 21 +++---- .../csv_scanner/sniffer/dialect_detection.cpp | 4 +- .../operator/csv_scanner/util/csv_error.cpp | 58 +++++++++---------- .../operator/csv_scanner/csv_error.hpp | 3 +- src/storage/serialization/serialize_nodes.cpp | 56 +++++++++--------- 6 files changed, 67 insertions(+), 76 deletions(-) diff --git a/.github/regression/micro_extended.csv b/.github/regression/micro_extended.csv index 6973785b4c98..a9517ef309b4 100644 --- a/.github/regression/micro_extended.csv +++ b/.github/regression/micro_extended.csv @@ -78,6 +78,7 @@ benchmark/micro/copy/to_parquet_partition_by_few.benchmark benchmark/micro/copy/to_parquet_partition_by_many.benchmark benchmark/micro/csv/16_byte_values.benchmark benchmark/micro/csv/1_byte_values.benchmark +benchmark/micro/csv/1brl.benchmark benchmark/micro/csv/multiple_read.benchmark benchmark/micro/csv/multiple_small_read_csv.benchmark benchmark/micro/csv/null_padding.benchmark diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index f8018e5fcdde..448dc8382c4d 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -380,10 +380,10 @@ bool StringValueResult::HandleError() { line_pos.GetGlobalPosition(requested_size, first_nl)); break; case CSVErrorType::CAST_ERROR: - csv_error = CSVError::CastError(state_machine.options, names[cur_error.col_idx], cur_error.error_message, - cur_error.col_idx, borked_line, lines_per_batch, - current_line_position.begin.GetGlobalPosition(requested_size, first_nl), - line_pos.GetGlobalPosition(requested_size, first_nl)); + csv_error = CSVError::CastError( + state_machine.options, names[cur_error.col_idx], cur_error.error_message, cur_error.col_idx, + borked_line, lines_per_batch, current_line_position.begin.GetGlobalPosition(requested_size, first_nl), + line_pos.GetGlobalPosition(requested_size, first_nl), parse_types[cur_error.col_idx].first); break; default: throw InvalidInputException("CSV Error not allowed when inserting row"); @@ -730,11 +730,8 @@ void StringValueScanner::Flush(DataChunk &insert_chunk) { auto csv_error = CSVError::CastError( state_machine->options, csv_file_scan->names[col_idx], error_message, col_idx, borked_line, lines_per_batch, - result.line_positions_per_row[line_error].begin.GetGlobalPosition(result.result_size, first_nl), - -1); - auto csv_error = - CSVError::CastError(state_machine->options, csv_file_scan->names[col_idx], error_message, col_idx, - row, lines_per_batch, result_vector.GetType().id()); + result.line_positions_per_row[line_error].begin.GetGlobalPosition(result.result_size, first_nl), -1, + result_vector.GetType().id()); error_handler->Error(csv_error); } @@ -758,11 +755,7 @@ void StringValueScanner::Flush(DataChunk &insert_chunk) { state_machine->options, csv_file_scan->names[col_idx], error_message, col_idx, borked_line, lines_per_batch, result.line_positions_per_row[line_error].begin.GetGlobalPosition(result.result_size, first_nl), - -1); - auto csv_error = - CSVError::CastError(state_machine->options, csv_file_scan->names[col_idx], error_message, - col_idx, row, lines_per_batch, result_vector.GetType().id()); - + -1, result_vector.GetType().id()); error_handler->Error(csv_error); } } diff --git a/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp b/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp index 03173547fa98..e6ffae7e30e7 100644 --- a/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp +++ b/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp @@ -185,7 +185,7 @@ void CSVSniffer::AnalyzeDialectCandidate(unique_ptr scanner, best_consistent_rows = consistent_rows; max_columns_found = num_cols; prev_padding_count = padding_count; - if (!options.null_padding && !options.ignore_errors) { + if (!options.null_padding && !options.ignore_errors.GetValue()) { sniffing_state_machine.dialect_options.skip_rows = start_row; } else { sniffing_state_machine.dialect_options.skip_rows = options.dialect_options.skip_rows.GetValue(); @@ -210,7 +210,7 @@ void CSVSniffer::AnalyzeDialectCandidate(unique_ptr scanner, } } if (!same_quote_is_candidate) { - if (!options.null_padding && !options.ignore_errors) { + if (!options.null_padding && !options.ignore_errors.GetValue()) { sniffing_state_machine.dialect_options.skip_rows = start_row; } else { sniffing_state_machine.dialect_options.skip_rows = options.dialect_options.skip_rows.GetValue(); diff --git a/src/execution/operator/csv_scanner/util/csv_error.cpp b/src/execution/operator/csv_scanner/util/csv_error.cpp index b6236553fbf0..7c7260c5d2e4 100644 --- a/src/execution/operator/csv_scanner/util/csv_error.cpp +++ b/src/execution/operator/csv_scanner/util/csv_error.cpp @@ -114,9 +114,8 @@ CSVError CSVError::ColumnTypesError(case_insensitive_map_t sql_types_per_ } CSVError CSVError::CastError(const CSVReaderOptions &options, string &column_name, string &cast_error, idx_t column_idx, - vector &row, LinesPerBoundary error_info, LogicalTypeId type) { string &csv_row, LinesPerBoundary error_info, idx_t row_byte_position, - int64_t byte_position) { + int64_t byte_position, LogicalTypeId type) { std::ostringstream error; // Which column error << "Error when converting column \"" << column_name << "\"." << '\n'; @@ -139,16 +138,16 @@ CSVError CSVError::CastError(const CSVReaderOptions &options, string &column_nam << '\n'; } -return CSVError(error.str(), CSVErrorType::CAST_ERROR, column_idx, csv_row, error_info, row_byte_position, + return CSVError(error.str(), CSVErrorType::CAST_ERROR, column_idx, csv_row, error_info, row_byte_position, byte_position, options); - } +} CSVError CSVError::LineSizeError(const CSVReaderOptions &options, idx_t actual_size, LinesPerBoundary error_info, string &csv_row, idx_t byte_position) { std::ostringstream error; error << "Maximum line size of " << options.maximum_line_size << " bytes exceeded. "; error << "Actual Size:" << actual_size << " bytes." << '\n'; -return CSVError(error.str(), CSVErrorType::MAXIMUM_LINE_SIZE, 0, csv_row, error_info, byte_position, byte_position, + return CSVError(error.str(), CSVErrorType::MAXIMUM_LINE_SIZE, 0, csv_row, error_info, byte_position, byte_position, options); } @@ -176,45 +175,44 @@ CSVError CSVError::UnterminatedQuotesError(const CSVReaderOptions &options, idx_ std::ostringstream error; error << "Value with unterminated quote found." << '\n'; error << '\n'; -return CSVError(error.str(), CSVErrorType::UNTERMINATED_QUOTES, current_column, csv_row, error_info, - row_byte_position, byte_position, options);} + return CSVError(error.str(), CSVErrorType::UNTERMINATED_QUOTES, current_column, csv_row, error_info, + row_byte_position, byte_position, options); +} CSVError CSVError::IncorrectColumnAmountError(const CSVReaderOptions &options, idx_t actual_columns, LinesPerBoundary error_info, string &csv_row, idx_t row_byte_position, int64_t byte_position) { std::ostringstream error; - // How many columns were expected and how many were found -// error << "Expected Number of Columns: " << options.dialect_options.num_cols << " Found: " << actual_columns + 1; -// if (actual_columns >= options.dialect_options.num_cols) { -// return CSVError(error.str(), CSVErrorType::TOO_MANY_COLUMNS, actual_columns, csv_row, error_info, -// row_byte_position, byte_position, options); -// } else { -// return CSVError(error.str(), CSVErrorType::TOO_FEW_COLUMNS, actual_columns, csv_row, error_info, -// row_byte_position, byte_position, options); -// } - - // How many columns were expected and how many were found - error << "Expected Number of Columns: " << options.dialect_options.num_cols << " Found: " << actual_columns << '\n'; - error << '\n' << "Possible fixes:" << '\n'; - if (!options.null_padding) { - error << "* Enable null padding (null_padding=true) to replace missing values with NULL" << '\n'; - } - if (!options.ignore_errors) { - error << "* Enable ignore errors (ignore_errors=true) to skip this row" << '\n'; + error << "Expected Number of Columns: " << options.dialect_options.num_cols << " Found: " << actual_columns + 1; + if (actual_columns >= options.dialect_options.num_cols) { + return CSVError(error.str(), CSVErrorType::TOO_MANY_COLUMNS, actual_columns, csv_row, error_info, + row_byte_position, byte_position, options); + } else { + return CSVError(error.str(), CSVErrorType::TOO_FEW_COLUMNS, actual_columns, csv_row, error_info, + row_byte_position, byte_position, options); } - error << '\n'; - // What were the options - error << options.ToString(); - return CSVError(error.str(), CSVErrorType::INCORRECT_COLUMN_AMOUNT, error_info); + + // // How many columns were expected and how many were found + // error << "Expected Number of Columns: " << options.dialect_options.num_cols << " Found: " << actual_columns << + //'\n'; error << '\n' << "Possible fixes:" << '\n'; if (!options.null_padding) { error << "* Enable null padding + //(null_padding=true) to replace missing values with NULL" << '\n'; + // } + // if (!options.ignore_errors) { + // error << "* Enable ignore errors (ignore_errors=true) to skip this row" << '\n'; + // } + // error << '\n'; + // // What were the options + // error << options.ToString(); + // return CSVError(error.str(), CSVErrorType::INCORRECT_COLUMN_AMOUNT, error_info); } CSVError CSVError::InvalidUTF8(const CSVReaderOptions &options, idx_t current_column, LinesPerBoundary error_info, string &csv_row, idx_t row_byte_position, int64_t byte_position) { std::ostringstream error; // How many columns were expected and how many were found - error << "Invalid unicode (byte sequence mismatch) detected."<< '\n'; + error << "Invalid unicode (byte sequence mismatch) detected." << '\n'; return CSVError(error.str(), CSVErrorType::INVALID_UNICODE, current_column, csv_row, error_info, row_byte_position, byte_position, options); } diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp index ea44c54462ff..1ffc21423344 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp @@ -59,8 +59,7 @@ class CSVError { //! Produces error messages for casting errors static CSVError CastError(const CSVReaderOptions &options, string &column_name, string &cast_error, idx_t column_idx, string &csv_row, LinesPerBoundary error_info, idx_t row_byte_position, - int64_t byte_position); - idx_t column_idx, vector &row, LinesPerBoundary error_info, LogicalTypeId type); + int64_t byte_position, LogicalTypeId type); //! Produces error for when the line size exceeds the maximum line size option static CSVError LineSizeError(const CSVReaderOptions &options, idx_t actual_size, LinesPerBoundary error_info, string &csv_row, idx_t byte_position); diff --git a/src/storage/serialization/serialize_nodes.cpp b/src/storage/serialization/serialize_nodes.cpp index 44531988a2cd..2359d127ef24 100644 --- a/src/storage/serialization/serialize_nodes.cpp +++ b/src/storage/serialization/serialize_nodes.cpp @@ -118,7 +118,7 @@ CSVOption CSVOption::Deserialize(Deserializer &deserializer) { } void CSVReaderOptions::Serialize(Serializer &serializer) const { - serializer.WritePropertyWithDefault(100, "ignore_errors", ignore_errors); + serializer.WriteProperty>(100, "ignore_errors", ignore_errors); serializer.WritePropertyWithDefault(101, "buffer_sample_size", buffer_sample_size); serializer.WritePropertyWithDefault(102, "null_str", null_str); serializer.WriteProperty(103, "compression", compression); @@ -135,26 +135,26 @@ void CSVReaderOptions::Serialize(Serializer &serializer) const { serializer.WritePropertyWithDefault(114, "buffer_size", buffer_size); serializer.WriteProperty(115, "file_options", file_options); serializer.WritePropertyWithDefault>(116, "force_quote", force_quote); - serializer.WritePropertyWithDefault(117, "rejects_table_name", rejects_table_name); + serializer.WriteProperty>(117, "store_rejects", store_rejects); serializer.WritePropertyWithDefault(118, "rejects_limit", rejects_limit); - serializer.WritePropertyWithDefault>(119, "rejects_recovery_columns", rejects_recovery_columns); - serializer.WritePropertyWithDefault>(120, "rejects_recovery_column_ids", rejects_recovery_column_ids); - serializer.WriteProperty>(121, "dialect_options.state_machine_options.delimiter", dialect_options.state_machine_options.delimiter); - serializer.WriteProperty>(122, "dialect_options.state_machine_options.quote", dialect_options.state_machine_options.quote); - serializer.WriteProperty>(123, "dialect_options.state_machine_options.escape", dialect_options.state_machine_options.escape); - serializer.WriteProperty>(124, "dialect_options.header", dialect_options.header); - serializer.WritePropertyWithDefault(125, "dialect_options.num_cols", dialect_options.num_cols); - serializer.WriteProperty>(126, "dialect_options.state_machine_options.new_line", dialect_options.state_machine_options.new_line); - serializer.WriteProperty>(127, "dialect_options.skip_rows", dialect_options.skip_rows); - serializer.WriteProperty>>(128, "dialect_options.date_format", dialect_options.date_format); - serializer.WritePropertyWithDefault(129, "sniffer_user_mismatch_error", sniffer_user_mismatch_error); - serializer.WritePropertyWithDefault(130, "parallel", parallel); + serializer.WriteProperty>(119, "dialect_options.state_machine_options.delimiter", dialect_options.state_machine_options.delimiter); + serializer.WriteProperty>(120, "dialect_options.state_machine_options.quote", dialect_options.state_machine_options.quote); + serializer.WriteProperty>(121, "dialect_options.state_machine_options.escape", dialect_options.state_machine_options.escape); + serializer.WriteProperty>(122, "dialect_options.header", dialect_options.header); + serializer.WritePropertyWithDefault(123, "dialect_options.num_cols", dialect_options.num_cols); + serializer.WriteProperty>(124, "dialect_options.state_machine_options.new_line", dialect_options.state_machine_options.new_line); + serializer.WriteProperty>(125, "dialect_options.skip_rows", dialect_options.skip_rows); + serializer.WriteProperty>>(126, "dialect_options.date_format", dialect_options.date_format); + serializer.WritePropertyWithDefault(127, "sniffer_user_mismatch_error", sniffer_user_mismatch_error); + serializer.WritePropertyWithDefault(128, "parallel", parallel); + serializer.WriteProperty>(129, "rejects_table_name", rejects_table_name); + serializer.WriteProperty>(130, "rejects_scan_name", rejects_scan_name); serializer.WritePropertyWithDefault>(131, "was_type_manually_set", was_type_manually_set); } CSVReaderOptions CSVReaderOptions::Deserialize(Deserializer &deserializer) { CSVReaderOptions result; - deserializer.ReadPropertyWithDefault(100, "ignore_errors", result.ignore_errors); + deserializer.ReadProperty>(100, "ignore_errors", result.ignore_errors); deserializer.ReadPropertyWithDefault(101, "buffer_sample_size", result.buffer_sample_size); deserializer.ReadPropertyWithDefault(102, "null_str", result.null_str); deserializer.ReadProperty(103, "compression", result.compression); @@ -171,20 +171,20 @@ CSVReaderOptions CSVReaderOptions::Deserialize(Deserializer &deserializer) { deserializer.ReadPropertyWithDefault(114, "buffer_size", result.buffer_size); deserializer.ReadProperty(115, "file_options", result.file_options); deserializer.ReadPropertyWithDefault>(116, "force_quote", result.force_quote); - deserializer.ReadPropertyWithDefault(117, "rejects_table_name", result.rejects_table_name); + deserializer.ReadProperty>(117, "store_rejects", result.store_rejects); deserializer.ReadPropertyWithDefault(118, "rejects_limit", result.rejects_limit); - deserializer.ReadPropertyWithDefault>(119, "rejects_recovery_columns", result.rejects_recovery_columns); - deserializer.ReadPropertyWithDefault>(120, "rejects_recovery_column_ids", result.rejects_recovery_column_ids); - deserializer.ReadProperty>(121, "dialect_options.state_machine_options.delimiter", result.dialect_options.state_machine_options.delimiter); - deserializer.ReadProperty>(122, "dialect_options.state_machine_options.quote", result.dialect_options.state_machine_options.quote); - deserializer.ReadProperty>(123, "dialect_options.state_machine_options.escape", result.dialect_options.state_machine_options.escape); - deserializer.ReadProperty>(124, "dialect_options.header", result.dialect_options.header); - deserializer.ReadPropertyWithDefault(125, "dialect_options.num_cols", result.dialect_options.num_cols); - deserializer.ReadProperty>(126, "dialect_options.state_machine_options.new_line", result.dialect_options.state_machine_options.new_line); - deserializer.ReadProperty>(127, "dialect_options.skip_rows", result.dialect_options.skip_rows); - deserializer.ReadProperty>>(128, "dialect_options.date_format", result.dialect_options.date_format); - deserializer.ReadPropertyWithDefault(129, "sniffer_user_mismatch_error", result.sniffer_user_mismatch_error); - deserializer.ReadPropertyWithDefault(130, "parallel", result.parallel); + deserializer.ReadProperty>(119, "dialect_options.state_machine_options.delimiter", result.dialect_options.state_machine_options.delimiter); + deserializer.ReadProperty>(120, "dialect_options.state_machine_options.quote", result.dialect_options.state_machine_options.quote); + deserializer.ReadProperty>(121, "dialect_options.state_machine_options.escape", result.dialect_options.state_machine_options.escape); + deserializer.ReadProperty>(122, "dialect_options.header", result.dialect_options.header); + deserializer.ReadPropertyWithDefault(123, "dialect_options.num_cols", result.dialect_options.num_cols); + deserializer.ReadProperty>(124, "dialect_options.state_machine_options.new_line", result.dialect_options.state_machine_options.new_line); + deserializer.ReadProperty>(125, "dialect_options.skip_rows", result.dialect_options.skip_rows); + deserializer.ReadProperty>>(126, "dialect_options.date_format", result.dialect_options.date_format); + deserializer.ReadPropertyWithDefault(127, "sniffer_user_mismatch_error", result.sniffer_user_mismatch_error); + deserializer.ReadPropertyWithDefault(128, "parallel", result.parallel); + deserializer.ReadProperty>(129, "rejects_table_name", result.rejects_table_name); + deserializer.ReadProperty>(130, "rejects_scan_name", result.rejects_scan_name); deserializer.ReadPropertyWithDefault>(131, "was_type_manually_set", result.was_type_manually_set); return result; } From 23680a061fde7a20ec711bc6645996e6c149d73e Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Tue, 2 Apr 2024 18:53:06 +0200 Subject: [PATCH 129/603] Forcing fix message to be set to all csv errors --- .../operator/csv_scanner/util/csv_error.cpp | 87 ++++++++++--------- .../operator/csv_scanner/csv_error.hpp | 10 ++- 2 files changed, 54 insertions(+), 43 deletions(-) diff --git a/src/execution/operator/csv_scanner/util/csv_error.cpp b/src/execution/operator/csv_scanner/util/csv_error.cpp index 7c7260c5d2e4..dbe43899e6e2 100644 --- a/src/execution/operator/csv_scanner/util/csv_error.cpp +++ b/src/execution/operator/csv_scanner/util/csv_error.cpp @@ -18,10 +18,10 @@ void CSVErrorHandler::ThrowError(CSVError csv_error) { if (PrintLineNumber(csv_error)) { error << "CSV Error on Line: " << GetLine(csv_error.error_info) << '\n'; } - if (csv_error.error_message_with_options.empty()) { + if (csv_error.full_error_message.empty()) { error << csv_error.error_message; } else { - error << csv_error.error_message_with_options; + error << csv_error.full_error_message; } switch (csv_error.type) { case CSVErrorType::CAST_ERROR: @@ -82,15 +82,16 @@ CSVError::CSVError(string error_message_p, CSVErrorType type_p, LinesPerBoundary CSVError::CSVError(string error_message_p, CSVErrorType type_p, idx_t column_idx_p, string csv_row_p, LinesPerBoundary error_info_p, idx_t row_byte_position, int64_t byte_position_p, - const CSVReaderOptions &reader_options) + const CSVReaderOptions &reader_options, const string &fixes) : error_message(std::move(error_message_p)), type(type_p), column_idx(column_idx_p), csv_row(std::move(csv_row_p)), error_info(error_info_p), row_byte_position(row_byte_position), byte_position(byte_position_p) { // What were the options std::ostringstream error; - error << error_message << std::endl; + error << error_message << '\n'; + error << fixes << '\n'; error << reader_options.ToString(); - error << std::endl; - error_message_with_options = error.str(); + error << '\n'; + full_error_message = error.str(); } CSVError CSVError::ColumnTypesError(case_insensitive_map_t sql_types_per_column, const vector &names) { @@ -118,28 +119,30 @@ CSVError CSVError::CastError(const CSVReaderOptions &options, string &column_nam int64_t byte_position, LogicalTypeId type) { std::ostringstream error; // Which column - error << "Error when converting column \"" << column_name << "\"." << '\n'; + error << "Error when converting column \"" << column_name << "\". "; // What was the cast error error << cast_error << '\n'; - - error << "Column " << column_name << " is being converted as type " << LogicalTypeIdToString(type) << '\n'; + std::ostringstream how_to_fix_it; + how_to_fix_it << "Column " << column_name << " is being converted as type " << LogicalTypeIdToString(type) << '\n'; if (!options.WasTypeManuallySet(column_idx)) { - error << "This type was auto-detected from the CSV file." << '\n'; - error << "Possible solutions:" << '\n'; - error << "* Override the type for this column manually by setting the type explicitly, e.g. types={'" - << column_name << "': 'VARCHAR'}" << '\n'; - error << "* Set the sample size to a larger value to enable the auto-detection to scan more values, e.g. " - "sample_size=-1" - << '\n'; - error << "* Use a COPY statement to automatically derive types from an existing table." << '\n'; + how_to_fix_it << "This type was auto-detected from the CSV file." << '\n'; + how_to_fix_it << "Possible solutions:" << '\n'; + how_to_fix_it << "* Override the type for this column manually by setting the type explicitly, e.g. types={'" + << column_name << "': 'VARCHAR'}" << '\n'; + how_to_fix_it + << "* Set the sample size to a larger value to enable the auto-detection to scan more values, e.g. " + "sample_size=-1" + << '\n'; + how_to_fix_it << "* Use a COPY statement to automatically derive types from an existing table." << '\n'; } else { - error << "This type was either manually set or derived from an existing table. Select a different type to " - "correctly parse this column." - << '\n'; + how_to_fix_it + << "This type was either manually set or derived from an existing table. Select a different type to " + "correctly parse this column." + << '\n'; } return CSVError(error.str(), CSVErrorType::CAST_ERROR, column_idx, csv_row, error_info, row_byte_position, - byte_position, options); + byte_position, options, how_to_fix_it.str()); } CSVError CSVError::LineSizeError(const CSVReaderOptions &options, idx_t actual_size, LinesPerBoundary error_info, @@ -147,8 +150,13 @@ CSVError CSVError::LineSizeError(const CSVReaderOptions &options, idx_t actual_s std::ostringstream error; error << "Maximum line size of " << options.maximum_line_size << " bytes exceeded. "; error << "Actual Size:" << actual_size << " bytes." << '\n'; + + std::ostringstream how_to_fix_it; + how_to_fix_it << "Possible Solution: Change the maximum length size, e.g., max_line_size=" << actual_size + 1 + << "\n"; + return CSVError(error.str(), CSVErrorType::MAXIMUM_LINE_SIZE, 0, csv_row, error_info, byte_position, byte_position, - options); + options, how_to_fix_it.str()); } CSVError CSVError::SniffingError(string &file_path) { @@ -175,37 +183,34 @@ CSVError CSVError::UnterminatedQuotesError(const CSVReaderOptions &options, idx_ std::ostringstream error; error << "Value with unterminated quote found." << '\n'; error << '\n'; + std::ostringstream how_to_fix_it; + how_to_fix_it << "Possible Solution: Enable ignore errors (ignore_errors=true) to skip this row" << '\n'; return CSVError(error.str(), CSVErrorType::UNTERMINATED_QUOTES, current_column, csv_row, error_info, - row_byte_position, byte_position, options); + row_byte_position, byte_position, options, how_to_fix_it.str()); } CSVError CSVError::IncorrectColumnAmountError(const CSVReaderOptions &options, idx_t actual_columns, LinesPerBoundary error_info, string &csv_row, idx_t row_byte_position, int64_t byte_position) { std::ostringstream error; - + // We don't have a fix for this + std::ostringstream how_to_fix_it; + how_to_fix_it << "Possible fixes:" << '\n'; + if (!options.null_padding) { + how_to_fix_it << "* Enable null padding (null_padding=true) to replace missing values with NULL" << '\n'; + } + if (!options.ignore_errors.GetValue()) { + how_to_fix_it << "* Enable ignore errors (ignore_errors=true) to skip this row" << '\n'; + } // How many columns were expected and how many were found error << "Expected Number of Columns: " << options.dialect_options.num_cols << " Found: " << actual_columns + 1; if (actual_columns >= options.dialect_options.num_cols) { return CSVError(error.str(), CSVErrorType::TOO_MANY_COLUMNS, actual_columns, csv_row, error_info, - row_byte_position, byte_position, options); + row_byte_position, byte_position, options, how_to_fix_it.str()); } else { return CSVError(error.str(), CSVErrorType::TOO_FEW_COLUMNS, actual_columns, csv_row, error_info, - row_byte_position, byte_position, options); + row_byte_position, byte_position, options, how_to_fix_it.str()); } - - // // How many columns were expected and how many were found - // error << "Expected Number of Columns: " << options.dialect_options.num_cols << " Found: " << actual_columns << - //'\n'; error << '\n' << "Possible fixes:" << '\n'; if (!options.null_padding) { error << "* Enable null padding - //(null_padding=true) to replace missing values with NULL" << '\n'; - // } - // if (!options.ignore_errors) { - // error << "* Enable ignore errors (ignore_errors=true) to skip this row" << '\n'; - // } - // error << '\n'; - // // What were the options - // error << options.ToString(); - // return CSVError(error.str(), CSVErrorType::INCORRECT_COLUMN_AMOUNT, error_info); } CSVError CSVError::InvalidUTF8(const CSVReaderOptions &options, idx_t current_column, LinesPerBoundary error_info, @@ -213,8 +218,10 @@ CSVError CSVError::InvalidUTF8(const CSVReaderOptions &options, idx_t current_co std::ostringstream error; // How many columns were expected and how many were found error << "Invalid unicode (byte sequence mismatch) detected." << '\n'; + std::ostringstream how_to_fix_it; + how_to_fix_it << "Possible Solution: Enable ignore errors (ignore_errors=true) to skip this row" << '\n'; return CSVError(error.str(), CSVErrorType::INVALID_UNICODE, current_column, csv_row, error_info, row_byte_position, - byte_position, options); + byte_position, options, how_to_fix_it.str()); } bool CSVErrorHandler::PrintLineNumber(CSVError &error) { diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp index 1ffc21423344..c0f556ffba32 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp @@ -52,7 +52,8 @@ class CSVError { public: CSVError() {}; CSVError(string error_message, CSVErrorType type, idx_t column_idx, string csv_row, LinesPerBoundary error_info, - idx_t row_byte_position, int64_t byte_position, const CSVReaderOptions &reader_options); + idx_t row_byte_position, int64_t byte_position, const CSVReaderOptions &reader_options, + const string &fixes); CSVError(string error_message, CSVErrorType type, LinesPerBoundary error_info); //! Produces error messages for column name -> type mismatch. static CSVError ColumnTypesError(case_insensitive_map_t sql_types_per_column, const vector &names); @@ -84,8 +85,11 @@ class CSVError { //! Actual error message string error_message; - //! Actual error message - string error_message_with_options; + //! Full error message used in throws + //! 1. The Actual error + //! 2. How to fix it + //! 3. Options that generated the error + string full_error_message; //! Error Type CSVErrorType type; //! Column Index where error happened From d5108ba2c8c2fa866a3b0d6cad8412bed06154dc Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 3 Apr 2024 10:44:27 +0200 Subject: [PATCH 130/603] Add docker alpine build to check on builds This is currently NOT invoked on any CI job, but can be invoked like ./scripts/test_docker_images.sh Idea and implementation by @szarnyasg --- scripts/test_docker_images.sh | 5 +++++ 1 file changed, 5 insertions(+) create mode 100755 scripts/test_docker_images.sh diff --git a/scripts/test_docker_images.sh b/scripts/test_docker_images.sh new file mode 100755 index 000000000000..904d4f90b529 --- /dev/null +++ b/scripts/test_docker_images.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +make clean +docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb alpine:latest <<< "apk add g++ git make cmake ninja && GEN=ninja make" 2>&1 +echo "alpine:latest completed" From 0c7008b3c0d74817b0bda3c2354ed082a546202f Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 3 Apr 2024 11:30:36 +0200 Subject: [PATCH 131/603] Fix small issue with newline nullpadding in parallel --- .../csv_scanner/scanner/string_value_scanner.cpp | 11 ++++++----- src/execution/operator/csv_scanner/util/csv_error.cpp | 1 - 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 448dc8382c4d..07bf849e4a9c 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -405,9 +405,8 @@ void StringValueResult::QuotedNewLine(StringValueResult &result) { void StringValueResult::NullPaddingQuotedNewlineCheck() { // We do some checks for null_padding correctness - if (state_machine.options.null_padding && iterator.IsBoundarySet() && quoted_new_line && iterator.done) { - // If we have null_padding set, we found a quoted new line, we are scanning the file in parallel, and it's the - // last row of this thread. + if (state_machine.options.null_padding && iterator.IsBoundarySet() && quoted_new_line) { + // If we have null_padding set, we found a quoted new line, we are scanning the file in parallel; We error. LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); auto csv_error = CSVError::NullPaddingFail(state_machine.options, lines_per_batch); error_handler.Error(csv_error); @@ -1035,14 +1034,16 @@ bool StringValueScanner::MoveToNextBuffer() { // And an extra empty value to represent what comes after the delimiter result.AddRow(result, previous_buffer_handle->actual_size); lines_read++; - } else if (states.IsQuotedCurrent()) { + } + else if (states.IsQuotedCurrent()) { // Unterminated quote LinePosition current_line_start = {iterator.pos.buffer_idx, iterator.pos.buffer_pos, result.buffer_size}; result.current_line_position.begin = result.current_line_position.end; result.current_line_position.end = current_line_start; result.InvalidState(result); - } else { + } + else { result.AddRow(result, previous_buffer_handle->actual_size); lines_read++; } diff --git a/src/execution/operator/csv_scanner/util/csv_error.cpp b/src/execution/operator/csv_scanner/util/csv_error.cpp index dbe43899e6e2..d8bafa91e72d 100644 --- a/src/execution/operator/csv_scanner/util/csv_error.cpp +++ b/src/execution/operator/csv_scanner/util/csv_error.cpp @@ -182,7 +182,6 @@ CSVError CSVError::UnterminatedQuotesError(const CSVReaderOptions &options, idx_ int64_t byte_position) { std::ostringstream error; error << "Value with unterminated quote found." << '\n'; - error << '\n'; std::ostringstream how_to_fix_it; how_to_fix_it << "Possible Solution: Enable ignore errors (ignore_errors=true) to skip this row" << '\n'; return CSVError(error.str(), CSVErrorType::UNTERMINATED_QUOTES, current_column, csv_row, error_info, From c483d282d5faf5917243c2914370b145b3e75bb8 Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 3 Apr 2024 12:38:01 +0200 Subject: [PATCH 132/603] properly serialize sequences to disk --- src/catalog/catalog_entry/sequence_catalog_entry.cpp | 8 +++++--- .../duckdb/parser/parsed_data/create_sequence_info.hpp | 2 ++ src/include/duckdb/storage/serialization/create_info.json | 5 +++++ src/parser/parsed_data/create_sequence_info.cpp | 1 + src/storage/serialization/serialize_create_info.cpp | 2 ++ 5 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/catalog/catalog_entry/sequence_catalog_entry.cpp b/src/catalog/catalog_entry/sequence_catalog_entry.cpp index 936fae03854f..e421e5f0695e 100644 --- a/src/catalog/catalog_entry/sequence_catalog_entry.cpp +++ b/src/catalog/catalog_entry/sequence_catalog_entry.cpp @@ -14,8 +14,9 @@ namespace duckdb { SequenceData::SequenceData(CreateSequenceInfo &info) - : usage_count(info.usage_count), counter(info.start_value), increment(info.increment), - start_value(info.start_value), min_value(info.min_value), max_value(info.max_value), cycle(info.cycle) { + : usage_count(info.usage_count), counter(info.start_value + (info.increment * info.usage_count)), + last_value(info.last_value), increment(info.increment), start_value(info.start_value), min_value(info.min_value), + max_value(info.max_value), cycle(info.cycle) { } SequenceCatalogEntry::SequenceCatalogEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateSequenceInfo &info) @@ -93,10 +94,11 @@ unique_ptr SequenceCatalogEntry::GetInfo() const { result->schema = schema.name; result->name = name; result->usage_count = seq_data.usage_count; + result->last_value = seq_data.last_value; result->increment = seq_data.increment; result->min_value = seq_data.min_value; result->max_value = seq_data.max_value; - result->start_value = seq_data.counter; + result->start_value = seq_data.start_value; result->cycle = seq_data.cycle; result->comment = comment; return std::move(result); diff --git a/src/include/duckdb/parser/parsed_data/create_sequence_info.hpp b/src/include/duckdb/parser/parsed_data/create_sequence_info.hpp index de8fcc572f75..0e5182863f06 100644 --- a/src/include/duckdb/parser/parsed_data/create_sequence_info.hpp +++ b/src/include/duckdb/parser/parsed_data/create_sequence_info.hpp @@ -45,6 +45,8 @@ struct CreateSequenceInfo : public CreateInfo { int64_t start_value; //! Whether or not the sequence cycles bool cycle; + //! Last value of the sequence + int64_t last_value; public: unique_ptr Copy() const override; diff --git a/src/include/duckdb/storage/serialization/create_info.json b/src/include/duckdb/storage/serialization/create_info.json index c850a71e854a..fa8fe7dc339f 100644 --- a/src/include/duckdb/storage/serialization/create_info.json +++ b/src/include/duckdb/storage/serialization/create_info.json @@ -280,6 +280,11 @@ "id": 206, "name": "cycle", "type": "bool" + }, + { + "id": 207, + "name": "last_value", + "type": "int64_t" } ] } diff --git a/src/parser/parsed_data/create_sequence_info.cpp b/src/parser/parsed_data/create_sequence_info.cpp index d7caac026f83..a5bbb604a470 100644 --- a/src/parser/parsed_data/create_sequence_info.cpp +++ b/src/parser/parsed_data/create_sequence_info.cpp @@ -21,6 +21,7 @@ unique_ptr CreateSequenceInfo::Copy() const { result->max_value = max_value; result->start_value = start_value; result->cycle = cycle; + result->last_value = last_value; return std::move(result); } diff --git a/src/storage/serialization/serialize_create_info.cpp b/src/storage/serialization/serialize_create_info.cpp index 3196a1d107d0..5eb7070e9ade 100644 --- a/src/storage/serialization/serialize_create_info.cpp +++ b/src/storage/serialization/serialize_create_info.cpp @@ -137,6 +137,7 @@ void CreateSequenceInfo::Serialize(Serializer &serializer) const { serializer.WritePropertyWithDefault(204, "max_value", max_value); serializer.WritePropertyWithDefault(205, "start_value", start_value); serializer.WritePropertyWithDefault(206, "cycle", cycle); + serializer.WritePropertyWithDefault(207, "last_value", last_value); } unique_ptr CreateSequenceInfo::Deserialize(Deserializer &deserializer) { @@ -148,6 +149,7 @@ unique_ptr CreateSequenceInfo::Deserialize(Deserializer &deserialize deserializer.ReadPropertyWithDefault(204, "max_value", result->max_value); deserializer.ReadPropertyWithDefault(205, "start_value", result->start_value); deserializer.ReadPropertyWithDefault(206, "cycle", result->cycle); + deserializer.ReadPropertyWithDefault(207, "last_value", result->last_value); return std::move(result); } From 2f76a7b72dd4427bce5ec0872c58269a667c7c75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Wed, 3 Apr 2024 12:41:41 +0200 Subject: [PATCH 133/603] arrays and jaro --- .../scalar/list/array_slice.cpp | 22 ++++++------ .../scalar/list/list_reduce.cpp | 2 +- src/core_functions/scalar/list/list_sort.cpp | 6 ++-- src/core_functions/scalar/list/list_value.cpp | 2 +- src/core_functions/scalar/list/range.cpp | 4 +-- src/core_functions/scalar/map/map_extract.cpp | 2 +- .../scalar/operators/bitwise.cpp | 4 +-- src/core_functions/scalar/string/bar.cpp | 2 +- src/core_functions/scalar/string/chr.cpp | 2 +- src/core_functions/scalar/string/hex.cpp | 29 ++++++++------- src/core_functions/scalar/string/instr.cpp | 2 +- .../function/scalar/string_functions.hpp | 2 +- third_party/jaro_winkler/details/common.hpp | 18 +++++----- .../jaro_winkler/details/intrinsics.hpp | 2 +- .../jaro_winkler/details/jaro_impl.hpp | 36 +++++++++---------- 15 files changed, 69 insertions(+), 66 deletions(-) diff --git a/src/core_functions/scalar/list/array_slice.cpp b/src/core_functions/scalar/list/array_slice.cpp index 7651f410d023..c91247ae9f69 100644 --- a/src/core_functions/scalar/list/array_slice.cpp +++ b/src/core_functions/scalar/list/array_slice.cpp @@ -40,7 +40,7 @@ unique_ptr ListSliceBindData::Copy() const { } template -static idx_t CalculateSliceLength(idx_t begin, idx_t end, INDEX_TYPE step, bool svalid) { +static idx_t CalculateSliceLength(INDEX_TYPE begin, INDEX_TYPE end, INDEX_TYPE step, bool svalid) { if (step < 0) { step = abs(step); } @@ -48,14 +48,14 @@ static idx_t CalculateSliceLength(idx_t begin, idx_t end, INDEX_TYPE step, bool throw InvalidInputException("Slice step cannot be zero"); } if (step == 1) { - return NumericCast(end - begin); - } else if (static_cast(step) >= (end - begin)) { + return UnsafeNumericCast(end - begin); + } else if (step >= (end - begin)) { return 1; } if ((end - begin) % step != 0) { - return (end - begin) / step + 1; + return UnsafeNumericCast((end - begin) / step + 1); } - return (end - begin) / step; + return UnsafeNumericCast((end - begin) / step); } template @@ -64,7 +64,7 @@ INDEX_TYPE ValueLength(const INPUT_TYPE &value) { } template <> -int64_t ValueLength(const list_entry_t &value) { +idx_t ValueLength(const list_entry_t &value) { return value.length; } @@ -119,8 +119,8 @@ INPUT_TYPE SliceValue(Vector &result, INPUT_TYPE input, INDEX_TYPE begin, INDEX_ template <> list_entry_t SliceValue(Vector &result, list_entry_t input, int64_t begin, int64_t end) { - input.offset += begin; - input.length = end - begin; + input.offset = UnsafeNumericCast(UnsafeNumericCast(input.offset) + begin); + input.length = UnsafeNumericCast(end - begin); return input; } @@ -145,14 +145,14 @@ list_entry_t SliceValueWithSteps(Vector &result, SelectionVector &sel, list_entr return input; } input.length = CalculateSliceLength(begin, end, step, true); - idx_t child_idx = input.offset + begin; + auto child_idx = UnsafeNumericCast(UnsafeNumericCast(input.offset) + begin); if (step < 0) { - child_idx = input.offset + end - 1; + child_idx = UnsafeNumericCast(UnsafeNumericCast(input.offset) + end - 1); } input.offset = sel_idx; for (idx_t i = 0; i < input.length; i++) { sel.set_index(sel_idx, child_idx); - child_idx += step; + child_idx = UnsafeNumericCast(UnsafeNumericCast(child_idx) + step); sel_idx++; } return input; diff --git a/src/core_functions/scalar/list/list_reduce.cpp b/src/core_functions/scalar/list/list_reduce.cpp index 1f619780af2f..b58a741294f5 100644 --- a/src/core_functions/scalar/list/list_reduce.cpp +++ b/src/core_functions/scalar/list/list_reduce.cpp @@ -99,7 +99,7 @@ static bool ExecuteReduce(idx_t loops, ReduceExecuteInfo &execute_info, LambdaFu } // create the index vector - Vector index_vector(Value::BIGINT(loops + 1)); + Vector index_vector(Value::BIGINT(UnsafeNumericCast(loops + 1))); // slice the left and right slice execute_info.left_slice.Slice(execute_info.left_slice, execute_info.left_sel, reduced_row_idx); diff --git a/src/core_functions/scalar/list/list_sort.cpp b/src/core_functions/scalar/list/list_sort.cpp index 1226430c3721..64733ec4ec29 100644 --- a/src/core_functions/scalar/list/list_sort.cpp +++ b/src/core_functions/scalar/list/list_sort.cpp @@ -52,8 +52,8 @@ ListSortBindData::ListSortBindData(OrderType order_type_p, OrderByNullType null_ payload_layout.Initialize(payload_types); // get the BoundOrderByNode - auto idx_col_expr = make_uniq_base(LogicalType::USMALLINT, 0); - auto lists_col_expr = make_uniq_base(child_type, 1); + auto idx_col_expr = make_uniq_base(LogicalType::USMALLINT, 0U); + auto lists_col_expr = make_uniq_base(child_type, 1U); orders.emplace_back(OrderType::ASCENDING, OrderByNullType::ORDER_DEFAULT, std::move(idx_col_expr)); orders.emplace_back(order_type, null_order, std::move(lists_col_expr)); } @@ -241,7 +241,7 @@ static void ListSortFunction(DataChunk &args, ExpressionState &state, Vector &re for (idx_t i = 0; i < count; i++) { for (idx_t j = result_data[i].offset; j < result_data[i].offset + result_data[i].length; j++) { auto b = sel_sorted.get_index(j) - result_data[i].offset; - result_entry.SetValue(j, Value::BIGINT(b + 1)); + result_entry.SetValue(j, Value::BIGINT(UnsafeNumericCast(b + 1))); } } } else { diff --git a/src/core_functions/scalar/list/list_value.cpp b/src/core_functions/scalar/list/list_value.cpp index 9aae09e2c8a4..e2ae537fb723 100644 --- a/src/core_functions/scalar/list/list_value.cpp +++ b/src/core_functions/scalar/list/list_value.cpp @@ -59,7 +59,7 @@ static unique_ptr ListValueBind(ClientContext &context, ScalarFunc StringUtil::Format("Cannot unpivot columns of types %s and %s - an explicit cast is required", child_type.ToString(), arg_type.ToString()); throw BinderException(arguments[i]->query_location, - QueryErrorContext::Format(list_arguments, error, int(error_index), false)); + QueryErrorContext::Format(list_arguments, error, error_index, false)); } else { throw BinderException(arguments[i]->query_location, "Cannot create a list of types %s and %s - an explicit cast is required", diff --git a/src/core_functions/scalar/list/range.cpp b/src/core_functions/scalar/list/range.cpp index 4bb8685388c4..d965eb30d06f 100644 --- a/src/core_functions/scalar/list/range.cpp +++ b/src/core_functions/scalar/list/range.cpp @@ -80,7 +80,7 @@ struct TimestampRangeInfo { if (start_value < end_value && is_negative) { return 0; } - int64_t total_values = 0; + uint64_t total_values = 0; if (is_negative) { // negative interval, start_value is going down while (inclusive_bound ? start_value >= end_value : start_value > end_value) { @@ -203,7 +203,7 @@ static void ListRangeFunction(DataChunk &args, ExpressionState &state, Vector &r } auto list_data = FlatVector::GetData(result); auto &result_validity = FlatVector::Validity(result); - int64_t total_size = 0; + uint64_t total_size = 0; for (idx_t i = 0; i < args_size; i++) { if (!info.RowIsValid(i)) { result_validity.SetInvalid(i); diff --git a/src/core_functions/scalar/map/map_extract.cpp b/src/core_functions/scalar/map/map_extract.cpp index 2986a7f6b48a..8f6c4a8b03f5 100644 --- a/src/core_functions/scalar/map/map_extract.cpp +++ b/src/core_functions/scalar/map/map_extract.cpp @@ -58,7 +58,7 @@ void FillResult(Vector &map, Vector &offsets, Vector &result, idx_t count) { auto &value_list_entry = UnifiedVectorFormat::GetData(map_data)[value_index]; // Add the values to the result - idx_t list_offset = value_list_entry.offset + offset; + idx_t list_offset = value_list_entry.offset + UnsafeNumericCast(offset); // All keys are unique, only one will ever match idx_t length = 1; ListVector::Append(result, values_entries, length + list_offset, list_offset); diff --git a/src/core_functions/scalar/operators/bitwise.cpp b/src/core_functions/scalar/operators/bitwise.cpp index 73f968bb5377..d9bb58950272 100644 --- a/src/core_functions/scalar/operators/bitwise.cpp +++ b/src/core_functions/scalar/operators/bitwise.cpp @@ -251,7 +251,7 @@ static void BitwiseShiftLeftOperation(DataChunk &args, ExpressionState &state, V Bit::SetEmptyBitString(target, input); return target; } - Bit::LeftShift(input, shift, target); + Bit::LeftShift(input, UnsafeNumericCast(shift), target); return target; }); } @@ -294,7 +294,7 @@ static void BitwiseShiftRightOperation(DataChunk &args, ExpressionState &state, Bit::SetEmptyBitString(target, input); return target; } - Bit::RightShift(input, shift, target); + Bit::RightShift(input, UnsafeNumericCast(shift), target); return target; }); } diff --git a/src/core_functions/scalar/string/bar.cpp b/src/core_functions/scalar/string/bar.cpp index 291553a3a2d0..e9cd400cc426 100644 --- a/src/core_functions/scalar/string/bar.cpp +++ b/src/core_functions/scalar/string/bar.cpp @@ -40,7 +40,7 @@ static string_t BarScalarFunction(double x, double min, double max, double max_w result.clear(); - int32_t width_as_int = static_cast(width * PARTIAL_BLOCKS_COUNT); + auto width_as_int = NumericCast(width * PARTIAL_BLOCKS_COUNT); idx_t full_blocks_count = (width_as_int / PARTIAL_BLOCKS_COUNT); for (idx_t i = 0; i < full_blocks_count; i++) { result += FULL_BLOCK; diff --git a/src/core_functions/scalar/string/chr.cpp b/src/core_functions/scalar/string/chr.cpp index e7bb62e1db57..20947cd2f816 100644 --- a/src/core_functions/scalar/string/chr.cpp +++ b/src/core_functions/scalar/string/chr.cpp @@ -16,7 +16,7 @@ struct ChrOperator { char c[5] = {'\0', '\0', '\0', '\0', '\0'}; int utf8_bytes; GetCodepoint(input, c, utf8_bytes); - return string_t(&c[0], utf8_bytes); + return string_t(&c[0], UnsafeNumericCast(utf8_bytes)); } }; diff --git a/src/core_functions/scalar/string/hex.cpp b/src/core_functions/scalar/string/hex.cpp index dffbae70d030..6afaeadb56a4 100644 --- a/src/core_functions/scalar/string/hex.cpp +++ b/src/core_functions/scalar/string/hex.cpp @@ -90,7 +90,8 @@ struct HexIntegralOperator { template static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) { - idx_t num_leading_zero = CountZeros::Leading(input); + auto num_leading_zero = + UnsafeNumericCast(CountZeros::Leading(UnsafeNumericCast(input))); idx_t num_bits_to_check = 64 - num_leading_zero; D_ASSERT(num_bits_to_check <= sizeof(INPUT_TYPE) * 8); @@ -109,7 +110,7 @@ struct HexIntegralOperator { auto target = StringVector::EmptyString(result, buffer_size); auto output = target.GetDataWriteable(); - WriteHexBytes(input, output, buffer_size); + WriteHexBytes(UnsafeNumericCast(input), output, buffer_size); target.Finalize(); return target; @@ -120,7 +121,7 @@ struct HexHugeIntOperator { template static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) { - idx_t num_leading_zero = CountZeros::Leading(input); + auto num_leading_zero = UnsafeNumericCast(CountZeros::Leading(input)); idx_t buffer_size = sizeof(INPUT_TYPE) * 2 - (num_leading_zero / 4); // Special case: All bits are zero @@ -147,7 +148,7 @@ struct HexUhugeIntOperator { template static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) { - idx_t num_leading_zero = CountZeros::Leading(input); + auto num_leading_zero = UnsafeNumericCast(CountZeros::Leading(input)); idx_t buffer_size = sizeof(INPUT_TYPE) * 2 - (num_leading_zero / 4); // Special case: All bits are zero @@ -189,7 +190,7 @@ struct BinaryStrOperator { auto output = target.GetDataWriteable(); for (idx_t i = 0; i < size; ++i) { - uint8_t byte = data[i]; + auto byte = UnsafeNumericCast(data[i]); for (idx_t i = 8; i >= 1; --i) { *output = ((byte >> (i - 1)) & 0x01) + '0'; output++; @@ -205,7 +206,8 @@ struct BinaryIntegralOperator { template static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) { - idx_t num_leading_zero = CountZeros::Leading(input); + auto num_leading_zero = + UnsafeNumericCast(CountZeros::Leading(UnsafeNumericCast(input))); idx_t num_bits_to_check = 64 - num_leading_zero; D_ASSERT(num_bits_to_check <= sizeof(INPUT_TYPE) * 8); @@ -224,7 +226,7 @@ struct BinaryIntegralOperator { auto target = StringVector::EmptyString(result, buffer_size); auto output = target.GetDataWriteable(); - WriteBinBytes(input, output, buffer_size); + WriteBinBytes(UnsafeNumericCast(input), output, buffer_size); target.Finalize(); return target; @@ -234,7 +236,7 @@ struct BinaryIntegralOperator { struct BinaryHugeIntOperator { template static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) { - idx_t num_leading_zero = CountZeros::Leading(input); + auto num_leading_zero = UnsafeNumericCast(CountZeros::Leading(input)); idx_t buffer_size = sizeof(INPUT_TYPE) * 8 - num_leading_zero; // Special case: All bits are zero @@ -259,7 +261,7 @@ struct BinaryHugeIntOperator { struct BinaryUhugeIntOperator { template static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) { - idx_t num_leading_zero = CountZeros::Leading(input); + auto num_leading_zero = UnsafeNumericCast(CountZeros::Leading(input)); idx_t buffer_size = sizeof(INPUT_TYPE) * 8 - num_leading_zero; // Special case: All bits are zero @@ -301,7 +303,7 @@ struct FromHexOperator { // Treated as a single byte idx_t i = 0; if (size % 2 != 0) { - *output = StringUtil::GetHexValue(data[i]); + *output = UnsafeNumericCast(StringUtil::GetHexValue(data[i])); i++; output++; } @@ -309,7 +311,7 @@ struct FromHexOperator { for (; i < size; i += 2) { uint8_t major = StringUtil::GetHexValue(data[i]); uint8_t minor = StringUtil::GetHexValue(data[i + 1]); - *output = UnsafeNumericCast((major << 4) | minor); + *output = UnsafeNumericCast((major << 4) | minor); output++; } @@ -343,7 +345,7 @@ struct FromBinaryOperator { byte |= StringUtil::GetBinaryValue(data[i]) << (j - 1); i++; } - *output = byte; + *output = UnsafeNumericCast(byte); output++; } @@ -353,7 +355,8 @@ struct FromBinaryOperator { byte |= StringUtil::GetBinaryValue(data[i]) << (j - 1); i++; } - *output = byte; + *output = UnsafeNumericCast(byte); + ; output++; } diff --git a/src/core_functions/scalar/string/instr.cpp b/src/core_functions/scalar/string/instr.cpp index becbbd4896f3..66608db60820 100644 --- a/src/core_functions/scalar/string/instr.cpp +++ b/src/core_functions/scalar/string/instr.cpp @@ -33,7 +33,7 @@ struct InstrAsciiOperator { template static inline TR Operation(TA haystack, TB needle) { auto location = ContainsFun::Find(haystack, needle); - return location == DConstants::INVALID_INDEX ? 0 : location + 1; + return UnsafeNumericCast(location == DConstants::INVALID_INDEX ? 0U : location + 1U); } }; diff --git a/src/include/duckdb/function/scalar/string_functions.hpp b/src/include/duckdb/function/scalar/string_functions.hpp index 3fc6cf62978d..c96c2b3c33c3 100644 --- a/src/include/duckdb/function/scalar/string_functions.hpp +++ b/src/include/duckdb/function/scalar/string_functions.hpp @@ -78,7 +78,7 @@ struct LengthFun { return length; } } - return input_length; + return UnsafeNumericCast(input_length); } }; diff --git a/third_party/jaro_winkler/details/common.hpp b/third_party/jaro_winkler/details/common.hpp index 0a3193fb5b65..6855ce60af01 100644 --- a/third_party/jaro_winkler/details/common.hpp +++ b/third_party/jaro_winkler/details/common.hpp @@ -91,7 +91,7 @@ struct BitvectorHashmap { void insert_mask(CharT key, uint64_t mask) { uint64_t i = lookup(static_cast(key)); - m_map[i].key = key; + m_map[i].key = static_cast(key); m_map[i].value |= mask; } @@ -150,7 +150,7 @@ struct PatternMatchVector { for (int64_t i = 0; i < std::distance(first, last); ++i) { auto key = first[i]; if (key >= 0 && key <= 255) { - m_extendedAscii[key] |= mask; + m_extendedAscii[static_cast(key)] |= mask; } else { m_map.insert_mask(key, mask); @@ -175,7 +175,7 @@ struct PatternMatchVector { uint64_t get(CharT key) const { if (key >= 0 && key <= 255) { - return m_extendedAscii[key]; + return m_extendedAscii[static_cast(key)]; } else { return m_map.get(key); @@ -215,10 +215,10 @@ struct BlockPatternMatchVector { assert(block < m_block_count); if (key >= 0 && key <= 255) { - m_extendedAscii[key * m_block_count + block] |= mask; + m_extendedAscii[static_cast(key * m_block_count + block)] |= mask; } else { - m_map[block].insert_mask(key, mask); + m_map[static_cast(block)].insert_mask(key, mask); } } @@ -227,8 +227,8 @@ struct BlockPatternMatchVector { { int64_t len = std::distance(first, last); m_block_count = ceildiv(len, 64); - m_map.resize(m_block_count); - m_extendedAscii.resize(m_block_count * 256); + m_map.resize(static_cast(m_block_count)); + m_extendedAscii.resize(static_cast(m_block_count * 256)); for (int64_t i = 0; i < len; ++i) { int64_t block = i / 64; @@ -251,10 +251,10 @@ struct BlockPatternMatchVector { { assert(block < m_block_count); if (key >= 0 && key <= 255) { - return m_extendedAscii[key * m_block_count + block]; + return m_extendedAscii[static_cast(key * m_block_count + block)]; } else { - return m_map[block].get(key); + return m_map[static_cast(block)].get(key); } } diff --git a/third_party/jaro_winkler/details/intrinsics.hpp b/third_party/jaro_winkler/details/intrinsics.hpp index 174bdccd841b..45f07c407f16 100644 --- a/third_party/jaro_winkler/details/intrinsics.hpp +++ b/third_party/jaro_winkler/details/intrinsics.hpp @@ -15,7 +15,7 @@ namespace intrinsics { template T bit_mask_lsb(int n) { - T mask = -1; + T mask = static_cast(-1); if (n < static_cast(sizeof(T) * 8)) { mask += static_cast(1) << n; } diff --git a/third_party/jaro_winkler/details/jaro_impl.hpp b/third_party/jaro_winkler/details/jaro_impl.hpp index 7b8f28b893ca..47e6fa506c1c 100644 --- a/third_party/jaro_winkler/details/jaro_impl.hpp +++ b/third_party/jaro_winkler/details/jaro_impl.hpp @@ -147,39 +147,39 @@ static inline void flag_similar_characters_step(const common::BlockPatternMatchV if (BoundMask.words == 1) { uint64_t PM_j = PM.get(word, T_j) & BoundMask.last_mask & BoundMask.first_mask & - (~flagged.P_flag[word]); + (~flagged.P_flag[static_cast(word)]); - flagged.P_flag[word] |= blsi(PM_j); - flagged.T_flag[j_word] |= static_cast(PM_j != 0) << j_pos; + flagged.P_flag[static_cast(word)] |= blsi(PM_j); + flagged.T_flag[static_cast(j_word)] |= static_cast(PM_j != 0) << j_pos; return; } if (BoundMask.first_mask) { - uint64_t PM_j = PM.get(word, T_j) & BoundMask.first_mask & (~flagged.P_flag[word]); + uint64_t PM_j = PM.get(word, T_j) & BoundMask.first_mask & (~flagged.P_flag[static_cast(word)]); if (PM_j) { - flagged.P_flag[word] |= blsi(PM_j); - flagged.T_flag[j_word] |= 1ull << j_pos; + flagged.P_flag[static_cast(word)] |= blsi(PM_j); + flagged.T_flag[static_cast(j_word)] |= 1ull << j_pos; return; } word++; } for (; word < last_word - 1; ++word) { - uint64_t PM_j = PM.get(word, T_j) & (~flagged.P_flag[word]); + uint64_t PM_j = PM.get(word, T_j) & (~flagged.P_flag[static_cast(word)]); if (PM_j) { - flagged.P_flag[word] |= blsi(PM_j); - flagged.T_flag[j_word] |= 1ull << j_pos; + flagged.P_flag[static_cast(word)] |= blsi(PM_j); + flagged.T_flag[static_cast(j_word)] |= 1ull << j_pos; return; } } if (BoundMask.last_mask) { - uint64_t PM_j = PM.get(word, T_j) & BoundMask.last_mask & (~flagged.P_flag[word]); + uint64_t PM_j = PM.get(word, T_j) & BoundMask.last_mask & (~flagged.P_flag[static_cast(word)]); - flagged.P_flag[word] |= blsi(PM_j); - flagged.T_flag[j_word] |= static_cast(PM_j != 0) << j_pos; + flagged.P_flag[static_cast(word)] |= blsi(PM_j); + flagged.T_flag[static_cast(j_word)] |= static_cast(PM_j != 0) << j_pos; } } @@ -199,8 +199,8 @@ flag_similar_characters_block(const common::BlockPatternMatchVector& PM, InputIt int64_t PatternWords = common::ceildiv(P_len, 64); FlaggedCharsMultiword flagged; - flagged.T_flag.resize(TextWords); - flagged.P_flag.resize(PatternWords); + flagged.T_flag.resize(static_cast(TextWords)); + flagged.P_flag.resize(static_cast(PatternWords)); SearchBoundMask BoundMask; int64_t start_range = std::min(Bound + 1, P_len); @@ -262,21 +262,21 @@ count_transpositions_block(const common::BlockPatternMatchVector& PM, InputIt1 T using namespace intrinsics; int64_t TextWord = 0; int64_t PatternWord = 0; - uint64_t T_flag = flagged.T_flag[TextWord]; - uint64_t P_flag = flagged.P_flag[PatternWord]; + uint64_t T_flag = flagged.T_flag[static_cast(TextWord)]; + uint64_t P_flag = flagged.P_flag[static_cast(PatternWord)]; int64_t Transpositions = 0; while (FlaggedChars) { while (!T_flag) { TextWord++; T_first += 64; - T_flag = flagged.T_flag[TextWord]; + T_flag = flagged.T_flag[static_cast(TextWord)]; } while (T_flag) { while (!P_flag) { PatternWord++; - P_flag = flagged.P_flag[PatternWord]; + P_flag = flagged.P_flag[static_cast(PatternWord)]; } uint64_t PatternFlagMask = blsi(P_flag); From c0b849d1d5c09a93aae12cf65ae4d238ce63c8d9 Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 3 Apr 2024 13:11:15 +0200 Subject: [PATCH 134/603] dont serialize 'last_value', just recreate it instead --- .../catalog_entry/sequence_catalog_entry.cpp | 18 ++++++++++++++---- .../parsed_data/create_sequence_info.hpp | 2 -- .../storage/serialization/create_info.json | 5 ----- .../parsed_data/create_sequence_info.cpp | 1 - .../serialization/serialize_create_info.cpp | 2 -- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/catalog/catalog_entry/sequence_catalog_entry.cpp b/src/catalog/catalog_entry/sequence_catalog_entry.cpp index e421e5f0695e..27307f956fbb 100644 --- a/src/catalog/catalog_entry/sequence_catalog_entry.cpp +++ b/src/catalog/catalog_entry/sequence_catalog_entry.cpp @@ -13,10 +13,21 @@ namespace duckdb { +// This should only be used for deserialization purposes, as we can guarantee that the result is within bounds in that +// case. +static int64_t LastValue(CreateSequenceInfo &info) { + auto usage_count = info.usage_count; + if (usage_count == 0) { + return info.start_value; + } + usage_count--; + // FIXME: this is not accounting for CYCLE behavior + return info.start_value + (info.increment * usage_count); +} + SequenceData::SequenceData(CreateSequenceInfo &info) - : usage_count(info.usage_count), counter(info.start_value + (info.increment * info.usage_count)), - last_value(info.last_value), increment(info.increment), start_value(info.start_value), min_value(info.min_value), - max_value(info.max_value), cycle(info.cycle) { + : usage_count(info.usage_count), counter(LastValue(info)), last_value(LastValue(info)), increment(info.increment), + start_value(info.start_value), min_value(info.min_value), max_value(info.max_value), cycle(info.cycle) { } SequenceCatalogEntry::SequenceCatalogEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateSequenceInfo &info) @@ -94,7 +105,6 @@ unique_ptr SequenceCatalogEntry::GetInfo() const { result->schema = schema.name; result->name = name; result->usage_count = seq_data.usage_count; - result->last_value = seq_data.last_value; result->increment = seq_data.increment; result->min_value = seq_data.min_value; result->max_value = seq_data.max_value; diff --git a/src/include/duckdb/parser/parsed_data/create_sequence_info.hpp b/src/include/duckdb/parser/parsed_data/create_sequence_info.hpp index 0e5182863f06..de8fcc572f75 100644 --- a/src/include/duckdb/parser/parsed_data/create_sequence_info.hpp +++ b/src/include/duckdb/parser/parsed_data/create_sequence_info.hpp @@ -45,8 +45,6 @@ struct CreateSequenceInfo : public CreateInfo { int64_t start_value; //! Whether or not the sequence cycles bool cycle; - //! Last value of the sequence - int64_t last_value; public: unique_ptr Copy() const override; diff --git a/src/include/duckdb/storage/serialization/create_info.json b/src/include/duckdb/storage/serialization/create_info.json index fa8fe7dc339f..c850a71e854a 100644 --- a/src/include/duckdb/storage/serialization/create_info.json +++ b/src/include/duckdb/storage/serialization/create_info.json @@ -280,11 +280,6 @@ "id": 206, "name": "cycle", "type": "bool" - }, - { - "id": 207, - "name": "last_value", - "type": "int64_t" } ] } diff --git a/src/parser/parsed_data/create_sequence_info.cpp b/src/parser/parsed_data/create_sequence_info.cpp index a5bbb604a470..d7caac026f83 100644 --- a/src/parser/parsed_data/create_sequence_info.cpp +++ b/src/parser/parsed_data/create_sequence_info.cpp @@ -21,7 +21,6 @@ unique_ptr CreateSequenceInfo::Copy() const { result->max_value = max_value; result->start_value = start_value; result->cycle = cycle; - result->last_value = last_value; return std::move(result); } diff --git a/src/storage/serialization/serialize_create_info.cpp b/src/storage/serialization/serialize_create_info.cpp index 5eb7070e9ade..3196a1d107d0 100644 --- a/src/storage/serialization/serialize_create_info.cpp +++ b/src/storage/serialization/serialize_create_info.cpp @@ -137,7 +137,6 @@ void CreateSequenceInfo::Serialize(Serializer &serializer) const { serializer.WritePropertyWithDefault(204, "max_value", max_value); serializer.WritePropertyWithDefault(205, "start_value", start_value); serializer.WritePropertyWithDefault(206, "cycle", cycle); - serializer.WritePropertyWithDefault(207, "last_value", last_value); } unique_ptr CreateSequenceInfo::Deserialize(Deserializer &deserializer) { @@ -149,7 +148,6 @@ unique_ptr CreateSequenceInfo::Deserialize(Deserializer &deserialize deserializer.ReadPropertyWithDefault(204, "max_value", result->max_value); deserializer.ReadPropertyWithDefault(205, "start_value", result->start_value); deserializer.ReadPropertyWithDefault(206, "cycle", result->cycle); - deserializer.ReadPropertyWithDefault(207, "last_value", result->last_value); return std::move(result); } From 4a8819fe1388431cbac1f290f135b2332ef023eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Wed, 3 Apr 2024 13:18:41 +0200 Subject: [PATCH 135/603] still stuck in scalars --- src/core_functions/scalar/string/pad.cpp | 16 ++--- .../scalar/string/string_split.cpp | 2 +- src/core_functions/scalar/string/to_base.cpp | 2 +- .../scalar/string/translate.cpp | 8 +-- src/core_functions/scalar/string/trim.cpp | 22 +++--- src/core_functions/scalar/string/unicode.cpp | 2 +- src/function/aggregate/distributive/first.cpp | 2 +- src/function/pragma/pragma_queries.cpp | 4 +- .../compress_string.cpp | 2 +- src/function/scalar/list/list_extract.cpp | 4 +- src/function/scalar/list/list_resize.cpp | 4 +- src/function/scalar/list/list_select.cpp | 3 +- src/function/scalar/strftime_format.cpp | 67 ++++++++++--------- src/function/scalar/string/caseconvert.cpp | 8 +-- src/include/duckdb/common/types/bit.hpp | 2 +- 15 files changed, 78 insertions(+), 70 deletions(-) diff --git a/src/core_functions/scalar/string/pad.cpp b/src/core_functions/scalar/string/pad.cpp index 859142293153..856544ea7d93 100644 --- a/src/core_functions/scalar/string/pad.cpp +++ b/src/core_functions/scalar/string/pad.cpp @@ -17,9 +17,9 @@ static pair PadCountChars(const idx_t len, const char *data, const idx_t nchars = 0; for (; nchars < len && nbytes < size; ++nchars) { utf8proc_int32_t codepoint; - auto bytes = utf8proc_iterate(str + nbytes, size - nbytes, &codepoint); + auto bytes = utf8proc_iterate(str + nbytes, UnsafeNumericCast(size - nbytes), &codepoint); D_ASSERT(bytes > 0); - nbytes += bytes; + nbytes += UnsafeNumericCast(bytes); } return pair(nbytes, nchars); @@ -47,9 +47,9 @@ static bool InsertPadding(const idx_t len, const string_t &pad, vector &re // Write the next character utf8proc_int32_t codepoint; - auto bytes = utf8proc_iterate(str + nbytes, size - nbytes, &codepoint); + auto bytes = utf8proc_iterate(str + nbytes, UnsafeNumericCast(size - nbytes), &codepoint); D_ASSERT(bytes > 0); - nbytes += bytes; + nbytes += UnsafeNumericCast(bytes); } // Flush the remaining pad @@ -67,10 +67,10 @@ static string_t LeftPadFunction(const string_t &str, const int32_t len, const st auto size_str = str.GetSize(); // Count how much of str will fit in the output - auto written = PadCountChars(len, data_str, size_str); + auto written = PadCountChars(UnsafeNumericCast(len), data_str, size_str); // Left pad by the number of characters still needed - if (!InsertPadding(len - written.second, pad, result)) { + if (!InsertPadding(UnsafeNumericCast(len) - written.second, pad, result)) { throw InvalidInputException("Insufficient padding in LPAD."); } @@ -96,13 +96,13 @@ static string_t RightPadFunction(const string_t &str, const int32_t len, const s auto size_str = str.GetSize(); // Count how much of str will fit in the output - auto written = PadCountChars(len, data_str, size_str); + auto written = PadCountChars(UnsafeNumericCast(len), data_str, size_str); // Append as much of the original string as fits result.insert(result.end(), data_str, data_str + written.first); // Right pad by the number of characters still needed - if (!InsertPadding(len - written.second, pad, result)) { + if (!InsertPadding(UnsafeNumericCast(len) - written.second, pad, result)) { throw InvalidInputException("Insufficient padding in RPAD."); }; diff --git a/src/core_functions/scalar/string/string_split.cpp b/src/core_functions/scalar/string/string_split.cpp index 9d3ea23be52c..c62cacd753e5 100644 --- a/src/core_functions/scalar/string/string_split.cpp +++ b/src/core_functions/scalar/string/string_split.cpp @@ -51,7 +51,7 @@ struct ConstantRegexpStringSplit { return DConstants::INVALID_INDEX; } match_size = match.size(); - return match.data() - input_data; + return UnsafeNumericCast(match.data() - input_data); } }; diff --git a/src/core_functions/scalar/string/to_base.cpp b/src/core_functions/scalar/string/to_base.cpp index ad5e1088ef24..963f4f562f42 100644 --- a/src/core_functions/scalar/string/to_base.cpp +++ b/src/core_functions/scalar/string/to_base.cpp @@ -48,7 +48,7 @@ static void ToBaseFunction(DataChunk &args, ExpressionState &state, Vector &resu length++; } - return StringVector::AddString(result, ptr, end - ptr); + return StringVector::AddString(result, ptr, UnsafeNumericCast(end - ptr)); }); } diff --git a/src/core_functions/scalar/string/translate.cpp b/src/core_functions/scalar/string/translate.cpp index 192438697b74..c01ec2a25093 100644 --- a/src/core_functions/scalar/string/translate.cpp +++ b/src/core_functions/scalar/string/translate.cpp @@ -37,10 +37,10 @@ static string_t TranslateScalarFunction(const string_t &haystack, const string_t while (i < size_needle && j < size_thread) { auto codepoint_needle = Utf8Proc::UTF8ToCodepoint(input_needle, sz); input_needle += sz; - i += sz; + i += UnsafeNumericCast(sz); auto codepoint_thread = Utf8Proc::UTF8ToCodepoint(input_thread, sz); input_thread += sz; - j += sz; + j += UnsafeNumericCast(sz); // Ignore unicode character that is existed in to_replace if (to_replace.count(codepoint_needle) == 0) { to_replace[codepoint_needle] = codepoint_thread; @@ -52,7 +52,7 @@ static string_t TranslateScalarFunction(const string_t &haystack, const string_t while (i < size_needle) { auto codepoint_needle = Utf8Proc::UTF8ToCodepoint(input_needle, sz); input_needle += sz; - i += sz; + i += UnsafeNumericCast(sz); // Add unicode character that will be deleted if (to_replace.count(codepoint_needle) == 0) { to_delete.insert(codepoint_needle); @@ -60,7 +60,7 @@ static string_t TranslateScalarFunction(const string_t &haystack, const string_t } char c[5] = {'\0', '\0', '\0', '\0', '\0'}; - for (i = 0; i < size_haystack; i += sz) { + for (i = 0; i < size_haystack; i += UnsafeNumericCast(sz)) { auto codepoint_haystack = Utf8Proc::UTF8ToCodepoint(input_haystack, sz); if (to_replace.count(codepoint_haystack) != 0) { Utf8Proc::CodepointToUtf8(to_replace[codepoint_haystack], c_sz, c); diff --git a/src/core_functions/scalar/string/trim.cpp b/src/core_functions/scalar/string/trim.cpp index 91e3b5dd1970..d89ebbaffb6d 100644 --- a/src/core_functions/scalar/string/trim.cpp +++ b/src/core_functions/scalar/string/trim.cpp @@ -23,12 +23,13 @@ struct TrimOperator { idx_t begin = 0; if (LTRIM) { while (begin < size) { - auto bytes = utf8proc_iterate(str + begin, size - begin, &codepoint); + auto bytes = + utf8proc_iterate(str + begin, UnsafeNumericCast(size - begin), &codepoint); D_ASSERT(bytes > 0); if (utf8proc_category(codepoint) != UTF8PROC_CATEGORY_ZS) { break; } - begin += bytes; + begin += UnsafeNumericCast(bytes); } } @@ -37,9 +38,9 @@ struct TrimOperator { if (RTRIM) { end = begin; for (auto next = begin; next < size;) { - auto bytes = utf8proc_iterate(str + next, size - next, &codepoint); + auto bytes = utf8proc_iterate(str + next, UnsafeNumericCast(size - next), &codepoint); D_ASSERT(bytes > 0); - next += bytes; + next += UnsafeNumericCast(bytes); if (utf8proc_category(codepoint) != UTF8PROC_CATEGORY_ZS) { end = next; } @@ -69,7 +70,8 @@ static void GetIgnoredCodepoints(string_t ignored, unordered_set( + utf8proc_iterate(dataptr + pos, UnsafeNumericCast(size - pos), &codepoint)); ignored_codepoints.insert(codepoint); } } @@ -91,11 +93,12 @@ static void BinaryTrimFunction(DataChunk &input, ExpressionState &state, Vector idx_t begin = 0; if (LTRIM) { while (begin < size) { - auto bytes = utf8proc_iterate(str + begin, size - begin, &codepoint); + auto bytes = + utf8proc_iterate(str + begin, UnsafeNumericCast(size - begin), &codepoint); if (ignored_codepoints.find(codepoint) == ignored_codepoints.end()) { break; } - begin += bytes; + begin += UnsafeNumericCast(bytes); } } @@ -104,9 +107,10 @@ static void BinaryTrimFunction(DataChunk &input, ExpressionState &state, Vector if (RTRIM) { end = begin; for (auto next = begin; next < size;) { - auto bytes = utf8proc_iterate(str + next, size - next, &codepoint); + auto bytes = + utf8proc_iterate(str + next, UnsafeNumericCast(size - next), &codepoint); D_ASSERT(bytes > 0); - next += bytes; + next += UnsafeNumericCast(bytes); if (ignored_codepoints.find(codepoint) == ignored_codepoints.end()) { end = next; } diff --git a/src/core_functions/scalar/string/unicode.cpp b/src/core_functions/scalar/string/unicode.cpp index b621c53202f1..b62a129aad61 100644 --- a/src/core_functions/scalar/string/unicode.cpp +++ b/src/core_functions/scalar/string/unicode.cpp @@ -15,7 +15,7 @@ struct UnicodeOperator { auto str = reinterpret_cast(input.GetData()); auto len = input.GetSize(); utf8proc_int32_t codepoint; - (void)utf8proc_iterate(str, len, &codepoint); + (void)utf8proc_iterate(str, UnsafeNumericCast(len), &codepoint); return codepoint; } }; diff --git a/src/function/aggregate/distributive/first.cpp b/src/function/aggregate/distributive/first.cpp index 2d2aa0530395..143cf4317573 100644 --- a/src/function/aggregate/distributive/first.cpp +++ b/src/function/aggregate/distributive/first.cpp @@ -88,7 +88,7 @@ struct FirstFunctionString : public FirstFunctionBase { auto ptr = LAST ? new char[len] : char_ptr_cast(input_data.allocator.Allocate(len)); memcpy(ptr, value.GetData(), len); - state.value = string_t(ptr, UnsafeNumericCast(len)); + state.value = string_t(ptr, UnsafeNumericCast(len)); } } } diff --git a/src/function/pragma/pragma_queries.cpp b/src/function/pragma/pragma_queries.cpp index a69d735bc826..d1033bbc55d2 100644 --- a/src/function/pragma/pragma_queries.cpp +++ b/src/function/pragma/pragma_queries.cpp @@ -141,9 +141,9 @@ string PragmaImportDatabase(ClientContext &context, const FunctionParameters &pa auto file_path = fs.JoinPath(parameters.values[0].ToString(), file); auto handle = fs.OpenFile(file_path, FileFlags::FILE_FLAGS_READ); auto fsize = fs.GetFileSize(*handle); - auto buffer = make_unsafe_uniq_array(fsize); + auto buffer = make_unsafe_uniq_array(UnsafeNumericCast(fsize)); fs.Read(*handle, buffer.get(), fsize); - auto query = string(buffer.get(), fsize); + auto query = string(buffer.get(), UnsafeNumericCast(fsize)); // Replace the placeholder with the path provided to IMPORT if (file == "load.sql") { Parser parser; diff --git a/src/function/scalar/compressed_materialization/compress_string.cpp b/src/function/scalar/compressed_materialization/compress_string.cpp index 1125322fcb6e..a8e046b60bcf 100644 --- a/src/function/scalar/compressed_materialization/compress_string.cpp +++ b/src/function/scalar/compressed_materialization/compress_string.cpp @@ -39,7 +39,7 @@ static inline RESULT_TYPE StringCompressInternal(const string_t &input) { ReverseMemCpy(result_ptr + remainder, data_ptr_cast(input.GetPointer()), input.GetSize()); memset(result_ptr, '\0', remainder); } - result_ptr[0] = UnsafeNumericCast(input.GetSize()); + result_ptr[0] = UnsafeNumericCast(input.GetSize()); return result; } diff --git a/src/function/scalar/list/list_extract.cpp b/src/function/scalar/list/list_extract.cpp index 822b9079cdc9..e1566641aafb 100644 --- a/src/function/scalar/list/list_extract.cpp +++ b/src/function/scalar/list/list_extract.cpp @@ -62,13 +62,13 @@ void ListExtractTemplate(idx_t count, UnifiedVectorFormat &list_data, UnifiedVec result_mask.SetInvalid(i); continue; } - child_offset = list_entry.offset + list_entry.length + offsets_entry; + child_offset = list_entry.offset + list_entry.length + UnsafeNumericCast(offsets_entry); } else { if ((idx_t)offsets_entry >= list_entry.length) { result_mask.SetInvalid(i); continue; } - child_offset = list_entry.offset + offsets_entry; + child_offset = list_entry.offset + UnsafeNumericCast(offsets_entry); } auto child_index = child_format.sel->get_index(child_offset); if (child_format.validity.RowIsValid(child_index)) { diff --git a/src/function/scalar/list/list_resize.cpp b/src/function/scalar/list/list_resize.cpp index edd6ca29108b..04f494139cd4 100644 --- a/src/function/scalar/list/list_resize.cpp +++ b/src/function/scalar/list/list_resize.cpp @@ -38,7 +38,7 @@ void ListResizeFunction(DataChunk &args, ExpressionState &state, Vector &result) for (idx_t i = 0; i < count; i++) { auto index = new_size_data.sel->get_index(i); if (new_size_data.validity.RowIsValid(index)) { - new_child_size += new_size_entries[index]; + new_child_size += UnsafeNumericCast(new_size_entries[index]); } } @@ -72,7 +72,7 @@ void ListResizeFunction(DataChunk &args, ExpressionState &state, Vector &result) idx_t new_size_entry = 0; if (new_size_data.validity.RowIsValid(new_index)) { - new_size_entry = new_size_entries[new_index]; + new_size_entry = UnsafeNumericCast(new_size_entries[new_index]); } // find the smallest size between lists and new_sizes diff --git a/src/function/scalar/list/list_select.cpp b/src/function/scalar/list/list_select.cpp index 1878f808ad30..9d45de5a1ecc 100644 --- a/src/function/scalar/list/list_select.cpp +++ b/src/function/scalar/list/list_select.cpp @@ -11,7 +11,8 @@ struct SetSelectionVectorSelect { ValidityMask &input_validity, Vector &selection_entry, idx_t child_idx, idx_t &target_offset, idx_t selection_offset, idx_t input_offset, idx_t target_length) { - idx_t sel_idx = selection_entry.GetValue(selection_offset + child_idx).GetValue() - 1; + auto sel_idx = + UnsafeNumericCast(selection_entry.GetValue(selection_offset + child_idx).GetValue() - 1); if (sel_idx < target_length) { selection_vector.set_index(target_offset, input_offset + sel_idx); if (!input_validity.RowIsValid(input_offset + sel_idx)) { diff --git a/src/function/scalar/strftime_format.cpp b/src/function/scalar/strftime_format.cpp index f50f3dac4b7f..9a948287c3a6 100644 --- a/src/function/scalar/strftime_format.cpp +++ b/src/function/scalar/strftime_format.cpp @@ -80,7 +80,7 @@ idx_t StrfTimeFormat::GetSpecifierLength(StrTimeSpecifier specifier, date_t date if (0 <= year && year <= 9999) { return 4; } else { - return NumericHelper::SignedLength(year); + return UnsafeNumericCast(NumericHelper::SignedLength(year)); } } case StrTimeSpecifier::MONTH_DECIMAL: { @@ -129,11 +129,14 @@ idx_t StrfTimeFormat::GetSpecifierLength(StrTimeSpecifier specifier, date_t date return len; } case StrTimeSpecifier::DAY_OF_MONTH: - return NumericHelper::UnsignedLength(Date::ExtractDay(date)); + return UnsafeNumericCast( + NumericHelper::UnsignedLength(UnsafeNumericCast(Date::ExtractDay(date)))); case StrTimeSpecifier::DAY_OF_YEAR_DECIMAL: - return NumericHelper::UnsignedLength(Date::ExtractDayOfTheYear(date)); + return UnsafeNumericCast( + NumericHelper::UnsignedLength(UnsafeNumericCast(Date::ExtractDayOfTheYear(date)))); case StrTimeSpecifier::YEAR_WITHOUT_CENTURY: - return NumericHelper::UnsignedLength(AbsValue(Date::ExtractYear(date)) % 100); + return UnsafeNumericCast(NumericHelper::UnsignedLength( + AbsValue(UnsafeNumericCast(Date::ExtractYear(date)) % 100))); default: throw InternalException("Unimplemented specifier for GetSpecifierLength"); } @@ -195,13 +198,13 @@ char *StrfTimeFormat::WritePadded(char *target, uint32_t value, size_t padding) D_ASSERT(padding > 1); if (padding % 2) { int decimals = value % 1000; - WritePadded3(target + padding - 3, decimals); + WritePadded3(target + padding - 3, UnsafeNumericCast(decimals)); value /= 1000; padding -= 3; } for (size_t i = 0; i < padding / 2; i++) { int decimals = value % 100; - WritePadded2(target + padding - 2 * (i + 1), decimals); + WritePadded2(target + padding - 2 * (i + 1), UnsafeNumericCast(decimals)); value /= 100; } return target + padding; @@ -245,26 +248,26 @@ char *StrfTimeFormat::WriteDateSpecifier(StrTimeSpecifier specifier, date_t date } case StrTimeSpecifier::DAY_OF_YEAR_PADDED: { int32_t doy = Date::ExtractDayOfTheYear(date); - target = WritePadded3(target, doy); + target = WritePadded3(target, UnsafeNumericCast(doy)); break; } case StrTimeSpecifier::WEEK_NUMBER_PADDED_MON_FIRST: - target = WritePadded2(target, Date::ExtractWeekNumberRegular(date, true)); + target = WritePadded2(target, UnsafeNumericCast(Date::ExtractWeekNumberRegular(date, true))); break; case StrTimeSpecifier::WEEK_NUMBER_PADDED_SUN_FIRST: - target = WritePadded2(target, Date::ExtractWeekNumberRegular(date, false)); + target = WritePadded2(target, UnsafeNumericCast(Date::ExtractWeekNumberRegular(date, false))); break; case StrTimeSpecifier::WEEK_NUMBER_ISO: - target = WritePadded2(target, Date::ExtractISOWeekNumber(date)); + target = WritePadded2(target, UnsafeNumericCast(Date::ExtractISOWeekNumber(date))); break; case StrTimeSpecifier::DAY_OF_YEAR_DECIMAL: { - uint32_t doy = Date::ExtractDayOfTheYear(date); + auto doy = UnsafeNumericCast(Date::ExtractDayOfTheYear(date)); target += NumericHelper::UnsignedLength(doy); NumericHelper::FormatUnsigned(doy, target); break; } case StrTimeSpecifier::YEAR_ISO: - target = WritePadded(target, Date::ExtractISOYearNumber(date), 4); + target = WritePadded(target, UnsafeNumericCast(Date::ExtractISOYearNumber(date)), 4); break; case StrTimeSpecifier::WEEKDAY_ISO: *target = char('0' + uint8_t(Date::ExtractISODayOfTheWeek(date))); @@ -281,7 +284,7 @@ char *StrfTimeFormat::WriteStandardSpecifier(StrTimeSpecifier specifier, int32_t // data contains [0] year, [1] month, [2] day, [3] hour, [4] minute, [5] second, [6] msec, [7] utc switch (specifier) { case StrTimeSpecifier::DAY_OF_MONTH_PADDED: - target = WritePadded2(target, data[2]); + target = WritePadded2(target, UnsafeNumericCast(data[2])); break; case StrTimeSpecifier::ABBREVIATED_MONTH_NAME: { auto &month_name = Date::MONTH_NAMES_ABBREVIATED[data[1] - 1]; @@ -292,14 +295,14 @@ char *StrfTimeFormat::WriteStandardSpecifier(StrTimeSpecifier specifier, int32_t return WriteString(target, month_name); } case StrTimeSpecifier::MONTH_DECIMAL_PADDED: - target = WritePadded2(target, data[1]); + target = WritePadded2(target, UnsafeNumericCast(data[1])); break; case StrTimeSpecifier::YEAR_WITHOUT_CENTURY_PADDED: - target = WritePadded2(target, AbsValue(data[0]) % 100); + target = WritePadded2(target, UnsafeNumericCast(AbsValue(data[0]) % 100)); break; case StrTimeSpecifier::YEAR_DECIMAL: if (data[0] >= 0 && data[0] <= 9999) { - target = WritePadded(target, data[0], 4); + target = WritePadded(target, UnsafeNumericCast(data[0]), 4); } else { int32_t year = data[0]; if (data[0] < 0) { @@ -307,13 +310,13 @@ char *StrfTimeFormat::WriteStandardSpecifier(StrTimeSpecifier specifier, int32_t year = -year; target++; } - auto len = NumericHelper::UnsignedLength(year); + auto len = NumericHelper::UnsignedLength(UnsafeNumericCast(year)); NumericHelper::FormatUnsigned(year, target + len); target += len; } break; case StrTimeSpecifier::HOUR_24_PADDED: { - target = WritePadded2(target, data[3]); + target = WritePadded2(target, UnsafeNumericCast(data[3])); break; } case StrTimeSpecifier::HOUR_12_PADDED: { @@ -321,7 +324,7 @@ char *StrfTimeFormat::WriteStandardSpecifier(StrTimeSpecifier specifier, int32_t if (hour == 0) { hour = 12; } - target = WritePadded2(target, hour); + target = WritePadded2(target, UnsafeNumericCast(hour)); break; } case StrTimeSpecifier::AM_PM: @@ -329,20 +332,20 @@ char *StrfTimeFormat::WriteStandardSpecifier(StrTimeSpecifier specifier, int32_t *target++ = 'M'; break; case StrTimeSpecifier::MINUTE_PADDED: { - target = WritePadded2(target, data[4]); + target = WritePadded2(target, UnsafeNumericCast(data[4])); break; } case StrTimeSpecifier::SECOND_PADDED: - target = WritePadded2(target, data[5]); + target = WritePadded2(target, UnsafeNumericCast(data[5])); break; case StrTimeSpecifier::NANOSECOND_PADDED: - target = WritePadded(target, data[6] * Interval::NANOS_PER_MICRO, 9); + target = WritePadded(target, UnsafeNumericCast(data[6] * Interval::NANOS_PER_MICRO), 9); break; case StrTimeSpecifier::MICROSECOND_PADDED: - target = WritePadded(target, data[6], 6); + target = WritePadded(target, UnsafeNumericCast(data[6]), 6); break; case StrTimeSpecifier::MILLISECOND_PADDED: - target = WritePadded3(target, data[6] / Interval::MICROS_PER_MSEC); + target = WritePadded3(target, UnsafeNumericCast(data[6] / Interval::MICROS_PER_MSEC)); break; case StrTimeSpecifier::UTC_OFFSET: { *target++ = (data[7] < 0) ? '-' : '+'; @@ -350,10 +353,10 @@ char *StrfTimeFormat::WriteStandardSpecifier(StrTimeSpecifier specifier, int32_t auto offset = abs(data[7]); auto offset_hours = offset / Interval::MINS_PER_HOUR; auto offset_minutes = offset % Interval::MINS_PER_HOUR; - target = WritePadded2(target, offset_hours); + target = WritePadded2(target, UnsafeNumericCast(offset_hours)); if (offset_minutes) { *target++ = ':'; - target = WritePadded2(target, offset_minutes); + target = WritePadded2(target, UnsafeNumericCast(offset_minutes)); } break; } @@ -364,7 +367,7 @@ char *StrfTimeFormat::WriteStandardSpecifier(StrTimeSpecifier specifier, int32_t } break; case StrTimeSpecifier::DAY_OF_MONTH: { - target = Write2(target, data[2] % 100); + target = Write2(target, UnsafeNumericCast(data[2] % 100)); break; } case StrTimeSpecifier::MONTH_DECIMAL: { @@ -372,7 +375,7 @@ char *StrfTimeFormat::WriteStandardSpecifier(StrTimeSpecifier specifier, int32_t break; } case StrTimeSpecifier::YEAR_WITHOUT_CENTURY: { - target = Write2(target, AbsValue(data[0]) % 100); + target = Write2(target, UnsafeNumericCast(AbsValue(data[0]) % 100)); break; } case StrTimeSpecifier::HOUR_24_DECIMAL: { @@ -845,9 +848,9 @@ bool StrpTimeFormat::Parse(string_t str, ParseResult &result) const { // numeric specifier: parse a number uint64_t number = 0; size_t start_pos = pos; - size_t end_pos = start_pos + numeric_width[i]; + size_t end_pos = start_pos + UnsafeNumericCast(numeric_width[i]); while (pos < size && pos < end_pos && StringUtil::CharacterIsDigit(data[pos])) { - number = number * 10 + data[pos] - '0'; + number = number * 10 + UnsafeNumericCast(data[pos]) - '0'; pos++; } if (pos == start_pos) { @@ -1229,7 +1232,7 @@ bool StrpTimeFormat::Parse(string_t str, ParseResult &result) const { // But tz must not be empty. if (tz_end == tz_begin) { error_message = "Empty Time Zone name"; - error_position = tz_begin - data; + error_position = UnsafeNumericCast(tz_begin - data); return false; } result.tz.assign(tz_begin, tz_end); @@ -1288,7 +1291,7 @@ bool StrpTimeFormat::Parse(string_t str, ParseResult &result) const { case StrTimeSpecifier::WEEK_NUMBER_PADDED_MON_FIRST: { // Adjust weekday to be 0-based for the week type if (has_weekday) { - weekday = (weekday + 7 - int(offset_specifier == StrTimeSpecifier::WEEK_NUMBER_PADDED_MON_FIRST)) % 7; + weekday = (weekday + 7 - uint64_t(offset_specifier == StrTimeSpecifier::WEEK_NUMBER_PADDED_MON_FIRST)) % 7; } // Get the start of week 1, move back 7 days and then weekno * 7 + weekday gives the date const auto jan1 = Date::FromDate(result_data[0], 1, 1); diff --git a/src/function/scalar/string/caseconvert.cpp b/src/function/scalar/string/caseconvert.cpp index 100ed9765a35..bf54b9ac38e3 100644 --- a/src/function/scalar/string/caseconvert.cpp +++ b/src/function/scalar/string/caseconvert.cpp @@ -58,11 +58,11 @@ static idx_t GetResultLength(const char *input_data, idx_t input_length) { if (input_data[i] & 0x80) { // unicode int sz = 0; - int codepoint = utf8proc_codepoint(input_data + i, sz); - int converted_codepoint = IS_UPPER ? utf8proc_toupper(codepoint) : utf8proc_tolower(codepoint); - int new_sz = utf8proc_codepoint_length(converted_codepoint); + auto codepoint = utf8proc_codepoint(input_data + i, sz); + auto converted_codepoint = IS_UPPER ? utf8proc_toupper(codepoint) : utf8proc_tolower(codepoint); + auto new_sz = utf8proc_codepoint_length(converted_codepoint); D_ASSERT(new_sz >= 0); - output_length += new_sz; + output_length += UnsafeNumericCast(new_sz); i += sz; } else { // ascii diff --git a/src/include/duckdb/common/types/bit.hpp b/src/include/duckdb/common/types/bit.hpp index 2dd594dd5524..903b3cc75ff9 100644 --- a/src/include/duckdb/common/types/bit.hpp +++ b/src/include/duckdb/common/types/bit.hpp @@ -104,7 +104,7 @@ void Bit::NumericToBit(T numeric, string_t &output_str) { *output = 0; // set padding to 0 ++output; for (idx_t idx = 0; idx < sizeof(T); ++idx) { - output[idx] = data[sizeof(T) - idx - 1]; + output[idx] = UnsafeNumericCast(data[sizeof(T) - idx - 1]); } Bit::Finalize(output_str); } From 8f15bb7a5ac3e21ebeca91eb12aafe4717f945a8 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 3 Apr 2024 13:42:33 +0200 Subject: [PATCH 136/603] Simplify error message stored --- .../operator/csv_scanner/util/csv_error.cpp | 9 ++++++ .../operator/csv_scanner/csv_error.hpp | 3 ++ .../csv/rejects/csv_rejects_flush_cast.test | 30 +++++-------------- 3 files changed, 19 insertions(+), 23 deletions(-) diff --git a/src/execution/operator/csv_scanner/util/csv_error.cpp b/src/execution/operator/csv_scanner/util/csv_error.cpp index d8bafa91e72d..d9119a4dd9d3 100644 --- a/src/execution/operator/csv_scanner/util/csv_error.cpp +++ b/src/execution/operator/csv_scanner/util/csv_error.cpp @@ -1,5 +1,7 @@ #include "duckdb/execution/operator/csv_scanner/csv_error.hpp" #include "duckdb/common/exception/conversion_exception.hpp" +#include "duckdb/common/string_util.hpp" + #include namespace duckdb { @@ -87,6 +89,9 @@ CSVError::CSVError(string error_message_p, CSVErrorType type_p, idx_t column_idx error_info(error_info_p), row_byte_position(row_byte_position), byte_position(byte_position_p) { // What were the options std::ostringstream error; + if (reader_options.ignore_errors.GetValue()){ + RemoveNewLine(error_message); + } error << error_message << '\n'; error << fixes << '\n'; error << reader_options.ToString(); @@ -114,6 +119,10 @@ CSVError CSVError::ColumnTypesError(case_insensitive_map_t sql_types_per_ return CSVError(exception, CSVErrorType::COLUMN_NAME_TYPE_MISMATCH, {}); } +void CSVError::RemoveNewLine(string &error){ + error = StringUtil::Split(error, "\n")[0]; +} + CSVError CSVError::CastError(const CSVReaderOptions &options, string &column_name, string &cast_error, idx_t column_idx, string &csv_row, LinesPerBoundary error_info, idx_t row_byte_position, int64_t byte_position, LogicalTypeId type) { diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp index c0f556ffba32..340c42cd1c40 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp @@ -83,6 +83,9 @@ class CSVError { return error_info.boundary_idx; } + //! We might want to remove newline in errors if we are doing them for the rejects tables + void RemoveNewLine(string &error); + //! Actual error message string error_message; //! Full error message used in throws diff --git a/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test b/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test index 69530026555e..7af3c29f373d 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test +++ b/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test @@ -11,32 +11,16 @@ query III SELECT typeof(first(a)), typeof(first(b)), COUNT(*) FROM read_csv( 'data/csv/error/flush_cast.csv', columns = {'a': 'DATE', 'b': 'VARCHAR'}, - rejects_table='csv_rejects_table', + store_rejects = true, delim = ',', - dateformat = '%d-%m-%Y', - ignore_errors=true); + dateformat = '%d-%m-%Y'); ---- DATE VARCHAR 2811 - -query IIIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position -FROM csv_rejects_table order by all; ----- -data/csv/error/flush_cast.csv 2813 1 "a" CAST c, bla 44971 -data/csv/error/flush_cast.csv 439 1 "a" CAST B, bla 6996 - -query I -SELECT error_message -FROM csv_rejects_table where byte_position = 6996; ----- -:.*Could not parse string "B" according to format specifier "%d-%m-%Y".* - -query I -SELECT error_message -FROM csv_rejects_table where byte_position = 44971; +query IIIIIIIIII +SELECT * +FROM reject_errors order by all; ---- -:.*Could not parse string "c" according to format specifier "%d-%m-%Y".* +3 0 439 6997 NULL 1 a CAST B, bla Error when converting column "a". Could not parse string "B" according to format specifier "%d-%m-%Y" +3 0 2813 44972 NULL 1 a CAST c, bla Error when converting column "a". Could not parse string "c" according to format specifier "%d-%m-%Y" -statement ok -DROP TABLE csv_rejects_table; \ No newline at end of file From 4b460b42186f8cea996ce0c7c3e97bf2ff7a8f5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Wed, 3 Apr 2024 13:44:55 +0200 Subject: [PATCH 137/603] moar --- src/catalog/dependency_manager.cpp | 584 ------------------ src/catalog/duck_catalog.cpp | 153 ----- src/function/scalar/string/caseconvert.cpp | 16 +- src/function/scalar/string/length.cpp | 18 +- src/function/scalar/string/substring.cpp | 2 +- .../rule/conjunction_simplification.cpp | 69 --- src/optimizer/rule/constant_folding.cpp | 42 -- .../rule/date_part_simplification.cpp | 103 --- src/optimizer/rule/empty_needle_removal.cpp | 53 -- src/optimizer/rule/enum_comparison.cpp | 69 --- .../rule/in_clause_simplification_rule.cpp | 56 -- src/optimizer/rule/like_optimizations.cpp | 161 ----- src/optimizer/rule/move_constants.cpp | 164 ----- .../rule/ordered_aggregate_optimizer.cpp | 101 --- src/planner/CMakeLists.txt | 26 - 15 files changed, 18 insertions(+), 1599 deletions(-) diff --git a/src/catalog/dependency_manager.cpp b/src/catalog/dependency_manager.cpp index 918bd3b276b7..8b137891791f 100644 --- a/src/catalog/dependency_manager.cpp +++ b/src/catalog/dependency_manager.cpp @@ -1,585 +1 @@ -#include "duckdb/catalog/dependency_manager.hpp" -#include "duckdb/catalog/catalog_entry/type_catalog_entry.hpp" -#include "duckdb/catalog/duck_catalog.hpp" -#include "duckdb/catalog/catalog_entry.hpp" -#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" -#include "duckdb/main/client_context.hpp" -#include "duckdb/main/database.hpp" -#include "duckdb/parser/expression/constant_expression.hpp" -#include "duckdb/catalog/dependency_list.hpp" -#include "duckdb/common/enums/catalog_type.hpp" -#include "duckdb/catalog/catalog_entry/dependency/dependency_entry.hpp" -#include "duckdb/catalog/catalog_entry/dependency/dependency_subject_entry.hpp" -#include "duckdb/catalog/catalog_entry/dependency/dependency_dependent_entry.hpp" -#include "duckdb/catalog/catalog_entry/duck_schema_entry.hpp" -#include "duckdb/catalog/dependency_catalog_set.hpp" -namespace duckdb { - -static void AssertMangledName(const string &mangled_name, idx_t expected_null_bytes) { -#ifdef DEBUG - idx_t nullbyte_count = 0; - for (auto &ch : mangled_name) { - nullbyte_count += ch == '\0'; - } - D_ASSERT(nullbyte_count == expected_null_bytes); -#endif -} - -MangledEntryName::MangledEntryName(const CatalogEntryInfo &info) { - auto &type = info.type; - auto &schema = info.schema; - auto &name = info.name; - - this->name = CatalogTypeToString(type) + '\0' + schema + '\0' + name; - AssertMangledName(this->name, 2); -} - -MangledDependencyName::MangledDependencyName(const MangledEntryName &from, const MangledEntryName &to) { - this->name = from.name + '\0' + to.name; - AssertMangledName(this->name, 5); -} - -DependencyManager::DependencyManager(DuckCatalog &catalog) : catalog(catalog), subjects(catalog), dependents(catalog) { -} - -string DependencyManager::GetSchema(CatalogEntry &entry) { - if (entry.type == CatalogType::SCHEMA_ENTRY) { - return entry.name; - } - return entry.ParentSchema().name; -} - -MangledEntryName DependencyManager::MangleName(const CatalogEntryInfo &info) { - return MangledEntryName(info); -} - -MangledEntryName DependencyManager::MangleName(CatalogEntry &entry) { - if (entry.type == CatalogType::DEPENDENCY_ENTRY) { - auto &dependency_entry = entry.Cast(); - return dependency_entry.EntryMangledName(); - } - auto type = entry.type; - auto schema = GetSchema(entry); - auto name = entry.name; - CatalogEntryInfo info {type, schema, name}; - - return MangleName(info); -} - -DependencyInfo DependencyInfo::FromSubject(DependencyEntry &dep) { - return DependencyInfo {/*dependent = */ dep.Dependent(), - /*subject = */ dep.Subject()}; -} - -DependencyInfo DependencyInfo::FromDependent(DependencyEntry &dep) { - return DependencyInfo {/*dependent = */ dep.Dependent(), - /*subject = */ dep.Subject()}; -} - -// ----------- DEPENDENCY_MANAGER ----------- - -bool DependencyManager::IsSystemEntry(CatalogEntry &entry) const { - if (entry.internal) { - return true; - } - - switch (entry.type) { - case CatalogType::DEPENDENCY_ENTRY: - case CatalogType::DATABASE_ENTRY: - case CatalogType::RENAMED_ENTRY: - return true; - default: - return false; - } -} - -CatalogSet &DependencyManager::Dependents() { - return dependents; -} - -CatalogSet &DependencyManager::Subjects() { - return subjects; -} - -void DependencyManager::ScanSetInternal(CatalogTransaction transaction, const CatalogEntryInfo &info, - bool scan_subjects, dependency_callback_t &callback) { - catalog_entry_set_t other_entries; - - auto cb = [&](CatalogEntry &other) { - D_ASSERT(other.type == CatalogType::DEPENDENCY_ENTRY); - auto &other_entry = other.Cast(); -#ifdef DEBUG - auto side = other_entry.Side(); - if (scan_subjects) { - D_ASSERT(side == DependencyEntryType::SUBJECT); - } else { - D_ASSERT(side == DependencyEntryType::DEPENDENT); - } - -#endif - - other_entries.insert(other_entry); - callback(other_entry); - }; - - if (scan_subjects) { - DependencyCatalogSet subjects(Subjects(), info); - subjects.Scan(transaction, cb); - } else { - DependencyCatalogSet dependents(Dependents(), info); - dependents.Scan(transaction, cb); - } - -#ifdef DEBUG - // Verify some invariants - // Every dependency should have a matching dependent in the other set - // And vice versa - auto mangled_name = MangleName(info); - - if (scan_subjects) { - for (auto &entry : other_entries) { - auto other_info = GetLookupProperties(entry); - DependencyCatalogSet other_dependents(Dependents(), other_info); - - // Verify that the other half of the dependency also exists - auto dependent = other_dependents.GetEntryDetailed(transaction, mangled_name); - D_ASSERT(dependent.reason != CatalogSet::EntryLookup::FailureReason::NOT_PRESENT); - } - } else { - for (auto &entry : other_entries) { - auto other_info = GetLookupProperties(entry); - DependencyCatalogSet other_subjects(Subjects(), other_info); - - // Verify that the other half of the dependent also exists - auto subject = other_subjects.GetEntryDetailed(transaction, mangled_name); - D_ASSERT(subject.reason != CatalogSet::EntryLookup::FailureReason::NOT_PRESENT); - } - } -#endif -} - -void DependencyManager::ScanDependents(CatalogTransaction transaction, const CatalogEntryInfo &info, - dependency_callback_t &callback) { - ScanSetInternal(transaction, info, false, callback); -} - -void DependencyManager::ScanSubjects(CatalogTransaction transaction, const CatalogEntryInfo &info, - dependency_callback_t &callback) { - ScanSetInternal(transaction, info, true, callback); -} - -void DependencyManager::RemoveDependency(CatalogTransaction transaction, const DependencyInfo &info) { - auto &dependent = info.dependent; - auto &subject = info.subject; - - // The dependents of the dependency (target) - DependencyCatalogSet dependents(Dependents(), subject.entry); - // The subjects of the dependencies of the dependent - DependencyCatalogSet subjects(Subjects(), dependent.entry); - - auto dependent_mangled = MangledEntryName(dependent.entry); - auto subject_mangled = MangledEntryName(subject.entry); - - auto dependent_p = dependents.GetEntry(transaction, dependent_mangled); - if (dependent_p) { - // 'dependent' is no longer inhibiting the deletion of 'dependency' - dependents.DropEntry(transaction, dependent_mangled, false); - } - auto subject_p = subjects.GetEntry(transaction, subject_mangled); - if (subject_p) { - // 'dependency' is no longer required by 'dependent' - subjects.DropEntry(transaction, subject_mangled, false); - } -} - -void DependencyManager::CreateSubject(CatalogTransaction transaction, const DependencyInfo &info) { - auto &from = info.dependent.entry; - - DependencyCatalogSet set(Subjects(), from); - auto dep = make_uniq_base(catalog, info); - auto entry_name = dep->EntryMangledName(); - - //! Add to the list of objects that 'dependent' has a dependency on - set.CreateEntry(transaction, entry_name, std::move(dep)); -} - -void DependencyManager::CreateDependent(CatalogTransaction transaction, const DependencyInfo &info) { - auto &from = info.subject.entry; - - DependencyCatalogSet set(Dependents(), from); - auto dep = make_uniq_base(catalog, info); - auto entry_name = dep->EntryMangledName(); - - //! Add to the list of object that depend on 'subject' - set.CreateEntry(transaction, entry_name, std::move(dep)); -} - -void DependencyManager::CreateDependency(CatalogTransaction transaction, DependencyInfo &info) { - DependencyCatalogSet subjects(Subjects(), info.dependent.entry); - DependencyCatalogSet dependents(Dependents(), info.subject.entry); - - auto subject_mangled = MangleName(info.subject.entry); - auto dependent_mangled = MangleName(info.dependent.entry); - - auto &dependent_flags = info.dependent.flags; - auto &subject_flags = info.subject.flags; - - auto existing_subject = subjects.GetEntry(transaction, subject_mangled); - auto existing_dependent = dependents.GetEntry(transaction, dependent_mangled); - - // Inherit the existing flags and drop the existing entry if present - if (existing_subject) { - auto &existing = existing_subject->Cast(); - auto existing_flags = existing.Subject().flags; - if (existing_flags != subject_flags) { - subject_flags.Apply(existing_flags); - } - subjects.DropEntry(transaction, subject_mangled, false, false); - } - if (existing_dependent) { - auto &existing = existing_dependent->Cast(); - auto existing_flags = existing.Dependent().flags; - if (existing_flags != dependent_flags) { - dependent_flags.Apply(existing_flags); - } - dependents.DropEntry(transaction, dependent_mangled, false, false); - } - - // Create an entry in the dependents map of the object that is the target of the dependency - CreateDependent(transaction, info); - // Create an entry in the subjects map of the object that is targeting another entry - CreateSubject(transaction, info); -} - -void DependencyManager::AddObject(CatalogTransaction transaction, CatalogEntry &object, - const DependencyList &dependencies) { - if (IsSystemEntry(object)) { - // Don't do anything for this - return; - } - - // check for each object in the sources if they were not deleted yet - for (auto &dep : dependencies.set) { - auto &dependency = dep.get(); - if (&dependency.ParentCatalog() != &object.ParentCatalog()) { - throw DependencyException( - "Error adding dependency for object \"%s\" - dependency \"%s\" is in catalog " - "\"%s\", which does not match the catalog \"%s\".\nCross catalog dependencies are not supported.", - object.name, dependency.name, dependency.ParentCatalog().GetName(), object.ParentCatalog().GetName()); - } - if (!dependency.set) { - throw InternalException("Dependency has no set"); - } - auto catalog_entry = dependency.set->GetEntry(transaction, dependency.name); - if (!catalog_entry) { - throw InternalException("Dependency has already been deleted?"); - } - } - - // indexes do not require CASCADE to be dropped, they are simply always dropped along with the table - DependencyDependentFlags dependency_flags; - if (object.type != CatalogType::INDEX_ENTRY) { - // indexes do not require CASCADE to be dropped, they are simply always dropped along with the table - dependency_flags.SetBlocking(); - } - - // add the object to the dependents_map of each object that it depends on - for (auto &dependency : dependencies.set) { - DependencyInfo info { - /*dependent = */ DependencyDependent {GetLookupProperties(object), dependency_flags}, - /*subject = */ DependencySubject {GetLookupProperties(dependency), DependencySubjectFlags()}}; - CreateDependency(transaction, info); - } -} - -static bool CascadeDrop(bool cascade, const DependencyDependentFlags &flags) { - if (cascade) { - return true; - } - if (flags.IsOwnedBy()) { - // We are owned by this object, while it exists we can not be dropped without cascade. - return false; - } - return !flags.IsBlocking(); -} - -CatalogEntryInfo DependencyManager::GetLookupProperties(CatalogEntry &entry) { - if (entry.type == CatalogType::DEPENDENCY_ENTRY) { - auto &dependency_entry = entry.Cast(); - return dependency_entry.EntryInfo(); - } else { - auto schema = DependencyManager::GetSchema(entry); - auto &name = entry.name; - auto &type = entry.type; - return CatalogEntryInfo {type, schema, name}; - } -} - -optional_ptr DependencyManager::LookupEntry(CatalogTransaction transaction, CatalogEntry &dependency) { - auto info = GetLookupProperties(dependency); - - auto &type = info.type; - auto &schema = info.schema; - auto &name = info.name; - - // Lookup the schema - auto schema_entry = catalog.GetSchema(transaction, schema, OnEntryNotFound::RETURN_NULL); - if (type == CatalogType::SCHEMA_ENTRY || !schema_entry) { - // This is a schema entry, perform the callback only providing the schema - return reinterpret_cast(schema_entry.get()); - } - auto entry = schema_entry->GetEntry(transaction, type, name); - return entry; -} - -void DependencyManager::CleanupDependencies(CatalogTransaction transaction, CatalogEntry &object) { - // Collect the dependencies - vector to_remove; - - auto info = GetLookupProperties(object); - ScanSubjects(transaction, info, - [&](DependencyEntry &dep) { to_remove.push_back(DependencyInfo::FromSubject(dep)); }); - ScanDependents(transaction, info, - [&](DependencyEntry &dep) { to_remove.push_back(DependencyInfo::FromDependent(dep)); }); - - // Remove the dependency entries - for (auto &dep : to_remove) { - RemoveDependency(transaction, dep); - } -} - -void DependencyManager::DropObject(CatalogTransaction transaction, CatalogEntry &object, bool cascade) { - if (IsSystemEntry(object)) { - // Don't do anything for this - return; - } - - auto info = GetLookupProperties(object); - // Check if there are any entries that block the DROP because they still depend on the object - catalog_entry_set_t to_drop; - ScanDependents(transaction, info, [&](DependencyEntry &dep) { - // It makes no sense to have a schema depend on anything - D_ASSERT(dep.EntryInfo().type != CatalogType::SCHEMA_ENTRY); - auto entry = LookupEntry(transaction, dep); - if (!entry) { - return; - } - - if (!CascadeDrop(cascade, dep.Dependent().flags)) { - // no cascade and there are objects that depend on this object: throw error - throw DependencyException("Cannot drop entry \"%s\" because there are entries that " - "depend on it. Use DROP...CASCADE to drop all dependents.", - object.name); - } - to_drop.insert(*entry); - }); - ScanSubjects(transaction, info, [&](DependencyEntry &dep) { - auto flags = dep.Subject().flags; - if (flags.IsOwnership()) { - // We own this object, it should be dropped along with the table - auto entry = LookupEntry(transaction, dep); - to_drop.insert(*entry); - } - }); - - CleanupDependencies(transaction, object); - - for (auto &entry : to_drop) { - auto set = entry.get().set; - D_ASSERT(set); - set->DropEntry(transaction, entry.get().name, cascade); - } -} - -void DependencyManager::AlterObject(CatalogTransaction transaction, CatalogEntry &old_obj, CatalogEntry &new_obj) { - if (IsSystemEntry(new_obj)) { - D_ASSERT(IsSystemEntry(old_obj)); - // Don't do anything for this - return; - } - - auto info = GetLookupProperties(old_obj); - dependency_set_t owned_objects; - ScanDependents(transaction, info, [&](DependencyEntry &dep) { - // It makes no sense to have a schema depend on anything - D_ASSERT(dep.EntryInfo().type != CatalogType::SCHEMA_ENTRY); - - auto entry = LookupEntry(transaction, dep); - if (!entry) { - return; - } - // conflict: attempting to alter this object but the dependent object still exists - // no cascade and there are objects that depend on this object: throw error - throw DependencyException("Cannot alter entry \"%s\" because there are entries that " - "depend on it.", - old_obj.name); - }); - - // Keep old dependencies - dependency_set_t dependents; - ScanSubjects(transaction, info, [&](DependencyEntry &dep) { - auto entry = LookupEntry(transaction, dep); - if (!entry) { - return; - } - if (dep.Subject().flags.IsOwnership()) { - owned_objects.insert(Dependency(*entry, dep.Dependent().flags)); - return; - } - dependents.insert(Dependency(*entry, dep.Dependent().flags)); - }); - - // FIXME: we should update dependencies in the future - // some alters could cause dependencies to change (imagine types of table columns) - // or DEFAULT depending on a sequence - if (StringUtil::CIEquals(old_obj.name, new_obj.name)) { - // The name was not changed, we do not need to recreate the dependency links - return; - } - CleanupDependencies(transaction, old_obj); - - for (auto &dep : dependents) { - auto &other = dep.entry.get(); - DependencyInfo info {/*dependent = */ DependencyDependent {GetLookupProperties(new_obj), dep.flags}, - /*subject = */ DependencySubject {GetLookupProperties(other), DependencySubjectFlags()}}; - CreateDependency(transaction, info); - } - - // For all the objects we own, re establish the dependency of the owner on the object - for (auto &object : owned_objects) { - auto &entry = object.entry.get(); - { - DependencyInfo info { - /*dependent = */ DependencyDependent {GetLookupProperties(new_obj), - DependencyDependentFlags().SetOwnedBy()}, - /*subject = */ DependencySubject {GetLookupProperties(entry), DependencySubjectFlags().SetOwnership()}}; - CreateDependency(transaction, info); - } - } -} - -void DependencyManager::Scan( - ClientContext &context, - const std::function &callback) { - lock_guard write_lock(catalog.GetWriteLock()); - auto transaction = catalog.GetCatalogTransaction(context); - - // All the objects registered in the dependency manager - catalog_entry_set_t entries; - dependents.Scan(transaction, [&](CatalogEntry &set) { - auto entry = LookupEntry(transaction, set); - entries.insert(*entry); - }); - - // For every registered entry, get the dependents - for (auto &entry : entries) { - auto entry_info = GetLookupProperties(entry); - // Scan all the dependents of the entry - ScanDependents(transaction, entry_info, [&](DependencyEntry &dependent) { - auto dep = LookupEntry(transaction, dependent); - if (!dep) { - return; - } - auto &dependent_entry = *dep; - callback(entry, dependent_entry, dependent.Dependent().flags); - }); - } -} - -void DependencyManager::AddOwnership(CatalogTransaction transaction, CatalogEntry &owner, CatalogEntry &entry) { - if (IsSystemEntry(entry) || IsSystemEntry(owner)) { - return; - } - - // If the owner is already owned by something else, throw an error - auto owner_info = GetLookupProperties(owner); - ScanDependents(transaction, owner_info, [&](DependencyEntry &dep) { - if (dep.Dependent().flags.IsOwnedBy()) { - throw DependencyException("%s can not become the owner, it is already owned by %s", owner.name, - dep.EntryInfo().name); - } - }); - - // If the entry is the owner of another entry, throw an error - auto entry_info = GetLookupProperties(entry); - ScanSubjects(transaction, entry_info, [&](DependencyEntry &other) { - auto dependent_entry = LookupEntry(transaction, other); - if (!dependent_entry) { - return; - } - auto &dep = *dependent_entry; - - auto flags = other.Dependent().flags; - if (!flags.IsOwnedBy()) { - return; - } - throw DependencyException("%s already owns %s. Cannot have circular dependencies", entry.name, dep.name); - }); - - // If the entry is already owned, throw an error - ScanDependents(transaction, entry_info, [&](DependencyEntry &other) { - auto dependent_entry = LookupEntry(transaction, other); - if (!dependent_entry) { - return; - } - - auto &dep = *dependent_entry; - auto flags = other.Subject().flags; - if (!flags.IsOwnership()) { - return; - } - if (&dep != &owner) { - throw DependencyException("%s is already owned by %s", entry.name, dep.name); - } - }); - - DependencyInfo info { - /*dependent = */ DependencyDependent {GetLookupProperties(owner), DependencyDependentFlags().SetOwnedBy()}, - /*subject = */ DependencySubject {GetLookupProperties(entry), DependencySubjectFlags().SetOwnership()}}; - CreateDependency(transaction, info); -} - -static string FormatString(const MangledEntryName &mangled) { - auto input = mangled.name; - for (size_t i = 0; i < input.size(); i++) { - if (input[i] == '\0') { - input[i] = '_'; - } - } - return input; -} - -void DependencyManager::PrintSubjects(CatalogTransaction transaction, const CatalogEntryInfo &info) { - auto name = MangleName(info); - Printer::Print(StringUtil::Format("Subjects of %s", FormatString(name))); - auto subjects = DependencyCatalogSet(Subjects(), info); - subjects.Scan(transaction, [&](CatalogEntry &dependency) { - auto &dep = dependency.Cast(); - auto &entry_info = dep.EntryInfo(); - auto type = entry_info.type; - auto schema = entry_info.schema; - auto name = entry_info.name; - Printer::Print(StringUtil::Format("Schema: %s | Name: %s | Type: %s | Dependent type: %s | Subject type: %s", - schema, name, CatalogTypeToString(type), dep.Dependent().flags.ToString(), - dep.Subject().flags.ToString())); - }); -} - -void DependencyManager::PrintDependents(CatalogTransaction transaction, const CatalogEntryInfo &info) { - auto name = MangleName(info); - Printer::Print(StringUtil::Format("Dependents of %s", FormatString(name))); - auto dependents = DependencyCatalogSet(Dependents(), info); - dependents.Scan(transaction, [&](CatalogEntry &dependent) { - auto &dep = dependent.Cast(); - auto &entry_info = dep.EntryInfo(); - auto type = entry_info.type; - auto schema = entry_info.schema; - auto name = entry_info.name; - Printer::Print(StringUtil::Format("Schema: %s | Name: %s | Type: %s | Dependent type: %s | Subject type: %s", - schema, name, CatalogTypeToString(type), dep.Dependent().flags.ToString(), - dep.Subject().flags.ToString())); - }); -} - -} // namespace duckdb diff --git a/src/catalog/duck_catalog.cpp b/src/catalog/duck_catalog.cpp index 8f712b843ed3..8b137891791f 100644 --- a/src/catalog/duck_catalog.cpp +++ b/src/catalog/duck_catalog.cpp @@ -1,154 +1 @@ -#include "duckdb/catalog/duck_catalog.hpp" -#include "duckdb/catalog/dependency_manager.hpp" -#include "duckdb/catalog/catalog_entry/duck_schema_entry.hpp" -#include "duckdb/storage/storage_manager.hpp" -#include "duckdb/parser/parsed_data/drop_info.hpp" -#include "duckdb/parser/parsed_data/create_schema_info.hpp" -#include "duckdb/catalog/default/default_schemas.hpp" -#include "duckdb/function/built_in_functions.hpp" -#include "duckdb/main/attached_database.hpp" -#ifndef DISABLE_CORE_FUNCTIONS_EXTENSION -#include "duckdb/core_functions/core_functions.hpp" -#endif -namespace duckdb { - -DuckCatalog::DuckCatalog(AttachedDatabase &db) - : Catalog(db), dependency_manager(make_uniq(*this)), - schemas(make_uniq(*this, make_uniq(*this))) { -} - -DuckCatalog::~DuckCatalog() { -} - -void DuckCatalog::Initialize(bool load_builtin) { - // first initialize the base system catalogs - // these are never written to the WAL - // we start these at 1 because deleted entries default to 0 - auto data = CatalogTransaction::GetSystemTransaction(GetDatabase()); - - // create the default schema - CreateSchemaInfo info; - info.schema = DEFAULT_SCHEMA; - info.internal = true; - CreateSchema(data, info); - - if (load_builtin) { - // initialize default functions - BuiltinFunctions builtin(data, *this); - builtin.Initialize(); - -#ifndef DISABLE_CORE_FUNCTIONS_EXTENSION - CoreFunctions::RegisterFunctions(*this, data); -#endif - } - - Verify(); -} - -bool DuckCatalog::IsDuckCatalog() { - return true; -} - -//===--------------------------------------------------------------------===// -// Schema -//===--------------------------------------------------------------------===// -optional_ptr DuckCatalog::CreateSchemaInternal(CatalogTransaction transaction, CreateSchemaInfo &info) { - DependencyList dependencies; - auto entry = make_uniq(*this, info); - auto result = entry.get(); - if (!schemas->CreateEntry(transaction, info.schema, std::move(entry), dependencies)) { - return nullptr; - } - return (CatalogEntry *)result; -} - -optional_ptr DuckCatalog::CreateSchema(CatalogTransaction transaction, CreateSchemaInfo &info) { - D_ASSERT(!info.schema.empty()); - auto result = CreateSchemaInternal(transaction, info); - if (!result) { - switch (info.on_conflict) { - case OnCreateConflict::ERROR_ON_CONFLICT: - throw CatalogException::EntryAlreadyExists(CatalogType::SCHEMA_ENTRY, info.schema); - case OnCreateConflict::REPLACE_ON_CONFLICT: { - DropInfo drop_info; - drop_info.type = CatalogType::SCHEMA_ENTRY; - drop_info.catalog = info.catalog; - drop_info.name = info.schema; - DropSchema(transaction, drop_info); - result = CreateSchemaInternal(transaction, info); - if (!result) { - throw InternalException("Failed to create schema entry in CREATE_OR_REPLACE"); - } - break; - } - case OnCreateConflict::IGNORE_ON_CONFLICT: - break; - default: - throw InternalException("Unsupported OnCreateConflict for CreateSchema"); - } - return nullptr; - } - return result; -} - -void DuckCatalog::DropSchema(CatalogTransaction transaction, DropInfo &info) { - D_ASSERT(!info.name.empty()); - ModifyCatalog(); - if (!schemas->DropEntry(transaction, info.name, info.cascade)) { - if (info.if_not_found == OnEntryNotFound::THROW_EXCEPTION) { - throw CatalogException::MissingEntry(CatalogType::SCHEMA_ENTRY, info.name, string()); - } - } -} - -void DuckCatalog::DropSchema(ClientContext &context, DropInfo &info) { - DropSchema(GetCatalogTransaction(context), info); -} - -void DuckCatalog::ScanSchemas(ClientContext &context, std::function callback) { - schemas->Scan(GetCatalogTransaction(context), - [&](CatalogEntry &entry) { callback(entry.Cast()); }); -} - -void DuckCatalog::ScanSchemas(std::function callback) { - schemas->Scan([&](CatalogEntry &entry) { callback(entry.Cast()); }); -} - -optional_ptr DuckCatalog::GetSchema(CatalogTransaction transaction, const string &schema_name, - OnEntryNotFound if_not_found, QueryErrorContext error_context) { - D_ASSERT(!schema_name.empty()); - auto entry = schemas->GetEntry(transaction, schema_name); - if (!entry) { - if (if_not_found == OnEntryNotFound::THROW_EXCEPTION) { - throw CatalogException(error_context, "Schema with name %s does not exist!", schema_name); - } - return nullptr; - } - return &entry->Cast(); -} - -DatabaseSize DuckCatalog::GetDatabaseSize(ClientContext &context) { - return db.GetStorageManager().GetDatabaseSize(); -} - -vector DuckCatalog::GetMetadataInfo(ClientContext &context) { - return db.GetStorageManager().GetMetadataInfo(); -} - -bool DuckCatalog::InMemory() { - return db.GetStorageManager().InMemory(); -} - -string DuckCatalog::GetDBPath() { - return db.GetStorageManager().GetDBPath(); -} - -void DuckCatalog::Verify() { -#ifdef DEBUG - Catalog::Verify(); - schemas->Verify(*this); -#endif -} - -} // namespace duckdb diff --git a/src/function/scalar/string/caseconvert.cpp b/src/function/scalar/string/caseconvert.cpp index bf54b9ac38e3..fa5b612f1860 100644 --- a/src/function/scalar/string/caseconvert.cpp +++ b/src/function/scalar/string/caseconvert.cpp @@ -44,8 +44,8 @@ static string_t ASCIICaseConvert(Vector &result, const char *input_data, idx_t i auto result_str = StringVector::EmptyString(result, output_length); auto result_data = result_str.GetDataWriteable(); for (idx_t i = 0; i < input_length; i++) { - result_data[i] = IS_UPPER ? UpperFun::ASCII_TO_UPPER_MAP[uint8_t(input_data[i])] - : LowerFun::ASCII_TO_LOWER_MAP[uint8_t(input_data[i])]; + result_data[i] = UnsafeNumericCast(IS_UPPER ? UpperFun::ASCII_TO_UPPER_MAP[uint8_t(input_data[i])] + : LowerFun::ASCII_TO_LOWER_MAP[uint8_t(input_data[i])]); } result_str.Finalize(); return result_str; @@ -63,7 +63,7 @@ static idx_t GetResultLength(const char *input_data, idx_t input_length) { auto new_sz = utf8proc_codepoint_length(converted_codepoint); D_ASSERT(new_sz >= 0); output_length += UnsafeNumericCast(new_sz); - i += sz; + i += UnsafeNumericCast(sz); } else { // ascii output_length++; @@ -79,17 +79,17 @@ static void CaseConvert(const char *input_data, idx_t input_length, char *result if (input_data[i] & 0x80) { // non-ascii character int sz = 0, new_sz = 0; - int codepoint = utf8proc_codepoint(input_data + i, sz); - int converted_codepoint = IS_UPPER ? utf8proc_toupper(codepoint) : utf8proc_tolower(codepoint); + auto codepoint = utf8proc_codepoint(input_data + i, sz); + auto converted_codepoint = IS_UPPER ? utf8proc_toupper(codepoint) : utf8proc_tolower(codepoint); auto success = utf8proc_codepoint_to_utf8(converted_codepoint, new_sz, result_data); D_ASSERT(success); (void)success; result_data += new_sz; - i += sz; + i += UnsafeNumericCast(sz); } else { // ascii - *result_data = IS_UPPER ? UpperFun::ASCII_TO_UPPER_MAP[uint8_t(input_data[i])] - : LowerFun::ASCII_TO_LOWER_MAP[uint8_t(input_data[i])]; + *result_data = UnsafeNumericCast(IS_UPPER ? UpperFun::ASCII_TO_UPPER_MAP[uint8_t(input_data[i])] + : LowerFun::ASCII_TO_LOWER_MAP[uint8_t(input_data[i])]); result_data++; i++; } diff --git a/src/function/scalar/string/length.cpp b/src/function/scalar/string/length.cpp index 218e4e84626d..129105bee80a 100644 --- a/src/function/scalar/string/length.cpp +++ b/src/function/scalar/string/length.cpp @@ -29,14 +29,14 @@ struct GraphemeCountOperator { struct StrLenOperator { template static inline TR Operation(TA input) { - return input.GetSize(); + return UnsafeNumericCast(input.GetSize()); } }; struct OctetLenOperator { template static inline TR Operation(TA input) { - return Bit::OctetLength(input); + return UnsafeNumericCast(Bit::OctetLength(input)); } }; @@ -44,7 +44,7 @@ struct OctetLenOperator { struct BitLenOperator { template static inline TR Operation(TA input) { - return 8 * input.GetSize(); + return UnsafeNumericCast(8 * input.GetSize()); } }; @@ -52,7 +52,7 @@ struct BitLenOperator { struct BitStringLenOperator { template static inline TR Operation(TA input) { - return Bit::BitLength(input); + return UnsafeNumericCast(Bit::BitLength(input)); } }; @@ -74,8 +74,8 @@ static unique_ptr LengthPropagateStats(ClientContext &context, F static void ListLengthFunction(DataChunk &args, ExpressionState &state, Vector &result) { auto &input = args.data[0]; D_ASSERT(input.GetType().id() == LogicalTypeId::LIST); - UnaryExecutor::Execute(input, result, args.size(), - [](list_entry_t input) { return input.length; }); + UnaryExecutor::Execute( + input, result, args.size(), [](list_entry_t input) { return UnsafeNumericCast(input.length); }); if (args.AllConstant()) { result.SetVectorType(VectorType::CONSTANT_VECTOR); } @@ -117,7 +117,7 @@ static void ListLengthBinaryFunction(DataChunk &args, ExpressionState &, Vector if (dimension != 1) { throw NotImplementedException("array_length for lists with dimensions other than 1 not implemented"); } - return input.length; + return UnsafeNumericCast(input.length); }); if (args.AllConstant()) { result.SetVectorType(VectorType::CONSTANT_VECTOR); @@ -153,7 +153,7 @@ static void ArrayLengthBinaryFunction(DataChunk &args, ExpressionState &state, V throw OutOfRangeException(StringUtil::Format( "array_length dimension '%lld' out of range (min: '1', max: '%lld')", dimension, max_dimension)); } - return dimensions[dimension - 1]; + return dimensions[UnsafeNumericCast(dimension - 1)]; }); if (args.AllConstant()) { @@ -175,7 +175,7 @@ static unique_ptr ArrayOrListLengthBinaryBind(ClientContext &conte vector dimensions; while (true) { if (type.id() == LogicalTypeId::ARRAY) { - dimensions.push_back(ArrayType::GetSize(type)); + dimensions.push_back(UnsafeNumericCast(ArrayType::GetSize(type))); type = ArrayType::GetChildType(type); } else { break; diff --git a/src/function/scalar/string/substring.cpp b/src/function/scalar/string/substring.cpp index b0b2d8161e00..a6d582c4ecd8 100644 --- a/src/function/scalar/string/substring.cpp +++ b/src/function/scalar/string/substring.cpp @@ -40,7 +40,7 @@ string_t SubstringEmptyString(Vector &result) { } string_t SubstringSlice(Vector &result, const char *input_data, int64_t offset, int64_t length) { - auto result_string = StringVector::EmptyString(result, length); + auto result_string = StringVector::EmptyString(result, UnsafeNumericCast(length)); auto result_data = result_string.GetDataWriteable(); memcpy(result_data, input_data + offset, length); result_string.Finalize(); diff --git a/src/optimizer/rule/conjunction_simplification.cpp b/src/optimizer/rule/conjunction_simplification.cpp index 646471b9412e..8b137891791f 100644 --- a/src/optimizer/rule/conjunction_simplification.cpp +++ b/src/optimizer/rule/conjunction_simplification.cpp @@ -1,70 +1 @@ -#include "duckdb/optimizer/rule/conjunction_simplification.hpp" -#include "duckdb/execution/expression_executor.hpp" -#include "duckdb/planner/expression/bound_conjunction_expression.hpp" -#include "duckdb/planner/expression/bound_constant_expression.hpp" - -namespace duckdb { - -ConjunctionSimplificationRule::ConjunctionSimplificationRule(ExpressionRewriter &rewriter) : Rule(rewriter) { - // match on a ComparisonExpression that has a ConstantExpression as a check - auto op = make_uniq(); - op->matchers.push_back(make_uniq()); - op->policy = SetMatcher::Policy::SOME; - root = std::move(op); -} - -unique_ptr ConjunctionSimplificationRule::RemoveExpression(BoundConjunctionExpression &conj, - const Expression &expr) { - for (idx_t i = 0; i < conj.children.size(); i++) { - if (conj.children[i].get() == &expr) { - // erase the expression - conj.children.erase_at(i); - break; - } - } - if (conj.children.size() == 1) { - // one expression remaining: simply return that expression and erase the conjunction - return std::move(conj.children[0]); - } - return nullptr; -} - -unique_ptr ConjunctionSimplificationRule::Apply(LogicalOperator &op, - vector> &bindings, bool &changes_made, - bool is_root) { - auto &conjunction = bindings[0].get().Cast(); - auto &constant_expr = bindings[1].get(); - // the constant_expr is a scalar expression that we have to fold - // use an ExpressionExecutor to execute the expression - D_ASSERT(constant_expr.IsFoldable()); - Value constant_value; - if (!ExpressionExecutor::TryEvaluateScalar(GetContext(), constant_expr, constant_value)) { - return nullptr; - } - constant_value = constant_value.DefaultCastAs(LogicalType::BOOLEAN); - if (constant_value.IsNull()) { - // we can't simplify conjunctions with a constant NULL - return nullptr; - } - if (conjunction.type == ExpressionType::CONJUNCTION_AND) { - if (!BooleanValue::Get(constant_value)) { - // FALSE in AND, result of expression is false - return make_uniq(Value::BOOLEAN(false)); - } else { - // TRUE in AND, remove the expression from the set - return RemoveExpression(conjunction, constant_expr); - } - } else { - D_ASSERT(conjunction.type == ExpressionType::CONJUNCTION_OR); - if (!BooleanValue::Get(constant_value)) { - // FALSE in OR, remove the expression from the set - return RemoveExpression(conjunction, constant_expr); - } else { - // TRUE in OR, result of expression is true - return make_uniq(Value::BOOLEAN(true)); - } - } -} - -} // namespace duckdb diff --git a/src/optimizer/rule/constant_folding.cpp b/src/optimizer/rule/constant_folding.cpp index 6b7d20c46aa5..8b137891791f 100644 --- a/src/optimizer/rule/constant_folding.cpp +++ b/src/optimizer/rule/constant_folding.cpp @@ -1,43 +1 @@ -#include "duckdb/optimizer/rule/constant_folding.hpp" -#include "duckdb/common/exception.hpp" -#include "duckdb/execution/expression_executor.hpp" -#include "duckdb/optimizer/expression_rewriter.hpp" -#include "duckdb/planner/expression/bound_constant_expression.hpp" - -namespace duckdb { - -//! The ConstantFoldingExpressionMatcher matches on any scalar expression (i.e. Expression::IsFoldable is true) -class ConstantFoldingExpressionMatcher : public FoldableConstantMatcher { -public: - bool Match(Expression &expr, vector> &bindings) override { - // we also do not match on ConstantExpressions, because we cannot fold those any further - if (expr.type == ExpressionType::VALUE_CONSTANT) { - return false; - } - return FoldableConstantMatcher::Match(expr, bindings); - } -}; - -ConstantFoldingRule::ConstantFoldingRule(ExpressionRewriter &rewriter) : Rule(rewriter) { - auto op = make_uniq(); - root = std::move(op); -} - -unique_ptr ConstantFoldingRule::Apply(LogicalOperator &op, vector> &bindings, - bool &changes_made, bool is_root) { - auto &root = bindings[0].get(); - // the root is a scalar expression that we have to fold - D_ASSERT(root.IsFoldable() && root.type != ExpressionType::VALUE_CONSTANT); - - // use an ExpressionExecutor to execute the expression - Value result_value; - if (!ExpressionExecutor::TryEvaluateScalar(GetContext(), root, result_value)) { - return nullptr; - } - D_ASSERT(result_value.type().InternalType() == root.return_type.InternalType()); - // now get the value from the result vector and insert it back into the plan as a constant expression - return make_uniq(result_value); -} - -} // namespace duckdb diff --git a/src/optimizer/rule/date_part_simplification.cpp b/src/optimizer/rule/date_part_simplification.cpp index 6737e576c7dd..8b137891791f 100644 --- a/src/optimizer/rule/date_part_simplification.cpp +++ b/src/optimizer/rule/date_part_simplification.cpp @@ -1,104 +1 @@ -#include "duckdb/optimizer/rule/date_part_simplification.hpp" -#include "duckdb/common/exception.hpp" -#include "duckdb/planner/expression/bound_constant_expression.hpp" -#include "duckdb/planner/expression/bound_function_expression.hpp" -#include "duckdb/optimizer/matcher/expression_matcher.hpp" -#include "duckdb/optimizer/expression_rewriter.hpp" -#include "duckdb/common/enums/date_part_specifier.hpp" -#include "duckdb/function/function.hpp" -#include "duckdb/function/function_binder.hpp" - -namespace duckdb { - -DatePartSimplificationRule::DatePartSimplificationRule(ExpressionRewriter &rewriter) : Rule(rewriter) { - auto func = make_uniq(); - func->function = make_uniq("date_part"); - func->matchers.push_back(make_uniq()); - func->matchers.push_back(make_uniq()); - func->policy = SetMatcher::Policy::ORDERED; - root = std::move(func); -} - -unique_ptr DatePartSimplificationRule::Apply(LogicalOperator &op, vector> &bindings, - bool &changes_made, bool is_root) { - auto &date_part = bindings[0].get().Cast(); - auto &constant_expr = bindings[1].get().Cast(); - auto &constant = constant_expr.value; - - if (constant.IsNull()) { - // NULL specifier: return constant NULL - return make_uniq(Value(date_part.return_type)); - } - // otherwise check the specifier - auto specifier = GetDatePartSpecifier(StringValue::Get(constant)); - string new_function_name; - switch (specifier) { - case DatePartSpecifier::YEAR: - new_function_name = "year"; - break; - case DatePartSpecifier::MONTH: - new_function_name = "month"; - break; - case DatePartSpecifier::DAY: - new_function_name = "day"; - break; - case DatePartSpecifier::DECADE: - new_function_name = "decade"; - break; - case DatePartSpecifier::CENTURY: - new_function_name = "century"; - break; - case DatePartSpecifier::MILLENNIUM: - new_function_name = "millennium"; - break; - case DatePartSpecifier::QUARTER: - new_function_name = "quarter"; - break; - case DatePartSpecifier::WEEK: - new_function_name = "week"; - break; - case DatePartSpecifier::YEARWEEK: - new_function_name = "yearweek"; - break; - case DatePartSpecifier::DOW: - new_function_name = "dayofweek"; - break; - case DatePartSpecifier::ISODOW: - new_function_name = "isodow"; - break; - case DatePartSpecifier::DOY: - new_function_name = "dayofyear"; - break; - case DatePartSpecifier::MICROSECONDS: - new_function_name = "microsecond"; - break; - case DatePartSpecifier::MILLISECONDS: - new_function_name = "millisecond"; - break; - case DatePartSpecifier::SECOND: - new_function_name = "second"; - break; - case DatePartSpecifier::MINUTE: - new_function_name = "minute"; - break; - case DatePartSpecifier::HOUR: - new_function_name = "hour"; - break; - default: - return nullptr; - } - // found a replacement function: bind it - vector> children; - children.push_back(std::move(date_part.children[1])); - - ErrorData error; - FunctionBinder binder(rewriter.context); - auto function = binder.BindScalarFunction(DEFAULT_SCHEMA, new_function_name, std::move(children), error, false); - if (!function) { - error.Throw(); - } - return function; -} - -} // namespace duckdb diff --git a/src/optimizer/rule/empty_needle_removal.cpp b/src/optimizer/rule/empty_needle_removal.cpp index 500d639a16df..8b137891791f 100644 --- a/src/optimizer/rule/empty_needle_removal.cpp +++ b/src/optimizer/rule/empty_needle_removal.cpp @@ -1,54 +1 @@ -#include "duckdb/optimizer/rule/empty_needle_removal.hpp" -#include "duckdb/execution/expression_executor.hpp" -#include "duckdb/planner/expression/bound_function_expression.hpp" -#include "duckdb/planner/expression/bound_constant_expression.hpp" -#include "duckdb/planner/expression/bound_operator_expression.hpp" -#include "duckdb/planner/expression/bound_case_expression.hpp" -#include "duckdb/optimizer/expression_rewriter.hpp" - -namespace duckdb { - -EmptyNeedleRemovalRule::EmptyNeedleRemovalRule(ExpressionRewriter &rewriter) : Rule(rewriter) { - // match on a FunctionExpression that has a foldable ConstantExpression - auto func = make_uniq(); - func->matchers.push_back(make_uniq()); - func->matchers.push_back(make_uniq()); - func->policy = SetMatcher::Policy::SOME; - - unordered_set functions = {"prefix", "contains", "suffix"}; - func->function = make_uniq(functions); - root = std::move(func); -} - -unique_ptr EmptyNeedleRemovalRule::Apply(LogicalOperator &op, vector> &bindings, - bool &changes_made, bool is_root) { - auto &root = bindings[0].get().Cast(); - D_ASSERT(root.children.size() == 2); - auto &prefix_expr = bindings[2].get(); - - // the constant_expr is a scalar expression that we have to fold - if (!prefix_expr.IsFoldable()) { - return nullptr; - } - D_ASSERT(root.return_type.id() == LogicalTypeId::BOOLEAN); - - auto prefix_value = ExpressionExecutor::EvaluateScalar(GetContext(), prefix_expr); - - if (prefix_value.IsNull()) { - return make_uniq(Value(LogicalType::BOOLEAN)); - } - - D_ASSERT(prefix_value.type() == prefix_expr.return_type); - auto &needle_string = StringValue::Get(prefix_value); - - // PREFIX('xyz', '') is TRUE - // PREFIX(NULL, '') is NULL - // so rewrite PREFIX(x, '') to TRUE_OR_NULL(x) - if (needle_string.empty()) { - return ExpressionRewriter::ConstantOrNull(std::move(root.children[0]), Value::BOOLEAN(true)); - } - return nullptr; -} - -} // namespace duckdb diff --git a/src/optimizer/rule/enum_comparison.cpp b/src/optimizer/rule/enum_comparison.cpp index 8b0525789974..8b137891791f 100644 --- a/src/optimizer/rule/enum_comparison.cpp +++ b/src/optimizer/rule/enum_comparison.cpp @@ -1,70 +1 @@ -#include "duckdb/optimizer/rule/enum_comparison.hpp" -#include "duckdb/execution/expression_executor.hpp" -#include "duckdb/planner/expression/bound_comparison_expression.hpp" -#include "duckdb/planner/expression/bound_cast_expression.hpp" -#include "duckdb/optimizer/matcher/type_matcher_id.hpp" -#include "duckdb/optimizer/expression_rewriter.hpp" -#include "duckdb/common/types.hpp" - -namespace duckdb { - -EnumComparisonRule::EnumComparisonRule(ExpressionRewriter &rewriter) : Rule(rewriter) { - // match on a ComparisonExpression that is an Equality and has a VARCHAR and ENUM as its children - auto op = make_uniq(); - // Enum requires expression to be root - op->expr_type = make_uniq(ExpressionType::COMPARE_EQUAL); - for (idx_t i = 0; i < 2; i++) { - auto child = make_uniq(); - child->type = make_uniq(LogicalTypeId::VARCHAR); - child->matcher = make_uniq(); - child->matcher->type = make_uniq(LogicalTypeId::ENUM); - op->matchers.push_back(std::move(child)); - } - root = std::move(op); -} - -bool AreMatchesPossible(LogicalType &left, LogicalType &right) { - LogicalType *small_enum, *big_enum; - if (EnumType::GetSize(left) < EnumType::GetSize(right)) { - small_enum = &left; - big_enum = &right; - } else { - small_enum = &right; - big_enum = &left; - } - auto &string_vec = EnumType::GetValuesInsertOrder(*small_enum); - auto string_vec_ptr = FlatVector::GetData(string_vec); - auto size = EnumType::GetSize(*small_enum); - for (idx_t i = 0; i < size; i++) { - auto key = string_vec_ptr[i].GetString(); - if (EnumType::GetPos(*big_enum, key) != -1) { - return true; - } - } - return false; -} -unique_ptr EnumComparisonRule::Apply(LogicalOperator &op, vector> &bindings, - bool &changes_made, bool is_root) { - - auto &root = bindings[0].get().Cast(); - auto &left_child = bindings[1].get().Cast(); - auto &right_child = bindings[3].get().Cast(); - - if (!AreMatchesPossible(left_child.child->return_type, right_child.child->return_type)) { - vector> children; - children.push_back(std::move(root.left)); - children.push_back(std::move(root.right)); - return ExpressionRewriter::ConstantOrNull(std::move(children), Value::BOOLEAN(false)); - } - - if (!is_root || op.type != LogicalOperatorType::LOGICAL_FILTER) { - return nullptr; - } - - auto cast_left_to_right = - BoundCastExpression::AddDefaultCastToType(std::move(left_child.child), right_child.child->return_type, true); - return make_uniq(root.type, std::move(cast_left_to_right), std::move(right_child.child)); -} - -} // namespace duckdb diff --git a/src/optimizer/rule/in_clause_simplification_rule.cpp b/src/optimizer/rule/in_clause_simplification_rule.cpp index e1ad4fd9e78e..8b137891791f 100644 --- a/src/optimizer/rule/in_clause_simplification_rule.cpp +++ b/src/optimizer/rule/in_clause_simplification_rule.cpp @@ -1,57 +1 @@ -#include "duckdb/execution/expression_executor.hpp" -#include "duckdb/optimizer/rule/in_clause_simplification.hpp" -#include "duckdb/planner/expression/list.hpp" -#include "duckdb/planner/expression/bound_operator_expression.hpp" -namespace duckdb { - -InClauseSimplificationRule::InClauseSimplificationRule(ExpressionRewriter &rewriter) : Rule(rewriter) { - // match on InClauseExpression that has a ConstantExpression as a check - auto op = make_uniq(); - op->policy = SetMatcher::Policy::SOME; - root = std::move(op); -} - -unique_ptr InClauseSimplificationRule::Apply(LogicalOperator &op, vector> &bindings, - bool &changes_made, bool is_root) { - auto &expr = bindings[0].get().Cast(); - if (expr.children[0]->expression_class != ExpressionClass::BOUND_CAST) { - return nullptr; - } - auto &cast_expression = expr.children[0]->Cast(); - if (cast_expression.child->expression_class != ExpressionClass::BOUND_COLUMN_REF) { - return nullptr; - } - //! Here we check if we can apply the expression on the constant side - auto target_type = cast_expression.source_type(); - if (!BoundCastExpression::CastIsInvertible(cast_expression.return_type, target_type)) { - return nullptr; - } - vector> cast_list; - //! First check if we can cast all children - for (size_t i = 1; i < expr.children.size(); i++) { - if (expr.children[i]->expression_class != ExpressionClass::BOUND_CONSTANT) { - return nullptr; - } - D_ASSERT(expr.children[i]->IsFoldable()); - auto constant_value = ExpressionExecutor::EvaluateScalar(GetContext(), *expr.children[i]); - auto new_constant = constant_value.DefaultTryCastAs(target_type); - if (!new_constant) { - return nullptr; - } else { - auto new_constant_expr = make_uniq(constant_value); - cast_list.push_back(std::move(new_constant_expr)); - } - } - //! We can cast, so we move the new constant - for (size_t i = 1; i < expr.children.size(); i++) { - expr.children[i] = std::move(cast_list[i - 1]); - - // expr->children[i] = std::move(new_constant_expr); - } - //! We can cast the full list, so we move the column - expr.children[0] = std::move(cast_expression.child); - return nullptr; -} - -} // namespace duckdb diff --git a/src/optimizer/rule/like_optimizations.cpp b/src/optimizer/rule/like_optimizations.cpp index 96f7b1501e8a..8b137891791f 100644 --- a/src/optimizer/rule/like_optimizations.cpp +++ b/src/optimizer/rule/like_optimizations.cpp @@ -1,162 +1 @@ -#include "duckdb/optimizer/rule/like_optimizations.hpp" -#include "duckdb/execution/expression_executor.hpp" -#include "duckdb/planner/expression/bound_function_expression.hpp" -#include "duckdb/planner/expression/bound_constant_expression.hpp" -#include "duckdb/planner/expression/bound_operator_expression.hpp" -#include "duckdb/planner/expression/bound_comparison_expression.hpp" - -namespace duckdb { - -LikeOptimizationRule::LikeOptimizationRule(ExpressionRewriter &rewriter) : Rule(rewriter) { - // match on a FunctionExpression that has a foldable ConstantExpression - auto func = make_uniq(); - func->matchers.push_back(make_uniq()); - func->matchers.push_back(make_uniq()); - func->policy = SetMatcher::Policy::ORDERED; - // we match on LIKE ("~~") and NOT LIKE ("!~~") - func->function = make_uniq(unordered_set {"!~~", "~~"}); - root = std::move(func); -} - -static bool PatternIsConstant(const string &pattern) { - for (idx_t i = 0; i < pattern.size(); i++) { - if (pattern[i] == '%' || pattern[i] == '_') { - return false; - } - } - return true; -} - -static bool PatternIsPrefix(const string &pattern) { - idx_t i; - for (i = pattern.size(); i > 0; i--) { - if (pattern[i - 1] != '%') { - break; - } - } - if (i == pattern.size()) { - // no trailing % - // cannot be a prefix - return false; - } - // continue to look in the string - // if there is a % or _ in the string (besides at the very end) this is not a prefix match - for (; i > 0; i--) { - if (pattern[i - 1] == '%' || pattern[i - 1] == '_') { - return false; - } - } - return true; -} - -static bool PatternIsSuffix(const string &pattern) { - idx_t i; - for (i = 0; i < pattern.size(); i++) { - if (pattern[i] != '%') { - break; - } - } - if (i == 0) { - // no leading % - // cannot be a suffix - return false; - } - // continue to look in the string - // if there is a % or _ in the string (besides at the beginning) this is not a suffix match - for (; i < pattern.size(); i++) { - if (pattern[i] == '%' || pattern[i] == '_') { - return false; - } - } - return true; -} - -static bool PatternIsContains(const string &pattern) { - idx_t start; - idx_t end; - for (start = 0; start < pattern.size(); start++) { - if (pattern[start] != '%') { - break; - } - } - for (end = pattern.size(); end > 0; end--) { - if (pattern[end - 1] != '%') { - break; - } - } - if (start == 0 || end == pattern.size()) { - // contains requires both a leading AND a trailing % - return false; - } - // check if there are any other special characters in the string - // if there is a % or _ in the string (besides at the beginning/end) this is not a contains match - for (idx_t i = start; i < end; i++) { - if (pattern[i] == '%' || pattern[i] == '_') { - return false; - } - } - return true; -} - -unique_ptr LikeOptimizationRule::Apply(LogicalOperator &op, vector> &bindings, - bool &changes_made, bool is_root) { - auto &root = bindings[0].get().Cast(); - auto &constant_expr = bindings[2].get().Cast(); - D_ASSERT(root.children.size() == 2); - - if (constant_expr.value.IsNull()) { - return make_uniq(Value(root.return_type)); - } - - // the constant_expr is a scalar expression that we have to fold - if (!constant_expr.IsFoldable()) { - return nullptr; - } - - auto constant_value = ExpressionExecutor::EvaluateScalar(GetContext(), constant_expr); - D_ASSERT(constant_value.type() == constant_expr.return_type); - auto &patt_str = StringValue::Get(constant_value); - - bool is_not_like = root.function.name == "!~~"; - if (PatternIsConstant(patt_str)) { - // Pattern is constant - return make_uniq(is_not_like ? ExpressionType::COMPARE_NOTEQUAL - : ExpressionType::COMPARE_EQUAL, - std::move(root.children[0]), std::move(root.children[1])); - } else if (PatternIsPrefix(patt_str)) { - // Prefix LIKE pattern : [^%_]*[%]+, ignoring underscore - return ApplyRule(root, PrefixFun::GetFunction(), patt_str, is_not_like); - } else if (PatternIsSuffix(patt_str)) { - // Suffix LIKE pattern: [%]+[^%_]*, ignoring underscore - return ApplyRule(root, SuffixFun::GetFunction(), patt_str, is_not_like); - } else if (PatternIsContains(patt_str)) { - // Contains LIKE pattern: [%]+[^%_]*[%]+, ignoring underscore - return ApplyRule(root, ContainsFun::GetFunction(), patt_str, is_not_like); - } - return nullptr; -} - -unique_ptr LikeOptimizationRule::ApplyRule(BoundFunctionExpression &expr, ScalarFunction function, - string pattern, bool is_not_like) { - // replace LIKE by an optimized function - unique_ptr result; - auto new_function = - make_uniq(expr.return_type, std::move(function), std::move(expr.children), nullptr); - - // removing "%" from the pattern - pattern.erase(std::remove(pattern.begin(), pattern.end(), '%'), pattern.end()); - - new_function->children[1] = make_uniq(Value(std::move(pattern))); - - result = std::move(new_function); - if (is_not_like) { - auto negation = make_uniq(ExpressionType::OPERATOR_NOT, LogicalType::BOOLEAN); - negation->children.push_back(std::move(result)); - result = std::move(negation); - } - - return result; -} - -} // namespace duckdb diff --git a/src/optimizer/rule/move_constants.cpp b/src/optimizer/rule/move_constants.cpp index 636265ff9131..8b137891791f 100644 --- a/src/optimizer/rule/move_constants.cpp +++ b/src/optimizer/rule/move_constants.cpp @@ -1,165 +1 @@ -#include "duckdb/optimizer/rule/move_constants.hpp" -#include "duckdb/common/exception.hpp" -#include "duckdb/common/value_operations/value_operations.hpp" -#include "duckdb/planner/expression/bound_comparison_expression.hpp" -#include "duckdb/planner/expression/bound_constant_expression.hpp" -#include "duckdb/planner/expression/bound_function_expression.hpp" -#include "duckdb/optimizer/expression_rewriter.hpp" - -namespace duckdb { - -MoveConstantsRule::MoveConstantsRule(ExpressionRewriter &rewriter) : Rule(rewriter) { - auto op = make_uniq(); - op->matchers.push_back(make_uniq()); - op->policy = SetMatcher::Policy::UNORDERED; - - auto arithmetic = make_uniq(); - // we handle multiplication, addition and subtraction because those are "easy" - // integer division makes the division case difficult - // e.g. [x / 2 = 3] means [x = 6 OR x = 7] because of truncation -> no clean rewrite rules - arithmetic->function = make_uniq(unordered_set {"+", "-", "*"}); - // we match only on integral numeric types - arithmetic->type = make_uniq(); - auto child_constant_matcher = make_uniq(); - auto child_expression_matcher = make_uniq(); - child_constant_matcher->type = make_uniq(); - child_expression_matcher->type = make_uniq(); - arithmetic->matchers.push_back(std::move(child_constant_matcher)); - arithmetic->matchers.push_back(std::move(child_expression_matcher)); - arithmetic->policy = SetMatcher::Policy::SOME; - op->matchers.push_back(std::move(arithmetic)); - root = std::move(op); -} - -unique_ptr MoveConstantsRule::Apply(LogicalOperator &op, vector> &bindings, - bool &changes_made, bool is_root) { - auto &comparison = bindings[0].get().Cast(); - auto &outer_constant = bindings[1].get().Cast(); - auto &arithmetic = bindings[2].get().Cast(); - auto &inner_constant = bindings[3].get().Cast(); - D_ASSERT(arithmetic.return_type.IsIntegral()); - D_ASSERT(arithmetic.children[0]->return_type.IsIntegral()); - if (inner_constant.value.IsNull() || outer_constant.value.IsNull()) { - return make_uniq(Value(comparison.return_type)); - } - auto &constant_type = outer_constant.return_type; - hugeint_t outer_value = IntegralValue::Get(outer_constant.value); - hugeint_t inner_value = IntegralValue::Get(inner_constant.value); - - idx_t arithmetic_child_index = arithmetic.children[0].get() == &inner_constant ? 1 : 0; - auto &op_type = arithmetic.function.name; - if (op_type == "+") { - // [x + 1 COMP 10] OR [1 + x COMP 10] - // order does not matter in addition: - // simply change right side to 10-1 (outer_constant - inner_constant) - if (!Hugeint::TrySubtractInPlace(outer_value, inner_value)) { - return nullptr; - } - auto result_value = Value::HUGEINT(outer_value); - if (!result_value.DefaultTryCastAs(constant_type)) { - if (comparison.type != ExpressionType::COMPARE_EQUAL) { - return nullptr; - } - // if the cast is not possible then the comparison is not possible - // for example, if we have x + 5 = 3, where x is an unsigned number, we will get x = -2 - // since this is not possible we can remove the entire branch here - return ExpressionRewriter::ConstantOrNull(std::move(arithmetic.children[arithmetic_child_index]), - Value::BOOLEAN(false)); - } - outer_constant.value = std::move(result_value); - } else if (op_type == "-") { - // [x - 1 COMP 10] O R [1 - x COMP 10] - // order matters in subtraction: - if (arithmetic_child_index == 0) { - // [x - 1 COMP 10] - // change right side to 10+1 (outer_constant + inner_constant) - if (!Hugeint::TryAddInPlace(outer_value, inner_value)) { - return nullptr; - } - auto result_value = Value::HUGEINT(outer_value); - if (!result_value.DefaultTryCastAs(constant_type)) { - // if the cast is not possible then an equality comparison is not possible - if (comparison.type != ExpressionType::COMPARE_EQUAL) { - return nullptr; - } - return ExpressionRewriter::ConstantOrNull(std::move(arithmetic.children[arithmetic_child_index]), - Value::BOOLEAN(false)); - } - outer_constant.value = std::move(result_value); - } else { - // [1 - x COMP 10] - // change right side to 1-10=-9 - if (!Hugeint::TrySubtractInPlace(inner_value, outer_value)) { - return nullptr; - } - auto result_value = Value::HUGEINT(inner_value); - if (!result_value.DefaultTryCastAs(constant_type)) { - // if the cast is not possible then an equality comparison is not possible - if (comparison.type != ExpressionType::COMPARE_EQUAL) { - return nullptr; - } - return ExpressionRewriter::ConstantOrNull(std::move(arithmetic.children[arithmetic_child_index]), - Value::BOOLEAN(false)); - } - outer_constant.value = std::move(result_value); - // in this case, we should also flip the comparison - // e.g. if we have [4 - x < 2] then we should have [x > 2] - comparison.type = FlipComparisonExpression(comparison.type); - } - } else { - D_ASSERT(op_type == "*"); - // [x * 2 COMP 10] OR [2 * x COMP 10] - // order does not matter in multiplication: - // change right side to 10/2 (outer_constant / inner_constant) - // but ONLY if outer_constant is cleanly divisible by the inner_constant - if (inner_value == 0) { - // x * 0, the result is either 0 or NULL - // we let the arithmetic_simplification rule take care of simplifying this first - return nullptr; - } - // check out of range for HUGEINT or not cleanly divisible - // HUGEINT is not cleanly divisible when outer_value == minimum and inner value == -1. (modulo overflow) - if ((outer_value == NumericLimits::Minimum() && inner_value == -1) || - outer_value % inner_value != 0) { - bool is_equality = comparison.type == ExpressionType::COMPARE_EQUAL; - bool is_inequality = comparison.type == ExpressionType::COMPARE_NOTEQUAL; - if (is_equality || is_inequality) { - // we know the values are not equal - // the result will be either FALSE or NULL (if COMPARE_EQUAL) - // or TRUE or NULL (if COMPARE_NOTEQUAL) - return ExpressionRewriter::ConstantOrNull(std::move(arithmetic.children[arithmetic_child_index]), - Value::BOOLEAN(is_inequality)); - } else { - // not cleanly divisible and we are doing > >= < <=, skip the simplification for now - return nullptr; - } - } - if (inner_value < 0) { - // multiply by negative value, need to flip expression - comparison.type = FlipComparisonExpression(comparison.type); - } - // else divide the RHS by the LHS - // we need to do a range check on the cast even though we do a division - // because e.g. -128 / -1 = 128, which is out of range - auto result_value = Value::HUGEINT(outer_value / inner_value); - if (!result_value.DefaultTryCastAs(constant_type)) { - return ExpressionRewriter::ConstantOrNull(std::move(arithmetic.children[arithmetic_child_index]), - Value::BOOLEAN(false)); - } - outer_constant.value = std::move(result_value); - } - // replace left side with x - // first extract x from the arithmetic expression - auto arithmetic_child = std::move(arithmetic.children[arithmetic_child_index]); - // then place in the comparison - if (comparison.left.get() == &outer_constant) { - comparison.right = std::move(arithmetic_child); - } else { - comparison.left = std::move(arithmetic_child); - } - changes_made = true; - return nullptr; -} - -} // namespace duckdb diff --git a/src/optimizer/rule/ordered_aggregate_optimizer.cpp b/src/optimizer/rule/ordered_aggregate_optimizer.cpp index 553c0e30f450..8b137891791f 100644 --- a/src/optimizer/rule/ordered_aggregate_optimizer.cpp +++ b/src/optimizer/rule/ordered_aggregate_optimizer.cpp @@ -1,102 +1 @@ -#include "duckdb/optimizer/rule/ordered_aggregate_optimizer.hpp" -#include "duckdb/catalog/catalog_entry/aggregate_function_catalog_entry.hpp" -#include "duckdb/function/function_binder.hpp" -#include "duckdb/optimizer/matcher/expression_matcher.hpp" -#include "duckdb/optimizer/expression_rewriter.hpp" -#include "duckdb/planner/expression/bound_aggregate_expression.hpp" -#include "duckdb/planner/expression/bound_constant_expression.hpp" -#include "duckdb/main/client_context.hpp" -#include "duckdb/planner/operator/logical_aggregate.hpp" - -namespace duckdb { - -OrderedAggregateOptimizer::OrderedAggregateOptimizer(ExpressionRewriter &rewriter) : Rule(rewriter) { - // we match on an OR expression within a LogicalFilter node - root = make_uniq(); - root->expr_class = ExpressionClass::BOUND_AGGREGATE; -} - -unique_ptr OrderedAggregateOptimizer::Apply(ClientContext &context, BoundAggregateExpression &aggr, - vector> &groups, bool &changes_made) { - if (!aggr.order_bys) { - // no ORDER BYs defined - return nullptr; - } - if (aggr.function.order_dependent == AggregateOrderDependent::NOT_ORDER_DEPENDENT) { - // not an order dependent aggregate but we have an ORDER BY clause - remove it - aggr.order_bys.reset(); - changes_made = true; - return nullptr; - } - - // Remove unnecessary ORDER BY clauses and return if nothing remains - if (aggr.order_bys->Simplify(groups)) { - aggr.order_bys.reset(); - changes_made = true; - return nullptr; - } - - // Rewrite first/last/arbitrary/any_value to use arg_xxx[_null] and create_sort_key - const auto &aggr_name = aggr.function.name; - string arg_xxx_name; - if (aggr_name == "last") { - arg_xxx_name = "arg_max_null"; - } else if (aggr_name == "first" || aggr_name == "arbitrary") { - arg_xxx_name = "arg_min_null"; - } else if (aggr_name == "any_value") { - arg_xxx_name = "arg_min"; - } else { - return nullptr; - } - - FunctionBinder binder(context); - vector> sort_children; - for (auto &order : aggr.order_bys->orders) { - sort_children.emplace_back(std::move(order.expression)); - - string modifier; - modifier += (order.type == OrderType::ASCENDING) ? "ASC" : "DESC"; - modifier += " NULLS"; - modifier += (order.null_order == OrderByNullType::NULLS_FIRST) ? " FIRST" : " LAST"; - sort_children.emplace_back(make_uniq(Value(modifier))); - } - aggr.order_bys.reset(); - - ErrorData error; - auto sort_key = binder.BindScalarFunction(DEFAULT_SCHEMA, "create_sort_key", std::move(sort_children), error); - if (!sort_key) { - error.Throw(); - } - - auto &children = aggr.children; - children.emplace_back(std::move(sort_key)); - - // Look up the arg_xxx_name function in the catalog - QueryErrorContext error_context; - auto &func = Catalog::GetEntry(context, SYSTEM_CATALOG, DEFAULT_SCHEMA, arg_xxx_name, - error_context); - D_ASSERT(func.type == CatalogType::AGGREGATE_FUNCTION_ENTRY); - - // bind the aggregate - vector types; - for (const auto &child : children) { - types.emplace_back(child->return_type); - } - auto best_function = binder.BindFunction(func.name, func.functions, types, error); - if (best_function == DConstants::INVALID_INDEX) { - error.Throw(); - } - // found a matching function! - auto bound_function = func.functions.GetFunctionByOffset(best_function); - return binder.BindAggregateFunction(bound_function, std::move(children), std::move(aggr.filter), - aggr.IsDistinct() ? AggregateType::DISTINCT : AggregateType::NON_DISTINCT); -} - -unique_ptr OrderedAggregateOptimizer::Apply(LogicalOperator &op, vector> &bindings, - bool &changes_made, bool is_root) { - auto &aggr = bindings[0].get().Cast(); - return Apply(rewriter.context, aggr, op.Cast().groups, changes_made); -} - -} // namespace duckdb diff --git a/src/planner/CMakeLists.txt b/src/planner/CMakeLists.txt index 19f4c28a0758..8b137891791f 100644 --- a/src/planner/CMakeLists.txt +++ b/src/planner/CMakeLists.txt @@ -1,27 +1 @@ -add_subdirectory(expression) -add_subdirectory(binder) -add_subdirectory(expression_binder) -add_subdirectory(filter) -add_subdirectory(operator) -add_subdirectory(subquery) -add_library_unity( - duckdb_planner - OBJECT - bound_result_modifier.cpp - bound_parameter_map.cpp - expression_iterator.cpp - expression.cpp - table_binding.cpp - expression_binder.cpp - joinside.cpp - logical_operator.cpp - binder.cpp - bind_context.cpp - planner.cpp - pragma_handler.cpp - logical_operator_visitor.cpp - table_filter.cpp) -set(ALL_OBJECT_FILES - ${ALL_OBJECT_FILES} $ - PARENT_SCOPE) From 7ce337e59cd5aeb1a34eed46503bed93ebcaa6d8 Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 3 Apr 2024 13:56:45 +0200 Subject: [PATCH 138/603] fixed it --- .../catalog_entry/sequence_catalog_entry.cpp | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/src/catalog/catalog_entry/sequence_catalog_entry.cpp b/src/catalog/catalog_entry/sequence_catalog_entry.cpp index 27307f956fbb..c51aef73186a 100644 --- a/src/catalog/catalog_entry/sequence_catalog_entry.cpp +++ b/src/catalog/catalog_entry/sequence_catalog_entry.cpp @@ -13,20 +13,8 @@ namespace duckdb { -// This should only be used for deserialization purposes, as we can guarantee that the result is within bounds in that -// case. -static int64_t LastValue(CreateSequenceInfo &info) { - auto usage_count = info.usage_count; - if (usage_count == 0) { - return info.start_value; - } - usage_count--; - // FIXME: this is not accounting for CYCLE behavior - return info.start_value + (info.increment * usage_count); -} - SequenceData::SequenceData(CreateSequenceInfo &info) - : usage_count(info.usage_count), counter(LastValue(info)), last_value(LastValue(info)), increment(info.increment), + : usage_count(0), counter(info.start_value), last_value(info.start_value), increment(info.increment), start_value(info.start_value), min_value(info.min_value), max_value(info.max_value), cycle(info.cycle) { } @@ -108,7 +96,9 @@ unique_ptr SequenceCatalogEntry::GetInfo() const { result->increment = seq_data.increment; result->min_value = seq_data.min_value; result->max_value = seq_data.max_value; - result->start_value = seq_data.start_value; + // To "persist" the last_value we create the sequence as if the provided START value is the current value + // Inside the SequenceData we set the usage_count to 0 so currvalue will throw if the sequence hasnt been updated yet + result->start_value = seq_data.counter; result->cycle = seq_data.cycle; result->comment = comment; return std::move(result); From 1b8e3eda8b4e452a3ba072eab87a9baea94a784f Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 3 Apr 2024 14:00:38 +0200 Subject: [PATCH 139/603] add clarifying comment --- src/catalog/catalog_entry/sequence_catalog_entry.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/catalog/catalog_entry/sequence_catalog_entry.cpp b/src/catalog/catalog_entry/sequence_catalog_entry.cpp index c51aef73186a..8d1918a2ca38 100644 --- a/src/catalog/catalog_entry/sequence_catalog_entry.cpp +++ b/src/catalog/catalog_entry/sequence_catalog_entry.cpp @@ -13,6 +13,9 @@ namespace duckdb { +// NOTE: usage_count is explicitly set to 0, +// if the sequence was serialized to disk the start_value +// was updated to the value of the last_value before serializing, keeping the state of the sequence. SequenceData::SequenceData(CreateSequenceInfo &info) : usage_count(0), counter(info.start_value), last_value(info.start_value), increment(info.increment), start_value(info.start_value), min_value(info.min_value), max_value(info.max_value), cycle(info.cycle) { From 39fb3162f968d101f331cec3fd18cbcb3c70a4fb Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 3 Apr 2024 14:27:05 +0200 Subject: [PATCH 140/603] format comment --- src/catalog/catalog_entry/sequence_catalog_entry.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/catalog/catalog_entry/sequence_catalog_entry.cpp b/src/catalog/catalog_entry/sequence_catalog_entry.cpp index 8d1918a2ca38..cc122108dcf6 100644 --- a/src/catalog/catalog_entry/sequence_catalog_entry.cpp +++ b/src/catalog/catalog_entry/sequence_catalog_entry.cpp @@ -100,7 +100,8 @@ unique_ptr SequenceCatalogEntry::GetInfo() const { result->min_value = seq_data.min_value; result->max_value = seq_data.max_value; // To "persist" the last_value we create the sequence as if the provided START value is the current value - // Inside the SequenceData we set the usage_count to 0 so currvalue will throw if the sequence hasnt been updated yet + // Inside the SequenceData we set the usage_count to 0 so currvalue will throw if the sequence hasnt been updated + // yet result->start_value = seq_data.counter; result->cycle = seq_data.cycle; result->comment = comment; From 00efd83289e8d51a14f171fd07b3815c1f116aa0 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 3 Apr 2024 15:06:04 +0200 Subject: [PATCH 141/603] Doing some extra checks to give the correct byte-position where errors happen --- .../scanner/string_value_scanner.cpp | 102 +++++++++++++----- .../table_function/global_csv_state.cpp | 2 +- .../operator/csv_scanner/util/csv_error.cpp | 4 +- .../csv_scanner/string_value_scanner.hpp | 7 +- .../csv/rejects/csv_buffer_size_rejects.test | 46 +++----- 5 files changed, 98 insertions(+), 63 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 07bf849e4a9c..4a00f957e300 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -339,10 +339,17 @@ void StringValueResult::HandleUnicodeError(idx_t col_idx, LinePosition &error_po Utf8Proc::MakeValid(&char_array[0], char_array.size()); borked_line = {char_array.begin(), char_array.end() - 1}; LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); - auto csv_error = CSVError::InvalidUTF8(state_machine.options, col_idx, lines_per_batch, borked_line, - current_line_position.begin.GetGlobalPosition(requested_size, first_nl), - error_position.GetGlobalPosition(requested_size, first_nl)); - error_handler.Error(csv_error, true); + if (current_line_position.begin == error_position) { + auto csv_error = CSVError::InvalidUTF8(state_machine.options, col_idx, lines_per_batch, borked_line, + current_line_position.begin.GetGlobalPosition(requested_size, first_nl), + error_position.GetGlobalPosition(requested_size, first_nl)); + error_handler.Error(csv_error, true); + } else { + auto csv_error = CSVError::InvalidUTF8(state_machine.options, col_idx, lines_per_batch, borked_line, + current_line_position.begin.GetGlobalPosition(requested_size, first_nl), + error_position.GetGlobalPosition(requested_size)); + error_handler.Error(csv_error, true); + } } bool StringValueResult::HandleError() { @@ -357,10 +364,17 @@ bool StringValueResult::HandleError() { switch (cur_error.type) { case CSVErrorType::TOO_MANY_COLUMNS: - csv_error = CSVError::IncorrectColumnAmountError( - state_machine.options, col_idx, lines_per_batch, borked_line, - current_line_position.begin.GetGlobalPosition(requested_size, first_nl), - line_pos.GetGlobalPosition(requested_size, first_nl)); + if (current_line_position.begin == line_pos) { + csv_error = CSVError::IncorrectColumnAmountError( + state_machine.options, col_idx, lines_per_batch, borked_line, + current_line_position.begin.GetGlobalPosition(requested_size, first_nl), + line_pos.GetGlobalPosition(requested_size, first_nl)); + } else { + csv_error = CSVError::IncorrectColumnAmountError( + state_machine.options, col_idx, lines_per_batch, borked_line, + current_line_position.begin.GetGlobalPosition(requested_size, first_nl), + line_pos.GetGlobalPosition(requested_size)); + } break; case CSVErrorType::INVALID_UNICODE: { // We have to sanitize the CSV line @@ -368,22 +382,47 @@ bool StringValueResult::HandleError() { char_array.push_back('\0'); // Null-terminate the character array Utf8Proc::MakeValid(&char_array[0], char_array.size()); borked_line = {char_array.begin(), char_array.end() - 1}; - csv_error = CSVError::InvalidUTF8(state_machine.options, col_idx, lines_per_batch, borked_line, - current_line_position.begin.GetGlobalPosition(requested_size, first_nl), - line_pos.GetGlobalPosition(requested_size, first_nl)); + if (current_line_position.begin == line_pos) { + csv_error = + CSVError::InvalidUTF8(state_machine.options, col_idx, lines_per_batch, borked_line, + current_line_position.begin.GetGlobalPosition(requested_size, first_nl), + line_pos.GetGlobalPosition(requested_size, first_nl)); + } else { + csv_error = + CSVError::InvalidUTF8(state_machine.options, col_idx, lines_per_batch, borked_line, + current_line_position.begin.GetGlobalPosition(requested_size, first_nl), + line_pos.GetGlobalPosition(requested_size)); + } break; } case CSVErrorType::UNTERMINATED_QUOTES: - csv_error = CSVError::UnterminatedQuotesError( - state_machine.options, col_idx, lines_per_batch, borked_line, - current_line_position.begin.GetGlobalPosition(requested_size, first_nl), - line_pos.GetGlobalPosition(requested_size, first_nl)); + if (current_line_position.begin == line_pos) { + csv_error = CSVError::UnterminatedQuotesError( + state_machine.options, col_idx, lines_per_batch, borked_line, + current_line_position.begin.GetGlobalPosition(requested_size, first_nl), + line_pos.GetGlobalPosition(requested_size, first_nl)); + } else { + csv_error = CSVError::UnterminatedQuotesError( + state_machine.options, col_idx, lines_per_batch, borked_line, + current_line_position.begin.GetGlobalPosition(requested_size, first_nl), + line_pos.GetGlobalPosition(requested_size)); + } break; case CSVErrorType::CAST_ERROR: - csv_error = CSVError::CastError( - state_machine.options, names[cur_error.col_idx], cur_error.error_message, cur_error.col_idx, - borked_line, lines_per_batch, current_line_position.begin.GetGlobalPosition(requested_size, first_nl), - line_pos.GetGlobalPosition(requested_size, first_nl), parse_types[cur_error.col_idx].first); + if (current_line_position.begin == line_pos) { + csv_error = CSVError::CastError( + state_machine.options, names[cur_error.col_idx], cur_error.error_message, cur_error.col_idx, + borked_line, lines_per_batch, + current_line_position.begin.GetGlobalPosition(requested_size, first_nl), + line_pos.GetGlobalPosition(requested_size, first_nl), parse_types[cur_error.col_idx].first); + } else { + csv_error = CSVError::CastError( + state_machine.options, names[cur_error.col_idx], cur_error.error_message, cur_error.col_idx, + borked_line, lines_per_batch, + current_line_position.begin.GetGlobalPosition(requested_size, first_nl), + line_pos.GetGlobalPosition(requested_size), parse_types[cur_error.col_idx].first); + } + break; default: throw InvalidInputException("CSV Error not allowed when inserting row"); @@ -499,11 +538,20 @@ bool StringValueResult::AddRowInternal() { bool first_nl; auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); - auto csv_error = CSVError::IncorrectColumnAmountError( - state_machine.options, cur_col_id - 1, lines_per_batch, borked_line, - current_line_position.begin.GetGlobalPosition(requested_size, first_nl), - last_position.GetGlobalPosition(requested_size, first_nl)); - error_handler.Error(csv_error); + if (current_line_position.begin == last_position) { + auto csv_error = CSVError::IncorrectColumnAmountError( + state_machine.options, cur_col_id - 1, lines_per_batch, borked_line, + current_line_position.begin.GetGlobalPosition(requested_size, first_nl), + last_position.GetGlobalPosition(requested_size, first_nl)); + error_handler.Error(csv_error); + } else { + auto csv_error = CSVError::IncorrectColumnAmountError( + state_machine.options, cur_col_id - 1, lines_per_batch, borked_line, + current_line_position.begin.GetGlobalPosition(requested_size, first_nl), + last_position.GetGlobalPosition(requested_size)); + error_handler.Error(csv_error); + } + // If we are here we ignore_errors, so we delete this line number_of_rows--; } @@ -1034,16 +1082,14 @@ bool StringValueScanner::MoveToNextBuffer() { // And an extra empty value to represent what comes after the delimiter result.AddRow(result, previous_buffer_handle->actual_size); lines_read++; - } - else if (states.IsQuotedCurrent()) { + } else if (states.IsQuotedCurrent()) { // Unterminated quote LinePosition current_line_start = {iterator.pos.buffer_idx, iterator.pos.buffer_pos, result.buffer_size}; result.current_line_position.begin = result.current_line_position.end; result.current_line_position.end = current_line_start; result.InvalidState(result); - } - else { + } else { result.AddRow(result, previous_buffer_handle->actual_size); lines_read++; } diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index 3ad83d0954f7..b16c2d48df79 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -279,7 +279,7 @@ void CSVGlobalState::FillRejectsTable() { // a null errors_appender.Append(Value()); } else { - errors_appender.Append(error.byte_position); + errors_appender.Append(error.byte_position + 1); } // 6. Column Index errors_appender.Append(col_idx + 1); diff --git a/src/execution/operator/csv_scanner/util/csv_error.cpp b/src/execution/operator/csv_scanner/util/csv_error.cpp index d9119a4dd9d3..18a37fa3f2ff 100644 --- a/src/execution/operator/csv_scanner/util/csv_error.cpp +++ b/src/execution/operator/csv_scanner/util/csv_error.cpp @@ -89,7 +89,7 @@ CSVError::CSVError(string error_message_p, CSVErrorType type_p, idx_t column_idx error_info(error_info_p), row_byte_position(row_byte_position), byte_position(byte_position_p) { // What were the options std::ostringstream error; - if (reader_options.ignore_errors.GetValue()){ + if (reader_options.ignore_errors.GetValue()) { RemoveNewLine(error_message); } error << error_message << '\n'; @@ -119,7 +119,7 @@ CSVError CSVError::ColumnTypesError(case_insensitive_map_t sql_types_per_ return CSVError(exception, CSVErrorType::COLUMN_NAME_TYPE_MISMATCH, {}); } -void CSVError::RemoveNewLine(string &error){ +void CSVError::RemoveNewLine(string &error) { error = StringUtil::Split(error, "\n")[0]; } diff --git a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp index 6332906f03ca..0039f9ade5b0 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp @@ -42,7 +42,12 @@ class LinePosition { } return other.buffer_size - other.buffer_pos + buffer_pos; } - idx_t GetGlobalPosition(idx_t requested_buffer_size, bool first_char_nl) { + + bool operator==(const LinePosition &other) const { + return buffer_pos == other.buffer_pos && buffer_idx == other.buffer_idx && buffer_size == other.buffer_size; + } + + idx_t GetGlobalPosition(idx_t requested_buffer_size, bool first_char_nl = false) { return requested_buffer_size * buffer_idx + buffer_pos + first_char_nl; } idx_t buffer_pos = 0; diff --git a/test/sql/copy/csv/rejects/csv_buffer_size_rejects.test b/test/sql/copy/csv/rejects/csv_buffer_size_rejects.test index 76b95cfbe731..35f44da755a0 100644 --- a/test/sql/copy/csv/rejects/csv_buffer_size_rejects.test +++ b/test/sql/copy/csv/rejects/csv_buffer_size_rejects.test @@ -7,7 +7,7 @@ require skip_reload # Test will fail on windows because byte_position is slightly different due to \r\n instead of \n require notwindows -loop buffer_size 5 10 +loop buffer_size 5 8 # Ensure that we can get the schema if we reduce the sample size and ignore errors query IIIII @@ -15,45 +15,29 @@ SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), M 'test/sql/copy/csv/data/error/mismatch/big_bad*.csv', sample_size=1, buffer_size=${buffer_size}, - rejects_table='csv_rejects_table', - ignore_errors=true); + store_rejects = true); ---- BIGINT VARCHAR 11044 11044 2 -query IIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), column_idx, column_name, error_type, csv_line, byte_position -FROM csv_rejects_table; +query IIIIIIIIIII rowsort +SELECT * EXCLUDE (scan_id, user_arguments) FROM reject_scans order by all; ---- -test/sql/copy/csv/data/error/mismatch/big_bad.csv 1 "column0" CAST B, A 10875 -test/sql/copy/csv/data/error/mismatch/big_bad.csv 1 "column0" CAST C, A 20875 -test/sql/copy/csv/data/error/mismatch/big_bad2.csv 1 "column0" CAST B, A 18395 -test/sql/copy/csv/data/error/mismatch/big_bad2.csv 1 "column0" CAST C, A 28395 - -query I -SELECT error_message -FROM csv_rejects_table where byte_position = 10875; ----- -:.*Could not convert string "B" to 'BIGINT'.* +0 test/sql/copy/csv/data/error/mismatch/big_bad.csv , \0 \0 \n 0 false {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL +1 test/sql/copy/csv/data/error/mismatch/big_bad2.csv , \0 \0 \n 0 false {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL -query I -SELECT error_message -FROM csv_rejects_table where byte_position = 20875; ----- -:.*Could not convert string "C" to 'BIGINT'.* -query I -SELECT error_message -FROM csv_rejects_table where byte_position = 18395; +query IIIIIIIII rowsort +SELECT * EXCLUDE (scan_id) FROM reject_errors order by all; ---- -:.*Could not convert string "B" to 'BIGINT'.* +0 2176 10876 10876 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +0 4176 20876 20876 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' +1 3680 18396 18396 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +1 5680 28396 28396 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' -query I -SELECT error_message -FROM csv_rejects_table where byte_position = 28395; ----- -:.*Could not convert string "C" to 'BIGINT'.* +statement ok +DROP TABLE reject_errors; statement ok -DROP TABLE csv_rejects_table; +DROP TABLE reject_scans; endloop \ No newline at end of file From b0966a06757f6935c87661869a4186b4268cb520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Wed, 3 Apr 2024 15:11:03 +0200 Subject: [PATCH 142/603] moar --- src/function/scalar/string/contains.cpp | 4 +- .../string/regexp/regexp_extract_all.cpp | 4 +- src/function/scalar/string/strip_accents.cpp | 3 +- src/function/scalar/string/substring.cpp | 18 ++- src/function/scalar/struct/struct_extract.cpp | 4 +- src/function/table/arrow.cpp | 7 +- src/function/table/arrow_conversion.cpp | 149 ++++++++++-------- 7 files changed, 109 insertions(+), 80 deletions(-) diff --git a/src/function/scalar/string/contains.cpp b/src/function/scalar/string/contains.cpp index fb68e00e8dd8..3e24ed6a71b4 100644 --- a/src/function/scalar/string/contains.cpp +++ b/src/function/scalar/string/contains.cpp @@ -22,7 +22,7 @@ static idx_t ContainsUnaligned(const unsigned char *haystack, idx_t haystack_siz UNSIGNED haystack_entry = 0; const UNSIGNED start = (sizeof(UNSIGNED) * 8) - 8; const UNSIGNED shift = (sizeof(UNSIGNED) - NEEDLE_SIZE) * 8; - for (int i = 0; i < NEEDLE_SIZE; i++) { + for (idx_t i = 0; i < NEEDLE_SIZE; i++) { needle_entry |= UNSIGNED(needle[i]) << UNSIGNED(start - i * 8); haystack_entry |= UNSIGNED(haystack[i]) << UNSIGNED(start - i * 8); } @@ -106,7 +106,7 @@ idx_t ContainsFun::Find(const unsigned char *haystack, idx_t haystack_size, cons if (location == nullptr) { return DConstants::INVALID_INDEX; } - idx_t base_offset = const_uchar_ptr_cast(location) - haystack; + idx_t base_offset = UnsafeNumericCast(const_uchar_ptr_cast(location) - haystack); haystack_size -= base_offset; haystack = const_uchar_ptr_cast(location); // switch algorithm depending on needle size diff --git a/src/function/scalar/string/regexp/regexp_extract_all.cpp b/src/function/scalar/string/regexp/regexp_extract_all.cpp index f8a5736b4374..0e6cfa7dd5c5 100644 --- a/src/function/scalar/string/regexp/regexp_extract_all.cpp +++ b/src/function/scalar/string/regexp/regexp_extract_all.cpp @@ -96,7 +96,7 @@ void ExtractSingleTuple(const string_t &string, duckdb_re2::RE2 &pattern, int32_ // Every group is a substring of the original, we can find out the offset using the pointer // the 'match_group' address is guaranteed to be bigger than that of the source D_ASSERT(const_char_ptr_cast(match_group.begin()) >= string.GetData()); - idx_t offset = match_group.begin() - string.GetData(); + auto offset = UnsafeNumericCast(match_group.begin() - string.GetData()); list_content[child_idx] = string_t(string.GetData() + offset, UnsafeNumericCast(match_group.size())); } @@ -199,7 +199,7 @@ void RegexpExtractAll::Execute(DataChunk &args, ExpressionState &state, Vector & if (group_count_p == -1) { throw InvalidInputException("Pattern failed to parse, error: '%s'", stored_re->error()); } - non_const_args->SetSize(group_count_p); + non_const_args->SetSize(UnsafeNumericCast(group_count_p)); } } diff --git a/src/function/scalar/string/strip_accents.cpp b/src/function/scalar/string/strip_accents.cpp index 758c72646ec7..1883c60f0f73 100644 --- a/src/function/scalar/string/strip_accents.cpp +++ b/src/function/scalar/string/strip_accents.cpp @@ -22,7 +22,8 @@ struct StripAccentsOperator { } // non-ascii, perform collation - auto stripped = utf8proc_remove_accents((const utf8proc_uint8_t *)input.GetData(), input.GetSize()); + auto stripped = utf8proc_remove_accents((const utf8proc_uint8_t *)input.GetData(), + UnsafeNumericCast(input.GetSize())); auto result_str = StringVector::AddString(result, const_char_ptr_cast(stripped)); free(stripped); return result_str; diff --git a/src/function/scalar/string/substring.cpp b/src/function/scalar/string/substring.cpp index a6d582c4ecd8..76f4859f2734 100644 --- a/src/function/scalar/string/substring.cpp +++ b/src/function/scalar/string/substring.cpp @@ -42,7 +42,7 @@ string_t SubstringEmptyString(Vector &result) { string_t SubstringSlice(Vector &result, const char *input_data, int64_t offset, int64_t length) { auto result_string = StringVector::EmptyString(result, UnsafeNumericCast(length)); auto result_data = result_string.GetDataWriteable(); - memcpy(result_data, input_data + offset, length); + memcpy(result_data, input_data + offset, UnsafeNumericCast(length)); result_string.Finalize(); return result_string; } @@ -88,10 +88,10 @@ string_t SubstringASCII(Vector &result, string_t input, int64_t offset, int64_t AssertInSupportedRange(input_size, offset, length); int64_t start, end; - if (!SubstringStartEnd(input_size, offset, length, start, end)) { + if (!SubstringStartEnd(UnsafeNumericCast(input_size), offset, length, start, end)) { return SubstringEmptyString(result); } - return SubstringSlice(result, input_data, start, end - start); + return SubstringSlice(result, input_data, start, UnsafeNumericCast(end - start)); } string_t SubstringFun::SubstringUnicode(Vector &result, string_t input, int64_t offset, int64_t length) { @@ -186,7 +186,8 @@ string_t SubstringFun::SubstringUnicode(Vector &result, string_t input, int64_t } D_ASSERT(end_pos >= start_pos); // after we have found these, we can slice the substring - return SubstringSlice(result, input_data, start_pos, end_pos - start_pos); + return SubstringSlice(result, input_data, UnsafeNumericCast(start_pos), + UnsafeNumericCast(end_pos - start_pos)); } string_t SubstringFun::SubstringGrapheme(Vector &result, string_t input, int64_t offset, int64_t length) { @@ -198,14 +199,14 @@ string_t SubstringFun::SubstringGrapheme(Vector &result, string_t input, int64_t // we don't know yet if the substring is ascii, but we assume it is (for now) // first get the start and end as if this was an ascii string int64_t start, end; - if (!SubstringStartEnd(input_size, offset, length, start, end)) { + if (!SubstringStartEnd(UnsafeNumericCast(input_size), offset, length, start, end)) { return SubstringEmptyString(result); } // now check if all the characters between 0 and end are ascii characters // note that we scan one further to check for a potential combining diacritics (e.g. i + diacritic is ï) bool is_ascii = true; - idx_t ascii_end = MinValue(end + 1, input_size); + idx_t ascii_end = MinValue(UnsafeNumericCast(end + 1), input_size); for (idx_t i = 0; i < ascii_end; i++) { if (input_data[i] & 0x80) { // found a non-ascii character: eek @@ -229,7 +230,7 @@ string_t SubstringFun::SubstringGrapheme(Vector &result, string_t input, int64_t return true; }); // now call substring start and end again, but with the number of unicode characters this time - SubstringStartEnd(num_characters, offset, length, start, end); + SubstringStartEnd(UnsafeNumericCast(num_characters), offset, length, start, end); } // now scan the graphemes of the string to find the positions of the start and end characters @@ -249,7 +250,8 @@ string_t SubstringFun::SubstringGrapheme(Vector &result, string_t input, int64_t return SubstringEmptyString(result); } // after we have found these, we can slice the substring - return SubstringSlice(result, input_data, start_pos, end_pos - start_pos); + return SubstringSlice(result, input_data, UnsafeNumericCast(start_pos), + UnsafeNumericCast(end_pos - start_pos)); } struct SubstringUnicodeOp { diff --git a/src/function/scalar/struct/struct_extract.cpp b/src/function/scalar/struct/struct_extract.cpp index 50e033b77079..2572cb51bc30 100644 --- a/src/function/scalar/struct/struct_extract.cpp +++ b/src/function/scalar/struct/struct_extract.cpp @@ -133,8 +133,8 @@ static unique_ptr StructExtractBindIndex(ClientContext &context, S throw BinderException("Key index %lld for struct_extract out of range - expected an index between 1 and %llu", index, struct_children.size()); } - bound_function.return_type = struct_children[index - 1].second; - return make_uniq(index - 1); + bound_function.return_type = struct_children[NumericCast(index - 1)].second; + return make_uniq(NumericCast(index - 1)); } static unique_ptr PropagateStructExtractStats(ClientContext &context, FunctionStatisticsInput &input) { diff --git a/src/function/table/arrow.cpp b/src/function/table/arrow.cpp index a65257a8363b..a38576170b31 100644 --- a/src/function/table/arrow.cpp +++ b/src/function/table/arrow.cpp @@ -117,7 +117,7 @@ static unique_ptr GetArrowLogicalTypeNoDictionary(ArrowSchema &schema return list_type; } else if (format[0] == '+' && format[1] == 'w') { std::string parameters = format.substr(format.find(':') + 1); - idx_t fixed_size = std::stoi(parameters); + auto fixed_size = NumericCast(std::stoi(parameters)); auto child_type = ArrowTableFunction::GetArrowLogicalType(*schema.children[0]); auto list_type = make_uniq(LogicalType::ARRAY(child_type->GetDuckType(), fixed_size), fixed_size); list_type->AddChild(std::move(child_type)); @@ -197,7 +197,7 @@ static unique_ptr GetArrowLogicalTypeNoDictionary(ArrowSchema &schema return make_uniq(LogicalType::BLOB, ArrowVariableSizeType::SUPER_SIZE); } else if (format[0] == 'w') { std::string parameters = format.substr(format.find(':') + 1); - idx_t fixed_size = std::stoi(parameters); + auto fixed_size = NumericCast(std::stoi(parameters)); return make_uniq(LogicalType::BLOB, fixed_size); } else if (format[0] == 't' && format[1] == 's') { // Timestamp with Timezone @@ -366,7 +366,8 @@ void ArrowTableFunction::ArrowScanFunction(ClientContext &context, TableFunction return; } } - int64_t output_size = MinValue(STANDARD_VECTOR_SIZE, state.chunk->arrow_array.length - state.chunk_offset); + auto output_size = + MinValue(STANDARD_VECTOR_SIZE, NumericCast(state.chunk->arrow_array.length) - state.chunk_offset); data.lines_read += output_size; if (global_state.CanRemoveFilterColumns()) { state.all_columns.Reset(); diff --git a/src/function/table/arrow_conversion.cpp b/src/function/table/arrow_conversion.cpp index 55058683b24d..c82b6a64ad4d 100644 --- a/src/function/table/arrow_conversion.cpp +++ b/src/function/table/arrow_conversion.cpp @@ -40,12 +40,12 @@ idx_t GetEffectiveOffset(ArrowArray &array, int64_t parent_offset, const ArrowSc if (nested_offset != -1) { // The parent of this array is a list // We just ignore the parent offset, it's already applied to the list - return array.offset + nested_offset; + return UnsafeNumericCast(array.offset + nested_offset); } // Parent offset is set in the case of a struct, it applies to all child arrays // 'chunk_offset' is how much of the chunk we've already scanned, in case the chunk size exceeds // STANDARD_VECTOR_SIZE - return array.offset + parent_offset + state.chunk_offset; + return UnsafeNumericCast(array.offset + parent_offset) + state.chunk_offset; } template @@ -158,7 +158,8 @@ static void ArrowToDuckDBList(Vector &vector, ArrowArray &array, ArrowArrayScanS ListVector::Reserve(vector, list_size); ListVector::SetListSize(vector, list_size); auto &child_vector = ListVector::GetEntry(vector); - SetValidityMask(child_vector, *array.children[0], scan_state, list_size, array.offset, start_offset); + SetValidityMask(child_vector, *array.children[0], scan_state, list_size, array.offset, + NumericCast(start_offset)); auto &list_mask = FlatVector::Validity(vector); if (parent_mask) { //! Since this List is owned by a struct we must guarantee their validity map matches on Null @@ -183,13 +184,16 @@ static void ArrowToDuckDBList(Vector &vector, ArrowArray &array, ArrowArrayScanS switch (array_physical_type) { case ArrowArrayPhysicalType::DICTIONARY_ENCODED: // TODO: add support for offsets - ColumnArrowToDuckDBDictionary(child_vector, child_array, child_state, list_size, child_type, start_offset); + ColumnArrowToDuckDBDictionary(child_vector, child_array, child_state, list_size, child_type, + NumericCast(start_offset)); break; case ArrowArrayPhysicalType::RUN_END_ENCODED: - ColumnArrowToDuckDBRunEndEncoded(child_vector, child_array, child_state, list_size, child_type, start_offset); + ColumnArrowToDuckDBRunEndEncoded(child_vector, child_array, child_state, list_size, child_type, + NumericCast(start_offset)); break; case ArrowArrayPhysicalType::DEFAULT: - ColumnArrowToDuckDB(child_vector, child_array, child_state, list_size, child_type, start_offset); + ColumnArrowToDuckDB(child_vector, child_array, child_state, list_size, child_type, + NumericCast(start_offset)); break; default: throw NotImplementedException("ArrowArrayPhysicalType not recognized"); @@ -209,7 +213,8 @@ static void ArrowToDuckDBArray(Vector &vector, ArrowArray &array, ArrowArrayScan SetValidityMask(vector, array, scan_state, size, parent_offset, nested_offset); auto &child_vector = ArrayVector::GetEntry(vector); - SetValidityMask(child_vector, *array.children[0], scan_state, child_count, array.offset, child_offset); + SetValidityMask(child_vector, *array.children[0], scan_state, child_count, array.offset, + NumericCast(child_offset)); auto &array_mask = FlatVector::Validity(vector); if (parent_mask) { @@ -244,9 +249,10 @@ static void ArrowToDuckDBArray(Vector &vector, ArrowArray &array, ArrowArrayScan } else { if (child_array.dictionary) { ColumnArrowToDuckDBDictionary(child_vector, child_array, child_state, child_count, child_type, - child_offset); + NumericCast(child_offset)); } else { - ColumnArrowToDuckDB(child_vector, child_array, child_state, child_count, child_type, child_offset); + ColumnArrowToDuckDB(child_vector, child_array, child_state, child_count, child_type, + NumericCast(child_offset)); } } } @@ -339,8 +345,9 @@ static void SetVectorString(Vector &vector, idx_t size, char *cdata, T *offsets) static void DirectConversion(Vector &vector, ArrowArray &array, const ArrowScanLocalState &scan_state, int64_t nested_offset, uint64_t parent_offset) { auto internal_type = GetTypeIdSize(vector.GetType().InternalType()); - auto data_ptr = ArrowBufferData(array, 1) + - internal_type * GetEffectiveOffset(array, parent_offset, scan_state, nested_offset); + auto data_ptr = + ArrowBufferData(array, 1) + + internal_type * GetEffectiveOffset(array, NumericCast(parent_offset), scan_state, nested_offset); FlatVector::SetData(vector, data_ptr); } @@ -595,7 +602,7 @@ static void ColumnArrowToDuckDBRunEndEncoded(Vector &vector, ArrowArray &array, auto &scan_state = array_state.state; D_ASSERT(run_ends_array.length == values_array.length); - auto compressed_size = run_ends_array.length; + auto compressed_size = NumericCast(run_ends_array.length); // Create a vector for the run ends and the values auto &run_end_encoding = array_state.RunEndEncoding(); if (!run_end_encoding.run_ends) { @@ -606,11 +613,12 @@ static void ColumnArrowToDuckDBRunEndEncoded(Vector &vector, ArrowArray &array, ColumnArrowToDuckDB(*run_end_encoding.run_ends, run_ends_array, array_state, compressed_size, run_ends_type); auto &values = *run_end_encoding.values; - SetValidityMask(values, values_array, scan_state, compressed_size, parent_offset, nested_offset); + SetValidityMask(values, values_array, scan_state, compressed_size, NumericCast(parent_offset), + nested_offset); ColumnArrowToDuckDB(values, values_array, array_state, compressed_size, values_type); } - idx_t scan_offset = GetEffectiveOffset(array, parent_offset, scan_state, nested_offset); + idx_t scan_offset = GetEffectiveOffset(array, NumericCast(parent_offset), scan_state, nested_offset); auto physical_type = run_ends_type.GetDuckType().InternalType(); switch (physical_type) { case PhysicalType::INT16: @@ -641,12 +649,12 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca //! Arrow bit-packs boolean values //! Lets first figure out where we are in the source array auto src_ptr = ArrowBufferData(array, 1) + - GetEffectiveOffset(array, parent_offset, scan_state, nested_offset) / 8; + GetEffectiveOffset(array, NumericCast(parent_offset), scan_state, nested_offset) / 8; auto tgt_ptr = (uint8_t *)FlatVector::GetData(vector); int src_pos = 0; idx_t cur_bit = scan_state.chunk_offset % 8; if (nested_offset != -1) { - cur_bit = nested_offset % 8; + cur_bit = NumericCast(nested_offset % 8); } for (idx_t row = 0; row < size; row++) { if ((src_ptr[src_pos] & (1 << cur_bit)) == 0) { @@ -686,11 +694,11 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca auto cdata = ArrowBufferData(array, 2); if (size_type == ArrowVariableSizeType::SUPER_SIZE) { auto offsets = ArrowBufferData(array, 1) + - GetEffectiveOffset(array, parent_offset, scan_state, nested_offset); + GetEffectiveOffset(array, NumericCast(parent_offset), scan_state, nested_offset); SetVectorString(vector, size, cdata, offsets); } else { auto offsets = ArrowBufferData(array, 1) + - GetEffectiveOffset(array, parent_offset, scan_state, nested_offset); + GetEffectiveOffset(array, NumericCast(parent_offset), scan_state, nested_offset); SetVectorString(vector, size, cdata, offsets); } break; @@ -706,7 +714,7 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca case ArrowDateTimeType::MILLISECONDS: { //! convert date from nanoseconds to days auto src_ptr = ArrowBufferData(array, 1) + - GetEffectiveOffset(array, parent_offset, scan_state, nested_offset); + GetEffectiveOffset(array, NumericCast(parent_offset), scan_state, nested_offset); auto tgt_ptr = FlatVector::GetData(vector); for (idx_t row = 0; row < size; row++) { tgt_ptr[row] = date_t( @@ -723,21 +731,24 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca auto precision = arrow_type.GetDateTimeType(); switch (precision) { case ArrowDateTimeType::SECONDS: { - TimeConversion(vector, array, scan_state, nested_offset, parent_offset, size, 1000000); + TimeConversion(vector, array, scan_state, nested_offset, NumericCast(parent_offset), size, + 1000000); break; } case ArrowDateTimeType::MILLISECONDS: { - TimeConversion(vector, array, scan_state, nested_offset, parent_offset, size, 1000); + TimeConversion(vector, array, scan_state, nested_offset, NumericCast(parent_offset), size, + 1000); break; } case ArrowDateTimeType::MICROSECONDS: { - TimeConversion(vector, array, scan_state, nested_offset, parent_offset, size, 1); + TimeConversion(vector, array, scan_state, nested_offset, NumericCast(parent_offset), size, + 1); break; } case ArrowDateTimeType::NANOSECONDS: { auto tgt_ptr = FlatVector::GetData(vector); auto src_ptr = ArrowBufferData(array, 1) + - GetEffectiveOffset(array, parent_offset, scan_state, nested_offset); + GetEffectiveOffset(array, NumericCast(parent_offset), scan_state, nested_offset); for (idx_t row = 0; row < size; row++) { tgt_ptr[row].micros = src_ptr[row] / 1000; } @@ -752,11 +763,13 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca auto precision = arrow_type.GetDateTimeType(); switch (precision) { case ArrowDateTimeType::SECONDS: { - TimestampTZConversion(vector, array, scan_state, nested_offset, parent_offset, size, 1000000); + TimestampTZConversion(vector, array, scan_state, nested_offset, NumericCast(parent_offset), size, + 1000000); break; } case ArrowDateTimeType::MILLISECONDS: { - TimestampTZConversion(vector, array, scan_state, nested_offset, parent_offset, size, 1000); + TimestampTZConversion(vector, array, scan_state, nested_offset, NumericCast(parent_offset), size, + 1000); break; } case ArrowDateTimeType::MICROSECONDS: { @@ -766,7 +779,7 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca case ArrowDateTimeType::NANOSECONDS: { auto tgt_ptr = FlatVector::GetData(vector); auto src_ptr = ArrowBufferData(array, 1) + - GetEffectiveOffset(array, parent_offset, scan_state, nested_offset); + GetEffectiveOffset(array, NumericCast(parent_offset), scan_state, nested_offset); for (idx_t row = 0; row < size; row++) { tgt_ptr[row].value = src_ptr[row] / 1000; } @@ -781,22 +794,25 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca auto precision = arrow_type.GetDateTimeType(); switch (precision) { case ArrowDateTimeType::SECONDS: { - IntervalConversionUs(vector, array, scan_state, nested_offset, parent_offset, size, 1000000); + IntervalConversionUs(vector, array, scan_state, nested_offset, NumericCast(parent_offset), size, + 1000000); break; } case ArrowDateTimeType::DAYS: case ArrowDateTimeType::MILLISECONDS: { - IntervalConversionUs(vector, array, scan_state, nested_offset, parent_offset, size, 1000); + IntervalConversionUs(vector, array, scan_state, nested_offset, NumericCast(parent_offset), size, + 1000); break; } case ArrowDateTimeType::MICROSECONDS: { - IntervalConversionUs(vector, array, scan_state, nested_offset, parent_offset, size, 1); + IntervalConversionUs(vector, array, scan_state, nested_offset, NumericCast(parent_offset), size, + 1); break; } case ArrowDateTimeType::NANOSECONDS: { auto tgt_ptr = FlatVector::GetData(vector); auto src_ptr = ArrowBufferData(array, 1) + - GetEffectiveOffset(array, parent_offset, scan_state, nested_offset); + GetEffectiveOffset(array, NumericCast(parent_offset), scan_state, nested_offset); for (idx_t row = 0; row < size; row++) { tgt_ptr[row].micros = src_ptr[row] / 1000; tgt_ptr[row].days = 0; @@ -805,11 +821,13 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca break; } case ArrowDateTimeType::MONTHS: { - IntervalConversionMonths(vector, array, scan_state, nested_offset, parent_offset, size); + IntervalConversionMonths(vector, array, scan_state, nested_offset, NumericCast(parent_offset), + size); break; } case ArrowDateTimeType::MONTH_DAY_NANO: { - IntervalConversionMonthDayNanos(vector, array, scan_state, nested_offset, parent_offset, size); + IntervalConversionMonthDayNanos(vector, array, scan_state, nested_offset, + NumericCast(parent_offset), size); break; } default: @@ -820,8 +838,8 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca case LogicalTypeId::DECIMAL: { auto val_mask = FlatVector::Validity(vector); //! We have to convert from INT128 - auto src_ptr = - ArrowBufferData(array, 1) + GetEffectiveOffset(array, parent_offset, scan_state, nested_offset); + auto src_ptr = ArrowBufferData(array, 1) + + GetEffectiveOffset(array, NumericCast(parent_offset), scan_state, nested_offset); switch (vector.GetType().InternalType()) { case PhysicalType::INT16: { auto tgt_ptr = FlatVector::GetData(vector); @@ -859,7 +877,8 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca case PhysicalType::INT128: { FlatVector::SetData(vector, ArrowBufferData(array, 1) + GetTypeIdSize(vector.GetType().InternalType()) * - GetEffectiveOffset(array, parent_offset, scan_state, nested_offset)); + GetEffectiveOffset(array, NumericCast(parent_offset), + scan_state, nested_offset)); break; } default: @@ -869,19 +888,23 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca break; } case LogicalTypeId::BLOB: { - ArrowToDuckDBBlob(vector, array, scan_state, size, arrow_type, nested_offset, parent_offset); + ArrowToDuckDBBlob(vector, array, scan_state, size, arrow_type, nested_offset, + NumericCast(parent_offset)); break; } case LogicalTypeId::LIST: { - ArrowToDuckDBList(vector, array, array_state, size, arrow_type, nested_offset, parent_mask, parent_offset); + ArrowToDuckDBList(vector, array, array_state, size, arrow_type, nested_offset, parent_mask, + NumericCast(parent_offset)); break; } case LogicalTypeId::ARRAY: { - ArrowToDuckDBArray(vector, array, array_state, size, arrow_type, nested_offset, parent_mask, parent_offset); + ArrowToDuckDBArray(vector, array, array_state, size, arrow_type, nested_offset, parent_mask, + NumericCast(parent_offset)); break; } case LogicalTypeId::MAP: { - ArrowToDuckDBList(vector, array, array_state, size, arrow_type, nested_offset, parent_mask, parent_offset); + ArrowToDuckDBList(vector, array, array_state, size, arrow_type, nested_offset, parent_mask, + NumericCast(parent_offset)); ArrowToDuckDBMapVerify(vector, size); break; } @@ -889,7 +912,7 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca //! Fill the children auto &child_entries = StructVector::GetEntries(vector); auto &struct_validity_mask = FlatVector::Validity(vector); - for (int64_t child_idx = 0; child_idx < array.n_children; child_idx++) { + for (idx_t child_idx = 0; child_idx < NumericCast(array.n_children); child_idx++) { auto &child_entry = *child_entries[child_idx]; auto &child_array = *array.children[child_idx]; auto &child_type = arrow_type[child_idx]; @@ -909,15 +932,15 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca switch (array_physical_type) { case ArrowArrayPhysicalType::DICTIONARY_ENCODED: ColumnArrowToDuckDBDictionary(child_entry, child_array, child_state, size, child_type, nested_offset, - &struct_validity_mask, array.offset); + &struct_validity_mask, NumericCast(array.offset)); break; case ArrowArrayPhysicalType::RUN_END_ENCODED: ColumnArrowToDuckDBRunEndEncoded(child_entry, child_array, child_state, size, child_type, nested_offset, - &struct_validity_mask, array.offset); + &struct_validity_mask, NumericCast(array.offset)); break; case ArrowArrayPhysicalType::DEFAULT: ColumnArrowToDuckDB(child_entry, child_array, child_state, size, child_type, nested_offset, - &struct_validity_mask, array.offset); + &struct_validity_mask, NumericCast(array.offset)); break; default: throw NotImplementedException("ArrowArrayPhysicalType not recognized"); @@ -933,13 +956,13 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca auto &validity_mask = FlatVector::Validity(vector); duckdb::vector children; - for (int64_t child_idx = 0; child_idx < array.n_children; child_idx++) { + for (idx_t child_idx = 0; child_idx < NumericCast(array.n_children); child_idx++) { Vector child(members[child_idx].second, size); auto &child_array = *array.children[child_idx]; auto &child_state = array_state.GetChild(child_idx); auto &child_type = arrow_type[child_idx]; - SetValidityMask(child, child_array, scan_state, size, parent_offset, nested_offset); + SetValidityMask(child, child_array, scan_state, size, NumericCast(parent_offset), nested_offset); auto array_physical_type = GetArrowArrayPhysicalType(child_type); switch (array_physical_type) { @@ -960,7 +983,7 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca } for (idx_t row_idx = 0; row_idx < size; row_idx++) { - auto tag = type_ids[row_idx]; + auto tag = NumericCast(type_ids[row_idx]); auto out_of_range = tag < 0 || tag >= array.n_children; if (out_of_range) { @@ -982,7 +1005,7 @@ template static void SetSelectionVectorLoop(SelectionVector &sel, data_ptr_t indices_p, idx_t size) { auto indices = reinterpret_cast(indices_p); for (idx_t row = 0; row < size; row++) { - sel.set_index(row, indices[row]); + sel.set_index(row, UnsafeNumericCast(indices[row])); } } @@ -994,7 +1017,7 @@ static void SetSelectionVectorLoopWithChecks(SelectionVector &sel, data_ptr_t in if (indices[row] > NumericLimits::Maximum()) { throw ConversionException("DuckDB only supports indices that fit on an uint32"); } - sel.set_index(row, indices[row]); + sel.set_index(row, NumericCast(indices[row])); } } @@ -1004,7 +1027,7 @@ static void SetMaskedSelectionVectorLoop(SelectionVector &sel, data_ptr_t indice auto indices = reinterpret_cast(indices_p); for (idx_t row = 0; row < size; row++) { if (mask.RowIsValid(row)) { - sel.set_index(row, indices[row]); + sel.set_index(row, UnsafeNumericCast(indices[row])); } else { //! Need to point out to last element sel.set_index(row, last_element_pos); @@ -1119,22 +1142,23 @@ static void ColumnArrowToDuckDBDictionary(Vector &vector, ArrowArray &array, Arr const bool has_nulls = CanContainNull(array, parent_mask); if (array_state.CacheOutdated(array.dictionary)) { //! We need to set the dictionary data for this column - auto base_vector = make_uniq(vector.GetType(), array.dictionary->length); - SetValidityMask(*base_vector, *array.dictionary, scan_state, array.dictionary->length, 0, 0, has_nulls); + auto base_vector = make_uniq(vector.GetType(), NumericCast(array.dictionary->length)); + SetValidityMask(*base_vector, *array.dictionary, scan_state, NumericCast(array.dictionary->length), 0, 0, + has_nulls); auto &dictionary_type = arrow_type.GetDictionary(); auto arrow_physical_type = GetArrowArrayPhysicalType(dictionary_type); switch (arrow_physical_type) { case ArrowArrayPhysicalType::DICTIONARY_ENCODED: - ColumnArrowToDuckDBDictionary(*base_vector, *array.dictionary, array_state, array.dictionary->length, - dictionary_type); + ColumnArrowToDuckDBDictionary(*base_vector, *array.dictionary, array_state, + NumericCast(array.dictionary->length), dictionary_type); break; case ArrowArrayPhysicalType::RUN_END_ENCODED: - ColumnArrowToDuckDBRunEndEncoded(*base_vector, *array.dictionary, array_state, array.dictionary->length, - dictionary_type); + ColumnArrowToDuckDBRunEndEncoded(*base_vector, *array.dictionary, array_state, + NumericCast(array.dictionary->length), dictionary_type); break; case ArrowArrayPhysicalType::DEFAULT: - ColumnArrowToDuckDB(*base_vector, *array.dictionary, array_state, array.dictionary->length, - dictionary_type); + ColumnArrowToDuckDB(*base_vector, *array.dictionary, array_state, + NumericCast(array.dictionary->length), dictionary_type); break; default: throw NotImplementedException("ArrowArrayPhysicalType not recognized"); @@ -1143,14 +1167,14 @@ static void ColumnArrowToDuckDBDictionary(Vector &vector, ArrowArray &array, Arr } auto offset_type = arrow_type.GetDuckType(); //! Get Pointer to Indices of Dictionary - auto indices = - ArrowBufferData(array, 1) + - GetTypeIdSize(offset_type.InternalType()) * GetEffectiveOffset(array, parent_offset, scan_state, nested_offset); + auto indices = ArrowBufferData(array, 1) + + GetTypeIdSize(offset_type.InternalType()) * + GetEffectiveOffset(array, NumericCast(parent_offset), scan_state, nested_offset); SelectionVector sel; if (has_nulls) { ValidityMask indices_validity; - GetValidityMask(indices_validity, array, scan_state, size, parent_offset); + GetValidityMask(indices_validity, array, scan_state, size, NumericCast(parent_offset)); if (parent_mask && !parent_mask->AllValid()) { auto &struct_validity_mask = *parent_mask; for (idx_t i = 0; i < size; i++) { @@ -1159,7 +1183,8 @@ static void ColumnArrowToDuckDBDictionary(Vector &vector, ArrowArray &array, Arr } } } - SetSelectionVector(sel, indices, offset_type, size, &indices_validity, array.dictionary->length); + SetSelectionVector(sel, indices, offset_type, size, &indices_validity, + NumericCast(array.dictionary->length)); } else { SetSelectionVector(sel, indices, offset_type, size); } From eca3caf1404d6aa75d8e3beff9ed212594373646 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 3 Apr 2024 15:42:30 +0200 Subject: [PATCH 143/603] Incorrect Column Amount --- .../scanner/string_value_scanner.cpp | 1 + .../operator/csv_scanner/util/csv_error.cpp | 4 +- .../csv_incorrect_columns_amount_rejects.test | 125 ++++++++++-------- 3 files changed, 71 insertions(+), 59 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 4a00f957e300..d3e980a57392 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -121,6 +121,7 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size // We error pointing to the current value error. current_errors.push_back({CSVErrorType::TOO_MANY_COLUMNS, cur_col_id, last_position}); } + cur_col_id++; return; } diff --git a/src/execution/operator/csv_scanner/util/csv_error.cpp b/src/execution/operator/csv_scanner/util/csv_error.cpp index 18a37fa3f2ff..da9f5d5e2435 100644 --- a/src/execution/operator/csv_scanner/util/csv_error.cpp +++ b/src/execution/operator/csv_scanner/util/csv_error.cpp @@ -214,10 +214,10 @@ CSVError CSVError::IncorrectColumnAmountError(const CSVReaderOptions &options, i error << "Expected Number of Columns: " << options.dialect_options.num_cols << " Found: " << actual_columns + 1; if (actual_columns >= options.dialect_options.num_cols) { return CSVError(error.str(), CSVErrorType::TOO_MANY_COLUMNS, actual_columns, csv_row, error_info, - row_byte_position, byte_position, options, how_to_fix_it.str()); + row_byte_position, byte_position - 1, options, how_to_fix_it.str()); } else { return CSVError(error.str(), CSVErrorType::TOO_FEW_COLUMNS, actual_columns, csv_row, error_info, - row_byte_position, byte_position, options, how_to_fix_it.str()); + row_byte_position, byte_position - 1, options, how_to_fix_it.str()); } } diff --git a/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test b/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test index 070b413a8497..2b59e17547d3 100644 --- a/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test +++ b/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test @@ -11,104 +11,115 @@ statement ok SELECT * FROM read_csv( 'data/csv/rejects/incorrect_columns/few_columns.csv', columns = {'a': 'INTEGER', 'b': 'INTEGER', 'c': 'INTEGER', 'd': 'INTEGER'}, - rejects_table='csv_rejects_table', - ignore_errors=true, auto_detect=false, header = 1); + store_rejects=true, auto_detect=false, header = 1); -query IIIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position -FROM csv_rejects_table; +query IIIIIIIIII rowsort +FROM reject_errors order by all; ---- -data/csv/rejects/incorrect_columns/few_columns.csv 1814 3 "d" MISSING COLUMNS 1,2,3 14504 -data/csv/rejects/incorrect_columns/few_columns.csv 1823 1 "b" MISSING COLUMNS 1 14574 -data/csv/rejects/incorrect_columns/few_columns.csv 2378 1 "b" MISSING COLUMNS 1 19008 -data/csv/rejects/incorrect_columns/few_columns.csv 2762 2 "c" MISSING COLUMNS 1,2 22074 +3 0 1814 14505 14510 3 d MISSING COLUMNS 1,2,3 Expected Number of Columns: 4 Found: 3 +3 0 1823 14575 14576 1 b MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 1 +3 0 2378 19009 19010 1 b MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 1 +3 0 2762 22075 22078 2 c MISSING COLUMNS 1,2 Expected Number of Columns: 4 Found: 2 statement ok -DROP TABLE csv_rejects_table; +DROP TABLE reject_errors; + +statement ok +DROP TABLE reject_scans; statement ok SELECT * FROM read_csv( 'data/csv/rejects/incorrect_columns/many_columns.csv', columns = {'a': 'INTEGER', 'b': 'INTEGER', 'c': 'INTEGER', 'd': 'INTEGER'}, - rejects_table='csv_rejects_table', - ignore_errors=true, auto_detect=false, header = 1); + store_rejects=true, auto_detect=false, header = 1); -query IIIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position -FROM csv_rejects_table; +query IIIIIIIIII rowsort +FROM reject_errors order by all; ---- -data/csv/rejects/incorrect_columns/many_columns.csv 1096 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 8760 -data/csv/rejects/incorrect_columns/many_columns.csv 1159 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 9268 -data/csv/rejects/incorrect_columns/many_columns.csv 1206 5 NULL TOO MANY COLUMNS 1,2,3,4,5 9648 -data/csv/rejects/incorrect_columns/many_columns.csv 2769 5 NULL TOO MANY COLUMNS 1,2,3,4,5 22154 +7 0 1096 8761 8768 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5 +7 0 1096 8761 8770 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6 +7 0 1159 9269 9276 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5 +7 0 1159 9269 9278 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6 +7 0 1206 9649 9656 5 NULL TOO MANY COLUMNS 1,2,3,4,5 Expected Number of Columns: 4 Found: 5 +7 0 2769 22155 22162 5 NULL TOO MANY COLUMNS 1,2,3,4,5 Expected Number of Columns: 4 Found: 5 statement ok -DROP TABLE csv_rejects_table; +DROP TABLE reject_errors; + +statement ok +DROP TABLE reject_scans; statement ok SELECT * FROM read_csv( 'data/csv/rejects/incorrect_columns/mix_columns.csv', columns = {'a': 'INTEGER', 'b': 'INTEGER', 'c': 'INTEGER', 'd': 'INTEGER'}, - rejects_table='csv_rejects_table', - ignore_errors=true, auto_detect=false, header = 1); + store_rejects=true, auto_detect=false, header = 1); -query IIIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position -FROM csv_rejects_table; +query IIIIIIIIII rowsort +FROM reject_errors order by all; ---- -data/csv/rejects/incorrect_columns/mix_columns.csv 1604 1 "b" MISSING COLUMNS 1 12824 -data/csv/rejects/incorrect_columns/mix_columns.csv 1671 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 13354 -data/csv/rejects/incorrect_columns/mix_columns.csv 2751 2 "c" MISSING COLUMNS 1,2 21998 -data/csv/rejects/incorrect_columns/mix_columns.csv 2768 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 22130 +11 0 1604 12825 12826 1 b MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 1 +11 0 1671 13355 13362 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5 +11 0 1671 13355 13364 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6 +11 0 2751 21999 22002 2 c MISSING COLUMNS 1,2 Expected Number of Columns: 4 Found: 2 +11 0 2768 22131 22138 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5 +11 0 2768 22131 22140 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6 + # Different Buffer Sizes loop buffer_size 10 15 statement ok -DROP TABLE csv_rejects_table; +DROP TABLE reject_errors; + +statement ok +DROP TABLE reject_scans; statement ok SELECT * FROM read_csv( 'data/csv/rejects/incorrect_columns/small_mix.csv', columns = {'a': 'INTEGER', 'b': 'INTEGER', 'c': 'INTEGER', 'd': 'INTEGER'}, - rejects_table='csv_rejects_table', - ignore_errors=true, auto_detect=false, header = 1); + store_rejects=true, auto_detect=false, header = 1); -query IIIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position -FROM csv_rejects_table; +query IIIIIIIII rowsort +SELECT * EXCLUDE (scan_id) FROM reject_errors order by all ---- -data/csv/rejects/incorrect_columns/small_mix.csv 3 5 NULL TOO MANY COLUMNS 1,2,3,4,5 16 -data/csv/rejects/incorrect_columns/small_mix.csv 4 3 "d" MISSING COLUMNS 1,2,3 26 +0 3 17 24 5 NULL TOO MANY COLUMNS 1,2,3,4,5 Expected Number of Columns: 4 Found: 5 +0 4 27 32 3 d MISSING COLUMNS 1,2,3 Expected Number of Columns: 4 Found: 3 endloop # All files statement ok -DROP TABLE csv_rejects_table; +DROP TABLE reject_errors; + +statement ok +DROP TABLE reject_scans; statement ok SELECT * FROM read_csv( 'data/csv/rejects/incorrect_columns/*.csv', columns = {'a': 'INTEGER', 'b': 'INTEGER', 'c': 'INTEGER', 'd': 'INTEGER'}, - rejects_table='csv_rejects_table', - ignore_errors=true, auto_detect=false, header = 1); + store_rejects=true, auto_detect=false, header = 1); -query IIIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position -FROM csv_rejects_table; +query IIIIIIIIII rowsort +FROM reject_errors order by all ---- -data/csv/rejects/incorrect_columns/few_columns.csv 1814 3 "d" MISSING COLUMNS 1,2,3 14504 -data/csv/rejects/incorrect_columns/few_columns.csv 1823 1 "b" MISSING COLUMNS 1 14574 -data/csv/rejects/incorrect_columns/few_columns.csv 2378 1 "b" MISSING COLUMNS 1 19008 -data/csv/rejects/incorrect_columns/few_columns.csv 2762 2 "c" MISSING COLUMNS 1,2 22074 -data/csv/rejects/incorrect_columns/many_columns.csv 1096 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 8760 -data/csv/rejects/incorrect_columns/many_columns.csv 1159 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 9268 -data/csv/rejects/incorrect_columns/many_columns.csv 1206 5 NULL TOO MANY COLUMNS 1,2,3,4,5 9648 -data/csv/rejects/incorrect_columns/many_columns.csv 2769 5 NULL TOO MANY COLUMNS 1,2,3,4,5 22154 -data/csv/rejects/incorrect_columns/mix_columns.csv 1604 1 "b" MISSING COLUMNS 1 12824 -data/csv/rejects/incorrect_columns/mix_columns.csv 1671 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 13354 -data/csv/rejects/incorrect_columns/mix_columns.csv 2751 2 "c" MISSING COLUMNS 1,2 21998 -data/csv/rejects/incorrect_columns/mix_columns.csv 2768 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 22130 -data/csv/rejects/incorrect_columns/small_mix.csv 3 5 NULL TOO MANY COLUMNS 1,2,3,4,5 16 -data/csv/rejects/incorrect_columns/small_mix.csv 4 3 "d" MISSING COLUMNS 1,2,3 26 +35 0 1814 14505 14510 3 d MISSING COLUMNS 1,2,3 Expected Number of Columns: 4 Found: 3 +35 0 1823 14575 14576 1 b MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 1 +35 0 2378 19009 19010 1 b MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 1 +35 0 2762 22075 22078 2 c MISSING COLUMNS 1,2 Expected Number of Columns: 4 Found: 2 +35 1 1096 8761 8768 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5 +35 1 1096 8761 8770 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6 +35 1 1159 9269 9276 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5 +35 1 1159 9269 9278 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6 +35 1 1206 9649 9656 5 NULL TOO MANY COLUMNS 1,2,3,4,5 Expected Number of Columns: 4 Found: 5 +35 1 2769 22155 22162 5 NULL TOO MANY COLUMNS 1,2,3,4,5 Expected Number of Columns: 4 Found: 5 +35 2 1604 12825 12826 1 b MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 1 +35 2 1671 13355 13362 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5 +35 2 1671 13355 13364 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6 +35 2 2751 21999 22002 2 c MISSING COLUMNS 1,2 Expected Number of Columns: 4 Found: 2 +35 2 2768 22131 22138 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5 +35 2 2768 22131 22140 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6 +35 3 3 17 24 5 NULL TOO MANY COLUMNS 1,2,3,4,5 Expected Number of Columns: 4 Found: 5 +35 3 4 27 32 3 d MISSING COLUMNS 1,2,3 Expected Number of Columns: 4 Found: 3 From 50d658c8cde7b5208a1b56feedd0ff3d9a4759dd Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 3 Apr 2024 16:43:43 +0200 Subject: [PATCH 144/603] Don't really care about copy from yet --- .../copy/csv/rejects/csv_rejects_auto.test | 123 +++--------------- 1 file changed, 15 insertions(+), 108 deletions(-) diff --git a/test/sql/copy/csv/rejects/csv_rejects_auto.test b/test/sql/copy/csv/rejects/csv_rejects_auto.test index bfa8073a6567..e673e9917287 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_auto.test +++ b/test/sql/copy/csv/rejects/csv_rejects_auto.test @@ -11,89 +11,44 @@ query IIIII SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto( 'test/sql/copy/csv/data/error/mismatch/big_bad*.csv', sample_size=1, - rejects_table='csv_rejects_table', - ignore_errors=true); + store_rejects=true); ---- BIGINT VARCHAR 11044 11044 2 -query IIIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position -FROM csv_rejects_table; +query IIIIIIIIII rowsort +FROM reject_errors order by all; ---- -test/sql/copy/csv/data/error/mismatch/big_bad.csv 2176 1 "column0" CAST B, A 10875 -test/sql/copy/csv/data/error/mismatch/big_bad.csv 4176 1 "column0" CAST C, A 20875 -test/sql/copy/csv/data/error/mismatch/big_bad2.csv 3680 1 "column0" CAST B, A 18395 -test/sql/copy/csv/data/error/mismatch/big_bad2.csv 5680 1 "column0" CAST C, A 28395 +3 0 2176 10876 10876 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +3 0 4176 20876 20876 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' +3 1 3680 18396 18396 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +3 1 5680 28396 28396 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' -query I -SELECT error_message -FROM csv_rejects_table where line=2176 and column_idx=1; ----- -:.*Could not convert string "B" to 'BIGINT'.* - -query I -SELECT error_message -FROM csv_rejects_table where line=4176 and column_idx=1; ----- -:.*Could not convert string "C" to 'BIGINT'.* - -query I -SELECT error_message -FROM csv_rejects_table where line=3680 and column_idx=1; ----- -:.*Could not convert string "B" to 'BIGINT'.* - -query I -SELECT error_message -FROM csv_rejects_table where line=5680 and column_idx=1; ----- -:.*Could not convert string "C" to 'BIGINT'.* +statement ok +DROP TABLE reject_errors; statement ok -DROP TABLE csv_rejects_table; +DROP TABLE reject_scans; # Test with lots of errors query I SELECT SUM(num) FROM read_csv_auto( 'test/sql/copy/csv/data/error/mismatch/half1.csv', header=true, - ignore_errors=true, sample_size=1, - rejects_table='csv_rejects_table') ----- -2464 - -query I -SELECT COUNT(*) FROM csv_rejects_table; ----- -1024 - -statement ok -DROP TABLE csv_rejects_table; - -# Test same with COPY -statement ok -CREATE TABLE tbl1 (col1 BIGINT, col2 VARCHAR); - -statement ok -COPY tbl1 FROM 'test/sql/copy/csv/data/error/mismatch/half1.csv' -WITH (HEADER, IGNORE_ERRORS TRUE, SAMPLE_SIZE 1000, REJECTS_TABLE 'csv_rejects_table'); - -query I -SELECT SUM(col1) FROM tbl1; + store_rejects=true) ---- 2464 query I -SELECT COUNT(*) FROM csv_rejects_table; +SELECT COUNT(*) FROM reject_errors; ---- 1024 statement ok -DROP TABLE csv_rejects_table; +DROP TABLE reject_errors; statement ok -DROP TABLE tbl1; +DROP TABLE reject_scans; # Test with more errors than STANDARD_VECTOR_SIZE query I @@ -112,52 +67,4 @@ SELECT COUNT(*) FROM csv_rejects_table; 3072 statement ok -DROP TABLE csv_rejects_table; - -statement ok -CREATE TABLE tbl1 (col1 BIGINT, col2 VARCHAR); - -statement ok -COPY tbl1 FROM 'test/sql/copy/csv/data/error/mismatch/half2.csv' -WITH (HEADER, IGNORE_ERRORS TRUE, SAMPLE_SIZE 1000, REJECTS_TABLE 'csv_rejects_table'); - -query I -SELECT SUM(col1) FROM tbl1; ----- -2542 - -query I -SELECT COUNT(*) FROM csv_rejects_table; ----- -3072 - -statement ok -DROP TABLE csv_rejects_table; - -statement ok -DROP TABLE tbl1; - -# Test with more errors than STANDARD_VECTOR_SIZE and limit -statement ok -CREATE TABLE tbl1 (col1 BIGINT, col2 VARCHAR); - -statement ok -COPY tbl1 FROM 'test/sql/copy/csv/data/error/mismatch/half2.csv' -WITH (HEADER, IGNORE_ERRORS TRUE, SAMPLE_SIZE 1000, REJECTS_TABLE 'csv_rejects_table', REJECTS_LIMIT 1337); - -query I -SELECT SUM(col1) FROM tbl1; ----- -2542 - -query I -SELECT COUNT(*) FROM csv_rejects_table; ----- -1337 - -statement ok -DROP TABLE csv_rejects_table; - -statement ok -DROP TABLE tbl1; - +DROP TABLE csv_rejects_table; \ No newline at end of file From 8b97a738ab147e76ea5d927f583b46d684019ddc Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 3 Apr 2024 16:55:51 +0200 Subject: [PATCH 145/603] More test fixes --- .../csv/rejects/csv_rejects_flush_cast.test | 1 - .../csv/rejects/csv_rejects_maximum_line.test | 71 ++++++++++--------- 2 files changed, 37 insertions(+), 35 deletions(-) diff --git a/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test b/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test index 7af3c29f373d..e6459aa5cd77 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test +++ b/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test @@ -18,7 +18,6 @@ SELECT typeof(first(a)), typeof(first(b)), COUNT(*) FROM read_csv( DATE VARCHAR 2811 query IIIIIIIIII -SELECT * FROM reject_errors order by all; ---- 3 0 439 6997 NULL 1 a CAST B, bla Error when converting column "a". Could not parse string "B" according to format specifier "%d-%m-%Y" diff --git a/test/sql/copy/csv/rejects/csv_rejects_maximum_line.test b/test/sql/copy/csv/rejects/csv_rejects_maximum_line.test index f6214aab0906..1095a90d70f8 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_maximum_line.test +++ b/test/sql/copy/csv/rejects/csv_rejects_maximum_line.test @@ -11,17 +11,18 @@ statement ok SELECT * FROM read_csv( 'data/csv/rejects/maximum_line/max_10.csv', columns = {'a': 'VARCHAR', 'b': 'INTEGER'}, - rejects_table='csv_rejects_table', - ignore_errors=true, auto_detect=false, header = 1, max_line_size=10); + store_rejects=true, auto_detect=false, header = 1, max_line_size=10); -query IIIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position -FROM csv_rejects_table; +query IIIIIIIIII +FROM reject_errors order by all; ---- -data/csv/rejects/maximum_line/max_10.csv 5 1 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 22 +3 0 5 23 23 1 a LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 Maximum line size of 10 bytes exceeded. Actual Size:19 bytes. statement ok -DROP TABLE csv_rejects_table; +DROP TABLE reject_errors; + +statement ok +DROP TABLE reject_scans; # Test with buffer sizes @@ -31,17 +32,18 @@ statement ok SELECT * FROM read_csv( 'data/csv/rejects/maximum_line/max_10.csv', columns = {'a': 'VARCHAR', 'b': 'INTEGER'}, - rejects_table='csv_rejects_table', - ignore_errors=true, auto_detect=false, header = 1, max_line_size=10, buffer_size=${buffer_size}); + store_rejects = true, auto_detect=false, header = 1, max_line_size=10, buffer_size=${buffer_size}); -query IIIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position -FROM csv_rejects_table; +query IIIIIIIII +SELECT * EXCLUDE (scan_id) FROM reject_errors order by all; ---- -data/csv/rejects/maximum_line/max_10.csv 5 1 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 22 +0 5 23 23 1 a LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 Maximum line size of 10 bytes exceeded. Actual Size:19 bytes. statement ok -DROP TABLE csv_rejects_table; +DROP TABLE reject_errors; + +statement ok +DROP TABLE reject_scans; endloop @@ -50,37 +52,38 @@ statement ok SELECT * FROM read_csv( 'data/csv/rejects/maximum_line/over_vector.csv', columns = {'a': 'VARCHAR', 'b': 'INTEGER'}, - rejects_table='csv_rejects_table', - ignore_errors=true, auto_detect=false, header = 1, max_line_size=20); + store_rejects = true, auto_detect=false, header = 1, max_line_size=20); -query IIIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position -FROM csv_rejects_table; +query IIIIIIIIII +FROM reject_errors order by all; ---- -data/csv/rejects/maximum_line/over_vector.csv 2282 1 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 13684 -data/csv/rejects/maximum_line/over_vector.csv 2591 1 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 15557 -data/csv/rejects/maximum_line/over_vector.csv 2923 1 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,3 17568 +27 0 2282 13685 13685 1 a LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 Maximum line size of 20 bytes exceeded. Actual Size:25 bytes. +27 0 2591 15558 15558 1 a LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 Maximum line size of 20 bytes exceeded. Actual Size:25 bytes. +27 0 2923 17569 17569 1 a LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,3 Maximum line size of 20 bytes exceeded. Actual Size:25 bytes. statement ok -DROP TABLE csv_rejects_table; +DROP TABLE reject_errors; -# Read Multiple Files +statement ok +DROP TABLE reject_scans; +# Read Multiple Files statement ok SELECT * FROM read_csv( 'data/csv/rejects/maximum_line/*.csv', columns = {'a': 'VARCHAR', 'b': 'INTEGER'}, - rejects_table='csv_rejects_table', - ignore_errors=true, auto_detect=false, header = 1, max_line_size=10); + store_rejects = true, auto_detect=false, header = 1, max_line_size=10); -query IIIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position -FROM csv_rejects_table; +query IIIIIIIIII +FROM reject_errors order by all; ---- -data/csv/rejects/maximum_line/max_10.csv 5 1 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 22 -data/csv/rejects/maximum_line/over_vector.csv 2282 1 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 13684 -data/csv/rejects/maximum_line/over_vector.csv 2591 1 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 15557 -data/csv/rejects/maximum_line/over_vector.csv 2923 1 "a" LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,3 17568 +31 0 5 23 23 1 a LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 Maximum line size of 10 bytes exceeded. Actual Size:19 bytes. +31 1 2282 13685 13685 1 a LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 Maximum line size of 10 bytes exceeded. Actual Size:25 bytes. +31 1 2591 15558 15558 1 a LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 Maximum line size of 10 bytes exceeded. Actual Size:25 bytes. +31 1 2923 17569 17569 1 a LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,3 Maximum line size of 10 bytes exceeded. Actual Size:25 bytes. + +statement ok +DROP TABLE reject_errors; statement ok -DROP TABLE csv_rejects_table; \ No newline at end of file +DROP TABLE reject_scans; \ No newline at end of file From 483033e0b0b4a73843f6ec98df6af4e4c5452572 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Thu, 4 Apr 2024 10:09:17 +0200 Subject: [PATCH 146/603] more casts, schema reflection should really use unsiged types everywhere --- src/function/table/copy_csv.cpp | 6 ++++-- src/function/table/read_file.cpp | 3 ++- src/function/table/repeat.cpp | 2 +- src/function/table/repeat_row.cpp | 2 +- src/function/table/system/duckdb_columns.cpp | 6 +++--- .../table/system/duckdb_constraints.cpp | 15 ++++++++------- src/function/table/system/duckdb_databases.cpp | 2 +- .../table/system/duckdb_dependencies.cpp | 4 ++-- src/function/table/system/duckdb_functions.cpp | 4 ++-- src/function/table/system/duckdb_indexes.cpp | 8 ++++---- src/function/table/system/duckdb_memory.cpp | 4 ++-- src/function/table/system/duckdb_schemas.cpp | 4 ++-- src/function/table/system/duckdb_sequences.cpp | 6 +++--- src/function/table/system/duckdb_tables.cpp | 17 +++++++++-------- .../table/system/duckdb_temporary_files.cpp | 2 +- src/function/table/system/duckdb_types.cpp | 12 +++++++----- src/function/table/system/duckdb_views.cpp | 8 ++++---- .../table/system/pragma_database_size.cpp | 8 ++++---- .../table/system/pragma_metadata_info.cpp | 6 +++--- .../table/system/pragma_storage_info.cpp | 12 ++++++------ src/function/table/unnest.cpp | 2 +- src/include/duckdb/execution/index/art/node.hpp | 2 +- 22 files changed, 71 insertions(+), 64 deletions(-) diff --git a/src/function/table/copy_csv.cpp b/src/function/table/copy_csv.cpp index 1fa2e46e7694..12ea326f1a31 100644 --- a/src/function/table/copy_csv.cpp +++ b/src/function/table/copy_csv.cpp @@ -111,8 +111,10 @@ static unique_ptr WriteCSVBind(ClientContext &context, CopyFunctio memset(bind_data->requires_quotes.get(), 0, sizeof(bool) * 256); bind_data->requires_quotes['\n'] = true; bind_data->requires_quotes['\r'] = true; - bind_data->requires_quotes[bind_data->options.dialect_options.state_machine_options.delimiter.GetValue()] = true; - bind_data->requires_quotes[bind_data->options.dialect_options.state_machine_options.quote.GetValue()] = true; + bind_data->requires_quotes[NumericCast( + bind_data->options.dialect_options.state_machine_options.delimiter.GetValue())] = true; + bind_data->requires_quotes[NumericCast( + bind_data->options.dialect_options.state_machine_options.quote.GetValue())] = true; if (!bind_data->options.write_newline.empty()) { bind_data->newline = TransformNewLine(bind_data->options.write_newline); diff --git a/src/function/table/read_file.cpp b/src/function/table/read_file.cpp index 7655d34c3ea8..c6aa707d026e 100644 --- a/src/function/table/read_file.cpp +++ b/src/function/table/read_file.cpp @@ -160,7 +160,8 @@ static void ReadFileExecute(ClientContext &context, TableFunctionInput &input, D } break; case ReadFileBindData::FILE_SIZE_COLUMN: { auto &file_size_vector = output.data[col_idx]; - FlatVector::GetData(file_size_vector)[out_idx] = file_handle->GetFileSize(); + FlatVector::GetData(file_size_vector)[out_idx] = + NumericCast(file_handle->GetFileSize()); } break; case ReadFileBindData::FILE_LAST_MODIFIED_COLUMN: { auto &last_modified_vector = output.data[col_idx]; diff --git a/src/function/table/repeat.cpp b/src/function/table/repeat.cpp index b62cbe15c87e..25fa43dc0675 100644 --- a/src/function/table/repeat.cpp +++ b/src/function/table/repeat.cpp @@ -26,7 +26,7 @@ static unique_ptr RepeatBind(ClientContext &context, TableFunction if (inputs[1].IsNull()) { throw BinderException("Repeat second parameter cannot be NULL"); } - return make_uniq(inputs[0], inputs[1].GetValue()); + return make_uniq(inputs[0], NumericCast(inputs[1].GetValue())); } static unique_ptr RepeatInit(ClientContext &context, TableFunctionInitInput &input) { diff --git a/src/function/table/repeat_row.cpp b/src/function/table/repeat_row.cpp index a81d8720691f..f68bf7cae439 100644 --- a/src/function/table/repeat_row.cpp +++ b/src/function/table/repeat_row.cpp @@ -32,7 +32,7 @@ static unique_ptr RepeatRowBind(ClientContext &context, TableFunct if (inputs.empty()) { throw BinderException("repeat_rows requires at least one column to be specified"); } - return make_uniq(inputs, entry->second.GetValue()); + return make_uniq(inputs, NumericCast(entry->second.GetValue())); } static unique_ptr RepeatRowInit(ClientContext &context, TableFunctionInitInput &input) { diff --git a/src/function/table/system/duckdb_columns.cpp b/src/function/table/system/duckdb_columns.cpp index 2d7790e1109c..1632a9e5c6c8 100644 --- a/src/function/table/system/duckdb_columns.cpp +++ b/src/function/table/system/duckdb_columns.cpp @@ -209,15 +209,15 @@ void ColumnHelper::WriteColumns(idx_t start_index, idx_t start_col, idx_t end_co // database_name, VARCHAR output.SetValue(col++, index, entry.catalog.GetName()); // database_oid, BIGINT - output.SetValue(col++, index, Value::BIGINT(entry.catalog.GetOid())); + output.SetValue(col++, index, Value::BIGINT(NumericCast(entry.catalog.GetOid()))); // schema_name, VARCHAR output.SetValue(col++, index, entry.schema.name); // schema_oid, BIGINT - output.SetValue(col++, index, Value::BIGINT(entry.schema.oid)); + output.SetValue(col++, index, Value::BIGINT(NumericCast(entry.schema.oid))); // table_name, VARCHAR output.SetValue(col++, index, entry.name); // table_oid, BIGINT - output.SetValue(col++, index, Value::BIGINT(entry.oid)); + output.SetValue(col++, index, Value::BIGINT(NumericCast(entry.oid))); // column_name, VARCHAR output.SetValue(col++, index, Value(ColumnName(i))); // column_index, INTEGER diff --git a/src/function/table/system/duckdb_constraints.cpp b/src/function/table/system/duckdb_constraints.cpp index 64aff39469ed..467aa2ae7605 100644 --- a/src/function/table/system/duckdb_constraints.cpp +++ b/src/function/table/system/duckdb_constraints.cpp @@ -180,15 +180,15 @@ void DuckDBConstraintsFunction(ClientContext &context, TableFunctionInput &data_ // database_name, LogicalType::VARCHAR output.SetValue(col++, count, Value(table.schema.catalog.GetName())); // database_oid, LogicalType::BIGINT - output.SetValue(col++, count, Value::BIGINT(table.schema.catalog.GetOid())); + output.SetValue(col++, count, Value::BIGINT(NumericCast(table.schema.catalog.GetOid()))); // schema_name, LogicalType::VARCHAR output.SetValue(col++, count, Value(table.schema.name)); // schema_oid, LogicalType::BIGINT - output.SetValue(col++, count, Value::BIGINT(table.schema.oid)); + output.SetValue(col++, count, Value::BIGINT(NumericCast(table.schema.oid))); // table_name, LogicalType::VARCHAR output.SetValue(col++, count, Value(table.name)); // table_oid, LogicalType::BIGINT - output.SetValue(col++, count, Value::BIGINT(table.oid)); + output.SetValue(col++, count, Value::BIGINT(NumericCast(table.oid))); // constraint_index, BIGINT UniqueKeyInfo uk_info; @@ -224,15 +224,16 @@ void DuckDBConstraintsFunction(ClientContext &context, TableFunctionInput &data_ } if (uk_info.columns.empty()) { - output.SetValue(col++, count, Value::BIGINT(data.unique_constraint_offset++)); + output.SetValue(col++, count, Value::BIGINT(NumericCast(data.unique_constraint_offset++))); } else { auto known_unique_constraint_offset = data.known_fk_unique_constraint_offsets.find(uk_info); if (known_unique_constraint_offset == data.known_fk_unique_constraint_offsets.end()) { data.known_fk_unique_constraint_offsets.insert(make_pair(uk_info, data.unique_constraint_offset)); - output.SetValue(col++, count, Value::BIGINT(data.unique_constraint_offset)); + output.SetValue(col++, count, Value::BIGINT(NumericCast(data.unique_constraint_offset))); data.unique_constraint_offset++; } else { - output.SetValue(col++, count, Value::BIGINT(known_unique_constraint_offset->second)); + output.SetValue(col++, count, + Value::BIGINT(NumericCast(known_unique_constraint_offset->second))); } } output.SetValue(col++, count, Value(constraint_type)); @@ -286,7 +287,7 @@ void DuckDBConstraintsFunction(ClientContext &context, TableFunctionInput &data_ vector index_list; vector column_name_list; for (auto column_index : column_index_list) { - index_list.push_back(Value::BIGINT(column_index.index)); + index_list.push_back(Value::BIGINT(NumericCast(column_index.index))); column_name_list.emplace_back(table.GetColumn(column_index).Name()); } diff --git a/src/function/table/system/duckdb_databases.cpp b/src/function/table/system/duckdb_databases.cpp index f0a1fc99b5b8..0aee93897a08 100644 --- a/src/function/table/system/duckdb_databases.cpp +++ b/src/function/table/system/duckdb_databases.cpp @@ -66,7 +66,7 @@ void DuckDBDatabasesFunction(ClientContext &context, TableFunctionInput &data_p, // database_name, VARCHAR output.SetValue(col++, count, attached.GetName()); // database_oid, BIGINT - output.SetValue(col++, count, Value::BIGINT(attached.oid)); + output.SetValue(col++, count, Value::BIGINT(NumericCast(attached.oid))); bool is_internal = attached.IsSystem() || attached.IsTemporary(); bool is_readonly = attached.IsReadOnly(); // path, VARCHAR diff --git a/src/function/table/system/duckdb_dependencies.cpp b/src/function/table/system/duckdb_dependencies.cpp index c262d7d47ae2..6f7370194ed5 100644 --- a/src/function/table/system/duckdb_dependencies.cpp +++ b/src/function/table/system/duckdb_dependencies.cpp @@ -85,13 +85,13 @@ void DuckDBDependenciesFunction(ClientContext &context, TableFunctionInput &data // classid, LogicalType::BIGINT output.SetValue(0, count, Value::BIGINT(0)); // objid, LogicalType::BIGINT - output.SetValue(1, count, Value::BIGINT(entry.object.oid)); + output.SetValue(1, count, Value::BIGINT(NumericCast(entry.object.oid))); // objsubid, LogicalType::INTEGER output.SetValue(2, count, Value::INTEGER(0)); // refclassid, LogicalType::BIGINT output.SetValue(3, count, Value::BIGINT(0)); // refobjid, LogicalType::BIGINT - output.SetValue(4, count, Value::BIGINT(entry.dependent.oid)); + output.SetValue(4, count, Value::BIGINT(NumericCast(entry.dependent.oid))); // refobjsubid, LogicalType::INTEGER output.SetValue(5, count, Value::INTEGER(0)); // deptype, LogicalType::VARCHAR diff --git a/src/function/table/system/duckdb_functions.cpp b/src/function/table/system/duckdb_functions.cpp index 107da17e2fde..4dd287fd3ff7 100644 --- a/src/function/table/system/duckdb_functions.cpp +++ b/src/function/table/system/duckdb_functions.cpp @@ -453,7 +453,7 @@ bool ExtractFunctionData(FunctionEntry &entry, idx_t function_idx, DataChunk &ou output.SetValue(col++, output_offset, Value(function.schema.catalog.GetName())); // database_oid, BIGINT - output.SetValue(col++, output_offset, Value::BIGINT(function.schema.catalog.GetOid())); + output.SetValue(col++, output_offset, Value::BIGINT(NumericCast(function.schema.catalog.GetOid()))); // schema_name, LogicalType::VARCHAR output.SetValue(col++, output_offset, Value(function.schema.name)); @@ -497,7 +497,7 @@ bool ExtractFunctionData(FunctionEntry &entry, idx_t function_idx, DataChunk &ou output.SetValue(col++, output_offset, Value::BOOLEAN(function.internal)); // function_oid, LogicalType::BIGINT - output.SetValue(col++, output_offset, Value::BIGINT(function.oid)); + output.SetValue(col++, output_offset, Value::BIGINT(NumericCast(function.oid))); // example, LogicalType::VARCHAR output.SetValue(col++, output_offset, entry.example.empty() ? Value() : entry.example); diff --git a/src/function/table/system/duckdb_indexes.cpp b/src/function/table/system/duckdb_indexes.cpp index 7cbb6e904544..0b8e78853b4c 100644 --- a/src/function/table/system/duckdb_indexes.cpp +++ b/src/function/table/system/duckdb_indexes.cpp @@ -91,22 +91,22 @@ void DuckDBIndexesFunction(ClientContext &context, TableFunctionInput &data_p, D // database_name, VARCHAR output.SetValue(col++, count, index.catalog.GetName()); // database_oid, BIGINT - output.SetValue(col++, count, Value::BIGINT(index.catalog.GetOid())); + output.SetValue(col++, count, Value::BIGINT(NumericCast(index.catalog.GetOid()))); // schema_name, VARCHAR output.SetValue(col++, count, Value(index.schema.name)); // schema_oid, BIGINT - output.SetValue(col++, count, Value::BIGINT(index.schema.oid)); + output.SetValue(col++, count, Value::BIGINT(NumericCast(index.schema.oid))); // index_name, VARCHAR output.SetValue(col++, count, Value(index.name)); // index_oid, BIGINT - output.SetValue(col++, count, Value::BIGINT(index.oid)); + output.SetValue(col++, count, Value::BIGINT(NumericCast(index.oid))); // find the table in the catalog auto &table_entry = index.schema.catalog.GetEntry(context, index.GetSchemaName(), index.GetTableName()); // table_name, VARCHAR output.SetValue(col++, count, Value(table_entry.name)); // table_oid, BIGINT - output.SetValue(col++, count, Value::BIGINT(table_entry.oid)); + output.SetValue(col++, count, Value::BIGINT(NumericCast(table_entry.oid))); // comment, VARCHAR output.SetValue(col++, count, Value(index.comment)); // is_unique, BOOLEAN diff --git a/src/function/table/system/duckdb_memory.cpp b/src/function/table/system/duckdb_memory.cpp index 9117c91a9a2d..e67fa51062b7 100644 --- a/src/function/table/system/duckdb_memory.cpp +++ b/src/function/table/system/duckdb_memory.cpp @@ -48,9 +48,9 @@ void DuckDBMemoryFunction(ClientContext &context, TableFunctionInput &data_p, Da // tag, VARCHAR output.SetValue(col++, count, EnumUtil::ToString(entry.tag)); // memory_usage_bytes, BIGINT - output.SetValue(col++, count, Value::BIGINT(entry.size)); + output.SetValue(col++, count, Value::BIGINT(NumericCast(entry.size))); // temporary_storage_bytes, BIGINT - output.SetValue(col++, count, Value::BIGINT(entry.evicted_data)); + output.SetValue(col++, count, Value::BIGINT(NumericCast(entry.evicted_data))); count++; } output.SetCardinality(count); diff --git a/src/function/table/system/duckdb_schemas.cpp b/src/function/table/system/duckdb_schemas.cpp index 65d4a48ca1d4..c8d9828e3603 100644 --- a/src/function/table/system/duckdb_schemas.cpp +++ b/src/function/table/system/duckdb_schemas.cpp @@ -66,11 +66,11 @@ void DuckDBSchemasFunction(ClientContext &context, TableFunctionInput &data_p, D // return values: idx_t col = 0; // "oid", PhysicalType::BIGINT - output.SetValue(col++, count, Value::BIGINT(entry.oid)); + output.SetValue(col++, count, Value::BIGINT(NumericCast(entry.oid))); // database_name, VARCHAR output.SetValue(col++, count, entry.catalog.GetName()); // database_oid, BIGINT - output.SetValue(col++, count, Value::BIGINT(entry.catalog.GetOid())); + output.SetValue(col++, count, Value::BIGINT(NumericCast(entry.catalog.GetOid()))); // "schema_name", PhysicalType::VARCHAR output.SetValue(col++, count, Value(entry.name)); // "comment", PhysicalType::VARCHAR diff --git a/src/function/table/system/duckdb_sequences.cpp b/src/function/table/system/duckdb_sequences.cpp index 7a38e1c65e96..44128380b37f 100644 --- a/src/function/table/system/duckdb_sequences.cpp +++ b/src/function/table/system/duckdb_sequences.cpp @@ -98,15 +98,15 @@ void DuckDBSequencesFunction(ClientContext &context, TableFunctionInput &data_p, // database_name, VARCHAR output.SetValue(col++, count, seq.catalog.GetName()); // database_oid, BIGINT - output.SetValue(col++, count, Value::BIGINT(seq.catalog.GetOid())); + output.SetValue(col++, count, Value::BIGINT(NumericCast(seq.catalog.GetOid()))); // schema_name, VARCHAR output.SetValue(col++, count, Value(seq.schema.name)); // schema_oid, BIGINT - output.SetValue(col++, count, Value::BIGINT(seq.schema.oid)); + output.SetValue(col++, count, Value::BIGINT(NumericCast(seq.schema.oid))); // sequence_name, VARCHAR output.SetValue(col++, count, Value(seq.name)); // sequence_oid, BIGINT - output.SetValue(col++, count, Value::BIGINT(seq.oid)); + output.SetValue(col++, count, Value::BIGINT(NumericCast(seq.oid))); // comment, VARCHAR output.SetValue(col++, count, Value(seq.comment)); // temporary, BOOLEAN diff --git a/src/function/table/system/duckdb_tables.cpp b/src/function/table/system/duckdb_tables.cpp index 2503cfbf80c9..a6f7d523aa3f 100644 --- a/src/function/table/system/duckdb_tables.cpp +++ b/src/function/table/system/duckdb_tables.cpp @@ -127,15 +127,15 @@ void DuckDBTablesFunction(ClientContext &context, TableFunctionInput &data_p, Da // database_name, VARCHAR output.SetValue(col++, count, table.catalog.GetName()); // database_oid, BIGINT - output.SetValue(col++, count, Value::BIGINT(table.catalog.GetOid())); + output.SetValue(col++, count, Value::BIGINT(NumericCast(table.catalog.GetOid()))); // schema_name, LogicalType::VARCHAR output.SetValue(col++, count, Value(table.schema.name)); // schema_oid, LogicalType::BIGINT - output.SetValue(col++, count, Value::BIGINT(table.schema.oid)); + output.SetValue(col++, count, Value::BIGINT(NumericCast(table.schema.oid))); // table_name, LogicalType::VARCHAR output.SetValue(col++, count, Value(table.name)); // table_oid, LogicalType::BIGINT - output.SetValue(col++, count, Value::BIGINT(table.oid)); + output.SetValue(col++, count, Value::BIGINT(NumericCast(table.oid))); // comment, LogicalType::VARCHAR output.SetValue(col++, count, Value(table.comment)); // internal, LogicalType::BOOLEAN @@ -145,15 +145,16 @@ void DuckDBTablesFunction(ClientContext &context, TableFunctionInput &data_p, Da // has_primary_key, LogicalType::BOOLEAN output.SetValue(col++, count, Value::BOOLEAN(TableHasPrimaryKey(table))); // estimated_size, LogicalType::BIGINT - Value card_val = - storage_info.cardinality == DConstants::INVALID_INDEX ? Value() : Value::BIGINT(storage_info.cardinality); + Value card_val = storage_info.cardinality == DConstants::INVALID_INDEX + ? Value() + : Value::BIGINT(NumericCast(storage_info.cardinality)); output.SetValue(col++, count, card_val); // column_count, LogicalType::BIGINT - output.SetValue(col++, count, Value::BIGINT(table.GetColumns().LogicalColumnCount())); + output.SetValue(col++, count, Value::BIGINT(NumericCast(table.GetColumns().LogicalColumnCount()))); // index_count, LogicalType::BIGINT - output.SetValue(col++, count, Value::BIGINT(storage_info.index_info.size())); + output.SetValue(col++, count, Value::BIGINT(NumericCast(storage_info.index_info.size()))); // check_constraint_count, LogicalType::BIGINT - output.SetValue(col++, count, Value::BIGINT(CheckConstraintCount(table))); + output.SetValue(col++, count, Value::BIGINT(NumericCast(CheckConstraintCount(table)))); // sql, LogicalType::VARCHAR output.SetValue(col++, count, Value(table.ToSQL())); diff --git a/src/function/table/system/duckdb_temporary_files.cpp b/src/function/table/system/duckdb_temporary_files.cpp index d4f043aff746..f1886639441c 100644 --- a/src/function/table/system/duckdb_temporary_files.cpp +++ b/src/function/table/system/duckdb_temporary_files.cpp @@ -45,7 +45,7 @@ void DuckDBTemporaryFilesFunction(ClientContext &context, TableFunctionInput &da // database_name, VARCHAR output.SetValue(col++, count, entry.path); // database_oid, BIGINT - output.SetValue(col++, count, Value::BIGINT(entry.size)); + output.SetValue(col++, count, Value::BIGINT(NumericCast(entry.size))); count++; } output.SetCardinality(count); diff --git a/src/function/table/system/duckdb_types.cpp b/src/function/table/system/duckdb_types.cpp index 2a7fc1fd1d61..1b7ca0c97488 100644 --- a/src/function/table/system/duckdb_types.cpp +++ b/src/function/table/system/duckdb_types.cpp @@ -89,17 +89,17 @@ void DuckDBTypesFunction(ClientContext &context, TableFunctionInput &data_p, Dat // database_name, VARCHAR output.SetValue(col++, count, type_entry.catalog.GetName()); // database_oid, BIGINT - output.SetValue(col++, count, Value::BIGINT(type_entry.catalog.GetOid())); + output.SetValue(col++, count, Value::BIGINT(NumericCast(type_entry.catalog.GetOid()))); // schema_name, LogicalType::VARCHAR output.SetValue(col++, count, Value(type_entry.schema.name)); // schema_oid, LogicalType::BIGINT - output.SetValue(col++, count, Value::BIGINT(type_entry.schema.oid)); + output.SetValue(col++, count, Value::BIGINT(NumericCast(type_entry.schema.oid))); // type_oid, BIGINT int64_t oid; if (type_entry.internal) { - oid = int64_t(type.id()); + oid = NumericCast(type.id()); } else { - oid = type_entry.oid; + oid = NumericCast(type_entry.oid); } Value oid_val; if (data.oids.find(oid) == data.oids.end()) { @@ -114,7 +114,9 @@ void DuckDBTypesFunction(ClientContext &context, TableFunctionInput &data_p, Dat // type_size, BIGINT auto internal_type = type.InternalType(); output.SetValue(col++, count, - internal_type == PhysicalType::INVALID ? Value() : Value::BIGINT(GetTypeIdSize(internal_type))); + internal_type == PhysicalType::INVALID + ? Value() + : Value::BIGINT(NumericCast(GetTypeIdSize(internal_type)))); // logical_type, VARCHAR output.SetValue(col++, count, Value(EnumUtil::ToString(type.id()))); // type_category, VARCHAR diff --git a/src/function/table/system/duckdb_views.cpp b/src/function/table/system/duckdb_views.cpp index bd17a7ef6a50..e2eb26603468 100644 --- a/src/function/table/system/duckdb_views.cpp +++ b/src/function/table/system/duckdb_views.cpp @@ -89,15 +89,15 @@ void DuckDBViewsFunction(ClientContext &context, TableFunctionInput &data_p, Dat // database_name, VARCHAR output.SetValue(col++, count, view.catalog.GetName()); // database_oid, BIGINT - output.SetValue(col++, count, Value::BIGINT(view.catalog.GetOid())); + output.SetValue(col++, count, Value::BIGINT(NumericCast(view.catalog.GetOid()))); // schema_name, LogicalType::VARCHAR output.SetValue(col++, count, Value(view.schema.name)); // schema_oid, LogicalType::BIGINT - output.SetValue(col++, count, Value::BIGINT(view.schema.oid)); + output.SetValue(col++, count, Value::BIGINT(NumericCast(view.schema.oid))); // view_name, LogicalType::VARCHAR output.SetValue(col++, count, Value(view.name)); // view_oid, LogicalType::BIGINT - output.SetValue(col++, count, Value::BIGINT(view.oid)); + output.SetValue(col++, count, Value::BIGINT(NumericCast(view.oid))); // comment, LogicalType::VARCHARs output.SetValue(col++, count, Value(view.comment)); // internal, LogicalType::BOOLEAN @@ -105,7 +105,7 @@ void DuckDBViewsFunction(ClientContext &context, TableFunctionInput &data_p, Dat // temporary, LogicalType::BOOLEAN output.SetValue(col++, count, Value::BOOLEAN(view.temporary)); // column_count, LogicalType::BIGINT - output.SetValue(col++, count, Value::BIGINT(view.types.size())); + output.SetValue(col++, count, Value::BIGINT(NumericCast(view.types.size()))); // sql, LogicalType::VARCHAR output.SetValue(col++, count, Value(view.ToSQL())); diff --git a/src/function/table/system/pragma_database_size.cpp b/src/function/table/system/pragma_database_size.cpp index c10eae0c9b60..ba8f2020e3d5 100644 --- a/src/function/table/system/pragma_database_size.cpp +++ b/src/function/table/system/pragma_database_size.cpp @@ -76,10 +76,10 @@ void PragmaDatabaseSizeFunction(ClientContext &context, TableFunctionInput &data idx_t col = 0; output.data[col++].SetValue(row, Value(db.GetName())); output.data[col++].SetValue(row, Value(StringUtil::BytesToHumanReadableString(ds.bytes))); - output.data[col++].SetValue(row, Value::BIGINT(ds.block_size)); - output.data[col++].SetValue(row, Value::BIGINT(ds.total_blocks)); - output.data[col++].SetValue(row, Value::BIGINT(ds.used_blocks)); - output.data[col++].SetValue(row, Value::BIGINT(ds.free_blocks)); + output.data[col++].SetValue(row, Value::BIGINT(NumericCast(ds.block_size))); + output.data[col++].SetValue(row, Value::BIGINT(NumericCast(ds.total_blocks))); + output.data[col++].SetValue(row, Value::BIGINT(NumericCast(ds.used_blocks))); + output.data[col++].SetValue(row, Value::BIGINT(NumericCast(ds.free_blocks))); output.data[col++].SetValue( row, ds.wal_size == idx_t(-1) ? Value() : Value(StringUtil::BytesToHumanReadableString(ds.wal_size))); output.data[col++].SetValue(row, data.memory_usage); diff --git a/src/function/table/system/pragma_metadata_info.cpp b/src/function/table/system/pragma_metadata_info.cpp index 92c180308db6..741a1d6c93c7 100644 --- a/src/function/table/system/pragma_metadata_info.cpp +++ b/src/function/table/system/pragma_metadata_info.cpp @@ -57,13 +57,13 @@ static void PragmaMetadataInfoFunction(ClientContext &context, TableFunctionInpu // block_id output.SetValue(col_idx++, count, Value::BIGINT(entry.block_id)); // total_blocks - output.SetValue(col_idx++, count, Value::BIGINT(entry.total_blocks)); + output.SetValue(col_idx++, count, Value::BIGINT(NumericCast(entry.total_blocks))); // free_blocks - output.SetValue(col_idx++, count, Value::BIGINT(entry.free_list.size())); + output.SetValue(col_idx++, count, Value::BIGINT(NumericCast(entry.free_list.size()))); // free_list vector list_values; for (auto &free_id : entry.free_list) { - list_values.push_back(Value::BIGINT(free_id)); + list_values.push_back(Value::BIGINT(NumericCast(free_id))); } output.SetValue(col_idx++, count, Value::LIST(LogicalType::BIGINT, std::move(list_values))); count++; diff --git a/src/function/table/system/pragma_storage_info.cpp b/src/function/table/system/pragma_storage_info.cpp index 90c60d15c040..0c33ebf01949 100644 --- a/src/function/table/system/pragma_storage_info.cpp +++ b/src/function/table/system/pragma_storage_info.cpp @@ -103,22 +103,22 @@ static void PragmaStorageInfoFunction(ClientContext &context, TableFunctionInput idx_t col_idx = 0; // row_group_id - output.SetValue(col_idx++, count, Value::BIGINT(entry.row_group_index)); + output.SetValue(col_idx++, count, Value::BIGINT(NumericCast(entry.row_group_index))); // column_name auto &col = columns.GetColumn(PhysicalIndex(entry.column_id)); output.SetValue(col_idx++, count, Value(col.Name())); // column_id - output.SetValue(col_idx++, count, Value::BIGINT(entry.column_id)); + output.SetValue(col_idx++, count, Value::BIGINT(NumericCast(entry.column_id))); // column_path output.SetValue(col_idx++, count, Value(entry.column_path)); // segment_id - output.SetValue(col_idx++, count, Value::BIGINT(entry.segment_idx)); + output.SetValue(col_idx++, count, Value::BIGINT(NumericCast(entry.segment_idx))); // segment_type output.SetValue(col_idx++, count, Value(entry.segment_type)); // start - output.SetValue(col_idx++, count, Value::BIGINT(entry.segment_start)); + output.SetValue(col_idx++, count, Value::BIGINT(NumericCast(entry.segment_start))); // count - output.SetValue(col_idx++, count, Value::BIGINT(entry.segment_count)); + output.SetValue(col_idx++, count, Value::BIGINT(NumericCast(entry.segment_count))); // compression output.SetValue(col_idx++, count, Value(entry.compression_type)); // stats @@ -131,7 +131,7 @@ static void PragmaStorageInfoFunction(ClientContext &context, TableFunctionInput // block_offset if (entry.persistent) { output.SetValue(col_idx++, count, Value::BIGINT(entry.block_id)); - output.SetValue(col_idx++, count, Value::BIGINT(entry.block_offset)); + output.SetValue(col_idx++, count, Value::BIGINT(NumericCast(entry.block_offset))); } else { output.SetValue(col_idx++, count, Value()); output.SetValue(col_idx++, count, Value()); diff --git a/src/function/table/unnest.cpp b/src/function/table/unnest.cpp index 15f3950893bf..b6485bc3d2aa 100644 --- a/src/function/table/unnest.cpp +++ b/src/function/table/unnest.cpp @@ -63,7 +63,7 @@ static unique_ptr UnnestLocalInit(ExecutionContext &con static unique_ptr UnnestInit(ClientContext &context, TableFunctionInitInput &input) { auto &bind_data = input.bind_data->Cast(); auto result = make_uniq(); - auto ref = make_uniq(bind_data.input_type, 0); + auto ref = make_uniq(bind_data.input_type, 0U); auto bound_unnest = make_uniq(ListType::GetChildType(bind_data.input_type)); bound_unnest->child = std::move(ref); result->select_list.push_back(std::move(bound_unnest)); diff --git a/src/include/duckdb/execution/index/art/node.hpp b/src/include/duckdb/execution/index/art/node.hpp index 2e170d410d91..34e2498dd01d 100644 --- a/src/include/duckdb/execution/index/art/node.hpp +++ b/src/include/duckdb/execution/index/art/node.hpp @@ -117,7 +117,7 @@ class Node : public IndexPointer { } //! Set the row ID (8th to 63rd bit) inline void SetRowId(const row_t row_id) { - Set((Get() & AND_METADATA) | row_id); + Set((Get() & AND_METADATA) | UnsafeNumericCast(row_id)); } //! Returns the type of the node, which is held in the metadata From f529ef99d5370478a3456018c9e6408185e87f37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Thu, 4 Apr 2024 11:02:10 +0200 Subject: [PATCH 147/603] moaaar --- src/common/adbc/nanoarrow/allocator.cpp | 4 +-- src/common/adbc/nanoarrow/metadata.cpp | 6 ++--- src/common/adbc/nanoarrow/schema.cpp | 13 +++++----- src/common/arrow/appender/bool_data.cpp | 1 + src/common/arrow/appender/struct_data.cpp | 2 +- src/common/arrow/appender/union_data.cpp | 3 ++- src/common/arrow/arrow_appender.cpp | 8 +++--- src/common/arrow/arrow_converter.cpp | 6 ++--- src/common/box_renderer.cpp | 2 +- src/common/compressed_file_system.cpp | 18 +++++++------ src/common/exception_format_value.cpp | 1 + src/common/file_system.cpp | 12 ++++----- src/common/gzip_file_system.cpp | 25 +++++++++++-------- src/common/hive_partitioning.cpp | 4 +-- src/common/local_file_system.cpp | 16 ++++++------ src/common/multi_file_reader.cpp | 2 +- src/common/pipe_file_system.cpp | 5 ++-- src/common/random_engine.cpp | 3 ++- src/common/re2_regex.cpp | 8 +++--- src/common/string_util.cpp | 5 ++-- .../common/arrow/appender/append_data.hpp | 1 + .../common/arrow/appender/enum_data.hpp | 6 ++--- .../duckdb/common/arrow/appender/map_data.hpp | 2 +- .../common/arrow/appender/varchar_data.hpp | 5 ++-- .../duckdb/storage/string_uncompressed.hpp | 4 +-- third_party/fsst/fsst.h | 2 +- 26 files changed, 92 insertions(+), 72 deletions(-) diff --git a/src/common/adbc/nanoarrow/allocator.cpp b/src/common/adbc/nanoarrow/allocator.cpp index 692cb58a6230..cea53b1880a7 100644 --- a/src/common/adbc/nanoarrow/allocator.cpp +++ b/src/common/adbc/nanoarrow/allocator.cpp @@ -23,11 +23,11 @@ namespace duckdb_nanoarrow { void *ArrowMalloc(int64_t size) { - return malloc(size); + return malloc(size_t(size)); } void *ArrowRealloc(void *ptr, int64_t size) { - return realloc(ptr, size); + return realloc(ptr, size_t(size)); } void ArrowFree(void *ptr) { diff --git a/src/common/adbc/nanoarrow/metadata.cpp b/src/common/adbc/nanoarrow/metadata.cpp index 742bbe4186c6..cb3009c01f3c 100644 --- a/src/common/adbc/nanoarrow/metadata.cpp +++ b/src/common/adbc/nanoarrow/metadata.cpp @@ -78,7 +78,7 @@ int64_t ArrowMetadataSizeOf(const char *metadata) { int64_t size = sizeof(int32_t); while (ArrowMetadataReaderRead(&reader, &key, &value) == NANOARROW_OK) { - size += sizeof(int32_t) + key.n_bytes + sizeof(int32_t) + value.n_bytes; + size += sizeof(int32_t) + uint64_t(key.n_bytes) + sizeof(int32_t) + uint64_t(value.n_bytes); } return size; @@ -89,7 +89,7 @@ ArrowErrorCode ArrowMetadataGetValue(const char *metadata, const char *key, cons struct ArrowStringView target_key_view = {key, static_cast(strlen(key))}; value_out->data = default_value; if (default_value != NULL) { - value_out->n_bytes = strlen(default_value); + value_out->n_bytes = int64_t(strlen(default_value)); } else { value_out->n_bytes = 0; } @@ -101,7 +101,7 @@ ArrowErrorCode ArrowMetadataGetValue(const char *metadata, const char *key, cons while (ArrowMetadataReaderRead(&reader, &key_view, &value) == NANOARROW_OK) { int key_equal = target_key_view.n_bytes == key_view.n_bytes && - strncmp(target_key_view.data, key_view.data, key_view.n_bytes) == 0; + strncmp(target_key_view.data, key_view.data, size_t(key_view.n_bytes)) == 0; if (key_equal) { value_out->data = value.data; value_out->n_bytes = value.n_bytes; diff --git a/src/common/adbc/nanoarrow/schema.cpp b/src/common/adbc/nanoarrow/schema.cpp index 1ed36f1f8f5f..38d1b314ff90 100644 --- a/src/common/adbc/nanoarrow/schema.cpp +++ b/src/common/adbc/nanoarrow/schema.cpp @@ -318,7 +318,7 @@ ArrowErrorCode ArrowSchemaSetFormat(struct ArrowSchema *schema, const char *form if (format != NULL) { size_t format_size = strlen(format) + 1; - schema->format = (const char *)ArrowMalloc(format_size); + schema->format = (const char *)ArrowMalloc(int64_t(format_size)); if (schema->format == NULL) { return ENOMEM; } @@ -338,7 +338,7 @@ ArrowErrorCode ArrowSchemaSetName(struct ArrowSchema *schema, const char *name) if (name != NULL) { size_t name_size = strlen(name) + 1; - schema->name = (const char *)ArrowMalloc(name_size); + schema->name = (const char *)ArrowMalloc(int64_t(name_size)); if (schema->name == NULL) { return ENOMEM; } @@ -357,13 +357,13 @@ ArrowErrorCode ArrowSchemaSetMetadata(struct ArrowSchema *schema, const char *me } if (metadata != NULL) { - size_t metadata_size = ArrowMetadataSizeOf(metadata); + auto metadata_size = ArrowMetadataSizeOf(metadata); schema->metadata = (const char *)ArrowMalloc(metadata_size); if (schema->metadata == NULL) { return ENOMEM; } - memcpy((void *)schema->metadata, metadata, metadata_size); + memcpy((void *)schema->metadata, metadata, size_t(metadata_size)); } else { schema->metadata = NULL; } @@ -377,7 +377,8 @@ ArrowErrorCode ArrowSchemaAllocateChildren(struct ArrowSchema *schema, int64_t n } if (n_children > 0) { - schema->children = (struct ArrowSchema **)ArrowMalloc(n_children * sizeof(struct ArrowSchema *)); + schema->children = + (struct ArrowSchema **)ArrowMalloc(int64_t(uint64_t(n_children) * sizeof(struct ArrowSchema *))); if (schema->children == NULL) { return ENOMEM; @@ -385,7 +386,7 @@ ArrowErrorCode ArrowSchemaAllocateChildren(struct ArrowSchema *schema, int64_t n schema->n_children = n_children; - memset(schema->children, 0, n_children * sizeof(struct ArrowSchema *)); + memset(schema->children, 0, uint64_t(n_children) * sizeof(struct ArrowSchema *)); for (int64_t i = 0; i < n_children; i++) { schema->children[i] = (struct ArrowSchema *)ArrowMalloc(sizeof(struct ArrowSchema)); diff --git a/src/common/arrow/appender/bool_data.cpp b/src/common/arrow/appender/bool_data.cpp index 6a5f67287387..d30b39339076 100644 --- a/src/common/arrow/appender/bool_data.cpp +++ b/src/common/arrow/appender/bool_data.cpp @@ -6,6 +6,7 @@ namespace duckdb { void ArrowBoolData::Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity) { auto byte_count = (capacity + 7) / 8; result.main_buffer.reserve(byte_count); + (void)AppendValidity; // silence a compiler warning about unused static function } void ArrowBoolData::Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) { diff --git a/src/common/arrow/appender/struct_data.cpp b/src/common/arrow/appender/struct_data.cpp index ce74a92a7b51..b2afa62d145e 100644 --- a/src/common/arrow/appender/struct_data.cpp +++ b/src/common/arrow/appender/struct_data.cpp @@ -35,7 +35,7 @@ void ArrowStructData::Finalize(ArrowAppendData &append_data, const LogicalType & auto &child_types = StructType::GetChildTypes(type); ArrowAppender::AddChildren(append_data, child_types.size()); result->children = append_data.child_pointers.data(); - result->n_children = child_types.size(); + result->n_children = NumericCast(child_types.size()); for (idx_t i = 0; i < child_types.size(); i++) { auto &child_type = child_types[i].second; append_data.child_arrays[i] = *ArrowAppender::FinalizeChild(child_type, std::move(append_data.child_data[i])); diff --git a/src/common/arrow/appender/union_data.cpp b/src/common/arrow/appender/union_data.cpp index 9797aa9f4772..270129c14dd1 100644 --- a/src/common/arrow/appender/union_data.cpp +++ b/src/common/arrow/appender/union_data.cpp @@ -14,6 +14,7 @@ void ArrowUnionData::Initialize(ArrowAppendData &result, const LogicalType &type auto child_buffer = ArrowAppender::InitializeChild(child.second, capacity, result.options); result.child_data.push_back(std::move(child_buffer)); } + (void)AppendValidity; // silence a compiler warning about unused static function } void ArrowUnionData::Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) { @@ -61,7 +62,7 @@ void ArrowUnionData::Finalize(ArrowAppendData &append_data, const LogicalType &t auto &child_types = UnionType::CopyMemberTypes(type); ArrowAppender::AddChildren(append_data, child_types.size()); result->children = append_data.child_pointers.data(); - result->n_children = child_types.size(); + result->n_children = NumericCast(child_types.size()); for (idx_t i = 0; i < child_types.size(); i++) { auto &child_type = child_types[i].second; append_data.child_arrays[i] = *ArrowAppender::FinalizeChild(child_type, std::move(append_data.child_data[i])); diff --git a/src/common/arrow/arrow_appender.cpp b/src/common/arrow/arrow_appender.cpp index 6dc0c14b3d24..e5ed4343c322 100644 --- a/src/common/arrow/arrow_appender.cpp +++ b/src/common/arrow/arrow_appender.cpp @@ -70,8 +70,8 @@ ArrowArray *ArrowAppender::FinalizeChild(const LogicalType &type, unique_ptroffset = 0; result->dictionary = nullptr; result->buffers = append_data.buffers.data(); - result->null_count = append_data.null_count; - result->length = append_data.row_count; + result->null_count = NumericCast(append_data.null_count); + result->length = NumericCast(append_data.row_count); result->buffers[0] = append_data.validity.data(); if (append_data.finalize) { @@ -90,10 +90,10 @@ ArrowArray ArrowAppender::Finalize() { ArrowArray result; AddChildren(*root_holder, types.size()); result.children = root_holder->child_pointers.data(); - result.n_children = types.size(); + result.n_children = NumericCast(types.size()); // Configure root array - result.length = row_count; + result.length = NumericCast(row_count); result.n_buffers = 1; result.buffers = root_holder->buffers.data(); // there is no actual buffer there since we don't have NULLs result.offset = 0; diff --git a/src/common/arrow/arrow_converter.cpp b/src/common/arrow/arrow_converter.cpp index 62be691b484f..b9f9ef836ed5 100644 --- a/src/common/arrow/arrow_converter.cpp +++ b/src/common/arrow/arrow_converter.cpp @@ -206,7 +206,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co case LogicalTypeId::STRUCT: { child.format = "+s"; auto &child_types = StructType::GetChildTypes(type); - child.n_children = child_types.size(); + child.n_children = NumericCast(child_types.size()); root_holder.nested_children.emplace_back(); root_holder.nested_children.back().resize(child_types.size()); root_holder.nested_children_ptr.emplace_back(); @@ -251,7 +251,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co std::string format = "+us:"; auto &child_types = UnionType::CopyMemberTypes(type); - child.n_children = child_types.size(); + child.n_children = NumericCast(child_types.size()); root_holder.nested_children.emplace_back(); root_holder.nested_children.back().resize(child_types.size()); root_holder.nested_children_ptr.emplace_back(); @@ -323,7 +323,7 @@ void ArrowConverter::ToArrowSchema(ArrowSchema *out_schema, const vectorchildren_ptrs[i] = &root_holder->children[i]; } out_schema->children = root_holder->children_ptrs.data(); - out_schema->n_children = column_count; + out_schema->n_children = NumericCast(column_count); // Store the schema out_schema->format = "+s"; // struct apparently diff --git a/src/common/box_renderer.cpp b/src/common/box_renderer.cpp index fc309150e1b7..629f6349fe69 100644 --- a/src/common/box_renderer.cpp +++ b/src/common/box_renderer.cpp @@ -399,7 +399,7 @@ vector BoxRenderer::ComputeRenderWidths(const vector &names, cons // e.g. if we have 10 columns, we remove #5, then #4, then #6, then #3, then #7, etc int64_t offset = 0; while (total_length > max_width) { - idx_t c = column_count / 2 + offset; + idx_t c = column_count / 2 + NumericCast(offset); total_length -= widths[c] + 3; pruned_columns.insert(c); if (offset >= 0) { diff --git a/src/common/compressed_file_system.cpp b/src/common/compressed_file_system.cpp index 725393904127..5727d4d7db93 100644 --- a/src/common/compressed_file_system.cpp +++ b/src/common/compressed_file_system.cpp @@ -1,4 +1,5 @@ #include "duckdb/common/compressed_file_system.hpp" +#include "duckdb/common/numeric_utils.hpp" namespace duckdb { @@ -37,7 +38,9 @@ int64_t CompressedFile::ReadData(void *buffer, int64_t remaining) { // first check if there are input bytes available in the output buffers if (stream_data.out_buff_start != stream_data.out_buff_end) { // there is! copy it into the output buffer - idx_t available = MinValue(remaining, stream_data.out_buff_end - stream_data.out_buff_start); + auto available = + MinValue(UnsafeNumericCast(remaining), + UnsafeNumericCast(stream_data.out_buff_end - stream_data.out_buff_start)); memcpy(data_ptr_t(buffer) + total_read, stream_data.out_buff_start, available); // increment the total read variables as required @@ -46,11 +49,11 @@ int64_t CompressedFile::ReadData(void *buffer, int64_t remaining) { remaining -= available; if (remaining == 0) { // done! read enough - return total_read; + return UnsafeNumericCast(total_read); } } if (!stream_wrapper) { - return total_read; + return UnsafeNumericCast(total_read); } // ran out of buffer: read more data from the child stream @@ -63,10 +66,11 @@ int64_t CompressedFile::ReadData(void *buffer, int64_t remaining) { if (stream_data.refresh && (stream_data.in_buff_end == stream_data.in_buff.get() + stream_data.in_buf_size)) { auto bufrem = stream_data.in_buff_end - stream_data.in_buff_start; // buffer not empty, move remaining bytes to the beginning - memmove(stream_data.in_buff.get(), stream_data.in_buff_start, bufrem); + memmove(stream_data.in_buff.get(), stream_data.in_buff_start, UnsafeNumericCast(bufrem)); stream_data.in_buff_start = stream_data.in_buff.get(); // refill the rest of input buffer - auto sz = child_handle->Read(stream_data.in_buff_start + bufrem, stream_data.in_buf_size - bufrem); + auto sz = child_handle->Read(stream_data.in_buff_start + bufrem, + stream_data.in_buf_size - UnsafeNumericCast(bufrem)); stream_data.in_buff_end = stream_data.in_buff_start + bufrem + sz; if (sz <= 0) { stream_wrapper.reset(); @@ -92,7 +96,7 @@ int64_t CompressedFile::ReadData(void *buffer, int64_t remaining) { stream_wrapper.reset(); } } - return total_read; + return UnsafeNumericCast(total_read); } int64_t CompressedFile::WriteData(data_ptr_t buffer, int64_t nr_bytes) { @@ -134,7 +138,7 @@ void CompressedFileSystem::Reset(FileHandle &handle) { int64_t CompressedFileSystem::GetFileSize(FileHandle &handle) { auto &compressed_file = handle.Cast(); - return compressed_file.child_handle->GetFileSize(); + return NumericCast(compressed_file.child_handle->GetFileSize()); } bool CompressedFileSystem::OnDiskFile(FileHandle &handle) { diff --git a/src/common/exception_format_value.cpp b/src/common/exception_format_value.cpp index 1eb9d45939f5..ddef4e10c872 100644 --- a/src/common/exception_format_value.cpp +++ b/src/common/exception_format_value.cpp @@ -1,5 +1,6 @@ #include "duckdb/common/exception.hpp" #include "duckdb/common/types.hpp" +#include "duckdb/common/helper.hpp" // defines DUCKDB_EXPLICIT_FALLTHROUGH which fmt will use to annotate #include "fmt/format.h" #include "fmt/printf.h" #include "duckdb/common/types/hugeint.hpp" diff --git a/src/common/file_system.cpp b/src/common/file_system.cpp index 7450c1fbec9a..ed912d590bc7 100644 --- a/src/common/file_system.cpp +++ b/src/common/file_system.cpp @@ -517,7 +517,7 @@ FileHandle::~FileHandle() { } int64_t FileHandle::Read(void *buffer, idx_t nr_bytes) { - return file_system.Read(*this, buffer, nr_bytes); + return file_system.Read(*this, buffer, UnsafeNumericCast(nr_bytes)); } bool FileHandle::Trim(idx_t offset_bytes, idx_t length_bytes) { @@ -525,15 +525,15 @@ bool FileHandle::Trim(idx_t offset_bytes, idx_t length_bytes) { } int64_t FileHandle::Write(void *buffer, idx_t nr_bytes) { - return file_system.Write(*this, buffer, nr_bytes); + return file_system.Write(*this, buffer, UnsafeNumericCast(nr_bytes)); } void FileHandle::Read(void *buffer, idx_t nr_bytes, idx_t location) { - file_system.Read(*this, buffer, nr_bytes, location); + file_system.Read(*this, buffer, UnsafeNumericCast(nr_bytes), location); } void FileHandle::Write(void *buffer, idx_t nr_bytes, idx_t location) { - file_system.Write(*this, buffer, nr_bytes, location); + file_system.Write(*this, buffer, UnsafeNumericCast(nr_bytes), location); } void FileHandle::Seek(idx_t location) { @@ -560,7 +560,7 @@ string FileHandle::ReadLine() { string result; char buffer[1]; while (true) { - idx_t tuples_read = Read(buffer, 1); + auto tuples_read = UnsafeNumericCast(Read(buffer, 1)); if (tuples_read == 0 || buffer[0] == '\n') { return result; } @@ -575,7 +575,7 @@ bool FileHandle::OnDiskFile() { } idx_t FileHandle::GetFileSize() { - return file_system.GetFileSize(*this); + return NumericCast(file_system.GetFileSize(*this)); } void FileHandle::Sync() { diff --git a/src/common/gzip_file_system.cpp b/src/common/gzip_file_system.cpp index d24d5e57149f..a4e39b90d39b 100644 --- a/src/common/gzip_file_system.cpp +++ b/src/common/gzip_file_system.cpp @@ -120,13 +120,13 @@ void MiniZStreamWrapper::Initialize(CompressedFile &file, bool write) { } else { idx_t data_start = GZIP_HEADER_MINSIZE; auto read_count = file.child_handle->Read(gzip_hdr, GZIP_HEADER_MINSIZE); - GZipFileSystem::VerifyGZIPHeader(gzip_hdr, read_count); + GZipFileSystem::VerifyGZIPHeader(gzip_hdr, NumericCast(read_count)); // Skip over the extra field if necessary if (gzip_hdr[3] & GZIP_FLAG_EXTRA) { uint8_t gzip_xlen[2]; file.child_handle->Seek(data_start); file.child_handle->Read(gzip_xlen, 2); - idx_t xlen = (uint8_t)gzip_xlen[0] | (uint8_t)gzip_xlen[1] << 8; + auto xlen = NumericCast((uint8_t)gzip_xlen[0] | (uint8_t)gzip_xlen[1] << 8); data_start += xlen + 2; } // Skip over the file name if necessary @@ -160,7 +160,7 @@ bool MiniZStreamWrapper::Read(StreamData &sd) { GZipFileSystem::VerifyGZIPHeader(gzip_hdr, GZIP_HEADER_MINSIZE); body_ptr += GZIP_HEADER_MINSIZE; if (gzip_hdr[3] & GZIP_FLAG_EXTRA) { - idx_t xlen = (uint8_t)*body_ptr | (uint8_t) * (body_ptr + 1) << 8; + auto xlen = NumericCast((uint8_t)*body_ptr | (uint8_t) * (body_ptr + 1) << 8); body_ptr += xlen + 2; if (GZIP_FOOTER_SIZE + GZIP_HEADER_MINSIZE + 2 + xlen >= GZIP_HEADER_MAXSIZE) { throw InternalException("Extra field resulting in GZIP header larger than defined maximum (%d)", @@ -170,7 +170,7 @@ bool MiniZStreamWrapper::Read(StreamData &sd) { if (gzip_hdr[3] & GZIP_FLAG_NAME) { char c; do { - c = *body_ptr; + c = UnsafeNumericCast(*body_ptr); body_ptr++; } while (c != '\0' && body_ptr < sd.in_buff_end); if ((idx_t)(body_ptr - sd.in_buff_start) >= GZIP_HEADER_MAXSIZE) { @@ -217,12 +217,13 @@ bool MiniZStreamWrapper::Read(StreamData &sd) { void MiniZStreamWrapper::Write(CompressedFile &file, StreamData &sd, data_ptr_t uncompressed_data, int64_t uncompressed_size) { // update the src and the total size - crc = duckdb_miniz::mz_crc32(crc, reinterpret_cast(uncompressed_data), uncompressed_size); - total_size += uncompressed_size; + crc = duckdb_miniz::mz_crc32(crc, reinterpret_cast(uncompressed_data), + UnsafeNumericCast(uncompressed_size)); + total_size += UnsafeNumericCast(uncompressed_size); auto remaining = uncompressed_size; while (remaining > 0) { - idx_t output_remaining = (sd.out_buff.get() + sd.out_buf_size) - sd.out_buff_start; + auto output_remaining = UnsafeNumericCast((sd.out_buff.get() + sd.out_buf_size) - sd.out_buff_start); mz_stream_ptr->next_in = reinterpret_cast(uncompressed_data); mz_stream_ptr->avail_in = NumericCast(remaining); @@ -237,10 +238,11 @@ void MiniZStreamWrapper::Write(CompressedFile &file, StreamData &sd, data_ptr_t sd.out_buff_start += output_remaining - mz_stream_ptr->avail_out; if (mz_stream_ptr->avail_out == 0) { // no more output buffer available: flush - file.child_handle->Write(sd.out_buff.get(), sd.out_buff_start - sd.out_buff.get()); + file.child_handle->Write(sd.out_buff.get(), + UnsafeNumericCast(sd.out_buff_start - sd.out_buff.get())); sd.out_buff_start = sd.out_buff.get(); } - idx_t written = remaining - mz_stream_ptr->avail_in; + auto written = UnsafeNumericCast(remaining - mz_stream_ptr->avail_in); uncompressed_data += written; remaining = mz_stream_ptr->avail_in; } @@ -258,7 +260,8 @@ void MiniZStreamWrapper::FlushStream() { auto res = mz_deflate(mz_stream_ptr.get(), duckdb_miniz::MZ_FINISH); sd.out_buff_start += (output_remaining - mz_stream_ptr->avail_out); if (sd.out_buff_start > sd.out_buff.get()) { - file->child_handle->Write(sd.out_buff.get(), sd.out_buff_start - sd.out_buff.get()); + file->child_handle->Write(sd.out_buff.get(), + UnsafeNumericCast(sd.out_buff_start - sd.out_buff.get())); sd.out_buff_start = sd.out_buff.get(); } if (res == duckdb_miniz::MZ_STREAM_END) { @@ -354,7 +357,7 @@ string GZipFileSystem::UncompressGZIPString(const string &in) { throw InternalException("Failed to initialize miniz"); } - auto bytes_remaining = in.size() - (body_ptr - in.data()); + auto bytes_remaining = in.size() - NumericCast(body_ptr - in.data()); mz_stream_ptr->next_in = const_uchar_ptr_cast(body_ptr); mz_stream_ptr->avail_in = NumericCast(bytes_remaining); diff --git a/src/common/hive_partitioning.cpp b/src/common/hive_partitioning.cpp index 0bb3306b7805..5dc8248cc3be 100644 --- a/src/common/hive_partitioning.cpp +++ b/src/common/hive_partitioning.cpp @@ -359,8 +359,8 @@ void HivePartitionedColumnData::GrowPartitions(PartitionedColumnDataAppendState void HivePartitionedColumnData::SynchronizeLocalMap() { // Synchronise global map into local, may contain changes from other threads too - for (auto it = global_state->partitions.begin() + local_partition_map.size(); it < global_state->partitions.end(); - it++) { + for (auto it = global_state->partitions.begin() + NumericCast(local_partition_map.size()); + it < global_state->partitions.end(); it++) { local_partition_map[(*it)->first] = (*it)->second; } } diff --git a/src/common/local_file_system.cpp b/src/common/local_file_system.cpp index b05189b4ea0f..43ef0a8badda 100644 --- a/src/common/local_file_system.cpp +++ b/src/common/local_file_system.cpp @@ -389,7 +389,7 @@ unique_ptr LocalFileSystem::OpenFile(const string &path_p, FileOpenF void LocalFileSystem::SetFilePointer(FileHandle &handle, idx_t location) { int fd = handle.Cast().fd; - off_t offset = lseek(fd, location, SEEK_SET); + off_t offset = lseek(fd, UnsafeNumericCast(location), SEEK_SET); if (offset == (off_t)-1) { throw IOException("Could not seek to location %lld for file \"%s\": %s", {{"errno", std::to_string(errno)}}, location, handle.path, strerror(errno)); @@ -403,14 +403,15 @@ idx_t LocalFileSystem::GetFilePointer(FileHandle &handle) { throw IOException("Could not get file position file \"%s\": %s", {{"errno", std::to_string(errno)}}, handle.path, strerror(errno)); } - return position; + return UnsafeNumericCast(position); } void LocalFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) { int fd = handle.Cast().fd; auto read_buffer = char_ptr_cast(buffer); while (nr_bytes > 0) { - int64_t bytes_read = pread(fd, read_buffer, nr_bytes, location); + int64_t bytes_read = + pread(fd, read_buffer, UnsafeNumericCast(nr_bytes), UnsafeNumericCast(location)); if (bytes_read == -1) { throw IOException("Could not read from file \"%s\": %s", {{"errno", std::to_string(errno)}}, handle.path, strerror(errno)); @@ -422,13 +423,13 @@ void LocalFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes, i } read_buffer += bytes_read; nr_bytes -= bytes_read; - location += bytes_read; + location += UnsafeNumericCast(bytes_read); } } int64_t LocalFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes) { int fd = handle.Cast().fd; - int64_t bytes_read = read(fd, buffer, nr_bytes); + int64_t bytes_read = read(fd, buffer, UnsafeNumericCast(nr_bytes)); if (bytes_read == -1) { throw IOException("Could not read from file \"%s\": %s", {{"errno", std::to_string(errno)}}, handle.path, strerror(errno)); @@ -440,7 +441,8 @@ void LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes, int fd = handle.Cast().fd; auto write_buffer = char_ptr_cast(buffer); while (nr_bytes > 0) { - int64_t bytes_written = pwrite(fd, write_buffer, nr_bytes, location); + int64_t bytes_written = + pwrite(fd, write_buffer, UnsafeNumericCast(nr_bytes), UnsafeNumericCast(location)); if (bytes_written < 0) { throw IOException("Could not write file \"%s\": %s", {{"errno", std::to_string(errno)}}, handle.path, strerror(errno)); @@ -451,7 +453,7 @@ void LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes, } write_buffer += bytes_written; nr_bytes -= bytes_written; - location += bytes_written; + location += UnsafeNumericCast(bytes_written); } } diff --git a/src/common/multi_file_reader.cpp b/src/common/multi_file_reader.cpp index bbb59705e29f..05ff0cb8ffa3 100644 --- a/src/common/multi_file_reader.cpp +++ b/src/common/multi_file_reader.cpp @@ -176,7 +176,7 @@ MultiFileReaderBindData MultiFileReader::BindOptions(MultiFileReaderOptions &opt auto lookup = std::find(names.begin(), names.end(), part.first); if (lookup != names.end()) { // hive partitioning column also exists in file - override - auto idx = lookup - names.begin(); + auto idx = NumericCast(lookup - names.begin()); hive_partitioning_index = idx; return_types[idx] = options.GetHiveLogicalType(part.first); } else { diff --git a/src/common/pipe_file_system.cpp b/src/common/pipe_file_system.cpp index 39a1877dd9bf..d6eb2c6aff48 100644 --- a/src/common/pipe_file_system.cpp +++ b/src/common/pipe_file_system.cpp @@ -2,6 +2,7 @@ #include "duckdb/common/exception.hpp" #include "duckdb/common/file_system.hpp" #include "duckdb/common/helper.hpp" +#include "duckdb/common/numeric_utils.hpp" namespace duckdb { class PipeFile : public FileHandle { @@ -22,10 +23,10 @@ class PipeFile : public FileHandle { }; int64_t PipeFile::ReadChunk(void *buffer, int64_t nr_bytes) { - return child_handle->Read(buffer, nr_bytes); + return child_handle->Read(buffer, UnsafeNumericCast(nr_bytes)); } int64_t PipeFile::WriteChunk(void *buffer, int64_t nr_bytes) { - return child_handle->Write(buffer, nr_bytes); + return child_handle->Write(buffer, UnsafeNumericCast(nr_bytes)); } void PipeFileSystem::Reset(FileHandle &handle) { diff --git a/src/common/random_engine.cpp b/src/common/random_engine.cpp index 0c9aec4e5119..acbda8b3e194 100644 --- a/src/common/random_engine.cpp +++ b/src/common/random_engine.cpp @@ -1,4 +1,5 @@ #include "duckdb/common/random_engine.hpp" +#include "duckdb/common/numeric_utils.hpp" #include "pcg_random.hpp" #include @@ -15,7 +16,7 @@ RandomEngine::RandomEngine(int64_t seed) : random_state(make_uniq() if (seed < 0) { random_state->pcg.seed(pcg_extras::seed_seq_from()); } else { - random_state->pcg.seed(seed); + random_state->pcg.seed(NumericCast(seed)); } } diff --git a/src/common/re2_regex.cpp b/src/common/re2_regex.cpp index f22b681acfe6..b67b3c6524c8 100644 --- a/src/common/re2_regex.cpp +++ b/src/common/re2_regex.cpp @@ -17,10 +17,11 @@ bool RegexSearchInternal(const char *input, Match &match, const Regex &r, RE2::A size_t end) { auto ®ex = r.GetRegex(); duckdb::vector target_groups; - auto group_count = regex.NumberOfCapturingGroups() + 1; + auto group_count = duckdb::UnsafeNumericCast(regex.NumberOfCapturingGroups() + 1); target_groups.resize(group_count); match.groups.clear(); - if (!regex.Match(StringPiece(input), start, end, anchor, target_groups.data(), group_count)) { + if (!regex.Match(StringPiece(input), start, end, anchor, target_groups.data(), + duckdb::UnsafeNumericCast(group_count))) { return false; } for (auto &group : target_groups) { @@ -41,7 +42,8 @@ bool RegexMatch(const std::string &input, Match &match, const Regex ®ex) { } bool RegexMatch(const char *start, const char *end, Match &match, const Regex ®ex) { - return RegexSearchInternal(start, match, regex, RE2::ANCHOR_BOTH, 0, end - start); + return RegexSearchInternal(start, match, regex, RE2::ANCHOR_BOTH, 0, + duckdb::UnsafeNumericCast(end - start)); } bool RegexMatch(const std::string &input, const Regex ®ex) { diff --git a/src/common/string_util.cpp b/src/common/string_util.cpp index 7898a343a927..7d33b1cd80f7 100644 --- a/src/common/string_util.cpp +++ b/src/common/string_util.cpp @@ -203,7 +203,8 @@ string StringUtil::Upper(const string &str) { string StringUtil::Lower(const string &str) { string copy(str); - transform(copy.begin(), copy.end(), copy.begin(), [](unsigned char c) { return StringUtil::CharacterToLower(c); }); + transform(copy.begin(), copy.end(), copy.begin(), + [](unsigned char c) { return StringUtil::CharacterToLower(UnsafeNumericCast(c)); }); return (copy); } @@ -215,7 +216,7 @@ bool StringUtil::IsLower(const string &str) { uint64_t StringUtil::CIHash(const string &str) { uint32_t hash = 0; for (auto c : str) { - hash += StringUtil::CharacterToLower(c); + hash += UnsafeNumericCast(StringUtil::CharacterToLower(UnsafeNumericCast(c))); hash += hash << 10; hash ^= hash >> 6; } diff --git a/src/include/duckdb/common/arrow/appender/append_data.hpp b/src/include/duckdb/common/arrow/appender/append_data.hpp index 8cbbf02c8ee7..52845cbd2dc6 100644 --- a/src/include/duckdb/common/arrow/appender/append_data.hpp +++ b/src/include/duckdb/common/arrow/appender/append_data.hpp @@ -59,6 +59,7 @@ struct ArrowAppendData { //===--------------------------------------------------------------------===// // Append Helper Functions //===--------------------------------------------------------------------===// + static void GetBitPosition(idx_t row_idx, idx_t ¤t_byte, uint8_t ¤t_bit) { current_byte = row_idx / 8; current_bit = row_idx % 8; diff --git a/src/include/duckdb/common/arrow/appender/enum_data.hpp b/src/include/duckdb/common/arrow/appender/enum_data.hpp index e6ed5c107b0a..f7f73e055141 100644 --- a/src/include/duckdb/common/arrow/appender/enum_data.hpp +++ b/src/include/duckdb/common/arrow/appender/enum_data.hpp @@ -44,14 +44,14 @@ struct ArrowEnumData : public ArrowScalarBaseData { auto string_length = GetLength(data[i]); // append the offset data - auto current_offset = last_offset + string_length; - offset_data[offset_idx] = UnsafeNumericCast(current_offset); + auto current_offset = UnsafeNumericCast(last_offset) + string_length; + offset_data[offset_idx] = UnsafeNumericCast(current_offset); // resize the string buffer if required, and write the string data append_data.aux_buffer.resize(current_offset); WriteData(append_data.aux_buffer.data() + last_offset, data[i]); - last_offset = UnsafeNumericCast(current_offset); + last_offset = UnsafeNumericCast(current_offset); } append_data.row_count += size; } diff --git a/src/include/duckdb/common/arrow/appender/map_data.hpp b/src/include/duckdb/common/arrow/appender/map_data.hpp index e881c532a15a..630ff1664f9c 100644 --- a/src/include/duckdb/common/arrow/appender/map_data.hpp +++ b/src/include/duckdb/common/arrow/appender/map_data.hpp @@ -75,7 +75,7 @@ struct ArrowMapData { struct_result->children = struct_data.child_pointers.data(); struct_result->n_buffers = 1; struct_result->n_children = struct_child_count; - struct_result->length = struct_data.child_data[0]->row_count; + struct_result->length = NumericCast(struct_data.child_data[0]->row_count); append_data.child_arrays[0] = *struct_result; diff --git a/src/include/duckdb/common/arrow/appender/varchar_data.hpp b/src/include/duckdb/common/arrow/appender/varchar_data.hpp index d6ffcd9db868..753756b9ea0b 100644 --- a/src/include/duckdb/common/arrow/appender/varchar_data.hpp +++ b/src/include/duckdb/common/arrow/appender/varchar_data.hpp @@ -77,8 +77,9 @@ struct ArrowVarcharData { auto string_length = OP::GetLength(data[source_idx]); // append the offset data - auto current_offset = last_offset + string_length; - if (!LARGE_STRING && (int64_t)last_offset + string_length > NumericLimits::Maximum()) { + auto current_offset = UnsafeNumericCast(last_offset) + string_length; + if (!LARGE_STRING && + UnsafeNumericCast(last_offset) + string_length > NumericLimits::Maximum()) { D_ASSERT(append_data.options.arrow_offset_size == ArrowOffsetSize::REGULAR); throw InvalidInputException( "Arrow Appender: The maximum total string size for regular string buffers is " diff --git a/src/include/duckdb/storage/string_uncompressed.hpp b/src/include/duckdb/storage/string_uncompressed.hpp index 6ad17066e58f..dacad03a0edb 100644 --- a/src/include/duckdb/storage/string_uncompressed.hpp +++ b/src/include/duckdb/storage/string_uncompressed.hpp @@ -148,7 +148,7 @@ struct UncompressedStringStorage { // place the dictionary offset into the set of vectors // note: for overflow strings we write negative value - result_data[target_idx] = -(*dictionary_size); + result_data[target_idx] = NumericCast(-(*dictionary_size)); } else { // string fits in block, append to dictionary and increment dictionary position D_ASSERT(string_length < NumericLimits::Maximum()); @@ -159,7 +159,7 @@ struct UncompressedStringStorage { memcpy(dict_pos, source_data[source_idx].GetData(), string_length); // place the dictionary offset into the set of vectors - result_data[target_idx] = *dictionary_size; + result_data[target_idx] = NumericCast(*dictionary_size); } D_ASSERT(RemainingSpace(segment, handle) <= Storage::BLOCK_SIZE); #ifdef DEBUG diff --git a/third_party/fsst/fsst.h b/third_party/fsst/fsst.h index 6970dedc053f..553c143cd021 100644 --- a/third_party/fsst/fsst.h +++ b/third_party/fsst/fsst.h @@ -184,7 +184,7 @@ duckdb_fsst_decompress( code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; } else { - unsigned long firstEscapePos=__builtin_ctzl((unsigned long long) escapeMask)>>3; + unsigned long firstEscapePos=static_cast(__builtin_ctzl((unsigned long long) escapeMask)>>3); switch(firstEscapePos) { /* Duff's device */ case 3: code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; DUCKDB_FSST_EXPLICIT_FALLTHROUGH; From 3db603c364c44af68fcdcfe97f60d1544072ea4c Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 4 Apr 2024 12:08:05 +0200 Subject: [PATCH 148/603] Fixing rejects_read to new model, properly increment scan_id --- .../scanner/string_value_scanner.cpp | 3 +- .../table_function/global_csv_state.cpp | 3 +- .../transaction/transaction_context.hpp | 2 + src/transaction/transaction_context.cpp | 6 + .../copy/csv/rejects/csv_rejects_read.test | 268 ++++++------------ 5 files changed, 89 insertions(+), 193 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index d3e980a57392..dd7c4364fb9d 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -120,8 +120,8 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size if (error) { // We error pointing to the current value error. current_errors.push_back({CSVErrorType::TOO_MANY_COLUMNS, cur_col_id, last_position}); + cur_col_id++; } - cur_col_id++; return; } @@ -552,7 +552,6 @@ bool StringValueResult::AddRowInternal() { last_position.GetGlobalPosition(requested_size)); error_handler.Error(csv_error); } - // If we are here we ignore_errors, so we delete this line number_of_rows--; } diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index b16c2d48df79..6646f4ab98b8 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -245,8 +245,8 @@ void CSVGlobalState::FillRejectsTable() { InternalAppender errors_appender(context, errors_table); InternalAppender scans_appender(context, scans_table); idx_t scan_idx = context.transaction.GetActiveQuery(); - idx_t file_idx = 0; for (auto &file : file_scans) { + idx_t file_idx = context.transaction.GetIncrementalIndex(); auto file_name = file->file_path; auto &errors = file->error_handler->errors; // We first insert the file into the file scans table @@ -309,7 +309,6 @@ void CSVGlobalState::FillRejectsTable() { rejects->count = 0; FillScanErrorTable(scans_appender, scan_idx, file_idx, *file); } - file_idx++; } errors_appender.Close(); scans_appender.Close(); diff --git a/src/include/duckdb/transaction/transaction_context.hpp b/src/include/duckdb/transaction/transaction_context.hpp index b0a50103bb46..b265c0131498 100644 --- a/src/include/duckdb/transaction/transaction_context.hpp +++ b/src/include/duckdb/transaction/transaction_context.hpp @@ -48,6 +48,7 @@ class TransactionContext { } idx_t GetActiveQuery(); + idx_t GetIncrementalIndex(); void ResetActiveQuery(); void SetActiveQuery(transaction_t query_number); @@ -56,6 +57,7 @@ class TransactionContext { bool auto_commit; unique_ptr current_transaction; + idx_t incremental_index = 0; TransactionContext(const TransactionContext &) = delete; }; diff --git a/src/transaction/transaction_context.cpp b/src/transaction/transaction_context.cpp index 7185a263894b..82d1fa43094f 100644 --- a/src/transaction/transaction_context.cpp +++ b/src/transaction/transaction_context.cpp @@ -89,13 +89,19 @@ idx_t TransactionContext::GetActiveQuery() { return current_transaction->GetActiveQuery(); } +idx_t TransactionContext::GetIncrementalIndex() { + return incremental_index++; +} + void TransactionContext::ResetActiveQuery() { + incremental_index = 0; if (current_transaction) { SetActiveQuery(MAXIMUM_QUERY_ID); } } void TransactionContext::SetActiveQuery(transaction_t query_number) { + incremental_index = 0; if (!current_transaction) { throw InternalException("SetActiveQuery called without active transaction"); } diff --git a/test/sql/copy/csv/rejects/csv_rejects_read.test b/test/sql/copy/csv/rejects/csv_rejects_read.test index 9917965558ba..b537833fd7dd 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_read.test +++ b/test/sql/copy/csv/rejects/csv_rejects_read.test @@ -10,101 +10,67 @@ query III rowsort SELECT * FROM read_csv( 'test/sql/copy/csv/data/error/mismatch/bad.csv', columns = {'col0': 'INTEGER', 'col1': 'INTEGER', 'col2': 'VARCHAR'}, - rejects_table='csv_rejects_table', - ignore_errors=true, auto_detect=false); + store_rejects = true, auto_detect=true); ---- 1 2 AAA 6 7 CCC -query IIIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position -FROM csv_rejects_table; +query IIIIIIIIII +FROM reject_errors; ---- -test/sql/copy/csv/data/error/mismatch/bad.csv 2 2 "col1" CAST 4,BBB,9, 9 +3 0 2 10 12 2 col1 CAST 4,BBB,9, Error when converting column "col1". Could not convert string "BBB" to 'INTEGER' -query I -SELECT error_message -FROM csv_rejects_table; ----- -:.*Could not convert string "BBB" to 'INTEGER'.* +statement ok +DROP TABLE reject_errors; statement ok -DROP TABLE csv_rejects_table; +DROP TABLE reject_scans; # Test with multiple columns on the same row query III rowsort SELECT * FROM read_csv( 'test/sql/copy/csv/data/error/mismatch/bad2.csv', columns = {'col0': 'INTEGER', 'col1': 'INTEGER', 'col2': 'INTEGER'}, - rejects_table='csv_rejects_table', - ignore_errors=true, auto_detect=false); + store_rejects = true, auto_detect=false); ---- 4 5 9 -query IIIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position -FROM csv_rejects_table; ----- -test/sql/copy/csv/data/error/mismatch/bad2.csv 1 3 "col2" CAST 1,2,DDD, 0 -test/sql/copy/csv/data/error/mismatch/bad2.csv 3 1 "col0" CAST EEE,7,FFF, 16 -test/sql/copy/csv/data/error/mismatch/bad2.csv 3 3 "col2" CAST EEE,7,FFF, 16 - -query I -SELECT error_message -FROM csv_rejects_table where line=1 and column_idx=3; +query IIIIIIIIII +FROM reject_errors ORDER BY ALL; ---- -:.*Could not convert string "DDD" to 'INTEGER'.* +7 0 1 1 5 3 col2 CAST 1,2,DDD, Error when converting column "col2". Could not convert string "DDD" to 'INTEGER' +7 0 3 17 17 1 col0 CAST EEE,7,FFF, Error when converting column "col0". Could not convert string "EEE" to 'INTEGER' +7 0 3 17 23 3 col2 CAST EEE,7,FFF, Error when converting column "col2". Could not convert string "FFF" to 'INTEGER' -query I -SELECT error_message -FROM csv_rejects_table where line=3 and column_idx=1; ----- -:.*Could not convert string "EEE" to 'INTEGER'.* - -query I -SELECT error_message -FROM csv_rejects_table where line=3 and column_idx=3; ----- -:.*Could not convert string "FFF" to 'INTEGER'.* +statement ok +DROP TABLE reject_errors; statement ok -DROP TABLE csv_rejects_table; +DROP TABLE reject_scans; # Test with multiple files query III rowsort SELECT * FROM read_csv( 'test/sql/copy/csv/data/error/mismatch/bad*.csv', columns = {'col0': 'INTEGER', 'col1': 'INTEGER', 'col2': 'VARCHAR'}, - rejects_table='csv_rejects_table', - ignore_errors=true, auto_detect=false); + store_rejects = true, auto_detect=false); ---- 1 2 AAA 1 2 DDD 4 5 9 6 7 CCC - -query IIIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position -FROM csv_rejects_table; +query IIIIIIIIII +FROM reject_errors ORDER BY ALL; ---- -test/sql/copy/csv/data/error/mismatch/bad.csv 2 2 "col1" CAST 4,BBB,9, 9 -test/sql/copy/csv/data/error/mismatch/bad2.csv 3 1 "col0" CAST EEE,7,FFF, 16 +11 0 2 10 12 2 col1 CAST 4,BBB,9, Error when converting column "col1". Could not convert string "BBB" to 'INTEGER' +11 1 3 17 17 1 col0 CAST EEE,7,FFF, Error when converting column "col0". Could not convert string "EEE" to 'INTEGER' -query I -SELECT error_message -FROM csv_rejects_table where line=2 and column_idx=2; ----- -:.*Could not convert string "BBB" to 'INTEGER'.* - -query I -SELECT error_message -FROM csv_rejects_table where line=3 and column_idx=1; ----- -:.*Could not convert string "EEE" to 'INTEGER'.* +statement ok +DROP TABLE reject_errors; statement ok -DROP TABLE csv_rejects_table; +DROP TABLE reject_scans; # Set limit @@ -112,9 +78,7 @@ query III rowsort SELECT * FROM read_csv( 'test/sql/copy/csv/data/error/mismatch/bad*.csv', columns = {'col0': 'INTEGER', 'col1': 'INTEGER', 'col2': 'VARCHAR'}, - rejects_table='csv_rejects_table', - rejects_limit=2, - ignore_errors=true, auto_detect=false); + store_rejects = true,rejects_limit=2, ignore_errors=true, auto_detect=false); ---- 1 2 AAA 1 2 DDD @@ -123,121 +87,80 @@ SELECT * FROM read_csv( # We should now only have two errors logged query I -SELECT COUNT(*) FROM csv_rejects_table +SELECT COUNT(*) FROM reject_errors ---- 2 statement ok -DROP TABLE csv_rejects_table; +DROP TABLE reject_errors; + +statement ok +DROP TABLE reject_scans; # Try with bigger files query I SELECT SUM(num) FROM read_csv( 'test/sql/copy/csv/data/error/mismatch/big_bad.csv', columns = {'num': 'INTEGER', 'str': 'VARCHAR'}, - rejects_table='csv_rejects_table', - ignore_errors=true, auto_detect=false); + store_rejects = true, auto_detect=false); ---- 4270 -query IIIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position -FROM csv_rejects_table; ----- -test/sql/copy/csv/data/error/mismatch/big_bad.csv 2176 1 "num" CAST B, A 10875 -test/sql/copy/csv/data/error/mismatch/big_bad.csv 4176 1 "num" CAST C, A 20875 - -query I -SELECT error_message -FROM csv_rejects_table where line=2176 and column_idx=1; +query IIIIIIIIII +FROM reject_errors ORDER BY ALL; ---- -:.*Could not convert string "B" to 'INTEGER'.* +19 0 2176 10876 10876 1 num CAST B, A Error when converting column "num". Could not convert string "B" to 'INTEGER' +19 0 4176 20876 20876 1 num CAST C, A Error when converting column "num". Could not convert string "C" to 'INTEGER' -query I -SELECT error_message -FROM csv_rejects_table where line=4176 and column_idx=1; ----- -:.*Could not convert string "C" to 'INTEGER'.* +statement ok +DROP TABLE reject_errors; statement ok -DROP TABLE csv_rejects_table; +DROP TABLE reject_scans; query I SELECT SUM(num) FROM read_csv( 'test/sql/copy/csv/data/error/mismatch/big_bad2.csv', columns = {'num': 'INTEGER', 'str': 'VARCHAR'}, - rejects_table='csv_rejects_table', - ignore_errors=true, auto_detect=false) + store_rejects = true, auto_detect=false) ---- 6774 -query IIIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position -FROM csv_rejects_table; +query IIIIIIIIII +FROM reject_errors ORDER BY ALL; ---- -test/sql/copy/csv/data/error/mismatch/big_bad2.csv 3680 1 "num" CAST B, A 18395 -test/sql/copy/csv/data/error/mismatch/big_bad2.csv 5680 1 "num" CAST C, A 28395 +23 0 3680 18396 18396 1 num CAST B, A Error when converting column "num". Could not convert string "B" to 'INTEGER' +23 0 5680 28396 28396 1 num CAST C, A Error when converting column "num". Could not convert string "C" to 'INTEGER' -query I -SELECT error_message -FROM csv_rejects_table where line=3680 and column_idx=1; ----- -:.*Could not convert string "B" to 'INTEGER'.* -query I -SELECT error_message -FROM csv_rejects_table where line=5680 and column_idx=1; ----- -:.*Could not convert string "C" to 'INTEGER'.* +statement ok +DROP TABLE reject_errors; statement ok -DROP TABLE csv_rejects_table; +DROP TABLE reject_scans; # Test with multiple big files query I SELECT SUM(num) FROM read_csv( 'test/sql/copy/csv/data/error/mismatch/big_*.csv', columns = {'num': 'INTEGER', 'str': 'VARCHAR'}, - rejects_table='csv_rejects_table', - ignore_errors=true, auto_detect=false); + store_rejects = true, auto_detect=false); ---- 11044 -query IIIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position -FROM csv_rejects_table; ----- -test/sql/copy/csv/data/error/mismatch/big_bad.csv 2176 1 "num" CAST B, A 10875 -test/sql/copy/csv/data/error/mismatch/big_bad.csv 4176 1 "num" CAST C, A 20875 -test/sql/copy/csv/data/error/mismatch/big_bad2.csv 3680 1 "num" CAST B, A 18395 -test/sql/copy/csv/data/error/mismatch/big_bad2.csv 5680 1 "num" CAST C, A 28395 - -query I -SELECT error_message -FROM csv_rejects_table where line=3680 and column_idx=1; ----- -:.*Could not convert string "B" to 'INTEGER'.* - -query I -SELECT error_message -FROM csv_rejects_table where line=5680 and column_idx=1; +query IIIIIIIIII +FROM reject_errors ORDER BY ALL; ---- -:.*Could not convert string "C" to 'INTEGER'.* +27 0 2176 10876 10876 1 num CAST B, A Error when converting column "num". Could not convert string "B" to 'INTEGER' +27 0 4176 20876 20876 1 num CAST C, A Error when converting column "num". Could not convert string "C" to 'INTEGER' +27 1 3680 18396 18396 1 num CAST B, A Error when converting column "num". Could not convert string "B" to 'INTEGER' +27 1 5680 28396 28396 1 num CAST C, A Error when converting column "num". Could not convert string "C" to 'INTEGER' -query I -SELECT error_message -FROM csv_rejects_table where line=2176 and column_idx=1; ----- -:.*Could not convert string "B" to 'INTEGER'.* - -query I -SELECT error_message -FROM csv_rejects_table where line=4176 and column_idx=1; ----- -:.*Could not convert string "C" to 'INTEGER'.* +statement ok +DROP TABLE reject_errors; statement ok -DROP TABLE csv_rejects_table; +DROP TABLE reject_scans; # Test with multiple rejects table in the same query query IIII rowsort @@ -245,61 +168,36 @@ SELECT * FROM read_csv( 'test/sql/copy/csv/data/error/mismatch/small1.csv', columns = {'num': 'INTEGER', 'str': 'VARCHAR'}, - rejects_table='csv_rejects_table_left', - ignore_errors=true) as L + store_rejects = true) as L JOIN read_csv( 'test/sql/copy/csv/data/error/mismatch/small2.csv', columns = {'num': 'INTEGER', 'str': 'VARCHAR'}, - rejects_table='csv_rejects_table_right', - ignore_errors=true) as R + store_rejects = true) as R ON L.num = R.num; ---- 1 A 1 A 3 C 3 C -query IIIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position -FROM csv_rejects_table_left; ----- -test/sql/copy/csv/data/error/mismatch/small1.csv 3 1 "num" CAST X,Y 14 -test/sql/copy/csv/data/error/mismatch/small1.csv 6 1 "num" CAST X,Y 26 - -query IIIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position -FROM csv_rejects_table_right; ----- -test/sql/copy/csv/data/error/mismatch/small2.csv 3 1 "num" CAST X,Y 14 -test/sql/copy/csv/data/error/mismatch/small2.csv 5 1 "num" CAST X,Y 22 - -query I -SELECT error_message -FROM csv_rejects_table_left where line=3 and column_idx=1; ----- -:.*Could not convert string "X" to 'INTEGER'.* - -query I -SELECT error_message -FROM csv_rejects_table_left where line=6 and column_idx=1; +query IIIIIIIIIIIII +FROM reject_scans ORDER BY ALL; ---- -:.*Could not convert string "X" to 'INTEGER'.* +31 0 test/sql/copy/csv/data/error/mismatch/small1.csv , " " \n 0 true {'num': 'INTEGER','str': 'VARCHAR'} NULL NULL store_rejects=true +31 1 test/sql/copy/csv/data/error/mismatch/small2.csv , " " \n 0 true {'num': 'INTEGER','str': 'VARCHAR'} NULL NULL store_rejects=true -query I -SELECT error_message -FROM csv_rejects_table_right where line=3 and column_idx=1; ----- -:.*Could not convert string "X" to 'INTEGER'.* -query I -SELECT error_message -FROM csv_rejects_table_right where line=5 and column_idx=1; +query IIIIIIIIII +FROM reject_errors ORDER BY ALL; ---- -:.*Could not convert string "X" to 'INTEGER'.* +31 0 3 15 15 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER' +31 0 6 27 27 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER' +31 1 3 15 15 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER' +31 1 5 23 23 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER' statement ok -DROP TABLE csv_rejects_table_left; +DROP TABLE reject_errors; statement ok -DROP TABLE csv_rejects_table_right; +DROP TABLE reject_scans; # Test with multiple rejects table in the same query, with different limits # (only one reject should be logged in right table) @@ -308,36 +206,28 @@ SELECT * FROM read_csv( 'test/sql/copy/csv/data/error/mismatch/small1.csv', columns = {'num': 'INTEGER', 'str': 'VARCHAR'}, - rejects_table='csv_rejects_table_left', - ignore_errors=true) as L + store_rejects = true) as L JOIN read_csv( 'test/sql/copy/csv/data/error/mismatch/small2.csv', columns = {'num': 'INTEGER', 'str': 'VARCHAR'}, - rejects_table='csv_rejects_table_right', - rejects_limit=1, - ignore_errors=true) as R + store_rejects = true, rejects_limit=1) as R ON L.num = R.num; ---- 1 A 1 A 3 C 3 C -query IIIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position -FROM csv_rejects_table_left; +query IIIIIIIIII +FROM reject_errors ORDER BY ALL; ---- -test/sql/copy/csv/data/error/mismatch/small1.csv 3 1 "num" CAST X,Y 14 -test/sql/copy/csv/data/error/mismatch/small1.csv 6 1 "num" CAST X,Y 26 +36 0 3 15 15 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER' +36 0 6 27 27 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER' +36 1 3 15 15 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER' -query I -SELECT COUNT(*) -FROM csv_rejects_table_right; ----- -1 statement ok -DROP TABLE csv_rejects_table_left; +DROP TABLE reject_errors; statement ok -DROP TABLE csv_rejects_table_right; +DROP TABLE reject_scans; From 3edaf0bf9e05af90577c7f123816256d0edf213e Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 4 Apr 2024 12:32:51 +0200 Subject: [PATCH 149/603] WIP - revert modifier_sql_types --- src/include/duckdb/planner/binder.hpp | 3 +- .../expression_binder/order_binder.hpp | 3 +- .../binder/query_node/bind_select_node.cpp | 36 +++---------------- .../expression_binder/order_binder.cpp | 22 ++++++------ 4 files changed, 19 insertions(+), 45 deletions(-) diff --git a/src/include/duckdb/planner/binder.hpp b/src/include/duckdb/planner/binder.hpp index d778bedfdc70..3dfee62f1ab9 100644 --- a/src/include/duckdb/planner/binder.hpp +++ b/src/include/duckdb/planner/binder.hpp @@ -325,8 +325,7 @@ class Binder : public std::enable_shared_from_this { BoundStatement BindCopyFrom(CopyStatement &stmt); void BindModifiers(OrderBinder &order_binder, QueryNode &statement, BoundQueryNode &result); - void BindModifierTypes(BoundQueryNode &result, const vector &sql_types, idx_t projection_index, - const vector &expansion_count = {}); + void BindModifierTypes(BoundQueryNode &result, const vector &sql_types, idx_t projection_index); unique_ptr BindLimit(OrderBinder &order_binder, LimitModifier &limit_mod); unique_ptr BindLimitPercent(OrderBinder &order_binder, LimitPercentModifier &limit_mod); diff --git a/src/include/duckdb/planner/expression_binder/order_binder.hpp b/src/include/duckdb/planner/expression_binder/order_binder.hpp index 3beed2c9506d..4868074d483a 100644 --- a/src/include/duckdb/planner/expression_binder/order_binder.hpp +++ b/src/include/duckdb/planner/expression_binder/order_binder.hpp @@ -42,8 +42,7 @@ class OrderBinder { unique_ptr CreateExtraReference(unique_ptr expr); private: - unique_ptr CreateProjectionReference(ParsedExpression &expr, const idx_t index, - const LogicalType &logical_type); + unique_ptr CreateProjectionReference(ParsedExpression &expr, const idx_t index); unique_ptr BindConstant(ParsedExpression &expr, const Value &val); private: diff --git a/src/planner/binder/query_node/bind_select_node.cpp b/src/planner/binder/query_node/bind_select_node.cpp index 568237abbda1..737e6ad020b7 100644 --- a/src/planner/binder/query_node/bind_select_node.cpp +++ b/src/planner/binder/query_node/bind_select_node.cpp @@ -244,8 +244,7 @@ static void AssignReturnType(unique_ptr &expr, const vector &sql_types, idx_t, - const vector &expansion_count) { +void Binder::BindModifierTypes(BoundQueryNode &result, const vector &sql_types, idx_t) { for (auto &bound_mod : result.modifiers) { switch (bound_mod->type) { case ResultModifierType::DISTINCT_MODIFIER: { @@ -258,13 +257,6 @@ void Binder::BindModifierTypes(BoundQueryNode &result, const vector if (bound_colref.binding.column_index == DConstants::INVALID_INDEX) { throw BinderException("Ambiguous name in DISTINCT ON!"); } - - idx_t max_count = sql_types.size(); - if (bound_colref.binding.column_index > max_count - 1) { - D_ASSERT(bound_colref.return_type == LogicalType::ANY); - throw BinderException("ORDER term out of range - should be between 1 and %lld", max_count); - } - bound_colref.return_type = sql_types[bound_colref.binding.column_index]; } for (auto &target_distinct : distinct.target_distincts) { @@ -281,7 +273,6 @@ void Binder::BindModifierTypes(BoundQueryNode &result, const vector break; } case ResultModifierType::ORDER_MODIFIER: { - auto &order = bound_mod->Cast(); for (auto &order_node : order.orders) { @@ -291,17 +282,6 @@ void Binder::BindModifierTypes(BoundQueryNode &result, const vector if (bound_colref.binding.column_index == DConstants::INVALID_INDEX) { throw BinderException("Ambiguous name in ORDER BY!"); } - - if (!expansion_count.empty() && bound_colref.return_type.id() != LogicalTypeId::ANY) { - bound_colref.binding.column_index = expansion_count[bound_colref.binding.column_index]; - } - - idx_t max_count = sql_types.size(); - if (bound_colref.binding.column_index > max_count - 1) { - D_ASSERT(bound_colref.return_type == LogicalType::ANY); - throw BinderException("ORDER term out of range - should be between 1 and %lld", max_count); - } - const auto &sql_type = sql_types[bound_colref.binding.column_index]; bound_colref.return_type = sql_type; @@ -492,11 +472,9 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ // if we expand select-list expressions, e.g., via UNNEST, then we need to possibly // adjust the column index of the already bound ORDER BY modifiers, and not only set their types - vector modifier_sql_types; - vector modifier_expansion_count; - vector group_by_all_indexes; vector new_names; + vector internal_sql_types; for (idx_t i = 0; i < statement.select_list.size(); i++) { bool is_window = statement.select_list[i]->IsWindow(); @@ -518,12 +496,11 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ auto &struct_expressions = select_binder.ExpandedExpressions(); D_ASSERT(!struct_expressions.empty()); - modifier_expansion_count.push_back(modifier_sql_types.size()); for (auto &struct_expr : struct_expressions) { - modifier_sql_types.push_back(struct_expr->return_type); new_names.push_back(struct_expr->GetName()); result->types.push_back(struct_expr->return_type); + internal_sql_types.push_back(struct_expr->return_type); result->select_list.push_back(std::move(struct_expr)); } @@ -531,10 +508,6 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ continue; } - // not an expanded expression - modifier_expansion_count.push_back(modifier_sql_types.size()); - modifier_sql_types.push_back(result_type); - if (can_group_by_all && select_binder.HasBoundColumns()) { if (select_binder.BoundAggregates()) { throw BinderException("Cannot mix aggregates with non-aggregated columns!"); @@ -555,6 +528,7 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ new_names.push_back(std::move(result->names[i])); result->types.push_back(result_type); } + internal_sql_types.push_back(result_type); if (can_group_by_all) { select_binder.ResetBindings(); @@ -617,7 +591,7 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ } // now that the SELECT list is bound, we set the types of DISTINCT/ORDER BY expressions - BindModifierTypes(*result, modifier_sql_types, result->projection_index, modifier_expansion_count); + BindModifierTypes(*result, internal_sql_types, result->projection_index); return std::move(result); } diff --git a/src/planner/expression_binder/order_binder.cpp b/src/planner/expression_binder/order_binder.cpp index 1173727182d3..c5fde60f0a4b 100644 --- a/src/planner/expression_binder/order_binder.cpp +++ b/src/planner/expression_binder/order_binder.cpp @@ -26,8 +26,7 @@ OrderBinder::OrderBinder(vector binders, idx_t projection_index, Selec this->extra_list = &node.select_list; } -unique_ptr OrderBinder::CreateProjectionReference(ParsedExpression &expr, const idx_t index, - const LogicalType &logical_type) { +unique_ptr OrderBinder::CreateProjectionReference(ParsedExpression &expr, const idx_t index) { string alias; if (extra_list && index < extra_list->size()) { alias = extra_list->at(index)->ToString(); @@ -36,7 +35,7 @@ unique_ptr OrderBinder::CreateProjectionReference(ParsedExpression & alias = expr.alias; } } - return make_uniq(std::move(alias), logical_type, ColumnBinding(projection_index, index)); + return make_uniq(std::move(alias), LogicalType::INVALID, ColumnBinding(projection_index, index)); } unique_ptr OrderBinder::CreateExtraReference(unique_ptr expr) { @@ -44,7 +43,7 @@ unique_ptr OrderBinder::CreateExtraReference(unique_ptrsize(); - auto result = CreateProjectionReference(*expr, extra_list->size(), LogicalType::INVALID); + auto result = CreateProjectionReference(*expr, extra_list->size()); extra_list->push_back(std::move(expr)); return result; } @@ -59,7 +58,10 @@ unique_ptr OrderBinder::BindConstant(ParsedExpression &expr, const V } // INTEGER constant: we use the integer as an index into the select list (e.g. ORDER BY 1) auto index = (idx_t)val.GetValue(); - return CreateProjectionReference(expr, index - 1, LogicalType::ANY); + if (index < 1 || index > max_count) { + throw BinderException("ORDER term out of range - should be between 1 and %lld", max_count); + } + return CreateProjectionReference(expr, index - 1); } unique_ptr OrderBinder::Bind(unique_ptr expr) { @@ -87,16 +89,16 @@ unique_ptr OrderBinder::Bind(unique_ptr expr) { auto entry = alias_map.find(colref.column_names[0]); if (entry != alias_map.end()) { // it does! point it to that entry - return CreateProjectionReference(*expr, entry->second, LogicalType::INVALID); + return CreateProjectionReference(*expr, entry->second); } break; } case ExpressionClass::POSITIONAL_REFERENCE: { auto &posref = expr->Cast(); if (posref.index < 1 || posref.index > max_count) { - throw BinderException("ORDER term out of range - should be between 1 and %lld", (idx_t)max_count); + throw BinderException("ORDER term out of range - should be between 1 and %lld", max_count); } - return CreateProjectionReference(*expr, posref.index - 1, LogicalType::ANY); + return CreateProjectionReference(*expr, posref.index - 1); } case ExpressionClass::PARAMETER: { throw ParameterNotAllowedException("Parameter not supported in ORDER BY clause"); @@ -107,7 +109,7 @@ unique_ptr OrderBinder::Bind(unique_ptr expr) { auto &constant = collation.child->Cast(); auto index = NumericCast(constant.value.GetValue()) - 1; if (index >= extra_list->size()) { - throw BinderException("ORDER term out of range - should be between 1 and %lld", (idx_t)max_count); + throw BinderException("ORDER term out of range - should be between 1 and %lld", max_count); } auto &sel_entry = extra_list->at(index); if (sel_entry->HasSubquery()) { @@ -135,7 +137,7 @@ unique_ptr OrderBinder::Bind(unique_ptr expr) { } // there is a matching entry in the projection list // just point to that entry - return CreateProjectionReference(*expr, entry->second, LogicalType::INVALID); + return CreateProjectionReference(*expr, entry->second); } if (!extra_list) { // no extra list specified: we cannot push an extra ORDER BY clause From ae853ab2467374f1ac2508733f7f2bd5b1fbd41f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Thu, 4 Apr 2024 12:49:27 +0200 Subject: [PATCH 150/603] moaaaar --- CMakeLists.txt | 1 - src/catalog/duck_catalog.cpp | 13 ++++++++++++ src/common/operator/cast_operators.cpp | 13 ++++++++---- src/common/operator/string_cast.cpp | 2 +- src/common/row_operations/row_aggregate.cpp | 20 +++++++++---------- src/common/row_operations/row_external.cpp | 7 ++++--- .../serializer/buffered_file_reader.cpp | 8 ++++---- .../serializer/buffered_file_writer.cpp | 17 ++++++++-------- src/common/sort/comparators.cpp | 2 +- src/common/sort/merge_sorter.cpp | 4 ++-- src/common/sort/partition_state.cpp | 12 +++++------ src/common/sort/sort_state.cpp | 6 +++--- src/common/sort/sorted_block.cpp | 10 +++++----- src/common/types/bit.cpp | 2 +- src/common/types/blob.cpp | 6 +++--- src/common/types/cast_helpers.cpp | 2 +- src/common/types/conflict_manager.cpp | 2 +- src/common/types/date.cpp | 4 ++-- src/common/types/decimal.cpp | 12 +++++------ src/common/types/hash.cpp | 2 +- src/common/types/hugeint.cpp | 4 ++-- src/function/table/system/duckdb_tables.cpp | 5 +++-- .../common/operator/integer_cast_operator.hpp | 9 +++++---- .../duckdb/common/sort/duckdb_pdqsort.hpp | 6 +++--- .../duckdb/common/types/cast_helpers.hpp | 16 ++++++++------- .../duckdb/common/types/validity_mask.hpp | 2 +- .../rule/ordered_aggregate_optimizer.cpp | 1 + 27 files changed, 106 insertions(+), 82 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index aebc060c3bd7..2d50d9c21828 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -553,7 +553,6 @@ include_directories(third_party/fast_float) include_directories(third_party/re2) include_directories(third_party/miniz) include_directories(third_party/utf8proc/include) -include_directories(third_party/miniparquet) include_directories(third_party/concurrentqueue) include_directories(third_party/pcg) include_directories(third_party/tdigest) diff --git a/src/catalog/duck_catalog.cpp b/src/catalog/duck_catalog.cpp index 56717e90f09f..ebf1bb53b72c 100644 --- a/src/catalog/duck_catalog.cpp +++ b/src/catalog/duck_catalog.cpp @@ -1,3 +1,16 @@ +#include "duckdb/catalog/duck_catalog.hpp" +#include "duckdb/catalog/dependency_manager.hpp" +#include "duckdb/catalog/catalog_entry/duck_schema_entry.hpp" +#include "duckdb/storage/storage_manager.hpp" +#include "duckdb/parser/parsed_data/drop_info.hpp" +#include "duckdb/parser/parsed_data/create_schema_info.hpp" +#include "duckdb/catalog/default/default_schemas.hpp" +#include "duckdb/function/built_in_functions.hpp" +#include "duckdb/main/attached_database.hpp" +#ifndef DISABLE_CORE_FUNCTIONS_EXTENSION +#include "duckdb/core_functions/core_functions.hpp" +#endif + namespace duckdb { DuckCatalog::DuckCatalog(AttachedDatabase &db) diff --git a/src/common/operator/cast_operators.cpp b/src/common/operator/cast_operators.cpp index c63adefcab7f..be59aabbe296 100644 --- a/src/common/operator/cast_operators.cpp +++ b/src/common/operator/cast_operators.cpp @@ -1593,12 +1593,12 @@ struct HugeIntCastData { using ResultType = T; using Operation = OP; ResultType result; - int64_t intermediate; + ResultType intermediate; uint8_t digits; ResultType decimal; uint16_t decimal_total_digits; - int64_t decimal_intermediate; + ResultType decimal_intermediate; uint16_t decimal_intermediate_digits; bool Flush() { @@ -1647,7 +1647,8 @@ struct HugeIntegerCastOperation { template static bool HandleDigit(T &state, uint8_t digit) { if (NEGATIVE) { - if (DUCKDB_UNLIKELY(state.intermediate < (NumericLimits::Minimum() + digit) / 10)) { + if (DUCKDB_UNLIKELY(UnsafeNumericCast(state.intermediate) < + (NumericLimits::Minimum() + digit) / 10)) { // intermediate is full: need to flush it if (!state.Flush()) { return false; @@ -1694,7 +1695,11 @@ struct HugeIntegerCastOperation { if (e < 0) { state.result = T::Operation::DivMod(state.result, T::Operation::POWERS_OF_TEN[-e], remainder); if (remainder < 0) { - remainder *= -1; + result_t negate_result; + if (!T::Operation::TryNegate(remainder, negate_result)) { + return false; + } + remainder = negate_result; } state.decimal = remainder; state.decimal_total_digits = UnsafeNumericCast(-e); diff --git a/src/common/operator/string_cast.cpp b/src/common/operator/string_cast.cpp index 2da6949abe50..0fd1f5f0a0ca 100644 --- a/src/common/operator/string_cast.cpp +++ b/src/common/operator/string_cast.cpp @@ -181,7 +181,7 @@ string_t StringCastTZ::Operation(dtime_tz_t input, Vector &vector) { auto ss = std::abs(offset); const auto hh = ss / Interval::SECS_PER_HOUR; - const auto hh_length = (hh < 100) ? 2 : NumericHelper::UnsignedLength(uint32_t(hh)); + const auto hh_length = UnsafeNumericCast((hh < 100) ? 2 : NumericHelper::UnsignedLength(uint32_t(hh))); length += hh_length; ss %= Interval::SECS_PER_HOUR; diff --git a/src/common/row_operations/row_aggregate.cpp b/src/common/row_operations/row_aggregate.cpp index f6e9e6cbb374..8be0ada4fa20 100644 --- a/src/common/row_operations/row_aggregate.cpp +++ b/src/common/row_operations/row_aggregate.cpp @@ -36,14 +36,14 @@ void RowOperations::DestroyStates(RowOperationsState &state, TupleDataLayout &la return; } // Move to the first aggregate state - VectorOperations::AddInPlace(addresses, layout.GetAggrOffset(), count); + VectorOperations::AddInPlace(addresses, UnsafeNumericCast(layout.GetAggrOffset()), count); for (const auto &aggr : layout.GetAggregates()) { if (aggr.function.destructor) { AggregateInputData aggr_input_data(aggr.GetFunctionData(), state.allocator); aggr.function.destructor(addresses, aggr_input_data, count); } // Move to the next aggregate state - VectorOperations::AddInPlace(addresses, aggr.payload_size, count); + VectorOperations::AddInPlace(addresses, UnsafeNumericCast(aggr.payload_size), count); } } @@ -74,8 +74,8 @@ void RowOperations::CombineStates(RowOperationsState &state, TupleDataLayout &la } // Move to the first aggregate states - VectorOperations::AddInPlace(sources, layout.GetAggrOffset(), count); - VectorOperations::AddInPlace(targets, layout.GetAggrOffset(), count); + VectorOperations::AddInPlace(sources, UnsafeNumericCast(layout.GetAggrOffset()), count); + VectorOperations::AddInPlace(targets, UnsafeNumericCast(layout.GetAggrOffset()), count); // Keep track of the offset idx_t offset = layout.GetAggrOffset(); @@ -87,16 +87,16 @@ void RowOperations::CombineStates(RowOperationsState &state, TupleDataLayout &la aggr.function.combine(sources, targets, aggr_input_data, count); // Move to the next aggregate states - VectorOperations::AddInPlace(sources, aggr.payload_size, count); - VectorOperations::AddInPlace(targets, aggr.payload_size, count); + VectorOperations::AddInPlace(sources, UnsafeNumericCast(aggr.payload_size), count); + VectorOperations::AddInPlace(targets, UnsafeNumericCast(aggr.payload_size), count); // Increment the offset offset += aggr.payload_size; } // Now subtract the offset to get back to the original position - VectorOperations::AddInPlace(sources, -offset, count); - VectorOperations::AddInPlace(targets, -offset, count); + VectorOperations::AddInPlace(sources, UnsafeNumericCast(-offset), count); + VectorOperations::AddInPlace(targets, UnsafeNumericCast(-offset), count); } void RowOperations::FinalizeStates(RowOperationsState &state, TupleDataLayout &layout, Vector &addresses, @@ -106,7 +106,7 @@ void RowOperations::FinalizeStates(RowOperationsState &state, TupleDataLayout &l VectorOperations::Copy(addresses, addresses_copy, result.size(), 0, 0); // Move to the first aggregate state - VectorOperations::AddInPlace(addresses_copy, layout.GetAggrOffset(), result.size()); + VectorOperations::AddInPlace(addresses_copy, UnsafeNumericCast(layout.GetAggrOffset()), result.size()); auto &aggregates = layout.GetAggregates(); for (idx_t i = 0; i < aggregates.size(); i++) { @@ -116,7 +116,7 @@ void RowOperations::FinalizeStates(RowOperationsState &state, TupleDataLayout &l aggr.function.finalize(addresses_copy, aggr_input_data, target, result.size(), 0); // Move to the next aggregate state - VectorOperations::AddInPlace(addresses_copy, aggr.payload_size, result.size()); + VectorOperations::AddInPlace(addresses_copy, UnsafeNumericCast(aggr.payload_size), result.size()); } } diff --git a/src/common/row_operations/row_external.cpp b/src/common/row_operations/row_external.cpp index 9e2fa071f86d..5aef76a00dd7 100644 --- a/src/common/row_operations/row_external.cpp +++ b/src/common/row_operations/row_external.cpp @@ -37,7 +37,8 @@ void RowOperations::SwizzleColumns(const RowLayout &layout, const data_ptr_t bas for (idx_t i = 0; i < next; i++) { if (Load(col_ptr) > string_t::INLINE_LENGTH) { // Overwrite the string pointer with the within-row offset (if not inlined) - Store(Load(string_ptr) - heap_row_ptrs[i], string_ptr); + Store(UnsafeNumericCast(Load(string_ptr) - heap_row_ptrs[i]), + string_ptr); } col_ptr += row_width; string_ptr += row_width; @@ -46,7 +47,7 @@ void RowOperations::SwizzleColumns(const RowLayout &layout, const data_ptr_t bas // Non-varchar blob columns for (idx_t i = 0; i < next; i++) { // Overwrite the column data pointer with the within-row offset - Store(Load(col_ptr) - heap_row_ptrs[i], col_ptr); + Store(UnsafeNumericCast(Load(col_ptr) - heap_row_ptrs[i]), col_ptr); col_ptr += row_width; } } @@ -79,7 +80,7 @@ void RowOperations::CopyHeapAndSwizzle(const RowLayout &layout, data_ptr_t row_p // Copy and swizzle memcpy(heap_ptr, source_heap_ptr, size); - Store(heap_ptr - heap_base_ptr, row_ptr + heap_offset); + Store(UnsafeNumericCast(heap_ptr - heap_base_ptr), row_ptr + heap_offset); // Increment for next iteration row_ptr += row_width; diff --git a/src/common/serializer/buffered_file_reader.cpp b/src/common/serializer/buffered_file_reader.cpp index 762e1ca889a0..a6ed87e9f4bc 100644 --- a/src/common/serializer/buffered_file_reader.cpp +++ b/src/common/serializer/buffered_file_reader.cpp @@ -11,20 +11,20 @@ BufferedFileReader::BufferedFileReader(FileSystem &fs, const char *path, FileLoc optional_ptr opener) : fs(fs), data(make_unsafe_uniq_array(FILE_BUFFER_SIZE)), offset(0), read_data(0), total_read(0) { handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ | lock_type, opener.get()); - file_size = fs.GetFileSize(*handle); + file_size = NumericCast(fs.GetFileSize(*handle)); } BufferedFileReader::BufferedFileReader(FileSystem &fs, unique_ptr handle_p) : fs(fs), data(make_unsafe_uniq_array(FILE_BUFFER_SIZE)), offset(0), read_data(0), handle(std::move(handle_p)), total_read(0) { - file_size = fs.GetFileSize(*handle); + file_size = NumericCast(fs.GetFileSize(*handle)); } void BufferedFileReader::ReadData(data_ptr_t target_buffer, uint64_t read_size) { // first copy anything we can from the buffer data_ptr_t end_ptr = target_buffer + read_size; while (true) { - idx_t to_read = MinValue(end_ptr - target_buffer, read_data - offset); + idx_t to_read = MinValue(UnsafeNumericCast(end_ptr - target_buffer), read_data - offset); if (to_read > 0) { memcpy(target_buffer, data.get() + offset, to_read); offset += to_read; @@ -36,7 +36,7 @@ void BufferedFileReader::ReadData(data_ptr_t target_buffer, uint64_t read_size) // did not finish reading yet but exhausted buffer // read data into buffer offset = 0; - read_data = fs.Read(*handle, data.get(), FILE_BUFFER_SIZE); + read_data = UnsafeNumericCast(fs.Read(*handle, data.get(), FILE_BUFFER_SIZE)); if (read_data == 0) { throw SerializationException("not enough data in file to deserialize result"); } diff --git a/src/common/serializer/buffered_file_writer.cpp b/src/common/serializer/buffered_file_writer.cpp index dcbe7d4f0a45..62d237e63ea3 100644 --- a/src/common/serializer/buffered_file_writer.cpp +++ b/src/common/serializer/buffered_file_writer.cpp @@ -15,7 +15,7 @@ BufferedFileWriter::BufferedFileWriter(FileSystem &fs, const string &path_p, Fil } int64_t BufferedFileWriter::GetFileSize() { - return fs.GetFileSize(*handle) + offset; + return fs.GetFileSize(*handle) + NumericCast(offset); } idx_t BufferedFileWriter::GetTotalWritten() { @@ -37,13 +37,14 @@ void BufferedFileWriter::WriteData(const_data_ptr_t buffer, idx_t write_size) { Flush(); // Flush buffer before writing every things else } idx_t remaining_to_write = write_size - to_copy; - fs.Write(*handle, const_cast(buffer + to_copy), remaining_to_write); // NOLINT: wrong API in Write + fs.Write(*handle, const_cast(buffer + to_copy), + UnsafeNumericCast(remaining_to_write)); // NOLINT: wrong API in Write total_written += remaining_to_write; } else { // first copy anything we can from the buffer const_data_ptr_t end_ptr = buffer + write_size; while (buffer < end_ptr) { - idx_t to_write = MinValue((end_ptr - buffer), FILE_BUFFER_SIZE - offset); + idx_t to_write = MinValue(UnsafeNumericCast((end_ptr - buffer)), FILE_BUFFER_SIZE - offset); D_ASSERT(to_write > 0); memcpy(data.get() + offset, buffer, to_write); offset += to_write; @@ -59,7 +60,7 @@ void BufferedFileWriter::Flush() { if (offset == 0) { return; } - fs.Write(*handle, data.get(), offset); + fs.Write(*handle, data.get(), UnsafeNumericCast(offset)); total_written += offset; offset = 0; } @@ -70,11 +71,11 @@ void BufferedFileWriter::Sync() { } void BufferedFileWriter::Truncate(int64_t size) { - uint64_t persistent = fs.GetFileSize(*handle); - D_ASSERT((uint64_t)size <= persistent + offset); - if (persistent <= (uint64_t)size) { + auto persistent = fs.GetFileSize(*handle); + D_ASSERT(size <= persistent + NumericCast(offset)); + if (persistent <= size) { // truncating into the pending write buffer. - offset = size - persistent; + offset = NumericCast(size - persistent); } else { // truncate the physical file on disk handle->Truncate(size); diff --git a/src/common/sort/comparators.cpp b/src/common/sort/comparators.cpp index 7084a3f993a3..82e8069d9211 100644 --- a/src/common/sort/comparators.cpp +++ b/src/common/sort/comparators.cpp @@ -501,7 +501,7 @@ void Comparators::SwizzleSingleValue(data_ptr_t data_ptr, const data_ptr_t &heap if (type.InternalType() == PhysicalType::VARCHAR) { data_ptr += string_t::HEADER_SIZE; } - Store(Load(data_ptr) - heap_ptr, data_ptr); + Store(UnsafeNumericCast(Load(data_ptr) - heap_ptr), data_ptr); } } // namespace duckdb diff --git a/src/common/sort/merge_sorter.cpp b/src/common/sort/merge_sorter.cpp index 7d2f6a1dc4e1..b36887e66a06 100644 --- a/src/common/sort/merge_sorter.cpp +++ b/src/common/sort/merge_sorter.cpp @@ -516,9 +516,9 @@ void MergeSorter::MergeData(SortedData &result_data, SortedData &l_data, SortedD entry_size = l_smaller * Load(l_heap_ptr_copy) + r_smaller * Load(r_heap_ptr_copy); D_ASSERT(entry_size >= sizeof(uint32_t)); - D_ASSERT(l_heap_ptr_copy - l.BaseHeapPtr(l_data) + l_smaller * entry_size <= + D_ASSERT(NumericCast(l_heap_ptr_copy - l.BaseHeapPtr(l_data)) + l_smaller * entry_size <= l_data.heap_blocks[l.block_idx]->byte_offset); - D_ASSERT(r_heap_ptr_copy - r.BaseHeapPtr(r_data) + r_smaller * entry_size <= + D_ASSERT(NumericCast(r_heap_ptr_copy - r.BaseHeapPtr(r_data)) + r_smaller * entry_size <= r_data.heap_blocks[r.block_idx]->byte_offset); l_heap_ptr_copy += l_smaller * entry_size; r_heap_ptr_copy += r_smaller * entry_size; diff --git a/src/common/sort/partition_state.cpp b/src/common/sort/partition_state.cpp index c123f275556f..5346d057a381 100644 --- a/src/common/sort/partition_state.cpp +++ b/src/common/sort/partition_state.cpp @@ -318,7 +318,7 @@ void PartitionLocalSinkState::Sink(DataChunk &input_chunk) { const auto entry_size = payload_layout.GetRowWidth(); const auto capacity = MaxValue(STANDARD_VECTOR_SIZE, (Storage::BLOCK_SIZE / entry_size) + 1); rows = make_uniq(gstate.buffer_manager, capacity, entry_size); - strings = make_uniq(gstate.buffer_manager, (idx_t)Storage::BLOCK_SIZE, 1, true); + strings = make_uniq(gstate.buffer_manager, (idx_t)Storage::BLOCK_SIZE, 1U, true); } const auto row_count = input_chunk.size(); const auto row_sel = FlatVector::IncrementalSelectionVector(); @@ -402,8 +402,8 @@ void PartitionLocalSinkState::Combine() { PartitionGlobalMergeState::PartitionGlobalMergeState(PartitionGlobalSinkState &sink, GroupDataPtr group_data_p, hash_t hash_bin) : sink(sink), group_data(std::move(group_data_p)), memory_per_thread(sink.memory_per_thread), - num_threads(TaskScheduler::GetScheduler(sink.context).NumberOfThreads()), stage(PartitionSortStage::INIT), - total_tasks(0), tasks_assigned(0), tasks_completed(0) { + num_threads(NumericCast(TaskScheduler::GetScheduler(sink.context).NumberOfThreads())), + stage(PartitionSortStage::INIT), total_tasks(0), tasks_assigned(0), tasks_completed(0) { const auto group_idx = sink.hash_groups.size(); auto new_group = make_uniq(sink.buffer_manager, sink.partitions, sink.orders, @@ -424,8 +424,8 @@ PartitionGlobalMergeState::PartitionGlobalMergeState(PartitionGlobalSinkState &s PartitionGlobalMergeState::PartitionGlobalMergeState(PartitionGlobalSinkState &sink) : sink(sink), memory_per_thread(sink.memory_per_thread), - num_threads(TaskScheduler::GetScheduler(sink.context).NumberOfThreads()), stage(PartitionSortStage::INIT), - total_tasks(0), tasks_assigned(0), tasks_completed(0) { + num_threads(NumericCast(TaskScheduler::GetScheduler(sink.context).NumberOfThreads())), + stage(PartitionSortStage::INIT), total_tasks(0), tasks_assigned(0), tasks_completed(0) { const hash_t hash_bin = 0; const size_t group_idx = 0; @@ -661,7 +661,7 @@ void PartitionMergeEvent::Schedule() { // Schedule tasks equal to the number of threads, which will each merge multiple partitions auto &ts = TaskScheduler::GetScheduler(context); - idx_t num_threads = ts.NumberOfThreads(); + auto num_threads = NumericCast(ts.NumberOfThreads()); vector> merge_tasks; for (idx_t tnum = 0; tnum < num_threads; tnum++) { diff --git a/src/common/sort/sort_state.cpp b/src/common/sort/sort_state.cpp index 9c8594866e8a..27650b46e76b 100644 --- a/src/common/sort/sort_state.cpp +++ b/src/common/sort/sort_state.cpp @@ -171,13 +171,13 @@ void LocalSortState::Initialize(GlobalSortState &global_sort_state, BufferManage auto blob_row_width = sort_layout->blob_layout.GetRowWidth(); blob_sorting_data = make_uniq( *buffer_manager, RowDataCollection::EntriesPerBlock(blob_row_width), blob_row_width); - blob_sorting_heap = make_uniq(*buffer_manager, (idx_t)Storage::BLOCK_SIZE, 1, true); + blob_sorting_heap = make_uniq(*buffer_manager, (idx_t)Storage::BLOCK_SIZE, 1U, true); } // Payload data auto payload_row_width = payload_layout->GetRowWidth(); payload_data = make_uniq(*buffer_manager, RowDataCollection::EntriesPerBlock(payload_row_width), payload_row_width); - payload_heap = make_uniq(*buffer_manager, (idx_t)Storage::BLOCK_SIZE, 1, true); + payload_heap = make_uniq(*buffer_manager, (idx_t)Storage::BLOCK_SIZE, 1U, true); // Init done initialized = true; } @@ -323,7 +323,7 @@ void LocalSortState::ReOrder(SortedData &sd, data_ptr_t sorting_ptr, RowDataColl std::accumulate(heap.blocks.begin(), heap.blocks.end(), (idx_t)0, [](idx_t a, const unique_ptr &b) { return a + b->byte_offset; }); idx_t heap_block_size = MaxValue(total_byte_offset, (idx_t)Storage::BLOCK_SIZE); - auto ordered_heap_block = make_uniq(MemoryTag::ORDER_BY, *buffer_manager, heap_block_size, 1); + auto ordered_heap_block = make_uniq(MemoryTag::ORDER_BY, *buffer_manager, heap_block_size, 1U); ordered_heap_block->count = count; ordered_heap_block->byte_offset = total_byte_offset; auto ordered_heap_handle = buffer_manager->Pin(ordered_heap_block->block); diff --git a/src/common/sort/sorted_block.cpp b/src/common/sort/sorted_block.cpp index 22127abe1032..9539302c3b4a 100644 --- a/src/common/sort/sorted_block.cpp +++ b/src/common/sort/sorted_block.cpp @@ -30,7 +30,7 @@ void SortedData::CreateBlock() { data_blocks.push_back(make_uniq(MemoryTag::ORDER_BY, buffer_manager, capacity, layout.GetRowWidth())); if (!layout.AllConstant() && state.external) { heap_blocks.push_back( - make_uniq(MemoryTag::ORDER_BY, buffer_manager, (idx_t)Storage::BLOCK_SIZE, 1)); + make_uniq(MemoryTag::ORDER_BY, buffer_manager, (idx_t)Storage::BLOCK_SIZE, 1U)); D_ASSERT(data_blocks.size() == heap_blocks.size()); } } @@ -291,10 +291,10 @@ PayloadScanner::PayloadScanner(SortedData &sorted_data, GlobalSortState &global_ auto &layout = sorted_data.layout; // Create collections to put the data into so we can use RowDataCollectionScanner - rows = make_uniq(global_sort_state.buffer_manager, (idx_t)Storage::BLOCK_SIZE, 1); + rows = make_uniq(global_sort_state.buffer_manager, (idx_t)Storage::BLOCK_SIZE, 1U); rows->count = count; - heap = make_uniq(global_sort_state.buffer_manager, (idx_t)Storage::BLOCK_SIZE, 1); + heap = make_uniq(global_sort_state.buffer_manager, (idx_t)Storage::BLOCK_SIZE, 1U); if (!sorted_data.layout.AllConstant()) { heap->count = count; } @@ -330,7 +330,7 @@ PayloadScanner::PayloadScanner(GlobalSortState &global_sort_state, idx_t block_i auto &layout = sorted_data.layout; // Create collections to put the data into so we can use RowDataCollectionScanner - rows = make_uniq(global_sort_state.buffer_manager, (idx_t)Storage::BLOCK_SIZE, 1); + rows = make_uniq(global_sort_state.buffer_manager, (idx_t)Storage::BLOCK_SIZE, 1U); if (flush_p) { rows->blocks.emplace_back(std::move(sorted_data.data_blocks[block_idx])); } else { @@ -338,7 +338,7 @@ PayloadScanner::PayloadScanner(GlobalSortState &global_sort_state, idx_t block_i } rows->count = count; - heap = make_uniq(global_sort_state.buffer_manager, (idx_t)Storage::BLOCK_SIZE, 1); + heap = make_uniq(global_sort_state.buffer_manager, (idx_t)Storage::BLOCK_SIZE, 1U); if (!sorted_data.layout.AllConstant() && sorted_data.swizzled) { if (flush_p) { heap->blocks.emplace_back(std::move(sorted_data.heap_blocks[block_idx])); diff --git a/src/common/types/bit.cpp b/src/common/types/bit.cpp index 6f2bd8a0d066..83f0ce8365df 100644 --- a/src/common/types/bit.cpp +++ b/src/common/types/bit.cpp @@ -179,7 +179,7 @@ void Bit::BitToBlob(string_t bit, string_t &output_blob) { auto output = output_blob.GetDataWriteable(); idx_t size = output_blob.GetSize(); - output[0] = GetFirstByte(bit); + output[0] = UnsafeNumericCast(GetFirstByte(bit)); if (size > 2) { ++output; // First byte in bitstring contains amount of padded bits, diff --git a/src/common/types/blob.cpp b/src/common/types/blob.cpp index 5cb8e5df606b..6f472c0e3b92 100644 --- a/src/common/types/blob.cpp +++ b/src/common/types/blob.cpp @@ -47,7 +47,7 @@ void Blob::ToString(string_t blob, char *output) { for (idx_t i = 0; i < len; i++) { if (IsRegularCharacter(data[i])) { // ascii characters are rendered as-is - output[str_idx++] = data[i]; + output[str_idx++] = UnsafeNumericCast(data[i]); } else { auto byte_a = data[i] >> 4; auto byte_b = data[i] & 0x0F; @@ -244,8 +244,8 @@ uint32_t DecodeBase64Bytes(const string_t &str, const_data_ptr_t input_data, idx input_data[base_idx + decode_idx], base_idx + decode_idx); } } - return (decoded_bytes[0] << 3 * 6) + (decoded_bytes[1] << 2 * 6) + (decoded_bytes[2] << 1 * 6) + - (decoded_bytes[3] << 0 * 6); + return UnsafeNumericCast((decoded_bytes[0] << 3 * 6) + (decoded_bytes[1] << 2 * 6) + + (decoded_bytes[2] << 1 * 6) + (decoded_bytes[3] << 0 * 6)); } void Blob::FromBase64(string_t str, data_ptr_t output, idx_t output_size) { diff --git a/src/common/types/cast_helpers.cpp b/src/common/types/cast_helpers.cpp index 1e37fbc79ea6..f37fbaa971e9 100644 --- a/src/common/types/cast_helpers.cpp +++ b/src/common/types/cast_helpers.cpp @@ -67,7 +67,7 @@ int NumericHelper::UnsignedLength(uint32_t value) { } template <> -idx_t NumericHelper::UnsignedLength(uint64_t value) { +int NumericHelper::UnsignedLength(uint64_t value) { if (value >= 10000000000ULL) { if (value >= 1000000000000000ULL) { int length = 16; diff --git a/src/common/types/conflict_manager.cpp b/src/common/types/conflict_manager.cpp index 38d61240dcca..171d45115fb4 100644 --- a/src/common/types/conflict_manager.cpp +++ b/src/common/types/conflict_manager.cpp @@ -159,7 +159,7 @@ bool ConflictManager::AddNull(idx_t chunk_index) { if (!IsConflict(LookupResultType::LOOKUP_NULL)) { return false; } - return AddHit(chunk_index, DConstants::INVALID_INDEX); + return AddHit(chunk_index, UnsafeNumericCast(DConstants::INVALID_INDEX)); } bool ConflictManager::SingleIndexTarget() const { diff --git a/src/common/types/date.cpp b/src/common/types/date.cpp index 1f06b884dd27..be3349035199 100644 --- a/src/common/types/date.cpp +++ b/src/common/types/date.cpp @@ -307,7 +307,7 @@ bool Date::TryConvertDate(const char *buf, idx_t len, idx_t &pos, date_t &result // in strict mode, check remaining string for non-space characters if (strict) { // skip trailing spaces - while (pos < len && StringUtil::CharacterIsSpace((unsigned char)buf[pos])) { + while (pos < len && StringUtil::CharacterIsSpace(buf[pos])) { pos++; } // check position. if end was not reached, non-space chars remaining @@ -316,7 +316,7 @@ bool Date::TryConvertDate(const char *buf, idx_t len, idx_t &pos, date_t &result } } else { // in non-strict mode, check for any direct trailing digits - if (pos < len && StringUtil::CharacterIsDigit((unsigned char)buf[pos])) { + if (pos < len && StringUtil::CharacterIsDigit(buf[pos])) { return false; } } diff --git a/src/common/types/decimal.cpp b/src/common/types/decimal.cpp index 323cec4e381d..2f38d76ea786 100644 --- a/src/common/types/decimal.cpp +++ b/src/common/types/decimal.cpp @@ -6,9 +6,9 @@ namespace duckdb { template string TemplatedDecimalToString(SIGNED value, uint8_t width, uint8_t scale) { auto len = DecimalToString::DecimalLength(value, width, scale); - auto data = make_unsafe_uniq_array(len + 1); - DecimalToString::FormatDecimal(value, width, scale, data.get(), len); - return string(data.get(), len); + auto data = make_unsafe_uniq_array(UnsafeNumericCast(len + 1)); + DecimalToString::FormatDecimal(value, width, scale, data.get(), UnsafeNumericCast(len)); + return string(data.get(), UnsafeNumericCast(len)); } string Decimal::ToString(int16_t value, uint8_t width, uint8_t scale) { @@ -25,9 +25,9 @@ string Decimal::ToString(int64_t value, uint8_t width, uint8_t scale) { string Decimal::ToString(hugeint_t value, uint8_t width, uint8_t scale) { auto len = HugeintToStringCast::DecimalLength(value, width, scale); - auto data = make_unsafe_uniq_array(len + 1); - HugeintToStringCast::FormatDecimal(value, width, scale, data.get(), len); - return string(data.get(), len); + auto data = make_unsafe_uniq_array(UnsafeNumericCast(len + 1)); + HugeintToStringCast::FormatDecimal(value, width, scale, data.get(), UnsafeNumericCast(len)); + return string(data.get(), UnsafeNumericCast(len)); } } // namespace duckdb diff --git a/src/common/types/hash.cpp b/src/common/types/hash.cpp index cdc9ba3020ec..83a1ef22310e 100644 --- a/src/common/types/hash.cpp +++ b/src/common/types/hash.cpp @@ -22,7 +22,7 @@ hash_t Hash(int64_t val) { template <> hash_t Hash(hugeint_t val) { - return MurmurHash64(val.lower) ^ MurmurHash64(val.upper); + return MurmurHash64(val.lower) ^ MurmurHash64(static_cast(val.upper)); } template <> diff --git a/src/common/types/hugeint.cpp b/src/common/types/hugeint.cpp index d5343fc6860d..bc4a13e0b98d 100644 --- a/src/common/types/hugeint.cpp +++ b/src/common/types/hugeint.cpp @@ -85,7 +85,7 @@ static uint8_t PositiveHugeintHighestBit(hugeint_t bits) { uint8_t out = 0; if (bits.upper) { out = 64; - uint64_t up = bits.upper; + uint64_t up = static_cast(bits.upper); while (up) { up >>= 1; out++; @@ -104,7 +104,7 @@ static bool PositiveHugeintIsBitSet(hugeint_t lhs, uint8_t bit_position) { if (bit_position < 64) { return lhs.lower & (uint64_t(1) << uint64_t(bit_position)); } else { - return lhs.upper & (uint64_t(1) << uint64_t(bit_position - 64)); + return static_cast(lhs.upper) & (uint64_t(1) << uint64_t(bit_position - 64)); } } diff --git a/src/function/table/system/duckdb_tables.cpp b/src/function/table/system/duckdb_tables.cpp index 95a0e79ad0f5..bcc0668cde01 100644 --- a/src/function/table/system/duckdb_tables.cpp +++ b/src/function/table/system/duckdb_tables.cpp @@ -146,8 +146,9 @@ void DuckDBTablesFunction(ClientContext &context, TableFunctionInput &data_p, Da output.SetValue(col++, count, Value::BOOLEAN(TableHasPrimaryKey(table))); // estimated_size, LogicalType::BIGINT - Value card_val = - !storage_info.cardinality.IsValid() ? Value() : Value::BIGINT(NumericCast(storage_info.cardinality.GetIndex())); + Value card_val = !storage_info.cardinality.IsValid() + ? Value() + : Value::BIGINT(NumericCast(storage_info.cardinality.GetIndex())); output.SetValue(col++, count, card_val); // column_count, LogicalType::BIGINT output.SetValue(col++, count, Value::BIGINT(NumericCast(table.GetColumns().LogicalColumnCount()))); diff --git a/src/include/duckdb/common/operator/integer_cast_operator.hpp b/src/include/duckdb/common/operator/integer_cast_operator.hpp index 0fdca6b15b16..decec991f7e6 100644 --- a/src/include/duckdb/common/operator/integer_cast_operator.hpp +++ b/src/include/duckdb/common/operator/integer_cast_operator.hpp @@ -234,7 +234,8 @@ static bool IntegerCastLoop(const char *buf, idx_t len, T &result, bool strict) if (!StringUtil::CharacterIsDigit(buf[pos])) { break; } - if (!OP::template HandleDecimal(result, buf[pos] - '0')) { + if (!OP::template HandleDecimal( + result, UnsafeNumericCast(buf[pos] - '0'))) { return false; } pos++; @@ -296,7 +297,7 @@ static bool IntegerCastLoop(const char *buf, idx_t len, T &result, bool strict) } return false; } - uint8_t digit = buf[pos++] - '0'; + auto digit = UnsafeNumericCast(buf[pos++] - '0'); if (!OP::template HandleDigit(result, digit)) { return false; } @@ -330,9 +331,9 @@ static bool IntegerHexCastLoop(const char *buf, idx_t len, T &result, bool stric } uint8_t digit; if (current_char >= 'a') { - digit = current_char - 'a' + 10; + digit = UnsafeNumericCast(current_char - 'a' + 10); } else { - digit = current_char - '0'; + digit = UnsafeNumericCast(current_char - '0'); } pos++; diff --git a/src/include/duckdb/common/sort/duckdb_pdqsort.hpp b/src/include/duckdb/common/sort/duckdb_pdqsort.hpp index 7b239a15ab9f..cae339b20f2e 100644 --- a/src/include/duckdb/common/sort/duckdb_pdqsort.hpp +++ b/src/include/duckdb/common/sort/duckdb_pdqsort.hpp @@ -154,9 +154,9 @@ struct PDQIterator { } inline friend idx_t operator-(const PDQIterator &lhs, const PDQIterator &rhs) { - D_ASSERT((*lhs - *rhs) % lhs.entry_size == 0); + D_ASSERT(duckdb::NumericCast(*lhs - *rhs) % lhs.entry_size == 0); D_ASSERT(*lhs - *rhs >= 0); - return (*lhs - *rhs) / lhs.entry_size; + return duckdb::NumericCast(*lhs - *rhs) / lhs.entry_size; } inline friend bool operator<(const PDQIterator &lhs, const PDQIterator &rhs) { @@ -320,7 +320,7 @@ inline T *align_cacheline(T *p) { #else std::size_t ip = reinterpret_cast(p); #endif - ip = (ip + cacheline_size - 1) & -cacheline_size; + ip = (ip + cacheline_size - 1) & duckdb::UnsafeNumericCast(-cacheline_size); return reinterpret_cast(ip); } diff --git a/src/include/duckdb/common/types/cast_helpers.hpp b/src/include/duckdb/common/types/cast_helpers.hpp index 2e071f3798d4..03c45b5a09df 100644 --- a/src/include/duckdb/common/types/cast_helpers.hpp +++ b/src/include/duckdb/common/types/cast_helpers.hpp @@ -62,7 +62,7 @@ class NumericHelper { template static string_t FormatSigned(SIGNED value, Vector &vector) { int sign = -(value < 0); - UNSIGNED unsigned_value = UnsafeNumericCast(UNSIGNED(value ^ sign) - sign); + UNSIGNED unsigned_value = UNSIGNED(value) ^ UNSIGNED(sign) - UNSIGNED(sign); int length = UnsignedLength(unsigned_value) - sign; string_t result = StringVector::EmptyString(vector, NumericCast(length)); auto dataptr = result.GetDataWriteable(); @@ -123,16 +123,18 @@ struct DecimalToString { *dst = '-'; } if (scale == 0) { - NumericHelper::FormatUnsigned(value, end); + NumericHelper::FormatUnsigned(UnsafeNumericCast(value), end); return; } // we write two numbers: // the numbers BEFORE the decimal (major) // and the numbers AFTER the decimal (minor) - UNSIGNED minor = value % (UNSIGNED)NumericHelper::POWERS_OF_TEN[scale]; - UNSIGNED major = value / (UNSIGNED)NumericHelper::POWERS_OF_TEN[scale]; + auto minor = + UnsafeNumericCast(value) % UnsafeNumericCast(NumericHelper::POWERS_OF_TEN[scale]); + auto major = + UnsafeNumericCast(value) / UnsafeNumericCast(NumericHelper::POWERS_OF_TEN[scale]); // write the number after the decimal - dst = NumericHelper::FormatUnsigned(minor, end); + dst = NumericHelper::FormatUnsigned(UnsafeNumericCast(minor), end); // (optionally) pad with zeros and add the decimal point while (dst > (end - scale)) { *--dst = '0'; @@ -142,7 +144,7 @@ struct DecimalToString { D_ASSERT(width > scale || major == 0); if (width > scale) { // there are numbers after the comma - dst = NumericHelper::FormatUnsigned(major, dst); + dst = NumericHelper::FormatUnsigned(UnsafeNumericCast(major), dst); } } @@ -150,7 +152,7 @@ struct DecimalToString { static string_t Format(SIGNED value, uint8_t width, uint8_t scale, Vector &vector) { int len = DecimalLength(value, width, scale); string_t result = StringVector::EmptyString(vector, NumericCast(len)); - FormatDecimal(value, width, scale, result.GetDataWriteable(), len); + FormatDecimal(value, width, scale, result.GetDataWriteable(), UnsafeNumericCast(len)); result.Finalize(); return result; } diff --git a/src/include/duckdb/common/types/validity_mask.hpp b/src/include/duckdb/common/types/validity_mask.hpp index b7dd548cf68a..fce5f0d1a09d 100644 --- a/src/include/duckdb/common/types/validity_mask.hpp +++ b/src/include/duckdb/common/types/validity_mask.hpp @@ -19,7 +19,7 @@ struct ValidityMask; template struct TemplatedValidityData { static constexpr const int BITS_PER_VALUE = sizeof(V) * 8; - static constexpr const V MAX_ENTRY = ~V(0); + static constexpr const V MAX_ENTRY = V(~V(0)); public: inline explicit TemplatedValidityData(idx_t count) { diff --git a/src/optimizer/rule/ordered_aggregate_optimizer.cpp b/src/optimizer/rule/ordered_aggregate_optimizer.cpp index b8f52e4916d7..93c9c80166a8 100644 --- a/src/optimizer/rule/ordered_aggregate_optimizer.cpp +++ b/src/optimizer/rule/ordered_aggregate_optimizer.cpp @@ -6,6 +6,7 @@ #include "duckdb/planner/expression/bound_constant_expression.hpp" #include "duckdb/main/client_context.hpp" #include "duckdb/planner/operator/logical_aggregate.hpp" +#include "duckdb/optimizer/rule/ordered_aggregate_optimizer.hpp" namespace duckdb { From f306b3149d9e1ce7fecd095b2f288e3afd6a93c1 Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 4 Apr 2024 12:55:14 +0200 Subject: [PATCH 151/603] recreate the last_value --- .../catalog_entry/sequence_catalog_entry.cpp | 66 +++++++++++++++++-- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/src/catalog/catalog_entry/sequence_catalog_entry.cpp b/src/catalog/catalog_entry/sequence_catalog_entry.cpp index cc122108dcf6..550e8bf0d444 100644 --- a/src/catalog/catalog_entry/sequence_catalog_entry.cpp +++ b/src/catalog/catalog_entry/sequence_catalog_entry.cpp @@ -13,12 +13,71 @@ namespace duckdb { +static int64_t LastValue(const CreateSequenceInfo &info) { + if (info.usage_count == 0) { + // last_value was never set, we don't need to recreate it + return info.start_value; + } + int64_t result = 0; + auto to_simulate = info.usage_count; + // The first use is always just initializing the counter to start_value + to_simulate--; + if (info.cycle) { + auto current = info.start_value; + auto increase = info.increment > 0; + if (increase) { + while (to_simulate > 0) { + auto maximum_increase = info.max_value - current; + uint64_t max_uses = maximum_increase / info.increment; + if (maximum_increase % info.increment != 0) { + // i.e 300 / 170 == 1.76, 2 would overflow + max_uses++; + } + if (to_simulate >= max_uses) { + // Uses would overflow, cycle around + to_simulate -= max_uses; + current = info.min_value; + result = current; + } else { + result = current + (info.increment * to_simulate); + break; + } + } + } else { + while (to_simulate > 0) { + auto maximum_decrease = current - info.min_value; + uint64_t max_uses = maximum_decrease / info.increment; + if (maximum_decrease % info.increment != 0) { + // If there's a remainder, one more decrement would overflow + max_uses++; + } + if (to_simulate >= max_uses) { + // Decrementing would overflow, cycle around + to_simulate -= max_uses; + current = info.max_value; + result = current; + } else { + result = current - (info.increment * to_simulate); + break; + } + } + } + return result; + } else { + // This is guaranteed to be in bounds, otherwise nextval would have thrown trying to create this state + return info.start_value + (info.increment * to_simulate); + } +} + // NOTE: usage_count is explicitly set to 0, // if the sequence was serialized to disk the start_value // was updated to the value of the last_value before serializing, keeping the state of the sequence. SequenceData::SequenceData(CreateSequenceInfo &info) - : usage_count(0), counter(info.start_value), last_value(info.start_value), increment(info.increment), + : usage_count(info.usage_count), counter(0), last_value(0), increment(info.increment), start_value(info.start_value), min_value(info.min_value), max_value(info.max_value), cycle(info.cycle) { + auto reconstructed_last_value = LastValue(info); + last_value = reconstructed_last_value; + counter = reconstructed_last_value; } SequenceCatalogEntry::SequenceCatalogEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateSequenceInfo &info) @@ -99,10 +158,7 @@ unique_ptr SequenceCatalogEntry::GetInfo() const { result->increment = seq_data.increment; result->min_value = seq_data.min_value; result->max_value = seq_data.max_value; - // To "persist" the last_value we create the sequence as if the provided START value is the current value - // Inside the SequenceData we set the usage_count to 0 so currvalue will throw if the sequence hasnt been updated - // yet - result->start_value = seq_data.counter; + result->start_value = seq_data.start_value; result->cycle = seq_data.cycle; result->comment = comment; return std::move(result); From 1cdfbd85d2d70fd0d79920252bf52f1508ac98f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Thu, 4 Apr 2024 12:59:15 +0200 Subject: [PATCH 152/603] hugeint --- src/common/types/hugeint.cpp | 15 ++++++++------- src/common/types/uhugeint.cpp | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/common/types/hugeint.cpp b/src/common/types/hugeint.cpp index bc4a13e0b98d..c99abaa9ee1e 100644 --- a/src/common/types/hugeint.cpp +++ b/src/common/types/hugeint.cpp @@ -112,7 +112,8 @@ static hugeint_t PositiveHugeintLeftShift(hugeint_t lhs, uint32_t amount) { D_ASSERT(amount > 0 && amount < 64); hugeint_t result; result.lower = lhs.lower << amount; - result.upper = (lhs.upper << amount) + (lhs.lower >> (64 - amount)); + result.upper = + UnsafeNumericCast((UnsafeNumericCast(lhs.upper) << amount) + (lhs.lower >> (64 - amount))); return result; } @@ -625,7 +626,7 @@ bool Hugeint::TryCast(hugeint_t input, uhugeint_t &result) { } result.lower = input.lower; - result.upper = input.upper; + result.upper = UnsafeNumericCast(input.upper); return true; } @@ -744,7 +745,7 @@ bool ConvertFloatingToBigint(REAL_T value, hugeint_t &result) { value = -value; } result.lower = (uint64_t)fmod(value, REAL_T(NumericLimits::Maximum())); - result.upper = (uint64_t)(value / REAL_T(NumericLimits::Maximum())); + result.upper = (int64_t)(value / REAL_T(NumericLimits::Maximum())); if (negative) { Hugeint::NegateInPlace(result); } @@ -829,14 +830,14 @@ hugeint_t hugeint_t::operator>>(const hugeint_t &rhs) const { return *this; } else if (shift == 64) { result.upper = (upper < 0) ? -1 : 0; - result.lower = upper; + result.lower = UnsafeNumericCast(upper); } else if (shift < 64) { // perform lower shift in unsigned integer, and mask away the most significant bit result.lower = (uint64_t(upper) << (64 - shift)) | (lower >> shift); result.upper = upper >> shift; } else { D_ASSERT(shift < 128); - result.lower = upper >> (shift - 64); + result.lower = UnsafeNumericCast(upper >> (shift - 64)); result.upper = (upper < 0) ? -1 : 0; } return result; @@ -851,7 +852,7 @@ hugeint_t hugeint_t::operator<<(const hugeint_t &rhs) const { if (rhs.upper != 0 || shift >= 128) { return hugeint_t(0); } else if (shift == 64) { - result.upper = lower; + result.upper = UnsafeNumericCast(lower); result.lower = 0; } else if (shift == 0) { return *this; @@ -859,7 +860,7 @@ hugeint_t hugeint_t::operator<<(const hugeint_t &rhs) const { // perform upper shift in unsigned integer, and mask away the most significant bit uint64_t upper_shift = ((uint64_t(upper) << shift) + (lower >> (64 - shift))) & 0x7FFFFFFFFFFFFFFF; result.lower = lower << shift; - result.upper = upper_shift; + result.upper = UnsafeNumericCast(upper_shift); } else { D_ASSERT(shift < 128); result.lower = 0; diff --git a/src/common/types/uhugeint.cpp b/src/common/types/uhugeint.cpp index 479f756d4300..7469dd809642 100644 --- a/src/common/types/uhugeint.cpp +++ b/src/common/types/uhugeint.cpp @@ -399,7 +399,7 @@ bool Uhugeint::TryCast(uhugeint_t input, hugeint_t &result) { } result.lower = input.lower; - result.upper = input.upper; + result.upper = UnsafeNumericCast(input.upper); return true; } From 836f84d4ec456474baa34a1ea616a019529b9aab Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 4 Apr 2024 13:11:57 +0200 Subject: [PATCH 153/603] Adjust test --- .../csv/rejects/csv_rejects_two_tables.test | 48 ++++++++----------- 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/test/sql/copy/csv/rejects/csv_rejects_two_tables.test b/test/sql/copy/csv/rejects/csv_rejects_two_tables.test index e9ad454f6052..f50128989810 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_two_tables.test +++ b/test/sql/copy/csv/rejects/csv_rejects_two_tables.test @@ -17,20 +17,18 @@ BIGINT VARCHAR 11044 11044 2 query IIIIIIIIIIIII -SELECT * FROM reject_scans order by all; ---- 3 0 test/sql/copy/csv/data/error/mismatch/big_bad.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL store_rejects=true, sample_size=1 3 1 test/sql/copy/csv/data/error/mismatch/big_bad2.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL store_rejects=true, sample_size=1 -query IIIIIIIII -SELECT * +query IIIIIIIIII FROM reject_errors order by all; ---- -3 0 2176 10875 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' -3 0 4176 20875 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' -3 1 3680 18395 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' -3 1 5680 28395 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' +3 0 2176 10876 10876 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +3 0 4176 20876 20876 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' +3 1 3680 18396 18396 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +3 1 5680 28396 28396 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' # Test giving the name of errors table statement error @@ -54,20 +52,18 @@ SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), M BIGINT VARCHAR 11044 11044 2 query IIIIIIIIIIIII -SELECT * FROM reject_scans order by all; ---- 8 0 test/sql/copy/csv/data/error/mismatch/big_bad.csv , \0 \0 \n 0 false {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_table='rejects_errors_2', sample_size=1 8 1 test/sql/copy/csv/data/error/mismatch/big_bad2.csv , \0 \0 \n 0 false {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_table='rejects_errors_2', sample_size=1 -query IIIIIIIII -SELECT * +query IIIIIIIIII FROM rejects_errors_2 order by all; ---- -8 0 2176 10875 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' -8 0 4176 20875 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' -8 1 3680 18395 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' -8 1 5680 28395 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' +8 0 2176 10876 10876 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +8 0 4176 20876 20876 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' +8 1 3680 18396 18396 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +8 1 5680 28396 28396 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' statement ok drop table reject_errors; @@ -82,21 +78,18 @@ SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), M BIGINT VARCHAR 11044 11044 2 query IIIIIIIIIIIII -SELECT * FROM rejects_scan_2 order by all; ---- 12 0 test/sql/copy/csv/data/error/mismatch/big_bad.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_scan='rejects_scan_2', sample_size=1 12 1 test/sql/copy/csv/data/error/mismatch/big_bad2.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_scan='rejects_scan_2', sample_size=1 -query IIIIIIIII -SELECT * +query IIIIIIIIII FROM reject_errors order by all; ---- -12 0 2176 10875 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' -12 0 4176 20875 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' -12 1 3680 18395 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' -12 1 5680 28395 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' - +12 0 2176 10876 10876 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +12 0 4176 20876 20876 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' +12 1 3680 18396 18396 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +12 1 5680 28396 28396 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' # Test giving the name of both tables query IIIII @@ -116,14 +109,13 @@ FROM rejects_scan_3 order by all; 15 0 test/sql/copy/csv/data/error/mismatch/big_bad.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_table='rejects_errors_3', rejects_scan='rejects_scan_3', sample_size=1 15 1 test/sql/copy/csv/data/error/mismatch/big_bad2.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_table='rejects_errors_3', rejects_scan='rejects_scan_3', sample_size=1 -query IIIIIIIII -SELECT * +query IIIIIIIIII FROM rejects_errors_3 order by all; ---- -15 0 2176 10875 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' -15 0 4176 20875 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' -15 1 3680 18395 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' -15 1 5680 28395 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' +15 0 2176 10876 10876 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +15 0 4176 20876 20876 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' +15 1 3680 18396 18396 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +15 1 5680 28396 28396 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' statement ok drop table reject_errors; From c761de72539b99f568d89cdcaa8ac9c2012e6a83 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 4 Apr 2024 13:13:04 +0200 Subject: [PATCH 154/603] BoundExpandedExpression --- src/common/enums/expression_type.cpp | 4 +++ .../duckdb/common/enums/expression_type.hpp | 6 ++-- .../expression/bound_expanded_expression.hpp | 34 +++++++++++++++++++ .../expression_binder/select_binder.hpp | 8 ----- .../expression/bind_unnest_expression.cpp | 4 +-- .../binder/query_node/bind_select_node.cpp | 8 ++--- src/planner/expression/CMakeLists.txt | 1 + .../expression/bound_expanded_expression.cpp | 23 +++++++++++++ 8 files changed, 72 insertions(+), 16 deletions(-) create mode 100644 src/include/duckdb/planner/expression/bound_expanded_expression.hpp create mode 100644 src/planner/expression/bound_expanded_expression.cpp diff --git a/src/common/enums/expression_type.cpp b/src/common/enums/expression_type.cpp index c017b5059a4b..105f4dfa781b 100644 --- a/src/common/enums/expression_type.cpp +++ b/src/common/enums/expression_type.cpp @@ -141,6 +141,8 @@ string ExpressionTypeToString(ExpressionType type) { return "LAMBDA"; case ExpressionType::ARROW: return "ARROW"; + case ExpressionType::BOUND_EXPANDED: + return "BOUND_EXPANDED"; case ExpressionType::INVALID: break; } @@ -224,6 +226,8 @@ string ExpressionClassToString(ExpressionClass type) { return "BOUND_LAMBDA"; case ExpressionClass::BOUND_EXPRESSION: return "BOUND_EXPRESSION"; + case ExpressionClass::BOUND_EXPANDED: + return "BOUND_EXPANDED"; default: return "ExpressionClass::!!UNIMPLEMENTED_CASE!!"; } diff --git a/src/include/duckdb/common/enums/expression_type.hpp b/src/include/duckdb/common/enums/expression_type.hpp index 0b6516de0279..858781aa92d4 100644 --- a/src/include/duckdb/common/enums/expression_type.hpp +++ b/src/include/duckdb/common/enums/expression_type.hpp @@ -145,7 +145,8 @@ enum class ExpressionType : uint8_t { COLLATE = 230, LAMBDA = 231, POSITIONAL_REFERENCE = 232, - BOUND_LAMBDA_REF = 233 + BOUND_LAMBDA_REF = 233, + BOUND_EXPANDED = 234 }; //===--------------------------------------------------------------------===// @@ -199,7 +200,8 @@ enum class ExpressionClass : uint8_t { //===--------------------------------------------------------------------===// // Miscellaneous //===--------------------------------------------------------------------===// - BOUND_EXPRESSION = 50 + BOUND_EXPRESSION = 50, + BOUND_EXPANDED = 51 }; DUCKDB_API string ExpressionTypeToString(ExpressionType type); diff --git a/src/include/duckdb/planner/expression/bound_expanded_expression.hpp b/src/include/duckdb/planner/expression/bound_expanded_expression.hpp new file mode 100644 index 000000000000..7b44002448e1 --- /dev/null +++ b/src/include/duckdb/planner/expression/bound_expanded_expression.hpp @@ -0,0 +1,34 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// duckdb/planner/expression/bound_expanded_expression.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/planner/expression.hpp" + +namespace duckdb { + +//! BoundExpression is an intermediate dummy expression used by the binder. +//! It holds a set of expressions that will be "expanded" in the select list of a query +class BoundExpandedExpression : public Expression { +public: + static constexpr const ExpressionClass TYPE = ExpressionClass::BOUND_EXPANDED; + +public: + explicit BoundExpandedExpression(vector> expanded_expressions); + + vector> expanded_expressions; + +public: + string ToString() const override; + + bool Equals(const BaseExpression &other) const override; + + unique_ptr Copy() override; +}; + +} // namespace duckdb diff --git a/src/include/duckdb/planner/expression_binder/select_binder.hpp b/src/include/duckdb/planner/expression_binder/select_binder.hpp index 9c2ca8a70c52..324492e5e101 100644 --- a/src/include/duckdb/planner/expression_binder/select_binder.hpp +++ b/src/include/duckdb/planner/expression_binder/select_binder.hpp @@ -19,18 +19,10 @@ class SelectBinder : public BaseSelectBinder { case_insensitive_map_t alias_map); SelectBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info); - bool HasExpandedExpressions() { - return !expanded_expressions.empty(); - } - vector> &ExpandedExpressions() { - return expanded_expressions; - } - protected: BindResult BindUnnest(FunctionExpression &function, idx_t depth, bool root_expression) override; idx_t unnest_level = 0; - vector> expanded_expressions; }; } // namespace duckdb diff --git a/src/planner/binder/expression/bind_unnest_expression.cpp b/src/planner/binder/expression/bind_unnest_expression.cpp index 3df73284d8b1..0883d5f29870 100644 --- a/src/planner/binder/expression/bind_unnest_expression.cpp +++ b/src/planner/binder/expression/bind_unnest_expression.cpp @@ -4,6 +4,7 @@ #include "duckdb/planner/expression/bound_aggregate_expression.hpp" #include "duckdb/planner/expression/bound_columnref_expression.hpp" #include "duckdb/planner/expression/bound_constant_expression.hpp" +#include "duckdb/planner/expression/bound_expanded_expression.hpp" #include "duckdb/planner/expression/bound_function_expression.hpp" #include "duckdb/planner/expression/bound_parameter_expression.hpp" #include "duckdb/planner/expression_binder/aggregate_binder.hpp" @@ -224,8 +225,7 @@ BindResult SelectBinder::BindUnnest(FunctionExpression &function, idx_t depth, b break; } } - expanded_expressions = std::move(struct_expressions); - unnest_expr = make_uniq(Value(42)); + unnest_expr = make_uniq(std::move(struct_expressions)); } return BindResult(std::move(unnest_expr)); } diff --git a/src/planner/binder/query_node/bind_select_node.cpp b/src/planner/binder/query_node/bind_select_node.cpp index 737e6ad020b7..7751b6e91dcd 100644 --- a/src/planner/binder/query_node/bind_select_node.cpp +++ b/src/planner/binder/query_node/bind_select_node.cpp @@ -14,6 +14,7 @@ #include "duckdb/parser/tableref/joinref.hpp" #include "duckdb/planner/binder.hpp" #include "duckdb/planner/expression/bound_aggregate_expression.hpp" +#include "duckdb/planner/expression/bound_expanded_expression.hpp" #include "duckdb/planner/expression_binder/column_alias_binder.hpp" #include "duckdb/planner/expression_binder/constant_binder.hpp" #include "duckdb/planner/expression_binder/group_binder.hpp" @@ -486,7 +487,7 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ statement.aggregate_handling == AggregateHandling::FORCE_AGGREGATES && is_original_column; result->bound_column_count++; - if (select_binder.HasExpandedExpressions()) { + if (expr->type == ExpressionType::BOUND_EXPANDED) { if (!is_original_column) { throw InternalException("Only original columns can have expanded expressions"); } @@ -494,7 +495,8 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ throw BinderException("UNNEST of struct cannot be combined with GROUP BY ALL"); } - auto &struct_expressions = select_binder.ExpandedExpressions(); + auto &expanded = expr->Cast(); + auto &struct_expressions = expanded.expanded_expressions; D_ASSERT(!struct_expressions.empty()); for (auto &struct_expr : struct_expressions) { @@ -503,8 +505,6 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ internal_sql_types.push_back(struct_expr->return_type); result->select_list.push_back(std::move(struct_expr)); } - - struct_expressions.clear(); continue; } diff --git a/src/planner/expression/CMakeLists.txt b/src/planner/expression/CMakeLists.txt index 4ed5171d8923..af9ba5a90028 100644 --- a/src/planner/expression/CMakeLists.txt +++ b/src/planner/expression/CMakeLists.txt @@ -10,6 +10,7 @@ add_library_unity( bound_comparison_expression.cpp bound_conjunction_expression.cpp bound_constant_expression.cpp + bound_expanded_expression.cpp bound_function_expression.cpp bound_lambda_expression.cpp bound_lambdaref_expression.cpp diff --git a/src/planner/expression/bound_expanded_expression.cpp b/src/planner/expression/bound_expanded_expression.cpp new file mode 100644 index 000000000000..15f0f5e045a4 --- /dev/null +++ b/src/planner/expression/bound_expanded_expression.cpp @@ -0,0 +1,23 @@ +#include "duckdb/planner/expression/bound_expanded_expression.hpp" + + +namespace duckdb { + +BoundExpandedExpression::BoundExpandedExpression(vector> expanded_expressions_p) + : Expression(ExpressionType::BOUND_EXPANDED, ExpressionClass::BOUND_EXPANDED, LogicalType::INTEGER), + expanded_expressions(std::move(expanded_expressions_p)) { +} + +string BoundExpandedExpression::ToString() const { + return "BOUND_EXPANDED"; +} + +bool BoundExpandedExpression::Equals(const BaseExpression &other_p) const { + return false; +} + +unique_ptr BoundExpandedExpression::Copy() { + throw SerializationException("Cannot copy BoundExpandedExpression"); +} + +} // namespace duckdb From a6924b638a0851cbf4ef9909106500c8769840d4 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 4 Apr 2024 13:13:24 +0200 Subject: [PATCH 155/603] Small message adjustment --- test/sql/copy/csv/rejects/test_invalid_parameters.test | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/sql/copy/csv/rejects/test_invalid_parameters.test b/test/sql/copy/csv/rejects/test_invalid_parameters.test index 9325f3780f24..950337a90ebb 100644 --- a/test/sql/copy/csv/rejects/test_invalid_parameters.test +++ b/test/sql/copy/csv/rejects/test_invalid_parameters.test @@ -12,10 +12,10 @@ SELECT * FROM read_csv( 'test/sql/copy/csv/data/error/mismatch/bad.csv', columns = {'col0': 'INTEGER', 'col1': 'INTEGER', 'col2': 'VARCHAR'}, ignore_errors=false, - rejects_table='csv_rejects_table' + store_rejects=true ) ---- -only supported when IGNORE_ERRORS is set to true +STORE_REJECTS option is only supported when IGNORE_ERRORS is not manually set to false statement error SELECT * FROM read_csv( @@ -63,7 +63,7 @@ SELECT * FROM read_csv_auto( rejects_table='csv_rejects_table' ) ---- -only supported when IGNORE_ERRORS is set to true +option is only supported when IGNORE_ERRORS is not manually set to false statement error SELECT * FROM read_csv_auto( From 5f2883ff4e8d576d9698344c8e9309be2c754d66 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 4 Apr 2024 13:15:48 +0200 Subject: [PATCH 156/603] Adjustment to utf rejects --- .../csv/rejects/test_invalid_utf_rejects.test | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/test/sql/copy/csv/rejects/test_invalid_utf_rejects.test b/test/sql/copy/csv/rejects/test_invalid_utf_rejects.test index 94c56cc71562..e579648f8794 100644 --- a/test/sql/copy/csv/rejects/test_invalid_utf_rejects.test +++ b/test/sql/copy/csv/rejects/test_invalid_utf_rejects.test @@ -9,13 +9,12 @@ require notwindows statement ok from read_csv('test/sql/copy/csv/data/test/invalid_utf_big.csv',columns = {'col1': 'VARCHAR','col2': 'VARCHAR','col3': 'VARCHAR'}, - auto_detect=false, rejects_table='csv_rejects_table', header = 0, delim = ',', ignore_errors=true) + auto_detect=false, header = 0, delim = ',', store_rejects=true) -query IIIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position -FROM csv_rejects_table; +query IIIIIIIIII rowsort +FROM reject_errors ORDER BY ALL; ---- -test/sql/copy/csv/data/test/invalid_utf_big.csv 3001 2 "col2" INVALID UNICODE valid,invalid_??_part,valid 54000 -test/sql/copy/csv/data/test/invalid_utf_big.csv 3012 3 "col3" INVALID UNICODE valid,valid,invalid_??_part 54208 -test/sql/copy/csv/data/test/invalid_utf_big.csv 3023 2 "col2" INVALID UNICODE valid,invalid_??_part,valid 54416 -test/sql/copy/csv/data/test/invalid_utf_big.csv 3034 3 "col3" INVALID UNICODE valid,valid,invalid_??_part 54624 \ No newline at end of file +3 0 3001 54001 54007 2 col2 INVALID UNICODE valid,invalid_??_part,valid Invalid unicode (byte sequence mismatch) detected. +3 0 3012 54209 54221 3 col3 INVALID UNICODE valid,valid,invalid_??_part Invalid unicode (byte sequence mismatch) detected. +3 0 3023 54417 54423 2 col2 INVALID UNICODE valid,invalid_??_part,valid Invalid unicode (byte sequence mismatch) detected. +3 0 3034 54625 54637 3 col3 INVALID UNICODE valid,valid,invalid_??_part Invalid unicode (byte sequence mismatch) detected. \ No newline at end of file From 1ae2d3c66dd6a847bcec18a5f10a9126924b8e4a Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 4 Apr 2024 13:27:13 +0200 Subject: [PATCH 157/603] More adjustments --- test/sql/copy/csv/rejects/test_mixed.test | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/test/sql/copy/csv/rejects/test_mixed.test b/test/sql/copy/csv/rejects/test_mixed.test index 45001a5e4b05..e5ced0ea3ae8 100644 --- a/test/sql/copy/csv/rejects/test_mixed.test +++ b/test/sql/copy/csv/rejects/test_mixed.test @@ -11,8 +11,7 @@ query III SELECT * FROM read_csv( 'data/csv/rejects/frankstein/nightmare.csv', columns = {'a': 'INTEGER', 'b': 'INTEGER', 'c': 'VARCHAR'}, - rejects_table='csv_rejects_table', - ignore_errors=true, auto_detect=false, header = 1, max_line_size=20); + store_rejects = true, auto_detect=false, header = 1, max_line_size=20); ---- 1 2 pedro 1 2 pedro @@ -56,13 +55,12 @@ SELECT * FROM read_csv( 1 2 pedro 1 2 pedro -query IIIIIIII rowsort -SELECT regexp_replace(file, '\\', '/', 'g'), line, column_idx, column_name, error_type, csv_line, byte_position, error_message -FROM csv_rejects_table; +query IIIIIIIIII rowsort +FROM reject_errors ORDER BY ALL; ---- -data/csv/rejects/frankstein/nightmare.csv 10 2 "c" MISSING COLUMNS 1,2 102 Expected Number of Columns: 3 Found: 2 -data/csv/rejects/frankstein/nightmare.csv 14 4 NULL TOO MANY COLUMNS 1,2,"pedro",5 142 Expected Number of Columns: 3 Found: 4 -data/csv/rejects/frankstein/nightmare.csv 19 2 "b" CAST 1,bla,"pedro" 204 Error when converting column "b". Could not convert string "bla" to 'INTEGER' -data/csv/rejects/frankstein/nightmare.csv 22 3 "c" UNQUOTED VALUE 1,2,"pedro"bla 242 Value with unterminated quote found. -data/csv/rejects/frankstein/nightmare.csv 32 1 "a" LINE SIZE OVER MAXIMUM 1,2,"pedro thiago timbo holanda" 365 Maximum line size of 20 bytes exceeded. Actual Size:33 bytes. -data/csv/rejects/frankstein/nightmare.csv 38 3 "c" INVALID UNICODE 1,2,"pedro??" 458 Invalid unicode (byte sequence mismatch) detected. \ No newline at end of file +3 0 10 103 106 2 c MISSING COLUMNS 1,2 Expected Number of Columns: 3 Found: 2 +3 0 14 143 154 4 NULL TOO MANY COLUMNS 1,2,"pedro",5 Expected Number of Columns: 3 Found: 4 +3 0 19 205 207 2 b CAST 1,bla,"pedro" Error when converting column "b". Could not convert string "bla" to 'INTEGER' +3 0 22 243 247 3 c UNQUOTED VALUE 1,2,"pedro"bla Value with unterminated quote found. +3 0 32 366 366 1 a LINE SIZE OVER MAXIMUM 1,2,"pedro thiago timbo holanda" Maximum line size of 20 bytes exceeded. Actual Size:33 bytes. +3 0 38 459 463 3 c INVALID UNICODE 1,2,"pedro??" Invalid unicode (byte sequence mismatch) detected. \ No newline at end of file From 29b42357cf3e2850c16aa897a1a4bbe5c5332f3a Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 4 Apr 2024 13:53:57 +0200 Subject: [PATCH 158/603] Move alias information to dedicated SelectBindState object --- .../duckdb/planner/expression_binder.hpp | 1 + .../expression_binder/base_select_binder.hpp | 7 ++-- .../expression_binder/column_alias_binder.hpp | 7 ++-- .../expression_binder/group_binder.hpp | 5 ++- .../expression_binder/having_binder.hpp | 2 +- .../expression_binder/order_binder.hpp | 10 ++--- .../expression_binder/qualify_binder.hpp | 3 +- .../expression_binder/select_binder.hpp | 2 - .../planner/query_node/bound_select_node.hpp | 7 ++-- .../duckdb/planner/select_bind_state.hpp | 30 ++++++++++++++ .../binder/query_node/bind_select_node.cpp | 23 ++++++----- .../binder/query_node/bind_setop_node.cpp | 39 +++++++++---------- src/planner/binder/statement/bind_create.cpp | 8 ++-- .../expression_binder/base_select_binder.cpp | 22 +++++------ .../expression_binder/column_alias_binder.cpp | 12 +++--- .../expression_binder/group_binder.cpp | 8 ++-- .../expression_binder/having_binder.cpp | 4 +- .../expression_binder/order_binder.cpp | 21 +++++----- .../expression_binder/qualify_binder.cpp | 4 +- .../expression_binder/select_binder.cpp | 7 +--- 20 files changed, 117 insertions(+), 105 deletions(-) create mode 100644 src/include/duckdb/planner/select_bind_state.hpp diff --git a/src/include/duckdb/planner/expression_binder.hpp b/src/include/duckdb/planner/expression_binder.hpp index b74e0d289705..2f43ab0a9271 100644 --- a/src/include/duckdb/planner/expression_binder.hpp +++ b/src/include/duckdb/planner/expression_binder.hpp @@ -34,6 +34,7 @@ class CatalogEntry; class SimpleFunction; struct DummyBinding; +struct SelectBindState; struct BoundColumnReferenceInfo { string name; diff --git a/src/include/duckdb/planner/expression_binder/base_select_binder.hpp b/src/include/duckdb/planner/expression_binder/base_select_binder.hpp index 80faa01535cc..777b3514e9be 100644 --- a/src/include/duckdb/planner/expression_binder/base_select_binder.hpp +++ b/src/include/duckdb/planner/expression_binder/base_select_binder.hpp @@ -28,9 +28,7 @@ struct BoundGroupInformation { //! functions. class BaseSelectBinder : public ExpressionBinder { public: - BaseSelectBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info, - case_insensitive_map_t alias_map); - BaseSelectBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info); + BaseSelectBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info, bool support_alias_binding = true); bool BoundAggregates() { return bound_aggregate; @@ -48,10 +46,11 @@ class BaseSelectBinder : public ExpressionBinder { bool inside_window; bool bound_aggregate = false; + bool support_alias_binding; BoundSelectNode &node; BoundGroupInformation &info; - case_insensitive_map_t alias_map; + const SelectBindState &bind_state; protected: BindResult BindColumnRef(unique_ptr &expr_ptr, idx_t depth); diff --git a/src/include/duckdb/planner/expression_binder/column_alias_binder.hpp b/src/include/duckdb/planner/expression_binder/column_alias_binder.hpp index 8795ef3af849..056c17b180f6 100644 --- a/src/include/duckdb/planner/expression_binder/column_alias_binder.hpp +++ b/src/include/duckdb/planner/expression_binder/column_alias_binder.hpp @@ -13,20 +13,19 @@ namespace duckdb { -class BoundSelectNode; class ColumnRefExpression; +struct SelectBindState; //! A helper binder for WhereBinder and HavingBinder which support alias as a columnref. class ColumnAliasBinder { public: - ColumnAliasBinder(BoundSelectNode &node, const case_insensitive_map_t &alias_map); + explicit ColumnAliasBinder(const SelectBindState &bind_state); bool BindAlias(ExpressionBinder &enclosing_binder, unique_ptr &expr_ptr, idx_t depth, bool root_expression, BindResult &result); private: - BoundSelectNode &node; - const case_insensitive_map_t &alias_map; + const SelectBindState &bind_state; unordered_set visited_select_indexes; }; diff --git a/src/include/duckdb/planner/expression_binder/group_binder.hpp b/src/include/duckdb/planner/expression_binder/group_binder.hpp index 10e32b6cd08f..40a4c7a3a455 100644 --- a/src/include/duckdb/planner/expression_binder/group_binder.hpp +++ b/src/include/duckdb/planner/expression_binder/group_binder.hpp @@ -14,12 +14,13 @@ namespace duckdb { class ConstantExpression; class ColumnRefExpression; +struct SelectBindState; //! The GROUP binder is responsible for binding expressions in the GROUP BY clause class GroupBinder : public ExpressionBinder { public: GroupBinder(Binder &binder, ClientContext &context, SelectNode &node, idx_t group_index, - case_insensitive_map_t &alias_map, case_insensitive_map_t &group_alias_map); + const SelectBindState &bind_state, case_insensitive_map_t &group_alias_map); //! The unbound root expression unique_ptr unbound_expression; @@ -36,7 +37,7 @@ class GroupBinder : public ExpressionBinder { BindResult BindConstant(ConstantExpression &expr); SelectNode &node; - case_insensitive_map_t &alias_map; + const SelectBindState &bind_state; case_insensitive_map_t &group_alias_map; unordered_set used_aliases; diff --git a/src/include/duckdb/planner/expression_binder/having_binder.hpp b/src/include/duckdb/planner/expression_binder/having_binder.hpp index 113bc68a1ade..b49b4f454af6 100644 --- a/src/include/duckdb/planner/expression_binder/having_binder.hpp +++ b/src/include/duckdb/planner/expression_binder/having_binder.hpp @@ -18,7 +18,7 @@ namespace duckdb { class HavingBinder : public BaseSelectBinder { public: HavingBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info, - case_insensitive_map_t &alias_map, AggregateHandling aggregate_handling); + const SelectBindState &bind_state, AggregateHandling aggregate_handling); protected: BindResult BindExpression(unique_ptr &expr_ptr, idx_t depth, diff --git a/src/include/duckdb/planner/expression_binder/order_binder.hpp b/src/include/duckdb/planner/expression_binder/order_binder.hpp index 4868074d483a..fb096f9f39f1 100644 --- a/src/include/duckdb/planner/expression_binder/order_binder.hpp +++ b/src/include/duckdb/planner/expression_binder/order_binder.hpp @@ -17,14 +17,13 @@ namespace duckdb { class Binder; class Expression; class SelectNode; +struct SelectBindState; //! The ORDER binder is responsible for binding an expression within the ORDER BY clause of a SQL statement class OrderBinder { public: - OrderBinder(vector binders, idx_t projection_index, case_insensitive_map_t &alias_map, - parsed_expression_map_t &projection_map, idx_t max_count); - OrderBinder(vector binders, idx_t projection_index, SelectNode &node, - case_insensitive_map_t &alias_map, parsed_expression_map_t &projection_map); + OrderBinder(vector binders, idx_t projection_index, SelectBindState &bind_state, idx_t max_count); + OrderBinder(vector binders, idx_t projection_index, SelectNode &node, SelectBindState &bind_state); public: unique_ptr Bind(unique_ptr expr); @@ -50,8 +49,7 @@ class OrderBinder { idx_t projection_index; idx_t max_count; vector> *extra_list; - case_insensitive_map_t &alias_map; - parsed_expression_map_t &projection_map; + SelectBindState &bind_state; }; } // namespace duckdb diff --git a/src/include/duckdb/planner/expression_binder/qualify_binder.hpp b/src/include/duckdb/planner/expression_binder/qualify_binder.hpp index ef5d6e596153..79aec66d0461 100644 --- a/src/include/duckdb/planner/expression_binder/qualify_binder.hpp +++ b/src/include/duckdb/planner/expression_binder/qualify_binder.hpp @@ -12,12 +12,13 @@ #include "duckdb/planner/expression_binder/column_alias_binder.hpp" namespace duckdb { +struct SelectBindState; //! The QUALIFY binder is responsible for binding an expression within the QUALIFY clause of a SQL statement class QualifyBinder : public BaseSelectBinder { public: QualifyBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info, - case_insensitive_map_t &alias_map); + const SelectBindState &bind_state); protected: BindResult BindExpression(unique_ptr &expr_ptr, idx_t depth, diff --git a/src/include/duckdb/planner/expression_binder/select_binder.hpp b/src/include/duckdb/planner/expression_binder/select_binder.hpp index 324492e5e101..86c88015803a 100644 --- a/src/include/duckdb/planner/expression_binder/select_binder.hpp +++ b/src/include/duckdb/planner/expression_binder/select_binder.hpp @@ -15,8 +15,6 @@ namespace duckdb { //! The SELECT binder is responsible for binding an expression within the SELECT clause of a SQL statement class SelectBinder : public BaseSelectBinder { public: - SelectBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info, - case_insensitive_map_t alias_map); SelectBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info); protected: diff --git a/src/include/duckdb/planner/query_node/bound_select_node.hpp b/src/include/duckdb/planner/query_node/bound_select_node.hpp index 0bab35cacb1e..7455abd8b935 100644 --- a/src/include/duckdb/planner/query_node/bound_select_node.hpp +++ b/src/include/duckdb/planner/query_node/bound_select_node.hpp @@ -14,6 +14,7 @@ #include "duckdb/planner/bound_tableref.hpp" #include "duckdb/parser/parsed_data/sample_options.hpp" #include "duckdb/parser/group_by_node.hpp" +#include "duckdb/planner/select_bind_state.hpp" namespace duckdb { @@ -41,10 +42,8 @@ class BoundSelectNode : public BoundQueryNode { BoundSelectNode() : BoundQueryNode(QueryNodeType::SELECT_NODE) { } - //! The original unparsed expressions. This is exported after binding, because the binding might change the - //! expressions (e.g. when a * clause is present) - vector> original_expressions; - + //! Bind information + SelectBindState bind_state; //! The projection list vector> select_list; //! The FROM clause diff --git a/src/include/duckdb/planner/select_bind_state.hpp b/src/include/duckdb/planner/select_bind_state.hpp new file mode 100644 index 000000000000..e62e9ad43c67 --- /dev/null +++ b/src/include/duckdb/planner/select_bind_state.hpp @@ -0,0 +1,30 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// duckdb/planner/select_bind_state.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/planner/bound_query_node.hpp" +#include "duckdb/planner/logical_operator.hpp" +#include "duckdb/parser/expression_map.hpp" +#include "duckdb/planner/bound_tableref.hpp" +#include "duckdb/parser/parsed_data/sample_options.hpp" +#include "duckdb/parser/group_by_node.hpp" + +namespace duckdb { + +//! Bind state during a SelectNode +struct SelectBindState { + // Mapping of (alias -> index) and a mapping of (Expression -> index) for the SELECT list + case_insensitive_map_t alias_map; + parsed_expression_map_t projection_map; + //! The original unparsed expressions. This is exported after binding, because the binding might change the + //! expressions (e.g. when a * clause is present) + vector> original_expressions; +}; + +} // namespace duckdb diff --git a/src/planner/binder/query_node/bind_select_node.cpp b/src/planner/binder/query_node/bind_select_node.cpp index 7751b6e91dcd..129978ac1dd5 100644 --- a/src/planner/binder/query_node/bind_select_node.cpp +++ b/src/planner/binder/query_node/bind_select_node.cpp @@ -26,6 +26,7 @@ #include "duckdb/planner/expression_iterator.hpp" #include "duckdb/planner/query_node/bound_select_node.hpp" #include "duckdb/parser/expression/function_expression.hpp" +#include "duckdb/planner/select_bind_state.hpp" namespace duckdb { @@ -361,19 +362,17 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ } statement.select_list = std::move(new_select_list); - // create a mapping of (alias -> index) and a mapping of (Expression -> index) for the SELECT list - case_insensitive_map_t alias_map; - parsed_expression_map_t projection_map; + auto &bind_state = result->bind_state; for (idx_t i = 0; i < statement.select_list.size(); i++) { auto &expr = statement.select_list[i]; result->names.push_back(expr->GetName()); ExpressionBinder::QualifyColumnNames(*this, expr); if (!expr->alias.empty()) { - alias_map[expr->alias] = i; + bind_state.alias_map[expr->alias] = i; result->names[i] = expr->alias; } - projection_map[*expr] = i; - result->original_expressions.push_back(expr->Copy()); + bind_state.projection_map[*expr] = i; + bind_state.original_expressions.push_back(expr->Copy()); } result->column_count = statement.select_list.size(); @@ -383,14 +382,14 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ // bind any star expressions in the WHERE clause BindWhereStarExpression(statement.where_clause); - ColumnAliasBinder alias_binder(*result, alias_map); + ColumnAliasBinder alias_binder(bind_state); WhereBinder where_binder(*this, context, &alias_binder); unique_ptr condition = std::move(statement.where_clause); result->where_clause = where_binder.Bind(condition); } // now bind all the result modifiers; including DISTINCT and ORDER BY targets - OrderBinder order_binder({this}, result->projection_index, statement, alias_map, projection_map); + OrderBinder order_binder({this}, result->projection_index, statement, bind_state); BindModifiers(order_binder, statement, *result); vector> unbound_groups; @@ -399,7 +398,7 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ if (!group_expressions.empty()) { // the statement has a GROUP BY clause, bind it unbound_groups.resize(group_expressions.size()); - GroupBinder group_binder(*this, context, statement, result->group_index, alias_map, info.alias_map); + GroupBinder group_binder(*this, context, statement, result->group_index, bind_state, info.alias_map); for (idx_t i = 0; i < group_expressions.size(); i++) { // we keep a copy of the unbound expression; @@ -449,7 +448,7 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ // bind the HAVING clause, if any if (statement.having) { - HavingBinder having_binder(*this, context, *result, info, alias_map, statement.aggregate_handling); + HavingBinder having_binder(*this, context, *result, info, bind_state, statement.aggregate_handling); ExpressionBinder::QualifyColumnNames(*this, statement.having); result->having = having_binder.Bind(statement.having); } @@ -460,7 +459,7 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ if (statement.aggregate_handling == AggregateHandling::FORCE_AGGREGATES) { throw BinderException("Combining QUALIFY with GROUP BY ALL is not supported yet"); } - qualify_binder = make_uniq(*this, context, *result, info, alias_map); + qualify_binder = make_uniq(*this, context, *result, info, bind_state); ExpressionBinder::QualifyColumnNames(*this, statement.qualify); result->qualify = qualify_binder->Bind(statement.qualify); if (qualify_binder->HasBoundColumns() && qualify_binder->BoundAggregates()) { @@ -469,7 +468,7 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ } // after that, we bind to the SELECT list - SelectBinder select_binder(*this, context, *result, info, alias_map); + SelectBinder select_binder(*this, context, *result, info); // if we expand select-list expressions, e.g., via UNNEST, then we need to possibly // adjust the column index of the already bound ORDER BY modifiers, and not only set their types diff --git a/src/planner/binder/query_node/bind_setop_node.cpp b/src/planner/binder/query_node/bind_setop_node.cpp index 0d8d4445f85f..b22d4e92e275 100644 --- a/src/planner/binder/query_node/bind_setop_node.cpp +++ b/src/planner/binder/query_node/bind_setop_node.cpp @@ -9,12 +9,12 @@ #include "duckdb/planner/expression_binder/order_binder.hpp" #include "duckdb/planner/query_node/bound_select_node.hpp" #include "duckdb/planner/query_node/bound_set_operation_node.hpp" +#include "duckdb/planner/select_bind_state.hpp" #include "duckdb/common/enum_util.hpp" namespace duckdb { -static void GatherAliases(BoundQueryNode &node, case_insensitive_map_t &aliases, - parsed_expression_map_t &expressions, const vector &reorder_idx) { +static void GatherAliases(BoundQueryNode &node, SelectBindState &bind_state, const vector &reorder_idx) { if (node.type == QueryNodeType::SET_OPERATION_NODE) { // setop, recurse auto &setop = node.Cast(); @@ -32,13 +32,13 @@ static void GatherAliases(BoundQueryNode &node, case_insensitive_map_t &a } // use new reorder index - GatherAliases(*setop.left, aliases, expressions, new_left_reorder_idx); - GatherAliases(*setop.right, aliases, expressions, new_right_reorder_idx); + GatherAliases(*setop.left, bind_state, new_left_reorder_idx); + GatherAliases(*setop.right, bind_state, new_right_reorder_idx); return; } - GatherAliases(*setop.left, aliases, expressions, reorder_idx); - GatherAliases(*setop.right, aliases, expressions, reorder_idx); + GatherAliases(*setop.left, bind_state, reorder_idx); + GatherAliases(*setop.right, bind_state, reorder_idx); } else { // query node D_ASSERT(node.type == QueryNodeType::SELECT_NODE); @@ -46,27 +46,27 @@ static void GatherAliases(BoundQueryNode &node, case_insensitive_map_t &a // fill the alias lists for (idx_t i = 0; i < select.names.size(); i++) { auto &name = select.names[i]; - auto &expr = select.original_expressions[i]; + auto &expr = select.bind_state.original_expressions[i]; // first check if the alias is already in there - auto entry = aliases.find(name); + auto entry = bind_state.alias_map.find(name); idx_t index = reorder_idx[i]; - if (entry == aliases.end()) { + if (entry == bind_state.alias_map.end()) { // the alias is not in there yet, just assign it - aliases[name] = index; + bind_state.alias_map[name] = index; } // now check if the node is already in the set of expressions - auto expr_entry = expressions.find(*expr); - if (expr_entry != expressions.end()) { + auto expr_entry = bind_state.projection_map.find(*expr); + if (expr_entry != bind_state.projection_map.end()) { // the node is in there // repeat the same as with the alias: if there is an ambiguity we insert "-1" if (expr_entry->second != index) { - expressions[*expr] = DConstants::INVALID_INDEX; + bind_state.projection_map[*expr] = DConstants::INVALID_INDEX; } } else { // not in there yet, just place it in there - expressions[*expr] = index; + bind_state.projection_map[*expr] = index; } } } @@ -230,22 +230,21 @@ unique_ptr Binder::BindNode(SetOperationNode &statement) { // we recursively visit the children of this node to extract aliases and expressions that can be referenced // in the ORDER BY - case_insensitive_map_t alias_map; - parsed_expression_map_t expression_map; + SelectBindState bind_state; if (result->setop_type == SetOperationType::UNION_BY_NAME) { - GatherAliases(*result->left, alias_map, expression_map, result->left_reorder_idx); - GatherAliases(*result->right, alias_map, expression_map, result->right_reorder_idx); + GatherAliases(*result->left, bind_state, result->left_reorder_idx); + GatherAliases(*result->right, bind_state, result->right_reorder_idx); } else { vector reorder_idx; for (idx_t i = 0; i < result->names.size(); i++) { reorder_idx.push_back(i); } - GatherAliases(*result, alias_map, expression_map, reorder_idx); + GatherAliases(*result, bind_state, reorder_idx); } // now we perform the actual resolution of the ORDER BY/DISTINCT expressions OrderBinder order_binder({result->left_binder.get(), result->right_binder.get()}, result->setop_index, - alias_map, expression_map, result->names.size()); + bind_state, result->names.size()); BindModifiers(order_binder, statement, *result); } diff --git a/src/planner/binder/statement/bind_create.cpp b/src/planner/binder/statement/bind_create.cpp index 69fdc4afca7a..c2d7650f8d32 100644 --- a/src/planner/binder/statement/bind_create.cpp +++ b/src/planner/binder/statement/bind_create.cpp @@ -42,6 +42,7 @@ #include "duckdb/catalog/duck_catalog.hpp" #include "duckdb/function/table/table_scan.hpp" #include "duckdb/parser/tableref/basetableref.hpp" +#include "duckdb/planner/select_bind_state.hpp" namespace duckdb { @@ -184,11 +185,10 @@ SchemaCatalogEntry &Binder::BindCreateFunctionInfo(CreateInfo &info) { // bind it to verify the function was defined correctly ErrorData error; - auto sel_node = make_uniq(); - auto group_info = make_uniq(); - SelectBinder binder(*this, context, *sel_node, *group_info); + BoundSelectNode sel_node; + BoundGroupInformation group_info; + SelectBinder binder(*this, context, sel_node, group_info); error = binder.Bind(expression, 0, false); - if (error.HasError()) { error.Throw(); } diff --git a/src/planner/expression_binder/base_select_binder.cpp b/src/planner/expression_binder/base_select_binder.cpp index 78ff2d903247..a90fd728bab8 100644 --- a/src/planner/expression_binder/base_select_binder.cpp +++ b/src/planner/expression_binder/base_select_binder.cpp @@ -8,17 +8,13 @@ #include "duckdb/planner/expression/bound_columnref_expression.hpp" #include "duckdb/planner/expression_binder/aggregate_binder.hpp" #include "duckdb/planner/query_node/bound_select_node.hpp" +#include "duckdb/planner/select_bind_state.hpp" namespace duckdb { BaseSelectBinder::BaseSelectBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, - BoundGroupInformation &info, case_insensitive_map_t alias_map) - : ExpressionBinder(binder, context), inside_window(false), node(node), info(info), alias_map(std::move(alias_map)) { -} - -BaseSelectBinder::BaseSelectBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, - BoundGroupInformation &info) - : BaseSelectBinder(binder, context, node, info, case_insensitive_map_t()) { + BoundGroupInformation &info, bool support_alias_binding) + : ExpressionBinder(binder, context), inside_window(false), support_alias_binding(support_alias_binding), node(node), info(info), bind_state(node.bind_state) { } BindResult BaseSelectBinder::BindExpression(unique_ptr &expr_ptr, idx_t depth, bool root_expression) { @@ -76,9 +72,9 @@ BindResult BaseSelectBinder::BindColumnRef(unique_ptr &expr_pt // binding failed // check in the alias map auto &colref = (expr_ptr.get())->Cast(); - if (!colref.IsQualified()) { - auto alias_entry = alias_map.find(colref.column_names[0]); - if (alias_entry != alias_map.end()) { + if (!colref.IsQualified() && support_alias_binding) { + auto alias_entry = bind_state.alias_map.find(colref.column_names[0]); + if (alias_entry != bind_state.alias_map.end()) { // found entry! auto index = alias_entry->second; if (index >= node.bound_column_count) { @@ -96,7 +92,7 @@ BindResult BaseSelectBinder::BindColumnRef(unique_ptr &expr_pt " This is not yet supported.", colref.column_names[0]); } - auto copied_expression = node.original_expressions[index]->Copy(); + auto copied_expression = node.bind_state.original_expressions[index]->Copy(); result = BindExpression(copied_expression, depth, false); return result; } @@ -146,8 +142,8 @@ BindResult BaseSelectBinder::BindGroup(ParsedExpression &expr, idx_t depth, idx_ } bool BaseSelectBinder::QualifyColumnAlias(const ColumnRefExpression &colref) { - if (!colref.IsQualified()) { - return alias_map.find(colref.column_names[0]) != alias_map.end(); + if (!colref.IsQualified() && support_alias_binding) { + return bind_state.alias_map.find(colref.column_names[0]) != bind_state.alias_map.end(); } return false; } diff --git a/src/planner/expression_binder/column_alias_binder.cpp b/src/planner/expression_binder/column_alias_binder.cpp index df83dee41496..95722820afa8 100644 --- a/src/planner/expression_binder/column_alias_binder.cpp +++ b/src/planner/expression_binder/column_alias_binder.cpp @@ -1,15 +1,15 @@ #include "duckdb/planner/expression_binder/column_alias_binder.hpp" #include "duckdb/parser/expression/columnref_expression.hpp" -#include "duckdb/planner/query_node/bound_select_node.hpp" #include "duckdb/common/string_util.hpp" #include "duckdb/planner/expression_binder.hpp" #include "duckdb/planner/binder.hpp" +#include "duckdb/planner/select_bind_state.hpp" namespace duckdb { -ColumnAliasBinder::ColumnAliasBinder(BoundSelectNode &node, const case_insensitive_map_t &alias_map) - : node(node), alias_map(alias_map), visited_select_indexes() { +ColumnAliasBinder::ColumnAliasBinder(const SelectBindState &bind_state) + : bind_state(bind_state), visited_select_indexes() { } bool ColumnAliasBinder::BindAlias(ExpressionBinder &enclosing_binder, unique_ptr &expr_ptr, @@ -24,8 +24,8 @@ bool ColumnAliasBinder::BindAlias(ExpressionBinder &enclosing_binder, unique_ptr } // We try to find the alias in the alias_map and return false, if no alias exists. - auto alias_entry = alias_map.find(expr.column_names[0]); - if (alias_entry == alias_map.end()) { + auto alias_entry = bind_state.alias_map.find(expr.column_names[0]); + if (alias_entry == bind_state.alias_map.end()) { return false; } @@ -35,7 +35,7 @@ bool ColumnAliasBinder::BindAlias(ExpressionBinder &enclosing_binder, unique_ptr } // We found an alias, so we copy the alias expression into this expression. - auto original_expr = node.original_expressions[alias_entry->second]->Copy(); + auto original_expr = bind_state.original_expressions[alias_entry->second]->Copy(); expr_ptr = std::move(original_expr); visited_select_indexes.insert(alias_entry->second); diff --git a/src/planner/expression_binder/group_binder.cpp b/src/planner/expression_binder/group_binder.cpp index f2fb36a53ba1..c86914ecfd7a 100644 --- a/src/planner/expression_binder/group_binder.cpp +++ b/src/planner/expression_binder/group_binder.cpp @@ -9,8 +9,8 @@ namespace duckdb { GroupBinder::GroupBinder(Binder &binder, ClientContext &context, SelectNode &node, idx_t group_index, - case_insensitive_map_t &alias_map, case_insensitive_map_t &group_alias_map) - : ExpressionBinder(binder, context), node(node), alias_map(alias_map), group_alias_map(group_alias_map), + const SelectBindState &bind_state, case_insensitive_map_t &group_alias_map) + : ExpressionBinder(binder, context), node(node), bind_state(bind_state), group_alias_map(group_alias_map), group_index(group_index) { } @@ -94,8 +94,8 @@ BindResult GroupBinder::BindColumnRef(ColumnRefExpression &colref) { // failed to bind the column and the node is the root expression with depth = 0 // check if refers to an alias in the select clause auto alias_name = colref.column_names[0]; - auto entry = alias_map.find(alias_name); - if (entry == alias_map.end()) { + auto entry = bind_state.alias_map.find(alias_name); + if (entry == bind_state.alias_map.end()) { // no matching alias found return result; } diff --git a/src/planner/expression_binder/having_binder.cpp b/src/planner/expression_binder/having_binder.cpp index 428ac15136ff..98268bb1e845 100644 --- a/src/planner/expression_binder/having_binder.cpp +++ b/src/planner/expression_binder/having_binder.cpp @@ -9,8 +9,8 @@ namespace duckdb { HavingBinder::HavingBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info, - case_insensitive_map_t &alias_map, AggregateHandling aggregate_handling) - : BaseSelectBinder(binder, context, node, info), column_alias_binder(node, alias_map), + const SelectBindState &bind_state, AggregateHandling aggregate_handling) + : BaseSelectBinder(binder, context, node, info, false), column_alias_binder(bind_state), aggregate_handling(aggregate_handling) { target_type = LogicalType(LogicalTypeId::BOOLEAN); } diff --git a/src/planner/expression_binder/order_binder.cpp b/src/planner/expression_binder/order_binder.cpp index c5fde60f0a4b..495a6a81aa28 100644 --- a/src/planner/expression_binder/order_binder.cpp +++ b/src/planner/expression_binder/order_binder.cpp @@ -13,15 +13,12 @@ namespace duckdb { -OrderBinder::OrderBinder(vector binders, idx_t projection_index, case_insensitive_map_t &alias_map, - parsed_expression_map_t &projection_map, idx_t max_count) +OrderBinder::OrderBinder(vector binders, idx_t projection_index, SelectBindState &bind_state, idx_t max_count) : binders(std::move(binders)), projection_index(projection_index), max_count(max_count), extra_list(nullptr), - alias_map(alias_map), projection_map(projection_map) { + bind_state(bind_state) { } -OrderBinder::OrderBinder(vector binders, idx_t projection_index, SelectNode &node, - case_insensitive_map_t &alias_map, parsed_expression_map_t &projection_map) - : binders(std::move(binders)), projection_index(projection_index), alias_map(alias_map), - projection_map(projection_map) { +OrderBinder::OrderBinder(vector binders, idx_t projection_index, SelectNode &node, SelectBindState &bind_state) + : binders(std::move(binders)), projection_index(projection_index), bind_state(bind_state) { this->max_count = node.select_list.size(); this->extra_list = &node.select_list; } @@ -42,7 +39,7 @@ unique_ptr OrderBinder::CreateExtraReference(unique_ptrsize(); + bind_state.projection_map[*expr] = extra_list->size(); auto result = CreateProjectionReference(*expr, extra_list->size()); extra_list->push_back(std::move(expr)); return result; @@ -86,8 +83,8 @@ unique_ptr OrderBinder::Bind(unique_ptr expr) { break; } // check the alias list - auto entry = alias_map.find(colref.column_names[0]); - if (entry != alias_map.end()) { + auto entry = bind_state.alias_map.find(colref.column_names[0]); + if (entry != bind_state.alias_map.end()) { // it does! point it to that entry return CreateProjectionReference(*expr, entry->second); } @@ -130,8 +127,8 @@ unique_ptr OrderBinder::Bind(unique_ptr expr) { ExpressionBinder::QualifyColumnNames(*binder, expr); } // first check if the ORDER BY clause already points to an entry in the projection list - auto entry = projection_map.find(*expr); - if (entry != projection_map.end()) { + auto entry = bind_state.projection_map.find(*expr); + if (entry != bind_state.projection_map.end()) { if (entry->second == DConstants::INVALID_INDEX) { throw BinderException("Ambiguous reference to column"); } diff --git a/src/planner/expression_binder/qualify_binder.cpp b/src/planner/expression_binder/qualify_binder.cpp index ff40ae9a2d1c..20ee60c3cdc8 100644 --- a/src/planner/expression_binder/qualify_binder.cpp +++ b/src/planner/expression_binder/qualify_binder.cpp @@ -10,8 +10,8 @@ namespace duckdb { QualifyBinder::QualifyBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info, - case_insensitive_map_t &alias_map) - : BaseSelectBinder(binder, context, node, info), column_alias_binder(node, alias_map) { + const SelectBindState &bind_state) + : BaseSelectBinder(binder, context, node, info, false), column_alias_binder(bind_state) { target_type = LogicalType(LogicalTypeId::BOOLEAN); } diff --git a/src/planner/expression_binder/select_binder.cpp b/src/planner/expression_binder/select_binder.cpp index d83b2e6c2f4a..33da0642c780 100644 --- a/src/planner/expression_binder/select_binder.cpp +++ b/src/planner/expression_binder/select_binder.cpp @@ -2,13 +2,8 @@ namespace duckdb { -SelectBinder::SelectBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info, - case_insensitive_map_t alias_map) - : BaseSelectBinder(binder, context, node, info, std::move(alias_map)) { -} - SelectBinder::SelectBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info) - : SelectBinder(binder, context, node, info, case_insensitive_map_t()) { + : BaseSelectBinder(binder, context, node, info) { } } // namespace duckdb From 0e5eff7caad6436ab09b615116a4e4c49e6f3e04 Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 4 Apr 2024 14:09:45 +0200 Subject: [PATCH 159/603] added test for the reconstructing of the last_value + counter --- .../catalog_entry/sequence_catalog_entry.cpp | 19 +-- .../sequence/test_duckdb_sequences.test | 129 ++++++++++++++++++ 2 files changed, 136 insertions(+), 12 deletions(-) diff --git a/src/catalog/catalog_entry/sequence_catalog_entry.cpp b/src/catalog/catalog_entry/sequence_catalog_entry.cpp index 550e8bf0d444..e61d216ee2fe 100644 --- a/src/catalog/catalog_entry/sequence_catalog_entry.cpp +++ b/src/catalog/catalog_entry/sequence_catalog_entry.cpp @@ -28,11 +28,8 @@ static int64_t LastValue(const CreateSequenceInfo &info) { if (increase) { while (to_simulate > 0) { auto maximum_increase = info.max_value - current; - uint64_t max_uses = maximum_increase / info.increment; - if (maximum_increase % info.increment != 0) { - // i.e 300 / 170 == 1.76, 2 would overflow - max_uses++; - } + uint64_t max_uses = 1 + (maximum_increase / info.increment); + if (to_simulate >= max_uses) { // Uses would overflow, cycle around to_simulate -= max_uses; @@ -44,20 +41,18 @@ static int64_t LastValue(const CreateSequenceInfo &info) { } } } else { + auto increment = info.increment * -1; while (to_simulate > 0) { auto maximum_decrease = current - info.min_value; - uint64_t max_uses = maximum_decrease / info.increment; - if (maximum_decrease % info.increment != 0) { - // If there's a remainder, one more decrement would overflow - max_uses++; - } + uint64_t max_uses = 1 + (maximum_decrease / increment); + if (to_simulate >= max_uses) { // Decrementing would overflow, cycle around to_simulate -= max_uses; current = info.max_value; result = current; } else { - result = current - (info.increment * to_simulate); + result = current - (increment * to_simulate); break; } } @@ -114,8 +109,8 @@ int64_t SequenceCatalogEntry::CurrentValue() { int64_t SequenceCatalogEntry::NextValue(DuckTransaction &transaction) { lock_guard seqlock(lock); int64_t result; - result = data.counter; bool overflow = !TryAddOperator::Operation(data.counter, data.increment, data.counter); + result = data.counter; if (data.cycle) { if (overflow) { data.counter = data.increment < 0 ? data.max_value : data.min_value; diff --git a/test/sql/catalog/sequence/test_duckdb_sequences.test b/test/sql/catalog/sequence/test_duckdb_sequences.test index 6df8643c3187..66cc21cc27d0 100644 --- a/test/sql/catalog/sequence/test_duckdb_sequences.test +++ b/test/sql/catalog/sequence/test_duckdb_sequences.test @@ -63,3 +63,132 @@ query IIIIII select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); ---- 20316564 1 9223372036854775807 1 false 20316564 + +# Reached max value (no cycle) + +statement ok +drop sequence seq1 cascade; + +statement ok +create sequence seq1 increment 2 minvalue 3 maxvalue 8 start 5 + +statement ok +create table tbl (a integer DEFAULT nextval('seq1')) + +statement ok +insert into tbl values(default); + +statement ok +insert into tbl values(default); + +query IIIIII +select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); +---- +5 3 8 2 false 7 + +statement error +insert into tbl values(default); +---- +Sequence Error: nextval: reached maximum value of sequence "seq1" (8) + +query IIIIII +select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); +---- +5 3 8 2 false 7 + +# Reached min value (no cycle) + +statement ok +drop sequence seq1 cascade; + +statement ok +create sequence seq1 increment -2 minvalue 3 maxvalue 8 start 5 + +statement ok +create table tbl (a integer DEFAULT nextval('seq1')) + +statement ok +insert into tbl values(default); + +statement ok +insert into tbl values(default); + +query IIIIII +select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); +---- +5 3 8 -2 false 3 + +statement error +insert into tbl values(default); +---- +Sequence Error: nextval: reached minimum value of sequence "seq1" (3) + +query IIIIII +select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); +---- +5 3 8 -2 false 3 + +# Cycle (positive) + +statement ok +drop sequence seq1 cascade; + +statement ok +create sequence seq1 increment 2 minvalue 3 maxvalue 8 start 5 cycle + +statement ok +create table tbl (a integer DEFAULT nextval('seq1')) + +# value is 5 here +statement ok +insert into tbl values(default); + +# value is 7 here +statement ok +insert into tbl values(default); + +query IIIIII +select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); +---- +5 3 8 2 true 7 + +# value is min_value (3) +statement ok +insert into tbl values(default); + +query IIIIII +select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); +---- +5 3 8 2 true 3 + +# Cycle (negative) + +statement ok +drop sequence seq1 cascade; + +statement ok +create sequence seq1 increment -2 minvalue 3 maxvalue 8 start 5 cycle + +statement ok +create table tbl (a integer DEFAULT nextval('seq1')) + +# value is 5 +statement ok +insert into tbl values(default); + +# value is 3 +statement ok +insert into tbl values(default); + +query IIIIII +select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); +---- +5 3 8 -2 true 3 + +statement ok +insert into tbl values(default); + +query IIIIII +select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); +---- +5 3 8 -2 true 8 From 69efd0a0a74267ef78863e7a80868b401bde03d0 Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 4 Apr 2024 14:34:05 +0200 Subject: [PATCH 160/603] reconstructing last_value and counter --- .../catalog_entry/sequence_catalog_entry.cpp | 26 ++++++++----------- .../sequence/test_duckdb_sequences.test | 3 ++- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/catalog/catalog_entry/sequence_catalog_entry.cpp b/src/catalog/catalog_entry/sequence_catalog_entry.cpp index e61d216ee2fe..7c9fd41b9953 100644 --- a/src/catalog/catalog_entry/sequence_catalog_entry.cpp +++ b/src/catalog/catalog_entry/sequence_catalog_entry.cpp @@ -13,15 +13,9 @@ namespace duckdb { -static int64_t LastValue(const CreateSequenceInfo &info) { - if (info.usage_count == 0) { - // last_value was never set, we don't need to recreate it - return info.start_value; - } - int64_t result = 0; - auto to_simulate = info.usage_count; - // The first use is always just initializing the counter to start_value - to_simulate--; +static int64_t ReconstructValue(const CreateSequenceInfo &info, idx_t usage_count) { + int64_t result = info.start_value; + auto to_simulate = usage_count; if (info.cycle) { auto current = info.start_value; auto increase = info.increment > 0; @@ -68,11 +62,13 @@ static int64_t LastValue(const CreateSequenceInfo &info) { // if the sequence was serialized to disk the start_value // was updated to the value of the last_value before serializing, keeping the state of the sequence. SequenceData::SequenceData(CreateSequenceInfo &info) - : usage_count(info.usage_count), counter(0), last_value(0), increment(info.increment), - start_value(info.start_value), min_value(info.min_value), max_value(info.max_value), cycle(info.cycle) { - auto reconstructed_last_value = LastValue(info); - last_value = reconstructed_last_value; - counter = reconstructed_last_value; + : usage_count(info.usage_count), counter(ReconstructValue(info, info.usage_count)), last_value(info.start_value), + increment(info.increment), start_value(info.start_value), min_value(info.min_value), max_value(info.max_value), + cycle(info.cycle) { + + if (info.usage_count) { + last_value = ReconstructValue(info, info.usage_count - 1); + } } SequenceCatalogEntry::SequenceCatalogEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateSequenceInfo &info) @@ -109,8 +105,8 @@ int64_t SequenceCatalogEntry::CurrentValue() { int64_t SequenceCatalogEntry::NextValue(DuckTransaction &transaction) { lock_guard seqlock(lock); int64_t result; - bool overflow = !TryAddOperator::Operation(data.counter, data.increment, data.counter); result = data.counter; + bool overflow = !TryAddOperator::Operation(data.counter, data.increment, data.counter); if (data.cycle) { if (overflow) { data.counter = data.increment < 0 ? data.max_value : data.min_value; diff --git a/test/sql/catalog/sequence/test_duckdb_sequences.test b/test/sql/catalog/sequence/test_duckdb_sequences.test index 66cc21cc27d0..865e0f69e598 100644 --- a/test/sql/catalog/sequence/test_duckdb_sequences.test +++ b/test/sql/catalog/sequence/test_duckdb_sequences.test @@ -28,12 +28,13 @@ select start_value, min_value, max_value, increment_by, cycle, last_value from d statement ok insert into tbl values(default); -# last_value is no longer NULL +# value is 0 query IIIIII select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); ---- 0 0 9223372036854775807 1 false 0 +# value is 1 statement ok insert into tbl values(default); From 41b91fa5b99d43816ec2697f619d834ca236bbfb Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 4 Apr 2024 14:35:06 +0200 Subject: [PATCH 161/603] Clean up BaseSelectBinder - move logic that is unique to SELECT list to SelectBinder --- .../expression_binder/base_select_binder.hpp | 10 ++-- .../expression_binder/having_binder.hpp | 7 +-- .../expression_binder/qualify_binder.hpp | 5 +- .../expression_binder/select_binder.hpp | 4 ++ .../expression_binder/base_select_binder.cpp | 51 ++----------------- .../expression_binder/having_binder.cpp | 22 ++------ .../expression_binder/qualify_binder.cpp | 22 +------- .../expression_binder/select_binder.cpp | 45 ++++++++++++++++ 8 files changed, 66 insertions(+), 100 deletions(-) diff --git a/src/include/duckdb/planner/expression_binder/base_select_binder.hpp b/src/include/duckdb/planner/expression_binder/base_select_binder.hpp index 777b3514e9be..1fb875c77b1e 100644 --- a/src/include/duckdb/planner/expression_binder/base_select_binder.hpp +++ b/src/include/duckdb/planner/expression_binder/base_select_binder.hpp @@ -28,7 +28,7 @@ struct BoundGroupInformation { //! functions. class BaseSelectBinder : public ExpressionBinder { public: - BaseSelectBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info, bool support_alias_binding = true); + BaseSelectBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info); bool BoundAggregates() { return bound_aggregate; @@ -46,23 +46,19 @@ class BaseSelectBinder : public ExpressionBinder { bool inside_window; bool bound_aggregate = false; - bool support_alias_binding; BoundSelectNode &node; BoundGroupInformation &info; - const SelectBindState &bind_state; protected: - BindResult BindColumnRef(unique_ptr &expr_ptr, idx_t depth); BindResult BindGroupingFunction(OperatorExpression &op, idx_t depth) override; //! Binds a WINDOW expression and returns the result. - BindResult BindWindow(WindowExpression &expr, idx_t depth); + virtual BindResult BindWindow(WindowExpression &expr, idx_t depth); + virtual BindResult BindColumnRef(unique_ptr &expr_ptr, idx_t depth, bool root_expression); idx_t TryBindGroup(ParsedExpression &expr); BindResult BindGroup(ParsedExpression &expr, idx_t depth, idx_t group_index); - - bool QualifyColumnAlias(const ColumnRefExpression &colref) override; }; } // namespace duckdb diff --git a/src/include/duckdb/planner/expression_binder/having_binder.hpp b/src/include/duckdb/planner/expression_binder/having_binder.hpp index b49b4f454af6..c0738861efbf 100644 --- a/src/include/duckdb/planner/expression_binder/having_binder.hpp +++ b/src/include/duckdb/planner/expression_binder/having_binder.hpp @@ -21,12 +21,9 @@ class HavingBinder : public BaseSelectBinder { const SelectBindState &bind_state, AggregateHandling aggregate_handling); protected: - BindResult BindExpression(unique_ptr &expr_ptr, idx_t depth, - bool root_expression = false) override; - + BindResult BindWindow(WindowExpression &expr, idx_t depth) override; + BindResult BindColumnRef(unique_ptr &expr_ptr, idx_t depth, bool root_expression) override; private: - BindResult BindColumnRef(unique_ptr &expr_ptr, idx_t depth, bool root_expression); - ColumnAliasBinder column_alias_binder; AggregateHandling aggregate_handling; }; diff --git a/src/include/duckdb/planner/expression_binder/qualify_binder.hpp b/src/include/duckdb/planner/expression_binder/qualify_binder.hpp index 79aec66d0461..8d6e5577bcc5 100644 --- a/src/include/duckdb/planner/expression_binder/qualify_binder.hpp +++ b/src/include/duckdb/planner/expression_binder/qualify_binder.hpp @@ -21,12 +21,9 @@ class QualifyBinder : public BaseSelectBinder { const SelectBindState &bind_state); protected: - BindResult BindExpression(unique_ptr &expr_ptr, idx_t depth, - bool root_expression = false) override; + BindResult BindColumnRef(unique_ptr &expr_ptr, idx_t depth, bool root_expression) override; private: - BindResult BindColumnRef(unique_ptr &expr_ptr, idx_t depth, bool root_expression); - ColumnAliasBinder column_alias_binder; }; diff --git a/src/include/duckdb/planner/expression_binder/select_binder.hpp b/src/include/duckdb/planner/expression_binder/select_binder.hpp index 86c88015803a..64e5914130ea 100644 --- a/src/include/duckdb/planner/expression_binder/select_binder.hpp +++ b/src/include/duckdb/planner/expression_binder/select_binder.hpp @@ -19,7 +19,11 @@ class SelectBinder : public BaseSelectBinder { protected: BindResult BindUnnest(FunctionExpression &function, idx_t depth, bool root_expression) override; + BindResult BindColumnRef(unique_ptr &expr_ptr, idx_t depth, bool root_expression) override; + bool QualifyColumnAlias(const ColumnRefExpression &colref) override; + +protected: idx_t unnest_level = 0; }; diff --git a/src/planner/expression_binder/base_select_binder.cpp b/src/planner/expression_binder/base_select_binder.cpp index a90fd728bab8..4930c2d6cc9b 100644 --- a/src/planner/expression_binder/base_select_binder.cpp +++ b/src/planner/expression_binder/base_select_binder.cpp @@ -13,8 +13,8 @@ namespace duckdb { BaseSelectBinder::BaseSelectBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, - BoundGroupInformation &info, bool support_alias_binding) - : ExpressionBinder(binder, context), inside_window(false), support_alias_binding(support_alias_binding), node(node), info(info), bind_state(node.bind_state) { + BoundGroupInformation &info) + : ExpressionBinder(binder, context), inside_window(false), node(node), info(info) { } BindResult BaseSelectBinder::BindExpression(unique_ptr &expr_ptr, idx_t depth, bool root_expression) { @@ -26,7 +26,7 @@ BindResult BaseSelectBinder::BindExpression(unique_ptr &expr_p } switch (expr.expression_class) { case ExpressionClass::COLUMN_REF: - return BindColumnRef(expr_ptr, depth); + return BindColumnRef(expr_ptr, depth, root_expression); case ExpressionClass::DEFAULT: return BindResult("SELECT clause cannot contain DEFAULT clause"); case ExpressionClass::WINDOW: @@ -63,42 +63,8 @@ idx_t BaseSelectBinder::TryBindGroup(ParsedExpression &expr) { return DConstants::INVALID_INDEX; } -BindResult BaseSelectBinder::BindColumnRef(unique_ptr &expr_ptr, idx_t depth) { - // first try to bind the column reference regularly - auto result = ExpressionBinder::BindExpression(expr_ptr, depth); - if (!result.HasError()) { - return result; - } - // binding failed - // check in the alias map - auto &colref = (expr_ptr.get())->Cast(); - if (!colref.IsQualified() && support_alias_binding) { - auto alias_entry = bind_state.alias_map.find(colref.column_names[0]); - if (alias_entry != bind_state.alias_map.end()) { - // found entry! - auto index = alias_entry->second; - if (index >= node.bound_column_count) { - throw BinderException("Column \"%s\" referenced that exists in the SELECT clause - but this column " - "cannot be referenced before it is defined", - colref.column_names[0]); - } - if (node.select_list[index]->IsVolatile()) { - throw BinderException("Alias \"%s\" referenced in a SELECT clause - but the expression has side " - "effects. This is not yet supported.", - colref.column_names[0]); - } - if (node.select_list[index]->HasSubquery()) { - throw BinderException("Alias \"%s\" referenced in a SELECT clause - but the expression has a subquery." - " This is not yet supported.", - colref.column_names[0]); - } - auto copied_expression = node.bind_state.original_expressions[index]->Copy(); - result = BindExpression(copied_expression, depth, false); - return result; - } - } - // entry was not found in the alias map: return the original error - return result; +BindResult BaseSelectBinder::BindColumnRef(unique_ptr &expr_ptr, idx_t depth, bool root_expression) { + return ExpressionBinder::BindExpression(expr_ptr, depth); } BindResult BaseSelectBinder::BindGroupingFunction(OperatorExpression &op, idx_t depth) { @@ -141,11 +107,4 @@ BindResult BaseSelectBinder::BindGroup(ParsedExpression &expr, idx_t depth, idx_ } } -bool BaseSelectBinder::QualifyColumnAlias(const ColumnRefExpression &colref) { - if (!colref.IsQualified() && support_alias_binding) { - return bind_state.alias_map.find(colref.column_names[0]) != bind_state.alias_map.end(); - } - return false; -} - } // namespace duckdb diff --git a/src/planner/expression_binder/having_binder.cpp b/src/planner/expression_binder/having_binder.cpp index 98268bb1e845..8536147b1f0f 100644 --- a/src/planner/expression_binder/having_binder.cpp +++ b/src/planner/expression_binder/having_binder.cpp @@ -10,13 +10,12 @@ namespace duckdb { HavingBinder::HavingBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info, const SelectBindState &bind_state, AggregateHandling aggregate_handling) - : BaseSelectBinder(binder, context, node, info, false), column_alias_binder(bind_state), + : BaseSelectBinder(binder, context, node, info), column_alias_binder(bind_state), aggregate_handling(aggregate_handling) { target_type = LogicalType(LogicalTypeId::BOOLEAN); } BindResult HavingBinder::BindColumnRef(unique_ptr &expr_ptr, idx_t depth, bool root_expression) { - // Keep the original column name to return a meaningful error message. auto column_name = expr_ptr->Cast().GetColumnName(); @@ -37,7 +36,7 @@ BindResult HavingBinder::BindColumnRef(unique_ptr &expr_ptr, i "Having clause cannot reference column \"%s\" in correlated subquery and group by all", column_name); } - auto expr = duckdb::BaseSelectBinder::BindExpression(expr_ptr, depth); + auto expr = duckdb::BaseSelectBinder::BindColumnRef(expr_ptr, depth, root_expression); if (expr.HasError()) { return expr; } @@ -54,21 +53,8 @@ BindResult HavingBinder::BindColumnRef(unique_ptr &expr_ptr, i "column %s must appear in the GROUP BY clause or be used in an aggregate function", column_name)); } -BindResult HavingBinder::BindExpression(unique_ptr &expr_ptr, idx_t depth, bool root_expression) { - auto &expr = *expr_ptr; - // check if the expression binds to one of the groups - auto group_index = TryBindGroup(expr); - if (group_index != DConstants::INVALID_INDEX) { - return BindGroup(expr, depth, group_index); - } - switch (expr.expression_class) { - case ExpressionClass::WINDOW: - return BindResult("HAVING clause cannot contain window functions!"); - case ExpressionClass::COLUMN_REF: - return BindColumnRef(expr_ptr, depth, root_expression); - default: - return duckdb::BaseSelectBinder::BindExpression(expr_ptr, depth); - } +BindResult HavingBinder::BindWindow(WindowExpression &expr, idx_t depth) { + return BindResult("HAVING clause cannot contain window functions!"); } } // namespace duckdb diff --git a/src/planner/expression_binder/qualify_binder.cpp b/src/planner/expression_binder/qualify_binder.cpp index 20ee60c3cdc8..c6e177139ee1 100644 --- a/src/planner/expression_binder/qualify_binder.cpp +++ b/src/planner/expression_binder/qualify_binder.cpp @@ -11,13 +11,12 @@ namespace duckdb { QualifyBinder::QualifyBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info, const SelectBindState &bind_state) - : BaseSelectBinder(binder, context, node, info, false), column_alias_binder(bind_state) { + : BaseSelectBinder(binder, context, node, info), column_alias_binder(bind_state) { target_type = LogicalType(LogicalTypeId::BOOLEAN); } BindResult QualifyBinder::BindColumnRef(unique_ptr &expr_ptr, idx_t depth, bool root_expression) { - - auto result = duckdb::BaseSelectBinder::BindExpression(expr_ptr, depth); + auto result = duckdb::BaseSelectBinder::BindColumnRef(expr_ptr, depth, root_expression); if (!result.HasError()) { return result; } @@ -36,21 +35,4 @@ BindResult QualifyBinder::BindColumnRef(unique_ptr &expr_ptr, StringUtil::Format("Referenced column %s not found in FROM clause and can't find in alias map.", expr_string)); } -BindResult QualifyBinder::BindExpression(unique_ptr &expr_ptr, idx_t depth, bool root_expression) { - auto &expr = *expr_ptr; - // check if the expression binds to one of the groups - auto group_index = TryBindGroup(expr); - if (group_index != DConstants::INVALID_INDEX) { - return BindGroup(expr, depth, group_index); - } - switch (expr.expression_class) { - case ExpressionClass::WINDOW: - return BindWindow(expr.Cast(), depth); - case ExpressionClass::COLUMN_REF: - return BindColumnRef(expr_ptr, depth, root_expression); - default: - return duckdb::BaseSelectBinder::BindExpression(expr_ptr, depth); - } -} - } // namespace duckdb diff --git a/src/planner/expression_binder/select_binder.cpp b/src/planner/expression_binder/select_binder.cpp index 33da0642c780..a6b29c56a8c0 100644 --- a/src/planner/expression_binder/select_binder.cpp +++ b/src/planner/expression_binder/select_binder.cpp @@ -6,4 +6,49 @@ SelectBinder::SelectBinder(Binder &binder, ClientContext &context, BoundSelectNo : BaseSelectBinder(binder, context, node, info) { } +BindResult SelectBinder::BindColumnRef(unique_ptr &expr_ptr, idx_t depth, bool root_expression) { + // first try to bind the column reference regularly + auto result = BaseSelectBinder::BindColumnRef(expr_ptr, depth, root_expression); + if (!result.HasError()) { + return result; + } + // binding failed + // check in the alias map + auto &colref = (expr_ptr.get())->Cast(); + if (!colref.IsQualified()) { + auto alias_entry = node.bind_state.alias_map.find(colref.column_names[0]); + if (alias_entry != node.bind_state.alias_map.end()) { + // found entry! + auto index = alias_entry->second; + if (index >= node.bound_column_count) { + throw BinderException("Column \"%s\" referenced that exists in the SELECT clause - but this column " + "cannot be referenced before it is defined", + colref.column_names[0]); + } + if (node.select_list[index]->IsVolatile()) { + throw BinderException("Alias \"%s\" referenced in a SELECT clause - but the expression has side " + "effects. This is not yet supported.", + colref.column_names[0]); + } + if (node.select_list[index]->HasSubquery()) { + throw BinderException("Alias \"%s\" referenced in a SELECT clause - but the expression has a subquery." + " This is not yet supported.", + colref.column_names[0]); + } + auto copied_expression = node.bind_state.original_expressions[index]->Copy(); + result = BindExpression(copied_expression, depth, false); + return result; + } + } + // entry was not found in the alias map: return the original error + return result; +} + +bool SelectBinder::QualifyColumnAlias(const ColumnRefExpression &colref) { + if (!colref.IsQualified()) { + return node.bind_state.alias_map.find(colref.column_names[0]) != node.bind_state.alias_map.end(); + } + return false; +} + } // namespace duckdb From a04637b5313ae406725e4fa97f1b747c5812ba87 Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 4 Apr 2024 14:35:46 +0200 Subject: [PATCH 162/603] add additional tests --- .../catalog_entry/sequence_catalog_entry.cpp | 3 -- .../catalog/sequence/sequence_currvalue.test | 33 +++++++++++++++++++ .../catalog/sequence/sequence_single_use.test | 10 ++++++ 3 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 test/sql/catalog/sequence/sequence_currvalue.test create mode 100644 test/sql/catalog/sequence/sequence_single_use.test diff --git a/src/catalog/catalog_entry/sequence_catalog_entry.cpp b/src/catalog/catalog_entry/sequence_catalog_entry.cpp index 7c9fd41b9953..239fd21a0bac 100644 --- a/src/catalog/catalog_entry/sequence_catalog_entry.cpp +++ b/src/catalog/catalog_entry/sequence_catalog_entry.cpp @@ -58,9 +58,6 @@ static int64_t ReconstructValue(const CreateSequenceInfo &info, idx_t usage_coun } } -// NOTE: usage_count is explicitly set to 0, -// if the sequence was serialized to disk the start_value -// was updated to the value of the last_value before serializing, keeping the state of the sequence. SequenceData::SequenceData(CreateSequenceInfo &info) : usage_count(info.usage_count), counter(ReconstructValue(info, info.usage_count)), last_value(info.start_value), increment(info.increment), start_value(info.start_value), min_value(info.min_value), max_value(info.max_value), diff --git a/test/sql/catalog/sequence/sequence_currvalue.test b/test/sql/catalog/sequence/sequence_currvalue.test new file mode 100644 index 000000000000..6efa36dbaa3f --- /dev/null +++ b/test/sql/catalog/sequence/sequence_currvalue.test @@ -0,0 +1,33 @@ +# name: test/sql/catalog/sequence/sequence_currvalue.test +# group: [sequence] + +load __TEST_DIR__/sequence_currvalue.db + +statement ok +create sequence seq; + +statement ok +create table tbl (i integer default nextval('seq')) + +statement error +select currval('seq') +---- +Sequence Error: currval: sequence is not yet defined in this session + +statement ok +insert into tbl values(default); + +query I +select currval('seq') +---- +1 + +restart + +# We can't replicate this behavior yet +mode skip + +statement error +select currval('seq') +---- +Sequence Error: currval: sequence is not yet defined in this session diff --git a/test/sql/catalog/sequence/sequence_single_use.test b/test/sql/catalog/sequence/sequence_single_use.test new file mode 100644 index 000000000000..1b4b9086400b --- /dev/null +++ b/test/sql/catalog/sequence/sequence_single_use.test @@ -0,0 +1,10 @@ +# name: test/sql/catalog/sequence/sequence_single_use.test +# group: [sequence] + +statement ok +create sequence seq1 increment 2 minvalue 3 maxvalue 8 start 8; + +query I +select nextval('seq1'); +---- +8 From a3c46b467fd98f0d8a68f870f03d7cb7b306b89f Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 4 Apr 2024 14:46:12 +0200 Subject: [PATCH 163/603] Add SelectBindState::BindAlias --- .../planner/expression_binder/column_alias_binder.hpp | 4 ++-- .../duckdb/planner/expression_binder/group_binder.hpp | 4 ++-- .../duckdb/planner/expression_binder/having_binder.hpp | 2 +- .../duckdb/planner/expression_binder/qualify_binder.hpp | 3 +-- .../{ => expression_binder}/select_bind_state.hpp | 5 ++++- .../duckdb/planner/query_node/bound_select_node.hpp | 2 +- src/planner/binder/query_node/bind_select_node.cpp | 6 +++--- src/planner/binder/query_node/bind_setop_node.cpp | 2 +- src/planner/binder/statement/bind_create.cpp | 2 +- src/planner/expression_binder/CMakeLists.txt | 1 + src/planner/expression_binder/base_select_binder.cpp | 2 +- src/planner/expression_binder/column_alias_binder.cpp | 8 +++----- src/planner/expression_binder/group_binder.cpp | 2 +- src/planner/expression_binder/having_binder.cpp | 4 ++-- src/planner/expression_binder/qualify_binder.cpp | 5 ++--- src/planner/expression_binder/select_bind_state.cpp | 9 +++++++++ 16 files changed, 35 insertions(+), 26 deletions(-) rename src/include/duckdb/planner/{ => expression_binder}/select_bind_state.hpp (89%) create mode 100644 src/planner/expression_binder/select_bind_state.cpp diff --git a/src/include/duckdb/planner/expression_binder/column_alias_binder.hpp b/src/include/duckdb/planner/expression_binder/column_alias_binder.hpp index 056c17b180f6..4f1e2c34abda 100644 --- a/src/include/duckdb/planner/expression_binder/column_alias_binder.hpp +++ b/src/include/duckdb/planner/expression_binder/column_alias_binder.hpp @@ -19,13 +19,13 @@ struct SelectBindState; //! A helper binder for WhereBinder and HavingBinder which support alias as a columnref. class ColumnAliasBinder { public: - explicit ColumnAliasBinder(const SelectBindState &bind_state); + explicit ColumnAliasBinder(SelectBindState &bind_state); bool BindAlias(ExpressionBinder &enclosing_binder, unique_ptr &expr_ptr, idx_t depth, bool root_expression, BindResult &result); private: - const SelectBindState &bind_state; + SelectBindState &bind_state; unordered_set visited_select_indexes; }; diff --git a/src/include/duckdb/planner/expression_binder/group_binder.hpp b/src/include/duckdb/planner/expression_binder/group_binder.hpp index 40a4c7a3a455..1c398daac4d0 100644 --- a/src/include/duckdb/planner/expression_binder/group_binder.hpp +++ b/src/include/duckdb/planner/expression_binder/group_binder.hpp @@ -20,7 +20,7 @@ struct SelectBindState; class GroupBinder : public ExpressionBinder { public: GroupBinder(Binder &binder, ClientContext &context, SelectNode &node, idx_t group_index, - const SelectBindState &bind_state, case_insensitive_map_t &group_alias_map); + SelectBindState &bind_state, case_insensitive_map_t &group_alias_map); //! The unbound root expression unique_ptr unbound_expression; @@ -37,7 +37,7 @@ class GroupBinder : public ExpressionBinder { BindResult BindConstant(ConstantExpression &expr); SelectNode &node; - const SelectBindState &bind_state; + SelectBindState &bind_state; case_insensitive_map_t &group_alias_map; unordered_set used_aliases; diff --git a/src/include/duckdb/planner/expression_binder/having_binder.hpp b/src/include/duckdb/planner/expression_binder/having_binder.hpp index c0738861efbf..fb5155f43d34 100644 --- a/src/include/duckdb/planner/expression_binder/having_binder.hpp +++ b/src/include/duckdb/planner/expression_binder/having_binder.hpp @@ -18,7 +18,7 @@ namespace duckdb { class HavingBinder : public BaseSelectBinder { public: HavingBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info, - const SelectBindState &bind_state, AggregateHandling aggregate_handling); + AggregateHandling aggregate_handling); protected: BindResult BindWindow(WindowExpression &expr, idx_t depth) override; diff --git a/src/include/duckdb/planner/expression_binder/qualify_binder.hpp b/src/include/duckdb/planner/expression_binder/qualify_binder.hpp index 8d6e5577bcc5..d7cbbf5b7cbb 100644 --- a/src/include/duckdb/planner/expression_binder/qualify_binder.hpp +++ b/src/include/duckdb/planner/expression_binder/qualify_binder.hpp @@ -17,8 +17,7 @@ struct SelectBindState; //! The QUALIFY binder is responsible for binding an expression within the QUALIFY clause of a SQL statement class QualifyBinder : public BaseSelectBinder { public: - QualifyBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info, - const SelectBindState &bind_state); + QualifyBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info); protected: BindResult BindColumnRef(unique_ptr &expr_ptr, idx_t depth, bool root_expression) override; diff --git a/src/include/duckdb/planner/select_bind_state.hpp b/src/include/duckdb/planner/expression_binder/select_bind_state.hpp similarity index 89% rename from src/include/duckdb/planner/select_bind_state.hpp rename to src/include/duckdb/planner/expression_binder/select_bind_state.hpp index e62e9ad43c67..89cd00bbd317 100644 --- a/src/include/duckdb/planner/select_bind_state.hpp +++ b/src/include/duckdb/planner/expression_binder/select_bind_state.hpp @@ -1,7 +1,7 @@ //===----------------------------------------------------------------------===// // DuckDB // -// duckdb/planner/select_bind_state.hpp +// duckdb/planner/expression_binder/select_bind_state.hpp // // //===----------------------------------------------------------------------===// @@ -25,6 +25,9 @@ struct SelectBindState { //! The original unparsed expressions. This is exported after binding, because the binding might change the //! expressions (e.g. when a * clause is present) vector> original_expressions; + +public: + unique_ptr BindAlias(idx_t index); }; } // namespace duckdb diff --git a/src/include/duckdb/planner/query_node/bound_select_node.hpp b/src/include/duckdb/planner/query_node/bound_select_node.hpp index 7455abd8b935..b3a22966a230 100644 --- a/src/include/duckdb/planner/query_node/bound_select_node.hpp +++ b/src/include/duckdb/planner/query_node/bound_select_node.hpp @@ -14,7 +14,7 @@ #include "duckdb/planner/bound_tableref.hpp" #include "duckdb/parser/parsed_data/sample_options.hpp" #include "duckdb/parser/group_by_node.hpp" -#include "duckdb/planner/select_bind_state.hpp" +#include "duckdb/planner/expression_binder/select_bind_state.hpp" namespace duckdb { diff --git a/src/planner/binder/query_node/bind_select_node.cpp b/src/planner/binder/query_node/bind_select_node.cpp index 129978ac1dd5..95f9544e9ea5 100644 --- a/src/planner/binder/query_node/bind_select_node.cpp +++ b/src/planner/binder/query_node/bind_select_node.cpp @@ -26,7 +26,7 @@ #include "duckdb/planner/expression_iterator.hpp" #include "duckdb/planner/query_node/bound_select_node.hpp" #include "duckdb/parser/expression/function_expression.hpp" -#include "duckdb/planner/select_bind_state.hpp" +#include "duckdb/planner/expression_binder/select_bind_state.hpp" namespace duckdb { @@ -448,7 +448,7 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ // bind the HAVING clause, if any if (statement.having) { - HavingBinder having_binder(*this, context, *result, info, bind_state, statement.aggregate_handling); + HavingBinder having_binder(*this, context, *result, info, statement.aggregate_handling); ExpressionBinder::QualifyColumnNames(*this, statement.having); result->having = having_binder.Bind(statement.having); } @@ -459,7 +459,7 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ if (statement.aggregate_handling == AggregateHandling::FORCE_AGGREGATES) { throw BinderException("Combining QUALIFY with GROUP BY ALL is not supported yet"); } - qualify_binder = make_uniq(*this, context, *result, info, bind_state); + qualify_binder = make_uniq(*this, context, *result, info); ExpressionBinder::QualifyColumnNames(*this, statement.qualify); result->qualify = qualify_binder->Bind(statement.qualify); if (qualify_binder->HasBoundColumns() && qualify_binder->BoundAggregates()) { diff --git a/src/planner/binder/query_node/bind_setop_node.cpp b/src/planner/binder/query_node/bind_setop_node.cpp index b22d4e92e275..26690880ac2d 100644 --- a/src/planner/binder/query_node/bind_setop_node.cpp +++ b/src/planner/binder/query_node/bind_setop_node.cpp @@ -9,7 +9,7 @@ #include "duckdb/planner/expression_binder/order_binder.hpp" #include "duckdb/planner/query_node/bound_select_node.hpp" #include "duckdb/planner/query_node/bound_set_operation_node.hpp" -#include "duckdb/planner/select_bind_state.hpp" +#include "duckdb/planner/expression_binder/select_bind_state.hpp" #include "duckdb/common/enum_util.hpp" namespace duckdb { diff --git a/src/planner/binder/statement/bind_create.cpp b/src/planner/binder/statement/bind_create.cpp index c2d7650f8d32..8f66ea03cc4f 100644 --- a/src/planner/binder/statement/bind_create.cpp +++ b/src/planner/binder/statement/bind_create.cpp @@ -42,7 +42,7 @@ #include "duckdb/catalog/duck_catalog.hpp" #include "duckdb/function/table/table_scan.hpp" #include "duckdb/parser/tableref/basetableref.hpp" -#include "duckdb/planner/select_bind_state.hpp" +#include "duckdb/planner/expression_binder/select_bind_state.hpp" namespace duckdb { diff --git a/src/planner/expression_binder/CMakeLists.txt b/src/planner/expression_binder/CMakeLists.txt index 187401f927d1..d5a18f02b21a 100644 --- a/src/planner/expression_binder/CMakeLists.txt +++ b/src/planner/expression_binder/CMakeLists.txt @@ -17,6 +17,7 @@ add_library_unity( relation_binder.cpp returning_binder.cpp select_binder.cpp + select_bind_state.cpp table_function_binder.cpp update_binder.cpp where_binder.cpp) diff --git a/src/planner/expression_binder/base_select_binder.cpp b/src/planner/expression_binder/base_select_binder.cpp index 4930c2d6cc9b..cadc3ac6aba6 100644 --- a/src/planner/expression_binder/base_select_binder.cpp +++ b/src/planner/expression_binder/base_select_binder.cpp @@ -8,7 +8,7 @@ #include "duckdb/planner/expression/bound_columnref_expression.hpp" #include "duckdb/planner/expression_binder/aggregate_binder.hpp" #include "duckdb/planner/query_node/bound_select_node.hpp" -#include "duckdb/planner/select_bind_state.hpp" +#include "duckdb/planner/expression_binder/select_bind_state.hpp" namespace duckdb { diff --git a/src/planner/expression_binder/column_alias_binder.cpp b/src/planner/expression_binder/column_alias_binder.cpp index 95722820afa8..ec6d9480b437 100644 --- a/src/planner/expression_binder/column_alias_binder.cpp +++ b/src/planner/expression_binder/column_alias_binder.cpp @@ -4,11 +4,11 @@ #include "duckdb/common/string_util.hpp" #include "duckdb/planner/expression_binder.hpp" #include "duckdb/planner/binder.hpp" -#include "duckdb/planner/select_bind_state.hpp" +#include "duckdb/planner/expression_binder/select_bind_state.hpp" namespace duckdb { -ColumnAliasBinder::ColumnAliasBinder(const SelectBindState &bind_state) +ColumnAliasBinder::ColumnAliasBinder(SelectBindState &bind_state) : bind_state(bind_state), visited_select_indexes() { } @@ -35,12 +35,10 @@ bool ColumnAliasBinder::BindAlias(ExpressionBinder &enclosing_binder, unique_ptr } // We found an alias, so we copy the alias expression into this expression. - auto original_expr = bind_state.original_expressions[alias_entry->second]->Copy(); + auto original_expr = bind_state.BindAlias(alias_entry->second); expr_ptr = std::move(original_expr); visited_select_indexes.insert(alias_entry->second); - // Since the alias has been found, we pass a depth of 0. See issue 4978 (#16). - // Only HAVING, QUALIFY, and WHERE binders contain ColumnAliasBinders. result = enclosing_binder.BindExpression(expr_ptr, depth, root_expression); visited_select_indexes.erase(alias_entry->second); return true; diff --git a/src/planner/expression_binder/group_binder.cpp b/src/planner/expression_binder/group_binder.cpp index c86914ecfd7a..24e5f9806e32 100644 --- a/src/planner/expression_binder/group_binder.cpp +++ b/src/planner/expression_binder/group_binder.cpp @@ -9,7 +9,7 @@ namespace duckdb { GroupBinder::GroupBinder(Binder &binder, ClientContext &context, SelectNode &node, idx_t group_index, - const SelectBindState &bind_state, case_insensitive_map_t &group_alias_map) + SelectBindState &bind_state, case_insensitive_map_t &group_alias_map) : ExpressionBinder(binder, context), node(node), bind_state(bind_state), group_alias_map(group_alias_map), group_index(group_index) { } diff --git a/src/planner/expression_binder/having_binder.cpp b/src/planner/expression_binder/having_binder.cpp index 8536147b1f0f..6df6962c48ff 100644 --- a/src/planner/expression_binder/having_binder.cpp +++ b/src/planner/expression_binder/having_binder.cpp @@ -9,8 +9,8 @@ namespace duckdb { HavingBinder::HavingBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info, - const SelectBindState &bind_state, AggregateHandling aggregate_handling) - : BaseSelectBinder(binder, context, node, info), column_alias_binder(bind_state), + AggregateHandling aggregate_handling) + : BaseSelectBinder(binder, context, node, info), column_alias_binder(node.bind_state), aggregate_handling(aggregate_handling) { target_type = LogicalType(LogicalTypeId::BOOLEAN); } diff --git a/src/planner/expression_binder/qualify_binder.cpp b/src/planner/expression_binder/qualify_binder.cpp index c6e177139ee1..e778c2034b1b 100644 --- a/src/planner/expression_binder/qualify_binder.cpp +++ b/src/planner/expression_binder/qualify_binder.cpp @@ -9,9 +9,8 @@ namespace duckdb { -QualifyBinder::QualifyBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info, - const SelectBindState &bind_state) - : BaseSelectBinder(binder, context, node, info), column_alias_binder(bind_state) { +QualifyBinder::QualifyBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info) + : BaseSelectBinder(binder, context, node, info), column_alias_binder(node.bind_state) { target_type = LogicalType(LogicalTypeId::BOOLEAN); } diff --git a/src/planner/expression_binder/select_bind_state.cpp b/src/planner/expression_binder/select_bind_state.cpp new file mode 100644 index 000000000000..cc11084cbdea --- /dev/null +++ b/src/planner/expression_binder/select_bind_state.cpp @@ -0,0 +1,9 @@ +#include "duckdb/planner/expression_binder/select_bind_state.hpp" + +namespace duckdb { + +unique_ptr SelectBindState::BindAlias(idx_t index) { + return original_expressions[index]->Copy(); +} + +} From 667d33043561caaad1b4118c8627700e251ebe23 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 4 Apr 2024 14:56:57 +0200 Subject: [PATCH 164/603] Keep track of which aliases are volatile and/or have subqueries so we can forbid copying of volatile expressions --- .../expression_binder/having_binder.hpp | 1 + .../expression_binder/select_bind_state.hpp | 13 ++++++++++ .../binder/query_node/bind_select_node.cpp | 7 ++++++ .../binder/query_node/bind_setop_node.cpp | 2 +- .../expression/bound_expanded_expression.cpp | 5 ++-- .../expression_binder/column_alias_binder.cpp | 3 +-- .../expression_binder/group_binder.cpp | 2 +- .../expression_binder/having_binder.cpp | 2 +- .../expression_binder/order_binder.cpp | 8 ++++--- .../expression_binder/select_bind_state.cpp | 24 +++++++++++++++++++ .../expression_binder/select_binder.cpp | 18 ++++++-------- test/sql/binder/alias_where_side_effects.test | 11 +++++++++ .../types/struct/struct_unnest_recursion.test | 13 +++++----- 13 files changed, 80 insertions(+), 29 deletions(-) create mode 100644 test/sql/binder/alias_where_side_effects.test diff --git a/src/include/duckdb/planner/expression_binder/having_binder.hpp b/src/include/duckdb/planner/expression_binder/having_binder.hpp index fb5155f43d34..919779fc2dc0 100644 --- a/src/include/duckdb/planner/expression_binder/having_binder.hpp +++ b/src/include/duckdb/planner/expression_binder/having_binder.hpp @@ -23,6 +23,7 @@ class HavingBinder : public BaseSelectBinder { protected: BindResult BindWindow(WindowExpression &expr, idx_t depth) override; BindResult BindColumnRef(unique_ptr &expr_ptr, idx_t depth, bool root_expression) override; + private: ColumnAliasBinder column_alias_binder; AggregateHandling aggregate_handling; diff --git a/src/include/duckdb/planner/expression_binder/select_bind_state.hpp b/src/include/duckdb/planner/expression_binder/select_bind_state.hpp index 89cd00bbd317..e65c0207a393 100644 --- a/src/include/duckdb/planner/expression_binder/select_bind_state.hpp +++ b/src/include/duckdb/planner/expression_binder/select_bind_state.hpp @@ -28,6 +28,19 @@ struct SelectBindState { public: unique_ptr BindAlias(idx_t index); + + void SetExpressionIsVolatile(idx_t index); + void SetExpressionHasSubquery(idx_t index); + + bool AliasHasSubquery(idx_t index); + +private: + //! The set of referenced aliases + unordered_set referenced_aliases; + //! The set of expressions that is volatile + unordered_set volatile_expressions; + //! The set of expressions that contains a subquery + unordered_set subquery_expressions; }; } // namespace duckdb diff --git a/src/planner/binder/query_node/bind_select_node.cpp b/src/planner/binder/query_node/bind_select_node.cpp index 95f9544e9ea5..a817697ecea6 100644 --- a/src/planner/binder/query_node/bind_select_node.cpp +++ b/src/planner/binder/query_node/bind_select_node.cpp @@ -507,6 +507,13 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ continue; } + if (expr->IsVolatile()) { + result->bind_state.SetExpressionIsVolatile(i); + } + if (expr->HasSubquery()) { + result->bind_state.SetExpressionHasSubquery(i); + } + if (can_group_by_all && select_binder.HasBoundColumns()) { if (select_binder.BoundAggregates()) { throw BinderException("Cannot mix aggregates with non-aggregated columns!"); diff --git a/src/planner/binder/query_node/bind_setop_node.cpp b/src/planner/binder/query_node/bind_setop_node.cpp index 26690880ac2d..fbdb8d02200a 100644 --- a/src/planner/binder/query_node/bind_setop_node.cpp +++ b/src/planner/binder/query_node/bind_setop_node.cpp @@ -244,7 +244,7 @@ unique_ptr Binder::BindNode(SetOperationNode &statement) { } // now we perform the actual resolution of the ORDER BY/DISTINCT expressions OrderBinder order_binder({result->left_binder.get(), result->right_binder.get()}, result->setop_index, - bind_state, result->names.size()); + bind_state, result->names.size()); BindModifiers(order_binder, statement, *result); } diff --git a/src/planner/expression/bound_expanded_expression.cpp b/src/planner/expression/bound_expanded_expression.cpp index 15f0f5e045a4..5a02a62a84e8 100644 --- a/src/planner/expression/bound_expanded_expression.cpp +++ b/src/planner/expression/bound_expanded_expression.cpp @@ -1,11 +1,10 @@ #include "duckdb/planner/expression/bound_expanded_expression.hpp" - namespace duckdb { BoundExpandedExpression::BoundExpandedExpression(vector> expanded_expressions_p) - : Expression(ExpressionType::BOUND_EXPANDED, ExpressionClass::BOUND_EXPANDED, LogicalType::INTEGER), - expanded_expressions(std::move(expanded_expressions_p)) { + : Expression(ExpressionType::BOUND_EXPANDED, ExpressionClass::BOUND_EXPANDED, LogicalType::INTEGER), + expanded_expressions(std::move(expanded_expressions_p)) { } string BoundExpandedExpression::ToString() const { diff --git a/src/planner/expression_binder/column_alias_binder.cpp b/src/planner/expression_binder/column_alias_binder.cpp index ec6d9480b437..fbe0f5a377fb 100644 --- a/src/planner/expression_binder/column_alias_binder.cpp +++ b/src/planner/expression_binder/column_alias_binder.cpp @@ -8,8 +8,7 @@ namespace duckdb { -ColumnAliasBinder::ColumnAliasBinder(SelectBindState &bind_state) - : bind_state(bind_state), visited_select_indexes() { +ColumnAliasBinder::ColumnAliasBinder(SelectBindState &bind_state) : bind_state(bind_state), visited_select_indexes() { } bool ColumnAliasBinder::BindAlias(ExpressionBinder &enclosing_binder, unique_ptr &expr_ptr, diff --git a/src/planner/expression_binder/group_binder.cpp b/src/planner/expression_binder/group_binder.cpp index 24e5f9806e32..ff78acbd379c 100644 --- a/src/planner/expression_binder/group_binder.cpp +++ b/src/planner/expression_binder/group_binder.cpp @@ -9,7 +9,7 @@ namespace duckdb { GroupBinder::GroupBinder(Binder &binder, ClientContext &context, SelectNode &node, idx_t group_index, - SelectBindState &bind_state, case_insensitive_map_t &group_alias_map) + SelectBindState &bind_state, case_insensitive_map_t &group_alias_map) : ExpressionBinder(binder, context), node(node), bind_state(bind_state), group_alias_map(group_alias_map), group_index(group_index) { } diff --git a/src/planner/expression_binder/having_binder.cpp b/src/planner/expression_binder/having_binder.cpp index 6df6962c48ff..5a987ef7f3f9 100644 --- a/src/planner/expression_binder/having_binder.cpp +++ b/src/planner/expression_binder/having_binder.cpp @@ -9,7 +9,7 @@ namespace duckdb { HavingBinder::HavingBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info, - AggregateHandling aggregate_handling) + AggregateHandling aggregate_handling) : BaseSelectBinder(binder, context, node, info), column_alias_binder(node.bind_state), aggregate_handling(aggregate_handling) { target_type = LogicalType(LogicalTypeId::BOOLEAN); diff --git a/src/planner/expression_binder/order_binder.cpp b/src/planner/expression_binder/order_binder.cpp index 495a6a81aa28..4345529bf57e 100644 --- a/src/planner/expression_binder/order_binder.cpp +++ b/src/planner/expression_binder/order_binder.cpp @@ -15,9 +15,10 @@ namespace duckdb { OrderBinder::OrderBinder(vector binders, idx_t projection_index, SelectBindState &bind_state, idx_t max_count) : binders(std::move(binders)), projection_index(projection_index), max_count(max_count), extra_list(nullptr), - bind_state(bind_state) { + bind_state(bind_state) { } -OrderBinder::OrderBinder(vector binders, idx_t projection_index, SelectNode &node, SelectBindState &bind_state) +OrderBinder::OrderBinder(vector binders, idx_t projection_index, SelectNode &node, + SelectBindState &bind_state) : binders(std::move(binders)), projection_index(projection_index), bind_state(bind_state) { this->max_count = node.select_list.size(); this->extra_list = &node.select_list; @@ -32,7 +33,8 @@ unique_ptr OrderBinder::CreateProjectionReference(ParsedExpression & alias = expr.alias; } } - return make_uniq(std::move(alias), LogicalType::INVALID, ColumnBinding(projection_index, index)); + return make_uniq(std::move(alias), LogicalType::INVALID, + ColumnBinding(projection_index, index)); } unique_ptr OrderBinder::CreateExtraReference(unique_ptr expr) { diff --git a/src/planner/expression_binder/select_bind_state.cpp b/src/planner/expression_binder/select_bind_state.cpp index cc11084cbdea..382ff182bd1e 100644 --- a/src/planner/expression_binder/select_bind_state.cpp +++ b/src/planner/expression_binder/select_bind_state.cpp @@ -3,7 +3,31 @@ namespace duckdb { unique_ptr SelectBindState::BindAlias(idx_t index) { + if (volatile_expressions.find(index) != volatile_expressions.end()) { + throw BinderException("Alias \"%s\" referenced - but the expression has side " + "effects. This is not yet supported.", + original_expressions[index]->alias); + } + referenced_aliases.insert(index); return original_expressions[index]->Copy(); } +void SelectBindState::SetExpressionIsVolatile(idx_t index) { + // check if this expression has been referenced before + if (referenced_aliases.find(index) != referenced_aliases.end()) { + throw BinderException("Alias \"%s\" referenced - but the expression has side " + "effects. This is not yet supported.", + original_expressions[index]->alias); + } + volatile_expressions.insert(index); } + +void SelectBindState::SetExpressionHasSubquery(idx_t index) { + subquery_expressions.insert(index); +} + +bool SelectBindState::AliasHasSubquery(idx_t index) { + return subquery_expressions.find(index) != subquery_expressions.end(); +} + +} // namespace duckdb diff --git a/src/planner/expression_binder/select_binder.cpp b/src/planner/expression_binder/select_binder.cpp index a6b29c56a8c0..badab70f16de 100644 --- a/src/planner/expression_binder/select_binder.cpp +++ b/src/planner/expression_binder/select_binder.cpp @@ -16,26 +16,22 @@ BindResult SelectBinder::BindColumnRef(unique_ptr &expr_ptr, i // check in the alias map auto &colref = (expr_ptr.get())->Cast(); if (!colref.IsQualified()) { + auto &bind_state = node.bind_state; auto alias_entry = node.bind_state.alias_map.find(colref.column_names[0]); if (alias_entry != node.bind_state.alias_map.end()) { // found entry! auto index = alias_entry->second; if (index >= node.bound_column_count) { throw BinderException("Column \"%s\" referenced that exists in the SELECT clause - but this column " - "cannot be referenced before it is defined", - colref.column_names[0]); + "cannot be referenced before it is defined", + colref.column_names[0]); } - if (node.select_list[index]->IsVolatile()) { - throw BinderException("Alias \"%s\" referenced in a SELECT clause - but the expression has side " - "effects. This is not yet supported.", - colref.column_names[0]); - } - if (node.select_list[index]->HasSubquery()) { + if (bind_state.AliasHasSubquery(index)) { throw BinderException("Alias \"%s\" referenced in a SELECT clause - but the expression has a subquery." - " This is not yet supported.", - colref.column_names[0]); + " This is not yet supported.", + colref.column_names[0]); } - auto copied_expression = node.bind_state.original_expressions[index]->Copy(); + auto copied_expression = node.bind_state.BindAlias(index); result = BindExpression(copied_expression, depth, false); return result; } diff --git a/test/sql/binder/alias_where_side_effects.test b/test/sql/binder/alias_where_side_effects.test new file mode 100644 index 000000000000..0ffd503958bc --- /dev/null +++ b/test/sql/binder/alias_where_side_effects.test @@ -0,0 +1,11 @@ +# name: test/sql/binder/alias_where_side_effects.test +# description: Issue #10099: Incorrect query result when using random value in where clause +# group: [binder] + +statement ok +PRAGMA enable_verification + +statement error +select count(*) from (select random() as num from range(20) where num > 0.9) where num <= 0.9 +---- +expression has side effects diff --git a/test/sql/types/struct/struct_unnest_recursion.test b/test/sql/types/struct/struct_unnest_recursion.test index 4078abc470e7..89268a8e6702 100644 --- a/test/sql/types/struct/struct_unnest_recursion.test +++ b/test/sql/types/struct/struct_unnest_recursion.test @@ -10,13 +10,12 @@ SELECT UNNEST ( ( '1,2,3,4,,6' , ( 1 ) ) ) , x x; ---- cannot be referenced before it is defined -mode skip - -# FIXME this should work -statement error -SELECT UNNEST ( ( '1,2,3,4,,6' , ( random() ) ) ), 42 x, x; +query IIII +SELECT UNNEST ( ( '1,2,3,4,,6' , ( case when random() < 10 then 0 else 1 end ) ) ), 42 x, x; ---- -1,2,3,4,,6 1 42 42 +1,2,3,4,,6 0 42 42 + + + -mode unskip From 7eef6a705c78e2c2e8cdc9ea849be81f200bda02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Thu, 4 Apr 2024 15:05:47 +0200 Subject: [PATCH 165/603] moaar42 --- .../types/row/partitioned_tuple_data.cpp | 4 +- .../types/row/row_data_collection_scanner.cpp | 11 +-- src/common/types/uuid.cpp | 13 ++-- src/common/types/value.cpp | 8 +- src/common/types/vector.cpp | 6 +- .../vector_operations/null_operations.cpp | 2 +- .../numeric_inplace_operators.cpp | 4 +- src/execution/adaptive_filter.cpp | 2 +- src/execution/aggregate_hashtable.cpp | 6 +- .../aggregate/distinct_aggregate_data.cpp | 2 +- .../aggregate/physical_hash_aggregate.cpp | 2 +- .../aggregate/physical_streaming_window.cpp | 6 +- .../physical_ungrouped_aggregate.cpp | 2 +- .../operator/aggregate/physical_window.cpp | 2 +- .../buffer_manager/csv_file_handle.cpp | 3 +- .../scanner/column_count_scanner.cpp | 2 +- .../scanner/string_value_scanner.cpp | 2 +- .../csv_scanner/sniffer/header_detection.cpp | 17 ++-- .../csv_scanner/sniffer/type_detection.cpp | 2 +- src/execution/perfect_aggregate_hashtable.cpp | 8 +- src/execution/physical_operator.cpp | 4 +- src/execution/radix_partitioned_hashtable.cpp | 20 ++--- src/execution/window_executor.cpp | 78 +++++++++---------- src/include/duckdb/common/vector_size.hpp | 2 +- .../duckdb/execution/merge_sort_tree.hpp | 22 +++--- src/include/duckdb/storage/storage_info.hpp | 4 +- 26 files changed, 120 insertions(+), 114 deletions(-) diff --git a/src/common/types/row/partitioned_tuple_data.cpp b/src/common/types/row/partitioned_tuple_data.cpp index 979b292294e7..f3b6469812d0 100644 --- a/src/common/types/row/partitioned_tuple_data.cpp +++ b/src/common/types/row/partitioned_tuple_data.cpp @@ -329,8 +329,8 @@ void PartitionedTupleData::Repartition(PartitionedTupleData &new_partitioned_dat const int64_t update = reverse ? -1 : 1; const int64_t adjustment = reverse ? -1 : 0; - for (idx_t partition_idx = start_idx; partition_idx != end_idx; partition_idx += update) { - auto actual_partition_idx = partition_idx + adjustment; + for (idx_t partition_idx = start_idx; partition_idx != end_idx; partition_idx += idx_t(update)) { + auto actual_partition_idx = partition_idx + idx_t(adjustment); auto &partition = *partitions[actual_partition_idx]; if (partition.Count() > 0) { diff --git a/src/common/types/row/row_data_collection_scanner.cpp b/src/common/types/row/row_data_collection_scanner.cpp index 50135d887707..efbd072e5406 100644 --- a/src/common/types/row/row_data_collection_scanner.cpp +++ b/src/common/types/row/row_data_collection_scanner.cpp @@ -56,7 +56,8 @@ void RowDataCollectionScanner::AlignHeapBlocks(RowDataCollection &swizzled_block if (!swizzled_string_heap.keep_pinned) { auto heap_ptr = Load(data_ptr + layout.GetHeapOffset()); auto heap_offset = heap_ptr - heap_handle.Ptr(); - RowOperations::SwizzleHeapPointer(layout, data_ptr, heap_ptr, data_block->count, heap_offset); + RowOperations::SwizzleHeapPointer(layout, data_ptr, heap_ptr, data_block->count, + NumericCast(heap_offset)); } else { swizzled_string_heap.pinned_blocks.emplace_back(std::move(heap_handle)); } @@ -84,7 +85,7 @@ void RowDataCollectionScanner::AlignHeapBlocks(RowDataCollection &swizzled_block auto heap_start_ptr = Load(data_ptr + layout.GetHeapOffset()); auto heap_end_ptr = Load(data_ptr + layout.GetHeapOffset() + (next - 1) * layout.GetRowWidth()); - idx_t size = heap_end_ptr - heap_start_ptr + Load(heap_end_ptr); + auto size = NumericCast(heap_end_ptr - heap_start_ptr + Load(heap_end_ptr)); ptrs_and_sizes.emplace_back(heap_start_ptr, size); D_ASSERT(size <= heap_blocks[heap_block_idx]->byte_offset); @@ -100,7 +101,7 @@ void RowDataCollectionScanner::AlignHeapBlocks(RowDataCollection &swizzled_block // Finally, we allocate a new heap block and copy data to it swizzled_string_heap.blocks.emplace_back(make_uniq( - MemoryTag::ORDER_BY, buffer_manager, MaxValue(total_size, (idx_t)Storage::BLOCK_SIZE), 1)); + MemoryTag::ORDER_BY, buffer_manager, MaxValue(total_size, (idx_t)Storage::BLOCK_SIZE), 1U)); auto new_heap_handle = buffer_manager.Pin(swizzled_string_heap.blocks.back()->block); auto new_heap_ptr = new_heap_handle.Ptr(); for (auto &ptr_and_size : ptrs_and_sizes) { @@ -174,7 +175,7 @@ RowDataCollectionScanner::RowDataCollectionScanner(RowDataCollection &rows_p, Ro // Pretend that we have scanned up to the start block // and will stop at the end auto begin = rows.blocks.begin(); - auto end = begin + block_idx; + auto end = begin + NumericCast(block_idx); total_scanned = std::accumulate(begin, end, idx_t(0), [&](idx_t c, const unique_ptr &b) { return c + b->count; }); total_count = total_scanned + (*end)->count; @@ -194,7 +195,7 @@ void RowDataCollectionScanner::SwizzleBlock(RowDataBlock &data_block, RowDataBlo auto heap_handle = heap.buffer_manager.Pin(heap_block.block); auto heap_ptr = Load(data_ptr + layout.GetHeapOffset()); auto heap_offset = heap_ptr - heap_handle.Ptr(); - RowOperations::SwizzleHeapPointer(layout, data_ptr, heap_ptr, data_block.count, heap_offset); + RowOperations::SwizzleHeapPointer(layout, data_ptr, heap_ptr, data_block.count, NumericCast(heap_offset)); } void RowDataCollectionScanner::ReSwizzle() { diff --git a/src/common/types/uuid.cpp b/src/common/types/uuid.cpp index 818d9e8ddeaa..82583fe91b36 100644 --- a/src/common/types/uuid.cpp +++ b/src/common/types/uuid.cpp @@ -6,13 +6,13 @@ namespace duckdb { bool UUID::FromString(const string &str, hugeint_t &result) { auto hex2char = [](char ch) -> unsigned char { if (ch >= '0' && ch <= '9') { - return ch - '0'; + return UnsafeNumericCast(ch - '0'); } if (ch >= 'a' && ch <= 'f') { - return 10 + ch - 'a'; + return UnsafeNumericCast(10 + ch - 'a'); } if (ch >= 'A' && ch <= 'F') { - return 10 + ch - 'A'; + return UnsafeNumericCast(10 + ch - 'A'); } return 0; }; @@ -23,7 +23,7 @@ bool UUID::FromString(const string &str, hugeint_t &result) { if (str.empty()) { return false; } - int has_braces = 0; + idx_t has_braces = 0; if (str.front() == '{') { has_braces = 1; } @@ -54,14 +54,15 @@ bool UUID::FromString(const string &str, hugeint_t &result) { } void UUID::ToString(hugeint_t input, char *buf) { - auto byte_to_hex = [](char byte_val, char *buf, idx_t &pos) { + auto byte_to_hex = [](uint64_t byte_val, char *buf, idx_t &pos) { + D_ASSERT(byte_val <= 0xFF); static char const HEX_DIGITS[] = "0123456789abcdef"; buf[pos++] = HEX_DIGITS[(byte_val >> 4) & 0xf]; buf[pos++] = HEX_DIGITS[byte_val & 0xf]; }; // Flip back before convert to string - int64_t upper = input.upper ^ (uint64_t(1) << 63); + int64_t upper = int64_t(uint64_t(input.upper) ^ (uint64_t(1) << 63)); idx_t pos = 0; byte_to_hex(upper >> 56 & 0xFF, buf, pos); byte_to_hex(upper >> 48 & 0xFF, buf, pos); diff --git a/src/common/types/value.cpp b/src/common/types/value.cpp index fb3ee10cea4d..94c7c3415134 100644 --- a/src/common/types/value.cpp +++ b/src/common/types/value.cpp @@ -1209,11 +1209,11 @@ Value Value::Numeric(const LogicalType &type, int64_t value) { return Value::UINTEGER((uint32_t)value); case LogicalTypeId::UBIGINT: D_ASSERT(value >= 0); - return Value::UBIGINT(value); + return Value::UBIGINT(NumericCast(value)); case LogicalTypeId::HUGEINT: return Value::HUGEINT(value); case LogicalTypeId::UHUGEINT: - return Value::UHUGEINT(value); + return Value::UHUGEINT(NumericCast(value)); case LogicalTypeId::DECIMAL: return Value::DECIMAL(value, DecimalType::GetWidth(type), DecimalType::GetScale(type)); case LogicalTypeId::FLOAT: @@ -1221,7 +1221,7 @@ Value Value::Numeric(const LogicalType &type, int64_t value) { case LogicalTypeId::DOUBLE: return Value((double)value); case LogicalTypeId::POINTER: - return Value::POINTER(value); + return Value::POINTER(NumericCast(value)); case LogicalTypeId::DATE: D_ASSERT(value >= NumericLimits::Minimum() && value <= NumericLimits::Maximum()); return Value::DATE(date_t(NumericCast(value))); @@ -1662,7 +1662,7 @@ hugeint_t IntegralValue::Get(const Value &value) { case PhysicalType::UINT32: return UIntegerValue::Get(value); case PhysicalType::UINT64: - return UBigIntValue::Get(value); + return NumericCast(UBigIntValue::Get(value)); case PhysicalType::UINT128: return static_cast(UhugeIntValue::Get(value)); default: diff --git a/src/common/types/vector.cpp b/src/common/types/vector.cpp index b177911278b0..e61ad46dd259 100644 --- a/src/common/types/vector.cpp +++ b/src/common/types/vector.cpp @@ -555,7 +555,7 @@ Value Vector::GetValueInternal(const Vector &v_p, idx_t index_p) { case VectorType::SEQUENCE_VECTOR: { int64_t start, increment; SequenceVector::GetSequence(*vector, start, increment); - return Value::Numeric(vector->GetType(), start + increment * index); + return Value::Numeric(vector->GetType(), start + increment * NumericCast(index)); } default: throw InternalException("Unimplemented vector type for Vector::GetValue"); @@ -788,7 +788,7 @@ string Vector::ToString(idx_t count) const { int64_t start, increment; SequenceVector::GetSequence(*this, start, increment); for (idx_t i = 0; i < count; i++) { - retval += to_string(start + increment * i) + (i == count - 1 ? "" : ", "); + retval += to_string(start + increment * UnsafeNumericCast(i)) + (i == count - 1 ? "" : ", "); } break; } @@ -1006,7 +1006,7 @@ void Vector::Flatten(idx_t count) { buffer = VectorBuffer::CreateStandardVector(GetType()); data = buffer->GetData(); - VectorOperations::GenerateSequence(*this, sequence_count, start, increment); + VectorOperations::GenerateSequence(*this, NumericCast(sequence_count), start, increment); break; } default: diff --git a/src/common/vector_operations/null_operations.cpp b/src/common/vector_operations/null_operations.cpp index 48bc904da474..dd34ac8edbc1 100644 --- a/src/common/vector_operations/null_operations.cpp +++ b/src/common/vector_operations/null_operations.cpp @@ -102,7 +102,7 @@ idx_t VectorOperations::CountNotNull(Vector &input, const idx_t count) { default: for (idx_t i = 0; i < count; ++i) { const auto row_idx = vdata.sel->get_index(i); - valid += int(vdata.validity.RowIsValid(row_idx)); + valid += idx_t(vdata.validity.RowIsValid(row_idx)); } break; } diff --git a/src/common/vector_operations/numeric_inplace_operators.cpp b/src/common/vector_operations/numeric_inplace_operators.cpp index d2bd0f313db2..86b507a3ce9a 100644 --- a/src/common/vector_operations/numeric_inplace_operators.cpp +++ b/src/common/vector_operations/numeric_inplace_operators.cpp @@ -23,14 +23,14 @@ void VectorOperations::AddInPlace(Vector &input, int64_t right, idx_t count) { case VectorType::CONSTANT_VECTOR: { D_ASSERT(!ConstantVector::IsNull(input)); auto data = ConstantVector::GetData(input); - *data += right; + *data += UnsafeNumericCast(right); break; } default: { D_ASSERT(input.GetVectorType() == VectorType::FLAT_VECTOR); auto data = FlatVector::GetData(input); for (idx_t i = 0; i < count; i++) { - data[i] += right; + data[i] += UnsafeNumericCast(right); } break; } diff --git a/src/execution/adaptive_filter.cpp b/src/execution/adaptive_filter.cpp index 37ffad99b240..166174d00b89 100644 --- a/src/execution/adaptive_filter.cpp +++ b/src/execution/adaptive_filter.cpp @@ -58,7 +58,7 @@ void AdaptiveFilter::AdaptRuntimeStatistics(double duration) { // get swap index and swap likeliness std::uniform_int_distribution distribution(1, NumericCast(right_random_border)); // a <= i <= b - idx_t random_number = distribution(generator) - 1; + auto random_number = UnsafeNumericCast(distribution(generator) - 1); swap_idx = random_number / 100; // index to be swapped idx_t likeliness = random_number - 100 * swap_idx; // random number between [0, 100) diff --git a/src/execution/aggregate_hashtable.cpp b/src/execution/aggregate_hashtable.cpp index 90029b0a4b6c..c8ab1ce6b965 100644 --- a/src/execution/aggregate_hashtable.cpp +++ b/src/execution/aggregate_hashtable.cpp @@ -246,7 +246,7 @@ idx_t GroupedAggregateHashTable::AddChunk(DataChunk &groups, Vector &group_hashe #endif const auto new_group_count = FindOrCreateGroups(groups, group_hashes, state.addresses, state.new_groups); - VectorOperations::AddInPlace(state.addresses, layout.GetAggrOffset(), payload.size()); + VectorOperations::AddInPlace(state.addresses, NumericCast(layout.GetAggrOffset()), payload.size()); // Now every cell has an entry, update the aggregates auto &aggregates = layout.GetAggregates(); @@ -258,7 +258,7 @@ idx_t GroupedAggregateHashTable::AddChunk(DataChunk &groups, Vector &group_hashe if (filter_idx >= filter.size() || i < filter[filter_idx]) { // Skip all the aggregates that are not in the filter payload_idx += aggr.child_count; - VectorOperations::AddInPlace(state.addresses, aggr.payload_size, payload.size()); + VectorOperations::AddInPlace(state.addresses, NumericCast(aggr.payload_size), payload.size()); continue; } D_ASSERT(i == filter[filter_idx]); @@ -272,7 +272,7 @@ idx_t GroupedAggregateHashTable::AddChunk(DataChunk &groups, Vector &group_hashe // Move to the next aggregate payload_idx += aggr.child_count; - VectorOperations::AddInPlace(state.addresses, aggr.payload_size, payload.size()); + VectorOperations::AddInPlace(state.addresses, NumericCast(aggr.payload_size), payload.size()); filter_idx++; } diff --git a/src/execution/operator/aggregate/distinct_aggregate_data.cpp b/src/execution/operator/aggregate/distinct_aggregate_data.cpp index cd76deee6482..d1d0d20d6eb5 100644 --- a/src/execution/operator/aggregate/distinct_aggregate_data.cpp +++ b/src/execution/operator/aggregate/distinct_aggregate_data.cpp @@ -151,7 +151,7 @@ idx_t DistinctAggregateCollectionInfo::CreateTableIndexMap() { std::find_if(table_inputs.begin(), table_inputs.end(), FindMatchingAggregate(std::ref(aggregate))); if (matching_inputs != table_inputs.end()) { //! Assign the existing table to the aggregate - idx_t found_idx = std::distance(table_inputs.begin(), matching_inputs); + auto found_idx = NumericCast(std::distance(table_inputs.begin(), matching_inputs)); table_map[agg_idx] = found_idx; continue; } diff --git a/src/execution/operator/aggregate/physical_hash_aggregate.cpp b/src/execution/operator/aggregate/physical_hash_aggregate.cpp index 5217c110bd26..bfed85859f4d 100644 --- a/src/execution/operator/aggregate/physical_hash_aggregate.cpp +++ b/src/execution/operator/aggregate/physical_hash_aggregate.cpp @@ -565,7 +565,7 @@ class HashAggregateDistinctFinalizeTask : public ExecutorTask { void HashAggregateDistinctFinalizeEvent::Schedule() { auto n_tasks = CreateGlobalSources(); - n_tasks = MinValue(n_tasks, TaskScheduler::GetScheduler(context).NumberOfThreads()); + n_tasks = MinValue(n_tasks, NumericCast(TaskScheduler::GetScheduler(context).NumberOfThreads())); vector> tasks; for (idx_t i = 0; i < n_tasks; i++) { tasks.push_back(make_uniq(*pipeline, shared_from_this(), op, gstate)); diff --git a/src/execution/operator/aggregate/physical_streaming_window.cpp b/src/execution/operator/aggregate/physical_streaming_window.cpp index 7f4974681d67..f1310350e623 100644 --- a/src/execution/operator/aggregate/physical_streaming_window.cpp +++ b/src/execution/operator/aggregate/physical_streaming_window.cpp @@ -142,7 +142,7 @@ OperatorResultType PhysicalStreamingWindow::Execute(ExecutionContext &context, D auto data = FlatVector::GetData(result); int64_t start_row = gstate.row_number; for (idx_t i = 0; i < input.size(); ++i) { - data[i] = start_row + i; + data[i] = NumericCast(start_row + NumericCast(i)); } break; } @@ -192,7 +192,7 @@ OperatorResultType PhysicalStreamingWindow::Execute(ExecutionContext &context, D int64_t start_row = gstate.row_number; auto rdata = FlatVector::GetData(chunk.data[col_idx]); for (idx_t i = 0; i < count; i++) { - rdata[i] = start_row + i; + rdata[i] = NumericCast(start_row + NumericCast(i)); } break; } @@ -200,7 +200,7 @@ OperatorResultType PhysicalStreamingWindow::Execute(ExecutionContext &context, D throw NotImplementedException("%s for StreamingWindow", ExpressionTypeToString(expr.GetExpressionType())); } } - gstate.row_number += count; + gstate.row_number += NumericCast(count); chunk.SetCardinality(count); return OperatorResultType::NEED_MORE_INPUT; } diff --git a/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp b/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp index f1ceeab4bc7c..b542d4b6362f 100644 --- a/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +++ b/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp @@ -454,7 +454,7 @@ void UngroupedDistinctAggregateFinalizeEvent::Schedule() { global_source_states.push_back(radix_table_p.GetGlobalSourceState(context)); } n_tasks = MaxValue(n_tasks, 1); - n_tasks = MinValue(n_tasks, TaskScheduler::GetScheduler(context).NumberOfThreads()); + n_tasks = MinValue(n_tasks, NumericCast(TaskScheduler::GetScheduler(context).NumberOfThreads())); vector> tasks; for (idx_t i = 0; i < n_tasks; i++) { diff --git a/src/execution/operator/aggregate/physical_window.cpp b/src/execution/operator/aggregate/physical_window.cpp index bcfe0a56bd3b..01e6752a2cd0 100644 --- a/src/execution/operator/aggregate/physical_window.cpp +++ b/src/execution/operator/aggregate/physical_window.cpp @@ -327,7 +327,7 @@ void WindowPartitionSourceState::MaterializeSortedData() { heap->blocks = std::move(sd.heap_blocks); hash_group.reset(); } else { - heap = make_uniq(buffer_manager, (idx_t)Storage::BLOCK_SIZE, 1, true); + heap = make_uniq(buffer_manager, Storage::BLOCK_SIZE, 1U, true); } heap->count = std::accumulate(heap->blocks.begin(), heap->blocks.end(), idx_t(0), [&](idx_t c, const unique_ptr &b) { return c + b->count; }); diff --git a/src/execution/operator/csv_scanner/buffer_manager/csv_file_handle.cpp b/src/execution/operator/csv_scanner/buffer_manager/csv_file_handle.cpp index 5a73815e2763..90d32eebaab8 100644 --- a/src/execution/operator/csv_scanner/buffer_manager/csv_file_handle.cpp +++ b/src/execution/operator/csv_scanner/buffer_manager/csv_file_handle.cpp @@ -1,5 +1,6 @@ #include "duckdb/execution/operator/csv_scanner/csv_file_handle.hpp" #include "duckdb/common/exception/binder_exception.hpp" +#include "duckdb/common/numeric_utils.hpp" namespace duckdb { @@ -57,7 +58,7 @@ idx_t CSVFileHandle::Read(void *buffer, idx_t nr_bytes) { if (!finished) { finished = bytes_read == 0; } - return bytes_read; + return UnsafeNumericCast(bytes_read); } string CSVFileHandle::ReadLine() { diff --git a/src/execution/operator/csv_scanner/scanner/column_count_scanner.cpp b/src/execution/operator/csv_scanner/scanner/column_count_scanner.cpp index 8894cbcf4e76..70542ef4b472 100644 --- a/src/execution/operator/csv_scanner/scanner/column_count_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/column_count_scanner.cpp @@ -50,7 +50,7 @@ ColumnCountScanner::ColumnCountScanner(shared_ptr buffer_manag } unique_ptr ColumnCountScanner::UpgradeToStringValueScanner() { - auto scanner = make_uniq(0, buffer_manager, state_machine, error_handler, nullptr, true); + auto scanner = make_uniq(0U, buffer_manager, state_machine, error_handler, nullptr, true); return scanner; } diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 9582e1c1af2f..3cff693b1bee 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -1075,7 +1075,7 @@ void StringValueScanner::SetStart() { } scan_finder = make_uniq( - 0, buffer_manager, state_machine, make_shared(true), csv_file_scan, false, iterator, 1); + 0U, buffer_manager, state_machine, make_shared(true), csv_file_scan, false, iterator, 1U); auto &tuples = scan_finder->ParseChunk(); line_found = true; if (tuples.number_of_rows != 1) { diff --git a/src/execution/operator/csv_scanner/sniffer/header_detection.cpp b/src/execution/operator/csv_scanner/sniffer/header_detection.cpp index a52e2aeae884..6617421e5726 100644 --- a/src/execution/operator/csv_scanner/sniffer/header_detection.cpp +++ b/src/execution/operator/csv_scanner/sniffer/header_detection.cpp @@ -7,9 +7,9 @@ namespace duckdb { // Helper function to generate column names static string GenerateColumnName(const idx_t total_cols, const idx_t col_number, const string &prefix = "column") { - int max_digits = NumericHelper::UnsignedLength(total_cols - 1); - int digits = NumericHelper::UnsignedLength(col_number); - string leading_zeros = string(max_digits - digits, '0'); + auto max_digits = NumericHelper::UnsignedLength(total_cols - 1); + auto digits = NumericHelper::UnsignedLength(col_number); + string leading_zeros = string(NumericCast(max_digits - digits), '0'); string value = to_string(col_number); return string(prefix + leading_zeros + value); } @@ -22,21 +22,21 @@ static string TrimWhitespace(const string &col_name) { // Find the first character that is not left trimmed idx_t begin = 0; while (begin < size) { - auto bytes = utf8proc_iterate(str + begin, size - begin, &codepoint); + auto bytes = utf8proc_iterate(str + begin, NumericCast(size - begin), &codepoint); D_ASSERT(bytes > 0); if (utf8proc_category(codepoint) != UTF8PROC_CATEGORY_ZS) { break; } - begin += bytes; + begin += NumericCast(bytes); } // Find the last character that is not right trimmed idx_t end; end = begin; for (auto next = begin; next < col_name.size();) { - auto bytes = utf8proc_iterate(str + next, size - next, &codepoint); + auto bytes = utf8proc_iterate(str + next, NumericCast(size - next), &codepoint); D_ASSERT(bytes > 0); - next += bytes; + next += NumericCast(bytes); if (utf8proc_category(codepoint) != UTF8PROC_CATEGORY_ZS) { end = next; } @@ -48,7 +48,8 @@ static string TrimWhitespace(const string &col_name) { static string NormalizeColumnName(const string &col_name) { // normalize UTF8 characters to NFKD - auto nfkd = utf8proc_NFKD(reinterpret_cast(col_name.c_str()), col_name.size()); + auto nfkd = utf8proc_NFKD(reinterpret_cast(col_name.c_str()), + NumericCast(col_name.size())); const string col_name_nfkd = string(const_char_ptr_cast(nfkd), strlen(const_char_ptr_cast(nfkd))); free(nfkd); diff --git a/src/execution/operator/csv_scanner/sniffer/type_detection.cpp b/src/execution/operator/csv_scanner/sniffer/type_detection.cpp index da9f7ed6983f..077cbdd2f8a0 100644 --- a/src/execution/operator/csv_scanner/sniffer/type_detection.cpp +++ b/src/execution/operator/csv_scanner/sniffer/type_detection.cpp @@ -50,7 +50,7 @@ static bool StartsWithNumericDate(string &separator, const string &value) { } // second literal must match first - if (((field3 - literal2) != (field2 - literal1)) || strncmp(literal1, literal2, (field2 - literal1)) != 0) { + if (((field3 - literal2) != (field2 - literal1)) || strncmp(literal1, literal2, NumericCast((field2 - literal1))) != 0) { return false; } diff --git a/src/execution/perfect_aggregate_hashtable.cpp b/src/execution/perfect_aggregate_hashtable.cpp index da7c20192452..a46e9499b5a3 100644 --- a/src/execution/perfect_aggregate_hashtable.cpp +++ b/src/execution/perfect_aggregate_hashtable.cpp @@ -64,7 +64,7 @@ static void ComputeGroupLocationTemplated(UnifiedVectorFormat &group_data, Value // we only need to handle non-null values here if (group_data.validity.RowIsValid(index)) { D_ASSERT(data[index] >= min_val); - uintptr_t adjusted_value = (data[index] - min_val) + 1; + auto adjusted_value = UnsafeNumericCast((data[index] - min_val) + 1); address_data[i] += adjusted_value << current_shift; } } @@ -72,7 +72,7 @@ static void ComputeGroupLocationTemplated(UnifiedVectorFormat &group_data, Value // no null values: we can directly compute the addresses for (idx_t i = 0; i < count; i++) { auto index = group_data.sel->get_index(i); - uintptr_t adjusted_value = (data[index] - min_val) + 1; + auto adjusted_value = UnsafeNumericCast((data[index] - min_val) + 1); address_data[i] += adjusted_value << current_shift; } } @@ -149,7 +149,7 @@ void PerfectAggregateHashTable::AddChunk(DataChunk &groups, DataChunk &payload) } // move to the next aggregate payload_idx += input_count; - VectorOperations::AddInPlace(addresses, aggregate.payload_size, payload.size()); + VectorOperations::AddInPlace(addresses, NumericCast(aggregate.payload_size), payload.size()); } } @@ -199,7 +199,7 @@ static void ReconstructGroupVectorTemplated(uint32_t group_values[], Value &min, auto min_data = min.GetValueUnsafe(); for (idx_t i = 0; i < entry_count; i++) { // extract the value of this group from the total group index - auto group_index = UnsafeNumericCast((group_values[i] >> shift) & mask); + auto group_index = UnsafeNumericCast((group_values[i] >> shift) & mask); if (group_index == 0) { // if it is 0, the value is NULL validity_mask.SetInvalid(i); diff --git a/src/execution/physical_operator.cpp b/src/execution/physical_operator.cpp index ba5ba3e22158..4934789ecfe7 100644 --- a/src/execution/physical_operator.cpp +++ b/src/execution/physical_operator.cpp @@ -122,8 +122,8 @@ unique_ptr PhysicalOperator::GetGlobalSinkState(ClientContext & idx_t PhysicalOperator::GetMaxThreadMemory(ClientContext &context) { // Memory usage per thread should scale with max mem / num threads // We take 1/4th of this, to be conservative - idx_t max_memory = BufferManager::GetBufferManager(context).GetQueryMaxMemory(); - idx_t num_threads = TaskScheduler::GetScheduler(context).NumberOfThreads(); + auto max_memory = BufferManager::GetBufferManager(context).GetQueryMaxMemory(); + auto num_threads = NumericCast(TaskScheduler::GetScheduler(context).NumberOfThreads()); return (max_memory / num_threads) / 4; } diff --git a/src/execution/radix_partitioned_hashtable.cpp b/src/execution/radix_partitioned_hashtable.cpp index fe8575cb11ae..f148fefd1709 100644 --- a/src/execution/radix_partitioned_hashtable.cpp +++ b/src/execution/radix_partitioned_hashtable.cpp @@ -195,7 +195,7 @@ RadixHTGlobalSinkState::RadixHTGlobalSinkState(ClientContext &context_p, const R auto ht_size = blocks_per_partition * Storage::BLOCK_ALLOC_SIZE + config.sink_capacity * sizeof(aggr_ht_entry_t); // This really is the minimum reservation that we can do - idx_t num_threads = TaskScheduler::GetScheduler(context).NumberOfThreads(); + auto num_threads = NumericCast(TaskScheduler::GetScheduler(context).NumberOfThreads()); auto minimum_reservation = num_threads * ht_size; temporary_memory_state->SetMinimumReservation(minimum_reservation); @@ -273,12 +273,12 @@ void RadixHTConfig::SetRadixBitsInternal(const idx_t radix_bits_p, bool external } idx_t RadixHTConfig::InitialSinkRadixBits(ClientContext &context) { - const idx_t active_threads = TaskScheduler::GetScheduler(context).NumberOfThreads(); + const auto active_threads = NumericCast(TaskScheduler::GetScheduler(context).NumberOfThreads()); return MinValue(RadixPartitioning::RadixBits(NextPowerOfTwo(active_threads)), MAXIMUM_INITIAL_SINK_RADIX_BITS); } idx_t RadixHTConfig::MaximumSinkRadixBits(ClientContext &context) { - const idx_t active_threads = TaskScheduler::GetScheduler(context).NumberOfThreads(); + const auto active_threads = NumericCast(TaskScheduler::GetScheduler(context).NumberOfThreads()); return MinValue(RadixPartitioning::RadixBits(NextPowerOfTwo(active_threads)), MAXIMUM_FINAL_SINK_RADIX_BITS); } @@ -288,7 +288,7 @@ idx_t RadixHTConfig::ExternalRadixBits(const idx_t &maximum_sink_radix_bits_p) { idx_t RadixHTConfig::SinkCapacity(ClientContext &context) { // Get active and maximum number of threads - const idx_t active_threads = TaskScheduler::GetScheduler(context).NumberOfThreads(); + const auto active_threads = NumericCast(TaskScheduler::GetScheduler(context).NumberOfThreads()); // Compute cache size per active thread (assuming cache is shared) const auto total_shared_cache_size = active_threads * L3_CACHE_SIZE; @@ -527,8 +527,8 @@ void RadixPartitionedHashTable::Finalize(ClientContext &context, GlobalSinkState // Minimum of combining one partition at a time gstate.temporary_memory_state->SetMinimumReservation(gstate.max_partition_size); // Maximum of combining all partitions - auto max_threads = - MinValue(TaskScheduler::GetScheduler(context).NumberOfThreads(), gstate.partitions.size()); + auto max_threads = MinValue(NumericCast(TaskScheduler::GetScheduler(context).NumberOfThreads()), + gstate.partitions.size()); gstate.temporary_memory_state->SetRemainingSize(context, max_threads * gstate.max_partition_size); gstate.finalized = true; } @@ -545,8 +545,8 @@ idx_t RadixPartitionedHashTable::MaxThreads(GlobalSinkState &sink_p) const { // This many partitions will fit given our reservation (at least 1)) auto partitions_fit = MaxValue(sink.temporary_memory_state->GetReservation() / sink.max_partition_size, 1); // Maximum is either the number of partitions, or the number of threads - auto max_possible = - MinValue(sink.partitions.size(), TaskScheduler::GetScheduler(sink.context).NumberOfThreads()); + auto max_possible = MinValue( + sink.partitions.size(), NumericCast(TaskScheduler::GetScheduler(sink.context).NumberOfThreads())); // Mininum of the two return MinValue(partitions_fit, max_possible); @@ -726,8 +726,8 @@ void RadixHTLocalSourceState::Finalize(RadixHTGlobalSinkState &sink, RadixHTGlob const auto capacity = GroupedAggregateHashTable::GetCapacityForCount(partition.data->Count()); // However, we will limit the initial capacity so we don't do a huge over-allocation - const idx_t n_threads = TaskScheduler::GetScheduler(gstate.context).NumberOfThreads(); - const idx_t memory_limit = BufferManager::GetBufferManager(gstate.context).GetMaxMemory(); + const auto n_threads = NumericCast(TaskScheduler::GetScheduler(gstate.context).NumberOfThreads()); + const auto memory_limit = BufferManager::GetBufferManager(gstate.context).GetMaxMemory(); const idx_t thread_limit = 0.6 * memory_limit / n_threads; const idx_t size_per_entry = partition.data->SizeInBytes() / partition.data->Count() + diff --git a/src/execution/window_executor.cpp b/src/execution/window_executor.cpp index 627e8df630da..4d6b9b099c37 100644 --- a/src/execution/window_executor.cpp +++ b/src/execution/window_executor.cpp @@ -139,11 +139,11 @@ struct WindowColumnIterator { // Random Access inline iterator &operator+=(difference_type n) { - pos += n; + pos += UnsafeNumericCast(n); return *this; } inline iterator &operator-=(difference_type n) { - pos -= n; + pos -= UnsafeNumericCast(n); return *this; } @@ -232,7 +232,7 @@ static idx_t FindTypedRangeBound(const WindowInputColumn &over, const idx_t orde const auto first = over.GetCell(prev.start); if (!comp(val, first)) { // prev.first <= val, so we can start further forward - begin += (prev.start - order_begin); + begin += UnsafeNumericCast(prev.start - order_begin); } } if (order_begin < prev.end && prev.end < order_end) { @@ -240,7 +240,7 @@ static idx_t FindTypedRangeBound(const WindowInputColumn &over, const idx_t orde if (!comp(second, val)) { // val <= prev.second, so we can end further back // (prev.second is the largest peer) - end -= (order_end - prev.end - 1); + end -= UnsafeNumericCast(order_end - prev.end - 1); } } } @@ -452,13 +452,13 @@ void WindowBoundariesState::Update(const idx_t row_idx, const WindowInputColumn switch (start_boundary) { case WindowBoundary::UNBOUNDED_PRECEDING: - window_start = partition_start; + window_start = NumericCast(partition_start); break; case WindowBoundary::CURRENT_ROW_ROWS: - window_start = row_idx; + window_start = NumericCast(row_idx); break; case WindowBoundary::CURRENT_ROW_RANGE: - window_start = peer_start; + window_start = NumericCast(peer_start); break; case WindowBoundary::EXPR_PRECEDING_ROWS: { if (!TrySubtractOperator::Operation(int64_t(row_idx), boundary_start.GetCell(chunk_idx), @@ -475,21 +475,21 @@ void WindowBoundariesState::Update(const idx_t row_idx, const WindowInputColumn } case WindowBoundary::EXPR_PRECEDING_RANGE: { if (boundary_start.CellIsNull(chunk_idx)) { - window_start = peer_start; + window_start = NumericCast(peer_start); } else { prev.start = FindOrderedRangeBound(range_collection, range_sense, valid_start, row_idx, start_boundary, boundary_start, chunk_idx, prev); - window_start = prev.start; + window_start = NumericCast(prev.start); } break; } case WindowBoundary::EXPR_FOLLOWING_RANGE: { if (boundary_start.CellIsNull(chunk_idx)) { - window_start = peer_start; + window_start = NumericCast(peer_start); } else { prev.start = FindOrderedRangeBound(range_collection, range_sense, row_idx, valid_end, start_boundary, boundary_start, chunk_idx, prev); - window_start = prev.start; + window_start = NumericCast(prev.start); } break; } @@ -499,13 +499,13 @@ void WindowBoundariesState::Update(const idx_t row_idx, const WindowInputColumn switch (end_boundary) { case WindowBoundary::CURRENT_ROW_ROWS: - window_end = row_idx + 1; + window_end = NumericCast(row_idx + 1); break; case WindowBoundary::CURRENT_ROW_RANGE: - window_end = peer_end; + window_end = NumericCast(peer_end); break; case WindowBoundary::UNBOUNDED_FOLLOWING: - window_end = partition_end; + window_end = NumericCast(partition_end); break; case WindowBoundary::EXPR_PRECEDING_ROWS: if (!TrySubtractOperator::Operation(int64_t(row_idx + 1), boundary_end.GetCell(chunk_idx), @@ -520,21 +520,21 @@ void WindowBoundariesState::Update(const idx_t row_idx, const WindowInputColumn break; case WindowBoundary::EXPR_PRECEDING_RANGE: { if (boundary_end.CellIsNull(chunk_idx)) { - window_end = peer_end; + window_end = NumericCast(peer_end); } else { prev.end = FindOrderedRangeBound(range_collection, range_sense, valid_start, row_idx, end_boundary, boundary_end, chunk_idx, prev); - window_end = prev.end; + window_end = NumericCast(prev.end); } break; } case WindowBoundary::EXPR_FOLLOWING_RANGE: { if (boundary_end.CellIsNull(chunk_idx)) { - window_end = peer_end; + window_end = NumericCast(peer_end); } else { prev.end = FindOrderedRangeBound(range_collection, range_sense, row_idx, valid_end, end_boundary, boundary_end, chunk_idx, prev); - window_end = prev.end; + window_end = NumericCast(prev.end); } break; } @@ -543,17 +543,17 @@ void WindowBoundariesState::Update(const idx_t row_idx, const WindowInputColumn } // clamp windows to partitions if they should exceed - if (window_start < (int64_t)partition_start) { - window_start = partition_start; + if (window_start < NumericCast(partition_start)) { + window_start = NumericCast(partition_start); } - if (window_start > (int64_t)partition_end) { - window_start = partition_end; + if (window_start > NumericCast(partition_end)) { + window_start = NumericCast(partition_end); } - if (window_end < (int64_t)partition_start) { - window_end = partition_start; + if (window_end < NumericCast(partition_start)) { + window_end = NumericCast(partition_start); } - if (window_end > (int64_t)partition_end) { - window_end = partition_end; + if (window_end > NumericCast(partition_end)) { + window_end = NumericCast(partition_end); } if (window_start < 0 || window_end < 0) { @@ -1029,7 +1029,7 @@ void WindowAggregateExecutor::Finalize() { // Estimate the frame statistics // Default to the entire partition if we don't know anything FrameStats stats; - const int64_t count = aggregator->GetInputs().size(); + const auto count = NumericCast(aggregator->GetInputs().size()); // First entry is the frame start stats[0] = FrameDelta(-count, count); @@ -1090,7 +1090,7 @@ void WindowRowNumberExecutor::EvaluateInternal(WindowExecutorState &lstate, Vect auto partition_begin = FlatVector::GetData(lbstate.bounds.data[PARTITION_BEGIN]); auto rdata = FlatVector::GetData(result); for (idx_t i = 0; i < count; ++i, ++row_idx) { - rdata[i] = row_idx - partition_begin[i] + 1; + rdata[i] = NumericCast(row_idx - partition_begin[i] + 1); } } @@ -1147,7 +1147,7 @@ void WindowRankExecutor::EvaluateInternal(WindowExecutorState &lstate, Vector &r for (idx_t i = 0; i < count; ++i, ++row_idx) { lpeer.NextRank(partition_begin[i], peer_begin[i], row_idx); - rdata[i] = lpeer.rank; + rdata[i] = NumericCast(lpeer.rank); } } @@ -1209,7 +1209,7 @@ void WindowDenseRankExecutor::EvaluateInternal(WindowExecutorState &lstate, Vect for (idx_t i = 0; i < count; ++i, ++row_idx) { lpeer.NextRank(partition_begin[i], peer_begin[i], row_idx); - rdata[i] = lpeer.dense_rank; + rdata[i] = NumericCast(lpeer.dense_rank); } } @@ -1237,7 +1237,7 @@ void WindowPercentRankExecutor::EvaluateInternal(WindowExecutorState &lstate, Ve for (idx_t i = 0; i < count; ++i, ++row_idx) { lpeer.NextRank(partition_begin[i], peer_begin[i], row_idx); - int64_t denom = partition_end[i] - partition_begin[i] - 1; + auto denom = NumericCast(partition_end[i] - partition_begin[i] - 1); double percent_rank = denom > 0 ? ((double)lpeer.rank - 1) / denom : 0; rdata[i] = percent_rank; } @@ -1260,7 +1260,7 @@ void WindowCumeDistExecutor::EvaluateInternal(WindowExecutorState &lstate, Vecto auto peer_end = FlatVector::GetData(lbstate.bounds.data[PEER_END]); auto rdata = FlatVector::GetData(result); for (idx_t i = 0; i < count; ++i, ++row_idx) { - int64_t denom = partition_end[i] - partition_begin[i]; + auto denom = NumericCast(partition_end[i] - partition_begin[i]); double cume_dist = denom > 0 ? ((double)(peer_end[i] - partition_begin[i])) / denom : 0; rdata[i] = cume_dist; } @@ -1365,7 +1365,7 @@ void WindowNtileExecutor::EvaluateInternal(WindowExecutorState &lstate, Vector & throw InvalidInputException("Argument for ntile must be greater than zero"); } // With thanks from SQLite's ntileValueFunc() - int64_t n_total = partition_end[i] - partition_begin[i]; + auto n_total = NumericCast(partition_end[i] - partition_begin[i]); if (n_param > n_total) { // more groups allowed than we have values // map every entry to a unique group @@ -1374,7 +1374,7 @@ void WindowNtileExecutor::EvaluateInternal(WindowExecutorState &lstate, Vector & int64_t n_size = (n_total / n_param); // find the row idx within the group D_ASSERT(row_idx >= partition_begin[i]); - int64_t adjusted_row_idx = row_idx - partition_begin[i]; + auto adjusted_row_idx = NumericCast(row_idx - partition_begin[i]); // now compute the ntile int64_t n_large = n_total - n_param * n_size; int64_t i_small = n_large * (n_size + 1); @@ -1452,16 +1452,16 @@ void WindowLeadLagExecutor::EvaluateInternal(WindowExecutorState &lstate, Vector idx_t delta = 0; if (val_idx < (int64_t)row_idx) { // Count backwards - delta = idx_t(row_idx - val_idx); - val_idx = FindPrevStart(ignore_nulls, partition_begin[i], row_idx, delta); + delta = idx_t(row_idx - idx_t(val_idx)); + val_idx = int64_t(FindPrevStart(ignore_nulls, partition_begin[i], row_idx, delta)); } else if (val_idx > (int64_t)row_idx) { - delta = idx_t(val_idx - row_idx); - val_idx = FindNextStart(ignore_nulls, row_idx + 1, partition_end[i], delta); + delta = idx_t(idx_t(val_idx) - row_idx); + val_idx = int64_t(FindNextStart(ignore_nulls, row_idx + 1, partition_end[i], delta)); } // else offset is zero, so don't move. if (!delta) { - CopyCell(payload_collection, 0, val_idx, result, i); + CopyCell(payload_collection, 0, NumericCast(val_idx), result, i); } else if (wexpr.default_expr) { llstate.leadlag_default.CopyCell(result, i); } else { diff --git a/src/include/duckdb/common/vector_size.hpp b/src/include/duckdb/common/vector_size.hpp index 0fa29086b220..3eb73b36b3bd 100644 --- a/src/include/duckdb/common/vector_size.hpp +++ b/src/include/duckdb/common/vector_size.hpp @@ -13,7 +13,7 @@ namespace duckdb { //! The default standard vector size -#define DEFAULT_STANDARD_VECTOR_SIZE 2048 +#define DEFAULT_STANDARD_VECTOR_SIZE 2048U //! The vector size used in the execution engine #ifndef STANDARD_VECTOR_SIZE diff --git a/src/include/duckdb/execution/merge_sort_tree.hpp b/src/include/duckdb/execution/merge_sort_tree.hpp index 4f752a75410b..825c05f872de 100644 --- a/src/include/duckdb/execution/merge_sort_tree.hpp +++ b/src/include/duckdb/execution/merge_sort_tree.hpp @@ -515,8 +515,8 @@ void MergeSortTree::AggregateLowerBound(const idx_t lower, cons entry.first = run_idx.first * level_width; entry.second = std::min(entry.first + level_width, static_cast(tree[0].first.size())); auto *level_data = tree[level].first.data(); - idx_t entry_idx = - std::lower_bound(level_data + entry.first, level_data + entry.second, needle) - level_data; + auto entry_idx = NumericCast( + std::lower_bound(level_data + entry.first, level_data + entry.second, needle) - level_data); cascading_idx.first = cascading_idx.second = (entry_idx / CASCADING + 2 * (entry.first / level_width)) * FANOUT; @@ -545,7 +545,7 @@ void MergeSortTree::AggregateLowerBound(const idx_t lower, cons const auto run_pos = std::lower_bound(search_begin, search_end, needle) - level_data; // Compute runBegin and pass it to our callback const auto run_begin = curr.first - level_width; - aggregate(level, run_begin, run_pos); + aggregate(level, run_begin, NumericCast(run_pos)); // Update state for next round curr.first -= level_width; --cascading_idx.first; @@ -554,7 +554,7 @@ void MergeSortTree::AggregateLowerBound(const idx_t lower, cons if (curr.first != lower) { const auto *search_begin = level_data + cascading_idcs[cascading_idx.first]; const auto *search_end = level_data + cascading_idcs[cascading_idx.first + FANOUT]; - auto idx = std::lower_bound(search_begin, search_end, needle) - level_data; + auto idx = NumericCast(std::lower_bound(search_begin, search_end, needle) - level_data); cascading_idx.first = (idx / CASCADING + 2 * (lower / level_width)) * FANOUT; } @@ -567,7 +567,7 @@ void MergeSortTree::AggregateLowerBound(const idx_t lower, cons const auto run_pos = std::lower_bound(search_begin, search_end, needle) - level_data; // Compute runBegin and pass it to our callback const auto run_begin = curr.second; - aggregate(level, run_begin, run_pos); + aggregate(level, run_begin, NumericCast(run_pos)); // Update state for next round curr.second += level_width; ++cascading_idx.second; @@ -576,7 +576,7 @@ void MergeSortTree::AggregateLowerBound(const idx_t lower, cons if (curr.second != upper) { const auto *search_begin = level_data + cascading_idcs[cascading_idx.second]; const auto *search_end = level_data + cascading_idcs[cascading_idx.second + FANOUT]; - auto idx = std::lower_bound(search_begin, search_end, needle) - level_data; + auto idx = NumericCast(std::lower_bound(search_begin, search_end, needle) - level_data); cascading_idx.second = (idx / CASCADING + 2 * (upper / level_width)) * FANOUT; } } while (level >= LowestCascadingLevel()); @@ -591,8 +591,9 @@ void MergeSortTree::AggregateLowerBound(const idx_t lower, cons while (curr.first - lower >= level_width) { const auto *search_end = level_data + curr.first; const auto *search_begin = search_end - level_width; - const auto run_pos = std::lower_bound(search_begin, search_end, needle) - level_data; - const auto run_begin = search_begin - level_data; + const auto run_pos = + NumericCast(std::lower_bound(search_begin, search_end, needle) - level_data); + const auto run_begin = NumericCast(search_begin - level_data); aggregate(level, run_begin, run_pos); curr.first -= level_width; } @@ -600,8 +601,9 @@ void MergeSortTree::AggregateLowerBound(const idx_t lower, cons while (upper - curr.second >= level_width) { const auto *search_begin = level_data + curr.second; const auto *search_end = search_begin + level_width; - const auto run_pos = std::lower_bound(search_begin, search_end, needle) - level_data; - const auto run_begin = search_begin - level_data; + const auto run_pos = + NumericCast(std::lower_bound(search_begin, search_end, needle) - level_data); + const auto run_begin = NumericCast(search_begin - level_data); aggregate(level, run_begin, run_pos); curr.second += level_width; } diff --git a/src/include/duckdb/storage/storage_info.hpp b/src/include/duckdb/storage/storage_info.hpp index 1acdbba67cd3..e3d36025e681 100644 --- a/src/include/duckdb/storage/storage_info.hpp +++ b/src/include/duckdb/storage/storage_info.hpp @@ -29,7 +29,7 @@ using block_id_t = int64_t; struct Storage { //! The size of a hard disk sector, only really needed for Direct IO - constexpr static idx_t SECTOR_SIZE = 4096; + constexpr static idx_t SECTOR_SIZE = 4096U; //! Block header size for blocks written to the storage constexpr static idx_t BLOCK_HEADER_SIZE = sizeof(uint64_t); //! Size of a memory slot managed by the StorageManager. This is the quantum of allocation for Blocks on DuckDB. We @@ -39,7 +39,7 @@ struct Storage { constexpr static idx_t BLOCK_SIZE = BLOCK_ALLOC_SIZE - BLOCK_HEADER_SIZE; //! The size of the headers. This should be small and written more or less atomically by the hard disk. We default //! to the page size, which is 4KB. (1 << 12) - constexpr static idx_t FILE_HEADER_SIZE = 4096; + constexpr static idx_t FILE_HEADER_SIZE = 4096U; //! The number of rows per row group (must be a multiple of the vector size) constexpr static const idx_t ROW_GROUP_SIZE = STANDARD_ROW_GROUPS_SIZE; //! The number of vectors per row group From e58c90b1013e97653d75bbbbff2c7f219dca60aa Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 4 Apr 2024 15:36:48 +0200 Subject: [PATCH 166/603] Bunch of tests for multiple error return on borked lines --- data/csv/rejects/multiple_errors.csv | 8 ++ .../multiple_errors/cast_and_less_col.csv | 5 + .../multiple_errors/cast_and_more_col.csv | 5 + .../multiple_cast_implicit.csv | 4 + .../multiple_errors/multiple_casts_flush.csv | 4 + .../multiple_errors/multiple_casts_mixed.csv | 4 + .../scanner/string_value_scanner.cpp | 15 +- .../test_multiple_errors_same_line.test | 131 ++++++++++++++++++ 8 files changed, 174 insertions(+), 2 deletions(-) create mode 100644 data/csv/rejects/multiple_errors.csv create mode 100644 data/csv/rejects/multiple_errors/cast_and_less_col.csv create mode 100644 data/csv/rejects/multiple_errors/cast_and_more_col.csv create mode 100644 data/csv/rejects/multiple_errors/multiple_cast_implicit.csv create mode 100644 data/csv/rejects/multiple_errors/multiple_casts_flush.csv create mode 100644 data/csv/rejects/multiple_errors/multiple_casts_mixed.csv create mode 100644 test/sql/copy/csv/rejects/test_multiple_errors_same_line.test diff --git a/data/csv/rejects/multiple_errors.csv b/data/csv/rejects/multiple_errors.csv new file mode 100644 index 000000000000..6d10e51c3cf5 --- /dev/null +++ b/data/csv/rejects/multiple_errors.csv @@ -0,0 +1,8 @@ +name,age,current_day, barks +oogie boogie,3, 2023-01-01, 2 +oogie boogie,3, 2023-01-02, 5 +oogie boogie,3, 2023-01-03, bla, 7 +oogie boogie,3, bla, bla, 7 +oogie boogie,3, 2023-01-04, 8 +oogie boogie,3, bla +oogie boogieoogie boogieoogie boogieoogie boogieoogie boogieoogie boogieoogie boogie,3, bla diff --git a/data/csv/rejects/multiple_errors/cast_and_less_col.csv b/data/csv/rejects/multiple_errors/cast_and_less_col.csv new file mode 100644 index 000000000000..25f4dfe159ed --- /dev/null +++ b/data/csv/rejects/multiple_errors/cast_and_less_col.csv @@ -0,0 +1,5 @@ +name,age,current_day, barks +oogie boogie,3, 2023-01-01, 2 +oogie boogie,3, 2023-01-02, 5 +oogie boogie,bla, 2023-01-03 +oogie boogie,bla diff --git a/data/csv/rejects/multiple_errors/cast_and_more_col.csv b/data/csv/rejects/multiple_errors/cast_and_more_col.csv new file mode 100644 index 000000000000..4e5a7d2321d3 --- /dev/null +++ b/data/csv/rejects/multiple_errors/cast_and_more_col.csv @@ -0,0 +1,5 @@ +name,age,current_day, barks +oogie boogie,3, 2023-01-01, 2 +oogie boogie,3, 2023-01-02, 5 +oogie boogie,3, 2023-01-03, bla, 7 +oogie boogie,3, 2023-01-03, bla, 7, 8 diff --git a/data/csv/rejects/multiple_errors/multiple_cast_implicit.csv b/data/csv/rejects/multiple_errors/multiple_cast_implicit.csv new file mode 100644 index 000000000000..26ad443f6fdc --- /dev/null +++ b/data/csv/rejects/multiple_errors/multiple_cast_implicit.csv @@ -0,0 +1,4 @@ +name,age,current_day, barks +oogie boogie,3, 2023-01-01, 2 +oogie boogie,3, 2023-01-02, 5 +oogie boogie,bla_2, 2023-01-02, bla_1 \ No newline at end of file diff --git a/data/csv/rejects/multiple_errors/multiple_casts_flush.csv b/data/csv/rejects/multiple_errors/multiple_casts_flush.csv new file mode 100644 index 000000000000..21d7a7b58b54 --- /dev/null +++ b/data/csv/rejects/multiple_errors/multiple_casts_flush.csv @@ -0,0 +1,4 @@ +name,age,current_day, tomorrow +oogie boogie,3, 2023-01-01, 2023-01-02 +oogie boogie,3, 2023-01-02, 2023-01-03 +oogie boogie,3, bla_2, bla_1 \ No newline at end of file diff --git a/data/csv/rejects/multiple_errors/multiple_casts_mixed.csv b/data/csv/rejects/multiple_errors/multiple_casts_mixed.csv new file mode 100644 index 000000000000..3931dbb1821b --- /dev/null +++ b/data/csv/rejects/multiple_errors/multiple_casts_mixed.csv @@ -0,0 +1,4 @@ +name,age,current_day, barks +oogie boogie,3, 2023-01-01, 2 +oogie boogie,3, 2023-01-02, 5 +oogie boogie,3, bla_2, bla_1 \ No newline at end of file diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index dd7c4364fb9d..0a0b286b4671 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -365,6 +365,7 @@ bool StringValueResult::HandleError() { switch (cur_error.type) { case CSVErrorType::TOO_MANY_COLUMNS: + case CSVErrorType::TOO_FEW_COLUMNS: if (current_line_position.begin == line_pos) { csv_error = CSVError::IncorrectColumnAmountError( state_machine.options, col_idx, lines_per_batch, borked_line, @@ -503,7 +504,15 @@ bool StringValueResult::AddRowInternal() { CSVError::LineSizeError(state_machine.options, current_line_size, lines_per_batch, borked_line, current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); error_handler.Error(csv_error); - number_of_rows--; + if (number_of_rows > 0) { + number_of_rows--; + } + } + if (!current_errors.empty()) { + // We need to add a few columns error + for (idx_t col_idx = cur_col_id; col_idx < number_of_columns; col_idx++) { + current_errors.push_back({CSVErrorType::TOO_FEW_COLUMNS, col_idx - 1, last_position}); + } } if (HandleError()) { return false; @@ -553,7 +562,9 @@ bool StringValueResult::AddRowInternal() { error_handler.Error(csv_error); } // If we are here we ignore_errors, so we delete this line - number_of_rows--; + if (number_of_rows > 0) { + number_of_rows--; + } } } line_positions_per_row[number_of_rows] = current_line_position; diff --git a/test/sql/copy/csv/rejects/test_multiple_errors_same_line.test b/test/sql/copy/csv/rejects/test_multiple_errors_same_line.test new file mode 100644 index 000000000000..22ed108fee20 --- /dev/null +++ b/test/sql/copy/csv/rejects/test_multiple_errors_same_line.test @@ -0,0 +1,131 @@ +# name: test/sql/copy/csv/rejects/test_multiple_errors_same_line.test +# description: Tests a mix of multiple errors and validate they get hit +# group: [rejects] + +require skip_reload + +# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n +require notwindows + +#query IIII +#FROM read_csv('data/csv/rejects/multiple_errors/cast_and_more_col.csv', +# columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, +# store_rejects = true, auto_detect=false, header = 1); +#---- +#oogie boogie 3 2023-01-01 2 +#oogie boogie 3 2023-01-02 5 +# +#query IIIIIIIII rowsort +#SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; +#---- +#0 4 89 116 4 barks CAST oogie boogie,3, 2023-01-03, bla, 7 Error when converting column "barks". Could not convert string " bla" to 'INTEGER' +#0 4 89 120 5 NULL TOO MANY COLUMNS oogie boogie,3, 2023-01-03, bla, 7 Expected Number of Columns: 4 Found: 5 +#0 5 124 151 4 barks CAST oogie boogie,3, 2023-01-03, bla, 7, 8 Error when converting column "barks". Could not convert string " bla" to 'INTEGER' +#0 5 124 155 5 NULL TOO MANY COLUMNS oogie boogie,3, 2023-01-03, bla, 7, 8 Expected Number of Columns: 4 Found: 5 +#0 5 124 158 6 NULL TOO MANY COLUMNS oogie boogie,3, 2023-01-03, bla, 7, 8 Expected Number of Columns: 4 Found: 6 +# +#statement ok +#DROP TABLE reject_errors; +# +#statement ok +#DROP TABLE reject_scans; +# +#query IIII +#FROM read_csv('data/csv/rejects/multiple_errors/multiple_cast_implicit.csv', +# columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, +# store_rejects = true, auto_detect=false, header = 1); +#---- +#oogie boogie 3 2023-01-01 2 +#oogie boogie 3 2023-01-02 5 +# +#query IIIIIIIII rowsort +#SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; +#---- +#0 4 89 102 2 age CAST oogie boogie,bla_2, 2023-01-02, bla_1 Error when converting column "age". Could not convert string "bla_2" to 'INTEGER' +#0 4 89 120 4 barks CAST oogie boogie,bla_2, 2023-01-02, bla_1 Error when converting column "barks". Could not convert string " bla_1" to 'INTEGER' +# +#statement ok +#DROP TABLE reject_errors; +# +#statement ok +#DROP TABLE reject_scans; +# +#query IIII +#FROM read_csv('data/csv/rejects/multiple_errors/multiple_casts_flush.csv', +# columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'tomorrow': 'DATE'}, +# store_rejects = true, auto_detect=false, header = 1); +#---- +#oogie boogie 3 2023-01-01 2023-01-02 +#oogie boogie 3 2023-01-02 2023-01-03 +# +#query IIIIIIIII rowsort +#SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; +#---- +#0 4 110 NULL 3 current_day CAST oogie boogie,3, bla_2, bla_1 Error when converting column "current_day". date field value out of range: " bla_2", expected format is (YYYY-MM-DD) +#0 4 110 NULL 4 tomorrow CAST oogie boogie,3, bla_2, bla_1 Error when converting column "tomorrow". date field value out of range: " bla_1", expected format is (YYYY-MM-DD) +# +#statement ok +#DROP TABLE reject_errors; +# +#statement ok +#DROP TABLE reject_scans; +# +#query IIII +#FROM read_csv('data/csv/rejects/multiple_errors/multiple_casts_mixed.csv', +# columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, +# store_rejects = true, auto_detect=false, header = 1); +#---- +#oogie boogie 3 2023-01-01 2 +#oogie boogie 3 2023-01-02 5 +# +## FIXME: This will not present the both cast errors :'(, should be alleviated the more types we add to implicit casting +#query IIIIIIIII rowsort +#SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; +#---- +#0 4 89 111 4 barks CAST oogie boogie,3, bla_2, bla_1 Error when converting column "barks". Could not convert string " bla_1" to 'INTEGER' +# +#statement ok +#DROP TABLE reject_errors; +# +#statement ok +#DROP TABLE reject_scans; + +query IIII +FROM read_csv('data/csv/rejects/multiple_errors/cast_and_less_col.csv', + columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, + store_rejects = true, auto_detect=false, header = 1); +---- +oogie boogie 3 2023-01-01 2 +oogie boogie 3 2023-01-02 5 + +query IIIIIIIII rowsort +SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; +---- +0 4 89 102 2 age CAST oogie boogie,bla, 2023-01-03 Error when converting column "age". Could not convert string "bla" to 'INTEGER' +0 4 89 117 3 barks MISSING COLUMNS oogie boogie,bla, 2023-01-03 Expected Number of Columns: 4 Found: 3 +0 5 118 131 2 age CAST oogie boogie,bla Error when converting column "age". Could not convert string "bla" to 'INTEGER' +0 5 118 134 2 current_day MISSING COLUMNS oogie boogie,bla Expected Number of Columns: 4 Found: 2 +0 5 118 134 3 barks MISSING COLUMNS oogie boogie,bla Expected Number of Columns: 4 Found: 3 + +statement ok +DROP TABLE reject_errors; + +statement ok +DROP TABLE reject_scans; + +#query IIII +#FROM read_csv('data/csv/rejects/multiple_errors.csv', +# columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, +# store_rejects = true, auto_detect=false, header = 1, max_line_size=40); +#---- +#oogie boogie 3 2023-01-01 2 +#oogie boogie 3 2023-01-02 5 +# +#query IIIIIIIIII rowsort +#FROM reject_errors ORDER BY ALL; +#---- +#3 0 4 89 116 4 barks CAST oogie boogie,3, 2023-01-03, bla, 7 Error when converting column "barks". Could not convert string " bla" to 'INTEGER' +#3 0 4 89 120 5 NULL TOO MANY COLUMNS oogie boogie,3, 2023-01-03, bla, 7 Expected Number of Columns: 4 Found: 5 +#3 0 5 124 144 4 barks CAST oogie boogie,3, bla, bla, 7 Error when converting column "barks". Could not convert string " bla" to 'INTEGER' +#3 0 5 124 148 5 NULL TOO MANY COLUMNS oogie boogie,3, bla, bla, 7 Expected Number of Columns: 4 Found: 5 +#3 0 6 152 171 3 barks MISSING COLUMNS oogie boogie,3, bla Expected Number of Columns: 4 Found: 3 \ No newline at end of file From 8591d72fdb0f3a1bd06cba858b74a946a8587a16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Thu, 4 Apr 2024 15:59:14 +0200 Subject: [PATCH 167/603] moo --- .../csv_scanner/sniffer/type_detection.cpp | 5 +++-- .../table_function/global_csv_state.cpp | 2 +- .../csv_scanner/util/csv_reader_options.cpp | 20 +++++++++---------- .../operator/helper/physical_execute.cpp | 2 +- .../helper/physical_reservoir_sample.cpp | 2 +- .../operator/join/physical_asof_join.cpp | 2 +- .../operator/join/physical_hash_join.cpp | 12 +++++------ .../operator/join/physical_iejoin.cpp | 8 ++++---- .../join/physical_piecewise_merge_join.cpp | 4 ++-- .../operator/join/physical_range_join.cpp | 2 +- .../operator/order/physical_order.cpp | 2 +- .../physical_batch_copy_to_file.cpp | 6 +++--- .../persistent/physical_batch_insert.cpp | 9 +++++---- .../persistent/physical_copy_to_file.cpp | 2 +- .../operator/persistent/physical_delete.cpp | 2 +- .../operator/persistent/physical_export.cpp | 2 +- .../operator/persistent/physical_insert.cpp | 4 ++-- .../operator/persistent/physical_update.cpp | 2 +- .../physical_plan/plan_comparison_join.cpp | 2 +- .../physical_plan/plan_create_table.cpp | 4 ++-- src/execution/physical_plan/plan_top_n.cpp | 4 ++-- src/main/capi/arrow-c.cpp | 8 ++++---- src/main/capi/result-c.cpp | 4 ++-- src/main/client_context.cpp | 3 ++- src/main/database.cpp | 2 +- src/main/query_profiler.cpp | 2 +- src/main/relation.cpp | 2 +- src/main/secret/secret.cpp | 2 +- src/main/settings/settings.cpp | 18 ++++++++--------- src/parallel/pipeline.cpp | 4 ++-- src/parallel/task_scheduler.cpp | 4 ++-- 31 files changed, 75 insertions(+), 72 deletions(-) diff --git a/src/execution/operator/csv_scanner/sniffer/type_detection.cpp b/src/execution/operator/csv_scanner/sniffer/type_detection.cpp index 077cbdd2f8a0..65c2b5b3534e 100644 --- a/src/execution/operator/csv_scanner/sniffer/type_detection.cpp +++ b/src/execution/operator/csv_scanner/sniffer/type_detection.cpp @@ -50,7 +50,8 @@ static bool StartsWithNumericDate(string &separator, const string &value) { } // second literal must match first - if (((field3 - literal2) != (field2 - literal1)) || strncmp(literal1, literal2, NumericCast((field2 - literal1))) != 0) { + if (((field3 - literal2) != (field2 - literal1)) || + strncmp(literal1, literal2, NumericCast((field2 - literal1))) != 0) { return false; } @@ -69,7 +70,7 @@ static bool StartsWithNumericDate(string &separator, const string &value) { string GenerateDateFormat(const string &separator, const char *format_template) { string format_specifier = format_template; - auto amount_of_dashes = std::count(format_specifier.begin(), format_specifier.end(), '-'); + auto amount_of_dashes = NumericCast(std::count(format_specifier.begin(), format_specifier.end(), '-')); // All our date formats must have at least one - D_ASSERT(amount_of_dashes); string result; diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index b5943a9f5b8b..f89bcfef5b18 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -22,7 +22,7 @@ CSVGlobalState::CSVGlobalState(ClientContext &context_p, const shared_ptr(context, files[0], options, 0, bind_data, column_ids, file_schema)); + make_uniq(context, files[0], options, 0U, bind_data, column_ids, file_schema)); }; //! There are situations where we only support single threaded scanning bool many_csv_files = files.size() > 1 && files.size() > system_threads * 2; diff --git a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp index 836ddd731827..70d909941f66 100644 --- a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp +++ b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp @@ -90,11 +90,11 @@ void CSVReaderOptions::SetEscape(const string &input) { } int64_t CSVReaderOptions::GetSkipRows() const { - return this->dialect_options.skip_rows.GetValue(); + return NumericCast(this->dialect_options.skip_rows.GetValue()); } void CSVReaderOptions::SetSkipRows(int64_t skip_rows) { - dialect_options.skip_rows.Set(skip_rows); + dialect_options.skip_rows.Set(NumericCast(skip_rows)); } string CSVReaderOptions::GetDelimiter() const { @@ -162,7 +162,7 @@ void CSVReaderOptions::SetReadOption(const string &loption, const Value &value, if (loption == "auto_detect") { auto_detect = ParseBoolean(value, loption); } else if (loption == "sample_size") { - int64_t sample_size_option = ParseInteger(value, loption); + auto sample_size_option = ParseInteger(value, loption); if (sample_size_option < 1 && sample_size_option != -1) { throw BinderException("Unsupported parameter for SAMPLE_SIZE: cannot be smaller than 1"); } @@ -170,7 +170,7 @@ void CSVReaderOptions::SetReadOption(const string &loption, const Value &value, // If -1, we basically read the whole thing sample_size_chunks = NumericLimits().Maximum(); } else { - sample_size_chunks = sample_size_option / STANDARD_VECTOR_SIZE; + sample_size_chunks = NumericCast(sample_size_option / STANDARD_VECTOR_SIZE); if (sample_size_option % STANDARD_VECTOR_SIZE != 0) { sample_size_chunks++; } @@ -179,7 +179,7 @@ void CSVReaderOptions::SetReadOption(const string &loption, const Value &value, } else if (loption == "skip") { SetSkipRows(ParseInteger(value, loption)); } else if (loption == "max_line_size" || loption == "maximum_line_size") { - maximum_line_size = ParseInteger(value, loption); + maximum_line_size = NumericCast(ParseInteger(value, loption)); } else if (loption == "force_not_null") { force_not_null = ParseColumnList(value, expected_names, loption); } else if (loption == "date_format" || loption == "dateformat") { @@ -191,7 +191,7 @@ void CSVReaderOptions::SetReadOption(const string &loption, const Value &value, } else if (loption == "ignore_errors") { ignore_errors = ParseBoolean(value, loption); } else if (loption == "buffer_size") { - buffer_size = ParseInteger(value, loption); + buffer_size = NumericCast(ParseInteger(value, loption)); if (buffer_size == 0) { throw InvalidInputException("Buffer Size option must be higher than 0"); } @@ -221,11 +221,11 @@ void CSVReaderOptions::SetReadOption(const string &loption, const Value &value, rejects_recovery_columns.push_back(col_name); } } else if (loption == "rejects_limit") { - int64_t limit = ParseInteger(value, loption); + auto limit = ParseInteger(value, loption); if (limit < 0) { throw BinderException("Unsupported parameter for REJECTS_LIMIT: cannot be negative"); } - rejects_limit = limit; + rejects_limit = NumericCast(limit); } else { throw BinderException("Unrecognized option for CSV reader \"%s\"", loption); } @@ -523,7 +523,7 @@ void CSVReaderOptions::ToNamedParameters(named_parameter_map_t &named_params) { if (header.IsSetByUser()) { named_params["header"] = Value(GetHeader()); } - named_params["max_line_size"] = Value::BIGINT(maximum_line_size); + named_params["max_line_size"] = Value::BIGINT(NumericCast(maximum_line_size)); if (dialect_options.skip_rows.IsSetByUser()) { named_params["skip"] = Value::BIGINT(GetSkipRows()); } @@ -541,7 +541,7 @@ void CSVReaderOptions::ToNamedParameters(named_parameter_map_t &named_params) { named_params["column_names"] = StringVectorToValue(name_list); } named_params["all_varchar"] = Value::BOOLEAN(all_varchar); - named_params["maximum_line_size"] = Value::BIGINT(maximum_line_size); + named_params["maximum_line_size"] = Value::BIGINT(NumericCast(maximum_line_size)); } } // namespace duckdb diff --git a/src/execution/operator/helper/physical_execute.cpp b/src/execution/operator/helper/physical_execute.cpp index 4a07692181ac..22cab9be8dcc 100644 --- a/src/execution/operator/helper/physical_execute.cpp +++ b/src/execution/operator/helper/physical_execute.cpp @@ -5,7 +5,7 @@ namespace duckdb { PhysicalExecute::PhysicalExecute(PhysicalOperator &plan) - : PhysicalOperator(PhysicalOperatorType::EXECUTE, plan.types, -1), plan(plan) { + : PhysicalOperator(PhysicalOperatorType::EXECUTE, plan.types, idx_t(-1)), plan(plan) { } vector> PhysicalExecute::GetChildren() const { diff --git a/src/execution/operator/helper/physical_reservoir_sample.cpp b/src/execution/operator/helper/physical_reservoir_sample.cpp index 1bc87d25fc3b..b253025f4728 100644 --- a/src/execution/operator/helper/physical_reservoir_sample.cpp +++ b/src/execution/operator/helper/physical_reservoir_sample.cpp @@ -17,7 +17,7 @@ class SampleGlobalSinkState : public GlobalSinkState { } sample = make_uniq(allocator, percentage, options.seed); } else { - auto size = options.sample_size.GetValue(); + auto size = NumericCast(options.sample_size.GetValue()); if (size == 0) { return; } diff --git a/src/execution/operator/join/physical_asof_join.cpp b/src/execution/operator/join/physical_asof_join.cpp index 3996063315c5..8d65a1d6e7eb 100644 --- a/src/execution/operator/join/physical_asof_join.cpp +++ b/src/execution/operator/join/physical_asof_join.cpp @@ -159,7 +159,7 @@ SinkFinalizeType PhysicalAsOfJoin::Finalize(Pipeline &pipeline, Event &event, Cl // The data is all in so we can initialise the left partitioning. const vector> partitions_stats; gstate.lhs_sink = make_uniq(context, lhs_partitions, lhs_orders, children[0]->types, - partitions_stats, 0); + partitions_stats, 0U); gstate.lhs_sink->SyncPartitioning(gstate.rhs_sink); // Find the first group to sort diff --git a/src/execution/operator/join/physical_hash_join.cpp b/src/execution/operator/join/physical_hash_join.cpp index 1e1e9fd12384..8632c44bda13 100644 --- a/src/execution/operator/join/physical_hash_join.cpp +++ b/src/execution/operator/join/physical_hash_join.cpp @@ -88,7 +88,7 @@ PhysicalHashJoin::PhysicalHashJoin(LogicalOperator &op, unique_ptr(TaskScheduler::GetScheduler(context).NumberOfThreads())), temporary_memory_update_count(0), temporary_memory_state(TemporaryMemoryManager::Get(context).Register(context)), finalized(false), scanned_data(false) { @@ -206,7 +206,7 @@ unique_ptr PhysicalHashJoin::InitializeHashTable(ClientContext &c auto count_fun = CountFun::GetFunction(); vector> children; // this is a dummy but we need it to make the hash table understand whats going on - children.push_back(make_uniq_base(count_fun.return_type, 0)); + children.push_back(make_uniq_base(count_fun.return_type, 0U)); aggr = function_binder.BindAggregateFunction(count_fun, std::move(children), nullptr, AggregateType::NON_DISTINCT); correlated_aggregates.push_back(&*aggr); @@ -321,11 +321,11 @@ class HashJoinFinalizeEvent : public BasePipelineEvent { vector> finalize_tasks; auto &ht = *sink.hash_table; const auto chunk_count = ht.GetDataCollection().ChunkCount(); - const idx_t num_threads = TaskScheduler::GetScheduler(context).NumberOfThreads(); + const auto num_threads = NumericCast(TaskScheduler::GetScheduler(context).NumberOfThreads()); if (num_threads == 1 || (ht.Count() < PARALLEL_CONSTRUCT_THRESHOLD && !context.config.verify_parallelism)) { // Single-threaded finalize finalize_tasks.push_back( - make_uniq(shared_from_this(), context, sink, 0, chunk_count, false)); + make_uniq(shared_from_this(), context, sink, 0U, chunk_count, false)); } else { // Parallel finalize auto chunks_per_thread = MaxValue((chunk_count + num_threads - 1) / num_threads, 1); @@ -816,7 +816,7 @@ void HashJoinGlobalSourceState::PrepareBuild(HashJoinGlobalSinkState &sink) { build_chunk_count = data_collection.ChunkCount(); build_chunk_done = 0; - auto num_threads = TaskScheduler::GetScheduler(sink.context).NumberOfThreads(); + auto num_threads = NumericCast(TaskScheduler::GetScheduler(sink.context).NumberOfThreads()); build_chunks_per_thread = MaxValue((build_chunk_count + num_threads - 1) / num_threads, 1); ht.InitializePointerTable(); @@ -847,7 +847,7 @@ void HashJoinGlobalSourceState::PrepareScanHT(HashJoinGlobalSinkState &sink) { full_outer_chunk_count = data_collection.ChunkCount(); full_outer_chunk_done = 0; - auto num_threads = TaskScheduler::GetScheduler(sink.context).NumberOfThreads(); + auto num_threads = NumericCast(TaskScheduler::GetScheduler(sink.context).NumberOfThreads()); full_outer_chunks_per_thread = MaxValue((full_outer_chunk_count + num_threads - 1) / num_threads, 1); global_stage = HashJoinSourceStage::SCAN_HT; diff --git a/src/execution/operator/join/physical_iejoin.cpp b/src/execution/operator/join/physical_iejoin.cpp index 6e21b8602919..bc99e9fbf0a8 100644 --- a/src/execution/operator/join/physical_iejoin.cpp +++ b/src/execution/operator/join/physical_iejoin.cpp @@ -325,7 +325,7 @@ idx_t IEJoinUnion::AppendKey(SortedTable &table, ExpressionExecutor &executor, S payload.data[0].Sequence(rid, increment, scan_count); payload.SetCardinality(scan_count); keys.Fuse(payload); - rid += increment * scan_count; + rid += UnsafeNumericCast(increment) * scan_count; // Sort on the sort columns (which will no longer be needed) keys.Split(payload, payload_idx); @@ -385,7 +385,7 @@ IEJoinUnion::IEJoinUnion(ClientContext &context, const PhysicalIEJoin &op, Sorte payload_layout.Initialize(types); // Sort on the first expression - auto ref = make_uniq(order1.expression->return_type, 0); + auto ref = make_uniq(order1.expression->return_type, 0U); vector orders; orders.emplace_back(order1.type, order1.null_order, std::move(ref)); @@ -426,7 +426,7 @@ IEJoinUnion::IEJoinUnion(ClientContext &context, const PhysicalIEJoin &op, Sorte // Sort on the first expression orders.clear(); - ref = make_uniq(order2.expression->return_type, 0); + ref = make_uniq(order2.expression->return_type, 0U); orders.emplace_back(order2.type, order2.null_order, std::move(ref)); ExpressionExecutor executor(context); @@ -434,7 +434,7 @@ IEJoinUnion::IEJoinUnion(ClientContext &context, const PhysicalIEJoin &op, Sorte l2 = make_uniq(context, orders, payload_layout); for (idx_t base = 0, block_idx = 0; block_idx < l1->BlockCount(); ++block_idx) { - base += AppendKey(*l1, executor, *l2, 1, base, block_idx); + base += AppendKey(*l1, executor, *l2, 1, NumericCast(base), block_idx); } Sort(*l2); diff --git a/src/execution/operator/join/physical_piecewise_merge_join.cpp b/src/execution/operator/join/physical_piecewise_merge_join.cpp index 433ff32982b0..287c797186b7 100644 --- a/src/execution/operator/join/physical_piecewise_merge_join.cpp +++ b/src/execution/operator/join/physical_piecewise_merge_join.cpp @@ -105,7 +105,7 @@ unique_ptr PhysicalPiecewiseMergeJoin::GetGlobalSinkState(Clien unique_ptr PhysicalPiecewiseMergeJoin::GetLocalSinkState(ExecutionContext &context) const { // We only sink the RHS - return make_uniq(context.client, *this, 1); + return make_uniq(context.client, *this, 1U); } SinkResultType PhysicalPiecewiseMergeJoin::Sink(ExecutionContext &context, DataChunk &chunk, @@ -223,7 +223,7 @@ class PiecewiseMergeJoinState : public CachingOperatorState { void ResolveJoinKeys(DataChunk &input) { // sort by join key lhs_global_state = make_uniq(buffer_manager, lhs_order, lhs_layout); - lhs_local_table = make_uniq(context, op, 0); + lhs_local_table = make_uniq(context, op, 0U); lhs_local_table->Sink(input, *lhs_global_state); // Set external (can be forced with the PRAGMA) diff --git a/src/execution/operator/join/physical_range_join.cpp b/src/execution/operator/join/physical_range_join.cpp index 5d6d44f47bff..8962663e96ad 100644 --- a/src/execution/operator/join/physical_range_join.cpp +++ b/src/execution/operator/join/physical_range_join.cpp @@ -126,7 +126,7 @@ class RangeJoinMergeEvent : public BasePipelineEvent { // Schedule tasks equal to the number of threads, which will each merge multiple partitions auto &ts = TaskScheduler::GetScheduler(context); - idx_t num_threads = ts.NumberOfThreads(); + auto num_threads = NumericCast(ts.NumberOfThreads()); vector> iejoin_tasks; for (idx_t tnum = 0; tnum < num_threads; tnum++) { diff --git a/src/execution/operator/order/physical_order.cpp b/src/execution/operator/order/physical_order.cpp index ac933f3fdc84..f74693bd86cb 100644 --- a/src/execution/operator/order/physical_order.cpp +++ b/src/execution/operator/order/physical_order.cpp @@ -143,7 +143,7 @@ class OrderMergeEvent : public BasePipelineEvent { // Schedule tasks equal to the number of threads, which will each merge multiple partitions auto &ts = TaskScheduler::GetScheduler(context); - idx_t num_threads = ts.NumberOfThreads(); + auto num_threads = NumericCast(ts.NumberOfThreads()); vector> merge_tasks; for (idx_t tnum = 0; tnum < num_threads; tnum++) { diff --git a/src/execution/operator/persistent/physical_batch_copy_to_file.cpp b/src/execution/operator/persistent/physical_batch_copy_to_file.cpp index 5447f433cb69..c2c4d06ff9dd 100644 --- a/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +++ b/src/execution/operator/persistent/physical_batch_copy_to_file.cpp @@ -434,7 +434,7 @@ void PhysicalBatchCopyToFile::RepartitionBatches(ClientContext &context, GlobalS // create an empty collection auto new_collection = make_uniq(context, children[0]->types, ColumnDataAllocatorType::HYBRID); - append_batch = make_uniq(0, std::move(new_collection)); + append_batch = make_uniq(0U, std::move(new_collection)); } if (append_batch) { append_batch->collection->InitializeAppend(append_state); @@ -459,7 +459,7 @@ void PhysicalBatchCopyToFile::RepartitionBatches(ClientContext &context, GlobalS auto new_collection = make_uniq(context, children[0]->types, ColumnDataAllocatorType::HYBRID); - append_batch = make_uniq(0, std::move(new_collection)); + append_batch = make_uniq(0U, std::move(new_collection)); append_batch->collection->InitializeAppend(append_state); } } @@ -605,7 +605,7 @@ SourceResultType PhysicalBatchCopyToFile::GetData(ExecutionContext &context, Dat auto &g = sink_state->Cast(); chunk.SetCardinality(1); - chunk.SetValue(0, 0, Value::BIGINT(g.rows_copied)); + chunk.SetValue(0, 0, Value::BIGINT(NumericCast(g.rows_copied.load()))); return SourceResultType::FINISHED; } diff --git a/src/execution/operator/persistent/physical_batch_insert.cpp b/src/execution/operator/persistent/physical_batch_insert.cpp index f2f242a90d78..9f54ec44c6c9 100644 --- a/src/execution/operator/persistent/physical_batch_insert.cpp +++ b/src/execution/operator/persistent/physical_batch_insert.cpp @@ -166,7 +166,8 @@ class BatchInsertLocalState : public LocalSinkState { void CreateNewCollection(DuckTableEntry &table, const vector &insert_types) { auto &table_info = table.GetStorage().info; auto &block_manager = TableIOManager::Get(table.GetStorage()).GetBlockManagerForRowData(); - current_collection = make_uniq(table_info, block_manager, insert_types, MAX_ROW_ID); + current_collection = + make_uniq(table_info, block_manager, insert_types, NumericCast(MAX_ROW_ID)); current_collection->InitializeEmpty(); current_collection->InitializeAppend(current_append_state); } @@ -303,8 +304,8 @@ void BatchInsertGlobalState::ScheduleMergeTasks(idx_t min_batch_index) { auto &scheduled_task = to_be_scheduled_tasks[i - 1]; if (scheduled_task.start_index + 1 < scheduled_task.end_index) { // erase all entries except the first one - collections.erase(collections.begin() + scheduled_task.start_index + 1, - collections.begin() + scheduled_task.end_index); + collections.erase(collections.begin() + NumericCast(scheduled_task.start_index) + 1, + collections.begin() + NumericCast(scheduled_task.end_index)); } } } @@ -613,7 +614,7 @@ SourceResultType PhysicalBatchInsert::GetData(ExecutionContext &context, DataChu auto &insert_gstate = sink_state->Cast(); chunk.SetCardinality(1); - chunk.SetValue(0, 0, Value::BIGINT(insert_gstate.insert_count)); + chunk.SetValue(0, 0, Value::BIGINT(NumericCast(insert_gstate.insert_count))); return SourceResultType::FINISHED; } diff --git a/src/execution/operator/persistent/physical_copy_to_file.cpp b/src/execution/operator/persistent/physical_copy_to_file.cpp index 5925e9f012eb..e4e58b3b234b 100644 --- a/src/execution/operator/persistent/physical_copy_to_file.cpp +++ b/src/execution/operator/persistent/physical_copy_to_file.cpp @@ -423,7 +423,7 @@ SourceResultType PhysicalCopyToFile::GetData(ExecutionContext &context, DataChun auto &g = sink_state->Cast(); chunk.SetCardinality(1); - chunk.SetValue(0, 0, Value::BIGINT(g.rows_copied)); + chunk.SetValue(0, 0, Value::BIGINT(NumericCast(g.rows_copied.load()))); return SourceResultType::FINISHED; } diff --git a/src/execution/operator/persistent/physical_delete.cpp b/src/execution/operator/persistent/physical_delete.cpp index 42085f0e6128..4fc17049032a 100644 --- a/src/execution/operator/persistent/physical_delete.cpp +++ b/src/execution/operator/persistent/physical_delete.cpp @@ -91,7 +91,7 @@ SourceResultType PhysicalDelete::GetData(ExecutionContext &context, DataChunk &c auto &g = sink_state->Cast(); if (!return_chunk) { chunk.SetCardinality(1); - chunk.SetValue(0, 0, Value::BIGINT(g.deleted_count)); + chunk.SetValue(0, 0, Value::BIGINT(NumericCast(g.deleted_count))); return SourceResultType::FINISHED; } diff --git a/src/execution/operator/persistent/physical_export.cpp b/src/execution/operator/persistent/physical_export.cpp index 3979a88eeb59..121a7aa2a85f 100644 --- a/src/execution/operator/persistent/physical_export.cpp +++ b/src/execution/operator/persistent/physical_export.cpp @@ -31,7 +31,7 @@ static void WriteStringStreamToFile(FileSystem &fs, stringstream &ss, const stri auto ss_string = ss.str(); auto handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE_NEW | FileLockType::WRITE_LOCK); - fs.Write(*handle, (void *)ss_string.c_str(), ss_string.size()); + fs.Write(*handle, (void *)ss_string.c_str(), NumericCast(ss_string.size())); handle.reset(); } diff --git a/src/execution/operator/persistent/physical_insert.cpp b/src/execution/operator/persistent/physical_insert.cpp index fd17fbb8ea69..768334b50abe 100644 --- a/src/execution/operator/persistent/physical_insert.cpp +++ b/src/execution/operator/persistent/physical_insert.cpp @@ -448,7 +448,7 @@ SinkResultType PhysicalInsert::Sink(ExecutionContext &context, DataChunk &chunk, auto &table_info = storage.info; auto &block_manager = TableIOManager::Get(storage).GetBlockManagerForRowData(); lstate.local_collection = - make_uniq(table_info, block_manager, insert_types, MAX_ROW_ID); + make_uniq(table_info, block_manager, insert_types, NumericCast(MAX_ROW_ID)); lstate.local_collection->InitializeEmpty(); lstate.local_collection->InitializeAppend(lstate.local_append_state); lstate.writer = &gstate.table.GetStorage().CreateOptimisticWriter(context.client); @@ -540,7 +540,7 @@ SourceResultType PhysicalInsert::GetData(ExecutionContext &context, DataChunk &c auto &insert_gstate = sink_state->Cast(); if (!return_chunk) { chunk.SetCardinality(1); - chunk.SetValue(0, 0, Value::BIGINT(insert_gstate.insert_count)); + chunk.SetValue(0, 0, Value::BIGINT(NumericCast(insert_gstate.insert_count))); return SourceResultType::FINISHED; } diff --git a/src/execution/operator/persistent/physical_update.cpp b/src/execution/operator/persistent/physical_update.cpp index fb7f1c30054d..c8bab2854c06 100644 --- a/src/execution/operator/persistent/physical_update.cpp +++ b/src/execution/operator/persistent/physical_update.cpp @@ -175,7 +175,7 @@ SourceResultType PhysicalUpdate::GetData(ExecutionContext &context, DataChunk &c auto &g = sink_state->Cast(); if (!return_chunk) { chunk.SetCardinality(1); - chunk.SetValue(0, 0, Value::BIGINT(g.updated_count)); + chunk.SetValue(0, 0, Value::BIGINT(NumericCast(g.updated_count))); return SourceResultType::FINISHED; } diff --git a/src/execution/physical_plan/plan_comparison_join.cpp b/src/execution/physical_plan/plan_comparison_join.cpp index 3397a0604fa6..743f8189b61c 100644 --- a/src/execution/physical_plan/plan_comparison_join.cpp +++ b/src/execution/physical_plan/plan_comparison_join.cpp @@ -109,7 +109,7 @@ void CheckForPerfectJoinOpt(LogicalComparisonJoin &op, PerfectHashJoinStats &joi join_state.build_min = NumericStats::Min(stats_build); join_state.build_max = NumericStats::Max(stats_build); join_state.estimated_cardinality = op.estimated_cardinality; - join_state.build_range = build_range; + join_state.build_range = NumericCast(build_range); if (join_state.build_range > MAX_BUILD_SIZE) { return; } diff --git a/src/execution/physical_plan/plan_create_table.cpp b/src/execution/physical_plan/plan_create_table.cpp index be854e0da308..06b728ee47eb 100644 --- a/src/execution/physical_plan/plan_create_table.cpp +++ b/src/execution/physical_plan/plan_create_table.cpp @@ -22,10 +22,10 @@ unique_ptr DuckCatalog::PlanCreateTableAs(ClientContext &conte auto num_threads = TaskScheduler::GetScheduler(context).NumberOfThreads(); unique_ptr create; if (!parallel_streaming_insert && use_batch_index) { - create = make_uniq(op, op.schema, std::move(op.info), 0); + create = make_uniq(op, op.schema, std::move(op.info), 0U); } else { - create = make_uniq(op, op.schema, std::move(op.info), 0, + create = make_uniq(op, op.schema, std::move(op.info), 0U, parallel_streaming_insert && num_threads > 1); } diff --git a/src/execution/physical_plan/plan_top_n.cpp b/src/execution/physical_plan/plan_top_n.cpp index b3043440dcd4..9748904c57a8 100644 --- a/src/execution/physical_plan/plan_top_n.cpp +++ b/src/execution/physical_plan/plan_top_n.cpp @@ -9,8 +9,8 @@ unique_ptr PhysicalPlanGenerator::CreatePlan(LogicalTopN &op) auto plan = CreatePlan(*op.children[0]); - auto top_n = - make_uniq(op.types, std::move(op.orders), (idx_t)op.limit, op.offset, op.estimated_cardinality); + auto top_n = make_uniq(op.types, std::move(op.orders), NumericCast(op.limit), + NumericCast(op.offset), op.estimated_cardinality); top_n->children.push_back(std::move(plan)); return std::move(top_n); } diff --git a/src/main/capi/arrow-c.cpp b/src/main/capi/arrow-c.cpp index 335b936950eb..f74d9e22fdb8 100644 --- a/src/main/capi/arrow-c.cpp +++ b/src/main/capi/arrow-c.cpp @@ -127,7 +127,7 @@ idx_t duckdb_arrow_rows_changed(duckdb_arrow result) { auto rows = collection.GetRows(); D_ASSERT(row_count == 1); D_ASSERT(rows.size() == 1); - rows_changed = rows[0].GetValue(0).GetValue(); + rows_changed = duckdb::NumericCast(rows[0].GetValue(0).GetValue()); } return rows_changed; } @@ -291,8 +291,8 @@ duckdb_state duckdb_arrow_scan(duckdb_connection connection, const char *table_n } typedef void (*release_fn_t)(ArrowSchema *); - std::vector release_fns(schema.n_children); - for (int64_t i = 0; i < schema.n_children; i++) { + std::vector release_fns(duckdb::NumericCast(schema.n_children)); + for (idx_t i = 0; i < duckdb::NumericCast(schema.n_children); i++) { auto child = schema.children[i]; release_fns[i] = child->release; child->release = arrow_array_stream_wrapper::EmptySchemaRelease; @@ -301,7 +301,7 @@ duckdb_state duckdb_arrow_scan(duckdb_connection connection, const char *table_n auto ret = arrow_array_stream_wrapper::Ingest(connection, table_name, stream); // Restore release functions. - for (int64_t i = 0; i < schema.n_children; i++) { + for (idx_t i = 0; i < duckdb::NumericCast(schema.n_children); i++) { schema.children[i]->release = release_fns[i]; } diff --git a/src/main/capi/result-c.cpp b/src/main/capi/result-c.cpp index 81c03cc5b260..5b17c0132751 100644 --- a/src/main/capi/result-c.cpp +++ b/src/main/capi/result-c.cpp @@ -117,7 +117,7 @@ struct CDecimalConverter : public CBaseConverter { template static DST Convert(SRC input) { duckdb_hugeint result; - result.lower = input; + result.lower = NumericCast(input); result.upper = 0; return result; } @@ -350,7 +350,7 @@ bool DeprecatedMaterializeResult(duckdb_result *result) { // update total changes auto row_changes = materialized.GetValue(0, 0); if (!row_changes.IsNull() && row_changes.DefaultTryCastAs(LogicalType::BIGINT)) { - result->__deprecated_rows_changed = row_changes.GetValue(); + result->__deprecated_rows_changed = NumericCast(row_changes.GetValue()); } } // now write the data diff --git a/src/main/client_context.cpp b/src/main/client_context.cpp index 909593aa66f8..8c753d2fadc8 100644 --- a/src/main/client_context.cpp +++ b/src/main/client_context.cpp @@ -482,7 +482,8 @@ ClientContext::PendingPreparedStatementInternal(ClientContextLock &lock, shared_ display_create_func = config.display_create_func ? config.display_create_func : ProgressBar::DefaultProgressBarDisplay; } - active_query->progress_bar = make_uniq(executor, config.wait_time, display_create_func); + active_query->progress_bar = + make_uniq(executor, NumericCast(config.wait_time), display_create_func); active_query->progress_bar->Start(); query_progress.Restart(); } diff --git a/src/main/database.cpp b/src/main/database.cpp index 2e3fd0204e3a..50cd320a261a 100644 --- a/src/main/database.cpp +++ b/src/main/database.cpp @@ -381,7 +381,7 @@ const DBConfig &DBConfig::GetConfig(const ClientContext &context) { } idx_t DatabaseInstance::NumberOfThreads() { - return scheduler->NumberOfThreads(); + return NumericCast(scheduler->NumberOfThreads()); } const unordered_set &DatabaseInstance::LoadedExtensions() { diff --git a/src/main/query_profiler.cpp b/src/main/query_profiler.cpp index 176b1e4359c8..79e3020679c7 100644 --- a/src/main/query_profiler.cpp +++ b/src/main/query_profiler.cpp @@ -313,7 +313,7 @@ static string DrawPadded(const string &str, idx_t width) { } else { width -= str.size(); auto half_spaces = width / 2; - auto extra_left_space = width % 2 != 0 ? 1 : 0; + auto extra_left_space = NumericCast(width % 2 != 0 ? 1 : 0); return string(half_spaces + extra_left_space, ' ') + str + string(half_spaces, ' '); } } diff --git a/src/main/relation.cpp b/src/main/relation.cpp index 970ab4a58add..76095c2b7832 100644 --- a/src/main/relation.cpp +++ b/src/main/relation.cpp @@ -374,7 +374,7 @@ unique_ptr Relation::GetQueryNode() { } void Relation::Head(idx_t limit) { - auto limit_node = Limit(limit); + auto limit_node = Limit(NumericCast(limit)); limit_node->Execute()->Print(); } // LCOV_EXCL_STOP diff --git a/src/main/secret/secret.cpp b/src/main/secret/secret.cpp index 450e5621c323..097a2194cc3f 100644 --- a/src/main/secret/secret.cpp +++ b/src/main/secret/secret.cpp @@ -15,7 +15,7 @@ int64_t BaseSecret::MatchScore(const string &path) const { continue; } if (StringUtil::StartsWith(path, prefix)) { - longest_match = MaxValue(prefix.length(), longest_match); + longest_match = MaxValue(NumericCast(prefix.length()), longest_match); } } return longest_match; diff --git a/src/main/settings/settings.cpp b/src/main/settings/settings.cpp index 3344b77dc017..0726d926355d 100644 --- a/src/main/settings/settings.cpp +++ b/src/main/settings/settings.cpp @@ -749,7 +749,7 @@ void ExternalThreadsSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, c if (new_val < 0) { throw SyntaxException("Must have a non-negative number of external threads!"); } - idx_t new_external_threads = new_val; + auto new_external_threads = NumericCast(new_val); if (db) { TaskScheduler::GetScheduler(*db).SetThreads(config.options.maximum_threads, new_external_threads); } @@ -766,7 +766,7 @@ void ExternalThreadsSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) Value ExternalThreadsSetting::GetSetting(ClientContext &context) { auto &config = DBConfig::GetConfig(context); - return Value::BIGINT(config.options.external_threads); + return Value::BIGINT(NumericCast(config.options.external_threads)); } //===--------------------------------------------------------------------===// @@ -1000,7 +1000,7 @@ void PartitionedWriteFlushThreshold::SetLocal(ClientContext &context, const Valu } Value PartitionedWriteFlushThreshold::GetSetting(ClientContext &context) { - return Value::BIGINT(ClientConfig::GetConfig(context).partitioned_write_flush_threshold); + return Value::BIGINT(NumericCast(ClientConfig::GetConfig(context).partitioned_write_flush_threshold)); } //===--------------------------------------------------------------------===// @@ -1030,11 +1030,11 @@ void PerfectHashThresholdSetting::SetLocal(ClientContext &context, const Value & if (bits < 0 || bits > 32) { throw ParserException("Perfect HT threshold out of range: should be within range 0 - 32"); } - ClientConfig::GetConfig(context).perfect_ht_threshold = bits; + ClientConfig::GetConfig(context).perfect_ht_threshold = NumericCast(bits); } Value PerfectHashThresholdSetting::GetSetting(ClientContext &context) { - return Value::BIGINT(ClientConfig::GetConfig(context).perfect_ht_threshold); + return Value::BIGINT(NumericCast(ClientConfig::GetConfig(context).perfect_ht_threshold)); } //===--------------------------------------------------------------------===// @@ -1049,7 +1049,7 @@ void PivotFilterThreshold::SetLocal(ClientContext &context, const Value &input) } Value PivotFilterThreshold::GetSetting(ClientContext &context) { - return Value::BIGINT(ClientConfig::GetConfig(context).pivot_filter_threshold); + return Value::BIGINT(NumericCast(ClientConfig::GetConfig(context).pivot_filter_threshold)); } //===--------------------------------------------------------------------===// @@ -1064,7 +1064,7 @@ void PivotLimitSetting::SetLocal(ClientContext &context, const Value &input) { } Value PivotLimitSetting::GetSetting(ClientContext &context) { - return Value::BIGINT(ClientConfig::GetConfig(context).pivot_limit); + return Value::BIGINT(NumericCast(ClientConfig::GetConfig(context).pivot_limit)); } //===--------------------------------------------------------------------===// @@ -1280,7 +1280,7 @@ void ThreadsSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Val if (new_val < 1) { throw SyntaxException("Must have at least 1 thread!"); } - idx_t new_maximum_threads = new_val; + auto new_maximum_threads = NumericCast(new_val); if (db) { TaskScheduler::GetScheduler(*db).SetThreads(new_maximum_threads, config.options.external_threads); } @@ -1297,7 +1297,7 @@ void ThreadsSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { Value ThreadsSetting::GetSetting(ClientContext &context) { auto &config = DBConfig::GetConfig(context); - return Value::BIGINT(config.options.maximum_threads); + return Value::BIGINT(NumericCast(config.options.maximum_threads)); } //===--------------------------------------------------------------------===// diff --git a/src/parallel/pipeline.cpp b/src/parallel/pipeline.cpp index e0ace608b19f..643cb7c8ee54 100644 --- a/src/parallel/pipeline.cpp +++ b/src/parallel/pipeline.cpp @@ -110,9 +110,9 @@ bool Pipeline::ScheduleParallel(shared_ptr &event) { "Attempting to schedule a pipeline where the sink requires batch index but source does not support it"); } } - idx_t max_threads = source_state->MaxThreads(); + auto max_threads = source_state->MaxThreads(); auto &scheduler = TaskScheduler::GetScheduler(executor.context); - idx_t active_threads = scheduler.NumberOfThreads(); + auto active_threads = NumericCast(scheduler.NumberOfThreads()); if (max_threads > active_threads) { max_threads = active_threads; } diff --git a/src/parallel/task_scheduler.cpp b/src/parallel/task_scheduler.cpp index 5036483388bc..b0f8997ee82a 100644 --- a/src/parallel/task_scheduler.cpp +++ b/src/parallel/task_scheduler.cpp @@ -260,7 +260,7 @@ void TaskScheduler::SetAllocatorFlushTreshold(idx_t threshold) { void TaskScheduler::Signal(idx_t n) { #ifndef DUCKDB_NO_THREADS - queue->semaphore.signal(n); + queue->semaphore.signal(NumericCast(n)); #endif } @@ -279,7 +279,7 @@ void TaskScheduler::RelaunchThreads() { void TaskScheduler::RelaunchThreadsInternal(int32_t n) { #ifndef DUCKDB_NO_THREADS auto &config = DBConfig::GetConfig(db); - idx_t new_thread_count = n; + auto new_thread_count = NumericCast(n); if (threads.size() == new_thread_count) { current_thread_count = NumericCast(threads.size() + config.options.external_threads); return; From a0f7ff1dabff4a626ccf1ca17695cc31860862cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Thu, 4 Apr 2024 16:27:38 +0200 Subject: [PATCH 168/603] orrrr --- src/include/duckdb/common/bitpacking.hpp | 2 +- .../duckdb/storage/buffer/block_handle.hpp | 1 + src/storage/arena_allocator.cpp | 8 ++++--- .../buffer/buffer_pool_reservation.cpp | 4 ++-- src/storage/compression/bitpacking.cpp | 12 +++++----- .../compression/dictionary_compression.cpp | 12 ++++++---- .../compression/fixed_size_uncompressed.cpp | 2 +- src/storage/compression/rle.cpp | 2 +- .../compression/string_uncompressed.cpp | 9 +++---- .../compression/validity_uncompressed.cpp | 2 +- src/storage/data_table.cpp | 4 ++-- src/storage/local_storage.cpp | 11 +++++---- src/storage/single_file_block_manager.cpp | 24 +++++++++---------- src/storage/standard_buffer_manager.cpp | 16 ++++++------- src/storage/storage_manager.cpp | 6 ++--- src/storage/temporary_file_manager.cpp | 6 ++--- src/storage/temporary_memory_manager.cpp | 2 +- 17 files changed, 65 insertions(+), 58 deletions(-) diff --git a/src/include/duckdb/common/bitpacking.hpp b/src/include/duckdb/common/bitpacking.hpp index 7bddedfc42d6..3b499a4b4368 100644 --- a/src/include/duckdb/common/bitpacking.hpp +++ b/src/include/duckdb/common/bitpacking.hpp @@ -112,7 +112,7 @@ class BitpackingPrimitives { return num_to_round; } - return num_to_round + BITPACKING_ALGORITHM_GROUP_SIZE - remainder; + return num_to_round + BITPACKING_ALGORITHM_GROUP_SIZE - NumericCast(remainder); } private: diff --git a/src/include/duckdb/storage/buffer/block_handle.hpp b/src/include/duckdb/storage/buffer/block_handle.hpp index 3de827ea8fcf..34ac9d1c4b7a 100644 --- a/src/include/duckdb/storage/buffer/block_handle.hpp +++ b/src/include/duckdb/storage/buffer/block_handle.hpp @@ -11,6 +11,7 @@ #include "duckdb/common/atomic.hpp" #include "duckdb/common/common.hpp" #include "duckdb/common/mutex.hpp" +#include "duckdb/common/numeric_utils.hpp" #include "duckdb/storage/storage_info.hpp" #include "duckdb/common/file_buffer.hpp" #include "duckdb/common/enums/memory_tag.hpp" diff --git a/src/storage/arena_allocator.cpp b/src/storage/arena_allocator.cpp index c659b2c13d11..d7198c06281e 100644 --- a/src/storage/arena_allocator.cpp +++ b/src/storage/arena_allocator.cpp @@ -1,6 +1,7 @@ #include "duckdb/storage/arena_allocator.hpp" #include "duckdb/common/assert.hpp" +#include "duckdb/common/numeric_utils.hpp" namespace duckdb { @@ -88,10 +89,11 @@ data_ptr_t ArenaAllocator::Reallocate(data_ptr_t pointer, idx_t old_size, idx_t } auto head_ptr = head->data.get() + head->current_position; - int64_t diff = size - old_size; - if (pointer == head_ptr && (size < old_size || head->current_position + diff <= head->maximum_size)) { + int64_t diff = NumericCast(size) - NumericCast(old_size); + if (pointer == head_ptr && (size < old_size || NumericCast(head->current_position) + diff <= + NumericCast(head->maximum_size))) { // passed pointer is the head pointer, and the diff fits on the current chunk - head->current_position += diff; + head->current_position += NumericCast(diff); return pointer; } else { // allocate new memory diff --git a/src/storage/buffer/buffer_pool_reservation.cpp b/src/storage/buffer/buffer_pool_reservation.cpp index f22a96ffa58f..60a7bc882e61 100644 --- a/src/storage/buffer/buffer_pool_reservation.cpp +++ b/src/storage/buffer/buffer_pool_reservation.cpp @@ -23,8 +23,8 @@ BufferPoolReservation::~BufferPoolReservation() { } void BufferPoolReservation::Resize(idx_t new_size) { - int64_t delta = (int64_t)new_size - size; - pool.IncreaseUsedMemory(tag, delta); + int64_t delta = NumericCast(new_size) - NumericCast(size); + pool.IncreaseUsedMemory(tag, NumericCast(delta)); size = new_size; } diff --git a/src/storage/compression/bitpacking.cpp b/src/storage/compression/bitpacking.cpp index fa7f4a7c79f3..efdd03087bc8 100644 --- a/src/storage/compression/bitpacking.cpp +++ b/src/storage/compression/bitpacking.cpp @@ -64,7 +64,7 @@ typedef uint32_t bitpacking_metadata_encoded_t; static bitpacking_metadata_encoded_t EncodeMeta(bitpacking_metadata_t metadata) { D_ASSERT(metadata.offset <= 0x00FFFFFF); // max uint24_t bitpacking_metadata_encoded_t encoded_value = metadata.offset; - encoded_value |= (uint8_t)metadata.mode << 24; + encoded_value |= UnsafeNumericCast((uint8_t)metadata.mode << 24); return encoded_value; } static bitpacking_metadata_t DecodeMeta(bitpacking_metadata_encoded_t *metadata_encoded) { @@ -471,8 +471,8 @@ struct BitpackingCompressState : public CompressionState { }; bool CanStore(idx_t data_bytes, idx_t meta_bytes) { - auto required_data_bytes = AlignValue((data_ptr + data_bytes) - data_ptr); - auto required_meta_bytes = Storage::BLOCK_SIZE - (metadata_ptr - data_ptr) + meta_bytes; + auto required_data_bytes = AlignValue(UnsafeNumericCast((data_ptr + data_bytes) - data_ptr)); + auto required_meta_bytes = Storage::BLOCK_SIZE - UnsafeNumericCast(metadata_ptr - data_ptr) + meta_bytes; return required_data_bytes + required_meta_bytes <= Storage::BLOCK_SIZE - BitpackingPrimitives::BITPACKING_HEADER_SIZE; @@ -514,8 +514,8 @@ struct BitpackingCompressState : public CompressionState { auto base_ptr = handle.Ptr(); // Compact the segment by moving the metadata next to the data. - idx_t metadata_offset = AlignValue(data_ptr - base_ptr); - idx_t metadata_size = base_ptr + Storage::BLOCK_SIZE - metadata_ptr; + auto metadata_offset = NumericCast(AlignValue(data_ptr - base_ptr)); + auto metadata_size = NumericCast(base_ptr + Storage::BLOCK_SIZE - metadata_ptr); idx_t total_segment_size = metadata_offset + metadata_size; // Asserting things are still sane here @@ -866,7 +866,7 @@ template void BitpackingFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id, Vector &result, idx_t result_idx) { BitpackingScanState scan_state(segment); - scan_state.Skip(segment, row_id); + scan_state.Skip(segment, NumericCast(row_id)); D_ASSERT(scan_state.current_group_offset < BITPACKING_METADATA_GROUP_SIZE); diff --git a/src/storage/compression/dictionary_compression.cpp b/src/storage/compression/dictionary_compression.cpp index 4f2bce492d11..79ccc64471b9 100644 --- a/src/storage/compression/dictionary_compression.cpp +++ b/src/storage/compression/dictionary_compression.cpp @@ -456,7 +456,8 @@ unique_ptr DictionaryCompressionStorage::StringInitScan(Column for (uint32_t i = 0; i < index_buffer_count; i++) { // NOTE: the passing of dict_child_vector, will not be used, its for big strings uint16_t str_len = GetStringLength(index_buffer_ptr, i); - dict_child_data[i] = FetchStringFromDict(segment, dict, baseptr, index_buffer_ptr[i], str_len); + dict_child_data[i] = + FetchStringFromDict(segment, dict, baseptr, UnsafeNumericCast(index_buffer_ptr[i]), str_len); } return std::move(state); @@ -509,7 +510,8 @@ void DictionaryCompressionStorage::StringScanPartial(ColumnSegment &segment, Col auto string_number = scan_state.sel_vec->get_index(i + start_offset); auto dict_offset = index_buffer_ptr[string_number]; auto str_len = GetStringLength(index_buffer_ptr, UnsafeNumericCast(string_number)); - result_data[result_offset + i] = FetchStringFromDict(segment, dict, baseptr, dict_offset, str_len); + result_data[result_offset + i] = + FetchStringFromDict(segment, dict, baseptr, UnsafeNumericCast(dict_offset), str_len); } } else { @@ -559,11 +561,11 @@ void DictionaryCompressionStorage::StringFetchRow(ColumnSegment &segment, Column auto result_data = FlatVector::GetData(result); // Handling non-bitpacking-group-aligned start values; - idx_t start_offset = row_id % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE; + idx_t start_offset = NumericCast(row_id) % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE; // Decompress part of selection buffer we need for this value. sel_t decompression_buffer[BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE]; - data_ptr_t src = data_ptr_cast(&base_data[((row_id - start_offset) * width) / 8]); + data_ptr_t src = data_ptr_cast(&base_data[((NumericCast(row_id) - start_offset) * width) / 8]); BitpackingPrimitives::UnPackBuffer(data_ptr_cast(decompression_buffer), src, BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE, width); @@ -571,7 +573,7 @@ void DictionaryCompressionStorage::StringFetchRow(ColumnSegment &segment, Column auto dict_offset = index_buffer_ptr[selection_value]; uint16_t str_len = GetStringLength(index_buffer_ptr, selection_value); - result_data[result_idx] = FetchStringFromDict(segment, dict, baseptr, dict_offset, str_len); + result_data[result_idx] = FetchStringFromDict(segment, dict, baseptr, NumericCast(dict_offset), str_len); } //===--------------------------------------------------------------------===// diff --git a/src/storage/compression/fixed_size_uncompressed.cpp b/src/storage/compression/fixed_size_uncompressed.cpp index c1b210d787e2..bbccd3f0e166 100644 --- a/src/storage/compression/fixed_size_uncompressed.cpp +++ b/src/storage/compression/fixed_size_uncompressed.cpp @@ -172,7 +172,7 @@ void FixedSizeFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t ro auto handle = buffer_manager.Pin(segment.block); // first fetch the data from the base table - auto data_ptr = handle.Ptr() + segment.GetBlockOffset() + row_id * sizeof(T); + auto data_ptr = handle.Ptr() + segment.GetBlockOffset() + NumericCast(row_id) * sizeof(T); memcpy(FlatVector::GetData(result) + result_idx * sizeof(T), data_ptr, sizeof(T)); } diff --git a/src/storage/compression/rle.cpp b/src/storage/compression/rle.cpp index edc111dd337a..e518b14602f8 100644 --- a/src/storage/compression/rle.cpp +++ b/src/storage/compression/rle.cpp @@ -378,7 +378,7 @@ void RLEScan(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, V template void RLEFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id, Vector &result, idx_t result_idx) { RLEScanState scan_state(segment); - scan_state.Skip(segment, row_id); + scan_state.Skip(segment, NumericCast(row_id)); auto data = scan_state.handle.Ptr() + segment.GetBlockOffset(); auto data_pointer = reinterpret_cast(data + RLEConstants::RLE_HEADER_SIZE); diff --git a/src/storage/compression/string_uncompressed.cpp b/src/storage/compression/string_uncompressed.cpp index bfdebbe35a36..dea2ef6fc9f7 100644 --- a/src/storage/compression/string_uncompressed.cpp +++ b/src/storage/compression/string_uncompressed.cpp @@ -87,7 +87,7 @@ void UncompressedStringStorage::StringScanPartial(ColumnSegment &segment, Column for (idx_t i = 0; i < scan_count; i++) { // std::abs used since offsets can be negative to indicate big strings - uint32_t string_length = std::abs(base_data[start + i]) - std::abs(previous_offset); + auto string_length = UnsafeNumericCast(std::abs(base_data[start + i]) - std::abs(previous_offset)); result_data[result_offset + i] = FetchStringFromDict(segment, dict, result, baseptr, base_data[start + i], string_length); previous_offset = base_data[start + i]; @@ -133,9 +133,9 @@ void UncompressedStringStorage::StringFetchRow(ColumnSegment &segment, ColumnFet uint32_t string_length; if ((idx_t)row_id == 0) { // edge case where this is the first string in the dict - string_length = std::abs(dict_offset); + string_length = NumericCast(std::abs(dict_offset)); } else { - string_length = std::abs(dict_offset) - std::abs(base_data[row_id - 1]); + string_length = NumericCast(std::abs(dict_offset) - std::abs(base_data[row_id - 1])); } result_data[result_idx] = FetchStringFromDict(segment, dict, result, baseptr, dict_offset, string_length); } @@ -347,7 +347,8 @@ string_t UncompressedStringStorage::ReadOverflowString(ColumnSegment &segment, V // now append the string to the single buffer while (remaining > 0) { - idx_t to_write = MinValue(remaining, Storage::BLOCK_SIZE - sizeof(block_id_t) - offset); + idx_t to_write = + MinValue(remaining, Storage::BLOCK_SIZE - sizeof(block_id_t) - UnsafeNumericCast(offset)); memcpy(target_ptr, handle.Ptr() + offset, to_write); remaining -= to_write; offset += to_write; diff --git a/src/storage/compression/validity_uncompressed.cpp b/src/storage/compression/validity_uncompressed.cpp index 3df0cae80f91..f2f01533b89c 100644 --- a/src/storage/compression/validity_uncompressed.cpp +++ b/src/storage/compression/validity_uncompressed.cpp @@ -384,7 +384,7 @@ void ValidityFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row auto dataptr = handle.Ptr() + segment.GetBlockOffset(); ValidityMask mask(reinterpret_cast(dataptr)); auto &result_mask = FlatVector::Validity(result); - if (!mask.RowIsValidUnsafe(row_id)) { + if (!mask.RowIsValidUnsafe(NumericCast(row_id))) { result_mask.SetInvalid(result_idx); } } diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 08c389557ee0..a48e6163e6c3 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -742,7 +742,7 @@ void DataTable::AppendLock(TableAppendState &state) { if (!is_root) { throw TransactionException("Transaction conflict: adding entries to a table that has been altered!"); } - state.row_start = row_groups->GetTotalRows(); + state.row_start = NumericCast(row_groups->GetTotalRows()); state.current_row = state.row_start; } @@ -855,7 +855,7 @@ void DataTable::RevertAppend(idx_t start_row, idx_t count) { idx_t scan_count = MinValue(count, row_groups->GetTotalRows() - start_row); ScanTableSegment(start_row, scan_count, [&](DataChunk &chunk) { for (idx_t i = 0; i < chunk.size(); i++) { - row_data[i] = current_row_base + i; + row_data[i] = NumericCast(current_row_base + i); } info->indexes.Scan([&](Index &index) { index.Delete(chunk, row_identifiers); diff --git a/src/storage/local_storage.cpp b/src/storage/local_storage.cpp index 513a18e2faf1..b695f27f880e 100644 --- a/src/storage/local_storage.cpp +++ b/src/storage/local_storage.cpp @@ -191,7 +191,7 @@ void LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, TableAppen return true; }); if (append_to_table) { - table.RevertAppendInternal(append_state.row_start); + table.RevertAppendInternal(NumericCast(append_state.row_start)); } // we need to vacuum the indexes to remove any buffers that are now empty @@ -361,8 +361,9 @@ void LocalStorage::InitializeAppend(LocalAppendState &state, DataTable &table) { void LocalStorage::Append(LocalAppendState &state, DataChunk &chunk) { // append to unique indices (if any) auto storage = state.storage; - idx_t base_id = MAX_ROW_ID + storage->row_groups->GetTotalRows() + state.append_state.total_append_count; - auto error = DataTable::AppendToIndexes(storage->indexes, chunk, base_id); + idx_t base_id = + NumericCast(MAX_ROW_ID) + storage->row_groups->GetTotalRows() + state.append_state.total_append_count; + auto error = DataTable::AppendToIndexes(storage->indexes, chunk, NumericCast(base_id)); if (error.HasError()) { error.Throw(); } @@ -383,7 +384,7 @@ void LocalStorage::LocalMerge(DataTable &table, RowGroupCollection &collection) auto &storage = table_manager.GetOrCreateStorage(table); if (!storage.indexes.Empty()) { // append data to indexes if required - row_t base_id = MAX_ROW_ID + storage.row_groups->GetTotalRows(); + row_t base_id = MAX_ROW_ID + NumericCast(storage.row_groups->GetTotalRows()); auto error = storage.AppendToIndexes(transaction, collection, storage.indexes, table.GetTypes(), base_id); if (error.HasError()) { error.Throw(); @@ -447,7 +448,7 @@ void LocalStorage::Flush(DataTable &table, LocalTableStorage &storage) { TableAppendState append_state; table.AppendLock(append_state); - transaction.PushAppend(table, append_state.row_start, append_count); + transaction.PushAppend(table, NumericCast(append_state.row_start), append_count); if ((append_state.row_start == 0 || storage.row_groups->GetTotalRows() >= MERGE_THRESHOLD) && storage.deleted_rows == 0) { // table is currently empty OR we are bulk appending: move over the storage directly diff --git a/src/storage/single_file_block_manager.cpp b/src/storage/single_file_block_manager.cpp index b46f4f9273c9..98775c1c745e 100644 --- a/src/storage/single_file_block_manager.cpp +++ b/src/storage/single_file_block_manager.cpp @@ -195,8 +195,8 @@ void SingleFileBlockManager::CreateNewDatabase() { DatabaseHeader h1; // header 1 h1.iteration = 0; - h1.meta_block = INVALID_BLOCK; - h1.free_list = INVALID_BLOCK; + h1.meta_block = idx_t(INVALID_BLOCK); + h1.free_list = idx_t(INVALID_BLOCK); h1.block_count = 0; h1.block_size = Storage::BLOCK_ALLOC_SIZE; h1.vector_size = STANDARD_VECTOR_SIZE; @@ -205,8 +205,8 @@ void SingleFileBlockManager::CreateNewDatabase() { // header 2 DatabaseHeader h2; h2.iteration = 0; - h2.meta_block = INVALID_BLOCK; - h2.free_list = INVALID_BLOCK; + h2.meta_block = idx_t(INVALID_BLOCK); + h2.free_list = idx_t(INVALID_BLOCK); h2.block_count = 0; h2.block_size = Storage::BLOCK_ALLOC_SIZE; h2.vector_size = STANDARD_VECTOR_SIZE; @@ -283,7 +283,7 @@ void SingleFileBlockManager::Initialize(DatabaseHeader &header) { free_list_id = header.free_list; meta_block = header.meta_block; iteration_count = header.iteration; - max_block = header.block_count; + max_block = NumericCast(header.block_count); } void SingleFileBlockManager::LoadFreeList() { @@ -386,7 +386,7 @@ idx_t SingleFileBlockManager::GetMetaBlock() { idx_t SingleFileBlockManager::TotalBlocks() { lock_guard lock(block_lock); - return max_block; + return NumericCast(max_block); } idx_t SingleFileBlockManager::FreeBlocks() { @@ -413,12 +413,12 @@ unique_ptr SingleFileBlockManager::CreateBlock(block_id_t block_id, FileB void SingleFileBlockManager::Read(Block &block) { D_ASSERT(block.id >= 0); D_ASSERT(std::find(free_list.begin(), free_list.end(), block.id) == free_list.end()); - ReadAndChecksum(block, BLOCK_START + block.id * Storage::BLOCK_ALLOC_SIZE); + ReadAndChecksum(block, BLOCK_START + NumericCast(block.id) * Storage::BLOCK_ALLOC_SIZE); } void SingleFileBlockManager::Write(FileBuffer &buffer, block_id_t block_id) { D_ASSERT(block_id >= 0); - ChecksumAndWrite(buffer, BLOCK_START + block_id * Storage::BLOCK_ALLOC_SIZE); + ChecksumAndWrite(buffer, BLOCK_START + NumericCast(block_id) * Storage::BLOCK_ALLOC_SIZE); } void SingleFileBlockManager::Truncate() { @@ -440,7 +440,7 @@ void SingleFileBlockManager::Truncate() { // truncate the file free_list.erase(free_list.lower_bound(max_block), free_list.end()); newly_freed_list.erase(newly_freed_list.lower_bound(max_block), newly_freed_list.end()); - handle->Truncate(BLOCK_START + max_block * Storage::BLOCK_ALLOC_SIZE); + handle->Truncate(NumericCast(BLOCK_START + NumericCast(max_block) * Storage::BLOCK_ALLOC_SIZE)); } vector SingleFileBlockManager::GetFreeListBlocks() { @@ -529,7 +529,7 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) { header.free_list = DConstants::INVALID_INDEX; } metadata_manager.Flush(); - header.block_count = max_block; + header.block_count = NumericCast(max_block); auto &config = DBConfig::Get(db); if (config.options.checkpoint_abort == CheckpointAbort::DEBUG_ABORT_AFTER_FREE_LIST_WRITE) { @@ -569,8 +569,8 @@ void SingleFileBlockManager::TrimFreeBlocks() { // We are now one too far. --itr; // Trim the range. - handle->Trim(BLOCK_START + (first * Storage::BLOCK_ALLOC_SIZE), - (last + 1 - first) * Storage::BLOCK_ALLOC_SIZE); + handle->Trim(BLOCK_START + (NumericCast(first) * Storage::BLOCK_ALLOC_SIZE), + NumericCast(last + 1 - first) * Storage::BLOCK_ALLOC_SIZE); } } newly_freed_list.clear(); diff --git a/src/storage/standard_buffer_manager.cpp b/src/storage/standard_buffer_manager.cpp index ab5ad560ba12..0b62bd8db48f 100644 --- a/src/storage/standard_buffer_manager.cpp +++ b/src/storage/standard_buffer_manager.cpp @@ -126,7 +126,7 @@ void StandardBufferManager::ReAllocate(shared_ptr &handle, idx_t bl D_ASSERT(handle->memory_usage == handle->memory_charge.size); auto req = handle->buffer->CalculateMemory(block_size); - int64_t memory_delta = NumericCast(req.alloc_size) - handle->memory_usage; + int64_t memory_delta = NumericCast(req.alloc_size) - NumericCast(handle->memory_usage); if (memory_delta == 0) { return; @@ -134,10 +134,10 @@ void StandardBufferManager::ReAllocate(shared_ptr &handle, idx_t bl // evict blocks until we have space to resize this block // unlock the handle lock during the call to EvictBlocksOrThrow lock.unlock(); - auto reservation = - EvictBlocksOrThrow(handle->tag, memory_delta, nullptr, "failed to resize block from %s to %s%s", - StringUtil::BytesToHumanReadableString(handle->memory_usage), - StringUtil::BytesToHumanReadableString(req.alloc_size)); + auto reservation = EvictBlocksOrThrow(handle->tag, NumericCast(memory_delta), nullptr, + "failed to resize block from %s to %s%s", + StringUtil::BytesToHumanReadableString(handle->memory_usage), + StringUtil::BytesToHumanReadableString(req.alloc_size)); lock.lock(); // EvictBlocks decrements 'current_memory' for us. @@ -183,10 +183,10 @@ BufferHandle StandardBufferManager::Pin(shared_ptr &handle) { auto buf = handle->Load(handle, std::move(reusable_buffer)); handle->memory_charge = std::move(reservation); // In the case of a variable sized block, the buffer may be smaller than a full block. - int64_t delta = handle->buffer->AllocSize() - handle->memory_usage; + int64_t delta = NumericCast(handle->buffer->AllocSize()) - NumericCast(handle->memory_usage); if (delta) { D_ASSERT(delta < 0); - handle->memory_usage += delta; + handle->memory_usage += NumericCast(delta); handle->memory_charge.Resize(handle->memory_usage); } D_ASSERT(handle->memory_usage == handle->buffer->AllocSize()); @@ -367,7 +367,7 @@ vector StandardBufferManager::GetTemporaryFiles() { TemporaryFileInformation info; info.path = name; auto handle = fs.OpenFile(name, FileFlags::FILE_FLAGS_READ); - info.size = fs.GetFileSize(*handle); + info.size = NumericCast(fs.GetFileSize(*handle)); handle.reset(); result.push_back(info); }); diff --git a/src/storage/storage_manager.cpp b/src/storage/storage_manager.cpp index 1a798e3c9a06..dcc5b150d611 100644 --- a/src/storage/storage_manager.cpp +++ b/src/storage/storage_manager.cpp @@ -200,7 +200,7 @@ class SingleFileStorageCommitState : public StorageCommitState { wal.skip_writing = false; if (wal.GetTotalWritten() > initial_written) { // remove any entries written into the WAL by truncating it - wal.Truncate(initial_wal_size); + wal.Truncate(NumericCast(initial_wal_size)); } } } @@ -284,7 +284,7 @@ DatabaseSize SingleFileStorageManager::GetDatabaseSize() { ds.used_blocks = ds.total_blocks - ds.free_blocks; ds.bytes = (ds.total_blocks * ds.block_size); if (auto wal = GetWriteAheadLog()) { - ds.wal_size = wal->GetWALSize(); + ds.wal_size = NumericCast(wal->GetWALSize()); } } return ds; @@ -302,7 +302,7 @@ bool SingleFileStorageManager::AutomaticCheckpoint(idx_t estimated_wal_bytes) { } auto &config = DBConfig::Get(db); - auto initial_size = log->GetWALSize(); + auto initial_size = NumericCast(log->GetWALSize()); idx_t expected_wal_size = initial_size + estimated_wal_bytes; return expected_wal_size > config.options.checkpoint_wal_size; } diff --git a/src/storage/temporary_file_manager.cpp b/src/storage/temporary_file_manager.cpp index c374829037ec..632f7075f755 100644 --- a/src/storage/temporary_file_manager.cpp +++ b/src/storage/temporary_file_manager.cpp @@ -106,7 +106,7 @@ void TemporaryFileHandle::EraseBlockIndex(block_id_t block_index) { // remove the block (and potentially truncate the temp file) TemporaryFileLock lock(file_lock); D_ASSERT(handle); - RemoveTempBlockIndex(lock, block_index); + RemoveTempBlockIndex(lock, NumericCast(block_index)); } bool TemporaryFileHandle::DeleteIfEmpty() { @@ -147,7 +147,7 @@ void TemporaryFileHandle::RemoveTempBlockIndex(TemporaryFileLock &, idx_t index) #ifndef WIN32 // this ended up causing issues when sorting auto max_index = index_manager.GetMaxIndex(); auto &fs = FileSystem::GetFileSystem(db); - fs.Truncate(*handle, GetPositionInFile(max_index + 1)); + fs.Truncate(*handle, NumericCast(GetPositionInFile(max_index + 1))); #endif } } @@ -310,7 +310,7 @@ void TemporaryFileManager::EraseUsedBlock(TemporaryManagerLock &lock, block_id_t throw InternalException("EraseUsedBlock - Block %llu not found in used blocks", id); } used_blocks.erase(entry); - handle->EraseBlockIndex(index.block_index); + handle->EraseBlockIndex(NumericCast(index.block_index)); if (handle->DeleteIfEmpty()) { EraseFileHandle(lock, index.file_index); } diff --git a/src/storage/temporary_memory_manager.cpp b/src/storage/temporary_memory_manager.cpp index a6d07b9bf635..ba046d30fb17 100644 --- a/src/storage/temporary_memory_manager.cpp +++ b/src/storage/temporary_memory_manager.cpp @@ -47,7 +47,7 @@ void TemporaryMemoryManager::UpdateConfiguration(ClientContext &context) { memory_limit = MAXIMUM_MEMORY_LIMIT_RATIO * double(buffer_manager.GetMaxMemory()); has_temporary_directory = buffer_manager.HasTemporaryDirectory(); - num_threads = task_scheduler.NumberOfThreads(); + num_threads = NumericCast(task_scheduler.NumberOfThreads()); query_max_memory = buffer_manager.GetQueryMaxMemory(); } From d8575d835221e2375b5f7f5bcaf2d9d188b0d9d7 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 4 Apr 2024 16:33:46 +0200 Subject: [PATCH 169/603] More tests on maxlinesize and some fixes --- .../multiple_errors/cast_and_maxline.csv | 4 + .../multiple_errors/less_col_and_max_line.csv | 4 + .../multiple_errors/more_col_and_max_line.csv | 4 + .../scanner/string_value_scanner.cpp | 22 +- .../table_function/global_csv_state.cpp | 7 +- .../csv_scanner/string_value_scanner.hpp | 1 + .../csv/rejects/csv_rejects_maximum_line.test | 18 +- test/sql/copy/csv/rejects/test_mixed.test | 2 +- .../test_multiple_errors_same_line.test | 226 +++++++++++------- 9 files changed, 181 insertions(+), 107 deletions(-) create mode 100644 data/csv/rejects/multiple_errors/cast_and_maxline.csv create mode 100644 data/csv/rejects/multiple_errors/less_col_and_max_line.csv create mode 100644 data/csv/rejects/multiple_errors/more_col_and_max_line.csv diff --git a/data/csv/rejects/multiple_errors/cast_and_maxline.csv b/data/csv/rejects/multiple_errors/cast_and_maxline.csv new file mode 100644 index 000000000000..e4e871e59462 --- /dev/null +++ b/data/csv/rejects/multiple_errors/cast_and_maxline.csv @@ -0,0 +1,4 @@ +name,age,current_day, barks +oogie boogie,3, 2023-01-01, 2 +oogie boogie,3, 2023-01-02, 5 +oogie boogieoogie boogieoogie boogieoogie boogie,bla, 2023-01-03, 4 diff --git a/data/csv/rejects/multiple_errors/less_col_and_max_line.csv b/data/csv/rejects/multiple_errors/less_col_and_max_line.csv new file mode 100644 index 000000000000..bb3dbe4dfc74 --- /dev/null +++ b/data/csv/rejects/multiple_errors/less_col_and_max_line.csv @@ -0,0 +1,4 @@ +name,age,current_day, barks +oogie boogie,3, 2023-01-01, 2 +oogie boogie,3, 2023-01-02, 5 +oogie boogieoogie boogieoogie boogieoogie boogie,bla, 2023-01-03 diff --git a/data/csv/rejects/multiple_errors/more_col_and_max_line.csv b/data/csv/rejects/multiple_errors/more_col_and_max_line.csv new file mode 100644 index 000000000000..27366cd56e5c --- /dev/null +++ b/data/csv/rejects/multiple_errors/more_col_and_max_line.csv @@ -0,0 +1,4 @@ +name,age,current_day, barks +oogie boogie,3, 2023-01-01, 2 +oogie boogie,3, 2023-01-02, 5 +oogie boogieoogie boogieoogie boogieoogie boogie,bla, 2023-01-03,4, bla diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 0a0b286b4671..acade767b83e 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -424,7 +424,11 @@ bool StringValueResult::HandleError() { current_line_position.begin.GetGlobalPosition(requested_size, first_nl), line_pos.GetGlobalPosition(requested_size), parse_types[cur_error.col_idx].first); } - + break; + case CSVErrorType::MAXIMUM_LINE_SIZE: + csv_error = CSVError::LineSizeError( + state_machine.options, cur_error.current_line_size, lines_per_batch, borked_line, + current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); break; default: throw InvalidInputException("CSV Error not allowed when inserting row"); @@ -497,16 +501,8 @@ bool StringValueResult::AddRowInternal() { current_line_position.begin = current_line_position.end; current_line_position.end = current_line_start; if (current_line_size > state_machine.options.maximum_line_size) { - bool first_nl; - auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); - LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); - auto csv_error = - CSVError::LineSizeError(state_machine.options, current_line_size, lines_per_batch, borked_line, - current_line_position.begin.GetGlobalPosition(requested_size, first_nl)); - error_handler.Error(csv_error); - if (number_of_rows > 0) { - number_of_rows--; - } + current_errors.push_back({CSVErrorType::MAXIMUM_LINE_SIZE, 1, last_position}); + current_errors.back().current_line_size = current_line_size; } if (!current_errors.empty()) { // We need to add a few columns error @@ -562,9 +558,7 @@ bool StringValueResult::AddRowInternal() { error_handler.Error(csv_error); } // If we are here we ignore_errors, so we delete this line - if (number_of_rows > 0) { - number_of_rows--; - } + number_of_rows--; } } line_positions_per_row[number_of_rows] = current_line_position; diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index 6646f4ab98b8..b156f0af8b95 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -282,10 +282,15 @@ void CSVGlobalState::FillRejectsTable() { errors_appender.Append(error.byte_position + 1); } // 6. Column Index - errors_appender.Append(col_idx + 1); + if (error.type == CSVErrorType::MAXIMUM_LINE_SIZE) { + errors_appender.Append(Value()); + } else { + errors_appender.Append(col_idx + 1); + } // 7. Column Name (If Applicable) switch (error.type) { case CSVErrorType::TOO_MANY_COLUMNS: + case CSVErrorType::MAXIMUM_LINE_SIZE: errors_appender.Append(Value()); break; case CSVErrorType::TOO_FEW_COLUMNS: diff --git a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp index 0039f9ade5b0..7f4e6d8f017f 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp @@ -74,6 +74,7 @@ class CurrentError { CSVErrorType type; idx_t col_idx; + idx_t current_line_size; string error_message; //! Exact Position where the error happened LinePosition error_position; diff --git a/test/sql/copy/csv/rejects/csv_rejects_maximum_line.test b/test/sql/copy/csv/rejects/csv_rejects_maximum_line.test index 1095a90d70f8..99cedc820614 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_maximum_line.test +++ b/test/sql/copy/csv/rejects/csv_rejects_maximum_line.test @@ -16,7 +16,7 @@ SELECT * FROM read_csv( query IIIIIIIIII FROM reject_errors order by all; ---- -3 0 5 23 23 1 a LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 Maximum line size of 10 bytes exceeded. Actual Size:19 bytes. +3 0 5 23 23 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 Maximum line size of 10 bytes exceeded. Actual Size:19 bytes. statement ok DROP TABLE reject_errors; @@ -37,7 +37,7 @@ SELECT * FROM read_csv( query IIIIIIIII SELECT * EXCLUDE (scan_id) FROM reject_errors order by all; ---- -0 5 23 23 1 a LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 Maximum line size of 10 bytes exceeded. Actual Size:19 bytes. +0 5 23 23 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 Maximum line size of 10 bytes exceeded. Actual Size:19 bytes. statement ok DROP TABLE reject_errors; @@ -57,9 +57,9 @@ SELECT * FROM read_csv( query IIIIIIIIII FROM reject_errors order by all; ---- -27 0 2282 13685 13685 1 a LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 Maximum line size of 20 bytes exceeded. Actual Size:25 bytes. -27 0 2591 15558 15558 1 a LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 Maximum line size of 20 bytes exceeded. Actual Size:25 bytes. -27 0 2923 17569 17569 1 a LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,3 Maximum line size of 20 bytes exceeded. Actual Size:25 bytes. +27 0 2282 13685 13685 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 Maximum line size of 20 bytes exceeded. Actual Size:25 bytes. +27 0 2591 15558 15558 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 Maximum line size of 20 bytes exceeded. Actual Size:25 bytes. +27 0 2923 17569 17569 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,3 Maximum line size of 20 bytes exceeded. Actual Size:25 bytes. statement ok DROP TABLE reject_errors; @@ -77,10 +77,10 @@ SELECT * FROM read_csv( query IIIIIIIIII FROM reject_errors order by all; ---- -31 0 5 23 23 1 a LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 Maximum line size of 10 bytes exceeded. Actual Size:19 bytes. -31 1 2282 13685 13685 1 a LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 Maximum line size of 10 bytes exceeded. Actual Size:25 bytes. -31 1 2591 15558 15558 1 a LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 Maximum line size of 10 bytes exceeded. Actual Size:25 bytes. -31 1 2923 17569 17569 1 a LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,3 Maximum line size of 10 bytes exceeded. Actual Size:25 bytes. +31 0 5 23 23 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 Maximum line size of 10 bytes exceeded. Actual Size:19 bytes. +31 1 2282 13685 13685 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 Maximum line size of 10 bytes exceeded. Actual Size:25 bytes. +31 1 2591 15558 15558 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 Maximum line size of 10 bytes exceeded. Actual Size:25 bytes. +31 1 2923 17569 17569 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,3 Maximum line size of 10 bytes exceeded. Actual Size:25 bytes. statement ok DROP TABLE reject_errors; diff --git a/test/sql/copy/csv/rejects/test_mixed.test b/test/sql/copy/csv/rejects/test_mixed.test index e5ced0ea3ae8..d1f9b1decedc 100644 --- a/test/sql/copy/csv/rejects/test_mixed.test +++ b/test/sql/copy/csv/rejects/test_mixed.test @@ -62,5 +62,5 @@ FROM reject_errors ORDER BY ALL; 3 0 14 143 154 4 NULL TOO MANY COLUMNS 1,2,"pedro",5 Expected Number of Columns: 3 Found: 4 3 0 19 205 207 2 b CAST 1,bla,"pedro" Error when converting column "b". Could not convert string "bla" to 'INTEGER' 3 0 22 243 247 3 c UNQUOTED VALUE 1,2,"pedro"bla Value with unterminated quote found. -3 0 32 366 366 1 a LINE SIZE OVER MAXIMUM 1,2,"pedro thiago timbo holanda" Maximum line size of 20 bytes exceeded. Actual Size:33 bytes. +3 0 32 366 366 NULL NULL LINE SIZE OVER MAXIMUM 1,2,"pedro thiago timbo holanda" Maximum line size of 20 bytes exceeded. Actual Size:33 bytes. 3 0 38 459 463 3 c INVALID UNICODE 1,2,"pedro??" Invalid unicode (byte sequence mismatch) detected. \ No newline at end of file diff --git a/test/sql/copy/csv/rejects/test_multiple_errors_same_line.test b/test/sql/copy/csv/rejects/test_multiple_errors_same_line.test index 22ed108fee20..9c2243435322 100644 --- a/test/sql/copy/csv/rejects/test_multiple_errors_same_line.test +++ b/test/sql/copy/csv/rejects/test_multiple_errors_same_line.test @@ -7,88 +7,88 @@ require skip_reload # Test will fail on windows because byte_position is slightly different due to \r\n instead of \n require notwindows -#query IIII -#FROM read_csv('data/csv/rejects/multiple_errors/cast_and_more_col.csv', -# columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, -# store_rejects = true, auto_detect=false, header = 1); -#---- -#oogie boogie 3 2023-01-01 2 -#oogie boogie 3 2023-01-02 5 -# -#query IIIIIIIII rowsort -#SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; -#---- -#0 4 89 116 4 barks CAST oogie boogie,3, 2023-01-03, bla, 7 Error when converting column "barks". Could not convert string " bla" to 'INTEGER' -#0 4 89 120 5 NULL TOO MANY COLUMNS oogie boogie,3, 2023-01-03, bla, 7 Expected Number of Columns: 4 Found: 5 -#0 5 124 151 4 barks CAST oogie boogie,3, 2023-01-03, bla, 7, 8 Error when converting column "barks". Could not convert string " bla" to 'INTEGER' -#0 5 124 155 5 NULL TOO MANY COLUMNS oogie boogie,3, 2023-01-03, bla, 7, 8 Expected Number of Columns: 4 Found: 5 -#0 5 124 158 6 NULL TOO MANY COLUMNS oogie boogie,3, 2023-01-03, bla, 7, 8 Expected Number of Columns: 4 Found: 6 -# -#statement ok -#DROP TABLE reject_errors; -# -#statement ok -#DROP TABLE reject_scans; -# -#query IIII -#FROM read_csv('data/csv/rejects/multiple_errors/multiple_cast_implicit.csv', -# columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, -# store_rejects = true, auto_detect=false, header = 1); -#---- -#oogie boogie 3 2023-01-01 2 -#oogie boogie 3 2023-01-02 5 -# -#query IIIIIIIII rowsort -#SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; -#---- -#0 4 89 102 2 age CAST oogie boogie,bla_2, 2023-01-02, bla_1 Error when converting column "age". Could not convert string "bla_2" to 'INTEGER' -#0 4 89 120 4 barks CAST oogie boogie,bla_2, 2023-01-02, bla_1 Error when converting column "barks". Could not convert string " bla_1" to 'INTEGER' -# -#statement ok -#DROP TABLE reject_errors; -# -#statement ok -#DROP TABLE reject_scans; -# -#query IIII -#FROM read_csv('data/csv/rejects/multiple_errors/multiple_casts_flush.csv', -# columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'tomorrow': 'DATE'}, -# store_rejects = true, auto_detect=false, header = 1); -#---- -#oogie boogie 3 2023-01-01 2023-01-02 -#oogie boogie 3 2023-01-02 2023-01-03 -# -#query IIIIIIIII rowsort -#SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; -#---- -#0 4 110 NULL 3 current_day CAST oogie boogie,3, bla_2, bla_1 Error when converting column "current_day". date field value out of range: " bla_2", expected format is (YYYY-MM-DD) -#0 4 110 NULL 4 tomorrow CAST oogie boogie,3, bla_2, bla_1 Error when converting column "tomorrow". date field value out of range: " bla_1", expected format is (YYYY-MM-DD) -# -#statement ok -#DROP TABLE reject_errors; -# -#statement ok -#DROP TABLE reject_scans; -# -#query IIII -#FROM read_csv('data/csv/rejects/multiple_errors/multiple_casts_mixed.csv', -# columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, -# store_rejects = true, auto_detect=false, header = 1); -#---- -#oogie boogie 3 2023-01-01 2 -#oogie boogie 3 2023-01-02 5 -# -## FIXME: This will not present the both cast errors :'(, should be alleviated the more types we add to implicit casting -#query IIIIIIIII rowsort -#SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; -#---- -#0 4 89 111 4 barks CAST oogie boogie,3, bla_2, bla_1 Error when converting column "barks". Could not convert string " bla_1" to 'INTEGER' -# -#statement ok -#DROP TABLE reject_errors; -# -#statement ok -#DROP TABLE reject_scans; +query IIII +FROM read_csv('data/csv/rejects/multiple_errors/cast_and_more_col.csv', + columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, + store_rejects = true, auto_detect=false, header = 1); +---- +oogie boogie 3 2023-01-01 2 +oogie boogie 3 2023-01-02 5 + +query IIIIIIIII rowsort +SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; +---- +0 4 89 116 4 barks CAST oogie boogie,3, 2023-01-03, bla, 7 Error when converting column "barks". Could not convert string " bla" to 'INTEGER' +0 4 89 120 5 NULL TOO MANY COLUMNS oogie boogie,3, 2023-01-03, bla, 7 Expected Number of Columns: 4 Found: 5 +0 5 124 151 4 barks CAST oogie boogie,3, 2023-01-03, bla, 7, 8 Error when converting column "barks". Could not convert string " bla" to 'INTEGER' +0 5 124 155 5 NULL TOO MANY COLUMNS oogie boogie,3, 2023-01-03, bla, 7, 8 Expected Number of Columns: 4 Found: 5 +0 5 124 158 6 NULL TOO MANY COLUMNS oogie boogie,3, 2023-01-03, bla, 7, 8 Expected Number of Columns: 4 Found: 6 + +statement ok +DROP TABLE reject_errors; + +statement ok +DROP TABLE reject_scans; + +query IIII +FROM read_csv('data/csv/rejects/multiple_errors/multiple_cast_implicit.csv', + columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, + store_rejects = true, auto_detect=false, header = 1); +---- +oogie boogie 3 2023-01-01 2 +oogie boogie 3 2023-01-02 5 + +query IIIIIIIII rowsort +SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; +---- +0 4 89 102 2 age CAST oogie boogie,bla_2, 2023-01-02, bla_1 Error when converting column "age". Could not convert string "bla_2" to 'INTEGER' +0 4 89 120 4 barks CAST oogie boogie,bla_2, 2023-01-02, bla_1 Error when converting column "barks". Could not convert string " bla_1" to 'INTEGER' + +statement ok +DROP TABLE reject_errors; + +statement ok +DROP TABLE reject_scans; + +query IIII +FROM read_csv('data/csv/rejects/multiple_errors/multiple_casts_flush.csv', + columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'tomorrow': 'DATE'}, + store_rejects = true, auto_detect=false, header = 1); +---- +oogie boogie 3 2023-01-01 2023-01-02 +oogie boogie 3 2023-01-02 2023-01-03 + +query IIIIIIIII rowsort +SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; +---- +0 4 110 NULL 3 current_day CAST oogie boogie,3, bla_2, bla_1 Error when converting column "current_day". date field value out of range: " bla_2", expected format is (YYYY-MM-DD) +0 4 110 NULL 4 tomorrow CAST oogie boogie,3, bla_2, bla_1 Error when converting column "tomorrow". date field value out of range: " bla_1", expected format is (YYYY-MM-DD) + +statement ok +DROP TABLE reject_errors; + +statement ok +DROP TABLE reject_scans; + +query IIII +FROM read_csv('data/csv/rejects/multiple_errors/multiple_casts_mixed.csv', + columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, + store_rejects = true, auto_detect=false, header = 1); +---- +oogie boogie 3 2023-01-01 2 +oogie boogie 3 2023-01-02 5 + +# FIXME: This will not present the both cast errors :'(, should be alleviated the more types we add to implicit casting +query IIIIIIIII rowsort +SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; +---- +0 4 89 111 4 barks CAST oogie boogie,3, bla_2, bla_1 Error when converting column "barks". Could not convert string " bla_1" to 'INTEGER' + +statement ok +DROP TABLE reject_errors; + +statement ok +DROP TABLE reject_scans; query IIII FROM read_csv('data/csv/rejects/multiple_errors/cast_and_less_col.csv', @@ -113,6 +113,68 @@ DROP TABLE reject_errors; statement ok DROP TABLE reject_scans; +query IIII +FROM read_csv('data/csv/rejects/multiple_errors/cast_and_maxline.csv', + columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, + store_rejects = true, auto_detect=false, header = 1, max_line_size=40); +---- +oogie boogie 3 2023-01-01 2 +oogie boogie 3 2023-01-02 5 + +query IIIIIIIII rowsort +SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; +---- +0 4 89 138 2 age CAST oogie boogieoogie boogieoogie boogieoogie boogie,bla, 2023-01-03, 4 Error when converting column "age". Could not convert string "bla" to 'INTEGER' +0 4 89 89 NULL NULL LINE SIZE OVER MAXIMUM oogie boogieoogie boogieoogie boogieoogie boogie,bla, 2023-01-03, 4 Maximum line size of 40 bytes exceeded. Actual Size:68 bytes. + +statement ok +DROP TABLE reject_errors; + +statement ok +DROP TABLE reject_scans; + +query IIII +FROM read_csv('data/csv/rejects/multiple_errors/less_col_and_max_line.csv', + columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, + store_rejects = true, auto_detect=false, header = 1, max_line_size=40); +---- +oogie boogie 3 2023-01-01 2 +oogie boogie 3 2023-01-02 5 + +query IIIIIIIII +SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY byte_position; +---- +0 4 89 89 NULL NULL LINE SIZE OVER MAXIMUM oogie boogieoogie boogieoogie boogieoogie boogie,bla, 2023-01-03 Maximum line size of 40 bytes exceeded. Actual Size:65 bytes. +0 4 89 138 2 age CAST oogie boogieoogie boogieoogie boogieoogie boogie,bla, 2023-01-03 Error when converting column "age". Could not convert string "bla" to 'INTEGER' +0 4 89 153 3 barks MISSING COLUMNS oogie boogieoogie boogieoogie boogieoogie boogie,bla, 2023-01-03 Expected Number of Columns: 4 Found: 3 + +statement ok +DROP TABLE reject_errors; + +statement ok +DROP TABLE reject_scans; + +query IIII +FROM read_csv('data/csv/rejects/multiple_errors/more_col_and_max_line.csv', + columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, + store_rejects = true, auto_detect=false, header = 1, max_line_size=40); +---- +oogie boogie 3 2023-01-01 2 +oogie boogie 3 2023-01-02 5 + +query IIIIIIIII rowsort +SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY byte_position; +---- +0 4 89 138 2 age CAST oogie boogieoogie boogieoogie boogieoogie boogie,bla, 2023-01-03,4, bla Error when converting column "age". Could not convert string "bla" to 'INTEGER' +0 4 89 155 5 NULL TOO MANY COLUMNS oogie boogieoogie boogieoogie boogieoogie boogie,bla, 2023-01-03,4, bla Expected Number of Columns: 4 Found: 5 +0 4 89 89 NULL NULL LINE SIZE OVER MAXIMUM oogie boogieoogie boogieoogie boogieoogie boogie,bla, 2023-01-03,4, bla Maximum line size of 40 bytes exceeded. Actual Size:72 bytes. + +statement ok +DROP TABLE reject_errors; + +statement ok +DROP TABLE reject_scans; + #query IIII #FROM read_csv('data/csv/rejects/multiple_errors.csv', # columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, From 0cfb19acc9e040fa3293b9ad63852ffcbeeb2059 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 4 Apr 2024 16:46:46 +0200 Subject: [PATCH 170/603] Adding unquoted mix tests --- .../rejects/multiple_errors/unquoted_cast.csv | 5 ++ .../rejects/multiple_errors/unquoted_less.csv | 5 ++ .../multiple_errors/unquoted_maxline.csv | 5 ++ .../rejects/multiple_errors/unquoted_more.csv | 5 ++ .../test_multiple_errors_same_line.test | 82 +++++++++++++++++++ 5 files changed, 102 insertions(+) create mode 100644 data/csv/rejects/multiple_errors/unquoted_cast.csv create mode 100644 data/csv/rejects/multiple_errors/unquoted_less.csv create mode 100644 data/csv/rejects/multiple_errors/unquoted_maxline.csv create mode 100644 data/csv/rejects/multiple_errors/unquoted_more.csv diff --git a/data/csv/rejects/multiple_errors/unquoted_cast.csv b/data/csv/rejects/multiple_errors/unquoted_cast.csv new file mode 100644 index 000000000000..9cb8bf160c8c --- /dev/null +++ b/data/csv/rejects/multiple_errors/unquoted_cast.csv @@ -0,0 +1,5 @@ +name,age,current_day, barks +oogie boogie,3, 2023-01-01, 2 +"oogie boogie"bla,bla, 2023-01-02, 5 +oogie boogie,3, 2023-01-02, 7 + diff --git a/data/csv/rejects/multiple_errors/unquoted_less.csv b/data/csv/rejects/multiple_errors/unquoted_less.csv new file mode 100644 index 000000000000..5cd602581222 --- /dev/null +++ b/data/csv/rejects/multiple_errors/unquoted_less.csv @@ -0,0 +1,5 @@ +name,age,current_day, barks +oogie boogie,3, 2023-01-01, 2 +"oogie boogie"bla,4, 2023-01-02 +oogie boogie,3, 2023-01-02, 7 + diff --git a/data/csv/rejects/multiple_errors/unquoted_maxline.csv b/data/csv/rejects/multiple_errors/unquoted_maxline.csv new file mode 100644 index 000000000000..1dc1f8f2d505 --- /dev/null +++ b/data/csv/rejects/multiple_errors/unquoted_maxline.csv @@ -0,0 +1,5 @@ +name,age,current_day, barks +oogie boogie,3, 2023-01-01, 2 +"oogie boogieoogie boogieoogie boogieoogie boogie"bla,4, 2023-01-02, 5 +oogie boogie,3, 2023-01-02, 7 + diff --git a/data/csv/rejects/multiple_errors/unquoted_more.csv b/data/csv/rejects/multiple_errors/unquoted_more.csv new file mode 100644 index 000000000000..051e8cc90b86 --- /dev/null +++ b/data/csv/rejects/multiple_errors/unquoted_more.csv @@ -0,0 +1,5 @@ +name,age,current_day, barks +oogie boogie,3, 2023-01-01, 2 +"oogie boogie"bla,4, 2023-01-02, 5, 8 +oogie boogie,3, 2023-01-02, 7 + diff --git a/test/sql/copy/csv/rejects/test_multiple_errors_same_line.test b/test/sql/copy/csv/rejects/test_multiple_errors_same_line.test index 9c2243435322..112e7e0e2575 100644 --- a/test/sql/copy/csv/rejects/test_multiple_errors_same_line.test +++ b/test/sql/copy/csv/rejects/test_multiple_errors_same_line.test @@ -175,6 +175,88 @@ DROP TABLE reject_errors; statement ok DROP TABLE reject_scans; +query IIII +FROM read_csv('data/csv/rejects/multiple_errors/unquoted_cast.csv', + columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, + store_rejects = true, auto_detect=false, header = 1, max_line_size=40); +---- +oogie boogie 3 2023-01-01 2 +oogie boogie 3 2023-01-02 7 + +query IIIIIIIII rowsort +SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY byte_position; +---- +0 3 59 59 1 name UNQUOTED VALUE "oogie boogie"bla,bla, 2023-01-02, 5 Value with unterminated quote found. +0 3 59 77 2 age CAST "oogie boogie"bla,bla, 2023-01-02, 5 Error when converting column "age". Could not convert string "bla" to 'INTEGER' + + +statement ok +DROP TABLE reject_errors; + +statement ok +DROP TABLE reject_scans; + +query IIII +FROM read_csv('data/csv/rejects/multiple_errors/unquoted_less.csv', + columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, + store_rejects = true, auto_detect=false, header = 1, max_line_size=40); +---- +oogie boogie 3 2023-01-01 2 +oogie boogie 3 2023-01-02 7 + +query IIIIIIIII rowsort +SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY byte_position; +---- +0 3 59 59 1 name UNQUOTED VALUE "oogie boogie"bla,4, 2023-01-02 Value with unterminated quote found. +0 3 59 90 3 barks MISSING COLUMNS "oogie boogie"bla,4, 2023-01-02 Expected Number of Columns: 4 Found: 3 + +statement ok +DROP TABLE reject_errors; + +statement ok +DROP TABLE reject_scans; + +query IIII +FROM read_csv('data/csv/rejects/multiple_errors/unquoted_maxline.csv', + columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, + store_rejects = true, auto_detect=false, header = 1, max_line_size=40); +---- +oogie boogie 3 2023-01-01 2 +oogie boogie 3 2023-01-02 7 + +query IIIIIIIII rowsort +SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY byte_position; +---- +0 3 59 59 1 name UNQUOTED VALUE "oogie boogieoogie boogieoogie boogieoogie boogie"bla,4, 2023-01-02, 5 Value with unterminated quote found. +0 3 59 59 NULL NULL LINE SIZE OVER MAXIMUM "oogie boogieoogie boogieoogie boogieoogie boogie"bla,4, 2023-01-02, 5 Maximum line size of 40 bytes exceeded. Actual Size:71 bytes. + + +statement ok +DROP TABLE reject_errors; + +statement ok +DROP TABLE reject_scans; + +query IIII +FROM read_csv('data/csv/rejects/multiple_errors/unquoted_more.csv', + columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, + store_rejects = true, auto_detect=false, header = 1, max_line_size=40); +---- +oogie boogie 3 2023-01-01 2 +oogie boogie 3 2023-01-02 7 + +query IIIIIIIII rowsort +SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY byte_position; +---- +0 3 59 59 1 name UNQUOTED VALUE "oogie boogie"bla,4, 2023-01-02, 5, 8 Value with unterminated quote found. +0 3 59 93 5 NULL TOO MANY COLUMNS "oogie boogie"bla,4, 2023-01-02, 5, 8 Expected Number of Columns: 4 Found: 5 + +statement ok +DROP TABLE reject_errors; + +statement ok +DROP TABLE reject_scans; + #query IIII #FROM read_csv('data/csv/rejects/multiple_errors.csv', # columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, From a2e1abfafd613f10ffe2f173c52f035af047c27a Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 4 Apr 2024 10:40:59 +0200 Subject: [PATCH 171/603] [Dev] Remove extra newline from duckdb_platform --- CMakeLists.txt | 2 +- scripts/create_local_extension_repo.py | 3 ++- tools/utils/test_platform.cpp | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index aebc060c3bd7..ca0f670140e6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1195,7 +1195,7 @@ else() add_custom_target( duckdb_platform ALL COMMAND - echo "${DUCKDB_EXPLICIT_PLATFORM}" > duckdb_platform_out + printf "${DUCKDB_EXPLICIT_PLATFORM}" > duckdb_platform_out ) endif() diff --git a/scripts/create_local_extension_repo.py b/scripts/create_local_extension_repo.py index f2894ad24abd..8cf71abec002 100644 --- a/scripts/create_local_extension_repo.py +++ b/scripts/create_local_extension_repo.py @@ -27,7 +27,8 @@ dst_path = dst_path.replace("/", "\\") with open(duckdb_platform_out, 'r') as f: - duckdb_platform = [line.split(None, 1)[0] for line in f][0] + lines = f.readlines() + duckdb_platform = lines[0] # Create destination path dest_path = os.path.join(dst_path, duckdb_version, duckdb_platform) diff --git a/tools/utils/test_platform.cpp b/tools/utils/test_platform.cpp index af524785d949..ee00e671c90e 100644 --- a/tools/utils/test_platform.cpp +++ b/tools/utils/test_platform.cpp @@ -2,6 +2,6 @@ #include int main() { - std::cout << duckdb::DuckDBPlatform() << "\n"; + std::cout << duckdb::DuckDBPlatform(); return 0; } From 54c4b7069bfd7737ee97df2d5df854997f19aae9 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 4 Apr 2024 10:43:32 +0200 Subject: [PATCH 172/603] [Dev] Propagate EXTENSION_VERSION to build_loadable_extension_directory --- CMakeLists.txt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ca0f670140e6..669aee087864 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -723,7 +723,7 @@ if (NOT EXTENSION_CONFIG_BUILD AND NOT ${EXTENSION_TESTS_ONLY} AND NOT CLANG_TID endif() endif() -function(build_loadable_extension_directory NAME OUTPUT_DIRECTORY PARAMETERS) +function(build_loadable_extension_directory NAME OUTPUT_DIRECTORY EXTENSION_VERSION PARAMETERS) set(TARGET_NAME ${NAME}_loadable_extension) if (LOCAL_EXTENSION_REPO) add_dependencies(duckdb_local_extension_repo ${NAME}_loadable_extension) @@ -734,6 +734,8 @@ function(build_loadable_extension_directory NAME OUTPUT_DIRECTORY PARAMETERS) list(REMOVE_AT FILES 0) # remove output_directory list(REMOVE_AT FILES 0) + # remove extension_version + list(REMOVE_AT FILES 0) # remove parameters list(REMOVE_AT FILES 0) @@ -827,8 +829,9 @@ function(build_loadable_extension NAME PARAMETERS) set(FILES ${ARGV}) list(REMOVE_AT FILES 0) list(REMOVE_AT FILES 0) + string(TOUPPER ${NAME} EXTENSION_NAME_UPPERCASE) - build_loadable_extension_directory(${NAME} "extension/${NAME}" "${PARAMETERS}" ${FILES}) + build_loadable_extension_directory(${NAME} "extension/${NAME}" "${DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG}" "${PARAMETERS}" ${FILES}) endfunction() function(build_static_extension NAME PARAMETERS) @@ -883,6 +886,7 @@ function(register_extension NAME DONT_LINK DONT_BUILD LOAD_TESTS PATH INCLUDE_PA set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_PATH ${PATH} PARENT_SCOPE) set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_INCLUDE_PATH ${INCLUDE_PATH} PARENT_SCOPE) set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_TEST_PATH ${TEST_PATH} PARENT_SCOPE) + set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG ${GIT_COMMIT_HASH} PARENT_SCOPE) endfunction() # Downloads the external extension repo at the specified commit and calls register_extension @@ -944,6 +948,7 @@ function(duckdb_extension_load NAME) error("Git URL specified but no valid git commit was found for ${NAME} extension") endif() register_external_extension(${NAME} "${duckdb_extension_load_GIT_URL}" "${duckdb_extension_load_GIT_TAG}" "${duckdb_extension_load_DONT_LINK}" "${duckdb_extension_load_DONT_BUILD}" "${duckdb_extension_load_LOAD_TESTS}" "${duckdb_extension_load_SOURCE_DIR}" "${duckdb_extension_load_INCLUDE_DIR}" "${duckdb_extension_load_TEST_DIR}" "${duckdb_extension_load_APPLY_PATCHES}" "${duckdb_extension_load_SUBMODULES}") + set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG "${duckdb_extension_load_GIT_TAG}" PARENT_SCOPE) elseif (NOT "${duckdb_extension_load_SOURCE_DIR}" STREQUAL "") # Local extension, custom path message(STATUS "Load extension '${NAME}' from '${duckdb_extension_load_SOURCE_DIR}'") @@ -971,6 +976,7 @@ function(duckdb_extension_load NAME) # Local extension, default path message(STATUS "Load extension '${NAME}' from '${CMAKE_CURRENT_SOURCE_DIR}/extensions'") register_extension(${NAME} ${duckdb_extension_load_DONT_LINK} "${duckdb_extension_load_DONT_BUILD}" "${duckdb_extension_load_LOAD_TESTS}" "${CMAKE_CURRENT_SOURCE_DIR}/extension/${NAME}" "${CMAKE_CURRENT_SOURCE_DIR}/extension/${NAME}/include" "${CMAKE_CURRENT_SOURCE_DIR}/extension/${NAME}/test/sql") + set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG ${DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG} PARENT_SCOPE) endif() # Propagate variables set by register_extension From 2965774beadcfc9ea11f92f28bd2eee624783a87 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 4 Apr 2024 10:44:06 +0200 Subject: [PATCH 173/603] [Dev] Adapt to commit adding parameter to build_loadable_extension_directory --- test/extension/CMakeLists.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/extension/CMakeLists.txt b/test/extension/CMakeLists.txt index a34dd624132b..ca149cc58c3e 100644 --- a/test/extension/CMakeLists.txt +++ b/test/extension/CMakeLists.txt @@ -1,13 +1,14 @@ set(PARAMETERS "-warnings") -build_loadable_extension_directory(loadable_extension_demo test/extension - ${PARAMETERS} loadable_extension_demo.cpp) +build_loadable_extension_directory( + loadable_extension_demo test/extension "default-version" ${PARAMETERS} + loadable_extension_demo.cpp) if(NOT WIN32 AND NOT SUN) add_definitions(-DDUCKDB_BUILD_DIRECTORY="${PROJECT_BINARY_DIR}") build_loadable_extension_directory( - loadable_extension_optimizer_demo test/extension ${PARAMETERS} - ../extension/loadable_extension_optimizer_demo.cpp) + loadable_extension_optimizer_demo test/extension "default-version" + ${PARAMETERS} ../extension/loadable_extension_optimizer_demo.cpp) set(TEST_EXT_OBJECTS test_remote_optimizer.cpp) From 69d31d59d8a73f582db4a2145c36e56804d6051e Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 4 Apr 2024 10:47:14 +0200 Subject: [PATCH 174/603] [Dev] Add metadata information in the last segment of the extension --- CMakeLists.txt | 14 ++++++++ scripts/append_metadata.sh | 52 ++++++++++++++++++++++++++++++ scripts/extension-upload-single.sh | 23 +------------ 3 files changed, 67 insertions(+), 22 deletions(-) create mode 100755 scripts/append_metadata.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 669aee087864..753129013cad 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -822,6 +822,20 @@ function(build_loadable_extension_directory NAME OUTPUT_DIRECTORY EXTENSION_VERS COMMAND emcc $.lib -o $ -sSIDE_MODULE=1 -O3 ) endif() + add_custom_command( + TARGET ${TARGET_NAME} + POST_BUILD + COMMAND + printf "${DUCKDB_NORMALIZED_VERSION}" > duckdb_version_file + COMMAND + printf "${EXTENSION_VERSION}" > extension_version_file + COMMAND + bash ${CMAKE_SOURCE_DIR}/scripts/append_metadata.sh $ ${CMAKE_BINARY_DIR}/duckdb_platform_out duckdb_version_file extension_version_file + ) + add_dependencies(${TARGET_NAME} duckdb_platform) + if (NOT EXTENSION_CONFIG_BUILD AND NOT ${EXTENSION_TESTS_ONLY} AND NOT CLANG_TIDY) + add_dependencies(duckdb_local_extension_repo ${TARGET_NAME}) + endif() endfunction() function(build_loadable_extension NAME PARAMETERS) diff --git a/scripts/append_metadata.sh b/scripts/append_metadata.sh new file mode 100755 index 000000000000..beb47ccabd98 --- /dev/null +++ b/scripts/append_metadata.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +# Usage: ./script/append_metadata.sh +# Currently hardcoded to host up to 8 fields +# Example: ./scripts/append_metadata.sh file.duckdb_extension git_hash_duckdb_file git_hash_extension_file platfrom_file + +if (($# >= 9)); then + echo "Too many parameters provided, current script can handle at maxium 8 fields" + exit 1 +fi + +# 0 for custom section +# 213 in hex = 531 in decimal, total lenght of what follows (1 + 16 + 2 + 8x32 + 256) +# [1(continuation) + 0010011(payload) = \x93, 0(continuation) + 10(payload) = \x04] +echo -n -e '\x00' >> "$1" +echo -n -e '\x93\x04' >> "$1" +# 10 in hex = 16 in decimal, lenght of name, 1 byte +echo -n -e '\x10' >> "$1" +echo -n -e 'duckdb_signature' >> "$1" +# the name of the WebAssembly custom section, 16 bytes +# 1000 in hex, 512 in decimal +# [1(continuation) + 0000000(payload) = ff, 0(continuation) + 100(payload)], +# for a grand total of 2 bytes +echo -n -e '\x80\x04' >> "$1" + +dd if=/dev/zero of="$1.empty_32" bs=32 count=1 &> /dev/null +dd if=/dev/zero of="$1.empty_256" bs=256 count=1 &> /dev/null + +# Write empty fields +for ((i=$#; i<8; i++)) +do + cat "$1.empty_32" >> "$1" +done + +# Write provided fiedls (backwards) +for ((i=$#; i>=2; i--)) +do + cat "${!i}" > "$1.add" + cat "$1.empty_32" >> "$1.add" + dd if="$1.add" of="$1.add_trunc" bs=32 count=1 &> /dev/null + cat "$1.add_trunc" >> "$1" +done + +# Write how many fields have been written during previous step + echo -n "$#" > "$1.add" + cat "$1.empty_32" >> "$1.add" + dd if="$1.add" of="$1.add_trunc" bs=32 count=1 &> /dev/null + cat "$1.add_trunc" >> "$1" + +cat "$1.empty_256" >> "$1" + +rm -f "$1.*" diff --git a/scripts/extension-upload-single.sh b/scripts/extension-upload-single.sh index a7d3e0db8725..30f5c0c673a7 100755 --- a/scripts/extension-upload-single.sh +++ b/scripts/extension-upload-single.sh @@ -33,21 +33,7 @@ script_dir="$(dirname "$(readlink -f "$0")")" # calculate SHA256 hash of extension binary cat $ext > $ext.append -if [[ $4 == wasm* ]]; then - # 0 for custom section - # 113 in hex = 275 in decimal, total lenght of what follows (1 + 16 + 2 + 256) - # [1(continuation) + 0010011(payload) = \x93, 0(continuation) + 10(payload) = \x02] - echo -n -e '\x00' >> $ext.append - echo -n -e '\x93\x02' >> $ext.append - # 10 in hex = 16 in decimal, lenght of name, 1 byte - echo -n -e '\x10' >> $ext.append - echo -n -e 'duckdb_signature' >> $ext.append - # the name of the WebAssembly custom section, 16 bytes - # 100 in hex, 256 in decimal - # [1(continuation) + 0000000(payload) = ff, 0(continuation) + 10(payload)], - # for a grand total of 2 bytes - echo -n -e '\x80\x02' >> $ext.append -fi +( command -v truncate && truncate -s -256 $ext.append ) || ( command -v gtruncate && gtruncate -s -256 $ext.append ) || exit 1 # (Optionally) Sign binary if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then @@ -78,13 +64,6 @@ if [ -z "$AWS_ACCESS_KEY_ID" ]; then exit 0 fi -if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then - echo "$DUCKDB_EXTENSION_SIGNING_PK" > private.pem - $script_dir/compute-extension-hash.sh $ext.append > $ext.hash - openssl pkeyutl -sign -in $ext.hash -inkey private.pem -pkeyopt digest:sha256 -out $ext.sign - rm -f private.pem -fi - # Set dry run unless guard var is set DRY_RUN_PARAM="--dryrun" if [ "$DUCKDB_DEPLOY_SCRIPT_MODE" == "for_real" ]; then From 694a9c2dbeb558ac9c5f40ddab8be0d8ca023250 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 4 Apr 2024 10:53:06 +0200 Subject: [PATCH 175/603] [Dev] Expand API of SetExtensionLoaded --- src/include/duckdb/main/database.hpp | 2 +- src/main/database.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/include/duckdb/main/database.hpp b/src/include/duckdb/main/database.hpp index 0b87cf8f0ea2..90f4dd3c58f9 100644 --- a/src/include/duckdb/main/database.hpp +++ b/src/include/duckdb/main/database.hpp @@ -46,7 +46,7 @@ class DatabaseInstance : public std::enable_shared_from_this { DUCKDB_API ObjectCache &GetObjectCache(); DUCKDB_API ConnectionManager &GetConnectionManager(); DUCKDB_API ValidChecker &GetValidChecker(); - DUCKDB_API void SetExtensionLoaded(const std::string &extension_name); + DUCKDB_API void SetExtensionLoaded(const std::string &extension_name, const std::string &extension_version = ""); idx_t NumberOfThreads(); diff --git a/src/main/database.cpp b/src/main/database.cpp index 2e3fd0204e3a..56202dd7356b 100644 --- a/src/main/database.cpp +++ b/src/main/database.cpp @@ -401,7 +401,7 @@ bool DuckDB::ExtensionIsLoaded(const std::string &name) { return instance->ExtensionIsLoaded(name); } -void DatabaseInstance::SetExtensionLoaded(const std::string &name) { +void DatabaseInstance::SetExtensionLoaded(const std::string &name, const std::string &extension_version) { auto extension_name = ExtensionHelper::GetExtensionName(name); loaded_extensions.insert(extension_name); From 660b100f51237d41bec1c0be4e366211fa94bf35 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 4 Apr 2024 10:53:50 +0200 Subject: [PATCH 176/603] [Dev] Add field to ExtensionInitResult --- src/include/duckdb/main/extension_helper.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/include/duckdb/main/extension_helper.hpp b/src/include/duckdb/main/extension_helper.hpp index cc29ec41b067..8d8864685044 100644 --- a/src/include/duckdb/main/extension_helper.hpp +++ b/src/include/duckdb/main/extension_helper.hpp @@ -31,6 +31,7 @@ struct ExtensionAlias { struct ExtensionInitResult { string filename; string filebase; + string extension_version; void *lib_hdl; }; From 512066ca899e826c34c6c0fb2bc367d2698fa139 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 4 Apr 2024 10:55:35 +0200 Subject: [PATCH 177/603] [Dev] Use new interface, currently with old (wrong) value from the dynamic function call --- src/main/extension/extension_load.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/extension/extension_load.cpp b/src/main/extension/extension_load.cpp index 7fedb8e8d1f3..9afef96dc47b 100644 --- a/src/main/extension/extension_load.cpp +++ b/src/main/extension/extension_load.cpp @@ -239,6 +239,7 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const str } result.filebase = lowercase_extension_name; + result.extension_version = extension_version; result.filename = filename; result.lib_hdl = lib_hdl; return true; @@ -304,7 +305,7 @@ void ExtensionHelper::LoadExternalExtension(DatabaseInstance &db, FileSystem &fs init_fun_name, res.filename, error.RawMessage()); } - db.SetExtensionLoaded(extension); + db.SetExtensionLoaded(extension, res.extension_version); #endif } From b07ee605652475ee24035de25118eadd81dc1384 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 4 Apr 2024 10:59:09 +0200 Subject: [PATCH 178/603] [Dev] Read metadata metadata fields and use them instead of dynamic call --- src/main/extension/extension_load.cpp | 98 +++++++++++++++++---------- 1 file changed, 64 insertions(+), 34 deletions(-) diff --git a/src/main/extension/extension_load.cpp b/src/main/extension/extension_load.cpp index 9afef96dc47b..1468fbeb3838 100644 --- a/src/main/extension/extension_load.cpp +++ b/src/main/extension/extension_load.cpp @@ -119,15 +119,50 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const str error = StringUtil::Format("Extension \"%s\" not found.\n%s", filename, message); return false; } - if (!config.options.allow_unsigned_extensions) { - auto handle = fs.OpenFile(filename, FileFlags::FILE_FLAGS_READ); - // signature is the last 256 bytes of the file + string metadata_segment; + metadata_segment.resize(512); + + const std::string engine_version = std::string(GetVersionDirectoryName()); + const std::string engine_platform = std::string(DuckDB::Platform()); + + auto handle = fs.OpenFile(filename, FileFlags::FILE_FLAGS_READ); + + idx_t file_size = handle->GetFileSize(); + + if (file_size < 1024) { + throw InvalidInputException("Extension \"%s\", version unknown, do not have metadata compatible with DuckDB " + "version (%s). File size in particular is %i lower than minimum threshold of 1024", + filename, engine_version, file_size); + } + + auto metadata_offset = file_size - metadata_segment.size(); + + handle->Read((void *)metadata_segment.data(), metadata_segment.size(), metadata_offset); - string signature; - signature.resize(256); + std::vector metadata_field; + for (idx_t i = 0; i < 8; i++) { + metadata_field.emplace_back(metadata_segment, i * 32, 32); + } + + std::reverse(metadata_field.begin(), metadata_field.end()); + + for (auto &m : metadata_field) { + auto zero_pos = m.find_first_of((char)0); + if (zero_pos < m.size()) { + m.resize(zero_pos); + } + } + + std::string extension_duckdb_platform = metadata_field[1]; + std::string extension_duckdb_version = metadata_field[2]; + std::string extension_version = metadata_field[3]; + + if (!config.options.allow_unsigned_extensions) { + // signature is the last 256 bytes of the file + string signature(metadata_segment, metadata_segment.size() - 256); - auto signature_offset = handle->GetFileSize() - signature.size(); + auto signature_offset = metadata_offset + metadata_segment.size() - signature.size(); const idx_t maxLenChunks = 1024ULL * 1024ULL; const idx_t numChunks = (signature_offset + maxLenChunks - 1) / maxLenChunks; @@ -177,8 +212,31 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const str } if (!any_valid) { throw IOException(config.error_manager->FormatException(ErrorType::UNSIGNED_EXTENSION, filename)); + if (engine_version != extension_duckdb_version || engine_platform != extension_duckdb_platform) { + // FIXME: Add info on platform || version not matching, hinting probably signature is missing + } } } + + if (engine_version != extension_duckdb_version || engine_platform != extension_duckdb_platform) { + throw InvalidInputException( + "Extension \"%s\" (version %s, platfrom %s) does not match DuckDB loading it (version %s, platform %s)", + filename, extension_duckdb_version, extension_duckdb_platform, engine_version, engine_platform); + } + + char a[32] = {0}; + a[0] = '4'; + + if (strcmp(a, metadata_field[0].data()) != 0) { + throw InvalidInputException("Extension \"%s\", version unknown, do not have metadata compatible with DuckDB " + "version (%s %s). Number of fields was different than 4", + filename, extension_duckdb_version, engine_version); + } + + auto number_metadata_fields = 3; + D_ASSERT(number_metadata_fields == 3); // Currently hardcoded value + metadata_field.resize(number_metadata_fields + 1); + auto filebase = fs.ExtractBaseName(filename); #ifdef WASM_LOADABLE_EXTENSIONS @@ -210,34 +268,6 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const str auto lowercase_extension_name = StringUtil::Lower(filebase); - ext_version_fun_t version_fun; - auto version_fun_name = lowercase_extension_name + "_version"; - - version_fun = LoadFunctionFromDLL(lib_hdl, version_fun_name, filename); - - std::string engine_version = std::string(DuckDB::LibraryVersion()); - - auto version_fun_result = (*version_fun)(); - if (version_fun_result == nullptr) { - throw InvalidInputException("Extension \"%s\" returned a nullptr", filename); - } - std::string extension_version = std::string(version_fun_result); - - // Trim v's if necessary - std::string extension_version_trimmed = extension_version; - std::string engine_version_trimmed = engine_version; - if (!extension_version.empty() && extension_version[0] == 'v') { - extension_version_trimmed = extension_version.substr(1); - } - if (!engine_version.empty() && engine_version[0] == 'v') { - engine_version_trimmed = engine_version.substr(1); - } - - if (extension_version_trimmed != engine_version_trimmed) { - throw InvalidInputException("Extension \"%s\" version (%s) does not match DuckDB version (%s)", filename, - extension_version, engine_version); - } - result.filebase = lowercase_extension_name; result.extension_version = extension_version; result.filename = filename; From 41f753fc90fbea1c65d5ee6dbf76a15345616747 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 4 Apr 2024 11:01:15 +0200 Subject: [PATCH 179/603] Propagate metadata information to SQL, via the duckdb_extension table Note that this do not currently work for extensions that are double Loaded This might happen (for example in tpch) if _init function calls Load explicitly --- src/function/table/system/duckdb_extensions.cpp | 14 ++++++++++++-- src/include/duckdb/main/database.hpp | 12 ++++++++++++ src/main/database.cpp | 5 +++++ test/extension/load_extension.test | 5 +++++ 4 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/function/table/system/duckdb_extensions.cpp b/src/function/table/system/duckdb_extensions.cpp index 75cfc537c6cb..7a2f45108876 100644 --- a/src/function/table/system/duckdb_extensions.cpp +++ b/src/function/table/system/duckdb_extensions.cpp @@ -17,6 +17,7 @@ struct ExtensionInformation { string file_path; string description; vector aliases; + string extension_version; }; struct DuckDBExtensionsData : public GlobalTableFunctionState { @@ -47,6 +48,9 @@ static unique_ptr DuckDBExtensionsBind(ClientContext &context, Tab names.emplace_back("aliases"); return_types.emplace_back(LogicalType::LIST(LogicalType::VARCHAR)); + names.emplace_back("extension_version"); + return_types.emplace_back(LogicalType::VARCHAR); + return nullptr; } @@ -98,16 +102,20 @@ unique_ptr DuckDBExtensionsInit(ClientContext &context }); #endif // now check the list of currently loaded extensions - auto &loaded_extensions = db.LoadedExtensions(); - for (auto &ext_name : loaded_extensions) { + auto &loaded_extensions = db.LoadedExtensionsData(); + for (auto &e : loaded_extensions) { + auto &ext_name = e.first; + auto &ext_info = e.second; auto entry = installed_extensions.find(ext_name); if (entry == installed_extensions.end()) { ExtensionInformation info; info.name = ext_name; info.loaded = true; + info.extension_version = ext_info.extension_version; installed_extensions[ext_name] = std::move(info); } else { entry->second.loaded = true; + entry->second.extension_version = ext_info.extension_version; } } @@ -143,6 +151,8 @@ void DuckDBExtensionsFunction(ClientContext &context, TableFunctionInput &data_p output.SetValue(4, count, Value(entry.description)); // aliases LogicalType::LIST(LogicalType::VARCHAR) output.SetValue(5, count, Value::LIST(LogicalType::VARCHAR, entry.aliases)); + // extension version LogicalType::LIST(LogicalType::VARCHAR) + output.SetValue(6, count, Value(entry.extension_version)); data.offset++; count++; diff --git a/src/include/duckdb/main/database.hpp b/src/include/duckdb/main/database.hpp index 90f4dd3c58f9..0d7659ee1b33 100644 --- a/src/include/duckdb/main/database.hpp +++ b/src/include/duckdb/main/database.hpp @@ -27,6 +27,16 @@ class ObjectCache; struct AttachInfo; class DatabaseFileSystem; +struct ExtensionInfo { + explicit ExtensionInfo(const std::string &version) : extension_version(version) { + } + ExtensionInfo() : ExtensionInfo("defaultme") { + } + ExtensionInfo(const ExtensionInfo &x) : ExtensionInfo(x.extension_version) { + } + std::string extension_version; +}; + class DatabaseInstance : public std::enable_shared_from_this { friend class DuckDB; @@ -53,6 +63,7 @@ class DatabaseInstance : public std::enable_shared_from_this { DUCKDB_API static DatabaseInstance &GetDatabase(ClientContext &context); DUCKDB_API const unordered_set &LoadedExtensions(); + DUCKDB_API const unordered_map &LoadedExtensionsData(); DUCKDB_API bool ExtensionIsLoaded(const string &name); DUCKDB_API SettingLookupResult TryGetCurrentSetting(const string &key, Value &result); @@ -73,6 +84,7 @@ class DatabaseInstance : public std::enable_shared_from_this { unique_ptr object_cache; unique_ptr connection_manager; unordered_set loaded_extensions; + unordered_map loaded_extensions_data; ValidChecker db_validity; unique_ptr db_file_system; }; diff --git a/src/main/database.cpp b/src/main/database.cpp index 56202dd7356b..148ac456080f 100644 --- a/src/main/database.cpp +++ b/src/main/database.cpp @@ -388,6 +388,10 @@ const unordered_set &DatabaseInstance::LoadedExtensions() { return loaded_extensions; } +const unordered_map &DatabaseInstance::LoadedExtensionsData() { + return loaded_extensions_data; +} + idx_t DuckDB::NumberOfThreads() { return instance->NumberOfThreads(); } @@ -404,6 +408,7 @@ bool DuckDB::ExtensionIsLoaded(const std::string &name) { void DatabaseInstance::SetExtensionLoaded(const std::string &name, const std::string &extension_version) { auto extension_name = ExtensionHelper::GetExtensionName(name); loaded_extensions.insert(extension_name); + loaded_extensions_data.insert({extension_name, ExtensionInfo(extension_version)}); auto &callbacks = DBConfig::GetConfig(*this).extension_callbacks; for (auto &callback : callbacks) { diff --git a/test/extension/load_extension.test b/test/extension/load_extension.test index b05a3a2f3a72..5a1fe822ea71 100644 --- a/test/extension/load_extension.test +++ b/test/extension/load_extension.test @@ -24,6 +24,11 @@ LOAD NULL; statement ok LOAD '__BUILD_DIRECTORY__/test/extension/loadable_extension_demo.duckdb_extension'; +query I +SELECT extension_version FROM duckdb_extensions() WHERE extension_name = 'loadable_extension_demo'; +---- +default-version + query I SELECT hello('World'); ---- From 1536a7b512f4b348f00beb10945fff5f8dca13cd Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 4 Apr 2024 15:08:11 +0200 Subject: [PATCH 180/603] [Dev] Refactor of filtering of nulls, slightly cleaner --- src/main/extension/extension_load.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/main/extension/extension_load.cpp b/src/main/extension/extension_load.cpp index 1468fbeb3838..9253fac0b22c 100644 --- a/src/main/extension/extension_load.cpp +++ b/src/main/extension/extension_load.cpp @@ -57,6 +57,13 @@ static void ComputeSHA256FileSegment(FileHandle *handle, const idx_t start, cons } #endif +static string FilterZeroAtEnd(string s) { + while (!s.empty() && s.back() == '\0') { + s.pop_back(); + } + return s; +} + bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const string &extension, ExtensionInitResult &result, string &error) { #ifdef DUCKDB_DISABLE_EXTENSION_LOAD @@ -147,16 +154,9 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const str std::reverse(metadata_field.begin(), metadata_field.end()); - for (auto &m : metadata_field) { - auto zero_pos = m.find_first_of((char)0); - if (zero_pos < m.size()) { - m.resize(zero_pos); - } - } - - std::string extension_duckdb_platform = metadata_field[1]; - std::string extension_duckdb_version = metadata_field[2]; - std::string extension_version = metadata_field[3]; + std::string extension_duckdb_platform = FilterZeroAtEnd(metadata_field[1]); + std::string extension_duckdb_version = FilterZeroAtEnd(metadata_field[2]); + std::string extension_version = FilterZeroAtEnd(metadata_field[3]); if (!config.options.allow_unsigned_extensions) { // signature is the last 256 bytes of the file From bee3a5bd5384733208159d4bfc00c0ba44e72fa6 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 4 Apr 2024 15:09:19 +0200 Subject: [PATCH 181/603] Invert metadata errors, and add PrettyPrint function (since we might be reading random bytes) --- src/main/extension/extension_load.cpp | 46 +++++++++++++++++++++------ 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/src/main/extension/extension_load.cpp b/src/main/extension/extension_load.cpp index 9253fac0b22c..b1a64ec3022d 100644 --- a/src/main/extension/extension_load.cpp +++ b/src/main/extension/extension_load.cpp @@ -64,6 +64,32 @@ static string FilterZeroAtEnd(string s) { return s; } +static string PrettyPrintString(const string &s) { + string res = ""; + for (auto c : s) { + if (StringUtil::CharacterIsAlpha(c) || StringUtil::CharacterIsDigit(c) || c == '_' || c == '-' || c == ' ' || + c == '.') { + res += c; + } else { + uint8_t value = c; + res += "\\x"; + uint8_t first = value / 16; + if (first < 10) { + res += '0' + first; + } else { + res += 'a' + first - 10; + } + uint8_t second = value % 16; + if (second < 10) { + res += '0' + second; + } else { + res += 'a' + second - 10; + } + } + } + return res; +} + bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const string &extension, ExtensionInitResult &result, string &error) { #ifdef DUCKDB_DISABLE_EXTENSION_LOAD @@ -218,19 +244,19 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const str } } - if (engine_version != extension_duckdb_version || engine_platform != extension_duckdb_platform) { - throw InvalidInputException( - "Extension \"%s\" (version %s, platfrom %s) does not match DuckDB loading it (version %s, platform %s)", - filename, extension_duckdb_version, extension_duckdb_platform, engine_version, engine_platform); - } - char a[32] = {0}; a[0] = '4'; - if (strcmp(a, metadata_field[0].data()) != 0) { - throw InvalidInputException("Extension \"%s\", version unknown, do not have metadata compatible with DuckDB " - "version (%s %s). Number of fields was different than 4", - filename, extension_duckdb_version, engine_version); + if (strncmp(a, metadata_field[0].data(), 32) != 0) { + throw InvalidInputException("Extension \"%s\" do not have metadata compatible with DuckDB " + "loading it (version %s, platform %s)", + filename, engine_version, engine_platform); + } + + if (engine_version != extension_duckdb_version || engine_platform != extension_duckdb_platform) { + throw InvalidInputException( + "Extension \"%s\" (version %s, platfrom %s) does not match DuckDB loading it (version %s, platform %s)", + filename, PrettyPrintString(extension_duckdb_version), PrettyPrintString(extension_duckdb_platform), engine_version, engine_platform); } auto number_metadata_fields = 3; From df7d0f35eed3b9b439a7481597167272faf001f5 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 4 Apr 2024 15:40:43 +0200 Subject: [PATCH 182/603] Unify extension metadata errors between signed and unsigned --- src/main/extension/extension_load.cpp | 43 ++++++++++++++------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/src/main/extension/extension_load.cpp b/src/main/extension/extension_load.cpp index b1a64ec3022d..afbaa73ee7cc 100644 --- a/src/main/extension/extension_load.cpp +++ b/src/main/extension/extension_load.cpp @@ -164,9 +164,9 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const str idx_t file_size = handle->GetFileSize(); if (file_size < 1024) { - throw InvalidInputException("Extension \"%s\", version unknown, do not have metadata compatible with DuckDB " - "version (%s). File size in particular is %i lower than minimum threshold of 1024", - filename, engine_version, file_size); + throw InvalidInputException("Extension \"%s\" do not have metadata compatible with DuckDB loading it " + "(version %s, platform %s). File size in particular is %i lower than minimum threshold of 1024", + filename, engine_version, engine_platform, file_size); } auto metadata_offset = file_size - metadata_segment.size(); @@ -184,6 +184,23 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const str std::string extension_duckdb_version = FilterZeroAtEnd(metadata_field[2]); std::string extension_version = FilterZeroAtEnd(metadata_field[3]); + string metadata_mismatch_error = ""; + { + char a[32] = {0}; + a[0] = '4'; + if (strncmp(a, metadata_field[0].data(), 32) != 0) { + // metadata do not looks right, add this to the error message + metadata_mismatch_error = "\n" + StringUtil::Format("Extension \"%s\" do not have metadata compatible with DuckDB " + "loading it (version %s, platform %s)", + filename, engine_version, engine_platform); + } else if (engine_version != extension_duckdb_version || engine_platform != extension_duckdb_platform) { + metadata_mismatch_error = "\n" + StringUtil::Format("Extension \"%s\" (version %s, platfrom %s) does not match DuckDB loading it (version %s, platform %s)", + filename, PrettyPrintString(extension_duckdb_version), PrettyPrintString(extension_duckdb_platform), engine_version, engine_platform); + } else { + // All looks good + } + } + if (!config.options.allow_unsigned_extensions) { // signature is the last 256 bytes of the file string signature(metadata_segment, metadata_segment.size() - 256); @@ -237,26 +254,12 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const str } } if (!any_valid) { - throw IOException(config.error_manager->FormatException(ErrorType::UNSIGNED_EXTENSION, filename)); - if (engine_version != extension_duckdb_version || engine_platform != extension_duckdb_platform) { - // FIXME: Add info on platform || version not matching, hinting probably signature is missing - } + throw IOException(config.error_manager->FormatException(ErrorType::UNSIGNED_EXTENSION, filename) + metadata_mismatch_error); } } - char a[32] = {0}; - a[0] = '4'; - - if (strncmp(a, metadata_field[0].data(), 32) != 0) { - throw InvalidInputException("Extension \"%s\" do not have metadata compatible with DuckDB " - "loading it (version %s, platform %s)", - filename, engine_version, engine_platform); - } - - if (engine_version != extension_duckdb_version || engine_platform != extension_duckdb_platform) { - throw InvalidInputException( - "Extension \"%s\" (version %s, platfrom %s) does not match DuckDB loading it (version %s, platform %s)", - filename, PrettyPrintString(extension_duckdb_version), PrettyPrintString(extension_duckdb_platform), engine_version, engine_platform); + if (!metadata_mismatch_error.empty()) { + throw InvalidInputException(metadata_mismatch_error.substr(1)); } auto number_metadata_fields = 3; From 05768faa83c28ca9c78028cda0891c589a5ed857 Mon Sep 17 00:00:00 2001 From: Laurens Kuiper Date: Thu, 4 Apr 2024 16:53:42 +0200 Subject: [PATCH 183/603] remove skips --- .../copy/parquet/dictionary_compression_ratio_threshold.test | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/sql/copy/parquet/dictionary_compression_ratio_threshold.test b/test/sql/copy/parquet/dictionary_compression_ratio_threshold.test index b417dd8c6ee9..b0bfacd4bbf7 100644 --- a/test/sql/copy/parquet/dictionary_compression_ratio_threshold.test +++ b/test/sql/copy/parquet/dictionary_compression_ratio_threshold.test @@ -7,8 +7,6 @@ require parquet statement ok CREATE TABLE test AS SELECT 'thisisaverylongstringbutitrepeatsmanytimessoitshighlycompressible' || (range % 10) i FROM range(100000) -mode skip - # cannot be negative statement error COPY test TO '__TEST_DIR__/dictionary_compression_ratio_threshold.parquet' (dictionary_compression_ratio_threshold -2) @@ -35,8 +33,6 @@ SELECT dictionary_page_offset IS NULL FROM parquet_metadata('__TEST_DIR__/dictio ---- true -mode unskip - # the data compresses more than 10x statement ok COPY test TO '__TEST_DIR__/dictionary_compression_ratio_threshold.parquet' (dictionary_compression_ratio_threshold 10) From 8d8044c5319406acb9d8a11f5251a41e6ad552e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Thu, 4 Apr 2024 16:58:33 +0200 Subject: [PATCH 184/603] moo --- src/include/duckdb/common/bitpacking.hpp | 4 ++-- third_party/concurrentqueue/concurrentqueue.h | 10 +++++----- third_party/utf8proc/include/utf8proc.hpp | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/include/duckdb/common/bitpacking.hpp b/src/include/duckdb/common/bitpacking.hpp index 3b499a4b4368..e0a1fe23edb6 100644 --- a/src/include/duckdb/common/bitpacking.hpp +++ b/src/include/duckdb/common/bitpacking.hpp @@ -174,7 +174,7 @@ class BitpackingPrimitives { if (bitwidth < sizeof(T) * 8 && bitwidth != 0) { if (is_signed) { D_ASSERT(max_value <= (T(1) << (bitwidth - 1)) - 1); - D_ASSERT(min_value >= (T(-1) * ((T(1) << (bitwidth - 1)) - 1) - 1)); + // D_ASSERT(min_value >= (T(-1) * ((T(1) << (bitwidth - 1)) - 1) - 1)); } else { D_ASSERT(max_value <= (T(1) << (bitwidth)) - 1); } @@ -192,7 +192,7 @@ class BitpackingPrimitives { T const mask = UnsafeNumericCast(T_U(1) << (width - 1)); for (idx_t i = 0; i < BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE; ++i) { T value = Load(dst + i * sizeof(T)); - value = UnsafeNumericCast(value & ((T_U(1) << width) - T_U(1))); + value = UnsafeNumericCast(T_U(value) & ((T_U(1) << width) - T_U(1))); T result = (value ^ mask) - mask; Store(result, dst + i * sizeof(T)); } diff --git a/third_party/concurrentqueue/concurrentqueue.h b/third_party/concurrentqueue/concurrentqueue.h index f3e2b1005eec..0f5ad0a4d625 100644 --- a/third_party/concurrentqueue/concurrentqueue.h +++ b/third_party/concurrentqueue/concurrentqueue.h @@ -1942,7 +1942,7 @@ class ConcurrentQueue // block size (in order to get a correct signed block count offset in all cases): auto headBase = localBlockIndex->entries[localBlockIndexHead].base; auto blockBaseIndex = index & ~static_cast(BLOCK_SIZE - 1); - auto offset = static_cast(static_cast::type>(blockBaseIndex - headBase) / BLOCK_SIZE); + auto offset = static_cast(static_cast::type>(blockBaseIndex - headBase) / static_cast::type>(BLOCK_SIZE)); auto block = localBlockIndex->entries[(localBlockIndexHead + offset) & (localBlockIndex->size - 1)].block; // Dequeue @@ -2202,7 +2202,7 @@ class ConcurrentQueue auto headBase = localBlockIndex->entries[localBlockIndexHead].base; auto firstBlockBaseIndex = firstIndex & ~static_cast(BLOCK_SIZE - 1); - auto offset = static_cast(static_cast::type>(firstBlockBaseIndex - headBase) / BLOCK_SIZE); + auto offset = static_cast(static_cast::type>(firstBlockBaseIndex - headBase) / static_cast::type>(BLOCK_SIZE)); auto indexIndex = (localBlockIndexHead + offset) & (localBlockIndex->size - 1); // Iterate the blocks and dequeue @@ -2875,7 +2875,7 @@ class ConcurrentQueue assert(tailBase != INVALID_BLOCK_BASE); // Note: Must use division instead of shift because the index may wrap around, causing a negative // offset, whose negativity we want to preserve - auto offset = static_cast(static_cast::type>(index - tailBase) / BLOCK_SIZE); + auto offset = static_cast(static_cast::type>(index - tailBase) / static_cast::type>(BLOCK_SIZE)); size_t idx = (tail + offset) & (localBlockIndex->capacity - 1); assert(localBlockIndex->index[idx]->key.load(std::memory_order_relaxed) == index && localBlockIndex->index[idx]->value.load(std::memory_order_relaxed) != nullptr); return idx; @@ -3630,7 +3630,7 @@ ConsumerToken::ConsumerToken(ConcurrentQueue& queue) : itemsConsumedFromCurrent(0), currentProducer(nullptr), desiredProducer(nullptr) { initialOffset = queue.nextExplicitConsumerId.fetch_add(1, std::memory_order_release); - lastKnownGlobalOffset = -1; + lastKnownGlobalOffset = uint32_t(-1); } template @@ -3638,7 +3638,7 @@ ConsumerToken::ConsumerToken(BlockingConcurrentQueue& queue) : itemsConsumedFromCurrent(0), currentProducer(nullptr), desiredProducer(nullptr) { initialOffset = reinterpret_cast*>(&queue)->nextExplicitConsumerId.fetch_add(1, std::memory_order_release); - lastKnownGlobalOffset = -1; + lastKnownGlobalOffset = uint32_t(-1); } template diff --git a/third_party/utf8proc/include/utf8proc.hpp b/third_party/utf8proc/include/utf8proc.hpp index 336f95e2ed5a..37fa1f2664b8 100644 --- a/third_party/utf8proc/include/utf8proc.hpp +++ b/third_party/utf8proc/include/utf8proc.hpp @@ -637,7 +637,7 @@ void utf8proc_grapheme_callback(const char *s, size_t len, T &&fun) { size_t start = 0; size_t cpos = 0; while(true) { - cpos += UnsafeNumericCast(sz); + cpos += static_cast(sz); if (cpos >= len) { fun(start, cpos); return; From 8c184d6248e84f9791ecfc9dc5a3180d565619ba Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 4 Apr 2024 17:05:11 +0200 Subject: [PATCH 185/603] More tests --- .../multiple_errors/invalid_utf_cast.csv | 3 + .../multiple_errors/invalid_utf_less.csv | 3 + .../multiple_errors/invalid_utf_max_line.csv | 3 + .../multiple_errors/invalid_utf_more.csv | 3 + .../multiple_errors/invalid_utf_unquoted.csv | 3 + .../{ => multiple_errors}/multiple_errors.csv | 2 +- .../scanner/string_value_scanner.cpp | 15 +- .../test_multiple_errors_same_line.test | 136 +++++++++++++++--- 8 files changed, 141 insertions(+), 27 deletions(-) create mode 100644 data/csv/rejects/multiple_errors/invalid_utf_cast.csv create mode 100644 data/csv/rejects/multiple_errors/invalid_utf_less.csv create mode 100644 data/csv/rejects/multiple_errors/invalid_utf_max_line.csv create mode 100644 data/csv/rejects/multiple_errors/invalid_utf_more.csv create mode 100644 data/csv/rejects/multiple_errors/invalid_utf_unquoted.csv rename data/csv/rejects/{ => multiple_errors}/multiple_errors.csv (89%) diff --git a/data/csv/rejects/multiple_errors/invalid_utf_cast.csv b/data/csv/rejects/multiple_errors/invalid_utf_cast.csv new file mode 100644 index 000000000000..a4b2844afd03 --- /dev/null +++ b/data/csv/rejects/multiple_errors/invalid_utf_cast.csv @@ -0,0 +1,3 @@ +name,age,current_day, barks +oogie boogie,3, 2023-01-01, 2 +oogie boÿÿgie,bla, 2023-01-01, 2 \ No newline at end of file diff --git a/data/csv/rejects/multiple_errors/invalid_utf_less.csv b/data/csv/rejects/multiple_errors/invalid_utf_less.csv new file mode 100644 index 000000000000..adf74fc1e5fd --- /dev/null +++ b/data/csv/rejects/multiple_errors/invalid_utf_less.csv @@ -0,0 +1,3 @@ +name,age,current_day, barks +oogie boogie,3, 2023-01-01, 2 +oogie boÿÿgie,3, 2023-01-01 \ No newline at end of file diff --git a/data/csv/rejects/multiple_errors/invalid_utf_max_line.csv b/data/csv/rejects/multiple_errors/invalid_utf_max_line.csv new file mode 100644 index 000000000000..1f017d2d8cf7 --- /dev/null +++ b/data/csv/rejects/multiple_errors/invalid_utf_max_line.csv @@ -0,0 +1,3 @@ +name,age,current_day, barks +oogie boogie,3, 2023-01-01, 2 +oogie boÿÿgieoogie boogieoogie boogieoogie boogie,3, 2023-01-01, 2, 5 \ No newline at end of file diff --git a/data/csv/rejects/multiple_errors/invalid_utf_more.csv b/data/csv/rejects/multiple_errors/invalid_utf_more.csv new file mode 100644 index 000000000000..17fa55e72875 --- /dev/null +++ b/data/csv/rejects/multiple_errors/invalid_utf_more.csv @@ -0,0 +1,3 @@ +name,age,current_day, barks +oogie boogie,3, 2023-01-01, 2 +oogie boÿÿgie,3, 2023-01-01, 2, 5 \ No newline at end of file diff --git a/data/csv/rejects/multiple_errors/invalid_utf_unquoted.csv b/data/csv/rejects/multiple_errors/invalid_utf_unquoted.csv new file mode 100644 index 000000000000..3aaedb256d05 --- /dev/null +++ b/data/csv/rejects/multiple_errors/invalid_utf_unquoted.csv @@ -0,0 +1,3 @@ +name,last_name, age,current_day, barks +oogie, boogie,3, 2023-01-01, 2 +"oogie"bla, boÿÿgie,3, 2023-01-01, 2 \ No newline at end of file diff --git a/data/csv/rejects/multiple_errors.csv b/data/csv/rejects/multiple_errors/multiple_errors.csv similarity index 89% rename from data/csv/rejects/multiple_errors.csv rename to data/csv/rejects/multiple_errors/multiple_errors.csv index 6d10e51c3cf5..784fdd9d9faf 100644 --- a/data/csv/rejects/multiple_errors.csv +++ b/data/csv/rejects/multiple_errors/multiple_errors.csv @@ -3,6 +3,6 @@ oogie boogie,3, 2023-01-01, 2 oogie boogie,3, 2023-01-02, 5 oogie boogie,3, 2023-01-03, bla, 7 oogie boogie,3, bla, bla, 7 -oogie boogie,3, 2023-01-04, 8 +"oogie boogie"bla,3, 2023-01-04 oogie boogie,3, bla oogie boogieoogie boogieoogie boogieoogie boogieoogie boogieoogie boogieoogie boogie,3, bla diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index acade767b83e..9f2682deadce 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -334,11 +334,6 @@ void StringValueResult::AddValue(StringValueResult &result, const idx_t buffer_p void StringValueResult::HandleUnicodeError(idx_t col_idx, LinePosition &error_position) { bool first_nl; auto borked_line = current_line_position.ReconstructCurrentLine(first_nl, buffer_handles); - // sanitize borked line - std::vector char_array(borked_line.begin(), borked_line.end()); - char_array.push_back('\0'); // Null-terminate the character array - Utf8Proc::MakeValid(&char_array[0], char_array.size()); - borked_line = {char_array.begin(), char_array.end() - 1}; LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read); if (current_line_position.begin == error_position) { auto csv_error = CSVError::InvalidUTF8(state_machine.options, col_idx, lines_per_batch, borked_line, @@ -379,11 +374,6 @@ bool StringValueResult::HandleError() { } break; case CSVErrorType::INVALID_UNICODE: { - // We have to sanitize the CSV line - std::vector char_array(borked_line.begin(), borked_line.end()); - char_array.push_back('\0'); // Null-terminate the character array - Utf8Proc::MakeValid(&char_array[0], char_array.size()); - borked_line = {char_array.begin(), char_array.end() - 1}; if (current_line_position.begin == line_pos) { csv_error = CSVError::InvalidUTF8(state_machine.options, col_idx, lines_per_batch, borked_line, @@ -489,6 +479,11 @@ string FullLinePosition::ReconstructCurrentLine(bool &first_char_nl, result += second_buffer[i]; } } + // sanitize borked line + std::vector char_array(result.begin(), result.end()); + char_array.push_back('\0'); // Null-terminate the character array + Utf8Proc::MakeValid(&char_array[0], char_array.size()); + result = {char_array.begin(), char_array.end() - 1}; return result; } diff --git a/test/sql/copy/csv/rejects/test_multiple_errors_same_line.test b/test/sql/copy/csv/rejects/test_multiple_errors_same_line.test index 112e7e0e2575..6d9f1fcb5baa 100644 --- a/test/sql/copy/csv/rejects/test_multiple_errors_same_line.test +++ b/test/sql/copy/csv/rejects/test_multiple_errors_same_line.test @@ -257,19 +257,123 @@ DROP TABLE reject_errors; statement ok DROP TABLE reject_scans; -#query IIII -#FROM read_csv('data/csv/rejects/multiple_errors.csv', -# columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, -# store_rejects = true, auto_detect=false, header = 1, max_line_size=40); -#---- -#oogie boogie 3 2023-01-01 2 -#oogie boogie 3 2023-01-02 5 -# -#query IIIIIIIIII rowsort -#FROM reject_errors ORDER BY ALL; -#---- -#3 0 4 89 116 4 barks CAST oogie boogie,3, 2023-01-03, bla, 7 Error when converting column "barks". Could not convert string " bla" to 'INTEGER' -#3 0 4 89 120 5 NULL TOO MANY COLUMNS oogie boogie,3, 2023-01-03, bla, 7 Expected Number of Columns: 4 Found: 5 -#3 0 5 124 144 4 barks CAST oogie boogie,3, bla, bla, 7 Error when converting column "barks". Could not convert string " bla" to 'INTEGER' -#3 0 5 124 148 5 NULL TOO MANY COLUMNS oogie boogie,3, bla, bla, 7 Expected Number of Columns: 4 Found: 5 -#3 0 6 152 171 3 barks MISSING COLUMNS oogie boogie,3, bla Expected Number of Columns: 4 Found: 3 \ No newline at end of file +query IIII +FROM read_csv('data/csv/rejects/multiple_errors/invalid_utf_cast.csv', + columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, + store_rejects = true, auto_detect=false, header = 1, max_line_size=40); +---- +oogie boogie 3 2023-01-01 2 + +query IIIIIIIII rowsort +SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY byte_position; +---- +0 3 59 59 1 name INVALID UNICODE oogie bo??gie,bla, 2023-01-01, 2 Invalid unicode (byte sequence mismatch) detected. +0 3 59 73 2 age CAST oogie bo??gie,bla, 2023-01-01, 2 Error when converting column "age". Could not convert string "bla" to 'INTEGER' + +statement ok +DROP TABLE reject_errors; + +statement ok +DROP TABLE reject_scans; + +query IIII +FROM read_csv('data/csv/rejects/multiple_errors/invalid_utf_less.csv', + columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, + store_rejects = true, auto_detect=false, header = 1, max_line_size=40); +---- +oogie boogie 3 2023-01-01 2 + + +query IIIIIIIII rowsort +SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY byte_position; +---- +0 3 59 59 1 name INVALID UNICODE oogie bo??gie,3, 2023-01-01 Invalid unicode (byte sequence mismatch) detected. +0 3 59 86 3 barks MISSING COLUMNS oogie bo??gie,3, 2023-01-01 Expected Number of Columns: 4 Found: 3 + +statement ok +DROP TABLE reject_errors; + +statement ok +DROP TABLE reject_scans; + +query IIII +FROM read_csv('data/csv/rejects/multiple_errors/invalid_utf_max_line.csv', + columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, + store_rejects = true, auto_detect=false, header = 1, max_line_size=40); +---- +oogie boogie 3 2023-01-01 2 + + +query IIIIIIIII rowsort +SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY byte_position, error_message; +---- +0 3 59 125 5 NULL TOO MANY COLUMNS oogie bo??gieoogie boogieoogie boogieoogie boogie,3, 2023-01-01, 2, 5 Expected Number of Columns: 4 Found: 5 +0 3 59 59 1 name INVALID UNICODE oogie bo??gieoogie boogieoogie boogieoogie boogie,3, 2023-01-01, 2, 5 Invalid unicode (byte sequence mismatch) detected. +0 3 59 59 NULL NULL LINE SIZE OVER MAXIMUM oogie bo??gieoogie boogieoogie boogieoogie boogie,3, 2023-01-01, 2, 5 Maximum line size of 40 bytes exceeded. Actual Size:70 bytes. + +statement ok +DROP TABLE reject_errors; + +statement ok +DROP TABLE reject_scans; + +query IIII +FROM read_csv('data/csv/rejects/multiple_errors/invalid_utf_more.csv', + columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, + store_rejects = true, auto_detect=false, header = 1, max_line_size=40); +---- +oogie boogie 3 2023-01-01 2 + + +query IIIIIIIII rowsort +SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY byte_position; +---- +0 3 59 59 1 name INVALID UNICODE oogie bo??gie,3, 2023-01-01, 2, 5 Invalid unicode (byte sequence mismatch) detected. +0 3 59 89 5 NULL TOO MANY COLUMNS oogie bo??gie,3, 2023-01-01, 2, 5 Expected Number of Columns: 4 Found: 5 + +statement ok +DROP TABLE reject_errors; + +statement ok +DROP TABLE reject_scans; + +query IIIII +FROM read_csv('data/csv/rejects/multiple_errors/invalid_utf_unquoted.csv', + columns = {'name': 'VARCHAR', 'last_name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, + store_rejects = true, auto_detect=false, header = 1, max_line_size=40); +---- +oogie boogie 3 2023-01-01 2 + + +query IIIIIIIII rowsort +SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY byte_position; +---- +0 3 71 71 1 name UNQUOTED VALUE "oogie"bla, bo??gie,3, 2023-01-01, 2 Value with unterminated quote found. +0 3 71 82 2 last_name INVALID UNICODE "oogie"bla, bo??gie,3, 2023-01-01, 2 Invalid unicode (byte sequence mismatch) detected. + +statement ok +DROP TABLE reject_errors; + +statement ok +DROP TABLE reject_scans; + +query IIII +FROM read_csv('data/csv/rejects/multiple_errors/multiple_errors.csv', + columns = {'name': 'VARCHAR', 'age': 'INTEGER', 'current_day': 'DATE', 'barks': 'INTEGER'}, + store_rejects = true, auto_detect=false, header = 1, max_line_size=40); +---- +oogie boogie 3 2023-01-01 2 +oogie boogie 3 2023-01-02 5 + +query IIIIIIIII rowsort +SELECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; +---- +0 4 89 116 4 barks CAST oogie boogie,3, 2023-01-03, bla, 7 Error when converting column "barks". Could not convert string " bla" to 'INTEGER' +0 4 89 120 5 NULL TOO MANY COLUMNS oogie boogie,3, 2023-01-03, bla, 7 Expected Number of Columns: 4 Found: 5 +0 5 124 144 4 barks CAST oogie boogie,3, bla, bla, 7 Error when converting column "barks". Could not convert string " bla" to 'INTEGER' +0 5 124 148 5 NULL TOO MANY COLUMNS oogie boogie,3, bla, bla, 7 Expected Number of Columns: 4 Found: 5 +0 6 152 152 1 name UNQUOTED VALUE "oogie boogie"bla,3, 2023-01-04 Value with unterminated quote found. +0 6 152 183 3 barks MISSING COLUMNS "oogie boogie"bla,3, 2023-01-04 Expected Number of Columns: 4 Found: 3 +0 7 184 203 3 barks MISSING COLUMNS oogie boogie,3, bla Expected Number of Columns: 4 Found: 3 +0 8 204 204 NULL NULL LINE SIZE OVER MAXIMUM oogie boogieoogie boogieoogie boogieoogie boogieoogie boogieoogie boogieoogie boogie,3, bla Maximum line size of 40 bytes exceeded. Actual Size:92 bytes. +0 8 204 295 3 barks MISSING COLUMNS oogie boogieoogie boogieoogie boogieoogie boogieoogie boogieoogie boogieoogie boogie,3, bla Expected Number of Columns: 4 Found: 3 From 8dd5df622cf116b41d847a9c33e8cc26718b47cd Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 4 Apr 2024 17:05:26 +0200 Subject: [PATCH 186/603] woopsie on gen files --- .github/regression/micro_extended.csv | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/regression/micro_extended.csv b/.github/regression/micro_extended.csv index a9517ef309b4..6973785b4c98 100644 --- a/.github/regression/micro_extended.csv +++ b/.github/regression/micro_extended.csv @@ -78,7 +78,6 @@ benchmark/micro/copy/to_parquet_partition_by_few.benchmark benchmark/micro/copy/to_parquet_partition_by_many.benchmark benchmark/micro/csv/16_byte_values.benchmark benchmark/micro/csv/1_byte_values.benchmark -benchmark/micro/csv/1brl.benchmark benchmark/micro/csv/multiple_read.benchmark benchmark/micro/csv/multiple_small_read_csv.benchmark benchmark/micro/csv/null_padding.benchmark From b134927d2af44dfde501d3db51838e38f52f6bf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Thu, 4 Apr 2024 17:14:08 +0200 Subject: [PATCH 187/603] un-funking build --- .../operator/aggregate/physical_window.cpp | 2 +- src/optimizer/move_constants.cpp | 165 ++++++++++++++++++ .../rule/conjunction_simplification.cpp | 70 ++++++++ src/optimizer/rule/constant_folding.cpp | 43 +++++ .../rule/date_part_simplification.cpp | 103 +++++++++++ src/optimizer/rule/empty_needle_removal.cpp | 54 ++++++ src/optimizer/rule/enum_comparison.cpp | 70 ++++++++ .../rule/in_clause_simplification_rule.cpp | 57 ++++++ src/optimizer/rule/like_optimizations.cpp | 161 +++++++++++++++++ src/optimizer/rule/move_constants.cpp | 164 +++++++++++++++++ src/planner/CMakeLists.txt | 26 +++ 11 files changed, 914 insertions(+), 1 deletion(-) create mode 100644 src/optimizer/move_constants.cpp diff --git a/src/execution/operator/aggregate/physical_window.cpp b/src/execution/operator/aggregate/physical_window.cpp index 01e6752a2cd0..c4cd9d4cbb98 100644 --- a/src/execution/operator/aggregate/physical_window.cpp +++ b/src/execution/operator/aggregate/physical_window.cpp @@ -327,7 +327,7 @@ void WindowPartitionSourceState::MaterializeSortedData() { heap->blocks = std::move(sd.heap_blocks); hash_group.reset(); } else { - heap = make_uniq(buffer_manager, Storage::BLOCK_SIZE, 1U, true); + heap = make_uniq(buffer_manager, (idx_t)Storage::BLOCK_SIZE, 1U, true); } heap->count = std::accumulate(heap->blocks.begin(), heap->blocks.end(), idx_t(0), [&](idx_t c, const unique_ptr &b) { return c + b->count; }); diff --git a/src/optimizer/move_constants.cpp b/src/optimizer/move_constants.cpp new file mode 100644 index 000000000000..636265ff9131 --- /dev/null +++ b/src/optimizer/move_constants.cpp @@ -0,0 +1,165 @@ +#include "duckdb/optimizer/rule/move_constants.hpp" + +#include "duckdb/common/exception.hpp" +#include "duckdb/common/value_operations/value_operations.hpp" +#include "duckdb/planner/expression/bound_comparison_expression.hpp" +#include "duckdb/planner/expression/bound_constant_expression.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" +#include "duckdb/optimizer/expression_rewriter.hpp" + +namespace duckdb { + +MoveConstantsRule::MoveConstantsRule(ExpressionRewriter &rewriter) : Rule(rewriter) { + auto op = make_uniq(); + op->matchers.push_back(make_uniq()); + op->policy = SetMatcher::Policy::UNORDERED; + + auto arithmetic = make_uniq(); + // we handle multiplication, addition and subtraction because those are "easy" + // integer division makes the division case difficult + // e.g. [x / 2 = 3] means [x = 6 OR x = 7] because of truncation -> no clean rewrite rules + arithmetic->function = make_uniq(unordered_set {"+", "-", "*"}); + // we match only on integral numeric types + arithmetic->type = make_uniq(); + auto child_constant_matcher = make_uniq(); + auto child_expression_matcher = make_uniq(); + child_constant_matcher->type = make_uniq(); + child_expression_matcher->type = make_uniq(); + arithmetic->matchers.push_back(std::move(child_constant_matcher)); + arithmetic->matchers.push_back(std::move(child_expression_matcher)); + arithmetic->policy = SetMatcher::Policy::SOME; + op->matchers.push_back(std::move(arithmetic)); + root = std::move(op); +} + +unique_ptr MoveConstantsRule::Apply(LogicalOperator &op, vector> &bindings, + bool &changes_made, bool is_root) { + auto &comparison = bindings[0].get().Cast(); + auto &outer_constant = bindings[1].get().Cast(); + auto &arithmetic = bindings[2].get().Cast(); + auto &inner_constant = bindings[3].get().Cast(); + D_ASSERT(arithmetic.return_type.IsIntegral()); + D_ASSERT(arithmetic.children[0]->return_type.IsIntegral()); + if (inner_constant.value.IsNull() || outer_constant.value.IsNull()) { + return make_uniq(Value(comparison.return_type)); + } + auto &constant_type = outer_constant.return_type; + hugeint_t outer_value = IntegralValue::Get(outer_constant.value); + hugeint_t inner_value = IntegralValue::Get(inner_constant.value); + + idx_t arithmetic_child_index = arithmetic.children[0].get() == &inner_constant ? 1 : 0; + auto &op_type = arithmetic.function.name; + if (op_type == "+") { + // [x + 1 COMP 10] OR [1 + x COMP 10] + // order does not matter in addition: + // simply change right side to 10-1 (outer_constant - inner_constant) + if (!Hugeint::TrySubtractInPlace(outer_value, inner_value)) { + return nullptr; + } + auto result_value = Value::HUGEINT(outer_value); + if (!result_value.DefaultTryCastAs(constant_type)) { + if (comparison.type != ExpressionType::COMPARE_EQUAL) { + return nullptr; + } + // if the cast is not possible then the comparison is not possible + // for example, if we have x + 5 = 3, where x is an unsigned number, we will get x = -2 + // since this is not possible we can remove the entire branch here + return ExpressionRewriter::ConstantOrNull(std::move(arithmetic.children[arithmetic_child_index]), + Value::BOOLEAN(false)); + } + outer_constant.value = std::move(result_value); + } else if (op_type == "-") { + // [x - 1 COMP 10] O R [1 - x COMP 10] + // order matters in subtraction: + if (arithmetic_child_index == 0) { + // [x - 1 COMP 10] + // change right side to 10+1 (outer_constant + inner_constant) + if (!Hugeint::TryAddInPlace(outer_value, inner_value)) { + return nullptr; + } + auto result_value = Value::HUGEINT(outer_value); + if (!result_value.DefaultTryCastAs(constant_type)) { + // if the cast is not possible then an equality comparison is not possible + if (comparison.type != ExpressionType::COMPARE_EQUAL) { + return nullptr; + } + return ExpressionRewriter::ConstantOrNull(std::move(arithmetic.children[arithmetic_child_index]), + Value::BOOLEAN(false)); + } + outer_constant.value = std::move(result_value); + } else { + // [1 - x COMP 10] + // change right side to 1-10=-9 + if (!Hugeint::TrySubtractInPlace(inner_value, outer_value)) { + return nullptr; + } + auto result_value = Value::HUGEINT(inner_value); + if (!result_value.DefaultTryCastAs(constant_type)) { + // if the cast is not possible then an equality comparison is not possible + if (comparison.type != ExpressionType::COMPARE_EQUAL) { + return nullptr; + } + return ExpressionRewriter::ConstantOrNull(std::move(arithmetic.children[arithmetic_child_index]), + Value::BOOLEAN(false)); + } + outer_constant.value = std::move(result_value); + // in this case, we should also flip the comparison + // e.g. if we have [4 - x < 2] then we should have [x > 2] + comparison.type = FlipComparisonExpression(comparison.type); + } + } else { + D_ASSERT(op_type == "*"); + // [x * 2 COMP 10] OR [2 * x COMP 10] + // order does not matter in multiplication: + // change right side to 10/2 (outer_constant / inner_constant) + // but ONLY if outer_constant is cleanly divisible by the inner_constant + if (inner_value == 0) { + // x * 0, the result is either 0 or NULL + // we let the arithmetic_simplification rule take care of simplifying this first + return nullptr; + } + // check out of range for HUGEINT or not cleanly divisible + // HUGEINT is not cleanly divisible when outer_value == minimum and inner value == -1. (modulo overflow) + if ((outer_value == NumericLimits::Minimum() && inner_value == -1) || + outer_value % inner_value != 0) { + bool is_equality = comparison.type == ExpressionType::COMPARE_EQUAL; + bool is_inequality = comparison.type == ExpressionType::COMPARE_NOTEQUAL; + if (is_equality || is_inequality) { + // we know the values are not equal + // the result will be either FALSE or NULL (if COMPARE_EQUAL) + // or TRUE or NULL (if COMPARE_NOTEQUAL) + return ExpressionRewriter::ConstantOrNull(std::move(arithmetic.children[arithmetic_child_index]), + Value::BOOLEAN(is_inequality)); + } else { + // not cleanly divisible and we are doing > >= < <=, skip the simplification for now + return nullptr; + } + } + if (inner_value < 0) { + // multiply by negative value, need to flip expression + comparison.type = FlipComparisonExpression(comparison.type); + } + // else divide the RHS by the LHS + // we need to do a range check on the cast even though we do a division + // because e.g. -128 / -1 = 128, which is out of range + auto result_value = Value::HUGEINT(outer_value / inner_value); + if (!result_value.DefaultTryCastAs(constant_type)) { + return ExpressionRewriter::ConstantOrNull(std::move(arithmetic.children[arithmetic_child_index]), + Value::BOOLEAN(false)); + } + outer_constant.value = std::move(result_value); + } + // replace left side with x + // first extract x from the arithmetic expression + auto arithmetic_child = std::move(arithmetic.children[arithmetic_child_index]); + // then place in the comparison + if (comparison.left.get() == &outer_constant) { + comparison.right = std::move(arithmetic_child); + } else { + comparison.left = std::move(arithmetic_child); + } + changes_made = true; + return nullptr; +} + +} // namespace duckdb diff --git a/src/optimizer/rule/conjunction_simplification.cpp b/src/optimizer/rule/conjunction_simplification.cpp index 8b137891791f..14fd80a9423b 100644 --- a/src/optimizer/rule/conjunction_simplification.cpp +++ b/src/optimizer/rule/conjunction_simplification.cpp @@ -1 +1,71 @@ +#include "duckdb/optimizer/rule/conjunction_simplification.hpp" + +#include "duckdb/execution/expression_executor.hpp" +#include "duckdb/planner/expression/bound_conjunction_expression.hpp" +#include "duckdb/planner/expression/bound_constant_expression.hpp" + +namespace duckdb { + +ConjunctionSimplificationRule::ConjunctionSimplificationRule(ExpressionRewriter &rewriter) : Rule(rewriter) { + // match on a ComparisonExpression that has a ConstantExpression as a check + auto op = make_uniq(); + op->matchers.push_back(make_uniq()); + op->policy = SetMatcher::Policy::SOME; + root = std::move(op); +} + +unique_ptr ConjunctionSimplificationRule::RemoveExpression(BoundConjunctionExpression &conj, + const Expression &expr) { + for (idx_t i = 0; i < conj.children.size(); i++) { + if (conj.children[i].get() == &expr) { + // erase the expression + conj.children.erase_at(i); + break; + } + } + if (conj.children.size() == 1) { + // one expression remaining: simply return that expression and erase the conjunction + return std::move(conj.children[0]); + } + return nullptr; +} + +unique_ptr ConjunctionSimplificationRule::Apply(LogicalOperator &op, + vector> &bindings, bool &changes_made, + bool is_root) { + auto &conjunction = bindings[0].get().Cast(); + auto &constant_expr = bindings[1].get(); + // the constant_expr is a scalar expression that we have to fold + // use an ExpressionExecutor to execute the expression + D_ASSERT(constant_expr.IsFoldable()); + Value constant_value; + if (!ExpressionExecutor::TryEvaluateScalar(GetContext(), constant_expr, constant_value)) { + return nullptr; + } + constant_value = constant_value.DefaultCastAs(LogicalType::BOOLEAN); + if (constant_value.IsNull()) { + // we can't simplify conjunctions with a constant NULL + return nullptr; + } + if (conjunction.type == ExpressionType::CONJUNCTION_AND) { + if (!BooleanValue::Get(constant_value)) { + // FALSE in AND, result of expression is false + return make_uniq(Value::BOOLEAN(false)); + } else { + // TRUE in AND, remove the expression from the set + return RemoveExpression(conjunction, constant_expr); + } + } else { + D_ASSERT(conjunction.type == ExpressionType::CONJUNCTION_OR); + if (!BooleanValue::Get(constant_value)) { + // FALSE in OR, remove the expression from the set + return RemoveExpression(conjunction, constant_expr); + } else { + // TRUE in OR, result of expression is true + return make_uniq(Value::BOOLEAN(true)); + } + } +} + +} // namespace duckdb diff --git a/src/optimizer/rule/constant_folding.cpp b/src/optimizer/rule/constant_folding.cpp index 8b137891791f..7702b46040f7 100644 --- a/src/optimizer/rule/constant_folding.cpp +++ b/src/optimizer/rule/constant_folding.cpp @@ -1 +1,44 @@ +#include "duckdb/optimizer/rule/constant_folding.hpp" + +#include "duckdb/common/exception.hpp" +#include "duckdb/execution/expression_executor.hpp" +#include "duckdb/optimizer/expression_rewriter.hpp" +#include "duckdb/planner/expression/bound_constant_expression.hpp" + +namespace duckdb { + +//! The ConstantFoldingExpressionMatcher matches on any scalar expression (i.e. Expression::IsFoldable is true) +class ConstantFoldingExpressionMatcher : public FoldableConstantMatcher { +public: + bool Match(Expression &expr, vector> &bindings) override { + // we also do not match on ConstantExpressions, because we cannot fold those any further + if (expr.type == ExpressionType::VALUE_CONSTANT) { + return false; + } + return FoldableConstantMatcher::Match(expr, bindings); + } +}; + +ConstantFoldingRule::ConstantFoldingRule(ExpressionRewriter &rewriter) : Rule(rewriter) { + auto op = make_uniq(); + root = std::move(op); +} + +unique_ptr ConstantFoldingRule::Apply(LogicalOperator &op, vector> &bindings, + bool &changes_made, bool is_root) { + auto &root = bindings[0].get(); + // the root is a scalar expression that we have to fold + D_ASSERT(root.IsFoldable() && root.type != ExpressionType::VALUE_CONSTANT); + + // use an ExpressionExecutor to execute the expression + Value result_value; + if (!ExpressionExecutor::TryEvaluateScalar(GetContext(), root, result_value)) { + return nullptr; + } + D_ASSERT(result_value.type().InternalType() == root.return_type.InternalType()); + // now get the value from the result vector and insert it back into the plan as a constant expression + return make_uniq(result_value); +} + +} // namespace duckdb diff --git a/src/optimizer/rule/date_part_simplification.cpp b/src/optimizer/rule/date_part_simplification.cpp index 8b137891791f..6737e576c7dd 100644 --- a/src/optimizer/rule/date_part_simplification.cpp +++ b/src/optimizer/rule/date_part_simplification.cpp @@ -1 +1,104 @@ +#include "duckdb/optimizer/rule/date_part_simplification.hpp" +#include "duckdb/common/exception.hpp" +#include "duckdb/planner/expression/bound_constant_expression.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" +#include "duckdb/optimizer/matcher/expression_matcher.hpp" +#include "duckdb/optimizer/expression_rewriter.hpp" +#include "duckdb/common/enums/date_part_specifier.hpp" +#include "duckdb/function/function.hpp" +#include "duckdb/function/function_binder.hpp" + +namespace duckdb { + +DatePartSimplificationRule::DatePartSimplificationRule(ExpressionRewriter &rewriter) : Rule(rewriter) { + auto func = make_uniq(); + func->function = make_uniq("date_part"); + func->matchers.push_back(make_uniq()); + func->matchers.push_back(make_uniq()); + func->policy = SetMatcher::Policy::ORDERED; + root = std::move(func); +} + +unique_ptr DatePartSimplificationRule::Apply(LogicalOperator &op, vector> &bindings, + bool &changes_made, bool is_root) { + auto &date_part = bindings[0].get().Cast(); + auto &constant_expr = bindings[1].get().Cast(); + auto &constant = constant_expr.value; + + if (constant.IsNull()) { + // NULL specifier: return constant NULL + return make_uniq(Value(date_part.return_type)); + } + // otherwise check the specifier + auto specifier = GetDatePartSpecifier(StringValue::Get(constant)); + string new_function_name; + switch (specifier) { + case DatePartSpecifier::YEAR: + new_function_name = "year"; + break; + case DatePartSpecifier::MONTH: + new_function_name = "month"; + break; + case DatePartSpecifier::DAY: + new_function_name = "day"; + break; + case DatePartSpecifier::DECADE: + new_function_name = "decade"; + break; + case DatePartSpecifier::CENTURY: + new_function_name = "century"; + break; + case DatePartSpecifier::MILLENNIUM: + new_function_name = "millennium"; + break; + case DatePartSpecifier::QUARTER: + new_function_name = "quarter"; + break; + case DatePartSpecifier::WEEK: + new_function_name = "week"; + break; + case DatePartSpecifier::YEARWEEK: + new_function_name = "yearweek"; + break; + case DatePartSpecifier::DOW: + new_function_name = "dayofweek"; + break; + case DatePartSpecifier::ISODOW: + new_function_name = "isodow"; + break; + case DatePartSpecifier::DOY: + new_function_name = "dayofyear"; + break; + case DatePartSpecifier::MICROSECONDS: + new_function_name = "microsecond"; + break; + case DatePartSpecifier::MILLISECONDS: + new_function_name = "millisecond"; + break; + case DatePartSpecifier::SECOND: + new_function_name = "second"; + break; + case DatePartSpecifier::MINUTE: + new_function_name = "minute"; + break; + case DatePartSpecifier::HOUR: + new_function_name = "hour"; + break; + default: + return nullptr; + } + // found a replacement function: bind it + vector> children; + children.push_back(std::move(date_part.children[1])); + + ErrorData error; + FunctionBinder binder(rewriter.context); + auto function = binder.BindScalarFunction(DEFAULT_SCHEMA, new_function_name, std::move(children), error, false); + if (!function) { + error.Throw(); + } + return function; +} + +} // namespace duckdb diff --git a/src/optimizer/rule/empty_needle_removal.cpp b/src/optimizer/rule/empty_needle_removal.cpp index 8b137891791f..a8985838b1d5 100644 --- a/src/optimizer/rule/empty_needle_removal.cpp +++ b/src/optimizer/rule/empty_needle_removal.cpp @@ -1 +1,55 @@ +#include "duckdb/optimizer/rule/empty_needle_removal.hpp" + +#include "duckdb/execution/expression_executor.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" +#include "duckdb/planner/expression/bound_constant_expression.hpp" +#include "duckdb/planner/expression/bound_operator_expression.hpp" +#include "duckdb/planner/expression/bound_case_expression.hpp" +#include "duckdb/optimizer/expression_rewriter.hpp" + +namespace duckdb { + +EmptyNeedleRemovalRule::EmptyNeedleRemovalRule(ExpressionRewriter &rewriter) : Rule(rewriter) { + // match on a FunctionExpression that has a foldable ConstantExpression + auto func = make_uniq(); + func->matchers.push_back(make_uniq()); + func->matchers.push_back(make_uniq()); + func->policy = SetMatcher::Policy::SOME; + + unordered_set functions = {"prefix", "contains", "suffix"}; + func->function = make_uniq(functions); + root = std::move(func); +} + +unique_ptr EmptyNeedleRemovalRule::Apply(LogicalOperator &op, vector> &bindings, + bool &changes_made, bool is_root) { + auto &root = bindings[0].get().Cast(); + D_ASSERT(root.children.size() == 2); + auto &prefix_expr = bindings[2].get(); + + // the constant_expr is a scalar expression that we have to fold + if (!prefix_expr.IsFoldable()) { + return nullptr; + } + D_ASSERT(root.return_type.id() == LogicalTypeId::BOOLEAN); + + auto prefix_value = ExpressionExecutor::EvaluateScalar(GetContext(), prefix_expr); + + if (prefix_value.IsNull()) { + return make_uniq(Value(LogicalType::BOOLEAN)); + } + + D_ASSERT(prefix_value.type() == prefix_expr.return_type); + auto &needle_string = StringValue::Get(prefix_value); + + // PREFIX('xyz', '') is TRUE + // PREFIX(NULL, '') is NULL + // so rewrite PREFIX(x, '') to TRUE_OR_NULL(x) + if (needle_string.empty()) { + return ExpressionRewriter::ConstantOrNull(std::move(root.children[0]), Value::BOOLEAN(true)); + } + return nullptr; +} + +} // namespace duckdb diff --git a/src/optimizer/rule/enum_comparison.cpp b/src/optimizer/rule/enum_comparison.cpp index 8b137891791f..aeb5e224800e 100644 --- a/src/optimizer/rule/enum_comparison.cpp +++ b/src/optimizer/rule/enum_comparison.cpp @@ -1 +1,71 @@ +#include "duckdb/optimizer/rule/enum_comparison.hpp" + +#include "duckdb/execution/expression_executor.hpp" +#include "duckdb/planner/expression/bound_comparison_expression.hpp" +#include "duckdb/planner/expression/bound_cast_expression.hpp" +#include "duckdb/optimizer/matcher/type_matcher_id.hpp" +#include "duckdb/optimizer/expression_rewriter.hpp" +#include "duckdb/common/types.hpp" + +namespace duckdb { + +EnumComparisonRule::EnumComparisonRule(ExpressionRewriter &rewriter) : Rule(rewriter) { + // match on a ComparisonExpression that is an Equality and has a VARCHAR and ENUM as its children + auto op = make_uniq(); + // Enum requires expression to be root + op->expr_type = make_uniq(ExpressionType::COMPARE_EQUAL); + for (idx_t i = 0; i < 2; i++) { + auto child = make_uniq(); + child->type = make_uniq(LogicalTypeId::VARCHAR); + child->matcher = make_uniq(); + child->matcher->type = make_uniq(LogicalTypeId::ENUM); + op->matchers.push_back(std::move(child)); + } + root = std::move(op); +} + +bool AreMatchesPossible(LogicalType &left, LogicalType &right) { + LogicalType *small_enum, *big_enum; + if (EnumType::GetSize(left) < EnumType::GetSize(right)) { + small_enum = &left; + big_enum = &right; + } else { + small_enum = &right; + big_enum = &left; + } + auto &string_vec = EnumType::GetValuesInsertOrder(*small_enum); + auto string_vec_ptr = FlatVector::GetData(string_vec); + auto size = EnumType::GetSize(*small_enum); + for (idx_t i = 0; i < size; i++) { + auto key = string_vec_ptr[i].GetString(); + if (EnumType::GetPos(*big_enum, key) != -1) { + return true; + } + } + return false; +} +unique_ptr EnumComparisonRule::Apply(LogicalOperator &op, vector> &bindings, + bool &changes_made, bool is_root) { + + auto &root = bindings[0].get().Cast(); + auto &left_child = bindings[1].get().Cast(); + auto &right_child = bindings[3].get().Cast(); + + if (!AreMatchesPossible(left_child.child->return_type, right_child.child->return_type)) { + vector> children; + children.push_back(std::move(root.left)); + children.push_back(std::move(root.right)); + return ExpressionRewriter::ConstantOrNull(std::move(children), Value::BOOLEAN(false)); + } + + if (!is_root || op.type != LogicalOperatorType::LOGICAL_FILTER) { + return nullptr; + } + + auto cast_left_to_right = + BoundCastExpression::AddDefaultCastToType(std::move(left_child.child), right_child.child->return_type, true); + return make_uniq(root.type, std::move(cast_left_to_right), std::move(right_child.child)); +} + +} // namespace duckdb diff --git a/src/optimizer/rule/in_clause_simplification_rule.cpp b/src/optimizer/rule/in_clause_simplification_rule.cpp index 8b137891791f..07e433773796 100644 --- a/src/optimizer/rule/in_clause_simplification_rule.cpp +++ b/src/optimizer/rule/in_clause_simplification_rule.cpp @@ -1 +1,58 @@ +#include "duckdb/execution/expression_executor.hpp" +#include "duckdb/optimizer/rule/in_clause_simplification.hpp" +#include "duckdb/planner/expression/list.hpp" +#include "duckdb/planner/expression/bound_operator_expression.hpp" + +namespace duckdb { + +InClauseSimplificationRule::InClauseSimplificationRule(ExpressionRewriter &rewriter) : Rule(rewriter) { + // match on InClauseExpression that has a ConstantExpression as a check + auto op = make_uniq(); + op->policy = SetMatcher::Policy::SOME; + root = std::move(op); +} + +unique_ptr InClauseSimplificationRule::Apply(LogicalOperator &op, vector> &bindings, + bool &changes_made, bool is_root) { + auto &expr = bindings[0].get().Cast(); + if (expr.children[0]->expression_class != ExpressionClass::BOUND_CAST) { + return nullptr; + } + auto &cast_expression = expr.children[0]->Cast(); + if (cast_expression.child->expression_class != ExpressionClass::BOUND_COLUMN_REF) { + return nullptr; + } + //! Here we check if we can apply the expression on the constant side + auto target_type = cast_expression.source_type(); + if (!BoundCastExpression::CastIsInvertible(cast_expression.return_type, target_type)) { + return nullptr; + } + vector> cast_list; + //! First check if we can cast all children + for (size_t i = 1; i < expr.children.size(); i++) { + if (expr.children[i]->expression_class != ExpressionClass::BOUND_CONSTANT) { + return nullptr; + } + D_ASSERT(expr.children[i]->IsFoldable()); + auto constant_value = ExpressionExecutor::EvaluateScalar(GetContext(), *expr.children[i]); + auto new_constant = constant_value.DefaultTryCastAs(target_type); + if (!new_constant) { + return nullptr; + } else { + auto new_constant_expr = make_uniq(constant_value); + cast_list.push_back(std::move(new_constant_expr)); + } + } + //! We can cast, so we move the new constant + for (size_t i = 1; i < expr.children.size(); i++) { + expr.children[i] = std::move(cast_list[i - 1]); + + // expr->children[i] = std::move(new_constant_expr); + } + //! We can cast the full list, so we move the column + expr.children[0] = std::move(cast_expression.child); + return nullptr; +} + +} // namespace duckdb diff --git a/src/optimizer/rule/like_optimizations.cpp b/src/optimizer/rule/like_optimizations.cpp index 8b137891791f..96f7b1501e8a 100644 --- a/src/optimizer/rule/like_optimizations.cpp +++ b/src/optimizer/rule/like_optimizations.cpp @@ -1 +1,162 @@ +#include "duckdb/optimizer/rule/like_optimizations.hpp" +#include "duckdb/execution/expression_executor.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" +#include "duckdb/planner/expression/bound_constant_expression.hpp" +#include "duckdb/planner/expression/bound_operator_expression.hpp" +#include "duckdb/planner/expression/bound_comparison_expression.hpp" + +namespace duckdb { + +LikeOptimizationRule::LikeOptimizationRule(ExpressionRewriter &rewriter) : Rule(rewriter) { + // match on a FunctionExpression that has a foldable ConstantExpression + auto func = make_uniq(); + func->matchers.push_back(make_uniq()); + func->matchers.push_back(make_uniq()); + func->policy = SetMatcher::Policy::ORDERED; + // we match on LIKE ("~~") and NOT LIKE ("!~~") + func->function = make_uniq(unordered_set {"!~~", "~~"}); + root = std::move(func); +} + +static bool PatternIsConstant(const string &pattern) { + for (idx_t i = 0; i < pattern.size(); i++) { + if (pattern[i] == '%' || pattern[i] == '_') { + return false; + } + } + return true; +} + +static bool PatternIsPrefix(const string &pattern) { + idx_t i; + for (i = pattern.size(); i > 0; i--) { + if (pattern[i - 1] != '%') { + break; + } + } + if (i == pattern.size()) { + // no trailing % + // cannot be a prefix + return false; + } + // continue to look in the string + // if there is a % or _ in the string (besides at the very end) this is not a prefix match + for (; i > 0; i--) { + if (pattern[i - 1] == '%' || pattern[i - 1] == '_') { + return false; + } + } + return true; +} + +static bool PatternIsSuffix(const string &pattern) { + idx_t i; + for (i = 0; i < pattern.size(); i++) { + if (pattern[i] != '%') { + break; + } + } + if (i == 0) { + // no leading % + // cannot be a suffix + return false; + } + // continue to look in the string + // if there is a % or _ in the string (besides at the beginning) this is not a suffix match + for (; i < pattern.size(); i++) { + if (pattern[i] == '%' || pattern[i] == '_') { + return false; + } + } + return true; +} + +static bool PatternIsContains(const string &pattern) { + idx_t start; + idx_t end; + for (start = 0; start < pattern.size(); start++) { + if (pattern[start] != '%') { + break; + } + } + for (end = pattern.size(); end > 0; end--) { + if (pattern[end - 1] != '%') { + break; + } + } + if (start == 0 || end == pattern.size()) { + // contains requires both a leading AND a trailing % + return false; + } + // check if there are any other special characters in the string + // if there is a % or _ in the string (besides at the beginning/end) this is not a contains match + for (idx_t i = start; i < end; i++) { + if (pattern[i] == '%' || pattern[i] == '_') { + return false; + } + } + return true; +} + +unique_ptr LikeOptimizationRule::Apply(LogicalOperator &op, vector> &bindings, + bool &changes_made, bool is_root) { + auto &root = bindings[0].get().Cast(); + auto &constant_expr = bindings[2].get().Cast(); + D_ASSERT(root.children.size() == 2); + + if (constant_expr.value.IsNull()) { + return make_uniq(Value(root.return_type)); + } + + // the constant_expr is a scalar expression that we have to fold + if (!constant_expr.IsFoldable()) { + return nullptr; + } + + auto constant_value = ExpressionExecutor::EvaluateScalar(GetContext(), constant_expr); + D_ASSERT(constant_value.type() == constant_expr.return_type); + auto &patt_str = StringValue::Get(constant_value); + + bool is_not_like = root.function.name == "!~~"; + if (PatternIsConstant(patt_str)) { + // Pattern is constant + return make_uniq(is_not_like ? ExpressionType::COMPARE_NOTEQUAL + : ExpressionType::COMPARE_EQUAL, + std::move(root.children[0]), std::move(root.children[1])); + } else if (PatternIsPrefix(patt_str)) { + // Prefix LIKE pattern : [^%_]*[%]+, ignoring underscore + return ApplyRule(root, PrefixFun::GetFunction(), patt_str, is_not_like); + } else if (PatternIsSuffix(patt_str)) { + // Suffix LIKE pattern: [%]+[^%_]*, ignoring underscore + return ApplyRule(root, SuffixFun::GetFunction(), patt_str, is_not_like); + } else if (PatternIsContains(patt_str)) { + // Contains LIKE pattern: [%]+[^%_]*[%]+, ignoring underscore + return ApplyRule(root, ContainsFun::GetFunction(), patt_str, is_not_like); + } + return nullptr; +} + +unique_ptr LikeOptimizationRule::ApplyRule(BoundFunctionExpression &expr, ScalarFunction function, + string pattern, bool is_not_like) { + // replace LIKE by an optimized function + unique_ptr result; + auto new_function = + make_uniq(expr.return_type, std::move(function), std::move(expr.children), nullptr); + + // removing "%" from the pattern + pattern.erase(std::remove(pattern.begin(), pattern.end(), '%'), pattern.end()); + + new_function->children[1] = make_uniq(Value(std::move(pattern))); + + result = std::move(new_function); + if (is_not_like) { + auto negation = make_uniq(ExpressionType::OPERATOR_NOT, LogicalType::BOOLEAN); + negation->children.push_back(std::move(result)); + result = std::move(negation); + } + + return result; +} + +} // namespace duckdb diff --git a/src/optimizer/rule/move_constants.cpp b/src/optimizer/rule/move_constants.cpp index 8b137891791f..636265ff9131 100644 --- a/src/optimizer/rule/move_constants.cpp +++ b/src/optimizer/rule/move_constants.cpp @@ -1 +1,165 @@ +#include "duckdb/optimizer/rule/move_constants.hpp" +#include "duckdb/common/exception.hpp" +#include "duckdb/common/value_operations/value_operations.hpp" +#include "duckdb/planner/expression/bound_comparison_expression.hpp" +#include "duckdb/planner/expression/bound_constant_expression.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" +#include "duckdb/optimizer/expression_rewriter.hpp" + +namespace duckdb { + +MoveConstantsRule::MoveConstantsRule(ExpressionRewriter &rewriter) : Rule(rewriter) { + auto op = make_uniq(); + op->matchers.push_back(make_uniq()); + op->policy = SetMatcher::Policy::UNORDERED; + + auto arithmetic = make_uniq(); + // we handle multiplication, addition and subtraction because those are "easy" + // integer division makes the division case difficult + // e.g. [x / 2 = 3] means [x = 6 OR x = 7] because of truncation -> no clean rewrite rules + arithmetic->function = make_uniq(unordered_set {"+", "-", "*"}); + // we match only on integral numeric types + arithmetic->type = make_uniq(); + auto child_constant_matcher = make_uniq(); + auto child_expression_matcher = make_uniq(); + child_constant_matcher->type = make_uniq(); + child_expression_matcher->type = make_uniq(); + arithmetic->matchers.push_back(std::move(child_constant_matcher)); + arithmetic->matchers.push_back(std::move(child_expression_matcher)); + arithmetic->policy = SetMatcher::Policy::SOME; + op->matchers.push_back(std::move(arithmetic)); + root = std::move(op); +} + +unique_ptr MoveConstantsRule::Apply(LogicalOperator &op, vector> &bindings, + bool &changes_made, bool is_root) { + auto &comparison = bindings[0].get().Cast(); + auto &outer_constant = bindings[1].get().Cast(); + auto &arithmetic = bindings[2].get().Cast(); + auto &inner_constant = bindings[3].get().Cast(); + D_ASSERT(arithmetic.return_type.IsIntegral()); + D_ASSERT(arithmetic.children[0]->return_type.IsIntegral()); + if (inner_constant.value.IsNull() || outer_constant.value.IsNull()) { + return make_uniq(Value(comparison.return_type)); + } + auto &constant_type = outer_constant.return_type; + hugeint_t outer_value = IntegralValue::Get(outer_constant.value); + hugeint_t inner_value = IntegralValue::Get(inner_constant.value); + + idx_t arithmetic_child_index = arithmetic.children[0].get() == &inner_constant ? 1 : 0; + auto &op_type = arithmetic.function.name; + if (op_type == "+") { + // [x + 1 COMP 10] OR [1 + x COMP 10] + // order does not matter in addition: + // simply change right side to 10-1 (outer_constant - inner_constant) + if (!Hugeint::TrySubtractInPlace(outer_value, inner_value)) { + return nullptr; + } + auto result_value = Value::HUGEINT(outer_value); + if (!result_value.DefaultTryCastAs(constant_type)) { + if (comparison.type != ExpressionType::COMPARE_EQUAL) { + return nullptr; + } + // if the cast is not possible then the comparison is not possible + // for example, if we have x + 5 = 3, where x is an unsigned number, we will get x = -2 + // since this is not possible we can remove the entire branch here + return ExpressionRewriter::ConstantOrNull(std::move(arithmetic.children[arithmetic_child_index]), + Value::BOOLEAN(false)); + } + outer_constant.value = std::move(result_value); + } else if (op_type == "-") { + // [x - 1 COMP 10] O R [1 - x COMP 10] + // order matters in subtraction: + if (arithmetic_child_index == 0) { + // [x - 1 COMP 10] + // change right side to 10+1 (outer_constant + inner_constant) + if (!Hugeint::TryAddInPlace(outer_value, inner_value)) { + return nullptr; + } + auto result_value = Value::HUGEINT(outer_value); + if (!result_value.DefaultTryCastAs(constant_type)) { + // if the cast is not possible then an equality comparison is not possible + if (comparison.type != ExpressionType::COMPARE_EQUAL) { + return nullptr; + } + return ExpressionRewriter::ConstantOrNull(std::move(arithmetic.children[arithmetic_child_index]), + Value::BOOLEAN(false)); + } + outer_constant.value = std::move(result_value); + } else { + // [1 - x COMP 10] + // change right side to 1-10=-9 + if (!Hugeint::TrySubtractInPlace(inner_value, outer_value)) { + return nullptr; + } + auto result_value = Value::HUGEINT(inner_value); + if (!result_value.DefaultTryCastAs(constant_type)) { + // if the cast is not possible then an equality comparison is not possible + if (comparison.type != ExpressionType::COMPARE_EQUAL) { + return nullptr; + } + return ExpressionRewriter::ConstantOrNull(std::move(arithmetic.children[arithmetic_child_index]), + Value::BOOLEAN(false)); + } + outer_constant.value = std::move(result_value); + // in this case, we should also flip the comparison + // e.g. if we have [4 - x < 2] then we should have [x > 2] + comparison.type = FlipComparisonExpression(comparison.type); + } + } else { + D_ASSERT(op_type == "*"); + // [x * 2 COMP 10] OR [2 * x COMP 10] + // order does not matter in multiplication: + // change right side to 10/2 (outer_constant / inner_constant) + // but ONLY if outer_constant is cleanly divisible by the inner_constant + if (inner_value == 0) { + // x * 0, the result is either 0 or NULL + // we let the arithmetic_simplification rule take care of simplifying this first + return nullptr; + } + // check out of range for HUGEINT or not cleanly divisible + // HUGEINT is not cleanly divisible when outer_value == minimum and inner value == -1. (modulo overflow) + if ((outer_value == NumericLimits::Minimum() && inner_value == -1) || + outer_value % inner_value != 0) { + bool is_equality = comparison.type == ExpressionType::COMPARE_EQUAL; + bool is_inequality = comparison.type == ExpressionType::COMPARE_NOTEQUAL; + if (is_equality || is_inequality) { + // we know the values are not equal + // the result will be either FALSE or NULL (if COMPARE_EQUAL) + // or TRUE or NULL (if COMPARE_NOTEQUAL) + return ExpressionRewriter::ConstantOrNull(std::move(arithmetic.children[arithmetic_child_index]), + Value::BOOLEAN(is_inequality)); + } else { + // not cleanly divisible and we are doing > >= < <=, skip the simplification for now + return nullptr; + } + } + if (inner_value < 0) { + // multiply by negative value, need to flip expression + comparison.type = FlipComparisonExpression(comparison.type); + } + // else divide the RHS by the LHS + // we need to do a range check on the cast even though we do a division + // because e.g. -128 / -1 = 128, which is out of range + auto result_value = Value::HUGEINT(outer_value / inner_value); + if (!result_value.DefaultTryCastAs(constant_type)) { + return ExpressionRewriter::ConstantOrNull(std::move(arithmetic.children[arithmetic_child_index]), + Value::BOOLEAN(false)); + } + outer_constant.value = std::move(result_value); + } + // replace left side with x + // first extract x from the arithmetic expression + auto arithmetic_child = std::move(arithmetic.children[arithmetic_child_index]); + // then place in the comparison + if (comparison.left.get() == &outer_constant) { + comparison.right = std::move(arithmetic_child); + } else { + comparison.left = std::move(arithmetic_child); + } + changes_made = true; + return nullptr; +} + +} // namespace duckdb diff --git a/src/planner/CMakeLists.txt b/src/planner/CMakeLists.txt index 8b137891791f..19f4c28a0758 100644 --- a/src/planner/CMakeLists.txt +++ b/src/planner/CMakeLists.txt @@ -1 +1,27 @@ +add_subdirectory(expression) +add_subdirectory(binder) +add_subdirectory(expression_binder) +add_subdirectory(filter) +add_subdirectory(operator) +add_subdirectory(subquery) +add_library_unity( + duckdb_planner + OBJECT + bound_result_modifier.cpp + bound_parameter_map.cpp + expression_iterator.cpp + expression.cpp + table_binding.cpp + expression_binder.cpp + joinside.cpp + logical_operator.cpp + binder.cpp + bind_context.cpp + planner.cpp + pragma_handler.cpp + logical_operator_visitor.cpp + table_filter.cpp) +set(ALL_OBJECT_FILES + ${ALL_OBJECT_FILES} $ + PARENT_SCOPE) From a08c250c010d41d9bfc2d4fa1e21228cef36bea9 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 4 Apr 2024 16:09:32 +0200 Subject: [PATCH 188/603] Allow (for debug) disabling metadata checks [only on unsigned mode] --- src/include/duckdb/main/config.hpp | 2 ++ src/include/duckdb/main/settings.hpp | 9 +++++++ src/main/config.cpp | 1 + src/main/extension/extension_load.cpp | 38 ++++++++++++++++++--------- src/main/settings/settings.cpp | 17 ++++++++++++ 5 files changed, 55 insertions(+), 12 deletions(-) diff --git a/src/include/duckdb/main/config.hpp b/src/include/duckdb/main/config.hpp index c90d43a7b347..1e20d144bd07 100644 --- a/src/include/duckdb/main/config.hpp +++ b/src/include/duckdb/main/config.hpp @@ -175,6 +175,8 @@ struct DBConfigOptions { string extension_directory; //! Whether unsigned extensions should be loaded bool allow_unsigned_extensions = false; + //! Whether extensions with missing metadata should be loaded + bool allow_extensions_metadata_mismatch = false; //! Enable emitting FSST Vectors bool enable_fsst_vectors = false; //! Start transactions immediately in all attached databases - instead of lazily when a database is referenced diff --git a/src/include/duckdb/main/settings.hpp b/src/include/duckdb/main/settings.hpp index a1c6560b2a39..f49e8c2314f8 100644 --- a/src/include/duckdb/main/settings.hpp +++ b/src/include/duckdb/main/settings.hpp @@ -225,6 +225,15 @@ struct AllowUnsignedExtensionsSetting { static Value GetSetting(ClientContext &context); }; +struct AllowExtensionsMetadataMismatchSetting { + static constexpr const char *Name = "allow_extensions_metadata_mismatch"; + static constexpr const char *Description = "Allow to load extensions with not compatible metadata"; + static constexpr const LogicalTypeId InputType = LogicalTypeId::BOOLEAN; + static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value ¶meter); + static void ResetGlobal(DatabaseInstance *db, DBConfig &config); + static Value GetSetting(ClientContext &context); +}; + struct AllowUnredactedSecretsSetting { static constexpr const char *Name = "allow_unredacted_secrets"; static constexpr const char *Description = "Allow printing unredacted secrets"; diff --git a/src/main/config.cpp b/src/main/config.cpp index dcf647b094b9..c1e86b96f46d 100644 --- a/src/main/config.cpp +++ b/src/main/config.cpp @@ -71,6 +71,7 @@ static const ConfigurationOption internal_options[] = { DUCKDB_GLOBAL(EnableExternalAccessSetting), DUCKDB_GLOBAL(EnableFSSTVectors), DUCKDB_GLOBAL(AllowUnsignedExtensionsSetting), + DUCKDB_GLOBAL(AllowExtensionsMetadataMismatchSetting), DUCKDB_GLOBAL(AllowUnredactedSecretsSetting), DUCKDB_GLOBAL(CustomExtensionRepository), DUCKDB_GLOBAL(AutoloadExtensionRepository), diff --git a/src/main/extension/extension_load.cpp b/src/main/extension/extension_load.cpp index afbaa73ee7cc..98a66bc4b19e 100644 --- a/src/main/extension/extension_load.cpp +++ b/src/main/extension/extension_load.cpp @@ -164,9 +164,10 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const str idx_t file_size = handle->GetFileSize(); if (file_size < 1024) { - throw InvalidInputException("Extension \"%s\" do not have metadata compatible with DuckDB loading it " - "(version %s, platform %s). File size in particular is %i lower than minimum threshold of 1024", - filename, engine_version, engine_platform, file_size); + throw InvalidInputException( + "Extension \"%s\" do not have metadata compatible with DuckDB loading it " + "(version %s, platform %s). File size in particular is %i lower than minimum threshold of 1024", + filename, engine_version, engine_platform, file_size); } auto metadata_offset = file_size - metadata_segment.size(); @@ -190,12 +191,17 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const str a[0] = '4'; if (strncmp(a, metadata_field[0].data(), 32) != 0) { // metadata do not looks right, add this to the error message - metadata_mismatch_error = "\n" + StringUtil::Format("Extension \"%s\" do not have metadata compatible with DuckDB " - "loading it (version %s, platform %s)", - filename, engine_version, engine_platform); + metadata_mismatch_error = + "\n" + StringUtil::Format("Extension \"%s\" do not have metadata compatible with DuckDB " + "loading it (version %s, platform %s)", + filename, engine_version, engine_platform); } else if (engine_version != extension_duckdb_version || engine_platform != extension_duckdb_platform) { - metadata_mismatch_error = "\n" + StringUtil::Format("Extension \"%s\" (version %s, platfrom %s) does not match DuckDB loading it (version %s, platform %s)", - filename, PrettyPrintString(extension_duckdb_version), PrettyPrintString(extension_duckdb_platform), engine_version, engine_platform); + metadata_mismatch_error = "\n" + StringUtil::Format("Extension \"%s\" (version %s, platfrom %s) does not " + "match DuckDB loading it (version %s, platform %s)", + filename, PrettyPrintString(extension_duckdb_version), + PrettyPrintString(extension_duckdb_platform), + engine_version, engine_platform); + } else { // All looks good } @@ -254,12 +260,20 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const str } } if (!any_valid) { - throw IOException(config.error_manager->FormatException(ErrorType::UNSIGNED_EXTENSION, filename) + metadata_mismatch_error); + throw IOException(config.error_manager->FormatException(ErrorType::UNSIGNED_EXTENSION, filename) + + metadata_mismatch_error); } - } - if (!metadata_mismatch_error.empty()) { - throw InvalidInputException(metadata_mismatch_error.substr(1)); + if (!metadata_mismatch_error.empty()) { + // Signed extensions perform the full check + throw InvalidInputException(metadata_mismatch_error.substr(1)); + } + } else if (!config.options.allow_extensions_metadata_mismatch) { + if (!metadata_mismatch_error.empty()) { + // Unsigned extensions AND configuration allowing metadata_mismatch_error, loading allowed, mainly for + // debugging purposes + throw InvalidInputException(metadata_mismatch_error.substr(1)); + } } auto number_metadata_fields = 3; diff --git a/src/main/settings/settings.cpp b/src/main/settings/settings.cpp index 3344b77dc017..13c9c74d7438 100644 --- a/src/main/settings/settings.cpp +++ b/src/main/settings/settings.cpp @@ -476,6 +476,23 @@ Value AllowUnsignedExtensionsSetting::GetSetting(ClientContext &context) { return Value::BOOLEAN(config.options.allow_unsigned_extensions); } +//===--------------------------------------------------------------------===// +// Allow Extensions Metadata Mismatch +//===--------------------------------------------------------------------===// +void AllowExtensionsMetadataMismatchSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) { + auto new_value = input.GetValue(); + config.options.allow_extensions_metadata_mismatch = new_value; +} + +void AllowExtensionsMetadataMismatchSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) { + config.options.allow_extensions_metadata_mismatch = DBConfig().options.allow_extensions_metadata_mismatch; +} + +Value AllowExtensionsMetadataMismatchSetting::GetSetting(ClientContext &context) { + auto &config = DBConfig::GetConfig(context); + return Value::BOOLEAN(config.options.allow_extensions_metadata_mismatch); +} + //===--------------------------------------------------------------------===// // Allow Unredacted Secrets //===--------------------------------------------------------------------===// From 48183adcee499b1f353b70110d3386c45eec00f5 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 4 Apr 2024 16:35:45 +0200 Subject: [PATCH 189/603] Add test on metadata checks --- test/sql/extensions/checked_load.test | 30 +++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 test/sql/extensions/checked_load.test diff --git a/test/sql/extensions/checked_load.test b/test/sql/extensions/checked_load.test new file mode 100644 index 000000000000..39f2d2b4d135 --- /dev/null +++ b/test/sql/extensions/checked_load.test @@ -0,0 +1,30 @@ +# name: test/sql/extensions/checked_load.test +# description: Test metadata checks on load +# group: [extensions] + +statement error +LOAD 'README.md'; +---- +Invalid Input Error: Extension "README.md" do not have metadata compatible with DuckDB loading it + +statement ok +SET allow_extensions_metadata_mismatch=true; + +# This is the error thrown by dlopen +statement error +LOAD 'data/csv/error/time.csv'; +---- +Error: Extension "data/csv/error/time.csv" could not be loaded + +statement error +LOAD 'data/csv/no_opt.csv'; +---- +File size in particular is 15 lower than minimum threshold of 1024 + +statement ok +SET allow_unsigned_extensions=false; + +statement error +LOAD 'README.md'; +---- +IO Error: Extension "README.md" could not be loaded because its signature is either missing or invalid and unsigned extensions are disabled by configuration From 38047b7c8d159dc077dff224c0c65549b8743fd2 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 4 Apr 2024 17:10:42 +0200 Subject: [PATCH 190/603] Fix test/api/test_reset.cpp --- test/api/test_reset.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/test/api/test_reset.cpp b/test/api/test_reset.cpp index 999530a30ae0..d220ac207046 100644 --- a/test/api/test_reset.cpp +++ b/test/api/test_reset.cpp @@ -82,6 +82,7 @@ OptionValueSet &GetValueForOption(const string &name) { {"file_search_path", {"test"}}, {"force_compression", {"uncompressed", "Uncompressed"}}, {"home_directory", {"test"}}, + {"allow_extensions_metadata_mismatch", {"true"}}, {"integer_division", {true}}, {"extension_directory", {"test"}}, {"immediate_transaction_mode", {true}}, From a8b861b06c24a28121fa2e72e9c2cb8fe4542139 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 4 Apr 2024 22:12:06 +0200 Subject: [PATCH 191/603] Extension metadata: fix tidy check --- src/main/extension/extension_load.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/extension/extension_load.cpp b/src/main/extension/extension_load.cpp index 98a66bc4b19e..20fc6647a309 100644 --- a/src/main/extension/extension_load.cpp +++ b/src/main/extension/extension_load.cpp @@ -75,15 +75,15 @@ static string PrettyPrintString(const string &s) { res += "\\x"; uint8_t first = value / 16; if (first < 10) { - res += '0' + first; + res += string('0' + first); } else { - res += 'a' + first - 10; + res += string('a' + first - 10); } uint8_t second = value % 16; if (second < 10) { - res += '0' + second; + res += string('0' + second); } else { - res += 'a' + second - 10; + res += string('a' + second - 10); } } } From b24468a610c22fd0a99f1f92e115a1981f728aa7 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 4 Apr 2024 22:16:21 +0200 Subject: [PATCH 192/603] Extension metadata: Fix windows path in test --- test/sql/extensions/checked_load.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/sql/extensions/checked_load.test b/test/sql/extensions/checked_load.test index 39f2d2b4d135..64dae601b973 100644 --- a/test/sql/extensions/checked_load.test +++ b/test/sql/extensions/checked_load.test @@ -12,9 +12,9 @@ SET allow_extensions_metadata_mismatch=true; # This is the error thrown by dlopen statement error -LOAD 'data/csv/error/time.csv'; +LOAD 'README.md'; ---- -Error: Extension "data/csv/error/time.csv" could not be loaded +Error: Extension "README.md" could not be loaded statement error LOAD 'data/csv/no_opt.csv'; From 12901a7293794949d98eadd6b9cffe03e4190c41 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 4 Apr 2024 22:20:16 +0200 Subject: [PATCH 193/603] Extension metadata: Fix tidy-check --- src/main/extension/extension_load.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/extension/extension_load.cpp b/src/main/extension/extension_load.cpp index 20fc6647a309..d1040f20f431 100644 --- a/src/main/extension/extension_load.cpp +++ b/src/main/extension/extension_load.cpp @@ -75,15 +75,15 @@ static string PrettyPrintString(const string &s) { res += "\\x"; uint8_t first = value / 16; if (first < 10) { - res += string('0' + first); + res.push_back((char)('0' + first)); } else { - res += string('a' + first - 10); + res.push_back((char)('a' + first - 10)); } uint8_t second = value % 16; if (second < 10) { - res += string('0' + second); + res.push_back((char)('0' + second)); } else { - res += string('a' + second - 10); + res.push_back((char)('a' + second - 10)); } } } From 56fb2052afba028b3023f468237c3cc09e0298ad Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 5 Apr 2024 06:14:35 +0200 Subject: [PATCH 194/603] Extension medatadata: Fix error message --- src/main/extension/extension_load.cpp | 4 ++-- test/sql/extensions/checked_load.test | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/extension/extension_load.cpp b/src/main/extension/extension_load.cpp index d1040f20f431..002cf4aefe15 100644 --- a/src/main/extension/extension_load.cpp +++ b/src/main/extension/extension_load.cpp @@ -166,8 +166,8 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const str if (file_size < 1024) { throw InvalidInputException( "Extension \"%s\" do not have metadata compatible with DuckDB loading it " - "(version %s, platform %s). File size in particular is %i lower than minimum threshold of 1024", - filename, engine_version, engine_platform, file_size); + "(version %s, platform %s). File size in particular is lower than minimum threshold of 1024", + filename, engine_version, engine_platform); } auto metadata_offset = file_size - metadata_segment.size(); diff --git a/test/sql/extensions/checked_load.test b/test/sql/extensions/checked_load.test index 64dae601b973..dcead8139aa0 100644 --- a/test/sql/extensions/checked_load.test +++ b/test/sql/extensions/checked_load.test @@ -19,7 +19,7 @@ Error: Extension "README.md" could not be loaded statement error LOAD 'data/csv/no_opt.csv'; ---- -File size in particular is 15 lower than minimum threshold of 1024 +File size in particular is lower than minimum threshold of 1024 statement ok SET allow_unsigned_extensions=false; From 5efa63eaaca2f7bb6cf1c1e9e22c8058ecb3a2ea Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 5 Apr 2024 10:55:38 +0200 Subject: [PATCH 195/603] make tidy happy --- .../execution/operator/persistent/csv_rejects_table.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp b/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp index d00aede6687a..88dd86377dc7 100644 --- a/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +++ b/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp @@ -1,5 +1,7 @@ #pragma once +#include + #include "duckdb/storage/object_cache.hpp" #include "duckdb/common/mutex.hpp" #include "duckdb/common/typedefs.hpp" @@ -15,7 +17,7 @@ class ClientContext; class CSVRejectsTable : public ObjectCacheEntry { public: CSVRejectsTable(string rejects_scan, string rejects_error) - : count(0), scan_table(rejects_scan), errors_table(rejects_error) { + : count(0), scan_table(std::move(rejects_scan)), errors_table(std::move(rejects_error)) { } mutex write_lock; string name; From 9c07c9318853a65ad36ca60f7184d9b3fc5ea332 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 5 Apr 2024 11:08:07 +0200 Subject: [PATCH 196/603] Make user_parameters ordered --- .../operator/csv_scanner/util/csv_reader_options.cpp | 6 +++++- .../execution/operator/csv_scanner/csv_reader_options.hpp | 1 + test/sql/copy/csv/rejects/csv_rejects_two_tables.test | 8 ++++---- test/sql/copy/csv/test_sniff_csv_options.test | 2 +- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp index ebfe1ae306bd..6ecdc72989ae 100644 --- a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp +++ b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp @@ -386,6 +386,7 @@ bool StoreUserDefinedParameter(string &option) { } void CSVReaderOptions::FromNamedParameters(named_parameter_map_t &in, ClientContext &context, vector &return_types, vector &names) { + map ordered_user_defined_parameters; for (auto &kv : in) { if (MultiFileReader::ParseOption(kv.first, kv.second, file_options, context)) { continue; @@ -393,7 +394,7 @@ void CSVReaderOptions::FromNamedParameters(named_parameter_map_t &in, ClientCont auto loption = StringUtil::Lower(kv.first); // skip variables that are specific to auto detection if (StoreUserDefinedParameter(loption)) { - user_defined_parameters += loption + "=" + kv.second.ToSQLString() + ", "; + ordered_user_defined_parameters[loption] = kv.second.ToSQLString(); } if (loption == "columns") { auto &child_type = kv.second.type(); @@ -499,6 +500,9 @@ void CSVReaderOptions::FromNamedParameters(named_parameter_map_t &in, ClientCont SetReadOption(loption, kv.second, names); } } + for (auto &udf_parameter : ordered_user_defined_parameters) { + user_defined_parameters += udf_parameter.first + "=" + udf_parameter.second + ", "; + } if (user_defined_parameters.size() >= 2) { user_defined_parameters.erase(user_defined_parameters.size() - 2); } diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp index b937ccfc11d9..53a66da77838 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp @@ -107,6 +107,7 @@ struct CSVReaderOptions { //! User defined parameters for the csv function concatenated on a string string user_defined_parameters; + //===--------------------------------------------------------------------===// // WriteCSVOptions //===--------------------------------------------------------------------===// diff --git a/test/sql/copy/csv/rejects/csv_rejects_two_tables.test b/test/sql/copy/csv/rejects/csv_rejects_two_tables.test index f50128989810..70fef75c473f 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_two_tables.test +++ b/test/sql/copy/csv/rejects/csv_rejects_two_tables.test @@ -19,8 +19,8 @@ BIGINT VARCHAR 11044 11044 2 query IIIIIIIIIIIII FROM reject_scans order by all; ---- -3 0 test/sql/copy/csv/data/error/mismatch/big_bad.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL store_rejects=true, sample_size=1 -3 1 test/sql/copy/csv/data/error/mismatch/big_bad2.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL store_rejects=true, sample_size=1 +3 0 test/sql/copy/csv/data/error/mismatch/big_bad.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL sample_size=1, store_rejects=true +3 1 test/sql/copy/csv/data/error/mismatch/big_bad2.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL sample_size=1, store_rejects=true query IIIIIIIIII FROM reject_errors order by all; @@ -106,8 +106,8 @@ query IIIIIIIIIIIII SELECT * FROM rejects_scan_3 order by all; ---- -15 0 test/sql/copy/csv/data/error/mismatch/big_bad.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_table='rejects_errors_3', rejects_scan='rejects_scan_3', sample_size=1 -15 1 test/sql/copy/csv/data/error/mismatch/big_bad2.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_table='rejects_errors_3', rejects_scan='rejects_scan_3', sample_size=1 +15 0 test/sql/copy/csv/data/error/mismatch/big_bad.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_scan='rejects_scan_3', rejects_table='rejects_errors_3', sample_size=1 +15 1 test/sql/copy/csv/data/error/mismatch/big_bad2.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_scan='rejects_scan_3', rejects_table='rejects_errors_3', sample_size=1 query IIIIIIIIII FROM rejects_errors_3 order by all; diff --git a/test/sql/copy/csv/test_sniff_csv_options.test b/test/sql/copy/csv/test_sniff_csv_options.test index 29144c0455bf..aa402aa58cb6 100644 --- a/test/sql/copy/csv/test_sniff_csv_options.test +++ b/test/sql/copy/csv/test_sniff_csv_options.test @@ -82,7 +82,7 @@ FROM sniff_csv('test/sql/copy/csv/data/auto/time_date_timestamp_yyyy.mm.dd.csv', query IIIIIIIIIII FROM sniff_csv('test/sql/copy/csv/data/auto/time_date_timestamp_yyyy.mm.dd.csv', dateformat='%Y.%m.%d', timestampformat='%Y.%m.%d %H:%M:%S') ---- -, " " \n 0 true {'a': 'BIGINT', 'b': 'VARCHAR', 't': 'TIME', 'd': 'DATE', 'ts': 'TIMESTAMP'} %Y.%m.%d %Y.%m.%d %H:%M:%S timestampformat='%Y.%m.%d %H:%M:%S', dateformat='%Y.%m.%d' FROM read_csv('test/sql/copy/csv/data/auto/time_date_timestamp_yyyy.mm.dd.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'a': 'BIGINT', 'b': 'VARCHAR', 't': 'TIME', 'd': 'DATE', 'ts': 'TIMESTAMP'}, timestampformat='%Y.%m.%d %H:%M:%S', dateformat='%Y.%m.%d'); +, " " \n 0 1 {'a': 'BIGINT', 'b': 'VARCHAR', 't': 'TIME', 'd': 'DATE', 'ts': 'TIMESTAMP'} %Y.%m.%d %Y.%m.%d %H:%M:%S dateformat='%Y.%m.%d', timestampformat='%Y.%m.%d %H:%M:%S' FROM read_csv('test/sql/copy/csv/data/auto/time_date_timestamp_yyyy.mm.dd.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'a': 'BIGINT', 'b': 'VARCHAR', 't': 'TIME', 'd': 'DATE', 'ts': 'TIMESTAMP'}, dateformat='%Y.%m.%d', timestampformat='%Y.%m.%d %H:%M:%S'); query IIIII FROM read_csv('test/sql/copy/csv/data/auto/time_date_timestamp_yyyy.mm.dd.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'a': 'BIGINT', 'b': 'VARCHAR', 't': 'TIME', 'd': 'DATE', 'ts': 'TIMESTAMP'}, timestampformat='%Y.%m.%d %H:%M:%S', dateformat='%Y.%m.%d') order by all limit 1; From dbff19eb9248bea017790c236ce5241fc30a1f98 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 5 Apr 2024 11:10:25 +0200 Subject: [PATCH 197/603] Apply @samansmink's remove_at_files.patch This clean-up CMake syntax and fixes a problem where empty strings would be conflated for no argument --- CMakeLists.txt | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 753129013cad..cdc52a43dfbd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -729,15 +729,9 @@ function(build_loadable_extension_directory NAME OUTPUT_DIRECTORY EXTENSION_VERS add_dependencies(duckdb_local_extension_repo ${NAME}_loadable_extension) endif() # all parameters after output_directory - set(FILES ${ARGV}) - # remove name - list(REMOVE_AT FILES 0) - # remove output_directory - list(REMOVE_AT FILES 0) - # remove extension_version - list(REMOVE_AT FILES 0) - # remove parameters - list(REMOVE_AT FILES 0) + set(FILES "${ARGV}") + # remove name, output_directory, extension_version, parameters + list(REMOVE_AT FILES 0 1 2 3) # parse parameters string(FIND "${PARAMETERS}" "-no-warnings" IGNORE_WARNINGS) @@ -840,9 +834,8 @@ endfunction() function(build_loadable_extension NAME PARAMETERS) # all parameters after name - set(FILES ${ARGV}) - list(REMOVE_AT FILES 0) - list(REMOVE_AT FILES 0) + set(FILES "${ARGV}") + list(REMOVE_AT FILES 0 1) string(TOUPPER ${NAME} EXTENSION_NAME_UPPERCASE) build_loadable_extension_directory(${NAME} "extension/${NAME}" "${DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG}" "${PARAMETERS}" ${FILES}) @@ -850,7 +843,7 @@ endfunction() function(build_static_extension NAME PARAMETERS) # all parameters after name - set(FILES ${ARGV}) + set(FILES "${ARGV}") list(REMOVE_AT FILES 0) add_library(${NAME}_extension STATIC ${FILES}) target_link_libraries(${NAME}_extension duckdb_static) From a1291a9880003c11ce4efe626f1654ea57416b26 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 5 Apr 2024 11:15:45 +0200 Subject: [PATCH 198/603] Extension metadata: fail more nicely on missing EXTENSION_VERSION --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cdc52a43dfbd..e8316c787bb1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -822,7 +822,7 @@ function(build_loadable_extension_directory NAME OUTPUT_DIRECTORY EXTENSION_VERS COMMAND printf "${DUCKDB_NORMALIZED_VERSION}" > duckdb_version_file COMMAND - printf "${EXTENSION_VERSION}" > extension_version_file + printf "${EXTENSION_VERSION}" > extension_version_file || (echo "CMake appears not to propagate correctly version information for the extension" && exit 1) COMMAND bash ${CMAKE_SOURCE_DIR}/scripts/append_metadata.sh $ ${CMAKE_BINARY_DIR}/duckdb_platform_out duckdb_version_file extension_version_file ) From e392ffc134cd17dd9bd928e0d84ee6f8223ed6ce Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 5 Apr 2024 11:16:55 +0200 Subject: [PATCH 199/603] Remove scan id from tests to make them more deterministic --- .../copy/csv/rejects/csv_rejects_auto.test | 12 +-- .../csv/rejects/csv_rejects_flush_cast.test | 8 +- .../csv/rejects/csv_rejects_maximum_line.test | 28 +++---- .../copy/csv/rejects/csv_rejects_read.test | 82 +++++++++---------- .../csv/rejects/csv_rejects_two_tables.test | 80 +++++++++--------- .../csv/rejects/test_invalid_utf_rejects.test | 12 +-- test/sql/copy/csv/rejects/test_mixed.test | 16 ++-- 7 files changed, 119 insertions(+), 119 deletions(-) diff --git a/test/sql/copy/csv/rejects/csv_rejects_auto.test b/test/sql/copy/csv/rejects/csv_rejects_auto.test index e673e9917287..15eecd053fc1 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_auto.test +++ b/test/sql/copy/csv/rejects/csv_rejects_auto.test @@ -15,13 +15,13 @@ SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), M ---- BIGINT VARCHAR 11044 11044 2 -query IIIIIIIIII rowsort -FROM reject_errors order by all; +query IIIIIIIII rowsort +SELECT * EXCLUDE (scan_id) FROM reject_errors order by all; ---- -3 0 2176 10876 10876 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' -3 0 4176 20876 20876 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' -3 1 3680 18396 18396 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' -3 1 5680 28396 28396 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' +0 2176 10876 10876 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +0 4176 20876 20876 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' +1 3680 18396 18396 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +1 5680 28396 28396 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' statement ok DROP TABLE reject_errors; diff --git a/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test b/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test index e6459aa5cd77..ba48d9fe2a99 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test +++ b/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test @@ -17,9 +17,9 @@ SELECT typeof(first(a)), typeof(first(b)), COUNT(*) FROM read_csv( ---- DATE VARCHAR 2811 -query IIIIIIIIII -FROM reject_errors order by all; +query IIIIIIIII +SELECT * EXCLUDE (scan_id) FROM reject_errors order by all; ---- -3 0 439 6997 NULL 1 a CAST B, bla Error when converting column "a". Could not parse string "B" according to format specifier "%d-%m-%Y" -3 0 2813 44972 NULL 1 a CAST c, bla Error when converting column "a". Could not parse string "c" according to format specifier "%d-%m-%Y" +0 439 6997 NULL 1 a CAST B, bla Error when converting column "a". Could not parse string "B" according to format specifier "%d-%m-%Y" +0 2813 44972 NULL 1 a CAST c, bla Error when converting column "a". Could not parse string "c" according to format specifier "%d-%m-%Y" diff --git a/test/sql/copy/csv/rejects/csv_rejects_maximum_line.test b/test/sql/copy/csv/rejects/csv_rejects_maximum_line.test index 99cedc820614..21ab80aacad5 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_maximum_line.test +++ b/test/sql/copy/csv/rejects/csv_rejects_maximum_line.test @@ -13,10 +13,10 @@ SELECT * FROM read_csv( columns = {'a': 'VARCHAR', 'b': 'INTEGER'}, store_rejects=true, auto_detect=false, header = 1, max_line_size=10); -query IIIIIIIIII -FROM reject_errors order by all; +query IIIIIIIII +SELECT * EXCLUDE (scan_id) FROM reject_errors order by all; ---- -3 0 5 23 23 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 Maximum line size of 10 bytes exceeded. Actual Size:19 bytes. +0 5 23 23 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 Maximum line size of 10 bytes exceeded. Actual Size:19 bytes. statement ok DROP TABLE reject_errors; @@ -54,12 +54,12 @@ SELECT * FROM read_csv( columns = {'a': 'VARCHAR', 'b': 'INTEGER'}, store_rejects = true, auto_detect=false, header = 1, max_line_size=20); -query IIIIIIIIII -FROM reject_errors order by all; +query IIIIIIIII +SELECT * EXCLUDE (scan_id) FROM reject_errors order by all; ---- -27 0 2282 13685 13685 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 Maximum line size of 20 bytes exceeded. Actual Size:25 bytes. -27 0 2591 15558 15558 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 Maximum line size of 20 bytes exceeded. Actual Size:25 bytes. -27 0 2923 17569 17569 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,3 Maximum line size of 20 bytes exceeded. Actual Size:25 bytes. +0 2282 13685 13685 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 Maximum line size of 20 bytes exceeded. Actual Size:25 bytes. +0 2591 15558 15558 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 Maximum line size of 20 bytes exceeded. Actual Size:25 bytes. +0 2923 17569 17569 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,3 Maximum line size of 20 bytes exceeded. Actual Size:25 bytes. statement ok DROP TABLE reject_errors; @@ -74,13 +74,13 @@ SELECT * FROM read_csv( columns = {'a': 'VARCHAR', 'b': 'INTEGER'}, store_rejects = true, auto_detect=false, header = 1, max_line_size=10); -query IIIIIIIIII -FROM reject_errors order by all; +query IIIIIIIII +SELECT * EXCLUDE (scan_id) FROM reject_errors order by all; ---- -31 0 5 23 23 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 Maximum line size of 10 bytes exceeded. Actual Size:19 bytes. -31 1 2282 13685 13685 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 Maximum line size of 10 bytes exceeded. Actual Size:25 bytes. -31 1 2591 15558 15558 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 Maximum line size of 10 bytes exceeded. Actual Size:25 bytes. -31 1 2923 17569 17569 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,3 Maximum line size of 10 bytes exceeded. Actual Size:25 bytes. +0 5 23 23 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaa,4 Maximum line size of 10 bytes exceeded. Actual Size:19 bytes. +1 2282 13685 13685 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 Maximum line size of 10 bytes exceeded. Actual Size:25 bytes. +1 2591 15558 15558 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,1 Maximum line size of 10 bytes exceeded. Actual Size:25 bytes. +1 2923 17569 17569 NULL NULL LINE SIZE OVER MAXIMUM blaaaaaaaaaaaaaaaaaaaa,3 Maximum line size of 10 bytes exceeded. Actual Size:25 bytes. statement ok DROP TABLE reject_errors; diff --git a/test/sql/copy/csv/rejects/csv_rejects_read.test b/test/sql/copy/csv/rejects/csv_rejects_read.test index b537833fd7dd..ba090366ac6e 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_read.test +++ b/test/sql/copy/csv/rejects/csv_rejects_read.test @@ -15,10 +15,10 @@ SELECT * FROM read_csv( 1 2 AAA 6 7 CCC -query IIIIIIIIII -FROM reject_errors; +query IIIIIIIII +SELECT * EXCLUDE (scan_id) FROM reject_errors; ---- -3 0 2 10 12 2 col1 CAST 4,BBB,9, Error when converting column "col1". Could not convert string "BBB" to 'INTEGER' +0 2 10 12 2 col1 CAST 4,BBB,9, Error when converting column "col1". Could not convert string "BBB" to 'INTEGER' statement ok DROP TABLE reject_errors; @@ -35,12 +35,12 @@ SELECT * FROM read_csv( ---- 4 5 9 -query IIIIIIIIII -FROM reject_errors ORDER BY ALL; +query IIIIIIIII +SELECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; ---- -7 0 1 1 5 3 col2 CAST 1,2,DDD, Error when converting column "col2". Could not convert string "DDD" to 'INTEGER' -7 0 3 17 17 1 col0 CAST EEE,7,FFF, Error when converting column "col0". Could not convert string "EEE" to 'INTEGER' -7 0 3 17 23 3 col2 CAST EEE,7,FFF, Error when converting column "col2". Could not convert string "FFF" to 'INTEGER' +0 1 1 5 3 col2 CAST 1,2,DDD, Error when converting column "col2". Could not convert string "DDD" to 'INTEGER' +0 3 17 17 1 col0 CAST EEE,7,FFF, Error when converting column "col0". Could not convert string "EEE" to 'INTEGER' +0 3 17 23 3 col2 CAST EEE,7,FFF, Error when converting column "col2". Could not convert string "FFF" to 'INTEGER' statement ok DROP TABLE reject_errors; @@ -60,11 +60,11 @@ SELECT * FROM read_csv( 4 5 9 6 7 CCC -query IIIIIIIIII -FROM reject_errors ORDER BY ALL; +query IIIIIIIII +SELECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; ---- -11 0 2 10 12 2 col1 CAST 4,BBB,9, Error when converting column "col1". Could not convert string "BBB" to 'INTEGER' -11 1 3 17 17 1 col0 CAST EEE,7,FFF, Error when converting column "col0". Could not convert string "EEE" to 'INTEGER' +0 2 10 12 2 col1 CAST 4,BBB,9, Error when converting column "col1". Could not convert string "BBB" to 'INTEGER' +1 3 17 17 1 col0 CAST EEE,7,FFF, Error when converting column "col0". Could not convert string "EEE" to 'INTEGER' statement ok DROP TABLE reject_errors; @@ -106,11 +106,11 @@ SELECT SUM(num) FROM read_csv( ---- 4270 -query IIIIIIIIII -FROM reject_errors ORDER BY ALL; +query IIIIIIIII +SELECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; ---- -19 0 2176 10876 10876 1 num CAST B, A Error when converting column "num". Could not convert string "B" to 'INTEGER' -19 0 4176 20876 20876 1 num CAST C, A Error when converting column "num". Could not convert string "C" to 'INTEGER' +0 2176 10876 10876 1 num CAST B, A Error when converting column "num". Could not convert string "B" to 'INTEGER' +0 4176 20876 20876 1 num CAST C, A Error when converting column "num". Could not convert string "C" to 'INTEGER' statement ok DROP TABLE reject_errors; @@ -126,11 +126,11 @@ SELECT SUM(num) FROM read_csv( ---- 6774 -query IIIIIIIIII -FROM reject_errors ORDER BY ALL; +query IIIIIIIII +SELECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; ---- -23 0 3680 18396 18396 1 num CAST B, A Error when converting column "num". Could not convert string "B" to 'INTEGER' -23 0 5680 28396 28396 1 num CAST C, A Error when converting column "num". Could not convert string "C" to 'INTEGER' +0 3680 18396 18396 1 num CAST B, A Error when converting column "num". Could not convert string "B" to 'INTEGER' +0 5680 28396 28396 1 num CAST C, A Error when converting column "num". Could not convert string "C" to 'INTEGER' statement ok @@ -148,13 +148,13 @@ SELECT SUM(num) FROM read_csv( ---- 11044 -query IIIIIIIIII -FROM reject_errors ORDER BY ALL; +query IIIIIIIII +SELECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; ---- -27 0 2176 10876 10876 1 num CAST B, A Error when converting column "num". Could not convert string "B" to 'INTEGER' -27 0 4176 20876 20876 1 num CAST C, A Error when converting column "num". Could not convert string "C" to 'INTEGER' -27 1 3680 18396 18396 1 num CAST B, A Error when converting column "num". Could not convert string "B" to 'INTEGER' -27 1 5680 28396 28396 1 num CAST C, A Error when converting column "num". Could not convert string "C" to 'INTEGER' +0 2176 10876 10876 1 num CAST B, A Error when converting column "num". Could not convert string "B" to 'INTEGER' +0 4176 20876 20876 1 num CAST C, A Error when converting column "num". Could not convert string "C" to 'INTEGER' +1 3680 18396 18396 1 num CAST B, A Error when converting column "num". Could not convert string "B" to 'INTEGER' +1 5680 28396 28396 1 num CAST C, A Error when converting column "num". Could not convert string "C" to 'INTEGER' statement ok DROP TABLE reject_errors; @@ -178,20 +178,20 @@ ON L.num = R.num; 1 A 1 A 3 C 3 C -query IIIIIIIIIIIII -FROM reject_scans ORDER BY ALL; +query IIIIIIIIIIII +SELECT * EXCLUDE (scan_id) FROM reject_scans ORDER BY ALL; ---- -31 0 test/sql/copy/csv/data/error/mismatch/small1.csv , " " \n 0 true {'num': 'INTEGER','str': 'VARCHAR'} NULL NULL store_rejects=true -31 1 test/sql/copy/csv/data/error/mismatch/small2.csv , " " \n 0 true {'num': 'INTEGER','str': 'VARCHAR'} NULL NULL store_rejects=true +0 test/sql/copy/csv/data/error/mismatch/small1.csv , " " \n 0 true {'num': 'INTEGER','str': 'VARCHAR'} NULL NULL store_rejects=true +1 test/sql/copy/csv/data/error/mismatch/small2.csv , " " \n 0 true {'num': 'INTEGER','str': 'VARCHAR'} NULL NULL store_rejects=true -query IIIIIIIIII -FROM reject_errors ORDER BY ALL; +query IIIIIIIII +SELECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; ---- -31 0 3 15 15 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER' -31 0 6 27 27 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER' -31 1 3 15 15 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER' -31 1 5 23 23 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER' +0 3 15 15 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER' +0 6 27 27 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER' +1 3 15 15 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER' +1 5 23 23 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER' statement ok DROP TABLE reject_errors; @@ -217,12 +217,12 @@ ON L.num = R.num; 3 C 3 C -query IIIIIIIIII -FROM reject_errors ORDER BY ALL; +query IIIIIIIII +SELECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; ---- -36 0 3 15 15 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER' -36 0 6 27 27 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER' -36 1 3 15 15 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER' +0 3 15 15 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER' +0 6 27 27 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER' +1 3 15 15 1 num CAST X,Y Error when converting column "num". Could not convert string "X" to 'INTEGER' statement ok diff --git a/test/sql/copy/csv/rejects/csv_rejects_two_tables.test b/test/sql/copy/csv/rejects/csv_rejects_two_tables.test index 70fef75c473f..f856d929fa2e 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_two_tables.test +++ b/test/sql/copy/csv/rejects/csv_rejects_two_tables.test @@ -16,19 +16,19 @@ SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), M BIGINT VARCHAR 11044 11044 2 -query IIIIIIIIIIIII -FROM reject_scans order by all; +query IIIIIIIIIIII +SELECT * EXCLUDE (scan_id) FROM reject_scans order by all; ---- -3 0 test/sql/copy/csv/data/error/mismatch/big_bad.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL sample_size=1, store_rejects=true -3 1 test/sql/copy/csv/data/error/mismatch/big_bad2.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL sample_size=1, store_rejects=true +0 test/sql/copy/csv/data/error/mismatch/big_bad.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL sample_size=1, store_rejects=true +1 test/sql/copy/csv/data/error/mismatch/big_bad2.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL sample_size=1, store_rejects=true -query IIIIIIIIII -FROM reject_errors order by all; +query IIIIIIIII +SELECT * EXCLUDE (scan_id) FROM reject_errors order by all; ---- -3 0 2176 10876 10876 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' -3 0 4176 20876 20876 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' -3 1 3680 18396 18396 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' -3 1 5680 28396 28396 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' +0 2176 10876 10876 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +0 4176 20876 20876 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' +1 3680 18396 18396 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +1 5680 28396 28396 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' # Test giving the name of errors table statement error @@ -51,19 +51,19 @@ SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), M ---- BIGINT VARCHAR 11044 11044 2 -query IIIIIIIIIIIII -FROM reject_scans order by all; +query IIIIIIIIIIII +SELECT * EXCLUDE (scan_id) FROM reject_scans order by all; ---- -8 0 test/sql/copy/csv/data/error/mismatch/big_bad.csv , \0 \0 \n 0 false {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_table='rejects_errors_2', sample_size=1 -8 1 test/sql/copy/csv/data/error/mismatch/big_bad2.csv , \0 \0 \n 0 false {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_table='rejects_errors_2', sample_size=1 +0 test/sql/copy/csv/data/error/mismatch/big_bad.csv , \0 \0 \n 0 false {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_table='rejects_errors_2', sample_size=1 +1 test/sql/copy/csv/data/error/mismatch/big_bad2.csv , \0 \0 \n 0 false {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_table='rejects_errors_2', sample_size=1 -query IIIIIIIIII -FROM rejects_errors_2 order by all; +query IIIIIIIII +SELECT * EXCLUDE (scan_id) FROM rejects_errors_2 order by all; ---- -8 0 2176 10876 10876 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' -8 0 4176 20876 20876 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' -8 1 3680 18396 18396 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' -8 1 5680 28396 28396 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' +0 2176 10876 10876 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +0 4176 20876 20876 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' +1 3680 18396 18396 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +1 5680 28396 28396 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' statement ok drop table reject_errors; @@ -77,19 +77,19 @@ SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), M ---- BIGINT VARCHAR 11044 11044 2 -query IIIIIIIIIIIII -FROM rejects_scan_2 order by all; +query IIIIIIIIIIII +SELECT * EXCLUDE (scan_id) FROM rejects_scan_2 order by all; ---- -12 0 test/sql/copy/csv/data/error/mismatch/big_bad.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_scan='rejects_scan_2', sample_size=1 -12 1 test/sql/copy/csv/data/error/mismatch/big_bad2.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_scan='rejects_scan_2', sample_size=1 +0 test/sql/copy/csv/data/error/mismatch/big_bad.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_scan='rejects_scan_2', sample_size=1 +1 test/sql/copy/csv/data/error/mismatch/big_bad2.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_scan='rejects_scan_2', sample_size=1 -query IIIIIIIIII -FROM reject_errors order by all; +query IIIIIIIII +SELECT * EXCLUDE (scan_id) FROM reject_errors order by all; ---- -12 0 2176 10876 10876 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' -12 0 4176 20876 20876 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' -12 1 3680 18396 18396 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' -12 1 5680 28396 28396 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' +0 2176 10876 10876 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +0 4176 20876 20876 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' +1 3680 18396 18396 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +1 5680 28396 28396 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' # Test giving the name of both tables query IIIII @@ -102,20 +102,20 @@ SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), M ---- BIGINT VARCHAR 11044 11044 2 -query IIIIIIIIIIIII -SELECT * +query IIIIIIIIIIII +SELECT * EXCLUDE (scan_id) FROM rejects_scan_3 order by all; ---- -15 0 test/sql/copy/csv/data/error/mismatch/big_bad.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_scan='rejects_scan_3', rejects_table='rejects_errors_3', sample_size=1 -15 1 test/sql/copy/csv/data/error/mismatch/big_bad2.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_scan='rejects_scan_3', rejects_table='rejects_errors_3', sample_size=1 +0 test/sql/copy/csv/data/error/mismatch/big_bad.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_scan='rejects_scan_3', rejects_table='rejects_errors_3', sample_size=1 +1 test/sql/copy/csv/data/error/mismatch/big_bad2.csv , \0 \0 \n 0 0 {'column0': 'BIGINT','column1': 'VARCHAR'} NULL NULL rejects_scan='rejects_scan_3', rejects_table='rejects_errors_3', sample_size=1 -query IIIIIIIIII -FROM rejects_errors_3 order by all; +query IIIIIIIII +SELECT * EXCLUDE (scan_id) FROM rejects_errors_3 order by all; ---- -15 0 2176 10876 10876 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' -15 0 4176 20876 20876 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' -15 1 3680 18396 18396 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' -15 1 5680 28396 28396 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' +0 2176 10876 10876 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +0 4176 20876 20876 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' +1 3680 18396 18396 1 column0 CAST B, A Error when converting column "column0". Could not convert string "B" to 'BIGINT' +1 5680 28396 28396 1 column0 CAST C, A Error when converting column "column0". Could not convert string "C" to 'BIGINT' statement ok drop table reject_errors; diff --git a/test/sql/copy/csv/rejects/test_invalid_utf_rejects.test b/test/sql/copy/csv/rejects/test_invalid_utf_rejects.test index e579648f8794..f6b9840b2130 100644 --- a/test/sql/copy/csv/rejects/test_invalid_utf_rejects.test +++ b/test/sql/copy/csv/rejects/test_invalid_utf_rejects.test @@ -11,10 +11,10 @@ statement ok from read_csv('test/sql/copy/csv/data/test/invalid_utf_big.csv',columns = {'col1': 'VARCHAR','col2': 'VARCHAR','col3': 'VARCHAR'}, auto_detect=false, header = 0, delim = ',', store_rejects=true) -query IIIIIIIIII rowsort -FROM reject_errors ORDER BY ALL; +query IIIIIIIII rowsort +SELECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; ---- -3 0 3001 54001 54007 2 col2 INVALID UNICODE valid,invalid_??_part,valid Invalid unicode (byte sequence mismatch) detected. -3 0 3012 54209 54221 3 col3 INVALID UNICODE valid,valid,invalid_??_part Invalid unicode (byte sequence mismatch) detected. -3 0 3023 54417 54423 2 col2 INVALID UNICODE valid,invalid_??_part,valid Invalid unicode (byte sequence mismatch) detected. -3 0 3034 54625 54637 3 col3 INVALID UNICODE valid,valid,invalid_??_part Invalid unicode (byte sequence mismatch) detected. \ No newline at end of file +0 3001 54001 54007 2 col2 INVALID UNICODE valid,invalid_??_part,valid Invalid unicode (byte sequence mismatch) detected. +0 3012 54209 54221 3 col3 INVALID UNICODE valid,valid,invalid_??_part Invalid unicode (byte sequence mismatch) detected. +0 3023 54417 54423 2 col2 INVALID UNICODE valid,invalid_??_part,valid Invalid unicode (byte sequence mismatch) detected. +0 3034 54625 54637 3 col3 INVALID UNICODE valid,valid,invalid_??_part Invalid unicode (byte sequence mismatch) detected. \ No newline at end of file diff --git a/test/sql/copy/csv/rejects/test_mixed.test b/test/sql/copy/csv/rejects/test_mixed.test index d1f9b1decedc..54ef879556ab 100644 --- a/test/sql/copy/csv/rejects/test_mixed.test +++ b/test/sql/copy/csv/rejects/test_mixed.test @@ -55,12 +55,12 @@ SELECT * FROM read_csv( 1 2 pedro 1 2 pedro -query IIIIIIIIII rowsort -FROM reject_errors ORDER BY ALL; +query IIIIIIIII rowsort +SELECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; ---- -3 0 10 103 106 2 c MISSING COLUMNS 1,2 Expected Number of Columns: 3 Found: 2 -3 0 14 143 154 4 NULL TOO MANY COLUMNS 1,2,"pedro",5 Expected Number of Columns: 3 Found: 4 -3 0 19 205 207 2 b CAST 1,bla,"pedro" Error when converting column "b". Could not convert string "bla" to 'INTEGER' -3 0 22 243 247 3 c UNQUOTED VALUE 1,2,"pedro"bla Value with unterminated quote found. -3 0 32 366 366 NULL NULL LINE SIZE OVER MAXIMUM 1,2,"pedro thiago timbo holanda" Maximum line size of 20 bytes exceeded. Actual Size:33 bytes. -3 0 38 459 463 3 c INVALID UNICODE 1,2,"pedro??" Invalid unicode (byte sequence mismatch) detected. \ No newline at end of file +0 10 103 106 2 c MISSING COLUMNS 1,2 Expected Number of Columns: 3 Found: 2 +0 14 143 154 4 NULL TOO MANY COLUMNS 1,2,"pedro",5 Expected Number of Columns: 3 Found: 4 +0 19 205 207 2 b CAST 1,bla,"pedro" Error when converting column "b". Could not convert string "bla" to 'INTEGER' +0 22 243 247 3 c UNQUOTED VALUE 1,2,"pedro"bla Value with unterminated quote found. +0 32 366 366 NULL NULL LINE SIZE OVER MAXIMUM 1,2,"pedro thiago timbo holanda" Maximum line size of 20 bytes exceeded. Actual Size:33 bytes. +0 38 459 463 3 c INVALID UNICODE 1,2,"pedro??" Invalid unicode (byte sequence mismatch) detected. \ No newline at end of file From 8e5ebeffbafcbb947ee973c96368f726bd21304b Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 5 Apr 2024 11:27:58 +0200 Subject: [PATCH 200/603] Letss see if our CI can take this --- test/sql/copy/csv/test_multiple_big_compressed_csvs.test_slow | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/sql/copy/csv/test_multiple_big_compressed_csvs.test_slow b/test/sql/copy/csv/test_multiple_big_compressed_csvs.test_slow index 07b99f83c601..e53ef286a495 100644 --- a/test/sql/copy/csv/test_multiple_big_compressed_csvs.test_slow +++ b/test/sql/copy/csv/test_multiple_big_compressed_csvs.test_slow @@ -13,9 +13,6 @@ copy lineitem to '__TEST_DIR__/lineitem.csv.gz'; statement ok SET temp_directory='' -# load the DB from disk (Avoids OOM when generating ze table) -load __TEST_DIR__/lineitem_compressed.db - statement ok CREATE TABLE lineitem_2(l_orderkey INTEGER NOT NULL, l_partkey INTEGER NOT NULL, l_suppkey INTEGER NOT NULL, l_linenumber INTEGER NOT NULL, l_quantity DECIMAL(15,2) NOT NULL, l_extendedprice DECIMAL(15,2) NOT NULL, l_discount DECIMAL(15,2) NOT NULL, l_tax DECIMAL(15,2) NOT NULL, l_returnflag VARCHAR NOT NULL, l_linestatus VARCHAR NOT NULL, l_shipdate DATE NOT NULL, l_commitdate DATE NOT NULL, l_receiptdate DATE NOT NULL, l_shipinstruct VARCHAR NOT NULL, l_shipmode VARCHAR NOT NULL, l_comment VARCHAR NOT NULL); From d5874e18c43e01676db6e7f267b0ad2a7be2687a Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 5 Apr 2024 12:39:42 +0200 Subject: [PATCH 201/603] append_metadata.sh: Properly remove temp files, fix comment --- scripts/append_metadata.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/append_metadata.sh b/scripts/append_metadata.sh index beb47ccabd98..774872259ba4 100755 --- a/scripts/append_metadata.sh +++ b/scripts/append_metadata.sh @@ -32,7 +32,7 @@ do cat "$1.empty_32" >> "$1" done -# Write provided fiedls (backwards) +# Write provided fields (backwards) for ((i=$#; i>=2; i--)) do cat "${!i}" > "$1.add" @@ -49,4 +49,4 @@ done cat "$1.empty_256" >> "$1" -rm -f "$1.*" +rm -f $1.* From 2a166bd61452b42abcbbe858d0d79f6203034f81 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 5 Apr 2024 12:41:32 +0200 Subject: [PATCH 202/603] remove logic incorrectly copied from ViewCatalogEntry --- src/catalog/catalog_entry/index_catalog_entry.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/catalog/catalog_entry/index_catalog_entry.cpp b/src/catalog/catalog_entry/index_catalog_entry.cpp index 4cbf5ca86b1f..9090922218bc 100644 --- a/src/catalog/catalog_entry/index_catalog_entry.cpp +++ b/src/catalog/catalog_entry/index_catalog_entry.cpp @@ -35,10 +35,6 @@ unique_ptr IndexCatalogEntry::GetInfo() const { } string IndexCatalogEntry::ToSQL() const { - if (sql.empty()) { - //! Return empty sql with view name so pragma view_tables don't complain - return sql; - } auto info = GetInfo(); auto result = info->ToString(); return result + ";\n"; From c14e21b77bbb1249b39c630428955b83a43f1b43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Fri, 5 Apr 2024 13:24:06 +0200 Subject: [PATCH 203/603] unfunking tests --- src/common/operator/cast_operators.cpp | 4 +- src/common/row_operations/row_aggregate.cpp | 4 +- src/common/string_util.cpp | 4 +- src/common/types/conflict_manager.cpp | 2 +- src/common/types/hugeint.cpp | 8 +-- .../numeric_inplace_operators.cpp | 2 +- src/core_functions/scalar/bit/bitstring.cpp | 27 +++++--- .../scalar/blob/create_sort_key.cpp | 4 +- src/core_functions/scalar/date/date_part.cpp | 4 +- .../scalar/list/array_slice.cpp | 22 +++--- src/core_functions/scalar/string/hex.cpp | 29 ++++---- .../operator/join/physical_iejoin.cpp | 2 +- src/execution/perfect_aggregate_hashtable.cpp | 8 +-- src/function/scalar/list/list_extract.cpp | 4 +- src/function/scalar/list/list_select.cpp | 3 +- src/function/scalar/strftime_format.cpp | 67 +++++++++---------- .../duckdb/common/sort/duckdb_pdqsort.hpp | 2 +- src/include/duckdb/common/types/bit.hpp | 2 +- .../duckdb/common/types/cast_helpers.hpp | 4 +- .../duckdb/storage/string_uncompressed.hpp | 2 +- src/main/capi/result-c.cpp | 2 +- .../transform/helpers/transform_typename.cpp | 4 +- .../buffer/buffer_pool_reservation.cpp | 4 +- 23 files changed, 106 insertions(+), 108 deletions(-) diff --git a/src/common/operator/cast_operators.cpp b/src/common/operator/cast_operators.cpp index be59aabbe296..769ff78cdaf7 100644 --- a/src/common/operator/cast_operators.cpp +++ b/src/common/operator/cast_operators.cpp @@ -1647,7 +1647,7 @@ struct HugeIntegerCastOperation { template static bool HandleDigit(T &state, uint8_t digit) { if (NEGATIVE) { - if (DUCKDB_UNLIKELY(UnsafeNumericCast(state.intermediate) < + if (DUCKDB_UNLIKELY(static_cast(state.intermediate) < (NumericLimits::Minimum() + digit) / 10)) { // intermediate is full: need to flush it if (!state.Flush()) { @@ -1702,7 +1702,7 @@ struct HugeIntegerCastOperation { remainder = negate_result; } state.decimal = remainder; - state.decimal_total_digits = UnsafeNumericCast(-e); + state.decimal_total_digits = static_cast(-e); state.decimal_intermediate = 0; state.decimal_intermediate_digits = 0; return Finalize(state); diff --git a/src/common/row_operations/row_aggregate.cpp b/src/common/row_operations/row_aggregate.cpp index 8be0ada4fa20..0ea80035e08d 100644 --- a/src/common/row_operations/row_aggregate.cpp +++ b/src/common/row_operations/row_aggregate.cpp @@ -95,8 +95,8 @@ void RowOperations::CombineStates(RowOperationsState &state, TupleDataLayout &la } // Now subtract the offset to get back to the original position - VectorOperations::AddInPlace(sources, UnsafeNumericCast(-offset), count); - VectorOperations::AddInPlace(targets, UnsafeNumericCast(-offset), count); + VectorOperations::AddInPlace(sources, -UnsafeNumericCast(offset), count); + VectorOperations::AddInPlace(targets, -UnsafeNumericCast(offset), count); } void RowOperations::FinalizeStates(RowOperationsState &state, TupleDataLayout &layout, Vector &addresses, diff --git a/src/common/string_util.cpp b/src/common/string_util.cpp index 7d33b1cd80f7..2e9eb676e527 100644 --- a/src/common/string_util.cpp +++ b/src/common/string_util.cpp @@ -204,7 +204,7 @@ string StringUtil::Upper(const string &str) { string StringUtil::Lower(const string &str) { string copy(str); transform(copy.begin(), copy.end(), copy.begin(), - [](unsigned char c) { return StringUtil::CharacterToLower(UnsafeNumericCast(c)); }); + [](unsigned char c) { return StringUtil::CharacterToLower(static_cast(c)); }); return (copy); } @@ -216,7 +216,7 @@ bool StringUtil::IsLower(const string &str) { uint64_t StringUtil::CIHash(const string &str) { uint32_t hash = 0; for (auto c : str) { - hash += UnsafeNumericCast(StringUtil::CharacterToLower(UnsafeNumericCast(c))); + hash += static_cast(StringUtil::CharacterToLower(static_cast(c))); hash += hash << 10; hash ^= hash >> 6; } diff --git a/src/common/types/conflict_manager.cpp b/src/common/types/conflict_manager.cpp index 171d45115fb4..8e7ce0b9bb93 100644 --- a/src/common/types/conflict_manager.cpp +++ b/src/common/types/conflict_manager.cpp @@ -159,7 +159,7 @@ bool ConflictManager::AddNull(idx_t chunk_index) { if (!IsConflict(LookupResultType::LOOKUP_NULL)) { return false; } - return AddHit(chunk_index, UnsafeNumericCast(DConstants::INVALID_INDEX)); + return AddHit(chunk_index, static_cast(DConstants::INVALID_INDEX)); } bool ConflictManager::SingleIndexTarget() const { diff --git a/src/common/types/hugeint.cpp b/src/common/types/hugeint.cpp index c99abaa9ee1e..d83d81caa13b 100644 --- a/src/common/types/hugeint.cpp +++ b/src/common/types/hugeint.cpp @@ -830,14 +830,14 @@ hugeint_t hugeint_t::operator>>(const hugeint_t &rhs) const { return *this; } else if (shift == 64) { result.upper = (upper < 0) ? -1 : 0; - result.lower = UnsafeNumericCast(upper); + result.lower = uint64_t(upper); } else if (shift < 64) { // perform lower shift in unsigned integer, and mask away the most significant bit result.lower = (uint64_t(upper) << (64 - shift)) | (lower >> shift); result.upper = upper >> shift; } else { D_ASSERT(shift < 128); - result.lower = UnsafeNumericCast(upper >> (shift - 64)); + result.lower = uint64_t(upper >> (shift - 64)); result.upper = (upper < 0) ? -1 : 0; } return result; @@ -852,7 +852,7 @@ hugeint_t hugeint_t::operator<<(const hugeint_t &rhs) const { if (rhs.upper != 0 || shift >= 128) { return hugeint_t(0); } else if (shift == 64) { - result.upper = UnsafeNumericCast(lower); + result.upper = int64_t(lower); result.lower = 0; } else if (shift == 0) { return *this; @@ -860,7 +860,7 @@ hugeint_t hugeint_t::operator<<(const hugeint_t &rhs) const { // perform upper shift in unsigned integer, and mask away the most significant bit uint64_t upper_shift = ((uint64_t(upper) << shift) + (lower >> (64 - shift))) & 0x7FFFFFFFFFFFFFFF; result.lower = lower << shift; - result.upper = UnsafeNumericCast(upper_shift); + result.upper = int64_t(upper_shift); } else { D_ASSERT(shift < 128); result.lower = 0; diff --git a/src/common/vector_operations/numeric_inplace_operators.cpp b/src/common/vector_operations/numeric_inplace_operators.cpp index 86b507a3ce9a..863f3ba8dd6f 100644 --- a/src/common/vector_operations/numeric_inplace_operators.cpp +++ b/src/common/vector_operations/numeric_inplace_operators.cpp @@ -30,7 +30,7 @@ void VectorOperations::AddInPlace(Vector &input, int64_t right, idx_t count) { D_ASSERT(input.GetVectorType() == VectorType::FLAT_VECTOR); auto data = FlatVector::GetData(input); for (idx_t i = 0; i < count; i++) { - data[i] += UnsafeNumericCast(right); + data[i] = UnsafeNumericCast(UnsafeNumericCast(data[i]) + right); } break; } diff --git a/src/core_functions/scalar/bit/bitstring.cpp b/src/core_functions/scalar/bit/bitstring.cpp index 8a3074250139..babfadfe01e7 100644 --- a/src/core_functions/scalar/bit/bitstring.cpp +++ b/src/core_functions/scalar/bit/bitstring.cpp @@ -8,13 +8,17 @@ namespace duckdb { // BitStringFunction //===--------------------------------------------------------------------===// static void BitStringFunction(DataChunk &args, ExpressionState &state, Vector &result) { - BinaryExecutor::Execute( - args.data[0], args.data[1], result, args.size(), [&](string_t input, idx_t n) { - if (n < input.GetSize()) { + BinaryExecutor::Execute( + args.data[0], args.data[1], result, args.size(), [&](string_t input, int32_t n) { + if (n < 0) { + throw InvalidInputException("The bitstring length cannot be negative"); + } + if (idx_t(n) < input.GetSize()) { throw InvalidInputException("Length must be equal or larger than input string"); } idx_t len; Bit::TryGetBitStringSize(input, len, nullptr); // string verification + len = Bit::ComputeBitstringLen(n); string_t target = StringVector::EmptyString(result, len); Bit::BitString(input, n, target); @@ -24,7 +28,7 @@ static void BitStringFunction(DataChunk &args, ExpressionState &state, Vector &r } ScalarFunction BitStringFun::GetFunction() { - return ScalarFunction({LogicalType::VARCHAR, LogicalType::UBIGINT}, LogicalType::BIT, BitStringFunction); + return ScalarFunction({LogicalType::VARCHAR, LogicalType::INTEGER}, LogicalType::BIT, BitStringFunction); } //===--------------------------------------------------------------------===// @@ -33,7 +37,7 @@ ScalarFunction BitStringFun::GetFunction() { struct GetBitOperator { template static inline TR Operation(TA input, TB n) { - if (n > Bit::BitLength(input) - 1) { + if (n < 0 || (idx_t)n > Bit::BitLength(input) - 1) { throw OutOfRangeException("bit index %s out of valid range (0..%s)", NumericHelper::ToString(n), NumericHelper::ToString(Bit::BitLength(input) - 1)); } @@ -42,20 +46,21 @@ struct GetBitOperator { }; ScalarFunction GetBitFun::GetFunction() { - return ScalarFunction({LogicalType::BIT, LogicalType::UBIGINT}, LogicalType::INTEGER, - ScalarFunction::BinaryFunction); + return ScalarFunction({LogicalType::BIT, LogicalType::INTEGER}, LogicalType::INTEGER, + ScalarFunction::BinaryFunction); } //===--------------------------------------------------------------------===// // set_bit //===--------------------------------------------------------------------===// static void SetBitOperation(DataChunk &args, ExpressionState &state, Vector &result) { - TernaryExecutor::Execute( - args.data[0], args.data[1], args.data[2], result, args.size(), [&](string_t input, idx_t n, idx_t new_value) { + TernaryExecutor::Execute( + args.data[0], args.data[1], args.data[2], result, args.size(), + [&](string_t input, int32_t n, int32_t new_value) { if (new_value != 0 && new_value != 1) { throw InvalidInputException("The new bit must be 1 or 0"); } - if (n > Bit::BitLength(input) - 1) { + if (n < 0 || (idx_t)n > Bit::BitLength(input) - 1) { throw OutOfRangeException("bit index %s out of valid range (0..%s)", NumericHelper::ToString(n), NumericHelper::ToString(Bit::BitLength(input) - 1)); } @@ -67,7 +72,7 @@ static void SetBitOperation(DataChunk &args, ExpressionState &state, Vector &res } ScalarFunction SetBitFun::GetFunction() { - return ScalarFunction({LogicalType::BIT, LogicalType::UBIGINT, LogicalType::UBIGINT}, LogicalType::BIT, + return ScalarFunction({LogicalType::BIT, LogicalType::INTEGER, LogicalType::INTEGER}, LogicalType::BIT, SetBitOperation); } diff --git a/src/core_functions/scalar/blob/create_sort_key.cpp b/src/core_functions/scalar/blob/create_sort_key.cpp index 09ee36f5167e..e47986674271 100644 --- a/src/core_functions/scalar/blob/create_sort_key.cpp +++ b/src/core_functions/scalar/blob/create_sort_key.cpp @@ -519,8 +519,8 @@ void ConstructSortKeyList(SortKeyVectorData &vector_data, SortKeyChunk chunk, So } // write the end-of-list delimiter - result_ptr[offset++] = UnsafeNumericCast(info.flip_bytes ? ~SortKeyVectorData::LIST_DELIMITER - : SortKeyVectorData::LIST_DELIMITER); + result_ptr[offset++] = static_cast(info.flip_bytes ? ~SortKeyVectorData::LIST_DELIMITER + : SortKeyVectorData::LIST_DELIMITER); } } diff --git a/src/core_functions/scalar/date/date_part.cpp b/src/core_functions/scalar/date/date_part.cpp index ebbe158b51fd..3d0b75775ceb 100644 --- a/src/core_functions/scalar/date/date_part.cpp +++ b/src/core_functions/scalar/date/date_part.cpp @@ -1432,8 +1432,8 @@ void DatePart::StructOperator::Operation(bigint_vec &bigint_values, double_vec & // Both define epoch, and the correct value is the sum. // So mask it out and compute it separately. - Operation(bigint_values, double_values, d, idx, mask & UnsafeNumericCast(~EPOCH)); - Operation(bigint_values, double_values, t, idx, mask & UnsafeNumericCast(~EPOCH)); + Operation(bigint_values, double_values, d, idx, mask & ~UnsafeNumericCast(EPOCH)); + Operation(bigint_values, double_values, t, idx, mask & ~UnsafeNumericCast(EPOCH)); if (mask & EPOCH) { auto part_data = HasPartValue(double_values, DatePartSpecifier::EPOCH); diff --git a/src/core_functions/scalar/list/array_slice.cpp b/src/core_functions/scalar/list/array_slice.cpp index c91247ae9f69..7651f410d023 100644 --- a/src/core_functions/scalar/list/array_slice.cpp +++ b/src/core_functions/scalar/list/array_slice.cpp @@ -40,7 +40,7 @@ unique_ptr ListSliceBindData::Copy() const { } template -static idx_t CalculateSliceLength(INDEX_TYPE begin, INDEX_TYPE end, INDEX_TYPE step, bool svalid) { +static idx_t CalculateSliceLength(idx_t begin, idx_t end, INDEX_TYPE step, bool svalid) { if (step < 0) { step = abs(step); } @@ -48,14 +48,14 @@ static idx_t CalculateSliceLength(INDEX_TYPE begin, INDEX_TYPE end, INDEX_TYPE s throw InvalidInputException("Slice step cannot be zero"); } if (step == 1) { - return UnsafeNumericCast(end - begin); - } else if (step >= (end - begin)) { + return NumericCast(end - begin); + } else if (static_cast(step) >= (end - begin)) { return 1; } if ((end - begin) % step != 0) { - return UnsafeNumericCast((end - begin) / step + 1); + return (end - begin) / step + 1; } - return UnsafeNumericCast((end - begin) / step); + return (end - begin) / step; } template @@ -64,7 +64,7 @@ INDEX_TYPE ValueLength(const INPUT_TYPE &value) { } template <> -idx_t ValueLength(const list_entry_t &value) { +int64_t ValueLength(const list_entry_t &value) { return value.length; } @@ -119,8 +119,8 @@ INPUT_TYPE SliceValue(Vector &result, INPUT_TYPE input, INDEX_TYPE begin, INDEX_ template <> list_entry_t SliceValue(Vector &result, list_entry_t input, int64_t begin, int64_t end) { - input.offset = UnsafeNumericCast(UnsafeNumericCast(input.offset) + begin); - input.length = UnsafeNumericCast(end - begin); + input.offset += begin; + input.length = end - begin; return input; } @@ -145,14 +145,14 @@ list_entry_t SliceValueWithSteps(Vector &result, SelectionVector &sel, list_entr return input; } input.length = CalculateSliceLength(begin, end, step, true); - auto child_idx = UnsafeNumericCast(UnsafeNumericCast(input.offset) + begin); + idx_t child_idx = input.offset + begin; if (step < 0) { - child_idx = UnsafeNumericCast(UnsafeNumericCast(input.offset) + end - 1); + child_idx = input.offset + end - 1; } input.offset = sel_idx; for (idx_t i = 0; i < input.length; i++) { sel.set_index(sel_idx, child_idx); - child_idx = UnsafeNumericCast(UnsafeNumericCast(child_idx) + step); + child_idx += step; sel_idx++; } return input; diff --git a/src/core_functions/scalar/string/hex.cpp b/src/core_functions/scalar/string/hex.cpp index 6afaeadb56a4..dffbae70d030 100644 --- a/src/core_functions/scalar/string/hex.cpp +++ b/src/core_functions/scalar/string/hex.cpp @@ -90,8 +90,7 @@ struct HexIntegralOperator { template static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) { - auto num_leading_zero = - UnsafeNumericCast(CountZeros::Leading(UnsafeNumericCast(input))); + idx_t num_leading_zero = CountZeros::Leading(input); idx_t num_bits_to_check = 64 - num_leading_zero; D_ASSERT(num_bits_to_check <= sizeof(INPUT_TYPE) * 8); @@ -110,7 +109,7 @@ struct HexIntegralOperator { auto target = StringVector::EmptyString(result, buffer_size); auto output = target.GetDataWriteable(); - WriteHexBytes(UnsafeNumericCast(input), output, buffer_size); + WriteHexBytes(input, output, buffer_size); target.Finalize(); return target; @@ -121,7 +120,7 @@ struct HexHugeIntOperator { template static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) { - auto num_leading_zero = UnsafeNumericCast(CountZeros::Leading(input)); + idx_t num_leading_zero = CountZeros::Leading(input); idx_t buffer_size = sizeof(INPUT_TYPE) * 2 - (num_leading_zero / 4); // Special case: All bits are zero @@ -148,7 +147,7 @@ struct HexUhugeIntOperator { template static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) { - auto num_leading_zero = UnsafeNumericCast(CountZeros::Leading(input)); + idx_t num_leading_zero = CountZeros::Leading(input); idx_t buffer_size = sizeof(INPUT_TYPE) * 2 - (num_leading_zero / 4); // Special case: All bits are zero @@ -190,7 +189,7 @@ struct BinaryStrOperator { auto output = target.GetDataWriteable(); for (idx_t i = 0; i < size; ++i) { - auto byte = UnsafeNumericCast(data[i]); + uint8_t byte = data[i]; for (idx_t i = 8; i >= 1; --i) { *output = ((byte >> (i - 1)) & 0x01) + '0'; output++; @@ -206,8 +205,7 @@ struct BinaryIntegralOperator { template static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) { - auto num_leading_zero = - UnsafeNumericCast(CountZeros::Leading(UnsafeNumericCast(input))); + idx_t num_leading_zero = CountZeros::Leading(input); idx_t num_bits_to_check = 64 - num_leading_zero; D_ASSERT(num_bits_to_check <= sizeof(INPUT_TYPE) * 8); @@ -226,7 +224,7 @@ struct BinaryIntegralOperator { auto target = StringVector::EmptyString(result, buffer_size); auto output = target.GetDataWriteable(); - WriteBinBytes(UnsafeNumericCast(input), output, buffer_size); + WriteBinBytes(input, output, buffer_size); target.Finalize(); return target; @@ -236,7 +234,7 @@ struct BinaryIntegralOperator { struct BinaryHugeIntOperator { template static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) { - auto num_leading_zero = UnsafeNumericCast(CountZeros::Leading(input)); + idx_t num_leading_zero = CountZeros::Leading(input); idx_t buffer_size = sizeof(INPUT_TYPE) * 8 - num_leading_zero; // Special case: All bits are zero @@ -261,7 +259,7 @@ struct BinaryHugeIntOperator { struct BinaryUhugeIntOperator { template static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) { - auto num_leading_zero = UnsafeNumericCast(CountZeros::Leading(input)); + idx_t num_leading_zero = CountZeros::Leading(input); idx_t buffer_size = sizeof(INPUT_TYPE) * 8 - num_leading_zero; // Special case: All bits are zero @@ -303,7 +301,7 @@ struct FromHexOperator { // Treated as a single byte idx_t i = 0; if (size % 2 != 0) { - *output = UnsafeNumericCast(StringUtil::GetHexValue(data[i])); + *output = StringUtil::GetHexValue(data[i]); i++; output++; } @@ -311,7 +309,7 @@ struct FromHexOperator { for (; i < size; i += 2) { uint8_t major = StringUtil::GetHexValue(data[i]); uint8_t minor = StringUtil::GetHexValue(data[i + 1]); - *output = UnsafeNumericCast((major << 4) | minor); + *output = UnsafeNumericCast((major << 4) | minor); output++; } @@ -345,7 +343,7 @@ struct FromBinaryOperator { byte |= StringUtil::GetBinaryValue(data[i]) << (j - 1); i++; } - *output = UnsafeNumericCast(byte); + *output = byte; output++; } @@ -355,8 +353,7 @@ struct FromBinaryOperator { byte |= StringUtil::GetBinaryValue(data[i]) << (j - 1); i++; } - *output = UnsafeNumericCast(byte); - ; + *output = byte; output++; } diff --git a/src/execution/operator/join/physical_iejoin.cpp b/src/execution/operator/join/physical_iejoin.cpp index bc99e9fbf0a8..8c89195b1989 100644 --- a/src/execution/operator/join/physical_iejoin.cpp +++ b/src/execution/operator/join/physical_iejoin.cpp @@ -325,7 +325,7 @@ idx_t IEJoinUnion::AppendKey(SortedTable &table, ExpressionExecutor &executor, S payload.data[0].Sequence(rid, increment, scan_count); payload.SetCardinality(scan_count); keys.Fuse(payload); - rid += UnsafeNumericCast(increment) * scan_count; + rid += increment * UnsafeNumericCast(scan_count); // Sort on the sort columns (which will no longer be needed) keys.Split(payload, payload_idx); diff --git a/src/execution/perfect_aggregate_hashtable.cpp b/src/execution/perfect_aggregate_hashtable.cpp index a46e9499b5a3..da7c20192452 100644 --- a/src/execution/perfect_aggregate_hashtable.cpp +++ b/src/execution/perfect_aggregate_hashtable.cpp @@ -64,7 +64,7 @@ static void ComputeGroupLocationTemplated(UnifiedVectorFormat &group_data, Value // we only need to handle non-null values here if (group_data.validity.RowIsValid(index)) { D_ASSERT(data[index] >= min_val); - auto adjusted_value = UnsafeNumericCast((data[index] - min_val) + 1); + uintptr_t adjusted_value = (data[index] - min_val) + 1; address_data[i] += adjusted_value << current_shift; } } @@ -72,7 +72,7 @@ static void ComputeGroupLocationTemplated(UnifiedVectorFormat &group_data, Value // no null values: we can directly compute the addresses for (idx_t i = 0; i < count; i++) { auto index = group_data.sel->get_index(i); - auto adjusted_value = UnsafeNumericCast((data[index] - min_val) + 1); + uintptr_t adjusted_value = (data[index] - min_val) + 1; address_data[i] += adjusted_value << current_shift; } } @@ -149,7 +149,7 @@ void PerfectAggregateHashTable::AddChunk(DataChunk &groups, DataChunk &payload) } // move to the next aggregate payload_idx += input_count; - VectorOperations::AddInPlace(addresses, NumericCast(aggregate.payload_size), payload.size()); + VectorOperations::AddInPlace(addresses, aggregate.payload_size, payload.size()); } } @@ -199,7 +199,7 @@ static void ReconstructGroupVectorTemplated(uint32_t group_values[], Value &min, auto min_data = min.GetValueUnsafe(); for (idx_t i = 0; i < entry_count; i++) { // extract the value of this group from the total group index - auto group_index = UnsafeNumericCast((group_values[i] >> shift) & mask); + auto group_index = UnsafeNumericCast((group_values[i] >> shift) & mask); if (group_index == 0) { // if it is 0, the value is NULL validity_mask.SetInvalid(i); diff --git a/src/function/scalar/list/list_extract.cpp b/src/function/scalar/list/list_extract.cpp index e1566641aafb..822b9079cdc9 100644 --- a/src/function/scalar/list/list_extract.cpp +++ b/src/function/scalar/list/list_extract.cpp @@ -62,13 +62,13 @@ void ListExtractTemplate(idx_t count, UnifiedVectorFormat &list_data, UnifiedVec result_mask.SetInvalid(i); continue; } - child_offset = list_entry.offset + list_entry.length + UnsafeNumericCast(offsets_entry); + child_offset = list_entry.offset + list_entry.length + offsets_entry; } else { if ((idx_t)offsets_entry >= list_entry.length) { result_mask.SetInvalid(i); continue; } - child_offset = list_entry.offset + UnsafeNumericCast(offsets_entry); + child_offset = list_entry.offset + offsets_entry; } auto child_index = child_format.sel->get_index(child_offset); if (child_format.validity.RowIsValid(child_index)) { diff --git a/src/function/scalar/list/list_select.cpp b/src/function/scalar/list/list_select.cpp index 9d45de5a1ecc..70ae15194fcf 100644 --- a/src/function/scalar/list/list_select.cpp +++ b/src/function/scalar/list/list_select.cpp @@ -11,8 +11,7 @@ struct SetSelectionVectorSelect { ValidityMask &input_validity, Vector &selection_entry, idx_t child_idx, idx_t &target_offset, idx_t selection_offset, idx_t input_offset, idx_t target_length) { - auto sel_idx = - UnsafeNumericCast(selection_entry.GetValue(selection_offset + child_idx).GetValue() - 1); + auto sel_idx = selection_entry.GetValue(selection_offset + child_idx).GetValue() - 1; if (sel_idx < target_length) { selection_vector.set_index(target_offset, input_offset + sel_idx); if (!input_validity.RowIsValid(input_offset + sel_idx)) { diff --git a/src/function/scalar/strftime_format.cpp b/src/function/scalar/strftime_format.cpp index e8d31f522b6f..5181b005ed45 100644 --- a/src/function/scalar/strftime_format.cpp +++ b/src/function/scalar/strftime_format.cpp @@ -80,7 +80,7 @@ idx_t StrfTimeFormat::GetSpecifierLength(StrTimeSpecifier specifier, date_t date if (0 <= year && year <= 9999) { return 4; } else { - return UnsafeNumericCast(NumericHelper::SignedLength(year)); + return NumericHelper::SignedLength(year); } } case StrTimeSpecifier::MONTH_DECIMAL: { @@ -129,14 +129,11 @@ idx_t StrfTimeFormat::GetSpecifierLength(StrTimeSpecifier specifier, date_t date return len; } case StrTimeSpecifier::DAY_OF_MONTH: - return UnsafeNumericCast( - NumericHelper::UnsignedLength(UnsafeNumericCast(Date::ExtractDay(date)))); + return NumericHelper::UnsignedLength(Date::ExtractDay(date)); case StrTimeSpecifier::DAY_OF_YEAR_DECIMAL: - return UnsafeNumericCast( - NumericHelper::UnsignedLength(UnsafeNumericCast(Date::ExtractDayOfTheYear(date)))); + return NumericHelper::UnsignedLength(Date::ExtractDayOfTheYear(date)); case StrTimeSpecifier::YEAR_WITHOUT_CENTURY: - return UnsafeNumericCast(NumericHelper::UnsignedLength( - AbsValue(UnsafeNumericCast(Date::ExtractYear(date)) % 100))); + return NumericHelper::UnsignedLength(AbsValue(Date::ExtractYear(date)) % 100); default: throw InternalException("Unimplemented specifier for GetSpecifierLength"); } @@ -198,13 +195,13 @@ char *StrfTimeFormat::WritePadded(char *target, uint32_t value, size_t padding) D_ASSERT(padding > 1); if (padding % 2) { int decimals = value % 1000; - WritePadded3(target + padding - 3, UnsafeNumericCast(decimals)); + WritePadded3(target + padding - 3, decimals); value /= 1000; padding -= 3; } for (size_t i = 0; i < padding / 2; i++) { int decimals = value % 100; - WritePadded2(target + padding - 2 * (i + 1), UnsafeNumericCast(decimals)); + WritePadded2(target + padding - 2 * (i + 1), decimals); value /= 100; } return target + padding; @@ -248,26 +245,26 @@ char *StrfTimeFormat::WriteDateSpecifier(StrTimeSpecifier specifier, date_t date } case StrTimeSpecifier::DAY_OF_YEAR_PADDED: { int32_t doy = Date::ExtractDayOfTheYear(date); - target = WritePadded3(target, UnsafeNumericCast(doy)); + target = WritePadded3(target, doy); break; } case StrTimeSpecifier::WEEK_NUMBER_PADDED_MON_FIRST: - target = WritePadded2(target, UnsafeNumericCast(Date::ExtractWeekNumberRegular(date, true))); + target = WritePadded2(target, Date::ExtractWeekNumberRegular(date, true)); break; case StrTimeSpecifier::WEEK_NUMBER_PADDED_SUN_FIRST: - target = WritePadded2(target, UnsafeNumericCast(Date::ExtractWeekNumberRegular(date, false))); + target = WritePadded2(target, Date::ExtractWeekNumberRegular(date, false)); break; case StrTimeSpecifier::WEEK_NUMBER_ISO: - target = WritePadded2(target, UnsafeNumericCast(Date::ExtractISOWeekNumber(date))); + target = WritePadded2(target, Date::ExtractISOWeekNumber(date)); break; case StrTimeSpecifier::DAY_OF_YEAR_DECIMAL: { - auto doy = UnsafeNumericCast(Date::ExtractDayOfTheYear(date)); + uint32_t doy = Date::ExtractDayOfTheYear(date); target += NumericHelper::UnsignedLength(doy); NumericHelper::FormatUnsigned(doy, target); break; } case StrTimeSpecifier::YEAR_ISO: - target = WritePadded(target, UnsafeNumericCast(Date::ExtractISOYearNumber(date)), 4); + target = WritePadded(target, Date::ExtractISOYearNumber(date), 4); break; case StrTimeSpecifier::WEEKDAY_ISO: *target = char('0' + uint8_t(Date::ExtractISODayOfTheWeek(date))); @@ -284,7 +281,7 @@ char *StrfTimeFormat::WriteStandardSpecifier(StrTimeSpecifier specifier, int32_t // data contains [0] year, [1] month, [2] day, [3] hour, [4] minute, [5] second, [6] msec, [7] utc switch (specifier) { case StrTimeSpecifier::DAY_OF_MONTH_PADDED: - target = WritePadded2(target, UnsafeNumericCast(data[2])); + target = WritePadded2(target, data[2]); break; case StrTimeSpecifier::ABBREVIATED_MONTH_NAME: { auto &month_name = Date::MONTH_NAMES_ABBREVIATED[data[1] - 1]; @@ -295,14 +292,14 @@ char *StrfTimeFormat::WriteStandardSpecifier(StrTimeSpecifier specifier, int32_t return WriteString(target, month_name); } case StrTimeSpecifier::MONTH_DECIMAL_PADDED: - target = WritePadded2(target, UnsafeNumericCast(data[1])); + target = WritePadded2(target, data[1]); break; case StrTimeSpecifier::YEAR_WITHOUT_CENTURY_PADDED: - target = WritePadded2(target, UnsafeNumericCast(AbsValue(data[0]) % 100)); + target = WritePadded2(target, AbsValue(data[0]) % 100); break; case StrTimeSpecifier::YEAR_DECIMAL: if (data[0] >= 0 && data[0] <= 9999) { - target = WritePadded(target, UnsafeNumericCast(data[0]), 4); + target = WritePadded(target, data[0], 4); } else { int32_t year = data[0]; if (data[0] < 0) { @@ -310,13 +307,13 @@ char *StrfTimeFormat::WriteStandardSpecifier(StrTimeSpecifier specifier, int32_t year = -year; target++; } - auto len = NumericHelper::UnsignedLength(UnsafeNumericCast(year)); + auto len = NumericHelper::UnsignedLength(year); NumericHelper::FormatUnsigned(year, target + len); target += len; } break; case StrTimeSpecifier::HOUR_24_PADDED: { - target = WritePadded2(target, UnsafeNumericCast(data[3])); + target = WritePadded2(target, data[3]); break; } case StrTimeSpecifier::HOUR_12_PADDED: { @@ -324,7 +321,7 @@ char *StrfTimeFormat::WriteStandardSpecifier(StrTimeSpecifier specifier, int32_t if (hour == 0) { hour = 12; } - target = WritePadded2(target, UnsafeNumericCast(hour)); + target = WritePadded2(target, hour); break; } case StrTimeSpecifier::AM_PM: @@ -332,20 +329,20 @@ char *StrfTimeFormat::WriteStandardSpecifier(StrTimeSpecifier specifier, int32_t *target++ = 'M'; break; case StrTimeSpecifier::MINUTE_PADDED: { - target = WritePadded2(target, UnsafeNumericCast(data[4])); + target = WritePadded2(target, data[4]); break; } case StrTimeSpecifier::SECOND_PADDED: - target = WritePadded2(target, UnsafeNumericCast(data[5])); + target = WritePadded2(target, data[5]); break; case StrTimeSpecifier::NANOSECOND_PADDED: - target = WritePadded(target, UnsafeNumericCast(data[6] * Interval::NANOS_PER_MICRO), 9); + target = WritePadded(target, data[6] * Interval::NANOS_PER_MICRO, 9); break; case StrTimeSpecifier::MICROSECOND_PADDED: - target = WritePadded(target, UnsafeNumericCast(data[6]), 6); + target = WritePadded(target, data[6], 6); break; case StrTimeSpecifier::MILLISECOND_PADDED: - target = WritePadded3(target, UnsafeNumericCast(data[6] / Interval::MICROS_PER_MSEC)); + target = WritePadded3(target, data[6] / Interval::MICROS_PER_MSEC); break; case StrTimeSpecifier::UTC_OFFSET: { *target++ = (data[7] < 0) ? '-' : '+'; @@ -353,10 +350,10 @@ char *StrfTimeFormat::WriteStandardSpecifier(StrTimeSpecifier specifier, int32_t auto offset = abs(data[7]); auto offset_hours = offset / Interval::MINS_PER_HOUR; auto offset_minutes = offset % Interval::MINS_PER_HOUR; - target = WritePadded2(target, UnsafeNumericCast(offset_hours)); + target = WritePadded2(target, offset_hours); if (offset_minutes) { *target++ = ':'; - target = WritePadded2(target, UnsafeNumericCast(offset_minutes)); + target = WritePadded2(target, offset_minutes); } break; } @@ -367,7 +364,7 @@ char *StrfTimeFormat::WriteStandardSpecifier(StrTimeSpecifier specifier, int32_t } break; case StrTimeSpecifier::DAY_OF_MONTH: { - target = Write2(target, UnsafeNumericCast(data[2] % 100)); + target = Write2(target, data[2] % 100); break; } case StrTimeSpecifier::MONTH_DECIMAL: { @@ -375,7 +372,7 @@ char *StrfTimeFormat::WriteStandardSpecifier(StrTimeSpecifier specifier, int32_t break; } case StrTimeSpecifier::YEAR_WITHOUT_CENTURY: { - target = Write2(target, UnsafeNumericCast(AbsValue(data[0]) % 100)); + target = Write2(target, AbsValue(data[0]) % 100); break; } case StrTimeSpecifier::HOUR_24_DECIMAL: { @@ -848,9 +845,9 @@ bool StrpTimeFormat::Parse(string_t str, ParseResult &result) const { // numeric specifier: parse a number uint64_t number = 0; size_t start_pos = pos; - size_t end_pos = start_pos + UnsafeNumericCast(numeric_width[i]); + size_t end_pos = start_pos + numeric_width[i]; while (pos < size && pos < end_pos && StringUtil::CharacterIsDigit(data[pos])) { - number = number * 10 + UnsafeNumericCast(data[pos]) - '0'; + number = number * 10 + data[pos] - '0'; pos++; } if (pos == start_pos) { @@ -1232,7 +1229,7 @@ bool StrpTimeFormat::Parse(string_t str, ParseResult &result) const { // But tz must not be empty. if (tz_end == tz_begin) { error_message = "Empty Time Zone name"; - error_position = UnsafeNumericCast(tz_begin - data); + error_position = tz_begin - data; return false; } result.tz.assign(tz_begin, tz_end); @@ -1291,7 +1288,7 @@ bool StrpTimeFormat::Parse(string_t str, ParseResult &result) const { case StrTimeSpecifier::WEEK_NUMBER_PADDED_MON_FIRST: { // Adjust weekday to be 0-based for the week type if (has_weekday) { - weekday = (weekday + 7 - uint64_t(offset_specifier == StrTimeSpecifier::WEEK_NUMBER_PADDED_MON_FIRST)) % 7; + weekday = (weekday + 7 - int(offset_specifier == StrTimeSpecifier::WEEK_NUMBER_PADDED_MON_FIRST)) % 7; } // Get the start of week 1, move back 7 days and then weekno * 7 + weekday gives the date const auto jan1 = Date::FromDate(result_data[0], 1, 1); diff --git a/src/include/duckdb/common/sort/duckdb_pdqsort.hpp b/src/include/duckdb/common/sort/duckdb_pdqsort.hpp index cae339b20f2e..a71788ebf4b4 100644 --- a/src/include/duckdb/common/sort/duckdb_pdqsort.hpp +++ b/src/include/duckdb/common/sort/duckdb_pdqsort.hpp @@ -320,7 +320,7 @@ inline T *align_cacheline(T *p) { #else std::size_t ip = reinterpret_cast(p); #endif - ip = (ip + cacheline_size - 1) & duckdb::UnsafeNumericCast(-cacheline_size); + ip = (ip + cacheline_size - 1) & -duckdb::UnsafeNumericCast(cacheline_size); return reinterpret_cast(ip); } diff --git a/src/include/duckdb/common/types/bit.hpp b/src/include/duckdb/common/types/bit.hpp index 903b3cc75ff9..12c64d8b37f2 100644 --- a/src/include/duckdb/common/types/bit.hpp +++ b/src/include/duckdb/common/types/bit.hpp @@ -104,7 +104,7 @@ void Bit::NumericToBit(T numeric, string_t &output_str) { *output = 0; // set padding to 0 ++output; for (idx_t idx = 0; idx < sizeof(T); ++idx) { - output[idx] = UnsafeNumericCast(data[sizeof(T) - idx - 1]); + output[idx] = static_cast(data[sizeof(T) - idx - 1]); } Bit::Finalize(output_str); } diff --git a/src/include/duckdb/common/types/cast_helpers.hpp b/src/include/duckdb/common/types/cast_helpers.hpp index 03c45b5a09df..358438da3c17 100644 --- a/src/include/duckdb/common/types/cast_helpers.hpp +++ b/src/include/duckdb/common/types/cast_helpers.hpp @@ -62,9 +62,9 @@ class NumericHelper { template static string_t FormatSigned(SIGNED value, Vector &vector) { int sign = -(value < 0); - UNSIGNED unsigned_value = UNSIGNED(value) ^ UNSIGNED(sign) - UNSIGNED(sign); + UNSIGNED unsigned_value = UnsafeNumericCast(UNSIGNED(value ^ sign) - sign); int length = UnsignedLength(unsigned_value) - sign; - string_t result = StringVector::EmptyString(vector, NumericCast(length)); + string_t result = StringVector::EmptyString(vector, NumericCast(length)); auto dataptr = result.GetDataWriteable(); auto endptr = dataptr + length; endptr = FormatUnsigned(unsigned_value, endptr); diff --git a/src/include/duckdb/storage/string_uncompressed.hpp b/src/include/duckdb/storage/string_uncompressed.hpp index dacad03a0edb..0561efe099a0 100644 --- a/src/include/duckdb/storage/string_uncompressed.hpp +++ b/src/include/duckdb/storage/string_uncompressed.hpp @@ -148,7 +148,7 @@ struct UncompressedStringStorage { // place the dictionary offset into the set of vectors // note: for overflow strings we write negative value - result_data[target_idx] = NumericCast(-(*dictionary_size)); + result_data[target_idx] = -NumericCast((*dictionary_size)); } else { // string fits in block, append to dictionary and increment dictionary position D_ASSERT(string_length < NumericLimits::Maximum()); diff --git a/src/main/capi/result-c.cpp b/src/main/capi/result-c.cpp index 5b17c0132751..fae2b0cb2c24 100644 --- a/src/main/capi/result-c.cpp +++ b/src/main/capi/result-c.cpp @@ -117,7 +117,7 @@ struct CDecimalConverter : public CBaseConverter { template static DST Convert(SRC input) { duckdb_hugeint result; - result.lower = NumericCast(input); + result.lower = static_cast(input); result.upper = 0; return result; } diff --git a/src/parser/transform/helpers/transform_typename.cpp b/src/parser/transform/helpers/transform_typename.cpp index edea1222fba0..01c1697107e1 100644 --- a/src/parser/transform/helpers/transform_typename.cpp +++ b/src/parser/transform/helpers/transform_typename.cpp @@ -245,7 +245,7 @@ LogicalType Transformer::TransformTypeName(duckdb_libpgquery::PGTypeName &type_n if (val->type != duckdb_libpgquery::T_PGInteger) { throw ParserException("Expected integer value as array bound"); } - auto array_size = NumericCast(val->val.ival); + auto array_size = val->val.ival; if (array_size < 0) { // -1 if bounds are empty result_type = LogicalType::LIST(result_type); @@ -255,7 +255,7 @@ LogicalType Transformer::TransformTypeName(duckdb_libpgquery::PGTypeName &type_n } else if (array_size > static_cast(ArrayType::MAX_ARRAY_SIZE)) { throw ParserException("Arrays must have a size of at most %d", ArrayType::MAX_ARRAY_SIZE); } else { - result_type = LogicalType::ARRAY(result_type, array_size); + result_type = LogicalType::ARRAY(result_type, NumericCast(array_size)); } } } diff --git a/src/storage/buffer/buffer_pool_reservation.cpp b/src/storage/buffer/buffer_pool_reservation.cpp index 60a7bc882e61..f22a96ffa58f 100644 --- a/src/storage/buffer/buffer_pool_reservation.cpp +++ b/src/storage/buffer/buffer_pool_reservation.cpp @@ -23,8 +23,8 @@ BufferPoolReservation::~BufferPoolReservation() { } void BufferPoolReservation::Resize(idx_t new_size) { - int64_t delta = NumericCast(new_size) - NumericCast(size); - pool.IncreaseUsedMemory(tag, NumericCast(delta)); + int64_t delta = (int64_t)new_size - size; + pool.IncreaseUsedMemory(tag, delta); size = new_size; } From 99aa556e096250142400e72aebd08ccd2d6affc8 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 5 Apr 2024 13:43:54 +0200 Subject: [PATCH 204/603] Fall back to default empty string --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e8316c787bb1..791287b52743 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -822,7 +822,7 @@ function(build_loadable_extension_directory NAME OUTPUT_DIRECTORY EXTENSION_VERS COMMAND printf "${DUCKDB_NORMALIZED_VERSION}" > duckdb_version_file COMMAND - printf "${EXTENSION_VERSION}" > extension_version_file || (echo "CMake appears not to propagate correctly version information for the extension" && exit 1) + printf "${EXTENSION_VERSION}" > extension_version_file || (echo "CMake appears not to propagate correctly version information for the extension" && > extension_version_file) COMMAND bash ${CMAKE_SOURCE_DIR}/scripts/append_metadata.sh $ ${CMAKE_BINARY_DIR}/duckdb_platform_out duckdb_version_file extension_version_file ) From 7bf4cc69691c8a8ec0f22727e3bc5fdb4391cb2d Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Fri, 5 Apr 2024 13:53:53 +0200 Subject: [PATCH 205/603] WIP OrderBinder rework - return a constant value instead of a BoundColumnReference --- src/include/duckdb/planner/binder.hpp | 4 +- .../expression_binder/order_binder.hpp | 11 +-- .../expression_binder/select_bind_state.hpp | 3 + .../binder/query_node/bind_select_node.cpp | 77 ++++++++++--------- .../binder/query_node/bind_setop_node.cpp | 9 +-- .../expression_binder/order_binder.cpp | 37 +++------ .../expression_binder/select_bind_state.cpp | 8 ++ 7 files changed, 72 insertions(+), 77 deletions(-) diff --git a/src/include/duckdb/planner/binder.hpp b/src/include/duckdb/planner/binder.hpp index 3dfee62f1ab9..3b67155e23ec 100644 --- a/src/include/duckdb/planner/binder.hpp +++ b/src/include/duckdb/planner/binder.hpp @@ -324,8 +324,8 @@ class Binder : public std::enable_shared_from_this { BoundStatement BindCopyTo(CopyStatement &stmt); BoundStatement BindCopyFrom(CopyStatement &stmt); - void BindModifiers(OrderBinder &order_binder, QueryNode &statement, BoundQueryNode &result); - void BindModifierTypes(BoundQueryNode &result, const vector &sql_types, idx_t projection_index); + void PrepareModifiers(OrderBinder &order_binder, QueryNode &statement, BoundQueryNode &result); + void BindModifiers(BoundQueryNode &result, idx_t table_index, const vector &sql_types, const SelectBindState &bind_state); unique_ptr BindLimit(OrderBinder &order_binder, LimitModifier &limit_mod); unique_ptr BindLimitPercent(OrderBinder &order_binder, LimitPercentModifier &limit_mod); diff --git a/src/include/duckdb/planner/expression_binder/order_binder.hpp b/src/include/duckdb/planner/expression_binder/order_binder.hpp index fb096f9f39f1..8e18161ac057 100644 --- a/src/include/duckdb/planner/expression_binder/order_binder.hpp +++ b/src/include/duckdb/planner/expression_binder/order_binder.hpp @@ -22,15 +22,12 @@ struct SelectBindState; //! The ORDER binder is responsible for binding an expression within the ORDER BY clause of a SQL statement class OrderBinder { public: - OrderBinder(vector binders, idx_t projection_index, SelectBindState &bind_state, idx_t max_count); - OrderBinder(vector binders, idx_t projection_index, SelectNode &node, SelectBindState &bind_state); + OrderBinder(vector binders, SelectBindState &bind_state); + OrderBinder(vector binders, SelectNode &node, SelectBindState &bind_state); public: unique_ptr Bind(unique_ptr expr); - idx_t MaxCount() const { - return max_count; - } bool HasExtraList() const { return extra_list; } @@ -46,9 +43,7 @@ class OrderBinder { private: vector binders; - idx_t projection_index; - idx_t max_count; - vector> *extra_list; + optional_ptr>> extra_list; SelectBindState &bind_state; }; diff --git a/src/include/duckdb/planner/expression_binder/select_bind_state.hpp b/src/include/duckdb/planner/expression_binder/select_bind_state.hpp index e65c0207a393..49ce92b143f2 100644 --- a/src/include/duckdb/planner/expression_binder/select_bind_state.hpp +++ b/src/include/duckdb/planner/expression_binder/select_bind_state.hpp @@ -34,6 +34,9 @@ struct SelectBindState { bool AliasHasSubquery(idx_t index); + void AddExpandedColumn(idx_t expand_count); + void AddRegularColumn(); + private: //! The set of referenced aliases unordered_set referenced_aliases; diff --git a/src/planner/binder/query_node/bind_select_node.cpp b/src/planner/binder/query_node/bind_select_node.cpp index a817697ecea6..e903d78bcc61 100644 --- a/src/planner/binder/query_node/bind_select_node.cpp +++ b/src/planner/binder/query_node/bind_select_node.cpp @@ -14,6 +14,7 @@ #include "duckdb/parser/tableref/joinref.hpp" #include "duckdb/planner/binder.hpp" #include "duckdb/planner/expression/bound_aggregate_expression.hpp" +#include "duckdb/planner/expression/bound_constant_expression.hpp" #include "duckdb/planner/expression/bound_expanded_expression.hpp" #include "duckdb/planner/expression_binder/column_alias_binder.hpp" #include "duckdb/planner/expression_binder/constant_binder.hpp" @@ -38,7 +39,7 @@ unique_ptr Binder::BindOrderExpression(OrderBinder &order_binder, un // remove the expression from the DISTINCT ON list return nullptr; } - D_ASSERT(bound_expr->type == ExpressionType::BOUND_COLUMN_REF); + D_ASSERT(bound_expr->type == ExpressionType::VALUE_CONSTANT); return bound_expr; } @@ -123,7 +124,7 @@ unique_ptr Binder::BindLimitPercent(OrderBinder &order_bind return std::move(result); } -void Binder::BindModifiers(OrderBinder &order_binder, QueryNode &statement, BoundQueryNode &result) { +void Binder::PrepareModifiers(OrderBinder &order_binder, QueryNode &statement, BoundQueryNode &result) { for (auto &mod : statement.modifiers) { unique_ptr bound_modifier; switch (mod->type) { @@ -161,14 +162,10 @@ void Binder::BindModifiers(OrderBinder &order_binder, QueryNode &statement, Boun // replace the order list with the all elements in the SELECT list auto order_type = order.orders[0].type; auto null_order = order.orders[0].null_order; - - vector new_orders; - for (idx_t i = 0; i < order_binder.MaxCount(); i++) { - new_orders.emplace_back( - order_type, null_order, - make_uniq(Value::INTEGER(UnsafeNumericCast(i + 1)))); - } - order.orders = std::move(new_orders); + auto constant_expr = make_uniq(Value("ALL")); + bound_order->orders.emplace_back(order_type, null_order, std::move(constant_expr)); + bound_modifier = std::move(bound_order); + break; } } #if 0 @@ -246,7 +243,28 @@ static void AssignReturnType(unique_ptr &expr, const vector &sql_types, idx_t) { +unique_ptr FinalizeBindOrderExpression(unique_ptr expr, idx_t table_index, const vector &sql_types, const SelectBindState &bind_state) { + auto &constant = expr->Cast(); + switch(constant.value.type().id()) { + case LogicalTypeId::UBIGINT: { + // index + auto index = UBigIntValue::Get(constant.value); + return make_uniq(std::move(expr->alias), sql_types[index], ColumnBinding(table_index, index)); + } + case LogicalTypeId::VARCHAR: { + // ORDER BY ALL + throw InternalException("FIXME: order by all"); + } + case LogicalTypeId::STRUCT: { + // collation + throw InternalException("FIXME: collate struct by number"); + } + default: + throw InternalException("Unknown type in FinalizeBindOrderExpression"); + } +} + +void Binder::BindModifiers(BoundQueryNode &result, idx_t table_index, const vector &sql_types, const SelectBindState &bind_state) { for (auto &bound_mod : result.modifiers) { switch (bound_mod->type) { case ResultModifierType::DISTINCT_MODIFIER: { @@ -254,17 +272,8 @@ void Binder::BindModifierTypes(BoundQueryNode &result, const vector D_ASSERT(!distinct.target_distincts.empty()); // set types of distinct targets for (auto &expr : distinct.target_distincts) { - D_ASSERT(expr->type == ExpressionType::BOUND_COLUMN_REF); - auto &bound_colref = expr->Cast(); - if (bound_colref.binding.column_index == DConstants::INVALID_INDEX) { - throw BinderException("Ambiguous name in DISTINCT ON!"); - } - bound_colref.return_type = sql_types[bound_colref.binding.column_index]; - } - for (auto &target_distinct : distinct.target_distincts) { - auto &bound_colref = target_distinct->Cast(); - const auto &sql_type = sql_types[bound_colref.binding.column_index]; - ExpressionBinder::PushCollation(context, target_distinct, sql_type, true); + expr = FinalizeBindOrderExpression(std::move(expr), table_index, sql_types, bind_state); + ExpressionBinder::PushCollation(context, expr, expr->return_type, true); } break; } @@ -277,17 +286,9 @@ void Binder::BindModifierTypes(BoundQueryNode &result, const vector case ResultModifierType::ORDER_MODIFIER: { auto &order = bound_mod->Cast(); for (auto &order_node : order.orders) { - auto &expr = order_node.expression; - D_ASSERT(expr->type == ExpressionType::BOUND_COLUMN_REF); - auto &bound_colref = expr->Cast(); - if (bound_colref.binding.column_index == DConstants::INVALID_INDEX) { - throw BinderException("Ambiguous name in ORDER BY!"); - } - const auto &sql_type = sql_types[bound_colref.binding.column_index]; - bound_colref.return_type = sql_type; - - ExpressionBinder::PushCollation(context, order_node.expression, sql_type); + expr = FinalizeBindOrderExpression(std::move(expr), table_index, sql_types, bind_state); + ExpressionBinder::PushCollation(context, order_node.expression, expr->return_type); } break; } @@ -389,8 +390,8 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ } // now bind all the result modifiers; including DISTINCT and ORDER BY targets - OrderBinder order_binder({this}, result->projection_index, statement, bind_state); - BindModifiers(order_binder, statement, *result); + OrderBinder order_binder({this}, statement, bind_state); + PrepareModifiers(order_binder, statement, *result); vector> unbound_groups; BoundGroupInformation info; @@ -504,15 +505,17 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ internal_sql_types.push_back(struct_expr->return_type); result->select_list.push_back(std::move(struct_expr)); } + bind_state.AddExpandedColumn(struct_expressions.size()); continue; } if (expr->IsVolatile()) { - result->bind_state.SetExpressionIsVolatile(i); + bind_state.SetExpressionIsVolatile(i); } if (expr->HasSubquery()) { - result->bind_state.SetExpressionHasSubquery(i); + bind_state.SetExpressionHasSubquery(i); } + bind_state.AddRegularColumn(); if (can_group_by_all && select_binder.HasBoundColumns()) { if (select_binder.BoundAggregates()) { @@ -597,7 +600,7 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ } // now that the SELECT list is bound, we set the types of DISTINCT/ORDER BY expressions - BindModifierTypes(*result, internal_sql_types, result->projection_index); + BindModifiers(*result, result->projection_index, internal_sql_types, bind_state); return std::move(result); } diff --git a/src/planner/binder/query_node/bind_setop_node.cpp b/src/planner/binder/query_node/bind_setop_node.cpp index fbdb8d02200a..60a86d7bb09c 100644 --- a/src/planner/binder/query_node/bind_setop_node.cpp +++ b/src/planner/binder/query_node/bind_setop_node.cpp @@ -225,12 +225,12 @@ unique_ptr Binder::BindNode(SetOperationNode &statement) { } } + SelectBindState bind_state; if (!statement.modifiers.empty()) { // handle the ORDER BY/DISTINCT clauses // we recursively visit the children of this node to extract aliases and expressions that can be referenced // in the ORDER BY - SelectBindState bind_state; if (result->setop_type == SetOperationType::UNION_BY_NAME) { GatherAliases(*result->left, bind_state, result->left_reorder_idx); @@ -243,13 +243,12 @@ unique_ptr Binder::BindNode(SetOperationNode &statement) { GatherAliases(*result, bind_state, reorder_idx); } // now we perform the actual resolution of the ORDER BY/DISTINCT expressions - OrderBinder order_binder({result->left_binder.get(), result->right_binder.get()}, result->setop_index, - bind_state, result->names.size()); - BindModifiers(order_binder, statement, *result); + OrderBinder order_binder({result->left_binder.get(), result->right_binder.get()}, bind_state); + PrepareModifiers(order_binder, statement, *result); } // finally bind the types of the ORDER/DISTINCT clause expressions - BindModifierTypes(*result, result->types, result->setop_index); + BindModifiers(*result, result->setop_index, result->types, bind_state); return std::move(result); } diff --git a/src/planner/expression_binder/order_binder.cpp b/src/planner/expression_binder/order_binder.cpp index 4345529bf57e..ede86e8ab2a0 100644 --- a/src/planner/expression_binder/order_binder.cpp +++ b/src/planner/expression_binder/order_binder.cpp @@ -10,17 +10,15 @@ #include "duckdb/planner/binder.hpp" #include "duckdb/planner/expression/bound_parameter_expression.hpp" #include "duckdb/planner/expression_binder.hpp" +#include "duckdb/common/pair.hpp" namespace duckdb { -OrderBinder::OrderBinder(vector binders, idx_t projection_index, SelectBindState &bind_state, idx_t max_count) - : binders(std::move(binders)), projection_index(projection_index), max_count(max_count), extra_list(nullptr), - bind_state(bind_state) { +OrderBinder::OrderBinder(vector binders, SelectBindState &bind_state) + : binders(std::move(binders)), extra_list(nullptr), bind_state(bind_state) { } -OrderBinder::OrderBinder(vector binders, idx_t projection_index, SelectNode &node, - SelectBindState &bind_state) - : binders(std::move(binders)), projection_index(projection_index), bind_state(bind_state) { - this->max_count = node.select_list.size(); +OrderBinder::OrderBinder(vector binders, SelectNode &node, SelectBindState &bind_state) + : binders(std::move(binders)), bind_state(bind_state) { this->extra_list = &node.select_list; } @@ -33,8 +31,9 @@ unique_ptr OrderBinder::CreateProjectionReference(ParsedExpression & alias = expr.alias; } } - return make_uniq(std::move(alias), LogicalType::INVALID, - ColumnBinding(projection_index, index)); + auto result = make_uniq(Value::UBIGINT(index)); + result->alias = std::move(alias); + return std::move(result); } unique_ptr OrderBinder::CreateExtraReference(unique_ptr expr) { @@ -57,9 +56,6 @@ unique_ptr OrderBinder::BindConstant(ParsedExpression &expr, const V } // INTEGER constant: we use the integer as an index into the select list (e.g. ORDER BY 1) auto index = (idx_t)val.GetValue(); - if (index < 1 || index > max_count) { - throw BinderException("ORDER term out of range - should be between 1 and %lld", max_count); - } return CreateProjectionReference(expr, index - 1); } @@ -94,9 +90,6 @@ unique_ptr OrderBinder::Bind(unique_ptr expr) { } case ExpressionClass::POSITIONAL_REFERENCE: { auto &posref = expr->Cast(); - if (posref.index < 1 || posref.index > max_count) { - throw BinderException("ORDER term out of range - should be between 1 and %lld", max_count); - } return CreateProjectionReference(*expr, posref.index - 1); } case ExpressionClass::PARAMETER: { @@ -107,16 +100,10 @@ unique_ptr OrderBinder::Bind(unique_ptr expr) { if (collation.child->expression_class == ExpressionClass::CONSTANT) { auto &constant = collation.child->Cast(); auto index = NumericCast(constant.value.GetValue()) - 1; - if (index >= extra_list->size()) { - throw BinderException("ORDER term out of range - should be between 1 and %lld", max_count); - } - auto &sel_entry = extra_list->at(index); - if (sel_entry->HasSubquery()) { - throw BinderException( - "OrderBy referenced a ColumnNumber in a SELECT clause - but the expression has a subquery." - " This is not yet supported."); - } - collation.child = sel_entry->Copy(); + child_list_t values; + values.push_back(make_pair("index", Value::UBIGINT(index))); + values.push_back(make_pair("collation", Value(std::move(collation.collation)))); + return make_uniq(Value::STRUCT(std::move(values))); } break; } diff --git a/src/planner/expression_binder/select_bind_state.cpp b/src/planner/expression_binder/select_bind_state.cpp index 382ff182bd1e..134a4d819b62 100644 --- a/src/planner/expression_binder/select_bind_state.cpp +++ b/src/planner/expression_binder/select_bind_state.cpp @@ -30,4 +30,12 @@ bool SelectBindState::AliasHasSubquery(idx_t index) { return subquery_expressions.find(index) != subquery_expressions.end(); } +void SelectBindState::AddExpandedColumn(idx_t expand_count) { + +} + +void SelectBindState::AddRegularColumn() { + +} + } // namespace duckdb From 2b6e45c785a7fa611b2fced55c28426d4759fd35 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 5 Apr 2024 13:58:46 +0200 Subject: [PATCH 206/603] Correct outdated check --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 791287b52743..6fa0e7d09c2c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -951,8 +951,8 @@ function(duckdb_extension_load NAME) if (${duckdb_extension_load_DONT_BUILD}) register_extension(${NAME} "${duckdb_extension_load_DONT_LINK}" "${duckdb_extension_load_DONT_BUILD}" "" "" "" "") elseif (NOT "${duckdb_extension_load_GIT_URL}" STREQUAL "") - if (NOT "${duckdb_extension_load_GIT_COMMIT}" STREQUAL "") - error("Git URL specified but no valid git commit was found for ${NAME} extension") + if ("${duckdb_extension_load_GIT_TAG}" STREQUAL "") + error("Git URL specified but no valid GIT_TAG was found for ${NAME} extension") endif() register_external_extension(${NAME} "${duckdb_extension_load_GIT_URL}" "${duckdb_extension_load_GIT_TAG}" "${duckdb_extension_load_DONT_LINK}" "${duckdb_extension_load_DONT_BUILD}" "${duckdb_extension_load_LOAD_TESTS}" "${duckdb_extension_load_SOURCE_DIR}" "${duckdb_extension_load_INCLUDE_DIR}" "${duckdb_extension_load_TEST_DIR}" "${duckdb_extension_load_APPLY_PATCHES}" "${duckdb_extension_load_SUBMODULES}") set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG "${duckdb_extension_load_GIT_TAG}" PARENT_SCOPE) From 1113e9e6fb200a39e4afeeff2617c2cdefa2dce8 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 5 Apr 2024 13:59:22 +0200 Subject: [PATCH 207/603] Extension metdatada: Propagate extension version in more cases --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6fa0e7d09c2c..59dbb58e3c1d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -979,11 +979,12 @@ function(duckdb_extension_load NAME) # Local extension, default path message(STATUS "Load extension '${NAME}' from '${CMAKE_CURRENT_SOURCE_DIR}/extension_external'") register_extension(${NAME} ${duckdb_extension_load_DONT_LINK} "${duckdb_extension_load_DONT_BUILD}" "${duckdb_extension_load_LOAD_TESTS}" "${CMAKE_CURRENT_SOURCE_DIR}/extension_external/${NAME}" "${CMAKE_CURRENT_SOURCE_DIR}/extension_external/${NAME}/src/include" "${CMAKE_CURRENT_SOURCE_DIR}/extension_external/${NAME}/test/sql") + set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG "${DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG}" PARENT_SCOPE) else() # Local extension, default path message(STATUS "Load extension '${NAME}' from '${CMAKE_CURRENT_SOURCE_DIR}/extensions'") register_extension(${NAME} ${duckdb_extension_load_DONT_LINK} "${duckdb_extension_load_DONT_BUILD}" "${duckdb_extension_load_LOAD_TESTS}" "${CMAKE_CURRENT_SOURCE_DIR}/extension/${NAME}" "${CMAKE_CURRENT_SOURCE_DIR}/extension/${NAME}/include" "${CMAKE_CURRENT_SOURCE_DIR}/extension/${NAME}/test/sql") - set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG ${DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG} PARENT_SCOPE) + set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG "${DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG}" PARENT_SCOPE) endif() # Propagate variables set by register_extension From 805a732a083b65916306b5bbfe91353a80a6906d Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 5 Apr 2024 14:00:14 +0200 Subject: [PATCH 208/603] Extension metadata: EXTENSION_VERSION takes precedence, then GIT_TAG --- CMakeLists.txt | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 59dbb58e3c1d..2ce510567fa3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -930,7 +930,7 @@ endmacro() function(duckdb_extension_load NAME) # Parameter parsing set(options DONT_LINK DONT_BUILD LOAD_TESTS APPLY_PATCHES) - set(oneValueArgs SOURCE_DIR INCLUDE_DIR TEST_DIR GIT_URL GIT_TAG SUBMODULES) + set(oneValueArgs SOURCE_DIR INCLUDE_DIR TEST_DIR GIT_URL GIT_TAG SUBMODULES EXTENSION_VERSION) cmake_parse_arguments(duckdb_extension_load "${options}" "${oneValueArgs}" "" ${ARGN}) string(TOLOWER ${NAME} EXTENSION_NAME_LOWERCASE) @@ -955,7 +955,11 @@ function(duckdb_extension_load NAME) error("Git URL specified but no valid GIT_TAG was found for ${NAME} extension") endif() register_external_extension(${NAME} "${duckdb_extension_load_GIT_URL}" "${duckdb_extension_load_GIT_TAG}" "${duckdb_extension_load_DONT_LINK}" "${duckdb_extension_load_DONT_BUILD}" "${duckdb_extension_load_LOAD_TESTS}" "${duckdb_extension_load_SOURCE_DIR}" "${duckdb_extension_load_INCLUDE_DIR}" "${duckdb_extension_load_TEST_DIR}" "${duckdb_extension_load_APPLY_PATCHES}" "${duckdb_extension_load_SUBMODULES}") - set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG "${duckdb_extension_load_GIT_TAG}" PARENT_SCOPE) + if (NOT "${duckdb_extension_load_EXTENSION_VERSION}" STREQUAL "") + set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG "${duckdb_extension_load_EXTENSION_VERSION}" PARENT_SCOPE) + else() + set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG "${duckdb_extension_load_GIT_TAG}" PARENT_SCOPE) + endif() elseif (NOT "${duckdb_extension_load_SOURCE_DIR}" STREQUAL "") # Local extension, custom path message(STATUS "Load extension '${NAME}' from '${duckdb_extension_load_SOURCE_DIR}'") @@ -974,6 +978,13 @@ function(duckdb_extension_load NAME) set(TEST_PATH_DEFAULT ${duckdb_extension_load_TEST_DIR}) endif() + if (NOT "${duckdb_extension_load_EXTENSION_VERSION}" STREQUAL "") + set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG "${duckdb_extension_load_EXTENSION_VERSION}" PARENT_SCOPE) + elseif (NOT "${duckdb_extension_load_GIT_TAG}" STREQUAL "") + set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG "${duckdb_extension_load_GIT_TAG}" PARENT_SCOPE) + else() + set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG "" PARENT_SCOPE) + endif() register_extension(${NAME} "${duckdb_extension_load_DONT_LINK}" "${duckdb_extension_load_DONT_BUILD}" "${duckdb_extension_load_LOAD_TESTS}" "${duckdb_extension_load_SOURCE_DIR}" "${INCLUDE_PATH_DEFAULT}" "${TEST_PATH_DEFAULT}") elseif(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/extension_external/${NAME}) # Local extension, default path From 65e8e636471c2409743353c8a9e88f6ca9b8bde7 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 5 Apr 2024 14:34:12 +0200 Subject: [PATCH 209/603] Extension metadata: when specifying a SOURCE_DIR, if the extension is part of a repository use that repo git reference --- CMakeLists.txt | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2ce510567fa3..2fea25b4acdf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -983,7 +983,20 @@ function(duckdb_extension_load NAME) elseif (NOT "${duckdb_extension_load_GIT_TAG}" STREQUAL "") set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG "${duckdb_extension_load_GIT_TAG}" PARENT_SCOPE) else() - set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG "" PARENT_SCOPE) + find_package(Git) + if(Git_FOUND) + message(STATUS "CIAO ${duckdb_extension_load_SOURCE_DIR}") + execute_process( + COMMAND ${GIT_EXECUTABLE} log -1 --format=%h + WORKING_DIRECTORY ${duckdb_extension_load_SOURCE_DIR} + RESULT_VARIABLE GIT_RESULT + OUTPUT_VARIABLE GIT_COMMIT + OUTPUT_STRIP_TRAILING_WHITESPACE) + set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG "${GIT_COMMIT}" PARENT_SCOPE) + message(STATUS "CIAO ${GIT_COMMIT}") + else() + set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG "" PARENT_SCOPE) + endif() endif() register_extension(${NAME} "${duckdb_extension_load_DONT_LINK}" "${duckdb_extension_load_DONT_BUILD}" "${duckdb_extension_load_LOAD_TESTS}" "${duckdb_extension_load_SOURCE_DIR}" "${INCLUDE_PATH_DEFAULT}" "${TEST_PATH_DEFAULT}") elseif(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/extension_external/${NAME}) From 4b92eff10c0e070df55c52df48f2e9ba2e8a8566 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Fri, 5 Apr 2024 14:34:55 +0200 Subject: [PATCH 210/603] ORDER BY ALL working --- .../common/exception/binder_exception.hpp | 4 +++ .../binder/query_node/bind_select_node.cpp | 31 +++++++++++++++++-- .../expression_binder/order_binder.cpp | 1 + 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/include/duckdb/common/exception/binder_exception.hpp b/src/include/duckdb/common/exception/binder_exception.hpp index 21e04b13c833..653c5e80bd94 100644 --- a/src/include/duckdb/common/exception/binder_exception.hpp +++ b/src/include/duckdb/common/exception/binder_exception.hpp @@ -30,6 +30,10 @@ class BinderException : public Exception { : BinderException(ConstructMessage(msg, params...), Exception::InitializeExtraInfo(expr)) { } template + explicit BinderException(const Expression &expr, const string &msg, ARGS... params) + : BinderException(ConstructMessage(msg, params...), Exception::InitializeExtraInfo(expr)) { + } + template explicit BinderException(QueryErrorContext error_context, const string &msg, ARGS... params) : BinderException(ConstructMessage(msg, params...), Exception::InitializeExtraInfo(error_context)) { } diff --git a/src/planner/binder/query_node/bind_select_node.cpp b/src/planner/binder/query_node/bind_select_node.cpp index e903d78bcc61..8e2a58c4b26e 100644 --- a/src/planner/binder/query_node/bind_select_node.cpp +++ b/src/planner/binder/query_node/bind_select_node.cpp @@ -160,8 +160,8 @@ void Binder::PrepareModifiers(OrderBinder &order_binder, QueryNode &statement, B if (star.exclude_list.empty() && star.replace_list.empty() && !star.expr) { // ORDER BY ALL // replace the order list with the all elements in the SELECT list - auto order_type = order.orders[0].type; - auto null_order = order.orders[0].null_order; + auto order_type = config.ResolveOrder(order.orders[0].type); + auto null_order = config.ResolveNullOrder(order_type, order.orders[0].null_order); auto constant_expr = make_uniq(Value("ALL")); bound_order->orders.emplace_back(order_type, null_order, std::move(constant_expr)); bound_modifier = std::move(bound_order); @@ -249,11 +249,14 @@ unique_ptr FinalizeBindOrderExpression(unique_ptr expr, case LogicalTypeId::UBIGINT: { // index auto index = UBigIntValue::Get(constant.value); + if (index >= sql_types.size()) { + throw BinderException(*expr, "ORDER term out of range - should be between 1 and %lld", sql_types.size()); + } return make_uniq(std::move(expr->alias), sql_types[index], ColumnBinding(table_index, index)); } case LogicalTypeId::VARCHAR: { // ORDER BY ALL - throw InternalException("FIXME: order by all"); + return nullptr; } case LogicalTypeId::STRUCT: { // collation @@ -273,6 +276,11 @@ void Binder::BindModifiers(BoundQueryNode &result, idx_t table_index, const vect // set types of distinct targets for (auto &expr : distinct.target_distincts) { expr = FinalizeBindOrderExpression(std::move(expr), table_index, sql_types, bind_state); + if (!expr) { + throw InternalException("DISTINCT ON ORDER BY ALL not supported"); + } + } + for (auto &expr : distinct.target_distincts) { ExpressionBinder::PushCollation(context, expr, expr->return_type, true); } break; @@ -285,9 +293,26 @@ void Binder::BindModifiers(BoundQueryNode &result, idx_t table_index, const vect } case ResultModifierType::ORDER_MODIFIER: { auto &order = bound_mod->Cast(); + bool order_by_all = false; for (auto &order_node : order.orders) { auto &expr = order_node.expression; expr = FinalizeBindOrderExpression(std::move(expr), table_index, sql_types, bind_state); + if (!expr) { + order_by_all = true; + } + } + if (order_by_all) { + D_ASSERT(order.orders.size() == 1); + auto order_type = order.orders[0].type; + auto null_order = order.orders[0].null_order; + order.orders.clear(); + for(idx_t i = 0; i < sql_types.size(); i++) { + auto expr = make_uniq(sql_types[i], ColumnBinding(table_index, i)); + order.orders.emplace_back(order_type, null_order, std::move(expr)); + } + } + for (auto &order_node : order.orders) { + auto &expr = order_node.expression; ExpressionBinder::PushCollation(context, order_node.expression, expr->return_type); } break; diff --git a/src/planner/expression_binder/order_binder.cpp b/src/planner/expression_binder/order_binder.cpp index ede86e8ab2a0..6335f3949c72 100644 --- a/src/planner/expression_binder/order_binder.cpp +++ b/src/planner/expression_binder/order_binder.cpp @@ -33,6 +33,7 @@ unique_ptr OrderBinder::CreateProjectionReference(ParsedExpression & } auto result = make_uniq(Value::UBIGINT(index)); result->alias = std::move(alias); + result->query_location = expr.query_location; return std::move(result); } From 6a0786cdf6ec1d5219ffa85ce47c277e38aeab4e Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 5 Apr 2024 14:39:02 +0200 Subject: [PATCH 211/603] Extension metadata: when specifying a SOURCE_DIR, if the extension is part of a repository use that repo git reference --- CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2fea25b4acdf..59b5b97ee580 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -985,7 +985,6 @@ function(duckdb_extension_load NAME) else() find_package(Git) if(Git_FOUND) - message(STATUS "CIAO ${duckdb_extension_load_SOURCE_DIR}") execute_process( COMMAND ${GIT_EXECUTABLE} log -1 --format=%h WORKING_DIRECTORY ${duckdb_extension_load_SOURCE_DIR} @@ -993,7 +992,6 @@ function(duckdb_extension_load NAME) OUTPUT_VARIABLE GIT_COMMIT OUTPUT_STRIP_TRAILING_WHITESPACE) set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG "${GIT_COMMIT}" PARENT_SCOPE) - message(STATUS "CIAO ${GIT_COMMIT}") else() set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG "" PARENT_SCOPE) endif() From 5752be9f5ce0d259b876b8a2c624895ae257c958 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 5 Apr 2024 15:07:45 +0200 Subject: [PATCH 212/603] make wrapper implementation for weak_ptr and shared_ptr --- .../catalog_entry/duck_table_entry.cpp | 2 +- src/common/http_state.cpp | 2 +- src/common/re2_regex.cpp | 2 +- src/common/symbols.cpp | 26 ++--- src/execution/physical_plan/plan_cte.cpp | 2 +- .../physical_plan/plan_recursive_cte.cpp | 2 +- .../catalog_entry/duck_table_entry.hpp | 4 +- src/include/duckdb/common/helper.hpp | 8 ++ src/include/duckdb/common/re2_regex.hpp | 2 +- src/include/duckdb/common/shared_ptr.hpp | 109 +++++++++++++++++- src/include/duckdb/common/types.hpp | 1 + src/include/duckdb/common/weak_ptr.hpp | 88 ++++++++++++++ .../execution/operator/set/physical_cte.hpp | 2 +- .../operator/set/physical_recursive_cte.hpp | 2 +- .../execution/physical_plan_generator.hpp | 2 +- .../main/buffered_data/buffered_data.hpp | 1 + src/include/duckdb/main/client_context.hpp | 2 +- src/include/duckdb/main/relation.hpp | 2 +- .../duckdb/main/relation/query_relation.hpp | 2 +- .../main/relation/table_function_relation.hpp | 4 +- .../duckdb/main/relation/table_relation.hpp | 2 +- .../duckdb/main/relation/value_relation.hpp | 6 +- .../duckdb/main/relation/view_relation.hpp | 2 +- src/include/duckdb/planner/bind_context.hpp | 8 +- .../duckdb/transaction/transaction.hpp | 1 + src/main/relation.cpp | 4 +- src/main/relation/query_relation.cpp | 2 +- src/main/relation/read_csv_relation.cpp | 2 +- src/main/relation/table_relation.cpp | 2 +- src/main/relation/value_relation.cpp | 4 +- src/main/relation/view_relation.cpp | 2 +- src/planner/bind_context.cpp | 2 +- src/planner/binder.cpp | 1 + 33 files changed, 253 insertions(+), 50 deletions(-) create mode 100644 src/include/duckdb/common/weak_ptr.hpp diff --git a/src/catalog/catalog_entry/duck_table_entry.cpp b/src/catalog/catalog_entry/duck_table_entry.cpp index eb7be8d7a451..05dd9410e220 100644 --- a/src/catalog/catalog_entry/duck_table_entry.cpp +++ b/src/catalog/catalog_entry/duck_table_entry.cpp @@ -72,7 +72,7 @@ IndexStorageInfo GetIndexInfo(const IndexConstraintType &constraint_type, unique } DuckTableEntry::DuckTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, BoundCreateTableInfo &info, - std::shared_ptr inherited_storage) + shared_ptr inherited_storage) : TableCatalogEntry(catalog, schema, info.Base()), storage(std::move(inherited_storage)), bound_constraints(std::move(info.bound_constraints)), column_dependency_manager(std::move(info.column_dependency_manager)) { diff --git a/src/common/http_state.cpp b/src/common/http_state.cpp index b07c0d4b31f2..9d583fd8f5f2 100644 --- a/src/common/http_state.cpp +++ b/src/common/http_state.cpp @@ -26,7 +26,7 @@ void CachedFileHandle::AllocateBuffer(idx_t size) { if (file->initialized) { throw InternalException("Cannot allocate a buffer for a cached file that was already initialized"); } - file->data = std::shared_ptr(new char[size], std::default_delete()); + file->data = shared_ptr(new char[size], std::default_delete()); file->capacity = size; } diff --git a/src/common/re2_regex.cpp b/src/common/re2_regex.cpp index f22b681acfe6..da239cd4a213 100644 --- a/src/common/re2_regex.cpp +++ b/src/common/re2_regex.cpp @@ -10,7 +10,7 @@ namespace duckdb_re2 { Regex::Regex(const std::string &pattern, RegexOptions options) { RE2::Options o; o.set_case_sensitive(options == RegexOptions::CASE_INSENSITIVE); - regex = std::make_shared(StringPiece(pattern), o); + regex = make_shared(StringPiece(pattern), o); } bool RegexSearchInternal(const char *input, Match &match, const Regex &r, RE2::Anchor anchor, size_t start, diff --git a/src/common/symbols.cpp b/src/common/symbols.cpp index 8b550f2c93f5..816a40b0387f 100644 --- a/src/common/symbols.cpp +++ b/src/common/symbols.cpp @@ -182,24 +182,24 @@ INSTANTIATE_VECTOR(vector>) INSTANTIATE_VECTOR(vector>) INSTANTIATE_VECTOR(vector>) INSTANTIATE_VECTOR(vector>) -INSTANTIATE_VECTOR(vector>) +INSTANTIATE_VECTOR(vector>) INSTANTIATE_VECTOR(vector>) -INSTANTIATE_VECTOR(vector>) -INSTANTIATE_VECTOR(vector>) -INSTANTIATE_VECTOR(vector>) +INSTANTIATE_VECTOR(vector>) +INSTANTIATE_VECTOR(vector>) +INSTANTIATE_VECTOR(vector>) INSTANTIATE_VECTOR(vector>) INSTANTIATE_VECTOR(vector>) -INSTANTIATE_VECTOR(vector>) +INSTANTIATE_VECTOR(vector>) INSTANTIATE_VECTOR(vector>) -template class std::shared_ptr; -template class std::shared_ptr; -template class std::shared_ptr; -template class std::shared_ptr; -template class std::shared_ptr; -template class std::shared_ptr; -template class std::shared_ptr; -template class std::weak_ptr; +template class shared_ptr; +template class shared_ptr; +template class shared_ptr; +template class shared_ptr; +template class shared_ptr; +template class shared_ptr; +template class shared_ptr; +template class weak_ptr; #if !defined(__clang__) template struct std::atomic; diff --git a/src/execution/physical_plan/plan_cte.cpp b/src/execution/physical_plan/plan_cte.cpp index f323aed463b4..7a306c3a54ca 100644 --- a/src/execution/physical_plan/plan_cte.cpp +++ b/src/execution/physical_plan/plan_cte.cpp @@ -12,7 +12,7 @@ unique_ptr PhysicalPlanGenerator::CreatePlan(LogicalMaterializ D_ASSERT(op.children.size() == 2); // Create the working_table that the PhysicalCTE will use for evaluation. - auto working_table = std::make_shared(context, op.children[0]->types); + auto working_table = make_shared(context, op.children[0]->types); // Add the ColumnDataCollection to the context of this PhysicalPlanGenerator recursive_cte_tables[op.table_index] = working_table; diff --git a/src/execution/physical_plan/plan_recursive_cte.cpp b/src/execution/physical_plan/plan_recursive_cte.cpp index 82ddd9c21d3e..89da0cd0706e 100644 --- a/src/execution/physical_plan/plan_recursive_cte.cpp +++ b/src/execution/physical_plan/plan_recursive_cte.cpp @@ -12,7 +12,7 @@ unique_ptr PhysicalPlanGenerator::CreatePlan(LogicalRecursiveC D_ASSERT(op.children.size() == 2); // Create the working_table that the PhysicalRecursiveCTE will use for evaluation. - auto working_table = std::make_shared(context, op.types); + auto working_table = make_shared(context, op.types); // Add the ColumnDataCollection to the context of this PhysicalPlanGenerator recursive_cte_tables[op.table_index] = working_table; diff --git a/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp b/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp index 0890ce829ab4..d8154f3a3615 100644 --- a/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp +++ b/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp @@ -17,7 +17,7 @@ class DuckTableEntry : public TableCatalogEntry { public: //! Create a TableCatalogEntry and initialize storage for it DuckTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, BoundCreateTableInfo &info, - std::shared_ptr inherited_storage = nullptr); + shared_ptr inherited_storage = nullptr); public: unique_ptr AlterEntry(ClientContext &context, AlterInfo &info) override; @@ -64,7 +64,7 @@ class DuckTableEntry : public TableCatalogEntry { private: //! A reference to the underlying storage unit used for this table - std::shared_ptr storage; + shared_ptr storage; //! A list of constraints that are part of this table vector> bound_constraints; //! Manages dependencies of the individual columns of the table diff --git a/src/include/duckdb/common/helper.hpp b/src/include/duckdb/common/helper.hpp index b19b85f6d851..4c57e51a272b 100644 --- a/src/include/duckdb/common/helper.hpp +++ b/src/include/duckdb/common/helper.hpp @@ -65,6 +65,14 @@ make_uniq(ARGS&&... args) // NOLINT: mimic std style return unique_ptr, true>(new DATA_TYPE(std::forward(args)...)); } +template +inline +shared_ptr +make_shared(ARGS&&... args) // NOLINT: mimic std style +{ + return shared_ptr(new DATA_TYPE(std::forward(args)...)); +} + template inline typename TemplatedUniqueIf::templated_unique_single_t diff --git a/src/include/duckdb/common/re2_regex.hpp b/src/include/duckdb/common/re2_regex.hpp index de4b3313e362..ae5c48fd51f7 100644 --- a/src/include/duckdb/common/re2_regex.hpp +++ b/src/include/duckdb/common/re2_regex.hpp @@ -22,7 +22,7 @@ class Regex { } private: - std::shared_ptr regex; + shared_ptr regex; }; struct GroupMatch { diff --git a/src/include/duckdb/common/shared_ptr.hpp b/src/include/duckdb/common/shared_ptr.hpp index 4d97075eb8d3..615273d7c27e 100644 --- a/src/include/duckdb/common/shared_ptr.hpp +++ b/src/include/duckdb/common/shared_ptr.hpp @@ -9,11 +9,114 @@ #pragma once #include +#include + +template +class weak_ptr; namespace duckdb { -using std::make_shared; -using std::shared_ptr; -using std::weak_ptr; +template +class shared_ptr { +private: + template + friend class weak_ptr; + std::shared_ptr internal; + +public: + // Constructors + shared_ptr() : internal() { + } + shared_ptr(std::nullptr_t) : internal(nullptr) { + } // Implicit conversion + template + explicit shared_ptr(U *ptr) : internal(ptr) { + } + shared_ptr(const shared_ptr &other) : internal(other.internal) { + } + shared_ptr(std::shared_ptr other) : internal(std::move(other)) { + } + + // Destructor + ~shared_ptr() = default; + + // Assignment operators + shared_ptr &operator=(const shared_ptr &other) { + internal = other.internal; + return *this; + } + + // Modifiers + void reset() { + internal.reset(); + } + + template + void reset(U *ptr) { + internal.reset(ptr); + } + + template + void reset(U *ptr, Deleter deleter) { + internal.reset(ptr, deleter); + } + + // Observers + T *get() const { + return internal.get(); + } + + long use_count() const { + return internal.use_count(); + } + + explicit operator bool() const noexcept { + return internal.operator bool(); + } + + // Element access + std::__add_lvalue_reference_t operator*() const { + return *internal; + } + + T *operator->() const { + return internal.operator->(); + } + + // Relational operators + template + bool operator==(const shared_ptr &other) const noexcept { + return internal == other.internal; + } + + bool operator==(std::nullptr_t) const noexcept { + return internal == nullptr; + } + + template + bool operator!=(const shared_ptr &other) const noexcept { + return internal != other.internal; + } + + template + bool operator<(const shared_ptr &other) const noexcept { + return internal < other.internal; + } + + template + bool operator<=(const shared_ptr &other) const noexcept { + return internal <= other.internal; + } + + template + bool operator>(const shared_ptr &other) const noexcept { + return internal > other.internal; + } + + template + bool operator>=(const shared_ptr &other) const noexcept { + return internal >= other.internal; + } +}; } // namespace duckdb diff --git a/src/include/duckdb/common/types.hpp b/src/include/duckdb/common/types.hpp index bd5778ec402a..0151bc31a85f 100644 --- a/src/include/duckdb/common/types.hpp +++ b/src/include/duckdb/common/types.hpp @@ -12,6 +12,7 @@ #include "duckdb/common/constants.hpp" #include "duckdb/common/optional_ptr.hpp" #include "duckdb/common/vector.hpp" +#include "duckdb/common/helper.hpp" #include diff --git a/src/include/duckdb/common/weak_ptr.hpp b/src/include/duckdb/common/weak_ptr.hpp new file mode 100644 index 000000000000..bf442e02ad6a --- /dev/null +++ b/src/include/duckdb/common/weak_ptr.hpp @@ -0,0 +1,88 @@ +#pragma once + +#include "duckdb/common/shared_ptr.hpp" +#include + +namespace duckdb { + +template +class weak_ptr { +private: + std::weak_ptr internal; + +public: + // Constructors + weak_ptr() : internal() { + } + template + weak_ptr(const shared_ptr &ptr) : internal(ptr.internal) { + } + weak_ptr(const weak_ptr &other) : internal(other.internal) { + } + + // Destructor + ~weak_ptr() = default; + + // Assignment operators + weak_ptr &operator=(const weak_ptr &other) { + internal = other.internal; + return *this; + } + + template + weak_ptr &operator=(const shared_ptr &ptr) { + internal = ptr; + return *this; + } + + // Modifiers + void reset() { + internal.reset(); + } + + // Observers + long use_count() const { + return internal.use_count(); + } + + bool expired() const { + return internal.expired(); + } + + shared_ptr lock() const { + return internal.lock(); + } + + // Relational operators + template + bool operator==(const weak_ptr &other) const noexcept { + return internal == other.internal; + } + + template + bool operator!=(const weak_ptr &other) const noexcept { + return internal != other.internal; + } + + template + bool operator<(const weak_ptr &other) const noexcept { + return internal < other.internal; + } + + template + bool operator<=(const weak_ptr &other) const noexcept { + return internal <= other.internal; + } + + template + bool operator>(const weak_ptr &other) const noexcept { + return internal > other.internal; + } + + template + bool operator>=(const weak_ptr &other) const noexcept { + return internal >= other.internal; + } +}; + +} // namespace duckdb diff --git a/src/include/duckdb/execution/operator/set/physical_cte.hpp b/src/include/duckdb/execution/operator/set/physical_cte.hpp index 3ff5fb8cf3e1..fc006d4699d6 100644 --- a/src/include/duckdb/execution/operator/set/physical_cte.hpp +++ b/src/include/duckdb/execution/operator/set/physical_cte.hpp @@ -26,7 +26,7 @@ class PhysicalCTE : public PhysicalOperator { vector> cte_scans; - std::shared_ptr working_table; + shared_ptr working_table; idx_t table_index; string ctename; diff --git a/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp b/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp index 88071368eb47..61d82fa09f97 100644 --- a/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp +++ b/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp @@ -29,7 +29,7 @@ class PhysicalRecursiveCTE : public PhysicalOperator { idx_t table_index; bool union_all; - std::shared_ptr working_table; + shared_ptr working_table; shared_ptr recursive_meta_pipeline; public: diff --git a/src/include/duckdb/execution/physical_plan_generator.hpp b/src/include/duckdb/execution/physical_plan_generator.hpp index 8024bc50ed09..24715b143a40 100644 --- a/src/include/duckdb/execution/physical_plan_generator.hpp +++ b/src/include/duckdb/execution/physical_plan_generator.hpp @@ -31,7 +31,7 @@ class PhysicalPlanGenerator { LogicalDependencyList dependencies; //! Recursive CTEs require at least one ChunkScan, referencing the working_table. //! This data structure is used to establish it. - unordered_map> recursive_cte_tables; + unordered_map> recursive_cte_tables; //! Materialized CTE ids must be collected. unordered_map>> materialized_ctes; diff --git a/src/include/duckdb/main/buffered_data/buffered_data.hpp b/src/include/duckdb/main/buffered_data/buffered_data.hpp index d831736d69ac..8065fbee2c73 100644 --- a/src/include/duckdb/main/buffered_data/buffered_data.hpp +++ b/src/include/duckdb/main/buffered_data/buffered_data.hpp @@ -15,6 +15,7 @@ #include "duckdb/common/optional_idx.hpp" #include "duckdb/execution/physical_operator_states.hpp" #include "duckdb/common/enums/pending_execution_result.hpp" +#include "duckdb/common/weak_ptr.hpp" namespace duckdb { diff --git a/src/include/duckdb/main/client_context.hpp b/src/include/duckdb/main/client_context.hpp index 91aad1307c80..9f608273b668 100644 --- a/src/include/duckdb/main/client_context.hpp +++ b/src/include/duckdb/main/client_context.hpp @@ -304,7 +304,7 @@ class ClientContextWrapper { } private: - std::weak_ptr client_context; + weak_ptr client_context; }; } // namespace duckdb diff --git a/src/include/duckdb/main/relation.hpp b/src/include/duckdb/main/relation.hpp index 7d1798712975..c494366208cb 100644 --- a/src/include/duckdb/main/relation.hpp +++ b/src/include/duckdb/main/relation.hpp @@ -36,7 +36,7 @@ class TableRef; class Relation : public std::enable_shared_from_this { public: - Relation(const std::shared_ptr &context, RelationType type) : context(context), type(type) { + Relation(const shared_ptr &context, RelationType type) : context(context), type(type) { } Relation(ClientContextWrapper &context, RelationType type) : context(context.GetContext()), type(type) { } diff --git a/src/include/duckdb/main/relation/query_relation.hpp b/src/include/duckdb/main/relation/query_relation.hpp index 67cfcc063b27..f35a8133cd73 100644 --- a/src/include/duckdb/main/relation/query_relation.hpp +++ b/src/include/duckdb/main/relation/query_relation.hpp @@ -16,7 +16,7 @@ class SelectStatement; class QueryRelation : public Relation { public: - QueryRelation(const std::shared_ptr &context, unique_ptr select_stmt, string alias); + QueryRelation(const shared_ptr &context, unique_ptr select_stmt, string alias); ~QueryRelation() override; unique_ptr select_stmt; diff --git a/src/include/duckdb/main/relation/table_function_relation.hpp b/src/include/duckdb/main/relation/table_function_relation.hpp index 394e92ba716b..9d605c68f74f 100644 --- a/src/include/duckdb/main/relation/table_function_relation.hpp +++ b/src/include/duckdb/main/relation/table_function_relation.hpp @@ -14,11 +14,11 @@ namespace duckdb { class TableFunctionRelation : public Relation { public: - TableFunctionRelation(const std::shared_ptr &context, string name, vector parameters, + TableFunctionRelation(const shared_ptr &context, string name, vector parameters, named_parameter_map_t named_parameters, shared_ptr input_relation_p = nullptr, bool auto_init = true); - TableFunctionRelation(const std::shared_ptr &context, string name, vector parameters, + TableFunctionRelation(const shared_ptr &context, string name, vector parameters, shared_ptr input_relation_p = nullptr, bool auto_init = true); ~TableFunctionRelation() override { } diff --git a/src/include/duckdb/main/relation/table_relation.hpp b/src/include/duckdb/main/relation/table_relation.hpp index 77a950cebe89..a14ce054ba6d 100644 --- a/src/include/duckdb/main/relation/table_relation.hpp +++ b/src/include/duckdb/main/relation/table_relation.hpp @@ -15,7 +15,7 @@ namespace duckdb { class TableRelation : public Relation { public: - TableRelation(const std::shared_ptr &context, unique_ptr description); + TableRelation(const shared_ptr &context, unique_ptr description); unique_ptr description; diff --git a/src/include/duckdb/main/relation/value_relation.hpp b/src/include/duckdb/main/relation/value_relation.hpp index b8aa47c01a48..81fb8c4e1ce7 100644 --- a/src/include/duckdb/main/relation/value_relation.hpp +++ b/src/include/duckdb/main/relation/value_relation.hpp @@ -15,9 +15,9 @@ namespace duckdb { class ValueRelation : public Relation { public: - ValueRelation(const std::shared_ptr &context, const vector> &values, - vector names, string alias = "values"); - ValueRelation(const std::shared_ptr &context, const string &values, vector names, + ValueRelation(const shared_ptr &context, const vector> &values, vector names, + string alias = "values"); + ValueRelation(const shared_ptr &context, const string &values, vector names, string alias = "values"); vector>> expressions; diff --git a/src/include/duckdb/main/relation/view_relation.hpp b/src/include/duckdb/main/relation/view_relation.hpp index 8a8afa26071e..75f63910a520 100644 --- a/src/include/duckdb/main/relation/view_relation.hpp +++ b/src/include/duckdb/main/relation/view_relation.hpp @@ -14,7 +14,7 @@ namespace duckdb { class ViewRelation : public Relation { public: - ViewRelation(const std::shared_ptr &context, string schema_name, string view_name); + ViewRelation(const shared_ptr &context, string schema_name, string view_name); string schema_name; string view_name; diff --git a/src/include/duckdb/planner/bind_context.hpp b/src/include/duckdb/planner/bind_context.hpp index cab1479db700..e4b63f830747 100644 --- a/src/include/duckdb/planner/bind_context.hpp +++ b/src/include/duckdb/planner/bind_context.hpp @@ -41,7 +41,7 @@ class BindContext { explicit BindContext(Binder &binder); //! Keep track of recursive CTE references - case_insensitive_map_t> cte_references; + case_insensitive_map_t> cte_references; public: //! Given a column name, find the matching table it belongs to. Throws an @@ -129,10 +129,10 @@ class BindContext { //! (e.g. "column_name" might return "COLUMN_NAME") string GetActualColumnName(const string &binding, const string &column_name); - case_insensitive_map_t> GetCTEBindings() { + case_insensitive_map_t> GetCTEBindings() { return cte_bindings; } - void SetCTEBindings(case_insensitive_map_t> bindings) { + void SetCTEBindings(case_insensitive_map_t> bindings) { cte_bindings = std::move(bindings); } @@ -165,6 +165,6 @@ class BindContext { vector> using_column_sets; //! The set of CTE bindings - case_insensitive_map_t> cte_bindings; + case_insensitive_map_t> cte_bindings; }; } // namespace duckdb diff --git a/src/include/duckdb/transaction/transaction.hpp b/src/include/duckdb/transaction/transaction.hpp index 1c47725c10dd..dff31db5701c 100644 --- a/src/include/duckdb/transaction/transaction.hpp +++ b/src/include/duckdb/transaction/transaction.hpp @@ -13,6 +13,7 @@ #include "duckdb/transaction/undo_buffer.hpp" #include "duckdb/common/atomic.hpp" #include "duckdb/transaction/transaction_data.hpp" +#include "duckdb/common/weak_ptr.hpp" namespace duckdb { class SequenceCatalogEntry; diff --git a/src/main/relation.cpp b/src/main/relation.cpp index 970ab4a58add..87be11809ff4 100644 --- a/src/main/relation.cpp +++ b/src/main/relation.cpp @@ -277,7 +277,7 @@ void Relation::Create(const string &schema_name, const string &table_name) { } shared_ptr Relation::WriteCSVRel(const string &csv_file, case_insensitive_map_t> options) { - return std::make_shared(shared_from_this(), csv_file, std::move(options)); + return make_shared(shared_from_this(), csv_file, std::move(options)); } void Relation::WriteCSV(const string &csv_file, case_insensitive_map_t> options) { @@ -292,7 +292,7 @@ void Relation::WriteCSV(const string &csv_file, case_insensitive_map_t Relation::WriteParquetRel(const string &parquet_file, case_insensitive_map_t> options) { auto write_parquet = - std::make_shared(shared_from_this(), parquet_file, std::move(options)); + make_shared(shared_from_this(), parquet_file, std::move(options)); return std::move(write_parquet); } diff --git a/src/main/relation/query_relation.cpp b/src/main/relation/query_relation.cpp index f6421601d4a0..0ce867f51574 100644 --- a/src/main/relation/query_relation.cpp +++ b/src/main/relation/query_relation.cpp @@ -6,7 +6,7 @@ namespace duckdb { -QueryRelation::QueryRelation(const std::shared_ptr &context, unique_ptr select_stmt_p, +QueryRelation::QueryRelation(const shared_ptr &context, unique_ptr select_stmt_p, string alias_p) : Relation(context, RelationType::QUERY_RELATION), select_stmt(std::move(select_stmt_p)), alias(std::move(alias_p)) { diff --git a/src/main/relation/read_csv_relation.cpp b/src/main/relation/read_csv_relation.cpp index 1500720e0069..1529de9a7637 100644 --- a/src/main/relation/read_csv_relation.cpp +++ b/src/main/relation/read_csv_relation.cpp @@ -30,7 +30,7 @@ static Value CreateValueFromFileList(const vector &file_list) { return Value::LIST(std::move(files)); } -ReadCSVRelation::ReadCSVRelation(const std::shared_ptr &context, const vector &input, +ReadCSVRelation::ReadCSVRelation(const shared_ptr &context, const vector &input, named_parameter_map_t &&options, string alias_p) : TableFunctionRelation(context, "read_csv_auto", {CreateValueFromFileList(input)}, nullptr, false), alias(std::move(alias_p)) { diff --git a/src/main/relation/table_relation.cpp b/src/main/relation/table_relation.cpp index b4954e3d4fc2..2cdc0d9d945a 100644 --- a/src/main/relation/table_relation.cpp +++ b/src/main/relation/table_relation.cpp @@ -9,7 +9,7 @@ namespace duckdb { -TableRelation::TableRelation(const std::shared_ptr &context, unique_ptr description) +TableRelation::TableRelation(const shared_ptr &context, unique_ptr description) : Relation(context, RelationType::TABLE_RELATION), description(std::move(description)) { } diff --git a/src/main/relation/value_relation.cpp b/src/main/relation/value_relation.cpp index fe611700b42a..3e11ed6bbcdb 100644 --- a/src/main/relation/value_relation.cpp +++ b/src/main/relation/value_relation.cpp @@ -8,7 +8,7 @@ namespace duckdb { -ValueRelation::ValueRelation(const std::shared_ptr &context, const vector> &values, +ValueRelation::ValueRelation(const shared_ptr &context, const vector> &values, vector names_p, string alias_p) : Relation(context, RelationType::VALUE_LIST_RELATION), names(std::move(names_p)), alias(std::move(alias_p)) { // create constant expressions for the values @@ -24,7 +24,7 @@ ValueRelation::ValueRelation(const std::shared_ptr &context, cons context->TryBindRelation(*this, this->columns); } -ValueRelation::ValueRelation(const std::shared_ptr &context, const string &values_list, +ValueRelation::ValueRelation(const shared_ptr &context, const string &values_list, vector names_p, string alias_p) : Relation(context, RelationType::VALUE_LIST_RELATION), names(std::move(names_p)), alias(std::move(alias_p)) { this->expressions = Parser::ParseValuesList(values_list, context->GetParserOptions()); diff --git a/src/main/relation/view_relation.cpp b/src/main/relation/view_relation.cpp index b432f5d3770a..f6f1806727a8 100644 --- a/src/main/relation/view_relation.cpp +++ b/src/main/relation/view_relation.cpp @@ -7,7 +7,7 @@ namespace duckdb { -ViewRelation::ViewRelation(const std::shared_ptr &context, string schema_name_p, string view_name_p) +ViewRelation::ViewRelation(const shared_ptr &context, string schema_name_p, string view_name_p) : Relation(context, RelationType::VIEW_RELATION), schema_name(std::move(schema_name_p)), view_name(std::move(view_name_p)) { context->TryBindRelation(*this, this->columns); diff --git a/src/planner/bind_context.cpp b/src/planner/bind_context.cpp index 6323ebad9b9e..eac1d69c2bd6 100644 --- a/src/planner/bind_context.cpp +++ b/src/planner/bind_context.cpp @@ -520,7 +520,7 @@ void BindContext::AddCTEBinding(idx_t index, const string &alias, const vector(0); + cte_references[alias] = make_shared(0); } void BindContext::AddContext(BindContext other) { diff --git a/src/planner/binder.cpp b/src/planner/binder.cpp index 8239fd292664..75e2a0482364 100644 --- a/src/planner/binder.cpp +++ b/src/planner/binder.cpp @@ -16,6 +16,7 @@ #include "duckdb/planner/operator/logical_projection.hpp" #include "duckdb/planner/operator/logical_sample.hpp" #include "duckdb/parser/query_node/list.hpp" +#include "duckdb/common/helper.hpp" #include From 86803d47be0d016d46e2cc0c9b9a13fc61452d63 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 5 Apr 2024 14:40:23 +0200 Subject: [PATCH 213/603] [Dev] Rename DUCKDB_EXTENSION__GIT_TAG to DUCKDB_EXTENSION__EXT_VERSION --- CMakeLists.txt | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 59b5b97ee580..80a4a77aaf36 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -838,7 +838,7 @@ function(build_loadable_extension NAME PARAMETERS) list(REMOVE_AT FILES 0 1) string(TOUPPER ${NAME} EXTENSION_NAME_UPPERCASE) - build_loadable_extension_directory(${NAME} "extension/${NAME}" "${DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG}" "${PARAMETERS}" ${FILES}) + build_loadable_extension_directory(${NAME} "extension/${NAME}" "${DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_EXT_VERSION}" "${PARAMETERS}" ${FILES}) endfunction() function(build_static_extension NAME PARAMETERS) @@ -893,7 +893,7 @@ function(register_extension NAME DONT_LINK DONT_BUILD LOAD_TESTS PATH INCLUDE_PA set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_PATH ${PATH} PARENT_SCOPE) set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_INCLUDE_PATH ${INCLUDE_PATH} PARENT_SCOPE) set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_TEST_PATH ${TEST_PATH} PARENT_SCOPE) - set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG ${GIT_COMMIT_HASH} PARENT_SCOPE) + set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_EXT_VERSION ${GIT_COMMIT_HASH} PARENT_SCOPE) endfunction() # Downloads the external extension repo at the specified commit and calls register_extension @@ -956,9 +956,9 @@ function(duckdb_extension_load NAME) endif() register_external_extension(${NAME} "${duckdb_extension_load_GIT_URL}" "${duckdb_extension_load_GIT_TAG}" "${duckdb_extension_load_DONT_LINK}" "${duckdb_extension_load_DONT_BUILD}" "${duckdb_extension_load_LOAD_TESTS}" "${duckdb_extension_load_SOURCE_DIR}" "${duckdb_extension_load_INCLUDE_DIR}" "${duckdb_extension_load_TEST_DIR}" "${duckdb_extension_load_APPLY_PATCHES}" "${duckdb_extension_load_SUBMODULES}") if (NOT "${duckdb_extension_load_EXTENSION_VERSION}" STREQUAL "") - set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG "${duckdb_extension_load_EXTENSION_VERSION}" PARENT_SCOPE) + set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_EXT_VERSION "${duckdb_extension_load_EXTENSION_VERSION}" PARENT_SCOPE) else() - set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG "${duckdb_extension_load_GIT_TAG}" PARENT_SCOPE) + set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_EXT_VERSION "${duckdb_extension_load_GIT_TAG}" PARENT_SCOPE) endif() elseif (NOT "${duckdb_extension_load_SOURCE_DIR}" STREQUAL "") # Local extension, custom path @@ -979,9 +979,9 @@ function(duckdb_extension_load NAME) endif() if (NOT "${duckdb_extension_load_EXTENSION_VERSION}" STREQUAL "") - set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG "${duckdb_extension_load_EXTENSION_VERSION}" PARENT_SCOPE) - elseif (NOT "${duckdb_extension_load_GIT_TAG}" STREQUAL "") - set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG "${duckdb_extension_load_GIT_TAG}" PARENT_SCOPE) + set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_EXT_VERSION "${duckdb_extension_load_EXTENSION_VERSION}" PARENT_SCOPE) + elseif (NOT "${duckdb_extension_load_EXT_VERSION}" STREQUAL "") + set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_EXT_VERSION "${duckdb_extension_load_GIT_TAG}" PARENT_SCOPE) else() find_package(Git) if(Git_FOUND) @@ -991,9 +991,9 @@ function(duckdb_extension_load NAME) RESULT_VARIABLE GIT_RESULT OUTPUT_VARIABLE GIT_COMMIT OUTPUT_STRIP_TRAILING_WHITESPACE) - set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG "${GIT_COMMIT}" PARENT_SCOPE) + set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_EXT_VERSION "${GIT_COMMIT}" PARENT_SCOPE) else() - set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG "" PARENT_SCOPE) + set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_EXT_VERSION "" PARENT_SCOPE) endif() endif() register_extension(${NAME} "${duckdb_extension_load_DONT_LINK}" "${duckdb_extension_load_DONT_BUILD}" "${duckdb_extension_load_LOAD_TESTS}" "${duckdb_extension_load_SOURCE_DIR}" "${INCLUDE_PATH_DEFAULT}" "${TEST_PATH_DEFAULT}") @@ -1001,12 +1001,12 @@ function(duckdb_extension_load NAME) # Local extension, default path message(STATUS "Load extension '${NAME}' from '${CMAKE_CURRENT_SOURCE_DIR}/extension_external'") register_extension(${NAME} ${duckdb_extension_load_DONT_LINK} "${duckdb_extension_load_DONT_BUILD}" "${duckdb_extension_load_LOAD_TESTS}" "${CMAKE_CURRENT_SOURCE_DIR}/extension_external/${NAME}" "${CMAKE_CURRENT_SOURCE_DIR}/extension_external/${NAME}/src/include" "${CMAKE_CURRENT_SOURCE_DIR}/extension_external/${NAME}/test/sql") - set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG "${DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG}" PARENT_SCOPE) + set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_EXT_VERSION "${DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_EXT_VERSION}" PARENT_SCOPE) else() # Local extension, default path message(STATUS "Load extension '${NAME}' from '${CMAKE_CURRENT_SOURCE_DIR}/extensions'") register_extension(${NAME} ${duckdb_extension_load_DONT_LINK} "${duckdb_extension_load_DONT_BUILD}" "${duckdb_extension_load_LOAD_TESTS}" "${CMAKE_CURRENT_SOURCE_DIR}/extension/${NAME}" "${CMAKE_CURRENT_SOURCE_DIR}/extension/${NAME}/include" "${CMAKE_CURRENT_SOURCE_DIR}/extension/${NAME}/test/sql") - set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG "${DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_GIT_TAG}" PARENT_SCOPE) + set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_EXT_VERSION "${DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_EXT_VERSION}" PARENT_SCOPE) endif() # Propagate variables set by register_extension From e5faa31f9d3efb6c09b67d4003070c372068de18 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 5 Apr 2024 15:13:11 +0200 Subject: [PATCH 214/603] fts extension: Avoid implicit Load, use LoadInternal pattern --- extension/fts/fts_extension.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/extension/fts/fts_extension.cpp b/extension/fts/fts_extension.cpp index a48c996ffdc8..0efec7b03b60 100644 --- a/extension/fts/fts_extension.cpp +++ b/extension/fts/fts_extension.cpp @@ -46,7 +46,7 @@ static void StemFunction(DataChunk &args, ExpressionState &state, Vector &result }); } -void FtsExtension::Load(DuckDB &db) { +static void LoadInternal(DuckDB &db) { auto &db_instance = *db.instance; ScalarFunction stem_func("stem", {LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::VARCHAR, StemFunction); @@ -68,6 +68,10 @@ void FtsExtension::Load(DuckDB &db) { ExtensionUtil::RegisterFunction(db_instance, drop_fts_index_func); } +void FtsExtension::Load(DuckDB &db) { + LoadInternal(db); +} + std::string FtsExtension::Name() { return "fts"; } @@ -78,7 +82,7 @@ extern "C" { DUCKDB_EXTENSION_API void fts_init(duckdb::DatabaseInstance &db) { duckdb::DuckDB db_wrapper(db); - db_wrapper.LoadExtension(); + duckdb::LoadInternal(db_wrapper); } DUCKDB_EXTENSION_API const char *fts_version() { From 57ed3d544bbcf953b3cee70a93086682c892c278 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 5 Apr 2024 09:37:49 +0200 Subject: [PATCH 215/603] tpch extension: Avoid implicit Load, use LoadInternal pattern --- extension/tpch/tpch_extension.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/extension/tpch/tpch_extension.cpp b/extension/tpch/tpch_extension.cpp index 68548438ab20..fe73e3ae8792 100644 --- a/extension/tpch/tpch_extension.cpp +++ b/extension/tpch/tpch_extension.cpp @@ -156,7 +156,7 @@ static string PragmaTpchQuery(ClientContext &context, const FunctionParameters & return tpch::DBGenWrapper::GetQuery(index); } -void TpchExtension::Load(DuckDB &db) { +static void LoadInternal(DuckDB &db) { auto &db_instance = *db.instance; TableFunction dbgen_func("dbgen", {}, DbgenFunction, DbgenBind); @@ -182,6 +182,10 @@ void TpchExtension::Load(DuckDB &db) { ExtensionUtil::RegisterFunction(db_instance, tpch_query_answer_func); } +void TpchExtension::Load(DuckDB &db) { + LoadInternal(db); +} + std::string TpchExtension::GetQuery(int query) { return tpch::DBGenWrapper::GetQuery(query); } @@ -200,7 +204,7 @@ extern "C" { DUCKDB_EXTENSION_API void tpch_init(duckdb::DatabaseInstance &db) { duckdb::DuckDB db_wrapper(db); - db_wrapper.LoadExtension(); + duckdb::LoadInternal(db_wrapper); } DUCKDB_EXTENSION_API const char *tpch_version() { From 1427054e8013954461a9a2b841891af19df97745 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 5 Apr 2024 09:38:10 +0200 Subject: [PATCH 216/603] tpcds extension: Avoid implicit Load, use LoadInternal pattern --- extension/tpcds/tpcds_extension.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/extension/tpcds/tpcds_extension.cpp b/extension/tpcds/tpcds_extension.cpp index c6157b67ae93..708165525d3f 100644 --- a/extension/tpcds/tpcds_extension.cpp +++ b/extension/tpcds/tpcds_extension.cpp @@ -146,7 +146,7 @@ static string PragmaTpcdsQuery(ClientContext &context, const FunctionParameters return tpcds::DSDGenWrapper::GetQuery(index); } -void TpcdsExtension::Load(DuckDB &db) { +static void LoadInternal(DuckDB &db) { auto &db_instance = *db.instance; TableFunction dsdgen_func("dsdgen", {}, DsdgenFunction, DsdgenBind); @@ -172,6 +172,10 @@ void TpcdsExtension::Load(DuckDB &db) { ExtensionUtil::RegisterFunction(db_instance, tpcds_query_answer_func); } +void TpcdsExtension::Load(DuckDB &db) { + LoadInternal(db); +} + std::string TpcdsExtension::GetQuery(int query) { return tpcds::DSDGenWrapper::GetQuery(query); } @@ -189,7 +193,7 @@ std::string TpcdsExtension::Name() { extern "C" { DUCKDB_EXTENSION_API void tpcds_init(duckdb::DatabaseInstance &db) { duckdb::DuckDB db_wrapper(db); - db_wrapper.LoadExtension(); + duckdb::LoadInternal(db_wrapper); } DUCKDB_EXTENSION_API const char *tpcds_version() { From 2685cbb674c863584caf94e6cc80db212263442e Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 5 Apr 2024 09:39:56 +0200 Subject: [PATCH 217/603] icu extension: Avoid implicit Load, use LoadInternal pattern --- extension/icu/icu_extension.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/extension/icu/icu_extension.cpp b/extension/icu/icu_extension.cpp index a798f2f2ea45..bf09f4e1970c 100644 --- a/extension/icu/icu_extension.cpp +++ b/extension/icu/icu_extension.cpp @@ -221,7 +221,7 @@ static void SetICUCalendar(ClientContext &context, SetScope scope, Value ¶me } } -void IcuExtension::Load(DuckDB &ddb) { +static void LoadInternal(DuckDB &ddb) { auto &db = *ddb.instance; // iterate over all the collations @@ -275,6 +275,10 @@ void IcuExtension::Load(DuckDB &ddb) { ExtensionUtil::RegisterFunction(db, cal_names); } +void IcuExtension::Load(DuckDB &ddb) { + LoadInternal(ddb); +} + std::string IcuExtension::Name() { return "icu"; } @@ -285,7 +289,7 @@ extern "C" { DUCKDB_EXTENSION_API void icu_init(duckdb::DatabaseInstance &db) { // NOLINT duckdb::DuckDB db_wrapper(db); - db_wrapper.LoadExtension(); + duckdb::LoadInternal(db_wrapper); } DUCKDB_EXTENSION_API const char *icu_version() { // NOLINT From 2212e02e2a027a2afb3a4eb78985aca091b44d3d Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Fri, 5 Apr 2024 15:30:57 +0200 Subject: [PATCH 218/603] spell check on comments --- src/optimizer/pushdown/pushdown_window.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/optimizer/pushdown/pushdown_window.cpp b/src/optimizer/pushdown/pushdown_window.cpp index 46e98ace3ea5..5bad0d1357fa 100644 --- a/src/optimizer/pushdown/pushdown_window.cpp +++ b/src/optimizer/pushdown/pushdown_window.cpp @@ -80,7 +80,7 @@ unique_ptr FilterPushdown::PushdownWindow(unique_ptr Date: Fri, 5 Apr 2024 15:42:19 +0200 Subject: [PATCH 219/603] Extension metadata: Do not use GIT_TAG when SOURCE_DIR is present --- CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 80a4a77aaf36..8c6ca4e082b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -980,8 +980,6 @@ function(duckdb_extension_load NAME) if (NOT "${duckdb_extension_load_EXTENSION_VERSION}" STREQUAL "") set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_EXT_VERSION "${duckdb_extension_load_EXTENSION_VERSION}" PARENT_SCOPE) - elseif (NOT "${duckdb_extension_load_EXT_VERSION}" STREQUAL "") - set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_EXT_VERSION "${duckdb_extension_load_GIT_TAG}" PARENT_SCOPE) else() find_package(Git) if(Git_FOUND) From af007bd453d885b6d73bb966ce2e9d0b60ac738c Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Fri, 5 Apr 2024 17:04:49 +0200 Subject: [PATCH 220/603] ORDER BY COLLATE working --- .../common/exception/binder_exception.hpp | 2 +- src/include/duckdb/planner/binder.hpp | 3 +- .../binder/query_node/bind_select_node.cpp | 38 ++++++++++++++----- .../expression_binder/order_binder.cpp | 2 +- .../expression_binder/select_bind_state.cpp | 2 - .../test_collate_orderby_column_number.test | 14 +++++-- 6 files changed, 44 insertions(+), 17 deletions(-) diff --git a/src/include/duckdb/common/exception/binder_exception.hpp b/src/include/duckdb/common/exception/binder_exception.hpp index 653c5e80bd94..cd5bfe9ff1e7 100644 --- a/src/include/duckdb/common/exception/binder_exception.hpp +++ b/src/include/duckdb/common/exception/binder_exception.hpp @@ -31,7 +31,7 @@ class BinderException : public Exception { } template explicit BinderException(const Expression &expr, const string &msg, ARGS... params) - : BinderException(ConstructMessage(msg, params...), Exception::InitializeExtraInfo(expr)) { + : BinderException(ConstructMessage(msg, params...), Exception::InitializeExtraInfo(expr)) { } template explicit BinderException(QueryErrorContext error_context, const string &msg, ARGS... params) diff --git a/src/include/duckdb/planner/binder.hpp b/src/include/duckdb/planner/binder.hpp index 3b67155e23ec..4d013873cc95 100644 --- a/src/include/duckdb/planner/binder.hpp +++ b/src/include/duckdb/planner/binder.hpp @@ -325,7 +325,8 @@ class Binder : public std::enable_shared_from_this { BoundStatement BindCopyFrom(CopyStatement &stmt); void PrepareModifiers(OrderBinder &order_binder, QueryNode &statement, BoundQueryNode &result); - void BindModifiers(BoundQueryNode &result, idx_t table_index, const vector &sql_types, const SelectBindState &bind_state); + void BindModifiers(BoundQueryNode &result, idx_t table_index, const vector &sql_types, + const SelectBindState &bind_state); unique_ptr BindLimit(OrderBinder &order_binder, LimitModifier &limit_mod); unique_ptr BindLimitPercent(OrderBinder &order_binder, LimitPercentModifier &limit_mod); diff --git a/src/planner/binder/query_node/bind_select_node.cpp b/src/planner/binder/query_node/bind_select_node.cpp index 8e2a58c4b26e..d2783dadac67 100644 --- a/src/planner/binder/query_node/bind_select_node.cpp +++ b/src/planner/binder/query_node/bind_select_node.cpp @@ -243,16 +243,24 @@ static void AssignReturnType(unique_ptr &expr, const vector FinalizeBindOrderExpression(unique_ptr expr, idx_t table_index, const vector &sql_types, const SelectBindState &bind_state) { +unique_ptr CreateOrderExpression(unique_ptr expr, const vector &sql_types, + idx_t table_index, idx_t index) { + if (index >= sql_types.size()) { + throw BinderException(*expr, "ORDER term out of range - should be between 1 and %lld", sql_types.size()); + } + return make_uniq(std::move(expr->alias), sql_types[index], + ColumnBinding(table_index, index)); +} + +unique_ptr FinalizeBindOrderExpression(unique_ptr expr, idx_t table_index, + const vector &sql_types, + const SelectBindState &bind_state) { auto &constant = expr->Cast(); - switch(constant.value.type().id()) { + switch (constant.value.type().id()) { case LogicalTypeId::UBIGINT: { // index auto index = UBigIntValue::Get(constant.value); - if (index >= sql_types.size()) { - throw BinderException(*expr, "ORDER term out of range - should be between 1 and %lld", sql_types.size()); - } - return make_uniq(std::move(expr->alias), sql_types[index], ColumnBinding(table_index, index)); + return CreateOrderExpression(std::move(expr), sql_types, table_index, index); } case LogicalTypeId::VARCHAR: { // ORDER BY ALL @@ -260,14 +268,26 @@ unique_ptr FinalizeBindOrderExpression(unique_ptr expr, } case LogicalTypeId::STRUCT: { // collation - throw InternalException("FIXME: collate struct by number"); + auto &struct_values = StructValue::GetChildren(constant.value); + if (struct_values.size() != 2) { + throw InternalException("Expected two children: index and collation"); + } + auto index = UBigIntValue::Get(struct_values[0]); + auto collation = StringValue::Get(struct_values[1]); + if (sql_types[index].id() != LogicalTypeId::VARCHAR) { + throw BinderException(*expr, "COLLATE can only be applied to varchar columns"); + } + auto result = CreateOrderExpression(std::move(expr), sql_types, table_index, index); + result->return_type = LogicalType::VARCHAR_COLLATION(std::move(collation)); + return result; } default: throw InternalException("Unknown type in FinalizeBindOrderExpression"); } } -void Binder::BindModifiers(BoundQueryNode &result, idx_t table_index, const vector &sql_types, const SelectBindState &bind_state) { +void Binder::BindModifiers(BoundQueryNode &result, idx_t table_index, const vector &sql_types, + const SelectBindState &bind_state) { for (auto &bound_mod : result.modifiers) { switch (bound_mod->type) { case ResultModifierType::DISTINCT_MODIFIER: { @@ -306,7 +326,7 @@ void Binder::BindModifiers(BoundQueryNode &result, idx_t table_index, const vect auto order_type = order.orders[0].type; auto null_order = order.orders[0].null_order; order.orders.clear(); - for(idx_t i = 0; i < sql_types.size(); i++) { + for (idx_t i = 0; i < sql_types.size(); i++) { auto expr = make_uniq(sql_types[i], ColumnBinding(table_index, i)); order.orders.emplace_back(order_type, null_order, std::move(expr)); } diff --git a/src/planner/expression_binder/order_binder.cpp b/src/planner/expression_binder/order_binder.cpp index 6335f3949c72..a80ceb07f213 100644 --- a/src/planner/expression_binder/order_binder.cpp +++ b/src/planner/expression_binder/order_binder.cpp @@ -15,7 +15,7 @@ namespace duckdb { OrderBinder::OrderBinder(vector binders, SelectBindState &bind_state) - : binders(std::move(binders)), extra_list(nullptr), bind_state(bind_state) { + : binders(std::move(binders)), extra_list(nullptr), bind_state(bind_state) { } OrderBinder::OrderBinder(vector binders, SelectNode &node, SelectBindState &bind_state) : binders(std::move(binders)), bind_state(bind_state) { diff --git a/src/planner/expression_binder/select_bind_state.cpp b/src/planner/expression_binder/select_bind_state.cpp index 134a4d819b62..a987258548e4 100644 --- a/src/planner/expression_binder/select_bind_state.cpp +++ b/src/planner/expression_binder/select_bind_state.cpp @@ -31,11 +31,9 @@ bool SelectBindState::AliasHasSubquery(idx_t index) { } void SelectBindState::AddExpandedColumn(idx_t expand_count) { - } void SelectBindState::AddRegularColumn() { - } } // namespace duckdb diff --git a/test/sql/collate/test_collate_orderby_column_number.test b/test/sql/collate/test_collate_orderby_column_number.test index 7cb7b885536f..fb409e61328e 100644 --- a/test/sql/collate/test_collate_orderby_column_number.test +++ b/test/sql/collate/test_collate_orderby_column_number.test @@ -42,7 +42,7 @@ SELECT 'a' FROM ORDER BY -0 COLLATE NOCASE; ---- statement ok -CREATE TABLE collate_test(s VARCHAR) +CREATE TABLE collate_test(s VARCHAR); statement ok INSERT INTO collate_test VALUES ('ã'),('B'),('a'),('A'); @@ -96,14 +96,22 @@ NULL NULL -statement error +query I SELECT (SELECT s) FROM collate_test ORDER BY 1 COLLATE NOCASE; ---- +a +A +B +ã statement error SELECT (SELECT s) AS c1 FROM collate_test ORDER BY c11 COLLATE NOCASE; ---- -statement error +query I SELECT concat('a', (SELECT s)) FROM collate_test ORDER BY 1 COLLATE NOCASE; ---- +aa +aA +aB +aã From 241aa450122953db4d2fc06788e008295b330891 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Fri, 5 Apr 2024 17:28:01 +0200 Subject: [PATCH 221/603] LIMIT/OFFSET also uses the OrderBinder for subqueries --- .../binder/query_node/bind_select_node.cpp | 32 +++++++++++-------- test/sql/order/negative_offset.test | 6 +++- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/planner/binder/query_node/bind_select_node.cpp b/src/planner/binder/query_node/bind_select_node.cpp index d2783dadac67..d3becb272e22 100644 --- a/src/planner/binder/query_node/bind_select_node.cpp +++ b/src/planner/binder/query_node/bind_select_node.cpp @@ -59,7 +59,6 @@ BoundLimitNode Binder::BindLimitValue(OrderBinder &order_binder, unique_ptrIsFoldable()) { @@ -232,17 +231,6 @@ void Binder::PrepareModifiers(OrderBinder &order_binder, QueryNode &statement, B } } -static void AssignReturnType(unique_ptr &expr, const vector &sql_types) { - if (!expr) { - return; - } - if (expr->type != ExpressionType::BOUND_COLUMN_REF) { - return; - } - auto &bound_colref = expr->Cast(); - bound_colref.return_type = sql_types[bound_colref.binding.column_index]; -} - unique_ptr CreateOrderExpression(unique_ptr expr, const vector &sql_types, idx_t table_index, idx_t index) { if (index >= sql_types.size()) { @@ -286,6 +274,22 @@ unique_ptr FinalizeBindOrderExpression(unique_ptr expr, } } +static void AssignReturnType(unique_ptr &expr, idx_t table_index, + const vector &sql_types, + const SelectBindState &bind_state) { + if (!expr) { + return; + } + if (expr->type == ExpressionType::VALUE_CONSTANT) { + expr = FinalizeBindOrderExpression(std::move(expr), table_index, sql_types, bind_state); + } + if (expr->type != ExpressionType::BOUND_COLUMN_REF) { + return; + } + auto &bound_colref = expr->Cast(); + bound_colref.return_type = sql_types[bound_colref.binding.column_index]; +} + void Binder::BindModifiers(BoundQueryNode &result, idx_t table_index, const vector &sql_types, const SelectBindState &bind_state) { for (auto &bound_mod : result.modifiers) { @@ -307,8 +311,8 @@ void Binder::BindModifiers(BoundQueryNode &result, idx_t table_index, const vect } case ResultModifierType::LIMIT_MODIFIER: { auto &limit = bound_mod->Cast(); - AssignReturnType(limit.limit_val.GetExpression(), sql_types); - AssignReturnType(limit.offset_val.GetExpression(), sql_types); + AssignReturnType(limit.limit_val.GetExpression(), table_index, sql_types, bind_state); + AssignReturnType(limit.offset_val.GetExpression(), table_index, sql_types, bind_state); break; } case ResultModifierType::ORDER_MODIFIER: { diff --git a/test/sql/order/negative_offset.test b/test/sql/order/negative_offset.test index aebc8bb35848..d19d58c26278 100644 --- a/test/sql/order/negative_offset.test +++ b/test/sql/order/negative_offset.test @@ -8,21 +8,25 @@ PRAGMA enable_verification statement error SELECT * FROM generate_series(0,10,1) LIMIT 3 OFFSET -1; ---- +cannot be negative statement error SELECT * FROM generate_series(0,10,1) LIMIT -3; ---- +cannot be negative statement error SELECT * FROM generate_series(0,10,1) LIMIT -1% ---- +out of range statement ok -CREATE TABLE integers AS SELECT -1 k +CREATE TABLE integers AS SELECT -1 k; statement error SELECT * FROM generate_series(0,10,1) LIMIT (SELECT k FROM integers); ---- +out of range statement error SELECT * FROM generate_series(0,10,1) LIMIT 1 OFFSET (SELECT k FROM integers); From 0cbaa36b84fffed565372c1faac03d229d885c23 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 5 Apr 2024 17:38:35 +0200 Subject: [PATCH 222/603] changes to get duckdb::shared_ptr working, almost in a functional state --- scripts/format.py | 2 + .../catalog_entry/duck_table_entry.cpp | 15 +- src/common/allocator.cpp | 2 +- src/common/arrow/arrow_wrapper.cpp | 2 +- src/common/extra_type_info.cpp | 8 +- src/common/http_state.cpp | 6 +- src/common/re2_regex.cpp | 2 +- src/common/symbols.cpp | 34 ++-- src/common/types.cpp | 28 ++-- .../types/column/column_data_collection.cpp | 14 +- .../column/column_data_collection_segment.cpp | 2 +- .../types/column/partitioned_column_data.cpp | 4 +- .../types/row/partitioned_tuple_data.cpp | 4 +- .../types/row/tuple_data_collection.cpp | 4 +- src/common/types/value.cpp | 36 ++--- src/common/types/vector_cache.cpp | 6 +- src/execution/aggregate_hashtable.cpp | 2 +- src/execution/index/art/art.cpp | 3 +- .../operator/aggregate/aggregate_object.cpp | 2 +- .../aggregate/physical_hash_aggregate.cpp | 4 +- .../physical_ungrouped_aggregate.cpp | 2 +- .../operator/aggregate/physical_window.cpp | 2 +- .../csv_scanner/buffer_manager/csv_buffer.cpp | 8 +- .../buffer_manager/csv_buffer_manager.cpp | 2 +- .../scanner/string_value_scanner.cpp | 15 +- .../csv_scanner/sniffer/csv_sniffer.cpp | 4 +- .../table_function/csv_file_scanner.cpp | 20 +-- .../table_function/global_csv_state.cpp | 18 +-- .../helper/physical_buffered_collector.cpp | 2 +- .../operator/join/physical_asof_join.cpp | 2 +- .../operator/join/physical_hash_join.cpp | 4 +- .../operator/join/physical_range_join.cpp | 2 +- .../operator/order/physical_order.cpp | 2 +- .../physical_batch_copy_to_file.cpp | 2 +- .../persistent/physical_copy_to_file.cpp | 2 +- .../schema/physical_create_art_index.cpp | 2 +- .../operator/set/physical_recursive_cte.cpp | 2 +- src/execution/physical_plan/plan_cte.cpp | 2 +- .../physical_plan/plan_recursive_cte.cpp | 2 +- src/function/table/copy_csv.cpp | 2 +- src/function/table/read_csv.cpp | 2 +- src/function/table/sniff_csv.cpp | 2 +- .../duckdb/common/enable_shared_from_this.ipp | 40 +++++ src/include/duckdb/common/exception.hpp | 1 - src/include/duckdb/common/helper.hpp | 9 +- src/include/duckdb/common/http_state.hpp | 2 +- .../duckdb/common/multi_file_reader.hpp | 2 +- src/include/duckdb/common/re2_regex.hpp | 5 +- src/include/duckdb/common/shared_ptr.hpp | 121 ++------------ src/include/duckdb/common/shared_ptr.ipp | 150 ++++++++++++++++++ src/include/duckdb/common/types.hpp | 2 +- .../duckdb/common/types/selection_vector.hpp | 2 +- src/include/duckdb/common/unique_ptr.hpp | 12 +- .../common/{weak_ptr.hpp => weak_ptr.ipp} | 12 +- .../main/buffered_data/buffered_data.hpp | 2 +- src/include/duckdb/main/client_context.hpp | 2 +- src/include/duckdb/main/database.hpp | 2 +- src/include/duckdb/main/relation.hpp | 2 +- src/include/duckdb/parallel/event.hpp | 2 +- src/include/duckdb/parallel/interrupt.hpp | 1 + src/include/duckdb/parallel/meta_pipeline.hpp | 2 +- src/include/duckdb/parallel/pipeline.hpp | 2 +- src/include/duckdb/parallel/task.hpp | 2 +- src/include/duckdb/planner/binder.hpp | 4 +- src/include/duckdb/storage/object_cache.hpp | 6 +- .../duckdb/storage/serialization/types.json | 2 +- .../duckdb/transaction/local_storage.hpp | 2 +- .../duckdb/transaction/transaction.hpp | 2 +- src/main/capi/table_function-c.cpp | 2 +- src/main/client_context.cpp | 2 +- src/main/client_data.cpp | 4 +- src/main/connection.cpp | 21 +-- src/main/database.cpp | 4 +- src/main/db_instance_cache.cpp | 2 +- src/main/relation.cpp | 58 +++---- src/main/relation/read_csv_relation.cpp | 2 +- src/main/relation/table_relation.cpp | 6 +- src/parallel/executor.cpp | 17 +- src/parallel/meta_pipeline.cpp | 4 +- src/planner/bind_context.cpp | 4 +- src/planner/binder.cpp | 2 +- src/planner/bound_parameter_map.cpp | 2 +- src/planner/planner.cpp | 2 +- src/storage/buffer/block_manager.cpp | 2 +- src/storage/checkpoint_manager.cpp | 2 +- src/storage/data_table.cpp | 4 +- src/storage/local_storage.cpp | 14 +- src/storage/serialization/serialize_types.cpp | 2 +- src/storage/standard_buffer_manager.cpp | 8 +- src/storage/statistics/column_statistics.cpp | 6 +- src/storage/table/row_group.cpp | 2 +- src/storage/table/row_group_collection.cpp | 8 +- src/storage/table/row_version_manager.cpp | 2 +- src/storage/wal_replay.cpp | 2 +- 94 files changed, 483 insertions(+), 373 deletions(-) create mode 100644 src/include/duckdb/common/enable_shared_from_this.ipp create mode 100644 src/include/duckdb/common/shared_ptr.ipp rename src/include/duckdb/common/{weak_ptr.hpp => weak_ptr.ipp} (87%) diff --git a/scripts/format.py b/scripts/format.py index 40cfbb95903b..e053a021f0e4 100644 --- a/scripts/format.py +++ b/scripts/format.py @@ -41,6 +41,7 @@ extensions = [ '.cpp', + '.ipp', '.c', '.hpp', '.h', @@ -240,6 +241,7 @@ def get_changed_files(revision): format_commands = { '.cpp': cpp_format_command, + '.ipp': cpp_format_command, '.c': cpp_format_command, '.hpp': cpp_format_command, '.h': cpp_format_command, diff --git a/src/catalog/catalog_entry/duck_table_entry.cpp b/src/catalog/catalog_entry/duck_table_entry.cpp index 05dd9410e220..53ca0cbba0c3 100644 --- a/src/catalog/catalog_entry/duck_table_entry.cpp +++ b/src/catalog/catalog_entry/duck_table_entry.cpp @@ -83,8 +83,9 @@ DuckTableEntry::DuckTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, Bou for (auto &col_def : columns.Physical()) { storage_columns.push_back(col_def.Copy()); } - storage = make_shared(catalog.GetAttached(), StorageManager::Get(catalog).GetTableIOManager(&info), - schema.name, name, std::move(storage_columns), std::move(info.data)); + storage = + make_refcounted(catalog.GetAttached(), StorageManager::Get(catalog).GetTableIOManager(&info), + schema.name, name, std::move(storage_columns), std::move(info.data)); // create the unique indexes for the UNIQUE and PRIMARY KEY and FOREIGN KEY constraints idx_t indexes_idx = 0; @@ -344,7 +345,7 @@ unique_ptr DuckTableEntry::AddColumn(ClientContext &context, AddCo auto binder = Binder::CreateBinder(context); auto bound_create_info = binder->BindCreateTableInfo(std::move(create_info), schema); auto new_storage = - make_shared(context, *storage, info.new_column, *bound_create_info->bound_defaults.back()); + make_refcounted(context, *storage, info.new_column, *bound_create_info->bound_defaults.back()); return make_uniq(catalog, schema, *bound_create_info, new_storage); } @@ -480,7 +481,7 @@ unique_ptr DuckTableEntry::RemoveColumn(ClientContext &context, Re return make_uniq(catalog, schema, *bound_create_info, storage); } auto new_storage = - make_shared(context, *storage, columns.LogicalToPhysical(LogicalIndex(removed_index)).index); + make_refcounted(context, *storage, columns.LogicalToPhysical(LogicalIndex(removed_index)).index); return make_uniq(catalog, schema, *bound_create_info, new_storage); } @@ -548,7 +549,7 @@ unique_ptr DuckTableEntry::SetNotNull(ClientContext &context, SetN } // Return with new storage info. Note that we need the bound column index here. - auto new_storage = make_shared( + auto new_storage = make_refcounted( context, *storage, make_uniq(columns.LogicalToPhysical(LogicalIndex(not_null_idx)))); return make_uniq(catalog, schema, *bound_create_info, new_storage); } @@ -659,8 +660,8 @@ unique_ptr DuckTableEntry::ChangeColumnType(ClientContext &context } auto new_storage = - make_shared(context, *storage, columns.LogicalToPhysical(LogicalIndex(change_idx)).index, - info.target_type, std::move(storage_oids), *bound_expression); + make_refcounted(context, *storage, columns.LogicalToPhysical(LogicalIndex(change_idx)).index, + info.target_type, std::move(storage_oids), *bound_expression); auto result = make_uniq(catalog, schema, *bound_create_info, new_storage); return std::move(result); } diff --git a/src/common/allocator.cpp b/src/common/allocator.cpp index 5487578f0258..835aba386ff2 100644 --- a/src/common/allocator.cpp +++ b/src/common/allocator.cpp @@ -195,7 +195,7 @@ data_ptr_t Allocator::DefaultReallocate(PrivateAllocatorData *private_data, data } shared_ptr &Allocator::DefaultAllocatorReference() { - static shared_ptr DEFAULT_ALLOCATOR = make_shared(); + static shared_ptr DEFAULT_ALLOCATOR = make_refcounted(); return DEFAULT_ALLOCATOR; } diff --git a/src/common/arrow/arrow_wrapper.cpp b/src/common/arrow/arrow_wrapper.cpp index d439d99079b1..1bcc48ce91b6 100644 --- a/src/common/arrow/arrow_wrapper.cpp +++ b/src/common/arrow/arrow_wrapper.cpp @@ -50,7 +50,7 @@ void ArrowArrayStreamWrapper::GetSchema(ArrowSchemaWrapper &schema) { } shared_ptr ArrowArrayStreamWrapper::GetNextChunk() { - auto current_chunk = make_shared(); + auto current_chunk = make_refcounted(); if (arrow_array_stream.get_next(&arrow_array_stream, ¤t_chunk->arrow_array)) { // LCOV_EXCL_START throw InvalidInputException("arrow_scan: get_next failed(): %s", string(GetError())); } // LCOV_EXCL_STOP diff --git a/src/common/extra_type_info.cpp b/src/common/extra_type_info.cpp index f8d27d86e8f0..c5c3ca1b6b8f 100644 --- a/src/common/extra_type_info.cpp +++ b/src/common/extra_type_info.cpp @@ -190,7 +190,7 @@ struct EnumTypeInfoTemplated : public EnumTypeInfo { deserializer.ReadList(201, "values", [&](Deserializer::List &list, idx_t i) { strings[i] = StringVector::AddStringOrBlob(values_insert_order, list.ReadElement()); }); - return make_shared(values_insert_order, size); + return make_refcounted(values_insert_order, size); } const string_map_t &GetValues() const { @@ -227,13 +227,13 @@ LogicalType EnumTypeInfo::CreateType(Vector &ordered_data, idx_t size) { auto enum_internal_type = EnumTypeInfo::DictType(size); switch (enum_internal_type) { case PhysicalType::UINT8: - info = make_shared>(ordered_data, size); + info = make_refcounted>(ordered_data, size); break; case PhysicalType::UINT16: - info = make_shared>(ordered_data, size); + info = make_refcounted>(ordered_data, size); break; case PhysicalType::UINT32: - info = make_shared>(ordered_data, size); + info = make_refcounted>(ordered_data, size); break; default: throw InternalException("Invalid Physical Type for ENUMs"); diff --git a/src/common/http_state.cpp b/src/common/http_state.cpp index 9d583fd8f5f2..880454b87a75 100644 --- a/src/common/http_state.cpp +++ b/src/common/http_state.cpp @@ -62,14 +62,14 @@ shared_ptr HTTPState::TryGetState(ClientContext &context, bool create auto lookup = context.registered_state.find("http_state"); if (lookup != context.registered_state.end()) { - return std::static_pointer_cast(lookup->second); + return shared_ptr_cast(lookup->second); } if (!create_on_missing) { return nullptr; } - auto http_state = make_shared(); + auto http_state = make_refcounted(); context.registered_state["http_state"] = http_state; return http_state; } @@ -87,7 +87,7 @@ shared_ptr &HTTPState::GetCachedFile(const string &path) { lock_guard lock(cached_files_mutex); auto &cache_entry_ref = cached_files[path]; if (!cache_entry_ref) { - cache_entry_ref = make_shared(); + cache_entry_ref = make_refcounted(); } return cache_entry_ref; } diff --git a/src/common/re2_regex.cpp b/src/common/re2_regex.cpp index da239cd4a213..4b3e2fb8e87b 100644 --- a/src/common/re2_regex.cpp +++ b/src/common/re2_regex.cpp @@ -10,7 +10,7 @@ namespace duckdb_re2 { Regex::Regex(const std::string &pattern, RegexOptions options) { RE2::Options o; o.set_case_sensitive(options == RegexOptions::CASE_INSENSITIVE); - regex = make_shared(StringPiece(pattern), o); + regex = duckdb::make_refcounted(StringPiece(pattern), o); } bool RegexSearchInternal(const char *input, Match &match, const Regex &r, RE2::Anchor anchor, size_t start, diff --git a/src/common/symbols.cpp b/src/common/symbols.cpp index 816a40b0387f..ee16effeac43 100644 --- a/src/common/symbols.cpp +++ b/src/common/symbols.cpp @@ -147,6 +147,15 @@ template class unique_ptr; template class unique_ptr; template class unique_ptr; +template class shared_ptr; +template class shared_ptr; +template class shared_ptr; +template class shared_ptr; +template class shared_ptr; +template class shared_ptr; +template class shared_ptr; +template class weak_ptr; + } // namespace duckdb #define INSTANTIATE_VECTOR(VECTOR_DEFINITION) \ @@ -160,15 +169,6 @@ template class unique_ptr; template std::VECTOR_DEFINITION::const_reference std::VECTOR_DEFINITION::front() const; \ template std::VECTOR_DEFINITION::reference std::VECTOR_DEFINITION::front(); -template class duckdb::vector; -template class duckdb::vector; -template class duckdb::vector; -template class duckdb::vector; -template class duckdb::vector; -template class duckdb::vector; -template class duckdb::vector>; -template class duckdb::vector; - INSTANTIATE_VECTOR(vector) INSTANTIATE_VECTOR(vector) INSTANTIATE_VECTOR(vector) @@ -192,14 +192,14 @@ INSTANTIATE_VECTOR(vector>) INSTANTIATE_VECTOR(vector>) INSTANTIATE_VECTOR(vector>) -template class shared_ptr; -template class shared_ptr; -template class shared_ptr; -template class shared_ptr; -template class shared_ptr; -template class shared_ptr; -template class shared_ptr; -template class weak_ptr; +template class duckdb::vector; +template class duckdb::vector; +template class duckdb::vector; +template class duckdb::vector; +template class duckdb::vector; +template class duckdb::vector; +template class duckdb::vector>; +template class duckdb::vector; #if !defined(__clang__) template struct std::atomic; diff --git a/src/common/types.cpp b/src/common/types.cpp index e4318c26ed22..f70d69d8102a 100644 --- a/src/common/types.cpp +++ b/src/common/types.cpp @@ -1127,7 +1127,7 @@ bool ApproxEqual(double ldecimal, double rdecimal) { //===--------------------------------------------------------------------===// void LogicalType::SetAlias(string alias) { if (!type_info_) { - type_info_ = make_shared(ExtraTypeInfoType::GENERIC_TYPE_INFO, std::move(alias)); + type_info_ = make_refcounted(ExtraTypeInfoType::GENERIC_TYPE_INFO, std::move(alias)); } else { type_info_->alias = std::move(alias); } @@ -1176,7 +1176,7 @@ uint8_t DecimalType::MaxWidth() { LogicalType LogicalType::DECIMAL(uint8_t width, uint8_t scale) { D_ASSERT(width >= scale); - auto type_info = make_shared(width, scale); + auto type_info = make_refcounted(width, scale); return LogicalType(LogicalTypeId::DECIMAL, std::move(type_info)); } @@ -1198,7 +1198,7 @@ string StringType::GetCollation(const LogicalType &type) { } LogicalType LogicalType::VARCHAR_COLLATION(string collation) { // NOLINT - auto string_info = make_shared(std::move(collation)); + auto string_info = make_refcounted(std::move(collation)); return LogicalType(LogicalTypeId::VARCHAR, std::move(string_info)); } @@ -1213,7 +1213,7 @@ const LogicalType &ListType::GetChildType(const LogicalType &type) { } LogicalType LogicalType::LIST(const LogicalType &child) { - auto info = make_shared(child); + auto info = make_refcounted(child); return LogicalType(LogicalTypeId::LIST, std::move(info)); } @@ -1285,12 +1285,12 @@ bool StructType::IsUnnamed(const LogicalType &type) { } LogicalType LogicalType::STRUCT(child_list_t children) { - auto info = make_shared(std::move(children)); + auto info = make_refcounted(std::move(children)); return LogicalType(LogicalTypeId::STRUCT, std::move(info)); } LogicalType LogicalType::AGGREGATE_STATE(aggregate_state_t state_type) { // NOLINT - auto info = make_shared(std::move(state_type)); + auto info = make_refcounted(std::move(state_type)); return LogicalType(LogicalTypeId::AGGREGATE_STATE, std::move(info)); } @@ -1315,7 +1315,7 @@ LogicalType LogicalType::MAP(const LogicalType &child_p) { new_children[1].first = "value"; auto child = LogicalType::STRUCT(std::move(new_children)); - auto info = make_shared(child); + auto info = make_refcounted(child); return LogicalType(LogicalTypeId::MAP, std::move(info)); } @@ -1344,7 +1344,7 @@ LogicalType LogicalType::UNION(child_list_t members) { D_ASSERT(members.size() <= UnionType::MAX_UNION_MEMBERS); // union types always have a hidden "tag" field in front members.insert(members.begin(), {"", LogicalType::UTINYINT}); - auto info = make_shared(std::move(members)); + auto info = make_refcounted(std::move(members)); return LogicalType(LogicalTypeId::UNION, std::move(info)); } @@ -1397,12 +1397,12 @@ const string &UserType::GetTypeName(const LogicalType &type) { } LogicalType LogicalType::USER(const string &user_type_name) { - auto info = make_shared(user_type_name); + auto info = make_refcounted(user_type_name); return LogicalType(LogicalTypeId::USER, std::move(info)); } LogicalType LogicalType::USER(string catalog, string schema, string name) { - auto info = make_shared(std::move(catalog), std::move(schema), std::move(name)); + auto info = make_refcounted(std::move(catalog), std::move(schema), std::move(name)); return LogicalType(LogicalTypeId::USER, std::move(info)); } @@ -1518,12 +1518,12 @@ LogicalType ArrayType::ConvertToList(const LogicalType &type) { LogicalType LogicalType::ARRAY(const LogicalType &child, idx_t size) { D_ASSERT(size > 0); D_ASSERT(size < ArrayType::MAX_ARRAY_SIZE); - auto info = make_shared(child, size); + auto info = make_refcounted(child, size); return LogicalType(LogicalTypeId::ARRAY, std::move(info)); } LogicalType LogicalType::ARRAY(const LogicalType &child) { - auto info = make_shared(child, 0); + auto info = make_refcounted(child, 0); return LogicalType(LogicalTypeId::ARRAY, std::move(info)); } @@ -1531,7 +1531,7 @@ LogicalType LogicalType::ARRAY(const LogicalType &child) { // Any Type //===--------------------------------------------------------------------===// LogicalType LogicalType::ANY_PARAMS(LogicalType target, idx_t cast_score) { // NOLINT - auto type_info = make_shared(std::move(target), cast_score); + auto type_info = make_refcounted(std::move(target), cast_score); return LogicalType(LogicalTypeId::ANY, std::move(type_info)); } @@ -1584,7 +1584,7 @@ LogicalType LogicalType::INTEGER_LITERAL(const Value &constant) { // NOLINT if (!constant.type().IsIntegral()) { throw InternalException("INTEGER_LITERAL can only be made from literals of integer types"); } - auto type_info = make_shared(constant); + auto type_info = make_refcounted(constant); return LogicalType(LogicalTypeId::INTEGER_LITERAL, std::move(type_info)); } diff --git a/src/common/types/column/column_data_collection.cpp b/src/common/types/column/column_data_collection.cpp index 8552332c2dc8..0f931d7cf909 100644 --- a/src/common/types/column/column_data_collection.cpp +++ b/src/common/types/column/column_data_collection.cpp @@ -51,17 +51,17 @@ ColumnDataCollection::ColumnDataCollection(Allocator &allocator_p) { types.clear(); count = 0; this->finished_append = false; - allocator = make_shared(allocator_p); + allocator = make_refcounted(allocator_p); } ColumnDataCollection::ColumnDataCollection(Allocator &allocator_p, vector types_p) { Initialize(std::move(types_p)); - allocator = make_shared(allocator_p); + allocator = make_refcounted(allocator_p); } ColumnDataCollection::ColumnDataCollection(BufferManager &buffer_manager, vector types_p) { Initialize(std::move(types_p)); - allocator = make_shared(buffer_manager); + allocator = make_refcounted(buffer_manager); } ColumnDataCollection::ColumnDataCollection(shared_ptr allocator_p, vector types_p) { @@ -71,7 +71,7 @@ ColumnDataCollection::ColumnDataCollection(shared_ptr alloc ColumnDataCollection::ColumnDataCollection(ClientContext &context, vector types_p, ColumnDataAllocatorType type) - : ColumnDataCollection(make_shared(context, type), std::move(types_p)) { + : ColumnDataCollection(make_refcounted(context, type), std::move(types_p)) { D_ASSERT(!types.empty()); } @@ -199,7 +199,7 @@ ColumnDataChunkIterationHelper::ColumnDataChunkIterationHelper(const ColumnDataC ColumnDataChunkIterationHelper::ColumnDataChunkIterator::ColumnDataChunkIterator( const ColumnDataCollection *collection_p, vector column_ids_p) - : collection(collection_p), scan_chunk(make_shared()), row_index(0) { + : collection(collection_p), scan_chunk(make_refcounted()), row_index(0) { if (!collection) { return; } @@ -246,7 +246,7 @@ ColumnDataRowIterationHelper::ColumnDataRowIterationHelper(const ColumnDataColle } ColumnDataRowIterationHelper::ColumnDataRowIterator::ColumnDataRowIterator(const ColumnDataCollection *collection_p) - : collection(collection_p), scan_chunk(make_shared()), current_row(*scan_chunk, 0, 0) { + : collection(collection_p), scan_chunk(make_refcounted()), current_row(*scan_chunk, 0, 0) { if (!collection) { return; } @@ -1041,7 +1041,7 @@ void ColumnDataCollection::Reset() { segments.clear(); // Refreshes the ColumnDataAllocator to prevent holding on to allocated data unnecessarily - allocator = make_shared(*allocator); + allocator = make_refcounted(*allocator); } struct ValueResultEquals { diff --git a/src/common/types/column/column_data_collection_segment.cpp b/src/common/types/column/column_data_collection_segment.cpp index 9713b66af09e..1f815d521974 100644 --- a/src/common/types/column/column_data_collection_segment.cpp +++ b/src/common/types/column/column_data_collection_segment.cpp @@ -7,7 +7,7 @@ namespace duckdb { ColumnDataCollectionSegment::ColumnDataCollectionSegment(shared_ptr allocator_p, vector types_p) : allocator(std::move(allocator_p)), types(std::move(types_p)), count(0), - heap(make_shared(allocator->GetAllocator())) { + heap(make_refcounted(allocator->GetAllocator())) { } idx_t ColumnDataCollectionSegment::GetDataSize(idx_t type_size) { diff --git a/src/common/types/column/partitioned_column_data.cpp b/src/common/types/column/partitioned_column_data.cpp index c785f346e869..7d47e129f26b 100644 --- a/src/common/types/column/partitioned_column_data.cpp +++ b/src/common/types/column/partitioned_column_data.cpp @@ -9,7 +9,7 @@ namespace duckdb { PartitionedColumnData::PartitionedColumnData(PartitionedColumnDataType type_p, ClientContext &context_p, vector types_p) : type(type_p), context(context_p), types(std::move(types_p)), - allocators(make_shared()) { + allocators(make_refcounted()) { } PartitionedColumnData::PartitionedColumnData(const PartitionedColumnData &other) @@ -165,7 +165,7 @@ vector> &PartitionedColumnData::GetPartitions() } void PartitionedColumnData::CreateAllocator() { - allocators->allocators.emplace_back(make_shared(BufferManager::GetBufferManager(context))); + allocators->allocators.emplace_back(make_refcounted(BufferManager::GetBufferManager(context))); allocators->allocators.back()->MakeShared(); } diff --git a/src/common/types/row/partitioned_tuple_data.cpp b/src/common/types/row/partitioned_tuple_data.cpp index 979b292294e7..cd67c32abb0d 100644 --- a/src/common/types/row/partitioned_tuple_data.cpp +++ b/src/common/types/row/partitioned_tuple_data.cpp @@ -9,7 +9,7 @@ namespace duckdb { PartitionedTupleData::PartitionedTupleData(PartitionedTupleDataType type_p, BufferManager &buffer_manager_p, const TupleDataLayout &layout_p) : type(type_p), buffer_manager(buffer_manager_p), layout(layout_p.Copy()), count(0), data_size(0), - allocators(make_shared()) { + allocators(make_refcounted()) { } PartitionedTupleData::PartitionedTupleData(const PartitionedTupleData &other) @@ -434,7 +434,7 @@ void PartitionedTupleData::Print() { // LCOV_EXCL_STOP void PartitionedTupleData::CreateAllocator() { - allocators->allocators.emplace_back(make_shared(buffer_manager, layout)); + allocators->allocators.emplace_back(make_refcounted(buffer_manager, layout)); } } // namespace duckdb diff --git a/src/common/types/row/tuple_data_collection.cpp b/src/common/types/row/tuple_data_collection.cpp index 8e548f9f1e28..7ffcac79abce 100644 --- a/src/common/types/row/tuple_data_collection.cpp +++ b/src/common/types/row/tuple_data_collection.cpp @@ -12,7 +12,7 @@ namespace duckdb { using ValidityBytes = TupleDataLayout::ValidityBytes; TupleDataCollection::TupleDataCollection(BufferManager &buffer_manager, const TupleDataLayout &layout_p) - : layout(layout_p.Copy()), allocator(make_shared(buffer_manager, layout)) { + : layout(layout_p.Copy()), allocator(make_refcounted(buffer_manager, layout)) { Initialize(); } @@ -377,7 +377,7 @@ void TupleDataCollection::Reset() { segments.clear(); // Refreshes the TupleDataAllocator to prevent holding on to allocated data unnecessarily - allocator = make_shared(*allocator); + allocator = make_refcounted(*allocator); } void TupleDataCollection::InitializeChunk(DataChunk &chunk) const { diff --git a/src/common/types/value.cpp b/src/common/types/value.cpp index fb3ee10cea4d..a2558d6cea42 100644 --- a/src/common/types/value.cpp +++ b/src/common/types/value.cpp @@ -162,7 +162,7 @@ Value::Value(string val) : type_(LogicalType::VARCHAR), is_null(false) { if (!Value::StringIsValid(val.c_str(), val.size())) { throw ErrorManager::InvalidUnicodeError(val, "value construction"); } - value_info_ = make_shared(std::move(val)); + value_info_ = make_refcounted(std::move(val)); } Value::~Value() { @@ -668,7 +668,7 @@ Value Value::STRUCT(const LogicalType &type, vector struct_values) { for (size_t i = 0; i < struct_values.size(); i++) { struct_values[i] = struct_values[i].DefaultCastAs(child_types[i].second); } - result.value_info_ = make_shared(std::move(struct_values)); + result.value_info_ = make_refcounted(std::move(struct_values)); result.type_ = type; result.is_null = false; return result; @@ -711,7 +711,7 @@ Value Value::MAP(const LogicalType &key_type, const LogicalType &value_type, vec new_children.push_back(std::make_pair("value", std::move(values[i]))); values[i] = Value::STRUCT(std::move(new_children)); } - result.value_info_ = make_shared(std::move(values)); + result.value_info_ = make_refcounted(std::move(values)); return result; } @@ -735,7 +735,7 @@ Value Value::UNION(child_list_t members, uint8_t tag, Value value) } } union_values[tag + 1] = std::move(value); - result.value_info_ = make_shared(std::move(union_values)); + result.value_info_ = make_refcounted(std::move(union_values)); result.type_ = LogicalType::UNION(std::move(members)); return result; } @@ -752,7 +752,7 @@ Value Value::LIST(vector values) { #endif Value result; result.type_ = LogicalType::LIST(values[0].type()); - result.value_info_ = make_shared(std::move(values)); + result.value_info_ = make_refcounted(std::move(values)); result.is_null = false; return result; } @@ -770,7 +770,7 @@ Value Value::LIST(const LogicalType &child_type, vector values) { Value Value::EMPTYLIST(const LogicalType &child_type) { Value result; result.type_ = LogicalType::LIST(child_type); - result.value_info_ = make_shared(); + result.value_info_ = make_refcounted(); result.is_null = false; return result; } @@ -787,7 +787,7 @@ Value Value::ARRAY(vector values) { #endif Value result; result.type_ = LogicalType::ARRAY(values[0].type(), values.size()); - result.value_info_ = make_shared(std::move(values)); + result.value_info_ = make_refcounted(std::move(values)); result.is_null = false; return result; } @@ -805,7 +805,7 @@ Value Value::ARRAY(const LogicalType &child_type, vector values) { Value Value::EMPTYARRAY(const LogicalType &child_type, uint32_t size) { Value result; result.type_ = LogicalType::ARRAY(child_type, size); - result.value_info_ = make_shared(); + result.value_info_ = make_refcounted(); result.is_null = false; return result; } @@ -813,35 +813,35 @@ Value Value::EMPTYARRAY(const LogicalType &child_type, uint32_t size) { Value Value::BLOB(const_data_ptr_t data, idx_t len) { Value result(LogicalType::BLOB); result.is_null = false; - result.value_info_ = make_shared(string(const_char_ptr_cast(data), len)); + result.value_info_ = make_refcounted(string(const_char_ptr_cast(data), len)); return result; } Value Value::BLOB(const string &data) { Value result(LogicalType::BLOB); result.is_null = false; - result.value_info_ = make_shared(Blob::ToBlob(string_t(data))); + result.value_info_ = make_refcounted(Blob::ToBlob(string_t(data))); return result; } Value Value::AGGREGATE_STATE(const LogicalType &type, const_data_ptr_t data, idx_t len) { // NOLINT Value result(type); result.is_null = false; - result.value_info_ = make_shared(string(const_char_ptr_cast(data), len)); + result.value_info_ = make_refcounted(string(const_char_ptr_cast(data), len)); return result; } Value Value::BIT(const_data_ptr_t data, idx_t len) { Value result(LogicalType::BIT); result.is_null = false; - result.value_info_ = make_shared(string(const_char_ptr_cast(data), len)); + result.value_info_ = make_refcounted(string(const_char_ptr_cast(data), len)); return result; } Value Value::BIT(const string &data) { Value result(LogicalType::BIT); result.is_null = false; - result.value_info_ = make_shared(Bit::ToBit(string_t(data))); + result.value_info_ = make_refcounted(Bit::ToBit(string_t(data))); return result; } @@ -1936,27 +1936,27 @@ Value Value::Deserialize(Deserializer &deserializer) { case PhysicalType::VARCHAR: { auto str = deserializer.ReadProperty(102, "value"); if (type.id() == LogicalTypeId::BLOB) { - new_value.value_info_ = make_shared(Blob::ToBlob(str)); + new_value.value_info_ = make_refcounted(Blob::ToBlob(str)); } else { - new_value.value_info_ = make_shared(str); + new_value.value_info_ = make_refcounted(str); } } break; case PhysicalType::LIST: { deserializer.ReadObject(102, "value", [&](Deserializer &obj) { auto children = obj.ReadProperty>(100, "children"); - new_value.value_info_ = make_shared(children); + new_value.value_info_ = make_refcounted(children); }); } break; case PhysicalType::STRUCT: { deserializer.ReadObject(102, "value", [&](Deserializer &obj) { auto children = obj.ReadProperty>(100, "children"); - new_value.value_info_ = make_shared(children); + new_value.value_info_ = make_refcounted(children); }); } break; case PhysicalType::ARRAY: { deserializer.ReadObject(102, "value", [&](Deserializer &obj) { auto children = obj.ReadProperty>(100, "children"); - new_value.value_info_ = make_shared(children); + new_value.value_info_ = make_refcounted(children); }); } break; default: diff --git a/src/common/types/vector_cache.cpp b/src/common/types/vector_cache.cpp index c0ea6fa7cc3e..0c5075a54b45 100644 --- a/src/common/types/vector_cache.cpp +++ b/src/common/types/vector_cache.cpp @@ -18,7 +18,7 @@ class VectorCacheBuffer : public VectorBuffer { auto &child_type = ListType::GetChildType(type); child_caches.push_back(make_buffer(allocator, child_type, capacity)); auto child_vector = make_uniq(child_type, false, false); - auxiliary = make_shared(std::move(child_vector)); + auxiliary = make_refcounted(std::move(child_vector)); break; } case PhysicalType::ARRAY: { @@ -26,7 +26,7 @@ class VectorCacheBuffer : public VectorBuffer { auto array_size = ArrayType::GetSize(type); child_caches.push_back(make_buffer(allocator, child_type, array_size * capacity)); auto child_vector = make_uniq(child_type, true, false, array_size * capacity); - auxiliary = make_shared(std::move(child_vector), array_size, capacity); + auxiliary = make_refcounted(std::move(child_vector), array_size, capacity); break; } case PhysicalType::STRUCT: { @@ -34,7 +34,7 @@ class VectorCacheBuffer : public VectorBuffer { for (auto &child_type : child_types) { child_caches.push_back(make_buffer(allocator, child_type.second, capacity)); } - auto struct_buffer = make_shared(type); + auto struct_buffer = make_refcounted(type); auxiliary = std::move(struct_buffer); break; } diff --git a/src/execution/aggregate_hashtable.cpp b/src/execution/aggregate_hashtable.cpp index 90029b0a4b6c..95f826a35596 100644 --- a/src/execution/aggregate_hashtable.cpp +++ b/src/execution/aggregate_hashtable.cpp @@ -40,7 +40,7 @@ GroupedAggregateHashTable::GroupedAggregateHashTable(ClientContext &context, All vector aggregate_objects_p, idx_t initial_capacity, idx_t radix_bits) : BaseAggregateHashTable(context, allocator, aggregate_objects_p, std::move(payload_types_p)), - radix_bits(radix_bits), count(0), capacity(0), aggregate_allocator(make_shared(allocator)) { + radix_bits(radix_bits), count(0), capacity(0), aggregate_allocator(make_refcounted(allocator)) { // Append hash column to the end and initialise the row layout group_types_p.emplace_back(LogicalType::HASH); diff --git a/src/execution/index/art/art.cpp b/src/execution/index/art/art.cpp index 9fa44deafedf..7e3c7f3f38d3 100644 --- a/src/execution/index/art/art.cpp +++ b/src/execution/index/art/art.cpp @@ -54,7 +54,8 @@ ART::ART(const string &name, const IndexConstraintType index_constraint_type, co make_uniq(sizeof(Node16), block_manager), make_uniq(sizeof(Node48), block_manager), make_uniq(sizeof(Node256), block_manager)}; - allocators = make_shared, ALLOCATOR_COUNT>>(std::move(allocator_array)); + allocators = + make_refcounted, ALLOCATOR_COUNT>>(std::move(allocator_array)); } // deserialize lazily diff --git a/src/execution/operator/aggregate/aggregate_object.cpp b/src/execution/operator/aggregate/aggregate_object.cpp index 05af824d676f..79a524ea76b8 100644 --- a/src/execution/operator/aggregate/aggregate_object.cpp +++ b/src/execution/operator/aggregate/aggregate_object.cpp @@ -9,7 +9,7 @@ AggregateObject::AggregateObject(AggregateFunction function, FunctionData *bind_ idx_t payload_size, AggregateType aggr_type, PhysicalType return_type, Expression *filter) : function(std::move(function)), - bind_data_wrapper(bind_data ? make_shared(bind_data->Copy()) : nullptr), + bind_data_wrapper(bind_data ? make_refcounted(bind_data->Copy()) : nullptr), child_count(child_count), payload_size(payload_size), aggr_type(aggr_type), return_type(return_type), filter(filter) { } diff --git a/src/execution/operator/aggregate/physical_hash_aggregate.cpp b/src/execution/operator/aggregate/physical_hash_aggregate.cpp index 5217c110bd26..420a9aecb918 100644 --- a/src/execution/operator/aggregate/physical_hash_aggregate.cpp +++ b/src/execution/operator/aggregate/physical_hash_aggregate.cpp @@ -608,7 +608,7 @@ idx_t HashAggregateDistinctFinalizeEvent::CreateGlobalSources() { void HashAggregateDistinctFinalizeEvent::FinishEvent() { // Now that everything is added to the main ht, we can actually finalize - auto new_event = make_shared(context, pipeline.get(), op, gstate); + auto new_event = make_refcounted(context, pipeline.get(), op, gstate); this->InsertEvent(std::move(new_event)); } @@ -755,7 +755,7 @@ SinkFinalizeType PhysicalHashAggregate::FinalizeDistinct(Pipeline &pipeline, Eve radix_table->Finalize(context, radix_state); } } - auto new_event = make_shared(context, pipeline, *this, gstate); + auto new_event = make_refcounted(context, pipeline, *this, gstate); event.InsertEvent(std::move(new_event)); return SinkFinalizeType::READY; } diff --git a/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp b/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp index f1ceeab4bc7c..97008097513f 100644 --- a/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +++ b/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp @@ -586,7 +586,7 @@ SinkFinalizeType PhysicalUngroupedAggregate::FinalizeDistinct(Pipeline &pipeline auto &radix_state = *distinct_state.radix_states[table_idx]; radix_table_p->Finalize(context, radix_state); } - auto new_event = make_shared(context, *this, gstate, pipeline); + auto new_event = make_refcounted(context, *this, gstate, pipeline); event.InsertEvent(std::move(new_event)); return SinkFinalizeType::READY; } diff --git a/src/execution/operator/aggregate/physical_window.cpp b/src/execution/operator/aggregate/physical_window.cpp index bcfe0a56bd3b..b945615bab16 100644 --- a/src/execution/operator/aggregate/physical_window.cpp +++ b/src/execution/operator/aggregate/physical_window.cpp @@ -171,7 +171,7 @@ SinkFinalizeType PhysicalWindow::Finalize(Pipeline &pipeline, Event &event, Clie } // Schedule all the sorts for maximum thread utilisation - auto new_event = make_shared(*state.global_partition, pipeline); + auto new_event = make_refcounted(*state.global_partition, pipeline); event.InsertEvent(std::move(new_event)); return SinkFinalizeType::READY; diff --git a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp index 8c29ae79fb43..e0f23f4eb8f6 100644 --- a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp +++ b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp @@ -39,8 +39,8 @@ shared_ptr CSVBuffer::Next(CSVFileHandle &file_handle, idx_t buffer_s file_handle.Seek(global_csv_start + actual_buffer_size); has_seaked = false; } - auto next_csv_buffer = make_shared(file_handle, context, buffer_size, - global_csv_start + actual_buffer_size, file_number_p, buffer_idx + 1); + auto next_csv_buffer = make_refcounted( + file_handle, context, buffer_size, global_csv_start + actual_buffer_size, file_number_p, buffer_idx + 1); if (next_csv_buffer->GetBufferSize() == 0) { // We are done reading return nullptr; @@ -73,8 +73,8 @@ shared_ptr CSVBuffer::Pin(CSVFileHandle &file_handle, bool &has Reload(file_handle); has_seeked = true; } - return make_shared(buffer_manager.Pin(block), actual_buffer_size, last_buffer, file_number, - buffer_idx); + return make_refcounted(buffer_manager.Pin(block), actual_buffer_size, last_buffer, file_number, + buffer_idx); } void CSVBuffer::Unpin() { diff --git a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp index 2a13158b6081..f3bcd5c403b6 100644 --- a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp +++ b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp @@ -28,7 +28,7 @@ void CSVBufferManager::UnpinBuffer(const idx_t cache_idx) { void CSVBufferManager::Initialize() { if (cached_buffers.empty()) { cached_buffers.emplace_back( - make_shared(context, buffer_size, *file_handle, global_csv_pos, file_idx)); + make_refcounted(context, buffer_size, *file_handle, global_csv_pos, file_idx)); last_buffer = cached_buffers.front(); } } diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 9582e1c1af2f..ed52797b01ac 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -525,14 +525,14 @@ StringValueScanner::StringValueScanner(const shared_ptr &buffe } unique_ptr StringValueScanner::GetCSVScanner(ClientContext &context, CSVReaderOptions &options) { - auto state_machine = make_shared(options, options.dialect_options.state_machine_options, - CSVStateMachineCache::Get(context)); + auto state_machine = make_refcounted(options, options.dialect_options.state_machine_options, + CSVStateMachineCache::Get(context)); state_machine->dialect_options.num_cols = options.dialect_options.num_cols; state_machine->dialect_options.header = options.dialect_options.header; - auto buffer_manager = make_shared(context, options, options.file_path, 0); - auto scanner = make_uniq(buffer_manager, state_machine, make_shared()); - scanner->csv_file_scan = make_shared(context, options.file_path, options); + auto buffer_manager = make_refcounted(context, options, options.file_path, 0); + auto scanner = make_uniq(buffer_manager, state_machine, make_refcounted()); + scanner->csv_file_scan = make_refcounted(context, options.file_path, options); scanner->csv_file_scan->InitializeProjection(); return scanner; } @@ -1074,8 +1074,9 @@ void StringValueScanner::SetStart() { return; } - scan_finder = make_uniq( - 0, buffer_manager, state_machine, make_shared(true), csv_file_scan, false, iterator, 1); + scan_finder = + make_uniq(0, buffer_manager, state_machine, make_refcounted(true), + csv_file_scan, false, iterator, 1); auto &tuples = scan_finder->ParseChunk(); line_found = true; if (tuples.number_of_rows != 1) { diff --git a/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp b/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp index 3b60f247aa27..af8788a1ad9c 100644 --- a/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp +++ b/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp @@ -13,8 +13,8 @@ CSVSniffer::CSVSniffer(CSVReaderOptions &options_p, shared_ptr } // Initialize max columns found to either 0 or however many were set max_columns_found = set_columns.Size(); - error_handler = make_shared(options.ignore_errors); - detection_error_handler = make_shared(true); + error_handler = make_refcounted(options.ignore_errors); + detection_error_handler = make_refcounted(true); } bool SetColumns::IsSet() { diff --git a/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp b/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp index 0532fc678a41..4f5732b934f9 100644 --- a/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp +++ b/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp @@ -10,7 +10,7 @@ CSVFileScan::CSVFileScan(ClientContext &context, shared_ptr bu vector &file_schema) : file_path(options_p.file_path), file_idx(0), buffer_manager(std::move(buffer_manager_p)), state_machine(std::move(state_machine_p)), file_size(buffer_manager->file_handle->FileSize()), - error_handler(make_shared(options_p.ignore_errors)), + error_handler(make_refcounted(options_p.ignore_errors)), on_disk_file(buffer_manager->file_handle->OnDiskFile()), options(options_p) { if (bind_data.initial_reader.get()) { auto &union_reader = *bind_data.initial_reader; @@ -43,7 +43,7 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, cons const idx_t file_idx_p, const ReadCSVData &bind_data, const vector &column_ids, const vector &file_schema) : file_path(file_path_p), file_idx(file_idx_p), - error_handler(make_shared(options_p.ignore_errors)), options(options_p) { + error_handler(make_refcounted(options_p.ignore_errors)), options(options_p) { if (file_idx < bind_data.union_readers.size()) { // we are doing UNION BY NAME - fetch the options from the union reader for this file optional_ptr union_reader_ptr; @@ -73,7 +73,7 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, cons } // Initialize Buffer Manager - buffer_manager = make_shared(context, options, file_path, file_idx); + buffer_manager = make_refcounted(context, options, file_path, file_idx); // Initialize On Disk and Size of file on_disk_file = buffer_manager->file_handle->OnDiskFile(); file_size = buffer_manager->file_handle->FileSize(); @@ -89,7 +89,7 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, cons CSVSniffer sniffer(options, buffer_manager, state_machine_cache); sniffer.SniffCSV(); } - state_machine = make_shared( + state_machine = make_refcounted( state_machine_cache.Get(options.dialect_options.state_machine_options), options); MultiFileReader::InitializeReader(*this, options.file_options, bind_data.reader_bind, bind_data.return_types, @@ -120,8 +120,8 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, cons names = bind_data.csv_names; types = bind_data.csv_types; - state_machine = - make_shared(state_machine_cache.Get(options.dialect_options.state_machine_options), options); + state_machine = make_refcounted( + state_machine_cache.Get(options.dialect_options.state_machine_options), options); MultiFileReader::InitializeReader(*this, options.file_options, bind_data.reader_bind, bind_data.return_types, bind_data.return_names, column_ids, nullptr, file_path, context); @@ -129,9 +129,9 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, cons } CSVFileScan::CSVFileScan(ClientContext &context, const string &file_name, CSVReaderOptions &options_p) - : file_path(file_name), file_idx(0), error_handler(make_shared(options_p.ignore_errors)), + : file_path(file_name), file_idx(0), error_handler(make_refcounted(options_p.ignore_errors)), options(options_p) { - buffer_manager = make_shared(context, options, file_path, file_idx); + buffer_manager = make_refcounted(context, options, file_path, file_idx); // Initialize On Disk and Size of file on_disk_file = buffer_manager->file_handle->OnDiskFile(); file_size = buffer_manager->file_handle->FileSize(); @@ -151,8 +151,8 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_name, CSVRea options.dialect_options.num_cols = options.sql_type_list.size(); } // Initialize State Machine - state_machine = - make_shared(state_machine_cache.Get(options.dialect_options.state_machine_options), options); + state_machine = make_refcounted( + state_machine_cache.Get(options.dialect_options.state_machine_options), options); } void CSVFileScan::InitializeFileNamesTypes() { diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index b5943a9f5b8b..bc65c6d13172 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -14,7 +14,7 @@ CSVGlobalState::CSVGlobalState(ClientContext &context_p, const shared_ptrGetFilePath() == files[0]) { - auto state_machine = make_shared( + auto state_machine = make_refcounted( CSVStateMachineCache::Get(context).Get(options.dialect_options.state_machine_options), options); // If we already have a buffer manager, we don't need to reconstruct it to the first file file_scans.emplace_back(make_uniq(context, buffer_manager, state_machine, options, bind_data, @@ -36,7 +36,7 @@ CSVGlobalState::CSVGlobalState(ClientContext &context_p, const shared_ptrbuffer_manager->GetBuffer(0)->actual_size; current_boundary = CSVIterator(0, 0, 0, 0, buffer_size); } - current_buffer_in_use = make_shared(*file_scans.back()->buffer_manager, 0); + current_buffer_in_use = make_refcounted(*file_scans.back()->buffer_manager, 0); } double CSVGlobalState::GetProgress(const ReadCSVData &bind_data_p) const { @@ -66,8 +66,8 @@ unique_ptr CSVGlobalState::Next() { if (cur_idx == 0) { current_file = file_scans.back(); } else { - current_file = make_shared(context, bind_data.files[cur_idx], bind_data.options, cur_idx, - bind_data, column_ids, file_schema); + current_file = make_refcounted(context, bind_data.files[cur_idx], bind_data.options, cur_idx, + bind_data, column_ids, file_schema); } auto csv_scanner = make_uniq(scanner_idx++, current_file->buffer_manager, current_file->state_machine, @@ -80,7 +80,7 @@ unique_ptr CSVGlobalState::Next() { } if (current_buffer_in_use->buffer_idx != current_boundary.GetBufferIdx()) { current_buffer_in_use = - make_shared(*file_scans.back()->buffer_manager, current_boundary.GetBufferIdx()); + make_refcounted(*file_scans.back()->buffer_manager, current_boundary.GetBufferIdx()); } // We first create the scanner for the current boundary auto ¤t_file = *file_scans.back(); @@ -96,13 +96,13 @@ unique_ptr CSVGlobalState::Next() { auto current_file_idx = current_file.file_idx + 1; if (current_file_idx < bind_data.files.size()) { // If we have a next file we have to construct the file scan for that - file_scans.emplace_back(make_shared(context, bind_data.files[current_file_idx], - bind_data.options, current_file_idx, bind_data, column_ids, - file_schema)); + file_scans.emplace_back(make_refcounted(context, bind_data.files[current_file_idx], + bind_data.options, current_file_idx, bind_data, + column_ids, file_schema)); // And re-start the boundary-iterator auto buffer_size = file_scans.back()->buffer_manager->GetBuffer(0)->actual_size; current_boundary = CSVIterator(current_file_idx, 0, 0, 0, buffer_size); - current_buffer_in_use = make_shared(*file_scans.back()->buffer_manager, 0); + current_buffer_in_use = make_refcounted(*file_scans.back()->buffer_manager, 0); } else { // If not we are done with this CSV Scanning finished = true; diff --git a/src/execution/operator/helper/physical_buffered_collector.cpp b/src/execution/operator/helper/physical_buffered_collector.cpp index fcf75496e6f7..90708953a643 100644 --- a/src/execution/operator/helper/physical_buffered_collector.cpp +++ b/src/execution/operator/helper/physical_buffered_collector.cpp @@ -55,7 +55,7 @@ SinkCombineResultType PhysicalBufferedCollector::Combine(ExecutionContext &conte unique_ptr PhysicalBufferedCollector::GetGlobalSinkState(ClientContext &context) const { auto state = make_uniq(); state->context = context.shared_from_this(); - state->buffered_data = make_shared(state->context); + state->buffered_data = make_refcounted(state->context); return std::move(state); } diff --git a/src/execution/operator/join/physical_asof_join.cpp b/src/execution/operator/join/physical_asof_join.cpp index 3996063315c5..05e45d7a455f 100644 --- a/src/execution/operator/join/physical_asof_join.cpp +++ b/src/execution/operator/join/physical_asof_join.cpp @@ -169,7 +169,7 @@ SinkFinalizeType PhysicalAsOfJoin::Finalize(Pipeline &pipeline, Event &event, Cl } // Schedule all the sorts for maximum thread utilisation - auto new_event = make_shared(gstate.rhs_sink, pipeline); + auto new_event = make_refcounted(gstate.rhs_sink, pipeline); event.InsertEvent(std::move(new_event)); return SinkFinalizeType::READY; diff --git a/src/execution/operator/join/physical_hash_join.cpp b/src/execution/operator/join/physical_hash_join.cpp index 1e1e9fd12384..09c336458300 100644 --- a/src/execution/operator/join/physical_hash_join.cpp +++ b/src/execution/operator/join/physical_hash_join.cpp @@ -359,7 +359,7 @@ void HashJoinGlobalSinkState::ScheduleFinalize(Pipeline &pipeline, Event &event) return; } hash_table->InitializePointerTable(); - auto new_event = make_shared(pipeline, *this); + auto new_event = make_refcounted(pipeline, *this); event.InsertEvent(std::move(new_event)); } @@ -474,7 +474,7 @@ SinkFinalizeType PhysicalHashJoin::Finalize(Pipeline &pipeline, Event &event, Cl // We have to repartition ht.SetRepartitionRadixBits(sink.local_hash_tables, sink.temporary_memory_state->GetReservation(), max_partition_size, max_partition_count); - auto new_event = make_shared(pipeline, sink, sink.local_hash_tables); + auto new_event = make_refcounted(pipeline, sink, sink.local_hash_tables); event.InsertEvent(std::move(new_event)); } else { // No repartitioning! diff --git a/src/execution/operator/join/physical_range_join.cpp b/src/execution/operator/join/physical_range_join.cpp index 5d6d44f47bff..d89360236a3b 100644 --- a/src/execution/operator/join/physical_range_join.cpp +++ b/src/execution/operator/join/physical_range_join.cpp @@ -149,7 +149,7 @@ class RangeJoinMergeEvent : public BasePipelineEvent { void PhysicalRangeJoin::GlobalSortedTable::ScheduleMergeTasks(Pipeline &pipeline, Event &event) { // Initialize global sort state for a round of merging global_sort_state.InitializeMergeRound(); - auto new_event = make_shared(*this, pipeline); + auto new_event = make_refcounted(*this, pipeline); event.InsertEvent(std::move(new_event)); } diff --git a/src/execution/operator/order/physical_order.cpp b/src/execution/operator/order/physical_order.cpp index ac933f3fdc84..7aa77a7a30c0 100644 --- a/src/execution/operator/order/physical_order.cpp +++ b/src/execution/operator/order/physical_order.cpp @@ -186,7 +186,7 @@ SinkFinalizeType PhysicalOrder::Finalize(Pipeline &pipeline, Event &event, Clien void PhysicalOrder::ScheduleMergeTasks(Pipeline &pipeline, Event &event, OrderGlobalSinkState &state) { // Initialize global sort state for a round of merging state.global_sort_state.InitializeMergeRound(); - auto new_event = make_shared(state, pipeline); + auto new_event = make_refcounted(state, pipeline); event.InsertEvent(std::move(new_event)); } diff --git a/src/execution/operator/persistent/physical_batch_copy_to_file.cpp b/src/execution/operator/persistent/physical_batch_copy_to_file.cpp index 5447f433cb69..831ea35b6e49 100644 --- a/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +++ b/src/execution/operator/persistent/physical_batch_copy_to_file.cpp @@ -308,7 +308,7 @@ SinkFinalizeType PhysicalBatchCopyToFile::Finalize(Pipeline &pipeline, Event &ev FinalFlush(context, input.global_state); } else { // we have multiple tasks remaining - launch an event to execute the tasks in parallel - auto new_event = make_shared(*this, gstate, pipeline, context); + auto new_event = make_refcounted(*this, gstate, pipeline, context); event.InsertEvent(std::move(new_event)); } return SinkFinalizeType::READY; diff --git a/src/execution/operator/persistent/physical_copy_to_file.cpp b/src/execution/operator/persistent/physical_copy_to_file.cpp index 5925e9f012eb..be9ca4dbbd61 100644 --- a/src/execution/operator/persistent/physical_copy_to_file.cpp +++ b/src/execution/operator/persistent/physical_copy_to_file.cpp @@ -284,7 +284,7 @@ unique_ptr PhysicalCopyToFile::GetGlobalSinkState(ClientContext } if (partition_output) { - state->partition_state = make_shared(); + state->partition_state = make_refcounted(); } return std::move(state); diff --git a/src/execution/operator/schema/physical_create_art_index.cpp b/src/execution/operator/schema/physical_create_art_index.cpp index e7325405fafe..fe88d7c3bfa4 100644 --- a/src/execution/operator/schema/physical_create_art_index.cpp +++ b/src/execution/operator/schema/physical_create_art_index.cpp @@ -178,7 +178,7 @@ SinkFinalizeType PhysicalCreateARTIndex::Finalize(Pipeline &pipeline, Event &eve auto &index = index_entry->Cast(); index.initial_index_size = state.global_index->GetInMemorySize(); - index.info = make_shared(storage.info, index.name); + index.info = make_refcounted(storage.info, index.name); for (auto &parsed_expr : info->parsed_expressions) { index.parsed_expressions.push_back(parsed_expr->Copy()); } diff --git a/src/execution/operator/set/physical_recursive_cte.cpp b/src/execution/operator/set/physical_recursive_cte.cpp index 57a847dd0775..5210987325ab 100644 --- a/src/execution/operator/set/physical_recursive_cte.cpp +++ b/src/execution/operator/set/physical_recursive_cte.cpp @@ -200,7 +200,7 @@ void PhysicalRecursiveCTE::BuildPipelines(Pipeline ¤t, MetaPipeline &meta_ initial_state_pipeline.Build(*children[0]); // the RHS is the recursive pipeline - recursive_meta_pipeline = make_shared(executor, state, this); + recursive_meta_pipeline = make_refcounted(executor, state, this); recursive_meta_pipeline->SetRecursiveCTE(); recursive_meta_pipeline->Build(*children[1]); diff --git a/src/execution/physical_plan/plan_cte.cpp b/src/execution/physical_plan/plan_cte.cpp index 7a306c3a54ca..c11286a8cf3f 100644 --- a/src/execution/physical_plan/plan_cte.cpp +++ b/src/execution/physical_plan/plan_cte.cpp @@ -12,7 +12,7 @@ unique_ptr PhysicalPlanGenerator::CreatePlan(LogicalMaterializ D_ASSERT(op.children.size() == 2); // Create the working_table that the PhysicalCTE will use for evaluation. - auto working_table = make_shared(context, op.children[0]->types); + auto working_table = make_refcounted(context, op.children[0]->types); // Add the ColumnDataCollection to the context of this PhysicalPlanGenerator recursive_cte_tables[op.table_index] = working_table; diff --git a/src/execution/physical_plan/plan_recursive_cte.cpp b/src/execution/physical_plan/plan_recursive_cte.cpp index 89da0cd0706e..5ddb767612b6 100644 --- a/src/execution/physical_plan/plan_recursive_cte.cpp +++ b/src/execution/physical_plan/plan_recursive_cte.cpp @@ -12,7 +12,7 @@ unique_ptr PhysicalPlanGenerator::CreatePlan(LogicalRecursiveC D_ASSERT(op.children.size() == 2); // Create the working_table that the PhysicalRecursiveCTE will use for evaluation. - auto working_table = make_shared(context, op.types); + auto working_table = make_refcounted(context, op.types); // Add the ColumnDataCollection to the context of this PhysicalPlanGenerator recursive_cte_tables[op.table_index] = working_table; diff --git a/src/function/table/copy_csv.cpp b/src/function/table/copy_csv.cpp index 1fa2e46e7694..e45a6a643120 100644 --- a/src/function/table/copy_csv.cpp +++ b/src/function/table/copy_csv.cpp @@ -156,7 +156,7 @@ static unique_ptr ReadCSVBind(ClientContext &context, CopyInfo &in } if (options.auto_detect) { - auto buffer_manager = make_shared(context, options, bind_data->files[0], 0); + auto buffer_manager = make_refcounted(context, options, bind_data->files[0], 0); CSVSniffer sniffer(options, buffer_manager, CSVStateMachineCache::Get(context), {&expected_types, &expected_names}); sniffer.SniffCSV(); diff --git a/src/function/table/read_csv.cpp b/src/function/table/read_csv.cpp index 8d2e1be0d780..5f462c8059f1 100644 --- a/src/function/table/read_csv.cpp +++ b/src/function/table/read_csv.cpp @@ -98,7 +98,7 @@ static unique_ptr ReadCSVBind(ClientContext &context, TableFunctio } if (options.auto_detect && !options.file_options.union_by_name) { options.file_path = result->files[0]; - result->buffer_manager = make_shared(context, options, result->files[0], 0); + result->buffer_manager = make_refcounted(context, options, result->files[0], 0); CSVSniffer sniffer(options, result->buffer_manager, CSVStateMachineCache::Get(context), {&return_types, &names}); auto sniffer_result = sniffer.SniffCSV(); diff --git a/src/function/table/sniff_csv.cpp b/src/function/table/sniff_csv.cpp index 3e859a65afe3..d29a82f1c583 100644 --- a/src/function/table/sniff_csv.cpp +++ b/src/function/table/sniff_csv.cpp @@ -120,7 +120,7 @@ static void CSVSniffFunction(ClientContext &context, TableFunctionInput &data_p, auto sniffer_options = data.options; sniffer_options.file_path = data.path; - auto buffer_manager = make_shared(context, sniffer_options, sniffer_options.file_path, 0); + auto buffer_manager = make_refcounted(context, sniffer_options, sniffer_options.file_path, 0); if (sniffer_options.name_list.empty()) { sniffer_options.name_list = data.names_csv; } diff --git a/src/include/duckdb/common/enable_shared_from_this.ipp b/src/include/duckdb/common/enable_shared_from_this.ipp new file mode 100644 index 000000000000..6472db9c2b12 --- /dev/null +++ b/src/include/duckdb/common/enable_shared_from_this.ipp @@ -0,0 +1,40 @@ +namespace duckdb { + +template +class enable_shared_from_this { + mutable weak_ptr<_Tp> __weak_this_; + +protected: + constexpr enable_shared_from_this() noexcept { + } + enable_shared_from_this(enable_shared_from_this const &) noexcept { + } + enable_shared_from_this &operator=(enable_shared_from_this const &) noexcept { + return *this; + } + ~enable_shared_from_this() { + } + +public: + shared_ptr<_Tp> shared_from_this() { + return shared_ptr<_Tp>(__weak_this_); + } + shared_ptr<_Tp const> shared_from_this() const { + return shared_ptr(__weak_this_); + } + +#if _LIBCPP_STD_VER >= 17 + weak_ptr<_Tp> weak_from_this() noexcept { + return __weak_this_; + } + + weak_ptr weak_from_this() const noexcept { + return __weak_this_; + } +#endif // _LIBCPP_STD_VER >= 17 + + template + friend class shared_ptr; +}; + +} // namespace duckdb diff --git a/src/include/duckdb/common/exception.hpp b/src/include/duckdb/common/exception.hpp index 3765c6ba58c8..4f8dc1e16bbe 100644 --- a/src/include/duckdb/common/exception.hpp +++ b/src/include/duckdb/common/exception.hpp @@ -10,7 +10,6 @@ #include "duckdb/common/assert.hpp" #include "duckdb/common/exception_format_value.hpp" -#include "duckdb/common/shared_ptr.hpp" #include "duckdb/common/unordered_map.hpp" #include "duckdb/common/typedefs.hpp" diff --git a/src/include/duckdb/common/helper.hpp b/src/include/duckdb/common/helper.hpp index 4c57e51a272b..81da4bd79513 100644 --- a/src/include/duckdb/common/helper.hpp +++ b/src/include/duckdb/common/helper.hpp @@ -68,7 +68,7 @@ make_uniq(ARGS&&... args) // NOLINT: mimic std style template inline shared_ptr -make_shared(ARGS&&... args) // NOLINT: mimic std style +make_refcounted(ARGS&&... args) // NOLINT: mimic std style { return shared_ptr(new DATA_TYPE(std::forward(args)...)); } @@ -117,10 +117,15 @@ unique_ptr unique_ptr_cast(unique_ptr src) { // NOLINT: mimic std style return unique_ptr(static_cast(src.release())); } +template +shared_ptr shared_ptr_cast(shared_ptr src) { + return shared_ptr(std::static_pointer_cast(src.internal)); +} + struct SharedConstructor { template static shared_ptr Create(ARGS &&...args) { - return make_shared(std::forward(args)...); + return make_refcounted(std::forward(args)...); } }; diff --git a/src/include/duckdb/common/http_state.hpp b/src/include/duckdb/common/http_state.hpp index 1341b921147c..6fc1fa92e291 100644 --- a/src/include/duckdb/common/http_state.hpp +++ b/src/include/duckdb/common/http_state.hpp @@ -20,7 +20,7 @@ namespace duckdb { class CachedFileHandle; //! Represents a file that is intended to be fully downloaded, then used in parallel by multiple threads -class CachedFile : public std::enable_shared_from_this { +class CachedFile : public enable_shared_from_this { friend class CachedFileHandle; public: diff --git a/src/include/duckdb/common/multi_file_reader.hpp b/src/include/duckdb/common/multi_file_reader.hpp index ca52810e8dfd..ec318dea76e0 100644 --- a/src/include/duckdb/common/multi_file_reader.hpp +++ b/src/include/duckdb/common/multi_file_reader.hpp @@ -151,7 +151,7 @@ struct MultiFileReader { return BindUnionReader(context, return_types, names, result, options); } else { shared_ptr reader; - reader = make_shared(context, result.files[0], options); + reader = make_refcounted(context, result.files[0], options); return_types = reader->return_types; names = reader->names; result.Initialize(std::move(reader)); diff --git a/src/include/duckdb/common/re2_regex.hpp b/src/include/duckdb/common/re2_regex.hpp index ae5c48fd51f7..77b5b261b60d 100644 --- a/src/include/duckdb/common/re2_regex.hpp +++ b/src/include/duckdb/common/re2_regex.hpp @@ -4,7 +4,8 @@ #include "duckdb/common/winapi.hpp" #include "duckdb/common/vector.hpp" -#include +#include "duckdb/common/shared_ptr.hpp" +#include "duckdb/common/string.hpp" #include namespace duckdb_re2 { @@ -22,7 +23,7 @@ class Regex { } private: - shared_ptr regex; + duckdb::shared_ptr regex; }; struct GroupMatch { diff --git a/src/include/duckdb/common/shared_ptr.hpp b/src/include/duckdb/common/shared_ptr.hpp index 615273d7c27e..fe9d31ee40c9 100644 --- a/src/include/duckdb/common/shared_ptr.hpp +++ b/src/include/duckdb/common/shared_ptr.hpp @@ -10,113 +10,22 @@ #include #include +#include "duckdb/common/unique_ptr.hpp" -template -class weak_ptr; +#if _LIBCPP_STD_VER >= 17 +template +struct __bounded_convertible_to_unbounded : false_type {}; -namespace duckdb { +template +struct __bounded_convertible_to_unbounded<_Up[_Np], _Tp> : is_same, _Up[]> {}; -template -class shared_ptr { -private: - template - friend class weak_ptr; - std::shared_ptr internal; +template +struct __compatible_with : _Or, __bounded_convertible_to_unbounded<_Yp, _Tp>> {}; +#else +template +struct __compatible_with : std::is_convertible<_Yp *, _Tp *> {}; +#endif // _LIBCPP_STD_VER >= 17 -public: - // Constructors - shared_ptr() : internal() { - } - shared_ptr(std::nullptr_t) : internal(nullptr) { - } // Implicit conversion - template - explicit shared_ptr(U *ptr) : internal(ptr) { - } - shared_ptr(const shared_ptr &other) : internal(other.internal) { - } - shared_ptr(std::shared_ptr other) : internal(std::move(other)) { - } - - // Destructor - ~shared_ptr() = default; - - // Assignment operators - shared_ptr &operator=(const shared_ptr &other) { - internal = other.internal; - return *this; - } - - // Modifiers - void reset() { - internal.reset(); - } - - template - void reset(U *ptr) { - internal.reset(ptr); - } - - template - void reset(U *ptr, Deleter deleter) { - internal.reset(ptr, deleter); - } - - // Observers - T *get() const { - return internal.get(); - } - - long use_count() const { - return internal.use_count(); - } - - explicit operator bool() const noexcept { - return internal.operator bool(); - } - - // Element access - std::__add_lvalue_reference_t operator*() const { - return *internal; - } - - T *operator->() const { - return internal.operator->(); - } - - // Relational operators - template - bool operator==(const shared_ptr &other) const noexcept { - return internal == other.internal; - } - - bool operator==(std::nullptr_t) const noexcept { - return internal == nullptr; - } - - template - bool operator!=(const shared_ptr &other) const noexcept { - return internal != other.internal; - } - - template - bool operator<(const shared_ptr &other) const noexcept { - return internal < other.internal; - } - - template - bool operator<=(const shared_ptr &other) const noexcept { - return internal <= other.internal; - } - - template - bool operator>(const shared_ptr &other) const noexcept { - return internal > other.internal; - } - - template - bool operator>=(const shared_ptr &other) const noexcept { - return internal >= other.internal; - } -}; - -} // namespace duckdb +#include "duckdb/common/shared_ptr.ipp" +#include "duckdb/common/weak_ptr.ipp" +#include "duckdb/common/enable_shared_from_this.ipp" diff --git a/src/include/duckdb/common/shared_ptr.ipp b/src/include/duckdb/common/shared_ptr.ipp new file mode 100644 index 000000000000..f95901521664 --- /dev/null +++ b/src/include/duckdb/common/shared_ptr.ipp @@ -0,0 +1,150 @@ + +namespace duckdb { + +template +class weak_ptr; + +template +class shared_ptr { +private: + template + friend class weak_ptr; + std::shared_ptr internal; + +public: + // Constructors + shared_ptr() : internal() { + } + shared_ptr(std::nullptr_t) : internal(nullptr) { + } // Implicit conversion + template + explicit shared_ptr(U *ptr) : internal(ptr) { + } + // Constructor with custom deleter + template + shared_ptr(T *ptr, Deleter deleter) : internal(ptr, deleter) { + } + + shared_ptr(const shared_ptr &other) : internal(other.internal) { + } + + shared_ptr(std::shared_ptr other) : internal(std::move(other)) { + } + shared_ptr(shared_ptr &&other) : internal(std::move(other.internal)) { + } + + template + explicit shared_ptr(weak_ptr other) : internal(other.internal) { + } + + template ::value && __compatible_with::value && + std::is_convertible::pointer, T *>::value, + int> = 0> + shared_ptr(unique_ptr other) : internal(other.release()) { + } + + template ::value && __compatible_with::value && + std::is_convertible::pointer, T *>::value, + int> = 0> + shared_ptr(unique_ptr &&other) : internal(other.release()) { + } + + // Destructor + ~shared_ptr() = default; + + // Assignment operators + shared_ptr &operator=(const shared_ptr &other) { + internal = other.internal; + return *this; + } + + template + shared_ptr &operator=(unique_ptr &&__r) { + shared_ptr(std::move(__r)).swap(*this); + return *this; + } + + // Modifiers + void reset() { + internal.reset(); + } + + template + void reset(U *ptr) { + internal.reset(ptr); + } + + template + void reset(U *ptr, Deleter deleter) { + internal.reset(ptr, deleter); + } + + // Observers + T *get() const { + return internal.get(); + } + + long use_count() const { + return internal.use_count(); + } + + explicit operator bool() const noexcept { + return internal.operator bool(); + } + + template + operator shared_ptr() const noexcept { + return shared_ptr(internal); + } + + // Element access + std::__add_lvalue_reference_t operator*() const { + return *internal; + } + + T *operator->() const { + return internal.operator->(); + } + + // Relational operators + template + bool operator==(const shared_ptr &other) const noexcept { + return internal == other.internal; + } + + bool operator==(std::nullptr_t) const noexcept { + return internal == nullptr; + } + + template + bool operator!=(const shared_ptr &other) const noexcept { + return internal != other.internal; + } + + template + bool operator<(const shared_ptr &other) const noexcept { + return internal < other.internal; + } + + template + bool operator<=(const shared_ptr &other) const noexcept { + return internal <= other.internal; + } + + template + bool operator>(const shared_ptr &other) const noexcept { + return internal > other.internal; + } + + template + bool operator>=(const shared_ptr &other) const noexcept { + return internal >= other.internal; + } + + template + friend shared_ptr shared_ptr_cast(shared_ptr src); +}; + +} // namespace duckdb diff --git a/src/include/duckdb/common/types.hpp b/src/include/duckdb/common/types.hpp index 0151bc31a85f..e9e31b488ea5 100644 --- a/src/include/duckdb/common/types.hpp +++ b/src/include/duckdb/common/types.hpp @@ -35,7 +35,7 @@ using buffer_ptr = shared_ptr; template buffer_ptr make_buffer(ARGS &&...args) { // NOLINT: mimic std casing - return make_shared(std::forward(args)...); + return make_refcounted(std::forward(args)...); } struct list_entry_t { // NOLINT: mimic std casing diff --git a/src/include/duckdb/common/types/selection_vector.hpp b/src/include/duckdb/common/types/selection_vector.hpp index db6d6e9bdece..a0f0b185beae 100644 --- a/src/include/duckdb/common/types/selection_vector.hpp +++ b/src/include/duckdb/common/types/selection_vector.hpp @@ -71,7 +71,7 @@ struct SelectionVector { sel_vector = sel; } void Initialize(idx_t count = STANDARD_VECTOR_SIZE) { - selection_data = make_shared(count); + selection_data = make_refcounted(count); sel_vector = selection_data->owned_data.get(); } void Initialize(buffer_ptr data) { diff --git a/src/include/duckdb/common/unique_ptr.hpp b/src/include/duckdb/common/unique_ptr.hpp index d9f0b835832c..b98f8da00030 100644 --- a/src/include/duckdb/common/unique_ptr.hpp +++ b/src/include/duckdb/common/unique_ptr.hpp @@ -9,10 +9,10 @@ namespace duckdb { -template , bool SAFE = true> -class unique_ptr : public std::unique_ptr { // NOLINT: naming +template , bool SAFE = true> +class unique_ptr : public std::unique_ptr { // NOLINT: naming public: - using original = std::unique_ptr; + using original = std::unique_ptr; using original::original; // NOLINT private: @@ -53,9 +53,9 @@ class unique_ptr : public std::unique_ptr { // NOLINT } }; -template -class unique_ptr - : public std::unique_ptr> { +// FIXME: DELETER is defined, but we use std::default_delete??? +template +class unique_ptr : public std::unique_ptr> { public: using original = std::unique_ptr>; using original::original; diff --git a/src/include/duckdb/common/weak_ptr.hpp b/src/include/duckdb/common/weak_ptr.ipp similarity index 87% rename from src/include/duckdb/common/weak_ptr.hpp rename to src/include/duckdb/common/weak_ptr.ipp index bf442e02ad6a..5fbe213c92bc 100644 --- a/src/include/duckdb/common/weak_ptr.hpp +++ b/src/include/duckdb/common/weak_ptr.ipp @@ -1,20 +1,18 @@ -#pragma once - -#include "duckdb/common/shared_ptr.hpp" -#include - namespace duckdb { template class weak_ptr { private: + template + friend class shared_ptr; std::weak_ptr internal; public: // Constructors weak_ptr() : internal() { } - template + // template ::value, int> = 0> + template weak_ptr(const shared_ptr &ptr) : internal(ptr.internal) { } weak_ptr(const weak_ptr &other) : internal(other.internal) { @@ -29,7 +27,7 @@ class weak_ptr { return *this; } - template + template ::value, int> = 0> weak_ptr &operator=(const shared_ptr &ptr) { internal = ptr; return *this; diff --git a/src/include/duckdb/main/buffered_data/buffered_data.hpp b/src/include/duckdb/main/buffered_data/buffered_data.hpp index 8065fbee2c73..a863d551be6f 100644 --- a/src/include/duckdb/main/buffered_data/buffered_data.hpp +++ b/src/include/duckdb/main/buffered_data/buffered_data.hpp @@ -15,7 +15,7 @@ #include "duckdb/common/optional_idx.hpp" #include "duckdb/execution/physical_operator_states.hpp" #include "duckdb/common/enums/pending_execution_result.hpp" -#include "duckdb/common/weak_ptr.hpp" +#include "duckdb/common/shared_ptr.hpp" namespace duckdb { diff --git a/src/include/duckdb/main/client_context.hpp b/src/include/duckdb/main/client_context.hpp index 9f608273b668..1af4e9007deb 100644 --- a/src/include/duckdb/main/client_context.hpp +++ b/src/include/duckdb/main/client_context.hpp @@ -59,7 +59,7 @@ struct PendingQueryParameters { //! The ClientContext holds information relevant to the current client session //! during execution -class ClientContext : public std::enable_shared_from_this { +class ClientContext : public enable_shared_from_this { friend class PendingQueryResult; // LockContext friend class SimpleBufferedData; // ExecuteTaskInternal friend class StreamQueryResult; // LockContext diff --git a/src/include/duckdb/main/database.hpp b/src/include/duckdb/main/database.hpp index 0b87cf8f0ea2..5ec2b68ab01f 100644 --- a/src/include/duckdb/main/database.hpp +++ b/src/include/duckdb/main/database.hpp @@ -27,7 +27,7 @@ class ObjectCache; struct AttachInfo; class DatabaseFileSystem; -class DatabaseInstance : public std::enable_shared_from_this { +class DatabaseInstance : public enable_shared_from_this { friend class DuckDB; public: diff --git a/src/include/duckdb/main/relation.hpp b/src/include/duckdb/main/relation.hpp index c494366208cb..6d0ffa1d1cab 100644 --- a/src/include/duckdb/main/relation.hpp +++ b/src/include/duckdb/main/relation.hpp @@ -34,7 +34,7 @@ class LogicalOperator; class QueryNode; class TableRef; -class Relation : public std::enable_shared_from_this { +class Relation : public enable_shared_from_this { public: Relation(const shared_ptr &context, RelationType type) : context(context), type(type) { } diff --git a/src/include/duckdb/parallel/event.hpp b/src/include/duckdb/parallel/event.hpp index 89a108d98a98..b65dd0443c68 100644 --- a/src/include/duckdb/parallel/event.hpp +++ b/src/include/duckdb/parallel/event.hpp @@ -16,7 +16,7 @@ namespace duckdb { class Executor; class Task; -class Event : public std::enable_shared_from_this { +class Event : public enable_shared_from_this { public: explicit Event(Executor &executor); virtual ~Event() = default; diff --git a/src/include/duckdb/parallel/interrupt.hpp b/src/include/duckdb/parallel/interrupt.hpp index fe5348bc9395..f3c54aa29cdf 100644 --- a/src/include/duckdb/parallel/interrupt.hpp +++ b/src/include/duckdb/parallel/interrupt.hpp @@ -11,6 +11,7 @@ #include "duckdb/common/atomic.hpp" #include "duckdb/common/mutex.hpp" #include "duckdb/parallel/task.hpp" +#include "duckdb/common/shared_ptr.hpp" #include #include diff --git a/src/include/duckdb/parallel/meta_pipeline.hpp b/src/include/duckdb/parallel/meta_pipeline.hpp index 5bf58ef80cfd..f8f954fb4c62 100644 --- a/src/include/duckdb/parallel/meta_pipeline.hpp +++ b/src/include/duckdb/parallel/meta_pipeline.hpp @@ -14,7 +14,7 @@ namespace duckdb { //! MetaPipeline represents a set of pipelines that all have the same sink -class MetaPipeline : public std::enable_shared_from_this { +class MetaPipeline : public enable_shared_from_this { //! We follow these rules when building: //! 1. For joins, build out the blocking side before going down the probe side //! - The current streaming pipeline will have a dependency on it (dependency across MetaPipelines) diff --git a/src/include/duckdb/parallel/pipeline.hpp b/src/include/duckdb/parallel/pipeline.hpp index 28781200abb8..cb53777a2d61 100644 --- a/src/include/duckdb/parallel/pipeline.hpp +++ b/src/include/duckdb/parallel/pipeline.hpp @@ -66,7 +66,7 @@ class PipelineBuildState { }; //! The Pipeline class represents an execution pipeline starting at a -class Pipeline : public std::enable_shared_from_this { +class Pipeline : public enable_shared_from_this { friend class Executor; friend class PipelineExecutor; friend class PipelineEvent; diff --git a/src/include/duckdb/parallel/task.hpp b/src/include/duckdb/parallel/task.hpp index 2deadcbe3bad..2bcafeeaad5b 100644 --- a/src/include/duckdb/parallel/task.hpp +++ b/src/include/duckdb/parallel/task.hpp @@ -22,7 +22,7 @@ enum class TaskExecutionMode : uint8_t { PROCESS_ALL, PROCESS_PARTIAL }; enum class TaskExecutionResult : uint8_t { TASK_FINISHED, TASK_NOT_FINISHED, TASK_ERROR, TASK_BLOCKED }; //! Generic parallel task -class Task : public std::enable_shared_from_this { +class Task : public enable_shared_from_this { public: virtual ~Task() { } diff --git a/src/include/duckdb/planner/binder.hpp b/src/include/duckdb/planner/binder.hpp index 2c73e761f940..8d096266439a 100644 --- a/src/include/duckdb/planner/binder.hpp +++ b/src/include/duckdb/planner/binder.hpp @@ -80,7 +80,7 @@ struct CorrelatedColumnInfo { tables and columns in the catalog. In the process, it also resolves types of all expressions. */ -class Binder : public std::enable_shared_from_this { +class Binder : public enable_shared_from_this { friend class ExpressionBinder; friend class RecursiveDependentJoinPlanner; @@ -376,7 +376,7 @@ class Binder : public std::enable_shared_from_this { unique_ptr BindSummarize(ShowRef &ref); public: - // This should really be a private constructor, but make_shared does not allow it... + // This should really be a private constructor, but make_refcounted does not allow it... // If you are thinking about calling this, you should probably call Binder::CreateBinder Binder(bool i_know_what_i_am_doing, ClientContext &context, shared_ptr parent, bool inherit_ctes); }; diff --git a/src/include/duckdb/storage/object_cache.hpp b/src/include/duckdb/storage/object_cache.hpp index 25b6ab69d2ea..06a5c2d3a767 100644 --- a/src/include/duckdb/storage/object_cache.hpp +++ b/src/include/duckdb/storage/object_cache.hpp @@ -43,7 +43,7 @@ class ObjectCache { if (!object || object->GetObjectType() != T::ObjectType()) { return nullptr; } - return std::static_pointer_cast(object); + return shared_ptr_cast(object); } template @@ -52,7 +52,7 @@ class ObjectCache { auto entry = cache.find(key); if (entry == cache.end()) { - auto value = make_shared(args...); + auto value = make_refcounted(args...); cache[key] = value; return value; } @@ -60,7 +60,7 @@ class ObjectCache { if (!object || object->GetObjectType() != T::ObjectType()) { return nullptr; } - return std::static_pointer_cast(object); + return shared_ptr_cast(object); } void Put(string key, shared_ptr value) { diff --git a/src/include/duckdb/storage/serialization/types.json b/src/include/duckdb/storage/serialization/types.json index dd4cf2b7f147..5433f50a9d15 100644 --- a/src/include/duckdb/storage/serialization/types.json +++ b/src/include/duckdb/storage/serialization/types.json @@ -155,7 +155,7 @@ "class": "GenericTypeInfo", "base": "ExtraTypeInfo", "enum": "GENERIC_TYPE_INFO", - "custom_switch_code": "result = make_shared(type);\nbreak;" + "custom_switch_code": "result = make_refcounted(type);\nbreak;" }, { "class": "AnyTypeInfo", diff --git a/src/include/duckdb/transaction/local_storage.hpp b/src/include/duckdb/transaction/local_storage.hpp index 7481abd70e8c..a2140444dea1 100644 --- a/src/include/duckdb/transaction/local_storage.hpp +++ b/src/include/duckdb/transaction/local_storage.hpp @@ -21,7 +21,7 @@ class WriteAheadLog; struct LocalAppendState; struct TableAppendState; -class LocalTableStorage : public std::enable_shared_from_this { +class LocalTableStorage : public enable_shared_from_this { public: // Create a new LocalTableStorage explicit LocalTableStorage(DataTable &table); diff --git a/src/include/duckdb/transaction/transaction.hpp b/src/include/duckdb/transaction/transaction.hpp index dff31db5701c..b1bc3952c330 100644 --- a/src/include/duckdb/transaction/transaction.hpp +++ b/src/include/duckdb/transaction/transaction.hpp @@ -13,7 +13,7 @@ #include "duckdb/transaction/undo_buffer.hpp" #include "duckdb/common/atomic.hpp" #include "duckdb/transaction/transaction_data.hpp" -#include "duckdb/common/weak_ptr.hpp" +#include "duckdb/common/shared_ptr.hpp" namespace duckdb { class SequenceCatalogEntry; diff --git a/src/main/capi/table_function-c.cpp b/src/main/capi/table_function-c.cpp index e6eb5e3549fd..57be51d01f4b 100644 --- a/src/main/capi/table_function-c.cpp +++ b/src/main/capi/table_function-c.cpp @@ -179,7 +179,7 @@ void CTableFunction(ClientContext &context, TableFunctionInput &data_p, DataChun duckdb_table_function duckdb_create_table_function() { auto function = new duckdb::TableFunction("", {}, duckdb::CTableFunction, duckdb::CTableFunctionBind, duckdb::CTableFunctionInit, duckdb::CTableFunctionLocalInit); - function->function_info = duckdb::make_shared(); + function->function_info = duckdb::make_refcounted(); function->cardinality = duckdb::CTableFunctionCardinality; return function; } diff --git a/src/main/client_context.cpp b/src/main/client_context.cpp index 909593aa66f8..835210c76d1e 100644 --- a/src/main/client_context.cpp +++ b/src/main/client_context.cpp @@ -312,7 +312,7 @@ ClientContext::CreatePreparedStatementInternal(ClientContextLock &lock, const st unique_ptr statement, optional_ptr> values) { StatementType statement_type = statement->type; - auto result = make_shared(statement_type); + auto result = make_refcounted(statement_type); auto &profiler = QueryProfiler::Get(*this); profiler.StartQuery(query, IsExplainAnalyze(statement.get()), true); diff --git a/src/main/client_data.cpp b/src/main/client_data.cpp index 1298df3ea84d..f00b237837c4 100644 --- a/src/main/client_data.cpp +++ b/src/main/client_data.cpp @@ -35,8 +35,8 @@ class ClientFileSystem : public OpenerFileSystem { ClientData::ClientData(ClientContext &context) : catalog_search_path(make_uniq(context)) { auto &db = DatabaseInstance::GetDatabase(context); - profiler = make_shared(context); - temporary_objects = make_shared(db, AttachedDatabaseType::TEMP_DATABASE); + profiler = make_refcounted(context); + temporary_objects = make_refcounted(db, AttachedDatabaseType::TEMP_DATABASE); temporary_objects->oid = DatabaseManager::Get(db).ModifyCatalog(); random_engine = make_uniq(); file_opener = make_uniq(context); diff --git a/src/main/connection.cpp b/src/main/connection.cpp index 432ca9c21727..b76d440647f7 100644 --- a/src/main/connection.cpp +++ b/src/main/connection.cpp @@ -18,7 +18,8 @@ namespace duckdb { -Connection::Connection(DatabaseInstance &database) : context(make_shared(database.shared_from_this())) { +Connection::Connection(DatabaseInstance &database) + : context(make_refcounted(database.shared_from_this())) { ConnectionManager::Get(database).AddConnection(*context); #ifdef DEBUG EnableProfiling(); @@ -186,7 +187,7 @@ shared_ptr Connection::Table(const string &schema_name, const string & if (!table_info) { throw CatalogException("Table '%s' does not exist!", table_name); } - return make_shared(context, std::move(table_info)); + return make_refcounted(context, std::move(table_info)); } shared_ptr Connection::View(const string &tname) { @@ -194,7 +195,7 @@ shared_ptr Connection::View(const string &tname) { } shared_ptr Connection::View(const string &schema_name, const string &table_name) { - return make_shared(context, schema_name, table_name); + return make_refcounted(context, schema_name, table_name); } shared_ptr Connection::TableFunction(const string &fname) { @@ -205,11 +206,11 @@ shared_ptr Connection::TableFunction(const string &fname) { shared_ptr Connection::TableFunction(const string &fname, const vector &values, const named_parameter_map_t &named_parameters) { - return make_shared(context, fname, values, named_parameters); + return make_refcounted(context, fname, values, named_parameters); } shared_ptr Connection::TableFunction(const string &fname, const vector &values) { - return make_shared(context, fname, values); + return make_refcounted(context, fname, values); } shared_ptr Connection::Values(const vector> &values) { @@ -219,7 +220,7 @@ shared_ptr Connection::Values(const vector> &values) { shared_ptr Connection::Values(const vector> &values, const vector &column_names, const string &alias) { - return make_shared(context, values, column_names, alias); + return make_refcounted(context, values, column_names, alias); } shared_ptr Connection::Values(const string &values) { @@ -228,7 +229,7 @@ shared_ptr Connection::Values(const string &values) { } shared_ptr Connection::Values(const string &values, const vector &column_names, const string &alias) { - return make_shared(context, values, column_names, alias); + return make_refcounted(context, values, column_names, alias); } shared_ptr Connection::ReadCSV(const string &csv_file) { @@ -237,7 +238,7 @@ shared_ptr Connection::ReadCSV(const string &csv_file) { } shared_ptr Connection::ReadCSV(const vector &csv_input, named_parameter_map_t &&options) { - return make_shared(context, csv_input, std::move(options)); + return make_refcounted(context, csv_input, std::move(options)); } shared_ptr Connection::ReadCSV(const string &csv_input, named_parameter_map_t &&options) { @@ -258,7 +259,7 @@ shared_ptr Connection::ReadCSV(const string &csv_file, const vector files {csv_file}; - return make_shared(context, files, std::move(options)); + return make_refcounted(context, files, std::move(options)); } shared_ptr Connection::ReadParquet(const string &parquet_file, bool binary_as_string) { @@ -277,7 +278,7 @@ shared_ptr Connection::RelationFromQuery(const string &query, const st } shared_ptr Connection::RelationFromQuery(unique_ptr select_stmt, const string &alias) { - return make_shared(context, std::move(select_stmt), alias); + return make_refcounted(context, std::move(select_stmt), alias); } void Connection::BeginTransaction() { diff --git a/src/main/database.cpp b/src/main/database.cpp index 2e3fd0204e3a..a350f04ad610 100644 --- a/src/main/database.cpp +++ b/src/main/database.cpp @@ -264,7 +264,7 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf scheduler->RelaunchThreads(); } -DuckDB::DuckDB(const char *path, DBConfig *new_config) : instance(make_shared()) { +DuckDB::DuckDB(const char *path, DBConfig *new_config) : instance(make_refcounted()) { instance->Initialize(path, new_config); if (instance->config.options.load_extensions) { ExtensionHelper::LoadAllExtensions(*this); @@ -368,7 +368,7 @@ void DatabaseInstance::Configure(DBConfig &new_config) { if (new_config.buffer_pool) { config.buffer_pool = std::move(new_config.buffer_pool); } else { - config.buffer_pool = make_shared(config.options.maximum_memory); + config.buffer_pool = make_refcounted(config.options.maximum_memory); } } diff --git a/src/main/db_instance_cache.cpp b/src/main/db_instance_cache.cpp index 342af27f3c75..6105066c6c88 100644 --- a/src/main/db_instance_cache.cpp +++ b/src/main/db_instance_cache.cpp @@ -66,7 +66,7 @@ shared_ptr DBInstanceCache::CreateInstanceInternal(const string &databas if (abs_database_path.rfind(IN_MEMORY_PATH, 0) == 0) { instance_path = IN_MEMORY_PATH; } - auto db_instance = make_shared(instance_path, &config); + auto db_instance = make_refcounted(instance_path, &config); if (cache_instance) { db_instances[abs_database_path] = db_instance; } diff --git a/src/main/relation.cpp b/src/main/relation.cpp index 87be11809ff4..f315906bd200 100644 --- a/src/main/relation.cpp +++ b/src/main/relation.cpp @@ -39,7 +39,7 @@ shared_ptr Relation::Project(const string &expression, const string &a shared_ptr Relation::Project(const string &select_list, const vector &aliases) { auto expressions = Parser::ParseExpressionList(select_list, context.GetContext()->GetParserOptions()); - return make_shared(shared_from_this(), std::move(expressions), aliases); + return make_refcounted(shared_from_this(), std::move(expressions), aliases); } shared_ptr Relation::Project(const vector &expressions) { @@ -49,7 +49,7 @@ shared_ptr Relation::Project(const vector &expressions) { shared_ptr Relation::Project(vector> expressions, const vector &aliases) { - return make_shared(shared_from_this(), std::move(expressions), aliases); + return make_refcounted(shared_from_this(), std::move(expressions), aliases); } static vector> StringListToExpressionList(ClientContext &context, @@ -70,7 +70,7 @@ static vector> StringListToExpressionList(ClientCon shared_ptr Relation::Project(const vector &expressions, const vector &aliases) { auto result_list = StringListToExpressionList(*context.GetContext(), expressions); - return make_shared(shared_from_this(), std::move(result_list), aliases); + return make_refcounted(shared_from_this(), std::move(result_list), aliases); } shared_ptr Relation::Filter(const string &expression) { @@ -82,7 +82,7 @@ shared_ptr Relation::Filter(const string &expression) { } shared_ptr Relation::Filter(unique_ptr expression) { - return make_shared(shared_from_this(), std::move(expression)); + return make_refcounted(shared_from_this(), std::move(expression)); } shared_ptr Relation::Filter(const vector &expressions) { @@ -95,11 +95,11 @@ shared_ptr Relation::Filter(const vector &expressions) { expr = make_uniq(ExpressionType::CONJUNCTION_AND, std::move(expr), std::move(expression_list[i])); } - return make_shared(shared_from_this(), std::move(expr)); + return make_refcounted(shared_from_this(), std::move(expr)); } shared_ptr Relation::Limit(int64_t limit, int64_t offset) { - return make_shared(shared_from_this(), limit, offset); + return make_refcounted(shared_from_this(), limit, offset); } shared_ptr Relation::Order(const string &expression) { @@ -108,7 +108,7 @@ shared_ptr Relation::Order(const string &expression) { } shared_ptr Relation::Order(vector order_list) { - return make_shared(shared_from_this(), std::move(order_list)); + return make_refcounted(shared_from_this(), std::move(order_list)); } shared_ptr Relation::Order(const vector &expressions) { @@ -149,51 +149,51 @@ shared_ptr Relation::Join(const shared_ptr &other, } using_columns.push_back(colref.column_names[0]); } - return make_shared(shared_from_this(), other, std::move(using_columns), type, ref_type); + return make_refcounted(shared_from_this(), other, std::move(using_columns), type, ref_type); } else { // single expression that is not a column reference: use the expression as a join condition - return make_shared(shared_from_this(), other, std::move(expression_list[0]), type, ref_type); + return make_refcounted(shared_from_this(), other, std::move(expression_list[0]), type, ref_type); } } shared_ptr Relation::CrossProduct(const shared_ptr &other, JoinRefType join_ref_type) { - return make_shared(shared_from_this(), other, join_ref_type); + return make_refcounted(shared_from_this(), other, join_ref_type); } shared_ptr Relation::Union(const shared_ptr &other) { - return make_shared(shared_from_this(), other, SetOperationType::UNION, true); + return make_refcounted(shared_from_this(), other, SetOperationType::UNION, true); } shared_ptr Relation::Except(const shared_ptr &other) { - return make_shared(shared_from_this(), other, SetOperationType::EXCEPT, true); + return make_refcounted(shared_from_this(), other, SetOperationType::EXCEPT, true); } shared_ptr Relation::Intersect(const shared_ptr &other) { - return make_shared(shared_from_this(), other, SetOperationType::INTERSECT, true); + return make_refcounted(shared_from_this(), other, SetOperationType::INTERSECT, true); } shared_ptr Relation::Distinct() { - return make_shared(shared_from_this()); + return make_refcounted(shared_from_this()); } shared_ptr Relation::Alias(const string &alias) { - return make_shared(shared_from_this(), alias); + return make_refcounted(shared_from_this(), alias); } shared_ptr Relation::Aggregate(const string &aggregate_list) { auto expression_list = Parser::ParseExpressionList(aggregate_list, context.GetContext()->GetParserOptions()); - return make_shared(shared_from_this(), std::move(expression_list)); + return make_refcounted(shared_from_this(), std::move(expression_list)); } shared_ptr Relation::Aggregate(const string &aggregate_list, const string &group_list) { auto expression_list = Parser::ParseExpressionList(aggregate_list, context.GetContext()->GetParserOptions()); auto groups = Parser::ParseGroupByList(group_list, context.GetContext()->GetParserOptions()); - return make_shared(shared_from_this(), std::move(expression_list), std::move(groups)); + return make_refcounted(shared_from_this(), std::move(expression_list), std::move(groups)); } shared_ptr Relation::Aggregate(const vector &aggregates) { auto aggregate_list = StringListToExpressionList(*context.GetContext(), aggregates); - return make_shared(shared_from_this(), std::move(aggregate_list)); + return make_refcounted(shared_from_this(), std::move(aggregate_list)); } shared_ptr Relation::Aggregate(const vector &aggregates, const vector &groups) { @@ -204,7 +204,7 @@ shared_ptr Relation::Aggregate(const vector &aggregates, const shared_ptr Relation::Aggregate(vector> expressions, const string &group_list) { auto groups = Parser::ParseGroupByList(group_list, context.GetContext()->GetParserOptions()); - return make_shared(shared_from_this(), std::move(expressions), std::move(groups)); + return make_refcounted(shared_from_this(), std::move(expressions), std::move(groups)); } string Relation::GetAlias() { @@ -237,7 +237,7 @@ BoundStatement Relation::Bind(Binder &binder) { } shared_ptr Relation::InsertRel(const string &schema_name, const string &table_name) { - return make_shared(shared_from_this(), schema_name, table_name); + return make_refcounted(shared_from_this(), schema_name, table_name); } void Relation::Insert(const string &table_name) { @@ -255,12 +255,12 @@ void Relation::Insert(const string &schema_name, const string &table_name) { void Relation::Insert(const vector> &values) { vector column_names; - auto rel = make_shared(context.GetContext(), values, std::move(column_names), "values"); + auto rel = make_refcounted(context.GetContext(), values, std::move(column_names), "values"); rel->Insert(GetAlias()); } shared_ptr Relation::CreateRel(const string &schema_name, const string &table_name) { - return make_shared(shared_from_this(), schema_name, table_name); + return make_refcounted(shared_from_this(), schema_name, table_name); } void Relation::Create(const string &table_name) { @@ -277,7 +277,7 @@ void Relation::Create(const string &schema_name, const string &table_name) { } shared_ptr Relation::WriteCSVRel(const string &csv_file, case_insensitive_map_t> options) { - return make_shared(shared_from_this(), csv_file, std::move(options)); + return make_refcounted(shared_from_this(), csv_file, std::move(options)); } void Relation::WriteCSV(const string &csv_file, case_insensitive_map_t> options) { @@ -292,7 +292,7 @@ void Relation::WriteCSV(const string &csv_file, case_insensitive_map_t Relation::WriteParquetRel(const string &parquet_file, case_insensitive_map_t> options) { auto write_parquet = - make_shared(shared_from_this(), parquet_file, std::move(options)); + make_refcounted(shared_from_this(), parquet_file, std::move(options)); return std::move(write_parquet); } @@ -310,7 +310,7 @@ shared_ptr Relation::CreateView(const string &name, bool replace, bool } shared_ptr Relation::CreateView(const string &schema_name, const string &name, bool replace, bool temporary) { - auto view = make_shared(shared_from_this(), schema_name, name, replace, temporary); + auto view = make_refcounted(shared_from_this(), schema_name, name, replace, temporary); auto res = view->Execute(); if (res->HasError()) { const string prepended_message = "Failed to create view '" + name + "': "; @@ -329,7 +329,7 @@ unique_ptr Relation::Query(const string &name, const string &sql) { } unique_ptr Relation::Explain(ExplainType type) { - auto explain = make_shared(shared_from_this(), type); + auto explain = make_refcounted(shared_from_this(), type); return explain->Execute(); } @@ -343,12 +343,12 @@ void Relation::Delete(const string &condition) { shared_ptr Relation::TableFunction(const std::string &fname, const vector &values, const named_parameter_map_t &named_parameters) { - return make_shared(context.GetContext(), fname, values, named_parameters, - shared_from_this()); + return make_refcounted(context.GetContext(), fname, values, named_parameters, + shared_from_this()); } shared_ptr Relation::TableFunction(const std::string &fname, const vector &values) { - return make_shared(context.GetContext(), fname, values, shared_from_this()); + return make_refcounted(context.GetContext(), fname, values, shared_from_this()); } string Relation::ToString() { diff --git a/src/main/relation/read_csv_relation.cpp b/src/main/relation/read_csv_relation.cpp index 1529de9a7637..f63d535c13ab 100644 --- a/src/main/relation/read_csv_relation.cpp +++ b/src/main/relation/read_csv_relation.cpp @@ -56,7 +56,7 @@ ReadCSVRelation::ReadCSVRelation(const shared_ptr &context, const shared_ptr buffer_manager; context->RunFunctionInTransaction([&]() { - buffer_manager = make_shared(*context, csv_options, files[0], 0); + buffer_manager = make_refcounted(*context, csv_options, files[0], 0); CSVSniffer sniffer(csv_options, buffer_manager, CSVStateMachineCache::Get(*context)); auto sniffer_result = sniffer.SniffCSV(); auto &types = sniffer_result.return_types; diff --git a/src/main/relation/table_relation.cpp b/src/main/relation/table_relation.cpp index 2cdc0d9d945a..c37c88507849 100644 --- a/src/main/relation/table_relation.cpp +++ b/src/main/relation/table_relation.cpp @@ -56,14 +56,14 @@ void TableRelation::Update(const string &update_list, const string &condition) { vector> expressions; auto cond = ParseCondition(*context.GetContext(), condition); Parser::ParseUpdateList(update_list, update_columns, expressions, context.GetContext()->GetParserOptions()); - auto update = make_shared(context, std::move(cond), description->schema, description->table, - std::move(update_columns), std::move(expressions)); + auto update = make_refcounted(context, std::move(cond), description->schema, description->table, + std::move(update_columns), std::move(expressions)); update->Execute(); } void TableRelation::Delete(const string &condition) { auto cond = ParseCondition(*context.GetContext(), condition); - auto del = make_shared(context, std::move(cond), description->schema, description->table); + auto del = make_refcounted(context, std::move(cond), description->schema, description->table); del->Execute(); } diff --git a/src/parallel/executor.cpp b/src/parallel/executor.cpp index 41e710284c0e..a3be9b315843 100644 --- a/src/parallel/executor.cpp +++ b/src/parallel/executor.cpp @@ -73,10 +73,11 @@ void Executor::SchedulePipeline(const shared_ptr &meta_pipeline, S // create events/stack for the base pipeline auto base_pipeline = meta_pipeline->GetBasePipeline(); - auto base_initialize_event = make_shared(base_pipeline); - auto base_event = make_shared(base_pipeline); - auto base_finish_event = make_shared(base_pipeline); - auto base_complete_event = make_shared(base_pipeline->executor, event_data.initial_schedule); + auto base_initialize_event = make_refcounted(base_pipeline); + auto base_event = make_refcounted(base_pipeline); + auto base_finish_event = make_refcounted(base_pipeline); + auto base_complete_event = + make_refcounted(base_pipeline->executor, event_data.initial_schedule); PipelineEventStack base_stack(*base_initialize_event, *base_event, *base_finish_event, *base_complete_event); events.push_back(std::move(base_initialize_event)); events.push_back(std::move(base_event)); @@ -96,7 +97,7 @@ void Executor::SchedulePipeline(const shared_ptr &meta_pipeline, S D_ASSERT(pipeline); // create events/stack for this pipeline - auto pipeline_event = make_shared(pipeline); + auto pipeline_event = make_refcounted(pipeline); auto finish_group = meta_pipeline->GetFinishGroup(*pipeline); if (finish_group) { @@ -115,7 +116,7 @@ void Executor::SchedulePipeline(const shared_ptr &meta_pipeline, S event_map.insert(make_pair(reference(*pipeline), pipeline_stack)); } else if (meta_pipeline->HasFinishEvent(*pipeline)) { // this pipeline has its own finish event (despite going into the same sink - Finalize twice!) - auto pipeline_finish_event = make_shared(pipeline); + auto pipeline_finish_event = make_refcounted(pipeline); PipelineEventStack pipeline_stack(base_stack.pipeline_initialize_event, *pipeline_event, *pipeline_finish_event, base_stack.pipeline_complete_event); events.push_back(std::move(pipeline_finish_event)); @@ -359,7 +360,7 @@ void Executor::InitializeInternal(PhysicalOperator &plan) { // build and ready the pipelines PipelineBuildState state; - auto root_pipeline = make_shared(*this, state, nullptr); + auto root_pipeline = make_refcounted(*this, state, nullptr); root_pipeline->Build(*physical_plan); root_pipeline->Ready(); @@ -570,7 +571,7 @@ shared_ptr Executor::CreateChildPipeline(Pipeline ¤t, PhysicalOp D_ASSERT(op.IsSource()); // found another operator that is a source, schedule a child pipeline // 'op' is the source, and the sink is the same - auto child_pipeline = make_shared(*this); + auto child_pipeline = make_refcounted(*this); child_pipeline->sink = current.sink; child_pipeline->source = &op; diff --git a/src/parallel/meta_pipeline.cpp b/src/parallel/meta_pipeline.cpp index ded1cb246112..b73515d8e694 100644 --- a/src/parallel/meta_pipeline.cpp +++ b/src/parallel/meta_pipeline.cpp @@ -82,7 +82,7 @@ void MetaPipeline::Ready() { } MetaPipeline &MetaPipeline::CreateChildMetaPipeline(Pipeline ¤t, PhysicalOperator &op) { - children.push_back(make_shared(executor, state, &op)); + children.push_back(make_refcounted(executor, state, &op)); auto child_meta_pipeline = children.back().get(); // child MetaPipeline must finish completely before this MetaPipeline can start current.AddDependency(child_meta_pipeline->GetBasePipeline()); @@ -92,7 +92,7 @@ MetaPipeline &MetaPipeline::CreateChildMetaPipeline(Pipeline ¤t, PhysicalO } Pipeline &MetaPipeline::CreatePipeline() { - pipelines.emplace_back(make_shared(executor)); + pipelines.emplace_back(make_refcounted(executor)); state.SetPipelineSink(*pipelines.back(), sink, next_batch_index++); return *pipelines.back(); } diff --git a/src/planner/bind_context.cpp b/src/planner/bind_context.cpp index eac1d69c2bd6..611c7b34414b 100644 --- a/src/planner/bind_context.cpp +++ b/src/planner/bind_context.cpp @@ -514,13 +514,13 @@ void BindContext::AddGenericBinding(idx_t index, const string &alias, const vect void BindContext::AddCTEBinding(idx_t index, const string &alias, const vector &names, const vector &types) { - auto binding = make_shared(BindingType::BASE, alias, types, names, index); + auto binding = make_refcounted(BindingType::BASE, alias, types, names, index); if (cte_bindings.find(alias) != cte_bindings.end()) { throw BinderException("Duplicate alias \"%s\" in query!", alias); } cte_bindings[alias] = std::move(binding); - cte_references[alias] = make_shared(0); + cte_references[alias] = make_refcounted(0); } void BindContext::AddContext(BindContext other) { diff --git a/src/planner/binder.cpp b/src/planner/binder.cpp index 75e2a0482364..9316c6f85f63 100644 --- a/src/planner/binder.cpp +++ b/src/planner/binder.cpp @@ -47,7 +47,7 @@ shared_ptr Binder::CreateBinder(ClientContext &context, optional_ptr(true, context, parent ? parent->shared_from_this() : nullptr, inherit_ctes); + return make_refcounted(true, context, parent ? parent->shared_from_this() : nullptr, inherit_ctes); } Binder::Binder(bool, ClientContext &context, shared_ptr parent_p, bool inherit_ctes_p) diff --git a/src/planner/bound_parameter_map.cpp b/src/planner/bound_parameter_map.cpp index 420fa3931fbb..61571cd8543f 100644 --- a/src/planner/bound_parameter_map.cpp +++ b/src/planner/bound_parameter_map.cpp @@ -33,7 +33,7 @@ shared_ptr BoundParameterMap::CreateOrGetData(const string & auto entry = parameters.find(identifier); if (entry == parameters.end()) { // no entry yet: create a new one - auto data = make_shared(); + auto data = make_refcounted(); data->return_type = GetReturnType(identifier); CreateNewParameter(identifier, data); diff --git a/src/planner/planner.cpp b/src/planner/planner.cpp index 2df3a698e185..37381c2b6ec9 100644 --- a/src/planner/planner.cpp +++ b/src/planner/planner.cpp @@ -101,7 +101,7 @@ shared_ptr Planner::PrepareSQLStatement(unique_ptr(copied_statement->type); + auto prepared_data = make_refcounted(copied_statement->type); prepared_data->unbound_statement = std::move(copied_statement); prepared_data->names = names; prepared_data->types = types; diff --git a/src/storage/buffer/block_manager.cpp b/src/storage/buffer/block_manager.cpp index 36c2b559f354..fdead1bc1e6b 100644 --- a/src/storage/buffer/block_manager.cpp +++ b/src/storage/buffer/block_manager.cpp @@ -23,7 +23,7 @@ shared_ptr BlockManager::RegisterBlock(block_id_t block_id) { } } // create a new block pointer for this block - auto result = make_shared(*this, block_id, MemoryTag::BASE_TABLE); + auto result = make_refcounted(*this, block_id, MemoryTag::BASE_TABLE); // register the block pointer in the set of blocks as a weak pointer blocks[block_id] = weak_ptr(result); return result; diff --git a/src/storage/checkpoint_manager.cpp b/src/storage/checkpoint_manager.cpp index 574fc0660d1b..fe91431d986b 100644 --- a/src/storage/checkpoint_manager.cpp +++ b/src/storage/checkpoint_manager.cpp @@ -425,7 +425,7 @@ void CheckpointReader::ReadIndex(ClientContext &context, Deserializer &deseriali // now we can look for the index in the catalog and assign the table info auto &index = catalog.CreateIndex(context, info)->Cast(); - index.info = make_shared(table.GetStorage().info, info.index_name); + index.info = make_refcounted(table.GetStorage().info, info.index_name); // insert the parsed expressions into the index so that we can (de)serialize them during consecutive checkpoints for (auto &parsed_expr : info.parsed_expressions) { diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index ac55613c9295..97b8bc3680ec 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -45,12 +45,12 @@ bool DataTableInfo::IsTemporary() const { DataTable::DataTable(AttachedDatabase &db, shared_ptr table_io_manager_p, const string &schema, const string &table, vector column_definitions_p, unique_ptr data) - : info(make_shared(db, std::move(table_io_manager_p), schema, table)), + : info(make_refcounted(db, std::move(table_io_manager_p), schema, table)), column_definitions(std::move(column_definitions_p)), db(db), is_root(true) { // initialize the table with the existing data from disk, if any auto types = GetTypes(); this->row_groups = - make_shared(info, TableIOManager::Get(*this).GetBlockManagerForRowData(), types, 0); + make_refcounted(info, TableIOManager::Get(*this).GetBlockManagerForRowData(), types, 0); if (data && data->row_group_count > 0) { this->row_groups->Initialize(*data); } else { diff --git a/src/storage/local_storage.cpp b/src/storage/local_storage.cpp index 2c7fb0fe1d79..235535607e51 100644 --- a/src/storage/local_storage.cpp +++ b/src/storage/local_storage.cpp @@ -18,8 +18,8 @@ LocalTableStorage::LocalTableStorage(DataTable &table) : table_ref(table), allocator(Allocator::Get(table.db)), deleted_rows(0), optimistic_writer(table), merged_storage(false) { auto types = table.GetTypes(); - row_groups = make_shared(table.info, TableIOManager::Get(table).GetBlockManagerForRowData(), - types, MAX_ROW_ID, 0); + row_groups = make_refcounted(table.info, TableIOManager::Get(table).GetBlockManagerForRowData(), + types, MAX_ROW_ID, 0); row_groups->InitializeEmpty(); table.info->indexes.Scan([&](Index &index) { @@ -250,7 +250,7 @@ LocalTableStorage &LocalTableManager::GetOrCreateStorage(DataTable &table) { lock_guard l(table_storage_lock); auto entry = table_storage.find(table); if (entry == table_storage.end()) { - auto new_storage = make_shared(table); + auto new_storage = make_refcounted(table); auto storage = new_storage.get(); table_storage.insert(make_pair(reference(table), std::move(new_storage))); return *storage; @@ -534,7 +534,7 @@ void LocalStorage::AddColumn(DataTable &old_dt, DataTable &new_dt, ColumnDefinit if (!storage) { return; } - auto new_storage = make_shared(context, new_dt, *storage, new_column, default_value); + auto new_storage = make_refcounted(context, new_dt, *storage, new_column, default_value); table_manager.InsertEntry(new_dt, std::move(new_storage)); } @@ -544,7 +544,7 @@ void LocalStorage::DropColumn(DataTable &old_dt, DataTable &new_dt, idx_t remove if (!storage) { return; } - auto new_storage = make_shared(new_dt, *storage, removed_column); + auto new_storage = make_refcounted(new_dt, *storage, removed_column); table_manager.InsertEntry(new_dt, std::move(new_storage)); } @@ -555,8 +555,8 @@ void LocalStorage::ChangeType(DataTable &old_dt, DataTable &new_dt, idx_t change if (!storage) { return; } - auto new_storage = - make_shared(context, new_dt, *storage, changed_idx, target_type, bound_columns, cast_expr); + auto new_storage = make_refcounted(context, new_dt, *storage, changed_idx, target_type, + bound_columns, cast_expr); table_manager.InsertEntry(new_dt, std::move(new_storage)); } diff --git a/src/storage/serialization/serialize_types.cpp b/src/storage/serialization/serialize_types.cpp index 0b75f518b8eb..1b3b37d87cd9 100644 --- a/src/storage/serialization/serialize_types.cpp +++ b/src/storage/serialization/serialize_types.cpp @@ -35,7 +35,7 @@ shared_ptr ExtraTypeInfo::Deserialize(Deserializer &deserializer) result = EnumTypeInfo::Deserialize(deserializer); break; case ExtraTypeInfoType::GENERIC_TYPE_INFO: - result = make_shared(type); + result = make_refcounted(type); break; case ExtraTypeInfoType::INTEGER_LITERAL_TYPE_INFO: result = IntegerLiteralTypeInfo::Deserialize(deserializer); diff --git a/src/storage/standard_buffer_manager.cpp b/src/storage/standard_buffer_manager.cpp index 5b801134f08e..f0055e8cb3ef 100644 --- a/src/storage/standard_buffer_manager.cpp +++ b/src/storage/standard_buffer_manager.cpp @@ -98,8 +98,8 @@ shared_ptr StandardBufferManager::RegisterSmallMemory(idx_t block_s auto buffer = ConstructManagedBuffer(block_size, nullptr, FileBufferType::TINY_BUFFER); // create a new block pointer for this block - auto result = make_shared(*temp_block_manager, ++temporary_id, MemoryTag::BASE_TABLE, - std::move(buffer), false, block_size, std::move(reservation)); + auto result = make_refcounted(*temp_block_manager, ++temporary_id, MemoryTag::BASE_TABLE, + std::move(buffer), false, block_size, std::move(reservation)); #ifdef DUCKDB_DEBUG_DESTROY_BLOCKS // Initialize the memory with garbage data WriteGarbageIntoBuffer(*result->buffer); @@ -118,8 +118,8 @@ shared_ptr StandardBufferManager::RegisterMemory(MemoryTag tag, idx auto buffer = ConstructManagedBuffer(block_size, std::move(reusable_buffer)); // create a new block pointer for this block - return make_shared(*temp_block_manager, ++temporary_id, tag, std::move(buffer), can_destroy, - alloc_size, std::move(res)); + return make_refcounted(*temp_block_manager, ++temporary_id, tag, std::move(buffer), can_destroy, + alloc_size, std::move(res)); } BufferHandle StandardBufferManager::Allocate(MemoryTag tag, idx_t block_size, bool can_destroy, diff --git a/src/storage/statistics/column_statistics.cpp b/src/storage/statistics/column_statistics.cpp index e2c2b45b97f9..67d67417b671 100644 --- a/src/storage/statistics/column_statistics.cpp +++ b/src/storage/statistics/column_statistics.cpp @@ -14,7 +14,7 @@ ColumnStatistics::ColumnStatistics(BaseStatistics stats_p, unique_ptr ColumnStatistics::CreateEmptyStats(const LogicalType &type) { - return make_shared(BaseStatistics::CreateEmpty(type)); + return make_refcounted(BaseStatistics::CreateEmpty(type)); } void ColumnStatistics::Merge(ColumnStatistics &other) { @@ -53,7 +53,7 @@ void ColumnStatistics::UpdateDistinctStatistics(Vector &v, idx_t count) { } shared_ptr ColumnStatistics::Copy() const { - return make_shared(stats.Copy(), distinct_stats ? distinct_stats->Copy() : nullptr); + return make_refcounted(stats.Copy(), distinct_stats ? distinct_stats->Copy() : nullptr); } void ColumnStatistics::Serialize(Serializer &serializer) const { @@ -65,7 +65,7 @@ shared_ptr ColumnStatistics::Deserialize(Deserializer &deseria auto stats = deserializer.ReadProperty(100, "statistics"); auto distinct_stats = deserializer.ReadPropertyWithDefault>( 101, "distinct", unique_ptr()); - return make_shared(std::move(stats), std::move(distinct_stats)); + return make_refcounted(std::move(stats), std::move(distinct_stats)); } } // namespace duckdb diff --git a/src/storage/table/row_group.cpp b/src/storage/table/row_group.cpp index 824827d76a97..4d232e3f53c9 100644 --- a/src/storage/table/row_group.cpp +++ b/src/storage/table/row_group.cpp @@ -624,7 +624,7 @@ shared_ptr &RowGroup::GetOrCreateVersionInfoPtr() { if (!vinfo) { lock_guard lock(row_group_lock); if (!version_info) { - version_info = make_shared(start); + version_info = make_refcounted(start); } } return version_info; diff --git a/src/storage/table/row_group_collection.cpp b/src/storage/table/row_group_collection.cpp index 00e42d5b7b7f..c333ea6b1c10 100644 --- a/src/storage/table/row_group_collection.cpp +++ b/src/storage/table/row_group_collection.cpp @@ -55,7 +55,7 @@ RowGroupCollection::RowGroupCollection(shared_ptr info_p, BlockMa vector types_p, idx_t row_start_p, idx_t total_rows_p) : block_manager(block_manager), total_rows(total_rows_p), info(std::move(info_p)), types(std::move(types_p)), row_start(row_start_p), allocation_size(0) { - row_groups = make_shared(*this); + row_groups = make_refcounted(*this); } idx_t RowGroupCollection::GetTotalRows() const { @@ -1031,7 +1031,7 @@ shared_ptr RowGroupCollection::AddColumn(ClientContext &cont auto new_types = types; new_types.push_back(new_column.GetType()); auto result = - make_shared(info, block_manager, std::move(new_types), row_start, total_rows.load()); + make_refcounted(info, block_manager, std::move(new_types), row_start, total_rows.load()); ExpressionExecutor executor(context); DataChunk dummy_chunk; @@ -1059,7 +1059,7 @@ shared_ptr RowGroupCollection::RemoveColumn(idx_t col_idx) { new_types.erase(new_types.begin() + col_idx); auto result = - make_shared(info, block_manager, std::move(new_types), row_start, total_rows.load()); + make_refcounted(info, block_manager, std::move(new_types), row_start, total_rows.load()); result->stats.InitializeRemoveColumn(stats, col_idx); for (auto ¤t_row_group : row_groups->Segments()) { @@ -1077,7 +1077,7 @@ shared_ptr RowGroupCollection::AlterType(ClientContext &cont new_types[changed_idx] = target_type; auto result = - make_shared(info, block_manager, std::move(new_types), row_start, total_rows.load()); + make_refcounted(info, block_manager, std::move(new_types), row_start, total_rows.load()); result->stats.InitializeAlterType(stats, changed_idx, target_type); vector scan_types; diff --git a/src/storage/table/row_version_manager.cpp b/src/storage/table/row_version_manager.cpp index ead21f89234f..711daa7d1881 100644 --- a/src/storage/table/row_version_manager.cpp +++ b/src/storage/table/row_version_manager.cpp @@ -212,7 +212,7 @@ shared_ptr RowVersionManager::Deserialize(MetaBlockPointer de if (!delete_pointer.IsValid()) { return nullptr; } - auto version_info = make_shared(start); + auto version_info = make_refcounted(start); MetadataReader source(manager, delete_pointer, &version_info->storage_pointers); auto chunk_count = source.Read(); D_ASSERT(chunk_count > 0); diff --git a/src/storage/wal_replay.cpp b/src/storage/wal_replay.cpp index 9699bbac3e9d..b3fea7dae501 100644 --- a/src/storage/wal_replay.cpp +++ b/src/storage/wal_replay.cpp @@ -580,7 +580,7 @@ void WriteAheadLogDeserializer::ReplayCreateIndex() { // create the index in the catalog auto &table = catalog.GetEntry(context, create_info->schema, info.table).Cast(); auto &index = catalog.CreateIndex(context, info)->Cast(); - index.info = make_shared(table.GetStorage().info, index.name); + index.info = make_refcounted(table.GetStorage().info, index.name); // insert the parsed expressions into the index so that we can (de)serialize them during consecutive checkpoints for (auto &parsed_expr : info.parsed_expressions) { From a3089af1a4d4fcfab0b8c6f9b57ddd329f902310 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 5 Apr 2024 17:29:41 +0200 Subject: [PATCH 223/603] CMake: Fix no-version case in Windows, `printf "" > file` was somehow not working there --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c6ca4e082b9..0e28c1aa4c61 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -822,7 +822,7 @@ function(build_loadable_extension_directory NAME OUTPUT_DIRECTORY EXTENSION_VERS COMMAND printf "${DUCKDB_NORMALIZED_VERSION}" > duckdb_version_file COMMAND - printf "${EXTENSION_VERSION}" > extension_version_file || (echo "CMake appears not to propagate correctly version information for the extension" && > extension_version_file) + printf "${EXTENSION_VERSION}" > extension_version_file || ( echo "CMake appears not to propagate correctly version information for the extension, using empty string" && rm extension_version_file && touch extension_version_file ) COMMAND bash ${CMAKE_SOURCE_DIR}/scripts/append_metadata.sh $ ${CMAKE_BINARY_DIR}/duckdb_platform_out duckdb_version_file extension_version_file ) @@ -1222,7 +1222,7 @@ if(NOT DUCKDB_EXPLICIT_PLATFORM) ${PROJECT_BINARY_DIR}) add_custom_target( duckdb_platform ALL - COMMAND duckdb_platform_binary > duckdb_platform_out || (echo "Provide explicit DUCKDB_PLATFORM=your_target_arch to avoid build-type detection of the platform" && exit 1) + COMMAND duckdb_platform_binary > duckdb_platform_out || ( echo "Provide explicit DUCKDB_PLATFORM=your_target_arch to avoid build-type detection of the platform" && exit 1 ) ) add_dependencies(duckdb_platform duckdb_platform_binary) else() From 1e0fe2a96910637abc0037bc762393093df3a3c9 Mon Sep 17 00:00:00 2001 From: Tishj Date: Sat, 6 Apr 2024 21:08:11 +0200 Subject: [PATCH 224/603] almost compiling --- extension/httpfs/httpfs.cpp | 4 +- extension/httpfs/s3fs.cpp | 2 +- extension/json/json_functions/copy_json.cpp | 2 +- extension/json/json_functions/read_json.cpp | 16 ++--- .../json/json_functions/read_json_objects.cpp | 10 +-- extension/parquet/column_reader.cpp | 10 +-- .../include/templated_column_reader.hpp | 2 +- extension/parquet/parquet_crypto.cpp | 13 ++-- extension/parquet/parquet_extension.cpp | 4 +- extension/parquet/parquet_reader.cpp | 4 +- extension/parquet/parquet_writer.cpp | 2 +- extension/sqlsmith/statement_generator.cpp | 2 +- .../sqlsmith/third_party/sqlsmith/expr.cc | 46 ++++++------- .../sqlsmith/third_party/sqlsmith/grammar.cc | 66 +++++++++---------- .../sqlsmith/third_party/sqlsmith/sqlsmith.cc | 12 ++-- .../csv_scanner/sniffer/dialect_detection.cpp | 1 + .../operator/order/physical_order.cpp | 1 + src/include/duckdb/common/shared_ptr.ipp | 32 ++++++--- src/include/duckdb/common/unique_ptr.hpp | 1 + src/include/duckdb/common/weak_ptr.ipp | 18 +++-- .../csv_scanner/column_count_scanner.hpp | 1 + test/api/test_object_cache.cpp | 2 +- test/api/test_relation_api.cpp | 16 ++--- test/sql/storage/test_buffer_manager.cpp | 6 +- 24 files changed, 151 insertions(+), 122 deletions(-) diff --git a/extension/httpfs/httpfs.cpp b/extension/httpfs/httpfs.cpp index 9240df3cb0ac..1127d18de541 100644 --- a/extension/httpfs/httpfs.cpp +++ b/extension/httpfs/httpfs.cpp @@ -556,7 +556,7 @@ static optional_ptr TryGetMetadataCache(optional_ptrregistered_state.find("http_cache"); if (lookup == client_context->registered_state.end()) { - auto cache = make_shared(true, true); + auto cache = make_refcounted(true, true); client_context->registered_state["http_cache"] = cache; return cache.get(); } else { @@ -571,7 +571,7 @@ void HTTPFileHandle::Initialize(optional_ptr opener) { auto &hfs = file_system.Cast(); state = HTTPState::TryGetState(opener); if (!state) { - state = make_shared(); + state = make_refcounted(); } auto current_cache = TryGetMetadataCache(opener, hfs); diff --git a/extension/httpfs/s3fs.cpp b/extension/httpfs/s3fs.cpp index bdc03ec10bb8..5ba265c52c80 100644 --- a/extension/httpfs/s3fs.cpp +++ b/extension/httpfs/s3fs.cpp @@ -567,7 +567,7 @@ shared_ptr S3FileHandle::GetBuffer(uint16_t write_buffer_idx) { auto buffer_handle = s3fs.Allocate(part_size, config_params.max_upload_threads); auto new_write_buffer = - make_shared(write_buffer_idx * part_size, part_size, std::move(buffer_handle)); + make_refcounted(write_buffer_idx * part_size, part_size, std::move(buffer_handle)); { unique_lock lck(write_buffers_lock); auto lookup_result = write_buffers.find(write_buffer_idx); diff --git a/extension/json/json_functions/copy_json.cpp b/extension/json/json_functions/copy_json.cpp index dd26e6d0c0b1..a42c75aede5c 100644 --- a/extension/json/json_functions/copy_json.cpp +++ b/extension/json/json_functions/copy_json.cpp @@ -184,7 +184,7 @@ CopyFunction JSONFunctions::GetJSONCopyFunction() { function.plan = CopyToJSONPlan; function.copy_from_bind = CopyFromJSONBind; - function.copy_from_function = JSONFunctions::GetReadJSONTableFunction(make_shared( + function.copy_from_function = JSONFunctions::GetReadJSONTableFunction(make_refcounted( JSONScanType::READ_JSON, JSONFormat::NEWLINE_DELIMITED, JSONRecordType::RECORDS, false)); return function; diff --git a/extension/json/json_functions/read_json.cpp b/extension/json/json_functions/read_json.cpp index 3640e0a7e42d..db4a43e751be 100644 --- a/extension/json/json_functions/read_json.cpp +++ b/extension/json/json_functions/read_json.cpp @@ -381,26 +381,26 @@ TableFunctionSet CreateJSONFunctionInfo(string name, shared_ptr in } TableFunctionSet JSONFunctions::GetReadJSONFunction() { - auto info = - make_shared(JSONScanType::READ_JSON, JSONFormat::AUTO_DETECT, JSONRecordType::AUTO_DETECT, true); + auto info = make_refcounted(JSONScanType::READ_JSON, JSONFormat::AUTO_DETECT, + JSONRecordType::AUTO_DETECT, true); return CreateJSONFunctionInfo("read_json", std::move(info)); } TableFunctionSet JSONFunctions::GetReadNDJSONFunction() { - auto info = make_shared(JSONScanType::READ_JSON, JSONFormat::NEWLINE_DELIMITED, - JSONRecordType::AUTO_DETECT, true); + auto info = make_refcounted(JSONScanType::READ_JSON, JSONFormat::NEWLINE_DELIMITED, + JSONRecordType::AUTO_DETECT, true); return CreateJSONFunctionInfo("read_ndjson", std::move(info)); } TableFunctionSet JSONFunctions::GetReadJSONAutoFunction() { - auto info = - make_shared(JSONScanType::READ_JSON, JSONFormat::AUTO_DETECT, JSONRecordType::AUTO_DETECT, true); + auto info = make_refcounted(JSONScanType::READ_JSON, JSONFormat::AUTO_DETECT, + JSONRecordType::AUTO_DETECT, true); return CreateJSONFunctionInfo("read_json_auto", std::move(info)); } TableFunctionSet JSONFunctions::GetReadNDJSONAutoFunction() { - auto info = make_shared(JSONScanType::READ_JSON, JSONFormat::NEWLINE_DELIMITED, - JSONRecordType::AUTO_DETECT, true); + auto info = make_refcounted(JSONScanType::READ_JSON, JSONFormat::NEWLINE_DELIMITED, + JSONRecordType::AUTO_DETECT, true); return CreateJSONFunctionInfo("read_ndjson_auto", std::move(info)); } diff --git a/extension/json/json_functions/read_json_objects.cpp b/extension/json/json_functions/read_json_objects.cpp index 197a6a34843b..891d142dfaf2 100644 --- a/extension/json/json_functions/read_json_objects.cpp +++ b/extension/json/json_functions/read_json_objects.cpp @@ -61,7 +61,7 @@ TableFunction GetReadJSONObjectsTableFunction(bool list_parameter, shared_ptr(JSONScanType::READ_JSON_OBJECTS, JSONFormat::ARRAY, JSONRecordType::RECORDS); + make_refcounted(JSONScanType::READ_JSON_OBJECTS, JSONFormat::ARRAY, JSONRecordType::RECORDS); function_set.AddFunction(GetReadJSONObjectsTableFunction(false, function_info)); function_set.AddFunction(GetReadJSONObjectsTableFunction(true, function_info)); return function_set; @@ -69,8 +69,8 @@ TableFunctionSet JSONFunctions::GetReadJSONObjectsFunction() { TableFunctionSet JSONFunctions::GetReadNDJSONObjectsFunction() { TableFunctionSet function_set("read_ndjson_objects"); - auto function_info = make_shared(JSONScanType::READ_JSON_OBJECTS, JSONFormat::NEWLINE_DELIMITED, - JSONRecordType::RECORDS); + auto function_info = make_refcounted(JSONScanType::READ_JSON_OBJECTS, JSONFormat::NEWLINE_DELIMITED, + JSONRecordType::RECORDS); function_set.AddFunction(GetReadJSONObjectsTableFunction(false, function_info)); function_set.AddFunction(GetReadJSONObjectsTableFunction(true, function_info)); return function_set; @@ -78,8 +78,8 @@ TableFunctionSet JSONFunctions::GetReadNDJSONObjectsFunction() { TableFunctionSet JSONFunctions::GetReadJSONObjectsAutoFunction() { TableFunctionSet function_set("read_json_objects_auto"); - auto function_info = - make_shared(JSONScanType::READ_JSON_OBJECTS, JSONFormat::AUTO_DETECT, JSONRecordType::RECORDS); + auto function_info = make_refcounted(JSONScanType::READ_JSON_OBJECTS, JSONFormat::AUTO_DETECT, + JSONRecordType::RECORDS); function_set.AddFunction(GetReadJSONObjectsTableFunction(false, function_info)); function_set.AddFunction(GetReadJSONObjectsTableFunction(true, function_info)); return function_set; diff --git a/extension/parquet/column_reader.cpp b/extension/parquet/column_reader.cpp index b21d85fde215..285e9572e7ad 100644 --- a/extension/parquet/column_reader.cpp +++ b/extension/parquet/column_reader.cpp @@ -303,7 +303,7 @@ void ColumnReader::PreparePageV2(PageHeader &page_hdr) { void ColumnReader::AllocateBlock(idx_t size) { if (!block) { - block = make_shared(GetAllocator(), size); + block = make_refcounted(GetAllocator(), size); } else { block->resize(GetAllocator(), size); } @@ -515,7 +515,7 @@ idx_t ColumnReader::Read(uint64_t num_values, parquet_filter_t &filter, data_ptr result); } else if (dbp_decoder) { // TODO keep this in the state - auto read_buf = make_shared(); + auto read_buf = make_refcounted(); switch (schema.type) { case duckdb_parquet::format::Type::INT32: @@ -536,7 +536,7 @@ idx_t ColumnReader::Read(uint64_t num_values, parquet_filter_t &filter, data_ptr } else if (rle_decoder) { // RLE encoding for boolean D_ASSERT(type.id() == LogicalTypeId::BOOLEAN); - auto read_buf = make_shared(); + auto read_buf = make_refcounted(); read_buf->resize(reader.allocator, sizeof(bool) * (read_now - null_count)); rle_decoder->GetBatch(read_buf->ptr, read_now - null_count); PlainTemplated>(read_buf, define_out, read_now, filter, @@ -545,7 +545,7 @@ idx_t ColumnReader::Read(uint64_t num_values, parquet_filter_t &filter, data_ptr // DELTA_BYTE_ARRAY or DELTA_LENGTH_BYTE_ARRAY DeltaByteArray(define_out, read_now, filter, result_offset, result); } else if (bss_decoder) { - auto read_buf = make_shared(); + auto read_buf = make_refcounted(); switch (schema.type) { case duckdb_parquet::format::Type::FLOAT: @@ -661,7 +661,7 @@ void StringColumnReader::Dictionary(shared_ptr data, idx_t num static shared_ptr ReadDbpData(Allocator &allocator, ResizeableBuffer &buffer, idx_t &value_count) { auto decoder = make_uniq(buffer.ptr, buffer.len); value_count = decoder->TotalValues(); - auto result = make_shared(); + auto result = make_refcounted(); result->resize(allocator, sizeof(uint32_t) * value_count); decoder->GetBatch(result->ptr, value_count); decoder->Finalize(); diff --git a/extension/parquet/include/templated_column_reader.hpp b/extension/parquet/include/templated_column_reader.hpp index 59a1c13c4781..c2c7740c0902 100644 --- a/extension/parquet/include/templated_column_reader.hpp +++ b/extension/parquet/include/templated_column_reader.hpp @@ -43,7 +43,7 @@ class TemplatedColumnReader : public ColumnReader { public: void AllocateDict(idx_t size) { if (!dict) { - dict = make_shared(GetAllocator(), size); + dict = make_refcounted(GetAllocator(), size); } else { dict->resize(GetAllocator(), size); } diff --git a/extension/parquet/parquet_crypto.cpp b/extension/parquet/parquet_crypto.cpp index 6982d366da4c..d629899a672f 100644 --- a/extension/parquet/parquet_crypto.cpp +++ b/extension/parquet/parquet_crypto.cpp @@ -13,7 +13,7 @@ namespace duckdb { ParquetKeys &ParquetKeys::Get(ClientContext &context) { auto &cache = ObjectCache::GetObjectCache(context); if (!cache.Get(ParquetKeys::ObjectType())) { - cache.Put(ParquetKeys::ObjectType(), make_shared()); + cache.Put(ParquetKeys::ObjectType(), make_refcounted()); } return *cache.Get(ParquetKeys::ObjectType()); } @@ -300,13 +300,14 @@ class SimpleReadTransport : public TTransport { uint32_t ParquetCrypto::Read(TBase &object, TProtocol &iprot, const string &key) { // Create decryption protocol TCompactProtocolFactoryT tproto_factory; - auto dprot = tproto_factory.getProtocol(make_shared(iprot, key)); + auto dprot = tproto_factory.getProtocol(std::make_shared(iprot, key)); auto &dtrans = reinterpret_cast(*dprot->getTransport()); // We have to read the whole thing otherwise thrift throws an error before we realize we're decryption is wrong auto all = dtrans.ReadAll(); TCompactProtocolFactoryT tsimple_proto_factory; - auto simple_prot = tsimple_proto_factory.getProtocol(make_shared(all.get(), all.GetSize())); + auto simple_prot = + tsimple_proto_factory.getProtocol(std::make_shared(all.get(), all.GetSize())); // Read the object object.read(simple_prot.get()); @@ -317,7 +318,7 @@ uint32_t ParquetCrypto::Read(TBase &object, TProtocol &iprot, const string &key) uint32_t ParquetCrypto::Write(const TBase &object, TProtocol &oprot, const string &key) { // Create encryption protocol TCompactProtocolFactoryT tproto_factory; - auto eprot = tproto_factory.getProtocol(make_shared(oprot, key)); + auto eprot = tproto_factory.getProtocol(std::make_shared(oprot, key)); auto &etrans = reinterpret_cast(*eprot->getTransport()); // Write the object in memory @@ -331,7 +332,7 @@ uint32_t ParquetCrypto::ReadData(TProtocol &iprot, const data_ptr_t buffer, cons const string &key) { // Create decryption protocol TCompactProtocolFactoryT tproto_factory; - auto dprot = tproto_factory.getProtocol(make_shared(iprot, key)); + auto dprot = tproto_factory.getProtocol(std::make_shared(iprot, key)); auto &dtrans = reinterpret_cast(*dprot->getTransport()); // Read buffer @@ -346,7 +347,7 @@ uint32_t ParquetCrypto::WriteData(TProtocol &oprot, const const_data_ptr_t buffe // FIXME: we know the size upfront so we could do a streaming write instead of this // Create encryption protocol TCompactProtocolFactoryT tproto_factory; - auto eprot = tproto_factory.getProtocol(make_shared(oprot, key)); + auto eprot = tproto_factory.getProtocol(std::make_shared(oprot, key)); auto &etrans = reinterpret_cast(*eprot->getTransport()); // Write the data in memory diff --git a/extension/parquet/parquet_extension.cpp b/extension/parquet/parquet_extension.cpp index 36919143c85f..fb900c266ae9 100644 --- a/extension/parquet/parquet_extension.cpp +++ b/extension/parquet/parquet_extension.cpp @@ -540,7 +540,7 @@ class ParquetScanFunction { result->initial_reader = result->readers[0]; } else { result->initial_reader = - make_shared(context, bind_data.files[0], bind_data.parquet_options); + make_refcounted(context, bind_data.files[0], bind_data.parquet_options); result->readers[0] = result->initial_reader; } result->file_states[0] = ParquetFileState::OPEN; @@ -746,7 +746,7 @@ class ParquetScanFunction { shared_ptr reader; try { - reader = make_shared(context, file, pq_options); + reader = make_refcounted(context, file, pq_options); InitializeParquetReader(*reader, bind_data, parallel_state.column_ids, parallel_state.filters, context); } catch (...) { diff --git a/extension/parquet/parquet_reader.cpp b/extension/parquet/parquet_reader.cpp index efc16ff0ede2..ecf73d1ec542 100644 --- a/extension/parquet/parquet_reader.cpp +++ b/extension/parquet/parquet_reader.cpp @@ -49,7 +49,7 @@ using duckdb_parquet::format::Type; static unique_ptr CreateThriftFileProtocol(Allocator &allocator, FileHandle &file_handle, bool prefetch_mode) { - auto transport = make_shared(allocator, file_handle, prefetch_mode); + auto transport = make_refcounted(allocator, file_handle, prefetch_mode); return make_uniq>(std::move(transport)); } @@ -112,7 +112,7 @@ LoadMetadata(Allocator &allocator, FileHandle &file_handle, metadata->read(file_proto.get()); } - return make_shared(std::move(metadata), current_time); + return make_refcounted(std::move(metadata), current_time); } LogicalType ParquetReader::DeriveLogicalType(const SchemaElement &s_ele, bool binary_as_string) { diff --git a/extension/parquet/parquet_writer.cpp b/extension/parquet/parquet_writer.cpp index 7c299fa793a1..1e130964780a 100644 --- a/extension/parquet/parquet_writer.cpp +++ b/extension/parquet/parquet_writer.cpp @@ -366,7 +366,7 @@ ParquetWriter::ParquetWriter(FileSystem &fs, string file_name_p, vectorWriteData(const_data_ptr_cast("PAR1"), 4); } TCompactProtocolFactoryT tproto_factory; - protocol = tproto_factory.getProtocol(make_shared(*writer)); + protocol = tproto_factory.getProtocol(std::make_shared(*writer)); file_meta_data.num_rows = 0; file_meta_data.version = 1; diff --git a/extension/sqlsmith/statement_generator.cpp b/extension/sqlsmith/statement_generator.cpp index 973287df5dab..01e8609043e9 100644 --- a/extension/sqlsmith/statement_generator.cpp +++ b/extension/sqlsmith/statement_generator.cpp @@ -46,7 +46,7 @@ StatementGenerator::~StatementGenerator() { } shared_ptr StatementGenerator::GetDatabaseState(ClientContext &context) { - auto result = make_shared(); + auto result = make_refcounted(); result->test_types = TestAllTypesFun::GetTestTypes(); auto schemas = Catalog::GetAllSchemas(context); diff --git a/extension/sqlsmith/third_party/sqlsmith/expr.cc b/extension/sqlsmith/third_party/sqlsmith/expr.cc index 12e34a2ddfca..95d5fbfc813d 100644 --- a/extension/sqlsmith/third_party/sqlsmith/expr.cc +++ b/extension/sqlsmith/third_party/sqlsmith/expr.cc @@ -17,21 +17,21 @@ using impedance::matched; shared_ptr value_expr::factory(prod *p, sqltype *type_constraint) { try { if (1 == d20() && p->level < d6() && window_function::allowed(p)) - return make_shared(p, type_constraint); + return make_refcounted(p, type_constraint); else if (1 == d42() && p->level < d6()) - return make_shared(p, type_constraint); + return make_refcounted(p, type_constraint); else if (1 == d42() && p->level < d6()) - return make_shared(p, type_constraint); + return make_refcounted(p, type_constraint); else if (p->level < d6() && d6() == 1) - return make_shared(p, type_constraint); + return make_refcounted(p, type_constraint); else if (d12() == 1) - return make_shared(p, type_constraint); + return make_refcounted(p, type_constraint); else if (p->level < d6() && d9() == 1) - return make_shared(p, type_constraint); + return make_refcounted(p, type_constraint); else if (p->scope->refs.size() && d20() > 1) - return make_shared(p, type_constraint); + return make_refcounted(p, type_constraint); else - return make_shared(p, type_constraint); + return make_refcounted(p, type_constraint); } catch (runtime_error &e) { } p->retry(); @@ -89,18 +89,18 @@ column_reference::column_reference(prod *p, sqltype *type_constraint) : value_ex shared_ptr bool_expr::factory(prod *p) { try { if (p->level > d100()) - return make_shared(p); + return make_refcounted(p); if (d6() < 4) - return make_shared(p); + return make_refcounted(p); else if (d6() < 4) - return make_shared(p); + return make_refcounted(p); else if (d6() < 4) - return make_shared(p); + return make_refcounted(p); else if (d6() < 4) - return make_shared(p); + return make_refcounted(p); else - return make_shared(p); - // return make_shared(q); + return make_refcounted(p); + // return make_refcounted(q); } catch (runtime_error &e) { } p->retry(); @@ -108,7 +108,7 @@ shared_ptr bool_expr::factory(prod *p) { } exists_predicate::exists_predicate(prod *p) : bool_expr(p) { - subquery = make_shared(this, scope); + subquery = make_refcounted(this, scope); } void exists_predicate::accept(prod_visitor *v) { @@ -123,8 +123,8 @@ void exists_predicate::out(std::ostream &out) { } distinct_pred::distinct_pred(prod *p) : bool_binop(p) { - lhs = make_shared(this); - rhs = make_shared(this, lhs->type); + lhs = make_refcounted(this); + rhs = make_refcounted(this, lhs->type); } comparison_op::comparison_op(prod *p) : bool_binop(p) { @@ -330,15 +330,15 @@ void window_function::out(std::ostream &out) { window_function::window_function(prod *p, sqltype *type_constraint) : value_expr(p) { match(); - aggregate = make_shared(this, type_constraint, true); + aggregate = make_refcounted(this, type_constraint, true); type = aggregate->type; - partition_by.push_back(make_shared(this)); + partition_by.push_back(make_refcounted(this)); while (d6() > 4) - partition_by.push_back(make_shared(this)); + partition_by.push_back(make_refcounted(this)); - order_by.push_back(make_shared(this)); + order_by.push_back(make_refcounted(this)); while (d6() > 4) - order_by.push_back(make_shared(this)); + order_by.push_back(make_refcounted(this)); } bool window_function::allowed(prod *p) { diff --git a/extension/sqlsmith/third_party/sqlsmith/grammar.cc b/extension/sqlsmith/third_party/sqlsmith/grammar.cc index 040c2d135f0d..3335371e8820 100644 --- a/extension/sqlsmith/third_party/sqlsmith/grammar.cc +++ b/extension/sqlsmith/third_party/sqlsmith/grammar.cc @@ -16,14 +16,14 @@ shared_ptr table_ref::factory(prod *p) { try { if (p->level < 3 + d6()) { if (d6() > 3 && p->level < d6()) - return make_shared(p); + return make_refcounted(p); if (d6() > 3) - return make_shared(p); + return make_refcounted(p); } if (d6() > 3) - return make_shared(p); + return make_refcounted(p); else - return make_shared(p); + return make_refcounted(p); } catch (runtime_error &e) { p->retry(); } @@ -32,7 +32,7 @@ shared_ptr table_ref::factory(prod *p) { table_or_query_name::table_or_query_name(prod *p) : table_ref(p) { t = random_pick(scope->tables); - refs.push_back(make_shared(scope->stmt_uid("ref"), t)); + refs.push_back(make_refcounted(scope->stmt_uid("ref"), t)); } void table_or_query_name::out(std::ostream &out) { @@ -46,7 +46,7 @@ target_table::target_table(prod *p, table *victim) : table_ref(p) { retry(); } victim_ = victim; - refs.push_back(make_shared(scope->stmt_uid("target"), victim)); + refs.push_back(make_refcounted(scope->stmt_uid("target"), victim)); } void target_table::out(std::ostream &out) { @@ -62,7 +62,7 @@ table_sample::table_sample(prod *p) : table_ref(p) { retry(); } while (!t || !t->is_base_table); - refs.push_back(make_shared(scope->stmt_uid("sample"), t)); + refs.push_back(make_refcounted(scope->stmt_uid("sample"), t)); percent = 0.1 * d100(); method = (d6() > 2) ? "system" : "bernoulli"; } @@ -72,10 +72,10 @@ void table_sample::out(std::ostream &out) { } table_subquery::table_subquery(prod *p, bool lateral) : table_ref(p), is_lateral(lateral) { - query = make_shared(this, scope, lateral); + query = make_refcounted(this, scope, lateral); string alias = scope->stmt_uid("subq"); relation *aliased_rel = &query->select_list->derived_table; - refs.push_back(make_shared(alias, aliased_rel)); + refs.push_back(make_refcounted(alias, aliased_rel)); } table_subquery::~table_subquery() { @@ -89,9 +89,9 @@ void table_subquery::accept(prod_visitor *v) { shared_ptr join_cond::factory(prod *p, table_ref &lhs, table_ref &rhs) { try { if (d6() < 6) - return make_shared(p, lhs, rhs); + return make_refcounted(p, lhs, rhs); else - return make_shared(p, lhs, rhs); + return make_refcounted(p, lhs, rhs); } catch (runtime_error &e) { p->retry(); } @@ -196,7 +196,7 @@ from_clause::from_clause(prod *p) : prod(p) { // add a lateral subquery if (!impedance::matched(typeid(lateral_subquery))) break; - reflist.push_back(make_shared(this)); + reflist.push_back(make_refcounted(this)); for (auto r : reflist.back()->refs) scope->refs.push_back(&*r); } @@ -302,8 +302,8 @@ query_spec::query_spec(prod *p, struct scope *s, bool lateral) : prod(p), myscop if (lateral) scope->refs = s->refs; - from_clause = make_shared(this); - select_list = make_shared(this); + from_clause = make_refcounted(this); + select_list = make_refcounted(this); set_quantifier = (d100() == 1) ? "distinct" : ""; @@ -341,7 +341,7 @@ delete_stmt::delete_stmt(prod *p, struct scope *s, table *v) : modifying_stmt(p, delete_returning::delete_returning(prod *p, struct scope *s, table *victim) : delete_stmt(p, s, victim) { match(); - select_list = make_shared(this); + select_list = make_refcounted(this); } insert_stmt::insert_stmt(prod *p, struct scope *s, table *v) : modifying_stmt(p, s, v) { @@ -399,7 +399,7 @@ void set_list::out(std::ostream &out) { update_stmt::update_stmt(prod *p, struct scope *s, table *v) : modifying_stmt(p, s, v) { scope->refs.push_back(victim); search = bool_expr::factory(this); - set_list = make_shared(this, victim); + set_list = make_refcounted(this, victim); } void update_stmt::out(std::ostream &out) { @@ -409,7 +409,7 @@ void update_stmt::out(std::ostream &out) { update_returning::update_returning(prod *p, struct scope *s, table *v) : update_stmt(p, s, v) { match(); - select_list = make_shared(this); + select_list = make_refcounted(this); } upsert_stmt::upsert_stmt(prod *p, struct scope *s, table *v) : insert_stmt(p, s, v) { @@ -427,20 +427,20 @@ shared_ptr statement_factory(struct scope *s) { try { s->new_stmt(); if (d42() == 1) - return make_shared((struct prod *)0, s); + return make_refcounted((struct prod *)0, s); if (d42() == 1) - return make_shared((struct prod *)0, s); + return make_refcounted((struct prod *)0, s); else if (d42() == 1) - return make_shared((struct prod *)0, s); + return make_refcounted((struct prod *)0, s); else if (d42() == 1) { - return make_shared((struct prod *)0, s); + return make_refcounted((struct prod *)0, s); } else if (d42() == 1) - return make_shared((struct prod *)0, s); + return make_refcounted((struct prod *)0, s); else if (d6() > 4) - return make_shared((struct prod *)0, s); + return make_refcounted((struct prod *)0, s); else if (d6() > 5) - return make_shared((struct prod *)0, s); - return make_shared((struct prod *)0, s); + return make_refcounted((struct prod *)0, s); + return make_refcounted((struct prod *)0, s); } catch (runtime_error &e) { return statement_factory(s); } @@ -456,11 +456,11 @@ void common_table_expression::accept(prod_visitor *v) { common_table_expression::common_table_expression(prod *parent, struct scope *s) : prod(parent), myscope(s) { scope = &myscope; do { - shared_ptr query = make_shared(this, s); + shared_ptr query = make_refcounted(this, s); with_queries.push_back(query); string alias = scope->stmt_uid("jennifer"); relation *relation = &query->select_list->derived_table; - auto aliased_rel = make_shared(alias, relation); + auto aliased_rel = make_refcounted(alias, relation); refs.push_back(aliased_rel); scope->tables.push_back(&*aliased_rel); @@ -472,7 +472,7 @@ common_table_expression::common_table_expression(prod *parent, struct scope *s) scope->tables.push_back(pick); } while (d6() > 3); try { - query = make_shared(this, scope); + query = make_refcounted(this, scope); } catch (runtime_error &e) { retry(); goto retry; @@ -495,11 +495,11 @@ void common_table_expression::out(std::ostream &out) { merge_stmt::merge_stmt(prod *p, struct scope *s, table *v) : modifying_stmt(p, s, v) { match(); - target_table_ = make_shared(this, victim); + target_table_ = make_refcounted(this, victim); data_source = table_ref::factory(this); // join_condition = join_cond::factory(this, *target_table_, // *data_source); - join_condition = make_shared(this, *target_table_, *data_source); + join_condition = make_refcounted(this, *target_table_, *data_source); /* Put data_source into scope but not target_table. Visibility of the latter varies depending on kind of when clause. */ @@ -604,12 +604,12 @@ shared_ptr when_clause::factory(struct merge_stmt *p) { switch (d6()) { case 1: case 2: - return make_shared(p); + return make_refcounted(p); case 3: case 4: - return make_shared(p); + return make_refcounted(p); default: - return make_shared(p); + return make_refcounted(p); } } catch (runtime_error &e) { p->retry(); diff --git a/extension/sqlsmith/third_party/sqlsmith/sqlsmith.cc b/extension/sqlsmith/third_party/sqlsmith/sqlsmith.cc index b9eced4b3506..7673b6ac4392 100644 --- a/extension/sqlsmith/third_party/sqlsmith/sqlsmith.cc +++ b/extension/sqlsmith/third_party/sqlsmith/sqlsmith.cc @@ -83,7 +83,7 @@ int32_t run_sqlsmith(duckdb::DatabaseInstance &database, SQLSmithOptions opt) { try { shared_ptr schema; - schema = make_shared(database, opt.exclude_catalog, opt.verbose_output); + schema = make_refcounted(database, opt.exclude_catalog, opt.verbose_output); scope scope; long queries_generated = 0; @@ -97,20 +97,20 @@ int32_t run_sqlsmith(duckdb::DatabaseInstance &database, SQLSmithOptions opt) { duckdb::vector> loggers; - loggers.push_back(make_shared()); + loggers.push_back(make_refcounted()); if (opt.verbose_output) { - auto l = make_shared(); + auto l = make_refcounted(); global_cerr_logger = &*l; loggers.push_back(l); signal(SIGINT, cerr_log_handler); } if (opt.dump_all_graphs) - loggers.push_back(make_shared()); + loggers.push_back(make_refcounted()); if (opt.dump_all_queries) - loggers.push_back(make_shared()); + loggers.push_back(make_refcounted()); // if (options.count("dry-run")) { // while (1) { @@ -128,7 +128,7 @@ int32_t run_sqlsmith(duckdb::DatabaseInstance &database, SQLSmithOptions opt) { shared_ptr dut; - dut = make_shared(database); + dut = make_refcounted(database); if (opt.verbose_output) cerr << "Running queries..." << endl; diff --git a/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp b/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp index 80bddc9d158f..45f1b1803b55 100644 --- a/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp +++ b/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp @@ -1,5 +1,6 @@ #include "duckdb/execution/operator/csv_scanner/csv_sniffer.hpp" #include "duckdb/main/client_data.hpp" +#include "duckdb/common/shared_ptr.hpp" namespace duckdb { diff --git a/src/execution/operator/order/physical_order.cpp b/src/execution/operator/order/physical_order.cpp index 7aa77a7a30c0..8687acfe5ee1 100644 --- a/src/execution/operator/order/physical_order.cpp +++ b/src/execution/operator/order/physical_order.cpp @@ -6,6 +6,7 @@ #include "duckdb/parallel/base_pipeline_event.hpp" #include "duckdb/parallel/executor_task.hpp" #include "duckdb/storage/buffer_manager.hpp" +#include "duckdb/common/shared_ptr.hpp" namespace duckdb { diff --git a/src/include/duckdb/common/shared_ptr.ipp b/src/include/duckdb/common/shared_ptr.ipp index f95901521664..5db96b1b8cb6 100644 --- a/src/include/duckdb/common/shared_ptr.ipp +++ b/src/include/duckdb/common/shared_ptr.ipp @@ -10,6 +10,8 @@ private: template friend class weak_ptr; std::shared_ptr internal; + template + friend class shared_ptr; public: // Constructors @@ -24,6 +26,9 @@ public: template shared_ptr(T *ptr, Deleter deleter) : internal(ptr, deleter) { } + template + shared_ptr(const shared_ptr &__r, T *__p) noexcept : internal(__r.internal, __p) { + } shared_ptr(const shared_ptr &other) : internal(other.internal) { } @@ -37,17 +42,16 @@ public: explicit shared_ptr(weak_ptr other) : internal(other.internal) { } - template ::value && __compatible_with::value && - std::is_convertible::pointer, T *>::value, - int> = 0> - shared_ptr(unique_ptr other) : internal(other.release()) { +#if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR) + template ::value, int> = 0> + shared_ptr(std::auto_ptr &&__r) : internal(__r.release()) { } +#endif template ::value && __compatible_with::value && - std::is_convertible::pointer, T *>::value, - int> = 0> + typename std::enable_if<__compatible_with::value && + std::is_convertible::pointer, T *>::value, + int>::type = 0> shared_ptr(unique_ptr &&other) : internal(other.release()) { } @@ -60,7 +64,10 @@ public: return *this; } - template + template ::value && + std::is_convertible::pointer, T *>::value, + int>::type = 0> shared_ptr &operator=(unique_ptr &&__r) { shared_ptr(std::move(__r)).swap(*this); return *this; @@ -81,6 +88,10 @@ public: internal.reset(ptr, deleter); } + void swap(shared_ptr &r) noexcept { + internal.swap(r.internal); + } + // Observers T *get() const { return internal.get(); @@ -122,6 +133,9 @@ public: bool operator!=(const shared_ptr &other) const noexcept { return internal != other.internal; } + bool operator!=(std::nullptr_t) const noexcept { + return internal != nullptr; + } template bool operator<(const shared_ptr &other) const noexcept { diff --git a/src/include/duckdb/common/unique_ptr.hpp b/src/include/duckdb/common/unique_ptr.hpp index b98f8da00030..0689aeb6624b 100644 --- a/src/include/duckdb/common/unique_ptr.hpp +++ b/src/include/duckdb/common/unique_ptr.hpp @@ -14,6 +14,7 @@ class unique_ptr : public std::unique_ptr { // NOLINT: namin public: using original = std::unique_ptr; using original::original; // NOLINT + using pointer = typename original::pointer; private: static inline void AssertNotNull(const bool null) { diff --git a/src/include/duckdb/common/weak_ptr.ipp b/src/include/duckdb/common/weak_ptr.ipp index 5fbe213c92bc..2fd95699de2d 100644 --- a/src/include/duckdb/common/weak_ptr.ipp +++ b/src/include/duckdb/common/weak_ptr.ipp @@ -11,13 +11,23 @@ public: // Constructors weak_ptr() : internal() { } - // template ::value, int> = 0> + template - weak_ptr(const shared_ptr &ptr) : internal(ptr.internal) { + weak_ptr(shared_ptr const &ptr, typename std::enable_if<__compatible_with::value, int>::type = 0) noexcept + : internal(ptr.internal) { } - weak_ptr(const weak_ptr &other) : internal(other.internal) { + weak_ptr(weak_ptr const &other) noexcept : internal(other.internal) { + } + template + weak_ptr(weak_ptr const &ptr, typename std::enable_if<__compatible_with::value, int>::type = 0) noexcept + : internal(ptr.internal) { + } + weak_ptr(weak_ptr &&ptr) noexcept : internal(ptr.internal) { + } + template + weak_ptr(weak_ptr &&ptr, typename std::enable_if<__compatible_with::value, int>::type = 0) noexcept + : internal(ptr.internal) { } - // Destructor ~weak_ptr() = default; diff --git a/src/include/duckdb/execution/operator/csv_scanner/column_count_scanner.hpp b/src/include/duckdb/execution/operator/csv_scanner/column_count_scanner.hpp index ce2da9606d66..de25f08f235c 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/column_count_scanner.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/column_count_scanner.hpp @@ -13,6 +13,7 @@ #include "duckdb/execution/operator/csv_scanner/scanner_boundary.hpp" #include "duckdb/execution/operator/csv_scanner/string_value_scanner.hpp" #include "duckdb/execution/operator/csv_scanner/base_scanner.hpp" +#include "duckdb/common/shared_ptr.hpp" namespace duckdb { diff --git a/test/api/test_object_cache.cpp b/test/api/test_object_cache.cpp index a2dd2b5d8ab2..04f4ac7c91de 100644 --- a/test/api/test_object_cache.cpp +++ b/test/api/test_object_cache.cpp @@ -42,7 +42,7 @@ TEST_CASE("Test ObjectCache", "[api]") { auto &cache = ObjectCache::GetObjectCache(context); REQUIRE(cache.GetObject("test") == nullptr); - cache.Put("test", make_shared(42)); + cache.Put("test", make_refcounted(42)); REQUIRE(cache.GetObject("test") != nullptr); diff --git a/test/api/test_relation_api.cpp b/test/api/test_relation_api.cpp index 6d63bb3bbc43..740adedef620 100644 --- a/test/api/test_relation_api.cpp +++ b/test/api/test_relation_api.cpp @@ -12,7 +12,7 @@ TEST_CASE("Test simple relation API", "[relation_api]") { Connection con(db); con.EnableQueryVerification(); duckdb::unique_ptr result; - shared_ptr tbl, filter, proj, proj2, v1, v2, v3; + duckdb::shared_ptr tbl, filter, proj, proj2, v1, v2, v3; // create some tables REQUIRE_NO_FAIL(con.Query("CREATE TABLE integers(i INTEGER)")); @@ -215,7 +215,7 @@ TEST_CASE("Test combinations of set operations", "[relation_api]") { Connection con(db); con.EnableQueryVerification(); duckdb::unique_ptr result; - shared_ptr values, v1, v2, v3; + duckdb::shared_ptr values, v1, v2, v3; REQUIRE_NOTHROW(values = con.Values({{1, 10}, {2, 5}, {3, 4}}, {"i", "j"})); @@ -282,7 +282,7 @@ TEST_CASE("Test combinations of joins", "[relation_api]") { Connection con(db); con.EnableQueryVerification(); duckdb::unique_ptr result; - shared_ptr values, vjoin; + duckdb::shared_ptr values, vjoin; REQUIRE_NOTHROW(values = con.Values({{1, 10}, {2, 5}, {3, 4}}, {"i", "j"})); @@ -370,7 +370,7 @@ TEST_CASE("Test crossproduct relation", "[relation_api]") { Connection con(db); con.EnableQueryVerification(); duckdb::unique_ptr result; - shared_ptr values, vcross; + duckdb::shared_ptr values, vcross; REQUIRE_NOTHROW(values = con.Values({{1, 10}, {2, 5}, {3, 4}}, {"i", "j"}), "v1"); REQUIRE_NOTHROW(values = con.Values({{1, 10}, {2, 5}, {3, 4}}, {"i", "j"}), "v2"); @@ -401,7 +401,7 @@ TEST_CASE("Test view creation of relations", "[relation_api]") { Connection con(db); con.EnableQueryVerification(); duckdb::unique_ptr result; - shared_ptr tbl, filter, proj, proj2; + duckdb::shared_ptr tbl, filter, proj, proj2; // create some tables REQUIRE_NO_FAIL(con.Query("CREATE TABLE integers(i INTEGER)")); @@ -478,7 +478,7 @@ TEST_CASE("Test table creations using the relation API", "[relation_api]") { Connection con(db); con.EnableQueryVerification(); duckdb::unique_ptr result; - shared_ptr values; + duckdb::shared_ptr values; // create a table from a Values statement REQUIRE_NOTHROW(values = con.Values({{1, 10}, {2, 5}, {3, 4}}, {"i", "j"})); @@ -878,7 +878,7 @@ TEST_CASE("Test query relation", "[relation_api]") { Connection con(db); con.EnableQueryVerification(); duckdb::unique_ptr result; - shared_ptr tbl; + duckdb::shared_ptr tbl; // create some tables REQUIRE_NO_FAIL(con.Query("CREATE TABLE integers(i INTEGER)")); @@ -905,7 +905,7 @@ TEST_CASE("Test TopK relation", "[relation_api]") { Connection con(db); con.EnableQueryVerification(); duckdb::unique_ptr result; - shared_ptr tbl; + duckdb::shared_ptr tbl; REQUIRE_NO_FAIL(con.Query("CREATE TABLE test (i integer,j VARCHAR, k varchar )")); REQUIRE_NO_FAIL(con.Query("insert into test values (10,'a','a'), (20,'a','b')")); diff --git a/test/sql/storage/test_buffer_manager.cpp b/test/sql/storage/test_buffer_manager.cpp index e730fe892b92..572a6be9bb41 100644 --- a/test/sql/storage/test_buffer_manager.cpp +++ b/test/sql/storage/test_buffer_manager.cpp @@ -152,7 +152,7 @@ TEST_CASE("Test buffer reallocation", "[storage][.]") { CHECK(buffer_manager.GetUsedMemory() == 0); idx_t requested_size = Storage::BLOCK_SIZE; - shared_ptr block; + duckdb::shared_ptr block; auto handle = buffer_manager.Allocate(MemoryTag::EXTENSION, requested_size, false, &block); CHECK(buffer_manager.GetUsedMemory() == BufferManager::GetAllocSize(requested_size)); for (; requested_size < limit; requested_size *= 2) { @@ -196,7 +196,7 @@ TEST_CASE("Test buffer manager variable size allocations", "[storage][.]") { CHECK(buffer_manager.GetUsedMemory() == 0); idx_t requested_size = 424242; - shared_ptr block; + duckdb::shared_ptr block; auto pin = buffer_manager.Allocate(MemoryTag::EXTENSION, requested_size, false, &block); CHECK(buffer_manager.GetUsedMemory() >= requested_size + Storage::BLOCK_HEADER_SIZE); @@ -224,7 +224,7 @@ TEST_CASE("Test buffer manager buffer re-use", "[storage][.]") { // Create 40 blocks, but don't hold the pin // They will be added to the eviction queue and the buffers will be re-used idx_t block_count = 40; - duckdb::vector> blocks; + duckdb::vector> blocks; blocks.reserve(block_count); for (idx_t i = 0; i < block_count; i++) { blocks.emplace_back(); From da66e1f49aff53d36ce7b627e9811504070307d2 Mon Sep 17 00:00:00 2001 From: Tishj Date: Sat, 6 Apr 2024 21:56:28 +0200 Subject: [PATCH 225/603] compiling now --- extension/parquet/parquet_reader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extension/parquet/parquet_reader.cpp b/extension/parquet/parquet_reader.cpp index ecf73d1ec542..51c028e569ab 100644 --- a/extension/parquet/parquet_reader.cpp +++ b/extension/parquet/parquet_reader.cpp @@ -49,7 +49,7 @@ using duckdb_parquet::format::Type; static unique_ptr CreateThriftFileProtocol(Allocator &allocator, FileHandle &file_handle, bool prefetch_mode) { - auto transport = make_refcounted(allocator, file_handle, prefetch_mode); + auto transport = std::make_shared(allocator, file_handle, prefetch_mode); return make_uniq>(std::move(transport)); } From fb62ea24dcdf7908f1623aff4d744e7518defbdc Mon Sep 17 00:00:00 2001 From: Tishj Date: Sun, 7 Apr 2024 13:27:36 +0200 Subject: [PATCH 226/603] initialize 'shared_from_this' --- src/include/duckdb/common/shared_ptr.ipp | 35 ++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/src/include/duckdb/common/shared_ptr.ipp b/src/include/duckdb/common/shared_ptr.ipp index 5db96b1b8cb6..433f58adf478 100644 --- a/src/include/duckdb/common/shared_ptr.ipp +++ b/src/include/duckdb/common/shared_ptr.ipp @@ -4,15 +4,20 @@ namespace duckdb { template class weak_ptr; +template +class enable_shared_from_this; + template class shared_ptr { private: template friend class weak_ptr; - std::shared_ptr internal; template friend class shared_ptr; +private: + std::shared_ptr internal; + public: // Constructors shared_ptr() : internal() { @@ -21,10 +26,12 @@ public: } // Implicit conversion template explicit shared_ptr(U *ptr) : internal(ptr) { + __enable_weak_this(internal.get(), internal.get()); } // Constructor with custom deleter template shared_ptr(T *ptr, Deleter deleter) : internal(ptr, deleter) { + __enable_weak_this(internal.get(), internal.get()); } template shared_ptr(const shared_ptr &__r, T *__p) noexcept : internal(__r.internal, __p) { @@ -33,9 +40,12 @@ public: shared_ptr(const shared_ptr &other) : internal(other.internal) { } - shared_ptr(std::shared_ptr other) : internal(std::move(other)) { + shared_ptr(std::shared_ptr other) : internal(other) { + // FIXME: should we __enable_weak_this here? + // *our* enable_shared_from_this hasn't initialized yet, so I think so? + __enable_weak_this(internal.get(), internal.get()); } - shared_ptr(shared_ptr &&other) : internal(std::move(other.internal)) { + shared_ptr(shared_ptr &&other) : internal(other.internal) { } template @@ -45,6 +55,7 @@ public: #if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR) template ::value, int> = 0> shared_ptr(std::auto_ptr &&__r) : internal(__r.release()) { + __enable_weak_this(internal.get(), internal.get()); } #endif @@ -53,6 +64,7 @@ public: std::is_convertible::pointer, T *>::value, int>::type = 0> shared_ptr(unique_ptr &&other) : internal(other.release()) { + __enable_weak_this(internal.get(), internal.get()); } // Destructor @@ -159,6 +171,23 @@ public: template friend shared_ptr shared_ptr_cast(shared_ptr src); + +private: + // This overload is used when the class inherits from 'enable_shared_from_this' + template *>::value, + int>::type = 0> + void __enable_weak_this(const enable_shared_from_this *__e, _OrigPtr *__ptr) noexcept { + typedef typename std::remove_cv::type NonConstU; + if (__e && __e->__weak_this_.expired()) { + // __weak_this__ is the mutable variable returned by 'shared_from_this' + // it is initialized here + __e->__weak_this_ = shared_ptr(*this, const_cast(static_cast(__ptr))); + } + } + + void __enable_weak_this(...) noexcept { + } }; } // namespace duckdb From 39d6fad82f92cfc8c59000646a967abd13a59aef Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Sun, 7 Apr 2024 13:41:42 +0200 Subject: [PATCH 227/603] Correctly handle the difference between explicit indexes (order by 1) and indexes selected through aliasing of expressions --- src/include/duckdb/planner/binder.hpp | 4 +- .../expression_binder/select_bind_state.hpp | 5 +- .../binder/query_node/bind_select_node.cpp | 50 ++++++++++++------- .../binder/query_node/bind_setop_node.cpp | 2 +- .../expression_binder/order_binder.cpp | 9 +++- .../expression_binder/select_bind_state.cpp | 14 +++++- test/planner/projection_binding.test | 3 +- 7 files changed, 60 insertions(+), 27 deletions(-) diff --git a/src/include/duckdb/planner/binder.hpp b/src/include/duckdb/planner/binder.hpp index 4d013873cc95..d55e40e3d249 100644 --- a/src/include/duckdb/planner/binder.hpp +++ b/src/include/duckdb/planner/binder.hpp @@ -325,8 +325,8 @@ class Binder : public std::enable_shared_from_this { BoundStatement BindCopyFrom(CopyStatement &stmt); void PrepareModifiers(OrderBinder &order_binder, QueryNode &statement, BoundQueryNode &result); - void BindModifiers(BoundQueryNode &result, idx_t table_index, const vector &sql_types, - const SelectBindState &bind_state); + void BindModifiers(BoundQueryNode &result, idx_t table_index, const vector &names, + const vector &sql_types, const SelectBindState &bind_state); unique_ptr BindLimit(OrderBinder &order_binder, LimitModifier &limit_mod); unique_ptr BindLimitPercent(OrderBinder &order_binder, LimitPercentModifier &limit_mod); diff --git a/src/include/duckdb/planner/expression_binder/select_bind_state.hpp b/src/include/duckdb/planner/expression_binder/select_bind_state.hpp index 49ce92b143f2..d11d94731eb0 100644 --- a/src/include/duckdb/planner/expression_binder/select_bind_state.hpp +++ b/src/include/duckdb/planner/expression_binder/select_bind_state.hpp @@ -32,10 +32,11 @@ struct SelectBindState { void SetExpressionIsVolatile(idx_t index); void SetExpressionHasSubquery(idx_t index); - bool AliasHasSubquery(idx_t index); + bool AliasHasSubquery(idx_t index) const; void AddExpandedColumn(idx_t expand_count); void AddRegularColumn(); + idx_t GetFinalIndex(idx_t index) const; private: //! The set of referenced aliases @@ -44,6 +45,8 @@ struct SelectBindState { unordered_set volatile_expressions; //! The set of expressions that contains a subquery unordered_set subquery_expressions; + //! Column indices after expansion of Expanded expressions (e.g. UNNEST(STRUCT) clauses) + vector expanded_column_indices; }; } // namespace duckdb diff --git a/src/planner/binder/query_node/bind_select_node.cpp b/src/planner/binder/query_node/bind_select_node.cpp index d3becb272e22..9d1a78e747a3 100644 --- a/src/planner/binder/query_node/bind_select_node.cpp +++ b/src/planner/binder/query_node/bind_select_node.cpp @@ -231,16 +231,21 @@ void Binder::PrepareModifiers(OrderBinder &order_binder, QueryNode &statement, B } } -unique_ptr CreateOrderExpression(unique_ptr expr, const vector &sql_types, +unique_ptr CreateOrderExpression(unique_ptr expr, const vector &names, const vector &sql_types, idx_t table_index, idx_t index) { if (index >= sql_types.size()) { throw BinderException(*expr, "ORDER term out of range - should be between 1 and %lld", sql_types.size()); } - return make_uniq(std::move(expr->alias), sql_types[index], + auto result = make_uniq(std::move(expr->alias), sql_types[index], ColumnBinding(table_index, index)); + if (result->alias.empty() && index < names.size()) { + result->alias = names[index]; + } + return std::move(result); } unique_ptr FinalizeBindOrderExpression(unique_ptr expr, idx_t table_index, + const vector &names, const vector &sql_types, const SelectBindState &bind_state) { auto &constant = expr->Cast(); @@ -248,7 +253,7 @@ unique_ptr FinalizeBindOrderExpression(unique_ptr expr, case LogicalTypeId::UBIGINT: { // index auto index = UBigIntValue::Get(constant.value); - return CreateOrderExpression(std::move(expr), sql_types, table_index, index); + return CreateOrderExpression(std::move(expr), names, sql_types, table_index, bind_state.GetFinalIndex(index)); } case LogicalTypeId::VARCHAR: { // ORDER BY ALL @@ -257,16 +262,21 @@ unique_ptr FinalizeBindOrderExpression(unique_ptr expr, case LogicalTypeId::STRUCT: { // collation auto &struct_values = StructValue::GetChildren(constant.value); - if (struct_values.size() != 2) { - throw InternalException("Expected two children: index and collation"); + if (struct_values.size() > 2) { + throw InternalException("Expected one or two children: index and optional collation"); } auto index = UBigIntValue::Get(struct_values[0]); - auto collation = StringValue::Get(struct_values[1]); - if (sql_types[index].id() != LogicalTypeId::VARCHAR) { - throw BinderException(*expr, "COLLATE can only be applied to varchar columns"); + string collation; + if (struct_values.size() == 2) { + collation = StringValue::Get(struct_values[1]); + } + auto result = CreateOrderExpression(std::move(expr), names, sql_types, table_index, index); + if (!collation.empty()) { + if (sql_types[index].id() != LogicalTypeId::VARCHAR) { + throw BinderException(*result, "COLLATE can only be applied to varchar columns"); + } + result->return_type = LogicalType::VARCHAR_COLLATION(std::move(collation)); } - auto result = CreateOrderExpression(std::move(expr), sql_types, table_index, index); - result->return_type = LogicalType::VARCHAR_COLLATION(std::move(collation)); return result; } default: @@ -275,13 +285,14 @@ unique_ptr FinalizeBindOrderExpression(unique_ptr expr, } static void AssignReturnType(unique_ptr &expr, idx_t table_index, + const vector &names, const vector &sql_types, const SelectBindState &bind_state) { if (!expr) { return; } if (expr->type == ExpressionType::VALUE_CONSTANT) { - expr = FinalizeBindOrderExpression(std::move(expr), table_index, sql_types, bind_state); + expr = FinalizeBindOrderExpression(std::move(expr), table_index, names, sql_types, bind_state); } if (expr->type != ExpressionType::BOUND_COLUMN_REF) { return; @@ -290,8 +301,8 @@ static void AssignReturnType(unique_ptr &expr, idx_t table_index, bound_colref.return_type = sql_types[bound_colref.binding.column_index]; } -void Binder::BindModifiers(BoundQueryNode &result, idx_t table_index, const vector &sql_types, - const SelectBindState &bind_state) { +void Binder::BindModifiers(BoundQueryNode &result, idx_t table_index, const vector &names, + const vector &sql_types, const SelectBindState &bind_state) { for (auto &bound_mod : result.modifiers) { switch (bound_mod->type) { case ResultModifierType::DISTINCT_MODIFIER: { @@ -299,7 +310,7 @@ void Binder::BindModifiers(BoundQueryNode &result, idx_t table_index, const vect D_ASSERT(!distinct.target_distincts.empty()); // set types of distinct targets for (auto &expr : distinct.target_distincts) { - expr = FinalizeBindOrderExpression(std::move(expr), table_index, sql_types, bind_state); + expr = FinalizeBindOrderExpression(std::move(expr), table_index, names, sql_types, bind_state); if (!expr) { throw InternalException("DISTINCT ON ORDER BY ALL not supported"); } @@ -311,8 +322,8 @@ void Binder::BindModifiers(BoundQueryNode &result, idx_t table_index, const vect } case ResultModifierType::LIMIT_MODIFIER: { auto &limit = bound_mod->Cast(); - AssignReturnType(limit.limit_val.GetExpression(), table_index, sql_types, bind_state); - AssignReturnType(limit.offset_val.GetExpression(), table_index, sql_types, bind_state); + AssignReturnType(limit.limit_val.GetExpression(), table_index, names, sql_types, bind_state); + AssignReturnType(limit.offset_val.GetExpression(), table_index, names, sql_types, bind_state); break; } case ResultModifierType::ORDER_MODIFIER: { @@ -320,7 +331,7 @@ void Binder::BindModifiers(BoundQueryNode &result, idx_t table_index, const vect bool order_by_all = false; for (auto &order_node : order.orders) { auto &expr = order_node.expression; - expr = FinalizeBindOrderExpression(std::move(expr), table_index, sql_types, bind_state); + expr = FinalizeBindOrderExpression(std::move(expr), table_index, names, sql_types, bind_state); if (!expr) { order_by_all = true; } @@ -332,6 +343,9 @@ void Binder::BindModifiers(BoundQueryNode &result, idx_t table_index, const vect order.orders.clear(); for (idx_t i = 0; i < sql_types.size(); i++) { auto expr = make_uniq(sql_types[i], ColumnBinding(table_index, i)); + if (i < names.size()) { + expr->alias = names[i]; + } order.orders.emplace_back(order_type, null_order, std::move(expr)); } } @@ -649,7 +663,7 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ } // now that the SELECT list is bound, we set the types of DISTINCT/ORDER BY expressions - BindModifiers(*result, result->projection_index, internal_sql_types, bind_state); + BindModifiers(*result, result->projection_index, result->names, internal_sql_types, bind_state); return std::move(result); } diff --git a/src/planner/binder/query_node/bind_setop_node.cpp b/src/planner/binder/query_node/bind_setop_node.cpp index 60a86d7bb09c..b34133e065e7 100644 --- a/src/planner/binder/query_node/bind_setop_node.cpp +++ b/src/planner/binder/query_node/bind_setop_node.cpp @@ -248,7 +248,7 @@ unique_ptr Binder::BindNode(SetOperationNode &statement) { } // finally bind the types of the ORDER/DISTINCT clause expressions - BindModifiers(*result, result->setop_index, result->types, bind_state); + BindModifiers(*result, result->setop_index, result->names, result->types, bind_state); return std::move(result); } diff --git a/src/planner/expression_binder/order_binder.cpp b/src/planner/expression_binder/order_binder.cpp index a80ceb07f213..67c883f824b3 100644 --- a/src/planner/expression_binder/order_binder.cpp +++ b/src/planner/expression_binder/order_binder.cpp @@ -56,8 +56,13 @@ unique_ptr OrderBinder::BindConstant(ParsedExpression &expr, const V return nullptr; } // INTEGER constant: we use the integer as an index into the select list (e.g. ORDER BY 1) - auto index = (idx_t)val.GetValue(); - return CreateProjectionReference(expr, index - 1); + auto index = idx_t(val.GetValue() - 1); + child_list_t values; + values.push_back(make_pair("index", Value::UBIGINT(index))); + auto result = make_uniq(Value::STRUCT(std::move(values))); + result->alias = std::move(expr.alias); + result->query_location = expr.query_location; + return std::move(result); } unique_ptr OrderBinder::Bind(unique_ptr expr) { diff --git a/src/planner/expression_binder/select_bind_state.cpp b/src/planner/expression_binder/select_bind_state.cpp index a987258548e4..87256565d248 100644 --- a/src/planner/expression_binder/select_bind_state.cpp +++ b/src/planner/expression_binder/select_bind_state.cpp @@ -26,14 +26,26 @@ void SelectBindState::SetExpressionHasSubquery(idx_t index) { subquery_expressions.insert(index); } -bool SelectBindState::AliasHasSubquery(idx_t index) { +bool SelectBindState::AliasHasSubquery(idx_t index) const { return subquery_expressions.find(index) != subquery_expressions.end(); } void SelectBindState::AddExpandedColumn(idx_t expand_count) { + if (expanded_column_indices.empty()) { + expanded_column_indices.push_back(0); + } + expanded_column_indices.push_back(expanded_column_indices.back() + expand_count); } void SelectBindState::AddRegularColumn() { + AddExpandedColumn(1); +} + +idx_t SelectBindState::GetFinalIndex(idx_t index) const { + if (expanded_column_indices.empty()) { + return index; + } + return expanded_column_indices[index]; } } // namespace duckdb diff --git a/test/planner/projection_binding.test b/test/planner/projection_binding.test index a1466df12b4b..fbd7f97fafdc 100644 --- a/test/planner/projection_binding.test +++ b/test/planner/projection_binding.test @@ -3,7 +3,7 @@ # group: [planner] statement ok -CREATE TABLE a (i INTEGER, j INTEGER) +CREATE TABLE a (i INTEGER, j INTEGER); statement ok PRAGMA explain_output = PHYSICAL_ONLY; @@ -12,7 +12,6 @@ PRAGMA explain_output = PHYSICAL_ONLY; query I nosort orderbinder EXPLAIN SELECT i FROM a ORDER BY 1 ---- - query I nosort orderbinder EXPLAIN SELECT i FROM a ORDER BY i ---- From fbdf2f86b5464871f8431221bff3efa47fd55e06 Mon Sep 17 00:00:00 2001 From: Max Gabrielsson Date: Mon, 8 Apr 2024 10:16:15 +0200 Subject: [PATCH 228/603] initializes unknown indexes on catalog lookup --- .../catalog_entry/duck_schema_entry.cpp | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/catalog/catalog_entry/duck_schema_entry.cpp b/src/catalog/catalog_entry/duck_schema_entry.cpp index 105270d49b41..cbd78d374847 100644 --- a/src/catalog/catalog_entry/duck_schema_entry.cpp +++ b/src/catalog/catalog_entry/duck_schema_entry.cpp @@ -36,8 +36,8 @@ namespace duckdb { -void FindForeignKeyInformation(CatalogEntry &entry, AlterForeignKeyType alter_fk_type, - vector> &fk_arrays) { +static void FindForeignKeyInformation(CatalogEntry &entry, AlterForeignKeyType alter_fk_type, + vector> &fk_arrays) { if (entry.type != CatalogType::TABLE_ENTRY) { return; } @@ -63,6 +63,19 @@ void FindForeignKeyInformation(CatalogEntry &entry, AlterForeignKeyType alter_fk } } +static void LazyLoadIndexes(ClientContext &context, CatalogEntry &entry) { + if (entry.type == CatalogType::TABLE_ENTRY) { + auto &table_entry = entry.Cast(); + table_entry.GetStorage().info->InitializeIndexes(context); + } else if (entry.type == CatalogType::INDEX_ENTRY) { + auto &index_entry = entry.Cast(); + auto &table_entry = Catalog::GetEntry(context, CatalogType::TABLE_ENTRY, index_entry.catalog.GetName(), + index_entry.GetSchemaName(), index_entry.GetTableName()) + .Cast(); + table_entry.GetStorage().info->InitializeIndexes(context); + } +} + DuckSchemaEntry::DuckSchemaEntry(Catalog &catalog, CreateSchemaInfo &info) : SchemaCatalogEntry(catalog, info), tables(catalog, make_uniq(catalog, *this)), indexes(catalog), table_functions(catalog), copy_functions(catalog), pragma_functions(catalog), @@ -287,6 +300,9 @@ void DuckSchemaEntry::DropEntry(ClientContext &context, DropInfo &info) { CatalogTypeToString(existing_entry->type), CatalogTypeToString(info.type)); } + // if this is a index or table with indexes, initialize any unknown index instances + LazyLoadIndexes(context, *existing_entry); + // if there is a foreign key constraint, get that information vector> fk_arrays; FindForeignKeyInformation(*existing_entry, AlterForeignKeyType::AFT_DELETE, fk_arrays); From 57d12c7803fe2b81de8bc804f4bc38f16d54d2bd Mon Sep 17 00:00:00 2001 From: Max Gabrielsson Date: Mon, 8 Apr 2024 10:35:10 +0200 Subject: [PATCH 229/603] remove other initialization --- src/function/table/table_scan.cpp | 3 --- src/storage/data_table.cpp | 4 ---- src/storage/local_storage.cpp | 4 ---- 3 files changed, 11 deletions(-) diff --git a/src/function/table/table_scan.cpp b/src/function/table/table_scan.cpp index 097642361756..26a9843775e7 100644 --- a/src/function/table/table_scan.cpp +++ b/src/function/table/table_scan.cpp @@ -306,9 +306,6 @@ void TableScanPushdownComplexFilter(ClientContext &context, LogicalGet &get, Fun return; } - // Lazily initialize any unknown indexes that might have been loaded by an extension - storage.info->InitializeIndexes(context); - // behold storage.info->indexes.Scan([&](Index &index) { // first rewrite the index expression so the ColumnBindings align with the column bindings of the current table diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index ac55613c9295..0a3e0ed034f3 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -89,8 +89,6 @@ DataTable::DataTable(ClientContext &context, DataTable &parent, idx_t removed_co column_definitions.emplace_back(column_def.Copy()); } - // try to initialize unknown indexes - info->InitializeIndexes(context); // first check if there are any indexes that exist that point to the removed column info->indexes.Scan([&](Index &index) { for (auto &column_id : index.column_ids) { @@ -155,8 +153,6 @@ DataTable::DataTable(ClientContext &context, DataTable &parent, idx_t changed_id for (auto &column_def : parent.column_definitions) { column_definitions.emplace_back(column_def.Copy()); } - // try to initialize unknown indexes - info->InitializeIndexes(context); // first check if there are any indexes that exist that point to the changed column info->indexes.Scan([&](Index &index) { diff --git a/src/storage/local_storage.cpp b/src/storage/local_storage.cpp index 2c7fb0fe1d79..791847b102eb 100644 --- a/src/storage/local_storage.cpp +++ b/src/storage/local_storage.cpp @@ -470,9 +470,6 @@ void LocalStorage::Flush(DataTable &table, LocalTableStorage &storage) { storage.AppendToIndexes(transaction, append_state, append_count, true); } - // try to initialize any unknown indexes - table.info->InitializeIndexes(context); - // possibly vacuum any excess index data table.info->indexes.Scan([&](Index &index) { index.Vacuum(); @@ -575,7 +572,6 @@ TableIndexList &LocalStorage::GetIndexes(DataTable &table) { if (!storage) { throw InternalException("LocalStorage::GetIndexes - local storage not found"); } - table.info->InitializeIndexes(context); return storage->indexes; } From 3b561cb8a6821a4bc0797b4b9253ec248437e90e Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 8 Apr 2024 10:59:22 +0200 Subject: [PATCH 230/603] Remove extra catalog function --- src/catalog/catalog.cpp | 11 ----------- .../operator/persistent/csv_rejects_table.cpp | 4 ++-- src/include/duckdb/catalog/catalog.hpp | 2 -- 3 files changed, 2 insertions(+), 15 deletions(-) diff --git a/src/catalog/catalog.cpp b/src/catalog/catalog.cpp index 7e865661f862..775af179588d 100644 --- a/src/catalog/catalog.cpp +++ b/src/catalog/catalog.cpp @@ -773,17 +773,6 @@ CatalogEntry &Catalog::GetEntry(ClientContext &context, const string &schema, co throw CatalogException("CatalogElement \"%s.%s\" does not exist!", schema, name); } -bool Catalog::EntryExists(ClientContext &context, const string &schema, const string &name) { - vector entry_types {CatalogType::TABLE_ENTRY, CatalogType::SEQUENCE_ENTRY}; - for (auto entry_type : entry_types) { - auto result = GetEntry(context, entry_type, schema, name, OnEntryNotFound::RETURN_NULL); - if (result) { - return true; - } - } - return false; -} - optional_ptr Catalog::GetEntry(ClientContext &context, CatalogType type, const string &schema_name, const string &name, OnEntryNotFound if_not_found, QueryErrorContext error_context) { diff --git a/src/execution/operator/persistent/csv_rejects_table.cpp b/src/execution/operator/persistent/csv_rejects_table.cpp index 31f63d0279b8..4d3248bf3944 100644 --- a/src/execution/operator/persistent/csv_rejects_table.cpp +++ b/src/execution/operator/persistent/csv_rejects_table.cpp @@ -25,8 +25,8 @@ shared_ptr CSVRejectsTable::GetOrCreate(ClientContext &context, "CSV_REJECTS_TABLE_CACHE_ENTRY_" + StringUtil::Upper(rejects_scan) + "_" + StringUtil::Upper(rejects_error); auto &cache = ObjectCache::GetObjectCache(context); auto &catalog = Catalog::GetCatalog(context, TEMP_CATALOG); - bool rejects_scan_exist = catalog.EntryExists(context, DEFAULT_SCHEMA, rejects_scan); - bool rejects_error_exist = catalog.EntryExists(context, DEFAULT_SCHEMA, rejects_error); + auto rejects_scan_exist = catalog.GetEntry(context, CatalogType::TABLE_ENTRY, DEFAULT_SCHEMA, rejects_scan, OnEntryNotFound::RETURN_NULL) != nullptr; + auto rejects_error_exist = catalog.GetEntry(context, CatalogType::TABLE_ENTRY, DEFAULT_SCHEMA, rejects_error, OnEntryNotFound::RETURN_NULL) != nullptr; if ((rejects_scan_exist || rejects_error_exist) && !cache.Get(key)) { std::ostringstream error; if (rejects_scan_exist) { diff --git a/src/include/duckdb/catalog/catalog.hpp b/src/include/duckdb/catalog/catalog.hpp index 654fe4a3569c..244a5f362037 100644 --- a/src/include/duckdb/catalog/catalog.hpp +++ b/src/include/duckdb/catalog/catalog.hpp @@ -232,8 +232,6 @@ class Catalog { //! Gets the "schema.name" entry without a specified type, if entry does not exist an exception is thrown DUCKDB_API CatalogEntry &GetEntry(ClientContext &context, const string &schema, const string &name); - //! Returns true if the "schema.name" entry without a specified type exists - DUCKDB_API bool EntryExists(ClientContext &context, const string &schema, const string &name); //! Fetches a logical type from the catalog DUCKDB_API LogicalType GetType(ClientContext &context, const string &schema, const string &names, From df3d7ff3d0c8cba66f14bc67f15dd383b081d811 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 8 Apr 2024 11:24:29 +0200 Subject: [PATCH 231/603] more pr requests, adding tests and assertions --- .../operator/persistent/csv_rejects_table.cpp | 12 +++++++-- src/include/duckdb/catalog/catalog.hpp | 1 - .../csv/rejects/csv_rejects_two_tables.test | 26 ++++++++++++++++--- third_party/utf8proc/utf8proc_wrapper.cpp | 4 ++- 4 files changed, 36 insertions(+), 7 deletions(-) diff --git a/src/execution/operator/persistent/csv_rejects_table.cpp b/src/execution/operator/persistent/csv_rejects_table.cpp index 4d3248bf3944..1cd98d85fe81 100644 --- a/src/execution/operator/persistent/csv_rejects_table.cpp +++ b/src/execution/operator/persistent/csv_rejects_table.cpp @@ -21,12 +21,19 @@ TableCatalogEntry &CSVRejectsTable::GetScansTable(ClientContext &context) { shared_ptr CSVRejectsTable::GetOrCreate(ClientContext &context, const string &rejects_scan, const string &rejects_error) { + // Check that these names can't be the same + if (rejects_scan == rejects_error) { + throw BinderException("The names of the rejects scan and rejects error tables can't be the same. Use different " + "names for these tables."); + } auto key = "CSV_REJECTS_TABLE_CACHE_ENTRY_" + StringUtil::Upper(rejects_scan) + "_" + StringUtil::Upper(rejects_error); auto &cache = ObjectCache::GetObjectCache(context); auto &catalog = Catalog::GetCatalog(context, TEMP_CATALOG); - auto rejects_scan_exist = catalog.GetEntry(context, CatalogType::TABLE_ENTRY, DEFAULT_SCHEMA, rejects_scan, OnEntryNotFound::RETURN_NULL) != nullptr; - auto rejects_error_exist = catalog.GetEntry(context, CatalogType::TABLE_ENTRY, DEFAULT_SCHEMA, rejects_error, OnEntryNotFound::RETURN_NULL) != nullptr; + auto rejects_scan_exist = catalog.GetEntry(context, CatalogType::TABLE_ENTRY, DEFAULT_SCHEMA, rejects_scan, + OnEntryNotFound::RETURN_NULL) != nullptr; + auto rejects_error_exist = catalog.GetEntry(context, CatalogType::TABLE_ENTRY, DEFAULT_SCHEMA, rejects_error, + OnEntryNotFound::RETURN_NULL) != nullptr; if ((rejects_scan_exist || rejects_error_exist) && !cache.Get(key)) { std::ostringstream error; if (rejects_scan_exist) { @@ -38,6 +45,7 @@ shared_ptr CSVRejectsTable::GetOrCreate(ClientContext &context, error << "Either drop the used name(s), or give other name options in the CSV Reader function.\n"; throw BinderException(error.str()); } + return cache.GetOrCreate(key, rejects_scan, rejects_error); } diff --git a/src/include/duckdb/catalog/catalog.hpp b/src/include/duckdb/catalog/catalog.hpp index 244a5f362037..871738a975de 100644 --- a/src/include/duckdb/catalog/catalog.hpp +++ b/src/include/duckdb/catalog/catalog.hpp @@ -232,7 +232,6 @@ class Catalog { //! Gets the "schema.name" entry without a specified type, if entry does not exist an exception is thrown DUCKDB_API CatalogEntry &GetEntry(ClientContext &context, const string &schema, const string &name); - //! Fetches a logical type from the catalog DUCKDB_API LogicalType GetType(ClientContext &context, const string &schema, const string &names, OnEntryNotFound if_not_found); diff --git a/test/sql/copy/csv/rejects/csv_rejects_two_tables.test b/test/sql/copy/csv/rejects/csv_rejects_two_tables.test index f856d929fa2e..902031695f9f 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_two_tables.test +++ b/test/sql/copy/csv/rejects/csv_rejects_two_tables.test @@ -156,8 +156,7 @@ SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), M rejects_scan = 't' ); ---- -Reject Scan Table name "t" is already in use. Reject Error Table name "t" is already in use. Either drop the used name(s), or give other name options in the CSV Reader function. - +The names of the rejects scan and rejects error tables can't be the same. Use different names for these tables. # Test giving the name of the tables with store_rejects and/or ignore_errors set to false throws statement error @@ -230,4 +229,25 @@ SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), M store_rejects = false ); ---- -REJECTS_SCAN option is only supported when store_rejects is not manually set to false \ No newline at end of file +REJECTS_SCAN option is only supported when store_rejects is not manually set to false + +# Add a test where both tables have the same name (This should fail, because they both have the same name) +statement error +SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto( + 'test/sql/copy/csv/data/error/mismatch/big_bad*.csv', + sample_size=1, + rejects_scan = 'same_name_because_why_not', + rejects_table = 'same_name_because_why_not', + store_rejects = true + ); +---- +The names of the rejects scan and rejects error tables can't be the same. Use different names for these tables. + +# This hopefully doesn't fail because the names don't get registered if they fail. +statement ok +SELECT typeof(first(column0)), typeof(first(column1)), COUNT(*), SUM(column0), MAX(len(column1)) FROM read_csv_auto( + 'test/sql/copy/csv/data/error/mismatch/big_bad*.csv', + sample_size=1, + rejects_scan = 'same_name_because_why_not', + rejects_table = 'same_name_because_why_not_2', + store_rejects = true); diff --git a/third_party/utf8proc/utf8proc_wrapper.cpp b/third_party/utf8proc/utf8proc_wrapper.cpp index 02f6c0efc5de..9ff3615e8a89 100644 --- a/third_party/utf8proc/utf8proc_wrapper.cpp +++ b/third_party/utf8proc/utf8proc_wrapper.cpp @@ -1,6 +1,6 @@ #include "utf8proc_wrapper.hpp" #include "utf8proc.hpp" - +#include "duckdb/common/assert.hpp" using namespace std; namespace duckdb { @@ -103,6 +103,7 @@ UnicodeType Utf8Proc::Analyze(const char *s, size_t len, UnicodeInvalidReason *i } void Utf8Proc::MakeValid(char *s, size_t len, char special_flag){ + D_ASSERT(special_flag <=127); UnicodeType type = UnicodeType::ASCII; for (size_t i = 0; i < len; i++) { int c = (int) s[i]; @@ -133,6 +134,7 @@ void Utf8Proc::MakeValid(char *s, size_t len, char special_flag){ type = UnicodeType::ASCII; } } + D_ASSERT(Utf8Proc::IsValid(s,len)); } char* Utf8Proc::Normalize(const char *s, size_t len) { From 7e078316e8aba6465ec2e7382ea9ce428a28aa9e Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 8 Apr 2024 12:42:09 +0200 Subject: [PATCH 232/603] Also print error_line when throwing --- src/execution/operator/csv_scanner/util/csv_error.cpp | 4 ++++ test/sql/copy/csv/csv_error_message.test | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/execution/operator/csv_scanner/util/csv_error.cpp b/src/execution/operator/csv_scanner/util/csv_error.cpp index da9f5d5e2435..d22738f21415 100644 --- a/src/execution/operator/csv_scanner/util/csv_error.cpp +++ b/src/execution/operator/csv_scanner/util/csv_error.cpp @@ -19,12 +19,16 @@ void CSVErrorHandler::ThrowError(CSVError csv_error) { std::ostringstream error; if (PrintLineNumber(csv_error)) { error << "CSV Error on Line: " << GetLine(csv_error.error_info) << '\n'; + if (!csv_error.csv_row.empty()) { + error << "Original Line: " << csv_error.csv_row << '\n'; + } } if (csv_error.full_error_message.empty()) { error << csv_error.error_message; } else { error << csv_error.full_error_message; } + switch (csv_error.type) { case CSVErrorType::CAST_ERROR: throw ConversionException(error.str()); diff --git a/test/sql/copy/csv/csv_error_message.test b/test/sql/copy/csv/csv_error_message.test index b41d049fe65d..b51e98dd87e9 100644 --- a/test/sql/copy/csv/csv_error_message.test +++ b/test/sql/copy/csv/csv_error_message.test @@ -26,3 +26,7 @@ SELECT * FROM read_csv('__TEST_DIR__/int_parse_error.csv', columns={'i': 'INT'}, ---- Line: 104 +statement error +SELECT * FROM read_csv('__TEST_DIR__/int_parse_error.csv', columns={'i': 'INT'}, header=False, auto_detect=false) +---- +Original Line: hello \ No newline at end of file From 2f7be72713dfaf306a62abf3d7d9374149bf2bb1 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 8 Apr 2024 12:53:21 +0200 Subject: [PATCH 233/603] Current File Index --- .../csv_scanner/table_function/global_csv_state.cpp | 2 +- src/execution/operator/persistent/csv_rejects_table.cpp | 8 ++++++++ .../execution/operator/persistent/csv_rejects_table.hpp | 9 ++++++++- src/include/duckdb/transaction/transaction_context.hpp | 2 -- src/transaction/transaction_context.cpp | 6 ------ 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index b156f0af8b95..1759450ef41a 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -246,7 +246,7 @@ void CSVGlobalState::FillRejectsTable() { InternalAppender scans_appender(context, scans_table); idx_t scan_idx = context.transaction.GetActiveQuery(); for (auto &file : file_scans) { - idx_t file_idx = context.transaction.GetIncrementalIndex(); + idx_t file_idx = rejects->GetCurrentFileIndex(scan_idx); auto file_name = file->file_path; auto &errors = file->error_handler->errors; // We first insert the file into the file scans table diff --git a/src/execution/operator/persistent/csv_rejects_table.cpp b/src/execution/operator/persistent/csv_rejects_table.cpp index 1cd98d85fe81..11e8c1b0edf0 100644 --- a/src/execution/operator/persistent/csv_rejects_table.cpp +++ b/src/execution/operator/persistent/csv_rejects_table.cpp @@ -19,6 +19,14 @@ TableCatalogEntry &CSVRejectsTable::GetScansTable(ClientContext &context) { return table_entry; } +idx_t CSVRejectsTable::GetCurrentFileIndex(idx_t query_id) { + if (current_query_id != query_id) { + current_query_id = query_id; + current_file_idx = 0; + } + return current_file_idx++; +} + shared_ptr CSVRejectsTable::GetOrCreate(ClientContext &context, const string &rejects_scan, const string &rejects_error) { // Check that these names can't be the same diff --git a/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp b/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp index 88dd86377dc7..d1075a4fbffe 100644 --- a/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +++ b/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp @@ -32,7 +32,8 @@ class CSVRejectsTable : public ObjectCacheEntry { TableCatalogEntry &GetErrorsTable(ClientContext &context); TableCatalogEntry &GetScansTable(ClientContext &context); -public: + idx_t GetCurrentFileIndex(idx_t query_id); + static string ObjectType() { return "csv_rejects_table_cache"; } @@ -40,6 +41,12 @@ class CSVRejectsTable : public ObjectCacheEntry { string GetObjectType() override { return ObjectType(); } + +private: + //! Current File Index being used in the query + idx_t current_file_idx = 0; + //! Current Query ID being executed + idx_t current_query_id = 0; }; } // namespace duckdb diff --git a/src/include/duckdb/transaction/transaction_context.hpp b/src/include/duckdb/transaction/transaction_context.hpp index b265c0131498..b0a50103bb46 100644 --- a/src/include/duckdb/transaction/transaction_context.hpp +++ b/src/include/duckdb/transaction/transaction_context.hpp @@ -48,7 +48,6 @@ class TransactionContext { } idx_t GetActiveQuery(); - idx_t GetIncrementalIndex(); void ResetActiveQuery(); void SetActiveQuery(transaction_t query_number); @@ -57,7 +56,6 @@ class TransactionContext { bool auto_commit; unique_ptr current_transaction; - idx_t incremental_index = 0; TransactionContext(const TransactionContext &) = delete; }; diff --git a/src/transaction/transaction_context.cpp b/src/transaction/transaction_context.cpp index 82d1fa43094f..7185a263894b 100644 --- a/src/transaction/transaction_context.cpp +++ b/src/transaction/transaction_context.cpp @@ -89,19 +89,13 @@ idx_t TransactionContext::GetActiveQuery() { return current_transaction->GetActiveQuery(); } -idx_t TransactionContext::GetIncrementalIndex() { - return incremental_index++; -} - void TransactionContext::ResetActiveQuery() { - incremental_index = 0; if (current_transaction) { SetActiveQuery(MAXIMUM_QUERY_ID); } } void TransactionContext::SetActiveQuery(transaction_t query_number) { - incremental_index = 0; if (!current_transaction) { throw InternalException("SetActiveQuery called without active transaction"); } From 88454e4ce7511ed656da38406096002a36c360f1 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 8 Apr 2024 14:57:15 +0200 Subject: [PATCH 234/603] Maybe this is fine for the serializer? --- .../duckdb/storage/serialization/nodes.json | 49 +++++++++------- src/storage/serialization/serialize_nodes.cpp | 58 ++++++++++--------- 2 files changed, 59 insertions(+), 48 deletions(-) diff --git a/src/include/duckdb/storage/serialization/nodes.json b/src/include/duckdb/storage/serialization/nodes.json index acf730100b86..034db172d9db 100644 --- a/src/include/duckdb/storage/serialization/nodes.json +++ b/src/include/duckdb/storage/serialization/nodes.json @@ -537,7 +537,8 @@ "members": [ {"id": 100, "name": "ignore_errors", - "type": "CSVOption" + "type": "CSVOption", + "default": "false" }, {"id": 101, "name": "buffer_sample_size", @@ -604,64 +605,72 @@ "type": "vector" }, {"id": 117, - "name": "store_rejects", - "type": "CSVOption" + "name": "rejects_table_name", + "type": "CSVOption", + "default": "{\"reject_errors\"}" }, {"id": 118, "name": "rejects_limit", "type": "idx_t" }, {"id": 119, + "name": "rejects_recovery_columns", + "type": "vector", + "deleted": true + }, + {"id": 120, + "name": "rejects_recovery_column_ids", + "type": "vector", + "deleted": true + }, + {"id": 121, "name": "dialect_options.state_machine_options.delimiter", "type": "CSVOption" }, - {"id": 120, + {"id": 122, "name": "dialect_options.state_machine_options.quote", "type": "CSVOption" }, - {"id": 121, + {"id": 123, "name": "dialect_options.state_machine_options.escape", "type": "CSVOption" }, - {"id": 122, + {"id": 124, "name": "dialect_options.header", "type": "CSVOption" }, - {"id": 123, + {"id": 125, "name": "dialect_options.num_cols", "type": "idx_t" }, - {"id": 124, + {"id": 126, "name": "dialect_options.state_machine_options.new_line", "type": "CSVOption" }, - {"id": 125, + {"id": 127, "name": "dialect_options.skip_rows", "type": "CSVOption" }, - {"id": 126, + {"id": 128, "name": "dialect_options.date_format", "type": "map>" }, - {"id": 127, + {"id": 129, "name": "sniffer_user_mismatch_error", "type": "string" }, - {"id": 128, + {"id": 130, "name": "parallel", "type": "bool" }, - {"id": 129, - "name": "rejects_table_name", - "type": "CSVOption" - }, - {"id": 130, - "name": "rejects_scan_name", - "type": "CSVOption" - }, {"id": 131, "name": "was_type_manually_set", "type": "vector" + }, + {"id": 132, + "name": "rejects_scan_name", + "type": "CSVOption", + "default": "{\"reject_scans\"}" } ], "pointer_type": "none" diff --git a/src/storage/serialization/serialize_nodes.cpp b/src/storage/serialization/serialize_nodes.cpp index 2359d127ef24..a0d459a938f0 100644 --- a/src/storage/serialization/serialize_nodes.cpp +++ b/src/storage/serialization/serialize_nodes.cpp @@ -118,7 +118,7 @@ CSVOption CSVOption::Deserialize(Deserializer &deserializer) { } void CSVReaderOptions::Serialize(Serializer &serializer) const { - serializer.WriteProperty>(100, "ignore_errors", ignore_errors); + serializer.WritePropertyWithDefault>(100, "ignore_errors", ignore_errors, false); serializer.WritePropertyWithDefault(101, "buffer_sample_size", buffer_sample_size); serializer.WritePropertyWithDefault(102, "null_str", null_str); serializer.WriteProperty(103, "compression", compression); @@ -135,26 +135,27 @@ void CSVReaderOptions::Serialize(Serializer &serializer) const { serializer.WritePropertyWithDefault(114, "buffer_size", buffer_size); serializer.WriteProperty(115, "file_options", file_options); serializer.WritePropertyWithDefault>(116, "force_quote", force_quote); - serializer.WriteProperty>(117, "store_rejects", store_rejects); + serializer.WritePropertyWithDefault>(117, "rejects_table_name", rejects_table_name, {"reject_errors"}); serializer.WritePropertyWithDefault(118, "rejects_limit", rejects_limit); - serializer.WriteProperty>(119, "dialect_options.state_machine_options.delimiter", dialect_options.state_machine_options.delimiter); - serializer.WriteProperty>(120, "dialect_options.state_machine_options.quote", dialect_options.state_machine_options.quote); - serializer.WriteProperty>(121, "dialect_options.state_machine_options.escape", dialect_options.state_machine_options.escape); - serializer.WriteProperty>(122, "dialect_options.header", dialect_options.header); - serializer.WritePropertyWithDefault(123, "dialect_options.num_cols", dialect_options.num_cols); - serializer.WriteProperty>(124, "dialect_options.state_machine_options.new_line", dialect_options.state_machine_options.new_line); - serializer.WriteProperty>(125, "dialect_options.skip_rows", dialect_options.skip_rows); - serializer.WriteProperty>>(126, "dialect_options.date_format", dialect_options.date_format); - serializer.WritePropertyWithDefault(127, "sniffer_user_mismatch_error", sniffer_user_mismatch_error); - serializer.WritePropertyWithDefault(128, "parallel", parallel); - serializer.WriteProperty>(129, "rejects_table_name", rejects_table_name); - serializer.WriteProperty>(130, "rejects_scan_name", rejects_scan_name); + /* [Deleted] (vector) "rejects_recovery_columns" */ + /* [Deleted] (vector) "rejects_recovery_column_ids" */ + serializer.WriteProperty>(121, "dialect_options.state_machine_options.delimiter", dialect_options.state_machine_options.delimiter); + serializer.WriteProperty>(122, "dialect_options.state_machine_options.quote", dialect_options.state_machine_options.quote); + serializer.WriteProperty>(123, "dialect_options.state_machine_options.escape", dialect_options.state_machine_options.escape); + serializer.WriteProperty>(124, "dialect_options.header", dialect_options.header); + serializer.WritePropertyWithDefault(125, "dialect_options.num_cols", dialect_options.num_cols); + serializer.WriteProperty>(126, "dialect_options.state_machine_options.new_line", dialect_options.state_machine_options.new_line); + serializer.WriteProperty>(127, "dialect_options.skip_rows", dialect_options.skip_rows); + serializer.WriteProperty>>(128, "dialect_options.date_format", dialect_options.date_format); + serializer.WritePropertyWithDefault(129, "sniffer_user_mismatch_error", sniffer_user_mismatch_error); + serializer.WritePropertyWithDefault(130, "parallel", parallel); serializer.WritePropertyWithDefault>(131, "was_type_manually_set", was_type_manually_set); + serializer.WritePropertyWithDefault>(132, "rejects_scan_name", rejects_scan_name, {"reject_scans"}); } CSVReaderOptions CSVReaderOptions::Deserialize(Deserializer &deserializer) { CSVReaderOptions result; - deserializer.ReadProperty>(100, "ignore_errors", result.ignore_errors); + deserializer.ReadPropertyWithDefault>(100, "ignore_errors", result.ignore_errors, false); deserializer.ReadPropertyWithDefault(101, "buffer_sample_size", result.buffer_sample_size); deserializer.ReadPropertyWithDefault(102, "null_str", result.null_str); deserializer.ReadProperty(103, "compression", result.compression); @@ -171,21 +172,22 @@ CSVReaderOptions CSVReaderOptions::Deserialize(Deserializer &deserializer) { deserializer.ReadPropertyWithDefault(114, "buffer_size", result.buffer_size); deserializer.ReadProperty(115, "file_options", result.file_options); deserializer.ReadPropertyWithDefault>(116, "force_quote", result.force_quote); - deserializer.ReadProperty>(117, "store_rejects", result.store_rejects); + deserializer.ReadPropertyWithDefault>(117, "rejects_table_name", result.rejects_table_name, {"reject_errors"}); deserializer.ReadPropertyWithDefault(118, "rejects_limit", result.rejects_limit); - deserializer.ReadProperty>(119, "dialect_options.state_machine_options.delimiter", result.dialect_options.state_machine_options.delimiter); - deserializer.ReadProperty>(120, "dialect_options.state_machine_options.quote", result.dialect_options.state_machine_options.quote); - deserializer.ReadProperty>(121, "dialect_options.state_machine_options.escape", result.dialect_options.state_machine_options.escape); - deserializer.ReadProperty>(122, "dialect_options.header", result.dialect_options.header); - deserializer.ReadPropertyWithDefault(123, "dialect_options.num_cols", result.dialect_options.num_cols); - deserializer.ReadProperty>(124, "dialect_options.state_machine_options.new_line", result.dialect_options.state_machine_options.new_line); - deserializer.ReadProperty>(125, "dialect_options.skip_rows", result.dialect_options.skip_rows); - deserializer.ReadProperty>>(126, "dialect_options.date_format", result.dialect_options.date_format); - deserializer.ReadPropertyWithDefault(127, "sniffer_user_mismatch_error", result.sniffer_user_mismatch_error); - deserializer.ReadPropertyWithDefault(128, "parallel", result.parallel); - deserializer.ReadProperty>(129, "rejects_table_name", result.rejects_table_name); - deserializer.ReadProperty>(130, "rejects_scan_name", result.rejects_scan_name); + deserializer.ReadDeletedProperty>(119, "rejects_recovery_columns"); + deserializer.ReadDeletedProperty>(120, "rejects_recovery_column_ids"); + deserializer.ReadProperty>(121, "dialect_options.state_machine_options.delimiter", result.dialect_options.state_machine_options.delimiter); + deserializer.ReadProperty>(122, "dialect_options.state_machine_options.quote", result.dialect_options.state_machine_options.quote); + deserializer.ReadProperty>(123, "dialect_options.state_machine_options.escape", result.dialect_options.state_machine_options.escape); + deserializer.ReadProperty>(124, "dialect_options.header", result.dialect_options.header); + deserializer.ReadPropertyWithDefault(125, "dialect_options.num_cols", result.dialect_options.num_cols); + deserializer.ReadProperty>(126, "dialect_options.state_machine_options.new_line", result.dialect_options.state_machine_options.new_line); + deserializer.ReadProperty>(127, "dialect_options.skip_rows", result.dialect_options.skip_rows); + deserializer.ReadProperty>>(128, "dialect_options.date_format", result.dialect_options.date_format); + deserializer.ReadPropertyWithDefault(129, "sniffer_user_mismatch_error", result.sniffer_user_mismatch_error); + deserializer.ReadPropertyWithDefault(130, "parallel", result.parallel); deserializer.ReadPropertyWithDefault>(131, "was_type_manually_set", result.was_type_manually_set); + deserializer.ReadPropertyWithDefault>(132, "rejects_scan_name", result.rejects_scan_name, {"reject_scans"}); return result; } From 2fb9648b2f33226a4d1254fa4a6da326e9f76030 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 8 Apr 2024 15:03:23 +0200 Subject: [PATCH 235/603] Go away utility --- .../duckdb/execution/operator/persistent/csv_rejects_table.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp b/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp index d1075a4fbffe..7b88e0de80cb 100644 --- a/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +++ b/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp @@ -1,7 +1,5 @@ #pragma once -#include - #include "duckdb/storage/object_cache.hpp" #include "duckdb/common/mutex.hpp" #include "duckdb/common/typedefs.hpp" From feea4ce0fc4f7ed0ad3fd1d97dff415e1fe19650 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 8 Apr 2024 15:08:29 +0200 Subject: [PATCH 236/603] Update this test for smaller SF --- .../test_multiple_big_compressed_csvs.test_slow | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/test/sql/copy/csv/test_multiple_big_compressed_csvs.test_slow b/test/sql/copy/csv/test_multiple_big_compressed_csvs.test_slow index e53ef286a495..4c8116b7d3cd 100644 --- a/test/sql/copy/csv/test_multiple_big_compressed_csvs.test_slow +++ b/test/sql/copy/csv/test_multiple_big_compressed_csvs.test_slow @@ -5,19 +5,22 @@ require tpch statement ok -CALL dbgen(sf=10); +CALL dbgen(sf=1); + +statement ok +SET temp_directory='' statement ok copy lineitem to '__TEST_DIR__/lineitem.csv.gz'; statement ok -SET temp_directory='' +DROP TABLE lineitem; statement ok -CREATE TABLE lineitem_2(l_orderkey INTEGER NOT NULL, l_partkey INTEGER NOT NULL, l_suppkey INTEGER NOT NULL, l_linenumber INTEGER NOT NULL, l_quantity DECIMAL(15,2) NOT NULL, l_extendedprice DECIMAL(15,2) NOT NULL, l_discount DECIMAL(15,2) NOT NULL, l_tax DECIMAL(15,2) NOT NULL, l_returnflag VARCHAR NOT NULL, l_linestatus VARCHAR NOT NULL, l_shipdate DATE NOT NULL, l_commitdate DATE NOT NULL, l_receiptdate DATE NOT NULL, l_shipinstruct VARCHAR NOT NULL, l_shipmode VARCHAR NOT NULL, l_comment VARCHAR NOT NULL); +CREATE TABLE lineitem(l_orderkey INTEGER NOT NULL, l_partkey INTEGER NOT NULL, l_suppkey INTEGER NOT NULL, l_linenumber INTEGER NOT NULL, l_quantity DECIMAL(15,2) NOT NULL, l_extendedprice DECIMAL(15,2) NOT NULL, l_discount DECIMAL(15,2) NOT NULL, l_tax DECIMAL(15,2) NOT NULL, l_returnflag VARCHAR NOT NULL, l_linestatus VARCHAR NOT NULL, l_shipdate DATE NOT NULL, l_commitdate DATE NOT NULL, l_receiptdate DATE NOT NULL, l_shipinstruct VARCHAR NOT NULL, l_shipmode VARCHAR NOT NULL, l_comment VARCHAR NOT NULL); statement ok -INSERT INTO lineitem_2 FROM read_csv([ +INSERT INTO lineitem FROM read_csv([ '__TEST_DIR__/lineitem.csv.gz', '__TEST_DIR__/lineitem.csv.gz', '__TEST_DIR__/lineitem.csv.gz', @@ -45,6 +48,6 @@ INSERT INTO lineitem_2 FROM read_csv([ ]); query I -select count(*) from lineitem_2 +select count(*) from lineitem ---- -1439665248 \ No newline at end of file +144029160 \ No newline at end of file From 18e58509356ad86b059139f2e1c07253293110d4 Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 8 Apr 2024 15:14:52 +0200 Subject: [PATCH 237/603] fix some compilation issues --- src/include/duckdb/common/shared_ptr.ipp | 25 ++++++++++++++++------- src/include/duckdb/common/types/value.hpp | 1 + 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/include/duckdb/common/shared_ptr.ipp b/src/include/duckdb/common/shared_ptr.ipp index 433f58adf478..3cec8de3b587 100644 --- a/src/include/duckdb/common/shared_ptr.ipp +++ b/src/include/duckdb/common/shared_ptr.ipp @@ -24,7 +24,7 @@ public: } shared_ptr(std::nullptr_t) : internal(nullptr) { } // Implicit conversion - template + template ::value, int>::type = 0> explicit shared_ptr(U *ptr) : internal(ptr) { __enable_weak_this(internal.get(), internal.get()); } @@ -36,6 +36,16 @@ public: template shared_ptr(const shared_ptr &__r, T *__p) noexcept : internal(__r.internal, __p) { } + template + shared_ptr(shared_ptr &&__r, T *__p) noexcept : internal(__r.internal, __p) { + } + + template ::value, int>::type = 0> + shared_ptr(const shared_ptr &__r) noexcept : internal(__r.internal) { + } + template ::value, int>::type = 0> + shared_ptr(shared_ptr &&__r) noexcept : internal(__r.internal) { + } shared_ptr(const shared_ptr &other) : internal(other.internal) { } @@ -72,7 +82,13 @@ public: // Assignment operators shared_ptr &operator=(const shared_ptr &other) { - internal = other.internal; + shared_ptr(other).swap(*this); + return *this; + } + + template ::value, int>::type = 0> + shared_ptr &operator=(const shared_ptr &other) { + shared_ptr(other).swap(*this); return *this; } @@ -117,11 +133,6 @@ public: return internal.operator bool(); } - template - operator shared_ptr() const noexcept { - return shared_ptr(internal); - } - // Element access std::__add_lvalue_reference_t operator*() const { return *internal; diff --git a/src/include/duckdb/common/types/value.hpp b/src/include/duckdb/common/types/value.hpp index 0328ee6f1c73..2e27fdc4409f 100644 --- a/src/include/duckdb/common/types/value.hpp +++ b/src/include/duckdb/common/types/value.hpp @@ -17,6 +17,7 @@ #include "duckdb/common/types/date.hpp" #include "duckdb/common/types/datetime.hpp" #include "duckdb/common/types/interval.hpp" +#include "duckdb/common/shared_ptr.hpp" namespace duckdb { From 3870eda4a057b20a35e8addf19c0a2f70a407b3d Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 8 Apr 2024 16:51:46 +0200 Subject: [PATCH 238/603] some special code for options --- .../duckdb/common/serializer/deserializer.hpp | 12 ++++++++++++ .../duckdb/common/serializer/serializer.hpp | 16 ++++++++++++++++ .../duckdb/storage/serialization/nodes.json | 6 +++--- src/storage/serialization/serialize_nodes.cpp | 8 ++++---- 4 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/include/duckdb/common/serializer/deserializer.hpp b/src/include/duckdb/common/serializer/deserializer.hpp index 000104c43c60..e4096878a810 100644 --- a/src/include/duckdb/common/serializer/deserializer.hpp +++ b/src/include/duckdb/common/serializer/deserializer.hpp @@ -114,6 +114,18 @@ class Deserializer { OnOptionalPropertyEnd(true); } + template + inline void ReadPropertyWithDefault(const field_id_t field_id, const char *tag, CSVOption &ret, + T &&default_value) { + if (!OnOptionalPropertyBegin(field_id, tag)) { + ret = std::forward(default_value); + OnOptionalPropertyEnd(false); + return; + } + ret = Read(); + OnOptionalPropertyEnd(true); + } + // Special case: // Read into an existing data_ptr_t inline void ReadProperty(const field_id_t field_id, const char *tag, data_ptr_t ret, idx_t count) { diff --git a/src/include/duckdb/common/serializer/serializer.hpp b/src/include/duckdb/common/serializer/serializer.hpp index 10b926d048e1..f791b4a892df 100644 --- a/src/include/duckdb/common/serializer/serializer.hpp +++ b/src/include/duckdb/common/serializer/serializer.hpp @@ -17,6 +17,7 @@ #include "duckdb/common/unordered_set.hpp" #include "duckdb/common/optional_idx.hpp" #include "duckdb/common/value_operations/value_operations.hpp" +#include "duckdb/execution/operator/csv_scanner/csv_option.hpp" namespace duckdb { @@ -86,6 +87,21 @@ class Serializer { OnOptionalPropertyEnd(true); } + // Specialization for Value (default Value comparison throws when comparing nulls) + template + void WritePropertyWithDefault(const field_id_t field_id, const char *tag, const CSVOption &value, + const T &&default_value) { + // If current value is default, don't write it + if (!serialize_default_values && (value == default_value)) { + OnOptionalPropertyBegin(field_id, tag, false); + OnOptionalPropertyEnd(false); + return; + } + OnOptionalPropertyBegin(field_id, tag, true); + WriteValue(value.GetValue()); + OnOptionalPropertyEnd(true); + } + // Special case: data_ptr_T void WriteProperty(const field_id_t field_id, const char *tag, const_data_ptr_t ptr, idx_t count) { OnPropertyBegin(field_id, tag); diff --git a/src/include/duckdb/storage/serialization/nodes.json b/src/include/duckdb/storage/serialization/nodes.json index 034db172d9db..547be7329ad8 100644 --- a/src/include/duckdb/storage/serialization/nodes.json +++ b/src/include/duckdb/storage/serialization/nodes.json @@ -537,7 +537,7 @@ "members": [ {"id": 100, "name": "ignore_errors", - "type": "CSVOption", + "type": "bool", "default": "false" }, {"id": 101, @@ -606,8 +606,8 @@ }, {"id": 117, "name": "rejects_table_name", - "type": "CSVOption", - "default": "{\"reject_errors\"}" + "type": "string", + "default": "\"reject_errors\"" }, {"id": 118, "name": "rejects_limit", diff --git a/src/storage/serialization/serialize_nodes.cpp b/src/storage/serialization/serialize_nodes.cpp index a0d459a938f0..39529e35e786 100644 --- a/src/storage/serialization/serialize_nodes.cpp +++ b/src/storage/serialization/serialize_nodes.cpp @@ -118,7 +118,7 @@ CSVOption CSVOption::Deserialize(Deserializer &deserializer) { } void CSVReaderOptions::Serialize(Serializer &serializer) const { - serializer.WritePropertyWithDefault>(100, "ignore_errors", ignore_errors, false); + serializer.WritePropertyWithDefault(100, "ignore_errors", ignore_errors, false); serializer.WritePropertyWithDefault(101, "buffer_sample_size", buffer_sample_size); serializer.WritePropertyWithDefault(102, "null_str", null_str); serializer.WriteProperty(103, "compression", compression); @@ -135,7 +135,7 @@ void CSVReaderOptions::Serialize(Serializer &serializer) const { serializer.WritePropertyWithDefault(114, "buffer_size", buffer_size); serializer.WriteProperty(115, "file_options", file_options); serializer.WritePropertyWithDefault>(116, "force_quote", force_quote); - serializer.WritePropertyWithDefault>(117, "rejects_table_name", rejects_table_name, {"reject_errors"}); + serializer.WritePropertyWithDefault(117, "rejects_table_name", rejects_table_name, "reject_errors"); serializer.WritePropertyWithDefault(118, "rejects_limit", rejects_limit); /* [Deleted] (vector) "rejects_recovery_columns" */ /* [Deleted] (vector) "rejects_recovery_column_ids" */ @@ -155,7 +155,7 @@ void CSVReaderOptions::Serialize(Serializer &serializer) const { CSVReaderOptions CSVReaderOptions::Deserialize(Deserializer &deserializer) { CSVReaderOptions result; - deserializer.ReadPropertyWithDefault>(100, "ignore_errors", result.ignore_errors, false); + deserializer.ReadPropertyWithDefault(100, "ignore_errors", result.ignore_errors, false); deserializer.ReadPropertyWithDefault(101, "buffer_sample_size", result.buffer_sample_size); deserializer.ReadPropertyWithDefault(102, "null_str", result.null_str); deserializer.ReadProperty(103, "compression", result.compression); @@ -172,7 +172,7 @@ CSVReaderOptions CSVReaderOptions::Deserialize(Deserializer &deserializer) { deserializer.ReadPropertyWithDefault(114, "buffer_size", result.buffer_size); deserializer.ReadProperty(115, "file_options", result.file_options); deserializer.ReadPropertyWithDefault>(116, "force_quote", result.force_quote); - deserializer.ReadPropertyWithDefault>(117, "rejects_table_name", result.rejects_table_name, {"reject_errors"}); + deserializer.ReadPropertyWithDefault(117, "rejects_table_name", result.rejects_table_name, "reject_errors"); deserializer.ReadPropertyWithDefault(118, "rejects_limit", result.rejects_limit); deserializer.ReadDeletedProperty>(119, "rejects_recovery_columns"); deserializer.ReadDeletedProperty>(120, "rejects_recovery_column_ids"); From 747965783289e9cbcf32ae33169e523024f14ee0 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Mon, 8 Apr 2024 20:15:44 +0200 Subject: [PATCH 239/603] Initial fixes for setop --- src/planner/binder/query_node/bind_select_node.cpp | 2 +- src/planner/binder/query_node/bind_setop_node.cpp | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/planner/binder/query_node/bind_select_node.cpp b/src/planner/binder/query_node/bind_select_node.cpp index 9d1a78e747a3..c34d05f66a85 100644 --- a/src/planner/binder/query_node/bind_select_node.cpp +++ b/src/planner/binder/query_node/bind_select_node.cpp @@ -552,7 +552,7 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ if (expr->type == ExpressionType::BOUND_EXPANDED) { if (!is_original_column) { - throw InternalException("Only original columns can have expanded expressions"); + throw BinderException("UNNEST of struct cannot be used in ORDER BY/DISTINCT ON clause"); } if (statement.aggregate_handling == AggregateHandling::FORCE_AGGREGATES) { throw BinderException("UNNEST of struct cannot be combined with GROUP BY ALL"); diff --git a/src/planner/binder/query_node/bind_setop_node.cpp b/src/planner/binder/query_node/bind_setop_node.cpp index b34133e065e7..f5f8e26e3a23 100644 --- a/src/planner/binder/query_node/bind_setop_node.cpp +++ b/src/planner/binder/query_node/bind_setop_node.cpp @@ -43,10 +43,9 @@ static void GatherAliases(BoundQueryNode &node, SelectBindState &bind_state, con // query node D_ASSERT(node.type == QueryNodeType::SELECT_NODE); auto &select = node.Cast(); - // fill the alias lists + // fill the alias lists with the names for (idx_t i = 0; i < select.names.size(); i++) { auto &name = select.names[i]; - auto &expr = select.bind_state.original_expressions[i]; // first check if the alias is already in there auto entry = bind_state.alias_map.find(name); @@ -56,6 +55,11 @@ static void GatherAliases(BoundQueryNode &node, SelectBindState &bind_state, con // the alias is not in there yet, just assign it bind_state.alias_map[name] = index; } + } + // check if the expression matches one of the expressions in the original expression liset + for (idx_t i = 0; i < select.bind_state.original_expressions.size(); i++) { + auto &expr = select.bind_state.original_expressions[i]; + idx_t index = reorder_idx[i]; // now check if the node is already in the set of expressions auto expr_entry = bind_state.projection_map.find(*expr); if (expr_entry != bind_state.projection_map.end()) { From ddef3dffd56768c7b0971bd079d378f8a656da9f Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Tue, 9 Apr 2024 10:33:42 +0200 Subject: [PATCH 240/603] Add tests --- test/sql/types/struct/unnest_struct_mix.test | 89 ++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 test/sql/types/struct/unnest_struct_mix.test diff --git a/test/sql/types/struct/unnest_struct_mix.test b/test/sql/types/struct/unnest_struct_mix.test new file mode 100644 index 000000000000..81d7c9154fad --- /dev/null +++ b/test/sql/types/struct/unnest_struct_mix.test @@ -0,0 +1,89 @@ +# name: test/sql/types/struct/unnest_struct_mix.test +# description: Test UNNEST of struct mixed with other constructs +# group: [struct] + +statement ok +PRAGMA enable_verification + +# unnest of struct in set operation +statement ok +CREATE TABLE tbl_structs AS SELECT {'a': 1, 'b': 2, 'c': 3} AS s; + +statement ok +INSERT INTO tbl_structs VALUES ({'a': 2, 'b': 3, 'c': 1}); + +statement ok +INSERT INTO tbl_structs VALUES ({'a': 3, 'b': 1, 'c': 2}); + +query III +SELECT UNNEST(s) FROM tbl_structs UNION ALL SELECT s.a, s.b, s.c FROM tbl_structs ORDER BY s.a, s.b, s.c; +---- +1 2 3 +1 2 3 +2 3 1 +2 3 1 +3 1 2 +3 1 2 + +# order by unnest of struct +statement error +SELECT * FROM tbl_structs ORDER BY UNNEST(s); +---- +cannot be used + +# order by all +statement ok +CREATE OR REPLACE TABLE tbl_structs AS SELECT {'a': 1, 'b': 2, 'c': 3} AS s; + +statement ok +INSERT INTO tbl_structs VALUES ({'a': 1, 'b': 3, 'c': 1}); + +statement ok +INSERT INTO tbl_structs VALUES ({'a': 1, 'b': 1, 'c': 2}); + +query III +select unnest(s) from tbl_structs order by all; +---- +1 1 2 +1 2 3 +1 3 1 + +# order by index with collate +statement error +select unnest(s) from tbl_structs order by 2 collate nocase; +---- +COLLATE can only be applied to varchar columns + +statement ok +CREATE OR REPLACE TABLE tbl_structs AS SELECT {'a': 'hello'} s; + +statement ok +INSERT INTO tbl_structs VALUES ({'a': 'WORLD'}); + +query I +SELECT UNNEST(s) FROM tbl_structs ORDER BY 1 COLLATE NOCASE; +---- +hello +WORLD + +statement ok +CREATE OR REPLACE TABLE tbl_structs AS SELECT {'a': 'hello', 'b': 1} s; + +statement ok +INSERT INTO tbl_structs VALUES ({'a': 'WORLD', 'b': 2}); + +query II +SELECT UNNEST(s) FROM tbl_structs UNION ALL SELECT s.a, s.b FROM tbl_structs ORDER BY 1 COLLATE NOCASE; +---- +hello 1 +hello 1 +WORLD 2 +WORLD 2 + +query III +SELECT UNNEST(s), -s.b AS id FROM tbl_structs UNION ALL SELECT s.a, s.b, s.b FROM tbl_structs ORDER BY id; +---- +WORLD 2 -2 +hello 1 -1 +hello 1 1 +WORLD 2 2 From 610941b6df13e4aeaf2a0d6aaebbdb7ba63c3522 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Tue, 9 Apr 2024 10:36:47 +0200 Subject: [PATCH 241/603] Format fix --- src/include/duckdb/planner/binder.hpp | 2 +- .../binder/query_node/bind_select_node.cpp | 17 +++++++---------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/include/duckdb/planner/binder.hpp b/src/include/duckdb/planner/binder.hpp index d55e40e3d249..3c0d15ac3d5c 100644 --- a/src/include/duckdb/planner/binder.hpp +++ b/src/include/duckdb/planner/binder.hpp @@ -326,7 +326,7 @@ class Binder : public std::enable_shared_from_this { void PrepareModifiers(OrderBinder &order_binder, QueryNode &statement, BoundQueryNode &result); void BindModifiers(BoundQueryNode &result, idx_t table_index, const vector &names, - const vector &sql_types, const SelectBindState &bind_state); + const vector &sql_types, const SelectBindState &bind_state); unique_ptr BindLimit(OrderBinder &order_binder, LimitModifier &limit_mod); unique_ptr BindLimitPercent(OrderBinder &order_binder, LimitPercentModifier &limit_mod); diff --git a/src/planner/binder/query_node/bind_select_node.cpp b/src/planner/binder/query_node/bind_select_node.cpp index c34d05f66a85..6da8b9d4d9be 100644 --- a/src/planner/binder/query_node/bind_select_node.cpp +++ b/src/planner/binder/query_node/bind_select_node.cpp @@ -231,13 +231,13 @@ void Binder::PrepareModifiers(OrderBinder &order_binder, QueryNode &statement, B } } -unique_ptr CreateOrderExpression(unique_ptr expr, const vector &names, const vector &sql_types, - idx_t table_index, idx_t index) { +unique_ptr CreateOrderExpression(unique_ptr expr, const vector &names, + const vector &sql_types, idx_t table_index, idx_t index) { if (index >= sql_types.size()) { throw BinderException(*expr, "ORDER term out of range - should be between 1 and %lld", sql_types.size()); } auto result = make_uniq(std::move(expr->alias), sql_types[index], - ColumnBinding(table_index, index)); + ColumnBinding(table_index, index)); if (result->alias.empty() && index < names.size()) { result->alias = names[index]; } @@ -245,8 +245,7 @@ unique_ptr CreateOrderExpression(unique_ptr expr, const } unique_ptr FinalizeBindOrderExpression(unique_ptr expr, idx_t table_index, - const vector &names, - const vector &sql_types, + const vector &names, const vector &sql_types, const SelectBindState &bind_state) { auto &constant = expr->Cast(); switch (constant.value.type().id()) { @@ -284,10 +283,8 @@ unique_ptr FinalizeBindOrderExpression(unique_ptr expr, } } -static void AssignReturnType(unique_ptr &expr, idx_t table_index, - const vector &names, - const vector &sql_types, - const SelectBindState &bind_state) { +static void AssignReturnType(unique_ptr &expr, idx_t table_index, const vector &names, + const vector &sql_types, const SelectBindState &bind_state) { if (!expr) { return; } @@ -302,7 +299,7 @@ static void AssignReturnType(unique_ptr &expr, idx_t table_index, } void Binder::BindModifiers(BoundQueryNode &result, idx_t table_index, const vector &names, - const vector &sql_types, const SelectBindState &bind_state) { + const vector &sql_types, const SelectBindState &bind_state) { for (auto &bound_mod : result.modifiers) { switch (bound_mod->type) { case ResultModifierType::DISTINCT_MODIFIER: { From 6dbf4fc0b4ac5766049d20f32e1d34239754d3d0 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Tue, 9 Apr 2024 10:45:03 +0200 Subject: [PATCH 242/603] Fix test + generate files --- src/common/enum_util.cpp | 10 ++++++++++ test/planner/projection_binding.test | 7 ------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/common/enum_util.cpp b/src/common/enum_util.cpp index da3539efdbb3..c09995b9144a 100644 --- a/src/common/enum_util.cpp +++ b/src/common/enum_util.cpp @@ -2012,6 +2012,8 @@ const char* EnumUtil::ToChars(ExpressionClass value) { return "BOUND_LAMBDA_REF"; case ExpressionClass::BOUND_EXPRESSION: return "BOUND_EXPRESSION"; + case ExpressionClass::BOUND_EXPANDED: + return "BOUND_EXPANDED"; default: throw NotImplementedException(StringUtil::Format("Enum value: '%d' not implemented", value)); } @@ -2136,6 +2138,9 @@ ExpressionClass EnumUtil::FromString(const char *value) { if (StringUtil::Equals(value, "BOUND_EXPRESSION")) { return ExpressionClass::BOUND_EXPRESSION; } + if (StringUtil::Equals(value, "BOUND_EXPANDED")) { + return ExpressionClass::BOUND_EXPANDED; + } throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value)); } @@ -2278,6 +2283,8 @@ const char* EnumUtil::ToChars(ExpressionType value) { return "POSITIONAL_REFERENCE"; case ExpressionType::BOUND_LAMBDA_REF: return "BOUND_LAMBDA_REF"; + case ExpressionType::BOUND_EXPANDED: + return "BOUND_EXPANDED"; default: throw NotImplementedException(StringUtil::Format("Enum value: '%d' not implemented", value)); } @@ -2489,6 +2496,9 @@ ExpressionType EnumUtil::FromString(const char *value) { if (StringUtil::Equals(value, "BOUND_LAMBDA_REF")) { return ExpressionType::BOUND_LAMBDA_REF; } + if (StringUtil::Equals(value, "BOUND_EXPANDED")) { + return ExpressionType::BOUND_EXPANDED; + } throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value)); } diff --git a/test/planner/projection_binding.test b/test/planner/projection_binding.test index fbd7f97fafdc..7d7dfa1e00a4 100644 --- a/test/planner/projection_binding.test +++ b/test/planner/projection_binding.test @@ -10,9 +10,6 @@ PRAGMA explain_output = PHYSICAL_ONLY; # verify that ORDER BY binding is consistent in presence of different "table.column" vs "column" bindings query I nosort orderbinder -EXPLAIN SELECT i FROM a ORDER BY 1 ----- -query I nosort orderbinder EXPLAIN SELECT i FROM a ORDER BY i ---- @@ -25,10 +22,6 @@ EXPLAIN SELECT i FROM a ORDER BY a.i ---- # with alias -query I nosort orderbinderk -EXPLAIN SELECT i AS k FROM a ORDER BY 1 ----- - query I nosort orderbinderk EXPLAIN SELECT i AS k FROM a ORDER BY k ---- From deaf43a38cc6637932c9df86f3ea9a08f42ca5f8 Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 9 Apr 2024 11:04:33 +0200 Subject: [PATCH 243/603] move constructor should actually move --- src/include/duckdb/common/helper.hpp | 2 +- src/include/duckdb/common/shared_ptr.ipp | 74 ++++++++++++++---------- 2 files changed, 46 insertions(+), 30 deletions(-) diff --git a/src/include/duckdb/common/helper.hpp b/src/include/duckdb/common/helper.hpp index 81da4bd79513..2f12dca961f0 100644 --- a/src/include/duckdb/common/helper.hpp +++ b/src/include/duckdb/common/helper.hpp @@ -70,7 +70,7 @@ inline shared_ptr make_refcounted(ARGS&&... args) // NOLINT: mimic std style { - return shared_ptr(new DATA_TYPE(std::forward(args)...)); + return shared_ptr(std::make_shared(std::forward(args)...)); } template diff --git a/src/include/duckdb/common/shared_ptr.ipp b/src/include/duckdb/common/shared_ptr.ipp index 3cec8de3b587..f6976b91f3d9 100644 --- a/src/include/duckdb/common/shared_ptr.ipp +++ b/src/include/duckdb/common/shared_ptr.ipp @@ -12,9 +12,13 @@ class shared_ptr { private: template friend class weak_ptr; + template friend class shared_ptr; + template + friend shared_ptr shared_ptr_cast(shared_ptr src); + private: std::shared_ptr internal; @@ -23,45 +27,55 @@ public: shared_ptr() : internal() { } shared_ptr(std::nullptr_t) : internal(nullptr) { - } // Implicit conversion + } + + // From raw pointer of type U convertible to T template ::value, int>::type = 0> explicit shared_ptr(U *ptr) : internal(ptr) { __enable_weak_this(internal.get(), internal.get()); } - // Constructor with custom deleter + // From raw pointer of type T with custom Deleter template shared_ptr(T *ptr, Deleter deleter) : internal(ptr, deleter) { __enable_weak_this(internal.get(), internal.get()); } + // Aliasing constructor: shares ownership information with __r but contains __p instead + // When the created shared_ptr goes out of scope, it will call the Deleter of __r, will not delete __p template shared_ptr(const shared_ptr &__r, T *__p) noexcept : internal(__r.internal, __p) { } +#if _LIBCPP_STD_VER >= 20 template - shared_ptr(shared_ptr &&__r, T *__p) noexcept : internal(__r.internal, __p) { + shared_ptr(shared_ptr &&__r, T *__p) noexcept : internal(std::move(__r.internal), __p) { } +#endif + // Copy constructor, share ownership with __r template ::value, int>::type = 0> shared_ptr(const shared_ptr &__r) noexcept : internal(__r.internal) { } + shared_ptr(const shared_ptr &other) : internal(other.internal) { + } + // Move constructor, share ownership with __r template ::value, int>::type = 0> - shared_ptr(shared_ptr &&__r) noexcept : internal(__r.internal) { + shared_ptr(shared_ptr &&__r) noexcept : internal(std::move(__r.internal)) { } - - shared_ptr(const shared_ptr &other) : internal(other.internal) { + shared_ptr(shared_ptr &&other) : internal(std::move(other.internal)) { } + // Construct from std::shared_ptr shared_ptr(std::shared_ptr other) : internal(other) { // FIXME: should we __enable_weak_this here? // *our* enable_shared_from_this hasn't initialized yet, so I think so? __enable_weak_this(internal.get(), internal.get()); } - shared_ptr(shared_ptr &&other) : internal(other.internal) { - } + // Construct from weak_ptr template explicit shared_ptr(weak_ptr other) : internal(other.internal) { } + // Construct from auto_ptr #if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR) template ::value, int> = 0> shared_ptr(std::auto_ptr &&__r) : internal(__r.release()) { @@ -69,29 +83,43 @@ public: } #endif + // Construct from unique_ptr, takes over ownership of the unique_ptr template ::value && std::is_convertible::pointer, T *>::value, int>::type = 0> - shared_ptr(unique_ptr &&other) : internal(other.release()) { + shared_ptr(unique_ptr &&other) : internal(std::move(other)) { __enable_weak_this(internal.get(), internal.get()); } // Destructor ~shared_ptr() = default; - // Assignment operators - shared_ptr &operator=(const shared_ptr &other) { + // Assign from shared_ptr copy + shared_ptr &operator=(const shared_ptr &other) noexcept { + // Create a new shared_ptr using the copy constructor, then swap out the ownership to *this shared_ptr(other).swap(*this); return *this; } - template ::value, int>::type = 0> - shared_ptr &operator=(const shared_ptr &other) { + shared_ptr &operator=(const shared_ptr &other) { shared_ptr(other).swap(*this); return *this; } + // Assign from moved shared_ptr + shared_ptr &operator=(shared_ptr &&other) noexcept { + // Create a new shared_ptr using the move constructor, then swap out the ownership to *this + shared_ptr(std::move(other)).swap(*this); + return *this; + } + template ::value, int>::type = 0> + shared_ptr &operator=(shared_ptr &&other) { + shared_ptr(std::move(other)).swap(*this); + return *this; + } + + // Assign from moved unique_ptr template ::value && std::is_convertible::pointer, T *>::value, @@ -101,16 +129,13 @@ public: return *this; } - // Modifiers void reset() { internal.reset(); } - template void reset(U *ptr) { internal.reset(ptr); } - template void reset(U *ptr, Deleter deleter) { internal.reset(ptr, deleter); @@ -120,7 +145,6 @@ public: internal.swap(r.internal); } - // Observers T *get() const { return internal.get(); } @@ -133,7 +157,6 @@ public: return internal.operator bool(); } - // Element access std::__add_lvalue_reference_t operator*() const { return *internal; } @@ -147,15 +170,14 @@ public: bool operator==(const shared_ptr &other) const noexcept { return internal == other.internal; } - - bool operator==(std::nullptr_t) const noexcept { - return internal == nullptr; - } - template bool operator!=(const shared_ptr &other) const noexcept { return internal != other.internal; } + + bool operator==(std::nullptr_t) const noexcept { + return internal == nullptr; + } bool operator!=(std::nullptr_t) const noexcept { return internal != nullptr; } @@ -164,25 +186,19 @@ public: bool operator<(const shared_ptr &other) const noexcept { return internal < other.internal; } - template bool operator<=(const shared_ptr &other) const noexcept { return internal <= other.internal; } - template bool operator>(const shared_ptr &other) const noexcept { return internal > other.internal; } - template bool operator>=(const shared_ptr &other) const noexcept { return internal >= other.internal; } - template - friend shared_ptr shared_ptr_cast(shared_ptr src); - private: // This overload is used when the class inherits from 'enable_shared_from_this' template Date: Tue, 9 Apr 2024 11:10:11 +0200 Subject: [PATCH 244/603] make constructor from std::shared_ptr explicit --- extension/parquet/include/parquet_reader.hpp | 2 +- extension/parquet/include/parquet_writer.hpp | 2 +- .../duckdb/common/serializer/serialization_traits.hpp | 11 +++++++++++ src/include/duckdb/common/shared_ptr.ipp | 2 +- src/include/duckdb/common/weak_ptr.ipp | 2 +- 5 files changed, 15 insertions(+), 4 deletions(-) diff --git a/extension/parquet/include/parquet_reader.hpp b/extension/parquet/include/parquet_reader.hpp index 3393749373c4..6e65d46d8375 100644 --- a/extension/parquet/include/parquet_reader.hpp +++ b/extension/parquet/include/parquet_reader.hpp @@ -53,7 +53,7 @@ struct ParquetReaderScanState { idx_t group_offset; unique_ptr file_handle; unique_ptr root_reader; - unique_ptr thrift_file_proto; + std::unique_ptr thrift_file_proto; bool finished; SelectionVector sel; diff --git a/extension/parquet/include/parquet_writer.hpp b/extension/parquet/include/parquet_writer.hpp index 6b71b8196b26..44625d5db69d 100644 --- a/extension/parquet/include/parquet_writer.hpp +++ b/extension/parquet/include/parquet_writer.hpp @@ -108,7 +108,7 @@ class ParquetWriter { shared_ptr encryption_config; unique_ptr writer; - shared_ptr protocol; + std::shared_ptr protocol; duckdb_parquet::format::FileMetaData file_meta_data; std::mutex lock; diff --git a/src/include/duckdb/common/serializer/serialization_traits.hpp b/src/include/duckdb/common/serializer/serialization_traits.hpp index 616d90f2320f..5230a75e4f29 100644 --- a/src/include/duckdb/common/serializer/serialization_traits.hpp +++ b/src/include/duckdb/common/serializer/serialization_traits.hpp @@ -50,6 +50,13 @@ struct has_deserialize< T, typename std::enable_if(Deserializer &)>::value, T>::type> : std::true_type {}; +// Accept `static shared_ptr Deserialize(Deserializer& deserializer)` +template +struct has_deserialize< + T, + typename std::enable_if(Deserializer &)>::value, T>::type> + : std::true_type {}; + // Accept `static T Deserialize(Deserializer& deserializer)` template struct has_deserialize< @@ -105,6 +112,10 @@ template struct is_shared_ptr> : std::true_type { typedef T ELEMENT_TYPE; }; +template +struct is_shared_ptr> : std::true_type { + typedef T ELEMENT_TYPE; +}; template struct is_optional_ptr : std::false_type {}; diff --git a/src/include/duckdb/common/shared_ptr.ipp b/src/include/duckdb/common/shared_ptr.ipp index f6976b91f3d9..80ffc96507c7 100644 --- a/src/include/duckdb/common/shared_ptr.ipp +++ b/src/include/duckdb/common/shared_ptr.ipp @@ -64,7 +64,7 @@ public: } // Construct from std::shared_ptr - shared_ptr(std::shared_ptr other) : internal(other) { + explicit shared_ptr(std::shared_ptr other) : internal(other) { // FIXME: should we __enable_weak_this here? // *our* enable_shared_from_this hasn't initialized yet, so I think so? __enable_weak_this(internal.get(), internal.get()); diff --git a/src/include/duckdb/common/weak_ptr.ipp b/src/include/duckdb/common/weak_ptr.ipp index 2fd95699de2d..9ab76a0553eb 100644 --- a/src/include/duckdb/common/weak_ptr.ipp +++ b/src/include/duckdb/common/weak_ptr.ipp @@ -58,7 +58,7 @@ public: } shared_ptr lock() const { - return internal.lock(); + return shared_ptr(internal.lock()); } // Relational operators From 2cbe49e96c70f370830ba73f0ae0de3a9e790765 Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 9 Apr 2024 11:26:50 +0200 Subject: [PATCH 245/603] fix move in weak_ptr constructor, add SAFE template parameter --- .../duckdb/common/enable_shared_from_this.ipp | 24 ++++++++++--------- src/include/duckdb/common/shared_ptr.ipp | 23 +++++++++++------- src/include/duckdb/common/weak_ptr.ipp | 16 +++++++++---- 3 files changed, 38 insertions(+), 25 deletions(-) diff --git a/src/include/duckdb/common/enable_shared_from_this.ipp b/src/include/duckdb/common/enable_shared_from_this.ipp index 6472db9c2b12..d68d20033aa7 100644 --- a/src/include/duckdb/common/enable_shared_from_this.ipp +++ b/src/include/duckdb/common/enable_shared_from_this.ipp @@ -1,8 +1,13 @@ namespace duckdb { -template +template class enable_shared_from_this { - mutable weak_ptr<_Tp> __weak_this_; +public: + template + friend class shared_ptr; + +private: + mutable weak_ptr __weak_this_; protected: constexpr enable_shared_from_this() noexcept { @@ -16,25 +21,22 @@ protected: } public: - shared_ptr<_Tp> shared_from_this() { - return shared_ptr<_Tp>(__weak_this_); + shared_ptr shared_from_this() { + return shared_ptr(__weak_this_); } - shared_ptr<_Tp const> shared_from_this() const { - return shared_ptr(__weak_this_); + shared_ptr shared_from_this() const { + return shared_ptr(__weak_this_); } #if _LIBCPP_STD_VER >= 17 - weak_ptr<_Tp> weak_from_this() noexcept { + weak_ptr weak_from_this() noexcept { return __weak_this_; } - weak_ptr weak_from_this() const noexcept { + weak_ptr weak_from_this() const noexcept { return __weak_this_; } #endif // _LIBCPP_STD_VER >= 17 - - template - friend class shared_ptr; }; } // namespace duckdb diff --git a/src/include/duckdb/common/shared_ptr.ipp b/src/include/duckdb/common/shared_ptr.ipp index 80ffc96507c7..78d5365a03cc 100644 --- a/src/include/duckdb/common/shared_ptr.ipp +++ b/src/include/duckdb/common/shared_ptr.ipp @@ -1,26 +1,31 @@ namespace duckdb { -template +template class weak_ptr; template class enable_shared_from_this; -template +template class shared_ptr { +public: + using original = std::shared_ptr; + using element_type = typename original::element_type; + using weak_type = weak_ptr; + private: - template + template friend class weak_ptr; - template + template friend class shared_ptr; template friend shared_ptr shared_ptr_cast(shared_ptr src); private: - std::shared_ptr internal; + original internal; public: // Constructors @@ -84,11 +89,11 @@ public: #endif // Construct from unique_ptr, takes over ownership of the unique_ptr - template ::value && std::is_convertible::pointer, T *>::value, int>::type = 0> - shared_ptr(unique_ptr &&other) : internal(std::move(other)) { + shared_ptr(unique_ptr &&other) : internal(std::move(other)) { __enable_weak_this(internal.get(), internal.get()); } @@ -120,11 +125,11 @@ public: } // Assign from moved unique_ptr - template ::value && std::is_convertible::pointer, T *>::value, int>::type = 0> - shared_ptr &operator=(unique_ptr &&__r) { + shared_ptr &operator=(unique_ptr &&__r) { shared_ptr(std::move(__r)).swap(*this); return *this; } diff --git a/src/include/duckdb/common/weak_ptr.ipp b/src/include/duckdb/common/weak_ptr.ipp index 9ab76a0553eb..4b04adaa49dc 100644 --- a/src/include/duckdb/common/weak_ptr.ipp +++ b/src/include/duckdb/common/weak_ptr.ipp @@ -1,11 +1,17 @@ namespace duckdb { -template +template class weak_ptr { +public: + using original = std::weak_ptr; + using element_type = typename original::element_type; + private: - template + template friend class shared_ptr; - std::weak_ptr internal; + +private: + original internal; public: // Constructors @@ -22,11 +28,11 @@ public: weak_ptr(weak_ptr const &ptr, typename std::enable_if<__compatible_with::value, int>::type = 0) noexcept : internal(ptr.internal) { } - weak_ptr(weak_ptr &&ptr) noexcept : internal(ptr.internal) { + weak_ptr(weak_ptr &&ptr) noexcept : internal(std::move(ptr.internal)) { } template weak_ptr(weak_ptr &&ptr, typename std::enable_if<__compatible_with::value, int>::type = 0) noexcept - : internal(ptr.internal) { + : internal(std::move(ptr.internal)) { } // Destructor ~weak_ptr() = default; From dfdf55cd829b508f60f5f429bcf4113d081d597d Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 9 Apr 2024 11:49:19 +0200 Subject: [PATCH 246/603] add memory safety for shared_ptr and weak_ptr operations --- src/include/duckdb/common/shared_ptr.hpp | 15 ++++++++- src/include/duckdb/common/shared_ptr.ipp | 43 +++++++++++++++++++++--- src/include/duckdb/common/weak_ptr.ipp | 9 ++--- 3 files changed, 58 insertions(+), 9 deletions(-) diff --git a/src/include/duckdb/common/shared_ptr.hpp b/src/include/duckdb/common/shared_ptr.hpp index fe9d31ee40c9..82cb63157313 100644 --- a/src/include/duckdb/common/shared_ptr.hpp +++ b/src/include/duckdb/common/shared_ptr.hpp @@ -8,9 +8,12 @@ #pragma once +#include "duckdb/common/unique_ptr.hpp" +#include "duckdb/common/likely.hpp" +#include "duckdb/common/memory_safety.hpp" + #include #include -#include "duckdb/common/unique_ptr.hpp" #if _LIBCPP_STD_VER >= 17 template @@ -29,3 +32,13 @@ struct __compatible_with : std::is_convertible<_Yp *, _Tp *> {}; #include "duckdb/common/shared_ptr.ipp" #include "duckdb/common/weak_ptr.ipp" #include "duckdb/common/enable_shared_from_this.ipp" + +namespace duckdb { + +template +using unsafe_shared_ptr = shared_ptr; + +template +using unsafe_weak_ptr = weak_ptr; + +} // namespace duckdb diff --git a/src/include/duckdb/common/shared_ptr.ipp b/src/include/duckdb/common/shared_ptr.ipp index 78d5365a03cc..44496781a2ec 100644 --- a/src/include/duckdb/common/shared_ptr.ipp +++ b/src/include/duckdb/common/shared_ptr.ipp @@ -1,4 +1,3 @@ - namespace duckdb { template @@ -14,6 +13,17 @@ public: using element_type = typename original::element_type; using weak_type = weak_ptr; +private: + static inline void AssertNotNull(const bool null) { +#if defined(DUCKDB_DEBUG_NO_SAFETY) || defined(DUCKDB_CLANG_TIDY) + return; +#else + if (DUCKDB_UNLIKELY(null)) { + throw duckdb::InternalException("Attempted to dereference shared_ptr that is NULL!"); + } +#endif + } + private: template friend class weak_ptr; @@ -134,13 +144,26 @@ public: return *this; } - void reset() { +#ifdef DUCKDB_CLANG_TIDY + // This is necessary to tell clang-tidy that it reinitializes the variable after a move + [[clang::reinitializes]] +#endif + void + reset() { internal.reset(); } +#ifdef DUCKDB_CLANG_TIDY + // This is necessary to tell clang-tidy that it reinitializes the variable after a move + [[clang::reinitializes]] +#endif template void reset(U *ptr) { internal.reset(ptr); } +#ifdef DUCKDB_CLANG_TIDY + // This is necessary to tell clang-tidy that it reinitializes the variable after a move + [[clang::reinitializes]] +#endif template void reset(U *ptr, Deleter deleter) { internal.reset(ptr, deleter); @@ -163,11 +186,23 @@ public: } std::__add_lvalue_reference_t operator*() const { - return *internal; + if (MemorySafety::ENABLED) { + const auto ptr = internal.get(); + AssertNotNull(!ptr); + return *ptr; + } else { + return *internal; + } } T *operator->() const { - return internal.operator->(); + if (MemorySafety::ENABLED) { + const auto ptr = internal.get(); + AssertNotNull(!ptr); + return ptr; + } else { + return internal.operator->(); + } } // Relational operators diff --git a/src/include/duckdb/common/weak_ptr.ipp b/src/include/duckdb/common/weak_ptr.ipp index 4b04adaa49dc..f602aacb4830 100644 --- a/src/include/duckdb/common/weak_ptr.ipp +++ b/src/include/duckdb/common/weak_ptr.ipp @@ -19,7 +19,8 @@ public: } template - weak_ptr(shared_ptr const &ptr, typename std::enable_if<__compatible_with::value, int>::type = 0) noexcept + weak_ptr(shared_ptr const &ptr, + typename std::enable_if<__compatible_with::value, int>::type = 0) noexcept : internal(ptr.internal) { } weak_ptr(weak_ptr const &other) noexcept : internal(other.internal) { @@ -44,7 +45,7 @@ public: } template ::value, int> = 0> - weak_ptr &operator=(const shared_ptr &ptr) { + weak_ptr &operator=(const shared_ptr &ptr) { internal = ptr; return *this; } @@ -63,8 +64,8 @@ public: return internal.expired(); } - shared_ptr lock() const { - return shared_ptr(internal.lock()); + shared_ptr lock() const { + return shared_ptr(internal.lock()); } // Relational operators From 624f8adad66a85001100add98ee2b63ea48550d3 Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 9 Apr 2024 12:06:57 +0200 Subject: [PATCH 247/603] fix compilation error --- src/include/duckdb/common/shared_ptr.ipp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/duckdb/common/shared_ptr.ipp b/src/include/duckdb/common/shared_ptr.ipp index 44496781a2ec..e5c96f6af02e 100644 --- a/src/include/duckdb/common/shared_ptr.ipp +++ b/src/include/duckdb/common/shared_ptr.ipp @@ -92,7 +92,7 @@ public: // Construct from auto_ptr #if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR) - template ::value, int> = 0> + template ::value, int>::type = 0> shared_ptr(std::auto_ptr &&__r) : internal(__r.release()) { __enable_weak_this(internal.get(), internal.get()); } From df40291af5fc7eb9b4a4fbfd27863920c9461d2e Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 9 Apr 2024 12:23:48 +0200 Subject: [PATCH 248/603] use INVALID_INDEX-1 to indicate unlimited swap space --- src/main/settings/settings.cpp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/main/settings/settings.cpp b/src/main/settings/settings.cpp index 5d25adc855c1..6d07bc839562 100644 --- a/src/main/settings/settings.cpp +++ b/src/main/settings/settings.cpp @@ -975,20 +975,18 @@ Value MaximumMemorySetting::GetSetting(ClientContext &context) { // Maximum Temp Directory Size //===--------------------------------------------------------------------===// void MaximumTempDirectorySize::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) { - idx_t maximum_swap_space = DConstants::INVALID_INDEX; - if (input.ToString() != "-1") { - maximum_swap_space = DBConfig::ParseMemoryLimit(input.ToString()); + auto maximum_swap_space = DBConfig::ParseMemoryLimit(input.ToString()); + if (maximum_swap_space == DConstants::INVALID_INDEX) { + // We use INVALID_INDEX to indicate that the value is not set by the user + // use one lower to indicate 'unlimited' + maximum_swap_space--; } if (!db) { config.options.maximum_swap_space = maximum_swap_space; return; } auto &buffer_manager = BufferManager::GetBufferManager(*db); - if (maximum_swap_space == DConstants::INVALID_INDEX) { - buffer_manager.SetSwapLimit(); - } else { - buffer_manager.SetSwapLimit(maximum_swap_space); - } + buffer_manager.SetSwapLimit(maximum_swap_space); config.options.maximum_swap_space = maximum_swap_space; } From 9ce6c39d4e2c052f52993542125f019ddf1cc024 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Tue, 9 Apr 2024 12:44:50 +0200 Subject: [PATCH 249/603] hex and array slice, my old nemesis --- src/core_functions/scalar/bit/bitstring.cpp | 8 +- .../scalar/list/array_slice.cpp | 34 ++-- src/core_functions/scalar/string/hex.cpp | 26 +-- src/include/duckdb/common/bit_utils.hpp | 161 +++++++----------- 4 files changed, 99 insertions(+), 130 deletions(-) diff --git a/src/core_functions/scalar/bit/bitstring.cpp b/src/core_functions/scalar/bit/bitstring.cpp index babfadfe01e7..fc1768850f07 100644 --- a/src/core_functions/scalar/bit/bitstring.cpp +++ b/src/core_functions/scalar/bit/bitstring.cpp @@ -19,9 +19,9 @@ static void BitStringFunction(DataChunk &args, ExpressionState &state, Vector &r idx_t len; Bit::TryGetBitStringSize(input, len, nullptr); // string verification - len = Bit::ComputeBitstringLen(n); + len = Bit::ComputeBitstringLen(UnsafeNumericCast(n)); string_t target = StringVector::EmptyString(result, len); - Bit::BitString(input, n, target); + Bit::BitString(input, UnsafeNumericCast(n), target); target.Finalize(); return target; }); @@ -41,7 +41,7 @@ struct GetBitOperator { throw OutOfRangeException("bit index %s out of valid range (0..%s)", NumericHelper::ToString(n), NumericHelper::ToString(Bit::BitLength(input) - 1)); } - return UnsafeNumericCast(Bit::GetBit(input, n)); + return UnsafeNumericCast(Bit::GetBit(input, UnsafeNumericCast(n))); } }; @@ -66,7 +66,7 @@ static void SetBitOperation(DataChunk &args, ExpressionState &state, Vector &res } string_t target = StringVector::EmptyString(result, input.GetSize()); memcpy(target.GetDataWriteable(), input.GetData(), input.GetSize()); - Bit::SetBit(target, n, new_value); + Bit::SetBit(target, UnsafeNumericCast(n), UnsafeNumericCast(new_value)); return target; }); } diff --git a/src/core_functions/scalar/list/array_slice.cpp b/src/core_functions/scalar/list/array_slice.cpp index 7651f410d023..f708caf431ff 100644 --- a/src/core_functions/scalar/list/array_slice.cpp +++ b/src/core_functions/scalar/list/array_slice.cpp @@ -47,15 +47,17 @@ static idx_t CalculateSliceLength(idx_t begin, idx_t end, INDEX_TYPE step, bool if (step == 0 && svalid) { throw InvalidInputException("Slice step cannot be zero"); } + auto step_unsigned = UnsafeNumericCast(step); // we called abs() on this above. + if (step == 1) { - return NumericCast(end - begin); - } else if (static_cast(step) >= (end - begin)) { + return NumericCast(end - begin); + } else if (step_unsigned >= (end - begin)) { return 1; } - if ((end - begin) % step != 0) { - return (end - begin) / step + 1; + if ((end - begin) % step_unsigned != 0) { + return (end - begin) / step_unsigned + 1; } - return (end - begin) / step; + return (end - begin) / step_unsigned; } template @@ -65,7 +67,7 @@ INDEX_TYPE ValueLength(const INPUT_TYPE &value) { template <> int64_t ValueLength(const list_entry_t &value) { - return value.length; + return UnsafeNumericCast(value.length); } template <> @@ -119,8 +121,8 @@ INPUT_TYPE SliceValue(Vector &result, INPUT_TYPE input, INDEX_TYPE begin, INDEX_ template <> list_entry_t SliceValue(Vector &result, list_entry_t input, int64_t begin, int64_t end) { - input.offset += begin; - input.length = end - begin; + input.offset = UnsafeNumericCast(UnsafeNumericCast(input.offset) + begin); + input.length = UnsafeNumericCast(end - begin); return input; } @@ -144,15 +146,15 @@ list_entry_t SliceValueWithSteps(Vector &result, SelectionVector &sel, list_entr input.offset = sel_idx; return input; } - input.length = CalculateSliceLength(begin, end, step, true); - idx_t child_idx = input.offset + begin; + input.length = CalculateSliceLength(UnsafeNumericCast(begin), UnsafeNumericCast(end), step, true); + int64_t child_idx = UnsafeNumericCast(input.offset) + begin; if (step < 0) { - child_idx = input.offset + end - 1; + child_idx = UnsafeNumericCast(input.offset) + end - 1; } input.offset = sel_idx; for (idx_t i = 0; i < input.length; i++) { - sel.set_index(sel_idx, child_idx); - child_idx += step; + sel.set_index(sel_idx, UnsafeNumericCast(child_idx)); + child_idx = UnsafeNumericCast(child_idx) + step; sel_idx++; } return input; @@ -194,7 +196,8 @@ static void ExecuteConstantSlice(Vector &result, Vector &str_vector, Vector &beg idx_t sel_length = 0; bool sel_valid = false; if (step_vector && step_valid && str_valid && begin_valid && end_valid && step != 1 && end - begin > 0) { - sel_length = CalculateSliceLength(begin, end, step, step_valid); + sel_length = + CalculateSliceLength(UnsafeNumericCast(begin), UnsafeNumericCast(end), step, step_valid); sel.Initialize(sel_length); sel_valid = true; } @@ -268,7 +271,8 @@ static void ExecuteFlatSlice(Vector &result, Vector &list_vector, Vector &begin_ idx_t length = 0; if (end - begin > 0) { - length = CalculateSliceLength(begin, end, step, step_valid); + length = + CalculateSliceLength(UnsafeNumericCast(begin), UnsafeNumericCast(end), step, step_valid); } sel_length += length; diff --git a/src/core_functions/scalar/string/hex.cpp b/src/core_functions/scalar/string/hex.cpp index dffbae70d030..f399b65cfbc2 100644 --- a/src/core_functions/scalar/string/hex.cpp +++ b/src/core_functions/scalar/string/hex.cpp @@ -90,7 +90,7 @@ struct HexIntegralOperator { template static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) { - idx_t num_leading_zero = CountZeros::Leading(input); + auto num_leading_zero = CountZeros::Leading(static_cast(input)); idx_t num_bits_to_check = 64 - num_leading_zero; D_ASSERT(num_bits_to_check <= sizeof(INPUT_TYPE) * 8); @@ -109,7 +109,7 @@ struct HexIntegralOperator { auto target = StringVector::EmptyString(result, buffer_size); auto output = target.GetDataWriteable(); - WriteHexBytes(input, output, buffer_size); + WriteHexBytes(static_cast(input), output, buffer_size); target.Finalize(); return target; @@ -120,7 +120,7 @@ struct HexHugeIntOperator { template static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) { - idx_t num_leading_zero = CountZeros::Leading(input); + idx_t num_leading_zero = CountZeros::Leading(UnsafeNumericCast(input)); idx_t buffer_size = sizeof(INPUT_TYPE) * 2 - (num_leading_zero / 4); // Special case: All bits are zero @@ -147,7 +147,7 @@ struct HexUhugeIntOperator { template static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) { - idx_t num_leading_zero = CountZeros::Leading(input); + idx_t num_leading_zero = CountZeros::Leading(UnsafeNumericCast(input)); idx_t buffer_size = sizeof(INPUT_TYPE) * 2 - (num_leading_zero / 4); // Special case: All bits are zero @@ -189,7 +189,7 @@ struct BinaryStrOperator { auto output = target.GetDataWriteable(); for (idx_t i = 0; i < size; ++i) { - uint8_t byte = data[i]; + auto byte = static_cast(data[i]); for (idx_t i = 8; i >= 1; --i) { *output = ((byte >> (i - 1)) & 0x01) + '0'; output++; @@ -205,7 +205,7 @@ struct BinaryIntegralOperator { template static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) { - idx_t num_leading_zero = CountZeros::Leading(input); + auto num_leading_zero = CountZeros::Leading(static_cast(input)); idx_t num_bits_to_check = 64 - num_leading_zero; D_ASSERT(num_bits_to_check <= sizeof(INPUT_TYPE) * 8); @@ -224,7 +224,7 @@ struct BinaryIntegralOperator { auto target = StringVector::EmptyString(result, buffer_size); auto output = target.GetDataWriteable(); - WriteBinBytes(input, output, buffer_size); + WriteBinBytes(static_cast(input), output, buffer_size); target.Finalize(); return target; @@ -234,7 +234,7 @@ struct BinaryIntegralOperator { struct BinaryHugeIntOperator { template static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) { - idx_t num_leading_zero = CountZeros::Leading(input); + auto num_leading_zero = CountZeros::Leading(UnsafeNumericCast(input)); idx_t buffer_size = sizeof(INPUT_TYPE) * 8 - num_leading_zero; // Special case: All bits are zero @@ -259,7 +259,7 @@ struct BinaryHugeIntOperator { struct BinaryUhugeIntOperator { template static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) { - idx_t num_leading_zero = CountZeros::Leading(input); + auto num_leading_zero = CountZeros::Leading(UnsafeNumericCast(input)); idx_t buffer_size = sizeof(INPUT_TYPE) * 8 - num_leading_zero; // Special case: All bits are zero @@ -301,7 +301,7 @@ struct FromHexOperator { // Treated as a single byte idx_t i = 0; if (size % 2 != 0) { - *output = StringUtil::GetHexValue(data[i]); + *output = static_cast(StringUtil::GetHexValue(data[i])); i++; output++; } @@ -309,7 +309,7 @@ struct FromHexOperator { for (; i < size; i += 2) { uint8_t major = StringUtil::GetHexValue(data[i]); uint8_t minor = StringUtil::GetHexValue(data[i + 1]); - *output = UnsafeNumericCast((major << 4) | minor); + *output = static_cast((major << 4) | minor); output++; } @@ -343,7 +343,7 @@ struct FromBinaryOperator { byte |= StringUtil::GetBinaryValue(data[i]) << (j - 1); i++; } - *output = byte; + *output = static_cast(byte); // binary eh output++; } @@ -353,7 +353,7 @@ struct FromBinaryOperator { byte |= StringUtil::GetBinaryValue(data[i]) << (j - 1); i++; } - *output = byte; + *output = static_cast(byte); output++; } diff --git a/src/include/duckdb/common/bit_utils.hpp b/src/include/duckdb/common/bit_utils.hpp index 3c4f4a6bfcb8..0b9c325d9c23 100644 --- a/src/include/duckdb/common/bit_utils.hpp +++ b/src/include/duckdb/common/bit_utils.hpp @@ -10,68 +10,7 @@ #include "duckdb/common/hugeint.hpp" #include "duckdb/common/uhugeint.hpp" - -#if defined(_MSC_VER) && !defined(__clang__) -#define __restrict__ -#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ -#define __ORDER_LITTLE_ENDIAN__ 2 -#include -static inline int __builtin_ctzll(unsigned long long x) { -#ifdef _WIN64 - unsigned long ret; - _BitScanForward64(&ret, x); - return (int)ret; -#else - unsigned long low, high; - bool low_set = _BitScanForward(&low, (unsigned __int32)(x)) != 0; - _BitScanForward(&high, (unsigned __int32)(x >> 32)); - high += 32; - return low_set ? low : high; -#endif -} -static inline int __builtin_clzll(unsigned long long mask) { - unsigned long where; -// BitScanReverse scans from MSB to LSB for first set bit. -// Returns 0 if no set bit is found. -#if defined(_WIN64) - if (_BitScanReverse64(&where, mask)) - return static_cast(63 - where); -#elif defined(_WIN32) - // Scan the high 32 bits. - if (_BitScanReverse(&where, static_cast(mask >> 32))) - return static_cast(63 - (where + 32)); // Create a bit offset from the MSB. - // Scan the low 32 bits. - if (_BitScanReverse(&where, static_cast(mask))) - return static_cast(63 - where); -#else -#error "Implementation of __builtin_clzll required" -#endif - return 64; // Undefined Behavior. -} - -static inline int __builtin_ctz(unsigned int value) { - unsigned long trailing_zero = 0; - - if (_BitScanForward(&trailing_zero, value)) { - return trailing_zero; - } else { - // This is undefined, I better choose 32 than 0 - return 32; - } -} - -static inline int __builtin_clz(unsigned int value) { - unsigned long leading_zero = 0; - - if (_BitScanReverse(&leading_zero, value)) { - return 31 - leading_zero; - } else { - // Same remarks as above - return 32; - } -} - -#endif +#include "duckdb/common/numeric_utils.hpp" namespace duckdb { @@ -79,60 +18,86 @@ template struct CountZeros {}; template <> -struct CountZeros { - inline static int Leading(uint32_t value) { - if (!value) { - return 32; +struct CountZeros { + // see here: https://en.wikipedia.org/wiki/De_Bruijn_sequence + inline static idx_t Leading(const uint64_t value_in) { + if (!value_in) { + return 64; } - return __builtin_clz(value); + + uint64_t value = value_in; + + constexpr uint64_t index64msb[] = {0, 47, 1, 56, 48, 27, 2, 60, 57, 49, 41, 37, 28, 16, 3, 61, + 54, 58, 35, 52, 50, 42, 21, 44, 38, 32, 29, 23, 17, 11, 4, 62, + 46, 55, 26, 59, 40, 36, 15, 53, 34, 51, 20, 43, 31, 22, 10, 45, + 25, 39, 14, 33, 19, 30, 9, 24, 13, 18, 8, 12, 7, 6, 5, 63}; + + constexpr uint64_t debruijn64msb = 0X03F79D71B4CB0A89; + + value |= value >> 1; + value |= value >> 2; + value |= value >> 4; + value |= value >> 8; + value |= value >> 16; + value |= value >> 32; + auto result = 63 - index64msb[(value * debruijn64msb) >> 58]; +#ifdef __clang__ + D_ASSERT(result == static_cast(__builtin_clzl(value_in))); +#endif + return result; } - inline static int Trailing(uint32_t value) { - if (!value) { - return 32; + inline static idx_t Trailing(uint64_t value_in) { + if (!value_in) { + return 64; } - return __builtin_ctz(value); + uint64_t value = value_in; + + constexpr uint64_t index64lsb[] = {63, 0, 58, 1, 59, 47, 53, 2, 60, 39, 48, 27, 54, 33, 42, 3, + 61, 51, 37, 40, 49, 18, 28, 20, 55, 30, 34, 11, 43, 14, 22, 4, + 62, 57, 46, 52, 38, 26, 32, 41, 50, 36, 17, 19, 29, 10, 13, 21, + 56, 45, 25, 31, 35, 16, 9, 12, 44, 24, 15, 8, 23, 7, 6, 5}; + constexpr uint64_t debruijn64lsb = 0x07EDD5E59A4E28C2ULL; + auto result = index64lsb[((value & -value) * debruijn64lsb) >> 58] - 1; +#ifdef __clang__ + D_ASSERT(result == static_cast(__builtin_clzl(value_in))); +#endif + return result; } }; template <> -struct CountZeros { - inline static int Leading(uint64_t value) { - if (!value) { - return 64; - } - return __builtin_clzll(value); +struct CountZeros { + inline static idx_t Leading(uint32_t value) { + return CountZeros::Leading(static_cast(value)) - 32; } - inline static int Trailing(uint64_t value) { - if (!value) { - return 64; - } - return __builtin_ctzll(value); + inline static idx_t Trailing(uint32_t value) { + return CountZeros::Trailing(static_cast(value)); } }; template <> struct CountZeros { - inline static int Leading(hugeint_t value) { - const uint64_t upper = (uint64_t)value.upper; + inline static idx_t Leading(hugeint_t value) { + const uint64_t upper = static_cast(value.upper); const uint64_t lower = value.lower; if (upper) { - return __builtin_clzll(upper); + return CountZeros::Leading(upper); } else if (lower) { - return 64 + __builtin_clzll(lower); + return 64 + CountZeros::Leading(lower); } else { return 128; } } - inline static int Trailing(hugeint_t value) { - const uint64_t upper = (uint64_t)value.upper; + inline static idx_t Trailing(hugeint_t value) { + const uint64_t upper = static_cast(value.upper); const uint64_t lower = value.lower; if (lower) { - return __builtin_ctzll(lower); + return CountZeros::Trailing(lower); } else if (upper) { - return 64 + __builtin_ctzll(upper); + return 64 + CountZeros::Trailing(upper); } else { return 128; } @@ -141,27 +106,27 @@ struct CountZeros { template <> struct CountZeros { - inline static int Leading(uhugeint_t value) { - const uint64_t upper = (uint64_t)value.upper; + inline static idx_t Leading(uhugeint_t value) { + const uint64_t upper = static_cast(value.upper); const uint64_t lower = value.lower; if (upper) { - return __builtin_clzll(upper); + return CountZeros::Leading(upper); } else if (lower) { - return 64 + __builtin_clzll(lower); + return 64 + CountZeros::Leading(lower); } else { return 128; } } - inline static int Trailing(uhugeint_t value) { - const uint64_t upper = (uint64_t)value.upper; + inline static idx_t Trailing(uhugeint_t value) { + const uint64_t upper = static_cast(value.upper); const uint64_t lower = value.lower; if (lower) { - return __builtin_ctzll(lower); + return CountZeros::Trailing(lower); } else if (upper) { - return 64 + __builtin_ctzll(upper); + return 64 + CountZeros::Trailing(upper); } else { return 128; } From 41cd77ae336fb61aae34947116862f17144d1c0d Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Tue, 9 Apr 2024 12:57:07 +0200 Subject: [PATCH 250/603] One more test still having scan_ids --- .../csv_incorrect_columns_amount_rejects.test | 84 +++++++++---------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test b/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test index 2b59e17547d3..f1b63112a4c4 100644 --- a/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test +++ b/test/sql/copy/csv/rejects/csv_incorrect_columns_amount_rejects.test @@ -13,13 +13,13 @@ SELECT * FROM read_csv( columns = {'a': 'INTEGER', 'b': 'INTEGER', 'c': 'INTEGER', 'd': 'INTEGER'}, store_rejects=true, auto_detect=false, header = 1); -query IIIIIIIIII rowsort -FROM reject_errors order by all; +query IIIIIIIII rowsort +SELECT * EXCLUDE (scan_id) FROM reject_errors order by all; ---- -3 0 1814 14505 14510 3 d MISSING COLUMNS 1,2,3 Expected Number of Columns: 4 Found: 3 -3 0 1823 14575 14576 1 b MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 1 -3 0 2378 19009 19010 1 b MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 1 -3 0 2762 22075 22078 2 c MISSING COLUMNS 1,2 Expected Number of Columns: 4 Found: 2 +0 1814 14505 14510 3 d MISSING COLUMNS 1,2,3 Expected Number of Columns: 4 Found: 3 +0 1823 14575 14576 1 b MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 1 +0 2378 19009 19010 1 b MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 1 +0 2762 22075 22078 2 c MISSING COLUMNS 1,2 Expected Number of Columns: 4 Found: 2 statement ok DROP TABLE reject_errors; @@ -33,15 +33,15 @@ SELECT * FROM read_csv( columns = {'a': 'INTEGER', 'b': 'INTEGER', 'c': 'INTEGER', 'd': 'INTEGER'}, store_rejects=true, auto_detect=false, header = 1); -query IIIIIIIIII rowsort -FROM reject_errors order by all; +query IIIIIIIII rowsort +SELECT * EXCLUDE (scan_id) FROM reject_errors order by all; ---- -7 0 1096 8761 8768 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5 -7 0 1096 8761 8770 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6 -7 0 1159 9269 9276 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5 -7 0 1159 9269 9278 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6 -7 0 1206 9649 9656 5 NULL TOO MANY COLUMNS 1,2,3,4,5 Expected Number of Columns: 4 Found: 5 -7 0 2769 22155 22162 5 NULL TOO MANY COLUMNS 1,2,3,4,5 Expected Number of Columns: 4 Found: 5 +0 1096 8761 8768 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5 +0 1096 8761 8770 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6 +0 1159 9269 9276 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5 +0 1159 9269 9278 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6 +0 1206 9649 9656 5 NULL TOO MANY COLUMNS 1,2,3,4,5 Expected Number of Columns: 4 Found: 5 +0 2769 22155 22162 5 NULL TOO MANY COLUMNS 1,2,3,4,5 Expected Number of Columns: 4 Found: 5 statement ok DROP TABLE reject_errors; @@ -55,15 +55,15 @@ SELECT * FROM read_csv( columns = {'a': 'INTEGER', 'b': 'INTEGER', 'c': 'INTEGER', 'd': 'INTEGER'}, store_rejects=true, auto_detect=false, header = 1); -query IIIIIIIIII rowsort -FROM reject_errors order by all; +query IIIIIIIII rowsort +SELECT * EXCLUDE (scan_id) FROM reject_errors order by all; ---- -11 0 1604 12825 12826 1 b MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 1 -11 0 1671 13355 13362 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5 -11 0 1671 13355 13364 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6 -11 0 2751 21999 22002 2 c MISSING COLUMNS 1,2 Expected Number of Columns: 4 Found: 2 -11 0 2768 22131 22138 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5 -11 0 2768 22131 22140 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6 +0 1604 12825 12826 1 b MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 1 +0 1671 13355 13362 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5 +0 1671 13355 13364 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6 +0 2751 21999 22002 2 c MISSING COLUMNS 1,2 Expected Number of Columns: 4 Found: 2 +0 2768 22131 22138 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5 +0 2768 22131 22140 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6 # Different Buffer Sizes @@ -102,24 +102,24 @@ SELECT * FROM read_csv( columns = {'a': 'INTEGER', 'b': 'INTEGER', 'c': 'INTEGER', 'd': 'INTEGER'}, store_rejects=true, auto_detect=false, header = 1); -query IIIIIIIIII rowsort -FROM reject_errors order by all +query IIIIIIIII rowsort +SELECT * EXCLUDE (scan_id) FROM reject_errors order by all ---- -35 0 1814 14505 14510 3 d MISSING COLUMNS 1,2,3 Expected Number of Columns: 4 Found: 3 -35 0 1823 14575 14576 1 b MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 1 -35 0 2378 19009 19010 1 b MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 1 -35 0 2762 22075 22078 2 c MISSING COLUMNS 1,2 Expected Number of Columns: 4 Found: 2 -35 1 1096 8761 8768 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5 -35 1 1096 8761 8770 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6 -35 1 1159 9269 9276 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5 -35 1 1159 9269 9278 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6 -35 1 1206 9649 9656 5 NULL TOO MANY COLUMNS 1,2,3,4,5 Expected Number of Columns: 4 Found: 5 -35 1 2769 22155 22162 5 NULL TOO MANY COLUMNS 1,2,3,4,5 Expected Number of Columns: 4 Found: 5 -35 2 1604 12825 12826 1 b MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 1 -35 2 1671 13355 13362 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5 -35 2 1671 13355 13364 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6 -35 2 2751 21999 22002 2 c MISSING COLUMNS 1,2 Expected Number of Columns: 4 Found: 2 -35 2 2768 22131 22138 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5 -35 2 2768 22131 22140 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6 -35 3 3 17 24 5 NULL TOO MANY COLUMNS 1,2,3,4,5 Expected Number of Columns: 4 Found: 5 -35 3 4 27 32 3 d MISSING COLUMNS 1,2,3 Expected Number of Columns: 4 Found: 3 +0 1814 14505 14510 3 d MISSING COLUMNS 1,2,3 Expected Number of Columns: 4 Found: 3 +0 1823 14575 14576 1 b MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 1 +0 2378 19009 19010 1 b MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 1 +0 2762 22075 22078 2 c MISSING COLUMNS 1,2 Expected Number of Columns: 4 Found: 2 +1 1096 8761 8768 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5 +1 1096 8761 8770 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6 +1 1159 9269 9276 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5 +1 1159 9269 9278 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6 +1 1206 9649 9656 5 NULL TOO MANY COLUMNS 1,2,3,4,5 Expected Number of Columns: 4 Found: 5 +1 2769 22155 22162 5 NULL TOO MANY COLUMNS 1,2,3,4,5 Expected Number of Columns: 4 Found: 5 +2 1604 12825 12826 1 b MISSING COLUMNS 1 Expected Number of Columns: 4 Found: 1 +2 1671 13355 13362 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5 +2 1671 13355 13364 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6 +2 2751 21999 22002 2 c MISSING COLUMNS 1,2 Expected Number of Columns: 4 Found: 2 +2 2768 22131 22138 5 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 5 +2 2768 22131 22140 6 NULL TOO MANY COLUMNS 1,2,3,4,5,6 Expected Number of Columns: 4 Found: 6 +3 3 17 24 5 NULL TOO MANY COLUMNS 1,2,3,4,5 Expected Number of Columns: 4 Found: 5 +3 4 27 32 3 d MISSING COLUMNS 1,2,3 Expected Number of Columns: 4 Found: 3 From 6877efe32cbe32a4e41b5bc525537ad31c6f25a8 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Tue, 9 Apr 2024 13:08:34 +0200 Subject: [PATCH 251/603] Bot does not have permission for drafting --- .github/workflows/DraftPR.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/DraftPR.yml b/.github/workflows/DraftPR.yml index d20ff54e2486..0fd408681cb9 100644 --- a/.github/workflows/DraftPR.yml +++ b/.github/workflows/DraftPR.yml @@ -43,7 +43,7 @@ jobs: - name: 'Actually move to draft' shell: bash run: | - echo ${{ secrets.DUCKDBLABS_BOT_TOKEN }} | gh auth login --with-token + echo ${{ secrets.GH_TOKEN }} | gh auth login --with-token gh api graphql -F id=${{ env.PR_NUMBER }} -f query=' mutation($id: ID!) { convertPullRequestToDraft(input: { pullRequestId: $id }) { From 4995a4b744e442972fe333755ef85e63c4823780 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Tue, 9 Apr 2024 13:36:25 +0200 Subject: [PATCH 252/603] Add missing includes --- src/planner/expression_binder/group_binder.cpp | 1 + src/planner/expression_binder/order_binder.cpp | 2 ++ src/planner/expression_binder/select_bind_state.cpp | 1 + src/planner/expression_binder/select_binder.cpp | 2 ++ 4 files changed, 6 insertions(+) diff --git a/src/planner/expression_binder/group_binder.cpp b/src/planner/expression_binder/group_binder.cpp index ff78acbd379c..cbf25d76d649 100644 --- a/src/planner/expression_binder/group_binder.cpp +++ b/src/planner/expression_binder/group_binder.cpp @@ -4,6 +4,7 @@ #include "duckdb/parser/expression/constant_expression.hpp" #include "duckdb/parser/query_node/select_node.hpp" #include "duckdb/planner/expression/bound_constant_expression.hpp" +#include "duckdb/planner/expression_binder/select_bind_state.hpp" #include "duckdb/common/to_string.hpp" namespace duckdb { diff --git a/src/planner/expression_binder/order_binder.cpp b/src/planner/expression_binder/order_binder.cpp index 67c883f824b3..7dc062d180f6 100644 --- a/src/planner/expression_binder/order_binder.cpp +++ b/src/planner/expression_binder/order_binder.cpp @@ -8,8 +8,10 @@ #include "duckdb/parser/expression/star_expression.hpp" #include "duckdb/parser/query_node/select_node.hpp" #include "duckdb/planner/binder.hpp" +#include "duckdb/planner/expression/bound_constant_expression.hpp" #include "duckdb/planner/expression/bound_parameter_expression.hpp" #include "duckdb/planner/expression_binder.hpp" +#include "duckdb/planner/expression_binder/select_bind_state.hpp" #include "duckdb/common/pair.hpp" namespace duckdb { diff --git a/src/planner/expression_binder/select_bind_state.cpp b/src/planner/expression_binder/select_bind_state.cpp index 87256565d248..4b0a19cd2e5b 100644 --- a/src/planner/expression_binder/select_bind_state.cpp +++ b/src/planner/expression_binder/select_bind_state.cpp @@ -1,4 +1,5 @@ #include "duckdb/planner/expression_binder/select_bind_state.hpp" +#include "duckdb/common/exception/binder_exception.hpp" namespace duckdb { diff --git a/src/planner/expression_binder/select_binder.cpp b/src/planner/expression_binder/select_binder.cpp index badab70f16de..69c1dcdf10c2 100644 --- a/src/planner/expression_binder/select_binder.cpp +++ b/src/planner/expression_binder/select_binder.cpp @@ -1,4 +1,6 @@ #include "duckdb/planner/expression_binder/select_binder.hpp" +#include "duckdb/parser/expression/columnref_expression.hpp" +#include "duckdb/planner/query_node/bound_select_node.hpp" namespace duckdb { From 7b6d8f09a15de9b438864283fc93a4ac85e402db Mon Sep 17 00:00:00 2001 From: Gabor Szarnyas Date: Tue, 9 Apr 2024 13:39:32 +0200 Subject: [PATCH 253/603] CI: Remove 'needs reproducible example' when 'reproduced' label is applied --- .github/workflows/InternalIssuesCreateMirror.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/InternalIssuesCreateMirror.yml b/.github/workflows/InternalIssuesCreateMirror.yml index 89271105e10e..ac2a2874d944 100644 --- a/.github/workflows/InternalIssuesCreateMirror.yml +++ b/.github/workflows/InternalIssuesCreateMirror.yml @@ -25,7 +25,7 @@ jobs: - name: Remove 'needs triage' / 'under review' if 'reproduced' if: github.event.label.name == 'reproduced' run: | - gh issue edit --repo duckdb/duckdb ${{ github.event.issue.number }} --remove-label "needs triage" --remove-label "under review" + gh issue edit --repo duckdb/duckdb ${{ github.event.issue.number }} --remove-label "needs triage" --remove-label "under review" --remove-label "needs reproducible example" - name: Remove 'needs triage' / 'reproduced' if 'under review' if: github.event.label.name == 'under review' From 0479f93ef4c6a284639d85cf1859ddff09952ffb Mon Sep 17 00:00:00 2001 From: Gabor Szarnyas Date: Tue, 9 Apr 2024 13:50:58 +0200 Subject: [PATCH 254/603] Issue template: Add more extensions and connectors --- .github/ISSUE_TEMPLATE/bug_report.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index f3e918909bf1..1ead7f8f1b27 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -16,6 +16,7 @@ body: * [duckdb-wasm](https://github.com/duckdb/duckdb-wasm/issues/new) * [go-duckdb](https://github.com/marcboeker/go-duckdb/issues/new) * Extensions: + * [Arrow extension](https://github.com/duckdb/duckdb_arrow/issues/new) * [AWS extension](https://github.com/duckdb/duckdb_aws/issues/new) * [Azure extension](https://github.com/duckdb/duckdb_azure/issues/new) * [Iceberg extension](https://github.com/duckdb/duckdb_iceberg/issues/new) @@ -23,6 +24,8 @@ body: * [Postgres scanner](https://github.com/duckdb/postgres_scanner/issues/new) * [Spatial extension](https://github.com/duckdb/duckdb_spatial/issues/new) * [SQLite scanner](https://github.com/duckdb/sqlite_scanner/issues/new) + * Connectors: + * [dbt-duckdb](https://github.com/duckdb/dbt-duckdb) If none of the above repositories are applicable, feel free to raise it in this one From 27b6d2864d9fb7924e52b00b6f1828d9d4a31ba3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Tue, 9 Apr 2024 14:07:44 +0200 Subject: [PATCH 255/603] strftime --- src/function/scalar/strftime_format.cpp | 69 +++++++++++++------------ src/include/duckdb/common/bit_utils.hpp | 4 +- 2 files changed, 39 insertions(+), 34 deletions(-) diff --git a/src/function/scalar/strftime_format.cpp b/src/function/scalar/strftime_format.cpp index 5181b005ed45..5784ed91b895 100644 --- a/src/function/scalar/strftime_format.cpp +++ b/src/function/scalar/strftime_format.cpp @@ -80,7 +80,7 @@ idx_t StrfTimeFormat::GetSpecifierLength(StrTimeSpecifier specifier, date_t date if (0 <= year && year <= 9999) { return 4; } else { - return NumericHelper::SignedLength(year); + return UnsafeNumericCast(NumericHelper::SignedLength(year)); } } case StrTimeSpecifier::MONTH_DECIMAL: { @@ -129,11 +129,14 @@ idx_t StrfTimeFormat::GetSpecifierLength(StrTimeSpecifier specifier, date_t date return len; } case StrTimeSpecifier::DAY_OF_MONTH: - return NumericHelper::UnsignedLength(Date::ExtractDay(date)); + return UnsafeNumericCast( + NumericHelper::UnsignedLength(UnsafeNumericCast(Date::ExtractDay(date)))); case StrTimeSpecifier::DAY_OF_YEAR_DECIMAL: - return NumericHelper::UnsignedLength(Date::ExtractDayOfTheYear(date)); + return UnsafeNumericCast( + NumericHelper::UnsignedLength(UnsafeNumericCast(Date::ExtractDayOfTheYear(date)))); case StrTimeSpecifier::YEAR_WITHOUT_CENTURY: - return NumericHelper::UnsignedLength(AbsValue(Date::ExtractYear(date)) % 100); + return UnsafeNumericCast(NumericHelper::UnsignedLength( + UnsafeNumericCast(AbsValue(Date::ExtractYear(date)) % 100))); default: throw InternalException("Unimplemented specifier for GetSpecifierLength"); } @@ -195,13 +198,13 @@ char *StrfTimeFormat::WritePadded(char *target, uint32_t value, size_t padding) D_ASSERT(padding > 1); if (padding % 2) { int decimals = value % 1000; - WritePadded3(target + padding - 3, decimals); + WritePadded3(target + padding - 3, UnsafeNumericCast(decimals)); value /= 1000; padding -= 3; } for (size_t i = 0; i < padding / 2; i++) { int decimals = value % 100; - WritePadded2(target + padding - 2 * (i + 1), decimals); + WritePadded2(target + padding - 2 * (i + 1), UnsafeNumericCast(decimals)); value /= 100; } return target + padding; @@ -245,26 +248,26 @@ char *StrfTimeFormat::WriteDateSpecifier(StrTimeSpecifier specifier, date_t date } case StrTimeSpecifier::DAY_OF_YEAR_PADDED: { int32_t doy = Date::ExtractDayOfTheYear(date); - target = WritePadded3(target, doy); + target = WritePadded3(target, UnsafeNumericCast(doy)); break; } case StrTimeSpecifier::WEEK_NUMBER_PADDED_MON_FIRST: - target = WritePadded2(target, Date::ExtractWeekNumberRegular(date, true)); + target = WritePadded2(target, UnsafeNumericCast(Date::ExtractWeekNumberRegular(date, true))); break; case StrTimeSpecifier::WEEK_NUMBER_PADDED_SUN_FIRST: - target = WritePadded2(target, Date::ExtractWeekNumberRegular(date, false)); + target = WritePadded2(target, UnsafeNumericCast(Date::ExtractWeekNumberRegular(date, false))); break; case StrTimeSpecifier::WEEK_NUMBER_ISO: - target = WritePadded2(target, Date::ExtractISOWeekNumber(date)); + target = WritePadded2(target, UnsafeNumericCast(Date::ExtractISOWeekNumber(date))); break; case StrTimeSpecifier::DAY_OF_YEAR_DECIMAL: { - uint32_t doy = Date::ExtractDayOfTheYear(date); + auto doy = UnsafeNumericCast(Date::ExtractDayOfTheYear(date)); target += NumericHelper::UnsignedLength(doy); NumericHelper::FormatUnsigned(doy, target); break; } case StrTimeSpecifier::YEAR_ISO: - target = WritePadded(target, Date::ExtractISOYearNumber(date), 4); + target = WritePadded(target, UnsafeNumericCast(Date::ExtractISOYearNumber(date)), 4); break; case StrTimeSpecifier::WEEKDAY_ISO: *target = char('0' + uint8_t(Date::ExtractISODayOfTheWeek(date))); @@ -281,7 +284,7 @@ char *StrfTimeFormat::WriteStandardSpecifier(StrTimeSpecifier specifier, int32_t // data contains [0] year, [1] month, [2] day, [3] hour, [4] minute, [5] second, [6] msec, [7] utc switch (specifier) { case StrTimeSpecifier::DAY_OF_MONTH_PADDED: - target = WritePadded2(target, data[2]); + target = WritePadded2(target, UnsafeNumericCast(data[2])); break; case StrTimeSpecifier::ABBREVIATED_MONTH_NAME: { auto &month_name = Date::MONTH_NAMES_ABBREVIATED[data[1] - 1]; @@ -292,14 +295,14 @@ char *StrfTimeFormat::WriteStandardSpecifier(StrTimeSpecifier specifier, int32_t return WriteString(target, month_name); } case StrTimeSpecifier::MONTH_DECIMAL_PADDED: - target = WritePadded2(target, data[1]); + target = WritePadded2(target, UnsafeNumericCast(data[1])); break; case StrTimeSpecifier::YEAR_WITHOUT_CENTURY_PADDED: - target = WritePadded2(target, AbsValue(data[0]) % 100); + target = WritePadded2(target, UnsafeNumericCast(AbsValue(data[0]) % 100)); break; case StrTimeSpecifier::YEAR_DECIMAL: if (data[0] >= 0 && data[0] <= 9999) { - target = WritePadded(target, data[0], 4); + target = WritePadded(target, UnsafeNumericCast(data[0]), 4); } else { int32_t year = data[0]; if (data[0] < 0) { @@ -307,13 +310,13 @@ char *StrfTimeFormat::WriteStandardSpecifier(StrTimeSpecifier specifier, int32_t year = -year; target++; } - auto len = NumericHelper::UnsignedLength(year); + auto len = NumericHelper::UnsignedLength(UnsafeNumericCast(year)); NumericHelper::FormatUnsigned(year, target + len); target += len; } break; case StrTimeSpecifier::HOUR_24_PADDED: { - target = WritePadded2(target, data[3]); + target = WritePadded2(target, UnsafeNumericCast(data[3])); break; } case StrTimeSpecifier::HOUR_12_PADDED: { @@ -321,7 +324,7 @@ char *StrfTimeFormat::WriteStandardSpecifier(StrTimeSpecifier specifier, int32_t if (hour == 0) { hour = 12; } - target = WritePadded2(target, hour); + target = WritePadded2(target, UnsafeNumericCast(hour)); break; } case StrTimeSpecifier::AM_PM: @@ -329,20 +332,20 @@ char *StrfTimeFormat::WriteStandardSpecifier(StrTimeSpecifier specifier, int32_t *target++ = 'M'; break; case StrTimeSpecifier::MINUTE_PADDED: { - target = WritePadded2(target, data[4]); + target = WritePadded2(target, UnsafeNumericCast(data[4])); break; } case StrTimeSpecifier::SECOND_PADDED: - target = WritePadded2(target, data[5]); + target = WritePadded2(target, UnsafeNumericCast(data[5])); break; case StrTimeSpecifier::NANOSECOND_PADDED: - target = WritePadded(target, data[6] * Interval::NANOS_PER_MICRO, 9); + target = WritePadded(target, UnsafeNumericCast(data[6] * Interval::NANOS_PER_MICRO), 9); break; case StrTimeSpecifier::MICROSECOND_PADDED: - target = WritePadded(target, data[6], 6); + target = WritePadded(target, UnsafeNumericCast(data[6]), 6); break; case StrTimeSpecifier::MILLISECOND_PADDED: - target = WritePadded3(target, data[6] / Interval::MICROS_PER_MSEC); + target = WritePadded3(target, UnsafeNumericCast(data[6] / Interval::MICROS_PER_MSEC)); break; case StrTimeSpecifier::UTC_OFFSET: { *target++ = (data[7] < 0) ? '-' : '+'; @@ -350,10 +353,10 @@ char *StrfTimeFormat::WriteStandardSpecifier(StrTimeSpecifier specifier, int32_t auto offset = abs(data[7]); auto offset_hours = offset / Interval::MINS_PER_HOUR; auto offset_minutes = offset % Interval::MINS_PER_HOUR; - target = WritePadded2(target, offset_hours); + target = WritePadded2(target, UnsafeNumericCast(offset_hours)); if (offset_minutes) { *target++ = ':'; - target = WritePadded2(target, offset_minutes); + target = WritePadded2(target, UnsafeNumericCast(offset_minutes)); } break; } @@ -364,7 +367,7 @@ char *StrfTimeFormat::WriteStandardSpecifier(StrTimeSpecifier specifier, int32_t } break; case StrTimeSpecifier::DAY_OF_MONTH: { - target = Write2(target, data[2] % 100); + target = Write2(target, UnsafeNumericCast(data[2] % 100)); break; } case StrTimeSpecifier::MONTH_DECIMAL: { @@ -372,7 +375,7 @@ char *StrfTimeFormat::WriteStandardSpecifier(StrTimeSpecifier specifier, int32_t break; } case StrTimeSpecifier::YEAR_WITHOUT_CENTURY: { - target = Write2(target, AbsValue(data[0]) % 100); + target = Write2(target, UnsafeNumericCast(AbsValue(data[0]) % 100)); break; } case StrTimeSpecifier::HOUR_24_DECIMAL: { @@ -845,9 +848,9 @@ bool StrpTimeFormat::Parse(string_t str, ParseResult &result) const { // numeric specifier: parse a number uint64_t number = 0; size_t start_pos = pos; - size_t end_pos = start_pos + numeric_width[i]; + size_t end_pos = start_pos + UnsafeNumericCast(numeric_width[i]); while (pos < size && pos < end_pos && StringUtil::CharacterIsDigit(data[pos])) { - number = number * 10 + data[pos] - '0'; + number = number * 10 + UnsafeNumericCast(data[pos]) - '0'; pos++; } if (pos == start_pos) { @@ -1229,7 +1232,7 @@ bool StrpTimeFormat::Parse(string_t str, ParseResult &result) const { // But tz must not be empty. if (tz_end == tz_begin) { error_message = "Empty Time Zone name"; - error_position = tz_begin - data; + error_position = UnsafeNumericCast(tz_begin - data); return false; } result.tz.assign(tz_begin, tz_end); @@ -1288,7 +1291,9 @@ bool StrpTimeFormat::Parse(string_t str, ParseResult &result) const { case StrTimeSpecifier::WEEK_NUMBER_PADDED_MON_FIRST: { // Adjust weekday to be 0-based for the week type if (has_weekday) { - weekday = (weekday + 7 - int(offset_specifier == StrTimeSpecifier::WEEK_NUMBER_PADDED_MON_FIRST)) % 7; + weekday = (weekday + 7 - + static_cast(offset_specifier == StrTimeSpecifier::WEEK_NUMBER_PADDED_MON_FIRST)) % + 7; } // Get the start of week 1, move back 7 days and then weekno * 7 + weekday gives the date const auto jan1 = Date::FromDate(result_data[0], 1, 1); diff --git a/src/include/duckdb/common/bit_utils.hpp b/src/include/duckdb/common/bit_utils.hpp index 0b9c325d9c23..766e951e9269 100644 --- a/src/include/duckdb/common/bit_utils.hpp +++ b/src/include/duckdb/common/bit_utils.hpp @@ -57,9 +57,9 @@ struct CountZeros { 62, 57, 46, 52, 38, 26, 32, 41, 50, 36, 17, 19, 29, 10, 13, 21, 56, 45, 25, 31, 35, 16, 9, 12, 44, 24, 15, 8, 23, 7, 6, 5}; constexpr uint64_t debruijn64lsb = 0x07EDD5E59A4E28C2ULL; - auto result = index64lsb[((value & -value) * debruijn64lsb) >> 58] - 1; + auto result = index64lsb[((value & -value) * debruijn64lsb) >> 58]; #ifdef __clang__ - D_ASSERT(result == static_cast(__builtin_clzl(value_in))); + D_ASSERT(result == static_cast(__builtin_ctzl(value_in))); #endif return result; } From 0ecc6820ee6c9b3f2fac2713eb253393fe97bdf2 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 8 Apr 2024 15:12:50 +0200 Subject: [PATCH 256/603] Extension metadata: Move logic from bash to cmake (invoked recursively) --- CMakeLists.txt | 10 ++--- scripts/append_metadata.cmake | 68 ++++++++++++++++++++++++++++++++++ scripts/append_metadata.sh | 52 -------------------------- scripts/null.txt | Bin 0 -> 1 bytes 4 files changed, 71 insertions(+), 59 deletions(-) create mode 100644 scripts/append_metadata.cmake delete mode 100755 scripts/append_metadata.sh create mode 100644 scripts/null.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e28c1aa4c61..f071e73d212d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -820,11 +820,7 @@ function(build_loadable_extension_directory NAME OUTPUT_DIRECTORY EXTENSION_VERS TARGET ${TARGET_NAME} POST_BUILD COMMAND - printf "${DUCKDB_NORMALIZED_VERSION}" > duckdb_version_file - COMMAND - printf "${EXTENSION_VERSION}" > extension_version_file || ( echo "CMake appears not to propagate correctly version information for the extension, using empty string" && rm extension_version_file && touch extension_version_file ) - COMMAND - bash ${CMAKE_SOURCE_DIR}/scripts/append_metadata.sh $ ${CMAKE_BINARY_DIR}/duckdb_platform_out duckdb_version_file extension_version_file + ${CMAKE_COMMAND} -DEXTENSION=$ -DPLATFORM_FILE=${CMAKE_BINARY_DIR}/duckdb_platform_out -DDUCKDB_VERSION="${DUCKDB_NORMALIZED_VERSION}" -DEXTENSION_VERSION="${EXTENSION_VERSION}" -DNULL_FILE=${CMAKE_SOURCE_DIR}/scripts/null.txt -P ${CMAKE_SOURCE_DIR}/scripts/append_metadata.cmake ) add_dependencies(${TARGET_NAME} duckdb_platform) if (NOT EXTENSION_CONFIG_BUILD AND NOT ${EXTENSION_TESTS_ONLY} AND NOT CLANG_TIDY) @@ -1228,8 +1224,8 @@ if(NOT DUCKDB_EXPLICIT_PLATFORM) else() add_custom_target( duckdb_platform ALL - COMMAND - printf "${DUCKDB_EXPLICIT_PLATFORM}" > duckdb_platform_out + COMMAND + ${CMAKE_COMMAND} -E echo_append \"${DUCKDB_EXPLICIT_PLATFORM}\" > duckdb_platform_out ) endif() diff --git a/scripts/append_metadata.cmake b/scripts/append_metadata.cmake new file mode 100644 index 000000000000..557db545d901 --- /dev/null +++ b/scripts/append_metadata.cmake @@ -0,0 +1,68 @@ +cmake_minimum_required(VERSION 3.5) + +# Usage: cmake -DEXTENSION=path/to/extension.duckdb_extension -DPLATFORM_FILE=README.md -DDUCKDB_VERSION=tag1 -DEXTENSION_VERSION=tag2 -P scripts/append_metadata.cmake +# Currently hardcoded to host up to 8 fields +# Example: ./scripts/append_metadata.sh file.duckdb_extension git_hash_duckdb_file git_hash_extension_file platfrom_file + +set(EXTENSION "" CACHE PATH "Path to the extension where to add metadata") +set(NULL_FILE "" CACHE PATH "Path to file containing a single 0 byte") +set(META1 "4" CACHE STRING "Metadata field" FORCE) +set(PLATFORM_FILE "" CACHE PATH "Metadata field: path of file containing duckdb_platform") +set(DUCKDB_VERSION "" CACHE STRING "Metadata field: path of file containing duckdb_version") +set(EXTENSION_VERSION "" CACHE STRING "Metadata field: path of file containing extension_version") +set(META5 "" CACHE STRING "Metadata field") +set(META6 "" CACHE STRING "Metadata field") +set(META7 "" CACHE STRING "Metadata field") +set(META8 "" CACHE STRING "Metadata field") + +# null.txt should contain exactly 1 byte of value \x00 +file(READ "${NULL_FILE}" EMPTY_BYTE) + +string(REPEAT "${EMPTY_BYTE}" 32 EMPTY_32) +string(REPEAT "${EMPTY_BYTE}" 256 EMPTY_256) + +# 0 for custom section +string(APPEND CUSTOM_SECTION "${EMPTY_BYTE}") +# 213 in hex = 531 in decimal, total lenght of what follows (1 + 16 + 2 + 8x32 + 256) +# [1(continuation) + 0010011(payload) = \x93 -> 147, 0(continuation) + 10(payload) = \x04 -> 4] +# 10 in hex = 16 in decimal, lenght of name, 1 byte +string(ASCII 147 4 16 CUSTOM_SECTION_2) +string(APPEND CUSTOM_SECTION "${CUSTOM_SECTION_2}") + +# the name of the WebAssembly custom section, 16 bytes +string(APPEND CUSTOM_SECTION "duckdb_signature") + +# 1000 in hex, 512 in decimal +# [1(continuation) + 0000000(payload) = -> 128, 0(continuation) + 100(payload) -> 4], +# for a grand total of 2 bytes +string(ASCII 128 4 CUSTOM_SECTION_3) +string(APPEND CUSTOM_SECTION "${CUSTOM_SECTION_3}") + +# Second metadata-field is special, since content comes from a file +file(READ "${PLATFORM_FILE}" META2) + +# Build METADATAx variable by appending and then truncating empty strings +string(SUBSTRING "${META1}${EMPTY_32}" 0 32 METADATA1) +string(SUBSTRING "${META2}${EMPTY_32}" 0 32 METADATA2) +string(SUBSTRING "${DUCKDB_VERSION}${EMPTY_32}" 0 32 METADATA3) +string(SUBSTRING "${EXTENSION_VERSION}${EMPTY_32}" 0 32 METADATA4) +string(SUBSTRING "${META5}${EMPTY_32}" 0 32 METADATA5) +string(SUBSTRING "${META6}${EMPTY_32}" 0 32 METADATA6) +string(SUBSTRING "${META7}${EMPTY_32}" 0 32 METADATA7) +string(SUBSTRING "${META8}${EMPTY_32}" 0 32 METADATA8) + +# Append metadata fields, backwards +string(APPEND CUSTOM_SECTION "${METADATA8}") +string(APPEND CUSTOM_SECTION "${METADATA7}") +string(APPEND CUSTOM_SECTION "${METADATA6}") +string(APPEND CUSTOM_SECTION "${METADATA5}") +string(APPEND CUSTOM_SECTION "${METADATA4}") +string(APPEND CUSTOM_SECTION "${METADATA3}") +string(APPEND CUSTOM_SECTION "${METADATA2}") +string(APPEND CUSTOM_SECTION "${METADATA1}") + +# Append signature (yet to be computed) +string(APPEND CUSTOM_SECTION "${EMPTY_256}") + +# Append generated custom section to the extension +file(APPEND "${EXTENSION}" "${CUSTOM_SECTION}") diff --git a/scripts/append_metadata.sh b/scripts/append_metadata.sh deleted file mode 100755 index 774872259ba4..000000000000 --- a/scripts/append_metadata.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -# Usage: ./script/append_metadata.sh -# Currently hardcoded to host up to 8 fields -# Example: ./scripts/append_metadata.sh file.duckdb_extension git_hash_duckdb_file git_hash_extension_file platfrom_file - -if (($# >= 9)); then - echo "Too many parameters provided, current script can handle at maxium 8 fields" - exit 1 -fi - -# 0 for custom section -# 213 in hex = 531 in decimal, total lenght of what follows (1 + 16 + 2 + 8x32 + 256) -# [1(continuation) + 0010011(payload) = \x93, 0(continuation) + 10(payload) = \x04] -echo -n -e '\x00' >> "$1" -echo -n -e '\x93\x04' >> "$1" -# 10 in hex = 16 in decimal, lenght of name, 1 byte -echo -n -e '\x10' >> "$1" -echo -n -e 'duckdb_signature' >> "$1" -# the name of the WebAssembly custom section, 16 bytes -# 1000 in hex, 512 in decimal -# [1(continuation) + 0000000(payload) = ff, 0(continuation) + 100(payload)], -# for a grand total of 2 bytes -echo -n -e '\x80\x04' >> "$1" - -dd if=/dev/zero of="$1.empty_32" bs=32 count=1 &> /dev/null -dd if=/dev/zero of="$1.empty_256" bs=256 count=1 &> /dev/null - -# Write empty fields -for ((i=$#; i<8; i++)) -do - cat "$1.empty_32" >> "$1" -done - -# Write provided fields (backwards) -for ((i=$#; i>=2; i--)) -do - cat "${!i}" > "$1.add" - cat "$1.empty_32" >> "$1.add" - dd if="$1.add" of="$1.add_trunc" bs=32 count=1 &> /dev/null - cat "$1.add_trunc" >> "$1" -done - -# Write how many fields have been written during previous step - echo -n "$#" > "$1.add" - cat "$1.empty_32" >> "$1.add" - dd if="$1.add" of="$1.add_trunc" bs=32 count=1 &> /dev/null - cat "$1.add_trunc" >> "$1" - -cat "$1.empty_256" >> "$1" - -rm -f $1.* diff --git a/scripts/null.txt b/scripts/null.txt new file mode 100644 index 0000000000000000000000000000000000000000..f76dd238ade08917e6712764a16a22005a50573d GIT binary patch literal 1 IcmZPo000310RR91 literal 0 HcmV?d00001 From 671659a8166f56a827250222c7ef8ae5aabb81dd Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Tue, 9 Apr 2024 14:21:09 +0200 Subject: [PATCH 257/603] add a test case as well --- .../duckdb/optimizer/filter_pushdown.hpp | 4 +- src/optimizer/filter_pushdown.cpp | 5 ++- src/optimizer/pushdown/pushdown_aggregate.cpp | 2 +- .../pushdown/pushdown_cross_product.cpp | 2 +- src/optimizer/pushdown/pushdown_left_join.cpp | 2 +- src/optimizer/pushdown/pushdown_mark_join.cpp | 6 +-- .../pushdown/pushdown_projection.cpp | 2 +- .../pushdown/pushdown_semi_anti_join.cpp | 2 +- .../pushdown/pushdown_set_operation.cpp | 2 +- .../pushdown/pushdown_single_join.cpp | 2 +- .../statistics/operator/propagate_join.cpp | 4 +- .../another-column-binding-issue.test | 18 ++++++++ .../pushdown/pushdown_after_statistics.test | 42 +++++++++++++++++++ 13 files changed, 78 insertions(+), 15 deletions(-) create mode 100644 test/optimizer/another-column-binding-issue.test create mode 100644 test/optimizer/pushdown/pushdown_after_statistics.test diff --git a/src/include/duckdb/optimizer/filter_pushdown.hpp b/src/include/duckdb/optimizer/filter_pushdown.hpp index 70ba4fdf3a31..9b40413606d7 100644 --- a/src/include/duckdb/optimizer/filter_pushdown.hpp +++ b/src/include/duckdb/optimizer/filter_pushdown.hpp @@ -18,7 +18,7 @@ class Optimizer; class FilterPushdown { public: - explicit FilterPushdown(Optimizer &optimizer); + explicit FilterPushdown(Optimizer &optimizer, bool convert_mark_joins = true); //! Perform filter pushdown unique_ptr Rewrite(unique_ptr op); @@ -40,7 +40,7 @@ class FilterPushdown { private: vector> filters; Optimizer &optimizer; - + bool convert_mark_joins; //! Push down a LogicalAggregate op unique_ptr PushdownAggregate(unique_ptr op); //! Push down a distinct operator diff --git a/src/optimizer/filter_pushdown.cpp b/src/optimizer/filter_pushdown.cpp index 35d756a94c75..856b031f1958 100644 --- a/src/optimizer/filter_pushdown.cpp +++ b/src/optimizer/filter_pushdown.cpp @@ -9,7 +9,8 @@ namespace duckdb { using Filter = FilterPushdown::Filter; -FilterPushdown::FilterPushdown(Optimizer &optimizer) : optimizer(optimizer), combiner(optimizer.context) { +FilterPushdown::FilterPushdown(Optimizer &optimizer, bool convert_mark_joins) + : optimizer(optimizer), combiner(optimizer.context), convert_mark_joins(convert_mark_joins) { } unique_ptr FilterPushdown::Rewrite(unique_ptr op) { @@ -144,7 +145,7 @@ unique_ptr FilterPushdown::PushFinalFilters(unique_ptr FilterPushdown::FinishPushdown(unique_ptr op) { // unhandled type, first perform filter pushdown in its children for (auto &child : op->children) { - FilterPushdown pushdown(optimizer); + FilterPushdown pushdown(optimizer, convert_mark_joins); child = pushdown.Rewrite(std::move(child)); } // now push any existing filters diff --git a/src/optimizer/pushdown/pushdown_aggregate.cpp b/src/optimizer/pushdown/pushdown_aggregate.cpp index 396980d54361..26342835914d 100644 --- a/src/optimizer/pushdown/pushdown_aggregate.cpp +++ b/src/optimizer/pushdown/pushdown_aggregate.cpp @@ -37,7 +37,7 @@ unique_ptr FilterPushdown::PushdownAggregate(unique_ptr FilterPushdown::PushdownCrossProduct(unique_ptr op) { D_ASSERT(op->children.size() > 1); - FilterPushdown left_pushdown(optimizer), right_pushdown(optimizer); + FilterPushdown left_pushdown(optimizer, convert_mark_joins), right_pushdown(optimizer, convert_mark_joins); vector> join_expressions; auto join_ref_type = JoinRefType::REGULAR; switch (op->type) { diff --git a/src/optimizer/pushdown/pushdown_left_join.cpp b/src/optimizer/pushdown/pushdown_left_join.cpp index 47cfdfd6b639..3c43fc67d5e8 100644 --- a/src/optimizer/pushdown/pushdown_left_join.cpp +++ b/src/optimizer/pushdown/pushdown_left_join.cpp @@ -64,7 +64,7 @@ unique_ptr FilterPushdown::PushdownLeftJoin(unique_ptrtype == LogicalOperatorType::LOGICAL_DELIM_JOIN) { return FinishPushdown(std::move(op)); } - FilterPushdown left_pushdown(optimizer), right_pushdown(optimizer); + FilterPushdown left_pushdown(optimizer, convert_mark_joins), right_pushdown(optimizer, convert_mark_joins); // for a comparison join we create a FilterCombiner that checks if we can push conditions on LHS join conditions // into the RHS of the join FilterCombiner filter_combiner(optimizer); diff --git a/src/optimizer/pushdown/pushdown_mark_join.cpp b/src/optimizer/pushdown/pushdown_mark_join.cpp index b336cd3eed97..6c45a3e921a6 100644 --- a/src/optimizer/pushdown/pushdown_mark_join.cpp +++ b/src/optimizer/pushdown/pushdown_mark_join.cpp @@ -16,7 +16,7 @@ unique_ptr FilterPushdown::PushdownMarkJoin(unique_ptrtype == LogicalOperatorType::LOGICAL_DELIM_JOIN || op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN); right_bindings.insert(comp_join.mark_index); - FilterPushdown left_pushdown(optimizer), right_pushdown(optimizer); + FilterPushdown left_pushdown(optimizer, convert_mark_joins), right_pushdown(optimizer, convert_mark_joins); #ifdef DEBUG bool simplified_mark_join = false; #endif @@ -35,7 +35,7 @@ unique_ptr FilterPushdown::PushdownMarkJoin(unique_ptrfilter->type == ExpressionType::BOUND_COLUMN_REF) { + if (filters[i]->filter->type == ExpressionType::BOUND_COLUMN_REF && convert_mark_joins) { // filter just references the marker: turn into semi join #ifdef DEBUG simplified_mark_join = true; @@ -61,7 +61,7 @@ unique_ptr FilterPushdown::PushdownMarkJoin(unique_ptr FilterPushdown::PushdownProjection(unique_ptr> remain_expressions; diff --git a/src/optimizer/pushdown/pushdown_semi_anti_join.cpp b/src/optimizer/pushdown/pushdown_semi_anti_join.cpp index c9506fe05d68..7d240e3f6f15 100644 --- a/src/optimizer/pushdown/pushdown_semi_anti_join.cpp +++ b/src/optimizer/pushdown/pushdown_semi_anti_join.cpp @@ -17,7 +17,7 @@ unique_ptr FilterPushdown::PushdownSemiAntiJoin(unique_ptrchildren[0] = Rewrite(std::move(op->children[0])); - FilterPushdown right_pushdown(optimizer); + FilterPushdown right_pushdown(optimizer, convert_mark_joins); op->children[1] = right_pushdown.Rewrite(std::move(op->children[1])); bool left_empty = op->children[0]->type == LogicalOperatorType::LOGICAL_EMPTY_RESULT; diff --git a/src/optimizer/pushdown/pushdown_set_operation.cpp b/src/optimizer/pushdown/pushdown_set_operation.cpp index 315fe5dc0c03..5d3dce97e3a3 100644 --- a/src/optimizer/pushdown/pushdown_set_operation.cpp +++ b/src/optimizer/pushdown/pushdown_set_operation.cpp @@ -40,7 +40,7 @@ unique_ptr FilterPushdown::PushdownSetOperation(unique_ptr(); diff --git a/src/optimizer/pushdown/pushdown_single_join.cpp b/src/optimizer/pushdown/pushdown_single_join.cpp index 028dc092646b..089060143d30 100644 --- a/src/optimizer/pushdown/pushdown_single_join.cpp +++ b/src/optimizer/pushdown/pushdown_single_join.cpp @@ -9,7 +9,7 @@ unique_ptr FilterPushdown::PushdownSingleJoin(unique_ptr &left_bindings, unordered_set &right_bindings) { D_ASSERT(op->Cast().join_type == JoinType::SINGLE); - FilterPushdown left_pushdown(optimizer), right_pushdown(optimizer); + FilterPushdown left_pushdown(optimizer, convert_mark_joins), right_pushdown(optimizer, convert_mark_joins); // now check the set of filters for (idx_t i = 0; i < filters.size(); i++) { auto side = JoinSide::GetJoinSide(filters[i]->bindings, left_bindings, right_bindings); diff --git a/src/optimizer/statistics/operator/propagate_join.cpp b/src/optimizer/statistics/operator/propagate_join.cpp index 16541cf65368..2cb99419db1d 100644 --- a/src/optimizer/statistics/operator/propagate_join.cpp +++ b/src/optimizer/statistics/operator/propagate_join.cpp @@ -357,7 +357,9 @@ void StatisticsPropagator::CreateFilterFromJoinStats(unique_ptr child->expressions.emplace_back(std::move(filter_expr)); } - FilterPushdown filter_pushdown(optimizer); + // not allowed to let filter pushdowwn change mark joins to semi joins. + // semi joins are potentially slower AND the conversion can ruin column binding information + FilterPushdown filter_pushdown(optimizer, false); child = filter_pushdown.Rewrite(std::move(child)); PropagateExpression(expr); } diff --git a/test/optimizer/another-column-binding-issue.test b/test/optimizer/another-column-binding-issue.test new file mode 100644 index 000000000000..a1e181877b29 --- /dev/null +++ b/test/optimizer/another-column-binding-issue.test @@ -0,0 +1,18 @@ +# name: test/optimizer/another-column-binding-issue.test +# description: Arithmetic test +# group: [optimizer] + +statement ok +attach 'bug.db' as bug; + +statement ok +use bug; + +statement ok +SELECT t1.Transaction_ID +FROM transactions t1 +WHERE t1.Transaction_ID IN + (SELECT t2.Referred_Transaction_ID + FROM transactions t2 + WHERE t2.Transaction_ID IN (123606, 123602, 131522, 123604, 131470) + AND t2.Transaction_ID NOT IN (SELECT t2_filter.Transaction_ID FROM transactions t2_filter)) diff --git a/test/optimizer/pushdown/pushdown_after_statistics.test b/test/optimizer/pushdown/pushdown_after_statistics.test new file mode 100644 index 000000000000..85d1e361eaa2 --- /dev/null +++ b/test/optimizer/pushdown/pushdown_after_statistics.test @@ -0,0 +1,42 @@ +# name: test/optimizer/pushdown/pushdown_after_statistics.test +# description: Test Table Filter Push Down +# group: [pushdown] + +statement ok +set explain_output='optimized_only'; + + +statement ok +create table big_probe as select range%3000 a, range%4000 b from range(100000); + +statement ok +create table into_semi as select range%300 c from range(10000); + +statement ok +create table into_get as select range d from range(100); + + +# the IN filter becomes a mark join. We should keep it a mark join at this point +query II +explain select * from big_probe, into_semi, into_get where c in (1, 3, 5, 7, 10, 14, 16, 20, 22) and c = d and a = c; +---- +logical_opt :.*MARK.* + + +statement ok +create table mark_join_build as select range e from range(200); + +# Now the in filter is a semi join. +query II +explain select * from big_probe, into_semi, into_get where c in (select e from mark_join_build) and c = d and a = c; +---- +logical_opt :.*SEMI.* + + +statement ok +select t1.a from big_probe t1 +where t1.a in + (select t2.b + from big_probe t2 + where t2.b in (1206, 1202, 1322, 1204, 1370) + and t2.b not in (select t2_filter.a from big_probe t2_filter)); From f72498717469088b339efcc29878b6fb2179b8c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Tue, 9 Apr 2024 14:25:58 +0200 Subject: [PATCH 258/603] list_extract --- src/function/scalar/list/list_extract.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/function/scalar/list/list_extract.cpp b/src/function/scalar/list/list_extract.cpp index 822b9079cdc9..f70f12c1629e 100644 --- a/src/function/scalar/list/list_extract.cpp +++ b/src/function/scalar/list/list_extract.cpp @@ -62,13 +62,13 @@ void ListExtractTemplate(idx_t count, UnifiedVectorFormat &list_data, UnifiedVec result_mask.SetInvalid(i); continue; } - child_offset = list_entry.offset + list_entry.length + offsets_entry; + child_offset = UnsafeNumericCast(UnsafeNumericCast(list_entry.offset + list_entry.length) + offsets_entry); } else { if ((idx_t)offsets_entry >= list_entry.length) { result_mask.SetInvalid(i); continue; } - child_offset = list_entry.offset + offsets_entry; + child_offset = UnsafeNumericCast(UnsafeNumericCast(list_entry.offset) + offsets_entry); } auto child_index = child_format.sel->get_index(child_offset); if (child_format.validity.RowIsValid(child_index)) { From 7dbe531b4522aea27b15e605cf4abeca0a17beb7 Mon Sep 17 00:00:00 2001 From: Gabor Szarnyas Date: Tue, 9 Apr 2024 14:00:16 +0200 Subject: [PATCH 259/603] Update issue template --- .github/ISSUE_TEMPLATE/bug_report.yml | 35 ++++++++++++++++++++------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 1ead7f8f1b27..79962938728c 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -89,25 +89,42 @@ body: value: | If the above is not given and is not obvious from your GitHub profile page, we might close your issue without further review. Please refer to the [reasoning behind this rule](https://berthub.eu/articles/posts/anonymous-help/) if you have questions. - # Before Submitting + # Before Submitting: - type: dropdown attributes: - label: Have you tried this on the latest [nightly build](https://duckdb.org/docs/installation/?version=main)? + label: What is the latest build you tested with? If possible, we recommend testing with the latest nightly build. description: | - * **Python**: `pip install duckdb --upgrade --pre` - * **R**: `install.packages('duckdb', repos=c('https://duckdb.r-universe.dev', 'https://cloud.r-project.org'))` - * **Other Platforms**: You can find links to binaries [here](https://duckdb.org/docs/installation/) or compile from source. + Visit the [installation page](https://duckdb.org/docs/installation/) for instructions. options: - - I have tested with a nightly build - - I have tested with a release build (and could not test with a nightly build) - I have not tested with any build + - I have tested with a stable release + - I have tested with a nightly build + - I have tested with a source build validations: required: true + - type: dropdown + attributes: + label: Did you include all relevant data sets for reproducing the issue? + options: + - "No - Other reason (please specify in the issue body)" + - "No - I cannot share the data sets because they are confidential" + - "No - I cannot easily share my data sets due to their large size" + - "Not applicable - the reproduction does not require a data set" + - "Yes" + default: 0 + validations: + required: true + + - type: checkboxes + attributes: + label: Did you include all code required to reproduce the issue? + options: + - label: Yes, I have + - type: checkboxes attributes: - label: Have you tried the steps to reproduce? Do they include all relevant data and configuration? Does the issue you report still appear there? + label: Did you include all relevant configuration (e.g., CPU architecture, Python version, Linux distribution) to reproduce the issue? options: - label: Yes, I have - required: true From a2bee34cdfcb6935662024ae2cecb4e45ceba598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Tue, 9 Apr 2024 15:00:54 +0200 Subject: [PATCH 260/603] list_extract --- src/function/scalar/list/list_extract.cpp | 3 ++- src/function/scalar/list/list_select.cpp | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/function/scalar/list/list_extract.cpp b/src/function/scalar/list/list_extract.cpp index f70f12c1629e..aa8278ce63c6 100644 --- a/src/function/scalar/list/list_extract.cpp +++ b/src/function/scalar/list/list_extract.cpp @@ -62,7 +62,8 @@ void ListExtractTemplate(idx_t count, UnifiedVectorFormat &list_data, UnifiedVec result_mask.SetInvalid(i); continue; } - child_offset = UnsafeNumericCast(UnsafeNumericCast(list_entry.offset + list_entry.length) + offsets_entry); + child_offset = UnsafeNumericCast(UnsafeNumericCast(list_entry.offset + list_entry.length) + + offsets_entry); } else { if ((idx_t)offsets_entry >= list_entry.length) { result_mask.SetInvalid(i); diff --git a/src/function/scalar/list/list_select.cpp b/src/function/scalar/list/list_select.cpp index 70ae15194fcf..bf77d9e9dcc0 100644 --- a/src/function/scalar/list/list_select.cpp +++ b/src/function/scalar/list/list_select.cpp @@ -12,9 +12,10 @@ struct SetSelectionVectorSelect { idx_t &target_offset, idx_t selection_offset, idx_t input_offset, idx_t target_length) { auto sel_idx = selection_entry.GetValue(selection_offset + child_idx).GetValue() - 1; - if (sel_idx < target_length) { - selection_vector.set_index(target_offset, input_offset + sel_idx); - if (!input_validity.RowIsValid(input_offset + sel_idx)) { + if (sel_idx >= 0 && sel_idx < UnsafeNumericCast(target_length)) { + auto sel_idx_unsigned = UnsafeNumericCast(sel_idx); + selection_vector.set_index(target_offset, input_offset + sel_idx_unsigned); + if (!input_validity.RowIsValid(input_offset + sel_idx_unsigned)) { validity_mask.SetInvalid(target_offset); } } else { From b1c1a6ffe0e2b402f7f164ced2c294924cae8fad Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Tue, 9 Apr 2024 15:01:11 +0200 Subject: [PATCH 261/603] fix header files --- src/include/duckdb/optimizer/filter_pushdown.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/include/duckdb/optimizer/filter_pushdown.hpp b/src/include/duckdb/optimizer/filter_pushdown.hpp index 9b40413606d7..c8c87fd5b92e 100644 --- a/src/include/duckdb/optimizer/filter_pushdown.hpp +++ b/src/include/duckdb/optimizer/filter_pushdown.hpp @@ -38,9 +38,11 @@ class FilterPushdown { }; private: - vector> filters; Optimizer &optimizer; + FilterCombiner combiner; bool convert_mark_joins; + + vector> filters; //! Push down a LogicalAggregate op unique_ptr PushdownAggregate(unique_ptr op); //! Push down a distinct operator @@ -90,8 +92,6 @@ class FilterPushdown { void GenerateFilters(); //! if there are filters in this FilterPushdown node, push them into the combiner void PushFilters(); - - FilterCombiner combiner; }; } // namespace duckdb From 36e8260677a9eb7e5b4440a139454f0c75cfb638 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Tue, 9 Apr 2024 15:20:01 +0200 Subject: [PATCH 262/603] cast_helpers my other nemesis --- src/common/operator/string_cast.cpp | 16 ++++++++-------- .../duckdb/common/types/cast_helpers.hpp | 17 ++++++++++------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/common/operator/string_cast.cpp b/src/common/operator/string_cast.cpp index 0fd1f5f0a0ca..be3d604910bf 100644 --- a/src/common/operator/string_cast.cpp +++ b/src/common/operator/string_cast.cpp @@ -24,37 +24,37 @@ string_t StringCast::Operation(bool input, Vector &vector) { template <> string_t StringCast::Operation(int8_t input, Vector &vector) { - return NumericHelper::FormatSigned(input, vector); + return NumericHelper::FormatSigned(input, vector); } template <> string_t StringCast::Operation(int16_t input, Vector &vector) { - return NumericHelper::FormatSigned(input, vector); + return NumericHelper::FormatSigned(input, vector); } template <> string_t StringCast::Operation(int32_t input, Vector &vector) { - return NumericHelper::FormatSigned(input, vector); + return NumericHelper::FormatSigned(input, vector); } template <> string_t StringCast::Operation(int64_t input, Vector &vector) { - return NumericHelper::FormatSigned(input, vector); + return NumericHelper::FormatSigned(input, vector); } template <> duckdb::string_t StringCast::Operation(uint8_t input, Vector &vector) { - return NumericHelper::FormatSigned(input, vector); + return NumericHelper::FormatSigned(input, vector); } template <> duckdb::string_t StringCast::Operation(uint16_t input, Vector &vector) { - return NumericHelper::FormatSigned(input, vector); + return NumericHelper::FormatSigned(input, vector); } template <> duckdb::string_t StringCast::Operation(uint32_t input, Vector &vector) { - return NumericHelper::FormatSigned(input, vector); + return NumericHelper::FormatSigned(input, vector); } template <> duckdb::string_t StringCast::Operation(uint64_t input, Vector &vector) { - return NumericHelper::FormatSigned(input, vector); + return NumericHelper::FormatSigned(input, vector); } template <> diff --git a/src/include/duckdb/common/types/cast_helpers.hpp b/src/include/duckdb/common/types/cast_helpers.hpp index 358438da3c17..7dd91f3a5f8a 100644 --- a/src/include/duckdb/common/types/cast_helpers.hpp +++ b/src/include/duckdb/common/types/cast_helpers.hpp @@ -59,16 +59,19 @@ class NumericHelper { return ptr; } - template - static string_t FormatSigned(SIGNED value, Vector &vector) { - int sign = -(value < 0); - UNSIGNED unsigned_value = UnsafeNumericCast(UNSIGNED(value ^ sign) - sign); - int length = UnsignedLength(unsigned_value) - sign; - string_t result = StringVector::EmptyString(vector, NumericCast(length)); + template + static string_t FormatSigned(T value, Vector &vector) { + auto is_negative = (value < 0); + auto unsigned_value = static_cast::type>(AbsValue(value)); + auto length = UnsignedLength(unsigned_value); + if (is_negative) { + length++; + } + auto result = StringVector::EmptyString(vector, UnsafeNumericCast(length)); auto dataptr = result.GetDataWriteable(); auto endptr = dataptr + length; endptr = FormatUnsigned(unsigned_value, endptr); - if (sign) { + if (is_negative) { *--endptr = '-'; } result.Finalize(); From 55a0e5500baa53a138a78a03f3365b4f0c995334 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Tue, 9 Apr 2024 14:29:41 +0200 Subject: [PATCH 263/603] Adapt to new interface for get_setting_function_t --- src/include/duckdb/main/settings.hpp | 2 +- src/main/settings/settings.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/include/duckdb/main/settings.hpp b/src/include/duckdb/main/settings.hpp index f49e8c2314f8..131dcf78284f 100644 --- a/src/include/duckdb/main/settings.hpp +++ b/src/include/duckdb/main/settings.hpp @@ -231,7 +231,7 @@ struct AllowExtensionsMetadataMismatchSetting { static constexpr const LogicalTypeId InputType = LogicalTypeId::BOOLEAN; static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value ¶meter); static void ResetGlobal(DatabaseInstance *db, DBConfig &config); - static Value GetSetting(ClientContext &context); + static Value GetSetting(const ClientContext &context); }; struct AllowUnredactedSecretsSetting { diff --git a/src/main/settings/settings.cpp b/src/main/settings/settings.cpp index 13c9c74d7438..fbf990fd4392 100644 --- a/src/main/settings/settings.cpp +++ b/src/main/settings/settings.cpp @@ -488,7 +488,7 @@ void AllowExtensionsMetadataMismatchSetting::ResetGlobal(DatabaseInstance *db, D config.options.allow_extensions_metadata_mismatch = DBConfig().options.allow_extensions_metadata_mismatch; } -Value AllowExtensionsMetadataMismatchSetting::GetSetting(ClientContext &context) { +Value AllowExtensionsMetadataMismatchSetting::GetSetting(const ClientContext &context) { auto &config = DBConfig::GetConfig(context); return Value::BOOLEAN(config.options.allow_extensions_metadata_mismatch); } From 1f81f01215620c284aed6f65800f34ef09708db5 Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Tue, 9 Apr 2024 16:54:39 +0200 Subject: [PATCH 264/603] remove unecessary test file --- .../another-column-binding-issue.test | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 test/optimizer/another-column-binding-issue.test diff --git a/test/optimizer/another-column-binding-issue.test b/test/optimizer/another-column-binding-issue.test deleted file mode 100644 index a1e181877b29..000000000000 --- a/test/optimizer/another-column-binding-issue.test +++ /dev/null @@ -1,18 +0,0 @@ -# name: test/optimizer/another-column-binding-issue.test -# description: Arithmetic test -# group: [optimizer] - -statement ok -attach 'bug.db' as bug; - -statement ok -use bug; - -statement ok -SELECT t1.Transaction_ID -FROM transactions t1 -WHERE t1.Transaction_ID IN - (SELECT t2.Referred_Transaction_ID - FROM transactions t2 - WHERE t2.Transaction_ID IN (123606, 123602, 131522, 123604, 131470) - AND t2.Transaction_ID NOT IN (SELECT t2_filter.Transaction_ID FROM transactions t2_filter)) From 9f6a64a953156185669285fc0a54482482164dad Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 9 Apr 2024 17:58:39 +0200 Subject: [PATCH 265/603] fix compilation issues --- src/include/duckdb/common/shared_ptr.ipp | 10 +--------- src/include/duckdb/common/weak_ptr.ipp | 4 ++-- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/src/include/duckdb/common/shared_ptr.ipp b/src/include/duckdb/common/shared_ptr.ipp index e5c96f6af02e..0be8ab5bdfe0 100644 --- a/src/include/duckdb/common/shared_ptr.ipp +++ b/src/include/duckdb/common/shared_ptr.ipp @@ -90,14 +90,6 @@ public: explicit shared_ptr(weak_ptr other) : internal(other.internal) { } - // Construct from auto_ptr -#if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR) - template ::value, int>::type = 0> - shared_ptr(std::auto_ptr &&__r) : internal(__r.release()) { - __enable_weak_this(internal.get(), internal.get()); - } -#endif - // Construct from unique_ptr, takes over ownership of the unique_ptr template ::value && @@ -185,7 +177,7 @@ public: return internal.operator bool(); } - std::__add_lvalue_reference_t operator*() const { + typename std::add_lvalue_reference::type operator*() const { if (MemorySafety::ENABLED) { const auto ptr = internal.get(); AssertNotNull(!ptr); diff --git a/src/include/duckdb/common/weak_ptr.ipp b/src/include/duckdb/common/weak_ptr.ipp index f602aacb4830..dc8115b33717 100644 --- a/src/include/duckdb/common/weak_ptr.ipp +++ b/src/include/duckdb/common/weak_ptr.ipp @@ -44,9 +44,9 @@ public: return *this; } - template ::value, int> = 0> + template ::value, int>::type = 0> weak_ptr &operator=(const shared_ptr &ptr) { - internal = ptr; + internal = ptr.internal; return *this; } From 6b8c631a6b5ac4699bf56f5366b167724facbe4b Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 9 Apr 2024 17:59:13 +0200 Subject: [PATCH 266/603] test setting the maximum swap space to unlimited --- .../max_swap_space_unlimited.test | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 test/sql/storage/temp_directory/max_swap_space_unlimited.test diff --git a/test/sql/storage/temp_directory/max_swap_space_unlimited.test b/test/sql/storage/temp_directory/max_swap_space_unlimited.test new file mode 100644 index 000000000000..86dfdd7f4570 --- /dev/null +++ b/test/sql/storage/temp_directory/max_swap_space_unlimited.test @@ -0,0 +1,18 @@ +# name: test/sql/storage/temp_directory/max_swap_space_unlimited.test +# group: [temp_directory] + +require skip_reload + +require noforcestorage + +# Set a temp_directory to offload data +statement ok +set temp_directory='__TEST_DIR__/max_swap_space_reached' + +statement ok +PRAGMA max_temp_directory_size='-1'; + +query I +select current_setting('max_temp_directory_size') +---- +16383.9 PiB From 437f8de28428234f864571d4db66b93d103a9ba9 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Tue, 9 Apr 2024 22:05:46 +0200 Subject: [PATCH 267/603] Respect read-only mode in dbgen and dsdgen --- extension/tpcds/tpcds_extension.cpp | 10 +++++-- extension/tpch/tpch_extension.cpp | 12 +++++--- .../duckdb/function/table_function.hpp | 5 ++-- .../binder/tableref/bind_table_function.cpp | 2 +- src/planner/operator/logical_get.cpp | 2 +- test/sql/tpcds/dsdgen_readonly.test | 30 +++++++++++++++++++ test/sql/tpch/dbgen_readonly.test | 30 +++++++++++++++++++ 7 files changed, 80 insertions(+), 11 deletions(-) create mode 100644 test/sql/tpcds/dsdgen_readonly.test create mode 100644 test/sql/tpch/dbgen_readonly.test diff --git a/extension/tpcds/tpcds_extension.cpp b/extension/tpcds/tpcds_extension.cpp index c6157b67ae93..44c92d91333c 100644 --- a/extension/tpcds/tpcds_extension.cpp +++ b/extension/tpcds/tpcds_extension.cpp @@ -43,13 +43,17 @@ static duckdb::unique_ptr DsdgenBind(ClientContext &context, Table result->keys = kv.second.GetValue(); } } + if (input.binder) { + auto &catalog = Catalog::GetCatalog(context, result->catalog); + input.binder->properties.modified_databases.insert(catalog.GetName()); + } return_types.emplace_back(LogicalType::BOOLEAN); names.emplace_back("Success"); return std::move(result); } static void DsdgenFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &data = (DSDGenFunctionData &)*data_p.bind_data; + auto &data = data_p.bind_data->CastNoConst(); if (data.finished) { return; } @@ -82,7 +86,7 @@ static duckdb::unique_ptr TPCDSQueryBind(ClientContext &context, T } static void TPCDSQueryFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &data = (TPCDSData &)*data_p.global_state; + auto &data = data_p.global_state->Cast(); idx_t tpcds_queries = tpcds::DSDGenWrapper::QueriesCount(); if (data.offset >= tpcds_queries) { // finished returning values @@ -116,7 +120,7 @@ static duckdb::unique_ptr TPCDSQueryAnswerBind(ClientContext &cont } static void TPCDSQueryAnswerFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &data = (TPCDSData &)*data_p.global_state; + auto &data = data_p.global_state->Cast(); idx_t tpcds_queries = tpcds::DSDGenWrapper::QueriesCount(); vector scale_factors {1, 10}; idx_t total_answers = tpcds_queries * scale_factors.size(); diff --git a/extension/tpch/tpch_extension.cpp b/extension/tpch/tpch_extension.cpp index 68548438ab20..e1d6016b0681 100644 --- a/extension/tpch/tpch_extension.cpp +++ b/extension/tpch/tpch_extension.cpp @@ -8,6 +8,7 @@ #include "duckdb/parser/parser.hpp" #include "duckdb/parser/statement/select_statement.hpp" #include "duckdb/main/extension_util.hpp" +#include "duckdb/transaction/transaction.hpp" #endif #include "dbgen/dbgen.hpp" @@ -51,14 +52,17 @@ static duckdb::unique_ptr DbgenBind(ClientContext &context, TableF if (result->children != 1 && result->step == -1) { throw InvalidInputException("Step must be defined when children are defined"); } - + if (input.binder) { + auto &catalog = Catalog::GetCatalog(context, result->catalog); + input.binder->properties.modified_databases.insert(catalog.GetName()); + } return_types.emplace_back(LogicalType::BOOLEAN); names.emplace_back("Success"); return std::move(result); } static void DbgenFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &data = (DBGenFunctionData &)*data_p.bind_data; + auto &data = data_p.bind_data->CastNoConst(); if (data.finished) { return; } @@ -92,7 +96,7 @@ static duckdb::unique_ptr TPCHQueryBind(ClientContext &context, Ta } static void TPCHQueryFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &data = (TPCHData &)*data_p.global_state; + auto &data = data_p.global_state->Cast(); idx_t tpch_queries = 22; if (data.offset >= tpch_queries) { // finished returning values @@ -126,7 +130,7 @@ static duckdb::unique_ptr TPCHQueryAnswerBind(ClientContext &conte } static void TPCHQueryAnswerFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &data = (TPCHData &)*data_p.global_state; + auto &data = data_p.global_state->Cast(); idx_t tpch_queries = 22; vector scale_factors {0.01, 0.1, 1}; idx_t total_answers = tpch_queries * scale_factors.size(); diff --git a/src/include/duckdb/function/table_function.hpp b/src/include/duckdb/function/table_function.hpp index 30cc0d9f8df7..3321d274f902 100644 --- a/src/include/duckdb/function/table_function.hpp +++ b/src/include/duckdb/function/table_function.hpp @@ -83,9 +83,9 @@ struct LocalTableFunctionState { struct TableFunctionBindInput { TableFunctionBindInput(vector &inputs, named_parameter_map_t &named_parameters, vector &input_table_types, vector &input_table_names, - optional_ptr info) + optional_ptr info, optional_ptr binder) : inputs(inputs), named_parameters(named_parameters), input_table_types(input_table_types), - input_table_names(input_table_names), info(info) { + input_table_names(input_table_names), info(info), binder(binder) { } vector &inputs; @@ -93,6 +93,7 @@ struct TableFunctionBindInput { vector &input_table_types; vector &input_table_names; optional_ptr info; + optional_ptr binder; }; struct TableFunctionInitInput { diff --git a/src/planner/binder/tableref/bind_table_function.cpp b/src/planner/binder/tableref/bind_table_function.cpp index 16a60ce1cd19..be9848683b54 100644 --- a/src/planner/binder/tableref/bind_table_function.cpp +++ b/src/planner/binder/tableref/bind_table_function.cpp @@ -140,7 +140,7 @@ Binder::BindTableFunctionInternal(TableFunction &table_function, const string &f vector return_names; if (table_function.bind || table_function.bind_replace) { TableFunctionBindInput bind_input(parameters, named_parameters, input_table_types, input_table_names, - table_function.function_info.get()); + table_function.function_info.get(), this); if (table_function.bind_replace) { auto new_plan = table_function.bind_replace(context, bind_input); if (new_plan != nullptr) { diff --git a/src/planner/operator/logical_get.cpp b/src/planner/operator/logical_get.cpp index 1ee9256189ed..077eb6cbbaa9 100644 --- a/src/planner/operator/logical_get.cpp +++ b/src/planner/operator/logical_get.cpp @@ -165,7 +165,7 @@ unique_ptr LogicalGet::Deserialize(Deserializer &deserializer) deserializer.ReadProperty(208, "input_table_types", result->input_table_types); deserializer.ReadProperty(209, "input_table_names", result->input_table_names); TableFunctionBindInput input(result->parameters, result->named_parameters, result->input_table_types, - result->input_table_names, function.function_info.get()); + result->input_table_names, function.function_info.get(), nullptr); vector bind_return_types; vector bind_names; diff --git a/test/sql/tpcds/dsdgen_readonly.test b/test/sql/tpcds/dsdgen_readonly.test new file mode 100644 index 000000000000..a037cade63ec --- /dev/null +++ b/test/sql/tpcds/dsdgen_readonly.test @@ -0,0 +1,30 @@ +# name: test/sql/tpcds/dsdgen_readonly.test +# description: Test that dsdgen respects read-only mode +# group: [tpcds] + +require tpcds + +load __TEST_DIR__/test_dsdgen_ro.db + +statement ok +CREATE TABLE tbl (i INTEGER); + +load __TEST_DIR__/test_dsdgen_ro.db readonly + +statement error +CALL dsdgen(sf=0); +---- +read-only + +load + +statement ok +ATTACH '__TEST_DIR__/test_dsdgen_ro.db' AS dsdgentest (READ_ONLY) + +statement error +CALL dsdgen(sf=0, catalog='dsdgentest'); +---- +read-only + +statement ok +CALL dsdgen(sf=0); diff --git a/test/sql/tpch/dbgen_readonly.test b/test/sql/tpch/dbgen_readonly.test new file mode 100644 index 000000000000..de5ecc8d3377 --- /dev/null +++ b/test/sql/tpch/dbgen_readonly.test @@ -0,0 +1,30 @@ +# name: test/sql/tpch/dbgen_readonly.test +# description: Test that dbgen respects read-only mode +# group: [tpch] + +require tpch + +load __TEST_DIR__/test_dbgen_ro.db + +statement ok +CREATE TABLE tbl (i INTEGER); + +load __TEST_DIR__/test_dbgen_ro.db readonly + +statement error +CALL dbgen(sf=0); +---- +read-only + +load + +statement ok +ATTACH '__TEST_DIR__/test_dbgen_ro.db' AS dbgentest (READ_ONLY) + +statement error +CALL dbgen(sf=0, catalog='dbgentest'); +---- +read-only + +statement ok +CALL dbgen(sf=0); From c4a109f97e017e9ba6733aceafca7c803d31bd62 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Tue, 9 Apr 2024 22:22:43 +0200 Subject: [PATCH 268/603] Correctly handle quoted database names in USE statement --- src/catalog/catalog_search_path.cpp | 5 +++ .../transform/statement/transform_use.cpp | 5 ++- test/sql/attach/attach_dbname_quotes.test | 42 +++++++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 test/sql/attach/attach_dbname_quotes.test diff --git a/src/catalog/catalog_search_path.cpp b/src/catalog/catalog_search_path.cpp index 6be5f491c0f0..9368e300f4ad 100644 --- a/src/catalog/catalog_search_path.cpp +++ b/src/catalog/catalog_search_path.cpp @@ -67,6 +67,11 @@ CatalogSearchEntry CatalogSearchEntry::ParseInternal(const string &input, idx_t if (input[idx] == '"') { //! unquote idx++; + if (idx < input.size() && input[idx] == '"') { + // escaped quote + entry += input[idx]; + continue; + } goto normal; } entry += input[idx]; diff --git a/src/parser/transform/statement/transform_use.cpp b/src/parser/transform/statement/transform_use.cpp index a4d76843725f..4bb8a2b86414 100644 --- a/src/parser/transform/statement/transform_use.cpp +++ b/src/parser/transform/statement/transform_use.cpp @@ -11,9 +11,10 @@ unique_ptr Transformer::TransformUse(duckdb_libpgquery::PGUseStmt } string name; if (IsInvalidSchema(qualified_name.schema)) { - name = qualified_name.name; + name = KeywordHelper::WriteOptionallyQuoted(qualified_name.name, '"'); } else { - name = qualified_name.schema + "." + qualified_name.name; + name = KeywordHelper::WriteOptionallyQuoted(qualified_name.schema, '"') + "." + + KeywordHelper::WriteOptionallyQuoted(qualified_name.name, '"'); } auto name_expr = make_uniq(Value(name)); return make_uniq("schema", std::move(name_expr), SetScope::AUTOMATIC); diff --git a/test/sql/attach/attach_dbname_quotes.test b/test/sql/attach/attach_dbname_quotes.test new file mode 100644 index 000000000000..6c1069dd95c6 --- /dev/null +++ b/test/sql/attach/attach_dbname_quotes.test @@ -0,0 +1,42 @@ +# name: test/sql/attach/attach_dbname_quotes.test +# description: Test ATTACH with a quoted database name +# group: [attach] + +require noforcestorage + +statement ok +ATTACH ':memory:' as "my""db"; + +statement ok +CREATE TABLE "my""db".tbl(i int); + +statement ok +INSERT INTO "my""db".tbl VALUES (42) + +statement ok +USE "my""db"; + +query I +SELECT * FROM tbl +---- +42 + +statement ok +USE memory + +statement ok +CREATE SCHEMA "my""db"."my""schema" + +statement ok +CREATE TABLE "my""db"."my""schema".tbl(i int); + +statement ok +INSERT INTO "my""db"."my""schema".tbl VALUES (84) + +statement ok +USE "my""db"."my""schema" + +query I +SELECT * FROM tbl +---- +84 From 7097f39819edc1cc40f7d55e854ce6628e2d05fb Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 9 Apr 2024 22:23:14 +0200 Subject: [PATCH 269/603] surprisingly painless conversion from std::shared_ptr to duckdb::shared_ptr for the python package --- tools/odbc/include/duckdb_odbc.hpp | 3 +- .../duckdb_python/expression/pyexpression.hpp | 6 +- .../conversions/pyconnection_default.hpp | 2 +- .../duckdb_python/pybind11/pybind_wrapper.hpp | 1 + .../pyconnection/pyconnection.hpp | 3 +- .../src/include/duckdb_python/pyrelation.hpp | 2 +- .../src/include/duckdb_python/pytype.hpp | 2 +- tools/pythonpkg/src/pyconnection.cpp | 18 +++--- .../src/pyconnection/type_creation.cpp | 16 ++--- tools/pythonpkg/src/pyexpression.cpp | 26 ++++---- tools/pythonpkg/src/pyrelation.cpp | 6 +- tools/pythonpkg/src/typing/pytype.cpp | 25 ++++---- tools/pythonpkg/src/typing/typing.cpp | 64 +++++++++---------- 13 files changed, 90 insertions(+), 84 deletions(-) diff --git a/tools/odbc/include/duckdb_odbc.hpp b/tools/odbc/include/duckdb_odbc.hpp index 580cc86e9bdf..14b92ff1df5c 100644 --- a/tools/odbc/include/duckdb_odbc.hpp +++ b/tools/odbc/include/duckdb_odbc.hpp @@ -3,6 +3,7 @@ // needs to be first because BOOL #include "duckdb.hpp" +#include "duckdb/common/shared_ptr.hpp" #include "duckdb/common/windows.hpp" #include "descriptor.hpp" @@ -41,7 +42,7 @@ struct OdbcHandleEnv : public OdbcHandle { OdbcHandleEnv() : OdbcHandle(OdbcHandleType::ENV) { duckdb::DBConfig ODBC_CONFIG; ODBC_CONFIG.SetOptionByName("duckdb_api", "odbc"); - db = make_shared(nullptr, &ODBC_CONFIG); + db = make_refcounted(nullptr, &ODBC_CONFIG); }; shared_ptr db; diff --git a/tools/pythonpkg/src/include/duckdb_python/expression/pyexpression.hpp b/tools/pythonpkg/src/include/duckdb_python/expression/pyexpression.hpp index 2083a488a239..2ac7bc1f91b5 100644 --- a/tools/pythonpkg/src/include/duckdb_python/expression/pyexpression.hpp +++ b/tools/pythonpkg/src/include/duckdb_python/expression/pyexpression.hpp @@ -22,14 +22,14 @@ namespace duckdb { -struct DuckDBPyExpression : public std::enable_shared_from_this { +struct DuckDBPyExpression : public enable_shared_from_this { public: explicit DuckDBPyExpression(unique_ptr expr, OrderType order_type = OrderType::ORDER_DEFAULT, OrderByNullType null_order = OrderByNullType::ORDER_DEFAULT); public: - std::shared_ptr shared_from_this() { - return std::enable_shared_from_this::shared_from_this(); + shared_ptr shared_from_this() { + return enable_shared_from_this::shared_from_this(); } public: diff --git a/tools/pythonpkg/src/include/duckdb_python/pybind11/conversions/pyconnection_default.hpp b/tools/pythonpkg/src/include/duckdb_python/pybind11/conversions/pyconnection_default.hpp index 1c8908b98966..d6ad6979111d 100644 --- a/tools/pythonpkg/src/include/duckdb_python/pybind11/conversions/pyconnection_default.hpp +++ b/tools/pythonpkg/src/include/duckdb_python/pybind11/conversions/pyconnection_default.hpp @@ -37,7 +37,7 @@ class type_caster> }; template <> -struct is_holder_type> : std::true_type {}; +struct is_holder_type> : std::true_type {}; } // namespace detail } // namespace PYBIND11_NAMESPACE diff --git a/tools/pythonpkg/src/include/duckdb_python/pybind11/pybind_wrapper.hpp b/tools/pythonpkg/src/include/duckdb_python/pybind11/pybind_wrapper.hpp index 5c3be44eb6eb..56200f0bb59e 100644 --- a/tools/pythonpkg/src/include/duckdb_python/pybind11/pybind_wrapper.hpp +++ b/tools/pythonpkg/src/include/duckdb_python/pybind11/pybind_wrapper.hpp @@ -17,6 +17,7 @@ #include PYBIND11_DECLARE_HOLDER_TYPE(T, duckdb::unique_ptr) +PYBIND11_DECLARE_HOLDER_TYPE(T, duckdb::shared_ptr) namespace pybind11 { diff --git a/tools/pythonpkg/src/include/duckdb_python/pyconnection/pyconnection.hpp b/tools/pythonpkg/src/include/duckdb_python/pyconnection/pyconnection.hpp index a0d572dc9a91..91cf2be1a898 100644 --- a/tools/pythonpkg/src/include/duckdb_python/pyconnection/pyconnection.hpp +++ b/tools/pythonpkg/src/include/duckdb_python/pyconnection/pyconnection.hpp @@ -22,6 +22,7 @@ #include "duckdb/function/scalar_function.hpp" #include "duckdb_python/pybind11/conversions/exception_handling_enum.hpp" #include "duckdb_python/pybind11/conversions/python_udf_type_enum.hpp" +#include "duckdb/common/shared_ptr.hpp" namespace duckdb { @@ -37,7 +38,7 @@ class RegisteredArrow : public RegisteredObject { unique_ptr arrow_factory; }; -struct DuckDBPyConnection : public std::enable_shared_from_this { +struct DuckDBPyConnection : public enable_shared_from_this { public: shared_ptr database; unique_ptr connection; diff --git a/tools/pythonpkg/src/include/duckdb_python/pyrelation.hpp b/tools/pythonpkg/src/include/duckdb_python/pyrelation.hpp index e6cd967558b4..629299f6b3db 100644 --- a/tools/pythonpkg/src/include/duckdb_python/pyrelation.hpp +++ b/tools/pythonpkg/src/include/duckdb_python/pyrelation.hpp @@ -69,7 +69,7 @@ struct DuckDBPyRelation { py::str GetAlias(); - static unique_ptr EmptyResult(const std::shared_ptr &context, + static unique_ptr EmptyResult(const shared_ptr &context, const vector &types, vector names); unique_ptr SetAlias(const string &expr); diff --git a/tools/pythonpkg/src/include/duckdb_python/pytype.hpp b/tools/pythonpkg/src/include/duckdb_python/pytype.hpp index 72758bf112f1..a6e13dfd68e9 100644 --- a/tools/pythonpkg/src/include/duckdb_python/pytype.hpp +++ b/tools/pythonpkg/src/include/duckdb_python/pytype.hpp @@ -21,7 +21,7 @@ class PyUnionType : public py::object { static bool check_(const py::handle &object); }; -class DuckDBPyType : public std::enable_shared_from_this { +class DuckDBPyType : public enable_shared_from_this { public: explicit DuckDBPyType(LogicalType type); diff --git a/tools/pythonpkg/src/pyconnection.cpp b/tools/pythonpkg/src/pyconnection.cpp index dd366334e71e..548d83a0331d 100644 --- a/tools/pythonpkg/src/pyconnection.cpp +++ b/tools/pythonpkg/src/pyconnection.cpp @@ -50,6 +50,7 @@ #include "duckdb/catalog/catalog_entry/scalar_function_catalog_entry.hpp" #include "duckdb/main/pending_query_result.hpp" #include "duckdb/parser/keyword_helper.hpp" +#include "duckdb/common/shared_ptr.hpp" #include @@ -639,7 +640,7 @@ void DuckDBPyConnection::RegisterArrowObject(const py::object &arrow_object, con } vector> dependencies; dependencies.push_back( - make_shared(make_uniq(std::move(stream_factory), arrow_object))); + make_refcounted(make_uniq(std::move(stream_factory), arrow_object))); connection->context->external_dependencies[name] = std::move(dependencies); } @@ -664,8 +665,8 @@ shared_ptr DuckDBPyConnection::RegisterPythonObject(const st // keep a reference vector> dependencies; - dependencies.push_back(make_shared(make_uniq(python_object), - make_uniq(new_df))); + dependencies.push_back(make_refcounted(make_uniq(python_object), + make_uniq(new_df))); connection->context->external_dependencies[name] = std::move(dependencies); } } else if (IsAcceptedArrowObject(python_object) || IsPolarsDataframe(python_object)) { @@ -774,7 +775,8 @@ unique_ptr DuckDBPyConnection::ReadJSON(const string &name, co auto_detect = true; } - auto read_json_relation = make_shared(connection->context, name, std::move(options), auto_detect); + auto read_json_relation = + make_refcounted(connection->context, name, std::move(options), auto_detect); if (read_json_relation == nullptr) { throw BinderException("read_json can only be used when the JSON extension is (statically) loaded"); } @@ -1317,7 +1319,7 @@ shared_ptr DuckDBPyConnection::Cursor() { if (!connection) { throw ConnectionException("Connection has already been closed"); } - auto res = make_shared(); + auto res = make_refcounted(); res->database = database; res->connection = make_uniq(*res->database); cursors.push_back(res); @@ -1596,7 +1598,7 @@ static void SetDefaultConfigArguments(ClientContext &context) { } static shared_ptr FetchOrCreateInstance(const string &database, DBConfig &config) { - auto res = make_shared(); + auto res = make_refcounted(); res->database = instance_cache.GetInstance(database, config); if (!res->database) { //! No cached database, we must create a new instance @@ -1674,7 +1676,7 @@ shared_ptr DuckDBPyConnection::DefaultConnection() { PythonImportCache *DuckDBPyConnection::ImportCache() { if (!import_cache) { - import_cache = make_shared(); + import_cache = make_refcounted(); } return import_cache.get(); } @@ -1688,7 +1690,7 @@ ModifiedMemoryFileSystem &DuckDBPyConnection::GetObjectFileSystem() { throw InvalidInputException( "This operation could not be completed because required module 'fsspec' is not installed"); } - internal_object_filesystem = make_shared(modified_memory_fs()); + internal_object_filesystem = make_refcounted(modified_memory_fs()); auto &abstract_fs = reinterpret_cast(*internal_object_filesystem); RegisterFilesystem(abstract_fs); } diff --git a/tools/pythonpkg/src/pyconnection/type_creation.cpp b/tools/pythonpkg/src/pyconnection/type_creation.cpp index 5888fd84bb24..91860e7f936e 100644 --- a/tools/pythonpkg/src/pyconnection/type_creation.cpp +++ b/tools/pythonpkg/src/pyconnection/type_creation.cpp @@ -5,17 +5,17 @@ namespace duckdb { shared_ptr DuckDBPyConnection::MapType(const shared_ptr &key_type, const shared_ptr &value_type) { auto map_type = LogicalType::MAP(key_type->Type(), value_type->Type()); - return make_shared(map_type); + return make_refcounted(map_type); } shared_ptr DuckDBPyConnection::ListType(const shared_ptr &type) { auto array_type = LogicalType::LIST(type->Type()); - return make_shared(array_type); + return make_refcounted(array_type); } shared_ptr DuckDBPyConnection::ArrayType(const shared_ptr &type, idx_t size) { auto array_type = LogicalType::ARRAY(type->Type(), size); - return make_shared(array_type); + return make_refcounted(array_type); } static child_list_t GetChildList(const py::object &container) { @@ -59,7 +59,7 @@ shared_ptr DuckDBPyConnection::StructType(const py::object &fields throw InvalidInputException("Can not create an empty struct type!"); } auto struct_type = LogicalType::STRUCT(std::move(types)); - return make_shared(struct_type); + return make_refcounted(struct_type); } shared_ptr DuckDBPyConnection::UnionType(const py::object &members) { @@ -69,7 +69,7 @@ shared_ptr DuckDBPyConnection::UnionType(const py::object &members throw InvalidInputException("Can not create an empty union type!"); } auto union_type = LogicalType::UNION(std::move(types)); - return make_shared(union_type); + return make_refcounted(union_type); } shared_ptr DuckDBPyConnection::EnumType(const string &name, const shared_ptr &type, @@ -79,7 +79,7 @@ shared_ptr DuckDBPyConnection::EnumType(const string &name, const shared_ptr DuckDBPyConnection::DecimalType(int width, int scale) { auto decimal_type = LogicalType::DECIMAL(width, scale); - return make_shared(decimal_type); + return make_refcounted(decimal_type); } shared_ptr DuckDBPyConnection::StringType(const string &collation) { @@ -89,14 +89,14 @@ shared_ptr DuckDBPyConnection::StringType(const string &collation) } else { type = LogicalType::VARCHAR_COLLATION(collation); } - return make_shared(type); + return make_refcounted(type); } shared_ptr DuckDBPyConnection::Type(const string &type_str) { if (!connection) { throw ConnectionException("Connection already closed!"); } - return make_shared(TransformStringToLogicalType(type_str, *connection->context)); + return make_refcounted(TransformStringToLogicalType(type_str, *connection->context)); } } // namespace duckdb diff --git a/tools/pythonpkg/src/pyexpression.cpp b/tools/pythonpkg/src/pyexpression.cpp index d389d1672ecb..09031706acdc 100644 --- a/tools/pythonpkg/src/pyexpression.cpp +++ b/tools/pythonpkg/src/pyexpression.cpp @@ -34,19 +34,19 @@ const ParsedExpression &DuckDBPyExpression::GetExpression() const { shared_ptr DuckDBPyExpression::Copy() const { auto expr = GetExpression().Copy(); - return make_shared(std::move(expr), order_type, null_order); + return make_refcounted(std::move(expr), order_type, null_order); } shared_ptr DuckDBPyExpression::SetAlias(const string &name) const { auto copied_expression = GetExpression().Copy(); copied_expression->alias = name; - return make_shared(std::move(copied_expression)); + return make_refcounted(std::move(copied_expression)); } shared_ptr DuckDBPyExpression::Cast(const DuckDBPyType &type) const { auto copied_expression = GetExpression().Copy(); auto case_expr = make_uniq(type.Type(), std::move(copied_expression)); - return make_shared(std::move(case_expr)); + return make_refcounted(std::move(case_expr)); } // Case Expression modifiers @@ -64,7 +64,7 @@ shared_ptr DuckDBPyExpression::InternalWhen(unique_ptrcase_checks.push_back(std::move(check)); - return make_shared(std::move(expr)); + return make_refcounted(std::move(expr)); } shared_ptr DuckDBPyExpression::When(const DuckDBPyExpression &condition, @@ -82,7 +82,7 @@ shared_ptr DuckDBPyExpression::Else(const DuckDBPyExpression auto expr = unique_ptr_cast(std::move(expr_p)); expr->else_expr = value.GetExpression().Copy(); - return make_shared(std::move(expr)); + return make_refcounted(std::move(expr)); } // Binary operators @@ -181,7 +181,7 @@ shared_ptr DuckDBPyExpression::In(const py::args &args) { expressions.push_back(std::move(expr)); } auto operator_expr = make_uniq(ExpressionType::COMPARE_IN, std::move(expressions)); - return make_shared(std::move(operator_expr)); + return make_refcounted(std::move(operator_expr)); } shared_ptr DuckDBPyExpression::NotIn(const py::args &args) { @@ -249,7 +249,7 @@ shared_ptr DuckDBPyExpression::StarExpression(const py::list case_insensitive_set_t exclude; auto star = make_uniq(); PopulateExcludeList(star->exclude_list, exclude_list); - return make_shared(std::move(star)); + return make_refcounted(std::move(star)); } shared_ptr DuckDBPyExpression::ColumnExpression(const string &column_name) { @@ -267,7 +267,7 @@ shared_ptr DuckDBPyExpression::ColumnExpression(const string } column_names.push_back(qualified_name.name); - return make_shared(make_uniq(std::move(column_names))); + return make_refcounted(make_uniq(std::move(column_names))); } shared_ptr DuckDBPyExpression::ConstantExpression(const py::object &value) { @@ -292,14 +292,14 @@ DuckDBPyExpression::InternalFunctionExpression(const string &function_name, vector> children, bool is_operator) { auto function_expression = make_uniq(function_name, std::move(children), nullptr, nullptr, false, is_operator); - return make_shared(std::move(function_expression)); + return make_refcounted(std::move(function_expression)); } shared_ptr DuckDBPyExpression::InternalUnaryOperator(ExpressionType type, const DuckDBPyExpression &arg) { auto expr = arg.GetExpression().Copy(); auto operator_expression = make_uniq(type, std::move(expr)); - return make_shared(std::move(operator_expression)); + return make_refcounted(std::move(operator_expression)); } shared_ptr DuckDBPyExpression::InternalConjunction(ExpressionType type, @@ -311,11 +311,11 @@ shared_ptr DuckDBPyExpression::InternalConjunction(Expressio children.push_back(other.GetExpression().Copy()); auto operator_expression = make_uniq(type, std::move(children)); - return make_shared(std::move(operator_expression)); + return make_refcounted(std::move(operator_expression)); } shared_ptr DuckDBPyExpression::InternalConstantExpression(Value val) { - return make_shared(make_uniq(std::move(val))); + return make_refcounted(make_uniq(std::move(val))); } shared_ptr DuckDBPyExpression::ComparisonExpression(ExpressionType type, @@ -323,7 +323,7 @@ shared_ptr DuckDBPyExpression::ComparisonExpression(Expressi const DuckDBPyExpression &right_p) { auto left = left_p.GetExpression().Copy(); auto right = right_p.GetExpression().Copy(); - return make_shared( + return make_refcounted( make_uniq(type, std::move(left), std::move(right))); } diff --git a/tools/pythonpkg/src/pyrelation.cpp b/tools/pythonpkg/src/pyrelation.cpp index 457e0f72e617..c71f09ee433f 100644 --- a/tools/pythonpkg/src/pyrelation.cpp +++ b/tools/pythonpkg/src/pyrelation.cpp @@ -146,7 +146,7 @@ unique_ptr DuckDBPyRelation::ProjectFromTypes(const py::object return ProjectFromExpression(projection); } -unique_ptr DuckDBPyRelation::EmptyResult(const std::shared_ptr &context, +unique_ptr DuckDBPyRelation::EmptyResult(const shared_ptr &context, const vector &types, vector names) { vector dummy_values; D_ASSERT(types.size() == names.size()); @@ -157,7 +157,7 @@ unique_ptr DuckDBPyRelation::EmptyResult(const std::shared_ptr } vector> single_row(1, dummy_values); auto values_relation = - make_uniq(make_shared(context, single_row, std::move(names))); + make_uniq(make_refcounted(context, single_row, std::move(names))); // Add a filter on an impossible condition return values_relation->FilterFromExpression("true = false"); } @@ -1198,7 +1198,7 @@ unique_ptr DuckDBPyRelation::Query(const string &view_name, co if (statement.type == StatementType::SELECT_STATEMENT) { auto select_statement = unique_ptr_cast(std::move(parser.statements[0])); auto query_relation = - make_shared(rel->context.GetContext(), std::move(select_statement), "query_relation"); + make_refcounted(rel->context.GetContext(), std::move(select_statement), "query_relation"); return make_uniq(std::move(query_relation)); } else if (IsDescribeStatement(statement)) { auto query = PragmaShow(view_name); diff --git a/tools/pythonpkg/src/typing/pytype.cpp b/tools/pythonpkg/src/typing/pytype.cpp index ad9828876b3d..00edd97af4f9 100644 --- a/tools/pythonpkg/src/typing/pytype.cpp +++ b/tools/pythonpkg/src/typing/pytype.cpp @@ -56,20 +56,20 @@ shared_ptr DuckDBPyType::GetAttribute(const string &name) const { for (idx_t i = 0; i < children.size(); i++) { auto &child = children[i]; if (StringUtil::CIEquals(child.first, name)) { - return make_shared(StructType::GetChildType(type, i)); + return make_refcounted(StructType::GetChildType(type, i)); } } } if (type.id() == LogicalTypeId::LIST && StringUtil::CIEquals(name, "child")) { - return make_shared(ListType::GetChildType(type)); + return make_refcounted(ListType::GetChildType(type)); } if (type.id() == LogicalTypeId::MAP) { auto is_key = StringUtil::CIEquals(name, "key"); auto is_value = StringUtil::CIEquals(name, "value"); if (is_key) { - return make_shared(MapType::KeyType(type)); + return make_refcounted(MapType::KeyType(type)); } else if (is_value) { - return make_shared(MapType::ValueType(type)); + return make_refcounted(MapType::ValueType(type)); } else { throw py::attribute_error(StringUtil::Format("Tried to get a child from a map by the name of '%s', but " "this type only has 'key' and 'value' children", @@ -313,19 +313,19 @@ void DuckDBPyType::Initialize(py::handle &m) { type_module.def_property_readonly("children", &DuckDBPyType::Children); type_module.def(py::init<>([](const string &type_str, shared_ptr connection = nullptr) { auto ltype = FromString(type_str, std::move(connection)); - return make_shared(ltype); + return make_refcounted(ltype); })); type_module.def(py::init<>([](const PyGenericAlias &obj) { auto ltype = FromGenericAlias(obj); - return make_shared(ltype); + return make_refcounted(ltype); })); type_module.def(py::init<>([](const PyUnionType &obj) { auto ltype = FromUnionType(obj); - return make_shared(ltype); + return make_refcounted(ltype); })); type_module.def(py::init<>([](const py::object &obj) { auto ltype = FromObject(obj); - return make_shared(ltype); + return make_refcounted(ltype); })); type_module.def("__getattr__", &DuckDBPyType::GetAttribute, "Get the child type by 'name'", py::arg("name")); type_module.def("__getitem__", &DuckDBPyType::GetAttribute, "Get the child type by 'name'", py::arg("name")); @@ -357,7 +357,7 @@ py::list DuckDBPyType::Children() const { py::list children; auto id = type.id(); if (id == LogicalTypeId::LIST) { - children.append(py::make_tuple("child", make_shared(ListType::GetChildType(type)))); + children.append(py::make_tuple("child", make_refcounted(ListType::GetChildType(type)))); return children; } // FIXME: where is ARRAY?? @@ -366,13 +366,14 @@ py::list DuckDBPyType::Children() const { auto &struct_children = StructType::GetChildTypes(type); for (idx_t i = 0; i < struct_children.size(); i++) { auto &child = struct_children[i]; - children.append(py::make_tuple(child.first, make_shared(StructType::GetChildType(type, i)))); + children.append( + py::make_tuple(child.first, make_refcounted(StructType::GetChildType(type, i)))); } return children; } if (id == LogicalTypeId::MAP) { - children.append(py::make_tuple("key", make_shared(MapType::KeyType(type)))); - children.append(py::make_tuple("value", make_shared(MapType::ValueType(type)))); + children.append(py::make_tuple("key", make_refcounted(MapType::KeyType(type)))); + children.append(py::make_tuple("value", make_refcounted(MapType::ValueType(type)))); return children; } if (id == LogicalTypeId::DECIMAL) { diff --git a/tools/pythonpkg/src/typing/typing.cpp b/tools/pythonpkg/src/typing/typing.cpp index 0c1793ed703e..8064acfc22e1 100644 --- a/tools/pythonpkg/src/typing/typing.cpp +++ b/tools/pythonpkg/src/typing/typing.cpp @@ -4,38 +4,38 @@ namespace duckdb { static void DefineBaseTypes(py::handle &m) { - m.attr("SQLNULL") = make_shared(LogicalType::SQLNULL); - m.attr("BOOLEAN") = make_shared(LogicalType::BOOLEAN); - m.attr("TINYINT") = make_shared(LogicalType::TINYINT); - m.attr("UTINYINT") = make_shared(LogicalType::UTINYINT); - m.attr("SMALLINT") = make_shared(LogicalType::SMALLINT); - m.attr("USMALLINT") = make_shared(LogicalType::USMALLINT); - m.attr("INTEGER") = make_shared(LogicalType::INTEGER); - m.attr("UINTEGER") = make_shared(LogicalType::UINTEGER); - m.attr("BIGINT") = make_shared(LogicalType::BIGINT); - m.attr("UBIGINT") = make_shared(LogicalType::UBIGINT); - m.attr("HUGEINT") = make_shared(LogicalType::HUGEINT); - m.attr("UHUGEINT") = make_shared(LogicalType::UHUGEINT); - m.attr("UUID") = make_shared(LogicalType::UUID); - m.attr("FLOAT") = make_shared(LogicalType::FLOAT); - m.attr("DOUBLE") = make_shared(LogicalType::DOUBLE); - m.attr("DATE") = make_shared(LogicalType::DATE); - - m.attr("TIMESTAMP") = make_shared(LogicalType::TIMESTAMP); - m.attr("TIMESTAMP_MS") = make_shared(LogicalType::TIMESTAMP_MS); - m.attr("TIMESTAMP_NS") = make_shared(LogicalType::TIMESTAMP_NS); - m.attr("TIMESTAMP_S") = make_shared(LogicalType::TIMESTAMP_S); - - m.attr("TIME") = make_shared(LogicalType::TIME); - - m.attr("TIME_TZ") = make_shared(LogicalType::TIME_TZ); - m.attr("TIMESTAMP_TZ") = make_shared(LogicalType::TIMESTAMP_TZ); - - m.attr("VARCHAR") = make_shared(LogicalType::VARCHAR); - - m.attr("BLOB") = make_shared(LogicalType::BLOB); - m.attr("BIT") = make_shared(LogicalType::BIT); - m.attr("INTERVAL") = make_shared(LogicalType::INTERVAL); + m.attr("SQLNULL") = make_refcounted(LogicalType::SQLNULL); + m.attr("BOOLEAN") = make_refcounted(LogicalType::BOOLEAN); + m.attr("TINYINT") = make_refcounted(LogicalType::TINYINT); + m.attr("UTINYINT") = make_refcounted(LogicalType::UTINYINT); + m.attr("SMALLINT") = make_refcounted(LogicalType::SMALLINT); + m.attr("USMALLINT") = make_refcounted(LogicalType::USMALLINT); + m.attr("INTEGER") = make_refcounted(LogicalType::INTEGER); + m.attr("UINTEGER") = make_refcounted(LogicalType::UINTEGER); + m.attr("BIGINT") = make_refcounted(LogicalType::BIGINT); + m.attr("UBIGINT") = make_refcounted(LogicalType::UBIGINT); + m.attr("HUGEINT") = make_refcounted(LogicalType::HUGEINT); + m.attr("UHUGEINT") = make_refcounted(LogicalType::UHUGEINT); + m.attr("UUID") = make_refcounted(LogicalType::UUID); + m.attr("FLOAT") = make_refcounted(LogicalType::FLOAT); + m.attr("DOUBLE") = make_refcounted(LogicalType::DOUBLE); + m.attr("DATE") = make_refcounted(LogicalType::DATE); + + m.attr("TIMESTAMP") = make_refcounted(LogicalType::TIMESTAMP); + m.attr("TIMESTAMP_MS") = make_refcounted(LogicalType::TIMESTAMP_MS); + m.attr("TIMESTAMP_NS") = make_refcounted(LogicalType::TIMESTAMP_NS); + m.attr("TIMESTAMP_S") = make_refcounted(LogicalType::TIMESTAMP_S); + + m.attr("TIME") = make_refcounted(LogicalType::TIME); + + m.attr("TIME_TZ") = make_refcounted(LogicalType::TIME_TZ); + m.attr("TIMESTAMP_TZ") = make_refcounted(LogicalType::TIMESTAMP_TZ); + + m.attr("VARCHAR") = make_refcounted(LogicalType::VARCHAR); + + m.attr("BLOB") = make_refcounted(LogicalType::BLOB); + m.attr("BIT") = make_refcounted(LogicalType::BIT); + m.attr("INTERVAL") = make_refcounted(LogicalType::INTERVAL); } void DuckDBPyTyping::Initialize(py::module_ &parent) { From 1ce844a7e7a9da99e43f7e925e261a576a02513b Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Tue, 9 Apr 2024 22:23:40 +0200 Subject: [PATCH 270/603] Only quotes --- test/sql/attach/attach_dbname_quotes.test | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/sql/attach/attach_dbname_quotes.test b/test/sql/attach/attach_dbname_quotes.test index 6c1069dd95c6..2c4157dcb1ac 100644 --- a/test/sql/attach/attach_dbname_quotes.test +++ b/test/sql/attach/attach_dbname_quotes.test @@ -40,3 +40,9 @@ query I SELECT * FROM tbl ---- 84 + +statement ok +CREATE SCHEMA """" + +statement ok +USE """" From eb760b8aae9eed69ef5087168d4085d86e991468 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Tue, 9 Apr 2024 23:54:16 +0200 Subject: [PATCH 271/603] Test fix --- test/sql/attach/attach_table_info.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/sql/attach/attach_table_info.test b/test/sql/attach/attach_table_info.test index d8609aab98be..7f312f0d530c 100644 --- a/test/sql/attach/attach_table_info.test +++ b/test/sql/attach/attach_table_info.test @@ -38,7 +38,7 @@ SELECT current_database() memory statement ok -USE "new_database.new_schema" +USE new_database.new_schema query ITTTTT nosort table_info PRAGMA table_info('integers'); From 684c10bb92d0fc11c351ee19794edcb69c1babbf Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Tue, 9 Apr 2024 23:54:52 +0200 Subject: [PATCH 272/603] Fix for Python client bind --- tools/pythonpkg/src/python_udf.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pythonpkg/src/python_udf.cpp b/tools/pythonpkg/src/python_udf.cpp index 8850e8e0218d..45f1f40c6f16 100644 --- a/tools/pythonpkg/src/python_udf.cpp +++ b/tools/pythonpkg/src/python_udf.cpp @@ -65,7 +65,7 @@ static void ConvertPyArrowToDataChunk(const py::object &table, Vector &out, Clie vector input_types; vector input_names; - auto bind_input = TableFunctionBindInput(children, named_params, input_types, input_names, nullptr); + TableFunctionBindInput bind_input(children, named_params, input_types, input_names, nullptr, nullptr); vector return_types; vector return_names; From 05fcc39e39f453b887188b9d7412118da501789c Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Wed, 10 Apr 2024 08:45:04 +0200 Subject: [PATCH 273/603] Update token --- .github/workflows/DraftPR.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/DraftPR.yml b/.github/workflows/DraftPR.yml index 0fd408681cb9..85111502f910 100644 --- a/.github/workflows/DraftPR.yml +++ b/.github/workflows/DraftPR.yml @@ -43,7 +43,7 @@ jobs: - name: 'Actually move to draft' shell: bash run: | - echo ${{ secrets.GH_TOKEN }} | gh auth login --with-token + echo ${{ secrets.MOVE_PR_TO_DRAFT_TOKEN }} | gh auth login --with-token gh api graphql -F id=${{ env.PR_NUMBER }} -f query=' mutation($id: ID!) { convertPullRequestToDraft(input: { pullRequestId: $id }) { From 797aa1e571d7a89ee094b958b4ad4f130913f914 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Wed, 10 Apr 2024 08:46:21 +0200 Subject: [PATCH 274/603] skip_reload because of ATTACH --- test/sql/tpcds/dsdgen_readonly.test | 2 ++ test/sql/tpch/dbgen_readonly.test | 2 ++ 2 files changed, 4 insertions(+) diff --git a/test/sql/tpcds/dsdgen_readonly.test b/test/sql/tpcds/dsdgen_readonly.test index a037cade63ec..ec0c448d8e2e 100644 --- a/test/sql/tpcds/dsdgen_readonly.test +++ b/test/sql/tpcds/dsdgen_readonly.test @@ -4,6 +4,8 @@ require tpcds +require skip_reload + load __TEST_DIR__/test_dsdgen_ro.db statement ok diff --git a/test/sql/tpch/dbgen_readonly.test b/test/sql/tpch/dbgen_readonly.test index de5ecc8d3377..33cdb551c609 100644 --- a/test/sql/tpch/dbgen_readonly.test +++ b/test/sql/tpch/dbgen_readonly.test @@ -4,6 +4,8 @@ require tpch +require skip_reload + load __TEST_DIR__/test_dbgen_ro.db statement ok From 9562296f692c5acba20b60b296e80585da803cc3 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 10 Apr 2024 09:35:22 +0200 Subject: [PATCH 275/603] [CI] Remove GITHUB_PAT variable from R-CMD-check --- .github/workflows/R_CMD_CHECK.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/R_CMD_CHECK.yml b/.github/workflows/R_CMD_CHECK.yml index a5fe7c191be1..d111e135547a 100644 --- a/.github/workflows/R_CMD_CHECK.yml +++ b/.github/workflows/R_CMD_CHECK.yml @@ -51,7 +51,6 @@ jobs: - {os: ubuntu-latest, r: 'release'} env: - GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} R_KEEP_PKG_SOURCE: yes steps: From 1a1cd7b565b5b7f6f87f85f249cd2fa46f6f0965 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Wed, 10 Apr 2024 09:43:00 +0200 Subject: [PATCH 276/603] Remove quotes in tests --- test/sql/attach/attach_did_you_mean.test | 2 +- test/sql/attach/attach_nested_types.test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/sql/attach/attach_did_you_mean.test b/test/sql/attach/attach_did_you_mean.test index 26c418a4ba6f..967d2103e3a5 100644 --- a/test/sql/attach/attach_did_you_mean.test +++ b/test/sql/attach/attach_did_you_mean.test @@ -55,7 +55,7 @@ SELECT * FROM memory.hello # what if we switch default database AND default schema? statement ok -USE "db1.myschema" +USE db1.myschema statement ok SELECT * FROM blablabla diff --git a/test/sql/attach/attach_nested_types.test b/test/sql/attach/attach_nested_types.test index 9b63c6e8c052..99369a286c99 100644 --- a/test/sql/attach/attach_nested_types.test +++ b/test/sql/attach/attach_nested_types.test @@ -49,7 +49,7 @@ SELECT "table" FROM database.schema.table {'col': {'field': 42}} statement ok -USE "database.schema" +USE database.schema query I SELECT "table" FROM "table" From 6f3e37475d983fece4b967ed7a438b3379b3bc4d Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 10 Apr 2024 10:17:32 +0200 Subject: [PATCH 277/603] Bump-back duckdb_azure to pre-lzma custom vcpkg-port moving bminor mirror --- .github/config/out_of_tree_extensions.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/config/out_of_tree_extensions.cmake b/.github/config/out_of_tree_extensions.cmake index 131d84e90812..c94a1e029af6 100644 --- a/.github/config/out_of_tree_extensions.cmake +++ b/.github/config/out_of_tree_extensions.cmake @@ -38,7 +38,7 @@ if (NOT MINGW) duckdb_extension_load(azure LOAD_TESTS GIT_URL https://github.com/duckdb/duckdb_azure - GIT_TAG 6620a32454c1eb2e455104d87262061d2464aad0 + GIT_TAG 506b1fa0f3f892000130feac7a0e1de346095e80 APPLY_PATCHES ) endif() From f41e41be8f49d6dc9f7bdd23dcc992aaefa098bc Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 10 Apr 2024 11:17:53 +0200 Subject: [PATCH 278/603] Bump-forward duckdb_azure to main, post-lzma custom vcpkg-port moving mirror --- .github/config/out_of_tree_extensions.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/config/out_of_tree_extensions.cmake b/.github/config/out_of_tree_extensions.cmake index c94a1e029af6..5d1795ef1168 100644 --- a/.github/config/out_of_tree_extensions.cmake +++ b/.github/config/out_of_tree_extensions.cmake @@ -38,7 +38,7 @@ if (NOT MINGW) duckdb_extension_load(azure LOAD_TESTS GIT_URL https://github.com/duckdb/duckdb_azure - GIT_TAG 506b1fa0f3f892000130feac7a0e1de346095e80 + GIT_TAG 4512a652479016d40d712f990cab9b9aab43d341 APPLY_PATCHES ) endif() From c16051a5513bda1c9ef006ba8af4bdea19a523c4 Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 10 Apr 2024 11:25:49 +0200 Subject: [PATCH 279/603] add missing include --- src/include/duckdb/storage/table_storage_info.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/include/duckdb/storage/table_storage_info.hpp b/src/include/duckdb/storage/table_storage_info.hpp index 9f1eece71cf8..49b3089c657a 100644 --- a/src/include/duckdb/storage/table_storage_info.hpp +++ b/src/include/duckdb/storage/table_storage_info.hpp @@ -13,6 +13,7 @@ #include "duckdb/storage/block.hpp" #include "duckdb/storage/index_storage_info.hpp" #include "duckdb/storage/storage_info.hpp" +#include "duckdb/common/optional_idx.hpp" namespace duckdb { From 8e1e498b03417d61885a6274dc06bdf3ea465f96 Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 10 Apr 2024 12:28:23 +0200 Subject: [PATCH 280/603] throw an error if the CREATE INDEX statement has a WHERE clause --- src/include/duckdb/storage/table_storage_info.hpp | 2 +- src/parser/parsed_data/create_index_info.cpp | 2 -- src/parser/transform/statement/transform_create_index.cpp | 3 +++ test/fuzzer/pedro/create_index_error.test | 5 +++++ 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/include/duckdb/storage/table_storage_info.hpp b/src/include/duckdb/storage/table_storage_info.hpp index ea7d378a7262..49b3089c657a 100644 --- a/src/include/duckdb/storage/table_storage_info.hpp +++ b/src/include/duckdb/storage/table_storage_info.hpp @@ -8,12 +8,12 @@ #pragma once -#include "duckdb/common/optional_idx.hpp" #include "duckdb/common/types/value.hpp" #include "duckdb/common/unordered_map.hpp" #include "duckdb/storage/block.hpp" #include "duckdb/storage/index_storage_info.hpp" #include "duckdb/storage/storage_info.hpp" +#include "duckdb/common/optional_idx.hpp" namespace duckdb { diff --git a/src/parser/parsed_data/create_index_info.cpp b/src/parser/parsed_data/create_index_info.cpp index e649047a94ca..739d03626625 100644 --- a/src/parser/parsed_data/create_index_info.cpp +++ b/src/parser/parsed_data/create_index_info.cpp @@ -36,7 +36,6 @@ string CreateIndexInfo::ToString() const { result += " UNIQUE"; } result += " INDEX "; - // FIXME: 'CONCURRENTLY' ?? if (on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT) { result += "IF NOT EXISTS "; } @@ -73,7 +72,6 @@ string CreateIndexInfo::ToString() const { } result += " )"; } - // FIXME: optional WHERE ??? return result; } diff --git a/src/parser/transform/statement/transform_create_index.cpp b/src/parser/transform/statement/transform_create_index.cpp index cf3c81f47a02..3b13d53ce341 100644 --- a/src/parser/transform/statement/transform_create_index.cpp +++ b/src/parser/transform/statement/transform_create_index.cpp @@ -77,6 +77,9 @@ unique_ptr Transformer::TransformCreateIndex(duckdb_libpgquery: for (auto &expr : info->expressions) { info->parsed_expressions.emplace_back(expr->Copy()); } + if (stmt.whereClause) { + throw NotImplementedException("Creating partial indexes is not supported currently"); + } result->info = std::move(info); return result; } diff --git a/test/fuzzer/pedro/create_index_error.test b/test/fuzzer/pedro/create_index_error.test index 476ab8e69bf6..2c33436dc207 100644 --- a/test/fuzzer/pedro/create_index_error.test +++ b/test/fuzzer/pedro/create_index_error.test @@ -61,3 +61,8 @@ TransactionContext Error: Failed to commit: Failure in decode: could not convert statement ok CREATE INDEX i1 ON t1 USING ART (c1); + +statement error +create index i2 ON t1(c1) WHERE c1 != TRUE; +---- +Creating partial indexes is not supported currently From bccb90dd6167c274d0d6d9fa2880a5cf3387f9c4 Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 10 Apr 2024 12:38:32 +0200 Subject: [PATCH 281/603] adjust existing tests --- test/sql/upsert/postgres/composite_key.test | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/sql/upsert/postgres/composite_key.test b/test/sql/upsert/postgres/composite_key.test index 774e046da778..054c8077e695 100644 --- a/test/sql/upsert/postgres/composite_key.test +++ b/test/sql/upsert/postgres/composite_key.test @@ -66,11 +66,15 @@ statement ok insert into insertconflicttest values (12, 'Date', 0) on conflict (fruit, key) do update set other = 1 # Partial index tests, no inference predicate specified -statement ok +statement error create unique index part_comp_key_index on insertconflicttest(key, fruit) where key < 5; +---- +Creating partial indexes is not supported currently -statement ok +statement error create unique index expr_part_comp_key_index on insertconflicttest(key, fruit) where key < 5; +---- +Creating partial indexes is not supported currently # Expression index tests statement ok From bbd74ac95dc58000d7add23c2ecf099a5ff284a5 Mon Sep 17 00:00:00 2001 From: Christina Sioula Date: Fri, 5 Apr 2024 12:59:37 +0200 Subject: [PATCH 282/603] support reading gzipped files with the test runner --- .gitignore | 3 +- data/storage/index_0-9-1.db.gz | Bin 0 -> 3332 bytes data/storage/test.db.gz | Bin 0 -> 591 bytes test/sql/storage/unzip.test | 62 +++++++++++++++++++++++++++ test/sqlite/sqllogic_command.cpp | 36 ++++++++++++++++ test/sqlite/sqllogic_command.hpp | 16 +++++++ test/sqlite/sqllogic_parser.cpp | 3 ++ test/sqlite/sqllogic_parser.hpp | 3 +- test/sqlite/sqllogic_test_runner.cpp | 34 +++++++++++++-- 9 files changed, 152 insertions(+), 5 deletions(-) create mode 100644 data/storage/index_0-9-1.db.gz create mode 100644 data/storage/test.db.gz create mode 100644 test/sql/storage/unzip.test diff --git a/.gitignore b/.gitignore index 67114a06f137..f754b6ec73c4 100644 --- a/.gitignore +++ b/.gitignore @@ -339,9 +339,10 @@ zig-cache/* # .db files *.db +*.db.gz # local cmake extension config extension/extension_config_local.cmake # extension_external dir -extension_external \ No newline at end of file +extension_external diff --git a/data/storage/index_0-9-1.db.gz b/data/storage/index_0-9-1.db.gz new file mode 100644 index 0000000000000000000000000000000000000000..c4f963ac8cc47a3dc49c7d763924dfe3a9547da8 GIT binary patch literal 3332 zcmd^<{WlwC8^_bqO=}aIJ~}PUTJ^F;Lx*s@WkTyQDlq~m zjN79D2C^ytM9Ww!z;beoz7LX;n|r++o?q@~6DsiXJ$xj=&zeRxw8!kg#$8;Rn4ah= zE;#(XWm`Y(bKk?3XR})R{e!*(`FcTafBmyNaKcP6;$Ki+TyPdYxXaO5YJpGh_J?-W zcGU_4>t5)31Zwt1sdtTjxU^Cbm@T7htn)orf!;Zyrpli^V>uxA_1+V6O3DGsJs)V! z)ksZTmDNrF0O0wK1kwcuKr?YJZ~nni{cnZ0HIZEm#a}8tXNfY7dpYZLz7^uF z&TlYB__~HvAbIHLecA(M0#nJWGZpm>Zf@KvGCY1X#nxh(#(a5iH%Z{qdDb}DvHpOq zaTQw_9b+&&GcQfr#`ftZ6jm&U$_Z0;M%q>JiA#Im(AkjQNX!rC)c%-@$~J_U48eM==DmP?Aj(hnxQIl z$hG=LAT_+a=F!I0)6ea}7#DL%<=3xc5c6p}0Sh~a-+Sy>EMSM*wSg8^22(d0+QzSG zk4Ip`Z1J^#Ui_SOFI2E-eErEWp42e+$(IC^uUiRZ<#%xRkkFh6w0-v3-%R=ba-}3V z=2UD^M-ugw*S;!|&ap+Pe8?bX_^tU>>F4yb(&0NTrI+50Y^2T_Iv+o71+}NOhiG6a z5R%BvUp~GsR2)T99k_6vrnFLC*Ne$(F#dh9=Hxun(u233SS6ytb)9gX)FCOe z|4O&m`o_zwGL&ErwDGH!ba}Q7xESHo|@HIy#SZ8#1`E zQrIkm46d%@&)tQsg-DCtPRn1@aMxI_KIe*u??i6Ke8ZlvH=t#P)+zSg)$SQ2^`%@q zKU7kgHzH{S@+~$$g6(~By(K4Ac-+julJ~LyIS`MGM)Go;s(mT~lB4A_Kd{`YZ=FPA zJxVGYE9gCT)z2y#VbwP}a4iULq-R*o2VWJp?%Li8Z9$j*5KRH{slg!asOP5u24v6R zp-=2aXBIGwieY0+BRkRO-UDG+B9Zs{{{4aX?&wQ0C8Y1*H)Klg@75*i=X+gye1GiB zw5c)JV$XIS)5Co-ft0%P%SCd^}SSl0lPWh;_674i3@zfmJy~T6e zdAyiHd4hXG8U@y&m;%6MDfJ42w*n2j$&=(8E9XVz^+ccZO?6;4+i_?DiGKoaD=L-P zO@?3~uxhFV$^OE(`?G&@BvClF<_&?cE(2PIVE}DuLFY{lLAtq!l}BqA-cHA*i~6xN dReSViC_VLRqwQ>@E!d1^2z?0cX8Vr<(@# zoSi49g$K+#YW#ucp5x8Po#+1SY)drhQme7^J-hz%?*F@XzqkAzW1Ay>`S0Q7Q_nu# zy3}CK=Agoj5mI;bL&ct*dV1^5pFaP3<`Tbty=Z?9266_%U#Jqp?6TeEH z4Q2%L?!T-nd;G5A*{+Gt|9W|xR$SYnBDOWa;?7O$Td&V>O^Hrk8*eHy>+0pS_0_`1 zX6~G?1Y_UjO^?n0Z%{{Ot2x zzi$y-eL;@%tHg)fZ$Iknu3cGm@ZV(N>idF64o^B|aD7eLbBRZRa~~B|m1fu0UU|>? z!~dAuUj~K)*_A6!n5#eEs8qT6l*Prk<}C*}-t+w0x4(Y>|HZW+TSgu@Aa2W?k>_}a I!GMJU0N>E9%K!iX literal 0 HcmV?d00001 diff --git a/test/sql/storage/unzip.test b/test/sql/storage/unzip.test new file mode 100644 index 000000000000..6592fc6c672c --- /dev/null +++ b/test/sql/storage/unzip.test @@ -0,0 +1,62 @@ +# name: test/sql/storage/unzip.test +# description: Support gzipped files in the test runner +# group: [storage] + +# unzip to specific path +unzip data/storage/test.db.gz __TEST_DIR__/test.db + +load __TEST_DIR__/test.db readonly + +query I +SELECT a+1 FROM tbl; +---- +6 + +# unzip a 1.8M file to default extraction path -> __TEST_DIR__/ +unzip data/storage/index_0-9-1.db.gz + +load __TEST_DIR__/index_0-9-1.db readonly + +query II +SELECT table_name, index_count FROM duckdb_tables() ORDER BY table_name; +---- +fk_tbl 1 +idx_tbl 2 +pk_tbl 2 + +# unzip to default extraction path from NULL input +unzip data/storage/test.db.gz NULL + +load __TEST_DIR__/test.db readonly + +query I +SELECT a+2 FROM tbl; +---- +7 + +## test invalid use +# unzip + +## not gzipped database +# unzip data/storage/test.db + +## not gzipped database +# unzip data/storage/test.db + +## test NULL input paths +# unzip NULL +# unzip NULL NULL +# unzip NULL data/storage/test.db + +## invalid input path +# unzip path/to/nowhere data/storage/not_existed.db + +## invalid extraction path +# unzip data/storage/test.db.gz path/to/nowhere + +## already existed database file in the extraction - warning: this will overwrite existed wal_test_092.db +# unzip data/storage/test.db.gz data/storage/wal_test_092.db + +## extraction path to directory +# unzip data/storage/test.db.gz __TEST_DIR__/ +# unzip data/storage/test.db.gz __TEST_DIR__ diff --git a/test/sqlite/sqllogic_command.cpp b/test/sqlite/sqllogic_command.cpp index b9f64b5f407a..bd1b3e459729 100644 --- a/test/sqlite/sqllogic_command.cpp +++ b/test/sqlite/sqllogic_command.cpp @@ -144,6 +144,10 @@ SleepCommand::SleepCommand(SQLLogicTestRunner &runner, idx_t duration, SleepUnit : Command(runner), duration(duration), unit(unit) { } +UnzipCommand::UnzipCommand(SQLLogicTestRunner &runner, string &input, string &output) + : Command(runner), input_path(input), extraction_path(output) { +} + struct ParallelExecuteContext { ParallelExecuteContext(SQLLogicTestRunner &runner, const vector> &loop_commands, LoopDefinition definition) @@ -387,4 +391,36 @@ void Statement::ExecuteInternal(ExecuteContext &context) const { } } +void UnzipCommand::ExecuteInternal(ExecuteContext &context) const { + VirtualFileSystem vfs; + + // input + FileOpenFlags in_flags(FileFlags::FILE_FLAGS_READ); + in_flags.SetCompression(FileCompressionType::GZIP); + auto compressed_file_handle = vfs.OpenFile(input_path, in_flags); + if (compressed_file_handle == nullptr) { + throw CatalogException("Cannot open the file \"%s\"", input_path); + } + + // read the compressed data from the file + int64_t file_size = vfs.GetFileSize(*compressed_file_handle); + std::unique_ptr compressed_buffer(new char[BUFFER_SIZE]); + int64_t bytes_read = vfs.Read(*compressed_file_handle, compressed_buffer.get(), BUFFER_SIZE); + if (bytes_read < file_size) { + throw CatalogException("Cannot read the file \"%s\"", input_path); + } + + // output + FileOpenFlags out_flags(FileOpenFlags::FILE_FLAGS_FILE_CREATE | FileOpenFlags::FILE_FLAGS_WRITE); + auto output_file = vfs.OpenFile(extraction_path, out_flags); + if (!output_file) { + throw CatalogException("Cannot open the file \"%s\"", extraction_path); + } + + int64_t bytes_written = vfs.Write(*output_file, compressed_buffer.get(), BUFFER_SIZE); + if (bytes_written < file_size) { + throw CatalogException("Cannot write the file \"%s\"", extraction_path); + } +} + } // namespace duckdb diff --git a/test/sqlite/sqllogic_command.hpp b/test/sqlite/sqllogic_command.hpp index 0f45d8aa2755..d6b2e6c9f71a 100644 --- a/test/sqlite/sqllogic_command.hpp +++ b/test/sqlite/sqllogic_command.hpp @@ -9,6 +9,7 @@ #pragma once #include "duckdb.hpp" +#include "duckdb/common/virtual_file_system.hpp" namespace duckdb { class SQLLogicTestRunner; @@ -142,4 +143,19 @@ class SleepCommand : public Command { SleepUnit unit; }; +class UnzipCommand : public Command { +public: + // 1 MB + static constexpr const int64_t BUFFER_SIZE = 1u << 23; + +public: + UnzipCommand(SQLLogicTestRunner &runner, string &input, string &output); + + void ExecuteInternal(ExecuteContext &context) const override; + +private: + string input_path; + string extraction_path; +}; + } // namespace duckdb diff --git a/test/sqlite/sqllogic_parser.cpp b/test/sqlite/sqllogic_parser.cpp index 8cccc18ff64e..8fdb3f1bb85c 100644 --- a/test/sqlite/sqllogic_parser.cpp +++ b/test/sqlite/sqllogic_parser.cpp @@ -166,6 +166,7 @@ bool SQLLogicParser::IsSingleLineStatement(SQLLogicToken &token) { case SQLLogicTokenType::SQLLOGIC_RESTART: case SQLLogicTokenType::SQLLOGIC_RECONNECT: case SQLLogicTokenType::SQLLOGIC_SLEEP: + case SQLLogicTokenType::SQLLOGIC_UNZIP: return true; case SQLLogicTokenType::SQLLOGIC_SKIP_IF: @@ -219,6 +220,8 @@ SQLLogicTokenType SQLLogicParser::CommandToToken(const string &token) { return SQLLogicTokenType::SQLLOGIC_RECONNECT; } else if (token == "sleep") { return SQLLogicTokenType::SQLLOGIC_SLEEP; + } else if (token == "unzip") { + return SQLLogicTokenType::SQLLOGIC_UNZIP; } Fail("Unrecognized parameter %s", token); return SQLLogicTokenType::SQLLOGIC_INVALID; diff --git a/test/sqlite/sqllogic_parser.hpp b/test/sqlite/sqllogic_parser.hpp index af32e943c509..87e79be2b3a1 100644 --- a/test/sqlite/sqllogic_parser.hpp +++ b/test/sqlite/sqllogic_parser.hpp @@ -34,7 +34,8 @@ enum class SQLLogicTokenType { SQLLOGIC_LOAD, SQLLOGIC_RESTART, SQLLOGIC_RECONNECT, - SQLLOGIC_SLEEP + SQLLOGIC_SLEEP, + SQLLOGIC_UNZIP }; class SQLLogicToken { diff --git a/test/sqlite/sqllogic_test_runner.cpp b/test/sqlite/sqllogic_test_runner.cpp index 7fbbc621445f..6c7f32d38859 100644 --- a/test/sqlite/sqllogic_test_runner.cpp +++ b/test/sqlite/sqllogic_test_runner.cpp @@ -1,12 +1,14 @@ #include "catch.hpp" - -#include "sqllogic_test_runner.hpp" #include "test_helpers.hpp" +#include "sqllogic_parser.hpp" +#include "sqllogic_test_runner.hpp" #include "duckdb/main/extension_helper.hpp" #include "duckdb/main/extension/generated_extension_loader.hpp" #include "duckdb/main/extension_entries.hpp" -#include "sqllogic_parser.hpp" +#include "duckdb/common/virtual_file_system.hpp" +#include "duckdb/common/file_open_flags.hpp" + #ifdef DUCKDB_OUT_OF_TREE #include DUCKDB_EXTENSION_HEADER #endif @@ -732,6 +734,32 @@ void SQLLogicTestRunner::ExecuteFile(string script) { auto sleep_unit = SleepCommand::ParseUnit(token.parameters[1]); auto command = make_uniq(*this, sleep_duration, sleep_unit); ExecuteCommand(std::move(command)); + } else if (token.type == SQLLogicTokenType::SQLLOGIC_UNZIP) { + if (token.parameters.size() != 1 && token.parameters.size() != 2) { + parser.Fail("unzip requires 1 argument: [optional: " + ", default: __TEST_DIR__/]"); + } + + // set input path + auto input_path = ReplaceKeywords(token.parameters[0]); + + // file name + idx_t filename_start_pos = input_path.find_last_of("/") + 1; + if (!StringUtil::EndsWith(input_path, ".gz")) { + parser.Fail("unzip: input has not a GZIP extension"); + } + string filename = input_path.substr(filename_start_pos, input_path.size() - filename_start_pos - 3); + + // extraction path + string default_extraction_path = ReplaceKeywords("__TEST_DIR__/" + filename); + string extraction_path = + (token.parameters.size() == 2) ? ReplaceKeywords(token.parameters[1]) : default_extraction_path; + if (extraction_path == "NULL") { + extraction_path = default_extraction_path; + } + + auto command = make_uniq(*this, input_path, extraction_path); + ExecuteCommand(std::move(command)); } } if (InLoop()) { From 8097fdc0baa9fb0b36a5df08df93734dc507dbca Mon Sep 17 00:00:00 2001 From: Christina Sioula Date: Wed, 10 Apr 2024 13:13:44 +0200 Subject: [PATCH 283/603] add README.md --- data/storage/README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 data/storage/README.md diff --git a/data/storage/README.md b/data/storage/README.md new file mode 100644 index 000000000000..82b1534a5a62 --- /dev/null +++ b/data/storage/README.md @@ -0,0 +1,23 @@ +# DuckDB database files +DuckDB v0.10 enables the creation of storage database files, ensuring seamless compatibility for subsequent versions to read files generated by older versions. *Note: it is recommended to compress large database files.* + +For further details, please refer to: [Storage & Compatibility](https://duckdb.org/docs/internals/storage#compatibility) + +### How to gzip a database file +``` +gzip +``` + +### How to decompress and load a gzipped database file in the test runner +``` +unzip + +load readonly +``` + +or by using the default extraction path `__TEST_DIR__/` (temporary space) +``` +unzip + +load __TEST_DIR__/ readonly +``` From 7990a3ea3affe30feb980f802bcd3d28ca611161 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Wed, 10 Apr 2024 13:31:10 +0200 Subject: [PATCH 284/603] bitpacking --- src/execution/perfect_aggregate_hashtable.cpp | 9 +++++---- .../duckdb/storage/buffer/buffer_pool.hpp | 2 +- .../compression/chimp/algorithm/bit_reader.hpp | 2 +- src/storage/buffer/buffer_pool.cpp | 13 ++++++++++--- src/storage/buffer/buffer_pool_reservation.cpp | 4 ++-- src/storage/compression/bitpacking.cpp | 16 ++++++++++------ 6 files changed, 29 insertions(+), 17 deletions(-) diff --git a/src/execution/perfect_aggregate_hashtable.cpp b/src/execution/perfect_aggregate_hashtable.cpp index da7c20192452..ba8e366e9414 100644 --- a/src/execution/perfect_aggregate_hashtable.cpp +++ b/src/execution/perfect_aggregate_hashtable.cpp @@ -64,7 +64,7 @@ static void ComputeGroupLocationTemplated(UnifiedVectorFormat &group_data, Value // we only need to handle non-null values here if (group_data.validity.RowIsValid(index)) { D_ASSERT(data[index] >= min_val); - uintptr_t adjusted_value = (data[index] - min_val) + 1; + auto adjusted_value = UnsafeNumericCast((data[index] - min_val) + 1); address_data[i] += adjusted_value << current_shift; } } @@ -72,7 +72,7 @@ static void ComputeGroupLocationTemplated(UnifiedVectorFormat &group_data, Value // no null values: we can directly compute the addresses for (idx_t i = 0; i < count; i++) { auto index = group_data.sel->get_index(i); - uintptr_t adjusted_value = (data[index] - min_val) + 1; + auto adjusted_value = UnsafeNumericCast((data[index] - min_val) + 1); address_data[i] += adjusted_value << current_shift; } } @@ -149,7 +149,7 @@ void PerfectAggregateHashTable::AddChunk(DataChunk &groups, DataChunk &payload) } // move to the next aggregate payload_idx += input_count; - VectorOperations::AddInPlace(addresses, aggregate.payload_size, payload.size()); + VectorOperations::AddInPlace(addresses, UnsafeNumericCast(aggregate.payload_size), payload.size()); } } @@ -205,7 +205,8 @@ static void ReconstructGroupVectorTemplated(uint32_t group_values[], Value &min, validity_mask.SetInvalid(i); } else { // otherwise we add the value (minus 1) to the min value - data[i] = UnsafeNumericCast(min_data + group_index - 1); + data[i] = UnsafeNumericCast(UnsafeNumericCast(min_data) + + UnsafeNumericCast(group_index) - 1); } } } diff --git a/src/include/duckdb/storage/buffer/buffer_pool.hpp b/src/include/duckdb/storage/buffer/buffer_pool.hpp index 9b9b3fd78d5f..57762b63d557 100644 --- a/src/include/duckdb/storage/buffer/buffer_pool.hpp +++ b/src/include/duckdb/storage/buffer/buffer_pool.hpp @@ -48,7 +48,7 @@ class BufferPool { //! blocks can be evicted void SetLimit(idx_t limit, const char *exception_postscript); - void IncreaseUsedMemory(MemoryTag tag, idx_t size); + void UpdateUsedMemory(MemoryTag tag, int64_t size); idx_t GetUsedMemory() const; diff --git a/src/include/duckdb/storage/compression/chimp/algorithm/bit_reader.hpp b/src/include/duckdb/storage/compression/chimp/algorithm/bit_reader.hpp index 57512f27be5f..a2910e63e46a 100644 --- a/src/include/duckdb/storage/compression/chimp/algorithm/bit_reader.hpp +++ b/src/include/duckdb/storage/compression/chimp/algorithm/bit_reader.hpp @@ -150,7 +150,7 @@ struct BitReader { result = result << 8 | InnerReadByte(i); } result = result << remainder | InnerRead(remainder, bytes); - index += (bytes << 3) + remainder; + index += static_cast(bytes << 3) + remainder; return result; } diff --git a/src/storage/buffer/buffer_pool.cpp b/src/storage/buffer/buffer_pool.cpp index 0b427b227306..a7741b268a4d 100644 --- a/src/storage/buffer/buffer_pool.cpp +++ b/src/storage/buffer/buffer_pool.cpp @@ -66,9 +66,16 @@ bool BufferPool::AddToEvictionQueue(shared_ptr &handle) { return false; } -void BufferPool::IncreaseUsedMemory(MemoryTag tag, idx_t size) { - current_memory += size; - memory_usage_per_tag[uint8_t(tag)] += size; +void BufferPool::UpdateUsedMemory(MemoryTag tag, int64_t size) { + if (size < 0) { + current_memory -= UnsafeNumericCast(-size); + memory_usage_per_tag[uint8_t(tag)] -= UnsafeNumericCast(-size); + } else { + current_memory += UnsafeNumericCast(size); + ; + memory_usage_per_tag[uint8_t(tag)] += UnsafeNumericCast(size); + ; + } } idx_t BufferPool::GetUsedMemory() const { diff --git a/src/storage/buffer/buffer_pool_reservation.cpp b/src/storage/buffer/buffer_pool_reservation.cpp index f22a96ffa58f..783a98079fb6 100644 --- a/src/storage/buffer/buffer_pool_reservation.cpp +++ b/src/storage/buffer/buffer_pool_reservation.cpp @@ -23,8 +23,8 @@ BufferPoolReservation::~BufferPoolReservation() { } void BufferPoolReservation::Resize(idx_t new_size) { - int64_t delta = (int64_t)new_size - size; - pool.IncreaseUsedMemory(tag, delta); + auto delta = UnsafeNumericCast(new_size) - UnsafeNumericCast(size); + pool.UpdateUsedMemory(tag, delta); size = new_size; } diff --git a/src/storage/compression/bitpacking.cpp b/src/storage/compression/bitpacking.cpp index efdd03087bc8..03472f4cd5d3 100644 --- a/src/storage/compression/bitpacking.cpp +++ b/src/storage/compression/bitpacking.cpp @@ -5,6 +5,7 @@ #include "duckdb/common/operator/add.hpp" #include "duckdb/common/operator/multiply.hpp" #include "duckdb/common/operator/subtract.hpp" +#include "duckdb/common/operator/cast_operators.hpp" #include "duckdb/function/compression/compression.hpp" #include "duckdb/function/compression_function.hpp" #include "duckdb/main/config.hpp" @@ -804,8 +805,10 @@ void BitpackingScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t T *target_ptr = result_data + result_offset + scanned; for (idx_t i = 0; i < to_scan; i++) { - target_ptr[i] = (static_cast(scan_state.current_group_offset + i) * scan_state.current_constant) + - scan_state.current_frame_of_reference; + T multiplier; + auto success = TryCast::Operation(scan_state.current_group_offset + i, multiplier); + D_ASSERT(success); + target_ptr[i] = (multiplier * scan_state.current_constant) + scan_state.current_frame_of_reference; } scanned += to_scan; @@ -890,16 +893,17 @@ void BitpackingFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t r } if (scan_state.current_group.mode == BitpackingMode::CONSTANT_DELTA) { + T multiplier; + auto cast = TryCast::Operation(scan_state.current_group_offset, multiplier); + D_ASSERT(cast); #ifdef DEBUG // overflow check T result; - bool multiply = TryMultiplyOperator::Operation(static_cast(scan_state.current_group_offset), - scan_state.current_constant, result); + bool multiply = TryMultiplyOperator::Operation(multiplier, scan_state.current_constant, result); bool add = TryAddOperator::Operation(result, scan_state.current_frame_of_reference, result); D_ASSERT(multiply && add); #endif - *current_result_ptr = (static_cast(scan_state.current_group_offset) * scan_state.current_constant) + - scan_state.current_frame_of_reference; + *current_result_ptr = (multiplier * scan_state.current_constant) + scan_state.current_frame_of_reference; return; } From df84ecac5f1ba28ba685f7bab0cd8606a4b0eff5 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 10 Apr 2024 13:55:07 +0200 Subject: [PATCH 285/603] Basics of getting a list as nullstr options --- data/csv/null/multiple_nulls.csv | 4 ++ .../scanner/string_value_scanner.cpp | 59 ++++++++++++------- .../csv_scanner/util/csv_reader_options.cpp | 24 +++++++- src/function/table/copy_csv.cpp | 26 ++++---- src/function/table/read_csv.cpp | 2 +- .../csv_scanner/csv_reader_options.hpp | 4 +- .../csv_scanner/string_value_scanner.hpp | 6 +- .../duckdb/storage/serialization/nodes.json | 2 +- src/storage/serialization/serialize_nodes.cpp | 4 +- test/sql/copy/csv/csv_nullstr_list.test | 25 ++++++++ 10 files changed, 113 insertions(+), 43 deletions(-) create mode 100644 data/csv/null/multiple_nulls.csv create mode 100644 test/sql/copy/csv/csv_nullstr_list.test diff --git a/data/csv/null/multiple_nulls.csv b/data/csv/null/multiple_nulls.csv new file mode 100644 index 000000000000..661bf8a6d5e0 --- /dev/null +++ b/data/csv/null/multiple_nulls.csv @@ -0,0 +1,4 @@ +name,age,height +Pedro,31,1.73 +Mark,null, +Thijs,26,none diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 9582e1c1af2f..daf095e83c6c 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -18,7 +18,6 @@ StringValueResult::StringValueResult(CSVStates &states, CSVStateMachine &state_m : ScannerResult(states, state_machine), number_of_columns(NumericCast(state_machine.dialect_options.num_cols)), null_padding(state_machine.options.null_padding), ignore_errors(state_machine.options.ignore_errors), - null_str_ptr(state_machine.options.null_str.c_str()), null_str_size(state_machine.options.null_str.size()), result_size(result_size_p), error_handler(error_hander_p), iterator(iterator_p), store_line_size(store_line_size_p), csv_file_scan(std::move(csv_file_scan_p)), lines_read(lines_read_p), sniffing(sniffing_p) { @@ -88,6 +87,15 @@ StringValueResult::StringValueResult(CSVStates &states, CSVStateMachine &state_m vector_ptr.push_back(FlatVector::GetData(col)); validity_mask.push_back(&FlatVector::Validity(col)); } + + // Setup the NullStr information + null_str_count = state_machine.options.null_str.size(); + null_str_ptr = make_unsafe_uniq_array(null_str_count); + null_str_size = make_unsafe_uniq_array(null_str_count); + for (idx_t i = 0; i < null_str_count; i++) { + null_str_ptr[i] = state_machine.options.null_str[i].c_str(); + null_str_size[i] = state_machine.options.null_str[i].size(); + } } StringValueResult::~StringValueResult() { @@ -114,7 +122,11 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size bool error = true; if (cur_col_id == number_of_columns && ((quoted && state_machine.options.allow_quoted_nulls) || !quoted)) { // we make an exception if the first over-value is null - error = !IsValueNull(null_str_ptr, value_ptr, size); + bool is_value_null = false; + for (idx_t i = 0; i < null_str_count; i++) { + is_value_null = is_value_null || IsValueNull(null_str_ptr[i], value_ptr, size); + } + error = !is_value_null; } if (error) { HandleOverLimitRows(); @@ -129,32 +141,35 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size return; } } - if (size == null_str_size) { - if (((quoted && state_machine.options.allow_quoted_nulls) || !quoted)) { - if (IsValueNull(null_str_ptr, value_ptr, size)) { - bool empty = false; - if (chunk_col_id < state_machine.options.force_not_null.size()) { - empty = state_machine.options.force_not_null[chunk_col_id]; - } - if (empty) { - if (parse_types[chunk_col_id].first != LogicalTypeId::VARCHAR) { - // If it is not a varchar, empty values are not accepted, we must error. - cast_errors[chunk_col_id] = std::string(""); + for (idx_t i = 0; i < null_str_count; i++) { + if (size == null_str_size[i]) { + if (((quoted && state_machine.options.allow_quoted_nulls) || !quoted)) { + if (IsValueNull(null_str_ptr[i], value_ptr, size)) { + bool empty = false; + if (chunk_col_id < state_machine.options.force_not_null.size()) { + empty = state_machine.options.force_not_null[chunk_col_id]; } - static_cast(vector_ptr[chunk_col_id])[number_of_rows] = string_t(); - } else { - if (chunk_col_id == number_of_columns) { - // We check for a weird case, where we ignore an extra value, if it is a null value - return; + if (empty) { + if (parse_types[chunk_col_id].first != LogicalTypeId::VARCHAR) { + // If it is not a varchar, empty values are not accepted, we must error. + cast_errors[chunk_col_id] = std::string(""); + } + static_cast(vector_ptr[chunk_col_id])[number_of_rows] = string_t(); + } else { + if (chunk_col_id == number_of_columns) { + // We check for a weird case, where we ignore an extra value, if it is a null value + return; + } + validity_mask[chunk_col_id]->SetInvalid(number_of_rows); } - validity_mask[chunk_col_id]->SetInvalid(number_of_rows); + cur_col_id++; + chunk_col_id++; + return; } - cur_col_id++; - chunk_col_id++; - return; } } } + bool success = true; switch (parse_types[chunk_col_id].first) { case LogicalTypeId::TINYINT: diff --git a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp index 836ddd731827..b7fd88e7f306 100644 --- a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp +++ b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp @@ -277,8 +277,28 @@ bool CSVReaderOptions::SetBaseOption(const string &loption, const Value &value) SetEscape(ParseString(value, loption)); } else if (loption == "header") { SetHeader(ParseBoolean(value, loption)); - } else if (loption == "null" || loption == "nullstr") { - null_str = ParseString(value, loption); + } else if (loption == "nullstr" || loption == "null") { + auto &child_type = value.type(); + null_str.clear(); + if (child_type.id() != LogicalTypeId::LIST && child_type.id() != LogicalTypeId::VARCHAR) { + throw BinderException("read_csv %s option requires a string or a list as input", loption); + } + if (!sql_type_list.empty()) { + throw BinderException("read_csv_auto nullstr can only be supplied once"); + } + if (child_type.id() == LogicalTypeId::LIST) { + auto &list_child = ListType::GetChildType(child_type); + if (list_child.id() != LogicalTypeId::VARCHAR) { + throw BinderException("read_csv_auto %s requires a list of types (varchar) as input", loption); + } + auto &children = ListValue::GetChildren(value); + for (auto &child : children) { + null_str.push_back(StringValue::Get(child)); + } + } else { + null_str.push_back(StringValue::Get(ParseString(value, loption))); + } + } else if (loption == "encoding") { auto encoding = StringUtil::Lower(ParseString(value, loption)); if (encoding != "utf8" && encoding != "utf-8") { diff --git a/src/function/table/copy_csv.cpp b/src/function/table/copy_csv.cpp index 1fa2e46e7694..8d1d50b3558a 100644 --- a/src/function/table/copy_csv.cpp +++ b/src/function/table/copy_csv.cpp @@ -61,17 +61,20 @@ void BaseCSVData::Finalize() { AreOptionsEqual(options.dialect_options.state_machine_options.quote.GetValue(), options.dialect_options.state_machine_options.escape.GetValue(), "QUOTE", "ESCAPE"); } - if (!options.null_str.empty()) { - // null string and delimiter must not be substrings of each other - SubstringDetection(options.dialect_options.state_machine_options.delimiter.GetValue(), options.null_str, - "DELIMITER", "NULL"); - // quote/escape and nullstr must not be substrings of each other - SubstringDetection(options.dialect_options.state_machine_options.quote.GetValue(), options.null_str, "QUOTE", - "NULL"); + // null string and delimiter must not be substrings of each other + for (auto &null_str : options.null_str) { + if (!null_str.empty()) { + SubstringDetection(options.dialect_options.state_machine_options.delimiter.GetValue(), null_str, + "DELIMITER", "NULL"); - SubstringDetection(options.dialect_options.state_machine_options.escape.GetValue(), options.null_str, "ESCAPE", - "NULL"); + // quote/escape and nullstr must not be substrings of each other + SubstringDetection(options.dialect_options.state_machine_options.quote.GetValue(), null_str, "QUOTE", + "NULL"); + + SubstringDetection(options.dialect_options.state_machine_options.escape.GetValue(), null_str, "ESCAPE", + "NULL"); + } } if (!options.prefix.empty() || !options.suffix.empty()) { @@ -194,7 +197,7 @@ static string AddEscapes(char to_be_escaped, const char escape, const string &va static bool RequiresQuotes(WriteCSVData &csv_data, const char *str, idx_t len) { auto &options = csv_data.options; // check if the string is equal to the null string - if (len == options.null_str.size() && memcmp(str, options.null_str.c_str(), len) == 0) { + if (len == options.null_str[0].size() && memcmp(str, options.null_str[0].c_str(), len) == 0) { return true; } auto str_data = reinterpret_cast(str); @@ -393,13 +396,14 @@ static void WriteCSVChunkInternal(ClientContext &context, FunctionData &bind_dat writer.WriteData(const_data_ptr_cast(csv_data.newline.c_str()), csv_data.newline.size()); } // write values + D_ASSERT(options.null_str.size() == 1); for (idx_t col_idx = 0; col_idx < cast_chunk.ColumnCount(); col_idx++) { if (col_idx != 0) { WriteQuoteOrEscape(writer, options.dialect_options.state_machine_options.delimiter.GetValue()); } if (FlatVector::IsNull(cast_chunk.data[col_idx], row_idx)) { // write null value - writer.WriteData(const_data_ptr_cast(options.null_str.c_str()), options.null_str.size()); + writer.WriteData(const_data_ptr_cast(options.null_str[0].c_str()), options.null_str[0].size()); continue; } diff --git a/src/function/table/read_csv.cpp b/src/function/table/read_csv.cpp index 8d2e1be0d780..fc6819a4fc20 100644 --- a/src/function/table/read_csv.cpp +++ b/src/function/table/read_csv.cpp @@ -234,7 +234,7 @@ void ReadCSVTableFunction::ReadCSVAddNamedParameters(TableFunction &table_functi table_function.named_parameters["quote"] = LogicalType::VARCHAR; table_function.named_parameters["new_line"] = LogicalType::VARCHAR; table_function.named_parameters["escape"] = LogicalType::VARCHAR; - table_function.named_parameters["nullstr"] = LogicalType::VARCHAR; + table_function.named_parameters["nullstr"] = LogicalType::ANY; table_function.named_parameters["columns"] = LogicalType::ANY; table_function.named_parameters["auto_type_candidates"] = LogicalType::ANY; table_function.named_parameters["header"] = LogicalType::BOOLEAN; diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp index 2f1bd3ea27c3..78a9c062d7c6 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp @@ -51,8 +51,8 @@ struct CSVReaderOptions { vector rejects_recovery_column_ids; //! Number of samples to buffer idx_t buffer_sample_size = (idx_t)STANDARD_VECTOR_SIZE * 50; - //! Specifies the string that represents a null value - string null_str; + //! Specifies the strings that represents a null value + vector null_str = {""}; //! Whether file is compressed or not, and if so which compression type //! AUTO_DETECT (default; infer from file extension) FileCompressionType compression = FileCompressionType::AUTO_DETECT; diff --git a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp index 35a4ca4aa48c..2a91622745a2 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp @@ -69,8 +69,10 @@ class StringValueResult : public ScannerResult { const uint32_t number_of_columns; const bool null_padding; const bool ignore_errors; - const char *null_str_ptr; - const idx_t null_str_size; + + unsafe_unique_array null_str_ptr; + unsafe_unique_array null_str_size; + idx_t null_str_count; //! Internal Data Chunk used for flushing DataChunk parse_chunk; diff --git a/src/include/duckdb/storage/serialization/nodes.json b/src/include/duckdb/storage/serialization/nodes.json index 48140d01a680..42dfb2f01e1b 100644 --- a/src/include/duckdb/storage/serialization/nodes.json +++ b/src/include/duckdb/storage/serialization/nodes.json @@ -545,7 +545,7 @@ }, {"id": 102, "name": "null_str", - "type": "string" + "type": "vector" }, {"id": 103, "name": "compression", diff --git a/src/storage/serialization/serialize_nodes.cpp b/src/storage/serialization/serialize_nodes.cpp index 44531988a2cd..a6430b361e0d 100644 --- a/src/storage/serialization/serialize_nodes.cpp +++ b/src/storage/serialization/serialize_nodes.cpp @@ -120,7 +120,7 @@ CSVOption CSVOption::Deserialize(Deserializer &deserializer) { void CSVReaderOptions::Serialize(Serializer &serializer) const { serializer.WritePropertyWithDefault(100, "ignore_errors", ignore_errors); serializer.WritePropertyWithDefault(101, "buffer_sample_size", buffer_sample_size); - serializer.WritePropertyWithDefault(102, "null_str", null_str); + serializer.WritePropertyWithDefault>(102, "null_str", null_str); serializer.WriteProperty(103, "compression", compression); serializer.WritePropertyWithDefault(104, "allow_quoted_nulls", allow_quoted_nulls); serializer.WritePropertyWithDefault(105, "maximum_line_size", maximum_line_size); @@ -156,7 +156,7 @@ CSVReaderOptions CSVReaderOptions::Deserialize(Deserializer &deserializer) { CSVReaderOptions result; deserializer.ReadPropertyWithDefault(100, "ignore_errors", result.ignore_errors); deserializer.ReadPropertyWithDefault(101, "buffer_sample_size", result.buffer_sample_size); - deserializer.ReadPropertyWithDefault(102, "null_str", result.null_str); + deserializer.ReadPropertyWithDefault>(102, "null_str", result.null_str); deserializer.ReadProperty(103, "compression", result.compression); deserializer.ReadPropertyWithDefault(104, "allow_quoted_nulls", result.allow_quoted_nulls); deserializer.ReadPropertyWithDefault(105, "maximum_line_size", result.maximum_line_size); diff --git a/test/sql/copy/csv/csv_nullstr_list.test b/test/sql/copy/csv/csv_nullstr_list.test new file mode 100644 index 000000000000..1ca27c095834 --- /dev/null +++ b/test/sql/copy/csv/csv_nullstr_list.test @@ -0,0 +1,25 @@ +# name: test/sql/copy/csv/csv_nullstr_list.test +# description: Test CSVs with multiple nullstring values +# group: [csv] + +statement ok +PRAGMA enable_verification + + +# Test List +query III +FROM read_csv('data/csv/null/multiple_nulls.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'name': 'VARCHAR', 'age': 'VARCHAR', 'height': 'VARCHAR'}, nullstr = ['','none','null']); +---- +Pedro 31 1.73 +Mark NULL NULL +Thijs 26 NULL + +# Test From Copy + +# Test Quoted + +# Test where the last value is null over different strings + +# Check that write only supports ONE null string + +# Test white spaces \ No newline at end of file From 4fad019905a4482fde4aa9fb901dfa2266fce3df Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Wed, 10 Apr 2024 13:57:24 +0200 Subject: [PATCH 286/603] Add a useful comment --- test/sql/attach/attach_dbname_quotes.test | 1 + 1 file changed, 1 insertion(+) diff --git a/test/sql/attach/attach_dbname_quotes.test b/test/sql/attach/attach_dbname_quotes.test index 2c4157dcb1ac..c231f066682a 100644 --- a/test/sql/attach/attach_dbname_quotes.test +++ b/test/sql/attach/attach_dbname_quotes.test @@ -13,6 +13,7 @@ CREATE TABLE "my""db".tbl(i int); statement ok INSERT INTO "my""db".tbl VALUES (42) +# use with a table name in quotes statement ok USE "my""db"; From 75aeb028914a9b58f3eb8c43691b035cfbceeaf2 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 10 Apr 2024 13:58:31 +0200 Subject: [PATCH 287/603] table creation is too much for the CI, time to count --- .../copy/csv/test_multiple_big_compressed_csvs.test_slow | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/test/sql/copy/csv/test_multiple_big_compressed_csvs.test_slow b/test/sql/copy/csv/test_multiple_big_compressed_csvs.test_slow index 4c8116b7d3cd..4b0fa97dc92c 100644 --- a/test/sql/copy/csv/test_multiple_big_compressed_csvs.test_slow +++ b/test/sql/copy/csv/test_multiple_big_compressed_csvs.test_slow @@ -19,8 +19,8 @@ DROP TABLE lineitem; statement ok CREATE TABLE lineitem(l_orderkey INTEGER NOT NULL, l_partkey INTEGER NOT NULL, l_suppkey INTEGER NOT NULL, l_linenumber INTEGER NOT NULL, l_quantity DECIMAL(15,2) NOT NULL, l_extendedprice DECIMAL(15,2) NOT NULL, l_discount DECIMAL(15,2) NOT NULL, l_tax DECIMAL(15,2) NOT NULL, l_returnflag VARCHAR NOT NULL, l_linestatus VARCHAR NOT NULL, l_shipdate DATE NOT NULL, l_commitdate DATE NOT NULL, l_receiptdate DATE NOT NULL, l_shipinstruct VARCHAR NOT NULL, l_shipmode VARCHAR NOT NULL, l_comment VARCHAR NOT NULL); -statement ok -INSERT INTO lineitem FROM read_csv([ +query I +select count(*) from read_csv([ '__TEST_DIR__/lineitem.csv.gz', '__TEST_DIR__/lineitem.csv.gz', '__TEST_DIR__/lineitem.csv.gz', @@ -46,8 +46,5 @@ INSERT INTO lineitem FROM read_csv([ '__TEST_DIR__/lineitem.csv.gz', '__TEST_DIR__/lineitem.csv.gz', ]); - -query I -select count(*) from lineitem ---- 144029160 \ No newline at end of file From 4bfe3f3fe3020e754d278ea39911eded230b66e0 Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Wed, 10 Apr 2024 14:06:22 +0200 Subject: [PATCH 288/603] add test --- src/optimizer/optimizer.cpp | 12 ++-- test/optimizer/arithmetic_simplification.test | 66 +++++++------------ test/optimizer/topn/complex_top_n.test | 43 ++++++++++++ test/optimizer/{ => topn}/topn_optimizer.test | 2 +- 4 files changed, 75 insertions(+), 48 deletions(-) create mode 100644 test/optimizer/topn/complex_top_n.test rename test/optimizer/{ => topn}/topn_optimizer.test (94%) diff --git a/src/optimizer/optimizer.cpp b/src/optimizer/optimizer.cpp index ce20ca7d8a5f..b4bbd37ce7d9 100644 --- a/src/optimizer/optimizer.cpp +++ b/src/optimizer/optimizer.cpp @@ -153,6 +153,12 @@ unique_ptr Optimizer::Optimize(unique_ptr plan cse_optimizer.VisitOperator(*plan); }); + // transform ORDER BY + LIMIT to TopN + RunOptimizer(OptimizerType::TOP_N, [&]() { + TopN topn; + plan = topn.Optimize(std::move(plan)); + }); + // creates projection maps so unused columns are projected out early RunOptimizer(OptimizerType::COLUMN_LIFETIME, [&]() { ColumnLifetimeAnalyzer column_lifetime(true); @@ -179,12 +185,6 @@ unique_ptr Optimizer::Optimize(unique_ptr plan column_lifetime.VisitOperator(*plan); }); - // transform ORDER BY + LIMIT to TopN - RunOptimizer(OptimizerType::TOP_N, [&]() { - TopN topn; - plan = topn.Optimize(std::move(plan)); - }); - // apply simple expression heuristics to get an initial reordering RunOptimizer(OptimizerType::REORDER_FILTER, [&]() { ExpressionHeuristics expression_heuristics(*this); diff --git a/test/optimizer/arithmetic_simplification.test b/test/optimizer/arithmetic_simplification.test index f55e0031a6d1..f3bdd71f9d9c 100644 --- a/test/optimizer/arithmetic_simplification.test +++ b/test/optimizer/arithmetic_simplification.test @@ -1,48 +1,32 @@ # name: test/optimizer/arithmetic_simplification.test -# description: Arithmetic simplification test +# description: topN # group: [optimizer] statement ok -CREATE TABLE test(X INTEGER); +attach 'appian.duckdb' as appian; statement ok -PRAGMA explain_output = OPTIMIZED_ONLY; - -# verify that nop arithmetic is flattened -query I nosort xnorm -EXPLAIN SELECT X FROM test ----- - -query I nosort xnorm -EXPLAIN SELECT X+0 FROM test ----- - -query I nosort xnorm -EXPLAIN SELECT 0+X FROM test ----- - -query I nosort xnorm -EXPLAIN SELECT X-0 FROM test ----- - -query I nosort xnorm -EXPLAIN SELECT X*1 FROM test ----- - -query I nosort xnorm -EXPLAIN SELECT 1*X FROM test ----- - -query I nosort xnorm -EXPLAIN SELECT X//1 FROM test ----- - -# division by zero results in a NULL -query I nosort xnull -EXPLAIN SELECT NULL FROM test ----- - -query I nosort xnull -EXPLAIN SELECT X//0 FROM test ----- +use appian; +statement ok +WITH CTE AS ( + SELECT J1P, CUSTOMER_PRIORITY, CUSTOMER_ID FROM CUSTOMERVIEW + LEFT JOIN ( + SELECT ORDER_CUSTOMERID, SUM(ORDERITEMVIEW.ORDERITEM_QUANTITY) AS J1P FROM ORDERVIEW + LEFT JOIN ORDERITEMVIEW ON (ORDERVIEW.ORDER_ID = ORDERITEMVIEW.ORDERITEM_ORDERID) + WHERE (ORDERVIEW.ORDER_ISEXPEDITEDSHIPPED IS TRUE) + GROUP BY ORDERVIEW.ORDER_CUSTOMERID + ) AS J1J ON (J1J.ORDER_CUSTOMERID = CUSTOMERVIEW.CUSTOMER_ID) + ORDER BY CUSTOMER_PRIORITY ASC + LIMIT 50 OFFSET 50 +) SELECT J1P, Q2P, Q3P FROM CTE +LEFT JOIN ( + SELECT ORDER_CUSTOMERID FROM ORDERVIEW +) AS Q1J ON (Q1J.ORDER_CUSTOMERID = CTE.CUSTOMER_ID) +LEFT JOIN ( + SELECT CREDITCARD_CUSTOMERID AS Q2P FROM CREDITCARDVIEW +) AS Q2J ON (Q2J.Q2P = CTE.CUSTOMER_ID) +LEFT JOIN ( + SELECT ORDER_CUSTOMERID Q3P FROM ORDERVIEW + LEFT JOIN ORDERITEMVIEW ON ORDERVIEW.ORDER_ID = ORDERITEM_ORDERID +) AS Q3J ON (Q3J.Q3P = CTE.CUSTOMER_ID); diff --git a/test/optimizer/topn/complex_top_n.test b/test/optimizer/topn/complex_top_n.test new file mode 100644 index 000000000000..db3d7556ca69 --- /dev/null +++ b/test/optimizer/topn/complex_top_n.test @@ -0,0 +1,43 @@ +# name: test/optimizer/topn/complex_top_n.test +# description: topN +# group: [optimizer] + +statement ok +SELECT SETSEED(0.42); + +statement ok +create table CUSTOMERVIEW as select range customer_id, range*random()::INT % 3 as customer_priority from range(1000, 2000); + +statement ok +create table OrderView as select range order_id, ((random()*2::INT)%2)::BOOL order_isExpeditedShipped, range + (random() * 3)::INT order_customerId from range(1000, 2000); + +statement ok +create table OrderItemView as select random()*25 orderItem_quantity, range orderItem_orderId from range(1000, 2000); + +statement ok +create table CREDITCARDVIEW as select range CREDITCARD_CUSTOMERID from range(1000, 2000); + +query III +WITH CTE AS ( + SELECT J1P, CUSTOMER_PRIORITY, CUSTOMER_ID FROM CUSTOMERVIEW + LEFT JOIN ( + SELECT ORDER_CUSTOMERID, SUM(ORDERITEMVIEW.ORDERITEM_QUANTITY) AS J1P FROM ORDERVIEW + LEFT JOIN ORDERITEMVIEW ON (ORDERVIEW.ORDER_ID = ORDERITEMVIEW.ORDERITEM_ORDERID) + WHERE (ORDERVIEW.ORDER_ISEXPEDITEDSHIPPED IS TRUE) + GROUP BY ORDERVIEW.ORDER_CUSTOMERID + ) AS J1J ON (J1J.ORDER_CUSTOMERID = CUSTOMERVIEW.CUSTOMER_ID) + ORDER BY CUSTOMER_PRIORITY ASC + LIMIT 50 OFFSET 50 +) SELECT J1P, Q2P, Q3P FROM CTE +LEFT JOIN ( + SELECT ORDER_CUSTOMERID FROM ORDERVIEW +) AS Q1J ON (Q1J.ORDER_CUSTOMERID = CTE.CUSTOMER_ID) +LEFT JOIN ( + SELECT CREDITCARD_CUSTOMERID AS Q2P FROM CREDITCARDVIEW +) AS Q2J ON (Q2J.Q2P = CTE.CUSTOMER_ID) +LEFT JOIN ( + SELECT ORDER_CUSTOMERID Q3P FROM ORDERVIEW + LEFT JOIN ORDERITEMVIEW ON ORDERVIEW.ORDER_ID = ORDERITEM_ORDERID +) AS Q3J ON (Q3J.Q3P = CTE.CUSTOMER_ID); +---- +432 values hashing to c51b3f7dd78a68c95de3f44866394cfb diff --git a/test/optimizer/topn_optimizer.test b/test/optimizer/topn/topn_optimizer.test similarity index 94% rename from test/optimizer/topn_optimizer.test rename to test/optimizer/topn/topn_optimizer.test index ceb042b6ef49..e899e1c7e974 100644 --- a/test/optimizer/topn_optimizer.test +++ b/test/optimizer/topn/topn_optimizer.test @@ -1,4 +1,4 @@ -# name: test/optimizer/topn_optimizer.test +# name: test/optimizer/topn/opn_optimizer.test # description: Test Top N optimization # group: [optimizer] From 6a188b987f566a7825c3443c81022253166c62dd Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 10 Apr 2024 14:10:12 +0200 Subject: [PATCH 289/603] Fix check for empty lines --- .../scanner/string_value_scanner.cpp | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index daf095e83c6c..98d1215b14f6 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -498,17 +498,19 @@ bool StringValueResult::EmptyLine(StringValueResult &result, const idx_t buffer_ result.last_position++; } if (result.number_of_columns == 1) { - if (result.null_str_size == 0) { - bool empty = false; - if (!result.state_machine.options.force_not_null.empty()) { - empty = result.state_machine.options.force_not_null[0]; - } - if (empty) { - static_cast(result.vector_ptr[0])[result.number_of_rows] = string_t(); - } else { - result.validity_mask[0]->SetInvalid(result.number_of_rows); + for (idx_t i = 0; i < result.null_str_count; i++) { + if (result.null_str_size[i] == 0) { + bool empty = false; + if (!result.state_machine.options.force_not_null.empty()) { + empty = result.state_machine.options.force_not_null[0]; + } + if (empty) { + static_cast(result.vector_ptr[0])[result.number_of_rows] = string_t(); + } else { + result.validity_mask[0]->SetInvalid(result.number_of_rows); + } + result.number_of_rows++; } - result.number_of_rows++; } if (result.number_of_rows >= result.result_size) { // We have a full chunk From df62014929d7de833d6b3c2e815bb679d0fed04f Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Wed, 10 Apr 2024 14:20:59 +0200 Subject: [PATCH 290/603] make format-fix, and fix old test case --- test/optimizer/arithmetic_simplification.test | 66 ++++++++++++------- test/optimizer/topn/complex_top_n.test | 4 +- test/optimizer/topn/topn_optimizer.test | 4 +- 3 files changed, 45 insertions(+), 29 deletions(-) diff --git a/test/optimizer/arithmetic_simplification.test b/test/optimizer/arithmetic_simplification.test index f3bdd71f9d9c..f55e0031a6d1 100644 --- a/test/optimizer/arithmetic_simplification.test +++ b/test/optimizer/arithmetic_simplification.test @@ -1,32 +1,48 @@ # name: test/optimizer/arithmetic_simplification.test -# description: topN +# description: Arithmetic simplification test # group: [optimizer] statement ok -attach 'appian.duckdb' as appian; +CREATE TABLE test(X INTEGER); statement ok -use appian; +PRAGMA explain_output = OPTIMIZED_ONLY; + +# verify that nop arithmetic is flattened +query I nosort xnorm +EXPLAIN SELECT X FROM test +---- + +query I nosort xnorm +EXPLAIN SELECT X+0 FROM test +---- + +query I nosort xnorm +EXPLAIN SELECT 0+X FROM test +---- + +query I nosort xnorm +EXPLAIN SELECT X-0 FROM test +---- + +query I nosort xnorm +EXPLAIN SELECT X*1 FROM test +---- + +query I nosort xnorm +EXPLAIN SELECT 1*X FROM test +---- + +query I nosort xnorm +EXPLAIN SELECT X//1 FROM test +---- + +# division by zero results in a NULL +query I nosort xnull +EXPLAIN SELECT NULL FROM test +---- + +query I nosort xnull +EXPLAIN SELECT X//0 FROM test +---- -statement ok -WITH CTE AS ( - SELECT J1P, CUSTOMER_PRIORITY, CUSTOMER_ID FROM CUSTOMERVIEW - LEFT JOIN ( - SELECT ORDER_CUSTOMERID, SUM(ORDERITEMVIEW.ORDERITEM_QUANTITY) AS J1P FROM ORDERVIEW - LEFT JOIN ORDERITEMVIEW ON (ORDERVIEW.ORDER_ID = ORDERITEMVIEW.ORDERITEM_ORDERID) - WHERE (ORDERVIEW.ORDER_ISEXPEDITEDSHIPPED IS TRUE) - GROUP BY ORDERVIEW.ORDER_CUSTOMERID - ) AS J1J ON (J1J.ORDER_CUSTOMERID = CUSTOMERVIEW.CUSTOMER_ID) - ORDER BY CUSTOMER_PRIORITY ASC - LIMIT 50 OFFSET 50 -) SELECT J1P, Q2P, Q3P FROM CTE -LEFT JOIN ( - SELECT ORDER_CUSTOMERID FROM ORDERVIEW -) AS Q1J ON (Q1J.ORDER_CUSTOMERID = CTE.CUSTOMER_ID) -LEFT JOIN ( - SELECT CREDITCARD_CUSTOMERID AS Q2P FROM CREDITCARDVIEW -) AS Q2J ON (Q2J.Q2P = CTE.CUSTOMER_ID) -LEFT JOIN ( - SELECT ORDER_CUSTOMERID Q3P FROM ORDERVIEW - LEFT JOIN ORDERITEMVIEW ON ORDERVIEW.ORDER_ID = ORDERITEM_ORDERID -) AS Q3J ON (Q3J.Q3P = CTE.CUSTOMER_ID); diff --git a/test/optimizer/topn/complex_top_n.test b/test/optimizer/topn/complex_top_n.test index db3d7556ca69..a26f364ae06c 100644 --- a/test/optimizer/topn/complex_top_n.test +++ b/test/optimizer/topn/complex_top_n.test @@ -1,6 +1,6 @@ # name: test/optimizer/topn/complex_top_n.test -# description: topN -# group: [optimizer] +# description: topN +# group: [topn] statement ok SELECT SETSEED(0.42); diff --git a/test/optimizer/topn/topn_optimizer.test b/test/optimizer/topn/topn_optimizer.test index e899e1c7e974..96e0c9ec1c60 100644 --- a/test/optimizer/topn/topn_optimizer.test +++ b/test/optimizer/topn/topn_optimizer.test @@ -1,6 +1,6 @@ -# name: test/optimizer/topn/opn_optimizer.test +# name: test/optimizer/topn/topn_optimizer.test # description: Test Top N optimization -# group: [optimizer] +# group: [topn] statement ok CREATE TABLE integers(i INTEGER, j INTEGER) From 132acb2e17a5975b669179abb9259ca720864a64 Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 10 Apr 2024 14:39:41 +0200 Subject: [PATCH 291/603] merged --- src/include/duckdb/storage/table_storage_info.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/include/duckdb/storage/table_storage_info.hpp b/src/include/duckdb/storage/table_storage_info.hpp index 711c1e2fb114..49b3089c657a 100644 --- a/src/include/duckdb/storage/table_storage_info.hpp +++ b/src/include/duckdb/storage/table_storage_info.hpp @@ -8,7 +8,6 @@ #pragma once -#include "duckdb/common/optional_idx.hpp" #include "duckdb/common/types/value.hpp" #include "duckdb/common/unordered_map.hpp" #include "duckdb/storage/block.hpp" From 7a5678f9594420d8c1f30dc98824299840cf6652 Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Wed, 10 Apr 2024 14:46:07 +0200 Subject: [PATCH 292/603] check for topn in test --- test/optimizer/topn/complex_top_n.test | 35 ++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/test/optimizer/topn/complex_top_n.test b/test/optimizer/topn/complex_top_n.test index a26f364ae06c..04d51fe117b3 100644 --- a/test/optimizer/topn/complex_top_n.test +++ b/test/optimizer/topn/complex_top_n.test @@ -6,16 +6,16 @@ statement ok SELECT SETSEED(0.42); statement ok -create table CUSTOMERVIEW as select range customer_id, range*random()::INT % 3 as customer_priority from range(1000, 2000); +create table CUSTOMERVIEW as select range customer_id, range*random()::INT % 3 as customer_priority from range(1000, 4000); statement ok -create table OrderView as select range order_id, ((random()*2::INT)%2)::BOOL order_isExpeditedShipped, range + (random() * 3)::INT order_customerId from range(1000, 2000); +create table OrderView as select range order_id, ((random()*2::INT)%2)::BOOL order_isExpeditedShipped, range + (random() * 3)::INT order_customerId from range(1000, 4000); statement ok -create table OrderItemView as select random()*25 orderItem_quantity, range orderItem_orderId from range(1000, 2000); +create table OrderItemView as select random()*25 orderItem_quantity, range orderItem_orderId from range(1000, 4000); statement ok -create table CREDITCARDVIEW as select range CREDITCARD_CUSTOMERID from range(1000, 2000); +create table CREDITCARDVIEW as select range CREDITCARD_CUSTOMERID from range(1000, 4000); query III WITH CTE AS ( @@ -40,4 +40,29 @@ LEFT JOIN ( LEFT JOIN ORDERITEMVIEW ON ORDERVIEW.ORDER_ID = ORDERITEM_ORDERID ) AS Q3J ON (Q3J.Q3P = CTE.CUSTOMER_ID); ---- -432 values hashing to c51b3f7dd78a68c95de3f44866394cfb +423 values hashing to 88bbd750b435b7616e6596774a8d5689 + +query II +explain WITH CTE AS ( + SELECT J1P, CUSTOMER_PRIORITY, CUSTOMER_ID FROM CUSTOMERVIEW + LEFT JOIN ( + SELECT ORDER_CUSTOMERID, SUM(ORDERITEMVIEW.ORDERITEM_QUANTITY) AS J1P FROM ORDERVIEW + LEFT JOIN ORDERITEMVIEW ON (ORDERVIEW.ORDER_ID = ORDERITEMVIEW.ORDERITEM_ORDERID) + WHERE (ORDERVIEW.ORDER_ISEXPEDITEDSHIPPED IS TRUE) + GROUP BY ORDERVIEW.ORDER_CUSTOMERID + ) AS J1J ON (J1J.ORDER_CUSTOMERID = CUSTOMERVIEW.CUSTOMER_ID) + ORDER BY CUSTOMER_PRIORITY ASC + LIMIT 50 OFFSET 50 +) SELECT J1P, Q2P, Q3P FROM CTE +LEFT JOIN ( + SELECT ORDER_CUSTOMERID FROM ORDERVIEW +) AS Q1J ON (Q1J.ORDER_CUSTOMERID = CTE.CUSTOMER_ID) +LEFT JOIN ( + SELECT CREDITCARD_CUSTOMERID AS Q2P FROM CREDITCARDVIEW +) AS Q2J ON (Q2J.Q2P = CTE.CUSTOMER_ID) +LEFT JOIN ( + SELECT ORDER_CUSTOMERID Q3P FROM ORDERVIEW + LEFT JOIN ORDERITEMVIEW ON ORDERVIEW.ORDER_ID = ORDERITEM_ORDERID +) AS Q3J ON (Q3J.Q3P = CTE.CUSTOMER_ID); +---- +physical_plan :.*TOP_N.* From e8cbb9f5f4e40ad461697c6e57853392665df483 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 10 Apr 2024 15:02:52 +0200 Subject: [PATCH 293/603] Bump postgres to latest main --- .github/config/out_of_tree_extensions.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/config/out_of_tree_extensions.cmake b/.github/config/out_of_tree_extensions.cmake index 5d1795ef1168..e766c56004f1 100644 --- a/.github/config/out_of_tree_extensions.cmake +++ b/.github/config/out_of_tree_extensions.cmake @@ -67,7 +67,7 @@ if (NOT MINGW) duckdb_extension_load(postgres_scanner DONT_LINK GIT_URL https://github.com/duckdb/postgres_scanner - GIT_TAG 375710fd22a35107b2c28e744f787e1a93a99998 + GIT_TAG 96206f41d5ca7015920a66b54e936c986fe0b0f8 ) endif() From 432b97983ef6eba4d60201cd469dbb5de385ddbf Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 10 Apr 2024 15:04:49 +0200 Subject: [PATCH 294/603] Enable arrow build also for windows (already enabled in nightly) --- .github/config/out_of_tree_extensions.cmake | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/config/out_of_tree_extensions.cmake b/.github/config/out_of_tree_extensions.cmake index e766c56004f1..0f919af9b1e8 100644 --- a/.github/config/out_of_tree_extensions.cmake +++ b/.github/config/out_of_tree_extensions.cmake @@ -16,13 +16,11 @@ # VCPKG_TARGET_TRIPLET=arm64-osx ################# ARROW -if (NOT WIN32) - duckdb_extension_load(arrow - LOAD_TESTS DONT_LINK - GIT_URL https://github.com/duckdb/arrow - GIT_TAG 9e10240da11f61ea7fbfe3fc9988ffe672ccd40f - ) -endif() +duckdb_extension_load(arrow + LOAD_TESTS DONT_LINK + GIT_URL https://github.com/duckdb/arrow + GIT_TAG 9e10240da11f61ea7fbfe3fc9988ffe672ccd40f + ) ################## AWS if (NOT MINGW) From e9d973aeb4586feebbad3e708cec30d72b5c387d Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 10 Apr 2024 16:07:36 +0200 Subject: [PATCH 295/603] wip --- .../pythonpkg/scripts/connection_methods.json | 749 ------------------ tools/pythonpkg/src/pyconnection.cpp | 5 +- 2 files changed, 4 insertions(+), 750 deletions(-) diff --git a/tools/pythonpkg/scripts/connection_methods.json b/tools/pythonpkg/scripts/connection_methods.json index dedaef4cb967..c53f2b2d47dd 100644 --- a/tools/pythonpkg/scripts/connection_methods.json +++ b/tools/pythonpkg/scripts/connection_methods.json @@ -103,754 +103,5 @@ } ], "return": "DuckDBPyConnection" - }, - - { - "name": ["sqltype", "dtype", "type"], - "function": "Type", - "docs": "Create a type object by parsing the 'type_str' string", - "args": [ - { - "name": "type_str" - } - ], - "return": - }, - { - "name": "array_type", - "function": "ArrayType", - "docs": "Create an array type object of 'type'", - "args": [ - - ] - { - "name": "type" - .none(false) - }, - { - "name": "size" - .none(false - }, - }, - { - "name": "list_type", - "function": "ListType", - "docs": "Create a list type object of 'type'", - "args": [ - - ] - { - "name": "type" - .none(false - }, - }, - { - "name": "union_type", - "function": "UnionType", - "docs": "Create a union type object from 'members'", - "args": [ - - ] - { - "name": "members" - .none(false) - }, - }, - { - "name": "string_type", - "function": "StringType", - "docs": "Create a string type with an optional collation", - "args": [ - - ] - { - "name": "collation" - = string() - }, - }, - { - "name": "enum_type", - "function": "EnumType", - "docs": "Create an enum type of underlying 'type', consisting of the list of 'values'", - "args": [ - - ] - { - "name": "name" - }, - { - "name": "type" - }, - { - "name": "values" - }, - }, - { - "name": "decimal_type", - "function": "DecimalType", - "docs": "Create a decimal type with 'width' and 'scale'", - "args": [ - - ] - { - "name": "width" - }, - { - "name": "scale" - }, - }, - { - "name": ["struct_type", "row_type"], - "function": "StructType", - "docs": "Create a struct type object from 'fields'", - { - "name": "fields" - }, - }, - { - "name": "map_type", - "function": "MapType", - "docs": "Create a map type object from 'key_type' and 'value_type'", - "args": [ - - ] - { - "name": "key" - .none(false) - }, - { - "name": "value" - .none(false) - }, - }, - { - "name": "duplicate", - "function": "Cursor", - "docs": "Create a duplicate of the current connection", - }, - { - "name": "execute", - "function": "Execute", - "docs": "Execute the given SQL query, optionally using prepared statements with parameters set", - "args": [ - - ] - { - "name": "query" - }, - { - "name": "parameters" - = py::none() - }, - { - "name": "multiple_parameter_sets" - = false - }, - }, - { - "name": "executemany", - "function": "ExecuteMany", - "docs": "Execute the given prepared statement multiple times using the list of parameter sets in parameters", - "args": [ - - ] - { - "name": "query" - }, - { - "name": "parameters" - = py::none() - }, - }, - { - "name": "close", - "function": "Close", - "docs": "Close the connection" - }, - { - "name": "interrupt", - "function": "Interrupt", - "docs": "Interrupt pending operations" - }, - { - "name": "fetchone", - "function": "FetchOne", - "docs": "Fetch a single row from a result following execute" - }, - { - "name": "fetchmany", - "function": "FetchMany", - "docs": "Fetch the next set of rows from a result following execute" - { - "name": "size" - = 1 - }, - }, - { - "name": "fetchall", - "function": "FetchAll", - "docs": "Fetch all rows from a result following execute" - }, - { - "name": "fetchnumpy", - "function": "FetchNumpy", - "docs": "Fetch a result as list of NumPy arrays following execute" - }, - { - "name": "fetchdf", - "function": "FetchDF", - "docs": "Fetch a result as DataFrame following execute()", - py::kw_only(), - { - "name": "date_as_object" - = false - }, - }, - { - "name": "fetch_df", - "function": "FetchDF", - "docs": "Fetch a result as DataFrame following execute()", - py::kw_only(), - { - "name": "date_as_object" - = false - }, - }, - { - "name": "fetch_df_chunk", - "function": "FetchDFChunk", - "docs": "Fetch a chunk of the result as Data.Frame following execute()", - "args": [ - - ] - { - "name": "vectors_per_chunk" - = 1 - }, - py::kw_only(), - { - "name": "date_as_object" - = false - }, - }, - { - "name": "df", - "function": "FetchDF", - "docs": "Fetch a result as DataFrame following execute()", - py::kw_only(), - { - "name": "date_as_object" - = false - }, - }, - { - "name": "pl", - "function": "FetchPolars", - "docs": "Fetch a result as Polars DataFrame following execute()", - "args": [ - - ] - { - "name": "rows_per_batch" - = 1000000 - }, - }, - { - "name": "fetch_arrow_table", - "function": "FetchArrow", - "docs": "Fetch a result as Arrow table following execute()", - "args": [ - - ] - { - "name": "rows_per_batch" - = 1000000 - }, - }, - { - "name": "fetch_record_batch", - "function": "FetchRecordBatchReader", - "docs": "Fetch an Arrow RecordBatchReader following execute()", - "args": [ - - ] - { - "name": "rows_per_batch" - = 1000000 - }, - }, - { - "name": "arrow", - "function": "FetchArrow", - "docs": "Fetch a result as Arrow table following execute()", - "args": [ - - ] - { - "name": "rows_per_batch" - = 1000000 - }, - }, - { - "name": "torch", - "function": "FetchPyTorch", - "docs": "Fetch a result as dict of PyTorch Tensors following execute()" - - }, - { - "name": "tf", - "function": "FetchTF", - "docs": "Fetch a result as dict of TensorFlow Tensors following execute()" - - }, - { - "name": "begin", - "function": "Begin", - "docs": "Start a new transaction" - - }, - { - "name": "commit", - "function": "Commit", - "docs": "Commit changes performed within a transaction" - }, - { - "name": "rollback", - "function": "Rollback", - "docs": "Roll back changes performed within a transaction" - }, - { - "name": "append", - "function": "Append", - "docs": "Append the passed DataFrame to the named table", - "args": [ - - ] - { - "name": "table_name" - }, - { - "name": "df" - }, - py::kw_only(), - { - "name": "by_name" - = false - }, - }, - { - "name": "register", - "function": "RegisterPythonObject", - "docs": "Register the passed Python Object value for querying with a view", - "args": [ - - ] - { - "name": "view_name" - }, - { - "name": "python_object" - }, - }, - { - "name": "unregister", - "function": "UnregisterPythonObject", - "docs": "Unregister the view name", - "args": [ - - ] - { - "name": "view_name" - }, - }, - { - "name": "table", - "function": "Table", - "docs": "Create a relation object for the name'd table", - "args": [ - - ] - { - "name": "table_name" - }, - }, - { - "name": "view", - "function": "View", - "docs": "Create a relation object for the name'd view", - "args": [ - - ] - { - "name": "view_name" - }, - }, - { - "name": "values", - "function": "Values", - "docs": "Create a relation object from the passed values", - "args": [ - - ] - { - "name": "values" - }, - }, - { - "name": "table_function", - "function": "TableFunction", - "docs": "Create a relation object from the name'd table function with given parameters", - "args": [ - - ] - { - "name": "name" - }, - { - "name": "parameters" - = py::none() - }, - }, - { - "name": "read_json", - "function": "ReadJSON", - "docs": "Create a relation object from the JSON file in 'name'", - "args": [ - - ] - { - "name": "name" - }, - py::kw_only(), - { - "name": "columns" - = py::none() - }, - { - "name": "sample_size" - = py::none() - }, - { - "name": "maximum_depth" - = py::none() - }, - { - "name": "records" - = py::none() - }, - { - "name": "format" - = py::none() - }, - }, - { - "name": "extract_statements", - "function": "ExtractStatements", - "docs": "Parse the query string and extract the Statement object(s) produced", - "args": [ - - ] - { - "name": "query" - }, - }, - - { - "name": ["sql", "query", "from_query"], - "function": "RunQuery", - "docs": "Run a SQL query. If it is a SELECT statement, create a relation object from the given SQL query, otherwise run the query as-is.", - "args": [ - - ] - { - "name": "query" - }, - py::kw_only(), - { - "name": "alias" - = "" - }, - { - "name": "params" - = py::none( - }, - }, - - { - "name": ["read_csv", "from_csv_auto"], - "function": "ReadCSV", - "docs": "Create a relation object from the CSV file in 'name'", - "args": [ - - ] - { - "name": "name" - }, - py::kw_only(), - { - "name": "header" - = py::none() - }, - { - "name": "compression" - = py::none() - }, - { - "name": "sep" - = py::none() - }, - { - "name": "delimiter" - = py::none() - }, - { - "name": "dtype" - = py::none() - }, - { - "name": "na_values" - = py::none() - }, - { - "name": "skiprows" - = py::none() - }, - { - "name": "quotechar" - = py::none() - }, - { - "name": "escapechar" - = py::none() - }, - { - "name": "encoding" - = py::none() - }, - { - "name": "parallel" - = py::none() - }, - { - "name": "date_format" - = py::none() - }, - { - "name": "timestamp_format" - = py::none() - }, - { - "name": "sample_size" - = py::none() - }, - { - "name": "all_varchar" - = py::none() - }, - { - "name": "normalize_names" - = py::none() - }, - { - "name": "filename" - = py::none() - }, - { - "name": "null_padding" - = py::none() - }, - { - "name": "names" - = py::none( - }, - }, - { - "name": "from_df", - "function": "FromDF", - "docs": "Create a relation object from the Data.Frame in df", - "args": [ - - ] - { - "name": "df" - = py::none() - }, - }, - { - "name": "from_arrow", - "function": "FromArrow", - "docs": "Create a relation object from an Arrow object", - "args": [ - - ] - { - "name": "arrow_object" - }, - }, - - { - "name": ["from_parquet", "read_parquet"], - "function": "FromParquet", - "docs": "Create a relation object from the Parquet files in file_glob", - "args": [ - - ] - { - "name": "file_glob" - }, - { - "name": "binary_as_string" - = false - }, - py::kw_only(), - { - "name": "file_row_number" - = false - }, - { - "name": "filename" - = false - }, - { - "name": "hive_partitioning" - = false - }, - { - "name": "union_by_name" - = false - }, - { - "name": "compression" - = py::none( - }, - }, - { - "name": ["from_parquet", "read_parquet"], - "function": "FromParquets", - "docs": "Create a relation object from the Parquet files in file_globs", - "args": [ - - ] - { - "name": "file_globs" - }, - { - "name": "binary_as_string" - = false - }, - py::kw_only(), - { - "name": "file_row_number" - = false - }, - { - "name": "filename" - = false - }, - { - "name": "hive_partitioning" - = false - }, - { - "name": "union_by_name" - = false - }, - { - "name": "compression" - = py::none( - }, - }, - { - "name": "from_substrait", - "function": "FromSubstrait", - "docs": "Create a query object from protobuf plan", - "args": [ - - ] - { - "name": "proto" - }, - }, - { - "name": "get_substrait", - "function": "GetSubstrait", - "docs": "Serialize a query to protobuf", - "args": [ - - ] - { - "name": "query" - }, - py::kw_only(), - { - "name": "enable_optimizer" - = true - }, - }, - { - "name": "get_substrait_json", - "function": "GetSubstraitJSON", - "docs": "Serialize a query to protobuf on the JSON format", - "args": [ - - ] - { - "name": "query" - }, - py::kw_only(), - { - "name": "enable_optimizer" - = true - }, - }, - { - "name": "from_substrait_json", - "function": "FromSubstraitJSON", - "docs": "Create a query object from a JSON protobuf plan", - "args": [ - - ] - { - "name": "json" - }, - }, - { - "name": "get_table_names", - "function": "GetTableNames", - "docs": "Extract the required table names from a query", - "args": [ - - ] - { - "name": "query" - }, - }, - - { - "name": "install_extension", - "function": "InstallExtension", - "docs": "Install an extension by name", - "args": [ - - ] - { - "name": "extension" - }, - py::kw_only(), - { - "name": "force_install" - = false - }, - }, - { - "name": "load_extension", - "function": "LoadExtension", - "docs": "Load an installed extension", - "args": [ - - ] - { - "name": "extension" - }, } ] diff --git a/tools/pythonpkg/src/pyconnection.cpp b/tools/pythonpkg/src/pyconnection.cpp index f4cd9100a142..2de161bc3d25 100644 --- a/tools/pythonpkg/src/pyconnection.cpp +++ b/tools/pythonpkg/src/pyconnection.cpp @@ -119,6 +119,9 @@ py::object ArrowTableFromDataframe(const py::object &df) { } } +// NOTE: this function is generated by tools/pythonpkg/scripts/generate_connection_methods.py. +// Do not edit this function manually, your changes will be overwritten! + static void InitializeConnectionMethods(py::class_> &m) { m.def("cursor", &DuckDBPyConnection::Cursor, "Create a duplicate of the current connection") .def("register_filesystem", &DuckDBPyConnection::RegisterFilesystem, "Register a fsspec compliant filesystem", @@ -266,7 +269,7 @@ static void InitializeConnectionMethods(py::class_GetFileSystem(); From 073fd1b184b8dbb7358fd1f139e5540f3d63d90d Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 10 Apr 2024 16:11:17 +0200 Subject: [PATCH 296/603] start of script to generate the definitions inside the DuckDBPyConnection class --- .../scripts/generate_connection_methods.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 tools/pythonpkg/scripts/generate_connection_methods.py diff --git a/tools/pythonpkg/scripts/generate_connection_methods.py b/tools/pythonpkg/scripts/generate_connection_methods.py new file mode 100644 index 000000000000..84bdc5a0ae59 --- /dev/null +++ b/tools/pythonpkg/scripts/generate_connection_methods.py @@ -0,0 +1,48 @@ +import os +import json + +os.chdir(os.path.dirname(__file__)) + +JSON_PATH = os.path.join("connection_methods.json") +PYCONNECTION_SOURCE = os.path.join("..", "src", "pyconnection.cpp") + +INITIALIZE_METHOD = ( + "static void InitializeConnectionMethods(py::class_> &m) {" +) +END_MARKER = "} // END_OF_CONNECTION_METHODS" + +# Read the PYCONNECTION_SOURCE file +with open(PYCONNECTION_SOURCE, 'r') as source_file: + source_code = source_file.readlines() + +# Locate the InitializeConnectionMethods function in it +start_index = -1 +end_index = -1 +for i, line in enumerate(source_code): + if line.startswith(INITIALIZE_METHOD): + start_index = i + elif line.startswith(END_MARKER): + end_index = i + +if start_index == -1 or end_index == -1: + raise ValueError("Couldn't find start or end marker in source file") + +start_section = source_code[: start_index + 1] +end_section = source_code[end_index:] + +# Generate the definition code from the json +# Read the JSON file +with open(JSON_PATH, 'r') as json_file: + connection_methods = json.load(json_file) + +regenerated_method = [] +regenerated_method.extend(['', '']) + +with_newlines = [x + '\n' for x in regenerated_method] +# Recreate the file content by concatenating all the pieces together + +new_content = start_section + with_newlines + end_section + +# Write out the modified PYCONNECTION_SOURCE file +with open(PYCONNECTION_SOURCE, 'w') as source_file: + source_file.write("".join(new_content)) From 0220a8d4fc65e433c85fcb6e4596da76d8b1e88a Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 10 Apr 2024 16:13:49 +0200 Subject: [PATCH 297/603] More tests --- data/csv/null/mix_multiple_quoted_nulls.csv | 0 data/csv/null/multiple_quoted_nulls.csv | 4 ++++ test/sql/copy/csv/csv_nullstr_list.test | 19 ++++++++++++++++++- 3 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 data/csv/null/mix_multiple_quoted_nulls.csv create mode 100644 data/csv/null/multiple_quoted_nulls.csv diff --git a/data/csv/null/mix_multiple_quoted_nulls.csv b/data/csv/null/mix_multiple_quoted_nulls.csv new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/data/csv/null/multiple_quoted_nulls.csv b/data/csv/null/multiple_quoted_nulls.csv new file mode 100644 index 000000000000..0912448ed70b --- /dev/null +++ b/data/csv/null/multiple_quoted_nulls.csv @@ -0,0 +1,4 @@ +name,age,height +Pedro,31,1.73 +"Mark","null","" +Thijs,26,"none" diff --git a/test/sql/copy/csv/csv_nullstr_list.test b/test/sql/copy/csv/csv_nullstr_list.test index 1ca27c095834..95323622426d 100644 --- a/test/sql/copy/csv/csv_nullstr_list.test +++ b/test/sql/copy/csv/csv_nullstr_list.test @@ -14,9 +14,26 @@ Pedro 31 1.73 Mark NULL NULL Thijs 26 NULL +# Test Quoted +query III +FROM read_csv('data/csv/null/multiple_quoted_nulls.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'name': 'VARCHAR', 'age': 'VARCHAR', 'height': 'VARCHAR'}, nullstr = ['','none','null']); +---- +Pedro 31 1.73 +Mark NULL NULL +Thijs 26 NULL + +query III +FROM read_csv('data/csv/null/multiple_quoted_nulls.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'name': 'VARCHAR', 'age': 'VARCHAR', 'height': 'VARCHAR'}, nullstr = ['','none','null'], allow_quoted_nulls = false); +---- +Pedro 31 1.73 +Mark null (empty) +Thijs 26 none + +#allow_quoted_nulls + +# Test Null Strings equal to delim quote escape # Test From Copy -# Test Quoted # Test where the last value is null over different strings From bbf679e0dc92729042c42c3dce9655fb8c2bea6f Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 10 Apr 2024 16:34:44 +0200 Subject: [PATCH 298/603] generate the c++ (pybind) definition code --- .../scripts/generate_connection_methods.py | 53 +++++++++++++++++-- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/tools/pythonpkg/scripts/generate_connection_methods.py b/tools/pythonpkg/scripts/generate_connection_methods.py index 84bdc5a0ae59..1882e6e0afa6 100644 --- a/tools/pythonpkg/scripts/generate_connection_methods.py +++ b/tools/pythonpkg/scripts/generate_connection_methods.py @@ -29,16 +29,61 @@ start_section = source_code[: start_index + 1] end_section = source_code[end_index:] +# ---- Generate the definition code from the json ---- -# Generate the definition code from the json # Read the JSON file with open(JSON_PATH, 'r') as json_file: connection_methods = json.load(json_file) -regenerated_method = [] -regenerated_method.extend(['', '']) +body = [] +body.extend(['', '']) -with_newlines = [x + '\n' for x in regenerated_method] +DEFAULT_ARGUMENT_MAP = {'True': 'true', 'False': 'false', 'None': 'py::none()'} + + +def map_default(val): + if val in DEFAULT_ARGUMENT_MAP: + return DEFAULT_ARGUMENT_MAP[val] + return val + + +for conn in connection_methods: + definition = f"m.def(\"{conn['name']}\"" + definition += ", " + definition += f"""&DuckDBPyConnection::{conn['function']}""" + definition += ", " + definition += f"\"{conn['docs']}\"" + if 'args' in conn: + definition += ", " + arguments = [] + for arg in conn['args']: + argument = f"py::arg(\"{arg['name']}\")" + # Add the default argument if present + if 'default' in arg: + default = map_default(arg['default']) + argument += f" = {default}" + arguments.append(argument) + definition += ', '.join(arguments) + if 'kwargs' in conn: + definition += ", " + definition += "py::kw_only(), " + keyword_arguments = [] + for kwarg in conn['kwargs']: + keyword_argument = f"py::arg(\"{kwarg['name']}\")" + # Add the default argument if present + if 'default' in arg: + default = map_default(arg['default']) + keyword_argument += f" = {default}" + keyword_arguments.append(keyword_argument) + definition += ', '.join(keyword_arguments) + definition += ");" + body.append(definition) + +# ---- End of generation code ---- + +with_newlines = [x + '\n' for x in body] +print(''.join(with_newlines)) +exit() # Recreate the file content by concatenating all the pieces together new_content = start_section + with_newlines + end_section From c6800bd85c5927dded2719d401c30c15cf6aebed Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 10 Apr 2024 16:58:08 +0200 Subject: [PATCH 299/603] extend the list of functions --- .../pythonpkg/scripts/connection_methods.json | 737 ++++++++++++++++++ .../scripts/generate_connection_methods.py | 63 +- 2 files changed, 772 insertions(+), 28 deletions(-) diff --git a/tools/pythonpkg/scripts/connection_methods.json b/tools/pythonpkg/scripts/connection_methods.json index c53f2b2d47dd..39e11bb0ba3c 100644 --- a/tools/pythonpkg/scripts/connection_methods.json +++ b/tools/pythonpkg/scripts/connection_methods.json @@ -103,5 +103,742 @@ } ], "return": "DuckDBPyConnection" + }, + { + "name": ["sqltype", "dtype", "type"], + "function": "Type", + "docs": "Create a type object by parsing the 'type_str' string", + "args": [ + { + "name": "type_str", + "type": "str" + } + ], + "return": "DuckDBPyType" + }, + { + "name": "array_type", + "function": "ArrayType", + "docs": "Create an array type object of 'type'", + "args": [ + { + "name": "type", + "type": "DuckDBPyType", + "allow_none": false + }, + { + "name": "size", + "type": "int" + } + ] + }, + { + "name": "list_type", + "function": "ListType", + "docs": "Create a list type object of 'type'", + "args": [ + { + "name": "type", + "type": "DuckDBPyType", + "allow_none": false + } + ] + }, + { + "name": "union_type", + "function": "UnionType", + "docs": "Create a union type object from 'members'", + "args": [ + { + "name": "members", + "type": "DuckDBPyType", + "allow_none": false + } + ] + }, + { + "name": "string_type", + "function": "StringType", + "docs": "Create a string type with an optional collation", + "args": [ + { + "name": "collation", + "type": "str", + "default": "\"\"" + } + ] + }, + { + "name": "enum_type", + "function": "EnumType", + "docs": "Create an enum type of underlying 'type', consisting of the list of 'values'", + "args": [ + { + "name": "name", + "type": "str" + }, + { + "name": "type", + "type": "DuckDBPyType" + }, + { + "name": "values", + "type": "List[Any]" + } + ] + }, + { + "name": "decimal_type", + "function": "DecimalType", + "docs": "Create a decimal type with 'width' and 'scale'", + "args": [ + { + "name": "width", + "type": "int" + }, + { + "name": "scale", + "type": "int" + } + ] + }, + { + "name": ["struct_type", "row_type"], + "function": "StructType", + "docs": "Create a struct type object from 'fields'", + "args": [ + { + "name": "fields", + "type": "Union[Dict[str, DuckDBPyType], List[str]]" + } + ] + }, + { + "name": "map_type", + "function": "MapType", + "docs": "Create a map type object from 'key_type' and 'value_type'", + "args": [ + { + "name": "key", + "allow_none": false + }, + { + "name": "value", + "allow_none": false + } + ] + }, + { + "name": "duplicate", + "function": "Cursor", + "docs": "Create a duplicate of the current connection" + }, + { + "name": "execute", + "function": "Execute", + "docs": "Execute the given SQL query, optionally using prepared statements with parameters set", + "args": [ + { + "name": "query" + }, + { + "name": "parameters", + "default": "None" + }, + { + "name": "multiple_parameter_sets", + "default": "false" + } + ] + }, + { + "name": "executemany", + "function": "ExecuteMany", + "docs": "Execute the given prepared statement multiple times using the list of parameter sets in parameters", + "args": [ + { + "name": "query" + }, + { + "name": "parameters", + "default": "None" + } + ] + }, + { + "name": "close", + "function": "Close", + "docs": "Close the connection" + }, + { + "name": "interrupt", + "function": "Interrupt", + "docs": "Interrupt pending operations" + }, + { + "name": "fetchone", + "function": "FetchOne", + "docs": "Fetch a single row from a result following execute" + }, + { + "name": "fetchmany", + "function": "FetchMany", + "docs": "Fetch the next set of rows from a result following execute", + "args": [ + { + "name": "size", + "default": "1" + } + ] + }, + { + "name": "fetchall", + "function": "FetchAll", + "docs": "Fetch all rows from a result following execute" + }, + { + "name": "fetchnumpy", + "function": "FetchNumpy", + "docs": "Fetch a result as list of NumPy arrays following execute" + }, + { + "name": "fetchdf", + "function": "FetchDF", + "docs": "Fetch a result as DataFrame following execute()", + "kwargs": [ + { + "name": "date_as_object", + "default": "false" + } + ] + }, + { + "name": "fetch_df", + "function": "FetchDF", + "docs": "Fetch a result as DataFrame following execute()", + "kwargs": [ + { + "name": "date_as_object", + "default": "false" + } + ] + }, + { + "name": "fetch_df_chunk", + "function": "FetchDFChunk", + "docs": "Fetch a chunk of the result as Data.Frame following execute()", + "args": [ + { + "name": "vectors_per_chunk", + "default": "1" + } + ], + "kwargs": [ + { + "name": "date_as_object", + "default": "false" + } + ] + }, + { + "name": "df", + "function": "FetchDF", + "docs": "Fetch a result as DataFrame following execute()", + "kwargs": [ + { + "name": "date_as_object", + "default": "false" + } + ] + }, + { + "name": "pl", + "function": "FetchPolars", + "docs": "Fetch a result as Polars DataFrame following execute()", + "args": [ + { + "name": "rows_per_batch", + "default": "1000000" + } + ] + }, + { + "name": "fetch_arrow_table", + "function": "FetchArrow", + "docs": "Fetch a result as Arrow table following execute()", + "args": [ + { + "name": "rows_per_batch", + "default": "1000000" + } + ] + }, + { + "name": "fetch_record_batch", + "function": "FetchRecordBatchReader", + "docs": "Fetch an Arrow RecordBatchReader following execute()", + "args": [ + { + "name": "rows_per_batch", + "default": "1000000" + } + ] + }, + { + "name": "arrow", + "function": "FetchArrow", + "docs": "Fetch a result as Arrow table following execute()", + "args": [ + { + "name": "rows_per_batch", + "default": "1000000" + } + ] + }, + { + "name": "torch", + "function": "FetchPyTorch", + "docs": "Fetch a result as dict of PyTorch Tensors following execute()" + + }, + { + "name": "tf", + "function": "FetchTF", + "docs": "Fetch a result as dict of TensorFlow Tensors following execute()" + + }, + { + "name": "begin", + "function": "Begin", + "docs": "Start a new transaction" + + }, + { + "name": "commit", + "function": "Commit", + "docs": "Commit changes performed within a transaction" + }, + { + "name": "rollback", + "function": "Rollback", + "docs": "Roll back changes performed within a transaction" + }, + { + "name": "append", + "function": "Append", + "docs": "Append the passed DataFrame to the named table", + "args": [ + { + "name": "table_name" + }, + { + "name": "df" + } + ], + "kwargs": [ + { + "name": "by_name", + "default": "false" + } + ] + }, + { + "name": "register", + "function": "RegisterPythonObject", + "docs": "Register the passed Python Object value for querying with a view", + "args": [ + { + "name": "view_name" + }, + { + "name": "python_object" + } + ] + }, + { + "name": "unregister", + "function": "UnregisterPythonObject", + "docs": "Unregister the view name", + "args": [ + { + "name": "view_name" + } + ] + }, + { + "name": "table", + "function": "Table", + "docs": "Create a relation object for the name'd table", + "args": [ + { + "name": "table_name" + } + ] + }, + { + "name": "view", + "function": "View", + "docs": "Create a relation object for the name'd view", + "args": [ + { + "name": "view_name" + } + ] + }, + { + "name": "values", + "function": "Values", + "docs": "Create a relation object from the passed values", + "args": [ + { + "name": "values" + } + ] + }, + { + "name": "table_function", + "function": "TableFunction", + "docs": "Create a relation object from the name'd table function with given parameters", + "args": [ + { + "name": "name" + }, + { + "name": "parameters", + "default": "None" + } + ] + }, + { + "name": "read_json", + "function": "ReadJSON", + "docs": "Create a relation object from the JSON file in 'name'", + "args": [ + { + "name": "name" + } + ], + "kwargs": [ + { + "name": "columns", + "default": "None" + }, + { + "name": "sample_size", + "default": "None" + }, + { + "name": "maximum_depth", + "default": "None" + }, + { + "name": "records", + "default": "None" + }, + { + "name": "format", + "default": "None" + } + ] + }, + { + "name": "extract_statements", + "function": "ExtractStatements", + "docs": "Parse the query string and extract the Statement object(s) produced", + "args": [ + { + "name": "query" + } + ] + }, + { + "name": ["sql", "query", "from_query"], + "function": "RunQuery", + "docs": "Run a SQL query. If it is a SELECT statement, create a relation object from the given SQL query, otherwise run the query as-is.", + "args": [ + { + "name": "query" + } + ], + "kwargs": [ + { + "name": "alias", + "default": "" + }, + { + "name": "params", + "default": "None" + } + ] + }, + + { + "name": ["read_csv", "from_csv_auto"], + "function": "ReadCSV", + "docs": "Create a relation object from the CSV file in 'name'", + "args": [ + { + "name": "name" + } + ], + "kwargs": [ + { + "name": "header", + "default": "None" + }, + { + "name": "compression", + "default": "None" + }, + { + "name": "sep", + "default": "None" + }, + { + "name": "delimiter", + "default": "None" + }, + { + "name": "dtype", + "default": "None" + }, + { + "name": "na_values", + "default": "None" + }, + { + "name": "skiprows", + "default": "None" + }, + { + "name": "quotechar", + "default": "None" + }, + { + "name": "escapechar", + "default": "None" + }, + { + "name": "encoding", + "default": "None" + }, + { + "name": "parallel", + "default": "None" + }, + { + "name": "date_format", + "default": "None" + }, + { + "name": "timestamp_format", + "default": "None" + }, + { + "name": "sample_size", + "default": "None" + }, + { + "name": "all_varchar", + "default": "None" + }, + { + "name": "normalize_names", + "default": "None" + }, + { + "name": "filename", + "default": "None" + }, + { + "name": "null_padding", + "default": "None" + }, + { + "name": "names", + "default": "None" + } + ] + }, + { + "name": "from_df", + "function": "FromDF", + "docs": "Create a relation object from the Data.Frame in df", + "args": [ + { + "name": "df", + "default": "None" + } + ] + }, + { + "name": "from_arrow", + "function": "FromArrow", + "docs": "Create a relation object from an Arrow object", + "args": [ + { + "name": "arrow_object" + } + ] + }, + { + "name": ["from_parquet", "read_parquet"], + "function": "FromParquet", + "docs": "Create a relation object from the Parquet files in file_glob", + "args": [ + { + "name": "file_glob" + }, + { + "name": "binary_as_string", + "default": "false" + } + ], + "kwargs": [ + { + "name": "file_row_number", + "default": "false" + }, + { + "name": "filename", + "default": "false" + }, + { + "name": "hive_partitioning", + "default": "false" + }, + { + "name": "union_by_name", + "default": "false" + }, + { + "name": "compression", + "default": "None" + } + ] + }, + { + "name": ["from_parquet", "read_parquet"], + "function": "FromParquets", + "docs": "Create a relation object from the Parquet files in file_globs", + "args": [ + { + "name": "file_globs" + }, + { + "name": "binary_as_string", + "default": "false" + } + ], + "kwargs": [ + { + "name": "file_row_number", + "default": "false" + }, + { + "name": "filename", + "default": "false" + }, + { + "name": "hive_partitioning", + "default": "false" + }, + { + "name": "union_by_name", + "default": "false" + }, + { + "name": "compression", + "default": "None" + } + ] + }, + { + "name": "from_substrait", + "function": "FromSubstrait", + "docs": "Create a query object from protobuf plan", + "args": [ + { + "name": "proto" + } + ] + }, + { + "name": "get_substrait", + "function": "GetSubstrait", + "docs": "Serialize a query to protobuf", + "args": [ + { + "name": "query" + } + ], + "kwargs": [ + { + "name": "enable_optimizer", + "default": "True" + } + ] + }, + { + "name": "get_substrait_json", + "function": "GetSubstraitJSON", + "docs": "Serialize a query to protobuf on the JSON format", + "args": [ + { + "name": "query" + } + ], + "kwargs": [ + { + "name": "enable_optimizer", + "default": "True" + } + ] + }, + { + "name": "from_substrait_json", + "function": "FromSubstraitJSON", + "docs": "Create a query object from a JSON protobuf plan", + "args": [ + { + "name": "json" + } + ] + }, + { + "name": "get_table_names", + "function": "GetTableNames", + "docs": "Extract the required table names from a query", + "args": [ + { + "name": "query" + } + ] + }, + { + "name": "install_extension", + "function": "InstallExtension", + "docs": "Install an extension by name", + "args": [ + { + "name": "extension" + } + ], + "kwargs": [ + { + "name": "force_install", + "default": "false" + } + ] + }, + { + "name": "load_extension", + "function": "LoadExtension", + "docs": "Load an installed extension", + "args": [ + { + "name": "extension" + } + ] } ] diff --git a/tools/pythonpkg/scripts/generate_connection_methods.py b/tools/pythonpkg/scripts/generate_connection_methods.py index 1882e6e0afa6..a251c90d4cf9 100644 --- a/tools/pythonpkg/scripts/generate_connection_methods.py +++ b/tools/pythonpkg/scripts/generate_connection_methods.py @@ -48,36 +48,43 @@ def map_default(val): for conn in connection_methods: - definition = f"m.def(\"{conn['name']}\"" - definition += ", " - definition += f"""&DuckDBPyConnection::{conn['function']}""" - definition += ", " - definition += f"\"{conn['docs']}\"" - if 'args' in conn: + if isinstance(conn['name'], list): + names = conn['name'] + else: + names = [conn['name']] + for name in names: + definition = f"m.def(\"{name}\"" definition += ", " - arguments = [] - for arg in conn['args']: - argument = f"py::arg(\"{arg['name']}\")" - # Add the default argument if present - if 'default' in arg: - default = map_default(arg['default']) - argument += f" = {default}" - arguments.append(argument) - definition += ', '.join(arguments) - if 'kwargs' in conn: + definition += f"""&DuckDBPyConnection::{conn['function']}""" definition += ", " - definition += "py::kw_only(), " - keyword_arguments = [] - for kwarg in conn['kwargs']: - keyword_argument = f"py::arg(\"{kwarg['name']}\")" - # Add the default argument if present - if 'default' in arg: - default = map_default(arg['default']) - keyword_argument += f" = {default}" - keyword_arguments.append(keyword_argument) - definition += ', '.join(keyword_arguments) - definition += ");" - body.append(definition) + definition += f"\"{conn['docs']}\"" + if 'args' in conn: + definition += ", " + arguments = [] + for arg in conn['args']: + argument = f"py::arg(\"{arg['name']}\")" + # TODO: add '.none(false)' if required (add 'allow_none' to the JSON) + # Add the default argument if present + if 'default' in arg: + default = map_default(arg['default']) + argument += f" = {default}" + arguments.append(argument) + definition += ', '.join(arguments) + if 'kwargs' in conn: + definition += ", " + definition += "py::kw_only(), " + keyword_arguments = [] + for kwarg in conn['kwargs']: + keyword_argument = f"py::arg(\"{kwarg['name']}\")" + # TODO: add '.none(false)' if required (add 'allow_none' to the JSON) + # Add the default argument if present + if 'default' in arg: + default = map_default(arg['default']) + keyword_argument += f" = {default}" + keyword_arguments.append(keyword_argument) + definition += ', '.join(keyword_arguments) + definition += ");" + body.append(definition) # ---- End of generation code ---- From eb87a36c1e38b6f3e7dcba85d25a17ac67a12d63 Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 10 Apr 2024 17:03:08 +0200 Subject: [PATCH 300/603] more default remapping --- tools/pythonpkg/scripts/generate_connection_methods.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/pythonpkg/scripts/generate_connection_methods.py b/tools/pythonpkg/scripts/generate_connection_methods.py index a251c90d4cf9..c4c73159337a 100644 --- a/tools/pythonpkg/scripts/generate_connection_methods.py +++ b/tools/pythonpkg/scripts/generate_connection_methods.py @@ -36,9 +36,13 @@ connection_methods = json.load(json_file) body = [] -body.extend(['', '']) -DEFAULT_ARGUMENT_MAP = {'True': 'true', 'False': 'false', 'None': 'py::none()'} +DEFAULT_ARGUMENT_MAP = { + 'True': 'true', + 'False': 'false', + 'None': 'py::none()', + 'PythonUDFType.NATIVE': 'PythonUDFType::NATIVE', +} def map_default(val): From 78254624b8f7515905736d0b422705b28504ccee Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 10 Apr 2024 17:10:20 +0200 Subject: [PATCH 301/603] generate the isnone(false) --- .../pythonpkg/scripts/connection_methods.json | 6 +- .../scripts/generate_connection_methods.py | 74 +++++++++---------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/tools/pythonpkg/scripts/connection_methods.json b/tools/pythonpkg/scripts/connection_methods.json index 39e11bb0ba3c..2fc427ca039d 100644 --- a/tools/pythonpkg/scripts/connection_methods.json +++ b/tools/pythonpkg/scripts/connection_methods.json @@ -468,7 +468,7 @@ { "name": "table", "function": "Table", - "docs": "Create a relation object for the name'd table", + "docs": "Create a relation object for the named table", "args": [ { "name": "table_name" @@ -478,7 +478,7 @@ { "name": "view", "function": "View", - "docs": "Create a relation object for the name'd view", + "docs": "Create a relation object for the named view", "args": [ { "name": "view_name" @@ -498,7 +498,7 @@ { "name": "table_function", "function": "TableFunction", - "docs": "Create a relation object from the name'd table function with given parameters", + "docs": "Create a relation object from the named table function with given parameters", "args": [ { "name": "name" diff --git a/tools/pythonpkg/scripts/generate_connection_methods.py b/tools/pythonpkg/scripts/generate_connection_methods.py index c4c73159337a..f28470986d59 100644 --- a/tools/pythonpkg/scripts/generate_connection_methods.py +++ b/tools/pythonpkg/scripts/generate_connection_methods.py @@ -50,45 +50,45 @@ def map_default(val): return DEFAULT_ARGUMENT_MAP[val] return val - -for conn in connection_methods: - if isinstance(conn['name'], list): - names = conn['name'] - else: - names = [conn['name']] - for name in names: - definition = f"m.def(\"{name}\"" +def create_arguments(arguments) -> list: + result = [] + for arg in arguments: + argument = f"py::arg(\"{arg['name']}\")" + if 'allow_none' in arg: + value = str(arg['allow_none']).lower() + argument += f".none({value})" + # Add the default argument if present + if 'default' in arg: + default = map_default(arg['default']) + argument += f" = {default}" + result.append(argument) + return result + +def create_definition(name, method) -> str: + definition = f"m.def(\"{name}\"" + definition += ", " + definition += f"""&DuckDBPyConnection::{method['function']}""" + definition += ", " + definition += f"\"{method['docs']}\"" + if 'args' in method: definition += ", " - definition += f"""&DuckDBPyConnection::{conn['function']}""" + arguments = create_arguments(method['args']) + definition += ', '.join(arguments) + if 'kwargs' in method: definition += ", " - definition += f"\"{conn['docs']}\"" - if 'args' in conn: - definition += ", " - arguments = [] - for arg in conn['args']: - argument = f"py::arg(\"{arg['name']}\")" - # TODO: add '.none(false)' if required (add 'allow_none' to the JSON) - # Add the default argument if present - if 'default' in arg: - default = map_default(arg['default']) - argument += f" = {default}" - arguments.append(argument) - definition += ', '.join(arguments) - if 'kwargs' in conn: - definition += ", " - definition += "py::kw_only(), " - keyword_arguments = [] - for kwarg in conn['kwargs']: - keyword_argument = f"py::arg(\"{kwarg['name']}\")" - # TODO: add '.none(false)' if required (add 'allow_none' to the JSON) - # Add the default argument if present - if 'default' in arg: - default = map_default(arg['default']) - keyword_argument += f" = {default}" - keyword_arguments.append(keyword_argument) - definition += ', '.join(keyword_arguments) - definition += ");" - body.append(definition) + definition += "py::kw_only(), " + arguments = create_arguments(method['kwargs']) + definition += ', '.join(arguments) + definition += ");" + return definition + +for method in connection_methods: + if isinstance(method['name'], list): + names = method['name'] + else: + names = [method['name']] + for name in names: + body.append(create_definition(name, method)) # ---- End of generation code ---- From 8684102b873c326ae7714c2ece43f2ceea32078e Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 10 Apr 2024 17:22:21 +0200 Subject: [PATCH 302/603] correctly generating the pybind definition code --- .../pythonpkg/scripts/connection_methods.json | 2 +- .../scripts/generate_connection_methods.py | 9 +- tools/pythonpkg/src/pyconnection.cpp | 287 ++++++++++-------- 3 files changed, 160 insertions(+), 138 deletions(-) diff --git a/tools/pythonpkg/scripts/connection_methods.json b/tools/pythonpkg/scripts/connection_methods.json index 2fc427ca039d..f944e17625f0 100644 --- a/tools/pythonpkg/scripts/connection_methods.json +++ b/tools/pythonpkg/scripts/connection_methods.json @@ -563,7 +563,7 @@ "kwargs": [ { "name": "alias", - "default": "" + "default": "\"\"" }, { "name": "params", diff --git a/tools/pythonpkg/scripts/generate_connection_methods.py b/tools/pythonpkg/scripts/generate_connection_methods.py index f28470986d59..efcf13304f0d 100644 --- a/tools/pythonpkg/scripts/generate_connection_methods.py +++ b/tools/pythonpkg/scripts/generate_connection_methods.py @@ -42,6 +42,8 @@ 'False': 'false', 'None': 'py::none()', 'PythonUDFType.NATIVE': 'PythonUDFType::NATIVE', + 'PythonExceptionHandling.DEFAULT': 'PythonExceptionHandling::FORWARD_ERROR', + 'FunctionNullHandling.DEFAULT': 'FunctionNullHandling::DEFAULT_NULL_HANDLING', } @@ -50,6 +52,7 @@ def map_default(val): return DEFAULT_ARGUMENT_MAP[val] return val + def create_arguments(arguments) -> list: result = [] for arg in arguments: @@ -64,6 +67,7 @@ def create_arguments(arguments) -> list: result.append(argument) return result + def create_definition(name, method) -> str: definition = f"m.def(\"{name}\"" definition += ", " @@ -82,6 +86,7 @@ def create_definition(name, method) -> str: definition += ");" return definition + for method in connection_methods: if isinstance(method['name'], list): names = method['name'] @@ -92,9 +97,7 @@ def create_definition(name, method) -> str: # ---- End of generation code ---- -with_newlines = [x + '\n' for x in body] -print(''.join(with_newlines)) -exit() +with_newlines = ['\t' + x + '\n' for x in body] # Recreate the file content by concatenating all the pieces together new_content = start_section + with_newlines + end_section diff --git a/tools/pythonpkg/src/pyconnection.cpp b/tools/pythonpkg/src/pyconnection.cpp index 2de161bc3d25..39cf30451f5d 100644 --- a/tools/pythonpkg/src/pyconnection.cpp +++ b/tools/pythonpkg/src/pyconnection.cpp @@ -123,152 +123,168 @@ py::object ArrowTableFromDataframe(const py::object &df) { // Do not edit this function manually, your changes will be overwritten! static void InitializeConnectionMethods(py::class_> &m) { - m.def("cursor", &DuckDBPyConnection::Cursor, "Create a duplicate of the current connection") - .def("register_filesystem", &DuckDBPyConnection::RegisterFilesystem, "Register a fsspec compliant filesystem", - py::arg("filesystem")) - .def("unregister_filesystem", &DuckDBPyConnection::UnregisterFilesystem, "Unregister a filesystem", - py::arg("name")) - .def("list_filesystems", &DuckDBPyConnection::ListFilesystems, - "List registered filesystems, including builtin ones") - .def("filesystem_is_registered", &DuckDBPyConnection::FileSystemIsRegistered, - "Check if a filesystem with the provided name is currently registered", py::arg("name")); - + m.def("cursor", &DuckDBPyConnection::Cursor, "Create a duplicate of the current connection"); + m.def("register_filesystem", &DuckDBPyConnection::RegisterFilesystem, "Register a fsspec compliant filesystem", + py::arg("filesystem")); + m.def("unregister_filesystem", &DuckDBPyConnection::UnregisterFilesystem, "Unregister a filesystem", + py::arg("name")); + m.def("list_filesystems", &DuckDBPyConnection::ListFilesystems, + "List registered filesystems, including builtin ones"); + m.def("filesystem_is_registered", &DuckDBPyConnection::FileSystemIsRegistered, + "Check if a filesystem with the provided name is currently registered", py::arg("name")); m.def("create_function", &DuckDBPyConnection::RegisterScalarUDF, "Create a DuckDB function out of the passing in Python function so it can be used in queries", py::arg("name"), py::arg("function"), py::arg("parameters") = py::none(), py::arg("return_type") = py::none(), - py::kw_only(), py::arg("type") = PythonUDFType::NATIVE, py::arg("null_handling") = 0, - py::arg("exception_handling") = 0, py::arg("side_effects") = false); - + py::kw_only(), py::arg("type") = PythonUDFType::NATIVE, + py::arg("null_handling") = FunctionNullHandling::DEFAULT_NULL_HANDLING, + py::arg("exception_handling") = PythonExceptionHandling::FORWARD_ERROR, py::arg("side_effects") = false); m.def("remove_function", &DuckDBPyConnection::UnregisterUDF, "Remove a previously created function", py::arg("name")); - - DefineMethod({"sqltype", "dtype", "type"}, m, &DuckDBPyConnection::Type, - "Create a type object by parsing the 'type_str' string", py::arg("type_str")); - + m.def("sqltype", &DuckDBPyConnection::Type, "Create a type object by parsing the 'type_str' string", + py::arg("type_str")); + m.def("dtype", &DuckDBPyConnection::Type, "Create a type object by parsing the 'type_str' string", + py::arg("type_str")); + m.def("type", &DuckDBPyConnection::Type, "Create a type object by parsing the 'type_str' string", + py::arg("type_str")); m.def("array_type", &DuckDBPyConnection::ArrayType, "Create an array type object of 'type'", - py::arg("type").none(false), py::arg("size").none(false)); + py::arg("type").none(false), py::arg("size")); m.def("list_type", &DuckDBPyConnection::ListType, "Create a list type object of 'type'", py::arg("type").none(false)); m.def("union_type", &DuckDBPyConnection::UnionType, "Create a union type object from 'members'", - py::arg("members").none(false)) - .def("string_type", &DuckDBPyConnection::StringType, "Create a string type with an optional collation", - py::arg("collation") = string()) - .def("enum_type", &DuckDBPyConnection::EnumType, - "Create an enum type of underlying 'type', consisting of the list of 'values'", py::arg("name"), - py::arg("type"), py::arg("values")) - .def("decimal_type", &DuckDBPyConnection::DecimalType, "Create a decimal type with 'width' and 'scale'", - py::arg("width"), py::arg("scale")); - DefineMethod({"struct_type", "row_type"}, m, &DuckDBPyConnection::StructType, - "Create a struct type object from 'fields'", py::arg("fields")); + py::arg("members").none(false)); + m.def("string_type", &DuckDBPyConnection::StringType, "Create a string type with an optional collation", + py::arg("collation") = ""); + m.def("enum_type", &DuckDBPyConnection::EnumType, + "Create an enum type of underlying 'type', consisting of the list of 'values'", py::arg("name"), + py::arg("type"), py::arg("values")); + m.def("decimal_type", &DuckDBPyConnection::DecimalType, "Create a decimal type with 'width' and 'scale'", + py::arg("width"), py::arg("scale")); + m.def("struct_type", &DuckDBPyConnection::StructType, "Create a struct type object from 'fields'", + py::arg("fields")); + m.def("row_type", &DuckDBPyConnection::StructType, "Create a struct type object from 'fields'", py::arg("fields")); m.def("map_type", &DuckDBPyConnection::MapType, "Create a map type object from 'key_type' and 'value_type'", - py::arg("key").none(false), py::arg("value").none(false)) - .def("duplicate", &DuckDBPyConnection::Cursor, "Create a duplicate of the current connection") - .def("execute", &DuckDBPyConnection::Execute, - "Execute the given SQL query, optionally using prepared statements with parameters set", py::arg("query"), - py::arg("parameters") = py::none(), py::arg("multiple_parameter_sets") = false) - .def("executemany", &DuckDBPyConnection::ExecuteMany, - "Execute the given prepared statement multiple times using the list of parameter sets in parameters", - py::arg("query"), py::arg("parameters") = py::none()) - .def("close", &DuckDBPyConnection::Close, "Close the connection") - .def("interrupt", &DuckDBPyConnection::Interrupt, "Interrupt pending operations") - .def("fetchone", &DuckDBPyConnection::FetchOne, "Fetch a single row from a result following execute") - .def("fetchmany", &DuckDBPyConnection::FetchMany, "Fetch the next set of rows from a result following execute", - py::arg("size") = 1) - .def("fetchall", &DuckDBPyConnection::FetchAll, "Fetch all rows from a result following execute") - .def("fetchnumpy", &DuckDBPyConnection::FetchNumpy, "Fetch a result as list of NumPy arrays following execute") - .def("fetchdf", &DuckDBPyConnection::FetchDF, "Fetch a result as DataFrame following execute()", py::kw_only(), - py::arg("date_as_object") = false) - .def("fetch_df", &DuckDBPyConnection::FetchDF, "Fetch a result as DataFrame following execute()", py::kw_only(), - py::arg("date_as_object") = false) - .def("fetch_df_chunk", &DuckDBPyConnection::FetchDFChunk, - "Fetch a chunk of the result as Data.Frame following execute()", py::arg("vectors_per_chunk") = 1, - py::kw_only(), py::arg("date_as_object") = false) - .def("df", &DuckDBPyConnection::FetchDF, "Fetch a result as DataFrame following execute()", py::kw_only(), - py::arg("date_as_object") = false) - .def("pl", &DuckDBPyConnection::FetchPolars, "Fetch a result as Polars DataFrame following execute()", - py::arg("rows_per_batch") = 1000000) - .def("fetch_arrow_table", &DuckDBPyConnection::FetchArrow, "Fetch a result as Arrow table following execute()", - py::arg("rows_per_batch") = 1000000) - .def("fetch_record_batch", &DuckDBPyConnection::FetchRecordBatchReader, - "Fetch an Arrow RecordBatchReader following execute()", py::arg("rows_per_batch") = 1000000) - .def("arrow", &DuckDBPyConnection::FetchArrow, "Fetch a result as Arrow table following execute()", - py::arg("rows_per_batch") = 1000000) - .def("torch", &DuckDBPyConnection::FetchPyTorch, - "Fetch a result as dict of PyTorch Tensors following execute()") - .def("tf", &DuckDBPyConnection::FetchTF, "Fetch a result as dict of TensorFlow Tensors following execute()") - .def("begin", &DuckDBPyConnection::Begin, "Start a new transaction") - .def("commit", &DuckDBPyConnection::Commit, "Commit changes performed within a transaction") - .def("rollback", &DuckDBPyConnection::Rollback, "Roll back changes performed within a transaction") - .def("append", &DuckDBPyConnection::Append, "Append the passed DataFrame to the named table", - py::arg("table_name"), py::arg("df"), py::kw_only(), py::arg("by_name") = false) - .def("register", &DuckDBPyConnection::RegisterPythonObject, - "Register the passed Python Object value for querying with a view", py::arg("view_name"), - py::arg("python_object")) - .def("unregister", &DuckDBPyConnection::UnregisterPythonObject, "Unregister the view name", - py::arg("view_name")) - .def("table", &DuckDBPyConnection::Table, "Create a relation object for the name'd table", - py::arg("table_name")) - .def("view", &DuckDBPyConnection::View, "Create a relation object for the name'd view", py::arg("view_name")) - .def("values", &DuckDBPyConnection::Values, "Create a relation object from the passed values", - py::arg("values")) - .def("table_function", &DuckDBPyConnection::TableFunction, - "Create a relation object from the name'd table function with given parameters", py::arg("name"), - py::arg("parameters") = py::none()) - .def("read_json", &DuckDBPyConnection::ReadJSON, "Create a relation object from the JSON file in 'name'", - py::arg("name"), py::kw_only(), py::arg("columns") = py::none(), py::arg("sample_size") = py::none(), - py::arg("maximum_depth") = py::none(), py::arg("records") = py::none(), py::arg("format") = py::none()) - .def("extract_statements", &DuckDBPyConnection::ExtractStatements, - "Parse the query string and extract the Statement object(s) produced", py::arg("query")); - - DefineMethod({"sql", "query", "from_query"}, m, &DuckDBPyConnection::RunQuery, - "Run a SQL query. If it is a SELECT statement, create a relation object from the given SQL query, " - "otherwise run the query as-is.", - py::arg("query"), py::kw_only(), py::arg("alias") = "", py::arg("params") = py::none()); - - DefineMethod({"read_csv", "from_csv_auto"}, m, &DuckDBPyConnection::ReadCSV, - "Create a relation object from the CSV file in 'name'", py::arg("name"), py::kw_only(), - py::arg("header") = py::none(), py::arg("compression") = py::none(), py::arg("sep") = py::none(), - py::arg("delimiter") = py::none(), py::arg("dtype") = py::none(), py::arg("na_values") = py::none(), - py::arg("skiprows") = py::none(), py::arg("quotechar") = py::none(), - py::arg("escapechar") = py::none(), py::arg("encoding") = py::none(), py::arg("parallel") = py::none(), - py::arg("date_format") = py::none(), py::arg("timestamp_format") = py::none(), - py::arg("sample_size") = py::none(), py::arg("all_varchar") = py::none(), - py::arg("normalize_names") = py::none(), py::arg("filename") = py::none(), - py::arg("null_padding") = py::none(), py::arg("names") = py::none()); - + py::arg("key").none(false), py::arg("value").none(false)); + m.def("duplicate", &DuckDBPyConnection::Cursor, "Create a duplicate of the current connection"); + m.def("execute", &DuckDBPyConnection::Execute, + "Execute the given SQL query, optionally using prepared statements with parameters set", py::arg("query"), + py::arg("parameters") = py::none(), py::arg("multiple_parameter_sets") = false); + m.def("executemany", &DuckDBPyConnection::ExecuteMany, + "Execute the given prepared statement multiple times using the list of parameter sets in parameters", + py::arg("query"), py::arg("parameters") = py::none()); + m.def("close", &DuckDBPyConnection::Close, "Close the connection"); + m.def("interrupt", &DuckDBPyConnection::Interrupt, "Interrupt pending operations"); + m.def("fetchone", &DuckDBPyConnection::FetchOne, "Fetch a single row from a result following execute"); + m.def("fetchmany", &DuckDBPyConnection::FetchMany, "Fetch the next set of rows from a result following execute", + py::arg("size") = 1); + m.def("fetchall", &DuckDBPyConnection::FetchAll, "Fetch all rows from a result following execute"); + m.def("fetchnumpy", &DuckDBPyConnection::FetchNumpy, "Fetch a result as list of NumPy arrays following execute"); + m.def("fetchdf", &DuckDBPyConnection::FetchDF, "Fetch a result as DataFrame following execute()", py::kw_only(), + py::arg("date_as_object") = false); + m.def("fetch_df", &DuckDBPyConnection::FetchDF, "Fetch a result as DataFrame following execute()", py::kw_only(), + py::arg("date_as_object") = false); + m.def("fetch_df_chunk", &DuckDBPyConnection::FetchDFChunk, + "Fetch a chunk of the result as Data.Frame following execute()", py::arg("vectors_per_chunk") = 1, + py::kw_only(), py::arg("date_as_object") = false); + m.def("df", &DuckDBPyConnection::FetchDF, "Fetch a result as DataFrame following execute()", py::kw_only(), + py::arg("date_as_object") = false); + m.def("pl", &DuckDBPyConnection::FetchPolars, "Fetch a result as Polars DataFrame following execute()", + py::arg("rows_per_batch") = 1000000); + m.def("fetch_arrow_table", &DuckDBPyConnection::FetchArrow, "Fetch a result as Arrow table following execute()", + py::arg("rows_per_batch") = 1000000); + m.def("fetch_record_batch", &DuckDBPyConnection::FetchRecordBatchReader, + "Fetch an Arrow RecordBatchReader following execute()", py::arg("rows_per_batch") = 1000000); + m.def("arrow", &DuckDBPyConnection::FetchArrow, "Fetch a result as Arrow table following execute()", + py::arg("rows_per_batch") = 1000000); + m.def("torch", &DuckDBPyConnection::FetchPyTorch, "Fetch a result as dict of PyTorch Tensors following execute()"); + m.def("tf", &DuckDBPyConnection::FetchTF, "Fetch a result as dict of TensorFlow Tensors following execute()"); + m.def("begin", &DuckDBPyConnection::Begin, "Start a new transaction"); + m.def("commit", &DuckDBPyConnection::Commit, "Commit changes performed within a transaction"); + m.def("rollback", &DuckDBPyConnection::Rollback, "Roll back changes performed within a transaction"); + m.def("append", &DuckDBPyConnection::Append, "Append the passed DataFrame to the named table", + py::arg("table_name"), py::arg("df"), py::kw_only(), py::arg("by_name") = false); + m.def("register", &DuckDBPyConnection::RegisterPythonObject, + "Register the passed Python Object value for querying with a view", py::arg("view_name"), + py::arg("python_object")); + m.def("unregister", &DuckDBPyConnection::UnregisterPythonObject, "Unregister the view name", py::arg("view_name")); + m.def("table", &DuckDBPyConnection::Table, "Create a relation object for the named table", py::arg("table_name")); + m.def("view", &DuckDBPyConnection::View, "Create a relation object for the named view", py::arg("view_name")); + m.def("values", &DuckDBPyConnection::Values, "Create a relation object from the passed values", py::arg("values")); + m.def("table_function", &DuckDBPyConnection::TableFunction, + "Create a relation object from the named table function with given parameters", py::arg("name"), + py::arg("parameters") = py::none()); + m.def("read_json", &DuckDBPyConnection::ReadJSON, "Create a relation object from the JSON file in 'name'", + py::arg("name"), py::kw_only(), py::arg("columns") = py::none(), py::arg("sample_size") = py::none(), + py::arg("maximum_depth") = py::none(), py::arg("records") = py::none(), py::arg("format") = py::none()); + m.def("extract_statements", &DuckDBPyConnection::ExtractStatements, + "Parse the query string and extract the Statement object(s) produced", py::arg("query")); + m.def("sql", &DuckDBPyConnection::RunQuery, + "Run a SQL query. If it is a SELECT statement, create a relation object from the given SQL query, otherwise " + "run the query as-is.", + py::arg("query"), py::kw_only(), py::arg("alias") = "", py::arg("params") = py::none()); + m.def("query", &DuckDBPyConnection::RunQuery, + "Run a SQL query. If it is a SELECT statement, create a relation object from the given SQL query, otherwise " + "run the query as-is.", + py::arg("query"), py::kw_only(), py::arg("alias") = "", py::arg("params") = py::none()); + m.def("from_query", &DuckDBPyConnection::RunQuery, + "Run a SQL query. If it is a SELECT statement, create a relation object from the given SQL query, otherwise " + "run the query as-is.", + py::arg("query"), py::kw_only(), py::arg("alias") = "", py::arg("params") = py::none()); + m.def("read_csv", &DuckDBPyConnection::ReadCSV, "Create a relation object from the CSV file in 'name'", + py::arg("name"), py::kw_only(), py::arg("header") = py::none(), py::arg("compression") = py::none(), + py::arg("sep") = py::none(), py::arg("delimiter") = py::none(), py::arg("dtype") = py::none(), + py::arg("na_values") = py::none(), py::arg("skiprows") = py::none(), py::arg("quotechar") = py::none(), + py::arg("escapechar") = py::none(), py::arg("encoding") = py::none(), py::arg("parallel") = py::none(), + py::arg("date_format") = py::none(), py::arg("timestamp_format") = py::none(), + py::arg("sample_size") = py::none(), py::arg("all_varchar") = py::none(), + py::arg("normalize_names") = py::none(), py::arg("filename") = py::none(), + py::arg("null_padding") = py::none(), py::arg("names") = py::none()); + m.def("from_csv_auto", &DuckDBPyConnection::ReadCSV, "Create a relation object from the CSV file in 'name'", + py::arg("name"), py::kw_only(), py::arg("header") = py::none(), py::arg("compression") = py::none(), + py::arg("sep") = py::none(), py::arg("delimiter") = py::none(), py::arg("dtype") = py::none(), + py::arg("na_values") = py::none(), py::arg("skiprows") = py::none(), py::arg("quotechar") = py::none(), + py::arg("escapechar") = py::none(), py::arg("encoding") = py::none(), py::arg("parallel") = py::none(), + py::arg("date_format") = py::none(), py::arg("timestamp_format") = py::none(), + py::arg("sample_size") = py::none(), py::arg("all_varchar") = py::none(), + py::arg("normalize_names") = py::none(), py::arg("filename") = py::none(), + py::arg("null_padding") = py::none(), py::arg("names") = py::none()); m.def("from_df", &DuckDBPyConnection::FromDF, "Create a relation object from the Data.Frame in df", - py::arg("df") = py::none()) - .def("from_arrow", &DuckDBPyConnection::FromArrow, "Create a relation object from an Arrow object", - py::arg("arrow_object")); - - DefineMethod({"from_parquet", "read_parquet"}, m, &DuckDBPyConnection::FromParquet, - "Create a relation object from the Parquet files in file_glob", py::arg("file_glob"), - py::arg("binary_as_string") = false, py::kw_only(), py::arg("file_row_number") = false, - py::arg("filename") = false, py::arg("hive_partitioning") = false, py::arg("union_by_name") = false, - py::arg("compression") = py::none()); - DefineMethod({"from_parquet", "read_parquet"}, m, &DuckDBPyConnection::FromParquets, - "Create a relation object from the Parquet files in file_globs", py::arg("file_globs"), - py::arg("binary_as_string") = false, py::kw_only(), py::arg("file_row_number") = false, - py::arg("filename") = false, py::arg("hive_partitioning") = false, py::arg("union_by_name") = false, - py::arg("compression") = py::none()); - + py::arg("df") = py::none()); + m.def("from_arrow", &DuckDBPyConnection::FromArrow, "Create a relation object from an Arrow object", + py::arg("arrow_object")); + m.def("from_parquet", &DuckDBPyConnection::FromParquet, + "Create a relation object from the Parquet files in file_glob", py::arg("file_glob"), + py::arg("binary_as_string") = false, py::kw_only(), py::arg("file_row_number") = false, + py::arg("filename") = false, py::arg("hive_partitioning") = false, py::arg("union_by_name") = false, + py::arg("compression") = py::none()); + m.def("read_parquet", &DuckDBPyConnection::FromParquet, + "Create a relation object from the Parquet files in file_glob", py::arg("file_glob"), + py::arg("binary_as_string") = false, py::kw_only(), py::arg("file_row_number") = false, + py::arg("filename") = false, py::arg("hive_partitioning") = false, py::arg("union_by_name") = false, + py::arg("compression") = py::none()); + m.def("from_parquet", &DuckDBPyConnection::FromParquets, + "Create a relation object from the Parquet files in file_globs", py::arg("file_globs"), + py::arg("binary_as_string") = false, py::kw_only(), py::arg("file_row_number") = false, + py::arg("filename") = false, py::arg("hive_partitioning") = false, py::arg("union_by_name") = false, + py::arg("compression") = py::none()); + m.def("read_parquet", &DuckDBPyConnection::FromParquets, + "Create a relation object from the Parquet files in file_globs", py::arg("file_globs"), + py::arg("binary_as_string") = false, py::kw_only(), py::arg("file_row_number") = false, + py::arg("filename") = false, py::arg("hive_partitioning") = false, py::arg("union_by_name") = false, + py::arg("compression") = py::none()); m.def("from_substrait", &DuckDBPyConnection::FromSubstrait, "Create a query object from protobuf plan", - py::arg("proto")) - .def("get_substrait", &DuckDBPyConnection::GetSubstrait, "Serialize a query to protobuf", py::arg("query"), - py::kw_only(), py::arg("enable_optimizer") = true) - .def("get_substrait_json", &DuckDBPyConnection::GetSubstraitJSON, - "Serialize a query to protobuf on the JSON format", py::arg("query"), py::kw_only(), - py::arg("enable_optimizer") = true) - .def("from_substrait_json", &DuckDBPyConnection::FromSubstraitJSON, - "Create a query object from a JSON protobuf plan", py::arg("json")) - .def("get_table_names", &DuckDBPyConnection::GetTableNames, "Extract the required table names from a query", - py::arg("query")) - .def_property_readonly("description", &DuckDBPyConnection::GetDescription, - "Get result set attributes, mainly column names") - .def_property_readonly("rowcount", &DuckDBPyConnection::GetRowcount, "Get result set row count") - .def("install_extension", &DuckDBPyConnection::InstallExtension, "Install an extension by name", - py::arg("extension"), py::kw_only(), py::arg("force_install") = false) - .def("load_extension", &DuckDBPyConnection::LoadExtension, "Load an installed extension", py::arg("extension")); + py::arg("proto")); + m.def("get_substrait", &DuckDBPyConnection::GetSubstrait, "Serialize a query to protobuf", py::arg("query"), + py::kw_only(), py::arg("enable_optimizer") = true); + m.def("get_substrait_json", &DuckDBPyConnection::GetSubstraitJSON, + "Serialize a query to protobuf on the JSON format", py::arg("query"), py::kw_only(), + py::arg("enable_optimizer") = true); + m.def("from_substrait_json", &DuckDBPyConnection::FromSubstraitJSON, + "Create a query object from a JSON protobuf plan", py::arg("json")); + m.def("get_table_names", &DuckDBPyConnection::GetTableNames, "Extract the required table names from a query", + py::arg("query")); + m.def("install_extension", &DuckDBPyConnection::InstallExtension, "Install an extension by name", + py::arg("extension"), py::kw_only(), py::arg("force_install") = false); + m.def("load_extension", &DuckDBPyConnection::LoadExtension, "Load an installed extension", py::arg("extension")); } // END_OF_CONNECTION_METHODS void DuckDBPyConnection::UnregisterFilesystem(const py::str &name) { @@ -396,6 +412,9 @@ void DuckDBPyConnection::Initialize(py::handle &m) { .def("__exit__", &DuckDBPyConnection::Exit, py::arg("exc_type"), py::arg("exc"), py::arg("traceback")); InitializeConnectionMethods(connection_module); + connection_module.def_property_readonly("description", &DuckDBPyConnection::GetDescription, + "Get result set attributes, mainly column names"); + connection_module.def_property_readonly("rowcount", &DuckDBPyConnection::GetRowcount, "Get result set row count"); PyDateTime_IMPORT; // NOLINT DuckDBPyConnection::ImportCache(); } From d698881eba0f51acff6b5751ab585c7e67be18d9 Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 10 Apr 2024 17:29:14 +0200 Subject: [PATCH 303/603] add markers to the stubs --- tools/pythonpkg/duckdb-stubs/__init__.pyi | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tools/pythonpkg/duckdb-stubs/__init__.pyi b/tools/pythonpkg/duckdb-stubs/__init__.pyi index 1f1efa86078f..b575bd039660 100644 --- a/tools/pythonpkg/duckdb-stubs/__init__.pyi +++ b/tools/pythonpkg/duckdb-stubs/__init__.pyi @@ -251,6 +251,19 @@ def FunctionExpression(function: str, *cols: Expression) -> Expression: ... class DuckDBPyConnection: def __init__(self, *args, **kwargs) -> None: ... + def __enter__(self) -> DuckDBPyConnection: ... + def __exit__(self, exc_type: object, exc: object, traceback: object) -> None: ... + @property + def description(self) -> Optional[List[Any]]: ... + @property + def rowcount(self) -> int: ... + + + + # NOTE: this section is generated by tools/pythonpkg/scripts/generate_connection_stubs.py. + # Do not edit this section manually, your changes will be overwritten! + + # START OF CONNECTION METHODS def append(self, table_name: str, df: pandas.DataFrame) -> DuckDBPyConnection: ... def arrow(self, rows_per_batch: int = ...) -> pyarrow.lib.Table: ... def begin(self) -> DuckDBPyConnection: ... @@ -386,12 +399,7 @@ class DuckDBPyConnection: def list_type(self, type: DuckDBPyType) -> DuckDBPyType: ... def array_type(self, type: DuckDBPyType, size: int) -> DuckDBPyType: ... def map_type(self, key: DuckDBPyType, value: DuckDBPyType) -> DuckDBPyType: ... - def __enter__(self) -> DuckDBPyConnection: ... - def __exit__(self, exc_type: object, exc: object, traceback: object) -> None: ... - @property - def description(self) -> Optional[List[Any]]: ... - @property - def rowcount(self) -> int: ... + # END OF CONNECTION METHODS class DuckDBPyRelation: def close(self) -> None: ... From fbc53d0293cce5c65cec54844848382c2ee9cc96 Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 10 Apr 2024 20:34:50 +0200 Subject: [PATCH 304/603] add return types and argument types --- .../pythonpkg/scripts/connection_methods.json | 443 +++++++++++------- .../scripts/generate_connection_stubs.py | 88 ++++ 2 files changed, 357 insertions(+), 174 deletions(-) create mode 100644 tools/pythonpkg/scripts/generate_connection_stubs.py diff --git a/tools/pythonpkg/scripts/connection_methods.json b/tools/pythonpkg/scripts/connection_methods.json index f944e17625f0..f0d204c3af94 100644 --- a/tools/pythonpkg/scripts/connection_methods.json +++ b/tools/pythonpkg/scripts/connection_methods.json @@ -14,7 +14,8 @@ "name": "filesystem", "type": "str" } - ] + ], + "return": "None" }, { "name": "unregister_filesystem", @@ -25,7 +26,8 @@ "name": "name", "type": "str" } - ] + ], + "return": "None" }, { "name": "list_filesystems", @@ -39,7 +41,8 @@ "docs": "Check if a filesystem with the provided name is currently registered", "args": [ { - "name": "name" + "name": "name", + "type": "str" } ], "return": "bool" @@ -130,7 +133,8 @@ "name": "size", "type": "int" } - ] + ], + "return": "DuckDBPyType" }, { "name": "list_type", @@ -142,7 +146,8 @@ "type": "DuckDBPyType", "allow_none": false } - ] + ], + "return": "DuckDBPyType" }, { "name": "union_type", @@ -154,7 +159,8 @@ "type": "DuckDBPyType", "allow_none": false } - ] + ], + "return": "DuckDBPyType" }, { "name": "string_type", @@ -166,7 +172,8 @@ "type": "str", "default": "\"\"" } - ] + ], + "return": "DuckDBPyType" }, { "name": "enum_type", @@ -185,7 +192,8 @@ "name": "values", "type": "List[Any]" } - ] + ], + "return": "DuckDBPyType" }, { "name": "decimal_type", @@ -200,7 +208,8 @@ "name": "scale", "type": "int" } - ] + ], + "return": "DuckDBPyType" }, { "name": ["struct_type", "row_type"], @@ -211,7 +220,8 @@ "name": "fields", "type": "Union[Dict[str, DuckDBPyType], List[str]]" } - ] + ], + "return": "DuckDBPyType" }, { "name": "map_type", @@ -220,18 +230,22 @@ "args": [ { "name": "key", - "allow_none": false + "allow_none": false, + "type": "DuckDBPyType" }, { "name": "value", - "allow_none": false + "allow_none": false, + "type": "DuckDBPyType" } - ] + ], + "return": "DuckDBPyType" }, { "name": "duplicate", "function": "Cursor", - "docs": "Create a duplicate of the current connection" + "docs": "Create a duplicate of the current connection", + "return": "DuckDBPyConnection" }, { "name": "execute", @@ -239,17 +253,21 @@ "docs": "Execute the given SQL query, optionally using prepared statements with parameters set", "args": [ { - "name": "query" + "name": "query", + "type": "object" }, { "name": "parameters", - "default": "None" + "default": "None", + "type": "object" }, { "name": "multiple_parameter_sets", - "default": "false" + "default": "false", + "type": "bool" } - ] + ], + "return": "DuckDBPyConnection" }, { "name": "executemany", @@ -257,28 +275,34 @@ "docs": "Execute the given prepared statement multiple times using the list of parameter sets in parameters", "args": [ { - "name": "query" + "name": "query", + "type": "object" }, { "name": "parameters", - "default": "None" + "default": "None", + "type": "object" } - ] + ], + "return": "DuckDBPyConnection" }, { "name": "close", "function": "Close", - "docs": "Close the connection" + "docs": "Close the connection", + "return": "None" }, { "name": "interrupt", "function": "Interrupt", - "docs": "Interrupt pending operations" + "docs": "Interrupt pending operations", + "return": "None" }, { "name": "fetchone", "function": "FetchOne", - "docs": "Fetch a single row from a result following execute" + "docs": "Fetch a single row from a result following execute", + "return": "Optional[tuple]" }, { "name": "fetchmany", @@ -287,69 +311,56 @@ "args": [ { "name": "size", - "default": "1" + "default": "1", + "type": "int" } - ] + ], + "return": "List[Any]" }, { "name": "fetchall", "function": "FetchAll", - "docs": "Fetch all rows from a result following execute" + "docs": "Fetch all rows from a result following execute", + "return": "List[Any]" }, { "name": "fetchnumpy", "function": "FetchNumpy", - "docs": "Fetch a result as list of NumPy arrays following execute" - }, - { - "name": "fetchdf", - "function": "FetchDF", - "docs": "Fetch a result as DataFrame following execute()", - "kwargs": [ - { - "name": "date_as_object", - "default": "false" - } - ] + "docs": "Fetch a result as list of NumPy arrays following execute", + "return": "dict" }, { - "name": "fetch_df", + "name": ["fetchdf", "fetch_df", "df"], "function": "FetchDF", "docs": "Fetch a result as DataFrame following execute()", "kwargs": [ { "name": "date_as_object", - "default": "false" + "default": "false", + "type": "bool" } - ] + ], + "return": "pandas.DataFrame" }, { "name": "fetch_df_chunk", "function": "FetchDFChunk", - "docs": "Fetch a chunk of the result as Data.Frame following execute()", + "docs": "Fetch a chunk of the result as DataFrame following execute()", "args": [ { "name": "vectors_per_chunk", - "default": "1" + "default": "1", + "type": "int" } ], "kwargs": [ { "name": "date_as_object", - "default": "false" + "default": "false", + "type": "bool" } - ] - }, - { - "name": "df", - "function": "FetchDF", - "docs": "Fetch a result as DataFrame following execute()", - "kwargs": [ - { - "name": "date_as_object", - "default": "false" - } - ] + ], + "return": "pandas.DataFrame" }, { "name": "pl", @@ -358,20 +369,24 @@ "args": [ { "name": "rows_per_batch", - "default": "1000000" + "default": "1000000", + "type": "int" } - ] + ], + "return": "polars.DataFrame" }, { - "name": "fetch_arrow_table", + "name": ["fetch_arrow_table", "arrow"], "function": "FetchArrow", "docs": "Fetch a result as Arrow table following execute()", "args": [ { "name": "rows_per_batch", - "default": "1000000" + "default": "1000000", + "type": "int" } - ] + ], + "return": "pyarrow.lib.Table" }, { "name": "fetch_record_batch", @@ -380,48 +395,41 @@ "args": [ { "name": "rows_per_batch", - "default": "1000000" - } - ] - }, - { - "name": "arrow", - "function": "FetchArrow", - "docs": "Fetch a result as Arrow table following execute()", - "args": [ - { - "name": "rows_per_batch", - "default": "1000000" + "default": "1000000", + "type": "int" } - ] + ], + "return": "pyarrow.lib.RecordBatchReader" }, { "name": "torch", "function": "FetchPyTorch", - "docs": "Fetch a result as dict of PyTorch Tensors following execute()" - + "docs": "Fetch a result as dict of PyTorch Tensors following execute()", + "return": "dict" }, { "name": "tf", "function": "FetchTF", - "docs": "Fetch a result as dict of TensorFlow Tensors following execute()" - + "docs": "Fetch a result as dict of TensorFlow Tensors following execute()", + "return": "dict" }, { "name": "begin", "function": "Begin", - "docs": "Start a new transaction" - + "docs": "Start a new transaction", + "return": "DuckDBPyConnection" }, { "name": "commit", "function": "Commit", - "docs": "Commit changes performed within a transaction" + "docs": "Commit changes performed within a transaction", + "return": "DuckDBPyConnection" }, { "name": "rollback", "function": "Rollback", - "docs": "Roll back changes performed within a transaction" + "docs": "Roll back changes performed within a transaction", + "return": "DuckDBPyConnection" }, { "name": "append", @@ -429,18 +437,22 @@ "docs": "Append the passed DataFrame to the named table", "args": [ { - "name": "table_name" + "name": "table_name", + "type": "str" }, { - "name": "df" + "name": "df", + "type": "pandas.DataFrame" } ], "kwargs": [ { "name": "by_name", - "default": "false" + "default": "false", + "type": "bool" } - ] + ], + "return": "DuckDBPyConnection" }, { "name": "register", @@ -448,12 +460,15 @@ "docs": "Register the passed Python Object value for querying with a view", "args": [ { - "name": "view_name" + "name": "view_name", + "type": "str" }, { - "name": "python_object" + "name": "python_object", + "type": "object" } - ] + ], + "return": "DuckDBPyConnection" }, { "name": "unregister", @@ -461,9 +476,11 @@ "docs": "Unregister the view name", "args": [ { - "name": "view_name" + "name": "view_name", + "type": "str" } - ] + ], + "return": "DuckDBPyConnection" }, { "name": "table", @@ -471,9 +488,11 @@ "docs": "Create a relation object for the named table", "args": [ { - "name": "table_name" + "name": "table_name", + "type": "str" } - ] + ], + "return": "DuckDBPyRelation" }, { "name": "view", @@ -481,9 +500,11 @@ "docs": "Create a relation object for the named view", "args": [ { - "name": "view_name" + "name": "view_name", + "type": "str" } - ] + ], + "return": "DuckDBPyRelation" }, { "name": "values", @@ -491,9 +512,11 @@ "docs": "Create a relation object from the passed values", "args": [ { - "name": "values" + "name": "values", + "type": "List[Any]" } - ] + ], + "return": "DuckDBPyRelation" }, { "name": "table_function", @@ -501,13 +524,16 @@ "docs": "Create a relation object from the named table function with given parameters", "args": [ { - "name": "name" + "name": "name", + "type": "str" }, { "name": "parameters", - "default": "None" + "default": "None", + "type": "object" } - ] + ], + "return": "DuckDBPyRelation" }, { "name": "read_json", @@ -515,31 +541,38 @@ "docs": "Create a relation object from the JSON file in 'name'", "args": [ { - "name": "name" + "name": "name", + "type": "str" } ], "kwargs": [ { "name": "columns", - "default": "None" + "default": "None", + "type": "Optional[Dict[str,str]]" }, { "name": "sample_size", - "default": "None" + "default": "None", + "type": "Optional[int]" }, { "name": "maximum_depth", - "default": "None" + "default": "None", + "type": "Optional[int]" }, { "name": "records", - "default": "None" + "default": "None", + "type": "Optional[str]" }, { "name": "format", - "default": "None" + "default": "None", + "type": "Optional[str]" } - ] + ], + "return": "DuckDBPyRelation" }, { "name": "extract_statements", @@ -547,9 +580,11 @@ "docs": "Parse the query string and extract the Statement object(s) produced", "args": [ { - "name": "query" + "name": "query", + "type": "str" } - ] + ], + "return": "List[Statement]" }, { "name": ["sql", "query", "from_query"], @@ -557,119 +592,144 @@ "docs": "Run a SQL query. If it is a SELECT statement, create a relation object from the given SQL query, otherwise run the query as-is.", "args": [ { - "name": "query" + "name": "query", + "type": "str" } ], "kwargs": [ { "name": "alias", - "default": "\"\"" + "default": "\"\"", + "type": "str" }, { "name": "params", - "default": "None" + "default": "None", + "type": "object" } - ] + ], + "return": "DuckDBPyRelation" }, - { "name": ["read_csv", "from_csv_auto"], "function": "ReadCSV", "docs": "Create a relation object from the CSV file in 'name'", "args": [ { - "name": "name" + "name": "path_or_buffer", + "type": "Union[str, StringIO, TextIOBase]" } ], "kwargs": [ { "name": "header", - "default": "None" + "default": "None", + "type": "Optional[bool | int]" }, { "name": "compression", - "default": "None" + "default": "None", + "type": "Optional[str]" }, { "name": "sep", - "default": "None" + "default": "None", + "type": "Optional[str]" }, { "name": "delimiter", - "default": "None" + "default": "None", + "type": "Optional[str]" }, { "name": "dtype", - "default": "None" + "default": "None", + "type": "Optional[Dict[str, str] | List[str]]" }, { "name": "na_values", - "default": "None" + "default": "None", + "type": "Optional[str]" }, { "name": "skiprows", - "default": "None" + "default": "None", + "type": "Optional[int]" }, { "name": "quotechar", - "default": "None" + "default": "None", + "type": "Optional[str]" }, { "name": "escapechar", - "default": "None" + "default": "None", + "type": "Optional[str]" }, { "name": "encoding", - "default": "None" + "default": "None", + "type": "Optional[str]" }, { "name": "parallel", - "default": "None" + "default": "None", + "type": "Optional[bool]" }, { "name": "date_format", - "default": "None" + "default": "None", + "type": "Optional[str]" }, { "name": "timestamp_format", - "default": "None" + "default": "None", + "type": "Optional[str]" }, { "name": "sample_size", - "default": "None" + "default": "None", + "type": "Optional[int]" }, { "name": "all_varchar", - "default": "None" + "default": "None", + "type": "Optional[bool]" }, { "name": "normalize_names", - "default": "None" + "default": "None", + "type": "Optional[bool]" }, { "name": "filename", - "default": "None" + "default": "None", + "type": "Optional[bool]" }, { "name": "null_padding", - "default": "None" + "default": "None", + "type": "Optional[bool]" }, { "name": "names", - "default": "None" + "default": "None", + "type": "Optional[List[str]]" } - ] + ], + "return": "DuckDBPyRelation" }, { "name": "from_df", "function": "FromDF", - "docs": "Create a relation object from the Data.Frame in df", + "docs": "Create a relation object from the DataFrame in df", "args": [ { "name": "df", - "default": "None" + "type": "pandas.DataFrame" } - ] + ], + "return": "DuckDBPyRelation" }, { "name": "from_arrow", @@ -677,9 +737,11 @@ "docs": "Create a relation object from an Arrow object", "args": [ { - "name": "arrow_object" + "name": "arrow_object", + "type": "object" } - ] + ], + "return": "DuckDBPyRelation" }, { "name": ["from_parquet", "read_parquet"], @@ -687,35 +749,43 @@ "docs": "Create a relation object from the Parquet files in file_glob", "args": [ { - "name": "file_glob" + "name": "file_glob", + "type": "str" }, { "name": "binary_as_string", - "default": "false" + "default": "false", + "type": "bool" } ], "kwargs": [ { "name": "file_row_number", - "default": "false" + "default": "false", + "type": "bool" }, { "name": "filename", - "default": "false" + "default": "false", + "type": "bool" }, { "name": "hive_partitioning", - "default": "false" + "default": "false", + "type": "bool" }, { "name": "union_by_name", - "default": "false" + "default": "false", + "type": "bool" }, { "name": "compression", - "default": "None" + "default": "None", + "type": "str" } - ] + ], + "return": "DuckDBPyRelation" }, { "name": ["from_parquet", "read_parquet"], @@ -723,35 +793,43 @@ "docs": "Create a relation object from the Parquet files in file_globs", "args": [ { - "name": "file_globs" + "name": "file_globs", + "type": "str" }, { "name": "binary_as_string", - "default": "false" + "default": "false", + "type": "bool" } ], "kwargs": [ { "name": "file_row_number", - "default": "false" + "default": "false", + "type": "bool" }, { "name": "filename", - "default": "false" + "default": "false", + "type": "bool" }, { "name": "hive_partitioning", - "default": "false" + "default": "false", + "type": "bool" }, { "name": "union_by_name", - "default": "false" + "default": "false", + "type": "bool" }, { "name": "compression", - "default": "None" + "default": "None", + "type": "str" } - ] + ], + "return": "DuckDBPyRelation" }, { "name": "from_substrait", @@ -759,9 +837,11 @@ "docs": "Create a query object from protobuf plan", "args": [ { - "name": "proto" + "name": "proto", + "type": "str" } - ] + ], + "return": "DuckDBPyRelation" }, { "name": "get_substrait", @@ -769,15 +849,18 @@ "docs": "Serialize a query to protobuf", "args": [ { - "name": "query" + "name": "query", + "type": "str" } ], "kwargs": [ { "name": "enable_optimizer", - "default": "True" + "default": "True", + "type": "bool" } - ] + ], + "return": "str" }, { "name": "get_substrait_json", @@ -785,15 +868,18 @@ "docs": "Serialize a query to protobuf on the JSON format", "args": [ { - "name": "query" + "name": "query", + "type": "str" } ], "kwargs": [ { "name": "enable_optimizer", - "default": "True" + "default": "True", + "type": "bool" } - ] + ], + "return": "str" }, { "name": "from_substrait_json", @@ -801,9 +887,11 @@ "docs": "Create a query object from a JSON protobuf plan", "args": [ { - "name": "json" + "name": "json", + "type": "str" } - ] + ], + "return": "DuckDBPyRelation" }, { "name": "get_table_names", @@ -811,9 +899,11 @@ "docs": "Extract the required table names from a query", "args": [ { - "name": "query" + "name": "query", + "type": "str" } - ] + ], + "return": "List[str]" }, { "name": "install_extension", @@ -821,15 +911,18 @@ "docs": "Install an extension by name", "args": [ { - "name": "extension" + "name": "extension", + "type": "str" } ], "kwargs": [ { "name": "force_install", - "default": "false" + "default": "false", + "type": "bool" } - ] + ], + "return": "None" }, { "name": "load_extension", @@ -837,8 +930,10 @@ "docs": "Load an installed extension", "args": [ { - "name": "extension" + "name": "extension", + "type": "str" } - ] + ], + "return": "None" } ] diff --git a/tools/pythonpkg/scripts/generate_connection_stubs.py b/tools/pythonpkg/scripts/generate_connection_stubs.py new file mode 100644 index 000000000000..2d6683be8970 --- /dev/null +++ b/tools/pythonpkg/scripts/generate_connection_stubs.py @@ -0,0 +1,88 @@ +import os +import json + +os.chdir(os.path.dirname(__file__)) + +JSON_PATH = os.path.join("connection_methods.json") +DUCKDB_STUBS_FILE = os.path.join("..", "duckdb-stubs", "__init__.pyi") + +START_MARKER = " # START OF CONNECTION METHODS" +END_MARKER = " # END OF CONNECTION METHODS" + +# Read the DUCKDB_STUBS_FILE file +with open(DUCKDB_STUBS_FILE, 'r') as source_file: + source_code = source_file.readlines() + +# Locate the InitializeConnectionMethods function in it +start_index = -1 +end_index = -1 +for i, line in enumerate(source_code): + if line.startswith(START_MARKER): + # TODO: handle the case where the start marker appears multiple times + start_index = i + elif line.startswith(END_MARKER): + # TODO: ditto ^ + end_index = i + +if start_index == -1 or end_index == -1: + raise ValueError("Couldn't find start or end marker in source file") + +start_section = source_code[: start_index + 1] +end_section = source_code[end_index:] +# ---- Generate the definition code from the json ---- + +# Read the JSON file +with open(JSON_PATH, 'r') as json_file: + connection_methods = json.load(json_file) + +body = [] + + +def create_arguments(arguments) -> list: + result = [] + for arg in arguments: + argument = f"{arg['name']}: {arg['type']}" + # Add the default argument if present + if 'default' in arg: + default = arg['default'] + argument += f" = {default}" + result.append(argument) + return result + + +def create_definition(name, method) -> str: + print(method) + definition = f"def {name}(self" + if 'args' in method: + definition += ", " + arguments = create_arguments(method['args']) + definition += ', '.join(arguments) + if 'kwargs' in method: + definition += ", **kwargs" + definition += ")" + definition += f" -> {method['return']}: ..." + return definition + + +for method in connection_methods: + if isinstance(method['name'], list): + names = method['name'] + else: + names = [method['name']] + for name in names: + body.append(create_definition(name, method)) + +# ---- End of generation code ---- + +with_newlines = ['\t' + x + '\n' for x in body] +# Recreate the file content by concatenating all the pieces together + +new_content = start_section + with_newlines + end_section + +print(with_newlines) + +exit() + +# Write out the modified DUCKDB_STUBS_FILE file +with open(DUCKDB_STUBS_FILE, 'w') as source_file: + source_file.write("".join(new_content)) From 1a061d48e5c532a97cdf9c858a75345aebd379df Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 10 Apr 2024 20:50:15 +0200 Subject: [PATCH 305/603] generating the stubs for DuckDBPyConnection --- tools/pythonpkg/duckdb-stubs/__init__.pyi | 181 ++++++------------ .../pythonpkg/scripts/connection_methods.json | 30 +-- .../scripts/generate_connection_stubs.py | 11 +- tools/pythonpkg/src/pyconnection.cpp | 17 +- 4 files changed, 87 insertions(+), 152 deletions(-) diff --git a/tools/pythonpkg/duckdb-stubs/__init__.pyi b/tools/pythonpkg/duckdb-stubs/__init__.pyi index b575bd039660..a39a8b1451ca 100644 --- a/tools/pythonpkg/duckdb-stubs/__init__.pyi +++ b/tools/pythonpkg/duckdb-stubs/__init__.pyi @@ -264,141 +264,72 @@ class DuckDBPyConnection: # Do not edit this section manually, your changes will be overwritten! # START OF CONNECTION METHODS - def append(self, table_name: str, df: pandas.DataFrame) -> DuckDBPyConnection: ... - def arrow(self, rows_per_batch: int = ...) -> pyarrow.lib.Table: ... - def begin(self) -> DuckDBPyConnection: ... - def close(self) -> None: ... - def commit(self) -> DuckDBPyConnection: ... def cursor(self) -> DuckDBPyConnection: ... - def df(self) -> pandas.DataFrame: ... - def duplicate(self) -> DuckDBPyConnection: ... - def execute(self, query: str, parameters: object = ..., multiple_parameter_sets: bool = ...) -> DuckDBPyConnection: ... - def executemany(self, query: str, parameters: object = ...) -> DuckDBPyConnection: ... - def fetch_arrow_table(self, rows_per_batch: int = ...) -> pyarrow.lib.Table: ... - def fetch_df(self, *args, **kwargs) -> pandas.DataFrame: ... - def fetch_df_chunk(self, *args, **kwargs) -> pandas.DataFrame: ... - def fetch_record_batch(self, rows_per_batch: int = ...) -> pyarrow.lib.RecordBatchReader: ... - def fetchall(self) -> List[Any]: ... - def fetchdf(self, *args, **kwargs) -> pandas.DataFrame: ... - def fetchmany(self, size: int = ...) -> List[Any]: ... - def fetchnumpy(self) -> dict: ... - def fetchone(self) -> Optional[tuple]: ... - def from_arrow(self, arrow_object: object) -> DuckDBPyRelation: ... - def read_json( - self, - file_name: str, - columns: Optional[Dict[str,str]] = None, - sample_size: Optional[int] = None, - maximum_depth: Optional[int] = None, - records: Optional[str] = None, - format: Optional[str] = None - ) -> DuckDBPyRelation: ... - def read_csv( - self, - path_or_buffer: Union[str, StringIO, TextIOBase], - header: Optional[bool | int] = None, - compression: Optional[str] = None, - sep: Optional[str] = None, - delimiter: Optional[str] = None, - dtype: Optional[Dict[str, str] | List[str]] = None, - na_values: Optional[str] = None, - skiprows: Optional[int] = None, - quotechar: Optional[str] = None, - escapechar: Optional[str] = None, - encoding: Optional[str] = None, - parallel: Optional[bool] = None, - date_format: Optional[str] = None, - timestamp_format: Optional[str] = None, - sample_size: Optional[int] = None, - all_varchar: Optional[bool] = None, - normalize_names: Optional[bool] = None, - filename: Optional[bool] = None, - null_padding: Optional[bool] = None, - names: Optional[List[str]] = None - ) -> DuckDBPyRelation: ... - def from_csv_auto( - self, - path_or_buffer: Union[str, StringIO, TextIOBase], - header: Optional[bool | int] = None, - compression: Optional[str] = None, - sep: Optional[str] = None, - delimiter: Optional[str] = None, - dtype: Optional[Dict[str, str] | List[str]] = None, - na_values: Optional[str] = None, - skiprows: Optional[int] = None, - quotechar: Optional[str] = None, - escapechar: Optional[str] = None, - encoding: Optional[str] = None, - parallel: Optional[bool] = None, - date_format: Optional[str] = None, - timestamp_format: Optional[str] = None, - sample_size: Optional[int] = None, - all_varchar: Optional[bool] = None, - normalize_names: Optional[bool] = None, - filename: Optional[bool] = None, - null_padding: Optional[bool] = None, - names: Optional[List[str]] = None - ) -> DuckDBPyRelation: ... - def from_df(self, df: pandas.DataFrame = ...) -> DuckDBPyRelation: ... - @overload - def read_parquet(self, file_glob: str, binary_as_string: bool = ..., *, file_row_number: bool = ..., filename: bool = ..., hive_partitioning: bool = ..., union_by_name: bool = ...) -> DuckDBPyRelation: ... - @overload - def read_parquet(self, file_globs: List[str], binary_as_string: bool = ..., *, file_row_number: bool = ..., filename: bool = ..., hive_partitioning: bool = ..., union_by_name: bool = ...) -> DuckDBPyRelation: ... - @overload - def from_parquet(self, file_glob: str, binary_as_string: bool = ..., *, file_row_number: bool = ..., filename: bool = ..., hive_partitioning: bool = ..., union_by_name: bool = ...) -> DuckDBPyRelation: ... - @overload - def from_parquet(self, file_globs: List[str], binary_as_string: bool = ..., *, file_row_number: bool = ..., filename: bool = ..., hive_partitioning: bool = ..., union_by_name: bool = ...) -> DuckDBPyRelation: ... - def from_substrait(self, proto: bytes) -> DuckDBPyRelation: ... - def get_substrait(self, query: str) -> DuckDBPyRelation: ... - def get_substrait_json(self, query: str) -> DuckDBPyRelation: ... - def from_substrait_json(self, json: str) -> DuckDBPyRelation: ... - def get_table_names(self, query: str) -> Set[str]: ... - def install_extension(self, *args, **kwargs) -> None: ... - def interrupt(self) -> None: ... - def list_filesystems(self) -> List[Any]: ... + def register_filesystem(self, filesystem: str) -> None: ... + def unregister_filesystem(self, name: str) -> None: ... + def list_filesystems(self) -> list: ... def filesystem_is_registered(self, name: str) -> bool: ... - def load_extension(self, extension: str) -> None: ... - def pl(self, rows_per_batch: int = ..., connection: DuckDBPyConnection = ...) -> polars.DataFrame: ... - def torch(self, connection: DuckDBPyConnection = ...) -> dict: ... - def tf(self, connection: DuckDBPyConnection = ...) -> dict: ... - - def from_query(self, query: str, **kwargs) -> DuckDBPyRelation: ... - def extract_statements(self, query: str) -> List[Statement]: ... - def query(self, query: str, **kwargs) -> DuckDBPyRelation: ... - def sql(self, query: str, **kwargs) -> DuckDBPyRelation: ... - - def register(self, view_name: str, python_object: object) -> DuckDBPyConnection: ... + def create_function(self, name: str, function: function, parameters: Optional[List[DuckDBPyType]] = None, return_type: Optional[DuckDBPyType] = None, **kwargs) -> DuckDBPyConnection: ... def remove_function(self, name: str) -> DuckDBPyConnection: ... - def create_function( - self, - name: str, - func: Callable, - parameters: Optional[List[DuckDBPyType]] = None, - return_type: Optional[DuckDBPyType] = None, - type: Optional[PythonUDFType] = PythonUDFType.NATIVE, - null_handling: Optional[FunctionNullHandling] = FunctionNullHandling.DEFAULT, - exception_handling: Optional[PythonExceptionHandling] = PythonExceptionHandling.DEFAULT, - side_effects: Optional[bool] = False) -> DuckDBPyConnection: ... - def register_filesystem(self, filesystem: fsspec.AbstractFileSystem) -> None: ... - def rollback(self) -> DuckDBPyConnection: ... - def table(self, table_name: str) -> DuckDBPyRelation: ... - def table_function(self, name: str, parameters: object = ...) -> DuckDBPyRelation: ... - def unregister(self, view_name: str) -> DuckDBPyConnection: ... - def unregister_filesystem(self, name: str) -> None: ... - def values(self, values: object) -> DuckDBPyRelation: ... - def view(self, view_name: str) -> DuckDBPyRelation: ... def sqltype(self, type_str: str) -> DuckDBPyType: ... def dtype(self, type_str: str) -> DuckDBPyType: ... def type(self, type_str: str) -> DuckDBPyType: ... - def struct_type(self, fields: Union[Dict[str, DuckDBPyType], List[str]]) -> DuckDBPyType: ... - def row_type(self, fields: Union[Dict[str, DuckDBPyType], List[str]]) -> DuckDBPyType: ... - def union_type(self, members: Union[Dict[str, DuckDBPyType], List[str]]) -> DuckDBPyType: ... + def array_type(self, type: DuckDBPyType, size: int) -> DuckDBPyType: ... + def list_type(self, type: DuckDBPyType) -> DuckDBPyType: ... + def union_type(self, members: DuckDBPyType) -> DuckDBPyType: ... def string_type(self, collation: str = "") -> DuckDBPyType: ... def enum_type(self, name: str, type: DuckDBPyType, values: List[Any]) -> DuckDBPyType: ... def decimal_type(self, width: int, scale: int) -> DuckDBPyType: ... - def list_type(self, type: DuckDBPyType) -> DuckDBPyType: ... - def array_type(self, type: DuckDBPyType, size: int) -> DuckDBPyType: ... + def struct_type(self, fields: Union[Dict[str, DuckDBPyType], List[str]]) -> DuckDBPyType: ... + def row_type(self, fields: Union[Dict[str, DuckDBPyType], List[str]]) -> DuckDBPyType: ... def map_type(self, key: DuckDBPyType, value: DuckDBPyType) -> DuckDBPyType: ... + def duplicate(self) -> DuckDBPyConnection: ... + def execute(self, query: object, parameters: object = None, multiple_parameter_sets: bool = False) -> DuckDBPyConnection: ... + def executemany(self, query: object, parameters: object = None) -> DuckDBPyConnection: ... + def close(self) -> None: ... + def interrupt(self) -> None: ... + def fetchone(self) -> Optional[tuple]: ... + def fetchmany(self, size: int = 1) -> List[Any]: ... + def fetchall(self) -> List[Any]: ... + def fetchnumpy(self) -> dict: ... + def fetchdf(self, **kwargs) -> pandas.DataFrame: ... + def fetch_df(self, **kwargs) -> pandas.DataFrame: ... + def df(self, **kwargs) -> pandas.DataFrame: ... + def fetch_df_chunk(self, vectors_per_chunk: int = 1, **kwargs) -> pandas.DataFrame: ... + def pl(self, rows_per_batch: int = 1000000) -> polars.DataFrame: ... + def fetch_arrow_table(self, rows_per_batch: int = 1000000) -> pyarrow.lib.Table: ... + def arrow(self, rows_per_batch: int = 1000000) -> pyarrow.lib.Table: ... + def fetch_record_batch(self, rows_per_batch: int = 1000000) -> pyarrow.lib.RecordBatchReader: ... + def torch(self) -> dict: ... + def tf(self) -> dict: ... + def begin(self) -> DuckDBPyConnection: ... + def commit(self) -> DuckDBPyConnection: ... + def rollback(self) -> DuckDBPyConnection: ... + def append(self, table_name: str, df: pandas.DataFrame, **kwargs) -> DuckDBPyConnection: ... + def register(self, view_name: str, python_object: object) -> DuckDBPyConnection: ... + def unregister(self, view_name: str) -> DuckDBPyConnection: ... + def table(self, table_name: str) -> DuckDBPyRelation: ... + def view(self, view_name: str) -> DuckDBPyRelation: ... + def values(self, values: List[Any]) -> DuckDBPyRelation: ... + def table_function(self, name: str, parameters: object = None) -> DuckDBPyRelation: ... + def read_json(self, name: str, **kwargs) -> DuckDBPyRelation: ... + def extract_statements(self, query: str) -> List[Statement]: ... + def sql(self, query: str, **kwargs) -> DuckDBPyRelation: ... + def query(self, query: str, **kwargs) -> DuckDBPyRelation: ... + def from_query(self, query: str, **kwargs) -> DuckDBPyRelation: ... + def read_csv(self, path_or_buffer: Union[str, StringIO, TextIOBase], **kwargs) -> DuckDBPyRelation: ... + def from_csv_auto(self, path_or_buffer: Union[str, StringIO, TextIOBase], **kwargs) -> DuckDBPyRelation: ... + def from_df(self, df: pandas.DataFrame) -> DuckDBPyRelation: ... + def from_arrow(self, arrow_object: object) -> DuckDBPyRelation: ... + def from_parquet(self, file_glob: str, binary_as_string: bool = False, **kwargs) -> DuckDBPyRelation: ... + def read_parquet(self, file_glob: str, binary_as_string: bool = False, **kwargs) -> DuckDBPyRelation: ... + def from_substrait(self, proto: str) -> DuckDBPyRelation: ... + def get_substrait(self, query: str, **kwargs) -> str: ... + def get_substrait_json(self, query: str, **kwargs) -> str: ... + def from_substrait_json(self, json: str) -> DuckDBPyRelation: ... + def get_table_names(self, query: str) -> List[str]: ... + def install_extension(self, extension: str, **kwargs) -> None: ... + def load_extension(self, extension: str) -> None: ... # END OF CONNECTION METHODS class DuckDBPyRelation: diff --git a/tools/pythonpkg/scripts/connection_methods.json b/tools/pythonpkg/scripts/connection_methods.json index f0d204c3af94..3184459c7615 100644 --- a/tools/pythonpkg/scripts/connection_methods.json +++ b/tools/pythonpkg/scripts/connection_methods.json @@ -263,7 +263,7 @@ }, { "name": "multiple_parameter_sets", - "default": "false", + "default": "False", "type": "bool" } ], @@ -336,7 +336,7 @@ "kwargs": [ { "name": "date_as_object", - "default": "false", + "default": "False", "type": "bool" } ], @@ -356,7 +356,7 @@ "kwargs": [ { "name": "date_as_object", - "default": "false", + "default": "False", "type": "bool" } ], @@ -448,7 +448,7 @@ "kwargs": [ { "name": "by_name", - "default": "false", + "default": "False", "type": "bool" } ], @@ -754,29 +754,29 @@ }, { "name": "binary_as_string", - "default": "false", + "default": "False", "type": "bool" } ], "kwargs": [ { "name": "file_row_number", - "default": "false", + "default": "False", "type": "bool" }, { "name": "filename", - "default": "false", + "default": "False", "type": "bool" }, { "name": "hive_partitioning", - "default": "false", + "default": "False", "type": "bool" }, { "name": "union_by_name", - "default": "false", + "default": "False", "type": "bool" }, { @@ -798,29 +798,29 @@ }, { "name": "binary_as_string", - "default": "false", + "default": "False", "type": "bool" } ], "kwargs": [ { "name": "file_row_number", - "default": "false", + "default": "False", "type": "bool" }, { "name": "filename", - "default": "false", + "default": "False", "type": "bool" }, { "name": "hive_partitioning", - "default": "false", + "default": "False", "type": "bool" }, { "name": "union_by_name", - "default": "false", + "default": "False", "type": "bool" }, { @@ -918,7 +918,7 @@ "kwargs": [ { "name": "force_install", - "default": "false", + "default": "False", "type": "bool" } ], diff --git a/tools/pythonpkg/scripts/generate_connection_stubs.py b/tools/pythonpkg/scripts/generate_connection_stubs.py index 2d6683be8970..cacddd3b0ced 100644 --- a/tools/pythonpkg/scripts/generate_connection_stubs.py +++ b/tools/pythonpkg/scripts/generate_connection_stubs.py @@ -64,25 +64,30 @@ def create_definition(name, method) -> str: return definition +# We have "duplicate" methods, which are overloaded +# maybe we should add @overload to these instead, but this is easier +written_methods = set() + for method in connection_methods: if isinstance(method['name'], list): names = method['name'] else: names = [method['name']] for name in names: + if name in written_methods: + continue body.append(create_definition(name, method)) + written_methods.add(name) # ---- End of generation code ---- -with_newlines = ['\t' + x + '\n' for x in body] +with_newlines = [' ' + x + '\n' for x in body] # Recreate the file content by concatenating all the pieces together new_content = start_section + with_newlines + end_section print(with_newlines) -exit() - # Write out the modified DUCKDB_STUBS_FILE file with open(DUCKDB_STUBS_FILE, 'w') as source_file: source_file.write("".join(new_content)) diff --git a/tools/pythonpkg/src/pyconnection.cpp b/tools/pythonpkg/src/pyconnection.cpp index 39cf30451f5d..7e5697bcbfc6 100644 --- a/tools/pythonpkg/src/pyconnection.cpp +++ b/tools/pythonpkg/src/pyconnection.cpp @@ -182,19 +182,19 @@ static void InitializeConnectionMethods(py::class_ Date: Wed, 10 Apr 2024 21:48:48 +0200 Subject: [PATCH 306/603] add vss --- .github/config/extensions.csv | 1 + .github/config/out_of_tree_extensions.cmake | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/.github/config/extensions.csv b/.github/config/extensions.csv index 9fe8eb827b5c..d2e68ab77fa7 100644 --- a/.github/config/extensions.csv +++ b/.github/config/extensions.csv @@ -15,3 +15,4 @@ aws,https://github.com/duckdb/duckdb_aws,f7b8729f1cce5ada5d4add70e1486de50763fb9 azure,https://github.com/duckdb/duckdb_azure,86f39d76157de970d16d6d6537bc90c0ee1c7d35, spatial,https://github.com/duckdb/duckdb_spatial,05c4ba01c500140287bf6946fb6910122e5c2acf, iceberg,https://github.com/duckdb/duckdb_iceberg,7aa3d8e4cb7b513d35fdacfa28dc328771bc4047, +vss,https://github.com/duckdb/duckdb_vss,f03fe75e16dd67e404d6325d4f901d6958504730, \ No newline at end of file diff --git a/.github/config/out_of_tree_extensions.cmake b/.github/config/out_of_tree_extensions.cmake index 5d1795ef1168..304583abb513 100644 --- a/.github/config/out_of_tree_extensions.cmake +++ b/.github/config/out_of_tree_extensions.cmake @@ -104,3 +104,12 @@ if (NOT WIN32) GIT_TAG 1116fb580edd3e26e675436dbdbdf4a0aa5e456e ) endif() + + +################# VSS +duckdb_extension_load(vss + LOAD_TESTS + GIT_URL https://github.com/duckdb/duckdb_vss + GIT_TAG f03fe75e16dd67e404d6325d4f901d6958504730 + TEST_DIR test/sql + ) \ No newline at end of file From 07028017e5153a003c0fa655822c81f44e0c7e82 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 10 Apr 2024 22:00:57 +0200 Subject: [PATCH 307/603] Add postgres's new extension_entries --- src/include/duckdb/main/extension_entries.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/include/duckdb/main/extension_entries.hpp b/src/include/duckdb/main/extension_entries.hpp index 3b84b1544e3c..df6741df05a6 100644 --- a/src/include/duckdb/main/extension_entries.hpp +++ b/src/include/duckdb/main/extension_entries.hpp @@ -88,6 +88,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = { {"pg_clear_cache", "postgres_scanner", CatalogType::TABLE_FUNCTION_ENTRY}, {"pg_timezone_names", "icu", CatalogType::TABLE_FUNCTION_ENTRY}, {"postgres_attach", "postgres_scanner", CatalogType::TABLE_FUNCTION_ENTRY}, + {"postgres_execute", "postgres_scanner", CatalogType::TABLE_FUNCTION_ENTRY}, {"postgres_query", "postgres_scanner", CatalogType::TABLE_FUNCTION_ENTRY}, {"postgres_scan", "postgres_scanner", CatalogType::TABLE_FUNCTION_ENTRY}, {"postgres_scan_pushdown", "postgres_scanner", CatalogType::TABLE_FUNCTION_ENTRY}, @@ -257,6 +258,7 @@ static constexpr ExtensionEntry EXTENSION_SETTINGS[] = { {"pg_experimental_filter_pushdown", "postgres_scanner"}, {"pg_pages_per_task", "postgres_scanner"}, {"pg_use_binary_copy", "postgres_scanner"}, + {"pg_use_ctid_scan", "postgres_scanner"}, {"s3_access_key_id", "httpfs"}, {"s3_endpoint", "httpfs"}, {"s3_region", "httpfs"}, From 81c245569753ca607a22e0b6d354059d43d85ec0 Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 10 Apr 2024 22:09:39 +0200 Subject: [PATCH 308/603] generating the connection wrapper code, placed in __init__.py --- tools/pythonpkg/duckdb-stubs/__init__.pyi | 2 - tools/pythonpkg/duckdb/__init__.py | 457 +++++++++++++++--- tools/pythonpkg/duckdb_python.cpp | 3 +- .../generate_connection_wrapper_methods.py | 142 ++++++ 4 files changed, 531 insertions(+), 73 deletions(-) create mode 100644 tools/pythonpkg/scripts/generate_connection_wrapper_methods.py diff --git a/tools/pythonpkg/duckdb-stubs/__init__.pyi b/tools/pythonpkg/duckdb-stubs/__init__.pyi index a39a8b1451ca..748e1102c290 100644 --- a/tools/pythonpkg/duckdb-stubs/__init__.pyi +++ b/tools/pythonpkg/duckdb-stubs/__init__.pyi @@ -258,8 +258,6 @@ class DuckDBPyConnection: @property def rowcount(self) -> int: ... - - # NOTE: this section is generated by tools/pythonpkg/scripts/generate_connection_stubs.py. # Do not edit this section manually, your changes will be overwritten! diff --git a/tools/pythonpkg/duckdb/__init__.py b/tools/pythonpkg/duckdb/__init__.py index 8f01da43bc4f..f4f6af797c6e 100644 --- a/tools/pythonpkg/duckdb/__init__.py +++ b/tools/pythonpkg/duckdb/__init__.py @@ -40,83 +40,402 @@ "CaseExpression", ]) -# ---- Wrap the connection methods - -def is_dunder_method(method_name: str) -> bool: - if len(method_name) < 4: - return False - return method_name[:2] == '__' and method_name[:-3:-1] == '__' - -# Takes the function to execute on the 'connection' -def create_wrapper(func): - def _wrapper(*args, **kwargs): - connection = duckdb.connect(':default:') - if 'connection' in kwargs: - connection = kwargs.pop('connection') - return func(connection, *args, **kwargs) - return _wrapper - -# Takes the name of a DuckDBPyConnection function to wrap (copying signature, docs, etc) -# The 'func' is what gets executed when the function is called -def create_connection_wrapper(name, func): - # Define a decorator function that forwards attribute lookup to the default connection - return functools.wraps(getattr(DuckDBPyConnection, name))(create_wrapper(func)) - # These are overloaded twice, we define them inside of C++ so pybind can deal with it -EXCLUDED_METHODS = [ +_exported_symbols.extend([ 'df', 'arrow' -] -_exported_symbols.extend(EXCLUDED_METHODS) +]) from .duckdb import ( df, arrow ) -methods = [method for method in dir(DuckDBPyConnection) if not is_dunder_method(method) and method not in EXCLUDED_METHODS] -for method_name in methods: - def create_method_wrapper(method_name): - def call_method(conn, *args, **kwargs): - return getattr(conn, method_name)(*args, **kwargs) - return call_method - wrapper_function = create_connection_wrapper(method_name, create_method_wrapper(method_name)) - globals()[method_name] = wrapper_function # Define the wrapper function in the module namespace - _exported_symbols.append(method_name) - - -# Specialized "wrapper" methods - -SPECIAL_METHODS = [ - 'project', - 'distinct', - 'write_csv', - 'aggregate', - 'alias', - 'filter', - 'limit', - 'order', - 'query_df' -] - -for method_name in SPECIAL_METHODS: - def create_method_wrapper(name): - def _closure(name=name): - mapping = { - 'alias': 'set_alias', - 'query_df': 'query' - } - def call_method(con, df, *args, **kwargs): - if name in mapping: - mapped_name = mapping[name] - else: - mapped_name = name - return getattr(con.from_df(df), mapped_name)(*args, **kwargs) - return call_method - return _closure(name) - - wrapper_function = create_wrapper(create_method_wrapper(method_name)) - globals()[method_name] = wrapper_function # Define the wrapper function in the module namespace - _exported_symbols.append(method_name) +def __get_connection__(**kwargs): + if 'connection' in kwargs: + return kwargs.pop('connection') + else: + return duckdb.connect(":default:") + +# NOTE: this section is generated by tools/pythonpkg/scripts/generate_connection_wrapper_methods.py. +# Do not edit this section manually, your changes will be overwritten! + +# START OF CONNECTION WRAPPER + +def cursor(**kwargs): + conn = __get_connection__(*kwargs) + return conn.cursor(**kwargs) +_exported_symbols.append('cursor') + +def register_filesystem(filesystem, **kwargs): + conn = __get_connection__(*kwargs) + return conn.register_filesystem(filesystem, **kwargs) +_exported_symbols.append('register_filesystem') + +def unregister_filesystem(name, **kwargs): + conn = __get_connection__(*kwargs) + return conn.unregister_filesystem(name, **kwargs) +_exported_symbols.append('unregister_filesystem') + +def list_filesystems(**kwargs): + conn = __get_connection__(*kwargs) + return conn.list_filesystems(**kwargs) +_exported_symbols.append('list_filesystems') + +def filesystem_is_registered(name, **kwargs): + conn = __get_connection__(*kwargs) + return conn.filesystem_is_registered(name, **kwargs) +_exported_symbols.append('filesystem_is_registered') + +def create_function(name, function, parameters = None, return_type = None, **kwargs): + conn = __get_connection__(*kwargs) + return conn.create_function(name, function, parameters, return_type, **kwargs) +_exported_symbols.append('create_function') + +def remove_function(name, **kwargs): + conn = __get_connection__(*kwargs) + return conn.remove_function(name, **kwargs) +_exported_symbols.append('remove_function') + +def sqltype(type_str, **kwargs): + conn = __get_connection__(*kwargs) + return conn.sqltype(type_str, **kwargs) +_exported_symbols.append('sqltype') + +def dtype(type_str, **kwargs): + conn = __get_connection__(*kwargs) + return conn.dtype(type_str, **kwargs) +_exported_symbols.append('dtype') + +def type(type_str, **kwargs): + conn = __get_connection__(*kwargs) + return conn.type(type_str, **kwargs) +_exported_symbols.append('type') + +def array_type(type, size, **kwargs): + conn = __get_connection__(*kwargs) + return conn.array_type(type, size, **kwargs) +_exported_symbols.append('array_type') + +def list_type(type, **kwargs): + conn = __get_connection__(*kwargs) + return conn.list_type(type, **kwargs) +_exported_symbols.append('list_type') + +def union_type(members, **kwargs): + conn = __get_connection__(*kwargs) + return conn.union_type(members, **kwargs) +_exported_symbols.append('union_type') + +def string_type(collation = "", **kwargs): + conn = __get_connection__(*kwargs) + return conn.string_type(collation, **kwargs) +_exported_symbols.append('string_type') + +def enum_type(name, type, values, **kwargs): + conn = __get_connection__(*kwargs) + return conn.enum_type(name, type, values, **kwargs) +_exported_symbols.append('enum_type') + +def decimal_type(width, scale, **kwargs): + conn = __get_connection__(*kwargs) + return conn.decimal_type(width, scale, **kwargs) +_exported_symbols.append('decimal_type') + +def struct_type(fields, **kwargs): + conn = __get_connection__(*kwargs) + return conn.struct_type(fields, **kwargs) +_exported_symbols.append('struct_type') + +def row_type(fields, **kwargs): + conn = __get_connection__(*kwargs) + return conn.row_type(fields, **kwargs) +_exported_symbols.append('row_type') + +def map_type(key, value, **kwargs): + conn = __get_connection__(*kwargs) + return conn.map_type(key, value, **kwargs) +_exported_symbols.append('map_type') + +def duplicate(**kwargs): + conn = __get_connection__(*kwargs) + return conn.duplicate(**kwargs) +_exported_symbols.append('duplicate') + +def execute(query, parameters = None, multiple_parameter_sets = False, **kwargs): + conn = __get_connection__(*kwargs) + return conn.execute(query, parameters, multiple_parameter_sets, **kwargs) +_exported_symbols.append('execute') + +def executemany(query, parameters = None, **kwargs): + conn = __get_connection__(*kwargs) + return conn.executemany(query, parameters, **kwargs) +_exported_symbols.append('executemany') + +def close(**kwargs): + conn = __get_connection__(*kwargs) + return conn.close(**kwargs) +_exported_symbols.append('close') + +def interrupt(**kwargs): + conn = __get_connection__(*kwargs) + return conn.interrupt(**kwargs) +_exported_symbols.append('interrupt') + +def fetchone(**kwargs): + conn = __get_connection__(*kwargs) + return conn.fetchone(**kwargs) +_exported_symbols.append('fetchone') + +def fetchmany(size = 1, **kwargs): + conn = __get_connection__(*kwargs) + return conn.fetchmany(size, **kwargs) +_exported_symbols.append('fetchmany') + +def fetchall(**kwargs): + conn = __get_connection__(*kwargs) + return conn.fetchall(**kwargs) +_exported_symbols.append('fetchall') + +def fetchnumpy(**kwargs): + conn = __get_connection__(*kwargs) + return conn.fetchnumpy(**kwargs) +_exported_symbols.append('fetchnumpy') + +def fetchdf(**kwargs): + conn = __get_connection__(*kwargs) + return conn.fetchdf(**kwargs) +_exported_symbols.append('fetchdf') + +def fetch_df(**kwargs): + conn = __get_connection__(*kwargs) + return conn.fetch_df(**kwargs) +_exported_symbols.append('fetch_df') + +def df(**kwargs): + conn = __get_connection__(*kwargs) + return conn.df(**kwargs) +_exported_symbols.append('df') + +def fetch_df_chunk(vectors_per_chunk = 1, **kwargs): + conn = __get_connection__(*kwargs) + return conn.fetch_df_chunk(vectors_per_chunk, **kwargs) +_exported_symbols.append('fetch_df_chunk') + +def pl(rows_per_batch = 1000000, **kwargs): + conn = __get_connection__(*kwargs) + return conn.pl(rows_per_batch, **kwargs) +_exported_symbols.append('pl') + +def fetch_arrow_table(rows_per_batch = 1000000, **kwargs): + conn = __get_connection__(*kwargs) + return conn.fetch_arrow_table(rows_per_batch, **kwargs) +_exported_symbols.append('fetch_arrow_table') + +def arrow(rows_per_batch = 1000000, **kwargs): + conn = __get_connection__(*kwargs) + return conn.arrow(rows_per_batch, **kwargs) +_exported_symbols.append('arrow') + +def fetch_record_batch(rows_per_batch = 1000000, **kwargs): + conn = __get_connection__(*kwargs) + return conn.fetch_record_batch(rows_per_batch, **kwargs) +_exported_symbols.append('fetch_record_batch') + +def torch(**kwargs): + conn = __get_connection__(*kwargs) + return conn.torch(**kwargs) +_exported_symbols.append('torch') + +def tf(**kwargs): + conn = __get_connection__(*kwargs) + return conn.tf(**kwargs) +_exported_symbols.append('tf') + +def begin(**kwargs): + conn = __get_connection__(*kwargs) + return conn.begin(**kwargs) +_exported_symbols.append('begin') + +def commit(**kwargs): + conn = __get_connection__(*kwargs) + return conn.commit(**kwargs) +_exported_symbols.append('commit') + +def rollback(**kwargs): + conn = __get_connection__(*kwargs) + return conn.rollback(**kwargs) +_exported_symbols.append('rollback') + +def append(table_name, df, **kwargs): + conn = __get_connection__(*kwargs) + return conn.append(table_name, df, **kwargs) +_exported_symbols.append('append') + +def register(view_name, python_object, **kwargs): + conn = __get_connection__(*kwargs) + return conn.register(view_name, python_object, **kwargs) +_exported_symbols.append('register') + +def unregister(view_name, **kwargs): + conn = __get_connection__(*kwargs) + return conn.unregister(view_name, **kwargs) +_exported_symbols.append('unregister') + +def table(table_name, **kwargs): + conn = __get_connection__(*kwargs) + return conn.table(table_name, **kwargs) +_exported_symbols.append('table') + +def view(view_name, **kwargs): + conn = __get_connection__(*kwargs) + return conn.view(view_name, **kwargs) +_exported_symbols.append('view') + +def values(values, **kwargs): + conn = __get_connection__(*kwargs) + return conn.values(values, **kwargs) +_exported_symbols.append('values') + +def table_function(name, parameters = None, **kwargs): + conn = __get_connection__(*kwargs) + return conn.table_function(name, parameters, **kwargs) +_exported_symbols.append('table_function') + +def read_json(name, **kwargs): + conn = __get_connection__(*kwargs) + return conn.read_json(name, **kwargs) +_exported_symbols.append('read_json') + +def extract_statements(query, **kwargs): + conn = __get_connection__(*kwargs) + return conn.extract_statements(query, **kwargs) +_exported_symbols.append('extract_statements') + +def sql(query, **kwargs): + conn = __get_connection__(*kwargs) + return conn.sql(query, **kwargs) +_exported_symbols.append('sql') + +def query(query, **kwargs): + conn = __get_connection__(*kwargs) + return conn.query(query, **kwargs) +_exported_symbols.append('query') + +def from_query(query, **kwargs): + conn = __get_connection__(*kwargs) + return conn.from_query(query, **kwargs) +_exported_symbols.append('from_query') + +def read_csv(path_or_buffer, **kwargs): + conn = __get_connection__(*kwargs) + return conn.read_csv(path_or_buffer, **kwargs) +_exported_symbols.append('read_csv') + +def from_csv_auto(path_or_buffer, **kwargs): + conn = __get_connection__(*kwargs) + return conn.from_csv_auto(path_or_buffer, **kwargs) +_exported_symbols.append('from_csv_auto') + +def from_df(df, **kwargs): + conn = __get_connection__(*kwargs) + return conn.from_df(df, **kwargs) +_exported_symbols.append('from_df') + +def from_arrow(arrow_object, **kwargs): + conn = __get_connection__(*kwargs) + return conn.from_arrow(arrow_object, **kwargs) +_exported_symbols.append('from_arrow') + +def from_parquet(file_glob, binary_as_string = False, **kwargs): + conn = __get_connection__(*kwargs) + return conn.from_parquet(file_glob, binary_as_string, **kwargs) +_exported_symbols.append('from_parquet') + +def read_parquet(file_glob, binary_as_string = False, **kwargs): + conn = __get_connection__(*kwargs) + return conn.read_parquet(file_glob, binary_as_string, **kwargs) +_exported_symbols.append('read_parquet') + +def from_substrait(proto, **kwargs): + conn = __get_connection__(*kwargs) + return conn.from_substrait(proto, **kwargs) +_exported_symbols.append('from_substrait') + +def get_substrait(query, **kwargs): + conn = __get_connection__(*kwargs) + return conn.get_substrait(query, **kwargs) +_exported_symbols.append('get_substrait') + +def get_substrait_json(query, **kwargs): + conn = __get_connection__(*kwargs) + return conn.get_substrait_json(query, **kwargs) +_exported_symbols.append('get_substrait_json') + +def from_substrait_json(json, **kwargs): + conn = __get_connection__(*kwargs) + return conn.from_substrait_json(json, **kwargs) +_exported_symbols.append('from_substrait_json') + +def get_table_names(query, **kwargs): + conn = __get_connection__(*kwargs) + return conn.get_table_names(query, **kwargs) +_exported_symbols.append('get_table_names') + +def install_extension(extension, **kwargs): + conn = __get_connection__(*kwargs) + return conn.install_extension(extension, **kwargs) +_exported_symbols.append('install_extension') + +def load_extension(extension, **kwargs): + conn = __get_connection__(*kwargs) + return conn.load_extension(extension, **kwargs) +_exported_symbols.append('load_extension') + +def project(df, *args, **kwargs): + conn = __get_connection__(*kwargs) + return conn.from_df(df).project(*args, **kwargs) +_exported_symbols.append('project') + +def distinct(df, *args, **kwargs): + conn = __get_connection__(*kwargs) + return conn.from_df(df).distinct(*args, **kwargs) +_exported_symbols.append('distinct') + +def write_csv(df, *args, **kwargs): + conn = __get_connection__(*kwargs) + return conn.from_df(df).write_csv(*args, **kwargs) +_exported_symbols.append('write_csv') + +def aggregate(df, *args, **kwargs): + conn = __get_connection__(*kwargs) + return conn.from_df(df).aggregate(*args, **kwargs) +_exported_symbols.append('aggregate') + +def alias(df, *args, **kwargs): + conn = __get_connection__(*kwargs) + return conn.from_df(df).set_alias(*args, **kwargs) +_exported_symbols.append('alias') + +def filter(df, *args, **kwargs): + conn = __get_connection__(*kwargs) + return conn.from_df(df).filter(*args, **kwargs) +_exported_symbols.append('filter') + +def limit(df, *args, **kwargs): + conn = __get_connection__(*kwargs) + return conn.from_df(df).limit(*args, **kwargs) +_exported_symbols.append('limit') + +def order(df, *args, **kwargs): + conn = __get_connection__(*kwargs) + return conn.from_df(df).order(*args, **kwargs) +_exported_symbols.append('order') + +def query_df(df, *args, **kwargs): + conn = __get_connection__(*kwargs) + return conn.from_df(df).query(*args, **kwargs) +_exported_symbols.append('query_df') +# END OF CONNECTION WRAPPER # Enums from .duckdb import ( diff --git a/tools/pythonpkg/duckdb_python.cpp b/tools/pythonpkg/duckdb_python.cpp index 1289c5743833..bb2ee522d72c 100644 --- a/tools/pythonpkg/duckdb_python.cpp +++ b/tools/pythonpkg/duckdb_python.cpp @@ -71,8 +71,7 @@ static py::list PyTokenize(const string &query) { } static void InitializeConnectionMethods(py::module_ &m) { - // We define these "wrapper" methods inside of C++ because they are overloaded - // every other wrapper method is defined inside of __init__.py + // We define these "wrapper" methods manually because they are overloaded m.def( "arrow", [](idx_t rows_per_batch, shared_ptr conn) -> duckdb::pyarrow::Table { diff --git a/tools/pythonpkg/scripts/generate_connection_wrapper_methods.py b/tools/pythonpkg/scripts/generate_connection_wrapper_methods.py new file mode 100644 index 000000000000..b76ee249e510 --- /dev/null +++ b/tools/pythonpkg/scripts/generate_connection_wrapper_methods.py @@ -0,0 +1,142 @@ +import os +import json + +os.chdir(os.path.dirname(__file__)) + +JSON_PATH = os.path.join("connection_methods.json") +DUCKDB_INIT_FILE = os.path.join("..", "duckdb", "__init__.py") + +START_MARKER = "# START OF CONNECTION WRAPPER" +END_MARKER = "# END OF CONNECTION WRAPPER" + +# Read the DUCKDB_INIT_FILE file +with open(DUCKDB_INIT_FILE, 'r') as source_file: + source_code = source_file.readlines() + +start_index = -1 +end_index = -1 +for i, line in enumerate(source_code): + if line.startswith(START_MARKER): + # TODO: handle the case where the start marker appears multiple times + start_index = i + elif line.startswith(END_MARKER): + # TODO: ditto ^ + end_index = i + +if start_index == -1 or end_index == -1: + raise ValueError("Couldn't find start or end marker in source file") + +start_section = source_code[: start_index + 1] +end_section = source_code[end_index:] +# ---- Generate the definition code from the json ---- + +# Read the JSON file +with open(JSON_PATH, 'r') as json_file: + connection_methods = json.load(json_file) + +# Artificial "wrapper" methods on pandas.DataFrames +SPECIAL_METHODS = [ + {'name': 'project', 'args': [{'name': "*args", 'type': 'Any'}], 'return': 'DuckDBPyRelation'}, + {'name': 'distinct', 'args': [{'name': "*args", 'type': 'Any'}], 'return': 'DuckDBPyRelation'}, + {'name': 'write_csv', 'args': [{'name': "*args", 'type': 'Any'}], 'return': 'None'}, + {'name': 'aggregate', 'args': [{'name': "*args", 'type': 'Any'}], 'return': 'DuckDBPyRelation'}, + {'name': 'alias', 'args': [{'name': "*args", 'type': 'Any'}], 'return': 'DuckDBPyRelation'}, + {'name': 'filter', 'args': [{'name': "*args", 'type': 'Any'}], 'return': 'DuckDBPyRelation'}, + {'name': 'limit', 'args': [{'name': "*args", 'type': 'Any'}], 'return': 'DuckDBPyRelation'}, + {'name': 'order', 'args': [{'name': "*args", 'type': 'Any'}], 'return': 'DuckDBPyRelation'}, + {'name': 'query_df', 'args': [{'name': "*args", 'type': 'Any'}], 'return': 'DuckDBPyRelation'}, +] + +connection_methods.extend(SPECIAL_METHODS) + +body = [] + +SPECIAL_METHOD_NAMES = [x['name'] for x in SPECIAL_METHODS] + + +def generate_arguments(name, method) -> str: + arguments = [] + if name in SPECIAL_METHOD_NAMES: + # We add 'df' to these methods because they operate on a DataFrame + arguments.append('df') + + if 'args' in method: + for arg in method['args']: + res = arg['name'] + if 'default' in arg: + res += f" = {arg['default']}" + arguments.append(res) + arguments.append('**kwargs') + return ', '.join(arguments) + + +def generate_parameters(method) -> str: + arguments = [] + if 'args' in method: + for arg in method['args']: + arguments.append(f"{arg['name']}") + arguments.append('**kwargs') + return ', '.join(arguments) + + +def generate_function_call(name, method) -> str: + function_call = '' + if name in SPECIAL_METHOD_NAMES: + function_call += 'from_df(df).' + + REMAPPED_FUNCTIONS = {'alias': 'set_alias', 'query_df': 'query'} + if name in REMAPPED_FUNCTIONS: + function_name = REMAPPED_FUNCTIONS[name] + else: + function_name = name + function_call += function_name + return function_call + + +def create_definition(name, method) -> str: + print(method) + arguments = generate_arguments(name, method) + parameters = generate_parameters(method) + function_call = generate_function_call(name, method) + + func = f""" +def {name}({arguments}): + conn = __get_connection__(*kwargs) + return conn.{function_call}({parameters}) +_exported_symbols.append('{name}') +""" + return func + + +# We have "duplicate" methods, which are overloaded +written_methods = set() + +for method in connection_methods: + if isinstance(method['name'], list): + names = method['name'] + else: + names = [method['name']] + + # Artificially add 'connection' keyword argument + if 'kwargs' not in method: + method['kwargs'] = [] + method['kwargs'].append({'name': 'connection', 'type': 'DuckDBPyConnection'}) + + for name in names: + if name in written_methods: + continue + body.append(create_definition(name, method)) + written_methods.add(name) + +# ---- End of generation code ---- + +with_newlines = body +# Recreate the file content by concatenating all the pieces together + +new_content = start_section + with_newlines + end_section + +print(''.join(with_newlines)) + +# Write out the modified DUCKDB_INIT_FILE file +with open(DUCKDB_INIT_FILE, 'w') as source_file: + source_file.write("".join(new_content)) From a2c33c959c89f20281d475b8272793f52e3a849e Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 10 Apr 2024 22:49:19 +0200 Subject: [PATCH 309/603] pass all tests --- tools/pythonpkg/duckdb/__init__.py | 397 ++++++++++++++---- .../generate_connection_wrapper_methods.py | 26 +- .../tests/fast/api/test_duckdb_connection.py | 7 + 3 files changed, 336 insertions(+), 94 deletions(-) diff --git a/tools/pythonpkg/duckdb/__init__.py b/tools/pythonpkg/duckdb/__init__.py index f4f6af797c6e..29252c856881 100644 --- a/tools/pythonpkg/duckdb/__init__.py +++ b/tools/pythonpkg/duckdb/__init__.py @@ -50,391 +50,610 @@ arrow ) -def __get_connection__(**kwargs): - if 'connection' in kwargs: - return kwargs.pop('connection') - else: - return duckdb.connect(":default:") - # NOTE: this section is generated by tools/pythonpkg/scripts/generate_connection_wrapper_methods.py. # Do not edit this section manually, your changes will be overwritten! # START OF CONNECTION WRAPPER def cursor(**kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.cursor(**kwargs) _exported_symbols.append('cursor') def register_filesystem(filesystem, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.register_filesystem(filesystem, **kwargs) _exported_symbols.append('register_filesystem') def unregister_filesystem(name, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.unregister_filesystem(name, **kwargs) _exported_symbols.append('unregister_filesystem') def list_filesystems(**kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.list_filesystems(**kwargs) _exported_symbols.append('list_filesystems') def filesystem_is_registered(name, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.filesystem_is_registered(name, **kwargs) _exported_symbols.append('filesystem_is_registered') def create_function(name, function, parameters = None, return_type = None, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.create_function(name, function, parameters, return_type, **kwargs) _exported_symbols.append('create_function') def remove_function(name, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.remove_function(name, **kwargs) _exported_symbols.append('remove_function') def sqltype(type_str, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.sqltype(type_str, **kwargs) _exported_symbols.append('sqltype') def dtype(type_str, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.dtype(type_str, **kwargs) _exported_symbols.append('dtype') def type(type_str, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.type(type_str, **kwargs) _exported_symbols.append('type') def array_type(type, size, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.array_type(type, size, **kwargs) _exported_symbols.append('array_type') def list_type(type, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.list_type(type, **kwargs) _exported_symbols.append('list_type') def union_type(members, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.union_type(members, **kwargs) _exported_symbols.append('union_type') def string_type(collation = "", **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.string_type(collation, **kwargs) _exported_symbols.append('string_type') def enum_type(name, type, values, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.enum_type(name, type, values, **kwargs) _exported_symbols.append('enum_type') def decimal_type(width, scale, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.decimal_type(width, scale, **kwargs) _exported_symbols.append('decimal_type') def struct_type(fields, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.struct_type(fields, **kwargs) _exported_symbols.append('struct_type') def row_type(fields, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.row_type(fields, **kwargs) _exported_symbols.append('row_type') def map_type(key, value, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.map_type(key, value, **kwargs) _exported_symbols.append('map_type') def duplicate(**kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.duplicate(**kwargs) _exported_symbols.append('duplicate') def execute(query, parameters = None, multiple_parameter_sets = False, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.execute(query, parameters, multiple_parameter_sets, **kwargs) _exported_symbols.append('execute') def executemany(query, parameters = None, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.executemany(query, parameters, **kwargs) _exported_symbols.append('executemany') def close(**kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.close(**kwargs) _exported_symbols.append('close') def interrupt(**kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.interrupt(**kwargs) _exported_symbols.append('interrupt') def fetchone(**kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.fetchone(**kwargs) _exported_symbols.append('fetchone') def fetchmany(size = 1, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.fetchmany(size, **kwargs) _exported_symbols.append('fetchmany') def fetchall(**kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.fetchall(**kwargs) _exported_symbols.append('fetchall') def fetchnumpy(**kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.fetchnumpy(**kwargs) _exported_symbols.append('fetchnumpy') def fetchdf(**kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.fetchdf(**kwargs) _exported_symbols.append('fetchdf') def fetch_df(**kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.fetch_df(**kwargs) _exported_symbols.append('fetch_df') -def df(**kwargs): - conn = __get_connection__(*kwargs) - return conn.df(**kwargs) -_exported_symbols.append('df') - def fetch_df_chunk(vectors_per_chunk = 1, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.fetch_df_chunk(vectors_per_chunk, **kwargs) _exported_symbols.append('fetch_df_chunk') def pl(rows_per_batch = 1000000, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.pl(rows_per_batch, **kwargs) _exported_symbols.append('pl') def fetch_arrow_table(rows_per_batch = 1000000, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.fetch_arrow_table(rows_per_batch, **kwargs) _exported_symbols.append('fetch_arrow_table') -def arrow(rows_per_batch = 1000000, **kwargs): - conn = __get_connection__(*kwargs) - return conn.arrow(rows_per_batch, **kwargs) -_exported_symbols.append('arrow') - def fetch_record_batch(rows_per_batch = 1000000, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.fetch_record_batch(rows_per_batch, **kwargs) _exported_symbols.append('fetch_record_batch') def torch(**kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.torch(**kwargs) _exported_symbols.append('torch') def tf(**kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.tf(**kwargs) _exported_symbols.append('tf') def begin(**kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.begin(**kwargs) _exported_symbols.append('begin') def commit(**kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.commit(**kwargs) _exported_symbols.append('commit') def rollback(**kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.rollback(**kwargs) _exported_symbols.append('rollback') def append(table_name, df, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.append(table_name, df, **kwargs) _exported_symbols.append('append') def register(view_name, python_object, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.register(view_name, python_object, **kwargs) _exported_symbols.append('register') def unregister(view_name, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.unregister(view_name, **kwargs) _exported_symbols.append('unregister') def table(table_name, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.table(table_name, **kwargs) _exported_symbols.append('table') def view(view_name, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.view(view_name, **kwargs) _exported_symbols.append('view') def values(values, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.values(values, **kwargs) _exported_symbols.append('values') def table_function(name, parameters = None, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.table_function(name, parameters, **kwargs) _exported_symbols.append('table_function') def read_json(name, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.read_json(name, **kwargs) _exported_symbols.append('read_json') def extract_statements(query, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.extract_statements(query, **kwargs) _exported_symbols.append('extract_statements') def sql(query, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.sql(query, **kwargs) _exported_symbols.append('sql') def query(query, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.query(query, **kwargs) _exported_symbols.append('query') def from_query(query, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.from_query(query, **kwargs) _exported_symbols.append('from_query') def read_csv(path_or_buffer, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.read_csv(path_or_buffer, **kwargs) _exported_symbols.append('read_csv') def from_csv_auto(path_or_buffer, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.from_csv_auto(path_or_buffer, **kwargs) _exported_symbols.append('from_csv_auto') def from_df(df, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.from_df(df, **kwargs) _exported_symbols.append('from_df') def from_arrow(arrow_object, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.from_arrow(arrow_object, **kwargs) _exported_symbols.append('from_arrow') def from_parquet(file_glob, binary_as_string = False, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.from_parquet(file_glob, binary_as_string, **kwargs) _exported_symbols.append('from_parquet') def read_parquet(file_glob, binary_as_string = False, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.read_parquet(file_glob, binary_as_string, **kwargs) _exported_symbols.append('read_parquet') def from_substrait(proto, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.from_substrait(proto, **kwargs) _exported_symbols.append('from_substrait') def get_substrait(query, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.get_substrait(query, **kwargs) _exported_symbols.append('get_substrait') def get_substrait_json(query, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.get_substrait_json(query, **kwargs) _exported_symbols.append('get_substrait_json') def from_substrait_json(json, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.from_substrait_json(json, **kwargs) _exported_symbols.append('from_substrait_json') def get_table_names(query, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.get_table_names(query, **kwargs) _exported_symbols.append('get_table_names') def install_extension(extension, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.install_extension(extension, **kwargs) _exported_symbols.append('install_extension') def load_extension(extension, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.load_extension(extension, **kwargs) _exported_symbols.append('load_extension') def project(df, *args, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.from_df(df).project(*args, **kwargs) _exported_symbols.append('project') def distinct(df, *args, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.from_df(df).distinct(*args, **kwargs) _exported_symbols.append('distinct') def write_csv(df, *args, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.from_df(df).write_csv(*args, **kwargs) _exported_symbols.append('write_csv') def aggregate(df, *args, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.from_df(df).aggregate(*args, **kwargs) _exported_symbols.append('aggregate') def alias(df, *args, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.from_df(df).set_alias(*args, **kwargs) _exported_symbols.append('alias') def filter(df, *args, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.from_df(df).filter(*args, **kwargs) _exported_symbols.append('filter') def limit(df, *args, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.from_df(df).limit(*args, **kwargs) _exported_symbols.append('limit') def order(df, *args, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.from_df(df).order(*args, **kwargs) _exported_symbols.append('order') def query_df(df, *args, **kwargs): - conn = __get_connection__(*kwargs) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") return conn.from_df(df).query(*args, **kwargs) _exported_symbols.append('query_df') + +def description(**kwargs): + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") + return conn.description +_exported_symbols.append('description') + +def rowcount(**kwargs): + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") + return conn.rowcount +_exported_symbols.append('rowcount') # END OF CONNECTION WRAPPER # Enums diff --git a/tools/pythonpkg/scripts/generate_connection_wrapper_methods.py b/tools/pythonpkg/scripts/generate_connection_wrapper_methods.py index b76ee249e510..c890bc8bf2da 100644 --- a/tools/pythonpkg/scripts/generate_connection_wrapper_methods.py +++ b/tools/pythonpkg/scripts/generate_connection_wrapper_methods.py @@ -47,11 +47,18 @@ {'name': 'query_df', 'args': [{'name': "*args", 'type': 'Any'}], 'return': 'DuckDBPyRelation'}, ] +READONLY_PROPERTIES = [ + {'name': 'description', 'return': 'str'}, + {'name': 'rowcount', 'return': 'int'}, +] + connection_methods.extend(SPECIAL_METHODS) +connection_methods.extend(READONLY_PROPERTIES) body = [] SPECIAL_METHOD_NAMES = [x['name'] for x in SPECIAL_METHODS] +READONLY_PROPERTY_NAMES = [x['name'] for x in READONLY_PROPERTIES] def generate_arguments(name, method) -> str: @@ -70,13 +77,16 @@ def generate_arguments(name, method) -> str: return ', '.join(arguments) -def generate_parameters(method) -> str: +def generate_parameters(name, method) -> str: + if name in READONLY_PROPERTY_NAMES: + return '' arguments = [] if 'args' in method: for arg in method['args']: arguments.append(f"{arg['name']}") arguments.append('**kwargs') - return ', '.join(arguments) + result = ', '.join(arguments) + return '(' + result + ')' def generate_function_call(name, method) -> str: @@ -96,13 +106,16 @@ def generate_function_call(name, method) -> str: def create_definition(name, method) -> str: print(method) arguments = generate_arguments(name, method) - parameters = generate_parameters(method) + parameters = generate_parameters(name, method) function_call = generate_function_call(name, method) func = f""" def {name}({arguments}): - conn = __get_connection__(*kwargs) - return conn.{function_call}({parameters}) + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") + return conn.{function_call}{parameters} _exported_symbols.append('{name}') """ return func @@ -125,6 +138,9 @@ def {name}({arguments}): for name in names: if name in written_methods: continue + if name in ['arrow', 'df']: + # These methods are ambiguous and are handled in C++ code instead + continue body.append(create_definition(name, method)) written_methods.add(name) diff --git a/tools/pythonpkg/tests/fast/api/test_duckdb_connection.py b/tools/pythonpkg/tests/fast/api/test_duckdb_connection.py index c9c26a867ed2..4d1afb7b3bf4 100644 --- a/tools/pythonpkg/tests/fast/api/test_duckdb_connection.py +++ b/tools/pythonpkg/tests/fast/api/test_duckdb_connection.py @@ -84,6 +84,13 @@ def test_duplicate(self): with pytest.raises(duckdb.CatalogException): dup_conn.table("tbl").fetchall() + def test_readonly_properties(self): + duckdb.execute("select 42") + description = duckdb.description() + rowcount = duckdb.rowcount() + assert description == [('42', 'NUMBER', None, None, None, None, None)] + assert rowcount == -1 + def test_execute(self): assert [([4, 2],)] == duckdb.execute("select [4,2]").fetchall() From 6353eebdeb9e1a695e4ba642415cf8fa1195892f Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 10 Apr 2024 23:03:33 +0200 Subject: [PATCH 310/603] cleanup --- .../scripts/connection_wrapper_methods.json | 100 ++++++++++++++++++ .../generate_connection_wrapper_methods.py | 44 ++++---- .../scripts/generate_function_definition.py | 3 - 3 files changed, 118 insertions(+), 29 deletions(-) create mode 100644 tools/pythonpkg/scripts/connection_wrapper_methods.json delete mode 100644 tools/pythonpkg/scripts/generate_function_definition.py diff --git a/tools/pythonpkg/scripts/connection_wrapper_methods.json b/tools/pythonpkg/scripts/connection_wrapper_methods.json new file mode 100644 index 000000000000..2be14ae79a4f --- /dev/null +++ b/tools/pythonpkg/scripts/connection_wrapper_methods.json @@ -0,0 +1,100 @@ +[ + { + "name": "project", + "args": [ + { + "name": "*args", + "type": "Any" + } + ], + "return": "DuckDBPyRelation" + }, + { + "name": "distinct", + "args": [ + { + "name": "*args", + "type": "Any" + } + ], + "return": "DuckDBPyRelation" + }, + { + "name": "write_csv", + "args": [ + { + "name": "*args", + "type": "Any" + } + ], + "return": "None" + }, + { + "name": "aggregate", + "args": [ + { + "name": "*args", + "type": "Any" + } + ], + "return": "DuckDBPyRelation" + }, + { + "name": "alias", + "args": [ + { + "name": "*args", + "type": "Any" + } + ], + "return": "DuckDBPyRelation" + }, + { + "name": "filter", + "args": [ + { + "name": "*args", + "type": "Any" + } + ], + "return": "DuckDBPyRelation" + }, + { + "name": "limit", + "args": [ + { + "name": "*args", + "type": "Any" + } + ], + "return": "DuckDBPyRelation" + }, + { + "name": "order", + "args": [ + { + "name": "*args", + "type": "Any" + } + ], + "return": "DuckDBPyRelation" + }, + { + "name": "query_df", + "args": [ + { + "name": "*args", + "type": "Any" + } + ], + "return": "DuckDBPyRelation" + }, + { + "name": "description", + "return": "str" + }, + { + "name": "rowcount", + "return": "int" + } +] diff --git a/tools/pythonpkg/scripts/generate_connection_wrapper_methods.py b/tools/pythonpkg/scripts/generate_connection_wrapper_methods.py index c890bc8bf2da..e716765f6816 100644 --- a/tools/pythonpkg/scripts/generate_connection_wrapper_methods.py +++ b/tools/pythonpkg/scripts/generate_connection_wrapper_methods.py @@ -4,6 +4,7 @@ os.chdir(os.path.dirname(__file__)) JSON_PATH = os.path.join("connection_methods.json") +WRAPPER_JSON_PATH = os.path.join("connection_wrapper_methods.json") DUCKDB_INIT_FILE = os.path.join("..", "duckdb", "__init__.py") START_MARKER = "# START OF CONNECTION WRAPPER" @@ -30,35 +31,25 @@ end_section = source_code[end_index:] # ---- Generate the definition code from the json ---- +methods = [] + # Read the JSON file with open(JSON_PATH, 'r') as json_file: connection_methods = json.load(json_file) -# Artificial "wrapper" methods on pandas.DataFrames -SPECIAL_METHODS = [ - {'name': 'project', 'args': [{'name': "*args", 'type': 'Any'}], 'return': 'DuckDBPyRelation'}, - {'name': 'distinct', 'args': [{'name': "*args", 'type': 'Any'}], 'return': 'DuckDBPyRelation'}, - {'name': 'write_csv', 'args': [{'name': "*args", 'type': 'Any'}], 'return': 'None'}, - {'name': 'aggregate', 'args': [{'name': "*args", 'type': 'Any'}], 'return': 'DuckDBPyRelation'}, - {'name': 'alias', 'args': [{'name': "*args", 'type': 'Any'}], 'return': 'DuckDBPyRelation'}, - {'name': 'filter', 'args': [{'name': "*args", 'type': 'Any'}], 'return': 'DuckDBPyRelation'}, - {'name': 'limit', 'args': [{'name': "*args", 'type': 'Any'}], 'return': 'DuckDBPyRelation'}, - {'name': 'order', 'args': [{'name': "*args", 'type': 'Any'}], 'return': 'DuckDBPyRelation'}, - {'name': 'query_df', 'args': [{'name': "*args", 'type': 'Any'}], 'return': 'DuckDBPyRelation'}, -] - -READONLY_PROPERTIES = [ - {'name': 'description', 'return': 'str'}, - {'name': 'rowcount', 'return': 'int'}, -] - -connection_methods.extend(SPECIAL_METHODS) -connection_methods.extend(READONLY_PROPERTIES) +with open(WRAPPER_JSON_PATH, 'r') as json_file: + wrapper_methods = json.load(json_file) -body = [] +methods.extend(connection_methods) +methods.extend(wrapper_methods) -SPECIAL_METHOD_NAMES = [x['name'] for x in SPECIAL_METHODS] -READONLY_PROPERTY_NAMES = [x['name'] for x in READONLY_PROPERTIES] +# On DuckDBPyConnection these are read_only_properties, they're basically functions without requiring () to invoke +# that's not possible on 'duckdb' so it becomes a function call with no arguments (i.e duckdb.description()) +READONLY_PROPERTY_NAMES = ['description', 'rowcount'] + +# These methods are not directly DuckDBPyConnection methods, +# they first call 'from_df' and then call a method on the created DuckDBPyRelation +SPECIAL_METHOD_NAMES = [x['name'] for x in wrapper_methods if x['name'] not in READONLY_PROPERTY_NAMES] def generate_arguments(name, method) -> str: @@ -89,7 +80,7 @@ def generate_parameters(name, method) -> str: return '(' + result + ')' -def generate_function_call(name, method) -> str: +def generate_function_call(name) -> str: function_call = '' if name in SPECIAL_METHOD_NAMES: function_call += 'from_df(df).' @@ -107,7 +98,7 @@ def create_definition(name, method) -> str: print(method) arguments = generate_arguments(name, method) parameters = generate_parameters(name, method) - function_call = generate_function_call(name, method) + function_call = generate_function_call(name) func = f""" def {name}({arguments}): @@ -124,7 +115,8 @@ def {name}({arguments}): # We have "duplicate" methods, which are overloaded written_methods = set() -for method in connection_methods: +body = [] +for method in methods: if isinstance(method['name'], list): names = method['name'] else: diff --git a/tools/pythonpkg/scripts/generate_function_definition.py b/tools/pythonpkg/scripts/generate_function_definition.py deleted file mode 100644 index aca68675eb43..000000000000 --- a/tools/pythonpkg/scripts/generate_function_definition.py +++ /dev/null @@ -1,3 +0,0 @@ -class FunctionDefinition: - def __init__(self): - pass From 2ff91ad07b0cf63e32f46e319baf052156fb5f3b Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 10 Apr 2024 23:19:49 +0200 Subject: [PATCH 311/603] add the right arguments for the (DuckDBPyRelation) functions we're wrapping --- tools/pythonpkg/duckdb/__init__.py | 32 ++++++------- .../scripts/connection_wrapper_methods.json | 48 +++++++++++-------- 2 files changed, 44 insertions(+), 36 deletions(-) diff --git a/tools/pythonpkg/duckdb/__init__.py b/tools/pythonpkg/duckdb/__init__.py index 29252c856881..5d2d56e2dd4e 100644 --- a/tools/pythonpkg/duckdb/__init__.py +++ b/tools/pythonpkg/duckdb/__init__.py @@ -567,20 +567,20 @@ def load_extension(extension, **kwargs): return conn.load_extension(extension, **kwargs) _exported_symbols.append('load_extension') -def project(df, *args, **kwargs): +def project(df, project_expr, **kwargs): if 'connection' in kwargs: conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") - return conn.from_df(df).project(*args, **kwargs) + return conn.from_df(df).project(project_expr, **kwargs) _exported_symbols.append('project') -def distinct(df, *args, **kwargs): +def distinct(df, **kwargs): if 'connection' in kwargs: conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") - return conn.from_df(df).distinct(*args, **kwargs) + return conn.from_df(df).distinct(**kwargs) _exported_symbols.append('distinct') def write_csv(df, *args, **kwargs): @@ -591,52 +591,52 @@ def write_csv(df, *args, **kwargs): return conn.from_df(df).write_csv(*args, **kwargs) _exported_symbols.append('write_csv') -def aggregate(df, *args, **kwargs): +def aggregate(df, aggr_expr, group_expr = "", **kwargs): if 'connection' in kwargs: conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") - return conn.from_df(df).aggregate(*args, **kwargs) + return conn.from_df(df).aggregate(aggr_expr, group_expr, **kwargs) _exported_symbols.append('aggregate') -def alias(df, *args, **kwargs): +def alias(df, alias, **kwargs): if 'connection' in kwargs: conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") - return conn.from_df(df).set_alias(*args, **kwargs) + return conn.from_df(df).set_alias(alias, **kwargs) _exported_symbols.append('alias') -def filter(df, *args, **kwargs): +def filter(df, filter_expr, **kwargs): if 'connection' in kwargs: conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") - return conn.from_df(df).filter(*args, **kwargs) + return conn.from_df(df).filter(filter_expr, **kwargs) _exported_symbols.append('filter') -def limit(df, *args, **kwargs): +def limit(df, n, offset = 0, **kwargs): if 'connection' in kwargs: conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") - return conn.from_df(df).limit(*args, **kwargs) + return conn.from_df(df).limit(n, offset, **kwargs) _exported_symbols.append('limit') -def order(df, *args, **kwargs): +def order(df, order_expr, **kwargs): if 'connection' in kwargs: conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") - return conn.from_df(df).order(*args, **kwargs) + return conn.from_df(df).order(order_expr, **kwargs) _exported_symbols.append('order') -def query_df(df, *args, **kwargs): +def query_df(df, virtual_table_name, sql_query, **kwargs): if 'connection' in kwargs: conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") - return conn.from_df(df).query(*args, **kwargs) + return conn.from_df(df).query(virtual_table_name, sql_query, **kwargs) _exported_symbols.append('query_df') def description(**kwargs): diff --git a/tools/pythonpkg/scripts/connection_wrapper_methods.json b/tools/pythonpkg/scripts/connection_wrapper_methods.json index 2be14ae79a4f..65ac4afd47f7 100644 --- a/tools/pythonpkg/scripts/connection_wrapper_methods.json +++ b/tools/pythonpkg/scripts/connection_wrapper_methods.json @@ -3,20 +3,14 @@ "name": "project", "args": [ { - "name": "*args", - "type": "Any" + "name": "project_expr", + "type": "str" } ], "return": "DuckDBPyRelation" }, { "name": "distinct", - "args": [ - { - "name": "*args", - "type": "Any" - } - ], "return": "DuckDBPyRelation" }, { @@ -33,8 +27,13 @@ "name": "aggregate", "args": [ { - "name": "*args", - "type": "Any" + "name": "aggr_expr", + "type": "str" + }, + { + "name": "group_expr", + "type" : "str", + "default": "\"\"" } ], "return": "DuckDBPyRelation" @@ -43,8 +42,8 @@ "name": "alias", "args": [ { - "name": "*args", - "type": "Any" + "name": "alias", + "type": "str" } ], "return": "DuckDBPyRelation" @@ -53,8 +52,8 @@ "name": "filter", "args": [ { - "name": "*args", - "type": "Any" + "name": "filter_expr", + "type": "str" } ], "return": "DuckDBPyRelation" @@ -63,8 +62,13 @@ "name": "limit", "args": [ { - "name": "*args", - "type": "Any" + "name": "n", + "type": "int" + }, + { + "name": "offset", + "type": "int", + "default": "0" } ], "return": "DuckDBPyRelation" @@ -73,8 +77,8 @@ "name": "order", "args": [ { - "name": "*args", - "type": "Any" + "name": "order_expr", + "type": "str" } ], "return": "DuckDBPyRelation" @@ -83,8 +87,12 @@ "name": "query_df", "args": [ { - "name": "*args", - "type": "Any" + "name": "virtual_table_name", + "type": "str" + }, + { + "name": "sql_query", + "type": "str" } ], "return": "DuckDBPyRelation" From c94e308be0b3e86895b2b3954e82c7e2d054fe44 Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 10 Apr 2024 23:40:33 +0200 Subject: [PATCH 312/603] generating the stubs for the connection wrappers too --- tools/pythonpkg/duckdb-stubs/__init__.pyi | 227 +++++++----------- .../scripts/connection_wrapper_methods.json | 2 +- 2 files changed, 84 insertions(+), 145 deletions(-) diff --git a/tools/pythonpkg/duckdb-stubs/__init__.pyi b/tools/pythonpkg/duckdb-stubs/__init__.pyi index 748e1102c290..0a5587e628be 100644 --- a/tools/pythonpkg/duckdb-stubs/__init__.pyi +++ b/tools/pythonpkg/duckdb-stubs/__init__.pyi @@ -572,149 +572,88 @@ class token_type: # stubgen override - this gets removed by stubgen but it shouldn't def __members__(self) -> object: ... -def aggregate(df: pandas.DataFrame, aggr_expr: str, group_expr: str = ..., connection: DuckDBPyConnection = ...) -> DuckDBPyRelation: ... -def alias(df: pandas.DataFrame, alias: str, connection: DuckDBPyConnection = ...) -> DuckDBPyRelation: ... def connect(database: str = ..., read_only: bool = ..., config: dict = ...) -> DuckDBPyConnection: ... -def distinct(df: pandas.DataFrame, connection: DuckDBPyConnection = ...) -> DuckDBPyRelation: ... -def filter(df: pandas.DataFrame, filter_expr: str, connection: DuckDBPyConnection = ...) -> DuckDBPyRelation: ... -def from_substrait_json(jsonm: str, connection: DuckDBPyConnection = ...) -> DuckDBPyRelation: ... -def limit(df: pandas.DataFrame, n: int, connection: DuckDBPyConnection = ...) -> DuckDBPyRelation: ... -def order(df: pandas.DataFrame, order_expr: str, connection: DuckDBPyConnection = ...) -> DuckDBPyRelation: ... -def project(df: pandas.DataFrame, project_expr: str, connection: DuckDBPyConnection = ...) -> DuckDBPyRelation: ... -def write_csv(df: pandas.DataFrame, file_name: str, connection: DuckDBPyConnection = ...) -> None: ... -def read_json( - file_name: str, - columns: Optional[Dict[str,str]] = None, - sample_size: Optional[int] = None, - maximum_depth: Optional[int] = None, - format: Optional[str] = None, - records: Optional[str] = None, - connection: DuckDBPyConnection = ... -) -> DuckDBPyRelation: ... -def read_csv( - path_or_buffer: Union[str, StringIO, TextIOBase], - header: Optional[bool | int] = None, - compression: Optional[str] = None, - sep: Optional[str] = None, - delimiter: Optional[str] = None, - dtype: Optional[Dict[str, str] | List[str]] = None, - na_values: Optional[str] = None, - skiprows: Optional[int] = None, - quotechar: Optional[str] = None, - escapechar: Optional[str] = None, - encoding: Optional[str] = None, - parallel: Optional[bool] = None, - date_format: Optional[str] = None, - timestamp_format: Optional[str] = None, - sample_size: Optional[int] = None, - all_varchar: Optional[bool] = None, - normalize_names: Optional[bool] = None, - filename: Optional[bool] = None, - connection: DuckDBPyConnection = ... -) -> DuckDBPyRelation: ... -def from_csv_auto( - name: str, - header: Optional[bool | int] = None, - compression: Optional[str] = None, - sep: Optional[str] = None, - delimiter: Optional[str] = None, - dtype: Optional[Dict[str, str] | List[str]] = None, - na_values: Optional[str] = None, - skiprows: Optional[int] = None, - quotechar: Optional[str] = None, - escapechar: Optional[str] = None, - encoding: Optional[str] = None, - parallel: Optional[bool] = None, - date_format: Optional[str] = None, - timestamp_format: Optional[str] = None, - sample_size: Optional[int] = None, - all_varchar: Optional[bool] = None, - normalize_names: Optional[bool] = None, - filename: Optional[bool] = None, - null_padding: Optional[bool] = None, - connection: DuckDBPyConnection = ... -) -> DuckDBPyRelation: ... - -def append(table_name: str, df: pandas.DataFrame, connection: DuckDBPyConnection = ...) -> DuckDBPyConnection: ... -def arrow(rows_per_batch: int = ..., connection: DuckDBPyConnection = ...) -> pyarrow.lib.Table: ... -def begin(connection: DuckDBPyConnection = ...) -> DuckDBPyConnection: ... -def close(connection: DuckDBPyConnection = ...) -> None: ... -def commit(connection: DuckDBPyConnection = ...) -> DuckDBPyConnection: ... -def cursor(connection: DuckDBPyConnection = ...) -> DuckDBPyConnection: ... -def df(connection: DuckDBPyConnection = ...) -> pandas.DataFrame: ... -def description(connection: DuckDBPyConnection = ...) -> Optional[List[Any]]: ... -def rowcount(connection: DuckDBPyConnection = ...) -> int: ... -def duplicate(connection: DuckDBPyConnection = ...) -> DuckDBPyConnection: ... -def execute(query: str, parameters: object = ..., multiple_parameter_sets: bool = ..., connection: DuckDBPyConnection = ...) -> DuckDBPyConnection: ... -def executemany(query: str, parameters: object = ..., connection: DuckDBPyConnection = ...) -> DuckDBPyConnection: ... -def fetch_arrow_table(rows_per_batch: int = ..., connection: DuckDBPyConnection = ...) -> pyarrow.lib.Table: ... -def fetch_df(*args, connection: DuckDBPyConnection = ..., **kwargs) -> pandas.DataFrame: ... -def fetch_df_chunk(*args, connection: DuckDBPyConnection = ..., **kwargs) -> pandas.DataFrame: ... -def fetch_record_batch(rows_per_batch: int = ..., connection: DuckDBPyConnection = ...) -> pyarrow.lib.RecordBatchReader: ... -def fetchall(connection: DuckDBPyConnection = ...) -> List[Any]: ... -def fetchdf(*args, connection: DuckDBPyConnection = ..., **kwargs) -> pandas.DataFrame: ... -def fetchmany(size: int = ..., connection: DuckDBPyConnection = ...) -> List[Any]: ... -def fetchnumpy(connection: DuckDBPyConnection = ...) -> dict: ... -def fetchone(connection: DuckDBPyConnection = ...) -> Optional[tuple]: ... -def from_arrow(arrow_object: object, connection: DuckDBPyConnection = ...) -> DuckDBPyRelation: ... -def from_df(df: pandas.DataFrame = ..., connection: DuckDBPyConnection = ...) -> DuckDBPyRelation: ... -@overload -def read_parquet(file_glob: str, binary_as_string: bool = ..., *, file_row_number: bool = ..., filename: bool = ..., hive_partitioning: bool = ..., union_by_name: bool = ..., connection: DuckDBPyConnection = ...) -> DuckDBPyRelation: ... -@overload -def read_parquet(file_globs: List[str], binary_as_string: bool = ..., *, file_row_number: bool = ..., filename: bool = ..., hive_partitioning: bool = ..., union_by_name: bool = ..., connection: DuckDBPyConnection = ...) -> DuckDBPyRelation: ... -@overload -def from_parquet(file_glob: str, binary_as_string: bool = ..., *, file_row_number: bool = ..., filename: bool = ..., hive_partitioning: bool = ..., union_by_name: bool = ..., connection: DuckDBPyConnection = ...) -> DuckDBPyRelation: ... -@overload -def from_parquet(file_globs: List[str], binary_as_string: bool = ..., *, file_row_number: bool = ..., filename: bool = ..., hive_partitioning: bool = ..., union_by_name: bool = ..., connection: DuckDBPyConnection = ...) -> DuckDBPyRelation: ... -def from_substrait(proto: bytes, connection: DuckDBPyConnection = ...) -> DuckDBPyRelation: ... -def get_substrait(query: str, connection: DuckDBPyConnection = ...) -> DuckDBPyRelation: ... -def get_substrait_json(query: str, connection: DuckDBPyConnection = ...) -> DuckDBPyRelation: ... -def get_table_names(query: str, connection: DuckDBPyConnection = ...) -> Set[str]: ... -def install_extension(*args, connection: DuckDBPyConnection = ..., **kwargs) -> None: ... -def interrupt(connection: DuckDBPyConnection = ...) -> None: ... -def list_filesystems(connection: DuckDBPyConnection = ...) -> List[Any]: ... -def filesystem_is_registered(name: str, connection: DuckDBPyConnection = ...) -> bool: ... -def load_extension(extension: str, connection: DuckDBPyConnection = ...) -> None: ... -def pl(rows_per_batch: int = ..., connection: DuckDBPyConnection = ...) -> polars.DataFrame: ... -def torch(connection: DuckDBPyConnection = ...) -> dict: ... -def tf(self, connection: DuckDBPyConnection = ...) -> dict: ... -def register(view_name: str, python_object: object, connection: DuckDBPyConnection = ...) -> DuckDBPyConnection: ... -def remove_function(name: str, connection : DuckDBPyConnection = ...) -> DuckDBPyConnection: ... -def create_function( - name: str, - func: Callable, - parameters: Optional[List[DuckDBPyType]] = None, - return_type: Optional[DuckDBPyType] = None, - type: Optional[PythonUDFType] = PythonUDFType.NATIVE, - null_handling: Optional[FunctionNullHandling] = FunctionNullHandling.DEFAULT, - exception_handling: Optional[PythonExceptionHandling] = PythonExceptionHandling.DEFAULT, - side_effects: Optional[bool] = False, - connection: DuckDBPyConnection = ...) -> DuckDBPyConnection: ... -def register_filesystem(filesystem: fsspec.AbstractFileSystem, connection: DuckDBPyConnection = ...) -> None: ... -def rollback(connection: DuckDBPyConnection = ...) -> DuckDBPyConnection: ... - -def query(query: str, connection: DuckDBPyConnection = ..., **kwargs) -> DuckDBPyRelation: ... -def sql(query: str, connection: DuckDBPyConnection = ..., **kwargs) -> DuckDBPyRelation: ... -def from_query(query: str, connection: DuckDBPyConnection = ..., **kwargs) -> DuckDBPyRelation: ... -def extract_statements(self, query: str, connection: DuckDBPyConnection = ...) -> List[Statement]: ... - -def table(table_name: str, connection: DuckDBPyConnection = ...) -> DuckDBPyRelation: ... -def table_function(name: str, parameters: object = ..., connection: DuckDBPyConnection = ...) -> DuckDBPyRelation: ... -def unregister(view_name: str, connection: DuckDBPyConnection = ...) -> DuckDBPyConnection: ... -def query_df(df: pandas.DataFrame, virtual_table_name: str, sql_query: str, connection: DuckDBPyConnection = ...) -> DuckDBPyRelation: ... -def unregister_filesystem(name: str, connection: DuckDBPyConnection = ...) -> None: ... def tokenize(query: str) -> List[Any]: ... -def values(values: object, connection: DuckDBPyConnection = ...) -> DuckDBPyRelation: ... -def view(view_name: str, connection: DuckDBPyConnection = ...) -> DuckDBPyRelation: ... -def sqltype(type_str: str, connection: DuckDBPyConnection = ...) -> DuckDBPyType: ... -def dtype(type_str: str, connection: DuckDBPyConnection = ...) -> DuckDBPyType: ... -def type(type_str: str, connection: DuckDBPyConnection = ...) -> DuckDBPyType: ... -def struct_type(fields: Union[Dict[str, DuckDBPyType], List[str]], connection: DuckDBPyConnection = ...) -> DuckDBPyType: ... -def row_type(fields: Union[Dict[str, DuckDBPyType], List[str]], connection: DuckDBPyConnection = ...) -> DuckDBPyType: ... -def union_type(members: Union[Dict[str, DuckDBPyType], List[str]], connection: DuckDBPyConnection = ...) -> DuckDBPyType: ... -def string_type(collation: str = "", connection: DuckDBPyConnection = ...) -> DuckDBPyType: ... -def enum_type(name: str, type: DuckDBPyType, values: List[Any], connection: DuckDBPyConnection = ...) -> DuckDBPyType: ... -def decimal_type(width: int, scale: int, connection: DuckDBPyConnection = ...) -> DuckDBPyType: ... -def array_type(type: DuckDBPyType, size: int, connection: DuckDBPyConnection = ...) -> DuckDBPyType: ... -def list_type(type: DuckDBPyType, connection: DuckDBPyConnection = ...) -> DuckDBPyType: ... -def map_type(key: DuckDBPyType, value: DuckDBPyType, connection: DuckDBPyConnection = ...) -> DuckDBPyType: ... + +# NOTE: this section is generated by tools/pythonpkg/scripts/generate_connection_wrapper_stubs.py. +# Do not edit this section manually, your changes will be overwritten! + +# START OF CONNECTION WRAPPER +def cursor(**kwargs) -> DuckDBPyConnection: ... +def register_filesystem(filesystem: str, **kwargs) -> None: ... +def unregister_filesystem(name: str, **kwargs) -> None: ... +def list_filesystems(**kwargs) -> list: ... +def filesystem_is_registered(name: str, **kwargs) -> bool: ... +def create_function(name: str, function: function, parameters: Optional[List[DuckDBPyType]] = None, return_type: Optional[DuckDBPyType] = None, **kwargs) -> DuckDBPyConnection: ... +def remove_function(name: str, **kwargs) -> DuckDBPyConnection: ... +def sqltype(type_str: str, **kwargs) -> DuckDBPyType: ... +def dtype(type_str: str, **kwargs) -> DuckDBPyType: ... +def type(type_str: str, **kwargs) -> DuckDBPyType: ... +def array_type(type: DuckDBPyType, size: int, **kwargs) -> DuckDBPyType: ... +def list_type(type: DuckDBPyType, **kwargs) -> DuckDBPyType: ... +def union_type(members: DuckDBPyType, **kwargs) -> DuckDBPyType: ... +def string_type(collation: str = "", **kwargs) -> DuckDBPyType: ... +def enum_type(name: str, type: DuckDBPyType, values: List[Any], **kwargs) -> DuckDBPyType: ... +def decimal_type(width: int, scale: int, **kwargs) -> DuckDBPyType: ... +def struct_type(fields: Union[Dict[str, DuckDBPyType], List[str]], **kwargs) -> DuckDBPyType: ... +def row_type(fields: Union[Dict[str, DuckDBPyType], List[str]], **kwargs) -> DuckDBPyType: ... +def map_type(key: DuckDBPyType, value: DuckDBPyType, **kwargs) -> DuckDBPyType: ... +def duplicate(**kwargs) -> DuckDBPyConnection: ... +def execute(query: object, parameters: object = None, multiple_parameter_sets: bool = False, **kwargs) -> DuckDBPyConnection: ... +def executemany(query: object, parameters: object = None, **kwargs) -> DuckDBPyConnection: ... +def close(**kwargs) -> None: ... +def interrupt(**kwargs) -> None: ... +def fetchone(**kwargs) -> Optional[tuple]: ... +def fetchmany(size: int = 1, **kwargs) -> List[Any]: ... +def fetchall(**kwargs) -> List[Any]: ... +def fetchnumpy(**kwargs) -> dict: ... +def fetchdf(**kwargs) -> pandas.DataFrame: ... +def fetch_df(**kwargs) -> pandas.DataFrame: ... +def df(**kwargs) -> pandas.DataFrame: ... +def fetch_df_chunk(vectors_per_chunk: int = 1, **kwargs) -> pandas.DataFrame: ... +def pl(rows_per_batch: int = 1000000, **kwargs) -> polars.DataFrame: ... +def fetch_arrow_table(rows_per_batch: int = 1000000, **kwargs) -> pyarrow.lib.Table: ... +def arrow(rows_per_batch: int = 1000000, **kwargs) -> pyarrow.lib.Table: ... +def fetch_record_batch(rows_per_batch: int = 1000000, **kwargs) -> pyarrow.lib.RecordBatchReader: ... +def torch(**kwargs) -> dict: ... +def tf(**kwargs) -> dict: ... +def begin(**kwargs) -> DuckDBPyConnection: ... +def commit(**kwargs) -> DuckDBPyConnection: ... +def rollback(**kwargs) -> DuckDBPyConnection: ... +def append(table_name: str, df: pandas.DataFrame, **kwargs) -> DuckDBPyConnection: ... +def register(view_name: str, python_object: object, **kwargs) -> DuckDBPyConnection: ... +def unregister(view_name: str, **kwargs) -> DuckDBPyConnection: ... +def table(table_name: str, **kwargs) -> DuckDBPyRelation: ... +def view(view_name: str, **kwargs) -> DuckDBPyRelation: ... +def values(values: List[Any], **kwargs) -> DuckDBPyRelation: ... +def table_function(name: str, parameters: object = None, **kwargs) -> DuckDBPyRelation: ... +def read_json(name: str, **kwargs) -> DuckDBPyRelation: ... +def extract_statements(query: str, **kwargs) -> List[Statement]: ... +def sql(query: str, **kwargs) -> DuckDBPyRelation: ... +def query(query: str, **kwargs) -> DuckDBPyRelation: ... +def from_query(query: str, **kwargs) -> DuckDBPyRelation: ... +def read_csv(path_or_buffer: Union[str, StringIO, TextIOBase], **kwargs) -> DuckDBPyRelation: ... +def from_csv_auto(path_or_buffer: Union[str, StringIO, TextIOBase], **kwargs) -> DuckDBPyRelation: ... +def from_df(df: pandas.DataFrame, **kwargs) -> DuckDBPyRelation: ... +def from_arrow(arrow_object: object, **kwargs) -> DuckDBPyRelation: ... +def from_parquet(file_glob: str, binary_as_string: bool = False, **kwargs) -> DuckDBPyRelation: ... +def read_parquet(file_glob: str, binary_as_string: bool = False, **kwargs) -> DuckDBPyRelation: ... +def from_substrait(proto: str, **kwargs) -> DuckDBPyRelation: ... +def get_substrait(query: str, **kwargs) -> str: ... +def get_substrait_json(query: str, **kwargs) -> str: ... +def from_substrait_json(json: str, **kwargs) -> DuckDBPyRelation: ... +def get_table_names(query: str, **kwargs) -> List[str]: ... +def install_extension(extension: str, **kwargs) -> None: ... +def load_extension(extension: str, **kwargs) -> None: ... +def project(df: pandas.DataFrame, project_expr: str, **kwargs) -> DuckDBPyRelation: ... +def distinct(df: pandas.DataFrame, **kwargs) -> DuckDBPyRelation: ... +def write_csv(df: pandas.DataFrame, *args: Any, **kwargs) -> None: ... +def aggregate(df: pandas.DataFrame, aggr_expr: str, group_expr: str = "", **kwargs) -> DuckDBPyRelation: ... +def alias(df: pandas.DataFrame, alias: str, **kwargs) -> DuckDBPyRelation: ... +def filter(df: pandas.DataFrame, filter_expr: str, **kwargs) -> DuckDBPyRelation: ... +def limit(df: pandas.DataFrame, n: int, offset: int = 0, **kwargs) -> DuckDBPyRelation: ... +def order(df: pandas.DataFrame, order_expr: str, **kwargs) -> DuckDBPyRelation: ... +def query_df(df: pandas.DataFrame, virtual_table_name: str, sql_query: str, **kwargs) -> DuckDBPyRelation: ... +def description(**kwargs) -> Optional[List[Any]]: ... +def rowcount(**kwargs) -> int: ... +# END OF CONNECTION WRAPPER diff --git a/tools/pythonpkg/scripts/connection_wrapper_methods.json b/tools/pythonpkg/scripts/connection_wrapper_methods.json index 65ac4afd47f7..74f4cf31eba1 100644 --- a/tools/pythonpkg/scripts/connection_wrapper_methods.json +++ b/tools/pythonpkg/scripts/connection_wrapper_methods.json @@ -99,7 +99,7 @@ }, { "name": "description", - "return": "str" + "return": "Optional[List[Any]]" }, { "name": "rowcount", From 5ee33e0ca8cfa054ab20bf927999980eb7b85246 Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 10 Apr 2024 23:41:45 +0200 Subject: [PATCH 313/603] generation script --- .../generate_connection_wrapper_stubs.py | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 tools/pythonpkg/scripts/generate_connection_wrapper_stubs.py diff --git a/tools/pythonpkg/scripts/generate_connection_wrapper_stubs.py b/tools/pythonpkg/scripts/generate_connection_wrapper_stubs.py new file mode 100644 index 000000000000..f20b967318f3 --- /dev/null +++ b/tools/pythonpkg/scripts/generate_connection_wrapper_stubs.py @@ -0,0 +1,116 @@ +import os +import json + +os.chdir(os.path.dirname(__file__)) + +JSON_PATH = os.path.join("connection_methods.json") +WRAPPER_JSON_PATH = os.path.join("connection_wrapper_methods.json") +DUCKDB_STUBS_FILE = os.path.join("..", "duckdb-stubs", "__init__.pyi") + +START_MARKER = "# START OF CONNECTION WRAPPER" +END_MARKER = "# END OF CONNECTION WRAPPER" + +# Read the DUCKDB_STUBS_FILE file +with open(DUCKDB_STUBS_FILE, 'r') as source_file: + source_code = source_file.readlines() + +start_index = -1 +end_index = -1 +for i, line in enumerate(source_code): + if line.startswith(START_MARKER): + # TODO: handle the case where the start marker appears multiple times + start_index = i + elif line.startswith(END_MARKER): + # TODO: ditto ^ + end_index = i + +if start_index == -1 or end_index == -1: + raise ValueError("Couldn't find start or end marker in source file") + +start_section = source_code[: start_index + 1] +end_section = source_code[end_index:] +# ---- Generate the definition code from the json ---- + +methods = [] + +# Read the JSON file +with open(JSON_PATH, 'r') as json_file: + connection_methods = json.load(json_file) + +with open(WRAPPER_JSON_PATH, 'r') as json_file: + wrapper_methods = json.load(json_file) + +methods.extend(connection_methods) +methods.extend(wrapper_methods) + +# On DuckDBPyConnection these are read_only_properties, they're basically functions without requiring () to invoke +# that's not possible on 'duckdb' so it becomes a function call with no arguments (i.e duckdb.description()) +READONLY_PROPERTY_NAMES = ['description', 'rowcount'] + +# These methods are not directly DuckDBPyConnection methods, +# they first call 'from_df' and then call a method on the created DuckDBPyRelation +SPECIAL_METHOD_NAMES = [x['name'] for x in wrapper_methods if x['name'] not in READONLY_PROPERTY_NAMES] + + +def create_arguments(arguments) -> list: + result = [] + for arg in arguments: + argument = f"{arg['name']}: {arg['type']}" + # Add the default argument if present + if 'default' in arg: + default = arg['default'] + argument += f" = {default}" + result.append(argument) + return result + + +def create_definition(name, method) -> str: + print(method) + definition = f"def {name}(" + arguments = [] + if name in SPECIAL_METHOD_NAMES: + arguments.append('df: pandas.DataFrame') + if 'args' in method: + arguments.extend(create_arguments(method['args'])) + if 'kwargs' in method: + arguments.append("**kwargs") + definition += ', '.join(arguments) + definition += ")" + definition += f" -> {method['return']}: ..." + return definition + + +# We have "duplicate" methods, which are overloaded +# maybe we should add @overload to these instead, but this is easier +written_methods = set() + +body = [] +for method in methods: + if isinstance(method['name'], list): + names = method['name'] + else: + names = [method['name']] + + # Artificially add 'connection' keyword argument + if 'kwargs' not in method: + method['kwargs'] = [] + method['kwargs'].append({'name': 'connection', 'type': 'DuckDBPyConnection'}) + + for name in names: + if name in written_methods: + continue + body.append(create_definition(name, method)) + written_methods.add(name) + +# ---- End of generation code ---- + +with_newlines = [x + '\n' for x in body] +# Recreate the file content by concatenating all the pieces together + +new_content = start_section + with_newlines + end_section + +print(''.join(with_newlines)) + +# Write out the modified DUCKDB_STUBS_FILE file +with open(DUCKDB_STUBS_FILE, 'w') as source_file: + source_file.write("".join(new_content)) From 1bbda5b844b04218339d64bb958d8182b84b7b30 Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 11 Apr 2024 00:10:26 +0200 Subject: [PATCH 314/603] format --- tools/pythonpkg/src/pyconnection.cpp | 191 ++++++++++++++++++++------- 1 file changed, 142 insertions(+), 49 deletions(-) diff --git a/tools/pythonpkg/src/pyconnection.cpp b/tools/pythonpkg/src/pyconnection.cpp index 3e3eab8127c4..937cb6f3edb4 100644 --- a/tools/pythonpkg/src/pyconnection.cpp +++ b/tools/pythonpkg/src/pyconnection.cpp @@ -127,72 +127,165 @@ py::object ArrowTableFromDataframe(const py::object &df) { static void InitializeConnectionMethods(py::class_> &m) { m.def("cursor", &DuckDBPyConnection::Cursor, "Create a duplicate of the current connection"); - m.def("register_filesystem", &DuckDBPyConnection::RegisterFilesystem, "Register a fsspec compliant filesystem", py::arg("filesystem")); - m.def("unregister_filesystem", &DuckDBPyConnection::UnregisterFilesystem, "Unregister a filesystem", py::arg("name")); - m.def("list_filesystems", &DuckDBPyConnection::ListFilesystems, "List registered filesystems, including builtin ones"); - m.def("filesystem_is_registered", &DuckDBPyConnection::FileSystemIsRegistered, "Check if a filesystem with the provided name is currently registered", py::arg("name")); - m.def("create_function", &DuckDBPyConnection::RegisterScalarUDF, "Create a DuckDB function out of the passing in Python function so it can be used in queries", py::arg("name"), py::arg("function"), py::arg("parameters") = py::none(), py::arg("return_type") = py::none(), py::kw_only(), py::arg("type") = PythonUDFType::NATIVE, py::arg("null_handling") = FunctionNullHandling::DEFAULT_NULL_HANDLING, py::arg("exception_handling") = PythonExceptionHandling::FORWARD_ERROR, py::arg("side_effects") = false); - m.def("remove_function", &DuckDBPyConnection::UnregisterUDF, "Remove a previously created function", py::arg("name")); - m.def("sqltype", &DuckDBPyConnection::Type, "Create a type object by parsing the 'type_str' string", py::arg("type_str")); - m.def("dtype", &DuckDBPyConnection::Type, "Create a type object by parsing the 'type_str' string", py::arg("type_str")); - m.def("type", &DuckDBPyConnection::Type, "Create a type object by parsing the 'type_str' string", py::arg("type_str")); - m.def("array_type", &DuckDBPyConnection::ArrayType, "Create an array type object of 'type'", py::arg("type").none(false), py::arg("size")); - m.def("list_type", &DuckDBPyConnection::ListType, "Create a list type object of 'type'", py::arg("type").none(false)); - m.def("union_type", &DuckDBPyConnection::UnionType, "Create a union type object from 'members'", py::arg("members").none(false)); - m.def("string_type", &DuckDBPyConnection::StringType, "Create a string type with an optional collation", py::arg("collation") = ""); - m.def("enum_type", &DuckDBPyConnection::EnumType, "Create an enum type of underlying 'type', consisting of the list of 'values'", py::arg("name"), py::arg("type"), py::arg("values")); - m.def("decimal_type", &DuckDBPyConnection::DecimalType, "Create a decimal type with 'width' and 'scale'", py::arg("width"), py::arg("scale")); - m.def("struct_type", &DuckDBPyConnection::StructType, "Create a struct type object from 'fields'", py::arg("fields")); + m.def("register_filesystem", &DuckDBPyConnection::RegisterFilesystem, "Register a fsspec compliant filesystem", + py::arg("filesystem")); + m.def("unregister_filesystem", &DuckDBPyConnection::UnregisterFilesystem, "Unregister a filesystem", + py::arg("name")); + m.def("list_filesystems", &DuckDBPyConnection::ListFilesystems, + "List registered filesystems, including builtin ones"); + m.def("filesystem_is_registered", &DuckDBPyConnection::FileSystemIsRegistered, + "Check if a filesystem with the provided name is currently registered", py::arg("name")); + m.def("create_function", &DuckDBPyConnection::RegisterScalarUDF, + "Create a DuckDB function out of the passing in Python function so it can be used in queries", + py::arg("name"), py::arg("function"), py::arg("parameters") = py::none(), py::arg("return_type") = py::none(), + py::kw_only(), py::arg("type") = PythonUDFType::NATIVE, + py::arg("null_handling") = FunctionNullHandling::DEFAULT_NULL_HANDLING, + py::arg("exception_handling") = PythonExceptionHandling::FORWARD_ERROR, py::arg("side_effects") = false); + m.def("remove_function", &DuckDBPyConnection::UnregisterUDF, "Remove a previously created function", + py::arg("name")); + m.def("sqltype", &DuckDBPyConnection::Type, "Create a type object by parsing the 'type_str' string", + py::arg("type_str")); + m.def("dtype", &DuckDBPyConnection::Type, "Create a type object by parsing the 'type_str' string", + py::arg("type_str")); + m.def("type", &DuckDBPyConnection::Type, "Create a type object by parsing the 'type_str' string", + py::arg("type_str")); + m.def("array_type", &DuckDBPyConnection::ArrayType, "Create an array type object of 'type'", + py::arg("type").none(false), py::arg("size")); + m.def("list_type", &DuckDBPyConnection::ListType, "Create a list type object of 'type'", + py::arg("type").none(false)); + m.def("union_type", &DuckDBPyConnection::UnionType, "Create a union type object from 'members'", + py::arg("members").none(false)); + m.def("string_type", &DuckDBPyConnection::StringType, "Create a string type with an optional collation", + py::arg("collation") = ""); + m.def("enum_type", &DuckDBPyConnection::EnumType, + "Create an enum type of underlying 'type', consisting of the list of 'values'", py::arg("name"), + py::arg("type"), py::arg("values")); + m.def("decimal_type", &DuckDBPyConnection::DecimalType, "Create a decimal type with 'width' and 'scale'", + py::arg("width"), py::arg("scale")); + m.def("struct_type", &DuckDBPyConnection::StructType, "Create a struct type object from 'fields'", + py::arg("fields")); m.def("row_type", &DuckDBPyConnection::StructType, "Create a struct type object from 'fields'", py::arg("fields")); - m.def("map_type", &DuckDBPyConnection::MapType, "Create a map type object from 'key_type' and 'value_type'", py::arg("key").none(false), py::arg("value").none(false)); + m.def("map_type", &DuckDBPyConnection::MapType, "Create a map type object from 'key_type' and 'value_type'", + py::arg("key").none(false), py::arg("value").none(false)); m.def("duplicate", &DuckDBPyConnection::Cursor, "Create a duplicate of the current connection"); - m.def("execute", &DuckDBPyConnection::Execute, "Execute the given SQL query, optionally using prepared statements with parameters set", py::arg("query"), py::arg("parameters") = py::none(), py::arg("multiple_parameter_sets") = false); - m.def("executemany", &DuckDBPyConnection::ExecuteMany, "Execute the given prepared statement multiple times using the list of parameter sets in parameters", py::arg("query"), py::arg("parameters") = py::none()); + m.def("execute", &DuckDBPyConnection::Execute, + "Execute the given SQL query, optionally using prepared statements with parameters set", py::arg("query"), + py::arg("parameters") = py::none(), py::arg("multiple_parameter_sets") = false); + m.def("executemany", &DuckDBPyConnection::ExecuteMany, + "Execute the given prepared statement multiple times using the list of parameter sets in parameters", + py::arg("query"), py::arg("parameters") = py::none()); m.def("close", &DuckDBPyConnection::Close, "Close the connection"); m.def("interrupt", &DuckDBPyConnection::Interrupt, "Interrupt pending operations"); m.def("fetchone", &DuckDBPyConnection::FetchOne, "Fetch a single row from a result following execute"); - m.def("fetchmany", &DuckDBPyConnection::FetchMany, "Fetch the next set of rows from a result following execute", py::arg("size") = 1); + m.def("fetchmany", &DuckDBPyConnection::FetchMany, "Fetch the next set of rows from a result following execute", + py::arg("size") = 1); m.def("fetchall", &DuckDBPyConnection::FetchAll, "Fetch all rows from a result following execute"); m.def("fetchnumpy", &DuckDBPyConnection::FetchNumpy, "Fetch a result as list of NumPy arrays following execute"); - m.def("fetchdf", &DuckDBPyConnection::FetchDF, "Fetch a result as DataFrame following execute()", py::kw_only(), py::arg("date_as_object") = false); - m.def("fetch_df", &DuckDBPyConnection::FetchDF, "Fetch a result as DataFrame following execute()", py::kw_only(), py::arg("date_as_object") = false); - m.def("df", &DuckDBPyConnection::FetchDF, "Fetch a result as DataFrame following execute()", py::kw_only(), py::arg("date_as_object") = false); - m.def("fetch_df_chunk", &DuckDBPyConnection::FetchDFChunk, "Fetch a chunk of the result as DataFrame following execute()", py::arg("vectors_per_chunk") = 1, py::kw_only(), py::arg("date_as_object") = false); - m.def("pl", &DuckDBPyConnection::FetchPolars, "Fetch a result as Polars DataFrame following execute()", py::arg("rows_per_batch") = 1000000); - m.def("fetch_arrow_table", &DuckDBPyConnection::FetchArrow, "Fetch a result as Arrow table following execute()", py::arg("rows_per_batch") = 1000000); - m.def("arrow", &DuckDBPyConnection::FetchArrow, "Fetch a result as Arrow table following execute()", py::arg("rows_per_batch") = 1000000); - m.def("fetch_record_batch", &DuckDBPyConnection::FetchRecordBatchReader, "Fetch an Arrow RecordBatchReader following execute()", py::arg("rows_per_batch") = 1000000); + m.def("fetchdf", &DuckDBPyConnection::FetchDF, "Fetch a result as DataFrame following execute()", py::kw_only(), + py::arg("date_as_object") = false); + m.def("fetch_df", &DuckDBPyConnection::FetchDF, "Fetch a result as DataFrame following execute()", py::kw_only(), + py::arg("date_as_object") = false); + m.def("df", &DuckDBPyConnection::FetchDF, "Fetch a result as DataFrame following execute()", py::kw_only(), + py::arg("date_as_object") = false); + m.def("fetch_df_chunk", &DuckDBPyConnection::FetchDFChunk, + "Fetch a chunk of the result as DataFrame following execute()", py::arg("vectors_per_chunk") = 1, + py::kw_only(), py::arg("date_as_object") = false); + m.def("pl", &DuckDBPyConnection::FetchPolars, "Fetch a result as Polars DataFrame following execute()", + py::arg("rows_per_batch") = 1000000); + m.def("fetch_arrow_table", &DuckDBPyConnection::FetchArrow, "Fetch a result as Arrow table following execute()", + py::arg("rows_per_batch") = 1000000); + m.def("arrow", &DuckDBPyConnection::FetchArrow, "Fetch a result as Arrow table following execute()", + py::arg("rows_per_batch") = 1000000); + m.def("fetch_record_batch", &DuckDBPyConnection::FetchRecordBatchReader, + "Fetch an Arrow RecordBatchReader following execute()", py::arg("rows_per_batch") = 1000000); m.def("torch", &DuckDBPyConnection::FetchPyTorch, "Fetch a result as dict of PyTorch Tensors following execute()"); m.def("tf", &DuckDBPyConnection::FetchTF, "Fetch a result as dict of TensorFlow Tensors following execute()"); m.def("begin", &DuckDBPyConnection::Begin, "Start a new transaction"); m.def("commit", &DuckDBPyConnection::Commit, "Commit changes performed within a transaction"); m.def("rollback", &DuckDBPyConnection::Rollback, "Roll back changes performed within a transaction"); - m.def("append", &DuckDBPyConnection::Append, "Append the passed DataFrame to the named table", py::arg("table_name"), py::arg("df"), py::kw_only(), py::arg("by_name") = false); - m.def("register", &DuckDBPyConnection::RegisterPythonObject, "Register the passed Python Object value for querying with a view", py::arg("view_name"), py::arg("python_object")); + m.def("append", &DuckDBPyConnection::Append, "Append the passed DataFrame to the named table", + py::arg("table_name"), py::arg("df"), py::kw_only(), py::arg("by_name") = false); + m.def("register", &DuckDBPyConnection::RegisterPythonObject, + "Register the passed Python Object value for querying with a view", py::arg("view_name"), + py::arg("python_object")); m.def("unregister", &DuckDBPyConnection::UnregisterPythonObject, "Unregister the view name", py::arg("view_name")); m.def("table", &DuckDBPyConnection::Table, "Create a relation object for the named table", py::arg("table_name")); m.def("view", &DuckDBPyConnection::View, "Create a relation object for the named view", py::arg("view_name")); m.def("values", &DuckDBPyConnection::Values, "Create a relation object from the passed values", py::arg("values")); - m.def("table_function", &DuckDBPyConnection::TableFunction, "Create a relation object from the named table function with given parameters", py::arg("name"), py::arg("parameters") = py::none()); - m.def("read_json", &DuckDBPyConnection::ReadJSON, "Create a relation object from the JSON file in 'name'", py::arg("name"), py::kw_only(), py::arg("columns") = py::none(), py::arg("sample_size") = py::none(), py::arg("maximum_depth") = py::none(), py::arg("records") = py::none(), py::arg("format") = py::none()); - m.def("extract_statements", &DuckDBPyConnection::ExtractStatements, "Parse the query string and extract the Statement object(s) produced", py::arg("query")); - m.def("sql", &DuckDBPyConnection::RunQuery, "Run a SQL query. If it is a SELECT statement, create a relation object from the given SQL query, otherwise run the query as-is.", py::arg("query"), py::kw_only(), py::arg("alias") = "", py::arg("params") = py::none()); - m.def("query", &DuckDBPyConnection::RunQuery, "Run a SQL query. If it is a SELECT statement, create a relation object from the given SQL query, otherwise run the query as-is.", py::arg("query"), py::kw_only(), py::arg("alias") = "", py::arg("params") = py::none()); - m.def("from_query", &DuckDBPyConnection::RunQuery, "Run a SQL query. If it is a SELECT statement, create a relation object from the given SQL query, otherwise run the query as-is.", py::arg("query"), py::kw_only(), py::arg("alias") = "", py::arg("params") = py::none()); - m.def("read_csv", &DuckDBPyConnection::ReadCSV, "Create a relation object from the CSV file in 'name'", py::arg("path_or_buffer"), py::kw_only(), py::arg("header") = py::none(), py::arg("compression") = py::none(), py::arg("sep") = py::none(), py::arg("delimiter") = py::none(), py::arg("dtype") = py::none(), py::arg("na_values") = py::none(), py::arg("skiprows") = py::none(), py::arg("quotechar") = py::none(), py::arg("escapechar") = py::none(), py::arg("encoding") = py::none(), py::arg("parallel") = py::none(), py::arg("date_format") = py::none(), py::arg("timestamp_format") = py::none(), py::arg("sample_size") = py::none(), py::arg("all_varchar") = py::none(), py::arg("normalize_names") = py::none(), py::arg("filename") = py::none(), py::arg("null_padding") = py::none(), py::arg("names") = py::none()); - m.def("from_csv_auto", &DuckDBPyConnection::ReadCSV, "Create a relation object from the CSV file in 'name'", py::arg("path_or_buffer"), py::kw_only(), py::arg("header") = py::none(), py::arg("compression") = py::none(), py::arg("sep") = py::none(), py::arg("delimiter") = py::none(), py::arg("dtype") = py::none(), py::arg("na_values") = py::none(), py::arg("skiprows") = py::none(), py::arg("quotechar") = py::none(), py::arg("escapechar") = py::none(), py::arg("encoding") = py::none(), py::arg("parallel") = py::none(), py::arg("date_format") = py::none(), py::arg("timestamp_format") = py::none(), py::arg("sample_size") = py::none(), py::arg("all_varchar") = py::none(), py::arg("normalize_names") = py::none(), py::arg("filename") = py::none(), py::arg("null_padding") = py::none(), py::arg("names") = py::none()); + m.def("table_function", &DuckDBPyConnection::TableFunction, + "Create a relation object from the named table function with given parameters", py::arg("name"), + py::arg("parameters") = py::none()); + m.def("read_json", &DuckDBPyConnection::ReadJSON, "Create a relation object from the JSON file in 'name'", + py::arg("name"), py::kw_only(), py::arg("columns") = py::none(), py::arg("sample_size") = py::none(), + py::arg("maximum_depth") = py::none(), py::arg("records") = py::none(), py::arg("format") = py::none()); + m.def("extract_statements", &DuckDBPyConnection::ExtractStatements, + "Parse the query string and extract the Statement object(s) produced", py::arg("query")); + m.def("sql", &DuckDBPyConnection::RunQuery, + "Run a SQL query. If it is a SELECT statement, create a relation object from the given SQL query, otherwise " + "run the query as-is.", + py::arg("query"), py::kw_only(), py::arg("alias") = "", py::arg("params") = py::none()); + m.def("query", &DuckDBPyConnection::RunQuery, + "Run a SQL query. If it is a SELECT statement, create a relation object from the given SQL query, otherwise " + "run the query as-is.", + py::arg("query"), py::kw_only(), py::arg("alias") = "", py::arg("params") = py::none()); + m.def("from_query", &DuckDBPyConnection::RunQuery, + "Run a SQL query. If it is a SELECT statement, create a relation object from the given SQL query, otherwise " + "run the query as-is.", + py::arg("query"), py::kw_only(), py::arg("alias") = "", py::arg("params") = py::none()); + m.def("read_csv", &DuckDBPyConnection::ReadCSV, "Create a relation object from the CSV file in 'name'", + py::arg("path_or_buffer"), py::kw_only(), py::arg("header") = py::none(), py::arg("compression") = py::none(), + py::arg("sep") = py::none(), py::arg("delimiter") = py::none(), py::arg("dtype") = py::none(), + py::arg("na_values") = py::none(), py::arg("skiprows") = py::none(), py::arg("quotechar") = py::none(), + py::arg("escapechar") = py::none(), py::arg("encoding") = py::none(), py::arg("parallel") = py::none(), + py::arg("date_format") = py::none(), py::arg("timestamp_format") = py::none(), + py::arg("sample_size") = py::none(), py::arg("all_varchar") = py::none(), + py::arg("normalize_names") = py::none(), py::arg("filename") = py::none(), + py::arg("null_padding") = py::none(), py::arg("names") = py::none()); + m.def("from_csv_auto", &DuckDBPyConnection::ReadCSV, "Create a relation object from the CSV file in 'name'", + py::arg("path_or_buffer"), py::kw_only(), py::arg("header") = py::none(), py::arg("compression") = py::none(), + py::arg("sep") = py::none(), py::arg("delimiter") = py::none(), py::arg("dtype") = py::none(), + py::arg("na_values") = py::none(), py::arg("skiprows") = py::none(), py::arg("quotechar") = py::none(), + py::arg("escapechar") = py::none(), py::arg("encoding") = py::none(), py::arg("parallel") = py::none(), + py::arg("date_format") = py::none(), py::arg("timestamp_format") = py::none(), + py::arg("sample_size") = py::none(), py::arg("all_varchar") = py::none(), + py::arg("normalize_names") = py::none(), py::arg("filename") = py::none(), + py::arg("null_padding") = py::none(), py::arg("names") = py::none()); m.def("from_df", &DuckDBPyConnection::FromDF, "Create a relation object from the DataFrame in df", py::arg("df")); - m.def("from_arrow", &DuckDBPyConnection::FromArrow, "Create a relation object from an Arrow object", py::arg("arrow_object")); - m.def("from_parquet", &DuckDBPyConnection::FromParquet, "Create a relation object from the Parquet files in file_glob", py::arg("file_glob"), py::arg("binary_as_string") = false, py::kw_only(), py::arg("file_row_number") = false, py::arg("filename") = false, py::arg("hive_partitioning") = false, py::arg("union_by_name") = false, py::arg("compression") = py::none()); - m.def("read_parquet", &DuckDBPyConnection::FromParquet, "Create a relation object from the Parquet files in file_glob", py::arg("file_glob"), py::arg("binary_as_string") = false, py::kw_only(), py::arg("file_row_number") = false, py::arg("filename") = false, py::arg("hive_partitioning") = false, py::arg("union_by_name") = false, py::arg("compression") = py::none()); - m.def("from_parquet", &DuckDBPyConnection::FromParquets, "Create a relation object from the Parquet files in file_globs", py::arg("file_globs"), py::arg("binary_as_string") = false, py::kw_only(), py::arg("file_row_number") = false, py::arg("filename") = false, py::arg("hive_partitioning") = false, py::arg("union_by_name") = false, py::arg("compression") = py::none()); - m.def("read_parquet", &DuckDBPyConnection::FromParquets, "Create a relation object from the Parquet files in file_globs", py::arg("file_globs"), py::arg("binary_as_string") = false, py::kw_only(), py::arg("file_row_number") = false, py::arg("filename") = false, py::arg("hive_partitioning") = false, py::arg("union_by_name") = false, py::arg("compression") = py::none()); - m.def("from_substrait", &DuckDBPyConnection::FromSubstrait, "Create a query object from protobuf plan", py::arg("proto")); - m.def("get_substrait", &DuckDBPyConnection::GetSubstrait, "Serialize a query to protobuf", py::arg("query"), py::kw_only(), py::arg("enable_optimizer") = true); - m.def("get_substrait_json", &DuckDBPyConnection::GetSubstraitJSON, "Serialize a query to protobuf on the JSON format", py::arg("query"), py::kw_only(), py::arg("enable_optimizer") = true); - m.def("from_substrait_json", &DuckDBPyConnection::FromSubstraitJSON, "Create a query object from a JSON protobuf plan", py::arg("json")); - m.def("get_table_names", &DuckDBPyConnection::GetTableNames, "Extract the required table names from a query", py::arg("query")); - m.def("install_extension", &DuckDBPyConnection::InstallExtension, "Install an extension by name", py::arg("extension"), py::kw_only(), py::arg("force_install") = false); + m.def("from_arrow", &DuckDBPyConnection::FromArrow, "Create a relation object from an Arrow object", + py::arg("arrow_object")); + m.def("from_parquet", &DuckDBPyConnection::FromParquet, + "Create a relation object from the Parquet files in file_glob", py::arg("file_glob"), + py::arg("binary_as_string") = false, py::kw_only(), py::arg("file_row_number") = false, + py::arg("filename") = false, py::arg("hive_partitioning") = false, py::arg("union_by_name") = false, + py::arg("compression") = py::none()); + m.def("read_parquet", &DuckDBPyConnection::FromParquet, + "Create a relation object from the Parquet files in file_glob", py::arg("file_glob"), + py::arg("binary_as_string") = false, py::kw_only(), py::arg("file_row_number") = false, + py::arg("filename") = false, py::arg("hive_partitioning") = false, py::arg("union_by_name") = false, + py::arg("compression") = py::none()); + m.def("from_parquet", &DuckDBPyConnection::FromParquets, + "Create a relation object from the Parquet files in file_globs", py::arg("file_globs"), + py::arg("binary_as_string") = false, py::kw_only(), py::arg("file_row_number") = false, + py::arg("filename") = false, py::arg("hive_partitioning") = false, py::arg("union_by_name") = false, + py::arg("compression") = py::none()); + m.def("read_parquet", &DuckDBPyConnection::FromParquets, + "Create a relation object from the Parquet files in file_globs", py::arg("file_globs"), + py::arg("binary_as_string") = false, py::kw_only(), py::arg("file_row_number") = false, + py::arg("filename") = false, py::arg("hive_partitioning") = false, py::arg("union_by_name") = false, + py::arg("compression") = py::none()); + m.def("from_substrait", &DuckDBPyConnection::FromSubstrait, "Create a query object from protobuf plan", + py::arg("proto")); + m.def("get_substrait", &DuckDBPyConnection::GetSubstrait, "Serialize a query to protobuf", py::arg("query"), + py::kw_only(), py::arg("enable_optimizer") = true); + m.def("get_substrait_json", &DuckDBPyConnection::GetSubstraitJSON, + "Serialize a query to protobuf on the JSON format", py::arg("query"), py::kw_only(), + py::arg("enable_optimizer") = true); + m.def("from_substrait_json", &DuckDBPyConnection::FromSubstraitJSON, + "Create a query object from a JSON protobuf plan", py::arg("json")); + m.def("get_table_names", &DuckDBPyConnection::GetTableNames, "Extract the required table names from a query", + py::arg("query")); + m.def("install_extension", &DuckDBPyConnection::InstallExtension, "Install an extension by name", + py::arg("extension"), py::kw_only(), py::arg("force_install") = false); m.def("load_extension", &DuckDBPyConnection::LoadExtension, "Load an installed extension", py::arg("extension")); } // END_OF_CONNECTION_METHODS From eb0ad4b8436e0074f3a54987e7e03b5ca4d9fa31 Mon Sep 17 00:00:00 2001 From: Guen Prawiroatmodjo Date: Wed, 10 Apr 2024 18:28:51 -0700 Subject: [PATCH 315/603] SQL_TYPE_TIMESTAMP should use SQL_TYPE_TIMESTAMP instead of SQL_DATETIME --- tools/odbc/src/api_info.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/odbc/src/api_info.cpp b/tools/odbc/src/api_info.cpp index 309479dfa229..3f2918caddd4 100644 --- a/tools/odbc/src/api_info.cpp +++ b/tools/odbc/src/api_info.cpp @@ -98,7 +98,7 @@ const vector ApiInfo::ODBC_SUPPORTED_SQL_TYPES = { { "'BIGINT'", SQL_BIGINT, 19, "NULL", "NULL", "NULL", SQL_NULLABLE, SQL_FALSE, SQL_PRED_BASIC, SQL_FALSE, SQL_FALSE, SQL_FALSE, "NULL", 0, 0, SQL_BIGINT, -1, 2, -1}, { "'DATE'", SQL_TYPE_DATE, 10, "''''", "''''", "NULL", SQL_NULLABLE, SQL_FALSE, SQL_PRED_BASIC, -1, SQL_FALSE, SQL_FALSE, "NULL", -1, -1, SQL_DATETIME, SQL_CODE_DATE, -1, -1}, { "'TIME'", SQL_TYPE_TIME, 8, "''''", "''''", "NULL", SQL_NULLABLE, SQL_FALSE, SQL_PRED_BASIC, -1, SQL_FALSE, SQL_FALSE, "NULL", 0, 0, SQL_DATETIME, SQL_CODE_TIME, -1, -1}, -{ "'TIMESTAMP'", SQL_TYPE_TIMESTAMP, 26, "''''", "''''", "NULL", SQL_NULLABLE, SQL_FALSE, SQL_PRED_BASIC, -1, SQL_FALSE, SQL_FALSE, "NULL", 0, 0, SQL_DATETIME, SQL_CODE_TIMESTAMP, -1, -1}, +{ "'TIMESTAMP'", SQL_TYPE_TIMESTAMP, 26, "''''", "''''", "NULL", SQL_NULLABLE, SQL_FALSE, SQL_PRED_BASIC, -1, SQL_FALSE, SQL_FALSE, "NULL", 0, 0, SQL_TYPE_TIMESTAMP, SQL_CODE_TIMESTAMP, -1, -1}, { "'DECIMAL'", SQL_DECIMAL, 38, "''''", "''''", "'precision,scale'", SQL_NULLABLE, SQL_FALSE, SQL_PRED_BASIC, -1, SQL_FALSE, SQL_FALSE, "NULL", 0, 38, SQL_DECIMAL, -1, 10, -1}, { "'NUMERIC'", SQL_NUMERIC, 38, "''''", "''''", "'precision,scale'", SQL_NULLABLE, SQL_FALSE, SQL_PRED_BASIC, -1, SQL_FALSE, SQL_FALSE, "NULL", 0, 38, SQL_NUMERIC, -1, 10, -1}, { "'FLOAT'", SQL_FLOAT, 24, "NULL", "NULL", "NULL", SQL_NULLABLE, SQL_FALSE, SQL_PRED_BASIC, SQL_FALSE, SQL_FALSE, SQL_FALSE, "NULL", 0, 0, SQL_FLOAT, -1, 2, -1}, From 16d451f312b45da66c40e605f1c6e421bcb39ca7 Mon Sep 17 00:00:00 2001 From: Guen Prawiroatmodjo Date: Wed, 10 Apr 2024 19:21:04 -0700 Subject: [PATCH 316/603] SQL_TYPE_DATE and TIME should use SQL_TYPE_DATE and TIME instead of SQL_DATETIME --- tools/odbc/src/api_info.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/odbc/src/api_info.cpp b/tools/odbc/src/api_info.cpp index 3f2918caddd4..2753b7928eaf 100644 --- a/tools/odbc/src/api_info.cpp +++ b/tools/odbc/src/api_info.cpp @@ -96,8 +96,8 @@ const vector ApiInfo::ODBC_SUPPORTED_SQL_TYPES = { { "'SMALLINT'", SQL_SMALLINT, 5, "NULL", "NULL", "NULL", SQL_NULLABLE, SQL_FALSE, SQL_PRED_BASIC, SQL_FALSE, SQL_FALSE, SQL_FALSE, "NULL", 0, 0, SQL_SMALLINT, -1, 2, -1}, { "'INTEGER'", SQL_INTEGER, 10, "NULL", "NULL", "NULL", SQL_NULLABLE, SQL_FALSE, SQL_PRED_BASIC, SQL_FALSE, SQL_FALSE, SQL_FALSE, "NULL", 0, 0, SQL_INTEGER, -1, 2, -1}, { "'BIGINT'", SQL_BIGINT, 19, "NULL", "NULL", "NULL", SQL_NULLABLE, SQL_FALSE, SQL_PRED_BASIC, SQL_FALSE, SQL_FALSE, SQL_FALSE, "NULL", 0, 0, SQL_BIGINT, -1, 2, -1}, -{ "'DATE'", SQL_TYPE_DATE, 10, "''''", "''''", "NULL", SQL_NULLABLE, SQL_FALSE, SQL_PRED_BASIC, -1, SQL_FALSE, SQL_FALSE, "NULL", -1, -1, SQL_DATETIME, SQL_CODE_DATE, -1, -1}, -{ "'TIME'", SQL_TYPE_TIME, 8, "''''", "''''", "NULL", SQL_NULLABLE, SQL_FALSE, SQL_PRED_BASIC, -1, SQL_FALSE, SQL_FALSE, "NULL", 0, 0, SQL_DATETIME, SQL_CODE_TIME, -1, -1}, +{ "'DATE'", SQL_TYPE_DATE, 10, "''''", "''''", "NULL", SQL_NULLABLE, SQL_FALSE, SQL_PRED_BASIC, -1, SQL_FALSE, SQL_FALSE, "NULL", -1, -1, SQL_TYPE_DATE, SQL_CODE_DATE, -1, -1}, +{ "'TIME'", SQL_TYPE_TIME, 8, "''''", "''''", "NULL", SQL_NULLABLE, SQL_FALSE, SQL_PRED_BASIC, -1, SQL_FALSE, SQL_FALSE, "NULL", 0, 0, SQL_TYPE_TIME, SQL_CODE_TIME, -1, -1}, { "'TIMESTAMP'", SQL_TYPE_TIMESTAMP, 26, "''''", "''''", "NULL", SQL_NULLABLE, SQL_FALSE, SQL_PRED_BASIC, -1, SQL_FALSE, SQL_FALSE, "NULL", 0, 0, SQL_TYPE_TIMESTAMP, SQL_CODE_TIMESTAMP, -1, -1}, { "'DECIMAL'", SQL_DECIMAL, 38, "''''", "''''", "'precision,scale'", SQL_NULLABLE, SQL_FALSE, SQL_PRED_BASIC, -1, SQL_FALSE, SQL_FALSE, "NULL", 0, 38, SQL_DECIMAL, -1, 10, -1}, { "'NUMERIC'", SQL_NUMERIC, 38, "''''", "''''", "'precision,scale'", SQL_NULLABLE, SQL_FALSE, SQL_PRED_BASIC, -1, SQL_FALSE, SQL_FALSE, "NULL", 0, 38, SQL_NUMERIC, -1, 10, -1}, From 6137227e900889c16261bd8b630de2dd026c86af Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 11 Apr 2024 10:01:05 +0200 Subject: [PATCH 317/603] the one script to rule them all --- .../scripts/generate_connection_code.py | 10 + .../scripts/generate_connection_methods.py | 189 ++++++------ .../scripts/generate_connection_stubs.py | 167 +++++------ .../generate_connection_wrapper_methods.py | 278 +++++++++--------- .../generate_connection_wrapper_stubs.py | 212 ++++++------- 5 files changed, 440 insertions(+), 416 deletions(-) create mode 100644 tools/pythonpkg/scripts/generate_connection_code.py diff --git a/tools/pythonpkg/scripts/generate_connection_code.py b/tools/pythonpkg/scripts/generate_connection_code.py new file mode 100644 index 000000000000..3737f83ad319 --- /dev/null +++ b/tools/pythonpkg/scripts/generate_connection_code.py @@ -0,0 +1,10 @@ +import generate_connection_methods +import generate_connection_stubs +import generate_connection_wrapper_methods +import generate_connection_wrapper_stubs + +if __name__ == '__main__': + generate_connection_methods.generate() + generate_connection_stubs.generate() + generate_connection_wrapper_methods.generate() + generate_connection_wrapper_stubs.generate() diff --git a/tools/pythonpkg/scripts/generate_connection_methods.py b/tools/pythonpkg/scripts/generate_connection_methods.py index efcf13304f0d..10aacd1418e3 100644 --- a/tools/pythonpkg/scripts/generate_connection_methods.py +++ b/tools/pythonpkg/scripts/generate_connection_methods.py @@ -11,97 +11,102 @@ ) END_MARKER = "} // END_OF_CONNECTION_METHODS" -# Read the PYCONNECTION_SOURCE file -with open(PYCONNECTION_SOURCE, 'r') as source_file: - source_code = source_file.readlines() - -# Locate the InitializeConnectionMethods function in it -start_index = -1 -end_index = -1 -for i, line in enumerate(source_code): - if line.startswith(INITIALIZE_METHOD): - start_index = i - elif line.startswith(END_MARKER): - end_index = i - -if start_index == -1 or end_index == -1: - raise ValueError("Couldn't find start or end marker in source file") - -start_section = source_code[: start_index + 1] -end_section = source_code[end_index:] -# ---- Generate the definition code from the json ---- - -# Read the JSON file -with open(JSON_PATH, 'r') as json_file: - connection_methods = json.load(json_file) - -body = [] - -DEFAULT_ARGUMENT_MAP = { - 'True': 'true', - 'False': 'false', - 'None': 'py::none()', - 'PythonUDFType.NATIVE': 'PythonUDFType::NATIVE', - 'PythonExceptionHandling.DEFAULT': 'PythonExceptionHandling::FORWARD_ERROR', - 'FunctionNullHandling.DEFAULT': 'FunctionNullHandling::DEFAULT_NULL_HANDLING', -} - - -def map_default(val): - if val in DEFAULT_ARGUMENT_MAP: - return DEFAULT_ARGUMENT_MAP[val] - return val - - -def create_arguments(arguments) -> list: - result = [] - for arg in arguments: - argument = f"py::arg(\"{arg['name']}\")" - if 'allow_none' in arg: - value = str(arg['allow_none']).lower() - argument += f".none({value})" - # Add the default argument if present - if 'default' in arg: - default = map_default(arg['default']) - argument += f" = {default}" - result.append(argument) - return result - - -def create_definition(name, method) -> str: - definition = f"m.def(\"{name}\"" - definition += ", " - definition += f"""&DuckDBPyConnection::{method['function']}""" - definition += ", " - definition += f"\"{method['docs']}\"" - if 'args' in method: + +def generate(): + # Read the PYCONNECTION_SOURCE file + with open(PYCONNECTION_SOURCE, 'r') as source_file: + source_code = source_file.readlines() + + start_index = -1 + end_index = -1 + for i, line in enumerate(source_code): + if line.startswith(INITIALIZE_METHOD): + if start_index != -1: + raise ValueError("Encountered the INITIALIZE_METHOD a second time, quitting!") + start_index = i + elif line.startswith(END_MARKER): + if end_index != -1: + raise ValueError("Encountered the END_MARKER a second time, quitting!") + end_index = i + + if start_index == -1 or end_index == -1: + raise ValueError("Couldn't find start or end marker in source file") + + start_section = source_code[: start_index + 1] + end_section = source_code[end_index:] + # ---- Generate the definition code from the json ---- + + # Read the JSON file + with open(JSON_PATH, 'r') as json_file: + connection_methods = json.load(json_file) + + DEFAULT_ARGUMENT_MAP = { + 'True': 'true', + 'False': 'false', + 'None': 'py::none()', + 'PythonUDFType.NATIVE': 'PythonUDFType::NATIVE', + 'PythonExceptionHandling.DEFAULT': 'PythonExceptionHandling::FORWARD_ERROR', + 'FunctionNullHandling.DEFAULT': 'FunctionNullHandling::DEFAULT_NULL_HANDLING', + } + + def map_default(val): + if val in DEFAULT_ARGUMENT_MAP: + return DEFAULT_ARGUMENT_MAP[val] + return val + + def create_arguments(arguments) -> list: + result = [] + for arg in arguments: + argument = f"py::arg(\"{arg['name']}\")" + if 'allow_none' in arg: + value = str(arg['allow_none']).lower() + argument += f".none({value})" + # Add the default argument if present + if 'default' in arg: + default = map_default(arg['default']) + argument += f" = {default}" + result.append(argument) + return result + + def create_definition(name, method) -> str: + definition = f"m.def(\"{name}\"" definition += ", " - arguments = create_arguments(method['args']) - definition += ', '.join(arguments) - if 'kwargs' in method: + definition += f"""&DuckDBPyConnection::{method['function']}""" definition += ", " - definition += "py::kw_only(), " - arguments = create_arguments(method['kwargs']) - definition += ', '.join(arguments) - definition += ");" - return definition - - -for method in connection_methods: - if isinstance(method['name'], list): - names = method['name'] - else: - names = [method['name']] - for name in names: - body.append(create_definition(name, method)) - -# ---- End of generation code ---- - -with_newlines = ['\t' + x + '\n' for x in body] -# Recreate the file content by concatenating all the pieces together - -new_content = start_section + with_newlines + end_section - -# Write out the modified PYCONNECTION_SOURCE file -with open(PYCONNECTION_SOURCE, 'w') as source_file: - source_file.write("".join(new_content)) + definition += f"\"{method['docs']}\"" + if 'args' in method: + definition += ", " + arguments = create_arguments(method['args']) + definition += ', '.join(arguments) + if 'kwargs' in method: + definition += ", " + definition += "py::kw_only(), " + arguments = create_arguments(method['kwargs']) + definition += ', '.join(arguments) + definition += ");" + return definition + + body = [] + for method in connection_methods: + if isinstance(method['name'], list): + names = method['name'] + else: + names = [method['name']] + for name in names: + body.append(create_definition(name, method)) + + # ---- End of generation code ---- + + with_newlines = ['\t' + x + '\n' for x in body] + # Recreate the file content by concatenating all the pieces together + + new_content = start_section + with_newlines + end_section + + # Write out the modified PYCONNECTION_SOURCE file + with open(PYCONNECTION_SOURCE, 'w') as source_file: + source_file.write("".join(new_content)) + + +if __name__ == '__main__': + raise ValueError("Please use 'generate_connection_code.py' instead of running the individual script(s)") + # generate() diff --git a/tools/pythonpkg/scripts/generate_connection_stubs.py b/tools/pythonpkg/scripts/generate_connection_stubs.py index cacddd3b0ced..fc8c5e9d456a 100644 --- a/tools/pythonpkg/scripts/generate_connection_stubs.py +++ b/tools/pythonpkg/scripts/generate_connection_stubs.py @@ -9,85 +9,88 @@ START_MARKER = " # START OF CONNECTION METHODS" END_MARKER = " # END OF CONNECTION METHODS" -# Read the DUCKDB_STUBS_FILE file -with open(DUCKDB_STUBS_FILE, 'r') as source_file: - source_code = source_file.readlines() - -# Locate the InitializeConnectionMethods function in it -start_index = -1 -end_index = -1 -for i, line in enumerate(source_code): - if line.startswith(START_MARKER): - # TODO: handle the case where the start marker appears multiple times - start_index = i - elif line.startswith(END_MARKER): - # TODO: ditto ^ - end_index = i - -if start_index == -1 or end_index == -1: - raise ValueError("Couldn't find start or end marker in source file") - -start_section = source_code[: start_index + 1] -end_section = source_code[end_index:] -# ---- Generate the definition code from the json ---- - -# Read the JSON file -with open(JSON_PATH, 'r') as json_file: - connection_methods = json.load(json_file) - -body = [] - - -def create_arguments(arguments) -> list: - result = [] - for arg in arguments: - argument = f"{arg['name']}: {arg['type']}" - # Add the default argument if present - if 'default' in arg: - default = arg['default'] - argument += f" = {default}" - result.append(argument) - return result - - -def create_definition(name, method) -> str: - print(method) - definition = f"def {name}(self" - if 'args' in method: - definition += ", " - arguments = create_arguments(method['args']) - definition += ', '.join(arguments) - if 'kwargs' in method: - definition += ", **kwargs" - definition += ")" - definition += f" -> {method['return']}: ..." - return definition - - -# We have "duplicate" methods, which are overloaded -# maybe we should add @overload to these instead, but this is easier -written_methods = set() - -for method in connection_methods: - if isinstance(method['name'], list): - names = method['name'] - else: - names = [method['name']] - for name in names: - if name in written_methods: - continue - body.append(create_definition(name, method)) - written_methods.add(name) - -# ---- End of generation code ---- - -with_newlines = [' ' + x + '\n' for x in body] -# Recreate the file content by concatenating all the pieces together - -new_content = start_section + with_newlines + end_section - -print(with_newlines) - -# Write out the modified DUCKDB_STUBS_FILE file -with open(DUCKDB_STUBS_FILE, 'w') as source_file: - source_file.write("".join(new_content)) + +def generate(): + # Read the DUCKDB_STUBS_FILE file + with open(DUCKDB_STUBS_FILE, 'r') as source_file: + source_code = source_file.readlines() + + start_index = -1 + end_index = -1 + for i, line in enumerate(source_code): + if line.startswith(START_MARKER): + if start_index != -1: + raise ValueError("Encountered the START_MARKER a second time, quitting!") + start_index = i + elif line.startswith(END_MARKER): + if end_index != -1: + raise ValueError("Encountered the END_MARKER a second time, quitting!") + end_index = i + + if start_index == -1 or end_index == -1: + raise ValueError("Couldn't find start or end marker in source file") + + start_section = source_code[: start_index + 1] + end_section = source_code[end_index:] + # ---- Generate the definition code from the json ---- + + # Read the JSON file + with open(JSON_PATH, 'r') as json_file: + connection_methods = json.load(json_file) + + body = [] + + def create_arguments(arguments) -> list: + result = [] + for arg in arguments: + argument = f"{arg['name']}: {arg['type']}" + # Add the default argument if present + if 'default' in arg: + default = arg['default'] + argument += f" = {default}" + result.append(argument) + return result + + def create_definition(name, method) -> str: + print(method) + definition = f"def {name}(self" + if 'args' in method: + definition += ", " + arguments = create_arguments(method['args']) + definition += ', '.join(arguments) + if 'kwargs' in method: + definition += ", **kwargs" + definition += ")" + definition += f" -> {method['return']}: ..." + return definition + + # We have "duplicate" methods, which are overloaded + # maybe we should add @overload to these instead, but this is easier + written_methods = set() + + for method in connection_methods: + if isinstance(method['name'], list): + names = method['name'] + else: + names = [method['name']] + for name in names: + if name in written_methods: + continue + body.append(create_definition(name, method)) + written_methods.add(name) + + # ---- End of generation code ---- + + with_newlines = [' ' + x + '\n' for x in body] + # Recreate the file content by concatenating all the pieces together + + new_content = start_section + with_newlines + end_section + + # Write out the modified DUCKDB_STUBS_FILE file + with open(DUCKDB_STUBS_FILE, 'w') as source_file: + source_file.write("".join(new_content)) + + +if __name__ == '__main__': + raise ValueError("Please use 'generate_connection_code.py' instead of running the individual script(s)") + # generate() diff --git a/tools/pythonpkg/scripts/generate_connection_wrapper_methods.py b/tools/pythonpkg/scripts/generate_connection_wrapper_methods.py index e716765f6816..9d1d1a0fefdc 100644 --- a/tools/pythonpkg/scripts/generate_connection_wrapper_methods.py +++ b/tools/pythonpkg/scripts/generate_connection_wrapper_methods.py @@ -10,141 +10,143 @@ START_MARKER = "# START OF CONNECTION WRAPPER" END_MARKER = "# END OF CONNECTION WRAPPER" -# Read the DUCKDB_INIT_FILE file -with open(DUCKDB_INIT_FILE, 'r') as source_file: - source_code = source_file.readlines() - -start_index = -1 -end_index = -1 -for i, line in enumerate(source_code): - if line.startswith(START_MARKER): - # TODO: handle the case where the start marker appears multiple times - start_index = i - elif line.startswith(END_MARKER): - # TODO: ditto ^ - end_index = i - -if start_index == -1 or end_index == -1: - raise ValueError("Couldn't find start or end marker in source file") - -start_section = source_code[: start_index + 1] -end_section = source_code[end_index:] -# ---- Generate the definition code from the json ---- - -methods = [] - -# Read the JSON file -with open(JSON_PATH, 'r') as json_file: - connection_methods = json.load(json_file) - -with open(WRAPPER_JSON_PATH, 'r') as json_file: - wrapper_methods = json.load(json_file) - -methods.extend(connection_methods) -methods.extend(wrapper_methods) - -# On DuckDBPyConnection these are read_only_properties, they're basically functions without requiring () to invoke -# that's not possible on 'duckdb' so it becomes a function call with no arguments (i.e duckdb.description()) -READONLY_PROPERTY_NAMES = ['description', 'rowcount'] - -# These methods are not directly DuckDBPyConnection methods, -# they first call 'from_df' and then call a method on the created DuckDBPyRelation -SPECIAL_METHOD_NAMES = [x['name'] for x in wrapper_methods if x['name'] not in READONLY_PROPERTY_NAMES] - - -def generate_arguments(name, method) -> str: - arguments = [] - if name in SPECIAL_METHOD_NAMES: - # We add 'df' to these methods because they operate on a DataFrame - arguments.append('df') - - if 'args' in method: - for arg in method['args']: - res = arg['name'] - if 'default' in arg: - res += f" = {arg['default']}" - arguments.append(res) - arguments.append('**kwargs') - return ', '.join(arguments) - - -def generate_parameters(name, method) -> str: - if name in READONLY_PROPERTY_NAMES: - return '' - arguments = [] - if 'args' in method: - for arg in method['args']: - arguments.append(f"{arg['name']}") - arguments.append('**kwargs') - result = ', '.join(arguments) - return '(' + result + ')' - - -def generate_function_call(name) -> str: - function_call = '' - if name in SPECIAL_METHOD_NAMES: - function_call += 'from_df(df).' - - REMAPPED_FUNCTIONS = {'alias': 'set_alias', 'query_df': 'query'} - if name in REMAPPED_FUNCTIONS: - function_name = REMAPPED_FUNCTIONS[name] - else: - function_name = name - function_call += function_name - return function_call - - -def create_definition(name, method) -> str: - print(method) - arguments = generate_arguments(name, method) - parameters = generate_parameters(name, method) - function_call = generate_function_call(name) - - func = f""" -def {name}({arguments}): - if 'connection' in kwargs: - conn = kwargs.pop('connection') - else: - conn = duckdb.connect(":default:") - return conn.{function_call}{parameters} -_exported_symbols.append('{name}') -""" - return func - - -# We have "duplicate" methods, which are overloaded -written_methods = set() - -body = [] -for method in methods: - if isinstance(method['name'], list): - names = method['name'] - else: - names = [method['name']] - - # Artificially add 'connection' keyword argument - if 'kwargs' not in method: - method['kwargs'] = [] - method['kwargs'].append({'name': 'connection', 'type': 'DuckDBPyConnection'}) - - for name in names: - if name in written_methods: - continue - if name in ['arrow', 'df']: - # These methods are ambiguous and are handled in C++ code instead - continue - body.append(create_definition(name, method)) - written_methods.add(name) - -# ---- End of generation code ---- - -with_newlines = body -# Recreate the file content by concatenating all the pieces together - -new_content = start_section + with_newlines + end_section - -print(''.join(with_newlines)) - -# Write out the modified DUCKDB_INIT_FILE file -with open(DUCKDB_INIT_FILE, 'w') as source_file: - source_file.write("".join(new_content)) + +def generate(): + # Read the DUCKDB_INIT_FILE file + with open(DUCKDB_INIT_FILE, 'r') as source_file: + source_code = source_file.readlines() + + start_index = -1 + end_index = -1 + for i, line in enumerate(source_code): + if line.startswith(START_MARKER): + if start_index != -1: + raise ValueError("Encountered the START_MARKER a second time, quitting!") + start_index = i + elif line.startswith(END_MARKER): + if end_index != -1: + raise ValueError("Encountered the END_MARKER a second time, quitting!") + end_index = i + + if start_index == -1 or end_index == -1: + raise ValueError("Couldn't find start or end marker in source file") + + start_section = source_code[: start_index + 1] + end_section = source_code[end_index:] + # ---- Generate the definition code from the json ---- + + methods = [] + + # Read the JSON file + with open(JSON_PATH, 'r') as json_file: + connection_methods = json.load(json_file) + + with open(WRAPPER_JSON_PATH, 'r') as json_file: + wrapper_methods = json.load(json_file) + + methods.extend(connection_methods) + methods.extend(wrapper_methods) + + # On DuckDBPyConnection these are read_only_properties, they're basically functions without requiring () to invoke + # that's not possible on 'duckdb' so it becomes a function call with no arguments (i.e duckdb.description()) + READONLY_PROPERTY_NAMES = ['description', 'rowcount'] + + # These methods are not directly DuckDBPyConnection methods, + # they first call 'from_df' and then call a method on the created DuckDBPyRelation + SPECIAL_METHOD_NAMES = [x['name'] for x in wrapper_methods if x['name'] not in READONLY_PROPERTY_NAMES] + + def generate_arguments(name, method) -> str: + arguments = [] + if name in SPECIAL_METHOD_NAMES: + # We add 'df' to these methods because they operate on a DataFrame + arguments.append('df') + + if 'args' in method: + for arg in method['args']: + res = arg['name'] + if 'default' in arg: + res += f" = {arg['default']}" + arguments.append(res) + arguments.append('**kwargs') + return ', '.join(arguments) + + def generate_parameters(name, method) -> str: + if name in READONLY_PROPERTY_NAMES: + return '' + arguments = [] + if 'args' in method: + for arg in method['args']: + arguments.append(f"{arg['name']}") + arguments.append('**kwargs') + result = ', '.join(arguments) + return '(' + result + ')' + + def generate_function_call(name) -> str: + function_call = '' + if name in SPECIAL_METHOD_NAMES: + function_call += 'from_df(df).' + + REMAPPED_FUNCTIONS = {'alias': 'set_alias', 'query_df': 'query'} + if name in REMAPPED_FUNCTIONS: + function_name = REMAPPED_FUNCTIONS[name] + else: + function_name = name + function_call += function_name + return function_call + + def create_definition(name, method) -> str: + print(method) + arguments = generate_arguments(name, method) + parameters = generate_parameters(name, method) + function_call = generate_function_call(name) + + func = f""" + def {name}({arguments}): + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") + return conn.{function_call}{parameters} + _exported_symbols.append('{name}') + """ + return func + + # We have "duplicate" methods, which are overloaded + written_methods = set() + + body = [] + for method in methods: + if isinstance(method['name'], list): + names = method['name'] + else: + names = [method['name']] + + # Artificially add 'connection' keyword argument + if 'kwargs' not in method: + method['kwargs'] = [] + method['kwargs'].append({'name': 'connection', 'type': 'DuckDBPyConnection'}) + + for name in names: + if name in written_methods: + continue + if name in ['arrow', 'df']: + # These methods are ambiguous and are handled in C++ code instead + continue + body.append(create_definition(name, method)) + written_methods.add(name) + + # ---- End of generation code ---- + + with_newlines = body + # Recreate the file content by concatenating all the pieces together + + new_content = start_section + with_newlines + end_section + + # Write out the modified DUCKDB_INIT_FILE file + with open(DUCKDB_INIT_FILE, 'w') as source_file: + source_file.write("".join(new_content)) + + +if __name__ == '__main__': + raise ValueError("Please use 'generate_connection_code.py' instead of running the individual script(s)") + # generate() diff --git a/tools/pythonpkg/scripts/generate_connection_wrapper_stubs.py b/tools/pythonpkg/scripts/generate_connection_wrapper_stubs.py index f20b967318f3..ce6d4a58d869 100644 --- a/tools/pythonpkg/scripts/generate_connection_wrapper_stubs.py +++ b/tools/pythonpkg/scripts/generate_connection_wrapper_stubs.py @@ -10,107 +10,111 @@ START_MARKER = "# START OF CONNECTION WRAPPER" END_MARKER = "# END OF CONNECTION WRAPPER" -# Read the DUCKDB_STUBS_FILE file -with open(DUCKDB_STUBS_FILE, 'r') as source_file: - source_code = source_file.readlines() - -start_index = -1 -end_index = -1 -for i, line in enumerate(source_code): - if line.startswith(START_MARKER): - # TODO: handle the case where the start marker appears multiple times - start_index = i - elif line.startswith(END_MARKER): - # TODO: ditto ^ - end_index = i - -if start_index == -1 or end_index == -1: - raise ValueError("Couldn't find start or end marker in source file") - -start_section = source_code[: start_index + 1] -end_section = source_code[end_index:] -# ---- Generate the definition code from the json ---- - -methods = [] - -# Read the JSON file -with open(JSON_PATH, 'r') as json_file: - connection_methods = json.load(json_file) - -with open(WRAPPER_JSON_PATH, 'r') as json_file: - wrapper_methods = json.load(json_file) - -methods.extend(connection_methods) -methods.extend(wrapper_methods) - -# On DuckDBPyConnection these are read_only_properties, they're basically functions without requiring () to invoke -# that's not possible on 'duckdb' so it becomes a function call with no arguments (i.e duckdb.description()) -READONLY_PROPERTY_NAMES = ['description', 'rowcount'] - -# These methods are not directly DuckDBPyConnection methods, -# they first call 'from_df' and then call a method on the created DuckDBPyRelation -SPECIAL_METHOD_NAMES = [x['name'] for x in wrapper_methods if x['name'] not in READONLY_PROPERTY_NAMES] - - -def create_arguments(arguments) -> list: - result = [] - for arg in arguments: - argument = f"{arg['name']}: {arg['type']}" - # Add the default argument if present - if 'default' in arg: - default = arg['default'] - argument += f" = {default}" - result.append(argument) - return result - - -def create_definition(name, method) -> str: - print(method) - definition = f"def {name}(" - arguments = [] - if name in SPECIAL_METHOD_NAMES: - arguments.append('df: pandas.DataFrame') - if 'args' in method: - arguments.extend(create_arguments(method['args'])) - if 'kwargs' in method: - arguments.append("**kwargs") - definition += ', '.join(arguments) - definition += ")" - definition += f" -> {method['return']}: ..." - return definition - - -# We have "duplicate" methods, which are overloaded -# maybe we should add @overload to these instead, but this is easier -written_methods = set() - -body = [] -for method in methods: - if isinstance(method['name'], list): - names = method['name'] - else: - names = [method['name']] - - # Artificially add 'connection' keyword argument - if 'kwargs' not in method: - method['kwargs'] = [] - method['kwargs'].append({'name': 'connection', 'type': 'DuckDBPyConnection'}) - - for name in names: - if name in written_methods: - continue - body.append(create_definition(name, method)) - written_methods.add(name) - -# ---- End of generation code ---- - -with_newlines = [x + '\n' for x in body] -# Recreate the file content by concatenating all the pieces together - -new_content = start_section + with_newlines + end_section - -print(''.join(with_newlines)) - -# Write out the modified DUCKDB_STUBS_FILE file -with open(DUCKDB_STUBS_FILE, 'w') as source_file: - source_file.write("".join(new_content)) + +def generate(): + # Read the DUCKDB_STUBS_FILE file + with open(DUCKDB_STUBS_FILE, 'r') as source_file: + source_code = source_file.readlines() + + start_index = -1 + end_index = -1 + for i, line in enumerate(source_code): + if line.startswith(START_MARKER): + if start_index != -1: + raise ValueError("Encountered the START_MARKER a second time, quitting!") + start_index = i + elif line.startswith(END_MARKER): + if end_index != -1: + raise ValueError("Encountered the END_MARKER a second time, quitting!") + end_index = i + + if start_index == -1 or end_index == -1: + raise ValueError("Couldn't find start or end marker in source file") + + start_section = source_code[: start_index + 1] + end_section = source_code[end_index:] + # ---- Generate the definition code from the json ---- + + methods = [] + + # Read the JSON file + with open(JSON_PATH, 'r') as json_file: + connection_methods = json.load(json_file) + + with open(WRAPPER_JSON_PATH, 'r') as json_file: + wrapper_methods = json.load(json_file) + + methods.extend(connection_methods) + methods.extend(wrapper_methods) + + # On DuckDBPyConnection these are read_only_properties, they're basically functions without requiring () to invoke + # that's not possible on 'duckdb' so it becomes a function call with no arguments (i.e duckdb.description()) + READONLY_PROPERTY_NAMES = ['description', 'rowcount'] + + # These methods are not directly DuckDBPyConnection methods, + # they first call 'from_df' and then call a method on the created DuckDBPyRelation + SPECIAL_METHOD_NAMES = [x['name'] for x in wrapper_methods if x['name'] not in READONLY_PROPERTY_NAMES] + + def create_arguments(arguments) -> list: + result = [] + for arg in arguments: + argument = f"{arg['name']}: {arg['type']}" + # Add the default argument if present + if 'default' in arg: + default = arg['default'] + argument += f" = {default}" + result.append(argument) + return result + + def create_definition(name, method) -> str: + print(method) + definition = f"def {name}(" + arguments = [] + if name in SPECIAL_METHOD_NAMES: + arguments.append('df: pandas.DataFrame') + if 'args' in method: + arguments.extend(create_arguments(method['args'])) + if 'kwargs' in method: + arguments.append("**kwargs") + definition += ', '.join(arguments) + definition += ")" + definition += f" -> {method['return']}: ..." + return definition + + # We have "duplicate" methods, which are overloaded + # maybe we should add @overload to these instead, but this is easier + written_methods = set() + + body = [] + for method in methods: + if isinstance(method['name'], list): + names = method['name'] + else: + names = [method['name']] + + # Artificially add 'connection' keyword argument + if 'kwargs' not in method: + method['kwargs'] = [] + method['kwargs'].append({'name': 'connection', 'type': 'DuckDBPyConnection'}) + + for name in names: + if name in written_methods: + continue + body.append(create_definition(name, method)) + written_methods.add(name) + + # ---- End of generation code ---- + + with_newlines = [x + '\n' for x in body] + # Recreate the file content by concatenating all the pieces together + + new_content = start_section + with_newlines + end_section + + # Write out the modified DUCKDB_STUBS_FILE file + with open(DUCKDB_STUBS_FILE, 'w') as source_file: + source_file.write("".join(new_content)) + + +if __name__ == '__main__': + raise ValueError("Please use 'generate_connection_code.py' instead of running the individual script(s)") + # generate() From 64c8921d7e3cc0c0396983ef09accb35bde72cb4 Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 11 Apr 2024 10:05:41 +0200 Subject: [PATCH 318/603] cleanup --- tools/pythonpkg/duckdb/__init__.py | 150 +++++++++--------- .../scripts/generate_connection_stubs.py | 1 - .../generate_connection_wrapper_methods.py | 17 +- .../generate_connection_wrapper_stubs.py | 1 - 4 files changed, 83 insertions(+), 86 deletions(-) diff --git a/tools/pythonpkg/duckdb/__init__.py b/tools/pythonpkg/duckdb/__init__.py index 5d2d56e2dd4e..c6683a712782 100644 --- a/tools/pythonpkg/duckdb/__init__.py +++ b/tools/pythonpkg/duckdb/__init__.py @@ -57,7 +57,7 @@ def cursor(**kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.cursor(**kwargs) @@ -65,7 +65,7 @@ def cursor(**kwargs): def register_filesystem(filesystem, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.register_filesystem(filesystem, **kwargs) @@ -73,7 +73,7 @@ def register_filesystem(filesystem, **kwargs): def unregister_filesystem(name, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.unregister_filesystem(name, **kwargs) @@ -81,7 +81,7 @@ def unregister_filesystem(name, **kwargs): def list_filesystems(**kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.list_filesystems(**kwargs) @@ -89,7 +89,7 @@ def list_filesystems(**kwargs): def filesystem_is_registered(name, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.filesystem_is_registered(name, **kwargs) @@ -97,7 +97,7 @@ def filesystem_is_registered(name, **kwargs): def create_function(name, function, parameters = None, return_type = None, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.create_function(name, function, parameters, return_type, **kwargs) @@ -105,7 +105,7 @@ def create_function(name, function, parameters = None, return_type = None, **kwa def remove_function(name, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.remove_function(name, **kwargs) @@ -113,7 +113,7 @@ def remove_function(name, **kwargs): def sqltype(type_str, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.sqltype(type_str, **kwargs) @@ -121,7 +121,7 @@ def sqltype(type_str, **kwargs): def dtype(type_str, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.dtype(type_str, **kwargs) @@ -129,7 +129,7 @@ def dtype(type_str, **kwargs): def type(type_str, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.type(type_str, **kwargs) @@ -137,7 +137,7 @@ def type(type_str, **kwargs): def array_type(type, size, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.array_type(type, size, **kwargs) @@ -145,7 +145,7 @@ def array_type(type, size, **kwargs): def list_type(type, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.list_type(type, **kwargs) @@ -153,7 +153,7 @@ def list_type(type, **kwargs): def union_type(members, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.union_type(members, **kwargs) @@ -161,7 +161,7 @@ def union_type(members, **kwargs): def string_type(collation = "", **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.string_type(collation, **kwargs) @@ -169,7 +169,7 @@ def string_type(collation = "", **kwargs): def enum_type(name, type, values, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.enum_type(name, type, values, **kwargs) @@ -177,7 +177,7 @@ def enum_type(name, type, values, **kwargs): def decimal_type(width, scale, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.decimal_type(width, scale, **kwargs) @@ -185,7 +185,7 @@ def decimal_type(width, scale, **kwargs): def struct_type(fields, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.struct_type(fields, **kwargs) @@ -193,7 +193,7 @@ def struct_type(fields, **kwargs): def row_type(fields, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.row_type(fields, **kwargs) @@ -201,7 +201,7 @@ def row_type(fields, **kwargs): def map_type(key, value, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.map_type(key, value, **kwargs) @@ -209,7 +209,7 @@ def map_type(key, value, **kwargs): def duplicate(**kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.duplicate(**kwargs) @@ -217,7 +217,7 @@ def duplicate(**kwargs): def execute(query, parameters = None, multiple_parameter_sets = False, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.execute(query, parameters, multiple_parameter_sets, **kwargs) @@ -225,7 +225,7 @@ def execute(query, parameters = None, multiple_parameter_sets = False, **kwargs) def executemany(query, parameters = None, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.executemany(query, parameters, **kwargs) @@ -233,7 +233,7 @@ def executemany(query, parameters = None, **kwargs): def close(**kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.close(**kwargs) @@ -241,7 +241,7 @@ def close(**kwargs): def interrupt(**kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.interrupt(**kwargs) @@ -249,7 +249,7 @@ def interrupt(**kwargs): def fetchone(**kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.fetchone(**kwargs) @@ -257,7 +257,7 @@ def fetchone(**kwargs): def fetchmany(size = 1, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.fetchmany(size, **kwargs) @@ -265,7 +265,7 @@ def fetchmany(size = 1, **kwargs): def fetchall(**kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.fetchall(**kwargs) @@ -273,7 +273,7 @@ def fetchall(**kwargs): def fetchnumpy(**kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.fetchnumpy(**kwargs) @@ -281,7 +281,7 @@ def fetchnumpy(**kwargs): def fetchdf(**kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.fetchdf(**kwargs) @@ -289,7 +289,7 @@ def fetchdf(**kwargs): def fetch_df(**kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.fetch_df(**kwargs) @@ -297,7 +297,7 @@ def fetch_df(**kwargs): def fetch_df_chunk(vectors_per_chunk = 1, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.fetch_df_chunk(vectors_per_chunk, **kwargs) @@ -305,7 +305,7 @@ def fetch_df_chunk(vectors_per_chunk = 1, **kwargs): def pl(rows_per_batch = 1000000, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.pl(rows_per_batch, **kwargs) @@ -313,7 +313,7 @@ def pl(rows_per_batch = 1000000, **kwargs): def fetch_arrow_table(rows_per_batch = 1000000, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.fetch_arrow_table(rows_per_batch, **kwargs) @@ -321,7 +321,7 @@ def fetch_arrow_table(rows_per_batch = 1000000, **kwargs): def fetch_record_batch(rows_per_batch = 1000000, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.fetch_record_batch(rows_per_batch, **kwargs) @@ -329,7 +329,7 @@ def fetch_record_batch(rows_per_batch = 1000000, **kwargs): def torch(**kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.torch(**kwargs) @@ -337,7 +337,7 @@ def torch(**kwargs): def tf(**kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.tf(**kwargs) @@ -345,7 +345,7 @@ def tf(**kwargs): def begin(**kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.begin(**kwargs) @@ -353,7 +353,7 @@ def begin(**kwargs): def commit(**kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.commit(**kwargs) @@ -361,7 +361,7 @@ def commit(**kwargs): def rollback(**kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.rollback(**kwargs) @@ -369,7 +369,7 @@ def rollback(**kwargs): def append(table_name, df, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.append(table_name, df, **kwargs) @@ -377,7 +377,7 @@ def append(table_name, df, **kwargs): def register(view_name, python_object, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.register(view_name, python_object, **kwargs) @@ -385,7 +385,7 @@ def register(view_name, python_object, **kwargs): def unregister(view_name, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.unregister(view_name, **kwargs) @@ -393,7 +393,7 @@ def unregister(view_name, **kwargs): def table(table_name, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.table(table_name, **kwargs) @@ -401,7 +401,7 @@ def table(table_name, **kwargs): def view(view_name, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.view(view_name, **kwargs) @@ -409,7 +409,7 @@ def view(view_name, **kwargs): def values(values, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.values(values, **kwargs) @@ -417,7 +417,7 @@ def values(values, **kwargs): def table_function(name, parameters = None, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.table_function(name, parameters, **kwargs) @@ -425,7 +425,7 @@ def table_function(name, parameters = None, **kwargs): def read_json(name, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.read_json(name, **kwargs) @@ -433,7 +433,7 @@ def read_json(name, **kwargs): def extract_statements(query, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.extract_statements(query, **kwargs) @@ -441,7 +441,7 @@ def extract_statements(query, **kwargs): def sql(query, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.sql(query, **kwargs) @@ -449,7 +449,7 @@ def sql(query, **kwargs): def query(query, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.query(query, **kwargs) @@ -457,7 +457,7 @@ def query(query, **kwargs): def from_query(query, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.from_query(query, **kwargs) @@ -465,7 +465,7 @@ def from_query(query, **kwargs): def read_csv(path_or_buffer, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.read_csv(path_or_buffer, **kwargs) @@ -473,7 +473,7 @@ def read_csv(path_or_buffer, **kwargs): def from_csv_auto(path_or_buffer, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.from_csv_auto(path_or_buffer, **kwargs) @@ -481,7 +481,7 @@ def from_csv_auto(path_or_buffer, **kwargs): def from_df(df, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.from_df(df, **kwargs) @@ -489,7 +489,7 @@ def from_df(df, **kwargs): def from_arrow(arrow_object, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.from_arrow(arrow_object, **kwargs) @@ -497,7 +497,7 @@ def from_arrow(arrow_object, **kwargs): def from_parquet(file_glob, binary_as_string = False, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.from_parquet(file_glob, binary_as_string, **kwargs) @@ -505,7 +505,7 @@ def from_parquet(file_glob, binary_as_string = False, **kwargs): def read_parquet(file_glob, binary_as_string = False, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.read_parquet(file_glob, binary_as_string, **kwargs) @@ -513,7 +513,7 @@ def read_parquet(file_glob, binary_as_string = False, **kwargs): def from_substrait(proto, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.from_substrait(proto, **kwargs) @@ -521,7 +521,7 @@ def from_substrait(proto, **kwargs): def get_substrait(query, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.get_substrait(query, **kwargs) @@ -529,7 +529,7 @@ def get_substrait(query, **kwargs): def get_substrait_json(query, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.get_substrait_json(query, **kwargs) @@ -537,7 +537,7 @@ def get_substrait_json(query, **kwargs): def from_substrait_json(json, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.from_substrait_json(json, **kwargs) @@ -545,7 +545,7 @@ def from_substrait_json(json, **kwargs): def get_table_names(query, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.get_table_names(query, **kwargs) @@ -553,7 +553,7 @@ def get_table_names(query, **kwargs): def install_extension(extension, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.install_extension(extension, **kwargs) @@ -561,7 +561,7 @@ def install_extension(extension, **kwargs): def load_extension(extension, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.load_extension(extension, **kwargs) @@ -569,7 +569,7 @@ def load_extension(extension, **kwargs): def project(df, project_expr, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.from_df(df).project(project_expr, **kwargs) @@ -577,7 +577,7 @@ def project(df, project_expr, **kwargs): def distinct(df, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.from_df(df).distinct(**kwargs) @@ -585,7 +585,7 @@ def distinct(df, **kwargs): def write_csv(df, *args, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.from_df(df).write_csv(*args, **kwargs) @@ -593,7 +593,7 @@ def write_csv(df, *args, **kwargs): def aggregate(df, aggr_expr, group_expr = "", **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.from_df(df).aggregate(aggr_expr, group_expr, **kwargs) @@ -601,7 +601,7 @@ def aggregate(df, aggr_expr, group_expr = "", **kwargs): def alias(df, alias, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.from_df(df).set_alias(alias, **kwargs) @@ -609,7 +609,7 @@ def alias(df, alias, **kwargs): def filter(df, filter_expr, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.from_df(df).filter(filter_expr, **kwargs) @@ -617,7 +617,7 @@ def filter(df, filter_expr, **kwargs): def limit(df, n, offset = 0, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.from_df(df).limit(n, offset, **kwargs) @@ -625,7 +625,7 @@ def limit(df, n, offset = 0, **kwargs): def order(df, order_expr, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.from_df(df).order(order_expr, **kwargs) @@ -633,7 +633,7 @@ def order(df, order_expr, **kwargs): def query_df(df, virtual_table_name, sql_query, **kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.from_df(df).query(virtual_table_name, sql_query, **kwargs) @@ -641,7 +641,7 @@ def query_df(df, virtual_table_name, sql_query, **kwargs): def description(**kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.description @@ -649,7 +649,7 @@ def description(**kwargs): def rowcount(**kwargs): if 'connection' in kwargs: - conn = kwargs.pop('connection') + conn = kwargs.pop('connection') else: conn = duckdb.connect(":default:") return conn.rowcount diff --git a/tools/pythonpkg/scripts/generate_connection_stubs.py b/tools/pythonpkg/scripts/generate_connection_stubs.py index fc8c5e9d456a..7343ec3ec889 100644 --- a/tools/pythonpkg/scripts/generate_connection_stubs.py +++ b/tools/pythonpkg/scripts/generate_connection_stubs.py @@ -52,7 +52,6 @@ def create_arguments(arguments) -> list: return result def create_definition(name, method) -> str: - print(method) definition = f"def {name}(self" if 'args' in method: definition += ", " diff --git a/tools/pythonpkg/scripts/generate_connection_wrapper_methods.py b/tools/pythonpkg/scripts/generate_connection_wrapper_methods.py index 9d1d1a0fefdc..ac6345212c80 100644 --- a/tools/pythonpkg/scripts/generate_connection_wrapper_methods.py +++ b/tools/pythonpkg/scripts/generate_connection_wrapper_methods.py @@ -95,20 +95,19 @@ def generate_function_call(name) -> str: return function_call def create_definition(name, method) -> str: - print(method) arguments = generate_arguments(name, method) parameters = generate_parameters(name, method) function_call = generate_function_call(name) func = f""" - def {name}({arguments}): - if 'connection' in kwargs: - conn = kwargs.pop('connection') - else: - conn = duckdb.connect(":default:") - return conn.{function_call}{parameters} - _exported_symbols.append('{name}') - """ +def {name}({arguments}): + if 'connection' in kwargs: + conn = kwargs.pop('connection') + else: + conn = duckdb.connect(":default:") + return conn.{function_call}{parameters} +_exported_symbols.append('{name}') +""" return func # We have "duplicate" methods, which are overloaded diff --git a/tools/pythonpkg/scripts/generate_connection_wrapper_stubs.py b/tools/pythonpkg/scripts/generate_connection_wrapper_stubs.py index ce6d4a58d869..7665a91906fd 100644 --- a/tools/pythonpkg/scripts/generate_connection_wrapper_stubs.py +++ b/tools/pythonpkg/scripts/generate_connection_wrapper_stubs.py @@ -67,7 +67,6 @@ def create_arguments(arguments) -> list: return result def create_definition(name, method) -> str: - print(method) definition = f"def {name}(" arguments = [] if name in SPECIAL_METHOD_NAMES: From cbf1b651d1cb8f966a4e608870afa4cf2ba83bf4 Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 11 Apr 2024 10:08:58 +0200 Subject: [PATCH 319/603] add connection code generation script to 'generate-files' --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index a6fd0dcd3ea0..903363ad5039 100644 --- a/Makefile +++ b/Makefile @@ -443,6 +443,7 @@ generate-files: python3 scripts/generate_functions.py python3 scripts/generate_serialization.py python3 scripts/generate_enum_util.py + python3 tools/pythonpkg/scripts/generate_connection_code.py ./scripts/generate_micro_extended.sh bundle-library: release From 1c1636ab4877949724440d302a2c3bf32f5cf3d6 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 11 Apr 2024 10:50:45 +0200 Subject: [PATCH 320/603] Pragma assignment with multiple parameters - ParserException instead of InternalException --- src/parser/transform/statement/transform_pragma.cpp | 2 +- ...sterfuzz-testcase-minimized-parse_fuzz_test-5041566491475968 | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 test/ossfuzz/cases/clusterfuzz-testcase-minimized-parse_fuzz_test-5041566491475968 diff --git a/src/parser/transform/statement/transform_pragma.cpp b/src/parser/transform/statement/transform_pragma.cpp index d55d6a2bb8bc..63a5c35d4559 100644 --- a/src/parser/transform/statement/transform_pragma.cpp +++ b/src/parser/transform/statement/transform_pragma.cpp @@ -49,7 +49,7 @@ unique_ptr Transformer::TransformPragma(duckdb_libpgquery::PGPragm break; case duckdb_libpgquery::PG_PRAGMA_TYPE_ASSIGNMENT: if (info.parameters.size() != 1) { - throw InternalException("PRAGMA statement with assignment should contain exactly one parameter"); + throw ParserException("PRAGMA statement with assignment should contain exactly one parameter"); } if (!info.named_parameters.empty()) { throw InternalException("PRAGMA statement with assignment cannot have named parameters"); diff --git a/test/ossfuzz/cases/clusterfuzz-testcase-minimized-parse_fuzz_test-5041566491475968 b/test/ossfuzz/cases/clusterfuzz-testcase-minimized-parse_fuzz_test-5041566491475968 new file mode 100644 index 000000000000..64f0ff9d3f63 --- /dev/null +++ b/test/ossfuzz/cases/clusterfuzz-testcase-minimized-parse_fuzz_test-5041566491475968 @@ -0,0 +1 @@ +pragma ÿ=2,a \ No newline at end of file From e92cf9550919961b9f81c5c595d3543b3cd6d6f2 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 11 Apr 2024 10:54:35 +0200 Subject: [PATCH 321/603] Parameters in SET not supported (yet?) --- src/planner/binder/statement/bind_set.cpp | 3 +++ ...terfuzz-testcase-minimized-parse_fuzz_test-5103220313423872 | 1 + 2 files changed, 4 insertions(+) create mode 100644 test/ossfuzz/cases/clusterfuzz-testcase-minimized-parse_fuzz_test-5103220313423872 diff --git a/src/planner/binder/statement/bind_set.cpp b/src/planner/binder/statement/bind_set.cpp index 15f33d31faea..77c23e22ffd9 100644 --- a/src/planner/binder/statement/bind_set.cpp +++ b/src/planner/binder/statement/bind_set.cpp @@ -15,6 +15,9 @@ BoundStatement Binder::Bind(SetVariableStatement &stmt) { // evaluate the scalar value ConstantBinder default_binder(*this, context, "SET value"); auto bound_value = default_binder.Bind(stmt.value); + if (bound_value->HasParameter()) { + throw NotImplementedException("SET statements cannot have parameters"); + } auto value = ExpressionExecutor::EvaluateScalar(context, *bound_value, true); result.plan = make_uniq(stmt.name, std::move(value), stmt.scope); diff --git a/test/ossfuzz/cases/clusterfuzz-testcase-minimized-parse_fuzz_test-5103220313423872 b/test/ossfuzz/cases/clusterfuzz-testcase-minimized-parse_fuzz_test-5103220313423872 new file mode 100644 index 000000000000..9949d4475074 --- /dev/null +++ b/test/ossfuzz/cases/clusterfuzz-testcase-minimized-parse_fuzz_test-5103220313423872 @@ -0,0 +1 @@ +set ÿ=?=? \ No newline at end of file From 3862ec717912da20785e1fa4698a4f1945e9ec14 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 11 Apr 2024 11:06:28 +0200 Subject: [PATCH 322/603] Correctly check for overflows in hugeint sum/avg --- .../aggregate/algebraic/avg.cpp | 4 +-- .../aggregate/distributive/sum.cpp | 4 +-- .../core_functions/aggregate/sum_helpers.hpp | 14 +++++++++- ...minimized-parse_fuzz_test-5145260887965696 | 1 + .../types/hugeint/hugeint_sum_overflow.test | 26 +++++++++++++++++++ 5 files changed, 44 insertions(+), 5 deletions(-) create mode 100644 test/ossfuzz/cases/clusterfuzz-testcase-minimized-parse_fuzz_test-5145260887965696 create mode 100644 test/sql/types/hugeint/hugeint_sum_overflow.test diff --git a/src/core_functions/aggregate/algebraic/avg.cpp b/src/core_functions/aggregate/algebraic/avg.cpp index 9cebfc4a0b02..d00e743ff6ca 100644 --- a/src/core_functions/aggregate/algebraic/avg.cpp +++ b/src/core_functions/aggregate/algebraic/avg.cpp @@ -93,7 +93,7 @@ struct IntegerAverageOperation : public BaseSumOperation { +struct IntegerAverageOperationHugeint : public BaseSumOperation { template static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) { if (state.count == 0) { @@ -105,7 +105,7 @@ struct IntegerAverageOperationHugeint : public BaseSumOperation { +struct HugeintAverageOperation : public BaseSumOperation { template static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) { if (state.count == 0) { diff --git a/src/core_functions/aggregate/distributive/sum.cpp b/src/core_functions/aggregate/distributive/sum.cpp index 9f243869ad16..0d855297e91d 100644 --- a/src/core_functions/aggregate/distributive/sum.cpp +++ b/src/core_functions/aggregate/distributive/sum.cpp @@ -32,7 +32,7 @@ struct IntegerSumOperation : public BaseSumOperation { +struct SumToHugeintOperation : public BaseSumOperation { template static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) { if (!state.isset) { @@ -58,7 +58,7 @@ struct DoubleSumOperation : public BaseSumOperation; using KahanSumOperation = DoubleSumOperation; -struct HugeintSumOperation : public BaseSumOperation { +struct HugeintSumOperation : public BaseSumOperation { template static void Finalize(STATE &state, T &target, AggregateFinalizeData &finalize_data) { if (!state.isset) { diff --git a/src/include/duckdb/core_functions/aggregate/sum_helpers.hpp b/src/include/duckdb/core_functions/aggregate/sum_helpers.hpp index 45f533a7f8c4..355701bdf25d 100644 --- a/src/include/duckdb/core_functions/aggregate/sum_helpers.hpp +++ b/src/include/duckdb/core_functions/aggregate/sum_helpers.hpp @@ -65,6 +65,18 @@ struct RegularAdd { } }; +struct HugeintAdd { + template + static void AddNumber(STATE &state, T input) { + state.value = Hugeint::Add(state.value, input); + } + + template + static void AddConstant(STATE &state, T input, idx_t count) { + AddNumber(state, Hugeint::Multiply(input, count)); + } +}; + struct KahanAdd { template static void AddNumber(STATE &state, T input) { @@ -77,7 +89,7 @@ struct KahanAdd { } }; -struct HugeintAdd { +struct AddToHugeint { static void AddValue(hugeint_t &result, uint64_t value, int positive) { // integer summation taken from Tim Gubner et al. - Efficient Query Processing // with Optimistically Compressed Hash Tables & Strings in the USSR diff --git a/test/ossfuzz/cases/clusterfuzz-testcase-minimized-parse_fuzz_test-5145260887965696 b/test/ossfuzz/cases/clusterfuzz-testcase-minimized-parse_fuzz_test-5145260887965696 new file mode 100644 index 000000000000..f6e76e22cab4 --- /dev/null +++ b/test/ossfuzz/cases/clusterfuzz-testcase-minimized-parse_fuzz_test-5145260887965696 @@ -0,0 +1 @@ +Summarize select-170141183460469231731687303715884105728 \ No newline at end of file diff --git a/test/sql/types/hugeint/hugeint_sum_overflow.test b/test/sql/types/hugeint/hugeint_sum_overflow.test new file mode 100644 index 000000000000..84a57e364474 --- /dev/null +++ b/test/sql/types/hugeint/hugeint_sum_overflow.test @@ -0,0 +1,26 @@ +# name: test/sql/types/hugeint/hugeint_sum_overflow.test +# description: Test hugeint sum overflow +# group: [hugeint] + +statement ok +PRAGMA enable_verification + +statement error +SELECT SUM(170141183460469231731687303715884105727) FROM range(10); +---- +Overflow + +statement error +SELECT SUM(x) FROM (VALUES (170141183460469231731687303715884105727), (170141183460469231731687303715884105727)) t(x) +---- +Overflow + +statement error +SELECT AVG(170141183460469231731687303715884105727) FROM range(10); +---- +Overflow + +statement error +SELECT AVG(x) FROM (VALUES (170141183460469231731687303715884105727), (170141183460469231731687303715884105727)) t(x) +---- +Overflow From ae79db86417afb7766e7df40ea04cfce0726ae01 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 11 Apr 2024 11:17:27 +0200 Subject: [PATCH 323/603] Push correct target scale in decimal rounding --- src/core_functions/scalar/math/numeric.cpp | 2 +- ...sterfuzz-testcase-minimized-parse_fuzz_test-4954980899422208 | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 test/ossfuzz/cases/clusterfuzz-testcase-minimized-parse_fuzz_test-4954980899422208 diff --git a/src/core_functions/scalar/math/numeric.cpp b/src/core_functions/scalar/math/numeric.cpp index d47887f93e6a..2594b2f96b54 100644 --- a/src/core_functions/scalar/math/numeric.cpp +++ b/src/core_functions/scalar/math/numeric.cpp @@ -675,7 +675,7 @@ unique_ptr BindDecimalRoundPrecision(ClientContext &context, Scala } bound_function.arguments[0] = decimal_type; bound_function.return_type = LogicalType::DECIMAL(width, target_scale); - return make_uniq(round_value); + return make_uniq(target_scale); } ScalarFunctionSet RoundFun::GetFunctions() { diff --git a/test/ossfuzz/cases/clusterfuzz-testcase-minimized-parse_fuzz_test-4954980899422208 b/test/ossfuzz/cases/clusterfuzz-testcase-minimized-parse_fuzz_test-4954980899422208 new file mode 100644 index 000000000000..5af47c2af954 --- /dev/null +++ b/test/ossfuzz/cases/clusterfuzz-testcase-minimized-parse_fuzz_test-4954980899422208 @@ -0,0 +1 @@ +sElecT round(.3333333333333333,~2) \ No newline at end of file From a106c42632615d2ee30a62a4393a62f6a22aa2bb Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 11 Apr 2024 11:18:00 +0200 Subject: [PATCH 324/603] Add all new ossfuzz cases --- ...usterfuzz-testcase-minimized-parse_fuzz_test-5177383552352256 | 1 + 1 file changed, 1 insertion(+) create mode 100644 test/ossfuzz/cases/clusterfuzz-testcase-minimized-parse_fuzz_test-5177383552352256 diff --git a/test/ossfuzz/cases/clusterfuzz-testcase-minimized-parse_fuzz_test-5177383552352256 b/test/ossfuzz/cases/clusterfuzz-testcase-minimized-parse_fuzz_test-5177383552352256 new file mode 100644 index 000000000000..1eba605f0071 --- /dev/null +++ b/test/ossfuzz/cases/clusterfuzz-testcase-minimized-parse_fuzz_test-5177383552352256 @@ -0,0 +1 @@ +SElECT-+-170141183460469231731687303715884105728 \ No newline at end of file From febcc869032027e5c61d0b768d1aec30da6c6622 Mon Sep 17 00:00:00 2001 From: Max Gabrielsson Date: Thu, 11 Apr 2024 11:30:27 +0200 Subject: [PATCH 325/603] add patch --- .github/config/out_of_tree_extensions.cmake | 1 + .github/patches/extensions/vss/vss_depenency.patch | 13 +++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 .github/patches/extensions/vss/vss_depenency.patch diff --git a/.github/config/out_of_tree_extensions.cmake b/.github/config/out_of_tree_extensions.cmake index 304583abb513..49e2cc6ead37 100644 --- a/.github/config/out_of_tree_extensions.cmake +++ b/.github/config/out_of_tree_extensions.cmake @@ -112,4 +112,5 @@ duckdb_extension_load(vss GIT_URL https://github.com/duckdb/duckdb_vss GIT_TAG f03fe75e16dd67e404d6325d4f901d6958504730 TEST_DIR test/sql + APPLY_PATCHES ) \ No newline at end of file diff --git a/.github/patches/extensions/vss/vss_depenency.patch b/.github/patches/extensions/vss/vss_depenency.patch new file mode 100644 index 000000000000..98cd7d1d3932 --- /dev/null +++ b/.github/patches/extensions/vss/vss_depenency.patch @@ -0,0 +1,13 @@ +diff --git a/src/hnsw/hnsw_index_scan.cpp b/src/hnsw/hnsw_index_scan.cpp +index d53d7ee..954b439 100644 +--- a/src/hnsw/hnsw_index_scan.cpp ++++ b/src/hnsw/hnsw_index_scan.cpp +@@ -104,7 +104,7 @@ static unique_ptr HNSWIndexScanStatistics(ClientContext &context + //------------------------------------------------------------------------- + // Dependency + //------------------------------------------------------------------------- +-void HNSWIndexScanDependency(DependencyList &entries, const FunctionData *bind_data_p) { ++void HNSWIndexScanDependency(LogicalDependencyList &entries, const FunctionData *bind_data_p) { + auto &bind_data = bind_data_p->Cast(); + entries.AddDependency(bind_data.table); + From 0db2708230604a637f01929f0d2bd7801dd4a651 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 11 Apr 2024 11:47:17 +0200 Subject: [PATCH 326/603] Fix for decimal round - use early-out correctly --- src/core_functions/scalar/math/numeric.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core_functions/scalar/math/numeric.cpp b/src/core_functions/scalar/math/numeric.cpp index 2594b2f96b54..40a2f520d48a 100644 --- a/src/core_functions/scalar/math/numeric.cpp +++ b/src/core_functions/scalar/math/numeric.cpp @@ -575,7 +575,7 @@ static void DecimalRoundNegativePrecisionFunction(DataChunk &input, ExpressionSt auto &info = func_expr.bind_info->Cast(); auto source_scale = DecimalType::GetScale(func_expr.children[0]->return_type); auto width = DecimalType::GetWidth(func_expr.children[0]->return_type); - if (info.target_scale <= -int32_t(width)) { + if (info.target_scale <= -int32_t(width - source_scale)) { // scale too big for width result.SetVectorType(VectorType::CONSTANT_VECTOR); result.SetValue(0, Value::INTEGER(0)); @@ -675,7 +675,7 @@ unique_ptr BindDecimalRoundPrecision(ClientContext &context, Scala } bound_function.arguments[0] = decimal_type; bound_function.return_type = LogicalType::DECIMAL(width, target_scale); - return make_uniq(target_scale); + return make_uniq(round_value); } ScalarFunctionSet RoundFun::GetFunctions() { From f8238ab335c65e49d7e709c4e500589102cf1c72 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 11 Apr 2024 11:54:05 +0200 Subject: [PATCH 327/603] add require skip reload --- test/optimizer/topn/complex_top_n.test | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/optimizer/topn/complex_top_n.test b/test/optimizer/topn/complex_top_n.test index 04d51fe117b3..8cc825736293 100644 --- a/test/optimizer/topn/complex_top_n.test +++ b/test/optimizer/topn/complex_top_n.test @@ -2,6 +2,8 @@ # description: topN # group: [topn] +require skip_reload + statement ok SELECT SETSEED(0.42); From 7099f6646272022f170909f25db383999dd410a6 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 11 Apr 2024 12:09:00 +0200 Subject: [PATCH 328/603] DynamicCastCheck to be done on const T --- src/include/duckdb/common/helper.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/include/duckdb/common/helper.hpp b/src/include/duckdb/common/helper.hpp index b19b85f6d851..d4c07cc47091 100644 --- a/src/include/duckdb/common/helper.hpp +++ b/src/include/duckdb/common/helper.hpp @@ -215,9 +215,9 @@ bool RefersToSameObject(const T &a, const T &b) { } template -void DynamicCastCheck(SRC *source) { +void DynamicCastCheck(const SRC *source) { #ifndef __APPLE__ - D_ASSERT(dynamic_cast(source)); + D_ASSERT(dynamic_cast(source)); #endif } From d110d7992e054130787544773bd29cd99f59a4c6 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 11 Apr 2024 12:09:59 +0200 Subject: [PATCH 329/603] Use DynamicCastCheck both for const and non-const Casts --- src/include/duckdb/catalog/catalog.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/duckdb/catalog/catalog.hpp b/src/include/duckdb/catalog/catalog.hpp index 871738a975de..31098ba10f6c 100644 --- a/src/include/duckdb/catalog/catalog.hpp +++ b/src/include/duckdb/catalog/catalog.hpp @@ -368,7 +368,7 @@ class Catalog { template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; From df14a380d307244e5e939451a02765c56f057840 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 11 Apr 2024 12:15:42 +0200 Subject: [PATCH 330/603] Out-of-range positional reference --- .../expression_binder/select_bind_state.cpp | 2 +- .../duckfuzz/order_by_positional_reference.test | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 test/fuzzer/duckfuzz/order_by_positional_reference.test diff --git a/src/planner/expression_binder/select_bind_state.cpp b/src/planner/expression_binder/select_bind_state.cpp index 4b0a19cd2e5b..23ada81d0451 100644 --- a/src/planner/expression_binder/select_bind_state.cpp +++ b/src/planner/expression_binder/select_bind_state.cpp @@ -43,7 +43,7 @@ void SelectBindState::AddRegularColumn() { } idx_t SelectBindState::GetFinalIndex(idx_t index) const { - if (expanded_column_indices.empty()) { + if (index >= expanded_column_indices.size()) { return index; } return expanded_column_indices[index]; diff --git a/test/fuzzer/duckfuzz/order_by_positional_reference.test b/test/fuzzer/duckfuzz/order_by_positional_reference.test new file mode 100644 index 000000000000..ec567b9a2536 --- /dev/null +++ b/test/fuzzer/duckfuzz/order_by_positional_reference.test @@ -0,0 +1,14 @@ +# name: test/fuzzer/duckfuzz/order_by_positional_reference.test +# description: Mix GROUP BY ALL and positional references +# group: [duckfuzz] + +statement ok +PRAGMA enable_verification + +statement ok +create table integers(c1 int, c2 int); + +statement error +SELECT c1, c2, NULL FROM integers ORDER BY #10 +---- +term out of range From 22bb67a06e19ed95876b0161f8deff676348dd92 Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 11 Apr 2024 12:17:30 +0200 Subject: [PATCH 331/603] make the python linter happy --- scripts/generate_extensions_function.py | 32 +++++++++++++++---------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/scripts/generate_extensions_function.py b/scripts/generate_extensions_function.py index dff1dfc9d640..c95fa95f4655 100644 --- a/scripts/generate_extensions_function.py +++ b/scripts/generate_extensions_function.py @@ -80,8 +80,9 @@ class ExtensionFunction(NamedTuple): name: str type: CatalogType + @staticmethod def create_map(input: List[Tuple[str, str, str]]) -> Dict[Function, "ExtensionFunction"]: - output: Dict[str, "ExtensionFunction"] = {} + output: Dict[Function, "ExtensionFunction"] = {} for x in input: key = Function(x[0], catalog_type_from_type(x[2])) output[key] = ExtensionFunction(x[1], key.name, key.type) @@ -92,6 +93,7 @@ class ExtensionSetting(NamedTuple): extension: str name: str + @staticmethod def create_map(input: List[Tuple[str, str]]) -> Dict[str, "ExtensionSetting"]: output: Dict[str, "ExtensionSetting"] = {} for x in input: @@ -103,6 +105,7 @@ class ExtensionCopyFunction(NamedTuple): extension: str name: str + @staticmethod def create_map(input: List[Tuple[str, str]]) -> Dict[str, "ExtensionCopyFunction"]: output: Dict[str, "ExtensionCopyFunction"] = {} for x in input: @@ -114,6 +117,7 @@ class ExtensionType(NamedTuple): extension: str name: str + @staticmethod def create_map(input: List[Tuple[str, str]]) -> Dict[str, "ExtensionType"]: output: Dict[str, "ExtensionType"] = {} for x in input: @@ -145,7 +149,7 @@ def get_extension_names() -> List[str]: return extension_names -def get_query(sql_query, load_query): +def get_query(sql_query, load_query) -> list: # Optionally perform a LOAD of an extension # Then perform a SQL query, fetch the output query = f'{DUCKDB_PATH} -csv -unsigned -c "{load_query}{sql_query}" ' @@ -171,7 +175,7 @@ def get_functions(load="") -> Set[Function]: return functions -def get_settings(load=""): +def get_settings(load="") -> Set[str]: GET_SETTINGS_QUERY = """ select distinct name @@ -192,12 +196,12 @@ def __init__(self): self.stored_functions: Dict[str, List[Function]] = { 'substrait': [ - Function("from_substrait", "table"), - Function("get_substrait", "table"), - Function("get_substrait_json", "table"), - Function("from_substrait_json", "table"), + Function("from_substrait", CatalogType.TABLE), + Function("get_substrait", CatalogType.TABLE), + Function("get_substrait_json", CatalogType.TABLE), + Function("from_substrait_json", CatalogType.TABLE), ], - 'arrow': [Function("scan_arrow_ipc", "table"), Function("to_arrow_ipc", "table")], + 'arrow': [Function("scan_arrow_ipc", CatalogType.TABLE), Function("to_arrow_ipc", CatalogType.TABLE)], 'spatial': [], } self.stored_settings: Dict[str, List[str]] = {'substrait': [], 'arrow': [], 'spatial': []} @@ -214,8 +218,8 @@ def add_extension(self, extension_name: str): print(f"Load {extension_name} at {extension_path}") load = f"LOAD '{extension_path}';" - extension_functions = get_functions(load) - extension_settings = get_settings(load) + extension_functions = list(get_functions(load)) + extension_settings = list(get_settings(load)) self.add_settings(extension_name, extension_settings) self.add_functions(extension_name, extension_functions) @@ -237,7 +241,7 @@ def add_settings(self, extension_name: str, settings_list: List[str]): extension_name = extension_name.lower() added_settings: Set[str] = set(settings_list) - self.base_settings - settings_to_add: Dict[str, str] = {} + settings_to_add: Dict[str, ExtensionSetting] = {} for setting in added_settings: setting_name = setting.lower() settings_to_add[setting_name] = ExtensionSetting(extension_name, setting_name) @@ -315,7 +319,7 @@ def get_slice_of_file(var_name, file_str): # Parses the extension_entries.hpp file def parse_extension_entries(file_path): - def parse_contents(input) -> tuple: + def parse_contents(input) -> list: # Split the string by comma and remove any leading or trailing spaces elements = input.split(",") # Strip any leading or trailing spaces and surrounding double quotes from each element @@ -330,24 +334,28 @@ def parse_contents(input) -> tuple: ext_functions_file_blob = get_slice_of_file("EXTENSION_FUNCTIONS", file_blob) res = pattern.findall(ext_functions_file_blob) res = [parse_contents(x) for x in res] + res = [(x[0], x[1], x[2]) for x in res] cur_function_map = ExtensionFunction.create_map(res) # Get the extension settings ext_settings_file_blob = get_slice_of_file("EXTENSION_SETTINGS", file_blob) res = pattern.findall(ext_settings_file_blob) res = [parse_contents(x) for x in res] + res = [(x[0], x[1]) for x in res] cur_settings_map = ExtensionSetting.create_map(res) # Get the extension types ext_copy_functions_blob = get_slice_of_file("EXTENSION_COPY_FUNCTIONS", file_blob) res = pattern.findall(ext_copy_functions_blob) res = [parse_contents(x) for x in res] + res = [(x[0], x[1]) for x in res] cur_copy_functions_map = ExtensionCopyFunction.create_map(res) # Get the extension types ext_types_file_blob = get_slice_of_file("EXTENSION_TYPES", file_blob) res = pattern.findall(ext_types_file_blob) res = [parse_contents(x) for x in res] + res = [(x[0], x[1]) for x in res] cur_types_map = ExtensionType.create_map(res) return { From 04baa831475b777f978d34cdca80529e8d27a869 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 11 Apr 2024 12:25:53 +0200 Subject: [PATCH 332/603] Correctly deal with impossible implicit casts from array -> list --- src/function/cast_rules.cpp | 6 +++++- test/sql/function/array/array_flatten.test | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 test/sql/function/array/array_flatten.test diff --git a/src/function/cast_rules.cpp b/src/function/cast_rules.cpp index 3dc1213c8c9b..98ca78b86655 100644 --- a/src/function/cast_rules.cpp +++ b/src/function/cast_rules.cpp @@ -386,8 +386,12 @@ int64_t CastRules::ImplicitCast(const LogicalType &from, const LogicalType &to) } if (from.id() == LogicalTypeId::ARRAY && to.id() == LogicalTypeId::LIST) { // Arrays can be cast to lists for the cost of casting the child type + auto child_cost = ImplicitCast(ArrayType::GetChildType(from), ListType::GetChildType(to)); + if (child_cost < 0) { + return -1; + } // add 1 because we prefer ARRAY->ARRAY casts over ARRAY->LIST casts - return ImplicitCast(ArrayType::GetChildType(from), ListType::GetChildType(to)) + 1; + return child_cost + 1; } if (from.id() == LogicalTypeId::LIST && (to.id() == LogicalTypeId::ARRAY && !ArrayType::IsAnySize(to))) { // Lists can be cast to arrays for the cost of casting the child type, if the target size is known diff --git a/test/sql/function/array/array_flatten.test b/test/sql/function/array/array_flatten.test new file mode 100644 index 000000000000..406fab2db72d --- /dev/null +++ b/test/sql/function/array/array_flatten.test @@ -0,0 +1,16 @@ +# name: test/sql/function/array/array_flatten.test +# description: Test array flatten function +# group: [array] + +statement ok +PRAGMA enable_verification + +statement error +select flatten(['a', 'b', 'c']::varchar[3]); +---- +No function matches the given name and argument types + +query I +select flatten([['a'], ['b'], ['c']]::varchar[1][3]); +---- +[a, b, c] From 79c1c4b682e4e5d1d811c2a37c619984c0ac1776 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 11 Apr 2024 12:33:55 +0200 Subject: [PATCH 333/603] Use ConstantVector::SetNull --- src/include/duckdb/core_functions/lambda_functions.hpp | 3 ++- test/sql/function/list/lambda_constant_null.test | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 test/sql/function/list/lambda_constant_null.test diff --git a/src/include/duckdb/core_functions/lambda_functions.hpp b/src/include/duckdb/core_functions/lambda_functions.hpp index f560bf4ba3c6..624e087dd46a 100644 --- a/src/include/duckdb/core_functions/lambda_functions.hpp +++ b/src/include/duckdb/core_functions/lambda_functions.hpp @@ -88,7 +88,8 @@ class LambdaFunctions { result_validity = &FlatVector::Validity(result); if (list_column.GetType().id() == LogicalTypeId::SQLNULL) { - result_validity->SetInvalid(0); + result.SetVectorType(VectorType::CONSTANT_VECTOR); + ConstantVector::SetNull(result, true); result_is_null = true; return; } diff --git a/test/sql/function/list/lambda_constant_null.test b/test/sql/function/list/lambda_constant_null.test new file mode 100644 index 000000000000..05b5885d44db --- /dev/null +++ b/test/sql/function/list/lambda_constant_null.test @@ -0,0 +1,8 @@ +# name: test/sql/function/list/lambda_constant_null.test +# description: Test constant NULL values in lambdas +# group: [list] + +statement error +select quantile(NULL, filter(NULL, (c103 -> 'babea54a-2261-4b0c-b14b-1d0e9b794e1a'))); +---- +QUANTILE parameter cannot be NULL From 08b17e8da297797fdac3d999ffb62b2ffa08ff6b Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 11 Apr 2024 12:42:51 +0200 Subject: [PATCH 334/603] More tests --- test/sql/copy/csv/csv_nullstr_list.test | 82 +++++++++++++++++++++++-- 1 file changed, 77 insertions(+), 5 deletions(-) diff --git a/test/sql/copy/csv/csv_nullstr_list.test b/test/sql/copy/csv/csv_nullstr_list.test index 95323622426d..5ab1ceda27fe 100644 --- a/test/sql/copy/csv/csv_nullstr_list.test +++ b/test/sql/copy/csv/csv_nullstr_list.test @@ -5,7 +5,6 @@ statement ok PRAGMA enable_verification - # Test List query III FROM read_csv('data/csv/null/multiple_nulls.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'name': 'VARCHAR', 'age': 'VARCHAR', 'height': 'VARCHAR'}, nullstr = ['','none','null']); @@ -22,6 +21,7 @@ Pedro 31 1.73 Mark NULL NULL Thijs 26 NULL +#allow_quoted_nulls = false query III FROM read_csv('data/csv/null/multiple_quoted_nulls.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'name': 'VARCHAR', 'age': 'VARCHAR', 'height': 'VARCHAR'}, nullstr = ['','none','null'], allow_quoted_nulls = false); ---- @@ -29,14 +29,86 @@ Pedro 31 1.73 Mark null (empty) Thijs 26 none -#allow_quoted_nulls # Test Null Strings equal to delim quote escape -# Test From Copy +statement error +FROM read_csv('data/csv/null/multiple_quoted_nulls.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'name': 'VARCHAR', 'age': 'VARCHAR', 'height': 'VARCHAR'}, nullstr = ['',',','null'], allow_quoted_nulls = false); +---- +DELIMITER must not appear in the NULL specification and vice versa +statement error +FROM read_csv('data/csv/null/multiple_quoted_nulls.csv', auto_detect=false, delim=',', quote='"', escape='\', new_line='\n', skip=0, header=true, columns={'name': 'VARCHAR', 'age': 'VARCHAR', 'height': 'VARCHAR'}, nullstr = ['','"','null'], allow_quoted_nulls = false); +---- +QUOTE must not appear in the NULL specification and vice versa -# Test where the last value is null over different strings +statement error +FROM read_csv('data/csv/null/multiple_quoted_nulls.csv', auto_detect=false, delim=',', quote='"', escape='\', new_line='\n', skip=0, header=true, columns={'name': 'VARCHAR', 'age': 'VARCHAR', 'height': 'VARCHAR'}, nullstr = ['','\','null'], allow_quoted_nulls = false); +---- +ESCAPE must not appear in the NULL specification and vice versa + +# What if we have repeated values in our nullstr list? +query III +FROM read_csv('data/csv/null/multiple_quoted_nulls.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'name': 'VARCHAR', 'age': 'VARCHAR', 'height': 'VARCHAR'}, nullstr = ['','none','null','','none','null'], allow_quoted_nulls = false); +---- +Pedro 31 1.73 +Mark null (empty) +Thijs 26 none + +# Test Copy To +statement ok +create table t (a integer); + +statement ok +insert into t values (0), (null) + +statement ok +COPY t TO '__TEST_DIR__/t_i.csv' (nullstr 'none') + +# Not accepted in copy to/from +statement error +COPY t TO '__TEST_DIR__/t_i.csv' (nullstr ['none', '']) +---- +syntax error at or near "[" + +statement error +COPY t FROM '__TEST_DIR__/t_i.csv' (nullstr ['none', '']) +---- +syntax error at or near "[" + +statement ok +COPY t FROM '__TEST_DIR__/t_i.csv' (nullstr 'none') + +# Test with force_not_null +query III +FROM read_csv('data/csv/null/multiple_nulls.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'name': 'VARCHAR', 'age': 'VARCHAR', 'height': 'VARCHAR'}, nullstr = ['','none','null'], force_not_null = ['height']); +---- +Pedro 31 1.73 +Mark NULL NULL +Thijs 26 NULL -# Check that write only supports ONE null string +query III +FROM read_csv('data/csv/null/multiple_nulls.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'name': 'VARCHAR', 'age': 'VARCHAR', 'height': 'VARCHAR'}, nullstr = ['','none','null'], force_not_null = ['age','height']); +---- +Pedro 31 1.73 +Mark NULL NULL +Thijs 26 NULL + + +# Test Quoted +query III +FROM read_csv('data/csv/null/multiple_quoted_nulls.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'name': 'VARCHAR', 'age': 'VARCHAR', 'height': 'VARCHAR'}, nullstr = ['','none','null'], force_not_null = ['height']); +---- +Pedro 31 1.73 +Mark NULL NULL +Thijs 26 NULL + +query III +FROM read_csv('data/csv/null/multiple_quoted_nulls.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'name': 'VARCHAR', 'age': 'VARCHAR', 'height': 'VARCHAR'}, nullstr = ['','none','null'], force_not_null = ['age','height']); +---- +Pedro 31 1.73 +Mark NULL NULL +Thijs 26 NULL + +# Test where the last value is null over different strings # Test white spaces \ No newline at end of file From 5c9658872056865febe5690acd841f750f8e70b5 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 11 Apr 2024 12:50:57 +0200 Subject: [PATCH 335/603] Shell fuzzer fixes - remove several problematic functions, and implement sqlite3_limit --- tools/shell/shell.c | 205 ------------------ .../sqlite3_api_wrapper.cpp | 25 ++- 2 files changed, 23 insertions(+), 207 deletions(-) diff --git a/tools/shell/shell.c b/tools/shell/shell.c index d228bee5ff98..90d975f8f2fc 100644 --- a/tools/shell/shell.c +++ b/tools/shell/shell.c @@ -1946,167 +1946,6 @@ static void sha3Func( sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); } -/* Compute a string using sqlite3_vsnprintf() with a maximum length -** of 50 bytes and add it to the hash. -*/ -static void hash_step_vformat( - SHA3Context *p, /* Add content to this context */ - const char *zFormat, - ... -){ - va_list ap; - int n; - char zBuf[50]; - va_start(ap, zFormat); - sqlite3_vsnprintf(sizeof(zBuf),zBuf,zFormat,ap); - va_end(ap); - n = (int)strlen(zBuf); - SHA3Update(p, (unsigned char*)zBuf, n); -} - -/* -** Implementation of the sha3_query(SQL,SIZE) function. -** -** This function compiles and runs the SQL statement(s) given in the -** argument. The results are hashed using a SIZE-bit SHA3. The default -** size is 256. -** -** The format of the byte stream that is hashed is summarized as follows: -** -** S: -** R -** N -** I -** F -** B: -** T: -** -** is the original SQL text for each statement run and is -** the size of that text. The SQL text is UTF-8. A single R character -** occurs before the start of each row. N means a NULL value. -** I mean an 8-byte little-endian integer . F is a floating point -** number with an 8-byte little-endian IEEE floating point value . -** B means blobs of bytes. T means text rendered as -** bytes of UTF-8. The and values are expressed as an ASCII -** text integers. -** -** For each SQL statement in the X input, there is one S segment. Each -** S segment is followed by zero or more R segments, one for each row in the -** result set. After each R, there are one or more N, I, F, B, or T segments, -** one for each column in the result set. Segments are concatentated directly -** with no delimiters of any kind. -*/ -static void sha3QueryFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - sqlite3 *db = sqlite3_context_db_handle(context); - const char *zSql = (const char*)sqlite3_value_text(argv[0]); - sqlite3_stmt *pStmt = 0; - int nCol; /* Number of columns in the result set */ - int i; /* Loop counter */ - int rc; - int n; - const char *z; - SHA3Context cx; - int iSize; - - if( argc==1 ){ - iSize = 256; - }else{ - iSize = sqlite3_value_int(argv[1]); - if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){ - sqlite3_result_error(context, "SHA3 size should be one of: 224 256 " - "384 512", -1); - return; - } - } - if( zSql==0 ) return; - SHA3Init(&cx, iSize); - while( zSql[0] ){ - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql); - if( rc ){ - char *zMsg = sqlite3_mprintf("error SQL statement [%s]: %s", - zSql, sqlite3_errmsg(db)); - sqlite3_finalize(pStmt); - sqlite3_result_error(context, zMsg, -1); - sqlite3_free(zMsg); - return; - } - if( !sqlite3_stmt_readonly(pStmt) ){ - char *zMsg = sqlite3_mprintf("non-query: [%s]", sqlite3_sql(pStmt)); - sqlite3_finalize(pStmt); - sqlite3_result_error(context, zMsg, -1); - sqlite3_free(zMsg); - return; - } - nCol = sqlite3_column_count(pStmt); - z = sqlite3_sql(pStmt); - n = (int)strlen(z); - hash_step_vformat(&cx,"S%d:",n); - SHA3Update(&cx,(unsigned char*)z,n); - - /* Compute a hash over the result of the query */ - while( SQLITE_ROW==sqlite3_step(pStmt) ){ - SHA3Update(&cx,(const unsigned char*)"R",1); - for(i=0; i=1; j--){ - x[j] = u & 0xff; - u >>= 8; - } - x[0] = 'I'; - SHA3Update(&cx, x, 9); - break; - } - case SQLITE_FLOAT: { - sqlite3_uint64 u; - int j; - unsigned char x[9]; - double r = sqlite3_column_double(pStmt,i); - memcpy(&u, &r, 8); - for(j=8; j>=1; j--){ - x[j] = u & 0xff; - u >>= 8; - } - x[0] = 'F'; - SHA3Update(&cx,x,9); - break; - } - case SQLITE_TEXT: { - int n2 = sqlite3_column_bytes(pStmt, i); - const unsigned char *z2 = sqlite3_column_text(pStmt, i); - hash_step_vformat(&cx,"T%d:",n2); - SHA3Update(&cx, z2, n2); - break; - } - case SQLITE_BLOB: { - int n2 = sqlite3_column_bytes(pStmt, i); - const unsigned char *z2 = sqlite3_column_blob(pStmt, i); - hash_step_vformat(&cx,"B%d:",n2); - SHA3Update(&cx, z2, n2); - break; - } - } - } - } - sqlite3_finalize(pStmt); - } - sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); -} - - #ifdef _WIN32 #endif @@ -2126,16 +1965,6 @@ int sqlite3_shathree_init( SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, 0, sha3Func, 0, 0); } - if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "sha3_query", 1, - SQLITE_UTF8 | SQLITE_DIRECTONLY, - 0, sha3QueryFunc, 0, 0); - } - if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "sha3_query", 2, - SQLITE_UTF8 | SQLITE_DIRECTONLY, - 0, sha3QueryFunc, 0, 0); - } return rc; } @@ -14081,36 +13910,6 @@ static unsigned char *readHexDb(ShellState *p, int *pnData){ } #endif /* SQLITE_ENABLE_DESERIALIZE */ -/* -** Scalar function "shell_int32". The first argument to this function -** must be a blob. The second a non-negative integer. This function -** reads and returns a 32-bit big-endian integer from byte -** offset (4*) of the blob. -*/ -static void shellInt32( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const unsigned char *pBlob; - int nBlob; - int iInt; - - UNUSED_PARAMETER(argc); - nBlob = sqlite3_value_bytes(argv[0]); - pBlob = (const unsigned char*)sqlite3_value_blob(argv[0]); - iInt = sqlite3_value_int(argv[1]); - - if( iInt>=0 && (iInt+1)*4<=nBlob ){ - const unsigned char *a = &pBlob[iInt*4]; - sqlite3_int64 iVal = ((sqlite3_int64)a[0]<<24) - + ((sqlite3_int64)a[1]<<16) - + ((sqlite3_int64)a[2]<< 8) - + ((sqlite3_int64)a[3]<< 0); - sqlite3_result_int64(context, iVal); - } -} - /* ** Scalar function "shell_idquote(X)" returns string X quoted as an identifier, ** using "..." with internal double-quote characters doubled. @@ -14288,8 +14087,6 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_fileio_init(p->db, 0, 0); sqlite3_shathree_init(p->db, 0, 0); sqlite3_completion_init(p->db, 0, 0); - sqlite3_uint_init(p->db, 0, 0); - sqlite3_decimal_init(p->db, 0, 0); #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) sqlite3_dbdata_init(p->db, 0, 0); #endif @@ -14305,8 +14102,6 @@ static void open_db(ShellState *p, int openFlags){ shellPutsFunc, 0, 0); sqlite3_create_function(p->db, "shell_escape_crnl", 1, SQLITE_UTF8, 0, shellEscapeCrnl, 0, 0); - sqlite3_create_function(p->db, "shell_int32", 2, SQLITE_UTF8, 0, - shellInt32, 0, 0); sqlite3_create_function(p->db, "shell_idquote", 1, SQLITE_UTF8, 0, shellIdQuote, 0, 0); #ifndef SQLITE_NOHAVE_SYSTEM diff --git a/tools/sqlite3_api_wrapper/sqlite3_api_wrapper.cpp b/tools/sqlite3_api_wrapper/sqlite3_api_wrapper.cpp index ad2cac292e6d..4d9a008e2457 100644 --- a/tools/sqlite3_api_wrapper/sqlite3_api_wrapper.cpp +++ b/tools/sqlite3_api_wrapper/sqlite3_api_wrapper.cpp @@ -1076,8 +1076,29 @@ int sqlite3_get_autocommit(sqlite3 *db) { } int sqlite3_limit(sqlite3 *, int id, int newVal) { - fprintf(stderr, "sqlite3_limit: unsupported.\n"); - return -1; + if (newVal >= 0) { + // attempting to set limit value + return SQLITE_OK; + } + switch (id) { + case SQLITE_LIMIT_LENGTH: + case SQLITE_LIMIT_SQL_LENGTH: + case SQLITE_LIMIT_COLUMN: + case SQLITE_LIMIT_LIKE_PATTERN_LENGTH: + return std::numeric_limits::max(); + case SQLITE_LIMIT_EXPR_DEPTH: + return 1000; + case SQLITE_LIMIT_FUNCTION_ARG: + case SQLITE_LIMIT_VARIABLE_NUMBER: + return 256; + case SQLITE_LIMIT_ATTACHED: + return 1000; + case SQLITE_LIMIT_WORKER_THREADS: + case SQLITE_LIMIT_TRIGGER_DEPTH: + return 0; + default: + return SQLITE_ERROR; + } } int sqlite3_stmt_readonly(sqlite3_stmt *pStmt) { From ad79569204a1f2f20562656d0863a26ad791b175 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Tue, 9 Apr 2024 17:35:44 -0400 Subject: [PATCH 336/603] pyodide build --- .github/workflows/Pyodide.yml | 99 +++++++++++++++++++ .gitignore | 4 + tools/pythonpkg/pyodide.md | 57 +++++++++++ tools/pythonpkg/setup.py | 10 ++ tools/pythonpkg/src/native/python_objects.cpp | 11 ++- 5 files changed, 180 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/Pyodide.yml create mode 100644 tools/pythonpkg/pyodide.md diff --git a/.github/workflows/Pyodide.yml b/.github/workflows/Pyodide.yml new file mode 100644 index 000000000000..290024c3c190 --- /dev/null +++ b/.github/workflows/Pyodide.yml @@ -0,0 +1,99 @@ +name: Pyodide +on: + workflow_call: + inputs: + override_git_describe: + type: string + git_ref: + type: string + skip_tests: + type: string + workflow_dispatch: + inputs: + override_git_describe: + type: string + git_ref: + type: string + skip_tests: + type: string + repository_dispatch: + push: + branches: + - "**" + - "!main" + - "!feature" + paths-ignore: + - "**" + - "!.github/workflows/Pyodide.yml" + + pull_request: + types: [opened, reopened, ready_for_review] + paths-ignore: + - "**" + - "!.github/workflows/Pyodide.yml" + +jobs: + build_pyodide: + name: Build pyodide wheel + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + version: + - python: "3.10" + pyodide-build: "0.22.1" + - python: "3.11" + pyodide-build: "0.25.1" + steps: + - uses: actions/checkout@v4 + with: + # fetch everything so that the version on the built wheel path is + # correct + fetch-depth: 0 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.version.python }} + + - run: pip install 'pyodide-build==${{ matrix.version.pyodide-build }}' 'pydantic<2' + + - name: get emscripten version + id: emscripten-version + run: | + echo "value=$(pyodide config get emscripten_version)" | tee -a "$GITHUB_OUTPUT" + + - uses: mymindstorm/setup-emsdk@v14 + with: + version: ${{ steps.emscripten-version.outputs.value }} + + - name: build wasm wheel + run: pyodide build --exports=whole_archive + working-directory: ./tools/pythonpkg + env: + DUCKDB_CUSTOM_PLATFORM: wasm_eh_pyodide + CFLAGS: "-fexceptions" + LDFLAGS: "-fexceptions" + + - name: smoke test duckdb on pyodide + run: | + pyodide venv .venv-pyodide + source .venv-pyodide/bin/activate + pip install ./tools/pythonpkg/dist/*.whl + + python -V + + python < + + + + + + + + +``` + +## Caveats + +Only Pythons 3.10 and 3.11 are supported right now, with 3.12 support on the way. + +Wheels are tied to a specific version of Pyodide. For example when using +Pyodide version 0.25.1, you must use the cp311-based wheel. + +Some functionality is known to not work, such as extension downloading. + +The default extensions (as well as the `httpfs` extension) that ship with +duckdb Python don't need to be `INSTALL`ed, but others, like `spatial`, won't +work because they cannot be downloaded in the pyodide runtime. diff --git a/tools/pythonpkg/setup.py b/tools/pythonpkg/setup.py index 41ee53f32b92..76ba6a2d13d0 100644 --- a/tools/pythonpkg/setup.py +++ b/tools/pythonpkg/setup.py @@ -121,8 +121,10 @@ class build_ext(CompilerLauncherMixin, _build_ext): extensions = ['parquet', 'icu', 'fts', 'tpch', 'json'] is_android = hasattr(sys, 'getandroidapilevel') +is_pyodide = 'PYODIDE' in os.environ use_jemalloc = ( not is_android + and not is_pyodide and platform.system() == 'Linux' and platform.architecture()[0] == '64bit' and platform.machine() == 'x86_64' @@ -183,12 +185,20 @@ def open_utf8(fpath, flags): define_macros = [('DUCKDB_PYTHON_LIB_NAME', lib_name)] +custom_platform = os.environ.get('DUCKDB_CUSTOM_PLATFORM') +if custom_platform is not None: + define_macros.append(('DUCKDB_CUSTOM_PLATFORM', custom_platform)) + if platform.system() == 'Darwin': toolchain_args.extend(['-stdlib=libc++', '-mmacosx-version-min=10.7']) if platform.system() == 'Windows': define_macros.extend([('DUCKDB_BUILD_LIBRARY', None), ('WIN32', None)]) +if is_pyodide: + # show more useful error messages in the browser + define_macros.append(('PYBIND11_DETAILED_ERROR_MESSAGES', None)) + if 'BUILD_HTTPFS' in os.environ: libraries += ['crypto', 'ssl'] extensions += ['httpfs'] diff --git a/tools/pythonpkg/src/native/python_objects.cpp b/tools/pythonpkg/src/native/python_objects.cpp index 44d0acca4bf7..5c19f2ae13e2 100644 --- a/tools/pythonpkg/src/native/python_objects.cpp +++ b/tools/pythonpkg/src/native/python_objects.cpp @@ -575,7 +575,16 @@ py::object PythonObject::FromValue(const Value &val, const LogicalType &type, auto &array_values = ArrayValue::GetChildren(val); auto array_size = ArrayType::GetSize(type); auto &child_type = ArrayType::GetChildType(type); - py::tuple arr(array_size); + + // do not remove the static cast here, it's required for building + // duckdb-python with Emscripten. + // + // without this cast, a static_assert fails in pybind11 + // because the return type of ArrayType::GetSize is idx_t, + // which is typedef'd to uint64_t and ssize_t is 4 bytes with Emscripten + // and pybind11 requires that the input be castable to ssize_t + py::tuple arr(static_cast(array_size)); + for (idx_t elem_idx = 0; elem_idx < array_size; elem_idx++) { arr[elem_idx] = FromValue(array_values[elem_idx], child_type, client_properties); } From 3e43b6bdfafce142e463c411e658fa7314d7bf77 Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 11 Apr 2024 13:04:23 +0200 Subject: [PATCH 337/603] let the generation be modular by default, only replace the extensions that were not build --- scripts/generate_extensions_function.py | 129 +++++++++++++----------- 1 file changed, 72 insertions(+), 57 deletions(-) diff --git a/scripts/generate_extensions_function.py b/scripts/generate_extensions_function.py index c95fa95f4655..316c50617e46 100644 --- a/scripts/generate_extensions_function.py +++ b/scripts/generate_extensions_function.py @@ -32,7 +32,6 @@ from enum import Enum - class CatalogType(str, Enum): SCALAR = "CatalogType::SCALAR_FUNCTION_ENTRY" TABLE = "CatalogType::TABLE_FUNCTION_ENTRY" @@ -125,6 +124,60 @@ def create_map(input: List[Tuple[str, str]]) -> Dict[str, "ExtensionType"]: return output +class ParsedEntries: + def __init__(self, file_path): + self.path = file_path + self.functions = {} + self.settings = {} + self.types = {} + self.copy_functions = {} + + def parse_contents(input) -> list: + # Split the string by comma and remove any leading or trailing spaces + elements = input.split(",") + # Strip any leading or trailing spaces and surrounding double quotes from each element + elements = [element.strip().strip('"') for element in elements] + return elements + + file = open(file_path, 'r') + pattern = re.compile("{(.*(?:, )?)}[,}\n]") + file_blob = file.read() + + # Get the extension functions + ext_functions_file_blob = get_slice_of_file("EXTENSION_FUNCTIONS", file_blob) + res = pattern.findall(ext_functions_file_blob) + res = [parse_contents(x) for x in res] + res = [(x[0], x[1], x[2]) for x in res] + self.functions = ExtensionFunction.create_map(res) + + # Get the extension settings + ext_settings_file_blob = get_slice_of_file("EXTENSION_SETTINGS", file_blob) + res = pattern.findall(ext_settings_file_blob) + res = [parse_contents(x) for x in res] + res = [(x[0], x[1]) for x in res] + self.settings = ExtensionSetting.create_map(res) + + # Get the extension types + ext_copy_functions_blob = get_slice_of_file("EXTENSION_COPY_FUNCTIONS", file_blob) + res = pattern.findall(ext_copy_functions_blob) + res = [parse_contents(x) for x in res] + res = [(x[0], x[1]) for x in res] + self.copy_functions = ExtensionCopyFunction.create_map(res) + + # Get the extension types + ext_types_file_blob = get_slice_of_file("EXTENSION_TYPES", file_blob) + res = pattern.findall(ext_types_file_blob) + res = [parse_contents(x) for x in res] + res = [(x[0], x[1]) for x in res] + self.types = ExtensionType.create_map(res) + + def filter_entries(self, extensions: List[str]): + self.functions = {k: v for k, v in self.functions.items() if v.extension not in extensions} + self.copy_functions = {k: v for k, v in self.copy_functions.items() if v.extension not in extensions} + self.settings = {k: v for k, v in self.settings.items() if v.extension not in extensions} + self.types = {k: v for k, v in self.types.items() if v.extension not in extensions} + + def check_prerequisites(): if not os.path.isfile(EXTENSIONS_PATH) or not os.path.isfile(DUCKDB_PATH): print( @@ -210,6 +263,10 @@ def set_base(self): self.base_functions: Set[Function] = get_functions() self.base_settings: Set[str] = get_settings() + def add_entries(self, entries: ParsedEntries): + self.function_map.update(entries.functions) + self.settings_map.update(entries.settings) + def add_extension(self, extension_name: str): if extension_name in self.extensions: # Perform a LOAD and add the added settings/functions @@ -259,19 +316,19 @@ def add_functions(self, extension_name: str, function_list: List[Function]): self.function_map.update(functions_to_add) def validate(self): - parsed_entries = parse_extension_entries(HEADER_PATH) - if self.function_map != parsed_entries['functions']: + parsed_entries = ParsedEntries(HEADER_PATH) + if self.function_map != parsed_entries.functions: print("Function map mismatches:") - print_map_diff(self.function_map, parsed_entries['functions']) + print_map_diff(self.function_map, parsed_entries.functions) exit(1) - if self.settings_map != parsed_entries['settings']: + if self.settings_map != parsed_entries.settings: print("Settings map mismatches:") - print_map_diff(self.settings_map, parsed_entries['settings']) + print_map_diff(self.settings_map, parsed_entries.settings) exit(1) print("All entries found: ") - print(" > functions: " + str(len(parsed_entries['functions']))) - print(" > settings: " + str(len(parsed_entries['settings']))) + print(" > functions: " + str(len(parsed_entries.functions))) + print(" > settings: " + str(len(parsed_entries.settings))) def verify_export(self): if len(self.function_map) == 0 or len(self.settings_map) == 0: @@ -317,55 +374,6 @@ def get_slice_of_file(var_name, file_str): return file_str[begin:end] -# Parses the extension_entries.hpp file -def parse_extension_entries(file_path): - def parse_contents(input) -> list: - # Split the string by comma and remove any leading or trailing spaces - elements = input.split(",") - # Strip any leading or trailing spaces and surrounding double quotes from each element - elements = [element.strip().strip('"') for element in elements] - return elements - - file = open(file_path, 'r') - pattern = re.compile("{(.*(?:, )?)}[,}\n]") - file_blob = file.read() - - # Get the extension functions - ext_functions_file_blob = get_slice_of_file("EXTENSION_FUNCTIONS", file_blob) - res = pattern.findall(ext_functions_file_blob) - res = [parse_contents(x) for x in res] - res = [(x[0], x[1], x[2]) for x in res] - cur_function_map = ExtensionFunction.create_map(res) - - # Get the extension settings - ext_settings_file_blob = get_slice_of_file("EXTENSION_SETTINGS", file_blob) - res = pattern.findall(ext_settings_file_blob) - res = [parse_contents(x) for x in res] - res = [(x[0], x[1]) for x in res] - cur_settings_map = ExtensionSetting.create_map(res) - - # Get the extension types - ext_copy_functions_blob = get_slice_of_file("EXTENSION_COPY_FUNCTIONS", file_blob) - res = pattern.findall(ext_copy_functions_blob) - res = [parse_contents(x) for x in res] - res = [(x[0], x[1]) for x in res] - cur_copy_functions_map = ExtensionCopyFunction.create_map(res) - - # Get the extension types - ext_types_file_blob = get_slice_of_file("EXTENSION_TYPES", file_blob) - res = pattern.findall(ext_types_file_blob) - res = [parse_contents(x) for x in res] - res = [(x[0], x[1]) for x in res] - cur_types_map = ExtensionType.create_map(res) - - return { - 'functions': cur_function_map, - 'settings': cur_settings_map, - 'types': cur_types_map, - 'copy_functions': cur_copy_functions_map, - } - - def print_map_diff(d1, d2): s1 = sorted(set(d1.items())) s2 = sorted(set(d2.items())) @@ -551,11 +559,18 @@ def main(): # Collect the list of functions/settings without any extensions loaded extension_data.set_base() + # TODO: add 'purge' option to ignore existing entries ?? + parsed_entries = ParsedEntries(HEADER_PATH) + parsed_entries.filter_entries(extension_names) + for extension_name in extension_names: print(extension_name) # For every extension, add the functions/settings added by the extension extension_data.add_extension(extension_name) + # Add the entries we initially parsed from the HEADER_PATH + extension_data.add_entries(parsed_entries) + if args.validate: extension_data.validate() return From 4465345de395c12cfda6a5d831aa7c2e1c1186f0 Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 11 Apr 2024 13:32:52 +0200 Subject: [PATCH 338/603] statically link mbedtls into parquet, as it's required to load parquet dynamically --- extension/parquet/CMakeLists.txt | 1 + scripts/generate_extensions_function.py | 1 + 2 files changed, 2 insertions(+) diff --git a/extension/parquet/CMakeLists.txt b/extension/parquet/CMakeLists.txt index 718bb3af91ae..f4d415dbeeb9 100644 --- a/extension/parquet/CMakeLists.txt +++ b/extension/parquet/CMakeLists.txt @@ -66,6 +66,7 @@ endif() build_static_extension(parquet ${PARQUET_EXTENSION_FILES}) set(PARAMETERS "-warnings") build_loadable_extension(parquet ${PARAMETERS} ${PARQUET_EXTENSION_FILES}) +target_link_libraries(parquet_loadable_extension duckdb_mbedtls) install( TARGETS parquet_extension diff --git a/scripts/generate_extensions_function.py b/scripts/generate_extensions_function.py index 316c50617e46..be00d406eda0 100644 --- a/scripts/generate_extensions_function.py +++ b/scripts/generate_extensions_function.py @@ -32,6 +32,7 @@ from enum import Enum + class CatalogType(str, Enum): SCALAR = "CatalogType::SCALAR_FUNCTION_ENTRY" TABLE = "CatalogType::TABLE_FUNCTION_ENTRY" From cdec49e5fd1cc3f9889c29f2e05e537f5082da56 Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 11 Apr 2024 14:16:10 +0200 Subject: [PATCH 339/603] reformat after generation --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 903363ad5039..eb70c8bd5fda 100644 --- a/Makefile +++ b/Makefile @@ -445,6 +445,8 @@ generate-files: python3 scripts/generate_enum_util.py python3 tools/pythonpkg/scripts/generate_connection_code.py ./scripts/generate_micro_extended.sh +# Run the formatter again after (re)generating the files + $(MAKE) format-main bundle-library: release cd build/release && \ From 07352ec6cf13798a63032e7cdabf12411ab422ed Mon Sep 17 00:00:00 2001 From: Max Gabrielsson Date: Thu, 11 Apr 2024 14:20:32 +0200 Subject: [PATCH 340/603] add extension entries --- src/include/duckdb/main/extension_entries.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/include/duckdb/main/extension_entries.hpp b/src/include/duckdb/main/extension_entries.hpp index df6741df05a6..ac32aa4feb4d 100644 --- a/src/include/duckdb/main/extension_entries.hpp +++ b/src/include/duckdb/main/extension_entries.hpp @@ -47,6 +47,8 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = { {"fuzzyduck", "sqlsmith", CatalogType::TABLE_FUNCTION_ENTRY}, {"get_substrait", "substrait", CatalogType::TABLE_FUNCTION_ENTRY}, {"get_substrait_json", "substrait", CatalogType::TABLE_FUNCTION_ENTRY}, + {"hnsw_compact_index", "vss", CatalogType::PRAGMA_FUNCTION_ENTRY}, + {"hnsw_index_scan", "vss", CatalogType::TABLE_FUNCTION_ENTRY}, {"host", "inet", CatalogType::SCALAR_FUNCTION_ENTRY}, {"iceberg_metadata", "iceberg", CatalogType::TABLE_FUNCTION_ENTRY}, {"iceberg_scan", "iceberg", CatalogType::TABLE_FUNCTION_ENTRY}, @@ -92,6 +94,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = { {"postgres_query", "postgres_scanner", CatalogType::TABLE_FUNCTION_ENTRY}, {"postgres_scan", "postgres_scanner", CatalogType::TABLE_FUNCTION_ENTRY}, {"postgres_scan_pushdown", "postgres_scanner", CatalogType::TABLE_FUNCTION_ENTRY}, + {"pragma_hnsw_index_info", "vss", CatalogType::TABLE_FUNCTION_ENTRY}, {"read_json", "json", CatalogType::TABLE_FUNCTION_ENTRY}, {"read_json_auto", "json", CatalogType::TABLE_FUNCTION_ENTRY}, {"read_json_objects", "json", CatalogType::TABLE_FUNCTION_ENTRY}, From f66fed072a8b964331d26d37b8ddc552936aadff Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 11 Apr 2024 14:24:22 +0200 Subject: [PATCH 341/603] the shared_ptr internals live in .ipp files so we can separate enable_shared_from_this, shared_ptr and weak_ptr --- scripts/package_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/package_build.py b/scripts/package_build.py index ac737e88c141..3499f02c3747 100644 --- a/scripts/package_build.py +++ b/scripts/package_build.py @@ -343,7 +343,7 @@ def generate_unity_builds(source_list, nsplits, linenumbers): unity_build = True # re-order the files in the unity build so that they follow the same order as the CMake scores = {} - filenames = [x[0] for x in re.findall('([a-zA-Z0-9_]+[.](cpp|cc|c|cxx))', text)] + filenames = [x[0] for x in re.findall('([a-zA-Z0-9_]+[.](cpp|ipp|cc|c|cxx))', text)] score = 0 for filename in filenames: scores[filename] = score From d6bcc62d239e892e3fba990a4ccb3e29d83cba03 Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 11 Apr 2024 14:25:58 +0200 Subject: [PATCH 342/603] changed in the wrong place --- scripts/amalgamation.py | 2 +- scripts/package_build.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/amalgamation.py b/scripts/amalgamation.py index e26c71049f31..325cc19f1521 100644 --- a/scripts/amalgamation.py +++ b/scripts/amalgamation.py @@ -376,7 +376,7 @@ def list_include_files_recursive(dname, file_list): fpath = os.path.join(dname, fname) if os.path.isdir(fpath): list_include_files_recursive(fpath, file_list) - elif fname.endswith(('.hpp', '.h', '.hh', '.tcc', '.inc')): + elif fname.endswith(('.hpp', '.ipp', '.h', '.hh', '.tcc', '.inc')): file_list.append(fpath) diff --git a/scripts/package_build.py b/scripts/package_build.py index 3499f02c3747..ac737e88c141 100644 --- a/scripts/package_build.py +++ b/scripts/package_build.py @@ -343,7 +343,7 @@ def generate_unity_builds(source_list, nsplits, linenumbers): unity_build = True # re-order the files in the unity build so that they follow the same order as the CMake scores = {} - filenames = [x[0] for x in re.findall('([a-zA-Z0-9_]+[.](cpp|ipp|cc|c|cxx))', text)] + filenames = [x[0] for x in re.findall('([a-zA-Z0-9_]+[.](cpp|cc|c|cxx))', text)] score = 0 for filename in filenames: scores[filename] = score From 80255fd67c9f125ad013a10666e8f2dd8e0b1bbe Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 11 Apr 2024 14:55:07 +0200 Subject: [PATCH 343/603] proper implement force_not_null --- .../csv_scanner/util/csv_reader_options.cpp | 14 +++++++++-- src/function/table/read_csv.cpp | 24 +++++++++++++++++- .../csv_scanner/csv_reader_options.hpp | 2 ++ test/sql/copy/csv/csv_nullstr_list.test | 25 +++++++++++-------- 4 files changed, 52 insertions(+), 13 deletions(-) diff --git a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp index b7fd88e7f306..62b1fe687cba 100644 --- a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp +++ b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp @@ -180,8 +180,6 @@ void CSVReaderOptions::SetReadOption(const string &loption, const Value &value, SetSkipRows(ParseInteger(value, loption)); } else if (loption == "max_line_size" || loption == "maximum_line_size") { maximum_line_size = ParseInteger(value, loption); - } else if (loption == "force_not_null") { - force_not_null = ParseColumnList(value, expected_names, loption); } else if (loption == "date_format" || loption == "dateformat") { string format = ParseString(value, loption); SetDateFormat(LogicalTypeId::DATE, format, true); @@ -206,6 +204,18 @@ void CSVReaderOptions::SetReadOption(const string &loption, const Value &value, parallel = ParseBoolean(value, loption); } else if (loption == "allow_quoted_nulls") { allow_quoted_nulls = ParseBoolean(value, loption); + } else if (loption == "force_not_null") { + if (!expected_names.empty()) { + force_not_null = ParseColumnList(value, expected_names, loption); + } else { + // Get the list of columns to use as a recovery key + auto &children = ListValue::GetChildren(value); + for (auto &child : children) { + auto col_name = child.GetValue(); + force_not_null_names.insert(col_name); + } + } + } else if (loption == "rejects_table") { // skip, handled in SetRejectsOptions auto table_name = ParseString(value, loption); diff --git a/src/function/table/read_csv.cpp b/src/function/table/read_csv.cpp index fc6819a4fc20..8b9483ee3dfc 100644 --- a/src/function/table/read_csv.cpp +++ b/src/function/table/read_csv.cpp @@ -142,7 +142,28 @@ static unique_ptr ReadCSVBind(ClientContext &context, TableFunctio } result->return_types = return_types; result->return_names = names; - + if (!options.force_not_null_names.empty()) { + // Lets first check all column names match + duckdb::unordered_set column_names; + for (auto &name : names) { + column_names.insert(name); + } + for (auto &force_name : options.force_not_null_names) { + if (column_names.find(force_name) == column_names.end()) { + throw BinderException( + "Column name %s specified in force_not_null parameter, does not exist in the CSV File.", + force_name); + } + } + D_ASSERT(options.force_not_null.empty()); + for (idx_t i = 0; i < names.size(); i++) { + if (options.force_not_null_names.find(names[i]) != options.force_not_null_names.end()) { + options.force_not_null.push_back(true); + } else { + options.force_not_null.push_back(false); + } + } + } result->FinalizeRead(context); return std::move(result); } @@ -252,6 +273,7 @@ void ReadCSVTableFunction::ReadCSVAddNamedParameters(TableFunction &table_functi table_function.named_parameters["rejects_table"] = LogicalType::VARCHAR; table_function.named_parameters["rejects_limit"] = LogicalType::BIGINT; table_function.named_parameters["rejects_recovery_columns"] = LogicalType::LIST(LogicalType::VARCHAR); + table_function.named_parameters["force_not_null"] = LogicalType::LIST(LogicalType::VARCHAR); table_function.named_parameters["buffer_size"] = LogicalType::UBIGINT; table_function.named_parameters["decimal_separator"] = LogicalType::VARCHAR; table_function.named_parameters["parallel"] = LogicalType::BOOLEAN; diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp index 78a9c062d7c6..b2c1e68b9a8b 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp @@ -85,6 +85,8 @@ struct CSVReaderOptions { //! Whether or not header names shall be normalized bool normalize_names = false; //! True, if column with that index must skip null check + unordered_set force_not_null_names; + //! True, if column with that index must skip null check vector force_not_null; //! Number of sample chunks used in auto-detection idx_t sample_size_chunks = 20480 / STANDARD_VECTOR_SIZE; diff --git a/test/sql/copy/csv/csv_nullstr_list.test b/test/sql/copy/csv/csv_nullstr_list.test index 5ab1ceda27fe..ac8922594ebe 100644 --- a/test/sql/copy/csv/csv_nullstr_list.test +++ b/test/sql/copy/csv/csv_nullstr_list.test @@ -83,31 +83,36 @@ query III FROM read_csv('data/csv/null/multiple_nulls.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'name': 'VARCHAR', 'age': 'VARCHAR', 'height': 'VARCHAR'}, nullstr = ['','none','null'], force_not_null = ['height']); ---- Pedro 31 1.73 -Mark NULL NULL -Thijs 26 NULL +Mark NULL (empty) +Thijs 26 (empty) query III FROM read_csv('data/csv/null/multiple_nulls.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'name': 'VARCHAR', 'age': 'VARCHAR', 'height': 'VARCHAR'}, nullstr = ['','none','null'], force_not_null = ['age','height']); ---- Pedro 31 1.73 -Mark NULL NULL -Thijs 26 NULL - +Mark (empty) (empty) +Thijs 26 (empty) # Test Quoted query III FROM read_csv('data/csv/null/multiple_quoted_nulls.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'name': 'VARCHAR', 'age': 'VARCHAR', 'height': 'VARCHAR'}, nullstr = ['','none','null'], force_not_null = ['height']); ---- Pedro 31 1.73 -Mark NULL NULL -Thijs 26 NULL +Mark NULL (empty) +Thijs 26 (empty) query III -FROM read_csv('data/csv/null/multiple_quoted_nulls.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'name': 'VARCHAR', 'age': 'VARCHAR', 'height': 'VARCHAR'}, nullstr = ['','none','null'], force_not_null = ['age','height']); +FROM read_csv('data/csv/null/multiple_quoted_nulls.csv', auto_detect=true, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, nullstr = ['','none','null'], force_not_null = ['age','height'], ALL_VARCHAR = 1); ---- Pedro 31 1.73 -Mark NULL NULL -Thijs 26 NULL +Mark (empty) (empty) +Thijs 26 (empty) + +# Test force_not_null fails for made-up column +statement error +FROM read_csv('data/csv/null/multiple_nulls.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'name': 'VARCHAR', 'age': 'VARCHAR', 'height': 'VARCHAR'}, nullstr = ['','none','null'], force_not_null = ['dont_exist']); +---- +Column name dont_exist specified in force_not_null parameter, does not exist in the CSV File. # Test where the last value is null over different strings From 453b32aaf518539c6a2b1449572d7dafb437b397 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 11 Apr 2024 15:28:22 +0200 Subject: [PATCH 344/603] array_length should return NULL when the input value is NULL for consistency --- src/function/scalar/string/length.cpp | 25 +++++++++++++++++-- test/sql/function/array/array_length.test | 30 ++++++++++++++++++++++- 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/src/function/scalar/string/length.cpp b/src/function/scalar/string/length.cpp index 218e4e84626d..dd88fd8d1441 100644 --- a/src/function/scalar/string/length.cpp +++ b/src/function/scalar/string/length.cpp @@ -70,7 +70,6 @@ static unique_ptr LengthPropagateStats(ClientContext &context, F //------------------------------------------------------------------ // ARRAY / LIST LENGTH //------------------------------------------------------------------ - static void ListLengthFunction(DataChunk &args, ExpressionState &state, Vector &result) { auto &input = args.data[0]; D_ASSERT(input.GetType().id() == LogicalTypeId::LIST); @@ -83,9 +82,31 @@ static void ListLengthFunction(DataChunk &args, ExpressionState &state, Vector & static void ArrayLengthFunction(DataChunk &args, ExpressionState &state, Vector &result) { auto &input = args.data[0]; - // If the input is an array, the length is constant + + UnifiedVectorFormat format; + args.data[0].ToUnifiedFormat(args.size(), format); + + // for arrays the length is constant result.SetVectorType(VectorType::CONSTANT_VECTOR); ConstantVector::GetData(result)[0] = static_cast(ArrayType::GetSize(input.GetType())); + + // but we do need to take null values into account + if (format.validity.AllValid()) { + // if there are no null values we can just return the constant + return; + } + // otherwise we flatten and inherit the null values of the parent + result.Flatten(args.size()); + auto &result_validity = FlatVector::Validity(result); + for (idx_t r = 0; r < args.size(); r++) { + auto idx = format.sel->get_index(r); + if (!format.validity.RowIsValid(idx)) { + result_validity.SetInvalid(r); + } + } + if (args.AllConstant()) { + result.SetVectorType(VectorType::CONSTANT_VECTOR); + } } static unique_ptr ArrayOrListLengthBind(ClientContext &context, ScalarFunction &bound_function, diff --git a/test/sql/function/array/array_length.test b/test/sql/function/array/array_length.test index 4c82583007f0..b82ec6a05e41 100644 --- a/test/sql/function/array/array_length.test +++ b/test/sql/function/array/array_length.test @@ -10,6 +10,35 @@ SELECT length(array_value(1, 2, 3)); ---- 3 +# array length for NULL values +statement ok +create table arrays(a int[3]); + +statement ok +insert into arrays values ([1, 2, 3]), ([4, 5, 6]) + +query I +select length(a) from arrays; +---- +3 +3 + +query I +select length(NULL::int[3]) from arrays; +---- +NULL +NULL + +statement ok +insert into arrays values (NULL); + +query I +select length(a) from arrays; +---- +3 +3 +NULL + # Array length with dimension argument query I SELECT array_length(array_value(array_value(1, 2, 2), array_value(3, 4, 3)), 1); @@ -26,7 +55,6 @@ SELECT array_length(array_value(array_value(1, 2, 2), array_value(3, 4, 3)), 3); ---- Out of Range Error: array_length dimension '3' out of range (min: '1', max: '2') - statement error SELECT array_length(array_value(array_value(1, 2, 2), array_value(3, 4, 3)), 0); ---- From 13e70f797a4744b9f8e4146869c161a39376370a Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 11 Apr 2024 15:38:03 +0200 Subject: [PATCH 345/603] Update fuzzer scripts --- scripts/fuzzer_helper.py | 89 ++++++++++++++++++++++------------------ scripts/reduce_sql.py | 21 +++------- scripts/run_fuzzer.py | 46 +++++++++------------ 3 files changed, 72 insertions(+), 84 deletions(-) diff --git a/scripts/fuzzer_helper.py b/scripts/fuzzer_helper.py index e36275e9b0c5..9d73f1985243 100644 --- a/scripts/fuzzer_helper.py +++ b/scripts/fuzzer_helper.py @@ -41,31 +41,24 @@ footer = ''' ```''' - -def get_github_hash(): - proc = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE) - return proc.stdout.read().decode('utf8').strip() - - # github stuff def issue_url(): return 'https://api.github.com/repos/%s/%s/issues' % (REPO_OWNER, REPO_NAME) - def create_session(): # Create an authenticated session to create the issue session = requests.Session() session.headers.update({'Authorization': 'token %s' % (TOKEN,)}) return session - def make_github_issue(title, body): if len(title) > 240: # avoid title is too long error (maximum is 256 characters) title = title[:240] + '...' session = create_session() url = issue_url() - issue = {'title': title, 'body': body} + issue = {'title': title, + 'body': body} r = session.post(url, json.dumps(issue)) if r.status_code == 201: print('Successfully created Issue "%s"' % title) @@ -74,10 +67,9 @@ def make_github_issue(title, body): print('Response:', r.content.decode('utf8')) raise Exception("Failed to create issue") - -def get_github_issues(): +def get_github_issues(page): session = create_session() - url = issue_url() + url = issue_url()+'?per_page=100&page='+str(page) r = session.get(url) if r.status_code != 200: print('Failed to get list of issues') @@ -85,7 +77,6 @@ def get_github_issues(): raise Exception("Failed to get list of issues") return json.loads(r.content.decode('utf8')) - def close_github_issue(number): session = create_session() url = issue_url() + '/' + str(number) @@ -98,73 +89,89 @@ def close_github_issue(number): print('Response:', r.content.decode('utf8')) raise Exception("Failed to close issue") +def label_github_issue(number, label): + session = create_session() + url = issue_url() + '/' + str(number) + params = {'labels': [label]} + r = session.patch(url, json.dumps(params)) + if r.status_code == 200: + print(f'Successfully labeled Issue "{number}"') + else: + print(f'Could not label Issue "{number}" (status code {r.status_code})') + print('Response:', r.content.decode('utf8')) + raise Exception("Failed to label issue") def extract_issue(body, nr): try: splits = body.split(middle) sql = splits[0].split(header)[1] - error = splits[1][: -len(footer)] + error = splits[1][:-len(footer)] return (sql, error) except: print(f"Failed to extract SQL/error message from issue {nr}") print(body) return None - def run_shell_command_batch(shell, cmd): command = [shell, '--batch', '-init', '/dev/null'] - res = subprocess.run(command, input=bytearray(cmd, 'utf8'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + res = subprocess.run(command, input=bytearray(cmd, 'utf8'), stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=300) + except subprocess.TimeoutExpired: + print(f"TIMEOUT... {cmd}") + return ("", "", 0, True) stdout = res.stdout.decode('utf8').strip() stderr = res.stderr.decode('utf8').strip() - return (stdout, stderr, res.returncode) - + return (stdout, stderr, res.returncode, False) -def test_reproducibility(shell, issue, current_errors): +def test_reproducibility(shell, issue, current_errors, perform_check): extract = extract_issue(issue['body'], issue['number']) + labels = issue['labels'] + label_timeout = False + for label in labels: + if label['name'] == 'timeout': + label_timeout = True if extract is None: # failed extract: leave the issue as-is return True sql = extract[0] + ';' error = extract[1] - (stdout, stderr, returncode) = run_shell_command_batch(shell, sql) - if returncode == 0: - return False - if not fuzzer_helper.is_internal_error(stderr): - return False + if perform_check is True and label_timeout is False: + print(f"Checking issue {issue['number']}...") + (stdout, stderr, returncode, is_timeout) = run_shell_command_batch(shell, sql) + if is_timeout: + label_github_issue(issue['number'], 'timeout') + else: + if returncode == 0: + return False + if not fuzzer_helper.is_internal_error(stderr): + return False # issue is still reproducible current_errors[error] = issue return True - -def extract_github_issues(shell): +def extract_github_issues(shell, perform_check): current_errors = dict() - issues = get_github_issues() - for issue in issues: - # check if the github issue is still reproducible - if not test_reproducibility(shell, issue, current_errors): - # the issue appears to be fixed - close the issue - print(f"Failed to reproduce issue {issue['number']}, closing...") - close_github_issue(int(issue['number'])) + for p in range(1,10): + issues = get_github_issues(p) + for issue in issues: + # check if the github issue is still reproducible + if not test_reproducibility(shell, issue, current_errors, perform_check): + # the issue appears to be fixed - close the issue + print(f"Failed to reproduce issue {issue['number']}, closing...") + close_github_issue(int(issue['number'])) return current_errors - def file_issue(cmd, error_msg, fuzzer, seed, hash): # issue is new, file it print("Filing new issue to Github") title = error_msg - body = ( - fuzzer_desc.replace("${FUZZER}", fuzzer) - .replace("${FULL_HASH}", hash) - .replace("${SHORT_HASH}", hash[:5]) - .replace("${SEED}", str(seed)) - ) + body = fuzzer_desc.replace("${FUZZER}", fuzzer).replace("${FULL_HASH}", hash).replace("${SHORT_HASH}", hash[:5]).replace("${SEED}", str(seed)) body += header + cmd + middle + error_msg + footer print(title, body) make_github_issue(title, body) - def is_internal_error(error): if 'differs from original result' in error: return True diff --git a/scripts/reduce_sql.py b/scripts/reduce_sql.py index e266fd1d1aa0..f81669a73391 100644 --- a/scripts/reduce_sql.py +++ b/scripts/reduce_sql.py @@ -11,9 +11,8 @@ SELECT * FROM reduce_sql_statement('${QUERY}'); ''' - def sanitize_error(err): - err = re.sub(r'Error: near line \d+: ', '', err) + err = re.sub('Error: near line \d+: ', '', err) err = err.replace(os.getcwd() + '/', '') err = err.replace(os.getcwd(), '') if 'AddressSanitizer' in err: @@ -21,7 +20,6 @@ def sanitize_error(err): err = 'AddressSanitizer error ' + match return err - def run_shell_command(shell, cmd): command = [shell, '-csv', '--batch', '-init', '/dev/null'] @@ -30,7 +28,6 @@ def run_shell_command(shell, cmd): stderr = res.stderr.decode('utf8').strip() return (stdout, stderr, res.returncode) - def get_reduced_sql(shell, sql_query): reduce_query = get_reduced_query.replace('${QUERY}', sql_query.replace("'", "''")) (stdout, stderr, returncode) = run_shell_command(shell, reduce_query) @@ -43,7 +40,6 @@ def get_reduced_sql(shell, sql_query): reduce_candidates.append(line.strip('"').replace('""', '"')) return reduce_candidates[1:] - def reduce(sql_query, data_load, shell, error_msg, max_time_seconds=300): start = time.time() while True: @@ -70,22 +66,18 @@ def reduce(sql_query, data_load, shell, error_msg, max_time_seconds=300): break return sql_query - def is_ddl_query(query): query = query.lower() if 'create' in query or 'insert' in query or 'update' in query or 'delete' in query: return True return False - def initial_cleanup(query_log): query_log = query_log.replace('SELECT * FROM pragma_version()\n', '') return query_log - def run_queries_until_crash_mp(queries, result_file): import duckdb - con = duckdb.connect() sqlite_con = sqlite3.connect(result_file) sqlite_con.execute('CREATE TABLE queries(id INT, text VARCHAR)') @@ -110,7 +102,7 @@ def run_queries_until_crash_mp(queries, result_file): keep_query = True sqlite_con.execute('UPDATE result SET text=?', (exception_error,)) if not keep_query: - sqlite_con.execute('DELETE FROM queries WHERE id=?', (id,)) + sqlite_con.execute('DELETE FROM queries WHERE id=?', (id, )) if is_internal_error: # found internal error: no need to try further queries break @@ -121,7 +113,6 @@ def run_queries_until_crash_mp(queries, result_file): sqlite_con.commit() sqlite_con.close() - def run_queries_until_crash(queries): sqlite_file = 'cleaned_queries.db' if os.path.isfile(sqlite_file): @@ -149,10 +140,8 @@ def cleanup_irrelevant_queries(query_log): queries = [x for x in query_log.split(';\n') if len(x) > 0] return run_queries_until_crash(queries) - # def reduce_internal(start, sql_query, data_load, queries_final, shell, error_msg, max_time_seconds=300): - def reduce_query_log_query(start, shell, queries, query_index, max_time_seconds): new_query_list = queries[:] sql_query = queries[query_index] @@ -184,7 +173,6 @@ def reduce_query_log_query(start, shell, queries, query_index, max_time_seconds) break return sql_query - def reduce_query_log(queries, shell, max_time_seconds=300): start = time.time() current_index = 0 @@ -195,7 +183,7 @@ def reduce_query_log(queries, shell, max_time_seconds=300): if current_time - start > max_time_seconds: break # remove the query at "current_index" - new_queries = queries[:current_index] + queries[current_index + 1 :] + new_queries = queries[:current_index] + queries[current_index + 1:] # try to run the queries and check if we still get the same error (new_queries_x, current_error) = run_queries_until_crash(new_queries) if current_error is None: @@ -215,6 +203,7 @@ def reduce_query_log(queries, shell, max_time_seconds=300): return queries + # Example usage: # error_msg = 'INTERNAL Error: Assertion triggered in file "/Users/myth/Programs/duckdb-bugfix/src/common/types/data_chunk.cpp" on line 41: !types.empty()' # shell = 'build/debug/duckdb' @@ -271,4 +260,4 @@ def reduce_query_log(queries, shell, max_time_seconds=300): # limit 88 # ''' # -# print(reduce(sql_query, data_load, shell, error_msg)) +# print(reduce(sql_query, data_load, shell, error_msg)) \ No newline at end of file diff --git a/scripts/run_fuzzer.py b/scripts/run_fuzzer.py index 86bf65537495..57f7a677b3fd 100644 --- a/scripts/run_fuzzer.py +++ b/scripts/run_fuzzer.py @@ -12,15 +12,22 @@ fuzzer = None db = None shell = None +perform_checks = True for param in sys.argv: if param == '--sqlsmith': fuzzer = 'sqlsmith' elif param == '--duckfuzz': fuzzer = 'duckfuzz' + elif param == '--duckfuzz_functions': + fuzzer = 'duckfuzz_functions' elif param == '--alltypes': db = 'alltypes' elif param == '--tpch': db = 'tpch' + elif param == '--emptyalltypes': + db = 'emptyalltypes' + elif param == '--no_checks': + perform_checks = False elif param.startswith('--shell='): shell = param.replace('--shell=', '') elif param.startswith('--seed='): @@ -31,7 +38,7 @@ exit(1) if db is None: - print("Unrecognized database to run on, expected either --tpch or --alltypes") + print("Unrecognized database to run on, expected either --tpch, --alltypes or --emptyalltypes") exit(1) if shell is None: @@ -41,18 +48,18 @@ if seed < 0: seed = random.randint(0, 2**30) -git_hash = fuzzer_helper.get_github_hash() - +git_hash = os.getenv('DUCKDB_HASH') def create_db_script(db): if db == 'alltypes': return 'create table all_types as select * exclude(small_enum, medium_enum, large_enum) from test_all_types();' elif db == 'tpch': return 'call dbgen(sf=0.1);' + elif db == 'emptyalltypes': + return 'create table all_types as select * exclude(small_enum, medium_enum, large_enum) from test_all_types() limit 0;' else: raise Exception("Unknown database creation script") - def run_fuzzer_script(fuzzer): if fuzzer == 'sqlsmith': return "call sqlsmith(max_queries=${MAX_QUERIES}, seed=${SEED}, verbose_output=1, log='${LAST_LOG_FILE}', complete_log='${COMPLETE_LOG_FILE}');" @@ -63,7 +70,6 @@ def run_fuzzer_script(fuzzer): else: raise Exception("Unknown fuzzer type") - def get_fuzzer_name(fuzzer): if fuzzer == 'sqlsmith': return 'SQLSmith' @@ -74,7 +80,6 @@ def get_fuzzer_name(fuzzer): else: return 'Unknown' - def run_shell_command(cmd): command = [shell, '--batch', '-init', '/dev/null'] @@ -85,27 +90,19 @@ def run_shell_command(cmd): # first get a list of all github issues, and check if we can still reproduce them -current_errors = fuzzer_helper.extract_github_issues(shell) +current_errors = fuzzer_helper.extract_github_issues(shell, perform_checks) -max_queries = 1000 +max_queries = 2000 last_query_log_file = 'sqlsmith.log' complete_log_file = 'sqlsmith.complete.log' -print( - f'''========================================== +print(f'''========================================== RUNNING {fuzzer} on {db} -==========================================''' -) +==========================================''') load_script = create_db_script(db) fuzzer_name = get_fuzzer_name(fuzzer) -fuzzer = ( - run_fuzzer_script(fuzzer) - .replace('${MAX_QUERIES}', str(max_queries)) - .replace('${LAST_LOG_FILE}', last_query_log_file) - .replace('${COMPLETE_LOG_FILE}', complete_log_file) - .replace('${SEED}', str(seed)) -) +fuzzer = run_fuzzer_script(fuzzer).replace('${MAX_QUERIES}', str(max_queries)).replace('${LAST_LOG_FILE}', last_query_log_file).replace('${COMPLETE_LOG_FILE}', complete_log_file).replace('${SEED}', str(seed)) print(load_script) print(fuzzer) @@ -116,11 +113,9 @@ def run_shell_command(cmd): (stdout, stderr, returncode) = run_shell_command(cmd) -print( - f'''========================================== +print(f'''========================================== FINISHED RUNNING -==========================================''' -) +==========================================''') print("============== STDOUT ================") print(stdout) print("============== STDERR =================") @@ -165,10 +160,7 @@ def run_shell_command(cmd): # check if this is a duplicate issue if error_msg in current_errors: print("Skip filing duplicate issue") - print( - "Issue already exists: https://github.com/duckdb/duckdb-fuzzer/issues/" - + str(current_errors[error_msg]['number']) - ) + print("Issue already exists: https://github.com/duckdb/duckdb-fuzzer/issues/" + str(current_errors[error_msg]['number'])) exit(0) print(last_query) From 22d120f941fe5b16f06b45ed8dea9ba41f4debcf Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 11 Apr 2024 15:49:44 +0200 Subject: [PATCH 346/603] Make it possible to use reduce_sql from the command line --- scripts/fuzzer_helper.py | 28 ++++++++++++++------------- scripts/reduce_sql.py | 42 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 55 insertions(+), 15 deletions(-) diff --git a/scripts/fuzzer_helper.py b/scripts/fuzzer_helper.py index 9d73f1985243..d457b3172d49 100644 --- a/scripts/fuzzer_helper.py +++ b/scripts/fuzzer_helper.py @@ -6,20 +6,8 @@ import reduce_sql import fuzzer_helper -if 'FUZZEROFDUCKSKEY' not in os.environ: - print("FUZZEROFDUCKSKEY not found in environment variables") - exit(1) USERNAME = 'fuzzerofducks' -TOKEN = os.environ['FUZZEROFDUCKSKEY'] - -if len(TOKEN) == 0: - print("FUZZEROFDUCKSKEY is set but is empty") - exit(1) - -if len(TOKEN) != 40: - print("Incorrect length for FUZZEROFDUCKSKEY") - exit(1) REPO_OWNER = 'duckdb' REPO_NAME = 'duckdb-fuzzer' @@ -45,10 +33,24 @@ def issue_url(): return 'https://api.github.com/repos/%s/%s/issues' % (REPO_OWNER, REPO_NAME) + +def get_token(): + if 'FUZZEROFDUCKSKEY' not in os.environ: + print("FUZZEROFDUCKSKEY not found in environment variables") + exit(1) + token = os.environ['FUZZEROFDUCKSKEY'] + if len(token) == 0: + print("FUZZEROFDUCKSKEY is set but is empty") + exit(1) + + if len(token) != 40: + print("Incorrect length for FUZZEROFDUCKSKEY") + exit(1) + return token def create_session(): # Create an authenticated session to create the issue session = requests.Session() - session.headers.update({'Authorization': 'token %s' % (TOKEN,)}) + session.headers.update({'Authorization': 'token %s' % (get_token(),)}) return session def make_github_issue(title, body): diff --git a/scripts/reduce_sql.py b/scripts/reduce_sql.py index f81669a73391..de30dea09872 100644 --- a/scripts/reduce_sql.py +++ b/scripts/reduce_sql.py @@ -6,13 +6,16 @@ import multiprocessing import sqlite3 -multiprocessing.set_start_method('fork') +try: + multiprocessing.set_start_method('fork') +except RuntimeError: + pass get_reduced_query = ''' SELECT * FROM reduce_sql_statement('${QUERY}'); ''' def sanitize_error(err): - err = re.sub('Error: near line \d+: ', '', err) + err = re.sub(r'Error: near line \d+: ', '', err) err = err.replace(os.getcwd() + '/', '') err = err.replace(os.getcwd(), '') if 'AddressSanitizer' in err: @@ -203,6 +206,41 @@ def reduce_query_log(queries, shell, max_time_seconds=300): return queries +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser(description='Reduce a problematic SQL query') + parser.add_argument('--shell', dest='shell', action='store', help='Path to the shell executable', default='build/debug/duckdb') + parser.add_argument('--load', dest='load', action='store', help='Path to the data load script', required=True) + parser.add_argument('--exec', dest='exec', action='store', help='Path to the executable script', required=True) + parser.add_argument('--inplace', dest='inplace', action='store_true', help='If true, overrides the exec script with the final query') + parser.add_argument('--max-time', dest='max_time', action='store', help='Maximum time in seconds to run the reducer', default=300) + + args = parser.parse_args() + print("Starting reduce process") + + shell = args.shell + data_load = open(args.load).read() + sql_query = open(args.exec).read() + (stdout, stderr, returncode) = run_shell_command(shell, data_load + sql_query) + expected_error = sanitize_error(stderr) + + print("===================================================") + print("Found expected error") + print("===================================================") + print(expected_error) + print("===================================================") + + + final_query = reduce(sql_query, data_load, shell, expected_error, args.max_time) + print("Found final reduced query") + print("===================================================") + print(final_query) + print("===================================================") + if args.inplace: + print(f"Writing to file {args.exec}") + with open(args.exec, 'w+') as f: + f.write(final_query) + # Example usage: # error_msg = 'INTERNAL Error: Assertion triggered in file "/Users/myth/Programs/duckdb-bugfix/src/common/types/data_chunk.cpp" on line 41: !types.empty()' From e23ed656532804fe0262d9de624af063a46a5030 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 11 Apr 2024 15:51:51 +0200 Subject: [PATCH 347/603] Add nullstr list on python api as well --- data/csv/null/mix_multiple_quoted_nulls.csv | 0 test/sql/copy/csv/csv_nullstr_list.test | 19 +++++++++++++++---- tools/pythonpkg/duckdb-stubs/__init__.pyi | 4 ++-- tools/pythonpkg/src/pyconnection.cpp | 17 ++++++++++++++--- .../pythonpkg/tests/fast/api/test_read_csv.py | 7 +++++++ 5 files changed, 38 insertions(+), 9 deletions(-) delete mode 100644 data/csv/null/mix_multiple_quoted_nulls.csv diff --git a/data/csv/null/mix_multiple_quoted_nulls.csv b/data/csv/null/mix_multiple_quoted_nulls.csv deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/test/sql/copy/csv/csv_nullstr_list.test b/test/sql/copy/csv/csv_nullstr_list.test index ac8922594ebe..3fe0de49bc1e 100644 --- a/test/sql/copy/csv/csv_nullstr_list.test +++ b/test/sql/copy/csv/csv_nullstr_list.test @@ -108,12 +108,23 @@ Pedro 31 1.73 Mark (empty) (empty) Thijs 26 (empty) +# Test with projection push-down +query I +select height FROM read_csv('data/csv/null/multiple_nulls.csv', delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, nullstr = ['','none','null']); +---- +1.73 +NULL +NULL + +query I +select age FROM read_csv('data/csv/null/multiple_nulls.csv', delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, nullstr = ['','none','null']); +---- +31 +NULL +26 + # Test force_not_null fails for made-up column statement error FROM read_csv('data/csv/null/multiple_nulls.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'name': 'VARCHAR', 'age': 'VARCHAR', 'height': 'VARCHAR'}, nullstr = ['','none','null'], force_not_null = ['dont_exist']); ---- Column name dont_exist specified in force_not_null parameter, does not exist in the CSV File. - -# Test where the last value is null over different strings - -# Test white spaces \ No newline at end of file diff --git a/tools/pythonpkg/duckdb-stubs/__init__.pyi b/tools/pythonpkg/duckdb-stubs/__init__.pyi index 8adf8ef208ac..1bd4c717d897 100644 --- a/tools/pythonpkg/duckdb-stubs/__init__.pyi +++ b/tools/pythonpkg/duckdb-stubs/__init__.pyi @@ -663,7 +663,7 @@ def read_csv( sep: Optional[str] = None, delimiter: Optional[str] = None, dtype: Optional[Dict[str, str] | List[str]] = None, - na_values: Optional[str] = None, + na_values: Optional[str | List[str]] = None, skiprows: Optional[int] = None, quotechar: Optional[str] = None, escapechar: Optional[str] = None, @@ -684,7 +684,7 @@ def from_csv_auto( sep: Optional[str] = None, delimiter: Optional[str] = None, dtype: Optional[Dict[str, str] | List[str]] = None, - na_values: Optional[str] = None, + na_values: Optional[str| List[str]] = None, skiprows: Optional[int] = None, quotechar: Optional[str] = None, escapechar: Optional[str] = None, diff --git a/tools/pythonpkg/src/pyconnection.cpp b/tools/pythonpkg/src/pyconnection.cpp index dd366334e71e..e532c5053ac5 100644 --- a/tools/pythonpkg/src/pyconnection.cpp +++ b/tools/pythonpkg/src/pyconnection.cpp @@ -885,10 +885,21 @@ unique_ptr DuckDBPyConnection::ReadCSV( } if (!py::none().is(na_values)) { - if (!py::isinstance(na_values)) { - throw InvalidInputException("read_csv only accepts 'na_values' as a string"); + vector null_values; + if (!py::isinstance(na_values) && !py::isinstance(na_values)) { + throw InvalidInputException("read_csv only accepts 'na_values' as a string or a list of strings"); + } else if (py::isinstance(na_values)) { + null_values.push_back(Value(py::str(na_values))); + } else { + py::list null_list = na_values; + for (auto &elem : null_list) { + if (!py::isinstance(elem)) { + throw InvalidInputException("read_csv 'na_values' list has to consist of only strings"); + } + null_values.push_back(Value(std::string(py::str(elem)))); + } } - bind_parameters["nullstr"] = Value(py::str(na_values)); + bind_parameters["nullstr"] = Value::LIST(LogicalType::VARCHAR, std::move(null_values)); } if (!py::none().is(skiprows)) { diff --git a/tools/pythonpkg/tests/fast/api/test_read_csv.py b/tools/pythonpkg/tests/fast/api/test_read_csv.py index 59c29ab71a02..49df1ac4f976 100644 --- a/tools/pythonpkg/tests/fast/api/test_read_csv.py +++ b/tools/pythonpkg/tests/fast/api/test_read_csv.py @@ -101,6 +101,13 @@ def test_na_values(self, duckdb_cursor): print(res) assert res == (1, None, datetime.datetime(2006, 2, 15, 4, 46, 27)) + def test_na_values_list(self, duckdb_cursor): + rel = duckdb_cursor.read_csv(TestFile('category.csv'), na_values=['Action', 'Animation']) + res = rel.fetchone() + assert res == (1, None, datetime.datetime(2006, 2, 15, 4, 46, 27)) + res = rel.fetchone() + assert res == (2, None, datetime.datetime(2006, 2, 15, 4, 46, 27)) + def test_skiprows(self, duckdb_cursor): rel = duckdb_cursor.read_csv(TestFile('category.csv'), skiprows=1) res = rel.fetchone() From 164a61d00d4911e4a388e4241515712d832e949f Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 11 Apr 2024 16:41:15 +0200 Subject: [PATCH 348/603] Expand statement simplifier to recurse correctly in more situations allowing for more complex queries to be simplified more --- .../sqlsmith/include/statement_simplifier.hpp | 12 +- extension/sqlsmith/statement_simplifier.cpp | 119 ++++++++++++------ scripts/reduce_sql.py | 4 + 3 files changed, 87 insertions(+), 48 deletions(-) diff --git a/extension/sqlsmith/include/statement_simplifier.hpp b/extension/sqlsmith/include/statement_simplifier.hpp index cda89a829dd0..2bdf2715cd18 100644 --- a/extension/sqlsmith/include/statement_simplifier.hpp +++ b/extension/sqlsmith/include/statement_simplifier.hpp @@ -50,16 +50,10 @@ class StatementSimplifier { template void SimplifyReplace(T &element, T &other); - template - void SimplifyListReplace(T &element, vector &list); - - template - void SimplifyListReplaceNull(vector &list); - template void SimplifyOptional(duckdb::unique_ptr &opt); - void Simplify(TableRef &ref); + void Simplify(unique_ptr &ref); void Simplify(SelectNode &node); void Simplify(SetOperationNode &node); @@ -69,6 +63,10 @@ class StatementSimplifier { void Simplify(OrderModifier &modifier); void SimplifyExpression(duckdb::unique_ptr &expr); + void SimplifyOptionalExpression(duckdb::unique_ptr &expr); + void SimplifyChildExpression(duckdb::unique_ptr &expr, unique_ptr &child); + void SimplifyExpressionList(duckdb::unique_ptr &expr, vector> &expression_list); + void SimplifyExpressionList(vector> &expression_list, bool is_optional = true); void Simplify(CommonTableExpressionMap &cte_map); void Simplify(UpdateSetInfo &info); diff --git a/extension/sqlsmith/statement_simplifier.cpp b/extension/sqlsmith/statement_simplifier.cpp index 05fd4de43288..edfd3437ee70 100644 --- a/extension/sqlsmith/statement_simplifier.cpp +++ b/extension/sqlsmith/statement_simplifier.cpp @@ -44,21 +44,6 @@ void StatementSimplifier::SimplifyList(vector &list, bool is_optional) { } } -template -void StatementSimplifier::SimplifyListReplaceNull(vector &list) { - for (idx_t i = 0; i < list.size(); i++) { - duckdb::unique_ptr constant = make_uniq(Value()); - SimplifyReplace(list[i], constant); - } -} - -template -void StatementSimplifier::SimplifyListReplace(T &element, vector &list) { - for (idx_t i = 0; i < list.size(); i++) { - SimplifyReplace(element, list[i]); - } -} - template void StatementSimplifier::SimplifyOptional(duckdb::unique_ptr &opt) { if (!opt) { @@ -69,21 +54,24 @@ void StatementSimplifier::SimplifyOptional(duckdb::unique_ptr &opt) { opt = std::move(n); } -void StatementSimplifier::Simplify(TableRef &ref) { - switch (ref.type) { +void StatementSimplifier::Simplify(unique_ptr &ref) { + switch (ref->type) { case TableReferenceType::SUBQUERY: { - auto &subquery = ref.Cast(); + auto &subquery = ref->Cast(); Simplify(*subquery.subquery->node); break; } case TableReferenceType::JOIN: { - auto &cp = ref.Cast(); - Simplify(*cp.left); - Simplify(*cp.right); + auto &cp = ref->Cast(); + Simplify(cp.left); + Simplify(cp.right); + SimplifyOptional(cp.condition); + SimplifyReplace(ref, cp.left); + SimplifyReplace(ref, cp.right); break; } case TableReferenceType::EXPRESSION_LIST: { - auto &expr_list = ref.Cast(); + auto &expr_list = ref->Cast(); if (expr_list.values.size() == 1) { SimplifyList(expr_list.values[0]); } else if (expr_list.values.size() > 1) { @@ -98,18 +86,18 @@ void StatementSimplifier::Simplify(TableRef &ref) { void StatementSimplifier::Simplify(SelectNode &node) { // simplify projection list - SimplifyList(node.select_list, false); + SimplifyExpressionList(node.select_list, false); // from clause SimplifyOptional(node.from_table); // simplify groups SimplifyList(node.groups.grouping_sets); // simplify filters - SimplifyOptional(node.where_clause); - SimplifyOptional(node.having); - SimplifyOptional(node.qualify); + SimplifyOptionalExpression(node.where_clause); + SimplifyOptionalExpression(node.having); + SimplifyOptionalExpression(node.qualify); SimplifyOptional(node.sample); - Simplify(*node.from_table); + Simplify(node.from_table); } void StatementSimplifier::Simplify(SetOperationNode &node) { @@ -154,13 +142,41 @@ void StatementSimplifier::Simplify(QueryNode &node) { SimplifyList(node.modifiers); } +void StatementSimplifier::SimplifyExpressionList(duckdb::unique_ptr &expr, vector> &expression_list) { + for(auto &child : expression_list) { + SimplifyChildExpression(expr, child); + } +} + +void StatementSimplifier::SimplifyExpressionList(vector> &expression_list, bool is_optional) { + SimplifyList(expression_list, is_optional); + for(auto &child : expression_list) { + SimplifyExpression(child); + } +} + +void StatementSimplifier::SimplifyChildExpression(duckdb::unique_ptr &expr, unique_ptr &child) { + if (!child) { + return; + } + SimplifyReplace(expr, child); + SimplifyExpression(child); +} + +void StatementSimplifier::SimplifyOptionalExpression(duckdb::unique_ptr &expr) { + if (!expr) { + return; + } + SimplifyOptional(expr); + SimplifyExpression(expr); +} + void StatementSimplifier::SimplifyExpression(duckdb::unique_ptr &expr) { if (!expr) { return; } auto expr_class = expr->GetExpressionClass(); switch (expr_class) { - case ExpressionClass::COLUMN_REF: case ExpressionClass::CONSTANT: return; default: @@ -171,37 +187,60 @@ void StatementSimplifier::SimplifyExpression(duckdb::unique_ptrCast(); - SimplifyListReplace(expr, conj.children); + SimplifyExpressionList(expr, conj.children); break; } case ExpressionClass::FUNCTION: { auto &func = expr->Cast(); - SimplifyListReplace(expr, func.children); - SimplifyListReplaceNull(func.children); + SimplifyExpressionList(expr, func.children); break; } case ExpressionClass::OPERATOR: { auto &op = expr->Cast(); - SimplifyListReplace(expr, op.children); + SimplifyExpressionList(expr, op.children); break; } case ExpressionClass::CASE: { auto &op = expr->Cast(); - SimplifyReplace(expr, op.else_expr); + SimplifyChildExpression(expr, op.else_expr); for (auto &case_check : op.case_checks) { - SimplifyReplace(expr, case_check.then_expr); - SimplifyReplace(expr, case_check.when_expr); + SimplifyChildExpression(expr, case_check.then_expr); + SimplifyChildExpression(expr, case_check.when_expr); } break; } case ExpressionClass::CAST: { auto &cast = expr->Cast(); - SimplifyReplace(expr, cast.child); + SimplifyChildExpression(expr, cast.child); break; } case ExpressionClass::COLLATE: { auto &collate = expr->Cast(); - SimplifyReplace(expr, collate.child); + SimplifyChildExpression(expr, collate.child); + break; + } + case ExpressionClass::SUBQUERY: { + auto &subq = expr->Cast(); + SimplifyChildExpression(expr, subq.child); + Simplify(*subq.subquery->node); + break; + } + case ExpressionClass::COMPARISON: { + auto &comp = expr->Cast(); + SimplifyChildExpression(expr, comp.left); + SimplifyChildExpression(expr, comp.right); + break; + } + case ExpressionClass::WINDOW: { + auto &window = expr->Cast(); + SimplifyExpressionList(expr, window.children); + SimplifyExpressionList(expr, window.partitions); + SimplifyList(window.orders); + SimplifyChildExpression(expr, window.filter_expr); + SimplifyChildExpression(expr, window.start_expr); + SimplifyChildExpression(expr, window.end_expr); + SimplifyChildExpression(expr, window.offset_expr); + SimplifyChildExpression(expr, window.default_expr); break; } default: @@ -212,7 +251,7 @@ void StatementSimplifier::SimplifyExpression(duckdb::unique_ptr()); break; default: break; @@ -267,9 +306,7 @@ void StatementSimplifier::Simplify(UpdateSetInfo &info) { void StatementSimplifier::Simplify(UpdateStatement &stmt) { Simplify(stmt.cte_map); - if (stmt.from_table) { - Simplify(*stmt.from_table); - } + SimplifyOptional(stmt.from_table); D_ASSERT(stmt.set_info); Simplify(*stmt.set_info); SimplifyList(stmt.returning_list); diff --git a/scripts/reduce_sql.py b/scripts/reduce_sql.py index de30dea09872..92a449b799b1 100644 --- a/scripts/reduce_sql.py +++ b/scripts/reduce_sql.py @@ -6,6 +6,10 @@ import multiprocessing import sqlite3 +# this script can be used as a library, but can also be directly called +# example usage: +# python3 scripts/reduce_sql.py --load load.sql --exec exec.sql + try: multiprocessing.set_start_method('fork') except RuntimeError: From f8f5387da9c5aa272a7d272a3974339ba5b6940e Mon Sep 17 00:00:00 2001 From: Joel Lubinitsky Date: Thu, 11 Apr 2024 10:51:48 -0400 Subject: [PATCH 349/603] remove catalog/schema filtering from adbc getobjects --- src/common/adbc/adbc.cpp | 74 ++-- test/api/adbc/test_adbc.cpp | 155 ++++++--- tools/pythonpkg/tests/fast/adbc/test_adbc.py | 336 ++++++++++++++++++- 3 files changed, 445 insertions(+), 120 deletions(-) diff --git a/src/common/adbc/adbc.cpp b/src/common/adbc/adbc.cpp index eb388dc0a305..65a147b08d3e 100644 --- a/src/common/adbc/adbc.cpp +++ b/src/common/adbc/adbc.cpp @@ -994,14 +994,6 @@ AdbcStatusCode ConnectionGetObjects(struct AdbcConnection *connection, int depth case ADBC_OBJECT_DEPTH_CATALOGS: // Return metadata on catalogs. query = duckdb::StringUtil::Format(R"( - WITH filtered_schemata AS ( - SELECT - catalog_name, - schema_name, - FROM - information_schema.schemata - WHERE catalog_name NOT IN ('system', 'temp') AND schema_name NOT IN ('information_schema', 'pg_catalog') - ) SELECT catalog_name, []::STRUCT( @@ -1039,7 +1031,7 @@ AdbcStatusCode ConnectionGetObjects(struct AdbcConnection *connection, int depth )[] )[] catalog_db_schemas FROM - filtered_schemata + information_schema.schemata WHERE catalog_name LIKE '%s' GROUP BY catalog_name )", @@ -1048,26 +1040,18 @@ AdbcStatusCode ConnectionGetObjects(struct AdbcConnection *connection, int depth case ADBC_OBJECT_DEPTH_DB_SCHEMAS: // Return metadata on catalogs and schemas. query = duckdb::StringUtil::Format(R"( - WITH filtered_schemata AS ( + WITH db_schemas AS ( SELECT catalog_name, schema_name, - FROM - information_schema.schemata - WHERE catalog_name NOT IN ('system', 'temp') AND schema_name NOT IN ('information_schema', 'pg_catalog') - ), - db_schemas AS ( - SELECT - * - FROM - filtered_schemata + FROM information_schema.schemata WHERE schema_name LIKE '%s' ) SELECT catalog_name, LIST({ - db_schema_name: dbs.schema_name, + db_schema_name: schema_name, db_schema_tables: []::STRUCT( table_name VARCHAR, table_type VARCHAR, @@ -1099,11 +1083,11 @@ AdbcStatusCode ConnectionGetObjects(struct AdbcConnection *connection, int depth constraint_column_usage STRUCT(fk_catalog VARCHAR, fk_db_schema VARCHAR, fk_table VARCHAR, fk_column_name VARCHAR)[] )[] )[], - }) FILTER (dbs.schema_name IS NOT null) AS catalog_db_schemas + }) FILTER (dbs.schema_name is not null) catalog_db_schemas FROM - filtered_schemata + information_schema.schemata LEFT JOIN db_schemas dbs - USING (catalog_name) + USING (catalog_name, schema_name) WHERE catalog_name LIKE '%s' GROUP BY catalog_name )", @@ -1112,15 +1096,7 @@ AdbcStatusCode ConnectionGetObjects(struct AdbcConnection *connection, int depth case ADBC_OBJECT_DEPTH_TABLES: // Return metadata on catalogs, schemas, and tables. query = duckdb::StringUtil::Format(R"( - WITH filtered_schemata AS ( - SELECT - catalog_name, - schema_name, - FROM - information_schema.schemata - WHERE catalog_name NOT IN ('system', 'temp') AND schema_name NOT IN ('information_schema', 'pg_catalog') - ), - tables AS ( + WITH tables AS ( SELECT table_catalog catalog_name, table_schema schema_name, @@ -1164,8 +1140,8 @@ AdbcStatusCode ConnectionGetObjects(struct AdbcConnection *connection, int depth catalog_name, schema_name, db_schema_tables, - FROM filtered_schemata fs - LEFT JOIN tables t + FROM information_schema.schemata + LEFT JOIN tables USING (catalog_name, schema_name) WHERE schema_name LIKE '%s' ) @@ -1173,13 +1149,13 @@ AdbcStatusCode ConnectionGetObjects(struct AdbcConnection *connection, int depth SELECT catalog_name, LIST({ - db_schema_name: dbs.schema_name, + db_schema_name: schema_name, db_schema_tables: db_schema_tables, - }) FILTER (dbs.schema_name is not null) AS catalog_db_schemas + }) FILTER (dbs.schema_name is not null) catalog_db_schemas FROM - filtered_schemata + information_schema.schemata LEFT JOIN db_schemas dbs - USING (catalog_name) + USING (catalog_name, schema_name) WHERE catalog_name LIKE '%s' GROUP BY catalog_name )", @@ -1188,15 +1164,7 @@ AdbcStatusCode ConnectionGetObjects(struct AdbcConnection *connection, int depth case ADBC_OBJECT_DEPTH_COLUMNS: // Return metadata on catalogs, schemas, tables, and columns. query = duckdb::StringUtil::Format(R"( - WITH filtered_schemata AS ( - SELECT - catalog_name, - schema_name, - FROM - information_schema.schemata - WHERE catalog_name NOT IN ('system', 'temp') AND schema_name NOT IN ('information_schema', 'pg_catalog') - ), - columns AS ( + WITH columns AS ( SELECT table_catalog, table_schema, @@ -1265,8 +1233,8 @@ AdbcStatusCode ConnectionGetObjects(struct AdbcConnection *connection, int depth catalog_name, schema_name, db_schema_tables, - FROM filtered_schemata fs - LEFT JOIN tables t + FROM information_schema.schemata + LEFT JOIN tables USING (catalog_name, schema_name) WHERE schema_name LIKE '%s' ) @@ -1274,13 +1242,13 @@ AdbcStatusCode ConnectionGetObjects(struct AdbcConnection *connection, int depth SELECT catalog_name, LIST({ - db_schema_name: dbs.schema_name, + db_schema_name: schema_name, db_schema_tables: db_schema_tables, - }) FILTER (dbs.schema_name is not null) AS catalog_db_schemas + }) FILTER (dbs.schema_name is not null) catalog_db_schemas FROM - filtered_schemata + information_schema.schemata LEFT JOIN db_schemas dbs - USING (catalog_name) + USING (catalog_name, schema_name) WHERE catalog_name LIKE '%s' GROUP BY catalog_name )", diff --git a/test/api/adbc/test_adbc.cpp b/test/api/adbc/test_adbc.cpp index fd840bced89f..1aeac5e272f8 100644 --- a/test/api/adbc/test_adbc.cpp +++ b/test/api/adbc/test_adbc.cpp @@ -1064,18 +1064,22 @@ TEST_CASE("Test AdbcConnectionGetObjects", "[adbc]") { AdbcConnectionGetObjects(&db.adbc_connection, ADBC_OBJECT_DEPTH_CATALOGS, nullptr, nullptr, nullptr, nullptr, nullptr, &arrow_stream, &adbc_error); db.CreateTable("result", arrow_stream); - auto res = db.Query("Select * from result"); + auto res = db.Query("Select * from result order by catalog_name asc"); REQUIRE(res->ColumnCount() == 2); - REQUIRE(res->RowCount() == 1); - REQUIRE(res->GetValue(0, 0).ToString() == "test_catalog_depth"); + REQUIRE(res->RowCount() == 3); + REQUIRE(res->GetValue(0, 0).ToString() == "system"); + REQUIRE(res->GetValue(0, 1).ToString() == "temp"); + REQUIRE(res->GetValue(0, 2).ToString() == "test_catalog_depth"); REQUIRE(res->GetValue(1, 0).ToString() == "[]"); + REQUIRE(res->GetValue(1, 1).ToString() == "[]"); + REQUIRE(res->GetValue(1, 2).ToString() == "[]"); db.Query("Drop table result;"); // Test Filters AdbcConnectionGetObjects(&db.adbc_connection, ADBC_OBJECT_DEPTH_CATALOGS, "bla", nullptr, nullptr, nullptr, nullptr, &arrow_stream, &adbc_error); db.CreateTable("result", arrow_stream); - res = db.Query("Select * from result"); + res = db.Query("Select * from result order by catalog_name asc"); REQUIRE(res->ColumnCount() == 2); REQUIRE(res->RowCount() == 0); db.Query("Drop table result;"); @@ -1095,25 +1099,39 @@ TEST_CASE("Test AdbcConnectionGetObjects", "[adbc]") { AdbcConnectionGetObjects(&db.adbc_connection, ADBC_OBJECT_DEPTH_DB_SCHEMAS, nullptr, nullptr, nullptr, nullptr, nullptr, &arrow_stream, &adbc_error); db.CreateTable("result", arrow_stream); - auto res = db.Query("Select * from result"); + auto res = db.Query("Select * from result order by catalog_name asc"); REQUIRE(res->ColumnCount() == 2); - REQUIRE(res->RowCount() == 1); + REQUIRE(res->RowCount() == 3); REQUIRE(res->GetValue(0, 0).ToString() == "ADBC_OBJECT_DEPTH_DB_SCHEMAS"); + REQUIRE(res->GetValue(0, 1).ToString() == "system"); + REQUIRE(res->GetValue(0, 2).ToString() == "temp"); string expected = R"([ + { + 'db_schema_name': information_schema, + 'db_schema_tables': [] + }, { 'db_schema_name': main, 'db_schema_tables': [] + }, + { + 'db_schema_name': pg_catalog, + 'db_schema_tables': [] } ])"; REQUIRE(StringUtil::Replace(res->GetValue(1, 0).ToString(), " ", "") == StringUtil::Replace(StringUtil::Replace(StringUtil::Replace(expected, "\n", ""), "\t", ""), " ", "")); + REQUIRE(StringUtil::Replace(res->GetValue(1, 1).ToString(), " ", "") == + StringUtil::Replace(StringUtil::Replace(StringUtil::Replace(expected, "\n", ""), "\t", ""), " ", "")); + REQUIRE(StringUtil::Replace(res->GetValue(1, 2).ToString(), " ", "") == + StringUtil::Replace(StringUtil::Replace(StringUtil::Replace(expected, "\n", ""), "\t", ""), " ", "")); db.Query("Drop table result;"); // Test Filters AdbcConnectionGetObjects(&db.adbc_connection, ADBC_OBJECT_DEPTH_DB_SCHEMAS, "bla", nullptr, nullptr, nullptr, nullptr, &arrow_stream, &adbc_error); db.CreateTable("result", arrow_stream); - res = db.Query("Select * from result"); + res = db.Query("Select * from result order by catalog_name asc"); REQUIRE(res->ColumnCount() == 2); REQUIRE(res->RowCount() == 0); db.Query("Drop table result;"); @@ -1121,10 +1139,12 @@ TEST_CASE("Test AdbcConnectionGetObjects", "[adbc]") { AdbcConnectionGetObjects(&db.adbc_connection, ADBC_OBJECT_DEPTH_DB_SCHEMAS, nullptr, "bla", nullptr, nullptr, nullptr, &arrow_stream, &adbc_error); db.CreateTable("result", arrow_stream); - res = db.Query("Select * from result"); + res = db.Query("Select * from result order by catalog_name asc"); REQUIRE(res->ColumnCount() == 2); - REQUIRE(res->RowCount() == 1); + REQUIRE(res->RowCount() == 3); REQUIRE(res->GetValue(1, 0).ToString() == "NULL"); + REQUIRE(res->GetValue(1, 1).ToString() == "NULL"); + REQUIRE(res->GetValue(1, 2).ToString() == "NULL"); db.Query("Drop table result;"); } // 3. Test ADBC_OBJECT_DEPTH_TABLES @@ -1141,11 +1161,17 @@ TEST_CASE("Test AdbcConnectionGetObjects", "[adbc]") { AdbcConnectionGetObjects(&db.adbc_connection, ADBC_OBJECT_DEPTH_TABLES, nullptr, nullptr, nullptr, nullptr, nullptr, &arrow_stream, &adbc_error); db.CreateTable("result", arrow_stream); - auto res = db.Query("Select * from result"); + auto res = db.Query("Select * from result order by catalog_name asc"); REQUIRE(res->ColumnCount() == 2); - REQUIRE(res->RowCount() == 1); - REQUIRE(res->GetValue(0, 0).ToString() == "test_table_depth"); + REQUIRE(res->RowCount() == 3); + REQUIRE(res->GetValue(0, 0).ToString() == "system"); + REQUIRE(res->GetValue(0, 1).ToString() == "temp"); + REQUIRE(res->GetValue(0, 2).ToString() == "test_table_depth"); string expected = R"([ + { + 'db_schema_name': information_schema, + 'db_schema_tables': NULL + }, { 'db_schema_name': main, 'db_schema_tables': [ @@ -1156,9 +1182,13 @@ TEST_CASE("Test AdbcConnectionGetObjects", "[adbc]") { 'table_constraints': [] } ] + }, + { + 'db_schema_name': pg_catalog, + 'db_schema_tables': NULL } ])"; - REQUIRE(StringUtil::Replace(res->GetValue(1, 0).ToString(), " ", "") == + REQUIRE(StringUtil::Replace(res->GetValue(1, 2).ToString(), " ", "") == StringUtil::Replace(StringUtil::Replace(StringUtil::Replace(expected, "\n", ""), "\t", ""), " ", "")); db.Query("Drop table result;"); @@ -1166,7 +1196,7 @@ TEST_CASE("Test AdbcConnectionGetObjects", "[adbc]") { AdbcConnectionGetObjects(&db.adbc_connection, ADBC_OBJECT_DEPTH_TABLES, "bla", nullptr, nullptr, nullptr, nullptr, &arrow_stream, &adbc_error); db.CreateTable("result", arrow_stream); - res = db.Query("Select * from result"); + res = db.Query("Select * from result order by catalog_name asc"); REQUIRE(res->ColumnCount() == 2); REQUIRE(res->RowCount() == 0); db.Query("Drop table result;"); @@ -1174,19 +1204,21 @@ TEST_CASE("Test AdbcConnectionGetObjects", "[adbc]") { AdbcConnectionGetObjects(&db.adbc_connection, ADBC_OBJECT_DEPTH_TABLES, nullptr, "bla", nullptr, nullptr, nullptr, &arrow_stream, &adbc_error); db.CreateTable("result", arrow_stream); - res = db.Query("Select * from result"); + res = db.Query("Select * from result order by catalog_name asc"); REQUIRE(res->ColumnCount() == 2); - REQUIRE(res->RowCount() == 1); + REQUIRE(res->RowCount() == 3); REQUIRE(res->GetValue(1, 0).ToString() == "NULL"); + REQUIRE(res->GetValue(1, 1).ToString() == "NULL"); + REQUIRE(res->GetValue(1, 2).ToString() == "NULL"); db.Query("Drop table result;"); AdbcConnectionGetObjects(&db.adbc_connection, ADBC_OBJECT_DEPTH_TABLES, nullptr, nullptr, "bla", nullptr, nullptr, &arrow_stream, &adbc_error); db.CreateTable("result", arrow_stream); - res = db.Query("Select * from result"); + res = db.Query("Select * from result order by catalog_name asc"); REQUIRE(res->ColumnCount() == 2); - REQUIRE(res->RowCount() == 1); - REQUIRE(res->GetValue(1, 0).ToString() == "[{'db_schema_name': main, 'db_schema_tables': NULL}]"); + REQUIRE(res->RowCount() == 3); + REQUIRE(res->GetValue(1, 2).ToString() == "[{'db_schema_name': information_schema, 'db_schema_tables': NULL}, {'db_schema_name': main, 'db_schema_tables': NULL}, {'db_schema_name': pg_catalog, 'db_schema_tables': NULL}]"); db.Query("Drop table result;"); } // 4.Test ADBC_OBJECT_DEPTH_COLUMNS @@ -1203,11 +1235,17 @@ TEST_CASE("Test AdbcConnectionGetObjects", "[adbc]") { AdbcConnectionGetObjects(&db.adbc_connection, ADBC_OBJECT_DEPTH_COLUMNS, nullptr, nullptr, nullptr, nullptr, nullptr, &arrow_stream, &adbc_error); db.CreateTable("result", arrow_stream); - auto res = db.Query("Select * from result"); + auto res = db.Query("Select * from result order by catalog_name asc"); REQUIRE(res->ColumnCount() == 2); - REQUIRE(res->RowCount() == 1); - REQUIRE(res->GetValue(0, 0).ToString() == "test_column_depth"); + REQUIRE(res->RowCount() == 3); + REQUIRE(res->GetValue(0, 0).ToString() == "system"); + REQUIRE(res->GetValue(0, 1).ToString() == "temp"); + REQUIRE(res->GetValue(0, 2).ToString() == "test_column_depth"); string expected = R"([ + { + 'db_schema_name': information_schema, + 'db_schema_tables': NULL + }, { 'db_schema_name': main, 'db_schema_tables': [ @@ -1240,9 +1278,13 @@ TEST_CASE("Test AdbcConnectionGetObjects", "[adbc]") { 'table_constraints': NULL } ] + }, + { + 'db_schema_name': pg_catalog, + 'db_schema_tables': NULL } ])"; - REQUIRE(StringUtil::Replace(res->GetValue(1, 0).ToString(), " ", "") == + REQUIRE(StringUtil::Replace(res->GetValue(1, 2).ToString(), " ", "") == StringUtil::Replace(StringUtil::Replace(StringUtil::Replace(expected, "\n", ""), "\t", ""), " ", "")); db.Query("Drop table result;"); @@ -1250,7 +1292,7 @@ TEST_CASE("Test AdbcConnectionGetObjects", "[adbc]") { AdbcConnectionGetObjects(&db.adbc_connection, ADBC_OBJECT_DEPTH_COLUMNS, "bla", nullptr, nullptr, nullptr, nullptr, &arrow_stream, &adbc_error); db.CreateTable("result", arrow_stream); - res = db.Query("Select * from result"); + res = db.Query("Select * from result order by catalog_name asc"); REQUIRE(res->ColumnCount() == 2); REQUIRE(res->RowCount() == 0); db.Query("Drop table result;"); @@ -1258,30 +1300,32 @@ TEST_CASE("Test AdbcConnectionGetObjects", "[adbc]") { AdbcConnectionGetObjects(&db.adbc_connection, ADBC_OBJECT_DEPTH_COLUMNS, nullptr, "bla", nullptr, nullptr, nullptr, &arrow_stream, &adbc_error); db.CreateTable("result", arrow_stream); - res = db.Query("Select * from result"); + res = db.Query("Select * from result order by catalog_name asc"); REQUIRE(res->ColumnCount() == 2); - REQUIRE(res->RowCount() == 1); + REQUIRE(res->RowCount() == 3); REQUIRE(res->GetValue(1, 0).ToString() == "NULL"); + REQUIRE(res->GetValue(1, 1).ToString() == "NULL"); + REQUIRE(res->GetValue(1, 2).ToString() == "NULL"); db.Query("Drop table result;"); AdbcConnectionGetObjects(&db.adbc_connection, ADBC_OBJECT_DEPTH_COLUMNS, nullptr, nullptr, "bla", nullptr, nullptr, &arrow_stream, &adbc_error); db.CreateTable("result", arrow_stream); - res = db.Query("Select * from result"); + res = db.Query("Select * from result order by catalog_name asc"); REQUIRE(res->ColumnCount() == 2); - REQUIRE(res->RowCount() == 1); - REQUIRE(res->GetValue(1, 0).ToString() == "[{'db_schema_name': main, 'db_schema_tables': NULL}]"); + REQUIRE(res->RowCount() == 3); + REQUIRE(res->GetValue(1, 2).ToString() == "[{'db_schema_name': information_schema, 'db_schema_tables': NULL}, {'db_schema_name': main, 'db_schema_tables': NULL}, {'db_schema_name': pg_catalog, 'db_schema_tables': NULL}]"); db.Query("Drop table result;"); AdbcConnectionGetObjects(&db.adbc_connection, ADBC_OBJECT_DEPTH_COLUMNS, nullptr, nullptr, nullptr, nullptr, "bla", &arrow_stream, &adbc_error); db.CreateTable("result", arrow_stream); - res = db.Query("Select * from result"); + res = db.Query("Select * from result order by catalog_name asc"); REQUIRE(res->ColumnCount() == 2); - REQUIRE(res->RowCount() == 1); - REQUIRE(res->GetValue(1, 0).ToString() == - "[{'db_schema_name': main, 'db_schema_tables': [{'table_name': my_table, 'table_type': BASE TABLE, " - "'table_columns': NULL, 'table_constraints': NULL}]}]"); + REQUIRE(res->RowCount() == 3); + REQUIRE(res->GetValue(1, 2).ToString() == + "[{'db_schema_name': information_schema, 'db_schema_tables': NULL}, {'db_schema_name': main, 'db_schema_tables': [{'table_name': my_table, 'table_type': BASE TABLE, " + "'table_columns': NULL, 'table_constraints': NULL}]}, {'db_schema_name': pg_catalog, 'db_schema_tables': NULL}]"); db.Query("Drop table result;"); } // 5.Test ADBC_OBJECT_DEPTH_ALL @@ -1298,11 +1342,16 @@ TEST_CASE("Test AdbcConnectionGetObjects", "[adbc]") { AdbcConnectionGetObjects(&db.adbc_connection, ADBC_OBJECT_DEPTH_ALL, nullptr, nullptr, nullptr, nullptr, nullptr, &arrow_stream, &adbc_error); db.CreateTable("result", arrow_stream); - auto res = db.Query("Select * from result"); - REQUIRE(res->ColumnCount() == 2); - REQUIRE(res->RowCount() == 1); - REQUIRE(res->GetValue(0, 0).ToString() == "test_all_depth"); + auto res = db.Query("Select * from result order by catalog_name asc"); + REQUIRE(res->RowCount() == 3); + REQUIRE(res->GetValue(0, 0).ToString() == "system"); + REQUIRE(res->GetValue(0, 1).ToString() == "temp"); + REQUIRE(res->GetValue(0, 2).ToString() == "test_all_depth"); string expected = R"([ + { + 'db_schema_name': information_schema, + 'db_schema_tables': NULL + }, { 'db_schema_name': main, 'db_schema_tables': [ @@ -1335,9 +1384,13 @@ TEST_CASE("Test AdbcConnectionGetObjects", "[adbc]") { 'table_constraints': NULL } ] + }, + { + 'db_schema_name': pg_catalog, + 'db_schema_tables': NULL } ])"; - REQUIRE(StringUtil::Replace(res->GetValue(1, 0).ToString(), " ", "") == + REQUIRE(StringUtil::Replace(res->GetValue(1, 2).ToString(), " ", "") == StringUtil::Replace(StringUtil::Replace(StringUtil::Replace(expected, "\n", ""), "\t", ""), " ", "")); db.Query("Drop table result;"); @@ -1345,7 +1398,7 @@ TEST_CASE("Test AdbcConnectionGetObjects", "[adbc]") { AdbcConnectionGetObjects(&db.adbc_connection, ADBC_OBJECT_DEPTH_COLUMNS, "bla", nullptr, nullptr, nullptr, nullptr, &arrow_stream, &adbc_error); db.CreateTable("result", arrow_stream); - res = db.Query("Select * from result"); + res = db.Query("Select * from result order by catalog_name asc"); REQUIRE(res->ColumnCount() == 2); REQUIRE(res->RowCount() == 0); db.Query("Drop table result;"); @@ -1353,30 +1406,32 @@ TEST_CASE("Test AdbcConnectionGetObjects", "[adbc]") { AdbcConnectionGetObjects(&db.adbc_connection, ADBC_OBJECT_DEPTH_COLUMNS, nullptr, "bla", nullptr, nullptr, nullptr, &arrow_stream, &adbc_error); db.CreateTable("result", arrow_stream); - res = db.Query("Select * from result"); + res = db.Query("Select * from result order by catalog_name asc"); REQUIRE(res->ColumnCount() == 2); - REQUIRE(res->RowCount() == 1); + REQUIRE(res->RowCount() == 3); REQUIRE(res->GetValue(1, 0).ToString() == "NULL"); + REQUIRE(res->GetValue(1, 1).ToString() == "NULL"); + REQUIRE(res->GetValue(1, 2).ToString() == "NULL"); db.Query("Drop table result;"); AdbcConnectionGetObjects(&db.adbc_connection, ADBC_OBJECT_DEPTH_COLUMNS, nullptr, nullptr, "bla", nullptr, nullptr, &arrow_stream, &adbc_error); db.CreateTable("result", arrow_stream); - res = db.Query("Select * from result"); + res = db.Query("Select * from result order by catalog_name asc"); REQUIRE(res->ColumnCount() == 2); - REQUIRE(res->RowCount() == 1); - REQUIRE(res->GetValue(1, 0).ToString() == "[{'db_schema_name': main, 'db_schema_tables': NULL}]"); + REQUIRE(res->RowCount() == 3); + REQUIRE(res->GetValue(1, 2).ToString() == "[{'db_schema_name': information_schema, 'db_schema_tables': NULL}, {'db_schema_name': main, 'db_schema_tables': NULL}, {'db_schema_name': pg_catalog, 'db_schema_tables': NULL}]"); db.Query("Drop table result;"); AdbcConnectionGetObjects(&db.adbc_connection, ADBC_OBJECT_DEPTH_COLUMNS, nullptr, nullptr, nullptr, nullptr, "bla", &arrow_stream, &adbc_error); db.CreateTable("result", arrow_stream); - res = db.Query("Select * from result"); + res = db.Query("Select * from result order by catalog_name asc"); REQUIRE(res->ColumnCount() == 2); - REQUIRE(res->RowCount() == 1); - REQUIRE(res->GetValue(1, 0).ToString() == - "[{'db_schema_name': main, 'db_schema_tables': [{'table_name': my_table, 'table_type': BASE TABLE, " - "'table_columns': NULL, 'table_constraints': NULL}]}]"); + REQUIRE(res->RowCount() == 3); + REQUIRE(res->GetValue(1, 2).ToString() == + "[{'db_schema_name': information_schema, 'db_schema_tables': NULL}, {'db_schema_name': main, 'db_schema_tables': [{'table_name': my_table, 'table_type': BASE TABLE, " + "'table_columns': NULL, 'table_constraints': NULL}]}, {'db_schema_name': pg_catalog, 'db_schema_tables': NULL}]"); db.Query("Drop table result;"); } // Now lets test some errors diff --git a/tools/pythonpkg/tests/fast/adbc/test_adbc.py b/tools/pythonpkg/tests/fast/adbc/test_adbc.py index e0e0ee3830b4..708e00a94d36 100644 --- a/tools/pythonpkg/tests/fast/adbc/test_adbc.py +++ b/tools/pythonpkg/tests/fast/adbc/test_adbc.py @@ -47,10 +47,48 @@ def test_connection_get_objects(duck_conn): with duck_conn.cursor() as cursor: cursor.execute("CREATE TABLE getobjects (ints BIGINT PRIMARY KEY)") depth_all = duck_conn.adbc_get_objects(depth="all").read_all() - assert depth_all.to_pylist() == [ + assert sorted_get_objects(depth_all.to_pylist()) == sorted_get_objects([ + { + 'catalog_name': 'system', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, + { + 'catalog_name': 'temp', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, { 'catalog_name': 'memory', 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, { 'db_schema_name': 'main', 'db_schema_tables': [ @@ -97,15 +135,61 @@ def test_connection_get_objects(duck_conn): } ], }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, ], } - ] + ]) depth_tables = duck_conn.adbc_get_objects(depth="tables").read_all() - assert depth_tables.to_pylist() == [ + assert sorted_get_objects(depth_tables.to_pylist()) == sorted_get_objects([ + { + 'catalog_name': 'system', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, + { + 'catalog_name': 'temp', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, { 'catalog_name': 'memory', 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, { 'db_schema_name': 'main', 'db_schema_tables': [ @@ -119,28 +203,78 @@ def test_connection_get_objects(duck_conn): }, ], } - ] + ]) depth_db_schemas = duck_conn.adbc_get_objects(depth="db_schemas").read_all() - assert depth_db_schemas.to_pylist() == [ + assert sorted_get_objects(depth_db_schemas.to_pylist()) == sorted_get_objects([ + { + 'catalog_name': 'system', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, + { + 'catalog_name': 'temp', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, { 'catalog_name': 'memory', 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, { 'db_schema_name': 'main', 'db_schema_tables': [], }, ], } - ] + ]) depth_catalogs = duck_conn.adbc_get_objects(depth="catalogs").read_all() - assert depth_catalogs.to_pylist() == [ + assert sorted_get_objects(depth_catalogs.to_pylist()) == sorted_get_objects([ + { + 'catalog_name': 'system', + 'catalog_db_schemas': [], + }, + { + 'catalog_name': 'temp', + 'catalog_db_schemas': [], + }, { 'catalog_name': 'memory', 'catalog_db_schemas': [], - } - ] + }, + ]) # All result schemas should be the same assert depth_all.schema == depth_tables.schema @@ -153,10 +287,48 @@ def test_connection_get_objects_filters(duck_conn): cursor.execute("CREATE TABLE getobjects (ints BIGINT PRIMARY KEY)") no_filter = duck_conn.adbc_get_objects(depth="all").read_all() - assert no_filter.to_pylist() == [ + assert sorted_get_objects(no_filter.to_pylist()) == sorted_get_objects([ + { + 'catalog_name': 'system', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, + { + 'catalog_name': 'temp', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, { 'catalog_name': 'memory', 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, { 'db_schema_name': 'main', 'db_schema_tables': [ @@ -203,15 +375,61 @@ def test_connection_get_objects_filters(duck_conn): } ], }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, ], } - ] + ]) column_filter = duck_conn.adbc_get_objects(depth="all", column_name_filter="notexist").read_all() - assert column_filter.to_pylist() == [ + assert sorted_get_objects(column_filter.to_pylist()) == sorted_get_objects([ + { + 'catalog_name': 'system', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, + { + 'catalog_name': 'temp', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, { 'catalog_name': 'memory', 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, { 'db_schema_name': 'main', 'db_schema_tables': [ @@ -238,28 +456,78 @@ def test_connection_get_objects_filters(duck_conn): }, ], } - ] + ]) table_name_filter = duck_conn.adbc_get_objects(depth="all", table_name_filter="notexist").read_all() - assert table_name_filter.to_pylist() == [ + assert sorted_get_objects(table_name_filter.to_pylist()) == sorted_get_objects([ + { + 'catalog_name': 'system', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, + { + 'catalog_name': 'temp', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, { 'catalog_name': 'memory', 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, { 'db_schema_name': 'main', 'db_schema_tables': None, }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, ], } - ] + ]) db_schema_filter = duck_conn.adbc_get_objects(depth="all", db_schema_filter="notexist").read_all() - assert db_schema_filter.to_pylist() == [ + assert sorted_get_objects(db_schema_filter.to_pylist()) == sorted_get_objects([ + { + 'catalog_name': 'system', + 'catalog_db_schemas': None, + }, + { + 'catalog_name': 'temp', + 'catalog_db_schemas': None, + }, { 'catalog_name': 'memory', 'catalog_db_schemas': None, } - ] + ]) catalog_filter = duck_conn.adbc_get_objects(depth="all", catalog_filter="notexist").read_all() assert catalog_filter.to_pylist() == [] @@ -450,3 +718,37 @@ def test_read(duck_conn): datetime.datetime(2006, 2, 15, 4, 46, 27), ], } + +def sorted_get_objects(catalogs): + res = [] + for catalog in sorted(catalogs, key=lambda cat: cat['catalog_name']): + new_catalog = { + "catalog_name": catalog['catalog_name'], + "catalog_db_schemas": [], + } + + for db_schema in sorted(catalog['catalog_db_schemas'] or [], key=lambda sch: sch['db_schema_name']): + new_db_schema = { + "db_schema_name": db_schema['db_schema_name'], + "db_schema_tables": [], + } + + for table in sorted(db_schema['db_schema_tables'] or [], key=lambda tab: tab['table_name']): + new_table = { + "table_name": table['table_name'], + "table_type": table['table_type'], + "table_columns": [], + "table_constraints": [], + } + + for column in sorted(table['table_columns'] or [], key=lambda col: col['ordinal_position']): + new_table["table_columns"].append(column) + + for constraint in sorted(table['table_constraints'] or [], key=lambda con: con['constraint_name']): + new_table["table_constraints"].append(constraint) + + new_db_schema["db_schema_tables"].append(new_table) + new_catalog["catalog_db_schemas"].append(new_db_schema) + res.append(new_catalog) + + return res \ No newline at end of file From 3ff7cb2a63f1f9d2bb2fd427beefab07674b29af Mon Sep 17 00:00:00 2001 From: Joel Lubinitsky Date: Thu, 11 Apr 2024 11:07:32 -0400 Subject: [PATCH 350/603] run formatter --- test/api/adbc/test_adbc.cpp | 24 +- tools/pythonpkg/tests/fast/adbc/test_adbc.py | 943 ++++++++++--------- 2 files changed, 497 insertions(+), 470 deletions(-) diff --git a/test/api/adbc/test_adbc.cpp b/test/api/adbc/test_adbc.cpp index 1aeac5e272f8..d612a43d7fc6 100644 --- a/test/api/adbc/test_adbc.cpp +++ b/test/api/adbc/test_adbc.cpp @@ -1218,7 +1218,9 @@ TEST_CASE("Test AdbcConnectionGetObjects", "[adbc]") { res = db.Query("Select * from result order by catalog_name asc"); REQUIRE(res->ColumnCount() == 2); REQUIRE(res->RowCount() == 3); - REQUIRE(res->GetValue(1, 2).ToString() == "[{'db_schema_name': information_schema, 'db_schema_tables': NULL}, {'db_schema_name': main, 'db_schema_tables': NULL}, {'db_schema_name': pg_catalog, 'db_schema_tables': NULL}]"); + REQUIRE(res->GetValue(1, 2).ToString() == + "[{'db_schema_name': information_schema, 'db_schema_tables': NULL}, {'db_schema_name': main, " + "'db_schema_tables': NULL}, {'db_schema_name': pg_catalog, 'db_schema_tables': NULL}]"); db.Query("Drop table result;"); } // 4.Test ADBC_OBJECT_DEPTH_COLUMNS @@ -1314,7 +1316,9 @@ TEST_CASE("Test AdbcConnectionGetObjects", "[adbc]") { res = db.Query("Select * from result order by catalog_name asc"); REQUIRE(res->ColumnCount() == 2); REQUIRE(res->RowCount() == 3); - REQUIRE(res->GetValue(1, 2).ToString() == "[{'db_schema_name': information_schema, 'db_schema_tables': NULL}, {'db_schema_name': main, 'db_schema_tables': NULL}, {'db_schema_name': pg_catalog, 'db_schema_tables': NULL}]"); + REQUIRE(res->GetValue(1, 2).ToString() == + "[{'db_schema_name': information_schema, 'db_schema_tables': NULL}, {'db_schema_name': main, " + "'db_schema_tables': NULL}, {'db_schema_name': pg_catalog, 'db_schema_tables': NULL}]"); db.Query("Drop table result;"); AdbcConnectionGetObjects(&db.adbc_connection, ADBC_OBJECT_DEPTH_COLUMNS, nullptr, nullptr, nullptr, nullptr, @@ -1324,8 +1328,10 @@ TEST_CASE("Test AdbcConnectionGetObjects", "[adbc]") { REQUIRE(res->ColumnCount() == 2); REQUIRE(res->RowCount() == 3); REQUIRE(res->GetValue(1, 2).ToString() == - "[{'db_schema_name': information_schema, 'db_schema_tables': NULL}, {'db_schema_name': main, 'db_schema_tables': [{'table_name': my_table, 'table_type': BASE TABLE, " - "'table_columns': NULL, 'table_constraints': NULL}]}, {'db_schema_name': pg_catalog, 'db_schema_tables': NULL}]"); + "[{'db_schema_name': information_schema, 'db_schema_tables': NULL}, {'db_schema_name': main, " + "'db_schema_tables': [{'table_name': my_table, 'table_type': BASE TABLE, " + "'table_columns': NULL, 'table_constraints': NULL}]}, {'db_schema_name': pg_catalog, " + "'db_schema_tables': NULL}]"); db.Query("Drop table result;"); } // 5.Test ADBC_OBJECT_DEPTH_ALL @@ -1420,7 +1426,9 @@ TEST_CASE("Test AdbcConnectionGetObjects", "[adbc]") { res = db.Query("Select * from result order by catalog_name asc"); REQUIRE(res->ColumnCount() == 2); REQUIRE(res->RowCount() == 3); - REQUIRE(res->GetValue(1, 2).ToString() == "[{'db_schema_name': information_schema, 'db_schema_tables': NULL}, {'db_schema_name': main, 'db_schema_tables': NULL}, {'db_schema_name': pg_catalog, 'db_schema_tables': NULL}]"); + REQUIRE(res->GetValue(1, 2).ToString() == + "[{'db_schema_name': information_schema, 'db_schema_tables': NULL}, {'db_schema_name': main, " + "'db_schema_tables': NULL}, {'db_schema_name': pg_catalog, 'db_schema_tables': NULL}]"); db.Query("Drop table result;"); AdbcConnectionGetObjects(&db.adbc_connection, ADBC_OBJECT_DEPTH_COLUMNS, nullptr, nullptr, nullptr, nullptr, @@ -1430,8 +1438,10 @@ TEST_CASE("Test AdbcConnectionGetObjects", "[adbc]") { REQUIRE(res->ColumnCount() == 2); REQUIRE(res->RowCount() == 3); REQUIRE(res->GetValue(1, 2).ToString() == - "[{'db_schema_name': information_schema, 'db_schema_tables': NULL}, {'db_schema_name': main, 'db_schema_tables': [{'table_name': my_table, 'table_type': BASE TABLE, " - "'table_columns': NULL, 'table_constraints': NULL}]}, {'db_schema_name': pg_catalog, 'db_schema_tables': NULL}]"); + "[{'db_schema_name': information_schema, 'db_schema_tables': NULL}, {'db_schema_name': main, " + "'db_schema_tables': [{'table_name': my_table, 'table_type': BASE TABLE, " + "'table_columns': NULL, 'table_constraints': NULL}]}, {'db_schema_name': pg_catalog, " + "'db_schema_tables': NULL}]"); db.Query("Drop table result;"); } // Now lets test some errors diff --git a/tools/pythonpkg/tests/fast/adbc/test_adbc.py b/tools/pythonpkg/tests/fast/adbc/test_adbc.py index 708e00a94d36..9b05345f0bb3 100644 --- a/tools/pythonpkg/tests/fast/adbc/test_adbc.py +++ b/tools/pythonpkg/tests/fast/adbc/test_adbc.py @@ -47,234 +47,242 @@ def test_connection_get_objects(duck_conn): with duck_conn.cursor() as cursor: cursor.execute("CREATE TABLE getobjects (ints BIGINT PRIMARY KEY)") depth_all = duck_conn.adbc_get_objects(depth="all").read_all() - assert sorted_get_objects(depth_all.to_pylist()) == sorted_get_objects([ - { - 'catalog_name': 'system', - 'catalog_db_schemas': [ - { - 'db_schema_name': 'information_schema', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'main', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'pg_catalog', - 'db_schema_tables': None, - }, - ], - }, - { - 'catalog_name': 'temp', - 'catalog_db_schemas': [ - { - 'db_schema_name': 'information_schema', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'main', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'pg_catalog', - 'db_schema_tables': None, - }, - ], - }, - { - 'catalog_name': 'memory', - 'catalog_db_schemas': [ - { - 'db_schema_name': 'information_schema', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'main', - 'db_schema_tables': [ - { - 'table_name': 'getobjects', - 'table_type': 'BASE TABLE', - 'table_columns': [ - { - 'column_name': 'ints', - 'ordinal_position': 1, - 'remarks': '', - 'xdbc_char_octet_length': None, - 'xdbc_column_def': None, - 'xdbc_column_size': None, - 'xdbc_data_type': None, - 'xdbc_datetime_sub': None, - 'xdbc_decimal_digits': None, - 'xdbc_is_autoincrement': None, - 'xdbc_is_generatedcolumn': None, - 'xdbc_is_nullable': None, - 'xdbc_nullable': None, - 'xdbc_num_prec_radix': None, - 'xdbc_scope_catalog': None, - 'xdbc_scope_schema': None, - 'xdbc_scope_table': None, - 'xdbc_sql_data_type': None, - 'xdbc_type_name': None, - }, - ], - 'table_constraints': [ - { - 'constraint_column_names': [], - 'constraint_column_usage': [], - 'constraint_name': 'getobjects_ints_pkey', - 'constraint_type': 'PRIMARY KEY', - }, - { - 'constraint_column_names': [], - 'constraint_column_usage': [], - 'constraint_name': 'getobjects_ints_not_null', - 'constraint_type': 'CHECK', - }, - ], - } - ], - }, - { - 'db_schema_name': 'pg_catalog', - 'db_schema_tables': None, - }, - ], - } - ]) + assert sorted_get_objects(depth_all.to_pylist()) == sorted_get_objects( + [ + { + 'catalog_name': 'system', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, + { + 'catalog_name': 'temp', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, + { + 'catalog_name': 'memory', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': [ + { + 'table_name': 'getobjects', + 'table_type': 'BASE TABLE', + 'table_columns': [ + { + 'column_name': 'ints', + 'ordinal_position': 1, + 'remarks': '', + 'xdbc_char_octet_length': None, + 'xdbc_column_def': None, + 'xdbc_column_size': None, + 'xdbc_data_type': None, + 'xdbc_datetime_sub': None, + 'xdbc_decimal_digits': None, + 'xdbc_is_autoincrement': None, + 'xdbc_is_generatedcolumn': None, + 'xdbc_is_nullable': None, + 'xdbc_nullable': None, + 'xdbc_num_prec_radix': None, + 'xdbc_scope_catalog': None, + 'xdbc_scope_schema': None, + 'xdbc_scope_table': None, + 'xdbc_sql_data_type': None, + 'xdbc_type_name': None, + }, + ], + 'table_constraints': [ + { + 'constraint_column_names': [], + 'constraint_column_usage': [], + 'constraint_name': 'getobjects_ints_pkey', + 'constraint_type': 'PRIMARY KEY', + }, + { + 'constraint_column_names': [], + 'constraint_column_usage': [], + 'constraint_name': 'getobjects_ints_not_null', + 'constraint_type': 'CHECK', + }, + ], + } + ], + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, + ] + ) depth_tables = duck_conn.adbc_get_objects(depth="tables").read_all() - assert sorted_get_objects(depth_tables.to_pylist()) == sorted_get_objects([ - { - 'catalog_name': 'system', - 'catalog_db_schemas': [ - { - 'db_schema_name': 'information_schema', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'main', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'pg_catalog', - 'db_schema_tables': None, - }, - ], - }, - { - 'catalog_name': 'temp', - 'catalog_db_schemas': [ - { - 'db_schema_name': 'information_schema', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'main', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'pg_catalog', - 'db_schema_tables': None, - }, - ], - }, - { - 'catalog_name': 'memory', - 'catalog_db_schemas': [ - { - 'db_schema_name': 'information_schema', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'pg_catalog', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'main', - 'db_schema_tables': [ - { - 'table_name': 'getobjects', - 'table_type': 'BASE TABLE', - 'table_columns': [], - 'table_constraints': [], - } - ], - }, - ], - } - ]) + assert sorted_get_objects(depth_tables.to_pylist()) == sorted_get_objects( + [ + { + 'catalog_name': 'system', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, + { + 'catalog_name': 'temp', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, + { + 'catalog_name': 'memory', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': [ + { + 'table_name': 'getobjects', + 'table_type': 'BASE TABLE', + 'table_columns': [], + 'table_constraints': [], + } + ], + }, + ], + }, + ] + ) depth_db_schemas = duck_conn.adbc_get_objects(depth="db_schemas").read_all() - assert sorted_get_objects(depth_db_schemas.to_pylist()) == sorted_get_objects([ - { - 'catalog_name': 'system', - 'catalog_db_schemas': [ - { - 'db_schema_name': 'information_schema', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'main', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'pg_catalog', - 'db_schema_tables': None, - }, - ], - }, - { - 'catalog_name': 'temp', - 'catalog_db_schemas': [ - { - 'db_schema_name': 'information_schema', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'main', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'pg_catalog', - 'db_schema_tables': None, - }, - ], - }, - { - 'catalog_name': 'memory', - 'catalog_db_schemas': [ - { - 'db_schema_name': 'information_schema', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'pg_catalog', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'main', - 'db_schema_tables': [], - }, - ], - } - ]) + assert sorted_get_objects(depth_db_schemas.to_pylist()) == sorted_get_objects( + [ + { + 'catalog_name': 'system', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, + { + 'catalog_name': 'temp', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, + { + 'catalog_name': 'memory', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': [], + }, + ], + }, + ] + ) depth_catalogs = duck_conn.adbc_get_objects(depth="catalogs").read_all() - assert sorted_get_objects(depth_catalogs.to_pylist()) == sorted_get_objects([ - { - 'catalog_name': 'system', - 'catalog_db_schemas': [], - }, - { - 'catalog_name': 'temp', - 'catalog_db_schemas': [], - }, - { - 'catalog_name': 'memory', - 'catalog_db_schemas': [], - }, - ]) + assert sorted_get_objects(depth_catalogs.to_pylist()) == sorted_get_objects( + [ + { + 'catalog_name': 'system', + 'catalog_db_schemas': [], + }, + { + 'catalog_name': 'temp', + 'catalog_db_schemas': [], + }, + { + 'catalog_name': 'memory', + 'catalog_db_schemas': [], + }, + ] + ) # All result schemas should be the same assert depth_all.schema == depth_tables.schema @@ -287,247 +295,255 @@ def test_connection_get_objects_filters(duck_conn): cursor.execute("CREATE TABLE getobjects (ints BIGINT PRIMARY KEY)") no_filter = duck_conn.adbc_get_objects(depth="all").read_all() - assert sorted_get_objects(no_filter.to_pylist()) == sorted_get_objects([ - { - 'catalog_name': 'system', - 'catalog_db_schemas': [ - { - 'db_schema_name': 'information_schema', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'main', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'pg_catalog', - 'db_schema_tables': None, - }, - ], - }, - { - 'catalog_name': 'temp', - 'catalog_db_schemas': [ - { - 'db_schema_name': 'information_schema', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'main', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'pg_catalog', - 'db_schema_tables': None, - }, - ], - }, - { - 'catalog_name': 'memory', - 'catalog_db_schemas': [ - { - 'db_schema_name': 'information_schema', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'main', - 'db_schema_tables': [ - { - 'table_name': 'getobjects', - 'table_type': 'BASE TABLE', - 'table_columns': [ - { - 'column_name': 'ints', - 'ordinal_position': 1, - 'remarks': '', - 'xdbc_char_octet_length': None, - 'xdbc_column_def': None, - 'xdbc_column_size': None, - 'xdbc_data_type': None, - 'xdbc_datetime_sub': None, - 'xdbc_decimal_digits': None, - 'xdbc_is_autoincrement': None, - 'xdbc_is_generatedcolumn': None, - 'xdbc_is_nullable': None, - 'xdbc_nullable': None, - 'xdbc_num_prec_radix': None, - 'xdbc_scope_catalog': None, - 'xdbc_scope_schema': None, - 'xdbc_scope_table': None, - 'xdbc_sql_data_type': None, - 'xdbc_type_name': None, - }, - ], - 'table_constraints': [ - { - 'constraint_column_names': [], - 'constraint_column_usage': [], - 'constraint_name': 'getobjects_ints_pkey', - 'constraint_type': 'PRIMARY KEY', - }, - { - 'constraint_column_names': [], - 'constraint_column_usage': [], - 'constraint_name': 'getobjects_ints_not_null', - 'constraint_type': 'CHECK', - }, - ], - } - ], - }, - { - 'db_schema_name': 'pg_catalog', - 'db_schema_tables': None, - }, - ], - } - ]) + assert sorted_get_objects(no_filter.to_pylist()) == sorted_get_objects( + [ + { + 'catalog_name': 'system', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, + { + 'catalog_name': 'temp', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, + { + 'catalog_name': 'memory', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': [ + { + 'table_name': 'getobjects', + 'table_type': 'BASE TABLE', + 'table_columns': [ + { + 'column_name': 'ints', + 'ordinal_position': 1, + 'remarks': '', + 'xdbc_char_octet_length': None, + 'xdbc_column_def': None, + 'xdbc_column_size': None, + 'xdbc_data_type': None, + 'xdbc_datetime_sub': None, + 'xdbc_decimal_digits': None, + 'xdbc_is_autoincrement': None, + 'xdbc_is_generatedcolumn': None, + 'xdbc_is_nullable': None, + 'xdbc_nullable': None, + 'xdbc_num_prec_radix': None, + 'xdbc_scope_catalog': None, + 'xdbc_scope_schema': None, + 'xdbc_scope_table': None, + 'xdbc_sql_data_type': None, + 'xdbc_type_name': None, + }, + ], + 'table_constraints': [ + { + 'constraint_column_names': [], + 'constraint_column_usage': [], + 'constraint_name': 'getobjects_ints_pkey', + 'constraint_type': 'PRIMARY KEY', + }, + { + 'constraint_column_names': [], + 'constraint_column_usage': [], + 'constraint_name': 'getobjects_ints_not_null', + 'constraint_type': 'CHECK', + }, + ], + } + ], + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, + ] + ) column_filter = duck_conn.adbc_get_objects(depth="all", column_name_filter="notexist").read_all() - assert sorted_get_objects(column_filter.to_pylist()) == sorted_get_objects([ - { - 'catalog_name': 'system', - 'catalog_db_schemas': [ - { - 'db_schema_name': 'information_schema', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'main', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'pg_catalog', - 'db_schema_tables': None, - }, - ], - }, - { - 'catalog_name': 'temp', - 'catalog_db_schemas': [ - { - 'db_schema_name': 'information_schema', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'main', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'pg_catalog', - 'db_schema_tables': None, - }, - ], - }, - { - 'catalog_name': 'memory', - 'catalog_db_schemas': [ - { - 'db_schema_name': 'information_schema', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'pg_catalog', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'main', - 'db_schema_tables': [ - { - 'table_name': 'getobjects', - 'table_type': 'BASE TABLE', - 'table_columns': None, - 'table_constraints': [ - { - 'constraint_column_names': [], - 'constraint_column_usage': [], - 'constraint_name': 'getobjects_ints_pkey', - 'constraint_type': 'PRIMARY KEY', - }, - { - 'constraint_column_names': [], - 'constraint_column_usage': [], - 'constraint_name': 'getobjects_ints_not_null', - 'constraint_type': 'CHECK', - }, - ], - } - ], - }, - ], - } - ]) + assert sorted_get_objects(column_filter.to_pylist()) == sorted_get_objects( + [ + { + 'catalog_name': 'system', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, + { + 'catalog_name': 'temp', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, + { + 'catalog_name': 'memory', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': [ + { + 'table_name': 'getobjects', + 'table_type': 'BASE TABLE', + 'table_columns': None, + 'table_constraints': [ + { + 'constraint_column_names': [], + 'constraint_column_usage': [], + 'constraint_name': 'getobjects_ints_pkey', + 'constraint_type': 'PRIMARY KEY', + }, + { + 'constraint_column_names': [], + 'constraint_column_usage': [], + 'constraint_name': 'getobjects_ints_not_null', + 'constraint_type': 'CHECK', + }, + ], + } + ], + }, + ], + }, + ] + ) table_name_filter = duck_conn.adbc_get_objects(depth="all", table_name_filter="notexist").read_all() - assert sorted_get_objects(table_name_filter.to_pylist()) == sorted_get_objects([ - { - 'catalog_name': 'system', - 'catalog_db_schemas': [ - { - 'db_schema_name': 'information_schema', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'main', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'pg_catalog', - 'db_schema_tables': None, - }, - ], - }, - { - 'catalog_name': 'temp', - 'catalog_db_schemas': [ - { - 'db_schema_name': 'information_schema', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'main', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'pg_catalog', - 'db_schema_tables': None, - }, - ], - }, - { - 'catalog_name': 'memory', - 'catalog_db_schemas': [ - { - 'db_schema_name': 'information_schema', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'main', - 'db_schema_tables': None, - }, - { - 'db_schema_name': 'pg_catalog', - 'db_schema_tables': None, - }, - ], - } - ]) + assert sorted_get_objects(table_name_filter.to_pylist()) == sorted_get_objects( + [ + { + 'catalog_name': 'system', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, + { + 'catalog_name': 'temp', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, + { + 'catalog_name': 'memory', + 'catalog_db_schemas': [ + { + 'db_schema_name': 'information_schema', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'main', + 'db_schema_tables': None, + }, + { + 'db_schema_name': 'pg_catalog', + 'db_schema_tables': None, + }, + ], + }, + ] + ) db_schema_filter = duck_conn.adbc_get_objects(depth="all", db_schema_filter="notexist").read_all() - assert sorted_get_objects(db_schema_filter.to_pylist()) == sorted_get_objects([ - { - 'catalog_name': 'system', - 'catalog_db_schemas': None, - }, - { - 'catalog_name': 'temp', - 'catalog_db_schemas': None, - }, - { - 'catalog_name': 'memory', - 'catalog_db_schemas': None, - } - ]) + assert sorted_get_objects(db_schema_filter.to_pylist()) == sorted_get_objects( + [ + { + 'catalog_name': 'system', + 'catalog_db_schemas': None, + }, + { + 'catalog_name': 'temp', + 'catalog_db_schemas': None, + }, + { + 'catalog_name': 'memory', + 'catalog_db_schemas': None, + }, + ] + ) catalog_filter = duck_conn.adbc_get_objects(depth="all", catalog_filter="notexist").read_all() assert catalog_filter.to_pylist() == [] @@ -719,6 +735,7 @@ def test_read(duck_conn): ], } + def sorted_get_objects(catalogs): res = [] for catalog in sorted(catalogs, key=lambda cat: cat['catalog_name']): @@ -726,13 +743,13 @@ def sorted_get_objects(catalogs): "catalog_name": catalog['catalog_name'], "catalog_db_schemas": [], } - + for db_schema in sorted(catalog['catalog_db_schemas'] or [], key=lambda sch: sch['db_schema_name']): new_db_schema = { "db_schema_name": db_schema['db_schema_name'], "db_schema_tables": [], } - + for table in sorted(db_schema['db_schema_tables'] or [], key=lambda tab: tab['table_name']): new_table = { "table_name": table['table_name'], @@ -740,15 +757,15 @@ def sorted_get_objects(catalogs): "table_columns": [], "table_constraints": [], } - + for column in sorted(table['table_columns'] or [], key=lambda col: col['ordinal_position']): new_table["table_columns"].append(column) - + for constraint in sorted(table['table_constraints'] or [], key=lambda con: con['constraint_name']): new_table["table_constraints"].append(constraint) - + new_db_schema["db_schema_tables"].append(new_table) new_catalog["catalog_db_schemas"].append(new_db_schema) res.append(new_catalog) - return res \ No newline at end of file + return res From e733fb7bf6218432356108349e1fc8c6cef7c725 Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 11 Apr 2024 17:33:11 +0200 Subject: [PATCH 351/603] duckdb::shared_ptr in duckdb_java --- tools/jdbc/src/jni/duckdb_java.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/jdbc/src/jni/duckdb_java.cpp b/tools/jdbc/src/jni/duckdb_java.cpp index e019526c65be..f7fed87b26d9 100644 --- a/tools/jdbc/src/jni/duckdb_java.cpp +++ b/tools/jdbc/src/jni/duckdb_java.cpp @@ -1,6 +1,7 @@ #include "functions.hpp" #include "duckdb.hpp" #include "duckdb/main/client_context.hpp" +#include "duckdb/common/shared_ptr.hpp" #include "duckdb/main/client_data.hpp" #include "duckdb/catalog/catalog_search_path.hpp" #include "duckdb/main/appender.hpp" @@ -348,10 +349,10 @@ static Value create_value_from_bigdecimal(JNIEnv *env, jobject decimal) { * DuckDB is released as well. */ struct ConnectionHolder { - const shared_ptr db; + const duckdb::shared_ptr db; const duckdb::unique_ptr connection; - ConnectionHolder(shared_ptr _db) : db(_db), connection(make_uniq(*_db)) { + ConnectionHolder(duckdb::shared_ptr _db) : db(_db), connection(make_uniq(*_db)) { } }; From a2b532266b55c77e7ccd6f1851ae2e66c133eee1 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 11 Apr 2024 18:00:55 +0200 Subject: [PATCH 352/603] Use references in subquery flattening code --- .../subquery/flatten_dependent_join.hpp | 4 +- .../binder/query_node/plan_subquery.cpp | 18 +++---- .../subquery/flatten_dependent_join.cpp | 53 +++++++++---------- 3 files changed, 37 insertions(+), 38 deletions(-) diff --git a/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp b/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp index efc41cf648a5..991e084c42ab 100644 --- a/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +++ b/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp @@ -23,7 +23,7 @@ struct FlattenDependentJoins { //! Detects which Logical Operators have correlated expressions that they are dependent upon, filling the //! has_correlated_expressions map. - bool DetectCorrelatedExpressions(LogicalOperator *op, bool lateral = false, idx_t lateral_depth = 0); + bool DetectCorrelatedExpressions(LogicalOperator &op, bool lateral = false, idx_t lateral_depth = 0); //! Mark entire subtree of Logical Operators as correlated by adding them to the has_correlated_expressions map. bool MarkSubtreeCorrelated(LogicalOperator &op); @@ -35,7 +35,7 @@ struct FlattenDependentJoins { ColumnBinding base_binding; idx_t delim_offset; idx_t data_offset; - unordered_map has_correlated_expressions; + reference_map_t has_correlated_expressions; column_binding_map_t correlated_map; column_binding_map_t replacement_map; const vector &correlated_columns; diff --git a/src/planner/binder/query_node/plan_subquery.cpp b/src/planner/binder/query_node/plan_subquery.cpp index 7370457a6031..29e8f36c3063 100644 --- a/src/planner/binder/query_node/plan_subquery.cpp +++ b/src/planner/binder/query_node/plan_subquery.cpp @@ -254,7 +254,7 @@ static unique_ptr PlanCorrelatedSubquery(Binder &binder, BoundSubque FlattenDependentJoins flatten(binder, correlated_columns, perform_delim); // first we check which logical operators have correlated expressions in the first place - flatten.DetectCorrelatedExpressions(plan.get()); + flatten.DetectCorrelatedExpressions(*plan); // now we push the dependent join down auto dependent_join = flatten.PushDownDependentJoin(std::move(plan)); @@ -279,7 +279,7 @@ static unique_ptr PlanCorrelatedSubquery(Binder &binder, BoundSubque delim_join->mark_index = mark_index; // RHS FlattenDependentJoins flatten(binder, correlated_columns, perform_delim, true); - flatten.DetectCorrelatedExpressions(plan.get()); + flatten.DetectCorrelatedExpressions(*plan); auto dependent_join = flatten.PushDownDependentJoin(std::move(plan)); // fetch the set of columns @@ -307,7 +307,7 @@ static unique_ptr PlanCorrelatedSubquery(Binder &binder, BoundSubque delim_join->mark_index = mark_index; // RHS FlattenDependentJoins flatten(binder, correlated_columns, true, true); - flatten.DetectCorrelatedExpressions(plan.get()); + flatten.DetectCorrelatedExpressions(*plan); auto dependent_join = flatten.PushDownDependentJoin(std::move(plan)); // fetch the columns @@ -411,7 +411,7 @@ void Binder::PlanSubqueries(unique_ptr &expr_ptr, unique_ptr Binder::PlanLateralJoin(unique_ptr left, unique_ptr right, - vector &correlated_columns, + vector &correlated, JoinType join_type, unique_ptr condition) { // scan the right operator for correlated columns // correlated LATERAL JOIN @@ -423,13 +423,13 @@ unique_ptr Binder::PlanLateralJoin(unique_ptr arbitrary_expressions); } - auto perform_delim = PerformDuplicateElimination(*this, correlated_columns); - auto delim_join = CreateDuplicateEliminatedJoin(correlated_columns, join_type, std::move(left), perform_delim); + auto perform_delim = PerformDuplicateElimination(*this, correlated); + auto delim_join = CreateDuplicateEliminatedJoin(correlated, join_type, std::move(left), perform_delim); - FlattenDependentJoins flatten(*this, correlated_columns, perform_delim); + FlattenDependentJoins flatten(*this, correlated, perform_delim); // first we check which logical operators have correlated expressions in the first place - flatten.DetectCorrelatedExpressions(right.get(), true); + flatten.DetectCorrelatedExpressions(*right, true); // now we push the dependent join down auto dependent_join = flatten.PushDownDependentJoin(std::move(right)); @@ -448,7 +448,7 @@ unique_ptr Binder::PlanLateralJoin(unique_ptr D_ASSERT(delim_join->conditions.empty()); delim_join->conditions = std::move(conditions); // then add the delim join conditions - CreateDelimJoinConditions(*delim_join, correlated_columns, plan_columns, flatten.delim_offset, perform_delim); + CreateDelimJoinConditions(*delim_join, correlated, plan_columns, flatten.delim_offset, perform_delim); delim_join->AddChild(std::move(dependent_join)); // check if there are any arbitrary expressions left diff --git a/src/planner/subquery/flatten_dependent_join.cpp b/src/planner/subquery/flatten_dependent_join.cpp index 4e5623cab48d..7e863703472c 100644 --- a/src/planner/subquery/flatten_dependent_join.cpp +++ b/src/planner/subquery/flatten_dependent_join.cpp @@ -27,21 +27,20 @@ FlattenDependentJoins::FlattenDependentJoins(Binder &binder, const vectortype == LogicalOperatorType::LOGICAL_DEPENDENT_JOIN) { + if (op.type == LogicalOperatorType::LOGICAL_DEPENDENT_JOIN) { is_lateral_join = true; } HasCorrelatedExpressions visitor(correlated_columns, lateral, lateral_depth); - visitor.VisitOperator(*op); + visitor.VisitOperator(op); bool has_correlation = visitor.has_correlated_expressions; int child_idx = 0; // now visit the children of this entry and check if they have correlated expressions - for (auto &child : op->children) { + for (auto &child : op.children) { auto new_lateral_depth = lateral_depth; if (is_lateral_join && child_idx == 1) { new_lateral_depth = lateral_depth + 1; @@ -49,7 +48,7 @@ bool FlattenDependentJoins::DetectCorrelatedExpressions(LogicalOperator *op, boo // we OR the property with its children such that has_correlation is true if either // (1) this node has a correlated expression or // (2) one of its children has a correlated expression - if (DetectCorrelatedExpressions(child.get(), lateral, new_lateral_depth)) { + if (DetectCorrelatedExpressions(*child, lateral, new_lateral_depth)) { has_correlation = true; } child_idx++; @@ -60,10 +59,10 @@ bool FlattenDependentJoins::DetectCorrelatedExpressions(LogicalOperator *op, boo // If we detect correlation in a materialized or recursive CTE, the entire right side of the operator // needs to be marked as correlated. Otherwise, function PushDownDependentJoinInternal does not do the // right thing. - if (op->type == LogicalOperatorType::LOGICAL_MATERIALIZED_CTE || - op->type == LogicalOperatorType::LOGICAL_RECURSIVE_CTE) { + if (op.type == LogicalOperatorType::LOGICAL_MATERIALIZED_CTE || + op.type == LogicalOperatorType::LOGICAL_RECURSIVE_CTE) { if (has_correlation) { - MarkSubtreeCorrelated(*op->children[1].get()); + MarkSubtreeCorrelated(*op.children[1].get()); } } return has_correlation; @@ -71,7 +70,7 @@ bool FlattenDependentJoins::DetectCorrelatedExpressions(LogicalOperator *op, boo bool FlattenDependentJoins::MarkSubtreeCorrelated(LogicalOperator &op) { // Do not mark base table scans as correlated - auto entry = has_correlated_expressions.find(&op); + auto entry = has_correlated_expressions.find(op); D_ASSERT(entry != has_correlated_expressions.end()); bool has_correlation = entry->second; for (auto &child : op.children) { @@ -79,10 +78,10 @@ bool FlattenDependentJoins::MarkSubtreeCorrelated(LogicalOperator &op) { } if (op.type != LogicalOperatorType::LOGICAL_GET || op.children.size() == 1) { if (op.type == LogicalOperatorType::LOGICAL_CTE_REF) { - has_correlated_expressions[&op] = true; + has_correlated_expressions[op] = true; return true; } else { - has_correlated_expressions[&op] = has_correlation; + has_correlated_expressions[op] = has_correlation; } } return has_correlation; @@ -99,17 +98,17 @@ unique_ptr FlattenDependentJoins::PushDownDependentJoin(unique_ return result; } -bool SubqueryDependentFilter(Expression *expr) { - if (expr->expression_class == ExpressionClass::BOUND_CONJUNCTION && - expr->GetExpressionType() == ExpressionType::CONJUNCTION_AND) { - auto &bound_conjuction = expr->Cast(); +bool SubqueryDependentFilter(Expression &expr) { + if (expr.expression_class == ExpressionClass::BOUND_CONJUNCTION && + expr.GetExpressionType() == ExpressionType::CONJUNCTION_AND) { + auto &bound_conjuction = expr.Cast(); for (auto &child : bound_conjuction.children) { - if (SubqueryDependentFilter(child.get())) { + if (SubqueryDependentFilter(*child)) { return true; } } } - if (expr->expression_class == ExpressionClass::BOUND_SUBQUERY) { + if (expr.expression_class == ExpressionClass::BOUND_SUBQUERY) { return true; } return false; @@ -119,7 +118,7 @@ unique_ptr FlattenDependentJoins::PushDownDependentJoinInternal bool &parent_propagate_null_values, idx_t lateral_depth) { // first check if the logical operator has correlated expressions - auto entry = has_correlated_expressions.find(plan.get()); + auto entry = has_correlated_expressions.find(*plan); D_ASSERT(entry != has_correlated_expressions.end()); if (!entry->second) { // we reached a node without correlated expressions @@ -151,7 +150,7 @@ unique_ptr FlattenDependentJoins::PushDownDependentJoinInternal // filter // first we flatten the dependent join in the child of the filter for (auto &expr : plan->expressions) { - any_join |= SubqueryDependentFilter(expr.get()); + any_join |= SubqueryDependentFilter(*expr); } plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]), parent_propagate_null_values, lateral_depth); @@ -288,8 +287,8 @@ unique_ptr FlattenDependentJoins::PushDownDependentJoinInternal case LogicalOperatorType::LOGICAL_CROSS_PRODUCT: { // cross product // push into both sides of the plan - bool left_has_correlation = has_correlated_expressions.find(plan->children[0].get())->second; - bool right_has_correlation = has_correlated_expressions.find(plan->children[1].get())->second; + bool left_has_correlation = has_correlated_expressions.find(*plan->children[0])->second; + bool right_has_correlation = has_correlated_expressions.find(*plan->children[1])->second; if (!right_has_correlation) { // only left has correlation: push into left plan->children[0] = PushDownDependentJoinInternal(std::move(plan->children[0]), @@ -350,8 +349,8 @@ unique_ptr FlattenDependentJoins::PushDownDependentJoinInternal auto &join = plan->Cast(); D_ASSERT(plan->children.size() == 2); // check the correlated expressions in the children of the join - bool left_has_correlation = has_correlated_expressions.find(plan->children[0].get())->second; - bool right_has_correlation = has_correlated_expressions.find(plan->children[1].get())->second; + bool left_has_correlation = has_correlated_expressions.find(*plan->children[0])->second; + bool right_has_correlation = has_correlated_expressions.find(*plan->children[1])->second; if (join.join_type == JoinType::INNER) { // inner join @@ -433,12 +432,12 @@ unique_ptr FlattenDependentJoins::PushDownDependentJoinInternal auto &comparison_join = join.Cast(); comparison_join.conditions.push_back(std::move(cond)); } else { - auto &any_join = join.Cast(); + auto &logical_any_join = join.Cast(); auto comparison = make_uniq(ExpressionType::COMPARE_NOT_DISTINCT_FROM, std::move(left), std::move(right)); auto conjunction = make_uniq( - ExpressionType::CONJUNCTION_AND, std::move(comparison), std::move(any_join.condition)); - any_join.condition = std::move(conjunction); + ExpressionType::CONJUNCTION_AND, std::move(comparison), std::move(logical_any_join.condition)); + logical_any_join.condition = std::move(conjunction); } } // then we replace any correlated expressions with the corresponding entry in the correlated_map From 3516894d197ef49604e5337da2d28144f355770f Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 11 Apr 2024 20:06:21 +0200 Subject: [PATCH 353/603] For ExpressionDepthReducerRecursive - correctly recurse into nested subqueries --- .../expression_binder/lateral_binder.cpp | 16 +++++------ .../subquery/lateral/lateral_fuzzer_1463.test | 28 +++++++++++++++++++ 2 files changed, 36 insertions(+), 8 deletions(-) create mode 100644 test/sql/subquery/lateral/lateral_fuzzer_1463.test diff --git a/src/planner/expression_binder/lateral_binder.cpp b/src/planner/expression_binder/lateral_binder.cpp index 13e3ae59980a..e58d78afd0b7 100644 --- a/src/planner/expression_binder/lateral_binder.cpp +++ b/src/planner/expression_binder/lateral_binder.cpp @@ -81,11 +81,6 @@ static void ReduceColumnDepth(vector &columns, } } -static void ReduceExpressionSubquery(BoundSubqueryExpression &expr, - const vector &correlated_columns) { - ReduceColumnDepth(expr.binder->correlated_columns, correlated_columns); -} - class ExpressionDepthReducerRecursive : public BoundNodeVisitor { public: explicit ExpressionDepthReducerRecursive(const vector &correlated) @@ -111,6 +106,13 @@ class ExpressionDepthReducerRecursive : public BoundNodeVisitor { BoundNodeVisitor::VisitBoundTableRef(ref); } + static void ReduceExpressionSubquery(BoundSubqueryExpression &expr, + const vector &correlated_columns) { + ReduceColumnDepth(expr.binder->correlated_columns, correlated_columns); + ExpressionDepthReducerRecursive recursive(correlated_columns); + recursive.VisitBoundQueryNode(*expr.subquery); + } + private: const vector &correlated_columns; }; @@ -127,9 +129,7 @@ class ExpressionDepthReducer : public LogicalOperatorVisitor { } unique_ptr VisitReplace(BoundSubqueryExpression &expr, unique_ptr *expr_ptr) override { - ReduceExpressionSubquery(expr, correlated_columns); - ExpressionDepthReducerRecursive recursive(correlated_columns); - recursive.VisitBoundQueryNode(*expr.subquery); + ExpressionDepthReducerRecursive::ReduceExpressionSubquery(expr, correlated_columns); return nullptr; } diff --git a/test/sql/subquery/lateral/lateral_fuzzer_1463.test b/test/sql/subquery/lateral/lateral_fuzzer_1463.test new file mode 100644 index 000000000000..003bfd424411 --- /dev/null +++ b/test/sql/subquery/lateral/lateral_fuzzer_1463.test @@ -0,0 +1,28 @@ +# name: test/sql/subquery/lateral/lateral_fuzzer_1463.test +# description: Test case for fuzzer issue 1463: Expression with depth > 1 detected in non-lateral join +# group: [lateral] + +query II +SELECT * +FROM + (SELECT 42 AS c1) AS ref, + (SELECT a + b + 1 + FROM + (SELECT 1) t1(a), + (SELECT (SELECT (SELECT ref.c1 + 1)) + 1) t2(b) + ) +; +---- +42 46 + +# postgres compatible variant +query I +SELECT NULL +FROM + (SELECT 42 AS c1) AS ref, + LATERAL (SELECT NULL + FROM + (SELECT NULL) AS r2, + (SELECT (SELECT (SELECT ref.c1))) AS r3) AS r4; +---- +NULL From 08ac807ee9d39a1179a195060f4cf06023d5d704 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 11 Apr 2024 20:08:22 +0200 Subject: [PATCH 354/603] Fix reduce SQL test --- test/sqlsmith/sql_reduce.test | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/sqlsmith/sql_reduce.test b/test/sqlsmith/sql_reduce.test index 73ee30559b1c..7b956e8db065 100644 --- a/test/sqlsmith/sql_reduce.test +++ b/test/sqlsmith/sql_reduce.test @@ -7,18 +7,33 @@ require sqlsmith query I SELECT * FROM reduce_sql_statement('SELECT a, b FROM tbl') ORDER BY 1 ---- +SELECT NULL, b FROM tbl +SELECT NULL, b FROM tbl SELECT a FROM tbl +SELECT a, NULL FROM tbl +SELECT a, NULL FROM tbl SELECT a, b SELECT b FROM tbl query I SELECT * FROM reduce_sql_statement('SELECT a, b FROM tbl WHERE a AND b') ORDER BY 1 ---- +SELECT NULL, b FROM tbl WHERE (a AND b) +SELECT NULL, b FROM tbl WHERE (a AND b) SELECT a FROM tbl WHERE (a AND b) +SELECT a, NULL FROM tbl WHERE (a AND b) +SELECT a, NULL FROM tbl WHERE (a AND b) SELECT a, b FROM tbl +SELECT a, b FROM tbl WHERE (NULL AND b) +SELECT a, b FROM tbl WHERE (NULL AND b) +SELECT a, b FROM tbl WHERE (a AND NULL) +SELECT a, b FROM tbl WHERE (a AND NULL) SELECT a, b FROM tbl WHERE NULL +SELECT a, b FROM tbl WHERE NULL +SELECT a, b FROM tbl WHERE a SELECT a, b FROM tbl WHERE a SELECT a, b FROM tbl WHERE b +SELECT a, b FROM tbl WHERE b SELECT a, b WHERE (a AND b) SELECT b FROM tbl WHERE (a AND b) @@ -29,19 +44,31 @@ INSERT INTO tbl (VALUES (1)) INSERT INTO tbl (VALUES (2)) INSERT INTO tbl SELECT * INSERT INTO tbl SELECT NULL FROM (VALUES (1, 2)) +INSERT INTO tbl SELECT NULL FROM (VALUES (1, 2)) AS valueslist query I SELECT * FROM reduce_sql_statement('UPDATE tbl SET i=3, j=4 WHERE z=5') ORDER BY 1 ---- UPDATE tbl SET i = 3 WHERE (z = 5) UPDATE tbl SET i = 3, j = 4 +UPDATE tbl SET i = 3, j = 4 WHERE (NULL = 5) +UPDATE tbl SET i = 3, j = 4 WHERE 5 UPDATE tbl SET i = 3, j = 4 WHERE NULL +UPDATE tbl SET i = 3, j = 4 WHERE z UPDATE tbl SET j = 4 WHERE (z = 5) query I SELECT * FROM reduce_sql_statement('DELETE FROM a WHERE i >= 2000 AND i < 5000;') ORDER BY 1 ---- DELETE FROM a +DELETE FROM a WHERE ((NULL >= 2000) AND (i < 5000)) +DELETE FROM a WHERE ((i >= 2000) AND (NULL < 5000)) +DELETE FROM a WHERE ((i >= 2000) AND 5000) +DELETE FROM a WHERE ((i >= 2000) AND NULL) +DELETE FROM a WHERE ((i >= 2000) AND i) +DELETE FROM a WHERE (2000 AND (i < 5000)) +DELETE FROM a WHERE (NULL AND (i < 5000)) DELETE FROM a WHERE (i < 5000) DELETE FROM a WHERE (i >= 2000) +DELETE FROM a WHERE (i AND (i < 5000)) DELETE FROM a WHERE NULL From 1795abd305f3c538ab93d70f7e4c0299c340baf6 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 11 Apr 2024 20:18:19 +0200 Subject: [PATCH 355/603] Format fix --- .../sqlsmith/include/statement_simplifier.hpp | 3 +- extension/sqlsmith/statement_simplifier.cpp | 13 +++++--- scripts/fuzzer_helper.py | 33 +++++++++++++++---- scripts/reduce_sql.py | 32 ++++++++++++++---- scripts/run_fuzzer.py | 29 ++++++++++++---- .../binder/query_node/plan_subquery.cpp | 4 +-- .../expression_binder/lateral_binder.cpp | 2 +- 7 files changed, 87 insertions(+), 29 deletions(-) diff --git a/extension/sqlsmith/include/statement_simplifier.hpp b/extension/sqlsmith/include/statement_simplifier.hpp index 2bdf2715cd18..9e6fa160ede4 100644 --- a/extension/sqlsmith/include/statement_simplifier.hpp +++ b/extension/sqlsmith/include/statement_simplifier.hpp @@ -65,7 +65,8 @@ class StatementSimplifier { void SimplifyExpression(duckdb::unique_ptr &expr); void SimplifyOptionalExpression(duckdb::unique_ptr &expr); void SimplifyChildExpression(duckdb::unique_ptr &expr, unique_ptr &child); - void SimplifyExpressionList(duckdb::unique_ptr &expr, vector> &expression_list); + void SimplifyExpressionList(duckdb::unique_ptr &expr, + vector> &expression_list); void SimplifyExpressionList(vector> &expression_list, bool is_optional = true); void Simplify(CommonTableExpressionMap &cte_map); diff --git a/extension/sqlsmith/statement_simplifier.cpp b/extension/sqlsmith/statement_simplifier.cpp index edfd3437ee70..998d32df9fcc 100644 --- a/extension/sqlsmith/statement_simplifier.cpp +++ b/extension/sqlsmith/statement_simplifier.cpp @@ -142,20 +142,23 @@ void StatementSimplifier::Simplify(QueryNode &node) { SimplifyList(node.modifiers); } -void StatementSimplifier::SimplifyExpressionList(duckdb::unique_ptr &expr, vector> &expression_list) { - for(auto &child : expression_list) { +void StatementSimplifier::SimplifyExpressionList(duckdb::unique_ptr &expr, + vector> &expression_list) { + for (auto &child : expression_list) { SimplifyChildExpression(expr, child); } } -void StatementSimplifier::SimplifyExpressionList(vector> &expression_list, bool is_optional) { +void StatementSimplifier::SimplifyExpressionList(vector> &expression_list, + bool is_optional) { SimplifyList(expression_list, is_optional); - for(auto &child : expression_list) { + for (auto &child : expression_list) { SimplifyExpression(child); } } -void StatementSimplifier::SimplifyChildExpression(duckdb::unique_ptr &expr, unique_ptr &child) { +void StatementSimplifier::SimplifyChildExpression(duckdb::unique_ptr &expr, + unique_ptr &child) { if (!child) { return; } diff --git a/scripts/fuzzer_helper.py b/scripts/fuzzer_helper.py index d457b3172d49..dd82e0622490 100644 --- a/scripts/fuzzer_helper.py +++ b/scripts/fuzzer_helper.py @@ -29,6 +29,7 @@ footer = ''' ```''' + # github stuff def issue_url(): return 'https://api.github.com/repos/%s/%s/issues' % (REPO_OWNER, REPO_NAME) @@ -47,20 +48,22 @@ def get_token(): print("Incorrect length for FUZZEROFDUCKSKEY") exit(1) return token + + def create_session(): # Create an authenticated session to create the issue session = requests.Session() session.headers.update({'Authorization': 'token %s' % (get_token(),)}) return session + def make_github_issue(title, body): if len(title) > 240: # avoid title is too long error (maximum is 256 characters) title = title[:240] + '...' session = create_session() url = issue_url() - issue = {'title': title, - 'body': body} + issue = {'title': title, 'body': body} r = session.post(url, json.dumps(issue)) if r.status_code == 201: print('Successfully created Issue "%s"' % title) @@ -69,9 +72,10 @@ def make_github_issue(title, body): print('Response:', r.content.decode('utf8')) raise Exception("Failed to create issue") + def get_github_issues(page): session = create_session() - url = issue_url()+'?per_page=100&page='+str(page) + url = issue_url() + '?per_page=100&page=' + str(page) r = session.get(url) if r.status_code != 200: print('Failed to get list of issues') @@ -79,6 +83,7 @@ def get_github_issues(page): raise Exception("Failed to get list of issues") return json.loads(r.content.decode('utf8')) + def close_github_issue(number): session = create_session() url = issue_url() + '/' + str(number) @@ -91,6 +96,7 @@ def close_github_issue(number): print('Response:', r.content.decode('utf8')) raise Exception("Failed to close issue") + def label_github_issue(number, label): session = create_session() url = issue_url() + '/' + str(number) @@ -103,22 +109,26 @@ def label_github_issue(number, label): print('Response:', r.content.decode('utf8')) raise Exception("Failed to label issue") + def extract_issue(body, nr): try: splits = body.split(middle) sql = splits[0].split(header)[1] - error = splits[1][:-len(footer)] + error = splits[1][: -len(footer)] return (sql, error) except: print(f"Failed to extract SQL/error message from issue {nr}") print(body) return None + def run_shell_command_batch(shell, cmd): command = [shell, '--batch', '-init', '/dev/null'] try: - res = subprocess.run(command, input=bytearray(cmd, 'utf8'), stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=300) + res = subprocess.run( + command, input=bytearray(cmd, 'utf8'), stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=300 + ) except subprocess.TimeoutExpired: print(f"TIMEOUT... {cmd}") return ("", "", 0, True) @@ -126,6 +136,7 @@ def run_shell_command_batch(shell, cmd): stderr = res.stderr.decode('utf8').strip() return (stdout, stderr, res.returncode, False) + def test_reproducibility(shell, issue, current_errors, perform_check): extract = extract_issue(issue['body'], issue['number']) labels = issue['labels'] @@ -152,9 +163,10 @@ def test_reproducibility(shell, issue, current_errors, perform_check): current_errors[error] = issue return True + def extract_github_issues(shell, perform_check): current_errors = dict() - for p in range(1,10): + for p in range(1, 10): issues = get_github_issues(p) for issue in issues: # check if the github issue is still reproducible @@ -164,16 +176,23 @@ def extract_github_issues(shell, perform_check): close_github_issue(int(issue['number'])) return current_errors + def file_issue(cmd, error_msg, fuzzer, seed, hash): # issue is new, file it print("Filing new issue to Github") title = error_msg - body = fuzzer_desc.replace("${FUZZER}", fuzzer).replace("${FULL_HASH}", hash).replace("${SHORT_HASH}", hash[:5]).replace("${SEED}", str(seed)) + body = ( + fuzzer_desc.replace("${FUZZER}", fuzzer) + .replace("${FULL_HASH}", hash) + .replace("${SHORT_HASH}", hash[:5]) + .replace("${SEED}", str(seed)) + ) body += header + cmd + middle + error_msg + footer print(title, body) make_github_issue(title, body) + def is_internal_error(error): if 'differs from original result' in error: return True diff --git a/scripts/reduce_sql.py b/scripts/reduce_sql.py index 92a449b799b1..9d96bc41fca7 100644 --- a/scripts/reduce_sql.py +++ b/scripts/reduce_sql.py @@ -18,6 +18,7 @@ SELECT * FROM reduce_sql_statement('${QUERY}'); ''' + def sanitize_error(err): err = re.sub(r'Error: near line \d+: ', '', err) err = err.replace(os.getcwd() + '/', '') @@ -27,6 +28,7 @@ def sanitize_error(err): err = 'AddressSanitizer error ' + match return err + def run_shell_command(shell, cmd): command = [shell, '-csv', '--batch', '-init', '/dev/null'] @@ -35,6 +37,7 @@ def run_shell_command(shell, cmd): stderr = res.stderr.decode('utf8').strip() return (stdout, stderr, res.returncode) + def get_reduced_sql(shell, sql_query): reduce_query = get_reduced_query.replace('${QUERY}', sql_query.replace("'", "''")) (stdout, stderr, returncode) = run_shell_command(shell, reduce_query) @@ -47,6 +50,7 @@ def get_reduced_sql(shell, sql_query): reduce_candidates.append(line.strip('"').replace('""', '"')) return reduce_candidates[1:] + def reduce(sql_query, data_load, shell, error_msg, max_time_seconds=300): start = time.time() while True: @@ -73,18 +77,22 @@ def reduce(sql_query, data_load, shell, error_msg, max_time_seconds=300): break return sql_query + def is_ddl_query(query): query = query.lower() if 'create' in query or 'insert' in query or 'update' in query or 'delete' in query: return True return False + def initial_cleanup(query_log): query_log = query_log.replace('SELECT * FROM pragma_version()\n', '') return query_log + def run_queries_until_crash_mp(queries, result_file): import duckdb + con = duckdb.connect() sqlite_con = sqlite3.connect(result_file) sqlite_con.execute('CREATE TABLE queries(id INT, text VARCHAR)') @@ -109,7 +117,7 @@ def run_queries_until_crash_mp(queries, result_file): keep_query = True sqlite_con.execute('UPDATE result SET text=?', (exception_error,)) if not keep_query: - sqlite_con.execute('DELETE FROM queries WHERE id=?', (id, )) + sqlite_con.execute('DELETE FROM queries WHERE id=?', (id,)) if is_internal_error: # found internal error: no need to try further queries break @@ -120,6 +128,7 @@ def run_queries_until_crash_mp(queries, result_file): sqlite_con.commit() sqlite_con.close() + def run_queries_until_crash(queries): sqlite_file = 'cleaned_queries.db' if os.path.isfile(sqlite_file): @@ -147,8 +156,10 @@ def cleanup_irrelevant_queries(query_log): queries = [x for x in query_log.split(';\n') if len(x) > 0] return run_queries_until_crash(queries) + # def reduce_internal(start, sql_query, data_load, queries_final, shell, error_msg, max_time_seconds=300): + def reduce_query_log_query(start, shell, queries, query_index, max_time_seconds): new_query_list = queries[:] sql_query = queries[query_index] @@ -180,6 +191,7 @@ def reduce_query_log_query(start, shell, queries, query_index, max_time_seconds) break return sql_query + def reduce_query_log(queries, shell, max_time_seconds=300): start = time.time() current_index = 0 @@ -190,7 +202,7 @@ def reduce_query_log(queries, shell, max_time_seconds=300): if current_time - start > max_time_seconds: break # remove the query at "current_index" - new_queries = queries[:current_index] + queries[current_index + 1:] + new_queries = queries[:current_index] + queries[current_index + 1 :] # try to run the queries and check if we still get the same error (new_queries_x, current_error) = run_queries_until_crash(new_queries) if current_error is None: @@ -212,12 +224,19 @@ def reduce_query_log(queries, shell, max_time_seconds=300): if __name__ == "__main__": import argparse + parser = argparse.ArgumentParser(description='Reduce a problematic SQL query') - parser.add_argument('--shell', dest='shell', action='store', help='Path to the shell executable', default='build/debug/duckdb') + parser.add_argument( + '--shell', dest='shell', action='store', help='Path to the shell executable', default='build/debug/duckdb' + ) parser.add_argument('--load', dest='load', action='store', help='Path to the data load script', required=True) parser.add_argument('--exec', dest='exec', action='store', help='Path to the executable script', required=True) - parser.add_argument('--inplace', dest='inplace', action='store_true', help='If true, overrides the exec script with the final query') - parser.add_argument('--max-time', dest='max_time', action='store', help='Maximum time in seconds to run the reducer', default=300) + parser.add_argument( + '--inplace', dest='inplace', action='store_true', help='If true, overrides the exec script with the final query' + ) + parser.add_argument( + '--max-time', dest='max_time', action='store', help='Maximum time in seconds to run the reducer', default=300 + ) args = parser.parse_args() print("Starting reduce process") @@ -234,7 +253,6 @@ def reduce_query_log(queries, shell, max_time_seconds=300): print(expected_error) print("===================================================") - final_query = reduce(sql_query, data_load, shell, expected_error, args.max_time) print("Found final reduced query") print("===================================================") @@ -302,4 +320,4 @@ def reduce_query_log(queries, shell, max_time_seconds=300): # limit 88 # ''' # -# print(reduce(sql_query, data_load, shell, error_msg)) \ No newline at end of file +# print(reduce(sql_query, data_load, shell, error_msg)) diff --git a/scripts/run_fuzzer.py b/scripts/run_fuzzer.py index 57f7a677b3fd..c7d096f11ed9 100644 --- a/scripts/run_fuzzer.py +++ b/scripts/run_fuzzer.py @@ -50,6 +50,7 @@ git_hash = os.getenv('DUCKDB_HASH') + def create_db_script(db): if db == 'alltypes': return 'create table all_types as select * exclude(small_enum, medium_enum, large_enum) from test_all_types();' @@ -60,6 +61,7 @@ def create_db_script(db): else: raise Exception("Unknown database creation script") + def run_fuzzer_script(fuzzer): if fuzzer == 'sqlsmith': return "call sqlsmith(max_queries=${MAX_QUERIES}, seed=${SEED}, verbose_output=1, log='${LAST_LOG_FILE}', complete_log='${COMPLETE_LOG_FILE}');" @@ -70,6 +72,7 @@ def run_fuzzer_script(fuzzer): else: raise Exception("Unknown fuzzer type") + def get_fuzzer_name(fuzzer): if fuzzer == 'sqlsmith': return 'SQLSmith' @@ -80,6 +83,7 @@ def get_fuzzer_name(fuzzer): else: return 'Unknown' + def run_shell_command(cmd): command = [shell, '--batch', '-init', '/dev/null'] @@ -96,13 +100,21 @@ def run_shell_command(cmd): last_query_log_file = 'sqlsmith.log' complete_log_file = 'sqlsmith.complete.log' -print(f'''========================================== +print( + f'''========================================== RUNNING {fuzzer} on {db} -==========================================''') +==========================================''' +) load_script = create_db_script(db) fuzzer_name = get_fuzzer_name(fuzzer) -fuzzer = run_fuzzer_script(fuzzer).replace('${MAX_QUERIES}', str(max_queries)).replace('${LAST_LOG_FILE}', last_query_log_file).replace('${COMPLETE_LOG_FILE}', complete_log_file).replace('${SEED}', str(seed)) +fuzzer = ( + run_fuzzer_script(fuzzer) + .replace('${MAX_QUERIES}', str(max_queries)) + .replace('${LAST_LOG_FILE}', last_query_log_file) + .replace('${COMPLETE_LOG_FILE}', complete_log_file) + .replace('${SEED}', str(seed)) +) print(load_script) print(fuzzer) @@ -113,9 +125,11 @@ def run_shell_command(cmd): (stdout, stderr, returncode) = run_shell_command(cmd) -print(f'''========================================== +print( + f'''========================================== FINISHED RUNNING -==========================================''') +==========================================''' +) print("============== STDOUT ================") print(stdout) print("============== STDERR =================") @@ -160,7 +174,10 @@ def run_shell_command(cmd): # check if this is a duplicate issue if error_msg in current_errors: print("Skip filing duplicate issue") - print("Issue already exists: https://github.com/duckdb/duckdb-fuzzer/issues/" + str(current_errors[error_msg]['number'])) + print( + "Issue already exists: https://github.com/duckdb/duckdb-fuzzer/issues/" + + str(current_errors[error_msg]['number']) + ) exit(0) print(last_query) diff --git a/src/planner/binder/query_node/plan_subquery.cpp b/src/planner/binder/query_node/plan_subquery.cpp index 29e8f36c3063..3f3aaa92c9aa 100644 --- a/src/planner/binder/query_node/plan_subquery.cpp +++ b/src/planner/binder/query_node/plan_subquery.cpp @@ -411,8 +411,8 @@ void Binder::PlanSubqueries(unique_ptr &expr_ptr, unique_ptr Binder::PlanLateralJoin(unique_ptr left, unique_ptr right, - vector &correlated, - JoinType join_type, unique_ptr condition) { + vector &correlated, JoinType join_type, + unique_ptr condition) { // scan the right operator for correlated columns // correlated LATERAL JOIN vector conditions; diff --git a/src/planner/expression_binder/lateral_binder.cpp b/src/planner/expression_binder/lateral_binder.cpp index e58d78afd0b7..21ceb4e50c3f 100644 --- a/src/planner/expression_binder/lateral_binder.cpp +++ b/src/planner/expression_binder/lateral_binder.cpp @@ -107,7 +107,7 @@ class ExpressionDepthReducerRecursive : public BoundNodeVisitor { } static void ReduceExpressionSubquery(BoundSubqueryExpression &expr, - const vector &correlated_columns) { + const vector &correlated_columns) { ReduceColumnDepth(expr.binder->correlated_columns, correlated_columns); ExpressionDepthReducerRecursive recursive(correlated_columns); recursive.VisitBoundQueryNode(*expr.subquery); From 31ca8e46e056588ace09654ae574d2f39f012f8b Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 11 Apr 2024 21:46:29 +0200 Subject: [PATCH 356/603] clang tidy fixes --- .clang-tidy | 2 +- .../duckdb/common/enable_shared_from_this.ipp | 14 +++--- src/include/duckdb/common/helper.hpp | 2 +- src/include/duckdb/common/shared_ptr.hpp | 14 +++--- src/include/duckdb/common/shared_ptr.ipp | 46 +++++++++---------- src/include/duckdb/common/weak_ptr.ipp | 14 +++--- 6 files changed, 46 insertions(+), 46 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index ee2b06b745d7..c1b42c62ec42 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -46,7 +46,7 @@ CheckOptions: - key: readability-identifier-naming.VariableCase value: lower_case - key: modernize-use-emplace.SmartPointers - value: '::std::shared_ptr;::duckdb::unique_ptr;::std::auto_ptr;::std::weak_ptr' + value: '::duckdb::shared_ptr;::duckdb::unique_ptr;::std::auto_ptr;::duckdb::weak_ptr' - key: cppcoreguidelines-rvalue-reference-param-not-moved.IgnoreUnnamedParams value: true diff --git a/src/include/duckdb/common/enable_shared_from_this.ipp b/src/include/duckdb/common/enable_shared_from_this.ipp index d68d20033aa7..85cdd2205411 100644 --- a/src/include/duckdb/common/enable_shared_from_this.ipp +++ b/src/include/duckdb/common/enable_shared_from_this.ipp @@ -1,18 +1,18 @@ namespace duckdb { template -class enable_shared_from_this { +class enable_shared_from_this { // NOLINT: invalid case style public: template friend class shared_ptr; private: - mutable weak_ptr __weak_this_; + mutable weak_ptr __weak_this_; // NOLINT: __weak_this_ is reserved protected: constexpr enable_shared_from_this() noexcept { } - enable_shared_from_this(enable_shared_from_this const &) noexcept { + enable_shared_from_this(enable_shared_from_this const &) noexcept { // NOLINT: not marked as explicit } enable_shared_from_this &operator=(enable_shared_from_this const &) noexcept { return *this; @@ -21,19 +21,19 @@ protected: } public: - shared_ptr shared_from_this() { + shared_ptr shared_from_this() { // NOLINT: invalid case style return shared_ptr(__weak_this_); } - shared_ptr shared_from_this() const { + shared_ptr shared_from_this() const { // NOLINT: invalid case style return shared_ptr(__weak_this_); } #if _LIBCPP_STD_VER >= 17 - weak_ptr weak_from_this() noexcept { + weak_ptr weak_from_this() noexcept { // NOLINT: invalid case style return __weak_this_; } - weak_ptr weak_from_this() const noexcept { + weak_ptr weak_from_this() const noexcept { // NOLINT: invalid case style return __weak_this_; } #endif // _LIBCPP_STD_VER >= 17 diff --git a/src/include/duckdb/common/helper.hpp b/src/include/duckdb/common/helper.hpp index 2f12dca961f0..ed3e6e1f1be3 100644 --- a/src/include/duckdb/common/helper.hpp +++ b/src/include/duckdb/common/helper.hpp @@ -118,7 +118,7 @@ unique_ptr unique_ptr_cast(unique_ptr src) { // NOLINT: mimic std style } template -shared_ptr shared_ptr_cast(shared_ptr src) { +shared_ptr shared_ptr_cast(shared_ptr src) { // NOLINT: mimic std style return shared_ptr(std::static_pointer_cast(src.internal)); } diff --git a/src/include/duckdb/common/shared_ptr.hpp b/src/include/duckdb/common/shared_ptr.hpp index 1e6e1b2523b0..f5ca7d762c3a 100644 --- a/src/include/duckdb/common/shared_ptr.hpp +++ b/src/include/duckdb/common/shared_ptr.hpp @@ -22,17 +22,17 @@ namespace duckdb { // originally named '__compatible_with' #if _LIBCPP_STD_VER >= 17 -template +template struct __bounded_convertible_to_unbounded : false_type {}; -template -struct __bounded_convertible_to_unbounded<_Up[_Np], _Tp> : is_same, _Up[]> {}; +template +struct __bounded_convertible_to_unbounded<_Up[_Np], T> : is_same, _Up[]> {}; -template -struct compatible_with_t : _Or, __bounded_convertible_to_unbounded<_Yp, _Tp>> {}; +template +struct compatible_with_t : _Or, __bounded_convertible_to_unbounded> {}; #else -template -struct compatible_with_t : std::is_convertible<_Yp *, _Tp *> {}; +template +struct compatible_with_t : std::is_convertible {}; // NOLINT: invalid case style #endif // _LIBCPP_STD_VER >= 17 } // namespace duckdb diff --git a/src/include/duckdb/common/shared_ptr.ipp b/src/include/duckdb/common/shared_ptr.ipp index c36f28622f02..840f101588d4 100644 --- a/src/include/duckdb/common/shared_ptr.ipp +++ b/src/include/duckdb/common/shared_ptr.ipp @@ -7,7 +7,7 @@ template class enable_shared_from_this; template -class shared_ptr { +class shared_ptr { // NOLINT: invalid case style public: using original = std::shared_ptr; using element_type = typename original::element_type; @@ -49,33 +49,33 @@ public: explicit shared_ptr(U *ptr) : internal(ptr) { __enable_weak_this(internal.get(), internal.get()); } - // From raw pointer of type T with custom Deleter - template - shared_ptr(T *ptr, Deleter deleter) : internal(ptr, deleter) { + // From raw pointer of type T with custom DELETER + template + shared_ptr(T *ptr, DELETER deleter) : internal(ptr, deleter) { __enable_weak_this(internal.get(), internal.get()); } - // Aliasing constructor: shares ownership information with __r but contains __p instead - // When the created shared_ptr goes out of scope, it will call the Deleter of __r, will not delete __p + // Aliasing constructor: shares ownership information with ref but contains ptr instead + // When the created shared_ptr goes out of scope, it will call the DELETER of ref, will not delete ptr template - shared_ptr(const shared_ptr &__r, T *__p) noexcept : internal(__r.internal, __p) { + shared_ptr(const shared_ptr &ref, T *ptr) noexcept : internal(ref.internal, ptr) { } #if _LIBCPP_STD_VER >= 20 template - shared_ptr(shared_ptr &&__r, T *__p) noexcept : internal(std::move(__r.internal), __p) { + shared_ptr(shared_ptr &&ref, T *ptr) noexcept : internal(std::move(ref.internal), ptr) { } #endif - // Copy constructor, share ownership with __r + // Copy constructor, share ownership with ref template ::value, int>::type = 0> - shared_ptr(const shared_ptr &__r) noexcept : internal(__r.internal) { + shared_ptr(const shared_ptr &ref) noexcept : internal(ref.internal) { // NOLINT: not marked as explicit } - shared_ptr(const shared_ptr &other) : internal(other.internal) { + shared_ptr(const shared_ptr &other) : internal(other.internal) { // NOLINT: not marked as explicit } - // Move constructor, share ownership with __r + // Move constructor, share ownership with ref template ::value, int>::type = 0> - shared_ptr(shared_ptr &&__r) noexcept : internal(std::move(__r.internal)) { + shared_ptr(shared_ptr &&ref) noexcept : internal(std::move(ref.internal)) { // NOLINT: not marked as explicit } - shared_ptr(shared_ptr &&other) : internal(std::move(other.internal)) { + shared_ptr(shared_ptr &&other) : internal(std::move(other.internal)) { // NOLINT: not marked as explicit } // Construct from std::shared_ptr @@ -131,8 +131,8 @@ public: typename std::enable_if::value && std::is_convertible::pointer, T *>::value, int>::type = 0> - shared_ptr &operator=(unique_ptr &&__r) { - shared_ptr(std::move(__r)).swap(*this); + shared_ptr &operator=(unique_ptr &&ref) { + shared_ptr(std::move(ref)).swap(*this); return *this; } @@ -141,7 +141,7 @@ public: [[clang::reinitializes]] #endif void - reset() { + reset() { // NOLINT: invalid case style internal.reset(); } #ifdef DUCKDB_CLANG_TIDY @@ -149,15 +149,15 @@ public: [[clang::reinitializes]] #endif template - void reset(U *ptr) { + void reset(U *ptr) { // NOLINT: invalid case style internal.reset(ptr); } #ifdef DUCKDB_CLANG_TIDY // This is necessary to tell clang-tidy that it reinitializes the variable after a move [[clang::reinitializes]] #endif - template - void reset(U *ptr, Deleter deleter) { + template + void reset(U *ptr, DELETER deleter) { // NOLINT: invalid case style internal.reset(ptr, deleter); } @@ -236,12 +236,12 @@ private: template *>::value, int>::type = 0> - void __enable_weak_this(const enable_shared_from_this *__e, _OrigPtr *__ptr) noexcept { + void __enable_weak_this(const enable_shared_from_this *object, _OrigPtr *ptr) noexcept { typedef typename std::remove_cv::type NonConstU; - if (__e && __e->__weak_this_.expired()) { + if (object && object->__weak_this_.expired()) { // __weak_this__ is the mutable variable returned by 'shared_from_this' // it is initialized here - __e->__weak_this_ = shared_ptr(*this, const_cast(static_cast(__ptr))); + object->__weak_this_ = shared_ptr(*this, const_cast(static_cast(ptr))); } } diff --git a/src/include/duckdb/common/weak_ptr.ipp b/src/include/duckdb/common/weak_ptr.ipp index aae7c95dabd6..fff31e251e04 100644 --- a/src/include/duckdb/common/weak_ptr.ipp +++ b/src/include/duckdb/common/weak_ptr.ipp @@ -1,7 +1,7 @@ namespace duckdb { template -class weak_ptr { +class weak_ptr { // NOLINT: invalid case style public: using original = std::weak_ptr; using element_type = typename original::element_type; @@ -23,13 +23,13 @@ public: typename std::enable_if::value, int>::type = 0) noexcept : internal(ptr.internal) { } - weak_ptr(weak_ptr const &other) noexcept : internal(other.internal) { + weak_ptr(weak_ptr const &other) noexcept : internal(other.internal) { // NOLINT: not marked as explicit } template weak_ptr(weak_ptr const &ptr, typename std::enable_if::value, int>::type = 0) noexcept : internal(ptr.internal) { } - weak_ptr(weak_ptr &&ptr) noexcept : internal(std::move(ptr.internal)) { + weak_ptr(weak_ptr &&ptr) noexcept : internal(std::move(ptr.internal)) { // NOLINT: not marked as explicit } template weak_ptr(weak_ptr &&ptr, typename std::enable_if::value, int>::type = 0) noexcept @@ -51,20 +51,20 @@ public: } // Modifiers - void reset() { + void reset() { // NOLINT: invalid case style internal.reset(); } // Observers - long use_count() const { + long use_count() const { // NOLINT: invalid case style return internal.use_count(); } - bool expired() const { + bool expired() const { // NOLINT: invalid case style return internal.expired(); } - shared_ptr lock() const { + shared_ptr lock() const { // NOLINT: invalid case style return shared_ptr(internal.lock()); } From 3a73a47be13f0bc856965e75ce7d9ce8ddf6c5ba Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 11 Apr 2024 21:53:25 +0200 Subject: [PATCH 357/603] We don't need to unpin these while scanning --- .../csv_scanner/buffer_manager/csv_buffer_manager.cpp | 4 +++- src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp | 3 ++- .../execution/operator/csv_scanner/csv_buffer_manager.hpp | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp index 66a6e6ab3cec..644bf0540566 100644 --- a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp +++ b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp @@ -71,7 +71,9 @@ shared_ptr CSVBufferManager::GetBuffer(const idx_t pos) { done = true; } } - if (pos != 0) { + if (pos != 0 && sniffing) { + // We don't need to unpin the buffers here if we are not sniffing since we + // control it per-thread on the scan if (cached_buffers[pos - 1]) { cached_buffers[pos - 1]->Unpin(); } diff --git a/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp b/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp index 057120591132..dec1458bb793 100644 --- a/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp +++ b/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp @@ -83,6 +83,7 @@ void CSVSniffer::SetResultOptions() { } SnifferResult CSVSniffer::SniffCSV(bool force_match) { + buffer_manager->sniffing = true; // 1. Dialect Detection DetectDialect(); // 2. Type Detection @@ -100,7 +101,7 @@ SnifferResult CSVSniffer::SniffCSV(bool force_match) { if (!buffer_manager->file_handle->uncompressed) { buffer_manager->ResetBufferManager(); } - + buffer_manager->sniffing = false; if (!best_candidate->error_handler->errors.empty() && !options.ignore_errors) { for (auto &error_vector : best_candidate->error_handler->errors) { for (auto &error : error_vector.second) { diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp index f8c6f246c3e6..efc40e724f5c 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp @@ -47,6 +47,7 @@ class CSVBufferManager { ClientContext &context; idx_t skip_rows = 0; + bool sniffing; private: //! Reads next buffer in reference to cached_buffers.front() From 0d77ca409657e7a9e25ee8da812169e973b3470a Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 11 Apr 2024 22:58:23 +0200 Subject: [PATCH 358/603] Fix shell tests --- tools/shell/tests/test_shell_basics.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tools/shell/tests/test_shell_basics.py b/tools/shell/tests/test_shell_basics.py index 41d605017079..7fecae9cd1b0 100644 --- a/tools/shell/tests/test_shell_basics.py +++ b/tools/shell/tests/test_shell_basics.py @@ -326,14 +326,6 @@ def test_show_basic(shell): result = test.run() result.check_stdout("rowseparator") -def test_limit_error(shell): - test = ( - ShellTest(shell) - .statement(".limit length 42") - ) - result = test.run() - result.check_stderr("sqlite3_limit") - def test_timeout(shell): test = ( ShellTest(shell) @@ -1039,8 +1031,6 @@ def test_nullbyte_error_rendering(shell): result.check_stderr('INT32') @pytest.mark.parametrize("stmt", [ - "select decimal_mul(NULL, NULL);", - "select decimal_mul(NULL, i) FROM range(3) t(i);", "select sha3(NULL);" ]) def test_sqlite_udf_null(shell, stmt): From 44025289ee0af2ee8edc313e70529a74fef78c0a Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 11 Apr 2024 23:56:54 +0200 Subject: [PATCH 359/603] Allow it if we can seek --- .../operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp | 2 +- test/sql/tpch/gzip_csv_auto_detect.test_slow | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp index 644bf0540566..691b0f2f5120 100644 --- a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp +++ b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp @@ -71,7 +71,7 @@ shared_ptr CSVBufferManager::GetBuffer(const idx_t pos) { done = true; } } - if (pos != 0 && sniffing) { + if (pos != 0 && (sniffing || file_handle->CanSeek())) { // We don't need to unpin the buffers here if we are not sniffing since we // control it per-thread on the scan if (cached_buffers[pos - 1]) { diff --git a/test/sql/tpch/gzip_csv_auto_detect.test_slow b/test/sql/tpch/gzip_csv_auto_detect.test_slow index f3afcdfaf6ed..6591b6a4ba55 100644 --- a/test/sql/tpch/gzip_csv_auto_detect.test_slow +++ b/test/sql/tpch/gzip_csv_auto_detect.test_slow @@ -4,6 +4,9 @@ require tpch +statement ok +PRAGMA enable_verification + # write lineitem to lineitem.csv.gz statement ok CALL dbgen(sf=1); From ec6186246b94cc7bfb387c0017f0f7c322a4cbdb Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 11 Apr 2024 12:10:24 +0200 Subject: [PATCH 360/603] LogicalDelete: Move from dynamic_cast to Cast --- src/planner/operator/logical_delete.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/planner/operator/logical_delete.cpp b/src/planner/operator/logical_delete.cpp index a028a1ea6f36..950f2eaa2ebb 100644 --- a/src/planner/operator/logical_delete.cpp +++ b/src/planner/operator/logical_delete.cpp @@ -14,7 +14,7 @@ LogicalDelete::LogicalDelete(TableCatalogEntry &table, idx_t table_index) LogicalDelete::LogicalDelete(ClientContext &context, const unique_ptr &table_info) : LogicalOperator(LogicalOperatorType::LOGICAL_DELETE), table(Catalog::GetEntry(context, table_info->catalog, table_info->schema, - dynamic_cast(*table_info).table)) { + table_info->Cast().table)) { } idx_t LogicalDelete::EstimateCardinality(ClientContext &context) { From aac4b382daea9a93baa7af23bf0b4b019c7f4c62 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 11 Apr 2024 12:11:01 +0200 Subject: [PATCH 361/603] LogicalUpdate: Move from dynamic_cast to Cast --- src/planner/operator/logical_update.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/planner/operator/logical_update.cpp b/src/planner/operator/logical_update.cpp index e66dd36d1a9c..edcfd5d891be 100644 --- a/src/planner/operator/logical_update.cpp +++ b/src/planner/operator/logical_update.cpp @@ -12,7 +12,7 @@ LogicalUpdate::LogicalUpdate(TableCatalogEntry &table) LogicalUpdate::LogicalUpdate(ClientContext &context, const unique_ptr &table_info) : LogicalOperator(LogicalOperatorType::LOGICAL_UPDATE), table(Catalog::GetEntry(context, table_info->catalog, table_info->schema, - dynamic_cast(*table_info).table)) { + table_info->Cast().table)) { } idx_t LogicalUpdate::EstimateCardinality(ClientContext &context) { From 55991db6925f17ea7623b8e5b6a9ef540f8945e4 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 11 Apr 2024 12:11:32 +0200 Subject: [PATCH 362/603] LogicalInsert: Move from dynamic_cast to Cast --- src/planner/operator/logical_insert.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/planner/operator/logical_insert.cpp b/src/planner/operator/logical_insert.cpp index 3846ed009742..518661616058 100644 --- a/src/planner/operator/logical_insert.cpp +++ b/src/planner/operator/logical_insert.cpp @@ -14,7 +14,7 @@ LogicalInsert::LogicalInsert(TableCatalogEntry &table, idx_t table_index) LogicalInsert::LogicalInsert(ClientContext &context, const unique_ptr table_info) : LogicalOperator(LogicalOperatorType::LOGICAL_INSERT), table(Catalog::GetEntry(context, table_info->catalog, table_info->schema, - dynamic_cast(*table_info).table)) { + table_info->Cast().table)) { } idx_t LogicalInsert::EstimateCardinality(ClientContext &context) { From 4c78ff0e4ecf83c70b0d73891fa02b755615b380 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 11 Apr 2024 16:53:55 +0200 Subject: [PATCH 363/603] Add stricter runtime check on dynamic_check being a no-op --- src/include/duckdb/common/helper.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/include/duckdb/common/helper.hpp b/src/include/duckdb/common/helper.hpp index d4c07cc47091..c989983ba013 100644 --- a/src/include/duckdb/common/helper.hpp +++ b/src/include/duckdb/common/helper.hpp @@ -217,7 +217,8 @@ bool RefersToSameObject(const T &a, const T &b) { template void DynamicCastCheck(const SRC *source) { #ifndef __APPLE__ - D_ASSERT(dynamic_cast(source)); + // Actual check is on the fact that dynamic_cast and reinterpret_cast are equivalent + D_ASSERT(reinterpret_cast(source) == dynamic_cast(source)); #endif } From 39200d51d683ceb12414cc061129cb0ef706b9bf Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 11 Apr 2024 14:44:23 +0200 Subject: [PATCH 364/603] More dynamic_casts mirroring non-const case --- src/include/duckdb/catalog/catalog_entry.hpp | 2 +- src/include/duckdb/common/allocator.hpp | 2 +- src/include/duckdb/common/extra_type_info.hpp | 2 +- .../common/types/column/partitioned_column_data.hpp | 2 +- .../common/types/row/partitioned_tuple_data.hpp | 2 +- src/include/duckdb/common/types/vector_buffer.hpp | 2 +- .../duckdb/execution/expression_executor_state.hpp | 2 +- .../duckdb/execution/physical_operator_states.hpp | 12 ++++++------ src/include/duckdb/execution/window_executor.hpp | 2 +- src/include/duckdb/execution/window_segment_tree.hpp | 2 +- src/include/duckdb/function/cast/default_casts.hpp | 4 ++-- src/include/duckdb/function/compression_function.hpp | 8 ++++---- src/include/duckdb/function/copy_function.hpp | 6 +++--- src/include/duckdb/function/function.hpp | 4 ++-- src/include/duckdb/function/scalar_function.hpp | 2 +- src/include/duckdb/function/table_function.hpp | 6 +++--- src/include/duckdb/main/relation.hpp | 2 +- src/include/duckdb/parallel/event.hpp | 2 +- .../duckdb/parser/parsed_data/extra_drop_info.hpp | 2 +- src/include/duckdb/parser/parsed_data/parse_info.hpp | 2 +- src/include/duckdb/storage/data_pointer.hpp | 2 +- src/include/duckdb/storage/index.hpp | 2 +- src/include/duckdb/storage/storage_manager.hpp | 2 +- .../duckdb/storage/table/column_checkpoint_state.hpp | 2 +- src/include/duckdb/storage/table/scan_state.hpp | 4 ++-- src/include/duckdb/transaction/transaction.hpp | 2 +- 26 files changed, 41 insertions(+), 41 deletions(-) diff --git a/src/include/duckdb/catalog/catalog_entry.hpp b/src/include/duckdb/catalog/catalog_entry.hpp index fd0ed6e222b2..29c46fa10578 100644 --- a/src/include/duckdb/catalog/catalog_entry.hpp +++ b/src/include/duckdb/catalog/catalog_entry.hpp @@ -102,7 +102,7 @@ class CatalogEntry { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; diff --git a/src/include/duckdb/common/allocator.hpp b/src/include/duckdb/common/allocator.hpp index 7c82f049aaa7..ac27d267a9db 100644 --- a/src/include/duckdb/common/allocator.hpp +++ b/src/include/duckdb/common/allocator.hpp @@ -36,7 +36,7 @@ struct PrivateAllocatorData { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; diff --git a/src/include/duckdb/common/extra_type_info.hpp b/src/include/duckdb/common/extra_type_info.hpp index 5157293ca50e..8c8f8c0a62a7 100644 --- a/src/include/duckdb/common/extra_type_info.hpp +++ b/src/include/duckdb/common/extra_type_info.hpp @@ -50,7 +50,7 @@ struct ExtraTypeInfo { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } diff --git a/src/include/duckdb/common/types/column/partitioned_column_data.hpp b/src/include/duckdb/common/types/column/partitioned_column_data.hpp index 70caddbae666..058151cd56e9 100644 --- a/src/include/duckdb/common/types/column/partitioned_column_data.hpp +++ b/src/include/duckdb/common/types/column/partitioned_column_data.hpp @@ -117,7 +117,7 @@ class PartitionedColumnData { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; diff --git a/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp b/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp index 3a4c7f56179b..6ca89aa7ea3c 100644 --- a/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +++ b/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp @@ -181,7 +181,7 @@ class PartitionedTupleData { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; diff --git a/src/include/duckdb/common/types/vector_buffer.hpp b/src/include/duckdb/common/types/vector_buffer.hpp index e1d49aacb629..7108b8dbf185 100644 --- a/src/include/duckdb/common/types/vector_buffer.hpp +++ b/src/include/duckdb/common/types/vector_buffer.hpp @@ -131,7 +131,7 @@ class VectorBuffer { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; diff --git a/src/include/duckdb/execution/expression_executor_state.hpp b/src/include/duckdb/execution/expression_executor_state.hpp index c4bbc40c6b6b..c0802db2a308 100644 --- a/src/include/duckdb/execution/expression_executor_state.hpp +++ b/src/include/duckdb/execution/expression_executor_state.hpp @@ -46,7 +46,7 @@ struct ExpressionState { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; diff --git a/src/include/duckdb/execution/physical_operator_states.hpp b/src/include/duckdb/execution/physical_operator_states.hpp index b0bb166a975d..621c8124338d 100644 --- a/src/include/duckdb/execution/physical_operator_states.hpp +++ b/src/include/duckdb/execution/physical_operator_states.hpp @@ -52,7 +52,7 @@ class OperatorState { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; @@ -69,7 +69,7 @@ class GlobalOperatorState { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; @@ -90,7 +90,7 @@ class GlobalSinkState { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } @@ -114,7 +114,7 @@ class LocalSinkState { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; @@ -135,7 +135,7 @@ class GlobalSourceState { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; @@ -152,7 +152,7 @@ class LocalSourceState { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; diff --git a/src/include/duckdb/execution/window_executor.hpp b/src/include/duckdb/execution/window_executor.hpp index 9a5051fa5fc0..8d134d05e188 100644 --- a/src/include/duckdb/execution/window_executor.hpp +++ b/src/include/duckdb/execution/window_executor.hpp @@ -131,7 +131,7 @@ class WindowExecutorState { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; diff --git a/src/include/duckdb/execution/window_segment_tree.hpp b/src/include/duckdb/execution/window_segment_tree.hpp index 0cfbff2874b7..bcdf87439f12 100644 --- a/src/include/duckdb/execution/window_segment_tree.hpp +++ b/src/include/duckdb/execution/window_segment_tree.hpp @@ -31,7 +31,7 @@ class WindowAggregatorState { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } diff --git a/src/include/duckdb/function/cast/default_casts.hpp b/src/include/duckdb/function/cast/default_casts.hpp index e3d6072d5a2c..5d13c4354f58 100644 --- a/src/include/duckdb/function/cast/default_casts.hpp +++ b/src/include/duckdb/function/cast/default_casts.hpp @@ -30,7 +30,7 @@ struct BindCastInfo { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; @@ -48,7 +48,7 @@ struct BoundCastData { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; diff --git a/src/include/duckdb/function/compression_function.hpp b/src/include/duckdb/function/compression_function.hpp index 4095e1826229..e34e2d0c35ef 100644 --- a/src/include/duckdb/function/compression_function.hpp +++ b/src/include/duckdb/function/compression_function.hpp @@ -39,7 +39,7 @@ struct AnalyzeState { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; @@ -55,7 +55,7 @@ struct CompressionState { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; @@ -76,7 +76,7 @@ struct CompressedSegmentState { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; @@ -96,7 +96,7 @@ struct CompressionAppendState { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; diff --git a/src/include/duckdb/function/copy_function.hpp b/src/include/duckdb/function/copy_function.hpp index 99a01f8e0ba1..7e3a4808b435 100644 --- a/src/include/duckdb/function/copy_function.hpp +++ b/src/include/duckdb/function/copy_function.hpp @@ -30,7 +30,7 @@ struct LocalFunctionData { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; @@ -45,7 +45,7 @@ struct GlobalFunctionData { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; @@ -60,7 +60,7 @@ struct PreparedBatchData { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; diff --git a/src/include/duckdb/function/function.hpp b/src/include/duckdb/function/function.hpp index 65a186f0525d..7e89476e2526 100644 --- a/src/include/duckdb/function/function.hpp +++ b/src/include/duckdb/function/function.hpp @@ -58,13 +58,13 @@ struct FunctionData { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } // FIXME: this function should be removed in the future template TARGET &CastNoConst() const { - return const_cast(reinterpret_cast(*this)); // NOLINT: FIXME + return const_cast(Cast()); // NOLINT: FIXME } }; diff --git a/src/include/duckdb/function/scalar_function.hpp b/src/include/duckdb/function/scalar_function.hpp index ab65c97a2139..917f09eed2b4 100644 --- a/src/include/duckdb/function/scalar_function.hpp +++ b/src/include/duckdb/function/scalar_function.hpp @@ -29,7 +29,7 @@ struct FunctionLocalState { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; diff --git a/src/include/duckdb/function/table_function.hpp b/src/include/duckdb/function/table_function.hpp index 3321d274f902..bd8e176973ee 100644 --- a/src/include/duckdb/function/table_function.hpp +++ b/src/include/duckdb/function/table_function.hpp @@ -36,7 +36,7 @@ struct TableFunctionInfo { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; @@ -60,7 +60,7 @@ struct GlobalTableFunctionState { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; @@ -75,7 +75,7 @@ struct LocalTableFunctionState { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; diff --git a/src/include/duckdb/main/relation.hpp b/src/include/duckdb/main/relation.hpp index 7d1798712975..c16cb2a3e829 100644 --- a/src/include/duckdb/main/relation.hpp +++ b/src/include/duckdb/main/relation.hpp @@ -185,7 +185,7 @@ class Relation : public std::enable_shared_from_this { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; diff --git a/src/include/duckdb/parallel/event.hpp b/src/include/duckdb/parallel/event.hpp index 89a108d98a98..794d1344f1a6 100644 --- a/src/include/duckdb/parallel/event.hpp +++ b/src/include/duckdb/parallel/event.hpp @@ -59,7 +59,7 @@ class Event : public std::enable_shared_from_this { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } diff --git a/src/include/duckdb/parser/parsed_data/extra_drop_info.hpp b/src/include/duckdb/parser/parsed_data/extra_drop_info.hpp index b85c6252359f..2812469deb5c 100644 --- a/src/include/duckdb/parser/parsed_data/extra_drop_info.hpp +++ b/src/include/duckdb/parser/parsed_data/extra_drop_info.hpp @@ -38,7 +38,7 @@ struct ExtraDropInfo { template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } virtual unique_ptr Copy() const = 0; diff --git a/src/include/duckdb/parser/parsed_data/parse_info.hpp b/src/include/duckdb/parser/parsed_data/parse_info.hpp index d547065e9ef7..5d395c6adfcf 100644 --- a/src/include/duckdb/parser/parsed_data/parse_info.hpp +++ b/src/include/duckdb/parser/parsed_data/parse_info.hpp @@ -48,7 +48,7 @@ struct ParseInfo { template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } diff --git a/src/include/duckdb/storage/data_pointer.hpp b/src/include/duckdb/storage/data_pointer.hpp index c0c51df679aa..97752ee5e141 100644 --- a/src/include/duckdb/storage/data_pointer.hpp +++ b/src/include/duckdb/storage/data_pointer.hpp @@ -34,7 +34,7 @@ struct ColumnSegmentState { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; diff --git a/src/include/duckdb/storage/index.hpp b/src/include/duckdb/storage/index.hpp index 179735e8c117..f5e89486b28f 100644 --- a/src/include/duckdb/storage/index.hpp +++ b/src/include/duckdb/storage/index.hpp @@ -160,7 +160,7 @@ class Index { template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; diff --git a/src/include/duckdb/storage/storage_manager.hpp b/src/include/duckdb/storage/storage_manager.hpp index 91fc96755719..e0c07b6b8903 100644 --- a/src/include/duckdb/storage/storage_manager.hpp +++ b/src/include/duckdb/storage/storage_manager.hpp @@ -96,7 +96,7 @@ class StorageManager { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; diff --git a/src/include/duckdb/storage/table/column_checkpoint_state.hpp b/src/include/duckdb/storage/table/column_checkpoint_state.hpp index 5ac11cf7cc28..1c1a68432375 100644 --- a/src/include/duckdb/storage/table/column_checkpoint_state.hpp +++ b/src/include/duckdb/storage/table/column_checkpoint_state.hpp @@ -51,7 +51,7 @@ struct ColumnCheckpointState { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; diff --git a/src/include/duckdb/storage/table/scan_state.hpp b/src/include/duckdb/storage/table/scan_state.hpp index 6c919cfba0ff..7b8160fdc089 100644 --- a/src/include/duckdb/storage/table/scan_state.hpp +++ b/src/include/duckdb/storage/table/scan_state.hpp @@ -44,7 +44,7 @@ struct SegmentScanState { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; @@ -60,7 +60,7 @@ struct IndexScanState { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; diff --git a/src/include/duckdb/transaction/transaction.hpp b/src/include/duckdb/transaction/transaction.hpp index 1c47725c10dd..723c99460c52 100644 --- a/src/include/duckdb/transaction/transaction.hpp +++ b/src/include/duckdb/transaction/transaction.hpp @@ -66,7 +66,7 @@ class Transaction { } template const TARGET &Cast() const { - D_ASSERT(dynamic_cast(this)); + DynamicCastCheck(this); return reinterpret_cast(*this); } }; From a9ef0e20190383425e591cf6feaa715999fdba20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Fri, 12 Apr 2024 07:55:58 +0200 Subject: [PATCH 365/603] forgotten merge conflict --- src/include/duckdb/storage/string_uncompressed.hpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/include/duckdb/storage/string_uncompressed.hpp b/src/include/duckdb/storage/string_uncompressed.hpp index fca222511961..a57b851eb86d 100644 --- a/src/include/duckdb/storage/string_uncompressed.hpp +++ b/src/include/duckdb/storage/string_uncompressed.hpp @@ -162,12 +162,8 @@ struct UncompressedStringStorage { memcpy(dict_pos, source_data[source_idx].GetData(), string_length); // place the dictionary offset into the set of vectors -<<<<<<< HEAD - result_data[target_idx] = NumericCast(*dictionary_size); -======= D_ASSERT(*dictionary_size <= int32_t(Storage::BLOCK_SIZE)); - result_data[target_idx] = *dictionary_size; ->>>>>>> 40e2ff4837e79f2cc75e0d805595158a5409e680 + result_data[target_idx] = NumericCast(*dictionary_size); } D_ASSERT(RemainingSpace(segment, handle) <= Storage::BLOCK_SIZE); #ifdef DEBUG From 507e771f827fea7d219daa269d522654c1c94704 Mon Sep 17 00:00:00 2001 From: Elliana May Date: Thu, 11 Apr 2024 18:29:11 +0800 Subject: [PATCH 366/603] fix: add DuckDBPyType#children implementation for array & enum --- tools/pythonpkg/src/typing/pytype.cpp | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/tools/pythonpkg/src/typing/pytype.cpp b/tools/pythonpkg/src/typing/pytype.cpp index ad9828876b3d..cfcac9104ed7 100644 --- a/tools/pythonpkg/src/typing/pytype.cpp +++ b/tools/pythonpkg/src/typing/pytype.cpp @@ -347,7 +347,8 @@ py::list DuckDBPyType::Children() const { case LogicalTypeId::STRUCT: case LogicalTypeId::UNION: case LogicalTypeId::MAP: - break; + case LogicalTypeId::ARRAY: + case LogicalTypeId::ENUM: case LogicalTypeId::DECIMAL: break; default: @@ -360,8 +361,21 @@ py::list DuckDBPyType::Children() const { children.append(py::make_tuple("child", make_shared(ListType::GetChildType(type)))); return children; } - // FIXME: where is ARRAY?? - // it should expose 'child' and 'size' + if (id == LogicalTypeId::ARRAY) { + children.append(py::make_tuple("child", make_shared(ArrayType::GetChildType(type)))); + children.append(py::make_tuple("size", ArrayType::GetSize(type))); + return children; + } + if (id == LogicalTypeId::ENUM) { + auto &values_insert_order = EnumType::GetValuesInsertOrder(type); + auto strings = FlatVector::GetData(values_insert_order); + py::list strings_list; + for (size_t i = 0; i < EnumType::GetSize(type); i++) { + strings_list.append(py::str(strings[i].GetString())); + } + children.append(py::make_tuple("values", strings_list)); + return children; + } if (id == LogicalTypeId::STRUCT || id == LogicalTypeId::UNION) { auto &struct_children = StructType::GetChildTypes(type); for (idx_t i = 0; i < struct_children.size(); i++) { From 3c4af044f51f0c5f804c2bebf1bac7173fd01fc3 Mon Sep 17 00:00:00 2001 From: Elliana May Date: Thu, 11 Apr 2024 18:44:48 +0800 Subject: [PATCH 367/603] feat: add array support to spark compat --- tools/pythonpkg/duckdb/experimental/spark/sql/type_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/pythonpkg/duckdb/experimental/spark/sql/type_utils.py b/tools/pythonpkg/duckdb/experimental/spark/sql/type_utils.py index 4fe4f8b88b19..a17d0f53d9a9 100644 --- a/tools/pythonpkg/duckdb/experimental/spark/sql/type_utils.py +++ b/tools/pythonpkg/duckdb/experimental/spark/sql/type_utils.py @@ -74,7 +74,7 @@ def convert_nested_type(dtype: DuckDBPyType) -> DataType: id = dtype.id - if id == 'list': + if id == 'list' or id == 'array': children = dtype.children return ArrayType(convert_type(children[0][1])) # TODO: add support for 'union' @@ -89,7 +89,7 @@ def convert_nested_type(dtype: DuckDBPyType) -> DataType: def convert_type(dtype: DuckDBPyType) -> DataType: id = dtype.id - if id in ['list', 'struct', 'map']: + if id in ['list', 'struct', 'map', 'array']: return convert_nested_type(dtype) if id == 'decimal': children: List[Tuple[str, DuckDBPyType]] = dtype.children From 976f43ddb34272541e3384243bb04d21886acccf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Fri, 12 Apr 2024 09:41:21 +0200 Subject: [PATCH 368/603] csv reader use optional idx --- .../csv_scanner/scanner/string_value_scanner.cpp | 6 +++--- .../table_function/global_csv_state.cpp | 4 ++-- .../operator/csv_scanner/util/csv_error.cpp | 14 +++++++------- .../core_functions/aggregate/sum_helpers.hpp | 2 +- .../execution/operator/csv_scanner/csv_error.hpp | 12 ++++++------ src/main/settings/settings.cpp | 1 - 6 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index ad42b6dafaa6..da0f70fea17d 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -777,8 +777,8 @@ void StringValueScanner::Flush(DataChunk &insert_chunk) { auto csv_error = CSVError::CastError( state_machine->options, csv_file_scan->names[col_idx], error_message, col_idx, borked_line, lines_per_batch, - result.line_positions_per_row[line_error].begin.GetGlobalPosition(result.result_size, first_nl), -1, - result_vector.GetType().id()); + result.line_positions_per_row[line_error].begin.GetGlobalPosition(result.result_size, first_nl), + optional_idx::Invalid(), result_vector.GetType().id()); error_handler->Error(csv_error); } @@ -802,7 +802,7 @@ void StringValueScanner::Flush(DataChunk &insert_chunk) { state_machine->options, csv_file_scan->names[col_idx], error_message, col_idx, borked_line, lines_per_batch, result.line_positions_per_row[line_error].begin.GetGlobalPosition(result.result_size, first_nl), - -1, result_vector.GetType().id()); + optional_idx::Invalid(), result_vector.GetType().id()); error_handler->Error(csv_error); } } diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index a56a9e687804..9cfc08d42084 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -289,12 +289,12 @@ void CSVGlobalState::FillRejectsTable() { // 4. Byte Position of the row error errors_appender.Append(error.row_byte_position + 1); // 5. Byte Position where error occurred - if (error.byte_position == -1) { + if (!error.byte_position.IsValid()) { // This means this error comes from a flush, and we don't support this yet, so we give it // a null errors_appender.Append(Value()); } else { - errors_appender.Append(error.byte_position + 1); + errors_appender.Append(error.byte_position.GetIndex() + 1); } // 6. Column Index if (error.type == CSVErrorType::MAXIMUM_LINE_SIZE) { diff --git a/src/execution/operator/csv_scanner/util/csv_error.cpp b/src/execution/operator/csv_scanner/util/csv_error.cpp index d22738f21415..208c23371183 100644 --- a/src/execution/operator/csv_scanner/util/csv_error.cpp +++ b/src/execution/operator/csv_scanner/util/csv_error.cpp @@ -87,7 +87,7 @@ CSVError::CSVError(string error_message_p, CSVErrorType type_p, LinesPerBoundary } CSVError::CSVError(string error_message_p, CSVErrorType type_p, idx_t column_idx_p, string csv_row_p, - LinesPerBoundary error_info_p, idx_t row_byte_position, int64_t byte_position_p, + LinesPerBoundary error_info_p, idx_t row_byte_position, optional_idx byte_position_p, const CSVReaderOptions &reader_options, const string &fixes) : error_message(std::move(error_message_p)), type(type_p), column_idx(column_idx_p), csv_row(std::move(csv_row_p)), error_info(error_info_p), row_byte_position(row_byte_position), byte_position(byte_position_p) { @@ -129,7 +129,7 @@ void CSVError::RemoveNewLine(string &error) { CSVError CSVError::CastError(const CSVReaderOptions &options, string &column_name, string &cast_error, idx_t column_idx, string &csv_row, LinesPerBoundary error_info, idx_t row_byte_position, - int64_t byte_position, LogicalTypeId type) { + optional_idx byte_position, LogicalTypeId type) { std::ostringstream error; // Which column error << "Error when converting column \"" << column_name << "\". "; @@ -192,7 +192,7 @@ CSVError CSVError::NullPaddingFail(const CSVReaderOptions &options, LinesPerBoun CSVError CSVError::UnterminatedQuotesError(const CSVReaderOptions &options, idx_t current_column, LinesPerBoundary error_info, string &csv_row, idx_t row_byte_position, - int64_t byte_position) { + optional_idx byte_position) { std::ostringstream error; error << "Value with unterminated quote found." << '\n'; std::ostringstream how_to_fix_it; @@ -203,7 +203,7 @@ CSVError CSVError::UnterminatedQuotesError(const CSVReaderOptions &options, idx_ CSVError CSVError::IncorrectColumnAmountError(const CSVReaderOptions &options, idx_t actual_columns, LinesPerBoundary error_info, string &csv_row, idx_t row_byte_position, - int64_t byte_position) { + optional_idx byte_position) { std::ostringstream error; // We don't have a fix for this std::ostringstream how_to_fix_it; @@ -218,15 +218,15 @@ CSVError CSVError::IncorrectColumnAmountError(const CSVReaderOptions &options, i error << "Expected Number of Columns: " << options.dialect_options.num_cols << " Found: " << actual_columns + 1; if (actual_columns >= options.dialect_options.num_cols) { return CSVError(error.str(), CSVErrorType::TOO_MANY_COLUMNS, actual_columns, csv_row, error_info, - row_byte_position, byte_position - 1, options, how_to_fix_it.str()); + row_byte_position, byte_position.GetIndex() - 1, options, how_to_fix_it.str()); } else { return CSVError(error.str(), CSVErrorType::TOO_FEW_COLUMNS, actual_columns, csv_row, error_info, - row_byte_position, byte_position - 1, options, how_to_fix_it.str()); + row_byte_position, byte_position.GetIndex() - 1, options, how_to_fix_it.str()); } } CSVError CSVError::InvalidUTF8(const CSVReaderOptions &options, idx_t current_column, LinesPerBoundary error_info, - string &csv_row, idx_t row_byte_position, int64_t byte_position) { + string &csv_row, idx_t row_byte_position, optional_idx byte_position) { std::ostringstream error; // How many columns were expected and how many were found error << "Invalid unicode (byte sequence mismatch) detected." << '\n'; diff --git a/src/include/duckdb/core_functions/aggregate/sum_helpers.hpp b/src/include/duckdb/core_functions/aggregate/sum_helpers.hpp index 554ef2390faa..562f61ade356 100644 --- a/src/include/duckdb/core_functions/aggregate/sum_helpers.hpp +++ b/src/include/duckdb/core_functions/aggregate/sum_helpers.hpp @@ -73,7 +73,7 @@ struct HugeintAdd { template static void AddConstant(STATE &state, T input, idx_t count) { - AddNumber(state, Hugeint::Multiply(input, count)); + AddNumber(state, Hugeint::Multiply(input, UnsafeNumericCast(count))); } }; diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp index 340c42cd1c40..5f3ef031178a 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp @@ -52,7 +52,7 @@ class CSVError { public: CSVError() {}; CSVError(string error_message, CSVErrorType type, idx_t column_idx, string csv_row, LinesPerBoundary error_info, - idx_t row_byte_position, int64_t byte_position, const CSVReaderOptions &reader_options, + idx_t row_byte_position, optional_idx byte_position, const CSVReaderOptions &reader_options, const string &fixes); CSVError(string error_message, CSVErrorType type, LinesPerBoundary error_info); //! Produces error messages for column name -> type mismatch. @@ -60,7 +60,7 @@ class CSVError { //! Produces error messages for casting errors static CSVError CastError(const CSVReaderOptions &options, string &column_name, string &cast_error, idx_t column_idx, string &csv_row, LinesPerBoundary error_info, idx_t row_byte_position, - int64_t byte_position, LogicalTypeId type); + optional_idx byte_position, LogicalTypeId type); //! Produces error for when the line size exceeds the maximum line size option static CSVError LineSizeError(const CSVReaderOptions &options, idx_t actual_size, LinesPerBoundary error_info, string &csv_row, idx_t byte_position); @@ -69,15 +69,15 @@ class CSVError { //! Produces error messages for unterminated quoted values static CSVError UnterminatedQuotesError(const CSVReaderOptions &options, idx_t current_column, LinesPerBoundary error_info, string &csv_row, idx_t row_byte_position, - int64_t byte_position); + optional_idx byte_position); //! Produces error messages for null_padding option is set and we have quoted new values in parallel static CSVError NullPaddingFail(const CSVReaderOptions &options, LinesPerBoundary error_info); //! Produces error for incorrect (e.g., smaller and lower than the predefined) number of columns in a CSV Line static CSVError IncorrectColumnAmountError(const CSVReaderOptions &state_machine, idx_t actual_columns, LinesPerBoundary error_info, string &csv_row, idx_t row_byte_position, - int64_t byte_position); + optional_idx byte_position); static CSVError InvalidUTF8(const CSVReaderOptions &options, idx_t current_column, LinesPerBoundary error_info, - string &csv_row, idx_t row_byte_position, int64_t byte_position); + string &csv_row, idx_t row_byte_position, optional_idx byte_position); idx_t GetBoundaryIndex() { return error_info.boundary_idx; @@ -104,7 +104,7 @@ class CSVError { //! Byte position of where the row starts idx_t row_byte_position; //! Byte Position where error occurred. - int64_t byte_position; + optional_idx byte_position; }; class CSVErrorHandler { diff --git a/src/main/settings/settings.cpp b/src/main/settings/settings.cpp index aecc7e767330..6209c0c1543d 100644 --- a/src/main/settings/settings.cpp +++ b/src/main/settings/settings.cpp @@ -999,7 +999,6 @@ void PartitionedWriteFlushThreshold::SetLocal(ClientContext &context, const Valu ClientConfig::GetConfig(context).partitioned_write_flush_threshold = input.GetValue(); } - Value PartitionedWriteFlushThreshold::GetSetting(const ClientContext &context) { return Value::BIGINT(NumericCast(ClientConfig::GetConfig(context).partitioned_write_flush_threshold)); } From 2973658f019928cbef971e25b768811877c84840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Fri, 12 Apr 2024 10:09:48 +0200 Subject: [PATCH 369/603] compression code yay --- .../duckdb/planner/operator/logical_top_n.hpp | 6 +++--- .../storage/compression/alp/algorithm/alp.hpp | 5 +++-- .../storage/compression/alp/alp_compress.hpp | 4 ++-- .../duckdb/storage/compression/alp/alp_fetch.hpp | 2 +- .../duckdb/storage/compression/alp/alp_utils.hpp | 4 ++-- .../storage/compression/alprd/alprd_compress.hpp | 4 ++-- .../storage/compression/alprd/alprd_fetch.hpp | 2 +- .../storage/compression/chimp/chimp_fetch.hpp | 2 +- .../storage/compression/patas/patas_fetch.hpp | 2 +- src/optimizer/topn_optimizer.cpp | 6 +++--- src/storage/compression/fsst.cpp | 16 ++++++++++------ src/storage/metadata/metadata_manager.cpp | 4 ++-- src/storage/metadata/metadata_writer.cpp | 2 +- 13 files changed, 32 insertions(+), 27 deletions(-) diff --git a/src/include/duckdb/planner/operator/logical_top_n.hpp b/src/include/duckdb/planner/operator/logical_top_n.hpp index 745ca6292ca8..cc19ea6b7ff6 100644 --- a/src/include/duckdb/planner/operator/logical_top_n.hpp +++ b/src/include/duckdb/planner/operator/logical_top_n.hpp @@ -19,15 +19,15 @@ class LogicalTopN : public LogicalOperator { static constexpr const LogicalOperatorType TYPE = LogicalOperatorType::LOGICAL_TOP_N; public: - LogicalTopN(vector orders, int64_t limit, int64_t offset) + LogicalTopN(vector orders, idx_t limit, idx_t offset) : LogicalOperator(LogicalOperatorType::LOGICAL_TOP_N), orders(std::move(orders)), limit(limit), offset(offset) { } vector orders; //! The maximum amount of elements to emit - int64_t limit; + idx_t limit; //! The offset from the start to begin emitting elements - int64_t offset; + idx_t offset; public: vector GetColumnBindings() override { diff --git a/src/include/duckdb/storage/compression/alp/algorithm/alp.hpp b/src/include/duckdb/storage/compression/alp/algorithm/alp.hpp index d71189ded9a3..d216f23e58af 100644 --- a/src/include/duckdb/storage/compression/alp/algorithm/alp.hpp +++ b/src/include/duckdb/storage/compression/alp/algorithm/alp.hpp @@ -254,7 +254,8 @@ struct AlpCompression { static void FindBestFactorAndExponent(const T *input_vector, idx_t n_values, State &state) { //! We sample equidistant values within a vector; to do this we skip a fixed number of values vector vector_sample; - uint32_t idx_increments = MaxValue(1, (int32_t)std::ceil((double)n_values / AlpConstants::SAMPLES_PER_VECTOR)); + auto idx_increments = MaxValue( + 1, UnsafeNumericCast(std::ceil((double)n_values / AlpConstants::SAMPLES_PER_VECTOR))); for (idx_t i = 0; i < n_values; i += idx_increments) { vector_sample.push_back(input_vector[i]); } @@ -360,7 +361,7 @@ struct AlpCompression { } state.bit_width = bit_width; // in bits state.bp_size = bp_size; // in bytes - state.frame_of_reference = min_value; + state.frame_of_reference = static_cast(min_value); // understood this can be negative } /* diff --git a/src/include/duckdb/storage/compression/alp/alp_compress.hpp b/src/include/duckdb/storage/compression/alp/alp_compress.hpp index fc7d88694df3..e4021d939190 100644 --- a/src/include/duckdb/storage/compression/alp/alp_compress.hpp +++ b/src/include/duckdb/storage/compression/alp/alp_compress.hpp @@ -183,10 +183,10 @@ struct AlpCompressionState : public CompressionState { // Verify that the metadata_ptr is not smaller than the space used by the data D_ASSERT(dataptr + metadata_offset <= metadata_ptr); - idx_t bytes_used_by_metadata = dataptr + Storage::BLOCK_SIZE - metadata_ptr; + auto bytes_used_by_metadata = UnsafeNumericCast(dataptr + Storage::BLOCK_SIZE - metadata_ptr); // Initially the total segment size is the size of the block - idx_t total_segment_size = Storage::BLOCK_SIZE; + auto total_segment_size = Storage::BLOCK_SIZE; //! We compact the block if the space used is less than a threshold const auto used_space_percentage = diff --git a/src/include/duckdb/storage/compression/alp/alp_fetch.hpp b/src/include/duckdb/storage/compression/alp/alp_fetch.hpp index 81d3d2ac0989..54a9964d4ad7 100644 --- a/src/include/duckdb/storage/compression/alp/alp_fetch.hpp +++ b/src/include/duckdb/storage/compression/alp/alp_fetch.hpp @@ -28,7 +28,7 @@ void AlpFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id, using EXACT_TYPE = typename FloatingToExact::TYPE; AlpScanState scan_state(segment); - scan_state.Skip(segment, row_id); + scan_state.Skip(segment, UnsafeNumericCast(row_id)); auto result_data = FlatVector::GetData(result); result_data[result_idx] = (EXACT_TYPE)0; diff --git a/src/include/duckdb/storage/compression/alp/alp_utils.hpp b/src/include/duckdb/storage/compression/alp/alp_utils.hpp index 1b77e219e489..b5e49a6f3027 100644 --- a/src/include/duckdb/storage/compression/alp/alp_utils.hpp +++ b/src/include/duckdb/storage/compression/alp/alp_utils.hpp @@ -40,8 +40,8 @@ class AlpUtils { auto n_lookup_values = NumericCast(MinValue(current_vector_n_values, (idx_t)AlpConstants::ALP_VECTOR_SIZE)); //! We sample equidistant values within a vector; to do this we jump a fixed number of values - uint32_t n_sampled_increments = - MaxValue(1, (int32_t)std::ceil((double)n_lookup_values / AlpConstants::SAMPLES_PER_VECTOR)); + uint32_t n_sampled_increments = MaxValue( + 1, UnsafeNumericCast(std::ceil((double)n_lookup_values / AlpConstants::SAMPLES_PER_VECTOR))); uint32_t n_sampled_values = std::ceil((double)n_lookup_values / n_sampled_increments); D_ASSERT(n_sampled_values < AlpConstants::ALP_VECTOR_SIZE); diff --git a/src/include/duckdb/storage/compression/alprd/alprd_compress.hpp b/src/include/duckdb/storage/compression/alprd/alprd_compress.hpp index 5ff07dd7bf9a..3f2a8aca329d 100644 --- a/src/include/duckdb/storage/compression/alprd/alprd_compress.hpp +++ b/src/include/duckdb/storage/compression/alprd/alprd_compress.hpp @@ -185,10 +185,10 @@ struct AlpRDCompressionState : public CompressionState { // Verify that the metadata_ptr is not smaller than the space used by the data D_ASSERT(dataptr + metadata_offset <= metadata_ptr); - idx_t bytes_used_by_metadata = dataptr + Storage::BLOCK_SIZE - metadata_ptr; + auto bytes_used_by_metadata = UnsafeNumericCast(dataptr + Storage::BLOCK_SIZE - metadata_ptr); // Initially the total segment size is the size of the block - idx_t total_segment_size = Storage::BLOCK_SIZE; + auto total_segment_size = Storage::BLOCK_SIZE; //! We compact the block if the space used is less than a threshold const auto used_space_percentage = diff --git a/src/include/duckdb/storage/compression/alprd/alprd_fetch.hpp b/src/include/duckdb/storage/compression/alprd/alprd_fetch.hpp index 0128b8db683e..35923019877c 100644 --- a/src/include/duckdb/storage/compression/alprd/alprd_fetch.hpp +++ b/src/include/duckdb/storage/compression/alprd/alprd_fetch.hpp @@ -27,7 +27,7 @@ template void AlpRDFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id, Vector &result, idx_t result_idx) { using EXACT_TYPE = typename FloatingToExact::TYPE; AlpRDScanState scan_state(segment); - scan_state.Skip(segment, row_id); + scan_state.Skip(segment, UnsafeNumericCast(row_id)); auto result_data = FlatVector::GetData(result); result_data[result_idx] = (EXACT_TYPE)0; diff --git a/src/include/duckdb/storage/compression/chimp/chimp_fetch.hpp b/src/include/duckdb/storage/compression/chimp/chimp_fetch.hpp index c8e4edb9b36a..5d6be51e64cb 100644 --- a/src/include/duckdb/storage/compression/chimp/chimp_fetch.hpp +++ b/src/include/duckdb/storage/compression/chimp/chimp_fetch.hpp @@ -29,7 +29,7 @@ void ChimpFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id using INTERNAL_TYPE = typename ChimpType::TYPE; ChimpScanState scan_state(segment); - scan_state.Skip(segment, row_id); + scan_state.Skip(segment, UnsafeNumericCast(row_id)); auto result_data = FlatVector::GetData(result); if (scan_state.GroupFinished() && scan_state.total_value_count < scan_state.segment_count) { diff --git a/src/include/duckdb/storage/compression/patas/patas_fetch.hpp b/src/include/duckdb/storage/compression/patas/patas_fetch.hpp index fd416cfc6aa9..8e20ae67fe4e 100644 --- a/src/include/duckdb/storage/compression/patas/patas_fetch.hpp +++ b/src/include/duckdb/storage/compression/patas/patas_fetch.hpp @@ -29,7 +29,7 @@ void PatasFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id using EXACT_TYPE = typename FloatingToExact::TYPE; PatasScanState scan_state(segment); - scan_state.Skip(segment, row_id); + scan_state.Skip(segment, UnsafeNumericCast(row_id)); auto result_data = FlatVector::GetData(result); result_data[result_idx] = (EXACT_TYPE)0; diff --git a/src/optimizer/topn_optimizer.cpp b/src/optimizer/topn_optimizer.cpp index 26f6ca99506e..5e83f0a05f79 100644 --- a/src/optimizer/topn_optimizer.cpp +++ b/src/optimizer/topn_optimizer.cpp @@ -29,10 +29,10 @@ unique_ptr TopN::Optimize(unique_ptr op) { if (CanOptimize(*op)) { auto &limit = op->Cast(); auto &order_by = (op->children[0])->Cast(); - auto limit_val = int64_t(limit.limit_val.GetConstantValue()); - int64_t offset_val = 0; + auto limit_val = NumericCast(limit.limit_val.GetConstantValue()); + idx_t offset_val = 0; if (limit.offset_val.Type() == LimitNodeType::CONSTANT_VALUE) { - offset_val = NumericCast(limit.offset_val.GetConstantValue()); + offset_val = NumericCast(limit.offset_val.GetConstantValue()); } auto topn = make_uniq(std::move(order_by.orders), limit_val, offset_val); topn->AddChild(std::move(order_by.children[0])); diff --git a/src/storage/compression/fsst.cpp b/src/storage/compression/fsst.cpp index 6c9562366f0c..02474963f31d 100644 --- a/src/storage/compression/fsst.cpp +++ b/src/storage/compression/fsst.cpp @@ -178,7 +178,8 @@ idx_t FSSTStorage::StringFinalAnalyze(AnalyzeState &state_p) { compressed_dict_size += size; max_compressed_string_length = MaxValue(max_compressed_string_length, size); } - D_ASSERT(compressed_dict_size == (compressed_ptrs[res - 1] - compressed_ptrs[0]) + compressed_sizes[res - 1]); + D_ASSERT(compressed_dict_size == + (uint64_t)(compressed_ptrs[res - 1] - compressed_ptrs[0]) + compressed_sizes[res - 1]); auto minimum_width = BitpackingPrimitives::MinimumBitWidth(max_compressed_string_length); auto bitpacked_offsets_size = @@ -606,7 +607,8 @@ void FSSTStorage::StringScanPartial(ColumnSegment &segment, ColumnScanState &sta for (idx_t i = 0; i < scan_count; i++) { uint32_t string_length = bitunpack_buffer[i + offsets.scan_offset]; result_data[i] = UncompressedStringStorage::FetchStringFromDict( - segment, dict, result, baseptr, delta_decode_buffer[i + offsets.unused_delta_decoded_values], + segment, dict, result, baseptr, + UnsafeNumericCast(delta_decode_buffer[i + offsets.unused_delta_decoded_values]), string_length); FSSTVector::SetCount(result, scan_count); } @@ -615,7 +617,8 @@ void FSSTStorage::StringScanPartial(ColumnSegment &segment, ColumnScanState &sta for (idx_t i = 0; i < scan_count; i++) { uint32_t str_len = bitunpack_buffer[i + offsets.scan_offset]; auto str_ptr = FSSTStorage::FetchStringPointer( - dict, baseptr, delta_decode_buffer[i + offsets.unused_delta_decoded_values]); + dict, baseptr, + UnsafeNumericCast(delta_decode_buffer[i + offsets.unused_delta_decoded_values])); if (str_len > 0) { result_data[i + result_offset] = @@ -627,7 +630,7 @@ void FSSTStorage::StringScanPartial(ColumnSegment &segment, ColumnScanState &sta } scan_state.StoreLastDelta(delta_decode_buffer[scan_count + offsets.unused_delta_decoded_values - 1], - start + scan_count - 1); + UnsafeNumericCast(start + scan_count - 1)); } void FSSTStorage::StringScan(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result) { @@ -655,7 +658,7 @@ void FSSTStorage::StringFetchRow(ColumnSegment &segment, ColumnFetchState &state if (have_symbol_table) { // We basically just do a scan of 1 which is kinda expensive as we need to repeatedly delta decode until we // reach the row we want, we could consider a more clever caching trick if this is slow - auto offsets = CalculateBpDeltaOffsets(-1, row_id, 1); + auto offsets = CalculateBpDeltaOffsets(-1, UnsafeNumericCast(row_id), 1); auto bitunpack_buffer = unique_ptr(new uint32_t[offsets.total_bitunpack_count]); BitUnpackRange(base_data, data_ptr_cast(bitunpack_buffer.get()), offsets.total_bitunpack_count, @@ -667,7 +670,8 @@ void FSSTStorage::StringFetchRow(ColumnSegment &segment, ColumnFetchState &state uint32_t string_length = bitunpack_buffer[offsets.scan_offset]; string_t compressed_string = UncompressedStringStorage::FetchStringFromDict( - segment, dict, result, base_ptr, delta_decode_buffer[offsets.unused_delta_decoded_values], string_length); + segment, dict, result, base_ptr, + UnsafeNumericCast(delta_decode_buffer[offsets.unused_delta_decoded_values]), string_length); result_data[result_idx] = FSSTPrimitives::DecompressValue((void *)&decoder, result, compressed_string.GetData(), compressed_string.GetSize()); diff --git a/src/storage/metadata/metadata_manager.cpp b/src/storage/metadata/metadata_manager.cpp index 8d25e037e1b5..2fba4fc1bbc7 100644 --- a/src/storage/metadata/metadata_manager.cpp +++ b/src/storage/metadata/metadata_manager.cpp @@ -33,7 +33,7 @@ MetadataHandle MetadataManager::AllocateHandle() { // select the first free metadata block we can find MetadataPointer pointer; - pointer.block_index = free_block; + pointer.block_index = UnsafeNumericCast(free_block); auto &block = blocks[free_block]; if (block.block->BlockId() < MAXIMUM_BLOCK) { // this block is a disk-backed block, yet we are planning to write to it @@ -134,7 +134,7 @@ MetadataPointer MetadataManager::FromDiskPointer(MetaBlockPointer pointer) { pointer.block_pointer); } // LCOV_EXCL_STOP MetadataPointer result; - result.block_index = block_id; + result.block_index = UnsafeNumericCast(block_id); result.index = UnsafeNumericCast(index); return result; } diff --git a/src/storage/metadata/metadata_writer.cpp b/src/storage/metadata/metadata_writer.cpp index e47708c21f6a..cc95b9086a81 100644 --- a/src/storage/metadata/metadata_writer.cpp +++ b/src/storage/metadata/metadata_writer.cpp @@ -48,7 +48,7 @@ void MetadataWriter::NextBlock() { current_pointer = block.pointer; offset = sizeof(idx_t); capacity = MetadataManager::METADATA_BLOCK_SIZE; - Store(-1, BasePtr()); + Store(static_cast(-1), BasePtr()); if (written_pointers) { written_pointers->push_back(manager.GetDiskPointer(current_pointer)); } From 86d65a93e510d860ce4be6fbb656b36b6babdee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Fri, 12 Apr 2024 10:14:42 +0200 Subject: [PATCH 370/603] column data --- src/storage/table/column_data.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/storage/table/column_data.cpp b/src/storage/table/column_data.cpp index e48495049d7e..1ccc4b8a19f9 100644 --- a/src/storage/table/column_data.cpp +++ b/src/storage/table/column_data.cpp @@ -105,7 +105,7 @@ idx_t ColumnData::ScanVector(ColumnScanState &state, Vector &result, idx_t remai if (state.scan_options && state.scan_options->force_fetch_row) { for (idx_t i = 0; i < scan_count; i++) { ColumnFetchState fetch_state; - state.current->FetchRow(fetch_state, state.row_index + i, result, result_offset + i); + state.current->FetchRow(fetch_state, UnsafeNumericCast(state.row_index + i), result, result_offset + i); } } else { state.current->Scan(state, scan_count, result, result_offset, @@ -325,24 +325,24 @@ void ColumnData::RevertAppend(row_t start_row) { return; } // find the segment index that the current row belongs to - idx_t segment_index = data.GetSegmentIndex(l, start_row); - auto segment = data.GetSegmentByIndex(l, segment_index); + idx_t segment_index = data.GetSegmentIndex(l, UnsafeNumericCast(start_row)); + auto segment = data.GetSegmentByIndex(l, UnsafeNumericCast(segment_index)); auto &transient = *segment; D_ASSERT(transient.segment_type == ColumnSegmentType::TRANSIENT); // remove any segments AFTER this segment: they should be deleted entirely data.EraseSegments(l, segment_index); - this->count = start_row - this->start; + this->count = UnsafeNumericCast(start_row) - this->start; segment->next = nullptr; - transient.RevertAppend(start_row); + transient.RevertAppend(UnsafeNumericCast(start_row)); } idx_t ColumnData::Fetch(ColumnScanState &state, row_t row_id, Vector &result) { D_ASSERT(row_id >= 0); D_ASSERT(idx_t(row_id) >= start); // perform the fetch within the segment - state.row_index = start + ((row_id - start) / STANDARD_VECTOR_SIZE * STANDARD_VECTOR_SIZE); + state.row_index = start + ((UnsafeNumericCast(row_id) - start) / STANDARD_VECTOR_SIZE * STANDARD_VECTOR_SIZE); state.current = data.GetSegment(state.row_index); state.internal_index = state.current->start; return ScanVector(state, result, STANDARD_VECTOR_SIZE, false); @@ -350,14 +350,14 @@ idx_t ColumnData::Fetch(ColumnScanState &state, row_t row_id, Vector &result) { void ColumnData::FetchRow(TransactionData transaction, ColumnFetchState &state, row_t row_id, Vector &result, idx_t result_idx) { - auto segment = data.GetSegment(row_id); + auto segment = data.GetSegment(UnsafeNumericCast(row_id)); // now perform the fetch within the segment segment->FetchRow(state, row_id, result, result_idx); // merge any updates made to this row lock_guard update_guard(update_lock); if (updates) { - updates->FetchRow(transaction, row_id, result, result_idx); + updates->FetchRow(transaction, UnsafeNumericCast(row_id), result, result_idx); } } @@ -422,7 +422,7 @@ void ColumnData::CheckpointScan(ColumnSegment &segment, ColumnScanState &state, if (state.scan_options && state.scan_options->force_fetch_row) { for (idx_t i = 0; i < count; i++) { ColumnFetchState fetch_state; - segment.FetchRow(fetch_state, state.row_index + i, scan_vector, i); + segment.FetchRow(fetch_state, UnsafeNumericCast(state.row_index + i), scan_vector, i); } } else { segment.Scan(state, count, scan_vector, 0, true); From 175c19dcf187816d8527632cffcee5926c5d3a93 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 12 Apr 2024 10:30:01 +0200 Subject: [PATCH 371/603] make_refcounted -> make_shared_ptr --- extension/httpfs/httpfs.cpp | 4 +- extension/httpfs/s3fs.cpp | 2 +- extension/json/json_functions/copy_json.cpp | 2 +- extension/json/json_functions/read_json.cpp | 8 +-- .../json/json_functions/read_json_objects.cpp | 6 +- extension/parquet/column_reader.cpp | 10 +-- .../include/templated_column_reader.hpp | 2 +- extension/parquet/parquet_crypto.cpp | 2 +- extension/parquet/parquet_extension.cpp | 4 +- extension/parquet/parquet_reader.cpp | 2 +- .../catalog_entry/duck_table_entry.cpp | 10 +-- src/common/allocator.cpp | 2 +- src/common/arrow/arrow_wrapper.cpp | 2 +- src/common/extra_type_info.cpp | 8 +-- src/common/http_state.cpp | 4 +- src/common/re2_regex.cpp | 2 +- src/common/types.cpp | 28 ++++---- .../types/column/column_data_collection.cpp | 14 ++-- .../column/column_data_collection_segment.cpp | 2 +- .../types/column/partitioned_column_data.cpp | 4 +- .../types/row/partitioned_tuple_data.cpp | 4 +- .../types/row/tuple_data_collection.cpp | 4 +- src/common/types/value.cpp | 36 +++++------ src/common/types/vector_cache.cpp | 6 +- src/execution/aggregate_hashtable.cpp | 2 +- src/execution/index/art/art.cpp | 2 +- .../operator/aggregate/aggregate_object.cpp | 2 +- .../aggregate/physical_hash_aggregate.cpp | 4 +- .../physical_ungrouped_aggregate.cpp | 2 +- .../operator/aggregate/physical_window.cpp | 2 +- .../csv_scanner/buffer_manager/csv_buffer.cpp | 4 +- .../buffer_manager/csv_buffer_manager.cpp | 2 +- .../scanner/string_value_scanner.cpp | 10 +-- .../csv_scanner/sniffer/csv_sniffer.cpp | 4 +- .../table_function/csv_file_scanner.cpp | 16 ++--- .../table_function/global_csv_state.cpp | 12 ++-- .../helper/physical_buffered_collector.cpp | 2 +- .../operator/join/physical_asof_join.cpp | 2 +- .../operator/join/physical_hash_join.cpp | 4 +- .../operator/join/physical_range_join.cpp | 2 +- .../operator/order/physical_order.cpp | 2 +- .../physical_batch_copy_to_file.cpp | 2 +- .../persistent/physical_copy_to_file.cpp | 2 +- .../schema/physical_create_art_index.cpp | 2 +- .../operator/set/physical_recursive_cte.cpp | 2 +- src/execution/physical_plan/plan_cte.cpp | 2 +- .../physical_plan/plan_recursive_cte.cpp | 2 +- src/function/table/copy_csv.cpp | 2 +- src/function/table/read_csv.cpp | 2 +- src/function/table/sniff_csv.cpp | 2 +- src/include/duckdb/common/helper.hpp | 4 +- .../duckdb/common/multi_file_reader.hpp | 2 +- src/include/duckdb/common/types.hpp | 2 +- .../duckdb/common/types/selection_vector.hpp | 2 +- src/include/duckdb/planner/binder.hpp | 2 +- src/include/duckdb/storage/object_cache.hpp | 2 +- .../duckdb/storage/serialization/types.json | 2 +- src/main/capi/table_function-c.cpp | 2 +- src/main/client_context.cpp | 2 +- src/main/client_data.cpp | 4 +- src/main/connection.cpp | 20 +++--- src/main/database.cpp | 4 +- src/main/db_instance_cache.cpp | 2 +- src/main/relation.cpp | 56 ++++++++-------- src/main/relation/read_csv_relation.cpp | 2 +- src/main/relation/table_relation.cpp | 4 +- src/parallel/executor.cpp | 16 ++--- src/parallel/meta_pipeline.cpp | 4 +- src/planner/bind_context.cpp | 4 +- src/planner/binder.cpp | 2 +- src/planner/bound_parameter_map.cpp | 2 +- src/planner/planner.cpp | 2 +- src/storage/buffer/block_manager.cpp | 2 +- src/storage/checkpoint_manager.cpp | 2 +- src/storage/data_table.cpp | 4 +- src/storage/local_storage.cpp | 10 +-- src/storage/serialization/serialize_types.cpp | 2 +- src/storage/standard_buffer_manager.cpp | 4 +- src/storage/statistics/column_statistics.cpp | 6 +- src/storage/table/row_group.cpp | 2 +- src/storage/table/row_group_collection.cpp | 8 +-- src/storage/table/row_version_manager.cpp | 2 +- src/storage/wal_replay.cpp | 2 +- test/api/test_object_cache.cpp | 2 +- tools/odbc/include/duckdb_odbc.hpp | 2 +- tools/pythonpkg/src/pyconnection.cpp | 14 ++-- .../src/pyconnection/type_creation.cpp | 16 ++--- tools/pythonpkg/src/pyexpression.cpp | 26 ++++---- tools/pythonpkg/src/pyrelation.cpp | 4 +- tools/pythonpkg/src/typing/pytype.cpp | 24 +++---- tools/pythonpkg/src/typing/typing.cpp | 64 +++++++++---------- 91 files changed, 299 insertions(+), 299 deletions(-) diff --git a/extension/httpfs/httpfs.cpp b/extension/httpfs/httpfs.cpp index 476409ab8820..8c0bd34262b2 100644 --- a/extension/httpfs/httpfs.cpp +++ b/extension/httpfs/httpfs.cpp @@ -557,7 +557,7 @@ static optional_ptr TryGetMetadataCache(optional_ptrregistered_state.find("http_cache"); if (lookup == client_context->registered_state.end()) { - auto cache = make_refcounted(true, true); + auto cache = make_shared_ptr(true, true); client_context->registered_state["http_cache"] = cache; return cache.get(); } else { @@ -572,7 +572,7 @@ void HTTPFileHandle::Initialize(optional_ptr opener) { auto &hfs = file_system.Cast(); state = HTTPState::TryGetState(opener); if (!state) { - state = make_refcounted(); + state = make_shared_ptr(); } auto current_cache = TryGetMetadataCache(opener, hfs); diff --git a/extension/httpfs/s3fs.cpp b/extension/httpfs/s3fs.cpp index 888b6cb2115d..b06120896c00 100644 --- a/extension/httpfs/s3fs.cpp +++ b/extension/httpfs/s3fs.cpp @@ -568,7 +568,7 @@ shared_ptr S3FileHandle::GetBuffer(uint16_t write_buffer_idx) { auto buffer_handle = s3fs.Allocate(part_size, config_params.max_upload_threads); auto new_write_buffer = - make_refcounted(write_buffer_idx * part_size, part_size, std::move(buffer_handle)); + make_shared_ptr(write_buffer_idx * part_size, part_size, std::move(buffer_handle)); { unique_lock lck(write_buffers_lock); auto lookup_result = write_buffers.find(write_buffer_idx); diff --git a/extension/json/json_functions/copy_json.cpp b/extension/json/json_functions/copy_json.cpp index 5d97c01e3b84..4744e5a78c82 100644 --- a/extension/json/json_functions/copy_json.cpp +++ b/extension/json/json_functions/copy_json.cpp @@ -185,7 +185,7 @@ CopyFunction JSONFunctions::GetJSONCopyFunction() { function.plan = CopyToJSONPlan; function.copy_from_bind = CopyFromJSONBind; - function.copy_from_function = JSONFunctions::GetReadJSONTableFunction(make_refcounted( + function.copy_from_function = JSONFunctions::GetReadJSONTableFunction(make_shared_ptr( JSONScanType::READ_JSON, JSONFormat::NEWLINE_DELIMITED, JSONRecordType::RECORDS, false)); return function; diff --git a/extension/json/json_functions/read_json.cpp b/extension/json/json_functions/read_json.cpp index a7624e287240..e207060a448b 100644 --- a/extension/json/json_functions/read_json.cpp +++ b/extension/json/json_functions/read_json.cpp @@ -382,25 +382,25 @@ TableFunctionSet CreateJSONFunctionInfo(string name, shared_ptr in } TableFunctionSet JSONFunctions::GetReadJSONFunction() { - auto info = make_refcounted(JSONScanType::READ_JSON, JSONFormat::AUTO_DETECT, + auto info = make_shared_ptr(JSONScanType::READ_JSON, JSONFormat::AUTO_DETECT, JSONRecordType::AUTO_DETECT, true); return CreateJSONFunctionInfo("read_json", std::move(info)); } TableFunctionSet JSONFunctions::GetReadNDJSONFunction() { - auto info = make_refcounted(JSONScanType::READ_JSON, JSONFormat::NEWLINE_DELIMITED, + auto info = make_shared_ptr(JSONScanType::READ_JSON, JSONFormat::NEWLINE_DELIMITED, JSONRecordType::AUTO_DETECT, true); return CreateJSONFunctionInfo("read_ndjson", std::move(info)); } TableFunctionSet JSONFunctions::GetReadJSONAutoFunction() { - auto info = make_refcounted(JSONScanType::READ_JSON, JSONFormat::AUTO_DETECT, + auto info = make_shared_ptr(JSONScanType::READ_JSON, JSONFormat::AUTO_DETECT, JSONRecordType::AUTO_DETECT, true); return CreateJSONFunctionInfo("read_json_auto", std::move(info)); } TableFunctionSet JSONFunctions::GetReadNDJSONAutoFunction() { - auto info = make_refcounted(JSONScanType::READ_JSON, JSONFormat::NEWLINE_DELIMITED, + auto info = make_shared_ptr(JSONScanType::READ_JSON, JSONFormat::NEWLINE_DELIMITED, JSONRecordType::AUTO_DETECT, true); return CreateJSONFunctionInfo("read_ndjson_auto", std::move(info)); } diff --git a/extension/json/json_functions/read_json_objects.cpp b/extension/json/json_functions/read_json_objects.cpp index c8e798111071..630c2e694689 100644 --- a/extension/json/json_functions/read_json_objects.cpp +++ b/extension/json/json_functions/read_json_objects.cpp @@ -62,7 +62,7 @@ TableFunction GetReadJSONObjectsTableFunction(bool list_parameter, shared_ptr(JSONScanType::READ_JSON_OBJECTS, JSONFormat::ARRAY, JSONRecordType::RECORDS); + make_shared_ptr(JSONScanType::READ_JSON_OBJECTS, JSONFormat::ARRAY, JSONRecordType::RECORDS); function_set.AddFunction(GetReadJSONObjectsTableFunction(false, function_info)); function_set.AddFunction(GetReadJSONObjectsTableFunction(true, function_info)); return function_set; @@ -70,7 +70,7 @@ TableFunctionSet JSONFunctions::GetReadJSONObjectsFunction() { TableFunctionSet JSONFunctions::GetReadNDJSONObjectsFunction() { TableFunctionSet function_set("read_ndjson_objects"); - auto function_info = make_refcounted(JSONScanType::READ_JSON_OBJECTS, JSONFormat::NEWLINE_DELIMITED, + auto function_info = make_shared_ptr(JSONScanType::READ_JSON_OBJECTS, JSONFormat::NEWLINE_DELIMITED, JSONRecordType::RECORDS); function_set.AddFunction(GetReadJSONObjectsTableFunction(false, function_info)); function_set.AddFunction(GetReadJSONObjectsTableFunction(true, function_info)); @@ -79,7 +79,7 @@ TableFunctionSet JSONFunctions::GetReadNDJSONObjectsFunction() { TableFunctionSet JSONFunctions::GetReadJSONObjectsAutoFunction() { TableFunctionSet function_set("read_json_objects_auto"); - auto function_info = make_refcounted(JSONScanType::READ_JSON_OBJECTS, JSONFormat::AUTO_DETECT, + auto function_info = make_shared_ptr(JSONScanType::READ_JSON_OBJECTS, JSONFormat::AUTO_DETECT, JSONRecordType::RECORDS); function_set.AddFunction(GetReadJSONObjectsTableFunction(false, function_info)); function_set.AddFunction(GetReadJSONObjectsTableFunction(true, function_info)); diff --git a/extension/parquet/column_reader.cpp b/extension/parquet/column_reader.cpp index 680623c67f4b..e02feca15208 100644 --- a/extension/parquet/column_reader.cpp +++ b/extension/parquet/column_reader.cpp @@ -304,7 +304,7 @@ void ColumnReader::PreparePageV2(PageHeader &page_hdr) { void ColumnReader::AllocateBlock(idx_t size) { if (!block) { - block = make_refcounted(GetAllocator(), size); + block = make_shared_ptr(GetAllocator(), size); } else { block->resize(GetAllocator(), size); } @@ -516,7 +516,7 @@ idx_t ColumnReader::Read(uint64_t num_values, parquet_filter_t &filter, data_ptr result); } else if (dbp_decoder) { // TODO keep this in the state - auto read_buf = make_refcounted(); + auto read_buf = make_shared_ptr(); switch (schema.type) { case duckdb_parquet::format::Type::INT32: @@ -537,7 +537,7 @@ idx_t ColumnReader::Read(uint64_t num_values, parquet_filter_t &filter, data_ptr } else if (rle_decoder) { // RLE encoding for boolean D_ASSERT(type.id() == LogicalTypeId::BOOLEAN); - auto read_buf = make_refcounted(); + auto read_buf = make_shared_ptr(); read_buf->resize(reader.allocator, sizeof(bool) * (read_now - null_count)); rle_decoder->GetBatch(read_buf->ptr, read_now - null_count); PlainTemplated>(read_buf, define_out, read_now, filter, @@ -546,7 +546,7 @@ idx_t ColumnReader::Read(uint64_t num_values, parquet_filter_t &filter, data_ptr // DELTA_BYTE_ARRAY or DELTA_LENGTH_BYTE_ARRAY DeltaByteArray(define_out, read_now, filter, result_offset, result); } else if (bss_decoder) { - auto read_buf = make_refcounted(); + auto read_buf = make_shared_ptr(); switch (schema.type) { case duckdb_parquet::format::Type::FLOAT: @@ -662,7 +662,7 @@ void StringColumnReader::Dictionary(shared_ptr data, idx_t num static shared_ptr ReadDbpData(Allocator &allocator, ResizeableBuffer &buffer, idx_t &value_count) { auto decoder = make_uniq(buffer.ptr, buffer.len); value_count = decoder->TotalValues(); - auto result = make_refcounted(); + auto result = make_shared_ptr(); result->resize(allocator, sizeof(uint32_t) * value_count); decoder->GetBatch(result->ptr, value_count); decoder->Finalize(); diff --git a/extension/parquet/include/templated_column_reader.hpp b/extension/parquet/include/templated_column_reader.hpp index c8ffb761eb5f..009cd6aeeb86 100644 --- a/extension/parquet/include/templated_column_reader.hpp +++ b/extension/parquet/include/templated_column_reader.hpp @@ -44,7 +44,7 @@ class TemplatedColumnReader : public ColumnReader { public: void AllocateDict(idx_t size) { if (!dict) { - dict = make_refcounted(GetAllocator(), size); + dict = make_shared_ptr(GetAllocator(), size); } else { dict->resize(GetAllocator(), size); } diff --git a/extension/parquet/parquet_crypto.cpp b/extension/parquet/parquet_crypto.cpp index 13d96ab76135..d6bb7f1b2200 100644 --- a/extension/parquet/parquet_crypto.cpp +++ b/extension/parquet/parquet_crypto.cpp @@ -14,7 +14,7 @@ namespace duckdb { ParquetKeys &ParquetKeys::Get(ClientContext &context) { auto &cache = ObjectCache::GetObjectCache(context); if (!cache.Get(ParquetKeys::ObjectType())) { - cache.Put(ParquetKeys::ObjectType(), make_refcounted()); + cache.Put(ParquetKeys::ObjectType(), make_shared_ptr()); } return *cache.Get(ParquetKeys::ObjectType()); } diff --git a/extension/parquet/parquet_extension.cpp b/extension/parquet/parquet_extension.cpp index 5e898669c016..4cde1e9ec399 100644 --- a/extension/parquet/parquet_extension.cpp +++ b/extension/parquet/parquet_extension.cpp @@ -544,7 +544,7 @@ class ParquetScanFunction { result->initial_reader = result->readers[0]; } else { result->initial_reader = - make_refcounted(context, bind_data.files[0], bind_data.parquet_options); + make_shared_ptr(context, bind_data.files[0], bind_data.parquet_options); result->readers[0] = result->initial_reader; } result->file_states[0] = ParquetFileState::OPEN; @@ -750,7 +750,7 @@ class ParquetScanFunction { shared_ptr reader; try { - reader = make_refcounted(context, file, pq_options); + reader = make_shared_ptr(context, file, pq_options); InitializeParquetReader(*reader, bind_data, parallel_state.column_ids, parallel_state.filters, context); } catch (...) { diff --git a/extension/parquet/parquet_reader.cpp b/extension/parquet/parquet_reader.cpp index ecd51fc3c968..896bc11b390c 100644 --- a/extension/parquet/parquet_reader.cpp +++ b/extension/parquet/parquet_reader.cpp @@ -113,7 +113,7 @@ LoadMetadata(Allocator &allocator, FileHandle &file_handle, metadata->read(file_proto.get()); } - return make_refcounted(std::move(metadata), current_time); + return make_shared_ptr(std::move(metadata), current_time); } LogicalType ParquetReader::DeriveLogicalType(const SchemaElement &s_ele, bool binary_as_string) { diff --git a/src/catalog/catalog_entry/duck_table_entry.cpp b/src/catalog/catalog_entry/duck_table_entry.cpp index 53ca0cbba0c3..d6a56d0451cd 100644 --- a/src/catalog/catalog_entry/duck_table_entry.cpp +++ b/src/catalog/catalog_entry/duck_table_entry.cpp @@ -84,7 +84,7 @@ DuckTableEntry::DuckTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, Bou storage_columns.push_back(col_def.Copy()); } storage = - make_refcounted(catalog.GetAttached(), StorageManager::Get(catalog).GetTableIOManager(&info), + make_shared_ptr(catalog.GetAttached(), StorageManager::Get(catalog).GetTableIOManager(&info), schema.name, name, std::move(storage_columns), std::move(info.data)); // create the unique indexes for the UNIQUE and PRIMARY KEY and FOREIGN KEY constraints @@ -345,7 +345,7 @@ unique_ptr DuckTableEntry::AddColumn(ClientContext &context, AddCo auto binder = Binder::CreateBinder(context); auto bound_create_info = binder->BindCreateTableInfo(std::move(create_info), schema); auto new_storage = - make_refcounted(context, *storage, info.new_column, *bound_create_info->bound_defaults.back()); + make_shared_ptr(context, *storage, info.new_column, *bound_create_info->bound_defaults.back()); return make_uniq(catalog, schema, *bound_create_info, new_storage); } @@ -481,7 +481,7 @@ unique_ptr DuckTableEntry::RemoveColumn(ClientContext &context, Re return make_uniq(catalog, schema, *bound_create_info, storage); } auto new_storage = - make_refcounted(context, *storage, columns.LogicalToPhysical(LogicalIndex(removed_index)).index); + make_shared_ptr(context, *storage, columns.LogicalToPhysical(LogicalIndex(removed_index)).index); return make_uniq(catalog, schema, *bound_create_info, new_storage); } @@ -549,7 +549,7 @@ unique_ptr DuckTableEntry::SetNotNull(ClientContext &context, SetN } // Return with new storage info. Note that we need the bound column index here. - auto new_storage = make_refcounted( + auto new_storage = make_shared_ptr( context, *storage, make_uniq(columns.LogicalToPhysical(LogicalIndex(not_null_idx)))); return make_uniq(catalog, schema, *bound_create_info, new_storage); } @@ -660,7 +660,7 @@ unique_ptr DuckTableEntry::ChangeColumnType(ClientContext &context } auto new_storage = - make_refcounted(context, *storage, columns.LogicalToPhysical(LogicalIndex(change_idx)).index, + make_shared_ptr(context, *storage, columns.LogicalToPhysical(LogicalIndex(change_idx)).index, info.target_type, std::move(storage_oids), *bound_expression); auto result = make_uniq(catalog, schema, *bound_create_info, new_storage); return std::move(result); diff --git a/src/common/allocator.cpp b/src/common/allocator.cpp index 835aba386ff2..c587aaf33442 100644 --- a/src/common/allocator.cpp +++ b/src/common/allocator.cpp @@ -195,7 +195,7 @@ data_ptr_t Allocator::DefaultReallocate(PrivateAllocatorData *private_data, data } shared_ptr &Allocator::DefaultAllocatorReference() { - static shared_ptr DEFAULT_ALLOCATOR = make_refcounted(); + static shared_ptr DEFAULT_ALLOCATOR = make_shared_ptr(); return DEFAULT_ALLOCATOR; } diff --git a/src/common/arrow/arrow_wrapper.cpp b/src/common/arrow/arrow_wrapper.cpp index 1bcc48ce91b6..0f0613bca1c7 100644 --- a/src/common/arrow/arrow_wrapper.cpp +++ b/src/common/arrow/arrow_wrapper.cpp @@ -50,7 +50,7 @@ void ArrowArrayStreamWrapper::GetSchema(ArrowSchemaWrapper &schema) { } shared_ptr ArrowArrayStreamWrapper::GetNextChunk() { - auto current_chunk = make_refcounted(); + auto current_chunk = make_shared_ptr(); if (arrow_array_stream.get_next(&arrow_array_stream, ¤t_chunk->arrow_array)) { // LCOV_EXCL_START throw InvalidInputException("arrow_scan: get_next failed(): %s", string(GetError())); } // LCOV_EXCL_STOP diff --git a/src/common/extra_type_info.cpp b/src/common/extra_type_info.cpp index c5c3ca1b6b8f..d6a632164916 100644 --- a/src/common/extra_type_info.cpp +++ b/src/common/extra_type_info.cpp @@ -190,7 +190,7 @@ struct EnumTypeInfoTemplated : public EnumTypeInfo { deserializer.ReadList(201, "values", [&](Deserializer::List &list, idx_t i) { strings[i] = StringVector::AddStringOrBlob(values_insert_order, list.ReadElement()); }); - return make_refcounted(values_insert_order, size); + return make_shared_ptr(values_insert_order, size); } const string_map_t &GetValues() const { @@ -227,13 +227,13 @@ LogicalType EnumTypeInfo::CreateType(Vector &ordered_data, idx_t size) { auto enum_internal_type = EnumTypeInfo::DictType(size); switch (enum_internal_type) { case PhysicalType::UINT8: - info = make_refcounted>(ordered_data, size); + info = make_shared_ptr>(ordered_data, size); break; case PhysicalType::UINT16: - info = make_refcounted>(ordered_data, size); + info = make_shared_ptr>(ordered_data, size); break; case PhysicalType::UINT32: - info = make_refcounted>(ordered_data, size); + info = make_shared_ptr>(ordered_data, size); break; default: throw InternalException("Invalid Physical Type for ENUMs"); diff --git a/src/common/http_state.cpp b/src/common/http_state.cpp index 880454b87a75..a2e91182f588 100644 --- a/src/common/http_state.cpp +++ b/src/common/http_state.cpp @@ -69,7 +69,7 @@ shared_ptr HTTPState::TryGetState(ClientContext &context, bool create return nullptr; } - auto http_state = make_refcounted(); + auto http_state = make_shared_ptr(); context.registered_state["http_state"] = http_state; return http_state; } @@ -87,7 +87,7 @@ shared_ptr &HTTPState::GetCachedFile(const string &path) { lock_guard lock(cached_files_mutex); auto &cache_entry_ref = cached_files[path]; if (!cache_entry_ref) { - cache_entry_ref = make_refcounted(); + cache_entry_ref = make_shared_ptr(); } return cache_entry_ref; } diff --git a/src/common/re2_regex.cpp b/src/common/re2_regex.cpp index 4b3e2fb8e87b..c82bf429a135 100644 --- a/src/common/re2_regex.cpp +++ b/src/common/re2_regex.cpp @@ -10,7 +10,7 @@ namespace duckdb_re2 { Regex::Regex(const std::string &pattern, RegexOptions options) { RE2::Options o; o.set_case_sensitive(options == RegexOptions::CASE_INSENSITIVE); - regex = duckdb::make_refcounted(StringPiece(pattern), o); + regex = duckdb::make_shared_ptr(StringPiece(pattern), o); } bool RegexSearchInternal(const char *input, Match &match, const Regex &r, RE2::Anchor anchor, size_t start, diff --git a/src/common/types.cpp b/src/common/types.cpp index f70d69d8102a..b9f5b8edbb40 100644 --- a/src/common/types.cpp +++ b/src/common/types.cpp @@ -1127,7 +1127,7 @@ bool ApproxEqual(double ldecimal, double rdecimal) { //===--------------------------------------------------------------------===// void LogicalType::SetAlias(string alias) { if (!type_info_) { - type_info_ = make_refcounted(ExtraTypeInfoType::GENERIC_TYPE_INFO, std::move(alias)); + type_info_ = make_shared_ptr(ExtraTypeInfoType::GENERIC_TYPE_INFO, std::move(alias)); } else { type_info_->alias = std::move(alias); } @@ -1176,7 +1176,7 @@ uint8_t DecimalType::MaxWidth() { LogicalType LogicalType::DECIMAL(uint8_t width, uint8_t scale) { D_ASSERT(width >= scale); - auto type_info = make_refcounted(width, scale); + auto type_info = make_shared_ptr(width, scale); return LogicalType(LogicalTypeId::DECIMAL, std::move(type_info)); } @@ -1198,7 +1198,7 @@ string StringType::GetCollation(const LogicalType &type) { } LogicalType LogicalType::VARCHAR_COLLATION(string collation) { // NOLINT - auto string_info = make_refcounted(std::move(collation)); + auto string_info = make_shared_ptr(std::move(collation)); return LogicalType(LogicalTypeId::VARCHAR, std::move(string_info)); } @@ -1213,7 +1213,7 @@ const LogicalType &ListType::GetChildType(const LogicalType &type) { } LogicalType LogicalType::LIST(const LogicalType &child) { - auto info = make_refcounted(child); + auto info = make_shared_ptr(child); return LogicalType(LogicalTypeId::LIST, std::move(info)); } @@ -1285,12 +1285,12 @@ bool StructType::IsUnnamed(const LogicalType &type) { } LogicalType LogicalType::STRUCT(child_list_t children) { - auto info = make_refcounted(std::move(children)); + auto info = make_shared_ptr(std::move(children)); return LogicalType(LogicalTypeId::STRUCT, std::move(info)); } LogicalType LogicalType::AGGREGATE_STATE(aggregate_state_t state_type) { // NOLINT - auto info = make_refcounted(std::move(state_type)); + auto info = make_shared_ptr(std::move(state_type)); return LogicalType(LogicalTypeId::AGGREGATE_STATE, std::move(info)); } @@ -1315,7 +1315,7 @@ LogicalType LogicalType::MAP(const LogicalType &child_p) { new_children[1].first = "value"; auto child = LogicalType::STRUCT(std::move(new_children)); - auto info = make_refcounted(child); + auto info = make_shared_ptr(child); return LogicalType(LogicalTypeId::MAP, std::move(info)); } @@ -1344,7 +1344,7 @@ LogicalType LogicalType::UNION(child_list_t members) { D_ASSERT(members.size() <= UnionType::MAX_UNION_MEMBERS); // union types always have a hidden "tag" field in front members.insert(members.begin(), {"", LogicalType::UTINYINT}); - auto info = make_refcounted(std::move(members)); + auto info = make_shared_ptr(std::move(members)); return LogicalType(LogicalTypeId::UNION, std::move(info)); } @@ -1397,12 +1397,12 @@ const string &UserType::GetTypeName(const LogicalType &type) { } LogicalType LogicalType::USER(const string &user_type_name) { - auto info = make_refcounted(user_type_name); + auto info = make_shared_ptr(user_type_name); return LogicalType(LogicalTypeId::USER, std::move(info)); } LogicalType LogicalType::USER(string catalog, string schema, string name) { - auto info = make_refcounted(std::move(catalog), std::move(schema), std::move(name)); + auto info = make_shared_ptr(std::move(catalog), std::move(schema), std::move(name)); return LogicalType(LogicalTypeId::USER, std::move(info)); } @@ -1518,12 +1518,12 @@ LogicalType ArrayType::ConvertToList(const LogicalType &type) { LogicalType LogicalType::ARRAY(const LogicalType &child, idx_t size) { D_ASSERT(size > 0); D_ASSERT(size < ArrayType::MAX_ARRAY_SIZE); - auto info = make_refcounted(child, size); + auto info = make_shared_ptr(child, size); return LogicalType(LogicalTypeId::ARRAY, std::move(info)); } LogicalType LogicalType::ARRAY(const LogicalType &child) { - auto info = make_refcounted(child, 0); + auto info = make_shared_ptr(child, 0); return LogicalType(LogicalTypeId::ARRAY, std::move(info)); } @@ -1531,7 +1531,7 @@ LogicalType LogicalType::ARRAY(const LogicalType &child) { // Any Type //===--------------------------------------------------------------------===// LogicalType LogicalType::ANY_PARAMS(LogicalType target, idx_t cast_score) { // NOLINT - auto type_info = make_refcounted(std::move(target), cast_score); + auto type_info = make_shared_ptr(std::move(target), cast_score); return LogicalType(LogicalTypeId::ANY, std::move(type_info)); } @@ -1584,7 +1584,7 @@ LogicalType LogicalType::INTEGER_LITERAL(const Value &constant) { // NOLINT if (!constant.type().IsIntegral()) { throw InternalException("INTEGER_LITERAL can only be made from literals of integer types"); } - auto type_info = make_refcounted(constant); + auto type_info = make_shared_ptr(constant); return LogicalType(LogicalTypeId::INTEGER_LITERAL, std::move(type_info)); } diff --git a/src/common/types/column/column_data_collection.cpp b/src/common/types/column/column_data_collection.cpp index 0f931d7cf909..46aed955ad8b 100644 --- a/src/common/types/column/column_data_collection.cpp +++ b/src/common/types/column/column_data_collection.cpp @@ -51,17 +51,17 @@ ColumnDataCollection::ColumnDataCollection(Allocator &allocator_p) { types.clear(); count = 0; this->finished_append = false; - allocator = make_refcounted(allocator_p); + allocator = make_shared_ptr(allocator_p); } ColumnDataCollection::ColumnDataCollection(Allocator &allocator_p, vector types_p) { Initialize(std::move(types_p)); - allocator = make_refcounted(allocator_p); + allocator = make_shared_ptr(allocator_p); } ColumnDataCollection::ColumnDataCollection(BufferManager &buffer_manager, vector types_p) { Initialize(std::move(types_p)); - allocator = make_refcounted(buffer_manager); + allocator = make_shared_ptr(buffer_manager); } ColumnDataCollection::ColumnDataCollection(shared_ptr allocator_p, vector types_p) { @@ -71,7 +71,7 @@ ColumnDataCollection::ColumnDataCollection(shared_ptr alloc ColumnDataCollection::ColumnDataCollection(ClientContext &context, vector types_p, ColumnDataAllocatorType type) - : ColumnDataCollection(make_refcounted(context, type), std::move(types_p)) { + : ColumnDataCollection(make_shared_ptr(context, type), std::move(types_p)) { D_ASSERT(!types.empty()); } @@ -199,7 +199,7 @@ ColumnDataChunkIterationHelper::ColumnDataChunkIterationHelper(const ColumnDataC ColumnDataChunkIterationHelper::ColumnDataChunkIterator::ColumnDataChunkIterator( const ColumnDataCollection *collection_p, vector column_ids_p) - : collection(collection_p), scan_chunk(make_refcounted()), row_index(0) { + : collection(collection_p), scan_chunk(make_shared_ptr()), row_index(0) { if (!collection) { return; } @@ -246,7 +246,7 @@ ColumnDataRowIterationHelper::ColumnDataRowIterationHelper(const ColumnDataColle } ColumnDataRowIterationHelper::ColumnDataRowIterator::ColumnDataRowIterator(const ColumnDataCollection *collection_p) - : collection(collection_p), scan_chunk(make_refcounted()), current_row(*scan_chunk, 0, 0) { + : collection(collection_p), scan_chunk(make_shared_ptr()), current_row(*scan_chunk, 0, 0) { if (!collection) { return; } @@ -1041,7 +1041,7 @@ void ColumnDataCollection::Reset() { segments.clear(); // Refreshes the ColumnDataAllocator to prevent holding on to allocated data unnecessarily - allocator = make_refcounted(*allocator); + allocator = make_shared_ptr(*allocator); } struct ValueResultEquals { diff --git a/src/common/types/column/column_data_collection_segment.cpp b/src/common/types/column/column_data_collection_segment.cpp index 1f815d521974..918680c13911 100644 --- a/src/common/types/column/column_data_collection_segment.cpp +++ b/src/common/types/column/column_data_collection_segment.cpp @@ -7,7 +7,7 @@ namespace duckdb { ColumnDataCollectionSegment::ColumnDataCollectionSegment(shared_ptr allocator_p, vector types_p) : allocator(std::move(allocator_p)), types(std::move(types_p)), count(0), - heap(make_refcounted(allocator->GetAllocator())) { + heap(make_shared_ptr(allocator->GetAllocator())) { } idx_t ColumnDataCollectionSegment::GetDataSize(idx_t type_size) { diff --git a/src/common/types/column/partitioned_column_data.cpp b/src/common/types/column/partitioned_column_data.cpp index 7d47e129f26b..0ac1e065b24f 100644 --- a/src/common/types/column/partitioned_column_data.cpp +++ b/src/common/types/column/partitioned_column_data.cpp @@ -9,7 +9,7 @@ namespace duckdb { PartitionedColumnData::PartitionedColumnData(PartitionedColumnDataType type_p, ClientContext &context_p, vector types_p) : type(type_p), context(context_p), types(std::move(types_p)), - allocators(make_refcounted()) { + allocators(make_shared_ptr()) { } PartitionedColumnData::PartitionedColumnData(const PartitionedColumnData &other) @@ -165,7 +165,7 @@ vector> &PartitionedColumnData::GetPartitions() } void PartitionedColumnData::CreateAllocator() { - allocators->allocators.emplace_back(make_refcounted(BufferManager::GetBufferManager(context))); + allocators->allocators.emplace_back(make_shared_ptr(BufferManager::GetBufferManager(context))); allocators->allocators.back()->MakeShared(); } diff --git a/src/common/types/row/partitioned_tuple_data.cpp b/src/common/types/row/partitioned_tuple_data.cpp index cd67c32abb0d..b134a9b9784e 100644 --- a/src/common/types/row/partitioned_tuple_data.cpp +++ b/src/common/types/row/partitioned_tuple_data.cpp @@ -9,7 +9,7 @@ namespace duckdb { PartitionedTupleData::PartitionedTupleData(PartitionedTupleDataType type_p, BufferManager &buffer_manager_p, const TupleDataLayout &layout_p) : type(type_p), buffer_manager(buffer_manager_p), layout(layout_p.Copy()), count(0), data_size(0), - allocators(make_refcounted()) { + allocators(make_shared_ptr()) { } PartitionedTupleData::PartitionedTupleData(const PartitionedTupleData &other) @@ -434,7 +434,7 @@ void PartitionedTupleData::Print() { // LCOV_EXCL_STOP void PartitionedTupleData::CreateAllocator() { - allocators->allocators.emplace_back(make_refcounted(buffer_manager, layout)); + allocators->allocators.emplace_back(make_shared_ptr(buffer_manager, layout)); } } // namespace duckdb diff --git a/src/common/types/row/tuple_data_collection.cpp b/src/common/types/row/tuple_data_collection.cpp index 7ffcac79abce..86cbb144c804 100644 --- a/src/common/types/row/tuple_data_collection.cpp +++ b/src/common/types/row/tuple_data_collection.cpp @@ -12,7 +12,7 @@ namespace duckdb { using ValidityBytes = TupleDataLayout::ValidityBytes; TupleDataCollection::TupleDataCollection(BufferManager &buffer_manager, const TupleDataLayout &layout_p) - : layout(layout_p.Copy()), allocator(make_refcounted(buffer_manager, layout)) { + : layout(layout_p.Copy()), allocator(make_shared_ptr(buffer_manager, layout)) { Initialize(); } @@ -377,7 +377,7 @@ void TupleDataCollection::Reset() { segments.clear(); // Refreshes the TupleDataAllocator to prevent holding on to allocated data unnecessarily - allocator = make_refcounted(*allocator); + allocator = make_shared_ptr(*allocator); } void TupleDataCollection::InitializeChunk(DataChunk &chunk) const { diff --git a/src/common/types/value.cpp b/src/common/types/value.cpp index a2558d6cea42..9a10ea324d37 100644 --- a/src/common/types/value.cpp +++ b/src/common/types/value.cpp @@ -162,7 +162,7 @@ Value::Value(string val) : type_(LogicalType::VARCHAR), is_null(false) { if (!Value::StringIsValid(val.c_str(), val.size())) { throw ErrorManager::InvalidUnicodeError(val, "value construction"); } - value_info_ = make_refcounted(std::move(val)); + value_info_ = make_shared_ptr(std::move(val)); } Value::~Value() { @@ -668,7 +668,7 @@ Value Value::STRUCT(const LogicalType &type, vector struct_values) { for (size_t i = 0; i < struct_values.size(); i++) { struct_values[i] = struct_values[i].DefaultCastAs(child_types[i].second); } - result.value_info_ = make_refcounted(std::move(struct_values)); + result.value_info_ = make_shared_ptr(std::move(struct_values)); result.type_ = type; result.is_null = false; return result; @@ -711,7 +711,7 @@ Value Value::MAP(const LogicalType &key_type, const LogicalType &value_type, vec new_children.push_back(std::make_pair("value", std::move(values[i]))); values[i] = Value::STRUCT(std::move(new_children)); } - result.value_info_ = make_refcounted(std::move(values)); + result.value_info_ = make_shared_ptr(std::move(values)); return result; } @@ -735,7 +735,7 @@ Value Value::UNION(child_list_t members, uint8_t tag, Value value) } } union_values[tag + 1] = std::move(value); - result.value_info_ = make_refcounted(std::move(union_values)); + result.value_info_ = make_shared_ptr(std::move(union_values)); result.type_ = LogicalType::UNION(std::move(members)); return result; } @@ -752,7 +752,7 @@ Value Value::LIST(vector values) { #endif Value result; result.type_ = LogicalType::LIST(values[0].type()); - result.value_info_ = make_refcounted(std::move(values)); + result.value_info_ = make_shared_ptr(std::move(values)); result.is_null = false; return result; } @@ -770,7 +770,7 @@ Value Value::LIST(const LogicalType &child_type, vector values) { Value Value::EMPTYLIST(const LogicalType &child_type) { Value result; result.type_ = LogicalType::LIST(child_type); - result.value_info_ = make_refcounted(); + result.value_info_ = make_shared_ptr(); result.is_null = false; return result; } @@ -787,7 +787,7 @@ Value Value::ARRAY(vector values) { #endif Value result; result.type_ = LogicalType::ARRAY(values[0].type(), values.size()); - result.value_info_ = make_refcounted(std::move(values)); + result.value_info_ = make_shared_ptr(std::move(values)); result.is_null = false; return result; } @@ -805,7 +805,7 @@ Value Value::ARRAY(const LogicalType &child_type, vector values) { Value Value::EMPTYARRAY(const LogicalType &child_type, uint32_t size) { Value result; result.type_ = LogicalType::ARRAY(child_type, size); - result.value_info_ = make_refcounted(); + result.value_info_ = make_shared_ptr(); result.is_null = false; return result; } @@ -813,35 +813,35 @@ Value Value::EMPTYARRAY(const LogicalType &child_type, uint32_t size) { Value Value::BLOB(const_data_ptr_t data, idx_t len) { Value result(LogicalType::BLOB); result.is_null = false; - result.value_info_ = make_refcounted(string(const_char_ptr_cast(data), len)); + result.value_info_ = make_shared_ptr(string(const_char_ptr_cast(data), len)); return result; } Value Value::BLOB(const string &data) { Value result(LogicalType::BLOB); result.is_null = false; - result.value_info_ = make_refcounted(Blob::ToBlob(string_t(data))); + result.value_info_ = make_shared_ptr(Blob::ToBlob(string_t(data))); return result; } Value Value::AGGREGATE_STATE(const LogicalType &type, const_data_ptr_t data, idx_t len) { // NOLINT Value result(type); result.is_null = false; - result.value_info_ = make_refcounted(string(const_char_ptr_cast(data), len)); + result.value_info_ = make_shared_ptr(string(const_char_ptr_cast(data), len)); return result; } Value Value::BIT(const_data_ptr_t data, idx_t len) { Value result(LogicalType::BIT); result.is_null = false; - result.value_info_ = make_refcounted(string(const_char_ptr_cast(data), len)); + result.value_info_ = make_shared_ptr(string(const_char_ptr_cast(data), len)); return result; } Value Value::BIT(const string &data) { Value result(LogicalType::BIT); result.is_null = false; - result.value_info_ = make_refcounted(Bit::ToBit(string_t(data))); + result.value_info_ = make_shared_ptr(Bit::ToBit(string_t(data))); return result; } @@ -1936,27 +1936,27 @@ Value Value::Deserialize(Deserializer &deserializer) { case PhysicalType::VARCHAR: { auto str = deserializer.ReadProperty(102, "value"); if (type.id() == LogicalTypeId::BLOB) { - new_value.value_info_ = make_refcounted(Blob::ToBlob(str)); + new_value.value_info_ = make_shared_ptr(Blob::ToBlob(str)); } else { - new_value.value_info_ = make_refcounted(str); + new_value.value_info_ = make_shared_ptr(str); } } break; case PhysicalType::LIST: { deserializer.ReadObject(102, "value", [&](Deserializer &obj) { auto children = obj.ReadProperty>(100, "children"); - new_value.value_info_ = make_refcounted(children); + new_value.value_info_ = make_shared_ptr(children); }); } break; case PhysicalType::STRUCT: { deserializer.ReadObject(102, "value", [&](Deserializer &obj) { auto children = obj.ReadProperty>(100, "children"); - new_value.value_info_ = make_refcounted(children); + new_value.value_info_ = make_shared_ptr(children); }); } break; case PhysicalType::ARRAY: { deserializer.ReadObject(102, "value", [&](Deserializer &obj) { auto children = obj.ReadProperty>(100, "children"); - new_value.value_info_ = make_refcounted(children); + new_value.value_info_ = make_shared_ptr(children); }); } break; default: diff --git a/src/common/types/vector_cache.cpp b/src/common/types/vector_cache.cpp index 0c5075a54b45..56664319c5da 100644 --- a/src/common/types/vector_cache.cpp +++ b/src/common/types/vector_cache.cpp @@ -18,7 +18,7 @@ class VectorCacheBuffer : public VectorBuffer { auto &child_type = ListType::GetChildType(type); child_caches.push_back(make_buffer(allocator, child_type, capacity)); auto child_vector = make_uniq(child_type, false, false); - auxiliary = make_refcounted(std::move(child_vector)); + auxiliary = make_shared_ptr(std::move(child_vector)); break; } case PhysicalType::ARRAY: { @@ -26,7 +26,7 @@ class VectorCacheBuffer : public VectorBuffer { auto array_size = ArrayType::GetSize(type); child_caches.push_back(make_buffer(allocator, child_type, array_size * capacity)); auto child_vector = make_uniq(child_type, true, false, array_size * capacity); - auxiliary = make_refcounted(std::move(child_vector), array_size, capacity); + auxiliary = make_shared_ptr(std::move(child_vector), array_size, capacity); break; } case PhysicalType::STRUCT: { @@ -34,7 +34,7 @@ class VectorCacheBuffer : public VectorBuffer { for (auto &child_type : child_types) { child_caches.push_back(make_buffer(allocator, child_type.second, capacity)); } - auto struct_buffer = make_refcounted(type); + auto struct_buffer = make_shared_ptr(type); auxiliary = std::move(struct_buffer); break; } diff --git a/src/execution/aggregate_hashtable.cpp b/src/execution/aggregate_hashtable.cpp index 95f826a35596..08293a8025f6 100644 --- a/src/execution/aggregate_hashtable.cpp +++ b/src/execution/aggregate_hashtable.cpp @@ -40,7 +40,7 @@ GroupedAggregateHashTable::GroupedAggregateHashTable(ClientContext &context, All vector aggregate_objects_p, idx_t initial_capacity, idx_t radix_bits) : BaseAggregateHashTable(context, allocator, aggregate_objects_p, std::move(payload_types_p)), - radix_bits(radix_bits), count(0), capacity(0), aggregate_allocator(make_refcounted(allocator)) { + radix_bits(radix_bits), count(0), capacity(0), aggregate_allocator(make_shared_ptr(allocator)) { // Append hash column to the end and initialise the row layout group_types_p.emplace_back(LogicalType::HASH); diff --git a/src/execution/index/art/art.cpp b/src/execution/index/art/art.cpp index 7e3c7f3f38d3..b150f7f4398a 100644 --- a/src/execution/index/art/art.cpp +++ b/src/execution/index/art/art.cpp @@ -55,7 +55,7 @@ ART::ART(const string &name, const IndexConstraintType index_constraint_type, co make_uniq(sizeof(Node48), block_manager), make_uniq(sizeof(Node256), block_manager)}; allocators = - make_refcounted, ALLOCATOR_COUNT>>(std::move(allocator_array)); + make_shared_ptr, ALLOCATOR_COUNT>>(std::move(allocator_array)); } // deserialize lazily diff --git a/src/execution/operator/aggregate/aggregate_object.cpp b/src/execution/operator/aggregate/aggregate_object.cpp index 79a524ea76b8..af61fa17fb4d 100644 --- a/src/execution/operator/aggregate/aggregate_object.cpp +++ b/src/execution/operator/aggregate/aggregate_object.cpp @@ -9,7 +9,7 @@ AggregateObject::AggregateObject(AggregateFunction function, FunctionData *bind_ idx_t payload_size, AggregateType aggr_type, PhysicalType return_type, Expression *filter) : function(std::move(function)), - bind_data_wrapper(bind_data ? make_refcounted(bind_data->Copy()) : nullptr), + bind_data_wrapper(bind_data ? make_shared_ptr(bind_data->Copy()) : nullptr), child_count(child_count), payload_size(payload_size), aggr_type(aggr_type), return_type(return_type), filter(filter) { } diff --git a/src/execution/operator/aggregate/physical_hash_aggregate.cpp b/src/execution/operator/aggregate/physical_hash_aggregate.cpp index 420a9aecb918..b17fd7fcdca5 100644 --- a/src/execution/operator/aggregate/physical_hash_aggregate.cpp +++ b/src/execution/operator/aggregate/physical_hash_aggregate.cpp @@ -608,7 +608,7 @@ idx_t HashAggregateDistinctFinalizeEvent::CreateGlobalSources() { void HashAggregateDistinctFinalizeEvent::FinishEvent() { // Now that everything is added to the main ht, we can actually finalize - auto new_event = make_refcounted(context, pipeline.get(), op, gstate); + auto new_event = make_shared_ptr(context, pipeline.get(), op, gstate); this->InsertEvent(std::move(new_event)); } @@ -755,7 +755,7 @@ SinkFinalizeType PhysicalHashAggregate::FinalizeDistinct(Pipeline &pipeline, Eve radix_table->Finalize(context, radix_state); } } - auto new_event = make_refcounted(context, pipeline, *this, gstate); + auto new_event = make_shared_ptr(context, pipeline, *this, gstate); event.InsertEvent(std::move(new_event)); return SinkFinalizeType::READY; } diff --git a/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp b/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp index 97008097513f..5dbbe6d993ea 100644 --- a/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +++ b/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp @@ -586,7 +586,7 @@ SinkFinalizeType PhysicalUngroupedAggregate::FinalizeDistinct(Pipeline &pipeline auto &radix_state = *distinct_state.radix_states[table_idx]; radix_table_p->Finalize(context, radix_state); } - auto new_event = make_refcounted(context, *this, gstate, pipeline); + auto new_event = make_shared_ptr(context, *this, gstate, pipeline); event.InsertEvent(std::move(new_event)); return SinkFinalizeType::READY; } diff --git a/src/execution/operator/aggregate/physical_window.cpp b/src/execution/operator/aggregate/physical_window.cpp index b945615bab16..d4dbf4982a26 100644 --- a/src/execution/operator/aggregate/physical_window.cpp +++ b/src/execution/operator/aggregate/physical_window.cpp @@ -171,7 +171,7 @@ SinkFinalizeType PhysicalWindow::Finalize(Pipeline &pipeline, Event &event, Clie } // Schedule all the sorts for maximum thread utilisation - auto new_event = make_refcounted(*state.global_partition, pipeline); + auto new_event = make_shared_ptr(*state.global_partition, pipeline); event.InsertEvent(std::move(new_event)); return SinkFinalizeType::READY; diff --git a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp index 79de691c2003..934025425b4a 100644 --- a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp +++ b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp @@ -41,7 +41,7 @@ shared_ptr CSVBuffer::Next(CSVFileHandle &file_handle, idx_t buffer_s file_handle.Seek(global_csv_start + actual_buffer_size); has_seaked = false; } - auto next_csv_buffer = make_refcounted( + auto next_csv_buffer = make_shared_ptr( file_handle, context, buffer_size, global_csv_start + actual_buffer_size, file_number_p, buffer_idx + 1); if (next_csv_buffer->GetBufferSize() == 0) { // We are done reading @@ -76,7 +76,7 @@ shared_ptr CSVBuffer::Pin(CSVFileHandle &file_handle, bool &has Reload(file_handle); has_seeked = true; } - return make_refcounted(buffer_manager.Pin(block), actual_buffer_size, requested_size, last_buffer, + return make_shared_ptr(buffer_manager.Pin(block), actual_buffer_size, requested_size, last_buffer, file_number, buffer_idx); } diff --git a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp index 494c4fd04b39..4aba439f1b6d 100644 --- a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp +++ b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp @@ -28,7 +28,7 @@ void CSVBufferManager::UnpinBuffer(const idx_t cache_idx) { void CSVBufferManager::Initialize() { if (cached_buffers.empty()) { cached_buffers.emplace_back( - make_refcounted(context, buffer_size, *file_handle, global_csv_pos, file_idx)); + make_shared_ptr(context, buffer_size, *file_handle, global_csv_pos, file_idx)); last_buffer = cached_buffers.front(); } } diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 791dfe725090..d23bd0e7a056 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -651,14 +651,14 @@ StringValueScanner::StringValueScanner(const shared_ptr &buffe } unique_ptr StringValueScanner::GetCSVScanner(ClientContext &context, CSVReaderOptions &options) { - auto state_machine = make_refcounted(options, options.dialect_options.state_machine_options, + auto state_machine = make_shared_ptr(options, options.dialect_options.state_machine_options, CSVStateMachineCache::Get(context)); state_machine->dialect_options.num_cols = options.dialect_options.num_cols; state_machine->dialect_options.header = options.dialect_options.header; - auto buffer_manager = make_refcounted(context, options, options.file_path, 0); - auto scanner = make_uniq(buffer_manager, state_machine, make_refcounted()); - scanner->csv_file_scan = make_refcounted(context, options.file_path, options); + auto buffer_manager = make_shared_ptr(context, options, options.file_path, 0); + auto scanner = make_uniq(buffer_manager, state_machine, make_shared_ptr()); + scanner->csv_file_scan = make_shared_ptr(context, options.file_path, options); scanner->csv_file_scan->InitializeProjection(); return scanner; } @@ -1222,7 +1222,7 @@ void StringValueScanner::SetStart() { } scan_finder = - make_uniq(0, buffer_manager, state_machine, make_refcounted(true), + make_uniq(0, buffer_manager, state_machine, make_shared_ptr(true), csv_file_scan, false, iterator, 1); auto &tuples = scan_finder->ParseChunk(); line_found = true; diff --git a/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp b/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp index 15fa8db67f03..d98797842994 100644 --- a/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp +++ b/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp @@ -13,8 +13,8 @@ CSVSniffer::CSVSniffer(CSVReaderOptions &options_p, shared_ptr } // Initialize max columns found to either 0 or however many were set max_columns_found = set_columns.Size(); - error_handler = make_refcounted(options.ignore_errors.GetValue()); - detection_error_handler = make_refcounted(true); + error_handler = make_shared_ptr(options.ignore_errors.GetValue()); + detection_error_handler = make_shared_ptr(true); } bool SetColumns::IsSet() { diff --git a/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp b/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp index a2ec291de390..eeb563cf771c 100644 --- a/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp +++ b/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp @@ -10,7 +10,7 @@ CSVFileScan::CSVFileScan(ClientContext &context, shared_ptr bu vector &file_schema) : file_path(options_p.file_path), file_idx(0), buffer_manager(std::move(buffer_manager_p)), state_machine(std::move(state_machine_p)), file_size(buffer_manager->file_handle->FileSize()), - error_handler(make_refcounted(options_p.ignore_errors.GetValue())), + error_handler(make_shared_ptr(options_p.ignore_errors.GetValue())), on_disk_file(buffer_manager->file_handle->OnDiskFile()), options(options_p) { if (bind_data.initial_reader.get()) { auto &union_reader = *bind_data.initial_reader; @@ -43,7 +43,7 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, cons const idx_t file_idx_p, const ReadCSVData &bind_data, const vector &column_ids, const vector &file_schema) : file_path(file_path_p), file_idx(file_idx_p), - error_handler(make_refcounted(options_p.ignore_errors.GetValue())), options(options_p) { + error_handler(make_shared_ptr(options_p.ignore_errors.GetValue())), options(options_p) { if (file_idx < bind_data.union_readers.size()) { // we are doing UNION BY NAME - fetch the options from the union reader for this file optional_ptr union_reader_ptr; @@ -73,7 +73,7 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, cons } // Initialize Buffer Manager - buffer_manager = make_refcounted(context, options, file_path, file_idx); + buffer_manager = make_shared_ptr(context, options, file_path, file_idx); // Initialize On Disk and Size of file on_disk_file = buffer_manager->file_handle->OnDiskFile(); file_size = buffer_manager->file_handle->FileSize(); @@ -89,7 +89,7 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, cons CSVSniffer sniffer(options, buffer_manager, state_machine_cache); sniffer.SniffCSV(); } - state_machine = make_refcounted( + state_machine = make_shared_ptr( state_machine_cache.Get(options.dialect_options.state_machine_options), options); MultiFileReader::InitializeReader(*this, options.file_options, bind_data.reader_bind, bind_data.return_types, @@ -120,7 +120,7 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, cons names = bind_data.csv_names; types = bind_data.csv_types; - state_machine = make_refcounted( + state_machine = make_shared_ptr( state_machine_cache.Get(options.dialect_options.state_machine_options), options); MultiFileReader::InitializeReader(*this, options.file_options, bind_data.reader_bind, bind_data.return_types, @@ -130,8 +130,8 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, cons CSVFileScan::CSVFileScan(ClientContext &context, const string &file_name, CSVReaderOptions &options_p) : file_path(file_name), file_idx(0), - error_handler(make_refcounted(options_p.ignore_errors.GetValue())), options(options_p) { - buffer_manager = make_refcounted(context, options, file_path, file_idx); + error_handler(make_shared_ptr(options_p.ignore_errors.GetValue())), options(options_p) { + buffer_manager = make_shared_ptr(context, options, file_path, file_idx); // Initialize On Disk and Size of file on_disk_file = buffer_manager->file_handle->OnDiskFile(); file_size = buffer_manager->file_handle->FileSize(); @@ -151,7 +151,7 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_name, CSVRea options.dialect_options.num_cols = options.sql_type_list.size(); } // Initialize State Machine - state_machine = make_refcounted( + state_machine = make_shared_ptr( state_machine_cache.Get(options.dialect_options.state_machine_options), options); } diff --git a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp index d358c19aeae7..c1182fc859bf 100644 --- a/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +++ b/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp @@ -14,7 +14,7 @@ CSVGlobalState::CSVGlobalState(ClientContext &context_p, const shared_ptrGetFilePath() == files[0]) { - auto state_machine = make_refcounted( + auto state_machine = make_shared_ptr( CSVStateMachineCache::Get(context).Get(options.dialect_options.state_machine_options), options); // If we already have a buffer manager, we don't need to reconstruct it to the first file file_scans.emplace_back(make_uniq(context, buffer_manager, state_machine, options, bind_data, @@ -36,7 +36,7 @@ CSVGlobalState::CSVGlobalState(ClientContext &context_p, const shared_ptrbuffer_manager->GetBuffer(0)->actual_size; current_boundary = CSVIterator(0, 0, 0, 0, buffer_size); } - current_buffer_in_use = make_refcounted(*file_scans.back()->buffer_manager, 0); + current_buffer_in_use = make_shared_ptr(*file_scans.back()->buffer_manager, 0); } double CSVGlobalState::GetProgress(const ReadCSVData &bind_data_p) const { @@ -67,7 +67,7 @@ unique_ptr CSVGlobalState::Next(optional_ptr parallel_lock(main_mutex); - file_scans.emplace_back(make_refcounted(context, bind_data.files[cur_idx], bind_data.options, + file_scans.emplace_back(make_shared_ptr(context, bind_data.files[cur_idx], bind_data.options, cur_idx, bind_data, column_ids, file_schema)); current_file = file_scans.back(); } @@ -88,7 +88,7 @@ unique_ptr CSVGlobalState::Next(optional_ptrbuffer_idx != current_boundary.GetBufferIdx()) { current_buffer_in_use = - make_refcounted(*file_scans.back()->buffer_manager, current_boundary.GetBufferIdx()); + make_shared_ptr(*file_scans.back()->buffer_manager, current_boundary.GetBufferIdx()); } // We first create the scanner for the current boundary auto ¤t_file = *file_scans.back(); @@ -111,13 +111,13 @@ unique_ptr CSVGlobalState::Next(optional_ptr(context, bind_data.files[current_file_idx], + file_scans.emplace_back(make_shared_ptr(context, bind_data.files[current_file_idx], bind_data.options, current_file_idx, bind_data, column_ids, file_schema)); // And re-start the boundary-iterator auto buffer_size = file_scans.back()->buffer_manager->GetBuffer(0)->actual_size; current_boundary = CSVIterator(current_file_idx, 0, 0, 0, buffer_size); - current_buffer_in_use = make_refcounted(*file_scans.back()->buffer_manager, 0); + current_buffer_in_use = make_shared_ptr(*file_scans.back()->buffer_manager, 0); } else { // If not we are done with this CSV Scanning finished = true; diff --git a/src/execution/operator/helper/physical_buffered_collector.cpp b/src/execution/operator/helper/physical_buffered_collector.cpp index 90708953a643..9f2ac70db09d 100644 --- a/src/execution/operator/helper/physical_buffered_collector.cpp +++ b/src/execution/operator/helper/physical_buffered_collector.cpp @@ -55,7 +55,7 @@ SinkCombineResultType PhysicalBufferedCollector::Combine(ExecutionContext &conte unique_ptr PhysicalBufferedCollector::GetGlobalSinkState(ClientContext &context) const { auto state = make_uniq(); state->context = context.shared_from_this(); - state->buffered_data = make_refcounted(state->context); + state->buffered_data = make_shared_ptr(state->context); return std::move(state); } diff --git a/src/execution/operator/join/physical_asof_join.cpp b/src/execution/operator/join/physical_asof_join.cpp index 05e45d7a455f..29bdef93bb5d 100644 --- a/src/execution/operator/join/physical_asof_join.cpp +++ b/src/execution/operator/join/physical_asof_join.cpp @@ -169,7 +169,7 @@ SinkFinalizeType PhysicalAsOfJoin::Finalize(Pipeline &pipeline, Event &event, Cl } // Schedule all the sorts for maximum thread utilisation - auto new_event = make_refcounted(gstate.rhs_sink, pipeline); + auto new_event = make_shared_ptr(gstate.rhs_sink, pipeline); event.InsertEvent(std::move(new_event)); return SinkFinalizeType::READY; diff --git a/src/execution/operator/join/physical_hash_join.cpp b/src/execution/operator/join/physical_hash_join.cpp index 09c336458300..f7333cd660fc 100644 --- a/src/execution/operator/join/physical_hash_join.cpp +++ b/src/execution/operator/join/physical_hash_join.cpp @@ -359,7 +359,7 @@ void HashJoinGlobalSinkState::ScheduleFinalize(Pipeline &pipeline, Event &event) return; } hash_table->InitializePointerTable(); - auto new_event = make_refcounted(pipeline, *this); + auto new_event = make_shared_ptr(pipeline, *this); event.InsertEvent(std::move(new_event)); } @@ -474,7 +474,7 @@ SinkFinalizeType PhysicalHashJoin::Finalize(Pipeline &pipeline, Event &event, Cl // We have to repartition ht.SetRepartitionRadixBits(sink.local_hash_tables, sink.temporary_memory_state->GetReservation(), max_partition_size, max_partition_count); - auto new_event = make_refcounted(pipeline, sink, sink.local_hash_tables); + auto new_event = make_shared_ptr(pipeline, sink, sink.local_hash_tables); event.InsertEvent(std::move(new_event)); } else { // No repartitioning! diff --git a/src/execution/operator/join/physical_range_join.cpp b/src/execution/operator/join/physical_range_join.cpp index d89360236a3b..996c0a4a7e4b 100644 --- a/src/execution/operator/join/physical_range_join.cpp +++ b/src/execution/operator/join/physical_range_join.cpp @@ -149,7 +149,7 @@ class RangeJoinMergeEvent : public BasePipelineEvent { void PhysicalRangeJoin::GlobalSortedTable::ScheduleMergeTasks(Pipeline &pipeline, Event &event) { // Initialize global sort state for a round of merging global_sort_state.InitializeMergeRound(); - auto new_event = make_refcounted(*this, pipeline); + auto new_event = make_shared_ptr(*this, pipeline); event.InsertEvent(std::move(new_event)); } diff --git a/src/execution/operator/order/physical_order.cpp b/src/execution/operator/order/physical_order.cpp index 8687acfe5ee1..f12076b80687 100644 --- a/src/execution/operator/order/physical_order.cpp +++ b/src/execution/operator/order/physical_order.cpp @@ -187,7 +187,7 @@ SinkFinalizeType PhysicalOrder::Finalize(Pipeline &pipeline, Event &event, Clien void PhysicalOrder::ScheduleMergeTasks(Pipeline &pipeline, Event &event, OrderGlobalSinkState &state) { // Initialize global sort state for a round of merging state.global_sort_state.InitializeMergeRound(); - auto new_event = make_refcounted(state, pipeline); + auto new_event = make_shared_ptr(state, pipeline); event.InsertEvent(std::move(new_event)); } diff --git a/src/execution/operator/persistent/physical_batch_copy_to_file.cpp b/src/execution/operator/persistent/physical_batch_copy_to_file.cpp index 831ea35b6e49..30699ee47c2f 100644 --- a/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +++ b/src/execution/operator/persistent/physical_batch_copy_to_file.cpp @@ -308,7 +308,7 @@ SinkFinalizeType PhysicalBatchCopyToFile::Finalize(Pipeline &pipeline, Event &ev FinalFlush(context, input.global_state); } else { // we have multiple tasks remaining - launch an event to execute the tasks in parallel - auto new_event = make_refcounted(*this, gstate, pipeline, context); + auto new_event = make_shared_ptr(*this, gstate, pipeline, context); event.InsertEvent(std::move(new_event)); } return SinkFinalizeType::READY; diff --git a/src/execution/operator/persistent/physical_copy_to_file.cpp b/src/execution/operator/persistent/physical_copy_to_file.cpp index be9ca4dbbd61..343551dd2c94 100644 --- a/src/execution/operator/persistent/physical_copy_to_file.cpp +++ b/src/execution/operator/persistent/physical_copy_to_file.cpp @@ -284,7 +284,7 @@ unique_ptr PhysicalCopyToFile::GetGlobalSinkState(ClientContext } if (partition_output) { - state->partition_state = make_refcounted(); + state->partition_state = make_shared_ptr(); } return std::move(state); diff --git a/src/execution/operator/schema/physical_create_art_index.cpp b/src/execution/operator/schema/physical_create_art_index.cpp index fe88d7c3bfa4..94aa2b74b9e9 100644 --- a/src/execution/operator/schema/physical_create_art_index.cpp +++ b/src/execution/operator/schema/physical_create_art_index.cpp @@ -178,7 +178,7 @@ SinkFinalizeType PhysicalCreateARTIndex::Finalize(Pipeline &pipeline, Event &eve auto &index = index_entry->Cast(); index.initial_index_size = state.global_index->GetInMemorySize(); - index.info = make_refcounted(storage.info, index.name); + index.info = make_shared_ptr(storage.info, index.name); for (auto &parsed_expr : info->parsed_expressions) { index.parsed_expressions.push_back(parsed_expr->Copy()); } diff --git a/src/execution/operator/set/physical_recursive_cte.cpp b/src/execution/operator/set/physical_recursive_cte.cpp index 5210987325ab..406299ac2939 100644 --- a/src/execution/operator/set/physical_recursive_cte.cpp +++ b/src/execution/operator/set/physical_recursive_cte.cpp @@ -200,7 +200,7 @@ void PhysicalRecursiveCTE::BuildPipelines(Pipeline ¤t, MetaPipeline &meta_ initial_state_pipeline.Build(*children[0]); // the RHS is the recursive pipeline - recursive_meta_pipeline = make_refcounted(executor, state, this); + recursive_meta_pipeline = make_shared_ptr(executor, state, this); recursive_meta_pipeline->SetRecursiveCTE(); recursive_meta_pipeline->Build(*children[1]); diff --git a/src/execution/physical_plan/plan_cte.cpp b/src/execution/physical_plan/plan_cte.cpp index c11286a8cf3f..190cb9319bc7 100644 --- a/src/execution/physical_plan/plan_cte.cpp +++ b/src/execution/physical_plan/plan_cte.cpp @@ -12,7 +12,7 @@ unique_ptr PhysicalPlanGenerator::CreatePlan(LogicalMaterializ D_ASSERT(op.children.size() == 2); // Create the working_table that the PhysicalCTE will use for evaluation. - auto working_table = make_refcounted(context, op.children[0]->types); + auto working_table = make_shared_ptr(context, op.children[0]->types); // Add the ColumnDataCollection to the context of this PhysicalPlanGenerator recursive_cte_tables[op.table_index] = working_table; diff --git a/src/execution/physical_plan/plan_recursive_cte.cpp b/src/execution/physical_plan/plan_recursive_cte.cpp index 5ddb767612b6..7933dd1660ba 100644 --- a/src/execution/physical_plan/plan_recursive_cte.cpp +++ b/src/execution/physical_plan/plan_recursive_cte.cpp @@ -12,7 +12,7 @@ unique_ptr PhysicalPlanGenerator::CreatePlan(LogicalRecursiveC D_ASSERT(op.children.size() == 2); // Create the working_table that the PhysicalRecursiveCTE will use for evaluation. - auto working_table = make_refcounted(context, op.types); + auto working_table = make_shared_ptr(context, op.types); // Add the ColumnDataCollection to the context of this PhysicalPlanGenerator recursive_cte_tables[op.table_index] = working_table; diff --git a/src/function/table/copy_csv.cpp b/src/function/table/copy_csv.cpp index e45a6a643120..0814d0f2d655 100644 --- a/src/function/table/copy_csv.cpp +++ b/src/function/table/copy_csv.cpp @@ -156,7 +156,7 @@ static unique_ptr ReadCSVBind(ClientContext &context, CopyInfo &in } if (options.auto_detect) { - auto buffer_manager = make_refcounted(context, options, bind_data->files[0], 0); + auto buffer_manager = make_shared_ptr(context, options, bind_data->files[0], 0); CSVSniffer sniffer(options, buffer_manager, CSVStateMachineCache::Get(context), {&expected_types, &expected_names}); sniffer.SniffCSV(); diff --git a/src/function/table/read_csv.cpp b/src/function/table/read_csv.cpp index dc8ec7f3abe7..6c949913762b 100644 --- a/src/function/table/read_csv.cpp +++ b/src/function/table/read_csv.cpp @@ -88,7 +88,7 @@ static unique_ptr ReadCSVBind(ClientContext &context, TableFunctio } if (options.auto_detect && !options.file_options.union_by_name) { options.file_path = result->files[0]; - result->buffer_manager = make_refcounted(context, options, result->files[0], 0); + result->buffer_manager = make_shared_ptr(context, options, result->files[0], 0); CSVSniffer sniffer(options, result->buffer_manager, CSVStateMachineCache::Get(context), {&return_types, &names}); auto sniffer_result = sniffer.SniffCSV(); diff --git a/src/function/table/sniff_csv.cpp b/src/function/table/sniff_csv.cpp index b7998902d93d..11dfb7797577 100644 --- a/src/function/table/sniff_csv.cpp +++ b/src/function/table/sniff_csv.cpp @@ -109,7 +109,7 @@ static void CSVSniffFunction(ClientContext &context, TableFunctionInput &data_p, auto sniffer_options = data.options; sniffer_options.file_path = data.path; - auto buffer_manager = make_refcounted(context, sniffer_options, sniffer_options.file_path, 0); + auto buffer_manager = make_shared_ptr(context, sniffer_options, sniffer_options.file_path, 0); if (sniffer_options.name_list.empty()) { sniffer_options.name_list = data.names_csv; } diff --git a/src/include/duckdb/common/helper.hpp b/src/include/duckdb/common/helper.hpp index ed3e6e1f1be3..789e9042febf 100644 --- a/src/include/duckdb/common/helper.hpp +++ b/src/include/duckdb/common/helper.hpp @@ -68,7 +68,7 @@ make_uniq(ARGS&&... args) // NOLINT: mimic std style template inline shared_ptr -make_refcounted(ARGS&&... args) // NOLINT: mimic std style +make_shared_ptr(ARGS&&... args) // NOLINT: mimic std style { return shared_ptr(std::make_shared(std::forward(args)...)); } @@ -125,7 +125,7 @@ shared_ptr shared_ptr_cast(shared_ptr src) { // NOLINT: mimic std style struct SharedConstructor { template static shared_ptr Create(ARGS &&...args) { - return make_refcounted(std::forward(args)...); + return make_shared_ptr(std::forward(args)...); } }; diff --git a/src/include/duckdb/common/multi_file_reader.hpp b/src/include/duckdb/common/multi_file_reader.hpp index ec318dea76e0..d7c106d00a4b 100644 --- a/src/include/duckdb/common/multi_file_reader.hpp +++ b/src/include/duckdb/common/multi_file_reader.hpp @@ -151,7 +151,7 @@ struct MultiFileReader { return BindUnionReader(context, return_types, names, result, options); } else { shared_ptr reader; - reader = make_refcounted(context, result.files[0], options); + reader = make_shared_ptr(context, result.files[0], options); return_types = reader->return_types; names = reader->names; result.Initialize(std::move(reader)); diff --git a/src/include/duckdb/common/types.hpp b/src/include/duckdb/common/types.hpp index e9e31b488ea5..6e38fa83b89d 100644 --- a/src/include/duckdb/common/types.hpp +++ b/src/include/duckdb/common/types.hpp @@ -35,7 +35,7 @@ using buffer_ptr = shared_ptr; template buffer_ptr make_buffer(ARGS &&...args) { // NOLINT: mimic std casing - return make_refcounted(std::forward(args)...); + return make_shared_ptr(std::forward(args)...); } struct list_entry_t { // NOLINT: mimic std casing diff --git a/src/include/duckdb/common/types/selection_vector.hpp b/src/include/duckdb/common/types/selection_vector.hpp index a0f0b185beae..969c785690d4 100644 --- a/src/include/duckdb/common/types/selection_vector.hpp +++ b/src/include/duckdb/common/types/selection_vector.hpp @@ -71,7 +71,7 @@ struct SelectionVector { sel_vector = sel; } void Initialize(idx_t count = STANDARD_VECTOR_SIZE) { - selection_data = make_refcounted(count); + selection_data = make_shared_ptr(count); sel_vector = selection_data->owned_data.get(); } void Initialize(buffer_ptr data) { diff --git a/src/include/duckdb/planner/binder.hpp b/src/include/duckdb/planner/binder.hpp index fa1734b1f360..c92559fab142 100644 --- a/src/include/duckdb/planner/binder.hpp +++ b/src/include/duckdb/planner/binder.hpp @@ -376,7 +376,7 @@ class Binder : public enable_shared_from_this { unique_ptr BindSummarize(ShowRef &ref); public: - // This should really be a private constructor, but make_refcounted does not allow it... + // This should really be a private constructor, but make_shared_ptr does not allow it... // If you are thinking about calling this, you should probably call Binder::CreateBinder Binder(bool i_know_what_i_am_doing, ClientContext &context, shared_ptr parent, bool inherit_ctes); }; diff --git a/src/include/duckdb/storage/object_cache.hpp b/src/include/duckdb/storage/object_cache.hpp index 06a5c2d3a767..5ef1a6733201 100644 --- a/src/include/duckdb/storage/object_cache.hpp +++ b/src/include/duckdb/storage/object_cache.hpp @@ -52,7 +52,7 @@ class ObjectCache { auto entry = cache.find(key); if (entry == cache.end()) { - auto value = make_refcounted(args...); + auto value = make_shared_ptr(args...); cache[key] = value; return value; } diff --git a/src/include/duckdb/storage/serialization/types.json b/src/include/duckdb/storage/serialization/types.json index 5433f50a9d15..8f3adec0837e 100644 --- a/src/include/duckdb/storage/serialization/types.json +++ b/src/include/duckdb/storage/serialization/types.json @@ -155,7 +155,7 @@ "class": "GenericTypeInfo", "base": "ExtraTypeInfo", "enum": "GENERIC_TYPE_INFO", - "custom_switch_code": "result = make_refcounted(type);\nbreak;" + "custom_switch_code": "result = make_shared_ptr(type);\nbreak;" }, { "class": "AnyTypeInfo", diff --git a/src/main/capi/table_function-c.cpp b/src/main/capi/table_function-c.cpp index 57be51d01f4b..a6916e8e2007 100644 --- a/src/main/capi/table_function-c.cpp +++ b/src/main/capi/table_function-c.cpp @@ -179,7 +179,7 @@ void CTableFunction(ClientContext &context, TableFunctionInput &data_p, DataChun duckdb_table_function duckdb_create_table_function() { auto function = new duckdb::TableFunction("", {}, duckdb::CTableFunction, duckdb::CTableFunctionBind, duckdb::CTableFunctionInit, duckdb::CTableFunctionLocalInit); - function->function_info = duckdb::make_refcounted(); + function->function_info = duckdb::make_shared_ptr(); function->cardinality = duckdb::CTableFunctionCardinality; return function; } diff --git a/src/main/client_context.cpp b/src/main/client_context.cpp index 49275d1dd7d0..515ab5fc732b 100644 --- a/src/main/client_context.cpp +++ b/src/main/client_context.cpp @@ -312,7 +312,7 @@ ClientContext::CreatePreparedStatementInternal(ClientContextLock &lock, const st unique_ptr statement, optional_ptr> values) { StatementType statement_type = statement->type; - auto result = make_refcounted(statement_type); + auto result = make_shared_ptr(statement_type); auto &profiler = QueryProfiler::Get(*this); profiler.StartQuery(query, IsExplainAnalyze(statement.get()), true); diff --git a/src/main/client_data.cpp b/src/main/client_data.cpp index 4dda60a7e053..54c5292cf339 100644 --- a/src/main/client_data.cpp +++ b/src/main/client_data.cpp @@ -35,8 +35,8 @@ class ClientFileSystem : public OpenerFileSystem { ClientData::ClientData(ClientContext &context) : catalog_search_path(make_uniq(context)) { auto &db = DatabaseInstance::GetDatabase(context); - profiler = make_refcounted(context); - temporary_objects = make_refcounted(db, AttachedDatabaseType::TEMP_DATABASE); + profiler = make_shared_ptr(context); + temporary_objects = make_shared_ptr(db, AttachedDatabaseType::TEMP_DATABASE); temporary_objects->oid = DatabaseManager::Get(db).ModifyCatalog(); random_engine = make_uniq(); file_opener = make_uniq(context); diff --git a/src/main/connection.cpp b/src/main/connection.cpp index b76d440647f7..11346a3a9846 100644 --- a/src/main/connection.cpp +++ b/src/main/connection.cpp @@ -19,7 +19,7 @@ namespace duckdb { Connection::Connection(DatabaseInstance &database) - : context(make_refcounted(database.shared_from_this())) { + : context(make_shared_ptr(database.shared_from_this())) { ConnectionManager::Get(database).AddConnection(*context); #ifdef DEBUG EnableProfiling(); @@ -187,7 +187,7 @@ shared_ptr Connection::Table(const string &schema_name, const string & if (!table_info) { throw CatalogException("Table '%s' does not exist!", table_name); } - return make_refcounted(context, std::move(table_info)); + return make_shared_ptr(context, std::move(table_info)); } shared_ptr Connection::View(const string &tname) { @@ -195,7 +195,7 @@ shared_ptr Connection::View(const string &tname) { } shared_ptr Connection::View(const string &schema_name, const string &table_name) { - return make_refcounted(context, schema_name, table_name); + return make_shared_ptr(context, schema_name, table_name); } shared_ptr Connection::TableFunction(const string &fname) { @@ -206,11 +206,11 @@ shared_ptr Connection::TableFunction(const string &fname) { shared_ptr Connection::TableFunction(const string &fname, const vector &values, const named_parameter_map_t &named_parameters) { - return make_refcounted(context, fname, values, named_parameters); + return make_shared_ptr(context, fname, values, named_parameters); } shared_ptr Connection::TableFunction(const string &fname, const vector &values) { - return make_refcounted(context, fname, values); + return make_shared_ptr(context, fname, values); } shared_ptr Connection::Values(const vector> &values) { @@ -220,7 +220,7 @@ shared_ptr Connection::Values(const vector> &values) { shared_ptr Connection::Values(const vector> &values, const vector &column_names, const string &alias) { - return make_refcounted(context, values, column_names, alias); + return make_shared_ptr(context, values, column_names, alias); } shared_ptr Connection::Values(const string &values) { @@ -229,7 +229,7 @@ shared_ptr Connection::Values(const string &values) { } shared_ptr Connection::Values(const string &values, const vector &column_names, const string &alias) { - return make_refcounted(context, values, column_names, alias); + return make_shared_ptr(context, values, column_names, alias); } shared_ptr Connection::ReadCSV(const string &csv_file) { @@ -238,7 +238,7 @@ shared_ptr Connection::ReadCSV(const string &csv_file) { } shared_ptr Connection::ReadCSV(const vector &csv_input, named_parameter_map_t &&options) { - return make_refcounted(context, csv_input, std::move(options)); + return make_shared_ptr(context, csv_input, std::move(options)); } shared_ptr Connection::ReadCSV(const string &csv_input, named_parameter_map_t &&options) { @@ -259,7 +259,7 @@ shared_ptr Connection::ReadCSV(const string &csv_file, const vector files {csv_file}; - return make_refcounted(context, files, std::move(options)); + return make_shared_ptr(context, files, std::move(options)); } shared_ptr Connection::ReadParquet(const string &parquet_file, bool binary_as_string) { @@ -278,7 +278,7 @@ shared_ptr Connection::RelationFromQuery(const string &query, const st } shared_ptr Connection::RelationFromQuery(unique_ptr select_stmt, const string &alias) { - return make_refcounted(context, std::move(select_stmt), alias); + return make_shared_ptr(context, std::move(select_stmt), alias); } void Connection::BeginTransaction() { diff --git a/src/main/database.cpp b/src/main/database.cpp index 4fb6ef547d99..5568ca26bffc 100644 --- a/src/main/database.cpp +++ b/src/main/database.cpp @@ -272,7 +272,7 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf scheduler->RelaunchThreads(); } -DuckDB::DuckDB(const char *path, DBConfig *new_config) : instance(make_refcounted()) { +DuckDB::DuckDB(const char *path, DBConfig *new_config) : instance(make_shared_ptr()) { instance->Initialize(path, new_config); if (instance->config.options.load_extensions) { ExtensionHelper::LoadAllExtensions(*this); @@ -380,7 +380,7 @@ void DatabaseInstance::Configure(DBConfig &new_config) { if (new_config.buffer_pool) { config.buffer_pool = std::move(new_config.buffer_pool); } else { - config.buffer_pool = make_refcounted(config.options.maximum_memory); + config.buffer_pool = make_shared_ptr(config.options.maximum_memory); } } diff --git a/src/main/db_instance_cache.cpp b/src/main/db_instance_cache.cpp index 6105066c6c88..a72592fd8346 100644 --- a/src/main/db_instance_cache.cpp +++ b/src/main/db_instance_cache.cpp @@ -66,7 +66,7 @@ shared_ptr DBInstanceCache::CreateInstanceInternal(const string &databas if (abs_database_path.rfind(IN_MEMORY_PATH, 0) == 0) { instance_path = IN_MEMORY_PATH; } - auto db_instance = make_refcounted(instance_path, &config); + auto db_instance = make_shared_ptr(instance_path, &config); if (cache_instance) { db_instances[abs_database_path] = db_instance; } diff --git a/src/main/relation.cpp b/src/main/relation.cpp index f315906bd200..46841d4e70ed 100644 --- a/src/main/relation.cpp +++ b/src/main/relation.cpp @@ -39,7 +39,7 @@ shared_ptr Relation::Project(const string &expression, const string &a shared_ptr Relation::Project(const string &select_list, const vector &aliases) { auto expressions = Parser::ParseExpressionList(select_list, context.GetContext()->GetParserOptions()); - return make_refcounted(shared_from_this(), std::move(expressions), aliases); + return make_shared_ptr(shared_from_this(), std::move(expressions), aliases); } shared_ptr Relation::Project(const vector &expressions) { @@ -49,7 +49,7 @@ shared_ptr Relation::Project(const vector &expressions) { shared_ptr Relation::Project(vector> expressions, const vector &aliases) { - return make_refcounted(shared_from_this(), std::move(expressions), aliases); + return make_shared_ptr(shared_from_this(), std::move(expressions), aliases); } static vector> StringListToExpressionList(ClientContext &context, @@ -70,7 +70,7 @@ static vector> StringListToExpressionList(ClientCon shared_ptr Relation::Project(const vector &expressions, const vector &aliases) { auto result_list = StringListToExpressionList(*context.GetContext(), expressions); - return make_refcounted(shared_from_this(), std::move(result_list), aliases); + return make_shared_ptr(shared_from_this(), std::move(result_list), aliases); } shared_ptr Relation::Filter(const string &expression) { @@ -82,7 +82,7 @@ shared_ptr Relation::Filter(const string &expression) { } shared_ptr Relation::Filter(unique_ptr expression) { - return make_refcounted(shared_from_this(), std::move(expression)); + return make_shared_ptr(shared_from_this(), std::move(expression)); } shared_ptr Relation::Filter(const vector &expressions) { @@ -95,11 +95,11 @@ shared_ptr Relation::Filter(const vector &expressions) { expr = make_uniq(ExpressionType::CONJUNCTION_AND, std::move(expr), std::move(expression_list[i])); } - return make_refcounted(shared_from_this(), std::move(expr)); + return make_shared_ptr(shared_from_this(), std::move(expr)); } shared_ptr Relation::Limit(int64_t limit, int64_t offset) { - return make_refcounted(shared_from_this(), limit, offset); + return make_shared_ptr(shared_from_this(), limit, offset); } shared_ptr Relation::Order(const string &expression) { @@ -108,7 +108,7 @@ shared_ptr Relation::Order(const string &expression) { } shared_ptr Relation::Order(vector order_list) { - return make_refcounted(shared_from_this(), std::move(order_list)); + return make_shared_ptr(shared_from_this(), std::move(order_list)); } shared_ptr Relation::Order(const vector &expressions) { @@ -149,51 +149,51 @@ shared_ptr Relation::Join(const shared_ptr &other, } using_columns.push_back(colref.column_names[0]); } - return make_refcounted(shared_from_this(), other, std::move(using_columns), type, ref_type); + return make_shared_ptr(shared_from_this(), other, std::move(using_columns), type, ref_type); } else { // single expression that is not a column reference: use the expression as a join condition - return make_refcounted(shared_from_this(), other, std::move(expression_list[0]), type, ref_type); + return make_shared_ptr(shared_from_this(), other, std::move(expression_list[0]), type, ref_type); } } shared_ptr Relation::CrossProduct(const shared_ptr &other, JoinRefType join_ref_type) { - return make_refcounted(shared_from_this(), other, join_ref_type); + return make_shared_ptr(shared_from_this(), other, join_ref_type); } shared_ptr Relation::Union(const shared_ptr &other) { - return make_refcounted(shared_from_this(), other, SetOperationType::UNION, true); + return make_shared_ptr(shared_from_this(), other, SetOperationType::UNION, true); } shared_ptr Relation::Except(const shared_ptr &other) { - return make_refcounted(shared_from_this(), other, SetOperationType::EXCEPT, true); + return make_shared_ptr(shared_from_this(), other, SetOperationType::EXCEPT, true); } shared_ptr Relation::Intersect(const shared_ptr &other) { - return make_refcounted(shared_from_this(), other, SetOperationType::INTERSECT, true); + return make_shared_ptr(shared_from_this(), other, SetOperationType::INTERSECT, true); } shared_ptr Relation::Distinct() { - return make_refcounted(shared_from_this()); + return make_shared_ptr(shared_from_this()); } shared_ptr Relation::Alias(const string &alias) { - return make_refcounted(shared_from_this(), alias); + return make_shared_ptr(shared_from_this(), alias); } shared_ptr Relation::Aggregate(const string &aggregate_list) { auto expression_list = Parser::ParseExpressionList(aggregate_list, context.GetContext()->GetParserOptions()); - return make_refcounted(shared_from_this(), std::move(expression_list)); + return make_shared_ptr(shared_from_this(), std::move(expression_list)); } shared_ptr Relation::Aggregate(const string &aggregate_list, const string &group_list) { auto expression_list = Parser::ParseExpressionList(aggregate_list, context.GetContext()->GetParserOptions()); auto groups = Parser::ParseGroupByList(group_list, context.GetContext()->GetParserOptions()); - return make_refcounted(shared_from_this(), std::move(expression_list), std::move(groups)); + return make_shared_ptr(shared_from_this(), std::move(expression_list), std::move(groups)); } shared_ptr Relation::Aggregate(const vector &aggregates) { auto aggregate_list = StringListToExpressionList(*context.GetContext(), aggregates); - return make_refcounted(shared_from_this(), std::move(aggregate_list)); + return make_shared_ptr(shared_from_this(), std::move(aggregate_list)); } shared_ptr Relation::Aggregate(const vector &aggregates, const vector &groups) { @@ -204,7 +204,7 @@ shared_ptr Relation::Aggregate(const vector &aggregates, const shared_ptr Relation::Aggregate(vector> expressions, const string &group_list) { auto groups = Parser::ParseGroupByList(group_list, context.GetContext()->GetParserOptions()); - return make_refcounted(shared_from_this(), std::move(expressions), std::move(groups)); + return make_shared_ptr(shared_from_this(), std::move(expressions), std::move(groups)); } string Relation::GetAlias() { @@ -237,7 +237,7 @@ BoundStatement Relation::Bind(Binder &binder) { } shared_ptr Relation::InsertRel(const string &schema_name, const string &table_name) { - return make_refcounted(shared_from_this(), schema_name, table_name); + return make_shared_ptr(shared_from_this(), schema_name, table_name); } void Relation::Insert(const string &table_name) { @@ -255,12 +255,12 @@ void Relation::Insert(const string &schema_name, const string &table_name) { void Relation::Insert(const vector> &values) { vector column_names; - auto rel = make_refcounted(context.GetContext(), values, std::move(column_names), "values"); + auto rel = make_shared_ptr(context.GetContext(), values, std::move(column_names), "values"); rel->Insert(GetAlias()); } shared_ptr Relation::CreateRel(const string &schema_name, const string &table_name) { - return make_refcounted(shared_from_this(), schema_name, table_name); + return make_shared_ptr(shared_from_this(), schema_name, table_name); } void Relation::Create(const string &table_name) { @@ -277,7 +277,7 @@ void Relation::Create(const string &schema_name, const string &table_name) { } shared_ptr Relation::WriteCSVRel(const string &csv_file, case_insensitive_map_t> options) { - return make_refcounted(shared_from_this(), csv_file, std::move(options)); + return make_shared_ptr(shared_from_this(), csv_file, std::move(options)); } void Relation::WriteCSV(const string &csv_file, case_insensitive_map_t> options) { @@ -292,7 +292,7 @@ void Relation::WriteCSV(const string &csv_file, case_insensitive_map_t Relation::WriteParquetRel(const string &parquet_file, case_insensitive_map_t> options) { auto write_parquet = - make_refcounted(shared_from_this(), parquet_file, std::move(options)); + make_shared_ptr(shared_from_this(), parquet_file, std::move(options)); return std::move(write_parquet); } @@ -310,7 +310,7 @@ shared_ptr Relation::CreateView(const string &name, bool replace, bool } shared_ptr Relation::CreateView(const string &schema_name, const string &name, bool replace, bool temporary) { - auto view = make_refcounted(shared_from_this(), schema_name, name, replace, temporary); + auto view = make_shared_ptr(shared_from_this(), schema_name, name, replace, temporary); auto res = view->Execute(); if (res->HasError()) { const string prepended_message = "Failed to create view '" + name + "': "; @@ -329,7 +329,7 @@ unique_ptr Relation::Query(const string &name, const string &sql) { } unique_ptr Relation::Explain(ExplainType type) { - auto explain = make_refcounted(shared_from_this(), type); + auto explain = make_shared_ptr(shared_from_this(), type); return explain->Execute(); } @@ -343,12 +343,12 @@ void Relation::Delete(const string &condition) { shared_ptr Relation::TableFunction(const std::string &fname, const vector &values, const named_parameter_map_t &named_parameters) { - return make_refcounted(context.GetContext(), fname, values, named_parameters, + return make_shared_ptr(context.GetContext(), fname, values, named_parameters, shared_from_this()); } shared_ptr Relation::TableFunction(const std::string &fname, const vector &values) { - return make_refcounted(context.GetContext(), fname, values, shared_from_this()); + return make_shared_ptr(context.GetContext(), fname, values, shared_from_this()); } string Relation::ToString() { diff --git a/src/main/relation/read_csv_relation.cpp b/src/main/relation/read_csv_relation.cpp index f63d535c13ab..cce7dba4ec8d 100644 --- a/src/main/relation/read_csv_relation.cpp +++ b/src/main/relation/read_csv_relation.cpp @@ -56,7 +56,7 @@ ReadCSVRelation::ReadCSVRelation(const shared_ptr &context, const shared_ptr buffer_manager; context->RunFunctionInTransaction([&]() { - buffer_manager = make_refcounted(*context, csv_options, files[0], 0); + buffer_manager = make_shared_ptr(*context, csv_options, files[0], 0); CSVSniffer sniffer(csv_options, buffer_manager, CSVStateMachineCache::Get(*context)); auto sniffer_result = sniffer.SniffCSV(); auto &types = sniffer_result.return_types; diff --git a/src/main/relation/table_relation.cpp b/src/main/relation/table_relation.cpp index c37c88507849..4a0ff6e0e50d 100644 --- a/src/main/relation/table_relation.cpp +++ b/src/main/relation/table_relation.cpp @@ -56,14 +56,14 @@ void TableRelation::Update(const string &update_list, const string &condition) { vector> expressions; auto cond = ParseCondition(*context.GetContext(), condition); Parser::ParseUpdateList(update_list, update_columns, expressions, context.GetContext()->GetParserOptions()); - auto update = make_refcounted(context, std::move(cond), description->schema, description->table, + auto update = make_shared_ptr(context, std::move(cond), description->schema, description->table, std::move(update_columns), std::move(expressions)); update->Execute(); } void TableRelation::Delete(const string &condition) { auto cond = ParseCondition(*context.GetContext(), condition); - auto del = make_refcounted(context, std::move(cond), description->schema, description->table); + auto del = make_shared_ptr(context, std::move(cond), description->schema, description->table); del->Execute(); } diff --git a/src/parallel/executor.cpp b/src/parallel/executor.cpp index a3be9b315843..827890080bea 100644 --- a/src/parallel/executor.cpp +++ b/src/parallel/executor.cpp @@ -73,11 +73,11 @@ void Executor::SchedulePipeline(const shared_ptr &meta_pipeline, S // create events/stack for the base pipeline auto base_pipeline = meta_pipeline->GetBasePipeline(); - auto base_initialize_event = make_refcounted(base_pipeline); - auto base_event = make_refcounted(base_pipeline); - auto base_finish_event = make_refcounted(base_pipeline); + auto base_initialize_event = make_shared_ptr(base_pipeline); + auto base_event = make_shared_ptr(base_pipeline); + auto base_finish_event = make_shared_ptr(base_pipeline); auto base_complete_event = - make_refcounted(base_pipeline->executor, event_data.initial_schedule); + make_shared_ptr(base_pipeline->executor, event_data.initial_schedule); PipelineEventStack base_stack(*base_initialize_event, *base_event, *base_finish_event, *base_complete_event); events.push_back(std::move(base_initialize_event)); events.push_back(std::move(base_event)); @@ -97,7 +97,7 @@ void Executor::SchedulePipeline(const shared_ptr &meta_pipeline, S D_ASSERT(pipeline); // create events/stack for this pipeline - auto pipeline_event = make_refcounted(pipeline); + auto pipeline_event = make_shared_ptr(pipeline); auto finish_group = meta_pipeline->GetFinishGroup(*pipeline); if (finish_group) { @@ -116,7 +116,7 @@ void Executor::SchedulePipeline(const shared_ptr &meta_pipeline, S event_map.insert(make_pair(reference(*pipeline), pipeline_stack)); } else if (meta_pipeline->HasFinishEvent(*pipeline)) { // this pipeline has its own finish event (despite going into the same sink - Finalize twice!) - auto pipeline_finish_event = make_refcounted(pipeline); + auto pipeline_finish_event = make_shared_ptr(pipeline); PipelineEventStack pipeline_stack(base_stack.pipeline_initialize_event, *pipeline_event, *pipeline_finish_event, base_stack.pipeline_complete_event); events.push_back(std::move(pipeline_finish_event)); @@ -360,7 +360,7 @@ void Executor::InitializeInternal(PhysicalOperator &plan) { // build and ready the pipelines PipelineBuildState state; - auto root_pipeline = make_refcounted(*this, state, nullptr); + auto root_pipeline = make_shared_ptr(*this, state, nullptr); root_pipeline->Build(*physical_plan); root_pipeline->Ready(); @@ -571,7 +571,7 @@ shared_ptr Executor::CreateChildPipeline(Pipeline ¤t, PhysicalOp D_ASSERT(op.IsSource()); // found another operator that is a source, schedule a child pipeline // 'op' is the source, and the sink is the same - auto child_pipeline = make_refcounted(*this); + auto child_pipeline = make_shared_ptr(*this); child_pipeline->sink = current.sink; child_pipeline->source = &op; diff --git a/src/parallel/meta_pipeline.cpp b/src/parallel/meta_pipeline.cpp index b73515d8e694..d383fe9592f8 100644 --- a/src/parallel/meta_pipeline.cpp +++ b/src/parallel/meta_pipeline.cpp @@ -82,7 +82,7 @@ void MetaPipeline::Ready() { } MetaPipeline &MetaPipeline::CreateChildMetaPipeline(Pipeline ¤t, PhysicalOperator &op) { - children.push_back(make_refcounted(executor, state, &op)); + children.push_back(make_shared_ptr(executor, state, &op)); auto child_meta_pipeline = children.back().get(); // child MetaPipeline must finish completely before this MetaPipeline can start current.AddDependency(child_meta_pipeline->GetBasePipeline()); @@ -92,7 +92,7 @@ MetaPipeline &MetaPipeline::CreateChildMetaPipeline(Pipeline ¤t, PhysicalO } Pipeline &MetaPipeline::CreatePipeline() { - pipelines.emplace_back(make_refcounted(executor)); + pipelines.emplace_back(make_shared_ptr(executor)); state.SetPipelineSink(*pipelines.back(), sink, next_batch_index++); return *pipelines.back(); } diff --git a/src/planner/bind_context.cpp b/src/planner/bind_context.cpp index 611c7b34414b..9e7da9c79bd2 100644 --- a/src/planner/bind_context.cpp +++ b/src/planner/bind_context.cpp @@ -514,13 +514,13 @@ void BindContext::AddGenericBinding(idx_t index, const string &alias, const vect void BindContext::AddCTEBinding(idx_t index, const string &alias, const vector &names, const vector &types) { - auto binding = make_refcounted(BindingType::BASE, alias, types, names, index); + auto binding = make_shared_ptr(BindingType::BASE, alias, types, names, index); if (cte_bindings.find(alias) != cte_bindings.end()) { throw BinderException("Duplicate alias \"%s\" in query!", alias); } cte_bindings[alias] = std::move(binding); - cte_references[alias] = make_refcounted(0); + cte_references[alias] = make_shared_ptr(0); } void BindContext::AddContext(BindContext other) { diff --git a/src/planner/binder.cpp b/src/planner/binder.cpp index 9316c6f85f63..66d0949ddd5f 100644 --- a/src/planner/binder.cpp +++ b/src/planner/binder.cpp @@ -47,7 +47,7 @@ shared_ptr Binder::CreateBinder(ClientContext &context, optional_ptr(true, context, parent ? parent->shared_from_this() : nullptr, inherit_ctes); + return make_shared_ptr(true, context, parent ? parent->shared_from_this() : nullptr, inherit_ctes); } Binder::Binder(bool, ClientContext &context, shared_ptr parent_p, bool inherit_ctes_p) diff --git a/src/planner/bound_parameter_map.cpp b/src/planner/bound_parameter_map.cpp index 61571cd8543f..aac752302bae 100644 --- a/src/planner/bound_parameter_map.cpp +++ b/src/planner/bound_parameter_map.cpp @@ -33,7 +33,7 @@ shared_ptr BoundParameterMap::CreateOrGetData(const string & auto entry = parameters.find(identifier); if (entry == parameters.end()) { // no entry yet: create a new one - auto data = make_refcounted(); + auto data = make_shared_ptr(); data->return_type = GetReturnType(identifier); CreateNewParameter(identifier, data); diff --git a/src/planner/planner.cpp b/src/planner/planner.cpp index 37381c2b6ec9..0ec47a7bfbd0 100644 --- a/src/planner/planner.cpp +++ b/src/planner/planner.cpp @@ -101,7 +101,7 @@ shared_ptr Planner::PrepareSQLStatement(unique_ptr(copied_statement->type); + auto prepared_data = make_shared_ptr(copied_statement->type); prepared_data->unbound_statement = std::move(copied_statement); prepared_data->names = names; prepared_data->types = types; diff --git a/src/storage/buffer/block_manager.cpp b/src/storage/buffer/block_manager.cpp index fdead1bc1e6b..e5e21b0e3cbd 100644 --- a/src/storage/buffer/block_manager.cpp +++ b/src/storage/buffer/block_manager.cpp @@ -23,7 +23,7 @@ shared_ptr BlockManager::RegisterBlock(block_id_t block_id) { } } // create a new block pointer for this block - auto result = make_refcounted(*this, block_id, MemoryTag::BASE_TABLE); + auto result = make_shared_ptr(*this, block_id, MemoryTag::BASE_TABLE); // register the block pointer in the set of blocks as a weak pointer blocks[block_id] = weak_ptr(result); return result; diff --git a/src/storage/checkpoint_manager.cpp b/src/storage/checkpoint_manager.cpp index fe91431d986b..429e0b9d1d53 100644 --- a/src/storage/checkpoint_manager.cpp +++ b/src/storage/checkpoint_manager.cpp @@ -425,7 +425,7 @@ void CheckpointReader::ReadIndex(ClientContext &context, Deserializer &deseriali // now we can look for the index in the catalog and assign the table info auto &index = catalog.CreateIndex(context, info)->Cast(); - index.info = make_refcounted(table.GetStorage().info, info.index_name); + index.info = make_shared_ptr(table.GetStorage().info, info.index_name); // insert the parsed expressions into the index so that we can (de)serialize them during consecutive checkpoints for (auto &parsed_expr : info.parsed_expressions) { diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index a1695d079b2e..96c670da95e3 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -45,12 +45,12 @@ bool DataTableInfo::IsTemporary() const { DataTable::DataTable(AttachedDatabase &db, shared_ptr table_io_manager_p, const string &schema, const string &table, vector column_definitions_p, unique_ptr data) - : info(make_refcounted(db, std::move(table_io_manager_p), schema, table)), + : info(make_shared_ptr(db, std::move(table_io_manager_p), schema, table)), column_definitions(std::move(column_definitions_p)), db(db), is_root(true) { // initialize the table with the existing data from disk, if any auto types = GetTypes(); this->row_groups = - make_refcounted(info, TableIOManager::Get(*this).GetBlockManagerForRowData(), types, 0); + make_shared_ptr(info, TableIOManager::Get(*this).GetBlockManagerForRowData(), types, 0); if (data && data->row_group_count > 0) { this->row_groups->Initialize(*data); } else { diff --git a/src/storage/local_storage.cpp b/src/storage/local_storage.cpp index 55d3a746a97f..1ba86783bbb5 100644 --- a/src/storage/local_storage.cpp +++ b/src/storage/local_storage.cpp @@ -18,7 +18,7 @@ LocalTableStorage::LocalTableStorage(DataTable &table) : table_ref(table), allocator(Allocator::Get(table.db)), deleted_rows(0), optimistic_writer(table), merged_storage(false) { auto types = table.GetTypes(); - row_groups = make_refcounted(table.info, TableIOManager::Get(table).GetBlockManagerForRowData(), + row_groups = make_shared_ptr(table.info, TableIOManager::Get(table).GetBlockManagerForRowData(), types, MAX_ROW_ID, 0); row_groups->InitializeEmpty(); @@ -250,7 +250,7 @@ LocalTableStorage &LocalTableManager::GetOrCreateStorage(DataTable &table) { lock_guard l(table_storage_lock); auto entry = table_storage.find(table); if (entry == table_storage.end()) { - auto new_storage = make_refcounted(table); + auto new_storage = make_shared_ptr(table); auto storage = new_storage.get(); table_storage.insert(make_pair(reference(table), std::move(new_storage))); return *storage; @@ -531,7 +531,7 @@ void LocalStorage::AddColumn(DataTable &old_dt, DataTable &new_dt, ColumnDefinit if (!storage) { return; } - auto new_storage = make_refcounted(context, new_dt, *storage, new_column, default_value); + auto new_storage = make_shared_ptr(context, new_dt, *storage, new_column, default_value); table_manager.InsertEntry(new_dt, std::move(new_storage)); } @@ -541,7 +541,7 @@ void LocalStorage::DropColumn(DataTable &old_dt, DataTable &new_dt, idx_t remove if (!storage) { return; } - auto new_storage = make_refcounted(new_dt, *storage, removed_column); + auto new_storage = make_shared_ptr(new_dt, *storage, removed_column); table_manager.InsertEntry(new_dt, std::move(new_storage)); } @@ -552,7 +552,7 @@ void LocalStorage::ChangeType(DataTable &old_dt, DataTable &new_dt, idx_t change if (!storage) { return; } - auto new_storage = make_refcounted(context, new_dt, *storage, changed_idx, target_type, + auto new_storage = make_shared_ptr(context, new_dt, *storage, changed_idx, target_type, bound_columns, cast_expr); table_manager.InsertEntry(new_dt, std::move(new_storage)); } diff --git a/src/storage/serialization/serialize_types.cpp b/src/storage/serialization/serialize_types.cpp index 1b3b37d87cd9..f937c0e0df73 100644 --- a/src/storage/serialization/serialize_types.cpp +++ b/src/storage/serialization/serialize_types.cpp @@ -35,7 +35,7 @@ shared_ptr ExtraTypeInfo::Deserialize(Deserializer &deserializer) result = EnumTypeInfo::Deserialize(deserializer); break; case ExtraTypeInfoType::GENERIC_TYPE_INFO: - result = make_refcounted(type); + result = make_shared_ptr(type); break; case ExtraTypeInfoType::INTEGER_LITERAL_TYPE_INFO: result = IntegerLiteralTypeInfo::Deserialize(deserializer); diff --git a/src/storage/standard_buffer_manager.cpp b/src/storage/standard_buffer_manager.cpp index f0055e8cb3ef..647a283b03e8 100644 --- a/src/storage/standard_buffer_manager.cpp +++ b/src/storage/standard_buffer_manager.cpp @@ -98,7 +98,7 @@ shared_ptr StandardBufferManager::RegisterSmallMemory(idx_t block_s auto buffer = ConstructManagedBuffer(block_size, nullptr, FileBufferType::TINY_BUFFER); // create a new block pointer for this block - auto result = make_refcounted(*temp_block_manager, ++temporary_id, MemoryTag::BASE_TABLE, + auto result = make_shared_ptr(*temp_block_manager, ++temporary_id, MemoryTag::BASE_TABLE, std::move(buffer), false, block_size, std::move(reservation)); #ifdef DUCKDB_DEBUG_DESTROY_BLOCKS // Initialize the memory with garbage data @@ -118,7 +118,7 @@ shared_ptr StandardBufferManager::RegisterMemory(MemoryTag tag, idx auto buffer = ConstructManagedBuffer(block_size, std::move(reusable_buffer)); // create a new block pointer for this block - return make_refcounted(*temp_block_manager, ++temporary_id, tag, std::move(buffer), can_destroy, + return make_shared_ptr(*temp_block_manager, ++temporary_id, tag, std::move(buffer), can_destroy, alloc_size, std::move(res)); } diff --git a/src/storage/statistics/column_statistics.cpp b/src/storage/statistics/column_statistics.cpp index 67d67417b671..8b3ac243e963 100644 --- a/src/storage/statistics/column_statistics.cpp +++ b/src/storage/statistics/column_statistics.cpp @@ -14,7 +14,7 @@ ColumnStatistics::ColumnStatistics(BaseStatistics stats_p, unique_ptr ColumnStatistics::CreateEmptyStats(const LogicalType &type) { - return make_refcounted(BaseStatistics::CreateEmpty(type)); + return make_shared_ptr(BaseStatistics::CreateEmpty(type)); } void ColumnStatistics::Merge(ColumnStatistics &other) { @@ -53,7 +53,7 @@ void ColumnStatistics::UpdateDistinctStatistics(Vector &v, idx_t count) { } shared_ptr ColumnStatistics::Copy() const { - return make_refcounted(stats.Copy(), distinct_stats ? distinct_stats->Copy() : nullptr); + return make_shared_ptr(stats.Copy(), distinct_stats ? distinct_stats->Copy() : nullptr); } void ColumnStatistics::Serialize(Serializer &serializer) const { @@ -65,7 +65,7 @@ shared_ptr ColumnStatistics::Deserialize(Deserializer &deseria auto stats = deserializer.ReadProperty(100, "statistics"); auto distinct_stats = deserializer.ReadPropertyWithDefault>( 101, "distinct", unique_ptr()); - return make_refcounted(std::move(stats), std::move(distinct_stats)); + return make_shared_ptr(std::move(stats), std::move(distinct_stats)); } } // namespace duckdb diff --git a/src/storage/table/row_group.cpp b/src/storage/table/row_group.cpp index 4d232e3f53c9..24c05e50463d 100644 --- a/src/storage/table/row_group.cpp +++ b/src/storage/table/row_group.cpp @@ -624,7 +624,7 @@ shared_ptr &RowGroup::GetOrCreateVersionInfoPtr() { if (!vinfo) { lock_guard lock(row_group_lock); if (!version_info) { - version_info = make_refcounted(start); + version_info = make_shared_ptr(start); } } return version_info; diff --git a/src/storage/table/row_group_collection.cpp b/src/storage/table/row_group_collection.cpp index c333ea6b1c10..e46a0e7663ff 100644 --- a/src/storage/table/row_group_collection.cpp +++ b/src/storage/table/row_group_collection.cpp @@ -55,7 +55,7 @@ RowGroupCollection::RowGroupCollection(shared_ptr info_p, BlockMa vector types_p, idx_t row_start_p, idx_t total_rows_p) : block_manager(block_manager), total_rows(total_rows_p), info(std::move(info_p)), types(std::move(types_p)), row_start(row_start_p), allocation_size(0) { - row_groups = make_refcounted(*this); + row_groups = make_shared_ptr(*this); } idx_t RowGroupCollection::GetTotalRows() const { @@ -1031,7 +1031,7 @@ shared_ptr RowGroupCollection::AddColumn(ClientContext &cont auto new_types = types; new_types.push_back(new_column.GetType()); auto result = - make_refcounted(info, block_manager, std::move(new_types), row_start, total_rows.load()); + make_shared_ptr(info, block_manager, std::move(new_types), row_start, total_rows.load()); ExpressionExecutor executor(context); DataChunk dummy_chunk; @@ -1059,7 +1059,7 @@ shared_ptr RowGroupCollection::RemoveColumn(idx_t col_idx) { new_types.erase(new_types.begin() + col_idx); auto result = - make_refcounted(info, block_manager, std::move(new_types), row_start, total_rows.load()); + make_shared_ptr(info, block_manager, std::move(new_types), row_start, total_rows.load()); result->stats.InitializeRemoveColumn(stats, col_idx); for (auto ¤t_row_group : row_groups->Segments()) { @@ -1077,7 +1077,7 @@ shared_ptr RowGroupCollection::AlterType(ClientContext &cont new_types[changed_idx] = target_type; auto result = - make_refcounted(info, block_manager, std::move(new_types), row_start, total_rows.load()); + make_shared_ptr(info, block_manager, std::move(new_types), row_start, total_rows.load()); result->stats.InitializeAlterType(stats, changed_idx, target_type); vector scan_types; diff --git a/src/storage/table/row_version_manager.cpp b/src/storage/table/row_version_manager.cpp index 711daa7d1881..e20d9b06a836 100644 --- a/src/storage/table/row_version_manager.cpp +++ b/src/storage/table/row_version_manager.cpp @@ -212,7 +212,7 @@ shared_ptr RowVersionManager::Deserialize(MetaBlockPointer de if (!delete_pointer.IsValid()) { return nullptr; } - auto version_info = make_refcounted(start); + auto version_info = make_shared_ptr(start); MetadataReader source(manager, delete_pointer, &version_info->storage_pointers); auto chunk_count = source.Read(); D_ASSERT(chunk_count > 0); diff --git a/src/storage/wal_replay.cpp b/src/storage/wal_replay.cpp index b3fea7dae501..04e38bc9d4aa 100644 --- a/src/storage/wal_replay.cpp +++ b/src/storage/wal_replay.cpp @@ -580,7 +580,7 @@ void WriteAheadLogDeserializer::ReplayCreateIndex() { // create the index in the catalog auto &table = catalog.GetEntry(context, create_info->schema, info.table).Cast(); auto &index = catalog.CreateIndex(context, info)->Cast(); - index.info = make_refcounted(table.GetStorage().info, index.name); + index.info = make_shared_ptr(table.GetStorage().info, index.name); // insert the parsed expressions into the index so that we can (de)serialize them during consecutive checkpoints for (auto &parsed_expr : info.parsed_expressions) { diff --git a/test/api/test_object_cache.cpp b/test/api/test_object_cache.cpp index 04f4ac7c91de..eebc48c94281 100644 --- a/test/api/test_object_cache.cpp +++ b/test/api/test_object_cache.cpp @@ -42,7 +42,7 @@ TEST_CASE("Test ObjectCache", "[api]") { auto &cache = ObjectCache::GetObjectCache(context); REQUIRE(cache.GetObject("test") == nullptr); - cache.Put("test", make_refcounted(42)); + cache.Put("test", make_shared_ptr(42)); REQUIRE(cache.GetObject("test") != nullptr); diff --git a/tools/odbc/include/duckdb_odbc.hpp b/tools/odbc/include/duckdb_odbc.hpp index 14b92ff1df5c..f7752d3b0682 100644 --- a/tools/odbc/include/duckdb_odbc.hpp +++ b/tools/odbc/include/duckdb_odbc.hpp @@ -42,7 +42,7 @@ struct OdbcHandleEnv : public OdbcHandle { OdbcHandleEnv() : OdbcHandle(OdbcHandleType::ENV) { duckdb::DBConfig ODBC_CONFIG; ODBC_CONFIG.SetOptionByName("duckdb_api", "odbc"); - db = make_refcounted(nullptr, &ODBC_CONFIG); + db = make_shared_ptr(nullptr, &ODBC_CONFIG); }; shared_ptr db; diff --git a/tools/pythonpkg/src/pyconnection.cpp b/tools/pythonpkg/src/pyconnection.cpp index 548d83a0331d..5d98994fc1a0 100644 --- a/tools/pythonpkg/src/pyconnection.cpp +++ b/tools/pythonpkg/src/pyconnection.cpp @@ -640,7 +640,7 @@ void DuckDBPyConnection::RegisterArrowObject(const py::object &arrow_object, con } vector> dependencies; dependencies.push_back( - make_refcounted(make_uniq(std::move(stream_factory), arrow_object))); + make_shared_ptr(make_uniq(std::move(stream_factory), arrow_object))); connection->context->external_dependencies[name] = std::move(dependencies); } @@ -665,7 +665,7 @@ shared_ptr DuckDBPyConnection::RegisterPythonObject(const st // keep a reference vector> dependencies; - dependencies.push_back(make_refcounted(make_uniq(python_object), + dependencies.push_back(make_shared_ptr(make_uniq(python_object), make_uniq(new_df))); connection->context->external_dependencies[name] = std::move(dependencies); } @@ -776,7 +776,7 @@ unique_ptr DuckDBPyConnection::ReadJSON(const string &name, co } auto read_json_relation = - make_refcounted(connection->context, name, std::move(options), auto_detect); + make_shared_ptr(connection->context, name, std::move(options), auto_detect); if (read_json_relation == nullptr) { throw BinderException("read_json can only be used when the JSON extension is (statically) loaded"); } @@ -1319,7 +1319,7 @@ shared_ptr DuckDBPyConnection::Cursor() { if (!connection) { throw ConnectionException("Connection has already been closed"); } - auto res = make_refcounted(); + auto res = make_shared_ptr(); res->database = database; res->connection = make_uniq(*res->database); cursors.push_back(res); @@ -1598,7 +1598,7 @@ static void SetDefaultConfigArguments(ClientContext &context) { } static shared_ptr FetchOrCreateInstance(const string &database, DBConfig &config) { - auto res = make_refcounted(); + auto res = make_shared_ptr(); res->database = instance_cache.GetInstance(database, config); if (!res->database) { //! No cached database, we must create a new instance @@ -1676,7 +1676,7 @@ shared_ptr DuckDBPyConnection::DefaultConnection() { PythonImportCache *DuckDBPyConnection::ImportCache() { if (!import_cache) { - import_cache = make_refcounted(); + import_cache = make_shared_ptr(); } return import_cache.get(); } @@ -1690,7 +1690,7 @@ ModifiedMemoryFileSystem &DuckDBPyConnection::GetObjectFileSystem() { throw InvalidInputException( "This operation could not be completed because required module 'fsspec' is not installed"); } - internal_object_filesystem = make_refcounted(modified_memory_fs()); + internal_object_filesystem = make_shared_ptr(modified_memory_fs()); auto &abstract_fs = reinterpret_cast(*internal_object_filesystem); RegisterFilesystem(abstract_fs); } diff --git a/tools/pythonpkg/src/pyconnection/type_creation.cpp b/tools/pythonpkg/src/pyconnection/type_creation.cpp index 91860e7f936e..1cf2e324c9e8 100644 --- a/tools/pythonpkg/src/pyconnection/type_creation.cpp +++ b/tools/pythonpkg/src/pyconnection/type_creation.cpp @@ -5,17 +5,17 @@ namespace duckdb { shared_ptr DuckDBPyConnection::MapType(const shared_ptr &key_type, const shared_ptr &value_type) { auto map_type = LogicalType::MAP(key_type->Type(), value_type->Type()); - return make_refcounted(map_type); + return make_shared_ptr(map_type); } shared_ptr DuckDBPyConnection::ListType(const shared_ptr &type) { auto array_type = LogicalType::LIST(type->Type()); - return make_refcounted(array_type); + return make_shared_ptr(array_type); } shared_ptr DuckDBPyConnection::ArrayType(const shared_ptr &type, idx_t size) { auto array_type = LogicalType::ARRAY(type->Type(), size); - return make_refcounted(array_type); + return make_shared_ptr(array_type); } static child_list_t GetChildList(const py::object &container) { @@ -59,7 +59,7 @@ shared_ptr DuckDBPyConnection::StructType(const py::object &fields throw InvalidInputException("Can not create an empty struct type!"); } auto struct_type = LogicalType::STRUCT(std::move(types)); - return make_refcounted(struct_type); + return make_shared_ptr(struct_type); } shared_ptr DuckDBPyConnection::UnionType(const py::object &members) { @@ -69,7 +69,7 @@ shared_ptr DuckDBPyConnection::UnionType(const py::object &members throw InvalidInputException("Can not create an empty union type!"); } auto union_type = LogicalType::UNION(std::move(types)); - return make_refcounted(union_type); + return make_shared_ptr(union_type); } shared_ptr DuckDBPyConnection::EnumType(const string &name, const shared_ptr &type, @@ -79,7 +79,7 @@ shared_ptr DuckDBPyConnection::EnumType(const string &name, const shared_ptr DuckDBPyConnection::DecimalType(int width, int scale) { auto decimal_type = LogicalType::DECIMAL(width, scale); - return make_refcounted(decimal_type); + return make_shared_ptr(decimal_type); } shared_ptr DuckDBPyConnection::StringType(const string &collation) { @@ -89,14 +89,14 @@ shared_ptr DuckDBPyConnection::StringType(const string &collation) } else { type = LogicalType::VARCHAR_COLLATION(collation); } - return make_refcounted(type); + return make_shared_ptr(type); } shared_ptr DuckDBPyConnection::Type(const string &type_str) { if (!connection) { throw ConnectionException("Connection already closed!"); } - return make_refcounted(TransformStringToLogicalType(type_str, *connection->context)); + return make_shared_ptr(TransformStringToLogicalType(type_str, *connection->context)); } } // namespace duckdb diff --git a/tools/pythonpkg/src/pyexpression.cpp b/tools/pythonpkg/src/pyexpression.cpp index 09031706acdc..3bc2c43ec47a 100644 --- a/tools/pythonpkg/src/pyexpression.cpp +++ b/tools/pythonpkg/src/pyexpression.cpp @@ -34,19 +34,19 @@ const ParsedExpression &DuckDBPyExpression::GetExpression() const { shared_ptr DuckDBPyExpression::Copy() const { auto expr = GetExpression().Copy(); - return make_refcounted(std::move(expr), order_type, null_order); + return make_shared_ptr(std::move(expr), order_type, null_order); } shared_ptr DuckDBPyExpression::SetAlias(const string &name) const { auto copied_expression = GetExpression().Copy(); copied_expression->alias = name; - return make_refcounted(std::move(copied_expression)); + return make_shared_ptr(std::move(copied_expression)); } shared_ptr DuckDBPyExpression::Cast(const DuckDBPyType &type) const { auto copied_expression = GetExpression().Copy(); auto case_expr = make_uniq(type.Type(), std::move(copied_expression)); - return make_refcounted(std::move(case_expr)); + return make_shared_ptr(std::move(case_expr)); } // Case Expression modifiers @@ -64,7 +64,7 @@ shared_ptr DuckDBPyExpression::InternalWhen(unique_ptrcase_checks.push_back(std::move(check)); - return make_refcounted(std::move(expr)); + return make_shared_ptr(std::move(expr)); } shared_ptr DuckDBPyExpression::When(const DuckDBPyExpression &condition, @@ -82,7 +82,7 @@ shared_ptr DuckDBPyExpression::Else(const DuckDBPyExpression auto expr = unique_ptr_cast(std::move(expr_p)); expr->else_expr = value.GetExpression().Copy(); - return make_refcounted(std::move(expr)); + return make_shared_ptr(std::move(expr)); } // Binary operators @@ -181,7 +181,7 @@ shared_ptr DuckDBPyExpression::In(const py::args &args) { expressions.push_back(std::move(expr)); } auto operator_expr = make_uniq(ExpressionType::COMPARE_IN, std::move(expressions)); - return make_refcounted(std::move(operator_expr)); + return make_shared_ptr(std::move(operator_expr)); } shared_ptr DuckDBPyExpression::NotIn(const py::args &args) { @@ -249,7 +249,7 @@ shared_ptr DuckDBPyExpression::StarExpression(const py::list case_insensitive_set_t exclude; auto star = make_uniq(); PopulateExcludeList(star->exclude_list, exclude_list); - return make_refcounted(std::move(star)); + return make_shared_ptr(std::move(star)); } shared_ptr DuckDBPyExpression::ColumnExpression(const string &column_name) { @@ -267,7 +267,7 @@ shared_ptr DuckDBPyExpression::ColumnExpression(const string } column_names.push_back(qualified_name.name); - return make_refcounted(make_uniq(std::move(column_names))); + return make_shared_ptr(make_uniq(std::move(column_names))); } shared_ptr DuckDBPyExpression::ConstantExpression(const py::object &value) { @@ -292,14 +292,14 @@ DuckDBPyExpression::InternalFunctionExpression(const string &function_name, vector> children, bool is_operator) { auto function_expression = make_uniq(function_name, std::move(children), nullptr, nullptr, false, is_operator); - return make_refcounted(std::move(function_expression)); + return make_shared_ptr(std::move(function_expression)); } shared_ptr DuckDBPyExpression::InternalUnaryOperator(ExpressionType type, const DuckDBPyExpression &arg) { auto expr = arg.GetExpression().Copy(); auto operator_expression = make_uniq(type, std::move(expr)); - return make_refcounted(std::move(operator_expression)); + return make_shared_ptr(std::move(operator_expression)); } shared_ptr DuckDBPyExpression::InternalConjunction(ExpressionType type, @@ -311,11 +311,11 @@ shared_ptr DuckDBPyExpression::InternalConjunction(Expressio children.push_back(other.GetExpression().Copy()); auto operator_expression = make_uniq(type, std::move(children)); - return make_refcounted(std::move(operator_expression)); + return make_shared_ptr(std::move(operator_expression)); } shared_ptr DuckDBPyExpression::InternalConstantExpression(Value val) { - return make_refcounted(make_uniq(std::move(val))); + return make_shared_ptr(make_uniq(std::move(val))); } shared_ptr DuckDBPyExpression::ComparisonExpression(ExpressionType type, @@ -323,7 +323,7 @@ shared_ptr DuckDBPyExpression::ComparisonExpression(Expressi const DuckDBPyExpression &right_p) { auto left = left_p.GetExpression().Copy(); auto right = right_p.GetExpression().Copy(); - return make_refcounted( + return make_shared_ptr( make_uniq(type, std::move(left), std::move(right))); } diff --git a/tools/pythonpkg/src/pyrelation.cpp b/tools/pythonpkg/src/pyrelation.cpp index 2809d2e41089..157eff1b2894 100644 --- a/tools/pythonpkg/src/pyrelation.cpp +++ b/tools/pythonpkg/src/pyrelation.cpp @@ -157,7 +157,7 @@ unique_ptr DuckDBPyRelation::EmptyResult(const shared_ptr> single_row(1, dummy_values); auto values_relation = - make_uniq(make_refcounted(context, single_row, std::move(names))); + make_uniq(make_shared_ptr(context, single_row, std::move(names))); // Add a filter on an impossible condition return values_relation->FilterFromExpression("true = false"); } @@ -1236,7 +1236,7 @@ unique_ptr DuckDBPyRelation::Query(const string &view_name, co if (statement.type == StatementType::SELECT_STATEMENT) { auto select_statement = unique_ptr_cast(std::move(parser.statements[0])); auto query_relation = - make_refcounted(rel->context.GetContext(), std::move(select_statement), "query_relation"); + make_shared_ptr(rel->context.GetContext(), std::move(select_statement), "query_relation"); return make_uniq(std::move(query_relation)); } else if (IsDescribeStatement(statement)) { auto query = PragmaShow(view_name); diff --git a/tools/pythonpkg/src/typing/pytype.cpp b/tools/pythonpkg/src/typing/pytype.cpp index 00edd97af4f9..5fa24fab9939 100644 --- a/tools/pythonpkg/src/typing/pytype.cpp +++ b/tools/pythonpkg/src/typing/pytype.cpp @@ -56,20 +56,20 @@ shared_ptr DuckDBPyType::GetAttribute(const string &name) const { for (idx_t i = 0; i < children.size(); i++) { auto &child = children[i]; if (StringUtil::CIEquals(child.first, name)) { - return make_refcounted(StructType::GetChildType(type, i)); + return make_shared_ptr(StructType::GetChildType(type, i)); } } } if (type.id() == LogicalTypeId::LIST && StringUtil::CIEquals(name, "child")) { - return make_refcounted(ListType::GetChildType(type)); + return make_shared_ptr(ListType::GetChildType(type)); } if (type.id() == LogicalTypeId::MAP) { auto is_key = StringUtil::CIEquals(name, "key"); auto is_value = StringUtil::CIEquals(name, "value"); if (is_key) { - return make_refcounted(MapType::KeyType(type)); + return make_shared_ptr(MapType::KeyType(type)); } else if (is_value) { - return make_refcounted(MapType::ValueType(type)); + return make_shared_ptr(MapType::ValueType(type)); } else { throw py::attribute_error(StringUtil::Format("Tried to get a child from a map by the name of '%s', but " "this type only has 'key' and 'value' children", @@ -313,19 +313,19 @@ void DuckDBPyType::Initialize(py::handle &m) { type_module.def_property_readonly("children", &DuckDBPyType::Children); type_module.def(py::init<>([](const string &type_str, shared_ptr connection = nullptr) { auto ltype = FromString(type_str, std::move(connection)); - return make_refcounted(ltype); + return make_shared_ptr(ltype); })); type_module.def(py::init<>([](const PyGenericAlias &obj) { auto ltype = FromGenericAlias(obj); - return make_refcounted(ltype); + return make_shared_ptr(ltype); })); type_module.def(py::init<>([](const PyUnionType &obj) { auto ltype = FromUnionType(obj); - return make_refcounted(ltype); + return make_shared_ptr(ltype); })); type_module.def(py::init<>([](const py::object &obj) { auto ltype = FromObject(obj); - return make_refcounted(ltype); + return make_shared_ptr(ltype); })); type_module.def("__getattr__", &DuckDBPyType::GetAttribute, "Get the child type by 'name'", py::arg("name")); type_module.def("__getitem__", &DuckDBPyType::GetAttribute, "Get the child type by 'name'", py::arg("name")); @@ -357,7 +357,7 @@ py::list DuckDBPyType::Children() const { py::list children; auto id = type.id(); if (id == LogicalTypeId::LIST) { - children.append(py::make_tuple("child", make_refcounted(ListType::GetChildType(type)))); + children.append(py::make_tuple("child", make_shared_ptr(ListType::GetChildType(type)))); return children; } // FIXME: where is ARRAY?? @@ -367,13 +367,13 @@ py::list DuckDBPyType::Children() const { for (idx_t i = 0; i < struct_children.size(); i++) { auto &child = struct_children[i]; children.append( - py::make_tuple(child.first, make_refcounted(StructType::GetChildType(type, i)))); + py::make_tuple(child.first, make_shared_ptr(StructType::GetChildType(type, i)))); } return children; } if (id == LogicalTypeId::MAP) { - children.append(py::make_tuple("key", make_refcounted(MapType::KeyType(type)))); - children.append(py::make_tuple("value", make_refcounted(MapType::ValueType(type)))); + children.append(py::make_tuple("key", make_shared_ptr(MapType::KeyType(type)))); + children.append(py::make_tuple("value", make_shared_ptr(MapType::ValueType(type)))); return children; } if (id == LogicalTypeId::DECIMAL) { diff --git a/tools/pythonpkg/src/typing/typing.cpp b/tools/pythonpkg/src/typing/typing.cpp index 8064acfc22e1..c0e2675ecf2d 100644 --- a/tools/pythonpkg/src/typing/typing.cpp +++ b/tools/pythonpkg/src/typing/typing.cpp @@ -4,38 +4,38 @@ namespace duckdb { static void DefineBaseTypes(py::handle &m) { - m.attr("SQLNULL") = make_refcounted(LogicalType::SQLNULL); - m.attr("BOOLEAN") = make_refcounted(LogicalType::BOOLEAN); - m.attr("TINYINT") = make_refcounted(LogicalType::TINYINT); - m.attr("UTINYINT") = make_refcounted(LogicalType::UTINYINT); - m.attr("SMALLINT") = make_refcounted(LogicalType::SMALLINT); - m.attr("USMALLINT") = make_refcounted(LogicalType::USMALLINT); - m.attr("INTEGER") = make_refcounted(LogicalType::INTEGER); - m.attr("UINTEGER") = make_refcounted(LogicalType::UINTEGER); - m.attr("BIGINT") = make_refcounted(LogicalType::BIGINT); - m.attr("UBIGINT") = make_refcounted(LogicalType::UBIGINT); - m.attr("HUGEINT") = make_refcounted(LogicalType::HUGEINT); - m.attr("UHUGEINT") = make_refcounted(LogicalType::UHUGEINT); - m.attr("UUID") = make_refcounted(LogicalType::UUID); - m.attr("FLOAT") = make_refcounted(LogicalType::FLOAT); - m.attr("DOUBLE") = make_refcounted(LogicalType::DOUBLE); - m.attr("DATE") = make_refcounted(LogicalType::DATE); - - m.attr("TIMESTAMP") = make_refcounted(LogicalType::TIMESTAMP); - m.attr("TIMESTAMP_MS") = make_refcounted(LogicalType::TIMESTAMP_MS); - m.attr("TIMESTAMP_NS") = make_refcounted(LogicalType::TIMESTAMP_NS); - m.attr("TIMESTAMP_S") = make_refcounted(LogicalType::TIMESTAMP_S); - - m.attr("TIME") = make_refcounted(LogicalType::TIME); - - m.attr("TIME_TZ") = make_refcounted(LogicalType::TIME_TZ); - m.attr("TIMESTAMP_TZ") = make_refcounted(LogicalType::TIMESTAMP_TZ); - - m.attr("VARCHAR") = make_refcounted(LogicalType::VARCHAR); - - m.attr("BLOB") = make_refcounted(LogicalType::BLOB); - m.attr("BIT") = make_refcounted(LogicalType::BIT); - m.attr("INTERVAL") = make_refcounted(LogicalType::INTERVAL); + m.attr("SQLNULL") = make_shared_ptr(LogicalType::SQLNULL); + m.attr("BOOLEAN") = make_shared_ptr(LogicalType::BOOLEAN); + m.attr("TINYINT") = make_shared_ptr(LogicalType::TINYINT); + m.attr("UTINYINT") = make_shared_ptr(LogicalType::UTINYINT); + m.attr("SMALLINT") = make_shared_ptr(LogicalType::SMALLINT); + m.attr("USMALLINT") = make_shared_ptr(LogicalType::USMALLINT); + m.attr("INTEGER") = make_shared_ptr(LogicalType::INTEGER); + m.attr("UINTEGER") = make_shared_ptr(LogicalType::UINTEGER); + m.attr("BIGINT") = make_shared_ptr(LogicalType::BIGINT); + m.attr("UBIGINT") = make_shared_ptr(LogicalType::UBIGINT); + m.attr("HUGEINT") = make_shared_ptr(LogicalType::HUGEINT); + m.attr("UHUGEINT") = make_shared_ptr(LogicalType::UHUGEINT); + m.attr("UUID") = make_shared_ptr(LogicalType::UUID); + m.attr("FLOAT") = make_shared_ptr(LogicalType::FLOAT); + m.attr("DOUBLE") = make_shared_ptr(LogicalType::DOUBLE); + m.attr("DATE") = make_shared_ptr(LogicalType::DATE); + + m.attr("TIMESTAMP") = make_shared_ptr(LogicalType::TIMESTAMP); + m.attr("TIMESTAMP_MS") = make_shared_ptr(LogicalType::TIMESTAMP_MS); + m.attr("TIMESTAMP_NS") = make_shared_ptr(LogicalType::TIMESTAMP_NS); + m.attr("TIMESTAMP_S") = make_shared_ptr(LogicalType::TIMESTAMP_S); + + m.attr("TIME") = make_shared_ptr(LogicalType::TIME); + + m.attr("TIME_TZ") = make_shared_ptr(LogicalType::TIME_TZ); + m.attr("TIMESTAMP_TZ") = make_shared_ptr(LogicalType::TIMESTAMP_TZ); + + m.attr("VARCHAR") = make_shared_ptr(LogicalType::VARCHAR); + + m.attr("BLOB") = make_shared_ptr(LogicalType::BLOB); + m.attr("BIT") = make_shared_ptr(LogicalType::BIT); + m.attr("INTERVAL") = make_shared_ptr(LogicalType::INTERVAL); } void DuckDBPyTyping::Initialize(py::module_ &parent) { From 39a05189f70d846140bdc8d112e2ad79fa33414e Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 12 Apr 2024 10:33:28 +0200 Subject: [PATCH 372/603] Upload pyodide build --- .github/workflows/Pyodide.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/Pyodide.yml b/.github/workflows/Pyodide.yml index 290024c3c190..ef96f95511d8 100644 --- a/.github/workflows/Pyodide.yml +++ b/.github/workflows/Pyodide.yml @@ -97,3 +97,11 @@ jobs: if-no-files-found: error path: | ./tools/pythonpkg/dist/*.whl + + - name: Deploy + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} + shell: bash + run: | + ./scripts/upload-assets-to-staging.sh pyodide tools/pythonpkg/dist/*.whl From 53a9ceed558acde2402e335c62548dfc12f418ed Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 12 Apr 2024 10:38:00 +0200 Subject: [PATCH 373/603] pyodide builds to not trigger autoinstall and autoload --- tools/pythonpkg/setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/pythonpkg/setup.py b/tools/pythonpkg/setup.py index 76ba6a2d13d0..253de360e7c4 100644 --- a/tools/pythonpkg/setup.py +++ b/tools/pythonpkg/setup.py @@ -206,7 +206,9 @@ def open_utf8(fpath, flags): for ext in extensions: define_macros.append(('DUCKDB_EXTENSION_{}_LINKED'.format(ext.upper()), None)) -define_macros.extend([('DUCKDB_EXTENSION_AUTOLOAD_DEFAULT', '1'), ('DUCKDB_EXTENSION_AUTOINSTALL_DEFAULT', '1')]) +if not is_pyodide: + # currently pyodide environment is not compatible with dynamic extesion loading + define_macros.extend([('DUCKDB_EXTENSION_AUTOLOAD_DEFAULT', '1'), ('DUCKDB_EXTENSION_AUTOINSTALL_DEFAULT', '1')]) linker_args = toolchain_args[:] if platform.system() == 'Windows': From 74bbb9730b5b2de00815e47fac31176a07fd0d23 Mon Sep 17 00:00:00 2001 From: Max Gabrielsson Date: Fri, 12 Apr 2024 10:48:35 +0200 Subject: [PATCH 374/603] bump vss --- .github/config/extensions.csv | 2 +- .github/config/out_of_tree_extensions.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/config/extensions.csv b/.github/config/extensions.csv index d2e68ab77fa7..9ce22e2b486f 100644 --- a/.github/config/extensions.csv +++ b/.github/config/extensions.csv @@ -15,4 +15,4 @@ aws,https://github.com/duckdb/duckdb_aws,f7b8729f1cce5ada5d4add70e1486de50763fb9 azure,https://github.com/duckdb/duckdb_azure,86f39d76157de970d16d6d6537bc90c0ee1c7d35, spatial,https://github.com/duckdb/duckdb_spatial,05c4ba01c500140287bf6946fb6910122e5c2acf, iceberg,https://github.com/duckdb/duckdb_iceberg,7aa3d8e4cb7b513d35fdacfa28dc328771bc4047, -vss,https://github.com/duckdb/duckdb_vss,f03fe75e16dd67e404d6325d4f901d6958504730, \ No newline at end of file +vss,https://github.com/duckdb/duckdb_vss,a85e973650a083e4b279126a0aec07924d84e765, \ No newline at end of file diff --git a/.github/config/out_of_tree_extensions.cmake b/.github/config/out_of_tree_extensions.cmake index c216a6a640c1..153ca357ff59 100644 --- a/.github/config/out_of_tree_extensions.cmake +++ b/.github/config/out_of_tree_extensions.cmake @@ -108,7 +108,7 @@ endif() duckdb_extension_load(vss LOAD_TESTS GIT_URL https://github.com/duckdb/duckdb_vss - GIT_TAG f03fe75e16dd67e404d6325d4f901d6958504730 + GIT_TAG a85e973650a083e4b279126a0aec07924d84e765 TEST_DIR test/sql APPLY_PATCHES ) \ No newline at end of file From 8e5178e1266d738d04dc9cf2bba66dd57ffd6e8d Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 12 Apr 2024 11:11:24 +0200 Subject: [PATCH 375/603] add patch for azure --- .../patches/extensions/azure/shared_ptr.patch | 400 ++++++++++++++++++ 1 file changed, 400 insertions(+) create mode 100644 .github/patches/extensions/azure/shared_ptr.patch diff --git a/.github/patches/extensions/azure/shared_ptr.patch b/.github/patches/extensions/azure/shared_ptr.patch new file mode 100644 index 000000000000..bee19cfb4b8a --- /dev/null +++ b/.github/patches/extensions/azure/shared_ptr.patch @@ -0,0 +1,400 @@ +diff --git a/src/azure_blob_filesystem.cpp b/src/azure_blob_filesystem.cpp +index bc34eb9..42f9323 100644 +--- a/src/azure_blob_filesystem.cpp ++++ b/src/azure_blob_filesystem.cpp +@@ -3,6 +3,8 @@ + #include "azure_storage_account_client.hpp" + #include "duckdb.hpp" + #include "duckdb/common/exception.hpp" ++#include "duckdb/common/helper.hpp" ++#include "duckdb/common/shared_ptr.hpp" + #include "duckdb/common/http_state.hpp" + #include "duckdb/common/file_opener.hpp" + #include "duckdb/common/string_util.hpp" +@@ -201,13 +203,13 @@ void AzureBlobStorageFileSystem::ReadRange(AzureFileHandle &handle, idx_t file_o + } + } + +-std::shared_ptr AzureBlobStorageFileSystem::CreateStorageContext(optional_ptr opener, +- const string &path, +- const AzureParsedUrl &parsed_url) { ++shared_ptr AzureBlobStorageFileSystem::CreateStorageContext(optional_ptr opener, ++ const string &path, ++ const AzureParsedUrl &parsed_url) { + auto azure_read_options = ParseAzureReadOptions(opener); + +- return std::make_shared(ConnectToBlobStorageAccount(opener, path, parsed_url), +- azure_read_options); ++ return make_shared_ptr(ConnectToBlobStorageAccount(opener, path, parsed_url), ++ azure_read_options); + } + + } // namespace duckdb +diff --git a/src/azure_dfs_filesystem.cpp b/src/azure_dfs_filesystem.cpp +index 5ccbed0..739078c 100644 +--- a/src/azure_dfs_filesystem.cpp ++++ b/src/azure_dfs_filesystem.cpp +@@ -1,6 +1,8 @@ + #include "azure_dfs_filesystem.hpp" + #include "azure_storage_account_client.hpp" + #include "duckdb/common/exception.hpp" ++#include "duckdb/common/helper.hpp" ++#include "duckdb/common/shared_ptr.hpp" + #include "duckdb/function/scalar/string_functions.hpp" + #include + #include +@@ -185,13 +187,13 @@ void AzureDfsStorageFileSystem::ReadRange(AzureFileHandle &handle, idx_t file_of + } + } + +-std::shared_ptr AzureDfsStorageFileSystem::CreateStorageContext(optional_ptr opener, +- const string &path, +- const AzureParsedUrl &parsed_url) { ++shared_ptr AzureDfsStorageFileSystem::CreateStorageContext(optional_ptr opener, ++ const string &path, ++ const AzureParsedUrl &parsed_url) { + auto azure_read_options = ParseAzureReadOptions(opener); + +- return std::make_shared(ConnectToDfsStorageAccount(opener, path, parsed_url), +- azure_read_options); ++ return make_shared_ptr(ConnectToDfsStorageAccount(opener, path, parsed_url), ++ azure_read_options); + } + + } // namespace duckdb +diff --git a/src/azure_filesystem.cpp b/src/azure_filesystem.cpp +index bbf5275..6175421 100644 +--- a/src/azure_filesystem.cpp ++++ b/src/azure_filesystem.cpp +@@ -1,5 +1,6 @@ + #include "azure_filesystem.hpp" + #include "duckdb/common/exception.hpp" ++#include "duckdb/common/shared_ptr.hpp" + #include "duckdb/common/types/value.hpp" + #include "duckdb/main/client_context.hpp" + #include +@@ -53,8 +54,8 @@ void AzureStorageFileSystem::LoadFileInfo(AzureFileHandle &handle) { + } + } + +-unique_ptr AzureStorageFileSystem::OpenFile(const string &path,FileOpenFlags flags, +- optional_ptr opener) { ++unique_ptr AzureStorageFileSystem::OpenFile(const string &path, FileOpenFlags flags, ++ optional_ptr opener) { + D_ASSERT(flags.Compression() == FileCompressionType::UNCOMPRESSED); + + if (flags.OpenForWriting()) { +@@ -153,16 +154,16 @@ int64_t AzureStorageFileSystem::Read(FileHandle &handle, void *buffer, int64_t n + return nr_bytes; + } + +-std::shared_ptr AzureStorageFileSystem::GetOrCreateStorageContext(optional_ptr opener, +- const string &path, +- const AzureParsedUrl &parsed_url) { ++shared_ptr AzureStorageFileSystem::GetOrCreateStorageContext(optional_ptr opener, ++ const string &path, ++ const AzureParsedUrl &parsed_url) { + Value value; + bool azure_context_caching = true; + if (FileOpener::TryGetCurrentSetting(opener, "azure_context_caching", value)) { + azure_context_caching = value.GetValue(); + } + +- std::shared_ptr result; ++ shared_ptr result; + if (azure_context_caching) { + auto client_context = FileOpener::TryGetClientContext(opener); + +@@ -182,7 +183,7 @@ std::shared_ptr AzureStorageFileSystem::GetOrCreateStorageCon + result = CreateStorageContext(opener, path, parsed_url); + registered_state[context_key] = result; + } else { +- result = std::shared_ptr(storage_account_it->second, azure_context_state); ++ result = shared_ptr(storage_account_it->second, azure_context_state); + } + } + } else { +diff --git a/src/azure_storage_account_client.cpp b/src/azure_storage_account_client.cpp +index 5a22e60..11ad859 100644 +--- a/src/azure_storage_account_client.cpp ++++ b/src/azure_storage_account_client.cpp +@@ -3,6 +3,8 @@ + #include "duckdb/catalog/catalog_transaction.hpp" + #include "duckdb/common/enums/statement_type.hpp" + #include "duckdb/common/exception.hpp" ++#include "duckdb/common/shared_ptr.hpp" ++#include "duckdb/common/helper.hpp" + #include "duckdb/common/file_opener.hpp" + #include "duckdb/common/string_util.hpp" + #include "duckdb/main/client_context.hpp" +@@ -75,12 +77,12 @@ static std::string AccountUrl(const AzureParsedUrl &azure_parsed_url) { + + template + static T ToClientOptions(const Azure::Core::Http::Policies::TransportOptions &transport_options, +- std::shared_ptr http_state) { ++ shared_ptr http_state) { + static_assert(std::is_base_of::value, + "type parameter must be an Azure ClientOptions"); + T options; + options.Transport = transport_options; +- if (nullptr != http_state) { ++ if (http_state != nullptr) { + // Because we mainly want to have stats on what has been needed and not on + // what has been used on the network, we register the policy on `PerOperationPolicies` + // part and not the `PerRetryPolicies`. Network issues will result in retry that can +@@ -92,13 +94,13 @@ static T ToClientOptions(const Azure::Core::Http::Policies::TransportOptions &tr + + static Azure::Storage::Blobs::BlobClientOptions + ToBlobClientOptions(const Azure::Core::Http::Policies::TransportOptions &transport_options, +- std::shared_ptr http_state) { ++ shared_ptr http_state) { + return ToClientOptions(transport_options, std::move(http_state)); + } + + static Azure::Storage::Files::DataLake::DataLakeClientOptions + ToDfsClientOptions(const Azure::Core::Http::Policies::TransportOptions &transport_options, +- std::shared_ptr http_state) { ++ shared_ptr http_state) { + return ToClientOptions(transport_options, + std::move(http_state)); + } +@@ -110,14 +112,14 @@ ToTokenCredentialOptions(const Azure::Core::Http::Policies::TransportOptions &tr + return options; + } + +-static std::shared_ptr GetHttpState(optional_ptr opener) { ++static shared_ptr GetHttpState(optional_ptr opener) { + Value value; + bool enable_http_stats = false; + if (FileOpener::TryGetCurrentSetting(opener, "azure_http_stats", value)) { + enable_http_stats = value.GetValue(); + } + +- std::shared_ptr http_state; ++ shared_ptr http_state; + if (enable_http_stats) { + http_state = HTTPState::TryGetState(opener); + } +@@ -168,7 +170,7 @@ CreateClientCredential(const std::string &tenant_id, const std::string &client_i + auto credential_options = ToTokenCredentialOptions(transport_options); + if (!client_secret.empty()) { + return std::make_shared(tenant_id, client_id, client_secret, +- credential_options); ++ credential_options); + } else if (!client_certificate_path.empty()) { + return std::make_shared( + tenant_id, client_id, client_certificate_path, credential_options); +@@ -408,8 +410,9 @@ GetDfsStorageAccountClientFromServicePrincipalProvider(optional_ptr + return Azure::Storage::Files::DataLake::DataLakeServiceClient(account_url, token_credential, dfs_options); + } + +-static Azure::Storage::Blobs::BlobServiceClient +-GetBlobStorageAccountClient(optional_ptr opener, const KeyValueSecret &secret, const AzureParsedUrl &azure_parsed_url) { ++static Azure::Storage::Blobs::BlobServiceClient GetBlobStorageAccountClient(optional_ptr opener, ++ const KeyValueSecret &secret, ++ const AzureParsedUrl &azure_parsed_url) { + auto &provider = secret.GetProvider(); + // default provider + if (provider == "config") { +@@ -424,7 +427,8 @@ GetBlobStorageAccountClient(optional_ptr opener, const KeyValueSecre + } + + static Azure::Storage::Files::DataLake::DataLakeServiceClient +-GetDfsStorageAccountClient(optional_ptr opener, const KeyValueSecret &secret, const AzureParsedUrl &azure_parsed_url) { ++GetDfsStorageAccountClient(optional_ptr opener, const KeyValueSecret &secret, ++ const AzureParsedUrl &azure_parsed_url) { + auto &provider = secret.GetProvider(); + // default provider + if (provider == "config") { +@@ -505,7 +509,8 @@ const SecretMatch LookupSecret(optional_ptr opener, const std::strin + return {}; + } + +-Azure::Storage::Blobs::BlobServiceClient ConnectToBlobStorageAccount(optional_ptr opener, const std::string &path, ++Azure::Storage::Blobs::BlobServiceClient ConnectToBlobStorageAccount(optional_ptr opener, ++ const std::string &path, + const AzureParsedUrl &azure_parsed_url) { + + auto secret_match = LookupSecret(opener, path); +@@ -519,7 +524,8 @@ Azure::Storage::Blobs::BlobServiceClient ConnectToBlobStorageAccount(optional_pt + } + + Azure::Storage::Files::DataLake::DataLakeServiceClient +-ConnectToDfsStorageAccount(optional_ptr opener, const std::string &path, const AzureParsedUrl &azure_parsed_url) { ++ConnectToDfsStorageAccount(optional_ptr opener, const std::string &path, ++ const AzureParsedUrl &azure_parsed_url) { + auto secret_match = LookupSecret(opener, path); + if (secret_match.HasMatch()) { + const auto &base_secret = secret_match.GetSecret(); +diff --git a/src/http_state_policy.cpp b/src/http_state_policy.cpp +index be2d61d..baa3f97 100644 +--- a/src/http_state_policy.cpp ++++ b/src/http_state_policy.cpp +@@ -1,5 +1,6 @@ + #include "http_state_policy.hpp" + #include ++#include "duckdb/common/shared_ptr.hpp" + #include + #include + #include +@@ -8,7 +9,7 @@ const static std::string CONTENT_LENGTH = "content-length"; + + namespace duckdb { + +-HttpStatePolicy::HttpStatePolicy(std::shared_ptr http_state) : http_state(std::move(http_state)) { ++HttpStatePolicy::HttpStatePolicy(shared_ptr http_state) : http_state(std::move(http_state)) { + } + + std::unique_ptr +@@ -34,12 +35,12 @@ HttpStatePolicy::Send(Azure::Core::Http::Request &request, Azure::Core::Http::Po + } + + const auto *body_stream = request.GetBodyStream(); +- if (nullptr != body_stream) { ++ if (body_stream != nullptr) { + http_state->total_bytes_sent += body_stream->Length(); + } + + auto result = next_policy.Send(request, context); +- if (nullptr != result) { ++ if (result != nullptr) { + const auto &response_body = result->GetBody(); + if (response_body.size() != 0) { + http_state->total_bytes_received += response_body.size(); +diff --git a/src/include/azure_blob_filesystem.hpp b/src/include/azure_blob_filesystem.hpp +index 4d10ebe..638864a 100644 +--- a/src/include/azure_blob_filesystem.hpp ++++ b/src/include/azure_blob_filesystem.hpp +@@ -1,6 +1,8 @@ + #pragma once + + #include "duckdb.hpp" ++#include "duckdb/common/shared_ptr.hpp" ++#include "duckdb/common/unique_ptr.hpp" + #include "azure_parsed_url.hpp" + #include "azure_filesystem.hpp" + #include +@@ -57,10 +59,10 @@ protected: + const string &GetContextPrefix() const override { + return PATH_PREFIX; + } +- std::shared_ptr CreateStorageContext(optional_ptr opener, const string &path, +- const AzureParsedUrl &parsed_url) override; +- duckdb::unique_ptr CreateHandle(const string &path, FileOpenFlags flags, +- optional_ptr opener) override; ++ shared_ptr CreateStorageContext(optional_ptr opener, const string &path, ++ const AzureParsedUrl &parsed_url) override; ++ unique_ptr CreateHandle(const string &path, FileOpenFlags flags, ++ optional_ptr opener) override; + + void ReadRange(AzureFileHandle &handle, idx_t file_offset, char *buffer_out, idx_t buffer_out_len) override; + }; +diff --git a/src/include/azure_dfs_filesystem.hpp b/src/include/azure_dfs_filesystem.hpp +index cdcdb23..8f35dc7 100644 +--- a/src/include/azure_dfs_filesystem.hpp ++++ b/src/include/azure_dfs_filesystem.hpp +@@ -2,6 +2,8 @@ + + #include "azure_filesystem.hpp" + #include "duckdb/common/file_opener.hpp" ++#include "duckdb/common/shared_ptr.hpp" ++#include "duckdb/common/unique_ptr.hpp" + #include + #include + #include +@@ -55,10 +57,10 @@ protected: + const string &GetContextPrefix() const override { + return PATH_PREFIX; + } +- std::shared_ptr CreateStorageContext(optional_ptr opener, const string &path, +- const AzureParsedUrl &parsed_url) override; +- duckdb::unique_ptr CreateHandle(const string &path, FileOpenFlags flags, +- optional_ptr opener) override; ++ shared_ptr CreateStorageContext(optional_ptr opener, const string &path, ++ const AzureParsedUrl &parsed_url) override; ++ unique_ptr CreateHandle(const string &path, FileOpenFlags flags, ++ optional_ptr opener) override; + + void ReadRange(AzureFileHandle &handle, idx_t file_offset, char *buffer_out, idx_t buffer_out_len) override; + }; +diff --git a/src/include/azure_filesystem.hpp b/src/include/azure_filesystem.hpp +index 338c744..02ed44b 100644 +--- a/src/include/azure_filesystem.hpp ++++ b/src/include/azure_filesystem.hpp +@@ -3,6 +3,7 @@ + #include "azure_parsed_url.hpp" + #include "duckdb/common/assert.hpp" + #include "duckdb/common/file_opener.hpp" ++#include "duckdb/common/shared_ptr.hpp" + #include "duckdb/common/file_system.hpp" + #include "duckdb/main/client_context_state.hpp" + #include +@@ -99,14 +100,14 @@ public: + + protected: + virtual duckdb::unique_ptr CreateHandle(const string &path, FileOpenFlags flags, +- optional_ptr opener) = 0; ++ optional_ptr opener) = 0; + virtual void ReadRange(AzureFileHandle &handle, idx_t file_offset, char *buffer_out, idx_t buffer_out_len) = 0; + + virtual const string &GetContextPrefix() const = 0; +- std::shared_ptr GetOrCreateStorageContext(optional_ptr opener, const string &path, +- const AzureParsedUrl &parsed_url); +- virtual std::shared_ptr CreateStorageContext(optional_ptr opener, const string &path, +- const AzureParsedUrl &parsed_url) = 0; ++ shared_ptr GetOrCreateStorageContext(optional_ptr opener, const string &path, ++ const AzureParsedUrl &parsed_url); ++ virtual shared_ptr CreateStorageContext(optional_ptr opener, const string &path, ++ const AzureParsedUrl &parsed_url) = 0; + + virtual void LoadRemoteFileInfo(AzureFileHandle &handle) = 0; + static AzureReadOptions ParseAzureReadOptions(optional_ptr opener); +diff --git a/src/include/azure_storage_account_client.hpp b/src/include/azure_storage_account_client.hpp +index 600fa10..aa9a6e5 100644 +--- a/src/include/azure_storage_account_client.hpp ++++ b/src/include/azure_storage_account_client.hpp +@@ -8,10 +8,12 @@ + + namespace duckdb { + +-Azure::Storage::Blobs::BlobServiceClient ConnectToBlobStorageAccount(optional_ptr opener, const std::string &path, ++Azure::Storage::Blobs::BlobServiceClient ConnectToBlobStorageAccount(optional_ptr opener, ++ const std::string &path, + const AzureParsedUrl &azure_parsed_url); + + Azure::Storage::Files::DataLake::DataLakeServiceClient +-ConnectToDfsStorageAccount(optional_ptr opener, const std::string &path, const AzureParsedUrl &azure_parsed_url); ++ConnectToDfsStorageAccount(optional_ptr opener, const std::string &path, ++ const AzureParsedUrl &azure_parsed_url); + + } // namespace duckdb +diff --git a/src/include/http_state_policy.hpp b/src/include/http_state_policy.hpp +index 310b9c3..9db73b6 100644 +--- a/src/include/http_state_policy.hpp ++++ b/src/include/http_state_policy.hpp +@@ -1,6 +1,7 @@ + #pragma once + + #include "duckdb/common/http_state.hpp" ++#include "duckdb/common/shared_ptr.hpp" + #include + #include + #include +@@ -11,7 +12,7 @@ namespace duckdb { + + class HttpStatePolicy : public Azure::Core::Http::Policies::HttpPolicy { + public: +- HttpStatePolicy(std::shared_ptr http_state); ++ HttpStatePolicy(shared_ptr http_state); + + std::unique_ptr Send(Azure::Core::Http::Request &request, + Azure::Core::Http::Policies::NextHttpPolicy next_policy, +@@ -20,7 +21,7 @@ public: + std::unique_ptr Clone() const override; + + private: +- std::shared_ptr http_state; ++ shared_ptr http_state; + }; + + } // namespace duckdb From 5e204ab3761d67e7bb15c3624a426176520f1358 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 12 Apr 2024 11:19:22 +0200 Subject: [PATCH 376/603] add patch for aws --- .github/config/out_of_tree_extensions.cmake | 1 + .../patches/extensions/aws/shared_ptr.patch | 37 +++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 .github/patches/extensions/aws/shared_ptr.patch diff --git a/.github/config/out_of_tree_extensions.cmake b/.github/config/out_of_tree_extensions.cmake index 0f919af9b1e8..d17b2e7ce508 100644 --- a/.github/config/out_of_tree_extensions.cmake +++ b/.github/config/out_of_tree_extensions.cmake @@ -28,6 +28,7 @@ if (NOT MINGW) LOAD_TESTS GIT_URL https://github.com/duckdb/duckdb_aws GIT_TAG f7b8729f1cce5ada5d4add70e1486de50763fb97 + APPLY_PATCHES ) endif() diff --git a/.github/patches/extensions/aws/shared_ptr.patch b/.github/patches/extensions/aws/shared_ptr.patch new file mode 100644 index 000000000000..7cd90185e268 --- /dev/null +++ b/.github/patches/extensions/aws/shared_ptr.patch @@ -0,0 +1,37 @@ +diff --git a/src/aws_secret.cpp b/src/aws_secret.cpp +index 75062b9..179cc29 100644 +--- a/src/aws_secret.cpp ++++ b/src/aws_secret.cpp +@@ -40,24 +40,24 @@ public: + + for (const auto &item : chain_list) { + if (item == "sts") { +- AddProvider(make_shared()); ++ AddProvider(std::make_shared()); + } else if (item == "sso") { + if (profile.empty()) { +- AddProvider(make_shared()); ++ AddProvider(std::make_shared()); + } else { +- AddProvider(make_shared(profile)); ++ AddProvider(std::make_shared(profile)); + } + } else if (item == "env") { +- AddProvider(make_shared()); ++ AddProvider(std::make_shared()); + } else if (item == "instance") { +- AddProvider(make_shared()); ++ AddProvider(std::make_shared()); + } else if (item == "process") { +- AddProvider(make_shared()); ++ AddProvider(std::make_shared()); + } else if (item == "config") { + if (profile.empty()) { +- AddProvider(make_shared()); ++ AddProvider(std::make_shared()); + } else { +- AddProvider(make_shared(profile.c_str())); ++ AddProvider(std::make_shared(profile.c_str())); + } + } else { + throw InvalidInputException("Unknown provider found while parsing AWS credential chain string: '%s'", From 810cb68db34ac35974d4b89506719a382bde3674 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 12 Apr 2024 11:30:27 +0200 Subject: [PATCH 377/603] avoid load error? --- .../operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp | 3 ++- .../execution/operator/csv_scanner/csv_buffer_manager.hpp | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp index 691b0f2f5120..68a52f78acf3 100644 --- a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp +++ b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp @@ -8,6 +8,7 @@ CSVBufferManager::CSVBufferManager(ClientContext &context_p, const CSVReaderOpti : context(context_p), file_idx(file_idx_p), file_path(file_path_p), buffer_size(CSVBuffer::CSV_BUFFER_SIZE) { D_ASSERT(!file_path.empty()); file_handle = ReadCSV::OpenCSV(file_path, options.compression, context); + can_seek = file_handle->CanSeek(); skip_rows = options.dialect_options.skip_rows.GetValue(); auto file_size = file_handle->FileSize(); if (file_size > 0 && file_size < buffer_size) { @@ -71,7 +72,7 @@ shared_ptr CSVBufferManager::GetBuffer(const idx_t pos) { done = true; } } - if (pos != 0 && (sniffing || file_handle->CanSeek())) { + if (pos != 0 && (sniffing || can_seek)) { // We don't need to unpin the buffers here if we are not sniffing since we // control it per-thread on the scan if (cached_buffers[pos - 1]) { diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp index efc40e724f5c..dc05ef378ab3 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp @@ -71,6 +71,7 @@ class CSVBufferManager { //! If the file_handle used seek bool has_seeked = false; unordered_set reset_when_possible; + bool can_seek; }; } // namespace duckdb From e46175d87f558550c0318282d128303160e3be06 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 12 Apr 2024 12:21:07 +0200 Subject: [PATCH 378/603] Update description Co-authored-by: Phillip Cloud <417981+cpcloud@users.noreply.github.com> --- tools/pythonpkg/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pythonpkg/setup.py b/tools/pythonpkg/setup.py index 253de360e7c4..c61e2e2b20d3 100644 --- a/tools/pythonpkg/setup.py +++ b/tools/pythonpkg/setup.py @@ -207,7 +207,7 @@ def open_utf8(fpath, flags): define_macros.append(('DUCKDB_EXTENSION_{}_LINKED'.format(ext.upper()), None)) if not is_pyodide: - # currently pyodide environment is not compatible with dynamic extesion loading + # currently pyodide environment is not compatible with dynamic extension loading define_macros.extend([('DUCKDB_EXTENSION_AUTOLOAD_DEFAULT', '1'), ('DUCKDB_EXTENSION_AUTOINSTALL_DEFAULT', '1')]) linker_args = toolchain_args[:] From 51f89479c773642c87b859553307571347de9b15 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 12 Apr 2024 12:23:40 +0200 Subject: [PATCH 379/603] add patch for postgres_scanner --- .github/config/out_of_tree_extensions.cmake | 1 + .../postgres_scanner/shared_ptr.patch | 214 ++++++++++++++++++ 2 files changed, 215 insertions(+) create mode 100644 .github/patches/extensions/postgres_scanner/shared_ptr.patch diff --git a/.github/config/out_of_tree_extensions.cmake b/.github/config/out_of_tree_extensions.cmake index d17b2e7ce508..a96f35e1d7be 100644 --- a/.github/config/out_of_tree_extensions.cmake +++ b/.github/config/out_of_tree_extensions.cmake @@ -67,6 +67,7 @@ if (NOT MINGW) DONT_LINK GIT_URL https://github.com/duckdb/postgres_scanner GIT_TAG 96206f41d5ca7015920a66b54e936c986fe0b0f8 + APPLY_PATCHES ) endif() diff --git a/.github/patches/extensions/postgres_scanner/shared_ptr.patch b/.github/patches/extensions/postgres_scanner/shared_ptr.patch new file mode 100644 index 000000000000..98d351393f23 --- /dev/null +++ b/.github/patches/extensions/postgres_scanner/shared_ptr.patch @@ -0,0 +1,214 @@ +diff --git a/src/include/postgres_binary_copy.hpp b/src/include/postgres_binary_copy.hpp +index 4d49b00..f69e4fa 100644 +--- a/src/include/postgres_binary_copy.hpp ++++ b/src/include/postgres_binary_copy.hpp +@@ -19,18 +19,19 @@ public: + PostgresBinaryCopyFunction(); + + static unique_ptr PostgresBinaryWriteBind(ClientContext &context, CopyFunctionBindInput &input, +- const vector &names, const vector &sql_types); +- +- static unique_ptr PostgresBinaryWriteInitializeGlobal(ClientContext &context, FunctionData &bind_data, +- const string &file_path); +- static unique_ptr PostgresBinaryWriteInitializeLocal(ExecutionContext &context, FunctionData &bind_data_p); +- static void PostgresBinaryWriteSink(ExecutionContext &context, FunctionData &bind_data_p, GlobalFunctionData &gstate, +- LocalFunctionData &lstate, DataChunk &input); +- static void PostgresBinaryWriteCombine(ExecutionContext &context, FunctionData &bind_data, GlobalFunctionData &gstate, +- LocalFunctionData &lstate); +- static void PostgresBinaryWriteFinalize(ClientContext &context, FunctionData &bind_data, GlobalFunctionData &gstate); ++ const vector &names, ++ const vector &sql_types); ++ ++ static unique_ptr ++ PostgresBinaryWriteInitializeGlobal(ClientContext &context, FunctionData &bind_data, const string &file_path); ++ static unique_ptr PostgresBinaryWriteInitializeLocal(ExecutionContext &context, ++ FunctionData &bind_data_p); ++ static void PostgresBinaryWriteSink(ExecutionContext &context, FunctionData &bind_data_p, ++ GlobalFunctionData &gstate, LocalFunctionData &lstate, DataChunk &input); ++ static void PostgresBinaryWriteCombine(ExecutionContext &context, FunctionData &bind_data, ++ GlobalFunctionData &gstate, LocalFunctionData &lstate); ++ static void PostgresBinaryWriteFinalize(ClientContext &context, FunctionData &bind_data, ++ GlobalFunctionData &gstate); + }; + +- +- + } // namespace duckdb +diff --git a/src/include/postgres_connection.hpp b/src/include/postgres_connection.hpp +index e4595e2..08fd19c 100644 +--- a/src/include/postgres_connection.hpp ++++ b/src/include/postgres_connection.hpp +@@ -10,6 +10,7 @@ + + #include "postgres_utils.hpp" + #include "postgres_result.hpp" ++#include "duckdb/common/shared_ptr.hpp" + + namespace duckdb { + class PostgresBinaryWriter; +diff --git a/src/include/storage/postgres_catalog_set.hpp b/src/include/storage/postgres_catalog_set.hpp +index e20a803..4fe45f6 100644 +--- a/src/include/storage/postgres_catalog_set.hpp ++++ b/src/include/storage/postgres_catalog_set.hpp +@@ -11,6 +11,7 @@ + #include "duckdb/transaction/transaction.hpp" + #include "duckdb/common/case_insensitive_map.hpp" + #include "duckdb/common/mutex.hpp" ++#include "duckdb/common/shared_ptr.hpp" + + namespace duckdb { + struct DropInfo; +diff --git a/src/postgres_binary_copy.cpp b/src/postgres_binary_copy.cpp +index f0d86a3..4c89c3f 100644 +--- a/src/postgres_binary_copy.cpp ++++ b/src/postgres_binary_copy.cpp +@@ -5,8 +5,7 @@ + + namespace duckdb { + +-PostgresBinaryCopyFunction::PostgresBinaryCopyFunction() : +- CopyFunction("postgres_binary") { ++PostgresBinaryCopyFunction::PostgresBinaryCopyFunction() : CopyFunction("postgres_binary") { + + copy_to_bind = PostgresBinaryWriteBind; + copy_to_initialize_global = PostgresBinaryWriteInitializeGlobal; +@@ -54,16 +53,18 @@ struct PostgresBinaryCopyGlobalState : public GlobalFunctionData { + } + }; + +-struct PostgresBinaryWriteBindData : public TableFunctionData { +-}; ++struct PostgresBinaryWriteBindData : public TableFunctionData {}; + +-unique_ptr PostgresBinaryCopyFunction::PostgresBinaryWriteBind(ClientContext &context, CopyFunctionBindInput &input, +- const vector &names, const vector &sql_types) { ++unique_ptr PostgresBinaryCopyFunction::PostgresBinaryWriteBind(ClientContext &context, ++ CopyFunctionBindInput &input, ++ const vector &names, ++ const vector &sql_types) { + return make_uniq(); + } + +-unique_ptr PostgresBinaryCopyFunction::PostgresBinaryWriteInitializeGlobal(ClientContext &context, FunctionData &bind_data, +- const string &file_path) { ++unique_ptr ++PostgresBinaryCopyFunction::PostgresBinaryWriteInitializeGlobal(ClientContext &context, FunctionData &bind_data, ++ const string &file_path) { + auto result = make_uniq(); + auto &fs = FileSystem::GetFileSystem(context); + result->file_writer = make_uniq(fs, file_path); +@@ -72,25 +73,27 @@ unique_ptr PostgresBinaryCopyFunction::PostgresBinaryWriteIn + return std::move(result); + } + +-unique_ptr PostgresBinaryCopyFunction::PostgresBinaryWriteInitializeLocal(ExecutionContext &context, FunctionData &bind_data_p) { ++unique_ptr ++PostgresBinaryCopyFunction::PostgresBinaryWriteInitializeLocal(ExecutionContext &context, FunctionData &bind_data_p) { + return make_uniq(); + } + +-void PostgresBinaryCopyFunction::PostgresBinaryWriteSink(ExecutionContext &context, FunctionData &bind_data_p, GlobalFunctionData &gstate_p, +- LocalFunctionData &lstate, DataChunk &input) { ++void PostgresBinaryCopyFunction::PostgresBinaryWriteSink(ExecutionContext &context, FunctionData &bind_data_p, ++ GlobalFunctionData &gstate_p, LocalFunctionData &lstate, ++ DataChunk &input) { + auto &gstate = gstate_p.Cast(); + gstate.WriteChunk(input); + } + +-void PostgresBinaryCopyFunction::PostgresBinaryWriteCombine(ExecutionContext &context, FunctionData &bind_data, GlobalFunctionData &gstate, +- LocalFunctionData &lstate) { ++void PostgresBinaryCopyFunction::PostgresBinaryWriteCombine(ExecutionContext &context, FunctionData &bind_data, ++ GlobalFunctionData &gstate, LocalFunctionData &lstate) { + } + +-void PostgresBinaryCopyFunction::PostgresBinaryWriteFinalize(ClientContext &context, FunctionData &bind_data, GlobalFunctionData &gstate_p) { ++void PostgresBinaryCopyFunction::PostgresBinaryWriteFinalize(ClientContext &context, FunctionData &bind_data, ++ GlobalFunctionData &gstate_p) { + auto &gstate = gstate_p.Cast(); + // write the footer and close the file + gstate.Flush(); + } + +- +-} +\ No newline at end of file ++} // namespace duckdb +\ No newline at end of file +diff --git a/src/postgres_connection.cpp b/src/postgres_connection.cpp +index 6372055..18fbd77 100644 +--- a/src/postgres_connection.cpp ++++ b/src/postgres_connection.cpp +@@ -3,6 +3,8 @@ + #include "duckdb/parser/parser.hpp" + #include "postgres_connection.hpp" + #include "duckdb/common/types/uuid.hpp" ++#include "duckdb/common/shared_ptr.hpp" ++#include "duckdb/common/helper.hpp" + + namespace duckdb { + +@@ -40,7 +42,7 @@ PostgresConnection &PostgresConnection::operator=(PostgresConnection &&other) no + + PostgresConnection PostgresConnection::Open(const string &connection_string) { + PostgresConnection result; +- result.connection = make_shared(PostgresUtils::PGConnect(connection_string)); ++ result.connection = make_shared_ptr(PostgresUtils::PGConnect(connection_string)); + result.dsn = connection_string; + return result; + } +diff --git a/src/postgres_extension.cpp b/src/postgres_extension.cpp +index 34d46d0..95988f2 100644 +--- a/src/postgres_extension.cpp ++++ b/src/postgres_extension.cpp +@@ -9,6 +9,7 @@ + #include "duckdb/catalog/catalog.hpp" + #include "duckdb/parser/parsed_data/create_table_function_info.hpp" + #include "duckdb/main/extension_util.hpp" ++#include "duckdb/common/helper.hpp" + #include "duckdb/main/database_manager.hpp" + #include "duckdb/main/attached_database.hpp" + #include "storage/postgres_catalog.hpp" +@@ -47,7 +48,7 @@ public: + class PostgresExtensionCallback : public ExtensionCallback { + public: + void OnConnectionOpened(ClientContext &context) override { +- context.registered_state.insert(make_pair("postgres_extension", make_shared())); ++ context.registered_state.insert(make_pair("postgres_extension", make_shared_ptr())); + } + }; + +@@ -123,7 +124,7 @@ static void LoadInternal(DatabaseInstance &db) { + + config.extension_callbacks.push_back(make_uniq()); + for (auto &connection : ConnectionManager::Get(db).GetConnectionList()) { +- connection->registered_state.insert(make_pair("postgres_extension", make_shared())); ++ connection->registered_state.insert(make_pair("postgres_extension", make_shared_ptr())); + } + } + +diff --git a/src/postgres_scanner.cpp b/src/postgres_scanner.cpp +index 449df0b..75c029f 100644 +--- a/src/postgres_scanner.cpp ++++ b/src/postgres_scanner.cpp +@@ -3,6 +3,8 @@ + #include + + #include "duckdb/main/extension_util.hpp" ++#include "duckdb/common/shared_ptr.hpp" ++#include "duckdb/common/helper.hpp" + #include "duckdb/parser/parsed_data/create_table_function_info.hpp" + #include "postgres_filter_pushdown.hpp" + #include "postgres_scanner.hpp" +diff --git a/src/storage/postgres_schema_set.cpp b/src/storage/postgres_schema_set.cpp +index 93c3f28..cd3b46f 100644 +--- a/src/storage/postgres_schema_set.cpp ++++ b/src/storage/postgres_schema_set.cpp +@@ -6,6 +6,7 @@ + #include "duckdb/parser/parsed_data/create_schema_info.hpp" + #include "storage/postgres_table_set.hpp" + #include "storage/postgres_catalog.hpp" ++#include "duckdb/common/shared_ptr.hpp" + + namespace duckdb { + From 01d9944d4bbcbde02cff83ce8d15e8a4830ccf1c Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 12 Apr 2024 12:46:15 +0200 Subject: [PATCH 380/603] Adjustments to make bill happy --- .../operator/csv_scanner/util/csv_reader_options.cpp | 2 +- src/function/table/read_csv.cpp | 5 ++--- test/sql/copy/csv/csv_nullstr_list.test | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp index d7500802d03d..f9da6ac8dc2b 100644 --- a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp +++ b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp @@ -295,7 +295,7 @@ bool CSVReaderOptions::SetBaseOption(const string &loption, const Value &value) if (child_type.id() != LogicalTypeId::LIST && child_type.id() != LogicalTypeId::VARCHAR) { throw BinderException("read_csv %s option requires a string or a list as input", loption); } - if (!sql_type_list.empty()) { + if (!null_str.empty()) { throw BinderException("read_csv_auto nullstr can only be supplied once"); } if (child_type.id() == LogicalTypeId::LIST) { diff --git a/src/function/table/read_csv.cpp b/src/function/table/read_csv.cpp index ec0b52951a8a..4434d7cd8bc7 100644 --- a/src/function/table/read_csv.cpp +++ b/src/function/table/read_csv.cpp @@ -140,9 +140,8 @@ static unique_ptr ReadCSVBind(ClientContext &context, TableFunctio } for (auto &force_name : options.force_not_null_names) { if (column_names.find(force_name) == column_names.end()) { - throw BinderException( - "Column name %s specified in force_not_null parameter, does not exist in the CSV File.", - force_name); + throw BinderException("\"force_not_null\" expected to find %s, but it was not found in the table", + force_name); } } D_ASSERT(options.force_not_null.empty()); diff --git a/test/sql/copy/csv/csv_nullstr_list.test b/test/sql/copy/csv/csv_nullstr_list.test index 3fe0de49bc1e..0e7914a6dc2a 100644 --- a/test/sql/copy/csv/csv_nullstr_list.test +++ b/test/sql/copy/csv/csv_nullstr_list.test @@ -127,4 +127,4 @@ NULL statement error FROM read_csv('data/csv/null/multiple_nulls.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'name': 'VARCHAR', 'age': 'VARCHAR', 'height': 'VARCHAR'}, nullstr = ['','none','null'], force_not_null = ['dont_exist']); ---- -Column name dont_exist specified in force_not_null parameter, does not exist in the CSV File. +"force_not_null" expected to find dont_exist, but it was not found in the table From dd40c1c7392723a3f8b001bcea9aa193be8d99ed Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 12 Apr 2024 12:46:26 +0200 Subject: [PATCH 381/603] add patch for substrait --- .github/config/out_of_tree_extensions.cmake | 1 + .../extensions/substrait/shared_ptr.patch | 130 ++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 .github/patches/extensions/substrait/shared_ptr.patch diff --git a/.github/config/out_of_tree_extensions.cmake b/.github/config/out_of_tree_extensions.cmake index a96f35e1d7be..3c20819cfd5d 100644 --- a/.github/config/out_of_tree_extensions.cmake +++ b/.github/config/out_of_tree_extensions.cmake @@ -102,5 +102,6 @@ if (NOT WIN32) LOAD_TESTS DONT_LINK GIT_URL https://github.com/duckdb/substrait GIT_TAG 1116fb580edd3e26e675436dbdbdf4a0aa5e456e + APPLY_PATCHES ) endif() diff --git a/.github/patches/extensions/substrait/shared_ptr.patch b/.github/patches/extensions/substrait/shared_ptr.patch new file mode 100644 index 000000000000..d04cfb9655af --- /dev/null +++ b/.github/patches/extensions/substrait/shared_ptr.patch @@ -0,0 +1,130 @@ +diff --git a/src/from_substrait.cpp b/src/from_substrait.cpp +index 566e21d..afbbb0b 100644 +--- a/src/from_substrait.cpp ++++ b/src/from_substrait.cpp +@@ -14,6 +14,8 @@ + #include "duckdb/main/connection.hpp" + #include "duckdb/parser/parser.hpp" + #include "duckdb/common/exception.hpp" ++#include "duckdb/common/helper.hpp" ++#include "duckdb/common/shared_ptr.hpp" + #include "duckdb/common/types.hpp" + #include "duckdb/common/enums/set_operation_type.hpp" + +@@ -404,25 +406,25 @@ shared_ptr SubstraitToDuckDB::TransformJoinOp(const substrait::Rel &so + throw InternalException("Unsupported join type"); + } + unique_ptr join_condition = TransformExpr(sjoin.expression()); +- return make_shared(TransformOp(sjoin.left())->Alias("left"), ++ return make_shared_ptr(TransformOp(sjoin.left())->Alias("left"), + TransformOp(sjoin.right())->Alias("right"), std::move(join_condition), djointype); + } + + shared_ptr SubstraitToDuckDB::TransformCrossProductOp(const substrait::Rel &sop) { + auto &sub_cross = sop.cross(); + +- return make_shared(TransformOp(sub_cross.left())->Alias("left"), ++ return make_shared_ptr(TransformOp(sub_cross.left())->Alias("left"), + TransformOp(sub_cross.right())->Alias("right")); + } + + shared_ptr SubstraitToDuckDB::TransformFetchOp(const substrait::Rel &sop) { + auto &slimit = sop.fetch(); +- return make_shared(TransformOp(slimit.input()), slimit.count(), slimit.offset()); ++ return make_shared_ptr(TransformOp(slimit.input()), slimit.count(), slimit.offset()); + } + + shared_ptr SubstraitToDuckDB::TransformFilterOp(const substrait::Rel &sop) { + auto &sfilter = sop.filter(); +- return make_shared(TransformOp(sfilter.input()), TransformExpr(sfilter.condition())); ++ return make_shared_ptr(TransformOp(sfilter.input()), TransformExpr(sfilter.condition())); + } + + shared_ptr SubstraitToDuckDB::TransformProjectOp(const substrait::Rel &sop) { +@@ -435,7 +437,7 @@ shared_ptr SubstraitToDuckDB::TransformProjectOp(const substrait::Rel + for (size_t i = 0; i < expressions.size(); i++) { + mock_aliases.push_back("expr_" + to_string(i)); + } +- return make_shared(TransformOp(sop.project().input()), std::move(expressions), ++ return make_shared_ptr(TransformOp(sop.project().input()), std::move(expressions), + std::move(mock_aliases)); + } + +@@ -463,7 +465,7 @@ shared_ptr SubstraitToDuckDB::TransformAggregateOp(const substrait::Re + expressions.push_back(make_uniq(RemapFunctionName(function_name), std::move(children))); + } + +- return make_shared(TransformOp(sop.aggregate().input()), std::move(expressions), ++ return make_shared_ptr(TransformOp(sop.aggregate().input()), std::move(expressions), + std::move(groups)); + } + +@@ -502,7 +504,7 @@ shared_ptr SubstraitToDuckDB::TransformReadOp(const substrait::Rel &so + } + + if (sget.has_filter()) { +- scan = make_shared(std::move(scan), TransformExpr(sget.filter())); ++ scan = make_shared_ptr(std::move(scan), TransformExpr(sget.filter())); + } + + if (sget.has_projection()) { +@@ -516,7 +518,7 @@ shared_ptr SubstraitToDuckDB::TransformReadOp(const substrait::Rel &so + expressions.push_back(make_uniq(sproj.field() + 1)); + } + +- scan = make_shared(std::move(scan), std::move(expressions), std::move(aliases)); ++ scan = make_shared_ptr(std::move(scan), std::move(expressions), std::move(aliases)); + } + + return scan; +@@ -527,7 +529,7 @@ shared_ptr SubstraitToDuckDB::TransformSortOp(const substrait::Rel &so + for (auto &sordf : sop.sort().sorts()) { + order_nodes.push_back(TransformOrder(sordf)); + } +- return make_shared(TransformOp(sop.sort().input()), std::move(order_nodes)); ++ return make_shared_ptr(TransformOp(sop.sort().input()), std::move(order_nodes)); + } + + static duckdb::SetOperationType TransformSetOperationType(substrait::SetRel_SetOp setop) { +@@ -562,7 +564,7 @@ shared_ptr SubstraitToDuckDB::TransformSetOp(const substrait::Rel &sop + auto lhs = TransformOp(inputs[0]); + auto rhs = TransformOp(inputs[1]); + +- return make_shared(std::move(lhs), std::move(rhs), type); ++ return make_shared_ptr(std::move(lhs), std::move(rhs), type); + } + + shared_ptr SubstraitToDuckDB::TransformOp(const substrait::Rel &sop) { +@@ -599,7 +601,7 @@ shared_ptr SubstraitToDuckDB::TransformRootOp(const substrait::RelRoot + aliases.push_back(column_name); + expressions.push_back(make_uniq(id++)); + } +- return make_shared(TransformOp(sop.input()), std::move(expressions), aliases); ++ return make_shared_ptr(TransformOp(sop.input()), std::move(expressions), aliases); + } + + shared_ptr SubstraitToDuckDB::TransformPlan() { +diff --git a/src/include/from_substrait.hpp b/src/include/from_substrait.hpp +index 8ea96cd..3a632ce 100644 +--- a/src/include/from_substrait.hpp ++++ b/src/include/from_substrait.hpp +@@ -5,6 +5,7 @@ + #include + #include "substrait/plan.pb.h" + #include "duckdb/main/connection.hpp" ++#include "duckdb/common/shared_ptr.hpp" + + namespace duckdb { + class SubstraitToDuckDB { +diff --git a/src/substrait_extension.cpp b/src/substrait_extension.cpp +index fae645c..6422ebd 100644 +--- a/src/substrait_extension.cpp ++++ b/src/substrait_extension.cpp +@@ -6,6 +6,7 @@ + + #ifndef DUCKDB_AMALGAMATION + #include "duckdb/common/enums/optimizer_type.hpp" ++#include "duckdb/common/shared_ptr.hpp" + #include "duckdb/function/table_function.hpp" + #include "duckdb/parser/parsed_data/create_table_function_info.hpp" + #include "duckdb/parser/parsed_data/create_pragma_function_info.hpp" From e056dd0525364e86530330ee6fe83637249d86d4 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Fri, 12 Apr 2024 12:48:27 +0200 Subject: [PATCH 382/603] Fuzzer - avoid signed overflow in ORDER BY --- src/planner/expression_binder/order_binder.cpp | 3 ++- test/sql/order/order_overflow.test | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 test/sql/order/order_overflow.test diff --git a/src/planner/expression_binder/order_binder.cpp b/src/planner/expression_binder/order_binder.cpp index 7dc062d180f6..c5a63de8ce84 100644 --- a/src/planner/expression_binder/order_binder.cpp +++ b/src/planner/expression_binder/order_binder.cpp @@ -58,7 +58,8 @@ unique_ptr OrderBinder::BindConstant(ParsedExpression &expr, const V return nullptr; } // INTEGER constant: we use the integer as an index into the select list (e.g. ORDER BY 1) - auto index = idx_t(val.GetValue() - 1); + auto order_value = val.GetValue(); + auto index = order_value <= 0 ? NumericLimits::Maximum() : idx_t(order_value - 1); child_list_t values; values.push_back(make_pair("index", Value::UBIGINT(index))); auto result = make_uniq(Value::STRUCT(std::move(values))); diff --git a/test/sql/order/order_overflow.test b/test/sql/order/order_overflow.test new file mode 100644 index 000000000000..64bc0dc81ee3 --- /dev/null +++ b/test/sql/order/order_overflow.test @@ -0,0 +1,8 @@ +# name: test/sql/order/order_overflow.test +# description: Test overflow in ORDER BY +# group: [tpch] + +statement error +SELECT 42 ORDER BY -9223372036854775808; +---- +ORDER term out of range From 612310ee2d5baf9e923e7be034c26f0307807dd3 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Fri, 12 Apr 2024 12:54:16 +0200 Subject: [PATCH 383/603] Add simplification to STAR expressions --- .../sqlsmith/include/statement_simplifier.hpp | 4 ++ extension/sqlsmith/statement_simplifier.cpp | 49 +++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/extension/sqlsmith/include/statement_simplifier.hpp b/extension/sqlsmith/include/statement_simplifier.hpp index 9e6fa160ede4..25f4848a849f 100644 --- a/extension/sqlsmith/include/statement_simplifier.hpp +++ b/extension/sqlsmith/include/statement_simplifier.hpp @@ -46,6 +46,10 @@ class StatementSimplifier { template void SimplifyList(vector &list, bool is_optional = true); + template + void SimplifyMap(T &map); + template + void SimplifySet(T &set); template void SimplifyReplace(T &element, T &other); diff --git a/extension/sqlsmith/statement_simplifier.cpp b/extension/sqlsmith/statement_simplifier.cpp index 998d32df9fcc..0395e594a6da 100644 --- a/extension/sqlsmith/statement_simplifier.cpp +++ b/extension/sqlsmith/statement_simplifier.cpp @@ -44,6 +44,46 @@ void StatementSimplifier::SimplifyList(vector &list, bool is_optional) { } } +template +void StatementSimplifier::SimplifyMap(T &map) { + if (map.empty()) { + return; + } + // copy the keys + vector keys; + for(auto &entry : map) { + keys.push_back(entry.first); + } + // try to remove all of the keys + for(idx_t i = 0; i < keys.size(); i++) { + auto entry = map.find(keys[i]); + auto n = std::move(entry->second); + map.erase(entry); + Simplification(); + map.insert(make_pair(std::move(keys[i]), std::move(n))); + } +} + +template +void StatementSimplifier::SimplifySet(T &set) { + if (set.empty()) { + return; + } + // copy the keys + vector keys; + for(auto &entry : set) { + keys.push_back(entry); + } + // try to remove all of the keys + for(idx_t i = 0; i < keys.size(); i++) { + auto entry = set.find(keys[i]); + set.erase(entry); + Simplification(); + set.insert(std::move(keys[i])); + } + +} + template void StatementSimplifier::SimplifyOptional(duckdb::unique_ptr &opt) { if (!opt) { @@ -234,6 +274,15 @@ void StatementSimplifier::SimplifyExpression(duckdb::unique_ptrCast(); + SimplifyMap(star.replace_list); + SimplifySet(star.exclude_list); + for(auto &entry : star.replace_list) { + SimplifyChildExpression(expr, entry.second); + } + break; + } case ExpressionClass::WINDOW: { auto &window = expr->Cast(); SimplifyExpressionList(expr, window.children); From 8247c57d95b736b0b258b8677cd753b48d0593c0 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 12 Apr 2024 12:59:49 +0200 Subject: [PATCH 384/603] add patch for arrow --- .github/config/out_of_tree_extensions.cmake | 1 + .../patches/extensions/arrow/shared_ptr.patch | 1134 +++++++++++++++++ 2 files changed, 1135 insertions(+) create mode 100644 .github/patches/extensions/arrow/shared_ptr.patch diff --git a/.github/config/out_of_tree_extensions.cmake b/.github/config/out_of_tree_extensions.cmake index 3c20819cfd5d..e1bcd5e2aec9 100644 --- a/.github/config/out_of_tree_extensions.cmake +++ b/.github/config/out_of_tree_extensions.cmake @@ -20,6 +20,7 @@ duckdb_extension_load(arrow LOAD_TESTS DONT_LINK GIT_URL https://github.com/duckdb/arrow GIT_TAG 9e10240da11f61ea7fbfe3fc9988ffe672ccd40f + APPLY_PATCHES ) ################## AWS diff --git a/.github/patches/extensions/arrow/shared_ptr.patch b/.github/patches/extensions/arrow/shared_ptr.patch new file mode 100644 index 000000000000..d523ffe01119 --- /dev/null +++ b/.github/patches/extensions/arrow/shared_ptr.patch @@ -0,0 +1,1134 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 72b1370..c95486c 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -7,56 +7,55 @@ set(EXTENSION_NAME ${TARGET_NAME}_extension) + project(${TARGET_NAME}) + include_directories(src/include) + +-set(EXTENSION_SOURCES +- src/arrow_extension.cpp +- src/arrow_stream_buffer.cpp +- src/arrow_scan_ipc.cpp +- src/arrow_to_ipc.cpp) ++set(EXTENSION_SOURCES src/arrow_extension.cpp src/arrow_stream_buffer.cpp ++ src/arrow_scan_ipc.cpp src/arrow_to_ipc.cpp) + + if(NOT "${OSX_BUILD_ARCH}" STREQUAL "") +- set(OSX_ARCH_FLAG -DCMAKE_OSX_ARCHITECTURES=${OSX_BUILD_ARCH}) ++ set(OSX_ARCH_FLAG -DCMAKE_OSX_ARCHITECTURES=${OSX_BUILD_ARCH}) + else() +- set(OSX_ARCH_FLAG "") ++ set(OSX_ARCH_FLAG "") + endif() + + # Building Arrow + include(ExternalProject) + ExternalProject_Add( +- ARROW_EP +- GIT_REPOSITORY "https://github.com/apache/arrow" +- GIT_TAG ea6875fd2a3ac66547a9a33c5506da94f3ff07f2 +- PREFIX "${CMAKE_BINARY_DIR}/third_party/arrow" +- INSTALL_DIR "${CMAKE_BINARY_DIR}/third_party/arrow/install" +- BUILD_BYPRODUCTS /lib/libarrow.a +- CONFIGURE_COMMAND +- ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} ${OSX_ARCH_FLAG} +- -DCMAKE_BUILD_TYPE=Release +- -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/third_party/arrow/install +- -DCMAKE_INSTALL_LIBDIR=lib -DARROW_BUILD_STATIC=ON -DARROW_BUILD_SHARED=OFF +- -DARROW_NO_DEPRECATED_API=ON -DARROW_POSITION_INDEPENDENT_CODE=ON +- -DARROW_SIMD_LEVEL=NONE -DARROW_ENABLE_TIMING_TESTS=OFF -DARROW_IPC=ON +- -DARROW_JEMALLOC=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED +- -DARROW_VERBOSE_THIRDPARTY_BUILD=OFF -DARROW_DEPENDENCY_USE_SHARED=OFF +- -DARROW_BOOST_USE_SHARED=OFF -DARROW_BROTLI_USE_SHARED=OFF +- -DARROW_BZ2_USE_SHARED=OFF -DARROW_GFLAGS_USE_SHARED=OFF +- -DARROW_GRPC_USE_SHARED=OFF -DARROW_JEMALLOC_USE_SHARED=OFF +- -DARROW_LZ4_USE_SHARED=OFF -DARROW_OPENSSL_USE_SHARED=OFF +- -DARROW_PROTOBUF_USE_SHARED=OFF -DARROW_SNAPPY_USE_SHARED=OFF +- -DARROW_THRIFT_USE_SHARED=OFF -DARROW_UTF8PROC_USE_SHARED=OFF +- -DARROW_ZSTD_USE_SHARED=OFF -DARROW_USE_GLOG=OFF -DARROW_WITH_BACKTRACE=OFF +- -DARROW_WITH_OPENTELEMETRY=OFF -DARROW_WITH_BROTLI=OFF -DARROW_WITH_BZ2=OFF +- -DARROW_WITH_LZ4=OFF -DARROW_WITH_SNAPPY=OFF -DARROW_WITH_ZLIB=OFF +- -DARROW_WITH_ZSTD=OFF -DARROW_WITH_UCX=OFF -DARROW_WITH_UTF8PROC=OFF +- -DARROW_WITH_RE2=OFF /cpp +- CMAKE_ARGS -Wno-dev +- UPDATE_COMMAND "") ++ ARROW_EP ++ GIT_REPOSITORY "https://github.com/apache/arrow" ++ GIT_TAG ea6875fd2a3ac66547a9a33c5506da94f3ff07f2 ++ PREFIX "${CMAKE_BINARY_DIR}/third_party/arrow" ++ INSTALL_DIR "${CMAKE_BINARY_DIR}/third_party/arrow/install" ++ BUILD_BYPRODUCTS /lib/libarrow.a ++ CONFIGURE_COMMAND ++ ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} ${OSX_ARCH_FLAG} ++ -DCMAKE_BUILD_TYPE=Release ++ -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/third_party/arrow/install ++ -DCMAKE_INSTALL_LIBDIR=lib -DARROW_BUILD_STATIC=ON -DARROW_BUILD_SHARED=OFF ++ -DARROW_NO_DEPRECATED_API=ON -DARROW_POSITION_INDEPENDENT_CODE=ON ++ -DARROW_SIMD_LEVEL=NONE -DARROW_ENABLE_TIMING_TESTS=OFF -DARROW_IPC=ON ++ -DARROW_JEMALLOC=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED ++ -DARROW_VERBOSE_THIRDPARTY_BUILD=OFF -DARROW_DEPENDENCY_USE_SHARED=OFF ++ -DARROW_BOOST_USE_SHARED=OFF -DARROW_BROTLI_USE_SHARED=OFF ++ -DARROW_BZ2_USE_SHARED=OFF -DARROW_GFLAGS_USE_SHARED=OFF ++ -DARROW_GRPC_USE_SHARED=OFF -DARROW_JEMALLOC_USE_SHARED=OFF ++ -DARROW_LZ4_USE_SHARED=OFF -DARROW_OPENSSL_USE_SHARED=OFF ++ -DARROW_PROTOBUF_USE_SHARED=OFF -DARROW_SNAPPY_USE_SHARED=OFF ++ -DARROW_THRIFT_USE_SHARED=OFF -DARROW_UTF8PROC_USE_SHARED=OFF ++ -DARROW_ZSTD_USE_SHARED=OFF -DARROW_USE_GLOG=OFF -DARROW_WITH_BACKTRACE=OFF ++ -DARROW_WITH_OPENTELEMETRY=OFF -DARROW_WITH_BROTLI=OFF -DARROW_WITH_BZ2=OFF ++ -DARROW_WITH_LZ4=OFF -DARROW_WITH_SNAPPY=OFF -DARROW_WITH_ZLIB=OFF ++ -DARROW_WITH_ZSTD=OFF -DARROW_WITH_UCX=OFF -DARROW_WITH_UTF8PROC=OFF ++ -DARROW_WITH_RE2=OFF /cpp ++ CMAKE_ARGS -Wno-dev ++ UPDATE_COMMAND "") + + ExternalProject_Get_Property(ARROW_EP install_dir) + add_library(arrow STATIC IMPORTED GLOBAL) + if(WIN32) +- set_target_properties(arrow PROPERTIES IMPORTED_LOCATION ${install_dir}/lib/arrow_static.lib) ++ set_target_properties(arrow PROPERTIES IMPORTED_LOCATION ++ ${install_dir}/lib/arrow_static.lib) + else() +- set_target_properties(arrow PROPERTIES IMPORTED_LOCATION ${install_dir}/lib/libarrow.a) ++ set_target_properties(arrow PROPERTIES IMPORTED_LOCATION ++ ${install_dir}/lib/libarrow.a) + endif() + + # create static library +@@ -71,12 +70,14 @@ build_loadable_extension(${TARGET_NAME} ${PARAMETERS} ${EXTENSION_SOURCES}) + add_dependencies(${TARGET_NAME}_loadable_extension ARROW_EP) + target_link_libraries(${TARGET_NAME}_loadable_extension arrow) + if(WIN32) +- target_compile_definitions(${TARGET_NAME}_loadable_extension PUBLIC ARROW_STATIC) ++ target_compile_definitions(${TARGET_NAME}_loadable_extension ++ PUBLIC ARROW_STATIC) + endif() +-target_include_directories(${TARGET_NAME}_loadable_extension PRIVATE ${install_dir}/include) ++target_include_directories(${TARGET_NAME}_loadable_extension ++ PRIVATE ${install_dir}/include) + + install( +- TARGETS ${EXTENSION_NAME} +- EXPORT "${DUCKDB_EXPORT_SET}" +- LIBRARY DESTINATION "${INSTALL_LIB_DIR}" +- ARCHIVE DESTINATION "${INSTALL_LIB_DIR}") +\ No newline at end of file ++ TARGETS ${EXTENSION_NAME} ++ EXPORT "${DUCKDB_EXPORT_SET}" ++ LIBRARY DESTINATION "${INSTALL_LIB_DIR}" ++ ARCHIVE DESTINATION "${INSTALL_LIB_DIR}") +diff --git a/src/arrow_extension.cpp b/src/arrow_extension.cpp +index e4daf26..6fadec0 100644 +--- a/src/arrow_extension.cpp ++++ b/src/arrow_extension.cpp +@@ -18,27 +18,24 @@ + namespace duckdb { + + static void LoadInternal(DatabaseInstance &instance) { +- ExtensionUtil::RegisterFunction(instance, ToArrowIPCFunction::GetFunction()); +- ExtensionUtil::RegisterFunction(instance, ArrowIPCTableFunction::GetFunction()); ++ ExtensionUtil::RegisterFunction(instance, ToArrowIPCFunction::GetFunction()); ++ ExtensionUtil::RegisterFunction(instance, ++ ArrowIPCTableFunction::GetFunction()); + } + +-void ArrowExtension::Load(DuckDB &db) { +- LoadInternal(*db.instance); +-} +-std::string ArrowExtension::Name() { +- return "arrow"; +-} ++void ArrowExtension::Load(DuckDB &db) { LoadInternal(*db.instance); } ++std::string ArrowExtension::Name() { return "arrow"; } + + } // namespace duckdb + + extern "C" { + + DUCKDB_EXTENSION_API void arrow_init(duckdb::DatabaseInstance &db) { +- LoadInternal(db); ++ LoadInternal(db); + } + + DUCKDB_EXTENSION_API const char *arrow_version() { +- return duckdb::DuckDB::LibraryVersion(); ++ return duckdb::DuckDB::LibraryVersion(); + } + } + +diff --git a/src/arrow_scan_ipc.cpp b/src/arrow_scan_ipc.cpp +index 7d5b2ff..a60d255 100644 +--- a/src/arrow_scan_ipc.cpp ++++ b/src/arrow_scan_ipc.cpp +@@ -3,111 +3,131 @@ + namespace duckdb { + + TableFunction ArrowIPCTableFunction::GetFunction() { +- child_list_t make_buffer_struct_children{{"ptr", LogicalType::UBIGINT}, +- {"size", LogicalType::UBIGINT}}; +- +- TableFunction scan_arrow_ipc_func( +- "scan_arrow_ipc", {LogicalType::LIST(LogicalType::STRUCT(make_buffer_struct_children))}, +- ArrowIPCTableFunction::ArrowScanFunction, ArrowIPCTableFunction::ArrowScanBind, +- ArrowTableFunction::ArrowScanInitGlobal, ArrowTableFunction::ArrowScanInitLocal); +- +- scan_arrow_ipc_func.cardinality = ArrowTableFunction::ArrowScanCardinality; +- scan_arrow_ipc_func.get_batch_index = nullptr; // TODO implement +- scan_arrow_ipc_func.projection_pushdown = true; +- scan_arrow_ipc_func.filter_pushdown = false; +- +- return scan_arrow_ipc_func; ++ child_list_t make_buffer_struct_children{ ++ {"ptr", LogicalType::UBIGINT}, {"size", LogicalType::UBIGINT}}; ++ ++ TableFunction scan_arrow_ipc_func( ++ "scan_arrow_ipc", ++ {LogicalType::LIST(LogicalType::STRUCT(make_buffer_struct_children))}, ++ ArrowIPCTableFunction::ArrowScanFunction, ++ ArrowIPCTableFunction::ArrowScanBind, ++ ArrowTableFunction::ArrowScanInitGlobal, ++ ArrowTableFunction::ArrowScanInitLocal); ++ ++ scan_arrow_ipc_func.cardinality = ArrowTableFunction::ArrowScanCardinality; ++ scan_arrow_ipc_func.get_batch_index = nullptr; // TODO implement ++ scan_arrow_ipc_func.projection_pushdown = true; ++ scan_arrow_ipc_func.filter_pushdown = false; ++ ++ return scan_arrow_ipc_func; + } + +-unique_ptr ArrowIPCTableFunction::ArrowScanBind(ClientContext &context, TableFunctionBindInput &input, +- vector &return_types, vector &names) { +- auto stream_decoder = make_uniq(); ++unique_ptr ArrowIPCTableFunction::ArrowScanBind( ++ ClientContext &context, TableFunctionBindInput &input, ++ vector &return_types, vector &names) { ++ auto stream_decoder = make_uniq(); + +- // Decode buffer ptr list +- auto buffer_ptr_list = ListValue::GetChildren(input.inputs[0]); +- for (auto &buffer_ptr_struct: buffer_ptr_list) { +- auto unpacked = StructValue::GetChildren(buffer_ptr_struct); +- uint64_t ptr = unpacked[0].GetValue(); +- uint64_t size = unpacked[1].GetValue(); ++ // Decode buffer ptr list ++ auto buffer_ptr_list = ListValue::GetChildren(input.inputs[0]); ++ for (auto &buffer_ptr_struct : buffer_ptr_list) { ++ auto unpacked = StructValue::GetChildren(buffer_ptr_struct); ++ uint64_t ptr = unpacked[0].GetValue(); ++ uint64_t size = unpacked[1].GetValue(); + +- // Feed stream into decoder +- auto res = stream_decoder->Consume((const uint8_t *) ptr, size); ++ // Feed stream into decoder ++ auto res = stream_decoder->Consume((const uint8_t *)ptr, size); + +- if (!res.ok()) { +- throw IOException("Invalid IPC stream"); +- } ++ if (!res.ok()) { ++ throw IOException("Invalid IPC stream"); + } +- +- if (!stream_decoder->buffer()->is_eos()) { +- throw IOException("IPC buffers passed to arrow scan should contain entire stream"); ++ } ++ ++ if (!stream_decoder->buffer()->is_eos()) { ++ throw IOException( ++ "IPC buffers passed to arrow scan should contain entire stream"); ++ } ++ ++ // These are the params I need to produce from the ipc buffers using the ++ // WebDB.cc code ++ auto stream_factory_ptr = (uintptr_t)&stream_decoder->buffer(); ++ auto stream_factory_produce = ++ (stream_factory_produce_t)&ArrowIPCStreamBufferReader::CreateStream; ++ auto stream_factory_get_schema = ++ (stream_factory_get_schema_t)&ArrowIPCStreamBufferReader::GetSchema; ++ auto res = make_uniq(stream_factory_produce, ++ stream_factory_ptr); ++ ++ // Store decoder ++ res->stream_decoder = std::move(stream_decoder); ++ ++ // TODO Everything below this is identical to the bind in ++ // duckdb/src/function/table/arrow.cpp ++ auto &data = *res; ++ stream_factory_get_schema((ArrowArrayStream *)stream_factory_ptr, ++ data.schema_root.arrow_schema); ++ for (idx_t col_idx = 0; ++ col_idx < (idx_t)data.schema_root.arrow_schema.n_children; col_idx++) { ++ auto &schema = *data.schema_root.arrow_schema.children[col_idx]; ++ if (!schema.release) { ++ throw InvalidInputException("arrow_scan: released schema passed"); + } +- +- // These are the params I need to produce from the ipc buffers using the WebDB.cc code +- auto stream_factory_ptr = (uintptr_t) & stream_decoder->buffer(); +- auto stream_factory_produce = (stream_factory_produce_t) & ArrowIPCStreamBufferReader::CreateStream; +- auto stream_factory_get_schema = (stream_factory_get_schema_t) & ArrowIPCStreamBufferReader::GetSchema; +- auto res = make_uniq(stream_factory_produce, stream_factory_ptr); +- +- // Store decoder +- res->stream_decoder = std::move(stream_decoder); +- +- // TODO Everything below this is identical to the bind in duckdb/src/function/table/arrow.cpp +- auto &data = *res; +- stream_factory_get_schema((ArrowArrayStream *) stream_factory_ptr, data.schema_root.arrow_schema); +- for (idx_t col_idx = 0; col_idx < (idx_t) data.schema_root.arrow_schema.n_children; col_idx++) { +- auto &schema = *data.schema_root.arrow_schema.children[col_idx]; +- if (!schema.release) { +- throw InvalidInputException("arrow_scan: released schema passed"); +- } +- auto arrow_type = GetArrowLogicalType(schema); +- if (schema.dictionary) { +- auto dictionary_type = GetArrowLogicalType(*schema.dictionary); +- return_types.emplace_back(dictionary_type->GetDuckType()); +- arrow_type->SetDictionary(std::move(dictionary_type)); +- } else { +- return_types.emplace_back(arrow_type->GetDuckType()); +- } +- res->arrow_table.AddColumn(col_idx, std::move(arrow_type)); +- auto format = string(schema.format); +- auto name = string(schema.name); +- if (name.empty()) { +- name = string("v") + to_string(col_idx); +- } +- names.push_back(name); ++ auto arrow_type = GetArrowLogicalType(schema); ++ if (schema.dictionary) { ++ auto dictionary_type = GetArrowLogicalType(*schema.dictionary); ++ return_types.emplace_back(dictionary_type->GetDuckType()); ++ arrow_type->SetDictionary(std::move(dictionary_type)); ++ } else { ++ return_types.emplace_back(arrow_type->GetDuckType()); + } +- QueryResult::DeduplicateColumns(names); +- return std::move(res); ++ res->arrow_table.AddColumn(col_idx, std::move(arrow_type)); ++ auto format = string(schema.format); ++ auto name = string(schema.name); ++ if (name.empty()) { ++ name = string("v") + to_string(col_idx); ++ } ++ names.push_back(name); ++ } ++ QueryResult::DeduplicateColumns(names); ++ return std::move(res); + } + +-// Same as regular arrow scan, except ArrowToDuckDB call TODO: refactor to allow nicely overriding this +-void ArrowIPCTableFunction::ArrowScanFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { +- if (!data_p.local_state) { +- return; +- } +- auto &data = data_p.bind_data->CastNoConst(); +- auto &state = data_p.local_state->Cast(); +- auto &global_state = data_p.global_state->Cast(); +- +- //! Out of tuples in this chunk +- if (state.chunk_offset >= (idx_t)state.chunk->arrow_array.length) { +- if (!ArrowScanParallelStateNext(context, data_p.bind_data.get(), state, global_state)) { +- return; +- } ++// Same as regular arrow scan, except ArrowToDuckDB call TODO: refactor to allow ++// nicely overriding this ++void ArrowIPCTableFunction::ArrowScanFunction(ClientContext &context, ++ TableFunctionInput &data_p, ++ DataChunk &output) { ++ if (!data_p.local_state) { ++ return; ++ } ++ auto &data = data_p.bind_data->CastNoConst(); ++ auto &state = data_p.local_state->Cast(); ++ auto &global_state = data_p.global_state->Cast(); ++ ++ //! Out of tuples in this chunk ++ if (state.chunk_offset >= (idx_t)state.chunk->arrow_array.length) { ++ if (!ArrowScanParallelStateNext(context, data_p.bind_data.get(), state, ++ global_state)) { ++ return; + } +- int64_t output_size = MinValue(STANDARD_VECTOR_SIZE, state.chunk->arrow_array.length - state.chunk_offset); +- data.lines_read += output_size; +- if (global_state.CanRemoveFilterColumns()) { +- state.all_columns.Reset(); +- state.all_columns.SetCardinality(output_size); +- ArrowToDuckDB(state, data.arrow_table.GetColumns(), state.all_columns, data.lines_read - output_size, false); +- output.ReferenceColumns(state.all_columns, global_state.projection_ids); +- } else { +- output.SetCardinality(output_size); +- ArrowToDuckDB(state, data.arrow_table.GetColumns(), output, data.lines_read - output_size, false); +- } +- +- output.Verify(); +- state.chunk_offset += output.size(); ++ } ++ int64_t output_size = ++ MinValue(STANDARD_VECTOR_SIZE, ++ state.chunk->arrow_array.length - state.chunk_offset); ++ data.lines_read += output_size; ++ if (global_state.CanRemoveFilterColumns()) { ++ state.all_columns.Reset(); ++ state.all_columns.SetCardinality(output_size); ++ ArrowToDuckDB(state, data.arrow_table.GetColumns(), state.all_columns, ++ data.lines_read - output_size, false); ++ output.ReferenceColumns(state.all_columns, global_state.projection_ids); ++ } else { ++ output.SetCardinality(output_size); ++ ArrowToDuckDB(state, data.arrow_table.GetColumns(), output, ++ data.lines_read - output_size, false); ++ } ++ ++ output.Verify(); ++ state.chunk_offset += output.size(); + } + + } // namespace duckdb +\ No newline at end of file +diff --git a/src/arrow_stream_buffer.cpp b/src/arrow_stream_buffer.cpp +index f097ca1..c9791e4 100644 +--- a/src/arrow_stream_buffer.cpp ++++ b/src/arrow_stream_buffer.cpp +@@ -1,95 +1,108 @@ + #include "arrow_stream_buffer.hpp" + + #include ++#include + + /// File copied from + /// https://github.com/duckdb/duckdb-wasm/blob/0ad10e7db4ef4025f5f4120be37addc4ebe29618/lib/src/arrow_stream_buffer.cc + namespace duckdb { + + /// Constructor +-ArrowIPCStreamBuffer::ArrowIPCStreamBuffer() : schema_(nullptr), batches_(), is_eos_(false) { +-} ++ArrowIPCStreamBuffer::ArrowIPCStreamBuffer() ++ : schema_(nullptr), batches_(), is_eos_(false) {} + /// Decoded a schema +-arrow::Status ArrowIPCStreamBuffer::OnSchemaDecoded(std::shared_ptr s) { +- schema_ = s; +- return arrow::Status::OK(); ++arrow::Status ++ArrowIPCStreamBuffer::OnSchemaDecoded(std::shared_ptr s) { ++ schema_ = s; ++ return arrow::Status::OK(); + } + /// Decoded a record batch +-arrow::Status ArrowIPCStreamBuffer::OnRecordBatchDecoded(std::shared_ptr batch) { +- batches_.push_back(batch); +- return arrow::Status::OK(); ++arrow::Status ArrowIPCStreamBuffer::OnRecordBatchDecoded( ++ std::shared_ptr batch) { ++ batches_.push_back(batch); ++ return arrow::Status::OK(); + } + /// Reached end of stream + arrow::Status ArrowIPCStreamBuffer::OnEOS() { +- is_eos_ = true; +- return arrow::Status::OK(); ++ is_eos_ = true; ++ return arrow::Status::OK(); + } + + /// Constructor +-ArrowIPCStreamBufferReader::ArrowIPCStreamBufferReader(std::shared_ptr buffer) +- : buffer_(buffer), next_batch_id_(0) { +-} ++ArrowIPCStreamBufferReader::ArrowIPCStreamBufferReader( ++ std::shared_ptr buffer) ++ : buffer_(buffer), next_batch_id_(0) {} + + /// Get the schema + std::shared_ptr ArrowIPCStreamBufferReader::schema() const { +- return buffer_->schema(); ++ return buffer_->schema(); + } + /// Read the next record batch in the stream. Return null for batch when + /// reaching end of stream +-arrow::Status ArrowIPCStreamBufferReader::ReadNext(std::shared_ptr *batch) { +- if (next_batch_id_ >= buffer_->batches().size()) { +- *batch = nullptr; +- return arrow::Status::OK(); +- } +- *batch = buffer_->batches()[next_batch_id_++]; +- return arrow::Status::OK(); ++arrow::Status ArrowIPCStreamBufferReader::ReadNext( ++ std::shared_ptr *batch) { ++ if (next_batch_id_ >= buffer_->batches().size()) { ++ *batch = nullptr; ++ return arrow::Status::OK(); ++ } ++ *batch = buffer_->batches()[next_batch_id_++]; ++ return arrow::Status::OK(); + } + + /// Arrow array stream factory function + duckdb::unique_ptr +-ArrowIPCStreamBufferReader::CreateStream(uintptr_t buffer_ptr, ArrowStreamParameters ¶meters) { +- assert(buffer_ptr != 0); +- auto buffer = reinterpret_cast *>(buffer_ptr); +- auto reader = std::make_shared(*buffer); ++ArrowIPCStreamBufferReader::CreateStream(uintptr_t buffer_ptr, ++ ArrowStreamParameters ¶meters) { ++ assert(buffer_ptr != 0); ++ auto buffer = ++ reinterpret_cast *>(buffer_ptr); ++ auto reader = std::make_shared(*buffer); + +- // Create arrow stream +- auto stream_wrapper = duckdb::make_uniq(); +- stream_wrapper->arrow_array_stream.release = nullptr; +- auto maybe_ok = arrow::ExportRecordBatchReader(reader, &stream_wrapper->arrow_array_stream); +- if (!maybe_ok.ok()) { +- if (stream_wrapper->arrow_array_stream.release) { +- stream_wrapper->arrow_array_stream.release(&stream_wrapper->arrow_array_stream); +- } +- return nullptr; +- } ++ // Create arrow stream ++ auto stream_wrapper = duckdb::make_uniq(); ++ stream_wrapper->arrow_array_stream.release = nullptr; ++ auto maybe_ok = arrow::ExportRecordBatchReader( ++ reader, &stream_wrapper->arrow_array_stream); ++ if (!maybe_ok.ok()) { ++ if (stream_wrapper->arrow_array_stream.release) { ++ stream_wrapper->arrow_array_stream.release( ++ &stream_wrapper->arrow_array_stream); ++ } ++ return nullptr; ++ } + +- // Release the stream +- return stream_wrapper; ++ // Release the stream ++ return stream_wrapper; + } + +-void ArrowIPCStreamBufferReader::GetSchema(uintptr_t buffer_ptr, duckdb::ArrowSchemaWrapper &schema) { +- assert(buffer_ptr != 0); +- auto buffer = reinterpret_cast *>(buffer_ptr); +- auto reader = std::make_shared(*buffer); ++void ArrowIPCStreamBufferReader::GetSchema(uintptr_t buffer_ptr, ++ duckdb::ArrowSchemaWrapper &schema) { ++ assert(buffer_ptr != 0); ++ auto buffer = ++ reinterpret_cast *>(buffer_ptr); ++ auto reader = std::make_shared(*buffer); + +- // Create arrow stream +- auto stream_wrapper = duckdb::make_uniq(); +- stream_wrapper->arrow_array_stream.release = nullptr; +- auto maybe_ok = arrow::ExportRecordBatchReader(reader, &stream_wrapper->arrow_array_stream); +- if (!maybe_ok.ok()) { +- if (stream_wrapper->arrow_array_stream.release) { +- stream_wrapper->arrow_array_stream.release(&stream_wrapper->arrow_array_stream); +- } +- return; +- } ++ // Create arrow stream ++ auto stream_wrapper = duckdb::make_uniq(); ++ stream_wrapper->arrow_array_stream.release = nullptr; ++ auto maybe_ok = arrow::ExportRecordBatchReader( ++ reader, &stream_wrapper->arrow_array_stream); ++ if (!maybe_ok.ok()) { ++ if (stream_wrapper->arrow_array_stream.release) { ++ stream_wrapper->arrow_array_stream.release( ++ &stream_wrapper->arrow_array_stream); ++ } ++ return; ++ } + +- // Pass ownership to caller +- stream_wrapper->arrow_array_stream.get_schema(&stream_wrapper->arrow_array_stream, &schema.arrow_schema); ++ // Pass ownership to caller ++ stream_wrapper->arrow_array_stream.get_schema( ++ &stream_wrapper->arrow_array_stream, &schema.arrow_schema); + } + + /// Constructor +-BufferingArrowIPCStreamDecoder::BufferingArrowIPCStreamDecoder(std::shared_ptr buffer) +- : arrow::ipc::StreamDecoder(buffer), buffer_(buffer) { +-} ++BufferingArrowIPCStreamDecoder::BufferingArrowIPCStreamDecoder( ++ std::shared_ptr buffer) ++ : arrow::ipc::StreamDecoder(buffer), buffer_(buffer) {} + + } // namespace duckdb +diff --git a/src/arrow_to_ipc.cpp b/src/arrow_to_ipc.cpp +index e282612..c316d85 100644 +--- a/src/arrow_to_ipc.cpp ++++ b/src/arrow_to_ipc.cpp +@@ -15,6 +15,8 @@ + #include "arrow/type_fwd.h" + #include "arrow/c/bridge.h" + ++#include ++ + #include "duckdb.hpp" + #ifndef DUCKDB_AMALGAMATION + #include "duckdb/common/arrow/result_arrow_wrapper.hpp" +@@ -28,165 +30,180 @@ + namespace duckdb { + + struct ToArrowIpcFunctionData : public TableFunctionData { +- ToArrowIpcFunctionData() { +- } +- shared_ptr schema; +- idx_t chunk_size; ++ ToArrowIpcFunctionData() {} ++ std::shared_ptr schema; ++ idx_t chunk_size; + }; + + struct ToArrowIpcGlobalState : public GlobalTableFunctionState { +- ToArrowIpcGlobalState() : sent_schema(false) { +- } +- atomic sent_schema; +- mutex lock; ++ ToArrowIpcGlobalState() : sent_schema(false) {} ++ atomic sent_schema; ++ mutex lock; + }; + + struct ToArrowIpcLocalState : public LocalTableFunctionState { +- unique_ptr appender; +- idx_t current_count = 0; +- bool checked_schema = false; ++ unique_ptr appender; ++ idx_t current_count = 0; ++ bool checked_schema = false; + }; + +- +-unique_ptr ToArrowIPCFunction::InitLocal(ExecutionContext &context, TableFunctionInitInput &input, +- GlobalTableFunctionState *global_state) { +- return make_uniq(); ++unique_ptr ++ToArrowIPCFunction::InitLocal(ExecutionContext &context, ++ TableFunctionInitInput &input, ++ GlobalTableFunctionState *global_state) { ++ return make_uniq(); + } + +-unique_ptr ToArrowIPCFunction::InitGlobal(ClientContext &context, +- TableFunctionInitInput &input) { +- return make_uniq(); ++unique_ptr ++ToArrowIPCFunction::InitGlobal(ClientContext &context, ++ TableFunctionInitInput &input) { ++ return make_uniq(); + } + +-unique_ptr ToArrowIPCFunction::Bind(ClientContext &context, TableFunctionBindInput &input, +- vector &return_types, vector &names) { +- auto result = make_uniq(); ++unique_ptr ++ToArrowIPCFunction::Bind(ClientContext &context, TableFunctionBindInput &input, ++ vector &return_types, ++ vector &names) { ++ auto result = make_uniq(); + +- result->chunk_size = DEFAULT_CHUNK_SIZE * STANDARD_VECTOR_SIZE; ++ result->chunk_size = DEFAULT_CHUNK_SIZE * STANDARD_VECTOR_SIZE; + +- // Set return schema +- return_types.emplace_back(LogicalType::BLOB); +- names.emplace_back("ipc"); +- return_types.emplace_back(LogicalType::BOOLEAN); +- names.emplace_back("header"); ++ // Set return schema ++ return_types.emplace_back(LogicalType::BLOB); ++ names.emplace_back("ipc"); ++ return_types.emplace_back(LogicalType::BOOLEAN); ++ names.emplace_back("header"); + +- // Create the Arrow schema +- ArrowSchema schema; +- ArrowConverter::ToArrowSchema(&schema, input.input_table_types, input.input_table_names, context.GetClientProperties()); +- result->schema = arrow::ImportSchema(&schema).ValueOrDie(); ++ // Create the Arrow schema ++ ArrowSchema schema; ++ ArrowConverter::ToArrowSchema(&schema, input.input_table_types, ++ input.input_table_names, ++ context.GetClientProperties()); ++ result->schema = arrow::ImportSchema(&schema).ValueOrDie(); + +- return std::move(result); ++ return std::move(result); + } + +-OperatorResultType ToArrowIPCFunction::Function(ExecutionContext &context, TableFunctionInput &data_p, DataChunk &input, +- DataChunk &output) { +- std::shared_ptr arrow_serialized_ipc_buffer; +- auto &data = (ToArrowIpcFunctionData &)*data_p.bind_data; +- auto &local_state = (ToArrowIpcLocalState &)*data_p.local_state; +- auto &global_state = (ToArrowIpcGlobalState &)*data_p.global_state; +- +- bool sending_schema = false; +- +- bool caching_disabled = !PhysicalOperator::OperatorCachingAllowed(context); +- +- if (!local_state.checked_schema) { +- if (!global_state.sent_schema) { +- lock_guard init_lock(global_state.lock); +- if (!global_state.sent_schema) { +- // This run will send the schema, other threads can just send the buffers +- global_state.sent_schema = true; +- sending_schema = true; +- } +- } +- local_state.checked_schema = true; ++OperatorResultType ToArrowIPCFunction::Function(ExecutionContext &context, ++ TableFunctionInput &data_p, ++ DataChunk &input, ++ DataChunk &output) { ++ std::shared_ptr arrow_serialized_ipc_buffer; ++ auto &data = (ToArrowIpcFunctionData &)*data_p.bind_data; ++ auto &local_state = (ToArrowIpcLocalState &)*data_p.local_state; ++ auto &global_state = (ToArrowIpcGlobalState &)*data_p.global_state; ++ ++ bool sending_schema = false; ++ ++ bool caching_disabled = !PhysicalOperator::OperatorCachingAllowed(context); ++ ++ if (!local_state.checked_schema) { ++ if (!global_state.sent_schema) { ++ lock_guard init_lock(global_state.lock); ++ if (!global_state.sent_schema) { ++ // This run will send the schema, other threads can just send the ++ // buffers ++ global_state.sent_schema = true; ++ sending_schema = true; ++ } ++ } ++ local_state.checked_schema = true; ++ } ++ ++ if (sending_schema) { ++ auto result = arrow::ipc::SerializeSchema(*data.schema); ++ arrow_serialized_ipc_buffer = result.ValueOrDie(); ++ output.data[1].SetValue(0, Value::BOOLEAN(1)); ++ } else { ++ if (!local_state.appender) { ++ local_state.appender = ++ make_uniq(input.GetTypes(), data.chunk_size, ++ context.client.GetClientProperties()); + } + +- if (sending_schema) { +- auto result = arrow::ipc::SerializeSchema(*data.schema); +- arrow_serialized_ipc_buffer = result.ValueOrDie(); +- output.data[1].SetValue(0, Value::BOOLEAN(1)); ++ // Append input chunk ++ local_state.appender->Append(input, 0, input.size(), input.size()); ++ local_state.current_count += input.size(); ++ ++ // If chunk size is reached, we can flush to IPC blob ++ if (caching_disabled || local_state.current_count >= data.chunk_size) { ++ // Construct record batch from DataChunk ++ ArrowArray arr = local_state.appender->Finalize(); ++ auto record_batch = ++ arrow::ImportRecordBatch(&arr, data.schema).ValueOrDie(); ++ ++ // Serialize recordbatch ++ auto options = arrow::ipc::IpcWriteOptions::Defaults(); ++ auto result = arrow::ipc::SerializeRecordBatch(*record_batch, options); ++ arrow_serialized_ipc_buffer = result.ValueOrDie(); ++ ++ // Reset appender ++ local_state.appender.reset(); ++ local_state.current_count = 0; ++ ++ output.data[1].SetValue(0, Value::BOOLEAN(0)); + } else { +- if (!local_state.appender) { +- local_state.appender = make_uniq(input.GetTypes(), data.chunk_size, context.client.GetClientProperties()); +- } +- +- // Append input chunk +- local_state.appender->Append(input, 0, input.size(), input.size()); +- local_state.current_count += input.size(); +- +- // If chunk size is reached, we can flush to IPC blob +- if (caching_disabled || local_state.current_count >= data.chunk_size) { +- // Construct record batch from DataChunk +- ArrowArray arr = local_state.appender->Finalize(); +- auto record_batch = arrow::ImportRecordBatch(&arr, data.schema).ValueOrDie(); +- +- // Serialize recordbatch +- auto options = arrow::ipc::IpcWriteOptions::Defaults(); +- auto result = arrow::ipc::SerializeRecordBatch(*record_batch, options); +- arrow_serialized_ipc_buffer = result.ValueOrDie(); +- +- // Reset appender +- local_state.appender.reset(); +- local_state.current_count = 0; +- +- output.data[1].SetValue(0, Value::BOOLEAN(0)); +- } else { +- return OperatorResultType::NEED_MORE_INPUT; +- } ++ return OperatorResultType::NEED_MORE_INPUT; + } ++ } ++ ++ // TODO clean up ++ auto wrapped_buffer = ++ make_buffer(arrow_serialized_ipc_buffer); ++ auto &vector = output.data[0]; ++ StringVector::AddBuffer(vector, wrapped_buffer); ++ auto data_ptr = (string_t *)vector.GetData(); ++ *data_ptr = string_t((const char *)arrow_serialized_ipc_buffer->data(), ++ arrow_serialized_ipc_buffer->size()); ++ output.SetCardinality(1); ++ ++ if (sending_schema) { ++ return OperatorResultType::HAVE_MORE_OUTPUT; ++ } else { ++ return OperatorResultType::NEED_MORE_INPUT; ++ } ++} + +- // TODO clean up +- auto wrapped_buffer = make_buffer(arrow_serialized_ipc_buffer); ++OperatorFinalizeResultType ToArrowIPCFunction::FunctionFinal( ++ ExecutionContext &context, TableFunctionInput &data_p, DataChunk &output) { ++ auto &data = (ToArrowIpcFunctionData &)*data_p.bind_data; ++ auto &local_state = (ToArrowIpcLocalState &)*data_p.local_state; ++ std::shared_ptr arrow_serialized_ipc_buffer; ++ ++ // TODO clean up ++ if (local_state.appender) { ++ ArrowArray arr = local_state.appender->Finalize(); ++ auto record_batch = ++ arrow::ImportRecordBatch(&arr, data.schema).ValueOrDie(); ++ ++ // Serialize recordbatch ++ auto options = arrow::ipc::IpcWriteOptions::Defaults(); ++ auto result = arrow::ipc::SerializeRecordBatch(*record_batch, options); ++ arrow_serialized_ipc_buffer = result.ValueOrDie(); ++ ++ auto wrapped_buffer = ++ make_buffer(arrow_serialized_ipc_buffer); + auto &vector = output.data[0]; + StringVector::AddBuffer(vector, wrapped_buffer); + auto data_ptr = (string_t *)vector.GetData(); +- *data_ptr = string_t((const char *)arrow_serialized_ipc_buffer->data(), arrow_serialized_ipc_buffer->size()); ++ *data_ptr = string_t((const char *)arrow_serialized_ipc_buffer->data(), ++ arrow_serialized_ipc_buffer->size()); + output.SetCardinality(1); ++ local_state.appender.reset(); ++ output.data[1].SetValue(0, Value::BOOLEAN(0)); ++ } + +- if (sending_schema) { +- return OperatorResultType::HAVE_MORE_OUTPUT; +- } else { +- return OperatorResultType::NEED_MORE_INPUT; +- } +-} +- +-OperatorFinalizeResultType ToArrowIPCFunction::FunctionFinal(ExecutionContext &context, TableFunctionInput &data_p, +- DataChunk &output) { +- auto &data = (ToArrowIpcFunctionData &)*data_p.bind_data; +- auto &local_state = (ToArrowIpcLocalState &)*data_p.local_state; +- std::shared_ptr arrow_serialized_ipc_buffer; +- +- // TODO clean up +- if (local_state.appender) { +- ArrowArray arr = local_state.appender->Finalize(); +- auto record_batch = arrow::ImportRecordBatch(&arr, data.schema).ValueOrDie(); +- +- // Serialize recordbatch +- auto options = arrow::ipc::IpcWriteOptions::Defaults(); +- auto result = arrow::ipc::SerializeRecordBatch(*record_batch, options); +- arrow_serialized_ipc_buffer = result.ValueOrDie(); +- +- auto wrapped_buffer = make_buffer(arrow_serialized_ipc_buffer); +- auto &vector = output.data[0]; +- StringVector::AddBuffer(vector, wrapped_buffer); +- auto data_ptr = (string_t *)vector.GetData(); +- *data_ptr = string_t((const char *)arrow_serialized_ipc_buffer->data(), arrow_serialized_ipc_buffer->size()); +- output.SetCardinality(1); +- local_state.appender.reset(); +- output.data[1].SetValue(0, Value::BOOLEAN(0)); +- } +- +- return OperatorFinalizeResultType::FINISHED; ++ return OperatorFinalizeResultType::FINISHED; + } + +- + TableFunction ToArrowIPCFunction::GetFunction() { +- TableFunction fun("to_arrow_ipc", {LogicalType::TABLE}, nullptr, ToArrowIPCFunction::Bind, +- ToArrowIPCFunction::InitGlobal,ToArrowIPCFunction::InitLocal); +- fun.in_out_function = ToArrowIPCFunction::Function; +- fun.in_out_function_final = ToArrowIPCFunction::FunctionFinal; ++ TableFunction fun("to_arrow_ipc", {LogicalType::TABLE}, nullptr, ++ ToArrowIPCFunction::Bind, ToArrowIPCFunction::InitGlobal, ++ ToArrowIPCFunction::InitLocal); ++ fun.in_out_function = ToArrowIPCFunction::Function; ++ fun.in_out_function_final = ToArrowIPCFunction::FunctionFinal; + +- return fun; ++ return fun; + } + + } // namespace duckdb +\ No newline at end of file +diff --git a/src/include/arrow_extension.hpp b/src/include/arrow_extension.hpp +index 8ad174e..7d600d1 100644 +--- a/src/include/arrow_extension.hpp ++++ b/src/include/arrow_extension.hpp +@@ -6,8 +6,8 @@ namespace duckdb { + + class ArrowExtension : public Extension { + public: +- void Load(DuckDB &db) override; +- std::string Name() override; ++ void Load(DuckDB &db) override; ++ std::string Name() override; + }; + + } // namespace duckdb +diff --git a/src/include/arrow_scan_ipc.hpp b/src/include/arrow_scan_ipc.hpp +index 4ec1b9f..66a7827 100644 +--- a/src/include/arrow_scan_ipc.hpp ++++ b/src/include/arrow_scan_ipc.hpp +@@ -9,20 +9,23 @@ namespace duckdb { + + struct ArrowIPCScanFunctionData : public ArrowScanFunctionData { + public: +- using ArrowScanFunctionData::ArrowScanFunctionData; +- unique_ptr stream_decoder = nullptr; ++ using ArrowScanFunctionData::ArrowScanFunctionData; ++ unique_ptr stream_decoder = nullptr; + }; + +-// IPC Table scan is identical to ArrowTableFunction arrow scan except instead of CDataInterface header pointers, it +-// takes a bunch of pointers pointing to buffers containing data in Arrow IPC format ++// IPC Table scan is identical to ArrowTableFunction arrow scan except instead ++// of CDataInterface header pointers, it takes a bunch of pointers pointing to ++// buffers containing data in Arrow IPC format + struct ArrowIPCTableFunction : public ArrowTableFunction { + public: +- static TableFunction GetFunction(); ++ static TableFunction GetFunction(); + + private: +- static unique_ptr ArrowScanBind(ClientContext &context, TableFunctionBindInput &input, +- vector &return_types, vector &names); +- static void ArrowScanFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output); ++ static unique_ptr ++ ArrowScanBind(ClientContext &context, TableFunctionBindInput &input, ++ vector &return_types, vector &names); ++ static void ArrowScanFunction(ClientContext &context, ++ TableFunctionInput &data_p, DataChunk &output); + }; + + } // namespace duckdb +diff --git a/src/include/arrow_stream_buffer.hpp b/src/include/arrow_stream_buffer.hpp +index a4cbe97..e486c72 100644 +--- a/src/include/arrow_stream_buffer.hpp ++++ b/src/include/arrow_stream_buffer.hpp +@@ -14,6 +14,7 @@ + #include + #include + #include ++#include + + /// File copied from + /// https://github.com/duckdb/duckdb-wasm/blob/0ad10e7db4ef4025f5f4120be37addc4ebe29618/lib/include/duckdb/web/arrow_stream_buffer.h +@@ -21,76 +22,72 @@ namespace duckdb { + + struct ArrowIPCStreamBuffer : public arrow::ipc::Listener { + protected: +- /// The schema +- std::shared_ptr schema_; +- /// The batches +- std::vector> batches_; +- /// Is eos? +- bool is_eos_; ++ /// The schema ++ std::shared_ptr schema_; ++ /// The batches ++ std::vector> batches_; ++ /// Is eos? ++ bool is_eos_; + +- /// Decoded a record batch +- arrow::Status OnSchemaDecoded(std::shared_ptr schema); +- /// Decoded a record batch +- arrow::Status OnRecordBatchDecoded(std::shared_ptr record_batch); +- /// Reached end of stream +- arrow::Status OnEOS(); ++ /// Decoded a record batch ++ arrow::Status OnSchemaDecoded(std::shared_ptr schema); ++ /// Decoded a record batch ++ arrow::Status ++ OnRecordBatchDecoded(std::shared_ptr record_batch); ++ /// Reached end of stream ++ arrow::Status OnEOS(); + + public: +- /// Constructor +- ArrowIPCStreamBuffer(); ++ /// Constructor ++ ArrowIPCStreamBuffer(); + +- /// Is end of stream? +- bool is_eos() const { +- return is_eos_; +- } +- /// Return the schema +- std::shared_ptr &schema() { +- return schema_; +- } +- /// Return the batches +- std::vector> &batches() { +- return batches_; +- } ++ /// Is end of stream? ++ bool is_eos() const { return is_eos_; } ++ /// Return the schema ++ std::shared_ptr &schema() { return schema_; } ++ /// Return the batches ++ std::vector> &batches() { ++ return batches_; ++ } + }; + + struct ArrowIPCStreamBufferReader : public arrow::RecordBatchReader { + protected: +- /// The buffer +- std::shared_ptr buffer_; +- /// The batch index +- size_t next_batch_id_; ++ /// The buffer ++ std::shared_ptr buffer_; ++ /// The batch index ++ size_t next_batch_id_; + + public: +- /// Constructor +- ArrowIPCStreamBufferReader(std::shared_ptr buffer); +- /// Destructor +- ~ArrowIPCStreamBufferReader() = default; ++ /// Constructor ++ ArrowIPCStreamBufferReader(std::shared_ptr buffer); ++ /// Destructor ++ ~ArrowIPCStreamBufferReader() = default; + +- /// Get the schema +- std::shared_ptr schema() const override; +- /// Read the next record batch in the stream. Return null for batch when reaching end of stream +- arrow::Status ReadNext(std::shared_ptr *batch) override; ++ /// Get the schema ++ std::shared_ptr schema() const override; ++ /// Read the next record batch in the stream. Return null for batch when ++ /// reaching end of stream ++ arrow::Status ReadNext(std::shared_ptr *batch) override; + +- /// Create arrow array stream wrapper +- static duckdb::unique_ptr CreateStream(uintptr_t buffer_ptr, +- ArrowStreamParameters ¶meters); +- /// Create arrow array stream wrapper +- static void GetSchema(uintptr_t buffer_ptr, ArrowSchemaWrapper &schema); ++ /// Create arrow array stream wrapper ++ static duckdb::unique_ptr ++ CreateStream(uintptr_t buffer_ptr, ArrowStreamParameters ¶meters); ++ /// Create arrow array stream wrapper ++ static void GetSchema(uintptr_t buffer_ptr, ArrowSchemaWrapper &schema); + }; + + struct BufferingArrowIPCStreamDecoder : public arrow::ipc::StreamDecoder { + protected: +- /// The buffer +- std::shared_ptr buffer_; ++ /// The buffer ++ std::shared_ptr buffer_; + + public: +- /// Constructor +- BufferingArrowIPCStreamDecoder( +- std::shared_ptr buffer = std::make_shared()); +- /// Get the buffer +- std::shared_ptr &buffer() { +- return buffer_; +- } ++ /// Constructor ++ BufferingArrowIPCStreamDecoder(std::shared_ptr buffer = ++ std::make_shared()); ++ /// Get the buffer ++ std::shared_ptr &buffer() { return buffer_; } + }; + + } // namespace duckdb +diff --git a/src/include/arrow_to_ipc.hpp b/src/include/arrow_to_ipc.hpp +index b4eb9d4..6c8995a 100644 +--- a/src/include/arrow_to_ipc.hpp ++++ b/src/include/arrow_to_ipc.hpp +@@ -3,36 +3,42 @@ + #include "arrow/buffer.h" + #include "duckdb.hpp" + ++#include ++ + namespace duckdb { + + class ArrowStringVectorBuffer : public VectorBuffer { + public: +- explicit ArrowStringVectorBuffer(std::shared_ptr buffer_p) +- : VectorBuffer(VectorBufferType::OPAQUE_BUFFER), buffer(std::move(buffer_p)) { +- } ++ explicit ArrowStringVectorBuffer(std::shared_ptr buffer_p) ++ : VectorBuffer(VectorBufferType::OPAQUE_BUFFER), ++ buffer(std::move(buffer_p)) {} + + private: +- std::shared_ptr buffer; ++ std::shared_ptr buffer; + }; + +- + class ToArrowIPCFunction { + public: +- //! note: this is the number of vectors per chunk +- static constexpr idx_t DEFAULT_CHUNK_SIZE = 120; ++ //! note: this is the number of vectors per chunk ++ static constexpr idx_t DEFAULT_CHUNK_SIZE = 120; + +- static TableFunction GetFunction(); ++ static TableFunction GetFunction(); + + private: +- static unique_ptr InitLocal(ExecutionContext &context, TableFunctionInitInput &input, +- GlobalTableFunctionState *global_state); +- static unique_ptr InitGlobal(ClientContext &context, +- TableFunctionInitInput &input); +- static unique_ptr Bind(ClientContext &context, TableFunctionBindInput &input, +- vector &return_types, vector &names); +- static OperatorResultType Function(ExecutionContext &context, TableFunctionInput &data_p, DataChunk &input, +- DataChunk &output); +- static OperatorFinalizeResultType FunctionFinal(ExecutionContext &context, TableFunctionInput &data_p, +- DataChunk &output); ++ static unique_ptr ++ InitLocal(ExecutionContext &context, TableFunctionInitInput &input, ++ GlobalTableFunctionState *global_state); ++ static unique_ptr ++ InitGlobal(ClientContext &context, TableFunctionInitInput &input); ++ static unique_ptr Bind(ClientContext &context, ++ TableFunctionBindInput &input, ++ vector &return_types, ++ vector &names); ++ static OperatorResultType Function(ExecutionContext &context, ++ TableFunctionInput &data_p, ++ DataChunk &input, DataChunk &output); ++ static OperatorFinalizeResultType FunctionFinal(ExecutionContext &context, ++ TableFunctionInput &data_p, ++ DataChunk &output); + }; // namespace duckdb +-} ++} // namespace duckdb From 9940483c695a0e1144d200149108d6d3539a9174 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Fri, 12 Apr 2024 12:59:53 +0200 Subject: [PATCH 385/603] Improved simplification for subqueries and set operations --- .../sqlsmith/include/statement_simplifier.hpp | 2 +- extension/sqlsmith/statement_simplifier.cpp | 48 +++++++++---------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/extension/sqlsmith/include/statement_simplifier.hpp b/extension/sqlsmith/include/statement_simplifier.hpp index 25f4848a849f..dfe9210f3831 100644 --- a/extension/sqlsmith/include/statement_simplifier.hpp +++ b/extension/sqlsmith/include/statement_simplifier.hpp @@ -61,7 +61,7 @@ class StatementSimplifier { void Simplify(SelectNode &node); void Simplify(SetOperationNode &node); - void Simplify(QueryNode &node); + void Simplify(unique_ptr &node); void Simplify(ResultModifier &modifier); void Simplify(OrderModifier &modifier); diff --git a/extension/sqlsmith/statement_simplifier.cpp b/extension/sqlsmith/statement_simplifier.cpp index 0395e594a6da..112b95365382 100644 --- a/extension/sqlsmith/statement_simplifier.cpp +++ b/extension/sqlsmith/statement_simplifier.cpp @@ -98,7 +98,11 @@ void StatementSimplifier::Simplify(unique_ptr &ref) { switch (ref->type) { case TableReferenceType::SUBQUERY: { auto &subquery = ref->Cast(); - Simplify(*subquery.subquery->node); + if (subquery.subquery->node->type == QueryNodeType::SELECT_NODE) { + auto &select_node = subquery.subquery->node->Cast(); + SimplifyReplace(ref, select_node.from_table); + } + Simplify(subquery.subquery->node); break; } case TableReferenceType::JOIN: { @@ -141,45 +145,41 @@ void StatementSimplifier::Simplify(SelectNode &node) { } void StatementSimplifier::Simplify(SetOperationNode &node) { - Simplify(*node.left); - Simplify(*node.right); + Simplify(node.left); + Simplify(node.right); } void StatementSimplifier::Simplify(CommonTableExpressionMap &cte) { // remove individual CTEs - vector cte_keys; - for (auto &kv : cte.map) { - cte_keys.push_back(kv.first); - } - for (idx_t i = 0; i < cte_keys.size(); i++) { - auto n = std::move(cte.map[cte_keys[i]]); - cte.map.erase(cte_keys[i]); - Simplification(); - cte.map[cte_keys[i]] = std::move(n); - + SimplifyMap(cte.map); + for (auto &cte_child : cte.map) { // simplify individual ctes - Simplify(*cte.map[cte_keys[i]]->query->node); + Simplify(cte_child.second->query->node); } } -void StatementSimplifier::Simplify(QueryNode &node) { - Simplify(node.cte_map); - switch (node.type) { +void StatementSimplifier::Simplify(unique_ptr &node) { + Simplify(node->cte_map); + switch (node->type) { case QueryNodeType::SELECT_NODE: - Simplify(node.Cast()); + Simplify(node->Cast()); break; - case QueryNodeType::SET_OPERATION_NODE: - Simplify(node.Cast()); + case QueryNodeType::SET_OPERATION_NODE: { + auto &setop = node->Cast(); + SimplifyReplace(node, setop.left); + SimplifyReplace(node, setop.right); + Simplify(setop); break; + } case QueryNodeType::RECURSIVE_CTE_NODE: case QueryNodeType::CTE_NODE: default: break; } - for (auto &modifier : node.modifiers) { + for (auto &modifier : node->modifiers) { Simplify(*modifier); } - SimplifyList(node.modifiers); + SimplifyList(node->modifiers); } void StatementSimplifier::SimplifyExpressionList(duckdb::unique_ptr &expr, @@ -265,7 +265,7 @@ void StatementSimplifier::SimplifyExpression(duckdb::unique_ptrCast(); SimplifyChildExpression(expr, subq.child); - Simplify(*subq.subquery->node); + Simplify(subq.subquery->node); break; } case ExpressionClass::COMPARISON: { @@ -318,7 +318,7 @@ void StatementSimplifier::Simplify(OrderModifier &modifier) { } void StatementSimplifier::Simplify(SelectStatement &stmt) { - Simplify(*stmt.node); + Simplify(stmt.node); ParsedExpressionIterator::EnumerateQueryNodeChildren( *stmt.node, [&](duckdb::unique_ptr &child) { SimplifyExpression(child); }); } From 75fcbbf13e897b257cf41587e062c76aa2ce6cd4 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Fri, 12 Apr 2024 13:18:48 +0200 Subject: [PATCH 386/603] Further extend query simplification --- .../sqlsmith/include/statement_simplifier.hpp | 9 +++ extension/sqlsmith/statement_simplifier.cpp | 64 ++++++++++++++++++- 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/extension/sqlsmith/include/statement_simplifier.hpp b/extension/sqlsmith/include/statement_simplifier.hpp index dfe9210f3831..bd1133c28d0f 100644 --- a/extension/sqlsmith/include/statement_simplifier.hpp +++ b/extension/sqlsmith/include/statement_simplifier.hpp @@ -25,6 +25,7 @@ class ParsedExpression; class ResultModifier; class OrderModifier; class UpdateSetInfo; +class GroupByNode; class StatementSimplifier { public: @@ -56,6 +57,10 @@ class StatementSimplifier { template void SimplifyOptional(duckdb::unique_ptr &opt); + template + void SimplifyAlias(T &input); + template + void SimplifyEnum(T &enum_ref, T default_value); void Simplify(unique_ptr &ref); @@ -73,8 +78,12 @@ class StatementSimplifier { vector> &expression_list); void SimplifyExpressionList(vector> &expression_list, bool is_optional = true); void Simplify(CommonTableExpressionMap &cte_map); + void Simplify(GroupByNode &groups); void Simplify(UpdateSetInfo &info); + +private: + vector>> query_nodes; }; } // namespace duckdb diff --git a/extension/sqlsmith/statement_simplifier.cpp b/extension/sqlsmith/statement_simplifier.cpp index 112b95365382..7a233c0adb15 100644 --- a/extension/sqlsmith/statement_simplifier.cpp +++ b/extension/sqlsmith/statement_simplifier.cpp @@ -94,10 +94,31 @@ void StatementSimplifier::SimplifyOptional(duckdb::unique_ptr &opt) { opt = std::move(n); } +template +void StatementSimplifier::SimplifyEnum(T &enum_ref, T default_value) { + if (enum_ref == default_value) { + return; + } + auto current = enum_ref; + enum_ref = default_value; + Simplification(); + enum_ref = current; +} + +template +void StatementSimplifier::SimplifyAlias(T &input) { + auto alias = std::move(input.alias); + auto column_name_alias = std::move(input.column_name_alias); + Simplification(); + input.alias = std::move(alias); + input.column_name_alias = std::move(column_name_alias); +} + void StatementSimplifier::Simplify(unique_ptr &ref) { switch (ref->type) { case TableReferenceType::SUBQUERY: { auto &subquery = ref->Cast(); + SimplifyAlias(subquery); if (subquery.subquery->node->type == QueryNodeType::SELECT_NODE) { auto &select_node = subquery.subquery->node->Cast(); SimplifyReplace(ref, select_node.from_table); @@ -109,7 +130,7 @@ void StatementSimplifier::Simplify(unique_ptr &ref) { auto &cp = ref->Cast(); Simplify(cp.left); Simplify(cp.right); - SimplifyOptional(cp.condition); + SimplifyOptionalExpression(cp.condition); SimplifyReplace(ref, cp.left); SimplifyReplace(ref, cp.right); break; @@ -123,23 +144,51 @@ void StatementSimplifier::Simplify(unique_ptr &ref) { } break; } + case TableReferenceType::TABLE_FUNCTION: { + auto &table_function = ref->Cast(); + // try to remove aliases + SimplifyAlias(table_function); + break; + } + case TableReferenceType::BASE_TABLE: { + auto &table_ref = ref->Cast(); + SimplifyAlias(table_ref); + break; + } default: break; } } +void StatementSimplifier::Simplify(GroupByNode &groups) { + // try to remove all groups + auto group_expr = std::move(groups.group_expressions); + auto group_sets = std::move(groups.grouping_sets); + Simplification(); + groups.group_expressions = std::move(group_expr); + groups.grouping_sets = std::move(group_sets); + + // try to remove grouping sets + SimplifyList(groups.grouping_sets, false); + // simplify expressions + for(auto &group : groups.group_expressions) { + SimplifyExpression(group); + } +} + void StatementSimplifier::Simplify(SelectNode &node) { // simplify projection list SimplifyExpressionList(node.select_list, false); // from clause SimplifyOptional(node.from_table); // simplify groups - SimplifyList(node.groups.grouping_sets); + Simplify(node.groups); // simplify filters SimplifyOptionalExpression(node.where_clause); SimplifyOptionalExpression(node.having); SimplifyOptionalExpression(node.qualify); SimplifyOptional(node.sample); + SimplifyEnum(node.aggregate_handling, AggregateHandling::STANDARD_HANDLING); Simplify(node.from_table); } @@ -159,6 +208,7 @@ void StatementSimplifier::Simplify(CommonTableExpressionMap &cte) { } void StatementSimplifier::Simplify(unique_ptr &node) { + query_nodes.push_back(node); Simplify(node->cte_map); switch (node->type) { case QueryNodeType::SELECT_NODE: @@ -180,6 +230,7 @@ void StatementSimplifier::Simplify(unique_ptr &node) { Simplify(*modifier); } SimplifyList(node->modifiers); + query_nodes.pop_back(); } void StatementSimplifier::SimplifyExpressionList(duckdb::unique_ptr &expr, @@ -264,6 +315,10 @@ void StatementSimplifier::SimplifyExpression(duckdb::unique_ptrCast(); + // try to move this subquery fully into the outer query + if (!query_nodes.empty()) { + SimplifyReplace(query_nodes.back().get(), subq.subquery->node); + } SimplifyChildExpression(expr, subq.child); Simplify(subq.subquery->node); break; @@ -293,6 +348,11 @@ void StatementSimplifier::SimplifyExpression(duckdb::unique_ptr Date: Fri, 12 Apr 2024 13:19:08 +0200 Subject: [PATCH 387/603] create an assertion out of this, TemporaryDirectoryHandle should never be made if the path is empty --- src/storage/temporary_file_manager.cpp | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/storage/temporary_file_manager.cpp b/src/storage/temporary_file_manager.cpp index aa17eaaea0a0..d90641bf4ad7 100644 --- a/src/storage/temporary_file_manager.cpp +++ b/src/storage/temporary_file_manager.cpp @@ -190,11 +190,10 @@ idx_t TemporaryFileHandle::GetPositionInFile(idx_t index) { TemporaryDirectoryHandle::TemporaryDirectoryHandle(DatabaseInstance &db, string path_p, optional_idx max_swap_space) : db(db), temp_directory(std::move(path_p)), temp_file(make_uniq(db, temp_directory)) { auto &fs = FileSystem::GetFileSystem(db); - if (!temp_directory.empty()) { - if (!fs.DirectoryExists(temp_directory)) { - fs.CreateDirectory(temp_directory); - created_directory = true; - } + D_ASSERT(!temp_directory.empty()); + if (!fs.DirectoryExists(temp_directory)) { + fs.CreateDirectory(temp_directory); + created_directory = true; } temp_file->SetMaxSwapSpace(max_swap_space); } @@ -253,14 +252,13 @@ bool TemporaryFileIndex::IsValid() const { //===--------------------------------------------------------------------===// static idx_t GetDefaultMax(const string &path) { - // Use the available disk space + D_ASSERT(!path.empty()); auto disk_space = FileSystem::GetAvailableDiskSpace(path); - idx_t default_value = 0; - if (disk_space.IsValid()) { - // Only use 90% of the available disk space - default_value = static_cast(static_cast(disk_space.GetIndex()) * 0.9); - } - return default_value; + // Use the available disk space + // We have made sure that the file exists before we call this, it shouldn't fail + D_ASSERT(disk_space.IsValid()); + // Only use 90% of the available disk space + return static_cast(static_cast(disk_space.GetIndex()) * 0.9); } TemporaryFileManager::TemporaryFileManager(DatabaseInstance &db, const string &temp_directory_p) From 3cf9d06af8dec6f64ca124e84c10465284065566 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 12 Apr 2024 13:22:54 +0200 Subject: [PATCH 388/603] Alright this is a bit hacky but annoying --- .../buffer_manager/csv_buffer_manager.cpp | 12 ++++++ test/sql/copy/csv/recursive_query_csv.test | 40 +++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 test/sql/copy/csv/recursive_query_csv.test diff --git a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp index 66a6e6ab3cec..f8efd8347161 100644 --- a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp +++ b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp @@ -63,6 +63,12 @@ bool CSVBufferManager::ReadNextAndCacheIt() { shared_ptr CSVBufferManager::GetBuffer(const idx_t pos) { lock_guard parallel_lock(main_mutex); +// if (pos == 0 && done && cached_buffers.empty()){ +// // This is a recursive CTE, we have to reset out whole buffer +// done = false; +// file_handle->Reset(); +// Initialize(); +// } while (pos >= cached_buffers.size()) { if (done) { return nullptr; @@ -98,6 +104,12 @@ void CSVBufferManager::ResetBuffer(const idx_t buffer_idx) { } // We only reset if previous one was also already reset if (buffer_idx > 0 && !cached_buffers[buffer_idx - 1]) { +// if (cached_buffers[buffer_idx]->last_buffer){ +// // We clear the whole shebang +// cached_buffers.clear(); +// reset_when_possible.clear(); +// return; +// } cached_buffers[buffer_idx].reset(); idx_t cur_buffer = buffer_idx + 1; while (reset_when_possible.find(cur_buffer) != reset_when_possible.end()) { diff --git a/test/sql/copy/csv/recursive_query_csv.test b/test/sql/copy/csv/recursive_query_csv.test new file mode 100644 index 000000000000..520e12362ba1 --- /dev/null +++ b/test/sql/copy/csv/recursive_query_csv.test @@ -0,0 +1,40 @@ +# name: test/sql/copy/csv/recursive_query_csv.test +# description: Test read CSV function in a recursive +# group: [csv] + +query I +with recursive + base as + ( select * + from '/Users/holanda/Desktop/Real_Estate_Sales_2001-2021_GL.csv' + where '2003-01-01' < "date recorded" and "date recorded" < '2010-01-01' and "sale amount" > 1000000 + ) + , chains as + ( + select + town + , "date recorded" as begTS + , "date recorded" as endTS + , [struct_pack(date:= "date recorded", amt:="sale amount", type:="property type")] as chain + from base + where "property type" = 'Condo' + union all + select + chains.town + , chains.begTS + , base."date recorded" as endTS + , list_append(chains.chain, struct_pack(date:= "date recorded", amt:="sale amount", type:="property type")) as chain + from base, chains + where + base.town = chains.town + and + ( + (len(chains.chain) = 1 and list_contains(['Residential', 'Single Family'], base."property type")) + or (len(chains.chain) = 2 and base."property type" = 'Condo') + or (len(chains.chain) = 3 and list_contains(['Residential', 'Single Family'], base."property type")) + ) + and chains.endTS < base."date recorded" + and base."date recorded" < (chains.endTS + interval 6 days) + ) + select * from chains; +---- \ No newline at end of file From da2d435c4ff9f877450cf7c30eedcdf0aa2a3868 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Fri, 12 Apr 2024 13:23:45 +0200 Subject: [PATCH 389/603] Correctly handle NULL argument in pragma_metadata_info --- .../sqlsmith/include/statement_simplifier.hpp | 4 ++-- extension/sqlsmith/statement_simplifier.cpp | 17 ++++++++--------- .../table/system/pragma_metadata_info.cpp | 11 +++++++++-- test/sql/order/order_overflow.test | 2 +- test/sql/pragma/test_metadata_info.test | 5 +++++ 5 files changed, 25 insertions(+), 14 deletions(-) diff --git a/extension/sqlsmith/include/statement_simplifier.hpp b/extension/sqlsmith/include/statement_simplifier.hpp index bd1133c28d0f..0f238437fe8a 100644 --- a/extension/sqlsmith/include/statement_simplifier.hpp +++ b/extension/sqlsmith/include/statement_simplifier.hpp @@ -57,9 +57,9 @@ class StatementSimplifier { template void SimplifyOptional(duckdb::unique_ptr &opt); - template + template void SimplifyAlias(T &input); - template + template void SimplifyEnum(T &enum_ref, T default_value); void Simplify(unique_ptr &ref); diff --git a/extension/sqlsmith/statement_simplifier.cpp b/extension/sqlsmith/statement_simplifier.cpp index 7a233c0adb15..ea98bd0e1f1d 100644 --- a/extension/sqlsmith/statement_simplifier.cpp +++ b/extension/sqlsmith/statement_simplifier.cpp @@ -51,11 +51,11 @@ void StatementSimplifier::SimplifyMap(T &map) { } // copy the keys vector keys; - for(auto &entry : map) { + for (auto &entry : map) { keys.push_back(entry.first); } // try to remove all of the keys - for(idx_t i = 0; i < keys.size(); i++) { + for (idx_t i = 0; i < keys.size(); i++) { auto entry = map.find(keys[i]); auto n = std::move(entry->second); map.erase(entry); @@ -71,17 +71,16 @@ void StatementSimplifier::SimplifySet(T &set) { } // copy the keys vector keys; - for(auto &entry : set) { + for (auto &entry : set) { keys.push_back(entry); } // try to remove all of the keys - for(idx_t i = 0; i < keys.size(); i++) { + for (idx_t i = 0; i < keys.size(); i++) { auto entry = set.find(keys[i]); set.erase(entry); Simplification(); set.insert(std::move(keys[i])); } - } template @@ -94,7 +93,7 @@ void StatementSimplifier::SimplifyOptional(duckdb::unique_ptr &opt) { opt = std::move(n); } -template +template void StatementSimplifier::SimplifyEnum(T &enum_ref, T default_value) { if (enum_ref == default_value) { return; @@ -105,7 +104,7 @@ void StatementSimplifier::SimplifyEnum(T &enum_ref, T default_value) { enum_ref = current; } -template +template void StatementSimplifier::SimplifyAlias(T &input) { auto alias = std::move(input.alias); auto column_name_alias = std::move(input.column_name_alias); @@ -171,7 +170,7 @@ void StatementSimplifier::Simplify(GroupByNode &groups) { // try to remove grouping sets SimplifyList(groups.grouping_sets, false); // simplify expressions - for(auto &group : groups.group_expressions) { + for (auto &group : groups.group_expressions) { SimplifyExpression(group); } } @@ -333,7 +332,7 @@ void StatementSimplifier::SimplifyExpression(duckdb::unique_ptrCast(); SimplifyMap(star.replace_list); SimplifySet(star.exclude_list); - for(auto &entry : star.replace_list) { + for (auto &entry : star.replace_list) { SimplifyChildExpression(expr, entry.second); } break; diff --git a/src/function/table/system/pragma_metadata_info.cpp b/src/function/table/system/pragma_metadata_info.cpp index 92c180308db6..d6717a9df606 100644 --- a/src/function/table/system/pragma_metadata_info.cpp +++ b/src/function/table/system/pragma_metadata_info.cpp @@ -34,8 +34,15 @@ static unique_ptr PragmaMetadataInfoBind(ClientContext &context, T names.emplace_back("free_list"); return_types.emplace_back(LogicalType::LIST(LogicalType::BIGINT)); - string db_name = - input.inputs.empty() ? DatabaseManager::GetDefaultDatabase(context) : StringValue::Get(input.inputs[0]); + string db_name; + if (input.inputs.empty()) { + db_name = DatabaseManager::GetDefaultDatabase(context); + } else { + if (input.inputs[0].IsNull()) { + throw BinderException("Database argument for pragma_metadata_info cannot be NULL"); + } + db_name = StringValue::Get(input.inputs[0]); + } auto &catalog = Catalog::GetCatalog(context, db_name); auto result = make_uniq(); result->metadata_info = catalog.GetMetadataInfo(context); diff --git a/test/sql/order/order_overflow.test b/test/sql/order/order_overflow.test index 64bc0dc81ee3..5385ac09a920 100644 --- a/test/sql/order/order_overflow.test +++ b/test/sql/order/order_overflow.test @@ -1,6 +1,6 @@ # name: test/sql/order/order_overflow.test # description: Test overflow in ORDER BY -# group: [tpch] +# group: [order] statement error SELECT 42 ORDER BY -9223372036854775808; diff --git a/test/sql/pragma/test_metadata_info.test b/test/sql/pragma/test_metadata_info.test index 767beb34fa9b..62a2b78b2ae4 100644 --- a/test/sql/pragma/test_metadata_info.test +++ b/test/sql/pragma/test_metadata_info.test @@ -33,3 +33,8 @@ CHECKPOINT db1 statement ok FROM pragma_metadata_info('db1'); + +statement error +FROM pragma_metadata_info(NULL) +---- +cannot be NULL From a31c43377c692102e5f23a1b079d847963d1ca22 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Fri, 12 Apr 2024 13:42:57 +0200 Subject: [PATCH 390/603] Fix for array_slice with numeric limits --- src/core_functions/scalar/list/array_slice.cpp | 2 +- test/sql/function/string/test_string_array_slice.test | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/core_functions/scalar/list/array_slice.cpp b/src/core_functions/scalar/list/array_slice.cpp index 7651f410d023..83415d96b670 100644 --- a/src/core_functions/scalar/list/array_slice.cpp +++ b/src/core_functions/scalar/list/array_slice.cpp @@ -97,7 +97,7 @@ static bool ClampSlice(const INPUT_TYPE &value, INDEX_TYPE &begin, INDEX_TYPE &e } const auto length = ValueLength(value); - if (begin < 0 && -begin > length && end < 0 && -end > length) { + if (begin < 0 && -begin > length && end < 0 && end < -length) { begin = 0; end = 0; return true; diff --git a/test/sql/function/string/test_string_array_slice.test b/test/sql/function/string/test_string_array_slice.test index a2aa458c0967..3f6349542093 100644 --- a/test/sql/function/string/test_string_array_slice.test +++ b/test/sql/function/string/test_string_array_slice.test @@ -231,6 +231,12 @@ SELECT array_slice(s, (-2147483646-1), -2147483647) FROM strings (empty) NULL +# out of bounds array slice +query I +select array_slice([], -1, -9223372036854775808) +---- +[] + # With constant null query I select * from (SELECT list_slice(NULL, 1, 3, 2)); From b28c9ea16f105e3c65a9a95deab1dc032018ce45 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 12 Apr 2024 13:49:05 +0200 Subject: [PATCH 391/603] Reset file over scanners and add proper test --- .../buffer_manager/csv_buffer_manager.cpp | 24 +++--- test/sql/copy/csv/recursive_query_csv.test | 73 ++++++++++++++++++- 2 files changed, 81 insertions(+), 16 deletions(-) diff --git a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp index f8efd8347161..87be194b1481 100644 --- a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp +++ b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp @@ -63,12 +63,12 @@ bool CSVBufferManager::ReadNextAndCacheIt() { shared_ptr CSVBufferManager::GetBuffer(const idx_t pos) { lock_guard parallel_lock(main_mutex); -// if (pos == 0 && done && cached_buffers.empty()){ -// // This is a recursive CTE, we have to reset out whole buffer -// done = false; -// file_handle->Reset(); -// Initialize(); -// } + if (pos == 0 && done && cached_buffers.empty()) { + // This is a recursive CTE, we have to reset out whole buffer + done = false; + file_handle->Reset(); + Initialize(); + } while (pos >= cached_buffers.size()) { if (done) { return nullptr; @@ -104,12 +104,12 @@ void CSVBufferManager::ResetBuffer(const idx_t buffer_idx) { } // We only reset if previous one was also already reset if (buffer_idx > 0 && !cached_buffers[buffer_idx - 1]) { -// if (cached_buffers[buffer_idx]->last_buffer){ -// // We clear the whole shebang -// cached_buffers.clear(); -// reset_when_possible.clear(); -// return; -// } + if (cached_buffers[buffer_idx]->last_buffer) { + // We clear the whole shebang + cached_buffers.clear(); + reset_when_possible.clear(); + return; + } cached_buffers[buffer_idx].reset(); idx_t cur_buffer = buffer_idx + 1; while (reset_when_possible.find(cur_buffer) != reset_when_possible.end()) { diff --git a/test/sql/copy/csv/recursive_query_csv.test b/test/sql/copy/csv/recursive_query_csv.test index 520e12362ba1..f6657e3a3c05 100644 --- a/test/sql/copy/csv/recursive_query_csv.test +++ b/test/sql/copy/csv/recursive_query_csv.test @@ -1,12 +1,14 @@ # name: test/sql/copy/csv/recursive_query_csv.test -# description: Test read CSV function in a recursive +# description: Test read CSV function in a recursive CTE # group: [csv] -query I +require httpfs + +query IIII with recursive base as ( select * - from '/Users/holanda/Desktop/Real_Estate_Sales_2001-2021_GL.csv' + from 'https://github.com/duckdb/duckdb-data/releases/download/v1.0/Real_Estate_Sales_2001-2021_GL.csv' where '2003-01-01' < "date recorded" and "date recorded" < '2010-01-01' and "sale amount" > 1000000 ) , chains as @@ -37,4 +39,67 @@ with recursive and base."date recorded" < (chains.endTS + interval 6 days) ) select * from chains; ----- \ No newline at end of file +---- +Greenwich 2007-05-15 2007-05-15 [{'date': 2007-05-15, 'amt': 1215000.0, 'type': Condo}] +Fairfield 2007-06-15 2007-06-15 [{'date': 2007-06-15, 'amt': 1100000.0, 'type': Condo}] +Greenwich 2007-01-31 2007-01-31 [{'date': 2007-01-31, 'amt': 4600000.0, 'type': Condo}] +Greenwich 2007-07-10 2007-07-10 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}] +Greenwich 2007-01-19 2007-01-19 [{'date': 2007-01-19, 'amt': 2227500.0, 'type': Condo}] +Greenwich 2006-11-20 2006-11-20 [{'date': 2006-11-20, 'amt': 2050000.0, 'type': Condo}] +Greenwich 2007-08-16 2007-08-16 [{'date': 2007-08-16, 'amt': 2430000.0, 'type': Condo}] +Greenwich 2007-07-17 2007-07-17 [{'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}] +Greenwich 2007-06-15 2007-06-15 [{'date': 2007-06-15, 'amt': 2264564.0, 'type': Condo}] +New Canaan 2007-02-15 2007-02-15 [{'date': 2007-02-15, 'amt': 2230000.0, 'type': Condo}] +Greenwich 2007-03-12 2007-03-12 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}] +Greenwich 2007-02-27 2007-02-27 [{'date': 2007-02-27, 'amt': 1120000.0, 'type': Condo}] +Danbury 2007-05-02 2007-05-02 [{'date': 2007-05-02, 'amt': 3105000.0, 'type': Condo}] +Darien 2007-09-12 2007-09-12 [{'date': 2007-09-12, 'amt': 1150000.0, 'type': Condo}] +Danbury 2007-05-09 2007-05-09 [{'date': 2007-05-09, 'amt': 1014205.0, 'type': Condo}] +Greenwich 2007-03-30 2007-03-30 [{'date': 2007-03-30, 'amt': 1200000.0, 'type': Condo}] +Greenwich 2007-03-20 2007-03-20 [{'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}] +Greenwich 2006-12-14 2006-12-14 [{'date': 2006-12-14, 'amt': 1800000.0, 'type': Condo}] +Clinton 2007-08-22 2007-08-22 [{'date': 2007-08-22, 'amt': 1175000.0, 'type': Condo}] +New Canaan 2007-01-22 2007-01-22 [{'date': 2007-01-22, 'amt': 1735000.0, 'type': Condo}] +Greenwich 2007-07-17 2007-07-19 [{'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-19, 'amt': 3600000.0, 'type': Single Family}] +Greenwich 2007-08-16 2007-08-21 [{'date': 2007-08-16, 'amt': 2430000.0, 'type': Condo}, {'date': 2007-08-21, 'amt': 4100000.0, 'type': Single Family}] +Greenwich 2007-02-27 2007-03-02 [{'date': 2007-02-27, 'amt': 1120000.0, 'type': Condo}, {'date': 2007-03-02, 'amt': 6500000.0, 'type': Single Family}] +Greenwich 2007-03-12 2007-03-16 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-16, 'amt': 12500000.0, 'type': Single Family}] +Greenwich 2007-06-15 2007-06-19 [{'date': 2007-06-15, 'amt': 2264564.0, 'type': Condo}, {'date': 2007-06-19, 'amt': 1965000.0, 'type': Single Family}] +Greenwich 2006-11-20 2006-11-21 [{'date': 2006-11-20, 'amt': 2050000.0, 'type': Condo}, {'date': 2006-11-21, 'amt': 6500000.0, 'type': Single Family}] +Greenwich 2007-07-10 2007-07-11 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-11, 'amt': 3250000.0, 'type': Single Family}] +Greenwich 2007-02-27 2007-02-28 [{'date': 2007-02-27, 'amt': 1120000.0, 'type': Condo}, {'date': 2007-02-28, 'amt': 2260000.0, 'type': Single Family}] +Greenwich 2007-07-17 2007-07-20 [{'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-20, 'amt': 18000000.0, 'type': Single Family}] +Greenwich 2007-07-10 2007-07-12 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 3565000.0, 'type': Single Family}] +Greenwich 2007-02-27 2007-03-01 [{'date': 2007-02-27, 'amt': 1120000.0, 'type': Condo}, {'date': 2007-03-01, 'amt': 1900000.0, 'type': Single Family}] +Greenwich 2007-03-12 2007-03-16 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-16, 'amt': 6537500.0, 'type': Single Family}] +Greenwich 2007-08-16 2007-08-17 [{'date': 2007-08-16, 'amt': 2430000.0, 'type': Condo}, {'date': 2007-08-17, 'amt': 3400000.0, 'type': Single Family}] +Greenwich 2007-07-10 2007-07-11 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-11, 'amt': 3150000.0, 'type': Single Family}] +Greenwich 2007-08-16 2007-08-17 [{'date': 2007-08-16, 'amt': 2430000.0, 'type': Condo}, {'date': 2007-08-17, 'amt': 1925000.0, 'type': Single Family}] +Greenwich 2007-07-10 2007-07-12 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 1269000.0, 'type': Single Family}] +Greenwich 2006-12-14 2006-12-15 [{'date': 2006-12-14, 'amt': 1800000.0, 'type': Condo}, {'date': 2006-12-15, 'amt': 2195000.0, 'type': Single Family}] +Greenwich 2007-03-20 2007-03-22 [{'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}, {'date': 2007-03-22, 'amt': 1580000.0, 'type': Single Family}] +Greenwich 2007-03-12 2007-03-13 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-13, 'amt': 1600000.0, 'type': Single Family}] +Greenwich 2007-07-17 2007-07-20 [{'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-20, 'amt': 7225000.0, 'type': Single Family}] +Greenwich 2007-08-16 2007-08-20 [{'date': 2007-08-16, 'amt': 2430000.0, 'type': Condo}, {'date': 2007-08-20, 'amt': 2590000.0, 'type': Single Family}] +Greenwich 2006-12-14 2006-12-15 [{'date': 2006-12-14, 'amt': 1800000.0, 'type': Condo}, {'date': 2006-12-15, 'amt': 5500000.0, 'type': Single Family}] +Greenwich 2007-01-19 2007-01-24 [{'date': 2007-01-19, 'amt': 2227500.0, 'type': Condo}, {'date': 2007-01-24, 'amt': 1750000.0, 'type': Single Family}] +Greenwich 2006-12-14 2006-12-18 [{'date': 2006-12-14, 'amt': 1800000.0, 'type': Condo}, {'date': 2006-12-18, 'amt': 5010000.0, 'type': Single Family}] +Greenwich 2007-03-20 2007-03-23 [{'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}, {'date': 2007-03-23, 'amt': 2850000.0, 'type': Single Family}] +Greenwich 2007-06-15 2007-06-19 [{'date': 2007-06-15, 'amt': 2264564.0, 'type': Condo}, {'date': 2007-06-19, 'amt': 1470000.0, 'type': Single Family}] +Greenwich 2007-07-10 2007-07-11 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-11, 'amt': 7050000.0, 'type': Single Family}] +Greenwich 2007-05-15 2007-05-17 [{'date': 2007-05-15, 'amt': 1215000.0, 'type': Condo}, {'date': 2007-05-17, 'amt': 2250000.0, 'type': Single Family}] +Greenwich 2007-07-10 2007-07-17 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 1269000.0, 'type': Single Family}, {'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}] +Greenwich 2007-07-10 2007-07-17 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 3565000.0, 'type': Single Family}, {'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}] +Greenwich 2007-03-12 2007-03-20 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-16, 'amt': 6537500.0, 'type': Single Family}, {'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}] +Greenwich 2007-03-12 2007-03-20 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-16, 'amt': 12500000.0, 'type': Single Family}, {'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}] +Greenwich 2007-07-10 2007-07-19 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 3565000.0, 'type': Single Family}, {'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-19, 'amt': 3600000.0, 'type': Single Family}] +Greenwich 2007-07-10 2007-07-19 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 1269000.0, 'type': Single Family}, {'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-19, 'amt': 3600000.0, 'type': Single Family}] +Greenwich 2007-07-10 2007-07-20 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 3565000.0, 'type': Single Family}, {'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-20, 'amt': 18000000.0, 'type': Single Family}] +Greenwich 2007-07-10 2007-07-20 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 1269000.0, 'type': Single Family}, {'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-20, 'amt': 18000000.0, 'type': Single Family}] +Greenwich 2007-03-12 2007-03-22 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-16, 'amt': 12500000.0, 'type': Single Family}, {'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}, {'date': 2007-03-22, 'amt': 1580000.0, 'type': Single Family}] +Greenwich 2007-03-12 2007-03-22 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-16, 'amt': 6537500.0, 'type': Single Family}, {'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}, {'date': 2007-03-22, 'amt': 1580000.0, 'type': Single Family}] +Greenwich 2007-07-10 2007-07-20 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 3565000.0, 'type': Single Family}, {'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-20, 'amt': 7225000.0, 'type': Single Family}] +Greenwich 2007-07-10 2007-07-20 [{'date': 2007-07-10, 'amt': 1240000.0, 'type': Condo}, {'date': 2007-07-12, 'amt': 1269000.0, 'type': Single Family}, {'date': 2007-07-17, 'amt': 3000000.0, 'type': Condo}, {'date': 2007-07-20, 'amt': 7225000.0, 'type': Single Family}] +Greenwich 2007-03-12 2007-03-23 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-16, 'amt': 12500000.0, 'type': Single Family}, {'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}, {'date': 2007-03-23, 'amt': 2850000.0, 'type': Single Family}] +Greenwich 2007-03-12 2007-03-23 [{'date': 2007-03-12, 'amt': 1084687.0, 'type': Condo}, {'date': 2007-03-16, 'amt': 6537500.0, 'type': Single Family}, {'date': 2007-03-20, 'amt': 4100000.0, 'type': Condo}, {'date': 2007-03-23, 'amt': 2850000.0, 'type': Single Family}] + From da8ac772bb383d55b080c9950ff8d7270efd16b3 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 12 Apr 2024 13:57:00 +0200 Subject: [PATCH 392/603] Extension metadata: Fix to use by default the result of `git log -1 --format=%h` --- CMakeLists.txt | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f071e73d212d..47cf1a5232dc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -908,6 +908,21 @@ macro(register_external_extension NAME URL COMMIT DONT_LINK DONT_BUILD LOAD_TEST message(STATUS "Load extension '${NAME}' from ${URL} @ ${COMMIT}") FETCHCONTENT_POPULATE(${NAME}_EXTENSION_FC) + find_package(Git) + if(Git_FOUND) + execute_process( + COMMAND ${GIT_EXECUTABLE} log -1 --format=%h + WORKING_DIRECTORY "${${NAME}_extension_fc_SOURCE_DIR}" + RESULT_VARIABLE GIT_RESULT + OUTPUT_VARIABLE GIT_SHORT_COMMIT + OUTPUT_STRIP_TRAILING_WHITESPACE) + else() + message(WARNING "Git not found, using ${COMMIT} as GIT_SHORT_COMMIT") + set(GIT_SHORT_COMMIT "${COMMIT}") + endif() + + set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_EXT_VERSION "${GIT_SHORT_COMMIT}") + if ("${INCLUDE_PATH}" STREQUAL "") set(INCLUDE_FULL_PATH "${${NAME}_extension_fc_SOURCE_DIR}/src/include") else() @@ -953,8 +968,6 @@ function(duckdb_extension_load NAME) register_external_extension(${NAME} "${duckdb_extension_load_GIT_URL}" "${duckdb_extension_load_GIT_TAG}" "${duckdb_extension_load_DONT_LINK}" "${duckdb_extension_load_DONT_BUILD}" "${duckdb_extension_load_LOAD_TESTS}" "${duckdb_extension_load_SOURCE_DIR}" "${duckdb_extension_load_INCLUDE_DIR}" "${duckdb_extension_load_TEST_DIR}" "${duckdb_extension_load_APPLY_PATCHES}" "${duckdb_extension_load_SUBMODULES}") if (NOT "${duckdb_extension_load_EXTENSION_VERSION}" STREQUAL "") set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_EXT_VERSION "${duckdb_extension_load_EXTENSION_VERSION}" PARENT_SCOPE) - else() - set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_EXT_VERSION "${duckdb_extension_load_GIT_TAG}" PARENT_SCOPE) endif() elseif (NOT "${duckdb_extension_load_SOURCE_DIR}" STREQUAL "") # Local extension, custom path From a0004970eef58eb6955f1305685d8e70b78a9bd7 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 12 Apr 2024 13:59:57 +0200 Subject: [PATCH 393/603] extensions: Change logging to log GIT_SHORT_COMMIT --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 47cf1a5232dc..15857696ebbb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -905,7 +905,6 @@ macro(register_external_extension NAME URL COMMIT DONT_LINK DONT_BUILD LOAD_TEST GIT_SUBMODULES "${SUBMODULES}" PATCH_COMMAND ${PATCH_COMMAND} ) - message(STATUS "Load extension '${NAME}' from ${URL} @ ${COMMIT}") FETCHCONTENT_POPULATE(${NAME}_EXTENSION_FC) find_package(Git) @@ -921,6 +920,7 @@ macro(register_external_extension NAME URL COMMIT DONT_LINK DONT_BUILD LOAD_TEST set(GIT_SHORT_COMMIT "${COMMIT}") endif() + message(STATUS "Load extension '${NAME}' from ${URL} @ ${GIT_SHORT_COMMIT}") set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_EXT_VERSION "${GIT_SHORT_COMMIT}") if ("${INCLUDE_PATH}" STREQUAL "") From d077157f88db892c6303f3af92d722641c001c56 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 12 Apr 2024 14:37:26 +0200 Subject: [PATCH 394/603] Throw error message if CTEs are attempted over a piped csv --- .../csv_scanner/buffer_manager/csv_buffer_manager.cpp | 4 ++++ .../execution/operator/csv_scanner/csv_buffer_manager.hpp | 1 + 2 files changed, 5 insertions(+) diff --git a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp index 87be194b1481..82cf1f846822 100644 --- a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp +++ b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp @@ -8,6 +8,7 @@ CSVBufferManager::CSVBufferManager(ClientContext &context_p, const CSVReaderOpti : context(context_p), file_idx(file_idx_p), file_path(file_path_p), buffer_size(CSVBuffer::CSV_BUFFER_SIZE) { D_ASSERT(!file_path.empty()); file_handle = ReadCSV::OpenCSV(file_path, options.compression, context); + is_pipe = file_handle->IsPipe(); skip_rows = options.dialect_options.skip_rows.GetValue(); auto file_size = file_handle->FileSize(); if (file_size > 0 && file_size < buffer_size) { @@ -64,6 +65,9 @@ bool CSVBufferManager::ReadNextAndCacheIt() { shared_ptr CSVBufferManager::GetBuffer(const idx_t pos) { lock_guard parallel_lock(main_mutex); if (pos == 0 && done && cached_buffers.empty()) { + if (is_pipe){ + throw InvalidInputException("Recursive CTEs are not allowed when using piped csv files"); + } // This is a recursive CTE, we have to reset out whole buffer done = false; file_handle->Reset(); diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp index f8c6f246c3e6..67ee6a3a18e4 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp @@ -70,6 +70,7 @@ class CSVBufferManager { //! If the file_handle used seek bool has_seeked = false; unordered_set reset_when_possible; + bool is_pipe; }; } // namespace duckdb From a9495df58dfc65675bb1be1ea2d4dc6169f8a6a3 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 12 Apr 2024 15:09:50 +0200 Subject: [PATCH 395/603] Durrr forgot to initialize the variable --- .../operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp | 3 +-- .../execution/operator/csv_scanner/csv_buffer_manager.hpp | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp index 68a52f78acf3..691b0f2f5120 100644 --- a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp +++ b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp @@ -8,7 +8,6 @@ CSVBufferManager::CSVBufferManager(ClientContext &context_p, const CSVReaderOpti : context(context_p), file_idx(file_idx_p), file_path(file_path_p), buffer_size(CSVBuffer::CSV_BUFFER_SIZE) { D_ASSERT(!file_path.empty()); file_handle = ReadCSV::OpenCSV(file_path, options.compression, context); - can_seek = file_handle->CanSeek(); skip_rows = options.dialect_options.skip_rows.GetValue(); auto file_size = file_handle->FileSize(); if (file_size > 0 && file_size < buffer_size) { @@ -72,7 +71,7 @@ shared_ptr CSVBufferManager::GetBuffer(const idx_t pos) { done = true; } } - if (pos != 0 && (sniffing || can_seek)) { + if (pos != 0 && (sniffing || file_handle->CanSeek())) { // We don't need to unpin the buffers here if we are not sniffing since we // control it per-thread on the scan if (cached_buffers[pos - 1]) { diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp index dc05ef378ab3..a305f204002d 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp @@ -47,7 +47,7 @@ class CSVBufferManager { ClientContext &context; idx_t skip_rows = 0; - bool sniffing; + bool sniffing = false; private: //! Reads next buffer in reference to cached_buffers.front() @@ -71,7 +71,6 @@ class CSVBufferManager { //! If the file_handle used seek bool has_seeked = false; unordered_set reset_when_possible; - bool can_seek; }; } // namespace duckdb From f94b09f0d03e3835b1d81b17680d881077984b08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Fri, 12 Apr 2024 15:11:37 +0200 Subject: [PATCH 396/603] woo --- .../storage/compression/alp/algorithm/alp.hpp | 4 +-- .../duckdb/storage/table/segment_tree.hpp | 10 +++---- src/storage/table/array_column_data.cpp | 6 ++-- src/storage/table/column_data.cpp | 6 ++-- src/storage/table/column_segment.cpp | 7 +++-- src/storage/table/list_column_data.cpp | 8 +++--- src/storage/table/row_group.cpp | 15 +++++----- src/storage/table/row_group_collection.cpp | 28 ++++++++++--------- src/storage/table/row_version_manager.cpp | 2 +- src/storage/table/struct_column_data.cpp | 2 +- src/storage/table/update_segment.cpp | 12 ++++---- src/transaction/cleanup_state.cpp | 4 +-- src/transaction/commit_state.cpp | 6 ++-- src/transaction/duck_transaction_manager.cpp | 4 +-- 14 files changed, 60 insertions(+), 54 deletions(-) diff --git a/src/include/duckdb/storage/compression/alp/algorithm/alp.hpp b/src/include/duckdb/storage/compression/alp/algorithm/alp.hpp index d216f23e58af..a8c7e91f3e66 100644 --- a/src/include/duckdb/storage/compression/alp/algorithm/alp.hpp +++ b/src/include/duckdb/storage/compression/alp/algorithm/alp.hpp @@ -359,8 +359,8 @@ struct AlpCompression { BitpackingPrimitives::PackBuffer(state.values_encoded, u_encoded_integers, n_values, bit_width); } - state.bit_width = bit_width; // in bits - state.bp_size = bp_size; // in bytes + state.bit_width = bit_width; // in bits + state.bp_size = bp_size; // in bytes state.frame_of_reference = static_cast(min_value); // understood this can be negative } diff --git a/src/include/duckdb/storage/table/segment_tree.hpp b/src/include/duckdb/storage/table/segment_tree.hpp index e2778bc33137..e267d2e351eb 100644 --- a/src/include/duckdb/storage/table/segment_tree.hpp +++ b/src/include/duckdb/storage/table/segment_tree.hpp @@ -83,11 +83,11 @@ class SegmentTree { if (index < 0) { // load all segments LoadAllSegments(l); - index = nodes.size() + index; + index += nodes.size(); if (index < 0) { return nullptr; } - return nodes[index].node.get(); + return nodes[UnsafeNumericCast(index)].node.get(); } else { // lazily load segments until we reach the specific segment while (idx_t(index) >= nodes.size() && LoadNextSegment(l)) { @@ -95,7 +95,7 @@ class SegmentTree { if (idx_t(index) >= nodes.size()) { return nullptr; } - return nodes[index].node.get(); + return nodes[UnsafeNumericCast(index)].node.get(); } } //! Gets the next segment @@ -116,7 +116,7 @@ class SegmentTree { #ifdef DEBUG D_ASSERT(nodes[segment->index].node.get() == segment); #endif - return GetSegmentByIndex(l, segment->index + 1); + return GetSegmentByIndex(l, UnsafeNumericCast(segment->index + 1)); } //! Gets a pointer to the last segment. Useful for appends. @@ -182,7 +182,7 @@ class SegmentTree { if (segment_start >= nodes.size() - 1) { return; } - nodes.erase(nodes.begin() + segment_start + 1, nodes.end()); + nodes.erase(nodes.begin() + UnsafeNumericCast(segment_start) + 1, nodes.end()); } //! Get the segment index of the column segment for the given row diff --git a/src/storage/table/array_column_data.cpp b/src/storage/table/array_column_data.cpp index 31cb93facf7f..be8b8a219c83 100644 --- a/src/storage/table/array_column_data.cpp +++ b/src/storage/table/array_column_data.cpp @@ -120,9 +120,9 @@ void ArrayColumnData::RevertAppend(row_t start_row) { validity.RevertAppend(start_row); // Revert child column auto array_size = ArrayType::GetSize(type); - child_column->RevertAppend(start_row * array_size); + child_column->RevertAppend(start_row * UnsafeNumericCast(array_size)); - this->count = start_row - this->start; + this->count = UnsafeNumericCast(start_row) - this->start; } idx_t ArrayColumnData::Fetch(ColumnScanState &state, row_t row_id, Vector &result) { @@ -162,7 +162,7 @@ void ArrayColumnData::FetchRow(TransactionData transaction, ColumnFetchState &st // We need to fetch between [row_id * array_size, (row_id + 1) * array_size) auto child_state = make_uniq(); child_state->Initialize(child_type, nullptr); - child_column->InitializeScanWithOffset(*child_state, row_id * array_size); + child_column->InitializeScanWithOffset(*child_state, UnsafeNumericCast(row_id) * array_size); Vector child_scan(child_type, array_size); child_column->ScanCount(*child_state, child_scan, array_size); VectorOperations::Copy(child_scan, child_vec, array_size, 0, result_idx * array_size); diff --git a/src/storage/table/column_data.cpp b/src/storage/table/column_data.cpp index 1ccc4b8a19f9..5e11c148f2fe 100644 --- a/src/storage/table/column_data.cpp +++ b/src/storage/table/column_data.cpp @@ -105,7 +105,8 @@ idx_t ColumnData::ScanVector(ColumnScanState &state, Vector &result, idx_t remai if (state.scan_options && state.scan_options->force_fetch_row) { for (idx_t i = 0; i < scan_count; i++) { ColumnFetchState fetch_state; - state.current->FetchRow(fetch_state, UnsafeNumericCast(state.row_index + i), result, result_offset + i); + state.current->FetchRow(fetch_state, UnsafeNumericCast(state.row_index + i), result, + result_offset + i); } } else { state.current->Scan(state, scan_count, result, result_offset, @@ -342,7 +343,8 @@ idx_t ColumnData::Fetch(ColumnScanState &state, row_t row_id, Vector &result) { D_ASSERT(row_id >= 0); D_ASSERT(idx_t(row_id) >= start); // perform the fetch within the segment - state.row_index = start + ((UnsafeNumericCast(row_id) - start) / STANDARD_VECTOR_SIZE * STANDARD_VECTOR_SIZE); + state.row_index = + start + ((UnsafeNumericCast(row_id) - start) / STANDARD_VECTOR_SIZE * STANDARD_VECTOR_SIZE); state.current = data.GetSegment(state.row_index); state.internal_index = state.current->start; return ScanVector(state, result, STANDARD_VECTOR_SIZE, false); diff --git a/src/storage/table/column_segment.cpp b/src/storage/table/column_segment.cpp index 175791b612a4..d0f2cd69545c 100644 --- a/src/storage/table/column_segment.cpp +++ b/src/storage/table/column_segment.cpp @@ -60,8 +60,8 @@ unique_ptr ColumnSegment::CreateTransientSegment(DatabaseInstance } else { buffer_manager.Allocate(MemoryTag::IN_MEMORY_TABLE, segment_size, false, &block); } - return make_uniq(db, std::move(block), type, ColumnSegmentType::TRANSIENT, start, 0, *function, - BaseStatistics::CreateEmpty(type), INVALID_BLOCK, 0, segment_size); + return make_uniq(db, std::move(block), type, ColumnSegmentType::TRANSIENT, start, 0U, *function, + BaseStatistics::CreateEmpty(type), INVALID_BLOCK, 0U, segment_size); } //===--------------------------------------------------------------------===// @@ -132,7 +132,8 @@ void ColumnSegment::ScanPartial(ColumnScanState &state, idx_t scan_count, Vector // Fetch //===--------------------------------------------------------------------===// void ColumnSegment::FetchRow(ColumnFetchState &state, row_t row_id, Vector &result, idx_t result_idx) { - function.get().fetch_row(*this, state, row_id - this->start, result, result_idx); + function.get().fetch_row(*this, state, UnsafeNumericCast(UnsafeNumericCast(row_id) - this->start), + result, result_idx); } //===--------------------------------------------------------------------===// diff --git a/src/storage/table/list_column_data.cpp b/src/storage/table/list_column_data.cpp index 418a4bd4644e..21aa9b5af410 100644 --- a/src/storage/table/list_column_data.cpp +++ b/src/storage/table/list_column_data.cpp @@ -44,7 +44,7 @@ uint64_t ListColumnData::FetchListOffset(idx_t row_idx) { auto segment = data.GetSegment(row_idx); ColumnFetchState fetch_state; Vector result(type, 1); - segment->FetchRow(fetch_state, row_idx, result, 0); + segment->FetchRow(fetch_state, UnsafeNumericCast(row_idx), result, 0U); // initialize the child scan with the required offset return FlatVector::GetData(result)[0]; @@ -235,7 +235,7 @@ void ListColumnData::RevertAppend(row_t start_row) { if (column_count > start) { // revert append in the child column auto list_offset = FetchListOffset(column_count - 1); - child_column->RevertAppend(list_offset); + child_column->RevertAppend(UnsafeNumericCast(list_offset)); } } @@ -269,8 +269,8 @@ void ListColumnData::FetchRow(TransactionData transaction, ColumnFetchState &sta } // now perform the fetch within the segment - auto start_offset = idx_t(row_id) == this->start ? 0 : FetchListOffset(row_id - 1); - auto end_offset = FetchListOffset(row_id); + auto start_offset = idx_t(row_id) == this->start ? 0 : FetchListOffset(UnsafeNumericCast(row_id - 1)); + auto end_offset = FetchListOffset(UnsafeNumericCast(row_id)); validity.FetchRow(transaction, *state.child_states[0], row_id, result, result_idx); auto &validity = FlatVector::Validity(result); diff --git a/src/storage/table/row_group.cpp b/src/storage/table/row_group.cpp index 824827d76a97..2de5faf7f735 100644 --- a/src/storage/table/row_group.cpp +++ b/src/storage/table/row_group.cpp @@ -487,7 +487,7 @@ void RowGroup::TemplatedScan(TransactionData transaction, CollectionScanState &s if (column == COLUMN_IDENTIFIER_ROW_ID) { // scan row id D_ASSERT(result.data[i].GetType().InternalType() == ROW_TYPE); - result.data[i].Sequence(this->start + current_row, 1, count); + result.data[i].Sequence(UnsafeNumericCast(this->start + current_row), 1, count); } else { auto &col_data = GetColumn(column); if (TYPE != TableScanType::TABLE_SCAN_REGULAR) { @@ -551,7 +551,8 @@ void RowGroup::TemplatedScan(TransactionData transaction, CollectionScanState &s result.data[i].SetVectorType(VectorType::FLAT_VECTOR); auto result_data = FlatVector::GetData(result.data[i]); for (size_t sel_idx = 0; sel_idx < approved_tuple_count; sel_idx++) { - result_data[sel_idx] = this->start + current_row + sel.get_index(sel_idx); + result_data[sel_idx] = + UnsafeNumericCast(this->start + current_row + sel.get_index(sel_idx)); } } else { auto &col_data = GetColumn(column); @@ -703,7 +704,7 @@ void RowGroup::RevertAppend(idx_t row_group_start) { auto &vinfo = GetOrCreateVersionInfo(); vinfo.RevertAppend(row_group_start - this->start); for (auto &column : columns) { - column->RevertAppend(row_group_start); + column->RevertAppend(UnsafeNumericCast(row_group_start)); } this->count = MinValue(row_group_start - this->start, this->count); Verify(); @@ -955,7 +956,7 @@ idx_t RowGroup::Delete(TransactionData transaction, DataTable &table, row_t *ids for (idx_t i = 0; i < count; i++) { D_ASSERT(ids[i] >= 0); D_ASSERT(idx_t(ids[i]) >= this->start && idx_t(ids[i]) < this->start + this->count); - del_state.Delete(ids[i] - this->start); + del_state.Delete(ids[i] - UnsafeNumericCast(this->start)); } del_state.Flush(); return del_state.delete_count; @@ -975,15 +976,15 @@ idx_t RowGroup::DeleteRows(idx_t vector_idx, transaction_t transaction_id, row_t void VersionDeleteState::Delete(row_t row_id) { D_ASSERT(row_id >= 0); - idx_t vector_idx = row_id / STANDARD_VECTOR_SIZE; - idx_t idx_in_vector = row_id - vector_idx * STANDARD_VECTOR_SIZE; + idx_t vector_idx = UnsafeNumericCast(row_id) / STANDARD_VECTOR_SIZE; + idx_t idx_in_vector = UnsafeNumericCast(row_id) - vector_idx * STANDARD_VECTOR_SIZE; if (current_chunk != vector_idx) { Flush(); current_chunk = vector_idx; chunk_row = vector_idx * STANDARD_VECTOR_SIZE; } - rows[count++] = idx_in_vector; + rows[count++] = UnsafeNumericCast(idx_in_vector); } void VersionDeleteState::Flush() { diff --git a/src/storage/table/row_group_collection.cpp b/src/storage/table/row_group_collection.cpp index 5fae55491020..00daedabcdc5 100644 --- a/src/storage/table/row_group_collection.cpp +++ b/src/storage/table/row_group_collection.cpp @@ -95,7 +95,7 @@ void RowGroupCollection::InitializeEmpty() { void RowGroupCollection::AppendRowGroup(SegmentLock &l, idx_t start_row) { D_ASSERT(start_row >= row_start); - auto new_row_group = make_uniq(*this, start_row, 0); + auto new_row_group = make_uniq(*this, start_row, 0U); new_row_group->InitializeEmpty(types); row_groups->AppendSegment(l, std::move(new_row_group)); } @@ -271,13 +271,13 @@ void RowGroupCollection::Fetch(TransactionData transaction, DataChunk &result, c { idx_t segment_index; auto l = row_groups->Lock(); - if (!row_groups->TryGetSegmentIndex(l, row_id, segment_index)) { + if (!row_groups->TryGetSegmentIndex(l, UnsafeNumericCast(row_id), segment_index)) { // in parallel append scenarios it is possible for the row_id continue; } - row_group = row_groups->GetSegmentByIndex(l, segment_index); + row_group = row_groups->GetSegmentByIndex(l, UnsafeNumericCast(segment_index)); } - if (!row_group->Fetch(transaction, row_id - row_group->start)) { + if (!row_group->Fetch(transaction, UnsafeNumericCast(row_id) - row_group->start)) { continue; } row_group->FetchRow(transaction, state, column_ids, row_id, result, count); @@ -306,7 +306,7 @@ bool RowGroupCollection::IsEmpty(SegmentLock &l) const { } void RowGroupCollection::InitializeAppend(TransactionData transaction, TableAppendState &state) { - state.row_start = total_rows; + state.row_start = UnsafeNumericCast(total_rows.load()); state.current_row = state.row_start; state.total_append_count = 0; @@ -433,7 +433,7 @@ void RowGroupCollection::RevertAppendInternal(idx_t start_row) { // revert from the last segment segment_index = segment_count - 1; } - auto &segment = *row_groups->GetSegmentByIndex(l, segment_index); + auto &segment = *row_groups->GetSegmentByIndex(l, UnsafeNumericCast(segment_index)); // remove any segments AFTER this segment: they should be deleted entirely row_groups->EraseSegments(l, segment_index); @@ -468,7 +468,7 @@ idx_t RowGroupCollection::Delete(TransactionData transaction, DataTable &table, idx_t pos = 0; do { idx_t start = pos; - auto row_group = row_groups->GetSegment(ids[start]); + auto row_group = row_groups->GetSegment(UnsafeNumericCast(ids[start])); for (pos++; pos < count; pos++) { D_ASSERT(ids[pos] >= 0); // check if this id still belongs to this row group @@ -494,10 +494,12 @@ void RowGroupCollection::Update(TransactionData transaction, row_t *ids, const v idx_t pos = 0; do { idx_t start = pos; - auto row_group = row_groups->GetSegment(ids[pos]); + auto row_group = row_groups->GetSegment(UnsafeNumericCast(ids[pos])); row_t base_id = - row_group->start + ((ids[pos] - row_group->start) / STANDARD_VECTOR_SIZE * STANDARD_VECTOR_SIZE); - row_t max_id = MinValue(base_id + STANDARD_VECTOR_SIZE, row_group->start + row_group->count); + UnsafeNumericCast(row_group->start + ((UnsafeNumericCast(ids[pos]) - row_group->start) / + STANDARD_VECTOR_SIZE * STANDARD_VECTOR_SIZE)); + auto max_id = MinValue(base_id + STANDARD_VECTOR_SIZE, + UnsafeNumericCast(row_group->start + row_group->count)); for (pos++; pos < updates.size(); pos++) { D_ASSERT(ids[pos] >= 0); // check if this id still belongs to this vector in this row group @@ -544,8 +546,8 @@ void RowGroupCollection::RemoveFromIndexes(TableIndexList &indexes, Vector &row_ result.Reset(); // figure out which row_group to fetch from auto row_id = row_ids[r]; - auto row_group = row_groups->GetSegment(row_id); - auto row_group_vector_idx = (row_id - row_group->start) / STANDARD_VECTOR_SIZE; + auto row_group = row_groups->GetSegment(UnsafeNumericCast(row_id)); + auto row_group_vector_idx = (UnsafeNumericCast(row_id) - row_group->start) / STANDARD_VECTOR_SIZE; auto base_row_id = row_group_vector_idx * STANDARD_VECTOR_SIZE + row_group->start; // fetch the current vector @@ -586,7 +588,7 @@ void RowGroupCollection::UpdateColumn(TransactionData transaction, Vector &row_i } // find the row_group this id belongs to auto primary_column_idx = column_path[0]; - auto row_group = row_groups->GetSegment(first_id); + auto row_group = row_groups->GetSegment(UnsafeNumericCast(first_id)); row_group->UpdateColumn(transaction, updates, row_ids, column_path); row_group->MergeIntoStatistics(primary_column_idx, stats.GetStats(primary_column_idx).Statistics()); diff --git a/src/storage/table/row_version_manager.cpp b/src/storage/table/row_version_manager.cpp index ead21f89234f..c317f246183b 100644 --- a/src/storage/table/row_version_manager.cpp +++ b/src/storage/table/row_version_manager.cpp @@ -69,7 +69,7 @@ bool RowVersionManager::Fetch(TransactionData transaction, idx_t row) { if (!info) { return true; } - return info->Fetch(transaction, row - vector_index * STANDARD_VECTOR_SIZE); + return info->Fetch(transaction, UnsafeNumericCast(row - vector_index * STANDARD_VECTOR_SIZE)); } void RowVersionManager::AppendVersionInfo(TransactionData transaction, idx_t count, idx_t row_group_start, diff --git a/src/storage/table/struct_column_data.cpp b/src/storage/table/struct_column_data.cpp index 5fca4ddae306..e31867a24125 100644 --- a/src/storage/table/struct_column_data.cpp +++ b/src/storage/table/struct_column_data.cpp @@ -157,7 +157,7 @@ void StructColumnData::RevertAppend(row_t start_row) { for (auto &sub_column : sub_columns) { sub_column->RevertAppend(start_row); } - this->count = start_row - this->start; + this->count = UnsafeNumericCast(start_row) - this->start; } idx_t StructColumnData::Fetch(ColumnScanState &state, row_t row_id, Vector &result) { diff --git a/src/storage/table/update_segment.cpp b/src/storage/table/update_segment.cpp index 624d987edab6..317023de8900 100644 --- a/src/storage/table/update_segment.cpp +++ b/src/storage/table/update_segment.cpp @@ -581,7 +581,7 @@ void UpdateSegment::InitializeUpdateInfo(UpdateInfo &info, row_t *ids, const Sel auto idx = sel.get_index(i); auto id = ids[idx]; D_ASSERT(idx_t(id) >= vector_offset && idx_t(id) < vector_offset + STANDARD_VECTOR_SIZE); - info.tuples[i] = NumericCast(id - vector_offset); + info.tuples[i] = NumericCast(NumericCast(id) - vector_offset); }; } @@ -697,7 +697,7 @@ static idx_t MergeLoop(row_t a[], sel_t b[], idx_t acount, idx_t bcount, idx_t a idx_t count = 0; while (aidx < acount && bidx < bcount) { auto a_index = asel.get_index(aidx); - auto a_id = a[a_index] - aoffset; + auto a_id = UnsafeNumericCast(a[a_index]) - aoffset; auto b_id = b[bidx]; if (a_id == b_id) { merge(a_id, a_index, bidx, count); @@ -716,7 +716,7 @@ static idx_t MergeLoop(row_t a[], sel_t b[], idx_t acount, idx_t bcount, idx_t a } for (; aidx < acount; aidx++) { auto a_index = asel.get_index(aidx); - pick_a(a[a_index] - aoffset, a_index, count); + pick_a(UnsafeNumericCast(a[a_index]) - aoffset, a_index, count); count++; } for (; bidx < bcount; bidx++) { @@ -777,7 +777,7 @@ static void MergeUpdateLoopInternal(UpdateInfo *base_info, V *base_table_data, U for (idx_t i = 0; i < count; i++) { auto idx = sel.get_index(i); // we have to merge the info for "ids[i]" - auto update_id = ids[idx] - base_id; + auto update_id = UnsafeNumericCast(ids[idx]) - base_id; while (update_info_offset < update_info->N && update_info->tuples[update_info_offset] < update_id) { // old id comes before the current id: write it @@ -1103,7 +1103,7 @@ void UpdateSegment::Update(TransactionData transaction, idx_t column_index, Vect // get the vector index based on the first id // we assert that all updates must be part of the same vector auto first_id = ids[sel.get_index(0)]; - idx_t vector_index = (first_id - column_data.start) / STANDARD_VECTOR_SIZE; + idx_t vector_index = (UnsafeNumericCast(first_id) - column_data.start) / STANDARD_VECTOR_SIZE; idx_t vector_offset = column_data.start + vector_index * STANDARD_VECTOR_SIZE; D_ASSERT(idx_t(first_id) >= column_data.start); @@ -1116,7 +1116,7 @@ void UpdateSegment::Update(TransactionData transaction, idx_t column_index, Vect // there is already a version here, check if there are any conflicts and search for the node that belongs to // this transaction in the version chain auto base_info = root->info[vector_index]->info.get(); - CheckForConflicts(base_info->next, transaction, ids, sel, count, vector_offset, node); + CheckForConflicts(base_info->next, transaction, ids, sel, count, UnsafeNumericCast(vector_offset), node); // there are no conflicts // first, check if this thread has already done any updates diff --git a/src/transaction/cleanup_state.cpp b/src/transaction/cleanup_state.cpp index 0ec438c20339..802405483aeb 100644 --- a/src/transaction/cleanup_state.cpp +++ b/src/transaction/cleanup_state.cpp @@ -72,12 +72,12 @@ void CleanupState::CleanupDelete(DeleteInfo &info) { count = 0; if (info.is_consecutive) { for (idx_t i = 0; i < info.count; i++) { - row_numbers[count++] = info.base_row + i; + row_numbers[count++] = UnsafeNumericCast(info.base_row + i); } } else { auto rows = info.GetRows(); for (idx_t i = 0; i < info.count; i++) { - row_numbers[count++] = info.base_row + rows[i]; + row_numbers[count++] = UnsafeNumericCast(info.base_row + rows[i]); } } Flush(); diff --git a/src/transaction/commit_state.cpp b/src/transaction/commit_state.cpp index 986aac3b17c1..ae78d0d0c840 100644 --- a/src/transaction/commit_state.cpp +++ b/src/transaction/commit_state.cpp @@ -206,12 +206,12 @@ void CommitState::WriteDelete(DeleteInfo &info) { auto rows = FlatVector::GetData(delete_chunk->data[0]); if (info.is_consecutive) { for (idx_t i = 0; i < info.count; i++) { - rows[i] = info.base_row + i; + rows[i] = UnsafeNumericCast(info.base_row + i); } } else { auto delete_rows = info.GetRows(); for (idx_t i = 0; i < info.count; i++) { - rows[i] = info.base_row + delete_rows[i]; + rows[i] = UnsafeNumericCast(info.base_row) + delete_rows[i]; } } delete_chunk->SetCardinality(info.count); @@ -245,7 +245,7 @@ void CommitState::WriteUpdate(UpdateInfo &info) { auto row_ids = FlatVector::GetData(update_chunk->data[1]); idx_t start = column_data.start + info.vector_index * STANDARD_VECTOR_SIZE; for (idx_t i = 0; i < info.N; i++) { - row_ids[info.tuples[i]] = start + info.tuples[i]; + row_ids[info.tuples[i]] = UnsafeNumericCast(start + info.tuples[i]); } if (column_data.type.id() == LogicalTypeId::VALIDITY) { // zero-initialize the booleans diff --git a/src/transaction/duck_transaction_manager.cpp b/src/transaction/duck_transaction_manager.cpp index bf7014c93411..a3ec6d4a3d9f 100644 --- a/src/transaction/duck_transaction_manager.cpp +++ b/src/transaction/duck_transaction_manager.cpp @@ -311,7 +311,7 @@ void DuckTransactionManager::RemoveTransaction(DuckTransaction &transaction) noe if (i > 0) { // we garbage collected transactions: remove them from the list recently_committed_transactions.erase(recently_committed_transactions.begin(), - recently_committed_transactions.begin() + i); + recently_committed_transactions.begin() + UnsafeNumericCast(i)); } // check if we can free the memory of any old transactions i = active_transactions.empty() ? old_transactions.size() : 0; @@ -326,7 +326,7 @@ void DuckTransactionManager::RemoveTransaction(DuckTransaction &transaction) noe } if (i > 0) { // we garbage collected transactions: remove them from the list - old_transactions.erase(old_transactions.begin(), old_transactions.begin() + i); + old_transactions.erase(old_transactions.begin(), old_transactions.begin() + UnsafeNumericCast(i)); } } From a75fc8a2e8677528179081fb452fe5827511968a Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 12 Apr 2024 15:12:59 +0200 Subject: [PATCH 397/603] format --- .../operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp index 82cf1f846822..34e4499ffadf 100644 --- a/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp +++ b/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp @@ -65,7 +65,7 @@ bool CSVBufferManager::ReadNextAndCacheIt() { shared_ptr CSVBufferManager::GetBuffer(const idx_t pos) { lock_guard parallel_lock(main_mutex); if (pos == 0 && done && cached_buffers.empty()) { - if (is_pipe){ + if (is_pipe) { throw InvalidInputException("Recursive CTEs are not allowed when using piped csv files"); } // This is a recursive CTE, we have to reset out whole buffer From 92141c56cece1981ef1e7bbf58a370250ebec5b1 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 12 Apr 2024 15:18:13 +0200 Subject: [PATCH 398/603] expand the keyword arguments in the stubs --- tools/pythonpkg/duckdb-stubs/__init__.pyi | 188 +++++++++--------- .../pythonpkg/scripts/connection_methods.json | 2 +- .../scripts/generate_connection_stubs.py | 12 +- .../generate_connection_wrapper_stubs.py | 4 +- 4 files changed, 105 insertions(+), 101 deletions(-) diff --git a/tools/pythonpkg/duckdb-stubs/__init__.pyi b/tools/pythonpkg/duckdb-stubs/__init__.pyi index f0630284282e..ff708ef930d5 100644 --- a/tools/pythonpkg/duckdb-stubs/__init__.pyi +++ b/tools/pythonpkg/duckdb-stubs/__init__.pyi @@ -269,7 +269,7 @@ class DuckDBPyConnection: def unregister_filesystem(self, name: str) -> None: ... def list_filesystems(self) -> list: ... def filesystem_is_registered(self, name: str) -> bool: ... - def create_function(self, name: str, function: function, parameters: Optional[List[DuckDBPyType]] = None, return_type: Optional[DuckDBPyType] = None, **kwargs) -> DuckDBPyConnection: ... + def create_function(self, name: str, function: function, parameters: Optional[List[DuckDBPyType]] = None, return_type: Optional[DuckDBPyType] = None, *, type: Optional[PythonUDFType] = PythonUDFType.NATIVE, null_handling: Optional[FunctionNullHandling] = FunctionNullHandling.DEFAULT, exception_handling: Optional[PythonExceptionHandling] = PythonExceptionHandling.DEFAULT, side_effects: bool = False) -> DuckDBPyConnection: ... def remove_function(self, name: str) -> DuckDBPyConnection: ... def sqltype(self, type_str: str) -> DuckDBPyType: ... def dtype(self, type_str: str) -> DuckDBPyType: ... @@ -292,10 +292,10 @@ class DuckDBPyConnection: def fetchmany(self, size: int = 1) -> List[Any]: ... def fetchall(self) -> List[Any]: ... def fetchnumpy(self) -> dict: ... - def fetchdf(self, **kwargs) -> pandas.DataFrame: ... - def fetch_df(self, **kwargs) -> pandas.DataFrame: ... - def df(self, **kwargs) -> pandas.DataFrame: ... - def fetch_df_chunk(self, vectors_per_chunk: int = 1, **kwargs) -> pandas.DataFrame: ... + def fetchdf(self, *, date_as_object: bool = False) -> pandas.DataFrame: ... + def fetch_df(self, *, date_as_object: bool = False) -> pandas.DataFrame: ... + def df(self, *, date_as_object: bool = False) -> pandas.DataFrame: ... + def fetch_df_chunk(self, vectors_per_chunk: int = 1, *, date_as_object: bool = False) -> pandas.DataFrame: ... def pl(self, rows_per_batch: int = 1000000) -> polars.DataFrame: ... def fetch_arrow_table(self, rows_per_batch: int = 1000000) -> pyarrow.lib.Table: ... def arrow(self, rows_per_batch: int = 1000000) -> pyarrow.lib.Table: ... @@ -305,30 +305,30 @@ class DuckDBPyConnection: def begin(self) -> DuckDBPyConnection: ... def commit(self) -> DuckDBPyConnection: ... def rollback(self) -> DuckDBPyConnection: ... - def append(self, table_name: str, df: pandas.DataFrame, **kwargs) -> DuckDBPyConnection: ... + def append(self, table_name: str, df: pandas.DataFrame, *, by_name: bool = False) -> DuckDBPyConnection: ... def register(self, view_name: str, python_object: object) -> DuckDBPyConnection: ... def unregister(self, view_name: str) -> DuckDBPyConnection: ... def table(self, table_name: str) -> DuckDBPyRelation: ... def view(self, view_name: str) -> DuckDBPyRelation: ... def values(self, values: List[Any]) -> DuckDBPyRelation: ... def table_function(self, name: str, parameters: object = None) -> DuckDBPyRelation: ... - def read_json(self, name: str, **kwargs) -> DuckDBPyRelation: ... + def read_json(self, name: str, *, columns: Optional[Dict[str,str]] = None, sample_size: Optional[int] = None, maximum_depth: Optional[int] = None, records: Optional[str] = None, format: Optional[str] = None) -> DuckDBPyRelation: ... def extract_statements(self, query: str) -> List[Statement]: ... - def sql(self, query: str, **kwargs) -> DuckDBPyRelation: ... - def query(self, query: str, **kwargs) -> DuckDBPyRelation: ... - def from_query(self, query: str, **kwargs) -> DuckDBPyRelation: ... - def read_csv(self, path_or_buffer: Union[str, StringIO, TextIOBase], **kwargs) -> DuckDBPyRelation: ... - def from_csv_auto(self, path_or_buffer: Union[str, StringIO, TextIOBase], **kwargs) -> DuckDBPyRelation: ... + def sql(self, query: str, *, alias: str = "", params: object = None) -> DuckDBPyRelation: ... + def query(self, query: str, *, alias: str = "", params: object = None) -> DuckDBPyRelation: ... + def from_query(self, query: str, *, alias: str = "", params: object = None) -> DuckDBPyRelation: ... + def read_csv(self, path_or_buffer: Union[str, StringIO, TextIOBase], *, header: Optional[bool | int] = None, compression: Optional[str] = None, sep: Optional[str] = None, delimiter: Optional[str] = None, dtype: Optional[Dict[str, str] | List[str]] = None, na_values: Optional[str] = None, skiprows: Optional[int] = None, quotechar: Optional[str] = None, escapechar: Optional[str] = None, encoding: Optional[str] = None, parallel: Optional[bool] = None, date_format: Optional[str] = None, timestamp_format: Optional[str] = None, sample_size: Optional[int] = None, all_varchar: Optional[bool] = None, normalize_names: Optional[bool] = None, filename: Optional[bool] = None, null_padding: Optional[bool] = None, names: Optional[List[str]] = None) -> DuckDBPyRelation: ... + def from_csv_auto(self, path_or_buffer: Union[str, StringIO, TextIOBase], *, header: Optional[bool | int] = None, compression: Optional[str] = None, sep: Optional[str] = None, delimiter: Optional[str] = None, dtype: Optional[Dict[str, str] | List[str]] = None, na_values: Optional[str] = None, skiprows: Optional[int] = None, quotechar: Optional[str] = None, escapechar: Optional[str] = None, encoding: Optional[str] = None, parallel: Optional[bool] = None, date_format: Optional[str] = None, timestamp_format: Optional[str] = None, sample_size: Optional[int] = None, all_varchar: Optional[bool] = None, normalize_names: Optional[bool] = None, filename: Optional[bool] = None, null_padding: Optional[bool] = None, names: Optional[List[str]] = None) -> DuckDBPyRelation: ... def from_df(self, df: pandas.DataFrame) -> DuckDBPyRelation: ... def from_arrow(self, arrow_object: object) -> DuckDBPyRelation: ... - def from_parquet(self, file_glob: str, binary_as_string: bool = False, **kwargs) -> DuckDBPyRelation: ... - def read_parquet(self, file_glob: str, binary_as_string: bool = False, **kwargs) -> DuckDBPyRelation: ... + def from_parquet(self, file_glob: str, binary_as_string: bool = False, *, file_row_number: bool = False, filename: bool = False, hive_partitioning: bool = False, union_by_name: bool = False, compression: Optional[str] = None) -> DuckDBPyRelation: ... + def read_parquet(self, file_glob: str, binary_as_string: bool = False, *, file_row_number: bool = False, filename: bool = False, hive_partitioning: bool = False, union_by_name: bool = False, compression: Optional[str] = None) -> DuckDBPyRelation: ... def from_substrait(self, proto: str) -> DuckDBPyRelation: ... - def get_substrait(self, query: str, **kwargs) -> str: ... - def get_substrait_json(self, query: str, **kwargs) -> str: ... + def get_substrait(self, query: str, *, enable_optimizer: bool = True) -> str: ... + def get_substrait_json(self, query: str, *, enable_optimizer: bool = True) -> str: ... def from_substrait_json(self, json: str) -> DuckDBPyRelation: ... def get_table_names(self, query: str) -> List[str]: ... - def install_extension(self, extension: str, **kwargs) -> None: ... + def install_extension(self, extension: str, *, force_install: bool = False) -> None: ... def load_extension(self, extension: str) -> None: ... # END OF CONNECTION METHODS @@ -585,81 +585,81 @@ def tokenize(query: str) -> List[Any]: ... # Do not edit this section manually, your changes will be overwritten! # START OF CONNECTION WRAPPER -def cursor(**kwargs) -> DuckDBPyConnection: ... -def register_filesystem(filesystem: str, **kwargs) -> None: ... -def unregister_filesystem(name: str, **kwargs) -> None: ... -def list_filesystems(**kwargs) -> list: ... -def filesystem_is_registered(name: str, **kwargs) -> bool: ... -def create_function(name: str, function: function, parameters: Optional[List[DuckDBPyType]] = None, return_type: Optional[DuckDBPyType] = None, **kwargs) -> DuckDBPyConnection: ... -def remove_function(name: str, **kwargs) -> DuckDBPyConnection: ... -def sqltype(type_str: str, **kwargs) -> DuckDBPyType: ... -def dtype(type_str: str, **kwargs) -> DuckDBPyType: ... -def type(type_str: str, **kwargs) -> DuckDBPyType: ... -def array_type(type: DuckDBPyType, size: int, **kwargs) -> DuckDBPyType: ... -def list_type(type: DuckDBPyType, **kwargs) -> DuckDBPyType: ... -def union_type(members: DuckDBPyType, **kwargs) -> DuckDBPyType: ... -def string_type(collation: str = "", **kwargs) -> DuckDBPyType: ... -def enum_type(name: str, type: DuckDBPyType, values: List[Any], **kwargs) -> DuckDBPyType: ... -def decimal_type(width: int, scale: int, **kwargs) -> DuckDBPyType: ... -def struct_type(fields: Union[Dict[str, DuckDBPyType], List[str]], **kwargs) -> DuckDBPyType: ... -def row_type(fields: Union[Dict[str, DuckDBPyType], List[str]], **kwargs) -> DuckDBPyType: ... -def map_type(key: DuckDBPyType, value: DuckDBPyType, **kwargs) -> DuckDBPyType: ... -def duplicate(**kwargs) -> DuckDBPyConnection: ... -def execute(query: object, parameters: object = None, multiple_parameter_sets: bool = False, **kwargs) -> DuckDBPyConnection: ... -def executemany(query: object, parameters: object = None, **kwargs) -> DuckDBPyConnection: ... -def close(**kwargs) -> None: ... -def interrupt(**kwargs) -> None: ... -def fetchone(**kwargs) -> Optional[tuple]: ... -def fetchmany(size: int = 1, **kwargs) -> List[Any]: ... -def fetchall(**kwargs) -> List[Any]: ... -def fetchnumpy(**kwargs) -> dict: ... -def fetchdf(**kwargs) -> pandas.DataFrame: ... -def fetch_df(**kwargs) -> pandas.DataFrame: ... -def df(**kwargs) -> pandas.DataFrame: ... -def fetch_df_chunk(vectors_per_chunk: int = 1, **kwargs) -> pandas.DataFrame: ... -def pl(rows_per_batch: int = 1000000, **kwargs) -> polars.DataFrame: ... -def fetch_arrow_table(rows_per_batch: int = 1000000, **kwargs) -> pyarrow.lib.Table: ... -def arrow(rows_per_batch: int = 1000000, **kwargs) -> pyarrow.lib.Table: ... -def fetch_record_batch(rows_per_batch: int = 1000000, **kwargs) -> pyarrow.lib.RecordBatchReader: ... -def torch(**kwargs) -> dict: ... -def tf(**kwargs) -> dict: ... -def begin(**kwargs) -> DuckDBPyConnection: ... -def commit(**kwargs) -> DuckDBPyConnection: ... -def rollback(**kwargs) -> DuckDBPyConnection: ... -def append(table_name: str, df: pandas.DataFrame, **kwargs) -> DuckDBPyConnection: ... -def register(view_name: str, python_object: object, **kwargs) -> DuckDBPyConnection: ... -def unregister(view_name: str, **kwargs) -> DuckDBPyConnection: ... -def table(table_name: str, **kwargs) -> DuckDBPyRelation: ... -def view(view_name: str, **kwargs) -> DuckDBPyRelation: ... -def values(values: List[Any], **kwargs) -> DuckDBPyRelation: ... -def table_function(name: str, parameters: object = None, **kwargs) -> DuckDBPyRelation: ... -def read_json(name: str, **kwargs) -> DuckDBPyRelation: ... -def extract_statements(query: str, **kwargs) -> List[Statement]: ... -def sql(query: str, **kwargs) -> DuckDBPyRelation: ... -def query(query: str, **kwargs) -> DuckDBPyRelation: ... -def from_query(query: str, **kwargs) -> DuckDBPyRelation: ... -def read_csv(path_or_buffer: Union[str, StringIO, TextIOBase], **kwargs) -> DuckDBPyRelation: ... -def from_csv_auto(path_or_buffer: Union[str, StringIO, TextIOBase], **kwargs) -> DuckDBPyRelation: ... -def from_df(df: pandas.DataFrame, **kwargs) -> DuckDBPyRelation: ... -def from_arrow(arrow_object: object, **kwargs) -> DuckDBPyRelation: ... -def from_parquet(file_glob: str, binary_as_string: bool = False, **kwargs) -> DuckDBPyRelation: ... -def read_parquet(file_glob: str, binary_as_string: bool = False, **kwargs) -> DuckDBPyRelation: ... -def from_substrait(proto: str, **kwargs) -> DuckDBPyRelation: ... -def get_substrait(query: str, **kwargs) -> str: ... -def get_substrait_json(query: str, **kwargs) -> str: ... -def from_substrait_json(json: str, **kwargs) -> DuckDBPyRelation: ... -def get_table_names(query: str, **kwargs) -> List[str]: ... -def install_extension(extension: str, **kwargs) -> None: ... -def load_extension(extension: str, **kwargs) -> None: ... -def project(df: pandas.DataFrame, project_expr: str, **kwargs) -> DuckDBPyRelation: ... -def distinct(df: pandas.DataFrame, **kwargs) -> DuckDBPyRelation: ... -def write_csv(df: pandas.DataFrame, *args: Any, **kwargs) -> None: ... -def aggregate(df: pandas.DataFrame, aggr_expr: str, group_expr: str = "", **kwargs) -> DuckDBPyRelation: ... -def alias(df: pandas.DataFrame, alias: str, **kwargs) -> DuckDBPyRelation: ... -def filter(df: pandas.DataFrame, filter_expr: str, **kwargs) -> DuckDBPyRelation: ... -def limit(df: pandas.DataFrame, n: int, offset: int = 0, **kwargs) -> DuckDBPyRelation: ... -def order(df: pandas.DataFrame, order_expr: str, **kwargs) -> DuckDBPyRelation: ... -def query_df(df: pandas.DataFrame, virtual_table_name: str, sql_query: str, **kwargs) -> DuckDBPyRelation: ... -def description(**kwargs) -> Optional[List[Any]]: ... -def rowcount(**kwargs) -> int: ... +def cursor(*, connection: DuckDBPyConnection) -> DuckDBPyConnection: ... +def register_filesystem(filesystem: str, *, connection: DuckDBPyConnection) -> None: ... +def unregister_filesystem(name: str, *, connection: DuckDBPyConnection) -> None: ... +def list_filesystems(*, connection: DuckDBPyConnection) -> list: ... +def filesystem_is_registered(name: str, *, connection: DuckDBPyConnection) -> bool: ... +def create_function(name: str, function: function, parameters: Optional[List[DuckDBPyType]] = None, return_type: Optional[DuckDBPyType] = None, *, type: Optional[PythonUDFType] = PythonUDFType.NATIVE, null_handling: Optional[FunctionNullHandling] = FunctionNullHandling.DEFAULT, exception_handling: Optional[PythonExceptionHandling] = PythonExceptionHandling.DEFAULT, side_effects: bool = False, connection: DuckDBPyConnection) -> DuckDBPyConnection: ... +def remove_function(name: str, *, connection: DuckDBPyConnection) -> DuckDBPyConnection: ... +def sqltype(type_str: str, *, connection: DuckDBPyConnection) -> DuckDBPyType: ... +def dtype(type_str: str, *, connection: DuckDBPyConnection) -> DuckDBPyType: ... +def type(type_str: str, *, connection: DuckDBPyConnection) -> DuckDBPyType: ... +def array_type(type: DuckDBPyType, size: int, *, connection: DuckDBPyConnection) -> DuckDBPyType: ... +def list_type(type: DuckDBPyType, *, connection: DuckDBPyConnection) -> DuckDBPyType: ... +def union_type(members: DuckDBPyType, *, connection: DuckDBPyConnection) -> DuckDBPyType: ... +def string_type(collation: str = "", *, connection: DuckDBPyConnection) -> DuckDBPyType: ... +def enum_type(name: str, type: DuckDBPyType, values: List[Any], *, connection: DuckDBPyConnection) -> DuckDBPyType: ... +def decimal_type(width: int, scale: int, *, connection: DuckDBPyConnection) -> DuckDBPyType: ... +def struct_type(fields: Union[Dict[str, DuckDBPyType], List[str]], *, connection: DuckDBPyConnection) -> DuckDBPyType: ... +def row_type(fields: Union[Dict[str, DuckDBPyType], List[str]], *, connection: DuckDBPyConnection) -> DuckDBPyType: ... +def map_type(key: DuckDBPyType, value: DuckDBPyType, *, connection: DuckDBPyConnection) -> DuckDBPyType: ... +def duplicate(*, connection: DuckDBPyConnection) -> DuckDBPyConnection: ... +def execute(query: object, parameters: object = None, multiple_parameter_sets: bool = False, *, connection: DuckDBPyConnection) -> DuckDBPyConnection: ... +def executemany(query: object, parameters: object = None, *, connection: DuckDBPyConnection) -> DuckDBPyConnection: ... +def close(*, connection: DuckDBPyConnection) -> None: ... +def interrupt(*, connection: DuckDBPyConnection) -> None: ... +def fetchone(*, connection: DuckDBPyConnection) -> Optional[tuple]: ... +def fetchmany(size: int = 1, *, connection: DuckDBPyConnection) -> List[Any]: ... +def fetchall(*, connection: DuckDBPyConnection) -> List[Any]: ... +def fetchnumpy(*, connection: DuckDBPyConnection) -> dict: ... +def fetchdf(*, date_as_object: bool = False, connection: DuckDBPyConnection) -> pandas.DataFrame: ... +def fetch_df(*, date_as_object: bool = False, connection: DuckDBPyConnection) -> pandas.DataFrame: ... +def df(*, date_as_object: bool = False, connection: DuckDBPyConnection) -> pandas.DataFrame: ... +def fetch_df_chunk(vectors_per_chunk: int = 1, *, date_as_object: bool = False, connection: DuckDBPyConnection) -> pandas.DataFrame: ... +def pl(rows_per_batch: int = 1000000, *, connection: DuckDBPyConnection) -> polars.DataFrame: ... +def fetch_arrow_table(rows_per_batch: int = 1000000, *, connection: DuckDBPyConnection) -> pyarrow.lib.Table: ... +def arrow(rows_per_batch: int = 1000000, *, connection: DuckDBPyConnection) -> pyarrow.lib.Table: ... +def fetch_record_batch(rows_per_batch: int = 1000000, *, connection: DuckDBPyConnection) -> pyarrow.lib.RecordBatchReader: ... +def torch(*, connection: DuckDBPyConnection) -> dict: ... +def tf(*, connection: DuckDBPyConnection) -> dict: ... +def begin(*, connection: DuckDBPyConnection) -> DuckDBPyConnection: ... +def commit(*, connection: DuckDBPyConnection) -> DuckDBPyConnection: ... +def rollback(*, connection: DuckDBPyConnection) -> DuckDBPyConnection: ... +def append(table_name: str, df: pandas.DataFrame, *, by_name: bool = False, connection: DuckDBPyConnection) -> DuckDBPyConnection: ... +def register(view_name: str, python_object: object, *, connection: DuckDBPyConnection) -> DuckDBPyConnection: ... +def unregister(view_name: str, *, connection: DuckDBPyConnection) -> DuckDBPyConnection: ... +def table(table_name: str, *, connection: DuckDBPyConnection) -> DuckDBPyRelation: ... +def view(view_name: str, *, connection: DuckDBPyConnection) -> DuckDBPyRelation: ... +def values(values: List[Any], *, connection: DuckDBPyConnection) -> DuckDBPyRelation: ... +def table_function(name: str, parameters: object = None, *, connection: DuckDBPyConnection) -> DuckDBPyRelation: ... +def read_json(name: str, *, columns: Optional[Dict[str,str]] = None, sample_size: Optional[int] = None, maximum_depth: Optional[int] = None, records: Optional[str] = None, format: Optional[str] = None, connection: DuckDBPyConnection) -> DuckDBPyRelation: ... +def extract_statements(query: str, *, connection: DuckDBPyConnection) -> List[Statement]: ... +def sql(query: str, *, alias: str = "", params: object = None, connection: DuckDBPyConnection) -> DuckDBPyRelation: ... +def query(query: str, *, alias: str = "", params: object = None, connection: DuckDBPyConnection) -> DuckDBPyRelation: ... +def from_query(query: str, *, alias: str = "", params: object = None, connection: DuckDBPyConnection) -> DuckDBPyRelation: ... +def read_csv(path_or_buffer: Union[str, StringIO, TextIOBase], *, header: Optional[bool | int] = None, compression: Optional[str] = None, sep: Optional[str] = None, delimiter: Optional[str] = None, dtype: Optional[Dict[str, str] | List[str]] = None, na_values: Optional[str] = None, skiprows: Optional[int] = None, quotechar: Optional[str] = None, escapechar: Optional[str] = None, encoding: Optional[str] = None, parallel: Optional[bool] = None, date_format: Optional[str] = None, timestamp_format: Optional[str] = None, sample_size: Optional[int] = None, all_varchar: Optional[bool] = None, normalize_names: Optional[bool] = None, filename: Optional[bool] = None, null_padding: Optional[bool] = None, names: Optional[List[str]] = None, connection: DuckDBPyConnection) -> DuckDBPyRelation: ... +def from_csv_auto(path_or_buffer: Union[str, StringIO, TextIOBase], *, header: Optional[bool | int] = None, compression: Optional[str] = None, sep: Optional[str] = None, delimiter: Optional[str] = None, dtype: Optional[Dict[str, str] | List[str]] = None, na_values: Optional[str] = None, skiprows: Optional[int] = None, quotechar: Optional[str] = None, escapechar: Optional[str] = None, encoding: Optional[str] = None, parallel: Optional[bool] = None, date_format: Optional[str] = None, timestamp_format: Optional[str] = None, sample_size: Optional[int] = None, all_varchar: Optional[bool] = None, normalize_names: Optional[bool] = None, filename: Optional[bool] = None, null_padding: Optional[bool] = None, names: Optional[List[str]] = None, connection: DuckDBPyConnection) -> DuckDBPyRelation: ... +def from_df(df: pandas.DataFrame, *, connection: DuckDBPyConnection) -> DuckDBPyRelation: ... +def from_arrow(arrow_object: object, *, connection: DuckDBPyConnection) -> DuckDBPyRelation: ... +def from_parquet(file_glob: str, binary_as_string: bool = False, *, file_row_number: bool = False, filename: bool = False, hive_partitioning: bool = False, union_by_name: bool = False, compression: Optional[str] = None, connection: DuckDBPyConnection) -> DuckDBPyRelation: ... +def read_parquet(file_glob: str, binary_as_string: bool = False, *, file_row_number: bool = False, filename: bool = False, hive_partitioning: bool = False, union_by_name: bool = False, compression: Optional[str] = None, connection: DuckDBPyConnection) -> DuckDBPyRelation: ... +def from_substrait(proto: str, *, connection: DuckDBPyConnection) -> DuckDBPyRelation: ... +def get_substrait(query: str, *, enable_optimizer: bool = True, connection: DuckDBPyConnection) -> str: ... +def get_substrait_json(query: str, *, enable_optimizer: bool = True, connection: DuckDBPyConnection) -> str: ... +def from_substrait_json(json: str, *, connection: DuckDBPyConnection) -> DuckDBPyRelation: ... +def get_table_names(query: str, *, connection: DuckDBPyConnection) -> List[str]: ... +def install_extension(extension: str, *, force_install: bool = False, connection: DuckDBPyConnection) -> None: ... +def load_extension(extension: str, *, connection: DuckDBPyConnection) -> None: ... +def project(df: pandas.DataFrame, project_expr: str, *, connection: DuckDBPyConnection) -> DuckDBPyRelation: ... +def distinct(df: pandas.DataFrame, *, connection: DuckDBPyConnection) -> DuckDBPyRelation: ... +def write_csv(df: pandas.DataFrame, *args: Any, connection: DuckDBPyConnection) -> None: ... +def aggregate(df: pandas.DataFrame, aggr_expr: str, group_expr: str = "", *, connection: DuckDBPyConnection) -> DuckDBPyRelation: ... +def alias(df: pandas.DataFrame, alias: str, *, connection: DuckDBPyConnection) -> DuckDBPyRelation: ... +def filter(df: pandas.DataFrame, filter_expr: str, *, connection: DuckDBPyConnection) -> DuckDBPyRelation: ... +def limit(df: pandas.DataFrame, n: int, offset: int = 0, *, connection: DuckDBPyConnection) -> DuckDBPyRelation: ... +def order(df: pandas.DataFrame, order_expr: str, *, connection: DuckDBPyConnection) -> DuckDBPyRelation: ... +def query_df(df: pandas.DataFrame, virtual_table_name: str, sql_query: str, *, connection: DuckDBPyConnection) -> DuckDBPyRelation: ... +def description(*, connection: DuckDBPyConnection) -> Optional[List[Any]]: ... +def rowcount(*, connection: DuckDBPyConnection) -> int: ... # END OF CONNECTION WRAPPER diff --git a/tools/pythonpkg/scripts/connection_methods.json b/tools/pythonpkg/scripts/connection_methods.json index 3184459c7615..06d51903ccc3 100644 --- a/tools/pythonpkg/scripts/connection_methods.json +++ b/tools/pythonpkg/scripts/connection_methods.json @@ -782,7 +782,7 @@ { "name": "compression", "default": "None", - "type": "str" + "type": "Optional[str]" } ], "return": "DuckDBPyRelation" diff --git a/tools/pythonpkg/scripts/generate_connection_stubs.py b/tools/pythonpkg/scripts/generate_connection_stubs.py index 7343ec3ec889..563ade3daefe 100644 --- a/tools/pythonpkg/scripts/generate_connection_stubs.py +++ b/tools/pythonpkg/scripts/generate_connection_stubs.py @@ -52,13 +52,15 @@ def create_arguments(arguments) -> list: return result def create_definition(name, method) -> str: - definition = f"def {name}(self" + definition = f"def {name}(" + arguments = ['self'] if 'args' in method: - definition += ", " - arguments = create_arguments(method['args']) - definition += ', '.join(arguments) + arguments.extend(create_arguments(method['args'])) if 'kwargs' in method: - definition += ", **kwargs" + if not any(x.startswith('*') for x in arguments): + arguments.append("*") + arguments.extend(create_arguments(method['kwargs'])) + definition += ", ".join(arguments) definition += ")" definition += f" -> {method['return']}: ..." return definition diff --git a/tools/pythonpkg/scripts/generate_connection_wrapper_stubs.py b/tools/pythonpkg/scripts/generate_connection_wrapper_stubs.py index 7665a91906fd..51d94db3dc2a 100644 --- a/tools/pythonpkg/scripts/generate_connection_wrapper_stubs.py +++ b/tools/pythonpkg/scripts/generate_connection_wrapper_stubs.py @@ -74,7 +74,9 @@ def create_definition(name, method) -> str: if 'args' in method: arguments.extend(create_arguments(method['args'])) if 'kwargs' in method: - arguments.append("**kwargs") + if not any(x.startswith('*') for x in arguments): + arguments.append("*") + arguments.extend(create_arguments(method['kwargs'])) definition += ', '.join(arguments) definition += ")" definition += f" -> {method['return']}: ..." From c93ab90358f7ded653549391f327ecac41e9fe44 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 12 Apr 2024 15:24:49 +0200 Subject: [PATCH 399/603] handle failing GetAvailableDiskSpace --- src/storage/temporary_file_manager.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/storage/temporary_file_manager.cpp b/src/storage/temporary_file_manager.cpp index d90641bf4ad7..ffb5a75a8644 100644 --- a/src/storage/temporary_file_manager.cpp +++ b/src/storage/temporary_file_manager.cpp @@ -256,7 +256,11 @@ static idx_t GetDefaultMax(const string &path) { auto disk_space = FileSystem::GetAvailableDiskSpace(path); // Use the available disk space // We have made sure that the file exists before we call this, it shouldn't fail - D_ASSERT(disk_space.IsValid()); + if (!disk_space.IsValid()) { + // But if it does (i.e because the system call is not implemented) + // we don't cap the available swap space + return DConstants::INVALID_INDEX - 1; + } // Only use 90% of the available disk space return static_cast(static_cast(disk_space.GetIndex()) * 0.9); } From 7bbd3ac56d230cedf632359c7c36f391fccd7bbd Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 12 Apr 2024 15:28:22 +0200 Subject: [PATCH 400/603] undo breaking changes --- .../catalog_entry/sequence_catalog_entry.cpp | 56 +---- .../duckdb/storage/table_storage_info.hpp | 2 +- .../catalog/sequence/sequence_currvalue.test | 33 --- .../catalog/sequence/sequence_single_use.test | 10 - .../sequence/test_duckdb_sequences.test | 195 ------------------ 5 files changed, 4 insertions(+), 292 deletions(-) delete mode 100644 test/sql/catalog/sequence/sequence_currvalue.test delete mode 100644 test/sql/catalog/sequence/sequence_single_use.test delete mode 100644 test/sql/catalog/sequence/test_duckdb_sequences.test diff --git a/src/catalog/catalog_entry/sequence_catalog_entry.cpp b/src/catalog/catalog_entry/sequence_catalog_entry.cpp index 239fd21a0bac..936fae03854f 100644 --- a/src/catalog/catalog_entry/sequence_catalog_entry.cpp +++ b/src/catalog/catalog_entry/sequence_catalog_entry.cpp @@ -13,59 +13,9 @@ namespace duckdb { -static int64_t ReconstructValue(const CreateSequenceInfo &info, idx_t usage_count) { - int64_t result = info.start_value; - auto to_simulate = usage_count; - if (info.cycle) { - auto current = info.start_value; - auto increase = info.increment > 0; - if (increase) { - while (to_simulate > 0) { - auto maximum_increase = info.max_value - current; - uint64_t max_uses = 1 + (maximum_increase / info.increment); - - if (to_simulate >= max_uses) { - // Uses would overflow, cycle around - to_simulate -= max_uses; - current = info.min_value; - result = current; - } else { - result = current + (info.increment * to_simulate); - break; - } - } - } else { - auto increment = info.increment * -1; - while (to_simulate > 0) { - auto maximum_decrease = current - info.min_value; - uint64_t max_uses = 1 + (maximum_decrease / increment); - - if (to_simulate >= max_uses) { - // Decrementing would overflow, cycle around - to_simulate -= max_uses; - current = info.max_value; - result = current; - } else { - result = current - (increment * to_simulate); - break; - } - } - } - return result; - } else { - // This is guaranteed to be in bounds, otherwise nextval would have thrown trying to create this state - return info.start_value + (info.increment * to_simulate); - } -} - SequenceData::SequenceData(CreateSequenceInfo &info) - : usage_count(info.usage_count), counter(ReconstructValue(info, info.usage_count)), last_value(info.start_value), - increment(info.increment), start_value(info.start_value), min_value(info.min_value), max_value(info.max_value), - cycle(info.cycle) { - - if (info.usage_count) { - last_value = ReconstructValue(info, info.usage_count - 1); - } + : usage_count(info.usage_count), counter(info.start_value), increment(info.increment), + start_value(info.start_value), min_value(info.min_value), max_value(info.max_value), cycle(info.cycle) { } SequenceCatalogEntry::SequenceCatalogEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateSequenceInfo &info) @@ -146,7 +96,7 @@ unique_ptr SequenceCatalogEntry::GetInfo() const { result->increment = seq_data.increment; result->min_value = seq_data.min_value; result->max_value = seq_data.max_value; - result->start_value = seq_data.start_value; + result->start_value = seq_data.counter; result->cycle = seq_data.cycle; result->comment = comment; return std::move(result); diff --git a/src/include/duckdb/storage/table_storage_info.hpp b/src/include/duckdb/storage/table_storage_info.hpp index 49b3089c657a..ea7d378a7262 100644 --- a/src/include/duckdb/storage/table_storage_info.hpp +++ b/src/include/duckdb/storage/table_storage_info.hpp @@ -8,12 +8,12 @@ #pragma once +#include "duckdb/common/optional_idx.hpp" #include "duckdb/common/types/value.hpp" #include "duckdb/common/unordered_map.hpp" #include "duckdb/storage/block.hpp" #include "duckdb/storage/index_storage_info.hpp" #include "duckdb/storage/storage_info.hpp" -#include "duckdb/common/optional_idx.hpp" namespace duckdb { diff --git a/test/sql/catalog/sequence/sequence_currvalue.test b/test/sql/catalog/sequence/sequence_currvalue.test deleted file mode 100644 index 6efa36dbaa3f..000000000000 --- a/test/sql/catalog/sequence/sequence_currvalue.test +++ /dev/null @@ -1,33 +0,0 @@ -# name: test/sql/catalog/sequence/sequence_currvalue.test -# group: [sequence] - -load __TEST_DIR__/sequence_currvalue.db - -statement ok -create sequence seq; - -statement ok -create table tbl (i integer default nextval('seq')) - -statement error -select currval('seq') ----- -Sequence Error: currval: sequence is not yet defined in this session - -statement ok -insert into tbl values(default); - -query I -select currval('seq') ----- -1 - -restart - -# We can't replicate this behavior yet -mode skip - -statement error -select currval('seq') ----- -Sequence Error: currval: sequence is not yet defined in this session diff --git a/test/sql/catalog/sequence/sequence_single_use.test b/test/sql/catalog/sequence/sequence_single_use.test deleted file mode 100644 index 1b4b9086400b..000000000000 --- a/test/sql/catalog/sequence/sequence_single_use.test +++ /dev/null @@ -1,10 +0,0 @@ -# name: test/sql/catalog/sequence/sequence_single_use.test -# group: [sequence] - -statement ok -create sequence seq1 increment 2 minvalue 3 maxvalue 8 start 8; - -query I -select nextval('seq1'); ----- -8 diff --git a/test/sql/catalog/sequence/test_duckdb_sequences.test b/test/sql/catalog/sequence/test_duckdb_sequences.test deleted file mode 100644 index 865e0f69e598..000000000000 --- a/test/sql/catalog/sequence/test_duckdb_sequences.test +++ /dev/null @@ -1,195 +0,0 @@ -# name: test/sql/catalog/sequence/test_duckdb_sequences.test -# group: [sequence] - -statement ok -pragma enable_verification; - -query IIIIIIIIIIIIIII -select * from duckdb_sequences(); ----- - -statement ok -create sequence seq1 start 0 minvalue 0; - -query IIIIII -select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); ----- -0 0 9223372036854775807 1 false NULL - -statement ok -create table tbl (a integer default nextval('seq1')) - -# No change, the sequence hasnt been used yet -query IIIIII -select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); ----- -0 0 9223372036854775807 1 false NULL - -statement ok -insert into tbl values(default); - -# value is 0 -query IIIIII -select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); ----- -0 0 9223372036854775807 1 false 0 - -# value is 1 -statement ok -insert into tbl values(default); - -query IIIIII -select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); ----- -0 0 9223372036854775807 1 false 1 - -statement ok -drop sequence seq1 cascade; - -statement ok -create sequence seq1 start 20316564; - -query IIIIII -select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); ----- -20316564 1 9223372036854775807 1 false NULL - -statement ok -create table tbl (a integer DEFAULT nextval('seq1')) - -statement ok -insert into tbl values(default); - -query IIIIII -select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); ----- -20316564 1 9223372036854775807 1 false 20316564 - -# Reached max value (no cycle) - -statement ok -drop sequence seq1 cascade; - -statement ok -create sequence seq1 increment 2 minvalue 3 maxvalue 8 start 5 - -statement ok -create table tbl (a integer DEFAULT nextval('seq1')) - -statement ok -insert into tbl values(default); - -statement ok -insert into tbl values(default); - -query IIIIII -select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); ----- -5 3 8 2 false 7 - -statement error -insert into tbl values(default); ----- -Sequence Error: nextval: reached maximum value of sequence "seq1" (8) - -query IIIIII -select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); ----- -5 3 8 2 false 7 - -# Reached min value (no cycle) - -statement ok -drop sequence seq1 cascade; - -statement ok -create sequence seq1 increment -2 minvalue 3 maxvalue 8 start 5 - -statement ok -create table tbl (a integer DEFAULT nextval('seq1')) - -statement ok -insert into tbl values(default); - -statement ok -insert into tbl values(default); - -query IIIIII -select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); ----- -5 3 8 -2 false 3 - -statement error -insert into tbl values(default); ----- -Sequence Error: nextval: reached minimum value of sequence "seq1" (3) - -query IIIIII -select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); ----- -5 3 8 -2 false 3 - -# Cycle (positive) - -statement ok -drop sequence seq1 cascade; - -statement ok -create sequence seq1 increment 2 minvalue 3 maxvalue 8 start 5 cycle - -statement ok -create table tbl (a integer DEFAULT nextval('seq1')) - -# value is 5 here -statement ok -insert into tbl values(default); - -# value is 7 here -statement ok -insert into tbl values(default); - -query IIIIII -select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); ----- -5 3 8 2 true 7 - -# value is min_value (3) -statement ok -insert into tbl values(default); - -query IIIIII -select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); ----- -5 3 8 2 true 3 - -# Cycle (negative) - -statement ok -drop sequence seq1 cascade; - -statement ok -create sequence seq1 increment -2 minvalue 3 maxvalue 8 start 5 cycle - -statement ok -create table tbl (a integer DEFAULT nextval('seq1')) - -# value is 5 -statement ok -insert into tbl values(default); - -# value is 3 -statement ok -insert into tbl values(default); - -query IIIIII -select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); ----- -5 3 8 -2 true 3 - -statement ok -insert into tbl values(default); - -query IIIIII -select start_value, min_value, max_value, increment_by, cycle, last_value from duckdb_sequences(); ----- -5 3 8 -2 true 8 From 92cc79039991c03722b2f0b7abed9e64dd4b21e6 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 12 Apr 2024 15:33:25 +0200 Subject: [PATCH 401/603] add test for last_value --- .../catalog/sequence/test_duckdb_sequences.test | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 test/sql/catalog/sequence/test_duckdb_sequences.test diff --git a/test/sql/catalog/sequence/test_duckdb_sequences.test b/test/sql/catalog/sequence/test_duckdb_sequences.test new file mode 100644 index 000000000000..6bcf21fc14d9 --- /dev/null +++ b/test/sql/catalog/sequence/test_duckdb_sequences.test @@ -0,0 +1,17 @@ +# name: test/sql/catalog/sequence/test_duckdb_sequences.test +# group: [sequence] + +require noforcestorage + +statement ok +create sequence my_seq; + +query I +select nextval('my_seq'); +---- +1 + +query I +select last_value from duckdb_sequences(); +---- +1 From a6d3e8f22b66ac7a67f1a470eedcd5513c7d00a1 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Fri, 12 Apr 2024 15:33:46 +0200 Subject: [PATCH 402/603] Check overflow for hugeint in Abs function correctly --- src/core_functions/scalar/math/numeric.cpp | 3 ++- src/include/duckdb/common/operator/abs.hpp | 16 ++++++++++++---- test/fuzzer/sqlsmith/test_abs_overflow.test | 4 ++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/core_functions/scalar/math/numeric.cpp b/src/core_functions/scalar/math/numeric.cpp index 40a2f520d48a..711f92608aec 100644 --- a/src/core_functions/scalar/math/numeric.cpp +++ b/src/core_functions/scalar/math/numeric.cpp @@ -167,7 +167,8 @@ ScalarFunctionSet AbsOperatorFun::GetFunctions() { case LogicalTypeId::TINYINT: case LogicalTypeId::SMALLINT: case LogicalTypeId::INTEGER: - case LogicalTypeId::BIGINT: { + case LogicalTypeId::BIGINT: + case LogicalTypeId::HUGEINT: { ScalarFunction func({type}, type, ScalarFunction::GetScalarUnaryFunction(type)); func.statistics = PropagateAbsStats; abs.AddFunction(func); diff --git a/src/include/duckdb/common/operator/abs.hpp b/src/include/duckdb/common/operator/abs.hpp index 26f48726d071..15a123132ea5 100644 --- a/src/include/duckdb/common/operator/abs.hpp +++ b/src/include/duckdb/common/operator/abs.hpp @@ -39,7 +39,7 @@ inline int8_t TryAbsOperator::Operation(int8_t input) { if (input == NumericLimits::Minimum()) { throw OutOfRangeException("Overflow on abs(%d)", input); } - return input < 0 ? -input : input; + return AbsOperator::Operation(input); } template <> @@ -47,7 +47,7 @@ inline int16_t TryAbsOperator::Operation(int16_t input) { if (input == NumericLimits::Minimum()) { throw OutOfRangeException("Overflow on abs(%d)", input); } - return input < 0 ? -input : input; + return AbsOperator::Operation(input); } template <> @@ -55,7 +55,7 @@ inline int32_t TryAbsOperator::Operation(int32_t input) { if (input == NumericLimits::Minimum()) { throw OutOfRangeException("Overflow on abs(%d)", input); } - return input < 0 ? -input : input; + return AbsOperator::Operation(input); } template <> @@ -63,7 +63,15 @@ inline int64_t TryAbsOperator::Operation(int64_t input) { if (input == NumericLimits::Minimum()) { throw OutOfRangeException("Overflow on abs(%d)", input); } - return input < 0 ? -input : input; + return AbsOperator::Operation(input); +} + +template <> +inline hugeint_t TryAbsOperator::Operation(hugeint_t input) { + if (input == NumericLimits::Minimum()) { + throw OutOfRangeException("Overflow on abs(%s)", input.ToString()); + } + return AbsOperator::Operation(input); } template <> diff --git a/test/fuzzer/sqlsmith/test_abs_overflow.test b/test/fuzzer/sqlsmith/test_abs_overflow.test index ebd4182bee34..0e0cd2ac5b30 100644 --- a/test/fuzzer/sqlsmith/test_abs_overflow.test +++ b/test/fuzzer/sqlsmith/test_abs_overflow.test @@ -8,9 +8,9 @@ PRAGMA enable_verification # test abs function on extremes statement ok -CREATE TABLE numerics AS SELECT tinyint, smallint, int, bigint FROM test_all_types(); +CREATE TABLE numerics AS SELECT tinyint, smallint, int, bigint, hugeint FROM test_all_types(); -foreach type tinyint smallint int bigint +foreach type tinyint smallint int bigint hugeint statement error SELECT abs(${type}) FROM numerics; From 7bd0c41e199044890e1570a1f7cd55eba395dc25 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 12 Apr 2024 15:37:39 +0200 Subject: [PATCH 403/603] initialize 'last_value' --- src/catalog/catalog_entry/sequence_catalog_entry.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/catalog/catalog_entry/sequence_catalog_entry.cpp b/src/catalog/catalog_entry/sequence_catalog_entry.cpp index 936fae03854f..5606ddc8ba8c 100644 --- a/src/catalog/catalog_entry/sequence_catalog_entry.cpp +++ b/src/catalog/catalog_entry/sequence_catalog_entry.cpp @@ -14,7 +14,7 @@ namespace duckdb { SequenceData::SequenceData(CreateSequenceInfo &info) - : usage_count(info.usage_count), counter(info.start_value), increment(info.increment), + : usage_count(info.usage_count), counter(info.start_value), last_value(info.start_value), increment(info.increment), start_value(info.start_value), min_value(info.min_value), max_value(info.max_value), cycle(info.cycle) { } From 60a1cdcd182caccfcbe5eea3f1d171fdcb5a438f Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Fri, 12 Apr 2024 15:38:38 +0200 Subject: [PATCH 404/603] Further expand statement simplifier --- extension/sqlsmith/statement_simplifier.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/extension/sqlsmith/statement_simplifier.cpp b/extension/sqlsmith/statement_simplifier.cpp index ea98bd0e1f1d..ff31a6fc1255 100644 --- a/extension/sqlsmith/statement_simplifier.cpp +++ b/extension/sqlsmith/statement_simplifier.cpp @@ -286,6 +286,12 @@ void StatementSimplifier::SimplifyExpression(duckdb::unique_ptrCast(); SimplifyExpressionList(expr, func.children); + SimplifyEnum(func.distinct, false); + SimplifyOptionalExpression(func.filter); + SimplifyOptional(func.order_bys); + if (func.order_bys) { + Simplify(*func.order_bys); + } break; } case ExpressionClass::OPERATOR: { @@ -349,8 +355,8 @@ void StatementSimplifier::SimplifyExpression(duckdb::unique_ptr Date: Fri, 12 Apr 2024 15:41:10 +0200 Subject: [PATCH 405/603] Correctly handle NULL value in quantile_cont --- src/core_functions/aggregate/holistic/quantile.cpp | 3 +++ test/sql/aggregate/quantile_fun.test | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/src/core_functions/aggregate/holistic/quantile.cpp b/src/core_functions/aggregate/holistic/quantile.cpp index 7a7693a6aa27..c82f7dfff0c9 100644 --- a/src/core_functions/aggregate/holistic/quantile.cpp +++ b/src/core_functions/aggregate/holistic/quantile.cpp @@ -1502,6 +1502,9 @@ unique_ptr BindQuantile(ClientContext &context, AggregateFunction throw BinderException("QUANTILE can only take constant parameters"); } Value quantile_val = ExpressionExecutor::EvaluateScalar(context, *arguments[1]); + if (quantile_val.IsNull()) { + throw BinderException("QUANTILE argument must not be NULL"); + } vector quantiles; if (quantile_val.type().id() != LogicalTypeId::LIST) { quantiles.push_back(CheckQuantile(quantile_val)); diff --git a/test/sql/aggregate/quantile_fun.test b/test/sql/aggregate/quantile_fun.test index cf3ce3d5d7b3..36fc4a4a1d8e 100644 --- a/test/sql/aggregate/quantile_fun.test +++ b/test/sql/aggregate/quantile_fun.test @@ -49,3 +49,8 @@ SELECT quantile_cont(r::${type}, [0.15, 0.5, 0.9]) FROM quantiles [14.8, 49.5, 89.1] endloop + +statement error +SELECT quantile_cont(NULL, CAST(NULL AS DOUBLE[])) +---- +not be NULL From fead11cb59373d4d179d8fa51fe2350a6fea342f Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 12 Apr 2024 12:57:55 +0200 Subject: [PATCH 406/603] Partial revert of 432b97983e - no _rtools --- .github/config/out_of_tree_extensions.cmake | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/config/out_of_tree_extensions.cmake b/.github/config/out_of_tree_extensions.cmake index 0f919af9b1e8..443f59fa14df 100644 --- a/.github/config/out_of_tree_extensions.cmake +++ b/.github/config/out_of_tree_extensions.cmake @@ -16,11 +16,13 @@ # VCPKG_TARGET_TRIPLET=arm64-osx ################# ARROW -duckdb_extension_load(arrow - LOAD_TESTS DONT_LINK - GIT_URL https://github.com/duckdb/arrow - GIT_TAG 9e10240da11f61ea7fbfe3fc9988ffe672ccd40f - ) +if (NOT MINGW) + duckdb_extension_load(arrow + LOAD_TESTS DONT_LINK + GIT_URL https://github.com/duckdb/arrow + GIT_TAG 9e10240da11f61ea7fbfe3fc9988ffe672ccd40f + ) +endif() ################## AWS if (NOT MINGW) From 6c0db1908ea3ea884c3925e210702ffeba8d39f9 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 12 Apr 2024 12:59:50 +0200 Subject: [PATCH 407/603] Bump sqlite and remove patches --- .github/config/out_of_tree_extensions.cmake | 3 +-- .../sqlite_scanner/wrap_idx_t.patch | 20 ------------------- 2 files changed, 1 insertion(+), 22 deletions(-) delete mode 100644 .github/patches/extensions/sqlite_scanner/wrap_idx_t.patch diff --git a/.github/config/out_of_tree_extensions.cmake b/.github/config/out_of_tree_extensions.cmake index 443f59fa14df..33c164eb1b17 100644 --- a/.github/config/out_of_tree_extensions.cmake +++ b/.github/config/out_of_tree_extensions.cmake @@ -92,8 +92,7 @@ endif() duckdb_extension_load(sqlite_scanner ${STATIC_LINK_SQLITE} LOAD_TESTS GIT_URL https://github.com/duckdb/sqlite_scanner - GIT_TAG 70a4c411434290b4af704940987a324d84af0024 - APPLY_PATCHES + GIT_TAG 091197efb34579c7195afa43dfb5925023c915c0 ) ################# SUBSTRAIT diff --git a/.github/patches/extensions/sqlite_scanner/wrap_idx_t.patch b/.github/patches/extensions/sqlite_scanner/wrap_idx_t.patch deleted file mode 100644 index 6dfe726431fe..000000000000 --- a/.github/patches/extensions/sqlite_scanner/wrap_idx_t.patch +++ /dev/null @@ -1,20 +0,0 @@ -diff --git a/src/storage/sqlite_table_entry.cpp b/src/storage/sqlite_table_entry.cpp -index 9d83f00..fadbb39 100644 ---- a/src/storage/sqlite_table_entry.cpp -+++ b/src/storage/sqlite_table_entry.cpp -@@ -53,10 +53,14 @@ TableStorageInfo SQLiteTableEntry::GetStorageInfo(ClientContext &context) { - auto &transaction = Transaction::Get(context, catalog).Cast(); - auto &db = transaction.GetDB(); - TableStorageInfo result; -- if (!db.GetMaxRowId(name, result.cardinality)) { -+ -+ idx_t cardinality; -+ if (!db.GetMaxRowId(name, cardinality)) { - // probably - result.cardinality = 10000; - } -+ result.cardinality = cardinality; -+ - result.index_info = db.GetIndexInfo(name); - return result; - } From a15af9f40833af61797ac180a7ea08ee0fd85624 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Fri, 12 Apr 2024 15:51:17 +0200 Subject: [PATCH 408/603] Fuzzer fixes for repeat blobs, clean up "repeat" code --- src/core_functions/scalar/string/repeat.cpp | 33 ++++++++------------- test/sql/function/string/test_repeat.test | 24 +++++++++++++++ 2 files changed, 36 insertions(+), 21 deletions(-) diff --git a/src/core_functions/scalar/string/repeat.cpp b/src/core_functions/scalar/string/repeat.cpp index 551095ea223c..afe3269f94b7 100644 --- a/src/core_functions/scalar/string/repeat.cpp +++ b/src/core_functions/scalar/string/repeat.cpp @@ -1,34 +1,25 @@ -#include "duckdb/common/exception.hpp" #include "duckdb/common/vector_operations/binary_executor.hpp" #include "duckdb/core_functions/scalar/string_functions.hpp" -#include -#include - namespace duckdb { -static string_t RepeatScalarFunction(const string_t &str, const int64_t cnt, vector &result) { - // Get information about the repeated string - auto input_str = str.GetData(); - auto size_str = str.GetSize(); - - // Reuse the buffer - result.clear(); - for (auto remaining = cnt; remaining-- > 0;) { - result.insert(result.end(), input_str, input_str + size_str); - } - - return string_t(result.data(), UnsafeNumericCast(result.size())); -} - -static void RepeatFunction(DataChunk &args, ExpressionState &state, Vector &result) { +static void RepeatFunction(DataChunk &args, ExpressionState &, Vector &result) { auto &str_vector = args.data[0]; auto &cnt_vector = args.data[1]; - vector buffer; BinaryExecutor::Execute( str_vector, cnt_vector, result, args.size(), [&](string_t str, int64_t cnt) { - return StringVector::AddString(result, RepeatScalarFunction(str, cnt, buffer)); + auto input_str = str.GetData(); + auto size_str = str.GetSize(); + + idx_t copy_count = cnt <= 0 || size_str == 0 ? 0 : idx_t(cnt); + auto result_str = StringVector::EmptyString(result, size_str * copy_count); + auto result_data = result_str.GetDataWriteable(); + for (idx_t i = 0; i < copy_count; i++) { + memcpy(result_data + i * size_str, input_str, size_str); + } + result_str.Finalize(); + return result_str; }); } diff --git a/test/sql/function/string/test_repeat.test b/test/sql/function/string/test_repeat.test index bc8c4685b8c9..98b52a98ac96 100644 --- a/test/sql/function/string/test_repeat.test +++ b/test/sql/function/string/test_repeat.test @@ -46,6 +46,30 @@ select REPEAT(a, 4) FROM strings WHERE b IS NOT NULL HelloHelloHelloHello MotörHeadMotörHeadMotörHeadMotörHead +# empty string +query I +SELECT repeat('', 99); +---- +(empty) + +# no repeat +query I +SELECT repeat('hello world', 0); +---- +(empty) + +# negative repeat +query I +SELECT repeat('hello world', -1); +---- +(empty) + +# repeat blob +query I +SELECT repeat(blob '00', 2); +---- +0000 + # test incorrect usage of reverse statement error select REPEAT() From 4cf13fed84d08c297864b78035e7a54f4f278a2d Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Fri, 12 Apr 2024 16:12:55 +0200 Subject: [PATCH 409/603] Fix interaction of LATERAL joins and QUALIFY clause --- .../binder/query_node/bind_select_node.cpp | 30 +++++++++---------- .../sql/subquery/lateral/lateral_qualify.test | 15 ++++++++++ 2 files changed, 30 insertions(+), 15 deletions(-) create mode 100644 test/sql/subquery/lateral/lateral_qualify.test diff --git a/src/planner/binder/query_node/bind_select_node.cpp b/src/planner/binder/query_node/bind_select_node.cpp index 6da8b9d4d9be..70bb67eeb0aa 100644 --- a/src/planner/binder/query_node/bind_select_node.cpp +++ b/src/planner/binder/query_node/bind_select_node.cpp @@ -515,16 +515,19 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ } // bind the QUALIFY clause, if any - unique_ptr qualify_binder; + vector bound_qualify_columns; if (statement.qualify) { if (statement.aggregate_handling == AggregateHandling::FORCE_AGGREGATES) { throw BinderException("Combining QUALIFY with GROUP BY ALL is not supported yet"); } - qualify_binder = make_uniq(*this, context, *result, info); + QualifyBinder qualify_binder(*this, context, *result, info); ExpressionBinder::QualifyColumnNames(*this, statement.qualify); - result->qualify = qualify_binder->Bind(statement.qualify); - if (qualify_binder->HasBoundColumns() && qualify_binder->BoundAggregates()) { - throw BinderException("Cannot mix aggregates with non-aggregated columns!"); + result->qualify = qualify_binder.Bind(statement.qualify); + if (qualify_binder.HasBoundColumns()) { + if (qualify_binder.BoundAggregates()) { + throw BinderException("Cannot mix aggregates with non-aggregated columns!"); + } + bound_qualify_columns = qualify_binder.GetBoundColumns(); } } @@ -625,17 +628,14 @@ unique_ptr Binder::BindSelectNode(SelectNode &statement, unique_ if (statement.aggregate_handling == AggregateHandling::NO_AGGREGATES_ALLOWED) { throw BinderException("Aggregates cannot be present in a Project relation!"); } else { - vector> to_check_binders; - to_check_binders.push_back(select_binder); - if (qualify_binder) { - to_check_binders.push_back(*qualify_binder); + vector bound_columns; + if (select_binder.HasBoundColumns()) { + bound_columns = select_binder.GetBoundColumns(); } - for (auto &binder : to_check_binders) { - auto &sel_binder = binder.get(); - if (!sel_binder.HasBoundColumns()) { - continue; - } - auto &bound_columns = sel_binder.GetBoundColumns(); + for (auto &bound_qualify_col : bound_qualify_columns) { + bound_columns.push_back(bound_qualify_col); + } + if (!bound_columns.empty()) { string error; error = "column \"%s\" must appear in the GROUP BY clause or must be part of an aggregate function."; if (statement.aggregate_handling == AggregateHandling::FORCE_AGGREGATES) { diff --git a/test/sql/subquery/lateral/lateral_qualify.test b/test/sql/subquery/lateral/lateral_qualify.test new file mode 100644 index 000000000000..983b10c1f525 --- /dev/null +++ b/test/sql/subquery/lateral/lateral_qualify.test @@ -0,0 +1,15 @@ +# name: test/sql/subquery/lateral/lateral_qualify.test +# description: Test LATERAL joins and QUALIFY clause interaction +# group: [lateral] + +statement ok +PRAGMA enable_verification + +query III +FROM (SELECT 42) t(x), (SELECT x, row_number() OVER () QUALIFY NULL); +---- + +query II +FROM (SELECT 42) t(x), (SELECT x * 2 QUALIFY row_number() OVER () < 10); +---- +42 84 From a8fcbe9b592243df1aed17722e75269213f2484d Mon Sep 17 00:00:00 2001 From: Florian Gerlinghoff Date: Fri, 12 Apr 2024 18:14:50 +0200 Subject: [PATCH 410/603] Make path to append_metadata.cmake relative to top-level CMakeLists.txt --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f071e73d212d..3b97d9e1597e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -820,7 +820,7 @@ function(build_loadable_extension_directory NAME OUTPUT_DIRECTORY EXTENSION_VERS TARGET ${TARGET_NAME} POST_BUILD COMMAND - ${CMAKE_COMMAND} -DEXTENSION=$ -DPLATFORM_FILE=${CMAKE_BINARY_DIR}/duckdb_platform_out -DDUCKDB_VERSION="${DUCKDB_NORMALIZED_VERSION}" -DEXTENSION_VERSION="${EXTENSION_VERSION}" -DNULL_FILE=${CMAKE_SOURCE_DIR}/scripts/null.txt -P ${CMAKE_SOURCE_DIR}/scripts/append_metadata.cmake + ${CMAKE_COMMAND} -DEXTENSION=$ -DPLATFORM_FILE=${CMAKE_BINARY_DIR}/duckdb_platform_out -DDUCKDB_VERSION="${DUCKDB_NORMALIZED_VERSION}" -DEXTENSION_VERSION="${EXTENSION_VERSION}" -DNULL_FILE=${CMAKE_CURRENT_FUNCTION_LIST_DIR}/scripts/null.txt -P ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/scripts/append_metadata.cmake ) add_dependencies(${TARGET_NAME} duckdb_platform) if (NOT EXTENSION_CONFIG_BUILD AND NOT ${EXTENSION_TESTS_ONLY} AND NOT CLANG_TIDY) From 51c52e4866ad6c48256fe12b9d7950f541fd2d4e Mon Sep 17 00:00:00 2001 From: Florian Gerlinghoff Date: Fri, 12 Apr 2024 18:31:59 +0200 Subject: [PATCH 411/603] Use top-level duckdb binary directory when looking for duckdb_platform_out file --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3b97d9e1597e..4c0a94696580 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -820,7 +820,7 @@ function(build_loadable_extension_directory NAME OUTPUT_DIRECTORY EXTENSION_VERS TARGET ${TARGET_NAME} POST_BUILD COMMAND - ${CMAKE_COMMAND} -DEXTENSION=$ -DPLATFORM_FILE=${CMAKE_BINARY_DIR}/duckdb_platform_out -DDUCKDB_VERSION="${DUCKDB_NORMALIZED_VERSION}" -DEXTENSION_VERSION="${EXTENSION_VERSION}" -DNULL_FILE=${CMAKE_CURRENT_FUNCTION_LIST_DIR}/scripts/null.txt -P ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/scripts/append_metadata.cmake + ${CMAKE_COMMAND} -DEXTENSION=$ -DPLATFORM_FILE=${DuckDB_BINARY_DIR}/duckdb_platform_out -DDUCKDB_VERSION="${DUCKDB_NORMALIZED_VERSION}" -DEXTENSION_VERSION="${EXTENSION_VERSION}" -DNULL_FILE=${CMAKE_CURRENT_FUNCTION_LIST_DIR}/scripts/null.txt -P ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/scripts/append_metadata.cmake ) add_dependencies(${TARGET_NAME} duckdb_platform) if (NOT EXTENSION_CONFIG_BUILD AND NOT ${EXTENSION_TESTS_ONLY} AND NOT CLANG_TIDY) From cd749b0083aaa3136d87c1a8859f47293a28ece6 Mon Sep 17 00:00:00 2001 From: Xiaoying Wang Date: Fri, 12 Apr 2024 15:46:13 -0700 Subject: [PATCH 412/603] fix #11638 filter out single relation predicates before join ordering --- .../duckdb/optimizer/join_order/cardinality_estimator.hpp | 2 +- src/optimizer/join_order/cardinality_estimator.cpp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp b/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp index a8a6057492ef..e4862ff57855 100644 --- a/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +++ b/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp @@ -82,7 +82,7 @@ class CardinalityEstimator { void PrintRelationToTdomInfo(); private: - bool SingleColumnFilter(FilterInfo &filter_info); + bool SingleRelationFilter(FilterInfo &filter_info); vector DetermineMatchingEquivalentSets(FilterInfo *filter_info); //! Given a filter, add the column bindings to the matching equivalent set at the index //! given in matching equivalent sets. diff --git a/src/optimizer/join_order/cardinality_estimator.cpp b/src/optimizer/join_order/cardinality_estimator.cpp index 18fd5c858386..669c05668d02 100644 --- a/src/optimizer/join_order/cardinality_estimator.cpp +++ b/src/optimizer/join_order/cardinality_estimator.cpp @@ -37,9 +37,9 @@ void CardinalityEstimator::AddRelationTdom(FilterInfo &filter_info) { relations_to_tdoms.emplace_back(new_r2tdom); } -bool CardinalityEstimator::SingleColumnFilter(FilterInfo &filter_info) { - if (filter_info.left_set && filter_info.right_set) { - // Both set +bool CardinalityEstimator::SingleRelationFilter(FilterInfo &filter_info) { + if (filter_info.left_set && filter_info.right_set && filter_info.set.count > 1) { + // Both set and are from different relations return false; } if (EmptyFilter(filter_info)) { @@ -100,7 +100,7 @@ void CardinalityEstimator::InitEquivalentRelations(const vector Date: Sat, 13 Apr 2024 09:23:32 +0200 Subject: [PATCH 413/603] sum_no_overflow is now for internal usage only - it can no longer be manually called --- .../aggregate/distributive/sum.cpp | 20 ++++++++----------- .../aggregate/distributive_functions.hpp | 2 +- test/sql/copy/parquet/parquet_hive2.test | 3 --- .../function/list/lambda_constant_null.test | 2 +- test/sql/window/test_empty_frames.test | 2 +- test/sqlsmith/sql_reduce.test | 5 +++++ 6 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/core_functions/aggregate/distributive/sum.cpp b/src/core_functions/aggregate/distributive/sum.cpp index 0d855297e91d..350f22d9d175 100644 --- a/src/core_functions/aggregate/distributive/sum.cpp +++ b/src/core_functions/aggregate/distributive/sum.cpp @@ -69,6 +69,11 @@ struct HugeintSumOperation : public BaseSumOperation SumNoOverflowBind(ClientContext &context, AggregateFunction &function, + vector> &arguments) { + throw BinderException("sum_no_overflow is for internal use only!"); +} + AggregateFunction GetSumAggregateNoOverflow(PhysicalType type) { switch (type) { case PhysicalType::INT32: { @@ -76,6 +81,7 @@ AggregateFunction GetSumAggregateNoOverflow(PhysicalType type) { LogicalType::INTEGER, LogicalType::HUGEINT); function.name = "sum_no_overflow"; function.order_dependent = AggregateOrderDependent::NOT_ORDER_DEPENDENT; + function.bind = SumNoOverflowBind; return function; } case PhysicalType::INT64: { @@ -83,6 +89,7 @@ AggregateFunction GetSumAggregateNoOverflow(PhysicalType type) { LogicalType::BIGINT, LogicalType::HUGEINT); function.name = "sum_no_overflow"; function.order_dependent = AggregateOrderDependent::NOT_ORDER_DEPENDENT; + function.bind = SumNoOverflowBind; return function; } default: @@ -173,17 +180,6 @@ unique_ptr BindDecimalSum(ClientContext &context, AggregateFunctio return nullptr; } -unique_ptr BindDecimalSumNoOverflow(ClientContext &context, AggregateFunction &function, - vector> &arguments) { - auto decimal_type = arguments[0]->return_type; - function = GetSumAggregateNoOverflow(decimal_type.InternalType()); - function.name = "sum_no_overflow"; - function.arguments[0] = decimal_type; - function.return_type = LogicalType::DECIMAL(Decimal::MAX_WIDTH_DECIMAL, DecimalType::GetScale(decimal_type)); - function.order_dependent = AggregateOrderDependent::NOT_ORDER_DEPENDENT; - return nullptr; -} - AggregateFunctionSet SumFun::GetFunctions() { AggregateFunctionSet sum; // decimal @@ -205,7 +201,7 @@ AggregateFunctionSet SumNoOverflowFun::GetFunctions() { sum_no_overflow.AddFunction(GetSumAggregateNoOverflow(PhysicalType::INT64)); sum_no_overflow.AddFunction( AggregateFunction({LogicalTypeId::DECIMAL}, LogicalTypeId::DECIMAL, nullptr, nullptr, nullptr, nullptr, nullptr, - FunctionNullHandling::DEFAULT_NULL_HANDLING, nullptr, BindDecimalSumNoOverflow)); + FunctionNullHandling::DEFAULT_NULL_HANDLING, nullptr, SumNoOverflowBind)); return sum_no_overflow; } diff --git a/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp b/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp index 0510f76a7a0d..3f65ca2666f3 100644 --- a/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +++ b/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp @@ -255,7 +255,7 @@ struct SumFun { struct SumNoOverflowFun { static constexpr const char *Name = "sum_no_overflow"; static constexpr const char *Parameters = "arg"; - static constexpr const char *Description = "Calculates the sum value for all tuples in arg without overflow checks."; + static constexpr const char *Description = "Internal only. Calculates the sum value for all tuples in arg without overflow checks."; static constexpr const char *Example = "sum_no_overflow(A)"; static AggregateFunctionSet GetFunctions(); diff --git a/test/sql/copy/parquet/parquet_hive2.test b/test/sql/copy/parquet/parquet_hive2.test index 0a50ee991e52..9d1e36c08d1f 100644 --- a/test/sql/copy/parquet/parquet_hive2.test +++ b/test/sql/copy/parquet/parquet_hive2.test @@ -17,6 +17,3 @@ copy (select 2000+(v//12)y,m,v,j from orders) TO '__TEST_DIR__/orders_m' (FORMAT statement ok copy (select 2000+(v//12)y,m,v,j from orders) TO '__TEST_DIR__/orders_y' (FORMAT PARQUET, PARTITION_BY (y)); - -statement ok -copy (select 2000+(v//12)y,m,v,j from orders) TO '__TEST_DIR__/orders_ym' (FORMAT PARQUET,PARTITION_BY (y,m)); diff --git a/test/sql/function/list/lambda_constant_null.test b/test/sql/function/list/lambda_constant_null.test index 05b5885d44db..02a584ce2194 100644 --- a/test/sql/function/list/lambda_constant_null.test +++ b/test/sql/function/list/lambda_constant_null.test @@ -5,4 +5,4 @@ statement error select quantile(NULL, filter(NULL, (c103 -> 'babea54a-2261-4b0c-b14b-1d0e9b794e1a'))); ---- -QUANTILE parameter cannot be NULL +QUANTILE argument must not be NULL diff --git a/test/sql/window/test_empty_frames.test b/test/sql/window/test_empty_frames.test index 250f007245cc..05e099674b55 100644 --- a/test/sql/window/test_empty_frames.test +++ b/test/sql/window/test_empty_frames.test @@ -53,7 +53,7 @@ endloop # # All NULLs # -foreach agg any_value arbitrary avg bit_and bit_or bit_xor favg first fsum group_concat histogram kahan_sum kurtosis last listagg mad max mean median min mode product sem skewness stddev stddev_pop stddev_samp sum sum_no_overflow sumkahan var_pop var_samp variance +foreach agg any_value arbitrary avg bit_and bit_or bit_xor favg first fsum group_concat histogram kahan_sum kurtosis last listagg mad max mean median min mode product sem skewness stddev stddev_pop stddev_samp sum sumkahan var_pop var_samp variance query II SELECT id, ${agg}(id) OVER (PARTITION BY ch ORDER BY id ROWS BETWEEN 1 FOLLOWING AND 2 FOLLOWING) diff --git a/test/sqlsmith/sql_reduce.test b/test/sqlsmith/sql_reduce.test index 7b956e8db065..432fc8581bb7 100644 --- a/test/sqlsmith/sql_reduce.test +++ b/test/sqlsmith/sql_reduce.test @@ -13,6 +13,8 @@ SELECT a FROM tbl SELECT a, NULL FROM tbl SELECT a, NULL FROM tbl SELECT a, b +SELECT a, b FROM tbl +SELECT a, b FROM tbl SELECT b FROM tbl query I @@ -28,6 +30,8 @@ SELECT a, b FROM tbl WHERE (NULL AND b) SELECT a, b FROM tbl WHERE (NULL AND b) SELECT a, b FROM tbl WHERE (a AND NULL) SELECT a, b FROM tbl WHERE (a AND NULL) +SELECT a, b FROM tbl WHERE (a AND b) +SELECT a, b FROM tbl WHERE (a AND b) SELECT a, b FROM tbl WHERE NULL SELECT a, b FROM tbl WHERE NULL SELECT a, b FROM tbl WHERE a @@ -41,6 +45,7 @@ query I SELECT * FROM reduce_sql_statement('INSERT INTO tbl VALUES (1, 2)') ORDER BY 1 ---- INSERT INTO tbl (VALUES (1)) +INSERT INTO tbl (VALUES (1, 2)) INSERT INTO tbl (VALUES (2)) INSERT INTO tbl SELECT * INSERT INTO tbl SELECT NULL FROM (VALUES (1, 2)) From aeaf47947a9f4a4da9be040fe561c98c35cb1ecc Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Sat, 13 Apr 2024 09:27:37 +0200 Subject: [PATCH 414/603] Add test back with mode skip --- test/sql/copy/parquet/parquet_hive2.test | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/sql/copy/parquet/parquet_hive2.test b/test/sql/copy/parquet/parquet_hive2.test index 9d1e36c08d1f..5fa4f7cad6a8 100644 --- a/test/sql/copy/parquet/parquet_hive2.test +++ b/test/sql/copy/parquet/parquet_hive2.test @@ -17,3 +17,9 @@ copy (select 2000+(v//12)y,m,v,j from orders) TO '__TEST_DIR__/orders_m' (FORMAT statement ok copy (select 2000+(v//12)y,m,v,j from orders) TO '__TEST_DIR__/orders_y' (FORMAT PARQUET, PARTITION_BY (y)); + +# FIXME - this opens many files concurrently which can lead to out of open file handles depending on your ulimit +mode skip + +statement ok +copy (select 2000+(v//12)y,m,v,j from orders) TO '__TEST_DIR__/orders_ym' (FORMAT PARQUET,PARTITION_BY (y,m)); From 6c2933b32bde64ffb44c7026e850f98a010e29bd Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Sat, 13 Apr 2024 09:30:43 +0200 Subject: [PATCH 415/603] Edit in json --- src/core_functions/aggregate/distributive/functions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core_functions/aggregate/distributive/functions.json b/src/core_functions/aggregate/distributive/functions.json index cdf5a19221f6..3bdfe3446fb6 100644 --- a/src/core_functions/aggregate/distributive/functions.json +++ b/src/core_functions/aggregate/distributive/functions.json @@ -153,7 +153,7 @@ { "name": "sum_no_overflow", "parameters": "arg", - "description": "Calculates the sum value for all tuples in arg without overflow checks.", + "description": "Internal only. Calculates the sum value for all tuples in arg without overflow checks.", "example": "sum_no_overflow(A)", "type": "aggregate_function_set" } From a47a8449fc282bcff9c6abee78fe8e304ff4e3a9 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Sat, 13 Apr 2024 09:46:07 +0200 Subject: [PATCH 416/603] Fix issue in debug code in cardinality estimator + add sum no overflow tests --- .../join_order/cardinality_estimator.cpp | 8 ++++++-- .../complex_join_cardinality_assert.test | 14 ++++++++++++++ .../list/aggregates/sum_no_overflow.test | 16 ++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 test/fuzzer/duckfuzz/complex_join_cardinality_assert.test create mode 100644 test/sql/function/list/aggregates/sum_no_overflow.test diff --git a/src/optimizer/join_order/cardinality_estimator.cpp b/src/optimizer/join_order/cardinality_estimator.cpp index 18fd5c858386..30508827d751 100644 --- a/src/optimizer/join_order/cardinality_estimator.cpp +++ b/src/optimizer/join_order/cardinality_estimator.cpp @@ -330,8 +330,12 @@ void CardinalityEstimator::AddRelationNamesToTdoms(vector &stats) for (auto &total_domain : relations_to_tdoms) { for (auto &binding : total_domain.equivalent_relations) { D_ASSERT(binding.table_index < stats.size()); - D_ASSERT(binding.column_index < stats.at(binding.table_index).column_names.size()); - string column_name = stats.at(binding.table_index).column_names.at(binding.column_index); + string column_name; + if (binding.column_index < stats[binding.table_index].column_names.size()) { + column_name = stats[binding.table_index].column_names[binding.column_index]; + } else { + column_name = "[unknown]"; + } total_domain.column_names.push_back(column_name); } } diff --git a/test/fuzzer/duckfuzz/complex_join_cardinality_assert.test b/test/fuzzer/duckfuzz/complex_join_cardinality_assert.test new file mode 100644 index 000000000000..a3540559868b --- /dev/null +++ b/test/fuzzer/duckfuzz/complex_join_cardinality_assert.test @@ -0,0 +1,14 @@ +# name: test/fuzzer/duckfuzz/complex_join_cardinality_assert.test +# description: Complex join that triggered an issue in debug code in the cardinality estimator +# group: [duckfuzz] + +statement ok +PRAGMA enable_verification + +statement ok +create table integers(i int); + +query I +SELECT NULL FROM integers AS t1 RIGHT JOIN integers AS t2 ON (NULL) WHERE EXISTS(SELECT NULL FROM integers AS t3 INNER JOIN integers ON ((t3.i + t1.i)) , (SELECT t2.i AS c0)) +---- + diff --git a/test/sql/function/list/aggregates/sum_no_overflow.test b/test/sql/function/list/aggregates/sum_no_overflow.test new file mode 100644 index 000000000000..ccbd7d1639bb --- /dev/null +++ b/test/sql/function/list/aggregates/sum_no_overflow.test @@ -0,0 +1,16 @@ +# name: test/sql/function/list/aggregates/sum_no_overflow.test +# description: Test the sum_no_overflow aggregate function +# group: [aggregates] + +statement ok +PRAGMA enable_verification + +statement error +SELECT sum_no_overflow(42) +---- +internal use only + +statement error +SELECT sum_no_overflow(42.5) +---- +internal use only From bac01038a6cc9da6a2911c149960cf38ea7cb076 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Sat, 13 Apr 2024 10:14:56 +0200 Subject: [PATCH 417/603] Implement serialize functions for sum_no_overflow --- .../aggregate/distributive/sum.cpp | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/src/core_functions/aggregate/distributive/sum.cpp b/src/core_functions/aggregate/distributive/sum.cpp index 350f22d9d175..9162f72ba742 100644 --- a/src/core_functions/aggregate/distributive/sum.cpp +++ b/src/core_functions/aggregate/distributive/sum.cpp @@ -74,6 +74,15 @@ unique_ptr SumNoOverflowBind(ClientContext &context, AggregateFunc throw BinderException("sum_no_overflow is for internal use only!"); } +void SumNoOverflowSerialize(Serializer &serializer, const optional_ptr bind_data, + const AggregateFunction &function) { + return; +} + +unique_ptr SumNoOverflowDeserialize(Deserializer &deserializer, AggregateFunction &function) { + return nullptr; +} + AggregateFunction GetSumAggregateNoOverflow(PhysicalType type) { switch (type) { case PhysicalType::INT32: { @@ -82,6 +91,8 @@ AggregateFunction GetSumAggregateNoOverflow(PhysicalType type) { function.name = "sum_no_overflow"; function.order_dependent = AggregateOrderDependent::NOT_ORDER_DEPENDENT; function.bind = SumNoOverflowBind; + function.serialize = SumNoOverflowSerialize; + function.deserialize = SumNoOverflowDeserialize; return function; } case PhysicalType::INT64: { @@ -90,6 +101,8 @@ AggregateFunction GetSumAggregateNoOverflow(PhysicalType type) { function.name = "sum_no_overflow"; function.order_dependent = AggregateOrderDependent::NOT_ORDER_DEPENDENT; function.bind = SumNoOverflowBind; + function.serialize = SumNoOverflowSerialize; + function.deserialize = SumNoOverflowDeserialize; return function; } default: @@ -97,6 +110,14 @@ AggregateFunction GetSumAggregateNoOverflow(PhysicalType type) { } } +AggregateFunction GetSumAggregateNoOverflowDecimal() { + AggregateFunction aggr({LogicalTypeId::DECIMAL}, LogicalTypeId::DECIMAL, nullptr, nullptr, nullptr, nullptr, + nullptr, FunctionNullHandling::DEFAULT_NULL_HANDLING, nullptr, SumNoOverflowBind); + aggr.serialize = SumNoOverflowSerialize; + aggr.deserialize = SumNoOverflowDeserialize; + return aggr; +} + unique_ptr SumPropagateStats(ClientContext &context, BoundAggregateExpression &expr, AggregateStatisticsInput &input) { if (input.node_stats && input.node_stats->has_max_cardinality) { @@ -199,9 +220,7 @@ AggregateFunctionSet SumNoOverflowFun::GetFunctions() { AggregateFunctionSet sum_no_overflow; sum_no_overflow.AddFunction(GetSumAggregateNoOverflow(PhysicalType::INT32)); sum_no_overflow.AddFunction(GetSumAggregateNoOverflow(PhysicalType::INT64)); - sum_no_overflow.AddFunction( - AggregateFunction({LogicalTypeId::DECIMAL}, LogicalTypeId::DECIMAL, nullptr, nullptr, nullptr, nullptr, nullptr, - FunctionNullHandling::DEFAULT_NULL_HANDLING, nullptr, SumNoOverflowBind)); + sum_no_overflow.AddFunction(GetSumAggregateNoOverflowDecimal()); return sum_no_overflow; } From 96335737bf71691bb32d742e0b7755719b81e9b6 Mon Sep 17 00:00:00 2001 From: Elliana May Date: Sat, 13 Apr 2024 22:23:33 +0800 Subject: [PATCH 418/603] fix(jdbc): support non-string config parameter values --- tools/jdbc/src/jni/duckdb_java.cpp | 12 ++++++++---- .../src/test/java/org/duckdb/TestDuckDBJDBC.java | 11 +++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/tools/jdbc/src/jni/duckdb_java.cpp b/tools/jdbc/src/jni/duckdb_java.cpp index e019526c65be..1fd9a281a935 100644 --- a/tools/jdbc/src/jni/duckdb_java.cpp +++ b/tools/jdbc/src/jni/duckdb_java.cpp @@ -86,6 +86,8 @@ static jmethodID J_UUID_getLeastSignificantBits; static jclass J_DuckDBDate; static jmethodID J_DuckDBDate_getDaysSinceEpoch; +static jmethodID J_Object_toString; + void ThrowJNI(JNIEnv *env, const char *message) { D_ASSERT(J_SQLException); env->ThrowNew(J_SQLException, message); @@ -250,6 +252,10 @@ JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *reserved) { J_ByteBuffer = (jclass)env->NewGlobalRef(tmpLocalRef); env->DeleteLocalRef(tmpLocalRef); + tmpLocalRef = env->FindClass("java/lang/Object"); + J_Object_toString = env->GetMethodID(tmpLocalRef, "toString", "()Ljava/lang/String;"); + env->DeleteLocalRef(tmpLocalRef); + return JNI_VERSION; } @@ -397,11 +403,9 @@ jobject _duckdb_jdbc_startup(JNIEnv *env, jclass, jbyteArray database_j, jboolea jobject key = env->CallObjectMethod(pair, J_Entry_getKey); jobject value = env->CallObjectMethod(pair, J_Entry_getValue); - D_ASSERT(env->IsInstanceOf(key, J_String)); - const string &key_str = jstring_to_string(env, (jstring)key); + const string &key_str = jstring_to_string(env, (jstring)env->CallObjectMethod(key, J_Object_toString)); - D_ASSERT(env->IsInstanceOf(value, J_String)); - const string &value_str = jstring_to_string(env, (jstring)value); + const string &value_str = jstring_to_string(env, (jstring)env->CallObjectMethod(value, J_Object_toString)); try { config.SetOptionByName(key_str, Value(value_str)); diff --git a/tools/jdbc/src/test/java/org/duckdb/TestDuckDBJDBC.java b/tools/jdbc/src/test/java/org/duckdb/TestDuckDBJDBC.java index 8307825bfad2..2c95b3cf57f1 100644 --- a/tools/jdbc/src/test/java/org/duckdb/TestDuckDBJDBC.java +++ b/tools/jdbc/src/test/java/org/duckdb/TestDuckDBJDBC.java @@ -3220,6 +3220,17 @@ public static void test_user_password() throws Exception { conn2.close(); } + public static void test_boolean_config() throws Exception { + Properties config = new Properties(); + config.put("enable_external_access", false); + try (Connection conn = DriverManager.getConnection(JDBC_URL, config); + PreparedStatement stmt = conn.prepareStatement("SELECT current_setting('enable_external_access')"); + ResultSet rs = stmt.executeQuery()) { + rs.next(); + assertEquals("false", rs.getString(1)); + } + } + public static void test_readonly_remains_bug5593() throws Exception { Path database_file = Files.createTempFile("duckdb-instance-cache-test-", ".duckdb"); database_file.toFile().delete(); From 41449aa6614bbe208ad4a361ca9789eab09b1a80 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Sun, 14 Apr 2024 09:56:48 +0200 Subject: [PATCH 419/603] json_extract_string - deal with NULL arguments correctly in json path --- extension/json/json_functions.cpp | 9 +++++++-- .../duckfuzz/json_extract_string_null.test | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 test/fuzzer/duckfuzz/json_extract_string_null.test diff --git a/extension/json/json_functions.cpp b/extension/json/json_functions.cpp index db2a6ed4ded2..e8241d6e35fe 100644 --- a/extension/json/json_functions.cpp +++ b/extension/json/json_functions.cpp @@ -15,6 +15,9 @@ namespace duckdb { using JSONPathType = JSONCommon::JSONPathType; static JSONPathType CheckPath(const Value &path_val, string &path, size_t &len) { + if (path_val.IsNull()) { + throw InternalException("JSON path cannot be NULL"); + } const auto path_str_val = path_val.DefaultCastAs(LogicalType::VARCHAR); auto path_str = path_str_val.GetValueUnsafe(); len = path_str.GetSize(); @@ -58,9 +61,11 @@ unique_ptr JSONReadFunctionData::Bind(ClientContext &context, Scal size_t len = 0; JSONPathType path_type = JSONPathType::REGULAR; if (arguments[1]->IsFoldable()) { - constant = true; const auto path_val = ExpressionExecutor::EvaluateScalar(context, *arguments[1]); - path_type = CheckPath(path_val, path, len); + if (!path_val.IsNull()) { + constant = true; + path_type = CheckPath(path_val, path, len); + } } bound_function.arguments[1] = LogicalType::VARCHAR; if (path_type == JSONCommon::JSONPathType::WILDCARD) { diff --git a/test/fuzzer/duckfuzz/json_extract_string_null.test b/test/fuzzer/duckfuzz/json_extract_string_null.test new file mode 100644 index 000000000000..f87b859b4af9 --- /dev/null +++ b/test/fuzzer/duckfuzz/json_extract_string_null.test @@ -0,0 +1,19 @@ +# name: test/fuzzer/duckfuzz/json_extract_string_null.test +# description: NULL input to json_extract_string - found by fuzzer +# group: [scalar] + +require json + +statement ok +pragma enable_verification + +statement ok +create table integers(i int); + +statement ok +insert into integers values (1); + +query I +SELECT json_extract_string(false, COALESCE(CASE WHEN (i) THEN (CAST(NULL AS "json")) ELSE NULL END, NULL)) FROM integers; +---- +NULL From 33094fb91cb0cf96580c577b5329526f1dabc7df Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Sun, 14 Apr 2024 10:04:00 +0200 Subject: [PATCH 420/603] Correctly detect overflows in negate of hugeint --- src/function/scalar/operators/arithmetic.cpp | 4 ++-- test/fuzzer/duckfuzz/hugeint_negate_underflow.test | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 test/fuzzer/duckfuzz/hugeint_negate_underflow.test diff --git a/src/function/scalar/operators/arithmetic.cpp b/src/function/scalar/operators/arithmetic.cpp index eb410c42fb9f..828395ff529e 100644 --- a/src/function/scalar/operators/arithmetic.cpp +++ b/src/function/scalar/operators/arithmetic.cpp @@ -446,8 +446,8 @@ void AddFun::RegisterFunction(BuiltinFunctions &set) { struct NegateOperator { template static bool CanNegate(T input) { - using Limits = std::numeric_limits; - return !(Limits::is_integer && Limits::is_signed && Limits::lowest() == input); + using Limits = NumericLimits; + return !(Limits::IsSigned() && Limits::Minimum() == input); } template diff --git a/test/fuzzer/duckfuzz/hugeint_negate_underflow.test b/test/fuzzer/duckfuzz/hugeint_negate_underflow.test new file mode 100644 index 000000000000..5418e5ea2f50 --- /dev/null +++ b/test/fuzzer/duckfuzz/hugeint_negate_underflow.test @@ -0,0 +1,8 @@ +# name: test/fuzzer/duckfuzz/hugeint_negate_underflow.test +# description: Negating of hugeint min leads to signed integer overflow +# group: [duckfuzz] + +statement error +select -(-170141183460469231731687303715884105728)::HUGEINT; +---- +Overflow From e29b82e1024a6ad504c3ff9964e412d21d88ec13 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Sun, 14 Apr 2024 10:14:37 +0200 Subject: [PATCH 421/603] Fix for subquery in limit/offset - because of macros we only know that an expression contains subqueries AFTER binding --- src/planner/binder/query_node/bind_select_node.cpp | 13 +++++++------ .../duckfuzz/limit_offset_macro_subquery.test | 12 ++++++++++++ 2 files changed, 19 insertions(+), 6 deletions(-) create mode 100644 test/fuzzer/duckfuzz/limit_offset_macro_subquery.test diff --git a/src/planner/binder/query_node/bind_select_node.cpp b/src/planner/binder/query_node/bind_select_node.cpp index 70bb67eeb0aa..8c557a45151f 100644 --- a/src/planner/binder/query_node/bind_select_node.cpp +++ b/src/planner/binder/query_node/bind_select_node.cpp @@ -46,21 +46,22 @@ unique_ptr Binder::BindOrderExpression(OrderBinder &order_binder, un BoundLimitNode Binder::BindLimitValue(OrderBinder &order_binder, unique_ptr limit_val, bool is_percentage, bool is_offset) { auto new_binder = Binder::CreateBinder(context, this, true); - if (limit_val->HasSubquery()) { + ExpressionBinder expr_binder(*new_binder, context); + auto target_type = is_percentage ? LogicalType::DOUBLE : LogicalType::BIGINT; + expr_binder.target_type = target_type; + auto original_limit = limit_val->Copy(); + auto expr = expr_binder.Bind(limit_val); + if (expr->HasSubquery()) { if (!order_binder.HasExtraList()) { throw BinderException("Subquery in LIMIT/OFFSET not supported in set operation"); } - auto bound_limit = order_binder.CreateExtraReference(std::move(limit_val)); + auto bound_limit = order_binder.CreateExtraReference(std::move(original_limit)); if (is_percentage) { return BoundLimitNode::ExpressionPercentage(std::move(bound_limit)); } else { return BoundLimitNode::ExpressionValue(std::move(bound_limit)); } } - ExpressionBinder expr_binder(*new_binder, context); - auto target_type = is_percentage ? LogicalType::DOUBLE : LogicalType::BIGINT; - expr_binder.target_type = target_type; - auto expr = expr_binder.Bind(limit_val); if (expr->IsFoldable()) { //! this is a constant auto val = ExpressionExecutor::EvaluateScalar(context, *expr).CastAs(context, target_type); diff --git a/test/fuzzer/duckfuzz/limit_offset_macro_subquery.test b/test/fuzzer/duckfuzz/limit_offset_macro_subquery.test new file mode 100644 index 000000000000..a1a1960135df --- /dev/null +++ b/test/fuzzer/duckfuzz/limit_offset_macro_subquery.test @@ -0,0 +1,12 @@ +# name: test/fuzzer/duckfuzz/limit_offset_macro_subquery.test +# description: Array slice underflow +# group: [duckfuzz] + +statement ok +PRAGMA enable_verification + +# test macro with subquery in limit/offset +query I +SELECT 42 OFFSET format_type(NULL, 2515); +---- +42 From 9ff5201134f05ed6882371e19b4cf969a3debab1 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Sun, 14 Apr 2024 10:21:28 +0200 Subject: [PATCH 422/603] reduce_sql - cancel if no expected error is found --- scripts/reduce_sql.py | 7 ++++++- test/fuzzer/duckfuzz/json_extract_string_null.test | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/reduce_sql.py b/scripts/reduce_sql.py index 9d96bc41fca7..bb5c62460686 100644 --- a/scripts/reduce_sql.py +++ b/scripts/reduce_sql.py @@ -245,7 +245,12 @@ def reduce_query_log(queries, shell, max_time_seconds=300): data_load = open(args.load).read() sql_query = open(args.exec).read() (stdout, stderr, returncode) = run_shell_command(shell, data_load + sql_query) - expected_error = sanitize_error(stderr) + expected_error = sanitize_error(stderr).strip() + if len(expected_error) == 0: + print("===================================================") + print("Could not find expected error - no error encountered") + print("===================================================") + exit(1) print("===================================================") print("Found expected error") diff --git a/test/fuzzer/duckfuzz/json_extract_string_null.test b/test/fuzzer/duckfuzz/json_extract_string_null.test index f87b859b4af9..8fc5ffe59286 100644 --- a/test/fuzzer/duckfuzz/json_extract_string_null.test +++ b/test/fuzzer/duckfuzz/json_extract_string_null.test @@ -1,6 +1,6 @@ # name: test/fuzzer/duckfuzz/json_extract_string_null.test # description: NULL input to json_extract_string - found by fuzzer -# group: [scalar] +# group: [duckfuzz] require json From 820f88eb4a9ac413b3e748d3fa9f256437f06767 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Sun, 14 Apr 2024 17:57:25 +0200 Subject: [PATCH 423/603] Avoid performing Apple codesign on extensions We have our own signing mechanism, and they conflict making the Apple signature invalid --- .github/workflows/OSX.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/OSX.yml b/.github/workflows/OSX.yml index 87099aef9fd3..d97a032dcbe6 100644 --- a/.github/workflows/OSX.yml +++ b/.github/workflows/OSX.yml @@ -262,17 +262,6 @@ jobs: run: | python3 scripts/run_tests_one_by_one.py ./build/release/test/unittest - - name: Sign Extension Binaries - env: - BUILD_CERTIFICATE_BASE64: ${{ secrets.OSX_CODESIGN_BUILD_CERTIFICATE_BASE64 }} - P12_PASSWORD: ${{ secrets.OSX_CODESIGN_P12_PASSWORD }} - KEYCHAIN_PASSWORD: ${{ secrets.OSX_CODESIGN_KEYCHAIN_PASSWORD }} - run: | - if [[ "$GITHUB_REPOSITORY" = "duckdb/duckdb" ]] ; then - . scripts/osx_import_codesign_certificate.sh - codesign --all-architectures --force --sign "Developer ID Application: Stichting DuckDB Foundation" build/release/extension/*/*.duckdb_extension - fi - - name: Deploy shell: bash env: From baa23b10d315d01642ee451b460ea0f8ca6c047b Mon Sep 17 00:00:00 2001 From: Max Gabrielsson Date: Sun, 14 Apr 2024 18:06:48 +0200 Subject: [PATCH 424/603] bump spatial --- .github/config/extensions.csv | 2 +- .github/config/out_of_tree_extensions.cmake | 2 +- src/include/duckdb/main/extension_entries.hpp | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/config/extensions.csv b/.github/config/extensions.csv index 9ce22e2b486f..28a87161e258 100644 --- a/.github/config/extensions.csv +++ b/.github/config/extensions.csv @@ -13,6 +13,6 @@ substrait,https://github.com/duckdb/substrait,1116fb580edd3e26e675436dbdbdf4a0aa arrow,https://github.com/duckdb/arrow,9e10240da11f61ea7fbfe3fc9988ffe672ccd40f,no-windows aws,https://github.com/duckdb/duckdb_aws,f7b8729f1cce5ada5d4add70e1486de50763fb97, azure,https://github.com/duckdb/duckdb_azure,86f39d76157de970d16d6d6537bc90c0ee1c7d35, -spatial,https://github.com/duckdb/duckdb_spatial,05c4ba01c500140287bf6946fb6910122e5c2acf, +spatial,https://github.com/duckdb/duckdb_spatial,8ac803e986ccda34f32dee82a7faae95b72b3492, iceberg,https://github.com/duckdb/duckdb_iceberg,7aa3d8e4cb7b513d35fdacfa28dc328771bc4047, vss,https://github.com/duckdb/duckdb_vss,a85e973650a083e4b279126a0aec07924d84e765, \ No newline at end of file diff --git a/.github/config/out_of_tree_extensions.cmake b/.github/config/out_of_tree_extensions.cmake index 0e02a41f6ad2..d6497c7bb8cb 100644 --- a/.github/config/out_of_tree_extensions.cmake +++ b/.github/config/out_of_tree_extensions.cmake @@ -75,7 +75,7 @@ endif() duckdb_extension_load(spatial DONT_LINK LOAD_TESTS GIT_URL https://github.com/duckdb/duckdb_spatial.git - GIT_TAG 05c4ba01c500140287bf6946fb6910122e5c2acf + GIT_TAG 8ac803e986ccda34f32dee82a7faae95b72b3492 INCLUDE_DIR spatial/include TEST_DIR test/sql APPLY_PATCHES diff --git a/src/include/duckdb/main/extension_entries.hpp b/src/include/duckdb/main/extension_entries.hpp index ac32aa4feb4d..c7b3b7e4607a 100644 --- a/src/include/duckdb/main/extension_entries.hpp +++ b/src/include/duckdb/main/extension_entries.hpp @@ -132,6 +132,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = { {"st_dimension", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, {"st_disjoint", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, {"st_distance", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, + {"st_distance_sphere", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, {"st_distance_spheroid", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, {"st_drivers", "spatial", CatalogType::TABLE_FUNCTION_ENTRY}, {"st_dump", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, @@ -154,6 +155,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = { {"st_geomfromhexwkb", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, {"st_geomfromtext", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, {"st_geomfromwkb", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, + {"st_hilbert", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, {"st_intersection", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, {"st_intersection_agg", "spatial", CatalogType::AGGREGATE_FUNCTION_ENTRY}, {"st_intersects", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, @@ -201,6 +203,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = { {"st_reduceprecision", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, {"st_removerepeatedpoints", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, {"st_reverse", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, + {"st_shortestline", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, {"st_simplify", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, {"st_simplifypreservetopology", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, {"st_startpoint", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY}, From a329e5822e60ddb053caab0682f11036c4baebd2 Mon Sep 17 00:00:00 2001 From: Max Gabrielsson Date: Sun, 14 Apr 2024 19:36:57 +0200 Subject: [PATCH 425/603] remove applied patch --- .../extensions/spatial/open_file.patch | 53 ------------------- 1 file changed, 53 deletions(-) delete mode 100644 .github/patches/extensions/spatial/open_file.patch diff --git a/.github/patches/extensions/spatial/open_file.patch b/.github/patches/extensions/spatial/open_file.patch deleted file mode 100644 index 573b7277d150..000000000000 --- a/.github/patches/extensions/spatial/open_file.patch +++ /dev/null @@ -1,53 +0,0 @@ -diff --git a/spatial/src/spatial/core/io/osm/st_read_osm.cpp b/spatial/src/spatial/core/io/osm/st_read_osm.cpp -index dfe396a..87bf598 100644 ---- a/spatial/src/spatial/core/io/osm/st_read_osm.cpp -+++ b/spatial/src/spatial/core/io/osm/st_read_osm.cpp -@@ -239,7 +239,7 @@ static unique_ptr InitGlobal(ClientContext &context, T - auto &fs = FileSystem::GetFileSystem(context); - auto file_name = bind_data.file_name; - -- auto handle = fs.OpenFile(file_name, FileFlags::FILE_FLAGS_READ, FileLockType::READ_LOCK); -+ auto handle = fs.OpenFile(file_name, FileFlags::FILE_FLAGS_READ | FileLockType::READ_LOCK); - auto file_size = handle->GetFileSize(); - - auto max_threads = context.db->NumberOfThreads(); -diff --git a/spatial/src/spatial/gdal/file_handler.cpp b/spatial/src/spatial/gdal/file_handler.cpp -index db449df..ebcefe5 100644 ---- a/spatial/src/spatial/gdal/file_handler.cpp -+++ b/spatial/src/spatial/gdal/file_handler.cpp -@@ -122,7 +122,7 @@ public: - auto &fs = FileSystem::GetFileSystem(context); - - // TODO: Double check that this is correct -- uint8_t flags; -+ FileOpenFlags flags; - auto len = strlen(access); - if (access[0] == 'r') { - flags = FileFlags::FILE_FLAGS_READ; -@@ -160,14 +160,14 @@ public: - // Check if the file is a directory - - #ifdef _WIN32 -- if (fs.DirectoryExists(path) && (flags & FileFlags::FILE_FLAGS_READ)) { -+ if (fs.DirectoryExists(path) && flags.OpenForReading()) { - // We can't open a directory for reading on windows without special flags - // so just open nul instead, gdal will reject it when it tries to read - auto file = fs.OpenFile("nul", flags); - return new DuckDBFileHandle(std::move(file)); - } - #endif -- auto file = fs.OpenFile(file_name, flags, FileSystem::DEFAULT_LOCK, FileCompressionType::AUTO_DETECT); -+ auto file = fs.OpenFile(file_name, flags | FileCompressionType::AUTO_DETECT); - return new DuckDBFileHandle(std::move(file)); - } catch (std::exception &ex) { - // Failed to open file via DuckDB File System. If this doesnt have a VSI prefix we can return an error here. -@@ -209,8 +209,7 @@ public: - - unique_ptr file; - try { -- file = fs.OpenFile(file_name, FileFlags::FILE_FLAGS_READ, FileSystem::DEFAULT_LOCK, -- FileCompressionType::AUTO_DETECT); -+ file = fs.OpenFile(file_name, FileFlags::FILE_FLAGS_READ | FileCompressionType::AUTO_DETECT); - } catch (std::exception &ex) { - return -1; - } From 273b03af7ece88b1fc612dfcc00fd12879af401c Mon Sep 17 00:00:00 2001 From: Max Gabrielsson Date: Sun, 14 Apr 2024 20:30:21 +0200 Subject: [PATCH 426/603] Update out_of_tree_extensions.cmake Co-authored-by: Carlo Piovesan --- .github/config/out_of_tree_extensions.cmake | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/config/out_of_tree_extensions.cmake b/.github/config/out_of_tree_extensions.cmake index d6497c7bb8cb..d907647e44c1 100644 --- a/.github/config/out_of_tree_extensions.cmake +++ b/.github/config/out_of_tree_extensions.cmake @@ -78,7 +78,6 @@ duckdb_extension_load(spatial GIT_TAG 8ac803e986ccda34f32dee82a7faae95b72b3492 INCLUDE_DIR spatial/include TEST_DIR test/sql - APPLY_PATCHES ) ################# SQLITE_SCANNER From 1198baa05b69dc90da48d45b45914fa49484d99f Mon Sep 17 00:00:00 2001 From: Xiaoying Wang Date: Sun, 14 Apr 2024 22:34:05 -0700 Subject: [PATCH 427/603] run format-fix --- src/optimizer/join_order/cardinality_estimator.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/optimizer/join_order/cardinality_estimator.cpp b/src/optimizer/join_order/cardinality_estimator.cpp index 669c05668d02..d69898e73f84 100644 --- a/src/optimizer/join_order/cardinality_estimator.cpp +++ b/src/optimizer/join_order/cardinality_estimator.cpp @@ -39,7 +39,7 @@ void CardinalityEstimator::AddRelationTdom(FilterInfo &filter_info) { bool CardinalityEstimator::SingleRelationFilter(FilterInfo &filter_info) { if (filter_info.left_set && filter_info.right_set && filter_info.set.count > 1) { - // Both set and are from different relations + // Both set and are from different relations return false; } if (EmptyFilter(filter_info)) { @@ -100,7 +100,7 @@ void CardinalityEstimator::InitEquivalentRelations(const vector Date: Mon, 15 Apr 2024 10:00:43 +0200 Subject: [PATCH 428/603] mc --- src/main/extension/extension_load.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/extension/extension_load.cpp b/src/main/extension/extension_load.cpp index 002cf4aefe15..58a1b698ab46 100644 --- a/src/main/extension/extension_load.cpp +++ b/src/main/extension/extension_load.cpp @@ -71,7 +71,7 @@ static string PrettyPrintString(const string &s) { c == '.') { res += c; } else { - uint8_t value = c; + auto value = UnsafeNumericCast(c); res += "\\x"; uint8_t first = value / 16; if (first < 10) { @@ -276,7 +276,7 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileSystem &fs, const str } } - auto number_metadata_fields = 3; + idx_t number_metadata_fields = 3; D_ASSERT(number_metadata_fields == 3); // Currently hardcoded value metadata_field.resize(number_metadata_fields + 1); From 7a35c390003464f3d24635af21b8210b15ed5d26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Mon, 15 Apr 2024 10:15:44 +0200 Subject: [PATCH 429/603] fixing platform binary --- src/include/duckdb/common/platform.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/include/duckdb/common/platform.h b/src/include/duckdb/common/platform.h index 32babf9c80eb..3166ff9684e2 100644 --- a/src/include/duckdb/common/platform.h +++ b/src/include/duckdb/common/platform.h @@ -1,5 +1,10 @@ #include -#include "duckdb/common/string_util.hpp" + +// duplicated from string_util.h to avoid linking issues +#ifndef DUCKDB_QUOTE_DEFINE +#define DUCKDB_QUOTE_DEFINE_IMPL(x) #x +#define DUCKDB_QUOTE_DEFINE(x) DUCKDB_QUOTE_DEFINE_IMPL(x) +#endif namespace duckdb { From f26de480f561f089f0955c34453afaa930070d33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Mon, 15 Apr 2024 10:19:40 +0200 Subject: [PATCH 430/603] so much nicer --- src/planner/operator/logical_top_n.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/planner/operator/logical_top_n.cpp b/src/planner/operator/logical_top_n.cpp index adf4019e84b0..e60adc0eaf84 100644 --- a/src/planner/operator/logical_top_n.cpp +++ b/src/planner/operator/logical_top_n.cpp @@ -4,8 +4,8 @@ namespace duckdb { idx_t LogicalTopN::EstimateCardinality(ClientContext &context) { auto child_cardinality = LogicalOperator::EstimateCardinality(context); - if (limit >= 0 && child_cardinality < idx_t(limit)) { - return NumericCast(limit); + if (child_cardinality < limit) { + return limit; } return child_cardinality; } From 35534478c304644d643390714a5d6566f4dee4ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Mon, 15 Apr 2024 10:33:30 +0200 Subject: [PATCH 431/603] mc2 --- src/function/table/arrow_conversion.cpp | 2 +- src/optimizer/topn_optimizer.cpp | 4 ++-- src/planner/expression_binder/order_binder.cpp | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/function/table/arrow_conversion.cpp b/src/function/table/arrow_conversion.cpp index c82b6a64ad4d..78d4cca859f6 100644 --- a/src/function/table/arrow_conversion.cpp +++ b/src/function/table/arrow_conversion.cpp @@ -985,7 +985,7 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca for (idx_t row_idx = 0; row_idx < size; row_idx++) { auto tag = NumericCast(type_ids[row_idx]); - auto out_of_range = tag < 0 || tag >= array.n_children; + auto out_of_range = tag >= array.n_children; if (out_of_range) { throw InvalidInputException("Arrow union tag out of range: %d", tag); } diff --git a/src/optimizer/topn_optimizer.cpp b/src/optimizer/topn_optimizer.cpp index 5e83f0a05f79..7c227cc70d5a 100644 --- a/src/optimizer/topn_optimizer.cpp +++ b/src/optimizer/topn_optimizer.cpp @@ -29,10 +29,10 @@ unique_ptr TopN::Optimize(unique_ptr op) { if (CanOptimize(*op)) { auto &limit = op->Cast(); auto &order_by = (op->children[0])->Cast(); - auto limit_val = NumericCast(limit.limit_val.GetConstantValue()); + auto limit_val = limit.limit_val.GetConstantValue(); idx_t offset_val = 0; if (limit.offset_val.Type() == LimitNodeType::CONSTANT_VALUE) { - offset_val = NumericCast(limit.offset_val.GetConstantValue()); + offset_val = limit.offset_val.GetConstantValue(); } auto topn = make_uniq(std::move(order_by.orders), limit_val, offset_val); topn->AddChild(std::move(order_by.children[0])); diff --git a/src/planner/expression_binder/order_binder.cpp b/src/planner/expression_binder/order_binder.cpp index c5a63de8ce84..2924a3a50170 100644 --- a/src/planner/expression_binder/order_binder.cpp +++ b/src/planner/expression_binder/order_binder.cpp @@ -108,7 +108,8 @@ unique_ptr OrderBinder::Bind(unique_ptr expr) { auto &collation = expr->Cast(); if (collation.child->expression_class == ExpressionClass::CONSTANT) { auto &constant = collation.child->Cast(); - auto index = NumericCast(constant.value.GetValue()) - 1; + D_ASSERT(constant.value.GetValue() > 0); + auto index = constant.value.GetValue() - 1; child_list_t values; values.push_back(make_pair("index", Value::UBIGINT(index))); values.push_back(make_pair("collation", Value(std::move(collation.collation)))); From 0ba9ac34b92aefa3b6366d5238d8aefca244147e Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 15 Apr 2024 10:47:54 +0200 Subject: [PATCH 432/603] add ALTERNATIVE_VERIFY2 --- CMakeLists.txt | 5 +++++ Makefile | 3 +++ 2 files changed, 8 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b1eaa7adfa2..f7c39b58b070 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -355,6 +355,7 @@ option(DISABLE_STR_INLINE "Debug setting: disable inlining of strings" FALSE) option(DISABLE_MEMORY_SAFETY "Debug setting: disable memory access checks at runtime" FALSE) option(DISABLE_ASSERTIONS "Debug setting: disable assertions" FALSE) option(ALTERNATIVE_VERIFY "Debug setting: use alternative verify mode" FALSE) +option(ALTERNATIVE_VERIFY2 "Debug setting: use alternative verify mode version 2" FALSE) option(RUN_SLOW_VERIFIERS "Debug setting: enable a more extensive set of verifiers" FALSE) option(DESTROY_UNPINNED_BLOCKS "Debug setting: destroy unpinned buffer-managed blocks" FALSE) option(FORCE_ASYNC_SINK_SOURCE "Debug setting: forces sinks/sources to block the first 2 times they're called" FALSE) @@ -432,6 +433,10 @@ if(ALTERNATIVE_VERIFY) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDUCKDB_ALTERNATIVE_VERIFY") endif() +if(ALTERNATIVE_VERIFY2) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDUCKDB_ALTERNATIVE_VERIFY2") +endif() + if(DEBUG_ALLOCATION) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDUCKDB_DEBUG_ALLOCATION") endif() diff --git a/Makefile b/Makefile index a6fd0dcd3ea0..a74d5be9479f 100644 --- a/Makefile +++ b/Makefile @@ -212,6 +212,9 @@ endif ifeq (${ALTERNATIVE_VERIFY}, 1) CMAKE_VARS:=${CMAKE_VARS} -DALTERNATIVE_VERIFY=1 endif +ifeq (${ALTERNATIVE_VERIFY2}, 1) + CMAKE_VARS:=${CMAKE_VARS} -DALTERNATIVE_VERIFY2=1 +endif ifneq (${VERIFY_VECTOR}, ) CMAKE_VARS:=${CMAKE_VARS} -DVERIFY_VECTOR=${VERIFY_VECTOR} endif From 161b646796618a54c562096046ec1a53b67a2482 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Mon, 15 Apr 2024 10:50:18 +0200 Subject: [PATCH 433/603] arrow tests --- test/api/capi/test_capi_arrow.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/api/capi/test_capi_arrow.cpp b/test/api/capi/test_capi_arrow.cpp index a8bdd2c9ae33..34ac028d56f5 100644 --- a/test/api/capi/test_capi_arrow.cpp +++ b/test/api/capi/test_capi_arrow.cpp @@ -237,7 +237,7 @@ TEST_CASE("Test arrow in C API", "[capi][arrow]") { auto data_chunk = &data_chunks[i]; data_chunk->Initialize(allocator, logical_types, STANDARD_VECTOR_SIZE); data_chunk->SetCardinality(STANDARD_VECTOR_SIZE); - for (int row = 0; row < STANDARD_VECTOR_SIZE; row++) { + for (idx_t row = 0; row < STANDARD_VECTOR_SIZE; row++) { data_chunk->SetValue(0, row, duckdb::Value(i)); } appender.Append(*data_chunk, 0, data_chunk->size(), data_chunk->size()); From 84a3e4a16f3c86139c0e650f8310da07ed11054a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Mon, 15 Apr 2024 10:57:27 +0200 Subject: [PATCH 434/603] mc test runner --- test/sqlite/sqllogic_test_runner.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/sqlite/sqllogic_test_runner.cpp b/test/sqlite/sqllogic_test_runner.cpp index 6c7f32d38859..628a1fc4922f 100644 --- a/test/sqlite/sqllogic_test_runner.cpp +++ b/test/sqlite/sqllogic_test_runner.cpp @@ -293,7 +293,7 @@ RequireResult SQLLogicTestRunner::CheckRequire(SQLLogicParser &parser, const vec parser.Fail("require vector_size requires a parameter"); } // require a specific vector size - auto required_vector_size = std::stoi(params[1]); + auto required_vector_size = NumericCast(std::stoi(params[1])); if (STANDARD_VECTOR_SIZE < required_vector_size) { // vector size is too low for this test: skip it return RequireResult::MISSING; @@ -306,7 +306,7 @@ RequireResult SQLLogicTestRunner::CheckRequire(SQLLogicParser &parser, const vec parser.Fail("require exact_vector_size requires a parameter"); } // require an exact vector size - auto required_vector_size = std::stoi(params[1]); + auto required_vector_size = NumericCast(std::stoi(params[1])); if (STANDARD_VECTOR_SIZE != required_vector_size) { // vector size does not match the required vector size: skip it return RequireResult::MISSING; @@ -319,7 +319,7 @@ RequireResult SQLLogicTestRunner::CheckRequire(SQLLogicParser &parser, const vec parser.Fail("require block_size requires a parameter"); } // require a specific block size - auto required_block_size = std::stoi(params[1]); + auto required_block_size = NumericCast(std::stoi(params[1])); if (Storage::BLOCK_ALLOC_SIZE != required_block_size) { // block size does not match the required block size: skip it return RequireResult::MISSING; From cc2bbeb973cb84d70e4cc3a7b2c91e918011e68e Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 15 Apr 2024 12:32:03 +0200 Subject: [PATCH 435/603] Skip ADBC tests if python version is not 3.9 or higher --- tools/pythonpkg/tests/fast/adbc/test_adbc.py | 2 ++ tools/pythonpkg/tests/fast/adbc/test_connection_get_info.py | 2 ++ tools/pythonpkg/tests/fast/adbc/test_statement_bind.py | 1 + 3 files changed, 5 insertions(+) diff --git a/tools/pythonpkg/tests/fast/adbc/test_adbc.py b/tools/pythonpkg/tests/fast/adbc/test_adbc.py index 9b05345f0bb3..efe1ab97ca32 100644 --- a/tools/pythonpkg/tests/fast/adbc/test_adbc.py +++ b/tools/pythonpkg/tests/fast/adbc/test_adbc.py @@ -4,6 +4,8 @@ import datetime import os +@pytest.mark.skipif(sys.version_info < (3, 9), reason="Requires python 3.9") + adbc_driver_manager = pytest.importorskip("adbc_driver_manager.dbapi") adbc_driver_manager_lib = pytest.importorskip("adbc_driver_manager._lib") diff --git a/tools/pythonpkg/tests/fast/adbc/test_connection_get_info.py b/tools/pythonpkg/tests/fast/adbc/test_connection_get_info.py index e50e9060642c..000ceff7f80b 100644 --- a/tools/pythonpkg/tests/fast/adbc/test_connection_get_info.py +++ b/tools/pythonpkg/tests/fast/adbc/test_connection_get_info.py @@ -6,6 +6,8 @@ pa = pytest.importorskip("pyarrow") adbc_driver_manager = pytest.importorskip("adbc_driver_manager") +@pytest.mark.skipif(sys.version_info < (3, 9), reason="Requires python 3.9") + try: adbc_driver_duckdb = pytest.importorskip("adbc_driver_duckdb.dbapi") con = adbc_driver_duckdb.connect() diff --git a/tools/pythonpkg/tests/fast/adbc/test_statement_bind.py b/tools/pythonpkg/tests/fast/adbc/test_statement_bind.py index b7c27ad6b5db..ddab8ea2ccea 100644 --- a/tools/pythonpkg/tests/fast/adbc/test_statement_bind.py +++ b/tools/pythonpkg/tests/fast/adbc/test_statement_bind.py @@ -9,6 +9,7 @@ con = adbc_driver_duckdb.connect() +@pytest.mark.skipif(sys.version_info < (3, 9), reason="Requires python 3.9") def _import(handle): """Helper to import a C Data Interface handle.""" if isinstance(handle, adbc_driver_manager.ArrowArrayStreamHandle): From a1ef32aa708a522f07df94364869c3cd824033a8 Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 15 Apr 2024 12:41:42 +0200 Subject: [PATCH 436/603] add getters/setters for 'table' property of VacuumInfo --- .../operator/helper/physical_vacuum.cpp | 10 +++++--- .../duckdb/parser/parsed_data/vacuum_info.hpp | 7 ++++-- .../storage/serialization/parse_info.json | 25 +++++++++++++++++++ src/parser/parsed_data/vacuum_info.cpp | 16 ++++++++++++ src/planner/binder/statement/bind_vacuum.cpp | 4 +-- .../serialization/serialize_parse_info.cpp | 10 ++++++++ .../fuzzer/pedro/vacuum_unsupported_type.test | 6 +++++ test/sql/explain/explain_all_statements.test | 3 +++ 8 files changed, 73 insertions(+), 8 deletions(-) diff --git a/src/execution/operator/helper/physical_vacuum.cpp b/src/execution/operator/helper/physical_vacuum.cpp index 10fd477901cb..898697a4180d 100644 --- a/src/execution/operator/helper/physical_vacuum.cpp +++ b/src/execution/operator/helper/physical_vacuum.cpp @@ -16,7 +16,8 @@ class VacuumLocalSinkState : public LocalSinkState { public: explicit VacuumLocalSinkState(VacuumInfo &info) { for (const auto &column_name : info.columns) { - auto &column = info.table->GetColumn(column_name); + auto &table = info.GetTable(); + auto &column = table.GetColumn(column_name); if (DistinctStatistics::TypeIsSupported(column.GetType())) { column_distinct_stats.push_back(make_uniq()); } else { @@ -37,7 +38,8 @@ class VacuumGlobalSinkState : public GlobalSinkState { explicit VacuumGlobalSinkState(VacuumInfo &info) { for (const auto &column_name : info.columns) { - auto &column = info.table->GetColumn(column_name); + auto &table = info.GetTable(); + auto &column = table.GetColumn(column_name); if (DistinctStatistics::TypeIsSupported(column.GetType())) { column_distinct_stats.push_back(make_uniq()); } else { @@ -89,9 +91,9 @@ SinkFinalizeType PhysicalVacuum::Finalize(Pipeline &pipeline, Event &event, Clie OperatorSinkFinalizeInput &input) const { auto &sink = input.global_state.Cast(); - auto table = info->table; + auto &table = info->GetTable(); for (idx_t col_idx = 0; col_idx < sink.column_distinct_stats.size(); col_idx++) { - table->GetStorage().SetDistinct(info->column_id_map.at(col_idx), + table.GetStorage().SetDistinct(info->column_id_map.at(col_idx), std::move(sink.column_distinct_stats[col_idx])); } diff --git a/src/include/duckdb/parser/parsed_data/vacuum_info.hpp b/src/include/duckdb/parser/parsed_data/vacuum_info.hpp index 08fdda15a6cc..c7d288b94314 100644 --- a/src/include/duckdb/parser/parsed_data/vacuum_info.hpp +++ b/src/include/duckdb/parser/parsed_data/vacuum_info.hpp @@ -37,14 +37,17 @@ struct VacuumInfo : public ParseInfo { explicit VacuumInfo(VacuumOptions options); const VacuumOptions options; + TableCatalogEntry &GetTable(); + bool HasTable() const; + void SetTable(TableCatalogEntry &table); public: bool has_table; unique_ptr ref; - optional_ptr table; unordered_map column_id_map; vector columns; - +private: + optional_ptr table; public: unique_ptr Copy(); diff --git a/src/include/duckdb/storage/serialization/parse_info.json b/src/include/duckdb/storage/serialization/parse_info.json index e2048b2173b7..1c72c7f508e3 100644 --- a/src/include/duckdb/storage/serialization/parse_info.json +++ b/src/include/duckdb/storage/serialization/parse_info.json @@ -555,6 +555,31 @@ "id": 200, "name": "options", "type": "VacuumOptions" + }, + { + "id": 201, + "name": "has_table", + "type": "bool" + }, + { + "id": 202, + "name": "ref", + "type": "unique_ptr" + }, + { + "id": 203, + "name": "table", + "type": "optional_ptr" + }, + { + "id": 204, + "name": "column_id_map", + "type": "unordered_map" + }, + { + "id": 205, + "name": "columns", + "type": "vector" } ], "constructor": ["options"] diff --git a/src/parser/parsed_data/vacuum_info.cpp b/src/parser/parsed_data/vacuum_info.cpp index bbbd9fb53404..e4b2f66c650a 100644 --- a/src/parser/parsed_data/vacuum_info.cpp +++ b/src/parser/parsed_data/vacuum_info.cpp @@ -11,7 +11,23 @@ unique_ptr VacuumInfo::Copy() { if (has_table) { result->ref = ref->Copy(); } + result->table = table; + result->column_id_map = column_id_map; + result->columns = columns; return result; } +TableCatalogEntry &VacuumInfo::GetTable() { + D_ASSERT(HasTable()); + return *table; +} + +bool VacuumInfo::HasTable() const { + return table != nullptr; +} + +void VacuumInfo::SetTable(TableCatalogEntry &table_p) { + table = &table_p; +} + } // namespace duckdb diff --git a/src/planner/binder/statement/bind_vacuum.cpp b/src/planner/binder/statement/bind_vacuum.cpp index c7aaa0abb61a..4dc69a6bdbed 100644 --- a/src/planner/binder/statement/bind_vacuum.cpp +++ b/src/planner/binder/statement/bind_vacuum.cpp @@ -14,7 +14,7 @@ BoundStatement Binder::Bind(VacuumStatement &stmt) { unique_ptr root; if (stmt.info->has_table) { - D_ASSERT(!stmt.info->table); + D_ASSERT(!stmt.info->HasTable()); D_ASSERT(stmt.info->column_id_map.empty()); auto bound_table = Bind(*stmt.info->ref); if (bound_table->type != TableReferenceType::BASE_TABLE) { @@ -22,7 +22,7 @@ BoundStatement Binder::Bind(VacuumStatement &stmt) { } auto ref = unique_ptr_cast(std::move(bound_table)); auto &table = ref->table; - stmt.info->table = &table; + stmt.info->SetTable(table); auto &columns = stmt.info->columns; vector> select_list; diff --git a/src/storage/serialization/serialize_parse_info.cpp b/src/storage/serialization/serialize_parse_info.cpp index cac770d7c517..f605c1d16f49 100644 --- a/src/storage/serialization/serialize_parse_info.cpp +++ b/src/storage/serialization/serialize_parse_info.cpp @@ -466,11 +466,21 @@ unique_ptr TransactionInfo::Deserialize(Deserializer &deserializer) { void VacuumInfo::Serialize(Serializer &serializer) const { ParseInfo::Serialize(serializer); serializer.WriteProperty(200, "options", options); + //serializer.WritePropertyWithDefault(201, "has_table", has_table); + //serializer.WritePropertyWithDefault>(202, "ref", ref); + //serializer.WritePropertyWithDefault>(203, "table", table); + //serializer.WritePropertyWithDefault>(204, "column_id_map", column_id_map); + //serializer.WritePropertyWithDefault>(205, "columns", columns); } unique_ptr VacuumInfo::Deserialize(Deserializer &deserializer) { auto options = deserializer.ReadProperty(200, "options"); auto result = duckdb::unique_ptr(new VacuumInfo(options)); + //deserializer.ReadPropertyWithDefault(201, "has_table", result->has_table); + //deserializer.ReadPropertyWithDefault>(202, "ref", result->ref); + //deserializer.ReadPropertyWithDefault>(203, "table", result->table); + //deserializer.ReadPropertyWithDefault>(204, "column_id_map", result->column_id_map); + //deserializer.ReadPropertyWithDefault>(205, "columns", result->columns); return std::move(result); } diff --git a/test/fuzzer/pedro/vacuum_unsupported_type.test b/test/fuzzer/pedro/vacuum_unsupported_type.test index eee1ec513de8..0b0d1470e8e0 100644 --- a/test/fuzzer/pedro/vacuum_unsupported_type.test +++ b/test/fuzzer/pedro/vacuum_unsupported_type.test @@ -2,6 +2,12 @@ # description: Vacuum a table with arrays # group: [pedro] +#statement ok +#pragma enable_verification + +statement ok +pragma verify_serializer + statement ok CREATE TABLE t1 AS (VALUES ([], {'a': []})); diff --git a/test/sql/explain/explain_all_statements.test b/test/sql/explain/explain_all_statements.test index 518f9f13b486..268ead8cddad 100644 --- a/test/sql/explain/explain_all_statements.test +++ b/test/sql/explain/explain_all_statements.test @@ -2,6 +2,9 @@ # description: Test explain on various statements # group: [explain] +statement ok +pragma enable_verification + statement ok explain PRAGMA enable_verification From 6a4baec3b6f96df088e4801b54c50984310f62d8 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Mon, 15 Apr 2024 13:17:27 +0200 Subject: [PATCH 437/603] Limit batch insert threads based on available memory, similar to Parquet write --- .../persistent/physical_batch_insert.cpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/execution/operator/persistent/physical_batch_insert.cpp b/src/execution/operator/persistent/physical_batch_insert.cpp index f2f242a90d78..7ddd838f2379 100644 --- a/src/execution/operator/persistent/physical_batch_insert.cpp +++ b/src/execution/operator/persistent/physical_batch_insert.cpp @@ -125,8 +125,9 @@ class BatchInsertTask { class BatchInsertGlobalState : public GlobalSinkState { public: - explicit BatchInsertGlobalState(ClientContext &context, DuckTableEntry &table, idx_t initial_memory) - : memory_manager(context, initial_memory), table(table), insert_count(0), optimistically_written(false) { + explicit BatchInsertGlobalState(ClientContext &context, DuckTableEntry &table, idx_t minimum_memory_per_thread) + : memory_manager(context, minimum_memory_per_thread), table(table), insert_count(0), + optimistically_written(false), minimum_memory_per_thread(minimum_memory_per_thread) { } BatchMemoryManager memory_manager; @@ -137,6 +138,7 @@ class BatchInsertGlobalState : public GlobalSinkState { vector collections; idx_t next_start = 0; atomic optimistically_written; + idx_t minimum_memory_per_thread; static bool ReadyToMerge(idx_t count); void ScheduleMergeTasks(idx_t min_batch_index); @@ -146,6 +148,13 @@ class BatchInsertGlobalState : public GlobalSinkState { void AddCollection(ClientContext &context, idx_t batch_index, idx_t min_batch_index, unique_ptr current_collection, optional_ptr writer = nullptr); + + idx_t MaxThreads(idx_t source_max_threads) override { + // try to request 4MB per column per thread + memory_manager.SetMemorySize(source_max_threads * minimum_memory_per_thread); + // cap the concurrent threads working on this task based on the amount of available memory + return MinValue(source_max_threads, memory_manager.AvailableMemory() / minimum_memory_per_thread + 1); + } }; class BatchInsertLocalState : public LocalSinkState { @@ -377,8 +386,8 @@ unique_ptr PhysicalBatchInsert::GetGlobalSinkState(ClientContex } // heuristic - we start off by allocating 4MB of cache space per column static constexpr const idx_t MINIMUM_MEMORY_PER_COLUMN = 4ULL * 1024ULL * 1024ULL; - auto initial_memory = table->GetColumns().PhysicalColumnCount() * MINIMUM_MEMORY_PER_COLUMN; - auto result = make_uniq(context, table->Cast(), initial_memory); + auto minimum_memory_per_thread = table->GetColumns().PhysicalColumnCount() * MINIMUM_MEMORY_PER_COLUMN; + auto result = make_uniq(context, table->Cast(), minimum_memory_per_thread); return std::move(result); } From 6dce9deff00d8ae88c168e143d2df2a72b236e28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Mon, 15 Apr 2024 14:13:34 +0200 Subject: [PATCH 438/603] fighting with FormatSigned --- src/include/duckdb/common/numeric_utils.hpp | 3 +++ src/include/duckdb/common/types/cast_helpers.hpp | 14 ++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/include/duckdb/common/numeric_utils.hpp b/src/include/duckdb/common/numeric_utils.hpp index 5b111c896fee..17ca6be0fe27 100644 --- a/src/include/duckdb/common/numeric_utils.hpp +++ b/src/include/duckdb/common/numeric_utils.hpp @@ -68,6 +68,9 @@ static void ThrowNumericCastError(FROM in, TO minval, TO maxval) { template TO NumericCast(FROM val) { + if (std::is_same::value) { + return static_cast(val); + } // some dance around signed-unsigned integer comparison below auto minval = NumericLimits::Minimum(); auto maxval = NumericLimits::Maximum(); diff --git a/src/include/duckdb/common/types/cast_helpers.hpp b/src/include/duckdb/common/types/cast_helpers.hpp index 7dd91f3a5f8a..fb312ace0d64 100644 --- a/src/include/duckdb/common/types/cast_helpers.hpp +++ b/src/include/duckdb/common/types/cast_helpers.hpp @@ -61,17 +61,15 @@ class NumericHelper { template static string_t FormatSigned(T value, Vector &vector) { - auto is_negative = (value < 0); - auto unsigned_value = static_cast::type>(AbsValue(value)); - auto length = UnsignedLength(unsigned_value); - if (is_negative) { - length++; - } - auto result = StringVector::EmptyString(vector, UnsafeNumericCast(length)); + typedef typename MakeUnsigned::type UNSIGNED; + int8_t sign = -(value < 0); + UNSIGNED unsigned_value = UNSIGNED(value ^ T(sign)) + UNSIGNED(AbsValue(sign)); + auto length = UnsafeNumericCast(UnsignedLength(unsigned_value) + AbsValue(sign)); + string_t result = StringVector::EmptyString(vector, length); auto dataptr = result.GetDataWriteable(); auto endptr = dataptr + length; endptr = FormatUnsigned(unsigned_value, endptr); - if (is_negative) { + if (sign) { *--endptr = '-'; } result.Finalize(); From f4a302e7f5734be4ba5cda0d0cbb4a7b68a90b60 Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 15 Apr 2024 14:46:10 +0200 Subject: [PATCH 439/603] split vacuum from logical-simple --- src/catalog/dependency_list.cpp | 8 ++ .../operator/helper/physical_vacuum.cpp | 27 ++-- src/execution/physical_plan/CMakeLists.txt | 1 + src/execution/physical_plan/plan_simple.cpp | 9 -- src/execution/physical_plan_generator.cpp | 4 +- .../duckdb/catalog/dependency_list.hpp | 2 + .../operator/helper/physical_vacuum.hpp | 5 +- .../execution/physical_plan_generator.hpp | 1 + .../duckdb/parser/parsed_data/vacuum_info.hpp | 12 +- src/include/duckdb/planner/binder.hpp | 3 + src/include/duckdb/planner/logical_tokens.hpp | 1 + src/include/duckdb/planner/operator/list.hpp | 1 + .../serialization/logical_operator.json | 2 +- .../storage/serialization/parse_info.json | 10 -- src/parser/parsed_data/vacuum_info.cpp | 15 -- src/planner/binder/statement/bind_vacuum.cpp | 131 +++++++++--------- src/planner/operator/CMakeLists.txt | 1 + .../serialize_logical_operator.cpp | 3 - .../serialization/serialize_parse_info.cpp | 16 +-- test/sql/vacuum/test_analyze.test | 2 +- 20 files changed, 114 insertions(+), 140 deletions(-) diff --git a/src/catalog/dependency_list.cpp b/src/catalog/dependency_list.cpp index 1ef3be15fbc4..a7a30468dc24 100644 --- a/src/catalog/dependency_list.cpp +++ b/src/catalog/dependency_list.cpp @@ -40,6 +40,10 @@ bool LogicalDependencyEquality::operator()(const LogicalDependency &a, const Log LogicalDependency::LogicalDependency() : entry(), catalog() { } +unique_ptr LogicalDependency::Copy() const { + return make_uniq(catalog, entry); +} + static string GetSchema(CatalogEntry &entry) { if (entry.type == CatalogType::SCHEMA_ENTRY) { return entry.name; @@ -61,6 +65,10 @@ LogicalDependency::LogicalDependency(CatalogEntry &entry) { } } +LogicalDependency::LogicalDependency(const string &catalog, const CatalogEntryInfo &entry) + : entry(entry), catalog(catalog) { +} + bool LogicalDependency::operator==(const LogicalDependency &other) const { return other.entry.name == entry.name && other.entry.schema == entry.schema && other.entry.type == entry.type; } diff --git a/src/execution/operator/helper/physical_vacuum.cpp b/src/execution/operator/helper/physical_vacuum.cpp index 898697a4180d..77c21d82ca70 100644 --- a/src/execution/operator/helper/physical_vacuum.cpp +++ b/src/execution/operator/helper/physical_vacuum.cpp @@ -7,17 +7,17 @@ namespace duckdb { -PhysicalVacuum::PhysicalVacuum(unique_ptr info_p, idx_t estimated_cardinality) +PhysicalVacuum::PhysicalVacuum(unique_ptr info_p, optional_ptr table, + unordered_map column_id_map, idx_t estimated_cardinality) : PhysicalOperator(PhysicalOperatorType::VACUUM, {LogicalType::BOOLEAN}, estimated_cardinality), - info(std::move(info_p)) { + info(std::move(info_p)), table(std::move(table)), column_id_map(std::move(column_id_map)) { } class VacuumLocalSinkState : public LocalSinkState { public: - explicit VacuumLocalSinkState(VacuumInfo &info) { + explicit VacuumLocalSinkState(VacuumInfo &info, optional_ptr table) { for (const auto &column_name : info.columns) { - auto &table = info.GetTable(); - auto &column = table.GetColumn(column_name); + auto &column = table->GetColumn(column_name); if (DistinctStatistics::TypeIsSupported(column.GetType())) { column_distinct_stats.push_back(make_uniq()); } else { @@ -30,16 +30,14 @@ class VacuumLocalSinkState : public LocalSinkState { }; unique_ptr PhysicalVacuum::GetLocalSinkState(ExecutionContext &context) const { - return make_uniq(*info); + return make_uniq(*info, table); } class VacuumGlobalSinkState : public GlobalSinkState { public: - explicit VacuumGlobalSinkState(VacuumInfo &info) { - + explicit VacuumGlobalSinkState(VacuumInfo &info, optional_ptr table) { for (const auto &column_name : info.columns) { - auto &table = info.GetTable(); - auto &column = table.GetColumn(column_name); + auto &column = table->GetColumn(column_name); if (DistinctStatistics::TypeIsSupported(column.GetType())) { column_distinct_stats.push_back(make_uniq()); } else { @@ -53,12 +51,12 @@ class VacuumGlobalSinkState : public GlobalSinkState { }; unique_ptr PhysicalVacuum::GetGlobalSinkState(ClientContext &context) const { - return make_uniq(*info); + return make_uniq(*info, table); } SinkResultType PhysicalVacuum::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const { auto &lstate = input.local_state.Cast(); - D_ASSERT(lstate.column_distinct_stats.size() == info->column_id_map.size()); + D_ASSERT(lstate.column_distinct_stats.size() == column_id_map.size()); for (idx_t col_idx = 0; col_idx < chunk.data.size(); col_idx++) { if (!DistinctStatistics::TypeIsSupported(chunk.data[col_idx].GetType())) { @@ -91,10 +89,9 @@ SinkFinalizeType PhysicalVacuum::Finalize(Pipeline &pipeline, Event &event, Clie OperatorSinkFinalizeInput &input) const { auto &sink = input.global_state.Cast(); - auto &table = info->GetTable(); + auto tbl = table; for (idx_t col_idx = 0; col_idx < sink.column_distinct_stats.size(); col_idx++) { - table.GetStorage().SetDistinct(info->column_id_map.at(col_idx), - std::move(sink.column_distinct_stats[col_idx])); + tbl->GetStorage().SetDistinct(column_id_map.at(col_idx), std::move(sink.column_distinct_stats[col_idx])); } return SinkFinalizeType::READY; diff --git a/src/execution/physical_plan/CMakeLists.txt b/src/execution/physical_plan/CMakeLists.txt index f9124752f6d7..b9f414c9ac7b 100644 --- a/src/execution/physical_plan/CMakeLists.txt +++ b/src/execution/physical_plan/CMakeLists.txt @@ -40,6 +40,7 @@ add_library_unity( plan_top_n.cpp plan_update.cpp plan_window.cpp + plan_vacuum.cpp plan_unnest.cpp plan_expression_get.cpp plan_recursive_cte.cpp diff --git a/src/execution/physical_plan/plan_simple.cpp b/src/execution/physical_plan/plan_simple.cpp index a32aef16fd51..3a7c2f9691cb 100644 --- a/src/execution/physical_plan/plan_simple.cpp +++ b/src/execution/physical_plan/plan_simple.cpp @@ -25,15 +25,6 @@ unique_ptr PhysicalPlanGenerator::CreatePlan(LogicalSimple &op case LogicalOperatorType::LOGICAL_TRANSACTION: return make_uniq(unique_ptr_cast(std::move(op.info)), op.estimated_cardinality); - case LogicalOperatorType::LOGICAL_VACUUM: { - auto result = make_uniq(unique_ptr_cast(std::move(op.info)), - op.estimated_cardinality); - if (!op.children.empty()) { - auto child = CreatePlan(*op.children[0]); - result->children.push_back(std::move(child)); - } - return std::move(result); - } case LogicalOperatorType::LOGICAL_LOAD: return make_uniq(unique_ptr_cast(std::move(op.info)), op.estimated_cardinality); diff --git a/src/execution/physical_plan_generator.cpp b/src/execution/physical_plan_generator.cpp index 14d74a9096cd..4f88c00cf797 100644 --- a/src/execution/physical_plan_generator.cpp +++ b/src/execution/physical_plan_generator.cpp @@ -176,10 +176,12 @@ unique_ptr PhysicalPlanGenerator::CreatePlan(LogicalOperator & case LogicalOperatorType::LOGICAL_PRAGMA: plan = CreatePlan(op.Cast()); break; + case LogicalOperatorType::LOGICAL_VACUUM: + plan = CreatePlan(op.Cast()); + break; case LogicalOperatorType::LOGICAL_TRANSACTION: case LogicalOperatorType::LOGICAL_ALTER: case LogicalOperatorType::LOGICAL_DROP: - case LogicalOperatorType::LOGICAL_VACUUM: case LogicalOperatorType::LOGICAL_LOAD: case LogicalOperatorType::LOGICAL_ATTACH: case LogicalOperatorType::LOGICAL_DETACH: diff --git a/src/include/duckdb/catalog/dependency_list.hpp b/src/include/duckdb/catalog/dependency_list.hpp index ed08f01ac945..7a76022a2121 100644 --- a/src/include/duckdb/catalog/dependency_list.hpp +++ b/src/include/duckdb/catalog/dependency_list.hpp @@ -30,7 +30,9 @@ struct LogicalDependency { public: explicit LogicalDependency(CatalogEntry &entry); + explicit LogicalDependency(const string &catalog, const CatalogEntryInfo &entry); LogicalDependency(); + unique_ptr Copy() const; bool operator==(const LogicalDependency &other) const; }; diff --git a/src/include/duckdb/execution/operator/helper/physical_vacuum.hpp b/src/include/duckdb/execution/operator/helper/physical_vacuum.hpp index 77238a0458fc..1bc3d53602fe 100644 --- a/src/include/duckdb/execution/operator/helper/physical_vacuum.hpp +++ b/src/include/duckdb/execution/operator/helper/physical_vacuum.hpp @@ -20,9 +20,12 @@ class PhysicalVacuum : public PhysicalOperator { static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::VACUUM; public: - PhysicalVacuum(unique_ptr info, idx_t estimated_cardinality); + PhysicalVacuum(unique_ptr info, optional_ptr table, + unordered_map column_id_map, idx_t estimated_cardinality); unique_ptr info; + optional_ptr table; + unordered_map column_id_map; public: // Source interface diff --git a/src/include/duckdb/execution/physical_plan_generator.hpp b/src/include/duckdb/execution/physical_plan_generator.hpp index 8024bc50ed09..647763213fc4 100644 --- a/src/include/duckdb/execution/physical_plan_generator.hpp +++ b/src/include/duckdb/execution/physical_plan_generator.hpp @@ -87,6 +87,7 @@ class PhysicalPlanGenerator { unique_ptr CreatePlan(LogicalSet &op); unique_ptr CreatePlan(LogicalReset &op); unique_ptr CreatePlan(LogicalSimple &op); + unique_ptr CreatePlan(LogicalVacuum &op); unique_ptr CreatePlan(LogicalUnnest &op); unique_ptr CreatePlan(LogicalRecursiveCTE &op); unique_ptr CreatePlan(LogicalMaterializedCTE &op); diff --git a/src/include/duckdb/parser/parsed_data/vacuum_info.hpp b/src/include/duckdb/parser/parsed_data/vacuum_info.hpp index c7d288b94314..6bd8fe6a60a0 100644 --- a/src/include/duckdb/parser/parsed_data/vacuum_info.hpp +++ b/src/include/duckdb/parser/parsed_data/vacuum_info.hpp @@ -13,6 +13,7 @@ #include "duckdb/planner/tableref/bound_basetableref.hpp" #include "duckdb/common/unordered_map.hpp" #include "duckdb/common/optional_ptr.hpp" +#include "duckdb/catalog/dependency_list.hpp" namespace duckdb { class Serializer; @@ -37,17 +38,10 @@ struct VacuumInfo : public ParseInfo { explicit VacuumInfo(VacuumOptions options); const VacuumOptions options; - TableCatalogEntry &GetTable(); - bool HasTable() const; - void SetTable(TableCatalogEntry &table); - -public: + vector columns; bool has_table; unique_ptr ref; - unordered_map column_id_map; - vector columns; -private: - optional_ptr table; + public: unique_ptr Copy(); diff --git a/src/include/duckdb/planner/binder.hpp b/src/include/duckdb/planner/binder.hpp index 804f78d3e0ae..ed646e6b04b7 100644 --- a/src/include/duckdb/planner/binder.hpp +++ b/src/include/duckdb/planner/binder.hpp @@ -37,6 +37,7 @@ class ViewCatalogEntry; class TableMacroCatalogEntry; class UpdateSetInfo; class LogicalProjection; +class LogicalVacuum; class ColumnList; class ExternalDependency; @@ -163,6 +164,8 @@ class Binder : public std::enable_shared_from_this { TableCatalogEntry &table, TableStorageInfo &storage_info); void BindOnConflictClause(LogicalInsert &insert, TableCatalogEntry &table, InsertStatement &stmt); + void BindVacuumTable(LogicalVacuum &vacuum, unique_ptr &root); + static void BindSchemaOrCatalog(ClientContext &context, string &catalog, string &schema); static void BindLogicalType(ClientContext &context, LogicalType &type, optional_ptr catalog = nullptr, const string &schema = INVALID_SCHEMA); diff --git a/src/include/duckdb/planner/logical_tokens.hpp b/src/include/duckdb/planner/logical_tokens.hpp index 7854bd393300..91050764207b 100644 --- a/src/include/duckdb/planner/logical_tokens.hpp +++ b/src/include/duckdb/planner/logical_tokens.hpp @@ -50,6 +50,7 @@ class LogicalMaterializedCTE; class LogicalSetOperation; class LogicalSample; class LogicalSimple; +class LogicalVacuum; class LogicalSet; class LogicalReset; class LogicalTopN; diff --git a/src/include/duckdb/planner/operator/list.hpp b/src/include/duckdb/planner/operator/list.hpp index 199f0a65234f..3fe4cb03a97b 100644 --- a/src/include/duckdb/planner/operator/list.hpp +++ b/src/include/duckdb/planner/operator/list.hpp @@ -41,4 +41,5 @@ #include "duckdb/planner/operator/logical_top_n.hpp" #include "duckdb/planner/operator/logical_unnest.hpp" #include "duckdb/planner/operator/logical_update.hpp" +#include "duckdb/planner/operator/logical_vacuum.hpp" #include "duckdb/planner/operator/logical_window.hpp" diff --git a/src/include/duckdb/storage/serialization/logical_operator.json b/src/include/duckdb/storage/serialization/logical_operator.json index d2439eadca76..78fd3eb08c32 100644 --- a/src/include/duckdb/storage/serialization/logical_operator.json +++ b/src/include/duckdb/storage/serialization/logical_operator.json @@ -806,7 +806,7 @@ { "class": "LogicalSimple", "base": "LogicalOperator", - "enum": ["LOGICAL_ALTER", "LOGICAL_VACUUM", "LOGICAL_LOAD", "LOGICAL_ATTACH", "LOGICAL_TRANSACTION", "LOGICAL_DROP", "LOGICAL_DETACH"], + "enum": ["LOGICAL_ALTER", "LOGICAL_LOAD", "LOGICAL_ATTACH", "LOGICAL_TRANSACTION", "LOGICAL_DROP", "LOGICAL_DETACH"], "members": [ { "id": 200, diff --git a/src/include/duckdb/storage/serialization/parse_info.json b/src/include/duckdb/storage/serialization/parse_info.json index 1c72c7f508e3..a44ccd10fc31 100644 --- a/src/include/duckdb/storage/serialization/parse_info.json +++ b/src/include/duckdb/storage/serialization/parse_info.json @@ -568,16 +568,6 @@ }, { "id": 203, - "name": "table", - "type": "optional_ptr" - }, - { - "id": 204, - "name": "column_id_map", - "type": "unordered_map" - }, - { - "id": 205, "name": "columns", "type": "vector" } diff --git a/src/parser/parsed_data/vacuum_info.cpp b/src/parser/parsed_data/vacuum_info.cpp index e4b2f66c650a..cf8326938e94 100644 --- a/src/parser/parsed_data/vacuum_info.cpp +++ b/src/parser/parsed_data/vacuum_info.cpp @@ -11,23 +11,8 @@ unique_ptr VacuumInfo::Copy() { if (has_table) { result->ref = ref->Copy(); } - result->table = table; - result->column_id_map = column_id_map; result->columns = columns; return result; } -TableCatalogEntry &VacuumInfo::GetTable() { - D_ASSERT(HasTable()); - return *table; -} - -bool VacuumInfo::HasTable() const { - return table != nullptr; -} - -void VacuumInfo::SetTable(TableCatalogEntry &table_p) { - table = &table_p; -} - } // namespace duckdb diff --git a/src/planner/binder/statement/bind_vacuum.cpp b/src/planner/binder/statement/bind_vacuum.cpp index 4dc69a6bdbed..5d48772eeab1 100644 --- a/src/planner/binder/statement/bind_vacuum.cpp +++ b/src/planner/binder/statement/bind_vacuum.cpp @@ -3,84 +3,85 @@ #include "duckdb/planner/binder.hpp" #include "duckdb/planner/operator/logical_get.hpp" #include "duckdb/planner/operator/logical_projection.hpp" -#include "duckdb/planner/operator/logical_simple.hpp" +#include "duckdb/planner/operator/logical_vacuum.hpp" #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" namespace duckdb { -BoundStatement Binder::Bind(VacuumStatement &stmt) { - BoundStatement result; +void Binder::BindVacuumTable(LogicalVacuum &vacuum, unique_ptr &root) { + auto &info = vacuum.GetInfo(); + if (!info.has_table) { + return; + } - unique_ptr root; + D_ASSERT(vacuum.column_id_map.empty()); + auto bound_table = Bind(*info.ref); + if (bound_table->type != TableReferenceType::BASE_TABLE) { + throw InvalidInputException("Can only vacuum/analyze base tables!"); + } + auto ref = unique_ptr_cast(std::move(bound_table)); + auto &table = ref->table; + vacuum.SetTable(table); - if (stmt.info->has_table) { - D_ASSERT(!stmt.info->HasTable()); - D_ASSERT(stmt.info->column_id_map.empty()); - auto bound_table = Bind(*stmt.info->ref); - if (bound_table->type != TableReferenceType::BASE_TABLE) { - throw InvalidInputException("Can only vacuum/analyze base tables!"); - } - auto ref = unique_ptr_cast(std::move(bound_table)); - auto &table = ref->table; - stmt.info->SetTable(table); + vector> select_list; + auto &columns = info.columns; + if (columns.empty()) { + // Empty means ALL columns should be vacuumed/analyzed + auto &get = ref->get->Cast(); + columns.insert(columns.end(), get.names.begin(), get.names.end()); + } - auto &columns = stmt.info->columns; - vector> select_list; - if (columns.empty()) { - // Empty means ALL columns should be vacuumed/analyzed - auto &get = ref->get->Cast(); - columns.insert(columns.end(), get.names.begin(), get.names.end()); + case_insensitive_set_t column_name_set; + vector non_generated_column_names; + for (auto &col_name : columns) { + if (column_name_set.count(col_name) > 0) { + throw BinderException("Vacuum the same column twice(same name in column name list)"); } - - case_insensitive_set_t column_name_set; - vector non_generated_column_names; - for (auto &col_name : columns) { - if (column_name_set.count(col_name) > 0) { - throw BinderException("Vacuum the same column twice(same name in column name list)"); - } - column_name_set.insert(col_name); - if (!table.ColumnExists(col_name)) { - throw BinderException("Column with name \"%s\" does not exist", col_name); - } - auto &col = table.GetColumn(col_name); - // ignore generated column - if (col.Generated()) { - continue; - } - non_generated_column_names.push_back(col_name); - ColumnRefExpression colref(col_name, table.name); - auto result = bind_context.BindColumn(colref, 0); - if (result.HasError()) { - result.error.Throw(); - } - select_list.push_back(std::move(result.expression)); + column_name_set.insert(col_name); + if (!table.ColumnExists(col_name)) { + throw BinderException("Column with name \"%s\" does not exist", col_name); } - stmt.info->columns = std::move(non_generated_column_names); - if (!select_list.empty()) { - auto table_scan = CreatePlan(*ref); - D_ASSERT(table_scan->type == LogicalOperatorType::LOGICAL_GET); - - auto &get = table_scan->Cast(); + auto &col = table.GetColumn(col_name); + // ignore generated column + if (col.Generated()) { + continue; + } + non_generated_column_names.push_back(col_name); + ColumnRefExpression colref(col_name, table.name); + auto result = bind_context.BindColumn(colref, 0); + if (result.HasError()) { + result.error.Throw(); + } + select_list.push_back(std::move(result.expression)); + } + info.columns = std::move(non_generated_column_names); + // Creating a table without any physical columns is not supported + D_ASSERT(!select_list.empty()); - D_ASSERT(select_list.size() == get.column_ids.size()); - D_ASSERT(stmt.info->columns.size() == get.column_ids.size()); - for (idx_t i = 0; i < get.column_ids.size(); i++) { - stmt.info->column_id_map[i] = - table.GetColumns().LogicalToPhysical(LogicalIndex(get.column_ids[i])).index; - } + auto table_scan = CreatePlan(*ref); + D_ASSERT(table_scan->type == LogicalOperatorType::LOGICAL_GET); - auto projection = make_uniq(GenerateTableIndex(), std::move(select_list)); - projection->children.push_back(std::move(table_scan)); + auto &get = table_scan->Cast(); - root = std::move(projection); - } else { - // eg. CREATE TABLE test (x AS (1)); - // ANALYZE test; - // Make it not a SINK so it doesn't have to do anything - stmt.info->has_table = false; - } + D_ASSERT(select_list.size() == get.column_ids.size()); + D_ASSERT(info.columns.size() == get.column_ids.size()); + for (idx_t i = 0; i < get.column_ids.size(); i++) { + vacuum.column_id_map[i] = table.GetColumns().LogicalToPhysical(LogicalIndex(get.column_ids[i])).index; } - auto vacuum = make_uniq(LogicalOperatorType::LOGICAL_VACUUM, std::move(stmt.info)); + + auto projection = make_uniq(GenerateTableIndex(), std::move(select_list)); + projection->children.push_back(std::move(table_scan)); + + root = std::move(projection); +} + +BoundStatement Binder::Bind(VacuumStatement &stmt) { + BoundStatement result; + + unique_ptr root; + + auto vacuum = make_uniq(std::move(stmt.info)); + BindVacuumTable(*vacuum, root); if (root) { vacuum->children.push_back(std::move(root)); } diff --git a/src/planner/operator/CMakeLists.txt b/src/planner/operator/CMakeLists.txt index 2823f1cee6c7..dacd4a44a1ea 100644 --- a/src/planner/operator/CMakeLists.txt +++ b/src/planner/operator/CMakeLists.txt @@ -42,6 +42,7 @@ add_library_unity( logical_unconditional_join.cpp logical_unnest.cpp logical_update.cpp + logical_vacuum.cpp logical_window.cpp) set(ALL_OBJECT_FILES ${ALL_OBJECT_FILES} $ diff --git a/src/storage/serialization/serialize_logical_operator.cpp b/src/storage/serialization/serialize_logical_operator.cpp index c88a681e242b..0fec4cbe1401 100644 --- a/src/storage/serialization/serialize_logical_operator.cpp +++ b/src/storage/serialization/serialize_logical_operator.cpp @@ -172,9 +172,6 @@ unique_ptr LogicalOperator::Deserialize(Deserializer &deseriali case LogicalOperatorType::LOGICAL_UPDATE: result = LogicalUpdate::Deserialize(deserializer); break; - case LogicalOperatorType::LOGICAL_VACUUM: - result = LogicalSimple::Deserialize(deserializer); - break; case LogicalOperatorType::LOGICAL_WINDOW: result = LogicalWindow::Deserialize(deserializer); break; diff --git a/src/storage/serialization/serialize_parse_info.cpp b/src/storage/serialization/serialize_parse_info.cpp index f605c1d16f49..fbd3d32c1e8b 100644 --- a/src/storage/serialization/serialize_parse_info.cpp +++ b/src/storage/serialization/serialize_parse_info.cpp @@ -466,21 +466,17 @@ unique_ptr TransactionInfo::Deserialize(Deserializer &deserializer) { void VacuumInfo::Serialize(Serializer &serializer) const { ParseInfo::Serialize(serializer); serializer.WriteProperty(200, "options", options); - //serializer.WritePropertyWithDefault(201, "has_table", has_table); - //serializer.WritePropertyWithDefault>(202, "ref", ref); - //serializer.WritePropertyWithDefault>(203, "table", table); - //serializer.WritePropertyWithDefault>(204, "column_id_map", column_id_map); - //serializer.WritePropertyWithDefault>(205, "columns", columns); + serializer.WritePropertyWithDefault(201, "has_table", has_table); + serializer.WritePropertyWithDefault>(202, "ref", ref); + serializer.WritePropertyWithDefault>(203, "columns", columns); } unique_ptr VacuumInfo::Deserialize(Deserializer &deserializer) { auto options = deserializer.ReadProperty(200, "options"); auto result = duckdb::unique_ptr(new VacuumInfo(options)); - //deserializer.ReadPropertyWithDefault(201, "has_table", result->has_table); - //deserializer.ReadPropertyWithDefault>(202, "ref", result->ref); - //deserializer.ReadPropertyWithDefault>(203, "table", result->table); - //deserializer.ReadPropertyWithDefault>(204, "column_id_map", result->column_id_map); - //deserializer.ReadPropertyWithDefault>(205, "columns", result->columns); + deserializer.ReadPropertyWithDefault(201, "has_table", result->has_table); + deserializer.ReadPropertyWithDefault>(202, "ref", result->ref); + deserializer.ReadPropertyWithDefault>(203, "columns", result->columns); return std::move(result); } diff --git a/test/sql/vacuum/test_analyze.test b/test/sql/vacuum/test_analyze.test index 4476abdeaa72..3bee1c017ed6 100644 --- a/test/sql/vacuum/test_analyze.test +++ b/test/sql/vacuum/test_analyze.test @@ -14,7 +14,7 @@ statement ok analyze statement ok -vacuum +vacuum; statement error vacuum test From 496977a2eaed20cf02df564821a526d8f12953d4 Mon Sep 17 00:00:00 2001 From: Christina Sioula Date: Thu, 11 Apr 2024 20:33:21 +0200 Subject: [PATCH 440/603] skip unzip tests in python - this functionality is not supported yet --- scripts/parser_test.py | 4 ++-- scripts/sqllogictest/__init__.py | 3 ++- scripts/sqllogictest/parser/__init__.py | 4 ++-- scripts/sqllogictest/parser/parser.py | 12 ++++++++---- scripts/sqllogictest/test.py | 6 +++++- test/sql/storage/unzip.test | 2 +- test/sqlite/sqllogic_command.hpp | 2 +- tools/pythonpkg/scripts/sqllogictest_python.py | 18 ++++++++++++++---- 8 files changed, 35 insertions(+), 16 deletions(-) diff --git a/scripts/parser_test.py b/scripts/parser_test.py index 8a70536ab6e8..e09fbe3264de 100644 --- a/scripts/parser_test.py +++ b/scripts/parser_test.py @@ -1,4 +1,4 @@ -from sqllogictest import SQLLogicParser, SQLLogicTest +from sqllogictest import SQLParserException, SQLLogicParser, SQLLogicTest from typing import Optional import argparse @@ -14,7 +14,7 @@ def main(): parser = SQLLogicParser() out: Optional[SQLLogicTest] = parser.parse(filename) if not out: - raise Exception(f"Test {filename} could not be parsed") + raise SQLParserException(f"Test {filename} could not be parsed") if __name__ == "__main__": diff --git a/scripts/sqllogictest/__init__.py b/scripts/sqllogictest/__init__.py index b6ed736103aa..02ae67a6f99c 100644 --- a/scripts/sqllogictest/__init__.py +++ b/scripts/sqllogictest/__init__.py @@ -24,7 +24,7 @@ ) from .decorator import SkipIf, OnlyIf from .expected_result import ExpectedResult -from .parser import SQLLogicParser +from .parser import SQLLogicParser, SQLParserException __all__ = [ TokenType, @@ -54,4 +54,5 @@ SkipIf, OnlyIf, SQLLogicParser, + SQLParserException, ] diff --git a/scripts/sqllogictest/parser/__init__.py b/scripts/sqllogictest/parser/__init__.py index 70a9a930b16b..90c98254ec73 100644 --- a/scripts/sqllogictest/parser/__init__.py +++ b/scripts/sqllogictest/parser/__init__.py @@ -1,3 +1,3 @@ -from .parser import SQLLogicParser, SQLLogicTest +from .parser import SQLLogicParser, SQLParserException, SQLLogicTest -__all__ = [SQLLogicParser, SQLLogicTest] +__all__ = [SQLLogicParser, SQLParserException, SQLLogicTest] diff --git a/scripts/sqllogictest/parser/parser.py b/scripts/sqllogictest/parser/parser.py index db78e60eddb9..f91706d1a9f6 100644 --- a/scripts/sqllogictest/parser/parser.py +++ b/scripts/sqllogictest/parser/parser.py @@ -53,6 +53,10 @@ def is_space(char: str): ### -------- PARSER ---------- +class SQLParserException(Exception): + def __init__(self, message): + self.message = "Parser Error: " + message + super().__init__(self.message) class SQLLogicParser: @@ -102,18 +106,18 @@ def peek(self): def peek_no_strip(self): if self.current_line >= len(self.lines): - raise Exception("File already fully consumed") + raise SQLParserException("File already fully consumed") return self.lines[self.current_line] def consume(self): if self.current_line >= len(self.lines): - raise Exception("File already fully consumed") + raise SQLParserException("File already fully consumed") self.current_line += 1 def fail(self, message): file_path = self.current_test.path error_message = f"{file_path}:{self.current_line + 1}: {message}" - raise Exception(error_message) + raise SQLParserException(error_message) def get_expected_result(self, statement_type: str) -> ExpectedResult: type_map = { @@ -530,7 +534,7 @@ def main(): parser = SQLLogicParser() out: Optional[SQLLogicTest] = parser.parse(filename) if not out: - raise Exception(f"Test {filename} could not be parsed") + raise SQLParserException(f"Test {filename} could not be parsed") if __name__ == "__main__": diff --git a/scripts/sqllogictest/test.py b/scripts/sqllogictest/test.py index 21d08d7a12ea..6a173b9d97b4 100644 --- a/scripts/sqllogictest/test.py +++ b/scripts/sqllogictest/test.py @@ -3,14 +3,18 @@ class SQLLogicTest: - __slots__ = ['path', 'statements'] + __slots__ = ['path', 'statements', 'skipped'] def __init__(self, path: str): self.path: str = path self.statements: List[BaseStatement] = [] + self.skipped = False def add_statement(self, statement: BaseStatement): self.statements.append(statement) def is_sqlite_test(self): return 'test/sqlite/select' in self.path or 'third_party/sqllogictest' in self.path + + def skip(self, val): + self.skipped = val diff --git a/test/sql/storage/unzip.test b/test/sql/storage/unzip.test index 6592fc6c672c..53fe95aea1ea 100644 --- a/test/sql/storage/unzip.test +++ b/test/sql/storage/unzip.test @@ -12,7 +12,7 @@ SELECT a+1 FROM tbl; ---- 6 -# unzip a 1.8M file to default extraction path -> __TEST_DIR__/ +# unzip a 1.8M file into the default extraction path -> __TEST_DIR__/ unzip data/storage/index_0-9-1.db.gz load __TEST_DIR__/index_0-9-1.db readonly diff --git a/test/sqlite/sqllogic_command.hpp b/test/sqlite/sqllogic_command.hpp index d6b2e6c9f71a..863d9a3419d8 100644 --- a/test/sqlite/sqllogic_command.hpp +++ b/test/sqlite/sqllogic_command.hpp @@ -146,7 +146,7 @@ class SleepCommand : public Command { class UnzipCommand : public Command { public: // 1 MB - static constexpr const int64_t BUFFER_SIZE = 1u << 23; + static constexpr const int64_t BUFFER_SIZE = 1u << 20; public: UnzipCommand(SQLLogicTestRunner &runner, string &input, string &output); diff --git a/tools/pythonpkg/scripts/sqllogictest_python.py b/tools/pythonpkg/scripts/sqllogictest_python.py index 0f04013af9ef..a1c8345e179c 100644 --- a/tools/pythonpkg/scripts/sqllogictest_python.py +++ b/tools/pythonpkg/scripts/sqllogictest_python.py @@ -8,6 +8,7 @@ script_path = os.path.dirname(os.path.abspath(__file__)) sys.path.append(os.path.join(script_path, '..', '..', '..', 'scripts')) from sqllogictest import ( + SQLParserException, SQLLogicParser, SQLLogicTest, ) @@ -95,6 +96,9 @@ def update_value(_: SQLLogicContext) -> Generator[Any, Any, Any]: # Yield once to represent one iteration, do not touch the keywords yield None + if self.test.skipped: + return ExecuteResult(ExecuteResult.Type.SKIPPED) + self.database = SQLLogicDatabase(':memory:', None) pool = self.database.connect() context = SQLLogicContext(pool, self, test.statements, keywords, update_value) @@ -168,10 +172,16 @@ def main(): continue if test_directory: file_path = os.path.join(test_directory, file_path) - test = sql_parser.parse(file_path) - if not test: - print(f'Failed to parse {file_path}') - exit(1) + + test = SQLLogicTest("") + try: + test = sql_parser.parse(file_path) + if not test: + raise SQLParserException(f'failed to parse {file_path}') + except SQLParserException as e: + test.skip(True) + executor.skip_log.append(str(e.message)) + print(f'[{i}/{total_tests}] {file_path}') # This is necessary to clean up databases/connections # So previously created databases are not still cached in the instance_cache From 9f5ec8d07cebe3839423cb6f416fabcafd0edfd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Mon, 15 Apr 2024 14:48:44 +0200 Subject: [PATCH 441/603] duplicated file --- src/optimizer/CMakeLists.txt | 10 +- src/optimizer/move_constants.cpp | 165 ------------------------------ src/optimizer/rule/CMakeLists.txt | 4 +- 3 files changed, 7 insertions(+), 172 deletions(-) delete mode 100644 src/optimizer/move_constants.cpp diff --git a/src/optimizer/CMakeLists.txt b/src/optimizer/CMakeLists.txt index 53f57e1c168d..bcdcb90d13f7 100644 --- a/src/optimizer/CMakeLists.txt +++ b/src/optimizer/CMakeLists.txt @@ -10,24 +10,24 @@ add_library_unity( duckdb_optimizer OBJECT column_binding_replacer.cpp + column_lifetime_analyzer.cpp common_aggregate_optimizer.cpp compressed_materialization.cpp cse_optimizer.cpp deliminator.cpp - unnest_rewriter.cpp - column_lifetime_analyzer.cpp expression_heuristics.cpp + expression_rewriter.cpp filter_combiner.cpp - filter_pushdown.cpp filter_pullup.cpp + filter_pushdown.cpp in_clause_rewriter.cpp optimizer.cpp - expression_rewriter.cpp regex_range_filter.cpp remove_duplicate_groups.cpp remove_unused_columns.cpp statistics_propagator.cpp - topn_optimizer.cpp) + topn_optimizer.cpp + unnest_rewriter.cpp) set(ALL_OBJECT_FILES ${ALL_OBJECT_FILES} $ PARENT_SCOPE) diff --git a/src/optimizer/move_constants.cpp b/src/optimizer/move_constants.cpp deleted file mode 100644 index 636265ff9131..000000000000 --- a/src/optimizer/move_constants.cpp +++ /dev/null @@ -1,165 +0,0 @@ -#include "duckdb/optimizer/rule/move_constants.hpp" - -#include "duckdb/common/exception.hpp" -#include "duckdb/common/value_operations/value_operations.hpp" -#include "duckdb/planner/expression/bound_comparison_expression.hpp" -#include "duckdb/planner/expression/bound_constant_expression.hpp" -#include "duckdb/planner/expression/bound_function_expression.hpp" -#include "duckdb/optimizer/expression_rewriter.hpp" - -namespace duckdb { - -MoveConstantsRule::MoveConstantsRule(ExpressionRewriter &rewriter) : Rule(rewriter) { - auto op = make_uniq(); - op->matchers.push_back(make_uniq()); - op->policy = SetMatcher::Policy::UNORDERED; - - auto arithmetic = make_uniq(); - // we handle multiplication, addition and subtraction because those are "easy" - // integer division makes the division case difficult - // e.g. [x / 2 = 3] means [x = 6 OR x = 7] because of truncation -> no clean rewrite rules - arithmetic->function = make_uniq(unordered_set {"+", "-", "*"}); - // we match only on integral numeric types - arithmetic->type = make_uniq(); - auto child_constant_matcher = make_uniq(); - auto child_expression_matcher = make_uniq(); - child_constant_matcher->type = make_uniq(); - child_expression_matcher->type = make_uniq(); - arithmetic->matchers.push_back(std::move(child_constant_matcher)); - arithmetic->matchers.push_back(std::move(child_expression_matcher)); - arithmetic->policy = SetMatcher::Policy::SOME; - op->matchers.push_back(std::move(arithmetic)); - root = std::move(op); -} - -unique_ptr MoveConstantsRule::Apply(LogicalOperator &op, vector> &bindings, - bool &changes_made, bool is_root) { - auto &comparison = bindings[0].get().Cast(); - auto &outer_constant = bindings[1].get().Cast(); - auto &arithmetic = bindings[2].get().Cast(); - auto &inner_constant = bindings[3].get().Cast(); - D_ASSERT(arithmetic.return_type.IsIntegral()); - D_ASSERT(arithmetic.children[0]->return_type.IsIntegral()); - if (inner_constant.value.IsNull() || outer_constant.value.IsNull()) { - return make_uniq(Value(comparison.return_type)); - } - auto &constant_type = outer_constant.return_type; - hugeint_t outer_value = IntegralValue::Get(outer_constant.value); - hugeint_t inner_value = IntegralValue::Get(inner_constant.value); - - idx_t arithmetic_child_index = arithmetic.children[0].get() == &inner_constant ? 1 : 0; - auto &op_type = arithmetic.function.name; - if (op_type == "+") { - // [x + 1 COMP 10] OR [1 + x COMP 10] - // order does not matter in addition: - // simply change right side to 10-1 (outer_constant - inner_constant) - if (!Hugeint::TrySubtractInPlace(outer_value, inner_value)) { - return nullptr; - } - auto result_value = Value::HUGEINT(outer_value); - if (!result_value.DefaultTryCastAs(constant_type)) { - if (comparison.type != ExpressionType::COMPARE_EQUAL) { - return nullptr; - } - // if the cast is not possible then the comparison is not possible - // for example, if we have x + 5 = 3, where x is an unsigned number, we will get x = -2 - // since this is not possible we can remove the entire branch here - return ExpressionRewriter::ConstantOrNull(std::move(arithmetic.children[arithmetic_child_index]), - Value::BOOLEAN(false)); - } - outer_constant.value = std::move(result_value); - } else if (op_type == "-") { - // [x - 1 COMP 10] O R [1 - x COMP 10] - // order matters in subtraction: - if (arithmetic_child_index == 0) { - // [x - 1 COMP 10] - // change right side to 10+1 (outer_constant + inner_constant) - if (!Hugeint::TryAddInPlace(outer_value, inner_value)) { - return nullptr; - } - auto result_value = Value::HUGEINT(outer_value); - if (!result_value.DefaultTryCastAs(constant_type)) { - // if the cast is not possible then an equality comparison is not possible - if (comparison.type != ExpressionType::COMPARE_EQUAL) { - return nullptr; - } - return ExpressionRewriter::ConstantOrNull(std::move(arithmetic.children[arithmetic_child_index]), - Value::BOOLEAN(false)); - } - outer_constant.value = std::move(result_value); - } else { - // [1 - x COMP 10] - // change right side to 1-10=-9 - if (!Hugeint::TrySubtractInPlace(inner_value, outer_value)) { - return nullptr; - } - auto result_value = Value::HUGEINT(inner_value); - if (!result_value.DefaultTryCastAs(constant_type)) { - // if the cast is not possible then an equality comparison is not possible - if (comparison.type != ExpressionType::COMPARE_EQUAL) { - return nullptr; - } - return ExpressionRewriter::ConstantOrNull(std::move(arithmetic.children[arithmetic_child_index]), - Value::BOOLEAN(false)); - } - outer_constant.value = std::move(result_value); - // in this case, we should also flip the comparison - // e.g. if we have [4 - x < 2] then we should have [x > 2] - comparison.type = FlipComparisonExpression(comparison.type); - } - } else { - D_ASSERT(op_type == "*"); - // [x * 2 COMP 10] OR [2 * x COMP 10] - // order does not matter in multiplication: - // change right side to 10/2 (outer_constant / inner_constant) - // but ONLY if outer_constant is cleanly divisible by the inner_constant - if (inner_value == 0) { - // x * 0, the result is either 0 or NULL - // we let the arithmetic_simplification rule take care of simplifying this first - return nullptr; - } - // check out of range for HUGEINT or not cleanly divisible - // HUGEINT is not cleanly divisible when outer_value == minimum and inner value == -1. (modulo overflow) - if ((outer_value == NumericLimits::Minimum() && inner_value == -1) || - outer_value % inner_value != 0) { - bool is_equality = comparison.type == ExpressionType::COMPARE_EQUAL; - bool is_inequality = comparison.type == ExpressionType::COMPARE_NOTEQUAL; - if (is_equality || is_inequality) { - // we know the values are not equal - // the result will be either FALSE or NULL (if COMPARE_EQUAL) - // or TRUE or NULL (if COMPARE_NOTEQUAL) - return ExpressionRewriter::ConstantOrNull(std::move(arithmetic.children[arithmetic_child_index]), - Value::BOOLEAN(is_inequality)); - } else { - // not cleanly divisible and we are doing > >= < <=, skip the simplification for now - return nullptr; - } - } - if (inner_value < 0) { - // multiply by negative value, need to flip expression - comparison.type = FlipComparisonExpression(comparison.type); - } - // else divide the RHS by the LHS - // we need to do a range check on the cast even though we do a division - // because e.g. -128 / -1 = 128, which is out of range - auto result_value = Value::HUGEINT(outer_value / inner_value); - if (!result_value.DefaultTryCastAs(constant_type)) { - return ExpressionRewriter::ConstantOrNull(std::move(arithmetic.children[arithmetic_child_index]), - Value::BOOLEAN(false)); - } - outer_constant.value = std::move(result_value); - } - // replace left side with x - // first extract x from the arithmetic expression - auto arithmetic_child = std::move(arithmetic.children[arithmetic_child_index]); - // then place in the comparison - if (comparison.left.get() == &outer_constant) { - comparison.right = std::move(arithmetic_child); - } else { - comparison.left = std::move(arithmetic_child); - } - changes_made = true; - return nullptr; -} - -} // namespace duckdb diff --git a/src/optimizer/rule/CMakeLists.txt b/src/optimizer/rule/CMakeLists.txt index f6e957fcad07..1ebbb4a0708f 100644 --- a/src/optimizer/rule/CMakeLists.txt +++ b/src/optimizer/rule/CMakeLists.txt @@ -11,9 +11,9 @@ add_library_unity( empty_needle_removal.cpp enum_comparison.cpp equal_or_null_simplification.cpp - move_constants.cpp - like_optimizations.cpp in_clause_simplification_rule.cpp + like_optimizations.cpp + move_constants.cpp ordered_aggregate_optimizer.cpp regex_optimizations.cpp) set(ALL_OBJECT_FILES From 967da94f7ba2aa5bd8a53ca5f233fae1049892b5 Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 15 Apr 2024 14:52:34 +0200 Subject: [PATCH 442/603] undo logical dependency --- src/catalog/dependency_list.cpp | 8 -------- src/include/duckdb/catalog/dependency_list.hpp | 2 -- 2 files changed, 10 deletions(-) diff --git a/src/catalog/dependency_list.cpp b/src/catalog/dependency_list.cpp index a7a30468dc24..1ef3be15fbc4 100644 --- a/src/catalog/dependency_list.cpp +++ b/src/catalog/dependency_list.cpp @@ -40,10 +40,6 @@ bool LogicalDependencyEquality::operator()(const LogicalDependency &a, const Log LogicalDependency::LogicalDependency() : entry(), catalog() { } -unique_ptr LogicalDependency::Copy() const { - return make_uniq(catalog, entry); -} - static string GetSchema(CatalogEntry &entry) { if (entry.type == CatalogType::SCHEMA_ENTRY) { return entry.name; @@ -65,10 +61,6 @@ LogicalDependency::LogicalDependency(CatalogEntry &entry) { } } -LogicalDependency::LogicalDependency(const string &catalog, const CatalogEntryInfo &entry) - : entry(entry), catalog(catalog) { -} - bool LogicalDependency::operator==(const LogicalDependency &other) const { return other.entry.name == entry.name && other.entry.schema == entry.schema && other.entry.type == entry.type; } diff --git a/src/include/duckdb/catalog/dependency_list.hpp b/src/include/duckdb/catalog/dependency_list.hpp index 7a76022a2121..ed08f01ac945 100644 --- a/src/include/duckdb/catalog/dependency_list.hpp +++ b/src/include/duckdb/catalog/dependency_list.hpp @@ -30,9 +30,7 @@ struct LogicalDependency { public: explicit LogicalDependency(CatalogEntry &entry); - explicit LogicalDependency(const string &catalog, const CatalogEntryInfo &entry); LogicalDependency(); - unique_ptr Copy() const; bool operator==(const LogicalDependency &other) const; }; From 3eac29080233248a44e948b4e647d2d78c1bb6e0 Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 15 Apr 2024 14:53:58 +0200 Subject: [PATCH 443/603] add missing files --- src/execution/physical_plan/plan_vacuum.cpp | 18 +++++ .../planner/operator/logical_vacuum.hpp | 52 +++++++++++++++ src/planner/operator/logical_vacuum.cpp | 65 +++++++++++++++++++ 3 files changed, 135 insertions(+) create mode 100644 src/execution/physical_plan/plan_vacuum.cpp create mode 100644 src/include/duckdb/planner/operator/logical_vacuum.hpp create mode 100644 src/planner/operator/logical_vacuum.cpp diff --git a/src/execution/physical_plan/plan_vacuum.cpp b/src/execution/physical_plan/plan_vacuum.cpp new file mode 100644 index 000000000000..804fd1fa22af --- /dev/null +++ b/src/execution/physical_plan/plan_vacuum.cpp @@ -0,0 +1,18 @@ +#include "duckdb/execution/operator/helper/physical_vacuum.hpp" +#include "duckdb/execution/physical_plan_generator.hpp" +#include "duckdb/planner/logical_operator.hpp" +#include "duckdb/planner/operator/logical_vacuum.hpp" + +namespace duckdb { + +unique_ptr PhysicalPlanGenerator::CreatePlan(LogicalVacuum &op) { + auto result = make_uniq(unique_ptr_cast(std::move(op.info)), + std::move(op.table), std::move(op.column_id_map), op.estimated_cardinality); + if (!op.children.empty()) { + auto child = CreatePlan(*op.children[0]); + result->children.push_back(std::move(child)); + } + return std::move(result); +} + +} // namespace duckdb diff --git a/src/include/duckdb/planner/operator/logical_vacuum.hpp b/src/include/duckdb/planner/operator/logical_vacuum.hpp new file mode 100644 index 000000000000..c22189e1da72 --- /dev/null +++ b/src/include/duckdb/planner/operator/logical_vacuum.hpp @@ -0,0 +1,52 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// duckdb/planner/operator/logical_vacuum.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/common/enums/statement_type.hpp" +#include "duckdb/parser/parsed_data/parse_info.hpp" +#include "duckdb/planner/logical_operator.hpp" +#include "duckdb/parser/parsed_data/vacuum_info.hpp" + +namespace duckdb { + +//! LogicalVacuum represents a simple logical operator that only passes on the parse info +class LogicalVacuum : public LogicalOperator { +public: + static constexpr const LogicalOperatorType TYPE = LogicalOperatorType::LOGICAL_VACUUM; + +public: + LogicalVacuum(); + LogicalVacuum(unique_ptr info); + +public: + VacuumInfo &GetInfo() { + return *info; + } + + TableCatalogEntry &GetTable(); + bool HasTable() const; + void SetTable(TableCatalogEntry &table_p); + +public: + optional_ptr table; + unordered_map column_id_map; + unique_ptr info; + +public: + void Serialize(Serializer &serializer) const override; + static unique_ptr Deserialize(Deserializer &deserializer); + idx_t EstimateCardinality(ClientContext &context) override; + +protected: + void ResolveTypes() override { + types.emplace_back(LogicalType::BOOLEAN); + } +}; + +} // namespace duckdb diff --git a/src/planner/operator/logical_vacuum.cpp b/src/planner/operator/logical_vacuum.cpp new file mode 100644 index 000000000000..f2a82d7b84f4 --- /dev/null +++ b/src/planner/operator/logical_vacuum.cpp @@ -0,0 +1,65 @@ +#include "duckdb/planner/operator/logical_vacuum.hpp" + +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/common/serializer/serializer.hpp" +#include "duckdb/common/serializer/deserializer.hpp" +#include "duckdb/parser/parsed_data/vacuum_info.hpp" + +namespace duckdb { + +LogicalVacuum::LogicalVacuum() : LogicalOperator(LogicalOperatorType::LOGICAL_VACUUM) { +} + +LogicalVacuum::LogicalVacuum(unique_ptr info) + : LogicalOperator(LogicalOperatorType::LOGICAL_VACUUM), info(std::move(info)) { +} + +TableCatalogEntry &LogicalVacuum::GetTable() { + D_ASSERT(HasTable()); + return *table; +} +bool LogicalVacuum::HasTable() const { + return table != nullptr; +} + +void LogicalVacuum::SetTable(TableCatalogEntry &table_p) { + table = &table_p; +} + +void LogicalVacuum::Serialize(Serializer &serializer) const { + LogicalOperator::Serialize(serializer); + + serializer.WriteProperty(200, "info", info); + serializer.WriteProperty(201, "column_id_map", column_id_map); +} + +unique_ptr LogicalVacuum::Deserialize(Deserializer &deserializer) { + auto result = unique_ptr(new LogicalVacuum()); + + auto tmp_info = deserializer.ReadPropertyWithDefault>(200, "info"); + deserializer.ReadProperty(201, "column_id_map", result->column_id_map); + + result->info = unique_ptr_cast(std::move(tmp_info)); + auto &info = *result->info; + if (info.has_table) { + // deserialize the 'table' + auto &context = deserializer.Get(); + auto binder = Binder::CreateBinder(context); + auto bound_table = binder->Bind(*info.ref); + if (bound_table->type != TableReferenceType::BASE_TABLE) { + throw InvalidInputException("Can only vacuum/analyze base tables!"); + } + auto ref = unique_ptr_cast(std::move(bound_table)); + auto &table = ref->table; + result->SetTable(table); + // FIXME: we should probably verify that the 'column_id_map' and 'columns' are the same on the bound table after + // deserialization? + } + return std::move(result); +} + +idx_t LogicalVacuum::EstimateCardinality(ClientContext &context) { + return 1; +} + +} // namespace duckdb From 707495d648326dfda2db61f23cb4e7882ab3fa26 Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 15 Apr 2024 14:55:06 +0200 Subject: [PATCH 444/603] add pragma enable_verification --- test/fuzzer/pedro/vacuum_unsupported_type.test | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/fuzzer/pedro/vacuum_unsupported_type.test b/test/fuzzer/pedro/vacuum_unsupported_type.test index 0b0d1470e8e0..f56804d38861 100644 --- a/test/fuzzer/pedro/vacuum_unsupported_type.test +++ b/test/fuzzer/pedro/vacuum_unsupported_type.test @@ -2,11 +2,8 @@ # description: Vacuum a table with arrays # group: [pedro] -#statement ok -#pragma enable_verification - statement ok -pragma verify_serializer +pragma enable_verification statement ok CREATE TABLE t1 AS (VALUES ([], {'a': []})); From 138e13511913fdd94d372b756d66da317f1608ce Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 15 Apr 2024 14:58:38 +0200 Subject: [PATCH 445/603] undo unrelated changes --- CMakeLists.txt | 5 ----- Makefile | 3 --- 2 files changed, 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f7c39b58b070..7b1eaa7adfa2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -355,7 +355,6 @@ option(DISABLE_STR_INLINE "Debug setting: disable inlining of strings" FALSE) option(DISABLE_MEMORY_SAFETY "Debug setting: disable memory access checks at runtime" FALSE) option(DISABLE_ASSERTIONS "Debug setting: disable assertions" FALSE) option(ALTERNATIVE_VERIFY "Debug setting: use alternative verify mode" FALSE) -option(ALTERNATIVE_VERIFY2 "Debug setting: use alternative verify mode version 2" FALSE) option(RUN_SLOW_VERIFIERS "Debug setting: enable a more extensive set of verifiers" FALSE) option(DESTROY_UNPINNED_BLOCKS "Debug setting: destroy unpinned buffer-managed blocks" FALSE) option(FORCE_ASYNC_SINK_SOURCE "Debug setting: forces sinks/sources to block the first 2 times they're called" FALSE) @@ -433,10 +432,6 @@ if(ALTERNATIVE_VERIFY) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDUCKDB_ALTERNATIVE_VERIFY") endif() -if(ALTERNATIVE_VERIFY2) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDUCKDB_ALTERNATIVE_VERIFY2") -endif() - if(DEBUG_ALLOCATION) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDUCKDB_DEBUG_ALLOCATION") endif() diff --git a/Makefile b/Makefile index a74d5be9479f..a6fd0dcd3ea0 100644 --- a/Makefile +++ b/Makefile @@ -212,9 +212,6 @@ endif ifeq (${ALTERNATIVE_VERIFY}, 1) CMAKE_VARS:=${CMAKE_VARS} -DALTERNATIVE_VERIFY=1 endif -ifeq (${ALTERNATIVE_VERIFY2}, 1) - CMAKE_VARS:=${CMAKE_VARS} -DALTERNATIVE_VERIFY2=1 -endif ifneq (${VERIFY_VECTOR}, ) CMAKE_VARS:=${CMAKE_VARS} -DVERIFY_VECTOR=${VERIFY_VECTOR} endif From 16942db17c6353d70a57225e25e56f74bb75211e Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 15 Apr 2024 16:11:48 +0200 Subject: [PATCH 446/603] fix compilation --- src/execution/operator/helper/physical_vacuum.cpp | 2 +- src/execution/physical_plan/plan_vacuum.cpp | 4 ++-- src/include/duckdb/planner/operator/logical_vacuum.hpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/execution/operator/helper/physical_vacuum.cpp b/src/execution/operator/helper/physical_vacuum.cpp index 77c21d82ca70..186c15411a08 100644 --- a/src/execution/operator/helper/physical_vacuum.cpp +++ b/src/execution/operator/helper/physical_vacuum.cpp @@ -10,7 +10,7 @@ namespace duckdb { PhysicalVacuum::PhysicalVacuum(unique_ptr info_p, optional_ptr table, unordered_map column_id_map, idx_t estimated_cardinality) : PhysicalOperator(PhysicalOperatorType::VACUUM, {LogicalType::BOOLEAN}, estimated_cardinality), - info(std::move(info_p)), table(std::move(table)), column_id_map(std::move(column_id_map)) { + info(std::move(info_p)), table(table), column_id_map(std::move(column_id_map)) { } class VacuumLocalSinkState : public LocalSinkState { diff --git a/src/execution/physical_plan/plan_vacuum.cpp b/src/execution/physical_plan/plan_vacuum.cpp index 804fd1fa22af..3f1f88b3dbf3 100644 --- a/src/execution/physical_plan/plan_vacuum.cpp +++ b/src/execution/physical_plan/plan_vacuum.cpp @@ -6,8 +6,8 @@ namespace duckdb { unique_ptr PhysicalPlanGenerator::CreatePlan(LogicalVacuum &op) { - auto result = make_uniq(unique_ptr_cast(std::move(op.info)), - std::move(op.table), std::move(op.column_id_map), op.estimated_cardinality); + auto result = make_uniq(unique_ptr_cast(std::move(op.info)), op.table, + std::move(op.column_id_map), op.estimated_cardinality); if (!op.children.empty()) { auto child = CreatePlan(*op.children[0]); result->children.push_back(std::move(child)); diff --git a/src/include/duckdb/planner/operator/logical_vacuum.hpp b/src/include/duckdb/planner/operator/logical_vacuum.hpp index c22189e1da72..2ecf1088c179 100644 --- a/src/include/duckdb/planner/operator/logical_vacuum.hpp +++ b/src/include/duckdb/planner/operator/logical_vacuum.hpp @@ -22,7 +22,7 @@ class LogicalVacuum : public LogicalOperator { public: LogicalVacuum(); - LogicalVacuum(unique_ptr info); + explicit LogicalVacuum(unique_ptr info); public: VacuumInfo &GetInfo() { From 2bd55e4aab53b7f6844c47e8c4630b5ebf86db35 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 15 Apr 2024 16:12:42 +0200 Subject: [PATCH 447/603] Testing more unhappy paths --- .../csv_scanner/util/csv_reader_options.cpp | 12 +++++++--- test/sql/copy/csv/csv_nullstr_list.test | 23 +++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp index f9da6ac8dc2b..c5cfd2d4f6bf 100644 --- a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp +++ b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp @@ -293,18 +293,24 @@ bool CSVReaderOptions::SetBaseOption(const string &loption, const Value &value) auto &child_type = value.type(); null_str.clear(); if (child_type.id() != LogicalTypeId::LIST && child_type.id() != LogicalTypeId::VARCHAR) { - throw BinderException("read_csv %s option requires a string or a list as input", loption); + throw BinderException("CSV Reader function option %s requires a string or a list as input", loption); } if (!null_str.empty()) { - throw BinderException("read_csv_auto nullstr can only be supplied once"); + throw BinderException("CSV Reader function option nullstr can only be supplied once"); } if (child_type.id() == LogicalTypeId::LIST) { auto &list_child = ListType::GetChildType(child_type); if (list_child.id() != LogicalTypeId::VARCHAR) { - throw BinderException("read_csv_auto %s requires a list of types (varchar) as input", loption); + throw BinderException("CSV Reader function option %s requires a non-empty list of possible null " + "strings (varchar) as input", + loption); } auto &children = ListValue::GetChildren(value); for (auto &child : children) { + if (child.IsNull()) { + throw BinderException( + "CSV Reader function option %s does not accept NULL values as a valid nullstr option", loption); + } null_str.push_back(StringValue::Get(child)); } } else { diff --git a/test/sql/copy/csv/csv_nullstr_list.test b/test/sql/copy/csv/csv_nullstr_list.test index 0e7914a6dc2a..0d3ca41175ba 100644 --- a/test/sql/copy/csv/csv_nullstr_list.test +++ b/test/sql/copy/csv/csv_nullstr_list.test @@ -29,6 +29,29 @@ Pedro 31 1.73 Mark null (empty) Thijs 26 none +# Test nullstr = [] +statement error +FROM read_csv('data/csv/null/multiple_nulls.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'name': 'VARCHAR', 'age': 'VARCHAR', 'height': 'VARCHAR'}, nullstr = []); +---- +CSV Reader function option nullstr requires a non-empty list of possible null strings (varchar) as input + +# Test nullstr = ['a', NULL] +statement error +FROM read_csv('data/csv/null/multiple_nulls.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'name': 'VARCHAR', 'age': 'VARCHAR', 'height': 'VARCHAR'}, nullstr = ['a', NULL]); +---- +CSV Reader function option nullstr does not accept NULL values as a valid nullstr option + +# Test nullstr = NULL +statement error +FROM read_csv('data/csv/null/multiple_nulls.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'name': 'VARCHAR', 'age': 'VARCHAR', 'height': 'VARCHAR'}, nullstr = NULL); +---- +CSV Reader function option nullstr requires a string or a list as input + +# Test nullstr = [42] +statement error +FROM read_csv('data/csv/null/multiple_nulls.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'name': 'VARCHAR', 'age': 'VARCHAR', 'height': 'VARCHAR'}, nullstr = [42]); +---- +CSV Reader function option nullstr requires a non-empty list of possible null strings (varchar) as input # Test Null Strings equal to delim quote escape statement error From d16dff09afe68ee7af8f94a433ec62b028cc4dd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Mon, 15 Apr 2024 16:14:04 +0200 Subject: [PATCH 448/603] orr array_slice --- src/core_functions/scalar/list/array_slice.cpp | 18 ++++++++---------- src/parallel/task_scheduler.cpp | 1 + 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/core_functions/scalar/list/array_slice.cpp b/src/core_functions/scalar/list/array_slice.cpp index a80bf1c7d890..c04e037b4a7c 100644 --- a/src/core_functions/scalar/list/array_slice.cpp +++ b/src/core_functions/scalar/list/array_slice.cpp @@ -47,17 +47,15 @@ static idx_t CalculateSliceLength(idx_t begin, idx_t end, INDEX_TYPE step, bool if (step == 0 && svalid) { throw InvalidInputException("Slice step cannot be zero"); } - auto step_unsigned = UnsafeNumericCast(step); // we called abs() on this above. - if (step == 1) { return NumericCast(end - begin); - } else if (step_unsigned >= (end - begin)) { + } else if (static_cast(step) >= (end - begin)) { return 1; } - if ((end - begin) % step_unsigned != 0) { - return (end - begin) / step_unsigned + 1; + if ((end - begin) % UnsafeNumericCast(step) != 0) { + return (end - begin) / UnsafeNumericCast(step) + 1; } - return (end - begin) / step_unsigned; + return (end - begin) / UnsafeNumericCast(step); } template @@ -147,14 +145,14 @@ list_entry_t SliceValueWithSteps(Vector &result, SelectionVector &sel, list_entr return input; } input.length = CalculateSliceLength(UnsafeNumericCast(begin), UnsafeNumericCast(end), step, true); - int64_t child_idx = UnsafeNumericCast(input.offset) + begin; + idx_t child_idx = input.offset + UnsafeNumericCast(begin); if (step < 0) { - child_idx = UnsafeNumericCast(input.offset) + end - 1; + child_idx = input.offset + UnsafeNumericCast(end) - 1; } input.offset = sel_idx; for (idx_t i = 0; i < input.length; i++) { - sel.set_index(sel_idx, UnsafeNumericCast(child_idx)); - child_idx = UnsafeNumericCast(child_idx) + step; + sel.set_index(sel_idx, child_idx); + child_idx += static_cast(step); // intentional overflow?? sel_idx++; } return input; diff --git a/src/parallel/task_scheduler.cpp b/src/parallel/task_scheduler.cpp index b0f8997ee82a..ea7dc0c6d963 100644 --- a/src/parallel/task_scheduler.cpp +++ b/src/parallel/task_scheduler.cpp @@ -2,6 +2,7 @@ #include "duckdb/common/chrono.hpp" #include "duckdb/common/exception.hpp" +#include "duckdb/common/numeric_utils.hpp" #include "duckdb/main/client_context.hpp" #include "duckdb/main/database.hpp" From f303d87f5f90112b7123e1b07f6c4460f1225cb8 Mon Sep 17 00:00:00 2001 From: Christina Sioula Date: Mon, 15 Apr 2024 09:18:18 +0200 Subject: [PATCH 449/603] change unzip.test vector's and block's sizes and to require 64bit --- test/sql/storage/unzip.test | 10 ++++++++++ tools/pythonpkg/scripts/sqllogictest_python.py | 5 ++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/test/sql/storage/unzip.test b/test/sql/storage/unzip.test index 53fe95aea1ea..f7f4525c199d 100644 --- a/test/sql/storage/unzip.test +++ b/test/sql/storage/unzip.test @@ -2,6 +2,16 @@ # description: Support gzipped files in the test runner # group: [storage] +# data/storage/index_0-9-1.db was written with a 64-bit version of duckdb +require 64bit + +require block_size 262144 + +require vector_size 2048 + +statement ok +PRAGMA enable_verification + # unzip to specific path unzip data/storage/test.db.gz __TEST_DIR__/test.db diff --git a/tools/pythonpkg/scripts/sqllogictest_python.py b/tools/pythonpkg/scripts/sqllogictest_python.py index a1c8345e179c..37cbc2b68815 100644 --- a/tools/pythonpkg/scripts/sqllogictest_python.py +++ b/tools/pythonpkg/scripts/sqllogictest_python.py @@ -173,12 +173,11 @@ def main(): if test_directory: file_path = os.path.join(test_directory, file_path) - test = SQLLogicTest("") try: test = sql_parser.parse(file_path) - if not test: - raise SQLParserException(f'failed to parse {file_path}') except SQLParserException as e: + if not ("test" in locals()): # test hasn't been initialized because of the raised exception + test = SQLLogicTest("") test.skip(True) executor.skip_log.append(str(e.message)) From 5168f7047516b4711e25fb6bd9ea22daf0d819c8 Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 15 Apr 2024 16:32:39 +0200 Subject: [PATCH 450/603] clang tidy --- src/include/duckdb/common/shared_ptr.ipp | 17 +++++++++-------- src/include/duckdb/common/weak_ptr.ipp | 6 ++++-- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/include/duckdb/common/shared_ptr.ipp b/src/include/duckdb/common/shared_ptr.ipp index 840f101588d4..6704724c21b1 100644 --- a/src/include/duckdb/common/shared_ptr.ipp +++ b/src/include/duckdb/common/shared_ptr.ipp @@ -32,7 +32,7 @@ private: friend class shared_ptr; template - friend shared_ptr shared_ptr_cast(shared_ptr src); + friend shared_ptr shared_ptr_cast(shared_ptr src); // NOLINT: invalid case style private: original internal; @@ -41,7 +41,7 @@ public: // Constructors shared_ptr() : internal() { } - shared_ptr(std::nullptr_t) : internal(nullptr) { + shared_ptr(std::nullptr_t) : internal(nullptr) { // NOLINT: not marked as explicit } // From raw pointer of type U convertible to T @@ -95,7 +95,7 @@ public: typename std::enable_if::value && std::is_convertible::pointer, T *>::value, int>::type = 0> - shared_ptr(unique_ptr &&other) : internal(std::move(other)) { + shared_ptr(unique_ptr &&other) : internal(std::move(other)) { // NOLINT: not marked as explicit __enable_weak_this(internal.get(), internal.get()); } @@ -161,15 +161,15 @@ public: internal.reset(ptr, deleter); } - void swap(shared_ptr &r) noexcept { + void swap(shared_ptr &r) noexcept { // NOLINT: invalid case style internal.swap(r.internal); } - T *get() const { + T *get() const { // NOLINT: invalid case style return internal.get(); } - long use_count() const { + long use_count() const { // NOLINT: invalid case style return internal.use_count(); } @@ -236,7 +236,8 @@ private: template *>::value, int>::type = 0> - void __enable_weak_this(const enable_shared_from_this *object, _OrigPtr *ptr) noexcept { + void __enable_weak_this(const enable_shared_from_this *object, + _OrigPtr *ptr) noexcept { // NOLINT: invalid case style typedef typename std::remove_cv::type NonConstU; if (object && object->__weak_this_.expired()) { // __weak_this__ is the mutable variable returned by 'shared_from_this' @@ -245,7 +246,7 @@ private: } } - void __enable_weak_this(...) noexcept { + void __enable_weak_this(...) noexcept { // NOLINT: invalid case style } }; diff --git a/src/include/duckdb/common/weak_ptr.ipp b/src/include/duckdb/common/weak_ptr.ipp index fff31e251e04..aef42e1f9e7e 100644 --- a/src/include/duckdb/common/weak_ptr.ipp +++ b/src/include/duckdb/common/weak_ptr.ipp @@ -18,23 +18,25 @@ public: weak_ptr() : internal() { } + // NOLINTBEGIN template weak_ptr(shared_ptr const &ptr, typename std::enable_if::value, int>::type = 0) noexcept : internal(ptr.internal) { } - weak_ptr(weak_ptr const &other) noexcept : internal(other.internal) { // NOLINT: not marked as explicit + weak_ptr(weak_ptr const &other) noexcept : internal(other.internal) { } template weak_ptr(weak_ptr const &ptr, typename std::enable_if::value, int>::type = 0) noexcept : internal(ptr.internal) { } - weak_ptr(weak_ptr &&ptr) noexcept : internal(std::move(ptr.internal)) { // NOLINT: not marked as explicit + weak_ptr(weak_ptr &&ptr) noexcept : internal(std::move(ptr.internal)) { } template weak_ptr(weak_ptr &&ptr, typename std::enable_if::value, int>::type = 0) noexcept : internal(std::move(ptr.internal)) { } + // NOLINTEND // Destructor ~weak_ptr() = default; From 2ef214f55eb41e9a80024d222ea0390926543d17 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Mon, 15 Apr 2024 16:45:35 +0200 Subject: [PATCH 451/603] Fix #11542 - correctly check if a column has updates by checking also if the validity component has updates --- .../duckdb/storage/table/column_data.hpp | 4 +- .../storage/table/standard_column_data.hpp | 2 + src/storage/table/column_data.cpp | 70 ++++++++++--------- src/storage/table/standard_column_data.cpp | 4 ++ 4 files changed, 46 insertions(+), 34 deletions(-) diff --git a/src/include/duckdb/storage/table/column_data.hpp b/src/include/duckdb/storage/table/column_data.hpp index 0ec917255f19..e7bc7a2e0a6c 100644 --- a/src/include/duckdb/storage/table/column_data.hpp +++ b/src/include/duckdb/storage/table/column_data.hpp @@ -75,6 +75,8 @@ class ColumnData { virtual void SetStart(idx_t new_start); //! The root type of the column const LogicalType &RootType() const; + //! Whether or not the column has any updates + virtual bool HasUpdates() const; //! Initialize a scan of the column virtual void InitializeScan(ColumnScanState &state); @@ -163,7 +165,7 @@ class ColumnData { //! The segments holding the data of this column segment ColumnSegmentTree data; //! The lock for the updates - mutex update_lock; + mutable mutex update_lock; //! The updates for this column segment unique_ptr updates; //! The stats of the root segment diff --git a/src/include/duckdb/storage/table/standard_column_data.hpp b/src/include/duckdb/storage/table/standard_column_data.hpp index 135024122f84..9ad0780d0ebd 100644 --- a/src/include/duckdb/storage/table/standard_column_data.hpp +++ b/src/include/duckdb/storage/table/standard_column_data.hpp @@ -23,6 +23,8 @@ class StandardColumnData : public ColumnData { ValidityColumnData validity; public: + bool HasUpdates() const override; + void SetStart(idx_t new_start) override; bool CheckZonemap(ColumnScanState &state, TableFilter &filter) override; diff --git a/src/storage/table/column_data.cpp b/src/storage/table/column_data.cpp index e48495049d7e..21f4888f6fe1 100644 --- a/src/storage/table/column_data.cpp +++ b/src/storage/table/column_data.cpp @@ -57,6 +57,11 @@ const LogicalType &ColumnData::RootType() const { return type; } +bool ColumnData::HasUpdates() const { + lock_guard update_guard(update_lock); + return updates.get(); +} + idx_t ColumnData::GetMaxEntry() { return count; } @@ -135,11 +140,7 @@ idx_t ColumnData::ScanVector(ColumnScanState &state, Vector &result, idx_t remai template idx_t ColumnData::ScanVector(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result) { - bool has_updates; - { - lock_guard update_guard(update_lock); - has_updates = updates ? true : false; - } + bool has_updates = HasUpdates(); idx_t current_row = vector_index * STANDARD_VECTOR_SIZE; auto vector_count = MinValue(STANDARD_VECTOR_SIZE, count - current_row); @@ -180,61 +181,62 @@ idx_t ColumnData::ScanCommitted(idx_t vector_index, ColumnScanState &state, Vect } } -void ColumnData::ScanCommittedRange(idx_t row_group_start, idx_t offset_in_row_group, idx_t count, Vector &result) { +void ColumnData::ScanCommittedRange(idx_t row_group_start, idx_t offset_in_row_group, idx_t s_count, Vector &result) { ColumnScanState child_state; InitializeScanWithOffset(child_state, row_group_start + offset_in_row_group); - auto scan_count = ScanVector(child_state, result, count, updates ? true : false); - if (updates) { + bool has_updates = HasUpdates(); + auto scan_count = ScanVector(child_state, result, s_count, has_updates); + if (has_updates) { result.Flatten(scan_count); - updates->FetchCommittedRange(offset_in_row_group, count, result); + updates->FetchCommittedRange(offset_in_row_group, s_count, result); } } -idx_t ColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t count) { - if (count == 0) { +idx_t ColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t scan_count) { + if (scan_count == 0) { return 0; } // ScanCount can only be used if there are no updates - D_ASSERT(!updates); - return ScanVector(state, result, count, false); + D_ASSERT(!HasUpdates()); + return ScanVector(state, result, scan_count, false); } void ColumnData::Select(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result, - SelectionVector &sel, idx_t &count, const TableFilter &filter) { + SelectionVector &sel, idx_t &s_count, const TableFilter &filter) { idx_t scan_count = Scan(transaction, vector_index, state, result); UnifiedVectorFormat vdata; result.ToUnifiedFormat(scan_count, vdata); - ColumnSegment::FilterSelection(sel, result, vdata, filter, scan_count, count); + ColumnSegment::FilterSelection(sel, result, vdata, filter, scan_count, s_count); } void ColumnData::FilterScan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result, - SelectionVector &sel, idx_t count) { + SelectionVector &sel, idx_t s_count) { Scan(transaction, vector_index, state, result); - result.Slice(sel, count); + result.Slice(sel, s_count); } void ColumnData::FilterScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, SelectionVector &sel, - idx_t count, bool allow_updates) { + idx_t s_count, bool allow_updates) { ScanCommitted(vector_index, state, result, allow_updates); - result.Slice(sel, count); + result.Slice(sel, s_count); } -void ColumnData::Skip(ColumnScanState &state, idx_t count) { - state.Next(count); +void ColumnData::Skip(ColumnScanState &state, idx_t s_count) { + state.Next(s_count); } -void ColumnData::Append(BaseStatistics &stats, ColumnAppendState &state, Vector &vector, idx_t count) { +void ColumnData::Append(BaseStatistics &append_stats, ColumnAppendState &state, Vector &vector, idx_t append_count) { UnifiedVectorFormat vdata; vector.ToUnifiedFormat(count, vdata); - AppendData(stats, state, vdata, count); + AppendData(append_stats, state, vdata, append_count); } -void ColumnData::Append(ColumnAppendState &state, Vector &vector, idx_t count) { +void ColumnData::Append(ColumnAppendState &state, Vector &vector, idx_t append_count) { if (parent || !stats) { throw InternalException("ColumnData::Append called on a column with a parent or without stats"); } - Append(stats->statistics, state, vector, count); + Append(stats->statistics, state, vector, append_count); } bool ColumnData::CheckZonemap(TableFilter &filter) { @@ -291,14 +293,15 @@ void ColumnData::InitializeAppend(ColumnAppendState &state) { D_ASSERT(state.current->function.get().append); } -void ColumnData::AppendData(BaseStatistics &stats, ColumnAppendState &state, UnifiedVectorFormat &vdata, idx_t count) { +void ColumnData::AppendData(BaseStatistics &append_stats, ColumnAppendState &state, UnifiedVectorFormat &vdata, + idx_t append_count) { idx_t offset = 0; - this->count += count; + this->count += append_count; while (true) { // append the data from the vector - idx_t copied_elements = state.current->Append(state, vdata, offset, count); - stats.Merge(state.current->stats.statistics); - if (copied_elements == count) { + idx_t copied_elements = state.current->Append(state, vdata, offset, append_count); + append_stats.Merge(state.current->stats.statistics); + if (copied_elements == append_count) { // finished copying everything break; } @@ -311,7 +314,7 @@ void ColumnData::AppendData(BaseStatistics &stats, ColumnAppendState &state, Uni state.current->InitializeAppend(state); } offset += copied_elements; - count -= copied_elements; + append_count -= copied_elements; } } @@ -419,16 +422,17 @@ unique_ptr ColumnData::CreateCheckpointState(RowGroup &ro void ColumnData::CheckpointScan(ColumnSegment &segment, ColumnScanState &state, idx_t row_group_start, idx_t count, Vector &scan_vector) { + bool has_updates = HasUpdates(); if (state.scan_options && state.scan_options->force_fetch_row) { for (idx_t i = 0; i < count; i++) { ColumnFetchState fetch_state; segment.FetchRow(fetch_state, state.row_index + i, scan_vector, i); } } else { - segment.Scan(state, count, scan_vector, 0, true); + segment.Scan(state, count, scan_vector, 0, !has_updates); } - if (updates) { + if (has_updates) { scan_vector.Flatten(count); updates->FetchCommittedRange(state.row_index - row_group_start, count, scan_vector); } diff --git a/src/storage/table/standard_column_data.cpp b/src/storage/table/standard_column_data.cpp index 9347e72e3fa6..ce851f00e747 100644 --- a/src/storage/table/standard_column_data.cpp +++ b/src/storage/table/standard_column_data.cpp @@ -21,6 +21,10 @@ void StandardColumnData::SetStart(idx_t new_start) { validity.SetStart(new_start); } +bool StandardColumnData::HasUpdates() const { + return ColumnData::HasUpdates() || validity.HasUpdates(); +} + bool StandardColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filter) { if (!state.segment_checked) { if (!state.current) { From 13c9caaef6d1a2ddcbc885ba841645a8475644a8 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Mon, 15 Apr 2024 16:55:31 +0200 Subject: [PATCH 452/603] Correctly check that updates are set here --- src/storage/table/column_data.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/storage/table/column_data.cpp b/src/storage/table/column_data.cpp index 21f4888f6fe1..e4b5648237d8 100644 --- a/src/storage/table/column_data.cpp +++ b/src/storage/table/column_data.cpp @@ -422,18 +422,17 @@ unique_ptr ColumnData::CreateCheckpointState(RowGroup &ro void ColumnData::CheckpointScan(ColumnSegment &segment, ColumnScanState &state, idx_t row_group_start, idx_t count, Vector &scan_vector) { - bool has_updates = HasUpdates(); if (state.scan_options && state.scan_options->force_fetch_row) { for (idx_t i = 0; i < count; i++) { ColumnFetchState fetch_state; segment.FetchRow(fetch_state, state.row_index + i, scan_vector, i); } } else { - segment.Scan(state, count, scan_vector, 0, !has_updates); + segment.Scan(state, count, scan_vector, 0, !HasUpdates()); } - if (has_updates) { - scan_vector.Flatten(count); + if (updates) { + D_ASSERT(scan_vector.GetVectorType() == VectorType::FLAT_VECTOR); updates->FetchCommittedRange(state.row_index - row_group_start, count, scan_vector); } } From 4f110e8ee38e6ed68f9a215672ce69842bd18b36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Mon, 15 Apr 2024 17:01:44 +0200 Subject: [PATCH 453/603] missing header for windows --- src/parallel/task_scheduler.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/parallel/task_scheduler.cpp b/src/parallel/task_scheduler.cpp index ea7dc0c6d963..d327f983213d 100644 --- a/src/parallel/task_scheduler.cpp +++ b/src/parallel/task_scheduler.cpp @@ -16,6 +16,8 @@ #include #endif +#include // ssize_t + namespace duckdb { struct SchedulerThread { From 3809f813e5d7c28c5c154ec987d5cb7b5594eb6c Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 15 Apr 2024 17:05:52 +0200 Subject: [PATCH 454/603] Extending the SQL Parser to handle [] in CSV options and adding copy from/to tests --- .../csv_scanner/util/csv_reader_options.cpp | 26 +- .../csv_scanner/csv_reader_options.hpp | 2 +- test/sql/copy/csv/csv_nullstr_list.test | 58 +- .../libpg_query/grammar/statements/copy.y | 1 + .../libpg_query/grammar/statements/select.y | 11 +- .../libpg_query/grammar/types/select.yh | 2 +- .../libpg_query/src_backend_parser_gram.cpp | 30708 ++++++++-------- 7 files changed, 15442 insertions(+), 15366 deletions(-) diff --git a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp index c5cfd2d4f6bf..c4a24ffbc054 100644 --- a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp +++ b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp @@ -250,7 +250,7 @@ void CSVReaderOptions::SetWriteOption(const string &loption, const Value &value) return; } - if (SetBaseOption(loption, value)) { + if (SetBaseOption(loption, value, true)) { return; } @@ -275,7 +275,7 @@ void CSVReaderOptions::SetWriteOption(const string &loption, const Value &value) } } -bool CSVReaderOptions::SetBaseOption(const string &loption, const Value &value) { +bool CSVReaderOptions::SetBaseOption(const string &loption, const Value &value, bool write_option) { // Make sure this function was only called after the option was turned into lowercase D_ASSERT(!std::any_of(loption.begin(), loption.end(), ::isupper)); @@ -300,13 +300,26 @@ bool CSVReaderOptions::SetBaseOption(const string &loption, const Value &value) } if (child_type.id() == LogicalTypeId::LIST) { auto &list_child = ListType::GetChildType(child_type); - if (list_child.id() != LogicalTypeId::VARCHAR) { + const vector *children = nullptr; + if (list_child.id() == LogicalTypeId::LIST) { + // This can happen if it comes from a copy FROM/TO + auto &list_grandchild = ListType::GetChildType(list_child); + auto &children_ref = ListValue::GetChildren(value); + if (list_grandchild.id() != LogicalTypeId::VARCHAR || children_ref.size() != 1) { + throw BinderException("CSV Reader function option %s requires a non-empty list of possible null " + "strings (varchar) as input", + loption); + } + children = &ListValue::GetChildren(children_ref.back()); + } else if (list_child.id() != LogicalTypeId::VARCHAR) { throw BinderException("CSV Reader function option %s requires a non-empty list of possible null " "strings (varchar) as input", loption); } - auto &children = ListValue::GetChildren(value); - for (auto &child : children) { + if (!children) { + children = &ListValue::GetChildren(value); + } + for (auto &child : *children) { if (child.IsNull()) { throw BinderException( "CSV Reader function option %s does not accept NULL values as a valid nullstr option", loption); @@ -316,6 +329,9 @@ bool CSVReaderOptions::SetBaseOption(const string &loption, const Value &value) } else { null_str.push_back(StringValue::Get(ParseString(value, loption))); } + if (null_str.size() > 1 && write_option) { + throw BinderException("CSV Writer function option %s only accepts one nullstr value.", loption); + } } else if (loption == "encoding") { auto encoding = StringUtil::Lower(ParseString(value, loption)); diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp index 15df01bf27d4..d929fb721aaf 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp @@ -150,7 +150,7 @@ struct CSVReaderOptions { void SetNewline(const string &input); //! Set an option that is supported by both reading and writing functions, called by //! the SetReadOption and SetWriteOption methods - bool SetBaseOption(const string &loption, const Value &value); + bool SetBaseOption(const string &loption, const Value &value, bool write_option = false); //! loption - lowercase string //! set - argument(s) to the option diff --git a/test/sql/copy/csv/csv_nullstr_list.test b/test/sql/copy/csv/csv_nullstr_list.test index 0d3ca41175ba..a20b4501febc 100644 --- a/test/sql/copy/csv/csv_nullstr_list.test +++ b/test/sql/copy/csv/csv_nullstr_list.test @@ -77,30 +77,6 @@ Pedro 31 1.73 Mark null (empty) Thijs 26 none -# Test Copy To -statement ok -create table t (a integer); - -statement ok -insert into t values (0), (null) - -statement ok -COPY t TO '__TEST_DIR__/t_i.csv' (nullstr 'none') - -# Not accepted in copy to/from -statement error -COPY t TO '__TEST_DIR__/t_i.csv' (nullstr ['none', '']) ----- -syntax error at or near "[" - -statement error -COPY t FROM '__TEST_DIR__/t_i.csv' (nullstr ['none', '']) ----- -syntax error at or near "[" - -statement ok -COPY t FROM '__TEST_DIR__/t_i.csv' (nullstr 'none') - # Test with force_not_null query III FROM read_csv('data/csv/null/multiple_nulls.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'name': 'VARCHAR', 'age': 'VARCHAR', 'height': 'VARCHAR'}, nullstr = ['','none','null'], force_not_null = ['height']); @@ -151,3 +127,37 @@ statement error FROM read_csv('data/csv/null/multiple_nulls.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, header=true, columns={'name': 'VARCHAR', 'age': 'VARCHAR', 'height': 'VARCHAR'}, nullstr = ['','none','null'], force_not_null = ['dont_exist']); ---- "force_not_null" expected to find dont_exist, but it was not found in the table + +# Lests add a few tests with copy from +statement ok +CREATE TABLE data (a VARCHAR, b VARCHAR, c VARCHAR) + +statement ok +COPY data FROM 'data/csv/null/multiple_nulls.csv' (nullstr ['','none','null'], HEADER 1); + +statement error +COPY data FROM 'data/csv/null/multiple_nulls.csv' (nullstr NULL, HEADER 1); +---- +syntax error at or near "NULL" + +statement error +COPY data FROM 'data/csv/null/multiple_nulls.csv' (nullstr [NULL], HEADER 1); +---- +Binder Error: CSV Reader function option nullstr requires a non-empty list of possible null strings (varchar) as input + +statement error +COPY data FROM 'data/csv/null/multiple_nulls.csv' (nullstr [42], HEADER 1); +---- +Binder Error: CSV Reader function option nullstr requires a non-empty list of possible null strings (varchar) as input + +query III +FROM data +---- +Pedro 31 1.73 +Mark NULL NULL +Thijs 26 NULL + +statement error +COPY data TO '__TEST_DIR__/multiple_nulls.csv' (nullstr ['a', 'b']); +---- +CSV Writer function option nullstr only accepts one nullstr value. \ No newline at end of file diff --git a/third_party/libpg_query/grammar/statements/copy.y b/third_party/libpg_query/grammar/statements/copy.y index c418f9f45a8d..ad2bcdd049a3 100644 --- a/third_party/libpg_query/grammar/statements/copy.y +++ b/third_party/libpg_query/grammar/statements/copy.y @@ -115,6 +115,7 @@ copy_options: copy_opt_list { $$ = $1; } copy_generic_opt_arg: opt_boolean_or_string { $$ = (PGNode *) makeString($1); } | NumericOnly { $$ = (PGNode *) $1; } + | list_expr { $$ = (PGNode *) $1; } | '*' { $$ = (PGNode *) makeNode(PGAStar); } | '(' copy_generic_opt_arg_list ')' { $$ = (PGNode *) $2; } | struct_expr { $$ = (PGNode *) $1; } diff --git a/third_party/libpg_query/grammar/statements/select.y b/third_party/libpg_query/grammar/statements/select.y index b549d3129e6d..2d0d0f82868b 100644 --- a/third_party/libpg_query/grammar/statements/select.y +++ b/third_party/libpg_query/grammar/statements/select.y @@ -2784,9 +2784,8 @@ indirection_expr: } | case_expr { $$ = $1; } - | '[' opt_expr_list_opt_comma ']' { - PGFuncCall *n = makeFuncCall(SystemFuncName("list_value"), $2, @2); - $$ = (PGNode *) n; + | list_expr { + $$ = $1; } | list_comprehension { $$ = $1; @@ -2820,7 +2819,11 @@ indirection_expr: } ; - +list_expr: '[' opt_expr_list_opt_comma ']' { + PGFuncCall *n = makeFuncCall(SystemFuncName("list_value"), $2, @2); + $$ = (PGNode *) n; + } + ; struct_expr: '{' dict_arguments_opt_comma '}' { diff --git a/third_party/libpg_query/grammar/types/select.yh b/third_party/libpg_query/grammar/types/select.yh index 937b93022adc..2e9b19ca14a8 100644 --- a/third_party/libpg_query/grammar/types/select.yh +++ b/third_party/libpg_query/grammar/types/select.yh @@ -9,7 +9,7 @@ %type opt_collate_clause %type indirection_expr indirection_expr_or_a_expr -%type struct_expr +%type struct_expr list_expr %type opt_nowait_or_skip diff --git a/third_party/libpg_query/src_backend_parser_gram.cpp b/third_party/libpg_query/src_backend_parser_gram.cpp index 871d973963fd..f8b6577fd638 100644 --- a/third_party/libpg_query/src_backend_parser_gram.cpp +++ b/third_party/libpg_query/src_backend_parser_gram.cpp @@ -1560,16 +1560,16 @@ union yyalloc /* YYFINAL -- State number of the termination state. */ #define YYFINAL 867 /* YYLAST -- Last index in YYTABLE. */ -#define YYLAST 72581 +#define YYLAST 72739 /* YYNTOKENS -- Number of terminals. */ #define YYNTOKENS 525 /* YYNNTS -- Number of nonterminals. */ -#define YYNNTS 459 +#define YYNNTS 460 /* YYNRULES -- Number of rules. */ -#define YYNRULES 2118 +#define YYNRULES 2120 /* YYNRULES -- Number of states. */ -#define YYNSTATES 3526 +#define YYNSTATES 3529 /* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX. */ #define YYUNDEFTOK 2 @@ -1711,110 +1711,110 @@ static const yytype_uint16 yyprhs[] = 1539, 1541, 1543, 1552, 1564, 1575, 1583, 1594, 1604, 1606, 1608, 1611, 1615, 1624, 1636, 1646, 1654, 1655, 1659, 1663, 1665, 1667, 1671, 1672, 1674, 1678, 1680, 1681, 1683, 1684, - 1686, 1687, 1689, 1693, 1695, 1697, 1699, 1703, 1705, 1706, - 1709, 1712, 1713, 1716, 1717, 1719, 1720, 1722, 1724, 1726, - 1730, 1734, 1736, 1738, 1742, 1746, 1750, 1754, 1758, 1762, - 1767, 1771, 1774, 1776, 1778, 1780, 1782, 1786, 1788, 1790, - 1794, 1796, 1798, 1802, 1806, 1810, 1812, 1815, 1820, 1825, - 1828, 1832, 1838, 1844, 1846, 1848, 1852, 1853, 1865, 1877, - 1888, 1901, 1903, 1906, 1912, 1917, 1922, 1927, 1932, 1940, - 1946, 1951, 1959, 1966, 1976, 1986, 1991, 1993, 1995, 1997, - 1999, 2001, 2003, 2005, 2011, 2013, 2015, 2019, 2021, 2024, - 2027, 2030, 2034, 2036, 2040, 2048, 2050, 2053, 2054, 2057, - 2058, 2062, 2066, 2071, 2076, 2081, 2086, 2090, 2093, 2095, - 2097, 2098, 2100, 2102, 2103, 2106, 2108, 2114, 2116, 2117, - 2120, 2123, 2124, 2126, 2127, 2131, 2137, 2139, 2143, 2148, - 2152, 2154, 2156, 2157, 2160, 2163, 2164, 2167, 2170, 2172, - 2174, 2176, 2177, 2180, 2185, 2191, 2196, 2199, 2203, 2206, - 2209, 2212, 2215, 2217, 2220, 2224, 2225, 2227, 2228, 2234, - 2236, 2241, 2248, 2251, 2253, 2254, 2259, 2260, 2262, 2264, - 2267, 2270, 2273, 2275, 2277, 2280, 2283, 2285, 2287, 2289, - 2291, 2293, 2295, 2299, 2303, 2304, 2306, 2310, 2312, 2315, - 2317, 2319, 2321, 2323, 2325, 2328, 2333, 2338, 2344, 2346, - 2348, 2351, 2352, 2355, 2356, 2358, 2362, 2364, 2365, 2367, - 2370, 2374, 2377, 2382, 2385, 2389, 2392, 2393, 2395, 2398, - 2399, 2404, 2410, 2412, 2415, 2418, 2419, 2421, 2425, 2427, - 2430, 2434, 2438, 2442, 2446, 2450, 2454, 2456, 2461, 2471, - 2481, 2485, 2486, 2489, 2492, 2493, 2499, 2503, 2504, 2506, - 2508, 2512, 2518, 2522, 2524, 2527, 2529, 2533, 2539, 2541, - 2544, 2548, 2553, 2559, 2564, 2570, 2575, 2582, 2588, 2593, - 2599, 2605, 2611, 2614, 2619, 2621, 2623, 2624, 2626, 2631, - 2637, 2642, 2643, 2646, 2649, 2652, 2654, 2656, 2658, 2660, - 2661, 2666, 2669, 2671, 2674, 2677, 2682, 2685, 2692, 2695, - 2697, 2701, 2706, 2707, 2710, 2711, 2714, 2715, 2717, 2721, - 2725, 2728, 2729, 2732, 2737, 2739, 2741, 2743, 2744, 2747, - 2751, 2757, 2764, 2767, 2771, 2773, 2779, 2785, 2791, 2795, - 2799, 2803, 2808, 2809, 2811, 2813, 2815, 2817, 2819, 2822, - 2827, 2829, 2831, 2833, 2835, 2838, 2842, 2843, 2845, 2847, - 2849, 2851, 2853, 2856, 2859, 2862, 2865, 2868, 2870, 2874, - 2875, 2877, 2879, 2881, 2883, 2889, 2892, 2894, 2896, 2898, - 2900, 2905, 2907, 2910, 2913, 2915, 2919, 2923, 2926, 2928, - 2929, 2935, 2938, 2944, 2947, 2949, 2953, 2957, 2958, 2960, + 1686, 1687, 1689, 1693, 1695, 1697, 1699, 1701, 1705, 1707, + 1708, 1711, 1714, 1715, 1718, 1719, 1721, 1722, 1724, 1726, + 1728, 1732, 1736, 1738, 1740, 1744, 1748, 1752, 1756, 1760, + 1764, 1769, 1773, 1776, 1778, 1780, 1782, 1784, 1788, 1790, + 1792, 1796, 1798, 1800, 1804, 1808, 1812, 1814, 1817, 1822, + 1827, 1830, 1834, 1840, 1846, 1848, 1850, 1854, 1855, 1867, + 1879, 1890, 1903, 1905, 1908, 1914, 1919, 1924, 1929, 1934, + 1942, 1948, 1953, 1961, 1968, 1978, 1988, 1993, 1995, 1997, + 1999, 2001, 2003, 2005, 2007, 2013, 2015, 2017, 2021, 2023, + 2026, 2029, 2032, 2036, 2038, 2042, 2050, 2052, 2055, 2056, + 2059, 2060, 2064, 2068, 2073, 2078, 2083, 2088, 2092, 2095, + 2097, 2099, 2100, 2102, 2104, 2105, 2108, 2110, 2116, 2118, + 2119, 2122, 2125, 2126, 2128, 2129, 2133, 2139, 2141, 2145, + 2150, 2154, 2156, 2158, 2159, 2162, 2165, 2166, 2169, 2172, + 2174, 2176, 2178, 2179, 2182, 2187, 2193, 2198, 2201, 2205, + 2208, 2211, 2214, 2217, 2219, 2222, 2226, 2227, 2229, 2230, + 2236, 2238, 2243, 2250, 2253, 2255, 2256, 2261, 2262, 2264, + 2266, 2269, 2272, 2275, 2277, 2279, 2282, 2285, 2287, 2289, + 2291, 2293, 2295, 2297, 2301, 2305, 2306, 2308, 2312, 2314, + 2317, 2319, 2321, 2323, 2325, 2327, 2330, 2335, 2340, 2346, + 2348, 2350, 2353, 2354, 2357, 2358, 2360, 2364, 2366, 2367, + 2369, 2372, 2376, 2379, 2384, 2387, 2391, 2394, 2395, 2397, + 2400, 2401, 2406, 2412, 2414, 2417, 2420, 2421, 2423, 2427, + 2429, 2432, 2436, 2440, 2444, 2448, 2452, 2456, 2458, 2463, + 2473, 2483, 2487, 2488, 2491, 2494, 2495, 2501, 2505, 2506, + 2508, 2510, 2514, 2520, 2524, 2526, 2529, 2531, 2535, 2541, + 2543, 2546, 2550, 2555, 2561, 2566, 2572, 2577, 2584, 2590, + 2595, 2601, 2607, 2613, 2616, 2621, 2623, 2625, 2626, 2628, + 2633, 2639, 2644, 2645, 2648, 2651, 2654, 2656, 2658, 2660, + 2662, 2663, 2668, 2671, 2673, 2676, 2679, 2684, 2687, 2694, + 2697, 2699, 2703, 2708, 2709, 2712, 2713, 2716, 2717, 2719, + 2723, 2727, 2730, 2731, 2734, 2739, 2741, 2743, 2745, 2746, + 2749, 2753, 2759, 2766, 2769, 2773, 2775, 2781, 2787, 2793, + 2797, 2801, 2805, 2810, 2811, 2813, 2815, 2817, 2819, 2821, + 2824, 2829, 2831, 2833, 2835, 2837, 2840, 2844, 2845, 2847, + 2849, 2851, 2853, 2855, 2858, 2861, 2864, 2867, 2870, 2872, + 2876, 2877, 2879, 2881, 2883, 2885, 2891, 2894, 2896, 2898, + 2900, 2902, 2907, 2909, 2912, 2915, 2917, 2921, 2925, 2928, + 2930, 2931, 2937, 2940, 2946, 2949, 2951, 2955, 2959, 2960, 2962, 2964, 2966, 2968, 2970, 2972, 2974, 2976, 2978, 2980, 2982, 2984, 2986, 2988, 2990, 2992, 2994, 2996, 2998, 3000, 3002, 3004, 3006, 3008, 3010, 3012, 3014, 3016, 3018, 3020, - 3022, 3024, 3026, 3028, 3030, 3034, 3038, 3042, 3046, 3050, - 3054, 3058, 3059, 3061, 3065, 3069, 3075, 3078, 3081, 3085, - 3089, 3093, 3097, 3101, 3105, 3109, 3113, 3117, 3121, 3125, - 3129, 3133, 3137, 3141, 3144, 3147, 3151, 3155, 3158, 3161, - 3165, 3169, 3175, 3180, 3187, 3191, 3197, 3202, 3209, 3214, - 3221, 3227, 3235, 3239, 3242, 3247, 3251, 3254, 3258, 3262, - 3266, 3270, 3275, 3279, 3284, 3288, 3293, 3299, 3306, 3313, - 3321, 3328, 3336, 3343, 3351, 3355, 3360, 3365, 3372, 3374, - 3379, 3383, 3389, 3391, 3395, 3398, 3401, 3405, 3409, 3413, - 3417, 3421, 3425, 3429, 3433, 3437, 3441, 3445, 3449, 3453, - 3457, 3461, 3464, 3467, 3473, 3480, 3487, 3495, 3497, 3500, - 3502, 3504, 3506, 3509, 3512, 3517, 3521, 3523, 3525, 3527, - 3529, 3531, 3536, 3538, 3540, 3544, 3546, 3549, 3554, 3557, - 3560, 3564, 3568, 3575, 3583, 3593, 3601, 3609, 3615, 3617, - 3619, 3621, 3627, 3634, 3641, 3646, 3651, 3656, 3661, 3668, - 3674, 3680, 3686, 3691, 3698, 3703, 3711, 3721, 3727, 3728, - 3734, 3739, 3740, 3742, 3743, 3746, 3747, 3749, 3753, 3757, - 3760, 3763, 3764, 3771, 3773, 3774, 3778, 3779, 3783, 3787, - 3791, 3792, 3794, 3799, 3802, 3805, 3808, 3811, 3814, 3818, - 3821, 3824, 3828, 3829, 3834, 3838, 3840, 3846, 3850, 3852, - 3856, 3858, 3861, 3865, 3867, 3871, 3873, 3876, 3878, 3879, - 3881, 3883, 3885, 3887, 3889, 3891, 3893, 3895, 3897, 3899, + 3022, 3024, 3026, 3028, 3030, 3032, 3036, 3040, 3044, 3048, + 3052, 3056, 3060, 3061, 3063, 3067, 3071, 3077, 3080, 3083, + 3087, 3091, 3095, 3099, 3103, 3107, 3111, 3115, 3119, 3123, + 3127, 3131, 3135, 3139, 3143, 3146, 3149, 3153, 3157, 3160, + 3163, 3167, 3171, 3177, 3182, 3189, 3193, 3199, 3204, 3211, + 3216, 3223, 3229, 3237, 3241, 3244, 3249, 3253, 3256, 3260, + 3264, 3268, 3272, 3277, 3281, 3286, 3290, 3295, 3301, 3308, + 3315, 3323, 3330, 3338, 3345, 3353, 3357, 3362, 3367, 3374, + 3376, 3381, 3385, 3391, 3393, 3397, 3400, 3403, 3407, 3411, + 3415, 3419, 3423, 3427, 3431, 3435, 3439, 3443, 3447, 3451, + 3455, 3459, 3463, 3466, 3469, 3475, 3482, 3489, 3497, 3499, + 3502, 3504, 3506, 3508, 3511, 3514, 3519, 3523, 3525, 3527, + 3529, 3531, 3533, 3538, 3540, 3542, 3544, 3546, 3549, 3554, + 3557, 3560, 3564, 3568, 3572, 3579, 3587, 3597, 3605, 3613, + 3619, 3621, 3623, 3625, 3631, 3638, 3645, 3650, 3655, 3660, + 3665, 3672, 3678, 3684, 3690, 3695, 3702, 3707, 3715, 3725, + 3731, 3732, 3738, 3743, 3744, 3746, 3747, 3750, 3751, 3753, + 3757, 3761, 3764, 3767, 3768, 3775, 3777, 3778, 3782, 3783, + 3787, 3791, 3795, 3796, 3798, 3803, 3806, 3809, 3812, 3815, + 3818, 3822, 3825, 3828, 3832, 3833, 3838, 3842, 3844, 3850, + 3854, 3856, 3860, 3862, 3865, 3869, 3871, 3875, 3877, 3880, + 3882, 3883, 3885, 3887, 3889, 3891, 3893, 3895, 3897, 3899, 3901, 3903, 3905, 3907, 3909, 3911, 3913, 3915, 3917, 3919, - 3924, 3926, 3931, 3933, 3938, 3940, 3943, 3945, 3948, 3950, - 3953, 3955, 3959, 3961, 3965, 3967, 3970, 3972, 3976, 3978, - 3981, 3983, 3984, 3986, 3990, 3992, 3996, 4000, 4002, 4006, - 4010, 4011, 4013, 4015, 4017, 4019, 4021, 4023, 4025, 4027, - 4029, 4031, 4033, 4035, 4037, 4039, 4044, 4048, 4051, 4055, - 4056, 4060, 4064, 4067, 4070, 4072, 4073, 4076, 4079, 4083, - 4086, 4088, 4090, 4094, 4100, 4102, 4105, 4110, 4113, 4114, - 4116, 4117, 4119, 4122, 4126, 4132, 4140, 4148, 4150, 4151, - 4152, 4155, 4156, 4159, 4163, 4167, 4171, 4177, 4185, 4193, - 4194, 4197, 4199, 4200, 4202, 4203, 4205, 4209, 4211, 4214, - 4218, 4221, 4223, 4228, 4231, 4233, 4234, 4238, 4240, 4244, - 4246, 4249, 4254, 4257, 4258, 4260, 4264, 4266, 4270, 4272, - 4275, 4277, 4281, 4283, 4285, 4288, 4290, 4292, 4295, 4297, - 4299, 4302, 4310, 4313, 4319, 4323, 4327, 4329, 4331, 4333, + 3921, 3923, 3928, 3930, 3935, 3937, 3942, 3944, 3947, 3949, + 3952, 3954, 3957, 3959, 3963, 3965, 3969, 3971, 3974, 3976, + 3980, 3982, 3985, 3987, 3988, 3990, 3994, 3996, 4000, 4004, + 4006, 4010, 4014, 4015, 4017, 4019, 4021, 4023, 4025, 4027, + 4029, 4031, 4033, 4035, 4037, 4039, 4041, 4043, 4048, 4052, + 4055, 4059, 4060, 4064, 4068, 4071, 4074, 4076, 4077, 4080, + 4083, 4087, 4090, 4092, 4094, 4098, 4104, 4106, 4109, 4114, + 4117, 4118, 4120, 4121, 4123, 4126, 4130, 4136, 4144, 4152, + 4154, 4155, 4156, 4159, 4160, 4163, 4167, 4171, 4175, 4181, + 4189, 4197, 4198, 4201, 4203, 4204, 4206, 4207, 4209, 4213, + 4215, 4218, 4222, 4225, 4227, 4232, 4235, 4237, 4238, 4242, + 4244, 4248, 4250, 4253, 4258, 4261, 4262, 4264, 4268, 4270, + 4274, 4276, 4279, 4281, 4285, 4287, 4289, 4292, 4294, 4296, + 4299, 4301, 4303, 4306, 4314, 4317, 4323, 4327, 4331, 4333, 4335, 4337, 4339, 4341, 4343, 4345, 4347, 4349, 4351, 4353, - 4355, 4358, 4361, 4365, 4369, 4370, 4372, 4374, 4376, 4382, - 4386, 4387, 4389, 4391, 4393, 4395, 4397, 4399, 4404, 4412, - 4419, 4422, 4423, 4425, 4427, 4429, 4431, 4445, 4462, 4464, - 4467, 4468, 4470, 4471, 4473, 4474, 4477, 4478, 4480, 4481, - 4488, 4497, 4504, 4513, 4520, 4529, 4533, 4536, 4538, 4539, - 4546, 4553, 4555, 4557, 4559, 4561, 4563, 4565, 4568, 4570, - 4572, 4574, 4576, 4578, 4583, 4590, 4594, 4597, 4602, 4606, - 4612, 4614, 4615, 4617, 4619, 4620, 4622, 4624, 4626, 4628, + 4355, 4357, 4359, 4362, 4365, 4369, 4373, 4374, 4376, 4378, + 4380, 4386, 4390, 4391, 4393, 4395, 4397, 4399, 4401, 4403, + 4408, 4416, 4423, 4426, 4427, 4429, 4431, 4433, 4435, 4449, + 4466, 4468, 4471, 4472, 4474, 4475, 4477, 4478, 4481, 4482, + 4484, 4485, 4492, 4501, 4508, 4517, 4524, 4533, 4537, 4540, + 4542, 4543, 4550, 4557, 4559, 4561, 4563, 4565, 4567, 4569, + 4572, 4574, 4576, 4578, 4580, 4582, 4587, 4594, 4598, 4601, + 4606, 4610, 4616, 4618, 4619, 4621, 4623, 4624, 4626, 4628, 4630, 4632, 4634, 4636, 4638, 4640, 4642, 4644, 4646, 4648, 4650, 4652, 4654, 4656, 4658, 4660, 4662, 4664, 4666, 4668, 4670, 4672, 4674, 4676, 4678, 4680, 4682, 4684, 4686, 4688, - 4690, 4692, 4696, 4698, 4700, 4702, 4704, 4706, 4708, 4711, - 4713, 4715, 4718, 4722, 4726, 4730, 4732, 4736, 4740, 4743, - 4747, 4751, 4753, 4755, 4757, 4761, 4767, 4769, 4771, 4773, - 4775, 4779, 4782, 4785, 4789, 4794, 4800, 4802, 4804, 4806, - 4808, 4813, 4820, 4826, 4831, 4838, 4840, 4842, 4844, 4846, - 4848, 4850, 4851, 4853, 4857, 4859, 4860, 4868, 4872, 4874, - 4877, 4881, 4884, 4885, 4888, 4889, 4892, 4897, 4903, 4912, - 4915, 4919, 4925, 4927, 4928, 4931, 4932, 4935, 4939, 4943, - 4947, 4949, 4951, 4953, 4956, 4960, 4963, 4966, 4969, 4972, - 4976, 4981, 4985, 4987, 4989, 4991, 4993, 4995, 4997, 4998, - 5000, 5004, 5006, 5010, 5013, 5023, 5036, 5048, 5061, 5076, - 5080, 5085, 5090, 5091, 5099, 5110, 5120, 5123, 5127, 5128, - 5133, 5135, 5137, 5139, 5141, 5143, 5145, 5147, 5149, 5151, + 4690, 4692, 4694, 4696, 4700, 4702, 4704, 4706, 4708, 4710, + 4712, 4715, 4717, 4719, 4722, 4726, 4730, 4734, 4736, 4740, + 4744, 4747, 4751, 4755, 4757, 4759, 4761, 4765, 4771, 4773, + 4775, 4777, 4779, 4783, 4786, 4789, 4793, 4798, 4804, 4806, + 4808, 4810, 4812, 4817, 4824, 4830, 4835, 4842, 4844, 4846, + 4848, 4850, 4852, 4854, 4855, 4857, 4861, 4863, 4864, 4872, + 4876, 4878, 4881, 4885, 4888, 4889, 4892, 4893, 4896, 4901, + 4907, 4916, 4919, 4923, 4929, 4931, 4932, 4935, 4936, 4939, + 4943, 4947, 4951, 4953, 4955, 4957, 4960, 4964, 4967, 4970, + 4973, 4976, 4980, 4985, 4989, 4991, 4993, 4995, 4997, 4999, + 5001, 5002, 5004, 5008, 5010, 5014, 5017, 5027, 5040, 5052, + 5065, 5080, 5084, 5089, 5094, 5095, 5103, 5114, 5124, 5127, + 5131, 5132, 5137, 5139, 5141, 5143, 5145, 5147, 5149, 5151, 5153, 5155, 5157, 5159, 5161, 5163, 5165, 5167, 5169, 5171, 5173, 5175, 5177, 5179, 5181, 5183, 5185, 5187, 5189, 5191, 5193, 5195, 5197, 5199, 5201, 5203, 5205, 5207, 5209, 5211, @@ -1875,21 +1875,22 @@ static const yytype_uint16 yyprhs[] = 6293, 6295, 6297, 6299, 6301, 6303, 6305, 6307, 6309, 6311, 6313, 6315, 6317, 6319, 6321, 6323, 6325, 6327, 6329, 6331, 6333, 6335, 6337, 6339, 6341, 6343, 6345, 6347, 6349, 6351, - 6353, 6355, 6357, 6359, 6361, 6363, 6365, 6367, 6369 + 6353, 6355, 6357, 6359, 6361, 6363, 6365, 6367, 6369, 6371, + 6373 }; /* YYRHS -- A `-1'-separated list of the rules' RHS. */ static const yytype_int16 yyrhs[] = { 526, 0, -1, 527, -1, 527, 517, 528, -1, 528, - -1, 920, -1, 586, -1, 529, -1, 957, -1, 958, - -1, 971, -1, 921, -1, 923, -1, 656, -1, 974, - -1, 652, -1, 910, -1, 578, -1, 576, -1, 598, - -1, 572, -1, 540, -1, 953, -1, 959, -1, 593, - -1, 646, -1, 582, -1, 928, -1, 926, -1, 927, - -1, 913, -1, 551, -1, 945, -1, 575, -1, 907, + -1, 921, -1, 586, -1, 529, -1, 958, -1, 959, + -1, 972, -1, 922, -1, 924, -1, 656, -1, 975, + -1, 652, -1, 911, -1, 578, -1, 576, -1, 598, + -1, 572, -1, 540, -1, 954, -1, 960, -1, 593, + -1, 646, -1, 582, -1, 929, -1, 927, -1, 928, + -1, 914, -1, 551, -1, 946, -1, 575, -1, 908, -1, 549, -1, 674, -1, 595, -1, 655, -1, 597, - -1, 948, -1, 962, -1, 939, -1, 965, -1, 972, + -1, 949, -1, 963, -1, 940, -1, 966, -1, 973, -1, -1, 32, 416, 761, 537, -1, 32, 416, 191, 152, 761, 537, -1, 32, 202, 541, 537, -1, 32, 202, 191, 152, 541, 537, -1, 32, 381, 541, 537, @@ -1910,50 +1911,50 @@ static const yytype_int16 yyrhs[] = 137, 190, 191, 152, -1, 137, 550, 191, 152, 542, 650, -1, 137, 550, 542, 650, -1, 32, 550, 542, 539, 439, 774, 770, 535, -1, 32, 550, 542, 538, - -1, 26, 615, -1, 32, 93, 895, 599, -1, 457, - 93, 895, -1, 137, 93, 191, 152, 895, 650, -1, - 137, 93, 895, 650, -1, 386, 244, -1, 386, 448, + -1, 26, 615, -1, 32, 93, 896, 599, -1, 457, + 93, 896, -1, 137, 93, 191, 152, 896, 650, -1, + 137, 93, 896, 650, -1, 386, 244, -1, 386, 448, -1, 386, 613, -1, 354, 613, -1, 538, -1, 454, 809, -1, -1, 609, -1, 386, 609, -1, 26, 609, -1, 137, 623, -1, 534, -1, 537, 518, 534, -1, 293, 514, 533, 515, -1, 386, 107, -1, 386, -1, - -1, 111, 895, -1, 111, 325, 895, -1, 111, 30, + -1, 111, 896, -1, 111, 325, 896, -1, 111, 30, -1, 111, 325, 30, -1, 543, -1, 542, 545, -1, - 3, -1, 977, -1, 978, -1, 542, -1, 5, -1, + 3, -1, 978, -1, 979, -1, 542, -1, 5, -1, 5, -1, 546, -1, 545, 546, -1, 516, 547, -1, - 548, -1, 3, -1, 981, -1, 977, -1, 983, -1, - 32, 370, 895, 350, 428, 895, -1, 32, 416, 761, - 350, 428, 895, -1, 32, 416, 191, 152, 761, 350, - 428, 895, -1, 32, 381, 541, 350, 428, 895, -1, - 32, 381, 191, 152, 541, 350, 428, 895, -1, 32, - 466, 541, 350, 428, 895, -1, 32, 466, 191, 152, - 541, 350, 428, 895, -1, 32, 202, 541, 350, 428, - 895, -1, 32, 202, 191, 152, 541, 350, 428, 895, - -1, 32, 416, 761, 350, 550, 895, 428, 895, -1, - 32, 416, 191, 152, 761, 350, 550, 895, 428, 895, - -1, 32, 416, 761, 350, 93, 895, 428, 895, -1, - 32, 416, 191, 152, 761, 350, 93, 895, 428, 895, + 548, -1, 3, -1, 982, -1, 978, -1, 984, -1, + 32, 370, 896, 350, 428, 896, -1, 32, 416, 761, + 350, 428, 896, -1, 32, 416, 191, 152, 761, 350, + 428, 896, -1, 32, 381, 541, 350, 428, 896, -1, + 32, 381, 191, 152, 541, 350, 428, 896, -1, 32, + 466, 541, 350, 428, 896, -1, 32, 466, 191, 152, + 541, 350, 428, 896, -1, 32, 202, 541, 350, 428, + 896, -1, 32, 202, 191, 152, 541, 350, 428, 896, + -1, 32, 416, 761, 350, 550, 896, 428, 896, -1, + 32, 416, 191, 152, 761, 350, 550, 896, 428, 896, + -1, 32, 416, 761, 350, 93, 896, 428, 896, -1, + 32, 416, 191, 152, 761, 350, 93, 896, 428, 896, -1, 82, -1, -1, 556, 212, 559, 219, 553, 554, 552, 560, 562, -1, 674, -1, 303, 563, 459, 674, -1, 514, 567, 515, 674, -1, 514, 567, 515, 303, 563, 459, 674, -1, 117, 460, -1, 541, -1, 541, 40, 542, -1, 59, 265, -1, 59, 320, -1, -1, - 514, 570, 515, 767, -1, 289, 93, 895, -1, -1, - 686, -1, -1, 542, 876, -1, 571, 503, 809, -1, + 514, 570, 515, 767, -1, 289, 93, 896, -1, -1, + 686, -1, -1, 542, 877, -1, 571, 503, 809, -1, 514, 564, 515, 503, 809, -1, 294, 352, -1, 294, 192, -1, -1, 289, 91, 555, 133, 451, 386, 569, 767, -1, 289, 91, 555, 133, 275, -1, -1, 542, - 565, 566, 702, 703, -1, 818, 565, 566, 702, 703, + 565, 566, 702, 703, -1, 819, 565, 566, 702, 703, -1, 514, 809, 515, 565, 566, 702, 703, -1, 358, - 882, -1, -1, 453, -1, 415, -1, 571, -1, 564, - 518, 571, -1, 80, 902, -1, -1, 902, -1, -1, + 883, -1, -1, 453, -1, 415, -1, 571, -1, 564, + 518, 571, -1, 80, 903, -1, -1, 903, -1, -1, 557, -1, 567, 518, 557, -1, 558, -1, 568, 518, 558, -1, 568, -1, 568, 518, -1, 561, -1, 570, - 518, 561, -1, 542, 876, -1, 100, 439, 541, 40, + 518, 561, -1, 542, 877, -1, 100, 439, 541, 40, 144, 675, -1, 100, 439, 541, 40, 144, 514, 573, 515, -1, 100, 439, 541, 40, 774, -1, 574, -1, -1, 544, -1, 574, 518, 544, -1, 322, 542, -1, - 322, 542, 503, 944, -1, 322, 542, 514, 856, 515, + 322, 542, 503, 945, -1, 322, 542, 514, 857, 515, -1, 100, 644, 381, 541, 577, -1, 100, 644, 381, 191, 274, 152, 541, 577, -1, 100, 294, 352, 644, 381, 541, 577, -1, 587, -1, -1, 100, 580, 377, @@ -1961,20 +1962,20 @@ static const yytype_int16 yyrhs[] = 274, 152, 579, 581, 514, 673, 515, -1, 100, 294, 352, 580, 377, 579, 581, 514, 673, 515, -1, -1, 542, -1, -1, 422, -1, 313, -1, -1, 198, 3, - -1, 151, 895, 585, -1, 100, 644, 416, 976, 40, - 151, 895, 585, 975, -1, 100, 644, 416, 191, 274, - 152, 976, 40, 151, 895, 585, 975, -1, 809, -1, - 905, 13, 809, -1, 583, -1, 584, 518, 583, -1, + -1, 151, 896, 585, -1, 100, 644, 416, 977, 40, + 151, 896, 585, 976, -1, 100, 644, 416, 191, 274, + 152, 977, 40, 151, 896, 585, 976, -1, 809, -1, + 906, 13, 809, -1, 583, -1, 584, 518, 583, -1, 514, 584, 515, -1, -1, 32, 381, 541, 587, -1, 32, 381, 191, 152, 541, 587, -1, 590, -1, 587, 590, -1, 476, -1, 500, -1, -1, 4, -1, 505, 4, -1, 506, 4, -1, 592, -1, 40, 777, -1, 60, 589, -1, 106, -1, 272, 106, -1, 201, 591, 589, -1, 250, 589, -1, 260, 589, -1, 272, 250, - -1, 272, 260, -1, 304, 59, 902, -1, 381, 265, - 902, -1, 400, 588, 589, -1, 356, -1, 356, 588, - 589, -1, 59, -1, -1, 898, -1, 505, 898, -1, - 506, 898, -1, 137, 580, 377, 542, 594, -1, 137, + -1, 272, 260, -1, 304, 59, 903, -1, 381, 265, + 903, -1, 400, 588, 589, -1, 356, -1, 356, 588, + 589, -1, 59, -1, -1, 899, -1, 505, 899, -1, + 506, 899, -1, 137, 580, 377, 542, 594, -1, 137, 580, 377, 191, 152, 542, 594, -1, -1, 171, 3, -1, 22, 596, -1, 52, 596, -1, 400, 596, -1, 86, 596, -1, 143, 596, -1, 363, 596, -1, 479, @@ -1982,13 +1983,13 @@ static const yytype_int16 yyrhs[] = 541, 514, 631, 515, 620, 612, -1, 100, 644, 416, 191, 274, 152, 541, 514, 631, 515, 620, 612, -1, 100, 294, 352, 644, 416, 541, 514, 631, 515, 620, - 612, -1, -1, 599, 624, -1, 639, -1, 983, -1, - 848, -1, 589, -1, 544, -1, 273, -1, 514, 587, + 612, -1, -1, 599, 624, -1, 639, -1, 984, -1, + 849, -1, 589, -1, 544, -1, 273, -1, 514, 587, 515, -1, -1, 544, -1, 272, 25, -1, 357, -1, - 63, -1, 386, 279, -1, 386, 117, -1, 93, 895, - 605, -1, 605, -1, 619, -1, 80, 902, -1, 274, + 63, -1, 386, 279, -1, 386, 117, -1, 93, 896, + 605, -1, 605, -1, 619, -1, 80, 903, -1, 274, 279, -1, 279, -1, 445, 630, -1, 328, 226, 630, - -1, 74, 514, 809, 515, 614, -1, 454, 88, 895, + -1, 74, 514, 809, 515, 614, -1, 454, 88, 896, -1, 117, 810, -1, 344, 541, 633, 642, 611, -1, 468, -1, 406, -1, 606, -1, -1, 175, 645, 40, 190, 601, -1, 175, 645, 40, 514, 809, 515, 607, @@ -1996,7 +1997,7 @@ static const yytype_int16 yyrhs[] = 289, 451, 603, -1, 610, -1, 635, -1, 610, 635, -1, 635, 610, -1, -1, 289, 86, 137, -1, 289, 86, 122, 366, -1, 289, 86, 327, 366, -1, -1, - 514, 617, 515, -1, 272, 204, -1, -1, 93, 895, + 514, 617, 515, -1, 272, 204, -1, -1, 93, 896, 640, -1, 640, -1, 85, -1, 94, -1, 118, -1, 190, -1, 203, -1, 402, -1, 405, -1, 30, -1, 636, -1, 617, 518, 636, -1, 454, 202, 627, -1, @@ -2013,8 +2014,8 @@ static const yytype_int16 yyrhs[] = 604, -1, -1, 289, 122, 603, -1, 548, 503, 600, -1, 548, -1, 548, 516, 548, 503, 600, -1, 548, 516, 548, -1, 632, -1, 637, 518, 632, -1, 637, - -1, 637, 518, -1, 774, -1, 899, 903, 509, 439, - -1, 387, 899, 903, 509, 439, -1, 74, 514, 809, + -1, 637, 518, -1, 774, -1, 900, 904, 509, 439, + -1, 387, 900, 904, 509, 439, -1, 74, 514, 809, 515, 599, -1, 445, 514, 638, 515, 630, 599, -1, 445, 618, 599, -1, 328, 226, 514, 638, 515, 630, 599, -1, 328, 226, 618, 599, -1, 168, 226, 514, @@ -2024,17 +2025,17 @@ static const yytype_int16 yyrhs[] = -1, 420, -1, 240, 422, -1, 240, 420, -1, 177, 422, -1, 177, 420, -1, 448, -1, -1, 33, -1, 59, 117, -1, 137, 647, 191, 152, 649, 650, -1, - 137, 647, 649, 650, -1, 137, 648, 191, 152, 892, - 650, -1, 137, 648, 892, 650, -1, 137, 651, 895, - 289, 902, 650, -1, 137, 651, 191, 152, 895, 289, - 902, 650, -1, 416, -1, 381, -1, 173, -1, 245, + 137, 647, 649, 650, -1, 137, 648, 191, 152, 893, + 650, -1, 137, 648, 893, 650, -1, 137, 651, 896, + 289, 903, 650, -1, 137, 651, 191, 152, 896, 289, + 903, 650, -1, 416, -1, 381, -1, 173, -1, 245, -1, 245, 416, -1, 466, -1, 249, 466, -1, 202, -1, 168, 416, -1, 81, -1, 97, -1, 370, -1, 402, -1, 423, 374, 307, -1, 423, 374, 129, -1, 423, 374, 421, -1, 423, 374, 90, -1, 439, -1, 24, 251, -1, 146, 433, -1, 156, -1, 168, 107, - 480, -1, 334, -1, 384, -1, 902, -1, 649, 518, - 902, -1, 63, -1, 357, -1, -1, 319, -1, 367, + 480, -1, 334, -1, 384, -1, 903, -1, 649, 518, + 903, -1, 63, -1, 357, -1, -1, 319, -1, 367, -1, 433, -1, 100, 644, 653, 541, 654, 40, 416, 674, -1, 100, 644, 653, 191, 274, 152, 541, 654, 40, 416, 674, -1, 100, 294, 352, 644, 653, 541, @@ -2042,483 +2043,483 @@ static const yytype_int16 yyrhs[] = 40, 809, -1, 100, 644, 653, 191, 274, 152, 541, 654, 40, 809, -1, 100, 294, 352, 644, 653, 541, 654, 40, 809, -1, 173, -1, 245, -1, 514, 515, - -1, 514, 856, 515, -1, 556, 451, 954, 386, 569, - 741, 955, 562, -1, 98, 669, 541, 633, 667, 658, + -1, 514, 857, 515, -1, 556, 451, 955, 386, 569, + 741, 956, 562, -1, 98, 669, 541, 633, 667, 658, 663, 672, 659, 588, 664, -1, 98, 514, 674, 515, 428, 663, 672, 588, 664, -1, 98, 171, 108, 542, 428, 542, 657, -1, -1, 514, 370, 515, -1, 514, 107, 515, -1, 171, -1, 428, -1, 661, 124, 544, -1, -1, 671, -1, 660, 518, 671, -1, 454, -1, -1, 40, -1, -1, 333, -1, -1, 668, -1, 514, - 673, 515, -1, 936, -1, 589, -1, 507, -1, 514, - 660, 515, -1, 815, -1, -1, 548, 665, -1, 476, - 287, -1, -1, 668, 670, -1, -1, 55, -1, -1, - 55, -1, 287, -1, 170, -1, 123, 662, 544, -1, - 279, 662, 544, -1, 102, -1, 186, -1, 336, 662, - 544, -1, 145, 662, 544, -1, 167, 336, 637, -1, - 167, 336, 507, -1, 309, 59, 637, -1, 309, 59, - 507, -1, 167, 274, 279, 637, -1, 167, 279, 637, - -1, 141, 544, -1, 936, -1, 544, -1, 403, -1, - 404, -1, 3, 516, 542, -1, 3, -1, 666, -1, - 673, 518, 666, -1, 676, -1, 675, -1, 514, 676, - 515, -1, 514, 675, 515, -1, 514, 965, 515, -1, - 679, -1, 677, 699, -1, 677, 698, 732, 705, -1, - 677, 698, 704, 733, -1, 686, 677, -1, 686, 677, - 699, -1, 686, 677, 698, 732, 705, -1, 686, 677, - 698, 704, 733, -1, 679, -1, 675, -1, 379, 696, - 881, -1, -1, 379, 696, 881, 690, 741, 767, 721, - 730, 824, 731, 709, -1, 379, 695, 883, 690, 741, - 767, 721, 730, 824, 731, 709, -1, 171, 742, 678, - 690, 767, 721, 730, 824, 731, 709, -1, 171, 742, - 379, 695, 883, 690, 767, 721, 730, 824, 731, 709, - -1, 740, -1, 416, 761, -1, 677, 444, 693, 694, - 677, -1, 677, 444, 693, 677, -1, 677, 217, 693, - 677, -1, 677, 147, 693, 677, -1, 681, 744, 454, - 883, -1, 681, 744, 454, 883, 180, 59, 894, -1, - 681, 744, 180, 59, 894, -1, 681, 744, 289, 685, - -1, 681, 744, 289, 685, 180, 59, 894, -1, 681, - 744, 289, 685, 454, 883, -1, 681, 744, 289, 685, - 454, 883, 180, 59, 894, -1, 682, 744, 289, 883, - 219, 265, 895, 680, 894, -1, 682, 744, 289, 883, - -1, 459, -1, 460, -1, 314, -1, 316, -1, 449, - -1, 315, -1, 810, -1, 810, 198, 514, 676, 515, - -1, 747, -1, 683, -1, 684, 518, 683, -1, 684, - -1, 684, 518, -1, 476, 687, -1, 500, 687, -1, - 476, 342, 687, -1, 688, -1, 687, 518, 688, -1, - 895, 904, 40, 689, 514, 909, 515, -1, 249, -1, - 274, 249, -1, -1, 219, 691, -1, -1, 422, 692, - 541, -1, 420, 692, 541, -1, 240, 422, 692, 541, - -1, 240, 420, 692, 541, -1, 177, 422, 692, 541, - -1, 177, 420, 692, 541, -1, 448, 692, 541, -1, - 416, 541, -1, 541, -1, 416, -1, -1, 30, -1, - 132, -1, -1, 59, 265, -1, 132, -1, 132, 289, - 514, 854, 515, -1, 30, -1, -1, 192, 281, -1, - 355, 281, -1, -1, 699, -1, -1, 295, 59, 700, - -1, 295, 59, 30, 702, 703, -1, 701, -1, 700, - 518, 701, -1, 809, 454, 848, 703, -1, 809, 702, - 703, -1, 41, -1, 126, -1, -1, 499, 163, -1, - 499, 230, -1, -1, 706, 707, -1, 707, 706, -1, - 706, -1, 707, -1, 704, -1, -1, 237, 715, -1, - 237, 715, 518, 716, -1, 161, 720, 717, 719, 290, - -1, 161, 720, 719, 290, -1, 286, 716, -1, 286, - 717, 719, -1, 4, 509, -1, 9, 509, -1, 4, - 312, -1, 9, 312, -1, 9, -1, 9, 366, -1, - 454, 368, 711, -1, -1, 542, -1, -1, 710, 514, - 708, 515, 714, -1, 708, -1, 708, 514, 542, 515, - -1, 708, 514, 542, 518, 9, 515, -1, 418, 711, - -1, 712, -1, -1, 351, 514, 9, 515, -1, -1, - 809, -1, 30, -1, 809, 509, -1, 4, 312, -1, - 9, 312, -1, 809, -1, 811, -1, 505, 718, -1, - 506, 718, -1, 898, -1, 4, -1, 365, -1, 366, - -1, 163, -1, 271, -1, 180, 59, 723, -1, 180, - 59, 30, -1, -1, 724, -1, 722, 518, 724, -1, - 722, -1, 722, 518, -1, 809, -1, 725, -1, 727, - -1, 726, -1, 728, -1, 514, 515, -1, 364, 514, - 854, 515, -1, 103, 514, 854, 515, -1, 181, 388, - 514, 723, 515, -1, 181, -1, 182, -1, 185, 809, - -1, -1, 335, 809, -1, -1, 734, -1, 166, 338, - 290, -1, 732, -1, -1, 735, -1, 734, 735, -1, - 736, 737, 738, -1, 166, 451, -1, 166, 272, 226, - 451, -1, 166, 389, -1, 166, 226, 389, -1, 284, - 891, -1, -1, 278, -1, 393, 243, -1, -1, 460, - 514, 854, 515, -1, 739, 518, 514, 854, 515, -1, - 739, -1, 739, 518, -1, 171, 743, -1, -1, 744, - -1, 742, 518, 744, -1, 742, -1, 742, 518, -1, - 761, 756, 713, -1, 762, 757, 713, -1, 740, 755, - 713, -1, 231, 762, 757, -1, 675, 756, 713, -1, - 231, 675, 756, -1, 754, -1, 514, 754, 515, 755, - -1, 744, 314, 514, 883, 166, 750, 745, 515, 756, - -1, 744, 449, 746, 514, 751, 166, 753, 515, 756, - -1, 180, 59, 893, -1, -1, 199, 281, -1, 148, - 281, -1, -1, 810, 198, 514, 883, 515, -1, 810, - 198, 543, -1, -1, 812, -1, 814, -1, 514, 852, - 515, -1, 748, 198, 514, 883, 515, -1, 748, 198, - 543, -1, 749, -1, 750, 749, -1, 543, -1, 514, - 893, 515, -1, 751, 198, 514, 883, 515, -1, 752, - -1, 753, 752, -1, 514, 754, 515, -1, 744, 101, - 224, 744, -1, 744, 758, 224, 744, 760, -1, 744, - 224, 744, 760, -1, 744, 268, 758, 224, 744, -1, - 744, 268, 224, 744, -1, 744, 42, 758, 224, 744, - 760, -1, 744, 42, 224, 744, 760, -1, 744, 321, - 224, 744, -1, 744, 37, 224, 744, 760, -1, 744, - 380, 224, 744, 760, -1, 40, 543, 514, 893, 515, - -1, 40, 543, -1, 542, 514, 893, 515, -1, 542, - -1, 755, -1, -1, 755, -1, 40, 514, 768, 515, - -1, 40, 543, 514, 768, 515, -1, 542, 514, 768, - 515, -1, -1, 172, 759, -1, 234, 759, -1, 361, - 759, -1, 380, -1, 37, -1, 208, -1, 299, -1, - -1, 454, 514, 893, 515, -1, 289, 809, -1, 541, - -1, 541, 507, -1, 290, 541, -1, 290, 514, 541, - 515, -1, 818, 766, -1, 366, 171, 514, 764, 515, - 766, -1, 818, 765, -1, 763, -1, 764, 518, 763, - -1, 40, 514, 768, 515, -1, -1, 500, 296, -1, - -1, 473, 809, -1, -1, 769, -1, 768, 518, 769, - -1, 543, 774, 770, -1, 80, 902, -1, -1, 542, - 774, -1, 771, 518, 542, 774, -1, 365, -1, 409, - -1, 774, -1, -1, 777, 776, -1, 387, 777, 776, - -1, 777, 39, 512, 898, 513, -1, 387, 777, 39, - 512, 898, 513, -1, 777, 39, -1, 387, 777, 39, - -1, 775, -1, 772, 514, 771, 515, 776, -1, 246, - 514, 858, 515, 776, -1, 444, 514, 771, 515, 776, - -1, 3, 516, 3, -1, 775, 516, 3, -1, 776, - 512, 513, -1, 776, 512, 898, 513, -1, -1, 779, - -1, 781, -1, 783, -1, 787, -1, 793, -1, 794, - 808, -1, 794, 514, 898, 515, -1, 781, -1, 784, - -1, 788, -1, 793, -1, 901, 780, -1, 514, 855, - 515, -1, -1, 215, -1, 216, -1, 394, -1, 54, - -1, 339, -1, 164, 782, -1, 136, 324, -1, 115, - 780, -1, 112, 780, -1, 282, 780, -1, 57, -1, - 514, 898, 515, -1, -1, 785, -1, 786, -1, 785, - -1, 786, -1, 56, 792, 514, 854, 515, -1, 56, - 792, -1, 789, -1, 790, -1, 789, -1, 790, -1, - 791, 514, 898, 515, -1, 791, -1, 72, 792, -1, - 71, 792, -1, 461, -1, 267, 72, 792, -1, 267, - 71, 792, -1, 269, 792, -1, 463, -1, -1, 427, - 514, 898, 515, 795, -1, 427, 795, -1, 426, 514, - 898, 515, 795, -1, 426, 795, -1, 218, -1, 500, - 426, 497, -1, 478, 426, 497, -1, -1, 494, -1, - 495, -1, 262, -1, 263, -1, 109, -1, 110, -1, - 188, -1, 189, -1, 258, -1, 259, -1, 375, -1, - 376, -1, 256, -1, 257, -1, 252, -1, 253, -1, - 470, -1, 471, -1, 113, -1, 114, -1, 69, -1, - 68, -1, 255, -1, 254, -1, 796, -1, 797, -1, - 798, -1, 799, -1, 800, -1, 801, -1, 802, -1, - 803, -1, 804, -1, 805, -1, 806, -1, 807, -1, - 796, 428, 797, -1, 798, 428, 799, -1, 798, 428, - 800, -1, 798, 428, 801, -1, 799, 428, 800, -1, - 799, 428, 801, -1, 800, 428, 801, -1, -1, 811, - -1, 809, 11, 774, -1, 809, 80, 902, -1, 809, - 46, 426, 497, 809, -1, 505, 809, -1, 506, 809, - -1, 809, 505, 809, -1, 809, 506, 809, -1, 809, - 507, 809, -1, 809, 508, 809, -1, 809, 15, 809, - -1, 809, 509, 809, -1, 809, 510, 809, -1, 809, - 16, 809, -1, 809, 501, 809, -1, 809, 502, 809, - -1, 809, 503, 809, -1, 809, 19, 809, -1, 809, - 20, 809, -1, 809, 21, 809, -1, 809, 847, 809, - -1, 847, 809, -1, 809, 847, -1, 809, 36, 809, - -1, 809, 294, 809, -1, 274, 809, -1, 498, 809, - -1, 809, 176, 809, -1, 809, 236, 809, -1, 809, - 236, 809, 145, 809, -1, 809, 498, 236, 809, -1, - 809, 498, 236, 809, 145, 809, -1, 809, 193, 809, - -1, 809, 193, 809, 145, 809, -1, 809, 498, 193, - 809, -1, 809, 498, 193, 809, 145, 809, -1, 809, - 391, 428, 809, -1, 809, 391, 428, 809, 145, 809, - -1, 809, 498, 391, 428, 809, -1, 809, 498, 391, - 428, 809, 145, 809, -1, 809, 221, 279, -1, 809, - 222, -1, 809, 221, 274, 279, -1, 809, 274, 279, - -1, 809, 277, -1, 809, 17, 809, -1, 809, 18, - 809, -1, 836, 301, 836, -1, 809, 221, 435, -1, - 809, 221, 274, 435, -1, 809, 221, 159, -1, 809, - 221, 274, 159, -1, 809, 221, 446, -1, 809, 221, - 274, 446, -1, 809, 221, 132, 171, 809, -1, 809, - 221, 274, 132, 171, 809, -1, 809, 221, 284, 514, - 858, 515, -1, 809, 221, 274, 284, 514, 858, 515, - -1, 809, 53, 880, 810, 36, 809, -1, 809, 498, - 53, 880, 810, 36, 809, -1, 809, 53, 413, 810, - 36, 809, -1, 809, 498, 53, 413, 810, 36, 809, - -1, 809, 198, 868, -1, 809, 498, 198, 868, -1, - 809, 849, 844, 675, -1, 809, 849, 844, 514, 809, - 515, -1, 117, -1, 83, 514, 809, 515, -1, 507, - 886, 890, -1, 542, 516, 507, 886, 890, -1, 811, - -1, 810, 11, 774, -1, 505, 810, -1, 506, 810, - -1, 810, 505, 810, -1, 810, 506, 810, -1, 810, - 507, 810, -1, 810, 508, 810, -1, 810, 15, 810, - -1, 810, 509, 810, -1, 810, 510, 810, -1, 810, - 16, 810, -1, 810, 501, 810, -1, 810, 502, 810, - -1, 810, 503, 810, -1, 810, 19, 810, -1, 810, - 20, 810, -1, 810, 21, 810, -1, 810, 847, 810, - -1, 847, 810, -1, 810, 847, -1, 810, 221, 132, - 171, 810, -1, 810, 221, 274, 132, 171, 810, -1, - 810, 221, 284, 514, 858, 515, -1, 810, 221, 274, - 284, 514, 858, 515, -1, 812, -1, 813, 879, -1, - 874, -1, 897, -1, 675, -1, 675, 545, -1, 152, - 675, -1, 729, 514, 854, 515, -1, 514, 809, 515, - -1, 814, -1, 836, -1, 519, -1, 10, -1, 815, - -1, 246, 520, 843, 521, -1, 817, -1, 869, -1, - 512, 855, 513, -1, 820, -1, 39, 675, -1, 39, - 512, 855, 513, -1, 522, 9, -1, 523, 548, -1, - 520, 839, 521, -1, 896, 514, 515, -1, 896, 514, - 856, 698, 697, 515, -1, 896, 514, 462, 857, 698, - 697, 515, -1, 896, 514, 856, 518, 462, 857, 698, - 697, 515, -1, 896, 514, 30, 856, 698, 697, 515, - -1, 896, 514, 132, 856, 698, 697, 515, -1, 816, - 821, 822, 823, 827, -1, 819, -1, 816, -1, 819, - -1, 81, 166, 514, 809, 515, -1, 66, 514, 809, - 40, 774, 515, -1, 438, 514, 809, 40, 774, 515, - -1, 158, 514, 859, 515, -1, 302, 514, 861, 515, - -1, 320, 514, 863, 515, -1, 411, 514, 864, 515, - -1, 432, 514, 809, 40, 774, 515, -1, 434, 514, - 58, 867, 515, -1, 434, 514, 232, 867, 515, -1, - 434, 514, 429, 867, 515, -1, 434, 514, 867, 515, - -1, 280, 514, 809, 518, 809, 515, -1, 79, 514, - 854, 515, -1, 512, 809, 166, 542, 198, 809, 513, - -1, 512, 809, 166, 542, 198, 811, 191, 809, 513, - -1, 477, 180, 514, 699, 515, -1, -1, 162, 514, - 473, 809, 515, -1, 162, 514, 809, 515, -1, -1, - 155, -1, -1, 475, 825, -1, -1, 826, -1, 825, - 518, 826, -1, 542, 40, 828, -1, 300, 828, -1, - 300, 542, -1, -1, 514, 829, 830, 698, 831, 515, - -1, 542, -1, -1, 309, 59, 853, -1, -1, 337, - 832, 834, -1, 366, 832, 834, -1, 183, 832, 834, - -1, -1, 833, -1, 53, 833, 36, 833, -1, 441, - 323, -1, 441, 165, -1, 104, 365, -1, 809, 323, - -1, 809, 165, -1, 148, 104, 365, -1, 148, 180, - -1, 148, 425, -1, 148, 272, 297, -1, -1, 365, - 514, 854, 515, -1, 365, 514, 515, -1, 835, -1, - 514, 853, 518, 809, 515, -1, 543, 524, 809, -1, - 837, -1, 838, 518, 837, -1, 838, -1, 838, 518, - -1, 809, 524, 809, -1, 840, -1, 841, 518, 840, - -1, 841, -1, 841, 518, -1, 842, -1, -1, 38, - -1, 396, -1, 30, -1, 8, -1, 846, -1, 505, - -1, 506, -1, 507, -1, 508, -1, 15, -1, 509, - -1, 510, -1, 16, -1, 501, -1, 502, -1, 503, - -1, 19, -1, 20, -1, 21, -1, 8, -1, 291, - 514, 850, 515, -1, 845, -1, 291, 514, 850, 515, - -1, 845, -1, 291, 514, 850, 515, -1, 236, -1, - 498, 236, -1, 176, -1, 498, 176, -1, 193, -1, - 498, 193, -1, 845, -1, 542, 516, 850, -1, 811, - -1, 851, 518, 811, -1, 851, -1, 851, 518, -1, - 809, -1, 853, 518, 809, -1, 853, -1, 853, 518, - -1, 854, -1, -1, 857, -1, 856, 518, 857, -1, - 809, -1, 905, 13, 809, -1, 905, 14, 809, -1, - 774, -1, 858, 518, 774, -1, 860, 171, 809, -1, - -1, 3, -1, 796, -1, 797, -1, 798, -1, 799, - -1, 800, -1, 801, -1, 802, -1, 803, -1, 804, - -1, 805, -1, 806, -1, 807, -1, 544, -1, 809, - 862, 865, 866, -1, 809, 862, 865, -1, 317, 809, - -1, 810, 198, 810, -1, -1, 809, 865, 866, -1, - 809, 866, 865, -1, 809, 865, -1, 809, 866, -1, - 853, -1, -1, 171, 809, -1, 166, 809, -1, 809, - 171, 854, -1, 171, 854, -1, 854, -1, 675, -1, - 514, 854, 515, -1, 65, 873, 870, 872, 143, -1, - 871, -1, 870, 871, -1, 472, 809, 424, 809, -1, - 139, 809, -1, -1, 809, -1, -1, 542, -1, 542, - 545, -1, 512, 809, 513, -1, 512, 875, 524, 875, - 513, -1, 512, 875, 524, 875, 524, 875, 513, -1, - 512, 875, 524, 506, 524, 875, 513, -1, 809, -1, - -1, -1, 876, 546, -1, -1, 514, 515, -1, 514, - 856, 515, -1, 516, 547, 877, -1, 512, 809, 513, - -1, 512, 875, 524, 875, 513, -1, 512, 875, 524, - 875, 524, 875, 513, -1, 512, 875, 524, 506, 524, - 875, 513, -1, -1, 879, 878, -1, 45, -1, -1, - 883, -1, -1, 884, -1, 882, 518, 884, -1, 882, - -1, 882, 518, -1, 809, 40, 906, -1, 809, 3, - -1, 809, -1, 148, 514, 893, 515, -1, 148, 542, - -1, 885, -1, -1, 809, 40, 542, -1, 887, -1, - 888, 518, 887, -1, 888, -1, 888, 518, -1, 352, - 514, 889, 515, -1, 352, 887, -1, -1, 541, -1, - 891, 518, 541, -1, 895, -1, 892, 518, 895, -1, - 892, -1, 892, 518, -1, 893, -1, 514, 893, 515, - -1, 543, -1, 900, -1, 542, 545, -1, 898, -1, - 4, -1, 544, 876, -1, 6, -1, 7, -1, 896, - 544, -1, 896, 514, 856, 698, 697, 515, 544, -1, - 778, 544, -1, 794, 514, 809, 515, 808, -1, 794, - 898, 808, -1, 794, 544, 808, -1, 435, -1, 159, - -1, 279, -1, 9, -1, 3, -1, 977, -1, 982, - -1, 3, -1, 977, -1, 979, -1, 3, -1, 977, - -1, 980, -1, 542, -1, 542, 903, -1, 516, 547, - -1, 903, 516, 547, -1, 514, 893, 515, -1, -1, - 899, -1, 548, -1, 5, -1, 325, 895, 908, 40, - 909, -1, 514, 858, 515, -1, -1, 674, -1, 551, - -1, 655, -1, 656, -1, 953, -1, 965, -1, 100, - 370, 541, 911, -1, 100, 370, 191, 274, 152, 541, - 911, -1, 100, 294, 352, 370, 541, 911, -1, 911, - 912, -1, -1, 598, -1, 913, -1, 576, -1, 972, - -1, 100, 919, 202, 916, 917, 289, 541, 915, 514, - 570, 515, 918, 767, -1, 100, 919, 202, 916, 191, - 274, 152, 627, 289, 541, 915, 514, 570, 515, 918, - 767, -1, 542, -1, 454, 914, -1, -1, 89, -1, - -1, 627, -1, -1, 476, 613, -1, -1, 445, -1, - -1, 32, 416, 761, 386, 370, 895, -1, 32, 416, - 191, 152, 761, 386, 370, 895, -1, 32, 381, 541, - 386, 370, 895, -1, 32, 381, 191, 152, 541, 386, - 370, 895, -1, 32, 466, 541, 386, 370, 895, -1, - 32, 466, 191, 152, 541, 386, 370, 895, -1, 167, - 75, 922, -1, 75, 922, -1, 542, -1, -1, 84, - 289, 925, 541, 221, 924, -1, 84, 289, 82, 809, - 221, 924, -1, 544, -1, 279, -1, 416, -1, 381, - -1, 173, -1, 245, -1, 245, 416, -1, 466, -1, - 108, -1, 202, -1, 370, -1, 439, -1, 154, 108, - 544, 664, -1, 154, 108, 542, 428, 544, 664, -1, - 197, 108, 544, -1, 153, 931, -1, 153, 935, 929, - 931, -1, 153, 464, 931, -1, 153, 514, 934, 515, - 931, -1, 464, -1, -1, 936, -1, 589, -1, -1, - 920, -1, 586, -1, 529, -1, 971, -1, 921, -1, - 656, -1, 974, -1, 652, -1, 910, -1, 576, -1, - 598, -1, 572, -1, 540, -1, 953, -1, 646, -1, - 582, -1, 913, -1, 551, -1, 945, -1, 575, -1, - 907, -1, 549, -1, 674, -1, 595, -1, 655, -1, - 948, -1, 962, -1, 939, -1, 965, -1, 972, -1, - 3, -1, 977, -1, 981, -1, 932, -1, 544, -1, - 937, -1, 934, 518, 937, -1, 35, -1, 34, -1, - 435, -1, 159, -1, 289, -1, 933, -1, 938, 930, - -1, 932, -1, 935, -1, 386, 940, -1, 386, 240, - 940, -1, 386, 385, 940, -1, 386, 177, 940, -1, - 941, -1, 969, 171, 104, -1, 426, 497, 943, -1, - 370, 544, -1, 969, 428, 944, -1, 969, 503, 944, - -1, 809, -1, 544, -1, 3, -1, 794, 544, 808, - -1, 794, 514, 898, 515, 544, -1, 589, -1, 117, - -1, 240, -1, 942, -1, 944, 518, 942, -1, 239, - 946, -1, 213, 946, -1, 167, 213, 946, -1, 213, - 946, 171, 947, -1, 167, 213, 946, 171, 947, -1, - 544, -1, 542, -1, 544, -1, 542, -1, 455, 950, - 952, 929, -1, 455, 950, 952, 929, 541, 904, -1, - 455, 950, 952, 929, 957, -1, 455, 514, 951, 515, - -1, 455, 514, 951, 515, 541, 904, -1, 935, -1, - 464, -1, 170, -1, 172, -1, 3, -1, 172, -1, - -1, 949, -1, 951, 518, 949, -1, 170, -1, -1, - 556, 122, 171, 954, 956, 955, 562, -1, 436, 692, - 954, -1, 761, -1, 761, 542, -1, 761, 40, 542, - -1, 473, 809, -1, -1, 454, 743, -1, -1, 935, - 929, -1, 935, 929, 541, 904, -1, 47, 960, 544, - 961, 664, -1, 47, 191, 274, 152, 960, 544, 961, - 664, -1, 128, 548, -1, 128, 108, 548, -1, 128, - 108, 191, 152, 548, -1, 108, -1, -1, 40, 542, - -1, -1, 354, 964, -1, 354, 240, 964, -1, 354, - 385, 964, -1, 354, 177, 964, -1, 969, -1, 30, - -1, 963, -1, 426, 497, -1, 430, 223, 235, -1, - 967, 674, -1, 412, 674, -1, 412, 970, -1, 967, - 970, -1, 967, 426, 497, -1, 967, 430, 223, 235, - -1, 967, 30, 968, -1, 967, -1, 127, -1, 126, - -1, 390, -1, 966, -1, 417, -1, -1, 542, -1, - 969, 516, 542, -1, 542, -1, 970, 516, 542, -1, - 61, 816, -1, 100, 644, 466, 541, 633, 918, 40, - 674, 973, -1, 100, 644, 466, 191, 274, 152, 541, - 633, 918, 40, 674, 973, -1, 100, 294, 352, 644, - 466, 541, 633, 918, 40, 674, 973, -1, 100, 644, - 342, 466, 541, 514, 637, 515, 918, 40, 674, 973, - -1, 100, 294, 352, 644, 342, 466, 541, 514, 637, - 515, 918, 40, 674, 973, -1, 476, 74, 292, -1, - 476, 64, 74, 292, -1, 476, 240, 74, 292, -1, - -1, 100, 644, 416, 976, 40, 674, 975, -1, 100, - 644, 416, 191, 274, 152, 976, 40, 674, 975, -1, - 100, 294, 352, 644, 416, 976, 40, 674, 975, -1, - 476, 107, -1, 476, 272, 107, -1, -1, 541, 633, - 620, 612, -1, 22, -1, 23, -1, 24, -1, 25, - -1, 26, -1, 27, -1, 28, -1, 29, -1, 31, - -1, 32, -1, 33, -1, 43, -1, 44, -1, 46, - -1, 47, -1, 48, -1, 50, -1, 51, -1, 52, - -1, 59, -1, 60, -1, 61, -1, 62, -1, 63, - -1, 64, -1, 67, -1, 68, -1, 69, -1, 70, - -1, 73, -1, 75, -1, 76, -1, 77, -1, 78, - -1, 84, -1, 85, -1, 86, -1, 87, -1, 88, - -1, 90, -1, 91, -1, 92, -1, 94, -1, 95, - -1, 96, -1, 97, -1, 98, -1, 99, -1, 102, - -1, 103, -1, 104, -1, 105, -1, 106, -1, 107, - -1, 108, -1, 109, -1, 110, -1, 111, -1, 113, - -1, 114, -1, 116, -1, 118, -1, 120, -1, 121, - -1, 122, -1, 123, -1, 124, -1, 125, -1, 128, - -1, 129, -1, 130, -1, 131, -1, 134, -1, 135, - -1, 136, -1, 137, -1, 138, -1, 140, -1, 141, - -1, 142, -1, 144, -1, 145, -1, 146, -1, 148, - -1, 149, -1, 150, -1, 151, -1, 153, -1, 154, - -1, 155, -1, 156, -1, 157, -1, 160, -1, 162, - -1, 163, -1, 165, -1, 167, -1, 169, -1, 173, - -1, 174, -1, 177, -1, 179, -1, 183, -1, 184, - -1, 186, -1, 187, -1, 188, -1, 189, -1, 190, - -1, 191, -1, 192, -1, 194, -1, 195, -1, 196, - -1, 197, -1, 199, -1, 200, -1, 201, -1, 202, - -1, 203, -1, 204, -1, 205, -1, 207, -1, 210, - -1, 211, -1, 212, -1, 213, -1, 214, -1, 220, - -1, 223, -1, 225, -1, 226, -1, 227, -1, 228, - -1, 229, -1, 230, -1, 233, -1, 235, -1, 238, - -1, 239, -1, 240, -1, 241, -1, 242, -1, 243, - -1, 244, -1, 245, -1, 247, -1, 248, -1, 249, - -1, 250, -1, 251, -1, 252, -1, 253, -1, 254, - -1, 255, -1, 256, -1, 257, -1, 258, -1, 259, - -1, 260, -1, 261, -1, 262, -1, 263, -1, 264, - -1, 265, -1, 266, -1, 270, -1, 271, -1, 272, - -1, 275, -1, 276, -1, 278, -1, 281, -1, 283, - -1, 284, -1, 285, -1, 287, -1, 288, -1, 291, - -1, 292, -1, 293, -1, 296, -1, 297, -1, 300, - -1, 303, -1, 304, -1, 305, -1, 306, -1, 307, - -1, 308, -1, 309, -1, 310, -1, 311, -1, 312, - -1, 313, -1, 318, -1, 319, -1, 322, -1, 323, - -1, 325, -1, 326, -1, 327, -1, 329, -1, 330, - -1, 331, -1, 332, -1, 333, -1, 334, -1, 336, - -1, 337, -1, 338, -1, 340, -1, 341, -1, 342, - -1, 343, -1, 345, -1, 346, -1, 347, -1, 348, - -1, 349, -1, 350, -1, 351, -1, 352, -1, 353, - -1, 354, -1, 355, -1, 356, -1, 357, -1, 359, - -1, 360, -1, 362, -1, 363, -1, 364, -1, 366, - -1, 367, -1, 368, -1, 369, -1, 370, -1, 371, - -1, 372, -1, 373, -1, 374, -1, 375, -1, 376, - -1, 377, -1, 378, -1, 381, -1, 382, -1, 383, - -1, 384, -1, 385, -1, 386, -1, 388, -1, 389, - -1, 392, -1, 393, -1, 395, -1, 397, -1, 398, - -1, 399, -1, 400, -1, 401, -1, 402, -1, 403, - -1, 404, -1, 405, -1, 406, -1, 407, -1, 408, - -1, 410, -1, 414, -1, 415, -1, 417, -1, 419, - -1, 420, -1, 421, -1, 422, -1, 423, -1, 425, - -1, 430, -1, 431, -1, 433, -1, 436, -1, 437, - -1, 439, -1, 440, -1, 441, -1, 442, -1, 443, - -1, 446, -1, 447, -1, 448, -1, 450, -1, 451, - -1, 452, -1, 453, -1, 455, -1, 456, -1, 457, - -1, 458, -1, 459, -1, 463, -1, 465, -1, 466, - -1, 467, -1, 468, -1, 469, -1, 470, -1, 471, - -1, 474, -1, 477, -1, 478, -1, 479, -1, 480, - -1, 481, -1, 482, -1, 494, -1, 495, -1, 496, - -1, 497, -1, 53, -1, 54, -1, 56, -1, 57, - -1, 71, -1, 72, -1, 79, -1, 83, -1, 112, - -1, 115, -1, 152, -1, 158, -1, 164, -1, 175, - -1, 181, -1, 182, -1, 209, -1, 215, -1, 216, - -1, 218, -1, 246, -1, 267, -1, 269, -1, 273, - -1, 280, -1, 282, -1, 298, -1, 302, -1, 320, - -1, 324, -1, 339, -1, 365, -1, 387, -1, 394, - -1, 409, -1, 411, -1, 426, -1, 427, -1, 432, - -1, 434, -1, 438, -1, 460, -1, 461, -1, 483, + 673, 515, -1, 937, -1, 589, -1, 815, -1, 507, + -1, 514, 660, 515, -1, 816, -1, -1, 548, 665, + -1, 476, 287, -1, -1, 668, 670, -1, -1, 55, + -1, -1, 55, -1, 287, -1, 170, -1, 123, 662, + 544, -1, 279, 662, 544, -1, 102, -1, 186, -1, + 336, 662, 544, -1, 145, 662, 544, -1, 167, 336, + 637, -1, 167, 336, 507, -1, 309, 59, 637, -1, + 309, 59, 507, -1, 167, 274, 279, 637, -1, 167, + 279, 637, -1, 141, 544, -1, 937, -1, 544, -1, + 403, -1, 404, -1, 3, 516, 542, -1, 3, -1, + 666, -1, 673, 518, 666, -1, 676, -1, 675, -1, + 514, 676, 515, -1, 514, 675, 515, -1, 514, 966, + 515, -1, 679, -1, 677, 699, -1, 677, 698, 732, + 705, -1, 677, 698, 704, 733, -1, 686, 677, -1, + 686, 677, 699, -1, 686, 677, 698, 732, 705, -1, + 686, 677, 698, 704, 733, -1, 679, -1, 675, -1, + 379, 696, 882, -1, -1, 379, 696, 882, 690, 741, + 767, 721, 730, 825, 731, 709, -1, 379, 695, 884, + 690, 741, 767, 721, 730, 825, 731, 709, -1, 171, + 742, 678, 690, 767, 721, 730, 825, 731, 709, -1, + 171, 742, 379, 695, 884, 690, 767, 721, 730, 825, + 731, 709, -1, 740, -1, 416, 761, -1, 677, 444, + 693, 694, 677, -1, 677, 444, 693, 677, -1, 677, + 217, 693, 677, -1, 677, 147, 693, 677, -1, 681, + 744, 454, 884, -1, 681, 744, 454, 884, 180, 59, + 895, -1, 681, 744, 180, 59, 895, -1, 681, 744, + 289, 685, -1, 681, 744, 289, 685, 180, 59, 895, + -1, 681, 744, 289, 685, 454, 884, -1, 681, 744, + 289, 685, 454, 884, 180, 59, 895, -1, 682, 744, + 289, 884, 219, 265, 896, 680, 895, -1, 682, 744, + 289, 884, -1, 459, -1, 460, -1, 314, -1, 316, + -1, 449, -1, 315, -1, 810, -1, 810, 198, 514, + 676, 515, -1, 747, -1, 683, -1, 684, 518, 683, + -1, 684, -1, 684, 518, -1, 476, 687, -1, 500, + 687, -1, 476, 342, 687, -1, 688, -1, 687, 518, + 688, -1, 896, 905, 40, 689, 514, 910, 515, -1, + 249, -1, 274, 249, -1, -1, 219, 691, -1, -1, + 422, 692, 541, -1, 420, 692, 541, -1, 240, 422, + 692, 541, -1, 240, 420, 692, 541, -1, 177, 422, + 692, 541, -1, 177, 420, 692, 541, -1, 448, 692, + 541, -1, 416, 541, -1, 541, -1, 416, -1, -1, + 30, -1, 132, -1, -1, 59, 265, -1, 132, -1, + 132, 289, 514, 855, 515, -1, 30, -1, -1, 192, + 281, -1, 355, 281, -1, -1, 699, -1, -1, 295, + 59, 700, -1, 295, 59, 30, 702, 703, -1, 701, + -1, 700, 518, 701, -1, 809, 454, 849, 703, -1, + 809, 702, 703, -1, 41, -1, 126, -1, -1, 499, + 163, -1, 499, 230, -1, -1, 706, 707, -1, 707, + 706, -1, 706, -1, 707, -1, 704, -1, -1, 237, + 715, -1, 237, 715, 518, 716, -1, 161, 720, 717, + 719, 290, -1, 161, 720, 719, 290, -1, 286, 716, + -1, 286, 717, 719, -1, 4, 509, -1, 9, 509, + -1, 4, 312, -1, 9, 312, -1, 9, -1, 9, + 366, -1, 454, 368, 711, -1, -1, 542, -1, -1, + 710, 514, 708, 515, 714, -1, 708, -1, 708, 514, + 542, 515, -1, 708, 514, 542, 518, 9, 515, -1, + 418, 711, -1, 712, -1, -1, 351, 514, 9, 515, + -1, -1, 809, -1, 30, -1, 809, 509, -1, 4, + 312, -1, 9, 312, -1, 809, -1, 811, -1, 505, + 718, -1, 506, 718, -1, 899, -1, 4, -1, 365, + -1, 366, -1, 163, -1, 271, -1, 180, 59, 723, + -1, 180, 59, 30, -1, -1, 724, -1, 722, 518, + 724, -1, 722, -1, 722, 518, -1, 809, -1, 725, + -1, 727, -1, 726, -1, 728, -1, 514, 515, -1, + 364, 514, 855, 515, -1, 103, 514, 855, 515, -1, + 181, 388, 514, 723, 515, -1, 181, -1, 182, -1, + 185, 809, -1, -1, 335, 809, -1, -1, 734, -1, + 166, 338, 290, -1, 732, -1, -1, 735, -1, 734, + 735, -1, 736, 737, 738, -1, 166, 451, -1, 166, + 272, 226, 451, -1, 166, 389, -1, 166, 226, 389, + -1, 284, 892, -1, -1, 278, -1, 393, 243, -1, + -1, 460, 514, 855, 515, -1, 739, 518, 514, 855, + 515, -1, 739, -1, 739, 518, -1, 171, 743, -1, + -1, 744, -1, 742, 518, 744, -1, 742, -1, 742, + 518, -1, 761, 756, 713, -1, 762, 757, 713, -1, + 740, 755, 713, -1, 231, 762, 757, -1, 675, 756, + 713, -1, 231, 675, 756, -1, 754, -1, 514, 754, + 515, 755, -1, 744, 314, 514, 884, 166, 750, 745, + 515, 756, -1, 744, 449, 746, 514, 751, 166, 753, + 515, 756, -1, 180, 59, 894, -1, -1, 199, 281, + -1, 148, 281, -1, -1, 810, 198, 514, 884, 515, + -1, 810, 198, 543, -1, -1, 812, -1, 814, -1, + 514, 853, 515, -1, 748, 198, 514, 884, 515, -1, + 748, 198, 543, -1, 749, -1, 750, 749, -1, 543, + -1, 514, 894, 515, -1, 751, 198, 514, 884, 515, + -1, 752, -1, 753, 752, -1, 514, 754, 515, -1, + 744, 101, 224, 744, -1, 744, 758, 224, 744, 760, + -1, 744, 224, 744, 760, -1, 744, 268, 758, 224, + 744, -1, 744, 268, 224, 744, -1, 744, 42, 758, + 224, 744, 760, -1, 744, 42, 224, 744, 760, -1, + 744, 321, 224, 744, -1, 744, 37, 224, 744, 760, + -1, 744, 380, 224, 744, 760, -1, 40, 543, 514, + 894, 515, -1, 40, 543, -1, 542, 514, 894, 515, + -1, 542, -1, 755, -1, -1, 755, -1, 40, 514, + 768, 515, -1, 40, 543, 514, 768, 515, -1, 542, + 514, 768, 515, -1, -1, 172, 759, -1, 234, 759, + -1, 361, 759, -1, 380, -1, 37, -1, 208, -1, + 299, -1, -1, 454, 514, 894, 515, -1, 289, 809, + -1, 541, -1, 541, 507, -1, 290, 541, -1, 290, + 514, 541, 515, -1, 819, 766, -1, 366, 171, 514, + 764, 515, 766, -1, 819, 765, -1, 763, -1, 764, + 518, 763, -1, 40, 514, 768, 515, -1, -1, 500, + 296, -1, -1, 473, 809, -1, -1, 769, -1, 768, + 518, 769, -1, 543, 774, 770, -1, 80, 903, -1, + -1, 542, 774, -1, 771, 518, 542, 774, -1, 365, + -1, 409, -1, 774, -1, -1, 777, 776, -1, 387, + 777, 776, -1, 777, 39, 512, 899, 513, -1, 387, + 777, 39, 512, 899, 513, -1, 777, 39, -1, 387, + 777, 39, -1, 775, -1, 772, 514, 771, 515, 776, + -1, 246, 514, 859, 515, 776, -1, 444, 514, 771, + 515, 776, -1, 3, 516, 3, -1, 775, 516, 3, + -1, 776, 512, 513, -1, 776, 512, 899, 513, -1, + -1, 779, -1, 781, -1, 783, -1, 787, -1, 793, + -1, 794, 808, -1, 794, 514, 899, 515, -1, 781, + -1, 784, -1, 788, -1, 793, -1, 902, 780, -1, + 514, 856, 515, -1, -1, 215, -1, 216, -1, 394, + -1, 54, -1, 339, -1, 164, 782, -1, 136, 324, + -1, 115, 780, -1, 112, 780, -1, 282, 780, -1, + 57, -1, 514, 899, 515, -1, -1, 785, -1, 786, + -1, 785, -1, 786, -1, 56, 792, 514, 855, 515, + -1, 56, 792, -1, 789, -1, 790, -1, 789, -1, + 790, -1, 791, 514, 899, 515, -1, 791, -1, 72, + 792, -1, 71, 792, -1, 461, -1, 267, 72, 792, + -1, 267, 71, 792, -1, 269, 792, -1, 463, -1, + -1, 427, 514, 899, 515, 795, -1, 427, 795, -1, + 426, 514, 899, 515, 795, -1, 426, 795, -1, 218, + -1, 500, 426, 497, -1, 478, 426, 497, -1, -1, + 494, -1, 495, -1, 262, -1, 263, -1, 109, -1, + 110, -1, 188, -1, 189, -1, 258, -1, 259, -1, + 375, -1, 376, -1, 256, -1, 257, -1, 252, -1, + 253, -1, 470, -1, 471, -1, 113, -1, 114, -1, + 69, -1, 68, -1, 255, -1, 254, -1, 796, -1, + 797, -1, 798, -1, 799, -1, 800, -1, 801, -1, + 802, -1, 803, -1, 804, -1, 805, -1, 806, -1, + 807, -1, 796, 428, 797, -1, 798, 428, 799, -1, + 798, 428, 800, -1, 798, 428, 801, -1, 799, 428, + 800, -1, 799, 428, 801, -1, 800, 428, 801, -1, + -1, 811, -1, 809, 11, 774, -1, 809, 80, 903, + -1, 809, 46, 426, 497, 809, -1, 505, 809, -1, + 506, 809, -1, 809, 505, 809, -1, 809, 506, 809, + -1, 809, 507, 809, -1, 809, 508, 809, -1, 809, + 15, 809, -1, 809, 509, 809, -1, 809, 510, 809, + -1, 809, 16, 809, -1, 809, 501, 809, -1, 809, + 502, 809, -1, 809, 503, 809, -1, 809, 19, 809, + -1, 809, 20, 809, -1, 809, 21, 809, -1, 809, + 848, 809, -1, 848, 809, -1, 809, 848, -1, 809, + 36, 809, -1, 809, 294, 809, -1, 274, 809, -1, + 498, 809, -1, 809, 176, 809, -1, 809, 236, 809, + -1, 809, 236, 809, 145, 809, -1, 809, 498, 236, + 809, -1, 809, 498, 236, 809, 145, 809, -1, 809, + 193, 809, -1, 809, 193, 809, 145, 809, -1, 809, + 498, 193, 809, -1, 809, 498, 193, 809, 145, 809, + -1, 809, 391, 428, 809, -1, 809, 391, 428, 809, + 145, 809, -1, 809, 498, 391, 428, 809, -1, 809, + 498, 391, 428, 809, 145, 809, -1, 809, 221, 279, + -1, 809, 222, -1, 809, 221, 274, 279, -1, 809, + 274, 279, -1, 809, 277, -1, 809, 17, 809, -1, + 809, 18, 809, -1, 837, 301, 837, -1, 809, 221, + 435, -1, 809, 221, 274, 435, -1, 809, 221, 159, + -1, 809, 221, 274, 159, -1, 809, 221, 446, -1, + 809, 221, 274, 446, -1, 809, 221, 132, 171, 809, + -1, 809, 221, 274, 132, 171, 809, -1, 809, 221, + 284, 514, 859, 515, -1, 809, 221, 274, 284, 514, + 859, 515, -1, 809, 53, 881, 810, 36, 809, -1, + 809, 498, 53, 881, 810, 36, 809, -1, 809, 53, + 413, 810, 36, 809, -1, 809, 498, 53, 413, 810, + 36, 809, -1, 809, 198, 869, -1, 809, 498, 198, + 869, -1, 809, 850, 845, 675, -1, 809, 850, 845, + 514, 809, 515, -1, 117, -1, 83, 514, 809, 515, + -1, 507, 887, 891, -1, 542, 516, 507, 887, 891, + -1, 811, -1, 810, 11, 774, -1, 505, 810, -1, + 506, 810, -1, 810, 505, 810, -1, 810, 506, 810, + -1, 810, 507, 810, -1, 810, 508, 810, -1, 810, + 15, 810, -1, 810, 509, 810, -1, 810, 510, 810, + -1, 810, 16, 810, -1, 810, 501, 810, -1, 810, + 502, 810, -1, 810, 503, 810, -1, 810, 19, 810, + -1, 810, 20, 810, -1, 810, 21, 810, -1, 810, + 848, 810, -1, 848, 810, -1, 810, 848, -1, 810, + 221, 132, 171, 810, -1, 810, 221, 274, 132, 171, + 810, -1, 810, 221, 284, 514, 859, 515, -1, 810, + 221, 274, 284, 514, 859, 515, -1, 812, -1, 813, + 880, -1, 875, -1, 898, -1, 675, -1, 675, 545, + -1, 152, 675, -1, 729, 514, 855, 515, -1, 514, + 809, 515, -1, 814, -1, 837, -1, 519, -1, 10, + -1, 816, -1, 246, 520, 844, 521, -1, 818, -1, + 870, -1, 815, -1, 821, -1, 39, 675, -1, 39, + 512, 856, 513, -1, 522, 9, -1, 523, 548, -1, + 512, 856, 513, -1, 520, 840, 521, -1, 897, 514, + 515, -1, 897, 514, 857, 698, 697, 515, -1, 897, + 514, 462, 858, 698, 697, 515, -1, 897, 514, 857, + 518, 462, 858, 698, 697, 515, -1, 897, 514, 30, + 857, 698, 697, 515, -1, 897, 514, 132, 857, 698, + 697, 515, -1, 817, 822, 823, 824, 828, -1, 820, + -1, 817, -1, 820, -1, 81, 166, 514, 809, 515, + -1, 66, 514, 809, 40, 774, 515, -1, 438, 514, + 809, 40, 774, 515, -1, 158, 514, 860, 515, -1, + 302, 514, 862, 515, -1, 320, 514, 864, 515, -1, + 411, 514, 865, 515, -1, 432, 514, 809, 40, 774, + 515, -1, 434, 514, 58, 868, 515, -1, 434, 514, + 232, 868, 515, -1, 434, 514, 429, 868, 515, -1, + 434, 514, 868, 515, -1, 280, 514, 809, 518, 809, + 515, -1, 79, 514, 855, 515, -1, 512, 809, 166, + 542, 198, 809, 513, -1, 512, 809, 166, 542, 198, + 811, 191, 809, 513, -1, 477, 180, 514, 699, 515, + -1, -1, 162, 514, 473, 809, 515, -1, 162, 514, + 809, 515, -1, -1, 155, -1, -1, 475, 826, -1, + -1, 827, -1, 826, 518, 827, -1, 542, 40, 829, + -1, 300, 829, -1, 300, 542, -1, -1, 514, 830, + 831, 698, 832, 515, -1, 542, -1, -1, 309, 59, + 854, -1, -1, 337, 833, 835, -1, 366, 833, 835, + -1, 183, 833, 835, -1, -1, 834, -1, 53, 834, + 36, 834, -1, 441, 323, -1, 441, 165, -1, 104, + 365, -1, 809, 323, -1, 809, 165, -1, 148, 104, + 365, -1, 148, 180, -1, 148, 425, -1, 148, 272, + 297, -1, -1, 365, 514, 855, 515, -1, 365, 514, + 515, -1, 836, -1, 514, 854, 518, 809, 515, -1, + 543, 524, 809, -1, 838, -1, 839, 518, 838, -1, + 839, -1, 839, 518, -1, 809, 524, 809, -1, 841, + -1, 842, 518, 841, -1, 842, -1, 842, 518, -1, + 843, -1, -1, 38, -1, 396, -1, 30, -1, 8, + -1, 847, -1, 505, -1, 506, -1, 507, -1, 508, + -1, 15, -1, 509, -1, 510, -1, 16, -1, 501, + -1, 502, -1, 503, -1, 19, -1, 20, -1, 21, + -1, 8, -1, 291, 514, 851, 515, -1, 846, -1, + 291, 514, 851, 515, -1, 846, -1, 291, 514, 851, + 515, -1, 236, -1, 498, 236, -1, 176, -1, 498, + 176, -1, 193, -1, 498, 193, -1, 846, -1, 542, + 516, 851, -1, 811, -1, 852, 518, 811, -1, 852, + -1, 852, 518, -1, 809, -1, 854, 518, 809, -1, + 854, -1, 854, 518, -1, 855, -1, -1, 858, -1, + 857, 518, 858, -1, 809, -1, 906, 13, 809, -1, + 906, 14, 809, -1, 774, -1, 859, 518, 774, -1, + 861, 171, 809, -1, -1, 3, -1, 796, -1, 797, + -1, 798, -1, 799, -1, 800, -1, 801, -1, 802, + -1, 803, -1, 804, -1, 805, -1, 806, -1, 807, + -1, 544, -1, 809, 863, 866, 867, -1, 809, 863, + 866, -1, 317, 809, -1, 810, 198, 810, -1, -1, + 809, 866, 867, -1, 809, 867, 866, -1, 809, 866, + -1, 809, 867, -1, 854, -1, -1, 171, 809, -1, + 166, 809, -1, 809, 171, 855, -1, 171, 855, -1, + 855, -1, 675, -1, 514, 855, 515, -1, 65, 874, + 871, 873, 143, -1, 872, -1, 871, 872, -1, 472, + 809, 424, 809, -1, 139, 809, -1, -1, 809, -1, + -1, 542, -1, 542, 545, -1, 512, 809, 513, -1, + 512, 876, 524, 876, 513, -1, 512, 876, 524, 876, + 524, 876, 513, -1, 512, 876, 524, 506, 524, 876, + 513, -1, 809, -1, -1, -1, 877, 546, -1, -1, + 514, 515, -1, 514, 857, 515, -1, 516, 547, 878, + -1, 512, 809, 513, -1, 512, 876, 524, 876, 513, + -1, 512, 876, 524, 876, 524, 876, 513, -1, 512, + 876, 524, 506, 524, 876, 513, -1, -1, 880, 879, + -1, 45, -1, -1, 884, -1, -1, 885, -1, 883, + 518, 885, -1, 883, -1, 883, 518, -1, 809, 40, + 907, -1, 809, 3, -1, 809, -1, 148, 514, 894, + 515, -1, 148, 542, -1, 886, -1, -1, 809, 40, + 542, -1, 888, -1, 889, 518, 888, -1, 889, -1, + 889, 518, -1, 352, 514, 890, 515, -1, 352, 888, + -1, -1, 541, -1, 892, 518, 541, -1, 896, -1, + 893, 518, 896, -1, 893, -1, 893, 518, -1, 894, + -1, 514, 894, 515, -1, 543, -1, 901, -1, 542, + 545, -1, 899, -1, 4, -1, 544, 877, -1, 6, + -1, 7, -1, 897, 544, -1, 897, 514, 857, 698, + 697, 515, 544, -1, 778, 544, -1, 794, 514, 809, + 515, 808, -1, 794, 899, 808, -1, 794, 544, 808, + -1, 435, -1, 159, -1, 279, -1, 9, -1, 3, + -1, 978, -1, 983, -1, 3, -1, 978, -1, 980, + -1, 3, -1, 978, -1, 981, -1, 542, -1, 542, + 904, -1, 516, 547, -1, 904, 516, 547, -1, 514, + 894, 515, -1, -1, 900, -1, 548, -1, 5, -1, + 325, 896, 909, 40, 910, -1, 514, 859, 515, -1, + -1, 674, -1, 551, -1, 655, -1, 656, -1, 954, + -1, 966, -1, 100, 370, 541, 912, -1, 100, 370, + 191, 274, 152, 541, 912, -1, 100, 294, 352, 370, + 541, 912, -1, 912, 913, -1, -1, 598, -1, 914, + -1, 576, -1, 973, -1, 100, 920, 202, 917, 918, + 289, 541, 916, 514, 570, 515, 919, 767, -1, 100, + 920, 202, 917, 191, 274, 152, 627, 289, 541, 916, + 514, 570, 515, 919, 767, -1, 542, -1, 454, 915, + -1, -1, 89, -1, -1, 627, -1, -1, 476, 613, + -1, -1, 445, -1, -1, 32, 416, 761, 386, 370, + 896, -1, 32, 416, 191, 152, 761, 386, 370, 896, + -1, 32, 381, 541, 386, 370, 896, -1, 32, 381, + 191, 152, 541, 386, 370, 896, -1, 32, 466, 541, + 386, 370, 896, -1, 32, 466, 191, 152, 541, 386, + 370, 896, -1, 167, 75, 923, -1, 75, 923, -1, + 542, -1, -1, 84, 289, 926, 541, 221, 925, -1, + 84, 289, 82, 809, 221, 925, -1, 544, -1, 279, + -1, 416, -1, 381, -1, 173, -1, 245, -1, 245, + 416, -1, 466, -1, 108, -1, 202, -1, 370, -1, + 439, -1, 154, 108, 544, 664, -1, 154, 108, 542, + 428, 544, 664, -1, 197, 108, 544, -1, 153, 932, + -1, 153, 936, 930, 932, -1, 153, 464, 932, -1, + 153, 514, 935, 515, 932, -1, 464, -1, -1, 937, + -1, 589, -1, -1, 921, -1, 586, -1, 529, -1, + 972, -1, 922, -1, 656, -1, 975, -1, 652, -1, + 911, -1, 576, -1, 598, -1, 572, -1, 540, -1, + 954, -1, 646, -1, 582, -1, 914, -1, 551, -1, + 946, -1, 575, -1, 908, -1, 549, -1, 674, -1, + 595, -1, 655, -1, 949, -1, 963, -1, 940, -1, + 966, -1, 973, -1, 3, -1, 978, -1, 982, -1, + 933, -1, 544, -1, 938, -1, 935, 518, 938, -1, + 35, -1, 34, -1, 435, -1, 159, -1, 289, -1, + 934, -1, 939, 931, -1, 933, -1, 936, -1, 386, + 941, -1, 386, 240, 941, -1, 386, 385, 941, -1, + 386, 177, 941, -1, 942, -1, 970, 171, 104, -1, + 426, 497, 944, -1, 370, 544, -1, 970, 428, 945, + -1, 970, 503, 945, -1, 809, -1, 544, -1, 3, + -1, 794, 544, 808, -1, 794, 514, 899, 515, 544, + -1, 589, -1, 117, -1, 240, -1, 943, -1, 945, + 518, 943, -1, 239, 947, -1, 213, 947, -1, 167, + 213, 947, -1, 213, 947, 171, 948, -1, 167, 213, + 947, 171, 948, -1, 544, -1, 542, -1, 544, -1, + 542, -1, 455, 951, 953, 930, -1, 455, 951, 953, + 930, 541, 905, -1, 455, 951, 953, 930, 958, -1, + 455, 514, 952, 515, -1, 455, 514, 952, 515, 541, + 905, -1, 936, -1, 464, -1, 170, -1, 172, -1, + 3, -1, 172, -1, -1, 950, -1, 952, 518, 950, + -1, 170, -1, -1, 556, 122, 171, 955, 957, 956, + 562, -1, 436, 692, 955, -1, 761, -1, 761, 542, + -1, 761, 40, 542, -1, 473, 809, -1, -1, 454, + 743, -1, -1, 936, 930, -1, 936, 930, 541, 905, + -1, 47, 961, 544, 962, 664, -1, 47, 191, 274, + 152, 961, 544, 962, 664, -1, 128, 548, -1, 128, + 108, 548, -1, 128, 108, 191, 152, 548, -1, 108, + -1, -1, 40, 542, -1, -1, 354, 965, -1, 354, + 240, 965, -1, 354, 385, 965, -1, 354, 177, 965, + -1, 970, -1, 30, -1, 964, -1, 426, 497, -1, + 430, 223, 235, -1, 968, 674, -1, 412, 674, -1, + 412, 971, -1, 968, 971, -1, 968, 426, 497, -1, + 968, 430, 223, 235, -1, 968, 30, 969, -1, 968, + -1, 127, -1, 126, -1, 390, -1, 967, -1, 417, + -1, -1, 542, -1, 970, 516, 542, -1, 542, -1, + 971, 516, 542, -1, 61, 817, -1, 100, 644, 466, + 541, 633, 919, 40, 674, 974, -1, 100, 644, 466, + 191, 274, 152, 541, 633, 919, 40, 674, 974, -1, + 100, 294, 352, 644, 466, 541, 633, 919, 40, 674, + 974, -1, 100, 644, 342, 466, 541, 514, 637, 515, + 919, 40, 674, 974, -1, 100, 294, 352, 644, 342, + 466, 541, 514, 637, 515, 919, 40, 674, 974, -1, + 476, 74, 292, -1, 476, 64, 74, 292, -1, 476, + 240, 74, 292, -1, -1, 100, 644, 416, 977, 40, + 674, 976, -1, 100, 644, 416, 191, 274, 152, 977, + 40, 674, 976, -1, 100, 294, 352, 644, 416, 977, + 40, 674, 976, -1, 476, 107, -1, 476, 272, 107, + -1, -1, 541, 633, 620, 612, -1, 22, -1, 23, + -1, 24, -1, 25, -1, 26, -1, 27, -1, 28, + -1, 29, -1, 31, -1, 32, -1, 33, -1, 43, + -1, 44, -1, 46, -1, 47, -1, 48, -1, 50, + -1, 51, -1, 52, -1, 59, -1, 60, -1, 61, + -1, 62, -1, 63, -1, 64, -1, 67, -1, 68, + -1, 69, -1, 70, -1, 73, -1, 75, -1, 76, + -1, 77, -1, 78, -1, 84, -1, 85, -1, 86, + -1, 87, -1, 88, -1, 90, -1, 91, -1, 92, + -1, 94, -1, 95, -1, 96, -1, 97, -1, 98, + -1, 99, -1, 102, -1, 103, -1, 104, -1, 105, + -1, 106, -1, 107, -1, 108, -1, 109, -1, 110, + -1, 111, -1, 113, -1, 114, -1, 116, -1, 118, + -1, 120, -1, 121, -1, 122, -1, 123, -1, 124, + -1, 125, -1, 128, -1, 129, -1, 130, -1, 131, + -1, 134, -1, 135, -1, 136, -1, 137, -1, 138, + -1, 140, -1, 141, -1, 142, -1, 144, -1, 145, + -1, 146, -1, 148, -1, 149, -1, 150, -1, 151, + -1, 153, -1, 154, -1, 155, -1, 156, -1, 157, + -1, 160, -1, 162, -1, 163, -1, 165, -1, 167, + -1, 169, -1, 173, -1, 174, -1, 177, -1, 179, + -1, 183, -1, 184, -1, 186, -1, 187, -1, 188, + -1, 189, -1, 190, -1, 191, -1, 192, -1, 194, + -1, 195, -1, 196, -1, 197, -1, 199, -1, 200, + -1, 201, -1, 202, -1, 203, -1, 204, -1, 205, + -1, 207, -1, 210, -1, 211, -1, 212, -1, 213, + -1, 214, -1, 220, -1, 223, -1, 225, -1, 226, + -1, 227, -1, 228, -1, 229, -1, 230, -1, 233, + -1, 235, -1, 238, -1, 239, -1, 240, -1, 241, + -1, 242, -1, 243, -1, 244, -1, 245, -1, 247, + -1, 248, -1, 249, -1, 250, -1, 251, -1, 252, + -1, 253, -1, 254, -1, 255, -1, 256, -1, 257, + -1, 258, -1, 259, -1, 260, -1, 261, -1, 262, + -1, 263, -1, 264, -1, 265, -1, 266, -1, 270, + -1, 271, -1, 272, -1, 275, -1, 276, -1, 278, + -1, 281, -1, 283, -1, 284, -1, 285, -1, 287, + -1, 288, -1, 291, -1, 292, -1, 293, -1, 296, + -1, 297, -1, 300, -1, 303, -1, 304, -1, 305, + -1, 306, -1, 307, -1, 308, -1, 309, -1, 310, + -1, 311, -1, 312, -1, 313, -1, 318, -1, 319, + -1, 322, -1, 323, -1, 325, -1, 326, -1, 327, + -1, 329, -1, 330, -1, 331, -1, 332, -1, 333, + -1, 334, -1, 336, -1, 337, -1, 338, -1, 340, + -1, 341, -1, 342, -1, 343, -1, 345, -1, 346, + -1, 347, -1, 348, -1, 349, -1, 350, -1, 351, + -1, 352, -1, 353, -1, 354, -1, 355, -1, 356, + -1, 357, -1, 359, -1, 360, -1, 362, -1, 363, + -1, 364, -1, 366, -1, 367, -1, 368, -1, 369, + -1, 370, -1, 371, -1, 372, -1, 373, -1, 374, + -1, 375, -1, 376, -1, 377, -1, 378, -1, 381, + -1, 382, -1, 383, -1, 384, -1, 385, -1, 386, + -1, 388, -1, 389, -1, 392, -1, 393, -1, 395, + -1, 397, -1, 398, -1, 399, -1, 400, -1, 401, + -1, 402, -1, 403, -1, 404, -1, 405, -1, 406, + -1, 407, -1, 408, -1, 410, -1, 414, -1, 415, + -1, 417, -1, 419, -1, 420, -1, 421, -1, 422, + -1, 423, -1, 425, -1, 430, -1, 431, -1, 433, + -1, 436, -1, 437, -1, 439, -1, 440, -1, 441, + -1, 442, -1, 443, -1, 446, -1, 447, -1, 448, + -1, 450, -1, 451, -1, 452, -1, 453, -1, 455, + -1, 456, -1, 457, -1, 458, -1, 459, -1, 463, + -1, 465, -1, 466, -1, 467, -1, 468, -1, 469, + -1, 470, -1, 471, -1, 474, -1, 477, -1, 478, + -1, 479, -1, 480, -1, 481, -1, 482, -1, 494, + -1, 495, -1, 496, -1, 497, -1, 53, -1, 54, + -1, 56, -1, 57, -1, 71, -1, 72, -1, 79, + -1, 83, -1, 112, -1, 115, -1, 152, -1, 158, + -1, 164, -1, 175, -1, 181, -1, 182, -1, 209, + -1, 215, -1, 216, -1, 218, -1, 246, -1, 267, + -1, 269, -1, 273, -1, 280, -1, 282, -1, 298, + -1, 302, -1, 320, -1, 324, -1, 339, -1, 365, + -1, 387, -1, 394, -1, 409, -1, 411, -1, 426, + -1, 427, -1, 432, -1, 434, -1, 438, -1, 460, + -1, 461, -1, 483, -1, 484, -1, 485, -1, 486, + -1, 487, -1, 488, -1, 489, -1, 490, -1, 491, + -1, 492, -1, 493, -1, 42, -1, 49, -1, 55, + -1, 81, -1, 89, -1, 101, -1, 170, -1, 172, + -1, 175, -1, 176, -1, 193, -1, 208, -1, 221, + -1, 222, -1, 224, -1, 234, -1, 236, -1, 246, + -1, 268, -1, 277, -1, 299, -1, 301, -1, 321, + -1, 361, -1, 391, -1, 409, -1, 418, -1, 464, + -1, 37, -1, 42, -1, 49, -1, 55, -1, 81, + -1, 83, -1, 89, -1, 101, -1, 170, -1, 172, + -1, 176, -1, 193, -1, 208, -1, 221, -1, 222, + -1, 224, -1, 234, -1, 236, -1, 268, -1, 277, + -1, 299, -1, 301, -1, 321, -1, 361, -1, 380, + -1, 391, -1, 418, -1, 438, -1, 464, -1, 37, + -1, 42, -1, 49, -1, 53, -1, 54, -1, 55, + -1, 56, -1, 57, -1, 72, -1, 71, -1, 79, + -1, 81, -1, 83, -1, 89, -1, 101, -1, 112, + -1, 115, -1, 152, -1, 158, -1, 164, -1, 170, + -1, 172, -1, 175, -1, 176, -1, 181, -1, 182, + -1, 193, -1, 208, -1, 209, -1, 216, -1, 218, + -1, 215, -1, 221, -1, 222, -1, 224, -1, 234, + -1, 236, -1, 246, -1, 267, -1, 268, -1, 269, + -1, 273, -1, 277, -1, 280, -1, 282, -1, 299, + -1, 298, -1, 301, -1, 302, -1, 320, -1, 321, + -1, 324, -1, 339, -1, 361, -1, 365, -1, 380, + -1, 387, -1, 391, -1, 394, -1, 409, -1, 411, + -1, 418, -1, 426, -1, 427, -1, 432, -1, 434, + -1, 438, -1, 460, -1, 461, -1, 464, -1, 483, -1, 484, -1, 485, -1, 486, -1, 487, -1, 488, -1, 489, -1, 490, -1, 491, -1, 492, -1, 493, - -1, 42, -1, 49, -1, 55, -1, 81, -1, 89, - -1, 101, -1, 170, -1, 172, -1, 175, -1, 176, - -1, 193, -1, 208, -1, 221, -1, 222, -1, 224, - -1, 234, -1, 236, -1, 246, -1, 268, -1, 277, - -1, 299, -1, 301, -1, 321, -1, 361, -1, 391, - -1, 409, -1, 418, -1, 464, -1, 37, -1, 42, - -1, 49, -1, 55, -1, 81, -1, 83, -1, 89, - -1, 101, -1, 170, -1, 172, -1, 176, -1, 193, - -1, 208, -1, 221, -1, 222, -1, 224, -1, 234, - -1, 236, -1, 268, -1, 277, -1, 299, -1, 301, - -1, 321, -1, 361, -1, 380, -1, 391, -1, 418, - -1, 438, -1, 464, -1, 37, -1, 42, -1, 49, - -1, 53, -1, 54, -1, 55, -1, 56, -1, 57, - -1, 72, -1, 71, -1, 79, -1, 81, -1, 83, - -1, 89, -1, 101, -1, 112, -1, 115, -1, 152, - -1, 158, -1, 164, -1, 170, -1, 172, -1, 175, - -1, 176, -1, 181, -1, 182, -1, 193, -1, 208, - -1, 209, -1, 216, -1, 218, -1, 215, -1, 221, + -1, 37, -1, 42, -1, 49, -1, 55, -1, 81, + -1, 83, -1, 89, -1, 101, -1, 170, -1, 172, + -1, 175, -1, 176, -1, 193, -1, 208, -1, 221, -1, 222, -1, 224, -1, 234, -1, 236, -1, 246, - -1, 267, -1, 268, -1, 269, -1, 273, -1, 277, - -1, 280, -1, 282, -1, 299, -1, 298, -1, 301, - -1, 302, -1, 320, -1, 321, -1, 324, -1, 339, - -1, 361, -1, 365, -1, 380, -1, 387, -1, 391, - -1, 394, -1, 409, -1, 411, -1, 418, -1, 426, - -1, 427, -1, 432, -1, 434, -1, 438, -1, 460, - -1, 461, -1, 464, -1, 483, -1, 484, -1, 485, - -1, 486, -1, 487, -1, 488, -1, 489, -1, 490, - -1, 491, -1, 492, -1, 493, -1, 37, -1, 42, - -1, 49, -1, 55, -1, 81, -1, 83, -1, 89, - -1, 101, -1, 170, -1, 172, -1, 175, -1, 176, - -1, 193, -1, 208, -1, 221, -1, 222, -1, 224, - -1, 234, -1, 236, -1, 246, -1, 268, -1, 277, - -1, 299, -1, 301, -1, 321, -1, 361, -1, 380, - -1, 391, -1, 409, -1, 418, -1, 438, -1, 464, - -1, 30, -1, 34, -1, 35, -1, 36, -1, 38, - -1, 39, -1, 40, -1, 41, -1, 45, -1, 58, - -1, 65, -1, 66, -1, 74, -1, 80, -1, 82, - -1, 93, -1, 100, -1, 117, -1, 119, -1, 126, - -1, 127, -1, 132, -1, 133, -1, 139, -1, 143, - -1, 147, -1, 159, -1, 161, -1, 166, -1, 168, - -1, 171, -1, 178, -1, 180, -1, 185, -1, 198, - -1, 206, -1, 217, -1, 219, -1, 231, -1, 232, - -1, 237, -1, 274, -1, 279, -1, 286, -1, 289, - -1, 290, -1, 294, -1, 295, -1, 314, -1, 315, - -1, 316, -1, 317, -1, 328, -1, 335, -1, 344, - -1, 358, -1, 379, -1, 390, -1, 396, -1, 412, - -1, 413, -1, 416, -1, 424, -1, 428, -1, 429, - -1, 435, -1, 444, -1, 445, -1, 449, -1, 454, - -1, 462, -1, 472, -1, 473, -1, 475, -1, 476, - -1 + -1, 268, -1, 277, -1, 299, -1, 301, -1, 321, + -1, 361, -1, 380, -1, 391, -1, 409, -1, 418, + -1, 438, -1, 464, -1, 30, -1, 34, -1, 35, + -1, 36, -1, 38, -1, 39, -1, 40, -1, 41, + -1, 45, -1, 58, -1, 65, -1, 66, -1, 74, + -1, 80, -1, 82, -1, 93, -1, 100, -1, 117, + -1, 119, -1, 126, -1, 127, -1, 132, -1, 133, + -1, 139, -1, 143, -1, 147, -1, 159, -1, 161, + -1, 166, -1, 168, -1, 171, -1, 178, -1, 180, + -1, 185, -1, 198, -1, 206, -1, 217, -1, 219, + -1, 231, -1, 232, -1, 237, -1, 274, -1, 279, + -1, 286, -1, 289, -1, 290, -1, 294, -1, 295, + -1, 314, -1, 315, -1, 316, -1, 317, -1, 328, + -1, 335, -1, 344, -1, 358, -1, 379, -1, 390, + -1, 396, -1, 412, -1, 413, -1, 416, -1, 424, + -1, 428, -1, 429, -1, 435, -1, 444, -1, 445, + -1, 449, -1, 454, -1, 462, -1, 472, -1, 473, + -1, 475, -1, 476, -1 }; /* YYRLINE[YYN] -- source line where rule number YYN was defined. */ @@ -2571,110 +2572,110 @@ static const yytype_uint16 yyrline[] = 119, 120, 8, 20, 33, 46, 58, 70, 86, 87, 91, 95, 7, 1, 30, 49, 61, 62, 63, 67, 68, 73, 77, 82, 86, 94, 95, 99, 100, 105, - 106, 110, 111, 116, 117, 118, 119, 120, 121, 126, - 134, 138, 143, 144, 149, 153, 158, 162, 166, 170, - 174, 178, 182, 186, 190, 194, 198, 202, 206, 210, - 214, 218, 226, 231, 232, 233, 234, 235, 241, 245, - 47, 48, 52, 53, 54, 72, 73, 80, 88, 96, - 104, 112, 120, 131, 132, 159, 164, 172, 188, 205, - 222, 239, 240, 259, 263, 267, 271, 275, 285, 296, - 306, 315, 326, 337, 349, 364, 382, 382, 386, 386, - 390, 390, 394, 400, 407, 411, 412, 416, 417, 431, - 438, 445, 455, 456, 459, 472, 473, 474, 478, 489, - 497, 502, 507, 512, 517, 525, 533, 538, 543, 550, - 551, 555, 556, 557, 561, 568, 569, 573, 574, 578, - 579, 580, 584, 585, 589, 590, 606, 607, 610, 619, - 630, 631, 632, 635, 636, 637, 641, 642, 643, 644, - 648, 649, 653, 655, 671, 673, 678, 681, 689, 693, - 697, 701, 705, 709, 716, 721, 728, 729, 733, 738, - 742, 746, 754, 761, 762, 767, 768, 772, 773, 778, - 780, 782, 787, 807, 808, 810, 815, 816, 820, 821, - 824, 825, 850, 851, 856, 860, 861, 865, 866, 870, - 871, 872, 873, 874, 878, 891, 898, 905, 912, 913, - 917, 918, 922, 923, 927, 928, 932, 933, 937, 938, - 942, 953, 954, 955, 956, 960, 961, 966, 967, 968, - 977, 983, 992, 993, 1006, 1007, 1011, 1012, 1016, 1017, - 1023, 1029, 1037, 1046, 1054, 1063, 1072, 1076, 1081, 1092, - 1106, 1107, 1110, 1111, 1112, 1115, 1123, 1132, 1133, 1134, - 1135, 1138, 1146, 1155, 1159, 1166, 1167, 1171, 1180, 1184, - 1209, 1213, 1226, 1240, 1255, 1267, 1280, 1294, 1308, 1321, - 1336, 1355, 1361, 1366, 1372, 1379, 1380, 1388, 1392, 1396, - 1402, 1409, 1414, 1415, 1416, 1417, 1418, 1419, 1423, 1424, - 1436, 1437, 1442, 1449, 1456, 1463, 1495, 1506, 1519, 1524, - 1525, 1528, 1529, 1532, 1533, 1538, 1539, 1544, 1548, 1554, - 1575, 1583, 1596, 1599, 1603, 1603, 1606, 1607, 1609, 1614, - 1621, 1626, 1632, 1637, 1643, 1647, 1654, 1661, 1671, 1672, - 1676, 1678, 1681, 1685, 1686, 1687, 1688, 1689, 1690, 1695, - 1715, 1716, 1717, 1718, 1729, 1743, 1744, 1750, 1755, 1760, - 1765, 1770, 1775, 1780, 1785, 1791, 1797, 1803, 1810, 1832, - 1841, 1845, 1853, 1857, 1865, 1877, 1898, 1902, 1908, 1912, - 1925, 1933, 1943, 1945, 1947, 1949, 1951, 1953, 1958, 1959, - 1966, 1975, 1983, 1992, 2003, 2011, 2012, 2013, 2017, 2017, - 2020, 2020, 2023, 2023, 2026, 2026, 2029, 2029, 2032, 2032, - 2035, 2035, 2038, 2038, 2041, 2041, 2044, 2044, 2047, 2047, - 2050, 2050, 2053, 2055, 2057, 2059, 2061, 2063, 2065, 2067, - 2069, 2071, 2073, 2075, 2077, 2082, 2087, 2093, 2100, 2105, - 2111, 2117, 2148, 2150, 2152, 2160, 2175, 2177, 2179, 2181, - 2183, 2185, 2187, 2189, 2191, 2193, 2195, 2197, 2199, 2201, - 2203, 2205, 2208, 2210, 2212, 2215, 2217, 2219, 2221, 2223, - 2228, 2233, 2240, 2245, 2252, 2257, 2264, 2269, 2277, 2285, - 2293, 2301, 2319, 2327, 2335, 2343, 2351, 2359, 2367, 2371, - 2387, 2395, 2403, 2411, 2419, 2427, 2435, 2439, 2443, 2447, - 2451, 2459, 2467, 2475, 2483, 2503, 2525, 2536, 2543, 2557, - 2565, 2573, 2593, 2595, 2597, 2599, 2601, 2603, 2605, 2607, - 2609, 2611, 2613, 2615, 2617, 2619, 2621, 2623, 2625, 2627, - 2629, 2631, 2633, 2635, 2639, 2643, 2647, 2661, 2662, 2676, - 2677, 2678, 2689, 2713, 2724, 2734, 2738, 2742, 2749, 2753, - 2760, 2764, 2781, 2785, 2787, 2791, 2794, 2805, 2810, 2817, - 2825, 2834, 2838, 2845, 2853, 2861, 2872, 2892, 2928, 2939, - 2940, 2947, 2953, 2955, 2957, 2961, 2970, 2975, 2982, 2997, - 3004, 3008, 3012, 3016, 3020, 3030, 3039, 3061, 3062, 3066, - 3067, 3068, 3072, 3073, 3080, 3081, 3085, 3086, 3091, 3099, - 3101, 3115, 3118, 3145, 3146, 3149, 3150, 3158, 3166, 3174, - 3183, 3193, 3211, 3257, 3266, 3275, 3284, 3293, 3305, 3306, - 3307, 3308, 3309, 3323, 3324, 3327, 3328, 3332, 3342, 3343, - 3347, 3348, 3352, 3359, 3360, 3365, 3366, 3371, 3372, 3375, - 3376, 3377, 3380, 3381, 3384, 3385, 3386, 3387, 3388, 3389, - 3390, 3391, 3392, 3393, 3394, 3395, 3396, 3397, 3400, 3402, - 3407, 3409, 3414, 3416, 3418, 3420, 3422, 3424, 3426, 3428, - 3442, 3444, 3449, 3453, 3460, 3465, 3471, 3475, 3482, 3487, - 3494, 3499, 3507, 3511, 3517, 3521, 3530, 3541, 3542, 3546, - 3550, 3557, 3558, 3559, 3560, 3561, 3562, 3563, 3564, 3565, - 3566, 3567, 3568, 3569, 3570, 3580, 3584, 3591, 3598, 3599, - 3615, 3619, 3624, 3628, 3643, 3648, 3652, 3655, 3658, 3659, - 3660, 3663, 3670, 3680, 3694, 3695, 3699, 3710, 3711, 3714, - 3715, 3718, 3722, 3729, 3737, 3745, 3753, 3763, 3764, 3769, - 3770, 3774, 3775, 3776, 3780, 3789, 3797, 3805, 3814, 3829, - 3830, 3835, 3836, 3846, 3847, 3851, 3852, 3856, 3857, 3860, - 3876, 3884, 3894, 3895, 3898, 3899, 3902, 3906, 3907, 3911, - 3912, 3915, 3916, 3917, 3927, 3928, 3932, 3934, 3940, 3941, - 3945, 3946, 3949, 3960, 3963, 3974, 3978, 3982, 3994, 3998, - 4007, 4014, 4052, 4056, 4060, 4064, 4068, 4072, 4076, 4082, - 4099, 4100, 4101, 4104, 4105, 4106, 4109, 4110, 4111, 4114, - 4115, 4118, 4120, 4125, 4126, 4129, 4133, 4134, 7, 18, - 19, 23, 24, 25, 26, 27, 28, 7, 26, 50, - 73, 80, 85, 86, 87, 88, 8, 33, 62, 66, - 67, 72, 73, 78, 79, 83, 84, 89, 90, 7, - 16, 25, 34, 43, 52, 5, 12, 22, 23, 7, - 15, 26, 27, 30, 31, 32, 33, 34, 35, 36, - 37, 38, 39, 7, 19, 33, 9, 16, 26, 33, - 44, 45, 50, 51, 52, 57, 58, 59, 60, 61, - 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, - 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, - 82, 83, 84, 85, 86, 90, 91, 92, 97, 98, - 103, 107, 115, 116, 121, 122, 123, 129, 134, 142, - 143, 10, 16, 22, 28, 38, 39, 47, 58, 70, - 78, 89, 95, 99, 103, 118, 125, 126, 127, 131, - 132, 7, 15, 22, 29, 36, 45, 46, 48, 49, - 8, 22, 36, 48, 56, 70, 71, 72, 73, 74, - 87, 88, 93, 94, 98, 99, 7, 18, 31, 35, - 42, 53, 54, 60, 61, 9, 19, 7, 16, 28, - 35, 42, 51, 52, 56, 57, 2, 7, 12, 17, - 26, 33, 43, 44, 51, 3, 10, 17, 24, 31, - 38, 45, 52, 61, 61, 63, 63, 65, 65, 67, - 68, 72, 73, 6, 8, 21, 34, 47, 65, 87, - 88, 89, 90, 11, 24, 37, 54, 55, 56, 61, - 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, + 106, 110, 111, 116, 117, 118, 119, 120, 121, 122, + 127, 135, 139, 144, 145, 150, 154, 159, 163, 167, + 171, 175, 179, 183, 187, 191, 195, 199, 203, 207, + 211, 215, 219, 227, 232, 233, 234, 235, 236, 242, + 246, 47, 48, 52, 53, 54, 72, 73, 80, 88, + 96, 104, 112, 120, 131, 132, 159, 164, 172, 188, + 205, 222, 239, 240, 259, 263, 267, 271, 275, 285, + 296, 306, 315, 326, 337, 349, 364, 382, 382, 386, + 386, 390, 390, 394, 400, 407, 411, 412, 416, 417, + 431, 438, 445, 455, 456, 459, 472, 473, 474, 478, + 489, 497, 502, 507, 512, 517, 525, 533, 538, 543, + 550, 551, 555, 556, 557, 561, 568, 569, 573, 574, + 578, 579, 580, 584, 585, 589, 590, 606, 607, 610, + 619, 630, 631, 632, 635, 636, 637, 641, 642, 643, + 644, 648, 649, 653, 655, 671, 673, 678, 681, 689, + 693, 697, 701, 705, 709, 716, 721, 728, 729, 733, + 738, 742, 746, 754, 761, 762, 767, 768, 772, 773, + 778, 780, 782, 787, 807, 808, 810, 815, 816, 820, + 821, 824, 825, 850, 851, 856, 860, 861, 865, 866, + 870, 871, 872, 873, 874, 878, 891, 898, 905, 912, + 913, 917, 918, 922, 923, 927, 928, 932, 933, 937, + 938, 942, 953, 954, 955, 956, 960, 961, 966, 967, + 968, 977, 983, 992, 993, 1006, 1007, 1011, 1012, 1016, + 1017, 1023, 1029, 1037, 1046, 1054, 1063, 1072, 1076, 1081, + 1092, 1106, 1107, 1110, 1111, 1112, 1115, 1123, 1132, 1133, + 1134, 1135, 1138, 1146, 1155, 1159, 1166, 1167, 1171, 1180, + 1184, 1209, 1213, 1226, 1240, 1255, 1267, 1280, 1294, 1308, + 1321, 1336, 1355, 1361, 1366, 1372, 1379, 1380, 1388, 1392, + 1396, 1402, 1409, 1414, 1415, 1416, 1417, 1418, 1419, 1423, + 1424, 1436, 1437, 1442, 1449, 1456, 1463, 1495, 1506, 1519, + 1524, 1525, 1528, 1529, 1532, 1533, 1538, 1539, 1544, 1548, + 1554, 1575, 1583, 1596, 1599, 1603, 1603, 1606, 1607, 1609, + 1614, 1621, 1626, 1632, 1637, 1643, 1647, 1654, 1661, 1671, + 1672, 1676, 1678, 1681, 1685, 1686, 1687, 1688, 1689, 1690, + 1695, 1715, 1716, 1717, 1718, 1729, 1743, 1744, 1750, 1755, + 1760, 1765, 1770, 1775, 1780, 1785, 1791, 1797, 1803, 1810, + 1832, 1841, 1845, 1853, 1857, 1865, 1877, 1898, 1902, 1908, + 1912, 1925, 1933, 1943, 1945, 1947, 1949, 1951, 1953, 1958, + 1959, 1966, 1975, 1983, 1992, 2003, 2011, 2012, 2013, 2017, + 2017, 2020, 2020, 2023, 2023, 2026, 2026, 2029, 2029, 2032, + 2032, 2035, 2035, 2038, 2038, 2041, 2041, 2044, 2044, 2047, + 2047, 2050, 2050, 2053, 2055, 2057, 2059, 2061, 2063, 2065, + 2067, 2069, 2071, 2073, 2075, 2077, 2082, 2087, 2093, 2100, + 2105, 2111, 2117, 2148, 2150, 2152, 2160, 2175, 2177, 2179, + 2181, 2183, 2185, 2187, 2189, 2191, 2193, 2195, 2197, 2199, + 2201, 2203, 2205, 2208, 2210, 2212, 2215, 2217, 2219, 2221, + 2223, 2228, 2233, 2240, 2245, 2252, 2257, 2264, 2269, 2277, + 2285, 2293, 2301, 2319, 2327, 2335, 2343, 2351, 2359, 2367, + 2371, 2387, 2395, 2403, 2411, 2419, 2427, 2435, 2439, 2443, + 2447, 2451, 2459, 2467, 2475, 2483, 2503, 2525, 2536, 2543, + 2557, 2565, 2573, 2593, 2595, 2597, 2599, 2601, 2603, 2605, + 2607, 2609, 2611, 2613, 2615, 2617, 2619, 2621, 2623, 2625, + 2627, 2629, 2631, 2633, 2635, 2639, 2643, 2647, 2661, 2662, + 2676, 2677, 2678, 2689, 2713, 2724, 2734, 2738, 2742, 2749, + 2753, 2760, 2764, 2781, 2785, 2787, 2790, 2793, 2804, 2809, + 2816, 2822, 2828, 2837, 2841, 2848, 2856, 2864, 2875, 2895, + 2931, 2942, 2943, 2950, 2956, 2958, 2960, 2964, 2973, 2978, + 2985, 3000, 3007, 3011, 3015, 3019, 3023, 3033, 3042, 3064, + 3065, 3069, 3070, 3071, 3075, 3076, 3083, 3084, 3088, 3089, + 3094, 3102, 3104, 3118, 3121, 3148, 3149, 3152, 3153, 3161, + 3169, 3177, 3186, 3196, 3214, 3260, 3269, 3278, 3287, 3296, + 3308, 3309, 3310, 3311, 3312, 3326, 3327, 3330, 3331, 3335, + 3345, 3346, 3350, 3351, 3355, 3362, 3363, 3368, 3369, 3374, + 3375, 3378, 3379, 3380, 3383, 3384, 3387, 3388, 3389, 3390, + 3391, 3392, 3393, 3394, 3395, 3396, 3397, 3398, 3399, 3400, + 3403, 3405, 3410, 3412, 3417, 3419, 3421, 3423, 3425, 3427, + 3429, 3431, 3445, 3447, 3452, 3456, 3463, 3468, 3474, 3478, + 3485, 3490, 3497, 3502, 3510, 3514, 3520, 3524, 3533, 3544, + 3545, 3549, 3553, 3560, 3561, 3562, 3563, 3564, 3565, 3566, + 3567, 3568, 3569, 3570, 3571, 3572, 3573, 3583, 3587, 3594, + 3601, 3602, 3618, 3622, 3627, 3631, 3646, 3651, 3655, 3658, + 3661, 3662, 3663, 3666, 3673, 3683, 3697, 3698, 3702, 3713, + 3714, 3717, 3718, 3721, 3725, 3732, 3740, 3748, 3756, 3766, + 3767, 3772, 3773, 3777, 3778, 3779, 3783, 3792, 3800, 3808, + 3817, 3832, 3833, 3838, 3839, 3849, 3850, 3854, 3855, 3859, + 3860, 3863, 3879, 3887, 3897, 3898, 3901, 3902, 3905, 3909, + 3910, 3914, 3915, 3918, 3919, 3920, 3930, 3931, 3935, 3937, + 3943, 3944, 3948, 3949, 3952, 3963, 3966, 3977, 3981, 3985, + 3997, 4001, 4010, 4017, 4055, 4059, 4063, 4067, 4071, 4075, + 4079, 4085, 4102, 4103, 4104, 4107, 4108, 4109, 4112, 4113, + 4114, 4117, 4118, 4121, 4123, 4128, 4129, 4132, 4136, 4137, + 7, 18, 19, 23, 24, 25, 26, 27, 28, 7, + 26, 50, 73, 80, 85, 86, 87, 88, 8, 33, + 62, 66, 67, 72, 73, 78, 79, 83, 84, 89, + 90, 7, 16, 25, 34, 43, 52, 5, 12, 22, + 23, 7, 15, 26, 27, 30, 31, 32, 33, 34, + 35, 36, 37, 38, 39, 7, 19, 33, 9, 16, + 26, 33, 44, 45, 50, 51, 52, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, + 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, + 80, 81, 82, 83, 84, 85, 86, 90, 91, 92, + 97, 98, 103, 107, 115, 116, 121, 122, 123, 129, + 134, 142, 143, 10, 16, 22, 28, 38, 39, 47, + 58, 70, 78, 89, 95, 99, 103, 118, 125, 126, + 127, 131, 132, 7, 15, 22, 29, 36, 45, 46, + 48, 49, 8, 22, 36, 48, 56, 70, 71, 72, + 73, 74, 87, 88, 93, 94, 98, 99, 7, 18, + 31, 35, 42, 53, 54, 60, 61, 9, 19, 7, + 16, 28, 35, 42, 51, 52, 56, 57, 2, 7, + 12, 17, 26, 33, 43, 44, 51, 3, 10, 17, + 24, 31, 38, 45, 52, 61, 61, 63, 63, 65, + 65, 67, 68, 72, 73, 6, 8, 21, 34, 47, + 65, 87, 88, 89, 90, 11, 24, 37, 54, 55, + 56, 61, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, @@ -2706,18 +2707,18 @@ static const yytype_uint16 yyrline[] = 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, 74, + 74, 74, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, - 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, - 75, 75, 75, 75, 76, 76, 76, 76, 76, 76, + 75, 75, 75, 75, 75, 75, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, - 76, 76, 77, 77, 77, 77, 77, 77, 77, 77, + 76, 76, 76, 76, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, - 77, 78, 78, 78, 78, 78, 78, 78, 78, 78, + 77, 77, 77, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, @@ -2725,17 +2726,18 @@ static const yytype_uint16 yyrline[] = 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, - 78, 78, 79, 79, 79, 79, 79, 79, 79, 79, + 78, 78, 78, 78, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, - 79, 79, 79, 79, 80, 80, 80, 80, 80, 80, + 79, 79, 79, 79, 79, 79, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80 + 80 }; #endif @@ -2906,8 +2908,8 @@ static const char *const yytname[] = "millisecond_keyword", "microsecond_keyword", "week_keyword", "decade_keyword", "century_keyword", "millennium_keyword", "opt_interval", "a_expr", "b_expr", "c_expr", "d_expr", - "indirection_expr_or_a_expr", "indirection_expr", "struct_expr", - "func_application", "func_expr", "func_expr_windowless", + "indirection_expr_or_a_expr", "indirection_expr", "list_expr", + "struct_expr", "func_application", "func_expr", "func_expr_windowless", "func_expr_common_subexpr", "list_comprehension", "within_group_clause", "filter_clause", "export_clause", "window_clause", "window_definition_list", "window_definition", "over_clause", @@ -3069,171 +3071,172 @@ static const yytype_uint16 yyr1[] = 651, 651, 652, 652, 652, 652, 652, 652, 653, 653, 654, 654, 655, 656, 656, 656, 657, 657, 657, 658, 658, 659, 659, 660, 660, 661, 661, 662, 662, 663, - 663, 664, 664, 665, 665, 665, 665, 665, 665, 666, - 667, 667, 668, 668, 669, 669, 670, 670, 670, 670, + 663, 664, 664, 665, 665, 665, 665, 665, 665, 665, + 666, 667, 667, 668, 668, 669, 669, 670, 670, 670, 670, 670, 670, 670, 670, 670, 670, 670, 670, 670, - 670, 670, 671, 672, 672, 672, 672, 672, 673, 673, - 674, 674, 675, 675, 675, 676, 676, 676, 676, 676, - 676, 676, 676, 677, 677, 678, 678, 679, 679, 679, + 670, 670, 670, 671, 672, 672, 672, 672, 672, 673, + 673, 674, 674, 675, 675, 675, 676, 676, 676, 676, + 676, 676, 676, 676, 677, 677, 678, 678, 679, 679, 679, 679, 679, 679, 679, 679, 679, 679, 679, 679, - 679, 679, 679, 679, 679, 679, 680, 680, 681, 681, - 682, 682, 683, 683, 683, 684, 684, 685, 685, 686, - 686, 686, 687, 687, 688, 689, 689, 689, 690, 690, - 691, 691, 691, 691, 691, 691, 691, 691, 691, 692, - 692, 693, 693, 693, 694, 695, 695, 696, 696, 697, - 697, 697, 698, 698, 699, 699, 700, 700, 701, 701, - 702, 702, 702, 703, 703, 703, 704, 704, 704, 704, - 705, 705, 706, 706, 706, 706, 707, 707, 708, 708, - 708, 708, 708, 708, 709, 709, 710, 710, 711, 711, - 711, 711, 712, 713, 713, 714, 714, 715, 715, 715, - 715, 715, 716, 717, 717, 717, 718, 718, 719, 719, - 720, 720, 721, 721, 721, 722, 722, 723, 723, 724, - 724, 724, 724, 724, 725, 726, 727, 728, 729, 729, - 730, 730, 731, 731, 732, 732, 733, 733, 734, 734, - 735, 736, 736, 736, 736, 737, 737, 738, 738, 738, - 739, 739, 740, 740, 741, 741, 742, 742, 743, 743, - 744, 744, 744, 744, 744, 744, 744, 744, 744, 744, - 745, 745, 746, 746, 746, 747, 747, 748, 748, 748, - 748, 749, 749, 750, 750, 751, 751, 752, 753, 753, - 754, 754, 754, 754, 754, 754, 754, 754, 754, 754, - 754, 755, 755, 755, 755, 756, 756, 757, 757, 757, - 757, 757, 758, 758, 758, 758, 758, 758, 759, 759, - 760, 760, 761, 761, 761, 761, 762, 762, 763, 764, - 764, 765, 765, 766, 766, 767, 767, 768, 768, 769, - 770, 770, 771, 771, 772, 772, 773, 773, 774, 774, - 774, 774, 774, 774, 774, 774, 774, 774, 775, 775, - 776, 776, 776, 777, 777, 777, 777, 777, 777, 777, - 778, 778, 778, 778, 779, 780, 780, 781, 781, 781, - 781, 781, 781, 781, 781, 781, 781, 781, 782, 782, - 783, 783, 784, 784, 785, 786, 787, 787, 788, 788, - 789, 790, 791, 791, 791, 791, 791, 791, 792, 792, - 793, 793, 793, 793, 794, 795, 795, 795, 796, 796, - 797, 797, 798, 798, 799, 799, 800, 800, 801, 801, - 802, 802, 803, 803, 804, 804, 805, 805, 806, 806, - 807, 807, 808, 808, 808, 808, 808, 808, 808, 808, + 679, 679, 679, 679, 679, 679, 679, 680, 680, 681, + 681, 682, 682, 683, 683, 683, 684, 684, 685, 685, + 686, 686, 686, 687, 687, 688, 689, 689, 689, 690, + 690, 691, 691, 691, 691, 691, 691, 691, 691, 691, + 692, 692, 693, 693, 693, 694, 695, 695, 696, 696, + 697, 697, 697, 698, 698, 699, 699, 700, 700, 701, + 701, 702, 702, 702, 703, 703, 703, 704, 704, 704, + 704, 705, 705, 706, 706, 706, 706, 707, 707, 708, + 708, 708, 708, 708, 708, 709, 709, 710, 710, 711, + 711, 711, 711, 712, 713, 713, 714, 714, 715, 715, + 715, 715, 715, 716, 717, 717, 717, 718, 718, 719, + 719, 720, 720, 721, 721, 721, 722, 722, 723, 723, + 724, 724, 724, 724, 724, 725, 726, 727, 728, 729, + 729, 730, 730, 731, 731, 732, 732, 733, 733, 734, + 734, 735, 736, 736, 736, 736, 737, 737, 738, 738, + 738, 739, 739, 740, 740, 741, 741, 742, 742, 743, + 743, 744, 744, 744, 744, 744, 744, 744, 744, 744, + 744, 745, 745, 746, 746, 746, 747, 747, 748, 748, + 748, 748, 749, 749, 750, 750, 751, 751, 752, 753, + 753, 754, 754, 754, 754, 754, 754, 754, 754, 754, + 754, 754, 755, 755, 755, 755, 756, 756, 757, 757, + 757, 757, 757, 758, 758, 758, 758, 758, 758, 759, + 759, 760, 760, 761, 761, 761, 761, 762, 762, 763, + 764, 764, 765, 765, 766, 766, 767, 767, 768, 768, + 769, 770, 770, 771, 771, 772, 772, 773, 773, 774, + 774, 774, 774, 774, 774, 774, 774, 774, 774, 775, + 775, 776, 776, 776, 777, 777, 777, 777, 777, 777, + 777, 778, 778, 778, 778, 779, 780, 780, 781, 781, + 781, 781, 781, 781, 781, 781, 781, 781, 781, 782, + 782, 783, 783, 784, 784, 785, 786, 787, 787, 788, + 788, 789, 790, 791, 791, 791, 791, 791, 791, 792, + 792, 793, 793, 793, 793, 794, 795, 795, 795, 796, + 796, 797, 797, 798, 798, 799, 799, 800, 800, 801, + 801, 802, 802, 803, 803, 804, 804, 805, 805, 806, + 806, 807, 807, 808, 808, 808, 808, 808, 808, 808, 808, 808, 808, 808, 808, 808, 808, 808, 808, 808, - 808, 808, 809, 809, 809, 809, 809, 809, 809, 809, + 808, 808, 808, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, 809, - 809, 809, 810, 810, 810, 810, 810, 810, 810, 810, + 809, 809, 809, 810, 810, 810, 810, 810, 810, 810, 810, 810, 810, 810, 810, 810, 810, 810, 810, 810, - 810, 810, 810, 810, 810, 810, 810, 811, 811, 812, - 812, 812, 812, 812, 812, 813, 813, 813, 814, 814, + 810, 810, 810, 810, 810, 810, 810, 810, 811, 811, + 812, 812, 812, 812, 812, 812, 813, 813, 813, 814, 814, 814, 814, 814, 814, 814, 814, 814, 814, 814, - 815, 816, 816, 816, 816, 816, 816, 817, 817, 818, - 818, 819, 819, 819, 819, 819, 819, 819, 819, 819, - 819, 819, 819, 819, 819, 820, 820, 821, 821, 822, - 822, 822, 823, 823, 824, 824, 825, 825, 826, 827, - 827, 827, 828, 829, 829, 830, 830, 831, 831, 831, - 831, 832, 832, 833, 833, 833, 833, 833, 834, 834, - 834, 834, 834, 835, 835, 836, 836, 837, 838, 838, - 839, 839, 840, 841, 841, 842, 842, 843, 843, 844, - 844, 844, 845, 845, 846, 846, 846, 846, 846, 846, - 846, 846, 846, 846, 846, 846, 846, 846, 847, 847, - 848, 848, 849, 849, 849, 849, 849, 849, 849, 849, + 814, 815, 816, 817, 817, 817, 817, 817, 817, 818, + 818, 819, 819, 820, 820, 820, 820, 820, 820, 820, + 820, 820, 820, 820, 820, 820, 820, 821, 821, 822, + 822, 823, 823, 823, 824, 824, 825, 825, 826, 826, + 827, 828, 828, 828, 829, 830, 830, 831, 831, 832, + 832, 832, 832, 833, 833, 834, 834, 834, 834, 834, + 835, 835, 835, 835, 835, 836, 836, 837, 837, 838, + 839, 839, 840, 840, 841, 842, 842, 843, 843, 844, + 844, 845, 845, 845, 846, 846, 847, 847, 847, 847, + 847, 847, 847, 847, 847, 847, 847, 847, 847, 847, + 848, 848, 849, 849, 850, 850, 850, 850, 850, 850, 850, 850, 851, 851, 852, 852, 853, 853, 854, 854, - 855, 855, 856, 856, 857, 857, 857, 858, 858, 859, - 859, 860, 860, 860, 860, 860, 860, 860, 860, 860, - 860, 860, 860, 860, 860, 861, 861, 862, 863, 863, - 864, 864, 864, 864, 864, 864, 865, 866, 867, 867, - 867, 868, 868, 869, 870, 870, 871, 872, 872, 873, - 873, 874, 874, 546, 546, 546, 546, 875, 875, 876, - 876, 877, 877, 877, 878, 878, 878, 878, 878, 879, + 855, 855, 856, 856, 857, 857, 858, 858, 858, 859, + 859, 860, 860, 861, 861, 861, 861, 861, 861, 861, + 861, 861, 861, 861, 861, 861, 861, 862, 862, 863, + 864, 864, 865, 865, 865, 865, 865, 865, 866, 867, + 868, 868, 868, 869, 869, 870, 871, 871, 872, 873, + 873, 874, 874, 875, 875, 546, 546, 546, 546, 876, + 876, 877, 877, 878, 878, 878, 879, 879, 879, 879, 879, 880, 880, 881, 881, 882, 882, 883, 883, 884, - 884, 884, 885, 885, 886, 886, 887, 888, 888, 889, - 889, 890, 890, 890, 891, 891, 892, 892, 893, 893, - 894, 894, 895, 896, 896, 897, 897, 897, 897, 897, - 897, 897, 897, 897, 897, 897, 897, 897, 897, 898, - 899, 899, 899, 900, 900, 900, 901, 901, 901, 902, - 902, 903, 903, 904, 904, 905, 906, 906, 907, 908, - 908, 909, 909, 909, 909, 909, 909, 910, 910, 910, - 911, 911, 912, 912, 912, 912, 913, 913, 914, 915, + 884, 885, 885, 885, 886, 886, 887, 887, 888, 889, + 889, 890, 890, 891, 891, 891, 892, 892, 893, 893, + 894, 894, 895, 895, 896, 897, 897, 898, 898, 898, + 898, 898, 898, 898, 898, 898, 898, 898, 898, 898, + 898, 899, 900, 900, 900, 901, 901, 901, 902, 902, + 902, 903, 903, 904, 904, 905, 905, 906, 907, 907, + 908, 909, 909, 910, 910, 910, 910, 910, 910, 911, + 911, 911, 912, 912, 913, 913, 913, 913, 914, 914, 915, 916, 916, 917, 917, 918, 918, 919, 919, 920, - 920, 920, 920, 920, 920, 921, 921, 922, 922, 923, - 923, 924, 924, 925, 925, 925, 925, 925, 925, 925, - 925, 925, 925, 926, 926, 927, 928, 928, 928, 928, - 929, 929, 930, 930, 930, 931, 931, 931, 931, 931, - 931, 931, 931, 931, 931, 931, 931, 931, 931, 931, - 931, 931, 931, 931, 931, 931, 931, 931, 931, 931, - 931, 931, 931, 931, 931, 932, 932, 932, 933, 933, - 934, 934, 935, 935, 936, 936, 936, 936, 937, 938, - 938, 939, 939, 939, 939, 940, 940, 940, 940, 941, - 941, 942, 943, 943, 943, 943, 943, 943, 943, 944, - 944, 945, 945, 945, 945, 945, 946, 946, 947, 947, - 948, 948, 948, 948, 948, 949, 949, 949, 949, 949, + 920, 921, 921, 921, 921, 921, 921, 922, 922, 923, + 923, 924, 924, 925, 925, 926, 926, 926, 926, 926, + 926, 926, 926, 926, 926, 927, 927, 928, 929, 929, + 929, 929, 930, 930, 931, 931, 931, 932, 932, 932, + 932, 932, 932, 932, 932, 932, 932, 932, 932, 932, + 932, 932, 932, 932, 932, 932, 932, 932, 932, 932, + 932, 932, 932, 932, 932, 932, 932, 933, 933, 933, + 934, 934, 935, 935, 936, 936, 937, 937, 937, 937, + 938, 939, 939, 940, 940, 940, 940, 941, 941, 941, + 941, 942, 942, 943, 944, 944, 944, 944, 944, 944, + 944, 945, 945, 946, 946, 946, 946, 946, 947, 947, + 948, 948, 949, 949, 949, 949, 949, 950, 950, 950, 950, 950, 951, 951, 952, 952, 953, 953, 954, 954, - 954, 955, 955, 956, 956, 957, 957, 958, 958, 959, - 959, 959, 960, 960, 961, 961, 962, 962, 962, 962, - 963, 963, 964, 964, 964, 965, 965, 965, 965, 965, - 965, 965, 965, 966, 966, 967, 967, 968, 968, 969, - 969, 970, 970, 971, 972, 972, 972, 972, 972, 973, - 973, 973, 973, 974, 974, 974, 975, 975, 975, 976, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, - 977, 977, 977, 977, 977, 977, 977, 977, 977, 977, + 955, 955, 955, 956, 956, 957, 957, 958, 958, 959, + 959, 960, 960, 960, 961, 961, 962, 962, 963, 963, + 963, 963, 964, 964, 965, 965, 965, 966, 966, 966, + 966, 966, 966, 966, 966, 967, 967, 968, 968, 969, + 969, 970, 970, 971, 971, 972, 973, 973, 973, 973, + 973, 974, 974, 974, 974, 975, 975, 975, 976, 976, + 976, 977, 978, 978, 978, 978, 978, 978, 978, 978, + 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, + 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, + 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, + 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, + 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, + 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, + 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, + 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, + 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, + 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, + 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, + 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, + 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, + 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, + 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, + 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, + 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, + 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, + 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, + 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, + 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, + 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, - 978, 978, 978, 978, 979, 979, 979, 979, 979, 979, + 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, + 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, + 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, + 978, 978, 978, 978, 978, 978, 978, 978, 978, 978, + 978, 978, 979, 979, 979, 979, 979, 979, 979, 979, 979, 979, 979, 979, 979, 979, 979, 979, 979, 979, 979, 979, 979, 979, 979, 979, 979, 979, 979, 979, - 979, 979, 980, 980, 980, 980, 980, 980, 980, 980, + 979, 979, 979, 979, 979, 979, 979, 979, 979, 979, + 979, 979, 979, 979, 979, 979, 979, 979, 979, 979, + 979, 979, 979, 979, 979, 979, 980, 980, 980, 980, 980, 980, 980, 980, 980, 980, 980, 980, 980, 980, 980, 980, 980, 980, 980, 980, 980, 980, 980, 980, - 980, 981, 981, 981, 981, 981, 981, 981, 981, 981, - 981, 981, 981, 981, 981, 981, 981, 981, 981, 981, - 981, 981, 981, 981, 981, 981, 981, 981, 981, 981, + 980, 980, 980, 980, 981, 981, 981, 981, 981, 981, 981, 981, 981, 981, 981, 981, 981, 981, 981, 981, 981, 981, 981, 981, 981, 981, 981, 981, 981, 981, - 981, 981, 981, 981, 981, 981, 981, 981, 981, 981, - 981, 981, 981, 981, 981, 981, 981, 981, 981, 981, - 981, 981, 981, 981, 981, 981, 981, 981, 981, 981, - 981, 981, 982, 982, 982, 982, 982, 982, 982, 982, + 981, 981, 981, 982, 982, 982, 982, 982, 982, 982, + 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, + 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, + 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, + 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, + 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, 983, 983, 983, 983, 983, 983, 983, 983, 983, 983, 983, 983, 983, 983, 983, 983, 983, 983, 983, 983, 983, 983, 983, 983, 983, 983, - 983, 983, 983, 983, 983, 983, 983, 983, 983, 983, - 983, 983, 983, 983, 983, 983, 983, 983, 983, 983, - 983, 983, 983, 983, 983, 983, 983, 983, 983, 983, - 983, 983, 983, 983, 983, 983, 983, 983, 983, 983, - 983, 983, 983, 983, 983, 983, 983, 983, 983 + 983, 983, 983, 983, 983, 983, 984, 984, 984, 984, + 984, 984, 984, 984, 984, 984, 984, 984, 984, 984, + 984, 984, 984, 984, 984, 984, 984, 984, 984, 984, + 984, 984, 984, 984, 984, 984, 984, 984, 984, 984, + 984, 984, 984, 984, 984, 984, 984, 984, 984, 984, + 984, 984, 984, 984, 984, 984, 984, 984, 984, 984, + 984, 984, 984, 984, 984, 984, 984, 984, 984, 984, + 984, 984, 984, 984, 984, 984, 984, 984, 984, 984, + 984 }; /* YYR2[YYN] -- Number of symbols composing right hand side of rule YYN. */ @@ -3286,109 +3289,110 @@ static const yytype_uint8 yyr2[] = 1, 1, 8, 11, 10, 7, 10, 9, 1, 1, 2, 3, 8, 11, 9, 7, 0, 3, 3, 1, 1, 3, 0, 1, 3, 1, 0, 1, 0, 1, - 0, 1, 3, 1, 1, 1, 3, 1, 0, 2, - 2, 0, 2, 0, 1, 0, 1, 1, 1, 3, - 3, 1, 1, 3, 3, 3, 3, 3, 3, 4, - 3, 2, 1, 1, 1, 1, 3, 1, 1, 3, - 1, 1, 3, 3, 3, 1, 2, 4, 4, 2, - 3, 5, 5, 1, 1, 3, 0, 11, 11, 10, - 12, 1, 2, 5, 4, 4, 4, 4, 7, 5, - 4, 7, 6, 9, 9, 4, 1, 1, 1, 1, - 1, 1, 1, 5, 1, 1, 3, 1, 2, 2, - 2, 3, 1, 3, 7, 1, 2, 0, 2, 0, - 3, 3, 4, 4, 4, 4, 3, 2, 1, 1, - 0, 1, 1, 0, 2, 1, 5, 1, 0, 2, - 2, 0, 1, 0, 3, 5, 1, 3, 4, 3, - 1, 1, 0, 2, 2, 0, 2, 2, 1, 1, - 1, 0, 2, 4, 5, 4, 2, 3, 2, 2, - 2, 2, 1, 2, 3, 0, 1, 0, 5, 1, - 4, 6, 2, 1, 0, 4, 0, 1, 1, 2, - 2, 2, 1, 1, 2, 2, 1, 1, 1, 1, - 1, 1, 3, 3, 0, 1, 3, 1, 2, 1, - 1, 1, 1, 1, 2, 4, 4, 5, 1, 1, - 2, 0, 2, 0, 1, 3, 1, 0, 1, 2, - 3, 2, 4, 2, 3, 2, 0, 1, 2, 0, - 4, 5, 1, 2, 2, 0, 1, 3, 1, 2, - 3, 3, 3, 3, 3, 3, 1, 4, 9, 9, - 3, 0, 2, 2, 0, 5, 3, 0, 1, 1, - 3, 5, 3, 1, 2, 1, 3, 5, 1, 2, - 3, 4, 5, 4, 5, 4, 6, 5, 4, 5, - 5, 5, 2, 4, 1, 1, 0, 1, 4, 5, - 4, 0, 2, 2, 2, 1, 1, 1, 1, 0, - 4, 2, 1, 2, 2, 4, 2, 6, 2, 1, - 3, 4, 0, 2, 0, 2, 0, 1, 3, 3, - 2, 0, 2, 4, 1, 1, 1, 0, 2, 3, - 5, 6, 2, 3, 1, 5, 5, 5, 3, 3, - 3, 4, 0, 1, 1, 1, 1, 1, 2, 4, - 1, 1, 1, 1, 2, 3, 0, 1, 1, 1, - 1, 1, 2, 2, 2, 2, 2, 1, 3, 0, - 1, 1, 1, 1, 5, 2, 1, 1, 1, 1, - 4, 1, 2, 2, 1, 3, 3, 2, 1, 0, - 5, 2, 5, 2, 1, 3, 3, 0, 1, 1, + 0, 1, 3, 1, 1, 1, 1, 3, 1, 0, + 2, 2, 0, 2, 0, 1, 0, 1, 1, 1, + 3, 3, 1, 1, 3, 3, 3, 3, 3, 3, + 4, 3, 2, 1, 1, 1, 1, 3, 1, 1, + 3, 1, 1, 3, 3, 3, 1, 2, 4, 4, + 2, 3, 5, 5, 1, 1, 3, 0, 11, 11, + 10, 12, 1, 2, 5, 4, 4, 4, 4, 7, + 5, 4, 7, 6, 9, 9, 4, 1, 1, 1, + 1, 1, 1, 1, 5, 1, 1, 3, 1, 2, + 2, 2, 3, 1, 3, 7, 1, 2, 0, 2, + 0, 3, 3, 4, 4, 4, 4, 3, 2, 1, + 1, 0, 1, 1, 0, 2, 1, 5, 1, 0, + 2, 2, 0, 1, 0, 3, 5, 1, 3, 4, + 3, 1, 1, 0, 2, 2, 0, 2, 2, 1, + 1, 1, 0, 2, 4, 5, 4, 2, 3, 2, + 2, 2, 2, 1, 2, 3, 0, 1, 0, 5, + 1, 4, 6, 2, 1, 0, 4, 0, 1, 1, + 2, 2, 2, 1, 1, 2, 2, 1, 1, 1, + 1, 1, 1, 3, 3, 0, 1, 3, 1, 2, + 1, 1, 1, 1, 1, 2, 4, 4, 5, 1, + 1, 2, 0, 2, 0, 1, 3, 1, 0, 1, + 2, 3, 2, 4, 2, 3, 2, 0, 1, 2, + 0, 4, 5, 1, 2, 2, 0, 1, 3, 1, + 2, 3, 3, 3, 3, 3, 3, 1, 4, 9, + 9, 3, 0, 2, 2, 0, 5, 3, 0, 1, + 1, 3, 5, 3, 1, 2, 1, 3, 5, 1, + 2, 3, 4, 5, 4, 5, 4, 6, 5, 4, + 5, 5, 5, 2, 4, 1, 1, 0, 1, 4, + 5, 4, 0, 2, 2, 2, 1, 1, 1, 1, + 0, 4, 2, 1, 2, 2, 4, 2, 6, 2, + 1, 3, 4, 0, 2, 0, 2, 0, 1, 3, + 3, 2, 0, 2, 4, 1, 1, 1, 0, 2, + 3, 5, 6, 2, 3, 1, 5, 5, 5, 3, + 3, 3, 4, 0, 1, 1, 1, 1, 1, 2, + 4, 1, 1, 1, 1, 2, 3, 0, 1, 1, + 1, 1, 1, 2, 2, 2, 2, 2, 1, 3, + 0, 1, 1, 1, 1, 5, 2, 1, 1, 1, + 1, 4, 1, 2, 2, 1, 3, 3, 2, 1, + 0, 5, 2, 5, 2, 1, 3, 3, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 3, 3, 3, 3, 3, 3, - 3, 0, 1, 3, 3, 5, 2, 2, 3, 3, + 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, + 3, 3, 0, 1, 3, 3, 5, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 2, 2, 3, 3, 2, 2, 3, - 3, 5, 4, 6, 3, 5, 4, 6, 4, 6, - 5, 7, 3, 2, 4, 3, 2, 3, 3, 3, - 3, 4, 3, 4, 3, 4, 5, 6, 6, 7, - 6, 7, 6, 7, 3, 4, 4, 6, 1, 4, - 3, 5, 1, 3, 2, 2, 3, 3, 3, 3, + 3, 3, 3, 3, 2, 2, 3, 3, 2, 2, + 3, 3, 5, 4, 6, 3, 5, 4, 6, 4, + 6, 5, 7, 3, 2, 4, 3, 2, 3, 3, + 3, 3, 4, 3, 4, 3, 4, 5, 6, 6, + 7, 6, 7, 6, 7, 3, 4, 4, 6, 1, + 4, 3, 5, 1, 3, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 2, 2, 5, 6, 6, 7, 1, 2, 1, - 1, 1, 2, 2, 4, 3, 1, 1, 1, 1, - 1, 4, 1, 1, 3, 1, 2, 4, 2, 2, - 3, 3, 6, 7, 9, 7, 7, 5, 1, 1, - 1, 5, 6, 6, 4, 4, 4, 4, 6, 5, - 5, 5, 4, 6, 4, 7, 9, 5, 0, 5, - 4, 0, 1, 0, 2, 0, 1, 3, 3, 2, - 2, 0, 6, 1, 0, 3, 0, 3, 3, 3, - 0, 1, 4, 2, 2, 2, 2, 2, 3, 2, - 2, 3, 0, 4, 3, 1, 5, 3, 1, 3, - 1, 2, 3, 1, 3, 1, 2, 1, 0, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 4, - 1, 4, 1, 4, 1, 2, 1, 2, 1, 2, - 1, 3, 1, 3, 1, 2, 1, 3, 1, 2, - 1, 0, 1, 3, 1, 3, 3, 1, 3, 3, + 3, 3, 2, 2, 5, 6, 6, 7, 1, 2, + 1, 1, 1, 2, 2, 4, 3, 1, 1, 1, + 1, 1, 4, 1, 1, 1, 1, 2, 4, 2, + 2, 3, 3, 3, 6, 7, 9, 7, 7, 5, + 1, 1, 1, 5, 6, 6, 4, 4, 4, 4, + 6, 5, 5, 5, 4, 6, 4, 7, 9, 5, + 0, 5, 4, 0, 1, 0, 2, 0, 1, 3, + 3, 2, 2, 0, 6, 1, 0, 3, 0, 3, + 3, 3, 0, 1, 4, 2, 2, 2, 2, 2, + 3, 2, 2, 3, 0, 4, 3, 1, 5, 3, + 1, 3, 1, 2, 3, 1, 3, 1, 2, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 4, 3, 2, 3, 0, - 3, 3, 2, 2, 1, 0, 2, 2, 3, 2, - 1, 1, 3, 5, 1, 2, 4, 2, 0, 1, - 0, 1, 2, 3, 5, 7, 7, 1, 0, 0, - 2, 0, 2, 3, 3, 3, 5, 7, 7, 0, - 2, 1, 0, 1, 0, 1, 3, 1, 2, 3, - 2, 1, 4, 2, 1, 0, 3, 1, 3, 1, - 2, 4, 2, 0, 1, 3, 1, 3, 1, 2, - 1, 3, 1, 1, 2, 1, 1, 2, 1, 1, - 2, 7, 2, 5, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 2, 2, 3, 3, 0, 1, 1, 1, 5, 3, - 0, 1, 1, 1, 1, 1, 1, 4, 7, 6, - 2, 0, 1, 1, 1, 1, 13, 16, 1, 2, - 0, 1, 0, 1, 0, 2, 0, 1, 0, 6, - 8, 6, 8, 6, 8, 3, 2, 1, 0, 6, - 6, 1, 1, 1, 1, 1, 1, 2, 1, 1, - 1, 1, 1, 4, 6, 3, 2, 4, 3, 5, - 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, + 1, 4, 1, 4, 1, 4, 1, 2, 1, 2, + 1, 2, 1, 3, 1, 3, 1, 2, 1, 3, + 1, 2, 1, 0, 1, 3, 1, 3, 3, 1, + 3, 3, 0, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 4, 3, 2, + 3, 0, 3, 3, 2, 2, 1, 0, 2, 2, + 3, 2, 1, 1, 3, 5, 1, 2, 4, 2, + 0, 1, 0, 1, 2, 3, 5, 7, 7, 1, + 0, 0, 2, 0, 2, 3, 3, 3, 5, 7, + 7, 0, 2, 1, 0, 1, 0, 1, 3, 1, + 2, 3, 2, 1, 4, 2, 1, 0, 3, 1, + 3, 1, 2, 4, 2, 0, 1, 3, 1, 3, + 1, 2, 1, 3, 1, 1, 2, 1, 1, 2, + 1, 1, 2, 7, 2, 5, 3, 3, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 2, 2, 3, 3, 0, 1, 1, 1, + 5, 3, 0, 1, 1, 1, 1, 1, 1, 4, + 7, 6, 2, 0, 1, 1, 1, 1, 13, 16, + 1, 2, 0, 1, 0, 1, 0, 2, 0, 1, + 0, 6, 8, 6, 8, 6, 8, 3, 2, 1, + 0, 6, 6, 1, 1, 1, 1, 1, 1, 2, + 1, 1, 1, 1, 1, 4, 6, 3, 2, 4, + 3, 5, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 3, 1, 1, 1, 1, 1, 1, 2, 1, - 1, 2, 3, 3, 3, 1, 3, 3, 2, 3, - 3, 1, 1, 1, 3, 5, 1, 1, 1, 1, - 3, 2, 2, 3, 4, 5, 1, 1, 1, 1, - 4, 6, 5, 4, 6, 1, 1, 1, 1, 1, - 1, 0, 1, 3, 1, 0, 7, 3, 1, 2, - 3, 2, 0, 2, 0, 2, 4, 5, 8, 2, - 3, 5, 1, 0, 2, 0, 2, 3, 3, 3, - 1, 1, 1, 2, 3, 2, 2, 2, 2, 3, - 4, 3, 1, 1, 1, 1, 1, 1, 0, 1, - 3, 1, 3, 2, 9, 12, 11, 12, 14, 3, - 4, 4, 0, 7, 10, 9, 2, 3, 0, 4, + 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, + 2, 1, 1, 2, 3, 3, 3, 1, 3, 3, + 2, 3, 3, 1, 1, 1, 3, 5, 1, 1, + 1, 1, 3, 2, 2, 3, 4, 5, 1, 1, + 1, 1, 4, 6, 5, 4, 6, 1, 1, 1, + 1, 1, 1, 0, 1, 3, 1, 0, 7, 3, + 1, 2, 3, 2, 0, 2, 0, 2, 4, 5, + 8, 2, 3, 5, 1, 0, 2, 0, 2, 3, + 3, 3, 1, 1, 1, 2, 3, 2, 2, 2, + 2, 3, 4, 3, 1, 1, 1, 1, 1, 1, + 0, 1, 3, 1, 3, 2, 9, 12, 11, 12, + 14, 3, 4, 4, 0, 7, 10, 9, 2, 3, + 0, 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -3450,7 +3454,7 @@ static const yytype_uint8 yyr2[] = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1 + 1 }; /* YYDEFACT[STATE-NAME] -- Default rule to reduce with in state @@ -3458,1944 +3462,1899 @@ static const yytype_uint8 yyr2[] = means the default is an error. */ static const yytype_uint16 yydefact[] = { - 155, 263, 0, 1383, 1382, 1453, 263, 0, 1318, 0, - 263, 485, 401, 0, 1474, 1473, 0, 207, 263, 0, - 155, 0, 0, 0, 0, 0, 0, 548, 551, 549, - 0, 0, 0, 263, 588, 0, 1475, 263, 0, 0, - 580, 550, 0, 1431, 0, 0, 0, 0, 0, 2, + 155, 263, 0, 1385, 1384, 1455, 263, 0, 1320, 0, + 263, 486, 401, 0, 1476, 1475, 0, 207, 263, 0, + 155, 0, 0, 0, 0, 0, 0, 549, 552, 550, + 0, 0, 0, 263, 589, 0, 1477, 263, 0, 0, + 581, 551, 0, 1433, 0, 0, 0, 0, 0, 2, 4, 7, 21, 35, 31, 0, 20, 33, 18, 17, 26, 6, 24, 37, 39, 19, 25, 15, 38, 13, - 36, 524, 510, 593, 523, 0, 0, 154, 692, 531, - 34, 16, 30, 5, 11, 12, 28, 29, 27, 1341, - 42, 32, 40, 22, 8, 9, 23, 41, 43, 1476, - 1472, 10, 44, 14, 262, 261, 255, 0, 0, 0, - 0, 0, 1452, 0, 0, 256, 111, 1500, 1501, 1502, - 1503, 1504, 1505, 1506, 1507, 1508, 1509, 1510, 1874, 1511, - 1512, 1513, 1514, 1515, 1875, 1516, 1517, 1518, 1820, 1821, - 1876, 1822, 1823, 1519, 1520, 1521, 1522, 1523, 1524, 1525, - 1526, 1527, 1528, 1824, 1825, 1529, 1530, 1531, 1532, 1533, - 1826, 1877, 1827, 1534, 1535, 1536, 1537, 1538, 1878, 1539, - 1540, 1541, 1542, 1543, 1544, 1545, 1546, 1547, 1879, 1548, - 1549, 1550, 1551, 1552, 1553, 1554, 1555, 1556, 1557, 1828, - 1558, 1559, 1829, 1560, 1561, 1562, 1563, 1564, 1565, 1566, - 1567, 1568, 1569, 1570, 1571, 1572, 1573, 1574, 1575, 1576, - 1577, 1578, 1579, 1580, 1581, 1582, 1583, 1584, 1585, 1586, - 1830, 1587, 1588, 1589, 1590, 1591, 1831, 1592, 1593, 1594, - 1832, 1595, 1596, 1597, 1880, 1881, 1598, 1599, 1833, 1883, - 1600, 1601, 1834, 1835, 1602, 1603, 1604, 1605, 1606, 1607, - 1608, 1609, 1610, 1884, 1611, 1612, 1613, 1614, 1615, 1616, - 1617, 1618, 1619, 1620, 1621, 1622, 1885, 1836, 1623, 1624, - 1625, 1626, 1627, 1837, 1838, 1839, 1628, 1886, 1887, 1629, - 1888, 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1889, 1637, - 1890, 1638, 1639, 1640, 1641, 1642, 1643, 1644, 1645, 1840, - 1646, 1647, 1648, 1649, 1650, 1651, 1652, 1653, 1654, 1655, - 1656, 1657, 1658, 1659, 1660, 1661, 1662, 1663, 1664, 1665, - 1841, 1892, 1842, 1666, 1667, 1668, 1843, 1669, 1670, 1893, - 1671, 1844, 1672, 1845, 1673, 1674, 1675, 1676, 1677, 1678, - 1679, 1680, 1681, 1682, 1846, 1894, 1683, 1895, 1847, 1684, - 1685, 1686, 1687, 1688, 1689, 1690, 1691, 1692, 1693, 1694, - 1695, 1696, 1848, 1896, 1697, 1698, 1849, 1699, 1700, 1701, - 1702, 1703, 1704, 1705, 1706, 1707, 1708, 1709, 1710, 1850, - 1711, 1712, 1713, 1714, 1715, 1716, 1717, 1718, 1719, 1720, - 1721, 1722, 1723, 1724, 1725, 1726, 1727, 1728, 1729, 1897, - 1730, 1731, 1732, 1851, 1733, 1734, 1735, 1736, 1737, 1738, - 1739, 1740, 1741, 1742, 1743, 1744, 1745, 1746, 1747, 1748, - 1749, 1750, 1751, 1852, 1752, 1753, 1898, 1754, 1755, 1853, - 1756, 1757, 1758, 1759, 1760, 1761, 1762, 1763, 1764, 1765, - 1766, 1767, 1768, 1854, 1769, 1855, 1770, 1771, 1772, 1900, - 1773, 1774, 1775, 1776, 1777, 1778, 1856, 1857, 1779, 1780, - 1858, 1781, 1859, 1782, 1783, 1860, 1784, 1785, 1786, 1787, - 1788, 1789, 1790, 1791, 1792, 1793, 1794, 1795, 1796, 1797, - 1798, 1799, 1800, 1861, 1862, 1801, 1901, 1802, 1803, 1804, - 1805, 1806, 1807, 1808, 1809, 1810, 1811, 1812, 1813, 1814, - 1815, 1863, 1864, 1865, 1866, 1867, 1868, 1869, 1870, 1871, - 1872, 1873, 1816, 1817, 1818, 1819, 0, 1483, 0, 1243, - 112, 113, 1265, 111, 1833, 1840, 1854, 1317, 1316, 112, - 0, 258, 484, 0, 0, 0, 0, 0, 0, 209, - 0, 395, 394, 0, 1307, 400, 0, 0, 0, 115, - 107, 1699, 114, 1242, 105, 121, 2044, 2045, 2046, 2047, - 1931, 2048, 2049, 2050, 2051, 1932, 2052, 1933, 1934, 1935, - 1936, 1937, 1938, 2053, 2054, 2055, 1940, 1939, 2056, 1941, - 2057, 1942, 2058, 1943, 1944, 2059, 2060, 1945, 1554, 1946, - 1947, 2061, 2062, 2063, 2064, 2065, 2066, 2067, 2068, 2069, - 1948, 1949, 2070, 2071, 1950, 2072, 2073, 1951, 2074, 1952, - 1953, 1954, 2075, 2076, 1955, 1956, 2077, 1957, 2078, 2079, - 1958, 1959, 1962, 1960, 2080, 1961, 2081, 1963, 1964, 1965, - 2082, 2083, 1966, 1967, 2084, 1968, 1969, 1970, 1971, 1972, - 2085, 1973, 2086, 1974, 1975, 2087, 2088, 2089, 2090, 2091, - 1977, 1976, 1978, 1979, 2092, 2093, 2094, 2095, 1980, 1981, - 1982, 2096, 2097, 1983, 2098, 2099, 1984, 1985, 2100, 1986, - 1987, 2101, 1988, 1989, 2102, 1990, 1991, 2103, 2104, 2105, - 1992, 2106, 1993, 1994, 2107, 2108, 1995, 1996, 2109, 1997, - 2110, 2111, 2112, 2113, 1998, 1999, 2114, 2000, 2115, 2116, - 2117, 2118, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, - 2009, 2010, 2011, 1449, 123, 122, 124, 0, 419, 420, + 36, 525, 511, 594, 524, 0, 0, 154, 693, 532, + 34, 16, 30, 5, 11, 12, 28, 29, 27, 1343, + 42, 32, 40, 22, 8, 9, 23, 41, 43, 1478, + 1474, 10, 44, 14, 262, 261, 255, 0, 0, 0, + 0, 0, 1454, 0, 0, 256, 111, 1502, 1503, 1504, + 1505, 1506, 1507, 1508, 1509, 1510, 1511, 1512, 1876, 1513, + 1514, 1515, 1516, 1517, 1877, 1518, 1519, 1520, 1822, 1823, + 1878, 1824, 1825, 1521, 1522, 1523, 1524, 1525, 1526, 1527, + 1528, 1529, 1530, 1826, 1827, 1531, 1532, 1533, 1534, 1535, + 1828, 1879, 1829, 1536, 1537, 1538, 1539, 1540, 1880, 1541, + 1542, 1543, 1544, 1545, 1546, 1547, 1548, 1549, 1881, 1550, + 1551, 1552, 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1830, + 1560, 1561, 1831, 1562, 1563, 1564, 1565, 1566, 1567, 1568, + 1569, 1570, 1571, 1572, 1573, 1574, 1575, 1576, 1577, 1578, + 1579, 1580, 1581, 1582, 1583, 1584, 1585, 1586, 1587, 1588, + 1832, 1589, 1590, 1591, 1592, 1593, 1833, 1594, 1595, 1596, + 1834, 1597, 1598, 1599, 1882, 1883, 1600, 1601, 1835, 1885, + 1602, 1603, 1836, 1837, 1604, 1605, 1606, 1607, 1608, 1609, + 1610, 1611, 1612, 1886, 1613, 1614, 1615, 1616, 1617, 1618, + 1619, 1620, 1621, 1622, 1623, 1624, 1887, 1838, 1625, 1626, + 1627, 1628, 1629, 1839, 1840, 1841, 1630, 1888, 1889, 1631, + 1890, 1632, 1633, 1634, 1635, 1636, 1637, 1638, 1891, 1639, + 1892, 1640, 1641, 1642, 1643, 1644, 1645, 1646, 1647, 1842, + 1648, 1649, 1650, 1651, 1652, 1653, 1654, 1655, 1656, 1657, + 1658, 1659, 1660, 1661, 1662, 1663, 1664, 1665, 1666, 1667, + 1843, 1894, 1844, 1668, 1669, 1670, 1845, 1671, 1672, 1895, + 1673, 1846, 1674, 1847, 1675, 1676, 1677, 1678, 1679, 1680, + 1681, 1682, 1683, 1684, 1848, 1896, 1685, 1897, 1849, 1686, + 1687, 1688, 1689, 1690, 1691, 1692, 1693, 1694, 1695, 1696, + 1697, 1698, 1850, 1898, 1699, 1700, 1851, 1701, 1702, 1703, + 1704, 1705, 1706, 1707, 1708, 1709, 1710, 1711, 1712, 1852, + 1713, 1714, 1715, 1716, 1717, 1718, 1719, 1720, 1721, 1722, + 1723, 1724, 1725, 1726, 1727, 1728, 1729, 1730, 1731, 1899, + 1732, 1733, 1734, 1853, 1735, 1736, 1737, 1738, 1739, 1740, + 1741, 1742, 1743, 1744, 1745, 1746, 1747, 1748, 1749, 1750, + 1751, 1752, 1753, 1854, 1754, 1755, 1900, 1756, 1757, 1855, + 1758, 1759, 1760, 1761, 1762, 1763, 1764, 1765, 1766, 1767, + 1768, 1769, 1770, 1856, 1771, 1857, 1772, 1773, 1774, 1902, + 1775, 1776, 1777, 1778, 1779, 1780, 1858, 1859, 1781, 1782, + 1860, 1783, 1861, 1784, 1785, 1862, 1786, 1787, 1788, 1789, + 1790, 1791, 1792, 1793, 1794, 1795, 1796, 1797, 1798, 1799, + 1800, 1801, 1802, 1863, 1864, 1803, 1903, 1804, 1805, 1806, + 1807, 1808, 1809, 1810, 1811, 1812, 1813, 1814, 1815, 1816, + 1817, 1865, 1866, 1867, 1868, 1869, 1870, 1871, 1872, 1873, + 1874, 1875, 1818, 1819, 1820, 1821, 0, 1485, 0, 1245, + 112, 113, 1267, 111, 1835, 1842, 1856, 1319, 1318, 112, + 0, 258, 485, 0, 0, 0, 0, 0, 0, 209, + 0, 395, 394, 0, 1309, 400, 0, 0, 0, 115, + 107, 1701, 114, 1244, 105, 121, 2046, 2047, 2048, 2049, + 1933, 2050, 2051, 2052, 2053, 1934, 2054, 1935, 1936, 1937, + 1938, 1939, 1940, 2055, 2056, 2057, 1942, 1941, 2058, 1943, + 2059, 1944, 2060, 1945, 1946, 2061, 2062, 1947, 1556, 1948, + 1949, 2063, 2064, 2065, 2066, 2067, 2068, 2069, 2070, 2071, + 1950, 1951, 2072, 2073, 1952, 2074, 2075, 1953, 2076, 1954, + 1955, 1956, 2077, 2078, 1957, 1958, 2079, 1959, 2080, 2081, + 1960, 1961, 1964, 1962, 2082, 1963, 2083, 1965, 1966, 1967, + 2084, 2085, 1968, 1969, 2086, 1970, 1971, 1972, 1973, 1974, + 2087, 1975, 2088, 1976, 1977, 2089, 2090, 2091, 2092, 2093, + 1979, 1978, 1980, 1981, 2094, 2095, 2096, 2097, 1982, 1983, + 1984, 2098, 2099, 1985, 2100, 2101, 1986, 1987, 2102, 1988, + 1989, 2103, 1990, 1991, 2104, 1992, 1993, 2105, 2106, 2107, + 1994, 2108, 1995, 1996, 2109, 2110, 1997, 1998, 2111, 1999, + 2112, 2113, 2114, 2115, 2000, 2001, 2116, 2002, 2117, 2118, + 2119, 2120, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, + 2011, 2012, 2013, 1451, 123, 122, 124, 0, 419, 420, 0, 430, 0, 412, 417, 413, 0, 439, 432, 440, 421, 411, 433, 422, 410, 208, 0, 441, 427, 415, 0, 0, 0, 0, 259, 220, 401, 0, 155, 0, - 1347, 1357, 1366, 1362, 1356, 1364, 1354, 1360, 1346, 1368, - 1355, 1359, 1352, 1369, 1350, 1367, 1365, 1353, 1361, 1345, - 1349, 1336, 1341, 1372, 1363, 1370, 1358, 1371, 1373, 1348, - 1374, 1351, 0, 1318, 0, 0, 1826, 1877, 1831, 0, - 1844, 0, 1847, 1848, 1733, 1855, 1858, 1859, 1860, 1861, - 0, 762, 114, 109, 746, 0, 526, 696, 706, 746, - 751, 1029, 774, 1030, 0, 116, 1417, 1416, 1412, 1411, - 194, 1280, 1461, 1600, 1640, 1750, 1856, 1779, 1479, 1462, - 1456, 1460, 260, 587, 585, 0, 1214, 1600, 1640, 1737, - 1750, 1856, 1391, 1395, 0, 257, 1481, 1466, 0, 1467, - 114, 532, 579, 0, 264, 1430, 0, 1435, 0, 1713, - 559, 562, 1274, 560, 524, 0, 0, 1, 155, 0, - 161, 0, 583, 583, 0, 583, 0, 516, 0, 0, - 524, 519, 523, 693, 1340, 1445, 1478, 1856, 1779, 1465, - 1468, 1609, 0, 0, 1609, 0, 1609, 0, 1609, 0, - 0, 1455, 1198, 0, 1244, 117, 0, 0, 1329, 1325, - 1330, 1326, 1331, 1324, 1323, 1332, 1328, 0, 0, 0, - 366, 399, 398, 397, 396, 401, 1609, 1291, 0, 205, - 448, 449, 0, 0, 0, 0, 0, 1302, 108, 106, - 1609, 1450, 428, 429, 0, 418, 414, 416, 0, 0, - 1609, 1269, 438, 434, 1609, 438, 1236, 1609, 0, 0, - 212, 0, 394, 1338, 1375, 1998, 1389, 0, 1390, 1380, - 1344, 1376, 1377, 155, 0, 483, 1315, 1413, 0, 0, - 0, 1150, 746, 751, 0, 0, 764, 0, 1169, 0, - 1175, 0, 0, 0, 746, 531, 0, 706, 763, 110, - 0, 744, 745, 634, 634, 588, 0, 569, 756, 0, - 0, 759, 757, 0, 759, 0, 0, 0, 759, 755, - 714, 0, 634, 0, 744, 747, 634, 0, 766, 1335, - 0, 0, 0, 0, 0, 1459, 1457, 1458, 1463, 0, - 0, 0, 1246, 1248, 1249, 1118, 1259, 1009, 0, 1821, - 1822, 1823, 1190, 1824, 1825, 1827, 1828, 1829, 968, 1574, - 1830, 1257, 1832, 1834, 1835, 1837, 1838, 1839, 1840, 1841, - 1842, 0, 1258, 1845, 1678, 1850, 1851, 1853, 1856, 1857, - 1256, 1862, 0, 0, 0, 1225, 1141, 0, 1008, 0, - 0, 0, 1191, 1199, 1001, 0, 0, 810, 811, 832, - 833, 812, 838, 839, 841, 813, 0, 1221, 902, 997, - 1209, 1006, 1010, 1048, 1012, 1028, 1015, 1085, 1007, 0, - 1013, 999, 1217, 569, 1215, 0, 1000, 1245, 569, 1213, - 1394, 1392, 1398, 1393, 0, 0, 0, 0, 0, 110, - 1438, 1437, 1429, 1427, 1428, 1426, 1425, 1432, 0, 1434, - 1341, 1136, 1138, 0, 561, 0, 0, 0, 513, 512, - 514, 3, 0, 0, 0, 0, 581, 582, 0, 0, - 0, 0, 0, 0, 0, 0, 677, 608, 609, 611, - 674, 678, 686, 0, 0, 0, 0, 0, 520, 0, - 1274, 1477, 1471, 1469, 0, 0, 0, 139, 139, 0, - 0, 0, 0, 0, 99, 48, 92, 0, 0, 0, - 0, 234, 247, 0, 0, 0, 0, 0, 244, 0, - 0, 227, 50, 221, 223, 0, 139, 0, 46, 0, - 0, 0, 52, 1453, 0, 483, 1197, 0, 119, 120, - 118, 111, 0, 2012, 1874, 1875, 1876, 1877, 1827, 1878, - 1879, 0, 1880, 1881, 1833, 1883, 1884, 1885, 1886, 1887, - 1888, 1889, 1890, 1840, 1892, 1893, 1894, 1895, 1896, 1897, - 2038, 1898, 1854, 1900, 1860, 0, 1901, 1021, 1144, 593, - 1142, 1275, 0, 112, 1262, 0, 1327, 0, 0, 0, - 0, 481, 0, 0, 0, 0, 1287, 0, 1609, 206, - 210, 0, 1609, 201, 1609, 366, 0, 1609, 366, 1609, - 0, 1301, 1304, 0, 431, 426, 424, 423, 425, 1609, - 253, 0, 0, 1270, 436, 437, 0, 405, 0, 0, - 407, 0, 0, 217, 0, 215, 0, 401, 155, 0, - 228, 1385, 1386, 1384, 0, 0, 1379, 1343, 231, 248, - 1388, 1378, 1387, 1342, 1337, 0, 0, 1333, 471, 0, - 0, 0, 0, 1151, 879, 878, 862, 863, 876, 877, - 864, 865, 872, 873, 881, 880, 870, 871, 866, 867, - 860, 861, 868, 869, 874, 875, 858, 859, 1164, 1152, - 1153, 1154, 1155, 1156, 1157, 1158, 1159, 1160, 1161, 1162, - 1163, 0, 0, 705, 703, 0, 0, 0, 0, 0, - 0, 1191, 0, 972, 1007, 0, 0, 0, 1136, 1174, - 0, 0, 0, 0, 0, 0, 1136, 1180, 0, 0, - 730, 742, 0, 627, 633, 704, 702, 0, 1214, 697, - 0, 776, 0, 756, 0, 755, 0, 0, 758, 752, - 0, 753, 0, 0, 0, 0, 754, 0, 0, 0, - 0, 0, 700, 0, 742, 0, 701, 773, 1419, 1418, - 1414, 1401, 1409, 195, 0, 1266, 1902, 1903, 1904, 820, - 1905, 849, 827, 849, 849, 1906, 1907, 1908, 1909, 816, - 816, 829, 1910, 1911, 1912, 1913, 1914, 817, 818, 854, - 1915, 1916, 1917, 1918, 1919, 0, 0, 1920, 849, 1921, - 816, 1922, 1923, 1924, 821, 1925, 784, 1926, 0, 1927, - 819, 785, 1928, 857, 857, 1929, 0, 844, 1930, 0, - 1147, 794, 802, 803, 804, 805, 830, 831, 806, 836, - 837, 807, 901, 0, 816, 1267, 1268, 155, 1464, 1480, - 0, 1141, 1016, 848, 835, 1189, 0, 843, 842, 0, - 1141, 825, 824, 823, 1003, 0, 822, 1098, 849, 849, - 847, 927, 826, 0, 0, 0, 0, 0, 853, 0, - 851, 928, 906, 907, 0, 1224, 1233, 1136, 1140, 0, - 1001, 1136, 0, 0, 1088, 1090, 0, 1018, 1019, 0, - 1192, 1247, 1002, 0, 1252, 0, 0, 901, 901, 1220, - 1118, 0, 1108, 1111, 0, 0, 1115, 1116, 1117, 0, - 0, 0, 1212, 0, 1126, 1128, 0, 0, 943, 1124, - 0, 946, 0, 0, 0, 0, 1112, 1113, 1114, 1104, - 1105, 1106, 1107, 1109, 1110, 1122, 1103, 924, 0, 998, - 0, 1051, 0, 923, 1218, 695, 0, 1250, 695, 1403, - 1407, 1408, 1402, 1406, 0, 1397, 1396, 1399, 1400, 1482, - 0, 1439, 1423, 0, 1420, 1139, 690, 563, 1238, 0, - 567, 1444, 160, 159, 0, 0, 536, 535, 602, 594, - 596, 602, 0, 534, 0, 650, 651, 0, 0, 0, - 0, 683, 681, 1246, 1259, 638, 612, 637, 0, 0, - 616, 0, 642, 902, 676, 518, 606, 607, 610, 517, - 0, 679, 0, 689, 0, 555, 557, 540, 554, 552, - 537, 545, 677, 611, 0, 1446, 1470, 0, 0, 0, - 0, 0, 1609, 0, 0, 787, 83, 64, 318, 138, - 0, 0, 0, 0, 0, 0, 0, 91, 88, 89, - 90, 0, 0, 0, 0, 1266, 232, 233, 246, 0, - 237, 238, 235, 239, 240, 0, 0, 225, 226, 0, - 0, 0, 0, 224, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1454, 1447, 1193, 1198, 593, 593, 593, - 0, 591, 592, 0, 0, 0, 0, 0, 470, 364, - 374, 0, 0, 0, 1291, 205, 0, 0, 0, 0, - 0, 0, 401, 1294, 1292, 1290, 1293, 1295, 1580, 189, - 0, 0, 0, 0, 0, 197, 200, 0, 363, 337, - 0, 0, 1306, 0, 0, 0, 1609, 353, 1303, 0, - 1451, 0, 0, 251, 438, 1271, 0, 435, 438, 1237, - 0, 438, 219, 0, 0, 1339, 1381, 229, 249, 230, - 250, 483, 478, 508, 0, 486, 491, 468, 0, 468, - 0, 488, 492, 468, 487, 0, 468, 482, 1415, 0, - 1044, 0, 1034, 0, 0, 765, 0, 0, 1035, 974, - 975, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 992, - 991, 1036, 769, 0, 772, 0, 0, 1172, 1173, 0, - 1037, 0, 0, 1179, 0, 0, 0, 1042, 0, 707, - 0, 0, 0, 622, 626, 629, 0, 632, 569, 525, - 1600, 1640, 0, 580, 580, 580, 578, 568, 0, 654, - 0, 0, 0, 731, 0, 0, 733, 735, 0, 0, - 738, 0, 713, 712, 0, 0, 0, 0, 777, 0, - 1242, 0, 0, 196, 0, 0, 0, 802, 0, 0, - 0, 792, 788, 0, 882, 883, 884, 885, 886, 887, - 888, 889, 890, 891, 892, 893, 808, 1279, 0, 814, - 1282, 1283, 1284, 1281, 1278, 1285, 1286, 0, 0, 0, - 0, 1188, 1184, 0, 0, 0, 0, 1093, 1095, 1097, - 0, 846, 845, 1102, 1108, 1111, 1115, 1116, 1117, 1112, - 1113, 1114, 1104, 1105, 1106, 1107, 1109, 1110, 0, 1130, - 0, 1084, 0, 0, 0, 0, 0, 0, 1223, 0, - 970, 0, 1014, 1005, 0, 0, 1091, 1020, 1225, 1200, - 0, 0, 0, 1255, 1254, 903, 912, 915, 947, 948, - 919, 920, 921, 925, 1277, 1276, 1219, 0, 1211, 0, - 0, 904, 929, 934, 0, 1181, 964, 0, 952, 0, - 942, 0, 950, 954, 930, 945, 0, 926, 0, 1212, - 1127, 1129, 0, 1125, 0, 916, 917, 918, 908, 909, - 910, 911, 913, 914, 922, 1101, 1099, 1100, 0, 1198, - 0, 1210, 0, 0, 1053, 0, 0, 949, 1216, 0, - 776, 593, 776, 0, 901, 1440, 1274, 1433, 1274, 1422, - 1137, 1239, 1273, 565, 0, 0, 0, 1442, 146, 150, - 0, 1199, 180, 182, 695, 0, 600, 601, 605, 0, - 0, 605, 584, 533, 1851, 1733, 0, 0, 0, 0, - 643, 684, 0, 675, 640, 641, 0, 639, 1246, 644, - 1245, 645, 648, 649, 617, 1234, 685, 687, 0, 680, - 0, 1240, 539, 558, 0, 0, 0, 0, 0, 522, - 521, 691, 0, 49, 0, 1609, 66, 0, 0, 0, - 0, 0, 0, 268, 0, 368, 268, 104, 1609, 438, - 1609, 438, 1504, 1575, 1751, 0, 62, 342, 95, 0, - 132, 371, 0, 327, 85, 100, 125, 0, 0, 51, - 222, 236, 241, 128, 245, 242, 1311, 243, 139, 0, - 47, 0, 126, 0, 1309, 0, 0, 53, 130, 1313, - 1455, 0, 1197, 0, 591, 591, 591, 0, 1143, 0, - 0, 0, 1145, 1146, 942, 1321, 1320, 1322, 1319, 456, - 469, 0, 365, 0, 480, 459, 460, 470, 1289, 210, - 0, 201, 366, 0, 366, 0, 1291, 0, 0, 191, - 187, 205, 211, 0, 0, 0, 0, 0, 364, 356, - 354, 387, 0, 361, 355, 0, 0, 313, 0, 1498, - 0, 0, 0, 0, 450, 0, 0, 0, 0, 253, - 254, 404, 1272, 406, 0, 408, 218, 216, 1334, 475, - 0, 474, 479, 477, 473, 472, 0, 467, 0, 501, - 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1149, - 0, 1167, 1166, 973, 980, 983, 987, 988, 989, 1168, - 0, 0, 0, 984, 985, 986, 976, 977, 978, 979, - 981, 982, 990, 774, 0, 0, 768, 1177, 1176, 1170, - 1171, 0, 1039, 1040, 1041, 1178, 0, 0, 743, 620, - 618, 621, 623, 619, 0, 0, 776, 580, 580, 580, - 580, 577, 0, 0, 0, 775, 0, 671, 739, 737, - 0, 761, 0, 734, 717, 740, 0, 725, 0, 732, - 781, 748, 0, 0, 750, 1410, 798, 0, 793, 789, - 0, 0, 0, 799, 0, 0, 0, 0, 0, 0, - 0, 1148, 586, 1017, 0, 0, 0, 1185, 0, 969, - 815, 828, 0, 1096, 1011, 0, 1119, 1083, 856, 855, - 857, 857, 0, 0, 0, 1232, 0, 1137, 1087, 1089, - 1233, 1004, 840, 901, 0, 0, 0, 0, 0, 0, - 0, 953, 944, 0, 951, 955, 0, 0, 0, 938, - 0, 0, 936, 965, 932, 0, 0, 966, 1197, 0, - 1201, 0, 0, 1052, 1061, 698, 694, 654, 591, 654, - 0, 1404, 1424, 1421, 566, 155, 1443, 0, 169, 0, - 0, 0, 0, 172, 186, 183, 1442, 0, 0, 595, - 597, 0, 1120, 605, 599, 647, 646, 0, 615, 682, - 613, 0, 688, 0, 556, 0, 542, 0, 716, 0, - 0, 0, 0, 0, 317, 0, 0, 0, 268, 0, - 376, 0, 383, 0, 0, 368, 349, 84, 0, 0, - 0, 58, 103, 76, 68, 54, 82, 0, 0, 87, - 0, 80, 97, 98, 96, 101, 0, 278, 303, 0, - 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 483, 1198, 1194, 1198, 0, 0, 0, - 593, 589, 590, 1022, 0, 455, 507, 504, 505, 503, - 227, 375, 0, 0, 0, 199, 363, 0, 1306, 0, - 1288, 401, 0, 192, 0, 190, 210, 0, 0, 201, - 366, 0, 341, 337, 362, 335, 334, 336, 0, 1499, - 220, 0, 1493, 366, 1305, 0, 0, 451, 0, 445, - 0, 1300, 252, 438, 0, 463, 502, 509, 489, 494, - 0, 500, 496, 495, 490, 498, 497, 493, 1032, 1043, - 1165, 0, 0, 0, 0, 767, 770, 0, 1038, 1033, - 741, 0, 0, 654, 0, 0, 0, 0, 571, 570, - 576, 0, 0, 1055, 736, 0, 0, 0, 723, 711, - 718, 719, 0, 0, 0, 779, 778, 749, 802, 0, - 782, 802, 0, 802, 0, 800, 0, 809, 894, 895, - 896, 897, 898, 899, 900, 834, 0, 1187, 1183, 1092, - 1094, 1131, 852, 850, 1222, 1136, 1227, 1229, 0, 0, - 0, 1086, 971, 1253, 905, 0, 0, 935, 1182, 956, - 0, 0, 0, 931, 1119, 0, 0, 0, 0, 0, - 940, 0, 1205, 1198, 0, 1204, 0, 0, 0, 0, - 1027, 699, 671, 0, 671, 0, 0, 1441, 0, 1436, - 147, 148, 149, 0, 0, 0, 164, 141, 0, 0, - 181, 169, 157, 603, 604, 0, 598, 614, 1235, 1241, - 541, 0, 1001, 0, 0, 538, 0, 133, 268, 0, - 0, 65, 0, 385, 329, 377, 360, 344, 0, 0, - 0, 269, 0, 402, 0, 0, 350, 0, 0, 0, - 0, 330, 0, 0, 289, 0, 0, 360, 0, 367, - 285, 286, 0, 57, 77, 0, 73, 0, 102, 0, - 0, 0, 0, 0, 60, 72, 0, 55, 0, 438, - 438, 63, 1266, 1902, 1903, 1904, 1905, 1906, 1907, 1908, - 1909, 1910, 1911, 2022, 1912, 1913, 1914, 1915, 1916, 1917, - 1918, 1919, 2031, 1920, 275, 1921, 1678, 1922, 1923, 1924, - 1925, 1926, 0, 1927, 785, 1928, 1929, 2110, 1930, 1104, - 1105, 274, 273, 370, 270, 378, 272, 0, 1267, 271, - 373, 328, 129, 1312, 0, 127, 0, 1310, 136, 134, - 131, 1314, 1448, 0, 0, 1025, 1026, 1023, 591, 0, - 0, 0, 483, 462, 0, 0, 0, 1498, 0, 0, - 0, 1609, 0, 188, 0, 0, 202, 1306, 198, 363, - 0, 393, 313, 388, 0, 1498, 1496, 0, 1306, 1492, - 0, 442, 0, 0, 0, 409, 476, 0, 499, 993, - 0, 0, 0, 0, 630, 0, 636, 671, 575, 574, - 573, 572, 653, 1549, 1834, 1732, 0, 657, 652, 655, - 660, 662, 661, 663, 659, 670, 0, 673, 760, 1132, - 1134, 0, 0, 0, 0, 724, 726, 0, 728, 0, - 780, 796, 0, 797, 0, 795, 790, 801, 1186, 1230, - 1231, 1226, 0, 902, 962, 960, 957, 0, 958, 939, - 0, 0, 937, 933, 0, 967, 0, 0, 1202, 0, - 1047, 0, 1050, 1064, 1060, 1059, 1055, 1022, 1055, 1405, - 564, 168, 145, 171, 170, 0, 1199, 178, 0, 0, - 169, 0, 173, 452, 0, 0, 553, 715, 546, 547, - 0, 381, 67, 0, 360, 0, 268, 346, 345, 348, - 343, 347, 0, 403, 0, 0, 287, 0, 294, 332, - 333, 331, 288, 360, 366, 290, 0, 0, 0, 69, - 59, 56, 61, 70, 0, 0, 71, 74, 781, 86, - 79, 1266, 2031, 2040, 0, 0, 0, 0, 0, 1196, - 1195, 0, 458, 457, 506, 454, 465, 227, 0, 0, - 0, 337, 1495, 0, 0, 447, 0, 0, 363, 193, - 0, 0, 0, 0, 1498, 0, 0, 265, 0, 310, - 0, 213, 1497, 0, 0, 1484, 0, 0, 1298, 1299, - 0, 464, 994, 0, 995, 771, 0, 0, 628, 1055, - 0, 0, 0, 664, 658, 0, 1054, 1056, 0, 625, - 1135, 720, 0, 722, 0, 746, 0, 746, 729, 791, - 783, 1228, 1045, 0, 959, 963, 961, 941, 1198, 1206, - 1198, 1203, 1049, 1063, 1066, 673, 1251, 673, 0, 0, - 156, 0, 0, 153, 140, 158, 1121, 543, 544, 0, - 268, 0, 359, 382, 299, 277, 0, 0, 0, 284, - 291, 392, 293, 0, 78, 94, 0, 0, 372, 137, - 135, 1024, 483, 0, 204, 1306, 313, 1492, 444, 0, - 0, 0, 0, 337, 220, 1494, 326, 319, 320, 321, - 322, 323, 324, 325, 340, 339, 311, 312, 0, 0, - 0, 0, 0, 446, 1300, 0, 175, 184, 0, 175, - 996, 631, 0, 673, 0, 0, 0, 656, 0, 0, - 672, 0, 529, 1133, 0, 710, 708, 0, 709, 0, - 0, 0, 0, 593, 625, 625, 142, 0, 143, 179, - 0, 0, 0, 366, 384, 358, 0, 351, 297, 296, - 298, 302, 0, 300, 0, 316, 0, 309, 277, 0, - 81, 0, 379, 453, 461, 0, 267, 1486, 363, 0, - 203, 1492, 313, 1498, 1492, 0, 1489, 0, 443, 0, - 0, 0, 177, 1306, 0, 177, 0, 625, 666, 0, - 665, 1058, 1057, 627, 721, 0, 1046, 1208, 1207, 0, - 1070, 528, 527, 0, 0, 0, 0, 392, 0, 338, - 0, 0, 299, 0, 292, 389, 390, 391, 0, 305, - 295, 306, 75, 93, 380, 0, 363, 1487, 266, 214, - 1485, 1490, 1491, 0, 175, 174, 602, 176, 776, 185, - 602, 635, 530, 667, 624, 727, 1065, 0, 0, 0, - 0, 0, 152, 776, 163, 0, 309, 357, 352, 276, - 301, 315, 0, 0, 0, 307, 0, 308, 1492, 0, - 177, 605, 1296, 605, 1820, 1550, 1786, 0, 1082, 1071, - 1082, 1082, 1062, 144, 151, 0, 268, 281, 0, 280, - 0, 369, 304, 1488, 1306, 602, 165, 166, 0, 1075, - 1074, 1073, 1077, 1076, 0, 1069, 1067, 1068, 776, 386, - 279, 283, 282, 776, 605, 0, 0, 1079, 0, 1080, - 162, 1297, 167, 1072, 1078, 1081 + 1349, 1359, 1368, 1364, 1358, 1366, 1356, 1362, 1348, 1370, + 1357, 1361, 1354, 1371, 1352, 1369, 1367, 1355, 1363, 1347, + 1351, 1338, 1343, 1374, 1365, 1372, 1360, 1373, 1375, 1350, + 1376, 1353, 0, 1320, 0, 0, 1828, 1879, 1833, 0, + 1846, 0, 1849, 1850, 1735, 1857, 1860, 1861, 1862, 1863, + 0, 763, 114, 109, 747, 0, 527, 697, 707, 747, + 752, 1031, 775, 1032, 0, 116, 1419, 1418, 1414, 1413, + 194, 1282, 1463, 1602, 1642, 1752, 1858, 1781, 1481, 1464, + 1458, 1462, 260, 588, 586, 0, 1216, 1602, 1642, 1739, + 1752, 1858, 1393, 1397, 0, 257, 1483, 1468, 0, 1469, + 114, 533, 580, 0, 264, 1432, 0, 1437, 0, 1715, + 560, 563, 1276, 561, 525, 0, 0, 1, 155, 0, + 161, 0, 584, 584, 0, 584, 0, 517, 0, 0, + 525, 520, 524, 694, 1342, 1447, 1480, 1858, 1781, 1467, + 1470, 1611, 0, 0, 1611, 0, 1611, 0, 1611, 0, + 0, 1457, 1200, 0, 1246, 117, 0, 0, 1331, 1327, + 1332, 1328, 1333, 1326, 1325, 1334, 1330, 0, 0, 0, + 366, 399, 398, 397, 396, 401, 1611, 1293, 0, 205, + 448, 449, 0, 0, 0, 0, 0, 1304, 108, 106, + 1611, 1452, 428, 429, 0, 418, 414, 416, 0, 0, + 1611, 1271, 438, 434, 1611, 438, 1238, 1611, 0, 0, + 212, 0, 394, 1340, 1377, 2000, 1391, 0, 1392, 1382, + 1346, 1378, 1379, 155, 0, 484, 1317, 1415, 0, 0, + 0, 1152, 747, 752, 0, 0, 765, 0, 1171, 0, + 1177, 0, 0, 0, 747, 532, 0, 707, 764, 110, + 0, 745, 746, 635, 635, 589, 0, 570, 757, 0, + 0, 760, 758, 0, 760, 0, 0, 0, 760, 756, + 715, 0, 635, 0, 745, 748, 635, 0, 767, 1337, + 0, 0, 0, 0, 0, 1461, 1459, 1460, 1465, 0, + 0, 0, 1248, 1250, 1251, 1120, 1261, 1010, 0, 1823, + 1824, 1825, 1192, 1826, 1827, 1829, 1830, 1831, 969, 1576, + 1832, 1259, 1834, 1836, 1837, 1839, 1840, 1841, 1842, 1843, + 1844, 0, 1260, 1847, 1680, 1852, 1853, 1855, 1858, 1859, + 1258, 1864, 0, 0, 0, 1227, 1143, 0, 1009, 0, + 0, 0, 1193, 1201, 1002, 0, 0, 811, 812, 833, + 834, 813, 839, 840, 842, 814, 0, 1223, 903, 998, + 1211, 1007, 1015, 1011, 1050, 1013, 1030, 1016, 1087, 1008, + 0, 1014, 1000, 1219, 570, 1217, 0, 1001, 1247, 570, + 1215, 1396, 1394, 1400, 1395, 0, 0, 0, 0, 0, + 110, 1440, 1439, 1431, 1429, 1430, 1428, 1427, 1434, 0, + 1436, 1343, 1138, 1140, 0, 562, 0, 0, 0, 514, + 513, 515, 3, 0, 0, 0, 0, 582, 583, 0, + 0, 0, 0, 0, 0, 0, 0, 678, 609, 610, + 612, 675, 679, 687, 0, 0, 0, 0, 0, 521, + 0, 1276, 1479, 1473, 1471, 0, 0, 0, 139, 139, + 0, 0, 0, 0, 0, 99, 48, 92, 0, 0, + 0, 0, 234, 247, 0, 0, 0, 0, 0, 244, + 0, 0, 227, 50, 221, 223, 0, 139, 0, 46, + 0, 0, 0, 52, 1455, 0, 484, 1199, 0, 119, + 120, 118, 111, 0, 2014, 1876, 1877, 1878, 1879, 1829, + 1880, 1881, 0, 1882, 1883, 1835, 1885, 1886, 1887, 1888, + 1889, 1890, 1891, 1892, 1842, 1894, 1895, 1896, 1897, 1898, + 1899, 2040, 1900, 1856, 1902, 1862, 0, 1903, 1023, 1146, + 594, 1144, 1277, 0, 112, 1264, 0, 1329, 0, 0, + 0, 0, 482, 0, 0, 0, 0, 1289, 0, 1611, + 206, 210, 0, 1611, 201, 1611, 366, 0, 1611, 366, + 1611, 0, 1303, 1306, 0, 431, 426, 424, 423, 425, + 1611, 253, 0, 0, 1272, 436, 437, 0, 405, 0, + 0, 407, 0, 0, 217, 0, 215, 0, 401, 155, + 0, 228, 1387, 1388, 1386, 0, 0, 1381, 1345, 231, + 248, 1390, 1380, 1389, 1344, 1339, 0, 0, 1335, 471, + 0, 0, 0, 0, 1153, 880, 879, 863, 864, 877, + 878, 865, 866, 873, 874, 882, 881, 871, 872, 867, + 868, 861, 862, 869, 870, 875, 876, 859, 860, 1166, + 1154, 1155, 1156, 1157, 1158, 1159, 1160, 1161, 1162, 1163, + 1164, 1165, 0, 0, 706, 704, 0, 0, 0, 0, + 0, 0, 1193, 0, 973, 1008, 0, 0, 0, 1138, + 1176, 0, 0, 0, 0, 0, 0, 1138, 1182, 0, + 0, 731, 743, 0, 628, 634, 705, 703, 0, 1216, + 698, 0, 777, 0, 757, 0, 756, 0, 0, 759, + 753, 0, 754, 0, 0, 0, 0, 755, 0, 0, + 0, 0, 0, 701, 0, 743, 0, 702, 774, 1421, + 1420, 1416, 1403, 1411, 195, 0, 1268, 1904, 1905, 1906, + 821, 1907, 850, 828, 850, 850, 1908, 1909, 1910, 1911, + 817, 817, 830, 1912, 1913, 1914, 1915, 1916, 818, 819, + 855, 1917, 1918, 1919, 1920, 1921, 0, 0, 1922, 850, + 1923, 817, 1924, 1925, 1926, 822, 1927, 785, 1928, 0, + 1929, 820, 786, 1930, 858, 858, 1931, 0, 845, 1932, + 0, 1149, 795, 803, 804, 805, 806, 831, 832, 807, + 837, 838, 808, 902, 0, 817, 1269, 1270, 155, 1466, + 1482, 0, 1143, 1017, 849, 836, 1191, 0, 844, 843, + 0, 1143, 826, 825, 824, 1004, 0, 823, 1100, 850, + 850, 848, 928, 827, 0, 0, 0, 0, 0, 854, + 0, 852, 929, 907, 908, 0, 1226, 1235, 1138, 1142, + 0, 1002, 1138, 0, 0, 1090, 1092, 0, 1019, 1020, + 0, 1194, 1249, 1003, 0, 1254, 0, 0, 902, 902, + 1222, 1120, 0, 1110, 1113, 0, 0, 1117, 1118, 1119, + 0, 0, 0, 1214, 0, 1128, 1130, 0, 0, 944, + 1126, 0, 947, 0, 0, 0, 0, 1114, 1115, 1116, + 1106, 1107, 1108, 1109, 1111, 1112, 1124, 1105, 925, 0, + 999, 0, 1053, 0, 924, 1220, 696, 0, 1252, 696, + 1405, 1409, 1410, 1404, 1408, 0, 1399, 1398, 1401, 1402, + 1484, 0, 1441, 1425, 0, 1422, 1141, 691, 564, 1240, + 0, 568, 1446, 160, 159, 0, 0, 537, 536, 603, + 595, 597, 603, 0, 535, 0, 651, 652, 0, 0, + 0, 0, 684, 682, 1248, 1261, 639, 613, 638, 0, + 0, 617, 0, 643, 903, 677, 519, 607, 608, 611, + 518, 0, 680, 0, 690, 0, 556, 558, 541, 555, + 553, 538, 546, 678, 612, 0, 1448, 1472, 0, 0, + 0, 0, 0, 1611, 0, 0, 788, 83, 64, 318, + 138, 0, 0, 0, 0, 0, 0, 0, 91, 88, + 89, 90, 0, 0, 0, 0, 1268, 232, 233, 246, + 0, 237, 238, 235, 239, 240, 0, 0, 225, 226, + 0, 0, 0, 0, 224, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1456, 1449, 1195, 1200, 594, 594, + 594, 0, 592, 593, 0, 0, 0, 0, 0, 470, + 364, 374, 0, 0, 0, 1293, 205, 0, 0, 0, + 0, 0, 0, 401, 1296, 1294, 1292, 1295, 1297, 1582, + 189, 0, 0, 0, 0, 0, 197, 200, 0, 363, + 337, 0, 0, 1308, 0, 0, 0, 1611, 353, 1305, + 0, 1453, 0, 0, 251, 438, 1273, 0, 435, 438, + 1239, 0, 438, 219, 0, 0, 1341, 1383, 229, 249, + 230, 250, 484, 479, 509, 0, 487, 492, 468, 0, + 468, 0, 489, 493, 468, 488, 0, 468, 483, 1417, + 0, 1046, 0, 1036, 0, 0, 766, 0, 0, 1037, + 975, 976, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 993, 992, 1038, 770, 0, 773, 0, 0, 1174, 1175, + 0, 1039, 0, 0, 1181, 0, 0, 0, 1044, 0, + 708, 0, 0, 0, 623, 627, 630, 0, 633, 570, + 526, 1602, 1642, 0, 581, 581, 581, 579, 569, 0, + 655, 0, 0, 0, 732, 0, 0, 734, 736, 0, + 0, 739, 0, 714, 713, 0, 0, 0, 0, 778, + 0, 1244, 0, 0, 196, 0, 0, 0, 803, 0, + 0, 0, 793, 789, 0, 883, 884, 885, 886, 887, + 888, 889, 890, 891, 892, 893, 894, 809, 1281, 0, + 815, 1284, 1285, 1286, 1283, 1280, 1287, 1288, 0, 0, + 0, 0, 1190, 1186, 0, 0, 0, 0, 1095, 1097, + 1099, 0, 847, 846, 1104, 1110, 1113, 1117, 1118, 1119, + 1114, 1115, 1116, 1106, 1107, 1108, 1109, 1111, 1112, 0, + 1132, 0, 1086, 0, 0, 0, 0, 0, 0, 1225, + 0, 971, 0, 1021, 1006, 0, 0, 1093, 1022, 1227, + 1202, 0, 0, 0, 1257, 1256, 904, 913, 916, 948, + 949, 920, 921, 922, 926, 1279, 1278, 1221, 0, 1213, + 0, 0, 905, 930, 935, 0, 1183, 965, 0, 953, + 0, 943, 0, 951, 955, 931, 946, 0, 927, 0, + 1214, 1129, 1131, 0, 1127, 0, 917, 918, 919, 909, + 910, 911, 912, 914, 915, 923, 1103, 1101, 1102, 0, + 1200, 0, 1212, 0, 0, 1055, 0, 0, 950, 1218, + 0, 777, 594, 777, 0, 902, 1442, 1276, 1435, 1276, + 1424, 1139, 1241, 1275, 566, 0, 0, 0, 1444, 146, + 150, 0, 1201, 180, 182, 696, 0, 601, 602, 606, + 0, 0, 606, 585, 534, 1853, 1735, 0, 0, 0, + 0, 644, 685, 0, 676, 641, 642, 0, 640, 1248, + 645, 1247, 646, 649, 650, 618, 1236, 686, 688, 0, + 681, 0, 1242, 540, 559, 0, 0, 0, 0, 0, + 523, 522, 692, 0, 49, 0, 1611, 66, 0, 0, + 0, 0, 0, 0, 268, 0, 368, 268, 104, 1611, + 438, 1611, 438, 1506, 1577, 1753, 0, 62, 342, 95, + 0, 132, 371, 0, 327, 85, 100, 125, 0, 0, + 51, 222, 236, 241, 128, 245, 242, 1313, 243, 139, + 0, 47, 0, 126, 0, 1311, 0, 0, 53, 130, + 1315, 1457, 0, 1199, 0, 592, 592, 592, 0, 1145, + 0, 0, 0, 1147, 1148, 943, 1323, 1322, 1324, 1321, + 456, 469, 0, 365, 0, 481, 459, 460, 470, 1291, + 210, 0, 201, 366, 0, 366, 0, 1293, 0, 0, + 191, 187, 205, 211, 0, 0, 0, 0, 0, 364, + 356, 354, 387, 0, 361, 355, 0, 0, 313, 0, + 1500, 0, 0, 0, 0, 450, 0, 0, 0, 0, + 253, 254, 404, 1274, 406, 0, 408, 218, 216, 1336, + 476, 1143, 0, 474, 480, 475, 478, 473, 472, 0, + 467, 0, 502, 0, 0, 0, 0, 0, 0, 0, + 0, 1033, 1151, 0, 1169, 1168, 974, 981, 984, 988, + 989, 990, 1170, 0, 0, 0, 985, 986, 987, 977, + 978, 979, 980, 982, 983, 991, 775, 0, 0, 769, + 1179, 1178, 1172, 1173, 0, 1041, 1042, 1043, 1180, 0, + 0, 744, 621, 619, 622, 624, 620, 0, 0, 777, + 581, 581, 581, 581, 578, 0, 0, 0, 776, 0, + 672, 740, 738, 0, 762, 0, 735, 718, 741, 0, + 726, 0, 733, 782, 749, 0, 0, 751, 1412, 799, + 0, 794, 790, 0, 0, 0, 800, 0, 0, 0, + 0, 0, 0, 0, 1150, 587, 1018, 0, 0, 0, + 1187, 0, 970, 816, 829, 0, 1098, 1012, 0, 1121, + 1085, 857, 856, 858, 858, 0, 0, 0, 1234, 0, + 1139, 1089, 1091, 1235, 1005, 841, 902, 0, 0, 0, + 0, 0, 0, 0, 954, 945, 0, 952, 956, 0, + 0, 0, 939, 0, 0, 937, 966, 933, 0, 0, + 967, 1199, 0, 1203, 0, 0, 1054, 1063, 699, 695, + 655, 592, 655, 0, 1406, 1426, 1423, 567, 155, 1445, + 0, 169, 0, 0, 0, 0, 172, 186, 183, 1444, + 0, 0, 596, 598, 0, 1122, 606, 600, 648, 647, + 0, 616, 683, 614, 0, 689, 0, 557, 0, 543, + 0, 717, 0, 0, 0, 0, 0, 317, 0, 0, + 0, 268, 0, 376, 0, 383, 0, 0, 368, 349, + 84, 0, 0, 0, 58, 103, 76, 68, 54, 82, + 0, 0, 87, 0, 80, 97, 98, 96, 101, 0, + 278, 303, 0, 0, 314, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 484, 1200, 1196, 1200, + 0, 0, 0, 594, 590, 591, 1024, 0, 455, 508, + 505, 506, 504, 227, 375, 0, 0, 0, 199, 363, + 0, 1308, 0, 1290, 401, 0, 192, 0, 190, 210, + 0, 0, 201, 366, 0, 341, 337, 362, 335, 334, + 336, 0, 1501, 220, 0, 1495, 366, 1307, 0, 0, + 451, 0, 445, 0, 1302, 252, 438, 0, 463, 503, + 510, 490, 495, 0, 501, 497, 496, 491, 499, 498, + 494, 1034, 1045, 1167, 0, 0, 0, 0, 768, 771, + 0, 1040, 1035, 742, 0, 0, 655, 0, 0, 0, + 0, 572, 571, 577, 0, 0, 1057, 737, 0, 0, + 0, 724, 712, 719, 720, 0, 0, 0, 780, 779, + 750, 803, 0, 783, 803, 0, 803, 0, 801, 0, + 810, 895, 896, 897, 898, 899, 900, 901, 835, 0, + 1189, 1185, 1094, 1096, 1133, 853, 851, 1224, 1138, 1229, + 1231, 0, 0, 0, 1088, 972, 1255, 906, 0, 0, + 936, 1184, 957, 0, 0, 0, 932, 1121, 0, 0, + 0, 0, 0, 941, 0, 1207, 1200, 0, 1206, 0, + 0, 0, 0, 1029, 700, 672, 0, 672, 0, 0, + 1443, 0, 1438, 147, 148, 149, 0, 0, 0, 164, + 141, 0, 0, 181, 169, 157, 604, 605, 0, 599, + 615, 1237, 1243, 542, 0, 1002, 0, 0, 539, 0, + 133, 268, 0, 0, 65, 0, 385, 329, 377, 360, + 344, 0, 0, 0, 269, 0, 402, 0, 0, 350, + 0, 0, 0, 0, 330, 0, 0, 289, 0, 0, + 360, 0, 367, 285, 286, 0, 57, 77, 0, 73, + 0, 102, 0, 0, 0, 0, 0, 60, 72, 0, + 55, 0, 438, 438, 63, 1268, 1904, 1905, 1906, 1907, + 1908, 1909, 1910, 1911, 1912, 1913, 2024, 1914, 1915, 1916, + 1917, 1918, 1919, 1920, 1921, 2033, 1922, 275, 1923, 1680, + 1924, 1925, 1926, 1927, 1928, 0, 1929, 786, 1930, 1931, + 2112, 1932, 1106, 1107, 274, 273, 370, 270, 378, 272, + 0, 1269, 271, 373, 328, 129, 1314, 0, 127, 0, + 1312, 136, 134, 131, 1316, 1450, 0, 0, 1027, 1028, + 1025, 592, 0, 0, 0, 484, 462, 0, 0, 0, + 1500, 0, 0, 0, 1611, 0, 188, 0, 0, 202, + 1308, 198, 363, 0, 393, 313, 388, 0, 1500, 1498, + 0, 1308, 1494, 0, 442, 0, 0, 0, 409, 477, + 0, 500, 994, 0, 0, 0, 0, 631, 0, 637, + 672, 576, 575, 574, 573, 654, 1551, 1836, 1734, 0, + 658, 653, 656, 661, 663, 662, 664, 660, 671, 0, + 674, 761, 1134, 1136, 0, 0, 0, 0, 725, 727, + 0, 729, 0, 781, 797, 0, 798, 0, 796, 791, + 802, 1188, 1232, 1233, 1228, 0, 903, 963, 961, 958, + 0, 959, 940, 0, 0, 938, 934, 0, 968, 0, + 0, 1204, 0, 1049, 0, 1052, 1066, 1062, 1061, 1057, + 1024, 1057, 1407, 565, 168, 145, 171, 170, 0, 1201, + 178, 0, 0, 169, 0, 173, 452, 0, 0, 554, + 716, 547, 548, 0, 381, 67, 0, 360, 0, 268, + 346, 345, 348, 343, 347, 0, 403, 0, 0, 287, + 0, 294, 332, 333, 331, 288, 360, 366, 290, 0, + 0, 0, 69, 59, 56, 61, 70, 0, 0, 71, + 74, 782, 86, 79, 1268, 2033, 2042, 0, 0, 0, + 0, 0, 1198, 1197, 0, 458, 457, 507, 454, 465, + 227, 0, 0, 0, 337, 1497, 0, 0, 447, 0, + 0, 363, 193, 0, 0, 0, 0, 1500, 0, 0, + 265, 0, 310, 0, 213, 1499, 0, 0, 1486, 0, + 0, 1300, 1301, 0, 464, 995, 0, 996, 772, 0, + 0, 629, 1057, 0, 0, 0, 665, 659, 0, 1056, + 1058, 0, 626, 1137, 721, 0, 723, 0, 747, 0, + 747, 730, 792, 784, 1230, 1047, 0, 960, 964, 962, + 942, 1200, 1208, 1200, 1205, 1051, 1065, 1068, 674, 1253, + 674, 0, 0, 156, 0, 0, 153, 140, 158, 1123, + 544, 545, 0, 268, 0, 359, 382, 299, 277, 0, + 0, 0, 284, 291, 392, 293, 0, 78, 94, 0, + 0, 372, 137, 135, 1026, 484, 0, 204, 1308, 313, + 1494, 444, 0, 0, 0, 0, 337, 220, 1496, 326, + 319, 320, 321, 322, 323, 324, 325, 340, 339, 311, + 312, 0, 0, 0, 0, 0, 446, 1302, 0, 175, + 184, 0, 175, 997, 632, 0, 674, 0, 0, 0, + 657, 0, 0, 673, 0, 530, 1135, 0, 711, 709, + 0, 710, 0, 0, 0, 0, 594, 626, 626, 142, + 0, 143, 179, 0, 0, 0, 366, 384, 358, 0, + 351, 297, 296, 298, 302, 0, 300, 0, 316, 0, + 309, 277, 0, 81, 0, 379, 453, 461, 0, 267, + 1488, 363, 0, 203, 1494, 313, 1500, 1494, 0, 1491, + 0, 443, 0, 0, 0, 177, 1308, 0, 177, 0, + 626, 667, 0, 666, 1060, 1059, 628, 722, 0, 1048, + 1210, 1209, 0, 1072, 529, 528, 0, 0, 0, 0, + 392, 0, 338, 0, 0, 299, 0, 292, 389, 390, + 391, 0, 305, 295, 306, 75, 93, 380, 0, 363, + 1489, 266, 214, 1487, 1492, 1493, 0, 175, 174, 603, + 176, 777, 185, 603, 636, 531, 668, 625, 728, 1067, + 0, 0, 0, 0, 0, 152, 777, 163, 0, 309, + 357, 352, 276, 301, 315, 0, 0, 0, 307, 0, + 308, 1494, 0, 177, 606, 1298, 606, 1822, 1552, 1788, + 0, 1084, 1073, 1084, 1084, 1064, 144, 151, 0, 268, + 281, 0, 280, 0, 369, 304, 1490, 1308, 603, 165, + 166, 0, 1077, 1076, 1075, 1079, 1078, 0, 1071, 1069, + 1070, 777, 386, 279, 283, 282, 777, 606, 0, 0, + 1081, 0, 1082, 162, 1299, 167, 1074, 1080, 1083 }; /* YYDEFGOTO[NTERM-NUM]. */ static const yytype_int16 yydefgoto[] = { - -1, 48, 49, 50, 750, 2593, 2594, 2595, 2235, 1204, - 3360, 2236, 1205, 1206, 2597, 751, 801, 1092, 803, 1093, - 1602, 905, 1238, 1239, 752, 1751, 753, 2816, 2159, 2541, - 3342, 55, 3087, 2162, 1164, 3090, 3307, 2809, 3085, 2542, - 3382, 3436, 3088, 2163, 2164, 3308, 2165, 754, 2654, 2655, - 755, 756, 1835, 59, 1300, 546, 1832, 757, 1333, 1334, - 960, 758, 1836, 1779, 2932, 1224, 1769, 1348, 62, 1853, - 759, 106, 64, 760, 2582, 2933, 3353, 2608, 3491, 2869, - 2870, 3350, 3351, 2585, 2238, 3419, 3420, 2669, 1760, 3414, - 2319, 3294, 2242, 2223, 2871, 2327, 3252, 2981, 2239, 2851, - 2320, 3346, 1848, 2321, 3347, 3106, 2322, 1810, 1839, 2586, - 3421, 2243, 1811, 2581, 2934, 1748, 2323, 3357, 2324, 547, - 2855, 761, 741, 742, 952, 1327, 743, 762, 936, 1845, - 763, 764, 2635, 2297, 3157, 2684, 3158, 2358, 2291, 1357, - 2352, 1873, 1813, 1358, 535, 1887, 2685, 2640, 1874, 765, - 1094, 72, 73, 1007, 74, 3100, 75, 76, 1725, 1726, - 1727, 848, 860, 861, 2155, 1441, 1957, 853, 1168, 1694, - 835, 836, 2281, 876, 1802, 1689, 1690, 2168, 2549, 1718, - 1719, 1177, 1178, 1945, 3322, 1946, 1947, 1434, 1435, 3198, - 1706, 1710, 1711, 2189, 2179, 1697, 2427, 3017, 3018, 3019, - 3020, 3021, 3022, 3023, 1095, 2723, 3209, 1714, 1715, 1180, - 1181, 1182, 1723, 2199, 78, 79, 2140, 2525, 2526, 807, - 3034, 1460, 1728, 2727, 2728, 2729, 3037, 3038, 3039, 808, - 1002, 1003, 1026, 1021, 1449, 1966, 809, 810, 1922, 1923, - 2396, 1028, 1959, 1977, 1978, 2735, 2451, 1529, 2224, 1530, - 1531, 1992, 1532, 1096, 1533, 1561, 1097, 1566, 1535, 1098, - 1099, 1100, 1538, 1101, 1102, 1103, 1104, 1554, 1105, 1106, - 1578, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, - 2003, 2004, 2005, 2006, 1151, 1729, 1108, 1109, 1110, 1111, - 1112, 1113, 1114, 812, 1115, 1116, 1651, 2134, 2524, 3027, - 3206, 3207, 2800, 3075, 3234, 3333, 3450, 3478, 3479, 3505, - 1117, 1118, 1594, 1595, 1596, 2027, 2028, 2029, 2030, 2128, - 1645, 1646, 1119, 2936, 1648, 2050, 3030, 3031, 1152, 1427, - 1589, 1279, 1280, 1543, 1401, 1402, 1408, 1897, 1416, 1420, - 1927, 1928, 1428, 2096, 1120, 2021, 2022, 2468, 1556, 1121, - 1237, 1601, 2795, 2131, 1649, 2090, 1128, 1122, 1129, 1124, - 1585, 1586, 2485, 2767, 2768, 2060, 2196, 1678, 2201, 2202, - 956, 1125, 1126, 1127, 1281, 519, 1544, 3437, 1323, 1157, - 1282, 2086, 766, 1034, 2014, 767, 1296, 1825, 768, 3189, - 2994, 1312, 1849, 2332, 548, 769, 770, 528, 85, 2286, - 917, 86, 87, 88, 885, 1350, 771, 1351, 1352, 967, - 89, 2686, 969, 970, 773, 842, 843, 1472, 1665, 1473, - 774, 818, 1470, 775, 1147, 857, 1148, 1150, 776, 1141, - 2538, 2157, 94, 95, 96, 114, 1235, 777, 829, 830, - 866, 99, 100, 1192, 831, 849, 779, 780, 3185, 781, - 2672, 1306, 529, 521, 522, 1546, 715, 1284, 716 + -1, 48, 49, 50, 750, 2596, 2597, 2598, 2236, 1205, + 3363, 2237, 1206, 1207, 2600, 751, 801, 1092, 803, 1093, + 1603, 905, 1239, 1240, 752, 1752, 753, 2819, 2160, 2544, + 3345, 55, 3090, 2163, 1165, 3093, 3310, 2812, 3088, 2545, + 3385, 3439, 3091, 2164, 2165, 3311, 2166, 754, 2657, 2658, + 755, 756, 1836, 59, 1301, 546, 1833, 757, 1334, 1335, + 960, 758, 1837, 1780, 2935, 1225, 1770, 1349, 62, 1854, + 759, 106, 64, 760, 2585, 2936, 3356, 2611, 3494, 2872, + 2873, 3353, 3354, 2588, 2239, 3422, 3423, 2672, 1761, 3417, + 2320, 3297, 2243, 2224, 2874, 2328, 3255, 2984, 2240, 2854, + 2321, 3349, 1849, 2322, 3350, 3109, 2323, 1811, 1840, 2589, + 3424, 2244, 1812, 2584, 2937, 1749, 2324, 3360, 2325, 547, + 2858, 761, 741, 742, 952, 1328, 743, 762, 936, 1846, + 763, 764, 2638, 2298, 3160, 2687, 3161, 2361, 2292, 1358, + 2354, 1874, 1814, 1359, 535, 1888, 2688, 2643, 1875, 765, + 1094, 72, 73, 1007, 74, 3103, 75, 76, 1726, 1727, + 1728, 848, 860, 861, 2156, 1442, 1958, 853, 1169, 1695, + 835, 836, 2282, 876, 1803, 1690, 1691, 2169, 2552, 1719, + 1720, 1178, 1179, 1946, 3325, 1947, 1948, 1435, 1436, 3201, + 1707, 1711, 1712, 2190, 2180, 1698, 2430, 3020, 3021, 3022, + 3023, 3024, 3025, 3026, 1095, 2726, 3212, 1715, 1716, 1181, + 1182, 1183, 1724, 2200, 78, 79, 2141, 2528, 2529, 807, + 3037, 1461, 1729, 2730, 2731, 2732, 3040, 3041, 3042, 808, + 1002, 1003, 1026, 1021, 1450, 1967, 809, 810, 1923, 1924, + 2399, 1028, 1960, 1978, 1979, 2738, 2454, 1530, 2225, 1531, + 1532, 1993, 1533, 1096, 1534, 1562, 1097, 1567, 1536, 1098, + 1099, 1100, 1539, 1101, 1102, 1103, 1104, 1555, 1105, 1106, + 1579, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, + 2004, 2005, 2006, 2007, 1152, 1730, 1108, 1109, 1110, 1111, + 1112, 1113, 1114, 1115, 812, 1116, 1117, 1652, 2135, 2527, + 3030, 3209, 3210, 2803, 3078, 3237, 3336, 3453, 3481, 3482, + 3508, 1118, 1119, 1595, 1596, 1597, 2028, 2029, 2030, 2031, + 2129, 1646, 1647, 1120, 2939, 1649, 2051, 3033, 3034, 1153, + 1428, 1590, 1280, 1281, 1544, 1402, 1403, 1409, 1898, 1417, + 1421, 1928, 1929, 1429, 2097, 1121, 2022, 2023, 2471, 1557, + 1122, 1238, 1602, 2798, 2132, 1650, 2091, 1129, 1123, 1130, + 1125, 1586, 1587, 2488, 2770, 2771, 2061, 2197, 1679, 2202, + 2203, 956, 1126, 1127, 1128, 1282, 519, 1545, 3440, 1324, + 1158, 1283, 2087, 766, 1034, 2015, 767, 1297, 1826, 768, + 3192, 2997, 1313, 1850, 2333, 548, 769, 770, 528, 85, + 2287, 917, 86, 87, 88, 885, 1351, 771, 1352, 1353, + 967, 89, 2689, 969, 970, 773, 842, 843, 1473, 1666, + 1474, 774, 818, 1471, 775, 1148, 857, 1149, 1151, 776, + 1142, 2541, 2158, 94, 95, 96, 114, 1236, 777, 829, + 830, 866, 99, 100, 1193, 831, 849, 779, 780, 3188, + 781, 2675, 1307, 529, 521, 522, 1547, 715, 1285, 716 }; /* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing STATE-NUM. */ -#define YYPACT_NINF -2987 +#define YYPACT_NINF -3059 static const int yypact[] = { - 6315, -13, 722, -2987, -2987, 285, -13, 49878, 64713, 72, - -13, 172, 4065, 51858, -2987, -2987, 46413, 41312, -13, 54828, - 72049, 349, 300, 31840, 382, 55323, 55323, -2987, -2987, -2987, - 64713, 54828, 55818, -13, 340, 65208, -2987, -13, 34315, 52353, - -27, -2987, 54828, 34, 16, 56313, 54828, 2732, 576, 224, - -2987, -2987, -2987, -2987, -2987, 126, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, 144, -2987, 116, 147, 31840, 31840, 1267, 318, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, 311, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - 33820, -2987, -2987, -2987, -2987, -2987, -2987, 56808, 54828, 57303, - 52848, 57798, -2987, 613, 819, -2987, 160, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, 181, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, 396, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, 188, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, 381, -2987, 402, -2987, - 189, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - 1569, -2987, -2987, 818, 2426, 54828, 20, 695, 580, -2987, - 58293, -2987, 591, 54828, -2987, -2987, 638, 856, 855, -2987, - -2987, 53343, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, 46908, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, 815, -2987, -2987, - 594, -2987, 154, -2987, -2987, 605, 625, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, 820, -2987, -2987, -2987, - 835, 65703, 58788, 59283, -2987, 704, 3136, 7105, 72067, 30848, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, 311, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, 55323, 64713, 55323, 717, 729, 1083, 739, 32335, - 756, 34811, 772, 781, 1128, 795, 800, 824, 830, 16, - 31344, 822, 381, -2987, 59778, 59778, -10, 1763, -2987, 59778, - 60273, -2987, 852, -2987, 819, -2987, -2987, -2987, 1185, -2987, - 419, 853, -2987, 60768, 60768, 60768, 877, 1157, -2987, -2987, - -2987, 875, -2987, -2987, 1116, 20517, 20517, 66198, 66198, 819, - 66198, 925, -2987, -2987, 37, -2987, -2987, -2987, 1267, 911, - 381, -2987, -2987, 52353, -2987, -2987, 225, 1287, 20517, 54828, - 943, -2987, 957, 943, 967, 969, 978, -2987, 6315, 1298, - 1209, 52353, 343, 343, 1463, 343, 1056, 1084, 6398, 6686, - -2987, 1859, -2987, 1019, -2987, 54828, 1124, 1049, 1325, -2987, - 911, 1409, 185, 1213, 1419, 4415, 1428, 559, 1443, 985, - 1450, 1576, 20517, 47403, 381, -2987, 11660, 20517, -2987, -2987, - -2987, 1207, -2987, -2987, -2987, -2987, -2987, 54828, 64713, 1112, - 1125, -2987, -2987, -2987, -2987, 1358, 1367, -2987, 1603, 66693, - -2987, -2987, 1179, 61263, 61758, 62253, 62748, 1561, -2987, -2987, - 1504, -2987, -2987, -2987, 1186, -2987, -2987, -2987, 332, 67188, - 1518, 1164, 175, -2987, 1530, 212, -2987, 1533, 1407, 15307, - -2987, 1346, -2987, -2987, -2987, 16, -2987, 284, -2987, -2987, - 43357, -2987, -2987, 72067, 1272, 1195, -2987, 1558, 20517, 20517, - 1223, 7242, 59778, 60273, 20517, 54828, -2987, 20517, 25206, 1227, - 20517, 20517, 12702, 20517, 29858, 59778, 1763, 1231, -2987, 588, - 54828, 1242, -2987, 1334, 1334, 340, 31840, 1540, -2987, 219, - 1568, 1467, -2987, 31840, 1467, 916, 1280, 1580, 1467, -2987, - 598, 1583, 1334, 35306, 1301, -2987, 1334, 1505, -2987, -2987, - 55323, 20517, 15307, 69663, 1779, -2987, -2987, -2987, -2987, 1596, - 64713, 1308, -2987, -2987, -2987, -2987, -2987, -2987, 669, 1827, - 141, 1830, 20517, 141, 141, 1329, 190, 190, -2987, 1524, - 1336, -2987, 195, 1337, 1338, 1849, 1850, 167, 163, 353, - 141, 20517, -2987, 190, 1345, 1855, 1347, 1857, 156, 187, - -2987, 199, 20517, 20517, 20517, 1715, 20517, 10618, -2987, 54828, - 1858, 47403, 692, -2987, 381, 1352, 819, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, 1354, -2987, 180, 6820, -2987, -2987, - -2987, -2987, -2987, 1392, -2987, -2987, -2987, -2987, 1572, 20517, - -2987, -2987, 1356, 1540, -2987, 204, -2987, -2987, 1540, -2987, - -2987, -2987, -2987, -2987, 220, 1766, 20517, 20517, 64713, 381, - 67683, -2987, -2987, -2987, -2987, -2987, -2987, -2987, 441, -2987, - 311, 45045, 1357, 1361, 943, 54828, 54828, 1837, -2987, -2987, - -2987, -2987, 52353, 142, 1660, 1492, -2987, -2987, 1267, 1267, - 15828, 737, 487, 1054, 16349, 21038, 1714, 1597, 227, 218, - 1719, -2987, 1600, 1829, 25206, 20517, 20517, 1056, 1084, 20517, - 957, -2987, -2987, -2987, 1655, 54828, 50373, 937, 941, 1377, - 1464, 1379, 81, 1808, -2987, 1384, -2987, 1485, 54828, 71598, - 192, -2987, 1856, 192, 192, 251, 1864, 1488, 272, 1652, - 362, 257, 1384, 202, -2987, 52353, 197, 761, 1384, 54828, - 1498, 788, 1384, 1820, 64713, 1195, 40944, 1406, -2987, -2987, - -2987, 164, 15307, -2987, 1156, 1311, 1320, 391, 153, 1373, - 1420, 15307, 1478, 1542, 168, 1577, 1587, 1601, 1659, 1665, - 1674, 1679, 1691, 149, 1710, 1712, 1718, 1720, 1726, 1730, - -2987, 1741, 170, 1760, 206, 15307, 1773, -2987, 45045, -1, - -2987, -2987, 1777, 174, -2987, 45134, -2987, 1713, 1508, 1509, - 64713, 1465, 54828, 1570, 865, 1793, 1846, 70143, 1675, -2987, - 1750, 54828, 1678, 202, 1680, 1439, 1916, 1684, 1125, 1685, - 1448, -2987, 68178, 47403, -2987, -2987, -2987, -2987, -2987, 1811, - 1794, 64713, 47403, 1452, -2987, -2987, 64713, -2987, 54828, 54828, - -2987, 54828, 64713, -2987, 526, 45045, 1957, 617, 72067, 48888, - -2987, -2987, -2987, -2987, 1143, 1201, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, 819, 47403, -2987, 3327, 55323, - 43975, 1459, 20517, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, 1461, 1807, -2987, -2987, 6597, 1466, 44015, 1468, 25206, - 25206, 381, 4715, -2987, -2987, 25206, 1469, 49383, 43888, 1462, - 1471, 44363, 16870, 20517, 16870, 16870, 44450, -2987, 1473, 44527, - 59778, 1475, 54828, 53838, -2987, -2987, -2987, 20517, 20517, 1763, - 54333, 1520, 31840, -2987, 31840, -2987, 1758, 31840, -2987, -2987, - 3920, -2987, 31840, 1770, 20517, 31840, -2987, 31840, 1717, 1721, - 1481, 31840, -2987, 54828, 1493, 54828, -2987, -2987, -2987, -2987, - -2987, 45045, -2987, 1491, 633, 1499, -2987, -2987, -2987, -2987, - -2987, 1551, -2987, 1551, 1551, -2987, -2987, -2987, -2987, 1502, - 1502, 1507, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, 1513, 353, -2987, 1551, -2987, - 1502, -2987, -2987, -2987, -2987, -2987, -2987, -2987, 71598, -2987, - -2987, -2987, -2987, 621, 645, -2987, 1514, -2987, -2987, 1515, - -2987, 1503, 1979, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, 8492, 742, 1502, -2987, -2987, 4693, -2987, -2987, - 20517, 20517, -2987, -2987, 1516, 45045, 1550, -2987, -2987, 20517, - 20517, -2987, -2987, -2987, -2987, 2024, -2987, 20517, 1551, 1551, - -2987, 5276, -2987, 39763, 17391, 1610, 1611, 2024, -2987, 2024, - -2987, 5276, 2033, 2033, 37781, -2987, 1695, 44614, -2987, 1536, - 1877, 7400, 1538, 1527, -2987, 1544, 1543, -2987, -2987, 41872, - 137, 381, 381, 20517, -2987, 2024, 20517, 8530, 8530, -2987, - 294, 69663, 20517, 20517, 20517, 20517, 20517, 20517, 20517, 20517, - 45918, 1628, 171, 64713, 20517, 20517, 1549, 881, -2987, 20517, - 1786, -2987, 1553, 20517, 1641, 896, 20517, 20517, 20517, 20517, - 20517, 20517, 20517, 20517, 20517, -2987, -2987, 28853, 347, 740, - 1890, 1909, -26, 1212, 20517, 1901, 11660, -2987, 1901, -2987, - -2987, -2987, -2987, -2987, 205, -2987, -2987, 1491, 1491, -2987, - 64713, -2987, 54828, 225, 51363, 20517, -2987, -2987, 1562, 1563, - 546, 1619, -2987, -2987, 54828, 38276, 1862, -2987, 357, 1567, - -2987, 43849, 1816, 1862, 1267, -2987, -2987, 26248, 1697, 1863, - 1800, -2987, -2987, 1781, 1785, -2987, 1573, 45267, 21559, 21559, - -2987, 1411, 45045, 1433, -2987, -2987, -2987, -2987, -2987, -2987, - 547, -2987, 54828, 78, 35801, -2987, 1581, 102, -2987, 5415, - 1920, 1883, 1714, 218, 1590, -2987, -2987, 996, 1593, 68673, - 54828, 1882, 1835, 1884, -45, 69663, -2987, -2987, -2987, -2987, - 54828, 64713, 63243, 69168, 47898, 54828, 47403, -2987, -2987, -2987, - -2987, 54828, 1102, 54828, 6396, -2987, -2987, -2987, -2987, 192, - -2987, -2987, -2987, -2987, -2987, 64713, 54828, -2987, -2987, 192, - 64713, 54828, 192, -2987, 1082, 54828, 54828, 54828, 54828, 1208, - 54828, 54828, 819, -2987, -2987, -2987, 22080, 8, 8, 1817, - 13223, 131, -2987, 20517, 20517, 901, 273, 64713, 1778, -2987, - -2987, 770, 1826, 200, -2987, 64713, 1649, 54828, 54828, 54828, - 54828, 54828, 1363, -2987, -2987, -2987, -2987, -2987, 1604, -2987, - 1965, 2116, 1606, 1614, 1978, -2987, 202, 1983, 50868, 159, - 2595, 1986, 1663, 1989, 13744, 2091, 1868, -2987, -2987, 1866, - -2987, 64713, 2141, -2987, 175, -2987, 47403, -2987, 212, -2987, - 1869, 241, -2987, 15307, 20517, -2987, -2987, -2987, -2987, -2987, - -2987, 1195, 29359, -2987, 776, -2987, -2987, 2108, 819, 2108, - 672, -2987, -2987, 2108, -2987, 2094, 2108, -2987, -2987, 69663, - -2987, 7699, -2987, 20517, 20517, -2987, 20517, 1994, -2987, 2146, - 2146, 69663, 25206, 25206, 25206, 25206, 25206, 25206, 213, 1345, - 25206, 25206, 25206, 25206, 25206, 25206, 25206, 25206, 25206, 26769, - 230, -2987, -2987, 786, 2119, 20517, 20517, 2001, 1994, 20517, - -2987, 69663, 1645, -2987, 1653, 1654, 20517, -2987, 69663, -2987, - 54828, 1656, -2, 561, -2987, 1664, 1669, -2987, 1540, -2987, - 810, 847, 54828, 2652, 3454, 3760, -2987, -2987, 20517, 1990, - 3920, 3920, 31840, -2987, 20517, 1670, -2987, -2987, 31840, 2007, - -2987, 3920, -2987, -2987, 36296, 3920, 69663, 825, -2987, 54828, - 69663, 833, 20517, -2987, 15307, 2171, 69663, 2138, 64713, 64713, - 2182, 1677, 1681, 2024, 1764, -2987, 1767, 1769, 1772, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, 69663, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, 1673, 1688, 20517, - 20517, 138, -2987, 7786, 1690, 1693, 5578, -2987, 1696, -2987, - 1682, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, 1702, -2987, - 1694, -2987, 1698, 1709, 1722, 1706, 1707, 54828, -2987, 22601, - -2987, 64713, -2987, -2987, 20517, 20517, 54828, -2987, 1715, -2987, - 1708, 1716, 8076, -2987, -2987, -2987, 239, 769, 8382, 1212, - 2227, 2227, 2227, 5276, -2987, -2987, -2987, 1731, -2987, 25206, - 25206, -2987, 2509, 256, 10618, -2987, -2987, 2058, -2987, 954, - -2987, 1727, -2987, -2987, 1988, -2987, 39763, 45338, 20517, 173, - -2987, 20517, 1549, 20517, 1802, 2227, 2227, 2227, 282, 282, - 239, 239, 239, 769, 1212, -2987, -2987, -2987, 1735, 20517, - 47403, -2987, 1739, 1752, 2077, 1347, 20517, -2987, -2987, 31840, - 1520, -1, 1520, 2024, 8530, -2987, 957, -2987, 957, -2987, - 45045, 54828, -2987, -2987, 1985, 1757, 31840, 1771, 2196, 2181, - 64713, -2987, -2987, 1745, 1901, 1742, -2987, -2987, 1776, 20517, - 1749, 1776, -2987, 1862, 5, 1982, 1202, 1202, 1411, 1987, - -2987, -2987, 1831, -2987, -2987, -2987, 20517, 14265, 1444, -2987, - 1447, -2987, -2987, -2987, -2987, -2987, 1765, -2987, 2035, -2987, - 54828, -2987, -2987, 25206, 2217, 20517, 36791, 2222, 2019, -2987, - -2987, -2987, 1860, 1384, 20517, 2011, -2987, 240, 1775, 2135, - 293, 2089, 64713, -2987, 281, 334, -2987, 755, 2142, 241, - 2144, 241, 47403, 47403, 47403, 843, -2987, -2987, -2987, 819, - -2987, 331, 854, -2987, -2987, -2987, -2987, 1865, 806, 1384, - 202, -2987, -2987, -2987, -2987, -2987, -2987, -2987, 198, 814, - 1384, 1870, -2987, 1872, -2987, 1874, 823, 1384, -2987, -2987, - 1576, 9053, 45045, 522, 131, 131, 131, 15307, -2987, 2016, - 2023, 1795, 45045, 45045, 135, -2987, -2987, -2987, -2987, 1801, - -2987, 210, -2987, 64713, -2987, -2987, -2987, 1778, 1846, 1750, - 54828, 202, 1803, 2268, 1125, 1448, -2987, 1962, 1043, 1676, - -2987, 64713, -2987, 47403, 64713, 54828, 54828, 54828, 63738, -2987, - -2987, -2987, 1804, 1805, -2987, 9, 2031, 2036, 54828, 1844, - 54828, 1379, 2282, 54828, -2987, 858, 17912, 2172, 54828, 1794, - -2987, -2987, -2987, -2987, 64713, -2987, -2987, 45045, -2987, -2987, - 48393, -2987, -2987, -2987, -2987, -2987, 47403, -2987, 819, -2987, - 819, 2057, 64713, 42367, 819, 42862, 819, 1822, -2987, 45045, - 8224, 45045, 2001, -2987, 148, 2146, 1825, 1825, 1825, 4327, - 2167, 215, 1828, 1825, 1825, 1825, 296, 296, 148, 148, - 148, 2146, 230, 852, 49383, 1832, -2987, 45045, 45045, -2987, - -2987, 1824, -2987, -2987, -2987, -2987, 1833, 1836, -2987, -2987, - -2987, -2987, -2987, -2987, 64713, 1238, 1520, -27, -27, -27, - -27, -2987, 54828, 54828, 54828, 45045, 2281, 2159, -2987, -2987, - 3920, 45045, 54828, -2987, 27811, -2987, 54828, -2987, 2179, -2987, - 2267, -2987, 54828, 863, -2987, -2987, -2987, 926, 1838, 1681, - 69663, 931, 955, -2987, 2024, 194, 1843, 1555, 1278, 492, - 1445, -2987, -2987, -2987, 1845, 44749, 20517, -2987, 2209, -2987, - -2987, -2987, 20517, 20517, -2987, 39763, -2987, -2987, -2987, -2987, - -50, -50, 1848, 10618, 44971, -2987, 2155, 8299, 45045, -2987, - 1695, -2987, -2987, 8530, 20517, 1205, 3014, 20517, 1851, 20517, - 2188, -2987, -2987, 1854, -2987, -2987, 69663, 20517, 1861, 2787, - 25206, 25206, 3657, -2987, 6043, 20517, 10618, -2987, 41030, 1840, - 1871, 1817, 18433, -2987, 2061, 1852, -2987, 1990, 131, 1990, - 1867, -2987, -2987, -2987, -2987, 4693, -2987, 20517, 2004, 64713, - 379, 2098, 961, -2987, 381, 38276, 1771, 20517, 247, -2987, - -2987, 1880, -2987, 1776, -2987, -2987, -2987, 2075, -2987, -2987, - -2987, 54828, -2987, 1873, -2987, 35801, 2189, 11139, -2987, 35801, - 54828, 54828, 40266, 2220, -2987, 64713, 64713, 64713, -2987, 64713, - 1878, 1876, 899, 1881, 752, -2987, 1847, 899, 2199, 651, - 1379, 272, 2556, 423, -2987, -2987, -2987, 1938, 54828, -2987, - 64713, -2987, -2987, -2987, -2987, -2987, 47898, -2987, -2987, 39267, - 47403, -2987, 47403, 54828, 54828, 54828, 54828, 54828, 54828, 54828, - 54828, 54828, 54828, 1195, 20517, -2987, 20517, 1887, 1891, 1892, - 1817, -2987, -2987, -2987, 223, -2987, 1889, -2987, -2987, -2987, - 257, -2987, 210, 1885, 1894, -2987, 50868, 2426, 1663, 2338, - 1846, 767, 64218, -2987, 1895, 1879, 1750, 963, 965, 202, - 1897, 2340, -2987, 159, 50868, -2987, -2987, -2987, 2297, -2987, - 704, 209, -2987, 1125, -2987, 2426, 1448, -2987, 2426, 45045, - 64713, 1932, -2987, 241, 971, -2987, -2987, -2987, -2987, -2987, - 64713, 1899, -2987, 1899, -2987, -2987, 1899, -2987, -2987, -2987, - -2987, 25206, 2238, 1905, 69663, -2987, -2987, 54828, -2987, -2987, - -2987, 1000, 1900, 1990, 54828, 54828, 54828, 54828, -2987, -2987, - -2987, 18954, 20517, 1946, -2987, 1907, 12181, 2225, -2987, 27290, - -2987, -2987, 1913, 36296, 64713, -2987, -2987, -2987, -2987, 2024, - -2987, -2987, 64713, -2987, 1922, -2987, 1923, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, 20517, 45045, -2987, 45045, - -2987, -2987, -2987, -2987, -2987, 7353, -2987, 1919, 1925, 64713, - 20517, -2987, -2987, -2987, 773, 20517, 20517, 2509, -2987, 45429, - 20517, 69663, 1002, 2509, 348, 20517, 5175, 5705, 20517, 20517, - 6302, 40305, -2987, 23122, 14786, -2987, 1926, 20517, 40344, 38771, - -2987, 31840, 2159, 1927, 2159, 819, 1931, 45045, 20517, -2987, - -2987, -2987, -2987, 1991, 67, 33325, 2149, -2987, 1944, 64713, - -2987, 2004, 45045, -2987, -2987, 39763, -2987, -2987, -2987, -2987, - -2987, 2389, 2000, 1935, 1937, -2987, 1365, -2987, -2987, 64713, - 1939, -2987, 1940, 899, -2987, 64713, 1980, -2987, 309, 2253, - 95, -2987, 20517, -2987, 2343, 2421, 1847, 1950, 64713, 54828, - 25206, -2987, 414, 246, -2987, 2241, 54828, 1980, 2380, -2987, - -2987, -2987, 752, -2987, 2279, 2194, -2987, 192, -2987, 20517, - 752, 2195, 222, 64713, -2987, -2987, 3395, -2987, 69663, 241, - 241, -2987, 1499, 1963, 1964, 1966, 1967, 1968, 1969, 1972, - 1973, 1975, 1997, -2987, 2003, 2005, 2006, 2010, 2012, 2013, - 2015, 2017, 1513, 2018, -2987, 2020, 1880, 2022, 2025, 2028, - 2030, 2032, 70623, 2034, 2045, 2047, 2048, 1514, 2050, 1143, - 1201, -2987, -2987, -2987, -2987, -2987, -2987, 1164, 2051, -2987, - 1998, -2987, -2987, -2987, 2053, -2987, 2074, -2987, -2987, -2987, - -2987, -2987, -2987, 1993, 2014, -2987, -2987, -2987, 131, 1992, - 1995, 64713, 1195, 113, 47403, 64713, 2055, 1844, 2438, 19475, - 1048, 2258, 2021, -2987, 819, 2037, -2987, 1663, -2987, 50868, - 2948, 245, 2036, -2987, 304, 1844, -2987, 2442, 1663, 2092, - 2533, -2987, 2286, 64713, 2065, -2987, -2987, 48393, 1899, 5624, - 25206, 69663, 1027, 1032, -2987, 2573, 2232, 2159, -2987, -2987, - -2987, -2987, -2987, 2072, -29, 2073, 10097, 2076, -2987, -2987, - -2987, -2987, -2987, -2987, 45045, 45045, 64713, 2255, -2987, -2987, - 2082, 2078, 37286, 2536, 2086, -2987, -2987, 2408, -2987, 30353, - -2987, 1681, 2095, 1681, 69663, 1681, -2987, -2987, 45045, 20517, - -2987, -2987, 41298, 2416, 2509, 2509, 45429, 1057, -2987, 2509, - 20517, 20517, 2509, 2509, 20517, -2987, 9575, 550, -2987, 1059, - -2987, 40391, -2987, 71103, -2987, -2987, 1946, 819, 1946, -2987, - -2987, 2093, -2987, -2987, -2987, 2150, -2987, -2987, 1061, 2519, - 2004, 20517, -2987, -2987, 2099, 35801, -2987, -2987, -2987, -2987, - 35801, 899, -2987, 2269, 1980, 2106, -2987, -2987, -2987, -2987, - -2987, -2987, 40430, -2987, 83, 20517, -2987, 184, 4327, -2987, - -2987, -2987, -2987, 1980, 1125, -2987, 54828, 2583, 2472, -2987, - -2987, 45045, -2987, -2987, 2024, 2024, -2987, -2987, 2267, -2987, - -2987, 2110, -2987, -2987, 1164, 375, 39267, 54828, 54828, -2987, - -2987, 2112, -2987, -2987, -2987, -2987, -2987, 257, 2504, 1069, - 1070, 159, -2987, 2426, 2426, 45045, 54828, 2478, 50868, -2987, - 47403, 2591, 2117, 54828, 1844, 437, 437, -2987, 2271, -2987, - 2273, -2987, -2987, 2593, 356, -2987, 19996, 54828, -2987, -2987, - 32830, -2987, 5624, 1071, -2987, -2987, 2120, 2127, -2987, 1946, - 20517, 2128, 20517, -2987, 23643, 2603, 2126, -2987, 20517, 2191, - 28332, -2987, 20517, -2987, 54828, 59778, 2133, 59778, -2987, -2987, - -2987, -2987, -2987, 20517, -2987, 2509, 2509, 2509, 20517, -2987, - 20517, -2987, -2987, -2987, 2339, 2255, -2987, 2255, 20517, 2426, - 381, 2758, 64713, 0, -2987, 45045, -2987, -2987, -2987, 54828, - -2987, 47403, -2987, 899, 12, 2136, 20517, 40780, 2379, -2987, - -2987, 2411, -2987, 2470, -2987, 2212, 640, 2228, -2987, -2987, - -2987, -2987, 1195, 819, -2987, 1663, 2036, 2092, -2987, 2154, - 54828, 1106, 2426, 159, 704, -2987, -2987, -2987, -2987, -2987, - -2987, -2987, -2987, -2987, -2987, -2987, -2987, -2987, 2426, 2596, - 2377, 2597, 2426, 45045, 1932, 20517, 91, -2987, 1110, 2592, - -2987, -2987, 2665, 2255, 2161, 23643, 2163, -2987, 2169, 64713, - 45045, 2313, -2987, -2987, 2173, -2987, -2987, 20517, -2987, 41380, - 2174, 2176, 2625, 1817, 2191, 2191, -2987, 67, -2987, -2987, - 2598, 32830, 2553, 1125, 899, 2187, 1137, -2987, -2987, -2987, - -2987, -2987, 202, -2987, 40866, 2420, 208, 2404, 2136, 20517, - -2987, 2256, -2987, -2987, -2987, 2656, -2987, -2987, 50868, 2184, - -2987, 2092, 2036, 1844, 2092, 2407, -2987, 2409, -2987, 2186, - 40905, 64713, 64713, 1663, 32830, 64713, 2190, 2191, -2987, 2197, - -2987, -2987, -2987, 53838, -2987, 2198, -2987, -2987, -2987, 20517, - 152, -2987, -2987, 2244, 54828, 1145, 40, 2411, 39267, -2987, - 47403, 1058, 12, 2500, -2987, -2987, -2987, -2987, 127, 2417, - -2987, 2422, -2987, 45045, -2987, 2426, 50868, -2987, -2987, -2987, - -2987, -2987, -2987, 32830, 2592, -2987, 357, -2987, 1520, -2987, - 357, -2987, -2987, -2987, -2987, -2987, 1462, 24164, 24164, 24164, - 2201, 2426, -2987, 1520, -2987, 2323, 2404, -2987, -2987, -2987, - -2987, -2987, 176, 176, 2588, -2987, 2266, -2987, 2092, 1146, - 64713, 1776, -2987, 1776, 25727, 2355, 177, 43927, 2575, -2987, - 2575, 2575, -2987, -2987, -2987, 38276, -2987, -2987, 2696, -2987, - 231, -2987, -2987, -2987, 1663, 357, -2987, -2987, 2688, -2987, - -2987, -2987, -2987, -2987, 196, -2987, -2987, -2987, 1520, 899, - -2987, -2987, -2987, 1520, 1776, 24685, 2361, -2987, 2430, -2987, - -2987, -2987, -2987, -2987, -2987, -2987 + 6375, 351, 1052, -3059, -3059, 657, 351, 50036, 64871, 293, + 351, 181, 3103, 52016, -3059, -3059, 46571, 4517, 351, 54986, + 72207, 328, 274, 31911, 355, 55481, 55481, -3059, -3059, -3059, + 64871, 54986, 55976, 351, 358, 65366, -3059, 351, 34386, 52511, + 198, -3059, 54986, 65, 257, 56471, 54986, 4726, 600, 263, + -3059, -3059, -3059, -3059, -3059, 178, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, 150, -3059, 189, 154, 31911, 31911, 1366, 118, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, 442, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + 33891, -3059, -3059, -3059, -3059, -3059, -3059, 56966, 54986, 57461, + 53006, 57956, -3059, 645, 965, -3059, 165, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, 170, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, 481, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, 183, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, 376, -3059, 516, -3059, + 195, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + 1178, -3059, -3059, 957, 3143, 54986, 432, 665, 718, -3059, + 58451, -3059, 725, 54986, -3059, -3059, 731, 774, 916, -3059, + -3059, 53501, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, 47066, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, 877, -3059, -3059, + 706, -3059, 194, -3059, -3059, 729, 686, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, 787, -3059, -3059, -3059, + 789, 65861, 58946, 59441, -3059, 667, 2978, 6731, 72225, 30919, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, 442, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, 55481, 64871, 55481, 709, 723, 1076, 734, 32406, + 737, 34882, 753, 769, 1117, 776, 778, 785, 796, 257, + 31415, 807, 376, -3059, 59936, 59936, -26, 2686, -3059, 59936, + 60431, -3059, 822, -3059, 965, -3059, -3059, -3059, 1157, -3059, + -61, 816, -3059, 60926, 60926, 60926, 842, 1124, -3059, -3059, + -3059, 866, -3059, -3059, 1095, 20588, 20588, 66356, 66356, 965, + 66356, 876, -3059, -3059, 91, -3059, -3059, -3059, 1366, 882, + 376, -3059, -3059, 52511, -3059, -3059, 237, 1219, 20588, 54986, + 884, -3059, 886, 884, 892, 906, 913, -3059, 6375, 1264, + 1145, 52511, 364, 364, 1386, 364, 212, 587, 4710, 2869, + -3059, 887, -3059, 944, -3059, 54986, 1048, 974, 1244, -3059, + 882, 1329, 1242, 1133, 1338, 5360, 1346, 1336, 1375, 1623, + 1382, 1523, 20588, 47561, 376, -3059, 11731, 20588, -3059, -3059, + -3059, 1149, -3059, -3059, -3059, -3059, -3059, 54986, 64871, 1072, + 1075, -3059, -3059, -3059, -3059, 1022, 1317, -3059, 1558, 66851, + -3059, -3059, 1160, 61421, 61916, 62411, 62906, 1514, -3059, -3059, + 1490, -3059, -3059, -3059, 1167, -3059, -3059, -3059, 176, 67346, + 1500, 1154, 110, -3059, 1509, 172, -3059, 1522, 1409, 15378, + -3059, 1353, -3059, -3059, -3059, 257, -3059, 403, -3059, -3059, + 43428, -3059, -3059, 72225, 1281, 1207, -3059, 1556, 20588, 20588, + 1221, 5787, 59936, 60431, 20588, 54986, -3059, 20588, 25277, 1235, + 20588, 20588, 12773, 20588, 29929, 59936, 2686, 1248, -3059, 582, + 54986, 1243, -3059, 1347, 1347, 358, 31911, 1548, -3059, 375, + 1545, 1479, -3059, 31911, 1479, 977, 1269, 1562, 1479, -3059, + 253, 1567, 1347, 35377, 1282, -3059, 1347, 1516, -3059, -3059, + 55481, 20588, 15378, 69821, 1776, -3059, -3059, -3059, -3059, 1585, + 64871, 1308, -3059, -3059, -3059, -3059, -3059, -3059, 615, 1818, + 158, 1819, 20588, 158, 158, 1311, 196, 196, -3059, 1503, + 1314, -3059, 200, 1315, 1316, 1827, 1831, 180, 134, 871, + 158, 20588, -3059, 196, 1326, 1839, 1334, 1846, 138, 173, + -3059, 201, 20588, 20588, 20588, 1704, 20588, 10689, -3059, 54986, + 1844, 47561, 545, -3059, 376, 1341, 965, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, 1356, -3059, 185, 7003, -3059, -3059, + -3059, -3059, -3059, -3059, 1377, -3059, -3059, -3059, -3059, 1559, + 20588, -3059, -3059, 1343, 1548, -3059, 202, -3059, -3059, 1548, + -3059, -3059, -3059, -3059, -3059, 217, 1768, 20588, 20588, 64871, + 376, 67841, -3059, -3059, -3059, -3059, -3059, -3059, -3059, 628, + -3059, 442, 45116, 1355, 1359, 884, 54986, 54986, 1835, -3059, + -3059, -3059, -3059, 52511, 159, 1657, 1491, -3059, -3059, 1366, + 1366, 15899, 1027, 220, 70, 16420, 21109, 1713, 1592, 225, + 586, 1715, -3059, 1598, 1825, 25277, 20588, 20588, 212, 587, + 20588, 886, -3059, -3059, -3059, 1651, 54986, 50531, 482, 553, + 1371, 1461, 1378, 29, 1798, -3059, 1376, -3059, 1465, 54986, + 71756, 244, -3059, 1855, 244, 244, 235, 1856, 1493, 261, + 1659, 630, -32, 1376, 2912, -3059, 52511, 156, 643, 1376, + 54986, 1494, 688, 1376, 1820, 64871, 1207, 41015, 1403, -3059, + -3059, -3059, 147, 15378, -3059, 1292, 1339, 1364, 367, 184, + 1400, 1542, 15378, 1602, 1650, 163, 1652, 1663, 1671, 1679, + 1683, 1687, 1703, 1705, 144, 1710, 1712, 1716, 1718, 1720, + 1724, -3059, 1728, 169, 1730, 199, 15378, 1733, -3059, 45116, + 8, -3059, -3059, 1738, 182, -3059, 45205, -3059, 1708, 1505, + 1506, 64871, 1455, 54986, 1560, 823, 1783, 1836, 70301, 1664, + -3059, 1741, 54986, 1666, 2912, 1667, 1429, 1904, 1672, 1075, + 1674, 1431, -3059, 68336, 47561, -3059, -3059, -3059, -3059, -3059, + 1797, 1780, 64871, 47561, 1437, -3059, -3059, 64871, -3059, 54986, + 54986, -3059, 54986, 64871, -3059, 644, 45116, 1941, 865, 72225, + 49046, -3059, -3059, -3059, -3059, 428, 951, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, 965, 47561, -3059, 2564, + 55481, 44046, 1444, 20588, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, 1449, 1801, -3059, -3059, 6587, 1453, 44086, 1454, + 25277, 25277, 376, 530, -3059, -3059, 25277, 1460, 49541, 43959, + 1463, 1467, 44434, 16941, 20588, 16941, 16941, 44521, -3059, 1471, + 44598, 59936, 1473, 54986, 53996, -3059, -3059, -3059, 20588, 20588, + 2686, 54491, 1515, 31911, -3059, 31911, -3059, 1765, 31911, -3059, + -3059, 2074, -3059, 31911, 1766, 20588, 31911, -3059, 31911, 1714, + 1717, 1477, 31911, -3059, 54986, 1482, 54986, -3059, -3059, -3059, + -3059, -3059, 45116, -3059, 1481, 690, 1485, -3059, -3059, -3059, + -3059, -3059, 1534, -3059, 1534, 1534, -3059, -3059, -3059, -3059, + 1492, 1492, 1497, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, 1498, 871, -3059, 1534, + -3059, 1492, -3059, -3059, -3059, -3059, -3059, -3059, -3059, 71756, + -3059, -3059, -3059, -3059, -50, 501, -3059, 1501, -3059, -3059, + 1507, -3059, 1489, 1978, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, 5696, 699, 1492, -3059, -3059, 5470, -3059, + -3059, 20588, 20588, -3059, -3059, 1508, 45116, 1547, -3059, -3059, + 20588, 20588, -3059, -3059, -3059, -3059, 2011, -3059, 20588, 1534, + 1534, -3059, 45587, -3059, 39834, 17462, 1597, 1599, 2011, -3059, + 2011, -3059, 45587, 2015, 2015, 37852, -3059, 1675, 44685, -3059, + 1518, 2122, 7670, 1510, 1511, -3059, 1519, 1513, -3059, -3059, + 41943, 166, 376, 376, 20588, -3059, 2011, 20588, 8642, 8642, + -3059, 295, 69821, 20588, 20588, 20588, 20588, 20588, 20588, 20588, + 20588, 46076, 1606, 127, 64871, 20588, 20588, 1524, 858, -3059, + 20588, 1754, -3059, 1525, 20588, 1608, 312, 20588, 20588, 20588, + 20588, 20588, 20588, 20588, 20588, 20588, -3059, -3059, 28924, 344, + 550, 1860, 1879, 46, 291, 20588, 1871, 11731, -3059, 1871, + -3059, -3059, -3059, -3059, -3059, 204, -3059, -3059, 1481, 1481, + -3059, 64871, -3059, 54986, 237, 51521, 20588, -3059, -3059, 1527, + 1531, 485, 1594, -3059, -3059, 54986, 38347, 1833, -3059, 350, + 1533, -3059, 43920, 1790, 1833, 1366, -3059, -3059, 26319, 1673, + 1830, 1770, -3059, -3059, 1749, 1751, -3059, 1546, 45338, 21630, + 21630, -3059, 646, 45116, 1293, -3059, -3059, -3059, -3059, -3059, + -3059, 733, -3059, 54986, 82, 35872, -3059, 1552, 111, -3059, + 3217, 1885, 1847, 1713, 586, 1557, -3059, -3059, 1781, 1563, + 68831, 54986, 1845, 1804, 1859, 308, 69821, -3059, -3059, -3059, + -3059, 54986, 64871, 63401, 69326, 48056, 54986, 47561, -3059, -3059, + -3059, -3059, 54986, 1218, 54986, 8381, -3059, -3059, -3059, -3059, + 244, -3059, -3059, -3059, -3059, -3059, 64871, 54986, -3059, -3059, + 244, 64871, 54986, 244, -3059, 1939, 54986, 54986, 54986, 54986, + 2069, 54986, 54986, 965, -3059, -3059, -3059, 22151, 24, 24, + 1786, 13294, 203, -3059, 20588, 20588, 306, 294, 64871, 1753, + -3059, -3059, 700, 1795, 103, -3059, 64871, 1621, 54986, 54986, + 54986, 54986, 54986, 2076, -3059, -3059, -3059, -3059, -3059, 1575, + -3059, 1940, 2090, 1582, 1586, 1947, -3059, 2912, 1951, 51026, + 698, 2699, 1952, 1630, 1956, 13815, 2072, 1840, -3059, -3059, + 1824, -3059, 64871, 2106, -3059, 110, -3059, 47561, -3059, 172, + -3059, 1829, 205, -3059, 15378, 20588, -3059, -3059, -3059, -3059, + -3059, -3059, 1207, 29430, -3059, 714, -3059, -3059, 2075, 965, + 2075, 565, -3059, -3059, 2075, -3059, 2060, 2075, -3059, -3059, + 69821, -3059, 7960, -3059, 20588, 20588, -3059, 20588, 1949, -3059, + 2110, 2110, 69821, 25277, 25277, 25277, 25277, 25277, 25277, 486, + 1326, 25277, 25277, 25277, 25277, 25277, 25277, 25277, 25277, 25277, + 26840, 301, -3059, -3059, 715, 2085, 20588, 20588, 1960, 1949, + 20588, -3059, 69821, 1613, -3059, 1614, 1617, 20588, -3059, 69821, + -3059, 54986, 1619, 9, -2, -3059, 1622, 1627, -3059, 1548, + -3059, 713, 775, 54986, 2465, 5151, 5636, -3059, -3059, 20588, + 1962, 2074, 2074, 31911, -3059, 20588, 1629, -3059, -3059, 31911, + 1979, -3059, 2074, -3059, -3059, 36367, 2074, 69821, 748, -3059, + 54986, 69821, 758, 20588, -3059, 15378, 2143, 69821, 2108, 64871, + 64871, 2145, 1637, 1639, 2011, 1725, -3059, 1727, 1729, 1731, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, 69821, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, 1643, 1647, + 20588, 20588, 80, -3059, 8114, 1646, 1653, 6163, -3059, 1644, + -3059, 1649, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, 1655, + -3059, 1661, -3059, 1669, 1677, 1682, 1676, 1681, 54986, -3059, + 22672, -3059, 64871, -3059, -3059, 20588, 20588, 54986, -3059, 1704, + -3059, 1685, 1688, 8303, -3059, -3059, -3059, 153, 394, 45444, + 291, 3594, 3594, 3594, 45587, -3059, -3059, -3059, 1692, -3059, + 25277, 25277, -3059, 2523, 2753, 10689, -3059, -3059, 1995, -3059, + 891, -3059, 1658, -3059, -3059, 3090, -3059, 39834, 45547, 20588, + 179, -3059, 20588, 1524, 20588, 1762, 3594, 3594, 3594, 268, + 268, 153, 153, 153, 394, 291, -3059, -3059, -3059, 1668, + 20588, 47561, -3059, 1680, 1684, 2038, 1334, 20588, -3059, -3059, + 31911, 1515, 8, 1515, 2011, 8642, -3059, 886, -3059, 886, + -3059, 45116, 54986, -3059, -3059, 1953, 1690, 31911, 1732, 2167, + 2157, 64871, -3059, -3059, 1693, 1871, 1719, -3059, -3059, 1721, + 20588, 3753, 1721, -3059, 1833, -5, 1929, 998, 998, 646, + 1931, -3059, -3059, 1774, -3059, -3059, -3059, 20588, 14336, 1389, + -3059, 1407, -3059, -3059, -3059, -3059, -3059, 1699, -3059, 1986, + -3059, 54986, -3059, -3059, 25277, 2172, 20588, 36862, 2174, 1974, + -3059, -3059, -3059, 1812, 1376, 20588, 1971, -3059, 187, 1734, + 2097, 371, 2049, 64871, -3059, 318, 353, -3059, 978, 2100, + 205, 2105, 205, 47561, 47561, 47561, 760, -3059, -3059, -3059, + 965, -3059, 271, 762, -3059, -3059, -3059, -3059, 1834, 933, + 1376, 2912, -3059, -3059, -3059, -3059, -3059, -3059, -3059, 222, + 948, 1376, 1842, -3059, 1849, -3059, 1851, 1015, 1376, -3059, + -3059, 1523, 9124, 45116, 336, 203, 203, 203, 15378, -3059, + 1982, 1983, 1758, 45116, 45116, 142, -3059, -3059, -3059, -3059, + 1760, -3059, 236, -3059, 64871, -3059, -3059, -3059, 1753, 1836, + 1741, 54986, 2912, 1767, 2236, 1075, 1431, -3059, 1928, 901, + 1678, -3059, 64871, -3059, 47561, 64871, 54986, 54986, 54986, 63896, + -3059, -3059, -3059, 1769, 1772, -3059, -11, 2004, 2005, 54986, + 1821, 54986, 1378, 2255, 54986, -3059, 766, 17983, 2144, 54986, + 1780, -3059, -3059, -3059, -3059, 64871, -3059, -3059, 45116, -3059, + -3059, 20588, 48551, -3059, -3059, -3059, -3059, -3059, -3059, 47561, + -3059, 965, -3059, 965, 2020, 64871, 42438, 965, 42933, 965, + 1785, -3059, 45116, 8353, 45116, 1960, -3059, 214, 2110, 1848, + 1848, 1848, 4127, 2130, 240, 1788, 1848, 1848, 1848, 282, + 282, 214, 214, 214, 2110, 301, 822, 49541, 1789, -3059, + 45116, 45116, -3059, -3059, 1794, -3059, -3059, -3059, -3059, 1802, + 1803, -3059, -3059, -3059, -3059, -3059, -3059, 64871, 1037, 1515, + 198, 198, 198, 198, -3059, 54986, 54986, 54986, 45116, 2245, + 2120, -3059, -3059, 2074, 45116, 54986, -3059, 27882, -3059, 54986, + -3059, 2153, -3059, 2240, -3059, 54986, 830, -3059, -3059, -3059, + 857, 1809, 1639, 69821, 873, 879, -3059, 2011, 136, 1807, + 1512, 1319, 750, 1413, -3059, -3059, -3059, 1808, 44820, 20588, + -3059, 2181, -3059, -3059, -3059, 20588, 20588, -3059, 39834, -3059, + -3059, -3059, -3059, -45, -45, 1811, 10689, 45042, -3059, 2139, + 8435, 45116, -3059, 1675, -3059, -3059, 8642, 20588, 1205, 1826, + 20588, 1823, 20588, 2169, -3059, -3059, 1832, -3059, -3059, 69821, + 20588, 1850, 5197, 25277, 25277, 6403, -3059, 7322, 20588, 10689, + -3059, 41101, 1817, 1852, 1786, 18504, -3059, 2047, 1843, -3059, + 1962, 203, 1962, 1854, -3059, -3059, -3059, -3059, 5470, -3059, + 20588, 1990, 64871, 518, 1927, 890, -3059, 376, 38347, 1732, + 20588, 252, -3059, -3059, 1857, -3059, 1721, -3059, -3059, -3059, + 2083, -3059, -3059, -3059, 54986, -3059, 1862, -3059, 35872, 2184, + 11210, -3059, 35872, 54986, 54986, 40337, 2223, -3059, 64871, 64871, + 64871, -3059, 64871, 1861, 1863, 653, 1866, 389, -3059, 1021, + 653, 2206, 296, 1378, 261, 1561, 521, -3059, -3059, -3059, + 1943, 54986, -3059, 64871, -3059, -3059, -3059, -3059, -3059, 48056, + -3059, -3059, 39338, 47561, -3059, 47561, 54986, 54986, 54986, 54986, + 54986, 54986, 54986, 54986, 54986, 54986, 1207, 20588, -3059, 20588, + 1868, 1869, 1874, 1786, -3059, -3059, -3059, 206, -3059, 1870, + -3059, -3059, -3059, -32, -3059, 236, 1877, 1878, -3059, 51026, + 3143, 1630, 2353, 1836, 850, 64376, -3059, 1882, 1876, 1741, + 897, 912, 2912, 1884, 2359, -3059, 698, 51026, -3059, -3059, + -3059, 2315, -3059, 667, 232, -3059, 1075, -3059, 3143, 1431, + -3059, 3143, 45116, 64871, 1948, -3059, 205, 922, -3059, -3059, + -3059, -3059, -3059, 64871, 1886, -3059, 1886, -3059, -3059, 1886, + -3059, -3059, -3059, -3059, 25277, 2234, 1892, 69821, -3059, -3059, + 54986, -3059, -3059, -3059, 942, 1894, 1962, 54986, 54986, 54986, + 54986, -3059, -3059, -3059, 19025, 20588, 1932, -3059, 1895, 12252, + 2213, -3059, 27361, -3059, -3059, 1899, 36367, 64871, -3059, -3059, + -3059, -3059, 2011, -3059, -3059, 64871, -3059, 1902, -3059, 1903, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, 20588, + 45116, -3059, 45116, -3059, -3059, -3059, -3059, -3059, 7369, -3059, + 1907, 1905, 64871, 20588, -3059, -3059, -3059, 747, 20588, 20588, + 2523, -3059, 6937, 20588, 69821, 962, 2523, 347, 20588, 2239, + 3853, 20588, 20588, 7350, 40376, -3059, 23193, 14857, -3059, 1906, + 20588, 40415, 38842, -3059, 31911, 2120, 1911, 2120, 965, 1913, + 45116, 20588, -3059, -3059, -3059, -3059, 1969, 349, 33396, 2142, + -3059, 1930, 64871, -3059, 1990, 45116, -3059, -3059, 39834, -3059, + -3059, -3059, -3059, -3059, 2373, 2635, 1923, 1924, -3059, 1340, + -3059, -3059, 64871, 1925, -3059, 1935, 653, -3059, 64871, 1968, + -3059, 272, 2241, 113, -3059, 20588, -3059, 2330, 2411, 1021, + 1942, 64871, 54986, 25277, -3059, 625, 243, -3059, 2227, 54986, + 1968, 2369, -3059, -3059, -3059, 389, -3059, 2268, 2185, -3059, + 244, -3059, 20588, 389, 2188, 137, 64871, -3059, -3059, 2738, + -3059, 69821, 205, 205, -3059, 1485, 1954, 1955, 1958, 1959, + 1961, 1965, 1967, 1972, 1973, 1976, -3059, 1988, 1989, 1992, + 1993, 1994, 1997, 1998, 1999, 1498, 2000, -3059, 2001, 1857, + 2006, 2013, 2016, 2019, 2021, 70781, 2026, 2028, 2031, 2033, + 1501, 2034, 428, 951, -3059, -3059, -3059, -3059, -3059, -3059, + 1154, 2035, -3059, 1975, -3059, -3059, -3059, 2044, -3059, 2056, + -3059, -3059, -3059, -3059, -3059, -3059, 1981, 2007, -3059, -3059, + -3059, 203, 1970, 1984, 64871, 1207, 151, 47561, 64871, 2037, + 1821, 2496, 19546, 1159, 2279, 2041, -3059, 965, 2053, -3059, + 1630, -3059, 51026, 3125, 669, 2005, -3059, 208, 1821, -3059, + 2451, 1630, 2094, 2521, -3059, 2282, 64871, 2059, -3059, -3059, + 48551, 1886, 4313, 25277, 69821, 964, 970, -3059, 2568, 2231, + 2120, -3059, -3059, -3059, -3059, -3059, 2064, -25, 2078, 10168, + 2065, -3059, -3059, -3059, -3059, -3059, -3059, 45116, 45116, 64871, + 2251, -3059, -3059, 2071, 2081, 37357, 2531, 2082, -3059, -3059, + 2395, -3059, 30424, -3059, 1639, 2086, 1639, 69821, 1639, -3059, + -3059, 45116, 20588, -3059, -3059, 41369, 2407, 2523, 2523, 6937, + 987, -3059, 2523, 20588, 20588, 2523, 2523, 20588, -3059, 9646, + 409, -3059, 995, -3059, 40462, -3059, 71261, -3059, -3059, 1932, + 965, 1932, -3059, -3059, 2089, -3059, -3059, -3059, 2141, -3059, + -3059, 1049, 2510, 1990, 20588, -3059, -3059, 2096, 35872, -3059, + -3059, -3059, -3059, 35872, 653, -3059, 2260, 1968, 2095, -3059, + -3059, -3059, -3059, -3059, -3059, 40501, -3059, 52, 20588, -3059, + 1046, 4127, -3059, -3059, -3059, -3059, 1968, 1075, -3059, 54986, + 2573, 2463, -3059, -3059, 45116, -3059, -3059, 2011, 2011, -3059, + -3059, 2240, -3059, -3059, 2101, -3059, -3059, 1154, 512, 39338, + 54986, 54986, -3059, -3059, 2103, -3059, -3059, -3059, -3059, -3059, + -32, 2497, 1064, 1065, 698, -3059, 3143, 3143, 45116, 54986, + 2471, 51026, -3059, 47561, 2587, 2113, 54986, 1821, 1137, 1137, + -3059, 2263, -3059, 2266, -3059, -3059, 2595, 276, -3059, 20067, + 54986, -3059, -3059, 32901, -3059, 4313, 1070, -3059, -3059, 2124, + 2126, -3059, 1932, 20588, 2127, 20588, -3059, 23714, 2602, 2125, + -3059, 20588, 2190, 28403, -3059, 20588, -3059, 54986, 59936, 2132, + 59936, -3059, -3059, -3059, -3059, -3059, 20588, -3059, 2523, 2523, + 2523, 20588, -3059, 20588, -3059, -3059, -3059, 2338, 2251, -3059, + 2251, 20588, 3143, 376, 2620, 64871, 20, -3059, 45116, -3059, + -3059, -3059, 54986, -3059, 47561, -3059, 653, 326, 2134, 20588, + 40851, 2372, -3059, -3059, 2405, -3059, 2464, -3059, 2201, 573, + 2222, -3059, -3059, -3059, -3059, 1207, 965, -3059, 1630, 2005, + 2094, -3059, 2148, 54986, 1084, 3143, 698, 667, -3059, -3059, + -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, -3059, + -3059, 3143, 2596, 2377, 2597, 3143, 45116, 1948, 20588, 79, + -3059, 1128, 2592, -3059, -3059, 2664, 2251, 2159, 23714, 2160, + -3059, 2162, 64871, 45116, 2309, -3059, -3059, 2166, -3059, -3059, + 20588, -3059, 41451, 2171, 2175, 2626, 1786, 2190, 2190, -3059, + 349, -3059, -3059, 2586, 32901, 2556, 1075, 653, 2187, 1136, + -3059, -3059, -3059, -3059, -3059, 2912, -3059, 40937, 2419, 122, + 2403, 2134, 20588, -3059, 2256, -3059, -3059, -3059, 2654, -3059, + -3059, 51026, 2182, -3059, 2094, 2005, 1821, 2094, 2406, -3059, + 2409, -3059, 2189, 40976, 64871, 64871, 1630, 32901, 64871, 2191, + 2190, -3059, 2192, -3059, -3059, -3059, 53996, -3059, 2193, -3059, + -3059, -3059, 20588, 135, -3059, -3059, 2252, 54986, 1138, 51, + 2405, 39338, -3059, 47561, 174, 326, 2500, -3059, -3059, -3059, + -3059, 209, 2421, -3059, 2423, -3059, 45116, -3059, 3143, 51026, + -3059, -3059, -3059, -3059, -3059, -3059, 32901, 2592, -3059, 350, + -3059, 1515, -3059, 350, -3059, -3059, -3059, -3059, -3059, 1463, + 24235, 24235, 24235, 2198, 3143, -3059, 1515, -3059, 2328, 2403, + -3059, -3059, -3059, -3059, -3059, 457, 457, 2598, -3059, 2273, + -3059, 2094, 1142, 64871, 1721, -3059, 1721, 25798, 2354, 238, + 43998, 2578, -3059, 2578, 2578, -3059, -3059, -3059, 38347, -3059, + -3059, 2702, -3059, 227, -3059, -3059, -3059, 1630, 350, -3059, + -3059, 2693, -3059, -3059, -3059, -3059, -3059, 343, -3059, -3059, + -3059, 1515, 653, -3059, -3059, -3059, 1515, 1721, 24756, 2365, + -3059, 2435, -3059, -3059, -3059, -3059, -3059, -3059, -3059 }; /* YYPGOTO[NTERM-NUM]. */ static const yytype_int16 yypgoto[] = { - -2987, -2987, -2987, 1875, 79, -2987, -2987, 146, -2987, 982, - -2987, 145, -814, 520, -2987, 86, 3842, 2546, 3549, 66, - -510, -886, -1210, 28, 87, -1138, 4, -2987, -2987, -2987, - -2987, -2987, -494, 211, -2987, -2987, -634, -2547, -583, -2987, - -2913, -2986, -2987, -2987, -727, -2954, -2059, 89, -2987, -2987, - 93, 1, -2096, -2987, -1645, 42, -2092, 94, 892, -2987, - -2555, 98, -883, -1185, -944, -1191, -2987, -123, -2987, 421, - 105, 1560, -2987, 11, -2163, -2858, -597, -2987, -701, -2987, - -353, -2987, -647, -2987, -929, -654, -686, -2783, -1130, -2987, - 1579, -400, -2987, 557, -2987, -2560, -2987, -2987, 551, -2987, - -1165, -2987, -2207, 115, -630, -2562, -2509, -2166, -885, 203, - -636, 179, -2079, -1212, -2987, 568, -2987, -615, -2987, -891, - -2026, 108, -2987, -2987, 1472, -947, -2987, 110, 1510, -2103, - 7, 10, -2987, -2987, -2987, -2987, -2987, -687, 500, -1211, - -2987, 451, -2987, -2987, -2987, -2987, -189, 169, -2252, 2, - 306, -44, -32, -2987, -25, -2987, -2987, -2987, 606, -2987, - -2987, 21, 29, 1657, -2987, -1033, -2987, -1587, 590, -2987, - 1809, 1810, -2136, -858, -68, -2987, 641, -1661, -2094, -643, - 1086, 1639, 1643, 406, -2325, -2987, -569, -2987, -62, -2987, - -2987, 639, 1129, -1560, -1556, -2987, -2175, -2987, -488, -375, - -2987, -2987, -2987, -2987, -2987, -2453, -2798, -642, 1099, -2987, - 1658, -2987, -2987, -2987, -2987, 43, -1510, 2811, 679, -48, - -2987, -2987, -2987, -2987, 107, -2987, 869, -200, -2987, 2046, - -696, -784, 1893, -153, 143, -1739, 41, 2059, 455, -2987, - -2987, 457, -2085, -1407, 409, -284, 866, -2987, -2987, -1221, - -2987, -1880, -1160, -2987, -2987, -673, 1107, -2987, -2987, -2987, - 1933, 2214, -2987, -2987, 2951, 3055, -2987, -896, 3102, 993, - -1029, 1896, -934, 1898, -930, -928, -933, 1902, 1903, 1904, - 1906, 1910, 1911, -1511, 4396, 2674, 2508, -2183, -2987, -2180, - 989, 55, -2987, -1374, 242, -2987, -2987, -2987, -2987, -2662, - -2987, -462, -2987, -456, -2987, -2987, -2987, -1622, -2703, -1652, - -2987, 2157, 797, -2987, -2987, 392, -2987, -2987, -2987, -2987, - -1527, -2987, 5650, 694, -2987, -2024, -2987, -2987, -970, -829, - -733, -999, -1208, -1916, -2987, -2987, -2987, -2987, -2987, -2987, - -1114, -1767, -161, 754, -2987, -2987, 849, -2987, -2987, -2987, - -1722, -2097, -2987, -2987, -2987, 759, 1436, 82, -822, -1600, - -2987, 812, -2357, -2987, -2987, 403, -2987, -620, -1119, -2424, - 2029, 77, -2987, -779, -2510, -2987, -2987, -702, -2617, -1134, - -894, -2987, 111, -2987, 360, 120, -1658, -2987, 14, -2987, - -408, -2987, -2987, -2575, -2987, 123, 124, 2114, -2987, 1092, - -2987, -2987, -2987, -2987, -575, -2987, -632, -628, -2987, -2987, - 18, -898, 1565, -2987, 125, 436, -2987, 917, -2987, 702, - 128, 80, 1546, 129, 1228, -2987, -2987, -2987, 17, -603, - 361, -2987, 1232, -2987, -2987, 1683, 643, 133, -2987, 302, - 22, -2987, -2987, -2987, 97, 2808, 134, 15, -2927, 136, - -2755, -1592, -7, -2987, -2987, -2987, -730, -2987, -2505 + -3059, -3059, -3059, 1865, 73, -3059, -3059, 139, -3059, 973, + -3059, 146, -787, 509, -3059, 77, 1252, 2549, 3668, 2713, + -515, -876, -1226, 1, 81, -1140, 3, -3059, -3059, -3059, + -3059, -3059, -502, 210, -3059, -3059, -634, -2561, -584, -3059, + -2881, -3004, -3059, -3059, -731, -2965, -2068, 86, -3059, -3059, + 88, 11, -2084, -3059, -1601, 36, -2072, 89, 898, -3059, + -2570, 90, -889, -1193, -933, -1200, -3059, -122, -3059, 425, + 96, 1228, -3059, 12, -2165, -2872, -594, -3059, -696, -3059, + -347, -3059, -641, -3059, -1047, -645, -679, -2818, -1139, -3059, + 1589, -396, -3059, 567, -3059, -2540, -3059, -3059, 558, -3059, + -1154, -3059, -2186, 140, -618, -2582, -2531, -2156, -912, 215, + -617, 193, -2122, -1331, -3059, 584, -3059, -600, -3059, -893, + -2019, 98, -3059, -3059, 1495, -909, -3059, 99, 1517, -2083, + 10, 22, -3059, -3059, -3059, -3059, -3059, -828, 513, -1206, + -3059, 454, -3059, -3059, -3059, -3059, -185, 171, -2234, 2, + 323, -43, -22, -3059, -16, -3059, -3059, -3059, 614, -3059, + -3059, 21, 32, 1665, -3059, -1020, -3059, -1587, 383, -3059, + 1814, 1841, -2144, -863, -66, -3059, 652, -1659, -2130, -629, + 1090, 1656, 1648, 407, -2525, -3059, -568, -3059, -88, -3059, + -3059, 647, 1132, -1544, -1559, -3059, -2203, -3059, -487, -375, + -3059, -3059, -3059, -3059, -3059, -2483, -2836, -620, 1103, -3059, + 1660, -3059, -3059, -3059, -3059, 61, -1497, 2814, 682, 41, + -3059, -3059, -3059, -3059, 108, -3059, 863, -195, -3059, 2054, + -675, -796, 1880, 34, 93, -1745, 31, 2068, 452, -3059, + -3059, 463, -2087, -1408, 408, -281, 872, -3059, -3059, -1246, + -3059, -1833, -1172, -3059, -3059, -806, 510, -3059, -3059, -3059, + 1645, 2152, -3059, -3059, 2178, 2764, -3059, -862, 2970, -781, + -1031, 1883, -924, 1887, -930, -946, -937, 1888, 1890, 1893, + 1896, 1897, 1898, -1534, 4692, 2325, 2519, -2179, -3059, -2178, + 993, 1003, 16, -3059, -1387, 128, -3059, -3059, -3059, -3059, + -2600, -3059, -442, -3059, -439, -3059, -3059, -3059, -1649, -3058, + -1674, -3059, 2641, 818, -3059, -3059, 410, -3059, -3059, -3059, + -3059, -1509, -3059, 5718, 712, -3059, -2024, -3059, -3059, -965, + -831, -638, -983, -1197, -1921, -3059, -3059, -3059, -3059, -3059, + -3059, -1488, -1772, -326, 777, -3059, -3059, 867, -3059, -3059, + -3059, -1703, -2098, -3059, -3059, -3059, 781, 1448, 84, -830, + -1621, -3059, 819, -2352, -3059, -3059, 399, -3059, -627, -1138, + -2428, 2164, 104, -3059, -925, -2536, -3059, -3059, -721, -2648, + -1131, -888, -3059, 100, -3059, 359, 105, -1650, -3059, 14, + -3059, -411, -3059, -3059, -2566, -3059, 106, 114, 2116, -3059, + 1094, -3059, -3059, -3059, -3059, -588, -3059, -635, -621, -3059, + -3059, 30, -898, 1564, -3059, 119, 372, -3059, 919, -3059, + 681, 120, 69, 1543, 121, 1231, -3059, -3059, -3059, 26, + -626, 363, -3059, 1240, -3059, -3059, 1694, 651, 124, -3059, + 43, 47, -3059, -3059, -3059, 72, 2816, 125, 15, -2904, + 129, -2759, -1672, -7, -3059, -3059, -3059, -713, -3059, -2530 }; /* YYTABLE[YYPACT[STATE-NUM]]. What to do in state STATE-NUM. If positive, shift that token. If negative, reduce the rule which number is the opposite. If zero, do what YYDEFACT says. If YYTABLE_NINF, syntax error. */ -#define YYTABLE_NINF -2044 +#define YYTABLE_NINF -2046 static const yytype_int16 yytable[] = { - 520, 58, 70, 865, 54, 877, 904, 68, 1330, 714, - 69, 65, 1223, 1123, 82, 102, 520, 93, 1240, 972, - 1419, 77, 98, 1187, 1794, 1022, 1347, 878, 879, 1153, - 2171, 1747, 1783, 1474, 1294, 1291, 1782, 1679, 772, 953, - 847, 77, 778, 1924, 713, 881, 2049, 1390, 1394, 1766, - 1580, 1392, 882, 1393, 2138, 2527, 1735, 2529, 1981, 740, - 1753, 2657, 517, 2587, 2544, 1336, 805, 1799, 520, 520, - 2447, 1757, 1353, 2968, 2273, 863, 1829, 2554, 811, 51, - 851, 1222, 2508, 1228, 518, 1232, 52, 53, 1787, 56, - 1655, 817, 817, 57, 60, 1658, 2073, 2074, 61, 2937, - 518, 2543, 889, 2982, 2939, 63, 819, 2449, 66, 1004, - 67, 80, 1855, 1240, 1025, 2985, 963, 1592, 805, 805, - 81, 966, 955, 83, 84, 90, 2766, 2641, 91, 92, - 811, 811, 844, 97, 101, -1322, 103, 2966, 2627, 2628, - 2629, 2830, -1244, 2580, -511, 2835, -849, -515, 2142, 2191, - 1361, 897, 518, 518, -1891, 2194, 2298, 1557, 1558, 1901, - 2399, -857, -2031, -2031, 1903, -1263, -2017, -2017, -1891, -1263, - 2299, 3381, -854, -1882, 1570, -1899, -854, -1260, -1260, -1264, - 901, -2022, -2022, -2040, -2040, 815, -1882, -1261, -1261, 1046, - 1663, 1349, -857, -1899, -1264, -816, 1340, 973, 1403, 3177, - -829, 1046, 2649, 1046, -844, 2645, 855, 2643, 1135, 815, - 815, 1196, 3162, 2636, 3110, 815, 2088, 1197, 2088, -2042, - -2042, 2428, 2429, 1659, 1340, 815, 2303, 532, 1142, 1046, - 3181, 1046, 2435, 1176, 1179, 2658, 2439, -466, 1324, 3487, - 972, 1901, 1209, 1797, -511, 1902, 1903, -515, 869, 3462, - 1611, 2730, 1798, 1240, 2731, 1613, 1443, 1588, 2857, 3, - 4, 944, 1210, 872, 1610, 813, 1767, 1611, 1165, 1770, - 1771, 1612, 1613, 3255, 3093, 1324, -227, 2466, 815, 1749, - 1749, -227, 2204, 2691, 2693, 1621, 2696, 1025, 3268, 3340, - 1785, 2615, 999, 1611, 874, -648, 2665, 1612, 1613, 1004, - 3516, 2860, 1621, 874, 1324, 3125, 71, 1901, 1211, -2044, - 2409, 1902, 1903, 1941, 1738, 3454, 2986, 813, 813, 1623, - 3145, 2583, 1198, 2279, -1102, 1758, 71, 1608, 1621, 804, - 2959, 2713, -1102, 873, 1682, 3447, 1623, 1660, 870, 2135, - 1139, 1354, 3500, 533, 71, 2380, 1941, 2702, 3511, 3076, - 3367, 3078, 2802, 864, 2804, 1349, 2197, 1772, 1987, 3201, - 1734, 530, 1623, 1730, 1731, 3121, 2422, 2423, 2424, 1005, - 833, 2295, 2844, 1166, -786, 783, 3517, 2125, -1123, 1172, - 3415, 804, 804, 880, 1562, 2126, -1123, 3405, 1172, 852, - 2075, 1011, 2803, 112, 3175, 1143, 3385, 1144, 2166, 3440, - 1572, 2497, 3171, 1212, -2016, -2016, 71, 2519, 1741, 2221, - 2823, 874, 3144, 3183, 3235, 2843, 3237, 104, 3348, 3285, - 3299, 1801, 1315, 1842, 1568, 1569, 3178, 1012, 1575, 3107, - 3300, 1349, -2044, -511, 1349, 1349, -515, 3334, 1499, 3335, - 921, 3179, 922, 1444, 3427, 3176, 1294, 3430, 3488, -2044, - 1576, 2761, 1213, 1014, -2044, 1174, 2584, 782, 3258, 2826, - 1661, 1316, 1214, 2864, 1174, 1136, 105, 3286, 3518, 2222, - 3172, 2198, 834, 2992, 1215, 1167, 113, 2824, 1199, 3469, - 3349, 2987, 3083, 2167, 3495, -668, 2280, 2381, 2136, 3448, - 814, 3455, -2044, 3366, 1683, 1580, 2580, 2382, 2580, 2703, - 3501, 1773, -511, 3108, 1175, -515, 1216, 2410, 1006, -786, - 3512, 1774, 2865, 784, 3341, 3387, 3416, 1800, 3449, 1574, - 3084, 3470, 3287, 1756, 2225, 3122, 1984, 3266, 2866, 1759, - 858, 3288, 1325, 3489, 3119, 1200, 919, 3313, 3007, 1201, - 1137, 3493, 3250, 3244, 1732, 1733, 2730, 1632, 856, 2731, - 3457, 3111, 2287, 1040, 3199, 3289, 2205, 980, 1218, 1681, - 875, 3260, 3490, 2978, 2975, 1868, 1870, 3156, 1743, 1325, - 945, 1202, 2443, 2990, 2216, 1674, 867, 871, 3463, 2049, - 1018, 714, 1600, 1219, 2089, 1196, 2510, 1557, 1558, 3428, - 2782, 1197, 2278, 2960, 1933, 1756, 3301, 3256, 1325, 1445, - 3417, 3276, 1221, 902, 1553, 2700, 1758, 903, 3120, 972, - 2020, 2998, 1570, 2637, 2638, 1948, 941, 2191, 3429, 953, - -511, 3519, 2557, -515, 1857, 1786, 2616, 3290, 2296, 2867, - 1861, 3180, 1969, 2531, 1575, 2325, 2937, 2326, 2868, 1317, - 3291, 2939, 1203, 2552, 2811, 1783, 2342, -2044, 2650, 902, - 1695, -1244, -1322, 903, 2546, -849, 1576, 2141, 1918, -511, - 2348, -511, -515, -1891, -515, 1575, 2656, 1559, 2367, 1567, - 1577, 3247, 2031, 2032, -1263, 3101, 3248, -1891, -1263, 2641, - 2373, -854, -1882, 1567, -1899, 1744, 534, 1576, -1264, 1145, - -1102, 2724, 3221, 1326, 1606, -1882, 1198, 1344, 1345, 2812, - 3365, 1579, -1899, -1264, 1560, 865, 1865, 2745, 1858, 1565, - 2401, 966, 3159, -844, 1240, 2069, 1240, 2406, 1656, 2143, - 993, 2017, 1588, 3372, 2661, 1344, 1345, 3134, 3135, 3373, - 1329, 1588, 1781, 1777, 1939, 1913, 1914, 1915, 1916, 1917, - 1918, 868, 971, 2127, -1123, 2052, 1458, 2221, 1777, 1644, - 1378, 1379, 996, 1318, -2044, 2440, 865, 1778, 1696, 2440, - 3092, 1639, 1640, 1641, 1642, 1643, 1644, 968, 2873, 77, - 778, 3498, 1778, 1698, 2070, 884, 2278, -227, -227, 2591, - 1611, 2588, 520, 2372, 1611, 2853, 2025, 2461, 3002, 1641, - 1642, 1643, 1644, 520, 536, 2153, 1692, 1459, 2055, 1338, - 2056, 3094, 1339, 1915, 1916, 1917, 1918, 2577, 3438, 2886, - 1759, 2854, 3523, 1188, 2400, 1621, 881, 1562, 2018, 1699, - 2154, 2407, 3151, 882, 815, 2251, 2071, 2024, 520, 520, - 2714, 2715, 2716, 2717, 2609, 2254, 883, 1572, 2257, 3292, - 71, 2874, 3293, 995, 811, 2335, 3127, 2610, 975, 1623, - 817, 520, 1199, 1623, 3132, 811, 1446, 537, 3041, 2966, - 2407, 3043, 1453, 3045, 977, 3057, 518, 1382, 1383, 58, - 70, 2009, 54, 2411, 1146, 68, 1756, 518, 69, 65, - 1029, 2250, 82, 102, 3267, 93, 3160, 900, 1154, 77, - 98, 1856, 2589, 902, 1140, 520, 714, 903, 2937, 1283, - 520, 1600, 2953, 2939, 2954, 1132, 815, 2341, 23, 1226, - -1891, 2343, 1140, 1201, 2345, 2416, 906, 3172, 3281, 3513, - 2520, 2091, 1031, 2213, 107, 2875, 918, 2412, 2351, 2190, - 2190, 2308, 925, 1032, 844, 844, 1701, 844, 2482, 2274, - 2275, 2276, 1436, 3253, 536, 1227, 2361, 51, 2049, 2109, - 2249, 2362, 1283, 1443, 52, 53, 1672, 56, 1439, 1673, - 1462, 57, 60, 971, 1466, 1450, 61, 1293, -208, 1336, - 2260, 520, 520, 63, 2354, 2267, 66, 520, 67, 80, - 520, 520, 2773, 520, 520, 520, 520, 1292, 81, 3240, - 1349, 83, 84, 90, 77, 778, 91, 92, 1702, 520, - 1349, 97, 101, 1349, 103, 1758, 520, 537, 2363, 3401, - 3402, 1196, 2532, 2097, 2533, 929, 1203, 1197, 2847, 1749, - 1924, 946, 1196, 1749, 520, 1283, 1545, 943, 1197, 930, - 1750, 813, 1758, 2097, 1752, 2625, 1346, 541, 930, 962, - 2098, 1862, 813, 865, 1863, 520, 2626, 1388, 1199, 805, - 1758, 27, 28, 29, 71, 864, 805, 937, 1758, 1783, - 2098, 811, 3442, 3229, 520, 545, 942, 1758, 811, 2630, - 2413, 3067, 2110, 2252, 3230, 520, 520, 520, 2255, 520, - 520, 2563, 2552, 518, 714, 3193, 2500, 3344, 1011, 2111, - 518, 947, 108, 1349, 2112, 982, 1469, 2225, 1209, 1575, - 902, 931, -1244, 109, 903, 2848, 994, 2405, 1196, 2590, - 931, 2591, 520, 2501, 1197, 923, 34, 924, 1210, 1598, - 2617, 1576, 1198, 1575, 1012, 1035, 1036, 1037, 1196, 520, - 520, 1788, 2113, 1198, 1197, 1577, 1686, 1687, 110, 1693, - 1452, 2592, 972, 882, 882, 1576, 882, 1867, 1983, 3361, - 1014, 1984, 1046, 39, 880, 2099, 1856, 1451, 1791, 1579, - 2100, 1456, 1604, 520, 1211, 2101, 1592, 520, 520, -2013, - -2013, 2849, 1607, 2850, 71, 2099, 2614, 520, 520, 520, - 2284, 1551, 520, 47, 2618, 2101, 41, 541, 111, 962, - 2464, 1657, 2360, 2622, 948, 2666, 2364, 44, 932, 2366, - 1662, 2674, 1545, 1140, 902, 1869, 2555, 1816, 1599, 1759, - 1046, 1046, 949, 1045, 2456, 545, 1901, 1172, 959, 1198, - 1902, 1903, 1173, 1611, 1904, 1905, 1906, 1612, 1613, 2740, - 2417, 978, 2418, 2502, 1196, 1283, 1759, 933, 2503, 1198, - 1197, 2775, 1942, 979, 1283, -592, 1817, 1943, 813, 980, - -592, 47, 2129, 981, 1759, 813, 2130, 2007, 1621, 1212, - 2008, 1932, 1759, 1934, 1935, 2498, 1784, 2419, 1283, 2420, - 984, 1759, 934, 1130, 1131, 1756, 1133, 1018, 1199, 71, - 1698, 1818, 2599, 2528, 2601, 2292, 987, 2114, 2293, 1199, - 1545, 2355, 1623, 1174, 2356, 988, 1445, 1823, 2049, 989, - 3003, 2393, 1756, 2602, 2394, 2604, 714, 1824, 1213, 990, - 1826, 1827, 804, 2725, 991, 714, 2102, 2732, 1214, 804, - 1756, -592, 935, 3509, -2014, -2014, 1699, 2103, 1756, 998, - 1215, 1819, 971, -2015, -2015, 1230, 2102, 1756, 992, 1201, - 2441, 1850, 1175, 2442, 993, 1198, 2212, 2103, 2444, 714, - 1201, 2442, 1027, 3472, 1552, 520, 1030, 968, 2605, 77, - 778, 2606, 1216, 2840, 2530, 2842, 1564, 1033, 3484, 2611, - -592, 1231, 2612, 2677, 1038, 1199, 1984, 3496, 2737, 3497, - 1039, 2442, 1202, 2566, 1872, 932, -2018, -2018, 2935, 2504, - 1816, 1040, 1700, 1590, 1960, 1199, 1961, 2556, 2556, 1963, - 2505, 2884, 520, 520, 1967, 1041, 2877, 1970, 520, 1971, - 520, 2841, 2952, 1975, 1218, 520, 520, 520, 520, 2648, - 3522, 1871, 1134, 3520, 933, 1469, 1908, 1138, 3521, 1817, - 520, 520, 2258, -2019, -2019, 520, 1201, 520, 23, 1219, - 520, 2738, 1203, 1701, 2008, 520, 2741, 520, 520, 2742, - 520, 2762, 2763, 1203, 520, 2962, 1201, 1149, 1221, 2652, - 2876, 1155, 2885, 1169, 3166, 1171, 1370, 1371, 2259, 1162, - 2743, 1156, 811, 2742, 880, 880, 2818, 880, 2976, 2819, - 2977, 2356, 1158, 2293, 1159, 805, 2996, 805, 1202, 2997, - 805, -2020, -2020, 1160, 518, 805, 1909, 811, 805, 811, - 805, 1199, 811, 1163, 805, 1702, 3330, 811, 3331, 935, - 811, 1545, 811, 1592, 1819, 3004, 811, 3058, 3005, 518, - 2008, 518, 1170, 2748, 518, 2751, 2753, 2754, 2749, 518, - 2750, 2752, 518, 1189, 518, 536, 1378, 1379, 518, 1203, - 536, 1191, 3194, 520, 520, 2008, 1193, 3195, 1194, 2013, - 2442, 2010, 520, 520, 2011, -2021, -2021, 2012, 2265, 1203, - 520, 1195, 1201, 1207, 2015, -1308, 115, 520, 77, 2016, - 531, 1208, 3224, 3459, 3231, 2008, 3241, 1984, 744, 3242, - 1225, 27, 28, 29, 3274, 3275, 3310, 2356, 2293, 2008, - -2023, -2023, 714, 832, 2266, 1229, 520, 845, 537, 520, - -2024, -2024, 1233, 537, 1545, 520, 520, 520, 520, 520, - 520, 520, 520, 714, -2025, -2025, 1234, 520, 520, 2552, - 972, 3370, 520, 1286, 2356, 3383, 520, 1289, 3384, 520, - 520, 520, 520, 520, 520, 520, 520, 520, 2138, 1290, - 520, 1295, 2683, 1297, 71, 1301, 34, 520, 2085, 1283, - 1311, 907, 3409, 1382, 1383, 3410, 1313, 2307, 2069, 813, - 3453, 3494, 2173, 3384, 3384, 1203, 1314, 3138, 520, 882, - 1321, 539, -2026, -2026, 3102, 2744, 2746, 908, -2027, -2027, - 1322, 815, 1328, 39, 813, 1331, 813, -2028, -2028, 813, - 520, 1146, -2029, -2029, 813, 2884, 1332, 813, 1337, 813, - 1355, 520, 520, 813, -2030, -2030, 1910, 1911, 1912, 1356, - 1913, 1914, 1915, 1916, 1917, 1918, 41, 1639, 1640, 1641, - 1642, 1643, 1644, -2032, -2032, -2033, -2033, 44, 1292, 1359, - 2144, -2034, -2034, -2035, -2035, -207, 2995, 1362, 1545, -2036, - -2036, 1417, 909, -2037, -2037, 2834, 1430, 714, 804, 714, - 804, 3155, 1433, 804, -2039, -2039, 1432, 2033, 804, 1440, - 2970, 804, 1987, 804, 2034, 2035, 1448, 804, 2036, 2037, - 2038, 910, 2958, -2041, -2041, 3471, 2192, 2193, 541, 3473, - 542, 47, 2237, 541, 2241, 962, -2043, -2043, 2988, 520, - 1803, 1804, 1447, 1283, 1454, 3069, 520, 520, -643, -643, - 1008, 1467, 14, 15, 1455, 1009, 545, 1461, 544, -647, - -647, 545, -646, -646, 911, 1465, 3309, 1380, 1381, 1547, - 1382, 1383, 1550, 3220, 3098, 3099, 3480, 3481, 3506, 3507, - 1349, 1548, -820, 1045, 3514, -827, 1901, 1283, 1667, 1668, - 1902, 1903, 2329, 1559, -2044, -2044, -2044, 23, 1563, 714, - 47, -668, -669, 71, -817, -818, 1283, 520, 2270, 1573, - -821, 1574, -819, 1584, 1010, 971, 1603, 1597, 1605, 1650, - 1666, 2285, 2285, 1652, 1654, 1675, 1676, 1680, 1685, 1684, - 1173, 2552, 1545, 1175, 1722, 1720, 520, 520, 1724, 520, - 1736, 1754, 1755, 1756, 1545, 520, 520, 520, 520, 520, - 520, 1761, 1762, 520, 520, 520, 520, 520, 520, 520, - 520, 520, 520, 1763, 2430, 1768, 1776, 1780, 520, 520, - 2433, 2857, 520, 1775, 1545, 2935, 1790, 2858, 112, 520, - 1796, 1545, 2095, 3130, 1806, 1011, 1807, 1808, 1346, 912, - 2859, 1812, 3139, 3140, 2359, 1821, 1822, 1815, 1831, 1830, - 913, 520, 1834, 1838, 1837, 520, 1840, 520, 1841, 1843, - 3042, 520, 1844, 1851, 2860, 1852, 2861, 3309, 1856, 1545, - 1864, 1012, 3272, 1545, 1890, 520, 1892, 1283, 1893, 1545, - 1929, 1895, 1962, 1898, 1921, 914, 1930, 1013, 1937, 1940, - 27, 28, 29, 1958, 1968, 1974, 1610, 1014, 1972, 1611, - 880, 1545, 1973, 1612, 1613, 805, 872, 1979, 915, 1982, - 3309, 805, 520, 520, 1553, 1985, 1560, 811, 1991, 1990, - -593, 1565, 2020, 811, -524, -593, 1542, 1986, 1988, 1989, - 2019, 1015, 3040, 1046, 1621, 916, 2053, 2054, -524, 518, - 2551, -2044, 554, -524, 1611, 518, 1592, 2059, 745, 2062, - 865, 2065, 520, 2862, 2087, 34, 2064, 520, 520, 3309, - 821, 3363, 2066, 2094, 2067, 2105, 36, 2106, 1623, 2108, - 2132, 2133, 2139, 2156, 862, 862, 873, 1016, 2152, 873, - 2151, 2172, 520, 520, 1017, 2169, 2181, 520, 38, 2182, - 2183, 2186, 39, 2184, -524, 3325, -593, 2185, 1349, 2203, - 2207, 520, 2208, 1349, 520, 2211, 520, 2214, 2218, 2219, - 2220, 2290, 874, 2294, -524, 2300, 1909, 2311, 2309, 2312, - 2313, 2863, 520, 714, 1018, 41, 2864, 1664, 2314, 520, - 2315, 2336, 520, 2507, 2310, 2316, 44, 893, 2330, 2331, - 1534, 2333, 2337, 1019, 2340, -593, 71, -524, 2357, 520, - 1868, 1870, 45, 2365, 874, 2338, 3116, 1901, 2344, 2395, - 2402, -524, 520, -524, -2044, 1926, -524, 1925, 2403, 2404, - 2426, 2408, -524, 2434, 2446, 2865, 46, 2448, 2414, 520, - 520, -2044, 805, 2415, 2432, 2453, -2044, 2935, 2462, 2454, - 47, 2866, 2457, 2455, 811, 2458, 520, 2459, 520, 805, - 2460, 2463, 1542, 2474, 813, 2470, 2478, 520, 2471, 2476, - 813, 811, 1020, 2477, 2473, 2813, 518, -524, 2475, 2479, - 1783, 2480, 2481, 2491, -2044, 714, 714, 714, 2494, 2499, - 2515, 2492, 2523, 518, 2534, 1610, 2539, -524, 1611, 3261, - 2540, 2506, 1612, 1613, 2537, 2547, -2044, -2044, -2044, 2516, - 2039, 2040, 2041, 2521, 2042, 2043, 2044, 2045, 2046, 2047, - 2237, 2237, 2237, 2545, 520, 865, 2522, 972, 804, 23, - 1283, 2535, -649, 1621, 804, 2548, 2565, 2558, 2562, 1632, - 1622, 2569, 2559, 2561, 2570, 2573, -524, 2576, 2571, 2575, - 1542, 2579, 2867, 2613, 2598, -524, 2600, 2631, 2619, 1823, - 2620, 2868, 2621, 875, 2632, 2607, 714, 1623, 2647, 1824, - 2633, 1545, 1826, 1827, 2651, 2634, 1534, 2646, 2667, 2663, - 2671, -524, 2675, 2664, 2680, 2668, -2044, -2044, -2044, 520, - 1913, 1914, 1915, 1916, 1917, 1918, 2690, 2698, 2701, 2708, - 2721, 1872, 2704, 971, 2722, 2733, 2707, 2734, 2709, 714, - 2739, 2710, 2758, 2770, 2069, 1868, 1870, 2639, 2747, 2780, - 2755, 2799, 2808, 2764, 2793, 2827, 2778, 1349, 2781, 2831, - 2801, 3314, 2839, 3316, 2872, 2653, 2784, 2888, 2969, -2044, - 2980, 813, 2805, 2984, 1872, 2794, 2993, 520, 2829, 902, - 3324, 2846, 1158, 903, 2825, 2852, 2845, 2974, 813, 2964, - 1590, 2814, 2955, 1624, 1534, 2961, 2956, 2957, 2965, 3000, - 2973, 2979, 27, 28, 29, 3006, 1346, 2293, 2095, 3001, - 1625, 3026, 3028, 3032, 2688, 1626, 2689, 520, 3036, 3446, - 2694, 3326, 2697, 3328, 2517, 3046, 3047, 3049, 3089, 865, - 3050, 3070, 3077, 1545, -524, 804, 3080, 3091, 3095, 811, - 3096, 3082, 3097, 2796, 3103, 3104, 3105, 3109, 3407, 520, - 3113, 3114, 804, 1629, 3115, 520, 520, 3123, 3126, 3411, - 3128, 518, 865, 3129, 3133, 3400, 520, 34, 3163, -2012, - -2013, 3147, -2014, -2015, -2016, -2017, -2044, 520, -2018, -2019, - 520, -2020, 520, 1639, 1640, 1641, 1642, 1643, 1644, 1545, - 520, 3146, 3148, 520, 520, 3395, 3149, 3152, 520, 520, - 3153, 1542, 902, -2021, 39, 520, 903, 1610, 1632, -2023, - 1611, -2024, -2025, 2833, 1612, 1613, -2026, 3150, -2027, -2028, - 520, -2029, 3167, -2030, -2032, 3168, -2033, 2013, -2034, 2010, - 520, -2035, 2011, 2817, -2036, 2012, -2037, 41, -2038, 3182, - -2039, 3170, 2015, 516, 527, 1621, 77, 2016, 44, 552, - 520, -2040, -2044, -2041, -2042, 552, -2043, -1261, 3184, 802, - 3161, 816, 816, 3186, 45, 3187, 820, 552, 828, 3190, - 939, 828, 3196, 3197, 846, 850, 3200, 3202, 850, 1623, - 3208, 552, 552, 3211, 3204, 3214, 1209, 23, 46, 714, - 3210, 3215, 2938, 714, 1542, 714, 3216, 3223, 3219, 3239, - 3243, 3238, 2815, 3249, 3246, 864, 1210, 520, 1634, 520, - 3251, 802, 802, 3263, 3264, 1534, -1260, 3271, 3273, 1349, - 3280, 3282, 3283, 3298, 2237, 3311, 813, 3296, 2940, 3297, - 2241, 3312, 3315, 3318, 3319, 3321, 846, 3327, 3332, 2967, - 3352, 1823, -1774, 850, 552, 850, 850, 850, 3122, 3356, - 3358, 1824, 1211, 2878, 1826, 1827, 3359, 3362, 3368, 3376, - 3375, 3377, 3381, 2879, 3386, 2931, 3388, 2989, 3390, 3435, - 2991, 3393, 865, 3073, 3399, -2044, 3406, 3397, 3394, 3398, - 3408, 3404, 3413, 3418, 520, 3424, 3425, 1545, 3426, 3431, - 3433, 3432, -2044, 3451, 3461, 3441, 3464, -2044, 2639, 3485, - 3462, 3466, 3443, 3445, 520, 520, 3482, 3463, 1534, 520, - 3499, 3510, 520, 3504, 3515, 1635, 3524, 3525, -2044, -2044, - -2044, 2880, 1639, 1640, 1641, 1642, 1643, 1644, 1542, 2887, - 27, 28, 29, 1161, 2245, -2044, 2328, 2596, 3339, 520, - 3439, 2891, -1774, 1439, 3403, 2346, 2820, 1212, 3508, 3136, - 2682, 3422, 3492, 520, 3259, 3460, 23, 3467, 520, 520, - 3486, 865, 958, 520, 1545, 1746, 3295, 2578, 520, 2983, - 3458, 520, 520, 3465, 2603, 2574, 520, 1283, 2856, 1590, - 520, 2941, 3456, 1854, 520, 1610, 904, 2642, 1611, -1774, - 1632, 520, 1612, 1613, 1820, 34, 1213, 2687, 3191, 2564, - 2550, 2963, 1677, -1774, 1437, 1438, 1214, 1717, -1774, 2210, - 1716, 2712, 1590, -1774, 3444, 2560, 2178, 3389, 1215, 3317, - 2881, 2209, -1774, 1621, 806, 2536, 3035, -1774, 1721, 3218, - -2044, 71, 39, 2438, 805, 520, 997, 71, 983, 2706, - 2705, 2736, 1534, 520, 3265, 2452, 811, 3392, 14, 15, - 1216, 2353, 3391, 2489, 2553, 2760, 2513, 1623, 2511, -1774, - 2467, 3079, 520, 2832, 1949, 41, 1404, 1389, 518, 1391, - 2490, 1545, 1542, 1395, 1396, 1397, 44, 1398, 862, -1774, - 3081, 1399, 1400, 2772, 1542, 2806, 3379, 976, 2288, 2445, - -2044, 2147, 45, 23, 1866, 1888, 2149, 2821, 890, 27, - 28, 29, 1218, 2623, 0, 2938, 1792, 0, 0, 0, - 0, 0, 0, 0, 1542, 0, 46, 0, 0, 23, - 0, 1542, 2785, 0, 0, 0, 0, 1219, -1774, 0, - 47, -1774, 0, 0, 0, 0, 0, -1774, 0, 0, - 0, 0, 0, 71, 0, 0, 1221, 714, 2882, 0, - 0, 2883, 520, -2044, 0, 0, 1536, 0, 0, 1542, - 0, 0, 865, 1542, 34, 0, 0, 0, 0, 1542, - -2044, 71, 3174, 0, 71, -2044, 0, -1774, 0, 0, - 971, 0, 1872, 520, 1545, 0, 1534, 0, 0, 0, - 0, 1542, 0, 0, 0, 0, 0, -2044, 1534, 520, - -1774, 39, 0, 0, 1639, 1640, 1641, 1642, 1643, 1644, - 0, 0, 1045, -2044, 0, 1901, 0, 0, 0, 1902, - 1903, 0, 1590, 1904, 1905, 1906, 0, 1545, 1534, 0, - 3169, 0, 520, 813, 41, 1534, 27, 28, 29, 0, - 2776, 0, 0, 520, 520, 44, 0, 520, 0, 520, - 0, 3337, 0, 1346, 0, 0, 0, 0, 852, 0, - 1756, 45, 27, 28, 29, 0, 0, 0, 1632, 0, - 0, 850, 0, 1534, 520, 0, 850, 1534, 0, 850, - 0, 0, 0, 1534, 0, 46, -1774, 552, 0, 3173, - 0, 0, 0, 0, 0, 0, -1774, 804, 520, 47, - 0, 34, 0, 0, 0, 1534, 0, 0, 0, 23, - 0, 864, 36, 0, 0, -1774, 0, -1774, -1774, 0, - 0, 0, 0, 0, 0, 0, 0, 34, 0, 2938, - 0, 0, 1536, 3236, 38, 1414, 0, 0, 39, 0, + 520, 904, 70, 54, 865, 1124, 1224, 877, 1292, 714, + 68, 58, 65, 1022, 82, 102, 520, 713, 1188, 1680, + 953, 77, 69, 517, 1784, 1420, 93, 1154, 1241, 1783, + 1795, 1925, 1295, 2172, 2139, 1394, 972, 1348, 1767, 811, + 847, 77, 2557, 1748, 1395, 1350, 1331, 98, 1581, 1475, + 772, 1393, 1830, 740, 2530, 881, 2532, 1391, 1982, 1754, + 1736, 882, 2590, 1758, 2547, 2050, 2450, 778, 520, 520, + 851, 1337, 1354, 51, 2074, 2075, 2940, 52, 863, 1800, + 2660, 53, 2942, 2511, 805, 2971, 56, 1788, 57, 60, + 61, 811, 811, 2546, 2274, 819, 63, 1856, 66, 67, + 80, 2583, 889, 2988, 1656, 81, 83, 844, 1223, 1659, + 1229, 518, 1233, 963, 84, 955, 878, 879, 2969, 90, + 91, 92, 1593, 1241, 97, 101, 2985, 518, 966, 103, + 1004, 2630, 2631, 2632, 2769, 1025, 805, 805, 2644, -1893, + 2833, 897, -1324, -858, 2838, 1046, 1046, 2304, 1362, -1893, + -512, 813, -1265, 2195, -516, 2452, 2402, -2033, -2033, 3384, + -1262, -1262, 2143, -850, 1612, 2299, 2192, 3180, -1884, 1614, + -1265, -1246, 2089, 1325, -1901, -1884, -2024, -2024, -858, 518, + 518, 1609, -2042, -2042, 973, -855, 1404, -1266, -1901, -855, + 815, 1558, 1559, 2661, 1046, -1263, -1263, -2019, -2019, 1622, + -1266, -817, 1664, 813, 813, -830, -845, 815, 1571, 815, + 1350, 3165, -2044, -2044, 1210, 2300, 2431, 2432, 2648, 2469, + 1660, 1341, 815, 2652, 2089, 1902, 1046, 2438, 2646, 3184, + 1904, 2442, 3113, 1624, 1211, 1325, 532, 855, 1750, 2639, + 1143, 815, 3258, 2694, 2696, 1166, 2699, 1177, 1341, 1786, + -512, 1563, 1543, 1046, -516, 1589, 1180, 972, 2733, 2734, + 1798, 1739, 1136, 3096, 1241, -227, 1316, 1573, 1325, 1799, + -227, 3, 4, 1759, 2296, -466, 2668, 3271, 1768, 1612, + 1212, 1771, 1772, 1613, 1614, -649, 1350, 999, 3128, 1350, + 1350, 2205, 3148, 1902, 3418, 1942, 1699, 1903, 1904, 815, + 869, 944, 1612, 874, 1750, 1317, 1613, 1614, 1025, 3343, + 2414, 815, 1902, 2962, 1622, 2618, 1903, 1904, 3450, 874, + 1004, 2412, 3079, 71, 3081, -1104, 3457, 2805, 1942, 2807, + 3181, 3465, 2716, -1104, 1661, 1140, 872, 1622, 1355, 2989, + 3302, 1773, 1700, 71, 3514, 3182, 804, 1988, 1624, 783, + 3303, 1683, 533, 1005, 1665, 1742, 1731, 1732, 2586, 1735, + 2198, 71, 3124, 3204, 2415, 2110, 2076, 2425, 2426, 2427, + 864, 1624, 2705, 1173, 2126, 1213, 3370, -1125, 1174, 3408, + -2018, -2018, 2127, 1696, 3443, -1125, 1173, 2806, 833, 3147, + 870, 2167, 3110, -787, 1167, 2280, 2847, 1843, 804, 804, + 880, 1459, 3337, 3503, 3338, 1612, 873, 1144, 1701, 1145, + 2375, 2136, 1444, 2876, 3174, 2826, 2846, 1802, 3288, 3501, + 1869, 1871, 2856, 71, 1214, 3186, 2829, 2522, 1576, 1543, + 3419, 3388, 1868, 1576, 1215, 1500, 782, 1046, 2098, -512, + 1622, 2403, 1031, -516, 1778, 1295, 1216, 3519, 2857, 1175, + 1577, 3175, 1460, 1032, 2764, 1577, 2583, 1662, 2583, 1702, + 3526, 3369, 1175, 814, 1578, 2099, 3111, 1326, 1779, 3498, + 3430, 3472, 3451, 3433, 1624, 2199, 2168, 1760, 1217, 3238, + 3390, 3240, 2827, 1318, 874, 1774, 2877, 784, 2111, -669, + 834, 1697, 1006, 2587, 1581, 1775, 1168, 2995, 1176, 3269, + 2226, 3452, 3458, 1757, 2990, 2112, 3515, 2416, -512, 1575, + 2113, 1684, -516, 3010, 3420, 1744, 3304, 1543, 2413, 1137, + 3490, 1703, 3125, 3520, 2706, 3253, 1801, 3202, -787, 1326, + 1219, 2297, 3247, 980, 3344, 3183, 919, 1682, 1045, 3460, + 2090, 1902, 1985, 1757, 3263, 1903, 1904, 1011, 2114, 1905, + 1906, 1907, 2021, 2733, 2734, 1220, 3473, 3431, 2281, 1733, + 2137, 3504, 1326, 1675, 1750, 2206, 3259, 3496, 1734, 3114, + 2878, 3001, 2446, 2288, 1222, 1751, 2963, 1601, 2981, 856, + 2100, 714, 530, 1012, 1787, 2285, 2217, 2978, 2785, 941, + 2102, 902, 2513, 1934, 1138, 903, 2993, 1319, 2050, 1445, + 867, 953, 3316, 2703, 2279, 3159, 1858, 1040, 1949, 1014, + 945, 2534, 1862, 2940, 852, 3521, 1576, 3432, 2383, 2942, + 2560, 1554, 1558, 1559, 3279, 1970, -512, 972, 1327, 871, + -516, 2343, 1745, 875, 2192, 1750, 883, 1784, 1577, 2640, + 2641, 2026, 3137, 3138, 2370, 2664, 1753, 1571, -1893, 2748, + 2619, 1576, 1578, 2056, 1568, 2057, 2376, 2653, -1893, -1324, + 3466, -1265, 2555, 1645, 1568, -512, 2349, -512, 2549, -516, + 3250, -516, -850, 1577, 2142, 3251, 3104, -1884, 902, -1265, + -1246, 2072, 903, -1901, -1884, 1563, 2404, 1580, 2727, 3462, + 1330, -1104, 2644, 2409, -855, 534, -1266, -1901, 1560, 1607, + 3224, 1146, 1859, 2115, 1866, 1573, 865, 2032, 2033, -1266, + 1561, 2659, 3368, 993, 1566, -845, 1657, 3376, 2144, 966, + 2018, 1589, 1345, 1346, 1919, 1241, 2070, 1241, 1908, 3491, + 1589, 2443, 3351, 3162, 2154, 2443, 1018, 1778, 1543, 2010, + 2128, 2103, 971, -1125, 2053, 3122, 3375, 1173, -593, 1345, + 1346, 1909, 2104, -593, 3095, 1446, 1940, 865, 1612, 2155, + 2384, 1779, 2222, 2464, 3086, 112, -227, -227, 3522, 77, + 2385, 858, 2850, 2071, 2612, 1642, 1643, 1644, 1645, 968, + 868, 104, 520, 2814, 2191, 2191, 3005, 2613, 2279, 1916, + 1917, 1918, 1919, 520, 3352, 778, 1640, 1641, 1642, 1643, + 1644, 1645, 3087, 2410, 3097, 811, 1914, 1915, 1916, 1917, + 1918, 1919, 3404, 3405, 3492, 1189, 811, 3154, 3178, 3123, + 3441, 1910, 2223, 1175, -593, 2222, 881, 1624, 520, 520, + 105, 1543, 882, 2717, 2718, 2719, 2720, 2252, 2815, 2364, + 2969, 996, 2410, 3493, 2365, 1350, 3163, 2255, 113, 2628, + 2258, 520, 921, 977, 922, 1350, 3130, 71, 1350, 2851, + 2629, 995, 2336, 3060, 3135, 3445, 1035, 1036, 1037, 3179, + 70, 54, 1176, -593, 1759, 2940, 2251, 2594, 68, 58, + 65, 2942, 82, 102, 1141, 2580, 1147, 1759, 902, 77, + 69, 1155, 903, 518, 93, 520, 714, 1601, 3175, 1284, + 520, 2366, 1141, 2092, 518, 2523, 884, 2889, 3044, 844, + 844, 3046, 844, 3048, 2019, 98, 1437, 813, 1339, 900, + 2485, 1340, 3232, 2025, 2956, 2852, 2957, 2853, 813, 2419, + 2309, 3516, 1759, 3233, 1463, 2275, 2276, 2277, 1467, 3284, + 2353, 51, 1569, 1570, 3256, 52, 2342, 930, 1350, 53, + 2344, 2214, 1284, 2346, 56, 1870, 57, 60, 61, 1699, + 1046, 1294, 2776, 971, 63, 1543, 66, 67, 80, 2050, + 815, 520, 520, 81, 83, 2357, 1337, 520, 2250, 1576, + 520, 520, 84, 520, 520, 520, 520, 90, 91, 92, + 2098, 3243, 97, 101, 77, -1893, 930, 103, 2261, 520, + 1782, 1577, 2558, 2268, 2591, 1700, 520, 1046, 1379, 1380, + 1925, 2193, 2194, 1789, 1444, 1580, 2535, 2099, 2536, 931, + 778, 3270, 811, 2503, 520, 1284, 1546, 536, 1857, 811, + 906, 1911, 1912, 1913, 872, 1914, 1915, 1916, 1917, 1918, + 1919, 1943, 536, 1447, 865, 520, 1944, 1440, -594, 1454, + 2504, 1784, 2363, -594, 1451, 2253, 2367, 902, 1792, 2369, + 2256, 1600, 2130, 2566, 520, 918, 2131, 805, 931, 2459, + 925, 71, 864, 2226, 805, 520, 520, 520, 1760, 520, + 520, 2633, 3364, 3196, 714, 923, 1693, 924, 3347, 1857, + 537, 1760, 1599, 3070, 902, 2860, -1246, 1933, 903, 1935, + 1936, 2861, -208, 2555, 873, 537, 2408, 1452, 929, 1543, + 518, 1457, 982, 520, 2862, 2592, 932, 518, 937, 2620, + 2860, 1543, 1702, 994, -594, 1383, 1384, 1552, 942, 47, + 520, 520, 2100, 2420, 813, 2421, 1760, 2101, 2863, 943, + 2864, 813, 2102, 1673, 1757, 946, 1674, 1687, 1688, 1011, + 1694, 1543, 947, 882, 882, 933, 882, 1757, 1543, 1863, + 972, 948, 1864, 2863, 520, 1817, 949, 3289, 520, 520, + 2505, 880, 1593, -594, 2326, 2506, 2327, 1759, 520, 520, + 520, 959, 874, 520, 1703, 1012, 2605, 2669, 2607, 2467, + 934, 71, 1759, 2677, 1141, 2422, 1543, 2423, 23, 536, + 1543, 1453, 1757, 1546, 1818, 1984, 1543, 2743, 1985, 1131, + 1132, 1014, 1134, 1045, 2008, 2293, 1902, 2009, 2294, 2533, + 1903, 1904, 3290, 978, 1905, 1906, 1907, 2865, 1543, 2358, + 2396, 3291, 2359, 2397, 115, 1293, 1284, 979, 531, 1819, + 935, 2778, 980, 932, 1197, 1284, 744, 2843, 981, 2845, + 1198, 984, 2559, 2559, 107, 3292, 1170, 1785, 1172, 1759, + 907, 832, 537, 2444, 2501, 845, 2445, 987, 1197, 1284, + 541, 1200, 962, 2447, 1198, 2608, 2445, 2614, 2609, 2531, + 2615, 2680, 933, 988, 1985, 541, 908, 962, 989, 1820, + 990, 1546, 991, 2103, 854, 2866, 71, 2728, 545, 992, + 2867, 2735, 3006, 2617, 2104, -2015, -2015, 714, 1824, 1825, + 993, 1827, 1828, 545, 998, 1851, 714, 2655, 2621, 2050, + 3261, 2602, 1027, 2604, 3512, 2867, 2507, 3293, 1030, 804, + 1033, 875, 2593, 971, 2594, 539, 804, 2508, 1018, 1038, + 3294, 27, 28, 29, 3499, 2740, 3500, 1039, 2445, 2868, + 714, 909, -2016, -2016, 3475, 1199, 520, 1446, 1873, 892, + 77, 895, 1197, 899, 2595, 2869, 2938, 935, 1198, 3487, + 968, 1553, 2741, 1135, 2868, 2009, 2569, -2017, -2017, 1199, + 910, 1760, 1040, 1565, 1041, 2625, 778, 3525, 2744, 1150, + 2869, 2745, 1293, 2651, 2746, 2887, 1760, 2745, 1139, -207, + 1157, 2880, 1156, 520, 520, 2821, 34, 1159, 2822, 520, + 1591, 520, 2979, -2020, -2020, 2359, 520, 520, 520, 520, + 2955, 1160, 108, 911, 3523, 2844, 1909, 2980, 1161, 3524, + 2294, 520, 520, 109, 811, 1163, 520, 2999, 520, 1164, + 3000, 520, 541, 39, 542, 1171, 520, 1757, 520, 520, + 2965, 520, 2765, 2766, 2879, 520, 2888, 3007, 1190, 811, + 3008, 811, 1757, 1760, 811, 1192, 2870, 1195, 110, 811, + 545, 1194, 811, 1199, 811, 2871, 41, 3061, 811, 3197, + 2009, 1196, 2009, 1208, 1961, 3198, 1962, 44, 2445, 1964, + 1209, 2870, 880, 880, 1968, 880, 1910, 1971, 1226, 1972, + 2871, 1817, 3227, 1976, 805, 2009, 805, 1371, 1372, 805, + 3234, 1200, 1546, 1985, 805, 2753, 2755, 805, 111, 805, + 1589, 1593, 518, 805, 2754, 2756, 2757, 1230, 3333, 1757, + 3334, 2752, 2747, 2749, 1234, 1200, 2751, 23, 1543, 3295, + 1818, 47, 3296, 1535, 520, 520, 813, 518, 912, 518, + 2014, 2011, 518, 520, 520, -2021, -2021, 518, 2012, 913, + 518, 520, 518, 1235, 3244, 1287, 518, 3245, 520, 77, + 2013, 813, 1202, 813, 2016, 3169, 813, 1379, 1380, 3277, + 3278, 813, 2359, 2294, 813, 3313, 813, 1290, 2009, 1291, + 813, 1296, 1201, 714, 914, 2017, 1202, 520, 1298, 3373, + 520, 1210, 2359, 1312, 1203, 1546, 520, 520, 520, 520, + 520, 520, 520, 520, 714, -2022, -2022, 915, 520, 520, + 2139, 1211, 2086, 520, 2686, 1820, 1302, 520, 1203, 1200, + 520, 520, 520, 520, 520, 520, 520, 520, 520, 972, + 2555, 520, 1314, 3386, 916, 3141, 3387, 1315, 520, 1197, + 1284, 3412, 1322, 3456, 3413, 1198, 3387, 3497, -644, -644, + 3387, 1329, 71, -2023, -2023, -2025, -2025, 1212, 2881, 520, + 1323, 2070, 1543, 2174, 1332, 1204, -2026, -2026, 2882, 882, + 27, 28, 29, 815, -2027, -2027, 1227, 1350, 3105, 2887, + 1202, 520, -2028, -2028, 1383, 1384, -2029, -2029, 1333, 1204, + -2030, -2030, 520, 520, 1147, 1338, 1911, 1912, 1913, 1356, + 1914, 1915, 1916, 1917, 1918, 1919, -2031, -2031, -2032, -2032, + 1535, 1357, 1228, -2034, -2034, -2035, -2035, 1360, 1543, -2036, + -2036, -2037, -2037, -2038, -2038, 1363, 2883, -2039, -2039, 1546, + 2837, -2041, -2041, -2043, -2043, 34, -2045, -2045, 714, 1418, + 714, 1804, 1805, 1988, -648, -648, 2238, 1433, 2242, 3158, + 1199, 2973, 1213, 1431, 2991, 1434, 804, 1441, 804, 1448, + 2961, 804, -647, -647, 1381, 1382, 804, 2998, 1449, 804, + 3474, 804, 39, 1455, 3476, 804, 1456, 920, 1383, 1384, + 520, 1462, 927, 1204, 1284, 928, 1466, 520, 520, 3101, + 3102, 3223, 3483, 3484, 14, 15, 3312, 1197, 1535, 3509, + 3510, 1214, 1468, 1198, 3072, 41, 1548, 3045, 1668, 1669, + 1549, 1215, 1551, -821, -828, 1560, 44, 1564, 47, -669, + -670, 1543, -818, 1216, 1045, 2884, -819, 1902, 1284, 3517, + 1574, 1903, 1904, 2330, -822, 1905, 1906, 1907, 1575, 23, + 714, -820, 1585, 1598, 1651, 1604, 1045, 1284, 520, 1902, + 1653, 1655, 2779, 1903, 1904, 1217, 971, -2046, -2046, -2046, + 1606, 71, 1667, 1676, 1677, 1681, 1685, 1686, 1176, 1174, + 47, 1721, 1723, 1546, 1725, 1755, 1737, 520, 520, 1756, + 520, 1762, 1757, 1764, 1763, 1546, 520, 520, 520, 520, + 520, 520, 2555, 2938, 520, 520, 520, 520, 520, 520, + 520, 520, 520, 520, 1769, 1776, 1200, 1219, 1199, 520, + 520, 1777, 1791, 520, 1781, 1546, 1543, 1797, 112, 1807, + 520, 1813, 1546, 1808, 1809, 1822, 1823, 1816, 1831, 1832, + 1835, 1838, 1220, 1839, 1841, 1845, 1842, 3133, 1844, 1852, + 2096, 1853, 520, 1857, 1865, 1350, 520, 3312, 520, 1891, + 1350, 1222, 520, 2885, 1893, 1197, 2886, 3275, 1896, 1899, + 1546, 1198, 1894, 1231, 1546, 1922, 520, 1202, 1284, 811, + 1546, 1930, 1931, 3142, 3143, 811, 1938, 1941, 1959, 1963, + 1969, 1975, 27, 28, 29, 1973, 1980, 1554, 1974, 1983, + 3312, 1986, 1546, 1543, 2433, 1991, 1561, 1869, 1871, 1232, + 2436, 1566, 1987, 520, 520, 1989, 3043, 1992, 880, 2021, + 1046, 1990, 2020, 2054, 805, 2055, 1612, 2060, 2065, 1535, + 805, 2063, 2088, 2106, 2068, 2066, 2109, 2067, 2095, 2107, + 2133, 2134, 2140, 986, 2816, 2152, 2153, 1909, 2157, 3312, + 873, 2170, 865, 520, 1593, 2173, 2183, 34, 520, 520, + 2184, 2185, 2182, 2186, 2187, 2208, 2209, 518, 36, 3366, + 2204, 2219, 2212, 518, 1200, 1757, 1199, 2215, 2220, 3328, + 1204, 874, 2295, 520, 520, 2221, 2291, 2301, 520, 2310, + 38, 813, 2312, 2313, 39, 1197, 2314, 813, 23, 2316, + 2315, 1198, 520, 2317, 2331, 520, 2332, 520, 2334, 2341, + 1543, 1008, 2337, 2339, 2338, 2360, 1009, 1910, 2345, 2368, + 1927, 1902, 1535, 520, 714, 2398, 1926, 41, 2405, 2406, + 520, 2213, 2407, 520, 2411, 1202, 2417, 1191, 44, 1910, + 3119, 2418, 2429, 2435, 1543, 2437, 2449, 2451, 2456, 2457, + 520, 2458, 2311, 2460, 45, 2461, 811, 2462, 2465, 2463, + 2466, 2473, 2476, 520, 71, 2938, 2502, 1203, 2474, 1288, + 2477, 2478, 2509, 811, 2481, 1010, 2479, 554, 46, 2482, + 520, 520, 2519, 745, 2480, 1304, 1306, 1309, 1311, 2497, + 2518, 2483, 47, 2526, 2524, 821, 2484, 520, 2525, 520, + 2494, 805, 2537, 2495, 2538, 2540, 1199, 2542, 520, 862, + 862, 2548, 1869, 1871, 1784, 3264, 2543, 2564, 805, -650, + 2551, 2561, 2550, 1543, 1350, 2562, 714, 714, 714, 2565, + 2817, 2568, 1200, 2572, 2238, 2238, 2238, 1407, 1204, 2573, + 2574, 27, 28, 29, 518, 2576, 1011, 1045, 2578, 2579, + 1902, 2582, 2601, 536, 1903, 1904, 1535, 2603, 1905, 1906, + 1907, 518, 2616, 2634, 2635, 520, 1543, 865, 813, -525, + 2622, 1284, 893, 2636, 2637, 3063, 2650, 2623, -1310, 2624, + 2654, 2649, 1012, -525, 2666, 813, 804, 972, -525, 2259, + 2667, 2670, 804, 1202, 2671, 2678, 2683, 2674, 1013, 2693, + 2701, 2704, 2707, 2710, 2724, 2725, 34, 714, 1014, 2711, + 1824, 1825, 1546, 1827, 1828, 1873, 537, 2712, 2713, 2736, + 2737, 2742, 2750, 2758, 2761, 2260, 2767, 1911, 1912, 1913, + 520, 1914, 1915, 1916, 1917, 1918, 1919, 2773, 2781, -525, + 2783, 2796, 1015, 39, 520, 971, 2784, 2802, 2811, -2046, + -2046, -2046, 714, 1914, 1915, 1916, 1917, 1918, 1919, -525, + 1873, 2804, 1200, 1965, 2834, 2787, 2797, 2070, 1543, 2808, + 2308, 2828, 3317, 2830, 3319, 2842, 41, 2832, 2849, 2848, + 2855, 2875, 2891, 2958, 2959, 3327, 2964, 44, 1016, 2960, + 520, 2967, 2968, 2972, 2977, 1017, 1204, 2976, 2982, 2983, + 1535, 2987, 2996, 45, 2294, 3003, 3004, 3029, -525, 3009, + 3031, 3035, 1535, 811, 3039, 3049, 3050, -525, 1591, 2266, + 3053, 3073, 3329, 1202, 3331, 3052, 3080, 46, 3083, 3085, + 520, 3092, 3098, 3094, 3410, 1018, 2096, 3449, 3099, 3100, + 3106, 2818, 1535, 865, 3108, 3112, 1546, 3116, 1738, 1535, + 3107, 3117, 2520, 3126, 1019, 2267, 3118, 3129, 2799, 3131, + 1909, 1765, 520, 804, 3132, -1776, 3414, 3136, 520, 520, + -2014, -2015, 3150, 3403, -2016, -2017, 865, -2018, 3149, 520, + 804, -2019, 1790, -2020, 3151, 3155, 1350, 1535, -2021, -2022, + 520, 1535, -2023, 520, 3152, 520, 541, 1535, 962, 3156, + 3398, 518, 1546, 520, -2025, -2026, 520, 520, -2027, -2028, + -2029, 520, 520, -2030, -2031, -2032, -2034, -2035, 520, 1535, + 3153, 544, -2036, 1020, 545, 813, 1204, 2836, 1966, -2037, + 1910, 1611, -2038, 520, 1612, -2039, 3166, -2040, 1613, 1614, + 2014, 2011, -2041, 520, -2042, 1815, 2820, -2043, 2012, -2044, + -2045, -1263, 3164, 3170, 1834, 3171, 516, 527, 3185, 77, + 2013, 3189, 552, 520, 2016, -1776, -525, 3173, 552, 1622, + 3187, 3190, 802, 3193, 816, 816, -2046, 3199, 3203, 820, + 552, 828, 3200, 3207, 828, 2017, 3211, 846, 850, 3213, + 3217, 850, 3205, 3219, 552, 552, 3214, 3218, 3226, 3222, + 3242, 3246, 714, 1624, 3252, 2941, 714, 3241, 714, 3254, + 2238, 3249, -1776, 3266, 2943, 3267, 2242, -1262, 3274, 1876, + 520, 3276, 520, 3283, 802, 802, -1776, 3285, 3286, 3299, + 1543, -1776, 3300, 864, 902, 3301, -1776, 1159, 903, 3314, + 3315, 3318, 3321, 3322, 3324, -1776, 3330, 3335, 3355, 846, + -1776, 3125, 2970, 3359, 3361, 3362, 850, 552, 850, 850, + 850, 3365, 3371, 3438, 1824, 1825, 1877, 1827, 1828, 3379, + 3378, 3380, 3384, 3389, 3391, 3393, 3076, 3396, 1537, 3407, + 2992, 3397, -1776, 2994, 3400, 3402, 865, 1878, 3401, 3409, + 3411, 3416, 3421, 1957, 3428, 3427, 3429, 520, 3434, -2046, + 1546, 3435, -1776, 3436, 3464, 1879, 3444, 3446, 3448, 1880, + 3467, 3454, 3469, 3485, 3488, 939, -2046, 520, 520, 3502, + 3465, -2046, 520, 1008, 3466, 520, 3507, 3513, 1009, 3518, + 3527, 1881, 3528, 1162, 1882, 2890, 2246, 2599, 817, 817, + 1911, 1912, 1913, 3342, 1914, 1915, 1916, 1917, 1918, 1919, + 1883, -1776, 520, 3442, -1776, 2894, 3406, 3511, 2823, -2046, + -1776, 1611, 2347, 3139, 1612, 2685, 520, 3425, 1613, 1614, + 3495, 520, 520, 3262, 3463, 865, 520, 1546, 1210, 3470, + 3489, 520, -525, 3298, 520, 520, 1747, 1010, 2581, 520, + 1284, 23, 2606, 520, 904, 3461, -525, 520, 1211, 1622, + -1776, -525, 2577, 2859, 520, 3468, -2046, 2986, 2944, 1591, + 3459, 2645, 1821, 2690, 1633, 3194, 2966, 1855, 2567, 1438, + 811, 1678, 2553, -1776, 2211, 2715, 1717, 901, 3447, 1535, + 2179, 3392, 3320, 1624, 2563, 1718, 2210, 806, 2441, 2539, + 3038, 1722, 1591, 1884, 1212, 1440, 1439, 3221, 520, 2709, + 2329, 1885, -525, 2739, 997, 1537, 520, 983, 1011, 2708, + 3268, 71, 2455, 1405, 1390, 805, 2355, 71, 1392, 1396, + 23, 1397, -525, 1886, 1398, 520, 2356, 1399, 1400, 1401, + 3395, 852, 3394, 2556, 1546, 2492, 2763, 1950, 2493, 2470, + 2516, 2514, 2775, 2835, 1012, 3084, 3382, 2809, 2500, 976, + 1887, 2289, 2448, 1889, 1867, 2148, 1008, 958, 518, -1776, + 1013, 1009, 2824, 2883, -2046, 2150, 890, 0, 2941, -1776, + 1014, -525, 2626, 3340, 0, 2147, 0, 2149, 1793, -2046, + -525, 0, 813, 0, 27, 28, 29, 2159, -1776, 1213, + -1776, -1776, 0, 1537, 0, 0, -2046, 0, 0, 0, + 0, -2046, 1210, 0, 1015, 0, 0, 0, 0, 0, + 714, 0, 0, 1535, 0, 520, 0, 0, 1873, 0, + 1010, 0, 1211, 71, 0, 2196, 865, -1776, 0, 0, + -1776, -1776, -1776, 0, 0, 3177, 0, 0, 1214, -2046, + 0, 0, 0, 971, 0, 0, 520, 1546, 1215, 34, + 1016, 71, 0, 0, 71, 0, 0, 1017, 0, 0, + 1216, 0, 520, 27, 28, 29, 0, 0, 1212, 1535, + 0, -2046, 0, 862, 0, 0, 0, 0, 1640, 1641, + 1642, 1643, 1644, 1645, 0, 0, 39, 0, 0, 0, + 1546, 1011, 1217, 0, 1633, 520, 0, 1018, 0, 0, + 0, 0, 1591, 0, 0, 0, 520, 520, 0, 0, + 520, 0, 520, 0, 0, 0, 1019, 0, 0, 41, + 2302, 2303, 2305, 2306, 2307, 0, 0, 1012, 34, -525, + 44, 0, 0, 0, 850, 0, 0, 520, 0, 850, + 0, 0, 850, 1013, 1219, 0, 45, 0, 1611, 0, + 552, 1612, 0, 1014, 0, 1613, 1614, 0, 0, 0, + 0, 520, 0, 1213, 0, 39, 0, 0, 0, 1220, + 46, 0, 1535, 0, 0, 0, 0, 804, 0, 0, + 0, 0, 0, 0, 47, 1020, 1622, 1015, 1222, 0, + 0, 864, 2941, -2046, -2046, 0, 0, 902, 41, 0, + 0, 903, 0, 0, 0, 536, 0, 0, 1187, 44, + 0, 0, 1214, 0, 1537, 0, 714, 0, 3280, 3281, + 1624, 0, 1215, 0, 1873, 45, 0, 0, 0, 0, + -1310, 0, 520, 1016, 1216, 1538, 520, 0, 0, 0, + 1017, 0, 0, 0, 0, 0, 520, 0, 520, 46, + 520, 0, 0, 0, 520, 2424, 520, 0, 520, 811, + 0, 1540, 0, 47, 0, 0, 1217, 1535, 537, 520, + 0, 0, 0, 0, 520, 1045, 520, 0, 1902, 0, + 1018, 0, 1903, 1904, 520, 2510, 1905, 1906, 1907, 0, + 0, 0, 0, 0, 3339, 0, 3341, 714, 0, 1019, + 0, -2046, 520, 0, 0, 3348, 0, 1537, 1640, 1641, + 1642, 1643, 1644, 1645, 0, 0, -2046, 0, 1219, 0, + 0, 0, 961, 0, 0, 0, 3176, 0, 0, 0, + 536, 0, 0, -2046, 0, 0, 0, 3374, -2046, 0, + 951, 552, 552, 1220, 1535, 0, 23, 518, 0, 0, + 0, 520, 0, 3377, 0, -1310, 71, 3381, 0, 0, + 0, 520, 1222, 1413, 23, 0, 0, 0, 1020, 0, + 862, 813, 0, 520, 0, 0, -2046, 0, 0, 0, + 0, 974, 527, 816, 0, 0, 0, 520, 516, 0, + 850, 0, 1591, 537, 0, 0, 0, 0, 540, 802, + 0, 0, 0, 1001, 1001, 520, 0, 0, 1001, 1024, + 811, 0, 1538, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 828, 828, 828, 0, 0, 0, 0, 0, + 520, 1633, 0, 0, 0, 0, 828, 828, 1540, 828, + 0, 1537, 0, 0, 0, 520, 0, 538, 541, 0, + 962, 1535, 850, 811, 2941, 0, 714, 0, 552, 0, + 0, 0, 0, 0, 3348, 2207, 539, 543, 0, 0, + 850, 0, 0, 544, 0, 0, 545, 0, 0, 520, + 3471, 0, 0, 0, 850, 1535, 0, 0, 1909, 27, + 28, 29, 0, 520, 520, 520, 0, 0, 518, 0, + 1538, 0, 811, 0, 0, 0, 3486, 27, 28, 29, + 0, 0, 0, 0, 0, 0, 850, 1289, 0, 0, + 520, 0, 813, 540, 0, 0, 1540, 0, 1300, 0, + -207, -2046, 850, 850, 850, 850, 0, 0, 0, 71, + 71, 518, 0, 0, 1860, 975, 1861, 817, 1321, 0, + 0, 0, 0, 0, 34, 0, 0, 1414, 1910, 0, + 0, 520, 0, 0, 1535, 813, 0, 0, 0, 0, + 0, 0, 34, 541, 0, 542, 0, 1029, 0, 0, + 0, 1001, 1024, 0, 850, 1537, 0, 1412, 0, 0, + 518, 39, 543, 1001, 1001, 0, 0, 1537, 544, 552, + 0, 545, 1133, 2647, 0, 802, 0, 1535, 0, 39, + 0, 0, 802, 0, 813, 71, 0, 71, 2662, 2663, + 2665, 0, 552, 0, 41, 0, 0, 1537, 0, 1469, + 0, 0, 0, 2676, 1537, 44, 2679, 0, -2046, 1550, + 0, 2684, 41, 0, 0, 1640, 1641, 1642, 1643, 1644, + 1645, 45, 1611, 44, 0, 1612, 0, 0, 71, 1613, + 1614, 0, 0, -2046, -2046, -2046, 0, 0, 0, 45, + 0, 0, 1537, 0, 71, 46, 1537, 0, 71, 1415, + 0, 0, 1537, 0, 0, 0, 0, 0, 552, 47, + 1622, 0, 0, 46, 0, 0, 0, 1623, 0, 0, + 0, 0, 0, 0, 1537, 0, 0, 47, 0, 1535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 714, -1774, 3277, 3278, -1774, -1774, -1774, - 0, 0, 0, 0, 39, 0, 0, 0, -2044, 520, - 0, 41, 0, 520, 862, 0, 0, 0, 0, 0, - 0, 0, 44, 520, 0, 520, 0, 520, 1872, 0, - 0, 520, 0, 520, 0, 520, 0, 41, 45, 0, - 0, 0, 2931, 0, 0, 0, 520, 0, 44, 0, - 0, 520, 0, 520, 0, 0, 0, 0, 0, 0, - 1536, 520, 46, 0, 45, 1908, 0, 0, 0, 0, - 0, 3336, 0, 3338, 714, 811, 47, 1537, 0, 520, - 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, - 0, 0, 27, 28, 29, 0, 0, 518, 0, 0, - 0, 0, 47, 0, 0, 0, 0, 0, 0, 3345, - 0, 0, 0, 0, 3371, -2044, 71, 951, 552, 552, - 0, 0, 1639, 1640, 1641, 1642, 1643, 1644, 520, 0, - 3374, 0, 0, 0, 3378, 1909, 0, 0, 520, 0, - 0, 1542, 0, 536, 0, 0, 0, 0, 0, 0, - 520, 0, 1590, 0, 0, 0, 0, 34, 974, 527, - 816, 0, 0, 0, 520, 516, 0, 850, -1308, 3364, - 0, 1414, 0, 0, 0, 0, 802, 0, 0, 0, - 1001, 1001, 520, 0, 0, 1001, 1024, 0, 1859, 0, - 1860, 0, 0, 0, 39, 0, 0, 0, 0, 828, - 828, 828, 0, 0, 0, 0, 537, 520, 0, 0, - 0, 0, 1875, 828, 828, 0, 828, 0, 0, 0, - 0, 0, 520, 0, 0, 0, 811, 41, 0, 850, - 0, 2938, 0, 714, 0, 552, 0, 0, 44, 0, - 0, 0, 0, 0, 0, 0, 0, 850, 518, 0, - 0, 0, 0, 1537, 45, 1534, 520, 3468, 0, 1876, - 961, 850, 813, 0, 0, 1209, 0, 0, 3345, 811, - 520, 520, 520, 1542, 0, 0, 0, 0, 46, 0, - 1877, 1536, 0, 3483, -1776, 1210, 0, 0, 0, 0, - 0, 518, 47, 850, 1288, 0, 0, 520, 1878, 71, - 71, 0, 1879, 0, 2931, 1299, 0, 0, 0, 850, - 850, 850, 850, 0, 0, 0, 0, 0, 811, 0, - 0, 0, 0, 0, 1880, 1320, 1413, 1881, 0, 1542, - 0, 1211, 0, 0, 0, 0, 540, 0, 520, 0, - 518, 1537, 0, 1882, 0, 1910, 1911, 1912, 0, 1913, - 1914, 1915, 1916, 1917, 1918, 0, 0, 0, 1001, 1024, - 0, 850, 0, 0, 1411, 0, 0, 0, 0, 0, - 1001, 1001, 0, 0, 1536, 71, 552, 71, 0, 0, - 0, 0, 802, 0, -1776, 0, 541, 1534, 962, 802, - 0, 0, 553, 0, 0, 0, 1414, 1414, 553, 552, - 2880, 0, 1414, 0, 0, 543, 1468, 0, 0, 0, - 553, 544, 0, 813, 545, 0, 1549, 0, 71, 0, - 0, 0, 0, 0, 553, 553, 1212, 0, 0, 0, - 0, -1776, 1542, 0, 71, 0, 1883, 0, 71, 0, - 0, 0, 0, 1534, 1884, -1776, 0, 0, 0, 0, - -1776, 0, 0, 0, 0, -1776, 813, 0, 0, 0, - 0, 0, 0, 0, -1776, 552, 1885, 0, 0, -1776, - 0, 0, 0, 0, 0, 1213, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1214, 0, 553, 0, 0, - 0, 0, 1412, 1886, 0, 1610, 0, 1215, 1611, 0, - 0, -1776, 1612, 1613, 0, 813, 0, 0, 1536, 0, - 0, 0, 0, 1713, 1669, 0, 1671, 0, 0, 0, - 0, -1776, 1413, 0, 0, 0, 0, 1542, 0, 1216, - 0, 552, 552, 1621, 0, 0, 0, 0, 850, 0, - -2044, 0, 0, 0, 0, 0, 1534, 0, 0, 0, + 0, 1538, 0, 0, 1624, 0, 0, 2721, 2722, 2723, + 0, 553, 0, 1347, 0, 0, 0, 553, 1670, 0, + 1672, 0, 0, 0, 1389, 1714, 0, 1540, 0, 553, + 0, 0, 0, 0, 1414, 552, 552, 0, 0, 0, + 0, 0, 850, 553, 553, 0, 0, 0, 1911, 1912, + 1913, 0, 1914, 1915, 1916, 1917, 1918, 1919, 0, 0, + 0, 0, 0, 0, 1412, 1900, 1901, 0, 0, 0, + 0, 1921, 0, 1470, 0, 850, 1746, 0, 0, 0, + 0, 71, 0, 0, 0, 0, 0, 0, 850, 0, + 0, 2034, 0, 0, 1538, 0, 0, 0, 2035, 2036, + 1625, 0, 2037, 2038, 2039, 850, 553, 71, 0, 850, + 0, 0, 0, 0, 1794, 0, 0, 1626, 0, 0, + 1540, 0, 1627, 0, 0, 0, 0, 1541, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1605, + 0, 0, 0, 0, 0, 0, 2831, 0, 0, 1608, + 0, 0, 0, 0, 0, 0, 1415, 0, 0, 0, + 1630, 0, 0, 0, 0, 0, 0, 0, 0, 1658, + 1810, 0, 850, 0, 0, 0, 0, 0, 1663, 0, + 0, 850, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1045, 1848, 0, 1902, 0, 0, 0, 1903, 1904, + 0, 951, 1905, 1906, 1907, 0, 951, 0, 552, 552, + 0, 552, 951, 0, 0, 1633, 0, 0, 0, 3064, + 0, 0, 0, 0, 0, 0, 0, 0, 1538, 0, + 0, 0, 0, 0, 0, 2218, 0, 2975, 0, 1469, + 0, 0, 0, 0, 0, 2227, 0, 2230, 0, 0, + 2241, 1535, 0, 0, 1540, 0, 2245, 0, 2247, 1414, + 1414, 0, 0, 0, 0, 1414, 0, 0, 0, 0, + 0, 2254, 0, 0, 0, 0, 2257, 0, 0, 0, + 2262, 2263, 2264, 2265, 0, 2269, 2270, 0, 0, 1412, + 1412, 0, 0, 0, 1537, 1412, 0, 516, 0, 3011, + 3012, 3013, 3014, 0, 1541, 0, 0, 0, 0, 0, + 1001, 0, 552, 1945, 0, 1635, 0, 0, 0, 0, + 850, 0, 802, 0, 802, 0, 0, 802, 0, 0, + 0, 0, 802, 1542, 0, 802, 0, 802, 0, 0, + 0, 802, 0, 552, 0, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1411, 71, 1537, 0, 0, 0, 0, 1623, 0, 0, - -1776, 850, 1745, -1776, 0, 0, 0, 0, 0, -1776, - 0, 1218, 0, 0, 850, 0, 0, 71, 0, 0, - -1791, 0, 0, 0, 0, 0, 0, 0, 0, 2217, - 0, 850, 0, 0, 1542, 850, 1219, 0, 0, 2226, - 1793, 2229, 0, 0, 2240, 0, 0, 0, 0, -1776, - 2244, 0, 2246, 0, 0, 1221, 0, 0, 0, 0, - 0, 0, 2788, 0, 0, 2253, 0, 0, 0, 2137, - 2256, 1534, -1776, 0, 2261, 2262, 2263, 2264, 0, 2268, - 2269, 0, 1536, 0, 0, 1537, 0, 0, 0, 0, - 0, 0, 0, -2044, 1536, 0, 1809, 0, 850, 0, - 0, 0, 0, 0, 0, 0, 0, 850, 0, 0, - -2044, 0, 0, 0, 1414, -2044, 0, 0, 1847, 0, - -1791, 0, 0, 0, 1536, 0, 0, 951, 0, 0, - 852, 1536, 951, 0, 552, 552, 0, 552, 951, 0, - 0, 1542, 0, 0, 854, 0, 0, 0, 1534, 0, - 0, 0, 0, -2044, 0, 0, 0, 0, -1776, 0, - 0, 0, 0, 0, 0, 1468, 0, -1791, -1776, 1536, - 0, 0, 0, 1536, 0, 1542, 0, 1413, 1413, 1536, - 0, -1791, 0, 1413, 0, 0, -1791, -1776, 0, -1776, - -1776, -1791, 0, 0, 0, 0, 0, 0, 0, 0, - -1791, 1536, 0, 0, 0, -1791, 0, 0, 1632, 892, - 0, 895, 0, 899, 0, 1411, 1411, 1008, 0, 1537, - 0, 1411, 1009, 516, 0, 0, -1776, 0, 0, -1776, - -1776, -1776, 0, 0, 0, 0, 1001, -1791, 552, 1944, - 0, 0, 0, 0, 1539, 0, 850, 0, 802, 0, - 802, 0, 0, 802, 1542, 1534, 0, -1791, 802, 0, - 0, 802, 0, 802, 0, 0, 0, 802, 0, 552, - 0, 552, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1010, 0, 0, 0, 0, 0, 0, 0, 1534, - 0, 0, 0, 0, 0, 0, 0, 1542, 0, 0, - 0, 0, 0, 0, 0, 0, -1791, 0, -2044, -1791, - 0, 0, 0, 0, 0, -1791, 0, 0, 0, 1414, - 1414, 1414, 1414, 1414, 1414, 0, 0, 1414, 1414, 1414, - 1414, 1414, 1414, 1414, 1414, 1414, 1414, 0, 0, 0, - 0, 0, 0, 1899, 1900, 0, 0, 0, 1540, 1920, - 0, 0, 1011, 0, 0, -1791, 0, 0, 0, 0, - 553, 0, 0, 1537, 0, 0, 0, 0, 1534, 0, - 0, 0, 0, 0, 0, 1537, 0, 0, -1791, 2048, - 0, 0, 0, 0, 0, 0, 0, 0, 1012, 0, - 2058, 0, 0, 0, 0, 1541, 0, 0, 0, 1542, - 0, 0, 0, 0, 1013, 1537, 0, 0, 0, 0, - 0, 1534, 1537, 0, 1014, -2044, 0, 0, 0, 0, - 1539, 0, 1639, 1640, 1641, 1642, 1643, 1644, 0, 951, - 0, 0, 0, 0, 0, 0, 852, 0, 0, 0, - 1859, 0, 0, 0, 0, 0, 0, 0, 1015, 0, - 1537, 0, 0, 0, 1537, 0, 0, 0, 0, 0, - 1537, 0, 0, 0, -1791, 2180, 0, 0, 0, 1964, - 0, 0, 0, 0, -1791, 0, 2145, 0, 850, 0, - 850, 0, 1537, 0, 0, 0, 0, 0, 0, 0, - 850, 2161, 0, -1791, 1016, -1791, -1791, 0, 0, 0, - 0, 1017, 536, 1411, 0, 0, 1414, 1414, 1539, 0, - 0, 1536, 0, 1534, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1540, 0, 0, -1308, 850, 0, - 552, 0, -1791, 0, 0, -1791, -1791, -1791, 0, 0, - 0, 1018, 0, 0, 0, 1745, 552, 0, 0, 0, - 0, 553, 553, 0, 0, 0, 552, 2227, 552, 2231, - 1019, 552, 0, 0, 0, 537, 0, 552, 0, 552, - 0, 1541, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 951, 552, 0, 0, 0, 951, 552, 0, 0, - 0, 552, 552, 552, 552, 1045, 552, 552, 1901, 0, - 0, 0, 1902, 1903, 0, 0, 1904, 1905, 1906, 0, - 0, 0, 1540, 2289, 0, 0, 0, 2670, 0, 538, - 1414, 1299, 0, 850, 850, 850, 850, 850, 0, 1020, - 0, 0, 0, 0, 1965, 0, 0, 920, 539, 0, - 0, 0, 927, 1536, 2318, 928, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 2339, 0, 1541, - 0, 1542, 0, 0, 0, 0, 0, 0, 553, 0, - 1413, 1413, 1413, 1413, 1413, 1413, 0, 0, 1413, 1413, - 1413, 1413, 1413, 1413, 1413, 1413, 1413, 1413, 0, 0, - 0, 0, 0, 0, 0, 540, 0, 0, 0, 1536, - 0, 1196, -207, 0, 0, 0, 0, 1197, 1411, 1411, - 1411, 1411, 1411, 1411, 0, 1209, 1411, 1411, 1411, 1411, - 1411, 1411, 1411, 1411, 1411, 1411, 0, 0, 0, 1539, - 0, 0, 0, 0, 0, 1210, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 541, 552, 542, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 850, 0, - 0, 0, 0, 0, 543, 0, 0, 0, 802, 0, - 544, 0, 0, 545, 802, 1534, 0, 0, 0, 0, - 552, 1211, 0, 0, 0, 552, 0, 0, 0, 0, - 0, 0, 1537, 0, 2450, 2450, 0, 0, 0, 0, - 0, 0, 1536, 0, 0, 0, 0, 0, 1908, 1431, - 0, 0, 1198, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1539, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1464, 1540, 0, 0, 2374, 2375, 2376, 2377, - 2378, 2379, 0, 0, 2383, 2384, 2385, 2386, 2387, 2388, - 2389, 2390, 2391, 2392, 0, 0, 0, 1413, 1413, 2836, - 2837, 0, 0, 552, 0, 0, 0, 2486, 0, 0, - 0, 0, 552, 0, 0, 0, 1212, 0, 1909, 0, - 1541, 0, 0, 0, 0, 0, 0, 2889, 0, 0, - 0, 0, 0, 986, 0, 1411, 1411, 1536, 1593, 0, - 0, 0, 2942, 2943, 2944, 2945, 2946, 2947, 2948, 2949, - 2950, 2951, 2048, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1537, 1213, 1540, 1414, 1414, 0, - 0, 0, 0, 0, 0, 1214, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 802, 0, 1215, 0, 0, - 0, 0, 0, 0, 0, 0, 1539, 552, 0, 0, - 0, 0, 802, 0, 553, 553, 2161, 0, 1199, 0, - 0, 1413, 0, 1541, 1536, 0, 0, 0, 0, 1216, - 1537, 0, 0, 1045, 0, 0, 1901, 1190, 0, 0, - 1902, 1903, 0, 0, 1904, 1905, 1906, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 552, 0, 0, 1411, - 0, 0, 552, 0, 0, 0, 0, 0, 0, 1287, - 0, 0, 0, 2495, 2496, 1217, 0, 0, 1809, 1201, - 0, 1218, 0, 0, 0, 1303, 1305, 1308, 1310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 11, 0, 0, 0, 0, 1219, 0, 0, 0, - 1540, 1220, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1221, 0, 0, 0, 14, - 15, 1536, 0, 1537, 0, 0, 0, 1406, 1910, 1911, - 1912, 0, 1913, 1914, 1915, 1916, 1917, 1918, 0, 1809, - 1539, 0, 0, 0, 0, 0, 850, 1541, 0, 0, - 0, 0, 1539, 0, 0, 1536, 0, 1299, 1414, 0, - 1809, 850, 850, 850, 23, 0, 0, 0, 0, 0, - 0, 0, 1203, 0, 552, 0, 850, 553, 553, 850, - 553, 0, 1539, 1414, 850, 0, 0, 0, 3117, 1539, - 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1809, 1809, - 0, 1809, 0, 1907, 0, 0, 0, 0, 1537, 0, - 0, 0, 0, 0, 0, 0, 0, 1539, 0, 0, - 0, 1539, 0, 0, 1536, 0, 1908, 1539, 0, 0, - 516, 0, 0, 0, 1540, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1540, 0, 0, 1539, - 2711, 0, 0, 0, 0, 0, 0, 0, 850, 850, - 850, 0, 0, 0, 0, 0, 0, 1536, 552, 0, - 1411, 553, 552, 0, 0, 0, 1540, 0, 552, 0, - 0, 1541, 0, 1540, 0, 1537, 0, 0, 0, 0, - 0, 0, 0, 1541, 0, 0, 1909, 27, 28, 29, - 0, 0, 1976, 0, 1980, 0, 0, 1414, 1413, 1413, - 0, 2048, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1540, 0, 1541, 0, 1540, 0, 1737, 0, 0, - 1541, 1540, 0, 0, 0, 0, 0, 0, 0, 0, - 1764, 0, 0, 0, 0, 0, 1411, 1411, 0, 0, - 0, 0, 0, 1540, 0, 0, 0, 0, 0, 0, - 0, 1789, 34, 0, 0, 0, 0, 0, 1541, 1536, - 0, 0, 1541, 36, 0, 2810, 0, 0, 1541, 0, - 0, 2161, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1537, 0, 0, 38, 0, 850, 0, 39, - 1541, 552, 0, 0, 0, 552, 552, 552, 0, 0, - 0, 1809, 1745, 1809, 0, 1847, 0, 0, 0, 40, - 0, 0, 0, 0, 1814, 0, 1537, 0, 0, 0, - 0, 0, 41, 1833, 552, 0, 2890, 0, 0, 0, - 0, 0, 0, 44, 0, 3262, 0, 1414, 0, 552, - 552, 552, 552, 552, 552, 552, 552, 552, 552, 45, - 0, 0, 0, 0, 0, 0, 3269, 3270, 0, 0, - 0, 0, 0, 1045, 2786, 2787, 1901, 0, 0, 0, - 1902, 1903, 2318, 46, 1904, 1905, 1906, 0, 850, 0, - 0, 0, 3284, 0, 0, 0, 0, 47, 0, 1413, - 1745, 3060, 0, 0, 0, 1537, 1910, 1911, 1912, 0, - 1913, 1914, 1915, 1916, 1917, 1918, 1847, 0, 0, 0, - 0, 1107, 1107, 0, 3029, 0, 1809, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1411, 0, 0, - 0, 0, 0, 552, 0, 0, 0, 0, 1537, 0, - 850, 850, 850, 850, 0, 0, 0, 0, 0, 1539, - 0, 0, 1411, 553, 0, 1411, 0, 0, 3053, 552, - 951, 0, 1956, 0, 1610, 0, 0, 1611, 3044, 553, - 0, 1612, 1613, 0, 0, 1616, 1617, 1618, 1236, 553, - 0, 553, 1278, 1285, 553, 0, 0, 0, 0, 0, - 553, 0, 553, 0, 0, 3051, 0, 0, 0, 0, - 0, 0, 1621, 0, 0, 553, 0, 0, 0, 1622, - 553, 0, 0, 0, 553, 553, 553, 553, 0, 553, - 553, 1536, 0, 0, 0, 3074, 0, 802, 0, 0, - 0, 0, 0, 0, 0, 1335, 1623, 0, 0, 0, - 1537, 3086, 0, 0, 0, 2161, 0, 1414, 1413, 0, - 0, 2048, 0, 1540, 1360, 2999, 0, 0, 0, 0, - 1405, 0, 0, 1407, 0, 1745, 1418, 1421, 1426, 1429, - 0, 1809, 0, 0, 0, 0, 1908, 0, 0, 0, - 0, 1539, 0, 0, 951, 552, 1411, 0, 0, 0, - 0, 0, 850, 0, 0, 0, 0, 0, 0, 0, - 1541, 0, 0, 1045, 0, 0, 1901, 1471, 1278, 3137, - 1902, 1903, 0, 3452, 1904, 1905, 1906, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1555, 0, - 0, 0, 1624, 0, 0, 0, 0, 1539, 0, 0, - 0, 0, 0, 0, 0, 0, 1909, 1571, 0, 1625, - 0, 0, 0, 0, 1626, 0, 0, 0, 1581, 1582, - 1583, 0, 1587, 1591, 0, 0, 0, 0, 0, 553, - 0, 0, 0, 0, 0, 0, 0, 1627, 1628, 0, - 0, 0, 0, 0, 0, 1540, 0, 3154, 1413, 0, - 0, 1809, 1629, 0, 2146, 1653, 2148, 0, 0, 0, - 0, 0, 0, 2437, 0, 2318, 2158, 0, 1980, 0, - 0, 0, 1471, 1471, 3118, 0, 0, 0, 0, 3188, - 0, 0, 0, 0, 0, 0, 1411, 0, 0, 0, - 1630, 0, 1541, 1631, 0, 0, 0, 0, 0, 0, - 1539, 1540, 0, 0, 2195, 0, 1691, 1632, 0, 0, - 1707, 1712, 3205, 0, 0, 0, 0, 0, 552, 0, - 0, 1107, 1107, 0, 0, 552, 1610, 0, 0, 1611, - 0, 0, 0, 1612, 1613, 1614, 1615, 1616, 1617, 1618, - 0, 0, 0, 0, 0, 0, 553, 0, 1541, 0, - 0, 0, 0, 2206, 1619, 1593, 0, 0, 0, 3233, - 0, 0, 1537, 0, 1621, 0, 0, 0, 0, 0, - 0, 1622, 1045, 0, 0, 1901, 1908, 0, 1278, 1902, - 1903, 552, 0, 1904, 1905, 1906, 552, 1278, 0, 0, - 0, 0, 0, 0, 0, 1539, 0, 0, 1623, 2301, - 2302, 2304, 2305, 2306, 1540, 0, 0, 1634, 0, 0, - 0, 1278, 552, 0, 3192, 0, 1910, 1911, 1912, 0, - 1913, 1914, 1915, 1916, 1917, 1918, 0, 0, 0, 0, - 0, 0, 0, 552, 552, 0, 0, 0, 0, 0, - 553, 0, 0, 0, 0, 0, 1909, 0, 0, 0, - 0, 1541, 850, 1045, 1745, 0, 1901, 0, 3323, 552, - 1902, 1903, 0, 0, 1904, 1905, 1906, 0, 0, 0, - 0, 0, 1539, 850, 0, 0, 3306, 0, 0, 0, - 0, 3061, 0, 0, 0, 0, 0, 0, 0, 553, - 0, 0, 0, 0, 1624, 2568, 1411, 0, 1891, 1540, - 552, 1001, 0, 1001, 0, 0, 0, 0, 0, 0, - 0, 1625, 0, 0, 1635, 0, 1626, 1636, 1637, 1638, - 0, 1639, 1640, 1641, 1642, 1643, 1644, 0, 3086, 0, - 0, 0, 0, 0, 2421, 850, 0, 0, 0, 1627, - 1628, 0, 0, 0, 0, 0, 1541, 0, 0, 0, - 0, 0, 0, 0, 1629, 0, 0, 0, 1426, 0, - 1426, 1426, 0, 0, 0, 0, 850, 0, 0, 0, - 0, 0, 0, 1107, 1107, 0, 1540, 0, 0, 1539, - 0, 0, 0, 0, 0, -2044, 0, 0, 0, 0, - 1107, 0, 1630, 0, 0, 1631, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 3205, 0, 0, 0, 1632, - 0, 0, 1633, 1539, 0, 0, 0, 553, 0, 0, - 0, 0, 0, 1541, 0, 0, 0, 3306, 0, 0, + 0, 0, 1538, 0, 2554, 0, 0, 0, 0, 0, + 0, 1415, 1415, 0, 1538, 0, 0, 1415, 0, 0, + 0, 0, 1541, 0, 0, 0, 0, 0, 1540, 1872, + 0, 0, 0, 1470, 1909, 0, 0, 0, 0, 0, + 1540, 0, 0, 0, 1538, 0, 0, 0, 0, 0, + 0, 1538, 1636, 0, 0, -2046, -2046, -2046, 1537, 1640, + 1641, 1642, 1643, 1644, 1645, 0, 0, 0, 0, 0, + 1540, 0, 0, 0, 0, 0, 0, 1540, 0, 0, + 0, 3127, 0, 2049, 0, 0, 0, 0, 0, 1538, + 0, 0, 0, 1538, 2059, 1045, 0, 0, 1902, 1538, + 0, 0, 1903, 1904, 1910, 0, 1905, 1906, 1907, 0, + 0, 0, 0, 0, 1537, 1540, 0, 0, 0, 1540, + 0, 1538, 0, 0, 0, 1540, 0, 0, 0, 0, + 0, 0, 0, 951, 0, 0, 0, 0, 0, 0, + 1542, 0, 0, 0, 0, 0, 0, 1540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1745, 1909, 1910, 1911, 1912, 0, - 1913, 1914, 1915, 1916, 1917, 1918, 1908, 951, 951, 0, - 3306, 951, 0, 0, 0, 0, 0, 0, 0, 1944, - 0, 0, 0, 1540, 0, 0, 0, 0, 0, 0, - 552, 0, 1539, 0, 0, 2023, 0, 0, 0, 0, - 0, 0, 0, 2026, 0, 0, 0, 0, 0, 1634, - 0, 0, 1745, 0, 0, 0, 0, 1540, 0, 3306, - 0, 553, 0, 0, 0, 553, 0, 0, 0, 0, - 1541, 1976, 0, 0, 0, 1539, 1909, 0, 0, 0, - 0, 0, 2072, 0, 0, 0, 0, 0, 2076, 2077, - 2078, 2079, 2080, 2081, 2082, 2083, 951, 0, 0, 0, - 2092, 2093, 0, 0, 1541, 2104, 0, 0, 0, 2107, - 0, 2161, 2115, 2116, 2117, 2118, 2119, 2120, 2121, 2122, - 2123, 0, 0, 2124, 0, 0, 0, 0, 0, 0, - 1107, 1610, 1278, 0, 1611, 0, 1540, 0, 1612, 1613, + 0, 0, 0, 0, 0, 0, 0, 2181, 0, 553, + 2146, 0, 850, 0, 850, 0, 0, 0, 2377, 2378, + 2379, 2380, 2381, 2382, 850, 2162, 2386, 2387, 2388, 2389, + 2390, 2391, 2392, 2393, 2394, 2395, 0, 1412, 0, 0, + 0, 0, 0, 0, 2040, 2041, 2042, 1537, 2043, 2044, + 2045, 2046, 2047, 2048, 0, 0, 0, 0, 1542, 0, + 0, 0, 850, 0, 552, 0, 0, 0, 0, 0, + 0, 0, 0, 1541, 0, 0, 0, 0, 0, 1746, + 552, 0, 0, 0, 2138, 0, 0, 0, 0, 0, + 552, 2228, 552, 2232, 0, 552, 0, 0, 0, 0, + 0, 552, 0, 552, 0, 0, 1860, 0, 0, 0, + 0, 1045, 0, 0, 1902, 951, 552, 0, 1903, 1904, + 951, 552, 1905, 1906, 1907, 552, 552, 552, 552, 1415, + 552, 552, 0, 0, 0, 0, 0, 0, 1909, 0, + 0, 0, 1537, 0, 1911, 1912, 1913, 2290, 1914, 1915, + 1916, 1917, 1918, 1919, 0, 1300, 0, 850, 850, 850, + 850, 850, 0, 0, 0, 0, 1541, 0, 2145, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 2319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 2150, 0, 0, 0, 0, 1635, 0, 0, 1636, - 1637, 1638, 0, 1639, 1640, 1641, 1642, 1643, 1644, 1621, - 0, 0, 0, 0, 0, 0, -2044, 1539, 0, 1540, - 0, 0, 2472, 1541, 1582, 1583, 0, 0, 0, 0, - 0, 0, 0, 0, 553, 0, 0, 0, 553, 553, - 553, 0, 0, 1623, 0, 1910, 1911, 1912, 0, 1913, - 1914, 1915, 1916, 1917, 1918, 0, 0, 0, 0, 0, - 0, 0, 2644, 0, 0, 0, 1541, 553, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 2659, 2660, 2662, - 0, 0, 553, 553, 553, 553, 553, 553, 553, 553, - 553, 553, 2673, 0, 0, 2676, 0, 0, 0, 0, - 2681, 0, 0, 0, 0, 0, 0, 0, 2789, 0, - 0, 0, 2272, 0, 0, 0, 1278, 0, 0, 2282, - 2283, 1540, 0, 0, 0, 0, 1910, 1911, 1912, 0, - 1913, 1914, 1915, 1916, 1917, 1918, 0, 0, 0, -2044, + 0, 2340, 0, 0, 0, 0, 0, 0, 0, 0, + 553, 553, 0, 0, 0, 2498, 2499, 0, 1910, 0, + 0, 3282, 1414, 1414, 1414, 1414, 1414, 1414, 0, 1537, + 1414, 1414, 1414, 1414, 1414, 1414, 1414, 1414, 1414, 1414, + 0, 0, 3307, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1412, 1412, 1412, 1412, 1412, 1412, 0, 0, + 1412, 1412, 1412, 1412, 1412, 1412, 1412, 1412, 1412, 1412, + 0, 1538, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1542, + 552, 0, 0, 2673, 0, 0, 0, 1540, 0, 0, + 0, 0, 850, 0, 3346, 0, 2271, 0, 0, 0, + 1541, 0, 802, 0, 0, 0, 0, 0, 802, 2286, + 2286, 0, 0, 0, 552, 0, 0, 553, 0, 552, + 0, 0, 0, 0, -2046, 3372, 1537, 0, 2453, 2453, + 0, 717, 0, 0, 1415, 1415, 1415, 1415, 1415, 1415, + 0, 0, 1415, 1415, 1415, 1415, 1415, 1415, 1415, 1415, + 1415, 1415, 0, 0, 0, 0, 0, 0, 0, 0, + 1537, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1542, 0, 0, 0, 1347, 0, 0, 0, + 0, 0, 2362, 0, 0, 0, 0, 0, 718, 0, + 0, 0, 0, 0, 1910, 1538, 0, 552, 0, 1414, + 1414, 2489, 0, 0, 719, 0, 552, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1911, 1912, + 1913, 1540, 1914, 1915, 1916, 1917, 1918, 1919, 0, 1412, + 1412, 0, 0, 0, 0, 0, 0, 0, 0, 1537, + 0, 0, 0, 0, 1541, 0, 2049, 0, 0, 0, + 0, 1538, 0, 720, 0, 0, 1541, 0, 1432, 0, + 0, 0, 0, 721, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 722, 0, 1540, 0, 802, + 723, 1465, 1537, 0, 0, 0, 1541, 0, 0, 0, + 0, 552, 0, 1541, 0, 0, 802, 0, 0, 0, + 2162, 0, 0, 0, 0, 0, 1542, 0, 0, 724, + 0, 0, 0, 1414, 0, 0, 0, 0, 0, 0, + 0, 1415, 1415, 0, 0, 0, 0, 2839, 2840, 0, + 0, 1541, 0, 0, 0, 1541, 0, 1008, 0, 0, + 552, 1541, 1009, 1412, 0, 0, 552, 1594, 0, 0, + 0, 0, 725, 0, 1538, 2892, 726, 0, 0, 0, + 0, 0, 1810, 1541, 0, 0, 0, 0, 0, 0, + 2945, 2946, 2947, 2948, 2949, 2950, 2951, 2952, 2953, 2954, + 1540, 0, 0, 0, 1537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, -2044, 0, 0, 0, - 1278, -2044, 0, 0, 0, 0, 0, 0, 1541, 0, - 0, 0, 0, 0, 0, 0, 1976, 0, 0, 1335, - 2347, 0, 0, 0, 2718, 2719, 2720, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, -2044, - 0, 0, 2437, 0, 0, 0, 0, 0, 0, 2369, - 2370, 0, 2371, 0, 0, 0, 0, 0, 0, 0, + 0, 1010, 0, 0, 1911, 1912, 1913, 0, 1914, 1915, + 1916, 1917, 1918, 1919, 553, 553, 0, 0, 0, 0, + 539, 0, 0, 0, 0, 0, 727, 0, 2789, 2790, + 0, 0, 0, 1810, 0, 1415, 0, 0, 0, 0, + 850, 728, 14, 15, 0, 0, 0, 0, 0, 1538, + 1542, 1300, 0, 0, 1810, 850, 850, 850, 0, 0, + 0, 0, 1542, 0, 0, 0, 0, 0, 552, 0, + 850, 0, 1011, 850, 729, 1540, 0, 730, 850, 0, + 1184, 0, 0, 0, 951, 0, 0, 23, 731, 0, + 0, 732, 1542, 0, 0, 0, 0, 0, 0, 1542, + 0, 0, 0, 0, 1810, 1810, 0, 1810, 1012, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1610, 0, 0, 1611, 0, -45, 0, 1612, 1613, 0, - 0, 2397, 2398, 0, 0, 2150, 0, 0, 0, 0, - 0, 0, 0, 0, 1632, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 2, 1621, 3, - 4, 0, 0, 0, 2425, -2044, 0, 0, 0, 1539, - 2431, 0, 5, 0, 0, 0, 0, 6, 0, 0, - 0, 0, 0, 0, 0, 0, 7, 0, 1471, 0, - 1278, 0, 1623, 0, 0, 0, 0, 0, 0, 0, - 8, 0, 0, 0, 0, 0, 0, 0, 0, 9, - 0, 10, 0, 2828, 0, 0, 0, 0, 553, 0, - 0, 0, 0, 11, 0, 12, 2465, 0, 0, 0, - 0, 0, 1196, 0, 0, 0, 13, 0, 1197, 0, - 0, 0, 0, 0, -2044, 1008, 1209, 0, 0, 0, - 1009, 14, 15, 16, 0, 0, 0, 3064, 0, 0, - 0, 0, 17, 0, 0, 2484, 1210, 0, 18, 0, - 2487, 2488, 0, 1540, 0, 0, 19, 0, 20, 21, - 0, 0, 0, 0, 0, 0, 0, 0, -2044, 0, - 0, 0, 22, 0, 0, 0, 23, 0, 0, 0, - 0, 0, 0, 0, 2972, -2044, 0, 0, 0, 1010, - -2044, 0, 1211, 0, 2509, 0, 0, 2512, 0, 2514, - 1541, 0, 24, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 2518, 0, 0, 25, 0, - 0, 0, 0, 1198, 0, 0, 0, 0, -2044, 0, - 0, -2044, 0, 0, 0, 0, 0, 0, 1639, 1640, - 1641, 1642, 1643, 1644, 26, 0, 3008, 3009, 3010, 3011, - 0, 0, 0, 0, 0, 1691, 0, 0, 0, 0, - 1011, 0, 0, 0, 0, 0, 0, 0, 1183, 0, - 0, 3213, 1712, 2122, 0, 0, 0, 0, 2437, 0, - 0, 0, 0, 1632, 0, 0, 0, 1212, 0, 0, - 0, 1107, 0, 0, 0, 1610, 1012, 0, 1611, 0, - 2572, 0, 1612, 1613, 1614, 1615, 1616, 1617, 1618, 0, - 0, 0, 1013, 0, 0, 0, 0, 0, 0, 27, - 28, 29, 1014, 1619, 0, 0, 0, 30, 1415, 0, - 31, 0, 0, 1621, 553, 0, 1213, 0, 0, 553, - 1622, 0, 0, 0, 0, 0, 1214, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1015, 1583, 1215, 32, - 0, 0, 0, 1278, 0, 553, 0, 1623, 33, 0, - 0, 0, 0, 0, 0, 0, 0, 1184, 0, 1199, - 0, 0, 0, -2044, 34, 0, 553, 553, 0, 0, - 1216, 35, 0, 0, 0, 36, 0, 0, 3124, 0, - 0, 0, 1016, 0, 0, 37, 0, 0, 0, 1017, - 0, 0, 553, 1008, 0, 0, 0, 38, 1009, 0, - 0, 39, 2679, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 2247, 0, 0, 0, - 1201, 40, 1218, 0, 0, 0, 0, 1647, 0, 1018, - 0, 0, 0, 553, 41, 0, 0, 42, 0, 0, - 43, 0, 0, 1624, 0, 44, 0, 1219, 1019, 0, - 0, 0, 2248, 0, 0, 0, 0, 1010, 0, 0, - 1625, 45, 0, 0, 0, 1626, 1221, 0, 0, 0, - -2044, 1647, 0, 0, 0, 0, 0, 1639, 1640, 1641, - 1642, 1643, 1644, 0, 0, 46, 0, 0, 1627, 1628, - 0, 0, 0, 1609, 0, 0, 0, 0, 1610, 47, - 0, 1611, -45, 1629, 1415, 1612, 1613, 1614, 1615, 1616, - 1617, 1618, 0, 0, 0, 0, 0, 1020, 0, 0, - 0, 0, 1185, 1203, 0, 0, 1619, 0, 1011, 0, - 1620, 0, 2757, 0, 0, 0, 1621, 0, 2759, 2026, - 0, 1630, 0, 1622, 1631, 0, 0, 0, 0, 2765, - 0, 0, 0, 0, 0, 0, 1647, 0, 1632, 0, - 2774, 1633, 0, 2777, 1012, 2779, 0, 0, 0, 0, - 1623, 0, 0, 2783, 0, 0, 0, 0, 0, 0, - 1013, 2790, 2791, 0, 0, 0, 0, 0, 2798, 0, - 1014, 0, 0, 0, 0, 0, 0, 0, 1647, 0, - 0, 0, 0, 2807, 0, 1647, 0, 0, 0, 0, - 0, 0, 0, 2822, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 553, 1015, 0, 0, 0, 0, 0, - 0, 0, 0, 1107, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1186, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1647, 0, 0, 1634, 0, - 0, 0, 0, 0, 0, 0, 1624, 0, 0, 0, - 1016, 0, 0, 0, 0, 0, 0, 1017, 3279, 0, - 1647, 0, 0, 1625, 0, 0, 0, 0, 1626, 0, - 2272, 0, 2272, 0, 0, 0, 0, 0, 0, 3304, + 0, 0, 0, 734, 1013, 0, 1538, 0, 0, 735, + 736, 0, 0, 0, 1014, 0, 516, 1542, 0, 0, + 737, 1542, 0, 2610, 0, 0, 738, 1542, 0, 0, + 0, 0, 1540, 0, 0, 0, 2714, 0, 0, 0, + 0, 0, 0, 0, 850, 850, 850, 0, 1015, 1542, + 0, 0, 0, 739, 552, 0, 1412, 0, 552, 0, + 0, 0, 0, 0, 552, 0, 0, 553, 553, 1185, + 553, 0, 0, 0, 0, 2642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1627, 1628, 0, 0, 0, 0, 1018, 0, 0, - 0, 0, 0, 0, 0, 1647, 1629, 1647, 0, 1415, - 1415, 0, 1919, 0, 0, 1415, 1019, 0, 1647, 0, - 0, 1647, 0, 0, 0, 0, 1647, 0, 0, 1647, + 0, 0, 0, 2656, 1016, 0, 3120, 2049, 0, 3002, + 0, 1017, 1414, 1414, 0, 0, 0, 0, 0, 0, + 27, 28, 29, 1538, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1537, 0, 0, 0, + 0, 0, 1412, 1412, 0, 1347, 0, 0, 0, 1540, + 0, 1018, 0, 0, 2691, 0, 2692, 1538, 0, 0, + 2697, 0, 2700, 1541, 0, 0, 0, 0, 0, 0, + 1019, 2813, 0, 0, 0, 0, 0, 2162, 0, 0, + 0, 553, 0, 1540, 0, 34, 0, 0, 0, 0, + 0, 0, 0, 850, 0, 0, 36, 552, 0, 0, + 0, 552, 552, 552, 0, 0, 0, 1810, 1746, 1810, + 0, 1848, 1977, 0, 1981, 0, 0, 0, 38, 0, + 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, + 552, -1778, 2893, 0, 1415, 1415, 1538, 0, 0, 1020, + 0, 0, 0, 0, 1186, 552, 552, 552, 552, 552, + 552, 552, 552, 552, 552, 41, 0, 0, 0, 0, + 0, 0, 1540, 0, 0, 0, 44, 0, 3121, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 2319, 1538, + 0, 0, 45, 0, 850, 1611, 0, 0, 1612, 0, + 0, 0, 1613, 1614, 0, 0, 1746, 1541, 0, 0, + 0, 0, 0, 1414, 0, 1540, 46, 0, 0, 0, + 0, 0, 1848, 0, 0, 0, 0, 0, 0, 0, + 47, 0, 1810, 1622, 0, 0, 0, 0, 3032, 0, + -2046, -1778, 0, 1412, 0, 0, 0, 0, 0, 552, + 0, 0, 0, 0, 0, 0, 850, 850, 850, 850, + 0, 0, 0, 1541, 0, 0, 0, 1624, 1412, 0, + 0, 1412, 0, 0, 0, 552, 951, 0, 0, 1542, + 0, 0, 3056, 3265, 3047, 0, 0, 0, -1778, 0, + 0, 1538, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, -1778, 0, 3272, 3273, 0, -1778, 0, 0, + 0, 3054, -1778, 0, 0, 2934, 0, 1540, 3195, 0, + 0, -1778, 0, 0, 0, 0, -1778, 0, 0, 0, + 3287, 0, 2788, 0, 0, 1415, 0, 0, 0, 0, + 0, 3077, 0, 802, 0, 0, 0, 0, 2642, 0, + 0, 0, 0, 0, 0, 0, 0, 3089, -1778, 0, + 1415, 2162, 0, -2046, 0, 0, 1541, 2049, 0, 0, + 0, 0, 1414, 0, 0, 0, 1197, 0, -1778, 0, + -2046, 1746, 1198, 553, 0, -2046, 0, 1810, 0, 0, + 1210, 0, 0, 0, 0, 0, 0, 0, 0, 553, + 951, 552, 1412, 0, 0, 0, 0, 0, 850, 553, + 1211, 553, 0, 1542, 553, 0, 0, 0, 0, 0, + 553, 0, 553, -2046, 0, 3140, 0, -1778, 0, 0, + -1778, 0, 0, 0, 0, 553, -1778, 0, 0, 0, + 553, 0, 0, 0, 553, 553, 553, 553, 0, 553, + 553, 0, 0, 0, 0, 0, 1212, 0, 0, 0, + 0, 1541, 0, 0, 0, 0, 0, 0, 0, 1542, + 0, 0, 0, 0, 0, 0, -1778, 0, 1633, 0, + 0, 0, 0, 0, 0, 0, 0, 1199, 0, 0, + 0, 0, 0, 0, 1415, 0, 0, 0, 0, -1778, + 0, 0, 0, 3157, 0, 0, 0, 1810, 0, 0, + 0, 3082, 1414, 0, 0, 0, 0, 1107, 1107, 0, + 0, 2319, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 3191, 0, 0, 1541, 0, + 0, 0, 1412, 0, 0, 0, 0, 0, 0, 0, + 0, 1213, 0, 1538, 0, 0, 0, 852, 11, 0, + 0, 3455, 0, 0, 0, 0, 0, 0, 3208, 0, + 0, 0, 1542, 0, 552, 0, 0, 0, -2046, 1540, + 0, 552, 0, 0, 1237, -1778, 14, 15, 1279, 1286, + 0, 0, 0, 0, 0, -1778, 0, 0, 0, 553, + 1214, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1215, 0, 0, 0, -1778, 3236, -1778, -1778, 0, 0, + 0, 0, 1216, 0, 0, 0, -1793, 0, 0, 0, + 0, 23, 0, 2440, 1415, 0, 0, 552, 1981, 0, + 0, 1336, 552, 1200, 0, 1541, 0, 0, 0, 0, + 0, 0, 0, -1778, 1217, 0, -1778, -1778, -1778, 0, + 1361, 0, 0, 0, 0, 0, 1406, 1542, 552, 1408, + 0, 0, 1419, 1422, 1427, 1430, 0, 0, 0, 1541, + 3172, 0, 0, 0, 0, -2046, 0, 0, 0, 552, + 552, 0, 1640, 1641, 1642, 1643, 1644, 1645, 0, 0, + 1218, 0, 0, 1347, 1202, 0, 1219, 0, 850, 0, + 1746, 0, 0, 1472, 1279, 552, 553, 0, 0, 0, + 0, 0, 3326, 0, 0, 1594, -1793, 0, 0, 850, + 0, 1220, 3309, 0, 1556, 0, 1221, 0, 0, 0, + 0, 0, 0, 0, 1542, 0, 0, 0, 0, 0, + 1222, 0, 1412, 1572, 1365, 1366, 552, 1001, 1541, 1001, + 0, 0, 0, 0, 1582, 1583, 1584, 0, 1588, 1592, + 0, 0, 0, -1793, 27, 28, 29, 0, 0, 0, + 1364, 0, 815, 3239, 3089, 0, 0, -1793, 0, 0, + 0, 850, -1793, 0, 0, 1367, 1368, -1793, 0, 1369, + 1370, 1541, 1654, 0, 0, 0, -1793, 1204, 0, 0, + 553, -1793, 0, 0, 0, 0, 0, 0, 0, 1472, + 1472, 0, 850, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, + 0, 0, 0, -1793, 1415, 1365, 1366, 0, 0, 0, + 36, 1542, 2934, 1692, 0, 0, 0, 1708, 1713, 553, + 0, 3208, 0, -1793, 0, 2571, 0, 0, 1107, 1107, + 0, 0, 38, 0, 1371, 1372, 39, 0, 0, 0, + 0, 0, 0, 3309, 0, 1542, 1367, 1368, 0, 0, + 1369, 1370, 0, 0, 0, 0, 40, 0, 0, 0, + 0, 0, 0, 1541, 0, 0, 0, 0, 0, 41, + 1746, 0, -1793, 0, 0, -1793, 0, 0, 0, 0, + 44, -1793, 0, 951, 951, 1279, 3309, 951, 0, 0, + 0, 0, 0, 0, 1279, 1945, 45, 0, 1373, 1374, + 1375, 1376, 1377, 1378, 1379, 1380, 552, 0, 1381, 1382, + 0, 0, 0, 0, 0, 0, 0, 0, 1279, 0, + 46, -1793, 0, 0, 1542, 1371, 1372, 0, 1746, 0, + 0, 0, 0, 0, 47, 3309, 0, 0, 0, 3367, + 0, 0, 0, 0, -1793, 0, 0, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 3343, 0, 0, 1630, 1635, 0, 1631, 1636, 1637, - 1638, 0, 1639, 1640, 1641, 1642, 1643, 1644, 0, 0, - 0, 1632, 0, 0, 1633, 1894, 0, 3024, 3025, 0, - 0, 1647, 3369, 0, 0, 0, 0, 0, 0, 717, - 0, 0, 0, 0, 0, 1020, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1542, 0, 0, + 0, 0, 951, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 2162, 0, 1373, + 1374, 1375, 1376, 1377, 1378, 1379, 1380, 0, 0, 1381, + 1382, 0, 852, 0, 0, 1892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 3048, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 3052, 0, 0, 0, - 0, 3054, 3055, 0, 0, 0, 3056, 0, 0, 0, - 0, 3059, 0, 0, 3062, 3063, 718, 0, 0, 2272, - 1278, 0, 0, 3071, 0, 0, 0, 0, 0, 0, - 0, 0, 719, 0, 1107, 1647, 0, 0, 0, 0, - 0, 1634, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1647, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1647, 1647, 1647, 0, 0, 0, 1647, 0, 0, - 0, 1647, 0, 0, 0, 1363, 0, 815, 3112, 0, - 0, 720, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 721, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 722, 0, 3131, 0, 0, 723, 0, + 0, 1383, 1384, 0, 0, 0, 0, 0, 0, 0, + -1793, 0, 0, 0, 0, 0, 0, 0, 0, 0, + -1793, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 553, 0, 0, 0, 553, 0, -1793, + 0, -1793, -1793, 1977, 0, 1427, 0, 1427, 1427, 1542, + 0, 0, 0, 0, 2934, 0, 0, 0, 0, 0, + 1107, 1107, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1107, -1793, 0, + 0, -1793, -1793, -1793, 0, 0, 0, 0, 0, 0, + 0, 0, 1383, 1384, 0, 0, 1385, 1386, 0, 0, + 0, 1611, 0, 0, 1612, 1541, 0, 0, 1613, 1614, + 1615, 1616, 1617, 1618, 1619, 0, 0, 0, 0, 0, + 1387, 1388, 0, 0, 0, 0, 0, 0, 0, 1620, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1622, + 1994, 0, 0, 0, 0, 0, 1623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 553, 0, 0, 0, + 553, 553, 553, 1624, 0, 0, 0, 0, 0, 0, + 0, 0, 2024, 0, 0, 0, 0, 1385, 1386, 0, + 2027, 0, 0, 0, 0, 0, 0, 0, 0, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1647, 0, 0, 0, 724, 0, 0, - 1364, 1365, 0, 0, 0, 0, 0, 0, 1635, 0, - 0, 1636, 1637, 1638, 0, 1639, 1640, 1641, 1642, 1643, - 1644, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1647, 0, 0, 0, 0, 0, 0, 0, 0, - 725, 1366, 1367, 0, 726, 1368, 1369, 1647, 0, 0, - 0, 1610, 1647, 0, 1611, 3165, 0, 0, 1612, 1613, - 1614, 1615, 1616, 1617, 1618, 0, 0, 0, 0, 1919, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1619, - 0, 0, 0, 2769, 0, 0, 0, 0, 0, 1621, - 0, 0, 0, 0, 0, 0, 1622, 0, 1610, 0, - 0, 1611, 1591, 0, 0, 1612, 1613, 1614, 1615, 1616, - 1617, 1618, 0, 0, 727, 0, 0, 0, 0, 0, - 1370, 1371, 0, 1623, 0, 0, 1619, 0, 0, 728, - 0, 0, 0, 0, 0, 2484, 1621, 0, 0, 0, - 0, 0, 0, 1622, 0, 0, 3225, 3226, 0, 0, - 3227, 0, 1583, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 729, 0, 0, 730, 0, 0, 0, 0, - 1623, 0, 0, 0, 0, 0, 731, 3245, 0, 732, - 0, 0, 0, 0, 1372, 1373, 1374, 1375, 1376, 1377, - 1378, 1379, 0, 0, 1380, 1381, 0, 733, 0, 0, - 0, 3257, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 734, 0, 0, 0, 0, 0, 0, 736, 1624, - 0, 0, 0, 0, 0, 0, 0, 0, 737, 0, - 0, 1647, 0, 0, 738, 0, 1625, 0, 0, 1919, - 1919, 1626, 1415, 1415, 1415, 1415, 1415, 1415, 0, 0, - 1415, 1415, 1415, 1415, 1415, 1415, 1415, 1415, 1415, 1415, - 1919, 739, 0, 0, 1627, 1628, 1624, 0, 0, 0, - 0, 0, 3303, 0, 0, 0, 0, 0, 0, 1629, - 0, 0, 0, 1625, 0, 0, 0, 0, 1626, 0, - 3024, 0, 0, 0, 3320, 0, 0, 0, 1107, 0, - 0, 0, 0, 0, 0, 0, 0, 1382, 1383, 3329, - 0, 1627, 1628, 0, 2272, 0, 2272, 1630, 0, 0, - 1631, 0, 0, 0, 1107, 0, 1629, 0, 0, 0, - 0, 0, 0, 0, 1632, 0, 0, 1633, 0, 0, - 0, 0, 3354, 0, 0, 0, 0, 0, 0, 0, + 0, 1387, 1388, 0, 553, 553, 553, 553, 553, 553, + 553, 553, 553, 553, 0, 0, 0, 0, 0, 2073, + 0, 0, 0, 0, 0, 2077, 2078, 2079, 2080, 2081, + 2082, 2083, 2084, 0, 0, 0, 0, 2093, 2094, 0, + 0, 0, 2105, 0, 0, 0, 2108, 0, 0, 2116, + 2117, 2118, 2119, 2120, 2121, 2122, 2123, 2124, 0, 1625, + 2125, 0, 0, 0, 0, 0, 0, 1107, 0, 1279, + 0, 0, 0, 0, 0, 0, 1626, 0, 0, 0, + 0, 1627, 0, 0, 0, 0, 0, 0, 2151, 0, + 0, 0, 0, 0, 0, -45, 0, 0, 1977, 0, + 0, 1542, 0, 0, 1628, 1629, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1, 0, 1630, + 0, 1583, 1584, 0, 2440, 0, 0, 2, 0, 3, + 4, 1611, 0, 0, 1612, 0, 0, 0, 1613, 1614, + 0, 0, 5, 0, 0, 0, 0, 6, 0, 0, + 0, 0, 0, 0, 0, 0, 7, 1631, 0, 0, + 1632, 0, 0, 0, 0, 0, 0, 0, 0, 1622, + 8, 0, 0, 0, 1633, 0, -2046, 1634, 0, 9, + 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 11, 0, 12, 0, 0, 0, 0, + 0, 0, 0, 1624, 0, 0, 13, 0, 0, 2273, + 0, 0, 0, 1279, 0, 0, 2283, 2284, 0, 0, + 0, 14, 15, 16, 0, 0, 0, 0, 0, 0, + 0, 0, 17, 0, 0, 0, 0, 0, 18, 0, + 0, 0, 0, 0, 0, 0, 19, 0, 20, 21, + 553, 0, 0, 0, 0, 0, 0, 1279, 0, 0, + 0, 0, 22, 0, 0, 0, 23, 0, 2791, 0, + 0, 0, 0, 0, 1635, 0, 1336, 2348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1647, 1630, 0, 1647, 1631, 0, 0, + 0, 0, 24, 0, 0, 0, 0, 0, 0, -2046, + 0, 0, 0, 0, 0, 0, 2372, 2373, 25, 2374, + 0, 0, 0, 0, 0, 1611, -2046, 0, 1612, 0, + 0, -2046, 1613, 1614, 1615, 1616, 1617, 1618, 1619, 0, + 0, 0, 0, 0, 26, 0, 0, 0, 2400, 2401, + 0, 0, 2151, 1620, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1622, 0, 0, 0, 0, 0, -2046, + 1623, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 2428, 0, 0, 0, 0, 0, 2434, 0, 0, + 0, 1636, 0, 0, 1637, 1638, 1639, 1624, 1640, 1641, + 1642, 1643, 1644, 1645, 0, 1472, 0, 1279, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 2475, 0, 27, + 28, 29, 0, 0, 1633, 0, 0, 30, 0, 0, + 31, 0, 0, 3216, 0, 0, 1416, 0, 0, 0, + 2440, 0, 0, 2468, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, + 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1632, 0, 0, 1633, 0, 0, 0, 0, 0, - 0, 3380, 0, 0, 0, 0, 0, 1610, 0, 0, - 1611, 3024, 1384, 1385, 1612, 1613, 1614, 1615, 1616, 1617, - 1618, 0, 1647, 1107, 0, 0, 1647, 1647, 1647, 1647, - 1647, 1647, 1647, 1647, 0, 1619, 1386, 1387, 0, 1415, - 1415, 0, 1647, 1647, 1634, 1621, 0, 0, 0, 0, - 0, 0, 1622, 0, 1647, 3423, 0, 1647, 0, 0, - 0, 0, 0, 0, 0, 1647, 1647, 1647, 1647, 1647, - 1647, 1647, 1647, 1647, 1647, 0, 0, 0, 0, 1623, + 0, 0, 2487, 0, 34, 717, 0, 2490, 2491, 0, + 0, 35, 0, 1625, 0, 36, 553, 0, 0, 0, + 0, 553, 0, 0, 0, 37, 0, 0, 0, 0, + 1626, 0, 0, 0, 0, 1627, 0, 38, 0, 0, + 0, 39, 0, 0, -2046, 0, 0, 553, 0, 0, + 0, 2512, 0, 0, 2515, 0, 2517, 0, 1628, 1629, + 0, 40, 718, 0, 0, 0, 0, 0, 553, 553, + 0, 0, 2521, 1630, 41, 1648, 0, 42, 719, 0, + 43, 0, 0, 0, 0, 44, 0, 0, 0, 0, + 0, 0, 0, 0, 553, 0, 0, 0, 0, 0, + 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1631, 1692, 0, 1632, 0, 0, 0, 0, 0, + 1648, 0, 0, 0, 0, 46, 0, 720, 1633, 1713, + 2123, 1634, 0, 0, 0, 553, 0, 721, 0, 47, + 0, 0, -45, 0, 0, 0, 0, 0, 1107, 722, + 0, -2046, 0, 1416, 723, 0, 0, 2575, 1640, 1641, + 1642, 1643, 1644, 1645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1634, 0, 0, 1610, 0, 0, 1611, 0, 0, - 1647, 1612, 1613, 1614, 1615, 1616, 1617, 1618, 0, 0, + 0, 0, 0, 724, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1611, 0, 0, 1612, 0, + 0, 0, 1613, 1614, 0, 1648, 1617, 1618, 1619, 0, + 0, 0, 0, 0, 1584, 0, 0, 0, 0, 0, + 1279, 0, 0, 0, 0, 0, 725, 0, 1635, 0, + 726, 0, 0, 1622, 0, 0, 0, 0, 0, 0, + 1623, 0, 0, 0, 0, 0, 0, 1648, 0, 0, + 0, 0, 0, 0, 1648, 0, 1610, 0, 0, 0, + 0, 1611, 0, 0, 1612, 0, 0, 1624, 1613, 1614, + 1615, 1616, 1617, 1618, 1619, 0, 0, 0, 0, 2682, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1620, + 0, 0, 0, 1621, 0, 0, 0, 0, 0, 1622, + 727, 0, 0, 0, 1648, 0, 1623, 0, 0, 0, + 0, 0, 0, 0, 0, 728, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 553, 0, 0, 0, 1648, + 0, 0, 0, 1624, 0, 1636, 0, 0, 1637, 1638, + 1639, 0, 1640, 1641, 1642, 1643, 1644, 1645, 729, 0, + 0, 730, 0, 0, 0, 1895, 0, 0, 0, 0, + 0, 0, 731, 1625, 0, 732, 0, 0, 0, 0, + 0, 0, 0, 0, 1648, 0, 1648, 0, 1416, 1416, + 1626, 1920, 0, 733, 1416, 1627, 0, 1648, 0, 0, + 1648, 0, 0, 0, 0, 1648, 0, 734, 1648, 0, + 0, 0, 0, 0, 736, 0, 0, 0, -2046, -2046, + 0, 2760, 0, 0, 737, 0, 0, 2762, 2027, 0, + 738, 0, 0, 1630, 0, 0, 0, 0, 2768, 1625, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 2777, + 1648, 0, 2780, 0, 2782, 0, 1626, 739, 0, 0, + 0, 1627, 2786, 0, 0, 0, 0, 0, 0, 0, + 2793, 2794, 0, 0, -2046, 0, 0, 2801, 0, 0, + 0, 0, 0, 0, 1628, 1629, 0, 0, 1633, 0, + 0, 0, 2810, 0, 0, 0, 0, 0, 0, 1630, + 0, 0, 2825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1619, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1621, 0, 0, 0, 0, 0, 0, 1622, - 0, 0, 0, 3477, 3477, 3477, 0, 0, 0, 0, - 0, 1635, 0, 1415, 1636, 1637, 1638, 0, 1639, 1640, - 1641, 1642, 1643, 1644, 0, 0, 1623, 0, 2063, 0, - 3477, 0, 0, 0, 0, 1624, 0, 0, 0, 0, + 0, 0, 1107, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1648, 0, 0, 1631, 0, 0, + 1632, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1648, 0, 0, 0, 1633, 0, 0, 1634, 0, 0, + 1648, 1648, 1648, 0, 0, 0, 1648, 0, 0, 0, + 1648, 0, 0, 0, 0, 0, 0, 0, 0, 2273, + 0, 2273, 0, 0, 0, 0, 0, 0, 1635, 0, + 1611, 0, 0, 1612, 0, 0, 0, 1613, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1625, 0, 0, 0, 0, 1626, 1635, 0, - 0, 1636, 1637, 1638, 0, 1639, 1640, 1641, 1642, 1643, - 1644, 3477, 0, 0, 0, 2063, 0, 0, 0, 0, - 1627, 1628, 1647, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1647, 1647, 0, 1629, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1611, 0, + 0, 1612, 0, 0, 0, 1613, 1614, 0, 1622, 0, + 0, 0, 1648, 0, 0, -2046, 0, 1611, 0, 0, + 1612, 0, 0, 0, 1613, 1614, 1615, 1616, 1617, 1618, + 1619, 0, 0, 0, 1635, 0, 1622, 0, 0, 0, + 0, 0, 1624, -2046, 0, 1620, 0, 0, 0, 2772, + 1648, 0, 0, 0, 0, 1622, 3027, 3028, 0, 0, + 0, 0, 1623, 0, 0, 0, 1648, 0, 0, 0, + 1624, 1648, 0, 0, 0, 1636, 0, 0, 1637, 1638, + 1639, 0, 1640, 1641, 1642, 1643, 1644, 1645, 1920, 1624, + 0, 3051, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 3055, 0, 2792, 0, 0, + 3057, 3058, 0, 0, 0, 3059, 0, 0, 0, 0, + 3062, 0, 0, 3065, 3066, 0, 0, 0, 2273, 1279, + 0, 0, 3074, 0, 0, 3067, 0, 0, -2046, 0, + 0, 1636, 0, 1107, 1637, 1638, 1639, 0, 1640, 1641, + 1642, 1643, 1644, 1645, 0, -2046, 0, 0, 0, 0, + -2046, 0, 0, 0, 0, 0, -2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, -2046, 0, 1625, 0, 3115, -2046, 0, + 0, 0, 0, 0, 0, 0, 0, 0, -2046, 0, + 0, 0, 1626, 0, 0, 0, 0, 1627, 0, 0, + 0, 0, 0, 0, 3134, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, -2046, 0, 0, 0, + 1628, 1629, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1630, 0, 0, 0, 0, + 1648, 0, 0, 1633, 0, 0, 0, 0, 1920, 1920, + 0, 1416, 1416, 1416, 1416, 1416, 1416, 0, 0, 1416, + 1416, 1416, 1416, 1416, 1416, 1416, 1416, 1416, 1416, 1920, + 0, 1633, 0, 1631, 0, 0, 1632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1624, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1630, 0, 0, 1631, 0, 0, 1625, - 0, 0, 0, 0, 1626, 0, 0, 0, 0, 0, - 1632, 0, 0, 1633, 0, 0, 0, 1647, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1627, 1628, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1647, - 1647, 1647, 1629, 0, 1919, 1919, 1919, 1919, 1919, 1919, - 0, 0, 0, 1919, 1919, 1919, 1919, 1919, 1919, 1919, - 1919, 1919, 1919, 0, 0, 0, 0, 1647, 1647, 0, + 1633, 0, 0, 1634, 3168, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1611, 0, + 0, 1612, 0, 0, 0, 1613, 1614, 1615, 1616, 1617, + 1618, 1619, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1620, 0, 0, 0, + 0, 1592, 0, -2046, 0, 0, 1622, 0, 0, 0, + 0, 0, 0, 1623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1630, 0, 0, 1631, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1647, 0, 1632, 0, 0, - 1633, 1647, 0, 0, 1610, 0, 0, 1611, 0, 0, - 1634, 1612, 1613, 1614, 1615, 1616, 1617, 1618, 0, 0, + 0, -2046, 1648, 0, 2487, 1648, 0, 0, 0, 0, + 1624, 0, 0, 0, 0, 3228, 3229, 0, 0, 3230, + 1635, 1584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1619, 0, 0, 1647, 0, 0, 0, 0, - 0, 0, 1621, 0, 0, 0, 0, 0, 0, 1622, - 0, 0, 0, 0, 1647, 0, 0, 1647, 1647, 0, - 0, 0, 0, 0, 0, 1919, 1919, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1623, 0, 0, 1647, - 1415, 1415, 1647, 0, 1647, 0, 0, 0, 1647, 0, - 0, 0, 0, 0, 0, 0, 0, 1634, 0, 0, + 0, 0, 0, 0, 0, 0, 3248, 0, 0, 0, + 0, 1648, 0, 0, 0, 1648, 1648, 1648, 1648, 1648, + 1648, 1648, 1648, 0, 0, 0, 0, 0, 1416, 1416, + 3260, 1648, 1648, 0, 0, 0, 0, 0, 0, 0, + -2046, 0, 0, 1648, 0, 0, 1648, 1640, 1641, 1642, + 1643, 1644, 1645, 0, 1648, 1648, 1648, 1648, 1648, 1648, + 1648, 1648, 1648, 1648, 0, 0, 1625, 0, -2046, 0, + 0, 0, 0, 0, 0, 1640, 1641, 1642, 1643, 1644, + 1645, 0, 0, 1626, 0, 0, 0, 1636, 1627, 1648, + 1637, 1638, 1639, 0, 1640, 1641, 1642, 1643, 1644, 1645, + 0, 3306, 0, 0, 2064, 0, 0, 0, 0, 0, + 0, 1628, 1629, 0, 0, 0, 0, 0, 0, 3027, + 0, 0, 0, 3323, 0, 0, 1630, 1107, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 3332, 0, + 0, 0, 1416, 2273, 0, 2273, 0, 0, 0, 0, + 0, 0, 0, 1107, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1631, 0, 0, 1632, 0, 0, + 0, 3357, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1633, 0, 0, 1634, 0, 0, 0, 1611, 0, + 0, 1612, 0, 0, 0, 1613, 1614, 1615, 1616, 1617, + 1618, 1619, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1648, 0, 0, 0, 0, 1620, 0, 0, 0, + 3383, 1648, 1648, 0, 0, 0, 1622, 0, 0, 0, + 3027, 0, 0, 1623, 0, 0, 0, 0, 0, 0, + 0, 0, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1635, 0, 0, - 1636, 1637, 1638, 0, 1639, 1640, 1641, 1642, 1643, 1644, - 0, 0, 0, 0, 2368, 0, 0, 0, 0, 0, - 0, 0, 1647, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1610, 0, 0, 1611, 0, 0, 0, 1612, - 1613, 1614, 1615, 1616, 1617, 1618, 0, 0, 0, 0, - 0, 0, 1624, 0, 0, 0, 0, 0, 0, 0, - 1619, 0, 0, 0, 0, 0, 0, 0, 0, 1625, - 1621, 0, 0, 0, 1626, 0, 0, 1622, 0, 0, - 0, 0, 0, 0, 1635, 0, 0, 1636, 1637, 1638, - 0, 1639, 1640, 1641, 1642, 1643, 1644, 1627, 1628, 0, - 0, 2469, 0, 0, 1623, 0, 0, 1610, 0, 0, - 1611, 0, 1629, 0, 1612, 1613, 1614, 1615, 1616, 1617, - 1618, 0, 0, 0, 0, 0, 0, 0, 0, 1647, - 0, 0, 0, 0, 0, 1619, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1621, 0, 0, 0, 0, - 1630, 1415, 1622, 1631, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1632, 0, 0, - 1633, 0, 0, 0, 0, 0, 0, 0, 0, 1623, + 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3426, 0, 0, 0, 0, 0, + 0, 1635, 0, 0, 0, 0, 1648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1610, 0, 0, 1611, 0, 0, 0, 1612, 1613, 0, - 1624, 1616, 1617, 1618, 0, 0, 0, 1647, 0, 1647, - 0, 0, 0, 0, 0, 1647, 0, 1625, 1619, 0, - 0, 0, 1626, 0, 1647, 0, 0, 1647, 1621, 1647, - 0, 0, 0, 1647, 0, 1622, 1919, 1919, 0, 0, - 1647, 1647, 0, 0, 0, 1627, 1628, 0, 1647, 0, - 0, 0, 0, 0, 0, 0, 0, 1647, 0, 0, - 1629, 0, 1623, 0, 0, 0, 0, 1634, 0, 0, - 0, 0, 1647, 0, 0, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1625, 0, 0, 0, 0, 1626, 1630, 0, - 0, 1631, 0, 0, 0, 0, 0, 0, 0, 0, - 1415, 0, 0, 0, 0, 1632, 0, 0, 1633, 0, - 1627, 1628, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1629, 0, 0, 0, 0, + 1648, 1648, 1648, 0, 0, 1920, 1920, 1920, 1920, 1920, + 1920, 0, 0, 0, 1920, 1920, 1920, 1920, 1920, 1920, + 1920, 1920, 1920, 1920, 0, 0, 0, 0, 1648, 1648, + 0, 0, 1611, 0, 0, 1612, 0, 0, 0, 1613, + 1614, 1615, 1616, 1617, 1618, 1619, 1625, 0, 0, 0, + 0, 0, 3480, 3480, 3480, 0, 1648, 0, 0, 0, + 1620, 0, 1648, 1626, 0, 0, 0, 0, 1627, 0, + 1622, 0, 0, 0, 0, 0, 0, 1623, 1636, 3480, + 0, 1637, 1638, 1639, 0, 1640, 1641, 1642, 1643, 1644, + 1645, 1628, 1629, 0, 0, 2064, 1648, 0, 0, 0, + 0, 0, 0, 0, 1624, 0, 1630, 0, 0, 0, + 0, 0, 0, 0, 0, 1648, 0, 0, 1648, 1648, + 3480, 0, 0, 0, 0, 0, 1920, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1624, 0, - 1364, 1365, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1630, 1635, 1625, 1631, 1636, 1637, 1638, - 1626, 1639, 1640, 1641, 1642, 1643, 1644, 0, 0, 0, - 1632, 2493, 0, 1633, 0, 0, 0, 0, 1364, 1365, - 0, 1366, 1367, 1627, 1628, 1368, 1369, 0, 0, 0, - 0, 0, 0, 0, 0, 1634, 0, 0, 1629, 0, + 1648, 1416, 1416, 1648, 1631, 1648, 0, 1632, 0, 1648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1366, - 1367, 0, 0, 1368, 1369, 0, 0, 0, 0, 1919, - 1415, 0, 0, 0, 0, 0, 1630, 0, 0, 1631, + 0, 1633, 0, 0, 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1632, 1647, 1647, 1633, 0, 0, 0, - 1370, 1371, 0, 0, 0, 0, 0, 0, 0, 0, - 1634, 0, 0, 0, 0, 0, 0, 0, 1647, 0, - 0, 0, 1647, 0, 1647, 1647, 1647, 0, 0, 1647, - 0, 0, 1647, 1647, 0, 0, 0, 0, 1370, 1371, - 0, 1647, 1635, 0, 0, 1636, 1637, 1638, 0, 1639, - 1640, 1641, 1642, 1643, 1644, 0, 0, 0, 0, 2699, - 0, 0, 0, 0, 1372, 1373, 1374, 1375, 1376, 1377, - 1378, 1379, 0, 0, 1380, 1381, 0, 0, 0, 0, - 0, 0, 1647, 0, 0, 0, 0, 0, 1919, 0, - 0, 0, 0, 1634, 0, 0, 0, 0, 0, 0, - 0, 1647, 1372, 1373, 1374, 1375, 1376, 1377, 1378, 1379, - 0, 0, 1380, 1381, 0, 0, 0, 1635, 0, 0, - 1636, 1637, 1638, 0, 1639, 1640, 1641, 1642, 1643, 1644, - 0, 0, 0, 0, 2771, 1647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1919, 0, 0, 0, 0, 0, 0, 0, + 1625, 0, 0, 1648, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1626, 0, 0, + 0, 1611, 1627, 0, 1612, 0, 0, 0, 1613, 1614, + 1615, 1616, 1617, 1618, 1619, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1628, 1629, 0, 0, 1620, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1622, + 1630, 1635, 0, 0, 0, 0, 1623, 0, 0, 0, + 0, 1611, 0, 0, 1612, 0, 0, 0, 1613, 1614, + 1615, 1616, 1617, 1618, 1619, 0, 0, 0, 0, 0, + 0, 0, 0, 1624, 0, 0, 0, 0, 1631, 1620, + 0, 1632, 0, 0, 0, 0, 0, 0, 0, 1622, + 1648, 0, 0, 0, 0, 1633, 1623, 1197, 1634, 0, + 0, 0, 0, 1198, 0, 0, 0, 0, 0, 0, + 0, 1210, 1416, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1624, 0, 0, 0, 0, 0, 0, + 0, 1211, 0, 1611, 0, 0, 1612, 0, 0, 0, + 1613, 1614, 1615, 1616, 1617, 1618, 1619, 0, 1636, 0, + 0, 1637, 1638, 1639, 0, 1640, 1641, 1642, 1643, 1644, + 1645, 1620, 0, 0, 0, 2371, 0, 0, 1648, 1625, + 1648, 1622, 0, 0, 0, 0, 1648, 1212, 1623, 0, + 0, 0, 0, 0, 0, 1648, 1626, 0, 1648, 0, + 1648, 1627, 0, 0, 1648, 1635, 0, 1920, 1920, 0, + 0, 1648, 1648, 0, 0, 1624, 0, 0, 1199, 1648, + 0, 0, 0, 0, 1628, 1629, 0, 0, 1648, 1625, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1630, + 0, 0, 0, 1648, 0, 0, 1626, 0, 0, 0, + 0, 1627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1382, 1383, 0, - 0, 0, 0, 0, 0, 1647, 1647, 1647, 0, 0, - 1635, 0, 0, 1636, 1637, 1638, 0, 1639, 1640, 1641, - 1642, 1643, 1644, 0, 0, 1647, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1382, 1383, 1647, 0, 0, + 0, 0, 0, 0, 1628, 1629, 0, 1631, 0, 0, + 1632, 1416, 1213, 0, 0, 0, 0, 0, 0, 1630, + 0, 0, 0, 0, 1633, 0, 0, 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1625, 1636, 0, 0, 1637, 1638, 1639, 0, 1640, + 1641, 1642, 1643, 1644, 1645, 0, 0, 1631, 1626, 2472, + 1632, 1214, 0, 1627, 0, 0, 0, 0, 0, 0, + 0, 1215, 0, 0, 1633, 0, 0, 1634, 0, 0, + 0, 0, 0, 1216, 0, 0, 1628, 1629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1630, 0, 0, 1200, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1217, 0, 0, 0, 0, + 0, 0, 0, 0, 1635, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1631, + 1365, 1366, 1632, 0, 0, 0, 0, 0, 0, 0, + 1920, 1416, 0, 0, 0, 0, 1633, 0, 0, 1634, + 0, 2248, 0, 0, 0, 1202, 0, 1219, 0, 0, + 0, 0, 0, 0, 1635, 1648, 1648, 0, 0, 0, + 0, 1367, 1368, 0, 0, 1369, 1370, 0, 0, 0, + 0, 0, 1220, 0, 0, 0, 0, 2249, 0, 1648, + 0, 0, 0, 1648, 0, 1648, 1648, 1648, 0, 0, + 1648, 1222, 0, 1648, 1648, 0, 0, 0, 0, 0, + 0, 0, 1648, 0, 0, 0, 0, 0, 0, 0, + 0, 1636, 0, 0, 1637, 1638, 1639, 0, 1640, 1641, + 1642, 1643, 1644, 1645, 0, 0, 0, 0, 2496, 0, + 0, 0, 0, 0, 0, 0, 1635, 0, 0, 0, + 1371, 1372, 0, 1648, 0, 0, 0, 0, 1204, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1636, 1648, 0, 1637, 1638, 1639, 0, 1640, 1641, + 1642, 1643, 1644, 1645, 0, 0, 0, 0, 2702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1647, 0, 0, 0, 0, 0, 0, - 0, 0, 1384, 1385, 0, 0, 0, 0, 0, 0, - 1647, 0, 0, 0, 0, 0, 0, 0, 0, 1647, - 0, 0, 0, 0, 0, 0, 1386, 1387, 0, 0, + 0, 0, 0, 0, 0, 0, 1648, 0, 0, 0, + 0, 0, 0, 0, 1373, 1374, 1375, 1376, 1377, 1378, + 1379, 1380, 0, 0, 1381, 1382, 0, 0, 0, 0, + 0, 0, 0, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1384, 1385, 0, 0, 1647, 0, 1993, 0, 0, 0, + 0, 0, 0, 1636, 0, 0, 1637, 1638, 1639, 0, + 1640, 1641, 1642, 1643, 1644, 1645, 1648, 1648, 1648, 0, + 2774, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1648, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1386, 1387, 0, 0, 0, 0, - 1647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 1042, 815, 1043, - 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1647, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 0, 0, 1048, 0, 0, 128, 129, 130, 0, 131, - 132, 133, 134, 135, 136, 137, 138, 1049, 140, 1050, - 1051, 0, 143, 144, 145, 146, 147, 148, 1052, 785, - 149, 150, 151, 152, 1053, 1054, 155, 1647, 156, 157, - 158, 159, 786, 0, 787, 0, 1055, 163, 164, 165, - 166, 167, 168, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 178, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 1056, 190, 191, 1057, 193, - 1058, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 1059, - 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 1060, 221, 222, 223, 224, - 225, 788, 1061, 227, 0, 228, 229, 1062, 231, 0, - 232, 0, 233, 234, 0, 235, 236, 237, 238, 239, - 240, 0, 241, 0, 1063, 1064, 244, 245, 0, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 266, 267, 268, 269, 270, 271, 272, 1065, 1066, - 0, 1067, 0, 276, 277, 278, 279, 280, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 288, 289, 290, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 1068, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 1069, 321, 1070, 323, 324, 325, 326, 1071, 327, 328, - 329, 330, 1072, 790, 332, 1073, 334, 335, 336, 0, - 337, 338, 0, 0, 1074, 340, 341, 0, 0, 342, - 343, 344, 345, 346, 347, 792, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 793, 363, 364, 365, 366, 367, 368, - 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 1075, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 399, 400, 401, 402, 1076, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 0, 426, 427, 428, 1077, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 443, 444, 795, 0, 0, 446, 447, 0, - 448, 449, 450, 451, 452, 453, 454, 0, 455, 1078, - 1079, 0, 0, 458, 459, 796, 461, 797, 1080, 463, - 464, 798, 466, 467, 468, 469, 470, 0, 0, 471, - 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 483, 1081, 0, 485, 486, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, - 515, 1082, 0, 0, 0, 0, 0, 0, 1083, 1084, - 1085, 0, 0, 0, 0, 1086, 0, 1087, 0, 0, - 0, 0, 1088, 1089, 0, 1090, 1091, 2624, 116, 1042, - 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, - 120, 121, 122, 123, 124, 0, 125, 126, 127, 0, - 0, 0, 0, 0, 1048, 0, 0, 128, 129, 130, - 0, 131, 132, 133, 134, 135, 136, 137, 138, 1049, - 140, 1050, 1051, 0, 143, 144, 145, 146, 147, 148, - 1052, 785, 149, 150, 151, 152, 1053, 1054, 155, 0, - 156, 157, 158, 159, 786, 0, 787, 0, 1055, 163, - 164, 165, 166, 167, 168, 169, 170, 171, 0, 172, - 173, 174, 175, 176, 177, 0, 178, 179, 180, 181, - 182, 183, 184, 185, 186, 187, 188, 1056, 190, 191, - 1057, 193, 1058, 194, 0, 195, 196, 197, 198, 199, - 200, 0, 0, 201, 202, 203, 204, 0, 0, 205, - 206, 1059, 208, 209, 0, 210, 211, 212, 0, 213, - 214, 215, 0, 216, 217, 218, 219, 1060, 221, 222, - 223, 224, 225, 788, 1061, 227, 0, 228, 229, 1062, - 231, 0, 232, 0, 233, 234, 0, 235, 236, 237, - 238, 239, 240, 0, 241, 0, 1063, 1064, 244, 245, - 0, 246, 247, 248, 249, 250, 251, 252, 253, 254, - 255, 256, 257, 0, 258, 259, 260, 261, 262, 263, - 264, 0, 265, 266, 267, 268, 269, 270, 271, 272, - 1065, 1066, 0, 1067, 0, 276, 277, 278, 279, 280, - 281, 282, 283, 284, 285, 286, 0, 0, 287, 288, - 289, 290, 0, 291, 292, 293, 294, 295, 296, 297, - 298, 1068, 300, 301, 302, 303, 304, 305, 306, 307, - 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, - 318, 319, 1069, 321, 1070, 323, 324, 325, 326, 1071, - 327, 328, 329, 330, 1072, 790, 332, 1073, 334, 335, - 336, 0, 337, 338, 0, 0, 1074, 340, 341, 0, - 0, 342, 343, 344, 345, 346, 347, 792, 349, 350, - 351, 352, 353, 354, 355, 356, 357, 358, 359, 0, - 0, 0, 0, 360, 361, 793, 363, 364, 365, 366, - 367, 368, 369, 0, 370, 371, 372, 373, 374, 375, - 0, 376, 377, 378, 1075, 380, 381, 382, 383, 0, - 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, - 394, 395, 396, 0, 397, 398, 399, 400, 401, 402, - 1076, 404, 405, 406, 407, 408, 409, 410, 411, 412, - 413, 414, 415, 416, 0, 0, 417, 418, 419, 420, - 421, 422, 423, 424, 425, 0, 426, 427, 428, 1077, - 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, - 439, 440, 441, 442, 443, 444, 795, 0, 0, 446, - 447, 0, 448, 449, 450, 451, 452, 453, 454, 0, - 455, 1078, 1079, 0, 0, 458, 459, 796, 461, 797, - 1080, 463, 464, 798, 466, 467, 468, 469, 470, 0, - 0, 471, 472, 473, 0, 474, 475, 476, 477, 0, - 478, 479, 480, 481, 482, 483, 1081, 0, 485, 486, - 487, 488, 489, 490, 491, 492, 493, 0, 0, 494, - 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, - 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, - 513, 514, 515, 1082, 0, 0, 0, 0, 0, 0, - 1083, 1084, 1085, 0, 0, 0, 0, 1086, 0, 1087, - 0, 0, 0, 0, 1088, 1089, 0, 1090, 1091, 3228, - 116, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, - 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, - 127, 0, 0, 0, 0, 0, 1048, 0, 0, 128, - 129, 130, 0, 131, 132, 133, 134, 135, 136, 137, - 138, 1049, 140, 1050, 1051, 0, 143, 144, 145, 146, - 147, 148, 1052, 785, 149, 150, 151, 152, 1053, 1054, - 155, 0, 156, 157, 158, 159, 786, 0, 787, 0, - 1055, 163, 164, 165, 166, 167, 168, 169, 170, 171, - 0, 172, 173, 174, 175, 176, 177, 0, 178, 179, - 180, 181, 182, 183, 184, 185, 186, 187, 188, 1056, - 190, 191, 1057, 193, 1058, 194, 0, 195, 196, 197, - 198, 199, 200, 14, 15, 201, 202, 203, 204, 0, - 0, 205, 206, 1059, 208, 209, 0, 210, 211, 212, - 0, 213, 214, 215, 0, 216, 217, 218, 219, 1060, - 221, 222, 223, 224, 225, 788, 1061, 227, 0, 228, - 229, 1062, 231, 0, 232, 0, 233, 234, 23, 235, - 236, 237, 238, 239, 240, 0, 241, 0, 1063, 1064, - 244, 245, 0, 246, 247, 248, 249, 250, 251, 252, - 253, 254, 255, 256, 257, 0, 258, 259, 260, 261, - 262, 263, 264, 0, 265, 266, 267, 268, 269, 270, - 271, 272, 1065, 1066, 0, 1067, 0, 276, 277, 278, - 279, 280, 281, 282, 283, 284, 285, 286, 0, 0, - 287, 288, 289, 290, 0, 291, 292, 293, 294, 295, - 296, 297, 298, 1068, 300, 301, 302, 303, 304, 305, - 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, - 316, 317, 318, 319, 1069, 321, 1070, 323, 324, 325, - 326, 1071, 327, 328, 329, 330, 1072, 790, 332, 1073, - 334, 335, 336, 0, 337, 338, 0, 0, 1074, 340, - 341, 0, 0, 342, 343, 344, 345, 346, 347, 792, - 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, - 359, 27, 28, 29, 0, 360, 361, 793, 363, 364, - 365, 366, 367, 368, 369, 0, 370, 371, 372, 373, - 374, 375, 0, 376, 377, 378, 1075, 380, 381, 382, - 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, - 392, 393, 394, 395, 396, 0, 397, 398, 399, 400, - 401, 402, 1076, 404, 405, 406, 407, 408, 409, 410, - 411, 412, 413, 414, 415, 416, 34, 0, 417, 418, - 419, 420, 421, 422, 423, 424, 425, 36, 426, 427, - 428, 1077, 430, 0, 431, 432, 433, 434, 435, 436, - 437, 438, 439, 440, 441, 442, 443, 444, 795, 38, - 0, 446, 447, 39, 448, 449, 450, 451, 452, 453, - 454, 0, 455, 1078, 1079, 0, 0, 458, 459, 796, - 461, 797, 1080, 463, 464, 798, 466, 467, 468, 469, - 470, 0, 0, 471, 472, 473, 41, 474, 475, 476, - 477, 0, 478, 479, 480, 481, 482, 799, 1081, 0, - 485, 486, 487, 488, 489, 490, 491, 492, 493, 0, - 0, 494, 0, 45, 495, 496, 497, 498, 499, 500, - 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, - 511, 512, 513, 514, 515, 1082, 0, 46, 0, 0, - 0, 0, 1083, 1084, 1085, 0, 0, 0, 0, 1086, - 0, 1087, 3203, 0, 0, 0, 1088, 1089, 0, 1090, - 1091, 116, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, - 126, 127, 0, 0, 0, 0, 0, 1048, 0, 0, - 128, 129, 130, 0, 131, 132, 133, 134, 135, 136, - 137, 138, 1049, 140, 1050, 1051, 0, 143, 144, 145, - 146, 147, 148, 1052, 785, 149, 150, 151, 152, 1053, - 1054, 155, 0, 156, 157, 158, 159, 786, 0, 787, - 0, 1055, 163, 164, 165, 166, 167, 168, 169, 170, - 171, 0, 172, 173, 174, 175, 176, 177, 0, 178, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 1056, 190, 191, 1057, 193, 1058, 194, 0, 195, 196, - 197, 198, 199, 200, 14, 15, 201, 202, 203, 204, - 0, 0, 205, 206, 1059, 208, 209, 0, 210, 211, - 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, - 1060, 221, 222, 223, 224, 225, 788, 1061, 227, 0, - 228, 229, 1062, 231, 0, 232, 0, 233, 234, 23, - 235, 236, 237, 238, 239, 240, 0, 241, 0, 1063, - 1064, 244, 245, 0, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 0, 258, 259, 260, - 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, - 270, 271, 272, 1065, 1066, 0, 1067, 0, 276, 277, - 278, 279, 280, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 288, 289, 290, 0, 291, 292, 293, 294, - 295, 296, 297, 298, 1068, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 1069, 321, 1070, 323, 324, - 325, 326, 1071, 327, 328, 329, 330, 1072, 790, 332, - 1073, 334, 335, 336, 0, 337, 338, 0, 0, 1074, - 340, 341, 0, 0, 342, 343, 344, 345, 346, 347, - 792, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 27, 28, 29, 0, 360, 361, 793, 363, - 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, - 373, 374, 375, 0, 376, 377, 378, 1075, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 397, 398, 399, - 400, 401, 402, 1076, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 34, 0, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 36, 426, - 427, 428, 1077, 430, 0, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 443, 444, 795, - 38, 0, 446, 447, 39, 448, 449, 450, 451, 452, - 453, 454, 0, 455, 1078, 1079, 0, 0, 458, 459, - 796, 461, 797, 1080, 463, 464, 798, 466, 467, 468, - 469, 470, 0, 0, 471, 472, 473, 41, 474, 475, - 476, 477, 0, 478, 479, 480, 481, 482, 799, 1081, - 0, 485, 486, 487, 488, 489, 490, 491, 492, 493, - 0, 0, 494, 0, 45, 495, 496, 497, 498, 499, - 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, + 0, 0, 0, 0, 0, 0, 0, 1383, 1384, 0, + 0, 0, 0, 0, 1648, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1648, 0, 0, 0, 0, 0, 0, 0, 0, + 1648, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1648, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1648, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1385, 1386, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 116, 1042, 815, + 1043, 1044, 1045, 1046, 1047, 0, 1387, 1388, 0, 0, + 0, 0, 0, 0, 1648, 0, 117, 118, 119, 120, + 121, 122, 123, 124, 0, 125, 126, 127, 0, 0, + 0, 0, 0, 1048, 0, 0, 128, 129, 130, 0, + 131, 132, 133, 134, 135, 136, 137, 138, 1049, 140, + 1050, 1051, 0, 143, 144, 145, 146, 147, 148, 1052, + 785, 149, 150, 151, 152, 1053, 1054, 155, 1648, 156, + 157, 158, 159, 786, 0, 787, 0, 1055, 163, 164, + 165, 166, 167, 168, 169, 170, 171, 0, 172, 173, + 174, 175, 176, 177, 0, 178, 179, 180, 181, 182, + 183, 184, 185, 186, 187, 188, 1056, 190, 191, 1057, + 193, 1058, 194, 0, 195, 196, 197, 198, 199, 200, + 0, 0, 201, 202, 203, 204, 0, 0, 205, 206, + 1059, 208, 209, 0, 210, 211, 212, 0, 213, 214, + 215, 0, 216, 217, 218, 219, 1060, 221, 222, 223, + 224, 225, 788, 1061, 227, 0, 228, 229, 1062, 231, + 0, 232, 0, 233, 234, 0, 235, 236, 237, 238, + 239, 240, 0, 241, 0, 1063, 1064, 244, 245, 0, + 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, + 256, 257, 0, 258, 259, 260, 261, 262, 263, 264, + 0, 265, 266, 267, 268, 269, 270, 271, 272, 1065, + 1066, 0, 1067, 0, 276, 277, 278, 279, 280, 281, + 282, 283, 284, 285, 286, 0, 0, 287, 288, 289, + 290, 0, 291, 292, 293, 294, 295, 296, 297, 298, + 1068, 300, 301, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, + 319, 1069, 321, 1070, 323, 324, 325, 326, 1071, 327, + 328, 329, 330, 1072, 790, 332, 1073, 334, 335, 336, + 0, 337, 338, 0, 0, 1074, 340, 341, 0, 0, + 342, 343, 344, 345, 346, 347, 792, 349, 350, 351, + 352, 353, 354, 355, 356, 357, 358, 359, 0, 0, + 0, 0, 360, 361, 793, 363, 364, 365, 366, 367, + 368, 369, 0, 370, 371, 372, 373, 374, 375, 0, + 376, 377, 378, 1075, 380, 381, 382, 383, 0, 384, + 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, + 395, 396, 0, 397, 398, 399, 400, 401, 402, 1076, + 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, + 414, 415, 416, 0, 0, 417, 418, 419, 420, 421, + 422, 423, 424, 425, 0, 426, 427, 428, 1077, 430, + 0, 431, 432, 433, 434, 435, 436, 437, 438, 439, + 440, 441, 442, 443, 444, 795, 0, 0, 446, 447, + 0, 448, 449, 450, 451, 452, 453, 454, 0, 455, + 1078, 1079, 0, 0, 458, 459, 796, 461, 797, 1080, + 463, 464, 798, 466, 467, 468, 469, 470, 0, 0, + 471, 472, 473, 0, 474, 475, 476, 477, 0, 478, + 479, 480, 481, 482, 483, 1081, 0, 485, 486, 487, + 488, 489, 490, 491, 492, 493, 0, 0, 494, 0, + 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, + 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, + 514, 515, 1082, 0, 0, 0, 0, 0, 0, 1083, + 1084, 1085, 0, 0, 0, 0, 1086, 0, 1087, 0, + 0, 0, 0, 1088, 1089, 0, 1090, 1091, 2627, 116, + 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, + 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, + 0, 0, 0, 0, 0, 1048, 0, 0, 128, 129, + 130, 0, 131, 132, 133, 134, 135, 136, 137, 138, + 1049, 140, 1050, 1051, 0, 143, 144, 145, 146, 147, + 148, 1052, 785, 149, 150, 151, 152, 1053, 1054, 155, + 0, 156, 157, 158, 159, 786, 0, 787, 0, 1055, + 163, 164, 165, 166, 167, 168, 169, 170, 171, 0, + 172, 173, 174, 175, 176, 177, 0, 178, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 1056, 190, + 191, 1057, 193, 1058, 194, 0, 195, 196, 197, 198, + 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, + 205, 206, 1059, 208, 209, 0, 210, 211, 212, 0, + 213, 214, 215, 0, 216, 217, 218, 219, 1060, 221, + 222, 223, 224, 225, 788, 1061, 227, 0, 228, 229, + 1062, 231, 0, 232, 0, 233, 234, 0, 235, 236, + 237, 238, 239, 240, 0, 241, 0, 1063, 1064, 244, + 245, 0, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, + 272, 1065, 1066, 0, 1067, 0, 276, 277, 278, 279, + 280, 281, 282, 283, 284, 285, 286, 0, 0, 287, + 288, 289, 290, 0, 291, 292, 293, 294, 295, 296, + 297, 298, 1068, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 1069, 321, 1070, 323, 324, 325, 326, + 1071, 327, 328, 329, 330, 1072, 790, 332, 1073, 334, + 335, 336, 0, 337, 338, 0, 0, 1074, 340, 341, + 0, 0, 342, 343, 344, 345, 346, 347, 792, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 0, 0, 0, 0, 360, 361, 793, 363, 364, 365, + 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, + 375, 0, 376, 377, 378, 1075, 380, 381, 382, 383, + 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 397, 398, 399, 400, 401, + 402, 1076, 404, 405, 406, 407, 408, 409, 410, 411, + 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 0, 426, 427, 428, + 1077, 430, 0, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 443, 444, 795, 0, 0, + 446, 447, 0, 448, 449, 450, 451, 452, 453, 454, + 0, 455, 1078, 1079, 0, 0, 458, 459, 796, 461, + 797, 1080, 463, 464, 798, 466, 467, 468, 469, 470, + 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 1081, 0, 485, + 486, 487, 488, 489, 490, 491, 492, 493, 0, 0, + 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, + 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, + 512, 513, 514, 515, 1082, 0, 0, 0, 0, 0, + 0, 1083, 1084, 1085, 0, 0, 0, 0, 1086, 0, + 1087, 0, 0, 0, 0, 1088, 1089, 0, 1090, 1091, + 3231, 116, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, + 126, 127, 0, 0, 0, 0, 0, 1048, 0, 0, + 128, 129, 130, 0, 131, 132, 133, 134, 135, 136, + 137, 138, 1049, 140, 1050, 1051, 0, 143, 144, 145, + 146, 147, 148, 1052, 785, 149, 150, 151, 152, 1053, + 1054, 155, 0, 156, 157, 158, 159, 786, 0, 787, + 0, 1055, 163, 164, 165, 166, 167, 168, 169, 170, + 171, 0, 172, 173, 174, 175, 176, 177, 0, 178, + 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, + 1056, 190, 191, 1057, 193, 1058, 194, 0, 195, 196, + 197, 198, 199, 200, 14, 15, 201, 202, 203, 204, + 0, 0, 205, 206, 1059, 208, 209, 0, 210, 211, + 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, + 1060, 221, 222, 223, 224, 225, 788, 1061, 227, 0, + 228, 229, 1062, 231, 0, 232, 0, 233, 234, 23, + 235, 236, 237, 238, 239, 240, 0, 241, 0, 1063, + 1064, 244, 245, 0, 246, 247, 248, 249, 250, 251, + 252, 253, 254, 255, 256, 257, 0, 258, 259, 260, + 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, + 270, 271, 272, 1065, 1066, 0, 1067, 0, 276, 277, + 278, 279, 280, 281, 282, 283, 284, 285, 286, 0, + 0, 287, 288, 289, 290, 0, 291, 292, 293, 294, + 295, 296, 297, 298, 1068, 300, 301, 302, 303, 304, + 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, + 315, 316, 317, 318, 319, 1069, 321, 1070, 323, 324, + 325, 326, 1071, 327, 328, 329, 330, 1072, 790, 332, + 1073, 334, 335, 336, 0, 337, 338, 0, 0, 1074, + 340, 341, 0, 0, 342, 343, 344, 345, 346, 347, + 792, 349, 350, 351, 352, 353, 354, 355, 356, 357, + 358, 359, 27, 28, 29, 0, 360, 361, 793, 363, + 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, + 373, 374, 375, 0, 376, 377, 378, 1075, 380, 381, + 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, + 391, 392, 393, 394, 395, 396, 0, 397, 398, 399, + 400, 401, 402, 1076, 404, 405, 406, 407, 408, 409, + 410, 411, 412, 413, 414, 415, 416, 34, 0, 417, + 418, 419, 420, 421, 422, 423, 424, 425, 36, 426, + 427, 428, 1077, 430, 0, 431, 432, 433, 434, 435, + 436, 437, 438, 439, 440, 441, 442, 443, 444, 795, + 38, 0, 446, 447, 39, 448, 449, 450, 451, 452, + 453, 454, 0, 455, 1078, 1079, 0, 0, 458, 459, + 796, 461, 797, 1080, 463, 464, 798, 466, 467, 468, + 469, 470, 0, 0, 471, 472, 473, 41, 474, 475, + 476, 477, 0, 478, 479, 480, 481, 482, 799, 1081, + 0, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 0, 0, 494, 0, 45, 495, 496, 497, 498, 499, + 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 1082, 0, 46, 0, 0, 0, 0, 1083, 1084, 1085, 0, 0, 0, 0, - 1086, 0, 1087, 0, 0, 0, 0, 1088, 1089, 0, + 1086, 0, 1087, 3206, 0, 0, 0, 1088, 1089, 0, 1090, 1091, 116, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, @@ -5408,7 +5367,7 @@ static const yytype_int16 yytable[] = 170, 171, 0, 172, 173, 174, 175, 176, 177, 0, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 1056, 190, 191, 1057, 193, 1058, 194, 0, 195, - 196, 197, 198, 199, 200, 0, 0, 201, 202, 203, + 196, 197, 198, 199, 200, 14, 15, 201, 202, 203, 204, 0, 0, 205, 206, 1059, 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, 1060, 221, 222, 223, 224, 225, 788, 1061, 227, @@ -5434,10 +5393,10 @@ static const yytype_int16 yytable[] = 390, 391, 392, 393, 394, 395, 396, 0, 397, 398, 399, 400, 401, 402, 1076, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 34, 0, - 417, 418, 419, 420, 421, 422, 423, 424, 425, 0, + 417, 418, 419, 420, 421, 422, 423, 424, 425, 36, 426, 427, 428, 1077, 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, - 795, 0, 0, 446, 447, 39, 448, 449, 450, 451, + 795, 38, 0, 446, 447, 39, 448, 449, 450, 451, 452, 453, 454, 0, 455, 1078, 1079, 0, 0, 458, 459, 796, 461, 797, 1080, 463, 464, 798, 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, 41, 474, @@ -5448,478 +5407,478 @@ static const yytype_int16 yytable[] = 509, 510, 511, 512, 513, 514, 515, 1082, 0, 46, 0, 0, 0, 0, 1083, 1084, 1085, 0, 0, 0, 0, 1086, 0, 1087, 0, 0, 0, 0, 1088, 1089, - 0, 1090, 1091, 1241, 1042, 815, 1043, 1044, 1045, 1046, + 0, 1090, 1091, 116, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, - 1242, 125, 126, 127, 0, 0, 0, 1243, 0, 1048, - 0, 0, 1244, 129, 130, 0, 131, 132, 133, 1245, - 135, 136, 137, 138, 1049, 1246, 1050, 1051, 0, 143, + 0, 125, 126, 127, 0, 0, 0, 0, 0, 1048, + 0, 0, 128, 129, 130, 0, 131, 132, 133, 134, + 135, 136, 137, 138, 1049, 140, 1050, 1051, 0, 143, 144, 145, 146, 147, 148, 1052, 785, 149, 150, 151, 152, 1053, 1054, 155, 0, 156, 157, 158, 159, 786, - 0, 1247, 0, 1248, 163, 164, 165, 166, 167, 1249, + 0, 787, 0, 1055, 163, 164, 165, 166, 167, 168, 169, 170, 171, 0, 172, 173, 174, 175, 176, 177, - 0, 1250, 179, 180, 181, 182, 183, 184, 185, 186, + 0, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 1056, 190, 191, 1057, 193, 1058, 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, 202, - 203, 204, 1251, 0, 205, 206, 1059, 208, 209, 0, + 203, 204, 0, 0, 205, 206, 1059, 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, 1060, 221, 222, 223, 224, 225, 788, 1061, 227, 0, 228, 229, 1062, 231, 0, 232, 0, 233, - 1252, 0, 1253, 236, 237, 1254, 1255, 240, 0, 241, + 234, 23, 235, 236, 237, 238, 239, 240, 0, 241, 0, 1063, 1064, 244, 245, 0, 246, 247, 248, 249, - 250, 251, 252, 1256, 254, 255, 256, 257, 0, 258, - 259, 260, 261, 262, 263, 264, 0, 265, 1257, 267, + 250, 251, 252, 253, 254, 255, 256, 257, 0, 258, + 259, 260, 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, 272, 1065, 1066, 0, 1067, 0, - 276, 1258, 1259, 279, 1260, 281, 282, 283, 284, 285, - 286, 0, 0, 287, 1261, 289, 1262, 0, 291, 292, - 293, 294, 295, 296, 297, 298, 1263, 300, 301, 302, + 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, + 286, 0, 0, 287, 288, 289, 290, 0, 291, 292, + 293, 294, 295, 296, 297, 298, 1068, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, - 313, 314, 315, 316, 317, 318, 319, 1069, 1264, 1070, - 323, 324, 325, 326, 1071, 327, 328, 1265, 330, 1072, + 313, 314, 315, 316, 317, 318, 319, 1069, 321, 1070, + 323, 324, 325, 326, 1071, 327, 328, 329, 330, 1072, 790, 332, 1073, 334, 335, 336, 0, 337, 338, 0, - 0, 1074, 340, 341, 0, 0, 342, 343, 344, 1266, - 346, 1267, 792, 349, 350, 351, 352, 353, 354, 355, - 356, 357, 358, 359, 0, 0, 0, 0, 360, 361, - 793, 1268, 364, 365, 366, 367, 368, 369, 0, 370, + 0, 1074, 340, 341, 0, 0, 342, 343, 344, 345, + 346, 347, 792, 349, 350, 351, 352, 353, 354, 355, + 356, 357, 358, 359, 27, 28, 29, 0, 360, 361, + 793, 363, 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, 1075, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, 397, - 398, 1269, 400, 401, 402, 1076, 404, 405, 406, 407, - 408, 409, 410, 411, 412, 413, 414, 415, 416, 0, - 1270, 417, 418, 419, 420, 421, 422, 423, 424, 425, - 0, 1271, 427, 428, 1077, 430, 0, 431, 432, 433, - 434, 435, 436, 437, 438, 439, 440, 441, 442, 1272, - 444, 795, 0, 0, 446, 447, 0, 448, 1273, 450, + 398, 399, 400, 401, 402, 1076, 404, 405, 406, 407, + 408, 409, 410, 411, 412, 413, 414, 415, 416, 34, + 0, 417, 418, 419, 420, 421, 422, 423, 424, 425, + 0, 426, 427, 428, 1077, 430, 0, 431, 432, 433, + 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, + 444, 795, 0, 0, 446, 447, 39, 448, 449, 450, 451, 452, 453, 454, 0, 455, 1078, 1079, 0, 0, - 458, 459, 796, 461, 797, 1080, 463, 464, 1274, 466, - 467, 468, 469, 470, 0, 0, 471, 472, 473, 0, + 458, 459, 796, 461, 797, 1080, 463, 464, 798, 466, + 467, 468, 469, 470, 0, 0, 471, 472, 473, 41, 474, 475, 476, 477, 0, 478, 479, 480, 481, 482, - 483, 1081, 1275, 485, 1276, 487, 488, 489, 490, 491, - 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, + 799, 1081, 0, 485, 486, 487, 488, 489, 490, 491, + 492, 493, 0, 0, 494, 0, 45, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 1082, 0, - 0, 0, 0, 0, 0, 1083, 1084, 1085, 0, 0, - 0, 0, 1086, 0, 1087, 1277, 0, 0, 0, 1088, - 1089, 0, 1090, 1091, 116, 1042, 815, 1043, 1044, 0, + 46, 0, 0, 0, 0, 1083, 1084, 1085, 0, 0, + 0, 0, 1086, 0, 1087, 0, 0, 0, 0, 1088, + 1089, 0, 1090, 1091, 1242, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, - 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, - 1048, 0, 0, 128, 129, 130, 0, 131, 132, 133, - 134, 135, 136, 137, 138, 1049, 140, 1050, 1051, 0, + 124, 1243, 125, 126, 127, 0, 0, 0, 1244, 0, + 1048, 0, 0, 1245, 129, 130, 0, 131, 132, 133, + 1246, 135, 136, 137, 138, 1049, 1247, 1050, 1051, 0, 143, 144, 145, 146, 147, 148, 1052, 785, 149, 150, 151, 152, 1053, 1054, 155, 0, 156, 157, 158, 159, - 786, 0, 787, 0, 162, 163, 164, 165, 166, 167, - 168, 169, 170, 171, 0, 172, 173, 174, 175, 176, - 177, 0, 178, 179, 180, 181, 182, 183, 184, 185, - 186, 187, 188, 1056, 190, 191, 1057, 193, 0, 194, - 0, 195, 196, 197, 198, 199, 200, 14, 15, 201, - 202, 203, 204, 0, 0, 205, 206, 1059, 208, 209, + 786, 0, 1248, 0, 1249, 163, 164, 165, 166, 167, + 1250, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 1251, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 1056, 190, 191, 1057, 193, 1058, 194, + 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, + 202, 203, 204, 1252, 0, 205, 206, 1059, 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, 1060, 221, 222, 223, 224, 225, 788, 1061, 227, 0, 228, 229, 1062, 231, 0, 232, 0, - 233, 234, 23, 235, 236, 237, 238, 239, 240, 0, + 233, 1253, 0, 1254, 236, 237, 1255, 1256, 240, 0, 241, 0, 1063, 1064, 244, 245, 0, 246, 247, 248, - 249, 250, 251, 252, 253, 254, 255, 256, 257, 0, - 258, 259, 260, 261, 262, 263, 264, 0, 265, 266, + 249, 250, 251, 252, 1257, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 1258, 267, 268, 269, 270, 271, 272, 1065, 1066, 0, 1067, - 0, 276, 277, 278, 279, 280, 281, 282, 283, 284, - 285, 286, 0, 0, 287, 288, 289, 290, 0, 291, - 292, 293, 294, 295, 296, 297, 298, 1068, 300, 301, + 0, 276, 1259, 1260, 279, 1261, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 1262, 289, 1263, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 1264, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, 314, 315, 316, 317, 318, 319, 1069, 321, - 1070, 323, 324, 325, 326, 0, 327, 328, 329, 330, + 312, 313, 314, 315, 316, 317, 318, 319, 1069, 1265, + 1070, 323, 324, 325, 326, 1071, 327, 328, 1266, 330, 1072, 790, 332, 1073, 334, 335, 336, 0, 337, 338, - 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, - 345, 346, 347, 792, 349, 350, 351, 352, 353, 354, - 355, 356, 357, 358, 359, 27, 28, 29, 0, 360, - 361, 793, 363, 364, 365, 366, 367, 368, 369, 0, + 0, 0, 1074, 340, 341, 0, 0, 342, 343, 344, + 1267, 346, 1268, 792, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 361, 793, 1269, 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, 1075, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, - 397, 398, 399, 400, 401, 402, 1076, 404, 405, 406, + 397, 398, 1270, 400, 401, 402, 1076, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, - 34, 0, 417, 418, 419, 420, 421, 422, 423, 424, - 425, 36, 426, 427, 428, 1077, 430, 0, 431, 432, + 0, 1271, 417, 418, 419, 420, 421, 422, 423, 424, + 425, 0, 1272, 427, 428, 1077, 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, - 443, 444, 795, 38, 0, 446, 447, 39, 448, 449, + 1273, 444, 795, 0, 0, 446, 447, 0, 448, 1274, 450, 451, 452, 453, 454, 0, 455, 1078, 1079, 0, - 0, 458, 459, 796, 461, 797, 1080, 463, 464, 798, + 0, 458, 459, 796, 461, 797, 1080, 463, 464, 1275, 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, - 41, 474, 475, 476, 477, 0, 478, 479, 480, 481, - 482, 799, 1081, 0, 485, 486, 487, 488, 489, 490, - 491, 492, 493, 0, 0, 494, 0, 45, 495, 496, + 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 1081, 1276, 485, 1277, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, - 507, 508, 509, 510, 511, 512, 513, 514, 515, 0, - 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1086, 0, 1087, 0, 0, 0, 0, + 507, 508, 509, 510, 511, 512, 513, 514, 515, 1082, + 0, 0, 0, 0, 0, 0, 1083, 1084, 1085, 0, + 0, 0, 0, 1086, 0, 1087, 1278, 0, 0, 0, 1088, 1089, 0, 1090, 1091, 116, 1042, 815, 1043, 1044, - 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, + 0, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, 1048, 0, 0, 128, 129, 130, 0, 131, 132, 133, 134, 135, 136, 137, 138, 1049, 140, 1050, 1051, - 1422, 143, 144, 145, 146, 147, 148, 1052, 785, 149, + 0, 143, 144, 145, 146, 147, 148, 1052, 785, 149, 150, 151, 152, 1053, 1054, 155, 0, 156, 157, 158, - 159, 786, 0, 787, 0, 1055, 163, 164, 165, 166, + 159, 786, 0, 787, 0, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 0, 172, 173, 174, 175, 176, 177, 0, 178, 179, 180, 181, 182, 183, 184, - 185, 186, 187, 188, 1056, 190, 191, 1057, 193, 1058, - 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, + 185, 186, 187, 188, 1056, 190, 191, 1057, 193, 0, + 194, 0, 195, 196, 197, 198, 199, 200, 14, 15, 201, 202, 203, 204, 0, 0, 205, 206, 1059, 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, 1060, 221, 222, 223, 224, 225, 788, 1061, 227, 0, 228, 229, 1062, 231, 0, 232, - 0, 233, 234, 1423, 235, 236, 237, 238, 239, 240, + 0, 233, 234, 23, 235, 236, 237, 238, 239, 240, 0, 241, 0, 1063, 1064, 244, 245, 0, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, 272, 1065, 1066, 0, 1067, 0, 276, 277, 278, 279, 280, 281, 282, 283, - 284, 285, 286, 0, 1424, 287, 288, 289, 290, 0, + 284, 285, 286, 0, 0, 287, 288, 289, 290, 0, 291, 292, 293, 294, 295, 296, 297, 298, 1068, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 1069, - 321, 1070, 323, 324, 325, 326, 1071, 327, 328, 329, + 321, 1070, 323, 324, 325, 326, 0, 327, 328, 329, 330, 1072, 790, 332, 1073, 334, 335, 336, 0, 337, - 338, 0, 0, 1074, 340, 341, 0, 0, 342, 343, + 338, 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, 345, 346, 347, 792, 349, 350, 351, 352, 353, - 354, 355, 356, 357, 358, 359, 0, 0, 0, 0, + 354, 355, 356, 357, 358, 359, 27, 28, 29, 0, 360, 361, 793, 363, 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, 1075, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, 397, 398, 399, 400, 401, 402, 1076, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, - 416, 0, 0, 417, 418, 419, 420, 421, 422, 423, - 424, 425, 0, 426, 427, 428, 1077, 430, 0, 431, + 416, 34, 0, 417, 418, 419, 420, 421, 422, 423, + 424, 425, 36, 426, 427, 428, 1077, 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, - 442, 443, 444, 795, 0, 0, 446, 447, 0, 448, + 442, 443, 444, 795, 38, 0, 446, 447, 39, 448, 449, 450, 451, 452, 453, 454, 0, 455, 1078, 1079, - 0, 1425, 458, 459, 796, 461, 797, 1080, 463, 464, + 0, 0, 458, 459, 796, 461, 797, 1080, 463, 464, 798, 466, 467, 468, 469, 470, 0, 0, 471, 472, - 473, 0, 474, 475, 476, 477, 0, 478, 479, 480, - 481, 482, 483, 1081, 0, 485, 486, 487, 488, 489, - 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, + 473, 41, 474, 475, 476, 477, 0, 478, 479, 480, + 481, 482, 799, 1081, 0, 485, 486, 487, 488, 489, + 490, 491, 492, 493, 0, 0, 494, 0, 45, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, - 1082, 0, 0, 0, 0, 0, 0, 1083, 1084, 1085, + 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 0, 1087, 0, 0, 0, - 0, 1088, 1089, 0, 1090, 1091, 1241, 1042, 815, 1043, + 0, 1088, 1089, 0, 1090, 1091, 116, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 1243, 0, 1048, 0, 0, 1244, 129, 130, 0, 131, - 132, 133, 1245, 135, 136, 137, 138, 1049, 1246, 1050, - 1051, 0, 143, 144, 145, 146, 147, 148, 1052, 785, + 0, 0, 1048, 0, 0, 128, 129, 130, 0, 131, + 132, 133, 134, 135, 136, 137, 138, 1049, 140, 1050, + 1051, 1423, 143, 144, 145, 146, 147, 148, 1052, 785, 149, 150, 151, 152, 1053, 1054, 155, 0, 156, 157, - 158, 159, 786, 0, 1247, 0, 1248, 163, 164, 165, - 166, 167, 1249, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 1250, 179, 180, 181, 182, 183, + 158, 159, 786, 0, 787, 0, 1055, 163, 164, 165, + 166, 167, 168, 169, 170, 171, 0, 172, 173, 174, + 175, 176, 177, 0, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 1056, 190, 191, 1057, 193, 1058, 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, 205, 206, 1059, 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, 1060, 221, 222, 223, 224, 225, 788, 1061, 227, 0, 228, 229, 1062, 231, 0, - 232, 0, 233, 1252, 0, 1253, 236, 237, 1254, 1255, + 232, 0, 233, 234, 1424, 235, 236, 237, 238, 239, 240, 0, 241, 0, 1063, 1064, 244, 245, 0, 246, - 247, 248, 249, 250, 251, 252, 1256, 254, 255, 256, + 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 1257, 267, 268, 269, 270, 271, 272, 1065, 1066, - 0, 1067, 0, 276, 1258, 1259, 279, 1260, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 1261, 289, 1262, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 1263, + 265, 266, 267, 268, 269, 270, 271, 272, 1065, 1066, + 0, 1067, 0, 276, 277, 278, 279, 280, 281, 282, + 283, 284, 285, 286, 0, 1425, 287, 288, 289, 290, + 0, 291, 292, 293, 294, 295, 296, 297, 298, 1068, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 1069, 1264, 1070, 323, 324, 325, 326, 1071, 327, 328, - 1265, 330, 1072, 790, 332, 1073, 334, 335, 336, 0, + 1069, 321, 1070, 323, 324, 325, 326, 1071, 327, 328, + 329, 330, 1072, 790, 332, 1073, 334, 335, 336, 0, 337, 338, 0, 0, 1074, 340, 341, 0, 0, 342, - 343, 344, 1266, 346, 1267, 792, 349, 350, 351, 352, + 343, 344, 345, 346, 347, 792, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 793, 1268, 364, 365, 366, 367, 368, + 0, 360, 361, 793, 363, 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, 1075, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 1269, 400, 401, 402, 1076, 404, + 396, 0, 397, 398, 399, 400, 401, 402, 1076, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 1270, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 0, 1271, 427, 428, 1077, 430, 0, + 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, + 423, 424, 425, 0, 426, 427, 428, 1077, 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 1272, 444, 795, 0, 0, 446, 447, 0, - 448, 1273, 450, 451, 452, 453, 454, 0, 455, 1078, - 1079, 0, 0, 458, 459, 796, 461, 797, 1080, 463, - 464, 1274, 466, 467, 468, 469, 470, 0, 0, 471, + 441, 442, 443, 444, 795, 0, 0, 446, 447, 0, + 448, 449, 450, 451, 452, 453, 454, 0, 455, 1078, + 1079, 0, 1426, 458, 459, 796, 461, 797, 1080, 463, + 464, 798, 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 483, 1081, 2277, 485, 1276, 487, 488, + 480, 481, 482, 483, 1081, 0, 485, 486, 487, 488, 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 1082, 0, 0, 0, 0, 0, 0, 1083, 1084, 1085, 0, 0, 0, 0, 1086, 0, 1087, 0, 0, - 0, 0, 1088, 1089, 0, 1090, 1091, 1241, 1042, 815, + 0, 0, 1088, 1089, 0, 1090, 1091, 1242, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, 0, 0, - 0, 1243, 0, 1048, 0, 0, 1244, 129, 130, 0, - 131, 132, 133, 1245, 135, 136, 137, 138, 1049, 1246, + 0, 1244, 0, 1048, 0, 0, 1245, 129, 130, 0, + 131, 132, 133, 1246, 135, 136, 137, 138, 1049, 1247, 1050, 1051, 0, 143, 144, 145, 146, 147, 148, 1052, 785, 149, 150, 151, 152, 1053, 1054, 155, 0, 156, - 157, 158, 159, 786, 0, 1247, 0, 1248, 163, 164, - 165, 166, 167, 1249, 169, 170, 171, 0, 172, 173, - 174, 175, 176, 177, 0, 1250, 179, 180, 181, 182, + 157, 158, 159, 786, 0, 1248, 0, 1249, 163, 164, + 165, 166, 167, 1250, 169, 170, 171, 0, 172, 173, + 174, 175, 176, 177, 0, 1251, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 1056, 190, 191, 1057, 193, 1058, 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, 205, 206, 1059, 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, 1060, 221, 222, 223, 224, 225, 788, 1061, 227, 0, 228, 229, 1062, 231, - 0, 232, 0, 233, 1252, 0, 1253, 236, 237, 1254, - 1255, 240, 0, 241, 0, 1063, 1064, 244, 245, 0, - 246, 247, 248, 249, 250, 251, 252, 1256, 254, 255, + 0, 232, 0, 233, 1253, 0, 1254, 236, 237, 1255, + 1256, 240, 0, 241, 0, 1063, 1064, 244, 245, 0, + 246, 247, 248, 249, 250, 251, 252, 1257, 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, 263, 264, - 0, 265, 1257, 267, 268, 269, 270, 271, 272, 1065, - 1066, 0, 1067, 0, 276, 1258, 1259, 279, 1260, 281, - 282, 283, 284, 285, 286, 0, 0, 287, 1261, 289, - 1262, 0, 291, 292, 293, 294, 295, 296, 297, 298, - 1263, 300, 301, 302, 303, 304, 305, 306, 307, 308, + 0, 265, 1258, 267, 268, 269, 270, 271, 272, 1065, + 1066, 0, 1067, 0, 276, 1259, 1260, 279, 1261, 281, + 282, 283, 284, 285, 286, 0, 0, 287, 1262, 289, + 1263, 0, 291, 292, 293, 294, 295, 296, 297, 298, + 1264, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, - 319, 1069, 1264, 1070, 323, 324, 325, 326, 1071, 327, - 328, 1265, 330, 1072, 790, 332, 1073, 334, 335, 336, + 319, 1069, 1265, 1070, 323, 324, 325, 326, 1071, 327, + 328, 1266, 330, 1072, 790, 332, 1073, 334, 335, 336, 0, 337, 338, 0, 0, 1074, 340, 341, 0, 0, - 342, 343, 344, 1266, 346, 1267, 792, 349, 350, 351, + 342, 343, 344, 1267, 346, 1268, 792, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 0, 0, - 0, 0, 360, 361, 793, 1268, 364, 365, 366, 367, + 0, 0, 360, 361, 793, 1269, 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, 1075, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, - 395, 396, 0, 397, 398, 1269, 400, 401, 402, 1076, + 395, 396, 0, 397, 398, 1270, 400, 401, 402, 1076, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, - 414, 415, 416, 0, 1270, 417, 418, 419, 420, 421, - 422, 423, 424, 425, 0, 1271, 427, 428, 1077, 430, + 414, 415, 416, 0, 1271, 417, 418, 419, 420, 421, + 422, 423, 424, 425, 0, 1272, 427, 428, 1077, 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, 439, - 440, 441, 442, 1272, 444, 795, 0, 0, 446, 447, - 0, 448, 1273, 450, 451, 452, 453, 454, 0, 455, + 440, 441, 442, 1273, 444, 795, 0, 0, 446, 447, + 0, 448, 1274, 450, 451, 452, 453, 454, 0, 455, 1078, 1079, 0, 0, 458, 459, 796, 461, 797, 1080, - 463, 464, 1274, 466, 467, 468, 469, 470, 0, 0, + 463, 464, 1275, 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, 0, 478, - 479, 480, 481, 482, 483, 1081, 0, 485, 1276, 487, + 479, 480, 481, 482, 483, 1081, 2278, 485, 1277, 487, 488, 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 1082, 0, 0, 0, 0, 0, 0, 1083, - 1084, 1085, 0, 0, 0, 0, 1086, 0, 1087, 2334, - 0, 0, 0, 1088, 1089, 0, 1090, 1091, 116, 1042, + 1084, 1085, 0, 0, 0, 0, 1086, 0, 1087, 0, + 0, 0, 0, 1088, 1089, 0, 1090, 1091, 1242, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, - 120, 121, 122, 123, 124, -1109, 125, 126, 127, 0, - 0, 0, 0, -1109, 1048, 0, 0, 128, 129, 130, - 0, 131, 132, 133, 134, 135, 136, 137, 138, 1049, - 140, 1050, 1051, 0, 143, 144, 145, 146, 147, 148, + 120, 121, 122, 123, 124, 0, 125, 126, 127, 0, + 0, 0, 1244, 0, 1048, 0, 0, 1245, 129, 130, + 0, 131, 132, 133, 1246, 135, 136, 137, 138, 1049, + 1247, 1050, 1051, 0, 143, 144, 145, 146, 147, 148, 1052, 785, 149, 150, 151, 152, 1053, 1054, 155, 0, - 156, 157, 158, 159, 786, 0, 787, 0, 1055, 163, - 164, 165, 166, 167, 168, 169, 170, 171, 0, 172, - 173, 174, 175, 176, 177, 0, 178, 179, 180, 181, + 156, 157, 158, 159, 786, 0, 1248, 0, 1249, 163, + 164, 165, 166, 167, 1250, 169, 170, 171, 0, 172, + 173, 174, 175, 176, 177, 0, 1251, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 1056, 190, 191, 1057, 193, 1058, 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, 205, 206, 1059, 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, 1060, 221, 222, 223, 224, 225, 788, 1061, 227, 0, 228, 229, 1062, - 231, 0, 232, 0, 233, 234, 0, 235, 236, 237, - 238, 239, 240, 0, 241, 0, 1063, 1064, 244, 245, - 0, 246, 247, 248, 249, 250, 251, 252, 253, 254, + 231, 0, 232, 0, 233, 1253, 0, 1254, 236, 237, + 1255, 1256, 240, 0, 241, 0, 1063, 1064, 244, 245, + 0, 246, 247, 248, 249, 250, 251, 252, 1257, 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, 263, - 264, 0, 265, 266, 267, 268, 269, 270, 271, 272, - 1065, 1066, 0, 1067, 0, 276, 277, 278, 279, 280, - 281, 282, 283, 284, 285, 286, 0, 0, 287, 288, - 289, 290, 0, 291, 292, 293, 294, 295, 296, 297, - 298, 1068, 300, 301, 302, 303, 304, 305, 306, 307, + 264, 0, 265, 1258, 267, 268, 269, 270, 271, 272, + 1065, 1066, 0, 1067, 0, 276, 1259, 1260, 279, 1261, + 281, 282, 283, 284, 285, 286, 0, 0, 287, 1262, + 289, 1263, 0, 291, 292, 293, 294, 295, 296, 297, + 298, 1264, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, - 318, 319, 1069, 321, 1070, 323, 324, 325, 326, 1071, - 327, 328, 329, 330, 1072, 790, 332, 1073, 334, 335, + 318, 319, 1069, 1265, 1070, 323, 324, 325, 326, 1071, + 327, 328, 1266, 330, 1072, 790, 332, 1073, 334, 335, 336, 0, 337, 338, 0, 0, 1074, 340, 341, 0, - 0, 342, 343, 344, 345, 346, 347, 792, 349, 350, + 0, 342, 343, 344, 1267, 346, 1268, 792, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 0, - 0, 0, 0, 360, 361, 793, 363, 364, 365, 366, + 0, 0, 0, 360, 361, 793, 1269, 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, 1075, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, - 394, 395, 396, 0, 397, 398, 399, 400, 401, 402, + 394, 395, 396, 0, 397, 398, 1270, 400, 401, 402, 1076, 404, 405, 406, 407, 408, 409, 410, 411, 412, - 413, 414, 415, 416, 0, 0, 417, 418, 419, 420, - 421, 422, 423, 424, 425, 0, 426, 427, 428, 1077, - 430, -1109, 431, 432, 433, 434, 435, 436, 437, 438, - 439, 440, 441, 442, 443, 444, 795, 0, 0, 446, - 447, 0, 448, 449, 450, 451, 452, 453, 454, 0, + 413, 414, 415, 416, 0, 1271, 417, 418, 419, 420, + 421, 422, 423, 424, 425, 0, 1272, 427, 428, 1077, + 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, + 439, 440, 441, 442, 1273, 444, 795, 0, 0, 446, + 447, 0, 448, 1274, 450, 451, 452, 453, 454, 0, 455, 1078, 1079, 0, 0, 458, 459, 796, 461, 797, - 1080, 463, 464, 798, 466, 467, 468, 469, 470, 0, + 1080, 463, 464, 1275, 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, 0, - 478, 479, 480, 481, 482, 483, 1081, 0, 485, 486, + 478, 479, 480, 481, 482, 483, 1081, 0, 485, 1277, 487, 488, 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 1082, 0, 0, 0, 0, 0, 0, 1083, 1084, 1085, 0, 0, 0, 0, 1086, 0, 1087, - 0, 0, 0, 0, 1088, 1089, 0, 1090, 1091, 1241, + 2335, 0, 0, 0, 1088, 1089, 0, 1090, 1091, 116, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, - 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, - 0, 0, 0, 1243, 0, 1048, 0, 0, 1244, 129, - 130, 0, 131, 132, 133, 1245, 135, 136, 137, 138, - 1049, 1246, 1050, 1051, 0, 143, 144, 145, 146, 147, + 119, 120, 121, 122, 123, 124, -1111, 125, 126, 127, + 0, 0, 0, 0, -1111, 1048, 0, 0, 128, 129, + 130, 0, 131, 132, 133, 134, 135, 136, 137, 138, + 1049, 140, 1050, 1051, 0, 143, 144, 145, 146, 147, 148, 1052, 785, 149, 150, 151, 152, 1053, 1054, 155, - 0, 156, 157, 158, 159, 786, 0, 1247, 0, 1248, - 163, 164, 165, 166, 167, 1249, 169, 170, 171, 0, - 172, 173, 174, 175, 176, 177, 0, 1250, 179, 180, + 0, 156, 157, 158, 159, 786, 0, 787, 0, 1055, + 163, 164, 165, 166, 167, 168, 169, 170, 171, 0, + 172, 173, 174, 175, 176, 177, 0, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 1056, 190, 191, 1057, 193, 1058, 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, 205, 206, 1059, 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, 1060, 221, 222, 223, 224, 225, 788, 1061, 227, 0, 228, 229, - 1062, 231, 0, 232, 0, 233, 1252, 0, 1253, 236, - 237, 1254, 1255, 240, 0, 241, 0, 1063, 1064, 244, - 245, 0, 246, 247, 248, 249, 250, 251, 252, 1256, + 1062, 231, 0, 232, 0, 233, 234, 0, 235, 236, + 237, 238, 239, 240, 0, 241, 0, 1063, 1064, 244, + 245, 0, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, - 263, 264, 0, 265, 1257, 267, 268, 269, 270, 271, - 272, 1065, 1066, 0, 1067, 0, 276, 1258, 1259, 279, - 1260, 281, 282, 283, 284, 285, 286, 0, 0, 287, - 1261, 289, 1262, 0, 291, 292, 293, 294, 295, 296, - 297, 298, 1263, 300, 301, 302, 303, 304, 305, 306, + 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, + 272, 1065, 1066, 0, 1067, 0, 276, 277, 278, 279, + 280, 281, 282, 283, 284, 285, 286, 0, 0, 287, + 288, 289, 290, 0, 291, 292, 293, 294, 295, 296, + 297, 298, 1068, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, - 317, 318, 319, 1069, 1264, 1070, 323, 324, 325, 326, - 1071, 327, 328, 1265, 330, 1072, 790, 332, 1073, 334, + 317, 318, 319, 1069, 321, 1070, 323, 324, 325, 326, + 1071, 327, 328, 329, 330, 1072, 790, 332, 1073, 334, 335, 336, 0, 337, 338, 0, 0, 1074, 340, 341, - 0, 0, 342, 343, 344, 1266, 346, 1267, 792, 349, + 0, 0, 342, 343, 344, 345, 346, 347, 792, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, - 0, 0, 0, 0, 360, 361, 793, 1268, 364, 365, + 0, 0, 0, 0, 360, 361, 793, 363, 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, 1075, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, - 393, 394, 395, 396, 0, 397, 398, 1269, 400, 401, + 393, 394, 395, 396, 0, 397, 398, 399, 400, 401, 402, 1076, 404, 405, 406, 407, 408, 409, 410, 411, - 412, 413, 414, 415, 416, 0, 1270, 417, 418, 419, - 420, 421, 422, 423, 424, 425, 0, 1271, 427, 428, - 1077, 430, 0, 431, 432, 433, 434, 435, 436, 437, - 438, 439, 440, 441, 442, 1272, 444, 795, 0, 0, - 446, 447, 0, 448, 1273, 450, 451, 452, 453, 454, + 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 0, 426, 427, 428, + 1077, 430, -1111, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 443, 444, 795, 0, 0, + 446, 447, 0, 448, 449, 450, 451, 452, 453, 454, 0, 455, 1078, 1079, 0, 0, 458, 459, 796, 461, - 797, 1080, 463, 464, 1274, 466, 467, 468, 469, 470, + 797, 1080, 463, 464, 798, 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, 482, 483, 1081, 0, 485, - 1276, 487, 488, 489, 490, 491, 492, 493, 0, 0, + 486, 487, 488, 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 1082, 0, 0, 0, 0, 0, 0, 1083, 1084, 1085, 0, 0, 0, 0, 1086, 0, - 1087, 3068, 0, 0, 0, 1088, 1089, 0, 1090, 1091, - 1241, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, + 1087, 0, 0, 0, 0, 1088, 1089, 0, 1090, 1091, + 1242, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, - 127, 0, 0, 0, 1243, 0, 1048, 0, 0, 1244, - 129, 130, 0, 131, 132, 133, 1245, 135, 136, 137, - 138, 1049, 1246, 1050, 1051, 0, 143, 144, 145, 146, + 127, 0, 0, 0, 1244, 0, 1048, 0, 0, 1245, + 129, 130, 0, 131, 132, 133, 1246, 135, 136, 137, + 138, 1049, 1247, 1050, 1051, 0, 143, 144, 145, 146, 147, 148, 1052, 785, 149, 150, 151, 152, 1053, 1054, - 155, 0, 156, 157, 158, 159, 786, 0, 1247, 0, - 1248, 163, 164, 165, 166, 167, 1249, 169, 170, 171, - 0, 172, 173, 174, 175, 176, 177, 0, 1250, 179, + 155, 0, 156, 157, 158, 159, 786, 0, 1248, 0, + 1249, 163, 164, 165, 166, 167, 1250, 169, 170, 171, + 0, 172, 173, 174, 175, 176, 177, 0, 1251, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 1056, 190, 191, 1057, 193, 1058, 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, 205, 206, 1059, 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, 1060, 221, 222, 223, 224, 225, 788, 1061, 227, 0, 228, - 229, 1062, 231, 0, 232, 0, 233, 1252, 0, 1253, - 236, 237, 1254, 1255, 240, 0, 241, 0, 1063, 1064, + 229, 1062, 231, 0, 232, 0, 233, 1253, 0, 1254, + 236, 237, 1255, 1256, 240, 0, 241, 0, 1063, 1064, 244, 245, 0, 246, 247, 248, 249, 250, 251, 252, - 1256, 254, 255, 256, 257, 0, 258, 259, 260, 261, - 262, 263, 264, 0, 265, 1257, 267, 268, 269, 270, - 271, 272, 1065, 1066, 0, 1067, 0, 276, 1258, 1259, - 279, 1260, 281, 282, 283, 284, 285, 286, 0, 0, - 287, 1261, 289, 1262, 0, 291, 292, 293, 294, 295, - 296, 297, 298, 1263, 300, 301, 302, 303, 304, 305, + 1257, 254, 255, 256, 257, 0, 258, 259, 260, 261, + 262, 263, 264, 0, 265, 1258, 267, 268, 269, 270, + 271, 272, 1065, 1066, 0, 1067, 0, 276, 1259, 1260, + 279, 1261, 281, 282, 283, 284, 285, 286, 0, 0, + 287, 1262, 289, 1263, 0, 291, 292, 293, 294, 295, + 296, 297, 298, 1264, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, - 316, 317, 318, 319, 1069, 1264, 1070, 323, 324, 325, - 326, 1071, 327, 328, 1265, 330, 1072, 790, 332, 1073, + 316, 317, 318, 319, 1069, 1265, 1070, 323, 324, 325, + 326, 1071, 327, 328, 1266, 330, 1072, 790, 332, 1073, 334, 335, 336, 0, 337, 338, 0, 0, 1074, 340, - 341, 0, 0, 342, 343, 344, 1266, 346, 1267, 792, + 341, 0, 0, 342, 343, 344, 1267, 346, 1268, 792, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, - 359, 0, 0, 0, 0, 360, 361, 793, 1268, 364, + 359, 0, 0, 0, 0, 360, 361, 793, 1269, 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, 1075, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, - 392, 393, 394, 395, 396, 0, 397, 398, 1269, 400, + 392, 393, 394, 395, 396, 0, 397, 398, 1270, 400, 401, 402, 1076, 404, 405, 406, 407, 408, 409, 410, - 411, 412, 413, 414, 415, 416, 0, 1270, 417, 418, - 419, 420, 421, 422, 423, 424, 425, 0, 1271, 427, + 411, 412, 413, 414, 415, 416, 0, 1271, 417, 418, + 419, 420, 421, 422, 423, 424, 425, 0, 1272, 427, 428, 1077, 430, 0, 431, 432, 433, 434, 435, 436, - 437, 438, 439, 440, 441, 442, 1272, 444, 795, 0, - 0, 446, 447, 0, 448, 1273, 450, 451, 452, 453, + 437, 438, 439, 440, 441, 442, 1273, 444, 795, 0, + 0, 446, 447, 0, 448, 1274, 450, 451, 452, 453, 454, 0, 455, 1078, 1079, 0, 0, 458, 459, 796, - 461, 797, 1080, 463, 464, 1274, 466, 467, 468, 469, + 461, 797, 1080, 463, 464, 1275, 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, 482, 483, 1081, 0, - 485, 1276, 487, 488, 489, 490, 491, 492, 493, 0, + 485, 1277, 487, 488, 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 1082, 0, 0, 0, 0, 0, 0, 1083, 1084, 1085, 0, 0, 0, 0, 1086, - 0, 1087, 0, 0, 0, 0, 1088, 1089, 0, 1090, - 1091, 116, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, + 0, 1087, 3071, 0, 0, 0, 1088, 1089, 0, 1090, + 1091, 1242, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 118, 119, 120, 121, 122, 123, 124, 1688, 125, - 126, 127, 0, 0, 0, 0, 0, 1048, 0, 0, - 128, 129, 130, 0, 131, 132, 133, 134, 135, 136, - 137, 138, 1049, 140, 1050, 1051, 0, 143, 144, 145, + 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, + 126, 127, 0, 0, 0, 1244, 0, 1048, 0, 0, + 1245, 129, 130, 0, 131, 132, 133, 1246, 135, 136, + 137, 138, 1049, 1247, 1050, 1051, 0, 143, 144, 145, 146, 147, 148, 1052, 785, 149, 150, 151, 152, 1053, - 1054, 155, 0, 156, 157, 158, 159, 786, 0, 787, - 0, 1055, 163, 164, 165, 166, 167, 168, 169, 170, - 171, 0, 172, 173, 174, 175, 176, 177, 0, 178, + 1054, 155, 0, 156, 157, 158, 159, 786, 0, 1248, + 0, 1249, 163, 164, 165, 166, 167, 1250, 169, 170, + 171, 0, 172, 173, 174, 175, 176, 177, 0, 1251, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 1056, 190, 191, 1057, 193, 1058, 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, 205, 206, 1059, 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, 1060, 221, 222, 223, 224, 225, 788, 1061, 227, 0, - 228, 229, 1062, 231, 0, 232, 0, 233, 234, 0, - 235, 236, 237, 238, 239, 240, 0, 241, 0, 1063, + 228, 229, 1062, 231, 0, 232, 0, 233, 1253, 0, + 1254, 236, 237, 1255, 1256, 240, 0, 241, 0, 1063, 1064, 244, 245, 0, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 0, 258, 259, 260, - 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, - 270, 271, 272, 1065, 1066, 0, 1067, 0, 276, 277, - 278, 279, 280, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 288, 289, 290, 0, 291, 292, 293, 294, - 295, 296, 297, 298, 1068, 300, 301, 302, 303, 304, + 252, 1257, 254, 255, 256, 257, 0, 258, 259, 260, + 261, 262, 263, 264, 0, 265, 1258, 267, 268, 269, + 270, 271, 272, 1065, 1066, 0, 1067, 0, 276, 1259, + 1260, 279, 1261, 281, 282, 283, 284, 285, 286, 0, + 0, 287, 1262, 289, 1263, 0, 291, 292, 293, 294, + 295, 296, 297, 298, 1264, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 1069, 321, 1070, 323, 324, - 325, 326, 1071, 327, 328, 329, 330, 1072, 790, 332, + 315, 316, 317, 318, 319, 1069, 1265, 1070, 323, 324, + 325, 326, 1071, 327, 328, 1266, 330, 1072, 790, 332, 1073, 334, 335, 336, 0, 337, 338, 0, 0, 1074, - 340, 341, 0, 0, 342, 343, 344, 345, 346, 347, + 340, 341, 0, 0, 342, 343, 344, 1267, 346, 1268, 792, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 0, 0, 0, 0, 360, 361, 793, 363, + 358, 359, 0, 0, 0, 0, 360, 361, 793, 1269, 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, 1075, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 397, 398, 399, + 391, 392, 393, 394, 395, 396, 0, 397, 398, 1270, 400, 401, 402, 1076, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 0, 426, + 410, 411, 412, 413, 414, 415, 416, 0, 1271, 417, + 418, 419, 420, 421, 422, 423, 424, 425, 0, 1272, 427, 428, 1077, 430, 0, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 443, 444, 795, - 0, 0, 446, 447, 0, 448, 449, 450, 451, 452, + 436, 437, 438, 439, 440, 441, 442, 1273, 444, 795, + 0, 0, 446, 447, 0, 448, 1274, 450, 451, 452, 453, 454, 0, 455, 1078, 1079, 0, 0, 458, 459, - 796, 461, 797, 1080, 463, 464, 798, 466, 467, 468, + 796, 461, 797, 1080, 463, 464, 1275, 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, 482, 483, 1081, - 0, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 0, 485, 1277, 487, 488, 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 1082, 0, 0, 0, 0, 0, 0, 1083, 1084, 1085, 0, 0, 0, 0, 1086, 0, 1087, 0, 0, 0, 0, 1088, 1089, 0, - 1090, 1091, 116, 1703, 815, 1043, 1044, 1045, 1704, 1047, + 1090, 1091, 116, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 117, 118, 119, 120, 121, 122, 123, 124, 1705, + 0, 117, 118, 119, 120, 121, 122, 123, 124, 1689, 125, 126, 127, 0, 0, 0, 0, 0, 1048, 0, 0, 128, 129, 130, 0, 131, 132, 133, 134, 135, 136, 137, 138, 1049, 140, 1050, 1051, 0, 143, 144, @@ -5969,10 +5928,10 @@ static const yytype_int16 yytable[] = 509, 510, 511, 512, 513, 514, 515, 1082, 0, 0, 0, 0, 0, 0, 1083, 1084, 1085, 0, 0, 0, 0, 1086, 0, 1087, 0, 0, 0, 0, 1088, 1089, - 0, 1090, 1091, 116, 1042, 815, 1043, 1044, 1045, 1046, + 0, 1090, 1091, 116, 1704, 815, 1043, 1044, 1045, 1705, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, - 0, 125, 126, 127, 0, 0, 0, 0, 0, 1048, + 1706, 125, 126, 127, 0, 0, 0, 0, 0, 1048, 0, 0, 128, 129, 130, 0, 131, 132, 133, 134, 135, 136, 137, 138, 1049, 140, 1050, 1051, 0, 143, 144, 145, 146, 147, 148, 1052, 785, 149, 150, 151, @@ -5986,7 +5945,7 @@ static const yytype_int16 yytable[] = 210, 211, 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, 1060, 221, 222, 223, 224, 225, 788, 1061, 227, 0, 228, 229, 1062, 231, 0, 232, 0, 233, - 234, 1423, 235, 236, 237, 238, 239, 240, 0, 241, + 234, 0, 235, 236, 237, 238, 239, 240, 0, 241, 0, 1063, 1064, 244, 245, 0, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, 265, 266, 267, @@ -6038,7 +5997,7 @@ static const yytype_int16 yytable[] = 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, 1060, 221, 222, 223, 224, 225, 788, 1061, 227, 0, 228, 229, 1062, 231, 0, 232, 0, - 233, 234, 0, 235, 236, 237, 238, 239, 240, 0, + 233, 234, 1424, 235, 236, 237, 238, 239, 240, 0, 241, 0, 1063, 1064, 244, 245, 0, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, 265, 266, @@ -6072,7 +6031,7 @@ static const yytype_int16 yytable[] = 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 1082, 0, 0, 0, 0, 0, 0, 1083, 1084, 1085, 0, - 0, 0, 0, 1086, 0, 1087, 2051, 0, 0, 0, + 0, 0, 0, 1086, 0, 1087, 0, 0, 0, 0, 1088, 1089, 0, 1090, 1091, 116, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, 122, @@ -6114,7 +6073,7 @@ static const yytype_int16 yytable[] = 416, 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, 425, 0, 426, 427, 428, 1077, 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, - 442, 443, 444, 795, 0, 0, 446, 447, 2678, 448, + 442, 443, 444, 795, 0, 0, 446, 447, 0, 448, 449, 450, 451, 452, 453, 454, 0, 455, 1078, 1079, 0, 0, 458, 459, 796, 461, 797, 1080, 463, 464, 798, 466, 467, 468, 469, 470, 0, 0, 471, 472, @@ -6124,7 +6083,7 @@ static const yytype_int16 yytable[] = 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 1082, 0, 0, 0, 0, 0, 0, 1083, 1084, 1085, - 0, 0, 0, 0, 1086, 0, 1087, 0, 0, 0, + 0, 0, 0, 0, 1086, 0, 1087, 2052, 0, 0, 0, 1088, 1089, 0, 1090, 1091, 116, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, @@ -6166,13 +6125,13 @@ static const yytype_int16 yytable[] = 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, 425, 0, 426, 427, 428, 1077, 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 443, 444, 795, 0, 0, 446, 447, 0, + 441, 442, 443, 444, 795, 0, 0, 446, 447, 2681, 448, 449, 450, 451, 452, 453, 454, 0, 455, 1078, 1079, 0, 0, 458, 459, 796, 461, 797, 1080, 463, 464, 798, 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, 482, 483, 1081, 0, 485, 486, 487, 488, - 489, 490, 491, 492, 493, 0, 2797, 494, 0, 0, + 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 1082, 0, 0, 0, 0, 0, 0, 1083, 1084, @@ -6180,14 +6139,14 @@ static const yytype_int16 yytable[] = 0, 0, 1088, 1089, 0, 1090, 1091, 116, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, - 121, 122, 123, 124, 3012, 125, 126, 127, 0, 0, + 121, 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, 1048, 0, 0, 128, 129, 130, 0, 131, 132, 133, 134, 135, 136, 137, 138, 1049, 140, 1050, 1051, 0, 143, 144, 145, 146, 147, 148, 1052, 785, 149, 150, 151, 152, 1053, 1054, 155, 0, 156, 157, 158, 159, 786, 0, 787, 0, 1055, 163, 164, 165, 166, 167, 168, 169, 170, 171, 0, 172, 173, - 174, 175, 176, 177, 0, 178, 179, 3013, 181, 182, + 174, 175, 176, 177, 0, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 1056, 190, 191, 1057, 193, 1058, 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, 205, 206, @@ -6195,7 +6154,7 @@ static const yytype_int16 yytable[] = 215, 0, 216, 217, 218, 219, 1060, 221, 222, 223, 224, 225, 788, 1061, 227, 0, 228, 229, 1062, 231, 0, 232, 0, 233, 234, 0, 235, 236, 237, 238, - 239, 240, 0, 241, 0, 3014, 1064, 244, 245, 0, + 239, 240, 0, 241, 0, 1063, 1064, 244, 245, 0, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, 272, 1065, @@ -6213,7 +6172,7 @@ static const yytype_int16 yytable[] = 368, 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, 1075, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, - 395, 396, 0, 397, 398, 399, 400, 401, 3015, 1076, + 395, 396, 0, 397, 398, 399, 400, 401, 402, 1076, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, 425, 0, 426, 427, 428, 1077, 430, @@ -6224,22 +6183,22 @@ static const yytype_int16 yytable[] = 463, 464, 798, 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, 482, 483, 1081, 0, 485, 486, 487, - 488, 489, 490, 491, 492, 493, 0, 0, 494, 0, + 488, 489, 490, 491, 492, 493, 0, 2800, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 1082, 0, 0, 0, 0, 0, 0, 1083, - 1084, 1085, 0, 0, 0, 0, 1086, 0, 3016, 0, + 1084, 1085, 0, 0, 0, 0, 1086, 0, 1087, 0, 0, 0, 0, 1088, 1089, 0, 1090, 1091, 116, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, - 120, 121, 122, 123, 124, 0, 125, 126, 127, 0, + 120, 121, 122, 123, 124, 3015, 125, 126, 127, 0, 0, 0, 0, 0, 1048, 0, 0, 128, 129, 130, 0, 131, 132, 133, 134, 135, 136, 137, 138, 1049, 140, 1050, 1051, 0, 143, 144, 145, 146, 147, 148, 1052, 785, 149, 150, 151, 152, 1053, 1054, 155, 0, 156, 157, 158, 159, 786, 0, 787, 0, 1055, 163, 164, 165, 166, 167, 168, 169, 170, 171, 0, 172, - 173, 174, 175, 176, 177, 0, 178, 179, 180, 181, + 173, 174, 175, 176, 177, 0, 178, 179, 3016, 181, 182, 183, 184, 185, 186, 187, 188, 1056, 190, 191, 1057, 193, 1058, 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, 205, @@ -6247,7 +6206,7 @@ static const yytype_int16 yytable[] = 214, 215, 0, 216, 217, 218, 219, 1060, 221, 222, 223, 224, 225, 788, 1061, 227, 0, 228, 229, 1062, 231, 0, 232, 0, 233, 234, 0, 235, 236, 237, - 238, 239, 240, 0, 241, 0, 1063, 1064, 244, 245, + 238, 239, 240, 0, 241, 0, 3017, 1064, 244, 245, 0, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, 272, @@ -6265,13 +6224,13 @@ static const yytype_int16 yytable[] = 367, 368, 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, 1075, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, - 394, 395, 396, 0, 397, 398, 399, 400, 401, 402, + 394, 395, 396, 0, 397, 398, 399, 400, 401, 3018, 1076, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, 425, 0, 426, 427, 428, 1077, 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 795, 0, 0, 446, - 447, 3164, 448, 449, 450, 451, 452, 453, 454, 0, + 447, 0, 448, 449, 450, 451, 452, 453, 454, 0, 455, 1078, 1079, 0, 0, 458, 459, 796, 461, 797, 1080, 463, 464, 798, 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, 0, @@ -6280,7 +6239,7 @@ static const yytype_int16 yytable[] = 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 1082, 0, 0, 0, 0, 0, 0, - 1083, 1084, 1085, 0, 0, 0, 0, 1086, 0, 1087, + 1083, 1084, 1085, 0, 0, 0, 0, 1086, 0, 3019, 0, 0, 0, 0, 1088, 1089, 0, 1090, 1091, 116, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, @@ -6323,7 +6282,7 @@ static const yytype_int16 yytable[] = 420, 421, 422, 423, 424, 425, 0, 426, 427, 428, 1077, 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 795, 0, 0, - 446, 447, 3302, 448, 449, 450, 451, 452, 453, 454, + 446, 447, 3167, 448, 449, 450, 451, 452, 453, 454, 0, 455, 1078, 1079, 0, 0, 458, 459, 796, 461, 797, 1080, 463, 464, 798, 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, @@ -6375,7 +6334,7 @@ static const yytype_int16 yytable[] = 419, 420, 421, 422, 423, 424, 425, 0, 426, 427, 428, 1077, 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 795, 0, - 0, 446, 447, 0, 448, 449, 450, 451, 452, 453, + 0, 446, 447, 3305, 448, 449, 450, 451, 452, 453, 454, 0, 455, 1078, 1079, 0, 0, 458, 459, 796, 461, 797, 1080, 463, 464, 798, 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, 476, @@ -6436,9 +6395,9 @@ static const yytype_int16 yytable[] = 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 1082, 0, 0, 0, - 0, 0, 0, 1708, 1709, 1085, 0, 0, 0, 0, + 0, 0, 0, 1083, 1084, 1085, 0, 0, 0, 0, 1086, 0, 1087, 0, 0, 0, 0, 1088, 1089, 0, - 1090, 1091, 116, 2188, 815, 1043, 1044, 1045, 1046, 1047, + 1090, 1091, 116, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, 1048, 0, @@ -6488,9 +6447,9 @@ static const yytype_int16 yytable[] = 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 1082, 0, 0, - 0, 0, 0, 0, 1083, 1084, 1085, 0, 0, 0, + 0, 0, 0, 0, 1709, 1710, 1085, 0, 0, 0, 0, 1086, 0, 1087, 0, 0, 0, 0, 1088, 1089, - 0, 1090, 1091, 116, 1042, 815, 1043, 1044, 1045, 1046, + 0, 1090, 1091, 116, 2189, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, 1048, @@ -6540,7 +6499,7 @@ static const yytype_int16 yytable[] = 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 1082, 0, - 0, 0, 0, 0, 0, 1083, 2271, 1085, 0, 0, + 0, 0, 0, 0, 0, 1083, 1084, 1085, 0, 0, 0, 0, 1086, 0, 1087, 0, 0, 0, 0, 1088, 1089, 0, 1090, 1091, 116, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6592,8 +6551,8 @@ static const yytype_int16 yytable[] = 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 1082, - 0, 0, 0, 0, 0, 0, 1083, 1084, 1085, 0, - 0, 0, 0, 1086, 0, 2483, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1083, 2272, 1085, 0, + 0, 0, 0, 1086, 0, 1087, 0, 0, 0, 0, 1088, 1089, 0, 1090, 1091, 116, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, 122, @@ -6644,8 +6603,8 @@ static const yytype_int16 yytable[] = 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, - 1082, 0, 0, 0, 0, 0, 0, 1083, 3066, 1085, - 0, 0, 0, 0, 1086, 0, 1087, 0, 0, 0, + 1082, 0, 0, 0, 0, 0, 0, 1083, 1084, 1085, + 0, 0, 0, 0, 1086, 0, 2486, 0, 0, 0, 0, 1088, 1089, 0, 1090, 1091, 116, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, @@ -6656,7 +6615,7 @@ static const yytype_int16 yytable[] = 149, 150, 151, 152, 1053, 1054, 155, 0, 156, 157, 158, 159, 786, 0, 787, 0, 1055, 163, 164, 165, 166, 167, 168, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 178, 179, 3013, 181, 182, 183, + 175, 176, 177, 0, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 1056, 190, 191, 1057, 193, 1058, 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, 205, 206, 1059, @@ -6664,7 +6623,7 @@ static const yytype_int16 yytable[] = 0, 216, 217, 218, 219, 1060, 221, 222, 223, 224, 225, 788, 1061, 227, 0, 228, 229, 1062, 231, 0, 232, 0, 233, 234, 0, 235, 236, 237, 238, 239, - 240, 0, 241, 0, 3014, 1064, 244, 245, 0, 246, + 240, 0, 241, 0, 1063, 1064, 244, 245, 0, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, 272, 1065, 1066, @@ -6682,7 +6641,7 @@ static const yytype_int16 yytable[] = 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, 1075, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 399, 400, 401, 3015, 1076, 404, + 396, 0, 397, 398, 399, 400, 401, 402, 1076, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, 425, 0, 426, 427, 428, 1077, 430, 0, @@ -6696,19 +6655,19 @@ static const yytype_int16 yytable[] = 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, - 515, 1082, 0, 0, 0, 0, 0, 0, 1083, 1084, - 1085, 0, 0, 0, 0, 1086, 0, 3016, 0, 0, + 515, 1082, 0, 0, 0, 0, 0, 0, 1083, 3069, + 1085, 0, 0, 0, 0, 1086, 0, 1087, 0, 0, 0, 0, 1088, 1089, 0, 1090, 1091, 116, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, 1048, 0, 0, 128, 129, 130, 0, - 131, 132, 133, 134, 135, 136, 137, 3474, 1049, 140, + 131, 132, 133, 134, 135, 136, 137, 138, 1049, 140, 1050, 1051, 0, 143, 144, 145, 146, 147, 148, 1052, 785, 149, 150, 151, 152, 1053, 1054, 155, 0, 156, 157, 158, 159, 786, 0, 787, 0, 1055, 163, 164, 165, 166, 167, 168, 169, 170, 171, 0, 172, 173, - 174, 175, 176, 177, 0, 178, 179, 180, 3475, 182, + 174, 175, 176, 177, 0, 178, 179, 3016, 181, 182, 183, 184, 185, 186, 187, 188, 1056, 190, 191, 1057, 193, 1058, 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, 205, 206, @@ -6716,7 +6675,7 @@ static const yytype_int16 yytable[] = 215, 0, 216, 217, 218, 219, 1060, 221, 222, 223, 224, 225, 788, 1061, 227, 0, 228, 229, 1062, 231, 0, 232, 0, 233, 234, 0, 235, 236, 237, 238, - 239, 240, 0, 241, 0, 1063, 1064, 244, 245, 0, + 239, 240, 0, 241, 0, 3017, 1064, 244, 245, 0, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, 272, 1065, @@ -6734,7 +6693,7 @@ static const yytype_int16 yytable[] = 368, 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, 1075, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, - 395, 396, 0, 397, 398, 399, 400, 401, 402, 1076, + 395, 396, 0, 397, 398, 399, 400, 401, 3018, 1076, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, 425, 0, 426, 427, 428, 1077, 430, @@ -6742,25 +6701,25 @@ static const yytype_int16 yytable[] = 440, 441, 442, 443, 444, 795, 0, 0, 446, 447, 0, 448, 449, 450, 451, 452, 453, 454, 0, 455, 1078, 1079, 0, 0, 458, 459, 796, 461, 797, 1080, - 463, 464, 798, 466, 467, 3476, 469, 470, 0, 0, + 463, 464, 798, 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, 482, 483, 1081, 0, 485, 486, 487, 488, 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 1082, 0, 0, 0, 0, 0, 0, 1083, - 1084, 1085, 0, 0, 0, 0, 1086, 0, 1087, 0, + 1084, 1085, 0, 0, 0, 0, 1086, 0, 3019, 0, 0, 0, 0, 1088, 1089, 0, 1090, 1091, 116, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, 1048, 0, 0, 128, 129, 130, - 0, 131, 132, 133, 134, 135, 136, 137, 138, 1049, + 0, 131, 132, 133, 134, 135, 136, 137, 3477, 1049, 140, 1050, 1051, 0, 143, 144, 145, 146, 147, 148, 1052, 785, 149, 150, 151, 152, 1053, 1054, 155, 0, 156, 157, 158, 159, 786, 0, 787, 0, 1055, 163, 164, 165, 166, 167, 168, 169, 170, 171, 0, 172, - 173, 174, 175, 176, 177, 0, 178, 179, 180, 3475, + 173, 174, 175, 176, 177, 0, 178, 179, 180, 3478, 182, 183, 184, 185, 186, 187, 188, 1056, 190, 191, 1057, 193, 1058, 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, 205, @@ -6794,7 +6753,7 @@ static const yytype_int16 yytable[] = 439, 440, 441, 442, 443, 444, 795, 0, 0, 446, 447, 0, 448, 449, 450, 451, 452, 453, 454, 0, 455, 1078, 1079, 0, 0, 458, 459, 796, 461, 797, - 1080, 463, 464, 798, 466, 467, 3476, 469, 470, 0, + 1080, 463, 464, 798, 466, 467, 3479, 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, 482, 483, 1081, 0, 485, 486, 487, 488, 489, 490, 491, 492, 493, 0, 0, 494, @@ -6810,11 +6769,11 @@ static const yytype_int16 yytable[] = 130, 0, 131, 132, 133, 134, 135, 136, 137, 138, 1049, 140, 1050, 1051, 0, 143, 144, 145, 146, 147, 148, 1052, 785, 149, 150, 151, 152, 1053, 1054, 155, - 0, 156, 157, 158, 159, 786, 0, 787, 0, 162, + 0, 156, 157, 158, 159, 786, 0, 787, 0, 1055, 163, 164, 165, 166, 167, 168, 169, 170, 171, 0, 172, 173, 174, 175, 176, 177, 0, 178, 179, 180, - 181, 182, 183, 184, 185, 186, 187, 188, 1056, 190, - 191, 1057, 193, 0, 194, 0, 195, 196, 197, 198, + 3478, 182, 183, 184, 185, 186, 187, 188, 1056, 190, + 191, 1057, 193, 1058, 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, 205, 206, 1059, 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, 1060, 221, @@ -6830,7 +6789,7 @@ static const yytype_int16 yytable[] = 297, 298, 1068, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 1069, 321, 1070, 323, 324, 325, 326, - 0, 327, 328, 329, 330, 1072, 790, 332, 1073, 334, + 1071, 327, 328, 329, 330, 1072, 790, 332, 1073, 334, 335, 336, 0, 337, 338, 0, 0, 1074, 340, 341, 0, 0, 342, 343, 344, 345, 346, 347, 792, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, @@ -6846,43 +6805,43 @@ static const yytype_int16 yytable[] = 438, 439, 440, 441, 442, 443, 444, 795, 0, 0, 446, 447, 0, 448, 449, 450, 451, 452, 453, 454, 0, 455, 1078, 1079, 0, 0, 458, 459, 796, 461, - 797, 1080, 463, 464, 798, 466, 467, 468, 469, 470, + 797, 1080, 463, 464, 798, 466, 467, 3479, 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, 482, 483, 1081, 0, 485, 486, 487, 488, 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, - 512, 513, 514, 515, 0, 0, 0, 0, 0, 0, - 0, 1409, 1410, 0, 0, 0, 0, 0, 1086, 0, + 512, 513, 514, 515, 1082, 0, 0, 0, 0, 0, + 0, 1083, 1084, 1085, 0, 0, 0, 0, 1086, 0, 1087, 0, 0, 0, 0, 1088, 1089, 0, 1090, 1091, 116, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, 1048, 0, 0, 128, 129, 130, 0, 131, 132, 133, 134, 135, 136, 137, - -2044, 1049, 140, 1050, 1051, 0, 143, 144, 145, 146, + 138, 1049, 140, 1050, 1051, 0, 143, 144, 145, 146, 147, 148, 1052, 785, 149, 150, 151, 152, 1053, 1054, 155, 0, 156, 157, 158, 159, 786, 0, 787, 0, - 1055, 163, 164, 165, 166, 167, 168, 169, 170, 171, + 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 0, 172, 173, 174, 175, 176, 177, 0, 178, 179, - 180, 3475, 182, 183, 184, 185, 186, 187, 188, 1056, - 190, 191, 1057, 193, 1058, 194, 0, 195, 196, 197, + 180, 181, 182, 183, 184, 185, 186, 187, 188, 1056, + 190, 191, 1057, 193, 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, 205, 206, 1059, 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, 1060, 221, 222, 223, 224, 225, 788, 1061, 227, 0, 228, 229, 1062, 231, 0, 232, 0, 233, 234, 0, 235, - 236, 237, 238, -2044, 240, 0, 241, 0, 1063, 1064, + 236, 237, 238, 239, 240, 0, 241, 0, 1063, 1064, 244, 245, 0, 246, 247, 248, 249, 250, 251, 252, - -2044, 254, 255, 256, 257, 0, 258, 259, 260, 261, + 253, 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, 270, - 271, 272, 1065, 1066, 0, 1067, 0, 276, 0, 0, + 271, 272, 1065, 1066, 0, 1067, 0, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 0, 0, - 287, 288, 289, -2044, 0, 291, 292, 293, 294, 295, + 287, 288, 289, 290, 0, 291, 292, 293, 294, 295, 296, 297, 298, 1068, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 1069, 321, 1070, 323, 324, 325, - 326, 0, 327, 328, 0, 330, 1072, 790, 332, 1073, + 326, 0, 327, 328, 329, 330, 1072, 790, 332, 1073, 334, 335, 336, 0, 337, 338, 0, 0, 1074, 340, 341, 0, 0, 342, 343, 344, 345, 346, 347, 792, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, @@ -6893,49 +6852,49 @@ static const yytype_int16 yytable[] = 392, 393, 394, 395, 396, 0, 397, 398, 399, 400, 401, 402, 1076, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, 418, - 419, 420, 421, 422, 423, 424, 425, 0, -2044, 427, + 419, 420, 421, 422, 423, 424, 425, 0, 426, 427, 428, 1077, 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 795, 0, 0, 446, 447, 0, 448, 449, 450, 451, 452, 453, 454, 0, 455, 1078, 1079, 0, 0, 458, 459, 796, - 461, 797, 1080, 463, 464, 798, 466, 467, 3476, 469, + 461, 797, 1080, 463, 464, 798, 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, 482, 483, 1081, 0, 485, 486, 487, 488, 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, - 511, 512, 513, 514, 515, -2044, 0, 0, 0, 0, - 0, 0, 1083, 1084, 1085, 0, 0, 0, 0, 1086, + 511, 512, 513, 514, 515, 0, 0, 0, 0, 0, + 0, 0, 1410, 1411, 0, 0, 0, 0, 0, 1086, 0, 1087, 0, 0, 0, 0, 1088, 1089, 0, 1090, - 1091, 116, 1042, 815, 1043, 1044, 0, 1046, 1047, 0, + 1091, 116, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, 1048, 0, 0, 128, 129, 130, 0, 131, 132, 133, 134, 135, 136, - 137, 138, 1049, 140, 1050, 1051, 0, 143, 144, 145, + 137, -2046, 1049, 140, 1050, 1051, 0, 143, 144, 145, 146, 147, 148, 1052, 785, 149, 150, 151, 152, 1053, 1054, 155, 0, 156, 157, 158, 159, 786, 0, 787, - 0, 162, 163, 164, 165, 166, 167, 168, 169, 170, + 0, 1055, 163, 164, 165, 166, 167, 168, 169, 170, 171, 0, 172, 173, 174, 175, 176, 177, 0, 178, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 1056, 190, 191, 1057, 193, 0, 194, 0, 195, 196, + 179, 180, 3478, 182, 183, 184, 185, 186, 187, 188, + 1056, 190, 191, 1057, 193, 1058, 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, 205, 206, 1059, 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, 1060, 221, 222, 223, 224, 225, 788, 1061, 227, 0, 228, 229, 1062, 231, 0, 232, 0, 233, 234, 0, - 235, 236, 237, 238, 239, 240, 0, 241, 0, 1063, + 235, 236, 237, 238, -2046, 240, 0, 241, 0, 1063, 1064, 244, 245, 0, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 0, 258, 259, 260, + 252, -2046, 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, - 270, 271, 272, 1065, 1066, 0, 1067, 0, 276, 277, - 278, 279, 280, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 288, 289, 290, 0, 291, 292, 293, 294, + 270, 271, 272, 1065, 1066, 0, 1067, 0, 276, 0, + 0, 279, 280, 281, 282, 283, 284, 285, 286, 0, + 0, 287, 288, 289, -2046, 0, 291, 292, 293, 294, 295, 296, 297, 298, 1068, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 1069, 321, 1070, 323, 324, - 325, 326, 0, 327, 328, 329, 330, 1072, 790, 332, - 1073, 334, 335, 336, 0, 337, 338, 0, 0, 339, + 325, 326, 0, 327, 328, 0, 330, 1072, 790, 332, + 1073, 334, 335, 336, 0, 337, 338, 0, 0, 1074, 340, 341, 0, 0, 342, 343, 344, 345, 346, 347, 792, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, 361, 793, 363, @@ -6943,23 +6902,23 @@ static const yytype_int16 yytable[] = 373, 374, 375, 0, 376, 377, 378, 1075, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, 397, 398, 399, - 400, 401, 402, 2174, 2175, 405, 406, 407, 408, 409, + 400, 401, 402, 1076, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 0, 426, + 418, 419, 420, 421, 422, 423, 424, 425, 0, -2046, 427, 428, 1077, 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 795, 0, 0, 446, 447, 0, 448, 449, 450, 451, 452, 453, 454, 0, 455, 1078, 1079, 0, 0, 458, 459, - 796, 461, 797, 1080, 463, 464, 798, 466, 467, 468, + 796, 461, 797, 1080, 463, 464, 798, 466, 467, 3479, 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, 482, 483, 1081, 0, 485, 486, 487, 488, 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, - 510, 511, 512, 513, 514, 515, 0, 0, 0, 0, - 0, 0, 0, 2176, 2177, 0, 0, 0, 0, 0, + 510, 511, 512, 513, 514, 515, -2046, 0, 0, 0, + 0, 0, 0, 1083, 1084, 1085, 0, 0, 0, 0, 1086, 0, 1087, 0, 0, 0, 0, 1088, 1089, 0, - 1090, 1091, 116, 1042, 815, 1043, 1044, 1045, 1046, 1047, + 1090, 1091, 116, 1042, 815, 1043, 1044, 0, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, 1048, 0, @@ -6981,21 +6940,21 @@ static const yytype_int16 yytable[] = 251, 252, 253, 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, 272, 1065, 1066, 0, 1067, 0, 276, - 0, 278, 279, 280, 281, 282, 283, 284, 285, 286, + 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 0, 0, 287, 288, 289, 290, 0, 291, 292, 293, 294, 295, 296, 297, 298, 1068, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 1069, 321, 1070, 323, 324, 325, 326, 0, 327, 328, 329, 330, 1072, 790, 332, 1073, 334, 335, 336, 0, 337, 338, 0, 0, - 1074, 340, 341, 0, 0, 342, 343, 344, 345, 346, + 339, 340, 341, 0, 0, 342, 343, 344, 345, 346, 347, 792, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, 361, 793, 363, 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, 1075, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, 397, 398, - 399, 400, 401, 402, 1076, 404, 405, 406, 407, 408, + 399, 400, 401, 402, 2175, 2176, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, 425, 0, 426, 427, 428, 1077, 430, 0, 431, 432, 433, 434, @@ -7009,9 +6968,9 @@ static const yytype_int16 yytable[] = 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 0, 0, 0, - 0, 0, 0, 0, 1409, 1410, 0, 0, 0, 0, + 0, 0, 0, 0, 2177, 2178, 0, 0, 0, 0, 0, 1086, 0, 1087, 0, 0, 0, 0, 1088, 1089, - 0, 1090, 1091, 116, 1042, 815, 1043, 1044, 0, 1046, + 0, 1090, 1091, 116, 1042, 815, 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, 1048, @@ -7029,25 +6988,25 @@ static const yytype_int16 yytable[] = 218, 219, 1060, 221, 222, 223, 224, 225, 788, 1061, 227, 0, 228, 229, 1062, 231, 0, 232, 0, 233, 234, 0, 235, 236, 237, 238, 239, 240, 0, 241, - 3033, 1063, 1064, 244, 245, 0, 246, 247, 248, 249, - 250, 251, 252, 253, 254, 255, 256, 257, -717, 258, + 0, 1063, 1064, 244, 245, 0, 246, 247, 248, 249, + 250, 251, 252, 253, 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, 272, 1065, 1066, 0, 1067, 0, - 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, + 276, 0, 278, 279, 280, 281, 282, 283, 284, 285, 286, 0, 0, 287, 288, 289, 290, 0, 291, 292, 293, 294, 295, 296, 297, 298, 1068, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 1069, 321, 1070, 323, 324, 325, 326, 0, 327, 328, 329, 330, 1072, 790, 332, 1073, 334, 335, 336, 0, 337, 338, 0, - 0, 339, 340, 341, 0, 0, 342, 343, 344, 345, + 0, 1074, 340, 341, 0, 0, 342, 343, 344, 345, 346, 347, 792, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, 361, 793, 363, 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, 1075, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, 397, - 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, + 398, 399, 400, 401, 402, 1076, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, 425, 0, 426, 427, 428, 1077, 430, 0, 431, 432, 433, @@ -7061,8 +7020,8 @@ static const yytype_int16 yytable[] = 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1086, 0, 2726, 0, 0, 0, 0, 1088, + 0, 0, 0, 0, 0, 1410, 1411, 0, 0, 0, + 0, 0, 1086, 0, 1087, 0, 0, 0, 0, 1088, 1089, 0, 1090, 1091, 116, 1042, 815, 1043, 1044, 0, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, @@ -7081,8 +7040,8 @@ static const yytype_int16 yytable[] = 217, 218, 219, 1060, 221, 222, 223, 224, 225, 788, 1061, 227, 0, 228, 229, 1062, 231, 0, 232, 0, 233, 234, 0, 235, 236, 237, 238, 239, 240, 0, - 241, 0, 1063, 1064, 244, 245, 0, 246, 247, 248, - 249, 250, 251, 252, 253, 254, 255, 256, 257, 0, + 241, 3036, 1063, 1064, 244, 245, 0, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, -718, 258, 259, 260, 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, 272, 1065, 1066, 0, 1067, 0, 276, 277, 278, 279, 280, 281, 282, 283, 284, @@ -7114,7 +7073,7 @@ static const yytype_int16 yytable[] = 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1086, 0, 2726, 0, 0, 0, 0, + 0, 0, 0, 1086, 0, 2729, 0, 0, 0, 0, 1088, 1089, 0, 1090, 1091, 116, 1042, 815, 1043, 1044, 0, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, 122, @@ -7151,7 +7110,7 @@ static const yytype_int16 yytable[] = 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, 1075, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, - 0, 397, 398, 399, 400, 401, 402, 1076, 404, 405, + 0, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, 425, 0, 426, 427, 428, 1077, 430, 0, 431, @@ -7166,37 +7125,37 @@ static const yytype_int16 yytable[] = 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1086, 0, 1087, 0, 0, 0, + 0, 0, 0, 0, 1086, 0, 2729, 0, 0, 0, 0, 1088, 1089, 0, 1090, 1091, 116, 1042, 815, 1043, - 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, 0, + 1044, 0, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, 1048, 0, 0, 128, 129, 130, 0, 131, - 132, 133, 134, 135, 136, 137, 0, 1049, 140, 1050, + 132, 133, 134, 135, 136, 137, 138, 1049, 140, 1050, 1051, 0, 143, 144, 145, 146, 147, 148, 1052, 785, 149, 150, 151, 152, 1053, 1054, 155, 0, 156, 157, - 158, 159, 786, 0, 787, 0, 1055, 163, 164, 165, + 158, 159, 786, 0, 787, 0, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 0, 172, 173, 174, 175, 176, 177, 0, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 1056, 190, 191, 1057, 193, - 1058, 194, 0, 195, 196, 197, 198, 199, 200, 0, + 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, 205, 206, 1059, - 208, 209, 0, 210, 211, 212, 0, 213, 0, 215, + 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, 1060, 221, 222, 223, 224, 225, 788, 1061, 227, 0, 228, 229, 1062, 231, 0, - 232, 0, 233, 234, 0, 235, 236, 237, 238, 0, + 232, 0, 233, 234, 0, 235, 236, 237, 238, 239, 240, 0, 241, 0, 1063, 1064, 244, 245, 0, 246, - 247, 248, 249, 250, 251, 252, 0, 254, 255, 256, + 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, 272, 1065, 1066, - 0, 1067, 0, 276, 0, 0, 279, 280, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 288, 289, 0, + 0, 1067, 0, 276, 277, 278, 279, 280, 281, 282, + 283, 284, 285, 286, 0, 0, 287, 288, 289, 290, 0, 291, 292, 293, 294, 295, 296, 297, 298, 1068, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 1069, 321, 1070, 323, 324, 325, 326, 0, 327, 328, - 0, 330, 1072, 790, 332, 1073, 334, 335, 336, 0, - 337, 338, 0, 0, 1074, 340, 341, 0, 0, 342, + 329, 330, 1072, 790, 332, 1073, 334, 335, 336, 0, + 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, 345, 346, 347, 792, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, 361, 793, 363, 364, 365, 366, 367, 368, @@ -7206,7 +7165,7 @@ static const yytype_int16 yytable[] = 396, 0, 397, 398, 399, 400, 401, 402, 1076, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 0, 0, 427, 428, 1077, 430, 0, + 423, 424, 425, 0, 426, 427, 428, 1077, 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 795, 0, 0, 446, 447, 0, 448, 449, 450, 451, 452, 453, 454, 0, 455, 1078, @@ -7217,307 +7176,260 @@ static const yytype_int16 yytable[] = 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, - 515, 0, 0, 0, 0, 0, 0, 0, 1083, 1084, - 1085, 0, 964, 1340, 815, 1086, 0, 1087, 1046, 0, - 0, 0, 1088, 1089, 0, 1090, 1091, 0, 0, 0, - 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, - 125, 126, 127, 0, 0, 0, 560, 0, 0, 0, - 0, 565, 129, 130, 0, 131, 132, 133, 567, 135, - 136, 137, 568, 569, 570, 571, 572, 0, 143, 144, - 145, 146, 147, 148, 0, 0, 149, 150, 151, 152, - 576, 577, 155, 0, 156, 157, 158, 159, 579, 0, - 581, 0, 583, 163, 164, 165, 166, 167, 584, 169, - 170, 171, 0, 172, 173, 174, 175, 176, 177, 0, - 587, 179, 180, 181, 182, 183, 184, 185, 186, 187, - 188, 589, 190, 191, 590, 193, 0, 194, 0, 195, - 196, 197, 198, 199, 200, 0, 0, 201, 202, 203, - 204, 0, 0, 205, 206, 207, 208, 209, 0, 210, - 211, 212, 0, 213, 214, 215, 0, 216, 217, 218, - 219, 600, 221, 222, 223, 224, 225, 601, 1341, 227, - 0, 228, 229, 604, 231, 0, 232, 0, 233, 607, - 0, 609, 236, 237, 610, 611, 240, 0, 241, 0, - 614, 615, 244, 245, 0, 246, 247, 248, 249, 250, - 251, 252, 617, 254, 255, 256, 257, 0, 258, 259, - 260, 261, 262, 263, 264, 0, 265, 620, 621, 268, - 269, 270, 271, 272, 622, 623, 0, 625, 0, 276, - 627, 628, 279, 629, 281, 282, 283, 284, 285, 286, - 0, 0, 287, 632, 289, 633, 0, 291, 292, 293, - 294, 295, 296, 297, 298, 635, 300, 301, 302, 303, - 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, - 314, 315, 316, 317, 318, 319, 636, 637, 638, 323, - 324, 325, 639, 0, 327, 328, 641, 330, 0, 643, - 332, 644, 334, 335, 336, 0, 337, 338, 1342, 0, - 339, 340, 341, 0, 0, 342, 343, 650, 651, 346, - 652, 653, 349, 350, 351, 352, 353, 354, 355, 356, - 357, 358, 359, 0, 0, 0, 0, 360, 361, 658, - 659, 364, 365, 660, 367, 368, 369, 0, 370, 371, - 372, 373, 374, 375, 0, 376, 377, 378, 663, 380, - 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, - 390, 391, 392, 393, 394, 395, 396, 0, 397, 398, - 666, 400, 401, 402, 667, 404, 405, 406, 407, 408, - 409, 410, 411, 412, 413, 414, 415, 416, 0, 669, - 417, 418, 419, 420, 421, 422, 670, 424, 425, 0, - 672, 427, 428, 673, 430, 0, 431, 432, 433, 434, - 435, 436, 437, 438, 439, 440, 441, 442, 675, 444, - 676, 0, 0, 446, 447, 0, 448, 680, 450, 451, - 452, 453, 454, 0, 455, 682, 683, 0, 0, 458, - 459, 686, 461, 687, 1343, 463, 464, 689, 466, 467, - 468, 469, 470, 0, 0, 471, 472, 473, 0, 474, - 475, 476, 477, 0, 478, 479, 480, 481, 482, 694, - 695, 0, 485, 697, 487, 488, 489, 490, 491, 492, - 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, - 499, 500, 702, 703, 704, 705, 706, 707, 708, 709, - 710, 711, 712, 512, 513, 514, 515, 0, 0, 0, - 0, 523, 0, 0, 1344, 1345, 2349, 0, 0, 0, - 0, 0, 0, 2350, 0, 0, 0, 0, 0, 1089, - 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, - 126, 127, 0, 0, 0, 0, 0, 0, 1000, 0, - 0, 129, 130, 0, 131, 132, 133, 0, 135, 136, - 137, 138, 139, 0, 141, 142, 0, 143, 144, 145, - 146, 147, 148, 0, 0, 149, 150, 151, 152, 153, - 154, 155, 0, 156, 157, 158, 159, 160, 0, 0, - 0, 162, 163, 164, 165, 166, 167, 0, 169, 170, - 171, 0, 172, 173, 174, 175, 176, 177, 0, 0, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, - 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, - 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, - 212, 0, 213, 214, 215, -524, 216, 217, 218, 219, - 220, 221, 222, 223, 224, 225, 226, 0, 227, -524, - 228, 229, 230, 231, -524, 232, 0, 233, 0, 0, - 0, 236, 237, 524, 0, 240, 0, 241, 0, 242, - 243, 244, 245, 0, 246, 247, 248, 249, 250, 251, - 252, 0, 254, 255, 256, 257, 0, 258, 259, 260, - 261, 262, 263, 264, 0, 265, 0, 267, 268, 269, - 270, 271, 272, 273, 274, -524, 275, 0, 276, 0, - 0, 279, 0, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 0, 289, 0, -524, 291, 292, 293, 294, - 295, 296, 297, 298, 525, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 0, 322, 323, 324, - 325, 326, 0, 327, 328, 0, 330, 0, 331, 332, - 333, 334, 335, 336, -524, 337, 338, 0, 0, 339, - 340, 341, 0, -524, 342, 343, 344, 0, 346, 0, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 0, 0, 0, 0, 360, 361, 362, 0, - 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, - 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 397, 398, 0, - 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 0, 0, - 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 526, 444, 445, - 0, 0, 446, 447, 0, 448, 0, 450, 451, 452, - 453, 454, 0, 455, 456, 457, 0, 0, 458, 459, - 460, 461, 462, 0, 463, 464, 465, 466, 467, 468, - 469, 470, -524, 0, 471, 472, 473, 0, 474, 475, - 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, - 0, 485, 0, 487, 488, 489, 490, 491, 492, 493, - 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, - 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, - 510, 511, 512, 513, 514, 515, 523, 0, 549, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1158, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 129, 130, 0, 131, - 132, 133, 0, 135, 136, 137, 138, 139, 0, 141, - 142, 0, 143, 144, 145, 146, 147, 148, 0, 0, - 149, 150, 151, 152, 153, 154, 155, 0, 156, 157, - 158, 159, 160, 0, 0, 0, 162, 163, 164, 165, - 166, 167, 0, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 0, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, - 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 226, 0, 227, 0, 228, 229, 230, 231, 0, - 232, 0, 233, 0, 0, 0, 236, 237, 524, 0, - 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, - 247, 248, 249, 250, 251, 252, 0, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 0, 267, 268, 269, 270, 271, 272, 273, 274, - 0, 275, 0, 276, 0, 0, 279, 0, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 0, 289, 0, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 525, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 0, 322, 323, 324, 325, 326, 0, 327, 328, - 0, 330, 0, 331, 332, 333, 334, 335, 336, 0, - 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, - 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 362, 0, 364, 365, 366, 367, 368, - 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 0, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 0, 0, 427, 428, 429, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 526, 444, 445, 0, 0, 446, 447, 0, - 448, 0, 450, 451, 452, 453, 454, 0, 455, 456, - 457, 0, 0, 458, 459, 460, 461, 462, 0, 463, - 464, 465, 466, 467, 468, 469, 470, 0, 0, 471, - 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 483, 484, 0, 485, 0, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, - 515, 964, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 2436, 3217, 0, - 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, - 126, 127, 3, 4, 0, 560, 0, 0, 0, 0, - 565, 129, 130, 0, 131, 132, 133, 567, 135, 136, - 137, 568, 569, 570, 571, 572, 0, 143, 144, 145, - 146, 147, 148, 0, 0, 149, 150, 151, 152, 576, - 577, 155, 0, 156, 157, 158, 159, 579, 0, 581, - 0, 583, 163, 164, 165, 166, 167, 584, 169, 170, - 171, 0, 172, 173, 174, 175, 176, 177, 0, 587, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 589, 190, 191, 590, 193, 0, 194, 0, 195, 196, - 197, 198, 199, 200, 14, 15, 201, 202, 203, 204, - 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, - 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, - 600, 221, 222, 223, 224, 225, 601, 0, 227, 0, - 228, 229, 604, 231, 0, 232, 0, 233, 607, 23, - 609, 236, 237, 610, 611, 240, 0, 241, 0, 614, - 615, 244, 245, 0, 246, 247, 248, 249, 250, 251, - 252, 617, 254, 255, 256, 257, 0, 258, 259, 260, - 261, 262, 263, 264, 0, 265, 620, 621, 268, 269, - 270, 271, 272, 622, 623, 0, 625, 0, 276, 627, - 628, 279, 629, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 632, 289, 633, 0, 291, 292, 293, 294, - 295, 296, 297, 298, 635, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 636, 637, 638, 323, 324, - 325, 639, 0, 327, 328, 641, 330, 0, 643, 332, - 644, 334, 335, 336, 0, 337, 338, 0, 0, 339, - 340, 341, 0, 0, 342, 343, 650, 651, 346, 652, - 653, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 27, 28, 29, 0, 360, 361, 658, 659, - 364, 365, 660, 367, 368, 369, 0, 370, 371, 372, - 373, 374, 375, 0, 376, 377, 378, 663, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 397, 398, 666, - 400, 401, 402, 667, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 34, 669, 417, - 418, 419, 420, 421, 422, 670, 424, 425, 36, 672, - 427, 428, 673, 430, 0, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 675, 444, 676, - 38, 0, 446, 447, 39, 448, 680, 450, 451, 452, - 453, 454, 0, 455, 682, 683, 0, 0, 458, 459, - 686, 461, 687, 0, 463, 464, 689, 466, 467, 468, - 469, 470, 0, 0, 471, 472, 473, 41, 474, 475, - 476, 477, 0, 478, 479, 480, 481, 482, 965, 695, - 0, 485, 697, 487, 488, 489, 490, 491, 492, 493, - 0, 0, 494, 0, 45, 495, 496, 497, 498, 499, - 500, 702, 703, 704, 705, 706, 707, 708, 709, 710, - 711, 712, 512, 513, 514, 515, 0, 116, 46, 549, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 47, 0, 0, 0, 117, 118, 119, 120, + 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1086, 0, 1087, 0, 0, + 0, 0, 1088, 1089, 0, 1090, 1091, 116, 1042, 815, + 1043, 1044, 1045, 1046, 1047, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, 0, 0, - 0, 0, 0, 0, 0, 0, 128, 129, 130, 0, - 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, - 141, 142, 0, 143, 144, 145, 146, 147, 148, 0, - 785, 149, 150, 151, 152, 153, 154, 155, 0, 156, - 157, 158, 159, 786, 0, 787, 0, 162, 163, 164, + 0, 0, 0, 1048, 0, 0, 128, 129, 130, 0, + 131, 132, 133, 134, 135, 136, 137, 0, 1049, 140, + 1050, 1051, 0, 143, 144, 145, 146, 147, 148, 1052, + 785, 149, 150, 151, 152, 1053, 1054, 155, 0, 156, + 157, 158, 159, 786, 0, 787, 0, 1055, 163, 164, 165, 166, 167, 168, 169, 170, 171, 0, 172, 173, 174, 175, 176, 177, 0, 178, 179, 180, 181, 182, - 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, - 193, 0, 194, 0, 195, 196, 197, 198, 199, 200, - 14, 15, 201, 202, 203, 204, 0, 0, 205, 206, - 207, 208, 209, 0, 210, 211, 212, 0, 213, 214, - 215, 0, 216, 217, 218, 219, 220, 221, 222, 223, - 224, 225, 788, 0, 227, 0, 228, 229, 230, 231, - 0, 232, 0, 233, 234, 23, 235, 236, 237, 238, - 239, 240, 0, 241, 0, 242, 243, 244, 245, 0, - 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, + 183, 184, 185, 186, 187, 188, 1056, 190, 191, 1057, + 193, 1058, 194, 0, 195, 196, 197, 198, 199, 200, + 0, 0, 201, 202, 203, 204, 0, 0, 205, 206, + 1059, 208, 209, 0, 210, 211, 212, 0, 213, 0, + 215, 0, 216, 217, 218, 219, 1060, 221, 222, 223, + 224, 225, 788, 1061, 227, 0, 228, 229, 1062, 231, + 0, 232, 0, 233, 234, 0, 235, 236, 237, 238, + 0, 240, 0, 241, 0, 1063, 1064, 244, 245, 0, + 246, 247, 248, 249, 250, 251, 252, 0, 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, 263, 264, - 0, 265, 266, 267, 268, 269, 270, 271, 272, 273, - 274, 0, 275, 0, 276, 277, 278, 279, 280, 281, - 282, 283, 284, 285, 286, 789, 0, 287, 288, 289, - 290, 0, 291, 292, 293, 294, 295, 296, 297, 298, - 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, + 0, 265, 266, 267, 268, 269, 270, 271, 272, 1065, + 1066, 0, 1067, 0, 276, 0, 0, 279, 280, 281, + 282, 283, 284, 285, 286, 0, 0, 287, 288, 289, + 0, 0, 291, 292, 293, 294, 295, 296, 297, 298, + 1068, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, - 319, 320, 321, 322, 323, 324, 325, 326, 0, 327, - 328, 329, 330, 0, 790, 332, 333, 334, 335, 336, - 0, 337, 338, 0, 791, 339, 340, 341, 0, 0, + 319, 1069, 321, 1070, 323, 324, 325, 326, 0, 327, + 328, 0, 330, 1072, 790, 332, 1073, 334, 335, 336, + 0, 337, 338, 0, 0, 1074, 340, 341, 0, 0, 342, 343, 344, 345, 346, 347, 792, 349, 350, 351, - 352, 353, 354, 355, 356, 357, 358, 359, 27, 28, - 29, 0, 360, 361, 793, 363, 364, 365, 366, 367, + 352, 353, 354, 355, 356, 357, 358, 359, 0, 0, + 0, 0, 360, 361, 793, 363, 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, 375, 0, - 376, 377, 378, 379, 380, 381, 382, 383, 0, 384, + 376, 377, 378, 1075, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, - 395, 396, 0, 397, 398, 399, 400, 401, 402, 403, - 794, 405, 406, 407, 408, 409, 410, 411, 412, 413, - 414, 415, 416, 34, 0, 417, 418, 419, 420, 421, - 422, 423, 424, 425, 36, 426, 427, 428, 429, 430, + 395, 396, 0, 397, 398, 399, 400, 401, 402, 1076, + 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, + 414, 415, 416, 0, 0, 417, 418, 419, 420, 421, + 422, 423, 424, 425, 0, 0, 427, 428, 1077, 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, 439, - 440, 441, 442, 443, 444, 795, 38, 0, 446, 447, - 39, 448, 449, 450, 451, 452, 453, 454, 0, 455, - 456, 457, 0, 0, 458, 459, 796, 461, 797, 0, + 440, 441, 442, 443, 444, 795, 0, 0, 446, 447, + 0, 448, 449, 450, 451, 452, 453, 454, 0, 455, + 1078, 1079, 0, 0, 458, 459, 796, 461, 797, 1080, 463, 464, 798, 466, 467, 468, 469, 470, 0, 0, - 471, 472, 473, 41, 474, 475, 476, 477, 0, 478, - 479, 480, 481, 482, 799, 484, 0, 485, 486, 487, + 471, 472, 473, 0, 474, 475, 476, 477, 0, 478, + 479, 480, 481, 482, 483, 1081, 0, 485, 486, 487, 488, 489, 490, 491, 492, 493, 0, 0, 494, 0, - 45, 495, 496, 497, 498, 499, 500, 501, 502, 503, + 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, - 514, 515, 0, 116, 46, 549, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 800, 0, + 514, 515, 0, 0, 0, 0, 0, 0, 0, 1083, + 1084, 1085, 0, 964, 1341, 815, 1086, 0, 1087, 1046, + 0, 0, 0, 1088, 1089, 0, 1090, 1091, 0, 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, - 0, 125, 126, 127, 0, 0, 0, 0, 0, 0, - 0, 0, 128, 129, 130, 0, 131, 132, 133, 134, - 135, 136, 137, 138, 139, 140, 141, 142, 0, 143, - 144, 145, 146, 147, 148, 0, 785, 149, 150, 151, - 152, 153, 154, 155, 0, 156, 157, 158, 159, 786, - 0, 787, 0, 162, 163, 164, 165, 166, 167, 168, + 0, 125, 126, 127, 0, 0, 0, 560, 0, 0, + 0, 0, 565, 129, 130, 0, 131, 132, 133, 567, + 135, 136, 137, 568, 569, 570, 571, 572, 0, 143, + 144, 145, 146, 147, 148, 0, 0, 149, 150, 151, + 152, 576, 577, 155, 0, 156, 157, 158, 159, 579, + 0, 581, 0, 583, 163, 164, 165, 166, 167, 584, 169, 170, 171, 0, 172, 173, 174, 175, 176, 177, - 0, 178, 179, 180, 181, 182, 183, 184, 185, 186, - 187, 188, 189, 190, 191, 192, 193, 0, 194, 0, + 0, 587, 179, 180, 181, 182, 183, 184, 185, 186, + 187, 188, 589, 190, 191, 590, 193, 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, 217, - 218, 219, 220, 221, 222, 223, 224, 225, 788, 0, - 227, 0, 228, 229, 230, 231, 0, 232, 0, 233, - 234, 0, 235, 236, 237, 238, 239, 240, 0, 241, - 0, 242, 243, 244, 245, 0, 246, 247, 248, 249, - 250, 251, 252, 253, 254, 255, 256, 257, 0, 258, - 259, 260, 261, 262, 263, 264, 0, 265, 266, 267, - 268, 269, 270, 271, 272, 273, 274, 0, 275, 0, - 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, - 286, 789, 0, 287, 288, 289, 290, 0, 291, 292, - 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, + 218, 219, 600, 221, 222, 223, 224, 225, 601, 1342, + 227, 0, 228, 229, 604, 231, 0, 232, 0, 233, + 607, 0, 609, 236, 237, 610, 611, 240, 0, 241, + 0, 614, 615, 244, 245, 0, 246, 247, 248, 249, + 250, 251, 252, 617, 254, 255, 256, 257, 0, 258, + 259, 260, 261, 262, 263, 264, 0, 265, 620, 621, + 268, 269, 270, 271, 272, 622, 623, 0, 625, 0, + 276, 627, 628, 279, 629, 281, 282, 283, 284, 285, + 286, 0, 0, 287, 632, 289, 633, 0, 291, 292, + 293, 294, 295, 296, 297, 298, 635, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, - 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, - 323, 324, 325, 326, 0, 327, 328, 329, 330, 0, - 790, 332, 333, 334, 335, 336, 0, 337, 338, 0, - 791, 339, 340, 341, 0, 0, 342, 343, 344, 345, - 346, 347, 792, 349, 350, 351, 352, 353, 354, 355, + 313, 314, 315, 316, 317, 318, 319, 636, 637, 638, + 323, 324, 325, 639, 0, 327, 328, 641, 330, 0, + 643, 332, 644, 334, 335, 336, 0, 337, 338, 1343, + 0, 339, 340, 341, 0, 0, 342, 343, 650, 651, + 346, 652, 653, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, 361, - 793, 363, 364, 365, 366, 367, 368, 369, 0, 370, - 371, 372, 373, 374, 375, 0, 376, 377, 378, 379, + 658, 659, 364, 365, 660, 367, 368, 369, 0, 370, + 371, 372, 373, 374, 375, 0, 376, 377, 378, 663, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, 397, - 398, 399, 400, 401, 402, 403, 794, 405, 406, 407, + 398, 666, 400, 401, 402, 667, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 0, - 0, 417, 418, 419, 420, 421, 422, 423, 424, 425, - 0, 426, 427, 428, 429, 430, 0, 431, 432, 433, - 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, - 444, 795, 0, 0, 446, 447, 0, 448, 449, 450, - 451, 452, 453, 454, 0, 455, 456, 457, 0, 0, - 458, 459, 796, 461, 797, 0, 463, 464, 798, 466, + 669, 417, 418, 419, 420, 421, 422, 670, 424, 425, + 0, 672, 427, 428, 673, 430, 0, 431, 432, 433, + 434, 435, 436, 437, 438, 439, 440, 441, 442, 675, + 444, 676, 0, 0, 446, 447, 0, 448, 680, 450, + 451, 452, 453, 454, 0, 455, 682, 683, 0, 0, + 458, 459, 686, 461, 687, 1344, 463, 464, 689, 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, 482, - 799, 484, 0, 485, 486, 487, 488, 489, 490, 491, + 694, 695, 0, 485, 697, 487, 488, 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, - 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, - 508, 509, 510, 511, 512, 513, 514, 515, 116, 0, + 498, 499, 500, 702, 703, 704, 705, 706, 707, 708, + 709, 710, 711, 712, 512, 513, 514, 515, 0, 0, + 0, 0, 523, 0, 0, 1345, 1346, 2350, 0, 0, + 0, 0, 2351, 0, 2352, 0, 0, 0, 0, 0, + 1089, 117, 118, 119, 120, 121, 122, 123, 124, 0, + 125, 126, 127, 0, 0, 0, 0, 0, 0, 1000, + 0, 0, 129, 130, 0, 131, 132, 133, 0, 135, + 136, 137, 138, 139, 0, 141, 142, 0, 143, 144, + 145, 146, 147, 148, 0, 0, 149, 150, 151, 152, + 153, 154, 155, 0, 156, 157, 158, 159, 160, 0, + 0, 0, 162, 163, 164, 165, 166, 167, 0, 169, + 170, 171, 0, 172, 173, 174, 175, 176, 177, 0, + 0, 179, 180, 181, 182, 183, 184, 185, 186, 187, + 188, 189, 190, 191, 192, 193, 0, 194, 0, 195, + 196, 197, 198, 199, 200, 0, 0, 201, 202, 203, + 204, 0, 0, 205, 206, 207, 208, 209, 0, 210, + 211, 212, 0, 213, 214, 215, -525, 216, 217, 218, + 219, 220, 221, 222, 223, 224, 225, 226, 0, 227, + -525, 228, 229, 230, 231, -525, 232, 0, 233, 0, + 0, 0, 236, 237, 524, 0, 240, 0, 241, 0, + 242, 243, 244, 245, 0, 246, 247, 248, 249, 250, + 251, 252, 0, 254, 255, 256, 257, 0, 258, 259, + 260, 261, 262, 263, 264, 0, 265, 0, 267, 268, + 269, 270, 271, 272, 273, 274, -525, 275, 0, 276, + 0, 0, 279, 0, 281, 282, 283, 284, 285, 286, + 0, 0, 287, 0, 289, 0, -525, 291, 292, 293, + 294, 295, 296, 297, 298, 525, 300, 301, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + 314, 315, 316, 317, 318, 319, 320, 0, 322, 323, + 324, 325, 326, 0, 327, 328, 0, 330, 0, 331, + 332, 333, 334, 335, 336, -525, 337, 338, 0, 0, + 339, 340, 341, 0, -525, 342, 343, 344, 0, 346, + 0, 348, 349, 350, 351, 352, 353, 354, 355, 356, + 357, 358, 359, 0, 0, 0, 0, 360, 361, 362, + 0, 364, 365, 366, 367, 368, 369, 0, 370, 371, + 372, 373, 374, 375, 0, 376, 377, 378, 379, 380, + 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, + 390, 391, 392, 393, 394, 395, 396, 0, 397, 398, + 0, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, 412, 413, 414, 415, 416, 0, 0, + 417, 418, 419, 420, 421, 422, 423, 424, 425, 0, + 0, 427, 428, 429, 430, 0, 431, 432, 433, 434, + 435, 436, 437, 438, 439, 440, 441, 442, 526, 444, + 445, 0, 0, 446, 447, 0, 448, 0, 450, 451, + 452, 453, 454, 0, 455, 456, 457, 0, 0, 458, + 459, 460, 461, 462, 0, 463, 464, 465, 466, 467, + 468, 469, 470, -525, 0, 471, 472, 473, 0, 474, + 475, 476, 477, 0, 478, 479, 480, 481, 482, 483, + 484, 0, 485, 0, 487, 488, 489, 490, 491, 492, + 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, + 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, + 509, 510, 511, 512, 513, 514, 515, 523, 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 800, 0, 0, 117, 118, 119, + 0, 0, 0, 0, 1159, 0, 117, 118, 119, 120, + 121, 122, 123, 124, 0, 125, 126, 127, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 129, 130, 0, + 131, 132, 133, 0, 135, 136, 137, 138, 139, 0, + 141, 142, 0, 143, 144, 145, 146, 147, 148, 0, + 0, 149, 150, 151, 152, 153, 154, 155, 0, 156, + 157, 158, 159, 160, 0, 0, 0, 162, 163, 164, + 165, 166, 167, 0, 169, 170, 171, 0, 172, 173, + 174, 175, 176, 177, 0, 0, 179, 180, 181, 182, + 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, + 193, 0, 194, 0, 195, 196, 197, 198, 199, 200, + 0, 0, 201, 202, 203, 204, 0, 0, 205, 206, + 207, 208, 209, 0, 210, 211, 212, 0, 213, 214, + 215, 0, 216, 217, 218, 219, 220, 221, 222, 223, + 224, 225, 226, 0, 227, 0, 228, 229, 230, 231, + 0, 232, 0, 233, 0, 0, 0, 236, 237, 524, + 0, 240, 0, 241, 0, 242, 243, 244, 245, 0, + 246, 247, 248, 249, 250, 251, 252, 0, 254, 255, + 256, 257, 0, 258, 259, 260, 261, 262, 263, 264, + 0, 265, 0, 267, 268, 269, 270, 271, 272, 273, + 274, 0, 275, 0, 276, 0, 0, 279, 0, 281, + 282, 283, 284, 285, 286, 0, 0, 287, 0, 289, + 0, 0, 291, 292, 293, 294, 295, 296, 297, 298, + 525, 300, 301, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, + 319, 320, 0, 322, 323, 324, 325, 326, 0, 327, + 328, 0, 330, 0, 331, 332, 333, 334, 335, 336, + 0, 337, 338, 0, 0, 339, 340, 341, 0, 0, + 342, 343, 344, 0, 346, 0, 348, 349, 350, 351, + 352, 353, 354, 355, 356, 357, 358, 359, 0, 0, + 0, 0, 360, 361, 362, 0, 364, 365, 366, 367, + 368, 369, 0, 370, 371, 372, 373, 374, 375, 0, + 376, 377, 378, 379, 380, 381, 382, 383, 0, 384, + 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, + 395, 396, 0, 397, 398, 0, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, + 414, 415, 416, 0, 0, 417, 418, 419, 420, 421, + 422, 423, 424, 425, 0, 0, 427, 428, 429, 430, + 0, 431, 432, 433, 434, 435, 436, 437, 438, 439, + 440, 441, 442, 526, 444, 445, 0, 0, 446, 447, + 0, 448, 0, 450, 451, 452, 453, 454, 0, 455, + 456, 457, 0, 0, 458, 459, 460, 461, 462, 0, + 463, 464, 465, 466, 467, 468, 469, 470, 0, 0, + 471, 472, 473, 0, 474, 475, 476, 477, 0, 478, + 479, 480, 481, 482, 483, 484, 0, 485, 0, 487, + 488, 489, 490, 491, 492, 493, 0, 0, 494, 0, + 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, + 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, + 514, 515, 964, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 2439, 3220, + 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, + 125, 126, 127, 3, 4, 0, 560, 0, 0, 0, + 0, 565, 129, 130, 0, 131, 132, 133, 567, 135, + 136, 137, 568, 569, 570, 571, 572, 0, 143, 144, + 145, 146, 147, 148, 0, 0, 149, 150, 151, 152, + 576, 577, 155, 0, 156, 157, 158, 159, 579, 0, + 581, 0, 583, 163, 164, 165, 166, 167, 584, 169, + 170, 171, 0, 172, 173, 174, 175, 176, 177, 0, + 587, 179, 180, 181, 182, 183, 184, 185, 186, 187, + 188, 589, 190, 191, 590, 193, 0, 194, 0, 195, + 196, 197, 198, 199, 200, 14, 15, 201, 202, 203, + 204, 0, 0, 205, 206, 207, 208, 209, 0, 210, + 211, 212, 0, 213, 214, 215, 0, 216, 217, 218, + 219, 600, 221, 222, 223, 224, 225, 601, 0, 227, + 0, 228, 229, 604, 231, 0, 232, 0, 233, 607, + 23, 609, 236, 237, 610, 611, 240, 0, 241, 0, + 614, 615, 244, 245, 0, 246, 247, 248, 249, 250, + 251, 252, 617, 254, 255, 256, 257, 0, 258, 259, + 260, 261, 262, 263, 264, 0, 265, 620, 621, 268, + 269, 270, 271, 272, 622, 623, 0, 625, 0, 276, + 627, 628, 279, 629, 281, 282, 283, 284, 285, 286, + 0, 0, 287, 632, 289, 633, 0, 291, 292, 293, + 294, 295, 296, 297, 298, 635, 300, 301, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + 314, 315, 316, 317, 318, 319, 636, 637, 638, 323, + 324, 325, 639, 0, 327, 328, 641, 330, 0, 643, + 332, 644, 334, 335, 336, 0, 337, 338, 0, 0, + 339, 340, 341, 0, 0, 342, 343, 650, 651, 346, + 652, 653, 349, 350, 351, 352, 353, 354, 355, 356, + 357, 358, 359, 27, 28, 29, 0, 360, 361, 658, + 659, 364, 365, 660, 367, 368, 369, 0, 370, 371, + 372, 373, 374, 375, 0, 376, 377, 378, 663, 380, + 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, + 390, 391, 392, 393, 394, 395, 396, 0, 397, 398, + 666, 400, 401, 402, 667, 404, 405, 406, 407, 408, + 409, 410, 411, 412, 413, 414, 415, 416, 34, 669, + 417, 418, 419, 420, 421, 422, 670, 424, 425, 36, + 672, 427, 428, 673, 430, 0, 431, 432, 433, 434, + 435, 436, 437, 438, 439, 440, 441, 442, 675, 444, + 676, 38, 0, 446, 447, 39, 448, 680, 450, 451, + 452, 453, 454, 0, 455, 682, 683, 0, 0, 458, + 459, 686, 461, 687, 0, 463, 464, 689, 466, 467, + 468, 469, 470, 0, 0, 471, 472, 473, 41, 474, + 475, 476, 477, 0, 478, 479, 480, 481, 482, 965, + 695, 0, 485, 697, 487, 488, 489, 490, 491, 492, + 493, 0, 0, 494, 0, 45, 495, 496, 497, 498, + 499, 500, 702, 703, 704, 705, 706, 707, 708, 709, + 710, 711, 712, 512, 513, 514, 515, 0, 116, 46, + 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 47, 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, 0, 0, 0, 128, 129, 130, 0, 131, 132, 133, 134, 135, 136, 137, 138, 139, @@ -7528,392 +7440,194 @@ static const yytype_int16 yytable[] = 173, 174, 175, 176, 177, 0, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, 199, - 200, 0, 0, 201, 202, 203, 204, 0, 0, 205, + 200, 14, 15, 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 788, 0, 227, 0, 228, 229, 230, - 231, 0, 232, 0, 233, 234, 0, 235, 236, 237, + 231, 0, 232, 0, 233, 234, 23, 235, 236, 237, 238, 239, 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 0, 275, 0, 276, 277, 278, 279, 280, - 281, 282, 283, 284, 285, 286, 0, 0, 287, 288, + 281, 282, 283, 284, 285, 286, 789, 0, 287, 288, 289, 290, 0, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 0, 327, 328, 329, 330, 0, 790, 332, 333, 334, 335, - 336, 0, 337, 338, 0, 0, 339, 340, 341, 0, + 336, 0, 337, 338, 0, 791, 339, 340, 341, 0, 0, 342, 343, 344, 345, 346, 347, 792, 349, 350, - 351, 352, 353, 354, 355, 356, 357, 358, 359, 0, - 0, 0, 0, 360, 361, 793, 363, 364, 365, 366, + 351, 352, 353, 354, 355, 356, 357, 358, 359, 27, + 28, 29, 0, 360, 361, 793, 363, 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, 397, 398, 399, 400, 401, 402, 403, 794, 405, 406, 407, 408, 409, 410, 411, 412, - 413, 414, 415, 416, 0, 0, 417, 418, 419, 420, - 421, 422, 423, 424, 425, 0, 426, 427, 428, 429, + 413, 414, 415, 416, 34, 0, 417, 418, 419, 420, + 421, 422, 423, 424, 425, 36, 426, 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, - 439, 440, 441, 442, 443, 444, 795, 0, 0, 446, - 447, 0, 448, 449, 450, 451, 452, 453, 454, 0, + 439, 440, 441, 442, 443, 444, 795, 38, 0, 446, + 447, 39, 448, 449, 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, 0, 458, 459, 796, 461, 797, 0, 463, 464, 798, 466, 467, 468, 469, 470, 0, - 0, 471, 472, 473, 0, 474, 475, 476, 477, 0, - 478, 479, 480, 481, 482, 483, 484, 0, 485, 486, + 0, 471, 472, 473, 41, 474, 475, 476, 477, 0, + 478, 479, 480, 481, 482, 799, 484, 0, 485, 486, 487, 488, 489, 490, 491, 492, 493, 0, 0, 494, - 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, + 0, 45, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, - 513, 514, 515, 116, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, - 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, - 0, 125, 126, 127, 0, 0, 0, 0, 0, 0, - 0, 0, 128, 129, 130, 0, 131, 132, 133, 134, - 135, 136, 137, 138, 139, 140, 141, 142, 0, 143, - 144, 145, 146, 147, 148, 0, 785, 149, 150, 151, - 152, 153, 154, 155, 0, 156, 157, 158, 159, 786, - 0, 787, 0, 162, 163, 164, 165, 166, 167, 168, - 169, 170, 171, 0, 172, 173, 174, 175, 176, 177, - 0, 178, 179, 180, 181, 182, 183, 184, 185, 186, - 187, 188, 189, 190, 191, 192, 193, 0, 194, 0, - 195, 196, 197, 198, 199, 200, 0, 0, 201, 202, - 203, 204, 0, 0, 205, 206, 207, 208, 209, 0, - 210, 211, 212, 0, 213, 214, 215, 0, 216, 217, - 218, 219, 220, 221, 222, 223, 224, 225, 788, 0, - 227, 0, 228, 229, 230, 231, 0, 232, 0, 233, - 234, 0, 235, 236, 237, 238, 239, 240, 0, 241, - 0, 242, 243, 244, 245, 0, 246, 247, 248, 249, - 250, 251, 252, 253, 254, 255, 256, 257, 0, 258, - 259, 260, 261, 262, 263, 264, 0, 265, 266, 267, - 268, 269, 270, 271, 272, 273, 274, 0, 275, 0, - 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, - 286, 0, 0, 287, 288, 289, 290, 0, 291, 292, - 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, - 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, - 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, - 323, 324, 325, 326, 0, 327, 328, 329, 330, 0, - 790, 332, 333, 334, 335, 336, 0, 337, 338, 0, - 0, 339, 340, 341, 0, 0, 342, 343, 344, 345, - 346, 347, 792, 349, 350, 351, 352, 353, 354, 355, - 356, 357, 358, 359, 0, 0, 0, 0, 360, 361, - 793, 363, 364, 365, 366, 367, 368, 369, 0, 370, - 371, 372, 373, 374, 375, 0, 376, 377, 378, 379, - 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, - 389, 390, 391, 392, 393, 394, 395, 396, 0, 397, - 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, - 408, 409, 410, 411, 412, 413, 414, 415, 416, 0, - 0, 417, 418, 419, 420, 421, 422, 423, 424, 425, - 0, 426, 427, 428, 429, 430, 0, 431, 432, 433, - 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, - 444, 795, 0, 0, 446, 447, 0, 448, 449, 450, - 451, 452, 453, 454, 0, 455, 456, 457, 0, 0, - 458, 459, 796, 461, 797, 0, 463, 464, 798, 466, - 467, 468, 469, 470, 0, 0, 471, 472, 473, 0, - 474, 475, 476, 477, 0, 478, 479, 480, 481, 482, - 483, 484, 0, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, - 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, - 508, 509, 510, 511, 512, 513, 514, 515, 523, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 3305, 0, 0, 117, 118, 119, - 120, 121, 122, 123, 124, 0, 125, 126, 127, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 129, 130, - 0, 131, 132, 133, 0, 135, 136, 137, 138, 139, - 0, 141, 142, 0, 143, 144, 145, 146, 147, 148, - 0, 0, 149, 150, 151, 152, 153, 154, 155, 0, - 156, 157, 158, 159, 160, 0, 0, 0, 162, 163, - 164, 165, 166, 167, 0, 169, 170, 171, 0, 172, - 173, 174, 175, 176, 177, 0, 0, 179, 180, 181, - 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, - 192, 193, 0, 194, 0, 195, 196, 197, 198, 199, - 200, 14, 15, 201, 202, 203, 204, 0, 0, 205, - 206, 207, 208, 209, 0, 210, 211, 212, 0, 213, - 214, 215, 0, 216, 217, 218, 219, 220, 221, 222, - 223, 224, 225, 226, 0, 227, 0, 228, 229, 230, - 231, 0, 232, 0, 233, 0, 23, 0, 236, 237, - 524, 0, 240, 0, 241, 0, 242, 243, 244, 245, - 0, 246, 247, 248, 249, 250, 251, 252, 0, 254, - 255, 256, 257, 0, 258, 259, 260, 261, 262, 263, - 264, 0, 265, 0, 267, 268, 269, 270, 271, 272, - 273, 274, 0, 275, 0, 276, 0, 0, 279, 0, - 281, 282, 283, 284, 285, 286, 0, 0, 287, 0, - 289, 0, 0, 291, 292, 293, 294, 295, 296, 297, - 298, 525, 300, 301, 302, 303, 304, 305, 306, 307, - 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, - 318, 319, 320, 0, 322, 323, 324, 325, 326, 0, - 327, 328, 0, 330, 0, 331, 332, 333, 334, 335, - 336, 0, 337, 338, 0, 0, 339, 340, 341, 0, - 0, 342, 343, 344, 0, 346, 0, 348, 349, 350, - 351, 352, 353, 354, 355, 356, 357, 358, 359, 27, - 28, 29, 0, 360, 361, 362, 0, 364, 365, 366, - 367, 368, 369, 0, 370, 371, 372, 373, 374, 375, - 0, 376, 377, 378, 379, 380, 381, 382, 383, 0, - 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, - 394, 395, 396, 0, 397, 398, 0, 400, 401, 402, - 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, - 413, 414, 415, 416, 34, 0, 417, 418, 419, 420, - 421, 422, 423, 424, 425, 36, 0, 427, 428, 429, - 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, - 439, 440, 441, 442, 526, 444, 445, 38, 0, 446, - 447, 39, 448, 0, 450, 451, 452, 453, 454, 0, - 455, 456, 457, 0, 0, 458, 459, 460, 461, 462, - 0, 463, 464, 465, 466, 467, 468, 469, 470, 0, - 0, 471, 472, 473, 41, 474, 475, 476, 477, 0, - 478, 479, 480, 481, 482, 799, 484, 0, 485, 0, - 487, 488, 489, 490, 491, 492, 493, 0, 0, 494, - 0, 45, 495, 496, 497, 498, 499, 500, 501, 502, - 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, - 513, 514, 515, 523, 0, 46, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, - 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, - 886, 125, 126, 127, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 129, 130, 0, 131, 132, 133, 0, - 135, 136, 137, 138, 139, 0, 141, 142, 0, 143, - 144, 145, 146, 147, 148, 0, 0, 149, 150, 151, - 152, 153, 154, 155, 0, 156, 157, 158, 159, 160, - 0, 0, 0, 162, 163, 164, 165, 166, 167, 0, - 169, 170, 171, 0, 172, 173, 174, 175, 176, 177, - 0, 0, 179, 180, 181, 182, 183, 184, 185, 186, - 187, 188, 189, 190, 191, 192, 193, 0, 194, 0, - 195, 196, 197, 198, 199, 200, 0, 0, 201, 202, - 203, 204, 0, 0, 205, 206, 207, 208, 209, 0, - 210, 211, 212, 0, 213, 214, 215, 0, 216, 217, - 218, 219, 220, 221, 222, 223, 224, 225, 226, 0, - 227, 0, 228, 229, 230, 231, 0, 232, 0, 233, - 0, 23, 0, 236, 237, 524, 0, 240, 0, 241, - 0, 242, 243, 244, 245, 0, 246, 247, 248, 249, - 250, 251, 252, 0, 254, 255, 256, 257, 0, 258, - 259, 260, 261, 262, 263, 264, 0, 265, 0, 267, - 268, 269, 270, 271, 272, 273, 274, 0, 275, 0, - 276, 0, 0, 279, 0, 281, 282, 283, 284, 285, - 286, 0, 0, 287, 0, 289, 0, 0, 291, 292, - 293, 294, 295, 296, 297, 298, 525, 300, 301, 302, - 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, - 313, 314, 315, 316, 317, 318, 319, 320, 0, 322, - 323, 324, 325, 326, 0, 327, 328, 0, 330, 0, - 331, 332, 333, 334, 335, 336, 0, 337, 338, 0, - 0, 339, 340, 341, 0, 0, 342, 343, 344, 0, - 346, 0, 348, 349, 350, 351, 352, 353, 354, 355, - 356, 357, 358, 359, 27, 28, 29, 0, 360, 361, - 362, 0, 364, 365, 366, 367, 368, 369, 0, 370, - 371, 372, 373, 374, 375, 0, 376, 377, 378, 379, - 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, - 389, 390, 391, 392, 393, 394, 395, 396, 0, 397, - 398, 0, 400, 401, 402, 403, 404, 405, 406, 407, - 408, 409, 410, 411, 412, 413, 414, 415, 416, 34, - 0, 417, 418, 419, 420, 421, 422, 423, 424, 425, - 0, 0, 427, 428, 429, 430, 0, 431, 432, 433, - 434, 435, 436, 437, 438, 439, 440, 441, 442, 526, - 444, 445, 0, 0, 446, 447, 39, 448, 0, 450, - 451, 452, 453, 454, 0, 455, 887, 457, 0, 0, - 888, 459, 460, 461, 462, 0, 463, 464, 465, 466, - 467, 468, 469, 470, 0, 0, 471, 472, 473, 41, - 474, 475, 476, 477, 0, 478, 479, 480, 481, 482, - 799, 484, 0, 485, 0, 487, 488, 489, 490, 491, - 492, 493, 0, 0, 494, 0, 45, 495, 496, 497, - 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, - 508, 509, 510, 511, 512, 513, 514, 515, 523, 0, - 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 47, 0, 0, 117, 118, 119, - 120, 121, 122, 123, 124, 0, 125, 126, 127, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 129, 130, - 0, 131, 132, 133, 0, 135, 136, 137, 138, 139, - 0, 141, 142, 0, 143, 144, 145, 146, 147, 148, - 0, 0, 149, 150, 151, 152, 153, 154, 155, 0, - 156, 157, 158, 159, 160, 0, 0, 0, 162, 163, - 164, 165, 166, 167, 0, 169, 170, 171, 0, 172, - 173, 174, 175, 176, 177, 0, 0, 179, 180, 181, - 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, - 192, 193, 0, 194, 0, 195, 196, 197, 198, 199, - 200, 0, 0, 201, 202, 203, 204, 0, 0, 205, - 206, 207, 208, 209, 0, 210, 211, 212, 0, 213, - 214, 215, 0, 216, 217, 218, 219, 220, 221, 222, - 223, 224, 225, 226, 0, 227, 0, 228, 229, 230, - 231, 0, 232, 0, 233, 0, 23, 0, 236, 237, - 524, 0, 240, 0, 241, 0, 242, 243, 244, 245, - 0, 246, 247, 248, 249, 250, 251, 252, 0, 254, - 255, 256, 257, 0, 258, 259, 260, 261, 262, 263, - 264, 0, 265, 0, 267, 268, 269, 270, 271, 272, - 273, 274, 0, 275, 0, 276, 0, 0, 279, 0, - 281, 282, 283, 284, 285, 286, 0, 0, 287, 0, - 289, 0, 0, 291, 292, 293, 294, 295, 296, 297, - 298, 525, 300, 301, 302, 303, 304, 305, 306, 307, - 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, - 318, 319, 320, 0, 322, 323, 324, 325, 326, 0, - 327, 328, 0, 330, 0, 331, 332, 333, 334, 335, - 336, 0, 337, 338, 0, 0, 339, 340, 341, 0, - 0, 342, 343, 344, 0, 346, 0, 348, 349, 350, - 351, 352, 353, 354, 355, 356, 357, 358, 359, 27, - 28, 29, 0, 360, 361, 362, 0, 364, 365, 366, - 367, 368, 369, 0, 370, 371, 372, 373, 374, 375, - 0, 376, 377, 378, 379, 380, 381, 382, 383, 0, - 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, - 394, 395, 396, 0, 397, 398, 0, 400, 401, 402, - 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, - 413, 414, 415, 416, 34, 0, 417, 418, 419, 420, - 421, 422, 423, 424, 425, 0, 0, 427, 428, 429, - 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, - 439, 440, 441, 442, 526, 444, 445, 0, 0, 446, - 447, 39, 448, 0, 450, 451, 452, 453, 454, 0, - 455, 456, 457, 0, 0, 458, 459, 460, 461, 462, - 0, 463, 464, 465, 466, 467, 468, 469, 470, 0, - 0, 471, 472, 473, 41, 474, 475, 476, 477, 0, - 478, 479, 480, 481, 482, 799, 484, 0, 485, 0, - 487, 488, 489, 490, 491, 492, 493, 0, 0, 494, - 0, 45, 495, 496, 497, 498, 499, 500, 501, 502, - 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, - 513, 514, 515, 0, 523, 46, 549, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, + 513, 514, 515, 0, 116, 46, 549, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 800, 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, - 0, 135, 136, 137, 138, 139, 0, 141, 142, 0, - 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, + 0, 0, 0, 128, 129, 130, 0, 131, 132, 133, + 134, 135, 136, 137, 138, 139, 140, 141, 142, 0, + 143, 144, 145, 146, 147, 148, 0, 785, 149, 150, 151, 152, 153, 154, 155, 0, 156, 157, 158, 159, - 160, 0, 0, 0, 162, 163, 164, 165, 166, 167, - 0, 169, 170, 171, 0, 172, 173, 174, 175, 176, - 177, 0, 0, 179, 180, 181, 182, 183, 184, 185, + 786, 0, 787, 0, 162, 163, 164, 165, 166, 167, + 168, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, - 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + 217, 218, 219, 220, 221, 222, 223, 224, 225, 788, 0, 227, 0, 228, 229, 230, 231, 0, 232, 0, - 233, 0, 0, 0, 236, 237, 524, 0, 240, 0, + 233, 234, 0, 235, 236, 237, 238, 239, 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, - 249, 250, 251, 252, 0, 254, 255, 256, 257, 0, - 258, 259, 260, 261, 262, 263, 264, 0, 265, 0, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 0, 275, - 0, 276, 0, 0, 279, 0, 281, 282, 283, 284, - 285, 286, 0, 0, 287, 0, 289, 0, 0, 291, - 292, 293, 294, 295, 296, 297, 298, 525, 300, 301, + 0, 276, 277, 278, 279, 280, 281, 282, 283, 284, + 285, 286, 789, 0, 287, 288, 289, 290, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, 314, 315, 316, 317, 318, 319, 320, 0, - 322, 323, 324, 325, 326, 0, 327, 328, 0, 330, - 0, 331, 332, 333, 334, 335, 336, 0, 337, 338, - 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, - 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, + 322, 323, 324, 325, 326, 0, 327, 328, 329, 330, + 0, 790, 332, 333, 334, 335, 336, 0, 337, 338, + 0, 791, 339, 340, 341, 0, 0, 342, 343, 344, + 345, 346, 347, 792, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, - 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, + 361, 793, 363, 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, - 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, + 397, 398, 399, 400, 401, 402, 403, 794, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, - 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, + 425, 0, 426, 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, - 526, 444, 445, 0, 0, 446, 447, 0, 448, 0, + 443, 444, 795, 0, 0, 446, 447, 0, 448, 449, 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, - 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, + 0, 458, 459, 796, 461, 797, 0, 463, 464, 798, 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, - 482, 483, 484, 0, 485, 0, 487, 488, 489, 490, + 482, 799, 484, 0, 485, 486, 487, 488, 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, - 507, 508, 509, 510, 511, 512, 513, 514, 515, 523, - 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 985, 0, 0, 117, 118, + 507, 508, 509, 510, 511, 512, 513, 514, 515, 116, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 800, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, - 130, 0, 131, 132, 133, 0, 135, 136, 137, 138, - 139, 0, 141, 142, 0, 143, 144, 145, 146, 147, - 148, 0, 0, 149, 150, 151, 152, 153, 154, 155, - 0, 156, 157, 158, 159, 160, 0, 0, 0, 162, - 163, 164, 165, 166, 167, 0, 169, 170, 171, 0, - 172, 173, 174, 175, 176, 177, 0, 0, 179, 180, + 0, 0, 0, 0, 0, 0, 0, 0, 128, 129, + 130, 0, 131, 132, 133, 134, 135, 136, 137, 138, + 139, 140, 141, 142, 0, 143, 144, 145, 146, 147, + 148, 0, 785, 149, 150, 151, 152, 153, 154, 155, + 0, 156, 157, 158, 159, 786, 0, 787, 0, 162, + 163, 164, 165, 166, 167, 168, 169, 170, 171, 0, + 172, 173, 174, 175, 176, 177, 0, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, 220, 221, - 222, 223, 224, 225, 226, 0, 227, 0, 228, 229, - 230, 231, 0, 232, 0, 233, 0, 0, 0, 236, - 237, 524, 0, 240, 0, 241, 0, 242, 243, 244, - 245, 0, 246, 247, 248, 249, 250, 251, 252, 0, + 222, 223, 224, 225, 788, 0, 227, 0, 228, 229, + 230, 231, 0, 232, 0, 233, 234, 0, 235, 236, + 237, 238, 239, 240, 0, 241, 0, 242, 243, 244, + 245, 0, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, - 263, 264, 0, 265, 0, 267, 268, 269, 270, 271, - 272, 273, 274, 0, 275, 0, 276, 0, 0, 279, - 0, 281, 282, 283, 284, 285, 286, 0, 0, 287, - 0, 289, 0, 0, 291, 292, 293, 294, 295, 296, - 297, 298, 525, 300, 301, 302, 303, 304, 305, 306, + 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, + 272, 273, 274, 0, 275, 0, 276, 277, 278, 279, + 280, 281, 282, 283, 284, 285, 286, 0, 0, 287, + 288, 289, 290, 0, 291, 292, 293, 294, 295, 296, + 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, - 317, 318, 319, 320, 0, 322, 323, 324, 325, 326, - 0, 327, 328, 0, 330, 0, 331, 332, 333, 334, + 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, + 0, 327, 328, 329, 330, 0, 790, 332, 333, 334, 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, - 0, 0, 342, 343, 344, 0, 346, 0, 348, 349, + 0, 0, 342, 343, 344, 345, 346, 347, 792, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, - 0, 0, 0, 0, 360, 361, 362, 0, 364, 365, + 0, 0, 0, 0, 360, 361, 793, 363, 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, - 393, 394, 395, 396, 0, 397, 398, 0, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, + 393, 394, 395, 396, 0, 397, 398, 399, 400, 401, + 402, 403, 794, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, - 420, 421, 422, 423, 424, 425, 0, 0, 427, 428, + 420, 421, 422, 423, 424, 425, 0, 426, 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, - 438, 439, 440, 441, 442, 526, 444, 445, 0, 0, - 446, 447, 0, 448, 0, 450, 451, 452, 453, 454, - 0, 455, 456, 457, 0, 0, 458, 459, 460, 461, - 462, 0, 463, 464, 465, 466, 467, 468, 469, 470, + 438, 439, 440, 441, 442, 443, 444, 795, 0, 0, + 446, 447, 0, 448, 449, 450, 451, 452, 453, 454, + 0, 455, 456, 457, 0, 0, 458, 459, 796, 461, + 797, 0, 463, 464, 798, 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, 0, 485, - 0, 487, 488, 489, 490, 491, 492, 493, 0, 0, + 486, 487, 488, 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, - 512, 513, 514, 515, 523, 0, 549, 0, 0, 0, + 512, 513, 514, 515, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1463, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 47, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, - 0, 135, 136, 137, 138, 139, 0, 141, 142, 0, - 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, + 0, 0, 0, 128, 129, 130, 0, 131, 132, 133, + 134, 135, 136, 137, 138, 139, 140, 141, 142, 0, + 143, 144, 145, 146, 147, 148, 0, 785, 149, 150, 151, 152, 153, 154, 155, 0, 156, 157, 158, 159, - 160, 0, 0, 0, 162, 163, 164, 165, 166, 167, - 0, 169, 170, 171, 0, 172, 173, 174, 175, 176, - 177, 0, 0, 179, 180, 181, 182, 183, 184, 185, + 786, 0, 787, 0, 162, 163, 164, 165, 166, 167, + 168, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, - 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + 217, 218, 219, 220, 221, 222, 223, 224, 225, 788, 0, 227, 0, 228, 229, 230, 231, 0, 232, 0, - 233, 0, 0, 0, 236, 237, 524, 0, 240, 0, + 233, 234, 0, 235, 236, 237, 238, 239, 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, - 249, 250, 251, 252, 0, 254, 255, 256, 257, 0, - 258, 259, 260, 261, 262, 263, 264, 0, 265, 0, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 0, 275, - 0, 276, 0, 0, 279, 0, 281, 282, 283, 284, - 285, 286, 0, 0, 287, 0, 289, 0, 0, 291, - 292, 293, 294, 295, 296, 297, 298, 525, 300, 301, + 0, 276, 277, 278, 279, 280, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 288, 289, 290, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, 314, 315, 316, 317, 318, 319, 320, 0, - 322, 323, 324, 325, 326, 0, 327, 328, 0, 330, - 0, 331, 332, 333, 334, 335, 336, 0, 337, 338, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, + 322, 323, 324, 325, 326, 0, 327, 328, 329, 330, + 0, 790, 332, 333, 334, 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, - 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, + 345, 346, 347, 792, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, - 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, + 361, 793, 363, 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, - 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, + 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, - 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, + 425, 0, 426, 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, - 526, 444, 445, 0, 0, 446, 447, 0, 448, 0, + 443, 444, 795, 0, 0, 446, 447, 0, 448, 449, 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, - 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, + 0, 458, 459, 796, 461, 797, 0, 463, 464, 798, 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, - 482, 483, 484, 0, 485, 0, 487, 488, 489, 490, + 482, 483, 484, 0, 485, 486, 487, 488, 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 523, - 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 2200, 0, 0, 117, 118, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 3308, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, 0, 135, 136, 137, 138, @@ -7924,11 +7638,11 @@ static const yytype_int16 yytable[] = 172, 173, 174, 175, 176, 177, 0, 0, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, - 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, + 199, 200, 14, 15, 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 0, 227, 0, 228, 229, - 230, 231, 0, 232, 0, 233, 0, 0, 0, 236, + 230, 231, 0, 232, 0, 233, 0, 23, 0, 236, 237, 524, 0, 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, 249, 250, 251, 252, 0, 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, @@ -7943,28 +7657,28 @@ static const yytype_int16 yytable[] = 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, - 0, 0, 0, 0, 360, 361, 362, 0, 364, 365, + 27, 28, 29, 0, 360, 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, - 420, 421, 422, 423, 424, 425, 0, 0, 427, 428, + 412, 413, 414, 415, 416, 34, 0, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 36, 0, 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, - 438, 439, 440, 441, 442, 526, 444, 445, 0, 0, - 446, 447, 0, 448, 0, 450, 451, 452, 453, 454, + 438, 439, 440, 441, 442, 526, 444, 445, 38, 0, + 446, 447, 39, 448, 0, 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, 466, 467, 468, 469, 470, - 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, - 0, 478, 479, 480, 481, 482, 483, 484, 0, 485, + 0, 0, 471, 472, 473, 41, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 799, 484, 0, 485, 0, 487, 488, 489, 490, 491, 492, 493, 0, 0, - 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, + 494, 0, 45, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, - 512, 513, 514, 515, 523, 0, 549, 0, 0, 0, + 512, 513, 514, 515, 523, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 2436, 0, 0, 117, 118, 119, 120, 121, 122, 123, - 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, + 47, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 886, 125, 126, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, 0, 135, 136, 137, 138, 139, 0, 141, 142, 0, 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, @@ -7978,7 +7692,7 @@ static const yytype_int16 yytable[] = 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 0, 227, 0, 228, 229, 230, 231, 0, 232, 0, - 233, 0, 0, 0, 236, 237, 524, 0, 240, 0, + 233, 0, 23, 0, 236, 237, 524, 0, 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, 249, 250, 251, 252, 0, 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, 265, 0, @@ -7992,27 +7706,27 @@ static const yytype_int16 yytable[] = 0, 331, 332, 333, 334, 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, - 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 355, 356, 357, 358, 359, 27, 28, 29, 0, 360, 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, - 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, + 34, 0, 417, 418, 419, 420, 421, 422, 423, 424, 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, - 526, 444, 445, 0, 0, 446, 447, 0, 448, 0, - 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, - 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, + 526, 444, 445, 0, 0, 446, 447, 39, 448, 0, + 450, 451, 452, 453, 454, 0, 455, 887, 457, 0, + 0, 888, 459, 460, 461, 462, 0, 463, 464, 465, 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, - 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, - 482, 483, 484, 0, 485, 0, 487, 488, 489, 490, - 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, + 41, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 799, 484, 0, 485, 0, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 45, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 523, - 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 2567, 0, 0, 117, 118, + 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 47, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, 0, 135, 136, 137, 138, @@ -8027,7 +7741,7 @@ static const yytype_int16 yytable[] = 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 0, 227, 0, 228, 229, - 230, 231, 0, 232, 0, 233, 0, 0, 0, 236, + 230, 231, 0, 232, 0, 233, 0, 23, 0, 236, 237, 524, 0, 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, 249, 250, 251, 252, 0, 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, @@ -8042,485 +7756,76 @@ static const yytype_int16 yytable[] = 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, - 0, 0, 0, 0, 360, 361, 362, 0, 364, 365, + 27, 28, 29, 0, 360, 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, + 412, 413, 414, 415, 416, 34, 0, 417, 418, 419, 420, 421, 422, 423, 424, 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 526, 444, 445, 0, 0, - 446, 447, 0, 448, 0, 450, 451, 452, 453, 454, + 446, 447, 39, 448, 0, 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, 466, 467, 468, 469, 470, - 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, - 0, 478, 479, 480, 481, 482, 483, 484, 0, 485, + 0, 0, 471, 472, 473, 41, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 799, 484, 0, 485, 0, 487, 488, 489, 490, 491, 492, 493, 0, 0, - 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, + 494, 0, 45, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, - 512, 513, 514, 515, 523, 0, 0, 0, 0, 0, + 512, 513, 514, 515, 0, 523, 46, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 3212, 0, 0, 117, 118, 119, 120, 121, 122, 123, - 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, - 0, 135, 136, 137, 138, 139, 0, 141, 142, 0, - 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, - 151, 152, 153, 154, 155, 0, 156, 157, 158, 159, - 160, 0, 0, 0, 162, 163, 164, 165, 166, 167, - 0, 169, 170, 171, 0, 172, 173, 174, 175, 176, - 177, 0, 0, 179, 180, 181, 182, 183, 184, 185, - 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, - 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, - 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, - 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, - 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, - 0, 227, 0, 228, 229, 230, 231, 0, 232, 0, - 233, 0, 0, 0, 236, 237, 524, 0, 240, 0, - 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, - 249, 250, 251, 252, 0, 254, 255, 256, 257, 0, - 258, 259, 260, 261, 262, 263, 264, 0, 265, 0, - 267, 268, 269, 270, 271, 272, 273, 274, 0, 275, - 0, 276, 0, 0, 279, 0, 281, 282, 283, 284, - 285, 286, 0, 0, 287, 0, 289, 0, 0, 291, - 292, 293, 294, 295, 296, 297, 298, 525, 300, 301, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, 314, 315, 316, 317, 318, 319, 320, 0, - 322, 323, 324, 325, 326, 0, 327, 328, 0, 330, - 0, 331, 332, 333, 334, 335, 336, 0, 337, 338, - 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, - 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, - 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, - 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, - 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, - 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, - 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, - 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, - 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, - 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, - 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, - 526, 444, 445, 0, 0, 446, 447, 0, 448, 0, - 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, - 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, - 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, - 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, - 482, 483, 484, 0, 485, 0, 487, 488, 489, 490, - 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, - 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, - 507, 508, 509, 510, 511, 512, 513, 514, 515, 523, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 2057, 0, 0, 117, 118, - 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, - 130, 0, 131, 132, 133, 0, 135, 136, 137, 138, - 139, 0, 141, 142, 0, 143, 144, 145, 146, 147, - 148, 0, 0, 149, 150, 151, 152, 153, 154, 155, - 0, 156, 157, 158, 159, 160, 0, 0, 0, 162, - 163, 164, 165, 166, 167, 0, 169, 170, 171, 0, - 172, 173, 174, 175, 176, 177, 0, 0, 179, 180, - 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, - 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, - 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, - 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, - 213, 214, 215, 0, 216, 217, 218, 219, 220, 221, - 222, 223, 224, 225, 226, 0, 227, 0, 228, 229, - 230, 231, 0, 232, 0, 233, 0, 0, 0, 236, - 237, 524, 0, 240, 0, 241, 0, 242, 243, 244, - 245, 0, 246, 247, 248, 249, 250, 251, 252, 0, - 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, - 263, 264, 0, 265, 0, 267, 268, 269, 270, 271, - 272, 273, 274, 0, 275, 0, 276, 0, 0, 279, - 0, 281, 282, 283, 284, 285, 286, 0, 0, 287, - 0, 289, 0, 0, 291, 292, 293, 294, 295, 296, - 297, 298, 525, 300, 301, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, - 317, 318, 319, 320, 0, 322, 323, 324, 325, 326, - 0, 327, 328, 0, 330, 0, 331, 332, 333, 334, - 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, - 0, 0, 342, 343, 344, 0, 346, 0, 348, 349, - 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, - 0, 0, 0, 0, 360, 361, 362, 0, 364, 365, - 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, - 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, - 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, - 393, 394, 395, 396, 0, 397, 398, 0, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, - 420, 421, 422, 423, 424, 425, 0, 0, 427, 428, - 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, - 438, 439, 440, 441, 442, 526, 444, 445, 0, 0, - 446, 447, 0, 448, 0, 450, 451, 452, 453, 454, - 0, 455, 456, 457, 0, 0, 458, 459, 460, 461, - 462, 0, 463, 464, 465, 466, 467, 468, 469, 470, - 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, - 0, 478, 479, 480, 481, 482, 483, 484, 0, 485, - 0, 487, 488, 489, 490, 491, 492, 493, 0, 0, - 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, - 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, - 512, 513, 514, 515, 523, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 2160, 0, 0, 117, 118, 119, 120, 121, 122, 123, - 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, - 0, 135, 136, 137, 138, 139, 0, 141, 142, 0, - 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, - 151, 152, 153, 154, 155, 0, 156, 157, 158, 159, - 160, 0, 0, 0, 162, 163, 164, 165, 166, 167, - 0, 169, 170, 171, 0, 172, 173, 174, 175, 176, - 177, 0, 0, 179, 180, 181, 182, 183, 184, 185, - 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, - 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, - 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, - 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, - 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, - 0, 227, 0, 228, 229, 230, 231, 0, 232, 0, - 233, 0, 0, 0, 236, 237, 524, 0, 240, 0, - 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, - 249, 250, 251, 252, 0, 254, 255, 256, 257, 0, - 258, 259, 260, 261, 262, 263, 264, 0, 265, 0, - 267, 268, 269, 270, 271, 272, 273, 274, 0, 275, - 0, 276, 0, 0, 279, 0, 281, 282, 283, 284, - 285, 286, 0, 0, 287, 0, 289, 0, 0, 291, - 292, 293, 294, 295, 296, 297, 298, 525, 300, 301, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, 314, 315, 316, 317, 318, 319, 320, 0, - 322, 323, 324, 325, 326, 0, 327, 328, 0, 330, - 0, 331, 332, 333, 334, 335, 336, 0, 337, 338, - 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, - 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, - 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, - 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, - 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, - 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, - 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, - 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, - 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, - 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, - 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, - 526, 444, 445, 0, 0, 446, 447, 0, 448, 0, - 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, - 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, - 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, - 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, - 482, 483, 484, 0, 485, 0, 487, 488, 489, 490, - 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, - 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, - 507, 508, 509, 510, 511, 512, 513, 514, 515, 0, - 2892, 1340, 815, 0, 0, 2033, 1046, 0, 0, 0, - 0, 0, 2034, 2035, 0, 3073, 2036, 2037, 2038, 117, - 118, 119, 120, 121, 122, 123, 124, 556, 125, 126, - 127, 557, 558, 559, 2893, 561, 562, 563, 564, 2894, - 129, 130, 566, 131, 132, 133, 2895, 135, 136, 137, - 0, 1479, 2896, 1481, 1482, 573, 143, 144, 145, 146, - 147, 148, 574, 575, 149, 150, 151, 152, 1483, 1484, - 155, 578, 156, 157, 158, 159, 0, 580, 2897, 582, - 2898, 163, 164, 165, 166, 167, 2899, 169, 170, 171, - 585, 172, 173, 174, 175, 176, 177, 586, 2900, 179, - 180, 181, 182, 183, 184, 185, 186, 187, 188, 1489, - 190, 191, 1490, 193, 591, 194, 592, 195, 196, 197, - 198, 199, 200, 593, 594, 201, 202, 203, 204, 595, - 596, 205, 206, 1059, 208, 209, 597, 210, 211, 212, - 598, 213, 214, 215, 599, 216, 217, 218, 219, 0, - 221, 222, 223, 224, 225, 0, 602, 227, 603, 228, - 229, 1491, 231, 605, 232, 606, 233, 2901, 608, 2902, - 236, 237, 2903, 2904, 240, 612, 241, 613, 0, 0, - 244, 245, 616, 246, 247, 248, 249, 250, 251, 252, - 2905, 254, 255, 256, 257, 618, 258, 259, 260, 261, - 262, 263, 264, 619, 265, 2906, 0, 268, 269, 270, - 271, 272, 1497, 1498, 624, 1499, 626, 276, 2907, 2908, - 279, 2909, 281, 282, 283, 284, 285, 286, 630, 631, - 287, 2910, 289, 2911, 634, 291, 292, 293, 294, 295, - 296, 297, 298, 2912, 300, 301, 302, 303, 304, 305, - 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, - 316, 317, 318, 319, 1506, 2913, 1508, 323, 324, 325, - 2914, 640, 327, 328, 2915, 330, 642, 0, 332, 1510, - 334, 335, 336, 645, 337, 338, 646, 647, 2916, 340, - 341, 648, 649, 342, 343, 0, 2917, 346, 2918, 0, - 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, - 359, 654, 655, 656, 657, 360, 361, 0, 2919, 364, - 365, 0, 367, 368, 369, 661, 370, 371, 372, 373, - 374, 375, 662, 376, 377, 378, 1514, 380, 381, 382, - 383, 664, 384, 385, 386, 387, 388, 389, 390, 391, - 392, 393, 394, 395, 396, 665, 397, 398, 2920, 400, - 401, 402, 1516, 404, 405, 406, 407, 408, 409, 410, - 411, 412, 413, 414, 415, 416, 668, 2921, 417, 418, - 419, 420, 421, 422, 2922, 424, 425, 671, 2923, 427, - 428, 1520, 430, 674, 431, 432, 433, 434, 435, 436, - 437, 438, 439, 440, 441, 442, 2924, 444, 0, 677, - 678, 446, 447, 679, 448, 2925, 450, 451, 452, 453, - 454, 681, 455, 1523, 1524, 684, 685, 458, 459, 0, - 461, 0, 688, 463, 464, 2926, 466, 467, 468, 469, - 470, 2927, 691, 471, 472, 473, 692, 474, 475, 476, - 477, 693, 478, 479, 480, 481, 482, 0, 1527, 696, - 485, 2928, 487, 488, 489, 490, 491, 492, 493, 698, - 699, 494, 700, 701, 495, 496, 497, 498, 499, 500, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 512, 513, 514, 515, 0, 523, 0, 2039, 2040, - 2041, 2033, 2929, 2930, 2044, 2045, 2046, 2047, 2034, 2035, - 0, 0, 2036, 2037, 2038, 117, 118, 119, 120, 121, - 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 129, 130, 0, 131, - 132, 133, 0, 135, 136, 137, 138, 139, 0, 141, - 142, 0, 143, 144, 145, 146, 147, 148, 0, 0, - 149, 150, 151, 152, 153, 154, 155, 0, 156, 157, - 158, 159, 160, 0, 0, 0, 162, 163, 164, 165, - 166, 167, 0, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 0, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, - 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 226, 0, 227, 0, 228, 229, 230, 231, 0, - 232, 0, 233, 0, 0, 0, 236, 237, 524, 0, - 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, - 247, 248, 249, 250, 251, 252, 0, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 0, 267, 268, 269, 270, 271, 272, 273, 274, - 0, 275, 0, 276, 0, 0, 279, 0, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 0, 289, 0, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 525, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 0, 322, 323, 324, 325, 326, 0, 327, 328, - 0, 330, 0, 331, 332, 333, 334, 335, 336, 0, - 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, - 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 362, 0, 364, 365, 366, 367, 368, - 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 0, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 0, 0, 427, 428, 429, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 526, 444, 445, 0, 0, 446, 447, 0, - 448, 0, 450, 451, 452, 453, 454, 0, 455, 456, - 457, 0, 0, 458, 459, 460, 461, 462, 0, 463, - 464, 465, 466, 467, 468, 469, 470, 0, 0, 471, - 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 483, 484, 0, 485, 0, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, - 515, 0, 0, 0, 2039, 2040, 2041, 0, 2042, 2043, - 2044, 2045, 2046, 2047, 1610, 0, 0, 1611, 0, 0, - 0, 1612, 1613, 1614, 1615, 1616, 1617, 1618, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1619, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1621, 1610, 0, 0, 1611, 0, 0, 1622, - 1612, 1613, 1614, 1615, 1616, 1617, 1618, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1619, 0, 0, 0, 0, 1623, 0, 0, 0, - 0, 1621, 1610, 0, 0, 1611, 0, 0, 1622, 1612, - 1613, 1614, 1615, 1616, 1617, 1618, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1619, 0, 0, 0, 0, 1623, 0, 0, 0, 0, - 1621, 0, 0, 0, 0, 0, 0, 1622, 0, 1610, - 0, 0, 1611, 0, 0, 0, 1612, 1613, 1614, 1615, - 1616, 1617, 1618, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1623, 0, 0, 1619, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1621, 1610, 0, - 0, 1611, 1624, 0, 1622, 1612, 1613, 1614, 1615, 1616, - 1617, 1618, 0, 0, 0, 0, 0, 0, 0, 1625, - 0, 0, 0, 0, 1626, 0, 1619, 0, 0, 0, - 0, 1623, 0, 0, 0, 0, 1621, 0, 0, 0, - 0, 1624, 0, 1622, 0, 0, 0, 1627, 1628, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1625, 0, - 0, 0, 1629, 1626, 0, 0, 0, 0, 0, 0, - 1623, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1624, 0, 0, 0, 0, 0, 1627, 1628, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1625, 0, 0, - 1630, 1629, 1626, 1631, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1632, 0, 0, - 1633, 0, 0, 0, 0, 1627, 1628, 1624, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1630, - 1629, 0, 1631, 0, 1625, 0, 0, 0, 0, 1626, - 0, 0, 0, 0, 0, 0, 1632, 0, 0, 1633, - 0, 0, 0, 0, 0, 0, 1624, 0, 0, 0, - 0, 0, 1627, 1628, 0, 0, 0, 0, 1630, 0, - 0, 1631, 0, 1625, 0, 0, 0, 1629, 1626, 0, - 0, 0, 0, 0, 0, 1632, 0, 0, 1633, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1627, 1628, 0, 0, 0, 0, 1634, 0, 0, - 0, 0, 0, 0, 0, 1630, 1629, 0, 1631, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1632, 0, 0, 1633, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1634, 0, 0, 0, - 0, 0, 0, 0, 1630, 0, 0, 1631, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1632, 0, 0, 1633, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1634, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1635, 0, 0, 1636, 1637, 1638, - 0, 1639, 1640, 1641, 1642, 1643, 1644, 0, 0, 0, - 0, 2838, 1634, 0, 0, 0, 0, 0, 1610, 0, - 0, 1611, 0, 0, 0, 1612, 1613, 1614, 1615, 1616, - 1617, 1618, 0, 1635, 0, 0, 1636, 1637, 1638, 0, - 1639, 1640, 1641, 1642, 1643, 1644, 1619, 0, 0, 0, - 3065, 1634, 0, 0, 0, 0, 1621, 0, 0, 0, - 0, 0, 0, 1622, 0, 0, 0, 0, 0, 0, - 0, 0, 1635, 0, 0, 1636, 1637, 1638, 0, 1639, - 1640, 1641, 1642, 1643, 1644, 0, 0, 0, 0, 3072, - 1623, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1610, 0, 0, 1611, 0, 0, - 0, 1612, 1613, 1614, 1615, 1616, 1617, 1618, 0, 1635, - 0, 0, 1636, 1637, 1638, 0, 1639, 1640, 1641, 1642, - 1643, 1644, 1619, 0, 0, 0, 3232, 0, 0, 0, - 0, 0, 1621, 1610, 0, 0, 1611, 0, 0, 1622, - 1612, 1613, 1614, 1615, 1616, 1617, 1618, 0, 1635, 0, - 0, 1636, 1637, 1638, 0, 1639, 1640, 1641, 1642, 1643, - 1644, 1619, 0, 0, 0, 3254, 1623, 0, 0, 0, - 0, 1621, 1610, 0, 0, 1611, 1624, 0, 1622, 1612, - 1613, 1614, 1615, 1616, 1617, 1618, 0, 0, 0, 0, - 0, 0, 0, 1625, 0, 0, 0, 0, 1626, 0, - 1619, 0, 0, 0, 0, 1623, 0, 0, 0, 0, - 1621, 0, 0, 0, 0, 0, 0, 1622, 0, 0, - 0, 1627, 1628, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1629, 0, 0, 0, - 0, 0, 0, 0, 1623, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1610, 0, - 0, 1611, 1624, 0, 0, 1612, 1613, 1614, 1615, 1616, - 1617, 1618, 0, 0, 1630, 0, 0, 1631, 0, 1625, - 0, 0, 0, 0, 1626, 0, 1619, 0, 0, 0, - 0, 1632, 0, 0, 1633, 0, 1621, 0, 0, 0, - 0, 1624, 0, 1622, 0, 0, 0, 1627, 1628, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1625, 0, - 0, 0, 1629, 1626, 0, 0, 0, 0, 0, 0, - 1623, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1624, 0, 0, 0, 0, 0, 1627, 1628, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1625, 0, 0, - 1630, 1629, 1626, 1631, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1632, 0, 0, - 1633, 0, 0, 0, 0, 1627, 1628, 0, 0, 0, - 0, 1634, 0, 0, 0, 0, 0, 0, 0, 1630, - 1629, 0, 1631, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1632, 0, 0, 1633, - 0, 0, 0, 0, 0, 0, 1624, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1630, 0, - 0, 1631, 0, 1625, 0, 0, 0, 0, 1626, 0, - 0, 0, 0, 0, 0, 1632, 0, 0, 1633, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1627, 1628, 0, 0, 0, 0, 1634, 0, 0, - 0, 0, 0, 0, 0, 0, 1629, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1635, 0, - 0, 1636, 1637, 1638, 0, 1639, 1640, 1641, 1642, 1643, - 1644, 0, 0, 0, 0, 3355, 1634, 0, 0, 0, - 0, 0, 0, 0, 1630, 0, 1610, 1631, 0, 1611, - 0, 0, 0, 1612, 1613, 1614, 1615, 1616, 1617, 1618, - 0, 1632, 0, 0, 1633, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1619, 1634, 717, 0, 0, 0, - 0, 0, 0, 0, 1621, 0, 0, 0, 0, 0, - 0, 1622, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1635, 0, 0, 1636, 1637, 1638, - 0, 1639, 1640, 1641, 1642, 1643, 1644, 0, 1623, 0, - 0, 3412, 0, 0, 0, 0, 0, 0, 1610, 0, - 0, 1611, 0, 718, 0, 1612, 1613, 1614, 1615, 1616, - 1617, 1618, 0, 1635, 0, 0, 1636, 1637, 1638, 719, - 1639, 1640, 1641, 1642, 1643, 1644, 1619, 0, 0, 0, - 3434, 1634, 0, 0, 0, 0, 1621, 0, 0, 0, - 0, 0, 0, 1622, 0, 0, 0, 0, 0, 0, - 0, 0, 1635, 0, 0, 1636, 1637, 1638, 0, 1639, - 1640, 1641, 1642, 1643, 1644, 0, 0, 1795, 720, 0, - 1623, 0, 0, 0, 0, 0, 0, 0, 721, 0, - 0, 0, 0, 0, 1624, 0, 0, 0, 0, 0, - 722, 0, 0, 0, 0, 723, 0, 0, 0, 0, - 0, 1625, 0, 0, 0, 0, 1626, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 724, 0, 0, 0, 0, 1627, - 1628, 0, 0, 0, 0, 0, 0, 0, 1635, 0, - 0, 1636, 1637, 1638, 1629, 1639, 1640, 1641, 1642, 1643, - 1644, 0, 0, 2792, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1624, 725, 0, 0, - 0, 726, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1630, 1625, 0, 1631, 0, 0, 1626, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1632, - 0, 0, 1633, 0, 0, 0, 0, 0, 0, 0, - 0, 1627, 1628, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1629, 0, 0, 0, - 0, 0, 0, 0, 0, 539, 0, 0, 0, 0, - 0, 727, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 728, 0, 0, 0, - 0, 0, 0, 0, 1630, 0, 0, 1631, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1632, 0, 0, 1633, 0, 0, 0, 0, 729, - 0, 0, 730, 0, 0, 0, 0, 0, 0, 1634, - 0, 0, 0, 731, 0, 0, 732, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 733, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 734, 0, - 0, 0, 0, 0, 735, 736, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 737, 0, 0, 0, 0, - 0, 738, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1634, 0, 0, 0, 0, 0, 0, 739, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1635, 0, 0, 1636, - 1637, 1638, 0, 1639, 1640, 1641, 1642, 1643, 1644, 0, - 0, 3222, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 555, 0, 0, 1635, 0, - 0, 1636, 1637, 1638, 0, 1639, 1640, 1641, 1642, 1643, - 1644, 0, 0, 3396, 117, 118, 119, 120, 121, 122, - 123, 124, 556, 125, 126, 127, 557, 558, 559, 560, - 561, 562, 563, 564, 565, 129, 130, 566, 131, 132, - 133, 567, 135, 136, 137, 568, 569, 570, 571, 572, - 573, 143, 144, 145, 146, 147, 148, 574, 575, 149, - 150, 151, 152, 576, 577, 155, 578, 156, 157, 158, - 159, 579, 580, 581, 582, 583, 163, 164, 165, 166, - 167, 584, 169, 170, 171, 585, 172, 173, 174, 175, - 176, 177, 586, 587, 179, 180, 181, 182, 183, 184, - 185, 186, 187, 188, 589, 190, 191, 590, 193, 591, - 194, 592, 195, 196, 197, 198, 199, 200, 593, 594, - 201, 202, 203, 204, 595, 596, 205, 206, 207, 208, - 209, 597, 210, 211, 212, 598, 213, 214, 215, 599, - 216, 217, 218, 219, 600, 221, 222, 223, 224, 225, - 601, 602, 227, 603, 228, 229, 604, 231, 605, 232, - 606, 233, 607, 608, 609, 236, 237, 610, 611, 240, - 612, 241, 613, 614, 615, 244, 245, 616, 246, 247, - 248, 249, 250, 251, 252, 617, 254, 255, 256, 257, - 618, 258, 259, 260, 261, 262, 263, 264, 619, 265, - 620, 621, 268, 269, 270, 271, 272, 622, 623, 624, - 625, 626, 276, 627, 628, 279, 629, 281, 282, 283, - 284, 285, 286, 630, 631, 287, 632, 289, 633, 634, - 291, 292, 293, 294, 295, 296, 297, 298, 635, 300, + 47, 0, 0, 0, 117, 118, 119, 120, 121, 122, + 123, 124, 0, 125, 126, 127, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 129, 130, 0, 131, 132, + 133, 0, 135, 136, 137, 138, 139, 0, 141, 142, + 0, 143, 144, 145, 146, 147, 148, 0, 0, 149, + 150, 151, 152, 153, 154, 155, 0, 156, 157, 158, + 159, 160, 0, 0, 0, 162, 163, 164, 165, 166, + 167, 0, 169, 170, 171, 0, 172, 173, 174, 175, + 176, 177, 0, 0, 179, 180, 181, 182, 183, 184, + 185, 186, 187, 188, 189, 190, 191, 192, 193, 0, + 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, + 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, + 209, 0, 210, 211, 212, 0, 213, 214, 215, 0, + 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, + 226, 0, 227, 0, 228, 229, 230, 231, 0, 232, + 0, 233, 0, 0, 0, 236, 237, 524, 0, 240, + 0, 241, 0, 242, 243, 244, 245, 0, 246, 247, + 248, 249, 250, 251, 252, 0, 254, 255, 256, 257, + 0, 258, 259, 260, 261, 262, 263, 264, 0, 265, + 0, 267, 268, 269, 270, 271, 272, 273, 274, 0, + 275, 0, 276, 0, 0, 279, 0, 281, 282, 283, + 284, 285, 286, 0, 0, 287, 0, 289, 0, 0, + 291, 292, 293, 294, 295, 296, 297, 298, 525, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, - 311, 312, 313, 314, 315, 316, 317, 318, 319, 636, - 637, 638, 323, 324, 325, 639, 640, 327, 328, 641, - 330, 642, 643, 332, 644, 334, 335, 336, 645, 337, - 338, 646, 647, 339, 340, 341, 648, 649, 342, 343, - 650, 651, 346, 652, 653, 349, 350, 351, 352, 353, - 354, 355, 356, 357, 358, 359, 654, 655, 656, 657, - 360, 361, 658, 659, 364, 365, 660, 367, 368, 369, - 661, 370, 371, 372, 373, 374, 375, 662, 376, 377, - 378, 663, 380, 381, 382, 383, 664, 384, 385, 386, + 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, + 0, 322, 323, 324, 325, 326, 0, 327, 328, 0, + 330, 0, 331, 332, 333, 334, 335, 336, 0, 337, + 338, 0, 0, 339, 340, 341, 0, 0, 342, 343, + 344, 0, 346, 0, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, 358, 359, 0, 0, 0, 0, + 360, 361, 362, 0, 364, 365, 366, 367, 368, 369, + 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, + 378, 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, - 665, 397, 398, 666, 400, 401, 402, 667, 404, 405, + 0, 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, - 416, 668, 669, 417, 418, 419, 420, 421, 422, 670, - 424, 425, 671, 672, 427, 428, 673, 430, 674, 431, + 416, 0, 0, 417, 418, 419, 420, 421, 422, 423, + 424, 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, - 442, 675, 444, 676, 677, 678, 446, 447, 679, 448, - 680, 450, 451, 452, 453, 454, 681, 455, 682, 683, - 684, 685, 458, 459, 686, 461, 687, 688, 463, 464, - 689, 466, 467, 468, 469, 470, 690, 691, 471, 472, - 473, 692, 474, 475, 476, 477, 693, 478, 479, 480, - 481, 482, 694, 695, 696, 485, 697, 487, 488, 489, - 490, 491, 492, 493, 698, 699, 494, 700, 701, 495, - 496, 497, 498, 499, 500, 702, 703, 704, 705, 706, - 707, 708, 709, 710, 711, 712, 512, 513, 514, 515, - 523, 0, 0, 0, 0, 0, 0, 0, 0, 2068, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, + 442, 526, 444, 445, 0, 0, 446, 447, 0, 448, + 0, 450, 451, 452, 453, 454, 0, 455, 456, 457, + 0, 0, 458, 459, 460, 461, 462, 0, 463, 464, + 465, 466, 467, 468, 469, 470, 0, 0, 471, 472, + 473, 0, 474, 475, 476, 477, 0, 478, 479, 480, + 481, 482, 483, 484, 0, 485, 0, 487, 488, 489, + 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, + 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, + 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, + 523, 0, 549, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 985, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, 0, 135, 136, 137, @@ -8568,9 +7873,9 @@ static const yytype_int16 yytable[] = 485, 0, 487, 488, 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, - 511, 512, 513, 514, 515, 523, 0, 0, 0, 0, - 0, 0, 0, 0, 2692, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 117, 118, 119, 120, 121, 122, + 511, 512, 513, 514, 515, 523, 0, 549, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1464, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, 0, 135, 136, 137, 138, 139, 0, 141, 142, @@ -8618,413 +7923,566 @@ static const yytype_int16 yytable[] = 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, - 964, 1340, 815, 0, 0, 0, 1046, 0, 0, 2695, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, + 523, 0, 549, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 2201, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, - 127, 0, 0, 0, 560, 0, 0, 0, 0, 565, - 129, 130, 0, 131, 132, 133, 567, 135, 136, 137, - 568, 569, 570, 571, 572, 0, 143, 144, 145, 146, - 147, 148, 0, 0, 149, 150, 151, 152, 576, 577, - 155, 0, 156, 157, 158, 159, 579, 0, 581, 0, - 583, 163, 164, 165, 166, 167, 584, 169, 170, 171, - 0, 172, 173, 174, 175, 176, 177, 0, 587, 179, - 180, 181, 182, 183, 184, 185, 186, 187, 188, 589, - 190, 191, 590, 193, 0, 194, 0, 195, 196, 197, + 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 129, 130, 0, 131, 132, 133, 0, 135, 136, 137, + 138, 139, 0, 141, 142, 0, 143, 144, 145, 146, + 147, 148, 0, 0, 149, 150, 151, 152, 153, 154, + 155, 0, 156, 157, 158, 159, 160, 0, 0, 0, + 162, 163, 164, 165, 166, 167, 0, 169, 170, 171, + 0, 172, 173, 174, 175, 176, 177, 0, 0, 179, + 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, + 190, 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, 212, - 0, 213, 214, 215, 0, 216, 217, 218, 219, 600, - 221, 222, 223, 224, 225, 601, 1341, 227, 0, 228, - 229, 604, 231, 0, 232, 0, 233, 607, 0, 609, - 236, 237, 610, 611, 240, 0, 241, 0, 614, 615, - 244, 245, 0, 246, 247, 248, 249, 250, 251, 252, - 617, 254, 255, 256, 257, 0, 258, 259, 260, 261, - 262, 263, 264, 0, 265, 620, 621, 268, 269, 270, - 271, 272, 622, 623, 0, 625, 0, 276, 627, 628, - 279, 629, 281, 282, 283, 284, 285, 286, 0, 0, - 287, 632, 289, 633, 0, 291, 292, 293, 294, 295, - 296, 297, 298, 635, 300, 301, 302, 303, 304, 305, - 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, - 316, 317, 318, 319, 636, 637, 638, 323, 324, 325, - 639, 0, 327, 328, 641, 330, 0, 643, 332, 644, - 334, 335, 336, 0, 337, 338, 1342, 0, 339, 340, - 341, 0, 0, 342, 343, 650, 651, 346, 652, 653, + 0, 213, 214, 215, 0, 216, 217, 218, 219, 220, + 221, 222, 223, 224, 225, 226, 0, 227, 0, 228, + 229, 230, 231, 0, 232, 0, 233, 0, 0, 0, + 236, 237, 524, 0, 240, 0, 241, 0, 242, 243, + 244, 245, 0, 246, 247, 248, 249, 250, 251, 252, + 0, 254, 255, 256, 257, 0, 258, 259, 260, 261, + 262, 263, 264, 0, 265, 0, 267, 268, 269, 270, + 271, 272, 273, 274, 0, 275, 0, 276, 0, 0, + 279, 0, 281, 282, 283, 284, 285, 286, 0, 0, + 287, 0, 289, 0, 0, 291, 292, 293, 294, 295, + 296, 297, 298, 525, 300, 301, 302, 303, 304, 305, + 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, + 316, 317, 318, 319, 320, 0, 322, 323, 324, 325, + 326, 0, 327, 328, 0, 330, 0, 331, 332, 333, + 334, 335, 336, 0, 337, 338, 0, 0, 339, 340, + 341, 0, 0, 342, 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, - 359, 0, 0, 0, 0, 360, 361, 658, 659, 364, - 365, 660, 367, 368, 369, 0, 370, 371, 372, 373, - 374, 375, 0, 376, 377, 378, 663, 380, 381, 382, + 359, 0, 0, 0, 0, 360, 361, 362, 0, 364, + 365, 366, 367, 368, 369, 0, 370, 371, 372, 373, + 374, 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, - 392, 393, 394, 395, 396, 0, 397, 398, 666, 400, - 401, 402, 667, 404, 405, 406, 407, 408, 409, 410, - 411, 412, 413, 414, 415, 416, 0, 669, 417, 418, - 419, 420, 421, 422, 670, 424, 425, 0, 672, 427, - 428, 673, 430, 0, 431, 432, 433, 434, 435, 436, - 437, 438, 439, 440, 441, 442, 675, 444, 676, 0, - 0, 446, 447, 0, 448, 680, 450, 451, 452, 453, - 454, 0, 455, 682, 683, 0, 0, 458, 459, 686, - 461, 687, 1343, 463, 464, 689, 466, 467, 468, 469, + 392, 393, 394, 395, 396, 0, 397, 398, 0, 400, + 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, + 411, 412, 413, 414, 415, 416, 0, 0, 417, 418, + 419, 420, 421, 422, 423, 424, 425, 0, 0, 427, + 428, 429, 430, 0, 431, 432, 433, 434, 435, 436, + 437, 438, 439, 440, 441, 442, 526, 444, 445, 0, + 0, 446, 447, 0, 448, 0, 450, 451, 452, 453, + 454, 0, 455, 456, 457, 0, 0, 458, 459, 460, + 461, 462, 0, 463, 464, 465, 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, 476, - 477, 0, 478, 479, 480, 481, 482, 694, 695, 0, - 485, 697, 487, 488, 489, 490, 491, 492, 493, 0, + 477, 0, 478, 479, 480, 481, 482, 483, 484, 0, + 485, 0, 487, 488, 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, - 702, 703, 704, 705, 706, 707, 708, 709, 710, 711, - 712, 512, 513, 514, 515, 0, 0, 1610, 0, 0, - 1611, 0, 1344, 1345, 1612, 1613, 1614, 1615, 1616, 1617, - 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1619, 0, 0, 0, 0, - 2166, 0, 0, 0, 0, 1621, 1610, 0, 0, 1611, - 0, 0, 1622, 1612, 1613, 1614, 1615, 1616, 1617, 1618, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1619, 0, 0, 0, 0, 1623, - 0, 0, 0, 0, 1621, 1610, 0, 0, 1611, 0, - 0, 1622, 1612, 1613, 1614, 1615, 1616, 1617, 1618, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1619, 0, 0, 0, 0, 1623, 0, - 0, 0, 0, 1621, 0, 2167, 0, 0, 0, 0, - 1622, 0, 0, 1610, 0, 0, 1611, 0, 0, 0, - 1612, 1613, 1614, 1615, 1616, 1617, 1618, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1623, 0, 0, - 0, 1619, 0, 0, 0, 1889, 0, 0, 0, 0, - 0, 1621, 0, 1610, 0, 1624, 1611, 0, 1622, 0, - 1612, 1613, 1614, 1615, 1616, 1617, 1618, 0, 0, 0, - 0, 0, 1625, 0, 0, 0, 0, 1626, 0, 0, - 0, 1619, 0, 0, 1925, 1623, 0, 0, 0, 1926, - 0, 1621, 0, 0, 1624, 0, 0, 0, 1622, 0, - 1627, 1628, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1625, 0, 0, 0, 1629, 1626, 0, 0, 0, - 0, 0, 3502, 0, 0, 1623, 0, 0, 0, 0, - 0, 0, 0, 1624, 0, 0, 0, 0, 0, 1627, - 1628, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1625, 0, 0, 1630, 1629, 1626, 1631, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1632, 0, 0, 1633, 0, 0, 0, 0, 1627, 1628, - 0, 1624, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1630, 1629, 0, 1631, 0, 0, 1625, 0, - 0, 0, 0, 1626, 0, 0, 0, 0, 0, 1632, - 0, 0, 1633, 0, 0, 0, 0, 0, 0, 0, - 0, 1624, 0, 0, 0, 0, 1627, 1628, 0, 0, - 0, 1630, 0, 0, 1631, 0, 0, 0, 1625, 0, - 0, 1629, 0, 1626, 0, 0, 0, 0, 1632, 0, - 0, 1633, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1627, 1628, 0, 0, - 1634, 0, 0, 0, 0, 0, 0, 0, 0, 1630, - 3503, 1629, 1631, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1632, 0, 0, 1633, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1634, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1630, - 0, 0, 1631, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 2170, 0, 0, 1632, 0, 0, 1633, - 0, 0, 0, 0, 0, 0, 0, 0, 1634, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1896, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1635, 0, 0, - 1636, 1637, 1638, 0, 1639, 1640, 1641, 1642, 1643, 1644, - 0, 0, 0, 0, 0, 0, 1634, 0, 0, 0, - 0, 1610, 0, 0, 1611, 0, 0, 0, 1612, 1613, - 1614, 1615, 1616, 1617, 1618, 0, 1635, 0, 0, 1636, - 1637, 1638, 0, 1639, 1640, 1641, 1642, 1643, 1644, 1619, - 0, 0, 0, 1931, 0, 0, 1634, 0, 0, 1621, - 0, 0, 0, 0, 0, 0, 1622, 0, 0, 0, - 0, 0, 0, 0, 0, 1635, 0, 0, 1636, 1637, - 1638, 0, 1639, 1640, 1641, 1642, 1643, 1644, 0, 0, - 0, 0, 0, 1623, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1610, 0, - 0, 1611, 0, 0, 0, 1612, 1613, 1614, 1615, 1616, - 1617, 1618, 0, 1635, 0, 0, 1636, 1637, 1638, 0, - 1639, 1640, 1641, 1642, 1643, 1644, 1619, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1621, 0, 0, 0, - 0, 0, 0, 1622, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1635, 0, 0, 1636, 1637, 1638, 0, - 1639, 1640, 1641, 1642, 1643, 1644, 0, 0, 0, 0, - 1623, 0, 0, 0, 0, 1610, 0, 0, 1611, 1624, - 0, 0, 1612, 1613, 1614, 1615, 1616, 1617, 1618, 0, - 0, 0, 0, 0, 0, 0, 1625, 0, 0, 0, - 0, 1626, 0, 1619, 0, 0, 0, 1938, 0, 0, - 0, 0, 0, 1621, 0, 0, 0, 0, 0, 0, - 1622, 0, 0, 0, 1627, 1628, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1629, - 0, 0, 0, 0, 0, 0, 0, 1623, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1936, 1610, 0, 0, 1611, 1624, 0, 0, 1612, - 1613, 1614, 1615, 1616, 1617, 1618, 0, 1630, 0, 0, - 1631, 0, 0, 1625, 0, 0, 0, 0, 1626, 0, - 1619, 0, 0, 0, 1632, 0, 0, 1633, 0, 0, - 1621, 0, 0, 0, 0, 0, 0, 1622, 0, 0, - 0, 1627, 1628, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1629, 0, 0, 0, - 0, 0, 0, 0, 1623, 0, 0, 0, 0, 0, - 0, 0, 0, 1624, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1625, 0, 0, 0, 1630, 1626, 0, 1631, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1632, 0, 0, 1633, 0, 0, 0, 1627, 1628, - 0, 0, 0, 0, 1634, 0, 0, 1610, 0, 0, - 1611, 0, 0, 1629, 1612, 1613, 1614, 1615, 1616, 1617, - 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 2061, 0, 0, 0, 0, 1619, 0, 0, 0, 0, - 1624, 0, 0, 0, 0, 1621, 0, 0, 0, 0, - 0, 1630, 1622, 0, 1631, 0, 0, 1625, 0, 0, - 0, 0, 1626, 0, 0, 0, 0, 0, 1632, 0, - 0, 1633, 0, 0, 0, 0, 0, 0, 0, 1623, - 0, 0, 0, 0, 0, 1627, 1628, 0, 0, 0, - 0, 1634, 0, 0, 0, 0, 0, 0, 0, 0, - 1629, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1635, 0, 0, 1636, 1637, 1638, 0, 1639, 1640, - 1641, 1642, 1643, 1644, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1630, 0, - 0, 1631, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1632, 0, 0, 1633, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1634, 0, - 0, 0, 0, 0, 0, 1624, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1625, 0, 0, 0, 0, 1626, 1635, 0, - 0, 1636, 1637, 1638, 0, 1639, 1640, 1641, 1642, 1643, - 1644, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1627, 1628, 0, 0, 0, 0, 0, 0, 0, 1610, - 0, 0, 1611, 0, 0, 1629, 1612, 1613, 1614, 1615, - 1616, 1617, 1618, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1634, 0, 1619, 0, 0, - 0, 2769, 0, 0, 0, 0, 0, 1621, 0, 0, - 0, 0, 0, 1630, 1622, 1635, 1631, 0, 1636, 1637, - 1638, 0, 1639, 1640, 1641, 1642, 1643, 1644, 0, 0, - 1632, 0, 0, 1633, 0, 0, 0, 0, 0, 0, - 0, 1623, 0, 1610, 0, 0, 1611, 0, 0, 0, - 1612, 1613, 1614, 1615, 1616, 1617, 1618, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1619, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1621, 0, 0, 0, 0, 0, 0, 1622, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1635, 0, 0, 1636, 1637, 1638, 0, 1639, - 1640, 1641, 1642, 1643, 1644, 1623, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1634, 0, 1610, 0, 0, 1611, 0, 1624, 0, 1612, - 1613, 1614, 1615, 1616, 1617, 1618, 0, 0, 0, 0, - 0, 0, 0, 0, 1625, 0, 0, 0, 0, 1626, - 1619, 0, 0, 2756, 0, 0, 0, 0, 0, 0, - 1621, 0, 0, 0, 0, 0, 0, 1622, 0, 0, - 0, 0, 1627, 1628, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1629, 0, 0, - 0, 0, 0, 0, 1623, 0, 0, 0, 0, 0, - 0, 1624, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1625, 0, - 0, 0, 0, 1626, 0, 1630, 0, 1635, 1631, 0, - 1636, 1637, 1638, 0, 1639, 1640, 1641, 1642, 1643, 1644, - 0, 0, 1632, 0, 0, 1633, 1627, 1628, 0, 0, - 0, 0, 0, 0, 0, 1610, 0, 0, 1611, 0, - 0, 1629, 1612, 1613, 1614, 1615, 1616, 1617, 1618, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1619, 0, 0, 0, 0, 0, 0, - 1624, 0, 0, 1621, 0, 0, 0, 0, 0, 1630, - 1622, 0, 1631, 0, 0, 0, 0, 1625, 0, 0, - 0, 0, 1626, 0, 0, 0, 1632, 0, 0, 1633, - 0, 0, 0, 0, 0, 0, 1610, 1623, 0, 1611, - 0, 0, 0, 1612, 1613, 1805, 1628, 1616, 1617, 1618, - 0, 0, 1634, 0, 0, 0, 0, 0, 0, 0, - 1629, 0, 0, 0, 1619, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1621, 0, 0, 0, 0, 0, - 0, 1622, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1630, 0, - 0, 1631, 0, 0, 0, 0, 0, 0, 1623, 0, - 0, 0, 0, 0, 0, 1632, 0, 0, 1633, 0, - 0, 0, 0, 0, 0, 0, 1634, 1610, 0, 0, - 1611, 0, 0, 1624, 1612, 1613, 0, 0, 1616, 1617, - 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1625, 0, 0, 0, 0, 1626, 0, 0, 0, 1635, - 0, 0, 1636, 1637, 1638, 1621, 1639, 1640, 1641, 1642, - 1643, 1644, 1622, 0, 0, 0, 0, 0, 1627, 1628, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1629, 0, 0, 0, 0, 0, 1623, - 0, 0, 0, 0, 1624, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1634, 0, 0, 0, 0, - 0, 1625, 0, 0, 0, 0, 1626, 0, 0, 0, - 0, 1630, 0, 1635, 1631, 0, 1636, 1637, 1638, 0, - 1639, 1640, 1641, 1642, 1643, 1644, 0, 0, 1632, 1627, - 1628, 1633, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1629, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1624, 0, 0, 0, 0, - 0, 0, 1630, 0, 0, 1631, 0, 0, 0, 0, - 0, 0, 1625, 0, 0, 0, 0, 1626, 0, 1632, - 0, 0, 1635, 0, 0, 1636, 1637, 1638, 0, 1639, - 1640, 1641, 1642, 1643, 1644, 0, 0, 0, 0, 0, - -2044, -2044, 0, 0, 0, 0, 0, 0, 1634, 0, - 0, 0, 0, 0, 0, 1629, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, -2044, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1632, 0, 0, 0, 0, 0, 0, 0, 0, 1634, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, + 511, 512, 513, 514, 515, 523, 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 2439, 0, 0, 117, 118, 119, 120, 121, 122, + 123, 124, 0, 125, 126, 127, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 129, 130, 0, 131, 132, + 133, 0, 135, 136, 137, 138, 139, 0, 141, 142, + 0, 143, 144, 145, 146, 147, 148, 0, 0, 149, + 150, 151, 152, 153, 154, 155, 0, 156, 157, 158, + 159, 160, 0, 0, 0, 162, 163, 164, 165, 166, + 167, 0, 169, 170, 171, 0, 172, 173, 174, 175, + 176, 177, 0, 0, 179, 180, 181, 182, 183, 184, + 185, 186, 187, 188, 189, 190, 191, 192, 193, 0, + 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, + 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, + 209, 0, 210, 211, 212, 0, 213, 214, 215, 0, + 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, + 226, 0, 227, 0, 228, 229, 230, 231, 0, 232, + 0, 233, 0, 0, 0, 236, 237, 524, 0, 240, + 0, 241, 0, 242, 243, 244, 245, 0, 246, 247, + 248, 249, 250, 251, 252, 0, 254, 255, 256, 257, + 0, 258, 259, 260, 261, 262, 263, 264, 0, 265, + 0, 267, 268, 269, 270, 271, 272, 273, 274, 0, + 275, 0, 276, 0, 0, 279, 0, 281, 282, 283, + 284, 285, 286, 0, 0, 287, 0, 289, 0, 0, + 291, 292, 293, 294, 295, 296, 297, 298, 525, 300, + 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, + 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, + 0, 322, 323, 324, 325, 326, 0, 327, 328, 0, + 330, 0, 331, 332, 333, 334, 335, 336, 0, 337, + 338, 0, 0, 339, 340, 341, 0, 0, 342, 343, + 344, 0, 346, 0, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, 358, 359, 0, 0, 0, 0, + 360, 361, 362, 0, 364, 365, 366, 367, 368, 369, + 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, + 378, 379, 380, 381, 382, 383, 0, 384, 385, 386, + 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, + 0, 397, 398, 0, 400, 401, 402, 403, 404, 405, + 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, + 416, 0, 0, 417, 418, 419, 420, 421, 422, 423, + 424, 425, 0, 0, 427, 428, 429, 430, 0, 431, + 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, + 442, 526, 444, 445, 0, 0, 446, 447, 0, 448, + 0, 450, 451, 452, 453, 454, 0, 455, 456, 457, + 0, 0, 458, 459, 460, 461, 462, 0, 463, 464, + 465, 466, 467, 468, 469, 470, 0, 0, 471, 472, + 473, 0, 474, 475, 476, 477, 0, 478, 479, 480, + 481, 482, 483, 484, 0, 485, 0, 487, 488, 489, + 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, + 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, + 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, + 523, 0, 549, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 2570, 0, 0, 117, + 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, + 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 129, 130, 0, 131, 132, 133, 0, 135, 136, 137, + 138, 139, 0, 141, 142, 0, 143, 144, 145, 146, + 147, 148, 0, 0, 149, 150, 151, 152, 153, 154, + 155, 0, 156, 157, 158, 159, 160, 0, 0, 0, + 162, 163, 164, 165, 166, 167, 0, 169, 170, 171, + 0, 172, 173, 174, 175, 176, 177, 0, 0, 179, + 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, + 190, 191, 192, 193, 0, 194, 0, 195, 196, 197, + 198, 199, 200, 0, 0, 201, 202, 203, 204, 0, + 0, 205, 206, 207, 208, 209, 0, 210, 211, 212, + 0, 213, 214, 215, 0, 216, 217, 218, 219, 220, + 221, 222, 223, 224, 225, 226, 0, 227, 0, 228, + 229, 230, 231, 0, 232, 0, 233, 0, 0, 0, + 236, 237, 524, 0, 240, 0, 241, 0, 242, 243, + 244, 245, 0, 246, 247, 248, 249, 250, 251, 252, + 0, 254, 255, 256, 257, 0, 258, 259, 260, 261, + 262, 263, 264, 0, 265, 0, 267, 268, 269, 270, + 271, 272, 273, 274, 0, 275, 0, 276, 0, 0, + 279, 0, 281, 282, 283, 284, 285, 286, 0, 0, + 287, 0, 289, 0, 0, 291, 292, 293, 294, 295, + 296, 297, 298, 525, 300, 301, 302, 303, 304, 305, + 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, + 316, 317, 318, 319, 320, 0, 322, 323, 324, 325, + 326, 0, 327, 328, 0, 330, 0, 331, 332, 333, + 334, 335, 336, 0, 337, 338, 0, 0, 339, 340, + 341, 0, 0, 342, 343, 344, 0, 346, 0, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, + 359, 0, 0, 0, 0, 360, 361, 362, 0, 364, + 365, 366, 367, 368, 369, 0, 370, 371, 372, 373, + 374, 375, 0, 376, 377, 378, 379, 380, 381, 382, + 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, + 392, 393, 394, 395, 396, 0, 397, 398, 0, 400, + 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, + 411, 412, 413, 414, 415, 416, 0, 0, 417, 418, + 419, 420, 421, 422, 423, 424, 425, 0, 0, 427, + 428, 429, 430, 0, 431, 432, 433, 434, 435, 436, + 437, 438, 439, 440, 441, 442, 526, 444, 445, 0, + 0, 446, 447, 0, 448, 0, 450, 451, 452, 453, + 454, 0, 455, 456, 457, 0, 0, 458, 459, 460, + 461, 462, 0, 463, 464, 465, 466, 467, 468, 469, + 470, 0, 0, 471, 472, 473, 0, 474, 475, 476, + 477, 0, 478, 479, 480, 481, 482, 483, 484, 0, + 485, 0, 487, 488, 489, 490, 491, 492, 493, 0, + 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, + 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, + 511, 512, 513, 514, 515, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1635, 0, 0, 1636, 1637, - 1638, 0, 1639, 1640, 1641, 1642, 2187, 1644, 0, 0, + 0, 3215, 0, 0, 117, 118, 119, 120, 121, 122, + 123, 124, 0, 125, 126, 127, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 129, 130, 0, 131, 132, + 133, 0, 135, 136, 137, 138, 139, 0, 141, 142, + 0, 143, 144, 145, 146, 147, 148, 0, 0, 149, + 150, 151, 152, 153, 154, 155, 0, 156, 157, 158, + 159, 160, 0, 0, 0, 162, 163, 164, 165, 166, + 167, 0, 169, 170, 171, 0, 172, 173, 174, 175, + 176, 177, 0, 0, 179, 180, 181, 182, 183, 184, + 185, 186, 187, 188, 189, 190, 191, 192, 193, 0, + 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, + 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, + 209, 0, 210, 211, 212, 0, 213, 214, 215, 0, + 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, + 226, 0, 227, 0, 228, 229, 230, 231, 0, 232, + 0, 233, 0, 0, 0, 236, 237, 524, 0, 240, + 0, 241, 0, 242, 243, 244, 245, 0, 246, 247, + 248, 249, 250, 251, 252, 0, 254, 255, 256, 257, + 0, 258, 259, 260, 261, 262, 263, 264, 0, 265, + 0, 267, 268, 269, 270, 271, 272, 273, 274, 0, + 275, 0, 276, 0, 0, 279, 0, 281, 282, 283, + 284, 285, 286, 0, 0, 287, 0, 289, 0, 0, + 291, 292, 293, 294, 295, 296, 297, 298, 525, 300, + 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, + 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, + 0, 322, 323, 324, 325, 326, 0, 327, 328, 0, + 330, 0, 331, 332, 333, 334, 335, 336, 0, 337, + 338, 0, 0, 339, 340, 341, 0, 0, 342, 343, + 344, 0, 346, 0, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, 358, 359, 0, 0, 0, 0, + 360, 361, 362, 0, 364, 365, 366, 367, 368, 369, + 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, + 378, 379, 380, 381, 382, 383, 0, 384, 385, 386, + 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, + 0, 397, 398, 0, 400, 401, 402, 403, 404, 405, + 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, + 416, 0, 0, 417, 418, 419, 420, 421, 422, 423, + 424, 425, 0, 0, 427, 428, 429, 430, 0, 431, + 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, + 442, 526, 444, 445, 0, 0, 446, 447, 0, 448, + 0, 450, 451, 452, 453, 454, 0, 455, 456, 457, + 0, 0, 458, 459, 460, 461, 462, 0, 463, 464, + 465, 466, 467, 468, 469, 470, 0, 0, 471, 472, + 473, 0, 474, 475, 476, 477, 0, 478, 479, 480, + 481, 482, 483, 484, 0, 485, 0, 487, 488, 489, + 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, + 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, + 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, + 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 2058, 0, 0, 117, + 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, + 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 129, 130, 0, 131, 132, 133, 0, 135, 136, 137, + 138, 139, 0, 141, 142, 0, 143, 144, 145, 146, + 147, 148, 0, 0, 149, 150, 151, 152, 153, 154, + 155, 0, 156, 157, 158, 159, 160, 0, 0, 0, + 162, 163, 164, 165, 166, 167, 0, 169, 170, 171, + 0, 172, 173, 174, 175, 176, 177, 0, 0, 179, + 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, + 190, 191, 192, 193, 0, 194, 0, 195, 196, 197, + 198, 199, 200, 0, 0, 201, 202, 203, 204, 0, + 0, 205, 206, 207, 208, 209, 0, 210, 211, 212, + 0, 213, 214, 215, 0, 216, 217, 218, 219, 220, + 221, 222, 223, 224, 225, 226, 0, 227, 0, 228, + 229, 230, 231, 0, 232, 0, 233, 0, 0, 0, + 236, 237, 524, 0, 240, 0, 241, 0, 242, 243, + 244, 245, 0, 246, 247, 248, 249, 250, 251, 252, + 0, 254, 255, 256, 257, 0, 258, 259, 260, 261, + 262, 263, 264, 0, 265, 0, 267, 268, 269, 270, + 271, 272, 273, 274, 0, 275, 0, 276, 0, 0, + 279, 0, 281, 282, 283, 284, 285, 286, 0, 0, + 287, 0, 289, 0, 0, 291, 292, 293, 294, 295, + 296, 297, 298, 525, 300, 301, 302, 303, 304, 305, + 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, + 316, 317, 318, 319, 320, 0, 322, 323, 324, 325, + 326, 0, 327, 328, 0, 330, 0, 331, 332, 333, + 334, 335, 336, 0, 337, 338, 0, 0, 339, 340, + 341, 0, 0, 342, 343, 344, 0, 346, 0, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, + 359, 0, 0, 0, 0, 360, 361, 362, 0, 364, + 365, 366, 367, 368, 369, 0, 370, 371, 372, 373, + 374, 375, 0, 376, 377, 378, 379, 380, 381, 382, + 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, + 392, 393, 394, 395, 396, 0, 397, 398, 0, 400, + 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, + 411, 412, 413, 414, 415, 416, 0, 0, 417, 418, + 419, 420, 421, 422, 423, 424, 425, 0, 0, 427, + 428, 429, 430, 0, 431, 432, 433, 434, 435, 436, + 437, 438, 439, 440, 441, 442, 526, 444, 445, 0, + 0, 446, 447, 0, 448, 0, 450, 451, 452, 453, + 454, 0, 455, 456, 457, 0, 0, 458, 459, 460, + 461, 462, 0, 463, 464, 465, 466, 467, 468, 469, + 470, 0, 0, 471, 472, 473, 0, 474, 475, 476, + 477, 0, 478, 479, 480, 481, 482, 483, 484, 0, + 485, 0, 487, 488, 489, 490, 491, 492, 493, 0, + 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, + 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, + 511, 512, 513, 514, 515, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 2161, 0, 0, 117, 118, 119, 120, 121, 122, + 123, 124, 0, 125, 126, 127, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 129, 130, 0, 131, 132, + 133, 0, 135, 136, 137, 138, 139, 0, 141, 142, + 0, 143, 144, 145, 146, 147, 148, 0, 0, 149, + 150, 151, 152, 153, 154, 155, 0, 156, 157, 158, + 159, 160, 0, 0, 0, 162, 163, 164, 165, 166, + 167, 0, 169, 170, 171, 0, 172, 173, 174, 175, + 176, 177, 0, 0, 179, 180, 181, 182, 183, 184, + 185, 186, 187, 188, 189, 190, 191, 192, 193, 0, + 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, + 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, + 209, 0, 210, 211, 212, 0, 213, 214, 215, 0, + 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, + 226, 0, 227, 0, 228, 229, 230, 231, 0, 232, + 0, 233, 0, 0, 0, 236, 237, 524, 0, 240, + 0, 241, 0, 242, 243, 244, 245, 0, 246, 247, + 248, 249, 250, 251, 252, 0, 254, 255, 256, 257, + 0, 258, 259, 260, 261, 262, 263, 264, 0, 265, + 0, 267, 268, 269, 270, 271, 272, 273, 274, 0, + 275, 0, 276, 0, 0, 279, 0, 281, 282, 283, + 284, 285, 286, 0, 0, 287, 0, 289, 0, 0, + 291, 292, 293, 294, 295, 296, 297, 298, 525, 300, + 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, + 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, + 0, 322, 323, 324, 325, 326, 0, 327, 328, 0, + 330, 0, 331, 332, 333, 334, 335, 336, 0, 337, + 338, 0, 0, 339, 340, 341, 0, 0, 342, 343, + 344, 0, 346, 0, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, 358, 359, 0, 0, 0, 0, + 360, 361, 362, 0, 364, 365, 366, 367, 368, 369, + 0, 370, 371, 372, 373, 374, 375, 0, 376, 377, + 378, 379, 380, 381, 382, 383, 0, 384, 385, 386, + 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, + 0, 397, 398, 0, 400, 401, 402, 403, 404, 405, + 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, + 416, 0, 0, 417, 418, 419, 420, 421, 422, 423, + 424, 425, 0, 0, 427, 428, 429, 430, 0, 431, + 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, + 442, 526, 444, 445, 0, 0, 446, 447, 0, 448, + 0, 450, 451, 452, 453, 454, 0, 455, 456, 457, + 0, 0, 458, 459, 460, 461, 462, 0, 463, 464, + 465, 466, 467, 468, 469, 470, 0, 0, 471, 472, + 473, 0, 474, 475, 476, 477, 0, 478, 479, 480, + 481, 482, 483, 484, 0, 485, 0, 487, 488, 489, + 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, + 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, + 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, + 0, 2895, 1341, 815, 0, 0, 2034, 1046, 0, 0, + 0, 0, 0, 2035, 2036, 0, 3076, 2037, 2038, 2039, + 117, 118, 119, 120, 121, 122, 123, 124, 556, 125, + 126, 127, 557, 558, 559, 2896, 561, 562, 563, 564, + 2897, 129, 130, 566, 131, 132, 133, 2898, 135, 136, + 137, 0, 1480, 2899, 1482, 1483, 573, 143, 144, 145, + 146, 147, 148, 574, 575, 149, 150, 151, 152, 1484, + 1485, 155, 578, 156, 157, 158, 159, 0, 580, 2900, + 582, 2901, 163, 164, 165, 166, 167, 2902, 169, 170, + 171, 585, 172, 173, 174, 175, 176, 177, 586, 2903, + 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, + 1490, 190, 191, 1491, 193, 591, 194, 592, 195, 196, + 197, 198, 199, 200, 593, 594, 201, 202, 203, 204, + 595, 596, 205, 206, 1059, 208, 209, 597, 210, 211, + 212, 598, 213, 214, 215, 599, 216, 217, 218, 219, + 0, 221, 222, 223, 224, 225, 0, 602, 227, 603, + 228, 229, 1492, 231, 605, 232, 606, 233, 2904, 608, + 2905, 236, 237, 2906, 2907, 240, 612, 241, 613, 0, + 0, 244, 245, 616, 246, 247, 248, 249, 250, 251, + 252, 2908, 254, 255, 256, 257, 618, 258, 259, 260, + 261, 262, 263, 264, 619, 265, 2909, 0, 268, 269, + 270, 271, 272, 1498, 1499, 624, 1500, 626, 276, 2910, + 2911, 279, 2912, 281, 282, 283, 284, 285, 286, 630, + 631, 287, 2913, 289, 2914, 634, 291, 292, 293, 294, + 295, 296, 297, 298, 2915, 300, 301, 302, 303, 304, + 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, + 315, 316, 317, 318, 319, 1507, 2916, 1509, 323, 324, + 325, 2917, 640, 327, 328, 2918, 330, 642, 0, 332, + 1511, 334, 335, 336, 645, 337, 338, 646, 647, 2919, + 340, 341, 648, 649, 342, 343, 0, 2920, 346, 2921, + 0, 349, 350, 351, 352, 353, 354, 355, 356, 357, + 358, 359, 654, 655, 656, 657, 360, 361, 0, 2922, + 364, 365, 0, 367, 368, 369, 661, 370, 371, 372, + 373, 374, 375, 662, 376, 377, 378, 1515, 380, 381, + 382, 383, 664, 384, 385, 386, 387, 388, 389, 390, + 391, 392, 393, 394, 395, 396, 665, 397, 398, 2923, + 400, 401, 402, 1517, 404, 405, 406, 407, 408, 409, + 410, 411, 412, 413, 414, 415, 416, 668, 2924, 417, + 418, 419, 420, 421, 422, 2925, 424, 425, 671, 2926, + 427, 428, 1521, 430, 674, 431, 432, 433, 434, 435, + 436, 437, 438, 439, 440, 441, 442, 2927, 444, 0, + 677, 678, 446, 447, 679, 448, 2928, 450, 451, 452, + 453, 454, 681, 455, 1524, 1525, 684, 685, 458, 459, + 0, 461, 0, 688, 463, 464, 2929, 466, 467, 468, + 469, 470, 2930, 691, 471, 472, 473, 692, 474, 475, + 476, 477, 693, 478, 479, 480, 481, 482, 0, 1528, + 696, 485, 2931, 487, 488, 489, 490, 491, 492, 493, + 698, 699, 494, 700, 701, 495, 496, 497, 498, 499, + 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 512, 513, 514, 515, 0, 523, 0, 2040, + 2041, 2042, 2034, 2932, 2933, 2045, 2046, 2047, 2048, 2035, + 2036, 0, 0, 2037, 2038, 2039, 117, 118, 119, 120, + 121, 122, 123, 124, 0, 125, 126, 127, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 129, 130, 0, + 131, 132, 133, 0, 135, 136, 137, 138, 139, 0, + 141, 142, 0, 143, 144, 145, 146, 147, 148, 0, + 0, 149, 150, 151, 152, 153, 154, 155, 0, 156, + 157, 158, 159, 160, 0, 0, 0, 162, 163, 164, + 165, 166, 167, 0, 169, 170, 171, 0, 172, 173, + 174, 175, 176, 177, 0, 0, 179, 180, 181, 182, + 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, + 193, 0, 194, 0, 195, 196, 197, 198, 199, 200, + 0, 0, 201, 202, 203, 204, 0, 0, 205, 206, + 207, 208, 209, 0, 210, 211, 212, 0, 213, 214, + 215, 0, 216, 217, 218, 219, 220, 221, 222, 223, + 224, 225, 226, 0, 227, 0, 228, 229, 230, 231, + 0, 232, 0, 233, 0, 0, 0, 236, 237, 524, + 0, 240, 0, 241, 0, 242, 243, 244, 245, 0, + 246, 247, 248, 249, 250, 251, 252, 0, 254, 255, + 256, 257, 0, 258, 259, 260, 261, 262, 263, 264, + 0, 265, 0, 267, 268, 269, 270, 271, 272, 273, + 274, 0, 275, 0, 276, 0, 0, 279, 0, 281, + 282, 283, 284, 285, 286, 0, 0, 287, 0, 289, + 0, 0, 291, 292, 293, 294, 295, 296, 297, 298, + 525, 300, 301, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, + 319, 320, 0, 322, 323, 324, 325, 326, 0, 327, + 328, 0, 330, 0, 331, 332, 333, 334, 335, 336, + 0, 337, 338, 0, 0, 339, 340, 341, 0, 0, + 342, 343, 344, 0, 346, 0, 348, 349, 350, 351, + 352, 353, 354, 355, 356, 357, 358, 359, 0, 0, + 0, 0, 360, 361, 362, 0, 364, 365, 366, 367, + 368, 369, 0, 370, 371, 372, 373, 374, 375, 0, + 376, 377, 378, 379, 380, 381, 382, 383, 0, 384, + 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, + 395, 396, 0, 397, 398, 0, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, + 414, 415, 416, 0, 0, 417, 418, 419, 420, 421, + 422, 423, 424, 425, 0, 0, 427, 428, 429, 430, + 0, 431, 432, 433, 434, 435, 436, 437, 438, 439, + 440, 441, 442, 526, 444, 445, 0, 0, 446, 447, + 0, 448, 0, 450, 451, 452, 453, 454, 0, 455, + 456, 457, 0, 0, 458, 459, 460, 461, 462, 0, + 463, 464, 465, 466, 467, 468, 469, 470, 0, 0, + 471, 472, 473, 0, 474, 475, 476, 477, 0, 478, + 479, 480, 481, 482, 483, 484, 0, 485, 0, 487, + 488, 489, 490, 491, 492, 493, 0, 0, 494, 0, + 0, 495, 496, 497, 498, 499, 500, 501, 502, 503, + 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, + 514, 515, 0, 0, 0, 2040, 2041, 2042, 0, 2043, + 2044, 2045, 2046, 2047, 2048, 1611, 0, 0, 1612, 0, + 0, 0, 1613, 1614, 1615, 1616, 1617, 1618, 1619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1620, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1622, 1611, 0, 0, 1612, 0, 0, + 1623, 1613, 1614, 1615, 1616, 1617, 1618, 1619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1620, 0, 0, 0, 0, 1624, 0, 0, + 0, 0, 1622, 1611, 0, 0, 1612, 0, 0, 1623, + 1613, 1614, 1615, 1616, 1617, 1618, 1619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1635, 0, 0, 1636, - 1637, 1638, 0, 1639, 1640, 1641, 1642, 1643, 1644, 0, + 0, 1620, 0, 0, 0, 0, 1624, 0, 0, 0, + 0, 1622, 0, 0, 0, 0, 0, 0, 1623, 0, + 1611, 0, 0, 1612, 0, 0, 0, 1613, 1614, 1615, + 1616, 1617, 1618, 1619, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1624, 0, 0, 1620, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1622, 1611, + 0, 0, 1612, 1625, 0, 1623, 1613, 1614, 1615, 1616, + 1617, 1618, 1619, 0, 0, 0, 0, 0, 0, 0, + 1626, 0, 0, 0, 0, 1627, 0, 1620, 0, 0, + 0, 0, 1624, 0, 0, 0, 0, 1622, 0, 0, + 0, 0, 1625, 0, 1623, 0, 0, 0, 1628, 1629, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1626, + 0, 0, 0, 1630, 1627, 0, 0, 0, 0, 0, + 0, 1624, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1625, 0, 0, 0, 0, 0, 1628, 1629, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1626, 0, + 0, 1631, 1630, 1627, 1632, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1633, 0, + 0, 1634, 0, 0, 0, 0, 1628, 1629, 1625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1631, 1630, 0, 1632, 0, 1626, 0, 0, 0, 0, + 1627, 0, 0, 0, 0, 0, 0, 1633, 0, 0, + 1634, 0, 0, 0, 0, 0, 0, 1625, 0, 0, + 0, 0, 0, 1628, 1629, 0, 0, 0, 0, 1631, + 0, 0, 1632, 0, 1626, 0, 0, 0, 1630, 1627, + 0, 0, 0, 0, 0, 0, 1633, 0, 0, 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1628, 1629, 0, 0, 0, 0, 1635, 0, + 0, 0, 0, 0, 0, 0, 1631, 1630, 0, 1632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1633, 0, 0, 1634, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1635, 0, 0, + 0, 0, 0, 0, 0, 1631, 0, 0, 1632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1633, 0, 0, 1634, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1636, 0, 0, 1637, 1638, + 1639, 0, 1640, 1641, 1642, 1643, 1644, 1645, 0, 0, + 0, 0, 2841, 1635, 0, 0, 0, 0, 0, 1611, + 0, 0, 1612, 0, 0, 0, 1613, 1614, 1615, 1616, + 1617, 1618, 1619, 0, 1636, 0, 0, 1637, 1638, 1639, + 0, 1640, 1641, 1642, 1643, 1644, 1645, 1620, 0, 0, + 0, 3068, 1635, 0, 0, 0, 0, 1622, 0, 0, + 0, 0, 0, 0, 1623, 0, 0, 0, 0, 0, + 0, 0, 0, 1636, 0, 0, 1637, 1638, 1639, 0, + 1640, 1641, 1642, 1643, 1644, 1645, 0, 0, 0, 0, + 3075, 1624, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1611, 0, 0, 1612, 0, + 0, 0, 1613, 1614, 1615, 1616, 1617, 1618, 1619, 0, + 1636, 0, 0, 1637, 1638, 1639, 0, 1640, 1641, 1642, + 1643, 1644, 1645, 1620, 0, 0, 0, 3235, 0, 0, + 0, 0, 0, 1622, 1611, 0, 0, 1612, 0, 0, + 1623, 1613, 1614, 1615, 1616, 1617, 1618, 1619, 0, 1636, + 0, 0, 1637, 1638, 1639, 0, 1640, 1641, 1642, 1643, + 1644, 1645, 1620, 0, 0, 0, 3257, 1624, 0, 0, + 0, 0, 1622, 1611, 0, 0, 1612, 1625, 0, 1623, + 1613, 1614, 1615, 1616, 1617, 1618, 1619, 0, 0, 0, + 0, 0, 0, 0, 1626, 0, 0, 0, 0, 1627, + 0, 1620, 0, 0, 0, 0, 1624, 0, 0, 0, + 0, 1622, 0, 0, 0, 0, 0, 0, 1623, 0, + 0, 0, 1628, 1629, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1630, 0, 0, + 0, 0, 0, 0, 0, 1624, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1611, + 0, 0, 1612, 1625, 0, 0, 1613, 1614, 1615, 1616, + 1617, 1618, 1619, 0, 0, 1631, 0, 0, 1632, 0, + 1626, 0, 0, 0, 0, 1627, 0, 1620, 0, 0, + 0, 0, 1633, 0, 0, 1634, 0, 1622, 0, 0, + 0, 0, 1625, 0, 1623, 0, 0, 0, 1628, 1629, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1626, + 0, 0, 0, 1630, 1627, 0, 0, 0, 0, 0, + 0, 1624, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1625, 0, 0, 0, 0, 0, 1628, 1629, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1626, 0, + 0, 1631, 1630, 1627, 1632, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1633, 0, + 0, 1634, 0, 0, 0, 0, 1628, 1629, 0, 0, + 0, 0, 1635, 0, 0, 0, 0, 0, 0, 0, + 1631, 1630, 0, 1632, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1633, 0, 0, + 1634, 0, 0, 0, 0, 0, 0, 1625, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1631, + 0, 0, 1632, 0, 1626, 0, 0, 0, 0, 1627, + 0, 0, 0, 0, 0, 0, 1633, 0, 0, 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 555, 0, 2084, 0, 0, 0, 1635, 0, 0, - 1636, 1637, 1638, 0, 1639, 1640, 1641, 1642, 1643, 1644, - 117, 118, 119, 120, 121, 122, 123, 124, 556, 125, - 126, 127, 557, 558, 559, 560, 561, 562, 563, 564, - 565, 129, 130, 566, 131, 132, 133, 567, 135, 136, - 137, 568, 569, 570, 571, 572, 573, 143, 144, 145, - 146, 147, 148, 574, 575, 149, 150, 151, 152, 576, - 577, 155, 578, 156, 157, 158, 159, 579, 580, 581, - 582, 583, 163, 164, 165, 166, 167, 584, 169, 170, - 171, 585, 172, 173, 174, 175, 176, 177, 586, 587, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 589, 190, 191, 590, 193, 591, 194, 592, 195, 196, - 197, 198, 199, 200, 593, 594, 201, 202, 203, 204, - 595, 596, 205, 206, 207, 208, 209, 597, 210, 211, - 212, 598, 213, 214, 215, 599, 216, 217, 218, 219, - 600, 221, 222, 223, 224, 225, 601, 602, 227, 603, - 228, 229, 604, 231, 605, 232, 606, 233, 607, 608, - 609, 236, 237, 610, 611, 240, 612, 241, 613, 614, - 615, 244, 245, 616, 246, 247, 248, 249, 250, 251, - 252, 617, 254, 255, 256, 257, 618, 258, 259, 260, - 261, 262, 263, 264, 619, 265, 620, 621, 268, 269, - 270, 271, 272, 622, 623, 624, 625, 626, 276, 627, - 628, 279, 629, 281, 282, 283, 284, 285, 286, 630, - 631, 287, 632, 289, 633, 634, 291, 292, 293, 294, - 295, 296, 297, 298, 635, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 636, 637, 638, 323, 324, - 325, 639, 640, 327, 328, 641, 330, 642, 643, 332, - 644, 334, 335, 336, 645, 337, 338, 646, 647, 339, - 340, 341, 648, 649, 342, 343, 650, 651, 346, 652, - 653, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 654, 655, 656, 657, 360, 361, 658, 659, - 364, 365, 660, 367, 368, 369, 661, 370, 371, 372, - 373, 374, 375, 662, 376, 377, 378, 663, 380, 381, - 382, 383, 664, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 665, 397, 398, 666, - 400, 401, 402, 667, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 668, 669, 417, - 418, 419, 420, 421, 422, 670, 424, 425, 671, 672, - 427, 428, 673, 430, 674, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 675, 444, 676, - 677, 678, 446, 447, 679, 448, 680, 450, 451, 452, - 453, 454, 681, 455, 682, 683, 684, 685, 458, 459, - 686, 461, 687, 688, 463, 464, 689, 466, 467, 468, - 469, 470, 690, 691, 471, 472, 473, 692, 474, 475, - 476, 477, 693, 478, 479, 480, 481, 482, 694, 695, - 696, 485, 697, 487, 488, 489, 490, 491, 492, 493, - 698, 699, 494, 700, 701, 495, 496, 497, 498, 499, - 500, 702, 703, 704, 705, 706, 707, 708, 709, 710, - 711, 712, 512, 513, 514, 515, 555, 0, 0, 0, + 0, 0, 1628, 1629, 0, 0, 0, 0, 1635, 0, + 0, 0, 0, 0, 0, 0, 0, 1630, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1636, + 0, 0, 1637, 1638, 1639, 0, 1640, 1641, 1642, 1643, + 1644, 1645, 0, 0, 0, 0, 3358, 1635, 0, 0, + 0, 0, 0, 0, 0, 1631, 0, 1611, 1632, 0, + 1612, 0, 0, 0, 1613, 1614, 1615, 1616, 1617, 1618, + 1619, 0, 1633, 0, 0, 1634, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1620, 1635, 0, 0, 0, + 0, 0, 0, 0, 0, 1622, 0, 0, 0, 0, + 0, 0, 1623, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1636, 0, 0, 1637, 1638, + 1639, 0, 1640, 1641, 1642, 1643, 1644, 1645, 0, 1624, + 0, 0, 3415, 0, 0, 0, 0, 0, 0, 1611, + 0, 0, 1612, 0, 0, 0, 1613, 1614, 1615, 1616, + 1617, 1618, 1619, 0, 1636, 0, 0, 1637, 1638, 1639, + 0, 1640, 1641, 1642, 1643, 1644, 1645, 1620, 0, 0, + 0, 3437, 1635, 0, 0, 0, 0, 1622, 0, 0, + 0, 0, 0, 0, 1623, 0, 0, 0, 0, 0, + 0, 0, 0, 1636, 0, 0, 1637, 1638, 1639, 0, + 1640, 1641, 1642, 1643, 1644, 1645, 0, 0, 1796, 0, + 0, 1624, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 556, 125, 126, 127, 557, 558, 559, - 560, 561, 562, 563, 564, 565, 129, 130, 566, 131, - 132, 133, 567, 135, 136, 137, 568, 569, 570, 571, - 572, 573, 143, 144, 145, 146, 147, 148, 574, 575, - 149, 150, 151, 152, 576, 577, 155, 578, 156, 157, - 158, 159, 579, 580, 581, 582, 583, 163, 164, 165, - 166, 167, 584, 169, 170, 171, 585, 172, 173, 174, - 175, 176, 177, 586, 587, 179, 180, 181, 182, 183, - 184, 588, 186, 187, 188, 589, 190, 191, 590, 193, - 591, 194, 592, 195, 196, 197, 198, 199, 200, 593, - 594, 201, 202, 203, 204, 595, 596, 205, 206, 207, - 208, 209, 597, 210, 211, 212, 598, 213, 214, 215, - 599, 216, 217, 218, 219, 600, 221, 222, 223, 224, - 225, 601, 602, 227, 603, 228, 229, 604, 231, 605, - 232, 606, 233, 607, 608, 609, 236, 237, 610, 611, - 240, 612, 241, 613, 614, 615, 244, 245, 616, 246, - 247, 248, 249, 250, 251, 252, 617, 254, 255, 256, - 257, 618, 258, 259, 260, 261, 262, 263, 264, 619, - 265, 620, 621, 268, 269, 270, 271, 272, 622, 623, - 624, 625, 626, 276, 627, 628, 279, 629, 281, 282, - 283, 284, 285, 286, 630, 631, 287, 632, 289, 633, - 634, 291, 292, 293, 294, 295, 296, 297, 298, 635, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 636, 637, 638, 323, 324, 325, 639, 640, 327, 328, - 641, 330, 642, 643, 332, 644, 334, 335, 336, 645, - 337, 338, 646, 647, 339, 340, 341, 648, 649, 342, - 343, 650, 651, 346, 652, 653, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 654, 655, 656, - 657, 360, 361, 658, 659, 364, 365, 660, 367, 368, - 369, 661, 370, 371, 372, 373, 374, 375, 662, 376, - 377, 378, 663, 380, 381, 382, 383, 664, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 665, 397, 398, 666, 400, 401, 402, 667, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 668, 669, 417, 418, 419, 420, 421, 422, - 670, 424, 425, 671, 672, 427, 428, 673, 430, 674, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 675, 444, 676, 677, 678, 446, 447, 679, - 448, 680, 450, 451, 452, 453, 454, 681, 455, 682, - 683, 684, 685, 458, 459, 686, 461, 687, 688, 463, - 464, 689, 466, 467, 468, 469, 470, 690, 691, 471, - 472, 473, 692, 474, 475, 476, 477, 693, 478, 479, - 480, 481, 482, 694, 695, 696, 485, 697, 487, 488, - 489, 490, 491, 492, 493, 698, 699, 494, 700, 701, - 495, 496, 497, 498, 499, 500, 702, 703, 704, 705, - 706, 707, 708, 709, 710, 711, 712, 512, 513, 514, - 515, 555, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1626, 0, 0, 0, 0, 1627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 118, 119, 120, 121, 122, 123, 124, 556, 125, - 126, 127, 557, 558, 559, 560, 561, 562, 563, 564, - 565, 129, 130, 566, 131, 132, 133, 567, 135, 136, - 137, 568, 569, 570, 571, 572, 573, 143, 144, 145, - 146, 147, 148, 574, 575, 149, 150, 151, 152, 576, - 577, 155, 578, 156, 157, 158, 159, 579, 580, 581, - 582, 583, 163, 164, 165, 166, 167, 584, 169, 170, - 171, 585, 172, 173, 174, 175, 176, 177, 586, 587, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 589, 190, 191, 590, 193, 591, 194, 592, 195, 196, - 197, 198, 199, 200, 593, 594, 201, 202, 203, 204, - 595, 596, 205, 206, 207, 208, 209, 597, 210, 211, - 212, 598, 213, 214, 215, 599, 216, 217, 218, 219, - 600, 221, 222, 223, 224, 225, 601, 602, 227, 603, - 228, 229, 604, 231, 605, 232, 606, 233, 607, 608, - 609, 236, 237, 610, 611, 240, 612, 241, 613, 614, - 615, 244, 245, 616, 246, 247, 248, 249, 250, 940, - 252, 617, 254, 255, 256, 257, 618, 258, 259, 260, - 261, 262, 263, 264, 619, 265, 620, 621, 268, 269, - 270, 271, 272, 622, 623, 624, 625, 626, 276, 627, - 628, 279, 629, 281, 282, 283, 284, 285, 286, 630, - 631, 287, 632, 289, 633, 634, 291, 292, 293, 294, - 295, 296, 297, 298, 635, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 636, 637, 638, 323, 324, - 325, 639, 640, 327, 328, 641, 330, 642, 643, 332, - 644, 334, 335, 336, 645, 337, 338, 646, 647, 339, - 340, 341, 648, 649, 342, 343, 650, 651, 346, 652, - 653, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 654, 655, 656, 657, 360, 361, 658, 659, - 364, 365, 660, 367, 368, 369, 661, 370, 371, 372, - 373, 374, 375, 662, 376, 377, 378, 663, 380, 381, - 382, 383, 664, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 665, 397, 398, 666, - 400, 401, 402, 667, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 668, 669, 417, - 418, 419, 420, 421, 422, 670, 424, 425, 671, 672, - 427, 428, 673, 430, 674, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 675, 444, 676, - 677, 678, 446, 447, 679, 448, 680, 450, 451, 452, - 453, 454, 681, 455, 682, 683, 684, 685, 458, 459, - 686, 461, 687, 688, 463, 464, 689, 466, 467, 468, - 469, 470, 690, 691, 471, 472, 473, 692, 474, 475, - 476, 477, 693, 478, 479, 480, 481, 482, 694, 695, - 696, 485, 697, 487, 488, 489, 490, 491, 492, 493, - 698, 699, 494, 700, 701, 495, 496, 497, 498, 499, - 500, 702, 703, 704, 705, 706, 707, 708, 709, 710, - 711, 712, 512, 513, 514, 515, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, + 1628, 1629, 0, 0, 0, 0, 0, 0, 0, 1636, + 0, 0, 1637, 1638, 1639, 1630, 1640, 1641, 1642, 1643, + 1644, 1645, 0, 0, 2795, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1625, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1631, 1626, 0, 1632, 0, 0, 1627, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1633, 0, 0, 1634, 0, 0, 0, 0, 0, 0, + 0, 0, 1628, 1629, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1630, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1631, 0, 0, 1632, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1633, 0, 0, 1634, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1635, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1635, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1636, 0, 0, + 1637, 1638, 1639, 0, 1640, 1641, 1642, 1643, 1644, 1645, + 0, 0, 3225, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 555, 0, 0, 1636, + 0, 0, 1637, 1638, 1639, 0, 1640, 1641, 1642, 1643, + 1644, 1645, 0, 0, 3399, 117, 118, 119, 120, 121, 122, 123, 124, 556, 125, 126, 127, 557, 558, 559, 560, 561, 562, 563, 564, 565, 129, 130, 566, 131, 132, 133, 567, 135, 136, 137, 568, 569, 570, 571, @@ -9072,109 +8530,109 @@ static const yytype_int16 yytable[] = 489, 490, 491, 492, 493, 698, 699, 494, 700, 701, 495, 496, 497, 498, 499, 500, 702, 703, 704, 705, 706, 707, 708, 709, 710, 711, 712, 512, 513, 514, - 515, 555, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 118, 119, 120, 2232, 122, 123, 124, 556, 125, - 126, 127, 557, 558, 559, 560, 561, 562, 563, 564, - 565, 129, 130, 566, 131, 132, 133, 567, 135, 136, - 137, 568, 569, 570, 571, 572, 573, 143, 144, 145, - 146, 147, 148, 574, 575, 149, 150, 151, 152, 576, - 577, 155, 578, 156, 157, 158, 159, 579, 580, 581, - 582, 583, 163, 164, 165, 166, 167, 584, 169, 170, - 171, 585, 172, 173, 174, 175, 176, 177, 586, 587, + 515, 523, 0, 0, 0, 0, 0, 0, 0, 0, + 2069, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, + 126, 127, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 129, 130, 0, 131, 132, 133, 0, 135, 136, + 137, 138, 139, 0, 141, 142, 0, 143, 144, 145, + 146, 147, 148, 0, 0, 149, 150, 151, 152, 153, + 154, 155, 0, 156, 157, 158, 159, 160, 0, 0, + 0, 162, 163, 164, 165, 166, 167, 0, 169, 170, + 171, 0, 172, 173, 174, 175, 176, 177, 0, 0, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 589, 190, 191, 590, 193, 591, 194, 592, 195, 196, - 197, 198, 199, 200, 593, 594, 201, 202, 203, 204, - 595, 596, 205, 206, 207, 2233, 209, 597, 210, 211, - 212, 598, 213, 214, 215, 599, 216, 217, 218, 219, - 600, 221, 222, 223, 224, 225, 601, 602, 227, 603, - 228, 229, 604, 231, 605, 232, 606, 233, 607, 608, - 609, 236, 237, 610, 611, 240, 612, 241, 613, 614, - 615, 244, 245, 616, 246, 247, 248, 249, 250, 251, - 252, 617, 254, 255, 256, 257, 618, 258, 259, 260, - 261, 262, 263, 264, 619, 265, 620, 621, 268, 269, - 270, 271, 272, 622, 623, 624, 625, 626, 276, 627, - 628, 279, 629, 281, 282, 283, 284, 285, 286, 630, - 631, 287, 632, 289, 633, 634, 291, 292, 293, 294, - 295, 296, 297, 298, 635, 300, 301, 302, 303, 304, + 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, + 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, + 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, + 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, + 220, 221, 222, 223, 224, 225, 226, 0, 227, 0, + 228, 229, 230, 231, 0, 232, 0, 233, 0, 0, + 0, 236, 237, 524, 0, 240, 0, 241, 0, 242, + 243, 244, 245, 0, 246, 247, 248, 249, 250, 251, + 252, 0, 254, 255, 256, 257, 0, 258, 259, 260, + 261, 262, 263, 264, 0, 265, 0, 267, 268, 269, + 270, 271, 272, 273, 274, 0, 275, 0, 276, 0, + 0, 279, 0, 281, 282, 283, 284, 285, 286, 0, + 0, 287, 0, 289, 0, 0, 291, 292, 293, 294, + 295, 296, 297, 298, 525, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 636, 637, 638, 323, 324, - 325, 639, 640, 327, 328, 641, 330, 642, 643, 332, - 644, 334, 335, 336, 645, 337, 338, 646, 647, 339, - 340, 341, 648, 649, 342, 343, 650, 651, 346, 652, - 653, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 654, 655, 656, 657, 360, 361, 658, 659, - 364, 365, 660, 367, 368, 369, 661, 370, 371, 372, - 373, 374, 375, 662, 376, 377, 378, 663, 380, 381, - 382, 383, 664, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 665, 397, 398, 666, - 400, 401, 402, 667, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 668, 669, 417, - 418, 419, 420, 421, 2234, 670, 424, 425, 671, 672, - 427, 428, 673, 430, 674, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 675, 444, 676, - 677, 678, 446, 447, 679, 448, 680, 450, 451, 452, - 453, 454, 681, 455, 682, 683, 684, 685, 458, 459, - 686, 461, 687, 688, 463, 464, 689, 466, 467, 468, - 469, 470, 690, 691, 471, 472, 473, 692, 474, 475, - 476, 477, 693, 478, 479, 480, 481, 482, 694, 695, - 696, 485, 697, 487, 488, 489, 490, 491, 492, 493, - 698, 699, 494, 700, 701, 495, 496, 497, 498, 499, - 500, 702, 703, 704, 705, 706, 707, 708, 709, 710, - 711, 712, 512, 513, 514, 515, 964, 0, 815, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 315, 316, 317, 318, 319, 320, 0, 322, 323, 324, + 325, 326, 0, 327, 328, 0, 330, 0, 331, 332, + 333, 334, 335, 336, 0, 337, 338, 0, 0, 339, + 340, 341, 0, 0, 342, 343, 344, 0, 346, 0, + 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, + 358, 359, 0, 0, 0, 0, 360, 361, 362, 0, + 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, + 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, + 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, + 391, 392, 393, 394, 395, 396, 0, 397, 398, 0, + 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, + 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, + 418, 419, 420, 421, 422, 423, 424, 425, 0, 0, + 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, + 436, 437, 438, 439, 440, 441, 442, 526, 444, 445, + 0, 0, 446, 447, 0, 448, 0, 450, 451, 452, + 453, 454, 0, 455, 456, 457, 0, 0, 458, 459, + 460, 461, 462, 0, 463, 464, 465, 466, 467, 468, + 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, + 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, + 0, 485, 0, 487, 488, 489, 490, 491, 492, 493, + 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, + 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, + 510, 511, 512, 513, 514, 515, 523, 0, 0, 0, + 0, 0, 0, 0, 0, 2695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 560, 0, 0, 0, 0, 565, 129, 130, 0, 131, - 132, 133, 567, 135, 136, 137, 568, 569, 570, 571, - 572, 0, 143, 144, 145, 146, 147, 148, 0, 0, - 149, 150, 151, 152, 576, 577, 155, 0, 156, 157, - 158, 159, 579, 0, 581, 0, 583, 163, 164, 165, - 166, 167, 584, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 587, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 589, 190, 191, 590, 193, + 0, 0, 0, 0, 0, 0, 129, 130, 0, 131, + 132, 133, 0, 135, 136, 137, 138, 139, 0, 141, + 142, 0, 143, 144, 145, 146, 147, 148, 0, 0, + 149, 150, 151, 152, 153, 154, 155, 0, 156, 157, + 158, 159, 160, 0, 0, 0, 162, 163, 164, 165, + 166, 167, 0, 169, 170, 171, 0, 172, 173, 174, + 175, 176, 177, 0, 0, 179, 180, 181, 182, 183, + 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 600, 221, 222, 223, 224, - 225, 601, 1341, 227, 0, 228, 229, 604, 231, 0, - 232, 0, 233, 607, 0, 609, 236, 237, 610, 611, - 240, 0, 241, 0, 614, 615, 244, 245, 0, 246, - 247, 248, 249, 250, 251, 252, 617, 254, 255, 256, + 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, + 225, 226, 0, 227, 0, 228, 229, 230, 231, 0, + 232, 0, 233, 0, 0, 0, 236, 237, 524, 0, + 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, + 247, 248, 249, 250, 251, 252, 0, 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 620, 621, 268, 269, 270, 271, 272, 622, 623, - 0, 625, 0, 276, 627, 628, 279, 629, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 632, 289, 633, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 635, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 636, 637, 638, 323, 324, 325, 639, 0, 327, 328, - 641, 330, 0, 643, 332, 644, 334, 335, 336, 0, - 337, 338, 1342, 0, 339, 340, 341, 0, 0, 342, - 343, 650, 651, 346, 652, 653, 349, 350, 351, 352, + 265, 0, 267, 268, 269, 270, 271, 272, 273, 274, + 0, 275, 0, 276, 0, 0, 279, 0, 281, 282, + 283, 284, 285, 286, 0, 0, 287, 0, 289, 0, + 0, 291, 292, 293, 294, 295, 296, 297, 298, 525, + 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, + 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, + 320, 0, 322, 323, 324, 325, 326, 0, 327, 328, + 0, 330, 0, 331, 332, 333, 334, 335, 336, 0, + 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, + 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 658, 659, 364, 365, 660, 367, 368, + 0, 360, 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 663, 380, 381, 382, 383, 0, 384, 385, + 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 666, 400, 401, 402, 667, 404, + 396, 0, 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 669, 417, 418, 419, 420, 421, 422, - 670, 424, 425, 0, 672, 427, 428, 673, 430, 0, + 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, + 423, 424, 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 675, 444, 676, 0, 0, 446, 447, 0, - 448, 680, 450, 451, 452, 453, 454, 0, 455, 682, - 683, 0, 0, 458, 459, 686, 461, 687, 1343, 463, - 464, 689, 466, 467, 468, 469, 470, 0, 0, 471, + 441, 442, 526, 444, 445, 0, 0, 446, 447, 0, + 448, 0, 450, 451, 452, 453, 454, 0, 455, 456, + 457, 0, 0, 458, 459, 460, 461, 462, 0, 463, + 464, 465, 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 694, 695, 0, 485, 697, 487, 488, + 480, 481, 482, 483, 484, 0, 485, 0, 487, 488, 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 702, 703, 704, 705, - 706, 707, 708, 709, 710, 711, 712, 512, 513, 514, - 515, 964, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, + 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, + 515, 964, 1341, 815, 0, 0, 0, 1046, 0, 0, + 2698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, - 126, 127, 3, 4, 0, 560, 0, 0, 0, 0, + 126, 127, 0, 0, 0, 560, 0, 0, 0, 0, 565, 129, 130, 0, 131, 132, 133, 567, 135, 136, 137, 568, 569, 570, 571, 572, 0, 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, 151, 152, 576, @@ -9186,7 +8644,7 @@ static const yytype_int16 yytable[] = 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, - 600, 221, 222, 223, 224, 225, 601, 0, 227, 0, + 600, 221, 222, 223, 224, 225, 601, 1342, 227, 0, 228, 229, 604, 231, 0, 232, 0, 233, 607, 0, 609, 236, 237, 610, 611, 240, 0, 241, 0, 614, 615, 244, 245, 0, 246, 247, 248, 249, 250, 251, @@ -9199,7 +8657,7 @@ static const yytype_int16 yytable[] = 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 636, 637, 638, 323, 324, 325, 639, 0, 327, 328, 641, 330, 0, 643, 332, - 644, 334, 335, 336, 0, 337, 338, 0, 0, 339, + 644, 334, 335, 336, 0, 337, 338, 1343, 0, 339, 340, 341, 0, 0, 342, 343, 650, 651, 346, 652, 653, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, 361, 658, 659, @@ -9214,7243 +8672,3932 @@ static const yytype_int16 yytable[] = 436, 437, 438, 439, 440, 441, 442, 675, 444, 676, 0, 0, 446, 447, 0, 448, 680, 450, 451, 452, 453, 454, 0, 455, 682, 683, 0, 0, 458, 459, - 686, 461, 687, 0, 463, 464, 689, 466, 467, 468, + 686, 461, 687, 1344, 463, 464, 689, 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, 482, 694, 695, 0, 485, 697, 487, 488, 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, 500, 702, 703, 704, 705, 706, 707, 708, 709, 710, - 711, 712, 512, 513, 514, 515, 116, 0, 0, 0, + 711, 712, 512, 513, 514, 515, 0, 0, 1611, 0, + 0, 1612, 0, 1345, 1346, 1613, 1614, 1615, 1616, 1617, + 1618, 1619, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1620, 0, 0, 0, + 0, 2167, 0, 0, 0, 0, 1622, 1611, 0, 0, + 1612, 0, 0, 1623, 1613, 1614, 1615, 1616, 1617, 1618, + 1619, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1620, 0, 0, 0, 0, + 1624, 0, 0, 0, 0, 1622, 1611, 0, 0, 1612, + 0, 0, 1623, 1613, 1614, 1615, 1616, 1617, 1618, 1619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 0, 0, 0, 0, 0, 128, 129, 130, 0, 131, - 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, - 142, 0, 143, 144, 145, 146, 147, 148, 0, 785, - 149, 150, 151, 152, 153, 154, 155, 0, 156, 157, - 158, 159, 786, 0, 787, 0, 162, 163, 164, 165, - 166, 167, 168, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 178, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, - 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 788, 0, 227, 0, 228, 229, 230, 231, 0, - 232, 0, 233, 234, 0, 235, 236, 237, 238, 239, - 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, - 0, 275, 0, 276, 277, 278, 279, 280, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 288, 289, 290, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 299, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 321, 322, 323, 324, 325, 326, 0, 327, 328, - 329, 330, 0, 790, 332, 333, 334, 335, 336, 0, - 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, - 343, 344, 345, 346, 347, 792, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 793, 363, 364, 365, 366, 367, 368, - 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 399, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 0, 426, 427, 428, 429, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 443, 444, 795, 0, 0, 446, 447, 0, - 448, 449, 450, 451, 452, 453, 454, 0, 455, 456, - 457, 0, 0, 458, 459, 796, 461, 797, 0, 463, - 464, 798, 466, 467, 468, 469, 470, 0, 0, 471, - 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 483, 484, 0, 485, 486, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, - 515, 116, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1620, 0, 0, 0, 0, 1624, + 0, 0, 0, 0, 1622, 0, 2168, 0, 0, 0, + 0, 1623, 0, 0, 1611, 0, 0, 1612, 0, 0, + 0, 1613, 1614, 1615, 1616, 1617, 1618, 1619, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1624, 0, + 0, 0, 1620, 0, 0, 0, 1890, 0, 0, 0, + 0, 0, 1622, 0, 1611, 0, 1625, 1612, 0, 1623, + 0, 1613, 1614, 1615, 1616, 1617, 1618, 1619, 0, 0, + 0, 0, 0, 1626, 0, 0, 0, 0, 1627, 0, + 0, 0, 1620, 0, 0, 1926, 1624, 0, 0, 0, + 1927, 0, 1622, 0, 0, 1625, 0, 0, 0, 1623, + 0, 1628, 1629, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1626, 0, 0, 0, 1630, 1627, 0, 0, + 0, 0, 0, 3505, 0, 0, 1624, 0, 0, 0, + 0, 0, 0, 0, 1625, 0, 0, 0, 0, 0, + 1628, 1629, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1626, 0, 0, 1631, 1630, 1627, 1632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, - 126, 127, 0, 0, 0, 0, 0, 0, 0, 0, - 128, 129, 130, 0, 131, 132, 133, 134, 135, 136, - 137, 138, 139, 140, 141, 142, 0, 143, 144, 145, - 146, 147, 148, 0, 0, 149, 150, 151, 152, 153, - 154, 155, 0, 156, 157, 158, 159, 160, 0, 161, - 0, 162, 163, 164, 165, 166, 167, 168, 169, 170, - 171, 0, 172, 173, 174, 175, 176, 177, 0, 178, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, - 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, - 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, - 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, - 220, 221, 222, 223, 224, 225, 226, 0, 227, 0, - 228, 229, 230, 231, 0, 232, 0, 233, 234, 0, - 235, 236, 237, 238, 239, 240, 0, 241, 0, 242, - 243, 244, 245, 0, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 0, 258, 259, 260, - 261, 262, 263, 264, 0, 265, 266, 267, 268, 269, - 270, 271, 272, 273, 274, 0, 275, 0, 276, 277, - 278, 279, 280, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 288, 289, 290, 0, 291, 292, 293, 294, - 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 0, 327, 328, 329, 330, 0, 331, 332, - 333, 334, 335, 336, 0, 337, 338, 0, 0, 339, - 340, 341, 0, 0, 342, 343, 344, 345, 346, 347, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 0, 0, 0, 0, 360, 361, 362, 363, - 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, - 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 397, 398, 399, - 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 0, 426, - 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, - 0, 0, 446, 447, 0, 448, 449, 450, 451, 452, - 453, 454, 0, 455, 456, 457, 0, 0, 458, 459, - 460, 461, 462, 0, 463, 464, 465, 466, 467, 468, - 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, - 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, - 0, 485, 486, 487, 488, 489, 490, 491, 492, 493, - 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, - 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, - 510, 511, 512, 513, 514, 515, 523, 0, 0, 0, + 0, 1633, 0, 0, 1634, 0, 0, 0, 0, 1628, + 1629, 0, 1625, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1631, 1630, 0, 1632, 0, 0, 1626, + 0, 0, 0, 0, 1627, 0, 0, 0, 0, 0, + 1633, 0, 0, 1634, 0, 0, 0, 0, 0, 0, + 0, 0, 1625, 0, 0, 0, 0, 1628, 1629, 0, + 0, 0, 1631, 0, 0, 1632, 0, 0, 0, 1626, + 0, 0, 1630, 0, 1627, 0, 0, 0, 0, 1633, + 0, 0, 1634, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1628, 1629, 0, + 0, 1635, 0, 0, 0, 0, 0, 0, 0, 0, + 1631, 3506, 1630, 1632, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1633, 0, 0, + 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1635, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1631, 0, 0, 1632, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 2171, 0, 0, 1633, 0, 0, + 1634, 0, 0, 0, 0, 0, 0, 0, 0, 1635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 129, 130, 0, 131, - 132, 133, 0, 135, 136, 137, 138, 139, 0, 141, - 142, 0, 143, 144, 145, 146, 147, 148, 0, 0, - 149, 150, 151, 152, 153, 154, 155, 1738, 156, 157, - 158, 159, 160, 0, 0, 1739, 162, 163, 164, 165, - 166, 167, 0, 169, 170, 171, 1740, 172, 173, 174, - 175, 176, 177, 0, 0, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, - 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 226, 0, 227, 0, 228, 229, 230, 231, 0, - 232, 1741, 233, 0, 0, 0, 236, 237, 524, 0, - 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, - 247, 248, 249, 250, 1742, 252, 0, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 0, 267, 268, 269, 270, 271, 272, 273, 274, - 0, 275, 0, 276, 0, 0, 279, 0, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 0, 289, 0, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 525, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 0, 322, 323, 324, 325, 326, 0, 327, 328, - 0, 330, 0, 331, 332, 333, 334, 335, 336, 0, - 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, - 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 362, 0, 364, 365, 366, 367, 368, - 369, 1743, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 0, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 0, 0, 427, 428, 429, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 526, 444, 445, 0, 0, 446, 447, 0, - 448, 0, 450, 451, 452, 453, 454, 0, 455, 456, - 457, 0, 0, 458, 459, 460, 461, 462, 0, 463, - 464, 465, 466, 467, 468, 469, 470, 0, 1744, 471, - 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 483, 484, 0, 485, 0, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, - 515, 523, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1897, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1636, 0, + 0, 1637, 1638, 1639, 0, 1640, 1641, 1642, 1643, 1644, + 1645, 0, 0, 0, 0, 0, 0, 1635, 0, 0, + 0, 0, 1611, 0, 0, 1612, 0, 0, 0, 1613, + 1614, 1615, 1616, 1617, 1618, 1619, 0, 1636, 0, 0, + 1637, 1638, 1639, 0, 1640, 1641, 1642, 1643, 1644, 1645, + 1620, 0, 0, 0, 1932, 0, 0, 1635, 0, 0, + 1622, 0, 0, 0, 0, 0, 0, 1623, 0, 0, + 0, 0, 0, 0, 0, 0, 1636, 0, 0, 1637, + 1638, 1639, 0, 1640, 1641, 1642, 1643, 1644, 1645, 0, + 0, 0, 0, 0, 1624, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1611, + 0, 0, 1612, 0, 0, 0, 1613, 1614, 1615, 1616, + 1617, 1618, 1619, 0, 1636, 0, 0, 1637, 1638, 1639, + 0, 1640, 1641, 1642, 1643, 1644, 1645, 1620, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1622, 0, 0, + 0, 0, 0, 0, 1623, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1636, 0, 0, 1637, 1638, 1639, + 0, 1640, 1641, 1642, 1643, 1644, 1645, 0, 0, 0, + 0, 1624, 0, 0, 0, 0, 1611, 0, 0, 1612, + 1625, 0, 0, 1613, 1614, 1615, 1616, 1617, 1618, 1619, + 0, 0, 0, 0, 0, 0, 0, 1626, 0, 0, + 0, 0, 1627, 0, 1620, 0, 0, 0, 1939, 0, + 0, 0, 0, 0, 1622, 0, 0, 0, 0, 0, + 0, 1623, 0, 0, 0, 1628, 1629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, - 126, 127, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 129, 130, 0, 131, 132, 133, 0, 135, 136, - 137, 138, 139, 0, 141, 142, 0, 143, 144, 145, - 146, 147, 148, 0, 0, 149, 150, 151, 152, 153, - 154, 155, 1738, 156, 157, 158, 159, 160, 0, 0, - 0, 162, 163, 164, 165, 166, 167, 0, 169, 170, - 171, 1740, 172, 173, 174, 175, 176, 177, 0, 0, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, - 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, - 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, - 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, - 220, 221, 222, 223, 224, 225, 226, 0, 227, 0, - 228, 229, 230, 231, 0, 232, 1741, 233, 0, 0, - 0, 236, 237, 524, 0, 240, 0, 241, 0, 242, - 243, 244, 245, 0, 246, 247, 248, 249, 250, 251, - 252, 0, 254, 255, 256, 257, 0, 258, 259, 260, - 261, 262, 263, 264, 0, 265, 0, 267, 268, 269, - 270, 271, 272, 273, 274, 0, 275, 0, 276, 0, - 0, 279, 0, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 0, 289, 2317, 0, 291, 292, 293, 294, - 295, 296, 297, 298, 525, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 0, 322, 323, 324, - 325, 326, 0, 327, 328, 0, 330, 0, 331, 332, - 333, 334, 335, 336, 0, 337, 338, 0, 0, 339, - 340, 341, 0, 0, 342, 343, 344, 0, 346, 0, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 0, 0, 0, 0, 360, 361, 362, 0, - 364, 365, 366, 367, 368, 369, 1743, 370, 371, 372, - 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 397, 398, 0, - 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 0, 0, - 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 526, 444, 445, - 0, 0, 446, 447, 0, 448, 0, 450, 451, 452, - 453, 454, 0, 455, 456, 457, 0, 0, 458, 459, - 460, 461, 462, 0, 463, 464, 465, 466, 467, 468, - 469, 470, 0, 1744, 471, 472, 473, 0, 474, 475, - 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, - 0, 485, 0, 487, 488, 489, 490, 491, 492, 493, - 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, - 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, - 510, 511, 512, 513, 514, 515, 523, 0, 549, 0, + 1630, 0, 0, 0, 0, 0, 0, 0, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 0, 125, 126, 127, 3, 4, 0, - 0, 0, 0, 0, 0, 0, 129, 130, 0, 131, - 132, 133, 0, 135, 136, 137, 138, 139, 0, 141, - 142, 0, 143, 144, 145, 146, 147, 148, 0, 0, - 149, 150, 151, 152, 153, 154, 155, 0, 156, 157, - 158, 159, 160, 0, 0, 0, 162, 163, 164, 165, - 166, 167, 0, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 0, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, - 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 226, 0, 227, 0, 228, 229, 230, 231, 0, - 232, 0, 233, 0, 0, 0, 236, 237, 524, 0, - 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, - 247, 248, 249, 250, 251, 252, 0, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 0, 267, 268, 269, 270, 271, 272, 273, 274, - 0, 275, 0, 276, 0, 0, 279, 0, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 0, 289, 0, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 525, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 0, 322, 323, 324, 325, 326, 0, 327, 328, - 0, 330, 0, 331, 332, 333, 334, 335, 336, 0, - 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, - 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 362, 0, 364, 365, 366, 367, 368, - 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 0, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 0, 0, 427, 428, 429, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 526, 444, 445, 0, 0, 446, 447, 0, - 448, 0, 450, 451, 452, 453, 454, 0, 455, 456, - 457, 0, 0, 458, 459, 460, 461, 462, 0, 463, - 464, 465, 466, 467, 468, 469, 470, 0, 0, 471, - 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 483, 484, 0, 485, 0, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, - 515, 523, 0, 549, 0, 0, 0, 0, 0, 0, + 0, 0, 1937, 1611, 0, 0, 1612, 1625, 0, 0, + 1613, 1614, 1615, 1616, 1617, 1618, 1619, 0, 1631, 0, + 0, 1632, 0, 0, 1626, 0, 0, 0, 0, 1627, + 0, 1620, 0, 0, 0, 1633, 0, 0, 1634, 0, + 0, 1622, 0, 0, 0, 0, 0, 0, 1623, 0, + 0, 0, 1628, 1629, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1630, 0, 0, + 0, 0, 0, 0, 0, 1624, 0, 0, 0, 0, + 0, 0, 0, 0, 1625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 118, 119, 120, 121, 122, 123, 124, 550, 125, - 126, 127, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 129, 130, 0, 131, 132, 133, 0, 135, 136, - 137, 138, 139, 0, 141, 142, 0, 143, 144, 145, - 146, 147, 148, 0, 0, 149, 150, 151, 152, 153, - 154, 155, 0, 156, 157, 158, 159, 160, 0, 0, - 0, 162, 163, 164, 165, 166, 167, 0, 169, 170, - 171, 0, 172, 173, 174, 175, 176, 177, 0, 0, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, - 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, - 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, - 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, - 220, 221, 222, 223, 224, 225, 226, 0, 227, 0, - 228, 229, 230, 231, 0, 232, 0, 233, 0, 0, - 0, 236, 237, 524, 0, 240, 0, 241, 0, 242, - 243, 244, 245, 0, 246, 247, 248, 249, 250, 251, - 252, 0, 254, 255, 256, 257, 0, 258, 259, 260, - 261, 262, 263, 264, 0, 265, 0, 267, 268, 269, - 270, 271, 272, 273, 274, 0, 275, 0, 276, 0, - 0, 279, 0, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 0, 289, 0, 0, 291, 292, 293, 294, - 295, 296, 297, 298, 525, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 0, 322, 323, 324, - 325, 326, 0, 327, 328, 0, 330, 0, 331, 332, - 333, 334, 335, 336, 0, 337, 338, 0, 0, 339, - 340, 341, 0, 0, 342, 343, 344, 0, 346, 0, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 0, 0, 0, 0, 360, 361, 362, 0, - 364, 365, 366, 551, 368, 369, 0, 370, 371, 372, - 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 397, 398, 0, - 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 0, 0, - 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 526, 444, 445, - 0, 0, 446, 447, 0, 448, 0, 450, 451, 452, - 453, 454, 0, 455, 456, 457, 0, 0, 458, 459, - 460, 461, 462, 0, 463, 464, 465, 466, 467, 468, - 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, - 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, - 0, 485, 0, 487, 488, 489, 490, 491, 492, 493, - 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, - 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, - 510, 511, 512, 513, 514, 515, 523, 0, 549, 0, + 0, 1626, 0, 0, 0, 1631, 1627, 0, 1632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 129, 130, 0, 131, - 132, 133, 0, 135, 136, 137, 138, 139, 0, 141, - 142, 0, 143, 144, 145, 146, 147, 148, 0, 0, - 149, 150, 151, 152, 153, 154, 155, 0, 156, 157, - 158, 159, 160, 0, 0, 0, 162, 163, 164, 165, - 166, 167, 0, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 0, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, - 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 226, 0, 227, 0, 228, 229, 230, 231, 0, - 232, 0, 233, 0, 0, 0, 236, 237, 524, 0, - 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, - 247, 248, 249, 250, 251, 252, 0, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 0, 267, 268, 269, 270, 271, 272, 273, 274, - 0, 275, 0, 276, 0, 0, 279, 0, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 0, 289, 0, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 525, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 0, 322, 323, 324, 325, 326, 0, 327, 328, - 0, 330, 0, 331, 332, 333, 334, 335, 336, 0, - 337, 338, 0, 791, 339, 340, 341, 0, 0, 342, - 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 362, 0, 364, 365, 366, 367, 368, - 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 0, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 0, 0, 427, 428, 429, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 526, 444, 445, 0, 0, 446, 447, 0, - 448, 0, 450, 451, 452, 453, 454, 0, 455, 456, - 457, 0, 0, 458, 459, 460, 461, 462, 0, 463, - 464, 465, 466, 467, 468, 469, 470, 0, 0, 471, - 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 483, 484, 0, 485, 0, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, - 515, 523, 0, 549, 0, 0, 0, 0, 0, 0, + 0, 0, 1633, 0, 0, 1634, 0, 0, 0, 1628, + 1629, 0, 0, 0, 0, 1635, 0, 0, 1611, 0, + 0, 1612, 0, 0, 1630, 1613, 1614, 1615, 1616, 1617, + 1618, 1619, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 2062, 0, 0, 0, 0, 1620, 0, 0, 0, + 0, 1625, 0, 0, 0, 0, 1622, 0, 0, 0, + 0, 0, 1631, 1623, 0, 1632, 0, 0, 1626, 0, + 0, 0, 0, 1627, 0, 0, 0, 0, 0, 1633, + 0, 0, 1634, 0, 0, 0, 0, 0, 0, 0, + 1624, 0, 0, 0, 0, 0, 1628, 1629, 0, 0, + 0, 0, 1635, 0, 0, 0, 0, 0, 0, 0, + 0, 1630, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1636, 0, 0, 1637, 1638, 1639, 0, 1640, + 1641, 1642, 1643, 1644, 1645, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1631, + 0, 0, 1632, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1633, 0, 0, 1634, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1635, + 0, 0, 0, 0, 0, 0, 1625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, - 126, 127, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 129, 130, 0, 131, 132, 133, 0, 135, 136, - 137, 138, 139, 0, 141, 142, 0, 143, 144, 145, - 146, 147, 148, 0, 0, 149, 150, 151, 152, 153, - 154, 155, 0, 156, 157, 158, 159, 160, 0, 0, - 0, 162, 163, 164, 165, 166, 167, 0, 169, 170, - 171, 0, 172, 173, 174, 175, 176, 177, 0, 0, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, - 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, - 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, - 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, - 220, 221, 222, 223, 224, 225, 226, 0, 227, 0, - 228, 229, 230, 231, 0, 232, 0, 233, 0, 0, - 0, 236, 237, 524, 0, 240, 0, 241, 0, 242, - 243, 244, 245, 0, 246, 247, 248, 249, 250, 896, - 252, 0, 254, 255, 256, 257, 0, 258, 259, 260, - 261, 262, 263, 264, 0, 265, 0, 267, 268, 269, - 270, 271, 272, 273, 274, 0, 275, 0, 276, 0, - 0, 279, 0, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 0, 289, 0, 0, 291, 292, 293, 294, - 295, 296, 297, 298, 525, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 0, 322, 323, 324, - 325, 326, 0, 327, 328, 0, 330, 0, 331, 332, - 333, 334, 335, 336, 0, 337, 338, 0, 791, 339, - 340, 341, 0, 0, 342, 343, 344, 0, 346, 0, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 0, 0, 0, 0, 360, 361, 362, 0, - 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, - 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 397, 398, 0, - 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 0, 0, - 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 526, 444, 445, - 0, 0, 446, 447, 0, 448, 0, 450, 451, 452, - 453, 454, 0, 455, 456, 457, 0, 0, 458, 459, - 460, 461, 462, 0, 463, 464, 465, 466, 467, 468, - 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, - 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, - 0, 485, 0, 487, 488, 489, 490, 491, 492, 493, - 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, - 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, - 510, 511, 512, 513, 514, 515, 523, 0, 549, 0, + 0, 0, 0, 1626, 0, 0, 0, 0, 1627, 1636, + 0, 0, 1637, 1638, 1639, 0, 1640, 1641, 1642, 1643, + 1644, 1645, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1628, 1629, 0, 0, 0, 0, 0, 0, 0, + 1611, 0, 0, 1612, 0, 0, 1630, 1613, 1614, 1615, + 1616, 1617, 1618, 1619, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1635, 0, 1620, 0, + 0, 0, 2772, 0, 0, 0, 0, 0, 1622, 0, + 0, 0, 0, 0, 1631, 1623, 1636, 1632, 0, 1637, + 1638, 1639, 0, 1640, 1641, 1642, 1643, 1644, 1645, 0, + 0, 1633, 0, 0, 1634, 0, 0, 0, 0, 0, + 0, 0, 1624, 0, 1611, 0, 0, 1612, 0, 0, + 0, 1613, 1614, 1615, 1616, 1617, 1618, 1619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 938, 125, 126, 127, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 129, 130, 0, 131, - 132, 133, 0, 135, 136, 137, 138, 139, 0, 141, - 142, 0, 143, 144, 145, 146, 147, 148, 0, 0, - 149, 150, 151, 152, 153, 154, 155, 0, 156, 157, - 158, 159, 160, 0, 0, 0, 162, 163, 164, 165, - 166, 167, 0, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 0, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, - 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 226, 0, 227, 0, 228, 229, 230, 231, 0, - 232, 0, 233, 0, 0, 0, 236, 237, 524, 0, - 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, - 247, 248, 249, 250, 251, 252, 0, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 0, 267, 268, 269, 270, 271, 272, 273, 274, - 0, 275, 0, 276, 0, 0, 279, 0, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 0, 289, 0, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 525, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 0, 322, 323, 324, 325, 326, 0, 327, 328, - 0, 330, 0, 331, 332, 333, 334, 335, 336, 0, - 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, - 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 362, 0, 364, 365, 366, 367, 368, - 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 0, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 0, 0, 427, 428, 429, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 526, 444, 445, 0, 0, 446, 447, 0, - 448, 0, 450, 451, 452, 453, 454, 0, 455, 456, - 457, 0, 0, 458, 459, 460, 461, 462, 0, 463, - 464, 465, 466, 467, 468, 469, 470, 0, 0, 471, - 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 483, 484, 0, 485, 0, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, - 515, 523, 1942, 0, 0, 0, 0, 1943, 0, 0, + 0, 0, 1620, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1622, 0, 0, 0, 0, 0, 0, 1623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, - 126, 127, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 129, 130, 0, 131, 132, 133, 0, 135, 136, - 137, 138, 139, 0, 141, 142, 0, 143, 144, 145, - 146, 147, 148, 0, 0, 149, 150, 151, 152, 153, - 154, 155, 0, 156, 157, 158, 159, 160, 0, 0, - 0, 162, 163, 164, 165, 166, 167, 0, 169, 170, - 171, 0, 172, 173, 174, 175, 176, 177, 0, 0, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, - 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, - 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, - 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, - 220, 221, 222, 223, 224, 225, 226, 0, 227, 0, - 228, 229, 230, 231, 0, 232, 0, 233, 0, 0, - 0, 236, 237, 524, 0, 240, 0, 241, 0, 242, - 243, 244, 245, 0, 246, 247, 248, 249, 250, 251, - 252, 0, 254, 255, 256, 257, 0, 258, 259, 260, - 261, 262, 263, 264, 0, 265, 0, 267, 268, 269, - 270, 271, 272, 273, 274, 0, 275, 0, 276, 0, - 0, 279, 0, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 0, 289, 0, 0, 291, 292, 293, 294, - 295, 296, 297, 298, 525, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 0, 322, 323, 324, - 325, 326, 0, 327, 328, 0, 330, 0, 331, 332, - 333, 334, 335, 336, 0, 337, 338, 0, 0, 339, - 340, 341, 0, 0, 342, 343, 344, 0, 346, 0, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 0, 0, 0, 0, 360, 361, 362, 0, - 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, - 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 397, 398, 0, - 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 0, 0, - 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 526, 444, 445, - 0, 0, 446, 447, 0, 448, 0, 450, 451, 452, - 453, 454, 0, 455, 456, 457, 0, 0, 458, 459, - 460, 461, 462, 0, 463, 464, 465, 466, 467, 468, - 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, - 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, - 0, 485, 0, 487, 488, 489, 490, 491, 492, 493, - 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, - 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, - 510, 511, 512, 513, 514, 515, 523, 0, 549, 0, + 0, 0, 0, 1636, 0, 0, 1637, 1638, 1639, 0, + 1640, 1641, 1642, 1643, 1644, 1645, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 129, 130, 0, 131, - 132, 133, 0, 135, 136, 137, 138, 139, 0, 141, - 142, 0, 143, 144, 145, 146, 147, 148, 0, 0, - 149, 150, 151, 152, 153, 154, 155, 0, 156, 157, - 158, 159, 160, 0, 0, 0, 162, 163, 164, 165, - 166, 167, 0, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 0, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, - 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 226, 0, 227, 0, 228, 229, 230, 231, 0, - 232, 0, 233, 0, 0, 0, 236, 237, 524, 0, - 1950, 0, 241, 0, 242, 243, 244, 245, 0, 246, - 247, 248, 249, 250, 251, 252, 0, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 0, 267, 268, 269, 270, 271, 272, 273, 274, - 0, 275, 0, 276, 0, 0, 279, 0, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 0, 289, 0, - 0, 291, 292, 1951, 294, 295, 296, 297, 298, 525, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 0, 322, 323, 324, 325, 326, 0, 327, 328, - 0, 330, 0, 331, 332, 333, 334, 335, 336, 0, - 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, - 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 362, 0, 364, 365, 366, 367, 368, - 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 0, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 0, 0, 427, 428, 429, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 526, 444, 445, 0, 0, 446, 447, 1952, - 448, 0, 450, 1953, 452, 1954, 454, 0, 455, 456, - 457, 0, 0, 458, 459, 460, 461, 462, 0, 463, - 464, 465, 466, 467, 468, 469, 470, 0, 0, 471, - 472, 1955, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 483, 484, 0, 485, 0, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, - 515, 523, 0, 549, 0, 0, 0, 0, 0, 0, + 0, 1635, 0, 1611, 0, 0, 1612, 0, 1625, 0, + 1613, 1614, 1615, 1616, 1617, 1618, 1619, 0, 0, 0, + 0, 0, 0, 0, 0, 1626, 0, 0, 0, 0, + 1627, 1620, 0, 0, 2759, 0, 0, 0, 0, 0, + 0, 1622, 0, 0, 0, 0, 0, 0, 1623, 0, + 0, 0, 0, 1628, 1629, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1630, 0, + 0, 0, 0, 0, 0, 1624, 0, 0, 0, 0, + 0, 0, 1625, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1626, + 0, 0, 0, 0, 1627, 0, 1631, 0, 1636, 1632, + 0, 1637, 1638, 1639, 0, 1640, 1641, 1642, 1643, 1644, + 1645, 0, 0, 1633, 0, 0, 1634, 1628, 1629, 0, + 0, 0, 0, 0, 0, 0, 1611, 0, 0, 1612, + 0, 0, 1630, 1613, 1614, 1615, 1616, 1617, 1618, 1619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, - 126, 127, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 129, 130, 0, 131, 132, 133, 0, 135, 136, - 137, 138, 139, 0, 141, 142, 0, 143, 144, 145, - 146, 147, 148, 0, 0, 149, 150, 151, 152, 153, - 154, 155, 0, 156, 157, 158, 159, 160, 0, 0, - 0, 162, 163, 164, 165, 166, 167, 0, 169, 170, - 171, 0, 172, 173, 174, 175, 176, 177, 0, 0, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, - 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, - 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, - 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, - 220, 221, 222, 223, 224, 225, 226, 0, 227, 0, - 228, 229, 230, 231, 0, 232, 0, 233, 0, 0, - 0, 236, 237, 524, 0, 240, 0, 241, 0, 242, - 243, 244, 245, 0, 246, 247, 248, 249, 250, 251, - 252, 0, 254, 255, 256, 257, 0, 258, 259, 260, - 261, 262, 263, 264, 0, 265, 0, 267, 268, 269, - 270, 271, 272, 273, 274, 0, 275, 0, 276, 0, - 0, 279, 0, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 0, 289, 0, 0, 291, 292, 293, 294, - 295, 296, 297, 298, 525, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 0, 322, 323, 324, - 325, 326, 0, 327, 328, 0, 330, 0, 331, 332, - 333, 334, 335, 336, 0, 337, 338, 0, 0, 339, - 340, 341, 0, 0, 342, 343, 344, 0, 346, 0, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 0, 0, 0, 0, 360, 361, 362, 0, - 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, - 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 397, 398, 0, - 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 0, 0, - 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 526, 444, 445, - 0, 0, 446, 447, 0, 448, 0, 450, 451, 452, - 453, 454, 0, 455, 456, 457, 0, 0, 458, 459, - 460, 461, 462, 0, 463, 464, 465, 466, 467, 468, - 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, - 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, - 0, 485, 0, 487, 488, 489, 490, 491, 492, 493, - 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, - 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, - 510, 511, 512, 513, 514, 515, 523, 0, 815, 0, + 0, 0, 0, 0, 1620, 0, 0, 0, 0, 0, + 0, 1625, 0, 0, 1622, 0, 0, 0, 0, 0, + 1631, 1623, 0, 1632, 0, 0, 0, 0, 1626, 0, + 0, 0, 0, 1627, 0, 0, 0, 1633, 0, 0, + 1634, 0, 0, 0, 0, 0, 0, 0, 1624, 0, + 0, 0, 0, 0, 0, 0, 1806, 1629, 0, 0, + 0, 0, 0, 1635, 0, 0, 0, 0, 0, 0, + 0, 1630, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1611, 0, 0, 1612, 0, 0, 0, 1613, + 1614, 0, 0, 1617, 1618, 1619, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1631, + 1620, 0, 1632, 0, 0, 0, 0, 0, 0, 0, + 1622, 0, 0, 0, 0, 0, 1633, 1623, 0, 1634, + 0, 0, 0, 0, 0, 0, 0, 1635, 0, 0, + 0, 0, 0, 0, 1625, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1624, 0, 0, 0, 0, 0, + 0, 1626, 0, 0, 0, 0, 1627, 0, 0, 0, + 1636, 0, 0, 1637, 1638, 1639, 0, 1640, 1641, 1642, + 1643, 1644, 1645, 0, 0, 1611, 0, 0, 1612, 1628, + 1629, 0, 1613, 1614, 0, 0, 1617, 1618, 1619, 0, + 0, 0, 0, 0, 1630, 0, 0, 0, 0, 0, + 0, 0, 0, 1620, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1622, 0, 1611, 1635, 0, 1612, 0, + 1623, 0, 1613, 1614, 0, 0, 1617, 1618, 1619, 0, + 0, 0, 1631, 0, 1636, 1632, 0, 1637, 1638, 1639, + 1625, 1640, 1641, 1642, 1643, 1644, 1645, 1624, 0, 1633, + 0, 0, 1634, 1622, 0, 0, 0, 1626, 0, 0, + 1623, 0, 1627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 129, 130, 0, 131, - 132, 133, 0, 135, 136, 137, 138, 139, 0, 141, - 142, 0, 143, 144, 145, 146, 147, 148, 0, 0, - 149, 150, 151, 152, 153, 154, 155, 0, 156, 157, - 158, 159, 160, 0, 0, 0, 162, 163, 164, 165, - 166, 167, 0, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 0, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, - 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 226, 0, 227, 0, 228, 229, 230, 231, 0, - 232, 0, 233, 0, 0, 0, 236, 237, 524, 0, - 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, - 247, 248, 249, 250, 251, 252, 0, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 0, 267, 268, 269, 270, 271, 272, 273, 274, - 0, 275, 0, 276, 0, 0, 279, 0, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 0, 289, 0, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 525, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 0, 322, 323, 324, 325, 326, 0, 327, 328, - 0, 330, 0, 331, 332, 333, 334, 335, 336, 0, - 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, - 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 362, 0, 364, 365, 366, 367, 368, - 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 0, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 0, 0, 427, 428, 429, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 526, 444, 445, 0, 0, 446, 447, 0, - 448, 0, 450, 451, 452, 453, 454, 0, 455, 456, - 457, 0, 0, 458, 459, 460, 461, 462, 0, 463, - 464, 465, 466, 467, 468, 469, 470, 0, 0, 471, - 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 483, 484, 0, 485, 0, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, - 515, 523, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1628, 1629, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 118, 119, 120, 121, 122, 123, 124, 822, 125, - 126, 127, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 129, 130, 0, 131, 132, 133, 0, 135, 136, - 137, 138, 139, 0, 141, 142, 0, 143, 144, 145, - 146, 147, 148, 0, 0, 149, 150, 151, 152, 153, - 154, 155, 0, 156, 157, 158, 159, 160, 0, 0, - 0, 162, 163, 164, 165, 166, 167, 0, 169, 170, - 171, 0, 172, 173, 174, 175, 176, 177, 0, 0, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, - 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, - 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, - 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, - 220, 221, 222, 223, 224, 225, 226, 0, 227, 0, - 228, 229, 230, 231, 0, 232, 0, 233, 0, 0, - 0, 236, 237, 524, 0, 823, 0, 241, 0, 242, - 243, 244, 245, 0, 246, 247, 248, 249, 250, 251, - 252, 0, 254, 255, 256, 257, 0, 258, 259, 260, - 261, 262, 263, 264, 0, 265, 0, 267, 268, 269, - 270, 271, 272, 273, 274, 0, 275, 0, 276, 0, - 0, 279, 0, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 0, 289, 0, 0, 291, 292, 824, 294, - 295, 296, 297, 298, 525, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 0, 322, 323, 324, - 325, 326, 0, 327, 328, 0, 330, 0, 331, 332, - 333, 334, 335, 336, 0, 337, 338, 0, 0, 339, - 340, 341, 0, 0, 342, 343, 344, 0, 346, 0, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 0, 0, 0, 0, 360, 361, 362, 0, - 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, - 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 397, 398, 0, - 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, - 418, 419, 420, 825, 422, 423, 424, 425, 0, 0, - 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 526, 444, 445, - 0, 0, 446, 447, 0, 448, 0, 450, 451, 452, - 453, 454, 0, 455, 826, 457, 0, 0, 827, 459, - 460, 461, 462, 0, 463, 464, 465, 466, 467, 468, - 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, - 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, - 0, 485, 0, 487, 488, 489, 490, 491, 492, 493, - 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, - 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, - 510, 511, 512, 513, 514, 515, 523, 0, 549, 0, + 1630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 129, 130, 0, 131, - 132, 133, 0, 135, 136, 137, 138, 139, 0, 141, - 142, 0, 143, 144, 145, 146, 147, 148, 0, 0, - 149, 150, 151, 152, 153, 154, 155, 0, 156, 157, - 158, 159, 160, 0, 0, 0, 162, 163, 164, 165, - 166, 167, 0, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 0, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, - 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 226, 0, 227, 0, 228, 229, 230, 231, 0, - 232, 0, 233, 0, 0, 0, 236, 237, 524, 0, - 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, - 247, 248, 249, 250, 251, 252, 0, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 0, 267, 268, 269, 270, 271, 272, 273, 274, - 0, 275, 0, 276, 0, 0, 279, 0, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 0, 289, 0, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 525, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 0, 322, 323, 324, 325, 326, 0, 327, 328, - 0, 330, 0, 331, 332, 333, 334, 335, 336, 0, - 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, - 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 362, 0, 364, 365, 366, 367, 368, - 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 379, 380, 381, 859, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 0, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 0, 0, 427, 428, 429, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 526, 444, 445, 0, 0, 446, 447, 0, - 448, 0, 450, 451, 452, 453, 454, 0, 455, 456, - 457, 0, 0, 458, 459, 460, 461, 462, 0, 463, - 464, 465, 466, 467, 468, 469, 470, 0, 0, 471, - 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 483, 484, 0, 485, 0, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, - 515, 523, 0, 549, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1636, 0, 0, 1637, 1638, 1639, 0, + 1640, 1641, 1642, 1643, 1644, 1645, 0, 0, 1631, 0, + 0, 1632, 0, 1625, 0, 0, 0, 0, 0, 1635, + 0, 0, 0, 0, 0, 1633, 0, 0, 1634, 0, + 1626, 0, 0, 0, 0, 1627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, - 126, 127, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 129, 130, 0, 131, 132, 133, 0, 135, 136, - 137, 138, 139, 0, 141, 142, 0, 143, 144, 145, - 146, 147, 148, 0, 0, 149, 150, 151, 152, 153, - 154, 155, 0, 156, 157, 158, 159, 160, 0, 0, - 0, 162, 163, 164, 165, 166, 167, 0, 169, 170, - 171, 0, 172, 173, 174, 175, 176, 177, 0, 0, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, - 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, - 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, - 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, - 220, 221, 222, 223, 224, 225, 226, 0, 227, 0, - 228, 229, 230, 231, 0, 232, 0, 233, 0, 0, - 0, 236, 237, 524, 0, 240, 0, 241, 0, 242, - 243, 244, 245, 0, 246, 247, 248, 249, 250, 891, - 252, 0, 254, 255, 256, 257, 0, 258, 259, 260, - 261, 262, 263, 264, 0, 265, 0, 267, 268, 269, - 270, 271, 272, 273, 274, 0, 275, 0, 276, 0, - 0, 279, 0, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 0, 289, 0, 0, 291, 292, 293, 294, - 295, 296, 297, 298, 525, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 0, 322, 323, 324, - 325, 326, 0, 327, 328, 0, 330, 0, 331, 332, - 333, 334, 335, 336, 0, 337, 338, 0, 0, 339, - 340, 341, 0, 0, 342, 343, 344, 0, 346, 0, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 0, 0, 0, 0, 360, 361, 362, 0, - 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, - 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 397, 398, 0, - 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 0, 0, - 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 526, 444, 445, - 0, 0, 446, 447, 0, 448, 0, 450, 451, 452, - 453, 454, 0, 455, 456, 457, 0, 0, 458, 459, - 460, 461, 462, 0, 463, 464, 465, 466, 467, 468, - 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, - 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, - 0, 485, 0, 487, 488, 489, 490, 491, 492, 493, - 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, - 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, - 510, 511, 512, 513, 514, 515, 523, 0, 549, 0, + 0, 0, 0, 1625, 0, 0, 0, 0, 1628, 1629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 129, 130, 0, 131, - 132, 133, 0, 135, 136, 137, 138, 139, 0, 141, - 142, 0, 143, 144, 145, 146, 147, 148, 0, 0, - 149, 150, 151, 152, 153, 154, 155, 0, 156, 157, - 158, 159, 160, 0, 0, 0, 162, 163, 164, 165, - 166, 167, 0, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 0, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, - 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 226, 0, 227, 0, 228, 229, 230, 231, 0, - 232, 0, 233, 0, 0, 0, 236, 237, 524, 0, - 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, - 247, 248, 249, 250, 894, 252, 0, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 0, 267, 268, 269, 270, 271, 272, 273, 274, - 0, 275, 0, 276, 0, 0, 279, 0, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 0, 289, 0, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 525, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 0, 322, 323, 324, 325, 326, 0, 327, 328, - 0, 330, 0, 331, 332, 333, 334, 335, 336, 0, - 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, - 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 362, 0, 364, 365, 366, 367, 368, - 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 0, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 0, 0, 427, 428, 429, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 526, 444, 445, 0, 0, 446, 447, 0, - 448, 0, 450, 451, 452, 453, 454, 0, 455, 456, - 457, 0, 0, 458, 459, 460, 461, 462, 0, 463, - 464, 465, 466, 467, 468, 469, 470, 0, 0, 471, - 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 483, 484, 0, 485, 0, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, - 515, 523, 0, 549, 0, 0, 0, 0, 0, 0, + 1626, 0, 0, 1630, 0, 1627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, - 126, 127, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 129, 130, 0, 131, 132, 133, 0, 135, 136, - 137, 138, 139, 0, 141, 142, 0, 143, 144, 145, - 146, 147, 148, 0, 0, 149, 150, 151, 152, 153, - 154, 155, 0, 156, 157, 158, 159, 160, 0, 0, - 0, 162, 163, 164, 165, 166, 167, 0, 169, 170, - 171, 0, 172, 173, 174, 175, 176, 177, 0, 0, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, - 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, - 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, - 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, - 220, 221, 222, 223, 224, 225, 226, 0, 227, 0, - 228, 229, 230, 231, 0, 232, 0, 233, 0, 0, - 0, 236, 237, 524, 0, 240, 0, 241, 0, 242, - 243, 244, 245, 0, 246, 247, 248, 249, 250, 898, - 252, 0, 254, 255, 256, 257, 0, 258, 259, 260, - 261, 262, 263, 264, 0, 265, 0, 267, 268, 269, - 270, 271, 272, 273, 274, 0, 275, 0, 276, 0, - 0, 279, 0, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 0, 289, 0, 0, 291, 292, 293, 294, - 295, 296, 297, 298, 525, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 0, 322, 323, 324, - 325, 326, 0, 327, 328, 0, 330, 0, 331, 332, - 333, 334, 335, 336, 0, 337, 338, 0, 0, 339, - 340, 341, 0, 0, 342, 343, 344, 0, 346, 0, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 0, 0, 0, 0, 360, 361, 362, 0, - 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, - 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 397, 398, 0, - 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 0, 0, - 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 526, 444, 445, - 0, 0, 446, 447, 0, 448, 0, 450, 451, 452, - 453, 454, 0, 455, 456, 457, 0, 0, 458, 459, - 460, 461, 462, 0, 463, 464, 465, 466, 467, 468, - 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, - 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, - 0, 485, 0, 487, 488, 489, 490, 491, 492, 493, - 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, - 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, - 510, 511, 512, 513, 514, 515, 523, 0, 549, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1628, 1629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 129, 130, 0, 131, - 132, 133, 0, 135, 136, 137, 138, 139, 0, 141, - 142, 0, 143, 144, 145, 146, 147, 148, 0, 0, - 149, 150, 151, 152, 153, 154, 155, 0, 156, 157, - 158, 159, 160, 0, 0, 0, 162, 163, 164, 165, - 166, 167, 0, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 0, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, - 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 226, 0, 227, 0, 228, 229, 230, 231, 0, - 232, 0, 233, 0, 0, 0, 236, 237, 524, 0, - 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, - 247, 248, 249, 250, 926, 252, 0, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 0, 267, 268, 269, 270, 271, 272, 273, 274, - 0, 275, 0, 276, 0, 0, 279, 0, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 0, 289, 0, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 525, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 0, 322, 323, 324, 325, 326, 0, 327, 328, - 0, 330, 0, 331, 332, 333, 334, 335, 336, 0, - 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, - 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 362, 0, 364, 365, 366, 367, 368, - 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 0, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 0, 0, 427, 428, 429, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 526, 444, 445, 0, 0, 446, 447, 0, - 448, 0, 450, 451, 452, 453, 454, 0, 455, 456, - 457, 0, 0, 458, 459, 460, 461, 462, 0, 463, - 464, 465, 466, 467, 468, 469, 470, 0, 0, 471, - 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 483, 484, 0, 485, 0, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, - 515, 523, 0, 549, 0, 0, 0, 0, 0, 0, + 0, 1631, 0, 1630, 1632, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1635, 1636, 0, 1633, 1637, + 1638, 1639, 0, 1640, 1641, 1642, 1643, 2188, 1645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, - 126, 127, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 129, 130, 0, 131, 132, 133, 0, 135, 136, - 137, 138, 139, 0, 141, 142, 0, 143, 144, 145, - 146, 147, 148, 0, 0, 149, 150, 151, 152, 153, - 154, 155, 0, 156, 157, 158, 159, 160, 0, 0, - 0, 162, 163, 164, 165, 166, 167, 0, 169, 170, - 171, 0, 172, 173, 174, 175, 176, 177, 0, 0, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, - 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, - 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, - 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, - 220, 221, 222, 223, 224, 225, 226, 0, 227, 0, - 228, 229, 230, 231, 0, 232, 0, 233, 0, 0, - 0, 236, 237, 524, 0, 240, 0, 241, 0, 242, - 243, 244, 245, 0, 246, 247, 248, 249, 250, 954, - 252, 0, 254, 255, 256, 257, 0, 258, 259, 260, - 261, 262, 263, 264, 0, 265, 0, 267, 268, 269, - 270, 271, 272, 273, 274, 0, 275, 0, 276, 0, - 0, 279, 0, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 0, 289, 0, 0, 291, 292, 293, 294, - 295, 296, 297, 298, 525, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 0, 322, 323, 324, - 325, 326, 0, 327, 328, 0, 330, 0, 331, 332, - 333, 334, 335, 336, 0, 337, 338, 0, 0, 339, - 340, 341, 0, 0, 342, 343, 344, 0, 346, 0, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 0, 0, 0, 0, 360, 361, 362, 0, - 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, - 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 397, 398, 0, - 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 0, 0, - 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 526, 444, 445, - 0, 0, 446, 447, 0, 448, 0, 450, 451, 452, - 453, 454, 0, 455, 456, 457, 0, 0, 458, 459, - 460, 461, 462, 0, 463, 464, 465, 466, 467, 468, - 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, - 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, - 0, 485, 0, 487, 488, 489, 490, 491, 492, 493, - 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, - 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, - 510, 511, 512, 513, 514, 515, 523, 0, 549, 0, + 0, 1631, 0, 0, 1632, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 129, 130, 0, 131, - 132, 133, 0, 135, 136, 137, 138, 139, 0, 141, - 142, 0, 143, 144, 145, 146, 147, 148, 0, 0, - 149, 150, 151, 152, 153, 154, 155, 0, 156, 157, - 158, 159, 160, 0, 0, 0, 162, 163, 164, 165, - 166, 167, 0, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 0, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, - 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 226, 0, 227, 0, 228, 229, 230, 231, 0, - 232, 0, 233, 0, 0, 0, 236, 237, 524, 0, - 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, - 247, 248, 249, 250, 957, 252, 0, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 0, 267, 268, 269, 270, 271, 272, 273, 274, - 0, 275, 0, 276, 0, 0, 279, 0, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 0, 289, 0, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 525, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 0, 322, 323, 324, 325, 326, 0, 327, 328, - 0, 330, 0, 331, 332, 333, 334, 335, 336, 0, - 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, - 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 362, 0, 364, 365, 366, 367, 368, - 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 0, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 0, 0, 427, 428, 429, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 526, 444, 445, 0, 0, 446, 447, 0, - 448, 0, 450, 451, 452, 453, 454, 0, 455, 456, - 457, 0, 0, 458, 459, 460, 461, 462, 0, 463, - 464, 465, 466, 467, 468, 469, 470, 0, 0, 471, - 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 483, 484, 0, 485, 0, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, - 515, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, - 126, 127, 0, 0, 0, 0, 0, 0, 1000, 0, - 0, 129, 130, 0, 131, 132, 133, 0, 135, 136, - 137, 138, 139, 0, 141, 142, 0, 143, 144, 145, - 146, 147, 148, 0, 0, 149, 150, 151, 152, 153, - 154, 155, 0, 156, 157, 158, 159, 160, 0, 0, - 0, 162, 163, 164, 165, 166, 167, 0, 169, 170, - 171, 0, 172, 173, 174, 175, 176, 177, 0, 0, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, - 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, - 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, - 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, - 220, 221, 222, 223, 224, 225, 226, 0, 227, 0, - 228, 229, 230, 231, 0, 232, 0, 233, 0, 0, - 0, 236, 237, 524, 0, 240, 0, 241, 0, 242, - 243, 244, 245, 0, 246, 247, 248, 249, 250, 251, - 252, 0, 254, 255, 256, 257, 0, 258, 259, 260, - 261, 262, 263, 264, 0, 265, 0, 267, 268, 269, - 270, 271, 272, 273, 274, 0, 275, 0, 276, 0, - 0, 279, 0, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 0, 289, 0, 0, 291, 292, 293, 294, - 295, 296, 297, 298, 525, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 0, 322, 323, 324, - 325, 326, 0, 327, 328, 0, 330, 0, 331, 332, - 333, 334, 335, 336, 0, 337, 338, 0, 0, 339, - 340, 341, 0, 0, 342, 343, 344, 0, 346, 0, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 0, 0, 0, 0, 360, 361, 362, 0, - 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, - 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 397, 398, 0, - 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 0, 0, - 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 526, 444, 445, - 0, 0, 446, 447, 0, 448, 0, 450, 451, 452, - 453, 454, 0, 455, 456, 457, 0, 0, 458, 459, - 460, 461, 462, 0, 463, 464, 465, 466, 467, 468, - 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, - 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, - 0, 485, 0, 487, 488, 489, 490, 491, 492, 493, - 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, - 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, - 510, 511, 512, 513, 514, 515, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 0, 0, 0, 1023, 0, 0, 129, 130, 0, 131, - 132, 133, 0, 135, 136, 137, 138, 139, 0, 141, - 142, 0, 143, 144, 145, 146, 147, 148, 0, 0, - 149, 150, 151, 152, 153, 154, 155, 0, 156, 157, - 158, 159, 160, 0, 0, 0, 162, 163, 164, 165, - 166, 167, 0, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 0, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, - 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 226, 0, 227, 0, 228, 229, 230, 231, 0, - 232, 0, 233, 0, 0, 0, 236, 237, 524, 0, - 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, - 247, 248, 249, 250, 251, 252, 0, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 0, 267, 268, 269, 270, 271, 272, 273, 274, - 0, 275, 0, 276, 0, 0, 279, 0, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 0, 289, 0, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 525, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 0, 322, 323, 324, 325, 326, 0, 327, 328, - 0, 330, 0, 331, 332, 333, 334, 335, 336, 0, - 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, - 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 362, 0, 364, 365, 366, 367, 368, - 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 0, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 0, 0, 427, 428, 429, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 526, 444, 445, 0, 0, 446, 447, 0, - 448, 0, 450, 451, 452, 453, 454, 0, 455, 456, - 457, 0, 0, 458, 459, 460, 461, 462, 0, 463, - 464, 465, 466, 467, 468, 469, 470, 0, 0, 471, - 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 483, 484, 0, 485, 0, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, - 515, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 118, 119, 120, 121, 122, 123, 124, 822, 125, - 126, 127, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 129, 130, 0, 131, 132, 133, 0, 135, 136, - 137, 138, 139, 0, 141, 142, 0, 143, 144, 145, - 146, 147, 148, 0, 0, 149, 150, 151, 152, 153, - 154, 155, 0, 156, 157, 158, 159, 160, 0, 0, - 0, 162, 163, 164, 165, 166, 167, 0, 169, 170, - 171, 0, 172, 173, 174, 175, 176, 177, 0, 0, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, - 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, - 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, - 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, - 220, 221, 222, 223, 224, 225, 226, 0, 227, 0, - 228, 229, 230, 231, 0, 232, 0, 233, 0, 0, - 0, 236, 237, 524, 0, 240, 0, 241, 0, 242, - 243, 244, 245, 0, 246, 247, 248, 249, 250, 251, - 252, 0, 254, 255, 256, 257, 0, 258, 259, 260, - 261, 262, 263, 264, 0, 265, 0, 267, 268, 269, - 270, 271, 272, 273, 274, 0, 275, 0, 276, 0, - 0, 279, 0, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 0, 289, 0, 0, 291, 292, 293, 294, - 295, 296, 297, 298, 525, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 0, 322, 323, 324, - 325, 326, 0, 327, 328, 0, 330, 0, 331, 332, - 333, 334, 335, 336, 0, 337, 338, 0, 0, 339, - 340, 341, 0, 0, 342, 343, 344, 0, 346, 0, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 0, 0, 0, 0, 360, 361, 362, 0, - 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, - 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 397, 398, 0, - 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 0, 0, - 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 526, 444, 445, - 0, 0, 446, 447, 0, 448, 0, 450, 451, 452, - 453, 454, 0, 455, 826, 457, 0, 0, 827, 459, - 460, 461, 462, 0, 463, 464, 465, 466, 467, 468, - 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, - 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, - 0, 485, 0, 487, 488, 489, 490, 491, 492, 493, - 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, - 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, - 510, 511, 512, 513, 514, 515, 523, 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 129, 130, 0, 131, - 132, 133, 0, 135, 136, 137, 138, 139, 0, 141, - 142, 0, 143, 144, 145, 146, 147, 148, 0, 0, - 149, 150, 151, 152, 153, 154, 155, 0, 156, 157, - 158, 159, 160, 0, 0, 0, 162, 163, 164, 165, - 166, 167, 0, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 0, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, - 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 226, 0, 227, 0, 228, 229, 230, 231, 0, - 232, 0, 233, 0, 0, 0, 236, 237, 524, 0, - 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, - 247, 248, 249, 250, 1302, 252, 0, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 0, 267, 268, 269, 270, 271, 272, 273, 274, - 0, 275, 0, 276, 0, 0, 279, 0, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 0, 289, 0, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 525, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 0, 322, 323, 324, 325, 326, 0, 327, 328, - 0, 330, 0, 331, 332, 333, 334, 335, 336, 0, - 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, - 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 362, 0, 364, 365, 366, 367, 368, - 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 0, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 0, 0, 427, 428, 429, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 526, 444, 445, 0, 0, 446, 447, 0, - 448, 0, 450, 451, 452, 453, 454, 0, 455, 456, - 457, 0, 0, 458, 459, 460, 461, 462, 0, 463, - 464, 465, 466, 467, 468, 469, 470, 0, 0, 471, - 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 483, 484, 0, 485, 0, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, - 515, 523, 0, 549, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1635, 0, + 0, 0, 1636, 0, 0, 1637, 1638, 1639, 0, 1640, + 1641, 1642, 1643, 1644, 1645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, - 126, 127, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 129, 130, 0, 131, 132, 133, 0, 135, 136, - 137, 138, 139, 0, 141, 142, 0, 143, 144, 145, - 146, 147, 148, 0, 0, 149, 150, 151, 152, 153, - 154, 155, 0, 156, 157, 158, 159, 160, 0, 0, - 0, 162, 163, 164, 165, 166, 167, 0, 169, 170, - 171, 0, 172, 173, 174, 175, 176, 177, 0, 0, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, - 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, - 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, - 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, - 220, 221, 222, 223, 224, 225, 226, 0, 227, 0, - 228, 229, 230, 231, 0, 232, 0, 233, 0, 0, - 0, 236, 237, 524, 0, 240, 0, 241, 0, 242, - 243, 244, 245, 0, 246, 247, 248, 249, 250, 1304, - 252, 0, 254, 255, 256, 257, 0, 258, 259, 260, - 261, 262, 263, 264, 0, 265, 0, 267, 268, 269, - 270, 271, 272, 273, 274, 0, 275, 0, 276, 0, - 0, 279, 0, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 0, 289, 0, 0, 291, 292, 293, 294, - 295, 296, 297, 298, 525, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 0, 322, 323, 324, - 325, 326, 0, 327, 328, 0, 330, 0, 331, 332, - 333, 334, 335, 336, 0, 337, 338, 0, 0, 339, - 340, 341, 0, 0, 342, 343, 344, 0, 346, 0, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 0, 0, 0, 0, 360, 361, 362, 0, - 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, - 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 397, 398, 0, - 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 0, 0, - 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 526, 444, 445, - 0, 0, 446, 447, 0, 448, 0, 450, 451, 452, - 453, 454, 0, 455, 456, 457, 0, 0, 458, 459, - 460, 461, 462, 0, 463, 464, 465, 466, 467, 468, - 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, - 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, - 0, 485, 0, 487, 488, 489, 490, 491, 492, 493, - 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, - 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, - 510, 511, 512, 513, 514, 515, 523, 0, 549, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 129, 130, 0, 131, - 132, 133, 0, 135, 136, 137, 138, 139, 0, 141, - 142, 0, 143, 144, 145, 146, 147, 148, 0, 0, - 149, 150, 151, 152, 153, 154, 155, 0, 156, 157, - 158, 159, 160, 0, 0, 0, 162, 163, 164, 165, - 166, 167, 0, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 0, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, - 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 226, 0, 227, 0, 228, 229, 230, 231, 0, - 232, 0, 233, 0, 0, 0, 236, 237, 524, 0, - 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, - 247, 248, 249, 250, 1307, 252, 0, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 0, 267, 268, 269, 270, 271, 272, 273, 274, - 0, 275, 0, 276, 0, 0, 279, 0, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 0, 289, 0, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 525, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 0, 322, 323, 324, 325, 326, 0, 327, 328, - 0, 330, 0, 331, 332, 333, 334, 335, 336, 0, - 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, - 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 362, 0, 364, 365, 366, 367, 368, - 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 0, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 0, 0, 427, 428, 429, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 526, 444, 445, 0, 0, 446, 447, 0, - 448, 0, 450, 451, 452, 453, 454, 0, 455, 456, - 457, 0, 0, 458, 459, 460, 461, 462, 0, 463, - 464, 465, 466, 467, 468, 469, 470, 0, 0, 471, - 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 483, 484, 0, 485, 0, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, - 515, 523, 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, - 126, 127, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 129, 130, 0, 131, 132, 133, 0, 135, 136, - 137, 138, 139, 0, 141, 142, 0, 143, 144, 145, - 146, 147, 148, 0, 0, 149, 150, 151, 152, 153, - 154, 155, 0, 156, 157, 158, 159, 160, 0, 0, - 0, 162, 163, 164, 165, 166, 167, 0, 169, 170, - 171, 0, 172, 173, 174, 175, 176, 177, 0, 0, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, - 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, - 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, - 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, - 220, 221, 222, 223, 224, 225, 226, 0, 227, 0, - 228, 229, 230, 231, 0, 232, 0, 233, 0, 0, - 0, 236, 237, 524, 0, 240, 0, 241, 0, 242, - 243, 244, 245, 0, 246, 247, 248, 249, 250, 1309, - 252, 0, 254, 255, 256, 257, 0, 258, 259, 260, - 261, 262, 263, 264, 0, 265, 0, 267, 268, 269, - 270, 271, 272, 273, 274, 0, 275, 0, 276, 0, - 0, 279, 0, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 0, 289, 0, 0, 291, 292, 293, 294, - 295, 296, 297, 298, 525, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 0, 322, 323, 324, - 325, 326, 0, 327, 328, 0, 330, 0, 331, 332, - 333, 334, 335, 336, 0, 337, 338, 0, 0, 339, - 340, 341, 0, 0, 342, 343, 344, 0, 346, 0, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 0, 0, 0, 0, 360, 361, 362, 0, - 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, - 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 397, 398, 0, - 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 0, 0, - 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 526, 444, 445, - 0, 0, 446, 447, 0, 448, 0, 450, 451, 452, - 453, 454, 0, 455, 456, 457, 0, 0, 458, 459, - 460, 461, 462, 0, 463, 464, 465, 466, 467, 468, - 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, - 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, - 0, 485, 0, 487, 488, 489, 490, 491, 492, 493, - 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, - 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, - 510, 511, 512, 513, 514, 515, 523, 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 129, 130, 0, 131, - 132, 133, 0, 135, 136, 137, 138, 139, 0, 141, - 142, 0, 143, 144, 145, 146, 147, 148, 0, 0, - 149, 150, 151, 152, 153, 154, 155, 0, 156, 157, - 158, 159, 160, 0, 0, 0, 162, 163, 164, 165, - 166, 167, 0, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 0, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, - 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 226, 0, 227, 0, 228, 229, 230, 231, 0, - 232, 0, 233, 0, 0, 0, 236, 237, 524, 0, - 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, - 247, 248, 249, 250, 2228, 252, 0, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 0, 267, 268, 269, 270, 271, 272, 273, 274, - 0, 275, 0, 276, 0, 0, 279, 0, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 0, 289, 0, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 525, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 0, 322, 323, 324, 325, 326, 0, 327, 328, - 0, 330, 0, 331, 332, 333, 334, 335, 336, 0, - 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, - 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 362, 0, 364, 365, 366, 367, 368, - 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 0, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 0, 0, 427, 428, 429, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 526, 444, 445, 0, 0, 446, 447, 0, - 448, 0, 450, 451, 452, 453, 454, 0, 455, 456, - 457, 0, 0, 458, 459, 460, 461, 462, 0, 463, - 464, 465, 466, 467, 468, 469, 470, 0, 0, 471, - 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 483, 484, 0, 485, 0, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, - 515, 1475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, - 126, 127, 0, 0, 0, 1476, 0, 0, -787, 0, - 1477, 129, 130, 0, 131, 132, 133, 1478, 135, 136, - 137, 0, 1479, 1480, 1481, 1482, 0, 143, 144, 145, - 146, 147, 148, 0, 0, 149, 150, 151, 152, 1483, - 1484, 155, 0, 156, 157, 158, 159, 0, 0, 1485, - 0, 1486, 163, 164, 165, 166, 167, 1487, 169, 170, - 171, 0, 172, 173, 174, 175, 176, 177, 0, 1488, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 1489, 190, 191, 1490, 193, 0, 194, 0, 195, 196, - 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, - 0, 0, 205, 206, 1059, 208, 209, 0, 210, 211, - 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, - 0, 221, 222, 223, 224, 225, 0, 0, 227, 0, - 228, 229, 1491, 231, 0, 232, 0, 233, 1492, 0, - 1493, 236, 237, -787, 1494, 240, 0, 241, 0, 0, - 0, 244, 245, 0, 246, 247, 248, 249, 250, 251, - 252, 1495, 254, 255, 256, 257, 0, 258, 259, 260, - 261, 262, 263, 264, 0, 265, 1496, 0, 268, 269, - 270, 271, 272, 1497, 1498, 0, 1499, 0, 276, 1500, - 1501, 279, 1502, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 1503, 289, 1504, 0, 291, 292, 293, 294, - 295, 296, 297, 298, 1505, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 1506, 1507, 1508, 323, 324, - 325, 0, 0, 327, 328, 1509, 330, 0, 0, 332, - 1510, 334, 335, 336, 0, 337, 338, 0, 0, 339, - 340, 341, 0, 0, 342, 343, 0, 1511, 346, 1512, - 0, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 0, 0, 0, 0, 360, 361, 0, 1513, - 364, 365, 0, 367, 368, 369, 0, 370, 371, 372, - 373, 374, 375, 0, 376, 377, 378, 1514, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 397, 398, 1515, - 400, 401, 402, 1516, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 0, 1517, 417, - 418, 419, 420, 421, 422, 1518, 424, 425, 0, 1519, - 427, 428, 1520, 430, 0, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 1521, 444, 0, - 0, 0, 446, 447, 0, 448, 1522, 450, 451, 452, - 453, 454, 0, 455, 1523, 1524, 0, 0, 458, 459, - 0, 461, 0, 0, 463, 464, 1525, 466, 467, 468, - 469, 470, 1526, 0, 471, 472, 473, 0, 474, 475, - 476, 477, 0, 478, 479, 480, 481, 482, 0, 1527, - 0, 485, 1528, 487, 488, 489, 490, 491, 492, 493, - 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, - 500, 523, 0, 549, 0, 0, 0, 0, 0, 0, - 0, 0, 512, 513, 514, 515, 0, 0, 0, 0, - 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, - 126, 127, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 129, 130, 0, 131, 132, 133, 0, 135, 136, - 137, 138, 139, 0, 141, 142, 0, 143, 144, 145, - 146, 147, 148, 0, 0, 149, 150, 151, 152, 153, - 154, 155, 0, 156, 157, 158, 159, 160, 0, 0, - 0, 162, 163, 164, 165, 166, 167, 0, 169, 170, - 171, 0, 172, 173, 174, 175, 176, 177, 0, 0, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, - 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, - 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, - 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, - 220, 221, 222, 223, 224, 225, 226, 0, 227, 0, - 228, 229, 230, 231, 0, 232, 0, 233, 0, 0, - 0, 236, 237, 524, 0, 240, 0, 241, 0, 242, - 243, 244, 245, 0, 246, 247, 248, 249, 250, 2971, - 252, 0, 254, 255, 256, 257, 0, 258, 259, 260, - 261, 262, 263, 264, 0, 265, 0, 267, 268, 269, - 270, 271, 272, 273, 274, 0, 275, 0, 276, 0, - 0, 279, 0, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 0, 289, 0, 0, 291, 292, 293, 294, - 295, 296, 297, 298, 525, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 0, 322, 323, 324, - 325, 326, 0, 327, 328, 0, 330, 0, 331, 332, - 333, 334, 335, 336, 0, 337, 338, 0, 0, 339, - 340, 341, 0, 0, 342, 343, 344, 0, 346, 0, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 0, 0, 0, 0, 360, 361, 362, 0, - 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, - 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 397, 398, 0, - 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 0, 0, - 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 526, 444, 445, - 0, 0, 446, 447, 0, 448, 0, 450, 451, 452, - 453, 454, 0, 455, 456, 457, 0, 0, 458, 459, - 460, 461, 462, 0, 463, 464, 465, 466, 467, 468, - 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, - 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, - 0, 485, 0, 487, 488, 489, 490, 491, 492, 493, - 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, - 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, - 510, 511, 512, 513, 514, 515, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 129, 130, 0, 131, - 132, 133, 0, 135, 136, 137, 138, 139, 0, 141, - 142, 0, 143, 144, 145, 146, 147, 148, 0, 0, - 149, 150, 151, 152, 153, 154, 155, 0, 156, 157, - 158, 159, 160, 0, 0, 0, 162, 163, 164, 165, - 166, 167, 0, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 0, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, - 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 226, 0, 227, 0, 228, 229, 230, 231, 0, - 232, 0, 233, 0, 0, 0, 236, 237, 524, 0, - 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, - 247, 248, 249, 250, 251, 252, 0, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 0, 267, 268, 269, 270, 271, 272, 273, 274, - 0, 275, 0, 276, 0, 0, 279, 0, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 0, 289, 0, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 525, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 0, 322, 323, 324, 325, 326, 0, 327, 328, - 0, 330, 0, 331, 332, 333, 334, 335, 336, 0, - 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, - 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 362, 0, 364, 365, 366, 367, 368, - 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 0, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 0, 0, 427, 428, 429, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 526, 444, 445, 0, 0, 446, 447, 0, - 448, 0, 450, 451, 452, 453, 454, 0, 455, 456, - 457, 0, 0, 458, 459, 460, 461, 462, 0, 463, - 464, 465, 466, 467, 468, 469, 470, 0, 0, 471, - 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 483, 484, 0, 485, 0, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, - 515, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, - 126, 127, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 129, 130, 0, 131, 132, 133, 0, 135, 136, - 137, 138, 139, 0, 141, 142, 0, 143, 144, 145, - 146, 147, 148, 0, 0, 149, 150, 151, 152, 153, - 154, 155, 0, 156, 157, 158, 159, 160, 0, 0, - 0, 162, 163, 164, 165, 166, 167, 0, 169, 170, - 171, 0, 172, 173, 174, 175, 176, 177, 0, 0, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, - 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, - 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, - 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, - 220, 221, 222, 223, 224, 225, 226, 0, 227, 0, - 228, 229, 230, 231, 0, 232, 0, 233, 0, 0, - 0, 236, 237, 524, 0, 837, 0, 241, 0, 242, - 243, 244, 245, 0, 246, 247, 248, 249, 250, 251, - 252, 0, 254, 255, 256, 257, 0, 258, 259, 260, - 261, 262, 263, 264, 0, 265, 0, 267, 268, 269, - 270, 271, 272, 273, 274, 0, 275, 0, 276, 0, - 0, 279, 0, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 0, 289, 0, 0, 291, 292, 838, 294, - 295, 296, 297, 298, 525, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 0, 322, 323, 324, - 325, 326, 0, 327, 328, 0, 330, 0, 331, 332, - 333, 334, 335, 336, 0, 337, 338, 0, 0, 339, - 340, 341, 0, 0, 342, 343, 344, 0, 346, 0, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 0, 0, 0, 0, 360, 361, 362, 0, - 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, - 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 397, 398, 0, - 400, 401, 402, 403, 404, 405, 406, 407, 839, 409, - 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, - 418, 419, 420, 840, 422, 423, 424, 425, 0, 0, - 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 526, 444, 445, - 0, 0, 446, 447, 0, 448, 0, 450, 451, 452, - 453, 454, 0, 455, 841, 457, 0, 0, 458, 459, - 460, 461, 462, 0, 463, 464, 465, 466, 467, 468, - 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, - 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, - 0, 485, 0, 487, 488, 489, 490, 491, 492, 493, - 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, - 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, - 510, 511, 512, 513, 514, 515, 523, 0, 0, 0, + 0, 0, 0, 0, 0, 1636, 0, 0, 1637, 1638, + 1639, 0, 1640, 1641, 1642, 1643, 1644, 1645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 129, 130, 0, 131, - 132, 133, 0, 135, 136, 137, 138, 139, 0, 141, - 142, 0, 143, 144, 145, 146, 147, 148, 0, 0, - 149, 150, 151, 152, 153, 154, 155, 0, 156, 157, - 158, 159, 160, 0, 0, 0, 162, 163, 164, 165, - 166, 167, 0, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 0, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, - 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 226, 0, 227, 0, 228, 229, 230, 231, 0, - 232, 0, 233, 0, 0, 0, 236, 237, 524, 0, - 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, - 247, 248, 249, 250, 950, 252, 0, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 0, 267, 268, 269, 270, 271, 272, 273, 274, - 0, 275, 0, 276, 0, 0, 279, 0, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 0, 289, 0, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 525, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 0, 322, 323, 324, 325, 326, 0, 327, 328, - 0, 330, 0, 331, 332, 333, 334, 335, 336, 0, - 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, - 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 362, 0, 364, 365, 366, 367, 368, - 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 0, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 0, 0, 427, 428, 429, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 526, 444, 445, 0, 0, 446, 447, 0, - 448, 0, 450, 451, 452, 453, 454, 0, 455, 456, - 457, 0, 0, 458, 459, 460, 461, 462, 0, 463, - 464, 465, 466, 467, 468, 469, 470, 0, 0, 471, - 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 483, 484, 0, 485, 0, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, - 515, 523, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, + 0, 2085, 0, 0, 0, 1636, 0, 0, 1637, 1638, + 1639, 0, 1640, 1641, 1642, 1643, 1644, 1645, 117, 118, + 119, 120, 121, 122, 123, 124, 556, 125, 126, 127, + 557, 558, 559, 560, 561, 562, 563, 564, 565, 129, + 130, 566, 131, 132, 133, 567, 135, 136, 137, 568, + 569, 570, 571, 572, 573, 143, 144, 145, 146, 147, + 148, 574, 575, 149, 150, 151, 152, 576, 577, 155, + 578, 156, 157, 158, 159, 579, 580, 581, 582, 583, + 163, 164, 165, 166, 167, 584, 169, 170, 171, 585, + 172, 173, 174, 175, 176, 177, 586, 587, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 589, 190, + 191, 590, 193, 591, 194, 592, 195, 196, 197, 198, + 199, 200, 593, 594, 201, 202, 203, 204, 595, 596, + 205, 206, 207, 208, 209, 597, 210, 211, 212, 598, + 213, 214, 215, 599, 216, 217, 218, 219, 600, 221, + 222, 223, 224, 225, 601, 602, 227, 603, 228, 229, + 604, 231, 605, 232, 606, 233, 607, 608, 609, 236, + 237, 610, 611, 240, 612, 241, 613, 614, 615, 244, + 245, 616, 246, 247, 248, 249, 250, 251, 252, 617, + 254, 255, 256, 257, 618, 258, 259, 260, 261, 262, + 263, 264, 619, 265, 620, 621, 268, 269, 270, 271, + 272, 622, 623, 624, 625, 626, 276, 627, 628, 279, + 629, 281, 282, 283, 284, 285, 286, 630, 631, 287, + 632, 289, 633, 634, 291, 292, 293, 294, 295, 296, + 297, 298, 635, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 636, 637, 638, 323, 324, 325, 639, + 640, 327, 328, 641, 330, 642, 643, 332, 644, 334, + 335, 336, 645, 337, 338, 646, 647, 339, 340, 341, + 648, 649, 342, 343, 650, 651, 346, 652, 653, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 654, 655, 656, 657, 360, 361, 658, 659, 364, 365, + 660, 367, 368, 369, 661, 370, 371, 372, 373, 374, + 375, 662, 376, 377, 378, 663, 380, 381, 382, 383, + 664, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 665, 397, 398, 666, 400, 401, + 402, 667, 404, 405, 406, 407, 408, 409, 410, 411, + 412, 413, 414, 415, 416, 668, 669, 417, 418, 419, + 420, 421, 422, 670, 424, 425, 671, 672, 427, 428, + 673, 430, 674, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 675, 444, 676, 677, 678, + 446, 447, 679, 448, 680, 450, 451, 452, 453, 454, + 681, 455, 682, 683, 684, 685, 458, 459, 686, 461, + 687, 688, 463, 464, 689, 466, 467, 468, 469, 470, + 690, 691, 471, 472, 473, 692, 474, 475, 476, 477, + 693, 478, 479, 480, 481, 482, 694, 695, 696, 485, + 697, 487, 488, 489, 490, 491, 492, 493, 698, 699, + 494, 700, 701, 495, 496, 497, 498, 499, 500, 702, + 703, 704, 705, 706, 707, 708, 709, 710, 711, 712, + 512, 513, 514, 515, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, - 126, 127, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 129, 130, 0, 131, 132, 133, 0, 135, 136, - 137, 138, 139, 0, 141, 142, 0, 143, 144, 145, - 146, 147, 148, 0, 0, 149, 150, 151, 152, 153, - 154, 155, 0, 156, 157, 158, 159, 160, 0, 0, - 0, 162, 163, 164, 165, 166, 167, 0, 169, 170, - 171, 0, 172, 173, 174, 175, 176, 177, 0, 0, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, - 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, - 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, - 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, - 220, 221, 222, 223, 224, 225, 226, 0, 227, 0, - 228, 229, 230, 231, 0, 232, 0, 233, 0, 0, - 0, 236, 237, 524, 0, 240, 0, 241, 0, 242, - 243, 244, 245, 0, 246, 247, 248, 249, 250, 251, - 252, 0, 254, 255, 256, 257, 0, 258, 259, 260, - 261, 262, 263, 264, 0, 265, 0, 267, 268, 269, - 270, 271, 272, 273, 274, 0, 275, 0, 276, 0, - 0, 279, 0, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 0, 289, 0, 0, 291, 292, 293, 294, - 295, 296, 297, 298, 525, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 0, 322, 323, 324, - 325, 326, 0, 327, 328, 0, 330, 0, 331, 332, - 333, 334, 335, 336, 0, 337, 338, 0, 0, 339, - 340, 341, 0, 0, 342, 343, 344, 0, 346, 0, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 0, 0, 0, 0, 360, 361, 362, 0, - 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, - 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 397, 398, 0, - 400, 401, 402, 403, 404, 405, 406, 407, 839, 409, - 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 0, 0, - 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 526, 444, 445, - 0, 0, 446, 447, 0, 448, 0, 450, 451, 452, - 453, 454, 0, 455, 841, 457, 0, 0, 458, 459, - 460, 461, 462, 0, 463, 464, 465, 466, 467, 468, - 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, - 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, - 0, 485, 0, 487, 488, 489, 490, 491, 492, 493, - 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, - 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, - 510, 511, 512, 513, 514, 515, 523, 0, 0, 0, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 556, 125, 126, 127, 557, 558, 559, 560, 561, + 562, 563, 564, 565, 129, 130, 566, 131, 132, 133, + 567, 135, 136, 137, 568, 569, 570, 571, 572, 573, + 143, 144, 145, 146, 147, 148, 574, 575, 149, 150, + 151, 152, 576, 577, 155, 578, 156, 157, 158, 159, + 579, 580, 581, 582, 583, 163, 164, 165, 166, 167, + 584, 169, 170, 171, 585, 172, 173, 174, 175, 176, + 177, 586, 587, 179, 180, 181, 182, 183, 184, 588, + 186, 187, 188, 589, 190, 191, 590, 193, 591, 194, + 592, 195, 196, 197, 198, 199, 200, 593, 594, 201, + 202, 203, 204, 595, 596, 205, 206, 207, 208, 209, + 597, 210, 211, 212, 598, 213, 214, 215, 599, 216, + 217, 218, 219, 600, 221, 222, 223, 224, 225, 601, + 602, 227, 603, 228, 229, 604, 231, 605, 232, 606, + 233, 607, 608, 609, 236, 237, 610, 611, 240, 612, + 241, 613, 614, 615, 244, 245, 616, 246, 247, 248, + 249, 250, 251, 252, 617, 254, 255, 256, 257, 618, + 258, 259, 260, 261, 262, 263, 264, 619, 265, 620, + 621, 268, 269, 270, 271, 272, 622, 623, 624, 625, + 626, 276, 627, 628, 279, 629, 281, 282, 283, 284, + 285, 286, 630, 631, 287, 632, 289, 633, 634, 291, + 292, 293, 294, 295, 296, 297, 298, 635, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 636, 637, + 638, 323, 324, 325, 639, 640, 327, 328, 641, 330, + 642, 643, 332, 644, 334, 335, 336, 645, 337, 338, + 646, 647, 339, 340, 341, 648, 649, 342, 343, 650, + 651, 346, 652, 653, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 654, 655, 656, 657, 360, + 361, 658, 659, 364, 365, 660, 367, 368, 369, 661, + 370, 371, 372, 373, 374, 375, 662, 376, 377, 378, + 663, 380, 381, 382, 383, 664, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 665, + 397, 398, 666, 400, 401, 402, 667, 404, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 668, 669, 417, 418, 419, 420, 421, 422, 670, 424, + 425, 671, 672, 427, 428, 673, 430, 674, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 675, 444, 676, 677, 678, 446, 447, 679, 448, 680, + 450, 451, 452, 453, 454, 681, 455, 682, 683, 684, + 685, 458, 459, 686, 461, 687, 688, 463, 464, 689, + 466, 467, 468, 469, 470, 690, 691, 471, 472, 473, + 692, 474, 475, 476, 477, 693, 478, 479, 480, 481, + 482, 694, 695, 696, 485, 697, 487, 488, 489, 490, + 491, 492, 493, 698, 699, 494, 700, 701, 495, 496, + 497, 498, 499, 500, 702, 703, 704, 705, 706, 707, + 708, 709, 710, 711, 712, 512, 513, 514, 515, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 129, 130, 0, 131, - 132, 133, 0, 135, 136, 137, 138, 139, 0, 141, - 142, 0, 143, 144, 145, 146, 147, 148, 0, 0, - 149, 150, 151, 152, 153, 154, 155, 0, 156, 157, - 158, 159, 160, 0, 0, 0, 162, 163, 164, 165, - 166, 167, 0, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 0, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, - 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 226, 0, 227, 0, 228, 229, 230, 231, 0, - 232, 0, 233, 0, 0, 0, 236, 237, 524, 0, - 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, - 247, 248, 249, 250, 1298, 252, 0, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 0, 267, 268, 269, 270, 271, 272, 273, 274, - 0, 275, 0, 276, 0, 0, 279, 0, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 0, 289, 0, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 525, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 0, 322, 323, 324, 325, 326, 0, 327, 328, - 0, 330, 0, 331, 332, 333, 334, 335, 336, 0, - 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, - 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 362, 0, 364, 365, 366, 367, 368, - 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 0, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 0, 0, 427, 428, 429, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 526, 444, 445, 0, 0, 446, 447, 0, - 448, 0, 450, 451, 452, 453, 454, 0, 455, 456, - 457, 0, 0, 458, 459, 460, 461, 462, 0, 463, - 464, 465, 466, 467, 468, 469, 470, 0, 0, 471, - 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 483, 484, 0, 485, 0, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, - 515, 523, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, + 119, 120, 121, 122, 123, 124, 556, 125, 126, 127, + 557, 558, 559, 560, 561, 562, 563, 564, 565, 129, + 130, 566, 131, 132, 133, 567, 135, 136, 137, 568, + 569, 570, 571, 572, 573, 143, 144, 145, 146, 147, + 148, 574, 575, 149, 150, 151, 152, 576, 577, 155, + 578, 156, 157, 158, 159, 579, 580, 581, 582, 583, + 163, 164, 165, 166, 167, 584, 169, 170, 171, 585, + 172, 173, 174, 175, 176, 177, 586, 587, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 589, 190, + 191, 590, 193, 591, 194, 592, 195, 196, 197, 198, + 199, 200, 593, 594, 201, 202, 203, 204, 595, 596, + 205, 206, 207, 208, 209, 597, 210, 211, 212, 598, + 213, 214, 215, 599, 216, 217, 218, 219, 600, 221, + 222, 223, 224, 225, 601, 602, 227, 603, 228, 229, + 604, 231, 605, 232, 606, 233, 607, 608, 609, 236, + 237, 610, 611, 240, 612, 241, 613, 614, 615, 244, + 245, 616, 246, 247, 248, 249, 250, 940, 252, 617, + 254, 255, 256, 257, 618, 258, 259, 260, 261, 262, + 263, 264, 619, 265, 620, 621, 268, 269, 270, 271, + 272, 622, 623, 624, 625, 626, 276, 627, 628, 279, + 629, 281, 282, 283, 284, 285, 286, 630, 631, 287, + 632, 289, 633, 634, 291, 292, 293, 294, 295, 296, + 297, 298, 635, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 636, 637, 638, 323, 324, 325, 639, + 640, 327, 328, 641, 330, 642, 643, 332, 644, 334, + 335, 336, 645, 337, 338, 646, 647, 339, 340, 341, + 648, 649, 342, 343, 650, 651, 346, 652, 653, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 654, 655, 656, 657, 360, 361, 658, 659, 364, 365, + 660, 367, 368, 369, 661, 370, 371, 372, 373, 374, + 375, 662, 376, 377, 378, 663, 380, 381, 382, 383, + 664, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 665, 397, 398, 666, 400, 401, + 402, 667, 404, 405, 406, 407, 408, 409, 410, 411, + 412, 413, 414, 415, 416, 668, 669, 417, 418, 419, + 420, 421, 422, 670, 424, 425, 671, 672, 427, 428, + 673, 430, 674, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 675, 444, 676, 677, 678, + 446, 447, 679, 448, 680, 450, 451, 452, 453, 454, + 681, 455, 682, 683, 684, 685, 458, 459, 686, 461, + 687, 688, 463, 464, 689, 466, 467, 468, 469, 470, + 690, 691, 471, 472, 473, 692, 474, 475, 476, 477, + 693, 478, 479, 480, 481, 482, 694, 695, 696, 485, + 697, 487, 488, 489, 490, 491, 492, 493, 698, 699, + 494, 700, 701, 495, 496, 497, 498, 499, 500, 702, + 703, 704, 705, 706, 707, 708, 709, 710, 711, 712, + 512, 513, 514, 515, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, - 126, 127, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 129, 130, 0, 131, 132, 133, 0, 135, 136, - 137, 138, 139, 0, 141, 142, 0, 143, 144, 145, - 146, 147, 148, 0, 0, 149, 150, 151, 152, 153, - 154, 155, 0, 156, 157, 158, 159, 160, 0, 0, - 0, 162, 163, 164, 165, 166, 167, 0, 169, 170, - 171, 0, 172, 173, 174, 175, 176, 177, 0, 0, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, - 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, - 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, - 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, - 220, 221, 222, 223, 224, 225, 226, 0, 227, 0, - 228, 229, 230, 231, 0, 232, 0, 233, 0, 0, - 0, 236, 237, 524, 0, 240, 0, 241, 0, 242, - 243, 244, 245, 0, 246, 247, 248, 249, 250, 1319, - 252, 0, 254, 255, 256, 257, 0, 258, 259, 260, - 261, 262, 263, 264, 0, 265, 0, 267, 268, 269, - 270, 271, 272, 273, 274, 0, 275, 0, 276, 0, - 0, 279, 0, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 0, 289, 0, 0, 291, 292, 293, 294, - 295, 296, 297, 298, 525, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 0, 322, 323, 324, - 325, 326, 0, 327, 328, 0, 330, 0, 331, 332, - 333, 334, 335, 336, 0, 337, 338, 0, 0, 339, - 340, 341, 0, 0, 342, 343, 344, 0, 346, 0, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 0, 0, 0, 0, 360, 361, 362, 0, - 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, - 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 397, 398, 0, - 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 0, 0, - 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 526, 444, 445, - 0, 0, 446, 447, 0, 448, 0, 450, 451, 452, - 453, 454, 0, 455, 456, 457, 0, 0, 458, 459, - 460, 461, 462, 0, 463, 464, 465, 466, 467, 468, - 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, - 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, - 0, 485, 0, 487, 488, 489, 490, 491, 492, 493, - 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, - 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, - 510, 511, 512, 513, 514, 515, 523, 0, 0, 0, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 556, 125, 126, 127, 557, 558, 559, 560, 561, + 562, 563, 564, 565, 129, 130, 566, 131, 132, 133, + 567, 135, 136, 137, 568, 569, 570, 571, 572, 573, + 143, 144, 145, 146, 147, 148, 574, 575, 149, 150, + 151, 152, 576, 577, 155, 578, 156, 157, 158, 159, + 579, 580, 581, 582, 583, 163, 164, 165, 166, 167, + 584, 169, 170, 171, 585, 172, 173, 174, 175, 176, + 177, 586, 587, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 589, 190, 191, 590, 193, 591, 194, + 592, 195, 196, 197, 198, 199, 200, 593, 594, 201, + 202, 203, 204, 595, 596, 205, 206, 207, 208, 209, + 597, 210, 211, 212, 598, 213, 214, 215, 599, 216, + 217, 218, 219, 600, 221, 222, 223, 224, 225, 601, + 602, 227, 603, 228, 229, 604, 231, 605, 232, 606, + 233, 607, 608, 609, 236, 237, 610, 611, 240, 612, + 241, 613, 614, 615, 244, 245, 616, 246, 247, 248, + 249, 250, 251, 252, 617, 254, 255, 256, 257, 618, + 258, 259, 260, 261, 262, 263, 264, 619, 265, 620, + 621, 268, 269, 270, 271, 272, 622, 623, 624, 625, + 626, 276, 627, 628, 279, 629, 281, 282, 283, 284, + 285, 286, 630, 631, 287, 632, 289, 633, 634, 291, + 292, 293, 294, 295, 296, 297, 298, 635, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 636, 637, + 638, 323, 324, 325, 639, 640, 327, 328, 641, 330, + 642, 643, 332, 644, 334, 335, 336, 645, 337, 338, + 646, 647, 339, 340, 341, 648, 649, 342, 343, 650, + 651, 346, 652, 653, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 654, 655, 656, 657, 360, + 361, 658, 659, 364, 365, 660, 367, 368, 369, 661, + 370, 371, 372, 373, 374, 375, 662, 376, 377, 378, + 663, 380, 381, 382, 383, 664, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 665, + 397, 398, 666, 400, 401, 402, 667, 404, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 668, 669, 417, 418, 419, 420, 421, 422, 670, 424, + 425, 671, 672, 427, 428, 673, 430, 674, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 675, 444, 676, 677, 678, 446, 447, 679, 448, 680, + 450, 451, 452, 453, 454, 681, 455, 682, 683, 684, + 685, 458, 459, 686, 461, 687, 688, 463, 464, 689, + 466, 467, 468, 469, 470, 690, 691, 471, 472, 473, + 692, 474, 475, 476, 477, 693, 478, 479, 480, 481, + 482, 694, 695, 696, 485, 697, 487, 488, 489, 490, + 491, 492, 493, 698, 699, 494, 700, 701, 495, 496, + 497, 498, 499, 500, 702, 703, 704, 705, 706, 707, + 708, 709, 710, 711, 712, 512, 513, 514, 515, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 0, 0, 0, 1670, 0, 0, 129, 130, 0, 131, - 132, 133, 0, 135, 136, 137, 138, 139, 0, 141, - 142, 0, 143, 144, 145, 146, 147, 148, 0, 0, - 149, 150, 151, 152, 153, 154, 155, 0, 156, 157, - 158, 159, 160, 0, 0, 0, 162, 163, 164, 165, - 166, 167, 0, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 0, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, - 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 226, 0, 227, 0, 228, 229, 230, 231, 0, - 232, 0, 233, 0, 0, 0, 236, 237, 524, 0, - 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, - 247, 248, 249, 250, 251, 252, 0, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 0, 267, 268, 269, 270, 271, 272, 273, 274, - 0, 275, 0, 276, 0, 0, 279, 0, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 0, 289, 0, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 525, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 0, 322, 323, 324, 325, 326, 0, 327, 328, - 0, 330, 0, 331, 332, 333, 334, 335, 336, 0, - 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, - 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 362, 0, 364, 365, 366, 367, 368, - 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 0, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 0, 417, 418, 419, 420, 421, 0, - 423, 424, 425, 0, 0, 427, 428, 429, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 526, 444, 445, 0, 0, 446, 447, 0, - 448, 0, 450, 451, 452, 453, 454, 0, 455, 456, - 457, 0, 0, 458, 459, 460, 461, 462, 0, 463, - 464, 465, 466, 467, 468, 469, 470, 0, 0, 471, - 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 483, 484, 0, 485, 0, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, - 515, 523, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, - 126, 127, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 129, 130, 0, 131, 132, 133, 0, 135, 136, - 137, 138, 139, 0, 141, 142, 0, 143, 144, 145, - 146, 147, 148, 0, 0, 149, 150, 151, 152, 153, - 154, 155, 0, 156, 157, 158, 159, 160, 0, 0, - 0, 162, 163, 164, 165, 166, 167, 0, 169, 170, - 171, 0, 172, 173, 174, 175, 176, 177, 0, 0, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, - 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, - 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, - 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, - 220, 221, 222, 223, 224, 225, 226, 0, 227, 0, - 228, 229, 230, 231, 0, 232, 0, 233, 0, 0, - 0, 236, 237, 524, 0, 240, 0, 241, 0, 242, - 243, 244, 245, 0, 246, 247, 248, 249, 250, 1846, - 252, 0, 254, 255, 256, 257, 0, 258, 259, 260, - 261, 262, 263, 264, 0, 265, 0, 267, 268, 269, - 270, 271, 272, 273, 274, 0, 275, 0, 276, 0, - 0, 279, 0, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 0, 289, 0, 0, 291, 292, 293, 294, - 295, 296, 297, 298, 525, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 0, 322, 323, 324, - 325, 326, 0, 327, 328, 0, 330, 0, 331, 332, - 333, 334, 335, 336, 0, 337, 338, 0, 0, 339, - 340, 341, 0, 0, 342, 343, 344, 0, 346, 0, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 0, 0, 0, 0, 360, 361, 362, 0, - 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, - 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 397, 398, 0, - 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 0, 0, - 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 526, 444, 445, - 0, 0, 446, 447, 0, 448, 0, 450, 451, 452, - 453, 454, 0, 455, 456, 457, 0, 0, 458, 459, - 460, 461, 462, 0, 463, 464, 465, 466, 467, 468, - 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, - 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, - 0, 485, 0, 487, 488, 489, 490, 491, 492, 493, - 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, - 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, - 510, 511, 512, 513, 514, 515, 523, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 129, 130, 0, 131, - 132, 133, 0, 135, 136, 137, 138, 139, 0, 141, - 142, 0, 143, 144, 145, 146, 147, 148, 0, 0, - 149, 150, 151, 152, 153, 154, 155, 0, 156, 157, - 158, 159, 160, 0, 0, 0, 162, 163, 164, 165, - 166, 167, 0, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 0, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, - 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 226, 0, 227, 0, 228, 229, 230, 231, 0, - 232, 0, 233, 0, 0, 0, 236, 237, 524, 0, - 240, 0, 241, 0, 242, 243, 244, 245, 0, 246, - 247, 248, 249, 250, 2215, 252, 0, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 0, 267, 268, 269, 270, 271, 272, 273, 274, - 0, 275, 0, 276, 0, 0, 279, 0, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 0, 289, 0, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 525, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 0, 322, 323, 324, 325, 326, 0, 327, 328, - 0, 330, 0, 331, 332, 333, 334, 335, 336, 0, - 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, - 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 362, 0, 364, 365, 366, 367, 368, - 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 0, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 0, 0, 427, 428, 429, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 526, 444, 445, 0, 0, 446, 447, 0, - 448, 0, 450, 451, 452, 453, 454, 0, 455, 456, - 457, 0, 0, 458, 459, 460, 461, 462, 0, 463, - 464, 465, 466, 467, 468, 469, 470, 0, 0, 471, - 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 483, 484, 0, 485, 0, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, - 515, 523, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, - 126, 127, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 129, 130, 0, 131, 132, 133, 0, 135, 136, - 137, 138, 139, 0, 141, 142, 0, 143, 144, 145, - 146, 147, 148, 0, 0, 149, 150, 151, 152, 153, - 154, 155, 0, 156, 157, 158, 159, 160, 0, 0, - 0, 162, 163, 164, 165, 166, 167, 0, 169, 170, - 171, 0, 172, 173, 174, 175, 176, 177, 0, 0, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 0, 194, 0, 195, 196, - 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, - 0, 0, 205, 206, 207, 208, 209, 0, 210, 211, - 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, - 220, 221, 222, 223, 224, 225, 226, 0, 227, 0, - 228, 229, 230, 231, 0, 232, 0, 233, 0, 0, - 0, 236, 237, 524, 0, 240, 0, 241, 0, 242, - 243, 244, 245, 0, 246, 247, 248, 249, 250, 2230, - 252, 0, 254, 255, 256, 257, 0, 258, 259, 260, - 261, 262, 263, 264, 0, 265, 0, 267, 268, 269, - 270, 271, 272, 273, 274, 0, 275, 0, 276, 0, - 0, 279, 0, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 0, 289, 0, 0, 291, 292, 293, 294, - 295, 296, 297, 298, 525, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 320, 0, 322, 323, 324, - 325, 326, 0, 327, 328, 0, 330, 0, 331, 332, - 333, 334, 335, 336, 0, 337, 338, 0, 0, 339, - 340, 341, 0, 0, 342, 343, 344, 0, 346, 0, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 0, 0, 0, 0, 360, 361, 362, 0, - 364, 365, 366, 367, 368, 369, 0, 370, 371, 372, - 373, 374, 375, 0, 376, 377, 378, 379, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 397, 398, 0, - 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 0, 0, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 0, 0, - 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 526, 444, 445, - 0, 0, 446, 447, 0, 448, 0, 450, 451, 452, - 453, 454, 0, 455, 456, 457, 0, 0, 458, 459, - 460, 461, 462, 0, 463, 464, 465, 466, 467, 468, - 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, - 476, 477, 0, 478, 479, 480, 481, 482, 483, 484, - 0, 485, 0, 487, 488, 489, 490, 491, 492, 493, - 0, 0, 494, 0, 0, 495, 496, 497, 498, 499, - 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, - 510, 511, 512, 513, 514, 515, 1475, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 1476, 0, 0, 0, 0, 1477, 129, 130, 0, 131, - 132, 133, 1478, 135, 136, 137, 0, 1479, 1480, 1481, - 1482, 0, 143, 144, 145, 146, 147, 148, 0, 0, - 149, 150, 151, 152, 1483, 1484, 155, 0, 156, 157, - 158, 159, 0, 0, 1485, 0, 1486, 163, 164, 165, - 166, 167, 1487, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 1488, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 1489, 190, 191, 1490, 193, - 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 1059, - 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 0, 221, 222, 223, 224, - 225, 0, 0, 227, 0, 228, 229, 1491, 231, 0, - 232, 0, 233, 1492, 0, 1493, 236, 237, 0, 1494, - 240, 0, 241, 0, 0, 0, 244, 245, 0, 246, - 247, 248, 249, 250, 251, 252, 1495, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 1496, 0, 268, 269, 270, 271, 272, 1497, 1498, - 0, 1499, 0, 276, 1500, 1501, 279, 1502, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 1503, 289, 1504, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 1505, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 1506, 1507, 1508, 323, 324, 325, 0, 0, 327, 328, - 1509, 330, 0, 0, 332, 1510, 334, 335, 336, 0, - 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, - 343, 0, 1511, 346, 1512, 0, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 0, 1513, 364, 365, 0, 367, 368, - 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 1514, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 1515, 400, 401, 402, 1516, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 1517, 417, 418, 419, 420, 421, 422, - 1518, 424, 425, 0, 1519, 427, 428, 1520, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 1521, 444, 0, 0, 0, 446, 447, 0, - 448, 1522, 450, 451, 452, 453, 454, 0, 455, 1523, - 1524, 0, 0, 458, 459, 0, 461, 0, 0, 463, - 464, 1525, 466, 467, 468, 469, 470, 1526, 0, 471, - 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 0, 1527, 0, 485, 1528, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 1475, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 512, 513, 514, - 515, 0, 0, 0, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 1476, 0, 0, 0, 0, 1477, 129, 130, 0, 131, - 132, 133, 1478, 135, 136, 137, 0, 1479, 1480, 1481, - 1482, 0, 143, 144, 145, 146, 147, 148, 0, 0, - 149, 150, 151, 152, 1483, 1484, 155, 0, 156, 157, - 158, 159, 0, 0, 1485, 0, 1486, 163, 164, 165, - 166, 167, 1487, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 1488, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 1489, 190, 191, 1490, 193, - 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 1059, - 208, 209, 0, 210, 211, 212, 0, 1828, 214, 215, - 0, 216, 217, 218, 219, 0, 221, 222, 223, 224, - 225, 0, 0, 227, 0, 228, 229, 1491, 231, 0, - 232, 0, 233, 1492, 0, 1493, 236, 237, 0, 1494, - 240, 0, 241, 0, 0, 0, 244, 245, 0, 246, - 247, 248, 249, 250, 251, 252, 1495, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 1496, 0, 268, 269, 270, 271, 272, 1497, 1498, - 0, 1499, 0, 276, 1500, 1501, 279, 1502, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 1503, 289, 1504, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 1505, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 1506, 1507, 1508, 323, 324, 325, 0, 0, 327, 328, - 1509, 330, 0, 0, 332, 1510, 334, 335, 336, 0, - 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, - 343, 0, 1511, 346, 1512, 0, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 0, 1513, 364, 365, 0, 367, 368, - 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 1514, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 1515, 400, 401, 402, 1516, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 1517, 417, 418, 419, 420, 421, 422, - 1518, 424, 425, 0, 1519, 427, 428, 1520, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 1521, 444, 0, 0, 0, 446, 447, 0, - 448, 1522, 450, 451, 452, 453, 454, 0, 455, 1523, - 1524, 0, 0, 458, 459, 0, 461, 0, 0, 463, - 464, 1525, 466, 467, 468, 469, 470, 1526, 0, 471, - 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 0, 1527, 0, 485, 1528, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 3141, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 512, 513, 514, - 515, 0, 0, 0, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 2893, 0, 0, 0, 0, 2894, 129, 130, 0, 131, - 132, 133, 2895, 135, 136, 137, 0, 1479, 2896, 1481, - 1482, 0, 143, 144, 145, 146, 147, 148, 0, 0, - 149, 150, 151, 152, 1483, 1484, 155, 0, 156, 157, - 158, 159, 0, 0, 2897, 0, 2898, 163, 164, 165, - 166, 167, 2899, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 2900, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 1489, 190, 191, 1490, 193, - 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 1059, - 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 0, 221, 222, 223, 224, - 225, 0, 0, 227, 0, 228, 229, 1491, 231, 0, - 232, 0, 233, 2901, 0, 2902, 236, 237, 2903, 2904, - 240, 0, 241, 0, 0, 0, 244, 245, 0, 246, - 247, 248, 249, 250, 251, 252, 2905, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 2906, 0, 268, 269, 270, 271, 272, 1497, 1498, - 0, 1499, 0, 276, 2907, 2908, 279, 2909, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 2910, 289, 2911, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 3142, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 1506, 2913, 1508, 323, 324, 325, 0, 0, 327, 328, - 2915, 330, 0, 0, 332, 1510, 334, 335, 336, 0, - 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, - 343, 0, 2917, 346, 2918, 0, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 0, 2919, 364, 365, 0, 367, 368, - 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 377, 378, 1514, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 2920, 400, 401, 402, 0, 404, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 2921, 417, 418, 419, 420, 421, 422, - 0, 424, 425, 0, 2923, 427, 428, 1520, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 3143, 444, 0, 0, 0, 446, 447, 0, - 448, 2925, 450, 451, 452, 453, 454, 0, 455, 1523, - 1524, 0, 0, 458, 459, 0, 461, 0, 0, 463, - 464, 2926, 466, 467, 468, 469, 470, 0, 0, 471, - 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 0, 1527, 0, 485, 2928, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 523, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 512, 513, 514, - 515, 0, 0, 0, 0, 117, 118, 119, 120, 121, - 122, 123, 124, 0, 125, 126, 127, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 129, 130, 0, 131, - 132, 133, 0, 135, 136, 137, 138, 139, 0, 141, - 142, 0, 143, 144, 145, 146, 147, 148, 0, 0, - 149, 150, 151, 152, 153, 154, 155, 0, 156, 157, - 158, 159, 160, 0, 0, 0, 162, 163, 164, 165, - 166, 167, 0, 169, 170, 171, 0, 172, 173, 174, - 175, 176, 177, 0, 0, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 0, 194, 0, 195, 196, 197, 198, 199, 200, 0, - 0, 201, 202, 203, 204, 0, 0, 205, 206, 207, - 208, 209, 0, 210, 211, 212, 0, 213, 214, 215, - 0, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 226, 0, 227, 0, 228, 229, 230, 231, 0, - 232, 0, 233, 0, 0, 0, 236, 237, 524, 0, - 240, 0, 241, 0, 242, 243, 0, 245, 0, 246, - 247, 248, 249, 250, 251, 252, 0, 254, 255, 256, - 257, 0, 258, 259, 260, 261, 262, 263, 264, 0, - 265, 0, 267, 268, 269, 270, 271, 272, 273, 274, - 0, 275, 0, 276, 0, 0, 279, 0, 281, 282, - 283, 284, 285, 286, 0, 0, 287, 0, 289, 0, - 0, 291, 292, 293, 294, 295, 296, 297, 298, 525, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, - 320, 0, 322, 323, 324, 325, 326, 0, 327, 328, - 0, 330, 0, 331, 332, 333, 334, 335, 336, 0, - 337, 338, 0, 0, 339, 340, 341, 0, 0, 342, - 343, 344, 0, 346, 0, 348, 349, 350, 351, 352, - 353, 354, 0, 356, 357, 358, 359, 0, 0, 0, - 0, 360, 361, 362, 0, 364, 365, 366, 367, 368, - 369, 0, 370, 371, 372, 373, 374, 375, 0, 376, - 0, 378, 379, 380, 381, 382, 383, 0, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 397, 398, 0, 400, 401, 402, 403, 0, - 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 0, 0, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 0, 0, 427, 428, 429, 430, 0, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 526, 444, 445, 0, 0, 446, 447, 0, - 448, 0, 450, 451, 452, 453, 454, 0, 455, 456, - 457, 0, 0, 458, 459, 460, 461, 462, 0, 463, - 464, 465, 466, 467, 468, 469, 470, 0, 0, 471, - 472, 473, 0, 474, 475, 476, 477, 0, 478, 479, - 480, 481, 482, 483, 484, 0, 485, 0, 487, 488, - 489, 490, 491, 492, 493, 0, 0, 494, 0, 0, - 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, - 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, - 515, 1765, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 117, 118, 119, 120, 121, 122, 123, 124, 0, 125, - 126, 127, 0, 0, 0, 1476, 0, 0, 0, 0, - 1477, 129, 130, 0, 131, 132, 133, 1478, 135, 136, - 137, 0, 1479, 1480, 1481, 1482, 0, 143, 144, 145, - 146, 147, 148, 0, 0, 149, 150, 151, 152, 1483, - 1484, 155, 0, 156, 157, 158, 159, 0, 0, 1485, - 0, 1486, 163, 164, 165, 166, 167, 1487, 169, 170, - 171, 0, 172, 173, 174, 175, 176, 177, 0, 1488, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 1489, 190, 191, 1490, 193, 0, 194, 0, 195, 196, - 197, 198, 199, 200, 0, 0, 201, 202, 203, 204, - 0, 0, 205, 206, 1059, 208, 209, 0, 210, 211, - 212, 0, 213, 214, 215, 0, 216, 217, 218, 219, - 0, 221, 222, 223, 224, 225, 0, 0, 227, 0, - 228, 229, 1491, 231, 0, 232, 0, 233, 1492, 0, - 1493, 236, 237, 0, 1494, 240, 0, 241, 0, 0, - 0, 244, 245, 0, 246, 247, 248, 249, 250, 251, - 252, 1495, 254, 255, 256, 257, 0, 258, 259, 260, - 261, 262, 263, 264, 0, 265, 1496, 0, 268, 269, - 270, 271, 272, 1497, 1498, 0, 1499, 0, 276, 1500, - 1501, 279, 1502, 281, 282, 283, 284, 285, 286, 0, - 0, 287, 1503, 289, 1504, 0, 291, 292, 293, 294, - 295, 296, 297, 298, 0, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, 317, 318, 319, 1506, 1507, 1508, 323, 324, - 325, 0, 0, 327, 328, 1509, 330, 0, 0, 332, - 1510, 334, 335, 336, 0, 337, 338, 0, 0, 339, - 340, 341, 0, 0, 342, 343, 0, 1511, 346, 1512, - 0, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 0, 0, 0, 0, 360, 361, 0, 1513, - 364, 365, 0, 367, 368, 369, 0, 370, 371, 372, - 373, 374, 375, 0, 376, 377, 378, 1514, 380, 381, - 382, 383, 0, 384, 385, 386, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 397, 398, 1515, - 400, 401, 402, 0, 404, 405, 406, 407, 408, 409, - 410, 411, 412, 413, 414, 415, 416, 0, 1517, 417, - 418, 419, 420, 421, 422, 0, 424, 425, 0, 1519, - 427, 428, 1520, 430, 0, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 0, 444, 0, - 0, 0, 446, 447, 0, 448, 1522, 450, 451, 452, - 453, 454, 0, 455, 1523, 1524, 0, 0, 458, 459, - 0, 461, 0, 0, 463, 464, 1525, 466, 467, 468, - 469, 470, 0, 0, 471, 472, 473, 0, 474, 475, - 476, 477, 0, 478, 479, 480, 481, 482, 0, 1527, - 0, 485, 1528, 487, 488, 489, 490, 491, 492, 493, - 0, 1, 494, 0, 0, 495, 496, 497, 498, 499, - 500, 2, 0, 3, 4, 0, 0, 0, 0, 1, - 0, 0, 512, 513, 514, 515, 0, 0, 0, 2, - 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 6, - 0, 0, 0, 0, 8, 0, 0, 0, 7, 0, - 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, - 0, 0, 8, 0, 0, 0, 0, 11, 0, 746, - 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, - 13, 0, 0, 0, 0, 11, 0, 746, 0, 0, - 0, 0, 0, 0, 0, 14, 15, 0, 13, 0, - 0, 0, 0, 0, 0, 0, 747, 0, 0, 0, - 0, 0, 18, 14, 15, 0, 0, 0, 0, 0, - 19, 0, 0, 0, 747, 0, 0, 0, 0, 0, - 18, 0, 0, 0, 0, 0, 22, 0, 19, 0, - 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 22, 0, 0, 0, 23, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 25, 0, 0, 0, 0, 0, 0, 0, 26, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, + 119, 120, 2233, 122, 123, 124, 556, 125, 126, 127, + 557, 558, 559, 560, 561, 562, 563, 564, 565, 129, + 130, 566, 131, 132, 133, 567, 135, 136, 137, 568, + 569, 570, 571, 572, 573, 143, 144, 145, 146, 147, + 148, 574, 575, 149, 150, 151, 152, 576, 577, 155, + 578, 156, 157, 158, 159, 579, 580, 581, 582, 583, + 163, 164, 165, 166, 167, 584, 169, 170, 171, 585, + 172, 173, 174, 175, 176, 177, 586, 587, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 589, 190, + 191, 590, 193, 591, 194, 592, 195, 196, 197, 198, + 199, 200, 593, 594, 201, 202, 203, 204, 595, 596, + 205, 206, 207, 2234, 209, 597, 210, 211, 212, 598, + 213, 214, 215, 599, 216, 217, 218, 219, 600, 221, + 222, 223, 224, 225, 601, 602, 227, 603, 228, 229, + 604, 231, 605, 232, 606, 233, 607, 608, 609, 236, + 237, 610, 611, 240, 612, 241, 613, 614, 615, 244, + 245, 616, 246, 247, 248, 249, 250, 251, 252, 617, + 254, 255, 256, 257, 618, 258, 259, 260, 261, 262, + 263, 264, 619, 265, 620, 621, 268, 269, 270, 271, + 272, 622, 623, 624, 625, 626, 276, 627, 628, 279, + 629, 281, 282, 283, 284, 285, 286, 630, 631, 287, + 632, 289, 633, 634, 291, 292, 293, 294, 295, 296, + 297, 298, 635, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 636, 637, 638, 323, 324, 325, 639, + 640, 327, 328, 641, 330, 642, 643, 332, 644, 334, + 335, 336, 645, 337, 338, 646, 647, 339, 340, 341, + 648, 649, 342, 343, 650, 651, 346, 652, 653, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 654, 655, 656, 657, 360, 361, 658, 659, 364, 365, + 660, 367, 368, 369, 661, 370, 371, 372, 373, 374, + 375, 662, 376, 377, 378, 663, 380, 381, 382, 383, + 664, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 665, 397, 398, 666, 400, 401, + 402, 667, 404, 405, 406, 407, 408, 409, 410, 411, + 412, 413, 414, 415, 416, 668, 669, 417, 418, 419, + 420, 421, 2235, 670, 424, 425, 671, 672, 427, 428, + 673, 430, 674, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 675, 444, 676, 677, 678, + 446, 447, 679, 448, 680, 450, 451, 452, 453, 454, + 681, 455, 682, 683, 684, 685, 458, 459, 686, 461, + 687, 688, 463, 464, 689, 466, 467, 468, 469, 470, + 690, 691, 471, 472, 473, 692, 474, 475, 476, 477, + 693, 478, 479, 480, 481, 482, 694, 695, 696, 485, + 697, 487, 488, 489, 490, 491, 492, 493, 698, 699, + 494, 700, 701, 495, 496, 497, 498, 499, 500, 702, + 703, 704, 705, 706, 707, 708, 709, 710, 711, 712, + 512, 513, 514, 515, 964, 0, 815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 0, 125, 126, 127, 0, 0, 0, 560, 0, + 0, 0, 0, 565, 129, 130, 0, 131, 132, 133, + 567, 135, 136, 137, 568, 569, 570, 571, 572, 0, + 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, + 151, 152, 576, 577, 155, 0, 156, 157, 158, 159, + 579, 0, 581, 0, 583, 163, 164, 165, 166, 167, + 584, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 587, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 589, 190, 191, 590, 193, 0, 194, + 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, + 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, + 217, 218, 219, 600, 221, 222, 223, 224, 225, 601, + 1342, 227, 0, 228, 229, 604, 231, 0, 232, 0, + 233, 607, 0, 609, 236, 237, 610, 611, 240, 0, + 241, 0, 614, 615, 244, 245, 0, 246, 247, 248, + 249, 250, 251, 252, 617, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 620, + 621, 268, 269, 270, 271, 272, 622, 623, 0, 625, + 0, 276, 627, 628, 279, 629, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 632, 289, 633, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 635, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 636, 637, + 638, 323, 324, 325, 639, 0, 327, 328, 641, 330, + 0, 643, 332, 644, 334, 335, 336, 0, 337, 338, + 1343, 0, 339, 340, 341, 0, 0, 342, 343, 650, + 651, 346, 652, 653, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 361, 658, 659, 364, 365, 660, 367, 368, 369, 0, + 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, + 663, 380, 381, 382, 383, 0, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 397, 398, 666, 400, 401, 402, 667, 404, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 0, 669, 417, 418, 419, 420, 421, 422, 670, 424, + 425, 0, 672, 427, 428, 673, 430, 0, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 675, 444, 676, 0, 0, 446, 447, 0, 448, 680, + 450, 451, 452, 453, 454, 0, 455, 682, 683, 0, + 0, 458, 459, 686, 461, 687, 1344, 463, 464, 689, + 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, + 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 694, 695, 0, 485, 697, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, + 497, 498, 499, 500, 702, 703, 704, 705, 706, 707, + 708, 709, 710, 711, 712, 512, 513, 514, 515, 964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, + 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, + 3, 4, 0, 560, 0, 0, 0, 0, 565, 129, + 130, 0, 131, 132, 133, 567, 135, 136, 137, 568, + 569, 570, 571, 572, 0, 143, 144, 145, 146, 147, + 148, 0, 0, 149, 150, 151, 152, 576, 577, 155, + 0, 156, 157, 158, 159, 579, 0, 581, 0, 583, + 163, 164, 165, 166, 167, 584, 169, 170, 171, 0, + 172, 173, 174, 175, 176, 177, 0, 587, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 589, 190, + 191, 590, 193, 0, 194, 0, 195, 196, 197, 198, + 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, + 213, 214, 215, 0, 216, 217, 218, 219, 600, 221, + 222, 223, 224, 225, 601, 0, 227, 0, 228, 229, + 604, 231, 0, 232, 0, 233, 607, 0, 609, 236, + 237, 610, 611, 240, 0, 241, 0, 614, 615, 244, + 245, 0, 246, 247, 248, 249, 250, 251, 252, 617, + 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 620, 621, 268, 269, 270, 271, + 272, 622, 623, 0, 625, 0, 276, 627, 628, 279, + 629, 281, 282, 283, 284, 285, 286, 0, 0, 287, + 632, 289, 633, 0, 291, 292, 293, 294, 295, 296, + 297, 298, 635, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 636, 637, 638, 323, 324, 325, 639, + 0, 327, 328, 641, 330, 0, 643, 332, 644, 334, + 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, + 0, 0, 342, 343, 650, 651, 346, 652, 653, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 0, 0, 0, 0, 360, 361, 658, 659, 364, 365, + 660, 367, 368, 369, 0, 370, 371, 372, 373, 374, + 375, 0, 376, 377, 378, 663, 380, 381, 382, 383, + 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 397, 398, 666, 400, 401, + 402, 667, 404, 405, 406, 407, 408, 409, 410, 411, + 412, 413, 414, 415, 416, 0, 669, 417, 418, 419, + 420, 421, 422, 670, 424, 425, 0, 672, 427, 428, + 673, 430, 0, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 675, 444, 676, 0, 0, + 446, 447, 0, 448, 680, 450, 451, 452, 453, 454, + 0, 455, 682, 683, 0, 0, 458, 459, 686, 461, + 687, 0, 463, 464, 689, 466, 467, 468, 469, 470, + 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 694, 695, 0, 485, + 697, 487, 488, 489, 490, 491, 492, 493, 0, 0, + 494, 0, 0, 495, 496, 497, 498, 499, 500, 702, + 703, 704, 705, 706, 707, 708, 709, 710, 711, 712, + 512, 513, 514, 515, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, + 0, 0, 0, 128, 129, 130, 0, 131, 132, 133, + 134, 135, 136, 137, 138, 139, 140, 141, 142, 0, + 143, 144, 145, 146, 147, 148, 0, 785, 149, 150, + 151, 152, 153, 154, 155, 0, 156, 157, 158, 159, + 786, 0, 787, 0, 162, 163, 164, 165, 166, 167, + 168, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 178, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, + 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, + 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, + 217, 218, 219, 220, 221, 222, 223, 224, 225, 788, + 0, 227, 0, 228, 229, 230, 231, 0, 232, 0, + 233, 234, 0, 235, 236, 237, 238, 239, 240, 0, + 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 266, + 267, 268, 269, 270, 271, 272, 273, 274, 0, 275, + 0, 276, 277, 278, 279, 280, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 288, 289, 290, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, + 322, 323, 324, 325, 326, 0, 327, 328, 329, 330, + 0, 790, 332, 333, 334, 335, 336, 0, 337, 338, + 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, + 345, 346, 347, 792, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 361, 793, 363, 364, 365, 366, 367, 368, 369, 0, + 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, + 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, + 425, 0, 426, 427, 428, 429, 430, 0, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 443, 444, 795, 0, 0, 446, 447, 0, 448, 449, + 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, + 0, 458, 459, 796, 461, 797, 0, 463, 464, 798, + 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, + 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 0, 485, 486, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, 512, 513, 514, 515, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, + 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, + 0, 0, 0, 0, 0, 0, 0, 0, 128, 129, + 130, 0, 131, 132, 133, 134, 135, 136, 137, 138, + 139, 140, 141, 142, 0, 143, 144, 145, 146, 147, + 148, 0, 0, 149, 150, 151, 152, 153, 154, 155, + 0, 156, 157, 158, 159, 160, 0, 161, 0, 162, + 163, 164, 165, 166, 167, 168, 169, 170, 171, 0, + 172, 173, 174, 175, 176, 177, 0, 178, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, + 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, + 213, 214, 215, 0, 216, 217, 218, 219, 220, 221, + 222, 223, 224, 225, 226, 0, 227, 0, 228, 229, + 230, 231, 0, 232, 0, 233, 234, 0, 235, 236, + 237, 238, 239, 240, 0, 241, 0, 242, 243, 244, + 245, 0, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 266, 267, 268, 269, 270, 271, + 272, 273, 274, 0, 275, 0, 276, 277, 278, 279, + 280, 281, 282, 283, 284, 285, 286, 0, 0, 287, + 288, 289, 290, 0, 291, 292, 293, 294, 295, 296, + 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, + 0, 327, 328, 329, 330, 0, 331, 332, 333, 334, + 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, + 0, 0, 342, 343, 344, 345, 346, 347, 348, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 0, 0, 0, 0, 360, 361, 362, 363, 364, 365, + 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, + 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, + 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 397, 398, 399, 400, 401, + 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, + 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 0, 426, 427, 428, + 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 443, 444, 445, 0, 0, + 446, 447, 0, 448, 449, 450, 451, 452, 453, 454, + 0, 455, 456, 457, 0, 0, 458, 459, 460, 461, + 462, 0, 463, 464, 465, 466, 467, 468, 469, 470, + 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 0, 485, + 486, 487, 488, 489, 490, 491, 492, 493, 0, 0, + 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, + 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, + 512, 513, 514, 515, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 27, 28, 29, 0, 0, 0, 0, - 0, 30, 0, 0, 31, 0, 0, 0, 0, 0, - 0, 27, 28, 29, 0, 0, 0, 0, 0, 30, - 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, - 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, - 0, 32, 0, 0, 0, 0, 0, 0, 34, 0, - 33, 0, 0, 0, 0, 35, 0, 0, 0, 36, - 0, 0, 0, 0, 0, 0, 34, 0, 0, 37, - 0, 0, 0, 35, 0, 0, 0, 36, 0, 0, - 0, 38, 0, 0, 0, 39, 0, 37, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, - 0, 0, 0, 39, 0, 40, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, - 0, 0, 0, 40, 43, 0, 0, 0, 0, 44, - 0, 0, 0, 748, 0, 0, 41, 0, 0, 0, - 0, 0, 43, 0, 0, 45, 0, 44, 0, 0, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, + 0, 135, 136, 137, 138, 139, 0, 141, 142, 0, + 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, + 151, 152, 153, 154, 155, 1739, 156, 157, 158, 159, + 160, 0, 0, 1740, 162, 163, 164, 165, 166, 167, + 0, 169, 170, 171, 1741, 172, 173, 174, 175, 176, + 177, 0, 0, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, + 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, + 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, + 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + 0, 227, 0, 228, 229, 230, 231, 0, 232, 1742, + 233, 0, 0, 0, 236, 237, 524, 0, 240, 0, + 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, + 249, 250, 1743, 252, 0, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 0, + 267, 268, 269, 270, 271, 272, 273, 274, 0, 275, + 0, 276, 0, 0, 279, 0, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 0, 289, 0, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 525, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 0, + 322, 323, 324, 325, 326, 0, 327, 328, 0, 330, + 0, 331, 332, 333, 334, 335, 336, 0, 337, 338, + 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, + 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 361, 362, 0, 364, 365, 366, 367, 368, 369, 1744, + 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, + 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, + 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 526, 444, 445, 0, 0, 446, 447, 0, 448, 0, + 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, + 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, + 466, 467, 468, 469, 470, 0, 1745, 471, 472, 473, + 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 0, 485, 0, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, 512, 513, 514, 515, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 45, 0, 0, 0, 0, 0, 46, + 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, + 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, + 130, 0, 131, 132, 133, 0, 135, 136, 137, 138, + 139, 0, 141, 142, 0, 143, 144, 145, 146, 147, + 148, 0, 0, 149, 150, 151, 152, 153, 154, 155, + 1739, 156, 157, 158, 159, 160, 0, 0, 0, 162, + 163, 164, 165, 166, 167, 0, 169, 170, 171, 1741, + 172, 173, 174, 175, 176, 177, 0, 0, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, + 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, + 213, 214, 215, 0, 216, 217, 218, 219, 220, 221, + 222, 223, 224, 225, 226, 0, 227, 0, 228, 229, + 230, 231, 0, 232, 1742, 233, 0, 0, 0, 236, + 237, 524, 0, 240, 0, 241, 0, 242, 243, 244, + 245, 0, 246, 247, 248, 249, 250, 251, 252, 0, + 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 0, 267, 268, 269, 270, 271, + 272, 273, 274, 0, 275, 0, 276, 0, 0, 279, + 0, 281, 282, 283, 284, 285, 286, 0, 0, 287, + 0, 289, 2318, 0, 291, 292, 293, 294, 295, 296, + 297, 298, 525, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 0, 322, 323, 324, 325, 326, + 0, 327, 328, 0, 330, 0, 331, 332, 333, 334, + 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, + 0, 0, 342, 343, 344, 0, 346, 0, 348, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 0, 0, 0, 0, 360, 361, 362, 0, 364, 365, + 366, 367, 368, 369, 1744, 370, 371, 372, 373, 374, + 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, + 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 397, 398, 0, 400, 401, + 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, + 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 0, 0, 427, 428, + 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 526, 444, 445, 0, 0, + 446, 447, 0, 448, 0, 450, 451, 452, 453, 454, + 0, 455, 456, 457, 0, 0, 458, 459, 460, 461, + 462, 0, 463, 464, 465, 466, 467, 468, 469, 470, + 0, 1745, 471, 472, 473, 0, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 0, 485, + 0, 487, 488, 489, 490, 491, 492, 493, 0, 0, + 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, + 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, + 512, 513, 514, 515, 523, 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 749, 0, 0, 0, 46, 0, 0, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 0, 125, 126, 127, 3, 4, 0, 0, 0, + 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, + 0, 135, 136, 137, 138, 139, 0, 141, 142, 0, + 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, + 151, 152, 153, 154, 155, 0, 156, 157, 158, 159, + 160, 0, 0, 0, 162, 163, 164, 165, 166, 167, + 0, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 0, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, + 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, + 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, + 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + 0, 227, 0, 228, 229, 230, 231, 0, 232, 0, + 233, 0, 0, 0, 236, 237, 524, 0, 240, 0, + 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, + 249, 250, 251, 252, 0, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 0, + 267, 268, 269, 270, 271, 272, 273, 274, 0, 275, + 0, 276, 0, 0, 279, 0, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 0, 289, 0, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 525, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 0, + 322, 323, 324, 325, 326, 0, 327, 328, 0, 330, + 0, 331, 332, 333, 334, 335, 336, 0, 337, 338, + 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, + 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, + 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, + 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, + 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 526, 444, 445, 0, 0, 446, 447, 0, 448, 0, + 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, + 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, + 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, + 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 0, 485, 0, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, 512, 513, 514, 515, 523, + 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, + 119, 120, 121, 122, 123, 124, 550, 125, 126, 127, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, + 130, 0, 131, 132, 133, 0, 135, 136, 137, 138, + 139, 0, 141, 142, 0, 143, 144, 145, 146, 147, + 148, 0, 0, 149, 150, 151, 152, 153, 154, 155, + 0, 156, 157, 158, 159, 160, 0, 0, 0, 162, + 163, 164, 165, 166, 167, 0, 169, 170, 171, 0, + 172, 173, 174, 175, 176, 177, 0, 0, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, + 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, + 213, 214, 215, 0, 216, 217, 218, 219, 220, 221, + 222, 223, 224, 225, 226, 0, 227, 0, 228, 229, + 230, 231, 0, 232, 0, 233, 0, 0, 0, 236, + 237, 524, 0, 240, 0, 241, 0, 242, 243, 244, + 245, 0, 246, 247, 248, 249, 250, 251, 252, 0, + 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 0, 267, 268, 269, 270, 271, + 272, 273, 274, 0, 275, 0, 276, 0, 0, 279, + 0, 281, 282, 283, 284, 285, 286, 0, 0, 287, + 0, 289, 0, 0, 291, 292, 293, 294, 295, 296, + 297, 298, 525, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 0, 322, 323, 324, 325, 326, + 0, 327, 328, 0, 330, 0, 331, 332, 333, 334, + 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, + 0, 0, 342, 343, 344, 0, 346, 0, 348, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 0, 0, 0, 0, 360, 361, 362, 0, 364, 365, + 366, 551, 368, 369, 0, 370, 371, 372, 373, 374, + 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, + 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 397, 398, 0, 400, 401, + 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, + 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 0, 0, 427, 428, + 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 526, 444, 445, 0, 0, + 446, 447, 0, 448, 0, 450, 451, 452, 453, 454, + 0, 455, 456, 457, 0, 0, 458, 459, 460, 461, + 462, 0, 463, 464, 465, 466, 467, 468, 469, 470, + 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 0, 485, + 0, 487, 488, 489, 490, 491, 492, 493, 0, 0, + 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, + 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, + 512, 513, 514, 515, 523, 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 47 -}; - -static const yytype_int16 yycheck[] = -{ - 7, 0, 0, 47, 0, 73, 516, 0, 955, 16, - 0, 0, 895, 835, 0, 0, 23, 0, 904, 749, - 990, 0, 0, 881, 1235, 809, 970, 75, 76, 858, - 1691, 1196, 1223, 1032, 925, 920, 1221, 1156, 20, 741, - 38, 20, 20, 1417, 16, 77, 1573, 981, 981, 1209, - 1079, 981, 77, 981, 1654, 2140, 1190, 2142, 1465, 17, - 1198, 2313, 7, 2226, 2161, 959, 23, 1275, 75, 76, - 1986, 1201, 970, 2648, 1796, 46, 1297, 2171, 23, 0, - 39, 895, 2106, 897, 7, 899, 0, 0, 1226, 0, - 1123, 25, 26, 0, 0, 1128, 1607, 1608, 0, 2609, - 23, 2160, 100, 2663, 2609, 0, 26, 1987, 0, 805, - 0, 0, 1322, 999, 810, 2670, 748, 1087, 75, 76, - 0, 749, 742, 0, 0, 0, 2483, 2293, 0, 0, - 75, 76, 35, 0, 0, 0, 0, 2646, 2274, 2275, - 2276, 2565, 5, 2222, 0, 2569, 5, 0, 1658, 1709, - 979, 110, 75, 76, 5, 1711, 1814, 1053, 1054, 11, - 1927, 5, 13, 14, 16, 5, 13, 14, 5, 5, - 1815, 80, 5, 5, 1070, 5, 9, 13, 14, 5, - 114, 13, 14, 13, 14, 5, 5, 13, 14, 9, - 1134, 970, 5, 5, 5, 5, 4, 772, 982, 2982, - 5, 9, 2305, 9, 5, 2301, 172, 2299, 171, 5, - 5, 26, 2967, 3, 119, 5, 45, 32, 45, 13, - 14, 1960, 1961, 3, 4, 5, 1818, 55, 3, 9, - 2985, 9, 1971, 876, 876, 2314, 1975, 124, 63, 63, - 970, 11, 40, 1242, 100, 15, 16, 100, 122, 122, - 11, 2434, 1251, 1139, 2434, 16, 37, 1086, 74, 34, - 35, 107, 60, 147, 8, 23, 1210, 11, 871, 1213, - 1214, 15, 16, 190, 2821, 63, 4, 139, 5, 82, - 82, 9, 180, 2362, 2363, 46, 2365, 983, 3146, 289, - 93, 93, 802, 11, 295, 290, 287, 15, 16, 995, - 104, 117, 46, 295, 63, 2867, 0, 11, 106, 53, - 312, 15, 16, 1432, 74, 275, 107, 75, 76, 80, - 2937, 40, 137, 192, 30, 244, 20, 1106, 46, 23, - 107, 2416, 38, 217, 192, 183, 80, 117, 212, 365, - 850, 973, 165, 171, 38, 132, 1465, 132, 117, 2802, - 3277, 2804, 2527, 47, 2529, 1134, 278, 106, 1518, 388, - 1189, 289, 80, 1185, 1186, 119, 1953, 1954, 1955, 379, - 30, 171, 2579, 30, 40, 75, 180, 30, 30, 161, - 172, 75, 76, 77, 1057, 38, 38, 3341, 161, 416, - 1611, 172, 2528, 108, 149, 170, 3309, 172, 41, 3385, - 1073, 145, 2977, 201, 13, 14, 100, 2129, 168, 454, - 163, 295, 2922, 2988, 3076, 2578, 3078, 430, 406, 3174, - 64, 1279, 90, 1308, 71, 72, 122, 208, 478, 120, - 74, 1210, 176, 289, 1213, 1214, 289, 3235, 218, 3237, - 420, 137, 422, 224, 3371, 200, 1337, 3374, 272, 193, - 500, 2475, 250, 234, 198, 237, 175, 108, 274, 2553, - 240, 129, 260, 279, 237, 428, 479, 30, 272, 514, - 2979, 393, 132, 2680, 272, 132, 191, 230, 293, 3433, - 468, 272, 415, 126, 3470, 514, 355, 274, 514, 337, - 108, 451, 236, 3276, 352, 1524, 2575, 284, 2577, 284, - 323, 250, 358, 194, 286, 358, 304, 509, 518, 175, - 279, 260, 328, 213, 514, 3313, 308, 518, 366, 514, - 453, 3434, 85, 514, 1745, 279, 518, 3144, 344, 448, - 514, 94, 357, 357, 120, 350, 534, 3199, 2713, 354, - 503, 3468, 3104, 3090, 1187, 1187, 2729, 291, 514, 2729, - 3408, 456, 279, 516, 3007, 118, 454, 166, 356, 1162, - 444, 3123, 386, 2659, 2656, 1344, 1345, 454, 328, 357, - 416, 386, 1979, 2676, 1739, 1150, 0, 451, 451, 2106, - 361, 588, 1092, 381, 413, 26, 413, 1483, 1484, 3372, - 2506, 32, 1800, 370, 1423, 514, 240, 514, 357, 380, - 392, 3161, 400, 512, 463, 2372, 244, 516, 194, 1339, - 472, 2690, 1508, 403, 404, 1437, 588, 2177, 3373, 1321, - 476, 425, 2178, 476, 1326, 428, 428, 190, 428, 445, - 1332, 327, 1454, 2144, 478, 476, 3146, 478, 454, 307, - 203, 3146, 457, 2170, 265, 1836, 1856, 391, 2306, 512, - 163, 514, 517, 516, 2164, 514, 500, 1656, 510, 515, - 1871, 517, 515, 514, 517, 478, 2311, 514, 1889, 520, - 514, 3095, 1568, 1569, 514, 2838, 3100, 514, 514, 2845, - 1901, 514, 514, 520, 514, 445, 514, 500, 514, 464, - 396, 2430, 3049, 518, 514, 514, 137, 505, 506, 320, - 3275, 514, 514, 514, 514, 749, 1338, 513, 1328, 514, - 1931, 1339, 2964, 514, 1600, 1601, 1602, 1938, 514, 514, - 514, 1550, 1551, 3283, 2316, 505, 506, 505, 506, 3284, - 518, 1560, 370, 476, 1430, 505, 506, 507, 508, 509, - 510, 517, 749, 396, 396, 1574, 148, 454, 476, 510, - 258, 259, 800, 421, 498, 1976, 800, 500, 271, 1980, - 2819, 505, 506, 507, 508, 509, 510, 749, 117, 748, - 748, 3474, 500, 226, 1603, 464, 1984, 505, 506, 356, - 11, 26, 789, 1897, 11, 33, 1565, 2008, 2704, 507, - 508, 509, 510, 800, 177, 249, 59, 199, 1577, 515, - 1579, 2825, 518, 507, 508, 509, 510, 514, 3383, 386, - 448, 59, 3515, 881, 1928, 46, 848, 1490, 1551, 272, - 274, 1940, 2958, 848, 5, 1769, 1605, 1560, 835, 836, - 2417, 2418, 2419, 2420, 503, 1779, 518, 1510, 1782, 402, - 534, 190, 405, 800, 789, 1844, 2872, 516, 782, 80, - 784, 858, 293, 80, 2880, 800, 1009, 240, 2738, 3368, - 1979, 2741, 1015, 2743, 784, 2781, 789, 375, 376, 868, - 868, 1544, 868, 312, 856, 868, 514, 800, 868, 868, - 814, 1764, 868, 868, 509, 868, 2965, 274, 859, 868, - 868, 516, 137, 512, 853, 902, 903, 516, 3408, 906, - 907, 1411, 2624, 3408, 2626, 839, 5, 1854, 171, 350, - 514, 1858, 871, 354, 1861, 1948, 514, 3426, 3170, 3494, - 2130, 1623, 503, 1737, 202, 274, 108, 366, 1872, 1708, - 1709, 1822, 352, 514, 837, 838, 389, 840, 2057, 1797, - 1798, 1799, 1004, 3106, 177, 386, 274, 868, 2475, 53, - 1764, 279, 959, 37, 868, 868, 515, 868, 1006, 518, - 1022, 868, 868, 970, 1026, 1013, 868, 925, 377, 1863, - 1784, 978, 979, 868, 1872, 1789, 868, 984, 868, 868, - 987, 988, 2493, 990, 991, 992, 993, 370, 868, 3086, - 1769, 868, 868, 868, 973, 973, 868, 868, 451, 1006, - 1779, 868, 868, 1782, 868, 244, 1013, 240, 336, 3334, - 3335, 26, 2146, 132, 2148, 377, 457, 32, 119, 82, - 2394, 416, 26, 82, 1031, 1032, 1033, 433, 32, 173, - 93, 789, 244, 132, 93, 513, 970, 420, 173, 422, - 159, 515, 800, 1087, 518, 1052, 524, 981, 293, 1006, - 244, 314, 315, 316, 748, 749, 1013, 202, 244, 2250, - 159, 1006, 3387, 513, 1071, 448, 251, 244, 1013, 2277, - 509, 2793, 176, 1775, 524, 1082, 1083, 1084, 1780, 1086, - 1087, 2200, 2609, 1006, 1091, 3001, 132, 3250, 172, 193, - 1013, 466, 370, 1872, 198, 789, 1030, 2318, 40, 478, - 512, 245, 514, 381, 516, 206, 800, 1936, 26, 354, - 245, 356, 1119, 159, 32, 420, 379, 422, 60, 1091, - 2258, 500, 137, 478, 208, 823, 824, 825, 26, 1136, - 1137, 370, 236, 137, 32, 514, 1168, 1169, 416, 1171, - 224, 386, 1872, 1168, 1169, 500, 1171, 4, 515, 509, - 234, 518, 9, 416, 848, 274, 516, 1014, 370, 514, - 279, 1018, 1096, 1170, 106, 284, 2136, 1174, 1175, 13, - 14, 272, 1106, 274, 868, 274, 370, 1184, 1185, 1186, - 279, 512, 1189, 514, 370, 284, 449, 420, 466, 422, - 2019, 1125, 1879, 370, 374, 2325, 1883, 460, 342, 1886, - 1134, 2331, 1209, 1162, 512, 4, 4, 342, 516, 448, - 9, 9, 377, 8, 1993, 448, 11, 161, 514, 137, - 15, 16, 166, 11, 19, 20, 21, 15, 16, 2450, - 420, 514, 422, 279, 26, 1242, 448, 381, 284, 137, - 32, 36, 4, 514, 1251, 161, 381, 9, 1006, 166, - 166, 514, 512, 514, 448, 1013, 516, 515, 46, 201, - 518, 1422, 448, 1424, 1425, 2094, 1225, 420, 1275, 422, - 514, 448, 416, 837, 838, 514, 840, 361, 293, 973, - 226, 416, 2229, 2141, 2231, 515, 514, 391, 518, 293, - 1297, 515, 80, 237, 518, 514, 380, 1296, 2825, 171, - 2707, 515, 514, 2232, 518, 2234, 1313, 1296, 250, 514, - 1296, 1296, 1006, 2432, 514, 1322, 435, 2436, 260, 1013, - 514, 237, 466, 3486, 13, 14, 272, 446, 514, 507, - 272, 466, 1339, 13, 14, 350, 435, 514, 514, 354, - 515, 1313, 286, 518, 514, 137, 350, 446, 515, 1356, - 354, 518, 500, 3438, 1048, 1362, 171, 1339, 515, 1338, - 1338, 518, 304, 2575, 2143, 2577, 1060, 514, 3453, 515, - 286, 386, 518, 515, 497, 293, 518, 3471, 515, 3473, - 223, 518, 386, 2205, 1356, 342, 13, 14, 2609, 435, - 342, 516, 338, 1087, 1442, 293, 1444, 2176, 2177, 1447, - 446, 2592, 1409, 1410, 1452, 289, 2591, 1455, 1415, 1457, - 1417, 2576, 2623, 1461, 356, 1422, 1423, 1424, 1425, 2304, - 3514, 1355, 497, 3508, 381, 1359, 221, 516, 3513, 381, - 1437, 1438, 350, 13, 14, 1442, 354, 1444, 171, 381, - 1447, 515, 457, 389, 518, 1452, 515, 1454, 1455, 518, - 1457, 2480, 2481, 457, 1461, 2640, 354, 170, 400, 416, - 2590, 518, 2592, 873, 416, 875, 188, 189, 386, 171, - 515, 514, 1417, 518, 1168, 1169, 515, 1171, 515, 518, - 515, 518, 515, 518, 515, 1442, 515, 1444, 386, 518, - 1447, 13, 14, 515, 1417, 1452, 291, 1442, 1455, 1444, - 1457, 293, 1447, 294, 1461, 451, 3228, 1452, 3230, 466, - 1455, 1518, 1457, 2483, 466, 515, 1461, 515, 518, 1442, - 518, 1444, 59, 2457, 1447, 2458, 2459, 2460, 2458, 1452, - 2458, 2459, 1455, 514, 1457, 177, 258, 259, 1461, 457, - 177, 417, 515, 1550, 1551, 518, 497, 515, 223, 1547, - 518, 1547, 1559, 1560, 1547, 13, 14, 1547, 350, 457, - 1567, 152, 354, 350, 1547, 202, 6, 1574, 1547, 1547, - 10, 152, 515, 515, 515, 518, 515, 518, 18, 518, - 152, 314, 315, 316, 515, 515, 515, 518, 518, 518, - 13, 14, 1599, 33, 386, 152, 1603, 37, 240, 1606, - 13, 14, 152, 240, 1611, 1612, 1613, 1614, 1615, 1616, - 1617, 1618, 1619, 1620, 13, 14, 40, 1624, 1625, 3146, - 2350, 515, 1629, 416, 518, 515, 1633, 515, 518, 1636, - 1637, 1638, 1639, 1640, 1641, 1642, 1643, 1644, 3238, 514, - 1647, 274, 2344, 40, 1338, 466, 379, 1654, 1620, 1656, - 89, 82, 515, 375, 376, 518, 152, 294, 2544, 1417, - 515, 515, 1694, 518, 518, 457, 480, 2888, 1675, 1694, - 152, 313, 13, 14, 2839, 2454, 2455, 108, 13, 14, - 516, 5, 152, 416, 1442, 152, 1444, 13, 14, 1447, - 1697, 1673, 13, 14, 1452, 2886, 289, 1455, 352, 1457, - 428, 1708, 1709, 1461, 13, 14, 501, 502, 503, 514, - 505, 506, 507, 508, 509, 510, 449, 505, 506, 507, - 508, 509, 510, 13, 14, 13, 14, 460, 370, 171, - 1664, 13, 14, 13, 14, 377, 2683, 514, 1745, 13, - 14, 514, 173, 13, 14, 2567, 515, 1754, 1442, 1756, - 1444, 2962, 418, 1447, 13, 14, 514, 8, 1452, 219, - 2651, 1455, 2922, 1457, 15, 16, 299, 1461, 19, 20, - 21, 202, 2630, 13, 14, 3436, 365, 366, 420, 3440, - 422, 514, 1754, 420, 1756, 422, 13, 14, 2673, 1796, - 13, 14, 224, 1800, 514, 2794, 1803, 1804, 365, 366, - 37, 296, 126, 127, 224, 42, 448, 224, 445, 365, - 366, 448, 365, 366, 245, 514, 3190, 262, 263, 40, - 375, 376, 514, 3044, 459, 460, 3448, 3449, 3480, 3481, - 2609, 235, 5, 8, 3495, 5, 11, 1844, 1136, 1137, - 15, 16, 1840, 514, 19, 20, 21, 171, 324, 1856, - 514, 514, 514, 1547, 5, 5, 1863, 1864, 1792, 514, - 5, 514, 5, 148, 101, 1872, 514, 9, 514, 477, - 104, 1805, 1806, 301, 518, 518, 515, 40, 386, 219, - 166, 3408, 1889, 286, 284, 166, 1893, 1894, 59, 1896, - 235, 514, 428, 514, 1901, 1902, 1903, 1904, 1905, 1906, - 1907, 93, 518, 1910, 1911, 1912, 1913, 1914, 1915, 1916, - 1917, 1918, 1919, 428, 1962, 59, 428, 265, 1925, 1926, - 1968, 74, 1929, 59, 1931, 3146, 428, 80, 108, 1936, - 524, 1938, 1626, 2877, 221, 172, 428, 428, 1872, 370, - 93, 476, 2889, 2890, 1878, 152, 100, 377, 198, 274, - 381, 1958, 274, 514, 274, 1962, 40, 1964, 274, 274, - 2739, 1968, 514, 152, 117, 171, 119, 3341, 516, 1976, - 13, 208, 3157, 1980, 515, 1982, 515, 1984, 171, 1986, - 518, 515, 224, 515, 515, 416, 515, 224, 515, 514, - 314, 315, 316, 473, 224, 514, 8, 234, 281, 11, - 1694, 2008, 281, 15, 16, 1962, 147, 514, 439, 518, - 3384, 1968, 2019, 2020, 463, 516, 514, 1962, 39, 516, - 161, 514, 472, 1968, 147, 166, 1033, 514, 514, 514, - 514, 268, 2734, 9, 46, 466, 426, 426, 161, 1962, - 291, 53, 13, 166, 11, 1968, 3016, 352, 19, 513, - 2094, 524, 2059, 206, 426, 379, 518, 2064, 2065, 3433, - 31, 3272, 518, 514, 521, 279, 390, 514, 80, 428, - 180, 162, 171, 454, 45, 46, 217, 314, 515, 217, - 518, 265, 2089, 2090, 321, 518, 389, 2094, 412, 226, - 290, 518, 416, 312, 217, 3214, 237, 312, 2877, 518, - 180, 2108, 219, 2882, 2111, 515, 2113, 514, 226, 274, - 226, 333, 295, 287, 237, 466, 291, 152, 514, 3, - 514, 274, 2129, 2130, 361, 449, 279, 1134, 514, 2136, - 152, 40, 2139, 145, 1828, 152, 460, 108, 152, 476, - 1033, 152, 274, 380, 3, 286, 1840, 147, 40, 2156, - 2929, 2930, 476, 59, 295, 289, 2858, 11, 289, 40, - 515, 161, 2169, 286, 176, 171, 166, 166, 515, 515, - 180, 515, 295, 166, 3, 328, 500, 39, 514, 2186, - 2187, 193, 2139, 514, 514, 3, 198, 3408, 515, 512, - 514, 344, 428, 512, 2139, 428, 2203, 428, 2205, 2156, - 428, 513, 1209, 521, 1962, 515, 497, 2214, 515, 515, - 1968, 2156, 449, 515, 518, 117, 2139, 217, 516, 497, - 3411, 515, 515, 515, 236, 2232, 2233, 2234, 497, 171, - 428, 515, 155, 2156, 249, 8, 40, 237, 11, 3124, - 59, 514, 15, 16, 473, 503, 19, 20, 21, 514, - 501, 502, 503, 514, 505, 506, 507, 508, 509, 510, - 2232, 2233, 2234, 518, 2271, 2309, 514, 2997, 1962, 171, - 2277, 514, 290, 46, 1968, 499, 59, 290, 243, 291, - 53, 59, 451, 518, 265, 274, 286, 152, 428, 514, - 1297, 202, 445, 428, 152, 295, 152, 281, 428, 2298, - 428, 454, 428, 444, 281, 2239, 2313, 80, 40, 2298, - 515, 2318, 2298, 2298, 352, 514, 1209, 514, 287, 515, - 476, 444, 40, 518, 152, 289, 501, 502, 503, 2336, - 505, 506, 507, 508, 509, 510, 279, 515, 171, 515, - 59, 2313, 514, 2350, 185, 166, 514, 80, 515, 2356, - 512, 515, 143, 198, 3240, 3134, 3135, 2291, 515, 171, - 515, 300, 358, 515, 524, 290, 515, 3146, 514, 180, - 518, 3200, 152, 3202, 175, 2309, 515, 439, 40, 391, - 40, 2139, 515, 86, 2356, 514, 454, 2394, 515, 512, - 3212, 515, 515, 516, 514, 514, 518, 518, 2156, 514, - 2094, 303, 515, 176, 1297, 516, 515, 515, 514, 171, - 515, 514, 314, 315, 316, 515, 2350, 518, 2112, 514, - 193, 475, 515, 198, 2358, 198, 2360, 2434, 515, 3399, - 2364, 3215, 2366, 3217, 2128, 513, 513, 518, 289, 2483, - 515, 515, 515, 2450, 444, 2139, 515, 503, 59, 2394, - 515, 460, 515, 2521, 515, 515, 476, 204, 3343, 2466, - 117, 40, 2156, 236, 514, 2472, 2473, 226, 88, 3352, - 191, 2394, 2516, 279, 279, 3333, 2483, 379, 40, 516, - 516, 428, 516, 516, 516, 516, 498, 2494, 516, 516, - 2497, 516, 2499, 505, 506, 507, 508, 509, 510, 2506, - 2507, 503, 428, 2510, 2511, 3327, 513, 515, 2515, 2516, - 515, 1518, 512, 516, 416, 2522, 516, 8, 291, 516, - 11, 516, 516, 2567, 15, 16, 516, 513, 516, 516, - 2537, 516, 274, 516, 516, 514, 516, 2535, 516, 2535, - 2547, 516, 2535, 2541, 516, 2535, 516, 449, 516, 107, - 516, 514, 2535, 7, 8, 46, 2535, 2535, 460, 13, - 2567, 516, 53, 516, 516, 19, 516, 516, 476, 23, - 515, 25, 26, 40, 476, 289, 30, 31, 32, 514, - 551, 35, 9, 351, 38, 39, 514, 514, 42, 80, - 335, 45, 46, 515, 518, 59, 40, 171, 500, 2606, - 518, 515, 2609, 2610, 1611, 2612, 198, 191, 513, 459, - 91, 518, 514, 344, 515, 2309, 60, 2624, 391, 2626, - 514, 75, 76, 40, 152, 1518, 516, 515, 124, 3408, - 152, 40, 515, 40, 2606, 515, 2394, 366, 2610, 366, - 2612, 514, 514, 40, 518, 454, 100, 514, 309, 2647, - 514, 2650, 0, 107, 108, 109, 110, 111, 279, 248, - 190, 2650, 106, 107, 2650, 2650, 454, 439, 514, 292, - 74, 74, 80, 117, 9, 2609, 515, 2675, 515, 3381, - 2678, 368, 2726, 514, 59, 176, 133, 513, 515, 513, - 503, 93, 272, 289, 2701, 439, 40, 2704, 514, 292, - 514, 292, 193, 459, 204, 515, 289, 198, 2642, 386, - 122, 289, 515, 515, 2721, 2722, 515, 451, 1611, 2726, - 365, 25, 2729, 148, 36, 498, 365, 297, 501, 502, - 503, 175, 505, 506, 507, 508, 509, 510, 1745, 2593, - 314, 315, 316, 868, 1762, 236, 151, 2227, 3242, 2756, - 3384, 2606, 100, 2801, 3337, 1863, 2545, 201, 3485, 2882, - 2339, 3358, 3463, 2770, 3117, 3412, 171, 3421, 2775, 2776, - 3456, 2815, 743, 2780, 2781, 1196, 3176, 2220, 2785, 2664, - 3410, 2788, 2789, 3419, 2233, 2217, 2793, 2794, 2585, 2483, - 2797, 2612, 3407, 1321, 2801, 8, 3306, 2297, 11, 147, - 291, 2808, 15, 16, 1294, 379, 250, 2356, 2997, 2203, - 2169, 2642, 1155, 161, 1005, 1005, 260, 1178, 166, 1733, - 1177, 2415, 2516, 171, 3393, 2186, 1697, 3315, 272, 3204, - 274, 1732, 180, 46, 23, 2156, 2729, 185, 1180, 3039, - 53, 2535, 416, 1974, 2801, 2852, 800, 2541, 789, 2394, - 2393, 2442, 1745, 2860, 3138, 1989, 2801, 3319, 126, 127, - 304, 1872, 3318, 2066, 2170, 2473, 2112, 80, 2109, 217, - 2021, 2805, 2879, 2567, 1438, 449, 983, 981, 2801, 981, - 2068, 2888, 1889, 981, 981, 981, 460, 981, 859, 237, - 2808, 981, 981, 2490, 1901, 2535, 3304, 783, 1806, 1982, - 391, 1673, 476, 171, 1339, 1359, 1674, 2546, 100, 314, - 315, 316, 356, 2270, -1, 2922, 1233, -1, -1, -1, - -1, -1, -1, -1, 1931, -1, 500, -1, -1, 171, - -1, 1938, 145, -1, -1, -1, -1, 381, 286, -1, - 514, 289, -1, -1, -1, -1, -1, 295, -1, -1, - -1, -1, -1, 2647, -1, -1, 400, 2964, 402, -1, - -1, 405, 2969, 176, -1, -1, 1033, -1, -1, 1976, - -1, -1, 3016, 1980, 379, -1, -1, -1, -1, 1986, - 193, 2675, 2980, -1, 2678, 198, -1, 335, -1, -1, - 2997, -1, 2964, 3000, 3001, -1, 1889, -1, -1, -1, - -1, 2008, -1, -1, -1, -1, -1, 498, 1901, 3016, - 358, 416, -1, -1, 505, 506, 507, 508, 509, 510, - -1, -1, 8, 236, -1, 11, -1, -1, -1, 15, - 16, -1, 2726, 19, 20, 21, -1, 3044, 1931, -1, - 2974, -1, 3049, 2801, 449, 1938, 314, 315, 316, -1, - 36, -1, -1, 3060, 3061, 460, -1, 3064, -1, 3066, - -1, 303, -1, 2997, -1, -1, -1, -1, 416, -1, - 514, 476, 314, 315, 316, -1, -1, -1, 291, -1, - -1, 535, -1, 1976, 3091, -1, 540, 1980, -1, 543, - -1, -1, -1, 1986, -1, 500, 444, 551, -1, 151, - -1, -1, -1, -1, -1, -1, 454, 2801, 3115, 514, - -1, 379, -1, -1, -1, 2008, -1, -1, -1, 171, - -1, 2815, 390, -1, -1, 473, -1, 475, 476, -1, - -1, -1, -1, -1, -1, -1, -1, 379, -1, 3146, - -1, -1, 1209, 3077, 412, 988, -1, -1, 416, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 3170, 512, 3163, 3164, 515, 516, 517, - -1, -1, -1, -1, 416, -1, -1, -1, 391, 3186, - -1, 449, -1, 3190, 1155, -1, -1, -1, -1, -1, - -1, -1, 460, 3200, -1, 3202, -1, 3204, 3170, -1, - -1, 3208, -1, 3210, -1, 3212, -1, 449, 476, -1, - -1, -1, 3146, -1, -1, -1, 3223, -1, 460, -1, - -1, 3228, -1, 3230, -1, -1, -1, -1, -1, -1, - 1297, 3238, 500, -1, 476, 221, -1, -1, -1, -1, - -1, 3239, -1, 3241, 3251, 3190, 514, 1033, -1, 3256, - -1, -1, -1, -1, -1, -1, -1, -1, 500, -1, - -1, -1, 314, 315, 316, -1, -1, 3190, -1, -1, - -1, -1, 514, -1, -1, -1, -1, -1, -1, 3251, - -1, -1, -1, -1, 3282, 498, 2980, 741, 742, 743, - -1, -1, 505, 506, 507, 508, 509, 510, 3305, -1, - 3298, -1, -1, -1, 3302, 291, -1, -1, 3315, -1, - -1, 2318, -1, 177, -1, -1, -1, -1, -1, -1, - 3327, -1, 3016, -1, -1, -1, -1, 379, 782, 783, - 784, -1, -1, -1, 3341, 789, -1, 791, 202, 3273, - -1, 1184, -1, -1, -1, -1, 800, -1, -1, -1, - 804, 805, 3359, -1, -1, 809, 810, -1, 1329, -1, - 1331, -1, -1, -1, 416, -1, -1, -1, -1, 823, - 824, 825, -1, -1, -1, -1, 240, 3384, -1, -1, - -1, -1, 55, 837, 838, -1, 840, -1, -1, -1, - -1, -1, 3399, -1, -1, -1, 3341, 449, -1, 853, - -1, 3408, -1, 3410, -1, 859, -1, -1, 460, -1, - -1, -1, -1, -1, -1, -1, -1, 871, 3341, -1, - -1, -1, -1, 1209, 476, 2318, 3433, 3425, -1, 102, - 294, 885, 3190, -1, -1, 40, -1, -1, 3410, 3384, - 3447, 3448, 3449, 2450, -1, -1, -1, -1, 500, -1, - 123, 1518, -1, 3451, 0, 60, -1, -1, -1, -1, - -1, 3384, 514, 917, 918, -1, -1, 3474, 141, 3163, - 3164, -1, 145, -1, 3408, 929, -1, -1, -1, 933, - 934, 935, 936, -1, -1, -1, -1, -1, 3433, -1, - -1, -1, -1, -1, 167, 949, 988, 170, -1, 2506, - -1, 106, -1, -1, -1, -1, 370, -1, 3515, -1, - 3433, 1297, -1, 186, -1, 501, 502, 503, -1, 505, - 506, 507, 508, 509, 510, -1, -1, -1, 982, 983, - -1, 985, -1, -1, 988, -1, -1, -1, -1, -1, - 994, 995, -1, -1, 1611, 3239, 1000, 3241, -1, -1, - -1, -1, 1006, -1, 100, -1, 420, 2450, 422, 1013, - -1, -1, 13, -1, -1, -1, 1409, 1410, 19, 1023, - 175, -1, 1415, -1, -1, 439, 1030, -1, -1, -1, - 31, 445, -1, 3341, 448, -1, 1040, -1, 3282, -1, - -1, -1, -1, -1, 45, 46, 201, -1, -1, -1, - -1, 147, 2609, -1, 3298, -1, 279, -1, 3302, -1, - -1, -1, -1, 2506, 287, 161, -1, -1, -1, -1, - 166, -1, -1, -1, -1, 171, 3384, -1, -1, -1, - -1, -1, -1, -1, 180, 1089, 309, -1, -1, 185, - -1, -1, -1, -1, -1, 250, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 260, -1, 108, -1, -1, - -1, -1, 988, 336, -1, 8, -1, 272, 11, -1, - -1, 217, 15, 16, -1, 3433, -1, -1, 1745, -1, - -1, -1, -1, 1175, 1138, -1, 1140, -1, -1, -1, - -1, 237, 1184, -1, -1, -1, -1, 2704, -1, 304, - -1, 1155, 1156, 46, -1, -1, -1, -1, 1162, -1, - 53, -1, -1, -1, -1, -1, 2609, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 1184, 3425, 1518, -1, -1, -1, -1, 80, -1, -1, - 286, 1195, 1196, 289, -1, -1, -1, -1, -1, 295, - -1, 356, -1, -1, 1208, -1, -1, 3451, -1, -1, - 0, -1, -1, -1, -1, -1, -1, -1, -1, 1740, - -1, 1225, -1, -1, 2781, 1229, 381, -1, -1, 1750, - 1234, 1752, -1, -1, 1755, -1, -1, -1, -1, 335, - 1761, -1, 1763, -1, -1, 400, -1, -1, -1, -1, - -1, -1, 145, -1, -1, 1776, -1, -1, -1, 1652, - 1781, 2704, 358, -1, 1785, 1786, 1787, 1788, -1, 1790, - 1791, -1, 1889, -1, -1, 1611, -1, -1, -1, -1, - -1, -1, -1, 176, 1901, -1, 1290, -1, 1292, -1, - -1, -1, -1, -1, -1, -1, -1, 1301, -1, -1, - 193, -1, -1, -1, 1697, 198, -1, -1, 1312, -1, - 100, -1, -1, -1, 1931, -1, -1, 1321, -1, -1, - 416, 1938, 1326, -1, 1328, 1329, -1, 1331, 1332, -1, - -1, 2888, -1, -1, 42, -1, -1, -1, 2781, -1, - -1, -1, -1, 236, -1, -1, -1, -1, 444, -1, - -1, -1, -1, -1, -1, 1359, -1, 147, 454, 1976, - -1, -1, -1, 1980, -1, 2922, -1, 1409, 1410, 1986, - -1, 161, -1, 1415, -1, -1, 166, 473, -1, 475, - 476, 171, -1, -1, -1, -1, -1, -1, -1, -1, - 180, 2008, -1, -1, -1, 185, -1, -1, 291, 107, - -1, 109, -1, 111, -1, 1409, 1410, 37, -1, 1745, - -1, 1415, 42, 1417, -1, -1, 512, -1, -1, 515, - 516, 517, -1, -1, -1, -1, 1430, 217, 1432, 1433, - -1, -1, -1, -1, 1033, -1, 1440, -1, 1442, -1, - 1444, -1, -1, 1447, 3001, 2888, -1, 237, 1452, -1, - -1, 1455, -1, 1457, -1, -1, -1, 1461, -1, 1463, - -1, 1465, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 101, -1, -1, -1, -1, -1, -1, -1, 2922, - -1, -1, -1, -1, -1, -1, -1, 3044, -1, -1, - -1, -1, -1, -1, -1, -1, 286, -1, 391, 289, - -1, -1, -1, -1, -1, 295, -1, -1, -1, 1902, - 1903, 1904, 1905, 1906, 1907, -1, -1, 1910, 1911, 1912, - 1913, 1914, 1915, 1916, 1917, 1918, 1919, -1, -1, -1, - -1, -1, -1, 1409, 1410, -1, -1, -1, 1033, 1415, - -1, -1, 172, -1, -1, 335, -1, -1, -1, -1, - 551, -1, -1, 1889, -1, -1, -1, -1, 3001, -1, - -1, -1, -1, -1, -1, 1901, -1, -1, 358, 1573, - -1, -1, -1, -1, -1, -1, -1, -1, 208, -1, - 1584, -1, -1, -1, -1, 1033, -1, -1, -1, 3146, - -1, -1, -1, -1, 224, 1931, -1, -1, -1, -1, - -1, 3044, 1938, -1, 234, 498, -1, -1, -1, -1, - 1209, -1, 505, 506, 507, 508, 509, 510, -1, 1623, - -1, -1, -1, -1, -1, -1, 416, -1, -1, -1, - 2151, -1, -1, -1, -1, -1, -1, -1, 268, -1, - 1976, -1, -1, -1, 1980, -1, -1, -1, -1, -1, - 1986, -1, -1, -1, 444, 1697, -1, -1, -1, 289, - -1, -1, -1, -1, 454, -1, 1670, -1, 1672, -1, - 1674, -1, 2008, -1, -1, -1, -1, -1, -1, -1, - 1684, 1685, -1, 473, 314, 475, 476, -1, -1, -1, - -1, 321, 177, 1697, -1, -1, 2089, 2090, 1297, -1, - -1, 2318, -1, 3146, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 1209, -1, -1, 202, 1722, -1, - 1724, -1, 512, -1, -1, 515, 516, 517, -1, -1, - -1, 361, -1, -1, -1, 1739, 1740, -1, -1, -1, - -1, 742, 743, -1, -1, -1, 1750, 1751, 1752, 1753, - 380, 1755, -1, -1, -1, 240, -1, 1761, -1, 1763, - -1, 1209, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 1775, 1776, -1, -1, -1, 1780, 1781, -1, -1, - -1, 1785, 1786, 1787, 1788, 8, 1790, 1791, 11, -1, - -1, -1, 15, 16, -1, -1, 19, 20, 21, -1, - -1, -1, 1297, 1807, -1, -1, -1, 2328, -1, 294, - 2203, 1815, -1, 1817, 1818, 1819, 1820, 1821, -1, 449, - -1, -1, -1, -1, 454, -1, -1, 535, 313, -1, - -1, -1, 540, 2450, 1838, 543, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 1851, -1, 1297, - -1, 3408, -1, -1, -1, -1, -1, -1, 859, -1, - 1902, 1903, 1904, 1905, 1906, 1907, -1, -1, 1910, 1911, - 1912, 1913, 1914, 1915, 1916, 1917, 1918, 1919, -1, -1, - -1, -1, -1, -1, -1, 370, -1, -1, -1, 2506, - -1, 26, 377, -1, -1, -1, -1, 32, 1902, 1903, - 1904, 1905, 1906, 1907, -1, 40, 1910, 1911, 1912, 1913, - 1914, 1915, 1916, 1917, 1918, 1919, -1, -1, -1, 1518, - -1, -1, -1, -1, -1, 60, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 420, 1940, 422, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 1952, -1, - -1, -1, -1, -1, 439, -1, -1, -1, 1962, -1, - 445, -1, -1, 448, 1968, 3408, -1, -1, -1, -1, - 1974, 106, -1, -1, -1, 1979, -1, -1, -1, -1, - -1, -1, 2318, -1, 1988, 1989, -1, -1, -1, -1, - -1, -1, 2609, -1, -1, -1, -1, -1, 221, 1000, - -1, -1, 137, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 1611, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 1023, 1518, -1, -1, 1902, 1903, 1904, 1905, - 1906, 1907, -1, -1, 1910, 1911, 1912, 1913, 1914, 1915, - 1916, 1917, 1918, 1919, -1, -1, -1, 2089, 2090, 2570, - 2571, -1, -1, 2057, -1, -1, -1, 2061, -1, -1, - -1, -1, 2066, -1, -1, -1, 201, -1, 291, -1, - 1518, -1, -1, -1, -1, -1, -1, 2598, -1, -1, - -1, -1, -1, 791, -1, 2089, 2090, 2704, 1089, -1, - -1, -1, 2613, 2614, 2615, 2616, 2617, 2618, 2619, 2620, - 2621, 2622, 2106, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 2450, 250, 1611, 2510, 2511, -1, - -1, -1, -1, -1, -1, 260, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 2139, -1, 272, -1, -1, - -1, -1, -1, -1, -1, -1, 1745, 2151, -1, -1, - -1, -1, 2156, -1, 1155, 1156, 2160, -1, 293, -1, - -1, 2203, -1, 1611, 2781, -1, -1, -1, -1, 304, - 2506, -1, -1, 8, -1, -1, 11, 885, -1, -1, - 15, 16, -1, -1, 19, 20, 21, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 2200, -1, -1, 2203, - -1, -1, 2206, -1, -1, -1, -1, -1, -1, 917, - -1, -1, -1, 2089, 2090, 350, -1, -1, 2222, 354, - -1, 356, -1, -1, -1, 933, 934, 935, 936, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 98, -1, -1, -1, -1, 381, -1, -1, -1, - 1745, 386, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 400, -1, -1, -1, 126, - 127, 2888, -1, 2609, -1, -1, -1, 985, 501, 502, - 503, -1, 505, 506, 507, 508, 509, 510, -1, 2293, - 1889, -1, -1, -1, -1, -1, 2300, 1745, -1, -1, - -1, -1, 1901, -1, -1, 2922, -1, 2311, 2701, -1, - 2314, 2315, 2316, 2317, 171, -1, -1, -1, -1, -1, - -1, -1, 457, -1, 2328, -1, 2330, 1328, 1329, 2333, - 1331, -1, 1931, 2726, 2338, -1, -1, -1, 2859, 1938, - 2344, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 2362, 2363, - -1, 2365, -1, 198, -1, -1, -1, -1, 2704, -1, - -1, -1, -1, -1, -1, -1, -1, 1976, -1, -1, - -1, 1980, -1, -1, 3001, -1, 221, 1986, -1, -1, - 2394, -1, -1, -1, 1889, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 1901, -1, -1, 2008, - 2414, -1, -1, -1, -1, -1, -1, -1, 2422, 2423, - 2424, -1, -1, -1, -1, -1, -1, 3044, 2432, -1, - 2434, 1432, 2436, -1, -1, -1, 1931, -1, 2442, -1, - -1, 1889, -1, 1938, -1, 2781, -1, -1, -1, -1, - -1, -1, -1, 1901, -1, -1, 291, 314, 315, 316, - -1, -1, 1463, -1, 1465, -1, -1, 2860, 2510, 2511, - -1, 2475, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 1976, -1, 1931, -1, 1980, -1, 1195, -1, -1, - 1938, 1986, -1, -1, -1, -1, -1, -1, -1, -1, - 1208, -1, -1, -1, -1, -1, 2510, 2511, -1, -1, - -1, -1, -1, 2008, -1, -1, -1, -1, -1, -1, - -1, 1229, 379, -1, -1, -1, -1, -1, 1976, 3146, - -1, -1, 1980, 390, -1, 2539, -1, -1, 1986, -1, - -1, 2545, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 2888, -1, -1, 412, -1, 2561, -1, 416, - 2008, 2565, -1, -1, -1, 2569, 2570, 2571, -1, -1, - -1, 2575, 2576, 2577, -1, 2579, -1, -1, -1, 436, - -1, -1, -1, -1, 1292, -1, 2922, -1, -1, -1, - -1, -1, 449, 1301, 2598, -1, 2600, -1, -1, -1, - -1, -1, -1, 460, -1, 3126, -1, 3000, -1, 2613, - 2614, 2615, 2616, 2617, 2618, 2619, 2620, 2621, 2622, 476, - -1, -1, -1, -1, -1, -1, 3147, 3148, -1, -1, - -1, -1, -1, 8, 2510, 2511, 11, -1, -1, -1, - 15, 16, 2646, 500, 19, 20, 21, -1, 2652, -1, - -1, -1, 3173, -1, -1, -1, -1, 514, -1, 2701, - 2664, 36, -1, -1, -1, 3001, 501, 502, 503, -1, - 505, 506, 507, 508, 509, 510, 2680, -1, -1, -1, - -1, 835, 836, -1, 2726, -1, 2690, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 2701, -1, -1, - -1, -1, -1, 2707, -1, -1, -1, -1, 3044, -1, - 2714, 2715, 2716, 2717, -1, -1, -1, -1, -1, 2318, - -1, -1, 2726, 1724, -1, 2729, -1, -1, 2770, 2733, - 2734, -1, 1440, -1, 8, -1, -1, 11, 2742, 1740, - -1, 15, 16, -1, -1, 19, 20, 21, 902, 1750, - -1, 1752, 906, 907, 1755, -1, -1, -1, -1, -1, - 1761, -1, 1763, -1, -1, 2769, -1, -1, -1, -1, - -1, -1, 46, -1, -1, 1776, -1, -1, -1, 53, - 1781, -1, -1, -1, 1785, 1786, 1787, 1788, -1, 1790, - 1791, 3408, -1, -1, -1, 2799, -1, 2801, -1, -1, - -1, -1, -1, -1, -1, 959, 80, -1, -1, -1, - 3146, 2815, -1, -1, -1, 2819, -1, 3210, 2860, -1, - -1, 2825, -1, 2318, 978, 2701, -1, -1, -1, -1, - 984, -1, -1, 987, -1, 2839, 990, 991, 992, 993, - -1, 2845, -1, -1, -1, -1, 221, -1, -1, -1, - -1, 2450, -1, -1, 2858, 2859, 2860, -1, -1, -1, - -1, -1, 2866, -1, -1, -1, -1, -1, -1, -1, - 2318, -1, -1, 8, -1, -1, 11, 1031, 1032, 2883, - 15, 16, -1, 3404, 19, 20, 21, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 1052, -1, - -1, -1, 176, -1, -1, -1, -1, 2506, -1, -1, - -1, -1, -1, -1, -1, -1, 291, 1071, -1, 193, - -1, -1, -1, -1, 198, -1, -1, -1, 1082, 1083, - 1084, -1, 1086, 1087, -1, -1, -1, -1, -1, 1940, - -1, -1, -1, -1, -1, -1, -1, 221, 222, -1, - -1, -1, -1, -1, -1, 2450, -1, 2961, 3000, -1, - -1, 2965, 236, -1, 1672, 1119, 1674, -1, -1, -1, - -1, -1, -1, 1974, -1, 2979, 1684, -1, 1979, -1, - -1, -1, 1136, 1137, 2860, -1, -1, -1, -1, 2993, - -1, -1, -1, -1, -1, -1, 3000, -1, -1, -1, - 274, -1, 2450, 277, -1, -1, -1, -1, -1, -1, - 2609, 2506, -1, -1, 1722, -1, 1170, 291, -1, -1, - 1174, 1175, 3026, -1, -1, -1, -1, -1, 3032, -1, - -1, 1185, 1186, -1, -1, 3039, 8, -1, -1, 11, - -1, -1, -1, 15, 16, 17, 18, 19, 20, 21, - -1, -1, -1, -1, -1, -1, 2057, -1, 2506, -1, - -1, -1, -1, 198, 36, 2066, -1, -1, -1, 3073, - -1, -1, 3408, -1, 46, -1, -1, -1, -1, -1, - -1, 53, 8, -1, -1, 11, 221, -1, 1242, 15, - 16, 3095, -1, 19, 20, 21, 3100, 1251, -1, -1, - -1, -1, -1, -1, -1, 2704, -1, -1, 80, 1817, - 1818, 1819, 1820, 1821, 2609, -1, -1, 391, -1, -1, - -1, 1275, 3126, -1, 3000, -1, 501, 502, 503, -1, - 505, 506, 507, 508, 509, 510, -1, -1, -1, -1, - -1, -1, -1, 3147, 3148, -1, -1, -1, -1, -1, - 2151, -1, -1, -1, -1, -1, 291, -1, -1, -1, - -1, 2609, 3166, 8, 3168, -1, 11, -1, 3210, 3173, - 15, 16, -1, -1, 19, 20, 21, -1, -1, -1, - -1, -1, 2781, 3187, -1, -1, 3190, -1, -1, -1, - -1, 36, -1, -1, -1, -1, -1, -1, -1, 2200, - -1, -1, -1, -1, 176, 2206, 3210, -1, 1362, 2704, - 3214, 3215, -1, 3217, -1, -1, -1, -1, -1, -1, - -1, 193, -1, -1, 498, -1, 198, 501, 502, 503, - -1, 505, 506, 507, 508, 509, 510, -1, 3242, -1, - -1, -1, -1, -1, 1952, 3249, -1, -1, -1, 221, - 222, -1, -1, -1, -1, -1, 2704, -1, -1, -1, - -1, -1, -1, -1, 236, -1, -1, -1, 1422, -1, - 1424, 1425, -1, -1, -1, -1, 3280, -1, -1, -1, - -1, -1, -1, 1437, 1438, -1, 2781, -1, -1, 2888, - -1, -1, -1, -1, -1, 221, -1, -1, -1, -1, - 1454, -1, 274, -1, -1, 277, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 3319, -1, -1, -1, 291, - -1, -1, 294, 2922, -1, -1, -1, 2328, -1, -1, - -1, -1, -1, 2781, -1, -1, -1, 3341, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 3368, 291, 501, 502, 503, -1, - 505, 506, 507, 508, 509, 510, 221, 3381, 3382, -1, - 3384, 3385, -1, -1, -1, -1, -1, -1, -1, 3393, - -1, -1, -1, 2888, -1, -1, -1, -1, -1, -1, - 3404, -1, 3001, -1, -1, 1559, -1, -1, -1, -1, - -1, -1, -1, 1567, -1, -1, -1, -1, -1, 391, - -1, -1, 3426, -1, -1, -1, -1, 2922, -1, 3433, - -1, 2432, -1, -1, -1, 2436, -1, -1, -1, -1, - 2888, 2442, -1, -1, -1, 3044, 291, -1, -1, -1, - -1, -1, 1606, -1, -1, -1, -1, -1, 1612, 1613, - 1614, 1615, 1616, 1617, 1618, 1619, 3470, -1, -1, -1, - 1624, 1625, -1, -1, 2922, 1629, -1, -1, -1, 1633, - -1, 3485, 1636, 1637, 1638, 1639, 1640, 1641, 1642, 1643, - 1644, -1, -1, 1647, -1, -1, -1, -1, -1, -1, - 1654, 8, 1656, -1, 11, -1, 3001, -1, 15, 16, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 1675, -1, -1, -1, -1, 498, -1, -1, 501, - 502, 503, -1, 505, 506, 507, 508, 509, 510, 46, - -1, -1, -1, -1, -1, -1, 53, 3146, -1, 3044, - -1, -1, 524, 3001, 1708, 1709, -1, -1, -1, -1, - -1, -1, -1, -1, 2565, -1, -1, -1, 2569, 2570, - 2571, -1, -1, 80, -1, 501, 502, 503, -1, 505, - 506, 507, 508, 509, 510, -1, -1, -1, -1, -1, - -1, -1, 2300, -1, -1, -1, 3044, 2598, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 2315, 2316, 2317, - -1, -1, 2613, 2614, 2615, 2616, 2617, 2618, 2619, 2620, - 2621, 2622, 2330, -1, -1, 2333, -1, -1, -1, -1, - 2338, -1, -1, -1, -1, -1, -1, -1, 145, -1, - -1, -1, 1796, -1, -1, -1, 1800, -1, -1, 1803, - 1804, 3146, -1, -1, -1, -1, 501, 502, 503, -1, - 505, 506, 507, 508, 509, 510, -1, -1, -1, 176, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 193, -1, -1, -1, - 1844, 198, -1, -1, -1, -1, -1, -1, 3146, -1, - -1, -1, -1, -1, -1, -1, 2707, -1, -1, 1863, - 1864, -1, -1, -1, 2422, 2423, 2424, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 236, - -1, -1, 2733, -1, -1, -1, -1, -1, -1, 1893, - 1894, -1, 1896, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 8, -1, -1, 11, -1, 0, -1, 15, 16, -1, - -1, 1925, 1926, -1, -1, 1929, -1, -1, -1, -1, - -1, -1, -1, -1, 291, -1, -1, 22, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 32, 46, 34, - 35, -1, -1, -1, 1958, 53, -1, -1, -1, 3408, - 1964, -1, 47, -1, -1, -1, -1, 52, -1, -1, - -1, -1, -1, -1, -1, -1, 61, -1, 1982, -1, - 1984, -1, 80, -1, -1, -1, -1, -1, -1, -1, - 75, -1, -1, -1, -1, -1, -1, -1, -1, 84, - -1, 86, -1, 2561, -1, -1, -1, -1, 2859, -1, - -1, -1, -1, 98, -1, 100, 2020, -1, -1, -1, - -1, -1, 26, -1, -1, -1, 111, -1, 32, -1, - -1, -1, -1, -1, 391, 37, 40, -1, -1, -1, - 42, 126, 127, 128, -1, -1, -1, 145, -1, -1, - -1, -1, 137, -1, -1, 2059, 60, -1, 143, -1, - 2064, 2065, -1, 3408, -1, -1, 151, -1, 153, 154, - -1, -1, -1, -1, -1, -1, -1, -1, 176, -1, - -1, -1, 167, -1, -1, -1, 171, -1, -1, -1, - -1, -1, -1, -1, 2652, 193, -1, -1, -1, 101, - 198, -1, 106, -1, 2108, -1, -1, 2111, -1, 2113, - 3408, -1, 197, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 2129, -1, -1, 213, -1, - -1, -1, -1, 137, -1, -1, -1, -1, 236, -1, - -1, 498, -1, -1, -1, -1, -1, -1, 505, 506, - 507, 508, 509, 510, 239, -1, 2714, 2715, 2716, 2717, - -1, -1, -1, -1, -1, 2169, -1, -1, -1, -1, - 172, -1, -1, -1, -1, -1, -1, -1, 180, -1, - -1, 3032, 2186, 2187, -1, -1, -1, -1, 3039, -1, - -1, -1, -1, 291, -1, -1, -1, 201, -1, -1, - -1, 2205, -1, -1, -1, 8, 208, -1, 11, -1, - 2214, -1, 15, 16, 17, 18, 19, 20, 21, -1, - -1, -1, 224, -1, -1, -1, -1, -1, -1, 314, - 315, 316, 234, 36, -1, -1, -1, 322, 988, -1, - 325, -1, -1, 46, 3095, -1, 250, -1, -1, 3100, - 53, -1, -1, -1, -1, -1, 260, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 268, 2271, 272, 354, - -1, -1, -1, 2277, -1, 3126, -1, 80, 363, -1, - -1, -1, -1, -1, -1, -1, -1, 289, -1, 293, - -1, -1, -1, 391, 379, -1, 3147, 3148, -1, -1, - 304, 386, -1, -1, -1, 390, -1, -1, 2866, -1, - -1, -1, 314, -1, -1, 400, -1, -1, -1, 321, - -1, -1, 3173, 37, -1, -1, -1, 412, 42, -1, - -1, 416, 2336, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 350, -1, -1, -1, - 354, 436, 356, -1, -1, -1, -1, 1107, -1, 361, - -1, -1, -1, 3214, 449, -1, -1, 452, -1, -1, - 455, -1, -1, 176, -1, 460, -1, 381, 380, -1, - -1, -1, 386, -1, -1, -1, -1, 101, -1, -1, - 193, 476, -1, -1, -1, 198, 400, -1, -1, -1, - 498, 1151, -1, -1, -1, -1, -1, 505, 506, 507, - 508, 509, 510, -1, -1, 500, -1, -1, 221, 222, - -1, -1, -1, 3, -1, -1, -1, -1, 8, 514, - -1, 11, 517, 236, 1184, 15, 16, 17, 18, 19, - 20, 21, -1, -1, -1, -1, -1, 449, -1, -1, - -1, -1, 454, 457, -1, -1, 36, -1, 172, -1, - 40, -1, 2466, -1, -1, -1, 46, -1, 2472, 2473, - -1, 274, -1, 53, 277, -1, -1, -1, -1, 2483, - -1, -1, -1, -1, -1, -1, 1236, -1, 291, -1, - 2494, 294, -1, 2497, 208, 2499, -1, -1, -1, -1, - 80, -1, -1, 2507, -1, -1, -1, -1, -1, -1, - 224, 2515, 2516, -1, -1, -1, -1, -1, 2522, -1, - 234, -1, -1, -1, -1, -1, -1, -1, 1278, -1, - -1, -1, -1, 2537, -1, 1285, -1, -1, -1, -1, - -1, -1, -1, 2547, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 3404, 268, -1, -1, -1, -1, -1, - -1, -1, -1, 2567, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 289, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 1335, -1, -1, 391, -1, - -1, -1, -1, -1, -1, -1, 176, -1, -1, -1, - 314, -1, -1, -1, -1, -1, -1, 321, 3166, -1, - 1360, -1, -1, 193, -1, -1, -1, -1, 198, -1, - 2624, -1, 2626, -1, -1, -1, -1, -1, -1, 3187, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 221, 222, -1, -1, -1, -1, 361, -1, -1, - -1, -1, -1, -1, -1, 1405, 236, 1407, -1, 1409, - 1410, -1, 1412, -1, -1, 1415, 380, -1, 1418, -1, - -1, 1421, -1, -1, -1, -1, 1426, -1, -1, 1429, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 3249, -1, -1, 274, 498, -1, 277, 501, 502, - 503, -1, 505, 506, 507, 508, 509, 510, -1, -1, - -1, 291, -1, -1, 294, 518, -1, 2721, 2722, -1, - -1, 1471, 3280, -1, -1, -1, -1, -1, -1, 24, - -1, -1, -1, -1, -1, 449, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 2756, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 2770, -1, -1, -1, - -1, 2775, 2776, -1, -1, -1, 2780, -1, -1, -1, - -1, 2785, -1, -1, 2788, 2789, 81, -1, -1, 2793, - 2794, -1, -1, 2797, -1, -1, -1, -1, -1, -1, - -1, -1, 97, -1, 2808, 1555, -1, -1, -1, -1, - -1, 391, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 1571, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 1581, 1582, 1583, -1, -1, -1, 1587, -1, -1, - -1, 1591, -1, -1, -1, 3, -1, 5, 2852, -1, - -1, 146, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 156, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 168, -1, 2879, -1, -1, 173, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 1653, -1, -1, -1, 202, -1, -1, - 68, 69, -1, -1, -1, -1, -1, -1, 498, -1, - -1, 501, 502, 503, -1, 505, 506, 507, 508, 509, - 510, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 1691, -1, -1, -1, -1, -1, -1, -1, -1, - 245, 109, 110, -1, 249, 113, 114, 1707, -1, -1, - -1, 8, 1712, -1, 11, 2969, -1, -1, 15, 16, - 17, 18, 19, 20, 21, -1, -1, -1, -1, 1729, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 36, - -1, -1, -1, 40, -1, -1, -1, -1, -1, 46, - -1, -1, -1, -1, -1, -1, 53, -1, 8, -1, - -1, 11, 3016, -1, -1, 15, 16, 17, 18, 19, - 20, 21, -1, -1, 319, -1, -1, -1, -1, -1, - 188, 189, -1, 80, -1, -1, 36, -1, -1, 334, - -1, -1, -1, -1, -1, 3049, 46, -1, -1, -1, - -1, -1, -1, 53, -1, -1, 3060, 3061, -1, -1, - 3064, -1, 3066, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 367, -1, -1, 370, -1, -1, -1, -1, - 80, -1, -1, -1, -1, -1, 381, 3091, -1, 384, - -1, -1, -1, -1, 252, 253, 254, 255, 256, 257, - 258, 259, -1, -1, 262, 263, -1, 402, -1, -1, - -1, 3115, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 416, -1, -1, -1, -1, -1, -1, 423, 176, - -1, -1, -1, -1, -1, -1, -1, -1, 433, -1, - -1, 1891, -1, -1, 439, -1, 193, -1, -1, 1899, - 1900, 198, 1902, 1903, 1904, 1905, 1906, 1907, -1, -1, - 1910, 1911, 1912, 1913, 1914, 1915, 1916, 1917, 1918, 1919, - 1920, 466, -1, -1, 221, 222, 176, -1, -1, -1, - -1, -1, 3186, -1, -1, -1, -1, -1, -1, 236, - -1, -1, -1, 193, -1, -1, -1, -1, 198, -1, - 3204, -1, -1, -1, 3208, -1, -1, -1, 3212, -1, - -1, -1, -1, -1, -1, -1, -1, 375, 376, 3223, - -1, 221, 222, -1, 3228, -1, 3230, 274, -1, -1, - 277, -1, -1, -1, 3238, -1, 236, -1, -1, -1, - -1, -1, -1, -1, 291, -1, -1, 294, -1, -1, - -1, -1, 3256, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 2023, 274, -1, 2026, 277, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 291, -1, -1, 294, -1, -1, -1, -1, -1, - -1, 3305, -1, -1, -1, -1, -1, 8, -1, -1, - 11, 3315, 470, 471, 15, 16, 17, 18, 19, 20, - 21, -1, 2072, 3327, -1, -1, 2076, 2077, 2078, 2079, - 2080, 2081, 2082, 2083, -1, 36, 494, 495, -1, 2089, - 2090, -1, 2092, 2093, 391, 46, -1, -1, -1, -1, - -1, -1, 53, -1, 2104, 3359, -1, 2107, -1, -1, - -1, -1, -1, -1, -1, 2115, 2116, 2117, 2118, 2119, - 2120, 2121, 2122, 2123, 2124, -1, -1, -1, -1, 80, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 391, -1, -1, 8, -1, -1, 11, -1, -1, - 2150, 15, 16, 17, 18, 19, 20, 21, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 36, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 46, -1, -1, -1, -1, -1, -1, 53, - -1, -1, -1, 3447, 3448, 3449, -1, -1, -1, -1, - -1, 498, -1, 2203, 501, 502, 503, -1, 505, 506, - 507, 508, 509, 510, -1, -1, 80, -1, 515, -1, - 3474, -1, -1, -1, -1, 176, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 193, -1, -1, -1, -1, 198, 498, -1, - -1, 501, 502, 503, -1, 505, 506, 507, 508, 509, - 510, 3515, -1, -1, -1, 515, -1, -1, -1, -1, - 221, 222, 2272, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 2282, 2283, -1, 236, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 176, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 274, -1, -1, 277, -1, -1, 193, - -1, -1, -1, -1, 198, -1, -1, -1, -1, -1, - 291, -1, -1, 294, -1, -1, -1, 2347, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 221, 222, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 2369, - 2370, 2371, 236, -1, 2374, 2375, 2376, 2377, 2378, 2379, - -1, -1, -1, 2383, 2384, 2385, 2386, 2387, 2388, 2389, - 2390, 2391, 2392, -1, -1, -1, -1, 2397, 2398, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 274, -1, -1, 277, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 2425, -1, 291, -1, -1, - 294, 2431, -1, -1, 8, -1, -1, 11, -1, -1, - 391, 15, 16, 17, 18, 19, 20, 21, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 36, -1, -1, 2465, -1, -1, -1, -1, - -1, -1, 46, -1, -1, -1, -1, -1, -1, 53, - -1, -1, -1, -1, 2484, -1, -1, 2487, 2488, -1, - -1, -1, -1, -1, -1, 2495, 2496, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 80, -1, -1, 2509, - 2510, 2511, 2512, -1, 2514, -1, -1, -1, 2518, -1, - -1, -1, -1, -1, -1, -1, -1, 391, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 498, -1, -1, - 501, 502, 503, -1, 505, 506, 507, 508, 509, 510, - -1, -1, -1, -1, 515, -1, -1, -1, -1, -1, - -1, -1, 2572, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 8, -1, -1, 11, -1, -1, -1, 15, - 16, 17, 18, 19, 20, 21, -1, -1, -1, -1, - -1, -1, 176, -1, -1, -1, -1, -1, -1, -1, - 36, -1, -1, -1, -1, -1, -1, -1, -1, 193, - 46, -1, -1, -1, 198, -1, -1, 53, -1, -1, - -1, -1, -1, -1, 498, -1, -1, 501, 502, 503, - -1, 505, 506, 507, 508, 509, 510, 221, 222, -1, - -1, 515, -1, -1, 80, -1, -1, 8, -1, -1, - 11, -1, 236, -1, 15, 16, 17, 18, 19, 20, - 21, -1, -1, -1, -1, -1, -1, -1, -1, 2679, - -1, -1, -1, -1, -1, 36, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 46, -1, -1, -1, -1, - 274, 2701, 53, 277, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 291, -1, -1, - 294, -1, -1, -1, -1, -1, -1, -1, -1, 80, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 8, -1, -1, 11, -1, -1, -1, 15, 16, -1, - 176, 19, 20, 21, -1, -1, -1, 2757, -1, 2759, - -1, -1, -1, -1, -1, 2765, -1, 193, 36, -1, - -1, -1, 198, -1, 2774, -1, -1, 2777, 46, 2779, - -1, -1, -1, 2783, -1, 53, 2786, 2787, -1, -1, - 2790, 2791, -1, -1, -1, 221, 222, -1, 2798, -1, - -1, -1, -1, -1, -1, -1, -1, 2807, -1, -1, - 236, -1, 80, -1, -1, -1, -1, 391, -1, -1, - -1, -1, 2822, -1, -1, 176, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 193, -1, -1, -1, -1, 198, 274, -1, - -1, 277, -1, -1, -1, -1, -1, -1, -1, -1, - 2860, -1, -1, -1, -1, 291, -1, -1, 294, -1, - 221, 222, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 236, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 176, -1, - 68, 69, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 274, 498, 193, 277, 501, 502, 503, - 198, 505, 506, 507, 508, 509, 510, -1, -1, -1, - 291, 515, -1, 294, -1, -1, -1, -1, 68, 69, - -1, 109, 110, 221, 222, 113, 114, -1, -1, -1, - -1, -1, -1, -1, -1, 391, -1, -1, 236, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 109, - 110, -1, -1, 113, 114, -1, -1, -1, -1, 2999, - 3000, -1, -1, -1, -1, -1, 274, -1, -1, 277, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 291, 3024, 3025, 294, -1, -1, -1, - 188, 189, -1, -1, -1, -1, -1, -1, -1, -1, - 391, -1, -1, -1, -1, -1, -1, -1, 3048, -1, - -1, -1, 3052, -1, 3054, 3055, 3056, -1, -1, 3059, - -1, -1, 3062, 3063, -1, -1, -1, -1, 188, 189, - -1, 3071, 498, -1, -1, 501, 502, 503, -1, 505, - 506, 507, 508, 509, 510, -1, -1, -1, -1, 515, - -1, -1, -1, -1, 252, 253, 254, 255, 256, 257, - 258, 259, -1, -1, 262, 263, -1, -1, -1, -1, - -1, -1, 3112, -1, -1, -1, -1, -1, 3118, -1, - -1, -1, -1, 391, -1, -1, -1, -1, -1, -1, - -1, 3131, 252, 253, 254, 255, 256, 257, 258, 259, - -1, -1, 262, 263, -1, -1, -1, 498, -1, -1, - 501, 502, 503, -1, 505, 506, 507, 508, 509, 510, - -1, -1, -1, -1, 515, 3165, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 3192, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 375, 376, -1, - -1, -1, -1, -1, -1, 3225, 3226, 3227, -1, -1, - 498, -1, -1, 501, 502, 503, -1, 505, 506, 507, - 508, 509, 510, -1, -1, 3245, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 375, 376, 3257, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 3303, -1, -1, -1, -1, -1, -1, - -1, -1, 470, 471, -1, -1, -1, -1, -1, -1, - 3320, -1, -1, -1, -1, -1, -1, -1, -1, 3329, - -1, -1, -1, -1, -1, -1, 494, 495, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 470, 471, -1, -1, 3354, -1, 514, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 494, 495, -1, -1, -1, -1, - 3380, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 3, 4, 5, 6, - 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 3423, -1, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - -1, -1, 39, -1, -1, 42, 43, 44, -1, 46, - 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - 57, -1, 59, 60, 61, 62, 63, 64, 65, 66, - 67, 68, 69, 70, 71, 72, 73, 3477, 75, 76, - 77, 78, 79, -1, 81, -1, 83, 84, 85, 86, - 87, 88, 89, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, 101, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - 117, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, - -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, 159, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, 170, -1, 172, 173, 174, 175, 176, - 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, 221, 222, 223, 224, 225, 226, - 227, 228, 229, 230, -1, -1, 233, 234, 235, 236, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, - 277, 278, 279, 280, 281, 282, 283, 284, 285, -1, - 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, 321, 322, 323, 324, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, 361, 362, 363, 364, 365, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, -1, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, 391, 392, 393, 394, 395, -1, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, - 417, 418, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, 435, 436, - 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, -1, 463, 464, 465, 466, - 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, - 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, - 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 498, -1, -1, -1, -1, -1, -1, 505, 506, - 507, -1, -1, -1, -1, 512, -1, 514, -1, -1, - -1, -1, 519, 520, -1, 522, 523, 524, 3, 4, - 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 22, 23, 24, - 25, 26, 27, 28, 29, -1, 31, 32, 33, -1, - -1, -1, -1, -1, 39, -1, -1, 42, 43, 44, - -1, 46, 47, 48, 49, 50, 51, 52, 53, 54, - 55, 56, 57, -1, 59, 60, 61, 62, 63, 64, - 65, 66, 67, 68, 69, 70, 71, 72, 73, -1, - 75, 76, 77, 78, 79, -1, 81, -1, 83, 84, - 85, 86, 87, 88, 89, 90, 91, 92, -1, 94, - 95, 96, 97, 98, 99, -1, 101, 102, 103, 104, - 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, - 115, 116, 117, 118, -1, 120, 121, 122, 123, 124, - 125, -1, -1, 128, 129, 130, 131, -1, -1, 134, - 135, 136, 137, 138, -1, 140, 141, 142, -1, 144, - 145, 146, -1, 148, 149, 150, 151, 152, 153, 154, - 155, 156, 157, 158, 159, 160, -1, 162, 163, 164, - 165, -1, 167, -1, 169, 170, -1, 172, 173, 174, - 175, 176, 177, -1, 179, -1, 181, 182, 183, 184, - -1, 186, 187, 188, 189, 190, 191, 192, 193, 194, - 195, 196, 197, -1, 199, 200, 201, 202, 203, 204, - 205, -1, 207, 208, 209, 210, 211, 212, 213, 214, - 215, 216, -1, 218, -1, 220, 221, 222, 223, 224, - 225, 226, 227, 228, 229, 230, -1, -1, 233, 234, - 235, 236, -1, 238, 239, 240, 241, 242, 243, 244, - 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, - 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, - 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, - 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, - 285, -1, 287, 288, -1, -1, 291, 292, 293, -1, - -1, 296, 297, 298, 299, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, -1, - -1, -1, -1, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 327, -1, 329, 330, 331, 332, 333, 334, - -1, 336, 337, 338, 339, 340, 341, 342, 343, -1, - 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, - 355, 356, 357, -1, 359, 360, 361, 362, 363, 364, - 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, - 375, 376, 377, 378, -1, -1, 381, 382, 383, 384, - 385, 386, 387, 388, 389, -1, 391, 392, 393, 394, - 395, -1, 397, 398, 399, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, -1, -1, 414, - 415, -1, 417, 418, 419, 420, 421, 422, 423, -1, - 425, 426, 427, -1, -1, 430, 431, 432, 433, 434, - 435, 436, 437, 438, 439, 440, 441, 442, 443, -1, - -1, 446, 447, 448, -1, 450, 451, 452, 453, -1, - 455, 456, 457, 458, 459, 460, 461, -1, 463, 464, - 465, 466, 467, 468, 469, 470, 471, -1, -1, 474, - -1, -1, 477, 478, 479, 480, 481, 482, 483, 484, - 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, - 495, 496, 497, 498, -1, -1, -1, -1, -1, -1, - 505, 506, 507, -1, -1, -1, -1, 512, -1, 514, - -1, -1, -1, -1, 519, 520, -1, 522, 523, 524, - 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 22, - 23, 24, 25, 26, 27, 28, 29, -1, 31, 32, - 33, -1, -1, -1, -1, -1, 39, -1, -1, 42, - 43, 44, -1, 46, 47, 48, 49, 50, 51, 52, - 53, 54, 55, 56, 57, -1, 59, 60, 61, 62, - 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, - 73, -1, 75, 76, 77, 78, 79, -1, 81, -1, - 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, - -1, 94, 95, 96, 97, 98, 99, -1, 101, 102, - 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, - 113, 114, 115, 116, 117, 118, -1, 120, 121, 122, - 123, 124, 125, 126, 127, 128, 129, 130, 131, -1, - -1, 134, 135, 136, 137, 138, -1, 140, 141, 142, - -1, 144, 145, 146, -1, 148, 149, 150, 151, 152, - 153, 154, 155, 156, 157, 158, 159, 160, -1, 162, - 163, 164, 165, -1, 167, -1, 169, 170, 171, 172, - 173, 174, 175, 176, 177, -1, 179, -1, 181, 182, - 183, 184, -1, 186, 187, 188, 189, 190, 191, 192, - 193, 194, 195, 196, 197, -1, 199, 200, 201, 202, - 203, 204, 205, -1, 207, 208, 209, 210, 211, 212, - 213, 214, 215, 216, -1, 218, -1, 220, 221, 222, - 223, 224, 225, 226, 227, 228, 229, 230, -1, -1, - 233, 234, 235, 236, -1, 238, 239, 240, 241, 242, - 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, - 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, - 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, - 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, - 283, 284, 285, -1, 287, 288, -1, -1, 291, 292, - 293, -1, -1, 296, 297, 298, 299, 300, 301, 302, - 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, - 313, 314, 315, 316, -1, 318, 319, 320, 321, 322, - 323, 324, 325, 326, 327, -1, 329, 330, 331, 332, - 333, 334, -1, 336, 337, 338, 339, 340, 341, 342, - 343, -1, 345, 346, 347, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, -1, 359, 360, 361, 362, - 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, - 373, 374, 375, 376, 377, 378, 379, -1, 381, 382, - 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, - 393, 394, 395, -1, 397, 398, 399, 400, 401, 402, - 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, - -1, 414, 415, 416, 417, 418, 419, 420, 421, 422, - 423, -1, 425, 426, 427, -1, -1, 430, 431, 432, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, + 0, 135, 136, 137, 138, 139, 0, 141, 142, 0, + 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, + 151, 152, 153, 154, 155, 0, 156, 157, 158, 159, + 160, 0, 0, 0, 162, 163, 164, 165, 166, 167, + 0, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 0, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, + 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, + 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, + 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + 0, 227, 0, 228, 229, 230, 231, 0, 232, 0, + 233, 0, 0, 0, 236, 237, 524, 0, 240, 0, + 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, + 249, 250, 251, 252, 0, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 0, + 267, 268, 269, 270, 271, 272, 273, 274, 0, 275, + 0, 276, 0, 0, 279, 0, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 0, 289, 0, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 525, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 0, + 322, 323, 324, 325, 326, 0, 327, 328, 0, 330, + 0, 331, 332, 333, 334, 335, 336, 0, 337, 338, + 0, 791, 339, 340, 341, 0, 0, 342, 343, 344, + 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, + 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, + 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, + 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, - 443, -1, -1, 446, 447, 448, 449, 450, 451, 452, - 453, -1, 455, 456, 457, 458, 459, 460, 461, -1, - 463, 464, 465, 466, 467, 468, 469, 470, 471, -1, - -1, 474, -1, 476, 477, 478, 479, 480, 481, 482, - 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, - 493, 494, 495, 496, 497, 498, -1, 500, -1, -1, - -1, -1, 505, 506, 507, -1, -1, -1, -1, 512, - -1, 514, 515, -1, -1, -1, 519, 520, -1, 522, - 523, 3, 4, 5, 6, 7, 8, 9, 10, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, - 32, 33, -1, -1, -1, -1, -1, 39, -1, -1, - 42, 43, 44, -1, 46, 47, 48, 49, 50, 51, - 52, 53, 54, 55, 56, 57, -1, 59, 60, 61, - 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, - 72, 73, -1, 75, 76, 77, 78, 79, -1, 81, - -1, 83, 84, 85, 86, 87, 88, 89, 90, 91, - 92, -1, 94, 95, 96, 97, 98, 99, -1, 101, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, 117, 118, -1, 120, 121, - 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, - -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, - 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, 159, 160, -1, - 162, 163, 164, 165, -1, 167, -1, 169, 170, 171, - 172, 173, 174, 175, 176, 177, -1, 179, -1, 181, - 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, - 192, 193, 194, 195, 196, 197, -1, 199, 200, 201, - 202, 203, 204, 205, -1, 207, 208, 209, 210, 211, - 212, 213, 214, 215, 216, -1, 218, -1, 220, 221, - 222, 223, 224, 225, 226, 227, 228, 229, 230, -1, - -1, 233, 234, 235, 236, -1, 238, 239, 240, 241, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, - 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, - 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, - 292, 293, -1, -1, 296, 297, 298, 299, 300, 301, + 526, 444, 445, 0, 0, 446, 447, 0, 448, 0, + 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, + 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, + 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, + 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 0, 485, 0, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, 512, 513, 514, 515, 523, + 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, + 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, + 130, 0, 131, 132, 133, 0, 135, 136, 137, 138, + 139, 0, 141, 142, 0, 143, 144, 145, 146, 147, + 148, 0, 0, 149, 150, 151, 152, 153, 154, 155, + 0, 156, 157, 158, 159, 160, 0, 0, 0, 162, + 163, 164, 165, 166, 167, 0, 169, 170, 171, 0, + 172, 173, 174, 175, 176, 177, 0, 0, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, + 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, + 213, 214, 215, 0, 216, 217, 218, 219, 220, 221, + 222, 223, 224, 225, 226, 0, 227, 0, 228, 229, + 230, 231, 0, 232, 0, 233, 0, 0, 0, 236, + 237, 524, 0, 240, 0, 241, 0, 242, 243, 244, + 245, 0, 246, 247, 248, 249, 250, 896, 252, 0, + 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 0, 267, 268, 269, 270, 271, + 272, 273, 274, 0, 275, 0, 276, 0, 0, 279, + 0, 281, 282, 283, 284, 285, 286, 0, 0, 287, + 0, 289, 0, 0, 291, 292, 293, 294, 295, 296, + 297, 298, 525, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 0, 322, 323, 324, 325, 326, + 0, 327, 328, 0, 330, 0, 331, 332, 333, 334, + 335, 336, 0, 337, 338, 0, 791, 339, 340, 341, + 0, 0, 342, 343, 344, 0, 346, 0, 348, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 0, 0, 0, 0, 360, 361, 362, 0, 364, 365, + 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, + 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, + 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 397, 398, 0, 400, 401, + 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, + 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 0, 0, 427, 428, + 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 526, 444, 445, 0, 0, + 446, 447, 0, 448, 0, 450, 451, 452, 453, 454, + 0, 455, 456, 457, 0, 0, 458, 459, 460, 461, + 462, 0, 463, 464, 465, 466, 467, 468, 469, 470, + 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 0, 485, + 0, 487, 488, 489, 490, 491, 492, 493, 0, 0, + 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, + 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, + 512, 513, 514, 515, 523, 0, 549, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 938, 125, 126, 127, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, + 0, 135, 136, 137, 138, 139, 0, 141, 142, 0, + 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, + 151, 152, 153, 154, 155, 0, 156, 157, 158, 159, + 160, 0, 0, 0, 162, 163, 164, 165, 166, 167, + 0, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 0, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, + 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, + 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, + 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + 0, 227, 0, 228, 229, 230, 231, 0, 232, 0, + 233, 0, 0, 0, 236, 237, 524, 0, 240, 0, + 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, + 249, 250, 251, 252, 0, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 0, + 267, 268, 269, 270, 271, 272, 273, 274, 0, 275, + 0, 276, 0, 0, 279, 0, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 0, 289, 0, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 525, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, 314, 315, 316, -1, 318, 319, 320, 321, - 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, - 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, - 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, -1, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, 379, -1, 381, - 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, - 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 0, + 322, 323, 324, 325, 326, 0, 327, 328, 0, 330, + 0, 331, 332, 333, 334, 335, 336, 0, 337, 338, + 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, + 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, + 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, + 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, + 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 526, 444, 445, 0, 0, 446, 447, 0, 448, 0, + 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, + 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, + 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, + 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 0, 485, 0, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, 512, 513, 514, 515, 523, + 1943, 0, 0, 0, 0, 1944, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, + 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, + 130, 0, 131, 132, 133, 0, 135, 136, 137, 138, + 139, 0, 141, 142, 0, 143, 144, 145, 146, 147, + 148, 0, 0, 149, 150, 151, 152, 153, 154, 155, + 0, 156, 157, 158, 159, 160, 0, 0, 0, 162, + 163, 164, 165, 166, 167, 0, 169, 170, 171, 0, + 172, 173, 174, 175, 176, 177, 0, 0, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, + 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, + 213, 214, 215, 0, 216, 217, 218, 219, 220, 221, + 222, 223, 224, 225, 226, 0, 227, 0, 228, 229, + 230, 231, 0, 232, 0, 233, 0, 0, 0, 236, + 237, 524, 0, 240, 0, 241, 0, 242, 243, 244, + 245, 0, 246, 247, 248, 249, 250, 251, 252, 0, + 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 0, 267, 268, 269, 270, 271, + 272, 273, 274, 0, 275, 0, 276, 0, 0, 279, + 0, 281, 282, 283, 284, 285, 286, 0, 0, 287, + 0, 289, 0, 0, 291, 292, 293, 294, 295, 296, + 297, 298, 525, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 0, 322, 323, 324, 325, 326, + 0, 327, 328, 0, 330, 0, 331, 332, 333, 334, + 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, + 0, 0, 342, 343, 344, 0, 346, 0, 348, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 0, 0, 0, 0, 360, 361, 362, 0, 364, 365, + 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, + 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, + 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - 412, -1, 414, 415, 416, 417, 418, 419, 420, 421, - 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, - 442, 443, -1, -1, 446, 447, 448, 449, 450, 451, - 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, - -1, 463, 464, 465, 466, 467, 468, 469, 470, 471, - -1, -1, 474, -1, 476, 477, 478, 479, 480, 481, - 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 498, -1, 500, -1, - -1, -1, -1, 505, 506, 507, -1, -1, -1, -1, - 512, -1, 514, -1, -1, -1, -1, 519, 520, -1, - 522, 523, 3, 4, 5, 6, 7, 8, 9, 10, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 22, 23, 24, 25, 26, 27, 28, 29, -1, - 31, 32, 33, -1, -1, -1, -1, -1, 39, -1, - -1, 42, 43, 44, -1, 46, 47, 48, 49, 50, - 51, 52, 53, 54, 55, 56, 57, -1, 59, 60, - 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, - 71, 72, 73, -1, 75, 76, 77, 78, 79, -1, - 81, -1, 83, 84, 85, 86, 87, 88, 89, 90, - 91, 92, -1, 94, 95, 96, 97, 98, 99, -1, - 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, - 111, 112, 113, 114, 115, 116, 117, 118, -1, 120, - 121, 122, 123, 124, 125, -1, -1, 128, 129, 130, - 131, -1, -1, 134, 135, 136, 137, 138, -1, 140, - 141, 142, -1, 144, 145, 146, -1, 148, 149, 150, - 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, - -1, 162, 163, 164, 165, -1, 167, -1, 169, 170, - 171, 172, 173, 174, 175, 176, 177, -1, 179, -1, - 181, 182, 183, 184, -1, 186, 187, 188, 189, 190, - 191, 192, 193, 194, 195, 196, 197, -1, 199, 200, - 201, 202, 203, 204, 205, -1, 207, 208, 209, 210, - 211, 212, 213, 214, 215, 216, -1, 218, -1, 220, - 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, - -1, -1, 233, 234, 235, 236, -1, 238, 239, 240, - 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, - 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, - 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, - 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, - 281, 282, 283, 284, 285, -1, 287, 288, -1, -1, - 291, 292, 293, -1, -1, 296, 297, 298, 299, 300, - 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, - 311, 312, 313, 314, 315, 316, -1, 318, 319, 320, - 321, 322, 323, 324, 325, 326, 327, -1, 329, 330, - 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, - 341, 342, 343, -1, 345, 346, 347, 348, 349, 350, - 351, 352, 353, 354, 355, 356, 357, -1, 359, 360, - 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, - 371, 372, 373, 374, 375, 376, 377, 378, 379, -1, - 381, 382, 383, 384, 385, 386, 387, 388, 389, -1, - 391, 392, 393, 394, 395, -1, 397, 398, 399, 400, - 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, - 411, -1, -1, 414, 415, 416, 417, 418, 419, 420, - 421, 422, 423, -1, 425, 426, 427, -1, -1, 430, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 443, -1, -1, 446, 447, 448, 449, 450, - 451, 452, 453, -1, 455, 456, 457, 458, 459, 460, - 461, -1, 463, 464, 465, 466, 467, 468, 469, 470, - 471, -1, -1, 474, -1, 476, 477, 478, 479, 480, - 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, - 491, 492, 493, 494, 495, 496, 497, 498, -1, 500, - -1, -1, -1, -1, 505, 506, 507, -1, -1, -1, - -1, 512, -1, 514, -1, -1, -1, -1, 519, 520, - -1, 522, 523, 3, 4, 5, 6, 7, 8, 9, - 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 22, 23, 24, 25, 26, 27, 28, 29, - 30, 31, 32, 33, -1, -1, -1, 37, -1, 39, - -1, -1, 42, 43, 44, -1, 46, 47, 48, 49, - 50, 51, 52, 53, 54, 55, 56, 57, -1, 59, - 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, - 70, 71, 72, 73, -1, 75, 76, 77, 78, 79, - -1, 81, -1, 83, 84, 85, 86, 87, 88, 89, - 90, 91, 92, -1, 94, 95, 96, 97, 98, 99, - -1, 101, 102, 103, 104, 105, 106, 107, 108, 109, - 110, 111, 112, 113, 114, 115, 116, 117, 118, -1, - 120, 121, 122, 123, 124, 125, -1, -1, 128, 129, - 130, 131, 132, -1, 134, 135, 136, 137, 138, -1, - 140, 141, 142, -1, 144, 145, 146, -1, 148, 149, - 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, - 160, -1, 162, 163, 164, 165, -1, 167, -1, 169, - 170, -1, 172, 173, 174, 175, 176, 177, -1, 179, - -1, 181, 182, 183, 184, -1, 186, 187, 188, 189, - 190, 191, 192, 193, 194, 195, 196, 197, -1, 199, - 200, 201, 202, 203, 204, 205, -1, 207, 208, 209, - 210, 211, 212, 213, 214, 215, 216, -1, 218, -1, - 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, - 230, -1, -1, 233, 234, 235, 236, -1, 238, 239, - 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, - 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, - 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, - 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, - 280, 281, 282, 283, 284, 285, -1, 287, 288, -1, - -1, 291, 292, 293, -1, -1, 296, 297, 298, 299, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, -1, -1, -1, -1, 318, 319, - 320, 321, 322, 323, 324, 325, 326, 327, -1, 329, - 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, - 340, 341, 342, 343, -1, 345, 346, 347, 348, 349, - 350, 351, 352, 353, 354, 355, 356, 357, -1, 359, - 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, - 370, 371, 372, 373, 374, 375, 376, 377, 378, -1, - 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, - -1, 391, 392, 393, 394, 395, -1, 397, 398, 399, - 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - 410, 411, -1, -1, 414, 415, -1, 417, 418, 419, - 420, 421, 422, 423, -1, 425, 426, 427, -1, -1, - 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, - 440, 441, 442, 443, -1, -1, 446, 447, 448, -1, - 450, 451, 452, 453, -1, 455, 456, 457, 458, 459, - 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, - 470, 471, -1, -1, 474, -1, -1, 477, 478, 479, - 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, - 490, 491, 492, 493, 494, 495, 496, 497, 498, -1, - -1, -1, -1, -1, -1, 505, 506, 507, -1, -1, - -1, -1, 512, -1, 514, 515, -1, -1, -1, 519, - 520, -1, 522, 523, 3, 4, 5, 6, 7, -1, - 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, - 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, - 39, -1, -1, 42, 43, 44, -1, 46, 47, 48, - 49, 50, 51, 52, 53, 54, 55, 56, 57, -1, - 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, - 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, - 79, -1, 81, -1, 83, 84, 85, 86, 87, 88, - 89, 90, 91, 92, -1, 94, 95, 96, 97, 98, - 99, -1, 101, 102, 103, 104, 105, 106, 107, 108, - 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, - -1, 120, 121, 122, 123, 124, 125, 126, 127, 128, - 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, - -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, - 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, - 159, 160, -1, 162, 163, 164, 165, -1, 167, -1, - 169, 170, 171, 172, 173, 174, 175, 176, 177, -1, - 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, - 189, 190, 191, 192, 193, 194, 195, 196, 197, -1, - 199, 200, 201, 202, 203, 204, 205, -1, 207, 208, - 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, - -1, 220, 221, 222, 223, 224, 225, 226, 227, 228, - 229, 230, -1, -1, 233, 234, 235, 236, -1, 238, - 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, - 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, - 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, - 269, 270, 271, 272, 273, -1, 275, 276, 277, 278, - 279, 280, 281, 282, 283, 284, 285, -1, 287, 288, - -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, - 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, - 309, 310, 311, 312, 313, 314, 315, 316, -1, 318, - 319, 320, 321, 322, 323, 324, 325, 326, 327, -1, - 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, - 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, - 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, - 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, - 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, - 379, -1, 381, 382, 383, 384, 385, 386, 387, 388, - 389, 390, 391, 392, 393, 394, 395, -1, 397, 398, - 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, - 409, 410, 411, 412, -1, 414, 415, 416, 417, 418, - 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, - -1, 430, 431, 432, 433, 434, 435, 436, 437, 438, - 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, - 449, 450, 451, 452, 453, -1, 455, 456, 457, 458, - 459, 460, 461, -1, 463, 464, 465, 466, 467, 468, - 469, 470, 471, -1, -1, 474, -1, 476, 477, 478, - 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, - 489, 490, 491, 492, 493, 494, 495, 496, 497, -1, - -1, 500, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 512, -1, 514, -1, -1, -1, -1, - 519, 520, -1, 522, 523, 3, 4, 5, 6, 7, - 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 22, 23, 24, 25, 26, 27, - 28, 29, -1, 31, 32, 33, -1, -1, -1, -1, - -1, 39, -1, -1, 42, 43, 44, -1, 46, 47, - 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, - 68, 69, 70, 71, 72, 73, -1, 75, 76, 77, - 78, 79, -1, 81, -1, 83, 84, 85, 86, 87, - 88, 89, 90, 91, 92, -1, 94, 95, 96, 97, - 98, 99, -1, 101, 102, 103, 104, 105, 106, 107, - 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, - 118, -1, 120, 121, 122, 123, 124, 125, -1, -1, - 128, 129, 130, 131, -1, -1, 134, 135, 136, 137, - 138, -1, 140, 141, 142, -1, 144, 145, 146, -1, - 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, - 158, 159, 160, -1, 162, 163, 164, 165, -1, 167, - -1, 169, 170, 171, 172, 173, 174, 175, 176, 177, - -1, 179, -1, 181, 182, 183, 184, -1, 186, 187, - 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, - -1, 199, 200, 201, 202, 203, 204, 205, -1, 207, - 208, 209, 210, 211, 212, 213, 214, 215, 216, -1, - 218, -1, 220, 221, 222, 223, 224, 225, 226, 227, - 228, 229, 230, -1, 232, 233, 234, 235, 236, -1, - 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, - 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, - 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, - 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, - 278, 279, 280, 281, 282, 283, 284, 285, -1, 287, - 288, -1, -1, 291, 292, 293, -1, -1, 296, 297, - 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, - 308, 309, 310, 311, 312, 313, -1, -1, -1, -1, - 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, - -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, - 338, 339, 340, 341, 342, 343, -1, 345, 346, 347, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - -1, 359, 360, 361, 362, 363, 364, 365, 366, 367, - 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, - 378, -1, -1, 381, 382, 383, 384, 385, 386, 387, - 388, 389, -1, 391, 392, 393, 394, 395, -1, 397, - 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, - 408, 409, 410, 411, -1, -1, 414, 415, -1, 417, - 418, 419, 420, 421, 422, 423, -1, 425, 426, 427, - -1, 429, 430, 431, 432, 433, 434, 435, 436, 437, - 438, 439, 440, 441, 442, 443, -1, -1, 446, 447, - 448, -1, 450, 451, 452, 453, -1, 455, 456, 457, - 458, 459, 460, 461, -1, 463, 464, 465, 466, 467, - 468, 469, 470, 471, -1, -1, 474, -1, -1, 477, - 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, - 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, - 498, -1, -1, -1, -1, -1, -1, 505, 506, 507, - -1, -1, -1, -1, 512, -1, 514, -1, -1, -1, - -1, 519, 520, -1, 522, 523, 3, 4, 5, 6, - 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - 37, -1, 39, -1, -1, 42, 43, 44, -1, 46, - 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - 57, -1, 59, 60, 61, 62, 63, 64, 65, 66, - 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, 79, -1, 81, -1, 83, 84, 85, 86, - 87, 88, 89, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, 101, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - 117, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, - -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, 159, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, 170, -1, 172, 173, 174, 175, 176, - 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, 221, 222, 223, 224, 225, 226, - 227, 228, 229, 230, -1, -1, 233, 234, 235, 236, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, - 277, 278, 279, 280, 281, 282, 283, 284, 285, -1, - 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, 321, 322, 323, 324, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, 361, 362, 363, 364, 365, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, 380, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, 391, 392, 393, 394, 395, -1, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, - 417, 418, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, 435, 436, - 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, - 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, - 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, - 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 498, -1, -1, -1, -1, -1, -1, 505, 506, - 507, -1, -1, -1, -1, 512, -1, 514, -1, -1, - -1, -1, 519, 520, -1, 522, 523, 3, 4, 5, - 6, 7, 8, 9, 10, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 22, 23, 24, 25, - 26, 27, 28, 29, -1, 31, 32, 33, -1, -1, - -1, 37, -1, 39, -1, -1, 42, 43, 44, -1, - 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, - 56, 57, -1, 59, 60, 61, 62, 63, 64, 65, - 66, 67, 68, 69, 70, 71, 72, 73, -1, 75, - 76, 77, 78, 79, -1, 81, -1, 83, 84, 85, - 86, 87, 88, 89, 90, 91, 92, -1, 94, 95, - 96, 97, 98, 99, -1, 101, 102, 103, 104, 105, - 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, - 116, 117, 118, -1, 120, 121, 122, 123, 124, 125, - -1, -1, 128, 129, 130, 131, -1, -1, 134, 135, - 136, 137, 138, -1, 140, 141, 142, -1, 144, 145, - 146, -1, 148, 149, 150, 151, 152, 153, 154, 155, - 156, 157, 158, 159, 160, -1, 162, 163, 164, 165, - -1, 167, -1, 169, 170, -1, 172, 173, 174, 175, - 176, 177, -1, 179, -1, 181, 182, 183, 184, -1, - 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, - 196, 197, -1, 199, 200, 201, 202, 203, 204, 205, - -1, 207, 208, 209, 210, 211, 212, 213, 214, 215, - 216, -1, 218, -1, 220, 221, 222, 223, 224, 225, - 226, 227, 228, 229, 230, -1, -1, 233, 234, 235, - 236, -1, 238, 239, 240, 241, 242, 243, 244, 245, - 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, - 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, - 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, - 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, - -1, 287, 288, -1, -1, 291, 292, 293, -1, -1, - 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, - 306, 307, 308, 309, 310, 311, 312, 313, -1, -1, - -1, -1, 318, 319, 320, 321, 322, 323, 324, 325, - 326, 327, -1, 329, 330, 331, 332, 333, 334, -1, - 336, 337, 338, 339, 340, 341, 342, 343, -1, 345, - 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, - 356, 357, -1, 359, 360, 361, 362, 363, 364, 365, - 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, - 376, 377, 378, -1, 380, 381, 382, 383, 384, 385, - 386, 387, 388, 389, -1, 391, 392, 393, 394, 395, - -1, 397, 398, 399, 400, 401, 402, 403, 404, 405, - 406, 407, 408, 409, 410, 411, -1, -1, 414, 415, - -1, 417, 418, 419, 420, 421, 422, 423, -1, 425, - 426, 427, -1, -1, 430, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 443, -1, -1, - 446, 447, 448, -1, 450, 451, 452, 453, -1, 455, - 456, 457, 458, 459, 460, 461, -1, 463, 464, 465, - 466, 467, 468, 469, 470, 471, -1, -1, 474, -1, - -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, - 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, - 496, 497, 498, -1, -1, -1, -1, -1, -1, 505, - 506, 507, -1, -1, -1, -1, 512, -1, 514, 515, - -1, -1, -1, 519, 520, -1, 522, 523, 3, 4, - 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 22, 23, 24, - 25, 26, 27, 28, 29, 30, 31, 32, 33, -1, - -1, -1, -1, 38, 39, -1, -1, 42, 43, 44, - -1, 46, 47, 48, 49, 50, 51, 52, 53, 54, - 55, 56, 57, -1, 59, 60, 61, 62, 63, 64, - 65, 66, 67, 68, 69, 70, 71, 72, 73, -1, - 75, 76, 77, 78, 79, -1, 81, -1, 83, 84, - 85, 86, 87, 88, 89, 90, 91, 92, -1, 94, - 95, 96, 97, 98, 99, -1, 101, 102, 103, 104, - 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, - 115, 116, 117, 118, -1, 120, 121, 122, 123, 124, - 125, -1, -1, 128, 129, 130, 131, -1, -1, 134, - 135, 136, 137, 138, -1, 140, 141, 142, -1, 144, - 145, 146, -1, 148, 149, 150, 151, 152, 153, 154, - 155, 156, 157, 158, 159, 160, -1, 162, 163, 164, - 165, -1, 167, -1, 169, 170, -1, 172, 173, 174, - 175, 176, 177, -1, 179, -1, 181, 182, 183, 184, - -1, 186, 187, 188, 189, 190, 191, 192, 193, 194, - 195, 196, 197, -1, 199, 200, 201, 202, 203, 204, - 205, -1, 207, 208, 209, 210, 211, 212, 213, 214, - 215, 216, -1, 218, -1, 220, 221, 222, 223, 224, - 225, 226, 227, 228, 229, 230, -1, -1, 233, 234, - 235, 236, -1, 238, 239, 240, 241, 242, 243, 244, - 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, - 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, - 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, - 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, - 285, -1, 287, 288, -1, -1, 291, 292, 293, -1, - -1, 296, 297, 298, 299, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, -1, - -1, -1, -1, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 327, -1, 329, 330, 331, 332, 333, 334, - -1, 336, 337, 338, 339, 340, 341, 342, 343, -1, - 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, - 355, 356, 357, -1, 359, 360, 361, 362, 363, 364, - 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, - 375, 376, 377, 378, -1, -1, 381, 382, 383, 384, - 385, 386, 387, 388, 389, -1, 391, 392, 393, 394, - 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, -1, -1, 414, - 415, -1, 417, 418, 419, 420, 421, 422, 423, -1, - 425, 426, 427, -1, -1, 430, 431, 432, 433, 434, - 435, 436, 437, 438, 439, 440, 441, 442, 443, -1, - -1, 446, 447, 448, -1, 450, 451, 452, 453, -1, - 455, 456, 457, 458, 459, 460, 461, -1, 463, 464, - 465, 466, 467, 468, 469, 470, 471, -1, -1, 474, - -1, -1, 477, 478, 479, 480, 481, 482, 483, 484, - 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, - 495, 496, 497, 498, -1, -1, -1, -1, -1, -1, - 505, 506, 507, -1, -1, -1, -1, 512, -1, 514, - -1, -1, -1, -1, 519, 520, -1, 522, 523, 3, - 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, - 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, - -1, -1, -1, 37, -1, 39, -1, -1, 42, 43, - 44, -1, 46, 47, 48, 49, 50, 51, 52, 53, - 54, 55, 56, 57, -1, 59, 60, 61, 62, 63, - 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, - -1, 75, 76, 77, 78, 79, -1, 81, -1, 83, - 84, 85, 86, 87, 88, 89, 90, 91, 92, -1, - 94, 95, 96, 97, 98, 99, -1, 101, 102, 103, - 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, - 114, 115, 116, 117, 118, -1, 120, 121, 122, 123, - 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, - 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, - 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, - 154, 155, 156, 157, 158, 159, 160, -1, 162, 163, - 164, 165, -1, 167, -1, 169, 170, -1, 172, 173, - 174, 175, 176, 177, -1, 179, -1, 181, 182, 183, - 184, -1, 186, 187, 188, 189, 190, 191, 192, 193, - 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, - 204, 205, -1, 207, 208, 209, 210, 211, 212, 213, - 214, 215, 216, -1, 218, -1, 220, 221, 222, 223, - 224, 225, 226, 227, 228, 229, 230, -1, -1, 233, - 234, 235, 236, -1, 238, 239, 240, 241, 242, 243, - 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, - 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, - 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, - 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, - 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, - -1, -1, 296, 297, 298, 299, 300, 301, 302, 303, - 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, - -1, -1, -1, -1, 318, 319, 320, 321, 322, 323, - 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, - 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, - -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, - 354, 355, 356, 357, -1, 359, 360, 361, 362, 363, - 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, - 374, 375, 376, 377, 378, -1, 380, 381, 382, 383, - 384, 385, 386, 387, 388, 389, -1, 391, 392, 393, - 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, - 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, - 414, 415, -1, 417, 418, 419, 420, 421, 422, 423, - -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, - 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, - -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, - -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, - 464, 465, 466, 467, 468, 469, 470, 471, -1, -1, - 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, - 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, - 494, 495, 496, 497, 498, -1, -1, -1, -1, -1, - -1, 505, 506, 507, -1, -1, -1, -1, 512, -1, - 514, 515, -1, -1, -1, 519, 520, -1, 522, 523, - 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 22, - 23, 24, 25, 26, 27, 28, 29, -1, 31, 32, - 33, -1, -1, -1, 37, -1, 39, -1, -1, 42, - 43, 44, -1, 46, 47, 48, 49, 50, 51, 52, - 53, 54, 55, 56, 57, -1, 59, 60, 61, 62, - 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, - 73, -1, 75, 76, 77, 78, 79, -1, 81, -1, - 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, - -1, 94, 95, 96, 97, 98, 99, -1, 101, 102, - 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, - 113, 114, 115, 116, 117, 118, -1, 120, 121, 122, - 123, 124, 125, -1, -1, 128, 129, 130, 131, -1, - -1, 134, 135, 136, 137, 138, -1, 140, 141, 142, - -1, 144, 145, 146, -1, 148, 149, 150, 151, 152, - 153, 154, 155, 156, 157, 158, 159, 160, -1, 162, - 163, 164, 165, -1, 167, -1, 169, 170, -1, 172, - 173, 174, 175, 176, 177, -1, 179, -1, 181, 182, - 183, 184, -1, 186, 187, 188, 189, 190, 191, 192, - 193, 194, 195, 196, 197, -1, 199, 200, 201, 202, - 203, 204, 205, -1, 207, 208, 209, 210, 211, 212, - 213, 214, 215, 216, -1, 218, -1, 220, 221, 222, - 223, 224, 225, 226, 227, 228, 229, 230, -1, -1, - 233, 234, 235, 236, -1, 238, 239, 240, 241, 242, - 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, - 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, - 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, - 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, - 283, 284, 285, -1, 287, 288, -1, -1, 291, 292, - 293, -1, -1, 296, 297, 298, 299, 300, 301, 302, - 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, - 313, -1, -1, -1, -1, 318, 319, 320, 321, 322, - 323, 324, 325, 326, 327, -1, 329, 330, 331, 332, - 333, 334, -1, 336, 337, 338, 339, 340, 341, 342, - 343, -1, 345, 346, 347, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, -1, 359, 360, 361, 362, - 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, - 373, 374, 375, 376, 377, 378, -1, 380, 381, 382, - 383, 384, 385, 386, 387, 388, 389, -1, 391, 392, - 393, 394, 395, -1, 397, 398, 399, 400, 401, 402, - 403, 404, 405, 406, 407, 408, 409, 410, 411, -1, - -1, 414, 415, -1, 417, 418, 419, 420, 421, 422, - 423, -1, 425, 426, 427, -1, -1, 430, 431, 432, - 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, - 443, -1, -1, 446, 447, 448, -1, 450, 451, 452, - 453, -1, 455, 456, 457, 458, 459, 460, 461, -1, - 463, 464, 465, 466, 467, 468, 469, 470, 471, -1, - -1, 474, -1, -1, 477, 478, 479, 480, 481, 482, - 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, - 493, 494, 495, 496, 497, 498, -1, -1, -1, -1, - -1, -1, 505, 506, 507, -1, -1, -1, -1, 512, - -1, 514, -1, -1, -1, -1, 519, 520, -1, 522, - 523, 3, 4, 5, 6, 7, 8, 9, 10, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, -1, -1, -1, -1, -1, 39, -1, -1, - 42, 43, 44, -1, 46, 47, 48, 49, 50, 51, - 52, 53, 54, 55, 56, 57, -1, 59, 60, 61, - 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, - 72, 73, -1, 75, 76, 77, 78, 79, -1, 81, - -1, 83, 84, 85, 86, 87, 88, 89, 90, 91, - 92, -1, 94, 95, 96, 97, 98, 99, -1, 101, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, 117, 118, -1, 120, 121, - 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, - -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, - 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, 159, 160, -1, - 162, 163, 164, 165, -1, 167, -1, 169, 170, -1, - 172, 173, 174, 175, 176, 177, -1, 179, -1, 181, - 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, - 192, 193, 194, 195, 196, 197, -1, 199, 200, 201, - 202, 203, 204, 205, -1, 207, 208, 209, 210, 211, - 212, 213, 214, 215, 216, -1, 218, -1, 220, 221, - 222, 223, 224, 225, 226, 227, 228, 229, 230, -1, - -1, 233, 234, 235, 236, -1, 238, 239, 240, 241, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, - 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, - 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, - 292, 293, -1, -1, 296, 297, 298, 299, 300, 301, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, -1, -1, -1, -1, 318, 319, 320, 321, - 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, - 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, - 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, -1, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, -1, -1, 381, - 382, 383, 384, 385, 386, 387, 388, 389, -1, 391, - 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - -1, -1, 414, 415, -1, 417, 418, 419, 420, 421, - 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, - 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, - 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, - -1, 463, 464, 465, 466, 467, 468, 469, 470, 471, - -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, - 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 498, -1, -1, -1, - -1, -1, -1, 505, 506, 507, -1, -1, -1, -1, - 512, -1, 514, -1, -1, -1, -1, 519, 520, -1, - 522, 523, 3, 4, 5, 6, 7, 8, 9, 10, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 22, 23, 24, 25, 26, 27, 28, 29, 30, - 31, 32, 33, -1, -1, -1, -1, -1, 39, -1, - -1, 42, 43, 44, -1, 46, 47, 48, 49, 50, - 51, 52, 53, 54, 55, 56, 57, -1, 59, 60, - 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, - 71, 72, 73, -1, 75, 76, 77, 78, 79, -1, - 81, -1, 83, 84, 85, 86, 87, 88, 89, 90, - 91, 92, -1, 94, 95, 96, 97, 98, 99, -1, - 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, - 111, 112, 113, 114, 115, 116, 117, 118, -1, 120, - 121, 122, 123, 124, 125, -1, -1, 128, 129, 130, - 131, -1, -1, 134, 135, 136, 137, 138, -1, 140, - 141, 142, -1, 144, 145, 146, -1, 148, 149, 150, - 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, - -1, 162, 163, 164, 165, -1, 167, -1, 169, 170, - -1, 172, 173, 174, 175, 176, 177, -1, 179, -1, - 181, 182, 183, 184, -1, 186, 187, 188, 189, 190, - 191, 192, 193, 194, 195, 196, 197, -1, 199, 200, - 201, 202, 203, 204, 205, -1, 207, 208, 209, 210, - 211, 212, 213, 214, 215, 216, -1, 218, -1, 220, - 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, - -1, -1, 233, 234, 235, 236, -1, 238, 239, 240, - 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, - 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, - 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, - 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, - 281, 282, 283, 284, 285, -1, 287, 288, -1, -1, - 291, 292, 293, -1, -1, 296, 297, 298, 299, 300, - 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, - 311, 312, 313, -1, -1, -1, -1, 318, 319, 320, - 321, 322, 323, 324, 325, 326, 327, -1, 329, 330, - 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, - 341, 342, 343, -1, 345, 346, 347, 348, 349, 350, - 351, 352, 353, 354, 355, 356, 357, -1, 359, 360, - 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, - 371, 372, 373, 374, 375, 376, 377, 378, -1, -1, - 381, 382, 383, 384, 385, 386, 387, 388, 389, -1, - 391, 392, 393, 394, 395, -1, 397, 398, 399, 400, - 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, - 411, -1, -1, 414, 415, -1, 417, 418, 419, 420, - 421, 422, 423, -1, 425, 426, 427, -1, -1, 430, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 443, -1, -1, 446, 447, 448, -1, 450, - 451, 452, 453, -1, 455, 456, 457, 458, 459, 460, - 461, -1, 463, 464, 465, 466, 467, 468, 469, 470, - 471, -1, -1, 474, -1, -1, 477, 478, 479, 480, - 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, - 491, 492, 493, 494, 495, 496, 497, 498, -1, -1, - -1, -1, -1, -1, 505, 506, 507, -1, -1, -1, - -1, 512, -1, 514, -1, -1, -1, -1, 519, 520, - -1, 522, 523, 3, 4, 5, 6, 7, 8, 9, - 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 22, 23, 24, 25, 26, 27, 28, 29, - -1, 31, 32, 33, -1, -1, -1, -1, -1, 39, - -1, -1, 42, 43, 44, -1, 46, 47, 48, 49, - 50, 51, 52, 53, 54, 55, 56, 57, -1, 59, - 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, - 70, 71, 72, 73, -1, 75, 76, 77, 78, 79, - -1, 81, -1, 83, 84, 85, 86, 87, 88, 89, - 90, 91, 92, -1, 94, 95, 96, 97, 98, 99, - -1, 101, 102, 103, 104, 105, 106, 107, 108, 109, - 110, 111, 112, 113, 114, 115, 116, 117, 118, -1, - 120, 121, 122, 123, 124, 125, -1, -1, 128, 129, - 130, 131, -1, -1, 134, 135, 136, 137, 138, -1, - 140, 141, 142, -1, 144, 145, 146, -1, 148, 149, - 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, - 160, -1, 162, 163, 164, 165, -1, 167, -1, 169, - 170, 171, 172, 173, 174, 175, 176, 177, -1, 179, - -1, 181, 182, 183, 184, -1, 186, 187, 188, 189, - 190, 191, 192, 193, 194, 195, 196, 197, -1, 199, - 200, 201, 202, 203, 204, 205, -1, 207, 208, 209, - 210, 211, 212, 213, 214, 215, 216, -1, 218, -1, - 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, - 230, -1, -1, 233, 234, 235, 236, -1, 238, 239, - 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, - 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, - 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, - 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, - 280, 281, 282, 283, 284, 285, -1, 287, 288, -1, - -1, 291, 292, 293, -1, -1, 296, 297, 298, 299, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, -1, -1, -1, -1, 318, 319, - 320, 321, 322, 323, 324, 325, 326, 327, -1, 329, - 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, - 340, 341, 342, 343, -1, 345, 346, 347, 348, 349, - 350, 351, 352, 353, 354, 355, 356, 357, -1, 359, - 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, - 370, 371, 372, 373, 374, 375, 376, 377, 378, -1, - -1, 381, 382, 383, 384, 385, 386, 387, 388, 389, - -1, 391, 392, 393, 394, 395, -1, 397, 398, 399, - 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - 410, 411, -1, -1, 414, 415, -1, 417, 418, 419, - 420, 421, 422, 423, -1, 425, 426, 427, -1, -1, - 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, - 440, 441, 442, 443, -1, -1, 446, 447, 448, -1, - 450, 451, 452, 453, -1, 455, 456, 457, 458, 459, - 460, 461, -1, 463, 464, 465, 466, 467, 468, 469, - 470, 471, -1, -1, 474, -1, -1, 477, 478, 479, - 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, - 490, 491, 492, 493, 494, 495, 496, 497, 498, -1, - -1, -1, -1, -1, -1, 505, 506, 507, -1, -1, - -1, -1, 512, -1, 514, -1, -1, -1, -1, 519, - 520, -1, 522, 523, 3, 4, 5, 6, 7, 8, - 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, - 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, - 39, -1, -1, 42, 43, 44, -1, 46, 47, 48, - 49, 50, 51, 52, 53, 54, 55, 56, 57, -1, - 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, - 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, - 79, -1, 81, -1, 83, 84, 85, 86, 87, 88, - 89, 90, 91, 92, -1, 94, 95, 96, 97, 98, - 99, -1, 101, 102, 103, 104, 105, 106, 107, 108, - 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, - -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, - 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, - -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, - 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, - 159, 160, -1, 162, 163, 164, 165, -1, 167, -1, - 169, 170, -1, 172, 173, 174, 175, 176, 177, -1, - 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, - 189, 190, 191, 192, 193, 194, 195, 196, 197, -1, - 199, 200, 201, 202, 203, 204, 205, -1, 207, 208, - 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, - -1, 220, 221, 222, 223, 224, 225, 226, 227, 228, - 229, 230, -1, -1, 233, 234, 235, 236, -1, 238, - 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, - 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, - 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, - 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, - 279, 280, 281, 282, 283, 284, 285, -1, 287, 288, - -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, - 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, - 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, - 319, 320, 321, 322, 323, 324, 325, 326, 327, -1, - 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, - 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, - 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, - 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, - 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, - -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, - 389, -1, 391, 392, 393, 394, 395, -1, 397, 398, - 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, - 409, 410, 411, -1, -1, 414, 415, -1, 417, 418, - 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, - -1, 430, 431, 432, 433, 434, 435, 436, 437, 438, - 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, - -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, - 459, 460, 461, -1, 463, 464, 465, 466, 467, 468, - 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, - 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, - 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, - -1, -1, -1, -1, -1, -1, 505, 506, 507, -1, - -1, -1, -1, 512, -1, 514, 515, -1, -1, -1, - 519, 520, -1, 522, 523, 3, 4, 5, 6, 7, - 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 22, 23, 24, 25, 26, 27, - 28, 29, -1, 31, 32, 33, -1, -1, -1, -1, - -1, 39, -1, -1, 42, 43, 44, -1, 46, 47, - 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - -1, 59, 60, 61, 62, 63, 64, 65, 66, 67, - 68, 69, 70, 71, 72, 73, -1, 75, 76, 77, - 78, 79, -1, 81, -1, 83, 84, 85, 86, 87, - 88, 89, 90, 91, 92, -1, 94, 95, 96, 97, - 98, 99, -1, 101, 102, 103, 104, 105, 106, 107, - 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, - 118, -1, 120, 121, 122, 123, 124, 125, -1, -1, - 128, 129, 130, 131, -1, -1, 134, 135, 136, 137, - 138, -1, 140, 141, 142, -1, 144, 145, 146, -1, - 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, - 158, 159, 160, -1, 162, 163, 164, 165, -1, 167, - -1, 169, 170, -1, 172, 173, 174, 175, 176, 177, - -1, 179, -1, 181, 182, 183, 184, -1, 186, 187, - 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, - -1, 199, 200, 201, 202, 203, 204, 205, -1, 207, - 208, 209, 210, 211, 212, 213, 214, 215, 216, -1, - 218, -1, 220, 221, 222, 223, 224, 225, 226, 227, - 228, 229, 230, -1, -1, 233, 234, 235, 236, -1, - 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, - 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, - 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, - 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, - 278, 279, 280, 281, 282, 283, 284, 285, -1, 287, - 288, -1, -1, 291, 292, 293, -1, -1, 296, 297, - 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, - 308, 309, 310, 311, 312, 313, -1, -1, -1, -1, - 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, - -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, - 338, 339, 340, 341, 342, 343, -1, 345, 346, 347, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - -1, 359, 360, 361, 362, 363, 364, 365, 366, 367, - 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, - 378, -1, -1, 381, 382, 383, 384, 385, 386, 387, - 388, 389, -1, 391, 392, 393, 394, 395, -1, 397, - 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, - 408, 409, 410, 411, -1, -1, 414, 415, 416, 417, - 418, 419, 420, 421, 422, 423, -1, 425, 426, 427, - -1, -1, 430, 431, 432, 433, 434, 435, 436, 437, - 438, 439, 440, 441, 442, 443, -1, -1, 446, 447, - 448, -1, 450, 451, 452, 453, -1, 455, 456, 457, - 458, 459, 460, 461, -1, 463, 464, 465, 466, 467, - 468, 469, 470, 471, -1, -1, 474, -1, -1, 477, - 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, - 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, - 498, -1, -1, -1, -1, -1, -1, 505, 506, 507, - -1, -1, -1, -1, 512, -1, 514, -1, -1, -1, - -1, 519, 520, -1, 522, 523, 3, 4, 5, 6, - 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - -1, -1, 39, -1, -1, 42, 43, 44, -1, 46, - 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - 57, -1, 59, 60, 61, 62, 63, 64, 65, 66, - 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, 79, -1, 81, -1, 83, 84, 85, 86, - 87, 88, 89, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, 101, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - 117, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, - -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, 159, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, 170, -1, 172, 173, 174, 175, 176, - 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, 221, 222, 223, 224, 225, 226, - 227, 228, 229, 230, -1, -1, 233, 234, 235, 236, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, - 277, 278, 279, 280, 281, 282, 283, 284, 285, -1, - 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, 321, 322, 323, 324, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, 361, 362, 363, 364, 365, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, -1, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, 391, 392, 393, 394, 395, -1, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, - 417, 418, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, 435, 436, - 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, -1, 463, 464, 465, 466, - 467, 468, 469, 470, 471, -1, 473, 474, -1, -1, - 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, - 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 498, -1, -1, -1, -1, -1, -1, 505, 506, - 507, -1, -1, -1, -1, 512, -1, 514, -1, -1, - -1, -1, 519, 520, -1, 522, 523, 3, 4, 5, - 6, 7, 8, 9, 10, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 22, 23, 24, 25, - 26, 27, 28, 29, 30, 31, 32, 33, -1, -1, - -1, -1, -1, 39, -1, -1, 42, 43, 44, -1, - 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, - 56, 57, -1, 59, 60, 61, 62, 63, 64, 65, - 66, 67, 68, 69, 70, 71, 72, 73, -1, 75, - 76, 77, 78, 79, -1, 81, -1, 83, 84, 85, - 86, 87, 88, 89, 90, 91, 92, -1, 94, 95, - 96, 97, 98, 99, -1, 101, 102, 103, 104, 105, - 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, - 116, 117, 118, -1, 120, 121, 122, 123, 124, 125, - -1, -1, 128, 129, 130, 131, -1, -1, 134, 135, - 136, 137, 138, -1, 140, 141, 142, -1, 144, 145, - 146, -1, 148, 149, 150, 151, 152, 153, 154, 155, - 156, 157, 158, 159, 160, -1, 162, 163, 164, 165, - -1, 167, -1, 169, 170, -1, 172, 173, 174, 175, - 176, 177, -1, 179, -1, 181, 182, 183, 184, -1, - 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, - 196, 197, -1, 199, 200, 201, 202, 203, 204, 205, - -1, 207, 208, 209, 210, 211, 212, 213, 214, 215, - 216, -1, 218, -1, 220, 221, 222, 223, 224, 225, - 226, 227, 228, 229, 230, -1, -1, 233, 234, 235, - 236, -1, 238, 239, 240, 241, 242, 243, 244, 245, - 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, - 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, - 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, - 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, - -1, 287, 288, -1, -1, 291, 292, 293, -1, -1, - 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, - 306, 307, 308, 309, 310, 311, 312, 313, -1, -1, - -1, -1, 318, 319, 320, 321, 322, 323, 324, 325, - 326, 327, -1, 329, 330, 331, 332, 333, 334, -1, - 336, 337, 338, 339, 340, 341, 342, 343, -1, 345, - 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, - 356, 357, -1, 359, 360, 361, 362, 363, 364, 365, - 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, - 376, 377, 378, -1, -1, 381, 382, 383, 384, 385, - 386, 387, 388, 389, -1, 391, 392, 393, 394, 395, - -1, 397, 398, 399, 400, 401, 402, 403, 404, 405, - 406, 407, 408, 409, 410, 411, -1, -1, 414, 415, - -1, 417, 418, 419, 420, 421, 422, 423, -1, 425, - 426, 427, -1, -1, 430, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 443, -1, -1, - 446, 447, 448, -1, 450, 451, 452, 453, -1, 455, - 456, 457, 458, 459, 460, 461, -1, 463, 464, 465, - 466, 467, 468, 469, 470, 471, -1, -1, 474, -1, - -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, - 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, - 496, 497, 498, -1, -1, -1, -1, -1, -1, 505, - 506, 507, -1, -1, -1, -1, 512, -1, 514, -1, - -1, -1, -1, 519, 520, -1, 522, 523, 3, 4, - 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 22, 23, 24, - 25, 26, 27, 28, 29, -1, 31, 32, 33, -1, - -1, -1, -1, -1, 39, -1, -1, 42, 43, 44, - -1, 46, 47, 48, 49, 50, 51, 52, 53, 54, - 55, 56, 57, -1, 59, 60, 61, 62, 63, 64, - 65, 66, 67, 68, 69, 70, 71, 72, 73, -1, - 75, 76, 77, 78, 79, -1, 81, -1, 83, 84, - 85, 86, 87, 88, 89, 90, 91, 92, -1, 94, - 95, 96, 97, 98, 99, -1, 101, 102, 103, 104, - 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, - 115, 116, 117, 118, -1, 120, 121, 122, 123, 124, - 125, -1, -1, 128, 129, 130, 131, -1, -1, 134, - 135, 136, 137, 138, -1, 140, 141, 142, -1, 144, - 145, 146, -1, 148, 149, 150, 151, 152, 153, 154, - 155, 156, 157, 158, 159, 160, -1, 162, 163, 164, - 165, -1, 167, -1, 169, 170, -1, 172, 173, 174, - 175, 176, 177, -1, 179, -1, 181, 182, 183, 184, - -1, 186, 187, 188, 189, 190, 191, 192, 193, 194, - 195, 196, 197, -1, 199, 200, 201, 202, 203, 204, - 205, -1, 207, 208, 209, 210, 211, 212, 213, 214, - 215, 216, -1, 218, -1, 220, 221, 222, 223, 224, - 225, 226, 227, 228, 229, 230, -1, -1, 233, 234, - 235, 236, -1, 238, 239, 240, 241, 242, 243, 244, - 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, - 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, - 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, - 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, - 285, -1, 287, 288, -1, -1, 291, 292, 293, -1, - -1, 296, 297, 298, 299, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, -1, - -1, -1, -1, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 327, -1, 329, 330, 331, 332, 333, 334, - -1, 336, 337, 338, 339, 340, 341, 342, 343, -1, - 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, - 355, 356, 357, -1, 359, 360, 361, 362, 363, 364, - 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, - 375, 376, 377, 378, -1, -1, 381, 382, 383, 384, - 385, 386, 387, 388, 389, -1, 391, 392, 393, 394, - 395, -1, 397, 398, 399, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, -1, -1, 414, - 415, 416, 417, 418, 419, 420, 421, 422, 423, -1, - 425, 426, 427, -1, -1, 430, 431, 432, 433, 434, - 435, 436, 437, 438, 439, 440, 441, 442, 443, -1, - -1, 446, 447, 448, -1, 450, 451, 452, 453, -1, - 455, 456, 457, 458, 459, 460, 461, -1, 463, 464, - 465, 466, 467, 468, 469, 470, 471, -1, -1, 474, - -1, -1, 477, 478, 479, 480, 481, 482, 483, 484, - 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, - 495, 496, 497, 498, -1, -1, -1, -1, -1, -1, - 505, 506, 507, -1, -1, -1, -1, 512, -1, 514, - -1, -1, -1, -1, 519, 520, -1, 522, 523, 3, - 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, - 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, - -1, -1, -1, -1, -1, 39, -1, -1, 42, 43, - 44, -1, 46, 47, 48, 49, 50, 51, 52, 53, - 54, 55, 56, 57, -1, 59, 60, 61, 62, 63, - 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, - -1, 75, 76, 77, 78, 79, -1, 81, -1, 83, - 84, 85, 86, 87, 88, 89, 90, 91, 92, -1, - 94, 95, 96, 97, 98, 99, -1, 101, 102, 103, - 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, - 114, 115, 116, 117, 118, -1, 120, 121, 122, 123, - 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, - 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, - 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, - 154, 155, 156, 157, 158, 159, 160, -1, 162, 163, - 164, 165, -1, 167, -1, 169, 170, -1, 172, 173, - 174, 175, 176, 177, -1, 179, -1, 181, 182, 183, - 184, -1, 186, 187, 188, 189, 190, 191, 192, 193, - 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, - 204, 205, -1, 207, 208, 209, 210, 211, 212, 213, - 214, 215, 216, -1, 218, -1, 220, 221, 222, 223, - 224, 225, 226, 227, 228, 229, 230, -1, -1, 233, - 234, 235, 236, -1, 238, 239, 240, 241, 242, 243, - 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, - 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, - 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, - 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, - 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, - -1, -1, 296, 297, 298, 299, 300, 301, 302, 303, - 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, - -1, -1, -1, -1, 318, 319, 320, 321, 322, 323, - 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, - 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, - -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, - 354, 355, 356, 357, -1, 359, 360, 361, 362, 363, - 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, - 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, - 384, 385, 386, 387, 388, 389, -1, 391, 392, 393, - 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, - 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, - 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, - -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, - 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, - -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, - -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, - 464, 465, 466, 467, 468, 469, 470, 471, -1, -1, - 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, - 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, - 494, 495, 496, 497, 498, -1, -1, -1, -1, -1, - -1, 505, 506, 507, -1, -1, -1, -1, 512, -1, - 514, -1, -1, -1, -1, 519, 520, -1, 522, 523, - 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 22, - 23, 24, 25, 26, 27, 28, 29, -1, 31, 32, - 33, -1, -1, -1, -1, -1, 39, -1, -1, 42, - 43, 44, -1, 46, 47, 48, 49, 50, 51, 52, - 53, 54, 55, 56, 57, -1, 59, 60, 61, 62, - 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, - 73, -1, 75, 76, 77, 78, 79, -1, 81, -1, - 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, - -1, 94, 95, 96, 97, 98, 99, -1, 101, 102, - 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, - 113, 114, 115, 116, 117, 118, -1, 120, 121, 122, - 123, 124, 125, -1, -1, 128, 129, 130, 131, -1, - -1, 134, 135, 136, 137, 138, -1, 140, 141, 142, - -1, 144, 145, 146, -1, 148, 149, 150, 151, 152, - 153, 154, 155, 156, 157, 158, 159, 160, -1, 162, - 163, 164, 165, -1, 167, -1, 169, 170, -1, 172, - 173, 174, 175, 176, 177, -1, 179, -1, 181, 182, - 183, 184, -1, 186, 187, 188, 189, 190, 191, 192, - 193, 194, 195, 196, 197, -1, 199, 200, 201, 202, - 203, 204, 205, -1, 207, 208, 209, 210, 211, 212, - 213, 214, 215, 216, -1, 218, -1, 220, 221, 222, - 223, 224, 225, 226, 227, 228, 229, 230, -1, -1, - 233, 234, 235, 236, -1, 238, 239, 240, 241, 242, - 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, - 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, - 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, - 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, - 283, 284, 285, -1, 287, 288, -1, -1, 291, 292, - 293, -1, -1, 296, 297, 298, 299, 300, 301, 302, - 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, - 313, -1, -1, -1, -1, 318, 319, 320, 321, 322, - 323, 324, 325, 326, 327, -1, 329, 330, 331, 332, - 333, 334, -1, 336, 337, 338, 339, 340, 341, 342, - 343, -1, 345, 346, 347, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, -1, 359, 360, 361, 362, - 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, - 373, 374, 375, 376, 377, 378, -1, -1, 381, 382, - 383, 384, 385, 386, 387, 388, 389, -1, 391, 392, - 393, 394, 395, -1, 397, 398, 399, 400, 401, 402, - 403, 404, 405, 406, 407, 408, 409, 410, 411, -1, - -1, 414, 415, -1, 417, 418, 419, 420, 421, 422, - 423, -1, 425, 426, 427, -1, -1, 430, 431, 432, - 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, - 443, -1, -1, 446, 447, 448, -1, 450, 451, 452, - 453, -1, 455, 456, 457, 458, 459, 460, 461, -1, - 463, 464, 465, 466, 467, 468, 469, 470, 471, -1, - -1, 474, -1, -1, 477, 478, 479, 480, 481, 482, - 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, - 493, 494, 495, 496, 497, 498, -1, -1, -1, -1, - -1, -1, 505, 506, 507, -1, -1, -1, -1, 512, - -1, 514, -1, -1, -1, -1, 519, 520, -1, 522, - 523, 3, 4, 5, 6, 7, 8, 9, 10, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, - 32, 33, -1, -1, -1, -1, -1, 39, -1, -1, - 42, 43, 44, -1, 46, 47, 48, 49, 50, 51, - 52, 53, 54, 55, 56, 57, -1, 59, 60, 61, - 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, - 72, 73, -1, 75, 76, 77, 78, 79, -1, 81, - -1, 83, 84, 85, 86, 87, 88, 89, 90, 91, - 92, -1, 94, 95, 96, 97, 98, 99, -1, 101, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, 117, 118, -1, 120, 121, - 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, - -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, - 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, 159, 160, -1, - 162, 163, 164, 165, -1, 167, -1, 169, 170, -1, - 172, 173, 174, 175, 176, 177, -1, 179, -1, 181, - 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, - 192, 193, 194, 195, 196, 197, -1, 199, 200, 201, - 202, 203, 204, 205, -1, 207, 208, 209, 210, 211, - 212, 213, 214, 215, 216, -1, 218, -1, 220, 221, - 222, 223, 224, 225, 226, 227, 228, 229, 230, -1, - -1, 233, 234, 235, 236, -1, 238, 239, 240, 241, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, - 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, - 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, - 292, 293, -1, -1, 296, 297, 298, 299, 300, 301, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, -1, -1, -1, -1, 318, 319, 320, 321, - 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, - 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, - 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, -1, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, -1, -1, 381, - 382, 383, 384, 385, 386, 387, 388, 389, -1, 391, - 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - -1, -1, 414, 415, -1, 417, 418, 419, 420, 421, - 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, - 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, - 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, - -1, 463, 464, 465, 466, 467, 468, 469, 470, 471, - -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, - 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 498, -1, -1, -1, - -1, -1, -1, 505, 506, 507, -1, -1, -1, -1, - 512, -1, 514, -1, -1, -1, -1, 519, 520, -1, - 522, 523, 3, 4, 5, 6, 7, 8, 9, 10, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 22, 23, 24, 25, 26, 27, 28, 29, -1, - 31, 32, 33, -1, -1, -1, -1, -1, 39, -1, - -1, 42, 43, 44, -1, 46, 47, 48, 49, 50, - 51, 52, 53, 54, 55, 56, 57, -1, 59, 60, - 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, - 71, 72, 73, -1, 75, 76, 77, 78, 79, -1, - 81, -1, 83, 84, 85, 86, 87, 88, 89, 90, - 91, 92, -1, 94, 95, 96, 97, 98, 99, -1, - 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, - 111, 112, 113, 114, 115, 116, 117, 118, -1, 120, - 121, 122, 123, 124, 125, -1, -1, 128, 129, 130, - 131, -1, -1, 134, 135, 136, 137, 138, -1, 140, - 141, 142, -1, 144, 145, 146, -1, 148, 149, 150, - 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, - -1, 162, 163, 164, 165, -1, 167, -1, 169, 170, - -1, 172, 173, 174, 175, 176, 177, -1, 179, -1, - 181, 182, 183, 184, -1, 186, 187, 188, 189, 190, - 191, 192, 193, 194, 195, 196, 197, -1, 199, 200, - 201, 202, 203, 204, 205, -1, 207, 208, 209, 210, - 211, 212, 213, 214, 215, 216, -1, 218, -1, 220, - 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, - -1, -1, 233, 234, 235, 236, -1, 238, 239, 240, - 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, - 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, - 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, - 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, - 281, 282, 283, 284, 285, -1, 287, 288, -1, -1, - 291, 292, 293, -1, -1, 296, 297, 298, 299, 300, - 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, - 311, 312, 313, -1, -1, -1, -1, 318, 319, 320, - 321, 322, 323, 324, 325, 326, 327, -1, 329, 330, - 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, - 341, 342, 343, -1, 345, 346, 347, 348, 349, 350, - 351, 352, 353, 354, 355, 356, 357, -1, 359, 360, - 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, - 371, 372, 373, 374, 375, 376, 377, 378, -1, -1, - 381, 382, 383, 384, 385, 386, 387, 388, 389, -1, - 391, 392, 393, 394, 395, -1, 397, 398, 399, 400, - 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, - 411, -1, -1, 414, 415, -1, 417, 418, 419, 420, - 421, 422, 423, -1, 425, 426, 427, -1, -1, 430, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 443, -1, -1, 446, 447, 448, -1, 450, - 451, 452, 453, -1, 455, 456, 457, 458, 459, 460, - 461, -1, 463, 464, 465, 466, 467, 468, 469, 470, - 471, -1, -1, 474, -1, -1, 477, 478, 479, 480, - 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, - 491, 492, 493, 494, 495, 496, 497, 498, -1, -1, - -1, -1, -1, -1, 505, 506, 507, -1, -1, -1, - -1, 512, -1, 514, -1, -1, -1, -1, 519, 520, - -1, 522, 523, 3, 4, 5, 6, 7, 8, 9, - 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 22, 23, 24, 25, 26, 27, 28, 29, - -1, 31, 32, 33, -1, -1, -1, -1, -1, 39, - -1, -1, 42, 43, 44, -1, 46, 47, 48, 49, - 50, 51, 52, 53, 54, 55, 56, 57, -1, 59, - 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, - 70, 71, 72, 73, -1, 75, 76, 77, 78, 79, - -1, 81, -1, 83, 84, 85, 86, 87, 88, 89, - 90, 91, 92, -1, 94, 95, 96, 97, 98, 99, - -1, 101, 102, 103, 104, 105, 106, 107, 108, 109, - 110, 111, 112, 113, 114, 115, 116, 117, 118, -1, - 120, 121, 122, 123, 124, 125, -1, -1, 128, 129, - 130, 131, -1, -1, 134, 135, 136, 137, 138, -1, - 140, 141, 142, -1, 144, 145, 146, -1, 148, 149, - 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, - 160, -1, 162, 163, 164, 165, -1, 167, -1, 169, - 170, -1, 172, 173, 174, 175, 176, 177, -1, 179, - -1, 181, 182, 183, 184, -1, 186, 187, 188, 189, - 190, 191, 192, 193, 194, 195, 196, 197, -1, 199, - 200, 201, 202, 203, 204, 205, -1, 207, 208, 209, - 210, 211, 212, 213, 214, 215, 216, -1, 218, -1, - 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, - 230, -1, -1, 233, 234, 235, 236, -1, 238, 239, - 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, - 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, - 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, - 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, - 280, 281, 282, 283, 284, 285, -1, 287, 288, -1, - -1, 291, 292, 293, -1, -1, 296, 297, 298, 299, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, -1, -1, -1, -1, 318, 319, - 320, 321, 322, 323, 324, 325, 326, 327, -1, 329, - 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, - 340, 341, 342, 343, -1, 345, 346, 347, 348, 349, - 350, 351, 352, 353, 354, 355, 356, 357, -1, 359, - 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, - 370, 371, 372, 373, 374, 375, 376, 377, 378, -1, - -1, 381, 382, 383, 384, 385, 386, 387, 388, 389, - -1, 391, 392, 393, 394, 395, -1, 397, 398, 399, - 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - 410, 411, -1, -1, 414, 415, -1, 417, 418, 419, - 420, 421, 422, 423, -1, 425, 426, 427, -1, -1, - 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, - 440, 441, 442, 443, -1, -1, 446, 447, 448, -1, - 450, 451, 452, 453, -1, 455, 456, 457, 458, 459, - 460, 461, -1, 463, 464, 465, 466, 467, 468, 469, - 470, 471, -1, -1, 474, -1, -1, 477, 478, 479, - 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, - 490, 491, 492, 493, 494, 495, 496, 497, 498, -1, - -1, -1, -1, -1, -1, 505, 506, 507, -1, -1, - -1, -1, 512, -1, 514, -1, -1, -1, -1, 519, - 520, -1, 522, 523, 3, 4, 5, 6, 7, 8, - 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, - 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, - 39, -1, -1, 42, 43, 44, -1, 46, 47, 48, - 49, 50, 51, 52, 53, 54, 55, 56, 57, -1, - 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, - 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, - 79, -1, 81, -1, 83, 84, 85, 86, 87, 88, - 89, 90, 91, 92, -1, 94, 95, 96, 97, 98, - 99, -1, 101, 102, 103, 104, 105, 106, 107, 108, - 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, - -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, - 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, - -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, - 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, - 159, 160, -1, 162, 163, 164, 165, -1, 167, -1, - 169, 170, -1, 172, 173, 174, 175, 176, 177, -1, - 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, - 189, 190, 191, 192, 193, 194, 195, 196, 197, -1, - 199, 200, 201, 202, 203, 204, 205, -1, 207, 208, - 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, - -1, 220, 221, 222, 223, 224, 225, 226, 227, 228, - 229, 230, -1, -1, 233, 234, 235, 236, -1, 238, - 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, - 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, - 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, - 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, - 279, 280, 281, 282, 283, 284, 285, -1, 287, 288, - -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, - 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, - 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, - 319, 320, 321, 322, 323, 324, 325, 326, 327, -1, - 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, - 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, - 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, - 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, - 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, - -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, - 389, -1, 391, 392, 393, 394, 395, -1, 397, 398, - 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, - 409, 410, 411, -1, -1, 414, 415, -1, 417, 418, - 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, - -1, 430, 431, 432, 433, 434, 435, 436, 437, 438, - 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, - -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, - 459, 460, 461, -1, 463, 464, 465, 466, 467, 468, - 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, - 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, - 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, - -1, -1, -1, -1, -1, -1, 505, 506, 507, -1, - -1, -1, -1, 512, -1, 514, -1, -1, -1, -1, - 519, 520, -1, 522, 523, 3, 4, 5, 6, 7, - 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 22, 23, 24, 25, 26, 27, - 28, 29, -1, 31, 32, 33, -1, -1, -1, -1, - -1, 39, -1, -1, 42, 43, 44, -1, 46, 47, - 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - -1, 59, 60, 61, 62, 63, 64, 65, 66, 67, - 68, 69, 70, 71, 72, 73, -1, 75, 76, 77, - 78, 79, -1, 81, -1, 83, 84, 85, 86, 87, - 88, 89, 90, 91, 92, -1, 94, 95, 96, 97, - 98, 99, -1, 101, 102, 103, 104, 105, 106, 107, - 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, - 118, -1, 120, 121, 122, 123, 124, 125, -1, -1, - 128, 129, 130, 131, -1, -1, 134, 135, 136, 137, - 138, -1, 140, 141, 142, -1, 144, 145, 146, -1, - 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, - 158, 159, 160, -1, 162, 163, 164, 165, -1, 167, - -1, 169, 170, -1, 172, 173, 174, 175, 176, 177, - -1, 179, -1, 181, 182, 183, 184, -1, 186, 187, - 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, - -1, 199, 200, 201, 202, 203, 204, 205, -1, 207, - 208, 209, 210, 211, 212, 213, 214, 215, 216, -1, - 218, -1, 220, 221, 222, 223, 224, 225, 226, 227, - 228, 229, 230, -1, -1, 233, 234, 235, 236, -1, - 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, - 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, - 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, - 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, - 278, 279, 280, 281, 282, 283, 284, 285, -1, 287, - 288, -1, -1, 291, 292, 293, -1, -1, 296, 297, - 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, - 308, 309, 310, 311, 312, 313, -1, -1, -1, -1, - 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, - -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, - 338, 339, 340, 341, 342, 343, -1, 345, 346, 347, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - -1, 359, 360, 361, 362, 363, 364, 365, 366, 367, - 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, - 378, -1, -1, 381, 382, 383, 384, 385, 386, 387, - 388, 389, -1, 391, 392, 393, 394, 395, -1, 397, - 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, - 408, 409, 410, 411, -1, -1, 414, 415, -1, 417, - 418, 419, 420, 421, 422, 423, -1, 425, 426, 427, - -1, -1, 430, 431, 432, 433, 434, 435, 436, 437, - 438, 439, 440, 441, 442, 443, -1, -1, 446, 447, - 448, -1, 450, 451, 452, 453, -1, 455, 456, 457, - 458, 459, 460, 461, -1, 463, 464, 465, 466, 467, - 468, 469, 470, 471, -1, -1, 474, -1, -1, 477, - 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, - 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, - 498, -1, -1, -1, -1, -1, -1, 505, 506, 507, - -1, -1, -1, -1, 512, -1, 514, -1, -1, -1, - -1, 519, 520, -1, 522, 523, 3, 4, 5, 6, - 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - -1, -1, 39, -1, -1, 42, 43, 44, -1, 46, - 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - 57, -1, 59, 60, 61, 62, 63, 64, 65, 66, - 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, 79, -1, 81, -1, 83, 84, 85, 86, - 87, 88, 89, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, 101, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - 117, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, - -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, 159, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, 170, -1, 172, 173, 174, 175, 176, - 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, 221, 222, 223, 224, 225, 226, - 227, 228, 229, 230, -1, -1, 233, 234, 235, 236, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, - 277, 278, 279, 280, 281, 282, 283, 284, 285, -1, - 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, 321, 322, 323, 324, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, 361, 362, 363, 364, 365, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, -1, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, 391, 392, 393, 394, 395, -1, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, - 417, 418, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, 435, 436, - 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, -1, 463, 464, 465, 466, - 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, - 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, - 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 498, -1, -1, -1, -1, -1, -1, 505, 506, - 507, -1, -1, -1, -1, 512, -1, 514, -1, -1, - -1, -1, 519, 520, -1, 522, 523, 3, 4, 5, - 6, 7, 8, 9, 10, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 22, 23, 24, 25, - 26, 27, 28, 29, -1, 31, 32, 33, -1, -1, - -1, -1, -1, 39, -1, -1, 42, 43, 44, -1, - 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, - 56, 57, -1, 59, 60, 61, 62, 63, 64, 65, - 66, 67, 68, 69, 70, 71, 72, 73, -1, 75, - 76, 77, 78, 79, -1, 81, -1, 83, 84, 85, - 86, 87, 88, 89, 90, 91, 92, -1, 94, 95, - 96, 97, 98, 99, -1, 101, 102, 103, 104, 105, - 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, - 116, 117, 118, -1, 120, 121, 122, 123, 124, 125, - -1, -1, 128, 129, 130, 131, -1, -1, 134, 135, - 136, 137, 138, -1, 140, 141, 142, -1, 144, 145, - 146, -1, 148, 149, 150, 151, 152, 153, 154, 155, - 156, 157, 158, 159, 160, -1, 162, 163, 164, 165, - -1, 167, -1, 169, 170, -1, 172, 173, 174, 175, - 176, 177, -1, 179, -1, 181, 182, 183, 184, -1, - 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, - 196, 197, -1, 199, 200, 201, 202, 203, 204, 205, - -1, 207, 208, 209, 210, 211, 212, 213, 214, 215, - 216, -1, 218, -1, 220, 221, 222, 223, 224, 225, - 226, 227, 228, 229, 230, -1, -1, 233, 234, 235, - 236, -1, 238, 239, 240, 241, 242, 243, 244, 245, - 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, - 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, - 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, - 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, - -1, 287, 288, -1, -1, 291, 292, 293, -1, -1, - 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, - 306, 307, 308, 309, 310, 311, 312, 313, -1, -1, - -1, -1, 318, 319, 320, 321, 322, 323, 324, 325, - 326, 327, -1, 329, 330, 331, 332, 333, 334, -1, - 336, 337, 338, 339, 340, 341, 342, 343, -1, 345, - 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, - 356, 357, -1, 359, 360, 361, 362, 363, 364, 365, - 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, - 376, 377, 378, -1, -1, 381, 382, 383, 384, 385, - 386, 387, 388, 389, -1, 391, 392, 393, 394, 395, - -1, 397, 398, 399, 400, 401, 402, 403, 404, 405, - 406, 407, 408, 409, 410, 411, -1, -1, 414, 415, - -1, 417, 418, 419, 420, 421, 422, 423, -1, 425, - 426, 427, -1, -1, 430, 431, 432, 433, 434, 435, - 436, 437, 438, 439, 440, 441, 442, 443, -1, -1, - 446, 447, 448, -1, 450, 451, 452, 453, -1, 455, - 456, 457, 458, 459, 460, 461, -1, 463, 464, 465, - 466, 467, 468, 469, 470, 471, -1, -1, 474, -1, - -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, - 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, - 496, 497, 498, -1, -1, -1, -1, -1, -1, 505, - 506, 507, -1, -1, -1, -1, 512, -1, 514, -1, - -1, -1, -1, 519, 520, -1, 522, 523, 3, 4, - 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 22, 23, 24, - 25, 26, 27, 28, 29, -1, 31, 32, 33, -1, - -1, -1, -1, -1, 39, -1, -1, 42, 43, 44, - -1, 46, 47, 48, 49, 50, 51, 52, 53, 54, - 55, 56, 57, -1, 59, 60, 61, 62, 63, 64, - 65, 66, 67, 68, 69, 70, 71, 72, 73, -1, - 75, 76, 77, 78, 79, -1, 81, -1, 83, 84, - 85, 86, 87, 88, 89, 90, 91, 92, -1, 94, - 95, 96, 97, 98, 99, -1, 101, 102, 103, 104, - 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, - 115, 116, 117, 118, -1, 120, 121, 122, 123, 124, - 125, -1, -1, 128, 129, 130, 131, -1, -1, 134, - 135, 136, 137, 138, -1, 140, 141, 142, -1, 144, - 145, 146, -1, 148, 149, 150, 151, 152, 153, 154, - 155, 156, 157, 158, 159, 160, -1, 162, 163, 164, - 165, -1, 167, -1, 169, 170, -1, 172, 173, 174, - 175, 176, 177, -1, 179, -1, 181, 182, 183, 184, - -1, 186, 187, 188, 189, 190, 191, 192, 193, 194, - 195, 196, 197, -1, 199, 200, 201, 202, 203, 204, - 205, -1, 207, 208, 209, 210, 211, 212, 213, 214, - 215, 216, -1, 218, -1, 220, 221, 222, 223, 224, - 225, 226, 227, 228, 229, 230, -1, -1, 233, 234, - 235, 236, -1, 238, 239, 240, 241, 242, 243, 244, - 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, - 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, - 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, - 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, - 285, -1, 287, 288, -1, -1, 291, 292, 293, -1, - -1, 296, 297, 298, 299, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, -1, - -1, -1, -1, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 327, -1, 329, 330, 331, 332, 333, 334, - -1, 336, 337, 338, 339, 340, 341, 342, 343, -1, - 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, - 355, 356, 357, -1, 359, 360, 361, 362, 363, 364, - 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, - 375, 376, 377, 378, -1, -1, 381, 382, 383, 384, - 385, 386, 387, 388, 389, -1, 391, 392, 393, 394, - 395, -1, 397, 398, 399, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, -1, -1, 414, - 415, -1, 417, 418, 419, 420, 421, 422, 423, -1, - 425, 426, 427, -1, -1, 430, 431, 432, 433, 434, - 435, 436, 437, 438, 439, 440, 441, 442, 443, -1, - -1, 446, 447, 448, -1, 450, 451, 452, 453, -1, - 455, 456, 457, 458, 459, 460, 461, -1, 463, 464, - 465, 466, 467, 468, 469, 470, 471, -1, -1, 474, - -1, -1, 477, 478, 479, 480, 481, 482, 483, 484, - 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, - 495, 496, 497, 498, -1, -1, -1, -1, -1, -1, - 505, 506, 507, -1, -1, -1, -1, 512, -1, 514, - -1, -1, -1, -1, 519, 520, -1, 522, 523, 3, - 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, - 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, - -1, -1, -1, -1, -1, 39, -1, -1, 42, 43, - 44, -1, 46, 47, 48, 49, 50, 51, 52, 53, - 54, 55, 56, 57, -1, 59, 60, 61, 62, 63, - 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, - -1, 75, 76, 77, 78, 79, -1, 81, -1, 83, - 84, 85, 86, 87, 88, 89, 90, 91, 92, -1, - 94, 95, 96, 97, 98, 99, -1, 101, 102, 103, - 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, - 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, - 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, - 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, - 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, - 154, 155, 156, 157, 158, 159, 160, -1, 162, 163, - 164, 165, -1, 167, -1, 169, 170, -1, 172, 173, - 174, 175, 176, 177, -1, 179, -1, 181, 182, 183, - 184, -1, 186, 187, 188, 189, 190, 191, 192, 193, - 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, - 204, 205, -1, 207, 208, 209, 210, 211, 212, 213, - 214, 215, 216, -1, 218, -1, 220, 221, 222, 223, - 224, 225, 226, 227, 228, 229, 230, -1, -1, 233, - 234, 235, 236, -1, 238, 239, 240, 241, 242, 243, - 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, - 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, - 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, - -1, 275, 276, 277, 278, 279, 280, 281, 282, 283, - 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, - -1, -1, 296, 297, 298, 299, 300, 301, 302, 303, - 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, - -1, -1, -1, -1, 318, 319, 320, 321, 322, 323, - 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, - 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, - -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, - 354, 355, 356, 357, -1, 359, 360, 361, 362, 363, - 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, - 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, - 384, 385, 386, 387, 388, 389, -1, 391, 392, 393, - 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, - 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, - 414, 415, -1, 417, 418, 419, 420, 421, 422, 423, - -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, - 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, - -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, - -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, - 464, 465, 466, 467, 468, 469, 470, 471, -1, -1, - 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, - 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, - 494, 495, 496, 497, -1, -1, -1, -1, -1, -1, - -1, 505, 506, -1, -1, -1, -1, -1, 512, -1, - 514, -1, -1, -1, -1, 519, 520, -1, 522, 523, - 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 22, - 23, 24, 25, 26, 27, 28, 29, -1, 31, 32, - 33, -1, -1, -1, -1, -1, 39, -1, -1, 42, - 43, 44, -1, 46, 47, 48, 49, 50, 51, 52, - 53, 54, 55, 56, 57, -1, 59, 60, 61, 62, - 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, - 73, -1, 75, 76, 77, 78, 79, -1, 81, -1, - 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, - -1, 94, 95, 96, 97, 98, 99, -1, 101, 102, - 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, - 113, 114, 115, 116, 117, 118, -1, 120, 121, 122, - 123, 124, 125, -1, -1, 128, 129, 130, 131, -1, - -1, 134, 135, 136, 137, 138, -1, 140, 141, 142, - -1, 144, 145, 146, -1, 148, 149, 150, 151, 152, - 153, 154, 155, 156, 157, 158, 159, 160, -1, 162, - 163, 164, 165, -1, 167, -1, 169, 170, -1, 172, - 173, 174, 175, 176, 177, -1, 179, -1, 181, 182, - 183, 184, -1, 186, 187, 188, 189, 190, 191, 192, - 193, 194, 195, 196, 197, -1, 199, 200, 201, 202, - 203, 204, 205, -1, 207, 208, 209, 210, 211, 212, - 213, 214, 215, 216, -1, 218, -1, 220, -1, -1, - 223, 224, 225, 226, 227, 228, 229, 230, -1, -1, - 233, 234, 235, 236, -1, 238, 239, 240, 241, 242, - 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, - 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, - 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, - 273, -1, 275, 276, -1, 278, 279, 280, 281, 282, - 283, 284, 285, -1, 287, 288, -1, -1, 291, 292, - 293, -1, -1, 296, 297, 298, 299, 300, 301, 302, - 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, - 313, -1, -1, -1, -1, 318, 319, 320, 321, 322, - 323, 324, 325, 326, 327, -1, 329, 330, 331, 332, - 333, 334, -1, 336, 337, 338, 339, 340, 341, 342, - 343, -1, 345, 346, 347, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, -1, 359, 360, 361, 362, - 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, - 373, 374, 375, 376, 377, 378, -1, -1, 381, 382, - 383, 384, 385, 386, 387, 388, 389, -1, 391, 392, - 393, 394, 395, -1, 397, 398, 399, 400, 401, 402, - 403, 404, 405, 406, 407, 408, 409, 410, 411, -1, - -1, 414, 415, -1, 417, 418, 419, 420, 421, 422, - 423, -1, 425, 426, 427, -1, -1, 430, 431, 432, - 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, - 443, -1, -1, 446, 447, 448, -1, 450, 451, 452, - 453, -1, 455, 456, 457, 458, 459, 460, 461, -1, - 463, 464, 465, 466, 467, 468, 469, 470, 471, -1, - -1, 474, -1, -1, 477, 478, 479, 480, 481, 482, - 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, - 493, 494, 495, 496, 497, 498, -1, -1, -1, -1, - -1, -1, 505, 506, 507, -1, -1, -1, -1, 512, - -1, 514, -1, -1, -1, -1, 519, 520, -1, 522, - 523, 3, 4, 5, 6, 7, -1, 9, 10, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, - 32, 33, -1, -1, -1, -1, -1, 39, -1, -1, - 42, 43, 44, -1, 46, 47, 48, 49, 50, 51, - 52, 53, 54, 55, 56, 57, -1, 59, 60, 61, - 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, - 72, 73, -1, 75, 76, 77, 78, 79, -1, 81, - -1, 83, 84, 85, 86, 87, 88, 89, 90, 91, - 92, -1, 94, 95, 96, 97, 98, 99, -1, 101, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, -1, 118, -1, 120, 121, - 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, - -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, - 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, 159, 160, -1, - 162, 163, 164, 165, -1, 167, -1, 169, 170, -1, - 172, 173, 174, 175, 176, 177, -1, 179, -1, 181, - 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, - 192, 193, 194, 195, 196, 197, -1, 199, 200, 201, - 202, 203, 204, 205, -1, 207, 208, 209, 210, 211, - 212, 213, 214, 215, 216, -1, 218, -1, 220, 221, - 222, 223, 224, 225, 226, 227, 228, 229, 230, -1, - -1, 233, 234, 235, 236, -1, 238, 239, 240, 241, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, - 272, 273, -1, 275, 276, 277, 278, 279, 280, 281, - 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, - 292, 293, -1, -1, 296, 297, 298, 299, 300, 301, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, -1, -1, -1, -1, 318, 319, 320, 321, - 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, - 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, - 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, -1, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, -1, -1, 381, - 382, 383, 384, 385, 386, 387, 388, 389, -1, 391, - 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - -1, -1, 414, 415, -1, 417, 418, 419, 420, 421, - 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, - 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, - 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, - -1, 463, 464, 465, 466, 467, 468, 469, 470, 471, - -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, - 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, -1, -1, -1, -1, - -1, -1, -1, 505, 506, -1, -1, -1, -1, -1, - 512, -1, 514, -1, -1, -1, -1, 519, 520, -1, - 522, 523, 3, 4, 5, 6, 7, 8, 9, 10, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 22, 23, 24, 25, 26, 27, 28, 29, -1, - 31, 32, 33, -1, -1, -1, -1, -1, 39, -1, - -1, 42, 43, 44, -1, 46, 47, 48, 49, 50, - 51, 52, 53, 54, 55, 56, 57, -1, 59, 60, - 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, - 71, 72, 73, -1, 75, 76, 77, 78, 79, -1, - 81, -1, 83, 84, 85, 86, 87, 88, 89, 90, - 91, 92, -1, 94, 95, 96, 97, 98, 99, -1, - 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, - 111, 112, 113, 114, 115, 116, -1, 118, -1, 120, - 121, 122, 123, 124, 125, -1, -1, 128, 129, 130, - 131, -1, -1, 134, 135, 136, 137, 138, -1, 140, - 141, 142, -1, 144, 145, 146, -1, 148, 149, 150, - 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, - -1, 162, 163, 164, 165, -1, 167, -1, 169, 170, - -1, 172, 173, 174, 175, 176, 177, -1, 179, -1, - 181, 182, 183, 184, -1, 186, 187, 188, 189, 190, - 191, 192, 193, 194, 195, 196, 197, -1, 199, 200, - 201, 202, 203, 204, 205, -1, 207, 208, 209, 210, - 211, 212, 213, 214, 215, 216, -1, 218, -1, 220, - -1, 222, 223, 224, 225, 226, 227, 228, 229, 230, - -1, -1, 233, 234, 235, 236, -1, 238, 239, 240, - 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, - 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, - 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, - 271, 272, 273, -1, 275, 276, 277, 278, 279, 280, - 281, 282, 283, 284, 285, -1, 287, 288, -1, -1, - 291, 292, 293, -1, -1, 296, 297, 298, 299, 300, - 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, - 311, 312, 313, -1, -1, -1, -1, 318, 319, 320, - 321, 322, 323, 324, 325, 326, 327, -1, 329, 330, - 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, - 341, 342, 343, -1, 345, 346, 347, 348, 349, 350, - 351, 352, 353, 354, 355, 356, 357, -1, 359, 360, - 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, - 371, 372, 373, 374, 375, 376, 377, 378, -1, -1, - 381, 382, 383, 384, 385, 386, 387, 388, 389, -1, - 391, 392, 393, 394, 395, -1, 397, 398, 399, 400, - 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, - 411, -1, -1, 414, 415, -1, 417, 418, 419, 420, - 421, 422, 423, -1, 425, 426, 427, -1, -1, 430, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 443, -1, -1, 446, 447, 448, -1, 450, - 451, 452, 453, -1, 455, 456, 457, 458, 459, 460, - 461, -1, 463, 464, 465, 466, 467, 468, 469, 470, - 471, -1, -1, 474, -1, -1, 477, 478, 479, 480, - 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, - 491, 492, 493, 494, 495, 496, 497, -1, -1, -1, - -1, -1, -1, -1, 505, 506, -1, -1, -1, -1, - -1, 512, -1, 514, -1, -1, -1, -1, 519, 520, - -1, 522, 523, 3, 4, 5, 6, 7, -1, 9, - 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 22, 23, 24, 25, 26, 27, 28, 29, - -1, 31, 32, 33, -1, -1, -1, -1, -1, 39, - -1, -1, 42, 43, 44, -1, 46, 47, 48, 49, - 50, 51, 52, 53, 54, 55, 56, 57, -1, 59, - 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, - 70, 71, 72, 73, -1, 75, 76, 77, 78, 79, - -1, 81, -1, 83, 84, 85, 86, 87, 88, 89, - 90, 91, 92, -1, 94, 95, 96, 97, 98, 99, - -1, 101, 102, 103, 104, 105, 106, 107, 108, 109, - 110, 111, 112, 113, 114, 115, 116, -1, 118, -1, - 120, 121, 122, 123, 124, 125, -1, -1, 128, 129, - 130, 131, -1, -1, 134, 135, 136, 137, 138, -1, - 140, 141, 142, -1, 144, 145, 146, -1, 148, 149, - 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, - 160, -1, 162, 163, 164, 165, -1, 167, -1, 169, - 170, -1, 172, 173, 174, 175, 176, 177, -1, 179, - 180, 181, 182, 183, 184, -1, 186, 187, 188, 189, - 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, - 200, 201, 202, 203, 204, 205, -1, 207, 208, 209, - 210, 211, 212, 213, 214, 215, 216, -1, 218, -1, - 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, - 230, -1, -1, 233, 234, 235, 236, -1, 238, 239, - 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, - 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, - 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, - 270, 271, 272, 273, -1, 275, 276, 277, 278, 279, - 280, 281, 282, 283, 284, 285, -1, 287, 288, -1, - -1, 291, 292, 293, -1, -1, 296, 297, 298, 299, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, -1, -1, -1, -1, 318, 319, - 320, 321, 322, 323, 324, 325, 326, 327, -1, 329, - 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, - 340, 341, 342, 343, -1, 345, 346, 347, 348, 349, - 350, 351, 352, 353, 354, 355, 356, 357, -1, 359, - 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, - 370, 371, 372, 373, 374, 375, 376, 377, 378, -1, - -1, 381, 382, 383, 384, 385, 386, 387, 388, 389, - -1, 391, 392, 393, 394, 395, -1, 397, 398, 399, - 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - 410, 411, -1, -1, 414, 415, -1, 417, 418, 419, - 420, 421, 422, 423, -1, 425, 426, 427, -1, -1, - 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, - 440, 441, 442, 443, -1, -1, 446, 447, 448, -1, - 450, 451, 452, 453, -1, 455, 456, 457, 458, 459, - 460, 461, -1, 463, 464, 465, 466, 467, 468, 469, - 470, 471, -1, -1, 474, -1, -1, 477, 478, 479, - 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, - 490, 491, 492, 493, 494, 495, 496, 497, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 512, -1, 514, -1, -1, -1, -1, 519, - 520, -1, 522, 523, 3, 4, 5, 6, 7, -1, - 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, - 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, - 39, -1, -1, 42, 43, 44, -1, 46, 47, 48, - 49, 50, 51, 52, 53, 54, 55, 56, 57, -1, - 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, - 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, - 79, -1, 81, -1, 83, 84, 85, 86, 87, 88, - 89, 90, 91, 92, -1, 94, 95, 96, 97, 98, - 99, -1, 101, 102, 103, 104, 105, 106, 107, 108, - 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, - -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, - 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, - -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, - 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, - 159, 160, -1, 162, 163, 164, 165, -1, 167, -1, - 169, 170, -1, 172, 173, 174, 175, 176, 177, -1, - 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, - 189, 190, 191, 192, 193, 194, 195, 196, 197, -1, - 199, 200, 201, 202, 203, 204, 205, -1, 207, 208, - 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, - -1, 220, 221, 222, 223, 224, 225, 226, 227, 228, - 229, 230, -1, -1, 233, 234, 235, 236, -1, 238, - 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, - 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, - 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, - 269, 270, 271, 272, 273, -1, 275, 276, 277, 278, - 279, 280, 281, 282, 283, 284, 285, -1, 287, 288, - -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, - 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, - 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, - 319, 320, 321, 322, 323, 324, 325, 326, 327, -1, - 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, - 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, - 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, - 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, - 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, - -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, - 389, -1, 391, 392, 393, 394, 395, -1, 397, 398, - 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, - 409, 410, 411, -1, -1, 414, 415, -1, 417, 418, - 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, - -1, 430, 431, 432, 433, 434, 435, 436, 437, 438, - 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, - -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, - 459, 460, 461, -1, 463, 464, 465, 466, 467, 468, - 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, - 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, - 489, 490, 491, 492, 493, 494, 495, 496, 497, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 512, -1, 514, -1, -1, -1, -1, - 519, 520, -1, 522, 523, 3, 4, 5, 6, 7, - -1, 9, 10, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 22, 23, 24, 25, 26, 27, - 28, 29, -1, 31, 32, 33, -1, -1, -1, -1, - -1, 39, -1, -1, 42, 43, 44, -1, 46, 47, - 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - -1, 59, 60, 61, 62, 63, 64, 65, 66, 67, - 68, 69, 70, 71, 72, 73, -1, 75, 76, 77, - 78, 79, -1, 81, -1, 83, 84, 85, 86, 87, - 88, 89, 90, 91, 92, -1, 94, 95, 96, 97, - 98, 99, -1, 101, 102, 103, 104, 105, 106, 107, - 108, 109, 110, 111, 112, 113, 114, 115, 116, -1, - 118, -1, 120, 121, 122, 123, 124, 125, -1, -1, - 128, 129, 130, 131, -1, -1, 134, 135, 136, 137, - 138, -1, 140, 141, 142, -1, 144, 145, 146, -1, - 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, - 158, 159, 160, -1, 162, 163, 164, 165, -1, 167, - -1, 169, 170, -1, 172, 173, 174, 175, 176, 177, - -1, 179, -1, 181, 182, 183, 184, -1, 186, 187, - 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, - -1, 199, 200, 201, 202, 203, 204, 205, -1, 207, - 208, 209, 210, 211, 212, 213, 214, 215, 216, -1, - 218, -1, 220, 221, 222, 223, 224, 225, 226, 227, - 228, 229, 230, -1, -1, 233, 234, 235, 236, -1, - 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, - 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, - 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, - 268, 269, 270, 271, 272, 273, -1, 275, 276, 277, - 278, 279, 280, 281, 282, 283, 284, 285, -1, 287, - 288, -1, -1, 291, 292, 293, -1, -1, 296, 297, - 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, - 308, 309, 310, 311, 312, 313, -1, -1, -1, -1, - 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, - -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, - 338, 339, 340, 341, 342, 343, -1, 345, 346, 347, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - -1, 359, 360, 361, 362, 363, 364, 365, 366, 367, - 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, - 378, -1, -1, 381, 382, 383, 384, 385, 386, 387, - 388, 389, -1, 391, 392, 393, 394, 395, -1, 397, - 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, - 408, 409, 410, 411, -1, -1, 414, 415, -1, 417, - 418, 419, 420, 421, 422, 423, -1, 425, 426, 427, - -1, -1, 430, 431, 432, 433, 434, 435, 436, 437, - 438, 439, 440, 441, 442, 443, -1, -1, 446, 447, - 448, -1, 450, 451, 452, 453, -1, 455, 456, 457, - 458, 459, 460, 461, -1, 463, 464, 465, 466, 467, - 468, 469, 470, 471, -1, -1, 474, -1, -1, 477, - 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, - 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 512, -1, 514, -1, -1, -1, - -1, 519, 520, -1, 522, 523, 3, 4, 5, 6, - 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - -1, -1, 39, -1, -1, 42, 43, 44, -1, 46, - 47, 48, 49, 50, 51, 52, -1, 54, 55, 56, - 57, -1, 59, 60, 61, 62, 63, 64, 65, 66, - 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, 79, -1, 81, -1, 83, 84, 85, 86, - 87, 88, 89, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, 101, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - 117, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, -1, 146, - -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, 159, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, 170, -1, 172, 173, 174, 175, -1, - 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, -1, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, -1, -1, 223, 224, 225, 226, - 227, 228, 229, 230, -1, -1, 233, 234, 235, -1, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, 268, 269, 270, 271, 272, 273, -1, 275, 276, - -1, 278, 279, 280, 281, 282, 283, 284, 285, -1, - 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, 321, 322, 323, 324, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, 361, 362, 363, 364, 365, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, -1, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, -1, 392, 393, 394, 395, -1, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, - 417, 418, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, 435, 436, - 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, -1, 463, 464, 465, 466, - 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, - 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, - 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, -1, -1, -1, -1, -1, -1, -1, 505, 506, - 507, -1, 3, 4, 5, 512, -1, 514, 9, -1, - -1, -1, 519, 520, -1, 522, 523, -1, -1, -1, - -1, 22, 23, 24, 25, 26, 27, 28, 29, -1, - 31, 32, 33, -1, -1, -1, 37, -1, -1, -1, - -1, 42, 43, 44, -1, 46, 47, 48, 49, 50, - 51, 52, 53, 54, 55, 56, 57, -1, 59, 60, - 61, 62, 63, 64, -1, -1, 67, 68, 69, 70, - 71, 72, 73, -1, 75, 76, 77, 78, 79, -1, - 81, -1, 83, 84, 85, 86, 87, 88, 89, 90, - 91, 92, -1, 94, 95, 96, 97, 98, 99, -1, - 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, - 111, 112, 113, 114, 115, 116, -1, 118, -1, 120, - 121, 122, 123, 124, 125, -1, -1, 128, 129, 130, - 131, -1, -1, 134, 135, 136, 137, 138, -1, 140, - 141, 142, -1, 144, 145, 146, -1, 148, 149, 150, - 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, - -1, 162, 163, 164, 165, -1, 167, -1, 169, 170, - -1, 172, 173, 174, 175, 176, 177, -1, 179, -1, - 181, 182, 183, 184, -1, 186, 187, 188, 189, 190, - 191, 192, 193, 194, 195, 196, 197, -1, 199, 200, - 201, 202, 203, 204, 205, -1, 207, 208, 209, 210, - 211, 212, 213, 214, 215, 216, -1, 218, -1, 220, - 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, - -1, -1, 233, 234, 235, 236, -1, 238, 239, 240, - 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, - 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, - 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, - 271, 272, 273, -1, 275, 276, 277, 278, -1, 280, - 281, 282, 283, 284, 285, -1, 287, 288, 289, -1, - 291, 292, 293, -1, -1, 296, 297, 298, 299, 300, - 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, - 311, 312, 313, -1, -1, -1, -1, 318, 319, 320, - 321, 322, 323, 324, 325, 326, 327, -1, 329, 330, - 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, - 341, 342, 343, -1, 345, 346, 347, 348, 349, 350, - 351, 352, 353, 354, 355, 356, 357, -1, 359, 360, - 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, - 371, 372, 373, 374, 375, 376, 377, 378, -1, 380, - 381, 382, 383, 384, 385, 386, 387, 388, 389, -1, - 391, 392, 393, 394, 395, -1, 397, 398, 399, 400, - 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, - 411, -1, -1, 414, 415, -1, 417, 418, 419, 420, - 421, 422, 423, -1, 425, 426, 427, -1, -1, 430, - 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, - 441, 442, 443, -1, -1, 446, 447, 448, -1, 450, - 451, 452, 453, -1, 455, 456, 457, 458, 459, 460, - 461, -1, 463, 464, 465, 466, 467, 468, 469, 470, - 471, -1, -1, 474, -1, -1, 477, 478, 479, 480, - 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, - 491, 492, 493, 494, 495, 496, 497, -1, -1, -1, - -1, 3, -1, -1, 505, 506, 507, -1, -1, -1, - -1, -1, -1, 514, -1, -1, -1, -1, -1, 520, - 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, - 32, 33, -1, -1, -1, -1, -1, -1, 40, -1, - -1, 43, 44, -1, 46, 47, 48, -1, 50, 51, - 52, 53, 54, -1, 56, 57, -1, 59, 60, 61, - 62, 63, 64, -1, -1, 67, 68, 69, 70, 71, - 72, 73, -1, 75, 76, 77, 78, 79, -1, -1, - -1, 83, 84, 85, 86, 87, 88, -1, 90, 91, - 92, -1, 94, 95, 96, 97, 98, 99, -1, -1, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, -1, 118, -1, 120, 121, - 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, - -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, - 142, -1, 144, 145, 146, 147, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, -1, 160, 161, - 162, 163, 164, 165, 166, 167, -1, 169, -1, -1, - -1, 173, 174, 175, -1, 177, -1, 179, -1, 181, - 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, - 192, -1, 194, 195, 196, 197, -1, 199, 200, 201, - 202, 203, 204, 205, -1, 207, -1, 209, 210, 211, - 212, 213, 214, 215, 216, 217, 218, -1, 220, -1, - -1, 223, -1, 225, 226, 227, 228, 229, 230, -1, - -1, 233, -1, 235, -1, 237, 238, 239, 240, 241, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, -1, 269, 270, 271, - 272, 273, -1, 275, 276, -1, 278, -1, 280, 281, - 282, 283, 284, 285, 286, 287, 288, -1, -1, 291, - 292, 293, -1, 295, 296, 297, 298, -1, 300, -1, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, -1, -1, -1, -1, 318, 319, 320, -1, - 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, - 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, - 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, -1, 359, 360, -1, - 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, -1, -1, 381, - 382, 383, 384, 385, 386, 387, 388, 389, -1, -1, - 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - -1, -1, 414, 415, -1, 417, -1, 419, 420, 421, - 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - 432, 433, 434, -1, 436, 437, 438, 439, 440, 441, - 442, 443, 444, -1, 446, 447, 448, -1, 450, 451, - 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, - -1, 463, -1, 465, 466, 467, 468, 469, 470, 471, - -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, - 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 3, -1, 5, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 515, -1, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 43, 44, -1, 46, - 47, 48, -1, 50, 51, 52, 53, 54, -1, 56, - 57, -1, 59, 60, 61, 62, 63, 64, -1, -1, - 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, 79, -1, -1, -1, 83, 84, 85, 86, - 87, 88, -1, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, -1, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, - -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, -1, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, -1, -1, -1, 173, 174, 175, -1, - 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, -1, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, -1, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, -1, -1, 223, -1, 225, 226, - 227, 228, 229, 230, -1, -1, 233, -1, 235, -1, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, -1, 269, 270, 271, 272, 273, -1, 275, 276, - -1, 278, -1, 280, 281, 282, 283, 284, 285, -1, - 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, 298, -1, 300, -1, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, -1, 322, 323, 324, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, -1, 362, 363, 364, 365, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, -1, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, -1, 392, 393, 394, 395, -1, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, - 417, -1, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, -1, 436, - 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, -1, 463, -1, 465, 466, - 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, - 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, - 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 514, 515, -1, - 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, - 32, 33, 34, 35, -1, 37, -1, -1, -1, -1, - 42, 43, 44, -1, 46, 47, 48, 49, 50, 51, - 52, 53, 54, 55, 56, 57, -1, 59, 60, 61, - 62, 63, 64, -1, -1, 67, 68, 69, 70, 71, - 72, 73, -1, 75, 76, 77, 78, 79, -1, 81, - -1, 83, 84, 85, 86, 87, 88, 89, 90, 91, - 92, -1, 94, 95, 96, 97, 98, 99, -1, 101, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, -1, 118, -1, 120, 121, - 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, - -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, - 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, -1, 160, -1, - 162, 163, 164, 165, -1, 167, -1, 169, 170, 171, - 172, 173, 174, 175, 176, 177, -1, 179, -1, 181, - 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, - 192, 193, 194, 195, 196, 197, -1, 199, 200, 201, - 202, 203, 204, 205, -1, 207, 208, 209, 210, 211, - 212, 213, 214, 215, 216, -1, 218, -1, 220, 221, - 222, 223, 224, 225, 226, 227, 228, 229, 230, -1, - -1, 233, 234, 235, 236, -1, 238, 239, 240, 241, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, - 272, 273, -1, 275, 276, 277, 278, -1, 280, 281, - 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, - 292, 293, -1, -1, 296, 297, 298, 299, 300, 301, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, 314, 315, 316, -1, 318, 319, 320, 321, - 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, - 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, - 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, -1, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, - 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, - 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - 412, -1, 414, 415, 416, 417, 418, 419, 420, 421, - 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - 432, 433, 434, -1, 436, 437, 438, 439, 440, 441, - 442, 443, -1, -1, 446, 447, 448, 449, 450, 451, - 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, - -1, 463, 464, 465, 466, 467, 468, 469, 470, 471, - -1, -1, 474, -1, 476, 477, 478, 479, 480, 481, - 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, -1, 3, 500, 5, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 514, -1, -1, -1, 22, 23, 24, 25, - 26, 27, 28, 29, -1, 31, 32, 33, -1, -1, - -1, -1, -1, -1, -1, -1, 42, 43, 44, -1, - 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, - 56, 57, -1, 59, 60, 61, 62, 63, 64, -1, - 66, 67, 68, 69, 70, 71, 72, 73, -1, 75, - 76, 77, 78, 79, -1, 81, -1, 83, 84, 85, - 86, 87, 88, 89, 90, 91, 92, -1, 94, 95, - 96, 97, 98, 99, -1, 101, 102, 103, 104, 105, - 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, - 116, -1, 118, -1, 120, 121, 122, 123, 124, 125, - 126, 127, 128, 129, 130, 131, -1, -1, 134, 135, - 136, 137, 138, -1, 140, 141, 142, -1, 144, 145, - 146, -1, 148, 149, 150, 151, 152, 153, 154, 155, - 156, 157, 158, -1, 160, -1, 162, 163, 164, 165, - -1, 167, -1, 169, 170, 171, 172, 173, 174, 175, - 176, 177, -1, 179, -1, 181, 182, 183, 184, -1, - 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, - 196, 197, -1, 199, 200, 201, 202, 203, 204, 205, - -1, 207, 208, 209, 210, 211, 212, 213, 214, 215, - 216, -1, 218, -1, 220, 221, 222, 223, 224, 225, - 226, 227, 228, 229, 230, 231, -1, 233, 234, 235, - 236, -1, 238, 239, 240, 241, 242, 243, 244, 245, - 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, - 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, - 266, 267, 268, 269, 270, 271, 272, 273, -1, 275, - 276, 277, 278, -1, 280, 281, 282, 283, 284, 285, - -1, 287, 288, -1, 290, 291, 292, 293, -1, -1, - 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, - 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, - 316, -1, 318, 319, 320, 321, 322, 323, 324, 325, - 326, 327, -1, 329, 330, 331, 332, 333, 334, -1, - 336, 337, 338, 339, 340, 341, 342, 343, -1, 345, - 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, - 356, 357, -1, 359, 360, 361, 362, 363, 364, 365, - 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, - 376, 377, 378, 379, -1, 381, 382, 383, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, - -1, 397, 398, 399, 400, 401, 402, 403, 404, 405, - 406, 407, 408, 409, 410, 411, 412, -1, 414, 415, - 416, 417, 418, 419, 420, 421, 422, 423, -1, 425, - 426, 427, -1, -1, 430, 431, 432, 433, 434, -1, - 436, 437, 438, 439, 440, 441, 442, 443, -1, -1, - 446, 447, 448, 449, 450, 451, 452, 453, -1, 455, - 456, 457, 458, 459, 460, 461, -1, 463, 464, 465, - 466, 467, 468, 469, 470, 471, -1, -1, 474, -1, - 476, 477, 478, 479, 480, 481, 482, 483, 484, 485, - 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, - 496, 497, -1, 3, 500, 5, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 514, -1, - -1, -1, 22, 23, 24, 25, 26, 27, 28, 29, - -1, 31, 32, 33, -1, -1, -1, -1, -1, -1, - -1, -1, 42, 43, 44, -1, 46, 47, 48, 49, - 50, 51, 52, 53, 54, 55, 56, 57, -1, 59, - 60, 61, 62, 63, 64, -1, 66, 67, 68, 69, - 70, 71, 72, 73, -1, 75, 76, 77, 78, 79, - -1, 81, -1, 83, 84, 85, 86, 87, 88, 89, - 90, 91, 92, -1, 94, 95, 96, 97, 98, 99, - -1, 101, 102, 103, 104, 105, 106, 107, 108, 109, - 110, 111, 112, 113, 114, 115, 116, -1, 118, -1, - 120, 121, 122, 123, 124, 125, -1, -1, 128, 129, - 130, 131, -1, -1, 134, 135, 136, 137, 138, -1, - 140, 141, 142, -1, 144, 145, 146, -1, 148, 149, - 150, 151, 152, 153, 154, 155, 156, 157, 158, -1, - 160, -1, 162, 163, 164, 165, -1, 167, -1, 169, - 170, -1, 172, 173, 174, 175, 176, 177, -1, 179, - -1, 181, 182, 183, 184, -1, 186, 187, 188, 189, - 190, 191, 192, 193, 194, 195, 196, 197, -1, 199, - 200, 201, 202, 203, 204, 205, -1, 207, 208, 209, - 210, 211, 212, 213, 214, 215, 216, -1, 218, -1, - 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, - 230, 231, -1, 233, 234, 235, 236, -1, 238, 239, - 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, - 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, - 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, - 270, 271, 272, 273, -1, 275, 276, 277, 278, -1, - 280, 281, 282, 283, 284, 285, -1, 287, 288, -1, - 290, 291, 292, 293, -1, -1, 296, 297, 298, 299, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, -1, -1, -1, -1, 318, 319, - 320, 321, 322, 323, 324, 325, 326, 327, -1, 329, - 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, - 340, 341, 342, 343, -1, 345, 346, 347, 348, 349, - 350, 351, 352, 353, 354, 355, 356, 357, -1, 359, - 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, - 370, 371, 372, 373, 374, 375, 376, 377, 378, -1, - -1, 381, 382, 383, 384, 385, 386, 387, 388, 389, - -1, 391, 392, 393, 394, 395, -1, 397, 398, 399, - 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - 410, 411, -1, -1, 414, 415, -1, 417, 418, 419, - 420, 421, 422, 423, -1, 425, 426, 427, -1, -1, - 430, 431, 432, 433, 434, -1, 436, 437, 438, 439, - 440, 441, 442, 443, -1, -1, 446, 447, 448, -1, - 450, 451, 452, 453, -1, 455, 456, 457, 458, 459, - 460, 461, -1, 463, 464, 465, 466, 467, 468, 469, - 470, 471, -1, -1, 474, -1, -1, 477, 478, 479, - 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, - 490, 491, 492, 493, 494, 495, 496, 497, 3, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 514, -1, -1, 22, 23, 24, - 25, 26, 27, 28, 29, -1, 31, 32, 33, -1, - -1, -1, -1, -1, -1, -1, -1, 42, 43, 44, - -1, 46, 47, 48, 49, 50, 51, 52, 53, 54, - 55, 56, 57, -1, 59, 60, 61, 62, 63, 64, - -1, 66, 67, 68, 69, 70, 71, 72, 73, -1, - 75, 76, 77, 78, 79, -1, 81, -1, 83, 84, - 85, 86, 87, 88, 89, 90, 91, 92, -1, 94, - 95, 96, 97, 98, 99, -1, 101, 102, 103, 104, - 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, - 115, 116, -1, 118, -1, 120, 121, 122, 123, 124, - 125, -1, -1, 128, 129, 130, 131, -1, -1, 134, - 135, 136, 137, 138, -1, 140, 141, 142, -1, 144, - 145, 146, -1, 148, 149, 150, 151, 152, 153, 154, - 155, 156, 157, 158, -1, 160, -1, 162, 163, 164, - 165, -1, 167, -1, 169, 170, -1, 172, 173, 174, - 175, 176, 177, -1, 179, -1, 181, 182, 183, 184, - -1, 186, 187, 188, 189, 190, 191, 192, 193, 194, - 195, 196, 197, -1, 199, 200, 201, 202, 203, 204, - 205, -1, 207, 208, 209, 210, 211, 212, 213, 214, - 215, 216, -1, 218, -1, 220, 221, 222, 223, 224, - 225, 226, 227, 228, 229, 230, -1, -1, 233, 234, - 235, 236, -1, 238, 239, 240, 241, 242, 243, 244, - 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, - 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, - 265, 266, 267, 268, 269, 270, 271, 272, 273, -1, - 275, 276, 277, 278, -1, 280, 281, 282, 283, 284, - 285, -1, 287, 288, -1, -1, 291, 292, 293, -1, - -1, 296, 297, 298, 299, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, -1, - -1, -1, -1, 318, 319, 320, 321, 322, 323, 324, - 325, 326, 327, -1, 329, 330, 331, 332, 333, 334, - -1, 336, 337, 338, 339, 340, 341, 342, 343, -1, - 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, - 355, 356, 357, -1, 359, 360, 361, 362, 363, 364, - 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, - 375, 376, 377, 378, -1, -1, 381, 382, 383, 384, - 385, 386, 387, 388, 389, -1, 391, 392, 393, 394, - 395, -1, 397, 398, 399, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, -1, -1, 414, - 415, -1, 417, 418, 419, 420, 421, 422, 423, -1, - 425, 426, 427, -1, -1, 430, 431, 432, 433, 434, - -1, 436, 437, 438, 439, 440, 441, 442, 443, -1, - -1, 446, 447, 448, -1, 450, 451, 452, 453, -1, - 455, 456, 457, 458, 459, 460, 461, -1, 463, 464, - 465, 466, 467, 468, 469, 470, 471, -1, -1, 474, - -1, -1, 477, 478, 479, 480, 481, 482, 483, 484, - 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, - 495, 496, 497, 3, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 514, - -1, -1, 22, 23, 24, 25, 26, 27, 28, 29, - -1, 31, 32, 33, -1, -1, -1, -1, -1, -1, - -1, -1, 42, 43, 44, -1, 46, 47, 48, 49, - 50, 51, 52, 53, 54, 55, 56, 57, -1, 59, - 60, 61, 62, 63, 64, -1, 66, 67, 68, 69, - 70, 71, 72, 73, -1, 75, 76, 77, 78, 79, - -1, 81, -1, 83, 84, 85, 86, 87, 88, 89, - 90, 91, 92, -1, 94, 95, 96, 97, 98, 99, - -1, 101, 102, 103, 104, 105, 106, 107, 108, 109, - 110, 111, 112, 113, 114, 115, 116, -1, 118, -1, - 120, 121, 122, 123, 124, 125, -1, -1, 128, 129, - 130, 131, -1, -1, 134, 135, 136, 137, 138, -1, - 140, 141, 142, -1, 144, 145, 146, -1, 148, 149, - 150, 151, 152, 153, 154, 155, 156, 157, 158, -1, - 160, -1, 162, 163, 164, 165, -1, 167, -1, 169, - 170, -1, 172, 173, 174, 175, 176, 177, -1, 179, - -1, 181, 182, 183, 184, -1, 186, 187, 188, 189, - 190, 191, 192, 193, 194, 195, 196, 197, -1, 199, - 200, 201, 202, 203, 204, 205, -1, 207, 208, 209, - 210, 211, 212, 213, 214, 215, 216, -1, 218, -1, - 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, - 230, -1, -1, 233, 234, 235, 236, -1, 238, 239, - 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, - 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, - 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, - 270, 271, 272, 273, -1, 275, 276, 277, 278, -1, - 280, 281, 282, 283, 284, 285, -1, 287, 288, -1, - -1, 291, 292, 293, -1, -1, 296, 297, 298, 299, - 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, -1, -1, -1, -1, 318, 319, - 320, 321, 322, 323, 324, 325, 326, 327, -1, 329, - 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, - 340, 341, 342, 343, -1, 345, 346, 347, 348, 349, - 350, 351, 352, 353, 354, 355, 356, 357, -1, 359, - 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, - 370, 371, 372, 373, 374, 375, 376, 377, 378, -1, - -1, 381, 382, 383, 384, 385, 386, 387, 388, 389, - -1, 391, 392, 393, 394, 395, -1, 397, 398, 399, - 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - 410, 411, -1, -1, 414, 415, -1, 417, 418, 419, - 420, 421, 422, 423, -1, 425, 426, 427, -1, -1, - 430, 431, 432, 433, 434, -1, 436, 437, 438, 439, - 440, 441, 442, 443, -1, -1, 446, 447, 448, -1, - 450, 451, 452, 453, -1, 455, 456, 457, 458, 459, - 460, 461, -1, 463, 464, 465, 466, 467, 468, 469, - 470, 471, -1, -1, 474, -1, -1, 477, 478, 479, - 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, - 490, 491, 492, 493, 494, 495, 496, 497, 3, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 514, -1, -1, 22, 23, 24, - 25, 26, 27, 28, 29, -1, 31, 32, 33, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 43, 44, - -1, 46, 47, 48, -1, 50, 51, 52, 53, 54, - -1, 56, 57, -1, 59, 60, 61, 62, 63, 64, - -1, -1, 67, 68, 69, 70, 71, 72, 73, -1, - 75, 76, 77, 78, 79, -1, -1, -1, 83, 84, - 85, 86, 87, 88, -1, 90, 91, 92, -1, 94, - 95, 96, 97, 98, 99, -1, -1, 102, 103, 104, - 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, - 115, 116, -1, 118, -1, 120, 121, 122, 123, 124, - 125, 126, 127, 128, 129, 130, 131, -1, -1, 134, - 135, 136, 137, 138, -1, 140, 141, 142, -1, 144, - 145, 146, -1, 148, 149, 150, 151, 152, 153, 154, - 155, 156, 157, 158, -1, 160, -1, 162, 163, 164, - 165, -1, 167, -1, 169, -1, 171, -1, 173, 174, - 175, -1, 177, -1, 179, -1, 181, 182, 183, 184, - -1, 186, 187, 188, 189, 190, 191, 192, -1, 194, - 195, 196, 197, -1, 199, 200, 201, 202, 203, 204, - 205, -1, 207, -1, 209, 210, 211, 212, 213, 214, - 215, 216, -1, 218, -1, 220, -1, -1, 223, -1, - 225, 226, 227, 228, 229, 230, -1, -1, 233, -1, - 235, -1, -1, 238, 239, 240, 241, 242, 243, 244, - 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, - 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, - 265, 266, 267, -1, 269, 270, 271, 272, 273, -1, - 275, 276, -1, 278, -1, 280, 281, 282, 283, 284, - 285, -1, 287, 288, -1, -1, 291, 292, 293, -1, - -1, 296, 297, 298, -1, 300, -1, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, -1, 318, 319, 320, -1, 322, 323, 324, - 325, 326, 327, -1, 329, 330, 331, 332, 333, 334, - -1, 336, 337, 338, 339, 340, 341, 342, 343, -1, - 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, - 355, 356, 357, -1, 359, 360, -1, 362, 363, 364, - 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, - 375, 376, 377, 378, 379, -1, 381, 382, 383, 384, - 385, 386, 387, 388, 389, 390, -1, 392, 393, 394, - 395, -1, 397, 398, 399, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, 412, -1, 414, - 415, 416, 417, -1, 419, 420, 421, 422, 423, -1, - 425, 426, 427, -1, -1, 430, 431, 432, 433, 434, - -1, 436, 437, 438, 439, 440, 441, 442, 443, -1, - -1, 446, 447, 448, 449, 450, 451, 452, 453, -1, - 455, 456, 457, 458, 459, 460, 461, -1, 463, -1, - 465, 466, 467, 468, 469, 470, 471, -1, -1, 474, - -1, 476, 477, 478, 479, 480, 481, 482, 483, 484, - 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, - 495, 496, 497, 3, -1, 500, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 514, - -1, -1, 22, 23, 24, 25, 26, 27, 28, 29, - 30, 31, 32, 33, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 43, 44, -1, 46, 47, 48, -1, - 50, 51, 52, 53, 54, -1, 56, 57, -1, 59, - 60, 61, 62, 63, 64, -1, -1, 67, 68, 69, - 70, 71, 72, 73, -1, 75, 76, 77, 78, 79, - -1, -1, -1, 83, 84, 85, 86, 87, 88, -1, - 90, 91, 92, -1, 94, 95, 96, 97, 98, 99, - -1, -1, 102, 103, 104, 105, 106, 107, 108, 109, - 110, 111, 112, 113, 114, 115, 116, -1, 118, -1, - 120, 121, 122, 123, 124, 125, -1, -1, 128, 129, - 130, 131, -1, -1, 134, 135, 136, 137, 138, -1, - 140, 141, 142, -1, 144, 145, 146, -1, 148, 149, - 150, 151, 152, 153, 154, 155, 156, 157, 158, -1, - 160, -1, 162, 163, 164, 165, -1, 167, -1, 169, - -1, 171, -1, 173, 174, 175, -1, 177, -1, 179, - -1, 181, 182, 183, 184, -1, 186, 187, 188, 189, - 190, 191, 192, -1, 194, 195, 196, 197, -1, 199, - 200, 201, 202, 203, 204, 205, -1, 207, -1, 209, - 210, 211, 212, 213, 214, 215, 216, -1, 218, -1, - 220, -1, -1, 223, -1, 225, 226, 227, 228, 229, - 230, -1, -1, 233, -1, 235, -1, -1, 238, 239, - 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, - 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, - 260, 261, 262, 263, 264, 265, 266, 267, -1, 269, - 270, 271, 272, 273, -1, 275, 276, -1, 278, -1, - 280, 281, 282, 283, 284, 285, -1, 287, 288, -1, - -1, 291, 292, 293, -1, -1, 296, 297, 298, -1, - 300, -1, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, -1, 318, 319, - 320, -1, 322, 323, 324, 325, 326, 327, -1, 329, - 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, - 340, 341, 342, 343, -1, 345, 346, 347, 348, 349, - 350, 351, 352, 353, 354, 355, 356, 357, -1, 359, - 360, -1, 362, 363, 364, 365, 366, 367, 368, 369, - 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, - -1, 381, 382, 383, 384, 385, 386, 387, 388, 389, - -1, -1, 392, 393, 394, 395, -1, 397, 398, 399, - 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - 410, 411, -1, -1, 414, 415, 416, 417, -1, 419, - 420, 421, 422, 423, -1, 425, 426, 427, -1, -1, - 430, 431, 432, 433, 434, -1, 436, 437, 438, 439, - 440, 441, 442, 443, -1, -1, 446, 447, 448, 449, - 450, 451, 452, 453, -1, 455, 456, 457, 458, 459, - 460, 461, -1, 463, -1, 465, 466, 467, 468, 469, - 470, 471, -1, -1, 474, -1, 476, 477, 478, 479, - 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, - 490, 491, 492, 493, 494, 495, 496, 497, 3, -1, - 500, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 514, -1, -1, 22, 23, 24, - 25, 26, 27, 28, 29, -1, 31, 32, 33, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 43, 44, - -1, 46, 47, 48, -1, 50, 51, 52, 53, 54, - -1, 56, 57, -1, 59, 60, 61, 62, 63, 64, - -1, -1, 67, 68, 69, 70, 71, 72, 73, -1, - 75, 76, 77, 78, 79, -1, -1, -1, 83, 84, - 85, 86, 87, 88, -1, 90, 91, 92, -1, 94, - 95, 96, 97, 98, 99, -1, -1, 102, 103, 104, - 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, - 115, 116, -1, 118, -1, 120, 121, 122, 123, 124, - 125, -1, -1, 128, 129, 130, 131, -1, -1, 134, - 135, 136, 137, 138, -1, 140, 141, 142, -1, 144, - 145, 146, -1, 148, 149, 150, 151, 152, 153, 154, - 155, 156, 157, 158, -1, 160, -1, 162, 163, 164, - 165, -1, 167, -1, 169, -1, 171, -1, 173, 174, - 175, -1, 177, -1, 179, -1, 181, 182, 183, 184, - -1, 186, 187, 188, 189, 190, 191, 192, -1, 194, - 195, 196, 197, -1, 199, 200, 201, 202, 203, 204, - 205, -1, 207, -1, 209, 210, 211, 212, 213, 214, - 215, 216, -1, 218, -1, 220, -1, -1, 223, -1, - 225, 226, 227, 228, 229, 230, -1, -1, 233, -1, - 235, -1, -1, 238, 239, 240, 241, 242, 243, 244, - 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, - 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, - 265, 266, 267, -1, 269, 270, 271, 272, 273, -1, - 275, 276, -1, 278, -1, 280, 281, 282, 283, 284, - 285, -1, 287, 288, -1, -1, 291, 292, 293, -1, - -1, 296, 297, 298, -1, 300, -1, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 315, 316, -1, 318, 319, 320, -1, 322, 323, 324, - 325, 326, 327, -1, 329, 330, 331, 332, 333, 334, - -1, 336, 337, 338, 339, 340, 341, 342, 343, -1, - 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, - 355, 356, 357, -1, 359, 360, -1, 362, 363, 364, - 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, - 375, 376, 377, 378, 379, -1, 381, 382, 383, 384, - 385, 386, 387, 388, 389, -1, -1, 392, 393, 394, - 395, -1, 397, 398, 399, 400, 401, 402, 403, 404, - 405, 406, 407, 408, 409, 410, 411, -1, -1, 414, - 415, 416, 417, -1, 419, 420, 421, 422, 423, -1, - 425, 426, 427, -1, -1, 430, 431, 432, 433, 434, - -1, 436, 437, 438, 439, 440, 441, 442, 443, -1, - -1, 446, 447, 448, 449, 450, 451, 452, 453, -1, - 455, 456, 457, 458, 459, 460, 461, -1, 463, -1, - 465, 466, 467, 468, 469, 470, 471, -1, -1, 474, - -1, 476, 477, 478, 479, 480, 481, 482, 483, 484, - 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, - 495, 496, 497, -1, 3, 500, 5, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 514, - -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, - 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 43, 44, -1, 46, 47, 48, - -1, 50, 51, 52, 53, 54, -1, 56, 57, -1, - 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, - 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, - 79, -1, -1, -1, 83, 84, 85, 86, 87, 88, - -1, 90, 91, 92, -1, 94, 95, 96, 97, 98, - 99, -1, -1, 102, 103, 104, 105, 106, 107, 108, - 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, - -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, - 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, - -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, - 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, - -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, - 169, -1, -1, -1, 173, 174, 175, -1, 177, -1, - 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, - 189, 190, 191, 192, -1, 194, 195, 196, 197, -1, - 199, 200, 201, 202, 203, 204, 205, -1, 207, -1, - 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, - -1, 220, -1, -1, 223, -1, 225, 226, 227, 228, - 229, 230, -1, -1, 233, -1, 235, -1, -1, 238, - 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, - 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, - 259, 260, 261, 262, 263, 264, 265, 266, 267, -1, - 269, 270, 271, 272, 273, -1, 275, 276, -1, 278, - -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, - -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, - -1, 300, -1, 302, 303, 304, 305, 306, 307, 308, - 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, - 319, 320, -1, 322, 323, 324, 325, 326, 327, -1, - 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, - 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, - 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, - 359, 360, -1, 362, 363, 364, 365, 366, 367, 368, - 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, - -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, - 389, -1, -1, 392, 393, 394, 395, -1, 397, 398, - 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, - 409, 410, 411, -1, -1, 414, 415, -1, 417, -1, - 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, - -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, - 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, - -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, - 459, 460, 461, -1, 463, -1, 465, 466, 467, 468, - 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, - 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, - 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, - -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 514, -1, -1, 22, 23, - 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 43, - 44, -1, 46, 47, 48, -1, 50, 51, 52, 53, - 54, -1, 56, 57, -1, 59, 60, 61, 62, 63, - 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, - -1, 75, 76, 77, 78, 79, -1, -1, -1, 83, - 84, 85, 86, 87, 88, -1, 90, 91, 92, -1, - 94, 95, 96, 97, 98, 99, -1, -1, 102, 103, - 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, - 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, - 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, - 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, - 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, - 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, - 164, 165, -1, 167, -1, 169, -1, -1, -1, 173, - 174, 175, -1, 177, -1, 179, -1, 181, 182, 183, - 184, -1, 186, 187, 188, 189, 190, 191, 192, -1, - 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, - 204, 205, -1, 207, -1, 209, 210, 211, 212, 213, - 214, 215, 216, -1, 218, -1, 220, -1, -1, 223, - -1, 225, 226, 227, 228, 229, 230, -1, -1, 233, - -1, 235, -1, -1, 238, 239, 240, 241, 242, 243, - 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, - 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, - 264, 265, 266, 267, -1, 269, 270, 271, 272, 273, - -1, 275, 276, -1, 278, -1, 280, 281, 282, 283, - 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, - -1, -1, 296, 297, 298, -1, 300, -1, 302, 303, - 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, - -1, -1, -1, -1, 318, 319, 320, -1, 322, 323, - 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, - 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, - -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, - 354, 355, 356, 357, -1, 359, 360, -1, 362, 363, - 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, - 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, - 384, 385, 386, 387, 388, 389, -1, -1, 392, 393, - 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, - 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, - 414, 415, -1, 417, -1, 419, 420, 421, 422, 423, - -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, - 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, - -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, - -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, - -1, 465, 466, 467, 468, 469, 470, 471, -1, -1, - 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, - 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, - 494, 495, 496, 497, 3, -1, 5, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 514, -1, -1, 22, 23, 24, 25, 26, 27, 28, - 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 43, 44, -1, 46, 47, 48, - -1, 50, 51, 52, 53, 54, -1, 56, 57, -1, - 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, - 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, - 79, -1, -1, -1, 83, 84, 85, 86, 87, 88, - -1, 90, 91, 92, -1, 94, 95, 96, 97, 98, - 99, -1, -1, 102, 103, 104, 105, 106, 107, 108, - 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, - -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, - 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, - -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, - 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, - -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, - 169, -1, -1, -1, 173, 174, 175, -1, 177, -1, - 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, - 189, 190, 191, 192, -1, 194, 195, 196, 197, -1, - 199, 200, 201, 202, 203, 204, 205, -1, 207, -1, - 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, - -1, 220, -1, -1, 223, -1, 225, 226, 227, 228, - 229, 230, -1, -1, 233, -1, 235, -1, -1, 238, - 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, - 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, - 259, 260, 261, 262, 263, 264, 265, 266, 267, -1, - 269, 270, 271, 272, 273, -1, 275, 276, -1, 278, - -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, - -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, - -1, 300, -1, 302, 303, 304, 305, 306, 307, 308, - 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, - 319, 320, -1, 322, 323, 324, 325, 326, 327, -1, - 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, - 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, - 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, - 359, 360, -1, 362, 363, 364, 365, 366, 367, 368, - 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, - -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, - 389, -1, -1, 392, 393, 394, 395, -1, 397, 398, - 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, - 409, 410, 411, -1, -1, 414, 415, -1, 417, -1, - 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, - -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, - 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, - -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, - 459, 460, 461, -1, 463, -1, 465, 466, 467, 468, - 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, - 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, - 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, - -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 514, -1, -1, 22, 23, - 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 43, - 44, -1, 46, 47, 48, -1, 50, 51, 52, 53, - 54, -1, 56, 57, -1, 59, 60, 61, 62, 63, - 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, - -1, 75, 76, 77, 78, 79, -1, -1, -1, 83, - 84, 85, 86, 87, 88, -1, 90, 91, 92, -1, - 94, 95, 96, 97, 98, 99, -1, -1, 102, 103, - 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, - 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, - 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, - 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, - 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, - 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, - 164, 165, -1, 167, -1, 169, -1, -1, -1, 173, - 174, 175, -1, 177, -1, 179, -1, 181, 182, 183, - 184, -1, 186, 187, 188, 189, 190, 191, 192, -1, - 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, - 204, 205, -1, 207, -1, 209, 210, 211, 212, 213, - 214, 215, 216, -1, 218, -1, 220, -1, -1, 223, - -1, 225, 226, 227, 228, 229, 230, -1, -1, 233, - -1, 235, -1, -1, 238, 239, 240, 241, 242, 243, - 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, - 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, - 264, 265, 266, 267, -1, 269, 270, 271, 272, 273, - -1, 275, 276, -1, 278, -1, 280, 281, 282, 283, - 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, - -1, -1, 296, 297, 298, -1, 300, -1, 302, 303, - 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, - -1, -1, -1, -1, 318, 319, 320, -1, 322, 323, - 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, - 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, - -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, - 354, 355, 356, 357, -1, 359, 360, -1, 362, 363, - 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, - 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, - 384, 385, 386, 387, 388, 389, -1, -1, 392, 393, - 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, - 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, - 414, 415, -1, 417, -1, 419, 420, 421, 422, 423, - -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, - 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, - -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, - -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, - -1, 465, 466, 467, 468, 469, 470, 471, -1, -1, - 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, - 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, - 494, 495, 496, 497, 3, -1, 5, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 514, -1, -1, 22, 23, 24, 25, 26, 27, 28, - 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 43, 44, -1, 46, 47, 48, - -1, 50, 51, 52, 53, 54, -1, 56, 57, -1, - 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, - 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, - 79, -1, -1, -1, 83, 84, 85, 86, 87, 88, - -1, 90, 91, 92, -1, 94, 95, 96, 97, 98, - 99, -1, -1, 102, 103, 104, 105, 106, 107, 108, - 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, - -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, - 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, - -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, - 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, - -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, - 169, -1, -1, -1, 173, 174, 175, -1, 177, -1, - 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, - 189, 190, 191, 192, -1, 194, 195, 196, 197, -1, - 199, 200, 201, 202, 203, 204, 205, -1, 207, -1, - 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, - -1, 220, -1, -1, 223, -1, 225, 226, 227, 228, - 229, 230, -1, -1, 233, -1, 235, -1, -1, 238, - 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, - 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, - 259, 260, 261, 262, 263, 264, 265, 266, 267, -1, - 269, 270, 271, 272, 273, -1, 275, 276, -1, 278, - -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, - -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, - -1, 300, -1, 302, 303, 304, 305, 306, 307, 308, - 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, - 319, 320, -1, 322, 323, 324, 325, 326, 327, -1, - 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, - 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, - 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, - 359, 360, -1, 362, 363, 364, 365, 366, 367, 368, - 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, - -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, - 389, -1, -1, 392, 393, 394, 395, -1, 397, 398, - 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, - 409, 410, 411, -1, -1, 414, 415, -1, 417, -1, - 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, - -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, - 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, - -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, - 459, 460, 461, -1, 463, -1, 465, 466, 467, 468, - 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, - 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, - 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, - -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 514, -1, -1, 22, 23, - 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 43, - 44, -1, 46, 47, 48, -1, 50, 51, 52, 53, - 54, -1, 56, 57, -1, 59, 60, 61, 62, 63, - 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, - -1, 75, 76, 77, 78, 79, -1, -1, -1, 83, - 84, 85, 86, 87, 88, -1, 90, 91, 92, -1, - 94, 95, 96, 97, 98, 99, -1, -1, 102, 103, - 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, - 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, - 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, - 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, - 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, - 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, - 164, 165, -1, 167, -1, 169, -1, -1, -1, 173, - 174, 175, -1, 177, -1, 179, -1, 181, 182, 183, - 184, -1, 186, 187, 188, 189, 190, 191, 192, -1, - 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, - 204, 205, -1, 207, -1, 209, 210, 211, 212, 213, - 214, 215, 216, -1, 218, -1, 220, -1, -1, 223, - -1, 225, 226, 227, 228, 229, 230, -1, -1, 233, - -1, 235, -1, -1, 238, 239, 240, 241, 242, 243, - 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, - 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, - 264, 265, 266, 267, -1, 269, 270, 271, 272, 273, - -1, 275, 276, -1, 278, -1, 280, 281, 282, 283, - 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, - -1, -1, 296, 297, 298, -1, 300, -1, 302, 303, - 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, - -1, -1, -1, -1, 318, 319, 320, -1, 322, 323, - 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, - 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, - -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, - 354, 355, 356, 357, -1, 359, 360, -1, 362, 363, - 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, - 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, - 384, 385, 386, 387, 388, 389, -1, -1, 392, 393, - 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, - 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, - 414, 415, -1, 417, -1, 419, 420, 421, 422, 423, - -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, - 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, - -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, - -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, - -1, 465, 466, 467, 468, 469, 470, 471, -1, -1, - 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, - 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, - 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 514, -1, -1, 22, 23, 24, 25, 26, 27, 28, - 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 43, 44, -1, 46, 47, 48, - -1, 50, 51, 52, 53, 54, -1, 56, 57, -1, - 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, - 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, - 79, -1, -1, -1, 83, 84, 85, 86, 87, 88, - -1, 90, 91, 92, -1, 94, 95, 96, 97, 98, - 99, -1, -1, 102, 103, 104, 105, 106, 107, 108, - 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, - -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, - 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, - -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, - 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, - -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, - 169, -1, -1, -1, 173, 174, 175, -1, 177, -1, - 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, - 189, 190, 191, 192, -1, 194, 195, 196, 197, -1, - 199, 200, 201, 202, 203, 204, 205, -1, 207, -1, - 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, - -1, 220, -1, -1, 223, -1, 225, 226, 227, 228, - 229, 230, -1, -1, 233, -1, 235, -1, -1, 238, - 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, - 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, - 259, 260, 261, 262, 263, 264, 265, 266, 267, -1, - 269, 270, 271, 272, 273, -1, 275, 276, -1, 278, - -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, - -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, - -1, 300, -1, 302, 303, 304, 305, 306, 307, 308, - 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, - 319, 320, -1, 322, 323, 324, 325, 326, 327, -1, - 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, - 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, - 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, - 359, 360, -1, 362, 363, 364, 365, 366, 367, 368, - 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, - -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, - 389, -1, -1, 392, 393, 394, 395, -1, 397, 398, - 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, - 409, 410, 411, -1, -1, 414, 415, -1, 417, -1, - 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, - -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, - 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, - -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, - 459, 460, 461, -1, 463, -1, 465, 466, 467, 468, - 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, - 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, - 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 514, -1, -1, 22, 23, - 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 43, - 44, -1, 46, 47, 48, -1, 50, 51, 52, 53, - 54, -1, 56, 57, -1, 59, 60, 61, 62, 63, - 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, - -1, 75, 76, 77, 78, 79, -1, -1, -1, 83, - 84, 85, 86, 87, 88, -1, 90, 91, 92, -1, - 94, 95, 96, 97, 98, 99, -1, -1, 102, 103, - 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, - 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, - 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, - 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, - 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, - 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, - 164, 165, -1, 167, -1, 169, -1, -1, -1, 173, - 174, 175, -1, 177, -1, 179, -1, 181, 182, 183, - 184, -1, 186, 187, 188, 189, 190, 191, 192, -1, - 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, - 204, 205, -1, 207, -1, 209, 210, 211, 212, 213, - 214, 215, 216, -1, 218, -1, 220, -1, -1, 223, - -1, 225, 226, 227, 228, 229, 230, -1, -1, 233, - -1, 235, -1, -1, 238, 239, 240, 241, 242, 243, - 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, - 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, - 264, 265, 266, 267, -1, 269, 270, 271, 272, 273, - -1, 275, 276, -1, 278, -1, 280, 281, 282, 283, - 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, - -1, -1, 296, 297, 298, -1, 300, -1, 302, 303, - 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, - -1, -1, -1, -1, 318, 319, 320, -1, 322, 323, - 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, - 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, - -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, - 354, 355, 356, 357, -1, 359, 360, -1, 362, 363, - 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, - 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, - 384, 385, 386, 387, 388, 389, -1, -1, 392, 393, - 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, - 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, - 414, 415, -1, 417, -1, 419, 420, 421, 422, 423, - -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, - 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, - -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, - -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, - -1, 465, 466, 467, 468, 469, 470, 471, -1, -1, - 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, - 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, - 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 514, -1, -1, 22, 23, 24, 25, 26, 27, 28, - 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 43, 44, -1, 46, 47, 48, - -1, 50, 51, 52, 53, 54, -1, 56, 57, -1, - 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, - 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, - 79, -1, -1, -1, 83, 84, 85, 86, 87, 88, - -1, 90, 91, 92, -1, 94, 95, 96, 97, 98, - 99, -1, -1, 102, 103, 104, 105, 106, 107, 108, - 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, - -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, - 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, - -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, - 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, - -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, - 169, -1, -1, -1, 173, 174, 175, -1, 177, -1, - 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, - 189, 190, 191, 192, -1, 194, 195, 196, 197, -1, - 199, 200, 201, 202, 203, 204, 205, -1, 207, -1, - 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, - -1, 220, -1, -1, 223, -1, 225, 226, 227, 228, - 229, 230, -1, -1, 233, -1, 235, -1, -1, 238, - 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, - 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, - 259, 260, 261, 262, 263, 264, 265, 266, 267, -1, - 269, 270, 271, 272, 273, -1, 275, 276, -1, 278, - -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, - -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, - -1, 300, -1, 302, 303, 304, 305, 306, 307, 308, - 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, - 319, 320, -1, 322, 323, 324, 325, 326, 327, -1, - 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, - 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, - 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, - 359, 360, -1, 362, 363, 364, 365, 366, 367, 368, - 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, - -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, - 389, -1, -1, 392, 393, 394, 395, -1, 397, 398, - 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, - 409, 410, 411, -1, -1, 414, 415, -1, 417, -1, - 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, - -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, - 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, - -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, - 459, 460, 461, -1, 463, -1, 465, 466, 467, 468, - 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, - 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, - 489, 490, 491, 492, 493, 494, 495, 496, 497, -1, - 3, 4, 5, -1, -1, 8, 9, -1, -1, -1, - -1, -1, 15, 16, -1, 514, 19, 20, 21, 22, - 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, - 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, - 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, - -1, 54, 55, 56, 57, 58, 59, 60, 61, 62, - 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, - 73, 74, 75, 76, 77, 78, -1, 80, 81, 82, - 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, - 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, - 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, - 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, - 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, - 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, - 143, 144, 145, 146, 147, 148, 149, 150, 151, -1, - 153, 154, 155, 156, 157, -1, 159, 160, 161, 162, - 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, - 173, 174, 175, 176, 177, 178, 179, 180, -1, -1, - 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, - 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, - 203, 204, 205, 206, 207, 208, -1, 210, 211, 212, - 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, - 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, - 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, - 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, - 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, - 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, - 273, 274, 275, 276, 277, 278, 279, -1, 281, 282, - 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, - 293, 294, 295, 296, 297, -1, 299, 300, 301, -1, - 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, - 313, 314, 315, 316, 317, 318, 319, -1, 321, 322, - 323, -1, 325, 326, 327, 328, 329, 330, 331, 332, - 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, - 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, - 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, - 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, - 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, - 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, - 403, 404, 405, 406, 407, 408, 409, 410, -1, 412, - 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, - 423, 424, 425, 426, 427, 428, 429, 430, 431, -1, - 433, -1, 435, 436, 437, 438, 439, 440, 441, 442, - 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, - 453, 454, 455, 456, 457, 458, 459, -1, 461, 462, - 463, 464, 465, 466, 467, 468, 469, 470, 471, 472, - 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 494, 495, 496, 497, -1, 3, -1, 501, 502, - 503, 8, 505, 506, 507, 508, 509, 510, 15, 16, - -1, -1, 19, 20, 21, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 43, 44, -1, 46, - 47, 48, -1, 50, 51, 52, 53, 54, -1, 56, - 57, -1, 59, 60, 61, 62, 63, 64, -1, -1, - 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, 79, -1, -1, -1, 83, 84, 85, 86, - 87, 88, -1, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, -1, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, - -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, -1, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, -1, -1, -1, 173, 174, 175, -1, - 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, -1, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, -1, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, -1, -1, 223, -1, 225, 226, - 227, 228, 229, 230, -1, -1, 233, -1, 235, -1, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, -1, 269, 270, 271, 272, 273, -1, 275, 276, - -1, 278, -1, 280, 281, 282, 283, 284, 285, -1, - 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, 298, -1, 300, -1, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, -1, 322, 323, 324, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, -1, 362, 363, 364, 365, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, -1, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, -1, 392, 393, 394, 395, -1, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, - 417, -1, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, -1, 436, - 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, -1, 463, -1, 465, 466, - 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, - 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, - 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, -1, -1, -1, 501, 502, 503, -1, 505, 506, - 507, 508, 509, 510, 8, -1, -1, 11, -1, -1, - -1, 15, 16, 17, 18, 19, 20, 21, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 36, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 46, 8, -1, -1, 11, -1, -1, 53, - 15, 16, 17, 18, 19, 20, 21, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 36, -1, -1, -1, -1, 80, -1, -1, -1, - -1, 46, 8, -1, -1, 11, -1, -1, 53, 15, - 16, 17, 18, 19, 20, 21, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 36, -1, -1, -1, -1, 80, -1, -1, -1, -1, - 46, -1, -1, -1, -1, -1, -1, 53, -1, 8, - -1, -1, 11, -1, -1, -1, 15, 16, 17, 18, - 19, 20, 21, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 80, -1, -1, 36, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 46, 8, -1, - -1, 11, 176, -1, 53, 15, 16, 17, 18, 19, - 20, 21, -1, -1, -1, -1, -1, -1, -1, 193, - -1, -1, -1, -1, 198, -1, 36, -1, -1, -1, - -1, 80, -1, -1, -1, -1, 46, -1, -1, -1, - -1, 176, -1, 53, -1, -1, -1, 221, 222, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 193, -1, - -1, -1, 236, 198, -1, -1, -1, -1, -1, -1, - 80, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 176, -1, -1, -1, -1, -1, 221, 222, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 193, -1, -1, - 274, 236, 198, 277, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 291, -1, -1, - 294, -1, -1, -1, -1, 221, 222, 176, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 274, - 236, -1, 277, -1, 193, -1, -1, -1, -1, 198, - -1, -1, -1, -1, -1, -1, 291, -1, -1, 294, - -1, -1, -1, -1, -1, -1, 176, -1, -1, -1, - -1, -1, 221, 222, -1, -1, -1, -1, 274, -1, - -1, 277, -1, 193, -1, -1, -1, 236, 198, -1, - -1, -1, -1, -1, -1, 291, -1, -1, 294, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 221, 222, -1, -1, -1, -1, 391, -1, -1, - -1, -1, -1, -1, -1, 274, 236, -1, 277, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 291, -1, -1, 294, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 391, -1, -1, -1, - -1, -1, -1, -1, 274, -1, -1, 277, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 291, -1, -1, 294, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 391, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 498, -1, -1, 501, 502, 503, - -1, 505, 506, 507, 508, 509, 510, -1, -1, -1, - -1, 515, 391, -1, -1, -1, -1, -1, 8, -1, - -1, 11, -1, -1, -1, 15, 16, 17, 18, 19, - 20, 21, -1, 498, -1, -1, 501, 502, 503, -1, - 505, 506, 507, 508, 509, 510, 36, -1, -1, -1, - 515, 391, -1, -1, -1, -1, 46, -1, -1, -1, - -1, -1, -1, 53, -1, -1, -1, -1, -1, -1, - -1, -1, 498, -1, -1, 501, 502, 503, -1, 505, - 506, 507, 508, 509, 510, -1, -1, -1, -1, 515, - 80, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 8, -1, -1, 11, -1, -1, - -1, 15, 16, 17, 18, 19, 20, 21, -1, 498, - -1, -1, 501, 502, 503, -1, 505, 506, 507, 508, - 509, 510, 36, -1, -1, -1, 515, -1, -1, -1, - -1, -1, 46, 8, -1, -1, 11, -1, -1, 53, - 15, 16, 17, 18, 19, 20, 21, -1, 498, -1, - -1, 501, 502, 503, -1, 505, 506, 507, 508, 509, - 510, 36, -1, -1, -1, 515, 80, -1, -1, -1, - -1, 46, 8, -1, -1, 11, 176, -1, 53, 15, - 16, 17, 18, 19, 20, 21, -1, -1, -1, -1, - -1, -1, -1, 193, -1, -1, -1, -1, 198, -1, - 36, -1, -1, -1, -1, 80, -1, -1, -1, -1, - 46, -1, -1, -1, -1, -1, -1, 53, -1, -1, - -1, 221, 222, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 236, -1, -1, -1, - -1, -1, -1, -1, 80, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 8, -1, - -1, 11, 176, -1, -1, 15, 16, 17, 18, 19, - 20, 21, -1, -1, 274, -1, -1, 277, -1, 193, - -1, -1, -1, -1, 198, -1, 36, -1, -1, -1, - -1, 291, -1, -1, 294, -1, 46, -1, -1, -1, - -1, 176, -1, 53, -1, -1, -1, 221, 222, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 193, -1, - -1, -1, 236, 198, -1, -1, -1, -1, -1, -1, - 80, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 176, -1, -1, -1, -1, -1, 221, 222, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 193, -1, -1, - 274, 236, 198, 277, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 291, -1, -1, - 294, -1, -1, -1, -1, 221, 222, -1, -1, -1, - -1, 391, -1, -1, -1, -1, -1, -1, -1, 274, - 236, -1, 277, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 291, -1, -1, 294, - -1, -1, -1, -1, -1, -1, 176, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 274, -1, - -1, 277, -1, 193, -1, -1, -1, -1, 198, -1, - -1, -1, -1, -1, -1, 291, -1, -1, 294, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 221, 222, -1, -1, -1, -1, 391, -1, -1, - -1, -1, -1, -1, -1, -1, 236, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 498, -1, - -1, 501, 502, 503, -1, 505, 506, 507, 508, 509, - 510, -1, -1, -1, -1, 515, 391, -1, -1, -1, - -1, -1, -1, -1, 274, -1, 8, 277, -1, 11, - -1, -1, -1, 15, 16, 17, 18, 19, 20, 21, - -1, 291, -1, -1, 294, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 36, 391, 24, -1, -1, -1, - -1, -1, -1, -1, 46, -1, -1, -1, -1, -1, - -1, 53, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 498, -1, -1, 501, 502, 503, - -1, 505, 506, 507, 508, 509, 510, -1, 80, -1, - -1, 515, -1, -1, -1, -1, -1, -1, 8, -1, - -1, 11, -1, 81, -1, 15, 16, 17, 18, 19, - 20, 21, -1, 498, -1, -1, 501, 502, 503, 97, - 505, 506, 507, 508, 509, 510, 36, -1, -1, -1, - 515, 391, -1, -1, -1, -1, 46, -1, -1, -1, - -1, -1, -1, 53, -1, -1, -1, -1, -1, -1, - -1, -1, 498, -1, -1, 501, 502, 503, -1, 505, - 506, 507, 508, 509, 510, -1, -1, 513, 146, -1, - 80, -1, -1, -1, -1, -1, -1, -1, 156, -1, - -1, -1, -1, -1, 176, -1, -1, -1, -1, -1, - 168, -1, -1, -1, -1, 173, -1, -1, -1, -1, - -1, 193, -1, -1, -1, -1, 198, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 202, -1, -1, -1, -1, 221, - 222, -1, -1, -1, -1, -1, -1, -1, 498, -1, - -1, 501, 502, 503, 236, 505, 506, 507, 508, 509, - 510, -1, -1, 513, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 176, 245, -1, -1, - -1, 249, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 274, 193, -1, 277, -1, -1, 198, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 291, - -1, -1, 294, -1, -1, -1, -1, -1, -1, -1, - -1, 221, 222, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 236, -1, -1, -1, - -1, -1, -1, -1, -1, 313, -1, -1, -1, -1, - -1, 319, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 334, -1, -1, -1, - -1, -1, -1, -1, 274, -1, -1, 277, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 291, -1, -1, 294, -1, -1, -1, -1, 367, - -1, -1, 370, -1, -1, -1, -1, -1, -1, 391, - -1, -1, -1, 381, -1, -1, 384, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 402, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 416, -1, - -1, -1, -1, -1, 422, 423, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 433, -1, -1, -1, -1, - -1, 439, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 391, -1, -1, -1, -1, -1, -1, 466, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 498, -1, -1, 501, - 502, 503, -1, 505, 506, 507, 508, 509, 510, -1, - -1, 513, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 3, -1, -1, 498, -1, - -1, 501, 502, 503, -1, 505, 506, 507, 508, 509, - 510, -1, -1, 513, 22, 23, 24, 25, 26, 27, - 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, - 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, - 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, - 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, - 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, - 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, - 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, - 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, - 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, - 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, - 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, - 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, - 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, - 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, - 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, - 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, - 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, - 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, - 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, - 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, - 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, - 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, - 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, - 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, - 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, - 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, - 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, - 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, - 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, - 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, - 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, - 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, - 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, - 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, - 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, - 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, - 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, - 428, 429, 430, 431, 432, 433, 434, 435, 436, 437, - 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, - 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, - 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, - 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, - 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, - 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, - 3, -1, -1, -1, -1, -1, -1, -1, -1, 507, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 22, - 23, 24, 25, 26, 27, 28, 29, -1, 31, 32, - 33, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 43, 44, -1, 46, 47, 48, -1, 50, 51, 52, - 53, 54, -1, 56, 57, -1, 59, 60, 61, 62, - 63, 64, -1, -1, 67, 68, 69, 70, 71, 72, - 73, -1, 75, 76, 77, 78, 79, -1, -1, -1, - 83, 84, 85, 86, 87, 88, -1, 90, 91, 92, - -1, 94, 95, 96, 97, 98, 99, -1, -1, 102, - 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, - 113, 114, 115, 116, -1, 118, -1, 120, 121, 122, - 123, 124, 125, -1, -1, 128, 129, 130, 131, -1, - -1, 134, 135, 136, 137, 138, -1, 140, 141, 142, - -1, 144, 145, 146, -1, 148, 149, 150, 151, 152, - 153, 154, 155, 156, 157, 158, -1, 160, -1, 162, - 163, 164, 165, -1, 167, -1, 169, -1, -1, -1, - 173, 174, 175, -1, 177, -1, 179, -1, 181, 182, - 183, 184, -1, 186, 187, 188, 189, 190, 191, 192, - -1, 194, 195, 196, 197, -1, 199, 200, 201, 202, - 203, 204, 205, -1, 207, -1, 209, 210, 211, 212, - 213, 214, 215, 216, -1, 218, -1, 220, -1, -1, - 223, -1, 225, 226, 227, 228, 229, 230, -1, -1, - 233, -1, 235, -1, -1, 238, 239, 240, 241, 242, - 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, - 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, - 263, 264, 265, 266, 267, -1, 269, 270, 271, 272, - 273, -1, 275, 276, -1, 278, -1, 280, 281, 282, - 283, 284, 285, -1, 287, 288, -1, -1, 291, 292, - 293, -1, -1, 296, 297, 298, -1, 300, -1, 302, - 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, - 313, -1, -1, -1, -1, 318, 319, 320, -1, 322, - 323, 324, 325, 326, 327, -1, 329, 330, 331, 332, - 333, 334, -1, 336, 337, 338, 339, 340, 341, 342, - 343, -1, 345, 346, 347, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, -1, 359, 360, -1, 362, - 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, - 373, 374, 375, 376, 377, 378, -1, -1, 381, 382, - 383, 384, 385, 386, 387, 388, 389, -1, -1, 392, - 393, 394, 395, -1, 397, 398, 399, 400, 401, 402, - 403, 404, 405, 406, 407, 408, 409, 410, 411, -1, - -1, 414, 415, -1, 417, -1, 419, 420, 421, 422, - 423, -1, 425, 426, 427, -1, -1, 430, 431, 432, - 433, 434, -1, 436, 437, 438, 439, 440, 441, 442, - 443, -1, -1, 446, 447, 448, -1, 450, 451, 452, - 453, -1, 455, 456, 457, 458, 459, 460, 461, -1, - 463, -1, 465, 466, 467, 468, 469, 470, 471, -1, - -1, 474, -1, -1, 477, 478, 479, 480, 481, 482, - 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, - 493, 494, 495, 496, 497, 3, -1, -1, -1, -1, - -1, -1, -1, -1, 507, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 22, 23, 24, 25, 26, 27, - 28, 29, -1, 31, 32, 33, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 43, 44, -1, 46, 47, - 48, -1, 50, 51, 52, 53, 54, -1, 56, 57, - -1, 59, 60, 61, 62, 63, 64, -1, -1, 67, - 68, 69, 70, 71, 72, 73, -1, 75, 76, 77, - 78, 79, -1, -1, -1, 83, 84, 85, 86, 87, - 88, -1, 90, 91, 92, -1, 94, 95, 96, 97, - 98, 99, -1, -1, 102, 103, 104, 105, 106, 107, - 108, 109, 110, 111, 112, 113, 114, 115, 116, -1, - 118, -1, 120, 121, 122, 123, 124, 125, -1, -1, - 128, 129, 130, 131, -1, -1, 134, 135, 136, 137, - 138, -1, 140, 141, 142, -1, 144, 145, 146, -1, - 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, - 158, -1, 160, -1, 162, 163, 164, 165, -1, 167, - -1, 169, -1, -1, -1, 173, 174, 175, -1, 177, - -1, 179, -1, 181, 182, 183, 184, -1, 186, 187, - 188, 189, 190, 191, 192, -1, 194, 195, 196, 197, - -1, 199, 200, 201, 202, 203, 204, 205, -1, 207, - -1, 209, 210, 211, 212, 213, 214, 215, 216, -1, - 218, -1, 220, -1, -1, 223, -1, 225, 226, 227, - 228, 229, 230, -1, -1, 233, -1, 235, -1, -1, - 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, - 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, - 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, - -1, 269, 270, 271, 272, 273, -1, 275, 276, -1, - 278, -1, 280, 281, 282, 283, 284, 285, -1, 287, - 288, -1, -1, 291, 292, 293, -1, -1, 296, 297, - 298, -1, 300, -1, 302, 303, 304, 305, 306, 307, - 308, 309, 310, 311, 312, 313, -1, -1, -1, -1, - 318, 319, 320, -1, 322, 323, 324, 325, 326, 327, - -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, - 338, 339, 340, 341, 342, 343, -1, 345, 346, 347, - 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, - -1, 359, 360, -1, 362, 363, 364, 365, 366, 367, - 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, - 378, -1, -1, 381, 382, 383, 384, 385, 386, 387, - 388, 389, -1, -1, 392, 393, 394, 395, -1, 397, - 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, - 408, 409, 410, 411, -1, -1, 414, 415, -1, 417, - -1, 419, 420, 421, 422, 423, -1, 425, 426, 427, - -1, -1, 430, 431, 432, 433, 434, -1, 436, 437, - 438, 439, 440, 441, 442, 443, -1, -1, 446, 447, - 448, -1, 450, 451, 452, 453, -1, 455, 456, 457, - 458, 459, 460, 461, -1, 463, -1, 465, 466, 467, - 468, 469, 470, 471, -1, -1, 474, -1, -1, 477, - 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, - 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, - 3, 4, 5, -1, -1, -1, 9, -1, -1, 507, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 22, - 23, 24, 25, 26, 27, 28, 29, -1, 31, 32, - 33, -1, -1, -1, 37, -1, -1, -1, -1, 42, - 43, 44, -1, 46, 47, 48, 49, 50, 51, 52, - 53, 54, 55, 56, 57, -1, 59, 60, 61, 62, - 63, 64, -1, -1, 67, 68, 69, 70, 71, 72, - 73, -1, 75, 76, 77, 78, 79, -1, 81, -1, - 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, - -1, 94, 95, 96, 97, 98, 99, -1, 101, 102, - 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, - 113, 114, 115, 116, -1, 118, -1, 120, 121, 122, - 123, 124, 125, -1, -1, 128, 129, 130, 131, -1, - -1, 134, 135, 136, 137, 138, -1, 140, 141, 142, - -1, 144, 145, 146, -1, 148, 149, 150, 151, 152, - 153, 154, 155, 156, 157, 158, 159, 160, -1, 162, - 163, 164, 165, -1, 167, -1, 169, 170, -1, 172, - 173, 174, 175, 176, 177, -1, 179, -1, 181, 182, - 183, 184, -1, 186, 187, 188, 189, 190, 191, 192, - 193, 194, 195, 196, 197, -1, 199, 200, 201, 202, - 203, 204, 205, -1, 207, 208, 209, 210, 211, 212, - 213, 214, 215, 216, -1, 218, -1, 220, 221, 222, - 223, 224, 225, 226, 227, 228, 229, 230, -1, -1, - 233, 234, 235, 236, -1, 238, 239, 240, 241, 242, - 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, - 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, - 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, - 273, -1, 275, 276, 277, 278, -1, 280, 281, 282, - 283, 284, 285, -1, 287, 288, 289, -1, 291, 292, - 293, -1, -1, 296, 297, 298, 299, 300, 301, 302, - 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, - 313, -1, -1, -1, -1, 318, 319, 320, 321, 322, - 323, 324, 325, 326, 327, -1, 329, 330, 331, 332, - 333, 334, -1, 336, 337, 338, 339, 340, 341, 342, - 343, -1, 345, 346, 347, 348, 349, 350, 351, 352, - 353, 354, 355, 356, 357, -1, 359, 360, 361, 362, - 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, - 373, 374, 375, 376, 377, 378, -1, 380, 381, 382, - 383, 384, 385, 386, 387, 388, 389, -1, 391, 392, - 393, 394, 395, -1, 397, 398, 399, 400, 401, 402, - 403, 404, 405, 406, 407, 408, 409, 410, 411, -1, - -1, 414, 415, -1, 417, 418, 419, 420, 421, 422, - 423, -1, 425, 426, 427, -1, -1, 430, 431, 432, + 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 0, 0, 427, 428, + 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 526, 444, 445, 0, 0, + 446, 447, 0, 448, 0, 450, 451, 452, 453, 454, + 0, 455, 456, 457, 0, 0, 458, 459, 460, 461, + 462, 0, 463, 464, 465, 466, 467, 468, 469, 470, + 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 0, 485, + 0, 487, 488, 489, 490, 491, 492, 493, 0, 0, + 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, + 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, + 512, 513, 514, 515, 523, 0, 549, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, + 0, 135, 136, 137, 138, 139, 0, 141, 142, 0, + 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, + 151, 152, 153, 154, 155, 0, 156, 157, 158, 159, + 160, 0, 0, 0, 162, 163, 164, 165, 166, 167, + 0, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 0, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, + 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, + 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, + 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + 0, 227, 0, 228, 229, 230, 231, 0, 232, 0, + 233, 0, 0, 0, 236, 237, 524, 0, 1951, 0, + 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, + 249, 250, 251, 252, 0, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 0, + 267, 268, 269, 270, 271, 272, 273, 274, 0, 275, + 0, 276, 0, 0, 279, 0, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 0, 289, 0, 0, 291, + 292, 1952, 294, 295, 296, 297, 298, 525, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 0, + 322, 323, 324, 325, 326, 0, 327, 328, 0, 330, + 0, 331, 332, 333, 334, 335, 336, 0, 337, 338, + 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, + 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, + 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, + 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, + 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 526, 444, 445, 0, 0, 446, 447, 1953, 448, 0, + 450, 1954, 452, 1955, 454, 0, 455, 456, 457, 0, + 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, + 466, 467, 468, 469, 470, 0, 0, 471, 472, 1956, + 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 0, 485, 0, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, 512, 513, 514, 515, 523, + 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, + 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, + 130, 0, 131, 132, 133, 0, 135, 136, 137, 138, + 139, 0, 141, 142, 0, 143, 144, 145, 146, 147, + 148, 0, 0, 149, 150, 151, 152, 153, 154, 155, + 0, 156, 157, 158, 159, 160, 0, 0, 0, 162, + 163, 164, 165, 166, 167, 0, 169, 170, 171, 0, + 172, 173, 174, 175, 176, 177, 0, 0, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, + 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, + 213, 214, 215, 0, 216, 217, 218, 219, 220, 221, + 222, 223, 224, 225, 226, 0, 227, 0, 228, 229, + 230, 231, 0, 232, 0, 233, 0, 0, 0, 236, + 237, 524, 0, 240, 0, 241, 0, 242, 243, 244, + 245, 0, 246, 247, 248, 249, 250, 251, 252, 0, + 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 0, 267, 268, 269, 270, 271, + 272, 273, 274, 0, 275, 0, 276, 0, 0, 279, + 0, 281, 282, 283, 284, 285, 286, 0, 0, 287, + 0, 289, 0, 0, 291, 292, 293, 294, 295, 296, + 297, 298, 525, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 0, 322, 323, 324, 325, 326, + 0, 327, 328, 0, 330, 0, 331, 332, 333, 334, + 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, + 0, 0, 342, 343, 344, 0, 346, 0, 348, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 0, 0, 0, 0, 360, 361, 362, 0, 364, 365, + 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, + 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, + 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 397, 398, 0, 400, 401, + 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, + 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 0, 0, 427, 428, + 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 526, 444, 445, 0, 0, + 446, 447, 0, 448, 0, 450, 451, 452, 453, 454, + 0, 455, 456, 457, 0, 0, 458, 459, 460, 461, + 462, 0, 463, 464, 465, 466, 467, 468, 469, 470, + 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 0, 485, + 0, 487, 488, 489, 490, 491, 492, 493, 0, 0, + 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, + 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, + 512, 513, 514, 515, 523, 0, 815, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, + 0, 135, 136, 137, 138, 139, 0, 141, 142, 0, + 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, + 151, 152, 153, 154, 155, 0, 156, 157, 158, 159, + 160, 0, 0, 0, 162, 163, 164, 165, 166, 167, + 0, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 0, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, + 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, + 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, + 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + 0, 227, 0, 228, 229, 230, 231, 0, 232, 0, + 233, 0, 0, 0, 236, 237, 524, 0, 240, 0, + 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, + 249, 250, 251, 252, 0, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 0, + 267, 268, 269, 270, 271, 272, 273, 274, 0, 275, + 0, 276, 0, 0, 279, 0, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 0, 289, 0, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 525, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 0, + 322, 323, 324, 325, 326, 0, 327, 328, 0, 330, + 0, 331, 332, 333, 334, 335, 336, 0, 337, 338, + 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, + 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, + 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, + 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, + 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 526, 444, 445, 0, 0, 446, 447, 0, 448, 0, + 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, + 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, + 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, + 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 0, 485, 0, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, 512, 513, 514, 515, 523, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, + 119, 120, 121, 122, 123, 124, 822, 125, 126, 127, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, + 130, 0, 131, 132, 133, 0, 135, 136, 137, 138, + 139, 0, 141, 142, 0, 143, 144, 145, 146, 147, + 148, 0, 0, 149, 150, 151, 152, 153, 154, 155, + 0, 156, 157, 158, 159, 160, 0, 0, 0, 162, + 163, 164, 165, 166, 167, 0, 169, 170, 171, 0, + 172, 173, 174, 175, 176, 177, 0, 0, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, + 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, + 213, 214, 215, 0, 216, 217, 218, 219, 220, 221, + 222, 223, 224, 225, 226, 0, 227, 0, 228, 229, + 230, 231, 0, 232, 0, 233, 0, 0, 0, 236, + 237, 524, 0, 823, 0, 241, 0, 242, 243, 244, + 245, 0, 246, 247, 248, 249, 250, 251, 252, 0, + 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 0, 267, 268, 269, 270, 271, + 272, 273, 274, 0, 275, 0, 276, 0, 0, 279, + 0, 281, 282, 283, 284, 285, 286, 0, 0, 287, + 0, 289, 0, 0, 291, 292, 824, 294, 295, 296, + 297, 298, 525, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 0, 322, 323, 324, 325, 326, + 0, 327, 328, 0, 330, 0, 331, 332, 333, 334, + 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, + 0, 0, 342, 343, 344, 0, 346, 0, 348, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 0, 0, 0, 0, 360, 361, 362, 0, 364, 365, + 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, + 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, + 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 397, 398, 0, 400, 401, + 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, + 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, + 420, 825, 422, 423, 424, 425, 0, 0, 427, 428, + 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 526, 444, 445, 0, 0, + 446, 447, 0, 448, 0, 450, 451, 452, 453, 454, + 0, 455, 826, 457, 0, 0, 827, 459, 460, 461, + 462, 0, 463, 464, 465, 466, 467, 468, 469, 470, + 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 0, 485, + 0, 487, 488, 489, 490, 491, 492, 493, 0, 0, + 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, + 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, + 512, 513, 514, 515, 523, 0, 549, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, + 0, 135, 136, 137, 138, 139, 0, 141, 142, 0, + 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, + 151, 152, 153, 154, 155, 0, 156, 157, 158, 159, + 160, 0, 0, 0, 162, 163, 164, 165, 166, 167, + 0, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 0, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, + 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, + 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, + 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + 0, 227, 0, 228, 229, 230, 231, 0, 232, 0, + 233, 0, 0, 0, 236, 237, 524, 0, 240, 0, + 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, + 249, 250, 251, 252, 0, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 0, + 267, 268, 269, 270, 271, 272, 273, 274, 0, 275, + 0, 276, 0, 0, 279, 0, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 0, 289, 0, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 525, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 0, + 322, 323, 324, 325, 326, 0, 327, 328, 0, 330, + 0, 331, 332, 333, 334, 335, 336, 0, 337, 338, + 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, + 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, + 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, + 379, 380, 381, 859, 383, 0, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, + 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 526, 444, 445, 0, 0, 446, 447, 0, 448, 0, + 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, + 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, + 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, + 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 0, 485, 0, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, 512, 513, 514, 515, 523, + 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, + 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, + 130, 0, 131, 132, 133, 0, 135, 136, 137, 138, + 139, 0, 141, 142, 0, 143, 144, 145, 146, 147, + 148, 0, 0, 149, 150, 151, 152, 153, 154, 155, + 0, 156, 157, 158, 159, 160, 0, 0, 0, 162, + 163, 164, 165, 166, 167, 0, 169, 170, 171, 0, + 172, 173, 174, 175, 176, 177, 0, 0, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, + 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, + 213, 214, 215, 0, 216, 217, 218, 219, 220, 221, + 222, 223, 224, 225, 226, 0, 227, 0, 228, 229, + 230, 231, 0, 232, 0, 233, 0, 0, 0, 236, + 237, 524, 0, 240, 0, 241, 0, 242, 243, 244, + 245, 0, 246, 247, 248, 249, 250, 891, 252, 0, + 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 0, 267, 268, 269, 270, 271, + 272, 273, 274, 0, 275, 0, 276, 0, 0, 279, + 0, 281, 282, 283, 284, 285, 286, 0, 0, 287, + 0, 289, 0, 0, 291, 292, 293, 294, 295, 296, + 297, 298, 525, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 0, 322, 323, 324, 325, 326, + 0, 327, 328, 0, 330, 0, 331, 332, 333, 334, + 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, + 0, 0, 342, 343, 344, 0, 346, 0, 348, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 0, 0, 0, 0, 360, 361, 362, 0, 364, 365, + 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, + 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, + 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 397, 398, 0, 400, 401, + 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, + 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 0, 0, 427, 428, + 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 526, 444, 445, 0, 0, + 446, 447, 0, 448, 0, 450, 451, 452, 453, 454, + 0, 455, 456, 457, 0, 0, 458, 459, 460, 461, + 462, 0, 463, 464, 465, 466, 467, 468, 469, 470, + 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 0, 485, + 0, 487, 488, 489, 490, 491, 492, 493, 0, 0, + 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, + 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, + 512, 513, 514, 515, 523, 0, 549, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, + 0, 135, 136, 137, 138, 139, 0, 141, 142, 0, + 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, + 151, 152, 153, 154, 155, 0, 156, 157, 158, 159, + 160, 0, 0, 0, 162, 163, 164, 165, 166, 167, + 0, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 0, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, + 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, + 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, + 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + 0, 227, 0, 228, 229, 230, 231, 0, 232, 0, + 233, 0, 0, 0, 236, 237, 524, 0, 240, 0, + 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, + 249, 250, 894, 252, 0, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 0, + 267, 268, 269, 270, 271, 272, 273, 274, 0, 275, + 0, 276, 0, 0, 279, 0, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 0, 289, 0, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 525, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 0, + 322, 323, 324, 325, 326, 0, 327, 328, 0, 330, + 0, 331, 332, 333, 334, 335, 336, 0, 337, 338, + 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, + 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, + 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, + 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, + 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 526, 444, 445, 0, 0, 446, 447, 0, 448, 0, + 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, + 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, + 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, + 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 0, 485, 0, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, 512, 513, 514, 515, 523, + 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, + 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, + 130, 0, 131, 132, 133, 0, 135, 136, 137, 138, + 139, 0, 141, 142, 0, 143, 144, 145, 146, 147, + 148, 0, 0, 149, 150, 151, 152, 153, 154, 155, + 0, 156, 157, 158, 159, 160, 0, 0, 0, 162, + 163, 164, 165, 166, 167, 0, 169, 170, 171, 0, + 172, 173, 174, 175, 176, 177, 0, 0, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, + 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, + 213, 214, 215, 0, 216, 217, 218, 219, 220, 221, + 222, 223, 224, 225, 226, 0, 227, 0, 228, 229, + 230, 231, 0, 232, 0, 233, 0, 0, 0, 236, + 237, 524, 0, 240, 0, 241, 0, 242, 243, 244, + 245, 0, 246, 247, 248, 249, 250, 898, 252, 0, + 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 0, 267, 268, 269, 270, 271, + 272, 273, 274, 0, 275, 0, 276, 0, 0, 279, + 0, 281, 282, 283, 284, 285, 286, 0, 0, 287, + 0, 289, 0, 0, 291, 292, 293, 294, 295, 296, + 297, 298, 525, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 0, 322, 323, 324, 325, 326, + 0, 327, 328, 0, 330, 0, 331, 332, 333, 334, + 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, + 0, 0, 342, 343, 344, 0, 346, 0, 348, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 0, 0, 0, 0, 360, 361, 362, 0, 364, 365, + 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, + 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, + 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 397, 398, 0, 400, 401, + 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, + 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 0, 0, 427, 428, + 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 526, 444, 445, 0, 0, + 446, 447, 0, 448, 0, 450, 451, 452, 453, 454, + 0, 455, 456, 457, 0, 0, 458, 459, 460, 461, + 462, 0, 463, 464, 465, 466, 467, 468, 469, 470, + 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 0, 485, + 0, 487, 488, 489, 490, 491, 492, 493, 0, 0, + 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, + 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, + 512, 513, 514, 515, 523, 0, 549, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, + 0, 135, 136, 137, 138, 139, 0, 141, 142, 0, + 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, + 151, 152, 153, 154, 155, 0, 156, 157, 158, 159, + 160, 0, 0, 0, 162, 163, 164, 165, 166, 167, + 0, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 0, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, + 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, + 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, + 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + 0, 227, 0, 228, 229, 230, 231, 0, 232, 0, + 233, 0, 0, 0, 236, 237, 524, 0, 240, 0, + 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, + 249, 250, 926, 252, 0, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 0, + 267, 268, 269, 270, 271, 272, 273, 274, 0, 275, + 0, 276, 0, 0, 279, 0, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 0, 289, 0, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 525, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 0, + 322, 323, 324, 325, 326, 0, 327, 328, 0, 330, + 0, 331, 332, 333, 334, 335, 336, 0, 337, 338, + 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, + 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, + 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, + 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, + 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 526, 444, 445, 0, 0, 446, 447, 0, 448, 0, + 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, + 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, + 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, + 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 0, 485, 0, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, 512, 513, 514, 515, 523, + 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, + 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, + 130, 0, 131, 132, 133, 0, 135, 136, 137, 138, + 139, 0, 141, 142, 0, 143, 144, 145, 146, 147, + 148, 0, 0, 149, 150, 151, 152, 153, 154, 155, + 0, 156, 157, 158, 159, 160, 0, 0, 0, 162, + 163, 164, 165, 166, 167, 0, 169, 170, 171, 0, + 172, 173, 174, 175, 176, 177, 0, 0, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, + 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, + 213, 214, 215, 0, 216, 217, 218, 219, 220, 221, + 222, 223, 224, 225, 226, 0, 227, 0, 228, 229, + 230, 231, 0, 232, 0, 233, 0, 0, 0, 236, + 237, 524, 0, 240, 0, 241, 0, 242, 243, 244, + 245, 0, 246, 247, 248, 249, 250, 954, 252, 0, + 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 0, 267, 268, 269, 270, 271, + 272, 273, 274, 0, 275, 0, 276, 0, 0, 279, + 0, 281, 282, 283, 284, 285, 286, 0, 0, 287, + 0, 289, 0, 0, 291, 292, 293, 294, 295, 296, + 297, 298, 525, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 0, 322, 323, 324, 325, 326, + 0, 327, 328, 0, 330, 0, 331, 332, 333, 334, + 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, + 0, 0, 342, 343, 344, 0, 346, 0, 348, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 0, 0, 0, 0, 360, 361, 362, 0, 364, 365, + 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, + 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, + 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 397, 398, 0, 400, 401, + 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, + 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 0, 0, 427, 428, + 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 526, 444, 445, 0, 0, + 446, 447, 0, 448, 0, 450, 451, 452, 453, 454, + 0, 455, 456, 457, 0, 0, 458, 459, 460, 461, + 462, 0, 463, 464, 465, 466, 467, 468, 469, 470, + 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 0, 485, + 0, 487, 488, 489, 490, 491, 492, 493, 0, 0, + 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, + 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, + 512, 513, 514, 515, 523, 0, 549, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, + 0, 135, 136, 137, 138, 139, 0, 141, 142, 0, + 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, + 151, 152, 153, 154, 155, 0, 156, 157, 158, 159, + 160, 0, 0, 0, 162, 163, 164, 165, 166, 167, + 0, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 0, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, + 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, + 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, + 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + 0, 227, 0, 228, 229, 230, 231, 0, 232, 0, + 233, 0, 0, 0, 236, 237, 524, 0, 240, 0, + 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, + 249, 250, 957, 252, 0, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 0, + 267, 268, 269, 270, 271, 272, 273, 274, 0, 275, + 0, 276, 0, 0, 279, 0, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 0, 289, 0, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 525, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 0, + 322, 323, 324, 325, 326, 0, 327, 328, 0, 330, + 0, 331, 332, 333, 334, 335, 336, 0, 337, 338, + 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, + 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, + 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, + 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, + 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 526, 444, 445, 0, 0, 446, 447, 0, 448, 0, + 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, + 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, + 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, + 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 0, 485, 0, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, 512, 513, 514, 515, 523, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, + 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, + 0, 0, 0, 0, 0, 0, 1000, 0, 0, 129, + 130, 0, 131, 132, 133, 0, 135, 136, 137, 138, + 139, 0, 141, 142, 0, 143, 144, 145, 146, 147, + 148, 0, 0, 149, 150, 151, 152, 153, 154, 155, + 0, 156, 157, 158, 159, 160, 0, 0, 0, 162, + 163, 164, 165, 166, 167, 0, 169, 170, 171, 0, + 172, 173, 174, 175, 176, 177, 0, 0, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, + 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, + 213, 214, 215, 0, 216, 217, 218, 219, 220, 221, + 222, 223, 224, 225, 226, 0, 227, 0, 228, 229, + 230, 231, 0, 232, 0, 233, 0, 0, 0, 236, + 237, 524, 0, 240, 0, 241, 0, 242, 243, 244, + 245, 0, 246, 247, 248, 249, 250, 251, 252, 0, + 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 0, 267, 268, 269, 270, 271, + 272, 273, 274, 0, 275, 0, 276, 0, 0, 279, + 0, 281, 282, 283, 284, 285, 286, 0, 0, 287, + 0, 289, 0, 0, 291, 292, 293, 294, 295, 296, + 297, 298, 525, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 0, 322, 323, 324, 325, 326, + 0, 327, 328, 0, 330, 0, 331, 332, 333, 334, + 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, + 0, 0, 342, 343, 344, 0, 346, 0, 348, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 0, 0, 0, 0, 360, 361, 362, 0, 364, 365, + 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, + 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, + 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 397, 398, 0, 400, 401, + 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, + 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 0, 0, 427, 428, + 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 526, 444, 445, 0, 0, + 446, 447, 0, 448, 0, 450, 451, 452, 453, 454, + 0, 455, 456, 457, 0, 0, 458, 459, 460, 461, + 462, 0, 463, 464, 465, 466, 467, 468, 469, 470, + 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 0, 485, + 0, 487, 488, 489, 490, 491, 492, 493, 0, 0, + 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, + 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, + 512, 513, 514, 515, 523, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, + 0, 1023, 0, 0, 129, 130, 0, 131, 132, 133, + 0, 135, 136, 137, 138, 139, 0, 141, 142, 0, + 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, + 151, 152, 153, 154, 155, 0, 156, 157, 158, 159, + 160, 0, 0, 0, 162, 163, 164, 165, 166, 167, + 0, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 0, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, + 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, + 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, + 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + 0, 227, 0, 228, 229, 230, 231, 0, 232, 0, + 233, 0, 0, 0, 236, 237, 524, 0, 240, 0, + 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, + 249, 250, 251, 252, 0, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 0, + 267, 268, 269, 270, 271, 272, 273, 274, 0, 275, + 0, 276, 0, 0, 279, 0, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 0, 289, 0, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 525, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 0, + 322, 323, 324, 325, 326, 0, 327, 328, 0, 330, + 0, 331, 332, 333, 334, 335, 336, 0, 337, 338, + 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, + 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, + 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, + 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, + 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 526, 444, 445, 0, 0, 446, 447, 0, 448, 0, + 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, + 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, + 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, + 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 0, 485, 0, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, 512, 513, 514, 515, 523, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, + 119, 120, 121, 122, 123, 124, 822, 125, 126, 127, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, + 130, 0, 131, 132, 133, 0, 135, 136, 137, 138, + 139, 0, 141, 142, 0, 143, 144, 145, 146, 147, + 148, 0, 0, 149, 150, 151, 152, 153, 154, 155, + 0, 156, 157, 158, 159, 160, 0, 0, 0, 162, + 163, 164, 165, 166, 167, 0, 169, 170, 171, 0, + 172, 173, 174, 175, 176, 177, 0, 0, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, + 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, + 213, 214, 215, 0, 216, 217, 218, 219, 220, 221, + 222, 223, 224, 225, 226, 0, 227, 0, 228, 229, + 230, 231, 0, 232, 0, 233, 0, 0, 0, 236, + 237, 524, 0, 240, 0, 241, 0, 242, 243, 244, + 245, 0, 246, 247, 248, 249, 250, 251, 252, 0, + 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 0, 267, 268, 269, 270, 271, + 272, 273, 274, 0, 275, 0, 276, 0, 0, 279, + 0, 281, 282, 283, 284, 285, 286, 0, 0, 287, + 0, 289, 0, 0, 291, 292, 293, 294, 295, 296, + 297, 298, 525, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 0, 322, 323, 324, 325, 326, + 0, 327, 328, 0, 330, 0, 331, 332, 333, 334, + 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, + 0, 0, 342, 343, 344, 0, 346, 0, 348, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 0, 0, 0, 0, 360, 361, 362, 0, 364, 365, + 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, + 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, + 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 397, 398, 0, 400, 401, + 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, + 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 0, 0, 427, 428, + 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 526, 444, 445, 0, 0, + 446, 447, 0, 448, 0, 450, 451, 452, 453, 454, + 0, 455, 826, 457, 0, 0, 827, 459, 460, 461, + 462, 0, 463, 464, 465, 466, 467, 468, 469, 470, + 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 0, 485, + 0, 487, 488, 489, 490, 491, 492, 493, 0, 0, + 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, + 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, + 512, 513, 514, 515, 523, 0, 549, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, + 0, 135, 136, 137, 138, 139, 0, 141, 142, 0, + 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, + 151, 152, 153, 154, 155, 0, 156, 157, 158, 159, + 160, 0, 0, 0, 162, 163, 164, 165, 166, 167, + 0, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 0, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, + 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, + 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, + 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + 0, 227, 0, 228, 229, 230, 231, 0, 232, 0, + 233, 0, 0, 0, 236, 237, 524, 0, 240, 0, + 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, + 249, 250, 1303, 252, 0, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 0, + 267, 268, 269, 270, 271, 272, 273, 274, 0, 275, + 0, 276, 0, 0, 279, 0, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 0, 289, 0, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 525, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 0, + 322, 323, 324, 325, 326, 0, 327, 328, 0, 330, + 0, 331, 332, 333, 334, 335, 336, 0, 337, 338, + 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, + 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, + 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, + 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, + 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 526, 444, 445, 0, 0, 446, 447, 0, 448, 0, + 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, + 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, + 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, + 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 0, 485, 0, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, 512, 513, 514, 515, 523, + 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, + 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, + 130, 0, 131, 132, 133, 0, 135, 136, 137, 138, + 139, 0, 141, 142, 0, 143, 144, 145, 146, 147, + 148, 0, 0, 149, 150, 151, 152, 153, 154, 155, + 0, 156, 157, 158, 159, 160, 0, 0, 0, 162, + 163, 164, 165, 166, 167, 0, 169, 170, 171, 0, + 172, 173, 174, 175, 176, 177, 0, 0, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, + 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, + 213, 214, 215, 0, 216, 217, 218, 219, 220, 221, + 222, 223, 224, 225, 226, 0, 227, 0, 228, 229, + 230, 231, 0, 232, 0, 233, 0, 0, 0, 236, + 237, 524, 0, 240, 0, 241, 0, 242, 243, 244, + 245, 0, 246, 247, 248, 249, 250, 1305, 252, 0, + 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 0, 267, 268, 269, 270, 271, + 272, 273, 274, 0, 275, 0, 276, 0, 0, 279, + 0, 281, 282, 283, 284, 285, 286, 0, 0, 287, + 0, 289, 0, 0, 291, 292, 293, 294, 295, 296, + 297, 298, 525, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 0, 322, 323, 324, 325, 326, + 0, 327, 328, 0, 330, 0, 331, 332, 333, 334, + 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, + 0, 0, 342, 343, 344, 0, 346, 0, 348, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 0, 0, 0, 0, 360, 361, 362, 0, 364, 365, + 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, + 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, + 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 397, 398, 0, 400, 401, + 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, + 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 0, 0, 427, 428, + 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 526, 444, 445, 0, 0, + 446, 447, 0, 448, 0, 450, 451, 452, 453, 454, + 0, 455, 456, 457, 0, 0, 458, 459, 460, 461, + 462, 0, 463, 464, 465, 466, 467, 468, 469, 470, + 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 0, 485, + 0, 487, 488, 489, 490, 491, 492, 493, 0, 0, + 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, + 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, + 512, 513, 514, 515, 523, 0, 549, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, + 0, 135, 136, 137, 138, 139, 0, 141, 142, 0, + 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, + 151, 152, 153, 154, 155, 0, 156, 157, 158, 159, + 160, 0, 0, 0, 162, 163, 164, 165, 166, 167, + 0, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 0, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, + 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, + 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, + 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + 0, 227, 0, 228, 229, 230, 231, 0, 232, 0, + 233, 0, 0, 0, 236, 237, 524, 0, 240, 0, + 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, + 249, 250, 1308, 252, 0, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 0, + 267, 268, 269, 270, 271, 272, 273, 274, 0, 275, + 0, 276, 0, 0, 279, 0, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 0, 289, 0, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 525, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 0, + 322, 323, 324, 325, 326, 0, 327, 328, 0, 330, + 0, 331, 332, 333, 334, 335, 336, 0, 337, 338, + 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, + 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, + 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, + 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, + 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 526, 444, 445, 0, 0, 446, 447, 0, 448, 0, + 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, + 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, + 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, + 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 0, 485, 0, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, 512, 513, 514, 515, 523, + 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, + 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, + 130, 0, 131, 132, 133, 0, 135, 136, 137, 138, + 139, 0, 141, 142, 0, 143, 144, 145, 146, 147, + 148, 0, 0, 149, 150, 151, 152, 153, 154, 155, + 0, 156, 157, 158, 159, 160, 0, 0, 0, 162, + 163, 164, 165, 166, 167, 0, 169, 170, 171, 0, + 172, 173, 174, 175, 176, 177, 0, 0, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, + 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, + 213, 214, 215, 0, 216, 217, 218, 219, 220, 221, + 222, 223, 224, 225, 226, 0, 227, 0, 228, 229, + 230, 231, 0, 232, 0, 233, 0, 0, 0, 236, + 237, 524, 0, 240, 0, 241, 0, 242, 243, 244, + 245, 0, 246, 247, 248, 249, 250, 1310, 252, 0, + 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 0, 267, 268, 269, 270, 271, + 272, 273, 274, 0, 275, 0, 276, 0, 0, 279, + 0, 281, 282, 283, 284, 285, 286, 0, 0, 287, + 0, 289, 0, 0, 291, 292, 293, 294, 295, 296, + 297, 298, 525, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 0, 322, 323, 324, 325, 326, + 0, 327, 328, 0, 330, 0, 331, 332, 333, 334, + 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, + 0, 0, 342, 343, 344, 0, 346, 0, 348, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 0, 0, 0, 0, 360, 361, 362, 0, 364, 365, + 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, + 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, + 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 397, 398, 0, 400, 401, + 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, + 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 0, 0, 427, 428, + 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 526, 444, 445, 0, 0, + 446, 447, 0, 448, 0, 450, 451, 452, 453, 454, + 0, 455, 456, 457, 0, 0, 458, 459, 460, 461, + 462, 0, 463, 464, 465, 466, 467, 468, 469, 470, + 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 0, 485, + 0, 487, 488, 489, 490, 491, 492, 493, 0, 0, + 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, + 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, + 512, 513, 514, 515, 523, 0, 549, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, + 0, 135, 136, 137, 138, 139, 0, 141, 142, 0, + 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, + 151, 152, 153, 154, 155, 0, 156, 157, 158, 159, + 160, 0, 0, 0, 162, 163, 164, 165, 166, 167, + 0, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 0, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, + 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, + 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, + 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + 0, 227, 0, 228, 229, 230, 231, 0, 232, 0, + 233, 0, 0, 0, 236, 237, 524, 0, 240, 0, + 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, + 249, 250, 2229, 252, 0, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 0, + 267, 268, 269, 270, 271, 272, 273, 274, 0, 275, + 0, 276, 0, 0, 279, 0, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 0, 289, 0, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 525, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 0, + 322, 323, 324, 325, 326, 0, 327, 328, 0, 330, + 0, 331, 332, 333, 334, 335, 336, 0, 337, 338, + 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, + 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, + 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, + 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, + 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 526, 444, 445, 0, 0, 446, 447, 0, 448, 0, + 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, + 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, + 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, + 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 0, 485, 0, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, 512, 513, 514, 515, 1476, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, + 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, + 0, 0, 0, 1477, 0, 0, -788, 0, 1478, 129, + 130, 0, 131, 132, 133, 1479, 135, 136, 137, 0, + 1480, 1481, 1482, 1483, 0, 143, 144, 145, 146, 147, + 148, 0, 0, 149, 150, 151, 152, 1484, 1485, 155, + 0, 156, 157, 158, 159, 0, 0, 1486, 0, 1487, + 163, 164, 165, 166, 167, 1488, 169, 170, 171, 0, + 172, 173, 174, 175, 176, 177, 0, 1489, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 1490, 190, + 191, 1491, 193, 0, 194, 0, 195, 196, 197, 198, + 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, + 205, 206, 1059, 208, 209, 0, 210, 211, 212, 0, + 213, 214, 215, 0, 216, 217, 218, 219, 0, 221, + 222, 223, 224, 225, 0, 0, 227, 0, 228, 229, + 1492, 231, 0, 232, 0, 233, 1493, 0, 1494, 236, + 237, -788, 1495, 240, 0, 241, 0, 0, 0, 244, + 245, 0, 246, 247, 248, 249, 250, 251, 252, 1496, + 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 1497, 0, 268, 269, 270, 271, + 272, 1498, 1499, 0, 1500, 0, 276, 1501, 1502, 279, + 1503, 281, 282, 283, 284, 285, 286, 0, 0, 287, + 1504, 289, 1505, 0, 291, 292, 293, 294, 295, 296, + 297, 298, 1506, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 1507, 1508, 1509, 323, 324, 325, 0, + 0, 327, 328, 1510, 330, 0, 0, 332, 1511, 334, + 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, + 0, 0, 342, 343, 0, 1512, 346, 1513, 0, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 0, 0, 0, 0, 360, 361, 0, 1514, 364, 365, + 0, 367, 368, 369, 0, 370, 371, 372, 373, 374, + 375, 0, 376, 377, 378, 1515, 380, 381, 382, 383, + 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 397, 398, 1516, 400, 401, + 402, 1517, 404, 405, 406, 407, 408, 409, 410, 411, + 412, 413, 414, 415, 416, 0, 1518, 417, 418, 419, + 420, 421, 422, 1519, 424, 425, 0, 1520, 427, 428, + 1521, 430, 0, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 1522, 444, 0, 0, 0, + 446, 447, 0, 448, 1523, 450, 451, 452, 453, 454, + 0, 455, 1524, 1525, 0, 0, 458, 459, 0, 461, + 0, 0, 463, 464, 1526, 466, 467, 468, 469, 470, + 1527, 0, 471, 472, 473, 0, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 0, 1528, 0, 485, + 1529, 487, 488, 489, 490, 491, 492, 493, 0, 0, + 494, 0, 0, 495, 496, 497, 498, 499, 500, 523, + 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, + 512, 513, 514, 515, 0, 0, 0, 0, 117, 118, + 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, + 130, 0, 131, 132, 133, 0, 135, 136, 137, 138, + 139, 0, 141, 142, 0, 143, 144, 145, 146, 147, + 148, 0, 0, 149, 150, 151, 152, 153, 154, 155, + 0, 156, 157, 158, 159, 160, 0, 0, 0, 162, + 163, 164, 165, 166, 167, 0, 169, 170, 171, 0, + 172, 173, 174, 175, 176, 177, 0, 0, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, + 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, + 213, 214, 215, 0, 216, 217, 218, 219, 220, 221, + 222, 223, 224, 225, 226, 0, 227, 0, 228, 229, + 230, 231, 0, 232, 0, 233, 0, 0, 0, 236, + 237, 524, 0, 240, 0, 241, 0, 242, 243, 244, + 245, 0, 246, 247, 248, 249, 250, 2974, 252, 0, + 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 0, 267, 268, 269, 270, 271, + 272, 273, 274, 0, 275, 0, 276, 0, 0, 279, + 0, 281, 282, 283, 284, 285, 286, 0, 0, 287, + 0, 289, 0, 0, 291, 292, 293, 294, 295, 296, + 297, 298, 525, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 0, 322, 323, 324, 325, 326, + 0, 327, 328, 0, 330, 0, 331, 332, 333, 334, + 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, + 0, 0, 342, 343, 344, 0, 346, 0, 348, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 0, 0, 0, 0, 360, 361, 362, 0, 364, 365, + 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, + 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, + 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 397, 398, 0, 400, 401, + 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, + 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 0, 0, 427, 428, + 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 526, 444, 445, 0, 0, + 446, 447, 0, 448, 0, 450, 451, 452, 453, 454, + 0, 455, 456, 457, 0, 0, 458, 459, 460, 461, + 462, 0, 463, 464, 465, 466, 467, 468, 469, 470, + 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 0, 485, + 0, 487, 488, 489, 490, 491, 492, 493, 0, 0, + 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, + 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, + 512, 513, 514, 515, 523, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, + 0, 135, 136, 137, 138, 139, 0, 141, 142, 0, + 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, + 151, 152, 153, 154, 155, 0, 156, 157, 158, 159, + 160, 0, 0, 0, 162, 163, 164, 165, 166, 167, + 0, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 0, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, + 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, + 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, + 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + 0, 227, 0, 228, 229, 230, 231, 0, 232, 0, + 233, 0, 0, 0, 236, 237, 524, 0, 240, 0, + 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, + 249, 250, 251, 252, 0, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 0, + 267, 268, 269, 270, 271, 272, 273, 274, 0, 275, + 0, 276, 0, 0, 279, 0, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 0, 289, 0, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 525, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 0, + 322, 323, 324, 325, 326, 0, 327, 328, 0, 330, + 0, 331, 332, 333, 334, 335, 336, 0, 337, 338, + 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, + 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, + 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, + 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, + 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 526, 444, 445, 0, 0, 446, 447, 0, 448, 0, + 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, + 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, + 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, + 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 0, 485, 0, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, 512, 513, 514, 515, 523, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, + 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, + 130, 0, 131, 132, 133, 0, 135, 136, 137, 138, + 139, 0, 141, 142, 0, 143, 144, 145, 146, 147, + 148, 0, 0, 149, 150, 151, 152, 153, 154, 155, + 0, 156, 157, 158, 159, 160, 0, 0, 0, 162, + 163, 164, 165, 166, 167, 0, 169, 170, 171, 0, + 172, 173, 174, 175, 176, 177, 0, 0, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, + 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, + 213, 214, 215, 0, 216, 217, 218, 219, 220, 221, + 222, 223, 224, 225, 226, 0, 227, 0, 228, 229, + 230, 231, 0, 232, 0, 233, 0, 0, 0, 236, + 237, 524, 0, 837, 0, 241, 0, 242, 243, 244, + 245, 0, 246, 247, 248, 249, 250, 251, 252, 0, + 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 0, 267, 268, 269, 270, 271, + 272, 273, 274, 0, 275, 0, 276, 0, 0, 279, + 0, 281, 282, 283, 284, 285, 286, 0, 0, 287, + 0, 289, 0, 0, 291, 292, 838, 294, 295, 296, + 297, 298, 525, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 0, 322, 323, 324, 325, 326, + 0, 327, 328, 0, 330, 0, 331, 332, 333, 334, + 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, + 0, 0, 342, 343, 344, 0, 346, 0, 348, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 0, 0, 0, 0, 360, 361, 362, 0, 364, 365, + 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, + 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, + 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 397, 398, 0, 400, 401, + 402, 403, 404, 405, 406, 407, 839, 409, 410, 411, + 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, + 420, 840, 422, 423, 424, 425, 0, 0, 427, 428, + 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 526, 444, 445, 0, 0, + 446, 447, 0, 448, 0, 450, 451, 452, 453, 454, + 0, 455, 841, 457, 0, 0, 458, 459, 460, 461, + 462, 0, 463, 464, 465, 466, 467, 468, 469, 470, + 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 0, 485, + 0, 487, 488, 489, 490, 491, 492, 493, 0, 0, + 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, + 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, + 512, 513, 514, 515, 523, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, + 0, 135, 136, 137, 138, 139, 0, 141, 142, 0, + 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, + 151, 152, 153, 154, 155, 0, 156, 157, 158, 159, + 160, 0, 0, 0, 162, 163, 164, 165, 166, 167, + 0, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 0, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, + 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, + 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, + 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + 0, 227, 0, 228, 229, 230, 231, 0, 232, 0, + 233, 0, 0, 0, 236, 237, 524, 0, 240, 0, + 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, + 249, 250, 950, 252, 0, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 0, + 267, 268, 269, 270, 271, 272, 273, 274, 0, 275, + 0, 276, 0, 0, 279, 0, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 0, 289, 0, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 525, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 0, + 322, 323, 324, 325, 326, 0, 327, 328, 0, 330, + 0, 331, 332, 333, 334, 335, 336, 0, 337, 338, + 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, + 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, + 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, + 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, + 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 526, 444, 445, 0, 0, 446, 447, 0, 448, 0, + 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, + 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, + 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, + 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 0, 485, 0, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, 512, 513, 514, 515, 523, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, + 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, + 130, 0, 131, 132, 133, 0, 135, 136, 137, 138, + 139, 0, 141, 142, 0, 143, 144, 145, 146, 147, + 148, 0, 0, 149, 150, 151, 152, 153, 154, 155, + 0, 156, 157, 158, 159, 160, 0, 0, 0, 162, + 163, 164, 165, 166, 167, 0, 169, 170, 171, 0, + 172, 173, 174, 175, 176, 177, 0, 0, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, + 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, + 213, 214, 215, 0, 216, 217, 218, 219, 220, 221, + 222, 223, 224, 225, 226, 0, 227, 0, 228, 229, + 230, 231, 0, 232, 0, 233, 0, 0, 0, 236, + 237, 524, 0, 240, 0, 241, 0, 242, 243, 244, + 245, 0, 246, 247, 248, 249, 250, 251, 252, 0, + 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 0, 267, 268, 269, 270, 271, + 272, 273, 274, 0, 275, 0, 276, 0, 0, 279, + 0, 281, 282, 283, 284, 285, 286, 0, 0, 287, + 0, 289, 0, 0, 291, 292, 293, 294, 295, 296, + 297, 298, 525, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 0, 322, 323, 324, 325, 326, + 0, 327, 328, 0, 330, 0, 331, 332, 333, 334, + 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, + 0, 0, 342, 343, 344, 0, 346, 0, 348, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 0, 0, 0, 0, 360, 361, 362, 0, 364, 365, + 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, + 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, + 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 397, 398, 0, 400, 401, + 402, 403, 404, 405, 406, 407, 839, 409, 410, 411, + 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 0, 0, 427, 428, + 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 526, 444, 445, 0, 0, + 446, 447, 0, 448, 0, 450, 451, 452, 453, 454, + 0, 455, 841, 457, 0, 0, 458, 459, 460, 461, + 462, 0, 463, 464, 465, 466, 467, 468, 469, 470, + 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 0, 485, + 0, 487, 488, 489, 490, 491, 492, 493, 0, 0, + 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, + 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, + 512, 513, 514, 515, 523, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, + 0, 135, 136, 137, 138, 139, 0, 141, 142, 0, + 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, + 151, 152, 153, 154, 155, 0, 156, 157, 158, 159, + 160, 0, 0, 0, 162, 163, 164, 165, 166, 167, + 0, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 0, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, + 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, + 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, + 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + 0, 227, 0, 228, 229, 230, 231, 0, 232, 0, + 233, 0, 0, 0, 236, 237, 524, 0, 240, 0, + 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, + 249, 250, 1299, 252, 0, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 0, + 267, 268, 269, 270, 271, 272, 273, 274, 0, 275, + 0, 276, 0, 0, 279, 0, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 0, 289, 0, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 525, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 0, + 322, 323, 324, 325, 326, 0, 327, 328, 0, 330, + 0, 331, 332, 333, 334, 335, 336, 0, 337, 338, + 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, + 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, + 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, + 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, + 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 526, 444, 445, 0, 0, 446, 447, 0, 448, 0, + 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, + 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, + 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, + 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 0, 485, 0, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, 512, 513, 514, 515, 523, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, + 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, + 130, 0, 131, 132, 133, 0, 135, 136, 137, 138, + 139, 0, 141, 142, 0, 143, 144, 145, 146, 147, + 148, 0, 0, 149, 150, 151, 152, 153, 154, 155, + 0, 156, 157, 158, 159, 160, 0, 0, 0, 162, + 163, 164, 165, 166, 167, 0, 169, 170, 171, 0, + 172, 173, 174, 175, 176, 177, 0, 0, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, + 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, + 213, 214, 215, 0, 216, 217, 218, 219, 220, 221, + 222, 223, 224, 225, 226, 0, 227, 0, 228, 229, + 230, 231, 0, 232, 0, 233, 0, 0, 0, 236, + 237, 524, 0, 240, 0, 241, 0, 242, 243, 244, + 245, 0, 246, 247, 248, 249, 250, 1320, 252, 0, + 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 0, 267, 268, 269, 270, 271, + 272, 273, 274, 0, 275, 0, 276, 0, 0, 279, + 0, 281, 282, 283, 284, 285, 286, 0, 0, 287, + 0, 289, 0, 0, 291, 292, 293, 294, 295, 296, + 297, 298, 525, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 0, 322, 323, 324, 325, 326, + 0, 327, 328, 0, 330, 0, 331, 332, 333, 334, + 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, + 0, 0, 342, 343, 344, 0, 346, 0, 348, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 0, 0, 0, 0, 360, 361, 362, 0, 364, 365, + 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, + 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, + 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 397, 398, 0, 400, 401, + 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, + 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 0, 0, 427, 428, + 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 526, 444, 445, 0, 0, + 446, 447, 0, 448, 0, 450, 451, 452, 453, 454, + 0, 455, 456, 457, 0, 0, 458, 459, 460, 461, + 462, 0, 463, 464, 465, 466, 467, 468, 469, 470, + 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 0, 485, + 0, 487, 488, 489, 490, 491, 492, 493, 0, 0, + 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, + 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, + 512, 513, 514, 515, 523, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, + 0, 1671, 0, 0, 129, 130, 0, 131, 132, 133, + 0, 135, 136, 137, 138, 139, 0, 141, 142, 0, + 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, + 151, 152, 153, 154, 155, 0, 156, 157, 158, 159, + 160, 0, 0, 0, 162, 163, 164, 165, 166, 167, + 0, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 0, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, + 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, + 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, + 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + 0, 227, 0, 228, 229, 230, 231, 0, 232, 0, + 233, 0, 0, 0, 236, 237, 524, 0, 240, 0, + 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, + 249, 250, 251, 252, 0, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 0, + 267, 268, 269, 270, 271, 272, 273, 274, 0, 275, + 0, 276, 0, 0, 279, 0, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 0, 289, 0, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 525, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 0, + 322, 323, 324, 325, 326, 0, 327, 328, 0, 330, + 0, 331, 332, 333, 334, 335, 336, 0, 337, 338, + 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, + 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, + 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, + 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 0, 0, 417, 418, 419, 420, 421, 0, 423, 424, + 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 526, 444, 445, 0, 0, 446, 447, 0, 448, 0, + 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, + 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, + 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, + 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 0, 485, 0, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, 512, 513, 514, 515, 523, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, + 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, + 130, 0, 131, 132, 133, 0, 135, 136, 137, 138, + 139, 0, 141, 142, 0, 143, 144, 145, 146, 147, + 148, 0, 0, 149, 150, 151, 152, 153, 154, 155, + 0, 156, 157, 158, 159, 160, 0, 0, 0, 162, + 163, 164, 165, 166, 167, 0, 169, 170, 171, 0, + 172, 173, 174, 175, 176, 177, 0, 0, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, + 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, + 213, 214, 215, 0, 216, 217, 218, 219, 220, 221, + 222, 223, 224, 225, 226, 0, 227, 0, 228, 229, + 230, 231, 0, 232, 0, 233, 0, 0, 0, 236, + 237, 524, 0, 240, 0, 241, 0, 242, 243, 244, + 245, 0, 246, 247, 248, 249, 250, 1847, 252, 0, + 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 0, 267, 268, 269, 270, 271, + 272, 273, 274, 0, 275, 0, 276, 0, 0, 279, + 0, 281, 282, 283, 284, 285, 286, 0, 0, 287, + 0, 289, 0, 0, 291, 292, 293, 294, 295, 296, + 297, 298, 525, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 0, 322, 323, 324, 325, 326, + 0, 327, 328, 0, 330, 0, 331, 332, 333, 334, + 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, + 0, 0, 342, 343, 344, 0, 346, 0, 348, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 0, 0, 0, 0, 360, 361, 362, 0, 364, 365, + 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, + 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, + 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 397, 398, 0, 400, 401, + 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, + 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 0, 0, 427, 428, + 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 526, 444, 445, 0, 0, + 446, 447, 0, 448, 0, 450, 451, 452, 453, 454, + 0, 455, 456, 457, 0, 0, 458, 459, 460, 461, + 462, 0, 463, 464, 465, 466, 467, 468, 469, 470, + 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 0, 485, + 0, 487, 488, 489, 490, 491, 492, 493, 0, 0, + 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, + 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, + 512, 513, 514, 515, 523, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, + 0, 135, 136, 137, 138, 139, 0, 141, 142, 0, + 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, + 151, 152, 153, 154, 155, 0, 156, 157, 158, 159, + 160, 0, 0, 0, 162, 163, 164, 165, 166, 167, + 0, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 0, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, + 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, + 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, + 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + 0, 227, 0, 228, 229, 230, 231, 0, 232, 0, + 233, 0, 0, 0, 236, 237, 524, 0, 240, 0, + 241, 0, 242, 243, 244, 245, 0, 246, 247, 248, + 249, 250, 2216, 252, 0, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 0, + 267, 268, 269, 270, 271, 272, 273, 274, 0, 275, + 0, 276, 0, 0, 279, 0, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 0, 289, 0, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 525, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 0, + 322, 323, 324, 325, 326, 0, 327, 328, 0, 330, + 0, 331, 332, 333, 334, 335, 336, 0, 337, 338, + 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, + 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, + 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, + 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 397, 398, 0, 400, 401, 402, 403, 404, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, + 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 526, 444, 445, 0, 0, 446, 447, 0, 448, 0, + 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, + 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, + 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, + 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 0, 485, 0, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, 512, 513, 514, 515, 523, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, + 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, + 130, 0, 131, 132, 133, 0, 135, 136, 137, 138, + 139, 0, 141, 142, 0, 143, 144, 145, 146, 147, + 148, 0, 0, 149, 150, 151, 152, 153, 154, 155, + 0, 156, 157, 158, 159, 160, 0, 0, 0, 162, + 163, 164, 165, 166, 167, 0, 169, 170, 171, 0, + 172, 173, 174, 175, 176, 177, 0, 0, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, + 191, 192, 193, 0, 194, 0, 195, 196, 197, 198, + 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, + 205, 206, 207, 208, 209, 0, 210, 211, 212, 0, + 213, 214, 215, 0, 216, 217, 218, 219, 220, 221, + 222, 223, 224, 225, 226, 0, 227, 0, 228, 229, + 230, 231, 0, 232, 0, 233, 0, 0, 0, 236, + 237, 524, 0, 240, 0, 241, 0, 242, 243, 244, + 245, 0, 246, 247, 248, 249, 250, 2231, 252, 0, + 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 0, 267, 268, 269, 270, 271, + 272, 273, 274, 0, 275, 0, 276, 0, 0, 279, + 0, 281, 282, 283, 284, 285, 286, 0, 0, 287, + 0, 289, 0, 0, 291, 292, 293, 294, 295, 296, + 297, 298, 525, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 0, 322, 323, 324, 325, 326, + 0, 327, 328, 0, 330, 0, 331, 332, 333, 334, + 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, + 0, 0, 342, 343, 344, 0, 346, 0, 348, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 0, 0, 0, 0, 360, 361, 362, 0, 364, 365, + 366, 367, 368, 369, 0, 370, 371, 372, 373, 374, + 375, 0, 376, 377, 378, 379, 380, 381, 382, 383, + 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 397, 398, 0, 400, 401, + 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, + 412, 413, 414, 415, 416, 0, 0, 417, 418, 419, + 420, 421, 422, 423, 424, 425, 0, 0, 427, 428, + 429, 430, 0, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 526, 444, 445, 0, 0, + 446, 447, 0, 448, 0, 450, 451, 452, 453, 454, + 0, 455, 456, 457, 0, 0, 458, 459, 460, 461, + 462, 0, 463, 464, 465, 466, 467, 468, 469, 470, + 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 483, 484, 0, 485, + 0, 487, 488, 489, 490, 491, 492, 493, 0, 0, + 494, 0, 0, 495, 496, 497, 498, 499, 500, 501, + 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, + 512, 513, 514, 515, 1476, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 0, 125, 126, 127, 0, 0, 0, 1477, 0, + 0, 0, 0, 1478, 129, 130, 0, 131, 132, 133, + 1479, 135, 136, 137, 0, 1480, 1481, 1482, 1483, 0, + 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, + 151, 152, 1484, 1485, 155, 0, 156, 157, 158, 159, + 0, 0, 1486, 0, 1487, 163, 164, 165, 166, 167, + 1488, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 1489, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 1490, 190, 191, 1491, 193, 0, 194, + 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, + 202, 203, 204, 0, 0, 205, 206, 1059, 208, 209, + 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, + 217, 218, 219, 0, 221, 222, 223, 224, 225, 0, + 0, 227, 0, 228, 229, 1492, 231, 0, 232, 0, + 233, 1493, 0, 1494, 236, 237, 0, 1495, 240, 0, + 241, 0, 0, 0, 244, 245, 0, 246, 247, 248, + 249, 250, 251, 252, 1496, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 1497, + 0, 268, 269, 270, 271, 272, 1498, 1499, 0, 1500, + 0, 276, 1501, 1502, 279, 1503, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 1504, 289, 1505, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 1506, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 1507, 1508, + 1509, 323, 324, 325, 0, 0, 327, 328, 1510, 330, + 0, 0, 332, 1511, 334, 335, 336, 0, 337, 338, + 0, 0, 339, 340, 341, 0, 0, 342, 343, 0, + 1512, 346, 1513, 0, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 361, 0, 1514, 364, 365, 0, 367, 368, 369, 0, + 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, + 1515, 380, 381, 382, 383, 0, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 397, 398, 1516, 400, 401, 402, 1517, 404, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 0, 1518, 417, 418, 419, 420, 421, 422, 1519, 424, + 425, 0, 1520, 427, 428, 1521, 430, 0, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, - 443, -1, -1, 446, 447, 448, -1, 450, 451, 452, - 453, -1, 455, 456, 457, 458, 459, 460, 461, -1, - 463, 464, 465, 466, 467, 468, 469, 470, 471, -1, - -1, 474, -1, -1, 477, 478, 479, 480, 481, 482, - 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, - 493, 494, 495, 496, 497, -1, -1, 8, -1, -1, - 11, -1, 505, 506, 15, 16, 17, 18, 19, 20, - 21, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 36, -1, -1, -1, -1, - 41, -1, -1, -1, -1, 46, 8, -1, -1, 11, - -1, -1, 53, 15, 16, 17, 18, 19, 20, 21, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 36, -1, -1, -1, -1, 80, - -1, -1, -1, -1, 46, 8, -1, -1, 11, -1, - -1, 53, 15, 16, 17, 18, 19, 20, 21, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 36, -1, -1, -1, -1, 80, -1, - -1, -1, -1, 46, -1, 126, -1, -1, -1, -1, - 53, -1, -1, 8, -1, -1, 11, -1, -1, -1, - 15, 16, 17, 18, 19, 20, 21, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 80, -1, -1, - -1, 36, -1, -1, -1, 40, -1, -1, -1, -1, - -1, 46, -1, 8, -1, 176, 11, -1, 53, -1, - 15, 16, 17, 18, 19, 20, 21, -1, -1, -1, - -1, -1, 193, -1, -1, -1, -1, 198, -1, -1, - -1, 36, -1, -1, 166, 80, -1, -1, -1, 171, - -1, 46, -1, -1, 176, -1, -1, -1, 53, -1, - 221, 222, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 193, -1, -1, -1, 236, 198, -1, -1, -1, - -1, -1, 165, -1, -1, 80, -1, -1, -1, -1, - -1, -1, -1, 176, -1, -1, -1, -1, -1, 221, - 222, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 193, -1, -1, 274, 236, 198, 277, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 291, -1, -1, 294, -1, -1, -1, -1, 221, 222, - -1, 176, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 274, 236, -1, 277, -1, -1, 193, -1, - -1, -1, -1, 198, -1, -1, -1, -1, -1, 291, - -1, -1, 294, -1, -1, -1, -1, -1, -1, -1, - -1, 176, -1, -1, -1, -1, 221, 222, -1, -1, - -1, 274, -1, -1, 277, -1, -1, -1, 193, -1, - -1, 236, -1, 198, -1, -1, -1, -1, 291, -1, - -1, 294, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 221, 222, -1, -1, - 391, -1, -1, -1, -1, -1, -1, -1, -1, 274, - 323, 236, 277, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 291, -1, -1, 294, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 391, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 274, - -1, -1, 277, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 454, -1, -1, 291, -1, -1, 294, - -1, -1, -1, -1, -1, -1, -1, -1, 391, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 317, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 498, -1, -1, - 501, 502, 503, -1, 505, 506, 507, 508, 509, 510, - -1, -1, -1, -1, -1, -1, 391, -1, -1, -1, - -1, 8, -1, -1, 11, -1, -1, -1, 15, 16, - 17, 18, 19, 20, 21, -1, 498, -1, -1, 501, - 502, 503, -1, 505, 506, 507, 508, 509, 510, 36, - -1, -1, -1, 40, -1, -1, 391, -1, -1, 46, - -1, -1, -1, -1, -1, -1, 53, -1, -1, -1, - -1, -1, -1, -1, -1, 498, -1, -1, 501, 502, - 503, -1, 505, 506, 507, 508, 509, 510, -1, -1, - -1, -1, -1, 80, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 8, -1, - -1, 11, -1, -1, -1, 15, 16, 17, 18, 19, - 20, 21, -1, 498, -1, -1, 501, 502, 503, -1, - 505, 506, 507, 508, 509, 510, 36, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 46, -1, -1, -1, - -1, -1, -1, 53, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 498, -1, -1, 501, 502, 503, -1, - 505, 506, 507, 508, 509, 510, -1, -1, -1, -1, - 80, -1, -1, -1, -1, 8, -1, -1, 11, 176, - -1, -1, 15, 16, 17, 18, 19, 20, 21, -1, - -1, -1, -1, -1, -1, -1, 193, -1, -1, -1, - -1, 198, -1, 36, -1, -1, -1, 40, -1, -1, - -1, -1, -1, 46, -1, -1, -1, -1, -1, -1, - 53, -1, -1, -1, 221, 222, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 236, - -1, -1, -1, -1, -1, -1, -1, 80, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 171, 8, -1, -1, 11, 176, -1, -1, 15, - 16, 17, 18, 19, 20, 21, -1, 274, -1, -1, - 277, -1, -1, 193, -1, -1, -1, -1, 198, -1, - 36, -1, -1, -1, 291, -1, -1, 294, -1, -1, - 46, -1, -1, -1, -1, -1, -1, 53, -1, -1, - -1, 221, 222, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 236, -1, -1, -1, - -1, -1, -1, -1, 80, -1, -1, -1, -1, -1, - -1, -1, -1, 176, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 193, -1, -1, -1, 274, 198, -1, 277, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 291, -1, -1, 294, -1, -1, -1, 221, 222, - -1, -1, -1, -1, 391, -1, -1, 8, -1, -1, - 11, -1, -1, 236, 15, 16, 17, 18, 19, 20, - 21, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 166, -1, -1, -1, -1, 36, -1, -1, -1, -1, - 176, -1, -1, -1, -1, 46, -1, -1, -1, -1, - -1, 274, 53, -1, 277, -1, -1, 193, -1, -1, - -1, -1, 198, -1, -1, -1, -1, -1, 291, -1, - -1, 294, -1, -1, -1, -1, -1, -1, -1, 80, - -1, -1, -1, -1, -1, 221, 222, -1, -1, -1, - -1, 391, -1, -1, -1, -1, -1, -1, -1, -1, - 236, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 498, -1, -1, 501, 502, 503, -1, 505, 506, - 507, 508, 509, 510, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 274, -1, - -1, 277, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 291, -1, -1, 294, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 391, -1, - -1, -1, -1, -1, -1, 176, -1, -1, -1, -1, + 1522, 444, 0, 0, 0, 446, 447, 0, 448, 1523, + 450, 451, 452, 453, 454, 0, 455, 1524, 1525, 0, + 0, 458, 459, 0, 461, 0, 0, 463, 464, 1526, + 466, 467, 468, 469, 470, 1527, 0, 471, 472, 473, + 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 0, 1528, 0, 485, 1529, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, + 497, 498, 499, 500, 1476, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 512, 513, 514, 515, 0, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 0, 125, 126, 127, 0, 0, 0, 1477, 0, + 0, 0, 0, 1478, 129, 130, 0, 131, 132, 133, + 1479, 135, 136, 137, 0, 1480, 1481, 1482, 1483, 0, + 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, + 151, 152, 1484, 1485, 155, 0, 156, 157, 158, 159, + 0, 0, 1486, 0, 1487, 163, 164, 165, 166, 167, + 1488, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 1489, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 1490, 190, 191, 1491, 193, 0, 194, + 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, + 202, 203, 204, 0, 0, 205, 206, 1059, 208, 209, + 0, 210, 211, 212, 0, 1829, 214, 215, 0, 216, + 217, 218, 219, 0, 221, 222, 223, 224, 225, 0, + 0, 227, 0, 228, 229, 1492, 231, 0, 232, 0, + 233, 1493, 0, 1494, 236, 237, 0, 1495, 240, 0, + 241, 0, 0, 0, 244, 245, 0, 246, 247, 248, + 249, 250, 251, 252, 1496, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 1497, + 0, 268, 269, 270, 271, 272, 1498, 1499, 0, 1500, + 0, 276, 1501, 1502, 279, 1503, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 1504, 289, 1505, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 1506, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 1507, 1508, + 1509, 323, 324, 325, 0, 0, 327, 328, 1510, 330, + 0, 0, 332, 1511, 334, 335, 336, 0, 337, 338, + 0, 0, 339, 340, 341, 0, 0, 342, 343, 0, + 1512, 346, 1513, 0, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 361, 0, 1514, 364, 365, 0, 367, 368, 369, 0, + 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, + 1515, 380, 381, 382, 383, 0, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 397, 398, 1516, 400, 401, 402, 1517, 404, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 0, 1518, 417, 418, 419, 420, 421, 422, 1519, 424, + 425, 0, 1520, 427, 428, 1521, 430, 0, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 1522, 444, 0, 0, 0, 446, 447, 0, 448, 1523, + 450, 451, 452, 453, 454, 0, 455, 1524, 1525, 0, + 0, 458, 459, 0, 461, 0, 0, 463, 464, 1526, + 466, 467, 468, 469, 470, 1527, 0, 471, 472, 473, + 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 0, 1528, 0, 485, 1529, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, + 497, 498, 499, 500, 3144, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 512, 513, 514, 515, 0, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 0, 125, 126, 127, 0, 0, 0, 2896, 0, + 0, 0, 0, 2897, 129, 130, 0, 131, 132, 133, + 2898, 135, 136, 137, 0, 1480, 2899, 1482, 1483, 0, + 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, + 151, 152, 1484, 1485, 155, 0, 156, 157, 158, 159, + 0, 0, 2900, 0, 2901, 163, 164, 165, 166, 167, + 2902, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 2903, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 1490, 190, 191, 1491, 193, 0, 194, + 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, + 202, 203, 204, 0, 0, 205, 206, 1059, 208, 209, + 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, + 217, 218, 219, 0, 221, 222, 223, 224, 225, 0, + 0, 227, 0, 228, 229, 1492, 231, 0, 232, 0, + 233, 2904, 0, 2905, 236, 237, 2906, 2907, 240, 0, + 241, 0, 0, 0, 244, 245, 0, 246, 247, 248, + 249, 250, 251, 252, 2908, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 2909, + 0, 268, 269, 270, 271, 272, 1498, 1499, 0, 1500, + 0, 276, 2910, 2911, 279, 2912, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 2913, 289, 2914, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 3145, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 1507, 2916, + 1509, 323, 324, 325, 0, 0, 327, 328, 2918, 330, + 0, 0, 332, 1511, 334, 335, 336, 0, 337, 338, + 0, 0, 339, 340, 341, 0, 0, 342, 343, 0, + 2920, 346, 2921, 0, 349, 350, 351, 352, 353, 354, + 355, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 361, 0, 2922, 364, 365, 0, 367, 368, 369, 0, + 370, 371, 372, 373, 374, 375, 0, 376, 377, 378, + 1515, 380, 381, 382, 383, 0, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 397, 398, 2923, 400, 401, 402, 0, 404, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 0, 2924, 417, 418, 419, 420, 421, 422, 0, 424, + 425, 0, 2926, 427, 428, 1521, 430, 0, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 3146, 444, 0, 0, 0, 446, 447, 0, 448, 2928, + 450, 451, 452, 453, 454, 0, 455, 1524, 1525, 0, + 0, 458, 459, 0, 461, 0, 0, 463, 464, 2929, + 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, + 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 0, 1528, 0, 485, 2931, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, + 497, 498, 499, 500, 523, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 512, 513, 514, 515, 0, + 0, 0, 0, 117, 118, 119, 120, 121, 122, 123, + 124, 0, 125, 126, 127, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 129, 130, 0, 131, 132, 133, + 0, 135, 136, 137, 138, 139, 0, 141, 142, 0, + 143, 144, 145, 146, 147, 148, 0, 0, 149, 150, + 151, 152, 153, 154, 155, 0, 156, 157, 158, 159, + 160, 0, 0, 0, 162, 163, 164, 165, 166, 167, + 0, 169, 170, 171, 0, 172, 173, 174, 175, 176, + 177, 0, 0, 179, 180, 181, 182, 183, 184, 185, + 186, 187, 188, 189, 190, 191, 192, 193, 0, 194, + 0, 195, 196, 197, 198, 199, 200, 0, 0, 201, + 202, 203, 204, 0, 0, 205, 206, 207, 208, 209, + 0, 210, 211, 212, 0, 213, 214, 215, 0, 216, + 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + 0, 227, 0, 228, 229, 230, 231, 0, 232, 0, + 233, 0, 0, 0, 236, 237, 524, 0, 240, 0, + 241, 0, 242, 243, 0, 245, 0, 246, 247, 248, + 249, 250, 251, 252, 0, 254, 255, 256, 257, 0, + 258, 259, 260, 261, 262, 263, 264, 0, 265, 0, + 267, 268, 269, 270, 271, 272, 273, 274, 0, 275, + 0, 276, 0, 0, 279, 0, 281, 282, 283, 284, + 285, 286, 0, 0, 287, 0, 289, 0, 0, 291, + 292, 293, 294, 295, 296, 297, 298, 525, 300, 301, + 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, 320, 0, + 322, 323, 324, 325, 326, 0, 327, 328, 0, 330, + 0, 331, 332, 333, 334, 335, 336, 0, 337, 338, + 0, 0, 339, 340, 341, 0, 0, 342, 343, 344, + 0, 346, 0, 348, 349, 350, 351, 352, 353, 354, + 0, 356, 357, 358, 359, 0, 0, 0, 0, 360, + 361, 362, 0, 364, 365, 366, 367, 368, 369, 0, + 370, 371, 372, 373, 374, 375, 0, 376, 0, 378, + 379, 380, 381, 382, 383, 0, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 397, 398, 0, 400, 401, 402, 403, 0, 405, 406, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 0, 0, 417, 418, 419, 420, 421, 422, 423, 424, + 425, 0, 0, 427, 428, 429, 430, 0, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 526, 444, 445, 0, 0, 446, 447, 0, 448, 0, + 450, 451, 452, 453, 454, 0, 455, 456, 457, 0, + 0, 458, 459, 460, 461, 462, 0, 463, 464, 465, + 466, 467, 468, 469, 470, 0, 0, 471, 472, 473, + 0, 474, 475, 476, 477, 0, 478, 479, 480, 481, + 482, 483, 484, 0, 485, 0, 487, 488, 489, 490, + 491, 492, 493, 0, 0, 494, 0, 0, 495, 496, + 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, + 507, 508, 509, 510, 511, 512, 513, 514, 515, 1766, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 117, 118, + 119, 120, 121, 122, 123, 124, 0, 125, 126, 127, + 0, 0, 0, 1477, 0, 0, 0, 0, 1478, 129, + 130, 0, 131, 132, 133, 1479, 135, 136, 137, 0, + 1480, 1481, 1482, 1483, 0, 143, 144, 145, 146, 147, + 148, 0, 0, 149, 150, 151, 152, 1484, 1485, 155, + 0, 156, 157, 158, 159, 0, 0, 1486, 0, 1487, + 163, 164, 165, 166, 167, 1488, 169, 170, 171, 0, + 172, 173, 174, 175, 176, 177, 0, 1489, 179, 180, + 181, 182, 183, 184, 185, 186, 187, 188, 1490, 190, + 191, 1491, 193, 0, 194, 0, 195, 196, 197, 198, + 199, 200, 0, 0, 201, 202, 203, 204, 0, 0, + 205, 206, 1059, 208, 209, 0, 210, 211, 212, 0, + 213, 214, 215, 0, 216, 217, 218, 219, 0, 221, + 222, 223, 224, 225, 0, 0, 227, 0, 228, 229, + 1492, 231, 0, 232, 0, 233, 1493, 0, 1494, 236, + 237, 0, 1495, 240, 0, 241, 0, 0, 0, 244, + 245, 0, 246, 247, 248, 249, 250, 251, 252, 1496, + 254, 255, 256, 257, 0, 258, 259, 260, 261, 262, + 263, 264, 0, 265, 1497, 0, 268, 269, 270, 271, + 272, 1498, 1499, 0, 1500, 0, 276, 1501, 1502, 279, + 1503, 281, 282, 283, 284, 285, 286, 0, 0, 287, + 1504, 289, 1505, 0, 291, 292, 293, 294, 295, 296, + 297, 298, 0, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 1507, 1508, 1509, 323, 324, 325, 0, + 0, 327, 328, 1510, 330, 0, 0, 332, 1511, 334, + 335, 336, 0, 337, 338, 0, 0, 339, 340, 341, + 0, 0, 342, 343, 0, 1512, 346, 1513, 0, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 0, 0, 0, 0, 360, 361, 0, 1514, 364, 365, + 0, 367, 368, 369, 0, 370, 371, 372, 373, 374, + 375, 0, 376, 377, 378, 1515, 380, 381, 382, 383, + 0, 384, 385, 386, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 397, 398, 1516, 400, 401, + 402, 0, 404, 405, 406, 407, 408, 409, 410, 411, + 412, 413, 414, 415, 416, 0, 1518, 417, 418, 419, + 420, 421, 422, 0, 424, 425, 0, 1520, 427, 428, + 1521, 430, 0, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 0, 444, 0, 0, 0, + 446, 447, 0, 448, 1523, 450, 451, 452, 453, 454, + 0, 455, 1524, 1525, 0, 0, 458, 459, 0, 461, + 0, 0, 463, 464, 1526, 466, 467, 468, 469, 470, + 0, 0, 471, 472, 473, 0, 474, 475, 476, 477, + 0, 478, 479, 480, 481, 482, 0, 1528, 0, 485, + 1529, 487, 488, 489, 490, 491, 492, 493, 0, 1, + 494, 0, 0, 495, 496, 497, 498, 499, 500, 2, + 0, 3, 4, 0, 0, 0, 0, 1, 0, 0, + 512, 513, 514, 515, 0, 0, 0, 2, 0, 6, + 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, + 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, + 0, 0, 8, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 11, 0, 746, 0, 0, + 0, 10, 0, 0, 0, 0, 0, 0, 13, 0, + 0, 0, 0, 11, 0, 746, 0, 0, 0, 0, + 0, 0, 0, 14, 15, 0, 13, 0, 0, 0, + 0, 0, 0, 0, 747, 0, 0, 0, 0, 0, + 18, 14, 15, 0, 0, 0, 0, 0, 19, 0, + 0, 0, 747, 0, 0, 0, 0, 0, 18, 0, + 0, 0, 0, 0, 22, 0, 19, 0, 23, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 22, 0, 0, 0, 23, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, + 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 27, 28, 29, 0, 0, 0, 0, 0, 30, + 0, 0, 31, 0, 0, 0, 0, 0, 0, 27, + 28, 29, 0, 0, 0, 0, 0, 30, 0, 0, + 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 0, 0, 0, 0, 0, 32, + 0, 0, 0, 0, 0, 0, 34, 0, 33, 0, + 0, 0, 0, 35, 0, 0, 0, 36, 0, 0, + 0, 0, 0, 0, 34, 0, 0, 37, 0, 0, + 0, 35, 0, 0, 0, 36, 0, 0, 0, 38, + 0, 0, 0, 39, 0, 37, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, + 0, 39, 0, 40, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, + 0, 40, 43, 0, 0, 0, 0, 44, 0, 0, + 0, 748, 0, 0, 41, 0, 0, 0, 0, 0, + 43, 0, 0, 45, 0, 44, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 45, 0, 0, 0, 0, 0, 46, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 749, 0, 0, 0, 46, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 47 +}; + +static const yytype_int16 yycheck[] = +{ + 7, 516, 0, 0, 47, 835, 895, 73, 920, 16, + 0, 0, 0, 809, 0, 0, 23, 16, 881, 1157, + 741, 0, 0, 7, 1224, 990, 0, 858, 904, 1222, + 1236, 1418, 925, 1692, 1655, 981, 749, 970, 1210, 23, + 38, 20, 2172, 1197, 981, 970, 955, 0, 1079, 1032, + 20, 981, 1298, 17, 2141, 77, 2143, 981, 1466, 1199, + 1191, 77, 2227, 1202, 2162, 1574, 1987, 20, 75, 76, + 39, 959, 970, 0, 1608, 1609, 2612, 0, 46, 1276, + 2314, 0, 2612, 2107, 23, 2651, 0, 1227, 0, 0, + 0, 75, 76, 2161, 1797, 26, 0, 1323, 0, 0, + 0, 2223, 100, 2673, 1124, 0, 0, 35, 895, 1129, + 897, 7, 899, 748, 0, 742, 75, 76, 2649, 0, + 0, 0, 1087, 999, 0, 0, 2666, 23, 749, 0, + 805, 2275, 2276, 2277, 2486, 810, 75, 76, 2294, 5, + 2568, 110, 0, 5, 2572, 9, 9, 1819, 979, 5, + 0, 23, 5, 1712, 0, 1988, 1928, 13, 14, 80, + 13, 14, 1659, 5, 11, 1815, 1710, 2985, 5, 16, + 5, 5, 45, 63, 5, 5, 13, 14, 5, 75, + 76, 1106, 13, 14, 772, 5, 982, 5, 5, 9, + 5, 1053, 1054, 2315, 9, 13, 14, 13, 14, 46, + 5, 5, 1135, 75, 76, 5, 5, 5, 1070, 5, + 1135, 2970, 13, 14, 40, 1816, 1961, 1962, 2302, 139, + 3, 4, 5, 2306, 45, 11, 9, 1972, 2300, 2988, + 16, 1976, 119, 80, 60, 63, 55, 172, 82, 3, + 3, 5, 190, 2365, 2366, 871, 2368, 876, 4, 93, + 100, 1057, 1033, 9, 100, 1086, 876, 970, 2437, 2437, + 1243, 74, 171, 2824, 1140, 4, 90, 1073, 63, 1252, + 9, 34, 35, 244, 171, 124, 287, 3149, 1211, 11, + 106, 1214, 1215, 15, 16, 290, 1211, 802, 2870, 1214, + 1215, 180, 2940, 11, 172, 1433, 226, 15, 16, 5, + 122, 107, 11, 295, 82, 129, 15, 16, 983, 289, + 312, 5, 11, 107, 46, 93, 15, 16, 183, 295, + 995, 312, 2805, 0, 2807, 30, 275, 2530, 1466, 2532, + 122, 122, 2419, 38, 117, 850, 147, 46, 973, 107, + 64, 106, 272, 20, 117, 137, 23, 1519, 80, 75, + 74, 192, 171, 379, 1135, 168, 1186, 1187, 40, 1190, + 278, 38, 119, 388, 366, 53, 1612, 1954, 1955, 1956, + 47, 80, 132, 161, 30, 201, 3280, 30, 166, 3344, + 13, 14, 38, 163, 3388, 38, 161, 2531, 30, 2925, + 212, 41, 120, 40, 30, 192, 2582, 1309, 75, 76, + 77, 148, 3238, 165, 3240, 11, 217, 170, 338, 172, + 1898, 365, 37, 117, 2980, 163, 2581, 1280, 3177, 3477, + 1345, 1346, 33, 100, 250, 2991, 2556, 2130, 478, 1210, + 308, 3312, 4, 478, 260, 218, 108, 9, 132, 289, + 46, 1929, 503, 289, 476, 1338, 272, 104, 59, 237, + 500, 2982, 199, 514, 2478, 500, 2578, 240, 2580, 389, + 3518, 3279, 237, 108, 514, 159, 194, 357, 500, 3473, + 3374, 3436, 337, 3377, 80, 393, 126, 448, 304, 3079, + 3316, 3081, 230, 307, 295, 250, 190, 213, 176, 514, + 132, 271, 518, 175, 1525, 260, 132, 2683, 286, 3147, + 1746, 366, 451, 514, 272, 193, 279, 509, 358, 514, + 198, 352, 358, 2716, 392, 328, 240, 1298, 509, 428, + 63, 451, 279, 180, 284, 3107, 518, 3010, 175, 357, + 356, 428, 3093, 166, 514, 327, 534, 1163, 8, 3411, + 413, 11, 518, 514, 3126, 15, 16, 172, 236, 19, + 20, 21, 472, 2732, 2732, 381, 3437, 3375, 355, 1188, + 514, 323, 357, 1151, 82, 454, 514, 3471, 1188, 456, + 274, 2693, 1980, 279, 400, 93, 370, 1092, 2662, 514, + 274, 588, 289, 208, 428, 279, 1740, 2659, 2509, 588, + 284, 512, 413, 1424, 503, 516, 2679, 421, 2107, 224, + 0, 1322, 3202, 2375, 1801, 454, 1327, 516, 1438, 234, + 416, 2145, 1333, 3149, 416, 272, 478, 3376, 132, 3149, + 2179, 463, 1484, 1485, 3164, 1455, 476, 1340, 518, 451, + 476, 1857, 445, 444, 2178, 82, 518, 1837, 500, 403, + 404, 1566, 505, 506, 1890, 2317, 93, 1509, 514, 513, + 428, 478, 514, 1578, 520, 1580, 1902, 2307, 514, 517, + 451, 514, 2171, 510, 520, 515, 1872, 517, 2165, 515, + 3098, 517, 514, 500, 1657, 3103, 2841, 514, 512, 514, + 514, 1606, 516, 514, 514, 1491, 1932, 514, 2433, 515, + 518, 396, 2848, 1939, 514, 514, 514, 514, 514, 514, + 3052, 464, 1329, 391, 1339, 1511, 749, 1569, 1570, 514, + 514, 2312, 3278, 514, 514, 514, 514, 3287, 514, 1340, + 1551, 1552, 505, 506, 510, 1601, 1602, 1603, 198, 272, + 1561, 1977, 406, 2967, 249, 1981, 361, 476, 1519, 1545, + 396, 435, 749, 396, 1575, 120, 3286, 161, 161, 505, + 506, 221, 446, 166, 2822, 380, 1431, 800, 11, 274, + 274, 500, 454, 2009, 415, 108, 505, 506, 425, 748, + 284, 514, 119, 1604, 503, 507, 508, 509, 510, 749, + 517, 430, 789, 265, 1709, 1710, 2707, 516, 1985, 507, + 508, 509, 510, 800, 468, 748, 505, 506, 507, 508, + 509, 510, 453, 1941, 2828, 789, 505, 506, 507, 508, + 509, 510, 3337, 3338, 357, 881, 800, 2961, 149, 194, + 3386, 291, 514, 237, 237, 454, 848, 80, 835, 836, + 479, 1612, 848, 2420, 2421, 2422, 2423, 1770, 320, 274, + 3371, 800, 1980, 386, 279, 1770, 2968, 1780, 191, 513, + 1783, 858, 420, 784, 422, 1780, 2875, 534, 1783, 206, + 524, 800, 1845, 2784, 2883, 3390, 823, 824, 825, 200, + 868, 868, 286, 286, 244, 3411, 1765, 356, 868, 868, + 868, 3411, 868, 868, 853, 514, 856, 244, 512, 868, + 868, 859, 516, 789, 868, 902, 903, 1412, 3429, 906, + 907, 336, 871, 1624, 800, 2131, 464, 386, 2741, 837, + 838, 2744, 840, 2746, 1552, 868, 1004, 789, 515, 274, + 2058, 518, 513, 1561, 2627, 272, 2629, 274, 800, 1949, + 1823, 3497, 244, 524, 1022, 1798, 1799, 1800, 1026, 3173, + 1873, 868, 71, 72, 3109, 868, 1855, 173, 1873, 868, + 1859, 1738, 959, 1862, 868, 4, 868, 868, 868, 226, + 9, 925, 2496, 970, 868, 1746, 868, 868, 868, 2478, + 5, 978, 979, 868, 868, 1873, 1864, 984, 1765, 478, + 987, 988, 868, 990, 991, 992, 993, 868, 868, 868, + 132, 3089, 868, 868, 973, 514, 173, 868, 1785, 1006, + 370, 500, 4, 1790, 26, 272, 1013, 9, 258, 259, + 2397, 365, 366, 370, 37, 514, 2147, 159, 2149, 245, + 973, 509, 1006, 132, 1031, 1032, 1033, 177, 516, 1013, + 514, 501, 502, 503, 147, 505, 506, 507, 508, 509, + 510, 4, 177, 1009, 1087, 1052, 9, 1006, 161, 1015, + 159, 2251, 1880, 166, 1013, 1776, 1884, 512, 370, 1887, + 1781, 516, 512, 2201, 1071, 108, 516, 1006, 245, 1994, + 352, 748, 749, 2319, 1013, 1082, 1083, 1084, 448, 1086, + 1087, 2278, 509, 3004, 1091, 420, 59, 422, 3253, 516, + 240, 448, 1091, 2796, 512, 74, 514, 1423, 516, 1425, + 1426, 80, 377, 2612, 217, 240, 1937, 1014, 377, 1890, + 1006, 1018, 789, 1120, 93, 137, 342, 1013, 202, 2259, + 74, 1902, 389, 800, 237, 375, 376, 512, 251, 514, + 1137, 1138, 274, 420, 1006, 422, 448, 279, 117, 433, + 119, 1013, 284, 515, 514, 416, 518, 1169, 1170, 172, + 1172, 1932, 466, 1169, 1170, 381, 1172, 514, 1939, 515, + 1873, 374, 518, 117, 1171, 342, 377, 30, 1175, 1176, + 279, 848, 2137, 286, 476, 284, 478, 244, 1185, 1186, + 1187, 514, 295, 1190, 451, 208, 2233, 2326, 2235, 2020, + 416, 868, 244, 2332, 1163, 420, 1977, 422, 171, 177, + 1981, 224, 514, 1210, 381, 515, 1987, 2453, 518, 837, + 838, 234, 840, 8, 515, 515, 11, 518, 518, 2144, + 15, 16, 85, 514, 19, 20, 21, 206, 2009, 515, + 515, 94, 518, 518, 6, 370, 1243, 514, 10, 416, + 466, 36, 166, 342, 26, 1252, 18, 2578, 514, 2580, + 32, 514, 2177, 2178, 202, 118, 873, 1226, 875, 244, + 82, 33, 240, 515, 2095, 37, 518, 514, 26, 1276, + 420, 293, 422, 515, 32, 515, 518, 515, 518, 2142, + 518, 515, 381, 514, 518, 420, 108, 422, 171, 466, + 514, 1298, 514, 435, 42, 274, 973, 2435, 448, 514, + 279, 2439, 2710, 370, 446, 13, 14, 1314, 1297, 1297, + 514, 1297, 1297, 448, 507, 1314, 1323, 416, 370, 2828, + 274, 2230, 500, 2232, 3489, 279, 435, 190, 171, 1006, + 514, 444, 354, 1340, 356, 313, 1013, 446, 361, 497, + 203, 314, 315, 316, 3474, 515, 3476, 223, 518, 328, + 1357, 173, 13, 14, 3441, 137, 1363, 380, 1357, 107, + 1339, 109, 26, 111, 386, 344, 2612, 466, 32, 3456, + 1340, 1048, 515, 497, 328, 518, 2206, 13, 14, 137, + 202, 448, 516, 1060, 289, 370, 1339, 3517, 515, 170, + 344, 518, 370, 2305, 515, 2595, 448, 518, 516, 377, + 514, 2594, 518, 1410, 1411, 515, 379, 515, 518, 1416, + 1087, 1418, 515, 13, 14, 518, 1423, 1424, 1425, 1426, + 2626, 515, 370, 245, 3511, 2579, 221, 515, 515, 3516, + 518, 1438, 1439, 381, 1418, 171, 1443, 515, 1445, 294, + 518, 1448, 420, 416, 422, 59, 1453, 514, 1455, 1456, + 2643, 1458, 2483, 2484, 2593, 1462, 2595, 515, 514, 1443, + 518, 1445, 514, 448, 1448, 417, 445, 223, 416, 1453, + 448, 497, 1456, 137, 1458, 454, 449, 515, 1462, 515, + 518, 152, 518, 350, 1443, 515, 1445, 460, 518, 1448, + 152, 445, 1169, 1170, 1453, 1172, 291, 1456, 152, 1458, + 454, 342, 515, 1462, 1443, 518, 1445, 188, 189, 1448, + 515, 293, 1519, 518, 1453, 2461, 2462, 1456, 466, 1458, + 2351, 2486, 1418, 1462, 2461, 2462, 2463, 152, 3231, 514, + 3233, 2461, 2457, 2458, 152, 293, 2460, 171, 2319, 402, + 381, 514, 405, 1033, 1551, 1552, 1418, 1443, 370, 1445, + 1548, 1548, 1448, 1560, 1561, 13, 14, 1453, 1548, 381, + 1456, 1568, 1458, 40, 515, 416, 1462, 518, 1575, 1548, + 1548, 1443, 354, 1445, 1548, 416, 1448, 258, 259, 515, + 515, 1453, 518, 518, 1456, 515, 1458, 515, 518, 514, + 1462, 274, 350, 1600, 416, 1548, 354, 1604, 40, 515, + 1607, 40, 518, 89, 386, 1612, 1613, 1614, 1615, 1616, + 1617, 1618, 1619, 1620, 1621, 13, 14, 439, 1625, 1626, + 3241, 60, 1621, 1630, 2345, 466, 466, 1634, 386, 293, + 1637, 1638, 1639, 1640, 1641, 1642, 1643, 1644, 1645, 2352, + 3149, 1648, 152, 515, 466, 2891, 518, 480, 1655, 26, + 1657, 515, 152, 515, 518, 32, 518, 515, 365, 366, + 518, 152, 1339, 13, 14, 13, 14, 106, 107, 1676, + 516, 2547, 2453, 1695, 152, 457, 13, 14, 117, 1695, + 314, 315, 316, 5, 13, 14, 350, 2612, 2842, 2889, + 354, 1698, 13, 14, 375, 376, 13, 14, 289, 457, + 13, 14, 1709, 1710, 1674, 352, 501, 502, 503, 428, + 505, 506, 507, 508, 509, 510, 13, 14, 13, 14, + 1210, 514, 386, 13, 14, 13, 14, 171, 2509, 13, + 14, 13, 14, 13, 14, 514, 175, 13, 14, 1746, + 2570, 13, 14, 13, 14, 379, 13, 14, 1755, 514, + 1757, 13, 14, 2925, 365, 366, 1755, 514, 1757, 2965, + 137, 2654, 201, 515, 2676, 418, 1443, 219, 1445, 224, + 2633, 1448, 365, 366, 262, 263, 1453, 2686, 299, 1456, + 3439, 1458, 416, 514, 3443, 1462, 224, 535, 375, 376, + 1797, 224, 540, 457, 1801, 543, 514, 1804, 1805, 459, + 460, 3047, 3451, 3452, 126, 127, 3193, 26, 1298, 3483, + 3484, 250, 296, 32, 2797, 449, 40, 2742, 1137, 1138, + 235, 260, 514, 5, 5, 514, 460, 324, 514, 514, + 514, 2612, 5, 272, 8, 274, 5, 11, 1845, 3498, + 514, 15, 16, 1841, 5, 19, 20, 21, 514, 171, + 1857, 5, 148, 9, 477, 514, 8, 1864, 1865, 11, + 301, 518, 36, 15, 16, 304, 1873, 19, 20, 21, + 514, 1548, 104, 518, 515, 40, 219, 386, 286, 166, + 514, 166, 284, 1890, 59, 514, 235, 1894, 1895, 428, + 1897, 93, 514, 428, 518, 1902, 1903, 1904, 1905, 1906, + 1907, 1908, 3411, 3149, 1911, 1912, 1913, 1914, 1915, 1916, + 1917, 1918, 1919, 1920, 59, 59, 293, 356, 137, 1926, + 1927, 428, 428, 1930, 265, 1932, 2707, 524, 108, 221, + 1937, 476, 1939, 428, 428, 152, 100, 377, 274, 198, + 274, 274, 381, 514, 40, 514, 274, 2880, 274, 152, + 1627, 171, 1959, 516, 13, 2880, 1963, 3344, 1965, 515, + 2885, 400, 1969, 402, 515, 26, 405, 3160, 515, 515, + 1977, 32, 171, 350, 1981, 515, 1983, 354, 1985, 1963, + 1987, 518, 515, 2892, 2893, 1969, 515, 514, 473, 224, + 224, 514, 314, 315, 316, 281, 514, 463, 281, 518, + 3387, 516, 2009, 2784, 1963, 516, 514, 2932, 2933, 386, + 1969, 514, 514, 2020, 2021, 514, 2737, 39, 1695, 472, + 9, 514, 514, 426, 1963, 426, 11, 352, 518, 1519, + 1969, 513, 426, 279, 521, 524, 428, 518, 514, 514, + 180, 162, 171, 791, 117, 518, 515, 221, 454, 3436, + 217, 518, 2095, 2060, 3019, 265, 226, 379, 2065, 2066, + 290, 312, 389, 312, 518, 180, 219, 1963, 390, 3275, + 518, 226, 515, 1969, 293, 514, 137, 514, 274, 3217, + 457, 295, 287, 2090, 2091, 226, 333, 466, 2095, 514, + 412, 1963, 152, 3, 416, 26, 514, 1969, 171, 152, + 514, 32, 2109, 152, 152, 2112, 476, 2114, 152, 3, + 2891, 37, 40, 289, 274, 40, 42, 291, 289, 59, + 171, 11, 1612, 2130, 2131, 40, 166, 449, 515, 515, + 2137, 350, 515, 2140, 515, 354, 514, 885, 460, 291, + 2861, 514, 180, 514, 2925, 166, 3, 39, 3, 512, + 2157, 512, 1829, 428, 476, 428, 2140, 428, 515, 428, + 513, 515, 518, 2170, 1841, 3411, 171, 386, 515, 917, + 521, 516, 514, 2157, 497, 101, 515, 13, 500, 497, + 2187, 2188, 514, 19, 515, 933, 934, 935, 936, 497, + 428, 515, 514, 155, 514, 31, 515, 2204, 514, 2206, + 515, 2140, 249, 515, 514, 473, 137, 40, 2215, 45, + 46, 518, 3137, 3138, 3414, 3127, 59, 518, 2157, 290, + 499, 290, 503, 3004, 3149, 451, 2233, 2234, 2235, 243, + 303, 59, 293, 59, 2233, 2234, 2235, 985, 457, 265, + 428, 314, 315, 316, 2140, 274, 172, 8, 514, 152, + 11, 202, 152, 177, 15, 16, 1746, 152, 19, 20, + 21, 2157, 428, 281, 281, 2272, 3047, 2310, 2140, 147, + 428, 2278, 108, 515, 514, 36, 40, 428, 202, 428, + 352, 514, 208, 161, 515, 2157, 1963, 3000, 166, 350, + 518, 287, 1969, 354, 289, 40, 152, 476, 224, 279, + 515, 171, 514, 514, 59, 185, 379, 2314, 234, 515, + 2299, 2299, 2319, 2299, 2299, 2314, 240, 515, 515, 166, + 80, 512, 515, 515, 143, 386, 515, 501, 502, 503, + 2337, 505, 506, 507, 508, 509, 510, 198, 515, 217, + 171, 524, 268, 416, 2351, 2352, 514, 300, 358, 501, + 502, 503, 2359, 505, 506, 507, 508, 509, 510, 237, + 2359, 518, 293, 289, 180, 515, 514, 3243, 3149, 515, + 294, 514, 3203, 290, 3205, 152, 449, 515, 515, 518, + 514, 175, 439, 515, 515, 3215, 516, 460, 314, 515, + 2397, 514, 514, 40, 518, 321, 457, 515, 514, 40, + 1890, 86, 454, 476, 518, 171, 514, 475, 286, 515, + 515, 198, 1902, 2397, 515, 513, 513, 295, 2095, 350, + 515, 515, 3218, 354, 3220, 518, 515, 500, 515, 460, + 2437, 289, 59, 503, 3346, 361, 2113, 3402, 515, 515, + 515, 514, 1932, 2486, 476, 204, 2453, 117, 1196, 1939, + 515, 40, 2129, 226, 380, 386, 514, 88, 2524, 191, + 221, 1209, 2469, 2140, 279, 0, 3355, 279, 2475, 2476, + 516, 516, 428, 3336, 516, 516, 2519, 516, 503, 2486, + 2157, 516, 1230, 516, 428, 515, 3411, 1977, 516, 516, + 2497, 1981, 516, 2500, 513, 2502, 420, 1987, 422, 515, + 3330, 2397, 2509, 2510, 516, 516, 2513, 2514, 516, 516, + 516, 2518, 2519, 516, 516, 516, 516, 516, 2525, 2009, + 513, 445, 516, 449, 448, 2397, 457, 2570, 454, 516, + 291, 8, 516, 2540, 11, 516, 40, 516, 15, 16, + 2538, 2538, 516, 2550, 516, 1293, 2544, 516, 2538, 516, + 516, 516, 515, 274, 1302, 514, 7, 8, 107, 2538, + 2538, 40, 13, 2570, 2538, 100, 444, 514, 19, 46, + 476, 289, 23, 514, 25, 26, 53, 9, 514, 30, + 31, 32, 351, 518, 35, 2538, 335, 38, 39, 518, + 59, 42, 514, 198, 45, 46, 515, 515, 191, 513, + 459, 91, 2609, 80, 344, 2612, 2613, 518, 2615, 514, + 2609, 515, 147, 40, 2613, 152, 2615, 516, 515, 55, + 2627, 124, 2629, 152, 75, 76, 161, 40, 515, 366, + 3411, 166, 366, 2310, 512, 40, 171, 515, 516, 515, + 514, 514, 40, 518, 454, 180, 514, 309, 514, 100, + 185, 279, 2650, 248, 190, 454, 107, 108, 109, 110, + 111, 439, 514, 3384, 2653, 2653, 102, 2653, 2653, 292, + 74, 74, 80, 9, 515, 515, 514, 368, 1033, 93, + 2678, 515, 217, 2681, 513, 59, 2729, 123, 513, 133, + 503, 272, 289, 1441, 40, 439, 514, 2704, 292, 176, + 2707, 292, 237, 514, 204, 141, 515, 515, 515, 145, + 289, 459, 289, 515, 386, 551, 193, 2724, 2725, 365, + 122, 198, 2729, 37, 451, 2732, 148, 25, 42, 36, + 365, 167, 297, 868, 170, 2596, 1763, 2228, 25, 26, + 501, 502, 503, 3245, 505, 506, 507, 508, 509, 510, + 186, 286, 2759, 3387, 289, 2609, 3340, 3488, 2548, 236, + 295, 8, 1864, 2885, 11, 2340, 2773, 3361, 15, 16, + 3466, 2778, 2779, 3120, 3415, 2818, 2783, 2784, 40, 3424, + 3459, 2788, 147, 3179, 2791, 2792, 1197, 101, 2221, 2796, + 2797, 171, 2234, 2800, 3309, 3413, 161, 2804, 60, 46, + 335, 166, 2218, 2588, 2811, 3422, 53, 2667, 2615, 2486, + 3410, 2298, 1295, 2359, 291, 3000, 2645, 1322, 2204, 1005, + 2804, 1156, 2170, 358, 1734, 2418, 1178, 114, 3396, 2319, + 1698, 3318, 3207, 80, 2187, 1179, 1733, 23, 1975, 2157, + 2732, 1181, 2519, 279, 106, 2804, 1005, 3042, 2855, 2397, + 151, 287, 217, 2445, 800, 1210, 2863, 789, 172, 2396, + 3141, 2538, 1990, 983, 981, 2804, 1873, 2544, 981, 981, + 171, 981, 237, 309, 981, 2882, 1873, 981, 981, 981, + 3322, 416, 3321, 2171, 2891, 2067, 2476, 1439, 2069, 2022, + 2113, 2110, 2493, 2570, 208, 2811, 3307, 2538, 145, 783, + 336, 1807, 1983, 1360, 1340, 1674, 37, 743, 2804, 444, + 224, 42, 2549, 175, 391, 1675, 100, -1, 2925, 454, + 234, 286, 2271, 303, -1, 1673, -1, 1675, 1234, 176, + 295, -1, 2804, -1, 314, 315, 316, 1685, 473, 201, + 475, 476, -1, 1298, -1, -1, 193, -1, -1, -1, + -1, 198, 40, -1, 268, -1, -1, -1, -1, -1, + 2967, -1, -1, 2453, -1, 2972, -1, -1, 2967, -1, + 101, -1, 60, 2650, -1, 1723, 3019, 512, -1, -1, + 515, 516, 517, -1, -1, 2983, -1, -1, 250, 236, + -1, -1, -1, 3000, -1, -1, 3003, 3004, 260, 379, + 314, 2678, -1, -1, 2681, -1, -1, 321, -1, -1, + 272, -1, 3019, 314, 315, 316, -1, -1, 106, 2509, + -1, 498, -1, 859, -1, -1, -1, -1, 505, 506, + 507, 508, 509, 510, -1, -1, 416, -1, -1, -1, + 3047, 172, 304, -1, 291, 3052, -1, 361, -1, -1, + -1, -1, 2729, -1, -1, -1, 3063, 3064, -1, -1, + 3067, -1, 3069, -1, -1, -1, 380, -1, -1, 449, + 1818, 1819, 1820, 1821, 1822, -1, -1, 208, 379, 444, + 460, -1, -1, -1, 535, -1, -1, 3094, -1, 540, + -1, -1, 543, 224, 356, -1, 476, -1, 8, -1, + 551, 11, -1, 234, -1, 15, 16, -1, -1, -1, + -1, 3118, -1, 201, -1, 416, -1, -1, -1, 381, + 500, -1, 2612, -1, -1, -1, -1, 2804, -1, -1, + -1, -1, -1, -1, 514, 449, 46, 268, 400, -1, + -1, 2818, 3149, 53, 391, -1, -1, 512, 449, -1, + -1, 516, -1, -1, -1, 177, -1, -1, 289, 460, + -1, -1, 250, -1, 1519, -1, 3173, -1, 3166, 3167, + 80, -1, 260, -1, 3173, 476, -1, -1, -1, -1, + 202, -1, 3189, 314, 272, 1033, 3193, -1, -1, -1, + 321, -1, -1, -1, -1, -1, 3203, -1, 3205, 500, + 3207, -1, -1, -1, 3211, 1953, 3213, -1, 3215, 3193, + -1, 1033, -1, 514, -1, -1, 304, 2707, 240, 3226, + -1, -1, -1, -1, 3231, 8, 3233, -1, 11, -1, + 361, -1, 15, 16, 3241, 145, 19, 20, 21, -1, + -1, -1, -1, -1, 3242, -1, 3244, 3254, -1, 380, + -1, 498, 3259, -1, -1, 3254, -1, 1612, 505, 506, + 507, 508, 509, 510, -1, -1, 176, -1, 356, -1, + -1, -1, 294, -1, -1, -1, 151, -1, -1, -1, + 177, -1, -1, 193, -1, -1, -1, 3285, 198, -1, + 741, 742, 743, 381, 2784, -1, 171, 3193, -1, -1, + -1, 3308, -1, 3301, -1, 202, 2983, 3305, -1, -1, + -1, 3318, 400, 988, 171, -1, -1, -1, 449, -1, + 1156, 3193, -1, 3330, -1, -1, 236, -1, -1, -1, + -1, 782, 783, 784, -1, -1, -1, 3344, 789, -1, + 791, -1, 3019, 240, -1, -1, -1, -1, 370, 800, + -1, -1, -1, 804, 805, 3362, -1, -1, 809, 810, + 3344, -1, 1210, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 823, 824, 825, -1, -1, -1, -1, -1, + 3387, 291, -1, -1, -1, -1, 837, 838, 1210, 840, + -1, 1746, -1, -1, -1, 3402, -1, 294, 420, -1, + 422, 2891, 853, 3387, 3411, -1, 3413, -1, 859, -1, + -1, -1, -1, -1, 3413, 198, 313, 439, -1, -1, + 871, -1, -1, 445, -1, -1, 448, -1, -1, 3436, + 3428, -1, -1, -1, 885, 2925, -1, -1, 221, 314, + 315, 316, -1, 3450, 3451, 3452, -1, -1, 3344, -1, + 1298, -1, 3436, -1, -1, -1, 3454, 314, 315, 316, + -1, -1, -1, -1, -1, -1, 917, 918, -1, -1, + 3477, -1, 3344, 370, -1, -1, 1298, -1, 929, -1, + 377, 391, 933, 934, 935, 936, -1, -1, -1, 3166, + 3167, 3387, -1, -1, 1330, 782, 1332, 784, 949, -1, + -1, -1, -1, -1, 379, -1, -1, 988, 291, -1, + -1, 3518, -1, -1, 3004, 3387, -1, -1, -1, -1, + -1, -1, 379, 420, -1, 422, -1, 814, -1, -1, + -1, 982, 983, -1, 985, 1890, -1, 988, -1, -1, + 3436, 416, 439, 994, 995, -1, -1, 1902, 445, 1000, + -1, 448, 839, 2301, -1, 1006, -1, 3047, -1, 416, + -1, -1, 1013, -1, 3436, 3242, -1, 3244, 2316, 2317, + 2318, -1, 1023, -1, 449, -1, -1, 1932, -1, 1030, + -1, -1, -1, 2331, 1939, 460, 2334, -1, 498, 1040, + -1, 2339, 449, -1, -1, 505, 506, 507, 508, 509, + 510, 476, 8, 460, -1, 11, -1, -1, 3285, 15, + 16, -1, -1, 19, 20, 21, -1, -1, -1, 476, + -1, -1, 1977, -1, 3301, 500, 1981, -1, 3305, 988, + -1, -1, 1987, -1, -1, -1, -1, -1, 1089, 514, + 46, -1, -1, 500, -1, -1, -1, 53, -1, -1, + -1, -1, -1, -1, 2009, -1, -1, 514, -1, 3149, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 193, -1, -1, -1, -1, 198, 498, -1, - -1, 501, 502, 503, -1, 505, 506, 507, 508, 509, - 510, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 221, 222, -1, -1, -1, -1, -1, -1, -1, 8, - -1, -1, 11, -1, -1, 236, 15, 16, 17, 18, - 19, 20, 21, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 391, -1, 36, -1, -1, - -1, 40, -1, -1, -1, -1, -1, 46, -1, -1, - -1, -1, -1, 274, 53, 498, 277, -1, 501, 502, + -1, 1519, -1, -1, 80, -1, -1, 2425, 2426, 2427, + -1, 13, -1, 970, -1, -1, -1, 19, 1139, -1, + 1141, -1, -1, -1, 981, 1176, -1, 1519, -1, 31, + -1, -1, -1, -1, 1185, 1156, 1157, -1, -1, -1, + -1, -1, 1163, 45, 46, -1, -1, -1, 501, 502, 503, -1, 505, 506, 507, 508, 509, 510, -1, -1, - 291, -1, -1, 294, -1, -1, -1, -1, -1, -1, - -1, 80, -1, 8, -1, -1, 11, -1, -1, -1, - 15, 16, 17, 18, 19, 20, 21, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 36, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 46, -1, -1, -1, -1, -1, -1, 53, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 498, -1, -1, 501, 502, 503, -1, 505, - 506, 507, 508, 509, 510, 80, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 391, -1, 8, -1, -1, 11, -1, 176, -1, 15, - 16, 17, 18, 19, 20, 21, -1, -1, -1, -1, - -1, -1, -1, -1, 193, -1, -1, -1, -1, 198, - 36, -1, -1, 424, -1, -1, -1, -1, -1, -1, - 46, -1, -1, -1, -1, -1, -1, 53, -1, -1, - -1, -1, 221, 222, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 236, -1, -1, - -1, -1, -1, -1, 80, -1, -1, -1, -1, -1, - -1, 176, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 193, -1, - -1, -1, -1, 198, -1, 274, -1, 498, 277, -1, - 501, 502, 503, -1, 505, 506, 507, 508, 509, 510, - -1, -1, 291, -1, -1, 294, 221, 222, -1, -1, - -1, -1, -1, -1, -1, 8, -1, -1, 11, -1, - -1, 236, 15, 16, 17, 18, 19, 20, 21, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 36, -1, -1, -1, -1, -1, -1, - 176, -1, -1, 46, -1, -1, -1, -1, -1, 274, - 53, -1, 277, -1, -1, -1, -1, 193, -1, -1, - -1, -1, 198, -1, -1, -1, 291, -1, -1, 294, - -1, -1, -1, -1, -1, -1, 8, 80, -1, 11, - -1, -1, -1, 15, 16, 221, 222, 19, 20, 21, - -1, -1, 391, -1, -1, -1, -1, -1, -1, -1, - 236, -1, -1, -1, 36, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 46, -1, -1, -1, -1, -1, - -1, 53, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 274, -1, - -1, 277, -1, -1, -1, -1, -1, -1, 80, -1, - -1, -1, -1, -1, -1, 291, -1, -1, 294, -1, - -1, -1, -1, -1, -1, -1, 391, 8, -1, -1, - 11, -1, -1, 176, 15, 16, -1, -1, 19, 20, - 21, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 193, -1, -1, -1, -1, 198, -1, -1, -1, 498, - -1, -1, 501, 502, 503, 46, 505, 506, 507, 508, - 509, 510, 53, -1, -1, -1, -1, -1, 221, 222, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 236, -1, -1, -1, -1, -1, 80, - -1, -1, -1, -1, 176, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 391, -1, -1, -1, -1, - -1, 193, -1, -1, -1, -1, 198, -1, -1, -1, - -1, 274, -1, 498, 277, -1, 501, 502, 503, -1, - 505, 506, 507, 508, 509, 510, -1, -1, 291, 221, - 222, 294, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 236, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 1185, 1410, 1411, -1, -1, -1, + -1, 1416, -1, 1030, -1, 1196, 1197, -1, -1, -1, + -1, 3428, -1, -1, -1, -1, -1, -1, 1209, -1, + -1, 8, -1, -1, 1612, -1, -1, -1, 15, 16, + 176, -1, 19, 20, 21, 1226, 108, 3454, -1, 1230, + -1, -1, -1, -1, 1235, -1, -1, 193, -1, -1, + 1612, -1, 198, -1, -1, -1, -1, 1033, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1096, + -1, -1, -1, -1, -1, -1, 2564, -1, -1, 1106, + -1, -1, -1, -1, -1, -1, 1185, -1, -1, -1, + 236, -1, -1, -1, -1, -1, -1, -1, -1, 1126, + 1291, -1, 1293, -1, -1, -1, -1, -1, 1135, -1, + -1, 1302, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 8, 1313, -1, 11, -1, -1, -1, 15, 16, + -1, 1322, 19, 20, 21, -1, 1327, -1, 1329, 1330, + -1, 1332, 1333, -1, -1, 291, -1, -1, -1, 36, + -1, -1, -1, -1, -1, -1, -1, -1, 1746, -1, + -1, -1, -1, -1, -1, 1741, -1, 2655, -1, 1360, + -1, -1, -1, -1, -1, 1751, -1, 1753, -1, -1, + 1756, 3411, -1, -1, 1746, -1, 1762, -1, 1764, 1410, + 1411, -1, -1, -1, -1, 1416, -1, -1, -1, -1, + -1, 1777, -1, -1, -1, -1, 1782, -1, -1, -1, + 1786, 1787, 1788, 1789, -1, 1791, 1792, -1, -1, 1410, + 1411, -1, -1, -1, 2319, 1416, -1, 1418, -1, 2717, + 2718, 2719, 2720, -1, 1210, -1, -1, -1, -1, -1, + 1431, -1, 1433, 1434, -1, 391, -1, -1, -1, -1, + 1441, -1, 1443, -1, 1445, -1, -1, 1448, -1, -1, + -1, -1, 1453, 1033, -1, 1456, -1, 1458, -1, -1, + -1, 1462, -1, 1464, -1, 1466, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 176, -1, -1, -1, -1, - -1, -1, 274, -1, -1, 277, -1, -1, -1, -1, - -1, -1, 193, -1, -1, -1, -1, 198, -1, 291, - -1, -1, 498, -1, -1, 501, 502, 503, -1, 505, + -1, -1, 1890, -1, 291, -1, -1, -1, -1, -1, + -1, 1410, 1411, -1, 1902, -1, -1, 1416, -1, -1, + -1, -1, 1298, -1, -1, -1, -1, -1, 1890, 1356, + -1, -1, -1, 1360, 221, -1, -1, -1, -1, -1, + 1902, -1, -1, -1, 1932, -1, -1, -1, -1, -1, + -1, 1939, 498, -1, -1, 501, 502, 503, 2453, 505, 506, 507, 508, 509, 510, -1, -1, -1, -1, -1, - 221, 222, -1, -1, -1, -1, -1, -1, 391, -1, - -1, -1, -1, -1, -1, 236, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 277, -1, -1, -1, + 1932, -1, -1, -1, -1, -1, -1, 1939, -1, -1, + -1, 2869, -1, 1574, -1, -1, -1, -1, -1, 1977, + -1, -1, -1, 1981, 1585, 8, -1, -1, 11, 1987, + -1, -1, 15, 16, 291, -1, 19, 20, 21, -1, + -1, -1, -1, -1, 2509, 1977, -1, -1, -1, 1981, + -1, 2009, -1, -1, -1, 1987, -1, -1, -1, -1, + -1, -1, -1, 1624, -1, -1, -1, -1, -1, -1, + 1210, -1, -1, -1, -1, -1, -1, 2009, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 291, -1, -1, -1, -1, -1, -1, -1, -1, 391, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 1698, -1, 551, + 1671, -1, 1673, -1, 1675, -1, -1, -1, 1903, 1904, + 1905, 1906, 1907, 1908, 1685, 1686, 1911, 1912, 1913, 1914, + 1915, 1916, 1917, 1918, 1919, 1920, -1, 1698, -1, -1, + -1, -1, -1, -1, 501, 502, 503, 2612, 505, 506, + 507, 508, 509, 510, -1, -1, -1, -1, 1298, -1, + -1, -1, 1723, -1, 1725, -1, -1, -1, -1, -1, + -1, -1, -1, 1519, -1, -1, -1, -1, -1, 1740, + 1741, -1, -1, -1, 1653, -1, -1, -1, -1, -1, + 1751, 1752, 1753, 1754, -1, 1756, -1, -1, -1, -1, + -1, 1762, -1, 1764, -1, -1, 2152, -1, -1, -1, + -1, 8, -1, -1, 11, 1776, 1777, -1, 15, 16, + 1781, 1782, 19, 20, 21, 1786, 1787, 1788, 1789, 1698, + 1791, 1792, -1, -1, -1, -1, -1, -1, 221, -1, + -1, -1, 2707, -1, 501, 502, 503, 1808, 505, 506, + 507, 508, 509, 510, -1, 1816, -1, 1818, 1819, 1820, + 1821, 1822, -1, -1, -1, -1, 1612, -1, 1665, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1839, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1852, -1, -1, -1, -1, -1, -1, -1, -1, + 742, 743, -1, -1, -1, 2090, 2091, -1, 291, -1, + -1, 3169, 1903, 1904, 1905, 1906, 1907, 1908, -1, 2784, + 1911, 1912, 1913, 1914, 1915, 1916, 1917, 1918, 1919, 1920, + -1, -1, 3190, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 1903, 1904, 1905, 1906, 1907, 1908, -1, -1, + 1911, 1912, 1913, 1914, 1915, 1916, 1917, 1918, 1919, 1920, + -1, 2319, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1519, + 1941, -1, -1, 2329, -1, -1, -1, 2319, -1, -1, + -1, -1, 1953, -1, 3252, -1, 1793, -1, -1, -1, + 1746, -1, 1963, -1, -1, -1, -1, -1, 1969, 1806, + 1807, -1, -1, -1, 1975, -1, -1, 859, -1, 1980, + -1, -1, -1, -1, 221, 3283, 2891, -1, 1989, 1990, + -1, 24, -1, -1, 1903, 1904, 1905, 1906, 1907, 1908, + -1, -1, 1911, 1912, 1913, 1914, 1915, 1916, 1917, 1918, + 1919, 1920, -1, -1, -1, -1, -1, -1, -1, -1, + 2925, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 1612, -1, -1, -1, 1873, -1, -1, -1, + -1, -1, 1879, -1, -1, -1, -1, -1, 81, -1, + -1, -1, -1, -1, 291, 2453, -1, 2058, -1, 2090, + 2091, 2062, -1, -1, 97, -1, 2067, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 501, 502, + 503, 2453, 505, 506, 507, 508, 509, 510, -1, 2090, + 2091, -1, -1, -1, -1, -1, -1, -1, -1, 3004, + -1, -1, -1, -1, 1890, -1, 2107, -1, -1, -1, + -1, 2509, -1, 146, -1, -1, 1902, -1, 1000, -1, + -1, -1, -1, 156, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 168, -1, 2509, -1, 2140, + 173, 1023, 3047, -1, -1, -1, 1932, -1, -1, -1, + -1, 2152, -1, 1939, -1, -1, 2157, -1, -1, -1, + 2161, -1, -1, -1, -1, -1, 1746, -1, -1, 202, + -1, -1, -1, 2204, -1, -1, -1, -1, -1, -1, + -1, 2090, 2091, -1, -1, -1, -1, 2573, 2574, -1, + -1, 1977, -1, -1, -1, 1981, -1, 37, -1, -1, + 2201, 1987, 42, 2204, -1, -1, 2207, 1089, -1, -1, + -1, -1, 245, -1, 2612, 2601, 249, -1, -1, -1, + -1, -1, 2223, 2009, -1, -1, -1, -1, -1, -1, + 2616, 2617, 2618, 2619, 2620, 2621, 2622, 2623, 2624, 2625, + 2612, -1, -1, -1, 3149, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 498, -1, -1, 501, 502, - 503, -1, 505, 506, 507, 508, 509, 510, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 101, -1, -1, 501, 502, 503, -1, 505, 506, + 507, 508, 509, 510, 1156, 1157, -1, -1, -1, -1, + 313, -1, -1, -1, -1, -1, 319, -1, 2513, 2514, + -1, -1, -1, 2294, -1, 2204, -1, -1, -1, -1, + 2301, 334, 126, 127, -1, -1, -1, -1, -1, 2707, + 1890, 2312, -1, -1, 2315, 2316, 2317, 2318, -1, -1, + -1, -1, 1902, -1, -1, -1, -1, -1, 2329, -1, + 2331, -1, 172, 2334, 367, 2707, -1, 370, 2339, -1, + 180, -1, -1, -1, 2345, -1, -1, 171, 381, -1, + -1, 384, 1932, -1, -1, -1, -1, -1, -1, 1939, + -1, -1, -1, -1, 2365, 2366, -1, 2368, 208, 402, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 416, 224, -1, 2784, -1, -1, 422, + 423, -1, -1, -1, 234, -1, 2397, 1977, -1, -1, + 433, 1981, -1, 2240, -1, -1, 439, 1987, -1, -1, + -1, -1, 2784, -1, -1, -1, 2417, -1, -1, -1, + -1, -1, -1, -1, 2425, 2426, 2427, -1, 268, 2009, + -1, -1, -1, 466, 2435, -1, 2437, -1, 2439, -1, + -1, -1, -1, -1, 2445, -1, -1, 1329, 1330, 289, + 1332, -1, -1, -1, -1, 2292, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 391, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 498, -1, -1, 501, - 502, 503, -1, 505, 506, 507, 508, 509, 510, -1, + -1, -1, -1, 2310, 314, -1, 2862, 2478, -1, 2704, + -1, 321, 2513, 2514, -1, -1, -1, -1, -1, -1, + 314, 315, 316, 2891, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 3411, -1, -1, -1, + -1, -1, 2513, 2514, -1, 2352, -1, -1, -1, 2891, + -1, 361, -1, -1, 2361, -1, 2363, 2925, -1, -1, + 2367, -1, 2369, 2319, -1, -1, -1, -1, -1, -1, + 380, 2542, -1, -1, -1, -1, -1, 2548, -1, -1, + -1, 1433, -1, 2925, -1, 379, -1, -1, -1, -1, + -1, -1, -1, 2564, -1, -1, 390, 2568, -1, -1, + -1, 2572, 2573, 2574, -1, -1, -1, 2578, 2579, 2580, + -1, 2582, 1464, -1, 1466, -1, -1, -1, 412, -1, + -1, -1, 416, -1, -1, -1, -1, -1, -1, -1, + 2601, 0, 2603, -1, 2513, 2514, 3004, -1, -1, 449, + -1, -1, -1, -1, 454, 2616, 2617, 2618, 2619, 2620, + 2621, 2622, 2623, 2624, 2625, 449, -1, -1, -1, -1, + -1, -1, 3004, -1, -1, -1, 460, -1, 2863, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 2649, 3047, + -1, -1, 476, -1, 2655, 8, -1, -1, 11, -1, + -1, -1, 15, 16, -1, -1, 2667, 2453, -1, -1, + -1, -1, -1, 2704, -1, 3047, 500, -1, -1, -1, + -1, -1, 2683, -1, -1, -1, -1, -1, -1, -1, + 514, -1, 2693, 46, -1, -1, -1, -1, 2729, -1, + 53, 100, -1, 2704, -1, -1, -1, -1, -1, 2710, + -1, -1, -1, -1, -1, -1, 2717, 2718, 2719, 2720, + -1, -1, -1, 2509, -1, -1, -1, 80, 2729, -1, + -1, 2732, -1, -1, -1, 2736, 2737, -1, -1, 2319, + -1, -1, 2773, 3129, 2745, -1, -1, -1, 147, -1, + -1, 3149, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 161, -1, 3150, 3151, -1, 166, -1, -1, + -1, 2772, 171, -1, -1, 2612, -1, 3149, 3003, -1, + -1, 180, -1, -1, -1, -1, 185, -1, -1, -1, + 3176, -1, 145, -1, -1, 2704, -1, -1, -1, -1, + -1, 2802, -1, 2804, -1, -1, -1, -1, 2645, -1, + -1, -1, -1, -1, -1, -1, -1, 2818, 217, -1, + 2729, 2822, -1, 176, -1, -1, 2612, 2828, -1, -1, + -1, -1, 2863, -1, -1, -1, 26, -1, 237, -1, + 193, 2842, 32, 1725, -1, 198, -1, 2848, -1, -1, + 40, -1, -1, -1, -1, -1, -1, -1, -1, 1741, + 2861, 2862, 2863, -1, -1, -1, -1, -1, 2869, 1751, + 60, 1753, -1, 2453, 1756, -1, -1, -1, -1, -1, + 1762, -1, 1764, 236, -1, 2886, -1, 286, -1, -1, + 289, -1, -1, -1, -1, 1777, 295, -1, -1, -1, + 1782, -1, -1, -1, 1786, 1787, 1788, 1789, -1, 1791, + 1792, -1, -1, -1, -1, -1, 106, -1, -1, -1, + -1, 2707, -1, -1, -1, -1, -1, -1, -1, 2509, + -1, -1, -1, -1, -1, -1, 335, -1, 291, -1, + -1, -1, -1, -1, -1, -1, -1, 137, -1, -1, + -1, -1, -1, -1, 2863, -1, -1, -1, -1, 358, + -1, -1, -1, 2964, -1, -1, -1, 2968, -1, -1, + -1, 2808, 3003, -1, -1, -1, -1, 835, 836, -1, + -1, 2982, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 2996, -1, -1, 2784, -1, + -1, -1, 3003, -1, -1, -1, -1, -1, -1, -1, + -1, 201, -1, 3411, -1, -1, -1, 416, 98, -1, + -1, 3407, -1, -1, -1, -1, -1, -1, 3029, -1, + -1, -1, 2612, -1, 3035, -1, -1, -1, 391, 3411, + -1, 3042, -1, -1, 902, 444, 126, 127, 906, 907, + -1, -1, -1, -1, -1, 454, -1, -1, -1, 1941, + 250, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 260, -1, -1, -1, 473, 3076, 475, 476, -1, -1, + -1, -1, 272, -1, -1, -1, 0, -1, -1, -1, + -1, 171, -1, 1975, 3003, -1, -1, 3098, 1980, -1, + -1, 959, 3103, 293, -1, 2891, -1, -1, -1, -1, + -1, -1, -1, 512, 304, -1, 515, 516, 517, -1, + 978, -1, -1, -1, -1, -1, 984, 2707, 3129, 987, + -1, -1, 990, 991, 992, 993, -1, -1, -1, 2925, + 2977, -1, -1, -1, -1, 498, -1, -1, -1, 3150, + 3151, -1, 505, 506, 507, 508, 509, 510, -1, -1, + 350, -1, -1, 3000, 354, -1, 356, -1, 3169, -1, + 3171, -1, -1, 1031, 1032, 3176, 2058, -1, -1, -1, + -1, -1, 3213, -1, -1, 2067, 100, -1, -1, 3190, + -1, 381, 3193, -1, 1052, -1, 386, -1, -1, -1, + -1, -1, -1, -1, 2784, -1, -1, -1, -1, -1, + 400, -1, 3213, 1071, 68, 69, 3217, 3218, 3004, 3220, + -1, -1, -1, -1, 1082, 1083, 1084, -1, 1086, 1087, + -1, -1, -1, 147, 314, 315, 316, -1, -1, -1, + 3, -1, 5, 3080, 3245, -1, -1, 161, -1, -1, + -1, 3252, 166, -1, -1, 109, 110, 171, -1, 113, + 114, 3047, 1120, -1, -1, -1, 180, 457, -1, -1, + 2152, 185, -1, -1, -1, -1, -1, -1, -1, 1137, + 1138, -1, 3283, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 379, + -1, -1, -1, 217, 3213, 68, 69, -1, -1, -1, + 390, 2891, 3149, 1171, -1, -1, -1, 1175, 1176, 2201, + -1, 3322, -1, 237, -1, 2207, -1, -1, 1186, 1187, + -1, -1, 412, -1, 188, 189, 416, -1, -1, -1, + -1, -1, -1, 3344, -1, 2925, 109, 110, -1, -1, + 113, 114, -1, -1, -1, -1, 436, -1, -1, -1, + -1, -1, -1, 3149, -1, -1, -1, -1, -1, 449, + 3371, -1, 286, -1, -1, 289, -1, -1, -1, -1, + 460, 295, -1, 3384, 3385, 1243, 3387, 3388, -1, -1, + -1, -1, -1, -1, 1252, 3396, 476, -1, 252, 253, + 254, 255, 256, 257, 258, 259, 3407, -1, 262, 263, + -1, -1, -1, -1, -1, -1, -1, -1, 1276, -1, + 500, 335, -1, -1, 3004, 188, 189, -1, 3429, -1, + -1, -1, -1, -1, 514, 3436, -1, -1, -1, 3276, + -1, -1, -1, -1, 358, -1, -1, 2329, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 3047, -1, -1, + -1, -1, 3473, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 3488, -1, 252, + 253, 254, 255, 256, 257, 258, 259, -1, -1, 262, + 263, -1, 416, -1, -1, 1363, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 375, 376, -1, -1, -1, -1, -1, -1, -1, + 444, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 454, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 2435, -1, -1, -1, 2439, -1, 473, + -1, 475, 476, 2445, -1, 1423, -1, 1425, 1426, 3149, + -1, -1, -1, -1, 3411, -1, -1, -1, -1, -1, + 1438, 1439, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 1455, 512, -1, + -1, 515, 516, 517, -1, -1, -1, -1, -1, -1, + -1, -1, 375, 376, -1, -1, 470, 471, -1, -1, + -1, 8, -1, -1, 11, 3411, -1, -1, 15, 16, + 17, 18, 19, 20, 21, -1, -1, -1, -1, -1, + 494, 495, -1, -1, -1, -1, -1, -1, -1, 36, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 46, + 514, -1, -1, -1, -1, -1, 53, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 2568, -1, -1, -1, + 2572, 2573, 2574, 80, -1, -1, -1, -1, -1, -1, + -1, -1, 1560, -1, -1, -1, -1, 470, 471, -1, + 1568, -1, -1, -1, -1, -1, -1, -1, -1, 2601, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 494, 495, -1, 2616, 2617, 2618, 2619, 2620, 2621, + 2622, 2623, 2624, 2625, -1, -1, -1, -1, -1, 1607, + -1, -1, -1, -1, -1, 1613, 1614, 1615, 1616, 1617, + 1618, 1619, 1620, -1, -1, -1, -1, 1625, 1626, -1, + -1, -1, 1630, -1, -1, -1, 1634, -1, -1, 1637, + 1638, 1639, 1640, 1641, 1642, 1643, 1644, 1645, -1, 176, + 1648, -1, -1, -1, -1, -1, -1, 1655, -1, 1657, + -1, -1, -1, -1, -1, -1, 193, -1, -1, -1, + -1, 198, -1, -1, -1, -1, -1, -1, 1676, -1, + -1, -1, -1, -1, -1, 0, -1, -1, 2710, -1, + -1, 3411, -1, -1, 221, 222, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 22, -1, 236, + -1, 1709, 1710, -1, 2736, -1, -1, 32, -1, 34, + 35, 8, -1, -1, 11, -1, -1, -1, 15, 16, + -1, -1, 47, -1, -1, -1, -1, 52, -1, -1, + -1, -1, -1, -1, -1, -1, 61, 274, -1, -1, + 277, -1, -1, -1, -1, -1, -1, -1, -1, 46, + 75, -1, -1, -1, 291, -1, 53, 294, -1, 84, + -1, 86, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 98, -1, 100, -1, -1, -1, -1, + -1, -1, -1, 80, -1, -1, 111, -1, -1, 1797, + -1, -1, -1, 1801, -1, -1, 1804, 1805, -1, -1, + -1, 126, 127, 128, -1, -1, -1, -1, -1, -1, + -1, -1, 137, -1, -1, -1, -1, -1, 143, -1, + -1, -1, -1, -1, -1, -1, 151, -1, 153, 154, + 2862, -1, -1, -1, -1, -1, -1, 1845, -1, -1, + -1, -1, 167, -1, -1, -1, 171, -1, 145, -1, + -1, -1, -1, -1, 391, -1, 1864, 1865, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 197, -1, -1, -1, -1, -1, -1, 176, + -1, -1, -1, -1, -1, -1, 1894, 1895, 213, 1897, + -1, -1, -1, -1, -1, 8, 193, -1, 11, -1, + -1, 198, 15, 16, 17, 18, 19, 20, 21, -1, + -1, -1, -1, -1, 239, -1, -1, -1, 1926, 1927, + -1, -1, 1930, 36, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 46, -1, -1, -1, -1, -1, 236, + 53, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1959, -1, -1, -1, -1, -1, 1965, -1, -1, + -1, 498, -1, -1, 501, 502, 503, 80, 505, 506, + 507, 508, 509, 510, -1, 1983, -1, 1985, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 524, -1, 314, + 315, 316, -1, -1, 291, -1, -1, 322, -1, -1, + 325, -1, -1, 3035, -1, -1, 988, -1, -1, -1, + 3042, -1, -1, 2021, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 354, + -1, -1, -1, -1, -1, -1, -1, -1, 363, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 2060, -1, 379, 24, -1, 2065, 2066, -1, + -1, 386, -1, 176, -1, 390, 3098, -1, -1, -1, + -1, 3103, -1, -1, -1, 400, -1, -1, -1, -1, + 193, -1, -1, -1, -1, 198, -1, 412, -1, -1, + -1, 416, -1, -1, 391, -1, -1, 3129, -1, -1, + -1, 2109, -1, -1, 2112, -1, 2114, -1, 221, 222, + -1, 436, 81, -1, -1, -1, -1, -1, 3150, 3151, + -1, -1, 2130, 236, 449, 1107, -1, 452, 97, -1, + 455, -1, -1, -1, -1, 460, -1, -1, -1, -1, + -1, -1, -1, -1, 3176, -1, -1, -1, -1, -1, + -1, 476, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 274, 2170, -1, 277, -1, -1, -1, -1, -1, + 1152, -1, -1, -1, -1, 500, -1, 146, 291, 2187, + 2188, 294, -1, -1, -1, 3217, -1, 156, -1, 514, + -1, -1, 517, -1, -1, -1, -1, -1, 2206, 168, + -1, 498, -1, 1185, 173, -1, -1, 2215, 505, 506, + 507, 508, 509, 510, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 3, -1, 5, -1, -1, -1, 498, -1, -1, - 501, 502, 503, -1, 505, 506, 507, 508, 509, 510, - 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, - 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, - 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, - 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, - 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, - 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, - 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, - 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, - 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, - 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, - 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, - 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, - 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, - 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, - 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, - 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, - 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, - 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, - 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, - 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, - 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, - 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, - 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, - 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, - 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, - 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, - 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, - 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, - 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, - 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, - 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, - 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 3, -1, -1, -1, + -1, -1, -1, 202, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 8, -1, -1, 11, -1, + -1, -1, 15, 16, -1, 1237, 19, 20, 21, -1, + -1, -1, -1, -1, 2272, -1, -1, -1, -1, -1, + 2278, -1, -1, -1, -1, -1, 245, -1, 391, -1, + 249, -1, -1, 46, -1, -1, -1, -1, -1, -1, + 53, -1, -1, -1, -1, -1, -1, 1279, -1, -1, + -1, -1, -1, -1, 1286, -1, 3, -1, -1, -1, + -1, 8, -1, -1, 11, -1, -1, 80, 15, 16, + 17, 18, 19, 20, 21, -1, -1, -1, -1, 2337, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 36, + -1, -1, -1, 40, -1, -1, -1, -1, -1, 46, + 319, -1, -1, -1, 1336, -1, 53, -1, -1, -1, + -1, -1, -1, -1, -1, 334, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 3407, -1, -1, -1, 1361, + -1, -1, -1, 80, -1, 498, -1, -1, 501, 502, + 503, -1, 505, 506, 507, 508, 509, 510, 367, -1, + -1, 370, -1, -1, -1, 518, -1, -1, -1, -1, + -1, -1, 381, 176, -1, 384, -1, -1, -1, -1, + -1, -1, -1, -1, 1406, -1, 1408, -1, 1410, 1411, + 193, 1413, -1, 402, 1416, 198, -1, 1419, -1, -1, + 1422, -1, -1, -1, -1, 1427, -1, 416, 1430, -1, + -1, -1, -1, -1, 423, -1, -1, -1, 221, 222, + -1, 2469, -1, -1, 433, -1, -1, 2475, 2476, -1, + 439, -1, -1, 236, -1, -1, -1, -1, 2486, 176, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 2497, + 1472, -1, 2500, -1, 2502, -1, 193, 466, -1, -1, + -1, 198, 2510, -1, -1, -1, -1, -1, -1, -1, + 2518, 2519, -1, -1, 277, -1, -1, 2525, -1, -1, + -1, -1, -1, -1, 221, 222, -1, -1, 291, -1, + -1, -1, 2540, -1, -1, -1, -1, -1, -1, 236, + -1, -1, 2550, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, - 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, - 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, - 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, - 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, - 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, - 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, - 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, - 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, - 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, - 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, - 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, - 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, - 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, - 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, - 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, - 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, - 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, - 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, - 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, - 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, - 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, - 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, - 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, - 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, - 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, - 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, - 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, - 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, - 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, - 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, - 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 2570, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 1556, -1, -1, 274, -1, -1, + 277, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 1572, -1, -1, -1, 291, -1, -1, 294, -1, -1, + 1582, 1583, 1584, -1, -1, -1, 1588, -1, -1, -1, + 1592, -1, -1, -1, -1, -1, -1, -1, -1, 2627, + -1, 2629, -1, -1, -1, -1, -1, -1, 391, -1, + 8, -1, -1, 11, -1, -1, -1, 15, 16, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, - 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, - 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, - 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, - 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, - 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, - 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, - 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, - 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, - 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, - 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, - 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, - 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, - 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, - 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, - 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, - 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, - 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, - 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, - 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, - 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, - 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, - 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, - 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, - 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, - 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, - 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, - 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, - 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, - 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, - 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, - 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 3, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 8, -1, + -1, 11, -1, -1, -1, 15, 16, -1, 46, -1, + -1, -1, 1654, -1, -1, 53, -1, 8, -1, -1, + 11, -1, -1, -1, 15, 16, 17, 18, 19, 20, + 21, -1, -1, -1, 391, -1, 46, -1, -1, -1, + -1, -1, 80, 53, -1, 36, -1, -1, -1, 40, + 1692, -1, -1, -1, -1, 46, 2724, 2725, -1, -1, + -1, -1, 53, -1, -1, -1, 1708, -1, -1, -1, + 80, 1713, -1, -1, -1, 498, -1, -1, 501, 502, + 503, -1, 505, 506, 507, 508, 509, 510, 1730, 80, + -1, 2759, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 2773, -1, 145, -1, -1, + 2778, 2779, -1, -1, -1, 2783, -1, -1, -1, -1, + 2788, -1, -1, 2791, 2792, -1, -1, -1, 2796, 2797, + -1, -1, 2800, -1, -1, 145, -1, -1, 176, -1, + -1, 498, -1, 2811, 501, 502, 503, -1, 505, 506, + 507, 508, 509, 510, -1, 193, -1, -1, -1, -1, + 198, -1, -1, -1, -1, -1, 176, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, - 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, - 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, - 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, - 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, - 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, - 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, - 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, - 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, - 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, - 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, - 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, - 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, - 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, - 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, - 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, - 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, - 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, - 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, - 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, - 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, - 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, - 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, - 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, - 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, - 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, - 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, - 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, - 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, - 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, - 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, - 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 193, -1, 176, -1, 2855, 198, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 236, -1, + -1, -1, 193, -1, -1, -1, -1, 198, -1, -1, + -1, -1, -1, -1, 2882, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 236, -1, -1, -1, + 221, 222, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 236, -1, -1, -1, -1, + 1892, -1, -1, 291, -1, -1, -1, -1, 1900, 1901, + -1, 1903, 1904, 1905, 1906, 1907, 1908, -1, -1, 1911, + 1912, 1913, 1914, 1915, 1916, 1917, 1918, 1919, 1920, 1921, + -1, 291, -1, 274, -1, -1, 277, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, - 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, - 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, - 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, - 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, - 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, - 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, - 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, - 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, - 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, - 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, - 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, - 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, - 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, - 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, - 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, - 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, - 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, - 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, - 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, - 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, - 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, - 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, - 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, - 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, - 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, - 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, - 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, - 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, - 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, - 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, - 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 3, -1, 5, -1, + 291, -1, -1, 294, 2972, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 8, -1, + -1, 11, -1, -1, -1, 15, 16, 17, 18, 19, + 20, 21, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 36, -1, -1, -1, + -1, 3019, -1, 391, -1, -1, 46, -1, -1, -1, + -1, -1, -1, 53, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - 37, -1, -1, -1, -1, 42, 43, 44, -1, 46, - 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - 57, -1, 59, 60, 61, 62, 63, 64, -1, -1, - 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, 79, -1, 81, -1, 83, 84, 85, 86, - 87, 88, 89, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, 101, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, - -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, 159, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, 170, -1, 172, 173, 174, 175, 176, - 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, 221, 222, 223, 224, 225, 226, - 227, 228, 229, 230, -1, -1, 233, 234, 235, 236, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, 268, 269, 270, 271, 272, 273, -1, 275, 276, - 277, 278, -1, 280, 281, 282, 283, 284, 285, -1, - 287, 288, 289, -1, 291, 292, 293, -1, -1, 296, - 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, 321, 322, 323, 324, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, 361, 362, 363, 364, 365, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, 380, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, 391, 392, 393, 394, 395, -1, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, - 417, 418, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, 435, 436, - 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, -1, 463, 464, 465, 466, - 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, - 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, - 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 391, 2024, -1, 3052, 2027, -1, -1, -1, -1, + 80, -1, -1, -1, -1, 3063, 3064, -1, -1, 3067, + 391, 3069, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 3094, -1, -1, -1, + -1, 2073, -1, -1, -1, 2077, 2078, 2079, 2080, 2081, + 2082, 2083, 2084, -1, -1, -1, -1, -1, 2090, 2091, + 3118, 2093, 2094, -1, -1, -1, -1, -1, -1, -1, + 498, -1, -1, 2105, -1, -1, 2108, 505, 506, 507, + 508, 509, 510, -1, 2116, 2117, 2118, 2119, 2120, 2121, + 2122, 2123, 2124, 2125, -1, -1, 176, -1, 498, -1, + -1, -1, -1, -1, -1, 505, 506, 507, 508, 509, + 510, -1, -1, 193, -1, -1, -1, 498, 198, 2151, + 501, 502, 503, -1, 505, 506, 507, 508, 509, 510, + -1, 3189, -1, -1, 515, -1, -1, -1, -1, -1, + -1, 221, 222, -1, -1, -1, -1, -1, -1, 3207, + -1, -1, -1, 3211, -1, -1, 236, 3215, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 3226, -1, + -1, -1, 2204, 3231, -1, 3233, -1, -1, -1, -1, + -1, -1, -1, 3241, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 274, -1, -1, 277, -1, -1, + -1, 3259, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 291, -1, -1, 294, -1, -1, -1, 8, -1, + -1, 11, -1, -1, -1, 15, 16, 17, 18, 19, + 20, 21, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 2273, -1, -1, -1, -1, 36, -1, -1, -1, + 3308, 2283, 2284, -1, -1, -1, 46, -1, -1, -1, + 3318, -1, -1, 53, -1, -1, -1, -1, -1, -1, + -1, -1, 3330, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 80, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 3362, -1, -1, -1, -1, -1, + -1, 391, -1, -1, -1, -1, 2348, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 2372, 2373, 2374, -1, -1, 2377, 2378, 2379, 2380, 2381, + 2382, -1, -1, -1, 2386, 2387, 2388, 2389, 2390, 2391, + 2392, 2393, 2394, 2395, -1, -1, -1, -1, 2400, 2401, + -1, -1, 8, -1, -1, 11, -1, -1, -1, 15, + 16, 17, 18, 19, 20, 21, 176, -1, -1, -1, + -1, -1, 3450, 3451, 3452, -1, 2428, -1, -1, -1, + 36, -1, 2434, 193, -1, -1, -1, -1, 198, -1, + 46, -1, -1, -1, -1, -1, -1, 53, 498, 3477, + -1, 501, 502, 503, -1, 505, 506, 507, 508, 509, + 510, 221, 222, -1, -1, 515, 2468, -1, -1, -1, + -1, -1, -1, -1, 80, -1, 236, -1, -1, -1, + -1, -1, -1, -1, -1, 2487, -1, -1, 2490, 2491, + 3518, -1, -1, -1, -1, -1, 2498, 2499, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 2512, 2513, 2514, 2515, 274, 2517, -1, 277, -1, 2521, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 291, -1, -1, 294, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 176, -1, -1, 2575, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 193, -1, -1, + -1, 8, 198, -1, 11, -1, -1, -1, 15, 16, + 17, 18, 19, 20, 21, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 221, 222, -1, -1, 36, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 46, + 236, 391, -1, -1, -1, -1, 53, -1, -1, -1, + -1, 8, -1, -1, 11, -1, -1, -1, 15, 16, + 17, 18, 19, 20, 21, -1, -1, -1, -1, -1, + -1, -1, -1, 80, -1, -1, -1, -1, 274, 36, + -1, 277, -1, -1, -1, -1, -1, -1, -1, 46, + 2682, -1, -1, -1, -1, 291, 53, 26, 294, -1, + -1, -1, -1, 32, -1, -1, -1, -1, -1, -1, + -1, 40, 2704, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 80, -1, -1, -1, -1, -1, -1, + -1, 60, -1, 8, -1, -1, 11, -1, -1, -1, + 15, 16, 17, 18, 19, 20, 21, -1, 498, -1, + -1, 501, 502, 503, -1, 505, 506, 507, 508, 509, + 510, 36, -1, -1, -1, 515, -1, -1, 2760, 176, + 2762, 46, -1, -1, -1, -1, 2768, 106, 53, -1, + -1, -1, -1, -1, -1, 2777, 193, -1, 2780, -1, + 2782, 198, -1, -1, 2786, 391, -1, 2789, 2790, -1, + -1, 2793, 2794, -1, -1, 80, -1, -1, 137, 2801, + -1, -1, -1, -1, 221, 222, -1, -1, 2810, 176, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 236, + -1, -1, -1, 2825, -1, -1, 193, -1, -1, -1, + -1, 198, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 221, 222, -1, 274, -1, -1, + 277, 2863, 201, -1, -1, -1, -1, -1, -1, 236, + -1, -1, -1, -1, 291, -1, -1, 294, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 176, 498, -1, -1, 501, 502, 503, -1, 505, + 506, 507, 508, 509, 510, -1, -1, 274, 193, 515, + 277, 250, -1, 198, -1, -1, -1, -1, -1, -1, + -1, 260, -1, -1, 291, -1, -1, 294, -1, -1, + -1, -1, -1, 272, -1, -1, 221, 222, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 236, -1, -1, 293, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 304, -1, -1, -1, -1, + -1, -1, -1, -1, 391, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 274, + 68, 69, 277, -1, -1, -1, -1, -1, -1, -1, + 3002, 3003, -1, -1, -1, -1, 291, -1, -1, 294, + -1, 350, -1, -1, -1, 354, -1, 356, -1, -1, + -1, -1, -1, -1, 391, 3027, 3028, -1, -1, -1, + -1, 109, 110, -1, -1, 113, 114, -1, -1, -1, + -1, -1, 381, -1, -1, -1, -1, 386, -1, 3051, + -1, -1, -1, 3055, -1, 3057, 3058, 3059, -1, -1, + 3062, 400, -1, 3065, 3066, -1, -1, -1, -1, -1, + -1, -1, 3074, -1, -1, -1, -1, -1, -1, -1, + -1, 498, -1, -1, 501, 502, 503, -1, 505, 506, + 507, 508, 509, 510, -1, -1, -1, -1, 515, -1, + -1, -1, -1, -1, -1, -1, 391, -1, -1, -1, + 188, 189, -1, 3115, -1, -1, -1, -1, 457, 3121, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 498, 3134, -1, 501, 502, 503, -1, 505, 506, + 507, 508, 509, 510, -1, -1, -1, -1, 515, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 3168, -1, -1, -1, + -1, -1, -1, -1, 252, 253, 254, 255, 256, 257, + 258, 259, -1, -1, 262, 263, -1, -1, -1, -1, + -1, -1, -1, 3195, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 498, -1, -1, 501, 502, 503, -1, + 505, 506, 507, 508, 509, 510, 3228, 3229, 3230, -1, + 515, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 3248, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 3260, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 375, 376, -1, + -1, -1, -1, -1, 3306, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 3323, -1, -1, -1, -1, -1, -1, -1, -1, + 3332, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 3357, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 3383, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 470, 471, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 3, 4, 5, + 6, 7, 8, 9, 10, -1, 494, 495, -1, -1, + -1, -1, -1, -1, 3426, -1, 22, 23, 24, 25, + 26, 27, 28, 29, -1, 31, 32, 33, -1, -1, + -1, -1, -1, 39, -1, -1, 42, 43, 44, -1, + 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, -1, 59, 60, 61, 62, 63, 64, 65, + 66, 67, 68, 69, 70, 71, 72, 73, 3480, 75, + 76, 77, 78, 79, -1, 81, -1, 83, 84, 85, + 86, 87, 88, 89, 90, 91, 92, -1, 94, 95, + 96, 97, 98, 99, -1, 101, 102, 103, 104, 105, + 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, + 116, 117, 118, -1, 120, 121, 122, 123, 124, 125, + -1, -1, 128, 129, 130, 131, -1, -1, 134, 135, + 136, 137, 138, -1, 140, 141, 142, -1, 144, 145, + 146, -1, 148, 149, 150, 151, 152, 153, 154, 155, + 156, 157, 158, 159, 160, -1, 162, 163, 164, 165, + -1, 167, -1, 169, 170, -1, 172, 173, 174, 175, + 176, 177, -1, 179, -1, 181, 182, 183, 184, -1, + 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, + 196, 197, -1, 199, 200, 201, 202, 203, 204, 205, + -1, 207, 208, 209, 210, 211, 212, 213, 214, 215, + 216, -1, 218, -1, 220, 221, 222, 223, 224, 225, + 226, 227, 228, 229, 230, -1, -1, 233, 234, 235, + 236, -1, 238, 239, 240, 241, 242, 243, 244, 245, + 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, + 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, + 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, + 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, + -1, 287, 288, -1, -1, 291, 292, 293, -1, -1, + 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, + 306, 307, 308, 309, 310, 311, 312, 313, -1, -1, + -1, -1, 318, 319, 320, 321, 322, 323, 324, 325, + 326, 327, -1, 329, 330, 331, 332, 333, 334, -1, + 336, 337, 338, 339, 340, 341, 342, 343, -1, 345, + 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, + 356, 357, -1, 359, 360, 361, 362, 363, 364, 365, + 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, + 376, 377, 378, -1, -1, 381, 382, 383, 384, 385, + 386, 387, 388, 389, -1, 391, 392, 393, 394, 395, + -1, 397, 398, 399, 400, 401, 402, 403, 404, 405, + 406, 407, 408, 409, 410, 411, -1, -1, 414, 415, + -1, 417, 418, 419, 420, 421, 422, 423, -1, 425, + 426, 427, -1, -1, 430, 431, 432, 433, 434, 435, + 436, 437, 438, 439, 440, 441, 442, 443, -1, -1, + 446, 447, 448, -1, 450, 451, 452, 453, -1, 455, + 456, 457, 458, 459, 460, 461, -1, 463, 464, 465, + 466, 467, 468, 469, 470, 471, -1, -1, 474, -1, + -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, + 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, + 496, 497, 498, -1, -1, -1, -1, -1, -1, 505, + 506, 507, -1, -1, -1, -1, 512, -1, 514, -1, + -1, -1, -1, 519, 520, -1, 522, 523, 524, 3, + 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, + -1, -1, -1, -1, -1, 39, -1, -1, 42, 43, + 44, -1, 46, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 57, -1, 59, 60, 61, 62, 63, + 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, 81, -1, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, 101, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, 117, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, 159, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, 170, -1, 172, 173, + 174, 175, 176, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, 193, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, 208, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, -1, -1, 233, + 234, 235, 236, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, + 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, 299, 300, 301, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, 320, 321, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, 391, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, -1, 417, 418, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + 464, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 498, -1, -1, -1, -1, -1, + -1, 505, 506, 507, -1, -1, -1, -1, 512, -1, + 514, -1, -1, -1, -1, 519, 520, -1, 522, 523, + 524, 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, - 32, 33, 34, 35, -1, 37, -1, -1, -1, -1, + 32, 33, -1, -1, -1, -1, -1, 39, -1, -1, 42, 43, 44, -1, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, -1, 59, 60, 61, - 62, 63, 64, -1, -1, 67, 68, 69, 70, 71, + 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, 79, -1, 81, -1, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, -1, 94, 95, 96, 97, 98, 99, -1, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, -1, 118, -1, 120, 121, - 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, + 112, 113, 114, 115, 116, 117, 118, -1, 120, 121, + 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, -1, 160, -1, - 162, 163, 164, 165, -1, 167, -1, 169, 170, -1, + 152, 153, 154, 155, 156, 157, 158, 159, 160, -1, + 162, 163, 164, 165, -1, 167, -1, 169, 170, 171, 172, 173, 174, 175, 176, 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, -1, 199, 200, 201, @@ -16461,57 +12608,268 @@ static const yytype_int16 yycheck[] = 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, - 272, 273, -1, 275, 276, 277, 278, -1, 280, 281, + 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, -1, -1, -1, -1, 318, 319, 320, 321, + 312, 313, 314, 315, 316, -1, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, -1, 380, 381, - 382, 383, 384, 385, 386, 387, 388, 389, -1, 391, + 372, 373, 374, 375, 376, 377, 378, 379, -1, 381, + 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - -1, -1, 414, 415, -1, 417, 418, 419, 420, 421, + 412, -1, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - 432, 433, 434, -1, 436, 437, 438, 439, 440, 441, - 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, + 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, + 442, 443, -1, -1, 446, 447, 448, 449, 450, 451, 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, 464, 465, 466, 467, 468, 469, 470, 471, - -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, + -1, -1, 474, -1, 476, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 3, -1, -1, -1, + 492, 493, 494, 495, 496, 497, 498, -1, 500, -1, + -1, -1, -1, 505, 506, 507, -1, -1, -1, -1, + 512, -1, 514, 515, -1, -1, -1, 519, 520, -1, + 522, 523, 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 22, 23, 24, 25, 26, 27, 28, 29, -1, + 31, 32, 33, -1, -1, -1, -1, -1, 39, -1, + -1, 42, 43, 44, -1, 46, 47, 48, 49, 50, + 51, 52, 53, 54, 55, 56, 57, -1, 59, 60, + 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, + 71, 72, 73, -1, 75, 76, 77, 78, 79, -1, + 81, -1, 83, 84, 85, 86, 87, 88, 89, 90, + 91, 92, -1, 94, 95, 96, 97, 98, 99, -1, + 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, + 111, 112, 113, 114, 115, 116, 117, 118, -1, 120, + 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, + 131, -1, -1, 134, 135, 136, 137, 138, -1, 140, + 141, 142, -1, 144, 145, 146, -1, 148, 149, 150, + 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, + -1, 162, 163, 164, 165, -1, 167, -1, 169, 170, + 171, 172, 173, 174, 175, 176, 177, -1, 179, -1, + 181, 182, 183, 184, -1, 186, 187, 188, 189, 190, + 191, 192, 193, 194, 195, 196, 197, -1, 199, 200, + 201, 202, 203, 204, 205, -1, 207, 208, 209, 210, + 211, 212, 213, 214, 215, 216, -1, 218, -1, 220, + 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, + -1, -1, 233, 234, 235, 236, -1, 238, 239, 240, + 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, + 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, + 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, + 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, + 281, 282, 283, 284, 285, -1, 287, 288, -1, -1, + 291, 292, 293, -1, -1, 296, 297, 298, 299, 300, + 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, + 311, 312, 313, 314, 315, 316, -1, 318, 319, 320, + 321, 322, 323, 324, 325, 326, 327, -1, 329, 330, + 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, + 341, 342, 343, -1, 345, 346, 347, 348, 349, 350, + 351, 352, 353, 354, 355, 356, 357, -1, 359, 360, + 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, + 371, 372, 373, 374, 375, 376, 377, 378, 379, -1, + 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, + 391, 392, 393, 394, 395, -1, 397, 398, 399, 400, + 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, + 411, 412, -1, 414, 415, 416, 417, 418, 419, 420, + 421, 422, 423, -1, 425, 426, 427, -1, -1, 430, + 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, + 441, 442, 443, -1, -1, 446, 447, 448, 449, 450, + 451, 452, 453, -1, 455, 456, 457, 458, 459, 460, + 461, -1, 463, 464, 465, 466, 467, 468, 469, 470, + 471, -1, -1, 474, -1, 476, 477, 478, 479, 480, + 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, + 491, 492, 493, 494, 495, 496, 497, 498, -1, 500, + -1, -1, -1, -1, 505, 506, 507, -1, -1, -1, + -1, 512, -1, 514, -1, -1, -1, -1, 519, 520, + -1, 522, 523, 3, 4, 5, 6, 7, 8, 9, + 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 22, 23, 24, 25, 26, 27, 28, 29, + -1, 31, 32, 33, -1, -1, -1, -1, -1, 39, + -1, -1, 42, 43, 44, -1, 46, 47, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 57, -1, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, + 70, 71, 72, 73, -1, 75, 76, 77, 78, 79, + -1, 81, -1, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, -1, 94, 95, 96, 97, 98, 99, + -1, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, 118, -1, + 120, 121, 122, 123, 124, 125, -1, -1, 128, 129, + 130, 131, -1, -1, 134, 135, 136, 137, 138, -1, + 140, 141, 142, -1, 144, 145, 146, -1, 148, 149, + 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, + 160, -1, 162, 163, 164, 165, -1, 167, -1, 169, + 170, 171, 172, 173, 174, 175, 176, 177, -1, 179, + -1, 181, 182, 183, 184, -1, 186, 187, 188, 189, + 190, 191, 192, 193, 194, 195, 196, 197, -1, 199, + 200, 201, 202, 203, 204, 205, -1, 207, 208, 209, + 210, 211, 212, 213, 214, 215, 216, -1, 218, -1, + 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, + 230, -1, -1, 233, 234, 235, 236, -1, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, + 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, + 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, + 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, + 280, 281, 282, 283, 284, 285, -1, 287, 288, -1, + -1, 291, 292, 293, -1, -1, 296, 297, 298, 299, + 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, + 310, 311, 312, 313, 314, 315, 316, -1, 318, 319, + 320, 321, 322, 323, 324, 325, 326, 327, -1, 329, + 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, + 340, 341, 342, 343, -1, 345, 346, 347, 348, 349, + 350, 351, 352, 353, 354, 355, 356, 357, -1, 359, + 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, + 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, + -1, 381, 382, 383, 384, 385, 386, 387, 388, 389, + -1, 391, 392, 393, 394, 395, -1, 397, 398, 399, + 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, + 410, 411, -1, -1, 414, 415, 416, 417, 418, 419, + 420, 421, 422, 423, -1, 425, 426, 427, -1, -1, + 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, + 440, 441, 442, 443, -1, -1, 446, 447, 448, 449, + 450, 451, 452, 453, -1, 455, 456, 457, 458, 459, + 460, 461, -1, 463, 464, 465, 466, 467, 468, 469, + 470, 471, -1, -1, 474, -1, 476, 477, 478, 479, + 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, + 490, 491, 492, 493, 494, 495, 496, 497, 498, -1, + 500, -1, -1, -1, -1, 505, 506, 507, -1, -1, + -1, -1, 512, -1, 514, -1, -1, -1, -1, 519, + 520, -1, 522, 523, 3, 4, 5, 6, 7, 8, + 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, 30, 31, 32, 33, -1, -1, -1, 37, -1, + 39, -1, -1, 42, 43, 44, -1, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, -1, + 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, 81, -1, 83, 84, 85, 86, 87, 88, + 89, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, 101, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, 132, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + 159, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, 170, -1, 172, 173, 174, 175, 176, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, 193, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, 208, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, 221, 222, 223, 224, 225, 226, 227, 228, + 229, 230, -1, -1, 233, 234, 235, 236, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, + 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, + 279, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, + 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, 321, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, 380, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, 391, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, 418, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, 435, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, + -1, -1, -1, -1, -1, -1, 505, 506, 507, -1, + -1, -1, -1, 512, -1, 514, 515, -1, -1, -1, + 519, 520, -1, 522, 523, 3, 4, 5, 6, 7, + -1, 9, 10, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 22, 23, 24, 25, 26, 27, + 28, 29, -1, 31, 32, 33, -1, -1, -1, -1, + -1, 39, -1, -1, 42, 43, 44, -1, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, + -1, 59, 60, 61, 62, 63, 64, 65, 66, 67, + 68, 69, 70, 71, 72, 73, -1, 75, 76, 77, + 78, 79, -1, 81, -1, 83, 84, 85, 86, 87, + 88, 89, 90, 91, 92, -1, 94, 95, 96, 97, + 98, 99, -1, 101, 102, 103, 104, 105, 106, 107, + 108, 109, 110, 111, 112, 113, 114, 115, 116, -1, + 118, -1, 120, 121, 122, 123, 124, 125, 126, 127, + 128, 129, 130, 131, -1, -1, 134, 135, 136, 137, + 138, -1, 140, 141, 142, -1, 144, 145, 146, -1, + 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, + 158, 159, 160, -1, 162, 163, 164, 165, -1, 167, + -1, 169, 170, 171, 172, 173, 174, 175, 176, 177, + -1, 179, -1, 181, 182, 183, 184, -1, 186, 187, + 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, + -1, 199, 200, 201, 202, 203, 204, 205, -1, 207, + 208, 209, 210, 211, 212, 213, 214, 215, 216, -1, + 218, -1, 220, 221, 222, 223, 224, 225, 226, 227, + 228, 229, 230, -1, -1, 233, 234, 235, 236, -1, + 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, + 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, + 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, + 268, 269, 270, 271, 272, 273, -1, 275, 276, 277, + 278, 279, 280, 281, 282, 283, 284, 285, -1, 287, + 288, -1, -1, 291, 292, 293, -1, -1, 296, 297, + 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, + 308, 309, 310, 311, 312, 313, 314, 315, 316, -1, + 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, + -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, + 338, 339, 340, 341, 342, 343, -1, 345, 346, 347, + 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, + -1, 359, 360, 361, 362, 363, 364, 365, 366, 367, + 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, + 378, 379, -1, 381, 382, 383, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, -1, 397, + 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, + 408, 409, 410, 411, 412, -1, 414, 415, 416, 417, + 418, 419, 420, 421, 422, 423, -1, 425, 426, 427, + -1, -1, 430, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 443, -1, -1, 446, 447, + 448, 449, 450, 451, 452, 453, -1, 455, 456, 457, + 458, 459, 460, 461, -1, 463, 464, 465, 466, 467, + 468, 469, 470, 471, -1, -1, 474, -1, 476, 477, + 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, + 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, + -1, -1, 500, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 512, -1, 514, -1, -1, -1, + -1, 519, 520, -1, 522, 523, 3, 4, 5, 6, + 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - -1, -1, -1, -1, -1, 42, 43, 44, -1, 46, + -1, -1, 39, -1, -1, 42, 43, 44, -1, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - 57, -1, 59, 60, 61, 62, 63, 64, -1, 66, + 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, 79, -1, 81, -1, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, -1, 94, 95, 96, 97, 98, 99, -1, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, + 117, 118, -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, -1, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, 170, -1, 172, 173, 174, 175, 176, + 157, 158, 159, 160, -1, 162, 163, 164, 165, -1, + 167, -1, 169, 170, 171, 172, 173, 174, 175, 176, 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, -1, 220, 221, 222, 223, 224, 225, 226, - 227, 228, 229, 230, -1, -1, 233, 234, 235, 236, + 227, 228, 229, 230, -1, 232, 233, 234, 235, 236, -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, 268, 269, 270, 271, 272, 273, -1, 275, 276, - 277, 278, -1, 280, 281, 282, 283, 284, 285, -1, + 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, + 277, 278, 279, 280, 281, 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, @@ -16526,29 +12884,240 @@ static const yytype_int16 yycheck[] = 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, 417, 418, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, -1, 436, + 427, -1, 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, 464, 465, 466, 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, + 497, 498, -1, -1, -1, -1, -1, -1, 505, 506, + 507, -1, -1, -1, -1, 512, -1, 514, -1, -1, + -1, -1, 519, 520, -1, 522, 523, 3, 4, 5, + 6, 7, 8, 9, 10, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 22, 23, 24, 25, + 26, 27, 28, 29, -1, 31, 32, 33, -1, -1, + -1, 37, -1, 39, -1, -1, 42, 43, 44, -1, + 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, -1, 59, 60, 61, 62, 63, 64, 65, + 66, 67, 68, 69, 70, 71, 72, 73, -1, 75, + 76, 77, 78, 79, -1, 81, -1, 83, 84, 85, + 86, 87, 88, 89, 90, 91, 92, -1, 94, 95, + 96, 97, 98, 99, -1, 101, 102, 103, 104, 105, + 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, + 116, 117, 118, -1, 120, 121, 122, 123, 124, 125, + -1, -1, 128, 129, 130, 131, -1, -1, 134, 135, + 136, 137, 138, -1, 140, 141, 142, -1, 144, 145, + 146, -1, 148, 149, 150, 151, 152, 153, 154, 155, + 156, 157, 158, 159, 160, -1, 162, 163, 164, 165, + -1, 167, -1, 169, 170, -1, 172, 173, 174, 175, + 176, 177, -1, 179, -1, 181, 182, 183, 184, -1, + 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, + 196, 197, -1, 199, 200, 201, 202, 203, 204, 205, + -1, 207, 208, 209, 210, 211, 212, 213, 214, 215, + 216, -1, 218, -1, 220, 221, 222, 223, 224, 225, + 226, 227, 228, 229, 230, -1, -1, 233, 234, 235, + 236, -1, 238, 239, 240, 241, 242, 243, 244, 245, + 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, + 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, + 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, + 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, + -1, 287, 288, -1, -1, 291, 292, 293, -1, -1, + 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, + 306, 307, 308, 309, 310, 311, 312, 313, -1, -1, + -1, -1, 318, 319, 320, 321, 322, 323, 324, 325, + 326, 327, -1, 329, 330, 331, 332, 333, 334, -1, + 336, 337, 338, 339, 340, 341, 342, 343, -1, 345, + 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, + 356, 357, -1, 359, 360, 361, 362, 363, 364, 365, + 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, + 376, 377, 378, -1, 380, 381, 382, 383, 384, 385, + 386, 387, 388, 389, -1, 391, 392, 393, 394, 395, + -1, 397, 398, 399, 400, 401, 402, 403, 404, 405, + 406, 407, 408, 409, 410, 411, -1, -1, 414, 415, + -1, 417, 418, 419, 420, 421, 422, 423, -1, 425, + 426, 427, -1, -1, 430, 431, 432, 433, 434, 435, + 436, 437, 438, 439, 440, 441, 442, 443, -1, -1, + 446, 447, 448, -1, 450, 451, 452, 453, -1, 455, + 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, + 466, 467, 468, 469, 470, 471, -1, -1, 474, -1, + -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, + 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, + 496, 497, 498, -1, -1, -1, -1, -1, -1, 505, + 506, 507, -1, -1, -1, -1, 512, -1, 514, -1, + -1, -1, -1, 519, 520, -1, 522, 523, 3, 4, + 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 22, 23, 24, + 25, 26, 27, 28, 29, -1, 31, 32, 33, -1, + -1, -1, 37, -1, 39, -1, -1, 42, 43, 44, + -1, 46, 47, 48, 49, 50, 51, 52, 53, 54, + 55, 56, 57, -1, 59, 60, 61, 62, 63, 64, + 65, 66, 67, 68, 69, 70, 71, 72, 73, -1, + 75, 76, 77, 78, 79, -1, 81, -1, 83, 84, + 85, 86, 87, 88, 89, 90, 91, 92, -1, 94, + 95, 96, 97, 98, 99, -1, 101, 102, 103, 104, + 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, + 115, 116, 117, 118, -1, 120, 121, 122, 123, 124, + 125, -1, -1, 128, 129, 130, 131, -1, -1, 134, + 135, 136, 137, 138, -1, 140, 141, 142, -1, 144, + 145, 146, -1, 148, 149, 150, 151, 152, 153, 154, + 155, 156, 157, 158, 159, 160, -1, 162, 163, 164, + 165, -1, 167, -1, 169, 170, -1, 172, 173, 174, + 175, 176, 177, -1, 179, -1, 181, 182, 183, 184, + -1, 186, 187, 188, 189, 190, 191, 192, 193, 194, + 195, 196, 197, -1, 199, 200, 201, 202, 203, 204, + 205, -1, 207, 208, 209, 210, 211, 212, 213, 214, + 215, 216, -1, 218, -1, 220, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 230, -1, -1, 233, 234, + 235, 236, -1, 238, 239, 240, 241, 242, 243, 244, + 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, + 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, + 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, + 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, + 285, -1, 287, 288, -1, -1, 291, 292, 293, -1, + -1, 296, 297, 298, 299, 300, 301, 302, 303, 304, + 305, 306, 307, 308, 309, 310, 311, 312, 313, -1, + -1, -1, -1, 318, 319, 320, 321, 322, 323, 324, + 325, 326, 327, -1, 329, 330, 331, 332, 333, 334, + -1, 336, 337, 338, 339, 340, 341, 342, 343, -1, + 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, -1, 359, 360, 361, 362, 363, 364, + 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, + 375, 376, 377, 378, -1, 380, 381, 382, 383, 384, + 385, 386, 387, 388, 389, -1, 391, 392, 393, 394, + 395, -1, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 409, 410, 411, -1, -1, 414, + 415, -1, 417, 418, 419, 420, 421, 422, 423, -1, + 425, 426, 427, -1, -1, 430, 431, 432, 433, 434, + 435, 436, 437, 438, 439, 440, 441, 442, 443, -1, + -1, 446, 447, 448, -1, 450, 451, 452, 453, -1, + 455, 456, 457, 458, 459, 460, 461, -1, 463, 464, + 465, 466, 467, 468, 469, 470, 471, -1, -1, 474, + -1, -1, 477, 478, 479, 480, 481, 482, 483, 484, + 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, + 495, 496, 497, 498, -1, -1, -1, -1, -1, -1, + 505, 506, 507, -1, -1, -1, -1, 512, -1, 514, + 515, -1, -1, -1, 519, 520, -1, 522, 523, 3, + 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, + -1, -1, -1, -1, 38, 39, -1, -1, 42, 43, + 44, -1, 46, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 57, -1, 59, 60, 61, 62, 63, + 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, 81, -1, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, 101, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, 117, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, 159, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, 170, -1, 172, 173, + 174, 175, 176, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, 193, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, 208, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, -1, -1, 233, + 234, 235, 236, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, + 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, 299, 300, 301, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, 320, 321, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, 391, 392, 393, + 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, -1, 417, 418, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + 464, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 498, -1, -1, -1, -1, -1, + -1, 505, 506, 507, -1, -1, -1, -1, 512, -1, + 514, -1, -1, -1, -1, 519, 520, -1, 522, 523, + 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 22, + 23, 24, 25, 26, 27, 28, 29, -1, 31, 32, + 33, -1, -1, -1, 37, -1, 39, -1, -1, 42, + 43, 44, -1, 46, 47, 48, 49, 50, 51, 52, + 53, 54, 55, 56, 57, -1, 59, 60, 61, 62, + 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, + 73, -1, 75, 76, 77, 78, 79, -1, 81, -1, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, + -1, 94, 95, 96, 97, 98, 99, -1, 101, 102, + 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + 113, 114, 115, 116, 117, 118, -1, 120, 121, 122, + 123, 124, 125, -1, -1, 128, 129, 130, 131, -1, + -1, 134, 135, 136, 137, 138, -1, 140, 141, 142, + -1, 144, 145, 146, -1, 148, 149, 150, 151, 152, + 153, 154, 155, 156, 157, 158, 159, 160, -1, 162, + 163, 164, 165, -1, 167, -1, 169, 170, -1, 172, + 173, 174, 175, 176, 177, -1, 179, -1, 181, 182, + 183, 184, -1, 186, 187, 188, 189, 190, 191, 192, + 193, 194, 195, 196, 197, -1, 199, 200, 201, 202, + 203, 204, 205, -1, 207, 208, 209, 210, 211, 212, + 213, 214, 215, 216, -1, 218, -1, 220, 221, 222, + 223, 224, 225, 226, 227, 228, 229, 230, -1, -1, + 233, 234, 235, 236, -1, 238, 239, 240, 241, 242, + 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, + 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, + 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, + 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, + 283, 284, 285, -1, 287, 288, -1, -1, 291, 292, + 293, -1, -1, 296, 297, 298, 299, 300, 301, 302, + 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, + 313, -1, -1, -1, -1, 318, 319, 320, 321, 322, + 323, 324, 325, 326, 327, -1, 329, 330, 331, 332, + 333, 334, -1, 336, 337, 338, 339, 340, 341, 342, + 343, -1, 345, 346, 347, 348, 349, 350, 351, 352, + 353, 354, 355, 356, 357, -1, 359, 360, 361, 362, + 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, + 373, 374, 375, 376, 377, 378, -1, 380, 381, 382, + 383, 384, 385, 386, 387, 388, 389, -1, 391, 392, + 393, 394, 395, -1, 397, 398, 399, 400, 401, 402, + 403, 404, 405, 406, 407, 408, 409, 410, 411, -1, + -1, 414, 415, -1, 417, 418, 419, 420, 421, 422, + 423, -1, 425, 426, 427, -1, -1, 430, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 443, -1, -1, 446, 447, 448, -1, 450, 451, 452, + 453, -1, 455, 456, 457, 458, 459, 460, 461, -1, + 463, 464, 465, 466, 467, 468, 469, 470, 471, -1, + -1, 474, -1, -1, 477, 478, 479, 480, 481, 482, + 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, + 493, 494, 495, 496, 497, 498, -1, -1, -1, -1, + -1, -1, 505, 506, 507, -1, -1, -1, -1, 512, + -1, 514, 515, -1, -1, -1, 519, 520, -1, 522, + 523, 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, - 32, 33, -1, -1, -1, -1, -1, -1, -1, -1, + 32, 33, -1, -1, -1, 37, -1, 39, -1, -1, 42, 43, 44, -1, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, -1, 59, 60, 61, - 62, 63, 64, -1, -1, 67, 68, 69, 70, 71, + 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, 79, -1, 81, -1, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, -1, 94, 95, 96, 97, 98, 99, -1, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, -1, 118, -1, 120, 121, + 112, 113, 114, 115, 116, 117, 118, -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, -1, 160, -1, + 152, 153, 154, 155, 156, 157, 158, 159, 160, -1, 162, 163, 164, 165, -1, 167, -1, 169, 170, -1, 172, 173, 174, 175, 176, 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, @@ -16560,7 +13129,7 @@ static const yytype_int16 yycheck[] = 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, - 272, 273, -1, 275, 276, 277, 278, -1, 280, 281, + 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, @@ -16570,664 +13139,2677 @@ static const yytype_int16 yycheck[] = 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, -1, -1, 381, + 372, 373, 374, 375, 376, 377, 378, -1, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, -1, 391, 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, 417, 418, 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - 432, 433, 434, -1, 436, 437, 438, 439, 440, 441, + 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, 464, 465, 466, 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 3, -1, -1, -1, + 492, 493, 494, 495, 496, 497, 498, -1, -1, -1, + -1, -1, -1, 505, 506, 507, -1, -1, -1, -1, + 512, -1, 514, -1, -1, -1, -1, 519, 520, -1, + 522, 523, 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 22, 23, 24, 25, 26, 27, 28, 29, 30, + 31, 32, 33, -1, -1, -1, -1, -1, 39, -1, + -1, 42, 43, 44, -1, 46, 47, 48, 49, 50, + 51, 52, 53, 54, 55, 56, 57, -1, 59, 60, + 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, + 71, 72, 73, -1, 75, 76, 77, 78, 79, -1, + 81, -1, 83, 84, 85, 86, 87, 88, 89, 90, + 91, 92, -1, 94, 95, 96, 97, 98, 99, -1, + 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, + 111, 112, 113, 114, 115, 116, 117, 118, -1, 120, + 121, 122, 123, 124, 125, -1, -1, 128, 129, 130, + 131, -1, -1, 134, 135, 136, 137, 138, -1, 140, + 141, 142, -1, 144, 145, 146, -1, 148, 149, 150, + 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, + -1, 162, 163, 164, 165, -1, 167, -1, 169, 170, + -1, 172, 173, 174, 175, 176, 177, -1, 179, -1, + 181, 182, 183, 184, -1, 186, 187, 188, 189, 190, + 191, 192, 193, 194, 195, 196, 197, -1, 199, 200, + 201, 202, 203, 204, 205, -1, 207, 208, 209, 210, + 211, 212, 213, 214, 215, 216, -1, 218, -1, 220, + 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, + -1, -1, 233, 234, 235, 236, -1, 238, 239, 240, + 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, + 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, + 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, + 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, + 281, 282, 283, 284, 285, -1, 287, 288, -1, -1, + 291, 292, 293, -1, -1, 296, 297, 298, 299, 300, + 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, + 311, 312, 313, -1, -1, -1, -1, 318, 319, 320, + 321, 322, 323, 324, 325, 326, 327, -1, 329, 330, + 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, + 341, 342, 343, -1, 345, 346, 347, 348, 349, 350, + 351, 352, 353, 354, 355, 356, 357, -1, 359, 360, + 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, + 371, 372, 373, 374, 375, 376, 377, 378, -1, -1, + 381, 382, 383, 384, 385, 386, 387, 388, 389, -1, + 391, 392, 393, 394, 395, -1, 397, 398, 399, 400, + 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, + 411, -1, -1, 414, 415, -1, 417, 418, 419, 420, + 421, 422, 423, -1, 425, 426, 427, -1, -1, 430, + 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, + 441, 442, 443, -1, -1, 446, 447, 448, -1, 450, + 451, 452, 453, -1, 455, 456, 457, 458, 459, 460, + 461, -1, 463, 464, 465, 466, 467, 468, 469, 470, + 471, -1, -1, 474, -1, -1, 477, 478, 479, 480, + 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, + 491, 492, 493, 494, 495, 496, 497, 498, -1, -1, + -1, -1, -1, -1, 505, 506, 507, -1, -1, -1, + -1, 512, -1, 514, -1, -1, -1, -1, 519, 520, + -1, 522, 523, 3, 4, 5, 6, 7, 8, 9, + 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, -1, -1, -1, -1, -1, 39, + -1, -1, 42, 43, 44, -1, 46, 47, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 57, -1, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, + 70, 71, 72, 73, -1, 75, 76, 77, 78, 79, + -1, 81, -1, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, -1, 94, 95, 96, 97, 98, 99, + -1, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, 118, -1, + 120, 121, 122, 123, 124, 125, -1, -1, 128, 129, + 130, 131, -1, -1, 134, 135, 136, 137, 138, -1, + 140, 141, 142, -1, 144, 145, 146, -1, 148, 149, + 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, + 160, -1, 162, 163, 164, 165, -1, 167, -1, 169, + 170, -1, 172, 173, 174, 175, 176, 177, -1, 179, + -1, 181, 182, 183, 184, -1, 186, 187, 188, 189, + 190, 191, 192, 193, 194, 195, 196, 197, -1, 199, + 200, 201, 202, 203, 204, 205, -1, 207, 208, 209, + 210, 211, 212, 213, 214, 215, 216, -1, 218, -1, + 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, + 230, -1, -1, 233, 234, 235, 236, -1, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, + 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, + 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, + 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, + 280, 281, 282, 283, 284, 285, -1, 287, 288, -1, + -1, 291, 292, 293, -1, -1, 296, 297, 298, 299, + 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, + 310, 311, 312, 313, -1, -1, -1, -1, 318, 319, + 320, 321, 322, 323, 324, 325, 326, 327, -1, 329, + 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, + 340, 341, 342, 343, -1, 345, 346, 347, 348, 349, + 350, 351, 352, 353, 354, 355, 356, 357, -1, 359, + 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, + 370, 371, 372, 373, 374, 375, 376, 377, 378, -1, + -1, 381, 382, 383, 384, 385, 386, 387, 388, 389, + -1, 391, 392, 393, 394, 395, -1, 397, 398, 399, + 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, + 410, 411, -1, -1, 414, 415, -1, 417, 418, 419, + 420, 421, 422, 423, -1, 425, 426, 427, -1, -1, + 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, + 440, 441, 442, 443, -1, -1, 446, 447, 448, -1, + 450, 451, 452, 453, -1, 455, 456, 457, 458, 459, + 460, 461, -1, 463, 464, 465, 466, 467, 468, 469, + 470, 471, -1, -1, 474, -1, -1, 477, 478, 479, + 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, + 490, 491, 492, 493, 494, 495, 496, 497, 498, -1, + -1, -1, -1, -1, -1, 505, 506, 507, -1, -1, + -1, -1, 512, -1, 514, -1, -1, -1, -1, 519, + 520, -1, 522, 523, 3, 4, 5, 6, 7, 8, + 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, + 39, -1, -1, 42, 43, 44, -1, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, -1, + 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, 81, -1, 83, 84, 85, 86, 87, 88, + 89, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, 101, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + 159, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, 170, 171, 172, 173, 174, 175, 176, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, 193, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, 208, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, 221, 222, 223, 224, 225, 226, 227, 228, + 229, 230, -1, -1, 233, 234, 235, 236, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, + 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, + 279, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, + 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, 321, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, 391, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, 418, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, 435, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, 464, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, + -1, -1, -1, -1, -1, -1, 505, 506, 507, -1, + -1, -1, -1, 512, -1, 514, -1, -1, -1, -1, + 519, 520, -1, 522, 523, 3, 4, 5, 6, 7, + 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 22, 23, 24, 25, 26, 27, + 28, 29, -1, 31, 32, 33, -1, -1, -1, -1, + -1, 39, -1, -1, 42, 43, 44, -1, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, + -1, 59, 60, 61, 62, 63, 64, 65, 66, 67, + 68, 69, 70, 71, 72, 73, -1, 75, 76, 77, + 78, 79, -1, 81, -1, 83, 84, 85, 86, 87, + 88, 89, 90, 91, 92, -1, 94, 95, 96, 97, + 98, 99, -1, 101, 102, 103, 104, 105, 106, 107, + 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, + 118, -1, 120, 121, 122, 123, 124, 125, -1, -1, + 128, 129, 130, 131, -1, -1, 134, 135, 136, 137, + 138, -1, 140, 141, 142, -1, 144, 145, 146, -1, + 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, + 158, 159, 160, -1, 162, 163, 164, 165, -1, 167, + -1, 169, 170, -1, 172, 173, 174, 175, 176, 177, + -1, 179, -1, 181, 182, 183, 184, -1, 186, 187, + 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, + -1, 199, 200, 201, 202, 203, 204, 205, -1, 207, + 208, 209, 210, 211, 212, 213, 214, 215, 216, -1, + 218, -1, 220, 221, 222, 223, 224, 225, 226, 227, + 228, 229, 230, -1, -1, 233, 234, 235, 236, -1, + 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, + 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, + 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, + 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, + 278, 279, 280, 281, 282, 283, 284, 285, -1, 287, + 288, -1, -1, 291, 292, 293, -1, -1, 296, 297, + 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, + 308, 309, 310, 311, 312, 313, -1, -1, -1, -1, + 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, + -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, + 338, 339, 340, 341, 342, 343, -1, 345, 346, 347, + 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, + -1, 359, 360, 361, 362, 363, 364, 365, 366, 367, + 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, + 378, -1, -1, 381, 382, 383, 384, 385, 386, 387, + 388, 389, -1, 391, 392, 393, 394, 395, -1, 397, + 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, + 408, 409, 410, 411, -1, -1, 414, 415, -1, 417, + 418, 419, 420, 421, 422, 423, -1, 425, 426, 427, + -1, -1, 430, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 443, -1, -1, 446, 447, + 448, -1, 450, 451, 452, 453, -1, 455, 456, 457, + 458, 459, 460, 461, -1, 463, 464, 465, 466, 467, + 468, 469, 470, 471, -1, -1, 474, -1, -1, 477, + 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, + 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, + 498, -1, -1, -1, -1, -1, -1, 505, 506, 507, + -1, -1, -1, -1, 512, -1, 514, 515, -1, -1, + -1, 519, 520, -1, 522, 523, 3, 4, 5, 6, + 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 43, 44, -1, 46, - 47, 48, -1, 50, 51, 52, 53, 54, -1, 56, - 57, -1, 59, 60, 61, 62, 63, 64, -1, -1, - 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, - 77, 78, 79, -1, -1, 82, 83, 84, 85, 86, - 87, 88, -1, 90, 91, 92, 93, 94, 95, 96, - 97, 98, 99, -1, -1, 102, 103, 104, 105, 106, + -1, -1, 39, -1, -1, 42, 43, 44, -1, 46, + 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + 57, -1, 59, 60, 61, 62, 63, 64, 65, 66, + 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, + 77, 78, 79, -1, 81, -1, 83, 84, 85, 86, + 87, 88, 89, 90, 91, 92, -1, 94, 95, 96, + 97, 98, 99, -1, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, + 117, 118, -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, -1, 160, -1, 162, 163, 164, 165, -1, - 167, 168, 169, -1, -1, -1, 173, 174, 175, -1, + 157, 158, 159, 160, -1, 162, 163, 164, 165, -1, + 167, -1, 169, 170, -1, 172, 173, 174, 175, 176, 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, -1, 194, 195, 196, + 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, -1, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, -1, -1, 223, -1, 225, 226, - 227, 228, 229, 230, -1, -1, 233, -1, 235, -1, + 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, + -1, 218, -1, 220, 221, 222, 223, 224, 225, 226, + 227, 228, 229, 230, -1, -1, 233, 234, 235, 236, -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, -1, 269, 270, 271, 272, 273, -1, 275, 276, - -1, 278, -1, 280, 281, 282, 283, 284, 285, -1, + 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, + 277, 278, 279, 280, 281, 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, 298, -1, 300, -1, 302, 303, 304, 305, 306, + 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, -1, 322, 323, 324, 325, 326, - 327, 328, 329, 330, 331, 332, 333, 334, -1, 336, + -1, 318, 319, 320, 321, 322, 323, 324, 325, 326, + 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, -1, 362, 363, 364, 365, 366, + 357, -1, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, -1, 392, 393, 394, 395, -1, + 387, 388, 389, -1, 391, 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, - 417, -1, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, -1, 436, - 437, 438, 439, 440, 441, 442, 443, -1, 445, 446, + 407, 408, 409, 410, 411, -1, -1, 414, 415, 416, + 417, 418, 419, 420, 421, 422, 423, -1, 425, 426, + 427, -1, -1, 430, 431, 432, 433, 434, 435, 436, + 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, -1, 463, -1, 465, 466, + 457, 458, 459, 460, 461, -1, 463, 464, 465, 466, 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, + 497, 498, -1, -1, -1, -1, -1, -1, 505, 506, + 507, -1, -1, -1, -1, 512, -1, 514, -1, -1, + -1, -1, 519, 520, -1, 522, 523, 3, 4, 5, + 6, 7, 8, 9, 10, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 22, 23, 24, 25, + 26, 27, 28, 29, -1, 31, 32, 33, -1, -1, + -1, -1, -1, 39, -1, -1, 42, 43, 44, -1, + 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, -1, 59, 60, 61, 62, 63, 64, 65, + 66, 67, 68, 69, 70, 71, 72, 73, -1, 75, + 76, 77, 78, 79, -1, 81, -1, 83, 84, 85, + 86, 87, 88, 89, 90, 91, 92, -1, 94, 95, + 96, 97, 98, 99, -1, 101, 102, 103, 104, 105, + 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, + 116, 117, 118, -1, 120, 121, 122, 123, 124, 125, + -1, -1, 128, 129, 130, 131, -1, -1, 134, 135, + 136, 137, 138, -1, 140, 141, 142, -1, 144, 145, + 146, -1, 148, 149, 150, 151, 152, 153, 154, 155, + 156, 157, 158, 159, 160, -1, 162, 163, 164, 165, + -1, 167, -1, 169, 170, -1, 172, 173, 174, 175, + 176, 177, -1, 179, -1, 181, 182, 183, 184, -1, + 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, + 196, 197, -1, 199, 200, 201, 202, 203, 204, 205, + -1, 207, 208, 209, 210, 211, 212, 213, 214, 215, + 216, -1, 218, -1, 220, 221, 222, 223, 224, 225, + 226, 227, 228, 229, 230, -1, -1, 233, 234, 235, + 236, -1, 238, 239, 240, 241, 242, 243, 244, 245, + 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, + 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, + 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, + 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, + -1, 287, 288, -1, -1, 291, 292, 293, -1, -1, + 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, + 306, 307, 308, 309, 310, 311, 312, 313, -1, -1, + -1, -1, 318, 319, 320, 321, 322, 323, 324, 325, + 326, 327, -1, 329, 330, 331, 332, 333, 334, -1, + 336, 337, 338, 339, 340, 341, 342, 343, -1, 345, + 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, + 356, 357, -1, 359, 360, 361, 362, 363, 364, 365, + 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, + 376, 377, 378, -1, -1, 381, 382, 383, 384, 385, + 386, 387, 388, 389, -1, 391, 392, 393, 394, 395, + -1, 397, 398, 399, 400, 401, 402, 403, 404, 405, + 406, 407, 408, 409, 410, 411, -1, -1, 414, 415, + -1, 417, 418, 419, 420, 421, 422, 423, -1, 425, + 426, 427, -1, -1, 430, 431, 432, 433, 434, 435, + 436, 437, 438, 439, 440, 441, 442, 443, -1, -1, + 446, 447, 448, -1, 450, 451, 452, 453, -1, 455, + 456, 457, 458, 459, 460, 461, -1, 463, 464, 465, + 466, 467, 468, 469, 470, 471, -1, 473, 474, -1, + -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, + 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, + 496, 497, 498, -1, -1, -1, -1, -1, -1, 505, + 506, 507, -1, -1, -1, -1, 512, -1, 514, -1, + -1, -1, -1, 519, 520, -1, 522, 523, 3, 4, + 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 33, -1, + -1, -1, -1, -1, 39, -1, -1, 42, 43, 44, + -1, 46, 47, 48, 49, 50, 51, 52, 53, 54, + 55, 56, 57, -1, 59, 60, 61, 62, 63, 64, + 65, 66, 67, 68, 69, 70, 71, 72, 73, -1, + 75, 76, 77, 78, 79, -1, 81, -1, 83, 84, + 85, 86, 87, 88, 89, 90, 91, 92, -1, 94, + 95, 96, 97, 98, 99, -1, 101, 102, 103, 104, + 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, + 115, 116, 117, 118, -1, 120, 121, 122, 123, 124, + 125, -1, -1, 128, 129, 130, 131, -1, -1, 134, + 135, 136, 137, 138, -1, 140, 141, 142, -1, 144, + 145, 146, -1, 148, 149, 150, 151, 152, 153, 154, + 155, 156, 157, 158, 159, 160, -1, 162, 163, 164, + 165, -1, 167, -1, 169, 170, -1, 172, 173, 174, + 175, 176, 177, -1, 179, -1, 181, 182, 183, 184, + -1, 186, 187, 188, 189, 190, 191, 192, 193, 194, + 195, 196, 197, -1, 199, 200, 201, 202, 203, 204, + 205, -1, 207, 208, 209, 210, 211, 212, 213, 214, + 215, 216, -1, 218, -1, 220, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 230, -1, -1, 233, 234, + 235, 236, -1, 238, 239, 240, 241, 242, 243, 244, + 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, + 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, + 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, + 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, + 285, -1, 287, 288, -1, -1, 291, 292, 293, -1, + -1, 296, 297, 298, 299, 300, 301, 302, 303, 304, + 305, 306, 307, 308, 309, 310, 311, 312, 313, -1, + -1, -1, -1, 318, 319, 320, 321, 322, 323, 324, + 325, 326, 327, -1, 329, 330, 331, 332, 333, 334, + -1, 336, 337, 338, 339, 340, 341, 342, 343, -1, + 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, -1, 359, 360, 361, 362, 363, 364, + 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, + 375, 376, 377, 378, -1, -1, 381, 382, 383, 384, + 385, 386, 387, 388, 389, -1, 391, 392, 393, 394, + 395, -1, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 409, 410, 411, -1, -1, 414, + 415, -1, 417, 418, 419, 420, 421, 422, 423, -1, + 425, 426, 427, -1, -1, 430, 431, 432, 433, 434, + 435, 436, 437, 438, 439, 440, 441, 442, 443, -1, + -1, 446, 447, 448, -1, 450, 451, 452, 453, -1, + 455, 456, 457, 458, 459, 460, 461, -1, 463, 464, + 465, 466, 467, 468, 469, 470, 471, -1, -1, 474, + -1, -1, 477, 478, 479, 480, 481, 482, 483, 484, + 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, + 495, 496, 497, 498, -1, -1, -1, -1, -1, -1, + 505, 506, 507, -1, -1, -1, -1, 512, -1, 514, + -1, -1, -1, -1, 519, 520, -1, 522, 523, 3, + 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, + -1, -1, -1, -1, -1, 39, -1, -1, 42, 43, + 44, -1, 46, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 57, -1, 59, 60, 61, 62, 63, + 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, 81, -1, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, 101, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, 117, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, 159, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, 170, -1, 172, 173, + 174, 175, 176, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, 193, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, 208, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, -1, -1, 233, + 234, 235, 236, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, + 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, 299, 300, 301, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, 320, 321, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, 391, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + 464, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 498, -1, -1, -1, -1, -1, + -1, 505, 506, 507, -1, -1, -1, -1, 512, -1, + 514, -1, -1, -1, -1, 519, 520, -1, 522, 523, + 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 22, + 23, 24, 25, 26, 27, 28, 29, -1, 31, 32, + 33, -1, -1, -1, -1, -1, 39, -1, -1, 42, + 43, 44, -1, 46, 47, 48, 49, 50, 51, 52, + 53, 54, 55, 56, 57, -1, 59, 60, 61, 62, + 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, + 73, -1, 75, 76, 77, 78, 79, -1, 81, -1, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, + -1, 94, 95, 96, 97, 98, 99, -1, 101, 102, + 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + 113, 114, 115, 116, 117, 118, -1, 120, 121, 122, + 123, 124, 125, -1, -1, 128, 129, 130, 131, -1, + -1, 134, 135, 136, 137, 138, -1, 140, 141, 142, + -1, 144, 145, 146, -1, 148, 149, 150, 151, 152, + 153, 154, 155, 156, 157, 158, 159, 160, -1, 162, + 163, 164, 165, -1, 167, -1, 169, 170, -1, 172, + 173, 174, 175, 176, 177, -1, 179, -1, 181, 182, + 183, 184, -1, 186, 187, 188, 189, 190, 191, 192, + 193, 194, 195, 196, 197, -1, 199, 200, 201, 202, + 203, 204, 205, -1, 207, 208, 209, 210, 211, 212, + 213, 214, 215, 216, -1, 218, -1, 220, 221, 222, + 223, 224, 225, 226, 227, 228, 229, 230, -1, -1, + 233, 234, 235, 236, -1, 238, 239, 240, 241, 242, + 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, + 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, + 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, + 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, + 283, 284, 285, -1, 287, 288, -1, -1, 291, 292, + 293, -1, -1, 296, 297, 298, 299, 300, 301, 302, + 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, + 313, -1, -1, -1, -1, 318, 319, 320, 321, 322, + 323, 324, 325, 326, 327, -1, 329, 330, 331, 332, + 333, 334, -1, 336, 337, 338, 339, 340, 341, 342, + 343, -1, 345, 346, 347, 348, 349, 350, 351, 352, + 353, 354, 355, 356, 357, -1, 359, 360, 361, 362, + 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, + 373, 374, 375, 376, 377, 378, -1, -1, 381, 382, + 383, 384, 385, 386, 387, 388, 389, -1, 391, 392, + 393, 394, 395, -1, 397, 398, 399, 400, 401, 402, + 403, 404, 405, 406, 407, 408, 409, 410, 411, -1, + -1, 414, 415, 416, 417, 418, 419, 420, 421, 422, + 423, -1, 425, 426, 427, -1, -1, 430, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 443, -1, -1, 446, 447, 448, -1, 450, 451, 452, + 453, -1, 455, 456, 457, 458, 459, 460, 461, -1, + 463, 464, 465, 466, 467, 468, 469, 470, 471, -1, + -1, 474, -1, -1, 477, 478, 479, 480, 481, 482, + 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, + 493, 494, 495, 496, 497, 498, -1, -1, -1, -1, + -1, -1, 505, 506, 507, -1, -1, -1, -1, 512, + -1, 514, -1, -1, -1, -1, 519, 520, -1, 522, + 523, 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, - 32, 33, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 43, 44, -1, 46, 47, 48, -1, 50, 51, - 52, 53, 54, -1, 56, 57, -1, 59, 60, 61, - 62, 63, 64, -1, -1, 67, 68, 69, 70, 71, - 72, 73, 74, 75, 76, 77, 78, 79, -1, -1, - -1, 83, 84, 85, 86, 87, 88, -1, 90, 91, - 92, 93, 94, 95, 96, 97, 98, 99, -1, -1, + 32, 33, -1, -1, -1, -1, -1, 39, -1, -1, + 42, 43, 44, -1, 46, 47, 48, 49, 50, 51, + 52, 53, 54, 55, 56, 57, -1, 59, 60, 61, + 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, + 72, 73, -1, 75, 76, 77, 78, 79, -1, 81, + -1, 83, 84, 85, 86, 87, 88, 89, 90, 91, + 92, -1, 94, 95, 96, 97, 98, 99, -1, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, -1, 118, -1, 120, 121, + 112, 113, 114, 115, 116, 117, 118, -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, -1, 160, -1, - 162, 163, 164, 165, -1, 167, 168, 169, -1, -1, - -1, 173, 174, 175, -1, 177, -1, 179, -1, 181, + 152, 153, 154, 155, 156, 157, 158, 159, 160, -1, + 162, 163, 164, 165, -1, 167, -1, 169, 170, -1, + 172, 173, 174, 175, 176, 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, - 192, -1, 194, 195, 196, 197, -1, 199, 200, 201, - 202, 203, 204, 205, -1, 207, -1, 209, 210, 211, - 212, 213, 214, 215, 216, -1, 218, -1, 220, -1, - -1, 223, -1, 225, 226, 227, 228, 229, 230, -1, - -1, 233, -1, 235, 236, -1, 238, 239, 240, 241, + 192, 193, 194, 195, 196, 197, -1, 199, 200, 201, + 202, 203, 204, 205, -1, 207, 208, 209, 210, 211, + 212, 213, 214, 215, 216, -1, 218, -1, 220, 221, + 222, 223, 224, 225, 226, 227, 228, 229, 230, -1, + -1, 233, 234, 235, 236, -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, -1, 269, 270, 271, - 272, 273, -1, 275, 276, -1, 278, -1, 280, 281, + 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, + 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, - 292, 293, -1, -1, 296, 297, 298, -1, 300, -1, + 292, 293, -1, -1, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, -1, -1, -1, -1, 318, 319, 320, -1, - 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, + 312, 313, -1, -1, -1, -1, 318, 319, 320, 321, + 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, -1, 359, 360, -1, + 352, 353, 354, 355, 356, 357, -1, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, -1, -1, 381, - 382, 383, 384, 385, 386, 387, 388, 389, -1, -1, + 382, 383, 384, 385, 386, 387, 388, 389, -1, 391, 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - -1, -1, 414, 415, -1, 417, -1, 419, 420, 421, + -1, -1, 414, 415, -1, 417, 418, 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - 432, 433, 434, -1, 436, 437, 438, 439, 440, 441, - 442, 443, -1, 445, 446, 447, 448, -1, 450, 451, + 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, + 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, - -1, 463, -1, 465, 466, 467, 468, 469, 470, 471, + -1, 463, 464, 465, 466, 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 3, -1, 5, -1, + 492, 493, 494, 495, 496, 497, 498, -1, -1, -1, + -1, -1, -1, 505, 506, 507, -1, -1, -1, -1, + 512, -1, 514, -1, -1, -1, -1, 519, 520, -1, + 522, 523, 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 22, 23, 24, 25, 26, 27, 28, 29, -1, + 31, 32, 33, -1, -1, -1, -1, -1, 39, -1, + -1, 42, 43, 44, -1, 46, 47, 48, 49, 50, + 51, 52, 53, 54, 55, 56, 57, -1, 59, 60, + 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, + 71, 72, 73, -1, 75, 76, 77, 78, 79, -1, + 81, -1, 83, 84, 85, 86, 87, 88, 89, 90, + 91, 92, -1, 94, 95, 96, 97, 98, 99, -1, + 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, + 111, 112, 113, 114, 115, 116, 117, 118, -1, 120, + 121, 122, 123, 124, 125, -1, -1, 128, 129, 130, + 131, -1, -1, 134, 135, 136, 137, 138, -1, 140, + 141, 142, -1, 144, 145, 146, -1, 148, 149, 150, + 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, + -1, 162, 163, 164, 165, -1, 167, -1, 169, 170, + -1, 172, 173, 174, 175, 176, 177, -1, 179, -1, + 181, 182, 183, 184, -1, 186, 187, 188, 189, 190, + 191, 192, 193, 194, 195, 196, 197, -1, 199, 200, + 201, 202, 203, 204, 205, -1, 207, 208, 209, 210, + 211, 212, 213, 214, 215, 216, -1, 218, -1, 220, + 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, + -1, -1, 233, 234, 235, 236, -1, 238, 239, 240, + 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, + 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, + 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, + 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, + 281, 282, 283, 284, 285, -1, 287, 288, -1, -1, + 291, 292, 293, -1, -1, 296, 297, 298, 299, 300, + 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, + 311, 312, 313, -1, -1, -1, -1, 318, 319, 320, + 321, 322, 323, 324, 325, 326, 327, -1, 329, 330, + 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, + 341, 342, 343, -1, 345, 346, 347, 348, 349, 350, + 351, 352, 353, 354, 355, 356, 357, -1, 359, 360, + 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, + 371, 372, 373, 374, 375, 376, 377, 378, -1, -1, + 381, 382, 383, 384, 385, 386, 387, 388, 389, -1, + 391, 392, 393, 394, 395, -1, 397, 398, 399, 400, + 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, + 411, -1, -1, 414, 415, -1, 417, 418, 419, 420, + 421, 422, 423, -1, 425, 426, 427, -1, -1, 430, + 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, + 441, 442, 443, -1, -1, 446, 447, 448, -1, 450, + 451, 452, 453, -1, 455, 456, 457, 458, 459, 460, + 461, -1, 463, 464, 465, 466, 467, 468, 469, 470, + 471, -1, -1, 474, -1, -1, 477, 478, 479, 480, + 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, + 491, 492, 493, 494, 495, 496, 497, 498, -1, -1, + -1, -1, -1, -1, 505, 506, 507, -1, -1, -1, + -1, 512, -1, 514, -1, -1, -1, -1, 519, 520, + -1, 522, 523, 3, 4, 5, 6, 7, 8, 9, + 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 22, 23, 24, 25, 26, 27, 28, 29, + -1, 31, 32, 33, -1, -1, -1, -1, -1, 39, + -1, -1, 42, 43, 44, -1, 46, 47, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 57, -1, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, + 70, 71, 72, 73, -1, 75, 76, 77, 78, 79, + -1, 81, -1, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, -1, 94, 95, 96, 97, 98, 99, + -1, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, 118, -1, + 120, 121, 122, 123, 124, 125, -1, -1, 128, 129, + 130, 131, -1, -1, 134, 135, 136, 137, 138, -1, + 140, 141, 142, -1, 144, 145, 146, -1, 148, 149, + 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, + 160, -1, 162, 163, 164, 165, -1, 167, -1, 169, + 170, -1, 172, 173, 174, 175, 176, 177, -1, 179, + -1, 181, 182, 183, 184, -1, 186, 187, 188, 189, + 190, 191, 192, 193, 194, 195, 196, 197, -1, 199, + 200, 201, 202, 203, 204, 205, -1, 207, 208, 209, + 210, 211, 212, 213, 214, 215, 216, -1, 218, -1, + 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, + 230, -1, -1, 233, 234, 235, 236, -1, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, + 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, + 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, + 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, + 280, 281, 282, 283, 284, 285, -1, 287, 288, -1, + -1, 291, 292, 293, -1, -1, 296, 297, 298, 299, + 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, + 310, 311, 312, 313, -1, -1, -1, -1, 318, 319, + 320, 321, 322, 323, 324, 325, 326, 327, -1, 329, + 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, + 340, 341, 342, 343, -1, 345, 346, 347, 348, 349, + 350, 351, 352, 353, 354, 355, 356, 357, -1, 359, + 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, + 370, 371, 372, 373, 374, 375, 376, 377, 378, -1, + -1, 381, 382, 383, 384, 385, 386, 387, 388, 389, + -1, 391, 392, 393, 394, 395, -1, 397, 398, 399, + 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, + 410, 411, -1, -1, 414, 415, -1, 417, 418, 419, + 420, 421, 422, 423, -1, 425, 426, 427, -1, -1, + 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, + 440, 441, 442, 443, -1, -1, 446, 447, 448, -1, + 450, 451, 452, 453, -1, 455, 456, 457, 458, 459, + 460, 461, -1, 463, 464, 465, 466, 467, 468, 469, + 470, 471, -1, -1, 474, -1, -1, 477, 478, 479, + 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, + 490, 491, 492, 493, 494, 495, 496, 497, 498, -1, + -1, -1, -1, -1, -1, 505, 506, 507, -1, -1, + -1, -1, 512, -1, 514, -1, -1, -1, -1, 519, + 520, -1, 522, 523, 3, 4, 5, 6, 7, 8, + 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, + 39, -1, -1, 42, 43, 44, -1, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, -1, + 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, 81, -1, 83, 84, 85, 86, 87, 88, + 89, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, 101, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + 159, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, 170, -1, 172, 173, 174, 175, 176, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, 193, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, 208, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, 221, 222, 223, 224, 225, 226, 227, 228, + 229, 230, -1, -1, 233, 234, 235, 236, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, + 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, + 279, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, + 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, 321, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, 391, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, 418, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, 435, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, 464, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, + -1, -1, -1, -1, -1, -1, 505, 506, 507, -1, + -1, -1, -1, 512, -1, 514, -1, -1, -1, -1, + 519, 520, -1, 522, 523, 3, 4, 5, 6, 7, + 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 22, 23, 24, 25, 26, 27, + 28, 29, -1, 31, 32, 33, -1, -1, -1, -1, + -1, 39, -1, -1, 42, 43, 44, -1, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, + -1, 59, 60, 61, 62, 63, 64, 65, 66, 67, + 68, 69, 70, 71, 72, 73, -1, 75, 76, 77, + 78, 79, -1, 81, -1, 83, 84, 85, 86, 87, + 88, 89, 90, 91, 92, -1, 94, 95, 96, 97, + 98, 99, -1, 101, 102, 103, 104, 105, 106, 107, + 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, + 118, -1, 120, 121, 122, 123, 124, 125, -1, -1, + 128, 129, 130, 131, -1, -1, 134, 135, 136, 137, + 138, -1, 140, 141, 142, -1, 144, 145, 146, -1, + 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, + 158, 159, 160, -1, 162, 163, 164, 165, -1, 167, + -1, 169, 170, -1, 172, 173, 174, 175, 176, 177, + -1, 179, -1, 181, 182, 183, 184, -1, 186, 187, + 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, + -1, 199, 200, 201, 202, 203, 204, 205, -1, 207, + 208, 209, 210, 211, 212, 213, 214, 215, 216, -1, + 218, -1, 220, 221, 222, 223, 224, 225, 226, 227, + 228, 229, 230, -1, -1, 233, 234, 235, 236, -1, + 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, + 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, + 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, + 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, + 278, 279, 280, 281, 282, 283, 284, 285, -1, 287, + 288, -1, -1, 291, 292, 293, -1, -1, 296, 297, + 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, + 308, 309, 310, 311, 312, 313, -1, -1, -1, -1, + 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, + -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, + 338, 339, 340, 341, 342, 343, -1, 345, 346, 347, + 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, + -1, 359, 360, 361, 362, 363, 364, 365, 366, 367, + 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, + 378, -1, -1, 381, 382, 383, 384, 385, 386, 387, + 388, 389, -1, 391, 392, 393, 394, 395, -1, 397, + 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, + 408, 409, 410, 411, -1, -1, 414, 415, -1, 417, + 418, 419, 420, 421, 422, 423, -1, 425, 426, 427, + -1, -1, 430, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 443, -1, -1, 446, 447, + 448, -1, 450, 451, 452, 453, -1, 455, 456, 457, + 458, 459, 460, 461, -1, 463, 464, 465, 466, 467, + 468, 469, 470, 471, -1, -1, 474, -1, -1, 477, + 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, + 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, + 498, -1, -1, -1, -1, -1, -1, 505, 506, 507, + -1, -1, -1, -1, 512, -1, 514, -1, -1, -1, + -1, 519, 520, -1, 522, 523, 3, 4, 5, 6, + 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, 34, 35, -1, - -1, -1, -1, -1, -1, -1, 43, 44, -1, 46, - 47, 48, -1, 50, 51, 52, 53, 54, -1, 56, - 57, -1, 59, 60, 61, 62, 63, 64, -1, -1, + 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, + -1, -1, 39, -1, -1, 42, 43, 44, -1, 46, + 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + 57, -1, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, 79, -1, -1, -1, 83, 84, 85, 86, - 87, 88, -1, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, -1, 102, 103, 104, 105, 106, + 77, 78, 79, -1, 81, -1, 83, 84, 85, 86, + 87, 88, 89, 90, 91, 92, -1, 94, 95, 96, + 97, 98, 99, -1, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, + 117, 118, -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, -1, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, -1, -1, -1, 173, 174, 175, -1, + 157, 158, 159, 160, -1, 162, 163, 164, 165, -1, + 167, -1, 169, 170, -1, 172, 173, 174, 175, 176, 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, -1, 194, 195, 196, + 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, -1, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, -1, -1, 223, -1, 225, 226, - 227, 228, 229, 230, -1, -1, 233, -1, 235, -1, + 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, + -1, 218, -1, 220, 221, 222, 223, 224, 225, 226, + 227, 228, 229, 230, -1, -1, 233, 234, 235, 236, -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, -1, 269, 270, 271, 272, 273, -1, 275, 276, - -1, 278, -1, 280, 281, 282, 283, 284, 285, -1, + 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, + 277, 278, 279, 280, 281, 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, 298, -1, 300, -1, 302, 303, 304, 305, 306, + 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, -1, 322, 323, 324, 325, 326, + -1, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, -1, 362, 363, 364, 365, 366, + 357, -1, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, -1, 392, 393, 394, 395, -1, + 387, 388, 389, -1, 391, 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, - 417, -1, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, -1, 436, + 417, 418, 419, 420, 421, 422, 423, -1, 425, 426, + 427, -1, -1, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, -1, 463, -1, 465, 466, + 457, 458, 459, 460, 461, -1, 463, 464, 465, 466, 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 3, -1, 5, -1, -1, -1, -1, -1, -1, + 497, 498, -1, -1, -1, -1, -1, -1, 505, 506, + 507, -1, -1, -1, -1, 512, -1, 514, -1, -1, + -1, -1, 519, 520, -1, 522, 523, 3, 4, 5, + 6, 7, 8, 9, 10, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 22, 23, 24, 25, + 26, 27, 28, 29, -1, 31, 32, 33, -1, -1, + -1, -1, -1, 39, -1, -1, 42, 43, 44, -1, + 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, -1, 59, 60, 61, 62, 63, 64, 65, + 66, 67, 68, 69, 70, 71, 72, 73, -1, 75, + 76, 77, 78, 79, -1, 81, -1, 83, 84, 85, + 86, 87, 88, 89, 90, 91, 92, -1, 94, 95, + 96, 97, 98, 99, -1, 101, 102, 103, 104, 105, + 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, + 116, 117, 118, -1, 120, 121, 122, 123, 124, 125, + -1, -1, 128, 129, 130, 131, -1, -1, 134, 135, + 136, 137, 138, -1, 140, 141, 142, -1, 144, 145, + 146, -1, 148, 149, 150, 151, 152, 153, 154, 155, + 156, 157, 158, 159, 160, -1, 162, 163, 164, 165, + -1, 167, -1, 169, 170, -1, 172, 173, 174, 175, + 176, 177, -1, 179, -1, 181, 182, 183, 184, -1, + 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, + 196, 197, -1, 199, 200, 201, 202, 203, 204, 205, + -1, 207, 208, 209, 210, 211, 212, 213, 214, 215, + 216, -1, 218, -1, 220, 221, 222, 223, 224, 225, + 226, 227, 228, 229, 230, -1, -1, 233, 234, 235, + 236, -1, 238, 239, 240, 241, 242, 243, 244, 245, + 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, + 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, + 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, + 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, + -1, 287, 288, -1, -1, 291, 292, 293, -1, -1, + 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, + 306, 307, 308, 309, 310, 311, 312, 313, -1, -1, + -1, -1, 318, 319, 320, 321, 322, 323, 324, 325, + 326, 327, -1, 329, 330, 331, 332, 333, 334, -1, + 336, 337, 338, 339, 340, 341, 342, 343, -1, 345, + 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, + 356, 357, -1, 359, 360, 361, 362, 363, 364, 365, + 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, + 376, 377, 378, -1, -1, 381, 382, 383, 384, 385, + 386, 387, 388, 389, -1, 391, 392, 393, 394, 395, + -1, 397, 398, 399, 400, 401, 402, 403, 404, 405, + 406, 407, 408, 409, 410, 411, -1, -1, 414, 415, + -1, 417, 418, 419, 420, 421, 422, 423, -1, 425, + 426, 427, -1, -1, 430, 431, 432, 433, 434, 435, + 436, 437, 438, 439, 440, 441, 442, 443, -1, -1, + 446, 447, 448, -1, 450, 451, 452, 453, -1, 455, + 456, 457, 458, 459, 460, 461, -1, 463, 464, 465, + 466, 467, 468, 469, 470, 471, -1, -1, 474, -1, + -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, + 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, + 496, 497, 498, -1, -1, -1, -1, -1, -1, 505, + 506, 507, -1, -1, -1, -1, 512, -1, 514, -1, + -1, -1, -1, 519, 520, -1, 522, 523, 3, 4, + 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 22, 23, 24, + 25, 26, 27, 28, 29, -1, 31, 32, 33, -1, + -1, -1, -1, -1, 39, -1, -1, 42, 43, 44, + -1, 46, 47, 48, 49, 50, 51, 52, 53, 54, + 55, 56, 57, -1, 59, 60, 61, 62, 63, 64, + 65, 66, 67, 68, 69, 70, 71, 72, 73, -1, + 75, 76, 77, 78, 79, -1, 81, -1, 83, 84, + 85, 86, 87, 88, 89, 90, 91, 92, -1, 94, + 95, 96, 97, 98, 99, -1, 101, 102, 103, 104, + 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, + 115, 116, 117, 118, -1, 120, 121, 122, 123, 124, + 125, -1, -1, 128, 129, 130, 131, -1, -1, 134, + 135, 136, 137, 138, -1, 140, 141, 142, -1, 144, + 145, 146, -1, 148, 149, 150, 151, 152, 153, 154, + 155, 156, 157, 158, 159, 160, -1, 162, 163, 164, + 165, -1, 167, -1, 169, 170, -1, 172, 173, 174, + 175, 176, 177, -1, 179, -1, 181, 182, 183, 184, + -1, 186, 187, 188, 189, 190, 191, 192, 193, 194, + 195, 196, 197, -1, 199, 200, 201, 202, 203, 204, + 205, -1, 207, 208, 209, 210, 211, 212, 213, 214, + 215, 216, -1, 218, -1, 220, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 230, -1, -1, 233, 234, + 235, 236, -1, 238, 239, 240, 241, 242, 243, 244, + 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, + 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, + 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, + 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, + 285, -1, 287, 288, -1, -1, 291, 292, 293, -1, + -1, 296, 297, 298, 299, 300, 301, 302, 303, 304, + 305, 306, 307, 308, 309, 310, 311, 312, 313, -1, + -1, -1, -1, 318, 319, 320, 321, 322, 323, 324, + 325, 326, 327, -1, 329, 330, 331, 332, 333, 334, + -1, 336, 337, 338, 339, 340, 341, 342, 343, -1, + 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, -1, 359, 360, 361, 362, 363, 364, + 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, + 375, 376, 377, 378, -1, -1, 381, 382, 383, 384, + 385, 386, 387, 388, 389, -1, 391, 392, 393, 394, + 395, -1, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 409, 410, 411, -1, -1, 414, + 415, -1, 417, 418, 419, 420, 421, 422, 423, -1, + 425, 426, 427, -1, -1, 430, 431, 432, 433, 434, + 435, 436, 437, 438, 439, 440, 441, 442, 443, -1, + -1, 446, 447, 448, -1, 450, 451, 452, 453, -1, + 455, 456, 457, 458, 459, 460, 461, -1, 463, 464, + 465, 466, 467, 468, 469, 470, 471, -1, -1, 474, + -1, -1, 477, 478, 479, 480, 481, 482, 483, 484, + 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, + 495, 496, 497, 498, -1, -1, -1, -1, -1, -1, + 505, 506, 507, -1, -1, -1, -1, 512, -1, 514, + -1, -1, -1, -1, 519, 520, -1, 522, 523, 3, + 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, + -1, -1, -1, -1, -1, 39, -1, -1, 42, 43, + 44, -1, 46, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 57, -1, 59, 60, 61, 62, 63, + 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, 81, -1, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, 101, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, 117, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, 159, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, 170, -1, 172, 173, + 174, 175, 176, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, 193, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, 208, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, -1, -1, 233, + 234, 235, 236, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, + 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, 299, 300, 301, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, 320, 321, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, 391, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, -1, 417, 418, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + 464, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 498, -1, -1, -1, -1, -1, + -1, 505, 506, 507, -1, -1, -1, -1, 512, -1, + 514, -1, -1, -1, -1, 519, 520, -1, 522, 523, + 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 22, + 23, 24, 25, 26, 27, 28, 29, -1, 31, 32, + 33, -1, -1, -1, -1, -1, 39, -1, -1, 42, + 43, 44, -1, 46, 47, 48, 49, 50, 51, 52, + 53, 54, 55, 56, 57, -1, 59, 60, 61, 62, + 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, + 73, -1, 75, 76, 77, 78, 79, -1, 81, -1, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, + -1, 94, 95, 96, 97, 98, 99, -1, 101, 102, + 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + 113, 114, 115, 116, -1, 118, -1, 120, 121, 122, + 123, 124, 125, -1, -1, 128, 129, 130, 131, -1, + -1, 134, 135, 136, 137, 138, -1, 140, 141, 142, + -1, 144, 145, 146, -1, 148, 149, 150, 151, 152, + 153, 154, 155, 156, 157, 158, 159, 160, -1, 162, + 163, 164, 165, -1, 167, -1, 169, 170, -1, 172, + 173, 174, 175, 176, 177, -1, 179, -1, 181, 182, + 183, 184, -1, 186, 187, 188, 189, 190, 191, 192, + 193, 194, 195, 196, 197, -1, 199, 200, 201, 202, + 203, 204, 205, -1, 207, 208, 209, 210, 211, 212, + 213, 214, 215, 216, -1, 218, -1, 220, 221, 222, + 223, 224, 225, 226, 227, 228, 229, 230, -1, -1, + 233, 234, 235, 236, -1, 238, 239, 240, 241, 242, + 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, + 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, + 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, + 273, -1, 275, 276, 277, 278, 279, 280, 281, 282, + 283, 284, 285, -1, 287, 288, -1, -1, 291, 292, + 293, -1, -1, 296, 297, 298, 299, 300, 301, 302, + 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, + 313, -1, -1, -1, -1, 318, 319, 320, 321, 322, + 323, 324, 325, 326, 327, -1, 329, 330, 331, 332, + 333, 334, -1, 336, 337, 338, 339, 340, 341, 342, + 343, -1, 345, 346, 347, 348, 349, 350, 351, 352, + 353, 354, 355, 356, 357, -1, 359, 360, 361, 362, + 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, + 373, 374, 375, 376, 377, 378, -1, -1, 381, 382, + 383, 384, 385, 386, 387, 388, 389, -1, 391, 392, + 393, 394, 395, -1, 397, 398, 399, 400, 401, 402, + 403, 404, 405, 406, 407, 408, 409, 410, 411, -1, + -1, 414, 415, -1, 417, 418, 419, 420, 421, 422, + 423, -1, 425, 426, 427, -1, -1, 430, 431, 432, + 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, + 443, -1, -1, 446, 447, 448, -1, 450, 451, 452, + 453, -1, 455, 456, 457, 458, 459, 460, 461, -1, + 463, 464, 465, 466, 467, 468, 469, 470, 471, -1, + -1, 474, -1, -1, 477, 478, 479, 480, 481, 482, + 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, + 493, 494, 495, 496, 497, -1, -1, -1, -1, -1, + -1, -1, 505, 506, -1, -1, -1, -1, -1, 512, + -1, 514, -1, -1, -1, -1, 519, 520, -1, 522, + 523, 3, 4, 5, 6, 7, 8, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 43, 44, -1, 46, 47, 48, -1, 50, 51, - 52, 53, 54, -1, 56, 57, -1, 59, 60, 61, - 62, 63, 64, -1, -1, 67, 68, 69, 70, 71, - 72, 73, -1, 75, 76, 77, 78, 79, -1, -1, - -1, 83, 84, 85, 86, 87, 88, -1, 90, 91, - 92, -1, 94, 95, 96, 97, 98, 99, -1, -1, + 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, + 32, 33, -1, -1, -1, -1, -1, 39, -1, -1, + 42, 43, 44, -1, 46, 47, 48, 49, 50, 51, + 52, 53, 54, 55, 56, 57, -1, 59, 60, 61, + 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, + 72, 73, -1, 75, 76, 77, 78, 79, -1, 81, + -1, 83, 84, 85, 86, 87, 88, 89, 90, 91, + 92, -1, 94, 95, 96, 97, 98, 99, -1, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, -1, 118, -1, 120, 121, + 112, 113, 114, 115, 116, 117, 118, -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, -1, 160, -1, - 162, 163, 164, 165, -1, 167, -1, 169, -1, -1, - -1, 173, 174, 175, -1, 177, -1, 179, -1, 181, + 152, 153, 154, 155, 156, 157, 158, 159, 160, -1, + 162, 163, 164, 165, -1, 167, -1, 169, 170, -1, + 172, 173, 174, 175, 176, 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, - 192, -1, 194, 195, 196, 197, -1, 199, 200, 201, - 202, 203, 204, 205, -1, 207, -1, 209, 210, 211, + 192, 193, 194, 195, 196, 197, -1, 199, 200, 201, + 202, 203, 204, 205, -1, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, -1, 220, -1, - -1, 223, -1, 225, 226, 227, 228, 229, 230, -1, - -1, 233, -1, 235, -1, -1, 238, 239, 240, 241, + -1, 223, 224, 225, 226, 227, 228, 229, 230, -1, + -1, 233, 234, 235, 236, -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, -1, 269, 270, 271, - 272, 273, -1, 275, 276, -1, 278, -1, 280, 281, + 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, + 272, 273, -1, 275, 276, -1, 278, 279, 280, 281, 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, - 292, 293, -1, -1, 296, 297, 298, -1, 300, -1, + 292, 293, -1, -1, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, -1, -1, -1, -1, 318, 319, 320, -1, + 312, 313, -1, -1, -1, -1, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, -1, 359, 360, -1, + 352, 353, 354, 355, 356, 357, -1, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, -1, -1, 381, - 382, 383, 384, 385, 386, 387, 388, 389, -1, -1, + 382, 383, 384, 385, 386, 387, 388, 389, -1, 391, 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - -1, -1, 414, 415, -1, 417, -1, 419, 420, 421, + -1, -1, 414, 415, -1, 417, 418, 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - 432, 433, 434, -1, 436, 437, 438, 439, 440, 441, + 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, - -1, 463, -1, 465, 466, 467, 468, 469, 470, 471, + -1, 463, 464, 465, 466, 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 3, -1, 5, -1, + 492, 493, 494, 495, 496, 497, 498, -1, -1, -1, + -1, -1, -1, 505, 506, 507, -1, -1, -1, -1, + 512, -1, 514, -1, -1, -1, -1, 519, 520, -1, + 522, 523, 3, 4, 5, 6, 7, -1, 9, 10, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 22, 23, 24, 25, 26, 27, 28, 29, -1, + 31, 32, 33, -1, -1, -1, -1, -1, 39, -1, + -1, 42, 43, 44, -1, 46, 47, 48, 49, 50, + 51, 52, 53, 54, 55, 56, 57, -1, 59, 60, + 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, + 71, 72, 73, -1, 75, 76, 77, 78, 79, -1, + 81, -1, 83, 84, 85, 86, 87, 88, 89, 90, + 91, 92, -1, 94, 95, 96, 97, 98, 99, -1, + 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, + 111, 112, 113, 114, 115, 116, -1, 118, -1, 120, + 121, 122, 123, 124, 125, -1, -1, 128, 129, 130, + 131, -1, -1, 134, 135, 136, 137, 138, -1, 140, + 141, 142, -1, 144, 145, 146, -1, 148, 149, 150, + 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, + -1, 162, 163, 164, 165, -1, 167, -1, 169, 170, + -1, 172, 173, 174, 175, 176, 177, -1, 179, -1, + 181, 182, 183, 184, -1, 186, 187, 188, 189, 190, + 191, 192, 193, 194, 195, 196, 197, -1, 199, 200, + 201, 202, 203, 204, 205, -1, 207, 208, 209, 210, + 211, 212, 213, 214, 215, 216, -1, 218, -1, 220, + 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, + -1, -1, 233, 234, 235, 236, -1, 238, 239, 240, + 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, + 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, + 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, + 271, 272, 273, -1, 275, 276, 277, 278, 279, 280, + 281, 282, 283, 284, 285, -1, 287, 288, -1, -1, + 291, 292, 293, -1, -1, 296, 297, 298, 299, 300, + 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, + 311, 312, 313, -1, -1, -1, -1, 318, 319, 320, + 321, 322, 323, 324, 325, 326, 327, -1, 329, 330, + 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, + 341, 342, 343, -1, 345, 346, 347, 348, 349, 350, + 351, 352, 353, 354, 355, 356, 357, -1, 359, 360, + 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, + 371, 372, 373, 374, 375, 376, 377, 378, -1, -1, + 381, 382, 383, 384, 385, 386, 387, 388, 389, -1, + 391, 392, 393, 394, 395, -1, 397, 398, 399, 400, + 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, + 411, -1, -1, 414, 415, -1, 417, 418, 419, 420, + 421, 422, 423, -1, 425, 426, 427, -1, -1, 430, + 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, + 441, 442, 443, -1, -1, 446, 447, 448, -1, 450, + 451, 452, 453, -1, 455, 456, 457, 458, 459, 460, + 461, -1, 463, 464, 465, 466, 467, 468, 469, 470, + 471, -1, -1, 474, -1, -1, 477, 478, 479, 480, + 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, + 491, 492, 493, 494, 495, 496, 497, -1, -1, -1, + -1, -1, -1, -1, 505, 506, -1, -1, -1, -1, + -1, 512, -1, 514, -1, -1, -1, -1, 519, 520, + -1, 522, 523, 3, 4, 5, 6, 7, 8, 9, + 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 22, 23, 24, 25, 26, 27, 28, 29, + -1, 31, 32, 33, -1, -1, -1, -1, -1, 39, + -1, -1, 42, 43, 44, -1, 46, 47, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 57, -1, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, + 70, 71, 72, 73, -1, 75, 76, 77, 78, 79, + -1, 81, -1, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, -1, 94, 95, 96, 97, 98, 99, + -1, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, -1, 118, -1, + 120, 121, 122, 123, 124, 125, -1, -1, 128, 129, + 130, 131, -1, -1, 134, 135, 136, 137, 138, -1, + 140, 141, 142, -1, 144, 145, 146, -1, 148, 149, + 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, + 160, -1, 162, 163, 164, 165, -1, 167, -1, 169, + 170, -1, 172, 173, 174, 175, 176, 177, -1, 179, + -1, 181, 182, 183, 184, -1, 186, 187, 188, 189, + 190, 191, 192, 193, 194, 195, 196, 197, -1, 199, + 200, 201, 202, 203, 204, 205, -1, 207, 208, 209, + 210, 211, 212, 213, 214, 215, 216, -1, 218, -1, + 220, -1, 222, 223, 224, 225, 226, 227, 228, 229, + 230, -1, -1, 233, 234, 235, 236, -1, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, + 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, + 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, + 270, 271, 272, 273, -1, 275, 276, 277, 278, 279, + 280, 281, 282, 283, 284, 285, -1, 287, 288, -1, + -1, 291, 292, 293, -1, -1, 296, 297, 298, 299, + 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, + 310, 311, 312, 313, -1, -1, -1, -1, 318, 319, + 320, 321, 322, 323, 324, 325, 326, 327, -1, 329, + 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, + 340, 341, 342, 343, -1, 345, 346, 347, 348, 349, + 350, 351, 352, 353, 354, 355, 356, 357, -1, 359, + 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, + 370, 371, 372, 373, 374, 375, 376, 377, 378, -1, + -1, 381, 382, 383, 384, 385, 386, 387, 388, 389, + -1, 391, 392, 393, 394, 395, -1, 397, 398, 399, + 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, + 410, 411, -1, -1, 414, 415, -1, 417, 418, 419, + 420, 421, 422, 423, -1, 425, 426, 427, -1, -1, + 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, + 440, 441, 442, 443, -1, -1, 446, 447, 448, -1, + 450, 451, 452, 453, -1, 455, 456, 457, 458, 459, + 460, 461, -1, 463, 464, 465, 466, 467, 468, 469, + 470, 471, -1, -1, 474, -1, -1, 477, 478, 479, + 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, + 490, 491, 492, 493, 494, 495, 496, 497, -1, -1, + -1, -1, -1, -1, -1, 505, 506, -1, -1, -1, + -1, -1, 512, -1, 514, -1, -1, -1, -1, 519, + 520, -1, 522, 523, 3, 4, 5, 6, 7, -1, + 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, + 39, -1, -1, 42, 43, 44, -1, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, -1, + 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, 81, -1, 83, 84, 85, 86, 87, 88, + 89, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, 101, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + 159, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, 170, -1, 172, 173, 174, 175, 176, 177, -1, + 179, 180, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, + 199, 200, 201, 202, 203, 204, 205, -1, 207, 208, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, 221, 222, 223, 224, 225, 226, 227, 228, + 229, 230, -1, -1, 233, 234, 235, 236, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, + 269, 270, 271, 272, 273, -1, 275, 276, 277, 278, + 279, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, + 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, 321, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, 391, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, 418, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, 435, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, 464, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 512, -1, 514, -1, -1, -1, -1, + 519, 520, -1, 522, 523, 3, 4, 5, 6, 7, + -1, 9, 10, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 22, 23, 24, 25, 26, 27, + 28, 29, -1, 31, 32, 33, -1, -1, -1, -1, + -1, 39, -1, -1, 42, 43, 44, -1, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, + -1, 59, 60, 61, 62, 63, 64, 65, 66, 67, + 68, 69, 70, 71, 72, 73, -1, 75, 76, 77, + 78, 79, -1, 81, -1, 83, 84, 85, 86, 87, + 88, 89, 90, 91, 92, -1, 94, 95, 96, 97, + 98, 99, -1, 101, 102, 103, 104, 105, 106, 107, + 108, 109, 110, 111, 112, 113, 114, 115, 116, -1, + 118, -1, 120, 121, 122, 123, 124, 125, -1, -1, + 128, 129, 130, 131, -1, -1, 134, 135, 136, 137, + 138, -1, 140, 141, 142, -1, 144, 145, 146, -1, + 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, + 158, 159, 160, -1, 162, 163, 164, 165, -1, 167, + -1, 169, 170, -1, 172, 173, 174, 175, 176, 177, + -1, 179, -1, 181, 182, 183, 184, -1, 186, 187, + 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, + -1, 199, 200, 201, 202, 203, 204, 205, -1, 207, + 208, 209, 210, 211, 212, 213, 214, 215, 216, -1, + 218, -1, 220, 221, 222, 223, 224, 225, 226, 227, + 228, 229, 230, -1, -1, 233, 234, 235, 236, -1, + 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, + 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, + 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, + 268, 269, 270, 271, 272, 273, -1, 275, 276, 277, + 278, 279, 280, 281, 282, 283, 284, 285, -1, 287, + 288, -1, -1, 291, 292, 293, -1, -1, 296, 297, + 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, + 308, 309, 310, 311, 312, 313, -1, -1, -1, -1, + 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, + -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, + 338, 339, 340, 341, 342, 343, -1, 345, 346, 347, + 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, + -1, 359, 360, 361, 362, 363, 364, 365, 366, 367, + 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, + 378, -1, -1, 381, 382, 383, 384, 385, 386, 387, + 388, 389, -1, 391, 392, 393, 394, 395, -1, 397, + 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, + 408, 409, 410, 411, -1, -1, 414, 415, -1, 417, + 418, 419, 420, 421, 422, 423, -1, 425, 426, 427, + -1, -1, 430, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 443, -1, -1, 446, 447, + 448, -1, 450, 451, 452, 453, -1, 455, 456, 457, + 458, 459, 460, 461, -1, 463, 464, 465, 466, 467, + 468, 469, 470, 471, -1, -1, 474, -1, -1, 477, + 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, + 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 512, -1, 514, -1, -1, -1, + -1, 519, 520, -1, 522, 523, 3, 4, 5, 6, + 7, -1, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 43, 44, -1, 46, - 47, 48, -1, 50, 51, 52, 53, 54, -1, 56, - 57, -1, 59, 60, 61, 62, 63, 64, -1, -1, + -1, -1, 39, -1, -1, 42, 43, 44, -1, 46, + 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + 57, -1, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, 79, -1, -1, -1, 83, 84, 85, 86, - 87, 88, -1, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, -1, 102, 103, 104, 105, 106, + 77, 78, 79, -1, 81, -1, 83, 84, 85, 86, + 87, 88, 89, 90, 91, 92, -1, 94, 95, 96, + 97, 98, 99, -1, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, -1, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, -1, -1, -1, 173, 174, 175, -1, + 157, 158, 159, 160, -1, 162, 163, 164, 165, -1, + 167, -1, 169, 170, -1, 172, 173, 174, 175, 176, 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, -1, 194, 195, 196, + 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, -1, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, -1, -1, 223, -1, 225, 226, - 227, 228, 229, 230, -1, -1, 233, -1, 235, -1, + 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, + -1, 218, -1, 220, 221, 222, 223, 224, 225, 226, + 227, 228, 229, 230, -1, -1, 233, 234, 235, 236, -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, -1, 269, 270, 271, 272, 273, -1, 275, 276, - -1, 278, -1, 280, 281, 282, 283, 284, 285, -1, - 287, 288, -1, 290, 291, 292, 293, -1, -1, 296, - 297, 298, -1, 300, -1, 302, 303, 304, 305, 306, + 267, 268, 269, 270, 271, 272, 273, -1, 275, 276, + 277, 278, 279, 280, 281, 282, 283, 284, 285, -1, + 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, + 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, -1, 322, 323, 324, 325, 326, + -1, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, -1, 362, 363, 364, 365, 366, + 357, -1, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, -1, 392, 393, 394, 395, -1, + 387, 388, 389, -1, 391, 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, - 417, -1, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, -1, 436, + 417, 418, 419, 420, 421, 422, 423, -1, 425, 426, + 427, -1, -1, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, -1, 463, -1, 465, 466, + 457, 458, 459, 460, 461, -1, 463, 464, 465, 466, 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 3, -1, 5, -1, -1, -1, -1, -1, -1, + 497, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 512, -1, 514, -1, -1, + -1, -1, 519, 520, -1, 522, 523, 3, 4, 5, + 6, 7, 8, 9, 10, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 22, 23, 24, 25, + 26, 27, 28, 29, -1, 31, 32, 33, -1, -1, + -1, -1, -1, 39, -1, -1, 42, 43, 44, -1, + 46, 47, 48, 49, 50, 51, 52, -1, 54, 55, + 56, 57, -1, 59, 60, 61, 62, 63, 64, 65, + 66, 67, 68, 69, 70, 71, 72, 73, -1, 75, + 76, 77, 78, 79, -1, 81, -1, 83, 84, 85, + 86, 87, 88, 89, 90, 91, 92, -1, 94, 95, + 96, 97, 98, 99, -1, 101, 102, 103, 104, 105, + 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, + 116, 117, 118, -1, 120, 121, 122, 123, 124, 125, + -1, -1, 128, 129, 130, 131, -1, -1, 134, 135, + 136, 137, 138, -1, 140, 141, 142, -1, 144, -1, + 146, -1, 148, 149, 150, 151, 152, 153, 154, 155, + 156, 157, 158, 159, 160, -1, 162, 163, 164, 165, + -1, 167, -1, 169, 170, -1, 172, 173, 174, 175, + -1, 177, -1, 179, -1, 181, 182, 183, 184, -1, + 186, 187, 188, 189, 190, 191, 192, -1, 194, 195, + 196, 197, -1, 199, 200, 201, 202, 203, 204, 205, + -1, 207, 208, 209, 210, 211, 212, 213, 214, 215, + 216, -1, 218, -1, 220, -1, -1, 223, 224, 225, + 226, 227, 228, 229, 230, -1, -1, 233, 234, 235, + -1, -1, 238, 239, 240, 241, 242, 243, 244, 245, + 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, + 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, + 266, 267, 268, 269, 270, 271, 272, 273, -1, 275, + 276, -1, 278, 279, 280, 281, 282, 283, 284, 285, + -1, 287, 288, -1, -1, 291, 292, 293, -1, -1, + 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, + 306, 307, 308, 309, 310, 311, 312, 313, -1, -1, + -1, -1, 318, 319, 320, 321, 322, 323, 324, 325, + 326, 327, -1, 329, 330, 331, 332, 333, 334, -1, + 336, 337, 338, 339, 340, 341, 342, 343, -1, 345, + 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, + 356, 357, -1, 359, 360, 361, 362, 363, 364, 365, + 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, + 376, 377, 378, -1, -1, 381, 382, 383, 384, 385, + 386, 387, 388, 389, -1, -1, 392, 393, 394, 395, + -1, 397, 398, 399, 400, 401, 402, 403, 404, 405, + 406, 407, 408, 409, 410, 411, -1, -1, 414, 415, + -1, 417, 418, 419, 420, 421, 422, 423, -1, 425, + 426, 427, -1, -1, 430, 431, 432, 433, 434, 435, + 436, 437, 438, 439, 440, 441, 442, 443, -1, -1, + 446, 447, 448, -1, 450, 451, 452, 453, -1, 455, + 456, 457, 458, 459, 460, 461, -1, 463, 464, 465, + 466, 467, 468, 469, 470, 471, -1, -1, 474, -1, + -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, + 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, + 496, 497, -1, -1, -1, -1, -1, -1, -1, 505, + 506, 507, -1, 3, 4, 5, 512, -1, 514, 9, + -1, -1, -1, 519, 520, -1, 522, 523, -1, -1, + -1, -1, 22, 23, 24, 25, 26, 27, 28, 29, + -1, 31, 32, 33, -1, -1, -1, 37, -1, -1, + -1, -1, 42, 43, 44, -1, 46, 47, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 57, -1, 59, + 60, 61, 62, 63, 64, -1, -1, 67, 68, 69, + 70, 71, 72, 73, -1, 75, 76, 77, 78, 79, + -1, 81, -1, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, -1, 94, 95, 96, 97, 98, 99, + -1, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, -1, 118, -1, + 120, 121, 122, 123, 124, 125, -1, -1, 128, 129, + 130, 131, -1, -1, 134, 135, 136, 137, 138, -1, + 140, 141, 142, -1, 144, 145, 146, -1, 148, 149, + 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, + 160, -1, 162, 163, 164, 165, -1, 167, -1, 169, + 170, -1, 172, 173, 174, 175, 176, 177, -1, 179, + -1, 181, 182, 183, 184, -1, 186, 187, 188, 189, + 190, 191, 192, 193, 194, 195, 196, 197, -1, 199, + 200, 201, 202, 203, 204, 205, -1, 207, 208, 209, + 210, 211, 212, 213, 214, 215, 216, -1, 218, -1, + 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, + 230, -1, -1, 233, 234, 235, 236, -1, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, + 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, + 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, + 270, 271, 272, 273, -1, 275, 276, 277, 278, -1, + 280, 281, 282, 283, 284, 285, -1, 287, 288, 289, + -1, 291, 292, 293, -1, -1, 296, 297, 298, 299, + 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, + 310, 311, 312, 313, -1, -1, -1, -1, 318, 319, + 320, 321, 322, 323, 324, 325, 326, 327, -1, 329, + 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, + 340, 341, 342, 343, -1, 345, 346, 347, 348, 349, + 350, 351, 352, 353, 354, 355, 356, 357, -1, 359, + 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, + 370, 371, 372, 373, 374, 375, 376, 377, 378, -1, + 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, + -1, 391, 392, 393, 394, 395, -1, 397, 398, 399, + 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, + 410, 411, -1, -1, 414, 415, -1, 417, 418, 419, + 420, 421, 422, 423, -1, 425, 426, 427, -1, -1, + 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, + 440, 441, 442, 443, -1, -1, 446, 447, 448, -1, + 450, 451, 452, 453, -1, 455, 456, 457, 458, 459, + 460, 461, -1, 463, 464, 465, 466, 467, 468, 469, + 470, 471, -1, -1, 474, -1, -1, 477, 478, 479, + 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, + 490, 491, 492, 493, 494, 495, 496, 497, -1, -1, + -1, -1, 3, -1, -1, 505, 506, 507, -1, -1, + -1, -1, 512, -1, 514, -1, -1, -1, -1, -1, + 520, 22, 23, 24, 25, 26, 27, 28, 29, -1, + 31, 32, 33, -1, -1, -1, -1, -1, -1, 40, + -1, -1, 43, 44, -1, 46, 47, 48, -1, 50, + 51, 52, 53, 54, -1, 56, 57, -1, 59, 60, + 61, 62, 63, 64, -1, -1, 67, 68, 69, 70, + 71, 72, 73, -1, 75, 76, 77, 78, 79, -1, + -1, -1, 83, 84, 85, 86, 87, 88, -1, 90, + 91, 92, -1, 94, 95, 96, 97, 98, 99, -1, + -1, 102, 103, 104, 105, 106, 107, 108, 109, 110, + 111, 112, 113, 114, 115, 116, -1, 118, -1, 120, + 121, 122, 123, 124, 125, -1, -1, 128, 129, 130, + 131, -1, -1, 134, 135, 136, 137, 138, -1, 140, + 141, 142, -1, 144, 145, 146, 147, 148, 149, 150, + 151, 152, 153, 154, 155, 156, 157, 158, -1, 160, + 161, 162, 163, 164, 165, 166, 167, -1, 169, -1, + -1, -1, 173, 174, 175, -1, 177, -1, 179, -1, + 181, 182, 183, 184, -1, 186, 187, 188, 189, 190, + 191, 192, -1, 194, 195, 196, 197, -1, 199, 200, + 201, 202, 203, 204, 205, -1, 207, -1, 209, 210, + 211, 212, 213, 214, 215, 216, 217, 218, -1, 220, + -1, -1, 223, -1, 225, 226, 227, 228, 229, 230, + -1, -1, 233, -1, 235, -1, 237, 238, 239, 240, + 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, + 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, + 261, 262, 263, 264, 265, 266, 267, -1, 269, 270, + 271, 272, 273, -1, 275, 276, -1, 278, -1, 280, + 281, 282, 283, 284, 285, 286, 287, 288, -1, -1, + 291, 292, 293, -1, 295, 296, 297, 298, -1, 300, + -1, 302, 303, 304, 305, 306, 307, 308, 309, 310, + 311, 312, 313, -1, -1, -1, -1, 318, 319, 320, + -1, 322, 323, 324, 325, 326, 327, -1, 329, 330, + 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, + 341, 342, 343, -1, 345, 346, 347, 348, 349, 350, + 351, 352, 353, 354, 355, 356, 357, -1, 359, 360, + -1, 362, 363, 364, 365, 366, 367, 368, 369, 370, + 371, 372, 373, 374, 375, 376, 377, 378, -1, -1, + 381, 382, 383, 384, 385, 386, 387, 388, 389, -1, + -1, 392, 393, 394, 395, -1, 397, 398, 399, 400, + 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, + 411, -1, -1, 414, 415, -1, 417, -1, 419, 420, + 421, 422, 423, -1, 425, 426, 427, -1, -1, 430, + 431, 432, 433, 434, -1, 436, 437, 438, 439, 440, + 441, 442, 443, 444, -1, 446, 447, 448, -1, 450, + 451, 452, 453, -1, 455, 456, 457, 458, 459, 460, + 461, -1, 463, -1, 465, 466, 467, 468, 469, 470, + 471, -1, -1, 474, -1, -1, 477, 478, 479, 480, + 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, + 491, 492, 493, 494, 495, 496, 497, 3, -1, 5, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 515, -1, 22, 23, 24, 25, + 26, 27, 28, 29, -1, 31, 32, 33, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 43, 44, -1, + 46, 47, 48, -1, 50, 51, 52, 53, 54, -1, + 56, 57, -1, 59, 60, 61, 62, 63, 64, -1, + -1, 67, 68, 69, 70, 71, 72, 73, -1, 75, + 76, 77, 78, 79, -1, -1, -1, 83, 84, 85, + 86, 87, 88, -1, 90, 91, 92, -1, 94, 95, + 96, 97, 98, 99, -1, -1, 102, 103, 104, 105, + 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, + 116, -1, 118, -1, 120, 121, 122, 123, 124, 125, + -1, -1, 128, 129, 130, 131, -1, -1, 134, 135, + 136, 137, 138, -1, 140, 141, 142, -1, 144, 145, + 146, -1, 148, 149, 150, 151, 152, 153, 154, 155, + 156, 157, 158, -1, 160, -1, 162, 163, 164, 165, + -1, 167, -1, 169, -1, -1, -1, 173, 174, 175, + -1, 177, -1, 179, -1, 181, 182, 183, 184, -1, + 186, 187, 188, 189, 190, 191, 192, -1, 194, 195, + 196, 197, -1, 199, 200, 201, 202, 203, 204, 205, + -1, 207, -1, 209, 210, 211, 212, 213, 214, 215, + 216, -1, 218, -1, 220, -1, -1, 223, -1, 225, + 226, 227, 228, 229, 230, -1, -1, 233, -1, 235, + -1, -1, 238, 239, 240, 241, 242, 243, 244, 245, + 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, + 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, + 266, 267, -1, 269, 270, 271, 272, 273, -1, 275, + 276, -1, 278, -1, 280, 281, 282, 283, 284, 285, + -1, 287, 288, -1, -1, 291, 292, 293, -1, -1, + 296, 297, 298, -1, 300, -1, 302, 303, 304, 305, + 306, 307, 308, 309, 310, 311, 312, 313, -1, -1, + -1, -1, 318, 319, 320, -1, 322, 323, 324, 325, + 326, 327, -1, 329, 330, 331, 332, 333, 334, -1, + 336, 337, 338, 339, 340, 341, 342, 343, -1, 345, + 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, + 356, 357, -1, 359, 360, -1, 362, 363, 364, 365, + 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, + 376, 377, 378, -1, -1, 381, 382, 383, 384, 385, + 386, 387, 388, 389, -1, -1, 392, 393, 394, 395, + -1, 397, 398, 399, 400, 401, 402, 403, 404, 405, + 406, 407, 408, 409, 410, 411, -1, -1, 414, 415, + -1, 417, -1, 419, 420, 421, 422, 423, -1, 425, + 426, 427, -1, -1, 430, 431, 432, 433, 434, -1, + 436, 437, 438, 439, 440, 441, 442, 443, -1, -1, + 446, 447, 448, -1, 450, 451, 452, 453, -1, 455, + 456, 457, 458, 459, 460, 461, -1, 463, -1, 465, + 466, 467, 468, 469, 470, 471, -1, -1, 474, -1, + -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, + 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, + 496, 497, 3, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 514, 515, + -1, 22, 23, 24, 25, 26, 27, 28, 29, -1, + 31, 32, 33, 34, 35, -1, 37, -1, -1, -1, + -1, 42, 43, 44, -1, 46, 47, 48, 49, 50, + 51, 52, 53, 54, 55, 56, 57, -1, 59, 60, + 61, 62, 63, 64, -1, -1, 67, 68, 69, 70, + 71, 72, 73, -1, 75, 76, 77, 78, 79, -1, + 81, -1, 83, 84, 85, 86, 87, 88, 89, 90, + 91, 92, -1, 94, 95, 96, 97, 98, 99, -1, + 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, + 111, 112, 113, 114, 115, 116, -1, 118, -1, 120, + 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, + 131, -1, -1, 134, 135, 136, 137, 138, -1, 140, + 141, 142, -1, 144, 145, 146, -1, 148, 149, 150, + 151, 152, 153, 154, 155, 156, 157, 158, -1, 160, + -1, 162, 163, 164, 165, -1, 167, -1, 169, 170, + 171, 172, 173, 174, 175, 176, 177, -1, 179, -1, + 181, 182, 183, 184, -1, 186, 187, 188, 189, 190, + 191, 192, 193, 194, 195, 196, 197, -1, 199, 200, + 201, 202, 203, 204, 205, -1, 207, 208, 209, 210, + 211, 212, 213, 214, 215, 216, -1, 218, -1, 220, + 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, + -1, -1, 233, 234, 235, 236, -1, 238, 239, 240, + 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, + 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, + 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, + 271, 272, 273, -1, 275, 276, 277, 278, -1, 280, + 281, 282, 283, 284, 285, -1, 287, 288, -1, -1, + 291, 292, 293, -1, -1, 296, 297, 298, 299, 300, + 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, + 311, 312, 313, 314, 315, 316, -1, 318, 319, 320, + 321, 322, 323, 324, 325, 326, 327, -1, 329, 330, + 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, + 341, 342, 343, -1, 345, 346, 347, 348, 349, 350, + 351, 352, 353, 354, 355, 356, 357, -1, 359, 360, + 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, + 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, + 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, + 391, 392, 393, 394, 395, -1, 397, 398, 399, 400, + 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, + 411, 412, -1, 414, 415, 416, 417, 418, 419, 420, + 421, 422, 423, -1, 425, 426, 427, -1, -1, 430, + 431, 432, 433, 434, -1, 436, 437, 438, 439, 440, + 441, 442, 443, -1, -1, 446, 447, 448, 449, 450, + 451, 452, 453, -1, 455, 456, 457, 458, 459, 460, + 461, -1, 463, 464, 465, 466, 467, 468, 469, 470, + 471, -1, -1, 474, -1, 476, 477, 478, 479, 480, + 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, + 491, 492, 493, 494, 495, 496, 497, -1, 3, 500, + 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 514, -1, -1, -1, 22, 23, 24, + 25, 26, 27, 28, 29, -1, 31, 32, 33, -1, + -1, -1, -1, -1, -1, -1, -1, 42, 43, 44, + -1, 46, 47, 48, 49, 50, 51, 52, 53, 54, + 55, 56, 57, -1, 59, 60, 61, 62, 63, 64, + -1, 66, 67, 68, 69, 70, 71, 72, 73, -1, + 75, 76, 77, 78, 79, -1, 81, -1, 83, 84, + 85, 86, 87, 88, 89, 90, 91, 92, -1, 94, + 95, 96, 97, 98, 99, -1, 101, 102, 103, 104, + 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, + 115, 116, -1, 118, -1, 120, 121, 122, 123, 124, + 125, 126, 127, 128, 129, 130, 131, -1, -1, 134, + 135, 136, 137, 138, -1, 140, 141, 142, -1, 144, + 145, 146, -1, 148, 149, 150, 151, 152, 153, 154, + 155, 156, 157, 158, -1, 160, -1, 162, 163, 164, + 165, -1, 167, -1, 169, 170, 171, 172, 173, 174, + 175, 176, 177, -1, 179, -1, 181, 182, 183, 184, + -1, 186, 187, 188, 189, 190, 191, 192, 193, 194, + 195, 196, 197, -1, 199, 200, 201, 202, 203, 204, + 205, -1, 207, 208, 209, 210, 211, 212, 213, 214, + 215, 216, -1, 218, -1, 220, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 230, 231, -1, 233, 234, + 235, 236, -1, 238, 239, 240, 241, 242, 243, 244, + 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, + 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, + 265, 266, 267, 268, 269, 270, 271, 272, 273, -1, + 275, 276, 277, 278, -1, 280, 281, 282, 283, 284, + 285, -1, 287, 288, -1, 290, 291, 292, 293, -1, + -1, 296, 297, 298, 299, 300, 301, 302, 303, 304, + 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, + 315, 316, -1, 318, 319, 320, 321, 322, 323, 324, + 325, 326, 327, -1, 329, 330, 331, 332, 333, 334, + -1, 336, 337, 338, 339, 340, 341, 342, 343, -1, + 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, + 355, 356, 357, -1, 359, 360, 361, 362, 363, 364, + 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, + 375, 376, 377, 378, 379, -1, 381, 382, 383, 384, + 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, + 395, -1, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407, 408, 409, 410, 411, 412, -1, 414, + 415, 416, 417, 418, 419, 420, 421, 422, 423, -1, + 425, 426, 427, -1, -1, 430, 431, 432, 433, 434, + -1, 436, 437, 438, 439, 440, 441, 442, 443, -1, + -1, 446, 447, 448, 449, 450, 451, 452, 453, -1, + 455, 456, 457, 458, 459, 460, 461, -1, 463, 464, + 465, 466, 467, 468, 469, 470, 471, -1, -1, 474, + -1, 476, 477, 478, 479, 480, 481, 482, 483, 484, + 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, + 495, 496, 497, -1, 3, 500, 5, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 514, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, + -1, -1, -1, 42, 43, 44, -1, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, 66, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, 81, -1, 83, 84, 85, 86, 87, 88, + 89, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, 101, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, 170, -1, 172, 173, 174, 175, 176, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, 193, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, 208, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, 221, 222, 223, 224, 225, 226, 227, 228, + 229, 230, 231, -1, 233, 234, 235, 236, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, + 269, 270, 271, 272, 273, -1, 275, 276, 277, 278, + -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, 290, 291, 292, 293, -1, -1, 296, 297, 298, + 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, 321, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, 391, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, 418, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, 464, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, - 32, 33, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 43, 44, -1, 46, 47, 48, -1, 50, 51, - 52, 53, 54, -1, 56, 57, -1, 59, 60, 61, - 62, 63, 64, -1, -1, 67, 68, 69, 70, 71, - 72, 73, -1, 75, 76, 77, 78, 79, -1, -1, - -1, 83, 84, 85, 86, 87, 88, -1, 90, 91, - 92, -1, 94, 95, 96, 97, 98, 99, -1, -1, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, -1, 118, -1, 120, 121, - 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, - -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, - 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, -1, 160, -1, - 162, 163, 164, 165, -1, 167, -1, 169, -1, -1, - -1, 173, 174, 175, -1, 177, -1, 179, -1, 181, - 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, - 192, -1, 194, 195, 196, 197, -1, 199, 200, 201, - 202, 203, 204, 205, -1, 207, -1, 209, 210, 211, - 212, 213, 214, 215, 216, -1, 218, -1, 220, -1, - -1, 223, -1, 225, 226, 227, 228, 229, 230, -1, - -1, 233, -1, 235, -1, -1, 238, 239, 240, 241, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, -1, 269, 270, 271, - 272, 273, -1, 275, 276, -1, 278, -1, 280, 281, - 282, 283, 284, 285, -1, 287, 288, -1, 290, 291, - 292, 293, -1, -1, 296, 297, 298, -1, 300, -1, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, -1, -1, -1, -1, 318, 319, 320, -1, - 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, - 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, - 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, -1, 359, 360, -1, - 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, -1, -1, 381, - 382, 383, 384, 385, 386, 387, 388, 389, -1, -1, - 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - -1, -1, 414, 415, -1, 417, -1, 419, 420, 421, - 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - 432, 433, 434, -1, 436, 437, 438, 439, 440, 441, - 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, - 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, - -1, 463, -1, 465, 466, 467, 468, 469, 470, 471, - -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, - 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 3, -1, 5, -1, + -1, -1, -1, -1, -1, 514, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, + -1, -1, -1, -1, -1, -1, -1, -1, 42, 43, + 44, -1, 46, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 57, -1, 59, 60, 61, 62, 63, + 64, -1, 66, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, 81, -1, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, 101, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, 170, -1, 172, 173, + 174, 175, 176, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, 193, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, 208, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, -1, -1, 233, + 234, 235, 236, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, + -1, 275, 276, 277, 278, -1, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, 299, 300, 301, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, 320, 321, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, 391, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, -1, 417, 418, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + 464, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, 30, 31, 32, 33, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 43, 44, -1, 46, - 47, 48, -1, 50, 51, 52, 53, 54, -1, 56, - 57, -1, 59, 60, 61, 62, 63, 64, -1, -1, - 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, 79, -1, -1, -1, 83, 84, 85, 86, - 87, 88, -1, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, -1, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, - -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, -1, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, -1, -1, -1, 173, 174, 175, -1, - 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, -1, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, -1, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, -1, -1, 223, -1, 225, 226, - 227, 228, 229, 230, -1, -1, 233, -1, 235, -1, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, -1, 269, 270, 271, 272, 273, -1, 275, 276, - -1, 278, -1, 280, 281, 282, 283, 284, 285, -1, - 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, 298, -1, 300, -1, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, -1, 322, 323, 324, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, -1, 362, 363, 364, 365, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, -1, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, -1, 392, 393, 394, 395, -1, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, - 417, -1, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, -1, 436, - 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, -1, 463, -1, 465, 466, - 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, - 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, - 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 3, 4, -1, -1, -1, -1, 9, -1, -1, + 514, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, + -1, -1, -1, 42, 43, 44, -1, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, 66, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, 81, -1, 83, 84, 85, 86, 87, 88, + 89, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, 101, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, 170, -1, 172, 173, 174, 175, 176, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, 193, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, 208, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, 221, 222, 223, 224, 225, 226, 227, 228, + 229, 230, -1, -1, 233, 234, 235, 236, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, + 269, 270, 271, 272, 273, -1, 275, 276, 277, 278, + -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, + 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, 321, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, 391, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, 418, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, 464, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, - 32, 33, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 43, 44, -1, 46, 47, 48, -1, 50, 51, - 52, 53, 54, -1, 56, 57, -1, 59, 60, 61, - 62, 63, 64, -1, -1, 67, 68, 69, 70, 71, - 72, 73, -1, 75, 76, 77, 78, 79, -1, -1, - -1, 83, 84, 85, 86, 87, 88, -1, 90, 91, - 92, -1, 94, 95, 96, 97, 98, 99, -1, -1, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, -1, 118, -1, 120, 121, - 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, - -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, - 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, -1, 160, -1, - 162, 163, 164, 165, -1, 167, -1, 169, -1, -1, - -1, 173, 174, 175, -1, 177, -1, 179, -1, 181, - 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, - 192, -1, 194, 195, 196, 197, -1, 199, 200, 201, - 202, 203, 204, 205, -1, 207, -1, 209, 210, 211, - 212, 213, 214, 215, 216, -1, 218, -1, 220, -1, - -1, 223, -1, 225, 226, 227, 228, 229, 230, -1, - -1, 233, -1, 235, -1, -1, 238, 239, 240, 241, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, -1, 269, 270, 271, - 272, 273, -1, 275, 276, -1, 278, -1, 280, 281, - 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, - 292, 293, -1, -1, 296, 297, 298, -1, 300, -1, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, -1, -1, -1, -1, 318, 319, 320, -1, - 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, - 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, - 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, -1, 359, 360, -1, - 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, -1, -1, 381, - 382, 383, 384, 385, 386, 387, 388, 389, -1, -1, - 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - -1, -1, 414, 415, -1, 417, -1, 419, 420, 421, - 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - 432, 433, 434, -1, 436, 437, 438, 439, 440, 441, - 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, - 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, - -1, 463, -1, 465, 466, 467, 468, 469, 470, 471, - -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, - 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 3, -1, 5, -1, + -1, -1, -1, -1, -1, 514, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 43, + 44, -1, 46, 47, 48, -1, 50, 51, 52, 53, + 54, -1, 56, 57, -1, 59, 60, 61, 62, 63, + 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, -1, -1, 83, + 84, 85, 86, 87, 88, -1, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, -1, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, + 124, 125, 126, 127, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, -1, 171, -1, 173, + 174, 175, -1, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, -1, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, -1, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, -1, -1, 223, + -1, 225, 226, 227, 228, 229, 230, -1, -1, 233, + -1, 235, -1, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, -1, 269, 270, 271, 272, 273, + -1, 275, 276, -1, 278, -1, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, -1, 300, -1, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + 314, 315, 316, -1, 318, 319, 320, -1, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, -1, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, 379, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, 390, -1, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, 412, -1, + 414, 415, 416, 417, -1, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, 449, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + -1, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, 476, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 3, -1, 500, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 514, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, 30, 31, 32, 33, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 43, 44, -1, 46, 47, 48, + -1, 50, 51, 52, 53, 54, -1, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, -1, -1, 83, 84, 85, 86, 87, 88, + -1, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, -1, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, -1, 171, -1, 173, 174, 175, -1, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, -1, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, -1, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, -1, -1, 223, -1, 225, 226, 227, 228, + 229, 230, -1, -1, 233, -1, 235, -1, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, -1, + 269, 270, 271, 272, 273, -1, 275, 276, -1, 278, + -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, + -1, 300, -1, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, 314, 315, 316, -1, 318, + 319, 320, -1, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, -1, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + 379, -1, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, -1, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, 416, 417, -1, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + 449, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, -1, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, 476, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, + -1, 500, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 514, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 43, + 44, -1, 46, 47, 48, -1, 50, 51, 52, 53, + 54, -1, 56, 57, -1, 59, 60, 61, 62, 63, + 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, -1, -1, 83, + 84, 85, 86, 87, 88, -1, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, -1, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, -1, 171, -1, 173, + 174, 175, -1, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, -1, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, -1, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, -1, -1, 223, + -1, 225, 226, 227, 228, 229, 230, -1, -1, 233, + -1, 235, -1, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, -1, 269, 270, 271, 272, 273, + -1, 275, 276, -1, 278, -1, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, -1, 300, -1, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + 314, 315, 316, -1, 318, 319, 320, -1, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, -1, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, 379, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, -1, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, 416, 417, -1, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, 449, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + -1, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, 476, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, -1, 3, 500, 5, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 514, -1, -1, -1, 22, 23, 24, 25, 26, 27, + 28, 29, -1, 31, 32, 33, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 43, 44, -1, 46, 47, + 48, -1, 50, 51, 52, 53, 54, -1, 56, 57, + -1, 59, 60, 61, 62, 63, 64, -1, -1, 67, + 68, 69, 70, 71, 72, 73, -1, 75, 76, 77, + 78, 79, -1, -1, -1, 83, 84, 85, 86, 87, + 88, -1, 90, 91, 92, -1, 94, 95, 96, 97, + 98, 99, -1, -1, 102, 103, 104, 105, 106, 107, + 108, 109, 110, 111, 112, 113, 114, 115, 116, -1, + 118, -1, 120, 121, 122, 123, 124, 125, -1, -1, + 128, 129, 130, 131, -1, -1, 134, 135, 136, 137, + 138, -1, 140, 141, 142, -1, 144, 145, 146, -1, + 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, + 158, -1, 160, -1, 162, 163, 164, 165, -1, 167, + -1, 169, -1, -1, -1, 173, 174, 175, -1, 177, + -1, 179, -1, 181, 182, 183, 184, -1, 186, 187, + 188, 189, 190, 191, 192, -1, 194, 195, 196, 197, + -1, 199, 200, 201, 202, 203, 204, 205, -1, 207, + -1, 209, 210, 211, 212, 213, 214, 215, 216, -1, + 218, -1, 220, -1, -1, 223, -1, 225, 226, 227, + 228, 229, 230, -1, -1, 233, -1, 235, -1, -1, + 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, + 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, + 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, + -1, 269, 270, 271, 272, 273, -1, 275, 276, -1, + 278, -1, 280, 281, 282, 283, 284, 285, -1, 287, + 288, -1, -1, 291, 292, 293, -1, -1, 296, 297, + 298, -1, 300, -1, 302, 303, 304, 305, 306, 307, + 308, 309, 310, 311, 312, 313, -1, -1, -1, -1, + 318, 319, 320, -1, 322, 323, 324, 325, 326, 327, + -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, + 338, 339, 340, 341, 342, 343, -1, 345, 346, 347, + 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, + -1, 359, 360, -1, 362, 363, 364, 365, 366, 367, + 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, + 378, -1, -1, 381, 382, 383, 384, 385, 386, 387, + 388, 389, -1, -1, 392, 393, 394, 395, -1, 397, + 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, + 408, 409, 410, 411, -1, -1, 414, 415, -1, 417, + -1, 419, 420, 421, 422, 423, -1, 425, 426, 427, + -1, -1, 430, 431, 432, 433, 434, -1, 436, 437, + 438, 439, 440, 441, 442, 443, -1, -1, 446, 447, + 448, -1, 450, 451, 452, 453, -1, 455, 456, 457, + 458, 459, 460, 461, -1, 463, -1, 465, 466, 467, + 468, 469, 470, 471, -1, -1, 474, -1, -1, 477, + 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, + 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, + 3, -1, 5, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 514, -1, -1, 22, + 23, 24, 25, 26, 27, 28, 29, -1, 31, 32, + 33, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 43, 44, -1, 46, 47, 48, -1, 50, 51, 52, + 53, 54, -1, 56, 57, -1, 59, 60, 61, 62, + 63, 64, -1, -1, 67, 68, 69, 70, 71, 72, + 73, -1, 75, 76, 77, 78, 79, -1, -1, -1, + 83, 84, 85, 86, 87, 88, -1, 90, 91, 92, + -1, 94, 95, 96, 97, 98, 99, -1, -1, 102, + 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + 113, 114, 115, 116, -1, 118, -1, 120, 121, 122, + 123, 124, 125, -1, -1, 128, 129, 130, 131, -1, + -1, 134, 135, 136, 137, 138, -1, 140, 141, 142, + -1, 144, 145, 146, -1, 148, 149, 150, 151, 152, + 153, 154, 155, 156, 157, 158, -1, 160, -1, 162, + 163, 164, 165, -1, 167, -1, 169, -1, -1, -1, + 173, 174, 175, -1, 177, -1, 179, -1, 181, 182, + 183, 184, -1, 186, 187, 188, 189, 190, 191, 192, + -1, 194, 195, 196, 197, -1, 199, 200, 201, 202, + 203, 204, 205, -1, 207, -1, 209, 210, 211, 212, + 213, 214, 215, 216, -1, 218, -1, 220, -1, -1, + 223, -1, 225, 226, 227, 228, 229, 230, -1, -1, + 233, -1, 235, -1, -1, 238, 239, 240, 241, 242, + 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, + 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, + 263, 264, 265, 266, 267, -1, 269, 270, 271, 272, + 273, -1, 275, 276, -1, 278, -1, 280, 281, 282, + 283, 284, 285, -1, 287, 288, -1, -1, 291, 292, + 293, -1, -1, 296, 297, 298, -1, 300, -1, 302, + 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, + 313, -1, -1, -1, -1, 318, 319, 320, -1, 322, + 323, 324, 325, 326, 327, -1, 329, 330, 331, 332, + 333, 334, -1, 336, 337, 338, 339, 340, 341, 342, + 343, -1, 345, 346, 347, 348, 349, 350, 351, 352, + 353, 354, 355, 356, 357, -1, 359, 360, -1, 362, + 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, + 373, 374, 375, 376, 377, 378, -1, -1, 381, 382, + 383, 384, 385, 386, 387, 388, 389, -1, -1, 392, + 393, 394, 395, -1, 397, 398, 399, 400, 401, 402, + 403, 404, 405, 406, 407, 408, 409, 410, 411, -1, + -1, 414, 415, -1, 417, -1, 419, 420, 421, 422, + 423, -1, 425, 426, 427, -1, -1, 430, 431, 432, + 433, 434, -1, 436, 437, 438, 439, 440, 441, 442, + 443, -1, -1, 446, 447, 448, -1, 450, 451, 452, + 453, -1, 455, 456, 457, 458, 459, 460, 461, -1, + 463, -1, 465, 466, 467, 468, 469, 470, 471, -1, + -1, 474, -1, -1, 477, 478, 479, 480, 481, 482, + 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, + 493, 494, 495, 496, 497, 3, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 43, 44, -1, 46, - 47, 48, -1, 50, 51, 52, 53, 54, -1, 56, - 57, -1, 59, 60, 61, 62, 63, 64, -1, -1, - 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, 79, -1, -1, -1, 83, 84, 85, 86, - 87, 88, -1, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, -1, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, - -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, -1, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, -1, -1, -1, 173, 174, 175, -1, - 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, -1, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, -1, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, -1, -1, 223, -1, 225, 226, - 227, 228, 229, 230, -1, -1, 233, -1, 235, -1, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, -1, 269, 270, 271, 272, 273, -1, 275, 276, - -1, 278, -1, 280, 281, 282, 283, 284, 285, -1, - 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, 298, -1, 300, -1, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, -1, 322, 323, 324, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, -1, 362, 363, 364, 365, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, -1, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, -1, 392, 393, 394, 395, -1, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, -1, -1, 414, 415, 416, - 417, -1, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, -1, 436, - 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, -1, 463, -1, 465, 466, - 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, - 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, - 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 3, -1, 5, -1, -1, -1, -1, -1, -1, + -1, 514, -1, -1, 22, 23, 24, 25, 26, 27, + 28, 29, -1, 31, 32, 33, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 43, 44, -1, 46, 47, + 48, -1, 50, 51, 52, 53, 54, -1, 56, 57, + -1, 59, 60, 61, 62, 63, 64, -1, -1, 67, + 68, 69, 70, 71, 72, 73, -1, 75, 76, 77, + 78, 79, -1, -1, -1, 83, 84, 85, 86, 87, + 88, -1, 90, 91, 92, -1, 94, 95, 96, 97, + 98, 99, -1, -1, 102, 103, 104, 105, 106, 107, + 108, 109, 110, 111, 112, 113, 114, 115, 116, -1, + 118, -1, 120, 121, 122, 123, 124, 125, -1, -1, + 128, 129, 130, 131, -1, -1, 134, 135, 136, 137, + 138, -1, 140, 141, 142, -1, 144, 145, 146, -1, + 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, + 158, -1, 160, -1, 162, 163, 164, 165, -1, 167, + -1, 169, -1, -1, -1, 173, 174, 175, -1, 177, + -1, 179, -1, 181, 182, 183, 184, -1, 186, 187, + 188, 189, 190, 191, 192, -1, 194, 195, 196, 197, + -1, 199, 200, 201, 202, 203, 204, 205, -1, 207, + -1, 209, 210, 211, 212, 213, 214, 215, 216, -1, + 218, -1, 220, -1, -1, 223, -1, 225, 226, 227, + 228, 229, 230, -1, -1, 233, -1, 235, -1, -1, + 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, + 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, + 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, + -1, 269, 270, 271, 272, 273, -1, 275, 276, -1, + 278, -1, 280, 281, 282, 283, 284, 285, -1, 287, + 288, -1, -1, 291, 292, 293, -1, -1, 296, 297, + 298, -1, 300, -1, 302, 303, 304, 305, 306, 307, + 308, 309, 310, 311, 312, 313, -1, -1, -1, -1, + 318, 319, 320, -1, 322, 323, 324, 325, 326, 327, + -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, + 338, 339, 340, 341, 342, 343, -1, 345, 346, 347, + 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, + -1, 359, 360, -1, 362, 363, 364, 365, 366, 367, + 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, + 378, -1, -1, 381, 382, 383, 384, 385, 386, 387, + 388, 389, -1, -1, 392, 393, 394, 395, -1, 397, + 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, + 408, 409, 410, 411, -1, -1, 414, 415, -1, 417, + -1, 419, 420, 421, 422, 423, -1, 425, 426, 427, + -1, -1, 430, 431, 432, 433, 434, -1, 436, 437, + 438, 439, 440, 441, 442, 443, -1, -1, 446, 447, + 448, -1, 450, 451, 452, 453, -1, 455, 456, 457, + 458, 459, 460, 461, -1, 463, -1, 465, 466, 467, + 468, 469, 470, 471, -1, -1, 474, -1, -1, 477, + 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, + 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, + 3, -1, 5, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 514, -1, -1, 22, + 23, 24, 25, 26, 27, 28, 29, -1, 31, 32, + 33, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 43, 44, -1, 46, 47, 48, -1, 50, 51, 52, + 53, 54, -1, 56, 57, -1, 59, 60, 61, 62, + 63, 64, -1, -1, 67, 68, 69, 70, 71, 72, + 73, -1, 75, 76, 77, 78, 79, -1, -1, -1, + 83, 84, 85, 86, 87, 88, -1, 90, 91, 92, + -1, 94, 95, 96, 97, 98, 99, -1, -1, 102, + 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + 113, 114, 115, 116, -1, 118, -1, 120, 121, 122, + 123, 124, 125, -1, -1, 128, 129, 130, 131, -1, + -1, 134, 135, 136, 137, 138, -1, 140, 141, 142, + -1, 144, 145, 146, -1, 148, 149, 150, 151, 152, + 153, 154, 155, 156, 157, 158, -1, 160, -1, 162, + 163, 164, 165, -1, 167, -1, 169, -1, -1, -1, + 173, 174, 175, -1, 177, -1, 179, -1, 181, 182, + 183, 184, -1, 186, 187, 188, 189, 190, 191, 192, + -1, 194, 195, 196, 197, -1, 199, 200, 201, 202, + 203, 204, 205, -1, 207, -1, 209, 210, 211, 212, + 213, 214, 215, 216, -1, 218, -1, 220, -1, -1, + 223, -1, 225, 226, 227, 228, 229, 230, -1, -1, + 233, -1, 235, -1, -1, 238, 239, 240, 241, 242, + 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, + 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, + 263, 264, 265, 266, 267, -1, 269, 270, 271, 272, + 273, -1, 275, 276, -1, 278, -1, 280, 281, 282, + 283, 284, 285, -1, 287, 288, -1, -1, 291, 292, + 293, -1, -1, 296, 297, 298, -1, 300, -1, 302, + 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, + 313, -1, -1, -1, -1, 318, 319, 320, -1, 322, + 323, 324, 325, 326, 327, -1, 329, 330, 331, 332, + 333, 334, -1, 336, 337, 338, 339, 340, 341, 342, + 343, -1, 345, 346, 347, 348, 349, 350, 351, 352, + 353, 354, 355, 356, 357, -1, 359, 360, -1, 362, + 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, + 373, 374, 375, 376, 377, 378, -1, -1, 381, 382, + 383, 384, 385, 386, 387, 388, 389, -1, -1, 392, + 393, 394, 395, -1, 397, 398, 399, 400, 401, 402, + 403, 404, 405, 406, 407, 408, 409, 410, 411, -1, + -1, 414, 415, -1, 417, -1, 419, 420, 421, 422, + 423, -1, 425, 426, 427, -1, -1, 430, 431, 432, + 433, 434, -1, 436, 437, 438, 439, 440, 441, 442, + 443, -1, -1, 446, 447, 448, -1, 450, 451, 452, + 453, -1, 455, 456, 457, 458, 459, 460, 461, -1, + 463, -1, 465, 466, 467, 468, 469, 470, 471, -1, + -1, 474, -1, -1, 477, 478, 479, 480, 481, 482, + 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, + 493, 494, 495, 496, 497, 3, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, - 32, 33, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 43, 44, -1, 46, 47, 48, -1, 50, 51, - 52, 53, 54, -1, 56, 57, -1, 59, 60, 61, - 62, 63, 64, -1, -1, 67, 68, 69, 70, 71, - 72, 73, -1, 75, 76, 77, 78, 79, -1, -1, - -1, 83, 84, 85, 86, 87, 88, -1, 90, 91, - 92, -1, 94, 95, 96, 97, 98, 99, -1, -1, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, -1, 118, -1, 120, 121, - 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, - -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, - 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, -1, 160, -1, - 162, 163, 164, 165, -1, 167, -1, 169, -1, -1, - -1, 173, 174, 175, -1, 177, -1, 179, -1, 181, - 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, - 192, -1, 194, 195, 196, 197, -1, 199, 200, 201, - 202, 203, 204, 205, -1, 207, -1, 209, 210, 211, - 212, 213, 214, 215, 216, -1, 218, -1, 220, -1, - -1, 223, -1, 225, 226, 227, 228, 229, 230, -1, - -1, 233, -1, 235, -1, -1, 238, 239, 240, 241, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, -1, 269, 270, 271, - 272, 273, -1, 275, 276, -1, 278, -1, 280, 281, - 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, - 292, 293, -1, -1, 296, 297, 298, -1, 300, -1, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, -1, -1, -1, -1, 318, 319, 320, -1, - 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, - 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, - 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, -1, 359, 360, -1, - 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, -1, -1, 381, - 382, 383, 384, 385, 386, 387, 388, 389, -1, -1, - 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - -1, -1, 414, 415, -1, 417, -1, 419, 420, 421, - 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - 432, 433, 434, -1, 436, 437, 438, 439, 440, 441, - 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, - 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, - -1, 463, -1, 465, 466, 467, 468, 469, 470, 471, - -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, - 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 3, -1, 5, -1, + -1, 514, -1, -1, 22, 23, 24, 25, 26, 27, + 28, 29, -1, 31, 32, 33, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 43, 44, -1, 46, 47, + 48, -1, 50, 51, 52, 53, 54, -1, 56, 57, + -1, 59, 60, 61, 62, 63, 64, -1, -1, 67, + 68, 69, 70, 71, 72, 73, -1, 75, 76, 77, + 78, 79, -1, -1, -1, 83, 84, 85, 86, 87, + 88, -1, 90, 91, 92, -1, 94, 95, 96, 97, + 98, 99, -1, -1, 102, 103, 104, 105, 106, 107, + 108, 109, 110, 111, 112, 113, 114, 115, 116, -1, + 118, -1, 120, 121, 122, 123, 124, 125, -1, -1, + 128, 129, 130, 131, -1, -1, 134, 135, 136, 137, + 138, -1, 140, 141, 142, -1, 144, 145, 146, -1, + 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, + 158, -1, 160, -1, 162, 163, 164, 165, -1, 167, + -1, 169, -1, -1, -1, 173, 174, 175, -1, 177, + -1, 179, -1, 181, 182, 183, 184, -1, 186, 187, + 188, 189, 190, 191, 192, -1, 194, 195, 196, 197, + -1, 199, 200, 201, 202, 203, 204, 205, -1, 207, + -1, 209, 210, 211, 212, 213, 214, 215, 216, -1, + 218, -1, 220, -1, -1, 223, -1, 225, 226, 227, + 228, 229, 230, -1, -1, 233, -1, 235, -1, -1, + 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, + 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, + 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, + -1, 269, 270, 271, 272, 273, -1, 275, 276, -1, + 278, -1, 280, 281, 282, 283, 284, 285, -1, 287, + 288, -1, -1, 291, 292, 293, -1, -1, 296, 297, + 298, -1, 300, -1, 302, 303, 304, 305, 306, 307, + 308, 309, 310, 311, 312, 313, -1, -1, -1, -1, + 318, 319, 320, -1, 322, 323, 324, 325, 326, 327, + -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, + 338, 339, 340, 341, 342, 343, -1, 345, 346, 347, + 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, + -1, 359, 360, -1, 362, 363, 364, 365, 366, 367, + 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, + 378, -1, -1, 381, 382, 383, 384, 385, 386, 387, + 388, 389, -1, -1, 392, 393, 394, 395, -1, 397, + 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, + 408, 409, 410, 411, -1, -1, 414, 415, -1, 417, + -1, 419, 420, 421, 422, 423, -1, 425, 426, 427, + -1, -1, 430, 431, 432, 433, 434, -1, 436, 437, + 438, 439, 440, 441, 442, 443, -1, -1, 446, 447, + 448, -1, 450, 451, 452, 453, -1, 455, 456, 457, + 458, 459, 460, 461, -1, 463, -1, 465, 466, 467, + 468, 469, 470, 471, -1, -1, 474, -1, -1, 477, + 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, + 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, + 3, -1, 5, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 514, -1, -1, 22, + 23, 24, 25, 26, 27, 28, 29, -1, 31, 32, + 33, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 43, 44, -1, 46, 47, 48, -1, 50, 51, 52, + 53, 54, -1, 56, 57, -1, 59, 60, 61, 62, + 63, 64, -1, -1, 67, 68, 69, 70, 71, 72, + 73, -1, 75, 76, 77, 78, 79, -1, -1, -1, + 83, 84, 85, 86, 87, 88, -1, 90, 91, 92, + -1, 94, 95, 96, 97, 98, 99, -1, -1, 102, + 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + 113, 114, 115, 116, -1, 118, -1, 120, 121, 122, + 123, 124, 125, -1, -1, 128, 129, 130, 131, -1, + -1, 134, 135, 136, 137, 138, -1, 140, 141, 142, + -1, 144, 145, 146, -1, 148, 149, 150, 151, 152, + 153, 154, 155, 156, 157, 158, -1, 160, -1, 162, + 163, 164, 165, -1, 167, -1, 169, -1, -1, -1, + 173, 174, 175, -1, 177, -1, 179, -1, 181, 182, + 183, 184, -1, 186, 187, 188, 189, 190, 191, 192, + -1, 194, 195, 196, 197, -1, 199, 200, 201, 202, + 203, 204, 205, -1, 207, -1, 209, 210, 211, 212, + 213, 214, 215, 216, -1, 218, -1, 220, -1, -1, + 223, -1, 225, 226, 227, 228, 229, 230, -1, -1, + 233, -1, 235, -1, -1, 238, 239, 240, 241, 242, + 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, + 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, + 263, 264, 265, 266, 267, -1, 269, 270, 271, 272, + 273, -1, 275, 276, -1, 278, -1, 280, 281, 282, + 283, 284, 285, -1, 287, 288, -1, -1, 291, 292, + 293, -1, -1, 296, 297, 298, -1, 300, -1, 302, + 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, + 313, -1, -1, -1, -1, 318, 319, 320, -1, 322, + 323, 324, 325, 326, 327, -1, 329, 330, 331, 332, + 333, 334, -1, 336, 337, 338, 339, 340, 341, 342, + 343, -1, 345, 346, 347, 348, 349, 350, 351, 352, + 353, 354, 355, 356, 357, -1, 359, 360, -1, 362, + 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, + 373, 374, 375, 376, 377, 378, -1, -1, 381, 382, + 383, 384, 385, 386, 387, 388, 389, -1, -1, 392, + 393, 394, 395, -1, 397, 398, 399, 400, 401, 402, + 403, 404, 405, 406, 407, 408, 409, 410, 411, -1, + -1, 414, 415, -1, 417, -1, 419, 420, 421, 422, + 423, -1, 425, 426, 427, -1, -1, 430, 431, 432, + 433, 434, -1, 436, 437, 438, 439, 440, 441, 442, + 443, -1, -1, 446, 447, 448, -1, 450, 451, 452, + 453, -1, 455, 456, 457, 458, 459, 460, 461, -1, + 463, -1, 465, 466, 467, 468, 469, 470, 471, -1, + -1, 474, -1, -1, 477, 478, 479, 480, 481, 482, + 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, + 493, 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 43, 44, -1, 46, - 47, 48, -1, 50, 51, 52, 53, 54, -1, 56, - 57, -1, 59, 60, 61, 62, 63, 64, -1, -1, - 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, 79, -1, -1, -1, 83, 84, 85, 86, - 87, 88, -1, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, -1, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, - -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, -1, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, -1, -1, -1, 173, 174, 175, -1, - 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, -1, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, -1, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, -1, -1, 223, -1, 225, 226, - 227, 228, 229, 230, -1, -1, 233, -1, 235, -1, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, -1, 269, 270, 271, 272, 273, -1, 275, 276, - -1, 278, -1, 280, 281, 282, 283, 284, 285, -1, - 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, 298, -1, 300, -1, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, -1, 322, 323, 324, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, -1, 362, 363, 364, 365, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, -1, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, -1, 392, 393, 394, 395, -1, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, - 417, -1, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, -1, 436, - 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, -1, 463, -1, 465, 466, - 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, - 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, - 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 514, -1, -1, 22, 23, 24, 25, 26, 27, + 28, 29, -1, 31, 32, 33, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 43, 44, -1, 46, 47, + 48, -1, 50, 51, 52, 53, 54, -1, 56, 57, + -1, 59, 60, 61, 62, 63, 64, -1, -1, 67, + 68, 69, 70, 71, 72, 73, -1, 75, 76, 77, + 78, 79, -1, -1, -1, 83, 84, 85, 86, 87, + 88, -1, 90, 91, 92, -1, 94, 95, 96, 97, + 98, 99, -1, -1, 102, 103, 104, 105, 106, 107, + 108, 109, 110, 111, 112, 113, 114, 115, 116, -1, + 118, -1, 120, 121, 122, 123, 124, 125, -1, -1, + 128, 129, 130, 131, -1, -1, 134, 135, 136, 137, + 138, -1, 140, 141, 142, -1, 144, 145, 146, -1, + 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, + 158, -1, 160, -1, 162, 163, 164, 165, -1, 167, + -1, 169, -1, -1, -1, 173, 174, 175, -1, 177, + -1, 179, -1, 181, 182, 183, 184, -1, 186, 187, + 188, 189, 190, 191, 192, -1, 194, 195, 196, 197, + -1, 199, 200, 201, 202, 203, 204, 205, -1, 207, + -1, 209, 210, 211, 212, 213, 214, 215, 216, -1, + 218, -1, 220, -1, -1, 223, -1, 225, 226, 227, + 228, 229, 230, -1, -1, 233, -1, 235, -1, -1, + 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, + 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, + 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, + -1, 269, 270, 271, 272, 273, -1, 275, 276, -1, + 278, -1, 280, 281, 282, 283, 284, 285, -1, 287, + 288, -1, -1, 291, 292, 293, -1, -1, 296, 297, + 298, -1, 300, -1, 302, 303, 304, 305, 306, 307, + 308, 309, 310, 311, 312, 313, -1, -1, -1, -1, + 318, 319, 320, -1, 322, 323, 324, 325, 326, 327, + -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, + 338, 339, 340, 341, 342, 343, -1, 345, 346, 347, + 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, + -1, 359, 360, -1, 362, 363, 364, 365, 366, 367, + 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, + 378, -1, -1, 381, 382, 383, 384, 385, 386, 387, + 388, 389, -1, -1, 392, 393, 394, 395, -1, 397, + 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, + 408, 409, 410, 411, -1, -1, 414, 415, -1, 417, + -1, 419, 420, 421, 422, 423, -1, 425, 426, 427, + -1, -1, 430, 431, 432, 433, 434, -1, 436, 437, + 438, 439, 440, 441, 442, 443, -1, -1, 446, 447, + 448, -1, 450, 451, 452, 453, -1, 455, 456, 457, + 458, 459, 460, 461, -1, 463, -1, 465, 466, 467, + 468, 469, 470, 471, -1, -1, 474, -1, -1, 477, + 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, + 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, + 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 514, -1, -1, 22, + 23, 24, 25, 26, 27, 28, 29, -1, 31, 32, + 33, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 43, 44, -1, 46, 47, 48, -1, 50, 51, 52, + 53, 54, -1, 56, 57, -1, 59, 60, 61, 62, + 63, 64, -1, -1, 67, 68, 69, 70, 71, 72, + 73, -1, 75, 76, 77, 78, 79, -1, -1, -1, + 83, 84, 85, 86, 87, 88, -1, 90, 91, 92, + -1, 94, 95, 96, 97, 98, 99, -1, -1, 102, + 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + 113, 114, 115, 116, -1, 118, -1, 120, 121, 122, + 123, 124, 125, -1, -1, 128, 129, 130, 131, -1, + -1, 134, 135, 136, 137, 138, -1, 140, 141, 142, + -1, 144, 145, 146, -1, 148, 149, 150, 151, 152, + 153, 154, 155, 156, 157, 158, -1, 160, -1, 162, + 163, 164, 165, -1, 167, -1, 169, -1, -1, -1, + 173, 174, 175, -1, 177, -1, 179, -1, 181, 182, + 183, 184, -1, 186, 187, 188, 189, 190, 191, 192, + -1, 194, 195, 196, 197, -1, 199, 200, 201, 202, + 203, 204, 205, -1, 207, -1, 209, 210, 211, 212, + 213, 214, 215, 216, -1, 218, -1, 220, -1, -1, + 223, -1, 225, 226, 227, 228, 229, 230, -1, -1, + 233, -1, 235, -1, -1, 238, 239, 240, 241, 242, + 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, + 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, + 263, 264, 265, 266, 267, -1, 269, 270, 271, 272, + 273, -1, 275, 276, -1, 278, -1, 280, 281, 282, + 283, 284, 285, -1, 287, 288, -1, -1, 291, 292, + 293, -1, -1, 296, 297, 298, -1, 300, -1, 302, + 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, + 313, -1, -1, -1, -1, 318, 319, 320, -1, 322, + 323, 324, 325, 326, 327, -1, 329, 330, 331, 332, + 333, 334, -1, 336, 337, 338, 339, 340, 341, 342, + 343, -1, 345, 346, 347, 348, 349, 350, 351, 352, + 353, 354, 355, 356, 357, -1, 359, 360, -1, 362, + 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, + 373, 374, 375, 376, 377, 378, -1, -1, 381, 382, + 383, 384, 385, 386, 387, 388, 389, -1, -1, 392, + 393, 394, 395, -1, 397, 398, 399, 400, 401, 402, + 403, 404, 405, 406, 407, 408, 409, 410, 411, -1, + -1, 414, 415, -1, 417, -1, 419, 420, 421, 422, + 423, -1, 425, 426, 427, -1, -1, 430, 431, 432, + 433, 434, -1, 436, 437, 438, 439, 440, 441, 442, + 443, -1, -1, 446, 447, 448, -1, 450, 451, 452, + 453, -1, 455, 456, 457, 458, 459, 460, 461, -1, + 463, -1, 465, 466, 467, 468, 469, 470, 471, -1, + -1, 474, -1, -1, 477, 478, 479, 480, 481, 482, + 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, + 493, 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 43, 44, -1, 46, 47, 48, -1, 50, 51, - 52, 53, 54, -1, 56, 57, -1, 59, 60, 61, - 62, 63, 64, -1, -1, 67, 68, 69, 70, 71, - 72, 73, -1, 75, 76, 77, 78, 79, -1, -1, - -1, 83, 84, 85, 86, 87, 88, -1, 90, 91, - 92, -1, 94, 95, 96, 97, 98, 99, -1, -1, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, -1, 118, -1, 120, 121, - 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, - -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, - 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, -1, 160, -1, - 162, 163, 164, 165, -1, 167, -1, 169, -1, -1, - -1, 173, 174, 175, -1, 177, -1, 179, -1, 181, - 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, - 192, -1, 194, 195, 196, 197, -1, 199, 200, 201, - 202, 203, 204, 205, -1, 207, -1, 209, 210, 211, - 212, 213, 214, 215, 216, -1, 218, -1, 220, -1, - -1, 223, -1, 225, 226, 227, 228, 229, 230, -1, - -1, 233, -1, 235, -1, -1, 238, 239, 240, 241, + -1, 514, -1, -1, 22, 23, 24, 25, 26, 27, + 28, 29, -1, 31, 32, 33, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 43, 44, -1, 46, 47, + 48, -1, 50, 51, 52, 53, 54, -1, 56, 57, + -1, 59, 60, 61, 62, 63, 64, -1, -1, 67, + 68, 69, 70, 71, 72, 73, -1, 75, 76, 77, + 78, 79, -1, -1, -1, 83, 84, 85, 86, 87, + 88, -1, 90, 91, 92, -1, 94, 95, 96, 97, + 98, 99, -1, -1, 102, 103, 104, 105, 106, 107, + 108, 109, 110, 111, 112, 113, 114, 115, 116, -1, + 118, -1, 120, 121, 122, 123, 124, 125, -1, -1, + 128, 129, 130, 131, -1, -1, 134, 135, 136, 137, + 138, -1, 140, 141, 142, -1, 144, 145, 146, -1, + 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, + 158, -1, 160, -1, 162, 163, 164, 165, -1, 167, + -1, 169, -1, -1, -1, 173, 174, 175, -1, 177, + -1, 179, -1, 181, 182, 183, 184, -1, 186, 187, + 188, 189, 190, 191, 192, -1, 194, 195, 196, 197, + -1, 199, 200, 201, 202, 203, 204, 205, -1, 207, + -1, 209, 210, 211, 212, 213, 214, 215, 216, -1, + 218, -1, 220, -1, -1, 223, -1, 225, 226, 227, + 228, 229, 230, -1, -1, 233, -1, 235, -1, -1, + 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, + 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, + 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, + -1, 269, 270, 271, 272, 273, -1, 275, 276, -1, + 278, -1, 280, 281, 282, 283, 284, 285, -1, 287, + 288, -1, -1, 291, 292, 293, -1, -1, 296, 297, + 298, -1, 300, -1, 302, 303, 304, 305, 306, 307, + 308, 309, 310, 311, 312, 313, -1, -1, -1, -1, + 318, 319, 320, -1, 322, 323, 324, 325, 326, 327, + -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, + 338, 339, 340, 341, 342, 343, -1, 345, 346, 347, + 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, + -1, 359, 360, -1, 362, 363, 364, 365, 366, 367, + 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, + 378, -1, -1, 381, 382, 383, 384, 385, 386, 387, + 388, 389, -1, -1, 392, 393, 394, 395, -1, 397, + 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, + 408, 409, 410, 411, -1, -1, 414, 415, -1, 417, + -1, 419, 420, 421, 422, 423, -1, 425, 426, 427, + -1, -1, 430, 431, 432, 433, 434, -1, 436, 437, + 438, 439, 440, 441, 442, 443, -1, -1, 446, 447, + 448, -1, 450, 451, 452, 453, -1, 455, 456, 457, + 458, 459, 460, 461, -1, 463, -1, 465, 466, 467, + 468, 469, 470, 471, -1, -1, 474, -1, -1, 477, + 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, + 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, + -1, 3, 4, 5, -1, -1, 8, 9, -1, -1, + -1, -1, -1, 15, 16, -1, 514, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, + 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, + 52, -1, 54, 55, 56, 57, 58, 59, 60, 61, + 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, + 72, 73, 74, 75, 76, 77, 78, -1, 80, 81, + 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, + 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, + 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, + 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, + 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, + 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, + 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, + -1, 153, 154, 155, 156, 157, -1, 159, 160, 161, + 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, + 172, 173, 174, 175, 176, 177, 178, 179, 180, -1, + -1, 183, 184, 185, 186, 187, 188, 189, 190, 191, + 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, + 202, 203, 204, 205, 206, 207, 208, -1, 210, 211, + 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, + 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, + 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, -1, 269, 270, 271, - 272, 273, -1, 275, 276, -1, 278, -1, 280, 281, - 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, - 292, 293, -1, -1, 296, 297, 298, -1, 300, -1, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, -1, -1, -1, -1, 318, 319, 320, -1, - 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, - 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, - 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, -1, 359, 360, -1, + 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, + 272, 273, 274, 275, 276, 277, 278, 279, -1, 281, + 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, + 292, 293, 294, 295, 296, 297, -1, 299, 300, 301, + -1, 303, 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, -1, 321, + 322, 323, -1, 325, 326, 327, 328, 329, 330, 331, + 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, + 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, + 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, -1, -1, 381, - 382, 383, 384, 385, 386, 387, 388, 389, -1, -1, - 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - -1, -1, 414, 415, -1, 417, -1, 419, 420, 421, - 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - 432, 433, 434, -1, 436, 437, 438, 439, 440, 441, - 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, - 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, - -1, 463, -1, 465, 466, 467, 468, 469, 470, 471, - -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, - 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 3, -1, 5, -1, + 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, + 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, + 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, + 402, 403, 404, 405, 406, 407, 408, 409, 410, -1, + 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, + 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, + -1, 433, -1, 435, 436, 437, 438, 439, 440, 441, + 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, + 452, 453, 454, 455, 456, 457, 458, 459, -1, 461, + 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, + 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, + 482, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 494, 495, 496, 497, -1, 3, -1, 501, + 502, 503, 8, 505, 506, 507, 508, 509, 510, 15, + 16, -1, -1, 19, 20, 21, 22, 23, 24, 25, + 26, 27, 28, 29, -1, 31, 32, 33, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 43, 44, -1, + 46, 47, 48, -1, 50, 51, 52, 53, 54, -1, + 56, 57, -1, 59, 60, 61, 62, 63, 64, -1, + -1, 67, 68, 69, 70, 71, 72, 73, -1, 75, + 76, 77, 78, 79, -1, -1, -1, 83, 84, 85, + 86, 87, 88, -1, 90, 91, 92, -1, 94, 95, + 96, 97, 98, 99, -1, -1, 102, 103, 104, 105, + 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, + 116, -1, 118, -1, 120, 121, 122, 123, 124, 125, + -1, -1, 128, 129, 130, 131, -1, -1, 134, 135, + 136, 137, 138, -1, 140, 141, 142, -1, 144, 145, + 146, -1, 148, 149, 150, 151, 152, 153, 154, 155, + 156, 157, 158, -1, 160, -1, 162, 163, 164, 165, + -1, 167, -1, 169, -1, -1, -1, 173, 174, 175, + -1, 177, -1, 179, -1, 181, 182, 183, 184, -1, + 186, 187, 188, 189, 190, 191, 192, -1, 194, 195, + 196, 197, -1, 199, 200, 201, 202, 203, 204, 205, + -1, 207, -1, 209, 210, 211, 212, 213, 214, 215, + 216, -1, 218, -1, 220, -1, -1, 223, -1, 225, + 226, 227, 228, 229, 230, -1, -1, 233, -1, 235, + -1, -1, 238, 239, 240, 241, 242, 243, 244, 245, + 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, + 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, + 266, 267, -1, 269, 270, 271, 272, 273, -1, 275, + 276, -1, 278, -1, 280, 281, 282, 283, 284, 285, + -1, 287, 288, -1, -1, 291, 292, 293, -1, -1, + 296, 297, 298, -1, 300, -1, 302, 303, 304, 305, + 306, 307, 308, 309, 310, 311, 312, 313, -1, -1, + -1, -1, 318, 319, 320, -1, 322, 323, 324, 325, + 326, 327, -1, 329, 330, 331, 332, 333, 334, -1, + 336, 337, 338, 339, 340, 341, 342, 343, -1, 345, + 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, + 356, 357, -1, 359, 360, -1, 362, 363, 364, 365, + 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, + 376, 377, 378, -1, -1, 381, 382, 383, 384, 385, + 386, 387, 388, 389, -1, -1, 392, 393, 394, 395, + -1, 397, 398, 399, 400, 401, 402, 403, 404, 405, + 406, 407, 408, 409, 410, 411, -1, -1, 414, 415, + -1, 417, -1, 419, 420, 421, 422, 423, -1, 425, + 426, 427, -1, -1, 430, 431, 432, 433, 434, -1, + 436, 437, 438, 439, 440, 441, 442, 443, -1, -1, + 446, 447, 448, -1, 450, 451, 452, 453, -1, 455, + 456, 457, 458, 459, 460, 461, -1, 463, -1, 465, + 466, 467, 468, 469, 470, 471, -1, -1, 474, -1, + -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, + 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, + 496, 497, -1, -1, -1, 501, 502, 503, -1, 505, + 506, 507, 508, 509, 510, 8, -1, -1, 11, -1, + -1, -1, 15, 16, 17, 18, 19, 20, 21, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 43, 44, -1, 46, - 47, 48, -1, 50, 51, 52, 53, 54, -1, 56, - 57, -1, 59, 60, 61, 62, 63, 64, -1, -1, - 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, 79, -1, -1, -1, 83, 84, 85, 86, - 87, 88, -1, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, -1, 102, 103, 104, 105, 106, + -1, -1, -1, 36, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 46, 8, -1, -1, 11, -1, -1, + 53, 15, 16, 17, 18, 19, 20, 21, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 36, -1, -1, -1, -1, 80, -1, -1, + -1, -1, 46, 8, -1, -1, 11, -1, -1, 53, + 15, 16, 17, 18, 19, 20, 21, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 36, -1, -1, -1, -1, 80, -1, -1, -1, + -1, 46, -1, -1, -1, -1, -1, -1, 53, -1, + 8, -1, -1, 11, -1, -1, -1, 15, 16, 17, + 18, 19, 20, 21, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 80, -1, -1, 36, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 46, 8, + -1, -1, 11, 176, -1, 53, 15, 16, 17, 18, + 19, 20, 21, -1, -1, -1, -1, -1, -1, -1, + 193, -1, -1, -1, -1, 198, -1, 36, -1, -1, + -1, -1, 80, -1, -1, -1, -1, 46, -1, -1, + -1, -1, 176, -1, 53, -1, -1, -1, 221, 222, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 193, + -1, -1, -1, 236, 198, -1, -1, -1, -1, -1, + -1, 80, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 176, -1, -1, -1, -1, -1, 221, 222, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 193, -1, + -1, 274, 236, 198, 277, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 291, -1, + -1, 294, -1, -1, -1, -1, 221, 222, 176, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 274, 236, -1, 277, -1, 193, -1, -1, -1, -1, + 198, -1, -1, -1, -1, -1, -1, 291, -1, -1, + 294, -1, -1, -1, -1, -1, -1, 176, -1, -1, + -1, -1, -1, 221, 222, -1, -1, -1, -1, 274, + -1, -1, 277, -1, 193, -1, -1, -1, 236, 198, + -1, -1, -1, -1, -1, -1, 291, -1, -1, 294, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 221, 222, -1, -1, -1, -1, 391, -1, + -1, -1, -1, -1, -1, -1, 274, 236, -1, 277, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 291, -1, -1, 294, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 391, -1, -1, + -1, -1, -1, -1, -1, 274, -1, -1, 277, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 291, -1, -1, 294, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 391, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 498, -1, -1, 501, 502, + 503, -1, 505, 506, 507, 508, 509, 510, -1, -1, + -1, -1, 515, 391, -1, -1, -1, -1, -1, 8, + -1, -1, 11, -1, -1, -1, 15, 16, 17, 18, + 19, 20, 21, -1, 498, -1, -1, 501, 502, 503, + -1, 505, 506, 507, 508, 509, 510, 36, -1, -1, + -1, 515, 391, -1, -1, -1, -1, 46, -1, -1, + -1, -1, -1, -1, 53, -1, -1, -1, -1, -1, + -1, -1, -1, 498, -1, -1, 501, 502, 503, -1, + 505, 506, 507, 508, 509, 510, -1, -1, -1, -1, + 515, 80, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 8, -1, -1, 11, -1, + -1, -1, 15, 16, 17, 18, 19, 20, 21, -1, + 498, -1, -1, 501, 502, 503, -1, 505, 506, 507, + 508, 509, 510, 36, -1, -1, -1, 515, -1, -1, + -1, -1, -1, 46, 8, -1, -1, 11, -1, -1, + 53, 15, 16, 17, 18, 19, 20, 21, -1, 498, + -1, -1, 501, 502, 503, -1, 505, 506, 507, 508, + 509, 510, 36, -1, -1, -1, 515, 80, -1, -1, + -1, -1, 46, 8, -1, -1, 11, 176, -1, 53, + 15, 16, 17, 18, 19, 20, 21, -1, -1, -1, + -1, -1, -1, -1, 193, -1, -1, -1, -1, 198, + -1, 36, -1, -1, -1, -1, 80, -1, -1, -1, + -1, 46, -1, -1, -1, -1, -1, -1, 53, -1, + -1, -1, 221, 222, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 236, -1, -1, + -1, -1, -1, -1, -1, 80, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 8, + -1, -1, 11, 176, -1, -1, 15, 16, 17, 18, + 19, 20, 21, -1, -1, 274, -1, -1, 277, -1, + 193, -1, -1, -1, -1, 198, -1, 36, -1, -1, + -1, -1, 291, -1, -1, 294, -1, 46, -1, -1, + -1, -1, 176, -1, 53, -1, -1, -1, 221, 222, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 193, + -1, -1, -1, 236, 198, -1, -1, -1, -1, -1, + -1, 80, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 176, -1, -1, -1, -1, -1, 221, 222, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 193, -1, + -1, 274, 236, 198, 277, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 291, -1, + -1, 294, -1, -1, -1, -1, 221, 222, -1, -1, + -1, -1, 391, -1, -1, -1, -1, -1, -1, -1, + 274, 236, -1, 277, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 291, -1, -1, + 294, -1, -1, -1, -1, -1, -1, 176, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 274, + -1, -1, 277, -1, 193, -1, -1, -1, -1, 198, + -1, -1, -1, -1, -1, -1, 291, -1, -1, 294, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 221, 222, -1, -1, -1, -1, 391, -1, + -1, -1, -1, -1, -1, -1, -1, 236, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 498, + -1, -1, 501, 502, 503, -1, 505, 506, 507, 508, + 509, 510, -1, -1, -1, -1, 515, 391, -1, -1, + -1, -1, -1, -1, -1, 274, -1, 8, 277, -1, + 11, -1, -1, -1, 15, 16, 17, 18, 19, 20, + 21, -1, 291, -1, -1, 294, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 36, 391, -1, -1, -1, + -1, -1, -1, -1, -1, 46, -1, -1, -1, -1, + -1, -1, 53, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 498, -1, -1, 501, 502, + 503, -1, 505, 506, 507, 508, 509, 510, -1, 80, + -1, -1, 515, -1, -1, -1, -1, -1, -1, 8, + -1, -1, 11, -1, -1, -1, 15, 16, 17, 18, + 19, 20, 21, -1, 498, -1, -1, 501, 502, 503, + -1, 505, 506, 507, 508, 509, 510, 36, -1, -1, + -1, 515, 391, -1, -1, -1, -1, 46, -1, -1, + -1, -1, -1, -1, 53, -1, -1, -1, -1, -1, + -1, -1, -1, 498, -1, -1, 501, 502, 503, -1, + 505, 506, 507, 508, 509, 510, -1, -1, 513, -1, + -1, 80, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 176, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 193, -1, -1, -1, -1, 198, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 221, 222, -1, -1, -1, -1, -1, -1, -1, 498, + -1, -1, 501, 502, 503, 236, 505, 506, 507, 508, + 509, 510, -1, -1, 513, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 176, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 274, 193, -1, 277, -1, -1, 198, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 291, -1, -1, 294, -1, -1, -1, -1, -1, -1, + -1, -1, 221, 222, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 236, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 274, -1, -1, 277, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 291, -1, -1, 294, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 391, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 391, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 498, -1, -1, + 501, 502, 503, -1, 505, 506, 507, 508, 509, 510, + -1, -1, 513, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 3, -1, -1, 498, + -1, -1, 501, 502, 503, -1, 505, 506, 507, 508, + 509, 510, -1, -1, 513, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, + 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, + 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, + 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, + 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, + 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, + 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, - -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, -1, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, -1, -1, -1, 173, 174, 175, -1, - 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, -1, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, -1, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, -1, -1, 223, -1, 225, 226, - 227, 228, 229, 230, -1, -1, 233, -1, 235, -1, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, + 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, + 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, + 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, + 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, + 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, + 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, + 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, + 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, + 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, + 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, + 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, -1, 269, 270, 271, 272, 273, -1, 275, 276, - -1, 278, -1, 280, 281, 282, 283, 284, 285, -1, - 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, 298, -1, 300, -1, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, -1, 322, 323, 324, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, + 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, + 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, + 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, + 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, + 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, + 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, + 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, -1, 362, 363, 364, 365, 366, + 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, -1, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, -1, 392, 393, 394, 395, -1, + 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, + 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, - 417, -1, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, -1, 436, - 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, -1, 463, -1, 465, 466, - 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, + 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, + 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, + 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, + 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, + 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, + 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, + 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 3, -1, 5, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, + 507, -1, -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, -1, -1, -1, -1, 43, 44, -1, 46, 47, 48, -1, 50, 51, @@ -17275,8 +15857,8 @@ static const yytype_int16 yycheck[] = -1, 463, -1, 465, 466, 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 3, -1, 5, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 492, 493, 494, 495, 496, 497, 3, -1, -1, -1, + -1, -1, -1, -1, -1, 507, -1, -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, -1, -1, -1, -1, 43, 44, -1, 46, @@ -17325,1485 +15907,2937 @@ static const yytype_int16 yycheck[] = 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 3, -1, 5, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 497, 3, 4, 5, -1, -1, -1, 9, -1, -1, + 507, -1, -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, - 32, 33, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 43, 44, -1, 46, 47, 48, -1, 50, 51, - 52, 53, 54, -1, 56, 57, -1, 59, 60, 61, + 32, 33, -1, -1, -1, 37, -1, -1, -1, -1, + 42, 43, 44, -1, 46, 47, 48, 49, 50, 51, + 52, 53, 54, 55, 56, 57, -1, 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, 69, 70, 71, - 72, 73, -1, 75, 76, 77, 78, 79, -1, -1, - -1, 83, 84, 85, 86, 87, 88, -1, 90, 91, - 92, -1, 94, 95, 96, 97, 98, 99, -1, -1, + 72, 73, -1, 75, 76, 77, 78, 79, -1, 81, + -1, 83, 84, 85, 86, 87, 88, 89, 90, 91, + 92, -1, 94, 95, 96, 97, 98, 99, -1, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, -1, 160, -1, - 162, 163, 164, 165, -1, 167, -1, 169, -1, -1, - -1, 173, 174, 175, -1, 177, -1, 179, -1, 181, + 152, 153, 154, 155, 156, 157, 158, 159, 160, -1, + 162, 163, 164, 165, -1, 167, -1, 169, 170, -1, + 172, 173, 174, 175, 176, 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, - 192, -1, 194, 195, 196, 197, -1, 199, 200, 201, - 202, 203, 204, 205, -1, 207, -1, 209, 210, 211, - 212, 213, 214, 215, 216, -1, 218, -1, 220, -1, - -1, 223, -1, 225, 226, 227, 228, 229, 230, -1, - -1, 233, -1, 235, -1, -1, 238, 239, 240, 241, + 192, 193, 194, 195, 196, 197, -1, 199, 200, 201, + 202, 203, 204, 205, -1, 207, 208, 209, 210, 211, + 212, 213, 214, 215, 216, -1, 218, -1, 220, 221, + 222, 223, 224, 225, 226, 227, 228, 229, 230, -1, + -1, 233, 234, 235, 236, -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, -1, 269, 270, 271, - 272, 273, -1, 275, 276, -1, 278, -1, 280, 281, - 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, - 292, 293, -1, -1, 296, 297, 298, -1, 300, -1, + 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, + 272, 273, -1, 275, 276, 277, 278, -1, 280, 281, + 282, 283, 284, 285, -1, 287, 288, 289, -1, 291, + 292, 293, -1, -1, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, -1, -1, -1, -1, 318, 319, 320, -1, + 312, 313, -1, -1, -1, -1, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, -1, 359, 360, -1, + 352, 353, 354, 355, 356, 357, -1, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, -1, -1, 381, - 382, 383, 384, 385, 386, 387, 388, 389, -1, -1, + 372, 373, 374, 375, 376, 377, 378, -1, 380, 381, + 382, 383, 384, 385, 386, 387, 388, 389, -1, 391, 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - -1, -1, 414, 415, -1, 417, -1, 419, 420, 421, + -1, -1, 414, 415, -1, 417, 418, 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - 432, 433, 434, -1, 436, 437, 438, 439, 440, 441, + 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, - -1, 463, -1, 465, 466, 467, 468, 469, 470, 471, + -1, 463, 464, 465, 466, 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 3, -1, 5, -1, + 492, 493, 494, 495, 496, 497, -1, -1, 8, -1, + -1, 11, -1, 505, 506, 15, 16, 17, 18, 19, + 20, 21, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 36, -1, -1, -1, + -1, 41, -1, -1, -1, -1, 46, 8, -1, -1, + 11, -1, -1, 53, 15, 16, 17, 18, 19, 20, + 21, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 36, -1, -1, -1, -1, + 80, -1, -1, -1, -1, 46, 8, -1, -1, 11, + -1, -1, 53, 15, 16, 17, 18, 19, 20, 21, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 36, -1, -1, -1, -1, 80, + -1, -1, -1, -1, 46, -1, 126, -1, -1, -1, + -1, 53, -1, -1, 8, -1, -1, 11, -1, -1, + -1, 15, 16, 17, 18, 19, 20, 21, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 80, -1, + -1, -1, 36, -1, -1, -1, 40, -1, -1, -1, + -1, -1, 46, -1, 8, -1, 176, 11, -1, 53, + -1, 15, 16, 17, 18, 19, 20, 21, -1, -1, + -1, -1, -1, 193, -1, -1, -1, -1, 198, -1, + -1, -1, 36, -1, -1, 166, 80, -1, -1, -1, + 171, -1, 46, -1, -1, 176, -1, -1, -1, 53, + -1, 221, 222, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 193, -1, -1, -1, 236, 198, -1, -1, + -1, -1, -1, 165, -1, -1, 80, -1, -1, -1, + -1, -1, -1, -1, 176, -1, -1, -1, -1, -1, + 221, 222, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 193, -1, -1, 274, 236, 198, 277, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 291, -1, -1, 294, -1, -1, -1, -1, 221, + 222, -1, 176, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 274, 236, -1, 277, -1, -1, 193, + -1, -1, -1, -1, 198, -1, -1, -1, -1, -1, + 291, -1, -1, 294, -1, -1, -1, -1, -1, -1, + -1, -1, 176, -1, -1, -1, -1, 221, 222, -1, + -1, -1, 274, -1, -1, 277, -1, -1, -1, 193, + -1, -1, 236, -1, 198, -1, -1, -1, -1, 291, + -1, -1, 294, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 221, 222, -1, + -1, 391, -1, -1, -1, -1, -1, -1, -1, -1, + 274, 323, 236, 277, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 291, -1, -1, + 294, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 391, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 274, -1, -1, 277, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 454, -1, -1, 291, -1, -1, + 294, -1, -1, -1, -1, -1, -1, -1, -1, 391, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 317, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 498, -1, + -1, 501, 502, 503, -1, 505, 506, 507, 508, 509, + 510, -1, -1, -1, -1, -1, -1, 391, -1, -1, + -1, -1, 8, -1, -1, 11, -1, -1, -1, 15, + 16, 17, 18, 19, 20, 21, -1, 498, -1, -1, + 501, 502, 503, -1, 505, 506, 507, 508, 509, 510, + 36, -1, -1, -1, 40, -1, -1, 391, -1, -1, + 46, -1, -1, -1, -1, -1, -1, 53, -1, -1, + -1, -1, -1, -1, -1, -1, 498, -1, -1, 501, + 502, 503, -1, 505, 506, 507, 508, 509, 510, -1, + -1, -1, -1, -1, 80, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 8, + -1, -1, 11, -1, -1, -1, 15, 16, 17, 18, + 19, 20, 21, -1, 498, -1, -1, 501, 502, 503, + -1, 505, 506, 507, 508, 509, 510, 36, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 46, -1, -1, + -1, -1, -1, -1, 53, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 498, -1, -1, 501, 502, 503, + -1, 505, 506, 507, 508, 509, 510, -1, -1, -1, + -1, 80, -1, -1, -1, -1, 8, -1, -1, 11, + 176, -1, -1, 15, 16, 17, 18, 19, 20, 21, + -1, -1, -1, -1, -1, -1, -1, 193, -1, -1, + -1, -1, 198, -1, 36, -1, -1, -1, 40, -1, + -1, -1, -1, -1, 46, -1, -1, -1, -1, -1, + -1, 53, -1, -1, -1, 221, 222, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 236, -1, -1, -1, -1, -1, -1, -1, 80, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 171, 8, -1, -1, 11, 176, -1, -1, + 15, 16, 17, 18, 19, 20, 21, -1, 274, -1, + -1, 277, -1, -1, 193, -1, -1, -1, -1, 198, + -1, 36, -1, -1, -1, 291, -1, -1, 294, -1, + -1, 46, -1, -1, -1, -1, -1, -1, 53, -1, + -1, -1, 221, 222, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 236, -1, -1, + -1, -1, -1, -1, -1, 80, -1, -1, -1, -1, + -1, -1, -1, -1, 176, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 193, -1, -1, -1, 274, 198, -1, 277, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 291, -1, -1, 294, -1, -1, -1, 221, + 222, -1, -1, -1, -1, 391, -1, -1, 8, -1, + -1, 11, -1, -1, 236, 15, 16, 17, 18, 19, + 20, 21, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 166, -1, -1, -1, -1, 36, -1, -1, -1, + -1, 176, -1, -1, -1, -1, 46, -1, -1, -1, + -1, -1, 274, 53, -1, 277, -1, -1, 193, -1, + -1, -1, -1, 198, -1, -1, -1, -1, -1, 291, + -1, -1, 294, -1, -1, -1, -1, -1, -1, -1, + 80, -1, -1, -1, -1, -1, 221, 222, -1, -1, + -1, -1, 391, -1, -1, -1, -1, -1, -1, -1, + -1, 236, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 498, -1, -1, 501, 502, 503, -1, 505, + 506, 507, 508, 509, 510, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 274, + -1, -1, 277, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 291, -1, -1, 294, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 391, + -1, -1, -1, -1, -1, -1, 176, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 193, -1, -1, -1, -1, 198, 498, + -1, -1, 501, 502, 503, -1, 505, 506, 507, 508, + 509, 510, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 221, 222, -1, -1, -1, -1, -1, -1, -1, + 8, -1, -1, 11, -1, -1, 236, 15, 16, 17, + 18, 19, 20, 21, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 391, -1, 36, -1, + -1, -1, 40, -1, -1, -1, -1, -1, 46, -1, + -1, -1, -1, -1, 274, 53, 498, 277, -1, 501, + 502, 503, -1, 505, 506, 507, 508, 509, 510, -1, + -1, 291, -1, -1, 294, -1, -1, -1, -1, -1, + -1, -1, 80, -1, 8, -1, -1, 11, -1, -1, + -1, 15, 16, 17, 18, 19, 20, 21, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 36, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 46, -1, -1, -1, -1, -1, -1, 53, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 498, -1, -1, 501, 502, 503, -1, + 505, 506, 507, 508, 509, 510, 80, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 391, -1, 8, -1, -1, 11, -1, 176, -1, + 15, 16, 17, 18, 19, 20, 21, -1, -1, -1, + -1, -1, -1, -1, -1, 193, -1, -1, -1, -1, + 198, 36, -1, -1, 424, -1, -1, -1, -1, -1, + -1, 46, -1, -1, -1, -1, -1, -1, 53, -1, + -1, -1, -1, 221, 222, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 236, -1, + -1, -1, -1, -1, -1, 80, -1, -1, -1, -1, + -1, -1, 176, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 193, + -1, -1, -1, -1, 198, -1, 274, -1, 498, 277, + -1, 501, 502, 503, -1, 505, 506, 507, 508, 509, + 510, -1, -1, 291, -1, -1, 294, 221, 222, -1, + -1, -1, -1, -1, -1, -1, 8, -1, -1, 11, + -1, -1, 236, 15, 16, 17, 18, 19, 20, 21, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 36, -1, -1, -1, -1, -1, + -1, 176, -1, -1, 46, -1, -1, -1, -1, -1, + 274, 53, -1, 277, -1, -1, -1, -1, 193, -1, + -1, -1, -1, 198, -1, -1, -1, 291, -1, -1, + 294, -1, -1, -1, -1, -1, -1, -1, 80, -1, + -1, -1, -1, -1, -1, -1, 221, 222, -1, -1, + -1, -1, -1, 391, -1, -1, -1, -1, -1, -1, + -1, 236, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 8, -1, -1, 11, -1, -1, -1, 15, + 16, -1, -1, 19, 20, 21, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 274, + 36, -1, 277, -1, -1, -1, -1, -1, -1, -1, + 46, -1, -1, -1, -1, -1, 291, 53, -1, 294, + -1, -1, -1, -1, -1, -1, -1, 391, -1, -1, + -1, -1, -1, -1, 176, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 80, -1, -1, -1, -1, -1, + -1, 193, -1, -1, -1, -1, 198, -1, -1, -1, + 498, -1, -1, 501, 502, 503, -1, 505, 506, 507, + 508, 509, 510, -1, -1, 8, -1, -1, 11, 221, + 222, -1, 15, 16, -1, -1, 19, 20, 21, -1, + -1, -1, -1, -1, 236, -1, -1, -1, -1, -1, + -1, -1, -1, 36, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 46, -1, 8, 391, -1, 11, -1, + 53, -1, 15, 16, -1, -1, 19, 20, 21, -1, + -1, -1, 274, -1, 498, 277, -1, 501, 502, 503, + 176, 505, 506, 507, 508, 509, 510, 80, -1, 291, + -1, -1, 294, 46, -1, -1, -1, 193, -1, -1, + 53, -1, 198, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 221, 222, 80, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 236, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 498, -1, -1, 501, 502, 503, -1, + 505, 506, 507, 508, 509, 510, -1, -1, 274, -1, + -1, 277, -1, 176, -1, -1, -1, -1, -1, 391, + -1, -1, -1, -1, -1, 291, -1, -1, 294, -1, + 193, -1, -1, -1, -1, 198, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 176, -1, -1, -1, -1, 221, 222, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 193, -1, -1, 236, -1, 198, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 221, 222, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 274, -1, 236, 277, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 391, 498, -1, 291, 501, + 502, 503, -1, 505, 506, 507, 508, 509, 510, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 274, -1, -1, 277, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 291, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 391, -1, + -1, -1, 498, -1, -1, 501, 502, 503, -1, 505, + 506, 507, 508, 509, 510, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 391, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 498, -1, -1, 501, 502, + 503, -1, 505, 506, 507, 508, 509, 510, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 3, + -1, 5, -1, -1, -1, 498, -1, -1, 501, 502, + 503, -1, 505, 506, 507, 508, 509, 510, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, + 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, + 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, + 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, + 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, + 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, + 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, + 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, + 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, + 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, + 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, + 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, + 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, + 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, + 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, + 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, + 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, + 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, + 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, + 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, + 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, + 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, + 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, + 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, + 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, + 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, + 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, + 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, + 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, + 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, + 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, + 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, + 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, + 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, + 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, + 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, + 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, + 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, + 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, + 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, + 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, + 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, + 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, + 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, + 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, + 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, + 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, + 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, + 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, + 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, + 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, + 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, + 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, + 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, + 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, + 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, + 449, 450, 451, 452, 453, 454, 455, 456, 457, 458, + 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, + 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, + 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, + 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, + 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, + 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, + 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, + 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, + 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, + 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, + 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, + 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, + 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, + 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, + 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, + 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, + 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, + 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, + 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, + 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, + 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, + 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, + 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, + 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, + 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, + 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, + 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, + 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, + 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, + 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, + 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, + 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, + 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, + 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, + 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, + 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, + 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, + 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, + 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, + 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, + 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, + 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, + 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, + 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, + 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, + 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, + 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, + 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, + 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, + 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, + 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, + 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, + 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, + 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, + 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, + 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, + 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, + 449, 450, 451, 452, 453, 454, 455, 456, 457, 458, + 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, + 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, + 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, + 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, + 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, + 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, + 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, + 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, + 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, + 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, + 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, + 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, + 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, + 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, + 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, + 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, + 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, + 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, + 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, + 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, + 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, + 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, + 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, + 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, + 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, + 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, + 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, + 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, + 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, + 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 3, -1, 5, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, 37, -1, + -1, -1, -1, 42, 43, 44, -1, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, 81, -1, 83, 84, 85, 86, 87, 88, + 89, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, 101, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + 159, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, 170, -1, 172, 173, 174, 175, 176, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, 193, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, 208, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, 221, 222, 223, 224, 225, 226, 227, 228, + 229, 230, -1, -1, 233, 234, 235, 236, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, + 269, 270, 271, 272, 273, -1, 275, 276, 277, 278, + -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, + 289, -1, 291, 292, 293, -1, -1, 296, 297, 298, + 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, 321, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, 380, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, 391, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, 418, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, 435, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, 464, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 43, 44, -1, 46, - 47, 48, -1, 50, 51, 52, 53, 54, -1, 56, - 57, -1, 59, 60, 61, 62, 63, 64, -1, -1, - 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, 79, -1, -1, -1, 83, 84, 85, 86, - 87, 88, -1, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, -1, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, - -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, -1, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, -1, -1, -1, 173, 174, 175, -1, - 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, -1, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, -1, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, -1, -1, 223, -1, 225, 226, - 227, 228, 229, 230, -1, -1, 233, -1, 235, -1, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, -1, 269, 270, 271, 272, 273, -1, 275, 276, - -1, 278, -1, 280, 281, 282, 283, 284, 285, -1, - 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, 298, -1, 300, -1, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, -1, 322, 323, 324, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, -1, 362, 363, 364, 365, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, -1, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, -1, 392, 393, 394, 395, -1, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, - 417, -1, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, -1, 436, - 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, -1, 463, -1, 465, 466, - 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, - 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, - 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 3, -1, 5, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, + 34, 35, -1, 37, -1, -1, -1, -1, 42, 43, + 44, -1, 46, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 57, -1, 59, 60, 61, 62, 63, + 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, 81, -1, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, 101, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, 170, -1, 172, 173, + 174, 175, 176, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, 193, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, 208, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, -1, -1, 233, + 234, 235, 236, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, + -1, 275, 276, 277, 278, -1, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, 299, 300, 301, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, 320, 321, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, 380, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, 391, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, -1, 417, 418, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + 464, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, - 32, 33, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 43, 44, -1, 46, 47, 48, -1, 50, 51, - 52, 53, 54, -1, 56, 57, -1, 59, 60, 61, - 62, 63, 64, -1, -1, 67, 68, 69, 70, 71, - 72, 73, -1, 75, 76, 77, 78, 79, -1, -1, - -1, 83, 84, 85, 86, 87, 88, -1, 90, 91, - 92, -1, 94, 95, 96, 97, 98, 99, -1, -1, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, -1, 118, -1, 120, 121, - 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, - -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, - 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, -1, 160, -1, - 162, 163, 164, 165, -1, 167, -1, 169, -1, -1, - -1, 173, 174, 175, -1, 177, -1, 179, -1, 181, - 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, - 192, -1, 194, 195, 196, 197, -1, 199, 200, 201, - 202, 203, 204, 205, -1, 207, -1, 209, 210, 211, - 212, 213, 214, 215, 216, -1, 218, -1, 220, -1, - -1, 223, -1, 225, 226, 227, 228, 229, 230, -1, - -1, 233, -1, 235, -1, -1, 238, 239, 240, 241, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, -1, 269, 270, 271, - 272, 273, -1, 275, 276, -1, 278, -1, 280, 281, - 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, - 292, 293, -1, -1, 296, 297, 298, -1, 300, -1, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, -1, -1, -1, -1, 318, 319, 320, -1, - 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, - 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, - 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, -1, 359, 360, -1, - 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, -1, -1, 381, - 382, 383, 384, 385, 386, 387, 388, 389, -1, -1, - 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - -1, -1, 414, 415, -1, 417, -1, 419, 420, 421, - 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - 432, 433, 434, -1, 436, 437, 438, 439, 440, 441, - 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, - 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, - -1, 463, -1, 465, 466, 467, 468, 469, 470, 471, - -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, - 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 3, -1, 5, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, + -1, -1, -1, 42, 43, 44, -1, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, 66, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, 81, -1, 83, 84, 85, 86, 87, 88, + 89, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, 101, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, 170, -1, 172, 173, 174, 175, 176, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, 193, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, 208, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, 221, 222, 223, 224, 225, 226, 227, 228, + 229, 230, -1, -1, 233, 234, 235, 236, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, + 269, 270, 271, 272, 273, -1, 275, 276, 277, 278, + -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, + 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, 321, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, 391, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, 418, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, 464, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, + -1, -1, -1, -1, -1, -1, -1, -1, 42, 43, + 44, -1, 46, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 57, -1, 59, 60, 61, 62, 63, + 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, 81, -1, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, 101, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, 170, -1, 172, 173, + 174, 175, 176, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, 193, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, 208, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, -1, -1, 233, + 234, 235, 236, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, + -1, 275, 276, 277, 278, -1, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, 299, 300, 301, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, 320, 321, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, 391, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, -1, 417, 418, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + 464, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 43, 44, -1, 46, - 47, 48, -1, 50, 51, 52, 53, 54, -1, 56, - 57, -1, 59, 60, 61, 62, 63, 64, -1, -1, - 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, 79, -1, -1, -1, 83, 84, 85, 86, - 87, 88, -1, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, -1, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, - -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, -1, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, -1, -1, -1, 173, 174, 175, -1, - 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, -1, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, -1, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, -1, -1, 223, -1, 225, 226, - 227, 228, 229, 230, -1, -1, 233, -1, 235, -1, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, -1, 269, 270, 271, 272, 273, -1, 275, 276, - -1, 278, -1, 280, 281, 282, 283, 284, 285, -1, - 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, 298, -1, 300, -1, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, -1, 322, 323, 324, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, -1, 362, 363, 364, 365, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, -1, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, -1, 392, 393, 394, 395, -1, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, - 417, -1, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, -1, 436, - 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, -1, 463, -1, 465, 466, - 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, - 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, - 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 43, 44, -1, 46, 47, 48, + -1, 50, 51, 52, 53, 54, -1, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, + 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, + 79, -1, -1, 82, 83, 84, 85, 86, 87, 88, + -1, 90, 91, 92, 93, 94, 95, 96, 97, 98, + 99, -1, -1, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + -1, 160, -1, 162, 163, 164, 165, -1, 167, 168, + 169, -1, -1, -1, 173, 174, 175, -1, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, -1, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, -1, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, -1, -1, 223, -1, 225, 226, 227, 228, + 229, 230, -1, -1, 233, -1, 235, -1, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, -1, + 269, 270, 271, 272, 273, -1, 275, 276, -1, 278, + -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, + -1, 300, -1, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, -1, 322, 323, 324, 325, 326, 327, 328, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, -1, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, -1, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, -1, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, + 439, 440, 441, 442, 443, -1, 445, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, -1, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, - 32, 33, -1, -1, -1, -1, -1, -1, 40, -1, - -1, 43, 44, -1, 46, 47, 48, -1, 50, 51, - 52, 53, 54, -1, 56, 57, -1, 59, 60, 61, - 62, 63, 64, -1, -1, 67, 68, 69, 70, 71, - 72, 73, -1, 75, 76, 77, 78, 79, -1, -1, - -1, 83, 84, 85, 86, 87, 88, -1, 90, 91, - 92, -1, 94, 95, 96, 97, 98, 99, -1, -1, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, -1, 118, -1, 120, 121, - 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, - -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, - 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, -1, 160, -1, - 162, 163, 164, 165, -1, 167, -1, 169, -1, -1, - -1, 173, 174, 175, -1, 177, -1, 179, -1, 181, - 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, - 192, -1, 194, 195, 196, 197, -1, 199, 200, 201, - 202, 203, 204, 205, -1, 207, -1, 209, 210, 211, - 212, 213, 214, 215, 216, -1, 218, -1, 220, -1, - -1, 223, -1, 225, 226, 227, 228, 229, 230, -1, - -1, 233, -1, 235, -1, -1, 238, 239, 240, 241, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, -1, 269, 270, 271, - 272, 273, -1, 275, 276, -1, 278, -1, 280, 281, - 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, - 292, 293, -1, -1, 296, 297, 298, -1, 300, -1, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, -1, -1, -1, -1, 318, 319, 320, -1, - 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, - 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, - 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, -1, 359, 360, -1, - 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, -1, -1, 381, - 382, 383, 384, 385, 386, 387, 388, 389, -1, -1, - 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - -1, -1, 414, 415, -1, 417, -1, 419, 420, 421, - 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - 432, 433, 434, -1, 436, 437, 438, 439, 440, 441, - 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, - 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, - -1, 463, -1, 465, 466, 467, 468, 469, 470, 471, - -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, - 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 3, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 43, + 44, -1, 46, 47, 48, -1, 50, 51, 52, 53, + 54, -1, 56, 57, -1, 59, 60, 61, 62, 63, + 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, + 74, 75, 76, 77, 78, 79, -1, -1, -1, 83, + 84, 85, 86, 87, 88, -1, 90, 91, 92, 93, + 94, 95, 96, 97, 98, 99, -1, -1, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, + 164, 165, -1, 167, 168, 169, -1, -1, -1, 173, + 174, 175, -1, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, -1, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, -1, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, -1, -1, 223, + -1, 225, 226, 227, 228, 229, 230, -1, -1, 233, + -1, 235, 236, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, -1, 269, 270, 271, 272, 273, + -1, 275, 276, -1, 278, -1, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, -1, 300, -1, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, 320, -1, 322, 323, + 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, -1, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, -1, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, -1, 417, -1, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, + -1, 445, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + -1, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 3, -1, 5, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, 34, 35, -1, -1, -1, + -1, -1, -1, -1, 43, 44, -1, 46, 47, 48, + -1, 50, 51, 52, 53, 54, -1, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, -1, -1, 83, 84, 85, 86, 87, 88, + -1, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, -1, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, -1, -1, -1, 173, 174, 175, -1, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, -1, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, -1, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, -1, -1, 223, -1, 225, 226, 227, 228, + 229, 230, -1, -1, 233, -1, 235, -1, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, -1, + 269, 270, 271, 272, 273, -1, 275, 276, -1, 278, + -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, + -1, 300, -1, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, -1, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, -1, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, -1, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, -1, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, -1, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, + -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 43, + 44, -1, 46, 47, 48, -1, 50, 51, 52, 53, + 54, -1, 56, 57, -1, 59, 60, 61, 62, 63, + 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, -1, -1, 83, + 84, 85, 86, 87, 88, -1, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, -1, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, -1, -1, -1, 173, + 174, 175, -1, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, -1, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, -1, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, -1, -1, 223, + -1, 225, 226, 227, 228, 229, 230, -1, -1, 233, + -1, 235, -1, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, -1, 269, 270, 271, 272, 273, + -1, 275, 276, -1, 278, -1, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, -1, 300, -1, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, 320, -1, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, -1, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, -1, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, -1, 417, -1, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + -1, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 3, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - -1, -1, -1, 40, -1, -1, 43, 44, -1, 46, - 47, 48, -1, 50, 51, 52, 53, 54, -1, 56, - 57, -1, 59, 60, 61, 62, 63, 64, -1, -1, - 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, 79, -1, -1, -1, 83, 84, 85, 86, - 87, 88, -1, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, -1, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, - -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, -1, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, -1, -1, -1, 173, 174, 175, -1, - 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, -1, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, -1, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, -1, -1, 223, -1, 225, 226, - 227, 228, 229, 230, -1, -1, 233, -1, 235, -1, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, -1, 269, 270, 271, 272, 273, -1, 275, 276, - -1, 278, -1, 280, 281, 282, 283, 284, 285, -1, - 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, 298, -1, 300, -1, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, -1, 322, 323, 324, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, -1, 362, 363, 364, 365, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, -1, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, -1, 392, 393, 394, 395, -1, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, - 417, -1, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, -1, 436, - 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, -1, 463, -1, 465, 466, - 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, - 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, - 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 43, 44, -1, 46, 47, 48, + -1, 50, 51, 52, 53, 54, -1, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, -1, -1, 83, 84, 85, 86, 87, 88, + -1, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, -1, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, -1, -1, -1, 173, 174, 175, -1, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, -1, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, -1, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, -1, -1, 223, -1, 225, 226, 227, 228, + 229, 230, -1, -1, 233, -1, 235, -1, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, -1, + 269, 270, 271, 272, 273, -1, 275, 276, -1, 278, + -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, 290, 291, 292, 293, -1, -1, 296, 297, 298, + -1, 300, -1, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, -1, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, -1, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, -1, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, -1, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, -1, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, + -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 43, + 44, -1, 46, 47, 48, -1, 50, 51, 52, 53, + 54, -1, 56, 57, -1, 59, 60, 61, 62, 63, + 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, -1, -1, 83, + 84, 85, 86, 87, 88, -1, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, -1, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, -1, -1, -1, 173, + 174, 175, -1, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, -1, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, -1, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, -1, -1, 223, + -1, 225, 226, 227, 228, 229, 230, -1, -1, 233, + -1, 235, -1, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, -1, 269, 270, 271, 272, 273, + -1, 275, 276, -1, 278, -1, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, 290, 291, 292, 293, + -1, -1, 296, 297, 298, -1, 300, -1, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, 320, -1, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, -1, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, -1, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, -1, 417, -1, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + -1, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 3, -1, 5, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, 30, 31, 32, 33, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 43, 44, -1, 46, 47, 48, + -1, 50, 51, 52, 53, 54, -1, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, -1, -1, 83, 84, 85, 86, 87, 88, + -1, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, -1, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, -1, -1, -1, 173, 174, 175, -1, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, -1, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, -1, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, -1, -1, 223, -1, 225, 226, 227, 228, + 229, 230, -1, -1, 233, -1, 235, -1, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, -1, + 269, 270, 271, 272, 273, -1, 275, 276, -1, 278, + -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, + -1, 300, -1, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, -1, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, -1, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, -1, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, -1, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, -1, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, + 4, -1, -1, -1, -1, 9, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 43, + 44, -1, 46, 47, 48, -1, 50, 51, 52, 53, + 54, -1, 56, 57, -1, 59, 60, 61, 62, 63, + 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, -1, -1, 83, + 84, 85, 86, 87, 88, -1, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, -1, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, -1, -1, -1, 173, + 174, 175, -1, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, -1, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, -1, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, -1, -1, 223, + -1, 225, 226, 227, 228, 229, 230, -1, -1, 233, + -1, 235, -1, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, -1, 269, 270, 271, 272, 273, + -1, 275, 276, -1, 278, -1, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, -1, 300, -1, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, 320, -1, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, -1, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, -1, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, -1, 417, -1, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + -1, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 3, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 43, 44, -1, 46, 47, 48, -1, 50, 51, - 52, 53, 54, -1, 56, 57, -1, 59, 60, 61, - 62, 63, 64, -1, -1, 67, 68, 69, 70, 71, - 72, 73, -1, 75, 76, 77, 78, 79, -1, -1, - -1, 83, 84, 85, 86, 87, 88, -1, 90, 91, - 92, -1, 94, 95, 96, 97, 98, 99, -1, -1, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, -1, 118, -1, 120, 121, - 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, - -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, - 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, -1, 160, -1, - 162, 163, 164, 165, -1, 167, -1, 169, -1, -1, - -1, 173, 174, 175, -1, 177, -1, 179, -1, 181, - 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, - 192, -1, 194, 195, 196, 197, -1, 199, 200, 201, - 202, 203, 204, 205, -1, 207, -1, 209, 210, 211, - 212, 213, 214, 215, 216, -1, 218, -1, 220, -1, - -1, 223, -1, 225, 226, 227, 228, 229, 230, -1, - -1, 233, -1, 235, -1, -1, 238, 239, 240, 241, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, -1, 269, 270, 271, - 272, 273, -1, 275, 276, -1, 278, -1, 280, 281, - 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, - 292, 293, -1, -1, 296, 297, 298, -1, 300, -1, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, -1, -1, -1, -1, 318, 319, 320, -1, - 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, - 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, - 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, -1, 359, 360, -1, - 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, -1, -1, 381, - 382, 383, 384, 385, 386, 387, 388, 389, -1, -1, - 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - -1, -1, 414, 415, -1, 417, -1, 419, 420, 421, - 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - 432, 433, 434, -1, 436, 437, 438, 439, 440, 441, - 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, - 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, - -1, 463, -1, 465, 466, 467, 468, 469, 470, 471, - -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, - 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 3, -1, 5, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 43, 44, -1, 46, 47, 48, + -1, 50, 51, 52, 53, 54, -1, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, -1, -1, 83, 84, 85, 86, 87, 88, + -1, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, -1, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, -1, -1, -1, 173, 174, 175, -1, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, -1, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, -1, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, -1, -1, 223, -1, 225, 226, 227, 228, + 229, 230, -1, -1, 233, -1, 235, -1, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, -1, + 269, 270, 271, 272, 273, -1, 275, 276, -1, 278, + -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, + -1, 300, -1, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, -1, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, -1, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, -1, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, 416, 417, -1, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, -1, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, + -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 43, + 44, -1, 46, 47, 48, -1, 50, 51, 52, 53, + 54, -1, 56, 57, -1, 59, 60, 61, 62, 63, + 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, -1, -1, 83, + 84, 85, 86, 87, 88, -1, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, -1, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, -1, -1, -1, 173, + 174, 175, -1, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, -1, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, -1, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, -1, -1, 223, + -1, 225, 226, 227, 228, 229, 230, -1, -1, 233, + -1, 235, -1, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, -1, 269, 270, 271, 272, 273, + -1, 275, 276, -1, 278, -1, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, -1, 300, -1, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, 320, -1, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, -1, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, -1, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, -1, 417, -1, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + -1, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 3, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 43, 44, -1, 46, - 47, 48, -1, 50, 51, 52, 53, 54, -1, 56, - 57, -1, 59, 60, 61, 62, 63, 64, -1, -1, - 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, 79, -1, -1, -1, 83, 84, 85, 86, - 87, 88, -1, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, -1, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, - -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, -1, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, -1, -1, -1, 173, 174, 175, -1, - 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, -1, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, -1, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, -1, -1, 223, -1, 225, 226, - 227, 228, 229, 230, -1, -1, 233, -1, 235, -1, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, -1, 269, 270, 271, 272, 273, -1, 275, 276, - -1, 278, -1, 280, 281, 282, 283, 284, 285, -1, - 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, 298, -1, 300, -1, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, -1, 322, 323, 324, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, -1, 362, 363, 364, 365, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, -1, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, -1, 392, 393, 394, 395, -1, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, - 417, -1, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, -1, 436, - 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, -1, 463, -1, 465, 466, - 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, - 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, - 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 3, -1, 5, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 43, 44, -1, 46, 47, 48, + -1, 50, 51, 52, 53, 54, -1, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, -1, -1, 83, 84, 85, 86, 87, 88, + -1, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, -1, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, -1, -1, -1, 173, 174, 175, -1, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, -1, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, -1, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, -1, -1, 223, -1, 225, 226, 227, 228, + 229, 230, -1, -1, 233, -1, 235, -1, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, -1, + 269, 270, 271, 272, 273, -1, 275, 276, -1, 278, + -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, + -1, 300, -1, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, -1, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, -1, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, -1, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, -1, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, -1, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 43, + 44, -1, 46, 47, 48, -1, 50, 51, 52, 53, + 54, -1, 56, 57, -1, 59, 60, 61, 62, 63, + 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, -1, -1, 83, + 84, 85, 86, 87, 88, -1, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, -1, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, -1, -1, -1, 173, + 174, 175, -1, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, -1, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, -1, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, -1, -1, 223, + -1, 225, 226, 227, 228, 229, 230, -1, -1, 233, + -1, 235, -1, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, -1, 269, 270, 271, 272, 273, + -1, 275, 276, -1, 278, -1, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, -1, 300, -1, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, 320, -1, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, -1, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, -1, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, -1, 417, -1, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + -1, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 3, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, - 32, 33, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 43, 44, -1, 46, 47, 48, -1, 50, 51, - 52, 53, 54, -1, 56, 57, -1, 59, 60, 61, - 62, 63, 64, -1, -1, 67, 68, 69, 70, 71, - 72, 73, -1, 75, 76, 77, 78, 79, -1, -1, - -1, 83, 84, 85, 86, 87, 88, -1, 90, 91, - 92, -1, 94, 95, 96, 97, 98, 99, -1, -1, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, -1, 118, -1, 120, 121, - 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, - -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, - 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, -1, 160, -1, - 162, 163, 164, 165, -1, 167, -1, 169, -1, -1, - -1, 173, 174, 175, -1, 177, -1, 179, -1, 181, - 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, - 192, -1, 194, 195, 196, 197, -1, 199, 200, 201, - 202, 203, 204, 205, -1, 207, -1, 209, 210, 211, - 212, 213, 214, 215, 216, -1, 218, -1, 220, -1, - -1, 223, -1, 225, 226, 227, 228, 229, 230, -1, - -1, 233, -1, 235, -1, -1, 238, 239, 240, 241, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, -1, 269, 270, 271, - 272, 273, -1, 275, 276, -1, 278, -1, 280, 281, - 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, - 292, 293, -1, -1, 296, 297, 298, -1, 300, -1, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, -1, -1, -1, -1, 318, 319, 320, -1, - 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, - 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, - 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, -1, 359, 360, -1, - 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, -1, -1, 381, - 382, 383, 384, 385, 386, 387, 388, 389, -1, -1, - 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - -1, -1, 414, 415, -1, 417, -1, 419, 420, 421, - 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - 432, 433, 434, -1, 436, 437, 438, 439, 440, 441, - 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, - 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, - -1, 463, -1, 465, 466, 467, 468, 469, 470, 471, - -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, - 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 3, -1, 5, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 43, 44, -1, 46, 47, 48, + -1, 50, 51, 52, 53, 54, -1, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, -1, -1, 83, 84, 85, 86, 87, 88, + -1, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, -1, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, -1, -1, -1, 173, 174, 175, -1, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, -1, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, -1, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, -1, -1, 223, -1, 225, 226, 227, 228, + 229, 230, -1, -1, 233, -1, 235, -1, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, -1, + 269, 270, 271, 272, 273, -1, 275, 276, -1, 278, + -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, + -1, 300, -1, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, -1, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, -1, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, -1, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, -1, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, -1, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, + -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 43, + 44, -1, 46, 47, 48, -1, 50, 51, 52, 53, + 54, -1, 56, 57, -1, 59, 60, 61, 62, 63, + 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, -1, -1, 83, + 84, 85, 86, 87, 88, -1, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, -1, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, -1, -1, -1, 173, + 174, 175, -1, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, -1, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, -1, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, -1, -1, 223, + -1, 225, 226, 227, 228, 229, 230, -1, -1, 233, + -1, 235, -1, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, -1, 269, 270, 271, 272, 273, + -1, 275, 276, -1, 278, -1, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, -1, 300, -1, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, 320, -1, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, -1, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, -1, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, -1, 417, -1, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + -1, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 3, -1, 5, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 43, 44, -1, 46, 47, 48, + -1, 50, 51, 52, 53, 54, -1, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, -1, -1, 83, 84, 85, 86, 87, 88, + -1, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, -1, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, -1, -1, -1, 173, 174, 175, -1, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, -1, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, -1, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, -1, -1, 223, -1, 225, 226, 227, 228, + 229, 230, -1, -1, 233, -1, 235, -1, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, -1, + 269, 270, 271, 272, 273, -1, 275, 276, -1, 278, + -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, + -1, 300, -1, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, -1, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, -1, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, -1, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, -1, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, -1, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, + -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 43, + 44, -1, 46, 47, 48, -1, 50, 51, 52, 53, + 54, -1, 56, 57, -1, 59, 60, 61, 62, 63, + 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, -1, -1, 83, + 84, 85, 86, 87, 88, -1, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, -1, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, -1, -1, -1, 173, + 174, 175, -1, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, -1, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, -1, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, -1, -1, 223, + -1, 225, 226, 227, 228, 229, 230, -1, -1, 233, + -1, 235, -1, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, -1, 269, 270, 271, 272, 273, + -1, 275, 276, -1, 278, -1, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, -1, 300, -1, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, 320, -1, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, -1, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, -1, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, -1, 417, -1, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + -1, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 3, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 43, 44, -1, 46, - 47, 48, -1, 50, 51, 52, 53, 54, -1, 56, - 57, -1, 59, 60, 61, 62, 63, 64, -1, -1, - 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, 79, -1, -1, -1, 83, 84, 85, 86, - 87, 88, -1, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, -1, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, - -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, -1, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, -1, -1, -1, 173, 174, 175, -1, - 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, -1, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, -1, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, -1, -1, 223, -1, 225, 226, - 227, 228, 229, 230, -1, -1, 233, -1, 235, -1, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, -1, 269, 270, 271, 272, 273, -1, 275, 276, - -1, 278, -1, 280, 281, 282, 283, 284, 285, -1, - 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, 298, -1, 300, -1, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, -1, 322, 323, 324, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, -1, 362, 363, 364, 365, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, -1, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, -1, 392, 393, 394, 395, -1, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, - 417, -1, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, -1, 436, - 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, -1, 463, -1, 465, 466, - 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, - 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, - 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 3, -1, 5, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 43, 44, -1, 46, 47, 48, + -1, 50, 51, 52, 53, 54, -1, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, -1, -1, 83, 84, 85, 86, 87, 88, + -1, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, -1, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, -1, -1, -1, 173, 174, 175, -1, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, -1, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, -1, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, -1, -1, 223, -1, 225, 226, 227, 228, + 229, 230, -1, -1, 233, -1, 235, -1, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, -1, + 269, 270, 271, 272, 273, -1, 275, 276, -1, 278, + -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, + -1, 300, -1, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, -1, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, -1, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, -1, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, -1, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, -1, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, + -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 43, + 44, -1, 46, 47, 48, -1, 50, 51, 52, 53, + 54, -1, 56, 57, -1, 59, 60, 61, 62, 63, + 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, -1, -1, 83, + 84, 85, 86, 87, 88, -1, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, -1, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, -1, -1, -1, 173, + 174, 175, -1, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, -1, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, -1, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, -1, -1, 223, + -1, 225, 226, 227, 228, 229, 230, -1, -1, 233, + -1, 235, -1, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, -1, 269, 270, 271, 272, 273, + -1, 275, 276, -1, 278, -1, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, -1, 300, -1, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, 320, -1, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, -1, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, -1, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, -1, 417, -1, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + -1, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 3, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, - 32, 33, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 43, 44, -1, 46, 47, 48, -1, 50, 51, - 52, 53, 54, -1, 56, 57, -1, 59, 60, 61, - 62, 63, 64, -1, -1, 67, 68, 69, 70, 71, - 72, 73, -1, 75, 76, 77, 78, 79, -1, -1, - -1, 83, 84, 85, 86, 87, 88, -1, 90, 91, - 92, -1, 94, 95, 96, 97, 98, 99, -1, -1, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, -1, 118, -1, 120, 121, - 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, - -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, - 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, -1, 160, -1, - 162, 163, 164, 165, -1, 167, -1, 169, -1, -1, - -1, 173, 174, 175, -1, 177, -1, 179, -1, 181, - 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, - 192, -1, 194, 195, 196, 197, -1, 199, 200, 201, - 202, 203, 204, 205, -1, 207, -1, 209, 210, 211, - 212, 213, 214, 215, 216, -1, 218, -1, 220, -1, - -1, 223, -1, 225, 226, 227, 228, 229, 230, -1, - -1, 233, -1, 235, -1, -1, 238, 239, 240, 241, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, -1, 269, 270, 271, - 272, 273, -1, 275, 276, -1, 278, -1, 280, 281, - 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, - 292, 293, -1, -1, 296, 297, 298, -1, 300, -1, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, -1, -1, -1, -1, 318, 319, 320, -1, - 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, - 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, - 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, -1, 359, 360, -1, - 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, -1, -1, 381, - 382, 383, 384, 385, 386, 387, 388, 389, -1, -1, - 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - -1, -1, 414, 415, -1, 417, -1, 419, 420, 421, - 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - 432, 433, 434, -1, 436, 437, 438, 439, 440, 441, - 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, - 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, - -1, 463, -1, 465, 466, 467, 468, 469, 470, 471, - -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, - 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 3, -1, 5, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 43, 44, -1, 46, 47, 48, + -1, 50, 51, 52, 53, 54, -1, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, -1, -1, 83, 84, 85, 86, 87, 88, + -1, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, -1, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, -1, -1, -1, 173, 174, 175, -1, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, -1, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, -1, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, -1, -1, 223, -1, 225, 226, 227, 228, + 229, 230, -1, -1, 233, -1, 235, -1, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, -1, + 269, 270, 271, 272, 273, -1, 275, 276, -1, 278, + -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, + -1, 300, -1, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, -1, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, -1, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, -1, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, -1, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, -1, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 43, 44, -1, 46, - 47, 48, -1, 50, 51, 52, 53, 54, -1, 56, - 57, -1, 59, 60, 61, 62, 63, 64, -1, -1, - 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, 79, -1, -1, -1, 83, 84, 85, 86, - 87, 88, -1, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, -1, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, - -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, -1, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, -1, -1, -1, 173, 174, 175, -1, - 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, -1, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, -1, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, -1, -1, 223, -1, 225, 226, - 227, 228, 229, 230, -1, -1, 233, -1, 235, -1, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, -1, 269, 270, 271, 272, 273, -1, 275, 276, - -1, 278, -1, 280, 281, 282, 283, 284, 285, -1, - 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, 298, -1, 300, -1, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, -1, 322, 323, 324, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, -1, 362, 363, 364, 365, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, -1, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, -1, 392, 393, 394, 395, -1, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, - 417, -1, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, -1, 436, - 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, -1, 463, -1, 465, 466, - 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, - 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, - 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, + -1, -1, -1, -1, -1, -1, 40, -1, -1, 43, + 44, -1, 46, 47, 48, -1, 50, 51, 52, 53, + 54, -1, 56, 57, -1, 59, 60, 61, 62, 63, + 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, -1, -1, 83, + 84, 85, 86, 87, 88, -1, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, -1, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, -1, -1, -1, 173, + 174, 175, -1, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, -1, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, -1, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, -1, -1, 223, + -1, 225, 226, 227, 228, 229, 230, -1, -1, 233, + -1, 235, -1, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, -1, 269, 270, 271, 272, 273, + -1, 275, 276, -1, 278, -1, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, -1, 300, -1, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, 320, -1, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, -1, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, -1, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, -1, 417, -1, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + -1, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, - 32, 33, -1, -1, -1, 37, -1, -1, 40, -1, - 42, 43, 44, -1, 46, 47, 48, 49, 50, 51, - 52, -1, 54, 55, 56, 57, -1, 59, 60, 61, - 62, 63, 64, -1, -1, 67, 68, 69, 70, 71, - 72, 73, -1, 75, 76, 77, 78, -1, -1, 81, - -1, 83, 84, 85, 86, 87, 88, 89, 90, 91, - 92, -1, 94, 95, 96, 97, 98, 99, -1, 101, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, -1, 118, -1, 120, 121, - 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, - -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, - 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - -1, 153, 154, 155, 156, 157, -1, -1, 160, -1, - 162, 163, 164, 165, -1, 167, -1, 169, 170, -1, - 172, 173, 174, 175, 176, 177, -1, 179, -1, -1, - -1, 183, 184, -1, 186, 187, 188, 189, 190, 191, - 192, 193, 194, 195, 196, 197, -1, 199, 200, 201, - 202, 203, 204, 205, -1, 207, 208, -1, 210, 211, - 212, 213, 214, 215, 216, -1, 218, -1, 220, 221, - 222, 223, 224, 225, 226, 227, 228, 229, 230, -1, - -1, 233, 234, 235, 236, -1, 238, 239, 240, 241, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, - 272, -1, -1, 275, 276, 277, 278, -1, -1, 281, - 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, - 292, 293, -1, -1, 296, 297, -1, 299, 300, 301, - -1, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, -1, -1, -1, -1, 318, 319, -1, 321, - 322, 323, -1, 325, 326, 327, -1, 329, 330, 331, - 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, - 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, -1, 359, 360, 361, - 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, -1, 380, 381, - 382, 383, 384, 385, 386, 387, 388, 389, -1, 391, - 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, -1, - -1, -1, 414, 415, -1, 417, 418, 419, 420, 421, - 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - -1, 433, -1, -1, 436, 437, 438, 439, 440, 441, - 442, 443, 444, -1, 446, 447, 448, -1, 450, 451, - 452, 453, -1, 455, 456, 457, 458, 459, -1, 461, - -1, 463, 464, 465, 466, 467, 468, 469, 470, 471, - -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, - 482, 3, -1, 5, -1, -1, -1, -1, -1, -1, - -1, -1, 494, 495, 496, 497, -1, -1, -1, -1, - 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, - 32, 33, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 43, 44, -1, 46, 47, 48, -1, 50, 51, - 52, 53, 54, -1, 56, 57, -1, 59, 60, 61, - 62, 63, 64, -1, -1, 67, 68, 69, 70, 71, - 72, 73, -1, 75, 76, 77, 78, 79, -1, -1, - -1, 83, 84, 85, 86, 87, 88, -1, 90, 91, - 92, -1, 94, 95, 96, 97, 98, 99, -1, -1, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, -1, 118, -1, 120, 121, - 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, - -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, - 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, -1, 160, -1, - 162, 163, 164, 165, -1, 167, -1, 169, -1, -1, - -1, 173, 174, 175, -1, 177, -1, 179, -1, 181, - 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, - 192, -1, 194, 195, 196, 197, -1, 199, 200, 201, - 202, 203, 204, 205, -1, 207, -1, 209, 210, 211, - 212, 213, 214, 215, 216, -1, 218, -1, 220, -1, - -1, 223, -1, 225, 226, 227, 228, 229, 230, -1, - -1, 233, -1, 235, -1, -1, 238, 239, 240, 241, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, -1, 269, 270, 271, - 272, 273, -1, 275, 276, -1, 278, -1, 280, 281, - 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, - 292, 293, -1, -1, 296, 297, 298, -1, 300, -1, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, -1, -1, -1, -1, 318, 319, 320, -1, - 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, - 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, - 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, -1, 359, 360, -1, - 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, -1, -1, 381, - 382, 383, 384, 385, 386, 387, 388, 389, -1, -1, - 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - -1, -1, 414, 415, -1, 417, -1, 419, 420, 421, - 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - 432, 433, 434, -1, 436, 437, 438, 439, 440, 441, - 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, - 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, - -1, 463, -1, 465, 466, 467, 468, 469, 470, 471, - -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, - 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 3, -1, -1, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, + -1, 40, -1, -1, 43, 44, -1, 46, 47, 48, + -1, 50, 51, 52, 53, 54, -1, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, -1, -1, 83, 84, 85, 86, 87, 88, + -1, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, -1, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, -1, -1, -1, 173, 174, 175, -1, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, -1, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, -1, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, -1, -1, 223, -1, 225, 226, 227, 228, + 229, 230, -1, -1, 233, -1, 235, -1, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, -1, + 269, 270, 271, 272, 273, -1, 275, 276, -1, 278, + -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, + -1, 300, -1, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, -1, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, -1, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, -1, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, -1, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, -1, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 43, + 44, -1, 46, 47, 48, -1, 50, 51, 52, 53, + 54, -1, 56, 57, -1, 59, 60, 61, 62, 63, + 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, -1, -1, 83, + 84, 85, 86, 87, 88, -1, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, -1, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, -1, -1, -1, 173, + 174, 175, -1, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, -1, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, -1, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, -1, -1, 223, + -1, 225, 226, 227, 228, 229, 230, -1, -1, 233, + -1, 235, -1, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, -1, 269, 270, 271, 272, 273, + -1, 275, 276, -1, 278, -1, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, -1, 300, -1, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, 320, -1, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, -1, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, -1, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, -1, 417, -1, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + -1, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 3, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 43, 44, -1, 46, - 47, 48, -1, 50, 51, 52, 53, 54, -1, 56, - 57, -1, 59, 60, 61, 62, 63, 64, -1, -1, - 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, 79, -1, -1, -1, 83, 84, 85, 86, - 87, 88, -1, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, -1, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, - -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, -1, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, -1, -1, -1, 173, 174, 175, -1, - 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, -1, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, -1, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, -1, -1, 223, -1, 225, 226, - 227, 228, 229, 230, -1, -1, 233, -1, 235, -1, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, -1, 269, 270, 271, 272, 273, -1, 275, 276, - -1, 278, -1, 280, 281, 282, 283, 284, 285, -1, - 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, 298, -1, 300, -1, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, -1, 322, 323, 324, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, -1, 362, 363, 364, 365, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, -1, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, -1, 392, 393, 394, 395, -1, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, - 417, -1, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, -1, 436, - 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, -1, 463, -1, 465, 466, - 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, - 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, - 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 43, 44, -1, 46, 47, 48, + -1, 50, 51, 52, 53, 54, -1, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, -1, -1, 83, 84, 85, 86, 87, 88, + -1, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, -1, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, -1, -1, -1, 173, 174, 175, -1, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, -1, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, -1, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, -1, -1, 223, -1, 225, 226, 227, 228, + 229, 230, -1, -1, 233, -1, 235, -1, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, -1, + 269, 270, 271, 272, 273, -1, 275, 276, -1, 278, + -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, + -1, 300, -1, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, -1, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, -1, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, -1, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, -1, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, -1, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, + -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 43, + 44, -1, 46, 47, 48, -1, 50, 51, 52, 53, + 54, -1, 56, 57, -1, 59, 60, 61, 62, 63, + 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, -1, -1, 83, + 84, 85, 86, 87, 88, -1, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, -1, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, -1, -1, -1, 173, + 174, 175, -1, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, -1, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, -1, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, -1, -1, 223, + -1, 225, 226, 227, 228, 229, 230, -1, -1, 233, + -1, 235, -1, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, -1, 269, 270, 271, 272, 273, + -1, 275, 276, -1, 278, -1, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, -1, 300, -1, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, 320, -1, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, -1, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, -1, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, -1, 417, -1, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + -1, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 3, -1, 5, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 43, 44, -1, 46, 47, 48, + -1, 50, 51, 52, 53, 54, -1, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, -1, -1, 83, 84, 85, 86, 87, 88, + -1, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, -1, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, -1, -1, -1, 173, 174, 175, -1, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, -1, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, -1, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, -1, -1, 223, -1, 225, 226, 227, 228, + 229, 230, -1, -1, 233, -1, 235, -1, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, -1, + 269, 270, 271, 272, 273, -1, 275, 276, -1, 278, + -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, + -1, 300, -1, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, -1, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, -1, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, -1, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, -1, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, -1, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, + -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 43, + 44, -1, 46, 47, 48, -1, 50, 51, 52, 53, + 54, -1, 56, 57, -1, 59, 60, 61, 62, 63, + 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, -1, -1, 83, + 84, 85, 86, 87, 88, -1, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, -1, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, -1, -1, -1, 173, + 174, 175, -1, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, -1, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, -1, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, -1, -1, 223, + -1, 225, 226, 227, 228, 229, 230, -1, -1, 233, + -1, 235, -1, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, -1, 269, 270, 271, 272, 273, + -1, 275, 276, -1, 278, -1, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, -1, 300, -1, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, 320, -1, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, -1, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, -1, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, -1, 417, -1, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + -1, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 3, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, - 32, 33, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 43, 44, -1, 46, 47, 48, -1, 50, 51, - 52, 53, 54, -1, 56, 57, -1, 59, 60, 61, - 62, 63, 64, -1, -1, 67, 68, 69, 70, 71, - 72, 73, -1, 75, 76, 77, 78, 79, -1, -1, - -1, 83, 84, 85, 86, 87, 88, -1, 90, 91, - 92, -1, 94, 95, 96, 97, 98, 99, -1, -1, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, -1, 118, -1, 120, 121, - 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, - -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, - 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, -1, 160, -1, - 162, 163, 164, 165, -1, 167, -1, 169, -1, -1, - -1, 173, 174, 175, -1, 177, -1, 179, -1, 181, - 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, - 192, -1, 194, 195, 196, 197, -1, 199, 200, 201, - 202, 203, 204, 205, -1, 207, -1, 209, 210, 211, - 212, 213, 214, 215, 216, -1, 218, -1, 220, -1, - -1, 223, -1, 225, 226, 227, 228, 229, 230, -1, - -1, 233, -1, 235, -1, -1, 238, 239, 240, 241, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, -1, 269, 270, 271, - 272, 273, -1, 275, 276, -1, 278, -1, 280, 281, - 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, - 292, 293, -1, -1, 296, 297, 298, -1, 300, -1, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, -1, -1, -1, -1, 318, 319, 320, -1, - 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, - 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, - 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, -1, 359, 360, -1, - 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, -1, -1, 381, - 382, 383, 384, 385, 386, 387, 388, 389, -1, -1, - 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - -1, -1, 414, 415, -1, 417, -1, 419, 420, 421, - 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - 432, 433, 434, -1, 436, 437, 438, 439, 440, 441, - 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, - 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, - -1, 463, -1, 465, 466, 467, 468, 469, 470, 471, - -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, - 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 3, -1, -1, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 43, 44, -1, 46, 47, 48, + -1, 50, 51, 52, 53, 54, -1, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, -1, -1, 83, 84, 85, 86, 87, 88, + -1, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, -1, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, -1, -1, -1, 173, 174, 175, -1, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, -1, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, -1, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, -1, -1, 223, -1, 225, 226, 227, 228, + 229, 230, -1, -1, 233, -1, 235, -1, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, -1, + 269, 270, 271, 272, 273, -1, 275, 276, -1, 278, + -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, + -1, 300, -1, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, -1, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, -1, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, -1, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, -1, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, -1, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 43, 44, -1, 46, - 47, 48, -1, 50, 51, 52, 53, 54, -1, 56, - 57, -1, 59, 60, 61, 62, 63, 64, -1, -1, - 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, 79, -1, -1, -1, 83, 84, 85, 86, - 87, 88, -1, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, -1, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, - -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, -1, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, -1, -1, -1, 173, 174, 175, -1, - 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, -1, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, -1, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, -1, -1, 223, -1, 225, 226, - 227, 228, 229, 230, -1, -1, 233, -1, 235, -1, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, -1, 269, 270, 271, 272, 273, -1, 275, 276, - -1, 278, -1, 280, 281, 282, 283, 284, 285, -1, - 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, 298, -1, 300, -1, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, -1, 322, 323, 324, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, -1, 362, 363, 364, 365, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, -1, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, -1, 392, 393, 394, 395, -1, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, - 417, -1, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, -1, 436, - 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, -1, 463, -1, 465, 466, - 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, - 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, - 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, + -1, -1, -1, 37, -1, -1, 40, -1, 42, 43, + 44, -1, 46, 47, 48, 49, 50, 51, 52, -1, + 54, 55, 56, 57, -1, 59, 60, 61, 62, 63, + 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, -1, -1, 81, -1, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, 101, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, -1, 153, + 154, 155, 156, 157, -1, -1, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, 170, -1, 172, 173, + 174, 175, 176, 177, -1, 179, -1, -1, -1, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, 193, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, 208, -1, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, -1, -1, 233, + 234, 235, 236, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, 268, 269, 270, 271, 272, -1, + -1, 275, 276, 277, 278, -1, -1, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, -1, 299, 300, 301, -1, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, -1, 321, 322, 323, + -1, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, 361, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, 380, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, 391, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, -1, -1, -1, + 414, 415, -1, 417, 418, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, -1, 433, + -1, -1, 436, 437, 438, 439, 440, 441, 442, 443, + 444, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, -1, 461, -1, 463, + 464, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 3, + -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, + 494, 495, 496, 497, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 43, + 44, -1, 46, 47, 48, -1, 50, 51, 52, 53, + 54, -1, 56, 57, -1, 59, 60, 61, 62, 63, + 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, -1, -1, 83, + 84, 85, 86, 87, 88, -1, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, -1, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, -1, -1, -1, 173, + 174, 175, -1, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, -1, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, -1, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, -1, -1, 223, + -1, 225, 226, 227, 228, 229, 230, -1, -1, 233, + -1, 235, -1, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, -1, 269, 270, 271, 272, 273, + -1, 275, 276, -1, 278, -1, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, -1, 300, -1, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, 320, -1, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, -1, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, -1, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, -1, 417, -1, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + -1, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 43, 44, -1, 46, 47, 48, + -1, 50, 51, 52, 53, 54, -1, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, -1, -1, 83, 84, 85, 86, 87, 88, + -1, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, -1, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, -1, -1, -1, 173, 174, 175, -1, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, -1, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, -1, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, -1, -1, 223, -1, 225, 226, 227, 228, + 229, 230, -1, -1, 233, -1, 235, -1, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, -1, + 269, 270, 271, 272, 273, -1, 275, 276, -1, 278, + -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, + -1, 300, -1, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, -1, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, -1, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, -1, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, -1, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, -1, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, - 32, 33, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 43, 44, -1, 46, 47, 48, -1, 50, 51, - 52, 53, 54, -1, 56, 57, -1, 59, 60, 61, - 62, 63, 64, -1, -1, 67, 68, 69, 70, 71, - 72, 73, -1, 75, 76, 77, 78, 79, -1, -1, - -1, 83, 84, 85, 86, 87, 88, -1, 90, 91, - 92, -1, 94, 95, 96, 97, 98, 99, -1, -1, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, -1, 118, -1, 120, 121, - 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, - -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, - 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, -1, 160, -1, - 162, 163, 164, 165, -1, 167, -1, 169, -1, -1, - -1, 173, 174, 175, -1, 177, -1, 179, -1, 181, - 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, - 192, -1, 194, 195, 196, 197, -1, 199, 200, 201, - 202, 203, 204, 205, -1, 207, -1, 209, 210, 211, - 212, 213, 214, 215, 216, -1, 218, -1, 220, -1, - -1, 223, -1, 225, 226, 227, 228, 229, 230, -1, - -1, 233, -1, 235, -1, -1, 238, 239, 240, 241, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, -1, 269, 270, 271, - 272, 273, -1, 275, 276, -1, 278, -1, 280, 281, - 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, - 292, 293, -1, -1, 296, 297, 298, -1, 300, -1, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, -1, -1, -1, -1, 318, 319, 320, -1, - 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, - 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, - 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, -1, 359, 360, -1, - 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, -1, -1, 381, - 382, 383, 384, 385, 386, 387, 388, 389, -1, -1, - 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - -1, -1, 414, 415, -1, 417, -1, 419, 420, 421, - 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - 432, 433, 434, -1, 436, 437, 438, 439, 440, 441, - 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, - 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, - -1, 463, -1, 465, 466, 467, 468, 469, 470, 471, - -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, - 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 3, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 43, + 44, -1, 46, 47, 48, -1, 50, 51, 52, 53, + 54, -1, 56, 57, -1, 59, 60, 61, 62, 63, + 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, -1, -1, 83, + 84, 85, 86, 87, 88, -1, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, -1, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, -1, -1, -1, 173, + 174, 175, -1, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, -1, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, -1, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, -1, -1, 223, + -1, 225, 226, 227, 228, 229, 230, -1, -1, 233, + -1, 235, -1, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, -1, 269, 270, 271, 272, 273, + -1, 275, 276, -1, 278, -1, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, -1, 300, -1, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, 320, -1, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, -1, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, -1, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, -1, 417, -1, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + -1, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 43, 44, -1, 46, - 47, 48, -1, 50, 51, 52, 53, 54, -1, 56, - 57, -1, 59, 60, 61, 62, 63, 64, -1, -1, - 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, 79, -1, -1, -1, 83, 84, 85, 86, - 87, 88, -1, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, -1, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, - -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, -1, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, -1, -1, -1, 173, 174, 175, -1, - 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, -1, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, -1, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, -1, -1, 223, -1, 225, 226, - 227, 228, 229, 230, -1, -1, 233, -1, 235, -1, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, -1, 269, 270, 271, 272, 273, -1, 275, 276, - -1, 278, -1, 280, 281, 282, 283, 284, 285, -1, - 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, 298, -1, 300, -1, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, -1, 322, 323, 324, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, -1, 362, 363, 364, 365, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, -1, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, -1, 392, 393, 394, 395, -1, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, - 417, -1, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, -1, 436, - 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, -1, 463, -1, 465, 466, - 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, - 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, - 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 43, 44, -1, 46, 47, 48, + -1, 50, 51, 52, 53, 54, -1, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, -1, -1, 83, 84, 85, 86, 87, 88, + -1, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, -1, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, -1, -1, -1, 173, 174, 175, -1, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, -1, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, -1, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, -1, -1, 223, -1, 225, 226, 227, 228, + 229, 230, -1, -1, 233, -1, 235, -1, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, -1, + 269, 270, 271, 272, 273, -1, 275, 276, -1, 278, + -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, + -1, 300, -1, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, -1, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, -1, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, -1, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, -1, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, -1, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 43, + 44, -1, 46, 47, 48, -1, 50, 51, 52, 53, + 54, -1, 56, 57, -1, 59, 60, 61, 62, 63, + 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, -1, -1, 83, + 84, 85, 86, 87, 88, -1, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, -1, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, -1, -1, -1, 173, + 174, 175, -1, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, -1, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, -1, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, -1, -1, 223, + -1, 225, 226, 227, 228, 229, 230, -1, -1, 233, + -1, 235, -1, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, -1, 269, 270, 271, 272, 273, + -1, 275, 276, -1, 278, -1, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, -1, 300, -1, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, 320, -1, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, -1, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, -1, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, -1, 417, -1, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + -1, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, - 32, 33, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 43, 44, -1, 46, 47, 48, -1, 50, 51, - 52, 53, 54, -1, 56, 57, -1, 59, 60, 61, - 62, 63, 64, -1, -1, 67, 68, 69, 70, 71, - 72, 73, -1, 75, 76, 77, 78, 79, -1, -1, - -1, 83, 84, 85, 86, 87, 88, -1, 90, 91, - 92, -1, 94, 95, 96, 97, 98, 99, -1, -1, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, -1, 118, -1, 120, 121, - 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, - -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, - 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, -1, 160, -1, - 162, 163, 164, 165, -1, 167, -1, 169, -1, -1, - -1, 173, 174, 175, -1, 177, -1, 179, -1, 181, - 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, - 192, -1, 194, 195, 196, 197, -1, 199, 200, 201, - 202, 203, 204, 205, -1, 207, -1, 209, 210, 211, - 212, 213, 214, 215, 216, -1, 218, -1, 220, -1, - -1, 223, -1, 225, 226, 227, 228, 229, 230, -1, - -1, 233, -1, 235, -1, -1, 238, 239, 240, 241, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, -1, 269, 270, 271, - 272, 273, -1, 275, 276, -1, 278, -1, 280, 281, - 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, - 292, 293, -1, -1, 296, 297, 298, -1, 300, -1, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, -1, -1, -1, -1, 318, 319, 320, -1, - 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, - 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, - 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, -1, 359, 360, -1, - 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, -1, -1, 381, - 382, 383, 384, 385, 386, 387, 388, 389, -1, -1, - 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - -1, -1, 414, 415, -1, 417, -1, 419, 420, 421, - 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - 432, 433, 434, -1, 436, 437, 438, 439, 440, 441, - 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, - 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, - -1, 463, -1, 465, 466, 467, 468, 469, 470, 471, - -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, - 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 3, -1, -1, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 43, 44, -1, 46, 47, 48, + -1, 50, 51, 52, 53, 54, -1, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, -1, -1, 83, 84, 85, 86, 87, 88, + -1, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, -1, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, -1, -1, -1, 173, 174, 175, -1, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, -1, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, -1, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, -1, -1, 223, -1, 225, 226, 227, 228, + 229, 230, -1, -1, 233, -1, 235, -1, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, -1, + 269, 270, 271, 272, 273, -1, 275, 276, -1, 278, + -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, + -1, 300, -1, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, -1, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, -1, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, -1, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, -1, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, -1, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - -1, -1, -1, 40, -1, -1, 43, 44, -1, 46, - 47, 48, -1, 50, 51, 52, 53, 54, -1, 56, - 57, -1, 59, 60, 61, 62, 63, 64, -1, -1, - 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, 79, -1, -1, -1, 83, 84, 85, 86, - 87, 88, -1, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, -1, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, - -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, -1, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, -1, -1, -1, 173, 174, 175, -1, - 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, -1, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, -1, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, -1, -1, 223, -1, 225, 226, - 227, 228, 229, 230, -1, -1, 233, -1, 235, -1, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, -1, 269, 270, 271, 272, 273, -1, 275, 276, - -1, 278, -1, 280, 281, 282, 283, 284, 285, -1, - 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, 298, -1, 300, -1, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, -1, 322, 323, 324, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, -1, 362, 363, 364, 365, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, -1, 381, 382, 383, 384, 385, -1, - 387, 388, 389, -1, -1, 392, 393, 394, 395, -1, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, - 417, -1, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, -1, 436, - 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, -1, 463, -1, 465, 466, - 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, - 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, - 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 43, + 44, -1, 46, 47, 48, -1, 50, 51, 52, 53, + 54, -1, 56, 57, -1, 59, 60, 61, 62, 63, + 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, -1, -1, 83, + 84, 85, 86, 87, 88, -1, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, -1, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, -1, -1, -1, 173, + 174, 175, -1, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, -1, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, -1, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, -1, -1, 223, + -1, 225, 226, 227, 228, 229, 230, -1, -1, 233, + -1, 235, -1, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, -1, 269, 270, 271, 272, 273, + -1, 275, 276, -1, 278, -1, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, -1, 300, -1, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, 320, -1, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, -1, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, -1, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, -1, 417, -1, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + -1, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, + -1, 40, -1, -1, 43, 44, -1, 46, 47, 48, + -1, 50, 51, 52, 53, 54, -1, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, -1, -1, 83, 84, 85, 86, 87, 88, + -1, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, -1, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, -1, -1, -1, 173, 174, 175, -1, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, -1, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, -1, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, -1, -1, 223, -1, 225, 226, 227, 228, + 229, 230, -1, -1, 233, -1, 235, -1, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, -1, + 269, 270, 271, 272, 273, -1, 275, 276, -1, 278, + -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, + -1, 300, -1, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, -1, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, -1, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, -1, 381, 382, 383, 384, 385, -1, 387, 388, + 389, -1, -1, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, -1, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, -1, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, - 32, 33, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 43, 44, -1, 46, 47, 48, -1, 50, 51, - 52, 53, 54, -1, 56, 57, -1, 59, 60, 61, - 62, 63, 64, -1, -1, 67, 68, 69, 70, 71, - 72, 73, -1, 75, 76, 77, 78, 79, -1, -1, - -1, 83, 84, 85, 86, 87, 88, -1, 90, 91, - 92, -1, 94, 95, 96, 97, 98, 99, -1, -1, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, -1, 118, -1, 120, 121, - 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, - -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, - 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, -1, 160, -1, - 162, 163, 164, 165, -1, 167, -1, 169, -1, -1, - -1, 173, 174, 175, -1, 177, -1, 179, -1, 181, - 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, - 192, -1, 194, 195, 196, 197, -1, 199, 200, 201, - 202, 203, 204, 205, -1, 207, -1, 209, 210, 211, - 212, 213, 214, 215, 216, -1, 218, -1, 220, -1, - -1, 223, -1, 225, 226, 227, 228, 229, 230, -1, - -1, 233, -1, 235, -1, -1, 238, 239, 240, 241, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, -1, 269, 270, 271, - 272, 273, -1, 275, 276, -1, 278, -1, 280, 281, - 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, - 292, 293, -1, -1, 296, 297, 298, -1, 300, -1, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, -1, -1, -1, -1, 318, 319, 320, -1, - 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, - 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, - 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, -1, 359, 360, -1, - 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, -1, -1, 381, - 382, 383, 384, 385, 386, 387, 388, 389, -1, -1, - 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - -1, -1, 414, 415, -1, 417, -1, 419, 420, 421, - 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - 432, 433, 434, -1, 436, 437, 438, 439, 440, 441, - 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, - 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, - -1, 463, -1, 465, 466, 467, 468, 469, 470, 471, - -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, - 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 3, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 43, + 44, -1, 46, 47, 48, -1, 50, 51, 52, 53, + 54, -1, 56, 57, -1, 59, 60, 61, 62, 63, + 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, -1, -1, 83, + 84, 85, 86, 87, 88, -1, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, -1, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, -1, -1, -1, 173, + 174, 175, -1, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, -1, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, -1, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, -1, -1, 223, + -1, 225, 226, 227, 228, 229, 230, -1, -1, 233, + -1, 235, -1, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, -1, 269, 270, 271, 272, 273, + -1, 275, 276, -1, 278, -1, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, -1, 300, -1, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, 320, -1, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, -1, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, -1, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, -1, 417, -1, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + -1, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 43, 44, -1, 46, - 47, 48, -1, 50, 51, 52, 53, 54, -1, 56, - 57, -1, 59, 60, 61, 62, 63, 64, -1, -1, - 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, 79, -1, -1, -1, 83, 84, 85, 86, - 87, 88, -1, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, -1, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, - -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, -1, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, -1, -1, -1, 173, 174, 175, -1, - 177, -1, 179, -1, 181, 182, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, -1, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, -1, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, -1, -1, 223, -1, 225, 226, - 227, 228, 229, 230, -1, -1, 233, -1, 235, -1, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, -1, 269, 270, 271, 272, 273, -1, 275, 276, - -1, 278, -1, 280, 281, 282, 283, 284, 285, -1, - 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, 298, -1, 300, -1, 302, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, -1, 322, 323, 324, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, -1, 362, 363, 364, 365, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, -1, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, -1, 392, 393, 394, 395, -1, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, - 417, -1, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, -1, 436, - 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, -1, 463, -1, 465, 466, - 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, - 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, - 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 43, 44, -1, 46, 47, 48, + -1, 50, 51, 52, 53, 54, -1, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, -1, -1, 83, 84, 85, 86, 87, 88, + -1, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, -1, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, -1, -1, -1, 173, 174, 175, -1, 177, -1, + 179, -1, 181, 182, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, -1, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, -1, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, -1, -1, 223, -1, 225, 226, 227, 228, + 229, 230, -1, -1, 233, -1, 235, -1, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, -1, + 269, 270, 271, 272, 273, -1, 275, 276, -1, 278, + -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, + -1, 300, -1, 302, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, -1, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, -1, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, -1, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, -1, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, -1, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, - 32, 33, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 43, 44, -1, 46, 47, 48, -1, 50, 51, - 52, 53, 54, -1, 56, 57, -1, 59, 60, 61, - 62, 63, 64, -1, -1, 67, 68, 69, 70, 71, - 72, 73, -1, 75, 76, 77, 78, 79, -1, -1, - -1, 83, 84, 85, 86, 87, 88, -1, 90, 91, - 92, -1, 94, 95, 96, 97, 98, 99, -1, -1, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, -1, 118, -1, 120, 121, - 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, - -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, - 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, -1, 160, -1, - 162, 163, 164, 165, -1, 167, -1, 169, -1, -1, - -1, 173, 174, 175, -1, 177, -1, 179, -1, 181, - 182, 183, 184, -1, 186, 187, 188, 189, 190, 191, - 192, -1, 194, 195, 196, 197, -1, 199, 200, 201, - 202, 203, 204, 205, -1, 207, -1, 209, 210, 211, - 212, 213, 214, 215, 216, -1, 218, -1, 220, -1, - -1, 223, -1, 225, 226, 227, 228, 229, 230, -1, - -1, 233, -1, 235, -1, -1, 238, 239, 240, 241, - 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, -1, 269, 270, 271, - 272, 273, -1, 275, 276, -1, 278, -1, 280, 281, - 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, - 292, 293, -1, -1, 296, 297, 298, -1, 300, -1, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, -1, -1, -1, -1, 318, 319, 320, -1, - 322, 323, 324, 325, 326, 327, -1, 329, 330, 331, - 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, - 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, -1, 359, 360, -1, - 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, -1, -1, 381, - 382, 383, 384, 385, 386, 387, 388, 389, -1, -1, - 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - -1, -1, 414, 415, -1, 417, -1, 419, 420, 421, - 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - 432, 433, 434, -1, 436, 437, 438, 439, 440, 441, - 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, - 452, 453, -1, 455, 456, 457, 458, 459, 460, 461, - -1, 463, -1, 465, 466, 467, 468, 469, 470, 471, - -1, -1, 474, -1, -1, 477, 478, 479, 480, 481, - 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 3, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 43, + 44, -1, 46, 47, 48, -1, 50, 51, 52, 53, + 54, -1, 56, 57, -1, 59, 60, 61, 62, 63, + 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, 79, -1, -1, -1, 83, + 84, 85, 86, 87, 88, -1, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, -1, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, 152, 153, + 154, 155, 156, 157, 158, -1, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, -1, -1, -1, 173, + 174, 175, -1, 177, -1, 179, -1, 181, 182, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, -1, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, -1, 209, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, -1, -1, 223, + -1, 225, 226, 227, 228, 229, 230, -1, -1, 233, + -1, 235, -1, -1, 238, 239, 240, 241, 242, 243, + 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, -1, 269, 270, 271, 272, 273, + -1, 275, 276, -1, 278, -1, 280, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, 298, -1, 300, -1, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, 320, -1, 322, 323, + 324, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, -1, 362, 363, + 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, -1, 381, 382, 383, + 384, 385, 386, 387, 388, 389, -1, -1, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, 409, 410, 411, -1, -1, + 414, 415, -1, 417, -1, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, 432, 433, + 434, -1, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, 460, 461, -1, 463, + -1, 465, 466, 467, 468, 469, 470, 471, -1, -1, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, + 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - 37, -1, -1, -1, -1, 42, 43, 44, -1, 46, - 47, 48, 49, 50, 51, 52, -1, 54, 55, 56, - 57, -1, 59, 60, 61, 62, 63, 64, -1, -1, - 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, -1, -1, 81, -1, 83, 84, 85, 86, - 87, 88, 89, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, 101, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, - -1, 148, 149, 150, 151, -1, 153, 154, 155, 156, - 157, -1, -1, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, 170, -1, 172, 173, 174, -1, 176, - 177, -1, 179, -1, -1, -1, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, 208, -1, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, 221, 222, 223, 224, 225, 226, - 227, 228, 229, 230, -1, -1, 233, 234, 235, 236, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, 268, 269, 270, 271, 272, -1, -1, 275, 276, - 277, 278, -1, -1, 281, 282, 283, 284, 285, -1, - 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, -1, 299, 300, 301, -1, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, -1, 321, 322, 323, -1, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, 361, 362, 363, 364, 365, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, 380, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, 391, 392, 393, 394, 395, -1, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, -1, -1, -1, 414, 415, -1, - 417, 418, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, -1, 433, -1, -1, 436, - 437, 438, 439, 440, 441, 442, 443, 444, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, -1, 461, -1, 463, 464, 465, 466, - 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, - 477, 478, 479, 480, 481, 482, 3, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 494, 495, 496, - 497, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - 37, -1, -1, -1, -1, 42, 43, 44, -1, 46, - 47, 48, 49, 50, 51, 52, -1, 54, 55, 56, - 57, -1, 59, 60, 61, 62, 63, 64, -1, -1, - 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, -1, -1, 81, -1, 83, 84, 85, 86, - 87, 88, 89, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, 101, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, - -1, 148, 149, 150, 151, -1, 153, 154, 155, 156, - 157, -1, -1, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, 170, -1, 172, 173, 174, -1, 176, - 177, -1, 179, -1, -1, -1, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, 208, -1, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, 221, 222, 223, 224, 225, 226, - 227, 228, 229, 230, -1, -1, 233, 234, 235, 236, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, 268, 269, 270, 271, 272, -1, -1, 275, 276, - 277, 278, -1, -1, 281, 282, 283, 284, 285, -1, - 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, -1, 299, 300, 301, -1, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, -1, 321, 322, 323, -1, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, 361, 362, 363, 364, 365, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, 380, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, 391, 392, 393, 394, 395, -1, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, -1, -1, -1, 414, 415, -1, - 417, 418, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, -1, 433, -1, -1, 436, - 437, 438, 439, 440, 441, 442, 443, 444, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, -1, 461, -1, 463, 464, 465, 466, - 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, - 477, 478, 479, 480, 481, 482, 3, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 494, 495, 496, - 497, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - 37, -1, -1, -1, -1, 42, 43, 44, -1, 46, - 47, 48, 49, 50, 51, 52, -1, 54, 55, 56, - 57, -1, 59, 60, 61, 62, 63, 64, -1, -1, - 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, -1, -1, 81, -1, 83, 84, 85, 86, - 87, 88, 89, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, 101, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, - -1, 148, 149, 150, 151, -1, 153, 154, 155, 156, - 157, -1, -1, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, 170, -1, 172, 173, 174, 175, 176, - 177, -1, 179, -1, -1, -1, 183, 184, -1, 186, - 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, 208, -1, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, 221, 222, 223, 224, 225, 226, - 227, 228, 229, 230, -1, -1, 233, 234, 235, 236, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, 268, 269, 270, 271, 272, -1, -1, 275, 276, - 277, 278, -1, -1, 281, 282, 283, 284, 285, -1, - 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, -1, 299, 300, 301, -1, 303, 304, 305, 306, - 307, 308, 309, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, -1, 321, 322, 323, -1, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - 337, 338, 339, 340, 341, 342, 343, -1, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, 361, 362, 363, 364, -1, 366, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, 380, 381, 382, 383, 384, 385, 386, - -1, 388, 389, -1, 391, 392, 393, 394, 395, -1, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, -1, -1, -1, 414, 415, -1, - 417, 418, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, -1, 433, -1, -1, 436, - 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, -1, 461, -1, 463, 464, 465, 466, - 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, - 477, 478, 479, 480, 481, 482, 3, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 494, 495, 496, - 497, -1, -1, -1, -1, 22, 23, 24, 25, 26, - 27, 28, 29, -1, 31, 32, 33, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 43, 44, -1, 46, - 47, 48, -1, 50, 51, 52, 53, 54, -1, 56, - 57, -1, 59, 60, 61, 62, 63, 64, -1, -1, - 67, 68, 69, 70, 71, 72, 73, -1, 75, 76, - 77, 78, 79, -1, -1, -1, 83, 84, 85, 86, - 87, 88, -1, 90, 91, 92, -1, 94, 95, 96, - 97, 98, 99, -1, -1, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, - -1, 118, -1, 120, 121, 122, 123, 124, 125, -1, - -1, 128, 129, 130, 131, -1, -1, 134, 135, 136, - 137, 138, -1, 140, 141, 142, -1, 144, 145, 146, - -1, 148, 149, 150, 151, 152, 153, 154, 155, 156, - 157, 158, -1, 160, -1, 162, 163, 164, 165, -1, - 167, -1, 169, -1, -1, -1, 173, 174, 175, -1, - 177, -1, 179, -1, 181, 182, -1, 184, -1, 186, - 187, 188, 189, 190, 191, 192, -1, 194, 195, 196, - 197, -1, 199, 200, 201, 202, 203, 204, 205, -1, - 207, -1, 209, 210, 211, 212, 213, 214, 215, 216, - -1, 218, -1, 220, -1, -1, 223, -1, 225, 226, - 227, 228, 229, 230, -1, -1, 233, -1, 235, -1, - -1, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, - 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 267, -1, 269, 270, 271, 272, 273, -1, 275, 276, - -1, 278, -1, 280, 281, 282, 283, 284, 285, -1, - 287, 288, -1, -1, 291, 292, 293, -1, -1, 296, - 297, 298, -1, 300, -1, 302, 303, 304, 305, 306, - 307, 308, -1, 310, 311, 312, 313, -1, -1, -1, - -1, 318, 319, 320, -1, 322, 323, 324, 325, 326, - 327, -1, 329, 330, 331, 332, 333, 334, -1, 336, - -1, 338, 339, 340, 341, 342, 343, -1, 345, 346, - 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, - 357, -1, 359, 360, -1, 362, 363, 364, 365, -1, - 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, - 377, 378, -1, -1, 381, 382, 383, 384, 385, 386, - 387, 388, 389, -1, -1, 392, 393, 394, 395, -1, - 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, - 407, 408, 409, 410, 411, -1, -1, 414, 415, -1, - 417, -1, 419, 420, 421, 422, 423, -1, 425, 426, - 427, -1, -1, 430, 431, 432, 433, 434, -1, 436, - 437, 438, 439, 440, 441, 442, 443, -1, -1, 446, - 447, 448, -1, 450, 451, 452, 453, -1, 455, 456, - 457, 458, 459, 460, 461, -1, 463, -1, 465, 466, - 467, 468, 469, 470, 471, -1, -1, 474, -1, -1, - 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, - 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, - 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, 37, -1, + -1, -1, -1, 42, 43, 44, -1, 46, 47, 48, + 49, 50, 51, 52, -1, 54, 55, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + -1, -1, 81, -1, 83, 84, 85, 86, 87, 88, + 89, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, 101, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, -1, 153, 154, 155, 156, 157, -1, + -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, 170, -1, 172, 173, 174, -1, 176, 177, -1, + 179, -1, -1, -1, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, 193, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, 208, + -1, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, 221, 222, 223, 224, 225, 226, 227, 228, + 229, 230, -1, -1, 233, 234, 235, 236, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, + 269, 270, 271, 272, -1, -1, 275, 276, 277, 278, + -1, -1, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, -1, + 299, 300, 301, -1, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, -1, 321, 322, 323, -1, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, 380, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, 391, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, -1, -1, -1, 414, 415, -1, 417, 418, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, -1, 433, -1, -1, 436, 437, 438, + 439, 440, 441, 442, 443, 444, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, -1, 461, -1, 463, 464, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 3, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 494, 495, 496, 497, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, 37, -1, + -1, -1, -1, 42, 43, 44, -1, 46, 47, 48, + 49, 50, 51, 52, -1, 54, 55, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + -1, -1, 81, -1, 83, 84, 85, 86, 87, 88, + 89, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, 101, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, -1, 153, 154, 155, 156, 157, -1, + -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, 170, -1, 172, 173, 174, -1, 176, 177, -1, + 179, -1, -1, -1, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, 193, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, 208, + -1, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, 221, 222, 223, 224, 225, 226, 227, 228, + 229, 230, -1, -1, 233, 234, 235, 236, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, + 269, 270, 271, 272, -1, -1, 275, 276, 277, 278, + -1, -1, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, -1, + 299, 300, 301, -1, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, -1, 321, 322, 323, -1, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, 380, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, 391, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, -1, -1, -1, 414, 415, -1, 417, 418, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, -1, 433, -1, -1, 436, 437, 438, + 439, 440, 441, 442, 443, 444, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, -1, 461, -1, 463, 464, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 3, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 494, 495, 496, 497, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, 37, -1, + -1, -1, -1, 42, 43, 44, -1, 46, 47, 48, + 49, 50, 51, 52, -1, 54, 55, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + -1, -1, 81, -1, 83, 84, 85, 86, 87, 88, + 89, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, 101, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, -1, 153, 154, 155, 156, 157, -1, + -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, 170, -1, 172, 173, 174, 175, 176, 177, -1, + 179, -1, -1, -1, 183, 184, -1, 186, 187, 188, + 189, 190, 191, 192, 193, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, 208, + -1, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, 221, 222, 223, 224, 225, 226, 227, 228, + 229, 230, -1, -1, 233, 234, 235, 236, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, + 269, 270, 271, 272, -1, -1, 275, 276, 277, 278, + -1, -1, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, -1, + 299, 300, 301, -1, 303, 304, 305, 306, 307, 308, + 309, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, -1, 321, 322, 323, -1, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, 337, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, 361, 362, 363, 364, -1, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, 380, 381, 382, 383, 384, 385, 386, -1, 388, + 389, -1, 391, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, -1, -1, -1, 414, 415, -1, 417, 418, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, -1, 433, -1, -1, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, -1, 461, -1, 463, 464, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 3, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 494, 495, 496, 497, -1, + -1, -1, -1, 22, 23, 24, 25, 26, 27, 28, + 29, -1, 31, 32, 33, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 43, 44, -1, 46, 47, 48, + -1, 50, 51, 52, 53, 54, -1, 56, 57, -1, + 59, 60, 61, 62, 63, 64, -1, -1, 67, 68, + 69, 70, 71, 72, 73, -1, 75, 76, 77, 78, + 79, -1, -1, -1, 83, 84, 85, 86, 87, 88, + -1, 90, 91, 92, -1, 94, 95, 96, 97, 98, + 99, -1, -1, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, -1, 118, + -1, 120, 121, 122, 123, 124, 125, -1, -1, 128, + 129, 130, 131, -1, -1, 134, 135, 136, 137, 138, + -1, 140, 141, 142, -1, 144, 145, 146, -1, 148, + 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, + -1, 160, -1, 162, 163, 164, 165, -1, 167, -1, + 169, -1, -1, -1, 173, 174, 175, -1, 177, -1, + 179, -1, 181, 182, -1, 184, -1, 186, 187, 188, + 189, 190, 191, 192, -1, 194, 195, 196, 197, -1, + 199, 200, 201, 202, 203, 204, 205, -1, 207, -1, + 209, 210, 211, 212, 213, 214, 215, 216, -1, 218, + -1, 220, -1, -1, 223, -1, 225, 226, 227, 228, + 229, 230, -1, -1, 233, -1, 235, -1, -1, 238, + 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, + 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, + 259, 260, 261, 262, 263, 264, 265, 266, 267, -1, + 269, 270, 271, 272, 273, -1, 275, 276, -1, 278, + -1, 280, 281, 282, 283, 284, 285, -1, 287, 288, + -1, -1, 291, 292, 293, -1, -1, 296, 297, 298, + -1, 300, -1, 302, 303, 304, 305, 306, 307, 308, + -1, 310, 311, 312, 313, -1, -1, -1, -1, 318, + 319, 320, -1, 322, 323, 324, 325, 326, 327, -1, + 329, 330, 331, 332, 333, 334, -1, 336, -1, 338, + 339, 340, 341, 342, 343, -1, 345, 346, 347, 348, + 349, 350, 351, 352, 353, 354, 355, 356, 357, -1, + 359, 360, -1, 362, 363, 364, 365, -1, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + -1, -1, 381, 382, 383, 384, 385, 386, 387, 388, + 389, -1, -1, 392, 393, 394, 395, -1, 397, 398, + 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, -1, -1, 414, 415, -1, 417, -1, + 419, 420, 421, 422, 423, -1, 425, 426, 427, -1, + -1, 430, 431, 432, 433, 434, -1, 436, 437, 438, + 439, 440, 441, 442, 443, -1, -1, 446, 447, 448, + -1, 450, 451, 452, 453, -1, 455, 456, 457, 458, + 459, 460, 461, -1, 463, -1, 465, 466, 467, 468, + 469, 470, 471, -1, -1, 474, -1, -1, 477, 478, + 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, + 489, 490, 491, 492, 493, 494, 495, 496, 497, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 22, 23, 24, 25, 26, 27, 28, 29, -1, 31, - 32, 33, -1, -1, -1, 37, -1, -1, -1, -1, - 42, 43, 44, -1, 46, 47, 48, 49, 50, 51, - 52, -1, 54, 55, 56, 57, -1, 59, 60, 61, - 62, 63, 64, -1, -1, 67, 68, 69, 70, 71, - 72, 73, -1, 75, 76, 77, 78, -1, -1, 81, - -1, 83, 84, 85, 86, 87, 88, 89, 90, 91, - 92, -1, 94, 95, 96, 97, 98, 99, -1, 101, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - 112, 113, 114, 115, 116, -1, 118, -1, 120, 121, - 122, 123, 124, 125, -1, -1, 128, 129, 130, 131, - -1, -1, 134, 135, 136, 137, 138, -1, 140, 141, - 142, -1, 144, 145, 146, -1, 148, 149, 150, 151, - -1, 153, 154, 155, 156, 157, -1, -1, 160, -1, - 162, 163, 164, 165, -1, 167, -1, 169, 170, -1, - 172, 173, 174, -1, 176, 177, -1, 179, -1, -1, - -1, 183, 184, -1, 186, 187, 188, 189, 190, 191, - 192, 193, 194, 195, 196, 197, -1, 199, 200, 201, - 202, 203, 204, 205, -1, 207, 208, -1, 210, 211, - 212, 213, 214, 215, 216, -1, 218, -1, 220, 221, - 222, 223, 224, 225, 226, 227, 228, 229, 230, -1, - -1, 233, 234, 235, 236, -1, 238, 239, 240, 241, - 242, 243, 244, 245, -1, 247, 248, 249, 250, 251, - 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, - 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, - 272, -1, -1, 275, 276, 277, 278, -1, -1, 281, - 282, 283, 284, 285, -1, 287, 288, -1, -1, 291, - 292, 293, -1, -1, 296, 297, -1, 299, 300, 301, - -1, 303, 304, 305, 306, 307, 308, 309, 310, 311, - 312, 313, -1, -1, -1, -1, 318, 319, -1, 321, - 322, 323, -1, 325, 326, 327, -1, 329, 330, 331, - 332, 333, 334, -1, 336, 337, 338, 339, 340, 341, - 342, 343, -1, 345, 346, 347, 348, 349, 350, 351, - 352, 353, 354, 355, 356, 357, -1, 359, 360, 361, - 362, 363, 364, -1, 366, 367, 368, 369, 370, 371, - 372, 373, 374, 375, 376, 377, 378, -1, 380, 381, - 382, 383, 384, 385, 386, -1, 388, 389, -1, 391, - 392, 393, 394, 395, -1, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, -1, 410, -1, - -1, -1, 414, 415, -1, 417, 418, 419, 420, 421, - 422, 423, -1, 425, 426, 427, -1, -1, 430, 431, - -1, 433, -1, -1, 436, 437, 438, 439, 440, 441, - 442, 443, -1, -1, 446, 447, 448, -1, 450, 451, - 452, 453, -1, 455, 456, 457, 458, 459, -1, 461, - -1, 463, 464, 465, 466, 467, 468, 469, 470, 471, - -1, 22, 474, -1, -1, 477, 478, 479, 480, 481, - 482, 32, -1, 34, 35, -1, -1, -1, -1, 22, - -1, -1, 494, 495, 496, 497, -1, -1, -1, 32, - -1, 52, -1, -1, -1, -1, -1, -1, -1, -1, - 61, -1, -1, -1, -1, -1, -1, -1, -1, 52, - -1, -1, -1, -1, 75, -1, -1, -1, 61, -1, - -1, -1, -1, -1, -1, 86, -1, -1, -1, -1, - -1, -1, 75, -1, -1, -1, -1, 98, -1, 100, + -1, -1, -1, -1, -1, -1, -1, -1, 22, 23, + 24, 25, 26, 27, 28, 29, -1, 31, 32, 33, + -1, -1, -1, 37, -1, -1, -1, -1, 42, 43, + 44, -1, 46, 47, 48, 49, 50, 51, 52, -1, + 54, 55, 56, 57, -1, 59, 60, 61, 62, 63, + 64, -1, -1, 67, 68, 69, 70, 71, 72, 73, + -1, 75, 76, 77, 78, -1, -1, 81, -1, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, -1, + 94, 95, 96, 97, 98, 99, -1, 101, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, -1, 118, -1, 120, 121, 122, 123, + 124, 125, -1, -1, 128, 129, 130, 131, -1, -1, + 134, 135, 136, 137, 138, -1, 140, 141, 142, -1, + 144, 145, 146, -1, 148, 149, 150, 151, -1, 153, + 154, 155, 156, 157, -1, -1, 160, -1, 162, 163, + 164, 165, -1, 167, -1, 169, 170, -1, 172, 173, + 174, -1, 176, 177, -1, 179, -1, -1, -1, 183, + 184, -1, 186, 187, 188, 189, 190, 191, 192, 193, + 194, 195, 196, 197, -1, 199, 200, 201, 202, 203, + 204, 205, -1, 207, 208, -1, 210, 211, 212, 213, + 214, 215, 216, -1, 218, -1, 220, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, -1, -1, 233, + 234, 235, 236, -1, 238, 239, 240, 241, 242, 243, + 244, 245, -1, 247, 248, 249, 250, 251, 252, 253, + 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, 268, 269, 270, 271, 272, -1, + -1, 275, 276, 277, 278, -1, -1, 281, 282, 283, + 284, 285, -1, 287, 288, -1, -1, 291, 292, 293, + -1, -1, 296, 297, -1, 299, 300, 301, -1, 303, + 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, + -1, -1, -1, -1, 318, 319, -1, 321, 322, 323, + -1, 325, 326, 327, -1, 329, 330, 331, 332, 333, + 334, -1, 336, 337, 338, 339, 340, 341, 342, 343, + -1, 345, 346, 347, 348, 349, 350, 351, 352, 353, + 354, 355, 356, 357, -1, 359, 360, 361, 362, 363, + 364, -1, 366, 367, 368, 369, 370, 371, 372, 373, + 374, 375, 376, 377, 378, -1, 380, 381, 382, 383, + 384, 385, 386, -1, 388, 389, -1, 391, 392, 393, + 394, 395, -1, 397, 398, 399, 400, 401, 402, 403, + 404, 405, 406, 407, 408, -1, 410, -1, -1, -1, + 414, 415, -1, 417, 418, 419, 420, 421, 422, 423, + -1, 425, 426, 427, -1, -1, 430, 431, -1, 433, + -1, -1, 436, 437, 438, 439, 440, 441, 442, 443, + -1, -1, 446, 447, 448, -1, 450, 451, 452, 453, + -1, 455, 456, 457, 458, 459, -1, 461, -1, 463, + 464, 465, 466, 467, 468, 469, 470, 471, -1, 22, + 474, -1, -1, 477, 478, 479, 480, 481, 482, 32, + -1, 34, 35, -1, -1, -1, -1, 22, -1, -1, + 494, 495, 496, 497, -1, -1, -1, 32, -1, 52, + -1, -1, -1, -1, -1, -1, -1, -1, 61, -1, + -1, -1, -1, -1, -1, -1, -1, 52, -1, -1, + -1, -1, 75, -1, -1, -1, 61, -1, -1, -1, -1, -1, -1, 86, -1, -1, -1, -1, -1, -1, - 111, -1, -1, -1, -1, 98, -1, 100, -1, -1, - -1, -1, -1, -1, -1, 126, 127, -1, 111, -1, - -1, -1, -1, -1, -1, -1, 137, -1, -1, -1, - -1, -1, 143, 126, 127, -1, -1, -1, -1, -1, - 151, -1, -1, -1, 137, -1, -1, -1, -1, -1, - 143, -1, -1, -1, -1, -1, 167, -1, 151, -1, - 171, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 167, -1, -1, -1, 171, -1, + 75, -1, -1, -1, -1, 98, -1, 100, -1, -1, + -1, 86, -1, -1, -1, -1, -1, -1, 111, -1, + -1, -1, -1, 98, -1, 100, -1, -1, -1, -1, + -1, -1, -1, 126, 127, -1, 111, -1, -1, -1, + -1, -1, -1, -1, 137, -1, -1, -1, -1, -1, + 143, 126, 127, -1, -1, -1, -1, -1, 151, -1, + -1, -1, 137, -1, -1, -1, -1, -1, 143, -1, + -1, -1, -1, -1, 167, -1, 151, -1, 171, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 167, -1, -1, -1, 171, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 213, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 213, -1, -1, -1, -1, -1, -1, -1, 239, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 213, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 213, -1, -1, -1, -1, -1, -1, -1, 239, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 239, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 314, 315, 316, -1, -1, -1, -1, - -1, 322, -1, -1, 325, -1, -1, -1, -1, -1, -1, 314, 315, 316, -1, -1, -1, -1, -1, 322, - -1, -1, 325, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 354, -1, -1, -1, -1, -1, -1, - -1, -1, 363, -1, -1, -1, -1, -1, -1, -1, - -1, 354, -1, -1, -1, -1, -1, -1, 379, -1, - 363, -1, -1, -1, -1, 386, -1, -1, -1, 390, - -1, -1, -1, -1, -1, -1, 379, -1, -1, 400, + -1, -1, 325, -1, -1, -1, -1, -1, -1, 314, + 315, 316, -1, -1, -1, -1, -1, 322, -1, -1, + 325, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 354, -1, -1, -1, -1, -1, -1, -1, -1, + 363, -1, -1, -1, -1, -1, -1, -1, -1, 354, + -1, -1, -1, -1, -1, -1, 379, -1, 363, -1, -1, -1, -1, 386, -1, -1, -1, 390, -1, -1, - -1, 412, -1, -1, -1, 416, -1, 400, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 412, - -1, -1, -1, 416, -1, 436, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 449, -1, - -1, -1, -1, 436, 455, -1, -1, -1, -1, 460, - -1, -1, -1, 464, -1, -1, 449, -1, -1, -1, - -1, -1, 455, -1, -1, 476, -1, 460, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 476, -1, -1, -1, -1, -1, 500, + -1, -1, -1, -1, 379, -1, -1, 400, -1, -1, + -1, 386, -1, -1, -1, 390, -1, -1, -1, 412, + -1, -1, -1, 416, -1, 400, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 412, -1, -1, + -1, 416, -1, 436, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 449, -1, -1, -1, + -1, 436, 455, -1, -1, -1, -1, 460, -1, -1, + -1, 464, -1, -1, 449, -1, -1, -1, -1, -1, + 455, -1, -1, 476, -1, 460, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 514, -1, -1, -1, 500, -1, -1, + -1, 476, -1, -1, -1, -1, -1, 500, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 514 + -1, 514, -1, -1, -1, 500, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 514 }; /* YYSTOS[STATE-NUM] -- The (internal number of the) accessing @@ -18818,10 +18852,10 @@ static const yytype_uint16 yystos[] = 528, 529, 540, 549, 551, 556, 572, 575, 576, 578, 582, 586, 593, 595, 597, 598, 646, 652, 655, 656, 674, 675, 676, 677, 679, 681, 682, 686, 739, 740, - 907, 910, 913, 920, 921, 923, 926, 927, 928, 935, - 939, 945, 948, 953, 957, 958, 959, 962, 965, 966, - 967, 971, 972, 974, 430, 479, 596, 202, 370, 381, - 416, 466, 108, 191, 960, 596, 3, 22, 23, 24, + 908, 911, 914, 921, 922, 924, 927, 928, 929, 936, + 940, 946, 949, 954, 958, 959, 960, 963, 966, 967, + 968, 972, 973, 975, 430, 479, 596, 202, 370, 381, + 416, 466, 108, 191, 961, 596, 3, 22, 23, 24, 25, 26, 27, 28, 29, 31, 32, 33, 42, 43, 44, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 59, 60, 61, 62, 63, 64, 67, @@ -18861,11 +18895,11 @@ static const yytype_uint16 yystos[] = 457, 458, 459, 460, 461, 463, 464, 465, 466, 467, 468, 469, 470, 471, 474, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, - 492, 493, 494, 495, 496, 497, 542, 816, 896, 900, - 977, 978, 979, 3, 175, 246, 409, 542, 922, 977, + 492, 493, 494, 495, 496, 497, 542, 817, 897, 901, + 978, 979, 980, 3, 175, 246, 409, 542, 923, 978, 289, 596, 55, 171, 514, 669, 177, 240, 294, 313, - 370, 420, 422, 439, 445, 448, 580, 644, 919, 5, - 30, 325, 542, 543, 895, 3, 30, 34, 35, 36, + 370, 420, 422, 439, 445, 448, 580, 644, 920, 5, + 30, 325, 542, 543, 896, 3, 30, 34, 35, 36, 37, 38, 39, 40, 41, 42, 45, 49, 53, 54, 55, 56, 57, 58, 65, 66, 71, 72, 74, 79, 80, 81, 82, 83, 89, 93, 100, 101, 108, 112, @@ -18881,39 +18915,39 @@ static const yytype_uint16 yystos[] = 418, 424, 426, 427, 428, 429, 432, 434, 435, 438, 444, 445, 449, 454, 460, 461, 462, 464, 472, 473, 475, 476, 483, 484, 485, 486, 487, 488, 489, 490, - 491, 492, 493, 548, 977, 981, 983, 24, 81, 97, + 491, 492, 493, 548, 978, 982, 984, 24, 81, 97, 146, 156, 168, 173, 202, 245, 249, 319, 334, 367, 370, 381, 384, 402, 416, 422, 423, 433, 439, 466, - 580, 647, 648, 651, 596, 895, 100, 137, 464, 514, + 580, 647, 648, 651, 596, 896, 100, 137, 464, 514, 529, 540, 549, 551, 572, 575, 576, 582, 586, 595, - 598, 646, 652, 655, 656, 674, 907, 910, 913, 920, - 921, 931, 935, 939, 945, 948, 953, 962, 965, 971, - 972, 974, 108, 75, 213, 66, 79, 81, 158, 231, + 598, 646, 652, 655, 656, 674, 908, 911, 914, 921, + 922, 932, 936, 940, 946, 949, 954, 963, 966, 972, + 973, 975, 108, 75, 213, 66, 79, 81, 158, 231, 280, 290, 302, 320, 366, 411, 432, 434, 438, 460, 514, 541, 542, 543, 675, 740, 742, 744, 754, 761, - 762, 816, 818, 819, 108, 5, 542, 544, 946, 946, - 542, 895, 30, 177, 240, 385, 426, 430, 542, 963, - 964, 969, 596, 30, 132, 695, 696, 177, 240, 370, - 385, 426, 940, 941, 969, 596, 542, 674, 686, 970, - 542, 761, 416, 692, 541, 172, 514, 950, 514, 342, - 687, 688, 895, 687, 675, 676, 965, 0, 517, 122, + 762, 817, 819, 820, 108, 5, 542, 544, 947, 947, + 542, 896, 30, 177, 240, 385, 426, 430, 542, 964, + 965, 970, 596, 30, 132, 695, 696, 177, 240, 370, + 385, 426, 941, 942, 970, 596, 542, 674, 686, 971, + 542, 761, 416, 692, 541, 172, 514, 951, 514, 342, + 687, 688, 896, 687, 675, 676, 966, 0, 517, 122, 212, 451, 147, 217, 295, 444, 698, 699, 744, 744, - 675, 677, 679, 518, 464, 929, 30, 426, 430, 674, - 970, 191, 541, 895, 191, 541, 191, 761, 191, 541, + 675, 677, 679, 518, 464, 930, 30, 426, 430, 674, + 971, 191, 541, 896, 191, 541, 191, 761, 191, 541, 274, 544, 512, 516, 545, 546, 514, 82, 108, 173, - 202, 245, 370, 381, 416, 439, 466, 925, 108, 674, + 202, 245, 370, 381, 416, 439, 466, 926, 108, 674, 541, 420, 422, 420, 422, 352, 191, 541, 541, 377, - 173, 245, 342, 381, 416, 466, 653, 202, 30, 895, + 173, 245, 342, 381, 416, 466, 653, 202, 30, 896, 191, 548, 251, 433, 107, 416, 416, 466, 374, 377, - 191, 542, 649, 902, 191, 892, 895, 191, 895, 514, - 585, 294, 422, 931, 3, 460, 932, 934, 935, 937, - 938, 977, 981, 929, 542, 544, 922, 946, 514, 514, + 191, 542, 649, 903, 191, 893, 896, 191, 896, 514, + 585, 294, 422, 932, 3, 460, 933, 935, 936, 938, + 939, 978, 982, 930, 542, 544, 923, 947, 514, 514, 166, 514, 675, 762, 514, 514, 541, 514, 514, 171, 514, 514, 514, 514, 675, 740, 744, 754, 507, 545, 40, 542, 755, 756, 755, 379, 518, 678, 37, 42, 101, 172, 208, 224, 234, 268, 314, 321, 361, 380, 449, 758, 756, 40, 542, 755, 757, 500, 766, 544, - 171, 503, 514, 514, 908, 964, 964, 964, 497, 223, + 171, 503, 514, 514, 909, 965, 965, 965, 497, 223, 516, 289, 4, 6, 7, 8, 9, 10, 39, 54, 56, 57, 65, 71, 72, 83, 112, 115, 117, 136, 152, 159, 164, 181, 182, 215, 216, 218, 246, 267, @@ -18921,248 +18955,248 @@ static const yytype_uint16 yystos[] = 435, 461, 498, 505, 506, 507, 512, 514, 519, 520, 522, 523, 542, 544, 675, 729, 778, 781, 784, 785, 786, 788, 789, 790, 791, 793, 794, 809, 811, 812, - 813, 814, 815, 816, 817, 819, 820, 835, 836, 847, - 869, 874, 882, 883, 884, 896, 897, 898, 881, 883, - 940, 940, 544, 940, 497, 171, 428, 503, 516, 545, - 761, 954, 3, 170, 172, 464, 935, 949, 951, 170, - 952, 809, 853, 854, 687, 518, 514, 904, 515, 515, - 515, 528, 171, 294, 559, 954, 30, 132, 693, 693, - 59, 693, 161, 166, 237, 286, 704, 706, 707, 732, - 734, 735, 736, 180, 289, 454, 289, 698, 699, 514, - 541, 417, 968, 497, 223, 152, 26, 32, 137, 293, - 350, 354, 386, 457, 534, 537, 538, 350, 152, 40, - 60, 106, 201, 250, 260, 272, 304, 350, 356, 381, - 386, 400, 537, 587, 590, 152, 350, 386, 537, 152, - 350, 386, 537, 152, 40, 961, 809, 875, 547, 548, - 546, 3, 30, 37, 42, 49, 55, 81, 83, 89, - 101, 132, 170, 172, 175, 176, 193, 208, 221, 222, - 224, 234, 236, 246, 268, 277, 299, 301, 321, 361, - 380, 391, 409, 418, 438, 462, 464, 515, 809, 856, - 857, 899, 905, 977, 982, 809, 416, 541, 542, 515, - 514, 633, 370, 580, 644, 274, 911, 40, 191, 542, - 579, 466, 191, 541, 191, 541, 976, 191, 541, 191, - 541, 89, 916, 152, 480, 90, 129, 307, 421, 191, - 542, 152, 516, 903, 63, 357, 518, 650, 152, 518, - 650, 152, 289, 583, 584, 809, 905, 352, 515, 518, - 4, 159, 289, 435, 505, 506, 544, 589, 592, 898, - 930, 932, 933, 936, 931, 428, 514, 664, 668, 171, - 809, 854, 514, 3, 68, 69, 109, 110, 113, 114, - 188, 189, 252, 253, 254, 255, 256, 257, 258, 259, - 262, 263, 375, 376, 470, 471, 494, 495, 544, 796, - 797, 798, 799, 800, 801, 802, 803, 804, 805, 806, - 807, 859, 860, 756, 757, 809, 541, 809, 861, 505, - 506, 542, 810, 811, 836, 847, 863, 514, 809, 853, - 864, 809, 58, 171, 232, 429, 809, 854, 867, 809, - 515, 543, 514, 418, 712, 713, 713, 695, 696, 744, - 219, 690, 224, 37, 224, 380, 758, 224, 299, 759, - 744, 759, 224, 758, 514, 224, 759, 224, 148, 199, - 746, 224, 713, 514, 543, 514, 713, 296, 542, 544, - 947, 809, 942, 944, 856, 3, 37, 42, 49, 54, - 55, 56, 57, 71, 72, 81, 83, 89, 101, 112, - 115, 164, 170, 172, 176, 193, 208, 215, 216, 218, - 221, 222, 224, 234, 236, 246, 267, 268, 269, 277, - 282, 299, 301, 321, 339, 361, 365, 380, 387, 391, - 394, 409, 418, 426, 427, 438, 444, 461, 464, 772, - 774, 775, 777, 779, 781, 783, 785, 786, 787, 789, - 790, 793, 794, 858, 901, 977, 980, 40, 235, 542, - 514, 512, 675, 463, 792, 809, 873, 792, 792, 514, - 514, 780, 780, 324, 675, 514, 782, 520, 71, 72, - 792, 809, 780, 514, 514, 478, 500, 514, 795, 514, - 795, 809, 809, 809, 148, 885, 886, 809, 854, 855, - 675, 809, 853, 543, 837, 838, 839, 9, 548, 516, - 545, 876, 545, 514, 544, 514, 514, 544, 898, 3, - 8, 11, 15, 16, 17, 18, 19, 20, 21, 36, - 40, 46, 53, 80, 176, 193, 198, 221, 222, 236, - 274, 277, 291, 294, 391, 498, 501, 502, 503, 505, - 506, 507, 508, 509, 510, 845, 846, 847, 849, 879, - 477, 821, 301, 809, 518, 690, 514, 544, 690, 3, - 117, 240, 544, 589, 794, 943, 104, 944, 944, 542, - 40, 542, 515, 518, 929, 518, 515, 688, 892, 893, - 40, 954, 192, 352, 219, 386, 677, 677, 30, 700, - 701, 809, 59, 677, 694, 163, 271, 720, 226, 272, - 338, 389, 451, 4, 9, 30, 715, 809, 505, 506, - 716, 717, 809, 811, 732, 733, 707, 706, 704, 705, - 166, 735, 284, 737, 59, 683, 684, 685, 747, 810, - 883, 883, 704, 732, 854, 904, 235, 541, 74, 82, - 93, 168, 191, 328, 445, 542, 615, 625, 640, 82, - 93, 550, 93, 550, 514, 428, 514, 613, 244, 448, - 613, 93, 518, 428, 541, 3, 777, 589, 59, 591, - 589, 589, 106, 250, 260, 59, 428, 476, 500, 588, - 265, 370, 588, 590, 761, 93, 428, 550, 370, 541, - 428, 370, 960, 542, 664, 513, 524, 856, 856, 857, - 518, 698, 699, 13, 14, 221, 221, 428, 428, 542, - 632, 637, 476, 667, 541, 377, 342, 381, 416, 466, - 653, 152, 100, 576, 598, 912, 913, 972, 144, 774, - 274, 198, 581, 541, 274, 577, 587, 274, 514, 633, - 40, 274, 633, 274, 514, 654, 191, 542, 627, 917, - 548, 152, 171, 594, 649, 547, 516, 902, 892, 895, - 895, 902, 515, 518, 13, 931, 937, 4, 898, 4, - 898, 544, 548, 666, 673, 55, 102, 123, 141, 145, - 167, 170, 186, 279, 287, 309, 336, 670, 947, 40, - 515, 809, 515, 171, 518, 515, 317, 862, 515, 810, - 810, 11, 15, 16, 19, 20, 21, 198, 221, 291, - 501, 502, 503, 505, 506, 507, 508, 509, 510, 847, - 810, 515, 763, 764, 818, 166, 171, 865, 866, 518, - 515, 40, 867, 854, 867, 867, 171, 515, 40, 755, - 514, 893, 4, 9, 542, 708, 710, 711, 883, 881, - 177, 240, 416, 420, 422, 448, 541, 691, 473, 767, - 744, 744, 224, 744, 289, 454, 760, 744, 224, 883, - 744, 744, 281, 281, 514, 744, 543, 768, 769, 514, - 543, 768, 518, 515, 518, 516, 514, 777, 514, 514, - 516, 39, 776, 514, 796, 797, 798, 799, 800, 801, - 802, 803, 804, 805, 806, 807, 808, 515, 518, 780, - 551, 655, 656, 674, 909, 953, 965, 854, 855, 514, - 472, 870, 871, 809, 855, 898, 809, 840, 841, 842, - 843, 792, 792, 8, 15, 16, 19, 20, 21, 501, - 502, 503, 505, 506, 507, 508, 509, 510, 542, 845, - 850, 515, 854, 426, 426, 898, 898, 514, 542, 352, - 890, 166, 513, 515, 518, 524, 518, 521, 507, 546, - 854, 898, 809, 808, 808, 774, 809, 809, 809, 809, - 809, 809, 809, 809, 5, 548, 906, 426, 45, 413, - 880, 902, 809, 809, 514, 675, 868, 132, 159, 274, - 279, 284, 435, 446, 809, 279, 514, 809, 428, 53, - 176, 193, 198, 236, 391, 809, 809, 809, 809, 809, - 809, 809, 809, 809, 809, 30, 38, 396, 844, 512, - 516, 878, 180, 162, 822, 365, 514, 836, 884, 171, - 741, 856, 741, 514, 544, 542, 541, 949, 541, 957, - 809, 518, 515, 249, 274, 689, 454, 956, 541, 553, - 514, 542, 558, 568, 569, 571, 41, 126, 702, 518, - 454, 702, 265, 677, 365, 366, 505, 506, 717, 719, - 811, 389, 226, 290, 312, 312, 518, 509, 4, 718, - 898, 718, 365, 366, 719, 541, 891, 278, 393, 738, - 514, 893, 894, 518, 180, 454, 198, 180, 219, 733, - 705, 515, 350, 537, 514, 191, 625, 895, 226, 274, - 226, 454, 514, 618, 773, 774, 895, 542, 191, 895, - 191, 542, 26, 137, 386, 533, 536, 548, 609, 623, - 895, 548, 617, 636, 895, 534, 895, 350, 386, 537, - 587, 589, 902, 895, 589, 902, 895, 589, 350, 386, - 537, 895, 895, 895, 895, 350, 386, 537, 895, 895, - 544, 506, 809, 875, 698, 698, 698, 462, 857, 192, - 355, 697, 809, 809, 279, 544, 924, 279, 924, 542, - 333, 663, 515, 518, 287, 171, 428, 658, 911, 579, - 466, 541, 541, 976, 541, 541, 541, 294, 644, 514, - 675, 152, 3, 514, 514, 152, 152, 236, 542, 615, - 625, 628, 631, 641, 643, 476, 478, 620, 151, 674, - 152, 476, 918, 152, 515, 856, 40, 274, 289, 542, - 3, 650, 547, 650, 289, 650, 583, 809, 664, 507, - 514, 589, 665, 815, 936, 515, 518, 40, 662, 544, - 662, 274, 279, 336, 662, 59, 662, 774, 515, 809, - 809, 809, 865, 774, 810, 810, 810, 810, 810, 810, - 132, 274, 284, 810, 810, 810, 810, 810, 810, 810, - 810, 810, 810, 515, 518, 40, 765, 809, 809, 866, - 865, 774, 515, 515, 515, 854, 774, 893, 515, 312, - 509, 312, 366, 509, 514, 514, 690, 420, 422, 420, - 422, 541, 692, 692, 692, 809, 180, 721, 760, 760, - 744, 809, 514, 744, 166, 760, 514, 543, 751, 760, - 774, 515, 518, 768, 515, 942, 3, 858, 39, 776, - 542, 771, 771, 3, 512, 512, 898, 428, 428, 428, - 428, 774, 515, 513, 854, 809, 139, 871, 872, 515, - 515, 515, 524, 518, 521, 516, 515, 515, 497, 497, - 515, 515, 893, 514, 809, 887, 542, 809, 809, 837, - 886, 515, 515, 515, 497, 810, 810, 145, 854, 171, - 132, 159, 279, 284, 435, 446, 514, 145, 850, 809, - 413, 880, 809, 868, 809, 428, 514, 675, 809, 875, - 547, 514, 514, 155, 823, 742, 743, 767, 698, 767, - 898, 808, 904, 904, 249, 514, 743, 473, 955, 40, - 59, 554, 564, 571, 876, 518, 741, 503, 499, 703, - 701, 291, 845, 848, 703, 4, 898, 719, 290, 451, - 716, 518, 243, 893, 683, 59, 883, 514, 543, 59, - 265, 428, 809, 274, 640, 514, 152, 514, 618, 202, - 637, 638, 599, 40, 175, 608, 634, 599, 26, 137, - 354, 356, 386, 530, 531, 532, 538, 539, 152, 650, - 152, 650, 609, 623, 609, 515, 518, 544, 602, 503, - 516, 515, 518, 428, 370, 93, 428, 550, 370, 428, - 428, 428, 370, 961, 524, 513, 524, 697, 697, 697, - 857, 281, 281, 515, 514, 657, 3, 403, 404, 544, - 672, 632, 663, 581, 541, 577, 514, 40, 633, 654, - 911, 352, 416, 544, 573, 574, 579, 673, 637, 541, - 541, 976, 541, 515, 518, 287, 613, 287, 289, 612, - 895, 476, 975, 541, 613, 40, 541, 515, 416, 809, - 152, 541, 594, 902, 660, 671, 936, 666, 544, 544, - 279, 637, 507, 637, 544, 507, 637, 544, 515, 515, - 866, 171, 132, 284, 514, 766, 763, 514, 515, 515, - 515, 542, 708, 767, 692, 692, 692, 692, 541, 541, - 541, 59, 185, 730, 760, 893, 514, 748, 749, 750, - 812, 814, 893, 166, 80, 770, 769, 515, 515, 512, - 774, 515, 518, 515, 898, 513, 898, 515, 797, 799, - 800, 801, 800, 801, 801, 515, 424, 809, 143, 809, - 840, 850, 795, 795, 515, 809, 887, 888, 889, 40, - 198, 515, 890, 808, 809, 36, 36, 809, 515, 809, - 171, 514, 858, 809, 515, 145, 810, 810, 145, 145, - 809, 809, 513, 524, 514, 877, 699, 473, 809, 300, - 827, 518, 721, 697, 721, 515, 909, 809, 358, 562, - 542, 265, 320, 117, 303, 514, 552, 674, 515, 518, - 558, 955, 809, 163, 230, 514, 703, 290, 541, 515, - 894, 180, 675, 676, 883, 894, 895, 895, 515, 152, - 638, 625, 638, 599, 627, 518, 515, 119, 206, 272, - 274, 624, 514, 33, 59, 645, 634, 74, 80, 93, - 117, 119, 206, 274, 279, 328, 344, 445, 454, 604, - 605, 619, 175, 117, 190, 274, 613, 588, 107, 117, - 175, 274, 402, 405, 590, 613, 386, 532, 439, 895, - 542, 536, 3, 37, 42, 49, 55, 81, 83, 89, - 101, 170, 172, 175, 176, 193, 208, 221, 222, 224, - 234, 236, 246, 268, 273, 277, 291, 299, 301, 321, - 361, 380, 387, 391, 409, 418, 438, 444, 464, 505, - 506, 544, 589, 600, 639, 774, 848, 899, 977, 983, - 548, 636, 895, 895, 895, 895, 895, 895, 895, 895, - 895, 895, 664, 875, 875, 515, 515, 515, 698, 107, - 370, 516, 588, 672, 514, 514, 631, 674, 918, 40, - 644, 191, 541, 515, 518, 581, 515, 515, 577, 514, - 40, 622, 620, 628, 86, 585, 107, 272, 633, 674, - 654, 674, 627, 454, 915, 650, 515, 518, 637, 810, - 171, 514, 858, 768, 515, 518, 515, 721, 541, 541, - 541, 541, 30, 103, 181, 364, 514, 722, 723, 724, - 725, 726, 727, 728, 809, 809, 475, 824, 515, 811, - 851, 852, 198, 180, 745, 749, 515, 751, 752, 753, - 902, 776, 898, 776, 542, 776, 513, 513, 809, 518, - 515, 542, 809, 811, 809, 809, 809, 858, 515, 809, - 36, 36, 809, 809, 145, 515, 506, 875, 515, 856, - 515, 809, 515, 514, 542, 828, 730, 515, 730, 544, - 515, 882, 460, 415, 453, 563, 542, 557, 567, 289, - 560, 503, 571, 562, 850, 59, 515, 515, 459, 460, - 680, 599, 625, 515, 515, 476, 630, 120, 194, 204, - 119, 456, 809, 117, 40, 514, 902, 895, 810, 120, - 194, 119, 279, 226, 541, 630, 88, 645, 191, 279, - 589, 809, 645, 279, 505, 506, 592, 542, 774, 650, - 650, 3, 246, 409, 899, 903, 503, 428, 428, 513, - 513, 697, 515, 515, 542, 664, 454, 659, 661, 673, - 637, 515, 975, 40, 416, 809, 416, 274, 514, 544, - 514, 918, 631, 151, 674, 149, 200, 612, 122, 137, - 327, 975, 107, 918, 476, 973, 40, 289, 542, 914, - 514, 671, 810, 858, 515, 515, 9, 351, 714, 730, - 514, 388, 514, 515, 518, 542, 825, 826, 335, 731, - 518, 515, 514, 543, 59, 515, 198, 515, 752, 513, - 774, 887, 513, 191, 515, 809, 809, 809, 524, 513, - 524, 515, 515, 542, 829, 824, 544, 824, 518, 459, - 876, 515, 518, 91, 562, 809, 515, 894, 894, 344, - 630, 514, 621, 599, 515, 190, 514, 809, 274, 605, - 630, 633, 895, 40, 152, 770, 903, 509, 600, 895, - 895, 515, 588, 124, 515, 515, 620, 674, 674, 541, - 152, 673, 40, 515, 895, 975, 30, 85, 94, 118, - 190, 203, 402, 405, 616, 616, 366, 366, 40, 64, - 74, 240, 416, 809, 541, 514, 542, 561, 570, 818, - 515, 515, 514, 824, 854, 514, 854, 724, 40, 518, - 809, 454, 709, 811, 883, 893, 756, 514, 756, 809, - 875, 875, 309, 830, 731, 731, 674, 303, 674, 557, - 289, 514, 555, 541, 599, 548, 626, 629, 406, 468, - 606, 607, 514, 601, 809, 515, 248, 642, 190, 454, - 535, 509, 439, 664, 544, 918, 612, 973, 514, 541, - 515, 674, 620, 585, 674, 74, 292, 74, 674, 915, - 809, 80, 565, 515, 518, 565, 9, 731, 515, 723, - 515, 828, 826, 368, 515, 883, 513, 513, 513, 59, - 698, 709, 709, 563, 93, 570, 133, 633, 503, 515, - 518, 587, 515, 272, 614, 172, 308, 392, 289, 610, - 611, 635, 601, 809, 439, 40, 514, 973, 612, 975, - 973, 292, 292, 514, 515, 902, 566, 902, 918, 561, - 566, 515, 709, 515, 711, 515, 853, 183, 337, 366, - 831, 459, 895, 515, 275, 451, 642, 600, 629, 515, - 607, 204, 122, 451, 289, 635, 289, 610, 674, 570, - 565, 702, 767, 702, 53, 104, 441, 809, 832, 833, - 832, 832, 515, 674, 767, 386, 611, 63, 272, 357, - 386, 603, 603, 973, 515, 566, 703, 703, 833, 365, - 165, 323, 165, 323, 148, 834, 834, 834, 569, 599, - 25, 117, 279, 918, 702, 36, 104, 180, 272, 425, - 767, 767, 703, 833, 365, 297 + 813, 814, 815, 816, 817, 818, 820, 821, 836, 837, + 848, 870, 875, 883, 884, 885, 897, 898, 899, 882, + 884, 941, 941, 544, 941, 497, 171, 428, 503, 516, + 545, 761, 955, 3, 170, 172, 464, 936, 950, 952, + 170, 953, 809, 854, 855, 687, 518, 514, 905, 515, + 515, 515, 528, 171, 294, 559, 955, 30, 132, 693, + 693, 59, 693, 161, 166, 237, 286, 704, 706, 707, + 732, 734, 735, 736, 180, 289, 454, 289, 698, 699, + 514, 541, 417, 969, 497, 223, 152, 26, 32, 137, + 293, 350, 354, 386, 457, 534, 537, 538, 350, 152, + 40, 60, 106, 201, 250, 260, 272, 304, 350, 356, + 381, 386, 400, 537, 587, 590, 152, 350, 386, 537, + 152, 350, 386, 537, 152, 40, 962, 809, 876, 547, + 548, 546, 3, 30, 37, 42, 49, 55, 81, 83, + 89, 101, 132, 170, 172, 175, 176, 193, 208, 221, + 222, 224, 234, 236, 246, 268, 277, 299, 301, 321, + 361, 380, 391, 409, 418, 438, 462, 464, 515, 809, + 857, 858, 900, 906, 978, 983, 809, 416, 541, 542, + 515, 514, 633, 370, 580, 644, 274, 912, 40, 191, + 542, 579, 466, 191, 541, 191, 541, 977, 191, 541, + 191, 541, 89, 917, 152, 480, 90, 129, 307, 421, + 191, 542, 152, 516, 904, 63, 357, 518, 650, 152, + 518, 650, 152, 289, 583, 584, 809, 906, 352, 515, + 518, 4, 159, 289, 435, 505, 506, 544, 589, 592, + 899, 931, 933, 934, 937, 932, 428, 514, 664, 668, + 171, 809, 855, 514, 3, 68, 69, 109, 110, 113, + 114, 188, 189, 252, 253, 254, 255, 256, 257, 258, + 259, 262, 263, 375, 376, 470, 471, 494, 495, 544, + 796, 797, 798, 799, 800, 801, 802, 803, 804, 805, + 806, 807, 860, 861, 756, 757, 809, 541, 809, 862, + 505, 506, 542, 810, 811, 837, 848, 864, 514, 809, + 854, 865, 809, 58, 171, 232, 429, 809, 855, 868, + 809, 515, 543, 514, 418, 712, 713, 713, 695, 696, + 744, 219, 690, 224, 37, 224, 380, 758, 224, 299, + 759, 744, 759, 224, 758, 514, 224, 759, 224, 148, + 199, 746, 224, 713, 514, 543, 514, 713, 296, 542, + 544, 948, 809, 943, 945, 857, 3, 37, 42, 49, + 54, 55, 56, 57, 71, 72, 81, 83, 89, 101, + 112, 115, 164, 170, 172, 176, 193, 208, 215, 216, + 218, 221, 222, 224, 234, 236, 246, 267, 268, 269, + 277, 282, 299, 301, 321, 339, 361, 365, 380, 387, + 391, 394, 409, 418, 426, 427, 438, 444, 461, 464, + 772, 774, 775, 777, 779, 781, 783, 785, 786, 787, + 789, 790, 793, 794, 859, 902, 978, 981, 40, 235, + 542, 514, 512, 675, 463, 792, 809, 874, 792, 792, + 514, 514, 780, 780, 324, 675, 514, 782, 520, 71, + 72, 792, 809, 780, 514, 514, 478, 500, 514, 795, + 514, 795, 809, 809, 809, 148, 886, 887, 809, 855, + 856, 675, 809, 854, 543, 838, 839, 840, 9, 548, + 516, 545, 877, 545, 514, 544, 514, 514, 544, 899, + 3, 8, 11, 15, 16, 17, 18, 19, 20, 21, + 36, 40, 46, 53, 80, 176, 193, 198, 221, 222, + 236, 274, 277, 291, 294, 391, 498, 501, 502, 503, + 505, 506, 507, 508, 509, 510, 846, 847, 848, 850, + 880, 477, 822, 301, 809, 518, 690, 514, 544, 690, + 3, 117, 240, 544, 589, 794, 944, 104, 945, 945, + 542, 40, 542, 515, 518, 930, 518, 515, 688, 893, + 894, 40, 955, 192, 352, 219, 386, 677, 677, 30, + 700, 701, 809, 59, 677, 694, 163, 271, 720, 226, + 272, 338, 389, 451, 4, 9, 30, 715, 809, 505, + 506, 716, 717, 809, 811, 732, 733, 707, 706, 704, + 705, 166, 735, 284, 737, 59, 683, 684, 685, 747, + 810, 884, 884, 704, 732, 855, 905, 235, 541, 74, + 82, 93, 168, 191, 328, 445, 542, 615, 625, 640, + 82, 93, 550, 93, 550, 514, 428, 514, 613, 244, + 448, 613, 93, 518, 428, 541, 3, 777, 589, 59, + 591, 589, 589, 106, 250, 260, 59, 428, 476, 500, + 588, 265, 370, 588, 590, 761, 93, 428, 550, 370, + 541, 428, 370, 961, 542, 664, 513, 524, 857, 857, + 858, 518, 698, 699, 13, 14, 221, 221, 428, 428, + 542, 632, 637, 476, 667, 541, 377, 342, 381, 416, + 466, 653, 152, 100, 576, 598, 913, 914, 973, 144, + 774, 274, 198, 581, 541, 274, 577, 587, 274, 514, + 633, 40, 274, 633, 274, 514, 654, 191, 542, 627, + 918, 548, 152, 171, 594, 649, 547, 516, 903, 893, + 896, 896, 903, 515, 518, 13, 932, 938, 4, 899, + 4, 899, 544, 548, 666, 673, 55, 102, 123, 141, + 145, 167, 170, 186, 279, 287, 309, 336, 670, 948, + 40, 515, 809, 515, 171, 518, 515, 317, 863, 515, + 810, 810, 11, 15, 16, 19, 20, 21, 198, 221, + 291, 501, 502, 503, 505, 506, 507, 508, 509, 510, + 848, 810, 515, 763, 764, 819, 166, 171, 866, 867, + 518, 515, 40, 868, 855, 868, 868, 171, 515, 40, + 755, 514, 894, 4, 9, 542, 708, 710, 711, 884, + 882, 177, 240, 416, 420, 422, 448, 541, 691, 473, + 767, 744, 744, 224, 744, 289, 454, 760, 744, 224, + 884, 744, 744, 281, 281, 514, 744, 543, 768, 769, + 514, 543, 768, 518, 515, 518, 516, 514, 777, 514, + 514, 516, 39, 776, 514, 796, 797, 798, 799, 800, + 801, 802, 803, 804, 805, 806, 807, 808, 515, 518, + 780, 551, 655, 656, 674, 910, 954, 966, 855, 856, + 514, 472, 871, 872, 809, 856, 899, 809, 841, 842, + 843, 844, 792, 792, 8, 15, 16, 19, 20, 21, + 501, 502, 503, 505, 506, 507, 508, 509, 510, 542, + 846, 851, 515, 855, 426, 426, 899, 899, 514, 542, + 352, 891, 166, 513, 515, 518, 524, 518, 521, 507, + 546, 855, 899, 809, 808, 808, 774, 809, 809, 809, + 809, 809, 809, 809, 809, 5, 548, 907, 426, 45, + 413, 881, 903, 809, 809, 514, 675, 869, 132, 159, + 274, 279, 284, 435, 446, 809, 279, 514, 809, 428, + 53, 176, 193, 198, 236, 391, 809, 809, 809, 809, + 809, 809, 809, 809, 809, 809, 30, 38, 396, 845, + 512, 516, 879, 180, 162, 823, 365, 514, 837, 885, + 171, 741, 857, 741, 514, 544, 542, 541, 950, 541, + 958, 809, 518, 515, 249, 274, 689, 454, 957, 541, + 553, 514, 542, 558, 568, 569, 571, 41, 126, 702, + 518, 454, 702, 265, 677, 365, 366, 505, 506, 717, + 719, 811, 389, 226, 290, 312, 312, 518, 509, 4, + 718, 899, 718, 365, 366, 719, 541, 892, 278, 393, + 738, 514, 894, 895, 518, 180, 454, 198, 180, 219, + 733, 705, 515, 350, 537, 514, 191, 625, 896, 226, + 274, 226, 454, 514, 618, 773, 774, 896, 542, 191, + 896, 191, 542, 26, 137, 386, 533, 536, 548, 609, + 623, 896, 548, 617, 636, 896, 534, 896, 350, 386, + 537, 587, 589, 903, 896, 589, 903, 896, 589, 350, + 386, 537, 896, 896, 896, 896, 350, 386, 537, 896, + 896, 544, 506, 809, 876, 698, 698, 698, 462, 858, + 192, 355, 697, 809, 809, 279, 544, 925, 279, 925, + 542, 333, 663, 515, 518, 287, 171, 428, 658, 912, + 579, 466, 541, 541, 977, 541, 541, 541, 294, 644, + 514, 675, 152, 3, 514, 514, 152, 152, 236, 542, + 615, 625, 628, 631, 641, 643, 476, 478, 620, 151, + 674, 152, 476, 919, 152, 515, 857, 40, 274, 289, + 542, 3, 650, 547, 650, 289, 650, 583, 809, 664, + 507, 512, 514, 589, 665, 815, 816, 937, 515, 518, + 40, 662, 544, 662, 274, 279, 336, 662, 59, 662, + 774, 515, 809, 809, 809, 866, 774, 810, 810, 810, + 810, 810, 810, 132, 274, 284, 810, 810, 810, 810, + 810, 810, 810, 810, 810, 810, 515, 518, 40, 765, + 809, 809, 867, 866, 774, 515, 515, 515, 855, 774, + 894, 515, 312, 509, 312, 366, 509, 514, 514, 690, + 420, 422, 420, 422, 541, 692, 692, 692, 809, 180, + 721, 760, 760, 744, 809, 514, 744, 166, 760, 514, + 543, 751, 760, 774, 515, 518, 768, 515, 943, 3, + 859, 39, 776, 542, 771, 771, 3, 512, 512, 899, + 428, 428, 428, 428, 774, 515, 513, 855, 809, 139, + 872, 873, 515, 515, 515, 524, 518, 521, 516, 515, + 515, 497, 497, 515, 515, 894, 514, 809, 888, 542, + 809, 809, 838, 887, 515, 515, 515, 497, 810, 810, + 145, 855, 171, 132, 159, 279, 284, 435, 446, 514, + 145, 851, 809, 413, 881, 809, 869, 809, 428, 514, + 675, 809, 876, 547, 514, 514, 155, 824, 742, 743, + 767, 698, 767, 899, 808, 905, 905, 249, 514, 743, + 473, 956, 40, 59, 554, 564, 571, 877, 518, 741, + 503, 499, 703, 701, 291, 846, 849, 703, 4, 899, + 719, 290, 451, 716, 518, 243, 894, 683, 59, 884, + 514, 543, 59, 265, 428, 809, 274, 640, 514, 152, + 514, 618, 202, 637, 638, 599, 40, 175, 608, 634, + 599, 26, 137, 354, 356, 386, 530, 531, 532, 538, + 539, 152, 650, 152, 650, 609, 623, 609, 515, 518, + 544, 602, 503, 516, 515, 518, 428, 370, 93, 428, + 550, 370, 428, 428, 428, 370, 962, 524, 513, 524, + 697, 697, 697, 858, 281, 281, 515, 514, 657, 3, + 403, 404, 544, 672, 632, 663, 581, 541, 577, 514, + 40, 633, 654, 912, 352, 416, 544, 573, 574, 579, + 673, 637, 541, 541, 977, 541, 515, 518, 287, 613, + 287, 289, 612, 896, 476, 976, 541, 613, 40, 541, + 515, 416, 809, 152, 541, 594, 903, 660, 671, 937, + 666, 544, 544, 279, 637, 507, 637, 544, 507, 637, + 544, 515, 515, 867, 171, 132, 284, 514, 766, 763, + 514, 515, 515, 515, 542, 708, 767, 692, 692, 692, + 692, 541, 541, 541, 59, 185, 730, 760, 894, 514, + 748, 749, 750, 812, 814, 894, 166, 80, 770, 769, + 515, 515, 512, 774, 515, 518, 515, 899, 513, 899, + 515, 797, 799, 800, 801, 800, 801, 801, 515, 424, + 809, 143, 809, 841, 851, 795, 795, 515, 809, 888, + 889, 890, 40, 198, 515, 891, 808, 809, 36, 36, + 809, 515, 809, 171, 514, 859, 809, 515, 145, 810, + 810, 145, 145, 809, 809, 513, 524, 514, 878, 699, + 473, 809, 300, 828, 518, 721, 697, 721, 515, 910, + 809, 358, 562, 542, 265, 320, 117, 303, 514, 552, + 674, 515, 518, 558, 956, 809, 163, 230, 514, 703, + 290, 541, 515, 895, 180, 675, 676, 884, 895, 896, + 896, 515, 152, 638, 625, 638, 599, 627, 518, 515, + 119, 206, 272, 274, 624, 514, 33, 59, 645, 634, + 74, 80, 93, 117, 119, 206, 274, 279, 328, 344, + 445, 454, 604, 605, 619, 175, 117, 190, 274, 613, + 588, 107, 117, 175, 274, 402, 405, 590, 613, 386, + 532, 439, 896, 542, 536, 3, 37, 42, 49, 55, + 81, 83, 89, 101, 170, 172, 175, 176, 193, 208, + 221, 222, 224, 234, 236, 246, 268, 273, 277, 291, + 299, 301, 321, 361, 380, 387, 391, 409, 418, 438, + 444, 464, 505, 506, 544, 589, 600, 639, 774, 849, + 900, 978, 984, 548, 636, 896, 896, 896, 896, 896, + 896, 896, 896, 896, 896, 664, 876, 876, 515, 515, + 515, 698, 107, 370, 516, 588, 672, 514, 514, 631, + 674, 919, 40, 644, 191, 541, 515, 518, 581, 515, + 515, 577, 514, 40, 622, 620, 628, 86, 585, 107, + 272, 633, 674, 654, 674, 627, 454, 916, 650, 515, + 518, 637, 810, 171, 514, 859, 768, 515, 518, 515, + 721, 541, 541, 541, 541, 30, 103, 181, 364, 514, + 722, 723, 724, 725, 726, 727, 728, 809, 809, 475, + 825, 515, 811, 852, 853, 198, 180, 745, 749, 515, + 751, 752, 753, 903, 776, 899, 776, 542, 776, 513, + 513, 809, 518, 515, 542, 809, 811, 809, 809, 809, + 859, 515, 809, 36, 36, 809, 809, 145, 515, 506, + 876, 515, 857, 515, 809, 515, 514, 542, 829, 730, + 515, 730, 544, 515, 883, 460, 415, 453, 563, 542, + 557, 567, 289, 560, 503, 571, 562, 851, 59, 515, + 515, 459, 460, 680, 599, 625, 515, 515, 476, 630, + 120, 194, 204, 119, 456, 809, 117, 40, 514, 903, + 896, 810, 120, 194, 119, 279, 226, 541, 630, 88, + 645, 191, 279, 589, 809, 645, 279, 505, 506, 592, + 542, 774, 650, 650, 3, 246, 409, 900, 904, 503, + 428, 428, 513, 513, 697, 515, 515, 542, 664, 454, + 659, 661, 673, 637, 515, 976, 40, 416, 809, 416, + 274, 514, 544, 514, 919, 631, 151, 674, 149, 200, + 612, 122, 137, 327, 976, 107, 919, 476, 974, 40, + 289, 542, 915, 514, 671, 810, 859, 515, 515, 9, + 351, 714, 730, 514, 388, 514, 515, 518, 542, 826, + 827, 335, 731, 518, 515, 514, 543, 59, 515, 198, + 515, 752, 513, 774, 888, 513, 191, 515, 809, 809, + 809, 524, 513, 524, 515, 515, 542, 830, 825, 544, + 825, 518, 459, 877, 515, 518, 91, 562, 809, 515, + 895, 895, 344, 630, 514, 621, 599, 515, 190, 514, + 809, 274, 605, 630, 633, 896, 40, 152, 770, 904, + 509, 600, 896, 896, 515, 588, 124, 515, 515, 620, + 674, 674, 541, 152, 673, 40, 515, 896, 976, 30, + 85, 94, 118, 190, 203, 402, 405, 616, 616, 366, + 366, 40, 64, 74, 240, 416, 809, 541, 514, 542, + 561, 570, 819, 515, 515, 514, 825, 855, 514, 855, + 724, 40, 518, 809, 454, 709, 811, 884, 894, 756, + 514, 756, 809, 876, 876, 309, 831, 731, 731, 674, + 303, 674, 557, 289, 514, 555, 541, 599, 548, 626, + 629, 406, 468, 606, 607, 514, 601, 809, 515, 248, + 642, 190, 454, 535, 509, 439, 664, 544, 919, 612, + 974, 514, 541, 515, 674, 620, 585, 674, 74, 292, + 74, 674, 916, 809, 80, 565, 515, 518, 565, 9, + 731, 515, 723, 515, 829, 827, 368, 515, 884, 513, + 513, 513, 59, 698, 709, 709, 563, 93, 570, 133, + 633, 503, 515, 518, 587, 515, 272, 614, 172, 308, + 392, 289, 610, 611, 635, 601, 809, 439, 40, 514, + 974, 612, 976, 974, 292, 292, 514, 515, 903, 566, + 903, 919, 561, 566, 515, 709, 515, 711, 515, 854, + 183, 337, 366, 832, 459, 896, 515, 275, 451, 642, + 600, 629, 515, 607, 204, 122, 451, 289, 635, 289, + 610, 674, 570, 565, 702, 767, 702, 53, 104, 441, + 809, 833, 834, 833, 833, 515, 674, 767, 386, 611, + 63, 272, 357, 386, 603, 603, 974, 515, 566, 703, + 703, 834, 365, 165, 323, 165, 323, 148, 835, 835, + 835, 569, 599, 25, 117, 279, 919, 702, 36, 104, + 180, 272, 425, 767, 767, 703, 834, 365, 297 }; #define yyerrok (yyerrstatus = 0) @@ -23500,190 +23534,190 @@ YYLTYPE yylloc; case 475: #line 118 "third_party/libpg_query/grammar/statements/copy.y" - { (yyval.node) = (PGNode *) makeNode(PGAStar); ;} + { (yyval.node) = (PGNode *) (yyvsp[(1) - (1)].node); ;} break; case 476: #line 119 "third_party/libpg_query/grammar/statements/copy.y" - { (yyval.node) = (PGNode *) (yyvsp[(2) - (3)].list); ;} + { (yyval.node) = (PGNode *) makeNode(PGAStar); ;} break; case 477: #line 120 "third_party/libpg_query/grammar/statements/copy.y" - { (yyval.node) = (PGNode *) (yyvsp[(1) - (1)].node); ;} + { (yyval.node) = (PGNode *) (yyvsp[(2) - (3)].list); ;} break; case 478: #line 121 "third_party/libpg_query/grammar/statements/copy.y" - { (yyval.node) = NULL; ;} + { (yyval.node) = (PGNode *) (yyvsp[(1) - (1)].node); ;} break; case 479: -#line 127 "third_party/libpg_query/grammar/statements/copy.y" +#line 122 "third_party/libpg_query/grammar/statements/copy.y" + { (yyval.node) = NULL; ;} + break; + + case 480: +#line 128 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem((yyvsp[(1) - (2)].str), (yyvsp[(2) - (2)].node), (yylsp[(1) - (2)])); ;} break; - case 480: -#line 135 "third_party/libpg_query/grammar/statements/copy.y" + case 481: +#line 136 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("oids", (PGNode *)makeInteger(true), (yylsp[(1) - (2)])); ;} break; - case 481: -#line 138 "third_party/libpg_query/grammar/statements/copy.y" + case 482: +#line 139 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = NULL; ;} break; - case 482: -#line 143 "third_party/libpg_query/grammar/statements/copy.y" + case 483: +#line 144 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.list) = lappend((yyvsp[(1) - (2)].list), (yyvsp[(2) - (2)].defelt)); ;} break; - case 483: -#line 144 "third_party/libpg_query/grammar/statements/copy.y" + case 484: +#line 145 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.list) = NIL; ;} break; - case 484: -#line 150 "third_party/libpg_query/grammar/statements/copy.y" + case 485: +#line 151 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("format", (PGNode *)makeString("binary"), (yylsp[(1) - (1)])); ;} break; - case 485: -#line 153 "third_party/libpg_query/grammar/statements/copy.y" + case 486: +#line 154 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = NULL; ;} break; - case 486: -#line 159 "third_party/libpg_query/grammar/statements/copy.y" + case 487: +#line 160 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("format", (PGNode *)makeString("binary"), (yylsp[(1) - (1)])); ;} break; - case 487: -#line 163 "third_party/libpg_query/grammar/statements/copy.y" + case 488: +#line 164 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("oids", (PGNode *)makeInteger(true), (yylsp[(1) - (1)])); ;} break; - case 488: -#line 167 "third_party/libpg_query/grammar/statements/copy.y" + case 489: +#line 168 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("freeze", (PGNode *)makeInteger(true), (yylsp[(1) - (1)])); ;} break; - case 489: -#line 171 "third_party/libpg_query/grammar/statements/copy.y" + case 490: +#line 172 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("delimiter", (PGNode *)makeString((yyvsp[(3) - (3)].str)), (yylsp[(1) - (3)])); ;} break; - case 490: -#line 175 "third_party/libpg_query/grammar/statements/copy.y" + case 491: +#line 176 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("null", (PGNode *)makeString((yyvsp[(3) - (3)].str)), (yylsp[(1) - (3)])); ;} break; - case 491: -#line 179 "third_party/libpg_query/grammar/statements/copy.y" + case 492: +#line 180 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("format", (PGNode *)makeString("csv"), (yylsp[(1) - (1)])); ;} break; - case 492: -#line 183 "third_party/libpg_query/grammar/statements/copy.y" + case 493: +#line 184 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("header", (PGNode *)makeInteger(true), (yylsp[(1) - (1)])); ;} break; - case 493: -#line 187 "third_party/libpg_query/grammar/statements/copy.y" + case 494: +#line 188 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("quote", (PGNode *)makeString((yyvsp[(3) - (3)].str)), (yylsp[(1) - (3)])); ;} break; - case 494: -#line 191 "third_party/libpg_query/grammar/statements/copy.y" + case 495: +#line 192 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("escape", (PGNode *)makeString((yyvsp[(3) - (3)].str)), (yylsp[(1) - (3)])); ;} break; - case 495: -#line 195 "third_party/libpg_query/grammar/statements/copy.y" + case 496: +#line 196 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("force_quote", (PGNode *)(yyvsp[(3) - (3)].list), (yylsp[(1) - (3)])); ;} break; - case 496: -#line 199 "third_party/libpg_query/grammar/statements/copy.y" + case 497: +#line 200 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("force_quote", (PGNode *)makeNode(PGAStar), (yylsp[(1) - (3)])); ;} break; - case 497: -#line 203 "third_party/libpg_query/grammar/statements/copy.y" + case 498: +#line 204 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("partition_by", (PGNode *)(yyvsp[(3) - (3)].list), (yylsp[(1) - (3)])); ;} break; - case 498: -#line 207 "third_party/libpg_query/grammar/statements/copy.y" + case 499: +#line 208 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("partition_by", (PGNode *)makeNode(PGAStar), (yylsp[(1) - (3)])); ;} break; - case 499: -#line 211 "third_party/libpg_query/grammar/statements/copy.y" + case 500: +#line 212 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("force_not_null", (PGNode *)(yyvsp[(4) - (4)].list), (yylsp[(1) - (4)])); ;} break; - case 500: -#line 215 "third_party/libpg_query/grammar/statements/copy.y" + case 501: +#line 216 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("force_null", (PGNode *)(yyvsp[(3) - (3)].list), (yylsp[(1) - (3)])); ;} break; - case 501: -#line 219 "third_party/libpg_query/grammar/statements/copy.y" + case 502: +#line 220 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.defelt) = makeDefElem("encoding", (PGNode *)makeString((yyvsp[(2) - (2)].str)), (yylsp[(1) - (2)])); ;} break; - case 502: -#line 226 "third_party/libpg_query/grammar/statements/copy.y" - { (yyval.node) = (PGNode *) makeString((yyvsp[(1) - (1)].str)); ;} - break; - case 503: -#line 231 "third_party/libpg_query/grammar/statements/copy.y" - { (yyval.str) = (yyvsp[(1) - (1)].str); ;} +#line 227 "third_party/libpg_query/grammar/statements/copy.y" + { (yyval.node) = (PGNode *) makeString((yyvsp[(1) - (1)].str)); ;} break; case 504: #line 232 "third_party/libpg_query/grammar/statements/copy.y" - { (yyval.str) = NULL; ;} + { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; case 505: @@ -23693,51 +23727,56 @@ YYLTYPE yylloc; case 506: #line 234 "third_party/libpg_query/grammar/statements/copy.y" - { (yyval.str) = psprintf("%s.%s", (yyvsp[(1) - (3)].str), (yyvsp[(3) - (3)].str)); ;} + { (yyval.str) = NULL; ;} break; case 507: #line 235 "third_party/libpg_query/grammar/statements/copy.y" - { (yyval.str) = (yyvsp[(1) - (1)].str); ;} + { (yyval.str) = psprintf("%s.%s", (yyvsp[(1) - (3)].str), (yyvsp[(3) - (3)].str)); ;} break; case 508: -#line 242 "third_party/libpg_query/grammar/statements/copy.y" +#line 236 "third_party/libpg_query/grammar/statements/copy.y" + { (yyval.str) = (yyvsp[(1) - (1)].str); ;} + break; + + case 509: +#line 243 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].defelt)); ;} break; - case 509: -#line 246 "third_party/libpg_query/grammar/statements/copy.y" + case 510: +#line 247 "third_party/libpg_query/grammar/statements/copy.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].defelt)); ;} break; - case 512: + case 513: #line 52 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (3)].node); ;} break; - case 513: + case 514: #line 53 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (3)].node); ;} break; - case 514: + case 515: #line 55 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (3)].node); ;} break; - case 515: + case 516: #line 72 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 516: + case 517: #line 74 "third_party/libpg_query/grammar/statements/select.y" { insertSelectOptions((PGSelectStmt *) (yyvsp[(1) - (2)].node), (yyvsp[(2) - (2)].list), NIL, @@ -23747,7 +23786,7 @@ YYLTYPE yylloc; ;} break; - case 517: + case 518: #line 81 "third_party/libpg_query/grammar/statements/select.y" { insertSelectOptions((PGSelectStmt *) (yyvsp[(1) - (4)].node), (yyvsp[(2) - (4)].list), (yyvsp[(3) - (4)].list), @@ -23758,7 +23797,7 @@ YYLTYPE yylloc; ;} break; - case 518: + case 519: #line 89 "third_party/libpg_query/grammar/statements/select.y" { insertSelectOptions((PGSelectStmt *) (yyvsp[(1) - (4)].node), (yyvsp[(2) - (4)].list), (yyvsp[(4) - (4)].list), @@ -23769,7 +23808,7 @@ YYLTYPE yylloc; ;} break; - case 519: + case 520: #line 97 "third_party/libpg_query/grammar/statements/select.y" { insertSelectOptions((PGSelectStmt *) (yyvsp[(2) - (2)].node), NULL, NIL, @@ -23780,7 +23819,7 @@ YYLTYPE yylloc; ;} break; - case 520: + case 521: #line 105 "third_party/libpg_query/grammar/statements/select.y" { insertSelectOptions((PGSelectStmt *) (yyvsp[(2) - (3)].node), (yyvsp[(3) - (3)].list), NIL, @@ -23791,7 +23830,7 @@ YYLTYPE yylloc; ;} break; - case 521: + case 522: #line 113 "third_party/libpg_query/grammar/statements/select.y" { insertSelectOptions((PGSelectStmt *) (yyvsp[(2) - (5)].node), (yyvsp[(3) - (5)].list), (yyvsp[(4) - (5)].list), @@ -23802,7 +23841,7 @@ YYLTYPE yylloc; ;} break; - case 522: + case 523: #line 121 "third_party/libpg_query/grammar/statements/select.y" { insertSelectOptions((PGSelectStmt *) (yyvsp[(2) - (5)].node), (yyvsp[(3) - (5)].list), (yyvsp[(5) - (5)].list), @@ -23813,24 +23852,24 @@ YYLTYPE yylloc; ;} break; - case 523: + case 524: #line 131 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 524: + case 525: #line 132 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 525: + case 526: #line 160 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(3) - (3)].list); ;} break; - case 526: + case 527: #line 164 "third_party/libpg_query/grammar/statements/select.y" { PGAStar *star = makeNode(PGAStar); @@ -23838,7 +23877,7 @@ YYLTYPE yylloc; ;} break; - case 527: + case 528: #line 175 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *n = makeNode(PGSelectStmt); @@ -23855,7 +23894,7 @@ YYLTYPE yylloc; ;} break; - case 528: + case 529: #line 191 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *n = makeNode(PGSelectStmt); @@ -23873,7 +23912,7 @@ YYLTYPE yylloc; ;} break; - case 529: + case 530: #line 208 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *n = makeNode(PGSelectStmt); @@ -23890,7 +23929,7 @@ YYLTYPE yylloc; ;} break; - case 530: + case 531: #line 225 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *n = makeNode(PGSelectStmt); @@ -23908,12 +23947,12 @@ YYLTYPE yylloc; ;} break; - case 531: + case 532: #line 239 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 532: + case 533: #line 241 "third_party/libpg_query/grammar/statements/select.y" { /* same as SELECT * FROM relation_expr */ @@ -23935,35 +23974,35 @@ YYLTYPE yylloc; ;} break; - case 533: + case 534: #line 260 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeSetOp(PG_SETOP_UNION_BY_NAME, (yyvsp[(3) - (5)].boolean), (yyvsp[(1) - (5)].node), (yyvsp[(5) - (5)].node)); ;} break; - case 534: + case 535: #line 264 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeSetOp(PG_SETOP_UNION, (yyvsp[(3) - (4)].boolean), (yyvsp[(1) - (4)].node), (yyvsp[(4) - (4)].node)); ;} break; - case 535: + case 536: #line 268 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeSetOp(PG_SETOP_INTERSECT, (yyvsp[(3) - (4)].boolean), (yyvsp[(1) - (4)].node), (yyvsp[(4) - (4)].node)); ;} break; - case 536: + case 537: #line 272 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeSetOp(PG_SETOP_EXCEPT, (yyvsp[(3) - (4)].boolean), (yyvsp[(1) - (4)].node), (yyvsp[(4) - (4)].node)); ;} break; - case 537: + case 538: #line 276 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *res = makeNode(PGSelectStmt); @@ -23976,7 +24015,7 @@ YYLTYPE yylloc; ;} break; - case 538: + case 539: #line 286 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *res = makeNode(PGSelectStmt); @@ -23990,7 +24029,7 @@ YYLTYPE yylloc; ;} break; - case 539: + case 540: #line 297 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *res = makeNode(PGSelectStmt); @@ -24003,7 +24042,7 @@ YYLTYPE yylloc; ;} break; - case 540: + case 541: #line 307 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *res = makeNode(PGSelectStmt); @@ -24015,7 +24054,7 @@ YYLTYPE yylloc; ;} break; - case 541: + case 542: #line 316 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *res = makeNode(PGSelectStmt); @@ -24029,7 +24068,7 @@ YYLTYPE yylloc; ;} break; - case 542: + case 543: #line 327 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *res = makeNode(PGSelectStmt); @@ -24043,7 +24082,7 @@ YYLTYPE yylloc; ;} break; - case 543: + case 544: #line 338 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *res = makeNode(PGSelectStmt); @@ -24058,7 +24097,7 @@ YYLTYPE yylloc; ;} break; - case 544: + case 545: #line 350 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *res = makeNode(PGSelectStmt); @@ -24076,7 +24115,7 @@ YYLTYPE yylloc; ;} break; - case 545: + case 546: #line 365 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *res = makeNode(PGSelectStmt); @@ -24094,7 +24133,7 @@ YYLTYPE yylloc; ;} break; - case 552: + case 553: #line 395 "third_party/libpg_query/grammar/statements/select.y" { PGPivot *n = makeNode(PGPivot); @@ -24103,7 +24142,7 @@ YYLTYPE yylloc; ;} break; - case 553: + case 554: #line 401 "third_party/libpg_query/grammar/statements/select.y" { PGPivot *n = makeNode(PGPivot); @@ -24113,32 +24152,32 @@ YYLTYPE yylloc; ;} break; - case 554: + case 555: #line 407 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 555: + case 556: #line 411 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 556: + case 557: #line 412 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} break; - case 557: + case 558: #line 416 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 558: + case 559: #line 417 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; - case 559: + case 560: #line 432 "third_party/libpg_query/grammar/statements/select.y" { (yyval.with) = makeNode(PGWithClause); @@ -24148,7 +24187,7 @@ YYLTYPE yylloc; ;} break; - case 560: + case 561: #line 439 "third_party/libpg_query/grammar/statements/select.y" { (yyval.with) = makeNode(PGWithClause); @@ -24158,7 +24197,7 @@ YYLTYPE yylloc; ;} break; - case 561: + case 562: #line 446 "third_party/libpg_query/grammar/statements/select.y" { (yyval.with) = makeNode(PGWithClause); @@ -24168,17 +24207,17 @@ YYLTYPE yylloc; ;} break; - case 562: + case 563: #line 455 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 563: + case 564: #line 456 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} break; - case 564: + case 565: #line 460 "third_party/libpg_query/grammar/statements/select.y" { PGCommonTableExpr *n = makeNode(PGCommonTableExpr); @@ -24191,22 +24230,22 @@ YYLTYPE yylloc; ;} break; - case 565: + case 566: #line 472 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ctematerialize) = PGCTEMaterializeAlways; ;} break; - case 566: + case 567: #line 473 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ctematerialize) = PGCTEMaterializeNever; ;} break; - case 567: + case 568: #line 474 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ctematerialize) = PGCTEMaterializeDefault; ;} break; - case 568: + case 569: #line 479 "third_party/libpg_query/grammar/statements/select.y" { (yyval.into) = makeNode(PGIntoClause); @@ -24219,12 +24258,12 @@ YYLTYPE yylloc; ;} break; - case 569: + case 570: #line 489 "third_party/libpg_query/grammar/statements/select.y" { (yyval.into) = NULL; ;} break; - case 570: + case 571: #line 498 "third_party/libpg_query/grammar/statements/select.y" { (yyval.range) = (yyvsp[(3) - (3)].range); @@ -24232,7 +24271,7 @@ YYLTYPE yylloc; ;} break; - case 571: + case 572: #line 503 "third_party/libpg_query/grammar/statements/select.y" { (yyval.range) = (yyvsp[(3) - (3)].range); @@ -24240,7 +24279,7 @@ YYLTYPE yylloc; ;} break; - case 572: + case 573: #line 508 "third_party/libpg_query/grammar/statements/select.y" { (yyval.range) = (yyvsp[(4) - (4)].range); @@ -24248,7 +24287,7 @@ YYLTYPE yylloc; ;} break; - case 573: + case 574: #line 513 "third_party/libpg_query/grammar/statements/select.y" { (yyval.range) = (yyvsp[(4) - (4)].range); @@ -24256,7 +24295,7 @@ YYLTYPE yylloc; ;} break; - case 574: + case 575: #line 518 "third_party/libpg_query/grammar/statements/select.y" { ereport(PGWARNING, @@ -24267,7 +24306,7 @@ YYLTYPE yylloc; ;} break; - case 575: + case 576: #line 526 "third_party/libpg_query/grammar/statements/select.y" { ereport(PGWARNING, @@ -24278,7 +24317,7 @@ YYLTYPE yylloc; ;} break; - case 576: + case 577: #line 534 "third_party/libpg_query/grammar/statements/select.y" { (yyval.range) = (yyvsp[(3) - (3)].range); @@ -24286,7 +24325,7 @@ YYLTYPE yylloc; ;} break; - case 577: + case 578: #line 539 "third_party/libpg_query/grammar/statements/select.y" { (yyval.range) = (yyvsp[(2) - (2)].range); @@ -24294,7 +24333,7 @@ YYLTYPE yylloc; ;} break; - case 578: + case 579: #line 544 "third_party/libpg_query/grammar/statements/select.y" { (yyval.range) = (yyvsp[(1) - (1)].range); @@ -24302,87 +24341,87 @@ YYLTYPE yylloc; ;} break; - case 579: + case 580: #line 550 "third_party/libpg_query/grammar/statements/select.y" {;} break; - case 580: + case 581: #line 551 "third_party/libpg_query/grammar/statements/select.y" {;} break; - case 581: + case 582: #line 555 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = true; ;} break; - case 582: + case 583: #line 556 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = false; ;} break; - case 583: + case 584: #line 557 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = false; ;} break; - case 584: + case 585: #line 561 "third_party/libpg_query/grammar/statements/select.y" { ;} break; - case 585: + case 586: #line 568 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(NIL); ;} break; - case 586: + case 587: #line 569 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(4) - (5)].list); ;} break; - case 587: + case 588: #line 573 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL;;} break; - case 588: + case 589: #line 574 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 589: + case 590: #line 578 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ignorenulls) = PG_IGNORE_NULLS;;} break; - case 590: + case 591: #line 579 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ignorenulls) = PG_RESPECT_NULLS;;} break; - case 591: + case 592: #line 580 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ignorenulls) = PG_DEFAULT_NULLS; ;} break; - case 592: + case 593: #line 584 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list);;} break; - case 593: + case 594: #line 585 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 594: + case 595: #line 589 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(3) - (3)].list); ;} break; - case 595: + case 596: #line 591 "third_party/libpg_query/grammar/statements/select.y" { PGSortBy *sort = makeNode(PGSortBy); @@ -24398,17 +24437,17 @@ YYLTYPE yylloc; ;} break; - case 596: + case 597: #line 606 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].sortby)); ;} break; - case 597: + case 598: #line 607 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].sortby)); ;} break; - case 598: + case 599: #line 611 "third_party/libpg_query/grammar/statements/select.y" { (yyval.sortby) = makeNode(PGSortBy); @@ -24420,7 +24459,7 @@ YYLTYPE yylloc; ;} break; - case 599: + case 600: #line 620 "third_party/libpg_query/grammar/statements/select.y" { (yyval.sortby) = makeNode(PGSortBy); @@ -24432,72 +24471,72 @@ YYLTYPE yylloc; ;} break; - case 600: + case 601: #line 630 "third_party/libpg_query/grammar/statements/select.y" { (yyval.sortorder) = PG_SORTBY_ASC; ;} break; - case 601: + case 602: #line 631 "third_party/libpg_query/grammar/statements/select.y" { (yyval.sortorder) = PG_SORTBY_DESC; ;} break; - case 602: + case 603: #line 632 "third_party/libpg_query/grammar/statements/select.y" { (yyval.sortorder) = PG_SORTBY_DEFAULT; ;} break; - case 603: + case 604: #line 635 "third_party/libpg_query/grammar/statements/select.y" { (yyval.nullorder) = PG_SORTBY_NULLS_FIRST; ;} break; - case 604: + case 605: #line 636 "third_party/libpg_query/grammar/statements/select.y" { (yyval.nullorder) = PG_SORTBY_NULLS_LAST; ;} break; - case 605: + case 606: #line 637 "third_party/libpg_query/grammar/statements/select.y" { (yyval.nullorder) = PG_SORTBY_NULLS_DEFAULT; ;} break; - case 606: + case 607: #line 641 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make2((yyvsp[(2) - (2)].node), (yyvsp[(1) - (2)].node)); ;} break; - case 607: + case 608: #line 642 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make2((yyvsp[(1) - (2)].node), (yyvsp[(2) - (2)].node)); ;} break; - case 608: + case 609: #line 643 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make2(NULL, (yyvsp[(1) - (1)].node)); ;} break; - case 609: + case 610: #line 644 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make2((yyvsp[(1) - (1)].node), NULL); ;} break; - case 610: + case 611: #line 648 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 611: + case 612: #line 649 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make2(NULL,NULL); ;} break; - case 612: + case 613: #line 654 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (2)].node); ;} break; - case 613: + case 614: #line 656 "third_party/libpg_query/grammar/statements/select.y" { /* Disabled because it was too confusing, bjm 2002-02-18 */ @@ -24509,91 +24548,91 @@ YYLTYPE yylloc; ;} break; - case 614: + case 615: #line 672 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(3) - (5)].node); ;} break; - case 615: + case 616: #line 674 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeIntConst(1, -1); ;} break; - case 616: + case 617: #line 679 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (2)].node); ;} break; - case 617: + case 618: #line 682 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (3)].node); ;} break; - case 618: + case 619: #line 690 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeSampleSize(makeFloat((yyvsp[(1) - (2)].str)), true); ;} break; - case 619: + case 620: #line 694 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeSampleSize(makeInteger((yyvsp[(1) - (2)].ival)), true); ;} break; - case 620: + case 621: #line 698 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeSampleSize(makeFloat((yyvsp[(1) - (2)].str)), true); ;} break; - case 621: + case 622: #line 702 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeSampleSize(makeInteger((yyvsp[(1) - (2)].ival)), true); ;} break; - case 622: + case 623: #line 706 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeSampleSize(makeInteger((yyvsp[(1) - (1)].ival)), false); ;} break; - case 623: + case 624: #line 710 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeSampleSize(makeInteger((yyvsp[(1) - (2)].ival)), false); ;} break; - case 624: + case 625: #line 717 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(3) - (3)].node); ;} break; - case 625: + case 626: #line 721 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 626: + case 627: #line 728 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 627: + case 628: #line 729 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = NULL; ;} break; - case 628: + case 629: #line 734 "third_party/libpg_query/grammar/statements/select.y" { int seed = (yyvsp[(5) - (5)].ival); @@ -24601,21 +24640,21 @@ YYLTYPE yylloc; ;} break; - case 629: + case 630: #line 739 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeSampleOptions((yyvsp[(1) - (1)].node), NULL, NULL, (yylsp[(1) - (1)])); ;} break; - case 630: + case 631: #line 743 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeSampleOptions((yyvsp[(1) - (4)].node), (yyvsp[(3) - (4)].str), NULL, (yylsp[(1) - (4)])); ;} break; - case 631: + case 632: #line 747 "third_party/libpg_query/grammar/statements/select.y" { int seed = (yyvsp[(5) - (6)].ival); @@ -24623,39 +24662,39 @@ YYLTYPE yylloc; ;} break; - case 632: + case 633: #line 755 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (2)].node); ;} break; - case 633: + case 634: #line 761 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 634: + case 635: #line 762 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 635: + case 636: #line 767 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ival) = (yyvsp[(3) - (4)].ival); ;} break; - case 636: + case 637: #line 768 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ival) = -1; ;} break; - case 637: + case 638: #line 772 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 638: + case 639: #line 774 "third_party/libpg_query/grammar/statements/select.y" { /* LIMIT ALL is represented as a NULL constant */ @@ -24663,77 +24702,77 @@ YYLTYPE yylloc; ;} break; - case 639: + case 640: #line 779 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeLimitPercent((yyvsp[(1) - (2)].node)); ;} break; - case 640: + case 641: #line 781 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeLimitPercent(makeFloatConst((yyvsp[(1) - (2)].str),(yylsp[(1) - (2)]))); ;} break; - case 641: + case 642: #line 783 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeLimitPercent(makeIntConst((yyvsp[(1) - (2)].ival),(yylsp[(1) - (2)]))); ;} break; - case 642: + case 643: #line 787 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 643: + case 644: #line 807 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 644: + case 645: #line 809 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "+", NULL, (yyvsp[(2) - (2)].node), (yylsp[(1) - (2)])); ;} break; - case 645: + case 646: #line 811 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = doNegate((yyvsp[(2) - (2)].node), (yylsp[(1) - (2)])); ;} break; - case 646: + case 647: #line 815 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeIntConst((yyvsp[(1) - (1)].ival),(yylsp[(1) - (1)])); ;} break; - case 647: + case 648: #line 816 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeFloatConst((yyvsp[(1) - (1)].str),(yylsp[(1) - (1)])); ;} break; - case 648: + case 649: #line 820 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ival) = 0; ;} break; - case 649: + case 650: #line 821 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ival) = 0; ;} break; - case 650: + case 651: #line 824 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ival) = 0; ;} break; - case 651: + case 652: #line 825 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ival) = 0; ;} break; - case 652: + case 653: #line 850 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(3) - (3)].list); ;} break; - case 653: + case 654: #line 852 "third_party/libpg_query/grammar/statements/select.y" { PGNode *node = (PGNode *) makeGroupingSet(GROUPING_SET_ALL, NIL, (yylsp[(3) - (3)])); @@ -24741,145 +24780,145 @@ YYLTYPE yylloc; ;} break; - case 654: + case 655: #line 856 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 655: + case 656: #line 860 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 656: + case 657: #line 861 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list),(yyvsp[(3) - (3)].node)); ;} break; - case 657: + case 658: #line 865 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 658: + case 659: #line 866 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; - case 659: + case 660: #line 870 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 660: + case 661: #line 871 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 661: + case 662: #line 872 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 662: + case 663: #line 873 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 663: + case 664: #line 874 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 664: + case 665: #line 879 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeGroupingSet(GROUPING_SET_EMPTY, NIL, (yylsp[(1) - (2)])); ;} break; - case 665: + case 666: #line 892 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeGroupingSet(GROUPING_SET_ROLLUP, (yyvsp[(3) - (4)].list), (yylsp[(1) - (4)])); ;} break; - case 666: + case 667: #line 899 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeGroupingSet(GROUPING_SET_CUBE, (yyvsp[(3) - (4)].list), (yylsp[(1) - (4)])); ;} break; - case 667: + case 668: #line 906 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeGroupingSet(GROUPING_SET_SETS, (yyvsp[(4) - (5)].list), (yylsp[(1) - (5)])); ;} break; - case 668: + case 669: #line 912 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 669: + case 670: #line 913 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 670: + case 671: #line 917 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (2)].node); ;} break; - case 671: + case 672: #line 918 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 672: + case 673: #line 922 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (2)].node); ;} break; - case 673: + case 674: #line 923 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 674: + case 675: #line 927 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 675: + case 676: #line 928 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 676: + case 677: #line 932 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 677: + case 678: #line 933 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 678: + case 679: #line 937 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 679: + case 680: #line 938 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (2)].list), (yyvsp[(2) - (2)].node)); ;} break; - case 680: + case 681: #line 943 "third_party/libpg_query/grammar/statements/select.y" { PGLockingClause *n = makeNode(PGLockingClause); @@ -24890,52 +24929,52 @@ YYLTYPE yylloc; ;} break; - case 681: + case 682: #line 953 "third_party/libpg_query/grammar/statements/select.y" { (yyval.lockstrength) = LCS_FORUPDATE; ;} break; - case 682: + case 683: #line 954 "third_party/libpg_query/grammar/statements/select.y" { (yyval.lockstrength) = PG_LCS_FORNOKEYUPDATE; ;} break; - case 683: + case 684: #line 955 "third_party/libpg_query/grammar/statements/select.y" { (yyval.lockstrength) = PG_LCS_FORSHARE; ;} break; - case 684: + case 685: #line 956 "third_party/libpg_query/grammar/statements/select.y" { (yyval.lockstrength) = PG_LCS_FORKEYSHARE; ;} break; - case 685: + case 686: #line 960 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(2) - (2)].list); ;} break; - case 686: + case 687: #line 961 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 687: + case 688: #line 966 "third_party/libpg_query/grammar/statements/select.y" { (yyval.lockwaitpolicy) = LockWaitError; ;} break; - case 688: + case 689: #line 967 "third_party/libpg_query/grammar/statements/select.y" { (yyval.lockwaitpolicy) = PGLockWaitSkip; ;} break; - case 689: + case 690: #line 968 "third_party/libpg_query/grammar/statements/select.y" { (yyval.lockwaitpolicy) = PGLockWaitBlock; ;} break; - case 690: + case 691: #line 978 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *n = makeNode(PGSelectStmt); @@ -24944,7 +24983,7 @@ YYLTYPE yylloc; ;} break; - case 691: + case 692: #line 984 "third_party/libpg_query/grammar/statements/select.y" { PGSelectStmt *n = (PGSelectStmt *) (yyvsp[(1) - (5)].node); @@ -24953,47 +24992,47 @@ YYLTYPE yylloc; ;} break; - case 692: + case 693: #line 992 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 693: + case 694: #line 993 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (2)].node); ;} break; - case 694: + case 695: #line 1006 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(2) - (2)].list); ;} break; - case 695: + case 696: #line 1007 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 696: + case 697: #line 1011 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 697: + case 698: #line 1012 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} break; - case 698: + case 699: #line 1016 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 699: + case 700: #line 1017 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; - case 700: + case 701: #line 1024 "third_party/libpg_query/grammar/statements/select.y" { (yyvsp[(1) - (3)].range)->alias = (yyvsp[(2) - (3)].alias); @@ -25002,7 +25041,7 @@ YYLTYPE yylloc; ;} break; - case 701: + case 702: #line 1030 "third_party/libpg_query/grammar/statements/select.y" { PGRangeFunction *n = (PGRangeFunction *) (yyvsp[(1) - (3)].node); @@ -25013,7 +25052,7 @@ YYLTYPE yylloc; ;} break; - case 702: + case 703: #line 1038 "third_party/libpg_query/grammar/statements/select.y" { PGRangeSubselect *n = makeNode(PGRangeSubselect); @@ -25025,7 +25064,7 @@ YYLTYPE yylloc; ;} break; - case 703: + case 704: #line 1047 "third_party/libpg_query/grammar/statements/select.y" { PGRangeFunction *n = (PGRangeFunction *) (yyvsp[(2) - (3)].node); @@ -25036,7 +25075,7 @@ YYLTYPE yylloc; ;} break; - case 704: + case 705: #line 1055 "third_party/libpg_query/grammar/statements/select.y" { PGRangeSubselect *n = makeNode(PGRangeSubselect); @@ -25048,7 +25087,7 @@ YYLTYPE yylloc; ;} break; - case 705: + case 706: #line 1064 "third_party/libpg_query/grammar/statements/select.y" { PGRangeSubselect *n = makeNode(PGRangeSubselect); @@ -25060,14 +25099,14 @@ YYLTYPE yylloc; ;} break; - case 706: + case 707: #line 1073 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) (yyvsp[(1) - (1)].jexpr); ;} break; - case 707: + case 708: #line 1077 "third_party/libpg_query/grammar/statements/select.y" { (yyvsp[(2) - (4)].jexpr)->alias = (yyvsp[(4) - (4)].alias); @@ -25075,7 +25114,7 @@ YYLTYPE yylloc; ;} break; - case 708: + case 709: #line 1082 "third_party/libpg_query/grammar/statements/select.y" { PGPivotExpr *n = makeNode(PGPivotExpr); @@ -25089,7 +25128,7 @@ YYLTYPE yylloc; ;} break; - case 709: + case 710: #line 1093 "third_party/libpg_query/grammar/statements/select.y" { PGPivotExpr *n = makeNode(PGPivotExpr); @@ -25103,32 +25142,32 @@ YYLTYPE yylloc; ;} break; - case 710: + case 711: #line 1106 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(3) - (3)].list); ;} break; - case 711: + case 712: #line 1107 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NULL; ;} break; - case 712: + case 713: #line 1110 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = true; ;} break; - case 713: + case 714: #line 1111 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = false; ;} break; - case 714: + case 715: #line 1112 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = false; ;} break; - case 715: + case 716: #line 1116 "third_party/libpg_query/grammar/statements/select.y" { PGPivot *n = makeNode(PGPivot); @@ -25138,7 +25177,7 @@ YYLTYPE yylloc; ;} break; - case 716: + case 717: #line 1124 "third_party/libpg_query/grammar/statements/select.y" { PGPivot *n = makeNode(PGPivot); @@ -25148,22 +25187,22 @@ YYLTYPE yylloc; ;} break; - case 718: + case 719: #line 1133 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 719: + case 720: #line 1134 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 720: + case 721: #line 1135 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(2) - (3)].list); ;} break; - case 721: + case 722: #line 1139 "third_party/libpg_query/grammar/statements/select.y" { PGPivot *n = makeNode(PGPivot); @@ -25173,7 +25212,7 @@ YYLTYPE yylloc; ;} break; - case 722: + case 723: #line 1147 "third_party/libpg_query/grammar/statements/select.y" { PGPivot *n = makeNode(PGPivot); @@ -25183,31 +25222,31 @@ YYLTYPE yylloc; ;} break; - case 723: + case 724: #line 1156 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 724: + case 725: #line 1160 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (2)].list), (yyvsp[(2) - (2)].node)); ;} break; - case 725: + case 726: #line 1166 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeString((yyvsp[(1) - (1)].str))); ;} break; - case 726: + case 727: #line 1167 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(2) - (3)].list); ;} break; - case 727: + case 728: #line 1172 "third_party/libpg_query/grammar/statements/select.y" { PGPivot *n = makeNode(PGPivot); @@ -25217,28 +25256,28 @@ YYLTYPE yylloc; ;} break; - case 728: + case 729: #line 1181 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 729: + case 730: #line 1185 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (2)].list), (yyvsp[(2) - (2)].node)); ;} break; - case 730: + case 731: #line 1210 "third_party/libpg_query/grammar/statements/select.y" { (yyval.jexpr) = (yyvsp[(2) - (3)].jexpr); ;} break; - case 731: + case 732: #line 1214 "third_party/libpg_query/grammar/statements/select.y" { /* CROSS JOIN is same as unqualified inner join */ @@ -25254,7 +25293,7 @@ YYLTYPE yylloc; ;} break; - case 732: + case 733: #line 1227 "third_party/libpg_query/grammar/statements/select.y" { PGJoinExpr *n = makeNode(PGJoinExpr); @@ -25271,7 +25310,7 @@ YYLTYPE yylloc; ;} break; - case 733: + case 734: #line 1241 "third_party/libpg_query/grammar/statements/select.y" { /* letting join_type reduce to empty doesn't work */ @@ -25289,7 +25328,7 @@ YYLTYPE yylloc; ;} break; - case 734: + case 735: #line 1256 "third_party/libpg_query/grammar/statements/select.y" { PGJoinExpr *n = makeNode(PGJoinExpr); @@ -25304,7 +25343,7 @@ YYLTYPE yylloc; ;} break; - case 735: + case 736: #line 1268 "third_party/libpg_query/grammar/statements/select.y" { /* letting join_type reduce to empty doesn't work */ @@ -25320,7 +25359,7 @@ YYLTYPE yylloc; ;} break; - case 736: + case 737: #line 1281 "third_party/libpg_query/grammar/statements/select.y" { PGJoinExpr *n = makeNode(PGJoinExpr); @@ -25337,7 +25376,7 @@ YYLTYPE yylloc; ;} break; - case 737: + case 738: #line 1295 "third_party/libpg_query/grammar/statements/select.y" { PGJoinExpr *n = makeNode(PGJoinExpr); @@ -25354,7 +25393,7 @@ YYLTYPE yylloc; ;} break; - case 738: + case 739: #line 1309 "third_party/libpg_query/grammar/statements/select.y" { /* POSITIONAL JOIN is a coordinated scan */ @@ -25370,7 +25409,7 @@ YYLTYPE yylloc; ;} break; - case 739: + case 740: #line 1322 "third_party/libpg_query/grammar/statements/select.y" { /* ANTI JOIN is a filter */ @@ -25388,7 +25427,7 @@ YYLTYPE yylloc; ;} break; - case 740: + case 741: #line 1337 "third_party/libpg_query/grammar/statements/select.y" { /* SEMI JOIN is also a filter */ @@ -25407,7 +25446,7 @@ YYLTYPE yylloc; ;} break; - case 741: + case 742: #line 1356 "third_party/libpg_query/grammar/statements/select.y" { (yyval.alias) = makeNode(PGAlias); @@ -25416,7 +25455,7 @@ YYLTYPE yylloc; ;} break; - case 742: + case 743: #line 1362 "third_party/libpg_query/grammar/statements/select.y" { (yyval.alias) = makeNode(PGAlias); @@ -25424,7 +25463,7 @@ YYLTYPE yylloc; ;} break; - case 743: + case 744: #line 1367 "third_party/libpg_query/grammar/statements/select.y" { (yyval.alias) = makeNode(PGAlias); @@ -25433,7 +25472,7 @@ YYLTYPE yylloc; ;} break; - case 744: + case 745: #line 1373 "third_party/libpg_query/grammar/statements/select.y" { (yyval.alias) = makeNode(PGAlias); @@ -25441,31 +25480,31 @@ YYLTYPE yylloc; ;} break; - case 745: + case 746: #line 1379 "third_party/libpg_query/grammar/statements/select.y" { (yyval.alias) = (yyvsp[(1) - (1)].alias); ;} break; - case 746: + case 747: #line 1380 "third_party/libpg_query/grammar/statements/select.y" { (yyval.alias) = NULL; ;} break; - case 747: + case 748: #line 1389 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make2((yyvsp[(1) - (1)].alias), NIL); ;} break; - case 748: + case 749: #line 1393 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make2(NULL, (yyvsp[(3) - (4)].list)); ;} break; - case 749: + case 750: #line 1397 "third_party/libpg_query/grammar/statements/select.y" { PGAlias *a = makeNode(PGAlias); @@ -25474,7 +25513,7 @@ YYLTYPE yylloc; ;} break; - case 750: + case 751: #line 1403 "third_party/libpg_query/grammar/statements/select.y" { PGAlias *a = makeNode(PGAlias); @@ -25483,64 +25522,64 @@ YYLTYPE yylloc; ;} break; - case 751: + case 752: #line 1409 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make2(NULL, NIL); ;} break; - case 752: + case 753: #line 1414 "third_party/libpg_query/grammar/statements/select.y" { (yyval.jtype) = PG_JOIN_FULL; ;} break; - case 753: + case 754: #line 1415 "third_party/libpg_query/grammar/statements/select.y" { (yyval.jtype) = PG_JOIN_LEFT; ;} break; - case 754: + case 755: #line 1416 "third_party/libpg_query/grammar/statements/select.y" { (yyval.jtype) = PG_JOIN_RIGHT; ;} break; - case 755: + case 756: #line 1417 "third_party/libpg_query/grammar/statements/select.y" { (yyval.jtype) = PG_JOIN_SEMI; ;} break; - case 756: + case 757: #line 1418 "third_party/libpg_query/grammar/statements/select.y" { (yyval.jtype) = PG_JOIN_ANTI; ;} break; - case 757: + case 758: #line 1419 "third_party/libpg_query/grammar/statements/select.y" { (yyval.jtype) = PG_JOIN_INNER; ;} break; - case 758: + case 759: #line 1423 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 759: + case 760: #line 1424 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 760: + case 761: #line 1436 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) (yyvsp[(3) - (4)].list); ;} break; - case 761: + case 762: #line 1437 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (2)].node); ;} break; - case 762: + case 763: #line 1443 "third_party/libpg_query/grammar/statements/select.y" { /* inheritance query, implicitly */ @@ -25550,7 +25589,7 @@ YYLTYPE yylloc; ;} break; - case 763: + case 764: #line 1450 "third_party/libpg_query/grammar/statements/select.y" { /* inheritance query, explicitly */ @@ -25560,7 +25599,7 @@ YYLTYPE yylloc; ;} break; - case 764: + case 765: #line 1457 "third_party/libpg_query/grammar/statements/select.y" { /* no inheritance */ @@ -25570,7 +25609,7 @@ YYLTYPE yylloc; ;} break; - case 765: + case 766: #line 1464 "third_party/libpg_query/grammar/statements/select.y" { /* no inheritance, SQL99-style syntax */ @@ -25580,7 +25619,7 @@ YYLTYPE yylloc; ;} break; - case 766: + case 767: #line 1496 "third_party/libpg_query/grammar/statements/select.y" { PGRangeFunction *n = makeNode(PGRangeFunction); @@ -25594,7 +25633,7 @@ YYLTYPE yylloc; ;} break; - case 767: + case 768: #line 1507 "third_party/libpg_query/grammar/statements/select.y" { PGRangeFunction *n = makeNode(PGRangeFunction); @@ -25608,66 +25647,66 @@ YYLTYPE yylloc; ;} break; - case 768: + case 769: #line 1520 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make2((yyvsp[(1) - (2)].node), (yyvsp[(2) - (2)].list)); ;} break; - case 769: + case 770: #line 1524 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].list)); ;} break; - case 770: + case 771: #line 1525 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].list)); ;} break; - case 771: + case 772: #line 1528 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(3) - (4)].list); ;} break; - case 772: + case 773: #line 1529 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 773: + case 774: #line 1532 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = true; ;} break; - case 774: + case 775: #line 1533 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = false; ;} break; - case 775: + case 776: #line 1538 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (2)].node); ;} break; - case 776: + case 777: #line 1539 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 777: + case 778: #line 1545 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 778: + case 779: #line 1549 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} break; - case 779: + case 780: #line 1555 "third_party/libpg_query/grammar/statements/select.y" { PGColumnDef *n = makeNode(PGColumnDef); @@ -25688,7 +25727,7 @@ YYLTYPE yylloc; ;} break; - case 780: + case 781: #line 1576 "third_party/libpg_query/grammar/statements/select.y" { PGCollateClause *n = makeNode(PGCollateClause); @@ -25699,36 +25738,36 @@ YYLTYPE yylloc; ;} break; - case 781: + case 782: #line 1583 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 782: + case 783: #line 1596 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(list_make2(makeString((yyvsp[(1) - (2)].str)), (yyvsp[(2) - (2)].typnam))); ;} break; - case 783: + case 784: #line 1599 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (4)].list), list_make2(makeString((yyvsp[(3) - (4)].str)), (yyvsp[(4) - (4)].typnam))); ;} break; - case 786: + case 787: #line 1606 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 787: + case 788: #line 1607 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = NULL; ;} break; - case 788: + case 789: #line 1610 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (2)].typnam); @@ -25736,7 +25775,7 @@ YYLTYPE yylloc; ;} break; - case 789: + case 790: #line 1615 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(2) - (3)].typnam); @@ -25745,7 +25784,7 @@ YYLTYPE yylloc; ;} break; - case 790: + case 791: #line 1622 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (5)].typnam); @@ -25753,7 +25792,7 @@ YYLTYPE yylloc; ;} break; - case 791: + case 792: #line 1627 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(2) - (6)].typnam); @@ -25762,7 +25801,7 @@ YYLTYPE yylloc; ;} break; - case 792: + case 793: #line 1633 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (2)].typnam); @@ -25770,7 +25809,7 @@ YYLTYPE yylloc; ;} break; - case 793: + case 794: #line 1638 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(2) - (3)].typnam); @@ -25779,14 +25818,14 @@ YYLTYPE yylloc; ;} break; - case 794: + case 795: #line 1644 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = makeTypeNameFromNameList((yyvsp[(1) - (1)].list)); ;} break; - case 795: + case 796: #line 1648 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("struct"); @@ -25796,7 +25835,7 @@ YYLTYPE yylloc; ;} break; - case 796: + case 797: #line 1655 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("map"); @@ -25806,7 +25845,7 @@ YYLTYPE yylloc; ;} break; - case 797: + case 798: #line 1662 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("union"); @@ -25816,57 +25855,57 @@ YYLTYPE yylloc; ;} break; - case 798: + case 799: #line 1671 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make2(makeString((yyvsp[(1) - (3)].str)), makeString((yyvsp[(3) - (3)].str))); ;} break; - case 799: + case 800: #line 1672 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), makeString((yyvsp[(3) - (3)].str))); ;} break; - case 800: + case 801: #line 1677 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), makeInteger(-1)); ;} break; - case 801: + case 802: #line 1679 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (4)].list), makeInteger((yyvsp[(3) - (4)].ival))); ;} break; - case 802: + case 803: #line 1681 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 803: + case 804: #line 1685 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 804: + case 805: #line 1686 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 805: + case 806: #line 1687 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 806: + case 807: #line 1688 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 807: + case 808: #line 1689 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 808: + case 809: #line 1691 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (2)].typnam); @@ -25874,7 +25913,7 @@ YYLTYPE yylloc; ;} break; - case 809: + case 810: #line 1696 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (4)].typnam); @@ -25883,27 +25922,27 @@ YYLTYPE yylloc; ;} break; - case 810: + case 811: #line 1715 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 811: + case 812: #line 1716 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 812: + case 813: #line 1717 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 813: + case 814: #line 1718 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 814: + case 815: #line 1730 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = makeTypeName((yyvsp[(1) - (2)].str)); @@ -25912,17 +25951,17 @@ YYLTYPE yylloc; ;} break; - case 815: + case 816: #line 1743 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(2) - (3)].list); ;} break; - case 816: + case 817: #line 1744 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 817: + case 818: #line 1751 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("int4"); @@ -25930,7 +25969,7 @@ YYLTYPE yylloc; ;} break; - case 818: + case 819: #line 1756 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("int4"); @@ -25938,7 +25977,7 @@ YYLTYPE yylloc; ;} break; - case 819: + case 820: #line 1761 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("int2"); @@ -25946,7 +25985,7 @@ YYLTYPE yylloc; ;} break; - case 820: + case 821: #line 1766 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("int8"); @@ -25954,7 +25993,7 @@ YYLTYPE yylloc; ;} break; - case 821: + case 822: #line 1771 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("float4"); @@ -25962,7 +26001,7 @@ YYLTYPE yylloc; ;} break; - case 822: + case 823: #line 1776 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(2) - (2)].typnam); @@ -25970,7 +26009,7 @@ YYLTYPE yylloc; ;} break; - case 823: + case 824: #line 1781 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("float8"); @@ -25978,7 +26017,7 @@ YYLTYPE yylloc; ;} break; - case 824: + case 825: #line 1786 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("numeric"); @@ -25987,7 +26026,7 @@ YYLTYPE yylloc; ;} break; - case 825: + case 826: #line 1792 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("numeric"); @@ -25996,7 +26035,7 @@ YYLTYPE yylloc; ;} break; - case 826: + case 827: #line 1798 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("numeric"); @@ -26005,7 +26044,7 @@ YYLTYPE yylloc; ;} break; - case 827: + case 828: #line 1804 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("bool"); @@ -26013,7 +26052,7 @@ YYLTYPE yylloc; ;} break; - case 828: + case 829: #line 1811 "third_party/libpg_query/grammar/statements/select.y" { /* @@ -26037,35 +26076,35 @@ YYLTYPE yylloc; ;} break; - case 829: + case 830: #line 1832 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("float4"); ;} break; - case 830: + case 831: #line 1842 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 831: + case 832: #line 1846 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 832: + case 833: #line 1854 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 833: + case 834: #line 1858 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); @@ -26073,7 +26112,7 @@ YYLTYPE yylloc; ;} break; - case 834: + case 835: #line 1866 "third_party/libpg_query/grammar/statements/select.y" { const char *typname; @@ -26085,7 +26124,7 @@ YYLTYPE yylloc; ;} break; - case 835: + case 836: #line 1878 "third_party/libpg_query/grammar/statements/select.y" { /* bit defaults to bit(1), varbit to no limit */ @@ -26102,28 +26141,28 @@ YYLTYPE yylloc; ;} break; - case 836: + case 837: #line 1899 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 837: + case 838: #line 1903 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 838: + case 839: #line 1909 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = (yyvsp[(1) - (1)].typnam); ;} break; - case 839: + case 840: #line 1913 "third_party/libpg_query/grammar/statements/select.y" { /* Length was not specified so allow to be unrestricted. @@ -26137,7 +26176,7 @@ YYLTYPE yylloc; ;} break; - case 840: + case 841: #line 1926 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName((yyvsp[(1) - (4)].conststr)); @@ -26146,7 +26185,7 @@ YYLTYPE yylloc; ;} break; - case 841: + case 842: #line 1934 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName((yyvsp[(1) - (1)].conststr)); @@ -26157,47 +26196,47 @@ YYLTYPE yylloc; ;} break; - case 842: + case 843: #line 1944 "third_party/libpg_query/grammar/statements/select.y" { (yyval.conststr) = (yyvsp[(2) - (2)].boolean) ? "varchar": "bpchar"; ;} break; - case 843: + case 844: #line 1946 "third_party/libpg_query/grammar/statements/select.y" { (yyval.conststr) = (yyvsp[(2) - (2)].boolean) ? "varchar": "bpchar"; ;} break; - case 844: + case 845: #line 1948 "third_party/libpg_query/grammar/statements/select.y" { (yyval.conststr) = "varchar"; ;} break; - case 845: + case 846: #line 1950 "third_party/libpg_query/grammar/statements/select.y" { (yyval.conststr) = (yyvsp[(3) - (3)].boolean) ? "varchar": "bpchar"; ;} break; - case 846: + case 847: #line 1952 "third_party/libpg_query/grammar/statements/select.y" { (yyval.conststr) = (yyvsp[(3) - (3)].boolean) ? "varchar": "bpchar"; ;} break; - case 847: + case 848: #line 1954 "third_party/libpg_query/grammar/statements/select.y" { (yyval.conststr) = (yyvsp[(2) - (2)].boolean) ? "varchar": "bpchar"; ;} break; - case 848: + case 849: #line 1958 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = true; ;} break; - case 849: + case 850: #line 1959 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = false; ;} break; - case 850: + case 851: #line 1967 "third_party/libpg_query/grammar/statements/select.y" { if ((yyvsp[(5) - (5)].boolean)) @@ -26209,7 +26248,7 @@ YYLTYPE yylloc; ;} break; - case 851: + case 852: #line 1976 "third_party/libpg_query/grammar/statements/select.y" { if ((yyvsp[(2) - (2)].boolean)) @@ -26220,7 +26259,7 @@ YYLTYPE yylloc; ;} break; - case 852: + case 853: #line 1984 "third_party/libpg_query/grammar/statements/select.y" { if ((yyvsp[(5) - (5)].boolean)) @@ -26232,7 +26271,7 @@ YYLTYPE yylloc; ;} break; - case 853: + case 854: #line 1993 "third_party/libpg_query/grammar/statements/select.y" { if ((yyvsp[(2) - (2)].boolean)) @@ -26243,7 +26282,7 @@ YYLTYPE yylloc; ;} break; - case 854: + case 855: #line 2004 "third_party/libpg_query/grammar/statements/select.y" { (yyval.typnam) = SystemTypeName("interval"); @@ -26251,82 +26290,82 @@ YYLTYPE yylloc; ;} break; - case 855: + case 856: #line 2011 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = true; ;} break; - case 856: + case 857: #line 2012 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = false; ;} break; - case 857: + case 858: #line 2013 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = false; ;} break; - case 882: + case 883: #line 2054 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(YEAR), (yylsp[(1) - (1)]))); ;} break; - case 883: + case 884: #line 2056 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(MONTH), (yylsp[(1) - (1)]))); ;} break; - case 884: + case 885: #line 2058 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(DAY), (yylsp[(1) - (1)]))); ;} break; - case 885: + case 886: #line 2060 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(HOUR), (yylsp[(1) - (1)]))); ;} break; - case 886: + case 887: #line 2062 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(MINUTE), (yylsp[(1) - (1)]))); ;} break; - case 887: + case 888: #line 2064 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(SECOND), (yylsp[(1) - (1)]))); ;} break; - case 888: + case 889: #line 2066 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(MILLISECOND), (yylsp[(1) - (1)]))); ;} break; - case 889: + case 890: #line 2068 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(MICROSECOND), (yylsp[(1) - (1)]))); ;} break; - case 890: + case 891: #line 2070 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(WEEK), (yylsp[(1) - (1)]))); ;} break; - case 891: + case 892: #line 2072 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(DECADE), (yylsp[(1) - (1)]))); ;} break; - case 892: + case 893: #line 2074 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(CENTURY), (yylsp[(1) - (1)]))); ;} break; - case 893: + case 894: #line 2076 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(MILLENNIUM), (yylsp[(1) - (1)]))); ;} break; - case 894: + case 895: #line 2078 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(YEAR) | @@ -26334,7 +26373,7 @@ YYLTYPE yylloc; ;} break; - case 895: + case 896: #line 2083 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(DAY) | @@ -26342,7 +26381,7 @@ YYLTYPE yylloc; ;} break; - case 896: + case 897: #line 2088 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(DAY) | @@ -26351,7 +26390,7 @@ YYLTYPE yylloc; ;} break; - case 897: + case 898: #line 2094 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(DAY) | @@ -26361,7 +26400,7 @@ YYLTYPE yylloc; ;} break; - case 898: + case 899: #line 2101 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(HOUR) | @@ -26369,7 +26408,7 @@ YYLTYPE yylloc; ;} break; - case 899: + case 900: #line 2106 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(HOUR) | @@ -26378,7 +26417,7 @@ YYLTYPE yylloc; ;} break; - case 900: + case 901: #line 2112 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeIntConst(INTERVAL_MASK(MINUTE) | @@ -26386,22 +26425,22 @@ YYLTYPE yylloc; ;} break; - case 901: + case 902: #line 2117 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 902: + case 903: #line 2148 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 903: + case 904: #line 2151 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeTypeCast((yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].typnam), 0, (yylsp[(2) - (3)])); ;} break; - case 904: + case 905: #line 2153 "third_party/libpg_query/grammar/statements/select.y" { PGCollateClause *n = makeNode(PGCollateClause); @@ -26412,7 +26451,7 @@ YYLTYPE yylloc; ;} break; - case 905: + case 906: #line 2161 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeFuncCall(SystemFuncName("timezone"), @@ -26421,122 +26460,122 @@ YYLTYPE yylloc; ;} break; - case 906: + case 907: #line 2176 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "+", NULL, (yyvsp[(2) - (2)].node), (yylsp[(1) - (2)])); ;} break; - case 907: + case 908: #line 2178 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = doNegate((yyvsp[(2) - (2)].node), (yylsp[(1) - (2)])); ;} break; - case 908: + case 909: #line 2180 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "+", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 909: + case 910: #line 2182 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "-", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 910: + case 911: #line 2184 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "*", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 911: + case 912: #line 2186 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "/", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 912: + case 913: #line 2188 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "//", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 913: + case 914: #line 2190 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "%", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 914: + case 915: #line 2192 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "^", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 915: + case 916: #line 2194 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "**", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 916: + case 917: #line 2196 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "<", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 917: + case 918: #line 2198 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, ">", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 918: + case 919: #line 2200 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "=", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 919: + case 920: #line 2202 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "<=", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 920: + case 921: #line 2204 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, ">=", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 921: + case 922: #line 2206 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "<>", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 922: + case 923: #line 2209 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeAExpr(PG_AEXPR_OP, (yyvsp[(2) - (3)].list), (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 923: + case 924: #line 2211 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeAExpr(PG_AEXPR_OP, (yyvsp[(1) - (2)].list), NULL, (yyvsp[(2) - (2)].node), (yylsp[(1) - (2)])); ;} break; - case 924: + case 925: #line 2213 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeAExpr(PG_AEXPR_OP, (yyvsp[(2) - (2)].list), (yyvsp[(1) - (2)].node), NULL, (yylsp[(2) - (2)])); ;} break; - case 925: + case 926: #line 2216 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeAndExpr((yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 926: + case 927: #line 2218 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeOrExpr((yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 927: + case 928: #line 2220 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeNotExpr((yyvsp[(2) - (2)].node), (yylsp[(1) - (2)])); ;} break; - case 928: + case 929: #line 2222 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeNotExpr((yyvsp[(2) - (2)].node), (yylsp[(1) - (2)])); ;} break; - case 929: + case 930: #line 2224 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_GLOB, "~~~", @@ -26544,7 +26583,7 @@ YYLTYPE yylloc; ;} break; - case 930: + case 931: #line 2229 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_LIKE, "~~", @@ -26552,7 +26591,7 @@ YYLTYPE yylloc; ;} break; - case 931: + case 932: #line 2234 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall(SystemFuncName("like_escape"), @@ -26562,7 +26601,7 @@ YYLTYPE yylloc; ;} break; - case 932: + case 933: #line 2241 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_LIKE, "!~~", @@ -26570,7 +26609,7 @@ YYLTYPE yylloc; ;} break; - case 933: + case 934: #line 2246 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall(SystemFuncName("not_like_escape"), @@ -26580,7 +26619,7 @@ YYLTYPE yylloc; ;} break; - case 934: + case 935: #line 2253 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_ILIKE, "~~*", @@ -26588,7 +26627,7 @@ YYLTYPE yylloc; ;} break; - case 935: + case 936: #line 2258 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall(SystemFuncName("ilike_escape"), @@ -26598,7 +26637,7 @@ YYLTYPE yylloc; ;} break; - case 936: + case 937: #line 2265 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_ILIKE, "!~~*", @@ -26606,7 +26645,7 @@ YYLTYPE yylloc; ;} break; - case 937: + case 938: #line 2270 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall(SystemFuncName("not_ilike_escape"), @@ -26616,7 +26655,7 @@ YYLTYPE yylloc; ;} break; - case 938: + case 939: #line 2278 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall(SystemFuncName("similar_escape"), @@ -26627,7 +26666,7 @@ YYLTYPE yylloc; ;} break; - case 939: + case 940: #line 2286 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall(SystemFuncName("similar_escape"), @@ -26638,7 +26677,7 @@ YYLTYPE yylloc; ;} break; - case 940: + case 941: #line 2294 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall(SystemFuncName("similar_escape"), @@ -26649,7 +26688,7 @@ YYLTYPE yylloc; ;} break; - case 941: + case 942: #line 2302 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall(SystemFuncName("similar_escape"), @@ -26660,7 +26699,7 @@ YYLTYPE yylloc; ;} break; - case 942: + case 943: #line 2320 "third_party/libpg_query/grammar/statements/select.y" { PGNullTest *n = makeNode(PGNullTest); @@ -26671,7 +26710,7 @@ YYLTYPE yylloc; ;} break; - case 943: + case 944: #line 2328 "third_party/libpg_query/grammar/statements/select.y" { PGNullTest *n = makeNode(PGNullTest); @@ -26682,7 +26721,7 @@ YYLTYPE yylloc; ;} break; - case 944: + case 945: #line 2336 "third_party/libpg_query/grammar/statements/select.y" { PGNullTest *n = makeNode(PGNullTest); @@ -26693,7 +26732,7 @@ YYLTYPE yylloc; ;} break; - case 945: + case 946: #line 2344 "third_party/libpg_query/grammar/statements/select.y" { PGNullTest *n = makeNode(PGNullTest); @@ -26704,7 +26743,7 @@ YYLTYPE yylloc; ;} break; - case 946: + case 947: #line 2352 "third_party/libpg_query/grammar/statements/select.y" { PGNullTest *n = makeNode(PGNullTest); @@ -26715,7 +26754,7 @@ YYLTYPE yylloc; ;} break; - case 947: + case 948: #line 2360 "third_party/libpg_query/grammar/statements/select.y" { PGLambdaFunction *n = makeNode(PGLambdaFunction); @@ -26726,14 +26765,14 @@ YYLTYPE yylloc; ;} break; - case 948: + case 949: #line 2368 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "->>", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 949: + case 950: #line 2372 "third_party/libpg_query/grammar/statements/select.y" { if (list_length((yyvsp[(1) - (3)].list)) != 2) @@ -26752,7 +26791,7 @@ YYLTYPE yylloc; ;} break; - case 950: + case 951: #line 2388 "third_party/libpg_query/grammar/statements/select.y" { PGBooleanTest *b = makeNode(PGBooleanTest); @@ -26763,7 +26802,7 @@ YYLTYPE yylloc; ;} break; - case 951: + case 952: #line 2396 "third_party/libpg_query/grammar/statements/select.y" { PGBooleanTest *b = makeNode(PGBooleanTest); @@ -26774,7 +26813,7 @@ YYLTYPE yylloc; ;} break; - case 952: + case 953: #line 2404 "third_party/libpg_query/grammar/statements/select.y" { PGBooleanTest *b = makeNode(PGBooleanTest); @@ -26785,7 +26824,7 @@ YYLTYPE yylloc; ;} break; - case 953: + case 954: #line 2412 "third_party/libpg_query/grammar/statements/select.y" { PGBooleanTest *b = makeNode(PGBooleanTest); @@ -26796,7 +26835,7 @@ YYLTYPE yylloc; ;} break; - case 954: + case 955: #line 2420 "third_party/libpg_query/grammar/statements/select.y" { PGBooleanTest *b = makeNode(PGBooleanTest); @@ -26807,7 +26846,7 @@ YYLTYPE yylloc; ;} break; - case 955: + case 956: #line 2428 "third_party/libpg_query/grammar/statements/select.y" { PGBooleanTest *b = makeNode(PGBooleanTest); @@ -26818,35 +26857,35 @@ YYLTYPE yylloc; ;} break; - case 956: + case 957: #line 2436 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_DISTINCT, "=", (yyvsp[(1) - (5)].node), (yyvsp[(5) - (5)].node), (yylsp[(2) - (5)])); ;} break; - case 957: + case 958: #line 2440 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_NOT_DISTINCT, "=", (yyvsp[(1) - (6)].node), (yyvsp[(6) - (6)].node), (yylsp[(2) - (6)])); ;} break; - case 958: + case 959: #line 2444 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OF, "=", (yyvsp[(1) - (6)].node), (PGNode *) (yyvsp[(5) - (6)].list), (yylsp[(2) - (6)])); ;} break; - case 959: + case 960: #line 2448 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OF, "<>", (yyvsp[(1) - (7)].node), (PGNode *) (yyvsp[(6) - (7)].list), (yylsp[(2) - (7)])); ;} break; - case 960: + case 961: #line 2452 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_BETWEEN, @@ -26857,7 +26896,7 @@ YYLTYPE yylloc; ;} break; - case 961: + case 962: #line 2460 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_NOT_BETWEEN, @@ -26868,7 +26907,7 @@ YYLTYPE yylloc; ;} break; - case 962: + case 963: #line 2468 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_BETWEEN_SYM, @@ -26879,7 +26918,7 @@ YYLTYPE yylloc; ;} break; - case 963: + case 964: #line 2476 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_NOT_BETWEEN_SYM, @@ -26890,7 +26929,7 @@ YYLTYPE yylloc; ;} break; - case 964: + case 965: #line 2484 "third_party/libpg_query/grammar/statements/select.y" { /* in_expr returns a PGSubLink or a list of a_exprs */ @@ -26913,7 +26952,7 @@ YYLTYPE yylloc; ;} break; - case 965: + case 966: #line 2504 "third_party/libpg_query/grammar/statements/select.y" { /* in_expr returns a PGSubLink or a list of a_exprs */ @@ -26938,7 +26977,7 @@ YYLTYPE yylloc; ;} break; - case 966: + case 967: #line 2526 "third_party/libpg_query/grammar/statements/select.y" { PGSubLink *n = makeNode(PGSubLink); @@ -26952,7 +26991,7 @@ YYLTYPE yylloc; ;} break; - case 967: + case 968: #line 2537 "third_party/libpg_query/grammar/statements/select.y" { if ((yyvsp[(3) - (6)].subquerytype) == PG_ANY_SUBLINK) @@ -26962,7 +27001,7 @@ YYLTYPE yylloc; ;} break; - case 968: + case 969: #line 2544 "third_party/libpg_query/grammar/statements/select.y" { /* @@ -26979,7 +27018,7 @@ YYLTYPE yylloc; ;} break; - case 969: + case 970: #line 2558 "third_party/libpg_query/grammar/statements/select.y" { PGAStar *star = makeNode(PGAStar); @@ -26990,7 +27029,7 @@ YYLTYPE yylloc; ;} break; - case 970: + case 971: #line 2566 "third_party/libpg_query/grammar/statements/select.y" { PGAStar *star = makeNode(PGAStar); @@ -27001,7 +27040,7 @@ YYLTYPE yylloc; ;} break; - case 971: + case 972: #line 2574 "third_party/libpg_query/grammar/statements/select.y" { PGAStar *star = makeNode(PGAStar); @@ -27013,140 +27052,140 @@ YYLTYPE yylloc; ;} break; - case 972: + case 973: #line 2594 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 973: + case 974: #line 2596 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeTypeCast((yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].typnam), 0, (yylsp[(2) - (3)])); ;} break; - case 974: + case 975: #line 2598 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "+", NULL, (yyvsp[(2) - (2)].node), (yylsp[(1) - (2)])); ;} break; - case 975: + case 976: #line 2600 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = doNegate((yyvsp[(2) - (2)].node), (yylsp[(1) - (2)])); ;} break; - case 976: + case 977: #line 2602 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "+", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 977: + case 978: #line 2604 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "-", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 978: + case 979: #line 2606 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "*", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 979: + case 980: #line 2608 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "/", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 980: + case 981: #line 2610 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "//", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 981: + case 982: #line 2612 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "%", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 982: + case 983: #line 2614 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "^", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 983: + case 984: #line 2616 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "**", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 984: + case 985: #line 2618 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "<", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 985: + case 986: #line 2620 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, ">", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 986: + case 987: #line 2622 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "=", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 987: + case 988: #line 2624 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "<=", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 988: + case 989: #line 2626 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, ">=", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 989: + case 990: #line 2628 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OP, "<>", (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 990: + case 991: #line 2630 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeAExpr(PG_AEXPR_OP, (yyvsp[(2) - (3)].list), (yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yylsp[(2) - (3)])); ;} break; - case 991: + case 992: #line 2632 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeAExpr(PG_AEXPR_OP, (yyvsp[(1) - (2)].list), NULL, (yyvsp[(2) - (2)].node), (yylsp[(1) - (2)])); ;} break; - case 992: + case 993: #line 2634 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeAExpr(PG_AEXPR_OP, (yyvsp[(2) - (2)].list), (yyvsp[(1) - (2)].node), NULL, (yylsp[(2) - (2)])); ;} break; - case 993: + case 994: #line 2636 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_DISTINCT, "=", (yyvsp[(1) - (5)].node), (yyvsp[(5) - (5)].node), (yylsp[(2) - (5)])); ;} break; - case 994: + case 995: #line 2640 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_NOT_DISTINCT, "=", (yyvsp[(1) - (6)].node), (yyvsp[(6) - (6)].node), (yylsp[(2) - (6)])); ;} break; - case 995: + case 996: #line 2644 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OF, "=", (yyvsp[(1) - (6)].node), (PGNode *) (yyvsp[(5) - (6)].list), (yylsp[(2) - (6)])); ;} break; - case 996: + case 997: #line 2648 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_OF, "<>", (yyvsp[(1) - (7)].node), (PGNode *) (yyvsp[(6) - (7)].list), (yylsp[(2) - (7)])); ;} break; - case 998: + case 999: #line 2663 "third_party/libpg_query/grammar/statements/select.y" { if ((yyvsp[(2) - (2)].list)) @@ -27161,17 +27200,17 @@ YYLTYPE yylloc; ;} break; - case 999: + case 1000: #line 2676 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 1000: + case 1001: #line 2677 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 1001: + case 1002: #line 2679 "third_party/libpg_query/grammar/statements/select.y" { PGSubLink *n = makeNode(PGSubLink); @@ -27185,7 +27224,7 @@ YYLTYPE yylloc; ;} break; - case 1002: + case 1003: #line 2690 "third_party/libpg_query/grammar/statements/select.y" { /* @@ -27212,7 +27251,7 @@ YYLTYPE yylloc; ;} break; - case 1003: + case 1004: #line 2714 "third_party/libpg_query/grammar/statements/select.y" { PGSubLink *n = makeNode(PGSubLink); @@ -27226,7 +27265,7 @@ YYLTYPE yylloc; ;} break; - case 1004: + case 1005: #line 2725 "third_party/libpg_query/grammar/statements/select.y" { PGGroupingFunc *g = makeNode(PGGroupingFunc); @@ -27236,21 +27275,21 @@ YYLTYPE yylloc; ;} break; - case 1005: + case 1006: #line 2735 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (3)].node); ;} break; - case 1006: + case 1007: #line 2739 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 1007: + case 1008: #line 2742 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall(SystemFuncName("row"), (yyvsp[(1) - (1)].list), (yylsp[(1) - (1)])); @@ -27258,14 +27297,14 @@ YYLTYPE yylloc; ;} break; - case 1008: + case 1009: #line 2750 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeParamRef(0, (yylsp[(1) - (1)])); ;} break; - case 1009: + case 1010: #line 2754 "third_party/libpg_query/grammar/statements/select.y" { PGParamRef *p = makeNode(PGParamRef); @@ -27275,14 +27314,14 @@ YYLTYPE yylloc; ;} break; - case 1010: + case 1011: #line 2761 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 1011: + case 1012: #line 2765 "third_party/libpg_query/grammar/statements/select.y" { PGList *key_list = NULL; @@ -27302,35 +27341,34 @@ YYLTYPE yylloc; ;} break; - case 1012: + case 1013: #line 2782 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 1013: + case 1014: #line 2786 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 1014: + case 1015: #line 2787 "third_party/libpg_query/grammar/statements/select.y" { - PGFuncCall *n = makeFuncCall(SystemFuncName("list_value"), (yyvsp[(2) - (3)].list), (yylsp[(2) - (3)])); - (yyval.node) = (PGNode *) n; + (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 1015: -#line 2791 "third_party/libpg_query/grammar/statements/select.y" + case 1016: +#line 2790 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 1016: -#line 2795 "third_party/libpg_query/grammar/statements/select.y" + case 1017: +#line 2794 "third_party/libpg_query/grammar/statements/select.y" { PGSubLink *n = makeNode(PGSubLink); n->subLinkType = PG_ARRAY_SUBLINK; @@ -27343,8 +27381,8 @@ YYLTYPE yylloc; ;} break; - case 1017: -#line 2805 "third_party/libpg_query/grammar/statements/select.y" + case 1018: +#line 2804 "third_party/libpg_query/grammar/statements/select.y" { PGList *func_name = list_make1(makeString("construct_array")); PGFuncCall *n = makeFuncCall(func_name, (yyvsp[(3) - (4)].list), (yylsp[(1) - (4)])); @@ -27352,8 +27390,8 @@ YYLTYPE yylloc; ;} break; - case 1018: -#line 2811 "third_party/libpg_query/grammar/statements/select.y" + case 1019: +#line 2810 "third_party/libpg_query/grammar/statements/select.y" { PGPositionalReference *n = makeNode(PGPositionalReference); n->position = (yyvsp[(2) - (2)].ival); @@ -27362,30 +27400,38 @@ YYLTYPE yylloc; ;} break; - case 1019: -#line 2818 "third_party/libpg_query/grammar/statements/select.y" + case 1020: +#line 2817 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeNamedParamRef((yyvsp[(2) - (2)].str), (yylsp[(1) - (2)])); ;} break; - case 1020: -#line 2826 "third_party/libpg_query/grammar/statements/select.y" + case 1021: +#line 2822 "third_party/libpg_query/grammar/statements/select.y" + { + PGFuncCall *n = makeFuncCall(SystemFuncName("list_value"), (yyvsp[(2) - (3)].list), (yylsp[(2) - (3)])); + (yyval.node) = (PGNode *) n; + ;} + break; + + case 1022: +#line 2829 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *f = makeFuncCall(SystemFuncName("struct_pack"), (yyvsp[(2) - (3)].list), (yylsp[(2) - (3)])); (yyval.node) = (PGNode *) f; ;} break; - case 1021: -#line 2835 "third_party/libpg_query/grammar/statements/select.y" + case 1023: +#line 2838 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeFuncCall((yyvsp[(1) - (3)].list), NIL, (yylsp[(1) - (3)])); ;} break; - case 1022: -#line 2839 "third_party/libpg_query/grammar/statements/select.y" + case 1024: +#line 2842 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall((yyvsp[(1) - (6)].list), (yyvsp[(3) - (6)].list), (yylsp[(1) - (6)])); n->agg_order = (yyvsp[(4) - (6)].list); @@ -27394,8 +27440,8 @@ YYLTYPE yylloc; ;} break; - case 1023: -#line 2846 "third_party/libpg_query/grammar/statements/select.y" + case 1025: +#line 2849 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall((yyvsp[(1) - (7)].list), list_make1((yyvsp[(4) - (7)].node)), (yylsp[(1) - (7)])); n->func_variadic = true; @@ -27405,8 +27451,8 @@ YYLTYPE yylloc; ;} break; - case 1024: -#line 2854 "third_party/libpg_query/grammar/statements/select.y" + case 1026: +#line 2857 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall((yyvsp[(1) - (9)].list), lappend((yyvsp[(3) - (9)].list), (yyvsp[(6) - (9)].node)), (yylsp[(1) - (9)])); n->func_variadic = true; @@ -27416,8 +27462,8 @@ YYLTYPE yylloc; ;} break; - case 1025: -#line 2862 "third_party/libpg_query/grammar/statements/select.y" + case 1027: +#line 2865 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall((yyvsp[(1) - (7)].list), (yyvsp[(4) - (7)].list), (yylsp[(1) - (7)])); n->agg_order = (yyvsp[(5) - (7)].list); @@ -27430,8 +27476,8 @@ YYLTYPE yylloc; ;} break; - case 1026: -#line 2873 "third_party/libpg_query/grammar/statements/select.y" + case 1028: +#line 2876 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = makeFuncCall((yyvsp[(1) - (7)].list), (yyvsp[(4) - (7)].list), (yylsp[(1) - (7)])); n->agg_order = (yyvsp[(5) - (7)].list); @@ -27441,8 +27487,8 @@ YYLTYPE yylloc; ;} break; - case 1027: -#line 2893 "third_party/libpg_query/grammar/statements/select.y" + case 1029: +#line 2896 "third_party/libpg_query/grammar/statements/select.y" { PGFuncCall *n = (PGFuncCall *) (yyvsp[(1) - (5)].node); /* @@ -27480,23 +27526,23 @@ YYLTYPE yylloc; ;} break; - case 1028: -#line 2929 "third_party/libpg_query/grammar/statements/select.y" + case 1030: +#line 2932 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 1029: -#line 2939 "third_party/libpg_query/grammar/statements/select.y" + case 1031: +#line 2942 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 1030: -#line 2940 "third_party/libpg_query/grammar/statements/select.y" + case 1032: +#line 2943 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 1031: -#line 2948 "third_party/libpg_query/grammar/statements/select.y" + case 1033: +#line 2951 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeFuncCall(SystemFuncName("pg_collation_for"), list_make1((yyvsp[(4) - (5)].node)), @@ -27504,25 +27550,25 @@ YYLTYPE yylloc; ;} break; - case 1032: -#line 2954 "third_party/libpg_query/grammar/statements/select.y" + case 1034: +#line 2957 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeTypeCast((yyvsp[(3) - (6)].node), (yyvsp[(5) - (6)].typnam), 0, (yylsp[(1) - (6)])); ;} break; - case 1033: -#line 2956 "third_party/libpg_query/grammar/statements/select.y" + case 1035: +#line 2959 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeTypeCast((yyvsp[(3) - (6)].node), (yyvsp[(5) - (6)].typnam), 1, (yylsp[(1) - (6)])); ;} break; - case 1034: -#line 2958 "third_party/libpg_query/grammar/statements/select.y" + case 1036: +#line 2961 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeFuncCall(SystemFuncName("date_part"), (yyvsp[(3) - (4)].list), (yylsp[(1) - (4)])); ;} break; - case 1035: -#line 2962 "third_party/libpg_query/grammar/statements/select.y" + case 1037: +#line 2965 "third_party/libpg_query/grammar/statements/select.y" { /* overlay(A PLACING B FROM C FOR D) is converted to * overlay(A, B, C, D) @@ -27533,16 +27579,16 @@ YYLTYPE yylloc; ;} break; - case 1036: -#line 2971 "third_party/libpg_query/grammar/statements/select.y" + case 1038: +#line 2974 "third_party/libpg_query/grammar/statements/select.y" { /* position(A in B) is converted to position_inverse(A, B) */ (yyval.node) = (PGNode *) makeFuncCall(SystemFuncName("__internal_position_operator"), (yyvsp[(3) - (4)].list), (yylsp[(1) - (4)])); ;} break; - case 1037: -#line 2976 "third_party/libpg_query/grammar/statements/select.y" + case 1039: +#line 2979 "third_party/libpg_query/grammar/statements/select.y" { /* substring(A from B for C) is converted to * substring(A, B, C) - thomas 2000-11-28 @@ -27551,8 +27597,8 @@ YYLTYPE yylloc; ;} break; - case 1038: -#line 2983 "third_party/libpg_query/grammar/statements/select.y" + case 1040: +#line 2986 "third_party/libpg_query/grammar/statements/select.y" { /* TREAT(expr AS target) converts expr of a particular type to target, * which is defined to be a subtype of the original expression. @@ -27569,8 +27615,8 @@ YYLTYPE yylloc; ;} break; - case 1039: -#line 2998 "third_party/libpg_query/grammar/statements/select.y" + case 1041: +#line 3001 "third_party/libpg_query/grammar/statements/select.y" { /* various trim expressions are defined in SQL * - thomas 1997-07-19 @@ -27579,36 +27625,36 @@ YYLTYPE yylloc; ;} break; - case 1040: -#line 3005 "third_party/libpg_query/grammar/statements/select.y" + case 1042: +#line 3008 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeFuncCall(SystemFuncName("ltrim"), (yyvsp[(4) - (5)].list), (yylsp[(1) - (5)])); ;} break; - case 1041: -#line 3009 "third_party/libpg_query/grammar/statements/select.y" + case 1043: +#line 3012 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeFuncCall(SystemFuncName("rtrim"), (yyvsp[(4) - (5)].list), (yylsp[(1) - (5)])); ;} break; - case 1042: -#line 3013 "third_party/libpg_query/grammar/statements/select.y" + case 1044: +#line 3016 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeFuncCall(SystemFuncName("trim"), (yyvsp[(3) - (4)].list), (yylsp[(1) - (4)])); ;} break; - case 1043: -#line 3017 "third_party/libpg_query/grammar/statements/select.y" + case 1045: +#line 3020 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *) makeSimpleAExpr(PG_AEXPR_NULLIF, "=", (yyvsp[(3) - (6)].node), (yyvsp[(5) - (6)].node), (yylsp[(1) - (6)])); ;} break; - case 1044: -#line 3021 "third_party/libpg_query/grammar/statements/select.y" + case 1046: +#line 3024 "third_party/libpg_query/grammar/statements/select.y" { PGCoalesceExpr *c = makeNode(PGCoalesceExpr); c->args = (yyvsp[(3) - (4)].list); @@ -27617,8 +27663,8 @@ YYLTYPE yylloc; ;} break; - case 1045: -#line 3031 "third_party/libpg_query/grammar/statements/select.y" + case 1047: +#line 3034 "third_party/libpg_query/grammar/statements/select.y" { PGLambdaFunction *lambda = makeNode(PGLambdaFunction); lambda->lhs = makeColumnRef((yyvsp[(4) - (7)].str), NIL, (yylsp[(4) - (7)]), yyscanner); @@ -27629,8 +27675,8 @@ YYLTYPE yylloc; ;} break; - case 1046: -#line 3040 "third_party/libpg_query/grammar/statements/select.y" + case 1048: +#line 3043 "third_party/libpg_query/grammar/statements/select.y" { PGLambdaFunction *lambda = makeNode(PGLambdaFunction); lambda->lhs = makeColumnRef((yyvsp[(4) - (9)].str), NIL, (yylsp[(4) - (9)]), yyscanner); @@ -27647,63 +27693,63 @@ YYLTYPE yylloc; ;} break; - case 1047: -#line 3061 "third_party/libpg_query/grammar/statements/select.y" + case 1049: +#line 3064 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(4) - (5)].list); ;} break; - case 1048: -#line 3062 "third_party/libpg_query/grammar/statements/select.y" + case 1050: +#line 3065 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 1049: -#line 3066 "third_party/libpg_query/grammar/statements/select.y" + case 1051: +#line 3069 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(4) - (5)].node); ;} break; - case 1050: -#line 3067 "third_party/libpg_query/grammar/statements/select.y" + case 1052: +#line 3070 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(3) - (4)].node); ;} break; - case 1051: -#line 3068 "third_party/libpg_query/grammar/statements/select.y" + case 1053: +#line 3071 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 1052: -#line 3072 "third_party/libpg_query/grammar/statements/select.y" + case 1054: +#line 3075 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = true; ;} break; - case 1053: -#line 3073 "third_party/libpg_query/grammar/statements/select.y" + case 1055: +#line 3076 "third_party/libpg_query/grammar/statements/select.y" { (yyval.boolean) = false; ;} break; - case 1054: -#line 3080 "third_party/libpg_query/grammar/statements/select.y" + case 1056: +#line 3083 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(2) - (2)].list); ;} break; - case 1055: -#line 3081 "third_party/libpg_query/grammar/statements/select.y" + case 1057: +#line 3084 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 1056: -#line 3085 "third_party/libpg_query/grammar/statements/select.y" + case 1058: +#line 3088 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].windef)); ;} break; - case 1057: -#line 3087 "third_party/libpg_query/grammar/statements/select.y" + case 1059: +#line 3090 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].windef)); ;} break; - case 1058: -#line 3092 "third_party/libpg_query/grammar/statements/select.y" + case 1060: +#line 3095 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n = (yyvsp[(3) - (3)].windef); n->name = (yyvsp[(1) - (3)].str); @@ -27711,13 +27757,13 @@ YYLTYPE yylloc; ;} break; - case 1059: -#line 3100 "third_party/libpg_query/grammar/statements/select.y" + case 1061: +#line 3103 "third_party/libpg_query/grammar/statements/select.y" { (yyval.windef) = (yyvsp[(2) - (2)].windef); ;} break; - case 1060: -#line 3102 "third_party/libpg_query/grammar/statements/select.y" + case 1062: +#line 3105 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n = makeNode(PGWindowDef); n->name = (yyvsp[(2) - (2)].str); @@ -27732,13 +27778,13 @@ YYLTYPE yylloc; ;} break; - case 1061: -#line 3115 "third_party/libpg_query/grammar/statements/select.y" + case 1063: +#line 3118 "third_party/libpg_query/grammar/statements/select.y" { (yyval.windef) = NULL; ;} break; - case 1062: -#line 3120 "third_party/libpg_query/grammar/statements/select.y" + case 1064: +#line 3123 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n = makeNode(PGWindowDef); n->name = NULL; @@ -27754,28 +27800,28 @@ YYLTYPE yylloc; ;} break; - case 1063: -#line 3145 "third_party/libpg_query/grammar/statements/select.y" + case 1065: +#line 3148 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1064: -#line 3146 "third_party/libpg_query/grammar/statements/select.y" + case 1066: +#line 3149 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = NULL; ;} break; - case 1065: -#line 3149 "third_party/libpg_query/grammar/statements/select.y" + case 1067: +#line 3152 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(3) - (3)].list); ;} break; - case 1066: -#line 3150 "third_party/libpg_query/grammar/statements/select.y" + case 1068: +#line 3153 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 1067: -#line 3159 "third_party/libpg_query/grammar/statements/select.y" + case 1069: +#line 3162 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n = (yyvsp[(2) - (3)].windef); @@ -27785,8 +27831,8 @@ YYLTYPE yylloc; ;} break; - case 1068: -#line 3167 "third_party/libpg_query/grammar/statements/select.y" + case 1070: +#line 3170 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n = (yyvsp[(2) - (3)].windef); @@ -27796,8 +27842,8 @@ YYLTYPE yylloc; ;} break; - case 1069: -#line 3175 "third_party/libpg_query/grammar/statements/select.y" + case 1071: +#line 3178 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n = (yyvsp[(2) - (3)].windef); @@ -27807,8 +27853,8 @@ YYLTYPE yylloc; ;} break; - case 1070: -#line 3183 "third_party/libpg_query/grammar/statements/select.y" + case 1072: +#line 3186 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n = makeNode(PGWindowDef); @@ -27819,8 +27865,8 @@ YYLTYPE yylloc; ;} break; - case 1071: -#line 3194 "third_party/libpg_query/grammar/statements/select.y" + case 1073: +#line 3197 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n = (yyvsp[(1) - (1)].windef); @@ -27840,8 +27886,8 @@ YYLTYPE yylloc; ;} break; - case 1072: -#line 3212 "third_party/libpg_query/grammar/statements/select.y" + case 1074: +#line 3215 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n1 = (yyvsp[(2) - (4)].windef); PGWindowDef *n2 = (yyvsp[(4) - (4)].windef); @@ -27881,8 +27927,8 @@ YYLTYPE yylloc; ;} break; - case 1073: -#line 3258 "third_party/libpg_query/grammar/statements/select.y" + case 1075: +#line 3261 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n = makeNode(PGWindowDef); @@ -27893,8 +27939,8 @@ YYLTYPE yylloc; ;} break; - case 1074: -#line 3267 "third_party/libpg_query/grammar/statements/select.y" + case 1076: +#line 3270 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n = makeNode(PGWindowDef); @@ -27905,8 +27951,8 @@ YYLTYPE yylloc; ;} break; - case 1075: -#line 3276 "third_party/libpg_query/grammar/statements/select.y" + case 1077: +#line 3279 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n = makeNode(PGWindowDef); @@ -27917,8 +27963,8 @@ YYLTYPE yylloc; ;} break; - case 1076: -#line 3285 "third_party/libpg_query/grammar/statements/select.y" + case 1078: +#line 3288 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n = makeNode(PGWindowDef); @@ -27929,8 +27975,8 @@ YYLTYPE yylloc; ;} break; - case 1077: -#line 3294 "third_party/libpg_query/grammar/statements/select.y" + case 1079: +#line 3297 "third_party/libpg_query/grammar/statements/select.y" { PGWindowDef *n = makeNode(PGWindowDef); @@ -27941,53 +27987,53 @@ YYLTYPE yylloc; ;} break; - case 1078: -#line 3305 "third_party/libpg_query/grammar/statements/select.y" + case 1080: +#line 3308 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ival) = FRAMEOPTION_EXCLUDE_CURRENT_ROW; ;} break; - case 1079: -#line 3306 "third_party/libpg_query/grammar/statements/select.y" + case 1081: +#line 3309 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ival) = FRAMEOPTION_EXCLUDE_GROUP; ;} break; - case 1080: -#line 3307 "third_party/libpg_query/grammar/statements/select.y" + case 1082: +#line 3310 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ival) = FRAMEOPTION_EXCLUDE_TIES; ;} break; - case 1081: -#line 3308 "third_party/libpg_query/grammar/statements/select.y" + case 1083: +#line 3311 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ival) = 0; ;} break; - case 1082: -#line 3309 "third_party/libpg_query/grammar/statements/select.y" + case 1084: +#line 3312 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ival) = 0; ;} break; - case 1083: -#line 3323 "third_party/libpg_query/grammar/statements/select.y" + case 1085: +#line 3326 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(3) - (4)].list); ;} break; - case 1084: -#line 3324 "third_party/libpg_query/grammar/statements/select.y" + case 1086: +#line 3327 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 1085: -#line 3327 "third_party/libpg_query/grammar/statements/select.y" + case 1087: +#line 3330 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list);;} break; - case 1086: -#line 3328 "third_party/libpg_query/grammar/statements/select.y" + case 1088: +#line 3331 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(2) - (5)].list), (yyvsp[(4) - (5)].node)); ;} break; - case 1087: -#line 3332 "third_party/libpg_query/grammar/statements/select.y" + case 1089: +#line 3335 "third_party/libpg_query/grammar/statements/select.y" { PGNamedArgExpr *na = makeNode(PGNamedArgExpr); na->name = (yyvsp[(1) - (3)].str); @@ -27998,321 +28044,321 @@ YYLTYPE yylloc; ;} break; - case 1088: -#line 3342 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} - break; - - case 1089: -#line 3343 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} - break; - case 1090: -#line 3347 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.list) = (yyvsp[(1) - (1)].list); ;} +#line 3345 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; case 1091: -#line 3348 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.list) = (yyvsp[(1) - (2)].list); ;} +#line 3346 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} break; case 1092: -#line 3353 "third_party/libpg_query/grammar/statements/select.y" - { - (yyval.list) = list_make2((yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node)); - ;} +#line 3350 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; case 1093: -#line 3359 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.list) = list_make1((yyvsp[(1) - (1)].list)); ;} +#line 3351 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; case 1094: -#line 3360 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].list)); ;} +#line 3356 "third_party/libpg_query/grammar/statements/select.y" + { + (yyval.list) = list_make2((yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node)); + ;} break; case 1095: -#line 3365 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.list) = (yyvsp[(1) - (1)].list); ;} +#line 3362 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.list) = list_make1((yyvsp[(1) - (1)].list)); ;} break; case 1096: -#line 3366 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.list) = (yyvsp[(1) - (2)].list); ;} +#line 3363 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].list)); ;} break; case 1097: -#line 3371 "third_party/libpg_query/grammar/statements/select.y" +#line 3368 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; case 1098: -#line 3372 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.list) = NULL; ;} +#line 3369 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; case 1099: -#line 3375 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.subquerytype) = PG_ANY_SUBLINK; ;} +#line 3374 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; case 1100: -#line 3376 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.subquerytype) = PG_ANY_SUBLINK; ;} +#line 3375 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.list) = NULL; ;} break; case 1101: -#line 3377 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.subquerytype) = PG_ALL_SUBLINK; ;} +#line 3378 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.subquerytype) = PG_ANY_SUBLINK; ;} break; case 1102: -#line 3380 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.str) = (yyvsp[(1) - (1)].str); ;} +#line 3379 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.subquerytype) = PG_ANY_SUBLINK; ;} break; case 1103: -#line 3381 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.str) = (char*) (yyvsp[(1) - (1)].conststr); ;} +#line 3380 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.subquerytype) = PG_ALL_SUBLINK; ;} break; case 1104: -#line 3384 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.conststr) = "+"; ;} +#line 3383 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; case 1105: -#line 3385 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.conststr) = "-"; ;} +#line 3384 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.str) = (char*) (yyvsp[(1) - (1)].conststr); ;} break; case 1106: -#line 3386 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.conststr) = "*"; ;} +#line 3387 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.conststr) = "+"; ;} break; case 1107: -#line 3387 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.conststr) = "/"; ;} +#line 3388 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.conststr) = "-"; ;} break; case 1108: -#line 3388 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.conststr) = "//"; ;} +#line 3389 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.conststr) = "*"; ;} break; case 1109: -#line 3389 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.conststr) = "%"; ;} +#line 3390 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.conststr) = "/"; ;} break; case 1110: -#line 3390 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.conststr) = "^"; ;} +#line 3391 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.conststr) = "//"; ;} break; case 1111: -#line 3391 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.conststr) = "**"; ;} +#line 3392 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.conststr) = "%"; ;} break; case 1112: -#line 3392 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.conststr) = "<"; ;} +#line 3393 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.conststr) = "^"; ;} break; case 1113: -#line 3393 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.conststr) = ">"; ;} +#line 3394 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.conststr) = "**"; ;} break; case 1114: -#line 3394 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.conststr) = "="; ;} +#line 3395 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.conststr) = "<"; ;} break; case 1115: -#line 3395 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.conststr) = "<="; ;} +#line 3396 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.conststr) = ">"; ;} break; case 1116: -#line 3396 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.conststr) = ">="; ;} +#line 3397 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.conststr) = "="; ;} break; case 1117: -#line 3397 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.conststr) = "<>"; ;} +#line 3398 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.conststr) = "<="; ;} break; case 1118: -#line 3401 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.list) = list_make1(makeString((yyvsp[(1) - (1)].str))); ;} +#line 3399 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.conststr) = ">="; ;} break; case 1119: -#line 3403 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.list) = (yyvsp[(3) - (4)].list); ;} +#line 3400 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.conststr) = "<>"; ;} break; case 1120: -#line 3408 "third_party/libpg_query/grammar/statements/select.y" +#line 3404 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeString((yyvsp[(1) - (1)].str))); ;} break; case 1121: -#line 3410 "third_party/libpg_query/grammar/statements/select.y" +#line 3406 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(3) - (4)].list); ;} break; case 1122: -#line 3415 "third_party/libpg_query/grammar/statements/select.y" +#line 3411 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeString((yyvsp[(1) - (1)].str))); ;} break; case 1123: -#line 3417 "third_party/libpg_query/grammar/statements/select.y" +#line 3413 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(3) - (4)].list); ;} break; case 1124: -#line 3419 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.list) = list_make1(makeString("~~")); ;} +#line 3418 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.list) = list_make1(makeString((yyvsp[(1) - (1)].str))); ;} break; case 1125: -#line 3421 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.list) = list_make1(makeString("!~~")); ;} +#line 3420 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.list) = (yyvsp[(3) - (4)].list); ;} break; case 1126: -#line 3423 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.list) = list_make1(makeString("~~~")); ;} +#line 3422 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.list) = list_make1(makeString("~~")); ;} break; case 1127: -#line 3425 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.list) = list_make1(makeString("!~~~")); ;} +#line 3424 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.list) = list_make1(makeString("!~~")); ;} break; case 1128: -#line 3427 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.list) = list_make1(makeString("~~*")); ;} +#line 3426 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.list) = list_make1(makeString("~~~")); ;} break; case 1129: -#line 3429 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.list) = list_make1(makeString("!~~*")); ;} +#line 3428 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.list) = list_make1(makeString("!~~~")); ;} break; case 1130: -#line 3443 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.list) = list_make1(makeString((yyvsp[(1) - (1)].str))); ;} +#line 3430 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.list) = list_make1(makeString("~~*")); ;} break; case 1131: -#line 3445 "third_party/libpg_query/grammar/statements/select.y" - { (yyval.list) = lcons(makeString((yyvsp[(1) - (3)].str)), (yyvsp[(3) - (3)].list)); ;} +#line 3432 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.list) = list_make1(makeString("!~~*")); ;} break; case 1132: -#line 3450 "third_party/libpg_query/grammar/statements/select.y" +#line 3446 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.list) = list_make1(makeString((yyvsp[(1) - (1)].str))); ;} + break; + + case 1133: +#line 3448 "third_party/libpg_query/grammar/statements/select.y" + { (yyval.list) = lcons(makeString((yyvsp[(1) - (3)].str)), (yyvsp[(3) - (3)].list)); ;} + break; + + case 1134: +#line 3453 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 1133: -#line 3454 "third_party/libpg_query/grammar/statements/select.y" + case 1135: +#line 3457 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} break; - case 1134: -#line 3461 "third_party/libpg_query/grammar/statements/select.y" + case 1136: +#line 3464 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1135: -#line 3466 "third_party/libpg_query/grammar/statements/select.y" + case 1137: +#line 3469 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; - case 1136: -#line 3472 "third_party/libpg_query/grammar/statements/select.y" + case 1138: +#line 3475 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 1137: -#line 3476 "third_party/libpg_query/grammar/statements/select.y" + case 1139: +#line 3479 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} break; - case 1138: -#line 3483 "third_party/libpg_query/grammar/statements/select.y" + case 1140: +#line 3486 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1139: -#line 3488 "third_party/libpg_query/grammar/statements/select.y" + case 1141: +#line 3491 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; - case 1140: -#line 3495 "third_party/libpg_query/grammar/statements/select.y" + case 1142: +#line 3498 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1141: -#line 3499 "third_party/libpg_query/grammar/statements/select.y" + case 1143: +#line 3502 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NULL; ;} break; - case 1142: -#line 3508 "third_party/libpg_query/grammar/statements/select.y" + case 1144: +#line 3511 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 1143: -#line 3512 "third_party/libpg_query/grammar/statements/select.y" + case 1145: +#line 3515 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} break; - case 1144: -#line 3518 "third_party/libpg_query/grammar/statements/select.y" + case 1146: +#line 3521 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 1145: -#line 3522 "third_party/libpg_query/grammar/statements/select.y" + case 1147: +#line 3525 "third_party/libpg_query/grammar/statements/select.y" { PGNamedArgExpr *na = makeNode(PGNamedArgExpr); na->name = (yyvsp[(1) - (3)].str); @@ -28323,8 +28369,8 @@ YYLTYPE yylloc; ;} break; - case 1146: -#line 3531 "third_party/libpg_query/grammar/statements/select.y" + case 1148: +#line 3534 "third_party/libpg_query/grammar/statements/select.y" { PGNamedArgExpr *na = makeNode(PGNamedArgExpr); na->name = (yyvsp[(1) - (3)].str); @@ -28335,151 +28381,151 @@ YYLTYPE yylloc; ;} break; - case 1147: -#line 3541 "third_party/libpg_query/grammar/statements/select.y" + case 1149: +#line 3544 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].typnam)); ;} break; - case 1148: -#line 3542 "third_party/libpg_query/grammar/statements/select.y" + case 1150: +#line 3545 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].typnam)); ;} break; - case 1149: -#line 3547 "third_party/libpg_query/grammar/statements/select.y" + case 1151: +#line 3550 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make2(makeStringConst((yyvsp[(1) - (3)].str), (yylsp[(1) - (3)])), (yyvsp[(3) - (3)].node)); ;} break; - case 1150: -#line 3550 "third_party/libpg_query/grammar/statements/select.y" + case 1152: +#line 3553 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 1151: -#line 3557 "third_party/libpg_query/grammar/statements/select.y" + case 1153: +#line 3560 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1152: -#line 3558 "third_party/libpg_query/grammar/statements/select.y" + case 1154: +#line 3561 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "year"; ;} break; - case 1153: -#line 3559 "third_party/libpg_query/grammar/statements/select.y" + case 1155: +#line 3562 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "month"; ;} break; - case 1154: -#line 3560 "third_party/libpg_query/grammar/statements/select.y" + case 1156: +#line 3563 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "day"; ;} break; - case 1155: -#line 3561 "third_party/libpg_query/grammar/statements/select.y" + case 1157: +#line 3564 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "hour"; ;} break; - case 1156: -#line 3562 "third_party/libpg_query/grammar/statements/select.y" + case 1158: +#line 3565 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "minute"; ;} break; - case 1157: -#line 3563 "third_party/libpg_query/grammar/statements/select.y" + case 1159: +#line 3566 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "second"; ;} break; - case 1158: -#line 3564 "third_party/libpg_query/grammar/statements/select.y" + case 1160: +#line 3567 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "millisecond"; ;} break; - case 1159: -#line 3565 "third_party/libpg_query/grammar/statements/select.y" + case 1161: +#line 3568 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "microsecond"; ;} break; - case 1160: -#line 3566 "third_party/libpg_query/grammar/statements/select.y" + case 1162: +#line 3569 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "week"; ;} break; - case 1161: -#line 3567 "third_party/libpg_query/grammar/statements/select.y" + case 1163: +#line 3570 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "decade"; ;} break; - case 1162: -#line 3568 "third_party/libpg_query/grammar/statements/select.y" + case 1164: +#line 3571 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "century"; ;} break; - case 1163: -#line 3569 "third_party/libpg_query/grammar/statements/select.y" + case 1165: +#line 3572 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (char*) "millennium"; ;} break; - case 1164: -#line 3570 "third_party/libpg_query/grammar/statements/select.y" + case 1166: +#line 3573 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1165: -#line 3581 "third_party/libpg_query/grammar/statements/select.y" + case 1167: +#line 3584 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make4((yyvsp[(1) - (4)].node), (yyvsp[(2) - (4)].node), (yyvsp[(3) - (4)].node), (yyvsp[(4) - (4)].node)); ;} break; - case 1166: -#line 3585 "third_party/libpg_query/grammar/statements/select.y" + case 1168: +#line 3588 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make3((yyvsp[(1) - (3)].node), (yyvsp[(2) - (3)].node), (yyvsp[(3) - (3)].node)); ;} break; - case 1167: -#line 3592 "third_party/libpg_query/grammar/statements/select.y" + case 1169: +#line 3595 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (2)].node); ;} break; - case 1168: -#line 3598 "third_party/libpg_query/grammar/statements/select.y" + case 1170: +#line 3601 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make2((yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node)); ;} break; - case 1169: -#line 3599 "third_party/libpg_query/grammar/statements/select.y" + case 1171: +#line 3602 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 1170: -#line 3616 "third_party/libpg_query/grammar/statements/select.y" + case 1172: +#line 3619 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make3((yyvsp[(1) - (3)].node), (yyvsp[(2) - (3)].node), (yyvsp[(3) - (3)].node)); ;} break; - case 1171: -#line 3620 "third_party/libpg_query/grammar/statements/select.y" + case 1173: +#line 3623 "third_party/libpg_query/grammar/statements/select.y" { /* not legal per SQL99, but might as well allow it */ (yyval.list) = list_make3((yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node), (yyvsp[(2) - (3)].node)); ;} break; - case 1172: -#line 3625 "third_party/libpg_query/grammar/statements/select.y" + case 1174: +#line 3628 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make2((yyvsp[(1) - (2)].node), (yyvsp[(2) - (2)].node)); ;} break; - case 1173: -#line 3629 "third_party/libpg_query/grammar/statements/select.y" + case 1175: +#line 3632 "third_party/libpg_query/grammar/statements/select.y" { /* * Since there are no cases where this syntax allows @@ -28496,45 +28542,45 @@ YYLTYPE yylloc; ;} break; - case 1174: -#line 3644 "third_party/libpg_query/grammar/statements/select.y" + case 1176: +#line 3647 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1175: -#line 3648 "third_party/libpg_query/grammar/statements/select.y" + case 1177: +#line 3651 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 1176: -#line 3652 "third_party/libpg_query/grammar/statements/select.y" + case 1178: +#line 3655 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (2)].node); ;} break; - case 1177: -#line 3655 "third_party/libpg_query/grammar/statements/select.y" + case 1179: +#line 3658 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (2)].node); ;} break; - case 1178: -#line 3658 "third_party/libpg_query/grammar/statements/select.y" + case 1180: +#line 3661 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(3) - (3)].list), (yyvsp[(1) - (3)].node)); ;} break; - case 1179: -#line 3659 "third_party/libpg_query/grammar/statements/select.y" + case 1181: +#line 3662 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(2) - (2)].list); ;} break; - case 1180: -#line 3660 "third_party/libpg_query/grammar/statements/select.y" + case 1182: +#line 3663 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1181: -#line 3664 "third_party/libpg_query/grammar/statements/select.y" + case 1183: +#line 3667 "third_party/libpg_query/grammar/statements/select.y" { PGSubLink *n = makeNode(PGSubLink); n->subselect = (yyvsp[(1) - (1)].node); @@ -28543,13 +28589,13 @@ YYLTYPE yylloc; ;} break; - case 1182: -#line 3670 "third_party/libpg_query/grammar/statements/select.y" + case 1184: +#line 3673 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (PGNode *)(yyvsp[(2) - (3)].list); ;} break; - case 1183: -#line 3681 "third_party/libpg_query/grammar/statements/select.y" + case 1185: +#line 3684 "third_party/libpg_query/grammar/statements/select.y" { PGCaseExpr *c = makeNode(PGCaseExpr); c->casetype = InvalidOid; /* not analyzed yet */ @@ -28561,18 +28607,18 @@ YYLTYPE yylloc; ;} break; - case 1184: -#line 3694 "third_party/libpg_query/grammar/statements/select.y" + case 1186: +#line 3697 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 1185: -#line 3695 "third_party/libpg_query/grammar/statements/select.y" + case 1187: +#line 3698 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (2)].list), (yyvsp[(2) - (2)].node)); ;} break; - case 1186: -#line 3700 "third_party/libpg_query/grammar/statements/select.y" + case 1188: +#line 3703 "third_party/libpg_query/grammar/statements/select.y" { PGCaseWhen *w = makeNode(PGCaseWhen); w->expr = (PGExpr *) (yyvsp[(2) - (4)].node); @@ -28582,42 +28628,42 @@ YYLTYPE yylloc; ;} break; - case 1187: -#line 3710 "third_party/libpg_query/grammar/statements/select.y" + case 1189: +#line 3713 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(2) - (2)].node); ;} break; - case 1188: -#line 3711 "third_party/libpg_query/grammar/statements/select.y" + case 1190: +#line 3714 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 1189: -#line 3714 "third_party/libpg_query/grammar/statements/select.y" + case 1191: +#line 3717 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 1190: -#line 3715 "third_party/libpg_query/grammar/statements/select.y" + case 1192: +#line 3718 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 1191: -#line 3719 "third_party/libpg_query/grammar/statements/select.y" + case 1193: +#line 3722 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeColumnRef((yyvsp[(1) - (1)].str), NIL, (yylsp[(1) - (1)]), yyscanner); ;} break; - case 1192: -#line 3723 "third_party/libpg_query/grammar/statements/select.y" + case 1194: +#line 3726 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeColumnRef((yyvsp[(1) - (2)].str), (yyvsp[(2) - (2)].list), (yylsp[(1) - (2)]), yyscanner); ;} break; - case 1193: -#line 3730 "third_party/libpg_query/grammar/statements/select.y" + case 1195: +#line 3733 "third_party/libpg_query/grammar/statements/select.y" { PGAIndices *ai = makeNode(PGAIndices); ai->is_slice = false; @@ -28627,8 +28673,8 @@ YYLTYPE yylloc; ;} break; - case 1194: -#line 3738 "third_party/libpg_query/grammar/statements/select.y" + case 1196: +#line 3741 "third_party/libpg_query/grammar/statements/select.y" { PGAIndices *ai = makeNode(PGAIndices); ai->is_slice = true; @@ -28638,8 +28684,8 @@ YYLTYPE yylloc; ;} break; - case 1195: -#line 3745 "third_party/libpg_query/grammar/statements/select.y" + case 1197: +#line 3748 "third_party/libpg_query/grammar/statements/select.y" { PGAIndices *ai = makeNode(PGAIndices); ai->is_slice = true; @@ -28650,8 +28696,8 @@ YYLTYPE yylloc; ;} break; - case 1196: -#line 3753 "third_party/libpg_query/grammar/statements/select.y" + case 1198: +#line 3756 "third_party/libpg_query/grammar/statements/select.y" { PGAIndices *ai = makeNode(PGAIndices); ai->is_slice = true; @@ -28661,43 +28707,43 @@ YYLTYPE yylloc; ;} break; - case 1197: -#line 3763 "third_party/libpg_query/grammar/statements/select.y" + case 1199: +#line 3766 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 1198: -#line 3764 "third_party/libpg_query/grammar/statements/select.y" + case 1200: +#line 3767 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = NULL; ;} break; - case 1199: -#line 3769 "third_party/libpg_query/grammar/statements/select.y" + case 1201: +#line 3772 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 1200: -#line 3770 "third_party/libpg_query/grammar/statements/select.y" + case 1202: +#line 3773 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (2)].list), (yyvsp[(2) - (2)].node)); ;} break; - case 1201: -#line 3774 "third_party/libpg_query/grammar/statements/select.y" + case 1203: +#line 3777 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NULL; ;} break; - case 1202: -#line 3775 "third_party/libpg_query/grammar/statements/select.y" + case 1204: +#line 3778 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(NULL); ;} break; - case 1203: -#line 3776 "third_party/libpg_query/grammar/statements/select.y" + case 1205: +#line 3779 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(2) - (3)].list); ;} break; - case 1204: -#line 3781 "third_party/libpg_query/grammar/statements/select.y" + case 1206: +#line 3784 "third_party/libpg_query/grammar/statements/select.y" { if ((yyvsp[(3) - (3)].list)) { PGFuncCall *n = makeFuncCall(list_make1(makeString((yyvsp[(2) - (3)].str))), (yyvsp[(3) - (3)].list)->head->data.ptr_value ? (yyvsp[(3) - (3)].list) : NULL, (yylsp[(2) - (3)])); @@ -28708,8 +28754,8 @@ YYLTYPE yylloc; ;} break; - case 1205: -#line 3790 "third_party/libpg_query/grammar/statements/select.y" + case 1207: +#line 3793 "third_party/libpg_query/grammar/statements/select.y" { PGAIndices *ai = makeNode(PGAIndices); ai->is_slice = false; @@ -28719,8 +28765,8 @@ YYLTYPE yylloc; ;} break; - case 1206: -#line 3798 "third_party/libpg_query/grammar/statements/select.y" + case 1208: +#line 3801 "third_party/libpg_query/grammar/statements/select.y" { PGAIndices *ai = makeNode(PGAIndices); ai->is_slice = true; @@ -28730,8 +28776,8 @@ YYLTYPE yylloc; ;} break; - case 1207: -#line 3805 "third_party/libpg_query/grammar/statements/select.y" + case 1209: +#line 3808 "third_party/libpg_query/grammar/statements/select.y" { PGAIndices *ai = makeNode(PGAIndices); ai->is_slice = true; @@ -28742,8 +28788,8 @@ YYLTYPE yylloc; ;} break; - case 1208: -#line 3814 "third_party/libpg_query/grammar/statements/select.y" + case 1210: +#line 3817 "third_party/libpg_query/grammar/statements/select.y" { PGAIndices *ai = makeNode(PGAIndices); ai->is_slice = true; @@ -28753,48 +28799,48 @@ YYLTYPE yylloc; ;} break; - case 1209: -#line 3829 "third_party/libpg_query/grammar/statements/select.y" + case 1211: +#line 3832 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 1210: -#line 3830 "third_party/libpg_query/grammar/statements/select.y" + case 1212: +#line 3833 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (2)].list), (yyvsp[(2) - (2)].node)); ;} break; - case 1213: -#line 3846 "third_party/libpg_query/grammar/statements/select.y" + case 1215: +#line 3849 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1214: -#line 3847 "third_party/libpg_query/grammar/statements/select.y" + case 1216: +#line 3850 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 1215: -#line 3851 "third_party/libpg_query/grammar/statements/select.y" + case 1217: +#line 3854 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].target)); ;} break; - case 1216: -#line 3852 "third_party/libpg_query/grammar/statements/select.y" + case 1218: +#line 3855 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].target)); ;} break; - case 1217: -#line 3856 "third_party/libpg_query/grammar/statements/select.y" + case 1219: +#line 3859 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1218: -#line 3857 "third_party/libpg_query/grammar/statements/select.y" + case 1220: +#line 3860 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; - case 1219: -#line 3861 "third_party/libpg_query/grammar/statements/select.y" + case 1221: +#line 3864 "third_party/libpg_query/grammar/statements/select.y" { (yyval.target) = makeNode(PGResTarget); (yyval.target)->name = (yyvsp[(3) - (3)].str); @@ -28804,8 +28850,8 @@ YYLTYPE yylloc; ;} break; - case 1220: -#line 3877 "third_party/libpg_query/grammar/statements/select.y" + case 1222: +#line 3880 "third_party/libpg_query/grammar/statements/select.y" { (yyval.target) = makeNode(PGResTarget); (yyval.target)->name = (yyvsp[(2) - (2)].str); @@ -28815,8 +28861,8 @@ YYLTYPE yylloc; ;} break; - case 1221: -#line 3885 "third_party/libpg_query/grammar/statements/select.y" + case 1223: +#line 3888 "third_party/libpg_query/grammar/statements/select.y" { (yyval.target) = makeNode(PGResTarget); (yyval.target)->name = NULL; @@ -28826,140 +28872,140 @@ YYLTYPE yylloc; ;} break; - case 1222: -#line 3894 "third_party/libpg_query/grammar/statements/select.y" + case 1224: +#line 3897 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(3) - (4)].list); ;} break; - case 1223: -#line 3895 "third_party/libpg_query/grammar/statements/select.y" + case 1225: +#line 3898 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeString((yyvsp[(2) - (2)].str))); ;} break; - case 1224: -#line 3898 "third_party/libpg_query/grammar/statements/select.y" + case 1226: +#line 3901 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1225: -#line 3899 "third_party/libpg_query/grammar/statements/select.y" + case 1227: +#line 3902 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NULL; ;} break; - case 1226: -#line 3902 "third_party/libpg_query/grammar/statements/select.y" + case 1228: +#line 3905 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make2((yyvsp[(1) - (3)].node), makeString((yyvsp[(3) - (3)].str))); ;} break; - case 1227: -#line 3906 "third_party/libpg_query/grammar/statements/select.y" + case 1229: +#line 3909 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].list)); ;} break; - case 1228: -#line 3907 "third_party/libpg_query/grammar/statements/select.y" + case 1230: +#line 3910 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].list)); ;} break; - case 1229: -#line 3911 "third_party/libpg_query/grammar/statements/select.y" + case 1231: +#line 3914 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1230: -#line 3912 "third_party/libpg_query/grammar/statements/select.y" + case 1232: +#line 3915 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; - case 1231: -#line 3915 "third_party/libpg_query/grammar/statements/select.y" + case 1233: +#line 3918 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(3) - (4)].list); ;} break; - case 1232: -#line 3916 "third_party/libpg_query/grammar/statements/select.y" + case 1234: +#line 3919 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(2) - (2)].list)); ;} break; - case 1233: -#line 3917 "third_party/libpg_query/grammar/statements/select.y" + case 1235: +#line 3920 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NULL; ;} break; - case 1234: -#line 3927 "third_party/libpg_query/grammar/statements/select.y" + case 1236: +#line 3930 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].range)); ;} break; - case 1235: -#line 3928 "third_party/libpg_query/grammar/statements/select.y" + case 1237: +#line 3931 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].range)); ;} break; - case 1236: -#line 3933 "third_party/libpg_query/grammar/statements/select.y" + case 1238: +#line 3936 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeString((yyvsp[(1) - (1)].str))); ;} break; - case 1237: -#line 3935 "third_party/libpg_query/grammar/statements/select.y" + case 1239: +#line 3938 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), makeString((yyvsp[(3) - (3)].str))); ;} break; - case 1238: -#line 3940 "third_party/libpg_query/grammar/statements/select.y" + case 1240: +#line 3943 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1239: -#line 3941 "third_party/libpg_query/grammar/statements/select.y" + case 1241: +#line 3944 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; - case 1240: -#line 3945 "third_party/libpg_query/grammar/statements/select.y" + case 1242: +#line 3948 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 1241: -#line 3946 "third_party/libpg_query/grammar/statements/select.y" + case 1243: +#line 3949 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(2) - (3)].list); ;} break; - case 1242: -#line 3949 "third_party/libpg_query/grammar/statements/select.y" + case 1244: +#line 3952 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1243: -#line 3961 "third_party/libpg_query/grammar/statements/select.y" + case 1245: +#line 3964 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeString((yyvsp[(1) - (1)].str))); ;} break; - case 1244: -#line 3964 "third_party/libpg_query/grammar/statements/select.y" + case 1246: +#line 3967 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = check_func_name(lcons(makeString((yyvsp[(1) - (2)].str)), (yyvsp[(2) - (2)].list)), yyscanner); ;} break; - case 1245: -#line 3975 "third_party/libpg_query/grammar/statements/select.y" + case 1247: +#line 3978 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeIntConst((yyvsp[(1) - (1)].ival), (yylsp[(1) - (1)])); ;} break; - case 1246: -#line 3979 "third_party/libpg_query/grammar/statements/select.y" + case 1248: +#line 3982 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeFloatConst((yyvsp[(1) - (1)].str), (yylsp[(1) - (1)])); ;} break; - case 1247: -#line 3983 "third_party/libpg_query/grammar/statements/select.y" + case 1249: +#line 3986 "third_party/libpg_query/grammar/statements/select.y" { if ((yyvsp[(2) - (2)].list)) { @@ -28973,15 +29019,15 @@ YYLTYPE yylloc; ;} break; - case 1248: -#line 3995 "third_party/libpg_query/grammar/statements/select.y" + case 1250: +#line 3998 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeBitStringConst((yyvsp[(1) - (1)].str), (yylsp[(1) - (1)])); ;} break; - case 1249: -#line 3999 "third_party/libpg_query/grammar/statements/select.y" + case 1251: +#line 4002 "third_party/libpg_query/grammar/statements/select.y" { /* This is a bit constant per SQL99: * Without Feature F511, "BIT data type", @@ -28992,8 +29038,8 @@ YYLTYPE yylloc; ;} break; - case 1250: -#line 4008 "third_party/libpg_query/grammar/statements/select.y" + case 1252: +#line 4011 "third_party/libpg_query/grammar/statements/select.y" { /* generic type 'literal' syntax */ PGTypeName *t = makeTypeNameFromNameList((yyvsp[(1) - (2)].list)); @@ -29002,8 +29048,8 @@ YYLTYPE yylloc; ;} break; - case 1251: -#line 4015 "third_party/libpg_query/grammar/statements/select.y" + case 1253: +#line 4018 "third_party/libpg_query/grammar/statements/select.y" { /* generic syntax with a type modifier */ PGTypeName *t = makeTypeNameFromNameList((yyvsp[(1) - (7)].list)); @@ -29043,146 +29089,146 @@ YYLTYPE yylloc; ;} break; - case 1252: -#line 4053 "third_party/libpg_query/grammar/statements/select.y" + case 1254: +#line 4056 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeStringConstCast((yyvsp[(2) - (2)].str), (yylsp[(2) - (2)]), (yyvsp[(1) - (2)].typnam)); ;} break; - case 1253: -#line 4057 "third_party/libpg_query/grammar/statements/select.y" + case 1255: +#line 4060 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeIntervalNode((yyvsp[(3) - (5)].node), (yylsp[(3) - (5)]), (yyvsp[(5) - (5)].list)); ;} break; - case 1254: -#line 4061 "third_party/libpg_query/grammar/statements/select.y" + case 1256: +#line 4064 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeIntervalNode((yyvsp[(2) - (3)].ival), (yylsp[(2) - (3)]), (yyvsp[(3) - (3)].list)); ;} break; - case 1255: -#line 4065 "third_party/libpg_query/grammar/statements/select.y" + case 1257: +#line 4068 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeIntervalNode((yyvsp[(2) - (3)].str), (yylsp[(2) - (3)]), (yyvsp[(3) - (3)].list)); ;} break; - case 1256: -#line 4069 "third_party/libpg_query/grammar/statements/select.y" + case 1258: +#line 4072 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeBoolAConst(true, (yylsp[(1) - (1)])); ;} break; - case 1257: -#line 4073 "third_party/libpg_query/grammar/statements/select.y" + case 1259: +#line 4076 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeBoolAConst(false, (yylsp[(1) - (1)])); ;} break; - case 1258: -#line 4077 "third_party/libpg_query/grammar/statements/select.y" + case 1260: +#line 4080 "third_party/libpg_query/grammar/statements/select.y" { (yyval.node) = makeNullAConst((yylsp[(1) - (1)])); ;} break; - case 1259: -#line 4082 "third_party/libpg_query/grammar/statements/select.y" + case 1261: +#line 4085 "third_party/libpg_query/grammar/statements/select.y" { (yyval.ival) = (yyvsp[(1) - (1)].ival); ;} break; - case 1260: -#line 4099 "third_party/libpg_query/grammar/statements/select.y" + case 1262: +#line 4102 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1261: -#line 4100 "third_party/libpg_query/grammar/statements/select.y" + case 1263: +#line 4103 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = pstrdup((yyvsp[(1) - (1)].keyword)); ;} break; - case 1262: -#line 4101 "third_party/libpg_query/grammar/statements/select.y" + case 1264: +#line 4104 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = pstrdup((yyvsp[(1) - (1)].keyword)); ;} break; - case 1263: -#line 4104 "third_party/libpg_query/grammar/statements/select.y" + case 1265: +#line 4107 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1264: -#line 4105 "third_party/libpg_query/grammar/statements/select.y" + case 1266: +#line 4108 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = pstrdup((yyvsp[(1) - (1)].keyword)); ;} break; - case 1265: -#line 4106 "third_party/libpg_query/grammar/statements/select.y" + case 1267: +#line 4109 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = pstrdup((yyvsp[(1) - (1)].keyword)); ;} break; - case 1266: -#line 4109 "third_party/libpg_query/grammar/statements/select.y" + case 1268: +#line 4112 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1267: -#line 4110 "third_party/libpg_query/grammar/statements/select.y" + case 1269: +#line 4113 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = pstrdup((yyvsp[(1) - (1)].keyword)); ;} break; - case 1268: -#line 4111 "third_party/libpg_query/grammar/statements/select.y" + case 1270: +#line 4114 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = pstrdup((yyvsp[(1) - (1)].keyword)); ;} break; - case 1269: -#line 4114 "third_party/libpg_query/grammar/statements/select.y" + case 1271: +#line 4117 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeString((yyvsp[(1) - (1)].str))); ;} break; - case 1270: -#line 4115 "third_party/libpg_query/grammar/statements/select.y" + case 1272: +#line 4118 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lcons(makeString((yyvsp[(1) - (2)].str)), (yyvsp[(2) - (2)].list)); ;} break; - case 1271: -#line 4119 "third_party/libpg_query/grammar/statements/select.y" + case 1273: +#line 4122 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = list_make1(makeString((yyvsp[(2) - (2)].str))); ;} break; - case 1272: -#line 4121 "third_party/libpg_query/grammar/statements/select.y" + case 1274: +#line 4124 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), makeString((yyvsp[(3) - (3)].str))); ;} break; - case 1273: -#line 4125 "third_party/libpg_query/grammar/statements/select.y" + case 1275: +#line 4128 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = (yyvsp[(2) - (3)].list); ;} break; - case 1274: -#line 4126 "third_party/libpg_query/grammar/statements/select.y" + case 1276: +#line 4129 "third_party/libpg_query/grammar/statements/select.y" { (yyval.list) = NIL; ;} break; - case 1276: -#line 4133 "third_party/libpg_query/grammar/statements/select.y" + case 1278: +#line 4136 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1277: -#line 4134 "third_party/libpg_query/grammar/statements/select.y" + case 1279: +#line 4137 "third_party/libpg_query/grammar/statements/select.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1278: + case 1280: #line 8 "third_party/libpg_query/grammar/statements/prepare.y" { PGPrepareStmt *n = makeNode(PGPrepareStmt); @@ -29193,17 +29239,17 @@ YYLTYPE yylloc; ;} break; - case 1279: + case 1281: #line 18 "third_party/libpg_query/grammar/statements/prepare.y" { (yyval.list) = (yyvsp[(2) - (3)].list); ;} break; - case 1280: + case 1282: #line 19 "third_party/libpg_query/grammar/statements/prepare.y" { (yyval.list) = NIL; ;} break; - case 1287: + case 1289: #line 8 "third_party/libpg_query/grammar/statements/create_schema.y" { PGCreateSchemaStmt *n = makeNode(PGCreateSchemaStmt); @@ -29225,7 +29271,7 @@ YYLTYPE yylloc; ;} break; - case 1288: + case 1290: #line 27 "third_party/libpg_query/grammar/statements/create_schema.y" { PGCreateSchemaStmt *n = makeNode(PGCreateSchemaStmt); @@ -29252,7 +29298,7 @@ YYLTYPE yylloc; ;} break; - case 1289: + case 1291: #line 51 "third_party/libpg_query/grammar/statements/create_schema.y" { PGCreateSchemaStmt *n = makeNode(PGCreateSchemaStmt); @@ -29274,7 +29320,7 @@ YYLTYPE yylloc; ;} break; - case 1290: + case 1292: #line 74 "third_party/libpg_query/grammar/statements/create_schema.y" { if ((yyloc) < 0) /* see comments for YYLLOC_DEFAULT */ @@ -29283,12 +29329,12 @@ YYLTYPE yylloc; ;} break; - case 1291: + case 1293: #line 80 "third_party/libpg_query/grammar/statements/create_schema.y" { (yyval.list) = NIL; ;} break; - case 1296: + case 1298: #line 11 "third_party/libpg_query/grammar/statements/index.y" { PGIndexStmt *n = makeNode(PGIndexStmt); @@ -29314,7 +29360,7 @@ YYLTYPE yylloc; ;} break; - case 1297: + case 1299: #line 36 "third_party/libpg_query/grammar/statements/index.y" { PGIndexStmt *n = makeNode(PGIndexStmt); @@ -29340,62 +29386,62 @@ YYLTYPE yylloc; ;} break; - case 1298: + case 1300: #line 62 "third_party/libpg_query/grammar/statements/index.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1299: + case 1301: #line 66 "third_party/libpg_query/grammar/statements/index.y" { (yyval.str) = (yyvsp[(2) - (2)].str); ;} break; - case 1300: + case 1302: #line 67 "third_party/libpg_query/grammar/statements/index.y" { (yyval.str) = (char*) DEFAULT_INDEX_TYPE; ;} break; - case 1301: + case 1303: #line 72 "third_party/libpg_query/grammar/statements/index.y" { (yyval.boolean) = true; ;} break; - case 1302: + case 1304: #line 73 "third_party/libpg_query/grammar/statements/index.y" { (yyval.boolean) = false; ;} break; - case 1303: + case 1305: #line 78 "third_party/libpg_query/grammar/statements/index.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1304: + case 1306: #line 79 "third_party/libpg_query/grammar/statements/index.y" { (yyval.str) = NULL; ;} break; - case 1305: + case 1307: #line 83 "third_party/libpg_query/grammar/statements/index.y" { (yyval.list) = (yyvsp[(2) - (2)].list); ;} break; - case 1306: + case 1308: #line 84 "third_party/libpg_query/grammar/statements/index.y" { (yyval.list) = NIL; ;} break; - case 1307: + case 1309: #line 89 "third_party/libpg_query/grammar/statements/index.y" { (yyval.boolean) = true; ;} break; - case 1308: + case 1310: #line 90 "third_party/libpg_query/grammar/statements/index.y" { (yyval.boolean) = false; ;} break; - case 1309: + case 1311: #line 8 "third_party/libpg_query/grammar/statements/alter_schema.y" { PGAlterObjectSchemaStmt *n = makeNode(PGAlterObjectSchemaStmt); @@ -29407,7 +29453,7 @@ YYLTYPE yylloc; ;} break; - case 1310: + case 1312: #line 17 "third_party/libpg_query/grammar/statements/alter_schema.y" { PGAlterObjectSchemaStmt *n = makeNode(PGAlterObjectSchemaStmt); @@ -29419,7 +29465,7 @@ YYLTYPE yylloc; ;} break; - case 1311: + case 1313: #line 26 "third_party/libpg_query/grammar/statements/alter_schema.y" { PGAlterObjectSchemaStmt *n = makeNode(PGAlterObjectSchemaStmt); @@ -29431,7 +29477,7 @@ YYLTYPE yylloc; ;} break; - case 1312: + case 1314: #line 35 "third_party/libpg_query/grammar/statements/alter_schema.y" { PGAlterObjectSchemaStmt *n = makeNode(PGAlterObjectSchemaStmt); @@ -29443,7 +29489,7 @@ YYLTYPE yylloc; ;} break; - case 1313: + case 1315: #line 44 "third_party/libpg_query/grammar/statements/alter_schema.y" { PGAlterObjectSchemaStmt *n = makeNode(PGAlterObjectSchemaStmt); @@ -29455,7 +29501,7 @@ YYLTYPE yylloc; ;} break; - case 1314: + case 1316: #line 53 "third_party/libpg_query/grammar/statements/alter_schema.y" { PGAlterObjectSchemaStmt *n = makeNode(PGAlterObjectSchemaStmt); @@ -29467,7 +29513,7 @@ YYLTYPE yylloc; ;} break; - case 1315: + case 1317: #line 6 "third_party/libpg_query/grammar/statements/checkpoint.y" { PGCheckPointStmt *n = makeNode(PGCheckPointStmt); @@ -29477,7 +29523,7 @@ YYLTYPE yylloc; ;} break; - case 1316: + case 1318: #line 13 "third_party/libpg_query/grammar/statements/checkpoint.y" { PGCheckPointStmt *n = makeNode(PGCheckPointStmt); @@ -29487,17 +29533,17 @@ YYLTYPE yylloc; ;} break; - case 1317: + case 1319: #line 22 "third_party/libpg_query/grammar/statements/checkpoint.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1318: + case 1320: #line 23 "third_party/libpg_query/grammar/statements/checkpoint.y" { (yyval.str) = NULL; ;} break; - case 1319: + case 1321: #line 8 "third_party/libpg_query/grammar/statements/comment_on.y" { PGCommentOnStmt *n = makeNode(PGCommentOnStmt); @@ -29508,7 +29554,7 @@ YYLTYPE yylloc; ;} break; - case 1320: + case 1322: #line 16 "third_party/libpg_query/grammar/statements/comment_on.y" { PGCommentOnStmt *n = makeNode(PGCommentOnStmt); @@ -29519,67 +29565,67 @@ YYLTYPE yylloc; ;} break; - case 1321: + case 1323: #line 26 "third_party/libpg_query/grammar/statements/comment_on.y" { (yyval.node) = makeStringConst((yyvsp[(1) - (1)].str), (yylsp[(1) - (1)])); ;} break; - case 1322: + case 1324: #line 27 "third_party/libpg_query/grammar/statements/comment_on.y" { (yyval.node) = makeNullAConst((yylsp[(1) - (1)])); ;} break; - case 1323: + case 1325: #line 30 "third_party/libpg_query/grammar/statements/comment_on.y" { (yyval.objtype) = PG_OBJECT_TABLE; ;} break; - case 1324: + case 1326: #line 31 "third_party/libpg_query/grammar/statements/comment_on.y" { (yyval.objtype) = PG_OBJECT_SEQUENCE; ;} break; - case 1325: + case 1327: #line 32 "third_party/libpg_query/grammar/statements/comment_on.y" { (yyval.objtype) = PG_OBJECT_FUNCTION; ;} break; - case 1326: + case 1328: #line 33 "third_party/libpg_query/grammar/statements/comment_on.y" { (yyval.objtype) = PG_OBJECT_FUNCTION; ;} break; - case 1327: + case 1329: #line 34 "third_party/libpg_query/grammar/statements/comment_on.y" { (yyval.objtype) = PG_OBJECT_TABLE_MACRO; ;} break; - case 1328: + case 1330: #line 35 "third_party/libpg_query/grammar/statements/comment_on.y" { (yyval.objtype) = PG_OBJECT_VIEW; ;} break; - case 1329: + case 1331: #line 36 "third_party/libpg_query/grammar/statements/comment_on.y" { (yyval.objtype) = PG_OBJECT_DATABASE; ;} break; - case 1330: + case 1332: #line 37 "third_party/libpg_query/grammar/statements/comment_on.y" { (yyval.objtype) = PG_OBJECT_INDEX; ;} break; - case 1331: + case 1333: #line 38 "third_party/libpg_query/grammar/statements/comment_on.y" { (yyval.objtype) = PG_OBJECT_SCHEMA; ;} break; - case 1332: + case 1334: #line 39 "third_party/libpg_query/grammar/statements/comment_on.y" { (yyval.objtype) = PG_OBJECT_TYPE; ;} break; - case 1333: + case 1335: #line 8 "third_party/libpg_query/grammar/statements/export.y" { PGExportStmt *n = makeNode(PGExportStmt); @@ -29593,7 +29639,7 @@ YYLTYPE yylloc; ;} break; - case 1334: + case 1336: #line 20 "third_party/libpg_query/grammar/statements/export.y" { PGExportStmt *n = makeNode(PGExportStmt); @@ -29607,7 +29653,7 @@ YYLTYPE yylloc; ;} break; - case 1335: + case 1337: #line 34 "third_party/libpg_query/grammar/statements/export.y" { PGImportStmt *n = makeNode(PGImportStmt); @@ -29616,7 +29662,7 @@ YYLTYPE yylloc; ;} break; - case 1336: + case 1338: #line 10 "third_party/libpg_query/grammar/statements/explain.y" { PGExplainStmt *n = makeNode(PGExplainStmt); @@ -29626,7 +29672,7 @@ YYLTYPE yylloc; ;} break; - case 1337: + case 1339: #line 17 "third_party/libpg_query/grammar/statements/explain.y" { PGExplainStmt *n = makeNode(PGExplainStmt); @@ -29639,7 +29685,7 @@ YYLTYPE yylloc; ;} break; - case 1338: + case 1340: #line 27 "third_party/libpg_query/grammar/statements/explain.y" { PGExplainStmt *n = makeNode(PGExplainStmt); @@ -29649,7 +29695,7 @@ YYLTYPE yylloc; ;} break; - case 1339: + case 1341: #line 34 "third_party/libpg_query/grammar/statements/explain.y" { PGExplainStmt *n = makeNode(PGExplainStmt); @@ -29659,118 +29705,118 @@ YYLTYPE yylloc; ;} break; - case 1340: + case 1342: #line 44 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.boolean) = true; ;} break; - case 1341: + case 1343: #line 45 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.boolean) = false; ;} break; - case 1342: + case 1344: #line 50 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.node) = (PGNode *) makeString((yyvsp[(1) - (1)].str)); ;} break; - case 1343: + case 1345: #line 51 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.node) = (PGNode *) (yyvsp[(1) - (1)].value); ;} break; - case 1344: + case 1346: #line 52 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.node) = NULL; ;} break; - case 1375: + case 1377: #line 90 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1376: + case 1378: #line 91 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.str) = pstrdup((yyvsp[(1) - (1)].keyword)); ;} break; - case 1377: + case 1379: #line 92 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.str) = pstrdup((yyvsp[(1) - (1)].keyword)); ;} break; - case 1378: + case 1380: #line 97 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1379: + case 1381: #line 98 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1380: + case 1382: #line 104 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].defelt)); ;} break; - case 1381: + case 1383: #line 108 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].defelt)); ;} break; - case 1382: + case 1384: #line 115 "third_party/libpg_query/grammar/statements/explain.y" {;} break; - case 1383: + case 1385: #line 116 "third_party/libpg_query/grammar/statements/explain.y" {;} break; - case 1384: + case 1386: #line 121 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.str) = (char*) "true"; ;} break; - case 1385: + case 1387: #line 122 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.str) = (char*) "false"; ;} break; - case 1386: + case 1388: #line 123 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.str) = (char*) "on"; ;} break; - case 1387: + case 1389: #line 129 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1388: + case 1390: #line 135 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.defelt) = makeDefElem((yyvsp[(1) - (2)].str), (yyvsp[(2) - (2)].node), (yylsp[(1) - (2)])); ;} break; - case 1389: + case 1391: #line 142 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1390: + case 1392: #line 143 "third_party/libpg_query/grammar/statements/explain.y" { (yyval.str) = (char*) "analyze"; ;} break; - case 1391: + case 1393: #line 11 "third_party/libpg_query/grammar/statements/variable_set.y" { PGVariableSetStmt *n = (yyvsp[(2) - (2)].vsetstmt); @@ -29779,7 +29825,7 @@ YYLTYPE yylloc; ;} break; - case 1392: + case 1394: #line 17 "third_party/libpg_query/grammar/statements/variable_set.y" { PGVariableSetStmt *n = (yyvsp[(3) - (3)].vsetstmt); @@ -29788,7 +29834,7 @@ YYLTYPE yylloc; ;} break; - case 1393: + case 1395: #line 23 "third_party/libpg_query/grammar/statements/variable_set.y" { PGVariableSetStmt *n = (yyvsp[(3) - (3)].vsetstmt); @@ -29797,7 +29843,7 @@ YYLTYPE yylloc; ;} break; - case 1394: + case 1396: #line 29 "third_party/libpg_query/grammar/statements/variable_set.y" { PGVariableSetStmt *n = (yyvsp[(3) - (3)].vsetstmt); @@ -29806,12 +29852,12 @@ YYLTYPE yylloc; ;} break; - case 1395: + case 1397: #line 38 "third_party/libpg_query/grammar/statements/variable_set.y" {(yyval.vsetstmt) = (yyvsp[(1) - (1)].vsetstmt);;} break; - case 1396: + case 1398: #line 40 "third_party/libpg_query/grammar/statements/variable_set.y" { PGVariableSetStmt *n = makeNode(PGVariableSetStmt); @@ -29821,7 +29867,7 @@ YYLTYPE yylloc; ;} break; - case 1397: + case 1399: #line 48 "third_party/libpg_query/grammar/statements/variable_set.y" { PGVariableSetStmt *n = makeNode(PGVariableSetStmt); @@ -29835,7 +29881,7 @@ YYLTYPE yylloc; ;} break; - case 1398: + case 1400: #line 59 "third_party/libpg_query/grammar/statements/variable_set.y" { PGVariableSetStmt *n = makeNode(PGVariableSetStmt); @@ -29846,7 +29892,7 @@ YYLTYPE yylloc; ;} break; - case 1399: + case 1401: #line 71 "third_party/libpg_query/grammar/statements/variable_set.y" { PGVariableSetStmt *n = makeNode(PGVariableSetStmt); @@ -29857,7 +29903,7 @@ YYLTYPE yylloc; ;} break; - case 1400: + case 1402: #line 79 "third_party/libpg_query/grammar/statements/variable_set.y" { PGVariableSetStmt *n = makeNode(PGVariableSetStmt); @@ -29868,26 +29914,26 @@ YYLTYPE yylloc; ;} break; - case 1401: + case 1403: #line 90 "third_party/libpg_query/grammar/statements/variable_set.y" { (yyval.node) = (yyvsp[(1) - (1)].node); ;} break; - case 1402: + case 1404: #line 96 "third_party/libpg_query/grammar/statements/variable_set.y" { (yyval.node) = makeStringConst((yyvsp[(1) - (1)].str), (yylsp[(1) - (1)])); ;} break; - case 1403: + case 1405: #line 100 "third_party/libpg_query/grammar/statements/variable_set.y" { (yyval.node) = makeStringConst((yyvsp[(1) - (1)].str), (yylsp[(1) - (1)])); ;} break; - case 1404: + case 1406: #line 104 "third_party/libpg_query/grammar/statements/variable_set.y" { PGTypeName *t = (yyvsp[(1) - (3)].typnam); @@ -29905,7 +29951,7 @@ YYLTYPE yylloc; ;} break; - case 1405: + case 1407: #line 119 "third_party/libpg_query/grammar/statements/variable_set.y" { PGTypeName *t = (yyvsp[(1) - (5)].typnam); @@ -29915,32 +29961,32 @@ YYLTYPE yylloc; ;} break; - case 1406: + case 1408: #line 125 "third_party/libpg_query/grammar/statements/variable_set.y" { (yyval.node) = makeAConst((yyvsp[(1) - (1)].value), (yylsp[(1) - (1)])); ;} break; - case 1407: + case 1409: #line 126 "third_party/libpg_query/grammar/statements/variable_set.y" { (yyval.node) = NULL; ;} break; - case 1408: + case 1410: #line 127 "third_party/libpg_query/grammar/statements/variable_set.y" { (yyval.node) = NULL; ;} break; - case 1409: + case 1411: #line 131 "third_party/libpg_query/grammar/statements/variable_set.y" { (yyval.list) = list_make1((yyvsp[(1) - (1)].node)); ;} break; - case 1410: + case 1412: #line 132 "third_party/libpg_query/grammar/statements/variable_set.y" { (yyval.list) = lappend((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].node)); ;} break; - case 1411: + case 1413: #line 8 "third_party/libpg_query/grammar/statements/load.y" { PGLoadStmt *n = makeNode(PGLoadStmt); @@ -29951,7 +29997,7 @@ YYLTYPE yylloc; ;} break; - case 1412: + case 1414: #line 15 "third_party/libpg_query/grammar/statements/load.y" { PGLoadStmt *n = makeNode(PGLoadStmt); @@ -29962,7 +30008,7 @@ YYLTYPE yylloc; ;} break; - case 1413: + case 1415: #line 22 "third_party/libpg_query/grammar/statements/load.y" { PGLoadStmt *n = makeNode(PGLoadStmt); @@ -29973,7 +30019,7 @@ YYLTYPE yylloc; ;} break; - case 1414: + case 1416: #line 29 "third_party/libpg_query/grammar/statements/load.y" { PGLoadStmt *n = makeNode(PGLoadStmt); @@ -29984,7 +30030,7 @@ YYLTYPE yylloc; ;} break; - case 1415: + case 1417: #line 36 "third_party/libpg_query/grammar/statements/load.y" { PGLoadStmt *n = makeNode(PGLoadStmt); @@ -29995,27 +30041,27 @@ YYLTYPE yylloc; ;} break; - case 1416: + case 1418: #line 45 "third_party/libpg_query/grammar/statements/load.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1417: + case 1419: #line 46 "third_party/libpg_query/grammar/statements/load.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1418: + case 1420: #line 48 "third_party/libpg_query/grammar/statements/load.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1419: + case 1421: #line 49 "third_party/libpg_query/grammar/statements/load.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1420: + case 1422: #line 9 "third_party/libpg_query/grammar/statements/vacuum.y" { PGVacuumStmt *n = makeNode(PGVacuumStmt); @@ -30032,7 +30078,7 @@ YYLTYPE yylloc; ;} break; - case 1421: + case 1423: #line 23 "third_party/libpg_query/grammar/statements/vacuum.y" { PGVacuumStmt *n = makeNode(PGVacuumStmt); @@ -30049,7 +30095,7 @@ YYLTYPE yylloc; ;} break; - case 1422: + case 1424: #line 37 "third_party/libpg_query/grammar/statements/vacuum.y" { PGVacuumStmt *n = (PGVacuumStmt *) (yyvsp[(5) - (5)].node); @@ -30064,7 +30110,7 @@ YYLTYPE yylloc; ;} break; - case 1423: + case 1425: #line 49 "third_party/libpg_query/grammar/statements/vacuum.y" { PGVacuumStmt *n = makeNode(PGVacuumStmt); @@ -30075,7 +30121,7 @@ YYLTYPE yylloc; ;} break; - case 1424: + case 1426: #line 57 "third_party/libpg_query/grammar/statements/vacuum.y" { PGVacuumStmt *n = makeNode(PGVacuumStmt); @@ -30088,27 +30134,27 @@ YYLTYPE yylloc; ;} break; - case 1425: + case 1427: #line 70 "third_party/libpg_query/grammar/statements/vacuum.y" { (yyval.ival) = PG_VACOPT_ANALYZE; ;} break; - case 1426: + case 1428: #line 71 "third_party/libpg_query/grammar/statements/vacuum.y" { (yyval.ival) = PG_VACOPT_VERBOSE; ;} break; - case 1427: + case 1429: #line 72 "third_party/libpg_query/grammar/statements/vacuum.y" { (yyval.ival) = PG_VACOPT_FREEZE; ;} break; - case 1428: + case 1430: #line 73 "third_party/libpg_query/grammar/statements/vacuum.y" { (yyval.ival) = PG_VACOPT_FULL; ;} break; - case 1429: + case 1431: #line 75 "third_party/libpg_query/grammar/statements/vacuum.y" { if (strcmp((yyvsp[(1) - (1)].str), "disable_page_skipping") == 0) @@ -30121,37 +30167,37 @@ YYLTYPE yylloc; ;} break; - case 1430: + case 1432: #line 87 "third_party/libpg_query/grammar/statements/vacuum.y" { (yyval.boolean) = true; ;} break; - case 1431: + case 1433: #line 88 "third_party/libpg_query/grammar/statements/vacuum.y" { (yyval.boolean) = false; ;} break; - case 1432: + case 1434: #line 93 "third_party/libpg_query/grammar/statements/vacuum.y" { (yyval.ival) = (yyvsp[(1) - (1)].ival); ;} break; - case 1433: + case 1435: #line 94 "third_party/libpg_query/grammar/statements/vacuum.y" { (yyval.ival) = (yyvsp[(1) - (3)].ival) | (yyvsp[(3) - (3)].ival); ;} break; - case 1434: + case 1436: #line 98 "third_party/libpg_query/grammar/statements/vacuum.y" { (yyval.boolean) = true; ;} break; - case 1435: + case 1437: #line 99 "third_party/libpg_query/grammar/statements/vacuum.y" { (yyval.boolean) = false; ;} break; - case 1436: + case 1438: #line 9 "third_party/libpg_query/grammar/statements/delete.y" { PGDeleteStmt *n = makeNode(PGDeleteStmt); @@ -30164,7 +30210,7 @@ YYLTYPE yylloc; ;} break; - case 1437: + case 1439: #line 19 "third_party/libpg_query/grammar/statements/delete.y" { PGDeleteStmt *n = makeNode(PGDeleteStmt); @@ -30177,14 +30223,14 @@ YYLTYPE yylloc; ;} break; - case 1438: + case 1440: #line 32 "third_party/libpg_query/grammar/statements/delete.y" { (yyval.range) = (yyvsp[(1) - (1)].range); ;} break; - case 1439: + case 1441: #line 36 "third_party/libpg_query/grammar/statements/delete.y" { PGAlias *alias = makeNode(PGAlias); @@ -30194,7 +30240,7 @@ YYLTYPE yylloc; ;} break; - case 1440: + case 1442: #line 43 "third_party/libpg_query/grammar/statements/delete.y" { PGAlias *alias = makeNode(PGAlias); @@ -30204,27 +30250,27 @@ YYLTYPE yylloc; ;} break; - case 1441: + case 1443: #line 53 "third_party/libpg_query/grammar/statements/delete.y" { (yyval.node) = (yyvsp[(2) - (2)].node); ;} break; - case 1442: + case 1444: #line 54 "third_party/libpg_query/grammar/statements/delete.y" { (yyval.node) = NULL; ;} break; - case 1443: + case 1445: #line 60 "third_party/libpg_query/grammar/statements/delete.y" { (yyval.list) = (yyvsp[(2) - (2)].list); ;} break; - case 1444: + case 1446: #line 61 "third_party/libpg_query/grammar/statements/delete.y" { (yyval.list) = NIL; ;} break; - case 1445: + case 1447: #line 10 "third_party/libpg_query/grammar/statements/analyze.y" { PGVacuumStmt *n = makeNode(PGVacuumStmt); @@ -30237,7 +30283,7 @@ YYLTYPE yylloc; ;} break; - case 1446: + case 1448: #line 20 "third_party/libpg_query/grammar/statements/analyze.y" { PGVacuumStmt *n = makeNode(PGVacuumStmt); @@ -30250,7 +30296,7 @@ YYLTYPE yylloc; ;} break; - case 1447: + case 1449: #line 8 "third_party/libpg_query/grammar/statements/attach.y" { PGAttachStmt *n = makeNode(PGAttachStmt); @@ -30262,7 +30308,7 @@ YYLTYPE yylloc; ;} break; - case 1448: + case 1450: #line 17 "third_party/libpg_query/grammar/statements/attach.y" { PGAttachStmt *n = makeNode(PGAttachStmt); @@ -30274,7 +30320,7 @@ YYLTYPE yylloc; ;} break; - case 1449: + case 1451: #line 29 "third_party/libpg_query/grammar/statements/attach.y" { PGDetachStmt *n = makeNode(PGDetachStmt); @@ -30284,7 +30330,7 @@ YYLTYPE yylloc; ;} break; - case 1450: + case 1452: #line 36 "third_party/libpg_query/grammar/statements/attach.y" { PGDetachStmt *n = makeNode(PGDetachStmt); @@ -30294,7 +30340,7 @@ YYLTYPE yylloc; ;} break; - case 1451: + case 1453: #line 43 "third_party/libpg_query/grammar/statements/attach.y" { PGDetachStmt *n = makeNode(PGDetachStmt); @@ -30304,27 +30350,27 @@ YYLTYPE yylloc; ;} break; - case 1452: + case 1454: #line 51 "third_party/libpg_query/grammar/statements/attach.y" {;} break; - case 1453: + case 1455: #line 52 "third_party/libpg_query/grammar/statements/attach.y" {;} break; - case 1454: + case 1456: #line 56 "third_party/libpg_query/grammar/statements/attach.y" { (yyval.str) = (yyvsp[(2) - (2)].str); ;} break; - case 1455: + case 1457: #line 57 "third_party/libpg_query/grammar/statements/attach.y" { (yyval.str) = NULL; ;} break; - case 1456: + case 1458: #line 3 "third_party/libpg_query/grammar/statements/variable_reset.y" { (yyvsp[(2) - (2)].vsetstmt)->scope = VAR_SET_SCOPE_DEFAULT; @@ -30332,7 +30378,7 @@ YYLTYPE yylloc; ;} break; - case 1457: + case 1459: #line 8 "third_party/libpg_query/grammar/statements/variable_reset.y" { (yyvsp[(3) - (3)].vsetstmt)->scope = VAR_SET_SCOPE_LOCAL; @@ -30340,7 +30386,7 @@ YYLTYPE yylloc; ;} break; - case 1458: + case 1460: #line 13 "third_party/libpg_query/grammar/statements/variable_reset.y" { (yyvsp[(3) - (3)].vsetstmt)->scope = VAR_SET_SCOPE_SESSION; @@ -30348,7 +30394,7 @@ YYLTYPE yylloc; ;} break; - case 1459: + case 1461: #line 18 "third_party/libpg_query/grammar/statements/variable_reset.y" { (yyvsp[(3) - (3)].vsetstmt)->scope = VAR_SET_SCOPE_GLOBAL; @@ -30356,7 +30402,7 @@ YYLTYPE yylloc; ;} break; - case 1460: + case 1462: #line 27 "third_party/libpg_query/grammar/statements/variable_reset.y" { PGVariableSetStmt *n = makeNode(PGVariableSetStmt); @@ -30366,7 +30412,7 @@ YYLTYPE yylloc; ;} break; - case 1461: + case 1463: #line 34 "third_party/libpg_query/grammar/statements/variable_reset.y" { PGVariableSetStmt *n = makeNode(PGVariableSetStmt); @@ -30375,12 +30421,12 @@ YYLTYPE yylloc; ;} break; - case 1462: + case 1464: #line 43 "third_party/libpg_query/grammar/statements/variable_reset.y" { (yyval.vsetstmt) = (yyvsp[(1) - (1)].vsetstmt); ;} break; - case 1463: + case 1465: #line 45 "third_party/libpg_query/grammar/statements/variable_reset.y" { PGVariableSetStmt *n = makeNode(PGVariableSetStmt); @@ -30390,7 +30436,7 @@ YYLTYPE yylloc; ;} break; - case 1464: + case 1466: #line 52 "third_party/libpg_query/grammar/statements/variable_reset.y" { PGVariableSetStmt *n = makeNode(PGVariableSetStmt); @@ -30400,7 +30446,7 @@ YYLTYPE yylloc; ;} break; - case 1465: + case 1467: #line 3 "third_party/libpg_query/grammar/statements/variable_show.y" { PGVariableShowSelectStmt *n = makeNode(PGVariableShowSelectStmt); @@ -30411,7 +30457,7 @@ YYLTYPE yylloc; ;} break; - case 1466: + case 1468: #line 10 "third_party/libpg_query/grammar/statements/variable_show.y" { PGVariableShowSelectStmt *n = makeNode(PGVariableShowSelectStmt); @@ -30422,7 +30468,7 @@ YYLTYPE yylloc; ;} break; - case 1467: + case 1469: #line 18 "third_party/libpg_query/grammar/statements/variable_show.y" { PGVariableShowStmt *n = makeNode(PGVariableShowStmt); @@ -30432,7 +30478,7 @@ YYLTYPE yylloc; ;} break; - case 1468: + case 1470: #line 25 "third_party/libpg_query/grammar/statements/variable_show.y" { PGVariableShowStmt *n = makeNode(PGVariableShowStmt); @@ -30442,7 +30488,7 @@ YYLTYPE yylloc; ;} break; - case 1469: + case 1471: #line 32 "third_party/libpg_query/grammar/statements/variable_show.y" { PGVariableShowStmt *n = makeNode(PGVariableShowStmt); @@ -30452,7 +30498,7 @@ YYLTYPE yylloc; ;} break; - case 1470: + case 1472: #line 39 "third_party/libpg_query/grammar/statements/variable_show.y" { PGVariableShowStmt *n = makeNode(PGVariableShowStmt); @@ -30462,7 +30508,7 @@ YYLTYPE yylloc; ;} break; - case 1471: + case 1473: #line 46 "third_party/libpg_query/grammar/statements/variable_show.y" { PGVariableShowStmt *n = makeNode(PGVariableShowStmt); @@ -30472,7 +30518,7 @@ YYLTYPE yylloc; ;} break; - case 1472: + case 1474: #line 53 "third_party/libpg_query/grammar/statements/variable_show.y" { PGVariableShowStmt *n = makeNode(PGVariableShowStmt); @@ -30482,27 +30528,27 @@ YYLTYPE yylloc; ;} break; - case 1479: + case 1481: #line 67 "third_party/libpg_query/grammar/statements/variable_show.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 1480: + case 1482: #line 69 "third_party/libpg_query/grammar/statements/variable_show.y" { (yyval.str) = psprintf("%s.%s", (yyvsp[(1) - (3)].str), (yyvsp[(3) - (3)].str)); ;} break; - case 1481: + case 1483: #line 72 "third_party/libpg_query/grammar/statements/variable_show.y" { (yyval.str) = psprintf("\"%s\"", (yyvsp[(1) - (1)].str)); ;} break; - case 1482: + case 1484: #line 74 "third_party/libpg_query/grammar/statements/variable_show.y" { (yyval.str) = psprintf("%s.\"%s\"", (yyvsp[(1) - (3)].str), (yyvsp[(3) - (3)].str)); ;} break; - case 1483: + case 1485: #line 7 "third_party/libpg_query/grammar/statements/call.y" { PGCallStmt *n = makeNode(PGCallStmt); @@ -30511,7 +30557,7 @@ YYLTYPE yylloc; ;} break; - case 1484: + case 1486: #line 10 "third_party/libpg_query/grammar/statements/view.y" { PGViewStmt *n = makeNode(PGViewStmt); @@ -30526,7 +30572,7 @@ YYLTYPE yylloc; ;} break; - case 1485: + case 1487: #line 23 "third_party/libpg_query/grammar/statements/view.y" { PGViewStmt *n = makeNode(PGViewStmt); @@ -30541,7 +30587,7 @@ YYLTYPE yylloc; ;} break; - case 1486: + case 1488: #line 36 "third_party/libpg_query/grammar/statements/view.y" { PGViewStmt *n = makeNode(PGViewStmt); @@ -30556,7 +30602,7 @@ YYLTYPE yylloc; ;} break; - case 1487: + case 1489: #line 49 "third_party/libpg_query/grammar/statements/view.y" { PGViewStmt *n = makeNode(PGViewStmt); @@ -30576,7 +30622,7 @@ YYLTYPE yylloc; ;} break; - case 1488: + case 1490: #line 67 "third_party/libpg_query/grammar/statements/view.y" { PGViewStmt *n = makeNode(PGViewStmt); @@ -30596,27 +30642,27 @@ YYLTYPE yylloc; ;} break; - case 1489: + case 1491: #line 87 "third_party/libpg_query/grammar/statements/view.y" { (yyval.viewcheckoption) = CASCADED_CHECK_OPTION; ;} break; - case 1490: + case 1492: #line 88 "third_party/libpg_query/grammar/statements/view.y" { (yyval.viewcheckoption) = CASCADED_CHECK_OPTION; ;} break; - case 1491: + case 1493: #line 89 "third_party/libpg_query/grammar/statements/view.y" { (yyval.viewcheckoption) = PG_LOCAL_CHECK_OPTION; ;} break; - case 1492: + case 1494: #line 90 "third_party/libpg_query/grammar/statements/view.y" { (yyval.viewcheckoption) = PG_NO_CHECK_OPTION; ;} break; - case 1493: + case 1495: #line 12 "third_party/libpg_query/grammar/statements/create_as.y" { PGCreateTableAsStmt *ctas = makeNode(PGCreateTableAsStmt); @@ -30632,7 +30678,7 @@ YYLTYPE yylloc; ;} break; - case 1494: + case 1496: #line 25 "third_party/libpg_query/grammar/statements/create_as.y" { PGCreateTableAsStmt *ctas = makeNode(PGCreateTableAsStmt); @@ -30648,7 +30694,7 @@ YYLTYPE yylloc; ;} break; - case 1495: + case 1497: #line 38 "third_party/libpg_query/grammar/statements/create_as.y" { PGCreateTableAsStmt *ctas = makeNode(PGCreateTableAsStmt); @@ -30664,22 +30710,22 @@ YYLTYPE yylloc; ;} break; - case 1496: + case 1498: #line 54 "third_party/libpg_query/grammar/statements/create_as.y" { (yyval.boolean) = true; ;} break; - case 1497: + case 1499: #line 55 "third_party/libpg_query/grammar/statements/create_as.y" { (yyval.boolean) = false; ;} break; - case 1498: + case 1500: #line 56 "third_party/libpg_query/grammar/statements/create_as.y" { (yyval.boolean) = true; ;} break; - case 1499: + case 1501: #line 62 "third_party/libpg_query/grammar/statements/create_as.y" { (yyval.into) = makeNode(PGIntoClause); @@ -30694,7 +30740,7 @@ YYLTYPE yylloc; /* Line 1267 of yacc.c. */ -#line 30698 "third_party/libpg_query/grammar/grammar_out.cpp" +#line 30744 "third_party/libpg_query/grammar/grammar_out.cpp" default: break; } YY_SYMBOL_PRINT ("-> $$ =", yyr1[yyn], &yyval, &yyloc); From f56c9c8c7c237cd651ca0fe516e6eb4c7f1d9833 Mon Sep 17 00:00:00 2001 From: Max Gabrielsson Date: Mon, 15 Apr 2024 17:15:35 +0200 Subject: [PATCH 455/603] more index initialization --- src/storage/data_table.cpp | 6 ++++++ src/storage/local_storage.cpp | 3 +++ 2 files changed, 9 insertions(+) diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 0a3e0ed034f3..60f16502b0bc 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -89,6 +89,8 @@ DataTable::DataTable(ClientContext &context, DataTable &parent, idx_t removed_co column_definitions.emplace_back(column_def.Copy()); } + info->InitializeIndexes(context); + // first check if there are any indexes that exist that point to the removed column info->indexes.Scan([&](Index &index) { for (auto &column_id : index.column_ids) { @@ -135,6 +137,8 @@ DataTable::DataTable(ClientContext &context, DataTable &parent, unique_ptrInitializeIndexes(context); + // Verify the new constraint against current persistent/local data VerifyNewConstraint(context, parent, constraint.get()); @@ -154,6 +158,8 @@ DataTable::DataTable(ClientContext &context, DataTable &parent, idx_t changed_id column_definitions.emplace_back(column_def.Copy()); } + info->InitializeIndexes(context); + // first check if there are any indexes that exist that point to the changed column info->indexes.Scan([&](Index &index) { for (auto &column_id : index.column_ids) { diff --git a/src/storage/local_storage.cpp b/src/storage/local_storage.cpp index 791847b102eb..ee249ce2d29f 100644 --- a/src/storage/local_storage.cpp +++ b/src/storage/local_storage.cpp @@ -354,6 +354,7 @@ bool LocalStorage::NextParallelScan(ClientContext &context, DataTable &table, Pa } void LocalStorage::InitializeAppend(LocalAppendState &state, DataTable &table) { + table.info->InitializeIndexes(context); state.storage = &table_manager.GetOrCreateStorage(table); state.storage->row_groups->InitializeAppend(TransactionData(transaction), state.append_state); } @@ -445,6 +446,8 @@ void LocalStorage::Flush(DataTable &table, LocalTableStorage &storage) { } idx_t append_count = storage.row_groups->GetTotalRows() - storage.deleted_rows; + table.info->InitializeIndexes(context); + TableAppendState append_state; table.AppendLock(append_state); transaction.PushAppend(table, append_state.row_start, append_count); From 780f5e47b793d9046e49ed94a9760ba78e16df19 Mon Sep 17 00:00:00 2001 From: Christina Sioula Date: Mon, 15 Apr 2024 17:17:11 +0200 Subject: [PATCH 456/603] rmv skipped member from SQLLogicTest --- scripts/sqllogictest/test.py | 6 +----- tools/pythonpkg/scripts/sqllogictest_python.py | 7 +------ 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/scripts/sqllogictest/test.py b/scripts/sqllogictest/test.py index 6a173b9d97b4..21d08d7a12ea 100644 --- a/scripts/sqllogictest/test.py +++ b/scripts/sqllogictest/test.py @@ -3,18 +3,14 @@ class SQLLogicTest: - __slots__ = ['path', 'statements', 'skipped'] + __slots__ = ['path', 'statements'] def __init__(self, path: str): self.path: str = path self.statements: List[BaseStatement] = [] - self.skipped = False def add_statement(self, statement: BaseStatement): self.statements.append(statement) def is_sqlite_test(self): return 'test/sqlite/select' in self.path or 'third_party/sqllogictest' in self.path - - def skip(self, val): - self.skipped = val diff --git a/tools/pythonpkg/scripts/sqllogictest_python.py b/tools/pythonpkg/scripts/sqllogictest_python.py index 37cbc2b68815..dc5c64a73869 100644 --- a/tools/pythonpkg/scripts/sqllogictest_python.py +++ b/tools/pythonpkg/scripts/sqllogictest_python.py @@ -96,9 +96,6 @@ def update_value(_: SQLLogicContext) -> Generator[Any, Any, Any]: # Yield once to represent one iteration, do not touch the keywords yield None - if self.test.skipped: - return ExecuteResult(ExecuteResult.Type.SKIPPED) - self.database = SQLLogicDatabase(':memory:', None) pool = self.database.connect() context = SQLLogicContext(pool, self, test.statements, keywords, update_value) @@ -176,10 +173,8 @@ def main(): try: test = sql_parser.parse(file_path) except SQLParserException as e: - if not ("test" in locals()): # test hasn't been initialized because of the raised exception - test = SQLLogicTest("") - test.skip(True) executor.skip_log.append(str(e.message)) + continue print(f'[{i}/{total_tests}] {file_path}') # This is necessary to clean up databases/connections From 0ebc97f53df465f1a3f679012b263cecee5fc429 Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 15 Apr 2024 17:33:33 +0200 Subject: [PATCH 457/603] tidy --- src/include/duckdb/common/shared_ptr.ipp | 4 ++-- src/include/duckdb/common/weak_ptr.ipp | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/include/duckdb/common/shared_ptr.ipp b/src/include/duckdb/common/shared_ptr.ipp index 6704724c21b1..b6e8ab8b40e9 100644 --- a/src/include/duckdb/common/shared_ptr.ipp +++ b/src/include/duckdb/common/shared_ptr.ipp @@ -144,19 +144,19 @@ public: reset() { // NOLINT: invalid case style internal.reset(); } + template #ifdef DUCKDB_CLANG_TIDY // This is necessary to tell clang-tidy that it reinitializes the variable after a move [[clang::reinitializes]] #endif - template void reset(U *ptr) { // NOLINT: invalid case style internal.reset(ptr); } + template #ifdef DUCKDB_CLANG_TIDY // This is necessary to tell clang-tidy that it reinitializes the variable after a move [[clang::reinitializes]] #endif - template void reset(U *ptr, DELETER deleter) { // NOLINT: invalid case style internal.reset(ptr, deleter); } diff --git a/src/include/duckdb/common/weak_ptr.ipp b/src/include/duckdb/common/weak_ptr.ipp index aef42e1f9e7e..2f1b9c1b506f 100644 --- a/src/include/duckdb/common/weak_ptr.ipp +++ b/src/include/duckdb/common/weak_ptr.ipp @@ -53,6 +53,10 @@ public: } // Modifiers +#ifdef DUCKDB_CLANG_TIDY + // This is necessary to tell clang-tidy that it reinitializes the variable after a move + [[clang::reinitializes]] +#endif void reset() { // NOLINT: invalid case style internal.reset(); } From dece8f16c8d17f3a021aae73bc3507e2a5107cda Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 15 Apr 2024 20:19:55 +0200 Subject: [PATCH 458/603] format --- src/include/duckdb/common/shared_ptr.ipp | 6 ++++-- src/include/duckdb/common/weak_ptr.ipp | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/include/duckdb/common/shared_ptr.ipp b/src/include/duckdb/common/shared_ptr.ipp index b6e8ab8b40e9..b7a76e395ed3 100644 --- a/src/include/duckdb/common/shared_ptr.ipp +++ b/src/include/duckdb/common/shared_ptr.ipp @@ -149,7 +149,8 @@ public: // This is necessary to tell clang-tidy that it reinitializes the variable after a move [[clang::reinitializes]] #endif - void reset(U *ptr) { // NOLINT: invalid case style + void + reset(U *ptr) { // NOLINT: invalid case style internal.reset(ptr); } template @@ -157,7 +158,8 @@ public: // This is necessary to tell clang-tidy that it reinitializes the variable after a move [[clang::reinitializes]] #endif - void reset(U *ptr, DELETER deleter) { // NOLINT: invalid case style + void + reset(U *ptr, DELETER deleter) { // NOLINT: invalid case style internal.reset(ptr, deleter); } diff --git a/src/include/duckdb/common/weak_ptr.ipp b/src/include/duckdb/common/weak_ptr.ipp index 2f1b9c1b506f..84e0d747d25f 100644 --- a/src/include/duckdb/common/weak_ptr.ipp +++ b/src/include/duckdb/common/weak_ptr.ipp @@ -57,7 +57,8 @@ public: // This is necessary to tell clang-tidy that it reinitializes the variable after a move [[clang::reinitializes]] #endif - void reset() { // NOLINT: invalid case style + void + reset() { // NOLINT: invalid case style internal.reset(); } From 6a76c352523801bd9b07846ac43b8c39a87829a6 Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 15 Apr 2024 21:57:52 +0200 Subject: [PATCH 459/603] tidy checks --- src/include/duckdb/common/shared_ptr.ipp | 16 ++++++++++------ src/include/duckdb/common/weak_ptr.ipp | 3 +++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/include/duckdb/common/shared_ptr.ipp b/src/include/duckdb/common/shared_ptr.ipp index b7a76e395ed3..e9e080aacbcc 100644 --- a/src/include/duckdb/common/shared_ptr.ipp +++ b/src/include/duckdb/common/shared_ptr.ipp @@ -104,6 +104,9 @@ public: // Assign from shared_ptr copy shared_ptr &operator=(const shared_ptr &other) noexcept { + if (this == &other) { + return *this; + } // Create a new shared_ptr using the copy constructor, then swap out the ownership to *this shared_ptr(other).swap(*this); return *this; @@ -235,16 +238,17 @@ public: private: // This overload is used when the class inherits from 'enable_shared_from_this' - template *>::value, + template *>::value, int>::type = 0> - void __enable_weak_this(const enable_shared_from_this *object, - _OrigPtr *ptr) noexcept { // NOLINT: invalid case style - typedef typename std::remove_cv::type NonConstU; + void __enable_weak_this(const enable_shared_from_this *object, // NOLINT: invalid case style + V *ptr) noexcept { + typedef typename std::remove_cv::type non_const_u_t; if (object && object->__weak_this_.expired()) { // __weak_this__ is the mutable variable returned by 'shared_from_this' // it is initialized here - object->__weak_this_ = shared_ptr(*this, const_cast(static_cast(ptr))); + auto non_const = const_cast(static_cast(ptr)); // NOLINT: const cast + object->__weak_this_ = shared_ptr(*this, non_const); } } diff --git a/src/include/duckdb/common/weak_ptr.ipp b/src/include/duckdb/common/weak_ptr.ipp index 84e0d747d25f..40688ded2ea5 100644 --- a/src/include/duckdb/common/weak_ptr.ipp +++ b/src/include/duckdb/common/weak_ptr.ipp @@ -42,6 +42,9 @@ public: // Assignment operators weak_ptr &operator=(const weak_ptr &other) { + if (this == &other) { + return *this; + } internal = other.internal; return *this; } From bd90b0a4b5c542a0fb0bcc9423b43917d0c850c9 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 15 Apr 2024 22:42:05 +0200 Subject: [PATCH 460/603] Bump azure, remove patch --- .github/config/out_of_tree_extensions.cmake | 3 +- .../patches/extensions/azure/open_file.patch | 469 ------------------ 2 files changed, 1 insertion(+), 471 deletions(-) delete mode 100644 .github/patches/extensions/azure/open_file.patch diff --git a/.github/config/out_of_tree_extensions.cmake b/.github/config/out_of_tree_extensions.cmake index d907647e44c1..7fa24bc79c69 100644 --- a/.github/config/out_of_tree_extensions.cmake +++ b/.github/config/out_of_tree_extensions.cmake @@ -38,8 +38,7 @@ if (NOT MINGW) duckdb_extension_load(azure LOAD_TESTS GIT_URL https://github.com/duckdb/duckdb_azure - GIT_TAG 4512a652479016d40d712f990cab9b9aab43d341 - APPLY_PATCHES + GIT_TAG 09623777a366572bfb8fa53e47acdf72133a360e ) endif() diff --git a/.github/patches/extensions/azure/open_file.patch b/.github/patches/extensions/azure/open_file.patch deleted file mode 100644 index 116a80eb7766..000000000000 --- a/.github/patches/extensions/azure/open_file.patch +++ /dev/null @@ -1,469 +0,0 @@ -diff --git a/src/azure_blob_filesystem.cpp b/src/azure_blob_filesystem.cpp -index 6f4d0dc..bc34eb9 100644 ---- a/src/azure_blob_filesystem.cpp -+++ b/src/azure_blob_filesystem.cpp -@@ -66,21 +66,20 @@ AzureBlobContextState::GetBlobContainerClient(const std::string &blobContainerNa - } - - //////// AzureBlobStorageFileHandle //////// --AzureBlobStorageFileHandle::AzureBlobStorageFileHandle(AzureBlobStorageFileSystem &fs, string path, uint8_t flags, -+AzureBlobStorageFileHandle::AzureBlobStorageFileHandle(AzureBlobStorageFileSystem &fs, string path, FileOpenFlags flags, - const AzureReadOptions &read_options, - Azure::Storage::Blobs::BlobClient blob_client) - : AzureFileHandle(fs, std::move(path), flags, read_options), blob_client(std::move(blob_client)) { - } - - //////// AzureBlobStorageFileSystem //////// --unique_ptr AzureBlobStorageFileSystem::CreateHandle(const string &path, uint8_t flags, -- FileLockType lock, FileCompressionType compression, -- FileOpener *opener) { -- if (opener == nullptr) { -+unique_ptr AzureBlobStorageFileSystem::CreateHandle(const string &path, FileOpenFlags flags, -+ optional_ptr opener) { -+ if (!opener) { - throw InternalException("Cannot do Azure storage CreateHandle without FileOpener"); - } - -- D_ASSERT(compression == FileCompressionType::UNCOMPRESSED); -+ D_ASSERT(flags.Compression() == FileCompressionType::UNCOMPRESSED); - - auto parsed_url = ParseUrl(path); - auto storage_context = GetOrCreateStorageContext(opener, path, parsed_url); -@@ -167,9 +166,9 @@ void AzureBlobStorageFileSystem::LoadRemoteFileInfo(AzureFileHandle &handle) { - hfh.last_modified = ToTimeT(res.Value.LastModified); - } - --bool AzureBlobStorageFileSystem::FileExists(const string &filename) { -+bool AzureBlobStorageFileSystem::FileExists(const string &filename, optional_ptr opener) { - try { -- auto handle = OpenFile(filename, FileFlags::FILE_FLAGS_READ); -+ auto handle = OpenFile(filename, FileFlags::FILE_FLAGS_READ, opener); - auto &sfh = handle->Cast(); - if (sfh.length == 0) { - return false; -@@ -202,7 +201,7 @@ void AzureBlobStorageFileSystem::ReadRange(AzureFileHandle &handle, idx_t file_o - } - } - --std::shared_ptr AzureBlobStorageFileSystem::CreateStorageContext(FileOpener *opener, -+std::shared_ptr AzureBlobStorageFileSystem::CreateStorageContext(optional_ptr opener, - const string &path, - const AzureParsedUrl &parsed_url) { - auto azure_read_options = ParseAzureReadOptions(opener); -diff --git a/src/azure_dfs_filesystem.cpp b/src/azure_dfs_filesystem.cpp -index 34435ae..5ccbed0 100644 ---- a/src/azure_dfs_filesystem.cpp -+++ b/src/azure_dfs_filesystem.cpp -@@ -83,21 +83,20 @@ AzureDfsContextState::GetDfsFileSystemClient(const std::string &file_system_name - } - - //////// AzureDfsContextState //////// --AzureDfsStorageFileHandle::AzureDfsStorageFileHandle(AzureDfsStorageFileSystem &fs, string path, uint8_t flags, -+AzureDfsStorageFileHandle::AzureDfsStorageFileHandle(AzureDfsStorageFileSystem &fs, string path, FileOpenFlags flags, - const AzureReadOptions &read_options, - Azure::Storage::Files::DataLake::DataLakeFileClient client) - : AzureFileHandle(fs, std::move(path), flags, read_options), file_client(std::move(client)) { - } - - //////// AzureDfsStorageFileSystem //////// --unique_ptr AzureDfsStorageFileSystem::CreateHandle(const string &path, uint8_t flags, -- FileLockType lock, FileCompressionType compression, -- FileOpener *opener) { -+unique_ptr AzureDfsStorageFileSystem::CreateHandle(const string &path, FileOpenFlags flags, -+ optional_ptr opener) { - if (opener == nullptr) { - throw InternalException("Cannot do Azure storage CreateHandle without FileOpener"); - } - -- D_ASSERT(compression == FileCompressionType::UNCOMPRESSED); -+ D_ASSERT(flags.Compression() == FileCompressionType::UNCOMPRESSED); - - auto parsed_url = ParseUrl(path); - auto storage_context = GetOrCreateStorageContext(opener, path, parsed_url); -@@ -186,7 +185,7 @@ void AzureDfsStorageFileSystem::ReadRange(AzureFileHandle &handle, idx_t file_of - } - } - --std::shared_ptr AzureDfsStorageFileSystem::CreateStorageContext(FileOpener *opener, -+std::shared_ptr AzureDfsStorageFileSystem::CreateStorageContext(optional_ptr opener, - const string &path, - const AzureParsedUrl &parsed_url) { - auto azure_read_options = ParseAzureReadOptions(opener); -diff --git a/src/azure_filesystem.cpp b/src/azure_filesystem.cpp -index 4c2caed..bbf5275 100644 ---- a/src/azure_filesystem.cpp -+++ b/src/azure_filesystem.cpp -@@ -18,7 +18,7 @@ void AzureContextState::QueryEnd() { - is_valid = false; - } - --AzureFileHandle::AzureFileHandle(AzureStorageFileSystem &fs, string path, uint8_t flags, -+AzureFileHandle::AzureFileHandle(AzureStorageFileSystem &fs, string path, FileOpenFlags flags, - const AzureReadOptions &read_options) - : FileHandle(fs, std::move(path)), flags(flags), - // File info -@@ -27,7 +27,7 @@ AzureFileHandle::AzureFileHandle(AzureStorageFileSystem &fs, string path, uint8_ - buffer_available(0), buffer_idx(0), file_offset(0), buffer_start(0), buffer_end(0), - // Options - read_options(read_options) { -- if (flags & FileFlags::FILE_FLAGS_READ) { -+ if (flags.OpenForReading()) { - read_buffer = duckdb::unique_ptr(new data_t[read_options.buffer_size]); - } - } -@@ -37,7 +37,7 @@ void AzureFileHandle::PostConstruct() { - } - - void AzureStorageFileSystem::LoadFileInfo(AzureFileHandle &handle) { -- if (handle.flags & FileFlags::FILE_FLAGS_READ) { -+ if (handle.flags.OpenForReading()) { - try { - LoadRemoteFileInfo(handle); - } catch (const Azure::Storage::StorageException &e) { -@@ -53,15 +53,15 @@ void AzureStorageFileSystem::LoadFileInfo(AzureFileHandle &handle) { - } - } - --unique_ptr AzureStorageFileSystem::OpenFile(const string &path, uint8_t flags, FileLockType lock, -- FileCompressionType compression, FileOpener *opener) { -- D_ASSERT(compression == FileCompressionType::UNCOMPRESSED); -+unique_ptr AzureStorageFileSystem::OpenFile(const string &path,FileOpenFlags flags, -+ optional_ptr opener) { -+ D_ASSERT(flags.Compression() == FileCompressionType::UNCOMPRESSED); - -- if (flags & FileFlags::FILE_FLAGS_WRITE) { -+ if (flags.OpenForWriting()) { - throw NotImplementedException("Writing to Azure containers is currently not supported"); - } - -- auto handle = CreateHandle(path, flags, lock, compression, opener); -+ auto handle = CreateHandle(path, flags, opener); - return std::move(handle); - } - -@@ -92,7 +92,7 @@ void AzureStorageFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_b - idx_t buffer_offset = 0; - - // Don't buffer when DirectIO is set. -- if (hfh.flags & FileFlags::FILE_FLAGS_DIRECT_IO && to_read > 0) { -+ if (hfh.flags.DirectIO() && to_read > 0) { - ReadRange(hfh, location, (char *)buffer, to_read); - hfh.buffer_available = 0; - hfh.buffer_idx = 0; -@@ -153,7 +153,7 @@ int64_t AzureStorageFileSystem::Read(FileHandle &handle, void *buffer, int64_t n - return nr_bytes; - } - --std::shared_ptr AzureStorageFileSystem::GetOrCreateStorageContext(FileOpener *opener, -+std::shared_ptr AzureStorageFileSystem::GetOrCreateStorageContext(optional_ptr opener, - const string &path, - const AzureParsedUrl &parsed_url) { - Value value; -@@ -164,7 +164,7 @@ std::shared_ptr AzureStorageFileSystem::GetOrCreateStorageCon - - std::shared_ptr result; - if (azure_context_caching) { -- auto *client_context = FileOpener::TryGetClientContext(opener); -+ auto client_context = FileOpener::TryGetClientContext(opener); - - auto context_key = GetContextPrefix() + parsed_url.storage_account_name; - -@@ -192,7 +192,7 @@ std::shared_ptr AzureStorageFileSystem::GetOrCreateStorageCon - return result; - } - --AzureReadOptions AzureStorageFileSystem::ParseAzureReadOptions(FileOpener *opener) { -+AzureReadOptions AzureStorageFileSystem::ParseAzureReadOptions(optional_ptr opener) { - AzureReadOptions options; - - Value concurrency_val; -diff --git a/src/azure_storage_account_client.cpp b/src/azure_storage_account_client.cpp -index e54ca93..5a22e60 100644 ---- a/src/azure_storage_account_client.cpp -+++ b/src/azure_storage_account_client.cpp -@@ -35,7 +35,7 @@ namespace duckdb { - const static std::string DEFAULT_BLOB_ENDPOINT = "blob.core.windows.net"; - const static std::string DEFAULT_DFS_ENDPOINT = "dfs.core.windows.net"; - --static std::string TryGetCurrentSetting(FileOpener *opener, const std::string &name) { -+static std::string TryGetCurrentSetting(optional_ptr opener, const std::string &name) { - Value val; - if (FileOpener::TryGetCurrentSetting(opener, name, val)) { - return val.ToString(); -@@ -110,7 +110,7 @@ ToTokenCredentialOptions(const Azure::Core::Http::Policies::TransportOptions &tr - return options; - } - --static std::shared_ptr GetHttpState(FileOpener *opener) { -+static std::shared_ptr GetHttpState(optional_ptr opener) { - Value value; - bool enable_http_stats = false; - if (FileOpener::TryGetCurrentSetting(opener, "azure_http_stats", value)) { -@@ -270,7 +270,7 @@ static Azure::Core::Http::Policies::TransportOptions GetTransportOptions(const s - return transport_options; - } - --static Azure::Core::Http::Policies::TransportOptions GetTransportOptions(FileOpener *opener, -+static Azure::Core::Http::Policies::TransportOptions GetTransportOptions(optional_ptr opener, - const KeyValueSecret &secret) { - auto transport_option_type = TryGetCurrentSetting(opener, "azure_transport_option_type"); - -@@ -302,7 +302,7 @@ static Azure::Core::Http::Policies::TransportOptions GetTransportOptions(FileOpe - } - - static Azure::Storage::Blobs::BlobServiceClient --GetBlobStorageAccountClientFromConfigProvider(FileOpener *opener, const KeyValueSecret &secret, -+GetBlobStorageAccountClientFromConfigProvider(optional_ptr opener, const KeyValueSecret &secret, - const AzureParsedUrl &azure_parsed_url) { - auto transport_options = GetTransportOptions(opener, secret); - -@@ -328,7 +328,7 @@ GetBlobStorageAccountClientFromConfigProvider(FileOpener *opener, const KeyValue - } - - static Azure::Storage::Files::DataLake::DataLakeServiceClient --GetDfsStorageAccountClientFromConfigProvider(FileOpener *opener, const KeyValueSecret &secret, -+GetDfsStorageAccountClientFromConfigProvider(optional_ptr opener, const KeyValueSecret &secret, - const AzureParsedUrl &azure_parsed_url) { - auto transport_options = GetTransportOptions(opener, secret); - -@@ -355,7 +355,7 @@ GetDfsStorageAccountClientFromConfigProvider(FileOpener *opener, const KeyValueS - } - - static Azure::Storage::Blobs::BlobServiceClient --GetBlobStorageAccountClientFromCredentialChainProvider(FileOpener *opener, const KeyValueSecret &secret, -+GetBlobStorageAccountClientFromCredentialChainProvider(optional_ptr opener, const KeyValueSecret &secret, - const AzureParsedUrl &azure_parsed_url) { - auto transport_options = GetTransportOptions(opener, secret); - // Create credential chain -@@ -369,7 +369,7 @@ GetBlobStorageAccountClientFromCredentialChainProvider(FileOpener *opener, const - } - - static Azure::Storage::Files::DataLake::DataLakeServiceClient --GetDfsStorageAccountClientFromCredentialChainProvider(FileOpener *opener, const KeyValueSecret &secret, -+GetDfsStorageAccountClientFromCredentialChainProvider(optional_ptr opener, const KeyValueSecret &secret, - const AzureParsedUrl &azure_parsed_url) { - auto transport_options = GetTransportOptions(opener, secret); - // Create credential chain -@@ -383,7 +383,7 @@ GetDfsStorageAccountClientFromCredentialChainProvider(FileOpener *opener, const - } - - static Azure::Storage::Blobs::BlobServiceClient --GetBlobStorageAccountClientFromServicePrincipalProvider(FileOpener *opener, const KeyValueSecret &secret, -+GetBlobStorageAccountClientFromServicePrincipalProvider(optional_ptr opener, const KeyValueSecret &secret, - const AzureParsedUrl &azure_parsed_url) { - auto transport_options = GetTransportOptions(opener, secret); - auto token_credential = CreateClientCredential(secret, transport_options); -@@ -396,7 +396,7 @@ GetBlobStorageAccountClientFromServicePrincipalProvider(FileOpener *opener, cons - } - - static Azure::Storage::Files::DataLake::DataLakeServiceClient --GetDfsStorageAccountClientFromServicePrincipalProvider(FileOpener *opener, const KeyValueSecret &secret, -+GetDfsStorageAccountClientFromServicePrincipalProvider(optional_ptr opener, const KeyValueSecret &secret, - const AzureParsedUrl &azure_parsed_url) { - auto transport_options = GetTransportOptions(opener, secret); - auto token_credential = CreateClientCredential(secret, transport_options); -@@ -409,7 +409,7 @@ GetDfsStorageAccountClientFromServicePrincipalProvider(FileOpener *opener, const - } - - static Azure::Storage::Blobs::BlobServiceClient --GetBlobStorageAccountClient(FileOpener *opener, const KeyValueSecret &secret, const AzureParsedUrl &azure_parsed_url) { -+GetBlobStorageAccountClient(optional_ptr opener, const KeyValueSecret &secret, const AzureParsedUrl &azure_parsed_url) { - auto &provider = secret.GetProvider(); - // default provider - if (provider == "config") { -@@ -424,7 +424,7 @@ GetBlobStorageAccountClient(FileOpener *opener, const KeyValueSecret &secret, co - } - - static Azure::Storage::Files::DataLake::DataLakeServiceClient --GetDfsStorageAccountClient(FileOpener *opener, const KeyValueSecret &secret, const AzureParsedUrl &azure_parsed_url) { -+GetDfsStorageAccountClient(optional_ptr opener, const KeyValueSecret &secret, const AzureParsedUrl &azure_parsed_url) { - auto &provider = secret.GetProvider(); - // default provider - if (provider == "config") { -@@ -438,7 +438,7 @@ GetDfsStorageAccountClient(FileOpener *opener, const KeyValueSecret &secret, con - throw InvalidInputException("Unsupported provider type %s for azure", provider); - } - --static Azure::Core::Http::Policies::TransportOptions GetTransportOptions(FileOpener *opener) { -+static Azure::Core::Http::Policies::TransportOptions GetTransportOptions(optional_ptr opener) { - auto azure_transport_option_type = TryGetCurrentSetting(opener, "azure_transport_option_type"); - - // Load proxy options -@@ -449,7 +449,7 @@ static Azure::Core::Http::Policies::TransportOptions GetTransportOptions(FileOpe - return GetTransportOptions(azure_transport_option_type, http_proxy, http_proxy_user_name, http_proxy_password); - } - --static Azure::Storage::Blobs::BlobServiceClient GetBlobStorageAccountClient(FileOpener *opener, -+static Azure::Storage::Blobs::BlobServiceClient GetBlobStorageAccountClient(optional_ptr opener, - const std::string &provided_storage_account, - const std::string &provided_endpoint) { - auto transport_options = GetTransportOptions(opener); -@@ -494,7 +494,7 @@ static Azure::Storage::Blobs::BlobServiceClient GetBlobStorageAccountClient(File - return Azure::Storage::Blobs::BlobServiceClient {account_url, blob_options}; - } - --const SecretMatch LookupSecret(FileOpener *opener, const std::string &path) { -+const SecretMatch LookupSecret(optional_ptr opener, const std::string &path) { - auto context = opener->TryGetClientContext(); - - if (context) { -@@ -505,7 +505,7 @@ const SecretMatch LookupSecret(FileOpener *opener, const std::string &path) { - return {}; - } - --Azure::Storage::Blobs::BlobServiceClient ConnectToBlobStorageAccount(FileOpener *opener, const std::string &path, -+Azure::Storage::Blobs::BlobServiceClient ConnectToBlobStorageAccount(optional_ptr opener, const std::string &path, - const AzureParsedUrl &azure_parsed_url) { - - auto secret_match = LookupSecret(opener, path); -@@ -519,7 +519,7 @@ Azure::Storage::Blobs::BlobServiceClient ConnectToBlobStorageAccount(FileOpener - } - - Azure::Storage::Files::DataLake::DataLakeServiceClient --ConnectToDfsStorageAccount(FileOpener *opener, const std::string &path, const AzureParsedUrl &azure_parsed_url) { -+ConnectToDfsStorageAccount(optional_ptr opener, const std::string &path, const AzureParsedUrl &azure_parsed_url) { - auto secret_match = LookupSecret(opener, path); - if (secret_match.HasMatch()) { - const auto &base_secret = secret_match.GetSecret(); -diff --git a/src/include/azure_blob_filesystem.hpp b/src/include/azure_blob_filesystem.hpp -index cb76d68..4d10ebe 100644 ---- a/src/include/azure_blob_filesystem.hpp -+++ b/src/include/azure_blob_filesystem.hpp -@@ -23,7 +23,7 @@ class AzureBlobStorageFileSystem; - - class AzureBlobStorageFileHandle : public AzureFileHandle { - public: -- AzureBlobStorageFileHandle(AzureBlobStorageFileSystem &fs, string path, uint8_t flags, -+ AzureBlobStorageFileHandle(AzureBlobStorageFileSystem &fs, string path, FileOpenFlags flags, - const AzureReadOptions &read_options, Azure::Storage::Blobs::BlobClient blob_client); - ~AzureBlobStorageFileHandle() override = default; - -@@ -36,7 +36,7 @@ public: - vector Glob(const string &path, FileOpener *opener = nullptr) override; - - // FS methods -- bool FileExists(const string &filename) override; -+ bool FileExists(const string &filename, optional_ptr opener = nullptr) override; - bool CanHandleFile(const string &fpath) override; - string GetName() const override { - return "AzureBlobStorageFileSystem"; -@@ -57,10 +57,10 @@ protected: - const string &GetContextPrefix() const override { - return PATH_PREFIX; - } -- std::shared_ptr CreateStorageContext(FileOpener *opener, const string &path, -+ std::shared_ptr CreateStorageContext(optional_ptr opener, const string &path, - const AzureParsedUrl &parsed_url) override; -- duckdb::unique_ptr CreateHandle(const string &path, uint8_t flags, FileLockType lock, -- FileCompressionType compression, FileOpener *opener) override; -+ duckdb::unique_ptr CreateHandle(const string &path, FileOpenFlags flags, -+ optional_ptr opener) override; - - void ReadRange(AzureFileHandle &handle, idx_t file_offset, char *buffer_out, idx_t buffer_out_len) override; - }; -diff --git a/src/include/azure_dfs_filesystem.hpp b/src/include/azure_dfs_filesystem.hpp -index c4bf8fe..cdcdb23 100644 ---- a/src/include/azure_dfs_filesystem.hpp -+++ b/src/include/azure_dfs_filesystem.hpp -@@ -25,7 +25,7 @@ class AzureDfsStorageFileSystem; - - class AzureDfsStorageFileHandle : public AzureFileHandle { - public: -- AzureDfsStorageFileHandle(AzureDfsStorageFileSystem &fs, string path, uint8_t flags, -+ AzureDfsStorageFileHandle(AzureDfsStorageFileSystem &fs, string path, FileOpenFlags flags, - const AzureReadOptions &read_options, - Azure::Storage::Files::DataLake::DataLakeFileClient client); - ~AzureDfsStorageFileHandle() override = default; -@@ -55,10 +55,10 @@ protected: - const string &GetContextPrefix() const override { - return PATH_PREFIX; - } -- std::shared_ptr CreateStorageContext(FileOpener *opener, const string &path, -+ std::shared_ptr CreateStorageContext(optional_ptr opener, const string &path, - const AzureParsedUrl &parsed_url) override; -- duckdb::unique_ptr CreateHandle(const string &path, uint8_t flags, FileLockType lock, -- FileCompressionType compression, FileOpener *opener) override; -+ duckdb::unique_ptr CreateHandle(const string &path, FileOpenFlags flags, -+ optional_ptr opener) override; - - void ReadRange(AzureFileHandle &handle, idx_t file_offset, char *buffer_out, idx_t buffer_out_len) override; - }; -diff --git a/src/include/azure_filesystem.hpp b/src/include/azure_filesystem.hpp -index a41d7a2..338c744 100644 ---- a/src/include/azure_filesystem.hpp -+++ b/src/include/azure_filesystem.hpp -@@ -52,10 +52,10 @@ public: - } - - protected: -- AzureFileHandle(AzureStorageFileSystem &fs, string path, uint8_t flags, const AzureReadOptions &read_options); -+ AzureFileHandle(AzureStorageFileSystem &fs, string path, FileOpenFlags flags, const AzureReadOptions &read_options); - - public: -- const uint8_t flags; -+ const FileOpenFlags flags; - - // File info - idx_t length; -@@ -76,9 +76,8 @@ public: - class AzureStorageFileSystem : public FileSystem { - public: - // FS methods -- duckdb::unique_ptr OpenFile(const string &path, uint8_t flags, FileLockType lock = DEFAULT_LOCK, -- FileCompressionType compression = DEFAULT_COMPRESSION, -- FileOpener *opener = nullptr) override; -+ duckdb::unique_ptr OpenFile(const string &path, FileOpenFlags flags, -+ optional_ptr opener = nullptr) override; - - void Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override; - int64_t Read(FileHandle &handle, void *buffer, int64_t nr_bytes) override; -@@ -88,7 +87,7 @@ public: - bool OnDiskFile(FileHandle &handle) override { - return false; - } -- bool IsPipe(const string &filename) override { -+ bool IsPipe(const string &filename, optional_ptr opener = nullptr) override { - return false; - } - int64_t GetFileSize(FileHandle &handle) override; -@@ -99,18 +98,18 @@ public: - void LoadFileInfo(AzureFileHandle &handle); - - protected: -- virtual duckdb::unique_ptr CreateHandle(const string &path, uint8_t flags, FileLockType lock, -- FileCompressionType compression, FileOpener *opener) = 0; -+ virtual duckdb::unique_ptr CreateHandle(const string &path, FileOpenFlags flags, -+ optional_ptr opener) = 0; - virtual void ReadRange(AzureFileHandle &handle, idx_t file_offset, char *buffer_out, idx_t buffer_out_len) = 0; - - virtual const string &GetContextPrefix() const = 0; -- std::shared_ptr GetOrCreateStorageContext(FileOpener *opener, const string &path, -+ std::shared_ptr GetOrCreateStorageContext(optional_ptr opener, const string &path, - const AzureParsedUrl &parsed_url); -- virtual std::shared_ptr CreateStorageContext(FileOpener *opener, const string &path, -+ virtual std::shared_ptr CreateStorageContext(optional_ptr opener, const string &path, - const AzureParsedUrl &parsed_url) = 0; - - virtual void LoadRemoteFileInfo(AzureFileHandle &handle) = 0; -- static AzureReadOptions ParseAzureReadOptions(FileOpener *opener); -+ static AzureReadOptions ParseAzureReadOptions(optional_ptr opener); - static time_t ToTimeT(const Azure::DateTime &dt); - }; - -diff --git a/src/include/azure_storage_account_client.hpp b/src/include/azure_storage_account_client.hpp -index 2e22ee0..600fa10 100644 ---- a/src/include/azure_storage_account_client.hpp -+++ b/src/include/azure_storage_account_client.hpp -@@ -8,10 +8,10 @@ - - namespace duckdb { - --Azure::Storage::Blobs::BlobServiceClient ConnectToBlobStorageAccount(FileOpener *opener, const std::string &path, -+Azure::Storage::Blobs::BlobServiceClient ConnectToBlobStorageAccount(optional_ptr opener, const std::string &path, - const AzureParsedUrl &azure_parsed_url); - - Azure::Storage::Files::DataLake::DataLakeServiceClient --ConnectToDfsStorageAccount(FileOpener *opener, const std::string &path, const AzureParsedUrl &azure_parsed_url); -+ConnectToDfsStorageAccount(optional_ptr opener, const std::string &path, const AzureParsedUrl &azure_parsed_url); - - } // namespace duckdb From caec6c545877fb4df176d606b70e05b5bb26b614 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 15 Apr 2024 22:42:23 +0200 Subject: [PATCH 461/603] Bump iceberg, remove patch --- .github/config/out_of_tree_extensions.cmake | 3 +- .../extensions/iceberg/open_file.patch | 66 ------------------- 2 files changed, 1 insertion(+), 68 deletions(-) delete mode 100644 .github/patches/extensions/iceberg/open_file.patch diff --git a/.github/config/out_of_tree_extensions.cmake b/.github/config/out_of_tree_extensions.cmake index 7fa24bc79c69..a80e71b7de7c 100644 --- a/.github/config/out_of_tree_extensions.cmake +++ b/.github/config/out_of_tree_extensions.cmake @@ -54,8 +54,7 @@ if (NOT MINGW) duckdb_extension_load(iceberg ${LOAD_ICEBERG_TESTS} GIT_URL https://github.com/duckdb/duckdb_iceberg - GIT_TAG 7aa3d8e4cb7b513d35fdacfa28dc328771bc4047 - APPLY_PATCHES + GIT_TAG d89423c2ff90a0b98a093a133c8dfe2a55b9e092 ) endif() diff --git a/.github/patches/extensions/iceberg/open_file.patch b/.github/patches/extensions/iceberg/open_file.patch deleted file mode 100644 index f564792e3957..000000000000 --- a/.github/patches/extensions/iceberg/open_file.patch +++ /dev/null @@ -1,66 +0,0 @@ -diff --git a/src/common/iceberg.cpp b/src/common/iceberg.cpp -index 3c15105..face972 100644 ---- a/src/common/iceberg.cpp -+++ b/src/common/iceberg.cpp -@@ -127,7 +127,7 @@ unique_ptr IcebergSnapshot::GetParseInfo(const string &path, - parse_info->doc = doc; - parse_info->document = std::move(metadata_json); - -- return std::move(parse_info); -+ return parse_info; - } - - IcebergSnapshot IcebergSnapshot::GetLatestSnapshot(const string &path, FileSystem &fs) { -diff --git a/src/common/utils.cpp b/src/common/utils.cpp -index c0272bf..d2b01d9 100644 ---- a/src/common/utils.cpp -+++ b/src/common/utils.cpp -@@ -4,8 +4,7 @@ - namespace duckdb { - - string IcebergUtils::FileToString(const string &path, FileSystem &fs) { -- auto handle = -- fs.OpenFile(path, FileFlags::FILE_FLAGS_READ, FileSystem::DEFAULT_LOCK, FileSystem::DEFAULT_COMPRESSION); -+ auto handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ); - auto file_size = handle->GetFileSize(); - string ret_val(file_size, ' '); - handle->Read((char *)ret_val.c_str(), file_size); -diff --git a/src/iceberg_extension.cpp b/src/iceberg_extension.cpp -index 0e59fdc..895b79d 100644 ---- a/src/iceberg_extension.cpp -+++ b/src/iceberg_extension.cpp -@@ -16,12 +16,6 @@ - namespace duckdb { - - static void LoadInternal(DatabaseInstance &instance) { -- Connection con(instance); -- con.BeginTransaction(); -- auto &context = *con.context; -- -- auto &catalog = Catalog::GetSystemCatalog(*con.context); -- - // Iceberg Table Functions - for (auto &fun : IcebergFunctions::GetTableFunctions()) { - ExtensionUtil::RegisterFunction(instance, fun); -@@ -31,8 +25,6 @@ static void LoadInternal(DatabaseInstance &instance) { - for (auto &fun : IcebergFunctions::GetScalarFunctions()) { - ExtensionUtil::RegisterFunction(instance, fun); - } -- -- con.Commit(); - } - - void IcebergExtension::Load(DuckDB &db) { -diff --git a/src/iceberg_functions/iceberg_snapshots.cpp b/src/iceberg_functions/iceberg_snapshots.cpp -index 6d6ef57..4b67a6d 100644 ---- a/src/iceberg_functions/iceberg_snapshots.cpp -+++ b/src/iceberg_functions/iceberg_snapshots.cpp -@@ -93,7 +93,7 @@ TableFunctionSet IcebergFunctions::GetIcebergSnapshotsFunction() { - TableFunction table_function({LogicalType::VARCHAR}, IcebergSnapshotsFunction, IcebergSnapshotsBind, - IcebergSnapshotGlobalTableFunctionState::Init); - function_set.AddFunction(table_function); -- return std::move(function_set); -+ return function_set; - } - - } // namespace duckdb From e93841c83a8a000269d3430083c5092a970673ab Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 15 Apr 2024 22:42:38 +0200 Subject: [PATCH 462/603] Bump vss, remove patch --- .github/config/out_of_tree_extensions.cmake | 5 ++--- .github/patches/extensions/vss/vss_depenency.patch | 13 ------------- 2 files changed, 2 insertions(+), 16 deletions(-) delete mode 100644 .github/patches/extensions/vss/vss_depenency.patch diff --git a/.github/config/out_of_tree_extensions.cmake b/.github/config/out_of_tree_extensions.cmake index a80e71b7de7c..a95bd411a18c 100644 --- a/.github/config/out_of_tree_extensions.cmake +++ b/.github/config/out_of_tree_extensions.cmake @@ -106,7 +106,6 @@ endif() duckdb_extension_load(vss LOAD_TESTS GIT_URL https://github.com/duckdb/duckdb_vss - GIT_TAG a85e973650a083e4b279126a0aec07924d84e765 + GIT_TAG 9038b50cefb8bfd6b8ab8a254d3c728f3f172d15 TEST_DIR test/sql - APPLY_PATCHES - ) \ No newline at end of file + ) diff --git a/.github/patches/extensions/vss/vss_depenency.patch b/.github/patches/extensions/vss/vss_depenency.patch deleted file mode 100644 index 98cd7d1d3932..000000000000 --- a/.github/patches/extensions/vss/vss_depenency.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/src/hnsw/hnsw_index_scan.cpp b/src/hnsw/hnsw_index_scan.cpp -index d53d7ee..954b439 100644 ---- a/src/hnsw/hnsw_index_scan.cpp -+++ b/src/hnsw/hnsw_index_scan.cpp -@@ -104,7 +104,7 @@ static unique_ptr HNSWIndexScanStatistics(ClientContext &context - //------------------------------------------------------------------------- - // Dependency - //------------------------------------------------------------------------- --void HNSWIndexScanDependency(DependencyList &entries, const FunctionData *bind_data_p) { -+void HNSWIndexScanDependency(LogicalDependencyList &entries, const FunctionData *bind_data_p) { - auto &bind_data = bind_data_p->Cast(); - entries.AddDependency(bind_data.table); - From b553e8deea66043a66764c6c010ae6f75f2a840f Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 15 Apr 2024 22:46:42 +0200 Subject: [PATCH 463/603] Update extensions.csv (to be removed) --- .github/config/extensions.csv | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/config/extensions.csv b/.github/config/extensions.csv index 28a87161e258..50efd8809ff4 100644 --- a/.github/config/extensions.csv +++ b/.github/config/extensions.csv @@ -7,12 +7,12 @@ json,,, parquet,,, tpcds,,, tpch,,, -sqlite_scanner,https://github.com/duckdb/sqlite_scanner,c1343464ef4397665b858db9c193d33fac591b1c, -postgres_scanner,https://github.com/duckdb/postgres_scanner,375710fd22a35107b2c28e744f787e1a93a99998, +sqlite_scanner,https://github.com/duckdb/sqlite_scanner,091197efb34579c7195afa43dfb5925023c915c0, +postgres_scanner,https://github.com/duckdb/postgres_scanner,96206f41d5ca7015920a66b54e936c986fe0b0f8, substrait,https://github.com/duckdb/substrait,1116fb580edd3e26e675436dbdbdf4a0aa5e456e,no-windows arrow,https://github.com/duckdb/arrow,9e10240da11f61ea7fbfe3fc9988ffe672ccd40f,no-windows aws,https://github.com/duckdb/duckdb_aws,f7b8729f1cce5ada5d4add70e1486de50763fb97, -azure,https://github.com/duckdb/duckdb_azure,86f39d76157de970d16d6d6537bc90c0ee1c7d35, +azure,https://github.com/duckdb/duckdb_azure,09623777a366572bfb8fa53e47acdf72133a360e, spatial,https://github.com/duckdb/duckdb_spatial,8ac803e986ccda34f32dee82a7faae95b72b3492, -iceberg,https://github.com/duckdb/duckdb_iceberg,7aa3d8e4cb7b513d35fdacfa28dc328771bc4047, -vss,https://github.com/duckdb/duckdb_vss,a85e973650a083e4b279126a0aec07924d84e765, \ No newline at end of file +iceberg,https://github.com/duckdb/duckdb_iceberg,d89423c2ff90a0b98a093a133c8dfe2a55b9e092, +vss,https://github.com/duckdb/duckdb_vss,9038b50cefb8bfd6b8ab8a254d3c728f3f172d15, From 439005fb04a69eafc01d689fee88686e0eed2415 Mon Sep 17 00:00:00 2001 From: Zeb Burke-Conte Date: Mon, 15 Apr 2024 14:38:25 -0700 Subject: [PATCH 464/603] Include falloc to fix build on some Linux systems --- src/common/local_file_system.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/common/local_file_system.cpp b/src/common/local_file_system.cpp index b05189b4ea0f..9345ed4d84b0 100644 --- a/src/common/local_file_system.cpp +++ b/src/common/local_file_system.cpp @@ -40,6 +40,7 @@ extern "C" WINBASEAPI BOOL WINAPI GetPhysicallyInstalledSystemMemory(PULONGLONG) #endif #if defined(__linux__) +#include #include // See e.g.: // https://opensource.apple.com/source/CarbonHeaders/CarbonHeaders-18.1/TargetConditionals.h.auto.html From 1d6e334b76b5f63d94f816a880ff97c6a3183bcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Tue, 16 Apr 2024 09:04:08 +0200 Subject: [PATCH 465/603] ci fixes --- src/include/duckdb/common/vector.hpp | 2 +- src/parallel/task_scheduler.cpp | 2 +- src/storage/compression/bitpacking.cpp | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/include/duckdb/common/vector.hpp b/src/include/duckdb/common/vector.hpp index 95f76fe4d94e..c767b76bab3b 100644 --- a/src/include/duckdb/common/vector.hpp +++ b/src/include/duckdb/common/vector.hpp @@ -101,7 +101,7 @@ class vector : public std::vector> { // NOL return get(original::size() - 1); } - void erase_at(idx_t idx) { + void erase_at(idx_t idx) { // NOLINT: not using camelcase on purpose here if (MemorySafety::ENABLED && idx > original::size()) { throw InternalException("Can't remove offset %d from vector of size %d", idx, original::size()); } diff --git a/src/parallel/task_scheduler.cpp b/src/parallel/task_scheduler.cpp index d327f983213d..db2dd6087d5f 100644 --- a/src/parallel/task_scheduler.cpp +++ b/src/parallel/task_scheduler.cpp @@ -16,7 +16,7 @@ #include #endif -#include // ssize_t +#include // ssize_t namespace duckdb { diff --git a/src/storage/compression/bitpacking.cpp b/src/storage/compression/bitpacking.cpp index 03472f4cd5d3..53605cd540a8 100644 --- a/src/storage/compression/bitpacking.cpp +++ b/src/storage/compression/bitpacking.cpp @@ -808,6 +808,7 @@ void BitpackingScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t T multiplier; auto success = TryCast::Operation(scan_state.current_group_offset + i, multiplier); D_ASSERT(success); + (void)success; target_ptr[i] = (multiplier * scan_state.current_constant) + scan_state.current_frame_of_reference; } From a376b015ead517d89e58ef9afabeef9c26bdfac0 Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 16 Apr 2024 09:57:59 +0200 Subject: [PATCH 466/603] add more 'reinitializes' attributes to silence erroneous 'use after move' clang tidy errors --- src/include/duckdb/common/shared_ptr.ipp | 12 +++++++++--- src/include/duckdb/common/weak_ptr.ipp | 6 ++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/include/duckdb/common/shared_ptr.ipp b/src/include/duckdb/common/shared_ptr.ipp index e9e080aacbcc..460ae98f9f3b 100644 --- a/src/include/duckdb/common/shared_ptr.ipp +++ b/src/include/duckdb/common/shared_ptr.ipp @@ -73,8 +73,14 @@ public: } // Move constructor, share ownership with ref template ::value, int>::type = 0> +#ifdef DUCKDB_CLANG_TIDY + [[clang::reinitializes]] +#endif shared_ptr(shared_ptr &&ref) noexcept : internal(std::move(ref.internal)) { // NOLINT: not marked as explicit } +#ifdef DUCKDB_CLANG_TIDY + [[clang::reinitializes]] +#endif shared_ptr(shared_ptr &&other) : internal(std::move(other.internal)) { // NOLINT: not marked as explicit } @@ -95,6 +101,9 @@ public: typename std::enable_if::value && std::is_convertible::pointer, T *>::value, int>::type = 0> +#ifdef DUCKDB_CLANG_TIDY + [[clang::reinitializes]] +#endif shared_ptr(unique_ptr &&other) : internal(std::move(other)) { // NOLINT: not marked as explicit __enable_weak_this(internal.get(), internal.get()); } @@ -140,7 +149,6 @@ public: } #ifdef DUCKDB_CLANG_TIDY - // This is necessary to tell clang-tidy that it reinitializes the variable after a move [[clang::reinitializes]] #endif void @@ -149,7 +157,6 @@ public: } template #ifdef DUCKDB_CLANG_TIDY - // This is necessary to tell clang-tidy that it reinitializes the variable after a move [[clang::reinitializes]] #endif void @@ -158,7 +165,6 @@ public: } template #ifdef DUCKDB_CLANG_TIDY - // This is necessary to tell clang-tidy that it reinitializes the variable after a move [[clang::reinitializes]] #endif void diff --git a/src/include/duckdb/common/weak_ptr.ipp b/src/include/duckdb/common/weak_ptr.ipp index 40688ded2ea5..a714eb0e67b0 100644 --- a/src/include/duckdb/common/weak_ptr.ipp +++ b/src/include/duckdb/common/weak_ptr.ipp @@ -30,9 +30,15 @@ public: weak_ptr(weak_ptr const &ptr, typename std::enable_if::value, int>::type = 0) noexcept : internal(ptr.internal) { } +#ifdef DUCKDB_CLANG_TIDY + [[clang::reinitializes]] +#endif weak_ptr(weak_ptr &&ptr) noexcept : internal(std::move(ptr.internal)) { } template +#ifdef DUCKDB_CLANG_TIDY + [[clang::reinitializes]] +#endif weak_ptr(weak_ptr &&ptr, typename std::enable_if::value, int>::type = 0) noexcept : internal(std::move(ptr.internal)) { } From a04e5f34da9477097b5cd1595f7b6c54acfde4d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Tue, 16 Apr 2024 10:06:15 +0200 Subject: [PATCH 467/603] moar fixes for ci --- src/include/duckdb/common/types/cast_helpers.hpp | 6 +++--- src/parallel/task_scheduler.cpp | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/include/duckdb/common/types/cast_helpers.hpp b/src/include/duckdb/common/types/cast_helpers.hpp index fb312ace0d64..369ae2df1533 100644 --- a/src/include/duckdb/common/types/cast_helpers.hpp +++ b/src/include/duckdb/common/types/cast_helpers.hpp @@ -61,10 +61,10 @@ class NumericHelper { template static string_t FormatSigned(T value, Vector &vector) { - typedef typename MakeUnsigned::type UNSIGNED; + typedef typename MakeUnsigned::type unsigned_t; int8_t sign = -(value < 0); - UNSIGNED unsigned_value = UNSIGNED(value ^ T(sign)) + UNSIGNED(AbsValue(sign)); - auto length = UnsafeNumericCast(UnsignedLength(unsigned_value) + AbsValue(sign)); + unsigned_t unsigned_value = UNSIGNED(value ^ T(sign)) + unsigned_t(AbsValue(sign)); + auto length = UnsafeNumericCast(UnsignedLength(unsigned_value) + AbsValue(sign)); string_t result = StringVector::EmptyString(vector, length); auto dataptr = result.GetDataWriteable(); auto endptr = dataptr + length; diff --git a/src/parallel/task_scheduler.cpp b/src/parallel/task_scheduler.cpp index db2dd6087d5f..137cf4c2ea73 100644 --- a/src/parallel/task_scheduler.cpp +++ b/src/parallel/task_scheduler.cpp @@ -16,8 +16,6 @@ #include #endif -#include // ssize_t - namespace duckdb { struct SchedulerThread { @@ -263,6 +261,7 @@ void TaskScheduler::SetAllocatorFlushTreshold(idx_t threshold) { void TaskScheduler::Signal(idx_t n) { #ifndef DUCKDB_NO_THREADS + typedef std::make_signed::type ssize_t; queue->semaphore.signal(NumericCast(n)); #endif } From 97b727f29f1fb412383bfdf051d55c795435af41 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Tue, 16 Apr 2024 10:07:37 +0200 Subject: [PATCH 468/603] Clean up the update code --- .../duckdb/storage/table/column_data.hpp | 7 ++ src/storage/table/column_data.cpp | 84 +++++++++++-------- 2 files changed, 57 insertions(+), 34 deletions(-) diff --git a/src/include/duckdb/storage/table/column_data.hpp b/src/include/duckdb/storage/table/column_data.hpp index e7bc7a2e0a6c..55020f589ccc 100644 --- a/src/include/duckdb/storage/table/column_data.hpp +++ b/src/include/duckdb/storage/table/column_data.hpp @@ -161,6 +161,13 @@ class ColumnData { template idx_t ScanVector(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result); + void ClearUpdates(); + void FetchUpdates(TransactionData transaction, idx_t vector_index, Vector &result, idx_t scan_count, + bool allow_updates, bool scan_committed); + void FetchUpdateRow(TransactionData transaction, row_t row_id, Vector &result, idx_t result_idx); + void UpdateInternal(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids, + idx_t update_count, Vector &base_vector); + protected: //! The segments holding the data of this column segment ColumnSegmentTree data; diff --git a/src/storage/table/column_data.cpp b/src/storage/table/column_data.cpp index e4b5648237d8..21924d036696 100644 --- a/src/storage/table/column_data.cpp +++ b/src/storage/table/column_data.cpp @@ -62,6 +62,11 @@ bool ColumnData::HasUpdates() const { return updates.get(); } +void ColumnData::ClearUpdates() { + lock_guard update_guard(update_lock); + updates.reset(); +} + idx_t ColumnData::GetMaxEntry() { return count; } @@ -138,25 +143,52 @@ idx_t ColumnData::ScanVector(ColumnScanState &state, Vector &result, idx_t remai return initial_remaining - remaining; } +unique_ptr ColumnData::GetUpdateStatistics() { + lock_guard update_guard(update_lock); + return updates ? updates->GetStatistics() : nullptr; +} + +void ColumnData::FetchUpdates(TransactionData transaction, idx_t vector_index, Vector &result, idx_t scan_count, + bool allow_updates, bool scan_committed) { + lock_guard update_guard(update_lock); + if (!updates) { + return; + } + if (!allow_updates && updates->HasUncommittedUpdates(vector_index)) { + throw TransactionException("Cannot create index with outstanding updates"); + } + result.Flatten(scan_count); + if (scan_committed) { + updates->FetchCommitted(vector_index, result); + } else { + updates->FetchUpdates(transaction, vector_index, result); + } +} + +void ColumnData::FetchUpdateRow(TransactionData transaction, row_t row_id, Vector &result, idx_t result_idx) { + lock_guard update_guard(update_lock); + if (!updates) { + return; + } + updates->FetchRow(transaction, row_id, result, result_idx); +} + +void ColumnData::UpdateInternal(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids, + idx_t update_count, Vector &base_vector) { + lock_guard update_guard(update_lock); + if (!updates) { + updates = make_uniq(*this); + } + updates->Update(transaction, column_index, update_vector, row_ids, update_count, base_vector); +} + template idx_t ColumnData::ScanVector(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result) { - bool has_updates = HasUpdates(); idx_t current_row = vector_index * STANDARD_VECTOR_SIZE; auto vector_count = MinValue(STANDARD_VECTOR_SIZE, count - current_row); - auto scan_count = ScanVector(state, result, vector_count, has_updates); - if (has_updates) { - lock_guard update_guard(update_lock); - if (!ALLOW_UPDATES && updates->HasUncommittedUpdates(vector_index)) { - throw TransactionException("Cannot create index with outstanding updates"); - } - result.Flatten(scan_count); - if (SCAN_COMMITTED) { - updates->FetchCommitted(vector_index, result); - } else { - updates->FetchUpdates(transaction, vector_index, result); - } - } + auto scan_count = ScanVector(state, result, vector_count, HasUpdates()); + FetchUpdates(transaction, vector_index, result, scan_count, ALLOW_UPDATES, SCAN_COMMITTED); return scan_count; } @@ -358,24 +390,17 @@ void ColumnData::FetchRow(TransactionData transaction, ColumnFetchState &state, // now perform the fetch within the segment segment->FetchRow(state, row_id, result, result_idx); // merge any updates made to this row - lock_guard update_guard(update_lock); - if (updates) { - updates->FetchRow(transaction, row_id, result, result_idx); - } + FetchUpdateRow(transaction, row_id, result, result_idx); } void ColumnData::Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids, idx_t update_count) { - lock_guard update_guard(update_lock); - if (!updates) { - updates = make_uniq(*this); - } Vector base_vector(type); ColumnScanState state; auto fetch_count = Fetch(state, row_ids[0], base_vector); base_vector.Flatten(fetch_count); - updates->Update(transaction, column_index, update_vector, row_ids, update_count, base_vector); + UpdateInternal(transaction, column_index, update_vector, row_ids, update_count, base_vector); } void ColumnData::UpdateColumn(TransactionData transaction, const vector &column_path, Vector &update_vector, @@ -385,11 +410,6 @@ void ColumnData::UpdateColumn(TransactionData transaction, const vector ColumnData::GetUpdateStatistics() { - lock_guard update_guard(update_lock); - return updates ? updates->GetStatistics() : nullptr; -} - void ColumnData::AppendTransientSegment(SegmentLock &l, idx_t start_row) { idx_t vector_segment_size = Storage::BLOCK_SIZE; @@ -451,14 +471,13 @@ unique_ptr ColumnData::Checkpoint(RowGroup &row_group, // empty table: flush the empty list return checkpoint_state; } - lock_guard update_guard(update_lock); ColumnDataCheckpointer checkpointer(*this, row_group, *checkpoint_state, checkpoint_info); checkpointer.Checkpoint(std::move(nodes)); // replace the old tree with the new one data.Replace(l, checkpoint_state->new_tree); - updates.reset(); + ClearUpdates(); return checkpoint_state; } @@ -533,10 +552,7 @@ void ColumnData::GetColumnSegmentInfo(idx_t row_group_index, vector col_p column_info.segment_count = segment->count; column_info.compression_type = CompressionTypeToString(segment->function.get().type); column_info.segment_stats = segment->stats.statistics.ToString(); - { - lock_guard ulock(update_lock); - column_info.has_updates = updates ? true : false; - } + column_info.has_updates = ColumnData::HasUpdates(); // persistent // block_id // block_offset From 446386b9a6382cacc5154a125ef4633c5001b598 Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 16 Apr 2024 10:07:42 +0200 Subject: [PATCH 469/603] format --- src/include/duckdb/common/shared_ptr.ipp | 9 ++++++--- src/include/duckdb/common/weak_ptr.ipp | 18 ++++++++---------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/include/duckdb/common/shared_ptr.ipp b/src/include/duckdb/common/shared_ptr.ipp index 460ae98f9f3b..d046dc1412f8 100644 --- a/src/include/duckdb/common/shared_ptr.ipp +++ b/src/include/duckdb/common/shared_ptr.ipp @@ -76,12 +76,14 @@ public: #ifdef DUCKDB_CLANG_TIDY [[clang::reinitializes]] #endif - shared_ptr(shared_ptr &&ref) noexcept : internal(std::move(ref.internal)) { // NOLINT: not marked as explicit + shared_ptr(shared_ptr &&ref) noexcept // NOLINT: not marked as explicit + : internal(std::move(ref.internal)) { } #ifdef DUCKDB_CLANG_TIDY [[clang::reinitializes]] #endif - shared_ptr(shared_ptr &&other) : internal(std::move(other.internal)) { // NOLINT: not marked as explicit + shared_ptr(shared_ptr &&other) // NOLINT: not marked as explicit + : internal(std::move(other.internal)) { } // Construct from std::shared_ptr @@ -104,7 +106,8 @@ public: #ifdef DUCKDB_CLANG_TIDY [[clang::reinitializes]] #endif - shared_ptr(unique_ptr &&other) : internal(std::move(other)) { // NOLINT: not marked as explicit + shared_ptr(unique_ptr &&other) // NOLINT: not marked as explicit + : internal(std::move(other)) { __enable_weak_this(internal.get(), internal.get()); } diff --git a/src/include/duckdb/common/weak_ptr.ipp b/src/include/duckdb/common/weak_ptr.ipp index a714eb0e67b0..076fde953258 100644 --- a/src/include/duckdb/common/weak_ptr.ipp +++ b/src/include/duckdb/common/weak_ptr.ipp @@ -19,27 +19,25 @@ public: } // NOLINTBEGIN - template - weak_ptr(shared_ptr const &ptr, - typename std::enable_if::value, int>::type = 0) noexcept - : internal(ptr.internal) { + template ::value, int>::type = 0> + weak_ptr(shared_ptr const &ptr) noexcept : internal(ptr.internal) { } weak_ptr(weak_ptr const &other) noexcept : internal(other.internal) { } - template - weak_ptr(weak_ptr const &ptr, typename std::enable_if::value, int>::type = 0) noexcept - : internal(ptr.internal) { + template ::value, int>::type = 0> + weak_ptr(weak_ptr const &ptr) noexcept : internal(ptr.internal) { } #ifdef DUCKDB_CLANG_TIDY [[clang::reinitializes]] #endif - weak_ptr(weak_ptr &&ptr) noexcept : internal(std::move(ptr.internal)) { + weak_ptr(weak_ptr &&ptr) noexcept + : internal(std::move(ptr.internal)) { } - template + template ::value, int>::type = 0> #ifdef DUCKDB_CLANG_TIDY [[clang::reinitializes]] #endif - weak_ptr(weak_ptr &&ptr, typename std::enable_if::value, int>::type = 0) noexcept + weak_ptr(weak_ptr &&ptr) noexcept : internal(std::move(ptr.internal)) { } // NOLINTEND From 10db9ca14093c68b8c1ea8c895a67f8058ad9d57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Tue, 16 Apr 2024 10:10:48 +0200 Subject: [PATCH 470/603] orr --- src/include/duckdb/common/types/cast_helpers.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/duckdb/common/types/cast_helpers.hpp b/src/include/duckdb/common/types/cast_helpers.hpp index 369ae2df1533..2ff2da7594b2 100644 --- a/src/include/duckdb/common/types/cast_helpers.hpp +++ b/src/include/duckdb/common/types/cast_helpers.hpp @@ -63,7 +63,7 @@ class NumericHelper { static string_t FormatSigned(T value, Vector &vector) { typedef typename MakeUnsigned::type unsigned_t; int8_t sign = -(value < 0); - unsigned_t unsigned_value = UNSIGNED(value ^ T(sign)) + unsigned_t(AbsValue(sign)); + unsigned_t unsigned_value = unsigned_t(value ^ T(sign)) + unsigned_t(AbsValue(sign)); auto length = UnsafeNumericCast(UnsignedLength(unsigned_value) + AbsValue(sign)); string_t result = StringVector::EmptyString(vector, length); auto dataptr = result.GetDataWriteable(); From 0cea49443621edd8f2c49d79237020050328d534 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Tue, 16 Apr 2024 10:25:31 +0200 Subject: [PATCH 471/603] Make UNION BY NAME also use ForceMaxLogicalType, similar to UNION --- src/planner/binder/query_node/bind_setop_node.cpp | 4 ++-- test/sql/setops/test_union_by_name.test | 13 +++++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/planner/binder/query_node/bind_setop_node.cpp b/src/planner/binder/query_node/bind_setop_node.cpp index f5f8e26e3a23..316e9c9edf27 100644 --- a/src/planner/binder/query_node/bind_setop_node.cpp +++ b/src/planner/binder/query_node/bind_setop_node.cpp @@ -118,8 +118,8 @@ static void BuildUnionByNameInfo(ClientContext &context, BoundSetOperationNode & bool right_exist = right_index != right_names_map.end(); LogicalType result_type; if (left_exist && right_exist) { - result_type = LogicalType::MaxLogicalType(context, left_node.types[left_index->second], - right_node.types[right_index->second]); + result_type = LogicalType::ForceMaxLogicalType(left_node.types[left_index->second], + right_node.types[right_index->second]); if (left_index->second != i || right_index->second != i) { need_reorder = true; } diff --git a/test/sql/setops/test_union_by_name.test b/test/sql/setops/test_union_by_name.test index 7d7cb34ecd3f..4e3b56591bd8 100644 --- a/test/sql/setops/test_union_by_name.test +++ b/test/sql/setops/test_union_by_name.test @@ -146,6 +146,15 @@ NULL 3 1 NULL 3 NULL +# union by name with different types +query I +select '0' as c union all select 0 as c; +---- +0 +0 - - +query I +select '0' as c union all by name select 0 as c; +---- +0 +0 From e75c31d2538885ebe6e8f591a899ebca04b19b33 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Tue, 16 Apr 2024 10:34:40 +0200 Subject: [PATCH 472/603] Fix #11469 - make unnest parameters case-insensitive --- src/planner/binder/expression/bind_unnest_expression.cpp | 2 +- test/sql/types/list/recursive_unnest.test | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/planner/binder/expression/bind_unnest_expression.cpp b/src/planner/binder/expression/bind_unnest_expression.cpp index 0883d5f29870..58d50a4603ed 100644 --- a/src/planner/binder/expression/bind_unnest_expression.cpp +++ b/src/planner/binder/expression/bind_unnest_expression.cpp @@ -71,7 +71,7 @@ BindResult SelectBinder::BindUnnest(FunctionExpression &function, idx_t depth, b if (!function.children[i]->IsScalar()) { break; } - auto alias = function.children[i]->alias; + auto alias = StringUtil::Lower(function.children[i]->alias); BindChild(function.children[i], depth, error); if (error.HasError()) { return BindResult(std::move(error)); diff --git a/test/sql/types/list/recursive_unnest.test b/test/sql/types/list/recursive_unnest.test index f8e1de0a394f..8b8cb53e3b25 100644 --- a/test/sql/types/list/recursive_unnest.test +++ b/test/sql/types/list/recursive_unnest.test @@ -22,6 +22,15 @@ SELECT UNNEST([[[[[1, 2], [3, 4]], [[5]]], [[[]]]]], recursive := true) 4 5 +query I +SELECT UNNEST([[[[[1, 2], [3, 4]], [[5]]], [[[]]]]], RECURSIVE := true) +---- +1 +2 +3 +4 +5 + query I SELECT UNNEST([[[[[1, 2], [3, 4]], [[5]]], [[[]]]]], max_depth := 1) ---- From 0a2f238eff5cb771d9ad98d9f97f12267f86c2db Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Tue, 16 Apr 2024 10:46:24 +0200 Subject: [PATCH 473/603] Fix #11467: correctly merge unnamed structs and structs in CombineEqualTypes --- src/common/types.cpp | 9 ++++++--- test/sql/types/struct/unnamed_struct_mix.test | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 test/sql/types/struct/unnamed_struct_mix.test diff --git a/src/common/types.cpp b/src/common/types.cpp index e4318c26ed22..862f86e5f900 100644 --- a/src/common/types.cpp +++ b/src/common/types.cpp @@ -868,6 +868,8 @@ static bool CombineEqualTypes(const LogicalType &left, const LogicalType &right, // struct: perform recursively on each child auto &left_child_types = StructType::GetChildTypes(left); auto &right_child_types = StructType::GetChildTypes(right); + bool left_unnamed = StructType::IsUnnamed(left); + auto any_unnamed = left_unnamed || StructType::IsUnnamed(right); if (left_child_types.size() != right_child_types.size()) { // child types are not of equal size, we can't cast // return false @@ -876,14 +878,15 @@ static bool CombineEqualTypes(const LogicalType &left, const LogicalType &right, child_list_t child_types; for (idx_t i = 0; i < left_child_types.size(); i++) { LogicalType child_type; - // Child names must be in the same order - if (!StringUtil::CIEquals(left_child_types[i].first, right_child_types[i].first)) { + // Child names must be in the same order OR either one of the structs must be unnamed + if (!any_unnamed && !StringUtil::CIEquals(left_child_types[i].first, right_child_types[i].first)) { return false; } if (!OP::Operation(left_child_types[i].second, right_child_types[i].second, child_type)) { return false; } - child_types.emplace_back(left_child_types[i].first, std::move(child_type)); + auto &child_name = left_unnamed ? right_child_types[i].first : left_child_types[i].first; + child_types.emplace_back(child_name, std::move(child_type)); } result = LogicalType::STRUCT(child_types); return true; diff --git a/test/sql/types/struct/unnamed_struct_mix.test b/test/sql/types/struct/unnamed_struct_mix.test new file mode 100644 index 000000000000..dcc6086e7ed1 --- /dev/null +++ b/test/sql/types/struct/unnamed_struct_mix.test @@ -0,0 +1,16 @@ +# name: test/sql/types/struct/unnamed_struct_mix.test +# description: Test mix of named and unnamed structs +# group: [struct] + +statement ok +PRAGMA enable_verification + +query I +select [{ t:'abc', len:5 }, ('abc', 2)] +---- +[{'t': abc, 'len': 5}, {'t': abc, 'len': 2}] + +query I +select [('abc', 2), { t:'abc', len:5 }] +---- +[{'t': abc, 'len': 2}, {'t': abc, 'len': 5}] From e77834112e152364f9f15582ee6ef59fee832543 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Tue, 16 Apr 2024 11:02:55 +0200 Subject: [PATCH 474/603] proper skip --- tools/pythonpkg/tests/fast/adbc/test_adbc.py | 6 +++++- .../pythonpkg/tests/fast/adbc/test_connection_get_info.py | 6 +++++- tools/pythonpkg/tests/fast/adbc/test_statement_bind.py | 7 ++++++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/tools/pythonpkg/tests/fast/adbc/test_adbc.py b/tools/pythonpkg/tests/fast/adbc/test_adbc.py index efe1ab97ca32..21b3b9bf7c0e 100644 --- a/tools/pythonpkg/tests/fast/adbc/test_adbc.py +++ b/tools/pythonpkg/tests/fast/adbc/test_adbc.py @@ -4,7 +4,11 @@ import datetime import os -@pytest.mark.skipif(sys.version_info < (3, 9), reason="Requires python 3.9") +if sys.version_info < (3, 9): + pytest.skip( + "Python Version must be higher or equal to 3.9 to run this test", + allow_module_level=True, + ) adbc_driver_manager = pytest.importorskip("adbc_driver_manager.dbapi") adbc_driver_manager_lib = pytest.importorskip("adbc_driver_manager._lib") diff --git a/tools/pythonpkg/tests/fast/adbc/test_connection_get_info.py b/tools/pythonpkg/tests/fast/adbc/test_connection_get_info.py index 000ceff7f80b..3744b7dadd2f 100644 --- a/tools/pythonpkg/tests/fast/adbc/test_connection_get_info.py +++ b/tools/pythonpkg/tests/fast/adbc/test_connection_get_info.py @@ -6,7 +6,11 @@ pa = pytest.importorskip("pyarrow") adbc_driver_manager = pytest.importorskip("adbc_driver_manager") -@pytest.mark.skipif(sys.version_info < (3, 9), reason="Requires python 3.9") +if sys.version_info < (3, 9): + pytest.skip( + "Python Version must be higher or equal to 3.9 to run this test", + allow_module_level=True, + ) try: adbc_driver_duckdb = pytest.importorskip("adbc_driver_duckdb.dbapi") diff --git a/tools/pythonpkg/tests/fast/adbc/test_statement_bind.py b/tools/pythonpkg/tests/fast/adbc/test_statement_bind.py index ddab8ea2ccea..5e9d7d457145 100644 --- a/tools/pythonpkg/tests/fast/adbc/test_statement_bind.py +++ b/tools/pythonpkg/tests/fast/adbc/test_statement_bind.py @@ -2,6 +2,12 @@ import pytest +if sys.version_info < (3, 9): + pytest.skip( + "Python Version must be higher or equal to 3.9 to run this test", + allow_module_level=True, + ) + pa = pytest.importorskip("pyarrow") adbc_driver_manager = pytest.importorskip("adbc_driver_manager") @@ -9,7 +15,6 @@ con = adbc_driver_duckdb.connect() -@pytest.mark.skipif(sys.version_info < (3, 9), reason="Requires python 3.9") def _import(handle): """Helper to import a C Data Interface handle.""" if isinstance(handle, adbc_driver_manager.ArrowArrayStreamHandle): From a885dac37e3ca52efa6c399a44268255711849dd Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Tue, 16 Apr 2024 11:31:39 +0200 Subject: [PATCH 475/603] Fix uninitialized values --- src/include/duckdb/storage/partial_block_manager.hpp | 2 +- tools/shell/linenoise/linenoise.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/include/duckdb/storage/partial_block_manager.hpp b/src/include/duckdb/storage/partial_block_manager.hpp index 935f0126993a..b08ee7296351 100644 --- a/src/include/duckdb/storage/partial_block_manager.hpp +++ b/src/include/duckdb/storage/partial_block_manager.hpp @@ -34,7 +34,7 @@ struct UninitializedRegion { //! The current state of a partial block struct PartialBlockState { //! The block id of the partial block - block_id_t block_id; + block_id_t block_id = INVALID_BLOCK; //! The total bytes that we can assign to this block uint32_t block_size; //! Next allocation offset, and also the current allocation size diff --git a/tools/shell/linenoise/linenoise.cpp b/tools/shell/linenoise/linenoise.cpp index 090965c0ca3a..17ecc3e209e6 100644 --- a/tools/shell/linenoise/linenoise.cpp +++ b/tools/shell/linenoise/linenoise.cpp @@ -1061,6 +1061,7 @@ Linenoise::Linenoise(int stdin_fd, int stdout_fd, char *buf, size_t buflen, cons render = true; continuation_markers = true; insert = false; + search_index = 0; /* Buffer starts empty. */ buf[0] = '\0'; From 4acf688751efa6990b8bde99dec534a36c9949d7 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Tue, 16 Apr 2024 11:48:33 +0200 Subject: [PATCH 476/603] Fix issue where padding bits in bitpacking were not zero-initialized --- scripts/test_zero_initialize.py | 1 + src/storage/compression/bitpacking.cpp | 7 ++++++- .../storage/nested/struct_of_lists_unaligned.test | 15 +++++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 test/sql/storage/nested/struct_of_lists_unaligned.test diff --git a/scripts/test_zero_initialize.py b/scripts/test_zero_initialize.py index 397e14e6c68b..966e1cde187d 100644 --- a/scripts/test_zero_initialize.py +++ b/scripts/test_zero_initialize.py @@ -27,6 +27,7 @@ 'test/sql/storage/update/test_store_null_updates.test', 'test/sql/storage/test_store_integers.test', 'test/sql/storage/mix/test_update_delete_string.test', + 'test/sql/storage/nested/struct_of_lists_unaligned.test', ] diff --git a/src/storage/compression/bitpacking.cpp b/src/storage/compression/bitpacking.cpp index fa7f4a7c79f3..5a82482ebf7e 100644 --- a/src/storage/compression/bitpacking.cpp +++ b/src/storage/compression/bitpacking.cpp @@ -514,7 +514,8 @@ struct BitpackingCompressState : public CompressionState { auto base_ptr = handle.Ptr(); // Compact the segment by moving the metadata next to the data. - idx_t metadata_offset = AlignValue(data_ptr - base_ptr); + idx_t unaligned_offset = data_ptr - base_ptr; + idx_t metadata_offset = AlignValue(unaligned_offset); idx_t metadata_size = base_ptr + Storage::BLOCK_SIZE - metadata_ptr; idx_t total_segment_size = metadata_offset + metadata_size; @@ -523,6 +524,10 @@ struct BitpackingCompressState : public CompressionState { throw InternalException("Error in bitpacking size calculation"); } + if (unaligned_offset != metadata_offset) { + // zero initialize any padding bits + memset(base_ptr + unaligned_offset, 0, metadata_offset - unaligned_offset); + } memmove(base_ptr + metadata_offset, metadata_ptr, metadata_size); // Store the offset of the metadata of the first group (which is at the highest address). diff --git a/test/sql/storage/nested/struct_of_lists_unaligned.test b/test/sql/storage/nested/struct_of_lists_unaligned.test new file mode 100644 index 000000000000..40d09347a4a7 --- /dev/null +++ b/test/sql/storage/nested/struct_of_lists_unaligned.test @@ -0,0 +1,15 @@ +# name: test/sql/storage/nested/struct_of_lists_unaligned.test +# description: Test storage of structs with lists in it +# group: [nested] + +# load the DB from disk +load __TEST_DIR__/test_store_list_of_structs.db + +statement ok +CREATE TABLE test_list_2 (a integer, b STRUCT(c VARCHAR[], d VARCHAR[], e INTEGER[])); + +statement ok +INSERT INTO test_list_2 SELECT 1, row(['a', 'b', 'c', 'd', 'e', 'f'], ['A', 'B'], [1, 5, 9]) FROM range(10); + +statement ok +CHECKPOINT; From eed84b5041fea3e114f4a35712ba3568f1bbfa42 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Tue, 16 Apr 2024 11:49:12 +0200 Subject: [PATCH 477/603] Revert --- src/include/duckdb/storage/partial_block_manager.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/duckdb/storage/partial_block_manager.hpp b/src/include/duckdb/storage/partial_block_manager.hpp index b08ee7296351..935f0126993a 100644 --- a/src/include/duckdb/storage/partial_block_manager.hpp +++ b/src/include/duckdb/storage/partial_block_manager.hpp @@ -34,7 +34,7 @@ struct UninitializedRegion { //! The current state of a partial block struct PartialBlockState { //! The block id of the partial block - block_id_t block_id = INVALID_BLOCK; + block_id_t block_id; //! The total bytes that we can assign to this block uint32_t block_size; //! Next allocation offset, and also the current allocation size From 5d1aa4833580dc60945bf0aa6b5bf2a18a9fbe3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Tue, 16 Apr 2024 11:49:37 +0200 Subject: [PATCH 478/603] moar ci 42 --- src/common/box_renderer.cpp | 2 +- src/parallel/task_scheduler.cpp | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/common/box_renderer.cpp b/src/common/box_renderer.cpp index 629f6349fe69..684e47ce3ad3 100644 --- a/src/common/box_renderer.cpp +++ b/src/common/box_renderer.cpp @@ -399,7 +399,7 @@ vector BoxRenderer::ComputeRenderWidths(const vector &names, cons // e.g. if we have 10 columns, we remove #5, then #4, then #6, then #3, then #7, etc int64_t offset = 0; while (total_length > max_width) { - idx_t c = column_count / 2 + NumericCast(offset); + auto c = NumericCast(NumericCast(column_count) / 2 + offset); total_length -= widths[c] + 3; pruned_columns.insert(c); if (offset >= 0) { diff --git a/src/parallel/task_scheduler.cpp b/src/parallel/task_scheduler.cpp index 137cf4c2ea73..d11b144bf3e1 100644 --- a/src/parallel/task_scheduler.cpp +++ b/src/parallel/task_scheduler.cpp @@ -109,7 +109,11 @@ TaskScheduler::TaskScheduler(DatabaseInstance &db) TaskScheduler::~TaskScheduler() { #ifndef DUCKDB_NO_THREADS - RelaunchThreadsInternal(0); + try { + RelaunchThreadsInternal(0); + } catch (...) { + // nothing we can do in the destructor if this fails + } #endif } From 511c7592051698266a437a2ec59c4b2479063337 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Tue, 16 Apr 2024 12:00:03 +0200 Subject: [PATCH 479/603] Test fixes + add more tests --- test/sql/setops/test_union_all_by_name.test | 4 ++-- test/sql/setops/test_union_by_name.test | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/test/sql/setops/test_union_all_by_name.test b/test/sql/setops/test_union_all_by_name.test index cdba78269c45..4642e92e3fac 100644 --- a/test/sql/setops/test_union_all_by_name.test +++ b/test/sql/setops/test_union_all_by_name.test @@ -220,10 +220,10 @@ NULL 1 ########## type cast -statement error +query I SELECT DISTINCT ON(x) x FROM (SELECT 1 as x UNION ALL BY NAME SELECT '1' as x); ---- -an explicit cast is required +1 # have to sort here too because distinct on is hash-based, does not preserve order query I sort diff --git a/test/sql/setops/test_union_by_name.test b/test/sql/setops/test_union_by_name.test index 4e3b56591bd8..56909128001f 100644 --- a/test/sql/setops/test_union_by_name.test +++ b/test/sql/setops/test_union_by_name.test @@ -158,3 +158,19 @@ select '0' as c union all by name select 0 as c; ---- 0 0 + +query I +select {'a': '0'} as c union all by name select {'a': 0} as c +---- +{'a': 0} +{'a': 0} + +statement error +select {'a': 'hello'} as c union all by name select {'b': 'hello'} as c; +---- +Type Error + +statement error +select {'a': 'hello'} as c union all by name select {'a': 'hello', 'b': 'world'} as c; +---- +Type Error From 823391639f14443f8bd15b2ecba72d67b7d73c57 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Tue, 16 Apr 2024 12:36:09 +0200 Subject: [PATCH 480/603] Fixup extension_version metadata CMake for external extensions --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b1eaa7adfa2..5f199d3518cf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -921,7 +921,7 @@ macro(register_external_extension NAME URL COMMIT DONT_LINK DONT_BUILD LOAD_TEST endif() message(STATUS "Load extension '${NAME}' from ${URL} @ ${GIT_SHORT_COMMIT}") - set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_EXT_VERSION "${GIT_SHORT_COMMIT}") + set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_EXT_VERSION "${GIT_SHORT_COMMIT}" PARENT_SCOPE) if ("${INCLUDE_PATH}" STREQUAL "") set(INCLUDE_FULL_PATH "${${NAME}_extension_fc_SOURCE_DIR}/src/include") From 49910f0f6a46424d18637bf339e7aed7242907c8 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Tue, 16 Apr 2024 12:36:53 +0200 Subject: [PATCH 481/603] extension_version: Add tests on extension_version being there --- .../extensions/version_is_valid_httpfs.test | 21 +++++++++++++++++++ .../extensions/version_is_valid_sqlite.test | 21 +++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 test/sql/extensions/version_is_valid_httpfs.test create mode 100644 test/sql/extensions/version_is_valid_sqlite.test diff --git a/test/sql/extensions/version_is_valid_httpfs.test b/test/sql/extensions/version_is_valid_httpfs.test new file mode 100644 index 000000000000..4fc2319c4ed7 --- /dev/null +++ b/test/sql/extensions/version_is_valid_httpfs.test @@ -0,0 +1,21 @@ +# name: test/sql/extensions/version_is_valid_httpfs.test +# description: Test version metadata on load +# group: [extensions] + +require-env LOCAL_EXTENSION_REPO + +require httpfs + +statement ok +SET autoinstall_known_extensions=true; + +statement ok +SET autoload_known_extensions=true; + +statement ok +SET enable_server_cert_verification = true; + +query I +SELECT count(*) FROM duckdb_extensions() WHERE extension_version != '' AND extension_name == 'httpfs'; +---- +1 diff --git a/test/sql/extensions/version_is_valid_sqlite.test b/test/sql/extensions/version_is_valid_sqlite.test new file mode 100644 index 000000000000..762dd1b5d0c7 --- /dev/null +++ b/test/sql/extensions/version_is_valid_sqlite.test @@ -0,0 +1,21 @@ +# name: test/sql/extensions/version_is_valid_sqlite.test +# description: Test version metadata on load +# group: [extensions] + +require-env LOCAL_EXTENSION_REPO + +require sqlite_scanner + +statement ok +SET autoinstall_known_extensions=true; + +statement ok +SET autoload_known_extensions=true; + +statement ok +SET GLOBAL sqlite_all_varchar = true; + +query I +SELECT count(*) FROM duckdb_extensions() WHERE extension_version != '' AND extension_name == 'sqlite_scanner'; +---- +1 From ec5926fc20f260bd9b64404a3b40540917f47a82 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Tue, 16 Apr 2024 14:18:46 +0200 Subject: [PATCH 482/603] Fix #11484: support constant indexes in ARRAY(.. ORDER BY) --- .../expression/transform_subquery.cpp | 18 ++++++++ .../subquery/scalar/array_order_subquery.test | 43 +++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/src/parser/transform/expression/transform_subquery.cpp b/src/parser/transform/expression/transform_subquery.cpp index 33a9b1604bc4..11279ff68324 100644 --- a/src/parser/transform/expression/transform_subquery.cpp +++ b/src/parser/transform/expression/transform_subquery.cpp @@ -2,6 +2,8 @@ #include "duckdb/parser/transformer.hpp" #include "duckdb/parser/query_node/select_node.hpp" #include "duckdb/parser/tableref/subqueryref.hpp" +#include "duckdb/parser/expression/constant_expression.hpp" +#include "duckdb/parser/expression/positional_reference_expression.hpp" namespace duckdb { @@ -69,12 +71,28 @@ unique_ptr Transformer::TransformSubquery(duckdb_libpgquery::P vector> children; children.push_back(std::move(columns_star)); auto aggr = make_uniq("array_agg", std::move(children)); + // push ORDER BY modifiers into the array_agg for (auto &modifier : subquery_expr->subquery->node->modifiers) { if (modifier->type == ResultModifierType::ORDER_MODIFIER) { aggr->order_bys = unique_ptr_cast(modifier->Copy()); break; } } + // transform constants (e.g. ORDER BY 1) into positional references (ORDER BY #1) + if (aggr->order_bys) { + for (auto &order : aggr->order_bys->orders) { + if (order.expression->type == ExpressionType::VALUE_CONSTANT) { + auto &constant_expr = order.expression->Cast(); + Value bigint_value; + string error; + if (constant_expr.value.DefaultTryCastAs(LogicalType::BIGINT, bigint_value, &error)) { + int64_t order_index = BigIntValue::Get(bigint_value); + idx_t positional_index = order_index < 0 ? NumericLimits::Maximum() : order_index; + order.expression = make_uniq(positional_index); + } + } + } + } // ARRAY_AGG(COLUMNS(*)) IS NULL auto agg_is_null = make_uniq(ExpressionType::OPERATOR_IS_NULL, aggr->Copy()); // empty list diff --git a/test/sql/subquery/scalar/array_order_subquery.test b/test/sql/subquery/scalar/array_order_subquery.test index d622e99df772..13af1aa1816d 100644 --- a/test/sql/subquery/scalar/array_order_subquery.test +++ b/test/sql/subquery/scalar/array_order_subquery.test @@ -37,3 +37,46 @@ select array(select unnest(l) AS i order by i desc nulls first) as a from (value [] [12, 11, 10] +# use integer literals +query I +SELECT ARRAY + (SELECT 1 UNION ALL + SELECT 2 UNION ALL + SELECT 3 + ORDER by 1) AS new_array; +---- +[1, 2, 3] + +query I +SELECT ARRAY + (SELECT 1 UNION ALL + SELECT 2 UNION ALL + SELECT 3 + ORDER by 1 DESC) AS new_array; +---- +[3, 2, 1] + +statement error +SELECT ARRAY + (SELECT 1 UNION ALL + SELECT 2 UNION ALL + SELECT 3 + ORDER by -1) AS new_array; +---- +ORDER term out of range + +statement error +SELECT ARRAY + (SELECT 1 UNION ALL + SELECT 2 UNION ALL + SELECT 3 + ORDER by 2) AS new_array; +---- +ORDER term out of range + +statement ok +SELECT ARRAY + (SELECT 1 UNION ALL + SELECT 2 UNION ALL + SELECT 3 + ORDER by 'hello world') AS new_array; From a1a9ffe08dfab13ff5f3d44739535916699c55bd Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Tue, 16 Apr 2024 14:19:43 +0200 Subject: [PATCH 483/603] Explicit cast --- src/parser/transform/expression/transform_subquery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parser/transform/expression/transform_subquery.cpp b/src/parser/transform/expression/transform_subquery.cpp index 11279ff68324..c8dbe23a48c8 100644 --- a/src/parser/transform/expression/transform_subquery.cpp +++ b/src/parser/transform/expression/transform_subquery.cpp @@ -87,7 +87,7 @@ unique_ptr Transformer::TransformSubquery(duckdb_libpgquery::P string error; if (constant_expr.value.DefaultTryCastAs(LogicalType::BIGINT, bigint_value, &error)) { int64_t order_index = BigIntValue::Get(bigint_value); - idx_t positional_index = order_index < 0 ? NumericLimits::Maximum() : order_index; + idx_t positional_index = order_index < 0 ? NumericLimits::Maximum() : idx_t(order_index); order.expression = make_uniq(positional_index); } } From 8b0ac433e2b75acb09491a4430bbb366741263c0 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Tue, 16 Apr 2024 14:24:04 +0200 Subject: [PATCH 484/603] allow decimal type in auto_type_candidates --- data/csv/decimal.csv | 2 + .../csv_scanner/util/csv_reader_options.cpp | 7 +- test/sql/copy/csv/test_decimal.test | 73 +++++++++++++++++++ 3 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 data/csv/decimal.csv create mode 100644 test/sql/copy/csv/test_decimal.test diff --git a/data/csv/decimal.csv b/data/csv/decimal.csv new file mode 100644 index 000000000000..f1728bde39a9 --- /dev/null +++ b/data/csv/decimal.csv @@ -0,0 +1,2 @@ +col_a +3.5215257120407011 \ No newline at end of file diff --git a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp index c4a24ffbc054..82a740b2d154 100644 --- a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp +++ b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp @@ -415,9 +415,10 @@ static uint8_t GetCandidateSpecificity(const LogicalType &candidate_type) { {(uint8_t)LogicalTypeId::VARCHAR, 0}, {(uint8_t)LogicalTypeId::TIMESTAMP, 1}, {(uint8_t)LogicalTypeId::DATE, 2}, {(uint8_t)LogicalTypeId::TIME, 3}, {(uint8_t)LogicalTypeId::DOUBLE, 4}, {(uint8_t)LogicalTypeId::FLOAT, 5}, - {(uint8_t)LogicalTypeId::BIGINT, 6}, {(uint8_t)LogicalTypeId::INTEGER, 7}, - {(uint8_t)LogicalTypeId::SMALLINT, 8}, {(uint8_t)LogicalTypeId::TINYINT, 9}, - {(uint8_t)LogicalTypeId::BOOLEAN, 10}, {(uint8_t)LogicalTypeId::SQLNULL, 11}}; + {(uint8_t)LogicalTypeId::DECIMAL, 6}, {(uint8_t)LogicalTypeId::BIGINT, 7}, + {(uint8_t)LogicalTypeId::INTEGER, 8}, {(uint8_t)LogicalTypeId::SMALLINT, 9}, + {(uint8_t)LogicalTypeId::TINYINT, 10}, {(uint8_t)LogicalTypeId::BOOLEAN, 11}, + {(uint8_t)LogicalTypeId::SQLNULL, 12}}; auto id = (uint8_t)candidate_type.id(); auto it = auto_type_candidates_specificity.find(id); diff --git a/test/sql/copy/csv/test_decimal.test b/test/sql/copy/csv/test_decimal.test new file mode 100644 index 000000000000..02de1007581b --- /dev/null +++ b/test/sql/copy/csv/test_decimal.test @@ -0,0 +1,73 @@ +# name: test/sql/copy/csv/test_decimal.test +# description: Test CSVs with a decimal value +# group: [csv] + +query I +FROM 'data/csv/decimal.csv' +---- +3.521525712040701 + +query I +select typeof (col_a) FROM 'data/csv/decimal.csv' +---- +DOUBLE + +# By Default our decimal is (18,3) +query I +SELECT * FROM read_csv( + 'data/csv/decimal.csv', + columns = {'col_a': 'DECIMAL'}); +---- +3.522 + +query I +SELECT typeof(col_a) FROM read_csv( + 'data/csv/decimal.csv', + columns = {'col_a': 'DECIMAL'}); +---- +DECIMAL(18,3) + +# We can define our decimal as (18,15) +query I +SELECT * FROM read_csv( + 'data/csv/decimal.csv', + columns = {'col_a': 'DECIMAL(18,15)'}); +---- +3.521525712040701 + +query I +SELECT typeof(col_a) FROM read_csv( + 'data/csv/decimal.csv', + columns = {'col_a': 'DECIMAL(18,15)'}); +---- +DECIMAL(18,15) + +query I +SELECT typeof(col_a) FROM read_csv( + 'data/csv/decimal.csv', + auto_type_candidates=['NULL', 'DECIMAL', 'VARCHAR']); +---- +DECIMAL(18,3) + +# If we have multiple decimal values being defined, it defaults to the last one here +query I +SELECT typeof(col_a) FROM read_csv( + 'data/csv/decimal.csv', + auto_type_candidates=['NULL', 'DECIMAL(18,3)','DECIMAL(18,15)', 'VARCHAR']); +---- +DECIMAL(18,15) + +query I +SELECT typeof(col_a) FROM read_csv( + 'data/csv/decimal.csv', + auto_type_candidates=['NULL','DECIMAL(18,15)', 'DECIMAL(18,3)', 'VARCHAR']); +---- +DECIMAL(18,3) + +query I +SELECT typeof(col_a) FROM read_csv( + 'data/csv/decimal.csv', + auto_type_candidates=['NULL', 'DECIMAL(18,15)', 'VARCHAR']); +---- +DECIMAL(18,15) + From 2cf2d13e121069d88f851fb70e5b987bb4617df4 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Tue, 16 Apr 2024 14:57:08 +0200 Subject: [PATCH 485/603] Improve hive type auto-casting so that it looks at all files instead of only the first file --- src/common/multi_file_reader.cpp | 74 ++++++++++++------- .../common/multi_file_reader_options.hpp | 2 +- test/sql/copy/hive_types.test | 20 ++++- 3 files changed, 66 insertions(+), 30 deletions(-) diff --git a/src/common/multi_file_reader.cpp b/src/common/multi_file_reader.cpp index 5ba38299ef32..2aa6521fb2d6 100644 --- a/src/common/multi_file_reader.cpp +++ b/src/common/multi_file_reader.cpp @@ -421,39 +421,59 @@ bool MultiFileReaderOptions::AutoDetectHivePartitioningInternal(const vector &files, ClientContext &context) { + const LogicalType candidates[] = {LogicalType::DATE, LogicalType::TIMESTAMP, LogicalType::BIGINT}; + auto &fs = FileSystem::GetFileSystem(context); - std::map partitions; - auto splits = StringUtil::Split(file, fs.PathSeparator(file)); - if (splits.size() < 2) { - return; - } - for (auto it = splits.begin(); it != std::prev(splits.end()); it++) { - auto part = StringUtil::Split(*it, "="); - if (part.size() == 2) { - partitions[part.front()] = part.back(); + unordered_map detected_types; + for (auto &file : files) { + unordered_map partitions; + auto splits = StringUtil::Split(file, fs.PathSeparator(file)); + if (splits.size() < 2) { + return; } - } - if (partitions.empty()) { - return; - } - - const LogicalType candidates[] = {LogicalType::DATE, LogicalType::TIMESTAMP, LogicalType::BIGINT}; - for (auto &part : partitions) { - const string &name = part.first; - if (hive_types_schema.find(name) != hive_types_schema.end()) { - continue; + for (auto it = splits.begin(); it != std::prev(splits.end()); it++) { + auto part = StringUtil::Split(*it, "="); + if (part.size() == 2) { + partitions[part.front()] = part.back(); + } } - Value value(part.second); - for (auto &candidate : candidates) { - const bool success = value.TryCastAs(context, candidate, true); - if (success) { - hive_types_schema[name] = candidate; - break; + if (partitions.empty()) { + return; + } + + for (auto &part : partitions) { + const string &name = part.first; + if (hive_types_schema.find(name) != hive_types_schema.end()) { + // type was explicitly provided by the user + continue; + } + LogicalType detected_type = LogicalType::VARCHAR; + Value value(part.second); + for (auto &candidate : candidates) { + const bool success = value.TryCastAs(context, candidate, true); + if (success) { + detected_type = candidate; + break; + } + } + auto entry = detected_types.find(name); + if (entry == detected_types.end()) { + // type was not yet detected - insert it + detected_types.insert(make_pair(name, std::move(detected_type))); + } else { + // type was already detected - check if the type matches + // if not promote to VARCHAR + if (entry->second != detected_type) { + entry->second = LogicalType::VARCHAR; + } } } } + for (auto &entry : detected_types) { + hive_types_schema.insert(make_pair(entry.first, std::move(entry.second))); + } } void MultiFileReaderOptions::AutoDetectHivePartitioning(const vector &files, ClientContext &context) { D_ASSERT(!files.empty()); @@ -471,7 +491,7 @@ void MultiFileReaderOptions::AutoDetectHivePartitioning(const vector &fi hive_partitioning = AutoDetectHivePartitioningInternal(files, context); } if (hive_partitioning && hive_types_autocast) { - AutoDetectHiveTypesInternal(files.front(), context); + AutoDetectHiveTypesInternal(files, context); } } void MultiFileReaderOptions::VerifyHiveTypesArePartitions(const std::map &partitions) const { diff --git a/src/include/duckdb/common/multi_file_reader_options.hpp b/src/include/duckdb/common/multi_file_reader_options.hpp index 74a737f398b3..5b8bd90668cd 100644 --- a/src/include/duckdb/common/multi_file_reader_options.hpp +++ b/src/include/duckdb/common/multi_file_reader_options.hpp @@ -29,7 +29,7 @@ struct MultiFileReaderOptions { DUCKDB_API void AddBatchInfo(BindInfo &bind_info) const; DUCKDB_API void AutoDetectHivePartitioning(const vector &files, ClientContext &context); DUCKDB_API static bool AutoDetectHivePartitioningInternal(const vector &files, ClientContext &context); - DUCKDB_API void AutoDetectHiveTypesInternal(const string &file, ClientContext &context); + DUCKDB_API void AutoDetectHiveTypesInternal(const vector &file, ClientContext &context); DUCKDB_API void VerifyHiveTypesArePartitions(const std::map &partitions) const; DUCKDB_API LogicalType GetHiveLogicalType(const string &hive_partition_column) const; DUCKDB_API Value GetHivePartitionValue(const string &base, const string &entry, ClientContext &context) const; diff --git a/test/sql/copy/hive_types.test b/test/sql/copy/hive_types.test index 69eb055c8427..30411309a04a 100644 --- a/test/sql/copy/hive_types.test +++ b/test/sql/copy/hive_types.test @@ -6,8 +6,8 @@ require parquet -# statement ok -# PRAGMA enable_verification +statement ok +PRAGMA enable_verification statement ok copy 'data/csv/hive-partitioning/hive_types/himym.csv' to '__TEST_DIR__/partition' (format parquet, partition_by(season,director,aired)); @@ -86,6 +86,22 @@ select typeof(season),typeof(director),typeof(aired) from read_parquet('__TEST_D ---- BIGINT VARCHAR DATE +# hive types mix +statement ok +copy (select 1 AS a, 1 AS b, '123' AS partition UNION ALL SELECT 2, 2, '1992-01-01' UNION ALL SELECT 3, 3, 'abc') TO '__TEST_DIR__/partition_types' (FORMAT PARQUET, PARTITION_BY(partition)); + +query III +SELECT * FROM '__TEST_DIR__/partition_types/**/*.parquet' ORDER BY 1 +---- +1 1 123 +2 2 1992-01-01 +3 3 abc + +# explicit overwrite +statement error +select * from read_parquet('__TEST_DIR__/partition_types/**/*.parquet', hive_types={'partition':smallint}) +---- +Unable to cast # Complex filter filtering first file, filter should be pruned completely if hive_partitioning=1 query II From e70a2aa5fce4c9a55282e17033624ba8bb059aeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Tue, 16 Apr 2024 15:27:29 +0200 Subject: [PATCH 486/603] fixing merge conflict, thanks @pdet --- src/execution/operator/csv_scanner/util/csv_reader_options.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp index 6fa228033e44..e1beb5ef7d21 100644 --- a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp +++ b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp @@ -180,8 +180,6 @@ void CSVReaderOptions::SetReadOption(const string &loption, const Value &value, SetSkipRows(ParseInteger(value, loption)); } else if (loption == "max_line_size" || loption == "maximum_line_size") { maximum_line_size = NumericCast(ParseInteger(value, loption)); - } else if (loption == "force_not_null") { - force_not_null = ParseColumnList(value, expected_names, loption); } else if (loption == "date_format" || loption == "dateformat") { string format = ParseString(value, loption); SetDateFormat(LogicalTypeId::DATE, format, true); From a03012d078c610b017f8a78c2e830291066a629f Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Tue, 16 Apr 2024 16:20:19 +0200 Subject: [PATCH 487/603] Fix #11669: deduplicate column names in pivot correctly --- src/planner/binder/tableref/bind_pivot.cpp | 2 ++ test/sql/pivot/pivot_case_insensitive.test | 34 ++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 test/sql/pivot/pivot_case_insensitive.test diff --git a/src/planner/binder/tableref/bind_pivot.cpp b/src/planner/binder/tableref/bind_pivot.cpp index 6c89b7b2cd91..1b61629675ec 100644 --- a/src/planner/binder/tableref/bind_pivot.cpp +++ b/src/planner/binder/tableref/bind_pivot.cpp @@ -344,7 +344,9 @@ unique_ptr Binder::BindBoundPivot(PivotRef &ref) { result->bound_pivot.group_count = ref.bound_group_names.size(); result->bound_pivot.types = types; auto subquery_alias = ref.alias.empty() ? "__unnamed_pivot" : ref.alias; + QueryResult::DeduplicateColumns(names); bind_context.AddGenericBinding(result->bind_index, subquery_alias, names, types); + MoveCorrelatedExpressions(*result->child_binder); return std::move(result); } diff --git a/test/sql/pivot/pivot_case_insensitive.test b/test/sql/pivot/pivot_case_insensitive.test new file mode 100644 index 000000000000..d2c3be4af06b --- /dev/null +++ b/test/sql/pivot/pivot_case_insensitive.test @@ -0,0 +1,34 @@ +# name: test/sql/pivot/pivot_case_insensitive.test +# description: Test case insensitivity in pivot names +# group: [pivot] + +statement ok +PRAGMA enable_verification + +statement ok +CREATE TABLE Cities (Name VARCHAR, id INTEGER); +INSERT INTO Cities VALUES ('Test', 1); +INSERT INTO Cities VALUES ('test',2); + +statement ok +SET pivot_filter_threshold=1; + +query II +FROM Cities +PIVOT ( + array_agg(id) + FOR + name IN ('test','Test') +); +---- +[2] [1] + +query IIII +FROM Cities +PIVOT ( + array_agg(id), sum(id) + FOR + name IN ('test','Test') +); +---- +[2] 2 [1] 1 From 747d6e2ed03af71beaa2549c006362f9e782eede Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Tue, 16 Apr 2024 16:26:59 +0200 Subject: [PATCH 488/603] consider not null values when doing export --- .../operator/persistent/physical_export.cpp | 5 +++- .../parsed_data/exported_table_data.hpp | 7 +++-- src/planner/binder/statement/bind_export.cpp | 11 +++++-- test/sql/copy/csv/test_export_not_null.test | 30 +++++++++++++++++++ 4 files changed, 48 insertions(+), 5 deletions(-) create mode 100644 test/sql/copy/csv/test_export_not_null.test diff --git a/src/execution/operator/persistent/physical_export.cpp b/src/execution/operator/persistent/physical_export.cpp index 3979a88eeb59..abdb948ca4d6 100644 --- a/src/execution/operator/persistent/physical_export.cpp +++ b/src/execution/operator/persistent/physical_export.cpp @@ -45,7 +45,6 @@ static void WriteCopyStatement(FileSystem &fs, stringstream &ss, CopyInfo &info, auto file_path = StringUtil::Replace(exported_table.file_path, "\\", "/"); ss << StringUtil::Format("%s FROM %s (", SQLIdentifier(exported_table.table_name), SQLString(file_path)); - // write the copy options ss << "FORMAT '" << info.format << "'"; if (info.format == "csv") { @@ -60,6 +59,10 @@ static void WriteCopyStatement(FileSystem &fs, stringstream &ss, CopyInfo &info, if (info.options.find("quote") == info.options.end()) { info.options["quote"].push_back(Value("\"")); } + info.options.erase("force_not_null"); + for (auto ¬_null_column : exported_table.not_null_columns) { + info.options["force_not_null"].push_back(not_null_column); + } } for (auto ©_option : info.options) { if (copy_option.first == "force_quote") { diff --git a/src/include/duckdb/parser/parsed_data/exported_table_data.hpp b/src/include/duckdb/parser/parsed_data/exported_table_data.hpp index a5246b4e5e62..b3f26a9be6db 100644 --- a/src/include/duckdb/parser/parsed_data/exported_table_data.hpp +++ b/src/include/duckdb/parser/parsed_data/exported_table_data.hpp @@ -26,11 +26,14 @@ struct ExportedTableData { //! Path to be exported string file_path; + //! Not Null columns, if any + vector not_null_columns; }; struct ExportedTableInfo { - ExportedTableInfo(TableCatalogEntry &entry, ExportedTableData table_data) - : entry(entry), table_data(std::move(table_data)) { + ExportedTableInfo(TableCatalogEntry &entry, ExportedTableData table_data_p, vector ¬_null_columns_p) + : entry(entry), table_data(std::move(table_data_p)) { + table_data.not_null_columns = not_null_columns_p; } TableCatalogEntry &entry; diff --git a/src/planner/binder/statement/bind_export.cpp b/src/planner/binder/statement/bind_export.cpp index f5370a20a915..4403586697de 100644 --- a/src/planner/binder/statement/bind_export.cpp +++ b/src/planner/binder/statement/bind_export.cpp @@ -303,7 +303,14 @@ BoundStatement Binder::Bind(ExportStatement &stmt) { // We can not export generated columns child_list_t select_list; - + // Let's verify if any on these columns have not null constraints + vector not_null_columns; + for (auto &constaint : table.GetConstraints()) { + if (constaint->type == ConstraintType::NOT_NULL) { + auto ¬_null_constraint = constaint->Cast(); + not_null_columns.push_back(table.GetColumn(not_null_constraint.index).GetName()); + } + } for (auto &col : table.GetColumns().Physical()) { select_list.push_back(std::make_pair(col.Name(), col.Type())); } @@ -315,7 +322,7 @@ BoundStatement Binder::Bind(ExportStatement &stmt) { exported_data.file_path = info->file_path; - ExportedTableInfo table_info(table, std::move(exported_data)); + ExportedTableInfo table_info(table, std::move(exported_data), not_null_columns); exported_tables.data.push_back(table_info); id++; diff --git a/test/sql/copy/csv/test_export_not_null.test b/test/sql/copy/csv/test_export_not_null.test new file mode 100644 index 000000000000..3242c1654cd7 --- /dev/null +++ b/test/sql/copy/csv/test_export_not_null.test @@ -0,0 +1,30 @@ +# name: test/sql/copy/csv/test_export_not_null.test +# description: Test Export function that is not null +# group: [csv] + +statement ok +PRAGMA enable_verification + +statement ok +begin transaction; + +statement ok +create table tbl(a VARCHAR NOT NULL) + +statement ok +insert into tbl values (''); + +statement ok +EXPORT DATABASE '__TEST_DIR__/broken_empty_string'; + +statement ok +abort; + +statement ok +IMPORT DATABASE '__TEST_DIR__/broken_empty_string'; + +# Force not null can't be used explicitly +statement error +EXPORT DATABASE '__TEST_DIR__/broken_empty_string_2' (FORCE_NOT_NULL ['A']); +---- +Unrecognized option \ No newline at end of file From 884c08a927b5575e34354c17d8f672b8db79af96 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Tue, 16 Apr 2024 16:33:41 +0200 Subject: [PATCH 489/603] found where the bug is. dont know how to fix it though --- .../duckdb/optimizer/join_order/join_node.hpp | 1 + src/optimizer/join_order/join_node.cpp | 16 ++++++++++ src/optimizer/join_order/plan_enumerator.cpp | 17 +++++++++++ ..._the_join_node_hash_map_has_no_errors.test | 30 +++++++++++++++++++ 4 files changed, 64 insertions(+) create mode 100644 test/optimizer/joins/updating_the_join_node_hash_map_has_no_errors.test diff --git a/src/include/duckdb/optimizer/join_order/join_node.hpp b/src/include/duckdb/optimizer/join_order/join_node.hpp index 8f973c95487d..b4fe144e3f39 100644 --- a/src/include/duckdb/optimizer/join_order/join_node.hpp +++ b/src/include/duckdb/optimizer/join_order/join_node.hpp @@ -48,6 +48,7 @@ class JoinNode { public: void Print(); string ToString(); + void Verify(); }; } // namespace duckdb diff --git a/src/optimizer/join_order/join_node.cpp b/src/optimizer/join_order/join_node.cpp index ef8baa084bcf..70df8c98ba58 100644 --- a/src/optimizer/join_order/join_node.cpp +++ b/src/optimizer/join_order/join_node.cpp @@ -37,4 +37,20 @@ void JoinNode::Print() { Printer::Print(ToString()); } +void JoinNode::Verify() { +#ifdef DEBUG + D_ASSERT(set.count >= 1); + idx_t left_count = 0, right_count = 0; + if (left) { + left->Verify(); + left_count = left->set.count; + } + if (right) { + right->Verify(); + right_count = right->set.count; + } + D_ASSERT(set.count == left_count + right_count || set.count == 1); +#endif +} + } // namespace duckdb diff --git a/src/optimizer/join_order/plan_enumerator.cpp b/src/optimizer/join_order/plan_enumerator.cpp index a6efb84b8e6e..a9d6b9485efb 100644 --- a/src/optimizer/join_order/plan_enumerator.cpp +++ b/src/optimizer/join_order/plan_enumerator.cpp @@ -139,9 +139,12 @@ JoinNode &PlanEnumerator::EmitPair(JoinRelationSet &left, JoinRelationSet &right if (left_plan == plans.end() || right_plan == plans.end()) { throw InternalException("No left or right plan: internal error in join order optimizer"); } + left_plan->second->Verify(); + right_plan->second->Verify(); auto &new_set = query_graph_manager.set_manager.Union(left, right); // create the join tree based on combining the two plans auto new_plan = CreateJoinTree(new_set, info, *left_plan->second, *right_plan->second); + new_plan->Verify(); // check if this plan is the optimal plan we found for this set of relations auto entry = plans.find(new_set); auto new_cost = new_plan->cost; @@ -167,14 +170,28 @@ JoinNode &PlanEnumerator::EmitPair(JoinRelationSet &left, JoinRelationSet &right // nodes in the SolveExactly plan // If we know a node in the full plan is updated, we can prevent ourselves from exiting the // DP algorithm until the last plan updated is a full plan + result.Verify(); UpdateJoinNodesInFullPlan(result); if (must_update_full_plan) { must_update_full_plan = false; } } + if (new_set.ToString() == "[0, 1, 2, 5, 6, 9]") { + auto break_here = 0; + } D_ASSERT(new_plan); plans[new_set] = std::move(new_plan); + std::cout << "updating set " << new_set.ToString() << "with children " << left.ToString() << " and " << right.ToString() << std::endl; + if (new_set.ToString() == "[0, 2, 5, 6]") { + unordered_set bindings = {0, 1, 2, 5, 6, 9}; + JoinRelationSet &desired_set = query_graph_manager.set_manager.GetJoinRelation(bindings); + auto desired_set_plan = plans.find(desired_set); + if (desired_set_plan != plans.end()) { + desired_set_plan->second->Verify(); + std::cout << "verify ok? I don't think so" << std::endl; + } + } return result; } return *entry->second; diff --git a/test/optimizer/joins/updating_the_join_node_hash_map_has_no_errors.test b/test/optimizer/joins/updating_the_join_node_hash_map_has_no_errors.test new file mode 100644 index 000000000000..e61263b4f545 --- /dev/null +++ b/test/optimizer/joins/updating_the_join_node_hash_map_has_no_errors.test @@ -0,0 +1,30 @@ +# name: test/optimizer/joins/updating_the_join_node_hash_map_has_no_errors.test +# description: +# group: [joins] + +require tpch + + +statement ok +call dbgen(sf=0.05); + +statement ok +SELECT NULL +FROM main.supplier AS ref_0 +INNER JOIN main.nation +INNER JOIN main.nation AS ref_2 +INNER JOIN main.customer AS ref_3 +INNER JOIN main.supplier AS ref_4 ON ((ref_3.c_phone = ref_4.s_name)) ON ( + (SELECT NULL)) +INNER JOIN main.orders AS ref_5 +INNER JOIN main.orders AS ref_6 ON (ref_5.o_clerk) ON (1) ON (ref_3.c_mktsegment) ON ((ref_0.s_acctbal = ref_5.o_totalprice)) +INNER JOIN main.lineitem AS ref_7 ON ((ref_4.s_suppkey = ref_7.l_orderkey)) +INNER JOIN main.supplier +INNER JOIN main.supplier AS ref_11 +INNER JOIN main.lineitem AS ref_12 ON ( + (SELECT NULL)) ON (( + (SELECT ps_comment FROM main.partsupp) ~~* ref_12.l_linestatus)) ON + ((ref_7.l_linestatus ~~* (SELECT s_name FROM main.supplier))) +INNER JOIN + (SELECT NULL) ON (ref_6.o_orderstatus); + From f686803f0acc6a9898c5377d7a2de728c9b4f6fb Mon Sep 17 00:00:00 2001 From: Max Gabrielsson Date: Tue, 16 Apr 2024 18:22:12 +0200 Subject: [PATCH 490/603] bump vss --- .github/config/extensions.csv | 2 +- .github/config/out_of_tree_extensions.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/config/extensions.csv b/.github/config/extensions.csv index 50efd8809ff4..453ee59d3a80 100644 --- a/.github/config/extensions.csv +++ b/.github/config/extensions.csv @@ -15,4 +15,4 @@ aws,https://github.com/duckdb/duckdb_aws,f7b8729f1cce5ada5d4add70e1486de50763fb9 azure,https://github.com/duckdb/duckdb_azure,09623777a366572bfb8fa53e47acdf72133a360e, spatial,https://github.com/duckdb/duckdb_spatial,8ac803e986ccda34f32dee82a7faae95b72b3492, iceberg,https://github.com/duckdb/duckdb_iceberg,d89423c2ff90a0b98a093a133c8dfe2a55b9e092, -vss,https://github.com/duckdb/duckdb_vss,9038b50cefb8bfd6b8ab8a254d3c728f3f172d15, +vss,https://github.com/duckdb/duckdb_vss,8145f41d97178e82bed3376215eb8d02bcf1eec5, diff --git a/.github/config/out_of_tree_extensions.cmake b/.github/config/out_of_tree_extensions.cmake index a95bd411a18c..a1e7d240cac1 100644 --- a/.github/config/out_of_tree_extensions.cmake +++ b/.github/config/out_of_tree_extensions.cmake @@ -106,6 +106,6 @@ endif() duckdb_extension_load(vss LOAD_TESTS GIT_URL https://github.com/duckdb/duckdb_vss - GIT_TAG 9038b50cefb8bfd6b8ab8a254d3c728f3f172d15 + GIT_TAG 8145f41d97178e82bed3376215eb8d02bcf1eec5 TEST_DIR test/sql ) From 209461b83b3dce7c91a558c64f5cf39c31b49ad2 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Tue, 16 Apr 2024 19:52:44 +0200 Subject: [PATCH 491/603] make bill happy --- src/planner/binder/statement/bind_export.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/planner/binder/statement/bind_export.cpp b/src/planner/binder/statement/bind_export.cpp index 4403586697de..f44fd983f961 100644 --- a/src/planner/binder/statement/bind_export.cpp +++ b/src/planner/binder/statement/bind_export.cpp @@ -16,6 +16,7 @@ #include "duckdb/parser/query_node/select_node.hpp" #include "duckdb/common/numeric_utils.hpp" #include "duckdb/common/string_util.hpp" +#include "duckdb/parser/constraints/not_null_constraint.hpp" #include From e2bff3423aa0d3c8b2e07e00a31e9e777e3baadc Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Tue, 16 Apr 2024 19:55:44 +0200 Subject: [PATCH 492/603] Disable setting console pages by default, and add .utf8 setting --- tools/shell/shell.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tools/shell/shell.c b/tools/shell/shell.c index 90d975f8f2fc..a0afa3e2cdb4 100644 --- a/tools/shell/shell.c +++ b/tools/shell/shell.c @@ -462,16 +462,17 @@ static char continuePromptSelected[20]; /* Selected continuation prompt. default ** output from UTF-8 into MBCS. */ #if defined(_WIN32) || defined(WIN32) +static int win_utf8_mode = 0; + void utf8_printf(FILE *out, const char *zFormat, ...){ va_list ap; va_start(ap, zFormat); if( stdout_is_console && (out==stdout || out==stderr) ){ char *z1 = sqlite3_vmprintf(zFormat, ap); - if (SetConsoleOutputCP(CP_UTF8)) { + if (win_utf8_mode && SetConsoleOutputCP(CP_UTF8)) { // we can write UTF8 directly fputs(z1, out); } else { - // failed to set code page // fallback to writing old style windows unicode char *z2 = sqlite3_win32_utf8_to_mbcs_v2(z1, 0); fputs(z2, out); @@ -688,7 +689,7 @@ static char *local_getline(char *zLine, FILE *in){ #if defined(_WIN32) || defined(WIN32) int is_stdin = stdin_is_interactive && in==stdin; int is_utf8 = 0; - if (is_stdin) { + if (is_stdin && win_utf8_mode) { if (SetConsoleCP(CP_UTF8)) { is_utf8 = 1; } @@ -13616,6 +13617,9 @@ static const char *(azHelp[]) = { #endif ".width NUM1 NUM2 ... Set minimum column widths for columnar output", " Negative values right-justify", +#if defined(_WIN32) || defined(WIN32) + ".utf8 Enable experimental UTF-8 console output mode" +#endif }; /* @@ -19371,7 +19375,13 @@ static int do_meta_command(char *zLine, ShellState *p){ for(j=1; jcolWidth[j-1] = (int)integerValue(azArg[j]); } - } else { + } +#if defined(_WIN32) || defined(WIN32) + else if( c=='u' && strncmp(azArg[0], "utf8", n)==0 ){ + win_utf8_mode = 1; + } +#endif + else { #ifdef HAVE_LINENOISE const char *error = NULL; if (linenoiseParseOption((const char**) azArg, nArg, &error)) { From 2038d45d06d3205b8f3f967ee8404f527bcbbd5f Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Tue, 16 Apr 2024 20:00:45 +0200 Subject: [PATCH 493/603] Add missing include --- src/planner/binder/tableref/bind_pivot.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/planner/binder/tableref/bind_pivot.cpp b/src/planner/binder/tableref/bind_pivot.cpp index 1b61629675ec..34a9ae6a7737 100644 --- a/src/planner/binder/tableref/bind_pivot.cpp +++ b/src/planner/binder/tableref/bind_pivot.cpp @@ -19,6 +19,7 @@ #include "duckdb/planner/expression/bound_aggregate_expression.hpp" #include "duckdb/main/client_config.hpp" #include "duckdb/catalog/catalog_entry/aggregate_function_catalog_entry.hpp" +#include "duckdb/main/query_result.hpp" namespace duckdb { From 44a0e68ce4d6110e930591c356e03b6c7f48b827 Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 16 Apr 2024 20:53:08 +0200 Subject: [PATCH 494/603] use ExpressionExecutor, enables overriding behavior through the Catalog (such as ICU) --- src/function/table/copy_csv.cpp | 82 +++++++++++++++++-- .../duckdb/function/table/read_csv.hpp | 2 + 2 files changed, 76 insertions(+), 8 deletions(-) diff --git a/src/function/table/copy_csv.cpp b/src/function/table/copy_csv.cpp index 8d1d50b3558a..bf8ff78d1d2f 100644 --- a/src/function/table/copy_csv.cpp +++ b/src/function/table/copy_csv.cpp @@ -110,6 +110,65 @@ static unique_ptr WriteCSVBind(ClientContext &context, CopyFunctio } bind_data->Finalize(); + auto &options = csv_data->options; + auto &formats = options.write_date_format; + + bool has_dateformat = !formats[LogicalTypeId::DATE].Empty(); + bool has_timestampformat = !formats[LogicalTypeId::TIMESTAMP].Empty(); + + // Create a binder + auto binder = Binder::CreateBinder(context); + + // Create a Binding, used by the ExpressionBinder to turn our columns into BoundReferenceExpressions + auto &bind_context = binder->bind_context; + auto table_index = binder->GenerateTableIndex(); + bind_context.AddGenericBinding(table_index, "copy_csv", names, sql_types); + + // Create the ParsedExpressions (cast, strftime, etc..) + vector> unbound_expressions; + for (idx_t i = 0; i < sql_types.size(); i++) { + auto &type = sql_types[i]; + auto &name = names[i]; + + bool is_timestamp = type.id() == LogicalTypeId::TIMESTAMP || type.id() == LogicalTypeId::TIMESTAMP_TZ; + if (has_dateformat && type.id() == LogicalTypeId::DATE) { + // strftime(, 'format') + vector> children; + children.push_back(make_uniq(name)); + // TODO: set from user-provided format + children.push_back(make_uniq("%m/%d/%Y, %-I:%-M %p")); + auto func = make_uniq("strftime", std::move(children)); + unbound_expressions.push_back(std::move(expr)); + } else if (has_timestampformat && is_timestamp) { + // strftime(, 'format') + vector> children; + children.push_back(make_uniq(name)); + // TODO: set from user-provided format + children.push_back(make_uniq("%Y-%m-%dT%H:%M:%S.%fZ")); + auto func = make_uniq("strftime", std::move(children)); + unbound_expressions.push_back(std::move(expr)); + } else { + // CAST AS VARCHAR + auto column = make_uniq(name); + auto expr = make_uniq(LogicalType::VARCHAR, std::move(column)); + unbound_expressions.push_back(std::move(expr)); + } + } + + // Create an ExpressionBinder, bind the Expressions + vector> expressions; + ExpressionBinder expression_binder(*binder, context); + expression_binder.target_type = LogicalType::VARCHAR; + for (auto &expr : unbound_expressions) { + expressions.push_back(expression_binder.Bind(expr)); + } + + bind_data->cast_expressions = std::move(expressions); + + // Move these into the WriteCSVData + // In 'WriteCSVInitializeLocal' we'll create an ExpressionExecutor, fed our expressions + // In 'WriteCSVChunkInternal' we use this expression executor to convert our input columns to VARCHAR + bind_data->requires_quotes = make_unsafe_uniq_array(256); memset(bind_data->requires_quotes.get(), 0, sizeof(bool) * 256); bind_data->requires_quotes['\n'] = true; @@ -262,6 +321,14 @@ static void WriteQuotedString(WriteStream &writer, WriteCSVData &csv_data, const // Sink //===--------------------------------------------------------------------===// struct LocalWriteCSVData : public LocalFunctionData { +public: + LocalWriteCSVData(ClientContext &context, vector> &expressions) + : executor(context, expressions) { + } + +public: + //! Used to execute the expressions that transform input -> string + ExpressionExecutor executor; //! The thread-local buffer to write data into MemoryStream stream; //! A chunk with VARCHAR columns to cast intermediates into @@ -314,7 +381,7 @@ struct GlobalWriteCSVData : public GlobalFunctionData { static unique_ptr WriteCSVInitializeLocal(ExecutionContext &context, FunctionData &bind_data) { auto &csv_data = bind_data.Cast(); - auto local_data = make_uniq(); + auto local_data = make_uniq(context.client, csv_data.cast_expressions); // create the chunk with VARCHAR types vector types; @@ -362,6 +429,7 @@ static void WriteCSVChunkInternal(ClientContext &context, FunctionData &bind_dat MemoryStream &writer, DataChunk &input, bool &written_anything) { auto &csv_data = bind_data.Cast(); auto &options = csv_data.options; + auto &formats = options.write_date_format; // first cast the columns of the chunk to varchar cast_chunk.Reset(); @@ -370,17 +438,15 @@ static void WriteCSVChunkInternal(ClientContext &context, FunctionData &bind_dat if (csv_data.sql_types[col_idx].id() == LogicalTypeId::VARCHAR) { // VARCHAR, just reinterpret (cannot reference, because LogicalTypeId::VARCHAR is used by the JSON type too) cast_chunk.data[col_idx].Reinterpret(input.data[col_idx]); - } else if (!csv_data.options.write_date_format[LogicalTypeId::DATE].Empty() && - csv_data.sql_types[col_idx].id() == LogicalTypeId::DATE) { + } else if (!formats[LogicalTypeId::DATE].Empty() && csv_data.sql_types[col_idx].id() == LogicalTypeId::DATE) { // use the date format to cast the chunk - csv_data.options.write_date_format[LogicalTypeId::DATE].ConvertDateVector( - input.data[col_idx], cast_chunk.data[col_idx], input.size()); - } else if (!csv_data.options.write_date_format[LogicalTypeId::TIMESTAMP].Empty() && + formats[LogicalTypeId::DATE].ConvertDateVector(input.data[col_idx], cast_chunk.data[col_idx], input.size()); + } else if (!formats[LogicalTypeId::TIMESTAMP].Empty() && (csv_data.sql_types[col_idx].id() == LogicalTypeId::TIMESTAMP || csv_data.sql_types[col_idx].id() == LogicalTypeId::TIMESTAMP_TZ)) { // use the timestamp format to cast the chunk - csv_data.options.write_date_format[LogicalTypeId::TIMESTAMP].ConvertTimestampVector( - input.data[col_idx], cast_chunk.data[col_idx], input.size()); + formats[LogicalTypeId::TIMESTAMP].ConvertTimestampVector(input.data[col_idx], cast_chunk.data[col_idx], + input.size()); } else { // non varchar column, perform the cast VectorOperations::Cast(context, input.data[col_idx], cast_chunk.data[col_idx], input.size()); diff --git a/src/include/duckdb/function/table/read_csv.hpp b/src/include/duckdb/function/table/read_csv.hpp index aeb5050214fe..272fbbf68550 100644 --- a/src/include/duckdb/function/table/read_csv.hpp +++ b/src/include/duckdb/function/table/read_csv.hpp @@ -56,6 +56,8 @@ struct WriteCSVData : public BaseCSVData { idx_t flush_size = 4096ULL * 8ULL; //! For each byte whether or not the CSV file requires quotes when containing the byte unsafe_unique_array requires_quotes; + //! Expressions used to convert the input into strings + vector> cast_expressions; }; struct ColumnInfo { From b07a2f4b50d6ba9b3efa113f39fba1e2abdb4753 Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 16 Apr 2024 21:38:58 +0200 Subject: [PATCH 495/603] reworked execution --- src/execution/column_binding_resolver.cpp | 4 + src/function/table/copy_csv.cpp | 103 +++++++++--------- .../execution/column_binding_resolver.hpp | 2 + 3 files changed, 60 insertions(+), 49 deletions(-) diff --git a/src/execution/column_binding_resolver.cpp b/src/execution/column_binding_resolver.cpp index 568381503d78..ed3b2c8b2f95 100644 --- a/src/execution/column_binding_resolver.cpp +++ b/src/execution/column_binding_resolver.cpp @@ -15,6 +15,10 @@ namespace duckdb { ColumnBindingResolver::ColumnBindingResolver(bool verify_only) : verify_only(verify_only) { } +void ColumnBindingResolver::SetBindings(vector &&bindings_p) { + bindings = std::move(bindings_p); +} + void ColumnBindingResolver::VisitOperator(LogicalOperator &op) { switch (op.type) { case LogicalOperatorType::LOGICAL_ASOF_JOIN: diff --git a/src/function/table/copy_csv.cpp b/src/function/table/copy_csv.cpp index bf8ff78d1d2f..73adcd53c52f 100644 --- a/src/function/table/copy_csv.cpp +++ b/src/function/table/copy_csv.cpp @@ -12,6 +12,11 @@ #include "duckdb/function/scalar/string_functions.hpp" #include "duckdb/function/table/read_csv.hpp" #include "duckdb/parser/parsed_data/copy_info.hpp" +#include "duckdb/parser/expression/cast_expression.hpp" +#include "duckdb/parser/expression/function_expression.hpp" +#include "duckdb/parser/expression/columnref_expression.hpp" +#include "duckdb/execution/column_binding_resolver.hpp" +#include "duckdb/planner/operator/logical_dummy_scan.hpp" #include namespace duckdb { @@ -93,24 +98,10 @@ string TransformNewLine(string new_line) { ; } -static unique_ptr WriteCSVBind(ClientContext &context, CopyFunctionBindInput &input, - const vector &names, const vector &sql_types) { - auto bind_data = make_uniq(input.info.file_path, sql_types, names); - - // check all the options in the copy info - for (auto &option : input.info.options) { - auto loption = StringUtil::Lower(option.first); - auto &set = option.second; - bind_data->options.SetWriteOption(loption, ConvertVectorToValue(set)); - } - // verify the parsed options - if (bind_data->options.force_quote.empty()) { - // no FORCE_QUOTE specified: initialize to false - bind_data->options.force_quote.resize(names.size(), false); - } - bind_data->Finalize(); - - auto &options = csv_data->options; +static vector> CreateCastExpressions(WriteCSVData &bind_data, ClientContext &context, + const vector &names, + const vector &sql_types) { + auto &options = bind_data.options; auto &formats = options.write_date_format; bool has_dateformat = !formats[LogicalTypeId::DATE].Empty(); @@ -137,20 +128,20 @@ static unique_ptr WriteCSVBind(ClientContext &context, CopyFunctio children.push_back(make_uniq(name)); // TODO: set from user-provided format children.push_back(make_uniq("%m/%d/%Y, %-I:%-M %p")); - auto func = make_uniq("strftime", std::move(children)); - unbound_expressions.push_back(std::move(expr)); + auto func = make_uniq_base("strftime", std::move(children)); + unbound_expressions.push_back(std::move(func)); } else if (has_timestampformat && is_timestamp) { // strftime(, 'format') vector> children; children.push_back(make_uniq(name)); // TODO: set from user-provided format children.push_back(make_uniq("%Y-%m-%dT%H:%M:%S.%fZ")); - auto func = make_uniq("strftime", std::move(children)); - unbound_expressions.push_back(std::move(expr)); + auto func = make_uniq_base("strftime", std::move(children)); + unbound_expressions.push_back(std::move(func)); } else { // CAST AS VARCHAR auto column = make_uniq(name); - auto expr = make_uniq(LogicalType::VARCHAR, std::move(column)); + auto expr = make_uniq_base(LogicalType::VARCHAR, std::move(column)); unbound_expressions.push_back(std::move(expr)); } } @@ -163,11 +154,38 @@ static unique_ptr WriteCSVBind(ClientContext &context, CopyFunctio expressions.push_back(expression_binder.Bind(expr)); } - bind_data->cast_expressions = std::move(expressions); + ColumnBindingResolver resolver; + vector bindings; + for (idx_t i = 0; i < sql_types.size(); i++) { + bindings.push_back(ColumnBinding(table_index, i)); + } + resolver.SetBindings(std::move(bindings)); - // Move these into the WriteCSVData - // In 'WriteCSVInitializeLocal' we'll create an ExpressionExecutor, fed our expressions - // In 'WriteCSVChunkInternal' we use this expression executor to convert our input columns to VARCHAR + for (auto &expr : expressions) { + resolver.VisitExpression(&expr); + } + return expressions; +} + +static unique_ptr WriteCSVBind(ClientContext &context, CopyFunctionBindInput &input, + const vector &names, const vector &sql_types) { + auto bind_data = make_uniq(input.info.file_path, sql_types, names); + + // check all the options in the copy info + for (auto &option : input.info.options) { + auto loption = StringUtil::Lower(option.first); + auto &set = option.second; + bind_data->options.SetWriteOption(loption, ConvertVectorToValue(set)); + } + // verify the parsed options + if (bind_data->options.force_quote.empty()) { + // no FORCE_QUOTE specified: initialize to false + bind_data->options.force_quote.resize(names.size(), false); + } + bind_data->Finalize(); + + auto expressions = CreateCastExpressions(*bind_data, context, names, sql_types); + bind_data->cast_expressions = std::move(expressions); bind_data->requires_quotes = make_unsafe_uniq_array(256); memset(bind_data->requires_quotes.get(), 0, sizeof(bool) * 256); @@ -426,32 +444,16 @@ idx_t WriteCSVFileSize(GlobalFunctionData &gstate) { } static void WriteCSVChunkInternal(ClientContext &context, FunctionData &bind_data, DataChunk &cast_chunk, - MemoryStream &writer, DataChunk &input, bool &written_anything) { + MemoryStream &writer, DataChunk &input, bool &written_anything, + ExpressionExecutor &executor) { auto &csv_data = bind_data.Cast(); auto &options = csv_data.options; - auto &formats = options.write_date_format; // first cast the columns of the chunk to varchar cast_chunk.Reset(); cast_chunk.SetCardinality(input); - for (idx_t col_idx = 0; col_idx < input.ColumnCount(); col_idx++) { - if (csv_data.sql_types[col_idx].id() == LogicalTypeId::VARCHAR) { - // VARCHAR, just reinterpret (cannot reference, because LogicalTypeId::VARCHAR is used by the JSON type too) - cast_chunk.data[col_idx].Reinterpret(input.data[col_idx]); - } else if (!formats[LogicalTypeId::DATE].Empty() && csv_data.sql_types[col_idx].id() == LogicalTypeId::DATE) { - // use the date format to cast the chunk - formats[LogicalTypeId::DATE].ConvertDateVector(input.data[col_idx], cast_chunk.data[col_idx], input.size()); - } else if (!formats[LogicalTypeId::TIMESTAMP].Empty() && - (csv_data.sql_types[col_idx].id() == LogicalTypeId::TIMESTAMP || - csv_data.sql_types[col_idx].id() == LogicalTypeId::TIMESTAMP_TZ)) { - // use the timestamp format to cast the chunk - formats[LogicalTypeId::TIMESTAMP].ConvertTimestampVector(input.data[col_idx], cast_chunk.data[col_idx], - input.size()); - } else { - // non varchar column, perform the cast - VectorOperations::Cast(context, input.data[col_idx], cast_chunk.data[col_idx], input.size()); - } - } + + executor.Execute(input, cast_chunk); cast_chunk.Flatten(); // now loop over the vectors and output the values @@ -492,7 +494,7 @@ static void WriteCSVSink(ExecutionContext &context, FunctionData &bind_data, Glo // write data into the local buffer WriteCSVChunkInternal(context.client, bind_data, local_data.cast_chunk, local_data.stream, input, - local_data.written_anything); + local_data.written_anything, local_data.executor); // check if we should flush what we have currently written auto &writer = local_data.stream; @@ -570,11 +572,14 @@ unique_ptr WriteCSVPrepareBatch(ClientContext &context, Funct DataChunk cast_chunk; cast_chunk.Initialize(Allocator::Get(context), types); + auto expressions = CreateCastExpressions(csv_data, context, csv_data.options.name_list, types); + ExpressionExecutor executor(context, expressions); + // write CSV chunks to the batch data bool written_anything = false; auto batch = make_uniq(); for (auto &chunk : collection->Chunks()) { - WriteCSVChunkInternal(context, bind_data, cast_chunk, batch->stream, chunk, written_anything); + WriteCSVChunkInternal(context, bind_data, cast_chunk, batch->stream, chunk, written_anything, executor); } return std::move(batch); } diff --git a/src/include/duckdb/execution/column_binding_resolver.hpp b/src/include/duckdb/execution/column_binding_resolver.hpp index f98aeb2cfef4..de98ea2166a4 100644 --- a/src/include/duckdb/execution/column_binding_resolver.hpp +++ b/src/include/duckdb/execution/column_binding_resolver.hpp @@ -23,6 +23,8 @@ class ColumnBindingResolver : public LogicalOperatorVisitor { void VisitOperator(LogicalOperator &op) override; static void Verify(LogicalOperator &op); + //! Manually set bindings + void SetBindings(vector &&bindings); protected: vector bindings; From e35b725951212b4f71f39d5dfe6bd1126a1802b4 Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 16 Apr 2024 21:50:15 +0200 Subject: [PATCH 496/603] update hardcoded timestamp (placeholder) --- src/function/table/copy_csv.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/function/table/copy_csv.cpp b/src/function/table/copy_csv.cpp index 73adcd53c52f..32eb32231bfe 100644 --- a/src/function/table/copy_csv.cpp +++ b/src/function/table/copy_csv.cpp @@ -135,7 +135,7 @@ static vector> CreateCastExpressions(WriteCSVData &bind_d vector> children; children.push_back(make_uniq(name)); // TODO: set from user-provided format - children.push_back(make_uniq("%Y-%m-%dT%H:%M:%S.%fZ")); + children.push_back(make_uniq("%x %X.%g%z")); auto func = make_uniq_base("strftime", std::move(children)); unbound_expressions.push_back(std::move(func)); } else { From 6c643f7f4cad3cc91799a7e4ac57c541941df34f Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 16 Apr 2024 22:32:11 +0200 Subject: [PATCH 497/603] save Values for the format, fix an issue in the PrepareBatch --- .../operator/csv_scanner/util/csv_reader_options.cpp | 2 +- src/function/table/copy_csv.cpp | 11 ++++++----- .../operator/csv_scanner/csv_reader_options.hpp | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp index c4a24ffbc054..c39d7890d2ec 100644 --- a/src/execution/operator/csv_scanner/util/csv_reader_options.cpp +++ b/src/execution/operator/csv_scanner/util/csv_reader_options.cpp @@ -148,7 +148,7 @@ void CSVReaderOptions::SetDateFormat(LogicalTypeId type, const string &format, b error = StrTimeFormat::ParseFormatSpecifier(format, strpformat); dialect_options.date_format[type].Set(strpformat); } else { - error = StrTimeFormat::ParseFormatSpecifier(format, write_date_format[type]); + write_date_format[type] = Value(format); } if (!error.empty()) { throw InvalidInputException("Could not parse DATEFORMAT: %s", error.c_str()); diff --git a/src/function/table/copy_csv.cpp b/src/function/table/copy_csv.cpp index 32eb32231bfe..dd9567a0807f 100644 --- a/src/function/table/copy_csv.cpp +++ b/src/function/table/copy_csv.cpp @@ -104,8 +104,8 @@ static vector> CreateCastExpressions(WriteCSVData &bind_d auto &options = bind_data.options; auto &formats = options.write_date_format; - bool has_dateformat = !formats[LogicalTypeId::DATE].Empty(); - bool has_timestampformat = !formats[LogicalTypeId::TIMESTAMP].Empty(); + bool has_dateformat = !formats[LogicalTypeId::DATE].IsNull(); + bool has_timestampformat = !formats[LogicalTypeId::TIMESTAMP].IsNull(); // Create a binder auto binder = Binder::CreateBinder(context); @@ -127,7 +127,7 @@ static vector> CreateCastExpressions(WriteCSVData &bind_d vector> children; children.push_back(make_uniq(name)); // TODO: set from user-provided format - children.push_back(make_uniq("%m/%d/%Y, %-I:%-M %p")); + children.push_back(make_uniq(formats[LogicalTypeId::DATE])); auto func = make_uniq_base("strftime", std::move(children)); unbound_expressions.push_back(std::move(func)); } else if (has_timestampformat && is_timestamp) { @@ -135,7 +135,7 @@ static vector> CreateCastExpressions(WriteCSVData &bind_d vector> children; children.push_back(make_uniq(name)); // TODO: set from user-provided format - children.push_back(make_uniq("%x %X.%g%z")); + children.push_back(make_uniq(formats[LogicalTypeId::TIMESTAMP])); auto func = make_uniq_base("strftime", std::move(children)); unbound_expressions.push_back(std::move(func)); } else { @@ -572,7 +572,8 @@ unique_ptr WriteCSVPrepareBatch(ClientContext &context, Funct DataChunk cast_chunk; cast_chunk.Initialize(Allocator::Get(context), types); - auto expressions = CreateCastExpressions(csv_data, context, csv_data.options.name_list, types); + auto &original_types = collection->Types(); + auto expressions = CreateCastExpressions(csv_data, context, csv_data.options.name_list, original_types); ExpressionExecutor executor(context, expressions); // write CSV chunks to the batch data diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp index d929fb721aaf..65ae58f0e0e5 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp @@ -123,7 +123,7 @@ struct CSVReaderOptions { //! The date format to use (if any is specified) map date_format = {{LogicalTypeId::DATE, {}}, {LogicalTypeId::TIMESTAMP, {}}}; //! The date format to use for writing (if any is specified) - map write_date_format = {{LogicalTypeId::DATE, {}}, {LogicalTypeId::TIMESTAMP, {}}}; + map write_date_format = {{LogicalTypeId::DATE, Value()}, {LogicalTypeId::TIMESTAMP, Value()}}; //! Whether or not a type format is specified map has_format = {{LogicalTypeId::DATE, false}, {LogicalTypeId::TIMESTAMP, false}}; From 1b1d7f62358f90f1aa3d24c8bb12932427d18d00 Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 16 Apr 2024 22:36:16 +0200 Subject: [PATCH 498/603] add test for the newly enabled functionality, using a strftime overload added by ICU --- test/sql/copy/csv/test_csv_timestamp_tz.test | 1 - .../copy/csv/test_csv_timestamp_tz_icu.test | 24 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 test/sql/copy/csv/test_csv_timestamp_tz_icu.test diff --git a/test/sql/copy/csv/test_csv_timestamp_tz.test b/test/sql/copy/csv/test_csv_timestamp_tz.test index 6bc16ad0df7e..ca6bc3e6381a 100644 --- a/test/sql/copy/csv/test_csv_timestamp_tz.test +++ b/test/sql/copy/csv/test_csv_timestamp_tz.test @@ -15,4 +15,3 @@ query II select * from read_csv_auto('__TEST_DIR__/timestamps.csv'); ---- Tuesday Tuesday - diff --git a/test/sql/copy/csv/test_csv_timestamp_tz_icu.test b/test/sql/copy/csv/test_csv_timestamp_tz_icu.test new file mode 100644 index 000000000000..f24d9f799147 --- /dev/null +++ b/test/sql/copy/csv/test_csv_timestamp_tz_icu.test @@ -0,0 +1,24 @@ +# name: test/sql/copy/csv/test_csv_timestamp_tz_icu.test +# description: Test CSV with timestamp_tz and timestampformat +# group: [csv] + +statement ok +pragma enable_verification + +require icu + +statement ok +SET Calendar = 'gregorian'; + +statement ok +SET TimeZone = 'America/Los_Angeles'; + +statement ok +COPY ( + SELECT make_timestamptz(1713193669561000) AS t +) TO '__TEST_DIR__/timestamp-format.csv' (FORMAT CSV, timestampformat '%x %X.%g%z'); + +query I +select * from read_csv('__TEST_DIR__/timestamp-format.csv', all_varchar=true) +---- +2024-04-15 08:07:49.561-07 From 0528a0f4e3e922d39bc6af599002fc57ed9a0dc4 Mon Sep 17 00:00:00 2001 From: Max Gabrielsson Date: Tue, 16 Apr 2024 22:48:00 +0200 Subject: [PATCH 499/603] catch errors when vacuuming index, dont try to rever unknown indexes --- src/storage/data_table.cpp | 8 ++++++-- src/storage/local_storage.cpp | 6 +++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 60f16502b0bc..aeadf22cc77f 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -860,7 +860,9 @@ void DataTable::RevertAppend(idx_t start_row, idx_t count) { row_data[i] = current_row_base + i; } info->indexes.Scan([&](Index &index) { - index.Delete(chunk, row_identifiers); + if (!index.IsUnknown()) { + index.Delete(chunk, row_identifiers); + } return false; }); current_row_base += chunk.size(); @@ -870,7 +872,9 @@ void DataTable::RevertAppend(idx_t start_row, idx_t count) { // we need to vacuum the indexes to remove any buffers that are now empty // due to reverting the appends info->indexes.Scan([&](Index &index) { - index.Vacuum(); + if (!index.IsUnknown()) { + index.Vacuum(); + } return false; }); diff --git a/src/storage/local_storage.cpp b/src/storage/local_storage.cpp index ee249ce2d29f..b8482e2de327 100644 --- a/src/storage/local_storage.cpp +++ b/src/storage/local_storage.cpp @@ -197,7 +197,11 @@ void LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, TableAppen // we need to vacuum the indexes to remove any buffers that are now empty // due to reverting the appends table.info->indexes.Scan([&](Index &index) { - index.Vacuum(); + try { + index.Vacuum(); + } catch (std::exception &ex) { // LCOV_EXCL_START + error = ErrorData(ex); + } // LCOV_EXCL_STOP return false; }); error.Throw(); From 49099e877a4196fa2d45cb56079eda9232a055c1 Mon Sep 17 00:00:00 2001 From: Max Gabrielsson Date: Wed, 17 Apr 2024 00:36:53 +0200 Subject: [PATCH 500/603] handle updates, deletes, more index initialization --- src/execution/index/unknown_index.cpp | 26 +++++++++---------- .../duckdb/storage/table/data_table_info.hpp | 5 ++-- .../duckdb/storage/table/table_index_list.hpp | 5 ++-- src/storage/data_table.cpp | 9 +++++-- src/storage/table_index_list.cpp | 5 +++- src/transaction/undo_buffer.cpp | 4 ++- 6 files changed, 33 insertions(+), 21 deletions(-) diff --git a/src/execution/index/unknown_index.cpp b/src/execution/index/unknown_index.cpp index 09b14581abd3..e2b6a0cbd3c3 100644 --- a/src/execution/index/unknown_index.cpp +++ b/src/execution/index/unknown_index.cpp @@ -22,44 +22,44 @@ string UnknownIndex::GenerateErrorMessage() const { } ErrorData UnknownIndex::Append(IndexLock &, DataChunk &, Vector &) { - throw NotImplementedException(GenerateErrorMessage()); + throw MissingExtensionException(GenerateErrorMessage()); } void UnknownIndex::VerifyAppend(DataChunk &) { - throw NotImplementedException(GenerateErrorMessage()); + throw MissingExtensionException(GenerateErrorMessage()); } void UnknownIndex::VerifyAppend(DataChunk &, ConflictManager &) { - throw NotImplementedException(GenerateErrorMessage()); + throw MissingExtensionException(GenerateErrorMessage()); } void UnknownIndex::CommitDrop(IndexLock &) { - throw NotImplementedException(GenerateErrorMessage()); + throw MissingExtensionException(GenerateErrorMessage()); } void UnknownIndex::Delete(IndexLock &, DataChunk &, Vector &) { - throw NotImplementedException(GenerateErrorMessage()); + throw MissingExtensionException(GenerateErrorMessage()); } ErrorData UnknownIndex::Insert(IndexLock &, DataChunk &, Vector &) { - throw NotImplementedException(GenerateErrorMessage()); + throw MissingExtensionException(GenerateErrorMessage()); } IndexStorageInfo UnknownIndex::GetStorageInfo(bool) { - throw NotImplementedException(GenerateErrorMessage()); + throw MissingExtensionException(GenerateErrorMessage()); } bool UnknownIndex::MergeIndexes(IndexLock &, Index &) { - throw NotImplementedException(GenerateErrorMessage()); + throw MissingExtensionException(GenerateErrorMessage()); } void UnknownIndex::Vacuum(IndexLock &) { - throw NotImplementedException(GenerateErrorMessage()); + throw MissingExtensionException(GenerateErrorMessage()); } idx_t UnknownIndex::GetInMemorySize(IndexLock &) { - throw NotImplementedException(GenerateErrorMessage()); + throw MissingExtensionException(GenerateErrorMessage()); } void UnknownIndex::CheckConstraintsForChunk(DataChunk &, ConflictManager &) { - throw NotImplementedException(GenerateErrorMessage()); + throw MissingExtensionException(GenerateErrorMessage()); } string UnknownIndex::VerifyAndToString(IndexLock &, bool) { - throw NotImplementedException(GenerateErrorMessage()); + throw MissingExtensionException(GenerateErrorMessage()); } string UnknownIndex::GetConstraintViolationMessage(VerifyExistenceType, idx_t, DataChunk &) { - throw NotImplementedException(GenerateErrorMessage()); + throw MissingExtensionException(GenerateErrorMessage()); } } // namespace duckdb diff --git a/src/include/duckdb/storage/table/data_table_info.hpp b/src/include/duckdb/storage/table/data_table_info.hpp index 2785486b5da3..598d80e222b4 100644 --- a/src/include/duckdb/storage/table/data_table_info.hpp +++ b/src/include/duckdb/storage/table/data_table_info.hpp @@ -19,8 +19,9 @@ class TableIOManager; struct DataTableInfo { DataTableInfo(AttachedDatabase &db, shared_ptr table_io_manager_p, string schema, string table); - //! Initialize any unknown indexes whose types might now be present after an extension load - void InitializeIndexes(ClientContext &context); + //! Initialize any unknown indexes whose types might now be present after an extension load, optionally throwing an + //! exception if an index can't be initialized + void InitializeIndexes(ClientContext &context, bool throw_on_failure = false); //! The database instance of the table AttachedDatabase &db; diff --git a/src/include/duckdb/storage/table/table_index_list.hpp b/src/include/duckdb/storage/table/table_index_list.hpp index be15497ad9fe..397bf28243cd 100644 --- a/src/include/duckdb/storage/table/table_index_list.hpp +++ b/src/include/duckdb/storage/table/table_index_list.hpp @@ -41,8 +41,9 @@ class TableIndexList { void CommitDrop(const string &name); //! Returns true, if the index name does not exist bool NameIsUnique(const string &name); - //! Initializes unknown indexes that might now be present after an extension load - void InitializeIndexes(ClientContext &context, DataTableInfo &table_info); + //! Initializes unknown indexes that might now be present after an extension load, optionally throwing an exception + //! if a index cant be initialized + void InitializeIndexes(ClientContext &context, DataTableInfo &table_info, bool throw_on_failure = false); bool Empty(); idx_t Count(); void Move(TableIndexList &other); diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index aeadf22cc77f..2b09cfa60b9d 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -34,8 +34,8 @@ DataTableInfo::DataTableInfo(AttachedDatabase &db, shared_ptr ta table(std::move(table)) { } -void DataTableInfo::InitializeIndexes(ClientContext &context) { - indexes.InitializeIndexes(context, *this); +void DataTableInfo::InitializeIndexes(ClientContext &context, bool throw_on_failure) { + indexes.InitializeIndexes(context, *this, throw_on_failure); } bool DataTableInfo::IsTemporary() const { @@ -1006,6 +1006,8 @@ idx_t DataTable::Delete(TableCatalogEntry &table, ClientContext &context, Vector return 0; } + info->InitializeIndexes(context, true); + auto &transaction = DuckTransaction::Get(context, db); auto &local_storage = LocalStorage::Get(transaction); bool has_delete_constraints = TableHasDeleteConstraints(table); @@ -1163,6 +1165,9 @@ void DataTable::Update(TableCatalogEntry &table, ClientContext &context, Vector throw TransactionException("Transaction conflict: cannot update a table that has been altered!"); } + // check that there are no unknown indexes + info->InitializeIndexes(context, true); + // first verify that no constraints are violated VerifyUpdateConstraints(context, table, updates, column_ids); diff --git a/src/storage/table_index_list.cpp b/src/storage/table_index_list.cpp index ef14073f302b..b15a7aab10e3 100644 --- a/src/storage/table_index_list.cpp +++ b/src/storage/table_index_list.cpp @@ -55,7 +55,7 @@ bool TableIndexList::NameIsUnique(const string &name) { return true; } -void TableIndexList::InitializeIndexes(ClientContext &context, DataTableInfo &table_info) { +void TableIndexList::InitializeIndexes(ClientContext &context, DataTableInfo &table_info, bool throw_on_failure) { lock_guard lock(indexes_lock); for (auto &index : indexes) { if (!index->IsUnknown()) { @@ -68,6 +68,9 @@ void TableIndexList::InitializeIndexes(ClientContext &context, DataTableInfo &ta // Do we know the type of this index now? auto index_type = context.db->config.GetIndexTypes().FindByName(index_type_name); if (!index_type) { + if (throw_on_failure) { + throw MissingExtensionException("Cannot initialize index '%s', unknown index type '%s'. You probably need to load an extension.", unknown_index.name, index_type_name); + } continue; } diff --git a/src/transaction/undo_buffer.cpp b/src/transaction/undo_buffer.cpp index 148bd3d04913..02f044cc4af2 100644 --- a/src/transaction/undo_buffer.cpp +++ b/src/transaction/undo_buffer.cpp @@ -142,7 +142,9 @@ void UndoBuffer::Cleanup() { // possibly vacuum indexes for (const auto &table : state.indexed_tables) { table.second->info->indexes.Scan([&](Index &index) { - index.Vacuum(); + if(!index.IsUnknown()) { + index.Vacuum(); + } return false; }); } From b33810e72ba07ac0d9b175ec248dc76f470747e6 Mon Sep 17 00:00:00 2001 From: Max Gabrielsson Date: Wed, 17 Apr 2024 00:47:51 +0200 Subject: [PATCH 501/603] format --- src/storage/table_index_list.cpp | 4 +++- src/transaction/undo_buffer.cpp | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/storage/table_index_list.cpp b/src/storage/table_index_list.cpp index b15a7aab10e3..1e07ce6fff6f 100644 --- a/src/storage/table_index_list.cpp +++ b/src/storage/table_index_list.cpp @@ -69,7 +69,9 @@ void TableIndexList::InitializeIndexes(ClientContext &context, DataTableInfo &ta auto index_type = context.db->config.GetIndexTypes().FindByName(index_type_name); if (!index_type) { if (throw_on_failure) { - throw MissingExtensionException("Cannot initialize index '%s', unknown index type '%s'. You probably need to load an extension.", unknown_index.name, index_type_name); + throw MissingExtensionException( + "Cannot initialize index '%s', unknown index type '%s'. You probably need to load an extension.", + unknown_index.name, index_type_name); } continue; } diff --git a/src/transaction/undo_buffer.cpp b/src/transaction/undo_buffer.cpp index 02f044cc4af2..2e94aa7fef66 100644 --- a/src/transaction/undo_buffer.cpp +++ b/src/transaction/undo_buffer.cpp @@ -142,7 +142,7 @@ void UndoBuffer::Cleanup() { // possibly vacuum indexes for (const auto &table : state.indexed_tables) { table.second->info->indexes.Scan([&](Index &index) { - if(!index.IsUnknown()) { + if (!index.IsUnknown()) { index.Vacuum(); } return false; From a7f8ca2bf812a076a1203e1765ae89e38549c244 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 17 Apr 2024 06:08:35 +0200 Subject: [PATCH 502/603] CMake port from duckdb-wasm: Add thread setting and LINKED_LIBS option --- CMakeLists.txt | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5f199d3518cf..058197542d96 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -83,6 +83,13 @@ if (EXTENSION_STATIC_BUILD AND "${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") endif() option(DISABLE_UNITY "Disable unity builds." FALSE) +option(USE_WASM_THREADS "Should threads be used" FALSE) +if (${USE_WASM_THREADS}) + set(WASM_THREAD_FLAGS + -pthread + -sSHARED_MEMORY=1 + ) +endif() option(FORCE_COLORED_OUTPUT "Always produce ANSI-colored output (GNU/Clang only)." FALSE) @@ -810,10 +817,11 @@ function(build_loadable_extension_directory NAME OUTPUT_DIRECTORY EXTENSION_VERS COMMAND ${CMAKE_COMMAND} -E copy $ $.lib ) # Compile the library into the actual wasm file + string(TOUPPER ${NAME} EXTENSION_NAME_UPPERCASE) add_custom_command( TARGET ${TARGET_NAME} POST_BUILD - COMMAND emcc $.lib -o $ -sSIDE_MODULE=1 -O3 + COMMAND emcc $.lib -o $ -O3 -sSIDE_MODULE=2 -sEXPORTED_FUNCTIONS="_${NAME}_init" ${WASM_THREAD_FLAGS} ${DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_LINKED_LIBS} ) endif() add_custom_command( @@ -846,7 +854,7 @@ function(build_static_extension NAME PARAMETERS) endfunction() # Internal extension register function -function(register_extension NAME DONT_LINK DONT_BUILD LOAD_TESTS PATH INCLUDE_PATH TEST_PATH) +function(register_extension NAME DONT_LINK DONT_BUILD LOAD_TESTS PATH INCLUDE_PATH TEST_PATH LINKED_LIBS) string(TOLOWER ${NAME} EXTENSION_NAME_LOWERCASE) string(TOUPPER ${NAME} EXTENSION_NAME_UPPERCASE) @@ -868,6 +876,8 @@ function(register_extension NAME DONT_LINK DONT_BUILD LOAD_TESTS PATH INCLUDE_PA endif() endif() + set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_LINKED_LIBS "${LINKED_LIBS}" PARENT_SCOPE) + # Allows explicitly disabling extensions that may be specified in other configurations if (NOT ${DONT_BUILD} AND NOT ${EXTENSION_TESTS_ONLY}) set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_SHOULD_BUILD TRUE PARENT_SCOPE) @@ -893,7 +903,7 @@ function(register_extension NAME DONT_LINK DONT_BUILD LOAD_TESTS PATH INCLUDE_PA endfunction() # Downloads the external extension repo at the specified commit and calls register_extension -macro(register_external_extension NAME URL COMMIT DONT_LINK DONT_BUILD LOAD_TESTS PATH INCLUDE_PATH TEST_PATH APPLY_PATCHES SUBMODULES) +macro(register_external_extension NAME URL COMMIT DONT_LINK DONT_BUILD LOAD_TESTS PATH INCLUDE_PATH TEST_PATH APPLY_PATCHES LINKED_LIBS SUBMODULES) include(FetchContent) if (${APPLY_PATCHES}) set(PATCH_COMMAND python3 ${CMAKE_SOURCE_DIR}/scripts/apply_extension_patches.py ${CMAKE_SOURCE_DIR}/.github/patches/extensions/${NAME}/) @@ -921,6 +931,8 @@ macro(register_external_extension NAME URL COMMIT DONT_LINK DONT_BUILD LOAD_TEST endif() message(STATUS "Load extension '${NAME}' from ${URL} @ ${GIT_SHORT_COMMIT}") + + string(TOUPPER ${NAME} EXTENSION_NAME_UPPERCASE) set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_EXT_VERSION "${GIT_SHORT_COMMIT}" PARENT_SCOPE) if ("${INCLUDE_PATH}" STREQUAL "") @@ -935,13 +947,13 @@ macro(register_external_extension NAME URL COMMIT DONT_LINK DONT_BUILD LOAD_TEST set(TEST_FULL_PATH "${${NAME}_extension_fc_SOURCE_DIR}/${TEST_PATH}") endif() - register_extension(${NAME} ${DONT_LINK} ${DONT_BUILD} ${LOAD_TESTS} ${${NAME}_extension_fc_SOURCE_DIR}/${PATH} "${INCLUDE_FULL_PATH}" "${TEST_FULL_PATH}") + register_extension(${NAME} ${DONT_LINK} ${DONT_BUILD} ${LOAD_TESTS} ${${NAME}_extension_fc_SOURCE_DIR}/${PATH} "${INCLUDE_FULL_PATH}" "${TEST_FULL_PATH}" "${LINKED_LIBS}") endmacro() function(duckdb_extension_load NAME) # Parameter parsing set(options DONT_LINK DONT_BUILD LOAD_TESTS APPLY_PATCHES) - set(oneValueArgs SOURCE_DIR INCLUDE_DIR TEST_DIR GIT_URL GIT_TAG SUBMODULES EXTENSION_VERSION) + set(oneValueArgs SOURCE_DIR INCLUDE_DIR TEST_DIR GIT_URL GIT_TAG SUBMODULES EXTENSION_VERSION LINKED_LIBS) cmake_parse_arguments(duckdb_extension_load "${options}" "${oneValueArgs}" "" ${ARGN}) string(TOLOWER ${NAME} EXTENSION_NAME_LOWERCASE) @@ -960,12 +972,12 @@ function(duckdb_extension_load NAME) # Remote Git extension if (${duckdb_extension_load_DONT_BUILD}) - register_extension(${NAME} "${duckdb_extension_load_DONT_LINK}" "${duckdb_extension_load_DONT_BUILD}" "" "" "" "") + register_extension(${NAME} "${duckdb_extension_load_DONT_LINK}" "${duckdb_extension_load_DONT_BUILD}" "" "" "" "" "") elseif (NOT "${duckdb_extension_load_GIT_URL}" STREQUAL "") if ("${duckdb_extension_load_GIT_TAG}" STREQUAL "") error("Git URL specified but no valid GIT_TAG was found for ${NAME} extension") endif() - register_external_extension(${NAME} "${duckdb_extension_load_GIT_URL}" "${duckdb_extension_load_GIT_TAG}" "${duckdb_extension_load_DONT_LINK}" "${duckdb_extension_load_DONT_BUILD}" "${duckdb_extension_load_LOAD_TESTS}" "${duckdb_extension_load_SOURCE_DIR}" "${duckdb_extension_load_INCLUDE_DIR}" "${duckdb_extension_load_TEST_DIR}" "${duckdb_extension_load_APPLY_PATCHES}" "${duckdb_extension_load_SUBMODULES}") + register_external_extension(${NAME} "${duckdb_extension_load_GIT_URL}" "${duckdb_extension_load_GIT_TAG}" "${duckdb_extension_load_DONT_LINK}" "${duckdb_extension_load_DONT_BUILD}" "${duckdb_extension_load_LOAD_TESTS}" "${duckdb_extension_load_SOURCE_DIR}" "${duckdb_extension_load_INCLUDE_DIR}" "${duckdb_extension_load_TEST_DIR}" "${duckdb_extension_load_APPLY_PATCHES}" "${duckdb_extension_load_LINKED_LIBS}" "${duckdb_extension_load_SUBMODULES}") if (NOT "${duckdb_extension_load_EXTENSION_VERSION}" STREQUAL "") set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_EXT_VERSION "${duckdb_extension_load_EXTENSION_VERSION}" PARENT_SCOPE) endif() @@ -1003,7 +1015,7 @@ function(duckdb_extension_load NAME) set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_EXT_VERSION "" PARENT_SCOPE) endif() endif() - register_extension(${NAME} "${duckdb_extension_load_DONT_LINK}" "${duckdb_extension_load_DONT_BUILD}" "${duckdb_extension_load_LOAD_TESTS}" "${duckdb_extension_load_SOURCE_DIR}" "${INCLUDE_PATH_DEFAULT}" "${TEST_PATH_DEFAULT}") + register_extension(${NAME} "${duckdb_extension_load_DONT_LINK}" "${duckdb_extension_load_DONT_BUILD}" "${duckdb_extension_load_LOAD_TESTS}" "${duckdb_extension_load_SOURCE_DIR}" "${INCLUDE_PATH_DEFAULT}" "${TEST_PATH_DEFAULT}" "${duckdb_extension_load_LINKED_LIBS}") elseif(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/extension_external/${NAME}) # Local extension, default path message(STATUS "Load extension '${NAME}' from '${CMAKE_CURRENT_SOURCE_DIR}/extension_external'") @@ -1012,7 +1024,7 @@ function(duckdb_extension_load NAME) else() # Local extension, default path message(STATUS "Load extension '${NAME}' from '${CMAKE_CURRENT_SOURCE_DIR}/extensions'") - register_extension(${NAME} ${duckdb_extension_load_DONT_LINK} "${duckdb_extension_load_DONT_BUILD}" "${duckdb_extension_load_LOAD_TESTS}" "${CMAKE_CURRENT_SOURCE_DIR}/extension/${NAME}" "${CMAKE_CURRENT_SOURCE_DIR}/extension/${NAME}/include" "${CMAKE_CURRENT_SOURCE_DIR}/extension/${NAME}/test/sql") + register_extension(${NAME} ${duckdb_extension_load_DONT_LINK} "${duckdb_extension_load_DONT_BUILD}" "${duckdb_extension_load_LOAD_TESTS}" "${CMAKE_CURRENT_SOURCE_DIR}/extension/${NAME}" "${CMAKE_CURRENT_SOURCE_DIR}/extension/${NAME}/include" "${CMAKE_CURRENT_SOURCE_DIR}/extension/${NAME}/test/sql" "${duckdb_extension_load_LINKED_LIBS}") set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_EXT_VERSION "${DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_EXT_VERSION}" PARENT_SCOPE) endif() @@ -1024,6 +1036,7 @@ function(duckdb_extension_load NAME) set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_PATH ${DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_PATH} PARENT_SCOPE) set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_INCLUDE_PATH ${DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_INCLUDE_PATH} PARENT_SCOPE) set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_TEST_PATH ${DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_TEST_PATH} PARENT_SCOPE) + set(DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_LINKED_LIBS ${DUCKDB_EXTENSION_${EXTENSION_NAME_UPPERCASE}_LINKED_LIBS} PARENT_SCOPE) endfunction() if(${EXPORT_DLL_SYMBOLS}) From d4d56b877027e11f29afb5277f8c7af2c1c43197 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 17 Apr 2024 06:10:48 +0200 Subject: [PATCH 503/603] Port duckdb-wasm changes to Makefile --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index a6fd0dcd3ea0..911beb115bff 100644 --- a/Makefile +++ b/Makefile @@ -294,17 +294,17 @@ release: ${EXTENSION_CONFIG_STEP} wasm_mvp: ${EXTENSION_CONFIG_STEP} mkdir -p ./build/wasm_mvp && \ - emcmake cmake $(GENERATOR) ${COMMON_CMAKE_VARS} -DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -Bbuild/wasm_mvp -DCMAKE_CXX_FLAGS="-DDUCKDB_CUSTOM_PLATFORM=wasm_mvp" && \ + emcmake cmake $(GENERATOR) -DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -Bbuild/wasm_mvp -DCMAKE_CXX_FLAGS="-DDUCKDB_CUSTOM_PLATFORM=wasm_mvp" -DDUCKDB_EXPLICIT_PLATFORM="wasm_mvp" ${COMMON_CMAKE_VARS} ${TOOLCHAIN_FLAGS} && \ emmake make -j8 -Cbuild/wasm_mvp wasm_eh: ${EXTENSION_CONFIG_STEP} mkdir -p ./build/wasm_eh && \ - emcmake cmake $(GENERATOR) ${COMMON_CMAKE_VARS} -DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -Bbuild/wasm_eh -DCMAKE_CXX_FLAGS="-fwasm-exceptions -DWEBDB_FAST_EXCEPTIONS=1 -DDUCKDB_CUSTOM_PLATFORM=wasm_eh" && \ + emcmake cmake $(GENERATOR) -DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -Bbuild/wasm_eh -DCMAKE_CXX_FLAGS="-fwasm-exceptions -DWEBDB_FAST_EXCEPTIONS=1 -DDUCKDB_CUSTOM_PLATFORM=wasm_eh" -DDUCKDB_EXPLICIT_PLATFORM="wasm_eh" ${COMMON_CMAKE_VARS} ${TOOLCHAIN_FLAGS} && \ emmake make -j8 -Cbuild/wasm_eh wasm_threads: ${EXTENSION_CONFIG_STEP} mkdir -p ./build/wasm_threads && \ - emcmake cmake $(GENERATOR) ${COMMON_CMAKE_VARS} -DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -Bbuild/wasm_threads -DCMAKE_CXX_FLAGS="-fwasm-exceptions -DWEBDB_FAST_EXCEPTIONS=1 -DWITH_WASM_THREADS=1 -DWITH_WASM_SIMD=1 -DWITH_WASM_BULK_MEMORY=1 -DDUCKDB_CUSTOM_PLATFORM=wasm_threads" && \ + emcmake cmake $(GENERATOR) -DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -Bbuild/wasm_threads -DCMAKE_CXX_FLAGS="-fwasm-exceptions -DWEBDB_FAST_EXCEPTIONS=1 -DWITH_WASM_THREADS=1 -DWITH_WASM_SIMD=1 -DWITH_WASM_BULK_MEMORY=1 -DDUCKDB_CUSTOM_PLATFORM=wasm_threads -pthread" -DDUCKDB_EXPLICIT_PLATFORM="wasm_threads" ${COMMON_CMAKE_VARS} -DUSE_WASM_THREADS=1 -DCMAKE_C_FLAGS="-pthread" ${TOOLCHAIN_FLAGS} && \ emmake make -j8 -Cbuild/wasm_threads cldebug: ${EXTENSION_CONFIG_STEP} From cee3b7387d73b5fa737931a1069998c9ca8cf2e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Wed, 17 Apr 2024 09:42:46 +0200 Subject: [PATCH 504/603] re-fixing bitpacking --- src/storage/compression/bitpacking.cpp | 4 ++-- src/transaction/duck_transaction_manager.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/storage/compression/bitpacking.cpp b/src/storage/compression/bitpacking.cpp index b41f9779ca2f..d5d06dac8f45 100644 --- a/src/storage/compression/bitpacking.cpp +++ b/src/storage/compression/bitpacking.cpp @@ -516,9 +516,9 @@ struct BitpackingCompressState : public CompressionState { // Compact the segment by moving the metadata next to the data. - idx_t unaligned_offset = data_ptr - base_ptr; + idx_t unaligned_offset = NumericCast(data_ptr - base_ptr); idx_t metadata_offset = AlignValue(unaligned_offset); - idx_t metadata_size = base_ptr + Storage::BLOCK_SIZE - metadata_ptr; + idx_t metadata_size = NumericCast(base_ptr + Storage::BLOCK_SIZE - metadata_ptr); idx_t total_segment_size = metadata_offset + metadata_size; // Asserting things are still sane here diff --git a/src/transaction/duck_transaction_manager.cpp b/src/transaction/duck_transaction_manager.cpp index a3ec6d4a3d9f..f53ab599670f 100644 --- a/src/transaction/duck_transaction_manager.cpp +++ b/src/transaction/duck_transaction_manager.cpp @@ -311,7 +311,7 @@ void DuckTransactionManager::RemoveTransaction(DuckTransaction &transaction) noe if (i > 0) { // we garbage collected transactions: remove them from the list recently_committed_transactions.erase(recently_committed_transactions.begin(), - recently_committed_transactions.begin() + UnsafeNumericCast(i)); + recently_committed_transactions.begin() + static_cast(i)); } // check if we can free the memory of any old transactions i = active_transactions.empty() ? old_transactions.size() : 0; @@ -326,7 +326,7 @@ void DuckTransactionManager::RemoveTransaction(DuckTransaction &transaction) noe } if (i > 0) { // we garbage collected transactions: remove them from the list - old_transactions.erase(old_transactions.begin(), old_transactions.begin() + UnsafeNumericCast(i)); + old_transactions.erase(old_transactions.begin(), old_transactions.begin() + static_cast(i)); } } From 8964b151fd37313401dbcdb18f8a64a15fee1762 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Wed, 17 Apr 2024 09:57:56 +0200 Subject: [PATCH 505/603] missed on in column_data --- src/storage/table/column_data.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/table/column_data.cpp b/src/storage/table/column_data.cpp index 5347ec69c440..cc68b4625294 100644 --- a/src/storage/table/column_data.cpp +++ b/src/storage/table/column_data.cpp @@ -171,7 +171,7 @@ void ColumnData::FetchUpdateRow(TransactionData transaction, row_t row_id, Vecto if (!updates) { return; } - updates->FetchRow(transaction, row_id, result, result_idx); + updates->FetchRow(transaction, NumericCast(row_id), result, result_idx); } void ColumnData::UpdateInternal(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids, From defaaa1578aa45dacc2f86db1cbca30054e329f4 Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 17 Apr 2024 10:05:04 +0200 Subject: [PATCH 506/603] add a reset to make clang-tidy happy --- src/storage/table/row_group.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/storage/table/row_group.cpp b/src/storage/table/row_group.cpp index 24c05e50463d..dba0a13ae1d5 100644 --- a/src/storage/table/row_group.cpp +++ b/src/storage/table/row_group.cpp @@ -273,6 +273,7 @@ unique_ptr RowGroup::AlterType(RowGroupCollection &new_collection, con if (i == changed_idx) { // this is the altered column: use the new column row_group->columns.push_back(std::move(column_data)); + column_data.reset(); } else { // this column was not altered: use the data directly row_group->columns.push_back(cols[i]); From 8df6f0502c572b7b3ef177dceff12d1a2486555e Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 17 Apr 2024 10:30:20 +0200 Subject: [PATCH 507/603] add patch to azure --- .github/config/out_of_tree_extensions.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/config/out_of_tree_extensions.cmake b/.github/config/out_of_tree_extensions.cmake index 50bdda371c96..7b489f8c14ff 100644 --- a/.github/config/out_of_tree_extensions.cmake +++ b/.github/config/out_of_tree_extensions.cmake @@ -41,6 +41,7 @@ if (NOT MINGW) LOAD_TESTS GIT_URL https://github.com/duckdb/duckdb_azure GIT_TAG 09623777a366572bfb8fa53e47acdf72133a360e + APPLY_PATCHES ) endif() From adce37dd2004c0f3d96c75e91bb7da3723461e03 Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 17 Apr 2024 10:35:12 +0200 Subject: [PATCH 508/603] this test should error, no resolution for TIMESTAMP_TZ is available --- test/sql/copy/csv/test_csv_timestamp_tz.test | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/test/sql/copy/csv/test_csv_timestamp_tz.test b/test/sql/copy/csv/test_csv_timestamp_tz.test index ca6bc3e6381a..36728603706c 100644 --- a/test/sql/copy/csv/test_csv_timestamp_tz.test +++ b/test/sql/copy/csv/test_csv_timestamp_tz.test @@ -5,13 +5,9 @@ statement ok pragma enable_verification -statement ok +statement error copy ( select '2021-05-25 04:55:03.382494 UTC'::timestamp as ts, '2021-05-25 04:55:03.382494 UTC'::timestamptz as tstz ) to '__TEST_DIR__/timestamps.csv' ( timestampformat '%A'); - - -query II -select * from read_csv_auto('__TEST_DIR__/timestamps.csv'); ---- -Tuesday Tuesday +No function matches the given name and argument types From fec25c383143c213acca7f7c56eb7b48045dd4de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Wed, 17 Apr 2024 12:39:58 +0200 Subject: [PATCH 509/603] good to go? --- src/common/serializer/buffered_file_writer.cpp | 4 ++-- src/include/duckdb/common/vector.hpp | 6 +++++- src/storage/buffer/buffer_pool.cpp | 2 -- src/transaction/duck_transaction_manager.cpp | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/common/serializer/buffered_file_writer.cpp b/src/common/serializer/buffered_file_writer.cpp index 62d237e63ea3..be4f51fc3c7a 100644 --- a/src/common/serializer/buffered_file_writer.cpp +++ b/src/common/serializer/buffered_file_writer.cpp @@ -37,8 +37,8 @@ void BufferedFileWriter::WriteData(const_data_ptr_t buffer, idx_t write_size) { Flush(); // Flush buffer before writing every things else } idx_t remaining_to_write = write_size - to_copy; - fs.Write(*handle, const_cast(buffer + to_copy), - UnsafeNumericCast(remaining_to_write)); // NOLINT: wrong API in Write + fs.Write(*handle, const_cast(buffer + to_copy), // NOLINT: wrong API in Write + UnsafeNumericCast(remaining_to_write)); total_written += remaining_to_write; } else { // first copy anything we can from the buffer diff --git a/src/include/duckdb/common/vector.hpp b/src/include/duckdb/common/vector.hpp index c767b76bab3b..676adac20788 100644 --- a/src/include/duckdb/common/vector.hpp +++ b/src/include/duckdb/common/vector.hpp @@ -101,11 +101,15 @@ class vector : public std::vector> { // NOL return get(original::size() - 1); } + void unsafe_erase_at(idx_t idx) { // NOLINT: not using camelcase on purpose here + original::erase(original::begin() + static_cast(idx)); + } + void erase_at(idx_t idx) { // NOLINT: not using camelcase on purpose here if (MemorySafety::ENABLED && idx > original::size()) { throw InternalException("Can't remove offset %d from vector of size %d", idx, original::size()); } - original::erase(original::begin() + static_cast(idx)); + unsafe_erase_at(idx); } }; diff --git a/src/storage/buffer/buffer_pool.cpp b/src/storage/buffer/buffer_pool.cpp index a7741b268a4d..d94b87d20cfa 100644 --- a/src/storage/buffer/buffer_pool.cpp +++ b/src/storage/buffer/buffer_pool.cpp @@ -72,9 +72,7 @@ void BufferPool::UpdateUsedMemory(MemoryTag tag, int64_t size) { memory_usage_per_tag[uint8_t(tag)] -= UnsafeNumericCast(-size); } else { current_memory += UnsafeNumericCast(size); - ; memory_usage_per_tag[uint8_t(tag)] += UnsafeNumericCast(size); - ; } } diff --git a/src/transaction/duck_transaction_manager.cpp b/src/transaction/duck_transaction_manager.cpp index f53ab599670f..42e099cdf39e 100644 --- a/src/transaction/duck_transaction_manager.cpp +++ b/src/transaction/duck_transaction_manager.cpp @@ -277,7 +277,7 @@ void DuckTransactionManager::RemoveTransaction(DuckTransaction &transaction) noe } } // remove the transaction from the set of currently active transactions - active_transactions.erase_at(t_index); + active_transactions.unsafe_erase_at(t_index); // traverse the recently_committed transactions to see if we can remove any idx_t i = 0; for (; i < recently_committed_transactions.size(); i++) { From 835e114257d1098da19adec4e91f16dbb3daf93f Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 17 Apr 2024 12:48:24 +0200 Subject: [PATCH 510/603] missing constant expression --- src/function/table/copy_csv.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/function/table/copy_csv.cpp b/src/function/table/copy_csv.cpp index dd9567a0807f..c581b6443f56 100644 --- a/src/function/table/copy_csv.cpp +++ b/src/function/table/copy_csv.cpp @@ -15,6 +15,7 @@ #include "duckdb/parser/expression/cast_expression.hpp" #include "duckdb/parser/expression/function_expression.hpp" #include "duckdb/parser/expression/columnref_expression.hpp" +#include "duckdb/parser/expression/constant_expression.hpp" #include "duckdb/execution/column_binding_resolver.hpp" #include "duckdb/planner/operator/logical_dummy_scan.hpp" #include From d6399e7792a2ca1a5534559d34a9abfcb230354c Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 17 Apr 2024 13:03:24 +0200 Subject: [PATCH 511/603] always create a Copy of a SQLStatement when ALTERNATIVE_VERIFY is set --- src/main/client_context.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/client_context.cpp b/src/main/client_context.cpp index c3aa1917a66e..cd517fc91f78 100644 --- a/src/main/client_context.cpp +++ b/src/main/client_context.cpp @@ -762,6 +762,11 @@ void ClientContext::SetActiveResult(ClientContextLock &lock, BaseQueryResult &re unique_ptr ClientContext::PendingStatementOrPreparedStatementInternal( ClientContextLock &lock, const string &query, unique_ptr statement, shared_ptr &prepared, const PendingQueryParameters ¶meters) { +#ifdef DUCKDB_ALTERNATIVE_VERIFY + if (statement && statement->type != StatementType::LOGICAL_PLAN_STATEMENT) { + statement = statement->Copy(); + } +#endif // check if we are on AutoCommit. In this case we should start a transaction. if (statement && config.AnyVerification()) { // query verification is enabled From 13f80d49feeee88ecffe13c072d2d26bb5935ab3 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 17 Apr 2024 13:09:23 +0200 Subject: [PATCH 512/603] Also handle tables with multiple constraints --- .../operator/persistent/physical_export.cpp | 11 ++++++-- test/sql/copy/csv/test_export_not_null.test | 27 ++++++++++++++++++- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/execution/operator/persistent/physical_export.cpp b/src/execution/operator/persistent/physical_export.cpp index abdb948ca4d6..bd4d0252ce56 100644 --- a/src/execution/operator/persistent/physical_export.cpp +++ b/src/execution/operator/persistent/physical_export.cpp @@ -76,8 +76,15 @@ static void WriteCopyStatement(FileSystem &fs, stringstream &ss, CopyInfo &info, if (copy_option.second.size() == 1) { ss << copy_option.second[0].ToSQLString(); } else { - // FIXME handle multiple options - throw NotImplementedException("FIXME: serialize list of options"); + // For Lists + ss << "("; + for (idx_t i = 0; i < copy_option.second.size(); i++) { + ss << copy_option.second[i].ToSQLString(); + if (i != copy_option.second.size() - 1) { + ss << ", "; + } + } + ss << ")"; } } ss << ");" << '\n'; diff --git a/test/sql/copy/csv/test_export_not_null.test b/test/sql/copy/csv/test_export_not_null.test index 3242c1654cd7..3996ffb086d6 100644 --- a/test/sql/copy/csv/test_export_not_null.test +++ b/test/sql/copy/csv/test_export_not_null.test @@ -27,4 +27,29 @@ IMPORT DATABASE '__TEST_DIR__/broken_empty_string'; statement error EXPORT DATABASE '__TEST_DIR__/broken_empty_string_2' (FORCE_NOT_NULL ['A']); ---- -Unrecognized option \ No newline at end of file +Unrecognized option + +# Try Table with multiple null constraints + +statement ok +begin transaction; + +statement ok +create table tbl_2(a VARCHAR NOT NULL, b VARCHAR NOT NULL, c VARCHAR NOT NULL, d VARCHAR) + +statement ok +insert into tbl_2 values ('','','',''); + +statement ok +EXPORT DATABASE '__TEST_DIR__/multiple'; + +statement ok +abort; + +statement ok +IMPORT DATABASE '__TEST_DIR__/multiple'; + +query IIII +select * from tbl_2 +---- +(empty) (empty) (empty) NULL \ No newline at end of file From d8c083e21c1a1e7a13b5b9f32cac2dcbef053d05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Wed, 17 Apr 2024 14:45:53 +0200 Subject: [PATCH 513/603] void --- src/storage/compression/bitpacking.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/storage/compression/bitpacking.cpp b/src/storage/compression/bitpacking.cpp index d5d06dac8f45..cb10b9c92817 100644 --- a/src/storage/compression/bitpacking.cpp +++ b/src/storage/compression/bitpacking.cpp @@ -902,6 +902,7 @@ void BitpackingFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t r if (scan_state.current_group.mode == BitpackingMode::CONSTANT_DELTA) { T multiplier; auto cast = TryCast::Operation(scan_state.current_group_offset, multiplier); + (void)cast; D_ASSERT(cast); #ifdef DEBUG // overflow check From edd43a28c10f1b0c98cae54a5503abb04122f9c6 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 17 Apr 2024 15:15:21 +0200 Subject: [PATCH 514/603] Basics of removing Value() from sniffing --- .../scanner/string_value_scanner.cpp | 11 ---- .../csv_scanner/sniffer/header_detection.cpp | 8 ++- .../csv_scanner/sniffer/type_detection.cpp | 50 ++++++++++++------- .../operator/csv_scanner/csv_sniffer.hpp | 6 +-- .../csv_scanner/string_value_scanner.hpp | 2 - 5 files changed, 40 insertions(+), 37 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 9e6271c82818..506406b5b7ae 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -264,17 +264,6 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size chunk_col_id++; } -Value StringValueResult::GetValue(idx_t row_idx, idx_t col_idx) { - if (validity_mask[col_idx]->AllValid()) { - return Value(static_cast(vector_ptr[col_idx])[row_idx]); - } else { - if (validity_mask[col_idx]->RowIsValid(row_idx)) { - return Value(static_cast(vector_ptr[col_idx])[row_idx]); - } else { - return Value(); - } - } -} DataChunk &StringValueResult::ToChunk() { parse_chunk.SetCardinality(number_of_rows); return parse_chunk; diff --git a/src/execution/operator/csv_scanner/sniffer/header_detection.cpp b/src/execution/operator/csv_scanner/sniffer/header_detection.cpp index a52e2aeae884..ac3dd98e2c5f 100644 --- a/src/execution/operator/csv_scanner/sniffer/header_detection.cpp +++ b/src/execution/operator/csv_scanner/sniffer/header_detection.cpp @@ -1,6 +1,8 @@ #include "duckdb/common/types/cast_helpers.hpp" #include "duckdb/execution/operator/csv_scanner/csv_sniffer.hpp" #include "duckdb/execution/operator/csv_scanner/csv_reader_options.hpp" +#include "duckdb/common/types/value.hpp" + #include "utf8proc.hpp" namespace duckdb { @@ -123,7 +125,8 @@ bool CSVSniffer::DetectHeaderWithSetColumn() { const auto &sql_type = (*set_columns.types)[col]; if (sql_type != LogicalType::VARCHAR) { all_varchar = false; - if (!TryCastValue(options.dialect_options, options.decimal_separator, dummy_val, sql_type)) { + if (!TryCastValue(options.dialect_options, options.decimal_separator, StringValue::Get(dummy_val), + sql_type, dummy_val.IsNull())) { first_row_consistent = false; } } @@ -176,7 +179,8 @@ void CSVSniffer::DetectHeader() { if (sql_type != LogicalType::VARCHAR) { all_varchar = false; if (!TryCastValue(sniffer_state_machine.dialect_options, - sniffer_state_machine.options.decimal_separator, dummy_val, sql_type)) { + sniffer_state_machine.options.decimal_separator, StringValue::Get(dummy_val), + sql_type, dummy_val.IsNull())) { first_row_consistent = false; } } diff --git a/src/execution/operator/csv_scanner/sniffer/type_detection.cpp b/src/execution/operator/csv_scanner/sniffer/type_detection.cpp index 0572bb626c88..de39873deda6 100644 --- a/src/execution/operator/csv_scanner/sniffer/type_detection.cpp +++ b/src/execution/operator/csv_scanner/sniffer/type_detection.cpp @@ -85,8 +85,8 @@ string GenerateDateFormat(const string &separator, const char *format_template) } bool CSVSniffer::TryCastValue(const DialectOptions &dialect_options, const string &decimal_separator, - const Value &value, const LogicalType &sql_type) { - if (value.IsNull()) { + const string_t &value, const LogicalType &sql_type, bool is_null) { + if (is_null) { return true; } if (!dialect_options.date_format.find(LogicalTypeId::DATE)->second.GetValue().Empty() && @@ -95,7 +95,7 @@ bool CSVSniffer::TryCastValue(const DialectOptions &dialect_options, const strin string error_message; return dialect_options.date_format.find(LogicalTypeId::DATE) ->second.GetValue() - .TryParseDate(string_t(StringValue::Get(value)), result, error_message); + .TryParseDate(value, result, error_message); } if (!dialect_options.date_format.find(LogicalTypeId::TIMESTAMP)->second.GetValue().Empty() && sql_type.id() == LogicalTypeId::TIMESTAMP) { @@ -103,14 +103,15 @@ bool CSVSniffer::TryCastValue(const DialectOptions &dialect_options, const strin string error_message; return dialect_options.date_format.find(LogicalTypeId::TIMESTAMP) ->second.GetValue() - .TryParseTimestamp(string_t(StringValue::Get(value)), result, error_message); + .TryParseTimestamp(value, result, error_message); } if (decimal_separator != "." && (sql_type.id() == LogicalTypeId::DOUBLE)) { - return TryCastFloatingOperator::Operation(StringValue::Get(value)); + return TryCastFloatingOperator::Operation(value); } Value new_value; string error_message; - return value.TryCastAs(buffer_manager->context, sql_type, new_value, &error_message, true); + Value str_value(value); + return str_value.TryCastAs(buffer_manager->context, sql_type, new_value, &error_message, true); } void CSVSniffer::SetDateFormat(CSVStateMachine &candidate, const string &format_specifier, @@ -149,7 +150,7 @@ void CSVSniffer::InitializeDateAndTimeStampDetection(CSVStateMachine &candidate, } void CSVSniffer::DetectDateAndTimeStampFormats(CSVStateMachine &candidate, const LogicalType &sql_type, - const string &separator, Value &dummy_val) { + const string &separator, string_t &dummy_val) { // If it is the first time running date/timestamp detection we must initilize the format variables InitializeDateAndTimeStampDetection(candidate, separator, sql_type); // generate date format candidates the first time through @@ -210,9 +211,10 @@ void CSVSniffer::DetectTypes() { auto candidate = candidate_cc->UpgradeToStringValueScanner(); // Parse chunk and read csv with info candidate - auto &tuples = candidate->ParseChunk(); + auto &data_chunk = candidate->ParseChunk().ToChunk(); idx_t row_idx = 0; - if (tuples.number_of_rows > 1 && + idx_t chunk_size = data_chunk.size(); + if (chunk_size > 1 && (!options.dialect_options.header.IsSetByUser() || (options.dialect_options.header.IsSetByUser() && options.dialect_options.header.GetValue()))) { // This means we have more than one row, hence we can use the first row to detect if we have a header @@ -220,14 +222,19 @@ void CSVSniffer::DetectTypes() { } // First line where we start our type detection const idx_t start_idx_detection = row_idx; - for (; row_idx < tuples.number_of_rows; row_idx++) { - for (idx_t col_idx = 0; col_idx < tuples.number_of_columns; col_idx++) { - auto &col_type_candidates = info_sql_types_candidates[col_idx]; + + for (idx_t col_idx = 0; col_idx < data_chunk.ColumnCount(); col_idx++) { + auto &cur_vector = data_chunk.data[col_idx]; + D_ASSERT(cur_vector.GetVectorType() == VectorType::FLAT_VECTOR); + D_ASSERT(cur_vector.GetType() == LogicalType::VARCHAR); + auto vector_data = FlatVector::GetData(cur_vector); + auto null_mask = FlatVector::Validity(cur_vector); + auto &col_type_candidates = info_sql_types_candidates[col_idx]; + for (; row_idx < chunk_size; row_idx++) { // col_type_candidates can't be empty since anything in a CSV file should at least be a string // and we validate utf-8 compatibility when creating the type D_ASSERT(!col_type_candidates.empty()); auto cur_top_candidate = col_type_candidates.back(); - auto dummy_val = tuples.GetValue(row_idx, col_idx); // try cast from string to sql_type while (col_type_candidates.size() > 1) { const auto &sql_type = col_type_candidates.back(); @@ -236,10 +243,13 @@ void CSVSniffer::DetectTypes() { string separator; // If Value is not Null, Has a numeric date format, and the current investigated candidate is // either a timestamp or a date - if (!dummy_val.IsNull() && StartsWithNumericDate(separator, StringValue::Get(dummy_val)) && + // fixme: make this string_t + auto str_val = vector_data[row_idx].GetString(); + if (!null_mask.RowIsValid(row_idx) && StartsWithNumericDate(separator, str_val) && (col_type_candidates.back().id() == LogicalTypeId::TIMESTAMP || col_type_candidates.back().id() == LogicalTypeId::DATE)) { - DetectDateAndTimeStampFormats(candidate->GetStateMachine(), sql_type, separator, dummy_val); + DetectDateAndTimeStampFormats(candidate->GetStateMachine(), sql_type, separator, + vector_data[row_idx]); } // try cast from string to sql_type if (sql_type == LogicalType::VARCHAR) { @@ -247,7 +257,8 @@ void CSVSniffer::DetectTypes() { continue; } if (TryCastValue(sniffing_state_machine.dialect_options, - sniffing_state_machine.options.decimal_separator, dummy_val, sql_type)) { + sniffing_state_machine.options.decimal_separator, vector_data[row_idx], sql_type, + !null_mask.RowIsValid(row_idx))) { break; } else { if (row_idx != start_idx_detection && cur_top_candidate == LogicalType::BOOLEAN) { @@ -288,9 +299,10 @@ void CSVSniffer::DetectTypes() { for (auto &format_candidate : format_candidates) { best_format_candidates[format_candidate.first] = format_candidate.second.format; } - if (tuples.number_of_rows > 0) { - for (idx_t col_idx = 0; col_idx < tuples.number_of_columns; col_idx++) { - best_header_row.emplace_back(tuples.GetValue(0, col_idx)); + if (chunk_size > 0) { + for (idx_t col_idx = 0; col_idx < data_chunk.ColumnCount(); col_idx++) { + // fixme: make this string_t + best_header_row.emplace_back(data_chunk.GetValue(col_idx, 0)); } } } diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp index 3bb3c9bd5958..faad602faa8d 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp @@ -142,8 +142,8 @@ class CSVSniffer { void DetectTypes(); //! Change the date format for the type to the string //! Try to cast a string value to the specified sql type - bool TryCastValue(const DialectOptions &dialect_options, const string &decimal_separator, const Value &value, - const LogicalType &sql_type); + bool TryCastValue(const DialectOptions &dialect_options, const string &decimal_separator, const string_t &value, + const LogicalType &sql_type, bool is_null); void SetDateFormat(CSVStateMachine &candidate, const string &format_specifier, const LogicalTypeId &sql_type); //! Function that initialized the necessary variables used for date and timestamp detection @@ -151,7 +151,7 @@ class CSVSniffer { const LogicalType &sql_type); //! Functions that performs detection for date and timestamp formats void DetectDateAndTimeStampFormats(CSVStateMachine &candidate, const LogicalType &sql_type, const string &separator, - Value &dummy_val); + string_t &dummy_val); //! Variables for Type Detection //! Format Candidates for Date and Timestamp Types diff --git a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp index 9ad42fd16635..fe63c0a365fd 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp @@ -168,8 +168,6 @@ class StringValueResult : public ScannerResult { inline void AddValueToVector(const char *value_ptr, const idx_t size, bool allocate = false); - Value GetValue(idx_t row_idx, idx_t col_idx); - DataChunk &ToChunk(); //! Resets the state of the result void Reset(); From a8c752062096ff161f42fdae7cb6cbd2750d9a5b Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 17 Apr 2024 15:26:01 +0200 Subject: [PATCH 515/603] dont use a ColumnBindingResolver, create BoundReferenceExpressions directly --- src/execution/column_binding_resolver.cpp | 4 ---- src/function/table/copy_csv.cpp | 17 +++-------------- .../execution/column_binding_resolver.hpp | 2 -- 3 files changed, 3 insertions(+), 20 deletions(-) diff --git a/src/execution/column_binding_resolver.cpp b/src/execution/column_binding_resolver.cpp index ed3b2c8b2f95..568381503d78 100644 --- a/src/execution/column_binding_resolver.cpp +++ b/src/execution/column_binding_resolver.cpp @@ -15,10 +15,6 @@ namespace duckdb { ColumnBindingResolver::ColumnBindingResolver(bool verify_only) : verify_only(verify_only) { } -void ColumnBindingResolver::SetBindings(vector &&bindings_p) { - bindings = std::move(bindings_p); -} - void ColumnBindingResolver::VisitOperator(LogicalOperator &op) { switch (op.type) { case LogicalOperatorType::LOGICAL_ASOF_JOIN: diff --git a/src/function/table/copy_csv.cpp b/src/function/table/copy_csv.cpp index c581b6443f56..f8916f62d71b 100644 --- a/src/function/table/copy_csv.cpp +++ b/src/function/table/copy_csv.cpp @@ -111,7 +111,6 @@ static vector> CreateCastExpressions(WriteCSVData &bind_d // Create a binder auto binder = Binder::CreateBinder(context); - // Create a Binding, used by the ExpressionBinder to turn our columns into BoundReferenceExpressions auto &bind_context = binder->bind_context; auto table_index = binder->GenerateTableIndex(); bind_context.AddGenericBinding(table_index, "copy_csv", names, sql_types); @@ -126,7 +125,7 @@ static vector> CreateCastExpressions(WriteCSVData &bind_d if (has_dateformat && type.id() == LogicalTypeId::DATE) { // strftime(, 'format') vector> children; - children.push_back(make_uniq(name)); + children.push_back(make_uniq(make_uniq(name, type, i))); // TODO: set from user-provided format children.push_back(make_uniq(formats[LogicalTypeId::DATE])); auto func = make_uniq_base("strftime", std::move(children)); @@ -134,14 +133,14 @@ static vector> CreateCastExpressions(WriteCSVData &bind_d } else if (has_timestampformat && is_timestamp) { // strftime(, 'format') vector> children; - children.push_back(make_uniq(name)); + children.push_back(make_uniq(make_uniq(name, type, i))); // TODO: set from user-provided format children.push_back(make_uniq(formats[LogicalTypeId::TIMESTAMP])); auto func = make_uniq_base("strftime", std::move(children)); unbound_expressions.push_back(std::move(func)); } else { // CAST AS VARCHAR - auto column = make_uniq(name); + auto column = make_uniq(make_uniq(name, type, i)); auto expr = make_uniq_base(LogicalType::VARCHAR, std::move(column)); unbound_expressions.push_back(std::move(expr)); } @@ -155,16 +154,6 @@ static vector> CreateCastExpressions(WriteCSVData &bind_d expressions.push_back(expression_binder.Bind(expr)); } - ColumnBindingResolver resolver; - vector bindings; - for (idx_t i = 0; i < sql_types.size(); i++) { - bindings.push_back(ColumnBinding(table_index, i)); - } - resolver.SetBindings(std::move(bindings)); - - for (auto &expr : expressions) { - resolver.VisitExpression(&expr); - } return expressions; } diff --git a/src/include/duckdb/execution/column_binding_resolver.hpp b/src/include/duckdb/execution/column_binding_resolver.hpp index de98ea2166a4..f98aeb2cfef4 100644 --- a/src/include/duckdb/execution/column_binding_resolver.hpp +++ b/src/include/duckdb/execution/column_binding_resolver.hpp @@ -23,8 +23,6 @@ class ColumnBindingResolver : public LogicalOperatorVisitor { void VisitOperator(LogicalOperator &op) override; static void Verify(LogicalOperator &op); - //! Manually set bindings - void SetBindings(vector &&bindings); protected: vector bindings; From 23c3553c8103bf61da63aa891959dd11a9caa0ff Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 17 Apr 2024 15:26:56 +0200 Subject: [PATCH 516/603] remove resolved TODOs --- src/function/table/copy_csv.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/function/table/copy_csv.cpp b/src/function/table/copy_csv.cpp index f8916f62d71b..8f6999dd9262 100644 --- a/src/function/table/copy_csv.cpp +++ b/src/function/table/copy_csv.cpp @@ -126,7 +126,6 @@ static vector> CreateCastExpressions(WriteCSVData &bind_d // strftime(, 'format') vector> children; children.push_back(make_uniq(make_uniq(name, type, i))); - // TODO: set from user-provided format children.push_back(make_uniq(formats[LogicalTypeId::DATE])); auto func = make_uniq_base("strftime", std::move(children)); unbound_expressions.push_back(std::move(func)); @@ -134,7 +133,6 @@ static vector> CreateCastExpressions(WriteCSVData &bind_d // strftime(, 'format') vector> children; children.push_back(make_uniq(make_uniq(name, type, i))); - // TODO: set from user-provided format children.push_back(make_uniq(formats[LogicalTypeId::TIMESTAMP])); auto func = make_uniq_base("strftime", std::move(children)); unbound_expressions.push_back(std::move(func)); From 98372166d7ed18a362f4f1665881fa5517f9a784 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 17 Apr 2024 16:34:51 +0200 Subject: [PATCH 517/603] fix some date bug --- .../csv_scanner/sniffer/header_detection.cpp | 15 +++++++++++---- .../csv_scanner/sniffer/type_detection.cpp | 17 +++++++++++------ .../operator/csv_scanner/csv_sniffer.hpp | 1 + 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/execution/operator/csv_scanner/sniffer/header_detection.cpp b/src/execution/operator/csv_scanner/sniffer/header_detection.cpp index ac3dd98e2c5f..88975ad769c3 100644 --- a/src/execution/operator/csv_scanner/sniffer/header_detection.cpp +++ b/src/execution/operator/csv_scanner/sniffer/header_detection.cpp @@ -125,8 +125,12 @@ bool CSVSniffer::DetectHeaderWithSetColumn() { const auto &sql_type = (*set_columns.types)[col]; if (sql_type != LogicalType::VARCHAR) { all_varchar = false; - if (!TryCastValue(options.dialect_options, options.decimal_separator, StringValue::Get(dummy_val), - sql_type, dummy_val.IsNull())) { + string_t val; + if (!dummy_val.IsNull()) { + val = StringValue::Get(dummy_val); + } + if (!TryCastValue(options.dialect_options, options.decimal_separator, val, sql_type, + dummy_val.IsNull())) { first_row_consistent = false; } } @@ -178,9 +182,12 @@ void CSVSniffer::DetectHeader() { const auto &sql_type = best_sql_types_candidates_per_column_idx[col].back(); if (sql_type != LogicalType::VARCHAR) { all_varchar = false; + string_t val; + if (!dummy_val.IsNull()) { + val = StringValue::Get(dummy_val); + } if (!TryCastValue(sniffer_state_machine.dialect_options, - sniffer_state_machine.options.decimal_separator, StringValue::Get(dummy_val), - sql_type, dummy_val.IsNull())) { + sniffer_state_machine.options.decimal_separator, val, sql_type, dummy_val.IsNull())) { first_row_consistent = false; } } diff --git a/src/execution/operator/csv_scanner/sniffer/type_detection.cpp b/src/execution/operator/csv_scanner/sniffer/type_detection.cpp index de39873deda6..38f76645d276 100644 --- a/src/execution/operator/csv_scanner/sniffer/type_detection.cpp +++ b/src/execution/operator/csv_scanner/sniffer/type_detection.cpp @@ -143,6 +143,7 @@ void CSVSniffer::InitializeDateAndTimeStampDetection(CSVStateMachine &candidate, } } } + original_format_candidates = format_candidates; } // initialise the first candidate // all formats are constructed to be valid @@ -160,11 +161,11 @@ void CSVSniffer::DetectDateAndTimeStampFormats(CSVStateMachine &candidate, const auto save_format_candidates = type_format_candidates; bool had_format_candidates = !save_format_candidates.empty(); bool initial_format_candidates = - save_format_candidates.size() == format_template_candidates.at(sql_type.id()).size(); + save_format_candidates.size() == original_format_candidates.at(sql_type.id()).format.size(); while (!type_format_candidates.empty()) { // avoid using exceptions for flow control... auto ¤t_format = candidate.dialect_options.date_format[sql_type.id()].GetValue(); - if (current_format.Parse(StringValue::Get(dummy_val), result)) { + if (current_format.Parse(dummy_val, result)) { break; } // doesn't work - move to the next one @@ -182,10 +183,11 @@ void CSVSniffer::DetectDateAndTimeStampFormats(CSVStateMachine &candidate, const // we reset the whole thing because we tried to sniff the wrong type. format_candidates[sql_type.id()].initialized = false; format_candidates[sql_type.id()].format.clear(); + SetDateFormat(candidate, "", sql_type.id()); return; } type_format_candidates.swap(save_format_candidates); - SetDateFormat(candidate, "", sql_type.id()); + SetDateFormat(candidate, type_format_candidates.back(), sql_type.id()); } } } @@ -230,7 +232,7 @@ void CSVSniffer::DetectTypes() { auto vector_data = FlatVector::GetData(cur_vector); auto null_mask = FlatVector::Validity(cur_vector); auto &col_type_candidates = info_sql_types_candidates[col_idx]; - for (; row_idx < chunk_size; row_idx++) { + for (row_idx = start_idx_detection; row_idx < chunk_size; row_idx++) { // col_type_candidates can't be empty since anything in a CSV file should at least be a string // and we validate utf-8 compatibility when creating the type D_ASSERT(!col_type_candidates.empty()); @@ -244,8 +246,11 @@ void CSVSniffer::DetectTypes() { // If Value is not Null, Has a numeric date format, and the current investigated candidate is // either a timestamp or a date // fixme: make this string_t - auto str_val = vector_data[row_idx].GetString(); - if (!null_mask.RowIsValid(row_idx) && StartsWithNumericDate(separator, str_val) && + string str_val; + if (null_mask.RowIsValid(row_idx)) { + str_val = vector_data[row_idx].GetString(); + } + if (null_mask.RowIsValid(row_idx) && StartsWithNumericDate(separator, str_val) && (col_type_candidates.back().id() == LogicalTypeId::TIMESTAMP || col_type_candidates.back().id() == LogicalTypeId::DATE)) { DetectDateAndTimeStampFormats(candidate->GetStateMachine(), sql_type, separator, diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp index faad602faa8d..e1e2b0477471 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp @@ -167,6 +167,7 @@ class CSVSniffer { vector best_header_row; //! Variable used for sniffing date and timestamp map format_candidates; + map original_format_candidates; //! ------------------------------------------------------// //! ------------------ Type Refinement ------------------ // From 5672877caa103c39d32d89b4935a668e7f800cb9 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 17 Apr 2024 16:45:41 +0200 Subject: [PATCH 518/603] Date fix --- .../operator/csv_scanner/sniffer/type_detection.cpp | 5 +++-- .../duckdb/execution/operator/csv_scanner/csv_sniffer.hpp | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/execution/operator/csv_scanner/sniffer/type_detection.cpp b/src/execution/operator/csv_scanner/sniffer/type_detection.cpp index 38f76645d276..2b5c680d8a93 100644 --- a/src/execution/operator/csv_scanner/sniffer/type_detection.cpp +++ b/src/execution/operator/csv_scanner/sniffer/type_detection.cpp @@ -152,7 +152,7 @@ void CSVSniffer::InitializeDateAndTimeStampDetection(CSVStateMachine &candidate, void CSVSniffer::DetectDateAndTimeStampFormats(CSVStateMachine &candidate, const LogicalType &sql_type, const string &separator, string_t &dummy_val) { - // If it is the first time running date/timestamp detection we must initilize the format variables + // If it is the first time running date/timestamp detection we must initialize the format variables InitializeDateAndTimeStampDetection(candidate, separator, sql_type); // generate date format candidates the first time through auto &type_format_candidates = format_candidates[sql_type.id()].format; @@ -166,6 +166,7 @@ void CSVSniffer::DetectDateAndTimeStampFormats(CSVStateMachine &candidate, const // avoid using exceptions for flow control... auto ¤t_format = candidate.dialect_options.date_format[sql_type.id()].GetValue(); if (current_format.Parse(dummy_val, result)) { + format_candidates[sql_type.id()].had_match = true; break; } // doesn't work - move to the next one @@ -179,7 +180,7 @@ void CSVSniffer::DetectDateAndTimeStampFormats(CSVStateMachine &candidate, const // so restore the candidates that did work. // or throw them out if they were generated by this value. if (had_format_candidates) { - if (initial_format_candidates) { + if (initial_format_candidates && !format_candidates[sql_type.id()].had_match) { // we reset the whole thing because we tried to sniff the wrong type. format_candidates[sql_type.id()].initialized = false; format_candidates[sql_type.id()].format.clear(); diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp index e1e2b0477471..5e497e8dd4b1 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp @@ -16,6 +16,7 @@ namespace duckdb { struct DateTimestampSniffing { bool initialized = false; + bool had_match = false; vector format; }; //! Struct to store the result of the Sniffer From 1f829be0640d9ec086a67185cf521d93a91a6890 Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Wed, 17 Apr 2024 17:10:52 +0200 Subject: [PATCH 519/603] have a fix, but need to clean it up --- .../optimizer/join_order/cost_model.hpp | 2 +- .../duckdb/optimizer/join_order/join_node.hpp | 35 ++++- .../optimizer/join_order/plan_enumerator.hpp | 11 +- .../join_order/cardinality_estimator.cpp | 4 +- src/optimizer/join_order/cost_model.cpp | 2 +- src/optimizer/join_order/join_node.cpp | 11 +- src/optimizer/join_order/plan_enumerator.cpp | 139 ++++++++++-------- .../joins/update_nodes_in_full_path.test | 42 ++++++ ..._the_join_node_hash_map_has_no_errors.test | 4 +- 9 files changed, 170 insertions(+), 80 deletions(-) create mode 100644 test/optimizer/joins/update_nodes_in_full_path.test diff --git a/src/include/duckdb/optimizer/join_order/cost_model.hpp b/src/include/duckdb/optimizer/join_order/cost_model.hpp index 49895406afe7..446a127e19e5 100644 --- a/src/include/duckdb/optimizer/join_order/cost_model.hpp +++ b/src/include/duckdb/optimizer/join_order/cost_model.hpp @@ -26,7 +26,7 @@ class CostModel { void InitCostModel(); //! Compute cost of a join relation set - double ComputeCost(JoinNode &left, JoinNode &right); + double ComputeCost(DPJoinNode &left, DPJoinNode &right); //! Cardinality Estimator used to calculate cost CardinalityEstimator cardinality_estimator; diff --git a/src/include/duckdb/optimizer/join_order/join_node.hpp b/src/include/duckdb/optimizer/join_order/join_node.hpp index b4fe144e3f39..98e0553529ca 100644 --- a/src/include/duckdb/optimizer/join_order/join_node.hpp +++ b/src/include/duckdb/optimizer/join_order/join_node.hpp @@ -14,6 +14,35 @@ namespace duckdb { struct NeighborInfo; +class DPJoinNode { +public: + //! Represents a node in the join plan + JoinRelationSet &set; + //! information on how left and right are connected + optional_ptr info; + bool is_leaf; + //! left and right plans + JoinRelationSet &left_set; + JoinRelationSet &right_set; + + //! The cost of the join node. The cost is stored here so that the cost of + //! a join node stays in sync with how the join node is constructed. Storing the cost in an unordered_set + //! in the cost model is error prone. If the plan enumerator join node is updated and not the cost model + //! the whole Join Order Optimizer can start exhibiting undesired behavior. + double cost; + //! used only to populate logical operators with estimated caridnalities after the best join plan has been found. + idx_t cardinality; + + //! Create an intermediate node in the join tree. base_cardinality = estimated_props.cardinality + DPJoinNode(JoinRelationSet &set, optional_ptr info, JoinRelationSet &left, JoinRelationSet &right, double cost); + + //! Create a leaf node in the join tree + //! set cost to 0 for leaf nodes + //! cost will be the cost to *produce* an intermediate table + explicit DPJoinNode(JoinRelationSet &set); +}; + + class JoinNode { public: //! Represents a node in the join plan @@ -21,8 +50,8 @@ class JoinNode { //! information on how left and right are connected optional_ptr info; //! left and right plans - optional_ptr left; - optional_ptr right; + unique_ptr left; + unique_ptr right; //! The cost of the join node. The cost is stored here so that the cost of //! a join node stays in sync with how the join node is constructed. Storing the cost in an unordered_set @@ -33,7 +62,7 @@ class JoinNode { idx_t cardinality; //! Create an intermediate node in the join tree. base_cardinality = estimated_props.cardinality - JoinNode(JoinRelationSet &set, optional_ptr info, JoinNode &left, JoinNode &right, double cost); + JoinNode(JoinRelationSet &set, optional_ptr info, unique_ptr left, unique_ptr right, double cost); //! Create a leaf node in the join tree //! set cost to 0 for leaf nodes diff --git a/src/include/duckdb/optimizer/join_order/plan_enumerator.hpp b/src/include/duckdb/optimizer/join_order/plan_enumerator.hpp index 19d6a40fe68f..754d4c26d395 100644 --- a/src/include/duckdb/optimizer/join_order/plan_enumerator.hpp +++ b/src/include/duckdb/optimizer/join_order/plan_enumerator.hpp @@ -49,23 +49,24 @@ class PlanEnumerator { //! Cost model to evaluate cost of joins CostModel &cost_model; //! A map to store the optimal join plan found for a specific JoinRelationSet* - reference_map_t> plans; + reference_map_t> plans; bool full_plan_found; bool must_update_full_plan; unordered_set join_nodes_in_full_plan; - unique_ptr CreateJoinTree(JoinRelationSet &set, - const vector> &possible_connections, JoinNode &left, - JoinNode &right); + unique_ptr CreateJoinTree(JoinRelationSet &set, + const vector> &possible_connections, DPJoinNode &left, + DPJoinNode &right); //! Emit a pair as a potential join candidate. Returns the best plan found for the (left, right) connection (either //! the newly created plan, or an existing plan) - JoinNode &EmitPair(JoinRelationSet &left, JoinRelationSet &right, const vector> &info); + unique_ptr EmitPair(JoinRelationSet &left, JoinRelationSet &right, const vector> &info); //! Tries to emit a potential join candidate pair. Returns false if too many pairs have already been emitted, //! cancelling the dynamic programming step. bool TryEmitPair(JoinRelationSet &left, JoinRelationSet &right, const vector> &info); + unique_ptr CreateJoinNodeFromDPJoinNode(DPJoinNode dp_node); bool EnumerateCmpRecursive(JoinRelationSet &left, JoinRelationSet &right, unordered_set &exclusion_set); //! Emit a relation set node bool EmitCSG(JoinRelationSet &node); diff --git a/src/optimizer/join_order/cardinality_estimator.cpp b/src/optimizer/join_order/cardinality_estimator.cpp index 8cf884e3c854..fc55343a5df0 100644 --- a/src/optimizer/join_order/cardinality_estimator.cpp +++ b/src/optimizer/join_order/cardinality_estimator.cpp @@ -246,7 +246,7 @@ double CardinalityEstimator::EstimateCardinalityWithSet(JoinRelationSet &new_set denom *= match.denom; } // can happen if a table has cardinality 0, or a tdom is set to 0 - if (denom == 0) { + if (denom <= 1) { denom = 1; } auto result = numerator / denom; @@ -259,7 +259,7 @@ template <> idx_t CardinalityEstimator::EstimateCardinalityWithSet(JoinRelationSet &new_set) { auto cardinality_as_double = EstimateCardinalityWithSet(new_set); auto max = NumericLimits::Maximum(); - if (cardinality_as_double > max) { + if (cardinality_as_double >= max) { return max; } return (idx_t)cardinality_as_double; diff --git a/src/optimizer/join_order/cost_model.cpp b/src/optimizer/join_order/cost_model.cpp index a0ceeb72b287..bfe64412f053 100644 --- a/src/optimizer/join_order/cost_model.cpp +++ b/src/optimizer/join_order/cost_model.cpp @@ -8,7 +8,7 @@ CostModel::CostModel(QueryGraphManager &query_graph_manager) : query_graph_manager(query_graph_manager), cardinality_estimator() { } -double CostModel::ComputeCost(JoinNode &left, JoinNode &right) { +double CostModel::ComputeCost(DPJoinNode &left, DPJoinNode &right) { auto &combination = query_graph_manager.set_manager.Union(left.set, right.set); auto join_card = cardinality_estimator.EstimateCardinalityWithSet(combination); auto join_cost = join_card; diff --git a/src/optimizer/join_order/join_node.cpp b/src/optimizer/join_order/join_node.cpp index 70df8c98ba58..767d916948ae 100644 --- a/src/optimizer/join_order/join_node.cpp +++ b/src/optimizer/join_order/join_node.cpp @@ -9,8 +9,8 @@ namespace duckdb { JoinNode::JoinNode(JoinRelationSet &set) : set(set) { } -JoinNode::JoinNode(JoinRelationSet &set, optional_ptr info, JoinNode &left, JoinNode &right, double cost) - : set(set), info(info), left(&left), right(&right), cost(cost) { +JoinNode::JoinNode(JoinRelationSet &set, optional_ptr info, unique_ptr left, unique_ptr right, double cost) + : set(set), info(info), left(std::move(left)), right(std::move(right)), cost(cost) { } unique_ptr EstimatedProperties::Copy() { @@ -53,4 +53,11 @@ void JoinNode::Verify() { #endif } +DPJoinNode::DPJoinNode(JoinRelationSet &set) : set(set), info(nullptr), is_leaf(true), left_set(set), right_set(set) { +} + +DPJoinNode::DPJoinNode(JoinRelationSet &set, optional_ptr info, JoinRelationSet &left, JoinRelationSet &right, double cost) + : set(set), info(info), is_leaf(false), left_set(left), right_set(right), cost(cost) { +} + } // namespace duckdb diff --git a/src/optimizer/join_order/plan_enumerator.cpp b/src/optimizer/join_order/plan_enumerator.cpp index a9d6b9485efb..9c385710c899 100644 --- a/src/optimizer/join_order/plan_enumerator.cpp +++ b/src/optimizer/join_order/plan_enumerator.cpp @@ -110,10 +110,29 @@ void PlanEnumerator::GenerateCrossProducts() { // query_graph = query_graph_manager.GetQueryGraph(); } +unique_ptr PlanEnumerator::CreateJoinNodeFromDPJoinNode(DPJoinNode dp_node) { + if (dp_node.is_leaf) { + auto res = make_uniq(dp_node.set); + res->cardinality = dp_node.cardinality; + return res; + } + else { + auto left_DPJoinNode = plans.find(dp_node.left_set); + auto right_DPJoinNode = plans.find(dp_node.right_set); + D_ASSERT(left_DPJoinNode->second); + D_ASSERT(right_DPJoinNode->second); + auto left = CreateJoinNodeFromDPJoinNode(*left_DPJoinNode->second); + auto right = CreateJoinNodeFromDPJoinNode(*right_DPJoinNode->second); + auto res = make_uniq(dp_node.set, dp_node.info, std::move(left), std::move(right), dp_node.cost); + res->cardinality = dp_node.cardinality; + return res; + } +} + //! Create a new JoinTree node by joining together two previous JoinTree nodes -unique_ptr PlanEnumerator::CreateJoinTree(JoinRelationSet &set, +unique_ptr PlanEnumerator::CreateJoinTree(JoinRelationSet &set, const vector> &possible_connections, - JoinNode &left, JoinNode &right) { + DPJoinNode &left, DPJoinNode &right) { // for the hash join we want the right side (build side) to have the smallest cardinality // also just a heuristic but for now... // FIXME: we should probably actually benchmark that as well @@ -126,12 +145,12 @@ unique_ptr PlanEnumerator::CreateJoinTree(JoinRelationSet &set, } auto cost = cost_model.ComputeCost(left, right); - auto result = make_uniq(set, best_connection, left, right, cost); + auto result = make_uniq(set, best_connection, left.set, right.set, cost); result->cardinality = cost_model.cardinality_estimator.EstimateCardinalityWithSet(set); return result; } -JoinNode &PlanEnumerator::EmitPair(JoinRelationSet &left, JoinRelationSet &right, +unique_ptr PlanEnumerator::EmitPair(JoinRelationSet &left, JoinRelationSet &right, const vector> &info) { // get the left and right join plans auto left_plan = plans.find(left); @@ -139,12 +158,9 @@ JoinNode &PlanEnumerator::EmitPair(JoinRelationSet &left, JoinRelationSet &right if (left_plan == plans.end() || right_plan == plans.end()) { throw InternalException("No left or right plan: internal error in join order optimizer"); } - left_plan->second->Verify(); - right_plan->second->Verify(); auto &new_set = query_graph_manager.set_manager.Union(left, right); // create the join tree based on combining the two plans auto new_plan = CreateJoinTree(new_set, info, *left_plan->second, *right_plan->second); - new_plan->Verify(); // check if this plan is the optimal plan we found for this set of relations auto entry = plans.find(new_set); auto new_cost = new_plan->cost; @@ -170,8 +186,7 @@ JoinNode &PlanEnumerator::EmitPair(JoinRelationSet &left, JoinRelationSet &right // nodes in the SolveExactly plan // If we know a node in the full plan is updated, we can prevent ourselves from exiting the // DP algorithm until the last plan updated is a full plan - result.Verify(); - UpdateJoinNodesInFullPlan(result); +// UpdateJoinNodesInFullPlan(result); if (must_update_full_plan) { must_update_full_plan = false; } @@ -182,19 +197,20 @@ JoinNode &PlanEnumerator::EmitPair(JoinRelationSet &left, JoinRelationSet &right } D_ASSERT(new_plan); plans[new_set] = std::move(new_plan); - std::cout << "updating set " << new_set.ToString() << "with children " << left.ToString() << " and " << right.ToString() << std::endl; +// std::cout << "updating set " << new_set.ToString() << "with children " << left.ToString() << " and " << right.ToString() << std::endl; if (new_set.ToString() == "[0, 2, 5, 6]") { unordered_set bindings = {0, 1, 2, 5, 6, 9}; JoinRelationSet &desired_set = query_graph_manager.set_manager.GetJoinRelation(bindings); auto desired_set_plan = plans.find(desired_set); if (desired_set_plan != plans.end()) { - desired_set_plan->second->Verify(); std::cout << "verify ok? I don't think so" << std::endl; } } - return result; + return CreateJoinNodeFromDPJoinNode(result); } - return *entry->second; + // Create new join node. + + return CreateJoinNodeFromDPJoinNode(*entry->second); } bool PlanEnumerator::TryEmitPair(JoinRelationSet &left, JoinRelationSet &right, @@ -371,43 +387,38 @@ bool PlanEnumerator::SolveJoinOrderExactly() { } void PlanEnumerator::UpdateDPTree(JoinNode &new_plan) { - if (!NodeInFullPlan(new_plan)) { - // if the new node is not in the full plan, feel free to return - // because you won't be updating the full plan. - return; - } - auto &new_set = new_plan.set; - // now update every plan that uses this plan - unordered_set exclusion_set; - for (idx_t i = 0; i < new_set.count; i++) { - exclusion_set.insert(new_set.relations[i]); - } - auto neighbors = query_graph.GetNeighbors(new_set, exclusion_set); - auto all_neighbors = GetAllNeighborSets(neighbors); - for (const auto &neighbor : all_neighbors) { - auto &neighbor_relation = query_graph_manager.set_manager.GetJoinRelation(neighbor); - auto &combined_set = query_graph_manager.set_manager.Union(new_set, neighbor_relation); - - auto combined_set_plan = plans.find(combined_set); - if (combined_set_plan == plans.end()) { - continue; - } - - double combined_set_plan_cost = combined_set_plan->second->cost; // combined_set_plan->second->GetCost(); - auto connections = query_graph.GetConnections(new_set, neighbor_relation); - // recurse and update up the tree if the combined set produces a plan with a lower cost - // only recurse on neighbor relations that have plans. - auto right_plan = plans.find(neighbor_relation); - if (right_plan == plans.end()) { - continue; - } - auto &updated_plan = EmitPair(new_set, neighbor_relation, connections); - // <= because the child node has already been replaced. You need to - // replace the parent node as well in this case - if (updated_plan.cost < combined_set_plan_cost) { - UpdateDPTree(updated_plan); - } - } + return; +// if (!NodeInFullPlan(new_plan)) { +// // if the new node is not in the full plan, feel free to return +// // because you won't be updating the full plan. +// return; +// } +// auto &new_set = new_plan.set; +// // now update every plan that uses this plan +// unordered_set exclusion_set; +// for (idx_t i = 0; i < new_set.count; i++) { +// exclusion_set.insert(new_set.relations[i]); +// } +// auto neighbors = query_graph.GetNeighbors(new_set, exclusion_set); +// auto all_neighbors = GetAllNeighborSets(neighbors); +// for (const auto &neighbor : all_neighbors) { +// auto &neighbor_relation = query_graph_manager.set_manager.GetJoinRelation(neighbor); +// auto &combined_set = query_graph_manager.set_manager.Union(new_set, neighbor_relation); +// +// auto combined_set_plan = plans.find(combined_set); +// if (combined_set_plan == plans.end()) { +// continue; +// } +// +// double combined_set_plan_cost = combined_set_plan->second->cost; // combined_set_plan->second->GetCost(); +// auto connections = query_graph.GetConnections(new_set, neighbor_relation); +// // recurse and update up the tree if the combined set produces a plan with a lower cost +// // only recurse on neighbor relations that have plans. +// auto right_plan = plans.find(neighbor_relation); +// if (right_plan == plans.end()) { +// continue; +// } +// } } void PlanEnumerator::SolveJoinOrderApproximately() { @@ -423,7 +434,7 @@ void PlanEnumerator::SolveJoinOrderApproximately() { // smallest cost. This is O(r^2) per step, and every step will reduce the total amount of relations to-be-joined // by 1, so the total cost is O(r^3) in the amount of relations idx_t best_left = 0, best_right = 0; - optional_ptr best_connection; + unique_ptr best_connection; for (idx_t i = 0; i < join_relations.size(); i++) { auto left = join_relations[i]; for (idx_t j = i + 1; j < join_relations.size(); j++) { @@ -432,17 +443,16 @@ void PlanEnumerator::SolveJoinOrderApproximately() { auto connection = query_graph.GetConnections(left, right); if (!connection.empty()) { // we can check the cost of this connection - auto &node = EmitPair(left, right, connection); + auto node = EmitPair(left, right, connection); // update the DP tree in case a plan created by the DP algorithm uses the node // that was potentially just updated by EmitPair. You will get a use-after-free // error if future plans rely on the old node that was just replaced. // if node in FullPath, then updateDP tree. - UpdateDPTree(node); - if (!best_connection || node.cost < best_connection->cost) { + if (!best_connection || node->cost < best_connection->cost) { // best pair found so far - best_connection = &node; + best_connection = std::move(node); best_left = i; best_right = j; } @@ -452,15 +462,15 @@ void PlanEnumerator::SolveJoinOrderApproximately() { if (!best_connection) { // could not find a connection, but we were not done with finding a completed plan // we have to add a cross product; we add it between the two smallest relations - optional_ptr smallest_plans[2]; + unique_ptr smallest_plans[2]; idx_t smallest_index[2]; D_ASSERT(join_relations.size() >= 2); // first just add the first two join relations. It doesn't matter the cost as the JOO // will swap them on estimated cardinality anyway. for (idx_t i = 0; i < 2; i++) { - auto current_plan = plans[join_relations[i]].get(); - smallest_plans[i] = current_plan; + auto current_plan = CreateJoinNodeFromDPJoinNode(*plans[join_relations[i]]); + smallest_plans[i] = std::move(current_plan); smallest_index[i] = i; } @@ -468,11 +478,11 @@ void PlanEnumerator::SolveJoinOrderApproximately() { // add them if they have lower estimated cardinality. for (idx_t i = 2; i < join_relations.size(); i++) { // get the plan for this relation - auto current_plan = plans[join_relations[i].get()].get(); + auto current_plan = CreateJoinNodeFromDPJoinNode(*plans[join_relations[i]]); // check if the cardinality is smaller than the smallest two found so far for (idx_t j = 0; j < 2; j++) { if (!smallest_plans[j] || smallest_plans[j]->cost > current_plan->cost) { - smallest_plans[j] = current_plan; + smallest_plans[j] = std::move(current_plan); smallest_index[j] = i; break; } @@ -491,11 +501,11 @@ void PlanEnumerator::SolveJoinOrderApproximately() { auto connections = query_graph.GetConnections(left, right); D_ASSERT(!connections.empty()); - best_connection = &EmitPair(left, right, connections); + best_connection = EmitPair(left, right, connections); best_left = smallest_index[0]; best_right = smallest_index[1]; - UpdateDPTree(*best_connection); +// UpdateDPTree(*best_connection); // the code below assumes best_right > best_left if (best_left > best_right) { std::swap(best_left, best_right); @@ -527,9 +537,10 @@ void PlanEnumerator::InitLeafPlans() { for (idx_t i = 0; i < relation_stats.size(); i++) { auto stats = relation_stats.at(i); auto &relation_set = query_graph_manager.set_manager.GetJoinRelation(i); - auto join_node = make_uniq(relation_set); + auto join_node = make_uniq(relation_set); join_node->cost = 0; join_node->cardinality = stats.cardinality; + D_ASSERT(join_node->set.count == 1); plans[relation_set] = std::move(join_node); cost_model.cardinality_estimator.InitCardinalityEstimatorProps(&relation_set, stats); } @@ -566,7 +577,7 @@ unique_ptr PlanEnumerator::SolveJoinOrder() { //! solve the join order again, returning the final plan return SolveJoinOrder(); } - return std::move(final_plan->second); + return CreateJoinNodeFromDPJoinNode(*final_plan->second); } } // namespace duckdb diff --git a/test/optimizer/joins/update_nodes_in_full_path.test b/test/optimizer/joins/update_nodes_in_full_path.test new file mode 100644 index 000000000000..83b3db332dd3 --- /dev/null +++ b/test/optimizer/joins/update_nodes_in_full_path.test @@ -0,0 +1,42 @@ +# name: test/optimizer/joins/update_nodes_in_full_path.test +# description: updating nodes in full path should throw no errors +# group: [joins] + +require tpch + +statement ok +call dbgen(sf=0.1); + +statement ok +SELECT NULL +FROM main.supplier AS ref_0 +LEFT JOIN main.nation AS ref_1 +LEFT JOIN main.nation AS ref_2 + INNER JOIN main.customer AS ref_3 + INNER JOIN main.supplier AS ref_4 ON (ref_3.c_phone = ref_4.s_name) + ON ((SELECT l_linestatus FROM main.lineitem LIMIT 1 OFFSET 2) IS NULL) +INNER JOIN main.orders AS ref_5 + INNER JOIN main.orders AS ref_6 ON (ref_5.o_clerk ~~~ ref_5.o_comment) + ON (1) +ON (ref_3.c_mktsegment ~~~ ref_4.s_phone) +ON (ref_0.s_acctbal = ref_5.o_totalprice) +INNER JOIN main.lineitem AS ref_7 ON (ref_4.s_suppkey = ref_7.l_orderkey) +INNER JOIN main.supplier AS ref_8 + INNER JOIN main.partsupp AS ref_9 + INNER JOIN main.supplier AS ref_10 + INNER JOIN main.supplier AS ref_11 + INNER JOIN main.lineitem AS ref_12 + INNER JOIN main.customer AS ref_13 ON (ref_12.l_linestatus = ref_13.c_name) + ON ((SELECT ps_comment FROM main.partsupp LIMIT 1 OFFSET 4) ^@ ref_11.s_address) + ON (ref_13.c_phone ~~~ ref_10.s_address) + ON (ref_9.ps_partkey = ref_11.s_suppkey) + ON ((SELECT ps_comment FROM main.partsupp LIMIT 1 OFFSET 6) ~~* ref_12.l_linestatus) + ON ((ref_6.o_orderpriority IS NULL) OR (ref_7.l_linestatus ~~* (SELECT s_name FROM main.supplier LIMIT 1 OFFSET 6))) +INNER JOIN ( + SELECT ref_14.p_container AS c0, ref_14.p_mfgr AS c1, ref_14.p_container AS c2, ref_15.c_custkey AS c3 + FROM main.part AS ref_14 + INNER JOIN main.customer AS ref_15 ON (ref_14.p_brand ~~* ref_15.c_mktsegment) + WHERE (ref_14.p_comment ~~~ ref_14.p_container) + LIMIT 101 +) AS subq_0 ON (ref_6.o_orderstatus ~~* ref_6.o_comment) +WHERE (ref_8.s_address ~~* ref_8.s_address); diff --git a/test/optimizer/joins/updating_the_join_node_hash_map_has_no_errors.test b/test/optimizer/joins/updating_the_join_node_hash_map_has_no_errors.test index e61263b4f545..74fd75721a55 100644 --- a/test/optimizer/joins/updating_the_join_node_hash_map_has_no_errors.test +++ b/test/optimizer/joins/updating_the_join_node_hash_map_has_no_errors.test @@ -17,7 +17,7 @@ INNER JOIN main.customer AS ref_3 INNER JOIN main.supplier AS ref_4 ON ((ref_3.c_phone = ref_4.s_name)) ON ( (SELECT NULL)) INNER JOIN main.orders AS ref_5 -INNER JOIN main.orders AS ref_6 ON (ref_5.o_clerk) ON (1) ON (ref_3.c_mktsegment) ON ((ref_0.s_acctbal = ref_5.o_totalprice)) +INNER JOIN main.orders AS ref_6 ON (ref_5.o_clerk like '%0000%') ON (1) ON (ref_3.c_mktsegment NOT NULL) ON ((ref_0.s_acctbal = ref_5.o_totalprice)) INNER JOIN main.lineitem AS ref_7 ON ((ref_4.s_suppkey = ref_7.l_orderkey)) INNER JOIN main.supplier INNER JOIN main.supplier AS ref_11 @@ -26,5 +26,5 @@ INNER JOIN main.lineitem AS ref_12 ON ( (SELECT ps_comment FROM main.partsupp) ~~* ref_12.l_linestatus)) ON ((ref_7.l_linestatus ~~* (SELECT s_name FROM main.supplier))) INNER JOIN - (SELECT NULL) ON (ref_6.o_orderstatus); + (SELECT NULL) ON (ref_6.o_orderstatus NOT NULL); From aae501cd3fc01df53ca093f57389a0407cd44cbf Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Wed, 17 Apr 2024 17:13:36 +0200 Subject: [PATCH 520/603] make format-fix --- .../duckdb/optimizer/join_order/join_node.hpp | 7 +- .../optimizer/join_order/plan_enumerator.hpp | 7 +- src/optimizer/join_order/join_node.cpp | 6 +- src/optimizer/join_order/plan_enumerator.cpp | 78 +++++++++---------- ..._the_join_node_hash_map_has_no_errors.test | 2 +- 5 files changed, 52 insertions(+), 48 deletions(-) diff --git a/src/include/duckdb/optimizer/join_order/join_node.hpp b/src/include/duckdb/optimizer/join_order/join_node.hpp index 98e0553529ca..da884f30bade 100644 --- a/src/include/duckdb/optimizer/join_order/join_node.hpp +++ b/src/include/duckdb/optimizer/join_order/join_node.hpp @@ -34,7 +34,8 @@ class DPJoinNode { idx_t cardinality; //! Create an intermediate node in the join tree. base_cardinality = estimated_props.cardinality - DPJoinNode(JoinRelationSet &set, optional_ptr info, JoinRelationSet &left, JoinRelationSet &right, double cost); + DPJoinNode(JoinRelationSet &set, optional_ptr info, JoinRelationSet &left, JoinRelationSet &right, + double cost); //! Create a leaf node in the join tree //! set cost to 0 for leaf nodes @@ -42,7 +43,6 @@ class DPJoinNode { explicit DPJoinNode(JoinRelationSet &set); }; - class JoinNode { public: //! Represents a node in the join plan @@ -62,7 +62,8 @@ class JoinNode { idx_t cardinality; //! Create an intermediate node in the join tree. base_cardinality = estimated_props.cardinality - JoinNode(JoinRelationSet &set, optional_ptr info, unique_ptr left, unique_ptr right, double cost); + JoinNode(JoinRelationSet &set, optional_ptr info, unique_ptr left, + unique_ptr right, double cost); //! Create a leaf node in the join tree //! set cost to 0 for leaf nodes diff --git a/src/include/duckdb/optimizer/join_order/plan_enumerator.hpp b/src/include/duckdb/optimizer/join_order/plan_enumerator.hpp index 754d4c26d395..62231edf2e53 100644 --- a/src/include/duckdb/optimizer/join_order/plan_enumerator.hpp +++ b/src/include/duckdb/optimizer/join_order/plan_enumerator.hpp @@ -56,12 +56,13 @@ class PlanEnumerator { unordered_set join_nodes_in_full_plan; unique_ptr CreateJoinTree(JoinRelationSet &set, - const vector> &possible_connections, DPJoinNode &left, - DPJoinNode &right); + const vector> &possible_connections, DPJoinNode &left, + DPJoinNode &right); //! Emit a pair as a potential join candidate. Returns the best plan found for the (left, right) connection (either //! the newly created plan, or an existing plan) - unique_ptr EmitPair(JoinRelationSet &left, JoinRelationSet &right, const vector> &info); + unique_ptr EmitPair(JoinRelationSet &left, JoinRelationSet &right, + const vector> &info); //! Tries to emit a potential join candidate pair. Returns false if too many pairs have already been emitted, //! cancelling the dynamic programming step. bool TryEmitPair(JoinRelationSet &left, JoinRelationSet &right, const vector> &info); diff --git a/src/optimizer/join_order/join_node.cpp b/src/optimizer/join_order/join_node.cpp index 767d916948ae..1c3f8b21ad84 100644 --- a/src/optimizer/join_order/join_node.cpp +++ b/src/optimizer/join_order/join_node.cpp @@ -9,7 +9,8 @@ namespace duckdb { JoinNode::JoinNode(JoinRelationSet &set) : set(set) { } -JoinNode::JoinNode(JoinRelationSet &set, optional_ptr info, unique_ptr left, unique_ptr right, double cost) +JoinNode::JoinNode(JoinRelationSet &set, optional_ptr info, unique_ptr left, + unique_ptr right, double cost) : set(set), info(info), left(std::move(left)), right(std::move(right)), cost(cost) { } @@ -56,7 +57,8 @@ void JoinNode::Verify() { DPJoinNode::DPJoinNode(JoinRelationSet &set) : set(set), info(nullptr), is_leaf(true), left_set(set), right_set(set) { } -DPJoinNode::DPJoinNode(JoinRelationSet &set, optional_ptr info, JoinRelationSet &left, JoinRelationSet &right, double cost) +DPJoinNode::DPJoinNode(JoinRelationSet &set, optional_ptr info, JoinRelationSet &left, + JoinRelationSet &right, double cost) : set(set), info(info), is_leaf(false), left_set(left), right_set(right), cost(cost) { } diff --git a/src/optimizer/join_order/plan_enumerator.cpp b/src/optimizer/join_order/plan_enumerator.cpp index 9c385710c899..d5dade672054 100644 --- a/src/optimizer/join_order/plan_enumerator.cpp +++ b/src/optimizer/join_order/plan_enumerator.cpp @@ -115,8 +115,7 @@ unique_ptr PlanEnumerator::CreateJoinNodeFromDPJoinNode(DPJoinNode dp_ auto res = make_uniq(dp_node.set); res->cardinality = dp_node.cardinality; return res; - } - else { + } else { auto left_DPJoinNode = plans.find(dp_node.left_set); auto right_DPJoinNode = plans.find(dp_node.right_set); D_ASSERT(left_DPJoinNode->second); @@ -131,8 +130,8 @@ unique_ptr PlanEnumerator::CreateJoinNodeFromDPJoinNode(DPJoinNode dp_ //! Create a new JoinTree node by joining together two previous JoinTree nodes unique_ptr PlanEnumerator::CreateJoinTree(JoinRelationSet &set, - const vector> &possible_connections, - DPJoinNode &left, DPJoinNode &right) { + const vector> &possible_connections, + DPJoinNode &left, DPJoinNode &right) { // for the hash join we want the right side (build side) to have the smallest cardinality // also just a heuristic but for now... // FIXME: we should probably actually benchmark that as well @@ -151,7 +150,7 @@ unique_ptr PlanEnumerator::CreateJoinTree(JoinRelationSet &set, } unique_ptr PlanEnumerator::EmitPair(JoinRelationSet &left, JoinRelationSet &right, - const vector> &info) { + const vector> &info) { // get the left and right join plans auto left_plan = plans.find(left); auto right_plan = plans.find(right); @@ -186,7 +185,7 @@ unique_ptr PlanEnumerator::EmitPair(JoinRelationSet &left, JoinRelatio // nodes in the SolveExactly plan // If we know a node in the full plan is updated, we can prevent ourselves from exiting the // DP algorithm until the last plan updated is a full plan -// UpdateJoinNodesInFullPlan(result); + // UpdateJoinNodesInFullPlan(result); if (must_update_full_plan) { must_update_full_plan = false; } @@ -197,7 +196,8 @@ unique_ptr PlanEnumerator::EmitPair(JoinRelationSet &left, JoinRelatio } D_ASSERT(new_plan); plans[new_set] = std::move(new_plan); -// std::cout << "updating set " << new_set.ToString() << "with children " << left.ToString() << " and " << right.ToString() << std::endl; + // std::cout << "updating set " << new_set.ToString() << "with children " << left.ToString() << " and " << + //right.ToString() << std::endl; if (new_set.ToString() == "[0, 2, 5, 6]") { unordered_set bindings = {0, 1, 2, 5, 6, 9}; JoinRelationSet &desired_set = query_graph_manager.set_manager.GetJoinRelation(bindings); @@ -388,37 +388,37 @@ bool PlanEnumerator::SolveJoinOrderExactly() { void PlanEnumerator::UpdateDPTree(JoinNode &new_plan) { return; -// if (!NodeInFullPlan(new_plan)) { -// // if the new node is not in the full plan, feel free to return -// // because you won't be updating the full plan. -// return; -// } -// auto &new_set = new_plan.set; -// // now update every plan that uses this plan -// unordered_set exclusion_set; -// for (idx_t i = 0; i < new_set.count; i++) { -// exclusion_set.insert(new_set.relations[i]); -// } -// auto neighbors = query_graph.GetNeighbors(new_set, exclusion_set); -// auto all_neighbors = GetAllNeighborSets(neighbors); -// for (const auto &neighbor : all_neighbors) { -// auto &neighbor_relation = query_graph_manager.set_manager.GetJoinRelation(neighbor); -// auto &combined_set = query_graph_manager.set_manager.Union(new_set, neighbor_relation); -// -// auto combined_set_plan = plans.find(combined_set); -// if (combined_set_plan == plans.end()) { -// continue; -// } -// -// double combined_set_plan_cost = combined_set_plan->second->cost; // combined_set_plan->second->GetCost(); -// auto connections = query_graph.GetConnections(new_set, neighbor_relation); -// // recurse and update up the tree if the combined set produces a plan with a lower cost -// // only recurse on neighbor relations that have plans. -// auto right_plan = plans.find(neighbor_relation); -// if (right_plan == plans.end()) { -// continue; -// } -// } + // if (!NodeInFullPlan(new_plan)) { + // // if the new node is not in the full plan, feel free to return + // // because you won't be updating the full plan. + // return; + // } + // auto &new_set = new_plan.set; + // // now update every plan that uses this plan + // unordered_set exclusion_set; + // for (idx_t i = 0; i < new_set.count; i++) { + // exclusion_set.insert(new_set.relations[i]); + // } + // auto neighbors = query_graph.GetNeighbors(new_set, exclusion_set); + // auto all_neighbors = GetAllNeighborSets(neighbors); + // for (const auto &neighbor : all_neighbors) { + // auto &neighbor_relation = query_graph_manager.set_manager.GetJoinRelation(neighbor); + // auto &combined_set = query_graph_manager.set_manager.Union(new_set, neighbor_relation); + // + // auto combined_set_plan = plans.find(combined_set); + // if (combined_set_plan == plans.end()) { + // continue; + // } + // + // double combined_set_plan_cost = combined_set_plan->second->cost; // combined_set_plan->second->GetCost(); + // auto connections = query_graph.GetConnections(new_set, neighbor_relation); + // // recurse and update up the tree if the combined set produces a plan with a lower cost + // // only recurse on neighbor relations that have plans. + // auto right_plan = plans.find(neighbor_relation); + // if (right_plan == plans.end()) { + // continue; + // } + // } } void PlanEnumerator::SolveJoinOrderApproximately() { @@ -505,7 +505,7 @@ void PlanEnumerator::SolveJoinOrderApproximately() { best_left = smallest_index[0]; best_right = smallest_index[1]; -// UpdateDPTree(*best_connection); + // UpdateDPTree(*best_connection); // the code below assumes best_right > best_left if (best_left > best_right) { std::swap(best_left, best_right); diff --git a/test/optimizer/joins/updating_the_join_node_hash_map_has_no_errors.test b/test/optimizer/joins/updating_the_join_node_hash_map_has_no_errors.test index 74fd75721a55..94b768fce270 100644 --- a/test/optimizer/joins/updating_the_join_node_hash_map_has_no_errors.test +++ b/test/optimizer/joins/updating_the_join_node_hash_map_has_no_errors.test @@ -1,5 +1,5 @@ # name: test/optimizer/joins/updating_the_join_node_hash_map_has_no_errors.test -# description: +# description: # group: [joins] require tpch From 651513da98a7523f3d7cd772b4c5a1ff5019d8f1 Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 17 Apr 2024 20:23:31 +0200 Subject: [PATCH 521/603] missing headers --- src/function/table/copy_csv.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/function/table/copy_csv.cpp b/src/function/table/copy_csv.cpp index 8f6999dd9262..eb5205b113e9 100644 --- a/src/function/table/copy_csv.cpp +++ b/src/function/table/copy_csv.cpp @@ -16,6 +16,8 @@ #include "duckdb/parser/expression/function_expression.hpp" #include "duckdb/parser/expression/columnref_expression.hpp" #include "duckdb/parser/expression/constant_expression.hpp" +#include "duckdb/parser/expression/bound_expression.hpp" +#include "duckdb/planner/expression/bound_reference_expression.hpp" #include "duckdb/execution/column_binding_resolver.hpp" #include "duckdb/planner/operator/logical_dummy_scan.hpp" #include From aea174052cc575b720bf33ce26b3adca025e929a Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Wed, 17 Apr 2024 21:28:14 +0200 Subject: [PATCH 522/603] Fix numeric cast warnings --- src/function/table/arrow_conversion.cpp | 9 +++++---- .../duckdb/common/arrow/appender/varchar_data.hpp | 3 ++- .../duckdb/common/types/arrow_string_view_type.hpp | 2 +- src/include/duckdb/storage/storage_info.hpp | 2 +- src/main/config.cpp | 1 - src/optimizer/filter_combiner.cpp | 2 +- src/storage/standard_buffer_manager.cpp | 6 +++--- 7 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/function/table/arrow_conversion.cpp b/src/function/table/arrow_conversion.cpp index db305e580cb5..db831604674a 100644 --- a/src/function/table/arrow_conversion.cpp +++ b/src/function/table/arrow_conversion.cpp @@ -351,7 +351,7 @@ static void SetVectorStringView(Vector &vector, idx_t size, ArrowArray &array, i if (FlatVector::IsNull(vector, row_idx)) { continue; } - int32_t length = arrow_string[row_idx].Length(); + auto length = UnsafeNumericCast(arrow_string[row_idx].Length()); if (arrow_string[row_idx].IsInline()) { // This string is inlined // | Bytes 0-3 | Bytes 4-15 | @@ -363,7 +363,7 @@ static void SetVectorStringView(Vector &vector, idx_t size, ArrowArray &array, i // | Bytes 0-3 | Bytes 4-7 | Bytes 8-11 | Bytes 12-15 | // |------------|------------|------------|-------------| // | length | prefix | buf. index | offset | - int32_t buffer_index = arrow_string[row_idx].GetBufferIndex(); + auto buffer_index = UnsafeNumericCast(arrow_string[row_idx].GetBufferIndex()); int32_t offset = arrow_string[row_idx].GetOffset(); D_ASSERT(array.n_buffers > 2 + buffer_index); auto c_data = ArrowBufferData(array, 2 + buffer_index); @@ -738,8 +738,9 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowArraySca break; } case ArrowVariableSizeType::VIEW: { - SetVectorStringView(vector, size, array, - GetEffectiveOffset(array, parent_offset, scan_state, nested_offset)); + SetVectorStringView( + vector, size, array, + GetEffectiveOffset(array, NumericCast(parent_offset), scan_state, nested_offset)); break; } } diff --git a/src/include/duckdb/common/arrow/appender/varchar_data.hpp b/src/include/duckdb/common/arrow/appender/varchar_data.hpp index d4e9ffbbd17b..52c627145504 100644 --- a/src/include/duckdb/common/arrow/appender/varchar_data.hpp +++ b/src/include/duckdb/common/arrow/appender/varchar_data.hpp @@ -183,7 +183,8 @@ struct ArrowVarcharToStringViewData { // Buffer 2 is our only data buffer, could theoretically be more [ buffers ] result->buffers[2] = append_data.GetAuxBuffer().data(); // Buffer 3 is the data-buffer lengths buffer, and we also populate it in to finalize - reinterpret_cast(append_data.GetBufferSizeBuffer().data())[0] = append_data.offset; + reinterpret_cast(append_data.GetBufferSizeBuffer().data())[0] = + UnsafeNumericCast(append_data.offset); result->buffers[3] = append_data.GetBufferSizeBuffer().data(); } }; diff --git a/src/include/duckdb/common/types/arrow_string_view_type.hpp b/src/include/duckdb/common/types/arrow_string_view_type.hpp index 3a935b438dbe..94cc549302f6 100644 --- a/src/include/duckdb/common/types/arrow_string_view_type.hpp +++ b/src/include/duckdb/common/types/arrow_string_view_type.hpp @@ -29,7 +29,7 @@ union arrow_string_view_t { arrow_string_view_t(int32_t length, const char *data) { D_ASSERT(length <= ArrowStringViewConstants::MAX_INLINED_BYTES); inlined.length = length; - memcpy(inlined.data, data, length); + memcpy(inlined.data, data, UnsafeNumericCast(length)); if (length < ArrowStringViewConstants::MAX_INLINED_BYTES) { // have to 0 pad uint8_t remaining_bytes = ArrowStringViewConstants::MAX_INLINED_BYTES - NumericCast(length); diff --git a/src/include/duckdb/storage/storage_info.hpp b/src/include/duckdb/storage/storage_info.hpp index c7dad5835909..bc4c7591fc31 100644 --- a/src/include/duckdb/storage/storage_info.hpp +++ b/src/include/duckdb/storage/storage_info.hpp @@ -23,7 +23,7 @@ struct FileHandle; //! The maximum block id is 2^62 #define MAXIMUM_BLOCK 4611686018427388000LL //! The default block allocation size. -#define DEFAULT_BLOCK_ALLOC_SIZE 262144 +#define DEFAULT_BLOCK_ALLOC_SIZE 262144ULL //! The minimum block allocation size. This is the minimum size we test in our nightly tests. #define MIN_BLOCK_ALLOC_SIZE 16384 diff --git a/src/main/config.cpp b/src/main/config.cpp index bf15507cc86b..cc268e7e6b6c 100644 --- a/src/main/config.cpp +++ b/src/main/config.cpp @@ -54,7 +54,6 @@ bool DBConfigOptions::debug_print_bindings = false; #define FINAL_SETTING \ { nullptr, nullptr, LogicalTypeId::INVALID, nullptr, nullptr, nullptr, nullptr, nullptr } - static const ConfigurationOption internal_options[] = { DUCKDB_GLOBAL(AccessModeSetting), DUCKDB_GLOBAL(AllowPersistentSecrets), diff --git a/src/optimizer/filter_combiner.cpp b/src/optimizer/filter_combiner.cpp index fa85bc680666..a9563713fe93 100644 --- a/src/optimizer/filter_combiner.cpp +++ b/src/optimizer/filter_combiner.cpp @@ -575,7 +575,7 @@ TableFilterSet FilterCombiner::GenerateTableScanFilters(vector &column_id make_uniq(ExpressionType::COMPARE_EQUAL, fst_const_value_expr.value); table_filters.PushFilter(column_index, std::move(bound_eq_comparison)); table_filters.PushFilter(column_index, make_uniq()); - remaining_filters.erase(remaining_filters.begin() + rem_fil_idx); + remaining_filters.erase_at(rem_fil_idx); continue; } diff --git a/src/storage/standard_buffer_manager.cpp b/src/storage/standard_buffer_manager.cpp index e5b6490a1690..0a3b8f73c24e 100644 --- a/src/storage/standard_buffer_manager.cpp +++ b/src/storage/standard_buffer_manager.cpp @@ -50,9 +50,9 @@ void StandardBufferManager::SetTemporaryDirectory(const string &new_dir) { } StandardBufferManager::StandardBufferManager(DatabaseInstance &db, string tmp) - : BufferManager(), db(db), buffer_pool(db.GetBufferPool()), - temporary_id(MAXIMUM_BLOCK), buffer_allocator(BufferAllocatorAllocate, BufferAllocatorFree, - BufferAllocatorRealloc, make_uniq(*this)) { + : BufferManager(), db(db), buffer_pool(db.GetBufferPool()), temporary_id(MAXIMUM_BLOCK), + buffer_allocator(BufferAllocatorAllocate, BufferAllocatorFree, BufferAllocatorRealloc, + make_uniq(*this)) { temp_block_manager = make_uniq(*this, DEFAULT_BLOCK_ALLOC_SIZE); temporary_directory.path = std::move(tmp); for (idx_t i = 0; i < MEMORY_TAG_COUNT; i++) { From 30afb3b8dfbed512395cefd11ac0871a4d48a5a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Wed, 17 Apr 2024 21:34:59 +0200 Subject: [PATCH 523/603] first part wconversion --- src/CMakeLists.txt | 2 +- src/common/operator/cast_operators.cpp | 4 ++-- src/common/progress_bar/progress_bar.cpp | 2 +- src/common/types.cpp | 2 +- src/core_functions/aggregate/holistic/quantile.cpp | 7 ++++--- src/core_functions/scalar/math/numeric.cpp | 4 ++-- src/core_functions/scalar/random/setseed.cpp | 2 +- src/execution/aggregate_hashtable.cpp | 4 ++-- src/execution/join_hashtable.cpp | 3 ++- src/execution/operator/join/physical_hash_join.cpp | 2 +- src/execution/radix_partitioned_hashtable.cpp | 8 +++++--- src/include/duckdb/common/operator/numeric_cast.hpp | 4 ++-- src/main/config.cpp | 2 +- src/optimizer/join_order/estimated_properties.cpp | 4 ++-- src/optimizer/join_order/relation_manager.cpp | 6 ++++-- src/optimizer/join_order/relation_statistics_helper.cpp | 6 +++--- src/parallel/executor.cpp | 4 +++- src/storage/temporary_memory_manager.cpp | 6 +++--- 18 files changed, 40 insertions(+), 32 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4c69853abd3c..21f47240848a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -24,7 +24,7 @@ if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") set(EXIT_TIME_DESTRUCTORS_WARNING TRUE) set(CMAKE_CXX_FLAGS_DEBUG - "${CMAKE_CXX_FLAGS_DEBUG} -Wexit-time-destructors -Wimplicit-int-conversion -Wshorten-64-to-32 -Wnarrowing -Wsign-conversion -Wsign-compare" + "${CMAKE_CXX_FLAGS_DEBUG} -Wexit-time-destructors -Wimplicit-int-conversion -Wshorten-64-to-32 -Wnarrowing -Wsign-conversion -Wsign-compare -Wconversion" ) endif() diff --git a/src/common/operator/cast_operators.cpp b/src/common/operator/cast_operators.cpp index 769ff78cdaf7..843acc70527a 100644 --- a/src/common/operator/cast_operators.cpp +++ b/src/common/operator/cast_operators.cpp @@ -1957,7 +1957,7 @@ struct DecimalCastOperation { for (idx_t i = 0; i < state.excessive_decimals; i++) { auto mod = state.result % 10; round_up = NEGATIVE ? mod <= -5 : mod >= 5; - state.result /= 10.0; + state.result /= static_cast(10.0); } //! Only round up when exponents are involved if (state.exponent_type == T::ExponentType::POSITIVE && round_up) { @@ -2486,7 +2486,7 @@ bool DoubleToDecimalCast(SRC input, DST &result, CastParameters ¶meters, uin HandleCastError::AssignError(error, parameters); return false; } - result = Cast::Operation(value); + result = Cast::Operation(UnsafeNumericCast(value)); return true; } diff --git a/src/common/progress_bar/progress_bar.cpp b/src/common/progress_bar/progress_bar.cpp index 9c6a75fb6f86..720f5499fe89 100644 --- a/src/common/progress_bar/progress_bar.cpp +++ b/src/common/progress_bar/progress_bar.cpp @@ -121,7 +121,7 @@ void ProgressBar::Update(bool final) { if (final) { FinishProgressBarPrint(); } else { - PrintProgress(query_progress.percentage); + PrintProgress(NumericCast(query_progress.percentage.load())); } #endif } diff --git a/src/common/types.cpp b/src/common/types.cpp index 862f86e5f900..b54c47e78b0d 100644 --- a/src/common/types.cpp +++ b/src/common/types.cpp @@ -1110,7 +1110,7 @@ bool ApproxEqual(float ldecimal, float rdecimal) { if (!Value::FloatIsFinite(ldecimal) || !Value::FloatIsFinite(rdecimal)) { return ldecimal == rdecimal; } - float epsilon = std::fabs(rdecimal) * 0.01 + 0.00000001; + auto epsilon = UnsafeNumericCast(std::fabs(rdecimal) * 0.01 + 0.00000001); return std::fabs(ldecimal - rdecimal) <= epsilon; } diff --git a/src/core_functions/aggregate/holistic/quantile.cpp b/src/core_functions/aggregate/holistic/quantile.cpp index 0b3c67fe01a1..84446a6cee44 100644 --- a/src/core_functions/aggregate/holistic/quantile.cpp +++ b/src/core_functions/aggregate/holistic/quantile.cpp @@ -157,7 +157,7 @@ struct CastInterpolation { template static inline TARGET_TYPE Interpolate(const TARGET_TYPE &lo, const double d, const TARGET_TYPE &hi) { const auto delta = hi - lo; - return lo + delta * d; + return UnsafeNumericCast(lo + delta * d); } }; @@ -295,7 +295,8 @@ bool operator==(const QuantileValue &x, const QuantileValue &y) { template struct Interpolator { Interpolator(const QuantileValue &q, const idx_t n_p, const bool desc_p) - : desc(desc_p), RN((double)(n_p - 1) * q.dbl), FRN(floor(RN)), CRN(ceil(RN)), begin(0), end(n_p) { + : desc(desc_p), RN((double)(n_p - 1) * q.dbl), FRN(UnsafeNumericCast(floor(RN))), + CRN(UnsafeNumericCast(ceil(RN))), begin(0), end(n_p) { } template > @@ -365,7 +366,7 @@ struct Interpolator { } default: const auto scaled_q = (double)(n * q.dbl); - floored = floor(n - scaled_q); + floored = UnsafeNumericCast(floor(n - scaled_q)); break; } diff --git a/src/core_functions/scalar/math/numeric.cpp b/src/core_functions/scalar/math/numeric.cpp index 711f92608aec..4a6055a91413 100644 --- a/src/core_functions/scalar/math/numeric.cpp +++ b/src/core_functions/scalar/math/numeric.cpp @@ -516,7 +516,7 @@ struct RoundOperatorPrecision { return input; } } - return rounded_value; + return UnsafeNumericCast(rounded_value); } }; @@ -527,7 +527,7 @@ struct RoundOperator { if (std::isinf(rounded_value) || std::isnan(rounded_value)) { return input; } - return rounded_value; + return UnsafeNumericCast(rounded_value); } }; diff --git a/src/core_functions/scalar/random/setseed.cpp b/src/core_functions/scalar/random/setseed.cpp index f2db16e6c5a6..32965cf18ded 100644 --- a/src/core_functions/scalar/random/setseed.cpp +++ b/src/core_functions/scalar/random/setseed.cpp @@ -39,7 +39,7 @@ static void SetSeedFunction(DataChunk &args, ExpressionState &state, Vector &res if (input_seeds[i] < -1.0 || input_seeds[i] > 1.0 || Value::IsNan(input_seeds[i])) { throw InvalidInputException("SETSEED accepts seed values between -1.0 and 1.0, inclusive"); } - uint32_t norm_seed = (input_seeds[i] + 1.0) * half_max; + auto norm_seed = NumericCast((input_seeds[i] + 1.0) * half_max); random_engine.SetSeed(norm_seed); } diff --git a/src/execution/aggregate_hashtable.cpp b/src/execution/aggregate_hashtable.cpp index c8ab1ce6b965..6027e8205286 100644 --- a/src/execution/aggregate_hashtable.cpp +++ b/src/execution/aggregate_hashtable.cpp @@ -122,7 +122,7 @@ idx_t GroupedAggregateHashTable::InitialCapacity() { idx_t GroupedAggregateHashTable::GetCapacityForCount(idx_t count) { count = MaxValue(InitialCapacity(), count); - return NextPowerOfTwo(count * LOAD_FACTOR); + return NextPowerOfTwo(NumericCast(static_cast(count) * LOAD_FACTOR)); } idx_t GroupedAggregateHashTable::Capacity() const { @@ -130,7 +130,7 @@ idx_t GroupedAggregateHashTable::Capacity() const { } idx_t GroupedAggregateHashTable::ResizeThreshold() const { - return Capacity() / LOAD_FACTOR; + return NumericCast(static_cast(Capacity()) / LOAD_FACTOR); } idx_t GroupedAggregateHashTable::ApplyBitMask(hash_t hash) const { diff --git a/src/execution/join_hashtable.cpp b/src/execution/join_hashtable.cpp index 53adf3a85ef8..c1f273915e6c 100644 --- a/src/execution/join_hashtable.cpp +++ b/src/execution/join_hashtable.cpp @@ -940,7 +940,8 @@ void JoinHashTable::SetRepartitionRadixBits(vector> &l auto new_estimated_size = double(max_partition_size) / partition_multiplier; auto new_estimated_count = double(max_partition_count) / partition_multiplier; - auto new_estimated_ht_size = new_estimated_size + PointerTableSize(new_estimated_count); + auto new_estimated_ht_size = + new_estimated_size + static_cast(PointerTableSize(NumericCast(new_estimated_count))); if (new_estimated_ht_size <= double(max_ht_size) / 4) { // Aim for an estimated partition size of max_ht_size / 4 diff --git a/src/execution/operator/join/physical_hash_join.cpp b/src/execution/operator/join/physical_hash_join.cpp index 8632c44bda13..9cd7e34a07f7 100644 --- a/src/execution/operator/join/physical_hash_join.cpp +++ b/src/execution/operator/join/physical_hash_join.cpp @@ -409,7 +409,7 @@ class HashJoinRepartitionEvent : public BasePipelineEvent { total_size += sink_collection.SizeInBytes(); total_count += sink_collection.Count(); } - auto total_blocks = (double(total_size) + Storage::BLOCK_SIZE - 1) / Storage::BLOCK_SIZE; + auto total_blocks = NumericCast((double(total_size) + Storage::BLOCK_SIZE - 1) / Storage::BLOCK_SIZE); auto count_per_block = total_count / total_blocks; auto blocks_per_vector = MaxValue(STANDARD_VECTOR_SIZE / count_per_block, 2); diff --git a/src/execution/radix_partitioned_hashtable.cpp b/src/execution/radix_partitioned_hashtable.cpp index d2a174dc546e..595c1c4c2c83 100644 --- a/src/execution/radix_partitioned_hashtable.cpp +++ b/src/execution/radix_partitioned_hashtable.cpp @@ -197,7 +197,8 @@ RadixHTGlobalSinkState::RadixHTGlobalSinkState(ClientContext &context_p, const R count_before_combining(0), max_partition_size(0) { auto tuples_per_block = Storage::BLOCK_ALLOC_SIZE / radix_ht.GetLayout().GetRowWidth(); - idx_t ht_count = config.sink_capacity / GroupedAggregateHashTable::LOAD_FACTOR; + idx_t ht_count = + NumericCast(static_cast(config.sink_capacity) / GroupedAggregateHashTable::LOAD_FACTOR); auto num_partitions = RadixPartitioning::NumberOfPartitions(config.GetRadixBits()); auto count_per_partition = ht_count / num_partitions; auto blocks_per_partition = (count_per_partition + tuples_per_block) / tuples_per_block + 1; @@ -305,7 +306,8 @@ idx_t RadixHTConfig::SinkCapacity(ClientContext &context) { // Divide cache per active thread by entry size, round up to next power of two, to get capacity const auto size_per_entry = sizeof(aggr_ht_entry_t) * GroupedAggregateHashTable::LOAD_FACTOR; - const auto capacity = NextPowerOfTwo(cache_per_active_thread / size_per_entry); + const auto capacity = + NextPowerOfTwo(NumericCast(static_cast(cache_per_active_thread) / size_per_entry)); // Capacity must be at least the minimum capacity return MaxValue(capacity, GroupedAggregateHashTable::InitialCapacity()); @@ -718,7 +720,7 @@ void RadixHTLocalSourceState::Finalize(RadixHTGlobalSinkState &sink, RadixHTGlob // However, we will limit the initial capacity so we don't do a huge over-allocation const auto n_threads = NumericCast(TaskScheduler::GetScheduler(gstate.context).NumberOfThreads()); const auto memory_limit = BufferManager::GetBufferManager(gstate.context).GetMaxMemory(); - const idx_t thread_limit = 0.6 * memory_limit / n_threads; + const idx_t thread_limit = NumericCast(0.6 * memory_limit / n_threads); const idx_t size_per_entry = partition.data->SizeInBytes() / MaxValue(partition.data->Count(), 1) + idx_t(GroupedAggregateHashTable::LOAD_FACTOR * sizeof(aggr_ht_entry_t)); diff --git a/src/include/duckdb/common/operator/numeric_cast.hpp b/src/include/duckdb/common/operator/numeric_cast.hpp index 26603a987ce2..b6d3b6742f80 100644 --- a/src/include/duckdb/common/operator/numeric_cast.hpp +++ b/src/include/duckdb/common/operator/numeric_cast.hpp @@ -75,7 +75,7 @@ bool TryCastWithOverflowCheckFloat(SRC value, T &result, SRC min, SRC max) { return false; } // PG FLOAT => INT casts use statistical rounding. - result = std::nearbyint(value); + result = UnsafeNumericCast(std::nearbyint(value)); return true; } @@ -182,7 +182,7 @@ bool TryCastWithOverflowCheck(double input, float &result) { return true; } auto res = float(input); - if (!Value::FloatIsFinite(input)) { + if (!Value::DoubleIsFinite(input)) { return false; } result = res; diff --git a/src/main/config.cpp b/src/main/config.cpp index b8ad98651f11..56df902575a6 100644 --- a/src/main/config.cpp +++ b/src/main/config.cpp @@ -423,7 +423,7 @@ idx_t DBConfig::ParseMemoryLimit(const string &arg) { throw ParserException("Unknown unit for memory_limit: %s (expected: KB, MB, GB, TB for 1000^i units or KiB, " "MiB, GiB, TiB for 1024^i unites)"); } - return (idx_t)multiplier * limit; + return NumericCast(multiplier * limit); } // Right now we only really care about access mode when comparing DBConfigs diff --git a/src/optimizer/join_order/estimated_properties.cpp b/src/optimizer/join_order/estimated_properties.cpp index d3841a1bb3fb..9e907331abc4 100644 --- a/src/optimizer/join_order/estimated_properties.cpp +++ b/src/optimizer/join_order/estimated_properties.cpp @@ -11,7 +11,7 @@ double EstimatedProperties::GetCardinality() const { template <> idx_t EstimatedProperties::GetCardinality() const { auto max_idx_t = NumericLimits::Maximum() - 10000; - return MinValue(cardinality, max_idx_t); + return MinValue(NumericCast(cardinality), max_idx_t); } template <> @@ -22,7 +22,7 @@ double EstimatedProperties::GetCost() const { template <> idx_t EstimatedProperties::GetCost() const { auto max_idx_t = NumericLimits::Maximum() - 10000; - return MinValue(cost, max_idx_t); + return MinValue(NumericCast(cost), max_idx_t); } void EstimatedProperties::SetCardinality(double new_card) { diff --git a/src/optimizer/join_order/relation_manager.cpp b/src/optimizer/join_order/relation_manager.cpp index 3a0fbf0e920b..3df21b77e7fe 100644 --- a/src/optimizer/join_order/relation_manager.cpp +++ b/src/optimizer/join_order/relation_manager.cpp @@ -183,7 +183,8 @@ bool RelationManager::ExtractJoinRelations(LogicalOperator &input_op, auto &aggr = op->Cast(); auto operator_stats = RelationStatisticsHelper::ExtractAggregationStats(aggr, child_stats); if (!datasource_filters.empty()) { - operator_stats.cardinality *= RelationStatisticsHelper::DEFAULT_SELECTIVITY; + operator_stats.cardinality = NumericCast(static_cast(operator_stats.cardinality) * + RelationStatisticsHelper::DEFAULT_SELECTIVITY); } AddAggregateOrWindowRelation(input_op, parent, operator_stats, op->type); return true; @@ -196,7 +197,8 @@ bool RelationManager::ExtractJoinRelations(LogicalOperator &input_op, auto &window = op->Cast(); auto operator_stats = RelationStatisticsHelper::ExtractWindowStats(window, child_stats); if (!datasource_filters.empty()) { - operator_stats.cardinality *= RelationStatisticsHelper::DEFAULT_SELECTIVITY; + operator_stats.cardinality = NumericCast(static_cast(operator_stats.cardinality) * + RelationStatisticsHelper::DEFAULT_SELECTIVITY); } AddAggregateOrWindowRelation(input_op, parent, operator_stats, op->type); return true; diff --git a/src/optimizer/join_order/relation_statistics_helper.cpp b/src/optimizer/join_order/relation_statistics_helper.cpp index 94f5ddeb8b47..79af3bd8ca33 100644 --- a/src/optimizer/join_order/relation_statistics_helper.cpp +++ b/src/optimizer/join_order/relation_statistics_helper.cpp @@ -121,8 +121,8 @@ RelationStats RelationStatisticsHelper::ExtractGetStats(LogicalGet &get, ClientC // and there are other table filters (i.e cost > 50), use default selectivity. bool has_equality_filter = (cardinality_after_filters != base_table_cardinality); if (!has_equality_filter && !get.table_filters.filters.empty()) { - cardinality_after_filters = - MaxValue(base_table_cardinality * RelationStatisticsHelper::DEFAULT_SELECTIVITY, 1); + cardinality_after_filters = MaxValue( + NumericCast(base_table_cardinality * RelationStatisticsHelper::DEFAULT_SELECTIVITY), 1U); } if (base_table_cardinality == 0) { cardinality_after_filters = 0; @@ -345,7 +345,7 @@ RelationStats RelationStatisticsHelper::ExtractAggregationStats(LogicalAggregate // most likely we are running on parquet files. Therefore we divide by 2. new_card = (double)child_stats.cardinality / 2; } - stats.cardinality = new_card; + stats.cardinality = NumericCast(new_card); stats.column_names = child_stats.column_names; stats.stats_initialized = true; auto num_child_columns = aggr.GetColumnBindings().size(); diff --git a/src/parallel/executor.cpp b/src/parallel/executor.cpp index 41e710284c0e..585d093eca37 100644 --- a/src/parallel/executor.cpp +++ b/src/parallel/executor.cpp @@ -639,7 +639,9 @@ bool Executor::GetPipelinesProgress(double ¤t_progress, uint64_t ¤t_ for (size_t i = 0; i < progress.size(); i++) { progress[i] = MaxValue(0.0, MinValue(100.0, progress[i])); - current_cardinality += double(progress[i]) * double(cardinality[i]) / double(100); + current_cardinality = NumericCast(static_cast( + current_cardinality + + static_cast(progress[i]) * static_cast(cardinality[i]) / static_cast(100))); current_progress += progress[i] * double(cardinality[i]) / double(total_cardinality); D_ASSERT(current_cardinality <= total_cardinality); } diff --git a/src/storage/temporary_memory_manager.cpp b/src/storage/temporary_memory_manager.cpp index ba046d30fb17..2564e11cfb0e 100644 --- a/src/storage/temporary_memory_manager.cpp +++ b/src/storage/temporary_memory_manager.cpp @@ -45,7 +45,7 @@ void TemporaryMemoryManager::UpdateConfiguration(ClientContext &context) { auto &buffer_manager = BufferManager::GetBufferManager(context); auto &task_scheduler = TaskScheduler::GetScheduler(context); - memory_limit = MAXIMUM_MEMORY_LIMIT_RATIO * double(buffer_manager.GetMaxMemory()); + memory_limit = NumericCast(MAXIMUM_MEMORY_LIMIT_RATIO * static_cast(buffer_manager.GetMaxMemory())); has_temporary_directory = buffer_manager.HasTemporaryDirectory(); num_threads = NumericCast(task_scheduler.NumberOfThreads()); query_max_memory = buffer_manager.GetQueryMaxMemory(); @@ -92,14 +92,14 @@ void TemporaryMemoryManager::UpdateState(ClientContext &context, TemporaryMemory // 3. MAXIMUM_FREE_MEMORY_RATIO * free memory auto upper_bound = MinValue(temporary_memory_state.remaining_size, query_max_memory); auto free_memory = memory_limit - (reservation - temporary_memory_state.reservation); - upper_bound = MinValue(upper_bound, MAXIMUM_FREE_MEMORY_RATIO * free_memory); + upper_bound = MinValue(upper_bound, NumericCast(MAXIMUM_FREE_MEMORY_RATIO * free_memory)); if (remaining_size > memory_limit) { // We're processing more data than fits in memory, so we must further limit memory usage. // The upper bound for the reservation of this state is now also the minimum of: // 3. The ratio of the remaining size of this state and the total remaining size * memory limit auto ratio_of_remaining = double(temporary_memory_state.remaining_size) / double(remaining_size); - upper_bound = MinValue(upper_bound, ratio_of_remaining * memory_limit); + upper_bound = MinValue(upper_bound, NumericCast(ratio_of_remaining * memory_limit)); } SetReservation(temporary_memory_state, MaxValue(lower_bound, upper_bound)); From fbeeb5ea4970ce36e9e55ea65b33f3dd5f82d1e5 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Wed, 17 Apr 2024 21:49:40 +0200 Subject: [PATCH 524/603] Minor improvements to statement reduction --- extension/sqlsmith/statement_simplifier.cpp | 2 ++ scripts/reduce_sql.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/extension/sqlsmith/statement_simplifier.cpp b/extension/sqlsmith/statement_simplifier.cpp index ff31a6fc1255..66662f4cc024 100644 --- a/extension/sqlsmith/statement_simplifier.cpp +++ b/extension/sqlsmith/statement_simplifier.cpp @@ -132,6 +132,8 @@ void StatementSimplifier::Simplify(unique_ptr &ref) { SimplifyOptionalExpression(cp.condition); SimplifyReplace(ref, cp.left); SimplifyReplace(ref, cp.right); + SimplifyEnum(cp.type, JoinType::INNER); + SimplifyEnum(cp.ref_type, JoinRefType::REGULAR); break; } case TableReferenceType::EXPRESSION_LIST: { diff --git a/scripts/reduce_sql.py b/scripts/reduce_sql.py index bb5c62460686..ab2cd70e96c4 100644 --- a/scripts/reduce_sql.py +++ b/scripts/reduce_sql.py @@ -258,7 +258,7 @@ def reduce_query_log(queries, shell, max_time_seconds=300): print(expected_error) print("===================================================") - final_query = reduce(sql_query, data_load, shell, expected_error, args.max_time) + final_query = reduce(sql_query, data_load, shell, expected_error, int(args.max_time)) print("Found final reduced query") print("===================================================") print(final_query) From c6a0b1a787ae100471e529c826efe7660401b6de Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Wed, 17 Apr 2024 21:50:17 +0200 Subject: [PATCH 525/603] Bump Julia to v0.10.2 --- tools/juliapkg/Project.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/juliapkg/Project.toml b/tools/juliapkg/Project.toml index da97b7535576..6914a8f38ee5 100644 --- a/tools/juliapkg/Project.toml +++ b/tools/juliapkg/Project.toml @@ -1,7 +1,7 @@ name = "DuckDB" uuid = "d2f5444f-75bc-4fdf-ac35-56f514c445e1" authors = ["Mark Raasveldt "] -version = "0.10.1" +version = "0.10.2" [deps] DBInterface = "a10d1c49-ce27-4219-8d33-6db1a4562965" @@ -14,7 +14,7 @@ WeakRefStrings = "ea10d353-3f73-51f8-a26c-33c1cb351aa5" [compat] DBInterface = "2.5" -DuckDB_jll = "0.10.1" +DuckDB_jll = "0.10.2" FixedPointDecimals = "0.4, 0.5" Tables = "1.7" WeakRefStrings = "1.4" From ce6d2cdc04cdbb1bcb2da8eb5ad31d1d97653ac4 Mon Sep 17 00:00:00 2001 From: Richard Wesley <13156216+hawkfish@users.noreply.github.com> Date: Wed, 17 Apr 2024 14:31:50 -0700 Subject: [PATCH 526/603] Internal #1848: Window Progress PhysicalWindow is a source but didn't implement GetProgress fixes: duckdblabs/duckdb-internal#1848 --- .../operator/aggregate/physical_window.cpp | 15 ++++++++++++++- .../operator/aggregate/physical_window.hpp | 2 ++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/execution/operator/aggregate/physical_window.cpp b/src/execution/operator/aggregate/physical_window.cpp index bcfe0a56bd3b..5cb0045d9e60 100644 --- a/src/execution/operator/aggregate/physical_window.cpp +++ b/src/execution/operator/aggregate/physical_window.cpp @@ -205,6 +205,8 @@ class WindowGlobalSourceState : public GlobalSourceState { mutable mutex built_lock; //! The number of unfinished tasks atomic tasks_remaining; + //! The number of rows returned + atomic returned; public: idx_t MaxThreads() override { @@ -217,7 +219,7 @@ class WindowGlobalSourceState : public GlobalSourceState { }; WindowGlobalSourceState::WindowGlobalSourceState(ClientContext &context_p, WindowGlobalSinkState &gsink_p) - : context(context_p), gsink(gsink_p), next_build(0), tasks_remaining(0) { + : context(context_p), gsink(gsink_p), next_build(0), tasks_remaining(0), returned(0) { auto &hash_groups = gsink.global_partition->hash_groups; auto &gpart = gsink.global_partition; @@ -681,6 +683,15 @@ OrderPreservationType PhysicalWindow::SourceOrder() const { return SupportsBatchIndex() ? OrderPreservationType::FIXED_ORDER : OrderPreservationType::NO_ORDER; } +double PhysicalWindow::GetProgress(ClientContext &context, GlobalSourceState &gsource_p) const { + auto &gsource = gsource_p.Cast(); + const auto returned = gsource.returned.load(); + + auto &gsink = gsource.gsink; + const auto count = gsink.global_partition->count.load(); + return count ? (returned / double(count)) : -1; +} + idx_t PhysicalWindow::GetBatchIndex(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate_p, LocalSourceState &lstate_p) const { auto &lstate = lstate_p.Cast(); @@ -689,6 +700,7 @@ idx_t PhysicalWindow::GetBatchIndex(ExecutionContext &context, DataChunk &chunk, SourceResultType PhysicalWindow::GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const { + auto &gsource = input.global_state.Cast(); auto &lsource = input.local_state.Cast(); while (chunk.size() == 0) { // Move to the next bin if we are done. @@ -699,6 +711,7 @@ SourceResultType PhysicalWindow::GetData(ExecutionContext &context, DataChunk &c } lsource.Scan(chunk); + gsource.returned += chunk.size(); } return chunk.size() == 0 ? SourceResultType::FINISHED : SourceResultType::HAVE_MORE_OUTPUT; diff --git a/src/include/duckdb/execution/operator/aggregate/physical_window.hpp b/src/include/duckdb/execution/operator/aggregate/physical_window.hpp index aad04b562776..a554a46bc741 100644 --- a/src/include/duckdb/execution/operator/aggregate/physical_window.hpp +++ b/src/include/duckdb/execution/operator/aggregate/physical_window.hpp @@ -50,6 +50,8 @@ class PhysicalWindow : public PhysicalOperator { bool SupportsBatchIndex() const override; OrderPreservationType SourceOrder() const override; + double GetProgress(ClientContext &context, GlobalSourceState &gstate_p) const override; + public: // Sink interface SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override; From a36c22ae5f423cfcd679573aca3b5d6a5245ab56 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 18 Apr 2024 10:08:44 +0200 Subject: [PATCH 527/603] remove unused/unneeded code --- .../optimizer/join_order/plan_enumerator.hpp | 5 - src/optimizer/join_order/plan_enumerator.cpp | 95 +------------------ 2 files changed, 2 insertions(+), 98 deletions(-) diff --git a/src/include/duckdb/optimizer/join_order/plan_enumerator.hpp b/src/include/duckdb/optimizer/join_order/plan_enumerator.hpp index 62231edf2e53..9cb04a9c8ace 100644 --- a/src/include/duckdb/optimizer/join_order/plan_enumerator.hpp +++ b/src/include/duckdb/optimizer/join_order/plan_enumerator.hpp @@ -81,11 +81,6 @@ class PlanEnumerator { bool SolveJoinOrderExactly(); //! Solve the join order approximately using a greedy algorithm void SolveJoinOrderApproximately(); - - void UpdateDPTree(JoinNode &new_plan); - - void UpdateJoinNodesInFullPlan(JoinNode &node); - bool NodeInFullPlan(JoinNode &node); }; } // namespace duckdb diff --git a/src/optimizer/join_order/plan_enumerator.cpp b/src/optimizer/join_order/plan_enumerator.cpp index d5dade672054..035192643225 100644 --- a/src/optimizer/join_order/plan_enumerator.cpp +++ b/src/optimizer/join_order/plan_enumerator.cpp @@ -8,25 +8,6 @@ namespace duckdb { -bool PlanEnumerator::NodeInFullPlan(JoinNode &node) { - return join_nodes_in_full_plan.find(node.set.ToString()) != join_nodes_in_full_plan.end(); -} - -void PlanEnumerator::UpdateJoinNodesInFullPlan(JoinNode &node) { - if (node.set.count == query_graph_manager.relation_manager.NumRelations()) { - join_nodes_in_full_plan.clear(); - } - if (node.set.count < query_graph_manager.relation_manager.NumRelations()) { - join_nodes_in_full_plan.insert(node.set.ToString()); - } - if (node.left) { - UpdateJoinNodesInFullPlan(*node.left); - } - if (node.right) { - UpdateJoinNodesInFullPlan(*node.right); - } -} - static vector> AddSuperSets(const vector> ¤t, const vector &all_neighbors) { vector> ret; @@ -169,47 +150,10 @@ unique_ptr PlanEnumerator::EmitPair(JoinRelationSet &left, JoinRelatio } if (entry == plans.end() || new_cost < old_cost) { // the new plan costs less than the old plan. Update our DP tree and cost tree - auto &result = *new_plan; - - if (full_plan_found && - join_nodes_in_full_plan.find(new_plan->set.ToString()) != join_nodes_in_full_plan.end()) { - must_update_full_plan = true; - } - if (new_set.count == query_graph_manager.relation_manager.NumRelations()) { - full_plan_found = true; - // If we find a full plan, we need to keep track of which nodes are in the full plan. - // It's possible the DP algorithm updates a node in the current full plan, then moves on - // to the SolveApproximately. SolveApproximately may find a full plan with a higher cost than - // what SolveExactly found. In this case, we revert to the SolveExactly plan, but it is - // possible to get use-after-free errors if the SolveApproximately algorithm updated some (but not all) - // nodes in the SolveExactly plan - // If we know a node in the full plan is updated, we can prevent ourselves from exiting the - // DP algorithm until the last plan updated is a full plan - // UpdateJoinNodesInFullPlan(result); - if (must_update_full_plan) { - must_update_full_plan = false; - } - } - - if (new_set.ToString() == "[0, 1, 2, 5, 6, 9]") { - auto break_here = 0; - } - D_ASSERT(new_plan); plans[new_set] = std::move(new_plan); - // std::cout << "updating set " << new_set.ToString() << "with children " << left.ToString() << " and " << - //right.ToString() << std::endl; - if (new_set.ToString() == "[0, 2, 5, 6]") { - unordered_set bindings = {0, 1, 2, 5, 6, 9}; - JoinRelationSet &desired_set = query_graph_manager.set_manager.GetJoinRelation(bindings); - auto desired_set_plan = plans.find(desired_set); - if (desired_set_plan != plans.end()) { - std::cout << "verify ok? I don't think so" << std::endl; - } - } - return CreateJoinNodeFromDPJoinNode(result); + return CreateJoinNodeFromDPJoinNode(*plans[new_set]); } - // Create new join node. - + // Create join node from the plan currently in the DP table. return CreateJoinNodeFromDPJoinNode(*entry->second); } @@ -386,41 +330,6 @@ bool PlanEnumerator::SolveJoinOrderExactly() { return true; } -void PlanEnumerator::UpdateDPTree(JoinNode &new_plan) { - return; - // if (!NodeInFullPlan(new_plan)) { - // // if the new node is not in the full plan, feel free to return - // // because you won't be updating the full plan. - // return; - // } - // auto &new_set = new_plan.set; - // // now update every plan that uses this plan - // unordered_set exclusion_set; - // for (idx_t i = 0; i < new_set.count; i++) { - // exclusion_set.insert(new_set.relations[i]); - // } - // auto neighbors = query_graph.GetNeighbors(new_set, exclusion_set); - // auto all_neighbors = GetAllNeighborSets(neighbors); - // for (const auto &neighbor : all_neighbors) { - // auto &neighbor_relation = query_graph_manager.set_manager.GetJoinRelation(neighbor); - // auto &combined_set = query_graph_manager.set_manager.Union(new_set, neighbor_relation); - // - // auto combined_set_plan = plans.find(combined_set); - // if (combined_set_plan == plans.end()) { - // continue; - // } - // - // double combined_set_plan_cost = combined_set_plan->second->cost; // combined_set_plan->second->GetCost(); - // auto connections = query_graph.GetConnections(new_set, neighbor_relation); - // // recurse and update up the tree if the combined set produces a plan with a lower cost - // // only recurse on neighbor relations that have plans. - // auto right_plan = plans.find(neighbor_relation); - // if (right_plan == plans.end()) { - // continue; - // } - // } -} - void PlanEnumerator::SolveJoinOrderApproximately() { // at this point, we exited the dynamic programming but did not compute the final join order because it took too // long instead, we use a greedy heuristic to obtain a join ordering now we use Greedy Operator Ordering to From 74b92cdd0da735e94dcc1dacd7a8a704fbe9534d Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 18 Apr 2024 10:36:49 +0200 Subject: [PATCH 528/603] remove more unneeded code --- .../duckdb/optimizer/join_order/plan_enumerator.hpp | 5 +---- src/optimizer/join_order/plan_enumerator.cpp | 7 +++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/include/duckdb/optimizer/join_order/plan_enumerator.hpp b/src/include/duckdb/optimizer/join_order/plan_enumerator.hpp index 9cb04a9c8ace..5eac876c73bb 100644 --- a/src/include/duckdb/optimizer/join_order/plan_enumerator.hpp +++ b/src/include/duckdb/optimizer/join_order/plan_enumerator.hpp @@ -30,8 +30,7 @@ class PlanEnumerator { public: explicit PlanEnumerator(QueryGraphManager &query_graph_manager, CostModel &cost_model, const QueryGraphEdges &query_graph) - : query_graph(query_graph), query_graph_manager(query_graph_manager), cost_model(cost_model), - full_plan_found(false), must_update_full_plan(false) { + : query_graph(query_graph), query_graph_manager(query_graph_manager), cost_model(cost_model) { } //! Perform the join order solving @@ -51,8 +50,6 @@ class PlanEnumerator { //! A map to store the optimal join plan found for a specific JoinRelationSet* reference_map_t> plans; - bool full_plan_found; - bool must_update_full_plan; unordered_set join_nodes_in_full_plan; unique_ptr CreateJoinTree(JoinRelationSet &set, diff --git a/src/optimizer/join_order/plan_enumerator.cpp b/src/optimizer/join_order/plan_enumerator.cpp index 035192643225..d5a38cda9754 100644 --- a/src/optimizer/join_order/plan_enumerator.cpp +++ b/src/optimizer/join_order/plan_enumerator.cpp @@ -149,7 +149,7 @@ unique_ptr PlanEnumerator::EmitPair(JoinRelationSet &left, JoinRelatio old_cost = entry->second->cost; } if (entry == plans.end() || new_cost < old_cost) { - // the new plan costs less than the old plan. Update our DP tree and cost tree + // the new plan costs less than the old plan. Update our DP table. plans[new_set] = std::move(new_plan); return CreateJoinNodeFromDPJoinNode(*plans[new_set]); } @@ -163,7 +163,7 @@ bool PlanEnumerator::TryEmitPair(JoinRelationSet &left, JoinRelationSet &right, // If a full plan is created, it's possible a node in the plan gets updated. When this happens, make sure you keep // emitting pairs until you emit another final plan. Another final plan is guaranteed to be produced because of // our symmetry guarantees. - if (pairs >= 10000 && !must_update_full_plan) { + if (pairs >= 10000) { // when the amount of pairs gets too large we exit the dynamic programming and resort to a greedy algorithm // FIXME: simple heuristic currently // at 10K pairs stop searching exactly and switch to heuristic @@ -195,7 +195,7 @@ bool PlanEnumerator::EmitCSG(JoinRelationSet &node) { D_ASSERT(neighbors[i] > neighbors[i + 1]); } - // Dphyp paper missiing this. + // Dphyp paper missing this. // Because we are traversing in reverse order, we need to add neighbors whose number is smaller than the current // node to exclusion_set // This avoids duplicated enumeration @@ -414,7 +414,6 @@ void PlanEnumerator::SolveJoinOrderApproximately() { best_left = smallest_index[0]; best_right = smallest_index[1]; - // UpdateDPTree(*best_connection); // the code below assumes best_right > best_left if (best_left > best_right) { std::swap(best_left, best_right); From 34ce07e62d1cfa6c8d7d34b28ed6cc8fd06632d3 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 18 Apr 2024 10:56:31 +0200 Subject: [PATCH 529/603] remove verify that is no longer called --- .../duckdb/optimizer/join_order/join_node.hpp | 1 - src/optimizer/join_order/join_node.cpp | 16 ---------------- 2 files changed, 17 deletions(-) diff --git a/src/include/duckdb/optimizer/join_order/join_node.hpp b/src/include/duckdb/optimizer/join_order/join_node.hpp index da884f30bade..9c9a54bfb591 100644 --- a/src/include/duckdb/optimizer/join_order/join_node.hpp +++ b/src/include/duckdb/optimizer/join_order/join_node.hpp @@ -78,7 +78,6 @@ class JoinNode { public: void Print(); string ToString(); - void Verify(); }; } // namespace duckdb diff --git a/src/optimizer/join_order/join_node.cpp b/src/optimizer/join_order/join_node.cpp index 1c3f8b21ad84..f4958563e4c2 100644 --- a/src/optimizer/join_order/join_node.cpp +++ b/src/optimizer/join_order/join_node.cpp @@ -38,22 +38,6 @@ void JoinNode::Print() { Printer::Print(ToString()); } -void JoinNode::Verify() { -#ifdef DEBUG - D_ASSERT(set.count >= 1); - idx_t left_count = 0, right_count = 0; - if (left) { - left->Verify(); - left_count = left->set.count; - } - if (right) { - right->Verify(); - right_count = right->set.count; - } - D_ASSERT(set.count == left_count + right_count || set.count == 1); -#endif -} - DPJoinNode::DPJoinNode(JoinRelationSet &set) : set(set), info(nullptr), is_leaf(true), left_set(set), right_set(set) { } From c5249bbe91df35b0189b7d8d24af1907d610d2f2 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 18 Apr 2024 10:55:05 +0200 Subject: [PATCH 530/603] Add CI check on capability to build duckdb in docker --- .github/workflows/DockerTests.yml | 59 +++++++++++++++++++++++++++++++ scripts/test_docker_images.sh | 4 +-- 2 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/DockerTests.yml diff --git a/.github/workflows/DockerTests.yml b/.github/workflows/DockerTests.yml new file mode 100644 index 000000000000..be2e8579f43b --- /dev/null +++ b/.github/workflows/DockerTests.yml @@ -0,0 +1,59 @@ +name: Docker tests +on: + workflow_call: + inputs: + override_git_describe: + type: string + git_ref: + type: string + skip_tests: + type: string + workflow_dispatch: + inputs: + override_git_describe: + type: string + git_ref: + type: string + skip_tests: + type: string + repository_dispatch: + push: + branches: + - '**' + - '!main' + - '!feature' + paths-ignore: + - '**' + - '!.github/workflows/DockerTests.yml' + - '!scripts/test_docker_images.sh' + pull_request: + types: [opened, reopened, ready_for_review] + paths-ignore: + - '**' + - '!.github/workflows/DockerTests.yml' + - '!scripts/test_docker_images.sh' + +concurrency: + group: docker-${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}-${{ inputs.override_git_describe }} + cancel-in-progress: true + +env: + GH_TOKEN: ${{ secrets.GH_TOKEN }} + OVERRIDE_GIT_DESCRIBE: ${{ inputs.override_git_describe }} + +jobs: + linux-x64-docker: + # Builds binaries for linux_amd64_gcc4 + name: Docker tests on Linux (x64) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.git_ref }} + + - name: Build + shell: bash + run: | + ./scripts/test_docker_images.sh diff --git a/scripts/test_docker_images.sh b/scripts/test_docker_images.sh index 904d4f90b529..c2217a8b6488 100755 --- a/scripts/test_docker_images.sh +++ b/scripts/test_docker_images.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash make clean -docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb alpine:latest <<< "apk add g++ git make cmake ninja && GEN=ninja make" 2>&1 -echo "alpine:latest completed" +docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb alpine:latest <<< "apk add g++ git make cmake ninja python3 && GEN=ninja make && make clean" 2>&1 +docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb ubuntu:20.04 <<< "apt-get update && export DEBIAN_FRONTEND=noninteractive && apt-get install g++ git make cmake ninja-build python3 -y && GEN=ninja make && make clean" 2>&1 From ee45dcc7ba69490f49d7c902dfc54a6b554fe6a0 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 18 Apr 2024 11:13:58 +0200 Subject: [PATCH 531/603] Move from linux/falloc.h to fcntl.h + _GNU_SOURCE --- src/common/local_file_system.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/common/local_file_system.cpp b/src/common/local_file_system.cpp index 50b39a32d63a..744bce887cb1 100644 --- a/src/common/local_file_system.cpp +++ b/src/common/local_file_system.cpp @@ -40,7 +40,11 @@ extern "C" WINBASEAPI BOOL WINAPI GetPhysicallyInstalledSystemMemory(PULONGLONG) #endif #if defined(__linux__) -#include +// See https://man7.org/linux/man-pages/man2/fallocate.2.html +#ifndef _GNU_SOURCE +#define _GNU_SOURCE /* See feature_test_macros(7) */ +#endif +#include #include // See e.g.: // https://opensource.apple.com/source/CarbonHeaders/CarbonHeaders-18.1/TargetConditionals.h.auto.html From fbec0e894527a72f760ebce9c60dabc18b2f7490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Thu, 18 Apr 2024 11:44:13 +0200 Subject: [PATCH 532/603] first pass done --- .../duckdb/storage/compression/alp/algorithm/alp.hpp | 6 +++--- .../duckdb/storage/compression/alp/alp_constants.hpp | 9 +++++---- src/include/duckdb/storage/compression/alp/alp_utils.hpp | 2 +- .../duckdb/storage/compression/alprd/algorithm/alprd.hpp | 3 ++- .../duckdb/storage/compression/alprd/alprd_analyze.hpp | 8 +++++--- src/storage/compression/dictionary_compression.cpp | 4 ++-- src/storage/compression/fsst.cpp | 2 +- src/storage/statistics/distinct_statistics.cpp | 2 +- 8 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/include/duckdb/storage/compression/alp/algorithm/alp.hpp b/src/include/duckdb/storage/compression/alp/algorithm/alp.hpp index a8c7e91f3e66..d0bf989755b2 100644 --- a/src/include/duckdb/storage/compression/alp/algorithm/alp.hpp +++ b/src/include/duckdb/storage/compression/alp/algorithm/alp.hpp @@ -107,10 +107,10 @@ struct AlpCompression { */ static int64_t NumberToInt64(T n) { if (IsImpossibleToEncode(n)) { - return AlpConstants::ENCODING_UPPER_LIMIT; + return NumericCast(AlpConstants::ENCODING_UPPER_LIMIT); } n = n + AlpTypedConstants::MAGIC_NUMBER - AlpTypedConstants::MAGIC_NUMBER; - return static_cast(n); + return NumericCast(n); } /* @@ -185,7 +185,7 @@ struct AlpCompression { // Evaluate factor/exponent compression size (we optimize for FOR) uint64_t delta = (static_cast(max_encoded_value) - static_cast(min_encoded_value)); - estimated_bits_per_value = std::ceil(std::log2(delta + 1)); + estimated_bits_per_value = NumericCast(std::ceil(std::log2(delta + 1))); estimated_compression_size += n_values * estimated_bits_per_value; estimated_compression_size += exceptions_count * (EXACT_TYPE_BITSIZE + (AlpConstants::EXCEPTION_POSITION_SIZE * 8)); diff --git a/src/include/duckdb/storage/compression/alp/alp_constants.hpp b/src/include/duckdb/storage/compression/alp/alp_constants.hpp index 55353dda1fb6..9a7a36f9136f 100644 --- a/src/include/duckdb/storage/compression/alp/alp_constants.hpp +++ b/src/include/duckdb/storage/compression/alp/alp_constants.hpp @@ -70,11 +70,12 @@ struct AlpTypedConstants { static constexpr float MAGIC_NUMBER = 12582912.0; //! 2^22 + 2^23 static constexpr uint8_t MAX_EXPONENT = 10; - static constexpr const float EXP_ARR[] = {1.0, 10.0, 100.0, 1000.0, 10000.0, 100000.0, - 1000000.0, 10000000.0, 100000000.0, 1000000000.0, 10000000000.0}; + static constexpr const float EXP_ARR[] = {1.0F, 10.0F, 100.0F, 1000.0F, + 10000.0F, 100000.0F, 1000000.0F, 10000000.0F, + 100000000.0F, 1000000000.0F, 10000000000.0F}; - static constexpr float FRAC_ARR[] = {1.0, 0.1, 0.01, 0.001, 0.0001, 0.00001, - 0.000001, 0.0000001, 0.00000001, 0.000000001, 0.0000000001}; + static constexpr float FRAC_ARR[] = {1.0F, 0.1F, 0.01F, 0.001F, 0.0001F, 0.00001F, + 0.000001F, 0.0000001F, 0.00000001F, 0.000000001F, 0.0000000001F}; }; template <> diff --git a/src/include/duckdb/storage/compression/alp/alp_utils.hpp b/src/include/duckdb/storage/compression/alp/alp_utils.hpp index b5e49a6f3027..75292b829d9b 100644 --- a/src/include/duckdb/storage/compression/alp/alp_utils.hpp +++ b/src/include/duckdb/storage/compression/alp/alp_utils.hpp @@ -42,7 +42,7 @@ class AlpUtils { //! We sample equidistant values within a vector; to do this we jump a fixed number of values uint32_t n_sampled_increments = MaxValue( 1, UnsafeNumericCast(std::ceil((double)n_lookup_values / AlpConstants::SAMPLES_PER_VECTOR))); - uint32_t n_sampled_values = std::ceil((double)n_lookup_values / n_sampled_increments); + uint32_t n_sampled_values = NumericCast(std::ceil((double)n_lookup_values / n_sampled_increments)); D_ASSERT(n_sampled_values < AlpConstants::ALP_VECTOR_SIZE); AlpSamplingParameters sampling_params = {n_lookup_values, n_sampled_increments, n_sampled_values}; diff --git a/src/include/duckdb/storage/compression/alprd/algorithm/alprd.hpp b/src/include/duckdb/storage/compression/alprd/algorithm/alprd.hpp index 66d8262aebc9..ce99e1a0148a 100644 --- a/src/include/duckdb/storage/compression/alprd/algorithm/alprd.hpp +++ b/src/include/duckdb/storage/compression/alprd/algorithm/alprd.hpp @@ -105,7 +105,8 @@ struct AlpRDCompression { // The left parts bit width after compression is determined by how many elements are in the dictionary uint64_t actual_dictionary_size = MinValue(AlpRDConstants::MAX_DICTIONARY_SIZE, left_parts_sorted_repetitions.size()); - uint8_t left_bit_width = MaxValue(1, std::ceil(std::log2(actual_dictionary_size))); + uint8_t left_bit_width = + MaxValue(1, NumericCast(std::ceil(std::log2(actual_dictionary_size)))); if (PERSIST_DICT) { for (idx_t dict_idx = 0; dict_idx < actual_dictionary_size; dict_idx++) { diff --git a/src/include/duckdb/storage/compression/alprd/alprd_analyze.hpp b/src/include/duckdb/storage/compression/alprd/alprd_analyze.hpp index e88fdae61b34..e37d873ac525 100644 --- a/src/include/duckdb/storage/compression/alprd/alprd_analyze.hpp +++ b/src/include/duckdb/storage/compression/alprd/alprd_analyze.hpp @@ -126,13 +126,15 @@ idx_t AlpRDFinalAnalyze(AnalyzeState &state) { //! Overhead per vector: Pointer to data + Exceptions count double per_vector_overhead = AlpRDConstants::METADATA_POINTER_SIZE + AlpRDConstants::EXCEPTIONS_COUNT_SIZE; - uint32_t n_vectors = std::ceil((double)analyze_state.total_values_count / AlpRDConstants::ALP_VECTOR_SIZE); + uint32_t n_vectors = + NumericCast(std::ceil((double)analyze_state.total_values_count / AlpRDConstants::ALP_VECTOR_SIZE)); auto estimated_size = (estimed_compressed_bytes * factor_of_sampling) + (n_vectors * per_vector_overhead); - uint32_t estimated_n_blocks = std::ceil(estimated_size / (Storage::BLOCK_SIZE - per_segment_overhead)); + uint32_t estimated_n_blocks = + NumericCast(std::ceil(estimated_size / (Storage::BLOCK_SIZE - per_segment_overhead))); auto final_analyze_size = estimated_size + (estimated_n_blocks * per_segment_overhead); - return final_analyze_size; + return NumericCast(final_analyze_size); } } // namespace duckdb diff --git a/src/storage/compression/dictionary_compression.cpp b/src/storage/compression/dictionary_compression.cpp index 79ccc64471b9..58529e72aa19 100644 --- a/src/storage/compression/dictionary_compression.cpp +++ b/src/storage/compression/dictionary_compression.cpp @@ -86,7 +86,7 @@ typedef struct { } dictionary_compression_header_t; struct DictionaryCompressionStorage { - static constexpr float MINIMUM_COMPRESSION_RATIO = 1.2; + static constexpr float MINIMUM_COMPRESSION_RATIO = 1.2F; static constexpr uint16_t DICTIONARY_HEADER_SIZE = sizeof(dictionary_compression_header_t); static constexpr size_t COMPACTION_FLUSH_LIMIT = (size_t)Storage::BLOCK_SIZE / 5 * 4; @@ -402,7 +402,7 @@ idx_t DictionaryCompressionStorage::StringFinalAnalyze(AnalyzeState &state_p) { auto req_space = RequiredSpace(state.current_tuple_count, state.current_unique_count, state.current_dict_size, width); - return MINIMUM_COMPRESSION_RATIO * (state.segment_count * Storage::BLOCK_SIZE + req_space); + return NumericCast(MINIMUM_COMPRESSION_RATIO * (state.segment_count * Storage::BLOCK_SIZE + req_space)); } //===--------------------------------------------------------------------===// diff --git a/src/storage/compression/fsst.cpp b/src/storage/compression/fsst.cpp index 02474963f31d..fcccab4af95a 100644 --- a/src/storage/compression/fsst.cpp +++ b/src/storage/compression/fsst.cpp @@ -191,7 +191,7 @@ idx_t FSSTStorage::StringFinalAnalyze(AnalyzeState &state_p) { auto estimated_size = estimated_base_size + symtable_size; - return estimated_size * MINIMUM_COMPRESSION_RATIO; + return NumericCast(estimated_size * MINIMUM_COMPRESSION_RATIO); } //===--------------------------------------------------------------------===// diff --git a/src/storage/statistics/distinct_statistics.cpp b/src/storage/statistics/distinct_statistics.cpp index 7d4e6e92a346..69f533071733 100644 --- a/src/storage/statistics/distinct_statistics.cpp +++ b/src/storage/statistics/distinct_statistics.cpp @@ -64,7 +64,7 @@ idx_t DistinctStatistics::GetCount() const { double u1 = pow(u / s, 2) * u; // Estimate total uniques using Good Turing Estimation - idx_t estimate = u + u1 / s * (n - s); + idx_t estimate = NumericCast(u + u1 / s * (n - s)); return MinValue(estimate, total_count); } From fb4c967a64ad5fe5084a87550b8ee96323802c3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Thu, 18 Apr 2024 12:07:46 +0200 Subject: [PATCH 533/603] whee --- src/common/operator/cast_operators.cpp | 2 +- src/common/types.cpp | 2 +- src/include/duckdb/common/operator/numeric_cast.hpp | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/common/operator/cast_operators.cpp b/src/common/operator/cast_operators.cpp index 843acc70527a..02e30431969c 100644 --- a/src/common/operator/cast_operators.cpp +++ b/src/common/operator/cast_operators.cpp @@ -2486,7 +2486,7 @@ bool DoubleToDecimalCast(SRC input, DST &result, CastParameters ¶meters, uin HandleCastError::AssignError(error, parameters); return false; } - result = Cast::Operation(UnsafeNumericCast(value)); + result = Cast::Operation(static_cast(value)); return true; } diff --git a/src/common/types.cpp b/src/common/types.cpp index b54c47e78b0d..45bb5edab003 100644 --- a/src/common/types.cpp +++ b/src/common/types.cpp @@ -1110,7 +1110,7 @@ bool ApproxEqual(float ldecimal, float rdecimal) { if (!Value::FloatIsFinite(ldecimal) || !Value::FloatIsFinite(rdecimal)) { return ldecimal == rdecimal; } - auto epsilon = UnsafeNumericCast(std::fabs(rdecimal) * 0.01 + 0.00000001); + float epsilon = static_cast(std::fabs(rdecimal) * 0.01 + 0.00000001); return std::fabs(ldecimal - rdecimal) <= epsilon; } diff --git a/src/include/duckdb/common/operator/numeric_cast.hpp b/src/include/duckdb/common/operator/numeric_cast.hpp index b6d3b6742f80..f48839c4175e 100644 --- a/src/include/duckdb/common/operator/numeric_cast.hpp +++ b/src/include/duckdb/common/operator/numeric_cast.hpp @@ -75,7 +75,7 @@ bool TryCastWithOverflowCheckFloat(SRC value, T &result, SRC min, SRC max) { return false; } // PG FLOAT => INT casts use statistical rounding. - result = UnsafeNumericCast(std::nearbyint(value)); + result = static_cast(std::nearbyint(value)); return true; } @@ -182,7 +182,7 @@ bool TryCastWithOverflowCheck(double input, float &result) { return true; } auto res = float(input); - if (!Value::DoubleIsFinite(input)) { + if (!Value::FloatIsFinite(res)) { return false; } result = res; From 9591a16f39083ac3333b1ab86ea9fc346138b927 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 18 Apr 2024 12:46:40 +0200 Subject: [PATCH 534/603] base of is castable --- .../csv_scanner/sniffer/type_detection.cpp | 52 +++++++++++++++++++ .../operator/csv_scanner/csv_sniffer.hpp | 2 + 2 files changed, 54 insertions(+) diff --git a/src/execution/operator/csv_scanner/sniffer/type_detection.cpp b/src/execution/operator/csv_scanner/sniffer/type_detection.cpp index 2b5c680d8a93..d0720e5c76d6 100644 --- a/src/execution/operator/csv_scanner/sniffer/type_detection.cpp +++ b/src/execution/operator/csv_scanner/sniffer/type_detection.cpp @@ -121,6 +121,58 @@ void CSVSniffer::SetDateFormat(CSVStateMachine &candidate, const string &format_ candidate.dialect_options.date_format[sql_type].Set(strpformat, false); } +inline bool CSVSniffer::ValueIsCastable(string_t &value, LogicalType &type) { + bool success = true; + switch (type.id()) { + case LogicalTypeId::TINYINT: + return TrySimpleIntegerCast(value_ptr, size, static_cast(vector_ptr[chunk_col_id])[number_of_rows], + false); + case LogicalTypeId::SMALLINT: + return TrySimpleIntegerCast(value_ptr, size, static_cast(vector_ptr[chunk_col_id])[number_of_rows], + false); + case LogicalTypeId::INTEGER: + return TrySimpleIntegerCast(value_ptr, size, + static_cast(vector_ptr[chunk_col_id])[number_of_rows], false); + case LogicalTypeId::BIGINT: + return TrySimpleIntegerCast(value_ptr, size, + static_cast(vector_ptr[chunk_col_id])[number_of_rows], false); + case LogicalTypeId::UTINYINT: + return TrySimpleIntegerCast( + value_ptr, size, static_cast(vector_ptr[chunk_col_id])[number_of_rows], false); + case LogicalTypeId::USMALLINT: + return TrySimpleIntegerCast( + value_ptr, size, static_cast(vector_ptr[chunk_col_id])[number_of_rows], false); + case LogicalTypeId::UINTEGER: + return TrySimpleIntegerCast( + value_ptr, size, static_cast(vector_ptr[chunk_col_id])[number_of_rows], false); + case LogicalTypeId::UBIGINT: + return TrySimpleIntegerCast( + value_ptr, size, static_cast(vector_ptr[chunk_col_id])[number_of_rows], false); + case LogicalTypeId::DOUBLE: + return TryDoubleCast(value_ptr, size, static_cast(vector_ptr[chunk_col_id])[number_of_rows], + false, state_machine.options.decimal_separator[0]); + case LogicalTypeId::FLOAT: + return TryDoubleCast(value_ptr, size, static_cast(vector_ptr[chunk_col_id])[number_of_rows], + false, state_machine.options.decimal_separator[0]); + case LogicalTypeId::DATE: { + idx_t pos; + bool special; + return Date::TryConvertDate(value_ptr, size, pos, + static_cast(vector_ptr[chunk_col_id])[number_of_rows], special, false); + } + case LogicalTypeId::TIMESTAMP: + return Timestamp::TryConvertTimestamp( + value_ptr, size, static_cast(vector_ptr[chunk_col_id])[number_of_rows]) == + TimestampCastResult::SUCCESS; + + case LogicalTypeId::VARCHAR: + return true; + + case default:{ + // We do Value Try Cast for non-basic types. + } + } +}; void CSVSniffer::InitializeDateAndTimeStampDetection(CSVStateMachine &candidate, const string &separator, const LogicalType &sql_type) { auto &format_candidate = format_candidates[sql_type.id()]; diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp index 5e497e8dd4b1..8a706b7f0ecb 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp @@ -153,6 +153,8 @@ class CSVSniffer { //! Functions that performs detection for date and timestamp formats void DetectDateAndTimeStampFormats(CSVStateMachine &candidate, const LogicalType &sql_type, const string &separator, string_t &dummy_val); + //! If a string_t value can be cast to a type + inline bool ValueIsCastable(string_t &value, LogicalType &type); //! Variables for Type Detection //! Format Candidates for Date and Timestamp Types From e3ae8be9d3a520b44ffad76f4c5f8b88b87e9d63 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 18 Apr 2024 13:00:56 +0200 Subject: [PATCH 535/603] Remove bound_defaults from BoundCreateTableInfo --- src/catalog/catalog_entry/duck_table_entry.cpp | 6 +++--- src/include/duckdb/planner/binder.hpp | 2 ++ .../parsed_data/bound_create_table_info.hpp | 2 -- .../binder/statement/bind_create_table.cpp | 15 +++++++++++---- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/catalog/catalog_entry/duck_table_entry.cpp b/src/catalog/catalog_entry/duck_table_entry.cpp index eb7be8d7a451..90c4de3ca0b7 100644 --- a/src/catalog/catalog_entry/duck_table_entry.cpp +++ b/src/catalog/catalog_entry/duck_table_entry.cpp @@ -342,9 +342,9 @@ unique_ptr DuckTableEntry::AddColumn(ClientContext &context, AddCo create_info->columns.AddColumn(std::move(col)); auto binder = Binder::CreateBinder(context); - auto bound_create_info = binder->BindCreateTableInfo(std::move(create_info), schema); - auto new_storage = - make_shared(context, *storage, info.new_column, *bound_create_info->bound_defaults.back()); + vector> bound_defaults; + auto bound_create_info = binder->BindCreateTableInfo(std::move(create_info), schema, bound_defaults); + auto new_storage = make_shared(context, *storage, info.new_column, *bound_defaults.back()); return make_uniq(catalog, schema, *bound_create_info, new_storage); } diff --git a/src/include/duckdb/planner/binder.hpp b/src/include/duckdb/planner/binder.hpp index ed646e6b04b7..9d992fb407e7 100644 --- a/src/include/duckdb/planner/binder.hpp +++ b/src/include/duckdb/planner/binder.hpp @@ -119,6 +119,8 @@ class Binder : public std::enable_shared_from_this { unique_ptr BindCreateTableInfo(unique_ptr info); unique_ptr BindCreateTableInfo(unique_ptr info, SchemaCatalogEntry &schema); + unique_ptr BindCreateTableInfo(unique_ptr info, SchemaCatalogEntry &schema, + vector> &bound_defaults); void BindCreateViewInfo(CreateViewInfo &base); SchemaCatalogEntry &BindSchema(CreateInfo &info); diff --git a/src/include/duckdb/planner/parsed_data/bound_create_table_info.hpp b/src/include/duckdb/planner/parsed_data/bound_create_table_info.hpp index fe71dfbda6fe..c7a1aac57705 100644 --- a/src/include/duckdb/planner/parsed_data/bound_create_table_info.hpp +++ b/src/include/duckdb/planner/parsed_data/bound_create_table_info.hpp @@ -38,8 +38,6 @@ struct BoundCreateTableInfo { vector> constraints; //! List of bound constraints on the table vector> bound_constraints; - //! Bound default values - vector> bound_defaults; //! Dependents of the table (in e.g. default values) LogicalDependencyList dependencies; //! The existing table data on disk (if any) diff --git a/src/planner/binder/statement/bind_create_table.cpp b/src/planner/binder/statement/bind_create_table.cpp index 4564cb0f04c6..780ef2380497 100644 --- a/src/planner/binder/statement/bind_create_table.cpp +++ b/src/planner/binder/statement/bind_create_table.cpp @@ -239,8 +239,8 @@ static void ExtractExpressionDependencies(Expression &expr, LogicalDependencyLis expr, [&](Expression &child) { ExtractExpressionDependencies(child, dependencies); }); } -static void ExtractDependencies(BoundCreateTableInfo &info) { - for (auto &default_value : info.bound_defaults) { +static void ExtractDependencies(BoundCreateTableInfo &info, vector> &bound_defaults) { + for (auto &default_value : bound_defaults) { if (default_value) { ExtractExpressionDependencies(*default_value, info.dependencies); } @@ -252,7 +252,14 @@ static void ExtractDependencies(BoundCreateTableInfo &info) { } } } + unique_ptr Binder::BindCreateTableInfo(unique_ptr info, SchemaCatalogEntry &schema) { + vector> bound_defaults; + return BindCreateTableInfo(std::move(info), schema, bound_defaults); +} + +unique_ptr Binder::BindCreateTableInfo(unique_ptr info, SchemaCatalogEntry &schema, + vector> &bound_defaults) { auto &base = info->Cast(); auto result = make_uniq(schema, std::move(info)); if (base.query) { @@ -279,10 +286,10 @@ unique_ptr Binder::BindCreateTableInfo(unique_ptrbound_defaults); + BindDefaultValues(base.columns, bound_defaults); } // extract dependencies from any default values or CHECK constraints - ExtractDependencies(*result); + ExtractDependencies(*result, bound_defaults); if (base.columns.PhysicalColumnCount() == 0) { throw BinderException("Creating a table without physical (non-generated) columns is not supported"); From 5adcf391dac6d12359e48e4884426901a17e4542 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 18 Apr 2024 13:14:01 +0200 Subject: [PATCH 536/603] Explicitly define DUCKDB_MODULE_DIR and use that in invoking scripts --- CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0064a8b4445d..c6678fdf710b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,6 +24,8 @@ project(DuckDB) find_package(Threads REQUIRED) +set(DUCKDB_MODULE_BASE_DIR "${CMAKE_CURRENT_LIST_DIR}") + set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set (CMAKE_CXX_STANDARD 11) @@ -827,7 +829,7 @@ function(build_loadable_extension_directory NAME OUTPUT_DIRECTORY EXTENSION_VERS TARGET ${TARGET_NAME} POST_BUILD COMMAND - ${CMAKE_COMMAND} -DEXTENSION=$ -DPLATFORM_FILE=${DuckDB_BINARY_DIR}/duckdb_platform_out -DDUCKDB_VERSION="${DUCKDB_NORMALIZED_VERSION}" -DEXTENSION_VERSION="${EXTENSION_VERSION}" -DNULL_FILE=${CMAKE_CURRENT_FUNCTION_LIST_DIR}/scripts/null.txt -P ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/scripts/append_metadata.cmake + ${CMAKE_COMMAND} -DEXTENSION=$ -DPLATFORM_FILE=${DuckDB_BINARY_DIR}/duckdb_platform_out -DDUCKDB_VERSION="${DUCKDB_NORMALIZED_VERSION}" -DEXTENSION_VERSION="${EXTENSION_VERSION}" -DNULL_FILE=${DUCKDB_MODULE_BASE_DIR}/scripts/null.txt -P ${DUCKDB_MODULE_BASE_DIR}/scripts/append_metadata.cmake ) add_dependencies(${TARGET_NAME} duckdb_platform) if (NOT EXTENSION_CONFIG_BUILD AND NOT ${EXTENSION_TESTS_ONLY} AND NOT CLANG_TIDY) From d2adbc8bb91008f3402143282292ae932c86682c Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 18 Apr 2024 13:38:12 +0200 Subject: [PATCH 537/603] Properly avoid build-time dependency on Python Triggered by https://github.com/duckdb/duckdb/pull/11710 --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0064a8b4445d..d9b7f2045b5a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -715,6 +715,8 @@ endif() set(LOCAL_EXTENSION_REPO FALSE) if (NOT EXTENSION_CONFIG_BUILD AND NOT ${EXTENSION_TESTS_ONLY} AND NOT CLANG_TIDY) if (NOT Python3_FOUND) + add_custom_target( + duckdb_local_extension_repo ALL) MESSAGE(STATUS "Could not find python3, create extension directory step will be skipped") else() add_custom_target( From c9c03b2c8731c6908f49f5f108ae31813d167cad Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 18 Apr 2024 14:01:49 +0200 Subject: [PATCH 538/603] add duckdb::make_shared with static assert --- src/include/duckdb/common/helper.hpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/include/duckdb/common/helper.hpp b/src/include/duckdb/common/helper.hpp index 52e0dd65ea63..d40395855a8e 100644 --- a/src/include/duckdb/common/helper.hpp +++ b/src/include/duckdb/common/helper.hpp @@ -152,6 +152,14 @@ static duckdb::unique_ptr make_unique(ARGS&&... __args) { // NOLINT: mimic st return unique_ptr(new T(std::forward(__args)...)); } +template +static duckdb::shared_ptr make_shared(ARGS&&... __args) { // NOLINT: mimic std style +#ifndef DUCKDB_ENABLE_DEPRECATED_API + static_assert(sizeof(T) == 0, "Use make_shared_ptr instead of make_shared!"); +#endif // DUCKDB_ENABLE_DEPRECATED_API + return shared_ptr(new T(std::forward(__args)...)); +} + template constexpr T MaxValue(T a, T b) { return a > b ? a : b; From 9b08e27f107812cbce0f6d09d3bf0598178b53ee Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 18 Apr 2024 14:25:15 +0200 Subject: [PATCH 539/603] Fixing a bunch of these casts --- .../csv_scanner/sniffer/type_detection.cpp | 91 +++++++++++-------- .../operator/csv_scanner/csv_sniffer.hpp | 2 +- 2 files changed, 55 insertions(+), 38 deletions(-) diff --git a/src/execution/operator/csv_scanner/sniffer/type_detection.cpp b/src/execution/operator/csv_scanner/sniffer/type_detection.cpp index d0720e5c76d6..81e8e16323b2 100644 --- a/src/execution/operator/csv_scanner/sniffer/type_detection.cpp +++ b/src/execution/operator/csv_scanner/sniffer/type_detection.cpp @@ -2,6 +2,8 @@ #include "duckdb/execution/operator/csv_scanner/csv_sniffer.hpp" #include "duckdb/common/algorithm.hpp" #include "duckdb/common/string.hpp" +#include "duckdb/common/operator/integer_cast_operator.hpp" +#include "duckdb/common/operator/double_cast_operator.hpp" namespace duckdb { struct TryCastFloatingOperator { @@ -121,55 +123,70 @@ void CSVSniffer::SetDateFormat(CSVStateMachine &candidate, const string &format_ candidate.dialect_options.date_format[sql_type].Set(strpformat, false); } -inline bool CSVSniffer::ValueIsCastable(string_t &value, LogicalType &type) { +inline bool CSVSniffer::ValueIsCastable(const char *value_ptr, const idx_t value_size, LogicalType &type, + bool is_null) { + if (is_null) { + return true; + } bool success = true; switch (type.id()) { - case LogicalTypeId::TINYINT: - return TrySimpleIntegerCast(value_ptr, size, static_cast(vector_ptr[chunk_col_id])[number_of_rows], - false); - case LogicalTypeId::SMALLINT: - return TrySimpleIntegerCast(value_ptr, size, static_cast(vector_ptr[chunk_col_id])[number_of_rows], - false); - case LogicalTypeId::INTEGER: - return TrySimpleIntegerCast(value_ptr, size, - static_cast(vector_ptr[chunk_col_id])[number_of_rows], false); - case LogicalTypeId::BIGINT: - return TrySimpleIntegerCast(value_ptr, size, - static_cast(vector_ptr[chunk_col_id])[number_of_rows], false); - case LogicalTypeId::UTINYINT: - return TrySimpleIntegerCast( - value_ptr, size, static_cast(vector_ptr[chunk_col_id])[number_of_rows], false); - case LogicalTypeId::USMALLINT: - return TrySimpleIntegerCast( - value_ptr, size, static_cast(vector_ptr[chunk_col_id])[number_of_rows], false); - case LogicalTypeId::UINTEGER: - return TrySimpleIntegerCast( - value_ptr, size, static_cast(vector_ptr[chunk_col_id])[number_of_rows], false); - case LogicalTypeId::UBIGINT: - return TrySimpleIntegerCast( - value_ptr, size, static_cast(vector_ptr[chunk_col_id])[number_of_rows], false); - case LogicalTypeId::DOUBLE: - return TryDoubleCast(value_ptr, size, static_cast(vector_ptr[chunk_col_id])[number_of_rows], - false, state_machine.options.decimal_separator[0]); - case LogicalTypeId::FLOAT: - return TryDoubleCast(value_ptr, size, static_cast(vector_ptr[chunk_col_id])[number_of_rows], - false, state_machine.options.decimal_separator[0]); + case LogicalTypeId::TINYINT: { + int8_t dummy_value; + return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, false); + } + case LogicalTypeId::SMALLINT: { + int16_t dummy_value; + return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, false); + } + case LogicalTypeId::INTEGER: { + int32_t dummy_value; + return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, false); + } + case LogicalTypeId::BIGINT: { + int64_t dummy_value; + return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, false); + } + case LogicalTypeId::UTINYINT: { + uint8_t dummy_value; + return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, false); + } + case LogicalTypeId::USMALLINT: { + uint16_t dummy_value; + return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, false); + } + case LogicalTypeId::UINTEGER: { + uint32_t dummy_value; + return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, false); + } + case LogicalTypeId::UBIGINT: { + uint64_t dummy_value; + return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, false); + } + case LogicalTypeId::DOUBLE: { + double dummy_value; + return TryDoubleCast(value_ptr, value_size, dummy_value, false, options.decimal_separator[0]); + } + case LogicalTypeId::FLOAT: { + float dummy_value; + return TryDoubleCast(value_ptr, value_size, dummy_value, false, options.decimal_separator[0]); + } case LogicalTypeId::DATE: { idx_t pos; bool special; - return Date::TryConvertDate(value_ptr, size, pos, - static_cast(vector_ptr[chunk_col_id])[number_of_rows], special, false); + return Date::TryConvertDate(value_ptr, value_size, pos, + static_cast(vector_ptr[chunk_col_id])[number_of_rows], special, false); } case LogicalTypeId::TIMESTAMP: - return Timestamp::TryConvertTimestamp( - value_ptr, size, static_cast(vector_ptr[chunk_col_id])[number_of_rows]) == - TimestampCastResult::SUCCESS; + return Timestamp::TryConvertTimestamp(value_ptr, value_size, + static_cast(vector_ptr[chunk_col_id])[number_of_rows]) == + TimestampCastResult::SUCCESS; case LogicalTypeId::VARCHAR: return true; - case default:{ + default: { // We do Value Try Cast for non-basic types. + return true; } } }; diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp index 8a706b7f0ecb..427912c9adfb 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp @@ -154,7 +154,7 @@ class CSVSniffer { void DetectDateAndTimeStampFormats(CSVStateMachine &candidate, const LogicalType &sql_type, const string &separator, string_t &dummy_val); //! If a string_t value can be cast to a type - inline bool ValueIsCastable(string_t &value, LogicalType &type); + inline bool ValueIsCastable(const char *value_ptr, const idx_t value_size, LogicalType &type, bool is_null); //! Variables for Type Detection //! Format Candidates for Date and Timestamp Types From bc183fe66fd86a3c3298a55b2fab3d1b5a4eb6bb Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 18 Apr 2024 14:37:35 +0200 Subject: [PATCH 540/603] cast for all basic types --- .../csv_scanner/sniffer/type_detection.cpp | 81 ++++++++----------- .../operator/csv_scanner/csv_sniffer.hpp | 5 +- 2 files changed, 36 insertions(+), 50 deletions(-) diff --git a/src/execution/operator/csv_scanner/sniffer/type_detection.cpp b/src/execution/operator/csv_scanner/sniffer/type_detection.cpp index 81e8e16323b2..533046b9650a 100644 --- a/src/execution/operator/csv_scanner/sniffer/type_detection.cpp +++ b/src/execution/operator/csv_scanner/sniffer/type_detection.cpp @@ -86,36 +86,6 @@ string GenerateDateFormat(const string &separator, const char *format_template) return result; } -bool CSVSniffer::TryCastValue(const DialectOptions &dialect_options, const string &decimal_separator, - const string_t &value, const LogicalType &sql_type, bool is_null) { - if (is_null) { - return true; - } - if (!dialect_options.date_format.find(LogicalTypeId::DATE)->second.GetValue().Empty() && - sql_type.id() == LogicalTypeId::DATE) { - date_t result; - string error_message; - return dialect_options.date_format.find(LogicalTypeId::DATE) - ->second.GetValue() - .TryParseDate(value, result, error_message); - } - if (!dialect_options.date_format.find(LogicalTypeId::TIMESTAMP)->second.GetValue().Empty() && - sql_type.id() == LogicalTypeId::TIMESTAMP) { - timestamp_t result; - string error_message; - return dialect_options.date_format.find(LogicalTypeId::TIMESTAMP) - ->second.GetValue() - .TryParseTimestamp(value, result, error_message); - } - if (decimal_separator != "." && (sql_type.id() == LogicalTypeId::DOUBLE)) { - return TryCastFloatingOperator::Operation(value); - } - Value new_value; - string error_message; - Value str_value(value); - return str_value.TryCastAs(buffer_manager->context, sql_type, new_value, &error_message, true); -} - void CSVSniffer::SetDateFormat(CSVStateMachine &candidate, const string &format_specifier, const LogicalTypeId &sql_type) { StrpTimeFormat strpformat; @@ -123,12 +93,13 @@ void CSVSniffer::SetDateFormat(CSVStateMachine &candidate, const string &format_ candidate.dialect_options.date_format[sql_type].Set(strpformat, false); } -inline bool CSVSniffer::ValueIsCastable(const char *value_ptr, const idx_t value_size, LogicalType &type, - bool is_null) { +bool CSVSniffer::IsCasteable(const string_t value, const LogicalType &type, const DialectOptions &dialect_options, + const bool is_null) { if (is_null) { return true; } - bool success = true; + auto value_ptr = value.GetData(); + auto value_size = value.GetSize(); switch (type.id()) { case LogicalTypeId::TINYINT: { int8_t dummy_value; @@ -171,22 +142,39 @@ inline bool CSVSniffer::ValueIsCastable(const char *value_ptr, const idx_t value return TryDoubleCast(value_ptr, value_size, dummy_value, false, options.decimal_separator[0]); } case LogicalTypeId::DATE: { - idx_t pos; - bool special; - return Date::TryConvertDate(value_ptr, value_size, pos, - static_cast(vector_ptr[chunk_col_id])[number_of_rows], special, false); + if (!dialect_options.date_format.find(LogicalTypeId::DATE)->second.GetValue().Empty()) { + date_t result; + string error_message; + return dialect_options.date_format.find(LogicalTypeId::DATE) + ->second.GetValue() + .TryParseDate(value, result, error_message); + } else { + idx_t pos; + bool special; + date_t dummy_value; + return Date::TryConvertDate(value_ptr, value_size, pos, dummy_value, special, false); + } + } + case LogicalTypeId::TIMESTAMP: { + if (!dialect_options.date_format.find(LogicalTypeId::TIMESTAMP)->second.GetValue().Empty()) { + timestamp_t result; + string error_message; + return dialect_options.date_format.find(LogicalTypeId::TIMESTAMP) + ->second.GetValue() + .TryParseTimestamp(value, result, error_message); + } else { + timestamp_t dummy_value; + return Timestamp::TryConvertTimestamp(value_ptr, value_size, dummy_value) == TimestampCastResult::SUCCESS; + } } - case LogicalTypeId::TIMESTAMP: - return Timestamp::TryConvertTimestamp(value_ptr, value_size, - static_cast(vector_ptr[chunk_col_id])[number_of_rows]) == - TimestampCastResult::SUCCESS; - case LogicalTypeId::VARCHAR: return true; - default: { // We do Value Try Cast for non-basic types. - return true; + Value new_value; + string error_message; + Value str_value(value); + return str_value.TryCastAs(buffer_manager->context, type, new_value, &error_message, true); } } }; @@ -331,9 +319,8 @@ void CSVSniffer::DetectTypes() { // Nothing to convert it to continue; } - if (TryCastValue(sniffing_state_machine.dialect_options, - sniffing_state_machine.options.decimal_separator, vector_data[row_idx], sql_type, - !null_mask.RowIsValid(row_idx))) { + if (IsCasteable(vector_data[row_idx], sql_type, sniffing_state_machine.dialect_options, + !null_mask.RowIsValid(row_idx))) { break; } else { if (row_idx != start_idx_detection && cur_top_candidate == LogicalType::BOOLEAN) { diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp index 427912c9adfb..8e8796daaf09 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp @@ -143,8 +143,6 @@ class CSVSniffer { void DetectTypes(); //! Change the date format for the type to the string //! Try to cast a string value to the specified sql type - bool TryCastValue(const DialectOptions &dialect_options, const string &decimal_separator, const string_t &value, - const LogicalType &sql_type, bool is_null); void SetDateFormat(CSVStateMachine &candidate, const string &format_specifier, const LogicalTypeId &sql_type); //! Function that initialized the necessary variables used for date and timestamp detection @@ -154,7 +152,8 @@ class CSVSniffer { void DetectDateAndTimeStampFormats(CSVStateMachine &candidate, const LogicalType &sql_type, const string &separator, string_t &dummy_val); //! If a string_t value can be cast to a type - inline bool ValueIsCastable(const char *value_ptr, const idx_t value_size, LogicalType &type, bool is_null); + inline bool IsCasteable(const string_t value, const LogicalType &type, const DialectOptions &dialect_options, + const bool is_null); //! Variables for Type Detection //! Format Candidates for Date and Timestamp Types From 8be31e1758097da59ee2257e5185217be6c25c28 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 18 Apr 2024 14:49:42 +0200 Subject: [PATCH 541/603] small header fix --- .../operator/csv_scanner/sniffer/header_detection.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/execution/operator/csv_scanner/sniffer/header_detection.cpp b/src/execution/operator/csv_scanner/sniffer/header_detection.cpp index 88975ad769c3..591aacdc7532 100644 --- a/src/execution/operator/csv_scanner/sniffer/header_detection.cpp +++ b/src/execution/operator/csv_scanner/sniffer/header_detection.cpp @@ -129,8 +129,8 @@ bool CSVSniffer::DetectHeaderWithSetColumn() { if (!dummy_val.IsNull()) { val = StringValue::Get(dummy_val); } - if (!TryCastValue(options.dialect_options, options.decimal_separator, val, sql_type, - dummy_val.IsNull())) { + + if (!IsCasteable(val, sql_type, options.dialect_options, dummy_val.IsNull())) { first_row_consistent = false; } } @@ -186,8 +186,7 @@ void CSVSniffer::DetectHeader() { if (!dummy_val.IsNull()) { val = StringValue::Get(dummy_val); } - if (!TryCastValue(sniffer_state_machine.dialect_options, - sniffer_state_machine.options.decimal_separator, val, sql_type, dummy_val.IsNull())) { + if (!IsCasteable(val, sql_type, options.dialect_options, dummy_val.IsNull())) { first_row_consistent = false; } } From 9136c3de9a4248649a00566f3890e3dcf7f582b5 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 18 Apr 2024 14:53:13 +0200 Subject: [PATCH 542/603] Upload staging: from 'git describe --tags --long' to 'git log -1 --format=%h' This makes so that target is independent of actual status (pre or post tagging) --- .github/workflows/Python.yml | 8 ++++---- .github/workflows/StagedUpload.yml | 3 ++- .github/workflows/TwineUpload.yml | 2 +- scripts/upload-assets-to-staging.sh | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/Python.yml b/.github/workflows/Python.yml index 9d8d43efa313..ced2a2c88304 100644 --- a/.github/workflows/Python.yml +++ b/.github/workflows/Python.yml @@ -81,7 +81,7 @@ jobs: elif [[ -z "${{ inputs.override_git_describe }}" ]]; then echo "No override_git_describe provided" else - echo "UPLOAD_ASSETS_TO_STAGING_TARGET=$(git describe --tags --long)" >> "$GITHUB_ENV" + echo "UPLOAD_ASSETS_TO_STAGING_TARGET=$(git log -1 --format=%h)" >> "$GITHUB_ENV" echo "override_git_describe ${{ inputs.override_git_describe }}: add tag" git tag ${{ inputs.override_git_describe }} fi @@ -212,7 +212,7 @@ jobs: elif [[ -z "${{ inputs.override_git_describe }}" ]]; then echo "No override_git_describe provided" else - echo "UPLOAD_ASSETS_TO_STAGING_TARGET=$(git describe --tags --long)" >> "$GITHUB_ENV" + echo "UPLOAD_ASSETS_TO_STAGING_TARGET=$(git log -1 --format=%h)" >> "$GITHUB_ENV" echo "override_git_describe ${{ inputs.override_git_describe }}: add tag" git tag ${{ inputs.override_git_describe }} fi @@ -302,7 +302,7 @@ jobs: elif [[ -z "${{ inputs.override_git_describe }}" ]]; then echo "No override_git_describe provided" else - echo "UPLOAD_ASSETS_TO_STAGING_TARGET=$(git describe --tags --long)" >> "$GITHUB_ENV" + echo "UPLOAD_ASSETS_TO_STAGING_TARGET=$(git log -1 --format=%h)" >> "$GITHUB_ENV" echo "override_git_describe ${{ inputs.override_git_describe }}: add tag" git tag ${{ inputs.override_git_describe }} fi @@ -380,7 +380,7 @@ jobs: elif [[ -z "${{ inputs.override_git_describe }}" ]]; then echo "No override_git_describe provided" else - echo "UPLOAD_ASSETS_TO_STAGING_TARGET=$(git describe --tags --long)" >> "$GITHUB_ENV" + echo "UPLOAD_ASSETS_TO_STAGING_TARGET=$(git log -1 --format=%h)" >> "$GITHUB_ENV" echo "override_git_describe ${{ inputs.override_git_describe }}: add tag" git tag ${{ inputs.override_git_describe }} fi diff --git a/.github/workflows/StagedUpload.yml b/.github/workflows/StagedUpload.yml index 322f636421db..654070aeff2a 100644 --- a/.github/workflows/StagedUpload.yml +++ b/.github/workflows/StagedUpload.yml @@ -31,8 +31,9 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} run: | + TARGET=$(git log -1 --format=%h) mkdir to_be_uploaded - aws s3 cp --recursive "s3://duckdb-staging/${{ inputs.target_git_describe }}/$GITHUB_REPOSITORY/github_release" to_be_uploaded --region us-east-2 + aws s3 cp --recursive "s3://duckdb-staging/$TARGET/${{ inputs.target_git_describe }}/$GITHUB_REPOSITORY/github_release" to_be_uploaded --region us-east-2 - name: Deploy shell: bash diff --git a/.github/workflows/TwineUpload.yml b/.github/workflows/TwineUpload.yml index 1343fb691f58..58d27c2ccd4e 100644 --- a/.github/workflows/TwineUpload.yml +++ b/.github/workflows/TwineUpload.yml @@ -37,7 +37,7 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} run: | - TARGET=$(git describe --tags --long) + TARGET=$(git log -1 --format=%h) # decide target for staging if [ "$OVERRIDE_GIT_DESCRIBE" ]; then TARGET="$TARGET/$OVERRIDE_GIT_DESCRIBE" diff --git a/scripts/upload-assets-to-staging.sh b/scripts/upload-assets-to-staging.sh index 3b0d71ec828b..d2111d894800 100755 --- a/scripts/upload-assets-to-staging.sh +++ b/scripts/upload-assets-to-staging.sh @@ -45,7 +45,7 @@ if [ -z "$AWS_ACCESS_KEY_ID" ]; then fi -TARGET=$(git describe --tags --long) +TARGET=$(git log -1 --format=%h) if [ "$UPLOAD_ASSETS_TO_STAGING_TARGET" ]; then TARGET="$UPLOAD_ASSETS_TO_STAGING_TARGET" From c3ed3276fcf26d34d6288d6ef85e015ff7ca3d1c Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 18 Apr 2024 15:07:03 +0200 Subject: [PATCH 543/603] make all casting strict --- .../csv_scanner/sniffer/type_detection.cpp | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/execution/operator/csv_scanner/sniffer/type_detection.cpp b/src/execution/operator/csv_scanner/sniffer/type_detection.cpp index 533046b9650a..78d5b2542ec9 100644 --- a/src/execution/operator/csv_scanner/sniffer/type_detection.cpp +++ b/src/execution/operator/csv_scanner/sniffer/type_detection.cpp @@ -107,39 +107,39 @@ bool CSVSniffer::IsCasteable(const string_t value, const LogicalType &type, cons } case LogicalTypeId::SMALLINT: { int16_t dummy_value; - return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, false); + return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, true); } case LogicalTypeId::INTEGER: { int32_t dummy_value; - return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, false); + return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, true); } case LogicalTypeId::BIGINT: { int64_t dummy_value; - return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, false); + return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, true); } case LogicalTypeId::UTINYINT: { uint8_t dummy_value; - return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, false); + return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, true); } case LogicalTypeId::USMALLINT: { uint16_t dummy_value; - return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, false); + return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, true); } case LogicalTypeId::UINTEGER: { uint32_t dummy_value; - return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, false); + return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, true); } case LogicalTypeId::UBIGINT: { uint64_t dummy_value; - return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, false); + return TrySimpleIntegerCast(value_ptr, value_size, dummy_value, true); } case LogicalTypeId::DOUBLE: { double dummy_value; - return TryDoubleCast(value_ptr, value_size, dummy_value, false, options.decimal_separator[0]); + return TryDoubleCast(value_ptr, value_size, dummy_value, true, options.decimal_separator[0]); } case LogicalTypeId::FLOAT: { float dummy_value; - return TryDoubleCast(value_ptr, value_size, dummy_value, false, options.decimal_separator[0]); + return TryDoubleCast(value_ptr, value_size, dummy_value, true, options.decimal_separator[0]); } case LogicalTypeId::DATE: { if (!dialect_options.date_format.find(LogicalTypeId::DATE)->second.GetValue().Empty()) { @@ -152,7 +152,7 @@ bool CSVSniffer::IsCasteable(const string_t value, const LogicalType &type, cons idx_t pos; bool special; date_t dummy_value; - return Date::TryConvertDate(value_ptr, value_size, pos, dummy_value, special, false); + return Date::TryConvertDate(value_ptr, value_size, pos, dummy_value, special, true); } } case LogicalTypeId::TIMESTAMP: { From e445c53157d136affe61cd94d3a279d781abf4b0 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 18 Apr 2024 15:07:39 +0200 Subject: [PATCH 544/603] Add OnTag.yml, to automatically trigger needed workflows on tag creations --- .github/workflows/OnTag.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/OnTag.yml diff --git a/.github/workflows/OnTag.yml b/.github/workflows/OnTag.yml new file mode 100644 index 000000000000..a88b6eb3df66 --- /dev/null +++ b/.github/workflows/OnTag.yml @@ -0,0 +1,22 @@ +name: On Tag +on: + workflow_dispatch: + inputs: + override_git_describe: + type: string + push: + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + +jobs: + twine_upload: + uses: ./.github/workflows/TwineUpload.yml + secrets: inherit + with: + override_git_describe: ${{ inputs.override_git_describe || github.event.release.tag_name }} + + staged_upload: + uses: ./.github/workflows/StagedUpload.yml + secrets: inherit + with: + override_git_describe: ${{ inputs.override_git_describe || github.event.release.tag_name }} From e3903a8a641992646c8e728a0784a49110e198aa Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 18 Apr 2024 15:26:51 +0200 Subject: [PATCH 545/603] we should use correct dialect options for the header detection --- src/execution/operator/csv_scanner/sniffer/header_detection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/execution/operator/csv_scanner/sniffer/header_detection.cpp b/src/execution/operator/csv_scanner/sniffer/header_detection.cpp index 591aacdc7532..79efd42a589d 100644 --- a/src/execution/operator/csv_scanner/sniffer/header_detection.cpp +++ b/src/execution/operator/csv_scanner/sniffer/header_detection.cpp @@ -186,7 +186,7 @@ void CSVSniffer::DetectHeader() { if (!dummy_val.IsNull()) { val = StringValue::Get(dummy_val); } - if (!IsCasteable(val, sql_type, options.dialect_options, dummy_val.IsNull())) { + if (!IsCasteable(val, sql_type, sniffer_state_machine.dialect_options, dummy_val.IsNull())) { first_row_consistent = false; } } From 68b635c544f51e3ef2f724bf8be63688f4006bd2 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 18 Apr 2024 16:01:26 +0200 Subject: [PATCH 546/603] starting to unify. but will revert soon --- .../duckdb/optimizer/filter_pushdown.hpp | 2 + src/optimizer/pushdown/pushdown_aggregate.cpp | 2 + src/optimizer/pushdown/pushdown_window.cpp | 79 +++++++++++-------- 3 files changed, 51 insertions(+), 32 deletions(-) diff --git a/src/include/duckdb/optimizer/filter_pushdown.hpp b/src/include/duckdb/optimizer/filter_pushdown.hpp index a2aa1e2e5b07..1cefe032fff8 100644 --- a/src/include/duckdb/optimizer/filter_pushdown.hpp +++ b/src/include/duckdb/optimizer/filter_pushdown.hpp @@ -93,6 +93,8 @@ class FilterPushdown { //! if there are filters in this FilterPushdown node, push them into the combiner void PushFilters(); + bool CanPushdownFilter(vector window_exprs_partition_bindings, vector bindings); + FilterCombiner combiner; }; diff --git a/src/optimizer/pushdown/pushdown_aggregate.cpp b/src/optimizer/pushdown/pushdown_aggregate.cpp index 47a2b24e9942..f4c17b502506 100644 --- a/src/optimizer/pushdown/pushdown_aggregate.cpp +++ b/src/optimizer/pushdown/pushdown_aggregate.cpp @@ -58,9 +58,11 @@ unique_ptr FilterPushdown::PushdownAggregate(unique_ptr grouping_sets_bindings; for (auto &grp : aggr.grouping_sets) { // check for each of the grouping sets if they contain all groups if (bindings.empty()) { + // ?? // we can never push down empty grouping sets can_pushdown_filter = false; break; diff --git a/src/optimizer/pushdown/pushdown_window.cpp b/src/optimizer/pushdown/pushdown_window.cpp index 46e98ace3ea5..02c9802a8ce5 100644 --- a/src/optimizer/pushdown/pushdown_window.cpp +++ b/src/optimizer/pushdown/pushdown_window.cpp @@ -8,6 +8,7 @@ namespace duckdb { +// if a filter expression is on one of the bindings in the partition set. static bool FilterIsOnPartition(column_binding_set_t partition_bindings, Expression &filter_expression) { bool filter_is_on_partition = false; ExpressionIterator::EnumerateChildren(filter_expression, [&](unique_ptr &child) { @@ -38,19 +39,32 @@ static bool FilterIsOnPartition(column_binding_set_t partition_bindings, Express return filter_is_on_partition; } +bool FilterPushdown::CanPushdownFilter(vector window_exprs_partition_bindings, vector bindings) { + bool ret = true; + for (auto &partition_bindings : window_exprs_partition_bindings) { + bool filter_on_partitions_bindings = true; + for (auto &binding : bindings) { + if (partition_bindings.find(binding) == partition_bindings.end()) { + filter_on_partitions_bindings = false; + } + } + ret = ret && filter_on_partitions_bindings; + } + return ret; +} + unique_ptr FilterPushdown::PushdownWindow(unique_ptr op) { D_ASSERT(op->type == LogicalOperatorType::LOGICAL_WINDOW); auto &window = op->Cast(); // go into expressions, then look into the partitions. // if the filter applies to a partition in each window expression then you can push the filter // into the children. - auto pushdown = FilterPushdown(optimizer); - vector filters_to_pushdown; - filters_to_pushdown.reserve(filters.size()); - for (idx_t i = 0; i < filters.size(); i++) { - filters_to_pushdown.push_back(0); - } - // 1. Check every window expression + FilterPushdown pushdown(optimizer); + vector> leftover_filters; + + // First loop through the window expressions. If all window expressions + // have partitions, maybe we can push down a filter through the partition. + vector window_exprs_partition_bindings; for (auto &expr : window.expressions) { if (expr->expression_class != ExpressionClass::BOUND_WINDOW) { continue; @@ -58,12 +72,13 @@ unique_ptr FilterPushdown::PushdownWindow(unique_ptrCast(); auto &partitions = window_expr.partitions; if (partitions.empty()) { + // If any window expression does not have partitions, we cannot push any filters. // all window expressions need to be partitioned by the same column // in order to push down the window. return FinishPushdown(std::move(op)); } column_binding_set_t partition_bindings; - // 2. Get the binding information of the partitions + // 2. Get the binding information of the partitions of the window expression for (auto &partition_expr : partitions) { switch (partition_expr->type) { // TODO: Add expressions for function expressions like FLOOR, CEIL etc. @@ -76,36 +91,36 @@ unique_ptr FilterPushdown::PushdownWindow(unique_ptrfilter) && filters_to_pushdown[i] >= 0) { - filters_to_pushdown[i] = 1; - continue; - } - filters_to_pushdown[i] = -1; - } + if (window_exprs_partition_bindings.empty()) { + return FinishPushdown(std::move(op)); } - // push the new filters into the child. - vector> leftover_filters; + + // Loop through the filters. If a filter is on a partition in every window expression + // it can be pushed down. for (idx_t i = 0; i < filters.size(); i++) { - if (filters_to_pushdown[i] == 1) { - pushdown.filters.push_back(std::move(filters.at(i))); - continue; + auto can_pushdown_filter = true; + + // the filter must be on all partition bindings + vector bindings; + ExtractFilterBindings(*filters.at(i)->filter, bindings); + if (CanPushdownFilter(window_exprs_partition_bindings, bindings)) { + } - leftover_filters.push_back(std::move(filters.at(i))); - } - if (!pushdown.filters.empty()) { - op->children[0] = pushdown.Rewrite(std::move(op->children[0])); +// for (auto &partition_bindings : window_exprs_partition_bindings) { +// can_pushdown_filter = can_pushdown_filter && FilterIsOnPartition(partition_bindings, *filters.at(i)->filter); +// } +// if (can_pushdown_filter) { +// pushdown.filters.push_back(std::move(filters.at(i))); +// } else { +// leftover_filters.push_back(std::move(filters.at(i))); +// } } + op->children[0] = pushdown.Rewrite(std::move(op->children[0])); filters = std::move(leftover_filters); - return FinishPushdown(std::move(op)); + } } // namespace duckdb From 62de06159551912da70313cf946621432316cdeb Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 18 Apr 2024 16:07:34 +0200 Subject: [PATCH 547/603] wrongly translated settings from the sqllogic_test_runner.cpp file --- scripts/sqllogictest/result.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/sqllogictest/result.py b/scripts/sqllogictest/result.py index 68b92964b153..535a1a5ba28f 100644 --- a/scripts/sqllogictest/result.py +++ b/scripts/sqllogictest/result.py @@ -772,10 +772,9 @@ def execute_load(self, load: Load): # set up the config file additional_config = {} if readonly: - additional_config['temp_directory'] = False + additional_config['temp_directory'] = "" additional_config['access_mode'] = 'read_only' else: - additional_config['temp_directory'] = True additional_config['access_mode'] = 'automatic' self.pool = None From 95bc85108169e6fd89b4dbaac75b35e99defda4e Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 18 Apr 2024 16:12:25 +0200 Subject: [PATCH 548/603] Alright getting the last values out of the sniffer --- .../csv_scanner/sniffer/header_detection.cpp | 26 ++++++------------- .../csv_scanner/sniffer/type_detection.cpp | 24 +++++++++-------- .../operator/csv_scanner/csv_sniffer.hpp | 15 ++++++++++- 3 files changed, 35 insertions(+), 30 deletions(-) diff --git a/src/execution/operator/csv_scanner/sniffer/header_detection.cpp b/src/execution/operator/csv_scanner/sniffer/header_detection.cpp index 79efd42a589d..523aff201b7c 100644 --- a/src/execution/operator/csv_scanner/sniffer/header_detection.cpp +++ b/src/execution/operator/csv_scanner/sniffer/header_detection.cpp @@ -111,7 +111,7 @@ bool CSVSniffer::DetectHeaderWithSetColumn() { if (best_header_row[i].IsNull()) { return false; } - if (best_header_row[i] != (*set_columns.names)[i]) { + if (best_header_row[i].value.GetString() != (*set_columns.names)[i]) { has_header = false; break; } @@ -120,17 +120,12 @@ bool CSVSniffer::DetectHeaderWithSetColumn() { if (!has_header) { // We verify if the types are consistent for (idx_t col = 0; col < set_columns.Size(); col++) { - auto dummy_val = best_header_row[col]; // try cast to sql_type of column const auto &sql_type = (*set_columns.types)[col]; if (sql_type != LogicalType::VARCHAR) { all_varchar = false; - string_t val; - if (!dummy_val.IsNull()) { - val = StringValue::Get(dummy_val); - } - - if (!IsCasteable(val, sql_type, options.dialect_options, dummy_val.IsNull())) { + if (!IsCasteable(best_header_row[col].value, sql_type, options.dialect_options, + best_header_row[col].IsNull())) { first_row_consistent = false; } } @@ -174,19 +169,15 @@ void CSVSniffer::DetectHeader() { has_header = DetectHeaderWithSetColumn(); } else { for (idx_t col = 0; col < best_header_row.size(); col++) { - auto dummy_val = best_header_row[col]; - if (!dummy_val.IsNull()) { + if (!best_header_row[col].IsNull()) { first_row_nulls = false; } // try cast to sql_type of column const auto &sql_type = best_sql_types_candidates_per_column_idx[col].back(); if (sql_type != LogicalType::VARCHAR) { all_varchar = false; - string_t val; - if (!dummy_val.IsNull()) { - val = StringValue::Get(dummy_val); - } - if (!IsCasteable(val, sql_type, sniffer_state_machine.dialect_options, dummy_val.IsNull())) { + if (!IsCasteable(best_header_row[col].value, sql_type, sniffer_state_machine.dialect_options, + best_header_row[col].IsNull())) { first_row_consistent = false; } } @@ -217,11 +208,10 @@ void CSVSniffer::DetectHeader() { // get header names from CSV for (idx_t col = 0; col < best_header_row.size(); col++) { - const auto &val = best_header_row[col]; - string col_name = val.ToString(); + string col_name = best_header_row[col].value.GetString(); // generate name if field is empty - if (col_name.empty() || val.IsNull()) { + if (col_name.empty() || best_header_row[col].IsNull()) { col_name = GenerateColumnName(sniffer_state_machine.dialect_options.num_cols, col); } diff --git a/src/execution/operator/csv_scanner/sniffer/type_detection.cpp b/src/execution/operator/csv_scanner/sniffer/type_detection.cpp index 78d5b2542ec9..85dad9ea7f27 100644 --- a/src/execution/operator/csv_scanner/sniffer/type_detection.cpp +++ b/src/execution/operator/csv_scanner/sniffer/type_detection.cpp @@ -16,9 +16,9 @@ struct TryCastFloatingOperator { } }; -static bool StartsWithNumericDate(string &separator, const string &value) { - auto begin = value.c_str(); - auto end = begin + value.size(); +static bool StartsWithNumericDate(string &separator, const string_t &value) { + auto begin = value.GetData(); + auto end = begin + value.GetSize(); // StrpTimeFormat::Parse will skip whitespace, so we can too auto field1 = std::find_if_not(begin, end, StringUtil::CharacterIsSpace); @@ -303,12 +303,7 @@ void CSVSniffer::DetectTypes() { string separator; // If Value is not Null, Has a numeric date format, and the current investigated candidate is // either a timestamp or a date - // fixme: make this string_t - string str_val; - if (null_mask.RowIsValid(row_idx)) { - str_val = vector_data[row_idx].GetString(); - } - if (null_mask.RowIsValid(row_idx) && StartsWithNumericDate(separator, str_val) && + if (null_mask.RowIsValid(row_idx) && StartsWithNumericDate(separator, vector_data[row_idx]) && (col_type_candidates.back().id() == LogicalTypeId::TIMESTAMP || col_type_candidates.back().id() == LogicalTypeId::DATE)) { DetectDateAndTimeStampFormats(candidate->GetStateMachine(), sql_type, separator, @@ -363,8 +358,15 @@ void CSVSniffer::DetectTypes() { } if (chunk_size > 0) { for (idx_t col_idx = 0; col_idx < data_chunk.ColumnCount(); col_idx++) { - // fixme: make this string_t - best_header_row.emplace_back(data_chunk.GetValue(col_idx, 0)); + auto &cur_vector = data_chunk.data[col_idx]; + auto vector_data = FlatVector::GetData(cur_vector); + auto null_mask = FlatVector::Validity(cur_vector); + if (null_mask.RowIsValid(0)) { + auto value = HeaderValue(vector_data[0]); + best_header_row.push_back(value); + } else { + best_header_row.push_back({}); + } } } } diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp index 8e8796daaf09..50d5a9f17a93 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp @@ -75,6 +75,19 @@ struct SetColumns { } }; +struct HeaderValue { + HeaderValue() : is_null(true) { + } + explicit HeaderValue(string_t value_p) { + value = value_p; + } + bool IsNull() { + return is_null; + } + bool is_null = false; + string_t value {}; +}; + //! Sniffer that detects Header, Dialect and Types of CSV Files class CSVSniffer { public: @@ -166,7 +179,7 @@ class CSVSniffer { unordered_map> best_sql_types_candidates_per_column_idx; map> best_format_candidates; unique_ptr best_candidate; - vector best_header_row; + vector best_header_row; //! Variable used for sniffing date and timestamp map format_candidates; map original_format_candidates; From 8924c13d5947e98df9b0866869ee72db04489746 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 18 Apr 2024 16:22:29 +0200 Subject: [PATCH 549/603] Add micro benchmark --- .../micro/csv/multiple_small_files.benchmark | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 benchmark/micro/csv/multiple_small_files.benchmark diff --git a/benchmark/micro/csv/multiple_small_files.benchmark b/benchmark/micro/csv/multiple_small_files.benchmark new file mode 100644 index 000000000000..860d2c9a5b5f --- /dev/null +++ b/benchmark/micro/csv/multiple_small_files.benchmark @@ -0,0 +1,46 @@ +# name: benchmark/micro/csv/multiple_small_files.benchmark +# description: Run CSV scan over multiple small (2048 rows) values +# group: [csv] + +name CSV Read Benchmark over multiple small files +group csv + +load +CREATE TABLE t1 AS select i,i,i,i,i,i,i,i,i,i from range(0,2048) tbl(i); +COPY t1 TO '${BENCHMARK_DIR}/small_csv.csv' (FORMAT CSV, HEADER 1); + +run +SELECT * from +read_csv(['${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv', +'${BENCHMARK_DIR}/small_csv.csv','${BENCHMARK_DIR}/small_csv.csv'],delim= ',', header = 0) From 5a4ace1506aabfa6109084cde35997638ef570ec Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 18 Apr 2024 16:23:30 +0200 Subject: [PATCH 550/603] run it on CI --- .github/regression/csv.csv | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/regression/csv.csv b/.github/regression/csv.csv index 272b7ced2a04..9935bbecce29 100644 --- a/.github/regression/csv.csv +++ b/.github/regression/csv.csv @@ -4,4 +4,5 @@ benchmark/micro/csv/small_csv.benchmark benchmark/micro/csv/null_padding.benchmark benchmark/micro/csv/projection_pushdown.benchmark benchmark/micro/csv/1_byte_values.benchmark -benchmark/micro/csv/16_byte_values.benchmark \ No newline at end of file +benchmark/micro/csv/16_byte_values.benchmark +benchmark/micro/csv/multiple_small_files.benchmark \ No newline at end of file From 180fbf24318247b4ee3ba840d98a5e78fc801645 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 18 Apr 2024 16:23:50 +0200 Subject: [PATCH 551/603] auto gen stuff --- .github/regression/micro_extended.csv | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/regression/micro_extended.csv b/.github/regression/micro_extended.csv index 6973785b4c98..44e75e349df1 100644 --- a/.github/regression/micro_extended.csv +++ b/.github/regression/micro_extended.csv @@ -79,6 +79,7 @@ benchmark/micro/copy/to_parquet_partition_by_many.benchmark benchmark/micro/csv/16_byte_values.benchmark benchmark/micro/csv/1_byte_values.benchmark benchmark/micro/csv/multiple_read.benchmark +benchmark/micro/csv/multiple_small_files.benchmark benchmark/micro/csv/multiple_small_read_csv.benchmark benchmark/micro/csv/null_padding.benchmark benchmark/micro/csv/projection_pushdown.benchmark From ae060350906f24bf35239e0c231771871ecd7f97 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 18 Apr 2024 16:24:59 +0200 Subject: [PATCH 552/603] add back in old code --- src/optimizer/pushdown/pushdown_window.cpp | 33 +++++----------------- 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/src/optimizer/pushdown/pushdown_window.cpp b/src/optimizer/pushdown/pushdown_window.cpp index 02c9802a8ce5..9840b71e2866 100644 --- a/src/optimizer/pushdown/pushdown_window.cpp +++ b/src/optimizer/pushdown/pushdown_window.cpp @@ -39,20 +39,6 @@ static bool FilterIsOnPartition(column_binding_set_t partition_bindings, Express return filter_is_on_partition; } -bool FilterPushdown::CanPushdownFilter(vector window_exprs_partition_bindings, vector bindings) { - bool ret = true; - for (auto &partition_bindings : window_exprs_partition_bindings) { - bool filter_on_partitions_bindings = true; - for (auto &binding : bindings) { - if (partition_bindings.find(binding) == partition_bindings.end()) { - filter_on_partitions_bindings = false; - } - } - ret = ret && filter_on_partitions_bindings; - } - return ret; -} - unique_ptr FilterPushdown::PushdownWindow(unique_ptr op) { D_ASSERT(op->type == LogicalOperatorType::LOGICAL_WINDOW); auto &window = op->Cast(); @@ -104,19 +90,14 @@ unique_ptr FilterPushdown::PushdownWindow(unique_ptr bindings; - ExtractFilterBindings(*filters.at(i)->filter, bindings); - if (CanPushdownFilter(window_exprs_partition_bindings, bindings)) { - + for (auto &partition_bindings : window_exprs_partition_bindings) { + can_pushdown_filter = can_pushdown_filter && FilterIsOnPartition(partition_bindings, *filters.at(i)->filter); + } + if (can_pushdown_filter) { + pushdown.filters.push_back(std::move(filters.at(i))); + } else { + leftover_filters.push_back(std::move(filters.at(i))); } -// for (auto &partition_bindings : window_exprs_partition_bindings) { -// can_pushdown_filter = can_pushdown_filter && FilterIsOnPartition(partition_bindings, *filters.at(i)->filter); -// } -// if (can_pushdown_filter) { -// pushdown.filters.push_back(std::move(filters.at(i))); -// } else { -// leftover_filters.push_back(std::move(filters.at(i))); -// } } op->children[0] = pushdown.Rewrite(std::move(op->children[0])); filters = std::move(leftover_filters); From 8820bb728b56e228b782ec1292585a9ef398c661 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 18 Apr 2024 16:30:05 +0200 Subject: [PATCH 553/603] make format-fix --- src/include/duckdb/optimizer/filter_pushdown.hpp | 3 ++- src/optimizer/pushdown/pushdown_window.cpp | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/include/duckdb/optimizer/filter_pushdown.hpp b/src/include/duckdb/optimizer/filter_pushdown.hpp index 1cefe032fff8..f569c2ad362d 100644 --- a/src/include/duckdb/optimizer/filter_pushdown.hpp +++ b/src/include/duckdb/optimizer/filter_pushdown.hpp @@ -93,7 +93,8 @@ class FilterPushdown { //! if there are filters in this FilterPushdown node, push them into the combiner void PushFilters(); - bool CanPushdownFilter(vector window_exprs_partition_bindings, vector bindings); + bool CanPushdownFilter(vector window_exprs_partition_bindings, + vector bindings); FilterCombiner combiner; }; diff --git a/src/optimizer/pushdown/pushdown_window.cpp b/src/optimizer/pushdown/pushdown_window.cpp index 9840b71e2866..25d83a83c7f2 100644 --- a/src/optimizer/pushdown/pushdown_window.cpp +++ b/src/optimizer/pushdown/pushdown_window.cpp @@ -91,7 +91,8 @@ unique_ptr FilterPushdown::PushdownWindow(unique_ptrfilter); + can_pushdown_filter = + can_pushdown_filter && FilterIsOnPartition(partition_bindings, *filters.at(i)->filter); } if (can_pushdown_filter) { pushdown.filters.push_back(std::move(filters.at(i))); @@ -102,6 +103,5 @@ unique_ptr FilterPushdown::PushdownWindow(unique_ptrchildren[0] = pushdown.Rewrite(std::move(op->children[0])); filters = std::move(leftover_filters); return FinishPushdown(std::move(op)); - } } // namespace duckdb From 7f3e09f63cd2a4a4f3947e74435380ea77016744 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 18 Apr 2024 17:48:19 +0200 Subject: [PATCH 554/603] clang tidy fizes --- src/optimizer/join_order/join_order_optimizer.cpp | 1 - src/optimizer/join_order/plan_enumerator.cpp | 12 ++++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/optimizer/join_order/join_order_optimizer.cpp b/src/optimizer/join_order/join_order_optimizer.cpp index 1b259a3c5bad..cafc47a8022b 100644 --- a/src/optimizer/join_order/join_order_optimizer.cpp +++ b/src/optimizer/join_order/join_order_optimizer.cpp @@ -49,7 +49,6 @@ unique_ptr JoinOrderOptimizer::Optimize(unique_ptr PlanEnumerator::CreateJoinNodeFromDPJoinNode(DPJoinNode dp_ res->cardinality = dp_node.cardinality; return res; } else { - auto left_DPJoinNode = plans.find(dp_node.left_set); - auto right_DPJoinNode = plans.find(dp_node.right_set); - D_ASSERT(left_DPJoinNode->second); - D_ASSERT(right_DPJoinNode->second); - auto left = CreateJoinNodeFromDPJoinNode(*left_DPJoinNode->second); - auto right = CreateJoinNodeFromDPJoinNode(*right_DPJoinNode->second); + auto left_dp_join_node = plans.find(dp_node.left_set); + auto right_dp_join_node = plans.find(dp_node.right_set); + D_ASSERT(left_dp_join_node->second); + D_ASSERT(right_dp_join_node->second); + auto left = CreateJoinNodeFromDPJoinNode(*left_dp_join_node->second); + auto right = CreateJoinNodeFromDPJoinNode(*right_dp_join_node->second); auto res = make_uniq(dp_node.set, dp_node.info, std::move(left), std::move(right), dp_node.cost); res->cardinality = dp_node.cardinality; return res; From 9da0f63eb3487ee13fed1020caf08c5c96b61756 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 18 Apr 2024 17:48:30 +0200 Subject: [PATCH 555/603] ble no need to inline and ; baddy --- src/execution/operator/csv_scanner/sniffer/type_detection.cpp | 3 ++- .../duckdb/execution/operator/csv_scanner/csv_sniffer.hpp | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/execution/operator/csv_scanner/sniffer/type_detection.cpp b/src/execution/operator/csv_scanner/sniffer/type_detection.cpp index 85dad9ea7f27..31d500896204 100644 --- a/src/execution/operator/csv_scanner/sniffer/type_detection.cpp +++ b/src/execution/operator/csv_scanner/sniffer/type_detection.cpp @@ -177,7 +177,8 @@ bool CSVSniffer::IsCasteable(const string_t value, const LogicalType &type, cons return str_value.TryCastAs(buffer_manager->context, type, new_value, &error_message, true); } } -}; +} + void CSVSniffer::InitializeDateAndTimeStampDetection(CSVStateMachine &candidate, const string &separator, const LogicalType &sql_type) { auto &format_candidate = format_candidates[sql_type.id()]; diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp index 50d5a9f17a93..a47ebb704dbc 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp @@ -165,8 +165,8 @@ class CSVSniffer { void DetectDateAndTimeStampFormats(CSVStateMachine &candidate, const LogicalType &sql_type, const string &separator, string_t &dummy_val); //! If a string_t value can be cast to a type - inline bool IsCasteable(const string_t value, const LogicalType &type, const DialectOptions &dialect_options, - const bool is_null); + bool IsCasteable(const string_t value, const LogicalType &type, const DialectOptions &dialect_options, + const bool is_null); //! Variables for Type Detection //! Format Candidates for Date and Timestamp Types From 59f3dcd3e8b2bf285a261c4238c8dd62a6583c50 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 18 Apr 2024 20:30:14 +0200 Subject: [PATCH 556/603] Improve mkdir error reporting --- src/common/local_file_system.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/common/local_file_system.cpp b/src/common/local_file_system.cpp index 744bce887cb1..029aa02cf584 100644 --- a/src/common/local_file_system.cpp +++ b/src/common/local_file_system.cpp @@ -540,7 +540,8 @@ void LocalFileSystem::CreateDirectory(const string &directory, optional_ptr Date: Thu, 18 Apr 2024 20:31:20 +0200 Subject: [PATCH 557/603] Consistent quotes --- src/common/local_file_system.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/local_file_system.cpp b/src/common/local_file_system.cpp index 029aa02cf584..a2e7ab0dce74 100644 --- a/src/common/local_file_system.cpp +++ b/src/common/local_file_system.cpp @@ -991,7 +991,7 @@ void LocalFileSystem::CreateDirectory(const string &directory, optional_ptr Date: Thu, 18 Apr 2024 20:26:43 +0200 Subject: [PATCH 558/603] Remove python3 build dependency from docker tests --- scripts/test_docker_images.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/test_docker_images.sh b/scripts/test_docker_images.sh index c2217a8b6488..ce12ef35ae47 100755 --- a/scripts/test_docker_images.sh +++ b/scripts/test_docker_images.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash make clean +docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb alpine:latest <<< "apk add g++ git make cmake ninja && GEN=ninja make && make clean" 2>&1 docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb alpine:latest <<< "apk add g++ git make cmake ninja python3 && GEN=ninja make && make clean" 2>&1 -docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb ubuntu:20.04 <<< "apt-get update && export DEBIAN_FRONTEND=noninteractive && apt-get install g++ git make cmake ninja-build python3 -y && GEN=ninja make && make clean" 2>&1 +docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb ubuntu:20.04 <<< "apt-get update && export DEBIAN_FRONTEND=noninteractive && apt-get install g++ git make cmake ninja-build -y && GEN=ninja make && make clean" 2>&1 From cd0e1435f68db5ffea9af049fb863965dde14a2a Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 18 Apr 2024 21:04:15 +0200 Subject: [PATCH 559/603] Allow overriding C++ standard from [C]Make --- CMakeLists.txt | 2 +- Makefile | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fb45218c66fa..ec210f000160 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,7 +28,7 @@ set(DUCKDB_MODULE_BASE_DIR "${CMAKE_CURRENT_LIST_DIR}") set(CMAKE_EXPORT_COMPILE_COMMANDS ON) -set (CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD "11" CACHE STRING "C++ standard to enforce") set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/Makefile b/Makefile index 31a4cdcb64de..248c541c2d5d 100644 --- a/Makefile +++ b/Makefile @@ -65,6 +65,9 @@ ifdef OVERRIDE_GIT_DESCRIBE else COMMON_CMAKE_VARS:=${COMMON_CMAKE_VARS} -DOVERRIDE_GIT_DESCRIBE="" endif +ifneq (${CXX_STANDARD}, ) + CMAKE_VARS:=${CMAKE_VARS} -DCMAKE_CXX_STANDARD="${CXX_STANDARD}" +endif ifneq (${DUCKDB_EXTENSIONS}, ) BUILD_EXTENSIONS:=${DUCKDB_EXTENSIONS} endif From da40bca1c2f5d8a57e50764ff754283e81a080ea Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 18 Apr 2024 21:04:59 +0200 Subject: [PATCH 560/603] Fix asymmetrical operator== [C++20 error] --- src/include/duckdb/parser/base_expression.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/duckdb/parser/base_expression.hpp b/src/include/duckdb/parser/base_expression.hpp index aed84f5eeee3..a64baf7243fd 100644 --- a/src/include/duckdb/parser/base_expression.hpp +++ b/src/include/duckdb/parser/base_expression.hpp @@ -79,7 +79,7 @@ class BaseExpression { static bool Equals(const BaseExpression &left, const BaseExpression &right) { return left.Equals(right); } - bool operator==(const BaseExpression &rhs) { + bool operator==(const BaseExpression &rhs) const { return Equals(rhs); } From b92bbf90f819795909cb018427d4367781d1b278 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 18 Apr 2024 21:05:38 +0200 Subject: [PATCH 561/603] Fix compound assigment on volatile [C++20 error] --- third_party/mbedtls/library/constant_time.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/mbedtls/library/constant_time.cpp b/third_party/mbedtls/library/constant_time.cpp index a797dce390e3..d2a2239e5d87 100644 --- a/third_party/mbedtls/library/constant_time.cpp +++ b/third_party/mbedtls/library/constant_time.cpp @@ -61,7 +61,7 @@ int mbedtls_ct_memcmp( const void *a, * This avoids IAR compiler warning: * 'the order of volatile accesses is undefined ..' */ unsigned char x = A[i], y = B[i]; - diff |= x ^ y; + diff = (diff | (x ^ y)); } return( (int)diff ); From a802e6ca350dec56bd8358c971f17ff1b141dada Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 18 Apr 2024 21:06:38 +0200 Subject: [PATCH 562/603] Add CI run on C++23 standard --- scripts/test_docker_images.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/test_docker_images.sh b/scripts/test_docker_images.sh index ce12ef35ae47..7f9deaf196ae 100755 --- a/scripts/test_docker_images.sh +++ b/scripts/test_docker_images.sh @@ -3,4 +3,5 @@ make clean docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb alpine:latest <<< "apk add g++ git make cmake ninja && GEN=ninja make && make clean" 2>&1 docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb alpine:latest <<< "apk add g++ git make cmake ninja python3 && GEN=ninja make && make clean" 2>&1 +docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb alpine:latest <<< "apk add g++ git make cmake ninja && CXX_STANDARD=23 GEN=ninja make && make clean" 2>&1 docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb ubuntu:20.04 <<< "apt-get update && export DEBIAN_FRONTEND=noninteractive && apt-get install g++ git make cmake ninja-build -y && GEN=ninja make && make clean" 2>&1 From 196b1d1fa875330fd8f93d9e5f9d1afcd008ba95 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 18 Apr 2024 21:11:06 +0200 Subject: [PATCH 563/603] Fix missing std:: namespaces [C++17 error] --- src/include/duckdb/common/shared_ptr.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/include/duckdb/common/shared_ptr.hpp b/src/include/duckdb/common/shared_ptr.hpp index f5ca7d762c3a..6d0910ca6bf7 100644 --- a/src/include/duckdb/common/shared_ptr.hpp +++ b/src/include/duckdb/common/shared_ptr.hpp @@ -23,13 +23,13 @@ namespace duckdb { #if _LIBCPP_STD_VER >= 17 template -struct __bounded_convertible_to_unbounded : false_type {}; +struct __bounded_convertible_to_unbounded : std::false_type {}; template -struct __bounded_convertible_to_unbounded<_Up[_Np], T> : is_same, _Up[]> {}; +struct __bounded_convertible_to_unbounded<_Up[_Np], T> : std::is_same, _Up[]> {}; template -struct compatible_with_t : _Or, __bounded_convertible_to_unbounded> {}; +struct compatible_with_t : std::_Or, __bounded_convertible_to_unbounded> {}; #else template struct compatible_with_t : std::is_convertible {}; // NOLINT: invalid case style From 044ec9e34128d65a95fcbbf78a6242a2ab2f3844 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 18 Apr 2024 21:52:57 +0200 Subject: [PATCH 564/603] Initial version of removal of BoundConstraint from TableCatalogEntry --- .../catalog_entry/duck_table_entry.cpp | 39 +++--- .../catalog_entry/table_catalog_entry.cpp | 10 +- .../persistent/physical_batch_insert.cpp | 8 +- .../operator/persistent/physical_delete.cpp | 12 +- .../operator/persistent/physical_insert.cpp | 25 +++- .../operator/persistent/physical_update.cpp | 24 +++- .../table/system/duckdb_constraints.cpp | 32 +++-- .../catalog_entry/duck_table_entry.hpp | 6 +- .../catalog_entry/table_catalog_entry.hpp | 5 +- src/include/duckdb/planner/binder.hpp | 4 + .../parsed_data/bound_create_table_info.hpp | 2 - src/include/duckdb/storage/data_table.hpp | 25 ++-- .../duckdb/storage/table/append_state.hpp | 9 ++ .../duckdb/storage/table/delete_state.hpp | 23 ++++ .../duckdb/storage/table/update_state.hpp | 20 +++ .../binder/statement/bind_create_table.cpp | 130 +++++++++++------- src/planner/binder/statement/bind_update.cpp | 2 +- src/storage/data_table.cpp | 116 +++++++++------- src/storage/wal_replay.cpp | 3 +- 19 files changed, 326 insertions(+), 169 deletions(-) create mode 100644 src/include/duckdb/storage/table/delete_state.hpp create mode 100644 src/include/duckdb/storage/table/update_state.hpp diff --git a/src/catalog/catalog_entry/duck_table_entry.cpp b/src/catalog/catalog_entry/duck_table_entry.cpp index 90c4de3ca0b7..f77e69f65eba 100644 --- a/src/catalog/catalog_entry/duck_table_entry.cpp +++ b/src/catalog/catalog_entry/duck_table_entry.cpp @@ -71,10 +71,13 @@ IndexStorageInfo GetIndexInfo(const IndexConstraintType &constraint_type, unique return IndexStorageInfo(constraint_name + create_table_info.table + "_" + to_string(idx)); } +vector GetUniqueConstraintKeys(const ColumnList &columns, const UniqueConstraint &constraint) { + throw InternalException("FIXME: GetUniqueConstraintKeys"); +} + DuckTableEntry::DuckTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, BoundCreateTableInfo &info, std::shared_ptr inherited_storage) : TableCatalogEntry(catalog, schema, info.Base()), storage(std::move(inherited_storage)), - bound_constraints(std::move(info.bound_constraints)), column_dependency_manager(std::move(info.column_dependency_manager)) { if (!storage) { @@ -88,21 +91,19 @@ DuckTableEntry::DuckTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, Bou // create the unique indexes for the UNIQUE and PRIMARY KEY and FOREIGN KEY constraints idx_t indexes_idx = 0; - for (idx_t i = 0; i < bound_constraints.size(); i++) { - - auto &constraint = bound_constraints[i]; - + for (idx_t i = 0; i < constraints.size(); i++) { + auto &constraint = constraints[i]; if (constraint->type == ConstraintType::UNIQUE) { // unique constraint: create a unique index - auto &unique = constraint->Cast(); + auto &unique = constraint->Cast(); IndexConstraintType constraint_type = IndexConstraintType::UNIQUE; if (unique.is_primary_key) { constraint_type = IndexConstraintType::PRIMARY; } - + auto unique_keys = GetUniqueConstraintKeys(columns, unique); if (info.indexes.empty()) { - AddDataTableIndex(*storage, columns, unique.keys, constraint_type, + AddDataTableIndex(*storage, columns, unique_keys, constraint_type, GetIndexInfo(constraint_type, info.base, i)); } else { // we read the index from an old storage version, so we have to apply a dummy name @@ -112,13 +113,12 @@ DuckTableEntry::DuckTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, Bou } // now add the index - AddDataTableIndex(*storage, columns, unique.keys, constraint_type, info.indexes[indexes_idx++]); + AddDataTableIndex(*storage, columns, unique_keys, constraint_type, info.indexes[indexes_idx++]); } } else if (constraint->type == ConstraintType::FOREIGN_KEY) { - // foreign key constraint: create a foreign key index - auto &bfk = constraint->Cast(); + auto &bfk = constraint->Cast(); if (bfk.info.type == ForeignKeyType::FK_TYPE_FOREIGN_KEY_TABLE || bfk.info.type == ForeignKeyType::FK_TYPE_SELF_REFERENCE_TABLE) { @@ -351,10 +351,10 @@ unique_ptr DuckTableEntry::AddColumn(ClientContext &context, AddCo void DuckTableEntry::UpdateConstraintsOnColumnDrop(const LogicalIndex &removed_index, const vector &adjusted_indices, const RemoveColumnInfo &info, CreateTableInfo &create_info, - bool is_generated) { + const vector> &bound_constraints, + bool is_generated) { // handle constraints for the new table D_ASSERT(constraints.size() == bound_constraints.size()); - for (idx_t constr_idx = 0; constr_idx < constraints.size(); constr_idx++) { auto &constraint = constraints[constr_idx]; auto &bound_constraint = bound_constraints[constr_idx]; @@ -472,9 +472,11 @@ unique_ptr DuckTableEntry::RemoveColumn(ClientContext &context, Re } auto adjusted_indices = column_dependency_manager.RemoveColumn(removed_index, columns.LogicalColumnCount()); - UpdateConstraintsOnColumnDrop(removed_index, adjusted_indices, info, *create_info, dropped_column_is_generated); - auto binder = Binder::CreateBinder(context); + auto bound_constraints = binder->BindConstraints(constraints, name, columns); + + UpdateConstraintsOnColumnDrop(removed_index, adjusted_indices, info, *create_info, bound_constraints, dropped_column_is_generated); + auto bound_create_info = binder->BindCreateTableInfo(std::move(create_info), schema); if (columns.GetColumn(LogicalIndex(removed_index)).Generated()) { return make_uniq(catalog, schema, *bound_create_info, storage); @@ -583,6 +585,8 @@ unique_ptr DuckTableEntry::ChangeColumnType(ClientContext &context create_info->temporary = temporary; create_info->comment = comment; + auto binder = Binder::CreateBinder(context); + auto bound_constraints = binder->BindConstraints(constraints, name, columns); for (auto &col : columns.Logical()) { auto copy = col.Copy(); if (change_idx == col.Logical()) { @@ -643,7 +647,6 @@ unique_ptr DuckTableEntry::ChangeColumnType(ClientContext &context create_info->constraints.push_back(std::move(constraint)); } - auto binder = Binder::CreateBinder(context); // bind the specified expression vector bound_columns; AlterBinder expr_binder(*binder, context, *this, bound_columns, info.target_type); @@ -785,10 +788,6 @@ DataTable &DuckTableEntry::GetStorage() { return *storage; } -const vector> &DuckTableEntry::GetBoundConstraints() { - return bound_constraints; -} - TableFunction DuckTableEntry::GetScanFunction(ClientContext &context, unique_ptr &bind_data) { bind_data = make_uniq(*this); return TableScanFunction::GetFunction(); diff --git a/src/catalog/catalog_entry/table_catalog_entry.cpp b/src/catalog/catalog_entry/table_catalog_entry.cpp index f89d214f5465..493403830e2a 100644 --- a/src/catalog/catalog_entry/table_catalog_entry.cpp +++ b/src/catalog/catalog_entry/table_catalog_entry.cpp @@ -172,11 +172,6 @@ const vector> &TableCatalogEntry::GetConstraints() { DataTable &TableCatalogEntry::GetStorage() { throw InternalException("Calling GetStorage on a TableCatalogEntry that is not a DuckTableEntry"); } - -const vector> &TableCatalogEntry::GetBoundConstraints() { - throw InternalException("Calling GetBoundConstraints on a TableCatalogEntry that is not a DuckTableEntry"); -} - // LCOV_EXCL_STOP static void BindExtraColumns(TableCatalogEntry &table, LogicalGet &get, LogicalProjection &proj, LogicalUpdate &update, @@ -239,14 +234,15 @@ vector TableCatalogEntry::GetColumnSegmentInfo() { return {}; } -void TableCatalogEntry::BindUpdateConstraints(LogicalGet &get, LogicalProjection &proj, LogicalUpdate &update, +void TableCatalogEntry::BindUpdateConstraints(Binder &binder, LogicalGet &get, LogicalProjection &proj, LogicalUpdate &update, ClientContext &context) { // check the constraints and indexes of the table to see if we need to project any additional columns // we do this for indexes with multiple columns and CHECK constraints in the UPDATE clause // suppose we have a constraint CHECK(i + j < 10); now we need both i and j to check the constraint // if we are only updating one of the two columns we add the other one to the UPDATE set // with a "useless" update (i.e. i=i) so we can verify that the CHECK constraint is not violated - for (auto &constraint : GetBoundConstraints()) { + auto bound_constraints = binder.BindConstraints(constraints, name, columns); + for (auto &constraint : bound_constraints) { if (constraint->type == ConstraintType::CHECK) { auto &check = constraint->Cast(); // check constraint! check if we need to add any extra columns to the UPDATE clause diff --git a/src/execution/operator/persistent/physical_batch_insert.cpp b/src/execution/operator/persistent/physical_batch_insert.cpp index 04a137aa3c47..210f0b3d201d 100644 --- a/src/execution/operator/persistent/physical_batch_insert.cpp +++ b/src/execution/operator/persistent/physical_batch_insert.cpp @@ -171,6 +171,7 @@ class BatchInsertLocalState : public LocalSinkState { TableAppendState current_append_state; unique_ptr current_collection; optional_ptr writer; + unique_ptr constraint_state; void CreateNewCollection(DuckTableEntry &table, const vector &insert_types) { auto &table_info = table.GetStorage().info; @@ -494,7 +495,10 @@ SinkResultType PhysicalBatchInsert::Sink(ExecutionContext &context, DataChunk &c throw InternalException("Current batch differs from batch - but NextBatch was not called!?"); } - table.GetStorage().VerifyAppendConstraints(table, context.client, lstate.insert_chunk); + if (!lstate.constraint_state) { + lstate.constraint_state = table.GetStorage().InitializeConstraintVerification(table, context.client); + } + table.GetStorage().VerifyAppendConstraints(*lstate.constraint_state, context.client, lstate.insert_chunk); auto new_row_group = lstate.current_collection->Append(lstate.insert_chunk, lstate.current_append_state); if (new_row_group) { @@ -595,7 +599,7 @@ SinkFinalizeType PhysicalBatchInsert::Finalize(Pipeline &pipeline, Event &event, auto &table = gstate.table; auto &storage = table.GetStorage(); LocalAppendState append_state; - storage.InitializeLocalAppend(append_state, context); + storage.InitializeLocalAppend(append_state, table, context); auto &transaction = DuckTransaction::Get(context, table.catalog); for (auto &entry : gstate.collections) { if (entry.type != RowGroupBatchType::NOT_FLUSHED) { diff --git a/src/execution/operator/persistent/physical_delete.cpp b/src/execution/operator/persistent/physical_delete.cpp index 4fc17049032a..300376ce27fe 100644 --- a/src/execution/operator/persistent/physical_delete.cpp +++ b/src/execution/operator/persistent/physical_delete.cpp @@ -6,6 +6,7 @@ #include "duckdb/storage/data_table.hpp" #include "duckdb/storage/table/scan_state.hpp" #include "duckdb/transaction/duck_transaction.hpp" +#include "duckdb/storage/table/delete_state.hpp" namespace duckdb { @@ -25,10 +26,12 @@ class DeleteGlobalState : public GlobalSinkState { class DeleteLocalState : public LocalSinkState { public: - DeleteLocalState(Allocator &allocator, const vector &table_types) { - delete_chunk.Initialize(allocator, table_types); + DeleteLocalState(ClientContext &context, TableCatalogEntry &table) { + delete_chunk.Initialize(Allocator::Get(context), table.GetTypes()); + delete_state = table.GetStorage().InitializeDelete(table, context); } DataChunk delete_chunk; + unique_ptr delete_state; }; SinkResultType PhysicalDelete::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const { @@ -52,8 +55,7 @@ SinkResultType PhysicalDelete::Sink(ExecutionContext &context, DataChunk &chunk, table.Fetch(transaction, ustate.delete_chunk, column_ids, row_identifiers, chunk.size(), cfs); gstate.return_collection.Append(ustate.delete_chunk); } - gstate.deleted_count += table.Delete(tableref, context.client, row_identifiers, chunk.size()); - + gstate.deleted_count += table.Delete(*ustate.delete_state, context.client, row_identifiers, chunk.size()); return SinkResultType::NEED_MORE_INPUT; } @@ -62,7 +64,7 @@ unique_ptr PhysicalDelete::GetGlobalSinkState(ClientContext &co } unique_ptr PhysicalDelete::GetLocalSinkState(ExecutionContext &context) const { - return make_uniq(Allocator::Get(context.client), table.GetTypes()); + return make_uniq(context.client, tableref); } //===--------------------------------------------------------------------===// diff --git a/src/execution/operator/persistent/physical_insert.cpp b/src/execution/operator/persistent/physical_insert.cpp index 768334b50abe..181216a15b7f 100644 --- a/src/execution/operator/persistent/physical_insert.cpp +++ b/src/execution/operator/persistent/physical_insert.cpp @@ -17,6 +17,7 @@ #include "duckdb/execution/index/art/art.hpp" #include "duckdb/transaction/duck_transaction.hpp" #include "duckdb/storage/table/append_state.hpp" +#include "duckdb/storage/table/update_state.hpp" namespace duckdb { @@ -105,6 +106,14 @@ class InsertLocalState : public LocalSinkState { // Rows in the transaction-local storage that have been updated by a DO UPDATE conflict unordered_set updated_local_rows; idx_t update_count = 0; + unique_ptr constraint_state; + + ConstraintVerificationState &GetConstraintState(DataTable &table, TableCatalogEntry &tableref, ClientContext &context) { + if (!constraint_state) { + constraint_state = table.InitializeConstraintVerification(tableref, context); + } + return *constraint_state; + } }; unique_ptr PhysicalInsert::GetGlobalSinkState(ClientContext &context) const { @@ -278,7 +287,8 @@ static idx_t PerformOnConflictAction(ExecutionContext &context, DataChunk &chunk auto &data_table = table.GetStorage(); // Perform the update, using the results of the SET expressions if (GLOBAL) { - data_table.Update(table, context.client, row_ids, set_columns, update_chunk); + auto update_state = data_table.InitializeUpdate(table, context.client); + data_table.Update(*update_state, context.client, row_ids, set_columns, update_chunk); } else { auto &local_storage = LocalStorage::Get(context.client, data_table.db); // Perform the update, using the results of the SET expressions @@ -320,7 +330,8 @@ static idx_t HandleInsertConflicts(TableCatalogEntry &table, ExecutionContext &c ConflictInfo conflict_info(conflict_target); ConflictManager conflict_manager(VerifyExistenceType::APPEND, lstate.insert_chunk.size(), &conflict_info); if (GLOBAL) { - data_table.VerifyAppendConstraints(table, context.client, lstate.insert_chunk, &conflict_manager); + auto &constraint_state = lstate.GetConstraintState(data_table, table, context.client); + data_table.VerifyAppendConstraints(constraint_state, context.client, lstate.insert_chunk, &conflict_manager); } else { DataTable::VerifyUniqueIndexes(local_storage.GetIndexes(data_table), context.client, lstate.insert_chunk, &conflict_manager); @@ -380,7 +391,8 @@ static idx_t HandleInsertConflicts(TableCatalogEntry &table, ExecutionContext &c combined_chunk.Slice(sel.Selection(), sel.Count()); row_ids.Slice(sel.Selection(), sel.Count()); if (GLOBAL) { - data_table.VerifyAppendConstraints(table, context.client, combined_chunk, nullptr); + auto &constraint_state = lstate.GetConstraintState(data_table, table, context.client); + data_table.VerifyAppendConstraints(constraint_state, context.client, combined_chunk, nullptr); } else { DataTable::VerifyUniqueIndexes(local_storage.GetIndexes(data_table), context.client, lstate.insert_chunk, nullptr); @@ -406,7 +418,8 @@ idx_t PhysicalInsert::OnConflictHandling(TableCatalogEntry &table, ExecutionCont InsertLocalState &lstate) const { auto &data_table = table.GetStorage(); if (action_type == OnConflictAction::THROW) { - data_table.VerifyAppendConstraints(table, context.client, lstate.insert_chunk, nullptr); + auto &constraint_state = lstate.GetConstraintState(data_table, table, context.client); + data_table.VerifyAppendConstraints(constraint_state, context.client, lstate.insert_chunk, nullptr); return 0; } // Check whether any conflicts arise, and if they all meet the conflict_target + condition @@ -429,7 +442,7 @@ SinkResultType PhysicalInsert::Sink(ExecutionContext &context, DataChunk &chunk, if (!parallel) { if (!gstate.initialized) { - storage.InitializeLocalAppend(gstate.append_state, context.client); + storage.InitializeLocalAppend(gstate.append_state, table, context.client); gstate.initialized = true; } @@ -487,7 +500,7 @@ SinkCombineResultType PhysicalInsert::Combine(ExecutionContext &context, Operato // we have few rows - append to the local storage directly auto &table = gstate.table; auto &storage = table.GetStorage(); - storage.InitializeLocalAppend(gstate.append_state, context.client); + storage.InitializeLocalAppend(gstate.append_state, table, context.client); auto &transaction = DuckTransaction::Get(context.client, table.catalog); lstate.local_collection->Scan(transaction, [&](DataChunk &insert_chunk) { storage.LocalAppend(gstate.append_state, table, context.client, insert_chunk); diff --git a/src/execution/operator/persistent/physical_update.cpp b/src/execution/operator/persistent/physical_update.cpp index c8bab2854c06..6c8c6c041028 100644 --- a/src/execution/operator/persistent/physical_update.cpp +++ b/src/execution/operator/persistent/physical_update.cpp @@ -8,6 +8,8 @@ #include "duckdb/parallel/thread_context.hpp" #include "duckdb/planner/expression/bound_reference_expression.hpp" #include "duckdb/storage/data_table.hpp" +#include "duckdb/storage/table/delete_state.hpp" +#include "duckdb/storage/table/update_state.hpp" namespace duckdb { @@ -55,6 +57,22 @@ class UpdateLocalState : public LocalSinkState { DataChunk update_chunk; DataChunk mock_chunk; ExpressionExecutor default_executor; + unique_ptr delete_state; + unique_ptr update_state; + + TableDeleteState &GetDeleteState(DataTable &table, TableCatalogEntry &tableref, ClientContext &context) { + if (!delete_state) { + delete_state = table.InitializeDelete(tableref, context); + } + return *delete_state; + } + + TableUpdateState &GetUpdateState(DataTable &table, TableCatalogEntry &tableref, ClientContext &context) { + if (!update_state) { + update_state = table.InitializeUpdate(tableref, context); + } + return *update_state; + } }; SinkResultType PhysicalUpdate::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const { @@ -106,7 +124,8 @@ SinkResultType PhysicalUpdate::Sink(ExecutionContext &context, DataChunk &chunk, // we need to slice here update_chunk.Slice(sel, update_count); } - table.Delete(tableref, context.client, row_ids, update_chunk.size()); + auto &delete_state = lstate.GetDeleteState(table, tableref, context.client); + table.Delete(delete_state, context.client, row_ids, update_chunk.size()); // for the append we need to arrange the columns in a specific manner (namely the "standard table order") mock_chunk.SetCardinality(update_chunk); for (idx_t i = 0; i < columns.size(); i++) { @@ -120,7 +139,8 @@ SinkResultType PhysicalUpdate::Sink(ExecutionContext &context, DataChunk &chunk, mock_chunk.data[columns[i].index].Reference(update_chunk.data[i]); } } - table.Update(tableref, context.client, row_ids, columns, update_chunk); + auto &update_state = lstate.GetUpdateState(table, tableref, context.client); + table.Update(update_state, context.client, row_ids, columns, update_chunk); } if (return_chunk) { diff --git a/src/function/table/system/duckdb_constraints.cpp b/src/function/table/system/duckdb_constraints.cpp index 467aa2ae7605..c35eaf0da9ad 100644 --- a/src/function/table/system/duckdb_constraints.cpp +++ b/src/function/table/system/duckdb_constraints.cpp @@ -15,6 +15,7 @@ #include "duckdb/planner/constraints/bound_not_null_constraint.hpp" #include "duckdb/planner/constraints/bound_foreign_key_constraint.hpp" #include "duckdb/storage/data_table.hpp" +#include "duckdb/planner/binder.hpp" namespace duckdb { @@ -49,11 +50,24 @@ struct hash { namespace duckdb { +struct ConstraintEntry { + ConstraintEntry(ClientContext &context, TableCatalogEntry &table) : table(table) { + if (!table.IsDuckTable()) { + return; + } + auto binder = Binder::CreateBinder(context); + bound_constraints = binder->BindConstraints(table.GetConstraints(), table.name, table.GetColumns()); + } + + TableCatalogEntry &table; + vector> bound_constraints; +}; + struct DuckDBConstraintsData : public GlobalTableFunctionState { DuckDBConstraintsData() : offset(0), constraint_offset(0), unique_constraint_offset(0) { } - vector> entries; + vector entries; idx_t offset; idx_t constraint_offset; idx_t unique_constraint_offset; @@ -118,8 +132,9 @@ unique_ptr DuckDBConstraintsInit(ClientContext &contex }); sort(entries.begin(), entries.end(), [&](CatalogEntry &x, CatalogEntry &y) { return (x.name < y.name); }); - - result->entries.insert(result->entries.end(), entries.begin(), entries.end()); + for(auto &entry : entries) { + result->entries.emplace_back(context, entry.get().Cast()); + } }; return std::move(result); @@ -135,10 +150,9 @@ void DuckDBConstraintsFunction(ClientContext &context, TableFunctionInput &data_ // either fill up the chunk or return all the remaining columns idx_t count = 0; while (data.offset < data.entries.size() && count < STANDARD_VECTOR_SIZE) { - auto &entry = data.entries[data.offset].get(); - D_ASSERT(entry.type == CatalogType::TABLE_ENTRY); + auto &entry = data.entries[data.offset]; - auto &table = entry.Cast(); + auto &table = entry.table; auto &constraints = table.GetConstraints(); bool is_duck_table = table.IsDuckTable(); for (; data.constraint_offset < constraints.size() && count < STANDARD_VECTOR_SIZE; data.constraint_offset++) { @@ -163,7 +177,7 @@ void DuckDBConstraintsFunction(ClientContext &context, TableFunctionInput &data_ if (!is_duck_table) { continue; } - auto &bound_constraints = table.GetBoundConstraints(); + auto &bound_constraints = entry.bound_constraints; auto &bound_foreign_key = bound_constraints[data.constraint_offset]->Cast(); if (bound_foreign_key.info.type == ForeignKeyType::FK_TYPE_PRIMARY_KEY_TABLE) { // Those are already covered by PRIMARY KEY and UNIQUE entries @@ -194,7 +208,7 @@ void DuckDBConstraintsFunction(ClientContext &context, TableFunctionInput &data_ UniqueKeyInfo uk_info; if (is_duck_table) { - auto &bound_constraint = *table.GetBoundConstraints()[data.constraint_offset]; + auto &bound_constraint = *entry.bound_constraints[data.constraint_offset]; switch (bound_constraint.type) { case ConstraintType::UNIQUE: { auto &bound_unique = bound_constraint.Cast(); @@ -251,7 +265,7 @@ void DuckDBConstraintsFunction(ClientContext &context, TableFunctionInput &data_ vector column_index_list; if (is_duck_table) { - auto &bound_constraint = *table.GetBoundConstraints()[data.constraint_offset]; + auto &bound_constraint = *entry.bound_constraints[data.constraint_offset]; switch (bound_constraint.type) { case ConstraintType::CHECK: { auto &bound_check = bound_constraint.Cast(); diff --git a/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp b/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp index 0890ce829ab4..7c1323a4f300 100644 --- a/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp +++ b/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp @@ -24,8 +24,6 @@ class DuckTableEntry : public TableCatalogEntry { void UndoAlter(ClientContext &context, AlterInfo &info) override; //! Returns the underlying storage of the table DataTable &GetStorage() override; - //! Returns a list of the bound constraints of the table - const vector> &GetBoundConstraints() override; //! Get statistics of a column (physical or virtual) within the table unique_ptr GetStatistics(ClientContext &context, column_t column_id) override; @@ -60,13 +58,11 @@ class DuckTableEntry : public TableCatalogEntry { unique_ptr SetColumnComment(ClientContext &context, SetColumnCommentInfo &info); void UpdateConstraintsOnColumnDrop(const LogicalIndex &removed_index, const vector &adjusted_indices, - const RemoveColumnInfo &info, CreateTableInfo &create_info, bool is_generated); + const RemoveColumnInfo &info, CreateTableInfo &create_info, const vector> &bound_constraints, bool is_generated); private: //! A reference to the underlying storage unit used for this table std::shared_ptr storage; - //! A list of constraints that are part of this table - vector> bound_constraints; //! Manages dependencies of the individual columns of the table ColumnDependencyManager column_dependency_manager; }; diff --git a/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp b/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp index 243765a45006..fa9a795f2b0f 100644 --- a/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp +++ b/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp @@ -37,6 +37,7 @@ struct SetColumnCommentInfo; class TableFunction; struct FunctionData; +class Binder; class TableColumnInfo; struct ColumnSegmentInfo; class TableStorageInfo; @@ -74,8 +75,6 @@ class TableCatalogEntry : public StandardEntry { DUCKDB_API const ColumnList &GetColumns() const; //! Returns the underlying storage of the table virtual DataTable &GetStorage(); - //! Returns a list of the bound constraints of the table - virtual const vector> &GetBoundConstraints(); //! Returns a list of the constraints of the table DUCKDB_API const vector> &GetConstraints(); @@ -105,7 +104,7 @@ class TableCatalogEntry : public StandardEntry { //! Returns the storage info of this table virtual TableStorageInfo GetStorageInfo(ClientContext &context) = 0; - virtual void BindUpdateConstraints(LogicalGet &get, LogicalProjection &proj, LogicalUpdate &update, + virtual void BindUpdateConstraints(Binder &binder, LogicalGet &get, LogicalProjection &proj, LogicalUpdate &update, ClientContext &context); protected: diff --git a/src/include/duckdb/planner/binder.hpp b/src/include/duckdb/planner/binder.hpp index 9d992fb407e7..67e2bc1ec6b1 100644 --- a/src/include/duckdb/planner/binder.hpp +++ b/src/include/duckdb/planner/binder.hpp @@ -43,6 +43,7 @@ class ColumnList; class ExternalDependency; class TableFunction; class TableStorageInfo; +class BoundConstraint; struct CreateInfo; struct BoundCreateTableInfo; @@ -121,6 +122,9 @@ class Binder : public std::enable_shared_from_this { unique_ptr BindCreateTableInfo(unique_ptr info, SchemaCatalogEntry &schema); unique_ptr BindCreateTableInfo(unique_ptr info, SchemaCatalogEntry &schema, vector> &bound_defaults); + static vector> BindConstraints(ClientContext &context, const vector> &constraints, const string &table_name, const ColumnList &columns); + vector> BindConstraints(const vector> &constraints, const string &table_name, const ColumnList &columns); + vector> BindNewConstraints(vector> &constraints, const string &table_name, const ColumnList &columns); void BindCreateViewInfo(CreateViewInfo &base); SchemaCatalogEntry &BindSchema(CreateInfo &info); diff --git a/src/include/duckdb/planner/parsed_data/bound_create_table_info.hpp b/src/include/duckdb/planner/parsed_data/bound_create_table_info.hpp index c7a1aac57705..049aca82246d 100644 --- a/src/include/duckdb/planner/parsed_data/bound_create_table_info.hpp +++ b/src/include/duckdb/planner/parsed_data/bound_create_table_info.hpp @@ -36,8 +36,6 @@ struct BoundCreateTableInfo { ColumnDependencyManager column_dependency_manager; //! List of constraints on the table vector> constraints; - //! List of bound constraints on the table - vector> bound_constraints; //! Dependents of the table (in e.g. default values) LogicalDependencyList dependencies; //! The existing table data on disk (if any) diff --git a/src/include/duckdb/storage/data_table.hpp b/src/include/duckdb/storage/data_table.hpp index c6ae98a9f24e..74a5c2283830 100644 --- a/src/include/duckdb/storage/data_table.hpp +++ b/src/include/duckdb/storage/data_table.hpp @@ -41,6 +41,9 @@ class WriteAheadLog; class TableDataWriter; class ConflictManager; class TableScanState; +struct TableDeleteState; +struct ConstraintVerificationState; +struct TableUpdateState; enum class VerifyExistenceType : uint8_t; //! DataTable represents a physical table on disk @@ -92,7 +95,7 @@ class DataTable { const Vector &row_ids, idx_t fetch_count, ColumnFetchState &state); //! Initializes an append to transaction-local storage - void InitializeLocalAppend(LocalAppendState &state, ClientContext &context); + void InitializeLocalAppend(LocalAppendState &state, TableCatalogEntry &table, ClientContext &context); //! Append a DataChunk to the transaction-local storage of the table. void LocalAppend(LocalAppendState &state, TableCatalogEntry &table, ClientContext &context, DataChunk &chunk, bool unsafe = false); @@ -108,10 +111,14 @@ class DataTable { OptimisticDataWriter &CreateOptimisticWriter(ClientContext &context); void FinalizeOptimisticWriter(ClientContext &context, OptimisticDataWriter &writer); + unique_ptr InitializeDelete(TableCatalogEntry &table, ClientContext &context); //! Delete the entries with the specified row identifier from the table - idx_t Delete(TableCatalogEntry &table, ClientContext &context, Vector &row_ids, idx_t count); + idx_t Delete(TableDeleteState &state, ClientContext &context, Vector &row_ids, idx_t count); + + + unique_ptr InitializeUpdate(TableCatalogEntry &table, ClientContext &context); //! Update the entries with the specified row identifier from the table - void Update(TableCatalogEntry &table, ClientContext &context, Vector &row_ids, + void Update(TableUpdateState &state, ClientContext &context, Vector &row_ids, const vector &column_ids, DataChunk &data); //! Update a single (sub-)column along a column path //! The column_path vector is a *path* towards a column within the table @@ -186,22 +193,24 @@ class DataTable { //! FIXME: This is only necessary until we treat all indexes as catalog entries, allowing to alter constraints bool IndexNameIsUnique(const string &name); + //! Initialize constraint verification + unique_ptr InitializeConstraintVerification(TableCatalogEntry &table, ClientContext &context); //! Verify constraints with a chunk from the Append containing all columns of the table - void VerifyAppendConstraints(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk, - ConflictManager *conflict_manager = nullptr); + void VerifyAppendConstraints(ConstraintVerificationState &state, ClientContext &context, DataChunk &chunk, + optional_ptr conflict_manager = nullptr); public: static void VerifyUniqueIndexes(TableIndexList &indexes, ClientContext &context, DataChunk &chunk, - ConflictManager *conflict_manager); + optional_ptr conflict_manager); private: //! Verify the new added constraints against current persistent&local data void VerifyNewConstraint(ClientContext &context, DataTable &parent, const BoundConstraint *constraint); //! Verify constraints with a chunk from the Update containing only the specified column_ids - void VerifyUpdateConstraints(ClientContext &context, TableCatalogEntry &table, DataChunk &chunk, + void VerifyUpdateConstraints(ConstraintVerificationState &state, ClientContext &context, DataChunk &chunk, const vector &column_ids); //! Verify constraints with a chunk from the Delete containing all columns of the table - void VerifyDeleteConstraints(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk); + void VerifyDeleteConstraints(TableDeleteState &state, ClientContext &context, DataChunk &chunk); void InitializeScanWithOffset(TableScanState &state, const vector &column_ids, idx_t start_row, idx_t end_row); diff --git a/src/include/duckdb/storage/table/append_state.hpp b/src/include/duckdb/storage/table/append_state.hpp index 42cd29befad0..475a59187f1f 100644 --- a/src/include/duckdb/storage/table/append_state.hpp +++ b/src/include/duckdb/storage/table/append_state.hpp @@ -14,6 +14,7 @@ #include "duckdb/common/vector.hpp" #include "duckdb/function/compression_function.hpp" #include "duckdb/transaction/transaction_data.hpp" +#include "duckdb/planner/bound_constraint.hpp" namespace duckdb { class ColumnSegment; @@ -69,9 +70,17 @@ struct TableAppendState { TransactionData transaction; }; +struct ConstraintVerificationState { + explicit ConstraintVerificationState(TableCatalogEntry &table_p) : table(table_p) {} + + TableCatalogEntry &table; + vector> bound_constraints; +}; + struct LocalAppendState { TableAppendState append_state; LocalTableStorage *storage; + unique_ptr constraint_state; }; } // namespace duckdb diff --git a/src/include/duckdb/storage/table/delete_state.hpp b/src/include/duckdb/storage/table/delete_state.hpp new file mode 100644 index 000000000000..ec2c2d57c547 --- /dev/null +++ b/src/include/duckdb/storage/table/delete_state.hpp @@ -0,0 +1,23 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// duckdb/storage/table/delete_state.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/storage/table/append_state.hpp" + +namespace duckdb { +class TableCatalogEntry; + +struct TableDeleteState { + vector> bound_constraints; + bool has_delete_constraints = false; + DataChunk verify_chunk; + vector col_ids; +}; + +} // namespace duckdb diff --git a/src/include/duckdb/storage/table/update_state.hpp b/src/include/duckdb/storage/table/update_state.hpp new file mode 100644 index 000000000000..50ce404e0132 --- /dev/null +++ b/src/include/duckdb/storage/table/update_state.hpp @@ -0,0 +1,20 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// duckdb/storage/table/update_state.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/storage/table/append_state.hpp" + +namespace duckdb { +class TableCatalogEntry; + +struct TableUpdateState { + unique_ptr constraint_state; +}; + +} // namespace duckdb diff --git a/src/planner/binder/statement/bind_create_table.cpp b/src/planner/binder/statement/bind_create_table.cpp index 780ef2380497..10ef50599a08 100644 --- a/src/planner/binder/statement/bind_create_table.cpp +++ b/src/planner/binder/statement/bind_create_table.cpp @@ -35,61 +35,56 @@ static void CreateColumnDependencyManager(BoundCreateTableInfo &info) { } } -static void BindCheckConstraint(Binder &binder, BoundCreateTableInfo &info, const unique_ptr &cond) { - auto &base = info.base->Cast(); - +static unique_ptr BindCheckConstraint(Binder &binder, const string &table_name, const ColumnList &columns, const unique_ptr &cond) { auto bound_constraint = make_uniq(); // check constraint: bind the expression - CheckBinder check_binder(binder, binder.context, base.table, base.columns, bound_constraint->bound_columns); + CheckBinder check_binder(binder, binder.context, table_name, columns, bound_constraint->bound_columns); auto &check = cond->Cast(); // create a copy of the unbound expression because the binding destroys the constraint auto unbound_expression = check.expression->Copy(); // now bind the constraint and create a new BoundCheckConstraint - bound_constraint->expression = check_binder.Bind(check.expression); - info.bound_constraints.push_back(std::move(bound_constraint)); - // move the unbound constraint back into the original check expression - check.expression = std::move(unbound_expression); + bound_constraint->expression = check_binder.Bind(unbound_expression); + return std::move(bound_constraint); } -static void BindConstraints(Binder &binder, BoundCreateTableInfo &info) { - auto &base = info.base->Cast(); +vector> Binder::BindConstraints(ClientContext &context, const vector> &constraints, const string &table_name, const ColumnList &columns) { + auto binder = Binder::CreateBinder(context); + return binder->BindConstraints(constraints, table_name, columns); +} - bool has_primary_key = false; - logical_index_set_t not_null_columns; - vector primary_keys; - for (idx_t i = 0; i < base.constraints.size(); i++) { - auto &cond = base.constraints[i]; - switch (cond->type) { +vector> Binder::BindConstraints(const vector> &constraints, const string &table_name, const ColumnList &columns) { + vector> bound_constraints; + for (auto &constr : constraints) { + switch (constr->type) { case ConstraintType::CHECK: { - BindCheckConstraint(binder, info, cond); + bound_constraints.push_back(BindCheckConstraint(*this, table_name, columns, constr)); break; } case ConstraintType::NOT_NULL: { - auto ¬_null = cond->Cast(); - auto &col = base.columns.GetColumn(LogicalIndex(not_null.index)); - info.bound_constraints.push_back(make_uniq(PhysicalIndex(col.StorageOid()))); - not_null_columns.insert(not_null.index); + auto ¬_null = constr->Cast(); + auto &col = columns.GetColumn(LogicalIndex(not_null.index)); + bound_constraints.push_back(make_uniq(PhysicalIndex(col.StorageOid()))); break; } case ConstraintType::UNIQUE: { - auto &unique = cond->Cast(); + auto &unique = constr->Cast(); // have to resolve columns of the unique constraint vector keys; logical_index_set_t key_set; if (unique.HasIndex()) { - D_ASSERT(unique.GetIndex().index < base.columns.LogicalColumnCount()); + D_ASSERT(unique.GetIndex().index < columns.LogicalColumnCount()); // unique constraint is given by single index - unique.SetColumnName(base.columns.GetColumn(unique.GetIndex()).Name()); + unique.SetColumnName(columns.GetColumn(unique.GetIndex()).Name()); keys.push_back(unique.GetIndex()); key_set.insert(unique.GetIndex()); } else { // unique constraint is given by list of names // have to resolve names for (auto &keyname : unique.GetColumnNames()) { - if (!base.columns.ColumnExists(keyname)) { + if (!columns.ColumnExists(keyname)) { throw ParserException("column \"%s\" named in key does not exist", keyname); } - auto &column = base.columns.GetColumn(keyname); + auto &column = columns.GetColumn(keyname); auto column_index = column.Logical(); if (key_set.find(column_index) != key_set.end()) { throw ParserException("column \"%s\" appears twice in " @@ -100,38 +95,29 @@ static void BindConstraints(Binder &binder, BoundCreateTableInfo &info) { key_set.insert(column_index); } } - - if (unique.IsPrimaryKey()) { - // we can only have one primary key per table - if (has_primary_key) { - throw ParserException("table \"%s\" has more than one primary key", base.table); - } - has_primary_key = true; - primary_keys = keys; - } - info.bound_constraints.push_back( + bound_constraints.push_back( make_uniq(std::move(keys), std::move(key_set), unique.IsPrimaryKey())); break; } case ConstraintType::FOREIGN_KEY: { - auto &fk = cond->Cast(); + auto &fk = constr->Cast(); D_ASSERT((fk.info.type == ForeignKeyType::FK_TYPE_FOREIGN_KEY_TABLE && !fk.info.pk_keys.empty()) || (fk.info.type == ForeignKeyType::FK_TYPE_PRIMARY_KEY_TABLE && !fk.info.pk_keys.empty()) || fk.info.type == ForeignKeyType::FK_TYPE_SELF_REFERENCE_TABLE); physical_index_set_t fk_key_set, pk_key_set; - for (idx_t i = 0; i < fk.info.pk_keys.size(); i++) { - if (pk_key_set.find(fk.info.pk_keys[i]) != pk_key_set.end()) { + for (auto &pk_key : fk.info.pk_keys) { + if (pk_key_set.find(pk_key) != pk_key_set.end()) { throw BinderException("Duplicate primary key referenced in FOREIGN KEY constraint"); } - pk_key_set.insert(fk.info.pk_keys[i]); + pk_key_set.insert(pk_key); } - for (idx_t i = 0; i < fk.info.fk_keys.size(); i++) { - if (fk_key_set.find(fk.info.fk_keys[i]) != fk_key_set.end()) { + for (auto &fk_key : fk.info.fk_keys) { + if (fk_key_set.find(fk_key) != fk_key_set.end()) { throw BinderException("Duplicate key specified in FOREIGN KEY constraint"); } - fk_key_set.insert(fk.info.fk_keys[i]); + fk_key_set.insert(fk_key); } - info.bound_constraints.push_back( + bound_constraints.push_back( make_uniq(fk.info, std::move(pk_key_set), std::move(fk_key_set))); break; } @@ -139,6 +125,43 @@ static void BindConstraints(Binder &binder, BoundCreateTableInfo &info) { throw NotImplementedException("unrecognized constraint type in bind"); } } + return bound_constraints; +} + +vector> Binder::BindNewConstraints(vector> &constraints, const string &table_name, const ColumnList &columns) { + auto bound_constraints = BindConstraints(constraints, table_name, columns); + + // handle primary keys/not null constraints + bool has_primary_key = false; + logical_index_set_t not_null_columns; + vector primary_keys; + for(idx_t c = 0; c < constraints.size(); c++) { + auto &constr = constraints[c]; + switch(constr->type) { + case ConstraintType::NOT_NULL: { + auto ¬_null = constr->Cast(); + auto &col = columns.GetColumn(LogicalIndex(not_null.index)); + bound_constraints.push_back(make_uniq(PhysicalIndex(col.StorageOid()))); + not_null_columns.insert(not_null.index); + break; + } + case ConstraintType::UNIQUE: { + auto &unique = constr->Cast(); + auto &bound_unique = bound_constraints[c]->Cast(); + if (unique.IsPrimaryKey()) { + // we can only have one primary key per table + if (has_primary_key) { + throw ParserException("table \"%s\" has more than one primary key", table_name); + } + has_primary_key = true; + primary_keys = bound_unique.keys; + } + break; + } + default: + break; + } + } if (has_primary_key) { // if there is a primary key index, also create a NOT NULL constraint for each of the columns for (auto &column_index : primary_keys) { @@ -146,11 +169,12 @@ static void BindConstraints(Binder &binder, BoundCreateTableInfo &info) { //! No need to create a NotNullConstraint, it's already present continue; } - auto physical_index = base.columns.LogicalToPhysical(column_index); - base.constraints.push_back(make_uniq(column_index)); - info.bound_constraints.push_back(make_uniq(physical_index)); + auto physical_index = columns.LogicalToPhysical(column_index); + constraints.push_back(make_uniq(column_index)); + bound_constraints.push_back(make_uniq(physical_index)); } } + return bound_constraints; } void Binder::BindGeneratedColumns(BoundCreateTableInfo &info) { @@ -239,13 +263,13 @@ static void ExtractExpressionDependencies(Expression &expr, LogicalDependencyLis expr, [&](Expression &child) { ExtractExpressionDependencies(child, dependencies); }); } -static void ExtractDependencies(BoundCreateTableInfo &info, vector> &bound_defaults) { - for (auto &default_value : bound_defaults) { +static void ExtractDependencies(BoundCreateTableInfo &info, vector> &defaults, vector> &constraints) { + for (auto &default_value : defaults) { if (default_value) { ExtractExpressionDependencies(*default_value, info.dependencies); } } - for (auto &constraint : info.bound_constraints) { + for (auto &constraint : constraints) { if (constraint->type == ConstraintType::CHECK) { auto &bound_check = constraint->Cast(); ExtractExpressionDependencies(*bound_check.expression, info.dependencies); @@ -262,6 +286,8 @@ unique_ptr Binder::BindCreateTableInfo(unique_ptr> &bound_defaults) { auto &base = info->Cast(); auto result = make_uniq(schema, std::move(info)); + + vector> bound_constraints; if (base.query) { // construct the result object auto query_obj = Bind(*base.query); @@ -284,12 +310,12 @@ unique_ptr Binder::BindCreateTableInfo(unique_ptr(std::move(proj_tmp)); // bind any extra columns necessary for CHECK constraints or indexes - table.BindUpdateConstraints(*get, *proj, *update, context); + table.BindUpdateConstraints(*this, *get, *proj, *update, context); // finally add the row id column to the projection list proj->expressions.push_back(make_uniq( diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 770a18ed43b2..8dab59b0e1a8 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -23,7 +23,9 @@ #include "duckdb/common/types/conflict_manager.hpp" #include "duckdb/common/types/constraint_conflict_info.hpp" #include "duckdb/storage/table/append_state.hpp" +#include "duckdb/storage/table/delete_state.hpp" #include "duckdb/storage/table/scan_state.hpp" +#include "duckdb/storage/table/update_state.hpp" #include "duckdb/common/exception/transaction_exception.hpp" namespace duckdb { @@ -549,7 +551,7 @@ bool HasUniqueIndexes(TableIndexList &list) { bool has_unique_index = false; list.Scan([&](Index &index) { if (index.IsUnique()) { - return has_unique_index = true; + has_unique_index = true; return true; } return false; @@ -558,7 +560,7 @@ bool HasUniqueIndexes(TableIndexList &list) { } void DataTable::VerifyUniqueIndexes(TableIndexList &indexes, ClientContext &context, DataChunk &chunk, - ConflictManager *conflict_manager) { + optional_ptr conflict_manager) { //! check whether or not the chunk can be inserted into the indexes if (!conflict_manager) { // Only need to verify that no unique constraints are violated @@ -614,8 +616,9 @@ void DataTable::VerifyUniqueIndexes(TableIndexList &indexes, ClientContext &cont }); } -void DataTable::VerifyAppendConstraints(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk, - ConflictManager *conflict_manager) { +void DataTable::VerifyAppendConstraints(ConstraintVerificationState &state, ClientContext &context, DataChunk &chunk, + optional_ptr conflict_manager) { + auto &table = state.table; if (table.HasGeneratedColumns()) { // Verify that the generated columns expression work with the inserted values auto binder = Binder::CreateBinder(context); @@ -638,10 +641,9 @@ void DataTable::VerifyAppendConstraints(TableCatalogEntry &table, ClientContext } auto &constraints = table.GetConstraints(); - auto &bound_constraints = table.GetBoundConstraints(); - for (idx_t i = 0; i < bound_constraints.size(); i++) { + for (idx_t i = 0; i < state.bound_constraints.size(); i++) { auto &base_constraint = constraints[i]; - auto &constraint = bound_constraints[i]; + auto &constraint = state.bound_constraints[i]; switch (base_constraint->type) { case ConstraintType::NOT_NULL: { auto &bound_not_null = *reinterpret_cast(constraint.get()); @@ -673,12 +675,21 @@ void DataTable::VerifyAppendConstraints(TableCatalogEntry &table, ClientContext } } -void DataTable::InitializeLocalAppend(LocalAppendState &state, ClientContext &context) { +unique_ptr DataTable::InitializeConstraintVerification(TableCatalogEntry &table, ClientContext &context) { + auto result = make_uniq(table); + auto binder = Binder::CreateBinder(context); + result->bound_constraints = binder->BindConstraints(table.GetConstraints(), table.name, table.GetColumns()); + return result; +} + +void DataTable::InitializeLocalAppend(LocalAppendState &state, TableCatalogEntry &table, ClientContext &context) { if (!is_root) { throw TransactionException("Transaction conflict: adding entries to a table that has been altered!"); } auto &local_storage = LocalStorage::Get(context, db); local_storage.InitializeAppend(state, *this); + + state.constraint_state = InitializeConstraintVerification(table, context); } void DataTable::LocalAppend(LocalAppendState &state, TableCatalogEntry &table, ClientContext &context, DataChunk &chunk, @@ -695,7 +706,7 @@ void DataTable::LocalAppend(LocalAppendState &state, TableCatalogEntry &table, C // verify any constraints on the new chunk if (!unsafe) { - VerifyAppendConstraints(table, context, chunk); + VerifyAppendConstraints(*state.constraint_state, context, chunk); } // append to the transaction local data @@ -724,7 +735,7 @@ void DataTable::LocalMerge(ClientContext &context, RowGroupCollection &collectio void DataTable::LocalAppend(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk) { LocalAppendState append_state; auto &storage = table.GetStorage(); - storage.InitializeLocalAppend(append_state, context); + storage.InitializeLocalAppend(append_state, table, context); storage.LocalAppend(append_state, table, context, chunk); storage.FinalizeLocalAppend(append_state); } @@ -732,7 +743,7 @@ void DataTable::LocalAppend(TableCatalogEntry &table, ClientContext &context, Da void DataTable::LocalAppend(TableCatalogEntry &table, ClientContext &context, ColumnDataCollection &collection) { LocalAppendState append_state; auto &storage = table.GetStorage(); - storage.InitializeLocalAppend(append_state, context); + storage.InitializeLocalAppend(append_state, table, context); for (auto &chunk : collection.Chunks()) { storage.LocalAppend(append_state, table, context, chunk); } @@ -956,15 +967,14 @@ void DataTable::RemoveFromIndexes(Vector &row_identifiers, idx_t count) { // Delete //===--------------------------------------------------------------------===// static bool TableHasDeleteConstraints(TableCatalogEntry &table) { - auto &bound_constraints = table.GetBoundConstraints(); - for (auto &constraint : bound_constraints) { + for (auto &constraint : table.GetConstraints()) { switch (constraint->type) { case ConstraintType::NOT_NULL: case ConstraintType::CHECK: case ConstraintType::UNIQUE: break; case ConstraintType::FOREIGN_KEY: { - auto &bfk = *reinterpret_cast(constraint.get()); + auto &bfk = constraint->Cast(); if (bfk.info.type == ForeignKeyType::FK_TYPE_PRIMARY_KEY_TABLE || bfk.info.type == ForeignKeyType::FK_TYPE_SELF_REFERENCE_TABLE) { return true; @@ -978,9 +988,8 @@ static bool TableHasDeleteConstraints(TableCatalogEntry &table) { return false; } -void DataTable::VerifyDeleteConstraints(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk) { - auto &bound_constraints = table.GetBoundConstraints(); - for (auto &constraint : bound_constraints) { +void DataTable::VerifyDeleteConstraints(TableDeleteState &state, ClientContext &context, DataChunk &chunk) { + for (auto &constraint : state.bound_constraints) { switch (constraint->type) { case ConstraintType::NOT_NULL: case ConstraintType::CHECK: @@ -1000,33 +1009,39 @@ void DataTable::VerifyDeleteConstraints(TableCatalogEntry &table, ClientContext } } -idx_t DataTable::Delete(TableCatalogEntry &table, ClientContext &context, Vector &row_identifiers, idx_t count) { +unique_ptr DataTable::InitializeDelete(TableCatalogEntry &table, ClientContext &context) { + // initialize indexes (if any) + info->InitializeIndexes(context, true); + + auto binder = Binder::CreateBinder(context); + vector> bound_constraints; + vector types; + auto result = make_uniq(); + result->has_delete_constraints = TableHasDeleteConstraints(table); + if (result->has_delete_constraints) { + // initialize the chunk if there are any constraints to verify + for (idx_t i = 0; i < column_definitions.size(); i++) { + result->col_ids.push_back(column_definitions[i].StorageOid()); + types.emplace_back(column_definitions[i].Type()); + } + result->verify_chunk.Initialize(Allocator::Get(context), types); + result->bound_constraints = binder->BindConstraints(table.GetConstraints(), table.name, table.GetColumns()); + } + return result; +} + +idx_t DataTable::Delete(TableDeleteState &state, ClientContext &context, Vector &row_identifiers, idx_t count) { D_ASSERT(row_identifiers.GetType().InternalType() == ROW_TYPE); if (count == 0) { return 0; } - info->InitializeIndexes(context, true); - auto &transaction = DuckTransaction::Get(context, db); auto &local_storage = LocalStorage::Get(transaction); - bool has_delete_constraints = TableHasDeleteConstraints(table); row_identifiers.Flatten(count); auto ids = FlatVector::GetData(row_identifiers); - DataChunk verify_chunk; - vector col_ids; - vector types; - ColumnFetchState fetch_state; - if (has_delete_constraints) { - // initialize the chunk if there are any constraints to verify - for (idx_t i = 0; i < column_definitions.size(); i++) { - col_ids.push_back(column_definitions[i].StorageOid()); - types.emplace_back(column_definitions[i].Type()); - } - verify_chunk.Initialize(Allocator::Get(context), types); - } idx_t pos = 0; idx_t delete_count = 0; while (pos < count) { @@ -1045,18 +1060,20 @@ idx_t DataTable::Delete(TableCatalogEntry &table, ClientContext &context, Vector Vector offset_ids(row_identifiers, current_offset, pos); if (is_transaction_delete) { // transaction-local delete - if (has_delete_constraints) { + if (state.has_delete_constraints) { // perform the constraint verification - local_storage.FetchChunk(*this, offset_ids, current_count, col_ids, verify_chunk, fetch_state); - VerifyDeleteConstraints(table, context, verify_chunk); + ColumnFetchState fetch_state; + local_storage.FetchChunk(*this, offset_ids, current_count, state.col_ids, state.verify_chunk, fetch_state); + VerifyDeleteConstraints(state, context, state.verify_chunk); } delete_count += local_storage.Delete(*this, offset_ids, current_count); } else { // regular table delete - if (has_delete_constraints) { + if (state.has_delete_constraints) { // perform the constraint verification - Fetch(transaction, verify_chunk, col_ids, offset_ids, current_count, fetch_state); - VerifyDeleteConstraints(table, context, verify_chunk); + ColumnFetchState fetch_state; + Fetch(transaction, state.verify_chunk, state.col_ids, offset_ids, current_count, fetch_state); + VerifyDeleteConstraints(state, context, state.verify_chunk); } delete_count += row_groups->Delete(transaction, *this, ids + current_offset, current_count); } @@ -1101,10 +1118,11 @@ static bool CreateMockChunk(TableCatalogEntry &table, const vector &column_ids) { + auto &table = state.table; auto &constraints = table.GetConstraints(); - auto &bound_constraints = table.GetBoundConstraints(); + auto &bound_constraints = state.bound_constraints; for (idx_t constr_idx = 0; constr_idx < bound_constraints.size(); constr_idx++) { auto &base_constraint = constraints[constr_idx]; auto &constraint = bound_constraints[constr_idx]; @@ -1150,7 +1168,16 @@ void DataTable::VerifyUpdateConstraints(ClientContext &context, TableCatalogEntr #endif } -void DataTable::Update(TableCatalogEntry &table, ClientContext &context, Vector &row_ids, +unique_ptr DataTable::InitializeUpdate(TableCatalogEntry &table, ClientContext &context) { + // check that there are no unknown indexes + info->InitializeIndexes(context, true); + + auto result = make_uniq(); + result->constraint_state = InitializeConstraintVerification(table, context); + return result; +} + +void DataTable::Update(TableUpdateState &state, ClientContext &context, Vector &row_ids, const vector &column_ids, DataChunk &updates) { D_ASSERT(row_ids.GetType().InternalType() == ROW_TYPE); D_ASSERT(column_ids.size() == updates.ColumnCount()); @@ -1165,11 +1192,8 @@ void DataTable::Update(TableCatalogEntry &table, ClientContext &context, Vector throw TransactionException("Transaction conflict: cannot update a table that has been altered!"); } - // check that there are no unknown indexes - info->InitializeIndexes(context, true); - // first verify that no constraints are violated - VerifyUpdateConstraints(context, table, updates, column_ids); + VerifyUpdateConstraints(*state.constraint_state, context, updates, column_ids); // now perform the actual update Vector max_row_id_vec(Value::BIGINT(MAX_ROW_ID)); diff --git a/src/storage/wal_replay.cpp b/src/storage/wal_replay.cpp index 9699bbac3e9d..6f5357e3d9c2 100644 --- a/src/storage/wal_replay.cpp +++ b/src/storage/wal_replay.cpp @@ -678,9 +678,10 @@ void WriteAheadLogDeserializer::ReplayDelete() { auto source_ids = FlatVector::GetData(chunk.data[0]); // delete the tuples from the current table + TableDeleteState delete_state; for (idx_t i = 0; i < chunk.size(); i++) { row_ids[0] = source_ids[i]; - state.current_table->GetStorage().Delete(*state.current_table, context, row_identifiers, 1); + state.current_table->GetStorage().Delete(delete_state, context, row_identifiers, 1); } } From c98a18eb9b2cf574868da91c0516e4c8af4b06a8 Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 18 Apr 2024 22:13:41 +0200 Subject: [PATCH 565/603] accept NULL type, if any of the inputs are NULL, the result becomes NULL --- src/core_functions/scalar/map/map.cpp | 33 +++++++++++++++++---------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/src/core_functions/scalar/map/map.cpp b/src/core_functions/scalar/map/map.cpp index e27fe3fd6503..62f54cdd25d0 100644 --- a/src/core_functions/scalar/map/map.cpp +++ b/src/core_functions/scalar/map/map.cpp @@ -28,6 +28,13 @@ static void MapFunction(DataChunk &args, ExpressionState &, Vector &result) { // - STRUCTs have exactly two fields, a key-field, and a value-field // - key names are unique + if (result.GetType().id() == LogicalTypeId::SQLNULL) { + auto &validity = FlatVector::Validity(result); + validity.SetInvalid(0); + result.SetVectorType(VectorType::CONSTANT_VECTOR); + return; + } + D_ASSERT(result.GetType().id() == LogicalTypeId::MAP); auto row_count = args.size(); @@ -63,13 +70,15 @@ static void MapFunction(DataChunk &args, ExpressionState &, Vector &result) { UnifiedVectorFormat result_data; result.ToUnifiedFormat(row_count, result_data); auto result_entries = UnifiedVectorFormat::GetDataNoConst(result_data); - result_data.validity.SetAllValid(row_count); + + auto &result_validity = FlatVector::Validity(result); // get the resulting size of the key/value child lists idx_t result_child_size = 0; for (idx_t row_idx = 0; row_idx < row_count; row_idx++) { auto keys_idx = keys_data.sel->get_index(row_idx); - if (!keys_data.validity.RowIsValid(keys_idx)) { + auto values_idx = values_data.sel->get_index(row_idx); + if (!keys_data.validity.RowIsValid(keys_idx) || !values_data.validity.RowIsValid(values_idx)) { continue; } auto keys_entry = keys_entries[keys_idx]; @@ -87,22 +96,15 @@ static void MapFunction(DataChunk &args, ExpressionState &, Vector &result) { auto values_idx = values_data.sel->get_index(row_idx); auto result_idx = result_data.sel->get_index(row_idx); - // empty map - if (!keys_data.validity.RowIsValid(keys_idx) && !values_data.validity.RowIsValid(values_idx)) { - result_entries[result_idx] = list_entry_t(); + // NULL MAP + if (!keys_data.validity.RowIsValid(keys_idx) || !values_data.validity.RowIsValid(values_idx)) { + result_validity.SetInvalid(row_idx); continue; } auto keys_entry = keys_entries[keys_idx]; auto values_entry = values_entries[values_idx]; - // validity checks - if (!keys_data.validity.RowIsValid(keys_idx)) { - MapVector::EvalMapInvalidReason(MapInvalidReason::NULL_KEY_LIST); - } - if (!values_data.validity.RowIsValid(values_idx)) { - MapVector::EvalMapInvalidReason(MapInvalidReason::NULL_VALUE_LIST); - } if (keys_entry.length != values_entry.length) { MapVector::EvalMapInvalidReason(MapInvalidReason::NOT_ALIGNED); } @@ -166,6 +168,13 @@ static unique_ptr MapBind(ClientContext &, ScalarFunction &bound_f return make_uniq(bound_function.return_type); } + auto key_id = arguments[0]->return_type.id(); + auto value_id = arguments[1]->return_type.id(); + if (key_id == LogicalTypeId::SQLNULL || value_id == LogicalTypeId::SQLNULL) { + bound_function.return_type = LogicalTypeId::SQLNULL; + return make_uniq(bound_function.return_type); + } + // bind a MAP with key-value pairs D_ASSERT(arguments.size() == 2); if (arguments[0]->return_type.id() != LogicalTypeId::LIST) { From 253bd13f90aaa067757baae47c771c980ef31933 Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 18 Apr 2024 22:14:54 +0200 Subject: [PATCH 566/603] add test for NULL maps --- test/sql/types/map/map_null.test | 35 ++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 test/sql/types/map/map_null.test diff --git a/test/sql/types/map/map_null.test b/test/sql/types/map/map_null.test new file mode 100644 index 000000000000..6bc138488856 --- /dev/null +++ b/test/sql/types/map/map_null.test @@ -0,0 +1,35 @@ +# name: test/sql/types/map/map_null.test +# group: [map] + +statement ok +pragma enable_verification; + +query I +select map(NULL::INT[], [1,2,3]) +---- +NULL + +query I +select map(NULL, [1,2,3]) +---- +NULL + +query I +select map(NULL, NULL) +---- +NULL + +query I +select map(NULL, [1,2,3]) IS NULL +---- +true + +query I +select map([1,2,3], NULL) +---- +NULL + +query I +select map([1,2,3], NULL::INT[]) +---- +NULL From a371e61859ff26c6ca65359a78f60c31a4667d5a Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 18 Apr 2024 22:22:38 +0200 Subject: [PATCH 567/603] change existing tests --- test/sql/types/nested/map/map_error.test | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/sql/types/nested/map/map_error.test b/test/sql/types/nested/map/map_error.test index 315a1620ea7a..f67c19d4ead2 100644 --- a/test/sql/types/nested/map/map_error.test +++ b/test/sql/types/nested/map/map_error.test @@ -75,10 +75,11 @@ CREATE TABLE null_keys_list (k INT[], v INT[]); statement ok INSERT INTO null_keys_list VALUES ([1], [2]), (NULL, [4]); -statement error +query I SELECT MAP(k, v) FROM null_keys_list; ---- -The list of map keys must not be NULL. +{1=2} +NULL statement ok CREATE TABLE null_values_list (k INT[], v INT[]); @@ -86,7 +87,8 @@ CREATE TABLE null_values_list (k INT[], v INT[]); statement ok INSERT INTO null_values_list VALUES ([1], [2]), ([4], NULL); -statement error +query I SELECT MAP(k, v) FROM null_values_list; ---- -The list of map values must not be NULL. \ No newline at end of file +{1=2} +NULL From c7b80a695e4b3784e14a5a35bce4196c2108bc95 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 18 Apr 2024 22:29:19 +0200 Subject: [PATCH 568/603] Add also amazonlinux container tests --- scripts/test_docker_images.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/test_docker_images.sh b/scripts/test_docker_images.sh index 7f9deaf196ae..86febb0328fd 100755 --- a/scripts/test_docker_images.sh +++ b/scripts/test_docker_images.sh @@ -1,7 +1,10 @@ #!/usr/bin/env bash make clean +docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb amazonlinux:2 <<< "yum install g++ git make cmake ninja-build -y && GEN=ninja make && make clean" 2>&1 +docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb amazonlinux:latest <<< "yum install clang git make cmake ninja-build -y && GEN=ninja make && make clean" 2>&1 docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb alpine:latest <<< "apk add g++ git make cmake ninja && GEN=ninja make && make clean" 2>&1 docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb alpine:latest <<< "apk add g++ git make cmake ninja python3 && GEN=ninja make && make clean" 2>&1 docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb alpine:latest <<< "apk add g++ git make cmake ninja && CXX_STANDARD=23 GEN=ninja make && make clean" 2>&1 docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb ubuntu:20.04 <<< "apt-get update && export DEBIAN_FRONTEND=noninteractive && apt-get install g++ git make cmake ninja-build -y && GEN=ninja make && make clean" 2>&1 + From 3ed2ae9372c57e7bb34213f39e36ff306228dbf0 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 18 Apr 2024 22:29:38 +0200 Subject: [PATCH 569/603] Add test also on ubuntu:devel --- scripts/test_docker_images.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/test_docker_images.sh b/scripts/test_docker_images.sh index 86febb0328fd..173d0b9f340a 100755 --- a/scripts/test_docker_images.sh +++ b/scripts/test_docker_images.sh @@ -7,4 +7,5 @@ docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb alpine:latest <<< "apk ad docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb alpine:latest <<< "apk add g++ git make cmake ninja python3 && GEN=ninja make && make clean" 2>&1 docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb alpine:latest <<< "apk add g++ git make cmake ninja && CXX_STANDARD=23 GEN=ninja make && make clean" 2>&1 docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb ubuntu:20.04 <<< "apt-get update && export DEBIAN_FRONTEND=noninteractive && apt-get install g++ git make cmake ninja-build -y && GEN=ninja make && make clean" 2>&1 +docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb ubuntu:devel <<< "apt-get update && export DEBIAN_FRONTEND=noninteractive && apt-get install g++ git make cmake ninja-build -y && GEN=ninja make && make clean" 2>&1 From d1883806b1537054d4890d0c53ae6619f9261057 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 18 Apr 2024 22:43:07 +0200 Subject: [PATCH 570/603] Add centos to containerized tests --- scripts/test_docker_images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test_docker_images.sh b/scripts/test_docker_images.sh index 173d0b9f340a..3b180e963784 100755 --- a/scripts/test_docker_images.sh +++ b/scripts/test_docker_images.sh @@ -8,4 +8,4 @@ docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb alpine:latest <<< "apk ad docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb alpine:latest <<< "apk add g++ git make cmake ninja && CXX_STANDARD=23 GEN=ninja make && make clean" 2>&1 docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb ubuntu:20.04 <<< "apt-get update && export DEBIAN_FRONTEND=noninteractive && apt-get install g++ git make cmake ninja-build -y && GEN=ninja make && make clean" 2>&1 docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb ubuntu:devel <<< "apt-get update && export DEBIAN_FRONTEND=noninteractive && apt-get install g++ git make cmake ninja-build -y && GEN=ninja make && make clean" 2>&1 - +docker run -i --rm -v $(pwd):/duckdb --workdir /duckdb centos <<< "sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* && sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* && yum install git make cmake clang -y && make && make clean" 2>&1 From 15803cfdd6eb54d17b1a5316503435fffcc0fa2c Mon Sep 17 00:00:00 2001 From: stephaniewang Date: Thu, 18 Apr 2024 16:54:57 -0400 Subject: [PATCH 571/603] feat: rewrite which_secret() into a table function --- src/core_functions/function_list.cpp | 1 - src/core_functions/scalar/CMakeLists.txt | 1 - .../scalar/secret/CMakeLists.txt | 4 - .../scalar/secret/functions.json | 9 --- .../scalar/secret/which_secret.cpp | 28 ------- src/function/table/system/CMakeLists.txt | 1 + .../table/system/duckdb_which_secret.cpp | 75 +++++++++++++++++++ src/function/table/system_functions.cpp | 1 + .../function/table/system_functions.hpp | 4 + test/secrets/test_custom_secret_storage.cpp | 14 ++-- .../secrets/create_secret_scope_matching.test | 23 +++++- 11 files changed, 107 insertions(+), 54 deletions(-) delete mode 100644 src/core_functions/scalar/secret/CMakeLists.txt delete mode 100644 src/core_functions/scalar/secret/functions.json delete mode 100644 src/core_functions/scalar/secret/which_secret.cpp create mode 100644 src/function/table/system/duckdb_which_secret.cpp diff --git a/src/core_functions/function_list.cpp b/src/core_functions/function_list.cpp index 540752a4f48c..23bf82242562 100644 --- a/src/core_functions/function_list.cpp +++ b/src/core_functions/function_list.cpp @@ -385,7 +385,6 @@ static const StaticFunctionDefinition internal_functions[] = { DUCKDB_SCALAR_FUNCTION_SET(WeekFun), DUCKDB_SCALAR_FUNCTION_SET(WeekDayFun), DUCKDB_SCALAR_FUNCTION_SET(WeekOfYearFun), - DUCKDB_SCALAR_FUNCTION(WhichSecretFun), DUCKDB_SCALAR_FUNCTION_SET(BitwiseXorFun), DUCKDB_SCALAR_FUNCTION_SET(YearFun), DUCKDB_SCALAR_FUNCTION_SET(YearWeekFun), diff --git a/src/core_functions/scalar/CMakeLists.txt b/src/core_functions/scalar/CMakeLists.txt index c6dd785c5b37..b4f0ff7c7bc4 100644 --- a/src/core_functions/scalar/CMakeLists.txt +++ b/src/core_functions/scalar/CMakeLists.txt @@ -9,7 +9,6 @@ add_subdirectory(map) add_subdirectory(math) add_subdirectory(operators) add_subdirectory(random) -add_subdirectory(secret) add_subdirectory(string) add_subdirectory(struct) add_subdirectory(union) diff --git a/src/core_functions/scalar/secret/CMakeLists.txt b/src/core_functions/scalar/secret/CMakeLists.txt deleted file mode 100644 index 6937dcd6003e..000000000000 --- a/src/core_functions/scalar/secret/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -add_library_unity(duckdb_func_secret OBJECT which_secret.cpp) -set(ALL_OBJECT_FILES - ${ALL_OBJECT_FILES} $ - PARENT_SCOPE) diff --git a/src/core_functions/scalar/secret/functions.json b/src/core_functions/scalar/secret/functions.json deleted file mode 100644 index fb24476be3d7..000000000000 --- a/src/core_functions/scalar/secret/functions.json +++ /dev/null @@ -1,9 +0,0 @@ -[ - { - "name": "which_secret", - "parameters": "path,type", - "description": "Print out the name of the secret that will be used for reading a path", - "example": "which_secret('s3://some/authenticated/path.csv', 's3')", - "type": "scalar_function" - } -] diff --git a/src/core_functions/scalar/secret/which_secret.cpp b/src/core_functions/scalar/secret/which_secret.cpp deleted file mode 100644 index dfa54e6278cc..000000000000 --- a/src/core_functions/scalar/secret/which_secret.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#include "duckdb/core_functions/scalar/secret_functions.hpp" -#include "duckdb/main/secret/secret_manager.hpp" - -namespace duckdb { - -static void WhichSecretFunction(DataChunk &args, ExpressionState &state, Vector &result) { - D_ASSERT(args.ColumnCount() == 2); - - auto &secret_manager = SecretManager::Get(state.GetContext()); - auto transaction = CatalogTransaction::GetSystemCatalogTransaction(state.GetContext()); - - BinaryExecutor::Execute( - args.data[0], args.data[1], result, args.size(), [&](string_t path, string_t type) { - auto secret_match = secret_manager.LookupSecret(transaction, path.GetString(), type.GetString()); - if (!secret_match.HasMatch()) { - return string_t(); - } - return StringVector::AddString(result, secret_match.GetSecret().GetName()); - }); -} - -ScalarFunction WhichSecretFun::GetFunction() { - ScalarFunction which_secret("which_secret", {LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::VARCHAR, - WhichSecretFunction, nullptr, nullptr, nullptr, nullptr); - return which_secret; -} - -} // namespace duckdb diff --git a/src/function/table/system/CMakeLists.txt b/src/function/table/system/CMakeLists.txt index 066f216de150..14d15264ca17 100644 --- a/src/function/table/system/CMakeLists.txt +++ b/src/function/table/system/CMakeLists.txt @@ -13,6 +13,7 @@ add_library_unity( duckdb_optimizers.cpp duckdb_schemas.cpp duckdb_secrets.cpp + duckdb_which_secret.cpp duckdb_sequences.cpp duckdb_settings.cpp duckdb_tables.cpp diff --git a/src/function/table/system/duckdb_which_secret.cpp b/src/function/table/system/duckdb_which_secret.cpp new file mode 100644 index 000000000000..3314fee95e69 --- /dev/null +++ b/src/function/table/system/duckdb_which_secret.cpp @@ -0,0 +1,75 @@ +#include "duckdb/function/table/system_functions.hpp" + +#include "duckdb/common/file_system.hpp" +#include "duckdb/common/map.hpp" +#include "duckdb/common/string_util.hpp" +#include "duckdb/common/multi_file_reader.hpp" +#include "duckdb/function/function_set.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/main/database.hpp" +#include "duckdb/main/extension_helper.hpp" +#include "duckdb/main/secret/secret_manager.hpp" + +namespace duckdb { + +struct DuckDBWhichSecretData : public GlobalTableFunctionState { + DuckDBWhichSecretData() : finished(false) { + } + bool finished; +}; + +struct DuckDBWhichSecretBindData : public TableFunctionData { + explicit DuckDBWhichSecretBindData(TableFunctionBindInput &tf_input) : inputs(tf_input.inputs) {}; + + duckdb::vector inputs; +}; + +static unique_ptr DuckDBWhichSecretBind(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names) { + names.emplace_back("name"); + return_types.emplace_back(LogicalType::VARCHAR); + + names.emplace_back("persistent"); + return_types.emplace_back(LogicalType::VARCHAR); + + names.emplace_back("storage"); + return_types.emplace_back(LogicalType::VARCHAR); + + return make_uniq(input); +} + +unique_ptr DuckDBWhichSecretInit(ClientContext &context, TableFunctionInitInput &input) { + return make_uniq(); +} + +void DuckDBWhichSecretFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { + auto &data = data_p.global_state->Cast(); + if (data.finished) { + // finished returning values + return; + } + auto &bind_data = data_p.bind_data->Cast(); + + auto &secret_manager = SecretManager::Get(context); + auto transaction = CatalogTransaction::GetSystemCatalogTransaction(context); + + auto &inputs = bind_data.inputs; + auto path = inputs[0].ToString(); + auto type = inputs[1].ToString(); + auto secret_match = secret_manager.LookupSecret(transaction, path, type); + if (secret_match.HasMatch()) { + auto &secret_entry = *secret_match.secret_entry; + output.SetCardinality(1); + output.SetValue(0, 0, secret_entry.secret->GetName()); + output.SetValue(1, 0, EnumUtil::ToString(secret_entry.persist_type)); + output.SetValue(2, 0, secret_entry.storage_mode); + } + data.finished = true; +} + +void DuckDBWhichSecretFun::RegisterFunction(BuiltinFunctions &set) { + set.AddFunction(TableFunction("which_secret", {duckdb::LogicalType::VARCHAR, duckdb::LogicalType::VARCHAR}, + DuckDBWhichSecretFunction, DuckDBWhichSecretBind, DuckDBWhichSecretInit)); +} + +} // namespace duckdb diff --git a/src/function/table/system_functions.cpp b/src/function/table/system_functions.cpp index a9c191543083..a1a821db79f4 100644 --- a/src/function/table/system_functions.cpp +++ b/src/function/table/system_functions.cpp @@ -30,6 +30,7 @@ void BuiltinFunctions::RegisterSQLiteFunctions() { DuckDBMemoryFun::RegisterFunction(*this); DuckDBOptimizersFun::RegisterFunction(*this); DuckDBSecretsFun::RegisterFunction(*this); + DuckDBWhichSecretFun::RegisterFunction(*this); DuckDBSequencesFun::RegisterFunction(*this); DuckDBSettingsFun::RegisterFunction(*this); DuckDBTablesFun::RegisterFunction(*this); diff --git a/src/include/duckdb/function/table/system_functions.hpp b/src/include/duckdb/function/table/system_functions.hpp index 1ab8f87690df..23f4d9cb9caf 100644 --- a/src/include/duckdb/function/table/system_functions.hpp +++ b/src/include/duckdb/function/table/system_functions.hpp @@ -57,6 +57,10 @@ struct DuckDBSecretsFun { static void RegisterFunction(BuiltinFunctions &set); }; +struct DuckDBWhichSecretFun { + static void RegisterFunction(BuiltinFunctions &set); +}; + struct DuckDBDatabasesFun { static void RegisterFunction(BuiltinFunctions &set); }; diff --git a/test/secrets/test_custom_secret_storage.cpp b/test/secrets/test_custom_secret_storage.cpp index e0fa40c8140c..ba344a954470 100644 --- a/test/secrets/test_custom_secret_storage.cpp +++ b/test/secrets/test_custom_secret_storage.cpp @@ -91,8 +91,8 @@ TEST_CASE("Test secret lookups by secret type", "[secret][.]") { REQUIRE_NO_FAIL(con.Query("CREATE SECRET s2 (TYPE secret_type_2, SCOPE '')")); // Note that the secrets collide completely, except for their types - auto res1 = con.Query("SELECT which_secret('blablabla', 'secret_type_1')"); - auto res2 = con.Query("SELECT which_secret('blablabla', 'secret_type_2')"); + auto res1 = con.Query("SELECT name FROM which_secret('blablabla', 'secret_type_1')"); + auto res2 = con.Query("SELECT name FROM which_secret('blablabla', 'secret_type_2')"); // Correct secret is selected REQUIRE(res1->GetValue(0, 0).ToString() == "s1"); @@ -154,14 +154,14 @@ TEST_CASE("Test adding a custom secret storage", "[secret][.]") { REQUIRE(secret_ptr->secret->GetName() == "s2"); // Now try resolve secret by path -> this will return s1 because its scope matches best - auto which_secret_result = con.Query("SELECT which_secret('s3://foo/bar.csv', 'S3');"); + auto which_secret_result = con.Query("SELECT name FROM which_secret('s3://foo/bar.csv', 'S3');"); REQUIRE(which_secret_result->GetValue(0, 0).ToString() == "s1"); // Exclude the storage from lookups storage_ref.include_in_lookups = false; // Now the lookup will choose the other storage - which_secret_result = con.Query("SELECT which_secret('s3://foo/bar.csv', 's3');"); + which_secret_result = con.Query("SELECT name FROM which_secret('s3://foo/bar.csv', 's3');"); REQUIRE(which_secret_result->GetValue(0, 0).ToString() == "s2"); // Lets drop stuff now @@ -222,17 +222,17 @@ TEST_CASE("Test tie-break behaviour for custom secret storage", "[secret][.]") { REQUIRE(result->GetValue(0, 2).ToString() == "s3"); REQUIRE(result->GetValue(1, 2).ToString() == "test_storage_before"); - result = con.Query("SELECT which_secret('s3://', 's3');"); + result = con.Query("SELECT name FROM which_secret('s3://', 's3');"); REQUIRE(result->GetValue(0, 0).ToString() == "s3"); REQUIRE_NO_FAIL(con.Query("DROP SECRET s3")); - result = con.Query("SELECT which_secret('s3://', 's3');"); + result = con.Query("SELECT name FROM which_secret('s3://', 's3');"); REQUIRE(result->GetValue(0, 0).ToString() == "s1"); REQUIRE_NO_FAIL(con.Query("DROP SECRET s1")); - result = con.Query("SELECT which_secret('s3://', 's3');"); + result = con.Query("SELECT name FROM which_secret('s3://', 's3');"); REQUIRE(result->GetValue(0, 0).ToString() == "s2"); REQUIRE_NO_FAIL(con.Query("DROP SECRET s2")); diff --git a/test/sql/secrets/create_secret_scope_matching.test b/test/sql/secrets/create_secret_scope_matching.test index a026ad7cfd50..8f27e1d8adf4 100644 --- a/test/sql/secrets/create_secret_scope_matching.test +++ b/test/sql/secrets/create_secret_scope_matching.test @@ -7,11 +7,21 @@ load __TEST_DIR__/create_secret_scope_matching.db statement ok PRAGMA enable_verification; -require httpfs +statement ok +SET autoinstall_known_extensions=1; + +statement ok +SET autoload_known_extensions=1; statement ok set secret_directory='__TEST_DIR__/create_secret_scope_matching' +# No match +query I +SELECT name FROM which_secret('s3://', 's3') +---- + + statement ok CREATE TEMPORARY SECRET t1 ( TYPE S3 ) @@ -24,16 +34,21 @@ CREATE SECRET p1 IN LOCAL_FILE ( TYPE S3 ) # This ties within the same storage: the two temporary secrets s1 and s2 both score identically. We solve this by # tie-breaking on secret name alphabetical ordering query I -SELECT which_secret('s3://', 's3') +SELECT name FROM which_secret('s3://', 's3') ---- t1 +query III +FROM which_secret('s3://', 's3') +---- +t1 TEMPORARY memory + statement ok DROP SECRET t1 # Temporary secrets take preference over temporary ones query I -SELECT which_secret('s3://', 's3') +SELECT name FROM which_secret('s3://', 's3') ---- t2 @@ -41,7 +56,7 @@ statement ok DROP SECRET t2 query I -SELECT which_secret('s3://', 's3') +SELECT name FROM which_secret('s3://', 's3') ---- p1 From 38216a2c7972e309f3ad0fcfd48f80f7e4345eec Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 18 Apr 2024 22:57:32 +0200 Subject: [PATCH 572/603] add extra test with pyarrow MapArrays with None --- tools/pythonpkg/tests/fast/arrow/test_nested_arrow.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/pythonpkg/tests/fast/arrow/test_nested_arrow.py b/tools/pythonpkg/tests/fast/arrow/test_nested_arrow.py index 592778146835..9c6ceb06b4fe 100644 --- a/tools/pythonpkg/tests/fast/arrow/test_nested_arrow.py +++ b/tools/pythonpkg/tests/fast/arrow/test_nested_arrow.py @@ -183,6 +183,15 @@ def test_map_arrow_to_duckdb(self, duckdb_cursor): ): rel = duckdb.from_arrow(arrow_table).fetchall() + def test_null_map_arrow_to_duckdb(self, duckdb_cursor): + if not can_run: + return + map_type = pa.map_(pa.int32(), pa.int32()) + values = [None, [(5, 42)]] + arrow_table = pa.table({'detail': pa.array(values, map_type)}) + res = duckdb_cursor.sql("select * from arrow_table").fetchall() + assert res == [(None,), ({'key': [5], 'value': [42]},)] + def test_map_arrow_to_pandas(self, duckdb_cursor): if not can_run: return From 2f9bc1e3ab5e51788e6bf60d86af109c500952e8 Mon Sep 17 00:00:00 2001 From: stephaniewang Date: Thu, 18 Apr 2024 18:06:13 -0400 Subject: [PATCH 573/603] update generate_functions.py --- scripts/generate_functions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/generate_functions.py b/scripts/generate_functions.py index 572d7b703221..f708eb1f5b72 100644 --- a/scripts/generate_functions.py +++ b/scripts/generate_functions.py @@ -15,7 +15,6 @@ 'math', 'operators', 'random', - 'secret', 'string', 'debug', 'struct', From 9718fb052f82b83e9beb3b18e0a96adb7ca3f15a Mon Sep 17 00:00:00 2001 From: stephaniewang Date: Thu, 18 Apr 2024 19:49:50 -0400 Subject: [PATCH 574/603] fix test --- test/sql/secrets/create_secret_scope_matching.test | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/test/sql/secrets/create_secret_scope_matching.test b/test/sql/secrets/create_secret_scope_matching.test index 8f27e1d8adf4..3d5dd2aac35a 100644 --- a/test/sql/secrets/create_secret_scope_matching.test +++ b/test/sql/secrets/create_secret_scope_matching.test @@ -7,11 +7,7 @@ load __TEST_DIR__/create_secret_scope_matching.db statement ok PRAGMA enable_verification; -statement ok -SET autoinstall_known_extensions=1; - -statement ok -SET autoload_known_extensions=1; +require httpfs statement ok set secret_directory='__TEST_DIR__/create_secret_scope_matching' @@ -21,7 +17,6 @@ query I SELECT name FROM which_secret('s3://', 's3') ---- - statement ok CREATE TEMPORARY SECRET t1 ( TYPE S3 ) From 4f7b3dee5512038dbadd8e4ba2cdeee62b7d0636 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Fri, 19 Apr 2024 09:30:35 +0200 Subject: [PATCH 575/603] Implement GetUniqueConstraintKeys --- src/catalog/catalog_entry/duck_table_entry.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/catalog/catalog_entry/duck_table_entry.cpp b/src/catalog/catalog_entry/duck_table_entry.cpp index f77e69f65eba..e453df2e66b7 100644 --- a/src/catalog/catalog_entry/duck_table_entry.cpp +++ b/src/catalog/catalog_entry/duck_table_entry.cpp @@ -72,7 +72,15 @@ IndexStorageInfo GetIndexInfo(const IndexConstraintType &constraint_type, unique } vector GetUniqueConstraintKeys(const ColumnList &columns, const UniqueConstraint &constraint) { - throw InternalException("FIXME: GetUniqueConstraintKeys"); + vector indexes; + if (constraint.HasIndex()) { + indexes.push_back(columns.LogicalToPhysical(constraint.GetIndex())); + } else { + for(auto &keyname : constraint.GetColumnNames()) { + indexes.push_back(columns.GetColumn(keyname).Physical()); + } + } + return indexes; } DuckTableEntry::DuckTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, BoundCreateTableInfo &info, From 8668a2f8e7b0015bfbfbdefeae8cbace92c2147e Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Fri, 19 Apr 2024 09:30:55 +0200 Subject: [PATCH 576/603] Format fix --- .../catalog_entry/duck_table_entry.cpp | 7 ++++--- .../catalog_entry/table_catalog_entry.cpp | 4 ++-- .../operator/persistent/physical_insert.cpp | 3 ++- .../table/system/duckdb_constraints.cpp | 2 +- .../catalog_entry/duck_table_entry.hpp | 3 ++- src/include/duckdb/planner/binder.hpp | 10 +++++++--- src/include/duckdb/storage/data_table.hpp | 8 ++++---- .../duckdb/storage/table/append_state.hpp | 3 ++- .../binder/statement/bind_create_table.cpp | 20 ++++++++++++------- src/storage/data_table.cpp | 10 ++++++---- 10 files changed, 43 insertions(+), 27 deletions(-) diff --git a/src/catalog/catalog_entry/duck_table_entry.cpp b/src/catalog/catalog_entry/duck_table_entry.cpp index e453df2e66b7..c25c76bb279b 100644 --- a/src/catalog/catalog_entry/duck_table_entry.cpp +++ b/src/catalog/catalog_entry/duck_table_entry.cpp @@ -76,7 +76,7 @@ vector GetUniqueConstraintKeys(const ColumnList &columns, const U if (constraint.HasIndex()) { indexes.push_back(columns.LogicalToPhysical(constraint.GetIndex())); } else { - for(auto &keyname : constraint.GetColumnNames()) { + for (auto &keyname : constraint.GetColumnNames()) { indexes.push_back(columns.GetColumn(keyname).Physical()); } } @@ -360,7 +360,7 @@ void DuckTableEntry::UpdateConstraintsOnColumnDrop(const LogicalIndex &removed_i const vector &adjusted_indices, const RemoveColumnInfo &info, CreateTableInfo &create_info, const vector> &bound_constraints, - bool is_generated) { + bool is_generated) { // handle constraints for the new table D_ASSERT(constraints.size() == bound_constraints.size()); for (idx_t constr_idx = 0; constr_idx < constraints.size(); constr_idx++) { @@ -483,7 +483,8 @@ unique_ptr DuckTableEntry::RemoveColumn(ClientContext &context, Re auto binder = Binder::CreateBinder(context); auto bound_constraints = binder->BindConstraints(constraints, name, columns); - UpdateConstraintsOnColumnDrop(removed_index, adjusted_indices, info, *create_info, bound_constraints, dropped_column_is_generated); + UpdateConstraintsOnColumnDrop(removed_index, adjusted_indices, info, *create_info, bound_constraints, + dropped_column_is_generated); auto bound_create_info = binder->BindCreateTableInfo(std::move(create_info), schema); if (columns.GetColumn(LogicalIndex(removed_index)).Generated()) { diff --git a/src/catalog/catalog_entry/table_catalog_entry.cpp b/src/catalog/catalog_entry/table_catalog_entry.cpp index 493403830e2a..8f73ec416e62 100644 --- a/src/catalog/catalog_entry/table_catalog_entry.cpp +++ b/src/catalog/catalog_entry/table_catalog_entry.cpp @@ -234,8 +234,8 @@ vector TableCatalogEntry::GetColumnSegmentInfo() { return {}; } -void TableCatalogEntry::BindUpdateConstraints(Binder &binder, LogicalGet &get, LogicalProjection &proj, LogicalUpdate &update, - ClientContext &context) { +void TableCatalogEntry::BindUpdateConstraints(Binder &binder, LogicalGet &get, LogicalProjection &proj, + LogicalUpdate &update, ClientContext &context) { // check the constraints and indexes of the table to see if we need to project any additional columns // we do this for indexes with multiple columns and CHECK constraints in the UPDATE clause // suppose we have a constraint CHECK(i + j < 10); now we need both i and j to check the constraint diff --git a/src/execution/operator/persistent/physical_insert.cpp b/src/execution/operator/persistent/physical_insert.cpp index 181216a15b7f..c7b499855c5a 100644 --- a/src/execution/operator/persistent/physical_insert.cpp +++ b/src/execution/operator/persistent/physical_insert.cpp @@ -108,7 +108,8 @@ class InsertLocalState : public LocalSinkState { idx_t update_count = 0; unique_ptr constraint_state; - ConstraintVerificationState &GetConstraintState(DataTable &table, TableCatalogEntry &tableref, ClientContext &context) { + ConstraintVerificationState &GetConstraintState(DataTable &table, TableCatalogEntry &tableref, + ClientContext &context) { if (!constraint_state) { constraint_state = table.InitializeConstraintVerification(tableref, context); } diff --git a/src/function/table/system/duckdb_constraints.cpp b/src/function/table/system/duckdb_constraints.cpp index c35eaf0da9ad..71fabb16b5ce 100644 --- a/src/function/table/system/duckdb_constraints.cpp +++ b/src/function/table/system/duckdb_constraints.cpp @@ -132,7 +132,7 @@ unique_ptr DuckDBConstraintsInit(ClientContext &contex }); sort(entries.begin(), entries.end(), [&](CatalogEntry &x, CatalogEntry &y) { return (x.name < y.name); }); - for(auto &entry : entries) { + for (auto &entry : entries) { result->entries.emplace_back(context, entry.get().Cast()); } }; diff --git a/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp b/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp index 7c1323a4f300..5396f31115cc 100644 --- a/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp +++ b/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp @@ -58,7 +58,8 @@ class DuckTableEntry : public TableCatalogEntry { unique_ptr SetColumnComment(ClientContext &context, SetColumnCommentInfo &info); void UpdateConstraintsOnColumnDrop(const LogicalIndex &removed_index, const vector &adjusted_indices, - const RemoveColumnInfo &info, CreateTableInfo &create_info, const vector> &bound_constraints, bool is_generated); + const RemoveColumnInfo &info, CreateTableInfo &create_info, + const vector> &bound_constraints, bool is_generated); private: //! A reference to the underlying storage unit used for this table diff --git a/src/include/duckdb/planner/binder.hpp b/src/include/duckdb/planner/binder.hpp index 67e2bc1ec6b1..705f35595017 100644 --- a/src/include/duckdb/planner/binder.hpp +++ b/src/include/duckdb/planner/binder.hpp @@ -122,9 +122,13 @@ class Binder : public std::enable_shared_from_this { unique_ptr BindCreateTableInfo(unique_ptr info, SchemaCatalogEntry &schema); unique_ptr BindCreateTableInfo(unique_ptr info, SchemaCatalogEntry &schema, vector> &bound_defaults); - static vector> BindConstraints(ClientContext &context, const vector> &constraints, const string &table_name, const ColumnList &columns); - vector> BindConstraints(const vector> &constraints, const string &table_name, const ColumnList &columns); - vector> BindNewConstraints(vector> &constraints, const string &table_name, const ColumnList &columns); + static vector> BindConstraints(ClientContext &context, + const vector> &constraints, + const string &table_name, const ColumnList &columns); + vector> BindConstraints(const vector> &constraints, + const string &table_name, const ColumnList &columns); + vector> BindNewConstraints(vector> &constraints, + const string &table_name, const ColumnList &columns); void BindCreateViewInfo(CreateViewInfo &base); SchemaCatalogEntry &BindSchema(CreateInfo &info); diff --git a/src/include/duckdb/storage/data_table.hpp b/src/include/duckdb/storage/data_table.hpp index 74a5c2283830..e6aedf464bc9 100644 --- a/src/include/duckdb/storage/data_table.hpp +++ b/src/include/duckdb/storage/data_table.hpp @@ -115,7 +115,6 @@ class DataTable { //! Delete the entries with the specified row identifier from the table idx_t Delete(TableDeleteState &state, ClientContext &context, Vector &row_ids, idx_t count); - unique_ptr InitializeUpdate(TableCatalogEntry &table, ClientContext &context); //! Update the entries with the specified row identifier from the table void Update(TableUpdateState &state, ClientContext &context, Vector &row_ids, @@ -194,14 +193,15 @@ class DataTable { bool IndexNameIsUnique(const string &name); //! Initialize constraint verification - unique_ptr InitializeConstraintVerification(TableCatalogEntry &table, ClientContext &context); + unique_ptr InitializeConstraintVerification(TableCatalogEntry &table, + ClientContext &context); //! Verify constraints with a chunk from the Append containing all columns of the table void VerifyAppendConstraints(ConstraintVerificationState &state, ClientContext &context, DataChunk &chunk, - optional_ptr conflict_manager = nullptr); + optional_ptr conflict_manager = nullptr); public: static void VerifyUniqueIndexes(TableIndexList &indexes, ClientContext &context, DataChunk &chunk, - optional_ptr conflict_manager); + optional_ptr conflict_manager); private: //! Verify the new added constraints against current persistent&local data diff --git a/src/include/duckdb/storage/table/append_state.hpp b/src/include/duckdb/storage/table/append_state.hpp index 475a59187f1f..8f9a56c5cba2 100644 --- a/src/include/duckdb/storage/table/append_state.hpp +++ b/src/include/duckdb/storage/table/append_state.hpp @@ -71,7 +71,8 @@ struct TableAppendState { }; struct ConstraintVerificationState { - explicit ConstraintVerificationState(TableCatalogEntry &table_p) : table(table_p) {} + explicit ConstraintVerificationState(TableCatalogEntry &table_p) : table(table_p) { + } TableCatalogEntry &table; vector> bound_constraints; diff --git a/src/planner/binder/statement/bind_create_table.cpp b/src/planner/binder/statement/bind_create_table.cpp index 10ef50599a08..8a032b9fe1ee 100644 --- a/src/planner/binder/statement/bind_create_table.cpp +++ b/src/planner/binder/statement/bind_create_table.cpp @@ -35,7 +35,8 @@ static void CreateColumnDependencyManager(BoundCreateTableInfo &info) { } } -static unique_ptr BindCheckConstraint(Binder &binder, const string &table_name, const ColumnList &columns, const unique_ptr &cond) { +static unique_ptr BindCheckConstraint(Binder &binder, const string &table_name, + const ColumnList &columns, const unique_ptr &cond) { auto bound_constraint = make_uniq(); // check constraint: bind the expression CheckBinder check_binder(binder, binder.context, table_name, columns, bound_constraint->bound_columns); @@ -47,12 +48,15 @@ static unique_ptr BindCheckConstraint(Binder &binder, const str return std::move(bound_constraint); } -vector> Binder::BindConstraints(ClientContext &context, const vector> &constraints, const string &table_name, const ColumnList &columns) { +vector> Binder::BindConstraints(ClientContext &context, + const vector> &constraints, + const string &table_name, const ColumnList &columns) { auto binder = Binder::CreateBinder(context); return binder->BindConstraints(constraints, table_name, columns); } -vector> Binder::BindConstraints(const vector> &constraints, const string &table_name, const ColumnList &columns) { +vector> Binder::BindConstraints(const vector> &constraints, + const string &table_name, const ColumnList &columns) { vector> bound_constraints; for (auto &constr : constraints) { switch (constr->type) { @@ -128,16 +132,17 @@ vector> Binder::BindConstraints(const vector> Binder::BindNewConstraints(vector> &constraints, const string &table_name, const ColumnList &columns) { +vector> Binder::BindNewConstraints(vector> &constraints, + const string &table_name, const ColumnList &columns) { auto bound_constraints = BindConstraints(constraints, table_name, columns); // handle primary keys/not null constraints bool has_primary_key = false; logical_index_set_t not_null_columns; vector primary_keys; - for(idx_t c = 0; c < constraints.size(); c++) { + for (idx_t c = 0; c < constraints.size(); c++) { auto &constr = constraints[c]; - switch(constr->type) { + switch (constr->type) { case ConstraintType::NOT_NULL: { auto ¬_null = constr->Cast(); auto &col = columns.GetColumn(LogicalIndex(not_null.index)); @@ -263,7 +268,8 @@ static void ExtractExpressionDependencies(Expression &expr, LogicalDependencyLis expr, [&](Expression &child) { ExtractExpressionDependencies(child, dependencies); }); } -static void ExtractDependencies(BoundCreateTableInfo &info, vector> &defaults, vector> &constraints) { +static void ExtractDependencies(BoundCreateTableInfo &info, vector> &defaults, + vector> &constraints) { for (auto &default_value : defaults) { if (default_value) { ExtractExpressionDependencies(*default_value, info.dependencies); diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 8dab59b0e1a8..8a610b38c7c8 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -617,7 +617,7 @@ void DataTable::VerifyUniqueIndexes(TableIndexList &indexes, ClientContext &cont } void DataTable::VerifyAppendConstraints(ConstraintVerificationState &state, ClientContext &context, DataChunk &chunk, - optional_ptr conflict_manager) { + optional_ptr conflict_manager) { auto &table = state.table; if (table.HasGeneratedColumns()) { // Verify that the generated columns expression work with the inserted values @@ -675,7 +675,8 @@ void DataTable::VerifyAppendConstraints(ConstraintVerificationState &state, Clie } } -unique_ptr DataTable::InitializeConstraintVerification(TableCatalogEntry &table, ClientContext &context) { +unique_ptr DataTable::InitializeConstraintVerification(TableCatalogEntry &table, + ClientContext &context) { auto result = make_uniq(table); auto binder = Binder::CreateBinder(context); result->bound_constraints = binder->BindConstraints(table.GetConstraints(), table.name, table.GetColumns()); @@ -1063,7 +1064,8 @@ idx_t DataTable::Delete(TableDeleteState &state, ClientContext &context, Vector if (state.has_delete_constraints) { // perform the constraint verification ColumnFetchState fetch_state; - local_storage.FetchChunk(*this, offset_ids, current_count, state.col_ids, state.verify_chunk, fetch_state); + local_storage.FetchChunk(*this, offset_ids, current_count, state.col_ids, state.verify_chunk, + fetch_state); VerifyDeleteConstraints(state, context, state.verify_chunk); } delete_count += local_storage.Delete(*this, offset_ids, current_count); @@ -1118,7 +1120,7 @@ static bool CreateMockChunk(TableCatalogEntry &table, const vector &column_ids) { auto &table = state.table; auto &constraints = table.GetConstraints(); From 3e8d2ec4901b8aec742c1f37d2be99697b1bd241 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 19 Apr 2024 10:27:20 +0200 Subject: [PATCH 577/603] slightly more extended tests --- test/sql/types/map/map_null.test | 34 ++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/test/sql/types/map/map_null.test b/test/sql/types/map/map_null.test index 6bc138488856..68dc58be808a 100644 --- a/test/sql/types/map/map_null.test +++ b/test/sql/types/map/map_null.test @@ -33,3 +33,37 @@ query I select map([1,2,3], NULL::INT[]) ---- NULL + +query I +SELECT * FROM ( VALUES + (MAP(NULL, NULL)), + (MAP(NULL::INT[], NULL::INT[])), + (MAP([1,2,3], [1,2,3])) +) +---- +NULL +NULL +{1=1, 2=2, 3=3} + +query I +select MAP(a, b) FROM ( VALUES + (NULL, ['b', 'c']), + (NULL::INT[], NULL), + (NULL::INT[], NULL::VARCHAR[]), + (NULL::INT[], ['a', 'b', 'c']), + (NULL, ['longer string than inlined', 'smol']), + (NULL, NULL), + ([1,2,3], NULL), + ([1,2,3], ['z', 'y', 'x']), + ([1,2,3], NULL::VARCHAR[]), +) t(a, b) +---- +NULL +NULL +NULL +NULL +NULL +NULL +NULL +{1=z, 2=y, 3=x} +NULL From 8838f07cb47885cdcd9e3a467df048154d4eb9d8 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 19 Apr 2024 10:41:00 +0200 Subject: [PATCH 578/603] removed unused enum constants --- src/common/enum_util.cpp | 10 ---------- src/common/types/vector.cpp | 4 ---- src/function/table/arrow_conversion.cpp | 3 --- src/include/duckdb/common/types/vector.hpp | 10 +--------- 4 files changed, 1 insertion(+), 26 deletions(-) diff --git a/src/common/enum_util.cpp b/src/common/enum_util.cpp index b2db02f412c9..12a94a2b0f1c 100644 --- a/src/common/enum_util.cpp +++ b/src/common/enum_util.cpp @@ -3802,14 +3802,10 @@ const char* EnumUtil::ToChars(MapInvalidReason value) { switch(value) { case MapInvalidReason::VALID: return "VALID"; - case MapInvalidReason::NULL_KEY_LIST: - return "NULL_KEY_LIST"; case MapInvalidReason::NULL_KEY: return "NULL_KEY"; case MapInvalidReason::DUPLICATE_KEY: return "DUPLICATE_KEY"; - case MapInvalidReason::NULL_VALUE_LIST: - return "NULL_VALUE_LIST"; case MapInvalidReason::NOT_ALIGNED: return "NOT_ALIGNED"; case MapInvalidReason::INVALID_PARAMS: @@ -3824,18 +3820,12 @@ MapInvalidReason EnumUtil::FromString(const char *value) { if (StringUtil::Equals(value, "VALID")) { return MapInvalidReason::VALID; } - if (StringUtil::Equals(value, "NULL_KEY_LIST")) { - return MapInvalidReason::NULL_KEY_LIST; - } if (StringUtil::Equals(value, "NULL_KEY")) { return MapInvalidReason::NULL_KEY; } if (StringUtil::Equals(value, "DUPLICATE_KEY")) { return MapInvalidReason::DUPLICATE_KEY; } - if (StringUtil::Equals(value, "NULL_VALUE_LIST")) { - return MapInvalidReason::NULL_VALUE_LIST; - } if (StringUtil::Equals(value, "NOT_ALIGNED")) { return MapInvalidReason::NOT_ALIGNED; } diff --git a/src/common/types/vector.cpp b/src/common/types/vector.cpp index e61ad46dd259..112dc0de96ab 100644 --- a/src/common/types/vector.cpp +++ b/src/common/types/vector.cpp @@ -2089,10 +2089,6 @@ void MapVector::EvalMapInvalidReason(MapInvalidReason reason) { throw InvalidInputException("Map keys must be unique."); case MapInvalidReason::NULL_KEY: throw InvalidInputException("Map keys can not be NULL."); - case MapInvalidReason::NULL_KEY_LIST: - throw InvalidInputException("The list of map keys must not be NULL."); - case MapInvalidReason::NULL_VALUE_LIST: - throw InvalidInputException("The list of map values must not be NULL."); case MapInvalidReason::NOT_ALIGNED: throw InvalidInputException("The map key list does not align with the map value list."); case MapInvalidReason::INVALID_PARAMS: diff --git a/src/function/table/arrow_conversion.cpp b/src/function/table/arrow_conversion.cpp index 78d4cca859f6..c1759ef8484c 100644 --- a/src/function/table/arrow_conversion.cpp +++ b/src/function/table/arrow_conversion.cpp @@ -317,9 +317,6 @@ static void ArrowToDuckDBMapVerify(Vector &vector, idx_t count) { case MapInvalidReason::NULL_KEY: { throw InvalidInputException("Arrow map contains NULL as map key, which isn't supported by DuckDB map type"); } - case MapInvalidReason::NULL_KEY_LIST: { - throw InvalidInputException("Arrow map contains NULL as key list, which isn't supported by DuckDB map type"); - } default: { throw InternalException("MapInvalidReason not implemented"); } diff --git a/src/include/duckdb/common/types/vector.hpp b/src/include/duckdb/common/types/vector.hpp index b0786597a662..49cb9111c464 100644 --- a/src/include/duckdb/common/types/vector.hpp +++ b/src/include/duckdb/common/types/vector.hpp @@ -464,15 +464,7 @@ struct FSSTVector { DUCKDB_API static idx_t GetCount(Vector &vector); }; -enum class MapInvalidReason : uint8_t { - VALID, - NULL_KEY_LIST, - NULL_KEY, - DUPLICATE_KEY, - NULL_VALUE_LIST, - NOT_ALIGNED, - INVALID_PARAMS -}; +enum class MapInvalidReason : uint8_t { VALID, NULL_KEY, DUPLICATE_KEY, NOT_ALIGNED, INVALID_PARAMS }; struct MapVector { DUCKDB_API static const Vector &GetKeys(const Vector &vector); From 738b30192333b3d36a510f3fe6d1d9f247e7a220 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 19 Apr 2024 11:15:22 +0200 Subject: [PATCH 579/603] remove NULL_KEY_LIST error from numpy scan --- tools/pythonpkg/src/numpy/numpy_scan.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/pythonpkg/src/numpy/numpy_scan.cpp b/tools/pythonpkg/src/numpy/numpy_scan.cpp index 032d3b97f014..b4b1d3dbe276 100644 --- a/tools/pythonpkg/src/numpy/numpy_scan.cpp +++ b/tools/pythonpkg/src/numpy/numpy_scan.cpp @@ -153,8 +153,6 @@ static void VerifyMapConstraints(Vector &vec, idx_t count) { return; case MapInvalidReason::DUPLICATE_KEY: throw InvalidInputException("Dict->Map conversion failed because 'key' list contains duplicates"); - case MapInvalidReason::NULL_KEY_LIST: - throw InvalidInputException("Dict->Map conversion failed because 'key' list is None"); case MapInvalidReason::NULL_KEY: throw InvalidInputException("Dict->Map conversion failed because 'key' list contains None"); default: From 87f3fe3fb9ffe97553367963f9a731b827535fca Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Fri, 19 Apr 2024 11:20:17 +0200 Subject: [PATCH 580/603] Move binding of constraints out of execution and into the binding phase of insert/update/delete --- .../catalog_entry/table_catalog_entry.cpp | 2 +- .../persistent/physical_batch_insert.cpp | 20 +++---- .../operator/persistent/physical_delete.cpp | 14 +++-- .../operator/persistent/physical_insert.cpp | 53 +++++++++---------- .../operator/persistent/physical_update.cpp | 21 +++++--- src/execution/physical_plan/plan_delete.cpp | 4 +- src/execution/physical_plan/plan_insert.cpp | 11 ++-- src/execution/physical_plan/plan_update.cpp | 6 +-- .../catalog_entry/table_catalog_entry.hpp | 2 +- .../persistent/physical_batch_insert.hpp | 4 +- .../operator/persistent/physical_delete.hpp | 9 ++-- .../operator/persistent/physical_insert.hpp | 13 +++-- .../operator/persistent/physical_update.hpp | 4 +- src/include/duckdb/planner/binder.hpp | 1 + .../planner/operator/logical_delete.hpp | 1 + .../planner/operator/logical_insert.hpp | 2 + .../planner/operator/logical_update.hpp | 1 + src/include/duckdb/storage/data_table.hpp | 27 ++++++---- .../duckdb/storage/table/append_state.hpp | 9 ++-- .../duckdb/storage/table/delete_state.hpp | 2 +- .../duckdb/storage/table/update_state.hpp | 2 +- src/main/appender.cpp | 4 +- src/main/client_context.cpp | 4 +- .../binder/statement/bind_create_table.cpp | 4 ++ src/planner/binder/statement/bind_delete.cpp | 1 + src/planner/binder/statement/bind_insert.cpp | 1 + src/planner/binder/statement/bind_update.cpp | 1 + src/planner/operator/logical_delete.cpp | 2 + src/planner/operator/logical_insert.cpp | 2 + src/planner/operator/logical_update.cpp | 2 + src/storage/data_table.cpp | 42 ++++++++------- src/storage/wal_replay.cpp | 4 +- 32 files changed, 164 insertions(+), 111 deletions(-) diff --git a/src/catalog/catalog_entry/table_catalog_entry.cpp b/src/catalog/catalog_entry/table_catalog_entry.cpp index 8f73ec416e62..271970a957de 100644 --- a/src/catalog/catalog_entry/table_catalog_entry.cpp +++ b/src/catalog/catalog_entry/table_catalog_entry.cpp @@ -164,7 +164,7 @@ const ColumnDefinition &TableCatalogEntry::GetColumn(LogicalIndex idx) { return columns.GetColumn(idx); } -const vector> &TableCatalogEntry::GetConstraints() { +const vector> &TableCatalogEntry::GetConstraints() const { return constraints; } diff --git a/src/execution/operator/persistent/physical_batch_insert.cpp b/src/execution/operator/persistent/physical_batch_insert.cpp index 210f0b3d201d..12eda91c7e82 100644 --- a/src/execution/operator/persistent/physical_batch_insert.cpp +++ b/src/execution/operator/persistent/physical_batch_insert.cpp @@ -13,12 +13,14 @@ namespace duckdb { -PhysicalBatchInsert::PhysicalBatchInsert(vector types, TableCatalogEntry &table, - physical_index_vector_t column_index_map, - vector> bound_defaults, idx_t estimated_cardinality) - : PhysicalOperator(PhysicalOperatorType::BATCH_INSERT, std::move(types), estimated_cardinality), - column_index_map(std::move(column_index_map)), insert_table(&table), insert_types(table.GetTypes()), - bound_defaults(std::move(bound_defaults)) { +PhysicalBatchInsert::PhysicalBatchInsert(vector types_p, TableCatalogEntry &table, + physical_index_vector_t column_index_map_p, + vector> bound_defaults_p, + vector> bound_constraints_p, + idx_t estimated_cardinality) + : PhysicalOperator(PhysicalOperatorType::BATCH_INSERT, std::move(types_p), estimated_cardinality), + column_index_map(std::move(column_index_map_p)), insert_table(&table), insert_types(table.GetTypes()), + bound_defaults(std::move(bound_defaults_p)), bound_constraints(std::move(bound_constraints_p)) { } PhysicalBatchInsert::PhysicalBatchInsert(LogicalOperator &op, SchemaCatalogEntry &schema, @@ -171,7 +173,7 @@ class BatchInsertLocalState : public LocalSinkState { TableAppendState current_append_state; unique_ptr current_collection; optional_ptr writer; - unique_ptr constraint_state; + unique_ptr constraint_state; void CreateNewCollection(DuckTableEntry &table, const vector &insert_types) { auto &table_info = table.GetStorage().info; @@ -496,7 +498,7 @@ SinkResultType PhysicalBatchInsert::Sink(ExecutionContext &context, DataChunk &c } if (!lstate.constraint_state) { - lstate.constraint_state = table.GetStorage().InitializeConstraintVerification(table, context.client); + lstate.constraint_state = table.GetStorage().InitializeConstraintState(table, bound_constraints); } table.GetStorage().VerifyAppendConstraints(*lstate.constraint_state, context.client, lstate.insert_chunk); @@ -599,7 +601,7 @@ SinkFinalizeType PhysicalBatchInsert::Finalize(Pipeline &pipeline, Event &event, auto &table = gstate.table; auto &storage = table.GetStorage(); LocalAppendState append_state; - storage.InitializeLocalAppend(append_state, table, context); + storage.InitializeLocalAppend(append_state, table, context, bound_constraints); auto &transaction = DuckTransaction::Get(context, table.catalog); for (auto &entry : gstate.collections) { if (entry.type != RowGroupBatchType::NOT_FLUSHED) { diff --git a/src/execution/operator/persistent/physical_delete.cpp b/src/execution/operator/persistent/physical_delete.cpp index 300376ce27fe..bb2b1a76057a 100644 --- a/src/execution/operator/persistent/physical_delete.cpp +++ b/src/execution/operator/persistent/physical_delete.cpp @@ -10,6 +10,13 @@ namespace duckdb { +PhysicalDelete::PhysicalDelete(vector types, TableCatalogEntry &tableref, DataTable &table, + vector> bound_constraints, idx_t row_id_index, + idx_t estimated_cardinality, bool return_chunk) + : PhysicalOperator(PhysicalOperatorType::DELETE_OPERATOR, std::move(types), estimated_cardinality), + tableref(tableref), table(table), bound_constraints(std::move(bound_constraints)), row_id_index(row_id_index), + return_chunk(return_chunk) { +} //===--------------------------------------------------------------------===// // Sink //===--------------------------------------------------------------------===// @@ -26,9 +33,10 @@ class DeleteGlobalState : public GlobalSinkState { class DeleteLocalState : public LocalSinkState { public: - DeleteLocalState(ClientContext &context, TableCatalogEntry &table) { + DeleteLocalState(ClientContext &context, TableCatalogEntry &table, + const vector> &bound_constraints) { delete_chunk.Initialize(Allocator::Get(context), table.GetTypes()); - delete_state = table.GetStorage().InitializeDelete(table, context); + delete_state = table.GetStorage().InitializeDelete(table, context, bound_constraints); } DataChunk delete_chunk; unique_ptr delete_state; @@ -64,7 +72,7 @@ unique_ptr PhysicalDelete::GetGlobalSinkState(ClientContext &co } unique_ptr PhysicalDelete::GetLocalSinkState(ExecutionContext &context) const { - return make_uniq(context.client, tableref); + return make_uniq(context.client, tableref, bound_constraints); } //===--------------------------------------------------------------------===// diff --git a/src/execution/operator/persistent/physical_insert.cpp b/src/execution/operator/persistent/physical_insert.cpp index c7b499855c5a..a2d3e9c61110 100644 --- a/src/execution/operator/persistent/physical_insert.cpp +++ b/src/execution/operator/persistent/physical_insert.cpp @@ -21,22 +21,20 @@ namespace duckdb { -PhysicalInsert::PhysicalInsert(vector types_p, TableCatalogEntry &table, - physical_index_vector_t column_index_map, - vector> bound_defaults, - vector> set_expressions, vector set_columns, - vector set_types, idx_t estimated_cardinality, bool return_chunk, - bool parallel, OnConflictAction action_type, - unique_ptr on_conflict_condition_p, - unique_ptr do_update_condition_p, unordered_set conflict_target_p, - vector columns_to_fetch_p) +PhysicalInsert::PhysicalInsert( + vector types_p, TableCatalogEntry &table, physical_index_vector_t column_index_map, + vector> bound_defaults, vector> bound_constraints_p, + vector> set_expressions, vector set_columns, vector set_types, + idx_t estimated_cardinality, bool return_chunk, bool parallel, OnConflictAction action_type, + unique_ptr on_conflict_condition_p, unique_ptr do_update_condition_p, + unordered_set conflict_target_p, vector columns_to_fetch_p) : PhysicalOperator(PhysicalOperatorType::INSERT, std::move(types_p), estimated_cardinality), column_index_map(std::move(column_index_map)), insert_table(&table), insert_types(table.GetTypes()), - bound_defaults(std::move(bound_defaults)), return_chunk(return_chunk), parallel(parallel), - action_type(action_type), set_expressions(std::move(set_expressions)), set_columns(std::move(set_columns)), - set_types(std::move(set_types)), on_conflict_condition(std::move(on_conflict_condition_p)), - do_update_condition(std::move(do_update_condition_p)), conflict_target(std::move(conflict_target_p)), - columns_to_fetch(std::move(columns_to_fetch_p)) { + bound_defaults(std::move(bound_defaults)), bound_constraints(std::move(bound_constraints_p)), + return_chunk(return_chunk), parallel(parallel), action_type(action_type), + set_expressions(std::move(set_expressions)), set_columns(std::move(set_columns)), set_types(std::move(set_types)), + on_conflict_condition(std::move(on_conflict_condition_p)), do_update_condition(std::move(do_update_condition_p)), + conflict_target(std::move(conflict_target_p)), columns_to_fetch(std::move(columns_to_fetch_p)) { if (action_type == OnConflictAction::THROW) { return; @@ -91,8 +89,9 @@ class InsertGlobalState : public GlobalSinkState { class InsertLocalState : public LocalSinkState { public: InsertLocalState(ClientContext &context, const vector &types, - const vector> &bound_defaults) - : default_executor(context, bound_defaults) { + const vector> &bound_defaults, + const vector> &bound_constraints) + : default_executor(context, bound_defaults), bound_constraints(bound_constraints) { insert_chunk.Initialize(Allocator::Get(context), types); } @@ -106,12 +105,12 @@ class InsertLocalState : public LocalSinkState { // Rows in the transaction-local storage that have been updated by a DO UPDATE conflict unordered_set updated_local_rows; idx_t update_count = 0; - unique_ptr constraint_state; + unique_ptr constraint_state; + const vector> &bound_constraints; - ConstraintVerificationState &GetConstraintState(DataTable &table, TableCatalogEntry &tableref, - ClientContext &context) { + ConstraintState &GetConstraintState(DataTable &table, TableCatalogEntry &tableref) { if (!constraint_state) { - constraint_state = table.InitializeConstraintVerification(tableref, context); + constraint_state = table.InitializeConstraintState(tableref, bound_constraints); } return *constraint_state; } @@ -135,7 +134,7 @@ unique_ptr PhysicalInsert::GetGlobalSinkState(ClientContext &co } unique_ptr PhysicalInsert::GetLocalSinkState(ExecutionContext &context) const { - return make_uniq(context.client, insert_types, bound_defaults); + return make_uniq(context.client, insert_types, bound_defaults, bound_constraints); } void PhysicalInsert::ResolveDefaults(const TableCatalogEntry &table, DataChunk &chunk, @@ -288,7 +287,7 @@ static idx_t PerformOnConflictAction(ExecutionContext &context, DataChunk &chunk auto &data_table = table.GetStorage(); // Perform the update, using the results of the SET expressions if (GLOBAL) { - auto update_state = data_table.InitializeUpdate(table, context.client); + auto update_state = data_table.InitializeUpdate(table, context.client, op.bound_constraints); data_table.Update(*update_state, context.client, row_ids, set_columns, update_chunk); } else { auto &local_storage = LocalStorage::Get(context.client, data_table.db); @@ -331,7 +330,7 @@ static idx_t HandleInsertConflicts(TableCatalogEntry &table, ExecutionContext &c ConflictInfo conflict_info(conflict_target); ConflictManager conflict_manager(VerifyExistenceType::APPEND, lstate.insert_chunk.size(), &conflict_info); if (GLOBAL) { - auto &constraint_state = lstate.GetConstraintState(data_table, table, context.client); + auto &constraint_state = lstate.GetConstraintState(data_table, table); data_table.VerifyAppendConstraints(constraint_state, context.client, lstate.insert_chunk, &conflict_manager); } else { DataTable::VerifyUniqueIndexes(local_storage.GetIndexes(data_table), context.client, lstate.insert_chunk, @@ -392,7 +391,7 @@ static idx_t HandleInsertConflicts(TableCatalogEntry &table, ExecutionContext &c combined_chunk.Slice(sel.Selection(), sel.Count()); row_ids.Slice(sel.Selection(), sel.Count()); if (GLOBAL) { - auto &constraint_state = lstate.GetConstraintState(data_table, table, context.client); + auto &constraint_state = lstate.GetConstraintState(data_table, table); data_table.VerifyAppendConstraints(constraint_state, context.client, combined_chunk, nullptr); } else { DataTable::VerifyUniqueIndexes(local_storage.GetIndexes(data_table), context.client, @@ -419,7 +418,7 @@ idx_t PhysicalInsert::OnConflictHandling(TableCatalogEntry &table, ExecutionCont InsertLocalState &lstate) const { auto &data_table = table.GetStorage(); if (action_type == OnConflictAction::THROW) { - auto &constraint_state = lstate.GetConstraintState(data_table, table, context.client); + auto &constraint_state = lstate.GetConstraintState(data_table, table); data_table.VerifyAppendConstraints(constraint_state, context.client, lstate.insert_chunk, nullptr); return 0; } @@ -443,7 +442,7 @@ SinkResultType PhysicalInsert::Sink(ExecutionContext &context, DataChunk &chunk, if (!parallel) { if (!gstate.initialized) { - storage.InitializeLocalAppend(gstate.append_state, table, context.client); + storage.InitializeLocalAppend(gstate.append_state, table, context.client, bound_constraints); gstate.initialized = true; } @@ -501,7 +500,7 @@ SinkCombineResultType PhysicalInsert::Combine(ExecutionContext &context, Operato // we have few rows - append to the local storage directly auto &table = gstate.table; auto &storage = table.GetStorage(); - storage.InitializeLocalAppend(gstate.append_state, table, context.client); + storage.InitializeLocalAppend(gstate.append_state, table, context.client, bound_constraints); auto &transaction = DuckTransaction::Get(context.client, table.catalog); lstate.local_collection->Scan(transaction, [&](DataChunk &insert_chunk) { storage.LocalAppend(gstate.append_state, table, context.client, insert_chunk); diff --git a/src/execution/operator/persistent/physical_update.cpp b/src/execution/operator/persistent/physical_update.cpp index 6c8c6c041028..8dc663430642 100644 --- a/src/execution/operator/persistent/physical_update.cpp +++ b/src/execution/operator/persistent/physical_update.cpp @@ -15,11 +15,13 @@ namespace duckdb { PhysicalUpdate::PhysicalUpdate(vector types, TableCatalogEntry &tableref, DataTable &table, vector columns, vector> expressions, - vector> bound_defaults, idx_t estimated_cardinality, + vector> bound_defaults, + vector> bound_constraints, idx_t estimated_cardinality, bool return_chunk) : PhysicalOperator(PhysicalOperatorType::UPDATE, std::move(types), estimated_cardinality), tableref(tableref), table(table), columns(std::move(columns)), expressions(std::move(expressions)), - bound_defaults(std::move(bound_defaults)), return_chunk(return_chunk) { + bound_defaults(std::move(bound_defaults)), bound_constraints(std::move(bound_constraints)), + return_chunk(return_chunk) { } //===--------------------------------------------------------------------===// @@ -40,8 +42,9 @@ class UpdateGlobalState : public GlobalSinkState { class UpdateLocalState : public LocalSinkState { public: UpdateLocalState(ClientContext &context, const vector> &expressions, - const vector &table_types, const vector> &bound_defaults) - : default_executor(context, bound_defaults) { + const vector &table_types, const vector> &bound_defaults, + const vector> &bound_constraints) + : default_executor(context, bound_defaults), bound_constraints(bound_constraints) { // initialize the update chunk auto &allocator = Allocator::Get(context); vector update_types; @@ -59,17 +62,18 @@ class UpdateLocalState : public LocalSinkState { ExpressionExecutor default_executor; unique_ptr delete_state; unique_ptr update_state; + const vector> &bound_constraints; TableDeleteState &GetDeleteState(DataTable &table, TableCatalogEntry &tableref, ClientContext &context) { if (!delete_state) { - delete_state = table.InitializeDelete(tableref, context); + delete_state = table.InitializeDelete(tableref, context, bound_constraints); } return *delete_state; } TableUpdateState &GetUpdateState(DataTable &table, TableCatalogEntry &tableref, ClientContext &context) { if (!update_state) { - update_state = table.InitializeUpdate(tableref, context); + update_state = table.InitializeUpdate(tableref, context, bound_constraints); } return *update_state; } @@ -131,7 +135,7 @@ SinkResultType PhysicalUpdate::Sink(ExecutionContext &context, DataChunk &chunk, for (idx_t i = 0; i < columns.size(); i++) { mock_chunk.data[columns[i].index].Reference(update_chunk.data[i]); } - table.LocalAppend(tableref, context.client, mock_chunk); + table.LocalAppend(tableref, context.client, mock_chunk, bound_constraints); } else { if (return_chunk) { mock_chunk.SetCardinality(update_chunk); @@ -157,7 +161,8 @@ unique_ptr PhysicalUpdate::GetGlobalSinkState(ClientContext &co } unique_ptr PhysicalUpdate::GetLocalSinkState(ExecutionContext &context) const { - return make_uniq(context.client, expressions, table.GetTypes(), bound_defaults); + return make_uniq(context.client, expressions, table.GetTypes(), bound_defaults, + bound_constraints); } SinkCombineResultType PhysicalUpdate::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const { diff --git a/src/execution/physical_plan/plan_delete.cpp b/src/execution/physical_plan/plan_delete.cpp index 3a748c00da4b..7550da083d44 100644 --- a/src/execution/physical_plan/plan_delete.cpp +++ b/src/execution/physical_plan/plan_delete.cpp @@ -12,8 +12,8 @@ unique_ptr DuckCatalog::PlanDelete(ClientContext &context, Log // get the index of the row_id column auto &bound_ref = op.expressions[0]->Cast(); - auto del = make_uniq(op.types, op.table, op.table.GetStorage(), bound_ref.index, - op.estimated_cardinality, op.return_chunk); + auto del = make_uniq(op.types, op.table, op.table.GetStorage(), std::move(op.bound_constraints), + bound_ref.index, op.estimated_cardinality, op.return_chunk); del->children.push_back(std::move(plan)); return std::move(del); } diff --git a/src/execution/physical_plan/plan_insert.cpp b/src/execution/physical_plan/plan_insert.cpp index 3aa87d9abd7e..0ce031d07b0b 100644 --- a/src/execution/physical_plan/plan_insert.cpp +++ b/src/execution/physical_plan/plan_insert.cpp @@ -94,13 +94,14 @@ unique_ptr DuckCatalog::PlanInsert(ClientContext &context, Log unique_ptr insert; if (use_batch_index && !parallel_streaming_insert) { insert = make_uniq(op.types, op.table, op.column_index_map, std::move(op.bound_defaults), - op.estimated_cardinality); + std::move(op.bound_constraints), op.estimated_cardinality); } else { insert = make_uniq( - op.types, op.table, op.column_index_map, std::move(op.bound_defaults), std::move(op.expressions), - std::move(op.set_columns), std::move(op.set_types), op.estimated_cardinality, op.return_chunk, - parallel_streaming_insert && num_threads > 1, op.action_type, std::move(op.on_conflict_condition), - std::move(op.do_update_condition), std::move(op.on_conflict_filter), std::move(op.columns_to_fetch)); + op.types, op.table, op.column_index_map, std::move(op.bound_defaults), std::move(op.bound_constraints), + std::move(op.expressions), std::move(op.set_columns), std::move(op.set_types), op.estimated_cardinality, + op.return_chunk, parallel_streaming_insert && num_threads > 1, op.action_type, + std::move(op.on_conflict_condition), std::move(op.do_update_condition), std::move(op.on_conflict_filter), + std::move(op.columns_to_fetch)); } D_ASSERT(plan); insert->children.push_back(std::move(plan)); diff --git a/src/execution/physical_plan/plan_update.cpp b/src/execution/physical_plan/plan_update.cpp index 2789008347b6..b3591423303f 100644 --- a/src/execution/physical_plan/plan_update.cpp +++ b/src/execution/physical_plan/plan_update.cpp @@ -8,9 +8,9 @@ namespace duckdb { unique_ptr DuckCatalog::PlanUpdate(ClientContext &context, LogicalUpdate &op, unique_ptr plan) { - auto update = - make_uniq(op.types, op.table, op.table.GetStorage(), op.columns, std::move(op.expressions), - std::move(op.bound_defaults), op.estimated_cardinality, op.return_chunk); + auto update = make_uniq(op.types, op.table, op.table.GetStorage(), op.columns, + std::move(op.expressions), std::move(op.bound_defaults), + std::move(op.bound_constraints), op.estimated_cardinality, op.return_chunk); update->update_is_del_and_insert = op.update_is_del_and_insert; update->children.push_back(std::move(plan)); diff --git a/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp b/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp index fa9a795f2b0f..6e8c0f248944 100644 --- a/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp +++ b/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp @@ -77,7 +77,7 @@ class TableCatalogEntry : public StandardEntry { virtual DataTable &GetStorage(); //! Returns a list of the constraints of the table - DUCKDB_API const vector> &GetConstraints(); + DUCKDB_API const vector> &GetConstraints() const; DUCKDB_API string ToSQL() const override; //! Get statistics of a column (physical or virtual) within the table diff --git a/src/include/duckdb/execution/operator/persistent/physical_batch_insert.hpp b/src/include/duckdb/execution/operator/persistent/physical_batch_insert.hpp index 1a7155f37076..bce3ac633cf9 100644 --- a/src/include/duckdb/execution/operator/persistent/physical_batch_insert.hpp +++ b/src/include/duckdb/execution/operator/persistent/physical_batch_insert.hpp @@ -20,7 +20,7 @@ class PhysicalBatchInsert : public PhysicalOperator { //! INSERT INTO PhysicalBatchInsert(vector types, TableCatalogEntry &table, physical_index_vector_t column_index_map, vector> bound_defaults, - idx_t estimated_cardinality); + vector> bound_constraints, idx_t estimated_cardinality); //! CREATE TABLE AS PhysicalBatchInsert(LogicalOperator &op, SchemaCatalogEntry &schema, unique_ptr info, idx_t estimated_cardinality); @@ -33,6 +33,8 @@ class PhysicalBatchInsert : public PhysicalOperator { vector insert_types; //! The default expressions of the columns for which no value is provided vector> bound_defaults; + //! The bound constraints for the table + vector> bound_constraints; //! Table schema, in case of CREATE TABLE AS optional_ptr schema; //! Create table info, in case of CREATE TABLE AS diff --git a/src/include/duckdb/execution/operator/persistent/physical_delete.hpp b/src/include/duckdb/execution/operator/persistent/physical_delete.hpp index 387df24d05bf..740db21ad759 100644 --- a/src/include/duckdb/execution/operator/persistent/physical_delete.hpp +++ b/src/include/duckdb/execution/operator/persistent/physical_delete.hpp @@ -19,14 +19,13 @@ class PhysicalDelete : public PhysicalOperator { static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::DELETE_OPERATOR; public: - PhysicalDelete(vector types, TableCatalogEntry &tableref, DataTable &table, idx_t row_id_index, - idx_t estimated_cardinality, bool return_chunk) - : PhysicalOperator(PhysicalOperatorType::DELETE_OPERATOR, std::move(types), estimated_cardinality), - tableref(tableref), table(table), row_id_index(row_id_index), return_chunk(return_chunk) { - } + PhysicalDelete(vector types, TableCatalogEntry &tableref, DataTable &table, + vector> bound_constraints, idx_t row_id_index, + idx_t estimated_cardinality, bool return_chunk); TableCatalogEntry &tableref; DataTable &table; + vector> bound_constraints; idx_t row_id_index; bool return_chunk; diff --git a/src/include/duckdb/execution/operator/persistent/physical_insert.hpp b/src/include/duckdb/execution/operator/persistent/physical_insert.hpp index 8609a928a6fd..9680b5837b71 100644 --- a/src/include/duckdb/execution/operator/persistent/physical_insert.hpp +++ b/src/include/duckdb/execution/operator/persistent/physical_insert.hpp @@ -26,11 +26,12 @@ class PhysicalInsert : public PhysicalOperator { public: //! INSERT INTO PhysicalInsert(vector types, TableCatalogEntry &table, physical_index_vector_t column_index_map, - vector> bound_defaults, vector> set_expressions, - vector set_columns, vector set_types, idx_t estimated_cardinality, - bool return_chunk, bool parallel, OnConflictAction action_type, - unique_ptr on_conflict_condition, unique_ptr do_update_condition, - unordered_set on_conflict_filter, vector columns_to_fetch); + vector> bound_defaults, vector> bound_constraints, + vector> set_expressions, vector set_columns, + vector set_types, idx_t estimated_cardinality, bool return_chunk, bool parallel, + OnConflictAction action_type, unique_ptr on_conflict_condition, + unique_ptr do_update_condition, unordered_set on_conflict_filter, + vector columns_to_fetch); //! CREATE TABLE AS PhysicalInsert(LogicalOperator &op, SchemaCatalogEntry &schema, unique_ptr info, idx_t estimated_cardinality, bool parallel); @@ -43,6 +44,8 @@ class PhysicalInsert : public PhysicalOperator { vector insert_types; //! The default expressions of the columns for which no value is provided vector> bound_defaults; + //! The bound constraints for the table + vector> bound_constraints; //! If the returning statement is present, return the whole chunk bool return_chunk; //! Table schema, in case of CREATE TABLE AS diff --git a/src/include/duckdb/execution/operator/persistent/physical_update.hpp b/src/include/duckdb/execution/operator/persistent/physical_update.hpp index 0cccd98d51ee..b064c8b2e98f 100644 --- a/src/include/duckdb/execution/operator/persistent/physical_update.hpp +++ b/src/include/duckdb/execution/operator/persistent/physical_update.hpp @@ -22,13 +22,15 @@ class PhysicalUpdate : public PhysicalOperator { public: PhysicalUpdate(vector types, TableCatalogEntry &tableref, DataTable &table, vector columns, vector> expressions, - vector> bound_defaults, idx_t estimated_cardinality, bool return_chunk); + vector> bound_defaults, vector> bound_constraints, + idx_t estimated_cardinality, bool return_chunk); TableCatalogEntry &tableref; DataTable &table; vector columns; vector> expressions; vector> bound_defaults; + vector> bound_constraints; bool update_is_del_and_insert; //! If the returning statement is present, return the whole chunk bool return_chunk; diff --git a/src/include/duckdb/planner/binder.hpp b/src/include/duckdb/planner/binder.hpp index 088b4ae858b8..c2afbc1b8f5a 100644 --- a/src/include/duckdb/planner/binder.hpp +++ b/src/include/duckdb/planner/binder.hpp @@ -127,6 +127,7 @@ class Binder : public enable_shared_from_this { const string &table_name, const ColumnList &columns); vector> BindConstraints(const vector> &constraints, const string &table_name, const ColumnList &columns); + vector> BindConstraints(const TableCatalogEntry &table); vector> BindNewConstraints(vector> &constraints, const string &table_name, const ColumnList &columns); diff --git a/src/include/duckdb/planner/operator/logical_delete.hpp b/src/include/duckdb/planner/operator/logical_delete.hpp index 005d955b6b82..9a243a1f730b 100644 --- a/src/include/duckdb/planner/operator/logical_delete.hpp +++ b/src/include/duckdb/planner/operator/logical_delete.hpp @@ -23,6 +23,7 @@ class LogicalDelete : public LogicalOperator { TableCatalogEntry &table; idx_t table_index; bool return_chunk; + vector> bound_constraints; public: void Serialize(Serializer &serializer) const override; diff --git a/src/include/duckdb/planner/operator/logical_insert.hpp b/src/include/duckdb/planner/operator/logical_insert.hpp index 0c31fd6e4bd4..c8eb3997f844 100644 --- a/src/include/duckdb/planner/operator/logical_insert.hpp +++ b/src/include/duckdb/planner/operator/logical_insert.hpp @@ -37,6 +37,8 @@ class LogicalInsert : public LogicalOperator { bool return_chunk; //! The default statements used by the table vector> bound_defaults; + //! The constraints used by the table + vector> bound_constraints; //! Which action to take on conflict OnConflictAction action_type; diff --git a/src/include/duckdb/planner/operator/logical_update.hpp b/src/include/duckdb/planner/operator/logical_update.hpp index 9215ff694685..e0356419b0d2 100644 --- a/src/include/duckdb/planner/operator/logical_update.hpp +++ b/src/include/duckdb/planner/operator/logical_update.hpp @@ -28,6 +28,7 @@ class LogicalUpdate : public LogicalOperator { bool return_chunk; vector columns; vector> bound_defaults; + vector> bound_constraints; bool update_is_del_and_insert; public: diff --git a/src/include/duckdb/storage/data_table.hpp b/src/include/duckdb/storage/data_table.hpp index e6aedf464bc9..dcde5af382b5 100644 --- a/src/include/duckdb/storage/data_table.hpp +++ b/src/include/duckdb/storage/data_table.hpp @@ -42,7 +42,7 @@ class TableDataWriter; class ConflictManager; class TableScanState; struct TableDeleteState; -struct ConstraintVerificationState; +struct ConstraintState; struct TableUpdateState; enum class VerifyExistenceType : uint8_t; @@ -95,27 +95,32 @@ class DataTable { const Vector &row_ids, idx_t fetch_count, ColumnFetchState &state); //! Initializes an append to transaction-local storage - void InitializeLocalAppend(LocalAppendState &state, TableCatalogEntry &table, ClientContext &context); + void InitializeLocalAppend(LocalAppendState &state, TableCatalogEntry &table, ClientContext &context, + const vector> &bound_constraints); //! Append a DataChunk to the transaction-local storage of the table. void LocalAppend(LocalAppendState &state, TableCatalogEntry &table, ClientContext &context, DataChunk &chunk, bool unsafe = false); //! Finalizes a transaction-local append void FinalizeLocalAppend(LocalAppendState &state); //! Append a chunk to the transaction-local storage of this table - void LocalAppend(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk); + void LocalAppend(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk, + const vector> &bound_constraints); //! Append a column data collection to the transaction-local storage of this table - void LocalAppend(TableCatalogEntry &table, ClientContext &context, ColumnDataCollection &collection); + void LocalAppend(TableCatalogEntry &table, ClientContext &context, ColumnDataCollection &collection, + const vector> &bound_constraints); //! Merge a row group collection into the transaction-local storage void LocalMerge(ClientContext &context, RowGroupCollection &collection); //! Creates an optimistic writer for this table - used for optimistically writing parallel appends OptimisticDataWriter &CreateOptimisticWriter(ClientContext &context); void FinalizeOptimisticWriter(ClientContext &context, OptimisticDataWriter &writer); - unique_ptr InitializeDelete(TableCatalogEntry &table, ClientContext &context); + unique_ptr InitializeDelete(TableCatalogEntry &table, ClientContext &context, + const vector> &bound_constraints); //! Delete the entries with the specified row identifier from the table idx_t Delete(TableDeleteState &state, ClientContext &context, Vector &row_ids, idx_t count); - unique_ptr InitializeUpdate(TableCatalogEntry &table, ClientContext &context); + unique_ptr InitializeUpdate(TableCatalogEntry &table, ClientContext &context, + const vector> &bound_constraints); //! Update the entries with the specified row identifier from the table void Update(TableUpdateState &state, ClientContext &context, Vector &row_ids, const vector &column_ids, DataChunk &data); @@ -192,11 +197,11 @@ class DataTable { //! FIXME: This is only necessary until we treat all indexes as catalog entries, allowing to alter constraints bool IndexNameIsUnique(const string &name); - //! Initialize constraint verification - unique_ptr InitializeConstraintVerification(TableCatalogEntry &table, - ClientContext &context); + //! Initialize constraint verification state + unique_ptr InitializeConstraintState(TableCatalogEntry &table, + const vector> &bound_constraints); //! Verify constraints with a chunk from the Append containing all columns of the table - void VerifyAppendConstraints(ConstraintVerificationState &state, ClientContext &context, DataChunk &chunk, + void VerifyAppendConstraints(ConstraintState &state, ClientContext &context, DataChunk &chunk, optional_ptr conflict_manager = nullptr); public: @@ -207,7 +212,7 @@ class DataTable { //! Verify the new added constraints against current persistent&local data void VerifyNewConstraint(ClientContext &context, DataTable &parent, const BoundConstraint *constraint); //! Verify constraints with a chunk from the Update containing only the specified column_ids - void VerifyUpdateConstraints(ConstraintVerificationState &state, ClientContext &context, DataChunk &chunk, + void VerifyUpdateConstraints(ConstraintState &state, ClientContext &context, DataChunk &chunk, const vector &column_ids); //! Verify constraints with a chunk from the Delete containing all columns of the table void VerifyDeleteConstraints(TableDeleteState &state, ClientContext &context, DataChunk &chunk); diff --git a/src/include/duckdb/storage/table/append_state.hpp b/src/include/duckdb/storage/table/append_state.hpp index 8f9a56c5cba2..4488e0aaa791 100644 --- a/src/include/duckdb/storage/table/append_state.hpp +++ b/src/include/duckdb/storage/table/append_state.hpp @@ -70,18 +70,19 @@ struct TableAppendState { TransactionData transaction; }; -struct ConstraintVerificationState { - explicit ConstraintVerificationState(TableCatalogEntry &table_p) : table(table_p) { +struct ConstraintState { + explicit ConstraintState(TableCatalogEntry &table_p, const vector> &bound_constraints) + : table(table_p), bound_constraints(bound_constraints) { } TableCatalogEntry &table; - vector> bound_constraints; + const vector> &bound_constraints; }; struct LocalAppendState { TableAppendState append_state; LocalTableStorage *storage; - unique_ptr constraint_state; + unique_ptr constraint_state; }; } // namespace duckdb diff --git a/src/include/duckdb/storage/table/delete_state.hpp b/src/include/duckdb/storage/table/delete_state.hpp index ec2c2d57c547..6d25df4a5aee 100644 --- a/src/include/duckdb/storage/table/delete_state.hpp +++ b/src/include/duckdb/storage/table/delete_state.hpp @@ -14,7 +14,7 @@ namespace duckdb { class TableCatalogEntry; struct TableDeleteState { - vector> bound_constraints; + unique_ptr constraint_state; bool has_delete_constraints = false; DataChunk verify_chunk; vector col_ids; diff --git a/src/include/duckdb/storage/table/update_state.hpp b/src/include/duckdb/storage/table/update_state.hpp index 50ce404e0132..0a8193c54776 100644 --- a/src/include/duckdb/storage/table/update_state.hpp +++ b/src/include/duckdb/storage/table/update_state.hpp @@ -14,7 +14,7 @@ namespace duckdb { class TableCatalogEntry; struct TableUpdateState { - unique_ptr constraint_state; + unique_ptr constraint_state; }; } // namespace duckdb diff --git a/src/main/appender.cpp b/src/main/appender.cpp index 28a622755b3a..1a8fae39cf96 100644 --- a/src/main/appender.cpp +++ b/src/main/appender.cpp @@ -376,7 +376,9 @@ void Appender::FlushInternal(ColumnDataCollection &collection) { } void InternalAppender::FlushInternal(ColumnDataCollection &collection) { - table.GetStorage().LocalAppend(table, context, collection); + auto binder = Binder::CreateBinder(context); + auto bound_constraints = binder->BindConstraints(table); + table.GetStorage().LocalAppend(table, context, collection, bound_constraints); } void BaseAppender::Close() { diff --git a/src/main/client_context.cpp b/src/main/client_context.cpp index 6cd4573d9168..e6d7eddda401 100644 --- a/src/main/client_context.cpp +++ b/src/main/client_context.cpp @@ -1121,7 +1121,9 @@ void ClientContext::Append(TableDescription &description, ColumnDataCollection & throw InvalidInputException("Failed to append: table entry has different number of columns!"); } } - table_entry.GetStorage().LocalAppend(table_entry, *this, collection); + auto binder = Binder::CreateBinder(*this); + auto bound_constraints = binder->BindConstraints(table_entry); + table_entry.GetStorage().LocalAppend(table_entry, *this, collection, bound_constraints); }); } diff --git a/src/planner/binder/statement/bind_create_table.cpp b/src/planner/binder/statement/bind_create_table.cpp index 8a032b9fe1ee..4651ede86deb 100644 --- a/src/planner/binder/statement/bind_create_table.cpp +++ b/src/planner/binder/statement/bind_create_table.cpp @@ -55,6 +55,10 @@ vector> Binder::BindConstraints(ClientContext &conte return binder->BindConstraints(constraints, table_name, columns); } +vector> Binder::BindConstraints(const TableCatalogEntry &table) { + return BindConstraints(table.GetConstraints(), table.name, table.GetColumns()); +} + vector> Binder::BindConstraints(const vector> &constraints, const string &table_name, const ColumnList &columns) { vector> bound_constraints; diff --git a/src/planner/binder/statement/bind_delete.cpp b/src/planner/binder/statement/bind_delete.cpp index 7ae86228250a..da1707160776 100644 --- a/src/planner/binder/statement/bind_delete.cpp +++ b/src/planner/binder/statement/bind_delete.cpp @@ -69,6 +69,7 @@ BoundStatement Binder::Bind(DeleteStatement &stmt) { } // create the delete node auto del = make_uniq(table, GenerateTableIndex()); + del->bound_constraints = BindConstraints(table); del->AddChild(std::move(root)); // set up the delete expression diff --git a/src/planner/binder/statement/bind_insert.cpp b/src/planner/binder/statement/bind_insert.cpp index cb3782b822db..13ad3ea7ca9a 100644 --- a/src/planner/binder/statement/bind_insert.cpp +++ b/src/planner/binder/statement/bind_insert.cpp @@ -472,6 +472,7 @@ BoundStatement Binder::Bind(InsertStatement &stmt) { // bind the default values BindDefaultValues(table.GetColumns(), insert->bound_defaults); + insert->bound_constraints = BindConstraints(table); if (!stmt.select_statement && !stmt.default_values) { result.plan = std::move(insert); return result; diff --git a/src/planner/binder/statement/bind_update.cpp b/src/planner/binder/statement/bind_update.cpp index d3ea3a5f4774..2d0f6687ca61 100644 --- a/src/planner/binder/statement/bind_update.cpp +++ b/src/planner/binder/statement/bind_update.cpp @@ -106,6 +106,7 @@ BoundStatement Binder::Bind(UpdateStatement &stmt) { } // bind the default values BindDefaultValues(table.GetColumns(), update->bound_defaults); + update->bound_constraints = BindConstraints(table); // project any additional columns required for the condition/expressions if (stmt.set_info->condition) { diff --git a/src/planner/operator/logical_delete.cpp b/src/planner/operator/logical_delete.cpp index 950f2eaa2ebb..d82c3bfd86f2 100644 --- a/src/planner/operator/logical_delete.cpp +++ b/src/planner/operator/logical_delete.cpp @@ -15,6 +15,8 @@ LogicalDelete::LogicalDelete(ClientContext &context, const unique_ptr(context, table_info->catalog, table_info->schema, table_info->Cast().table)) { + auto binder = Binder::CreateBinder(context); + bound_constraints = binder->BindConstraints(table); } idx_t LogicalDelete::EstimateCardinality(ClientContext &context) { diff --git a/src/planner/operator/logical_insert.cpp b/src/planner/operator/logical_insert.cpp index 518661616058..dd5bf92aaeab 100644 --- a/src/planner/operator/logical_insert.cpp +++ b/src/planner/operator/logical_insert.cpp @@ -15,6 +15,8 @@ LogicalInsert::LogicalInsert(ClientContext &context, const unique_ptr(context, table_info->catalog, table_info->schema, table_info->Cast().table)) { + auto binder = Binder::CreateBinder(context); + bound_constraints = binder->BindConstraints(table); } idx_t LogicalInsert::EstimateCardinality(ClientContext &context) { diff --git a/src/planner/operator/logical_update.cpp b/src/planner/operator/logical_update.cpp index edcfd5d891be..386264478417 100644 --- a/src/planner/operator/logical_update.cpp +++ b/src/planner/operator/logical_update.cpp @@ -13,6 +13,8 @@ LogicalUpdate::LogicalUpdate(ClientContext &context, const unique_ptr(context, table_info->catalog, table_info->schema, table_info->Cast().table)) { + auto binder = Binder::CreateBinder(context); + bound_constraints = binder->BindConstraints(table); } idx_t LogicalUpdate::EstimateCardinality(ClientContext &context) { diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 39c855a20e2b..588b7c38139a 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -616,7 +616,7 @@ void DataTable::VerifyUniqueIndexes(TableIndexList &indexes, ClientContext &cont }); } -void DataTable::VerifyAppendConstraints(ConstraintVerificationState &state, ClientContext &context, DataChunk &chunk, +void DataTable::VerifyAppendConstraints(ConstraintState &state, ClientContext &context, DataChunk &chunk, optional_ptr conflict_manager) { auto &table = state.table; if (table.HasGeneratedColumns()) { @@ -675,22 +675,21 @@ void DataTable::VerifyAppendConstraints(ConstraintVerificationState &state, Clie } } -unique_ptr DataTable::InitializeConstraintVerification(TableCatalogEntry &table, - ClientContext &context) { - auto result = make_uniq(table); - auto binder = Binder::CreateBinder(context); - result->bound_constraints = binder->BindConstraints(table.GetConstraints(), table.name, table.GetColumns()); - return result; +unique_ptr +DataTable::InitializeConstraintState(TableCatalogEntry &table, + const vector> &bound_constraints) { + return make_uniq(table, bound_constraints); } -void DataTable::InitializeLocalAppend(LocalAppendState &state, TableCatalogEntry &table, ClientContext &context) { +void DataTable::InitializeLocalAppend(LocalAppendState &state, TableCatalogEntry &table, ClientContext &context, + const vector> &bound_constraints) { if (!is_root) { throw TransactionException("Transaction conflict: adding entries to a table that has been altered!"); } auto &local_storage = LocalStorage::Get(context, db); local_storage.InitializeAppend(state, *this); - state.constraint_state = InitializeConstraintVerification(table, context); + state.constraint_state = InitializeConstraintState(table, bound_constraints); } void DataTable::LocalAppend(LocalAppendState &state, TableCatalogEntry &table, ClientContext &context, DataChunk &chunk, @@ -733,18 +732,20 @@ void DataTable::LocalMerge(ClientContext &context, RowGroupCollection &collectio local_storage.LocalMerge(*this, collection); } -void DataTable::LocalAppend(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk) { +void DataTable::LocalAppend(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk, + const vector> &bound_constraints) { LocalAppendState append_state; auto &storage = table.GetStorage(); - storage.InitializeLocalAppend(append_state, table, context); + storage.InitializeLocalAppend(append_state, table, context, bound_constraints); storage.LocalAppend(append_state, table, context, chunk); storage.FinalizeLocalAppend(append_state); } -void DataTable::LocalAppend(TableCatalogEntry &table, ClientContext &context, ColumnDataCollection &collection) { +void DataTable::LocalAppend(TableCatalogEntry &table, ClientContext &context, ColumnDataCollection &collection, + const vector> &bound_constraints) { LocalAppendState append_state; auto &storage = table.GetStorage(); - storage.InitializeLocalAppend(append_state, table, context); + storage.InitializeLocalAppend(append_state, table, context, bound_constraints); for (auto &chunk : collection.Chunks()) { storage.LocalAppend(append_state, table, context, chunk); } @@ -990,7 +991,7 @@ static bool TableHasDeleteConstraints(TableCatalogEntry &table) { } void DataTable::VerifyDeleteConstraints(TableDeleteState &state, ClientContext &context, DataChunk &chunk) { - for (auto &constraint : state.bound_constraints) { + for (auto &constraint : state.constraint_state->bound_constraints) { switch (constraint->type) { case ConstraintType::NOT_NULL: case ConstraintType::CHECK: @@ -1010,12 +1011,12 @@ void DataTable::VerifyDeleteConstraints(TableDeleteState &state, ClientContext & } } -unique_ptr DataTable::InitializeDelete(TableCatalogEntry &table, ClientContext &context) { +unique_ptr DataTable::InitializeDelete(TableCatalogEntry &table, ClientContext &context, + const vector> &bound_constraints) { // initialize indexes (if any) info->InitializeIndexes(context, true); auto binder = Binder::CreateBinder(context); - vector> bound_constraints; vector types; auto result = make_uniq(); result->has_delete_constraints = TableHasDeleteConstraints(table); @@ -1026,7 +1027,7 @@ unique_ptr DataTable::InitializeDelete(TableCatalogEntry &tabl types.emplace_back(column_definitions[i].Type()); } result->verify_chunk.Initialize(Allocator::Get(context), types); - result->bound_constraints = binder->BindConstraints(table.GetConstraints(), table.name, table.GetColumns()); + result->constraint_state = make_uniq(table, bound_constraints); } return result; } @@ -1120,7 +1121,7 @@ static bool CreateMockChunk(TableCatalogEntry &table, const vector &column_ids) { auto &table = state.table; auto &constraints = table.GetConstraints(); @@ -1170,12 +1171,13 @@ void DataTable::VerifyUpdateConstraints(ConstraintVerificationState &state, Clie #endif } -unique_ptr DataTable::InitializeUpdate(TableCatalogEntry &table, ClientContext &context) { +unique_ptr DataTable::InitializeUpdate(TableCatalogEntry &table, ClientContext &context, + const vector> &bound_constraints) { // check that there are no unknown indexes info->InitializeIndexes(context, true); auto result = make_uniq(); - result->constraint_state = InitializeConstraintVerification(table, context); + result->constraint_state = InitializeConstraintState(table, bound_constraints); return result; } diff --git a/src/storage/wal_replay.cpp b/src/storage/wal_replay.cpp index ab819a017145..ddd0e7422187 100644 --- a/src/storage/wal_replay.cpp +++ b/src/storage/wal_replay.cpp @@ -659,7 +659,9 @@ void WriteAheadLogDeserializer::ReplayInsert() { } // append to the current table - state.current_table->GetStorage().LocalAppend(*state.current_table, context, chunk); + // we don't do any constraint verification here + vector> bound_constraints; + state.current_table->GetStorage().LocalAppend(*state.current_table, context, chunk, bound_constraints); } void WriteAheadLogDeserializer::ReplayDelete() { From bc18ac803a8d95c9c7c9a0fc3017e4cd961a34e0 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 19 Apr 2024 12:28:40 +0200 Subject: [PATCH 581/603] Rename method as a homage to my tenage years --- .../operator/csv_scanner/sniffer/header_detection.cpp | 8 ++++---- .../operator/csv_scanner/sniffer/type_detection.cpp | 8 ++++---- .../duckdb/execution/operator/csv_scanner/csv_sniffer.hpp | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/execution/operator/csv_scanner/sniffer/header_detection.cpp b/src/execution/operator/csv_scanner/sniffer/header_detection.cpp index 523aff201b7c..fe37ab86a9b0 100644 --- a/src/execution/operator/csv_scanner/sniffer/header_detection.cpp +++ b/src/execution/operator/csv_scanner/sniffer/header_detection.cpp @@ -124,8 +124,8 @@ bool CSVSniffer::DetectHeaderWithSetColumn() { const auto &sql_type = (*set_columns.types)[col]; if (sql_type != LogicalType::VARCHAR) { all_varchar = false; - if (!IsCasteable(best_header_row[col].value, sql_type, options.dialect_options, - best_header_row[col].IsNull())) { + if (!CanYouCastIt(best_header_row[col].value, sql_type, options.dialect_options, + best_header_row[col].IsNull())) { first_row_consistent = false; } } @@ -176,8 +176,8 @@ void CSVSniffer::DetectHeader() { const auto &sql_type = best_sql_types_candidates_per_column_idx[col].back(); if (sql_type != LogicalType::VARCHAR) { all_varchar = false; - if (!IsCasteable(best_header_row[col].value, sql_type, sniffer_state_machine.dialect_options, - best_header_row[col].IsNull())) { + if (!CanYouCastIt(best_header_row[col].value, sql_type, sniffer_state_machine.dialect_options, + best_header_row[col].IsNull())) { first_row_consistent = false; } } diff --git a/src/execution/operator/csv_scanner/sniffer/type_detection.cpp b/src/execution/operator/csv_scanner/sniffer/type_detection.cpp index 31d500896204..75869c3601f0 100644 --- a/src/execution/operator/csv_scanner/sniffer/type_detection.cpp +++ b/src/execution/operator/csv_scanner/sniffer/type_detection.cpp @@ -93,8 +93,8 @@ void CSVSniffer::SetDateFormat(CSVStateMachine &candidate, const string &format_ candidate.dialect_options.date_format[sql_type].Set(strpformat, false); } -bool CSVSniffer::IsCasteable(const string_t value, const LogicalType &type, const DialectOptions &dialect_options, - const bool is_null) { +bool CSVSniffer::CanYouCastIt(const string_t value, const LogicalType &type, const DialectOptions &dialect_options, + const bool is_null) { if (is_null) { return true; } @@ -315,8 +315,8 @@ void CSVSniffer::DetectTypes() { // Nothing to convert it to continue; } - if (IsCasteable(vector_data[row_idx], sql_type, sniffing_state_machine.dialect_options, - !null_mask.RowIsValid(row_idx))) { + if (CanYouCastIt(vector_data[row_idx], sql_type, sniffing_state_machine.dialect_options, + !null_mask.RowIsValid(row_idx))) { break; } else { if (row_idx != start_idx_detection && cur_top_candidate == LogicalType::BOOLEAN) { diff --git a/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp b/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp index a47ebb704dbc..cbceddd9f0b4 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp @@ -165,8 +165,8 @@ class CSVSniffer { void DetectDateAndTimeStampFormats(CSVStateMachine &candidate, const LogicalType &sql_type, const string &separator, string_t &dummy_val); //! If a string_t value can be cast to a type - bool IsCasteable(const string_t value, const LogicalType &type, const DialectOptions &dialect_options, - const bool is_null); + bool CanYouCastIt(const string_t value, const LogicalType &type, const DialectOptions &dialect_options, + const bool is_null); //! Variables for Type Detection //! Format Candidates for Date and Timestamp Types From b25f3c6607e406de7bd9529eb3d8e45c9dba2f9b Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Fri, 19 Apr 2024 12:29:37 +0200 Subject: [PATCH 582/603] more changes to pushdown window partitions and make format-fix --- .../duckdb/optimizer/filter_pushdown.hpp | 2 +- src/optimizer/pushdown/pushdown_window.cpp | 58 +++++++------------ .../pushdown_window_partition_filter.test | 27 ++++++++- 3 files changed, 47 insertions(+), 40 deletions(-) diff --git a/src/include/duckdb/optimizer/filter_pushdown.hpp b/src/include/duckdb/optimizer/filter_pushdown.hpp index f569c2ad362d..421c9209427e 100644 --- a/src/include/duckdb/optimizer/filter_pushdown.hpp +++ b/src/include/duckdb/optimizer/filter_pushdown.hpp @@ -94,7 +94,7 @@ class FilterPushdown { void PushFilters(); bool CanPushdownFilter(vector window_exprs_partition_bindings, - vector bindings); + const vector &bindings); FilterCombiner combiner; }; diff --git a/src/optimizer/pushdown/pushdown_window.cpp b/src/optimizer/pushdown/pushdown_window.cpp index 25d83a83c7f2..4733b0531419 100644 --- a/src/optimizer/pushdown/pushdown_window.cpp +++ b/src/optimizer/pushdown/pushdown_window.cpp @@ -8,48 +8,35 @@ namespace duckdb { -// if a filter expression is on one of the bindings in the partition set. -static bool FilterIsOnPartition(column_binding_set_t partition_bindings, Expression &filter_expression) { - bool filter_is_on_partition = false; - ExpressionIterator::EnumerateChildren(filter_expression, [&](unique_ptr &child) { - switch (child->type) { - case ExpressionType::BOUND_COLUMN_REF: { - auto &col_ref = child->Cast(); - if (partition_bindings.find(col_ref.binding) != partition_bindings.end()) { - filter_is_on_partition = true; - // if the filter has more columns, break to make sure the other - // columns of the filter are also partitioned on. +using Filter = FilterPushdown::Filter; + +bool FilterPushdown::CanPushdownFilter(vector window_exprs_partition_bindings, + const vector &bindings) { + auto filter_on_all_partitions = true; + for (auto &partition_binding_set : window_exprs_partition_bindings) { + auto filter_on_binding_set = true; + for (auto &binding : bindings) { + if (partition_binding_set.find(binding) == partition_binding_set.end()) { + filter_on_binding_set = false; break; } - // the filter is not on a column that is partitioned. immediately return. - filter_is_on_partition = false; - return; - } - // for simplicity, we don't push down filters that are functions. - // CONJUNCTION_AND filters are also not pushed down - case ExpressionType::BOUND_FUNCTION: - case ExpressionType::CONJUNCTION_AND: { - filter_is_on_partition = false; - return; } - default: + filter_on_all_partitions = filter_on_all_partitions && filter_on_binding_set; + if (!filter_on_all_partitions) { break; } - }); - return filter_is_on_partition; + } + return filter_on_all_partitions; } unique_ptr FilterPushdown::PushdownWindow(unique_ptr op) { D_ASSERT(op->type == LogicalOperatorType::LOGICAL_WINDOW); auto &window = op->Cast(); - // go into expressions, then look into the partitions. - // if the filter applies to a partition in each window expression then you can push the filter - // into the children. FilterPushdown pushdown(optimizer); - vector> leftover_filters; - // First loop through the window expressions. If all window expressions - // have partitions, maybe we can push down a filter through the partition. + // 1. Loop throguh the expressions, find the window expressions and investigate the partitions + // if a filter applies to a partition in each window expression then you can push the filter + // into the children. vector window_exprs_partition_bindings; for (auto &expr : window.expressions) { if (expr->expression_class != ExpressionClass::BOUND_WINDOW) { @@ -84,17 +71,14 @@ unique_ptr FilterPushdown::PushdownWindow(unique_ptr> leftover_filters; // Loop through the filters. If a filter is on a partition in every window expression // it can be pushed down. for (idx_t i = 0; i < filters.size(); i++) { - auto can_pushdown_filter = true; - // the filter must be on all partition bindings - for (auto &partition_bindings : window_exprs_partition_bindings) { - can_pushdown_filter = - can_pushdown_filter && FilterIsOnPartition(partition_bindings, *filters.at(i)->filter); - } - if (can_pushdown_filter) { + vector bindings; + ExtractFilterBindings(*filters.at(i)->filter, bindings); + if (CanPushdownFilter(window_exprs_partition_bindings, bindings)) { pushdown.filters.push_back(std::move(filters.at(i))); } else { leftover_filters.push_back(std::move(filters.at(i))); diff --git a/test/optimizer/pushdown/pushdown_window_partition_filter.test b/test/optimizer/pushdown/pushdown_window_partition_filter.test index 83229d8bcfe7..24e7ec60c875 100644 --- a/test/optimizer/pushdown/pushdown_window_partition_filter.test +++ b/test/optimizer/pushdown/pushdown_window_partition_filter.test @@ -6,7 +6,17 @@ statement ok pragma enable_verification; statement ok -create table t1 as from VALUES ('A', 1), ('B', 3), ('C', 12), ('A', 5), ('B', 8), ('C', 9), ('A', 10), ('B', 20), ('C', 3) t(a, b); +create table t1 as from VALUES + ('A', 1), + ('B', 3), + ('C', 12), + ('A', 5), + ('B', 8), + ('C', 9), + ('A', 10), + ('B', 20), + ('C', 3) + t(a, b); statement ok pragma explain_output=OPTIMIZED_ONLY @@ -17,7 +27,6 @@ create view window_with_filter as select a, b, LEAD(b) OVER (partition by a) as statement ok create view window_no_filter as select a, b, LEAD(b) OVER (partition by a) as lead from t1; - query III no_sort result_1 select * from window_with_filter where a != 'C' order by a; ---- @@ -127,4 +136,18 @@ explain select * from window_no_filter where a != 'C' order by a; ---- logical_opt :.*WINDOW.*FILTER.* +statement ok +create table t2 as select range a, range%50 b, range%25 c from range(500); + +# second window expression is not paritioned on b, so filter expression cannot be +# pushed down +query II +explain select * from (select a, b, c, sum(a) OVER (PARTITION BY b, c), sum(b) OVER (PARTITION BY a, c) from t2) where b > 25; +---- +logical_opt :.*FILTER.*WINDOW.* + +query II +explain select * from (select a, b, c, sum(a) OVER (PARTITION BY b, c), sum(b) OVER (PARTITION BY a, c) from t2) where c = 20; +---- +logical_opt :.*WINDOW.*c=20.* From 1b6d476e7adb9f6d3f87cc5816182b4b99d9c207 Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Fri, 19 Apr 2024 12:43:04 +0200 Subject: [PATCH 583/603] move CanPushdown Filter function --- src/include/duckdb/optimizer/filter_pushdown.hpp | 3 --- src/optimizer/pushdown/pushdown_window.cpp | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/include/duckdb/optimizer/filter_pushdown.hpp b/src/include/duckdb/optimizer/filter_pushdown.hpp index 421c9209427e..a2aa1e2e5b07 100644 --- a/src/include/duckdb/optimizer/filter_pushdown.hpp +++ b/src/include/duckdb/optimizer/filter_pushdown.hpp @@ -93,9 +93,6 @@ class FilterPushdown { //! if there are filters in this FilterPushdown node, push them into the combiner void PushFilters(); - bool CanPushdownFilter(vector window_exprs_partition_bindings, - const vector &bindings); - FilterCombiner combiner; }; diff --git a/src/optimizer/pushdown/pushdown_window.cpp b/src/optimizer/pushdown/pushdown_window.cpp index 4733b0531419..5c860783df5a 100644 --- a/src/optimizer/pushdown/pushdown_window.cpp +++ b/src/optimizer/pushdown/pushdown_window.cpp @@ -10,7 +10,7 @@ namespace duckdb { using Filter = FilterPushdown::Filter; -bool FilterPushdown::CanPushdownFilter(vector window_exprs_partition_bindings, +bool CanPushdownFilter(vector window_exprs_partition_bindings, const vector &bindings) { auto filter_on_all_partitions = true; for (auto &partition_binding_set : window_exprs_partition_bindings) { From 1eddb21b033817b67c4f56ff3d332c621e79fdaf Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Fri, 19 Apr 2024 12:46:49 +0200 Subject: [PATCH 584/603] make format-fix --- src/optimizer/pushdown/pushdown_window.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/optimizer/pushdown/pushdown_window.cpp b/src/optimizer/pushdown/pushdown_window.cpp index 5c860783df5a..be4fe70c2afe 100644 --- a/src/optimizer/pushdown/pushdown_window.cpp +++ b/src/optimizer/pushdown/pushdown_window.cpp @@ -11,7 +11,7 @@ namespace duckdb { using Filter = FilterPushdown::Filter; bool CanPushdownFilter(vector window_exprs_partition_bindings, - const vector &bindings) { + const vector &bindings) { auto filter_on_all_partitions = true; for (auto &partition_binding_set : window_exprs_partition_bindings) { auto filter_on_binding_set = true; From 2bbebda1e09ef81969d9814a3521b30a74048ae6 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Fri, 19 Apr 2024 13:11:19 +0200 Subject: [PATCH 585/603] Add missing includes --- src/execution/operator/persistent/physical_delete.cpp | 1 + .../duckdb/execution/operator/persistent/physical_delete.hpp | 1 + .../duckdb/execution/operator/persistent/physical_update.hpp | 1 + src/include/duckdb/planner/operator/logical_delete.hpp | 1 + src/include/duckdb/planner/operator/logical_update.hpp | 1 + src/storage/wal_replay.cpp | 1 + 6 files changed, 6 insertions(+) diff --git a/src/execution/operator/persistent/physical_delete.cpp b/src/execution/operator/persistent/physical_delete.cpp index bb2b1a76057a..83c92b67f655 100644 --- a/src/execution/operator/persistent/physical_delete.cpp +++ b/src/execution/operator/persistent/physical_delete.cpp @@ -7,6 +7,7 @@ #include "duckdb/storage/table/scan_state.hpp" #include "duckdb/transaction/duck_transaction.hpp" #include "duckdb/storage/table/delete_state.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" namespace duckdb { diff --git a/src/include/duckdb/execution/operator/persistent/physical_delete.hpp b/src/include/duckdb/execution/operator/persistent/physical_delete.hpp index 740db21ad759..e9c9c24c0ce0 100644 --- a/src/include/duckdb/execution/operator/persistent/physical_delete.hpp +++ b/src/include/duckdb/execution/operator/persistent/physical_delete.hpp @@ -9,6 +9,7 @@ #pragma once #include "duckdb/execution/physical_operator.hpp" +#include "duckdb/planner/bound_constraint.hpp" namespace duckdb { class DataTable; diff --git a/src/include/duckdb/execution/operator/persistent/physical_update.hpp b/src/include/duckdb/execution/operator/persistent/physical_update.hpp index b064c8b2e98f..9556b48c92b6 100644 --- a/src/include/duckdb/execution/operator/persistent/physical_update.hpp +++ b/src/include/duckdb/execution/operator/persistent/physical_update.hpp @@ -10,6 +10,7 @@ #include "duckdb/execution/physical_operator.hpp" #include "duckdb/planner/expression.hpp" +#include "duckdb/planner/bound_constraint.hpp" namespace duckdb { class DataTable; diff --git a/src/include/duckdb/planner/operator/logical_delete.hpp b/src/include/duckdb/planner/operator/logical_delete.hpp index 9a243a1f730b..513370ea254b 100644 --- a/src/include/duckdb/planner/operator/logical_delete.hpp +++ b/src/include/duckdb/planner/operator/logical_delete.hpp @@ -9,6 +9,7 @@ #pragma once #include "duckdb/planner/logical_operator.hpp" +#include "duckdb/planner/bound_constraint.hpp" namespace duckdb { class TableCatalogEntry; diff --git a/src/include/duckdb/planner/operator/logical_update.hpp b/src/include/duckdb/planner/operator/logical_update.hpp index e0356419b0d2..587341ab00c2 100644 --- a/src/include/duckdb/planner/operator/logical_update.hpp +++ b/src/include/duckdb/planner/operator/logical_update.hpp @@ -9,6 +9,7 @@ #pragma once #include "duckdb/planner/logical_operator.hpp" +#include "duckdb/planner/bound_constraint.hpp" namespace duckdb { class TableCatalogEntry; diff --git a/src/storage/wal_replay.cpp b/src/storage/wal_replay.cpp index ddd0e7422187..f0d78083e422 100644 --- a/src/storage/wal_replay.cpp +++ b/src/storage/wal_replay.cpp @@ -25,6 +25,7 @@ #include "duckdb/common/checksum.hpp" #include "duckdb/execution/index/index_type_set.hpp" #include "duckdb/execution/index/art/art.hpp" +#include "duckdb/storage/table/delete_state.hpp" namespace duckdb { From 239a1d11a5143b8c9aa9ce68fbf09e5b0284a43d Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 19 Apr 2024 13:22:04 +0200 Subject: [PATCH 586/603] Basics of implicit time/timestamp cast --- .../scanner/string_value_scanner.cpp | 43 ++++++++++--------- src/function/scalar/strftime_format.cpp | 23 +++++++--- .../function/scalar/strftime_format.hpp | 4 ++ 3 files changed, 43 insertions(+), 27 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 9e6271c82818..438fb1b8f0c9 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -215,16 +215,32 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size false, state_machine.options.decimal_separator[0]); break; case LogicalTypeId::DATE: { - idx_t pos; - bool special; - success = Date::TryConvertDate(value_ptr, size, pos, - static_cast(vector_ptr[chunk_col_id])[number_of_rows], special, false); + if (!state_machine.dialect_options.date_format.find(LogicalTypeId::DATE)->second.GetValue().Empty()) { + string error_message; + success = state_machine.dialect_options.date_format.find(LogicalTypeId::DATE) + ->second.GetValue() + .TryParseDate(value_ptr, size, + static_cast(vector_ptr[chunk_col_id])[number_of_rows], error_message); + } else { + idx_t pos; + bool special; + success = Date::TryConvertDate( + value_ptr, size, pos, static_cast(vector_ptr[chunk_col_id])[number_of_rows], special, false); + } break; } case LogicalTypeId::TIMESTAMP: { - success = Timestamp::TryConvertTimestamp( - value_ptr, size, static_cast(vector_ptr[chunk_col_id])[number_of_rows]) == - TimestampCastResult::SUCCESS; + if (!state_machine.dialect_options.date_format.find(LogicalTypeId::TIMESTAMP)->second.GetValue().Empty()) { + timestamp_t result; + string error_message; + // success = state_machine.dialect_options.date_format.find(LogicalTypeId::TIMESTAMP) + // ->second.GetValue() + // .TryParseTimestamp(value, result, error_message); + } else { + success = Timestamp::TryConvertTimestamp( + value_ptr, size, static_cast(vector_ptr[chunk_col_id])[number_of_rows]) == + TimestampCastResult::SUCCESS; + } break; } default: { @@ -1185,7 +1201,6 @@ bool StringValueScanner::CanDirectlyCast(const LogicalType &type, const map> &format_options) { switch (type.id()) { - // All Integers (Except HugeInt) case LogicalTypeId::TINYINT: case LogicalTypeId::SMALLINT: case LogicalTypeId::INTEGER: @@ -1196,20 +1211,8 @@ bool StringValueScanner::CanDirectlyCast(const LogicalType &type, case LogicalTypeId::UBIGINT: case LogicalTypeId::DOUBLE: case LogicalTypeId::FLOAT: - return true; case LogicalTypeId::DATE: - // We can only internally cast YYYY-MM-DD - if (format_options.at(LogicalTypeId::DATE).GetValue().format_specifier == "%Y-%m-%d") { - return true; - } else { - return false; - } case LogicalTypeId::TIMESTAMP: - if (format_options.at(LogicalTypeId::TIMESTAMP).GetValue().format_specifier == "%Y-%m-%d %H:%M:%S") { - return true; - } else { - return false; - } case LogicalType::VARCHAR: return true; default: diff --git a/src/function/scalar/strftime_format.cpp b/src/function/scalar/strftime_format.cpp index f0ec24920a59..4a8416f6e8e3 100644 --- a/src/function/scalar/strftime_format.cpp +++ b/src/function/scalar/strftime_format.cpp @@ -740,8 +740,7 @@ int32_t StrpTimeFormat::TryParseCollection(const char *data, idx_t &pos, idx_t s return -1; } -//! Parses a timestamp using the given specifier -bool StrpTimeFormat::Parse(string_t str, ParseResult &result) const { +bool StrpTimeFormat::Parse(const char *data, size_t size, ParseResult &result) const { auto &result_data = result.data; auto &error_message = result.error_message; auto &error_position = result.error_position; @@ -755,15 +754,11 @@ bool StrpTimeFormat::Parse(string_t str, ParseResult &result) const { result_data[5] = 0; result_data[6] = 0; result_data[7] = 0; - - auto data = str.GetData(); - idx_t size = str.GetSize(); // skip leading spaces while (StringUtil::CharacterIsSpace(*data)) { data++; size--; } - // Check for specials // Precheck for alphas for performance. idx_t pos = 0; @@ -1067,7 +1062,6 @@ bool StrpTimeFormat::Parse(string_t str, ParseResult &result) const { case StrTimeSpecifier::YEAR_DECIMAL: // Just validate, don't use break; - break; case StrTimeSpecifier::WEEKDAY_DECIMAL: // First offset specifier offset_specifier = specifiers[i]; @@ -1324,6 +1318,13 @@ bool StrpTimeFormat::Parse(string_t str, ParseResult &result) const { return true; } +//! Parses a timestamp using the given specifier +bool StrpTimeFormat::Parse(string_t str, ParseResult &result) const { + auto data = str.GetData(); + idx_t size = str.GetSize(); + return Parse(data, size, result); +} + StrpTimeFormat::ParseResult StrpTimeFormat::Parse(const string &format_string, const string &text) { StrpTimeFormat format; format.format_specifier = format_string; @@ -1413,6 +1414,14 @@ bool StrpTimeFormat::TryParseDate(string_t input, date_t &result, string &error_ return parse_result.TryToDate(result); } +bool StrpTimeFormat::TryParseDate(const char *data, size_t size, date_t &result, string &error_message) const { + ParseResult parse_result; + if (!Parse(data, size, parse_result)) { + return false; + } + return parse_result.TryToDate(result); +} + bool StrpTimeFormat::TryParseTime(string_t input, dtime_t &result, string &error_message) const { ParseResult parse_result; if (!Parse(input, parse_result)) { diff --git a/src/include/duckdb/function/scalar/strftime_format.hpp b/src/include/duckdb/function/scalar/strftime_format.hpp index a7b3addde7d2..f65e10131d5e 100644 --- a/src/include/duckdb/function/scalar/strftime_format.hpp +++ b/src/include/duckdb/function/scalar/strftime_format.hpp @@ -161,6 +161,10 @@ struct StrpTimeFormat : public StrTimeFormat { // NOLINT: work-around bug in cla DUCKDB_API bool Parse(string_t str, ParseResult &result) const; + DUCKDB_API bool Parse(const char *data, size_t size, ParseResult &result) const; + + DUCKDB_API bool TryParseDate(const char *data, size_t size, date_t &result, string &error_message) const; + DUCKDB_API bool TryParseDate(string_t str, date_t &result, string &error_message) const; DUCKDB_API bool TryParseTime(string_t str, dtime_t &result, string &error_message) const; DUCKDB_API bool TryParseTimestamp(string_t str, timestamp_t &result, string &error_message) const; From d78effa2ae09efb373e1b038f289004a42dd81c3 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Fri, 19 Apr 2024 13:32:41 +0200 Subject: [PATCH 587/603] Extension fixes --- .github/config/out_of_tree_extensions.cmake | 1 + .../postgres_scanner/shared_ptr.patch | 27 +++++++++++++++++++ .../sqlite_scanner/binder_update.patch | 27 +++++++++++++++++++ 3 files changed, 55 insertions(+) create mode 100644 .github/patches/extensions/sqlite_scanner/binder_update.patch diff --git a/.github/config/out_of_tree_extensions.cmake b/.github/config/out_of_tree_extensions.cmake index 7b489f8c14ff..159ed15ebe6f 100644 --- a/.github/config/out_of_tree_extensions.cmake +++ b/.github/config/out_of_tree_extensions.cmake @@ -94,6 +94,7 @@ duckdb_extension_load(sqlite_scanner ${STATIC_LINK_SQLITE} LOAD_TESTS GIT_URL https://github.com/duckdb/sqlite_scanner GIT_TAG 091197efb34579c7195afa43dfb5925023c915c0 + APPLY_PATCHES ) ################# SUBSTRAIT diff --git a/.github/patches/extensions/postgres_scanner/shared_ptr.patch b/.github/patches/extensions/postgres_scanner/shared_ptr.patch index 98d351393f23..e94920829735 100644 --- a/.github/patches/extensions/postgres_scanner/shared_ptr.patch +++ b/.github/patches/extensions/postgres_scanner/shared_ptr.patch @@ -58,6 +58,19 @@ index e20a803..4fe45f6 100644 namespace duckdb { struct DropInfo; +diff --git a/src/include/storage/postgres_table_entry.hpp b/src/include/storage/postgres_table_entry.hpp +index d96dfad..529c234 100644 +--- a/src/include/storage/postgres_table_entry.hpp ++++ b/src/include/storage/postgres_table_entry.hpp +@@ -50,7 +50,7 @@ public: + + TableStorageInfo GetStorageInfo(ClientContext &context) override; + +- void BindUpdateConstraints(LogicalGet &get, LogicalProjection &proj, LogicalUpdate &update, ++ void BindUpdateConstraints(Binder &binder, LogicalGet &get, LogicalProjection &proj, LogicalUpdate &update, + ClientContext &context) override; + + //! Get the copy format (text or binary) that should be used when writing data to this table diff --git a/src/postgres_binary_copy.cpp b/src/postgres_binary_copy.cpp index f0d86a3..4c89c3f 100644 --- a/src/postgres_binary_copy.cpp @@ -212,3 +225,17 @@ index 93c3f28..cd3b46f 100644 namespace duckdb { +diff --git a/src/storage/postgres_table_entry.cpp b/src/storage/postgres_table_entry.cpp +index d791678..7ba1ad6 100644 +--- a/src/storage/postgres_table_entry.cpp ++++ b/src/storage/postgres_table_entry.cpp +@@ -31,7 +31,8 @@ unique_ptr PostgresTableEntry::GetStatistics(ClientContext &cont + return nullptr; + } + +-void PostgresTableEntry::BindUpdateConstraints(LogicalGet &, LogicalProjection &, LogicalUpdate &, ClientContext &) { ++void PostgresTableEntry::BindUpdateConstraints(Binder &binder, LogicalGet &, LogicalProjection &, LogicalUpdate &, ++ ClientContext &) { + } + + TableFunction PostgresTableEntry::GetScanFunction(ClientContext &context, unique_ptr &bind_data) { diff --git a/.github/patches/extensions/sqlite_scanner/binder_update.patch b/.github/patches/extensions/sqlite_scanner/binder_update.patch new file mode 100644 index 000000000000..7973f54281e5 --- /dev/null +++ b/.github/patches/extensions/sqlite_scanner/binder_update.patch @@ -0,0 +1,27 @@ +diff --git a/src/include/storage/sqlite_table_entry.hpp b/src/include/storage/sqlite_table_entry.hpp +index 6e64d55..b08319b 100644 +--- a/src/include/storage/sqlite_table_entry.hpp ++++ b/src/include/storage/sqlite_table_entry.hpp +@@ -25,7 +25,7 @@ public: + + TableStorageInfo GetStorageInfo(ClientContext &context) override; + +- void BindUpdateConstraints(LogicalGet &get, LogicalProjection &proj, LogicalUpdate &update, ++ void BindUpdateConstraints(Binder &binder, LogicalGet &get, LogicalProjection &proj, LogicalUpdate &update, + ClientContext &context) override; + }; + +diff --git a/src/storage/sqlite_table_entry.cpp b/src/storage/sqlite_table_entry.cpp +index fadbb39..47378b0 100644 +--- a/src/storage/sqlite_table_entry.cpp ++++ b/src/storage/sqlite_table_entry.cpp +@@ -16,7 +16,8 @@ unique_ptr SQLiteTableEntry::GetStatistics(ClientContext &contex + return nullptr; + } + +-void SQLiteTableEntry::BindUpdateConstraints(LogicalGet &, LogicalProjection &, LogicalUpdate &, ClientContext &) { ++void SQLiteTableEntry::BindUpdateConstraints(Binder &, LogicalGet &, LogicalProjection &, LogicalUpdate &, ++ ClientContext &) { + } + + TableFunction SQLiteTableEntry::GetScanFunction(ClientContext &context, unique_ptr &bind_data) { From 09662ad5def7811a1bc5da642545b86178532de7 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 19 Apr 2024 13:38:01 +0200 Subject: [PATCH 588/603] Finishing up the work of implicit casting and fixing up error messages --- .../scanner/string_value_scanner.cpp | 18 ++++++++---------- src/function/scalar/strftime_format.cpp | 10 +++++++++- .../duckdb/function/scalar/strftime_format.hpp | 3 ++- test/sql/copy/csv/csv_hive_filename_union.test | 2 +- .../csv/rejects/csv_rejects_flush_cast.test | 4 ++-- .../test_multiple_errors_same_line.test | 10 +++++++--- test/sql/copy/csv/timestamp_with_tz.test | 2 +- 7 files changed, 30 insertions(+), 19 deletions(-) diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 438fb1b8f0c9..473a07a93c4c 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -216,11 +216,10 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size break; case LogicalTypeId::DATE: { if (!state_machine.dialect_options.date_format.find(LogicalTypeId::DATE)->second.GetValue().Empty()) { - string error_message; - success = state_machine.dialect_options.date_format.find(LogicalTypeId::DATE) - ->second.GetValue() - .TryParseDate(value_ptr, size, - static_cast(vector_ptr[chunk_col_id])[number_of_rows], error_message); + success = + state_machine.dialect_options.date_format.find(LogicalTypeId::DATE) + ->second.GetValue() + .TryParseDate(value_ptr, size, static_cast(vector_ptr[chunk_col_id])[number_of_rows]); } else { idx_t pos; bool special; @@ -231,11 +230,10 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size } case LogicalTypeId::TIMESTAMP: { if (!state_machine.dialect_options.date_format.find(LogicalTypeId::TIMESTAMP)->second.GetValue().Empty()) { - timestamp_t result; - string error_message; - // success = state_machine.dialect_options.date_format.find(LogicalTypeId::TIMESTAMP) - // ->second.GetValue() - // .TryParseTimestamp(value, result, error_message); + success = state_machine.dialect_options.date_format.find(LogicalTypeId::TIMESTAMP) + ->second.GetValue() + .TryParseTimestamp(value_ptr, size, + static_cast(vector_ptr[chunk_col_id])[number_of_rows]); } else { success = Timestamp::TryConvertTimestamp( value_ptr, size, static_cast(vector_ptr[chunk_col_id])[number_of_rows]) == diff --git a/src/function/scalar/strftime_format.cpp b/src/function/scalar/strftime_format.cpp index 4a8416f6e8e3..f8ff39c37247 100644 --- a/src/function/scalar/strftime_format.cpp +++ b/src/function/scalar/strftime_format.cpp @@ -1414,7 +1414,7 @@ bool StrpTimeFormat::TryParseDate(string_t input, date_t &result, string &error_ return parse_result.TryToDate(result); } -bool StrpTimeFormat::TryParseDate(const char *data, size_t size, date_t &result, string &error_message) const { +bool StrpTimeFormat::TryParseDate(const char *data, size_t size, date_t &result) const { ParseResult parse_result; if (!Parse(data, size, parse_result)) { return false; @@ -1440,4 +1440,12 @@ bool StrpTimeFormat::TryParseTimestamp(string_t input, timestamp_t &result, stri return parse_result.TryToTimestamp(result); } +bool StrpTimeFormat::TryParseTimestamp(const char *data, size_t size, timestamp_t &result) const { + ParseResult parse_result; + if (!Parse(data, size, parse_result)) { + return false; + } + return parse_result.TryToTimestamp(result); +} + } // namespace duckdb diff --git a/src/include/duckdb/function/scalar/strftime_format.hpp b/src/include/duckdb/function/scalar/strftime_format.hpp index f65e10131d5e..a538db4255b3 100644 --- a/src/include/duckdb/function/scalar/strftime_format.hpp +++ b/src/include/duckdb/function/scalar/strftime_format.hpp @@ -163,7 +163,8 @@ struct StrpTimeFormat : public StrTimeFormat { // NOLINT: work-around bug in cla DUCKDB_API bool Parse(const char *data, size_t size, ParseResult &result) const; - DUCKDB_API bool TryParseDate(const char *data, size_t size, date_t &result, string &error_message) const; + DUCKDB_API bool TryParseDate(const char *data, size_t size, date_t &result) const; + DUCKDB_API bool TryParseTimestamp(const char *data, size_t size, timestamp_t &result) const; DUCKDB_API bool TryParseDate(string_t str, date_t &result, string &error_message) const; DUCKDB_API bool TryParseTime(string_t str, dtime_t &result, string &error_message) const; diff --git a/test/sql/copy/csv/csv_hive_filename_union.test b/test/sql/copy/csv/csv_hive_filename_union.test index 79f84efcdfc0..ff145d929f77 100644 --- a/test/sql/copy/csv/csv_hive_filename_union.test +++ b/test/sql/copy/csv/csv_hive_filename_union.test @@ -56,7 +56,7 @@ xxx 42 statement error select * from read_csv_auto(['data/csv/hive-partitioning/mismatching_contents/part=1/test.csv', 'data/csv/hive-partitioning/mismatching_contents/part=2/test.csv']) order by 1 ---- -date field value out of range +Error when converting column "c". Could not convert string "world" to 'DATE' query III select a, b, c from read_csv_auto('data/csv/hive-partitioning/mismatching_contents/*/*.csv', UNION_BY_NAME=1) order by 2 NULLS LAST diff --git a/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test b/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test index ba48d9fe2a99..09daa735d6eb 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test +++ b/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test @@ -20,6 +20,6 @@ DATE VARCHAR 2811 query IIIIIIIII SELECT * EXCLUDE (scan_id) FROM reject_errors order by all; ---- -0 439 6997 NULL 1 a CAST B, bla Error when converting column "a". Could not parse string "B" according to format specifier "%d-%m-%Y" -0 2813 44972 NULL 1 a CAST c, bla Error when converting column "a". Could not parse string "c" according to format specifier "%d-%m-%Y" +0 439 6997 6997 1 a CAST B, bla Error when converting column "a". Could not convert string "B" to 'DATE' +0 2813 44972 44972 1 a CAST c, bla Error when converting column "a". Could not convert string "c" to 'DATE' diff --git a/test/sql/copy/csv/rejects/test_multiple_errors_same_line.test b/test/sql/copy/csv/rejects/test_multiple_errors_same_line.test index 6d9f1fcb5baa..85c3140097bc 100644 --- a/test/sql/copy/csv/rejects/test_multiple_errors_same_line.test +++ b/test/sql/copy/csv/rejects/test_multiple_errors_same_line.test @@ -61,8 +61,8 @@ oogie boogie 3 2023-01-02 2023-01-03 query IIIIIIIII rowsort SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; ---- -0 4 110 NULL 3 current_day CAST oogie boogie,3, bla_2, bla_1 Error when converting column "current_day". date field value out of range: " bla_2", expected format is (YYYY-MM-DD) -0 4 110 NULL 4 tomorrow CAST oogie boogie,3, bla_2, bla_1 Error when converting column "tomorrow". date field value out of range: " bla_1", expected format is (YYYY-MM-DD) +0 4 110 125 3 current_day CAST oogie boogie,3, bla_2, bla_1 Error when converting column "current_day". Could not convert string " bla_2" to 'DATE' +0 4 110 132 4 tomorrow CAST oogie boogie,3, bla_2, bla_1 Error when converting column "tomorrow". Could not convert string " bla_1" to 'DATE' statement ok DROP TABLE reject_errors; @@ -82,6 +82,7 @@ oogie boogie 3 2023-01-02 5 query IIIIIIIII rowsort SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; ---- +0 4 89 104 3 current_day CAST oogie boogie,3, bla_2, bla_1 Error when converting column "current_day". Could not convert string " bla_2" to 'DATE' 0 4 89 111 4 barks CAST oogie boogie,3, bla_2, bla_1 Error when converting column "barks". Could not convert string " bla_1" to 'INTEGER' statement ok @@ -370,10 +371,13 @@ SELECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; ---- 0 4 89 116 4 barks CAST oogie boogie,3, 2023-01-03, bla, 7 Error when converting column "barks". Could not convert string " bla" to 'INTEGER' 0 4 89 120 5 NULL TOO MANY COLUMNS oogie boogie,3, 2023-01-03, bla, 7 Expected Number of Columns: 4 Found: 5 +0 5 124 139 3 current_day CAST oogie boogie,3, bla, bla, 7 Error when converting column "current_day". Could not convert string " bla" to 'DATE' 0 5 124 144 4 barks CAST oogie boogie,3, bla, bla, 7 Error when converting column "barks". Could not convert string " bla" to 'INTEGER' 0 5 124 148 5 NULL TOO MANY COLUMNS oogie boogie,3, bla, bla, 7 Expected Number of Columns: 4 Found: 5 0 6 152 152 1 name UNQUOTED VALUE "oogie boogie"bla,3, 2023-01-04 Value with unterminated quote found. 0 6 152 183 3 barks MISSING COLUMNS "oogie boogie"bla,3, 2023-01-04 Expected Number of Columns: 4 Found: 3 +0 7 184 199 3 current_day CAST oogie boogie,3, bla Error when converting column "current_day". Could not convert string " bla" to 'DATE' 0 7 184 203 3 barks MISSING COLUMNS oogie boogie,3, bla Expected Number of Columns: 4 Found: 3 0 8 204 204 NULL NULL LINE SIZE OVER MAXIMUM oogie boogieoogie boogieoogie boogieoogie boogieoogie boogieoogie boogieoogie boogie,3, bla Maximum line size of 40 bytes exceeded. Actual Size:92 bytes. -0 8 204 295 3 barks MISSING COLUMNS oogie boogieoogie boogieoogie boogieoogie boogieoogie boogieoogie boogieoogie boogie,3, bla Expected Number of Columns: 4 Found: 3 +0 8 204 291 3 current_day CAST oogie boogieoogie boogieoogie boogieoogie boogieoogie boogieoogie boogieoogie boogie,3, bla Error when converting column "current_day". Could not convert string " bla" to 'DATE' +0 8 204 295 3 barks MISSING COLUMNS oogie boogieoogie boogieoogie boogieoogie boogieoogie boogieoogie boogieoogie boogie,3, bla Expected Number of Columns: 4 Found: 3 \ No newline at end of file diff --git a/test/sql/copy/csv/timestamp_with_tz.test b/test/sql/copy/csv/timestamp_with_tz.test index 9478957d222e..b8dbd3de4e74 100644 --- a/test/sql/copy/csv/timestamp_with_tz.test +++ b/test/sql/copy/csv/timestamp_with_tz.test @@ -9,7 +9,7 @@ CREATE TABLE tbl(id int, ts timestamp); statement error COPY tbl FROM 'data/csv/timestamp_with_tz.csv' (HEADER) ---- -timestamp that is not UTC +Error when converting column "ts". Could not convert string "2021-05-25 04:55:03.382494 EST" to 'TIMESTAMP' require icu From 58c44f6d14bd4059cddf2e1f0a828d54a28cf647 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 19 Apr 2024 13:56:39 +0200 Subject: [PATCH 589/603] return properly typed NULL value, fix up python tests and behavior --- src/core_functions/scalar/map/map.cpp | 29 ++++++++++----- .../src/native/python_conversion.cpp | 37 +++++++++++++++++-- tools/pythonpkg/src/pandas/analyzer.cpp | 4 ++ .../fast/pandas/test_df_object_resolution.py | 5 +-- 4 files changed, 60 insertions(+), 15 deletions(-) diff --git a/src/core_functions/scalar/map/map.cpp b/src/core_functions/scalar/map/map.cpp index 62f54cdd25d0..664946267cee 100644 --- a/src/core_functions/scalar/map/map.cpp +++ b/src/core_functions/scalar/map/map.cpp @@ -21,21 +21,28 @@ static void MapFunctionEmptyInput(Vector &result, const idx_t row_count) { result.Verify(row_count); } +static bool MapIsNull(const LogicalType &map) { + D_ASSERT(map.id() == LogicalTypeId::MAP); + auto &key = MapType::KeyType(map); + auto &value = MapType::ValueType(map); + return (key.id() == LogicalTypeId::SQLNULL && value.id() == LogicalTypeId::SQLNULL); +} + static void MapFunction(DataChunk &args, ExpressionState &, Vector &result) { // internal MAP representation // - LIST-vector that contains STRUCTs as child entries // - STRUCTs have exactly two fields, a key-field, and a value-field // - key names are unique + D_ASSERT(result.GetType().id() == LogicalTypeId::MAP); - if (result.GetType().id() == LogicalTypeId::SQLNULL) { + if (MapIsNull(result.GetType())) { auto &validity = FlatVector::Validity(result); validity.SetInvalid(0); result.SetVectorType(VectorType::CONSTANT_VECTOR); return; } - D_ASSERT(result.GetType().id() == LogicalTypeId::MAP); auto row_count = args.size(); // early-out, if no data @@ -162,16 +169,20 @@ static unique_ptr MapBind(ClientContext &, ScalarFunction &bound_f MapVector::EvalMapInvalidReason(MapInvalidReason::INVALID_PARAMS); } - // bind an empty MAP + bool is_null = false; if (arguments.empty()) { - bound_function.return_type = LogicalType::MAP(LogicalTypeId::SQLNULL, LogicalTypeId::SQLNULL); - return make_uniq(bound_function.return_type); + is_null = true; + } + if (!is_null) { + auto key_id = arguments[0]->return_type.id(); + auto value_id = arguments[1]->return_type.id(); + if (key_id == LogicalTypeId::SQLNULL || value_id == LogicalTypeId::SQLNULL) { + is_null = true; + } } - auto key_id = arguments[0]->return_type.id(); - auto value_id = arguments[1]->return_type.id(); - if (key_id == LogicalTypeId::SQLNULL || value_id == LogicalTypeId::SQLNULL) { - bound_function.return_type = LogicalTypeId::SQLNULL; + if (is_null) { + bound_function.return_type = LogicalType::MAP(LogicalTypeId::SQLNULL, LogicalTypeId::SQLNULL); return make_uniq(bound_function.return_type); } diff --git a/tools/pythonpkg/src/native/python_conversion.cpp b/tools/pythonpkg/src/native/python_conversion.cpp index cbdbb9f57f00..133d3fb768f1 100644 --- a/tools/pythonpkg/src/native/python_conversion.cpp +++ b/tools/pythonpkg/src/native/python_conversion.cpp @@ -37,6 +37,20 @@ vector TransformStructKeys(py::handle keys, idx_t size, const LogicalTyp return res; } +static bool IsValidMapComponent(const py::handle &component) { + // The component is either NULL + if (py::none().is(component)) { + return true; + } + if (!py::hasattr(component, "__getitem__")) { + return false; + } + if (!py::hasattr(component, "__len__")) { + return false; + } + return true; +} + bool DictionaryHasMapFormat(const PyDictionary &dict) { if (dict.len != 2) { return false; @@ -51,13 +65,19 @@ bool DictionaryHasMapFormat(const PyDictionary &dict) { return false; } - // Dont check for 'py::list' to allow ducktyping - if (!py::hasattr(keys, "__getitem__") || !py::hasattr(keys, "__len__")) { + if (!IsValidMapComponent(keys)) { return false; } - if (!py::hasattr(values, "__getitem__") || !py::hasattr(values, "__len__")) { + if (!IsValidMapComponent(values)) { return false; } + + // If either of the components is NULL, return early + if (py::none().is(keys) || py::none().is(values)) { + return true; + } + + // Verify that both the keys and values are of the same length auto size = py::len(keys); if (size != py::len(values)) { return false; @@ -91,6 +111,11 @@ Value TransformStructFormatDictionaryToMap(const PyDictionary &dict, const Logic if (target_type.id() != LogicalTypeId::MAP) { throw InvalidInputException("Please provide a valid target type for transform from Python to Value"); } + + if (py::none().is(dict.keys) || py::none().is(dict.values)) { + return Value(LogicalType::MAP(LogicalTypeId::SQLNULL, LogicalTypeId::SQLNULL)); + } + auto size = py::len(dict.keys); D_ASSERT(size == py::len(dict.values)); @@ -130,12 +155,18 @@ Value TransformDictionaryToMap(const PyDictionary &dict, const LogicalType &targ auto keys = dict.values.attr("__getitem__")(0); auto values = dict.values.attr("__getitem__")(1); + if (py::none().is(keys) || py::none().is(values)) { + // Either 'key' or 'value' is None, return early with a NULL value + return Value(LogicalType::MAP(LogicalTypeId::SQLNULL, LogicalTypeId::SQLNULL)); + } + auto key_size = py::len(keys); D_ASSERT(key_size == py::len(values)); if (key_size == 0) { // dict == { 'key': [], 'value': [] } return EmptyMapValue(); } + // dict == { 'key': [ ... ], 'value' : [ ... ] } LogicalType key_target = LogicalTypeId::UNKNOWN; LogicalType value_target = LogicalTypeId::UNKNOWN; diff --git a/tools/pythonpkg/src/pandas/analyzer.cpp b/tools/pythonpkg/src/pandas/analyzer.cpp index 508270894403..660d1fb2b3d2 100644 --- a/tools/pythonpkg/src/pandas/analyzer.cpp +++ b/tools/pythonpkg/src/pandas/analyzer.cpp @@ -331,6 +331,10 @@ LogicalType PandasAnalyzer::DictToMap(const PyDictionary &dict, bool &can_conver auto keys = dict.values.attr("__getitem__")(0); auto values = dict.values.attr("__getitem__")(1); + if (py::none().is(keys) || py::none().is(values)) { + return LogicalType::MAP(LogicalTypeId::SQLNULL, LogicalTypeId::SQLNULL); + } + auto key_type = GetListType(keys, can_convert); if (!can_convert) { return EmptyMap(); diff --git a/tools/pythonpkg/tests/fast/pandas/test_df_object_resolution.py b/tools/pythonpkg/tests/fast/pandas/test_df_object_resolution.py index 0f20f9fe0309..1a07e47fc8f4 100644 --- a/tools/pythonpkg/tests/fast/pandas/test_df_object_resolution.py +++ b/tools/pythonpkg/tests/fast/pandas/test_df_object_resolution.py @@ -324,7 +324,7 @@ def test_map_duplicate(self, pandas, duckdb_cursor): with pytest.raises( duckdb.InvalidInputException, match="Dict->Map conversion failed because 'key' list contains duplicates" ): - converted_col = duckdb_cursor.sql("select * from x").df() + duckdb_cursor.sql("select * from x").show() @pytest.mark.parametrize('pandas', [NumpyPandas(), ArrowPandas()]) def test_map_nullkey(self, pandas, duckdb_cursor): @@ -337,9 +337,8 @@ def test_map_nullkey(self, pandas, duckdb_cursor): @pytest.mark.parametrize('pandas', [NumpyPandas(), ArrowPandas()]) def test_map_nullkeylist(self, pandas, duckdb_cursor): x = pandas.DataFrame([[{'key': None, 'value': None}]]) - # Isn't actually converted to MAP because isinstance(None, list) != True converted_col = duckdb_cursor.sql("select * from x").df() - duckdb_col = duckdb_cursor.sql("SELECT {key: NULL, value: NULL} as '0'").df() + duckdb_col = duckdb_cursor.sql("SELECT MAP(NULL, NULL) as '0'").df() pandas.testing.assert_frame_equal(duckdb_col, converted_col) @pytest.mark.parametrize('pandas', [NumpyPandas(), ArrowPandas()]) From 9f57957c13bbdd798b82d99ce71696d884478cc3 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 19 Apr 2024 13:59:05 +0200 Subject: [PATCH 590/603] Removing 4x sleep function --- benchmark/micro/csv/time_type.benchmark | 0 .../scanner/string_value_scanner.cpp | 18 ++++++++---------- .../csv_scanner/string_value_scanner.hpp | 2 +- 3 files changed, 9 insertions(+), 11 deletions(-) create mode 100644 benchmark/micro/csv/time_type.benchmark diff --git a/benchmark/micro/csv/time_type.benchmark b/benchmark/micro/csv/time_type.benchmark new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 473a07a93c4c..49d761ce6ca9 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -97,6 +97,8 @@ StringValueResult::StringValueResult(CSVStates &states, CSVStateMachine &state_m null_str_ptr[i] = state_machine.options.null_str[i].c_str(); null_str_size[i] = state_machine.options.null_str[i].size(); } + date_format = state_machine.options.dialect_options.date_format.at(LogicalTypeId::DATE).GetValue(); + timestamp_format = state_machine.options.dialect_options.date_format.at(LogicalTypeId::TIMESTAMP).GetValue(); } StringValueResult::~StringValueResult() { @@ -215,11 +217,9 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size false, state_machine.options.decimal_separator[0]); break; case LogicalTypeId::DATE: { - if (!state_machine.dialect_options.date_format.find(LogicalTypeId::DATE)->second.GetValue().Empty()) { - success = - state_machine.dialect_options.date_format.find(LogicalTypeId::DATE) - ->second.GetValue() - .TryParseDate(value_ptr, size, static_cast(vector_ptr[chunk_col_id])[number_of_rows]); + if (!date_format.Empty()) { + success = date_format.TryParseDate(value_ptr, size, + static_cast(vector_ptr[chunk_col_id])[number_of_rows]); } else { idx_t pos; bool special; @@ -229,11 +229,9 @@ void StringValueResult::AddValueToVector(const char *value_ptr, const idx_t size break; } case LogicalTypeId::TIMESTAMP: { - if (!state_machine.dialect_options.date_format.find(LogicalTypeId::TIMESTAMP)->second.GetValue().Empty()) { - success = state_machine.dialect_options.date_format.find(LogicalTypeId::TIMESTAMP) - ->second.GetValue() - .TryParseTimestamp(value_ptr, size, - static_cast(vector_ptr[chunk_col_id])[number_of_rows]); + if (!timestamp_format.Empty()) { + success = timestamp_format.TryParseTimestamp( + value_ptr, size, static_cast(vector_ptr[chunk_col_id])[number_of_rows]); } else { success = Timestamp::TryConvertTimestamp( value_ptr, size, static_cast(vector_ptr[chunk_col_id])[number_of_rows]) == diff --git a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp index 9ad42fd16635..a1fa130e4ffd 100644 --- a/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +++ b/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp @@ -145,7 +145,7 @@ class StringValueResult : public ScannerResult { //! Errors happening in the current line (if any) vector current_errors; - + StrpTimeFormat date_format, timestamp_format; bool sniffing; //! Specialized code for quoted values, makes sure to remove quotes and escapes static inline void AddQuotedValue(StringValueResult &result, const idx_t buffer_pos); From f9ebe008a3ae0475ef829b98fa9fec5402e62c59 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 19 Apr 2024 13:59:42 +0200 Subject: [PATCH 591/603] Add Benchmark --- benchmark/micro/csv/time_type.benchmark | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/benchmark/micro/csv/time_type.benchmark b/benchmark/micro/csv/time_type.benchmark index e69de29bb2d1..98f4a556bb06 100644 --- a/benchmark/micro/csv/time_type.benchmark +++ b/benchmark/micro/csv/time_type.benchmark @@ -0,0 +1,14 @@ +# name: benchmark/micro/csv/time_type.benchmark +# description: Run CSV scan with timestamp and date types +# group: [csv] + +name CSV Read Benchmark with timestamp and date types +group csv + +load +CREATE TABLE t1 AS select '30/07/1992', '30/07/1992 17:15:30'; +insert into t1 select '30/07/1992', '30/07/1992 17:15:30' from range(0,10000000) tbl(i); +COPY t1 TO '${BENCHMARK_DIR}/time_timestamp.csv' (FORMAT CSV, HEADER 0); + +run +SELECT * from read_csv('${BENCHMARK_DIR}/time_timestamp.csv',delim= ',', header = 0) From 788e25787f2083ae855266320b6acc9a70e2f806 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 19 Apr 2024 14:02:38 +0200 Subject: [PATCH 592/603] Add it to csv benchmarks --- .github/regression/csv.csv | 3 ++- .github/regression/micro_extended.csv | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/regression/csv.csv b/.github/regression/csv.csv index 272b7ced2a04..ddfe10bad1e8 100644 --- a/.github/regression/csv.csv +++ b/.github/regression/csv.csv @@ -4,4 +4,5 @@ benchmark/micro/csv/small_csv.benchmark benchmark/micro/csv/null_padding.benchmark benchmark/micro/csv/projection_pushdown.benchmark benchmark/micro/csv/1_byte_values.benchmark -benchmark/micro/csv/16_byte_values.benchmark \ No newline at end of file +benchmark/micro/csv/16_byte_values.benchmark +benchmark/micro/csv/time_type.benchmark \ No newline at end of file diff --git a/.github/regression/micro_extended.csv b/.github/regression/micro_extended.csv index 6973785b4c98..365347ad3ad3 100644 --- a/.github/regression/micro_extended.csv +++ b/.github/regression/micro_extended.csv @@ -86,6 +86,7 @@ benchmark/micro/csv/read.benchmark benchmark/micro/csv/small_csv.benchmark benchmark/micro/csv/sniffer.benchmark benchmark/micro/csv/sniffer_quotes.benchmark +benchmark/micro/csv/time_type.benchmark benchmark/micro/cte/cte.benchmark benchmark/micro/cte/materialized_cte.benchmark benchmark/micro/date/extract_month.benchmark From e13c77e8ea44f1b7d1fc2991c31bf5dda0697a2b Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Fri, 19 Apr 2024 15:55:14 +0200 Subject: [PATCH 593/603] clang tidy fixes and make format fixes --- src/include/duckdb/optimizer/filter_pushdown.hpp | 3 +++ src/optimizer/pushdown/pushdown_aggregate.cpp | 16 ++++++++-------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/include/duckdb/optimizer/filter_pushdown.hpp b/src/include/duckdb/optimizer/filter_pushdown.hpp index a2aa1e2e5b07..1e07b0a6d625 100644 --- a/src/include/duckdb/optimizer/filter_pushdown.hpp +++ b/src/include/duckdb/optimizer/filter_pushdown.hpp @@ -88,6 +88,9 @@ class FilterPushdown { //! Adds a filter to the set of filters. Returns FilterResult::UNSATISFIABLE if the subtree should be stripped, or //! FilterResult::SUCCESS otherwise FilterResult AddFilter(unique_ptr expr); + //! Extract filter bindings to compare them with expressions in an operator and determine if the filter + //! can be pushed down + void ExtractFilterBindings(Expression &expr, vector &bindings); //! Generate filters from the current set of filters stored in the FilterCombiner void GenerateFilters(); //! if there are filters in this FilterPushdown node, push them into the combiner diff --git a/src/optimizer/pushdown/pushdown_aggregate.cpp b/src/optimizer/pushdown/pushdown_aggregate.cpp index 8b4cd94de853..c7341862ea8d 100644 --- a/src/optimizer/pushdown/pushdown_aggregate.cpp +++ b/src/optimizer/pushdown/pushdown_aggregate.cpp @@ -9,14 +9,6 @@ namespace duckdb { using Filter = FilterPushdown::Filter; -static void ExtractFilterBindings(Expression &expr, vector &bindings) { - if (expr.type == ExpressionType::BOUND_COLUMN_REF) { - auto &colref = expr.Cast(); - bindings.push_back(colref.binding); - } - ExpressionIterator::EnumerateChildren(expr, [&](Expression &child) { ExtractFilterBindings(child, bindings); }); -} - static unique_ptr ReplaceGroupBindings(LogicalAggregate &proj, unique_ptr expr) { if (expr->type == ExpressionType::BOUND_COLUMN_REF) { auto &colref = expr->Cast(); @@ -31,6 +23,14 @@ static unique_ptr ReplaceGroupBindings(LogicalAggregate &proj, uniqu return expr; } +void FilterPushdown::ExtractFilterBindings(Expression &expr, vector &bindings) { + if (expr.type == ExpressionType::BOUND_COLUMN_REF) { + auto &colref = expr.Cast(); + bindings.push_back(colref.binding); + } + ExpressionIterator::EnumerateChildren(expr, [&](Expression &child) { ExtractFilterBindings(child, bindings); }); +} + unique_ptr FilterPushdown::PushdownAggregate(unique_ptr op) { D_ASSERT(op->type == LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY); auto &aggr = op->Cast(); From d54e152722a577ec8406f843318601d091d814d0 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 19 Apr 2024 16:12:30 +0200 Subject: [PATCH 594/603] fix compilation --- src/core_functions/scalar/map/map.cpp | 22 ++++++++++++++----- .../types/nested/map/test_map_subscript.test | 3 +++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/core_functions/scalar/map/map.cpp b/src/core_functions/scalar/map/map.cpp index 664946267cee..ab67475d151b 100644 --- a/src/core_functions/scalar/map/map.cpp +++ b/src/core_functions/scalar/map/map.cpp @@ -21,11 +21,21 @@ static void MapFunctionEmptyInput(Vector &result, const idx_t row_count) { result.Verify(row_count); } -static bool MapIsNull(const LogicalType &map) { - D_ASSERT(map.id() == LogicalTypeId::MAP); - auto &key = MapType::KeyType(map); - auto &value = MapType::ValueType(map); - return (key.id() == LogicalTypeId::SQLNULL && value.id() == LogicalTypeId::SQLNULL); +static bool MapIsNull(DataChunk &chunk) { + if (chunk.data.empty()) { + return false; + } + D_ASSERT(chunk.data.size() == 2); + auto &keys = chunk.data[0]; + auto &values = chunk.data[1]; + + if (keys.GetType().id() == LogicalTypeId::SQLNULL) { + return true; + } + if (values.GetType().id() == LogicalTypeId::SQLNULL) { + return true; + } + return false; } static void MapFunction(DataChunk &args, ExpressionState &, Vector &result) { @@ -36,7 +46,7 @@ static void MapFunction(DataChunk &args, ExpressionState &, Vector &result) { // - key names are unique D_ASSERT(result.GetType().id() == LogicalTypeId::MAP); - if (MapIsNull(result.GetType())) { + if (MapIsNull(args)) { auto &validity = FlatVector::Validity(result); validity.SetInvalid(0); result.SetVectorType(VectorType::CONSTANT_VECTOR); diff --git a/test/sql/types/nested/map/test_map_subscript.test b/test/sql/types/nested/map/test_map_subscript.test index f75482857dad..8ad48d29e48b 100644 --- a/test/sql/types/nested/map/test_map_subscript.test +++ b/test/sql/types/nested/map/test_map_subscript.test @@ -2,6 +2,9 @@ # description: Test cardinality function for maps # group: [map] +statement ok +pragma enable_verification + # Single element on map query I select m[1] from (select MAP(LIST_VALUE(1, 2, 3, 4),LIST_VALUE(10, 9, 8, 7)) as m) as T From 82f35baa7cf2d37ba384fcdcca4da0e47e4abc92 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 19 Apr 2024 16:36:50 +0200 Subject: [PATCH 595/603] fixing flush error message --- data/csv/rejects/flush.csv | 0 .../scanner/string_value_scanner.cpp | 13 ++++- .../csv/rejects/csv_rejects_flush_cast.test | 5 +- .../rejects/csv_rejects_flush_message.test | 27 +++++++++++ .../test_multiple_errors_same_line.test | 4 +- .../interquery/test_concurrent_index.cpp | 48 +++++++++---------- 6 files changed, 66 insertions(+), 31 deletions(-) create mode 100644 data/csv/rejects/flush.csv create mode 100644 test/sql/copy/csv/rejects/csv_rejects_flush_message.test diff --git a/data/csv/rejects/flush.csv b/data/csv/rejects/flush.csv new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp index 9e6271c82818..195bb39368c0 100644 --- a/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +++ b/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp @@ -790,8 +790,12 @@ void StringValueScanner::Flush(DataChunk &insert_chunk) { bool first_nl; auto borked_line = result.line_positions_per_row[line_error].ReconstructCurrentLine(first_nl, result.buffer_handles); + std::ostringstream error; + error << "Could not convert string \"" << parse_vector.GetValue(line_error) << "\" to \'" + << LogicalTypeIdToString(type.id()) << "\'"; + string error_msg = error.str(); auto csv_error = CSVError::CastError( - state_machine->options, csv_file_scan->names[col_idx], error_message, col_idx, borked_line, + state_machine->options, csv_file_scan->names[col_idx], error_msg, col_idx, borked_line, lines_per_batch, result.line_positions_per_row[line_error].begin.GetGlobalPosition(result.result_size, first_nl), -1, result_vector.GetType().id()); @@ -814,8 +818,13 @@ void StringValueScanner::Flush(DataChunk &insert_chunk) { bool first_nl; auto borked_line = result.line_positions_per_row[line_error].ReconstructCurrentLine( first_nl, result.buffer_handles); + std::ostringstream error; + // Casting Error Message + error << "Could not convert string \"" << parse_vector.GetValue(line_error) << "\" to \'" + << LogicalTypeIdToString(type.id()) << "\'"; + string error_msg = error.str(); auto csv_error = CSVError::CastError( - state_machine->options, csv_file_scan->names[col_idx], error_message, col_idx, borked_line, + state_machine->options, csv_file_scan->names[col_idx], error_msg, col_idx, borked_line, lines_per_batch, result.line_positions_per_row[line_error].begin.GetGlobalPosition(result.result_size, first_nl), -1, result_vector.GetType().id()); diff --git a/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test b/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test index ba48d9fe2a99..859fbdb267f6 100644 --- a/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test +++ b/test/sql/copy/csv/rejects/csv_rejects_flush_cast.test @@ -20,6 +20,5 @@ DATE VARCHAR 2811 query IIIIIIIII SELECT * EXCLUDE (scan_id) FROM reject_errors order by all; ---- -0 439 6997 NULL 1 a CAST B, bla Error when converting column "a". Could not parse string "B" according to format specifier "%d-%m-%Y" -0 2813 44972 NULL 1 a CAST c, bla Error when converting column "a". Could not parse string "c" according to format specifier "%d-%m-%Y" - +0 439 6997 NULL 1 a CAST B, bla Error when converting column "a". Could not convert string "B" to 'DATE' +0 2813 44972 NULL 1 a CAST c, bla Error when converting column "a". Could not convert string "NULL" to 'DATE' diff --git a/test/sql/copy/csv/rejects/csv_rejects_flush_message.test b/test/sql/copy/csv/rejects/csv_rejects_flush_message.test new file mode 100644 index 000000000000..cb409c240f23 --- /dev/null +++ b/test/sql/copy/csv/rejects/csv_rejects_flush_message.test @@ -0,0 +1,27 @@ +# name: test/sql/copy/csv/rejects/csv_rejects_flush_message.test +# description: Test that Flush Cast gives reasonable messages +# group: [rejects] + +require skip_reload + +# Test will fail on windows because byte_position is slightly different due to \r\n instead of \n +require notwindows + +query I +SELECT * FROM read_csv( + 'data/csv/rejects/flush.csv', + columns = {'a': 'DECIMAL'}, + store_rejects = true); +---- +1521000.000 +33.439 +-21060000.000 + +query IIIIIIIII +SELECT * EXCLUDE (scan_id) FROM reject_errors order by all; +---- +0 2 10 NULL 1 a CAST "not-a-number" Error when converting column "a". Could not convert string "not-a-number" to 'DECIMAL' +0 3 25 NULL 1 a CAST "-26,9568" Error when converting column "a". Could not convert string "-26,9568" to 'DECIMAL' +0 5 44 NULL 1 a CAST "33,4386" Error when converting column "a". Could not convert string "33,4386" to 'DECIMAL' +0 6 54 NULL 1 a CAST "33.4386,00" Error when converting column "a". Could not convert string "33.4386,00" to 'DECIMAL' + diff --git a/test/sql/copy/csv/rejects/test_multiple_errors_same_line.test b/test/sql/copy/csv/rejects/test_multiple_errors_same_line.test index 6d9f1fcb5baa..89ece06be587 100644 --- a/test/sql/copy/csv/rejects/test_multiple_errors_same_line.test +++ b/test/sql/copy/csv/rejects/test_multiple_errors_same_line.test @@ -61,8 +61,8 @@ oogie boogie 3 2023-01-02 2023-01-03 query IIIIIIIII rowsort SElECT * EXCLUDE (scan_id) FROM reject_errors ORDER BY ALL; ---- -0 4 110 NULL 3 current_day CAST oogie boogie,3, bla_2, bla_1 Error when converting column "current_day". date field value out of range: " bla_2", expected format is (YYYY-MM-DD) -0 4 110 NULL 4 tomorrow CAST oogie boogie,3, bla_2, bla_1 Error when converting column "tomorrow". date field value out of range: " bla_1", expected format is (YYYY-MM-DD) +0 4 110 NULL 3 current_day CAST oogie boogie,3, bla_2, bla_1 Error when converting column "current_day". Could not convert string " bla_2" to 'DATE' +0 4 110 NULL 4 tomorrow CAST oogie boogie,3, bla_2, bla_1 Error when converting column "tomorrow". Could not convert string " bla_1" to 'DATE' statement ok DROP TABLE reject_errors; diff --git a/test/sql/parallelism/interquery/test_concurrent_index.cpp b/test/sql/parallelism/interquery/test_concurrent_index.cpp index 5e4896e0b370..36f329d6c441 100644 --- a/test/sql/parallelism/interquery/test_concurrent_index.cpp +++ b/test/sql/parallelism/interquery/test_concurrent_index.cpp @@ -267,30 +267,30 @@ TEST_CASE("Mix updates and inserts on PRIMARY KEY", "[index][.]") { REQUIRE(CHECK_COLUMN(result, 1, {Value::BIGINT(atomic_count)})); } -static void TransactionalAppendToPK(DuckDB *db, idx_t thread_idx) { - - Connection con(*db); - REQUIRE_NO_FAIL(con.Query("BEGIN TRANSACTION")); - - // get the initial count - auto result = con.Query("SELECT COUNT(*) FROM integers WHERE i >= 0"); - REQUIRE_NO_FAIL(*result); - - auto chunk = result->Fetch(); - auto initial_count = chunk->GetValue(0, 0).GetValue(); - - for (idx_t i = 0; i < 50; i++) { - - auto loop_result = con.Query("INSERT INTO integers VALUES ($1)", (int32_t)(thread_idx * 1000 + i)); - REQUIRE_NO_FAIL(*result); - - // check the count - loop_result = con.Query("SELECT COUNT(*), COUNT(DISTINCT i) FROM integers WHERE i >= 0"); - REQUIRE(CHECK_COLUMN(loop_result, 0, {Value::INTEGER(initial_count + i + 1)})); - } - - REQUIRE_NO_FAIL(con.Query("COMMIT")); -} +// static void TransactionalAppendToPK(DuckDB *db, idx_t thread_idx) { +// +// Connection con(*db); +// REQUIRE_NO_FAIL(con.Query("BEGIN TRANSACTION")); +// +// // get the initial count +// auto result = con.Query("SELECT COUNT(*) FROM integers WHERE i >= 0"); +// REQUIRE_NO_FAIL(*result); +// +// auto chunk = result->Fetch(); +// auto initial_count = chunk->GetValue(0, 0).GetValue(); +// +// for (idx_t i = 0; i < 50; i++) { +// +// auto loop_result = con.Query("INSERT INTO integers VALUES ($1)", (int32_t)(thread_idx * 1000 + i)); +// REQUIRE_NO_FAIL(*result); +// +// // check the count +// loop_result = con.Query("SELECT COUNT(*), COUNT(DISTINCT i) FROM integers WHERE i >= 0"); +// REQUIRE(CHECK_COLUMN(loop_result, 0, {Value::INTEGER(initial_count + i + 1)})); +// } +// +// REQUIRE_NO_FAIL(con.Query("COMMIT")); +//} TEST_CASE("Parallel transactional appends to indexed table", "[index][.]") { From 58354a26bebd4515aab5615d7275c4e31534b84d Mon Sep 17 00:00:00 2001 From: stephaniewang Date: Fri, 19 Apr 2024 10:59:20 -0400 Subject: [PATCH 596/603] add patch --- .../extensions/aws/0001-update-tests.patch | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/patches/extensions/aws/0001-update-tests.patch diff --git a/.github/patches/extensions/aws/0001-update-tests.patch b/.github/patches/extensions/aws/0001-update-tests.patch new file mode 100644 index 000000000000..8e668b4e0615 --- /dev/null +++ b/.github/patches/extensions/aws/0001-update-tests.patch @@ -0,0 +1,53 @@ +From e8e6c286376d97e0a695284fc32b3b67a77e35af Mon Sep 17 00:00:00 2001 +From: stephaniewang +Date: Thu, 18 Apr 2024 21:41:29 -0400 +Subject: [PATCH] update tests + +--- + test/sql/aws_minio_secret.test | 2 +- + test/sql/aws_secret_gcs.test | 2 +- + test/sql/aws_secret_r2.test | 2 +- + 3 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/test/sql/aws_minio_secret.test b/test/sql/aws_minio_secret.test +index 2ddc29f..34c4c92 100644 +--- a/test/sql/aws_minio_secret.test ++++ b/test/sql/aws_minio_secret.test +@@ -28,7 +28,7 @@ CREATE SECRET my_aws_secret ( + ); + + query I +-SELECT which_secret('s3://test-bucket/aws_minio_secret/secret1/test.csv', 's3') ++SELECT name FROM which_secret('s3://test-bucket/aws_minio_secret/secret1/test.csv', 's3') + ---- + my_aws_secret + +diff --git a/test/sql/aws_secret_gcs.test b/test/sql/aws_secret_gcs.test +index 0b1fd40..cbed048 100644 +--- a/test/sql/aws_secret_gcs.test ++++ b/test/sql/aws_secret_gcs.test +@@ -18,7 +18,7 @@ CREATE SECRET s1 ( + ); + + query I +-SELECT which_secret('gcs://haha/hoehoe.parkoe', 'gcs') ++SELECT name FROM which_secret('gcs://haha/hoehoe.parkoe', 'gcs') + ---- + s1 + +diff --git a/test/sql/aws_secret_r2.test b/test/sql/aws_secret_r2.test +index 01be38b..19ebd1e 100644 +--- a/test/sql/aws_secret_r2.test ++++ b/test/sql/aws_secret_r2.test +@@ -19,7 +19,7 @@ CREATE SECRET s1 ( + ); + + query I +-SELECT which_secret('r2://haha/hoehoe.parkoe', 'r2') ++SELECT name FROM which_secret('r2://haha/hoehoe.parkoe', 'r2') + ---- + s1 + +-- +2.39.2 (Apple Git-143) + From c789ffd01bfff8ac0890b1585b4a98fd75d58438 Mon Sep 17 00:00:00 2001 From: Elliana May Date: Fri, 12 Apr 2024 15:10:52 +0800 Subject: [PATCH 597/603] test: add DuckDBPyType tests --- tools/pythonpkg/tests/fast/test_type.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/pythonpkg/tests/fast/test_type.py b/tools/pythonpkg/tests/fast/test_type.py index d8723d45e22b..7ce38dfe9326 100644 --- a/tools/pythonpkg/tests/fast/test_type.py +++ b/tools/pythonpkg/tests/fast/test_type.py @@ -26,6 +26,7 @@ TIMESTAMP_MS, TIMESTAMP_NS, TIMESTAMP_S, + DuckDBPyType, TIME, TIME_TZ, TIMESTAMP_TZ, @@ -227,3 +228,9 @@ def test_optional(self): def test_optional_310(self): type = duckdb.typing.DuckDBPyType(str | None) assert type == 'VARCHAR' + + def test_children_attribute(self): + assert DuckDBPyType('INTEGER[]').children == [('child', DuckDBPyType('INTEGER'))] + assert DuckDBPyType('INTEGER[2]').children == [('child', DuckDBPyType('INTEGER')), ('size', 2)] + assert DuckDBPyType('INTEGER[2][3]').children == [('child', DuckDBPyType('INTEGER[2]')), ('size', 3)] + assert DuckDBPyType("ENUM('a', 'b', 'c')").children == [('values', ['a', 'b', 'c'])] From c56dda6a6a2248e46409af936bb4d36d857f93c9 Mon Sep 17 00:00:00 2001 From: Elliana May Date: Mon, 22 Apr 2024 01:34:48 +0800 Subject: [PATCH 598/603] lint --- tools/pythonpkg/src/typing/pytype.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pythonpkg/src/typing/pytype.cpp b/tools/pythonpkg/src/typing/pytype.cpp index be0c0661881e..529c8a043683 100644 --- a/tools/pythonpkg/src/typing/pytype.cpp +++ b/tools/pythonpkg/src/typing/pytype.cpp @@ -362,7 +362,7 @@ py::list DuckDBPyType::Children() const { return children; } if (id == LogicalTypeId::ARRAY) { - children.append(py::make_tuple("child", make_shared(ArrayType::GetChildType(type)))); + children.append(py::make_tuple("child", make_shared_ptr(ArrayType::GetChildType(type)))); children.append(py::make_tuple("size", ArrayType::GetSize(type))); return children; } From 7d4c2f326b056066ce4af1f0a72f774b4a5a8f7f Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Sun, 21 Apr 2024 06:18:43 +0200 Subject: [PATCH 599/603] Pyodide: avoid packaging data_files --- tools/pythonpkg/setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/pythonpkg/setup.py b/tools/pythonpkg/setup.py index c61e2e2b20d3..31265393696a 100644 --- a/tools/pythonpkg/setup.py +++ b/tools/pythonpkg/setup.py @@ -355,6 +355,8 @@ def setup_data_files(data_files): data_files = setup_data_files(extra_files + header_files) +if is_pyodide: + data_files = [] packages = [ lib_name, From aaa56f621587a1a6ea9e9487af2f465490cbe1be Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 22 Apr 2024 10:05:55 +0200 Subject: [PATCH 600/603] CI-Pyodide: trigger also on changes to tools/pythonpkg/setup.py --- .github/workflows/Pyodide.yml | 2 ++ tools/pythonpkg/setup.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/Pyodide.yml b/.github/workflows/Pyodide.yml index ef96f95511d8..ea93504a1ba5 100644 --- a/.github/workflows/Pyodide.yml +++ b/.github/workflows/Pyodide.yml @@ -25,12 +25,14 @@ on: paths-ignore: - "**" - "!.github/workflows/Pyodide.yml" + - "!tools/pythonpkg/setup.py" pull_request: types: [opened, reopened, ready_for_review] paths-ignore: - "**" - "!.github/workflows/Pyodide.yml" + - "!tools/pythonpkg/setup.py" jobs: build_pyodide: diff --git a/tools/pythonpkg/setup.py b/tools/pythonpkg/setup.py index 31265393696a..0b33a3bc0c3d 100644 --- a/tools/pythonpkg/setup.py +++ b/tools/pythonpkg/setup.py @@ -122,6 +122,7 @@ class build_ext(CompilerLauncherMixin, _build_ext): is_android = hasattr(sys, 'getandroidapilevel') is_pyodide = 'PYODIDE' in os.environ +no_source_wheel = is_pyodide use_jemalloc = ( not is_android and not is_pyodide @@ -355,7 +356,7 @@ def setup_data_files(data_files): data_files = setup_data_files(extra_files + header_files) -if is_pyodide: +if no_source_wheel: data_files = [] packages = [ From 574b101b0789edb3b1455f4fdbb02bf9c527ebb5 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 22 Apr 2024 10:07:06 +0200 Subject: [PATCH 601/603] CI Pyodide: Log package size --- .github/workflows/Pyodide.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/Pyodide.yml b/.github/workflows/Pyodide.yml index ea93504a1ba5..160bcc759f56 100644 --- a/.github/workflows/Pyodide.yml +++ b/.github/workflows/Pyodide.yml @@ -93,6 +93,10 @@ jobs: assert platform == "wasm_eh_pyodide", platform EOF + - name: Wheel sizes + run: | + ls -lah ./tools/pythonpkg/dist/*.whl + - uses: actions/upload-artifact@v3 with: name: pyodide-python${{ matrix.version.python }} From 42db2148aeecf3c3f12e223357a03daecb69712f Mon Sep 17 00:00:00 2001 From: Gabor Szarnyas Date: Mon, 22 Apr 2024 10:44:02 +0200 Subject: [PATCH 602/603] Add missing space in error message --- src/catalog/catalog_entry/duck_schema_entry.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/catalog/catalog_entry/duck_schema_entry.cpp b/src/catalog/catalog_entry/duck_schema_entry.cpp index cbd78d374847..ccdba5acbd0f 100644 --- a/src/catalog/catalog_entry/duck_schema_entry.cpp +++ b/src/catalog/catalog_entry/duck_schema_entry.cpp @@ -224,7 +224,7 @@ optional_ptr DuckSchemaEntry::CreateIndex(ClientContext &context, // currently, we can not alter PK/FK/UNIQUE constraints // concurrency-safe name checks against other INDEX catalog entries happens in the catalog if (!table.GetStorage().IndexNameIsUnique(info.index_name)) { - throw CatalogException("An index with the name " + info.index_name + "already exists!"); + throw CatalogException("An index with the name " + info.index_name + " already exists!"); } auto index = make_uniq(catalog, *this, info); From bb63a046fb6e51b0cfe7307e90170e4c536a4a91 Mon Sep 17 00:00:00 2001 From: Tom Ebergen Date: Mon, 22 Apr 2024 14:25:12 +0200 Subject: [PATCH 603/603] regression script should calculate diff with base branch, not main --- .github/workflows/Regression.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Regression.yml b/.github/workflows/Regression.yml index d87ebf9f1a4d..e318752a365b 100644 --- a/.github/workflows/Regression.yml +++ b/.github/workflows/Regression.yml @@ -133,7 +133,7 @@ jobs: if: always() shell: bash run: | - git diff --name-only origin/main | (grep "benchmark/micro/*" || true) > .github/regression/updated_micro_benchmarks.csv + git diff --name-only origin/${{ env.BASE_BRANCH }} | (grep "benchmark/micro/*" || true) > .github/regression/updated_micro_benchmarks.csv if [ $(wc -l < .github/regression/updated_micro_benchmarks.csv) -gt 0 ]; then echo "Detected the following modified benchmarks. Running a regression test" cat .github/regression/updated_micro_benchmarks.csv