Skip to content

Commit

Permalink
Arrow list buffer - suggest setting arrow_large_buffer_size to true…
Browse files Browse the repository at this point in the history
… when regular list buffer size is exceeded (duckdb#14384)

This can happen when emitting arrow tables with large lists.
  • Loading branch information
Mytherin authored Oct 16, 2024
2 parents a411527 + e44ce5d commit fb51e97
Show file tree
Hide file tree
Showing 2 changed files with 4 additions and 2 deletions.
3 changes: 2 additions & 1 deletion src/include/duckdb/common/arrow/appender/list_data.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,8 @@ struct ArrowListData {
(uint64_t)last_offset + list_length > NumericLimits<int32_t>::Maximum()) {
throw InvalidInputException(
"Arrow Appender: The maximum combined list offset for regular list buffers is "
"%u but the offset of %lu exceeds this.",
"%u but the offset of %lu exceeds this.\n* SET arrow_large_buffer_size=true to use large list "
"buffers",
NumericLimits<int32_t>::Maximum(), last_offset);
}
last_offset += list_length;
Expand Down
3 changes: 2 additions & 1 deletion src/include/duckdb/common/arrow/appender/list_view_data.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,8 @@ struct ArrowListViewData {
(uint64_t)last_offset + list_length > NumericLimits<int32_t>::Maximum()) {
throw InvalidInputException(
"Arrow Appender: The maximum combined list offset for regular list buffers is "
"%u but the offset of %lu exceeds this.",
"%u but the offset of %lu exceeds this.\n* SET arrow_large_buffer_size=true to use large list "
"buffers",
NumericLimits<int32_t>::Maximum(), last_offset);
}
offset_data[offset_idx] = last_offset;
Expand Down

0 comments on commit fb51e97

Please sign in to comment.