Skip to content

Commit 2078af7

Browse files
save-bufferpitrou
save-buffer
and
pitrou
authored
ARROW-17836: [C++] Allow specifying alignment of buffers (#14225)
Lead-authored-by: Sasha Krassovsky <krassovskysasha@gmail.com> Co-authored-by: Antoine Pitrou <antoine@python.org> Signed-off-by: Antoine Pitrou <antoine@python.org>
1 parent 63f013c commit 2078af7

36 files changed

+827
-392
lines changed

‎cpp/src/arrow/array/builder_adaptive.cc‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,11 @@ namespace arrow {
3333

3434
using internal::AdaptiveIntBuilderBase;
3535

36-
AdaptiveIntBuilderBase::AdaptiveIntBuilderBase(uint8_t start_int_size, MemoryPool* pool)
37-
: ArrayBuilder(pool), start_int_size_(start_int_size), int_size_(start_int_size) {}
36+
AdaptiveIntBuilderBase::AdaptiveIntBuilderBase(uint8_t start_int_size, MemoryPool* pool,
37+
int64_t alignment)
38+
: ArrayBuilder(pool, alignment),
39+
start_int_size_(start_int_size),
40+
int_size_(start_int_size) {}
3841

3942
void AdaptiveIntBuilderBase::Reset() {
4043
ArrayBuilder::Reset();
@@ -125,8 +128,9 @@ std::shared_ptr<DataType> AdaptiveIntBuilder::type() const {
125128
return nullptr;
126129
}
127130

128-
AdaptiveIntBuilder::AdaptiveIntBuilder(uint8_t start_int_size, MemoryPool* pool)
129-
: AdaptiveIntBuilderBase(start_int_size, pool) {}
131+
AdaptiveIntBuilder::AdaptiveIntBuilder(uint8_t start_int_size, MemoryPool* pool,
132+
int64_t alignment)
133+
: AdaptiveIntBuilderBase(start_int_size, pool, alignment) {}
130134

131135
Status AdaptiveIntBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
132136
RETURN_NOT_OK(CommitPendingData());

‎cpp/src/arrow/array/builder_adaptive.h‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,12 @@ namespace internal {
3939

4040
class ARROW_EXPORT AdaptiveIntBuilderBase : public ArrayBuilder {
4141
public:
42-
AdaptiveIntBuilderBase(uint8_t start_int_size, MemoryPool* pool);
42+
AdaptiveIntBuilderBase(uint8_t start_int_size, MemoryPool* pool,
43+
int64_t alignment = kDefaultBufferAlignment);
4344

44-
explicit AdaptiveIntBuilderBase(MemoryPool* pool)
45-
: AdaptiveIntBuilderBase(sizeof(uint8_t), pool) {}
45+
explicit AdaptiveIntBuilderBase(MemoryPool* pool,
46+
int64_t alignment = kDefaultBufferAlignment)
47+
: AdaptiveIntBuilderBase(sizeof(uint8_t), pool, alignment) {}
4648

4749
/// \brief Append multiple nulls
4850
/// \param[in] length the number of nulls to append
@@ -173,10 +175,12 @@ class ARROW_EXPORT AdaptiveUIntBuilder : public internal::AdaptiveIntBuilderBase
173175
class ARROW_EXPORT AdaptiveIntBuilder : public internal::AdaptiveIntBuilderBase {
174176
public:
175177
explicit AdaptiveIntBuilder(uint8_t start_int_size,
176-
MemoryPool* pool = default_memory_pool());
178+
MemoryPool* pool = default_memory_pool(),
179+
int64_t alignment = kDefaultBufferAlignment);
177180

178-
explicit AdaptiveIntBuilder(MemoryPool* pool = default_memory_pool())
179-
: AdaptiveIntBuilder(sizeof(uint8_t), pool) {}
181+
explicit AdaptiveIntBuilder(MemoryPool* pool = default_memory_pool(),
182+
int64_t alignment = kDefaultBufferAlignment)
183+
: AdaptiveIntBuilder(sizeof(uint8_t), pool, alignment) {}
180184

181185
using ArrayBuilder::Advance;
182186
using internal::AdaptiveIntBuilderBase::Reset;

‎cpp/src/arrow/array/builder_base.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,8 @@ constexpr int64_t kListMaximumElements = std::numeric_limits<int32_t>::max() - 1
6969
/// For example, ArrayBuilder* pointing to BinaryBuilder should be downcast before use.
7070
class ARROW_EXPORT ArrayBuilder {
7171
public:
72-
explicit ArrayBuilder(MemoryPool* pool) : pool_(pool), null_bitmap_builder_(pool) {}
72+
explicit ArrayBuilder(MemoryPool* pool, int64_t alignment = kDefaultBufferAlignment)
73+
: pool_(pool), alignment_(alignment), null_bitmap_builder_(pool, alignment) {}
7374

7475
ARROW_DEFAULT_MOVE_AND_ASSIGN(ArrayBuilder);
7576

@@ -283,6 +284,7 @@ class ARROW_EXPORT ArrayBuilder {
283284
const char* message);
284285

285286
MemoryPool* pool_;
287+
int64_t alignment_;
286288

287289
TypedBufferBuilder<bool> null_bitmap_builder_;
288290
int64_t null_count_ = 0;

‎cpp/src/arrow/array/builder_binary.cc‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,10 @@ using internal::checked_cast;
4444
// Fixed width binary
4545

4646
FixedSizeBinaryBuilder::FixedSizeBinaryBuilder(const std::shared_ptr<DataType>& type,
47-
MemoryPool* pool)
48-
: ArrayBuilder(pool),
47+
MemoryPool* pool, int64_t alignment)
48+
: ArrayBuilder(pool, alignment),
4949
byte_width_(checked_cast<const FixedSizeBinaryType&>(*type).byte_width()),
50-
byte_builder_(pool) {}
50+
byte_builder_(pool, alignment) {}
5151

5252
void FixedSizeBinaryBuilder::CheckValueSize(int64_t size) {
5353
DCHECK_EQ(size, byte_width_) << "Appending wrong size to FixedSizeBinaryBuilder";

‎cpp/src/arrow/array/builder_binary.h‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,11 @@ class BaseBinaryBuilder : public ArrayBuilder {
5454
using TypeClass = TYPE;
5555
using offset_type = typename TypeClass::offset_type;
5656

57-
explicit BaseBinaryBuilder(MemoryPool* pool = default_memory_pool())
58-
: ArrayBuilder(pool), offsets_builder_(pool), value_data_builder_(pool) {}
57+
explicit BaseBinaryBuilder(MemoryPool* pool = default_memory_pool(),
58+
int64_t alignment = kDefaultBufferAlignment)
59+
: ArrayBuilder(pool, alignment),
60+
offsets_builder_(pool, alignment),
61+
value_data_builder_(pool, alignment) {}
5962

6063
BaseBinaryBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool)
6164
: BaseBinaryBuilder(pool) {}
@@ -464,7 +467,8 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
464467
using TypeClass = FixedSizeBinaryType;
465468

466469
explicit FixedSizeBinaryBuilder(const std::shared_ptr<DataType>& type,
467-
MemoryPool* pool = default_memory_pool());
470+
MemoryPool* pool = default_memory_pool(),
471+
int64_t alignment = kDefaultBufferAlignment);
468472

469473
Status Append(const uint8_t* value) {
470474
ARROW_RETURN_NOT_OK(Reserve(1));

‎cpp/src/arrow/array/builder_decimal.cc‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@ class MemoryPool;
3636
// Decimal128Builder
3737

3838
Decimal128Builder::Decimal128Builder(const std::shared_ptr<DataType>& type,
39-
MemoryPool* pool)
40-
: FixedSizeBinaryBuilder(type, pool),
39+
MemoryPool* pool, int64_t alignment)
40+
: FixedSizeBinaryBuilder(type, pool, alignment),
4141
decimal_type_(internal::checked_pointer_cast<Decimal128Type>(type)) {}
4242

4343
Status Decimal128Builder::Append(Decimal128 value) {
@@ -71,8 +71,8 @@ Status Decimal128Builder::FinishInternal(std::shared_ptr<ArrayData>* out) {
7171
// Decimal256Builder
7272

7373
Decimal256Builder::Decimal256Builder(const std::shared_ptr<DataType>& type,
74-
MemoryPool* pool)
75-
: FixedSizeBinaryBuilder(type, pool),
74+
MemoryPool* pool, int64_t alignment)
75+
: FixedSizeBinaryBuilder(type, pool, alignment),
7676
decimal_type_(internal::checked_pointer_cast<Decimal256Type>(type)) {}
7777

7878
Status Decimal256Builder::Append(const Decimal256& value) {

‎cpp/src/arrow/array/builder_decimal.h‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,8 @@ class ARROW_EXPORT Decimal128Builder : public FixedSizeBinaryBuilder {
3939
using ValueType = Decimal128;
4040

4141
explicit Decimal128Builder(const std::shared_ptr<DataType>& type,
42-
MemoryPool* pool = default_memory_pool());
42+
MemoryPool* pool = default_memory_pool(),
43+
int64_t alignment = kDefaultBufferAlignment);
4344

4445
using FixedSizeBinaryBuilder::Append;
4546
using FixedSizeBinaryBuilder::AppendValues;
@@ -69,7 +70,8 @@ class ARROW_EXPORT Decimal256Builder : public FixedSizeBinaryBuilder {
6970
using ValueType = Decimal256;
7071

7172
explicit Decimal256Builder(const std::shared_ptr<DataType>& type,
72-
MemoryPool* pool = default_memory_pool());
73+
MemoryPool* pool = default_memory_pool(),
74+
int64_t alignment = kDefaultBufferAlignment);
7375

7476
using FixedSizeBinaryBuilder::Append;
7577
using FixedSizeBinaryBuilder::AppendValues;

‎cpp/src/arrow/array/builder_dict.h‎

Lines changed: 28 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -146,37 +146,40 @@ class DictionaryBuilderBase : public ArrayBuilder {
146146
!is_fixed_size_binary_type<T1>::value,
147147
const std::shared_ptr<DataType>&>
148148
value_type,
149-
MemoryPool* pool = default_memory_pool())
150-
: ArrayBuilder(pool),
149+
MemoryPool* pool = default_memory_pool(),
150+
int64_t alignment = kDefaultBufferAlignment)
151+
: ArrayBuilder(pool, alignment),
151152
memo_table_(new internal::DictionaryMemoTable(pool, value_type)),
152153
delta_offset_(0),
153154
byte_width_(-1),
154-
indices_builder_(start_int_size, pool),
155+
indices_builder_(start_int_size, pool, alignment),
155156
value_type_(value_type) {}
156157

157158
template <typename T1 = T>
158159
explicit DictionaryBuilderBase(
159160
enable_if_t<!is_fixed_size_binary_type<T1>::value, const std::shared_ptr<DataType>&>
160161
value_type,
161-
MemoryPool* pool = default_memory_pool())
162-
: ArrayBuilder(pool),
162+
MemoryPool* pool = default_memory_pool(),
163+
int64_t alignment = kDefaultBufferAlignment)
164+
: ArrayBuilder(pool, alignment),
163165
memo_table_(new internal::DictionaryMemoTable(pool, value_type)),
164166
delta_offset_(0),
165167
byte_width_(-1),
166-
indices_builder_(pool),
168+
indices_builder_(pool, alignment),
167169
value_type_(value_type) {}
168170

169171
template <typename T1 = T>
170172
explicit DictionaryBuilderBase(
171173
const std::shared_ptr<DataType>& index_type,
172174
enable_if_t<!is_fixed_size_binary_type<T1>::value, const std::shared_ptr<DataType>&>
173175
value_type,
174-
MemoryPool* pool = default_memory_pool())
175-
: ArrayBuilder(pool),
176+
MemoryPool* pool = default_memory_pool(),
177+
int64_t alignment = kDefaultBufferAlignment)
178+
: ArrayBuilder(pool, alignment),
176179
memo_table_(new internal::DictionaryMemoTable(pool, value_type)),
177180
delta_offset_(0),
178181
byte_width_(-1),
179-
indices_builder_(index_type, pool),
182+
indices_builder_(index_type, pool, alignment),
180183
value_type_(value_type) {}
181184

182185
template <typename B = BuilderType, typename T1 = T>
@@ -185,35 +188,38 @@ class DictionaryBuilderBase : public ArrayBuilder {
185188
is_fixed_size_binary_type<T1>::value,
186189
const std::shared_ptr<DataType>&>
187190
value_type,
188-
MemoryPool* pool = default_memory_pool())
189-
: ArrayBuilder(pool),
191+
MemoryPool* pool = default_memory_pool(),
192+
int64_t alignment = kDefaultBufferAlignment)
193+
: ArrayBuilder(pool, alignment),
190194
memo_table_(new internal::DictionaryMemoTable(pool, value_type)),
191195
delta_offset_(0),
192196
byte_width_(static_cast<const T1&>(*value_type).byte_width()),
193-
indices_builder_(start_int_size, pool),
197+
indices_builder_(start_int_size, pool, alignment),
194198
value_type_(value_type) {}
195199

196200
template <typename T1 = T>
197201
explicit DictionaryBuilderBase(
198202
enable_if_fixed_size_binary<T1, const std::shared_ptr<DataType>&> value_type,
199-
MemoryPool* pool = default_memory_pool())
200-
: ArrayBuilder(pool),
203+
MemoryPool* pool = default_memory_pool(),
204+
int64_t alignment = kDefaultBufferAlignment)
205+
: ArrayBuilder(pool, alignment),
201206
memo_table_(new internal::DictionaryMemoTable(pool, value_type)),
202207
delta_offset_(0),
203208
byte_width_(static_cast<const T1&>(*value_type).byte_width()),
204-
indices_builder_(pool),
209+
indices_builder_(pool, alignment),
205210
value_type_(value_type) {}
206211

207212
template <typename T1 = T>
208213
explicit DictionaryBuilderBase(
209214
const std::shared_ptr<DataType>& index_type,
210215
enable_if_fixed_size_binary<T1, const std::shared_ptr<DataType>&> value_type,
211-
MemoryPool* pool = default_memory_pool())
212-
: ArrayBuilder(pool),
216+
MemoryPool* pool = default_memory_pool(),
217+
int64_t alignment = kDefaultBufferAlignment)
218+
: ArrayBuilder(pool, alignment),
213219
memo_table_(new internal::DictionaryMemoTable(pool, value_type)),
214220
delta_offset_(0),
215221
byte_width_(static_cast<const T1&>(*value_type).byte_width()),
216-
indices_builder_(index_type, pool),
222+
indices_builder_(index_type, pool, alignment),
217223
value_type_(value_type) {}
218224

219225
template <typename T1 = T>
@@ -223,12 +229,13 @@ class DictionaryBuilderBase : public ArrayBuilder {
223229

224230
// This constructor doesn't check for errors. Use InsertMemoValues instead.
225231
explicit DictionaryBuilderBase(const std::shared_ptr<Array>& dictionary,
226-
MemoryPool* pool = default_memory_pool())
227-
: ArrayBuilder(pool),
232+
MemoryPool* pool = default_memory_pool(),
233+
int64_t alignment = kDefaultBufferAlignment)
234+
: ArrayBuilder(pool, alignment),
228235
memo_table_(new internal::DictionaryMemoTable(pool, dictionary)),
229236
delta_offset_(0),
230237
byte_width_(-1),
231-
indices_builder_(pool),
238+
indices_builder_(pool, alignment),
232239
value_type_(dictionary->type()) {}
233240

234241
~DictionaryBuilderBase() override = default;

‎cpp/src/arrow/array/builder_nested.h‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -51,14 +51,16 @@ class BaseListBuilder : public ArrayBuilder {
5151
/// Use this constructor to incrementally build the value array along with offsets and
5252
/// null bitmap.
5353
BaseListBuilder(MemoryPool* pool, std::shared_ptr<ArrayBuilder> const& value_builder,
54-
const std::shared_ptr<DataType>& type)
55-
: ArrayBuilder(pool),
56-
offsets_builder_(pool),
54+
const std::shared_ptr<DataType>& type,
55+
int64_t alignment = kDefaultBufferAlignment)
56+
: ArrayBuilder(pool, alignment),
57+
offsets_builder_(pool, alignment),
5758
value_builder_(value_builder),
5859
value_field_(type->field(0)->WithType(NULLPTR)) {}
5960

60-
BaseListBuilder(MemoryPool* pool, std::shared_ptr<ArrayBuilder> const& value_builder)
61-
: BaseListBuilder(pool, value_builder, list(value_builder->type())) {}
61+
BaseListBuilder(MemoryPool* pool, std::shared_ptr<ArrayBuilder> const& value_builder,
62+
int64_t alignment = kDefaultBufferAlignment)
63+
: BaseListBuilder(pool, value_builder, list(value_builder->type()), alignment) {}
6264

6365
Status Resize(int64_t capacity) override {
6466
if (capacity > maximum_elements()) {

‎cpp/src/arrow/array/builder_primitive.cc‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,11 +44,12 @@ Status NullBuilder::FinishInternal(std::shared_ptr<ArrayData>* out) {
4444
return Status::OK();
4545
}
4646

47-
BooleanBuilder::BooleanBuilder(MemoryPool* pool)
48-
: ArrayBuilder(pool), data_builder_(pool) {}
47+
BooleanBuilder::BooleanBuilder(MemoryPool* pool, int64_t alignment)
48+
: ArrayBuilder(pool, alignment), data_builder_(pool, alignment) {}
4949

50-
BooleanBuilder::BooleanBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool)
51-
: BooleanBuilder(pool) {
50+
BooleanBuilder::BooleanBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool,
51+
int64_t alignment)
52+
: BooleanBuilder(pool, alignment) {
5253
ARROW_CHECK_EQ(Type::BOOL, type->id());
5354
}
5455

0 commit comments

Comments
 (0)