Skip to content

Commit

Permalink
GH-39704: [C++][Parquet] Benchmark levels decoding (#39705)
Browse files Browse the repository at this point in the history
### Rationale for this change

This patch add the level-decoding benchmark. It test:
1. Different max-level (for flat type, maximum level would be 1, for nested type, it would grows)
2. With different repeat ( repeated null / non-null is different from non-repeated data)
3. With different read-batch size. This part of logic is a bit tricky in original code

### What changes are included in this PR?

Add Level decoding benchmark

### Are these changes tested?

No need

### Are there any user-facing changes?

no

* Closes: #39704

Authored-by: mwish <maplewish117@gmail.com>
Signed-off-by: Antoine Pitrou <antoine@python.org>
  • Loading branch information
mapleFU authored Feb 5, 2024
1 parent cb5c109 commit 0c88d13
Show file tree
Hide file tree
Showing 2 changed files with 100 additions and 2 deletions.
98 changes: 98 additions & 0 deletions cpp/src/parquet/column_reader_benchmark.cc
Original file line number Diff line number Diff line change
Expand Up @@ -219,5 +219,103 @@ BENCHMARK(RecordReaderReadRecords)
->Args({2, 1000, true})
->Args({2, 1000, false});

void GenerateLevels(int level_repeats, int max_level, int num_levels,
std::vector<int16_t>* levels) {
// Generate random levels
std::default_random_engine gen(/*seed=*/1943);
std::uniform_int_distribution<int16_t> d(0, max_level);
for (int i = 0; i < num_levels;) {
int16_t current_level = d(gen); // level repeat `level_repeats` times
const int current_repeated = std::min(level_repeats, num_levels - i);
levels->insert(levels->end(), current_repeated, current_level);
i += current_repeated;
}
}

void EncodeLevels(Encoding::type encoding, int16_t max_level, int num_levels,
const int16_t* input_levels, std::vector<uint8_t>* bytes) {
LevelEncoder encoder;
// encode levels
if (encoding == Encoding::RLE) {
int rle_size = LevelEncoder::MaxBufferSize(encoding, max_level, num_levels);
bytes->resize(rle_size + sizeof(int32_t));
// leave space to write the rle length value
encoder.Init(encoding, max_level, num_levels, bytes->data() + sizeof(int32_t),
rle_size);
encoder.Encode(num_levels, input_levels);
int data_length = encoder.len();
memcpy(bytes->data(), &data_length, sizeof(int32_t));
} else {
int bitpack_size =
LevelEncoder::MaxBufferSize(encoding, max_level, num_levels) + sizeof(int32_t);
bytes->resize(bitpack_size);
encoder.Init(encoding, max_level, num_levels, bytes->data(),
static_cast<int>(bytes->size()));
encoder.Encode(num_levels, input_levels);
}
}

static void DecodeLevels(Encoding::type level_encoding, int16_t max_level, int num_levels,
int batch_size, int level_repeat_count,
::benchmark::State& state) {
std::vector<uint8_t> bytes;
{
std::vector<int16_t> input_levels;
GenerateLevels(/*level_repeats=*/level_repeat_count, /*max_repeat_factor=*/max_level,
num_levels, &input_levels);
EncodeLevels(level_encoding, max_level, num_levels, input_levels.data(), &bytes);
}

LevelDecoder decoder;
std::vector<int16_t> output_levels(batch_size);
for (auto _ : state) {
state.PauseTiming();
decoder.SetData(level_encoding, max_level, num_levels, bytes.data(),
static_cast<int>(bytes.size()));
state.ResumeTiming();
// Decode multiple times with batch_size
while (true) {
int levels_decoded = decoder.Decode(batch_size, output_levels.data());
if (levels_decoded == 0) {
break;
}
}
}
state.SetBytesProcessed(state.iterations() * num_levels * sizeof(int16_t));
state.SetItemsProcessed(state.iterations() * num_levels);
}

static void ReadLevels_Rle(::benchmark::State& state) {
int16_t max_level = static_cast<int16_t>(state.range(0));
int num_levels = static_cast<int>(state.range(1));
int batch_size = static_cast<int>(state.range(2));
int level_repeat_count = static_cast<int>(state.range(3));
DecodeLevels(Encoding::RLE, max_level, num_levels, batch_size, level_repeat_count,
state);
}

static void ReadLevels_BitPack(::benchmark::State& state) {
int16_t max_level = static_cast<int16_t>(state.range(0));
int num_levels = static_cast<int>(state.range(1));
int batch_size = static_cast<int>(state.range(2));
int level_repeat_count = static_cast<int>(state.range(3));
DecodeLevels(Encoding::BIT_PACKED, max_level, num_levels, batch_size,
level_repeat_count, state);
}

static void ReadLevelsArguments(::benchmark::internal::Benchmark* b) {
b->ArgNames({"MaxLevel", "NumLevels", "BatchSize", "LevelRepeatCount"})
->Args({1, 8096, 1024, 1})
->Args({1, 8096, 1024, 7})
->Args({1, 8096, 1024, 1024})
->Args({1, 8096, 2048, 1})
->Args({3, 8096, 1024, 1})
->Args({3, 8096, 2048, 1})
->Args({3, 8096, 1024, 7});
}

BENCHMARK(ReadLevels_Rle)->Apply(ReadLevelsArguments);
BENCHMARK(ReadLevels_BitPack)->Apply(ReadLevelsArguments);

} // namespace benchmark
} // namespace parquet
4 changes: 2 additions & 2 deletions cpp/src/parquet/column_writer_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1021,7 +1021,7 @@ void EncodeLevels(Encoding::type encoding, int16_t max_level, int num_levels,
}

void VerifyDecodingLevels(Encoding::type encoding, int16_t max_level,
std::vector<int16_t>& input_levels,
const std::vector<int16_t>& input_levels,
std::vector<uint8_t>& bytes) {
LevelDecoder decoder;
int levels_count = 0;
Expand Down Expand Up @@ -1060,7 +1060,7 @@ void VerifyDecodingLevels(Encoding::type encoding, int16_t max_level,
}

void VerifyDecodingMultipleSetData(Encoding::type encoding, int16_t max_level,
std::vector<int16_t>& input_levels,
const std::vector<int16_t>& input_levels,
std::vector<std::vector<uint8_t>>& bytes) {
LevelDecoder decoder;
int levels_count = 0;
Expand Down

0 comments on commit 0c88d13

Please sign in to comment.