Skip to content

[Doc] Add docs to OLAP_SCAN_NODE query profile #3808

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morningman merged 4 commits into from
Jun 13, 2020
Merged

[Doc] Add docs to OLAP_SCAN_NODE query profile #3808

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
541570b
first
morningman-cmy Jun 8, 2020
dcd7963
add doc
morningman-cmy Jun 9, 2020
d0f584b
fix
morningman Jun 12, 2020
d2493d7
fix by review
morningman Jun 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 12 additions & 21 deletions be/src/exec/olap_scan_node.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,32 +92,23 @@ void OlapScanNode::_init_counter(RuntimeState* state) {
ADD_TIMER(_runtime_profile, "ShowHintsTime");

_reader_init_timer = ADD_TIMER(_runtime_profile, "ReaderInitTime");
_read_compressed_counter =
ADD_COUNTER(_runtime_profile, "CompressedBytesRead", TUnit::BYTES);
_read_uncompressed_counter =
ADD_COUNTER(_runtime_profile, "UncompressedBytesRead", TUnit::BYTES);
_read_compressed_counter = ADD_COUNTER(_runtime_profile, "CompressedBytesRead", TUnit::BYTES);
_read_uncompressed_counter = ADD_COUNTER(_runtime_profile, "UncompressedBytesRead", TUnit::BYTES);
_block_load_timer = ADD_TIMER(_runtime_profile, "BlockLoadTime");
_block_load_counter =
ADD_COUNTER(_runtime_profile, "BlocksLoad", TUnit::UNIT);
_block_fetch_timer =
ADD_TIMER(_runtime_profile, "BlockFetchTime");
_raw_rows_counter =
ADD_COUNTER(_runtime_profile, "RawRowsRead", TUnit::UNIT);
_block_load_counter = ADD_COUNTER(_runtime_profile, "BlocksLoad", TUnit::UNIT);
_block_fetch_timer = ADD_TIMER(_runtime_profile, "BlockFetchTime");
_raw_rows_counter = ADD_COUNTER(_runtime_profile, "RawRowsRead", TUnit::UNIT);
_block_convert_timer = ADD_TIMER(_runtime_profile, "BlockConvertTime");
_block_seek_timer = ADD_TIMER(_runtime_profile, "BlockSeekTime");
_block_seek_counter = ADD_COUNTER(_runtime_profile, "BlockSeekCount", TUnit::UNIT);

_rows_vec_cond_counter =
ADD_COUNTER(_runtime_profile, "RowsVectorPredFiltered", TUnit::UNIT);
_vec_cond_timer =
ADD_TIMER(_runtime_profile, "VectorPredEvalTime");
_rows_vec_cond_counter = ADD_COUNTER(_runtime_profile, "RowsVectorPredFiltered", TUnit::UNIT);
_vec_cond_timer = ADD_TIMER(_runtime_profile, "VectorPredEvalTime");

_stats_filtered_counter =
ADD_COUNTER(_runtime_profile, "RowsStatsFiltered", TUnit::UNIT);
_bf_filtered_counter =
ADD_COUNTER(_runtime_profile, "RowsBloomFilterFiltered", TUnit::UNIT);
_del_filtered_counter =
ADD_COUNTER(_runtime_profile, "RowsDelFiltered", TUnit::UNIT);
_stats_filtered_counter = ADD_COUNTER(_runtime_profile, "RowsStatsFiltered", TUnit::UNIT);
_bf_filtered_counter = ADD_COUNTER(_runtime_profile, "RowsBloomFilterFiltered", TUnit::UNIT);
_del_filtered_counter = ADD_COUNTER(_runtime_profile, "RowsDelFiltered", TUnit::UNIT);
_key_range_filtered_counter = ADD_COUNTER(_runtime_profile, "RowsKeyRangeFiltered", TUnit::UNIT);

_io_timer = ADD_TIMER(_runtime_profile, "IOTimer");
_decompressor_timer = ADD_TIMER(_runtime_profile, "DecompressorTimer");
Expand All @@ -128,7 +119,7 @@ void OlapScanNode::_init_counter(RuntimeState* state) {
_total_pages_num_counter = ADD_COUNTER(_runtime_profile, "TotalPagesNum", TUnit::UNIT);
_cached_pages_num_counter = ADD_COUNTER(_runtime_profile, "CachedPagesNum", TUnit::UNIT);

_bitmap_index_filter_counter = ADD_COUNTER(_runtime_profile, "BitmapIndexFilterCount", TUnit::UNIT);
_bitmap_index_filter_counter = ADD_COUNTER(_runtime_profile, "RowsBitmapIndexFiltered", TUnit::UNIT);
_bitmap_index_filter_timer = ADD_TIMER(_runtime_profile, "BitmapIndexFilterTimer");

_num_scanners = ADD_COUNTER(_runtime_profile, "NumScanners", TUnit::UNIT);
Expand Down
1 change: 1 addition & 0 deletions be/src/exec/olap_scan_node.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,7 @@ class OlapScanNode : public ScanNode {
RuntimeProfile::Counter* _stats_filtered_counter = nullptr;
RuntimeProfile::Counter* _bf_filtered_counter = nullptr;
RuntimeProfile::Counter* _del_filtered_counter = nullptr;
RuntimeProfile::Counter* _key_range_filtered_counter = nullptr;

RuntimeProfile::Counter* _block_seek_timer = nullptr;
RuntimeProfile::Counter* _block_seek_counter = nullptr;
Expand Down
3 changes: 2 additions & 1 deletion be/src/exec/olap_scanner.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -469,13 +469,14 @@ void OlapScanner::update_counter() {
COUNTER_UPDATE(_parent->_stats_filtered_counter, _reader->stats().rows_stats_filtered);
COUNTER_UPDATE(_parent->_bf_filtered_counter, _reader->stats().rows_bf_filtered);
COUNTER_UPDATE(_parent->_del_filtered_counter, _reader->stats().rows_del_filtered);
COUNTER_UPDATE(_parent->_key_range_filtered_counter, _reader->stats().rows_key_range_filtered);

COUNTER_UPDATE(_parent->_index_load_timer, _reader->stats().index_load_ns);

COUNTER_UPDATE(_parent->_total_pages_num_counter, _reader->stats().total_pages_num);
COUNTER_UPDATE(_parent->_cached_pages_num_counter, _reader->stats().cached_pages_num);

COUNTER_UPDATE(_parent->_bitmap_index_filter_counter, _reader->stats().bitmap_index_filter_count);
COUNTER_UPDATE(_parent->_bitmap_index_filter_counter, _reader->stats().rows_bitmap_index_filtered);
COUNTER_UPDATE(_parent->_bitmap_index_filter_timer, _reader->stats().bitmap_index_filter_timer);
COUNTER_UPDATE(_parent->_block_seek_counter, _reader->stats().block_seek_num);

Expand Down
3 changes: 0 additions & 3 deletions be/src/exec/scan_node.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ const string ScanNode::_s_materialize_tuple_timer = "MaterializeTupleTime(*)";
const string ScanNode::_s_per_read_thread_throughput_counter =
"PerReadThreadRawHdfsThroughput";
const string ScanNode::_s_num_disks_accessed_counter = "NumDiskAccess";
const string ScanNode::_s_scan_ranges_complete_counter = "ScanRangesComplete";
const string ScanNode::_s_scanner_thread_counters_prefix = "ScannerThreads";
const string ScanNode::_s_scanner_thread_total_wallclock_time =
"ScannerThreadsTotalWallClockTime";
Expand Down Expand Up @@ -59,8 +58,6 @@ Status ScanNode::prepare(RuntimeState* state) {
_bytes_read_counter,
_read_timer),
"");
_scan_ranges_complete_counter =
ADD_COUNTER(runtime_profile(), _s_scan_ranges_complete_counter, TUnit::UNIT);
_num_disks_accessed_counter =
ADD_COUNTER(runtime_profile(), _s_num_disks_accessed_counter, TUnit::UNIT);

Expand Down
5 changes: 0 additions & 5 deletions be/src/exec/scan_node.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,9 +108,6 @@ class ScanNode : public ExecNode {
RuntimeProfile::Counter* materialize_tuple_timer() const {
return _materialize_tuple_timer;
}
RuntimeProfile::Counter* scan_ranges_complete_counter() const {
return _scan_ranges_complete_counter;
}
RuntimeProfile::ThreadCounters* scanner_thread_counters() const {
return _scanner_thread_counters;
}
Expand All @@ -123,7 +120,6 @@ class ScanNode : public ExecNode {
static const std::string _s_per_read_thread_throughput_counter;
static const std::string _s_num_disks_accessed_counter;
static const std::string _s_materialize_tuple_timer;
static const std::string _s_scan_ranges_complete_counter;
static const std::string _s_scanner_thread_counters_prefix;
static const std::string _s_scanner_thread_total_wallclock_time;
static const std::string _s_average_io_mgr_queue_capacity;
Expand All @@ -140,7 +136,6 @@ class ScanNode : public ExecNode {
RuntimeProfile::Counter* _per_read_thread_throughput_counter;
RuntimeProfile::Counter* _num_disks_accessed_counter;
RuntimeProfile::Counter* _materialize_tuple_timer; // time writing tuple slots
RuntimeProfile::Counter* _scan_ranges_complete_counter;
// Aggregated scanner thread counters
RuntimeProfile::ThreadCounters* _scanner_thread_counters;
RuntimeProfile::Counter* _num_scanner_threads_started_counter;
Expand Down
2 changes: 1 addition & 1 deletion be/src/olap/file_stream.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,9 +98,9 @@ OLAPStatus ReadOnlyFileStream::_assure_data() {
OLAP_LOG_WARNING("fail to decompress err=%d", res);
return res;
}
_stats->uncompressed_bytes_read += _compressed_helper->limit();
}
}
_stats->uncompressed_bytes_read += _compressed_helper->limit();

_uncompressed = _compressed_helper;
_current_compress_position = file_cursor_used;
Expand Down
7 changes: 4 additions & 3 deletions be/src/olap/olap_common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,9 +243,9 @@ struct OlapReaderStatistics {
// total read bytes in memory
int64_t bytes_read = 0;

int64_t block_load_ns = 0;
int64_t block_load_ns = 0;
int64_t blocks_load = 0;
int64_t block_fetch_ns = 0;
int64_t block_fetch_ns = 0; // time of rowset reader's `next_batch()` call
int64_t block_seek_num = 0;
int64_t block_seek_ns = 0;
int64_t block_convert_ns = 0;
Expand All @@ -255,6 +255,7 @@ struct OlapReaderStatistics {
int64_t rows_vec_cond_filtered = 0;
int64_t vec_cond_ns = 0;

int64_t rows_key_range_filtered = 0;
int64_t rows_stats_filtered = 0;
int64_t rows_bf_filtered = 0;
int64_t rows_del_filtered = 0;
Expand All @@ -264,7 +265,7 @@ struct OlapReaderStatistics {
int64_t total_pages_num = 0;
int64_t cached_pages_num = 0;

int64_t bitmap_index_filter_count = 0;
int64_t rows_bitmap_index_filtered = 0;
int64_t bitmap_index_filter_timer = 0;
};

Expand Down
2 changes: 1 addition & 1 deletion be/src/olap/rowset/beta_rowset_reader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,10 +114,10 @@ OLAPStatus BetaRowsetReader::init(RowsetReaderContext* read_context) {
}

OLAPStatus BetaRowsetReader::next_block(RowBlock** block) {
SCOPED_RAW_TIMER(&_stats->block_fetch_ns);
// read next input block
_input_block->clear();
{
SCOPED_RAW_TIMER(&_stats->block_fetch_ns);
auto s = _iterator->next_batch(_input_block.get());
if (!s.ok()) {
if (s.is_end_of_file()) {
Expand Down
2 changes: 2 additions & 0 deletions be/src/olap/rowset/segment_v2/page_io.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,8 @@ Status PageIO::read_and_decompress_page(const PageReadOptions& opts,
page = std::move(decompressed_page);
page_slice = Slice(page.get(), footer->uncompressed_size() + footer_size + 4);
opts.stats->uncompressed_bytes_read += page_slice.size;
} else {
opts.stats->uncompressed_bytes_read += body_size;
}

*body = Slice(page_slice.data, page_slice.size - 4 - footer_size);
Expand Down
4 changes: 3 additions & 1 deletion be/src/olap/rowset/segment_v2/segment_iterator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,9 @@ Status SegmentIterator::_get_row_ranges_by_keys() {
RowRanges::ranges_union(result_ranges, row_range, &result_ranges);
}
// pre-condition: _row_ranges == [0, num_rows)
size_t pre_size = _row_bitmap.cardinality();
_row_bitmap = RowRanges::ranges_to_roaring(result_ranges);
_opts.stats->rows_key_range_filtered += (pre_size - _row_bitmap.cardinality());
DorisMetrics::instance()->segment_rows_by_short_key.increment(_row_bitmap.cardinality());

return Status::OK();
Expand Down Expand Up @@ -301,7 +303,7 @@ Status SegmentIterator::_apply_bitmap_index() {
}
}
_col_predicates = std::move(remaining_predicates);
_opts.stats->bitmap_index_filter_count += (input_rows - _row_bitmap.cardinality());
_opts.stats->rows_bitmap_index_filtered += (input_rows - _row_bitmap.cardinality());
return Status::OK();
}

Expand Down
Loading