From 547326edb644f872d8ca8b9fc7ff88f683d6936b Mon Sep 17 00:00:00 2001 From: "zihe.liu" Date: Wed, 25 Oct 2023 15:25:22 +0800 Subject: [PATCH] [Enhancement] Make scan metrics not duplicated for tablet internal parallel Signed-off-by: zihe.liu --- be/src/connector/lake_connector.cpp | 4 + be/src/exec/pipeline/scan/morsel.cpp | 50 ++- be/src/exec/pipeline/scan/morsel.h | 34 +- .../exec/pipeline/scan/olap_chunk_source.cpp | 19 +- be/src/exec/pipeline/scan/olap_chunk_source.h | 2 + be/src/storage/olap_common.h | 1 + be/src/storage/rowset/rowid_range_option.cpp | 13 +- be/src/storage/rowset/rowid_range_option.h | 13 +- be/src/storage/rowset/rowset.cpp | 16 +- be/src/storage/rowset/rowset_options.h | 6 +- be/src/storage/rowset/segment.cpp | 4 +- be/src/storage/rowset/segment_iterator.cpp | 74 ++-- be/src/storage/rowset/segment_options.cpp | 1 + be/src/storage/rowset/segment_options.h | 1 + .../storage/rowset/short_key_range_option.h | 13 + be/src/storage/tablet_reader.cpp | 6 +- be/src/storage/tablet_reader_params.h | 6 +- be/src/util/runtime_profile.cpp | 46 +-- be/src/util/runtime_profile.h | 26 +- fe/fe-core/pom.xml | 2 +- .../com/starrocks/common/util/Counter.java | 5 + .../starrocks/common/util/RuntimeProfile.java | 43 +-- gensrc/script/gen_functions.py | 2 +- gensrc/thrift/RuntimeProfile.thrift | 6 + test/lib/sr_sql_lib.py | 78 +++-- .../R/test_profile | 316 ++++++++++++++++++ .../T/test_profile | 249 ++++++++++++++ 27 files changed, 888 insertions(+), 148 deletions(-) create mode 100644 test/sql/test_tablet_internal_parallel/R/test_profile create mode 100644 test/sql/test_tablet_internal_parallel/T/test_profile diff --git a/be/src/connector/lake_connector.cpp b/be/src/connector/lake_connector.cpp index f51f4084dcf59a..97c29a9150a4d4 100644 --- a/be/src/connector/lake_connector.cpp +++ b/be/src/connector/lake_connector.cpp @@ -132,6 +132,7 @@ class LakeDataSource final : public DataSource { RuntimeProfile::Counter* _seg_zm_filtered_counter = nullptr; RuntimeProfile::Counter* _seg_rt_filtered_counter = nullptr; RuntimeProfile::Counter* _sk_filtered_counter = nullptr; + RuntimeProfile::Counter* _rows_after_sk_filtered_counter = nullptr; RuntimeProfile::Counter* _block_seek_timer = nullptr; RuntimeProfile::Counter* _block_seek_counter = nullptr; RuntimeProfile::Counter* _block_load_timer = nullptr; @@ -542,6 +543,8 @@ void LakeDataSource::init_counter(RuntimeState* state) { _zm_filtered_counter = ADD_CHILD_COUNTER(_runtime_profile, "ZoneMapIndexFilterRows", TUnit::UNIT, segment_init_name); _sk_filtered_counter = ADD_CHILD_COUNTER(_runtime_profile, "ShortKeyFilterRows", TUnit::UNIT, segment_init_name); + _rows_after_sk_filtered_counter = + ADD_CHILD_COUNTER(_runtime_profile, "RemainingRowsAfterShortKeyFilter", TUnit::UNIT, segment_init_name); _rows_key_range_counter = ADD_CHILD_COUNTER(_runtime_profile, "ShortKeyRangeNumber", TUnit::UNIT, segment_init_name); _column_iterator_init_timer = ADD_CHILD_TIMER(_runtime_profile, "ColumnIteratorInit", segment_init_name); @@ -657,6 +660,7 @@ void LakeDataSource::update_counter() { COUNTER_UPDATE(_zm_filtered_counter, _reader->stats().rows_stats_filtered); COUNTER_UPDATE(_bf_filtered_counter, _reader->stats().rows_bf_filtered); COUNTER_UPDATE(_sk_filtered_counter, _reader->stats().rows_key_range_filtered); + COUNTER_UPDATE(_rows_after_sk_filtered_counter, _reader->stats().rows_after_key_range); COUNTER_UPDATE(_rows_key_range_counter, _reader->stats().rows_key_range_num); COUNTER_UPDATE(_bi_filtered_counter, _reader->stats().rows_bitmap_index_filtered); diff --git a/be/src/exec/pipeline/scan/morsel.cpp b/be/src/exec/pipeline/scan/morsel.cpp index 5c9c1c9f632cad..abd898f2d62559 100644 --- a/be/src/exec/pipeline/scan/morsel.cpp +++ b/be/src/exec/pipeline/scan/morsel.cpp @@ -35,12 +35,49 @@ namespace starrocks::pipeline { const std::vector Morsel::kEmptyRowsets; +class PhysicalSplitScanMorsel final : public ScanMorsel { +public: + PhysicalSplitScanMorsel(int32_t plan_node_id, const TScanRange& scan_range, RowidRangeOptionPtr rowid_range_option) + : ScanMorsel(plan_node_id, scan_range), _rowid_range_option(std::move(rowid_range_option)) {} + + ~PhysicalSplitScanMorsel() override = default; + + void init_tablet_reader_params(TabletReaderParams* params) override; + + const std::unordered_set& skip_min_max_metrics() const override { + static const std::unordered_set metrics{"ShortKeyFilterRows", "SegmentZoneMapFilterRows"}; + return metrics; + } + +private: + RowidRangeOptionPtr _rowid_range_option; +}; + +class LogicalSplitScanMorsel final : public ScanMorsel { +public: + LogicalSplitScanMorsel(int32_t plan_node_id, const TScanRange& scan_range, + ShortKeyRangesOptionPtr short_key_ranges_option) + : ScanMorsel(plan_node_id, scan_range), _short_key_ranges_option(std::move(short_key_ranges_option)) {} + + ~LogicalSplitScanMorsel() override = default; + + void init_tablet_reader_params(TabletReaderParams* params) override; + + const std::unordered_set& skip_min_max_metrics() const override { + static const std::unordered_set metrics{"ShortKeyFilterRows", "SegmentZoneMapFilterRows"}; + return metrics; + } + +private: + ShortKeyRangesOptionPtr _short_key_ranges_option; +}; + void PhysicalSplitScanMorsel::init_tablet_reader_params(TabletReaderParams* params) { params->rowid_range_option = _rowid_range_option; } void LogicalSplitScanMorsel::init_tablet_reader_params(TabletReaderParams* params) { - params->short_key_ranges = _short_key_ranges; + params->short_key_ranges_option = _short_key_ranges_option; } /// MorselQueueFactory. @@ -277,7 +314,9 @@ StatusOr PhysicalSplitMorselQueue::_try_get_split_from_sing << "[range=" << taken_range.to_string() << "] "; num_taken_rows += taken_range.span_size(); - rowid_range->add(_cur_rowset(), _cur_segment(), std::make_shared>(std::move(taken_range))); + rowid_range->add(_cur_rowset(), _cur_segment(), std::make_shared>(std::move(taken_range)), + _is_first_split_of_segment); + _is_first_split_of_segment = false; if (_is_last_split_of_current_morsel()) { return rowid_range; @@ -403,6 +442,8 @@ bool PhysicalSplitMorselQueue::_next_segment() { } Status PhysicalSplitMorselQueue::_init_segment() { + _is_first_split_of_segment = true; + // Load the meta of the new rowset and the index of the new segment。 if (0 == _segment_idx) { // Read a new tablet. @@ -588,7 +629,9 @@ StatusOr LogicalSplitMorselQueue::try_get() { auto* scan_morsel = down_cast(_morsels[_tablet_idx].get()); auto morsel = std::make_unique( - scan_morsel->get_plan_node_id(), *(scan_morsel->get_scan_range()), std::move(short_key_ranges)); + scan_morsel->get_plan_node_id(), *(scan_morsel->get_scan_range()), + std::make_shared(std::move(short_key_ranges), _is_first_split_of_tablet)); + _is_first_split_of_tablet = false; morsel->set_rowsets(_tablet_rowsets[_tablet_idx]); _inc_num_splits(_is_last_split_of_current_morsel()); return morsel; @@ -730,6 +773,7 @@ Status LogicalSplitMorselQueue::_init_tablet() { _block_ranges_per_seek_range.clear(); _num_rest_blocks_per_seek_range.clear(); _range_idx = 0; + _is_first_split_of_tablet = true; if (_tablet_idx == 0) { // All the tablets have the same schema, so parse seek range with the first table schema. diff --git a/be/src/exec/pipeline/scan/morsel.h b/be/src/exec/pipeline/scan/morsel.h index d5b5d79d952df9..3d5fff13dc078b 100644 --- a/be/src/exec/pipeline/scan/morsel.h +++ b/be/src/exec/pipeline/scan/morsel.h @@ -91,6 +91,11 @@ class Morsel { } } + virtual const std::unordered_set& skip_min_max_metrics() const { + static const std::unordered_set metrics; + return metrics; + } + private: int32_t _plan_node_id; int64_t _from_version = 0; @@ -139,33 +144,6 @@ class ScanMorsel : public Morsel { int64_t _version = 0; }; -class PhysicalSplitScanMorsel final : public ScanMorsel { -public: - PhysicalSplitScanMorsel(int32_t plan_node_id, const TScanRange& scan_range, RowidRangeOptionPtr rowid_range_option) - : ScanMorsel(plan_node_id, scan_range), _rowid_range_option(std::move(rowid_range_option)) {} - - ~PhysicalSplitScanMorsel() override = default; - - void init_tablet_reader_params(TabletReaderParams* params) override; - -private: - RowidRangeOptionPtr _rowid_range_option; -}; - -class LogicalSplitScanMorsel final : public ScanMorsel { -public: - LogicalSplitScanMorsel(int32_t plan_node_id, const TScanRange& scan_range, - std::vector short_key_ranges) - : ScanMorsel(plan_node_id, scan_range), _short_key_ranges(std::move(short_key_ranges)) {} - - ~LogicalSplitScanMorsel() override = default; - - void init_tablet_reader_params(TabletReaderParams* params) override; - -private: - std::vector _short_key_ranges; -}; - /// MorselQueueFactory. class MorselQueueFactory { public: @@ -418,6 +396,7 @@ class PhysicalSplitMorselQueue final : public SplitMorselQueue { std::vector> _tablet_rowsets; bool _has_init_any_segment = false; + bool _is_first_split_of_segment = true; size_t _rowset_idx = 0; size_t _segment_idx = 0; @@ -480,6 +459,7 @@ class LogicalSplitMorselQueue final : public SplitMorselQueue { std::vector> _tablet_rowsets; bool _has_init_any_tablet = false; + bool _is_first_split_of_tablet = true; // Used to allocate memory for _tablet_seek_ranges. MemPool _mempool; diff --git a/be/src/exec/pipeline/scan/olap_chunk_source.cpp b/be/src/exec/pipeline/scan/olap_chunk_source.cpp index 1d935e283284c6..784159dd14423a 100644 --- a/be/src/exec/pipeline/scan/olap_chunk_source.cpp +++ b/be/src/exec/pipeline/scan/olap_chunk_source.cpp @@ -81,6 +81,15 @@ Status OlapChunkSource::prepare(RuntimeState* state) { return Status::OK(); } +TCounterMinMaxType::type OlapChunkSource::_get_counter_min_max_type(const std::string& metric_name) { + const auto& skip_min_max_metrics = _morsel->skip_min_max_metrics(); + if (skip_min_max_metrics.find(metric_name) != skip_min_max_metrics.end()) { + return TCounterMinMaxType::SKIP_ALL; + } + + return TCounterMinMaxType::MIN_MAX_ALL; +} + void OlapChunkSource::_init_counter(RuntimeState* state) { _bytes_read_counter = ADD_COUNTER(_runtime_profile, "BytesRead", TUnit::BYTES); _rows_read_counter = ADD_COUNTER(_runtime_profile, "RowsRead", TUnit::UNIT); @@ -108,12 +117,17 @@ void OlapChunkSource::_init_counter(RuntimeState* state) { _bi_filtered_counter = ADD_CHILD_COUNTER(_runtime_profile, "BitmapIndexFilterRows", TUnit::UNIT, segment_init_name); _bf_filtered_counter = ADD_CHILD_COUNTER(_runtime_profile, "BloomFilterFilterRows", TUnit::UNIT, segment_init_name); _seg_zm_filtered_counter = - ADD_CHILD_COUNTER(_runtime_profile, "SegmentZoneMapFilterRows", TUnit::UNIT, segment_init_name); + ADD_CHILD_COUNTER_SKIP_MIN_MAX(_runtime_profile, "SegmentZoneMapFilterRows", TUnit::UNIT, + _get_counter_min_max_type("SegmentZoneMapFilterRows"), segment_init_name); _seg_rt_filtered_counter = ADD_CHILD_COUNTER(_runtime_profile, "SegmentRuntimeZoneMapFilterRows", TUnit::UNIT, segment_init_name); _zm_filtered_counter = ADD_CHILD_COUNTER(_runtime_profile, "ZoneMapIndexFilterRows", TUnit::UNIT, segment_init_name); - _sk_filtered_counter = ADD_CHILD_COUNTER(_runtime_profile, "ShortKeyFilterRows", TUnit::UNIT, segment_init_name); + _sk_filtered_counter = + ADD_CHILD_COUNTER_SKIP_MIN_MAX(_runtime_profile, "ShortKeyFilterRows", TUnit::UNIT, + _get_counter_min_max_type("ShortKeyFilterRows"), segment_init_name); + _rows_after_sk_filtered_counter = + ADD_CHILD_COUNTER(_runtime_profile, "RemainingRowsAfterShortKeyFilter", TUnit::UNIT, segment_init_name); _column_iterator_init_timer = ADD_CHILD_TIMER(_runtime_profile, "ColumnIteratorInit", segment_init_name); _bitmap_index_iterator_init_timer = ADD_CHILD_TIMER(_runtime_profile, "BitmapIndexIteratorInit", segment_init_name); _zone_map_filter_timer = ADD_CHILD_TIMER(_runtime_profile, "ZoneMapIndexFiter", segment_init_name); @@ -481,6 +495,7 @@ void OlapChunkSource::_update_counter() { COUNTER_UPDATE(_zm_filtered_counter, _reader->stats().rows_stats_filtered); COUNTER_UPDATE(_bf_filtered_counter, _reader->stats().rows_bf_filtered); COUNTER_UPDATE(_sk_filtered_counter, _reader->stats().rows_key_range_filtered); + COUNTER_UPDATE(_rows_after_sk_filtered_counter, _reader->stats().rows_after_key_range); COUNTER_UPDATE(_rows_key_range_counter, _reader->stats().rows_key_range_num); COUNTER_UPDATE(_read_pages_num_counter, _reader->stats().total_pages_num); diff --git a/be/src/exec/pipeline/scan/olap_chunk_source.h b/be/src/exec/pipeline/scan/olap_chunk_source.h index ac3c909fab0677..d31557796d5c10 100644 --- a/be/src/exec/pipeline/scan/olap_chunk_source.h +++ b/be/src/exec/pipeline/scan/olap_chunk_source.h @@ -59,6 +59,7 @@ class OlapChunkSource final : public ChunkSource { Status _init_scanner_columns(std::vector& scanner_columns); Status _init_unused_output_columns(const std::vector& unused_output_columns); Status _init_olap_reader(RuntimeState* state); + TCounterMinMaxType::type _get_counter_min_max_type(const std::string& metric_name); void _init_counter(RuntimeState* state); Status _init_global_dicts(TabletReaderParams* params); Status _read_chunk_from_storage([[maybe_unused]] RuntimeState* state, Chunk* chunk); @@ -134,6 +135,7 @@ class OlapChunkSource final : public ChunkSource { RuntimeProfile::Counter* _seg_zm_filtered_counter = nullptr; RuntimeProfile::Counter* _seg_rt_filtered_counter = nullptr; RuntimeProfile::Counter* _sk_filtered_counter = nullptr; + RuntimeProfile::Counter* _rows_after_sk_filtered_counter = nullptr; RuntimeProfile::Counter* _block_seek_timer = nullptr; RuntimeProfile::Counter* _block_seek_counter = nullptr; RuntimeProfile::Counter* _block_load_timer = nullptr; diff --git a/be/src/storage/olap_common.h b/be/src/storage/olap_common.h index 18cf2e9fe9ca8f..4e675e10f0d1c0 100644 --- a/be/src/storage/olap_common.h +++ b/be/src/storage/olap_common.h @@ -249,6 +249,7 @@ struct OlapReaderStatistics { int64_t segment_stats_filtered = 0; int64_t rows_key_range_filtered = 0; + int64_t rows_after_key_range = 0; int64_t rows_key_range_num = 0; int64_t rows_stats_filtered = 0; int64_t rows_bf_filtered = 0; diff --git a/be/src/storage/rowset/rowid_range_option.cpp b/be/src/storage/rowset/rowid_range_option.cpp index 9a8ccf2d20f9bd..5d601d1ea9c6a1 100644 --- a/be/src/storage/rowset/rowid_range_option.cpp +++ b/be/src/storage/rowset/rowid_range_option.cpp @@ -21,30 +21,31 @@ namespace starrocks { -void RowidRangeOption::add(const Rowset* rowset, const Segment* segment, SparseRangePtr rowid_range) { +void RowidRangeOption::add(const Rowset* rowset, const Segment* segment, SparseRangePtr rowid_range, + bool is_first_split_of_segment) { auto rowset_it = rowid_range_per_segment_per_rowset.find(rowset->rowset_id()); if (rowset_it == rowid_range_per_segment_per_rowset.end()) { rowset_it = rowid_range_per_segment_per_rowset.emplace(rowset->rowset_id(), SetgmentRowidRangeMap()).first; } auto& segment_map = rowset_it->second; - segment_map.emplace(segment->id(), std::move(rowid_range)); + segment_map.emplace(segment->id(), SegmentSplit{std::move(rowid_range), is_first_split_of_segment}); } -bool RowidRangeOption::match_rowset(const Rowset* rowset) const { +bool RowidRangeOption::contains_rowset(const Rowset* rowset) const { return rowid_range_per_segment_per_rowset.find(rowset->rowset_id()) != rowid_range_per_segment_per_rowset.end(); } -SparseRangePtr RowidRangeOption::get_segment_rowid_range(const Rowset* rowset, const Segment* segment) { +RowidRangeOption::SegmentSplit RowidRangeOption::get_segment_rowid_range(const Rowset* rowset, const Segment* segment) { auto rowset_it = rowid_range_per_segment_per_rowset.find(rowset->rowset_id()); if (rowset_it == rowid_range_per_segment_per_rowset.end()) { - return nullptr; + return {nullptr, false}; } auto& segment_map = rowset_it->second; auto segment_it = segment_map.find(segment->id()); if (segment_it == segment_map.end()) { - return nullptr; + return {nullptr, false}; } return segment_it->second; } diff --git a/be/src/storage/rowset/rowid_range_option.h b/be/src/storage/rowset/rowid_range_option.h index 8a3eea94dff06d..43d3521f0b6eaa 100644 --- a/be/src/storage/rowset/rowid_range_option.h +++ b/be/src/storage/rowset/rowid_range_option.h @@ -27,15 +27,20 @@ class Segment; // It represents a specific rowid range on the segment with `segment_id` of the rowset with `rowset_id`. struct RowidRangeOption { public: + struct SegmentSplit { + SparseRangePtr row_id_range; + bool is_first_split_of_segment; + }; + RowidRangeOption() = default; - void add(const Rowset* rowset, const Segment* segment, SparseRangePtr rowid_range); + void add(const Rowset* rowset, const Segment* segment, SparseRangePtr rowid_range, bool is_first_split_of_segment); - bool match_rowset(const Rowset* rowset) const; - SparseRangePtr get_segment_rowid_range(const Rowset* rowset, const Segment* segment); + bool contains_rowset(const Rowset* rowset) const; + SegmentSplit get_segment_rowid_range(const Rowset* rowset, const Segment* segment); public: - using SetgmentRowidRangeMap = std::unordered_map; + using SetgmentRowidRangeMap = std::unordered_map; using RowsetRowidRangeMap = std::map; RowsetRowidRangeMap rowid_range_per_segment_per_rowset; diff --git a/be/src/storage/rowset/rowset.cpp b/be/src/storage/rowset/rowset.cpp index 06307bcb0f5126..132130ab9b45d3 100644 --- a/be/src/storage/rowset/rowset.cpp +++ b/be/src/storage/rowset/rowset.cpp @@ -51,6 +51,7 @@ #include "storage/merge_iterator.h" #include "storage/projection_iterator.h" #include "storage/rowset/rowid_range_option.h" +#include "storage/rowset/short_key_range_option.h" #include "storage/storage_engine.h" #include "storage/tablet_manager.h" #include "storage/tablet_meta_manager.h" @@ -598,7 +599,9 @@ Status Rowset::get_segment_iterators(const Schema& schema, const RowsetReadOptio seg_options.tablet_id = rowset_meta()->tablet_id(); seg_options.rowsetid = rowset_meta()->rowset_id(); seg_options.dcg_loader = std::make_shared(options.meta); - seg_options.short_key_ranges = options.short_key_ranges; + if (options.short_key_ranges_option != nullptr) { + seg_options.short_key_ranges = options.short_key_ranges_option->short_key_ranges; + } if (options.runtime_state != nullptr) { seg_options.is_cancelled = &options.runtime_state->cancelled_ref(); } @@ -626,10 +629,17 @@ Status Rowset::get_segment_iterators(const Schema& schema, const RowsetReadOptio } if (options.rowid_range_option != nullptr) { - seg_options.rowid_range_option = options.rowid_range_option->get_segment_rowid_range(this, seg_ptr.get()); - if (seg_options.rowid_range_option == nullptr) { + auto [rowid_range, is_first_split_of_segment] = + options.rowid_range_option->get_segment_rowid_range(this, seg_ptr.get()); + if (rowid_range == nullptr) { continue; } + seg_options.rowid_range_option = std::move(rowid_range); + seg_options.is_first_split_of_segment = is_first_split_of_segment; + } else if (options.short_key_ranges_option != nullptr) { + seg_options.is_first_split_of_segment = options.short_key_ranges_option->is_first_split_of_tablet; + } else { + seg_options.is_first_split_of_segment = true; } auto res = seg_ptr->new_iterator(segment_schema, seg_options); diff --git a/be/src/storage/rowset/rowset_options.h b/be/src/storage/rowset/rowset_options.h index 04678270867fce..160b5928fea229 100644 --- a/be/src/storage/rowset/rowset_options.h +++ b/be/src/storage/rowset/rowset_options.h @@ -37,11 +37,11 @@ class TabletSchema; class ColumnPredicate; class DeletePredicates; struct RowidRangeOption; -struct ShortKeyRangeOption; +struct ShortKeyRangesOption; class RowsetReadOptions { using RowidRangeOptionPtr = std::shared_ptr; - using ShortKeyRangeOptionPtr = std::shared_ptr; + using ShortKeyRangesOptionPtr = std::shared_ptr; using PredicateList = std::vector; public: @@ -74,7 +74,7 @@ class RowsetReadOptions { const std::unordered_set* unused_output_column_ids = nullptr; RowidRangeOptionPtr rowid_range_option = nullptr; - std::vector short_key_ranges; + ShortKeyRangesOptionPtr short_key_ranges_option = nullptr; OlapRuntimeScanRangePruner runtime_range_pruner; diff --git a/be/src/storage/rowset/segment.cpp b/be/src/storage/rowset/segment.cpp index 2753370dcc5c50..7cd1d8ba2f3b2b 100644 --- a/be/src/storage/rowset/segment.cpp +++ b/be/src/storage/rowset/segment.cpp @@ -252,7 +252,9 @@ StatusOr Segment::_new_iterator(const Schema& schema, const Se if (!_column_readers.at(column_unique_id)->segment_zone_map_filter(pair.second)) { // skip segment zonemap filter when this segment has column files link to it. if (tablet_column.is_key() || _use_segment_zone_map_filter(read_options)) { - read_options.stats->segment_stats_filtered += _column_readers.at(column_unique_id)->num_rows(); + if (read_options.is_first_split_of_segment) { + read_options.stats->segment_stats_filtered += _column_readers.at(column_unique_id)->num_rows(); + } return Status::EndOfFile(strings::Substitute("End of file $0, empty iterator", _fname)); } else { break; diff --git a/be/src/storage/rowset/segment_iterator.cpp b/be/src/storage/rowset/segment_iterator.cpp index 6aceaee032c126..820fdff703161e 100644 --- a/be/src/storage/rowset/segment_iterator.cpp +++ b/be/src/storage/rowset/segment_iterator.cpp @@ -192,8 +192,8 @@ class SegmentIterator final : public ChunkIterator { template Status _init_column_iterators(const Schema& schema); Status _get_row_ranges_by_keys(); - Status _get_row_ranges_by_key_ranges(); - Status _get_row_ranges_by_short_key_ranges(); + StatusOr> _get_row_ranges_by_key_ranges(); + StatusOr> _get_row_ranges_by_short_key_ranges(); Status _get_row_ranges_by_zone_map(); Status _get_row_ranges_by_bloom_filter(); Status _get_row_ranges_by_rowid_range(); @@ -412,8 +412,8 @@ Status SegmentIterator::_init() { RETURN_IF_ERROR(_init_column_iterators(_schema)); // filter by index stage // Use indexes and predicates to filter some data page - RETURN_IF_ERROR(_get_row_ranges_by_keys()); RETURN_IF_ERROR(_get_row_ranges_by_rowid_range()); + RETURN_IF_ERROR(_get_row_ranges_by_keys()); RETURN_IF_ERROR(_apply_del_vector()); RETURN_IF_ERROR(_apply_bitmap_index()); RETURN_IF_ERROR(_get_row_ranges_by_zone_map()); @@ -653,28 +653,46 @@ void SegmentIterator::_init_column_predicates() { } Status SegmentIterator::_get_row_ranges_by_keys() { - StarRocksMetrics::instance()->segment_row_total.increment(num_rows()); + if (_opts.is_first_split_of_segment) { + StarRocksMetrics::instance()->segment_row_total.increment(num_rows()); + } SCOPED_RAW_TIMER(&_opts.stats->rows_key_range_filter_ns); - if (!_opts.short_key_ranges.empty()) { - RETURN_IF_ERROR(_get_row_ranges_by_short_key_ranges()); + const uint32_t prev_num_rows = _scan_range.span_size(); + const bool is_logical_split = !_opts.short_key_ranges.empty(); + + SparseRange<> scan_range_by_keys; + if (is_logical_split) { + ASSIGN_OR_RETURN(scan_range_by_keys, _get_row_ranges_by_short_key_ranges()); _opts.stats->rows_key_range_num += _opts.short_key_ranges.size(); } else { - RETURN_IF_ERROR(_get_row_ranges_by_key_ranges()); + ASSIGN_OR_RETURN(scan_range_by_keys, _get_row_ranges_by_key_ranges()); } - _opts.stats->rows_key_range_filtered += num_rows() - _scan_range.span_size(); + _scan_range &= scan_range_by_keys; + + if (!is_logical_split) { + _opts.stats->rows_key_range_filtered += prev_num_rows - _scan_range.span_size(); + } else { + _opts.stats->rows_key_range_filtered += -static_cast(_scan_range.span_size()); + if (_opts.is_first_split_of_segment) { + _opts.stats->rows_key_range_filtered += prev_num_rows; + } + } + _opts.stats->rows_after_key_range += _scan_range.span_size(); StarRocksMetrics::instance()->segment_rows_by_short_key.increment(_scan_range.span_size()); + return Status::OK(); } -Status SegmentIterator::_get_row_ranges_by_key_ranges() { +StatusOr> SegmentIterator::_get_row_ranges_by_key_ranges() { DCHECK(_opts.short_key_ranges.empty()); - DCHECK_EQ(0, _scan_range.span_size()); + + SparseRange<> res; if (_opts.ranges.empty()) { - _scan_range.add(Range<>(0, num_rows())); - return Status::OK(); + res.add(Range<>(0, num_rows())); + return res; } RETURN_IF_ERROR(_segment->load_index(_skip_fill_data_cache())); @@ -691,21 +709,22 @@ Status SegmentIterator::_get_row_ranges_by_key_ranges() { RETURN_IF_ERROR(_lookup_ordinal(range.lower(), range.inclusive_lower(), upper_rowid, &lower_rowid)); } if (lower_rowid <= upper_rowid) { - _scan_range.add(Range{lower_rowid, upper_rowid}); + res.add(Range{lower_rowid, upper_rowid}); } } - return Status::OK(); + return res; } -Status SegmentIterator::_get_row_ranges_by_short_key_ranges() { +StatusOr> SegmentIterator::_get_row_ranges_by_short_key_ranges() { DCHECK(!_opts.short_key_ranges.empty()); - DCHECK_EQ(0, _scan_range.span_size()); + + SparseRange<> res; if (_opts.short_key_ranges.size() == 1 && _opts.short_key_ranges[0]->lower->is_infinite() && _opts.short_key_ranges[0]->upper->is_infinite()) { - _scan_range.add(Range<>(0, num_rows())); - return Status::OK(); + res.add(Range<>(0, num_rows())); + return res; } RETURN_IF_ERROR(_segment->load_index(_skip_fill_data_cache())); @@ -736,11 +755,11 @@ Status SegmentIterator::_get_row_ranges_by_short_key_ranges() { } if (lower_rowid <= upper_rowid) { - _scan_range.add(Range{lower_rowid, upper_rowid}); + res.add(Range{lower_rowid, upper_rowid}); } } - return Status::OK(); + return res; } Status SegmentIterator::_get_row_ranges_by_zone_map() { @@ -1809,8 +1828,19 @@ Status SegmentIterator::_get_row_ranges_by_bloom_filter() { } Status SegmentIterator::_get_row_ranges_by_rowid_range() { - RETURN_IF(_opts.rowid_range_option == nullptr || _scan_range.empty(), Status::OK()); - _scan_range = _scan_range.intersection(*_opts.rowid_range_option); + DCHECK_EQ(0, _scan_range.span_size()); + + if (_opts.rowid_range_option == nullptr) { + _scan_range.add(Range<>(0, num_rows())); + } else { + _scan_range |= (*_opts.rowid_range_option); + + _opts.stats->rows_key_range_filtered += -static_cast(_scan_range.span_size()); + if (_opts.is_first_split_of_segment) { + _opts.stats->rows_key_range_filtered += num_rows(); + } + } + return Status::OK(); } diff --git a/be/src/storage/rowset/segment_options.cpp b/be/src/storage/rowset/segment_options.cpp index 7553a18868847a..061f6d4d2c84cf 100644 --- a/be/src/storage/rowset/segment_options.cpp +++ b/be/src/storage/rowset/segment_options.cpp @@ -46,6 +46,7 @@ Status SegmentReadOptions::convert_to(SegmentReadOptions* dst, const std::vector dst->global_dictmaps = global_dictmaps; dst->rowid_range_option = rowid_range_option; dst->short_key_ranges = short_key_ranges; + dst->is_first_split_of_segment = is_first_split_of_segment; return Status::OK(); } diff --git a/be/src/storage/rowset/segment_options.h b/be/src/storage/rowset/segment_options.h index 9001dd06333b00..816e68d6597d9b 100644 --- a/be/src/storage/rowset/segment_options.h +++ b/be/src/storage/rowset/segment_options.h @@ -82,6 +82,7 @@ class SegmentReadOptions { bool has_delete_pred = false; + bool is_first_split_of_segment = true; SparseRangePtr rowid_range_option = nullptr; std::vector short_key_ranges; diff --git a/be/src/storage/rowset/short_key_range_option.h b/be/src/storage/rowset/short_key_range_option.h index fd49e0495326b1..10889228e74912 100644 --- a/be/src/storage/rowset/short_key_range_option.h +++ b/be/src/storage/rowset/short_key_range_option.h @@ -26,6 +26,10 @@ class Schema; using SchemaPtr = std::shared_ptr; struct ShortKeyOption; using ShortKeyOptionPtr = std::unique_ptr; +struct ShortKeyRangeOption; +using ShortKeyRangeOptionPtr = std::shared_ptr; +struct ShortKeyRangesOption; +using ShortKeyRangesOptionPtr = std::shared_ptr; // ShortKeyOption represents a sub key range endpoint splitted from a key range. // It could be a completed tuple key, or a short key with specific short_key_schema. @@ -65,4 +69,13 @@ struct ShortKeyRangeOption { const ShortKeyOptionPtr upper; }; +struct ShortKeyRangesOption { +public: + ShortKeyRangesOption(vector&& short_key_ranges, bool is_first_split_of_tablet) + : short_key_ranges(std::move(short_key_ranges)), is_first_split_of_tablet(is_first_split_of_tablet) {} + + std::vector short_key_ranges; + const bool is_first_split_of_tablet; +}; + } // namespace starrocks diff --git a/be/src/storage/tablet_reader.cpp b/be/src/storage/tablet_reader.cpp index f8c480072b0b0c..c83c357086f95e 100644 --- a/be/src/storage/tablet_reader.cpp +++ b/be/src/storage/tablet_reader.cpp @@ -208,7 +208,7 @@ Status TabletReader::_init_collector_for_pk_index_read() { return Status::InternalError(strings::Substitute("segment_idx out of range tablet:$0 $1 >= $2", _tablet->tablet_id(), segment_idx, rowset->num_segments())); } - rs_opts.rowid_range_option->add(rowset.get(), rowset->segments()[segment_idx].get(), rowid_range); + rs_opts.rowid_range_option->add(rowset.get(), rowset->segments()[segment_idx].get(), rowid_range, true); std::vector iters; RETURN_IF_ERROR(rowset->get_segment_iterators(schema(), rs_opts, &iters)); @@ -277,11 +277,11 @@ Status TabletReader::get_segment_iterators(const TabletReaderParams& params, std } rs_opts.meta = _tablet->data_dir()->get_meta(); rs_opts.rowid_range_option = params.rowid_range_option; - rs_opts.short_key_ranges = params.short_key_ranges; + rs_opts.short_key_ranges_option = params.short_key_ranges_option; SCOPED_RAW_TIMER(&_stats.create_segment_iter_ns); for (auto& rowset : _rowsets) { - if (params.rowid_range_option != nullptr && !params.rowid_range_option->match_rowset(rowset.get())) { + if (params.rowid_range_option != nullptr && !params.rowid_range_option->contains_rowset(rowset.get())) { continue; } diff --git a/be/src/storage/tablet_reader_params.h b/be/src/storage/tablet_reader_params.h index 4881075bd2590d..ed761bdf5e02f4 100644 --- a/be/src/storage/tablet_reader_params.h +++ b/be/src/storage/tablet_reader_params.h @@ -33,8 +33,8 @@ class RuntimeState; class ColumnPredicate; struct RowidRangeOption; using RowidRangeOptionPtr = std::shared_ptr; -struct ShortKeyRangeOption; -using ShortKeyRangeOptionPtr = std::shared_ptr; +struct ShortKeyRangesOption; +using ShortKeyRangesOptionPtr = std::shared_ptr; static inline std::unordered_set EMPTY_FILTERED_COLUMN_IDS; @@ -77,7 +77,7 @@ struct TabletReaderParams { const std::unordered_set* unused_output_column_ids = &EMPTY_FILTERED_COLUMN_IDS; RowidRangeOptionPtr rowid_range_option = nullptr; - std::vector short_key_ranges; + ShortKeyRangesOptionPtr short_key_ranges_option = nullptr; bool sorted_by_keys_per_tablet = false; OlapRuntimeScanRangePruner runtime_range_pruner; diff --git a/be/src/util/runtime_profile.cpp b/be/src/util/runtime_profile.cpp index 88fa4e4de9c39c..7927a7aaaaf367 100644 --- a/be/src/util/runtime_profile.cpp +++ b/be/src/util/runtime_profile.cpp @@ -812,9 +812,6 @@ RuntimeProfile* RuntimeProfile::merge_isomorphic_profiles(ObjectPool* obj_pool, bool require_identical) { DCHECK(!profiles.empty()); - static const std::string MERGED_INFO_PREFIX_MIN = "__MIN_OF_"; - static const std::string MERGED_INFO_PREFIX_MAX = "__MAX_OF_"; - // all metrics will be merged into the first profile auto* merged_profile = obj_pool->add(new RuntimeProfile(profiles[0]->name(), profiles[0]->_is_averaged_profile)); @@ -926,20 +923,23 @@ RuntimeProfile* RuntimeProfile::merge_isomorphic_profiles(ObjectPool* obj_pool, break; } - auto* min_counter = profile->get_counter(strings::Substitute("$0$1", MERGED_INFO_PREFIX_MIN, name)); - if (min_counter != nullptr) { - already_merged = true; - if (min_counter->value() < min_value) { - min_value = min_counter->value(); + if (!counter->skip_min_max()) { + auto* min_counter = profile->get_counter(strings::Substitute("$0$1", MERGED_INFO_PREFIX_MIN, name)); + if (min_counter != nullptr) { + already_merged = true; + if (min_counter->value() < min_value) { + min_value = min_counter->value(); + } } - } - auto* max_counter = profile->get_counter(strings::Substitute("$0$1", MERGED_INFO_PREFIX_MAX, name)); - if (max_counter != nullptr) { - already_merged = true; - if (max_counter->value() > max_value) { - max_value = max_counter->value(); + auto* max_counter = profile->get_counter(strings::Substitute("$0$1", MERGED_INFO_PREFIX_MAX, name)); + if (max_counter != nullptr) { + already_merged = true; + if (max_counter->value() > max_value) { + max_value = max_counter->value(); + } } } + counters.push_back(counter); } @@ -966,14 +966,16 @@ RuntimeProfile* RuntimeProfile::merge_isomorphic_profiles(ObjectPool* obj_pool, merged_counter->set(merged_value); - auto* min_counter = - merged_profile->add_child_counter(strings::Substitute("$0$1", MERGED_INFO_PREFIX_MIN, name), - type, merged_counter->strategy(), name); - auto* max_counter = - merged_profile->add_child_counter(strings::Substitute("$0$1", MERGED_INFO_PREFIX_MAX, name), - type, merged_counter->strategy(), name); - min_counter->set(min_value); - max_counter->set(max_value); + if (!merged_counter->skip_min_max()) { + auto* min_counter = + merged_profile->add_child_counter(strings::Substitute("$0$1", MERGED_INFO_PREFIX_MIN, name), + type, merged_counter->strategy(), name); + auto* max_counter = + merged_profile->add_child_counter(strings::Substitute("$0$1", MERGED_INFO_PREFIX_MAX, name), + type, merged_counter->strategy(), name); + min_counter->set(min_value); + max_counter->set(max_value); + } } } } diff --git a/be/src/util/runtime_profile.h b/be/src/util/runtime_profile.h index 16b5df43229903..94d62f791b8d08 100644 --- a/be/src/util/runtime_profile.h +++ b/be/src/util/runtime_profile.h @@ -80,6 +80,10 @@ inline unsigned long long operator"" _ms(unsigned long long x) { (profile)->add_child_counter(name, type, RuntimeProfile::Counter::create_strategy(type), parent) #define ADD_CHILD_COUNTER_SKIP_MERGE(profile, name, type, merge_type, parent) \ (profile)->add_child_counter(name, type, RuntimeProfile::Counter::create_strategy(type, merge_type), parent) +#define ADD_CHILD_COUNTER_SKIP_MIN_MAX(profile, name, type, min_max_type, parent) \ + (profile)->add_child_counter( \ + name, type, RuntimeProfile::Counter::create_strategy(type, TCounterMergeType::MERGE_ALL, 0, min_max_type), \ + parent) #define ADD_CHILD_TIMER_THESHOLD(profile, name, parent, threshold) \ (profile)->add_child_counter( \ name, TUnit::TIME_NS, \ @@ -122,23 +126,29 @@ class ObjectPool; // Thread-safe. class RuntimeProfile { public: + inline static const std::string MERGED_INFO_PREFIX_MIN = "__MIN_OF_"; + inline static const std::string MERGED_INFO_PREFIX_MAX = "__MAX_OF_"; + class Counter { public: - static TCounterStrategy create_strategy(TCounterAggregateType::type aggregate_type, - TCounterMergeType::type merge_type = TCounterMergeType::MERGE_ALL, - int64_t display_threshold = 0) { + static TCounterStrategy create_strategy( + TCounterAggregateType::type aggregate_type, + TCounterMergeType::type merge_type = TCounterMergeType::MERGE_ALL, int64_t display_threshold = 0, + TCounterMinMaxType::type min_max_type = TCounterMinMaxType::MIN_MAX_ALL) { TCounterStrategy strategy; strategy.aggregate_type = aggregate_type; strategy.merge_type = merge_type; strategy.display_threshold = display_threshold; + strategy.min_max_type = min_max_type; return strategy; } - static TCounterStrategy create_strategy(TUnit::type type, - TCounterMergeType::type merge_type = TCounterMergeType::MERGE_ALL, - int64_t display_threshold = 0) { + static TCounterStrategy create_strategy( + TUnit::type type, TCounterMergeType::type merge_type = TCounterMergeType::MERGE_ALL, + int64_t display_threshold = 0, + TCounterMinMaxType::type min_max_type = TCounterMinMaxType::MIN_MAX_ALL) { auto aggregate_type = is_time_type(type) ? TCounterAggregateType::AVG : TCounterAggregateType::SUM; - return create_strategy(aggregate_type, merge_type, display_threshold); + return create_strategy(aggregate_type, merge_type, display_threshold, min_max_type); } explicit Counter(TUnit::type type, int64_t value = 0) @@ -179,6 +189,8 @@ class RuntimeProfile { _strategy.merge_type == TCounterMergeType::SKIP_FIRST_MERGE; } + bool skip_min_max() const { return _strategy.min_max_type == TCounterMinMaxType::SKIP_ALL; } + int64_t display_threshold() const { return _strategy.display_threshold; } private: diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index eca2093db2cfa5..c04d18e8522acd 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -73,7 +73,7 @@ under the License. - ${env.STARROCKS_THIRDPARTY}/installed/bin/thrift + thrift diff --git a/fe/fe-core/src/main/java/com/starrocks/common/util/Counter.java b/fe/fe-core/src/main/java/com/starrocks/common/util/Counter.java index e0b8dc56721163..c14edc9fdb325d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/common/util/Counter.java +++ b/fe/fe-core/src/main/java/com/starrocks/common/util/Counter.java @@ -36,6 +36,7 @@ import com.starrocks.thrift.TCounterAggregateType; import com.starrocks.thrift.TCounterMergeType; +import com.starrocks.thrift.TCounterMinMaxType; import com.starrocks.thrift.TCounterStrategy; import com.starrocks.thrift.TUnit; @@ -81,6 +82,10 @@ public boolean isSkipMerge() { || Objects.equals(strategy.merge_type, TCounterMergeType.SKIP_SECOND_MERGE); } + public boolean isSkipMinMax() { + return Objects.equals(strategy.min_max_type, TCounterMinMaxType.SKIP_ALL); + } + public void setStrategy(TCounterStrategy strategy) { this.strategy = strategy; } diff --git a/fe/fe-core/src/main/java/com/starrocks/common/util/RuntimeProfile.java b/fe/fe-core/src/main/java/com/starrocks/common/util/RuntimeProfile.java index 44e325bde912fb..f4ff3e070f2cb9 100644 --- a/fe/fe-core/src/main/java/com/starrocks/common/util/RuntimeProfile.java +++ b/fe/fe-core/src/main/java/com/starrocks/common/util/RuntimeProfile.java @@ -651,20 +651,23 @@ public static RuntimeProfile mergeIsomorphicProfiles(List profil break; } - Counter minCounter = profile.getCounter(MERGED_INFO_PREFIX_MIN + name); - if (minCounter != null) { - alreadyMerged = true; - if (minCounter.getValue() < minValue) { - minValue = minCounter.getValue(); + if (!counter.isSkipMinMax()) { + Counter minCounter = profile.getCounter(MERGED_INFO_PREFIX_MIN + name); + if (minCounter != null) { + alreadyMerged = true; + if (minCounter.getValue() < minValue) { + minValue = minCounter.getValue(); + } } - } - Counter maxCounter = profile.getCounter(MERGED_INFO_PREFIX_MAX + name); - if (maxCounter != null) { - alreadyMerged = true; - if (maxCounter.getValue() > maxValue) { - maxValue = maxCounter.getValue(); + Counter maxCounter = profile.getCounter(MERGED_INFO_PREFIX_MAX + name); + if (maxCounter != null) { + alreadyMerged = true; + if (maxCounter.getValue() > maxValue) { + maxValue = maxCounter.getValue(); + } } } + counters.add(counter); } Counter mergedCounter; @@ -688,14 +691,16 @@ public static RuntimeProfile mergeIsomorphicProfiles(List profil } mergedCounter.setValue(mergedValue); - Counter minCounter = - mergedProfile.addCounter(MERGED_INFO_PREFIX_MIN + name, type, mergedCounter.getStrategy(), - name); - Counter maxCounter = - mergedProfile.addCounter(MERGED_INFO_PREFIX_MAX + name, type, mergedCounter.getStrategy(), - name); - minCounter.setValue(minValue); - maxCounter.setValue(maxValue); + if (!mergedCounter.isSkipMinMax()) { + Counter minCounter = + mergedProfile.addCounter(MERGED_INFO_PREFIX_MIN + name, type, mergedCounter.getStrategy(), + name); + Counter maxCounter = + mergedProfile.addCounter(MERGED_INFO_PREFIX_MAX + name, type, mergedCounter.getStrategy(), + name); + minCounter.setValue(minValue); + maxCounter.setValue(maxValue); + } } } diff --git a/gensrc/script/gen_functions.py b/gensrc/script/gen_functions.py index b4128cc605822d..5d34ec4b122808 100755 --- a/gensrc/script/gen_functions.py +++ b/gensrc/script/gen_functions.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # encoding: utf-8 """ diff --git a/gensrc/thrift/RuntimeProfile.thrift b/gensrc/thrift/RuntimeProfile.thrift index 6d003a49e4d27c..e6a50c11d5c887 100644 --- a/gensrc/thrift/RuntimeProfile.thrift +++ b/gensrc/thrift/RuntimeProfile.thrift @@ -32,10 +32,16 @@ enum TCounterMergeType { SKIP_SECOND_MERGE, } +enum TCounterMinMaxType { + MIN_MAX_ALL = 0, + SKIP_ALL = 1 +} + struct TCounterStrategy { 1: required TCounterAggregateType aggregate_type 2: required TCounterMergeType merge_type 3: required i64 display_threshold = 0 + 4: optional TCounterMinMaxType min_max_type = TCounterMinMaxType.MIN_MAX_ALL } // Counter data diff --git a/test/lib/sr_sql_lib.py b/test/lib/sr_sql_lib.py index 75c1a47a8d7ba2..4cc6770ad12133 100644 --- a/test/lib/sr_sql_lib.py +++ b/test/lib/sr_sql_lib.py @@ -37,8 +37,10 @@ import uuid import pymysql as _mysql +import requests from nose import tools from cup import log +from requests.auth import HTTPBasicAuth from lib import skip from lib import data_delete_lib @@ -51,7 +53,6 @@ common_data_path = os.path.join(root_path, "common/data") common_result_path = os.path.join(root_path, "common/result") - LOG_DIR = os.path.join(root_path, "log") if not os.path.exists(LOG_DIR): os.mkdir(LOG_DIR) @@ -478,12 +479,12 @@ def check(sql_id, sql, exp, act, order=False, ori_sql=None): return if any(re.compile(condition).search(sql) is not None for condition in skip.skip_res_cmd) or any( - condition in sql for condition in skip.skip_res_cmd + condition in sql for condition in skip.skip_res_cmd ): log.info("[%s.check] skip check" % sql_id) return - tmp_ori_sql = ori_sql[len(UNCHECK_FLAG) :] if ori_sql.startswith(UNCHECK_FLAG) else ori_sql + tmp_ori_sql = ori_sql[len(UNCHECK_FLAG):] if ori_sql.startswith(UNCHECK_FLAG) else ori_sql if tmp_ori_sql.startswith(SHELL_FLAG): tools.assert_equal(int(exp.split("\n")[0]), act[0], "shell %s error: %s" % (sql, act)) @@ -533,12 +534,12 @@ def check(sql_id, sql, exp, act, order=False, ori_sql=None): tools.assert_equal(str(exp), str(act)) else: if exp.startswith(REGEX_FLAG): - log.info("[check regex]: %s" % exp[len(REGEX_FLAG) :]) + log.info("[check regex]: %s" % exp[len(REGEX_FLAG):]) tools.assert_regexp_matches( r"%s" % str(act), - exp[len(REGEX_FLAG) :], + exp[len(REGEX_FLAG):], "sql result not match regex:\n- [SQL]: %s\n- [exp]: %s\n- [act]: %s\n---" - % (sql, exp[len(REGEX_FLAG) :], act), + % (sql, exp[len(REGEX_FLAG):], act), ) return @@ -664,7 +665,7 @@ def save_r_into_db(self, test_filepath, case_name, case_log, version): insert_round = 1 while len(new_log) > 0: current_log = new_log[: min(len(new_log), 65533)] - new_log = new_log[len(current_log) :] + new_log = new_log[len(current_log):] arg_dict = { "database_name": T_R_DB, @@ -693,7 +694,7 @@ def save_r_into_file(self, part=False): tools.assert_true(use_res["status"], "use db: [%s] error" % T_R_DB) self.execute_sql("set group_concat_max_len = 1024000;", True) - + # get records query_sql = """ select file, log_type, name, group_concat(log, ""), group_concat(hex(sequence), ",") @@ -921,7 +922,8 @@ def wait_for_pipe_finish(self, db_name, pipe_name, check_count=60): wait pipe load finish """ status = "" - show_sql = "select state from information_schema.pipes where database_name='{}' and pipe_name='{}'".format(db_name, pipe_name) + show_sql = "select state from information_schema.pipes where database_name='{}' and pipe_name='{}'".format( + db_name, pipe_name) count = 0 print("waiting for pipe {}.{} finish".format(db_name, pipe_name)) while count < check_count: @@ -938,7 +940,6 @@ def wait_for_pipe_finish(self, db_name, pipe_name, check_count=60): count += 1 tools.assert_equal("FINISHED", status, "didn't wait pipe finish") - def check_hit_materialized_view(self, query, mv_name): """ assert mv_name is hit in query @@ -1079,8 +1080,8 @@ def check_es_table_metadata_ready(self, table_name): return else: if ( - res["msg"][1].find("EsTable metadata has not been synced, Try it later") == -1 - and res["msg"][1].find("metadata failure: null") == -1 + res["msg"][1].find("EsTable metadata has not been synced, Try it later") == -1 + and res["msg"][1].find("metadata failure: null") == -1 ): log.info("==========check success: es table metadata is ready==========") return @@ -1092,15 +1093,15 @@ def check_es_table_metadata_ready(self, table_name): def _stream_load(self, label, database_name, table_name, filepath, headers=None, meta_sync=True): """ """ url = ( - "http://" - + self.mysql_host - + ":" - + self.http_port - + "/api/" - + database_name - + "/" - + table_name - + "/_stream_load" + "http://" + + self.mysql_host + + ":" + + self.http_port + + "/api/" + + database_name + + "/" + + table_name + + "/_stream_load" ) params = [ "curl", @@ -1246,6 +1247,15 @@ def execute_cmd(self, exec_url): ) return str(res) + def post_http_request(self, exec_url) -> str: + """Sends a POST request. + + :return: the response content. + """ + res = requests.post(exec_url, auth=HTTPBasicAuth(self.mysql_user, self.mysql_password)) + tools.assert_equal(200, res.status_code, f"failed to post http request [res={res}] [url={exec_url}]") + return res.content.decode("utf-8") + def manual_compact(self, database_name, table_name): sql = "show tablet from " + database_name + "." + table_name res = self.execute_sql(sql, "dml") @@ -1304,3 +1314,29 @@ def wait_analyze_finish(self, database_name, table_name, sql): counter = counter + 1 tools.assert_true(finished, "analyze timeout") + + def _get_backend_http_endpoints(self): + res = self.execute_sql("show backends;", ori=True) + tools.assert_true(res["status"], res["msg"]) + + backends = [] + for row in res["result"]: + backends.append({ + "ip": row[1], + "port": row[4], + }) + + return backends + + def update_be_config(self, key, value): + """Update the config to all the backends. + """ + backends = self._get_backend_http_endpoints() + for backend in backends: + exec_url = f"http://{backend['ip']}:{backend['port']}/api/update_config?{key}={value}" + print(f"fetch {exec_url}") + res = self.post_http_request(exec_url) + + res_json = json.loads(res) + tools.assert_dict_contains_subset({"status": "OK"}, res_json, + f"failed to update be config [response={res}] [url={exec_url}]") diff --git a/test/sql/test_tablet_internal_parallel/R/test_profile b/test/sql/test_tablet_internal_parallel/R/test_profile new file mode 100644 index 00000000000000..c098feaddffb48 --- /dev/null +++ b/test/sql/test_tablet_internal_parallel/R/test_profile @@ -0,0 +1,316 @@ +-- name: test_tablet_internal_parallel_profile @sequential +function: update_be_config("tablet_internal_parallel_min_splitted_scan_rows", "1") +-- result: +None +-- !result +function: update_be_config("tablet_internal_parallel_max_splitted_scan_rows", "1") +-- result: +None +-- !result +function: update_be_config("tablet_internal_parallel_max_splitted_scan_bytes", "1") +-- result: +None +-- !result +function: update_be_config("tablet_internal_parallel_min_scan_dop", "1") +-- result: +None +-- !result +create table dup_t ( + k1 int, + k2 int, + c1 string +) +duplicate key(k1, k2) +distributed by hash(k1) buckets 3 +properties("replication_num" = "1"); +-- result: +-- !result +insert into dup_t +select generate_series, generate_series + 10000, concat('a', generate_series) from TABLE(generate_series(0, 10000 - 1)); +-- result: +-- !result +create table uniq_t ( + k1 int, + k2 int, + c1 string +) +unique key(k1, k2) +distributed by hash(k1) buckets 3 +properties("replication_num" = "1"); +-- result: +-- !result +insert into uniq_t select * from dup_t; +-- result: +-- !result +select /*+SET_VAR(tablet_internal_parallel_mode="force_split")*/ sum(murmur_hash3_32(k2)) + sum(murmur_hash3_32(k1)) from dup_t where k1 between 10 and 1000; +-- result: +-31585402830 +-- !result +with + profile as ( + select unnest as line from (values(1))t(v) join unnest(split(get_query_profile(last_query_id()), "\n") ) + ), result as ( + -- format: "ShortKeyFilterRows: 9.009K (9009)" or "ShortKeyFilterRows: 100" + select "ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- ShortKeyFilterRows%" + UNION ALL + select "__MAX_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_ShortKeyFilterRows%" + UNION ALL + select "__MIN_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_ShortKeyFilterRows%" + UNION ALL + + select "SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- SegmentZoneMapFilterRows%" + UNION ALL + select "__MAX_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_SegmentZoneMapFilterRows%" + UNION ALL + select "__MIN_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_SegmentZoneMapFilterRows%" + UNION ALL + + select "RemainingRowsAfterShortKeyFilter" as `key`, regexp_extract(line, ".*- RemainingRowsAfterShortKeyFilter: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- RemainingRowsAfterShortKeyFilter%" + ) +select * from result order by `key`, value; +-- result: +RemainingRowsAfterShortKeyFilter 991 +SegmentZoneMapFilterRows 0 +ShortKeyFilterRows 9009 +-- !result +select /*+SET_VAR(enable_tablet_internal_parallel="false")*/ sum(murmur_hash3_32(k2)) + sum(murmur_hash3_32(k1)) from dup_t where k1 between 10 and 1000; +-- result: +-31585402830 +-- !result +with + profile as ( + select unnest as line from (values(1))t(v) join unnest(split(get_query_profile(last_query_id()), "\n") ) + ), result as ( + -- format: "ShortKeyFilterRows: 9.009K (9009)" or "ShortKeyFilterRows: 100" + select "ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- ShortKeyFilterRows%" + UNION ALL + select "__MAX_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_ShortKeyFilterRows%" + UNION ALL + select "__MIN_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_ShortKeyFilterRows%" + UNION ALL + + select "SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- SegmentZoneMapFilterRows%" + UNION ALL + select "__MAX_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_SegmentZoneMapFilterRows%" + UNION ALL + select "__MIN_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_SegmentZoneMapFilterRows%" + UNION ALL + + select "RemainingRowsAfterShortKeyFilter" as `key`, regexp_extract(line, ".*- RemainingRowsAfterShortKeyFilter: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- RemainingRowsAfterShortKeyFilter%" + ) +select * from result order by `key`, value; +-- result: +RemainingRowsAfterShortKeyFilter 991 +SegmentZoneMapFilterRows 0 +ShortKeyFilterRows 9009 +__MAX_OF_ShortKeyFilterRows 3021 +__MIN_OF_ShortKeyFilterRows 2986 +-- !result +select /*+SET_VAR(tablet_internal_parallel_mode="force_split")*/ sum(murmur_hash3_32(k2)) + sum(murmur_hash3_32(k1)) from uniq_t where k1 between 10 and 1000; +-- result: +-31585402830 +-- !result +with + profile as ( + select unnest as line from (values(1))t(v) join unnest(split(get_query_profile(last_query_id()), "\n") ) + ), result as ( + -- format: "ShortKeyFilterRows: 9.009K (9009)" or "ShortKeyFilterRows: 100" + select "ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- ShortKeyFilterRows%" + UNION ALL + select "__MAX_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_ShortKeyFilterRows%" + UNION ALL + select "__MIN_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_ShortKeyFilterRows%" + UNION ALL + + select "SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- SegmentZoneMapFilterRows%" + UNION ALL + select "__MAX_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_SegmentZoneMapFilterRows%" + UNION ALL + select "__MIN_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_SegmentZoneMapFilterRows%" + UNION ALL + + select "RemainingRowsAfterShortKeyFilter" as `key`, regexp_extract(line, ".*- RemainingRowsAfterShortKeyFilter: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- RemainingRowsAfterShortKeyFilter%" + ) +select * from result order by `key`, value; +-- result: +RemainingRowsAfterShortKeyFilter 991 +SegmentZoneMapFilterRows 0 +ShortKeyFilterRows 9009 +-- !result +select /*+SET_VAR(enable_tablet_internal_parallel="false")*/ sum(murmur_hash3_32(k2)) + sum(murmur_hash3_32(k1)) from uniq_t where k1 between 10 and 1000; +-- result: +-31585402830 +-- !result +with + profile as ( + select unnest as line from (values(1))t(v) join unnest(split(get_query_profile(last_query_id()), "\n") ) + ), result as ( + -- format: "ShortKeyFilterRows: 9.009K (9009)" or "ShortKeyFilterRows: 100" + select "ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- ShortKeyFilterRows%" + UNION ALL + select "__MAX_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_ShortKeyFilterRows%" + UNION ALL + select "__MIN_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_ShortKeyFilterRows%" + UNION ALL + + select "SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- SegmentZoneMapFilterRows%" + UNION ALL + select "__MAX_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_SegmentZoneMapFilterRows%" + UNION ALL + select "__MIN_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_SegmentZoneMapFilterRows%" + UNION ALL + + select "RemainingRowsAfterShortKeyFilter" as `key`, regexp_extract(line, ".*- RemainingRowsAfterShortKeyFilter: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- RemainingRowsAfterShortKeyFilter%" + ) +select * from result order by `key`, value; +-- result: +RemainingRowsAfterShortKeyFilter 991 +SegmentZoneMapFilterRows 0 +ShortKeyFilterRows 9009 +__MAX_OF_ShortKeyFilterRows 3021 +__MIN_OF_ShortKeyFilterRows 2986 +-- !result +select /*+SET_VAR(tablet_internal_parallel_mode="force_split")*/ sum(murmur_hash3_32(k2)) + sum(murmur_hash3_32(k1)) from dup_t where k1 between 20000 and 20100; +-- result: +None +-- !result +with + profile as ( + select unnest as line from (values(1))t(v) join unnest(split(get_query_profile(last_query_id()), "\n") ) + ), result as ( + -- format: "ShortKeyFilterRows: 9.009K (9009)" or "ShortKeyFilterRows: 100" + select "ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- ShortKeyFilterRows%" + UNION ALL + select "__MAX_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_ShortKeyFilterRows%" + UNION ALL + select "__MIN_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_ShortKeyFilterRows%" + UNION ALL + + select "SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- SegmentZoneMapFilterRows%" + UNION ALL + select "__MAX_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_SegmentZoneMapFilterRows%" + UNION ALL + select "__MIN_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_SegmentZoneMapFilterRows%" + UNION ALL + + select "RemainingRowsAfterShortKeyFilter" as `key`, regexp_extract(line, ".*- RemainingRowsAfterShortKeyFilter: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- RemainingRowsAfterShortKeyFilter%" + ) +select * from result order by `key`, value; +-- result: +RemainingRowsAfterShortKeyFilter 0 +SegmentZoneMapFilterRows 10000 +ShortKeyFilterRows 0 +-- !result +select /*+SET_VAR(enable_tablet_internal_parallel="false")*/ sum(murmur_hash3_32(k2)) + sum(murmur_hash3_32(k1)) from dup_t where k1 between 20000 and 20100; +-- result: +None +-- !result +with + profile as ( + select unnest as line from (values(1))t(v) join unnest(split(get_query_profile(last_query_id()), "\n") ) + ), result as ( + -- format: "ShortKeyFilterRows: 9.009K (9009)" or "ShortKeyFilterRows: 100" + select "ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- ShortKeyFilterRows%" + UNION ALL + select "__MAX_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_ShortKeyFilterRows%" + UNION ALL + select "__MIN_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_ShortKeyFilterRows%" + UNION ALL + + select "SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- SegmentZoneMapFilterRows%" + UNION ALL + select "__MAX_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_SegmentZoneMapFilterRows%" + UNION ALL + select "__MIN_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_SegmentZoneMapFilterRows%" + UNION ALL + + select "RemainingRowsAfterShortKeyFilter" as `key`, regexp_extract(line, ".*- RemainingRowsAfterShortKeyFilter: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- RemainingRowsAfterShortKeyFilter%" + ) +select * from result order by `key`, value; +-- result: +RemainingRowsAfterShortKeyFilter 0 +SegmentZoneMapFilterRows 10000 +ShortKeyFilterRows 0 +__MAX_OF_SegmentZoneMapFilterRows 3365 +__MIN_OF_SegmentZoneMapFilterRows 3302 +-- !result +select /*+SET_VAR(tablet_internal_parallel_mode="force_split")*/ sum(murmur_hash3_32(k2)) + sum(murmur_hash3_32(k1)) from uniq_t where k1 between 20000 and 20100; +-- result: +None +-- !result +with + profile as ( + select unnest as line from (values(1))t(v) join unnest(split(get_query_profile(last_query_id()), "\n") ) + ), result as ( + -- format: "ShortKeyFilterRows: 9.009K (9009)" or "ShortKeyFilterRows: 100" + select "ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- ShortKeyFilterRows%" + UNION ALL + select "__MAX_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_ShortKeyFilterRows%" + UNION ALL + select "__MIN_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_ShortKeyFilterRows%" + UNION ALL + + select "SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- SegmentZoneMapFilterRows%" + UNION ALL + select "__MAX_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_SegmentZoneMapFilterRows%" + UNION ALL + select "__MIN_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_SegmentZoneMapFilterRows%" + UNION ALL + + select "RemainingRowsAfterShortKeyFilter" as `key`, regexp_extract(line, ".*- RemainingRowsAfterShortKeyFilter: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- RemainingRowsAfterShortKeyFilter%" + ) +select * from result order by `key`, value; +-- result: +RemainingRowsAfterShortKeyFilter 0 +SegmentZoneMapFilterRows 10000 +ShortKeyFilterRows 0 +-- !result +select /*+SET_VAR(enable_tablet_internal_parallel="false")*/ sum(murmur_hash3_32(k2)) + sum(murmur_hash3_32(k1)) from uniq_t where k1 between 20000 and 20100; +-- result: +None +-- !result +with + profile as ( + select unnest as line from (values(1))t(v) join unnest(split(get_query_profile(last_query_id()), "\n") ) + ), result as ( + -- format: "ShortKeyFilterRows: 9.009K (9009)" or "ShortKeyFilterRows: 100" + select "ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- ShortKeyFilterRows%" + UNION ALL + select "__MAX_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_ShortKeyFilterRows%" + UNION ALL + select "__MIN_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_ShortKeyFilterRows%" + UNION ALL + + select "SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- SegmentZoneMapFilterRows%" + UNION ALL + select "__MAX_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_SegmentZoneMapFilterRows%" + UNION ALL + select "__MIN_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_SegmentZoneMapFilterRows%" + UNION ALL + + select "RemainingRowsAfterShortKeyFilter" as `key`, regexp_extract(line, ".*- RemainingRowsAfterShortKeyFilter: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- RemainingRowsAfterShortKeyFilter%" + ) +select * from result order by `key`, value; +-- result: +RemainingRowsAfterShortKeyFilter 0 +SegmentZoneMapFilterRows 10000 +ShortKeyFilterRows 0 +__MAX_OF_SegmentZoneMapFilterRows 3365 +__MIN_OF_SegmentZoneMapFilterRows 3302 +-- !result +function: update_be_config("tablet_internal_parallel_min_splitted_scan_rows", "16384") +-- result: +None +-- !result +function: update_be_config("tablet_internal_parallel_max_splitted_scan_rows", "1048576") +-- result: +None +-- !result +function: update_be_config("tablet_internal_parallel_max_splitted_scan_bytes", "536870912") +-- result: +None +-- !result +function: update_be_config("tablet_internal_parallel_min_scan_dop", "4") +-- result: +None +-- !result \ No newline at end of file diff --git a/test/sql/test_tablet_internal_parallel/T/test_profile b/test/sql/test_tablet_internal_parallel/T/test_profile new file mode 100644 index 00000000000000..e42c2845f33636 --- /dev/null +++ b/test/sql/test_tablet_internal_parallel/T/test_profile @@ -0,0 +1,249 @@ +-- name: test_tablet_internal_parallel_profile @sequential + +-- setup be config to minimize the split size. +function: update_be_config("tablet_internal_parallel_min_splitted_scan_rows", "1") +function: update_be_config("tablet_internal_parallel_max_splitted_scan_rows", "1") +function: update_be_config("tablet_internal_parallel_max_splitted_scan_bytes", "1") +function: update_be_config("tablet_internal_parallel_min_scan_dop", "1") + +-- prepare data. +create table dup_t ( + k1 int, + k2 int, + c1 string +) +duplicate key(k1, k2) +distributed by hash(k1) buckets 3 +properties("replication_num" = "1"); + +insert into dup_t +select generate_series, generate_series + 10000, concat('a', generate_series) from TABLE(generate_series(0, 10000 - 1)); + +create table uniq_t ( + k1 int, + k2 int, + c1 string +) +unique key(k1, k2) +distributed by hash(k1) buckets 3 +properties("replication_num" = "1"); + +insert into uniq_t select * from dup_t; + +-- 1. ShortKeyFilterRows for physical split. +select /*+SET_VAR(tablet_internal_parallel_mode="force_split")*/ sum(murmur_hash3_32(k2)) + sum(murmur_hash3_32(k1)) from dup_t where k1 between 10 and 1000; + +-- There are no min and max for "ShortKeyFilterRows". +with + profile as ( + select unnest as line from (values(1))t(v) join unnest(split(get_query_profile(last_query_id()), "\n") ) + ), result as ( + -- format: "ShortKeyFilterRows: 9.009K (9009)" or "ShortKeyFilterRows: 100" + select "ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- ShortKeyFilterRows%" + UNION ALL + select "__MAX_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_ShortKeyFilterRows%" + UNION ALL + select "__MIN_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_ShortKeyFilterRows%" + UNION ALL + + select "SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- SegmentZoneMapFilterRows%" + UNION ALL + select "__MAX_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_SegmentZoneMapFilterRows%" + UNION ALL + select "__MIN_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_SegmentZoneMapFilterRows%" + UNION ALL + + select "RemainingRowsAfterShortKeyFilter" as `key`, regexp_extract(line, ".*- RemainingRowsAfterShortKeyFilter: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- RemainingRowsAfterShortKeyFilter%" + ) +select * from result order by `key`, value; + + +select /*+SET_VAR(enable_tablet_internal_parallel="false")*/ sum(murmur_hash3_32(k2)) + sum(murmur_hash3_32(k1)) from dup_t where k1 between 10 and 1000; + +with + profile as ( + select unnest as line from (values(1))t(v) join unnest(split(get_query_profile(last_query_id()), "\n") ) + ), result as ( + -- format: "ShortKeyFilterRows: 9.009K (9009)" or "ShortKeyFilterRows: 100" + select "ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- ShortKeyFilterRows%" + UNION ALL + select "__MAX_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_ShortKeyFilterRows%" + UNION ALL + select "__MIN_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_ShortKeyFilterRows%" + UNION ALL + + select "SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- SegmentZoneMapFilterRows%" + UNION ALL + select "__MAX_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_SegmentZoneMapFilterRows%" + UNION ALL + select "__MIN_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_SegmentZoneMapFilterRows%" + UNION ALL + + select "RemainingRowsAfterShortKeyFilter" as `key`, regexp_extract(line, ".*- RemainingRowsAfterShortKeyFilter: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- RemainingRowsAfterShortKeyFilter%" + ) +select * from result order by `key`, value; + +-- 2. ShortKeyFilterRows for logical split. +select /*+SET_VAR(tablet_internal_parallel_mode="force_split")*/ sum(murmur_hash3_32(k2)) + sum(murmur_hash3_32(k1)) from uniq_t where k1 between 10 and 1000; + +-- There are no min and max for "ShortKeyFilterRows". +with + profile as ( + select unnest as line from (values(1))t(v) join unnest(split(get_query_profile(last_query_id()), "\n") ) + ), result as ( + -- format: "ShortKeyFilterRows: 9.009K (9009)" or "ShortKeyFilterRows: 100" + select "ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- ShortKeyFilterRows%" + UNION ALL + select "__MAX_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_ShortKeyFilterRows%" + UNION ALL + select "__MIN_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_ShortKeyFilterRows%" + UNION ALL + + select "SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- SegmentZoneMapFilterRows%" + UNION ALL + select "__MAX_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_SegmentZoneMapFilterRows%" + UNION ALL + select "__MIN_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_SegmentZoneMapFilterRows%" + UNION ALL + + select "RemainingRowsAfterShortKeyFilter" as `key`, regexp_extract(line, ".*- RemainingRowsAfterShortKeyFilter: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- RemainingRowsAfterShortKeyFilter%" + ) +select * from result order by `key`, value; + +select /*+SET_VAR(enable_tablet_internal_parallel="false")*/ sum(murmur_hash3_32(k2)) + sum(murmur_hash3_32(k1)) from uniq_t where k1 between 10 and 1000; + +with + profile as ( + select unnest as line from (values(1))t(v) join unnest(split(get_query_profile(last_query_id()), "\n") ) + ), result as ( + -- format: "ShortKeyFilterRows: 9.009K (9009)" or "ShortKeyFilterRows: 100" + select "ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- ShortKeyFilterRows%" + UNION ALL + select "__MAX_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_ShortKeyFilterRows%" + UNION ALL + select "__MIN_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_ShortKeyFilterRows%" + UNION ALL + + select "SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- SegmentZoneMapFilterRows%" + UNION ALL + select "__MAX_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_SegmentZoneMapFilterRows%" + UNION ALL + select "__MIN_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_SegmentZoneMapFilterRows%" + UNION ALL + + select "RemainingRowsAfterShortKeyFilter" as `key`, regexp_extract(line, ".*- RemainingRowsAfterShortKeyFilter: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- RemainingRowsAfterShortKeyFilter%" + ) +select * from result order by `key`, value; + + +-- 3. SegmentZoneMapFilterRows for physical split. +select /*+SET_VAR(tablet_internal_parallel_mode="force_split")*/ sum(murmur_hash3_32(k2)) + sum(murmur_hash3_32(k1)) from dup_t where k1 between 20000 and 20100; + +-- There are no min and max for "SegmentZoneMapFilterRows". +with + profile as ( + select unnest as line from (values(1))t(v) join unnest(split(get_query_profile(last_query_id()), "\n") ) + ), result as ( + -- format: "ShortKeyFilterRows: 9.009K (9009)" or "ShortKeyFilterRows: 100" + select "ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- ShortKeyFilterRows%" + UNION ALL + select "__MAX_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_ShortKeyFilterRows%" + UNION ALL + select "__MIN_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_ShortKeyFilterRows%" + UNION ALL + + select "SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- SegmentZoneMapFilterRows%" + UNION ALL + select "__MAX_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_SegmentZoneMapFilterRows%" + UNION ALL + select "__MIN_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_SegmentZoneMapFilterRows%" + UNION ALL + + select "RemainingRowsAfterShortKeyFilter" as `key`, regexp_extract(line, ".*- RemainingRowsAfterShortKeyFilter: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- RemainingRowsAfterShortKeyFilter%" + ) +select * from result order by `key`, value; + + +select /*+SET_VAR(enable_tablet_internal_parallel="false")*/ sum(murmur_hash3_32(k2)) + sum(murmur_hash3_32(k1)) from dup_t where k1 between 20000 and 20100; + +with + profile as ( + select unnest as line from (values(1))t(v) join unnest(split(get_query_profile(last_query_id()), "\n") ) + ), result as ( + -- format: "ShortKeyFilterRows: 9.009K (9009)" or "ShortKeyFilterRows: 100" + select "ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- ShortKeyFilterRows%" + UNION ALL + select "__MAX_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_ShortKeyFilterRows%" + UNION ALL + select "__MIN_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_ShortKeyFilterRows%" + UNION ALL + + select "SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- SegmentZoneMapFilterRows%" + UNION ALL + select "__MAX_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_SegmentZoneMapFilterRows%" + UNION ALL + select "__MIN_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_SegmentZoneMapFilterRows%" + UNION ALL + + select "RemainingRowsAfterShortKeyFilter" as `key`, regexp_extract(line, ".*- RemainingRowsAfterShortKeyFilter: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- RemainingRowsAfterShortKeyFilter%" + ) +select * from result order by `key`, value; + +-- 4. SegmentZoneMapFilterRows for logical split. +select /*+SET_VAR(tablet_internal_parallel_mode="force_split")*/ sum(murmur_hash3_32(k2)) + sum(murmur_hash3_32(k1)) from uniq_t where k1 between 20000 and 20100; + +-- There are no min and max for "SegmentZoneMapFilterRows". +with + profile as ( + select unnest as line from (values(1))t(v) join unnest(split(get_query_profile(last_query_id()), "\n") ) + ), result as ( + -- format: "ShortKeyFilterRows: 9.009K (9009)" or "ShortKeyFilterRows: 100" + select "ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- ShortKeyFilterRows%" + UNION ALL + select "__MAX_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_ShortKeyFilterRows%" + UNION ALL + select "__MIN_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_ShortKeyFilterRows%" + UNION ALL + + select "SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- SegmentZoneMapFilterRows%" + UNION ALL + select "__MAX_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_SegmentZoneMapFilterRows%" + UNION ALL + select "__MIN_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_SegmentZoneMapFilterRows%" + UNION ALL + + select "RemainingRowsAfterShortKeyFilter" as `key`, regexp_extract(line, ".*- RemainingRowsAfterShortKeyFilter: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- RemainingRowsAfterShortKeyFilter%" + ) +select * from result order by `key`, value; + +select /*+SET_VAR(enable_tablet_internal_parallel="false")*/ sum(murmur_hash3_32(k2)) + sum(murmur_hash3_32(k1)) from uniq_t where k1 between 20000 and 20100; + +with + profile as ( + select unnest as line from (values(1))t(v) join unnest(split(get_query_profile(last_query_id()), "\n") ) + ), result as ( + -- format: "ShortKeyFilterRows: 9.009K (9009)" or "ShortKeyFilterRows: 100" + select "ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- ShortKeyFilterRows%" + UNION ALL + select "__MAX_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_ShortKeyFilterRows%" + UNION ALL + select "__MIN_OF_ShortKeyFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_ShortKeyFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_ShortKeyFilterRows%" + UNION ALL + + select "SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- SegmentZoneMapFilterRows%" + UNION ALL + select "__MAX_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MAX_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MAX_OF_SegmentZoneMapFilterRows%" + UNION ALL + select "__MIN_OF_SegmentZoneMapFilterRows" as `key`, regexp_extract(line, ".*- __MIN_OF_SegmentZoneMapFilterRows: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- __MIN_OF_SegmentZoneMapFilterRows%" + UNION ALL + + select "RemainingRowsAfterShortKeyFilter" as `key`, regexp_extract(line, ".*- RemainingRowsAfterShortKeyFilter: (?:.*\\()?(\\d+)\\)?", 1) as value from profile where line like "%- RemainingRowsAfterShortKeyFilter%" + ) +select * from result order by `key`, value; + + +-- reset be config to default values. +function: update_be_config("tablet_internal_parallel_min_splitted_scan_rows", "16384") +function: update_be_config("tablet_internal_parallel_max_splitted_scan_rows", "1048576") +function: update_be_config("tablet_internal_parallel_max_splitted_scan_bytes", "536870912") +function: update_be_config("tablet_internal_parallel_min_scan_dop", "4")