diff --git a/be/src/exprs/bloom_filter_func.h b/be/src/exprs/bloom_filter_func.h index e88f692a23db8a..de093dfa5285df 100644 --- a/be/src/exprs/bloom_filter_func.h +++ b/be/src/exprs/bloom_filter_func.h @@ -100,14 +100,14 @@ class BloomFilterFuncBase : public RuntimeFilterFuncBase { virtual ~BloomFilterFuncBase() = default; void init_params(const RuntimeFilterParams* params) { - _bloom_filter_length = - params->runtime_bloom_filter_min_size > 0 - ? std::max(params->bloom_filter_size, params->runtime_bloom_filter_min_size) - : params->bloom_filter_size; + _bloom_filter_length = params->bloom_filter_size; + _build_bf_exactly = params->build_bf_exactly; _runtime_bloom_filter_min_size = params->runtime_bloom_filter_min_size; + _runtime_bloom_filter_max_size = params->runtime_bloom_filter_max_size; _null_aware = params->null_aware; _bloom_filter_size_calculated_by_ndv = params->bloom_filter_size_calculated_by_ndv; + _limit_length(); } Status init_with_fixed_length() { return init_with_fixed_length(_bloom_filter_length); } @@ -128,17 +128,11 @@ class BloomFilterFuncBase : public RuntimeFilterFuncBase { // if FE do use ndv stat to predict the bf size, BE only use the row count. FE have more // exactly row count stat. which one is min is more correctly. if (_bloom_filter_size_calculated_by_ndv) { - _bloom_filter_length = - _runtime_bloom_filter_min_size > 0 - ? std::max(_runtime_bloom_filter_min_size, - std::min(be_calculate_size, _bloom_filter_length)) - : std::min(be_calculate_size, _bloom_filter_length); + _bloom_filter_length = std::min(be_calculate_size, _bloom_filter_length); } else { - _bloom_filter_length = - _runtime_bloom_filter_min_size > 0 - ? std::max(_runtime_bloom_filter_min_size, be_calculate_size) - : be_calculate_size; + _bloom_filter_length = be_calculate_size; } + _limit_length(); } return init_with_fixed_length(_bloom_filter_length); } @@ -228,6 +222,16 @@ class BloomFilterFuncBase : public RuntimeFilterFuncBase { uint16_t* offsets, int number, bool is_parse_column) = 0; +private: + void _limit_length() { + if (_runtime_bloom_filter_min_size > 0) { + _bloom_filter_length = std::max(_bloom_filter_length, _runtime_bloom_filter_min_size); + } + if (_runtime_bloom_filter_max_size > 0) { + _bloom_filter_length = std::min(_bloom_filter_length, _runtime_bloom_filter_max_size); + } + } + protected: // bloom filter size int32_t _bloom_filter_alloced; @@ -235,6 +239,7 @@ class BloomFilterFuncBase : public RuntimeFilterFuncBase { bool _inited = false; int64_t _bloom_filter_length; int64_t _runtime_bloom_filter_min_size; + int64_t _runtime_bloom_filter_max_size; bool _build_bf_exactly = false; bool _bloom_filter_size_calculated_by_ndv = false; }; diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp index d7fc2ff7490761..c6fd3338b14656 100644 --- a/be/src/exprs/runtime_filter.cpp +++ b/be/src/exprs/runtime_filter.cpp @@ -1289,6 +1289,9 @@ Status IRuntimeFilter::init_with_desc(const TRuntimeFilterDesc* desc, const TQue params.runtime_bloom_filter_min_size = options->__isset.runtime_bloom_filter_min_size ? options->runtime_bloom_filter_min_size : 0; + params.runtime_bloom_filter_max_size = options->__isset.runtime_bloom_filter_max_size + ? options->runtime_bloom_filter_max_size + : 0; // We build runtime filter by exact distinct count iff three conditions are met: // 1. Only 1 join key // 2. Do not have remote target (e.g. do not need to merge), or broadcast join diff --git a/be/src/exprs/runtime_filter.h b/be/src/exprs/runtime_filter.h index f199e173e84cd4..c4a38517ab4ba0 100644 --- a/be/src/exprs/runtime_filter.h +++ b/be/src/exprs/runtime_filter.h @@ -128,6 +128,7 @@ struct RuntimeFilterParams { int64_t bloom_filter_size; int32_t max_in_num; int64_t runtime_bloom_filter_min_size; + int64_t runtime_bloom_filter_max_size; int32_t filter_id; bool bitmap_filter_not_in; bool build_bf_exactly; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 1aff6159d9a469..04c8ba705a49d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -3591,6 +3591,7 @@ public TQueryOptions toThrift() { tResult.setRuntimeFilterWaitTimeMs(runtimeFilterWaitTimeMs); tResult.setRuntimeFilterMaxInNum(runtimeFilterMaxInNum); tResult.setRuntimeBloomFilterMinSize(runtimeBloomFilterMinSize); + tResult.setRuntimeBloomFilterMaxSize(runtimeBloomFilterMaxSize); tResult.setRuntimeFilterWaitInfinitely(runtimeFilterWaitInfinitely); if (cpuResourceLimit > 0) { diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index a75e06f358b54f..4fd0b897e526af 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -332,6 +332,8 @@ struct TQueryOptions { 125: optional bool enable_segment_cache = true; + 126: optional i32 runtime_bloom_filter_max_size = 16777216; + // For cloud, to control if the content would be written into file cache // In write path, to control if the content would be written into file cache. // In read path, read from file cache or remote storage when execute query.