forked from apache/doris
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[bugfix](topn) fix topn optimzation wrong result for NULL values (apa…
…che#18121) 1. add PassNullPredicate to fix topn wrong result for NULL values 2. refactor RuntimePredicate to avoid using TCondition 3. refactor using ordering_exprs in fe and vsort_node
- Loading branch information
Showing
369 changed files
with
3,193 additions
and
161 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,168 @@ | ||
| // Licensed to the Apache Software Foundation (ASF) under one | ||
| // or more contributor license agreements. See the NOTICE file | ||
| // distributed with this work for additional information | ||
| // regarding copyright ownership. The ASF licenses this file | ||
| // to you under the Apache License, Version 2.0 (the | ||
| // "License"); you may not use this file except in compliance | ||
| // with the License. You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, | ||
| // software distributed under the License is distributed on an | ||
| // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| // KIND, either express or implied. See the License for the | ||
| // specific language governing permissions and limitations | ||
| // under the License. | ||
|
|
||
| #pragma once | ||
|
|
||
| #include <cstdint> | ||
|
|
||
| #include "olap/column_predicate.h" | ||
| #include "olap/rowset/segment_v2/bloom_filter.h" | ||
| #include "olap/rowset/segment_v2/inverted_index_reader.h" | ||
| #include "olap/wrapper_field.h" | ||
| #include "vec/columns/column_dictionary.h" | ||
|
|
||
| namespace doris { | ||
|
|
||
| /** | ||
| * A wrapper predicate that delegate to nested predicate | ||
| * but pass (set/return true) for NULL value rows. | ||
| * | ||
| * At parent, it's used for topn runtime predicate. | ||
| */ | ||
| class AcceptNullPredicate : public ColumnPredicate { | ||
| public: | ||
| AcceptNullPredicate(ColumnPredicate* nested) | ||
| : ColumnPredicate(nested->column_id(), nested->opposite()), _nested {nested} {} | ||
|
|
||
| PredicateType type() const override { return _nested->type(); } | ||
|
|
||
| Status evaluate(BitmapIndexIterator* iterator, uint32_t num_rows, | ||
| roaring::Roaring* roaring) const override { | ||
| return _nested->evaluate(iterator, num_rows, roaring); | ||
| } | ||
|
|
||
| Status evaluate(const Schema& schema, InvertedIndexIterator* iterator, uint32_t num_rows, | ||
| roaring::Roaring* bitmap) const override { | ||
| return _nested->evaluate(schema, iterator, num_rows, bitmap); | ||
| } | ||
|
|
||
| uint16_t evaluate(const vectorized::IColumn& column, uint16_t* sel, | ||
| uint16_t size) const override { | ||
| LOG(FATAL) << "evaluate without flags not supported"; | ||
| // return _nested->evaluate(column, sel, size); | ||
| } | ||
|
|
||
| void evaluate_and(const vectorized::IColumn& column, const uint16_t* sel, uint16_t size, | ||
| bool* flags) const override { | ||
| if (column.has_null()) { | ||
| // copy original flags | ||
| bool* original_flags = new bool[size]; | ||
| memcpy(original_flags, flags, size * sizeof(bool)); | ||
|
|
||
| // call evaluate_and and restore true for NULL rows | ||
| _nested->evaluate_and(column, sel, size, flags); | ||
| for (uint16_t i = 0; i < size; ++i) { | ||
| uint16_t idx = sel[i]; | ||
| if (original_flags[idx] && !flags[idx] && column.is_null_at(idx)) { | ||
| flags[i] = true; | ||
| } | ||
| } | ||
| } else { | ||
| _nested->evaluate_and(column, sel, size, flags); | ||
| } | ||
| } | ||
|
|
||
| void evaluate_or(const vectorized::IColumn& column, const uint16_t* sel, uint16_t size, | ||
| bool* flags) const override { | ||
| if (column.has_null()) { | ||
| // call evaluate_or and set true for NULL rows | ||
| _nested->evaluate_or(column, sel, size, flags); | ||
| for (uint16_t i = 0; i < size; ++i) { | ||
| uint16_t idx = sel[i]; | ||
| if (!flags[idx] && column.is_null_at(idx)) { | ||
| flags[i] = true; | ||
| } | ||
| } | ||
| } else { | ||
| _nested->evaluate_or(column, sel, size, flags); | ||
| } | ||
| } | ||
|
|
||
| bool evaluate_and(const std::pair<WrapperField*, WrapperField*>& statistic) const override { | ||
| // there is null in range, accept it | ||
| if (statistic.first->is_null() || statistic.second->is_null()) { | ||
| return true; | ||
| } | ||
| return _nested->evaluate_and(statistic); | ||
| } | ||
|
|
||
| bool evaluate_del(const std::pair<WrapperField*, WrapperField*>& statistic) const override { | ||
| return _nested->evaluate_del(statistic); | ||
| } | ||
|
|
||
| bool evaluate_and(const BloomFilter* bf) const override { return _nested->evaluate_and(bf); } | ||
|
|
||
| bool can_do_bloom_filter() const override { return _nested->can_do_bloom_filter(); } | ||
|
|
||
| void evaluate_vec(const vectorized::IColumn& column, uint16_t size, | ||
| bool* flags) const override { | ||
| _nested->evaluate_vec(column, size, flags); | ||
| if (column.has_null()) { | ||
| for (uint16_t i = 0; i < size; ++i) { | ||
| if (!flags[i] && column.is_null_at(i)) { | ||
| // set true for NULL rows | ||
| flags[i] = true; | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| void evaluate_and_vec(const vectorized::IColumn& column, uint16_t size, | ||
| bool* flags) const override { | ||
| if (column.has_null()) { | ||
| // copy original flags | ||
| bool* original_flags = new bool[size]; | ||
| memcpy(original_flags, flags, size * sizeof(bool)); | ||
|
|
||
| // call evaluate_and_vec and restore true for NULL rows | ||
| _nested->evaluate_and_vec(column, size, flags); | ||
| for (uint16_t i = 0; i < size; ++i) { | ||
| if (original_flags[i] && !flags[i] && column.is_null_at(i)) { | ||
| flags[i] = true; | ||
| } | ||
| } | ||
| } else { | ||
| _nested->evaluate_and_vec(column, size, flags); | ||
| } | ||
| } | ||
|
|
||
| std::string get_search_str() const override { return _nested->get_search_str(); } | ||
|
|
||
| std::string debug_string() const override { | ||
| return "passnull predicate for " + _nested->debug_string(); | ||
| } | ||
|
|
||
| /// Some predicates need to be cloned for each segment. | ||
| bool need_to_clone() const override { return _nested->need_to_clone(); } | ||
|
|
||
| void clone(ColumnPredicate** to) const override { | ||
| if (need_to_clone()) { | ||
| ColumnPredicate* clone_nested; | ||
| _nested->clone(&clone_nested); | ||
| *to = new AcceptNullPredicate(clone_nested); | ||
| } | ||
| } | ||
|
|
||
| private: | ||
| std::string _debug_string() const override { | ||
| return "passnull predicate for " + _nested->debug_string(); | ||
| } | ||
|
|
||
| ColumnPredicate* _nested; | ||
| }; | ||
|
|
||
| } //namespace doris |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
13 changes: 13 additions & 0 deletions
13
regression-test/data/datatype_p0/scalar_types/sql/dup_key_topn_q01_asc_nulls_last.out
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| -- This file is automatically generated. You should know what you did if you want to edit this | ||
| -- !dup_key_topn_q01_asc_nulls_last -- | ||
| 2022-01-10T16:04:49 -744222108 64562066838726736.278 false 85 25637 -21148 -593493650 -11450.651 -1.167904321387118E9 32585085266732240.402 2022-09-10 2022-08-08T18:00:04 2022-11-03 54.143.193.60 vitae@Oyoloo.mil Ruskin Court 16 | ||
| 2022-01-10T16:06:02 1769095688 95858385508825939.586 false -61 -24768 8981 -1811059535 26550.553 1.898948089429888E9 6302075696085095.930 2022-04-12 2022-05-06T18:56:26 2022-01-18 70.75.107.213 JacquelineRamirez@Edgetag.edu Schlimgen Way 58 | ||
| 2022-01-10T16:11:53 1906472648 30323570439371776.353 true -64 23204 -27723 1753599187 -28017.578 -8.7412093534361E7 34175877199258167.909 2023-01-07 2022-07-02T15:40:57 2022-10-09 60.230.5.23 asperiores_aut@Chatterpoint.name Summit Plaza 70 | ||
| 2022-01-10T16:13:51 2105767399 29408006992196347.603 true 86 18588 13544 -679723410 21493.826 1.885770666296631E9 85804384237665278.429 2023-01-07 2022-04-09T20:48:14 2022-06-22 98.145.190.138 yHarvey@Flipstorm.edu Southridge Street 43 | ||
| 2022-01-10T16:17:55 1863090100 90399389135889759.150 false -45 1656 -29181 -40413678 -32730.72 4.51575752697888E8 33932341853184569.730 2022-05-23 2022-04-24T13:12:53 2022-05-06 31.182.27.9 qui@Oodoo.mil Glendale Park 70 | ||
| 2022-01-10T16:19:43 -331752585 78238523737497869.470 true 0 -18857 26728 -1593527511 -3940.6157 1.011426330009696E9 30383576870508748.746 2022-10-18 2022-12-13T04:03:22 2022-04-25 114.191.125.61 et_natus@Chatterpoint.edu Merchant Circle 4 | ||
| 2022-01-10T16:26:01 694891812 89716246258986314.603 false -81 -31141 -32045 -1462025516 1286.6216 1.408000357577549E9 82173119133223782.614 2022-06-28 2022-08-30T06:48:08 2022-06-16 6.202.144.1 yFox@Flashpoint.biz Barnett Drive 12 | ||
| 2022-01-10T16:34:44 1969608603 60215082108365548.380 true 77 10466 7024 -1111293463 -3119.6628 -1.98697170405241E9 47421181841164225.577 2022-01-27 2022-06-05T03:27:17 2022-05-18 150.153.19.134 StephanieEdwards@Topicstorm.gov Onsgard Trail 10 | ||
| 2022-01-10T16:38:44 397842904 11471760929161672.212 false 48 22098 -23304 -647850805 -28931.523 -2.2117236773336E8 45351058658553553.851 2022-11-28 2022-08-13T12:22:09 2022-03-18 142.44.9.89 KathrynHarris@Riffpedia.biz Iowa Lane 43 | ||
| 2022-01-10T16:42:51 -914436742 2714763078802815.280 true -77 30692 -23058 1557401365 -25909.229 1.807346678696413E9 68043251987061973.806 2022-11-09 2022-08-04T09:16:13 2022-04-17 88.83.49.144 DouglasBailey@Skyndu.info Macpherson Center 25 | ||
|
|
Oops, something went wrong.