From bcae5ebcedd0384a3d13fe5766bf8f2b5e708aff Mon Sep 17 00:00:00 2001 From: seawinde <149132972+seawinde@users.noreply.github.com> Date: Fri, 26 Jan 2024 17:41:48 +0800 Subject: [PATCH] [Fix](Nereids) Fix lost predicate when query has a filter at the right input of the outer join (#30374) materialized view def is as following: > select l_shipdate, o_orderdate, l_partkey, l_suppkey, o_orderkey > from lineitem > left join (select * from orders where o_orderdate = '2023-12-10' ) t2 > on lineitem.l_orderkey = t2.o_orderkey; the query as following, should add filter `o_orderdate = '2023-12-10'` on mv when query rewrite by materialized view > select l_shipdate, o_orderdate, l_partkey, l_suppkey, o_orderkey > from lineitem > left join orders > on lineitem.l_orderkey = orders.o_orderkey > where o_orderdate = '2023-12-10' order by 1, 2, 3, 4, 5; --- .../mv/AbstractMaterializedViewRule.java | 66 ++++++++++++------- .../mv/join/left_outer/outer_join.out | 8 +++ .../mv/join/left_outer/outer_join.groovy | 24 ++++++- 3 files changed, 73 insertions(+), 25 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 57e06d4e16e516..a6b6bdeeeb320f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -441,6 +441,25 @@ protected SplitPredicate predicatesCompensate( viewStructInfo = viewStructInfo.withPredicates( viewStructInfo.getPredicates().merge(viewPulledUpExpressions)); } + // if the join type in query and mv plan is different, we should check query is have the + // filters which rejects null + Set> requireNoNullableViewSlot = comparisonResult.getViewNoNullableSlot(); + // check query is use the null reject slot which view comparison need + if (!requireNoNullableViewSlot.isEmpty()) { + SlotMapping queryToViewMapping = viewToQuerySlotMapping.inverse(); + // try to use + boolean valid = containsNullRejectSlot(requireNoNullableViewSlot, + queryStructInfo.getPredicates().getPulledUpPredicates(), queryToViewMapping, cascadesContext); + if (!valid) { + queryStructInfo = queryStructInfo.withPredicates( + queryStructInfo.getPredicates().merge(comparisonResult.getQueryAllPulledUpExpressions())); + valid = containsNullRejectSlot(requireNoNullableViewSlot, + queryStructInfo.getPredicates().getPulledUpPredicates(), queryToViewMapping, cascadesContext); + } + if (!valid) { + return SplitPredicate.INVALID_INSTANCE; + } + } // viewEquivalenceClass to query based // equal predicate compensate final Set equalCompensateConjunctions = Predicates.compensateEquivalence( @@ -464,30 +483,6 @@ protected SplitPredicate predicatesCompensate( || residualCompensatePredicates == null) { return SplitPredicate.INVALID_INSTANCE; } - // if the join type in query and mv plan is different, we should check query is have the - // filters which rejects null - Set> requireNoNullableViewSlot = comparisonResult.getViewNoNullableSlot(); - // check query is use the null reject slot which view comparison need - if (!requireNoNullableViewSlot.isEmpty()) { - Set queryPulledUpPredicates = comparisonResult.getQueryAllPulledUpExpressions().stream() - .flatMap(expr -> ExpressionUtils.extractConjunction(expr).stream()) - .collect(Collectors.toSet()); - Set nullRejectPredicates = ExpressionUtils.inferNotNull(queryPulledUpPredicates, - cascadesContext); - SlotMapping queryToViewMapping = viewToQuerySlotMapping.inverse(); - Set queryUsedNeedRejectNullSlotsViewBased = nullRejectPredicates.stream() - .map(expression -> TypeUtils.isNotNull(expression).orElse(null)) - .filter(Objects::nonNull) - .map(expr -> ExpressionUtils.replace((Expression) expr, queryToViewMapping.toSlotReferenceMap())) - .collect(Collectors.toSet()); - // query pulledUp predicates should have null reject predicates and contains any require noNullable slot - boolean valid = !queryPulledUpPredicates.containsAll(nullRejectPredicates) - && requireNoNullableViewSlot.stream().noneMatch( - set -> Sets.intersection(set, queryUsedNeedRejectNullSlotsViewBased).isEmpty()); - if (!valid) { - return SplitPredicate.INVALID_INSTANCE; - } - } return SplitPredicate.of(equalCompensateConjunctions.isEmpty() ? BooleanLiteral.TRUE : ExpressionUtils.and(equalCompensateConjunctions), rangeCompensatePredicates.isEmpty() ? BooleanLiteral.TRUE @@ -496,6 +491,29 @@ protected SplitPredicate predicatesCompensate( : ExpressionUtils.and(residualCompensatePredicates)); } + /** + * Check the queryPredicates contains the required nullable slot + */ + private boolean containsNullRejectSlot(Set> requireNoNullableViewSlot, + Set queryPredicates, + SlotMapping queryToViewMapping, + CascadesContext cascadesContext) { + Set queryPulledUpPredicates = queryPredicates.stream() + .flatMap(expr -> ExpressionUtils.extractConjunction(expr).stream()) + .collect(Collectors.toSet()); + Set nullRejectPredicates = ExpressionUtils.inferNotNull(queryPulledUpPredicates, + cascadesContext); + Set queryUsedNeedRejectNullSlotsViewBased = nullRejectPredicates.stream() + .map(expression -> TypeUtils.isNotNull(expression).orElse(null)) + .filter(Objects::nonNull) + .map(expr -> ExpressionUtils.replace((Expression) expr, queryToViewMapping.toSlotReferenceMap())) + .collect(Collectors.toSet()); + // query pulledUp predicates should have null reject predicates and contains any require noNullable slot + return !queryPulledUpPredicates.containsAll(nullRejectPredicates) + && requireNoNullableViewSlot.stream().noneMatch( + set -> Sets.intersection(set, queryUsedNeedRejectNullSlotsViewBased).isEmpty()); + } + /** * Decide the match mode * diff --git a/regression-test/data/nereids_rules_p0/mv/join/left_outer/outer_join.out b/regression-test/data/nereids_rules_p0/mv/join/left_outer/outer_join.out index 73c5193f136f96..0b171742bcaade 100644 --- a/regression-test/data/nereids_rules_p0/mv/join/left_outer/outer_join.out +++ b/regression-test/data/nereids_rules_p0/mv/join/left_outer/outer_join.out @@ -279,6 +279,14 @@ 2023-12-10 2023-12-10 2 4 3 2023-12-10 2023-12-10 2 4 3 +-- !query6_2_before -- +2023-12-10 2023-12-10 2 4 3 +2023-12-10 2023-12-10 2 4 3 + +-- !query6_2_after -- +2023-12-10 2023-12-10 2 4 3 +2023-12-10 2023-12-10 2 4 3 + -- !query7_0_before -- 3 3 2023-12-11 diff --git a/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy index ffeef5e149ffef..ef980aa2d9c81a 100644 --- a/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy @@ -179,7 +179,9 @@ suite("outer_join") { (2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), (3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), (4, 3, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-11', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), - (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx'); + (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx'), + (6, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx'), + (7, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx'); """ sql """ @@ -490,6 +492,26 @@ suite("outer_join") { sql """ DROP MATERIALIZED VIEW IF EXISTS mv6_1""" + // should compensate predicate o_orderdate = '2023-12-10' on mv + def mv6_2 = """ + select l_shipdate, o_orderdate, l_partkey, l_suppkey, o_orderkey + from lineitem + left join (select * from orders where o_orderdate = '2023-12-10' ) t2 + on lineitem.l_orderkey = t2.o_orderkey; + """ + def query6_2 = """ + select l_shipdate, o_orderdate, l_partkey, l_suppkey, o_orderkey + from lineitem + left join orders + on lineitem.l_orderkey = orders.o_orderkey + where o_orderdate = '2023-12-10' order by 1, 2, 3, 4, 5; + """ + order_qt_query6_2_before "${query6_2}" + check_rewrite(mv6_2, query6_2, "mv6_2") + order_qt_query6_2_after "${query6_2}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv6_2""" + + // filter inside + left + right def mv7_0 = "select l_shipdate, o_orderdate, l_partkey, l_suppkey " + "from lineitem " +