[Fix](Nereids) Fix lost predicate when query has a filter at the righ…

…t input of the outer join (apache#30374) materialized view def is as following: > select l_shipdate, o_orderdate, l_partkey, l_suppkey, o_orderkey > from lineitem > left join (select * from orders where o_orderdate = '2023-12-10' ) t2 > on lineitem.l_orderkey = t2.o_orderkey; the query as following, should add filter `o_orderdate = '2023-12-10'` on mv when query rewrite by materialized view > select l_shipdate, o_orderdate, l_partkey, l_suppkey, o_orderkey > from lineitem > left join orders > on lineitem.l_orderkey = orders.o_orderkey > where o_orderdate = '2023-12-10' order by 1, 2, 3, 4, 5;
seawinde · Jan 26, 2024 · bcae5eb · bcae5eb
1 parent ae3bb02
commit bcae5eb
Show file tree

Hide file tree

Showing 3 changed files with 73 additions and 25 deletions.
diff --git a/...main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/...main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
@@ -441,6 +441,25 @@ protected SplitPredicate predicatesCompensate(
             viewStructInfo = viewStructInfo.withPredicates(
                     viewStructInfo.getPredicates().merge(viewPulledUpExpressions));
         }
+        // if the join type in query and mv plan is different, we should check query is have the
+        // filters which rejects null
+        Set<Set<Slot>> requireNoNullableViewSlot = comparisonResult.getViewNoNullableSlot();
+        // check query is use the null reject slot which view comparison need
+        if (!requireNoNullableViewSlot.isEmpty()) {
+            SlotMapping queryToViewMapping = viewToQuerySlotMapping.inverse();
+            // try to use
+            boolean valid = containsNullRejectSlot(requireNoNullableViewSlot,
+                    queryStructInfo.getPredicates().getPulledUpPredicates(), queryToViewMapping, cascadesContext);
+            if (!valid) {
+                queryStructInfo = queryStructInfo.withPredicates(
+                        queryStructInfo.getPredicates().merge(comparisonResult.getQueryAllPulledUpExpressions()));
+                valid = containsNullRejectSlot(requireNoNullableViewSlot,
+                        queryStructInfo.getPredicates().getPulledUpPredicates(), queryToViewMapping, cascadesContext);
+            }
+            if (!valid) {
+                return SplitPredicate.INVALID_INSTANCE;
+            }
+        }
         // viewEquivalenceClass to query based
         // equal predicate compensate
         final Set<Expression> equalCompensateConjunctions = Predicates.compensateEquivalence(
@@ -464,30 +483,6 @@ protected SplitPredicate predicatesCompensate(
                 || residualCompensatePredicates == null) {
             return SplitPredicate.INVALID_INSTANCE;
         }
-        // if the join type in query and mv plan is different, we should check query is have the
-        // filters which rejects null
-        Set<Set<Slot>> requireNoNullableViewSlot = comparisonResult.getViewNoNullableSlot();
-        // check query is use the null reject slot which view comparison need
-        if (!requireNoNullableViewSlot.isEmpty()) {
-            Set<Expression> queryPulledUpPredicates = comparisonResult.getQueryAllPulledUpExpressions().stream()
-                    .flatMap(expr -> ExpressionUtils.extractConjunction(expr).stream())
-                    .collect(Collectors.toSet());
-            Set<Expression> nullRejectPredicates = ExpressionUtils.inferNotNull(queryPulledUpPredicates,
-                    cascadesContext);
-            SlotMapping queryToViewMapping = viewToQuerySlotMapping.inverse();
-            Set<Expression> queryUsedNeedRejectNullSlotsViewBased = nullRejectPredicates.stream()
-                    .map(expression -> TypeUtils.isNotNull(expression).orElse(null))
-                    .filter(Objects::nonNull)
-                    .map(expr -> ExpressionUtils.replace((Expression) expr, queryToViewMapping.toSlotReferenceMap()))
-                    .collect(Collectors.toSet());
-            // query pulledUp predicates should have null reject predicates and contains any require noNullable slot
-            boolean valid = !queryPulledUpPredicates.containsAll(nullRejectPredicates)
-                    && requireNoNullableViewSlot.stream().noneMatch(
-                            set -> Sets.intersection(set, queryUsedNeedRejectNullSlotsViewBased).isEmpty());
-            if (!valid) {
-                return SplitPredicate.INVALID_INSTANCE;
-            }
-        }
         return SplitPredicate.of(equalCompensateConjunctions.isEmpty() ? BooleanLiteral.TRUE
                         : ExpressionUtils.and(equalCompensateConjunctions),
                 rangeCompensatePredicates.isEmpty() ? BooleanLiteral.TRUE
@@ -496,6 +491,29 @@ protected SplitPredicate predicatesCompensate(
                         : ExpressionUtils.and(residualCompensatePredicates));
     }
 
+    /**
+     * Check the queryPredicates contains the required nullable slot
+     */
+    private boolean containsNullRejectSlot(Set<Set<Slot>> requireNoNullableViewSlot,
+            Set<Expression> queryPredicates,
+            SlotMapping queryToViewMapping,
+            CascadesContext cascadesContext) {
+        Set<Expression> queryPulledUpPredicates = queryPredicates.stream()
+                .flatMap(expr -> ExpressionUtils.extractConjunction(expr).stream())
+                .collect(Collectors.toSet());
+        Set<Expression> nullRejectPredicates = ExpressionUtils.inferNotNull(queryPulledUpPredicates,
+                cascadesContext);
+        Set<Expression> queryUsedNeedRejectNullSlotsViewBased = nullRejectPredicates.stream()
+                .map(expression -> TypeUtils.isNotNull(expression).orElse(null))
+                .filter(Objects::nonNull)
+                .map(expr -> ExpressionUtils.replace((Expression) expr, queryToViewMapping.toSlotReferenceMap()))
+                .collect(Collectors.toSet());
+        // query pulledUp predicates should have null reject predicates and contains any require noNullable slot
+        return !queryPulledUpPredicates.containsAll(nullRejectPredicates)
+                && requireNoNullableViewSlot.stream().noneMatch(
+                        set -> Sets.intersection(set, queryUsedNeedRejectNullSlotsViewBased).isEmpty());
+    }
+
     /**
      * Decide the match mode
      *

diff --git a/regression-test/data/nereids_rules_p0/mv/join/left_outer/outer_join.out b/regression-test/data/nereids_rules_p0/mv/join/left_outer/outer_join.out
@@ -279,6 +279,14 @@
 2023-12-10	2023-12-10	2	4	3
 2023-12-10	2023-12-10	2	4	3
 
+-- !query6_2_before --
+2023-12-10	2023-12-10	2	4	3
+2023-12-10	2023-12-10	2	4	3
+
+-- !query6_2_after --
+2023-12-10	2023-12-10	2	4	3
+2023-12-10	2023-12-10	2	4	3
+
 -- !query7_0_before --
 3	3	2023-12-11
 

diff --git a/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy
@@ -179,7 +179,9 @@ suite("outer_join") {
     (2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'),
     (3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'),
     (4, 3, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-11', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'),
-    (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx');
+    (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx'),
+    (6, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx'),
+    (7, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx');
     """
 
     sql """
@@ -490,6 +492,26 @@ suite("outer_join") {
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv6_1"""
 
 
+    // should compensate predicate o_orderdate = '2023-12-10' on mv
+    def mv6_2 = """
+        select l_shipdate, o_orderdate, l_partkey, l_suppkey, o_orderkey  
+        from lineitem 
+        left join (select * from orders where o_orderdate = '2023-12-10' ) t2 
+        on lineitem.l_orderkey = t2.o_orderkey;
+    """
+    def query6_2 = """
+        select l_shipdate, o_orderdate, l_partkey, l_suppkey, o_orderkey 
+        from lineitem 
+        left join orders 
+        on lineitem.l_orderkey = orders.o_orderkey 
+        where o_orderdate = '2023-12-10' order by 1, 2, 3, 4, 5;
+    """
+    order_qt_query6_2_before "${query6_2}"
+    check_rewrite(mv6_2, query6_2, "mv6_2")
+    order_qt_query6_2_after "${query6_2}"
+    sql """ DROP MATERIALIZED VIEW IF EXISTS mv6_2"""
+
+
     // filter inside + left + right
     def mv7_0 = "select l_shipdate, o_orderdate, l_partkey, l_suppkey " +
             "from lineitem " +