[fix](Nereids) When col stats is Unknow, not expression should return…

… the stats with selectivity of 1
realize096 · Sep 1, 2023 · 797d9de · 797d9de
1 parent e3bbba8
commit 797d9de
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 2 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
@@ -348,8 +348,16 @@ A not in (1, 2, 3, 100):
         return estimated;
     }
 
+    // Right Now, we just assume the selectivity is 1 when stats is Unknown
+    private Statistics handleUnknownCase(EstimationContext context) {
+        return context.statistics;
+    }
+
     @Override
     public Statistics visitNot(Not not, EstimationContext context) {
+        if (context.statistics.isInputSlotsUnknown(not.getInputSlots())) {
+            return handleUnknownCase(context);
+        }
         Statistics childStats = new FilterEstimation().estimate(not.child(), context.statistics);
         //if estimated rowCount is 0, adjust to 1 to make upper join reorder reasonable.
         double rowCount = Math.max(context.statistics.getRowCount() - childStats.getRowCount(), 1);

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java
@@ -19,11 +19,13 @@
 
 import org.apache.doris.nereids.stats.StatsMathUtil;
 import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Slot;
 
 import java.text.DecimalFormat;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
 
 public class Statistics {
     private static final int K_BYTES = 1024;
@@ -131,6 +133,12 @@ public Statistics addColumnStats(Expression expression, ColumnStatistic columnSt
         return this;
     }
 
+    public boolean isInputSlotsUnknown(Set<Slot> inputs) {
+        return inputs.stream()
+                .allMatch(s -> expressionToColumnStats.containsKey(s)
+                        && expressionToColumnStats.get(s).isUnKnown);
+    }
+
     public Statistics merge(Statistics statistics) {
         expressionToColumnStats.putAll(statistics.expressionToColumnStats);
         return this;

diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java
@@ -133,12 +133,12 @@ public void testNotInNaN() {
         Map<Expression, ColumnStatistic> slotToColumnStat = new HashMap<>();
         ColumnStatisticBuilder builder = new ColumnStatisticBuilder()
                 .setNdv(500)
-                .setIsUnknown(true);
+                .setIsUnknown(false);
         slotToColumnStat.put(a, builder.build());
         Statistics stat = new Statistics(1000, slotToColumnStat);
         FilterEstimation filterEstimation = new FilterEstimation();
         Statistics expected = filterEstimation.estimate(notIn, stat);
-        Assertions.assertTrue(Precision.equals(666.666, expected.getRowCount(), 0.01));
+        Assertions.assertTrue(Precision.equals(1000, expected.getRowCount(), 0.01));
     }
 
     /**