[opt](nereids) fix non-null selectivity computing

apache · Oct 23, 2024 · 437d65b · 437d65b
1 parent b3b0cb4
commit 437d65b
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 4 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java
@@ -119,7 +119,8 @@ public void normalizeColumnStatistics(double inputRowCount, boolean isNumNullsDe
             // the following columnStatistic.isUnKnown() judgment is loop inside since current doris
             // supports partial stats deriving, i.e, allowing part of tables have stats and other parts don't,
             // or part of columns have stats but other parts don't, especially join and filter estimation.
-            if (!checkColumnStatsValid(columnStatistic, rowCount) && !columnStatistic.isUnKnown()) {
+            if (!columnStatistic.isUnKnown() && (!checkColumnStatsValid(columnStatistic, rowCount)
+                    || isNumNullsDecreaseByProportion && columnStatistic.numNulls != 0)) {
                 ColumnStatisticBuilder columnStatisticBuilder = new ColumnStatisticBuilder(columnStatistic);
                 double ndv = Math.min(columnStatistic.ndv, rowCount);
                 double numNulls = Math.min(columnStatistic.numNulls * factor, rowCount - ndv);

diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java
@@ -1144,7 +1144,7 @@ public void testNumNullsAndTwoCol() {
         Statistics result = filterEstimation.estimate(and, stats);
         // result 1.0->2.0 bc happens because the calculation from normalization of
         // "Math.min(columnStatistic.numNulls * factor, rowCount - ndv);"
-        Assertions.assertEquals(result.getRowCount(), 2.0, 0.01);
+        Assertions.assertEquals(result.getRowCount(), 3.5, 0.01);
     }
 
     /**

diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
@@ -145,13 +145,13 @@ public void testFilter() {
         GroupExpression groupExpression = new GroupExpression(logicalFilter, ImmutableList.of(childGroup));
         Group ownerGroup = new Group(null, groupExpression, null);
         StatsCalculator.estimate(groupExpression, null);
-        Assertions.assertEquals(49.45, ownerGroup.getStatistics().getRowCount(), 0.001);
+        Assertions.assertEquals(49.945, ownerGroup.getStatistics().getRowCount(), 0.001);
 
         LogicalFilter<GroupPlan> logicalFilterOr = new LogicalFilter<>(or, groupPlan);
         GroupExpression groupExpressionOr = new GroupExpression(logicalFilterOr, ImmutableList.of(childGroup));
         Group ownerGroupOr = new Group(null, groupExpressionOr, null);
         StatsCalculator.estimate(groupExpressionOr, null);
-        Assertions.assertEquals(1449.05,
+        Assertions.assertEquals(1448.555,
                 ownerGroupOr.getStatistics().getRowCount(), 0.001);
     }