[Feature](neireids) Add support of ProjectNode in PlanTranslator (apa…

…che#10499) Since we will do the column prune with project node, so we need compact the project outputs to the PlanNode in PhysicalPlanTranslator::visitPhysicalProject 1. Add support for ProjectNode to make column prune available. 2. Add SortNode to PlanFragment when it is unpartitioned piggyback
starocean999 · Jul 1, 2022 · a44a222 · a44a222
1 parent d0b757c
commit a44a222
Show file tree

Hide file tree

Showing 14 changed files with 225 additions and 68 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfo.java
@@ -234,6 +234,10 @@ public static AggregateInfo create(
         AggregateInfo result = new AggregateInfo(groupingExprs, aggExprs, phase);
         result.outputTupleDesc = tupleDesc;
         result.intermediateTupleDesc = intermediateTupleDesc;
+        int aggExprSize = result.getAggregateExprs().size();
+        for (int i = 0; i < aggExprSize; i++) {
+            result.materializedSlots.add(i);
+        }
         return result;
     }
 

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -17,6 +17,7 @@
 
 package org.apache.doris.nereids;
 
+import org.apache.doris.analysis.DescriptorTable;
 import org.apache.doris.analysis.StatementBase;
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.UserException;
@@ -25,12 +26,12 @@
 import org.apache.doris.nereids.memo.GroupExpression;
 import org.apache.doris.nereids.memo.Memo;
 import org.apache.doris.nereids.properties.PhysicalProperties;
-import org.apache.doris.nereids.trees.plans.PhysicalPlanTranslator;
 import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.trees.plans.PlanTranslatorContext;
 import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalPlanAdapter;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
+import org.apache.doris.nereids.trees.plans.translator.PhysicalPlanTranslator;
+import org.apache.doris.nereids.trees.plans.translator.PlanTranslatorContext;
 import org.apache.doris.planner.PlanFragment;
 import org.apache.doris.planner.Planner;
 import org.apache.doris.planner.ScanNode;
@@ -39,6 +40,7 @@
 import com.google.common.collect.Lists;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 /**
@@ -49,6 +51,7 @@ public class NereidsPlanner extends Planner {
     private PlannerContext plannerContext;
     private final ConnectContext ctx;
     private List<ScanNode> scanNodeList = null;
+    private DescriptorTable descTable;
 
     public NereidsPlanner(ConnectContext ctx) {
         this.ctx = ctx;
@@ -67,11 +70,16 @@ public void plan(StatementBase queryStmt,
         physicalPlanTranslator.translatePlan(physicalPlan, planContext);
         fragments = new ArrayList<>(planContext.getPlanFragmentList());
         PlanFragment root = fragments.get(fragments.size() - 1);
+        for (PlanFragment fragment : fragments) {
+            fragment.finalize(queryStmt);
+        }
         root.setOutputExprs(queryStmt.getResultExprs());
         if (VectorizedUtil.isVectorized()) {
             root.getPlanRoot().convertToVectoriezd();
         }
         scanNodeList = planContext.getScanNodeList();
+        descTable = planContext.getDescTable();
+        Collections.reverse(fragments);
     }
 
     /**
@@ -146,4 +154,9 @@ private PhysicalPlan chooseBestPlan(Group rootGroup, PhysicalProperties physical
     public boolean isBlockQuery() {
         return true;
     }
+
+    @Override
+    public DescriptorTable getDescTable() {
+        return descTable;
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/operators/plans/physical/PhysicalScan.java
@@ -39,4 +39,8 @@ public PhysicalScan(OperatorType type, List<String> qualifier) {
         super(type);
         this.qualifier = Objects.requireNonNull(qualifier, "qualifier can not be null");
     }
+
+    public List<String> getQualifier() {
+        return qualifier;
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java
@@ -20,6 +20,8 @@
 import org.apache.doris.nereids.memo.GroupExpression;
 import org.apache.doris.nereids.operators.Operator;
 
+import com.alibaba.google.common.collect.ImmutableList;
+
 import java.util.List;
 import java.util.Optional;
 import java.util.function.Consumer;
@@ -75,4 +77,18 @@ default boolean anyMatch(Predicate<TreeNode<NODE_TYPE>> predicate) {
         }
         return false;
     }
+
+    /**
+     * Collect the nodes that satisfied the predicate.
+     */
+    default <T> T collect(Predicate<TreeNode<NODE_TYPE>> predicate) {
+        ImmutableList.Builder<TreeNode<NODE_TYPE>> result = ImmutableList.builder();
+        foreach(node -> {
+            if (predicate.test(node)) {
+                result.add(node);
+            }
+        });
+        return (T) result.build();
+    }
+
 }
diff --git a/...rees/expressions/ExpressionConverter.java → ...lans/translator/ExpressionTranslator.java b/...rees/expressions/ExpressionConverter.java → ...lans/translator/ExpressionTranslator.java
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package org.apache.doris.nereids.trees.expressions;
+package org.apache.doris.nereids.trees.plans.translator;
 
 import org.apache.doris.analysis.ArithmeticExpr;
 import org.apache.doris.analysis.BinaryPredicate;
@@ -29,8 +29,21 @@
 import org.apache.doris.analysis.StringLiteral;
 import org.apache.doris.catalog.Type;
 import org.apache.doris.nereids.trees.NodeType;
+import org.apache.doris.nereids.trees.expressions.Arithmetic;
+import org.apache.doris.nereids.trees.expressions.Between;
+import org.apache.doris.nereids.trees.expressions.CompoundPredicate;
+import org.apache.doris.nereids.trees.expressions.DefaultExpressionVisitor;
+import org.apache.doris.nereids.trees.expressions.EqualTo;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.GreaterThan;
+import org.apache.doris.nereids.trees.expressions.GreaterThanEqual;
+import org.apache.doris.nereids.trees.expressions.LessThan;
+import org.apache.doris.nereids.trees.expressions.LessThanEqual;
+import org.apache.doris.nereids.trees.expressions.Literal;
+import org.apache.doris.nereids.trees.expressions.Not;
+import org.apache.doris.nereids.trees.expressions.NullSafeEqual;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
 import org.apache.doris.nereids.trees.expressions.functions.BoundFunction;
-import org.apache.doris.nereids.trees.plans.PlanTranslatorContext;
 import org.apache.doris.nereids.types.BooleanType;
 import org.apache.doris.nereids.types.DataType;
 import org.apache.doris.nereids.types.DoubleType;
@@ -42,15 +55,15 @@
 import java.util.List;
 
 /**
- * Used to convert expression of new optimizer to stale expr.
+ * Used to translate expression of new optimizer to stale expr.
  */
 @SuppressWarnings("rawtypes")
-public class ExpressionConverter extends DefaultExpressionVisitor<Expr, PlanTranslatorContext> {
+public class ExpressionTranslator extends DefaultExpressionVisitor<Expr, PlanTranslatorContext> {
 
-    public static ExpressionConverter converter = new ExpressionConverter();
+    public static ExpressionTranslator INSTANCE = new ExpressionTranslator();
 
-    public static Expr convert(Expression expression, PlanTranslatorContext planContext) {
-        return converter.visit(expression, planContext);
+    public static Expr translate(Expression expression, PlanTranslatorContext planContext) {
+        return expression.accept(INSTANCE, planContext);
     }
 
     @Override
@@ -61,55 +74,55 @@ public Expr visitSlotReference(SlotReference slotReference, PlanTranslatorContex
     @Override
     public Expr visitEqualTo(EqualTo equalTo, PlanTranslatorContext context) {
         return new BinaryPredicate(Operator.EQ,
-                visit(equalTo.child(0), context),
-                visit(equalTo.child(1), context));
+                equalTo.child(0).accept(this, context),
+                equalTo.child(1).accept(this, context));
     }
 
     @Override
     public Expr visitGreaterThan(GreaterThan greaterThan, PlanTranslatorContext context) {
         return new BinaryPredicate(Operator.GT,
-                visit(greaterThan.child(0), context),
-                visit(greaterThan.child(1), context));
+                greaterThan.child(0).accept(this, context),
+                greaterThan.child(1).accept(this, context));
     }
 
     @Override
     public Expr visitGreaterThanEqual(GreaterThanEqual greaterThanEqual, PlanTranslatorContext context) {
         return new BinaryPredicate(Operator.GE,
-                visit(greaterThanEqual.child(0), context),
-                visit(greaterThanEqual.child(1), context));
+                greaterThanEqual.child(0).accept(this, context),
+                greaterThanEqual.child(1).accept(this, context));
     }
 
     @Override
     public Expr visitLessThan(LessThan lessThan, PlanTranslatorContext context) {
         return new BinaryPredicate(Operator.LT,
-                visit(lessThan.child(0), context),
-                visit(lessThan.child(1), context));
+                lessThan.child(0).accept(this, context),
+                lessThan.child(1).accept(this, context));
     }
 
     @Override
     public Expr visitLessThanEqual(LessThanEqual lessThanEqual, PlanTranslatorContext context) {
         return new BinaryPredicate(Operator.LE,
-                visit(lessThanEqual.child(0), context),
-                visit(lessThanEqual.child(1), context));
+                lessThanEqual.child(0).accept(this, context),
+                lessThanEqual.child(1).accept(this, context));
     }
 
     @Override
     public Expr visitNot(Not not, PlanTranslatorContext context) {
         return new org.apache.doris.analysis.CompoundPredicate(
                 org.apache.doris.analysis.CompoundPredicate.Operator.NOT,
-                visit(not.child(0), context),
+                not.child(0).accept(this, context),
                 null);
     }
 
     @Override
     public Expr visitNullSafeEqual(NullSafeEqual nullSafeEqual, PlanTranslatorContext context) {
         return new BinaryPredicate(Operator.EQ_FOR_NULL,
-                visit(nullSafeEqual.child(0), context),
-                visit(nullSafeEqual.child(1), context));
+                nullSafeEqual.child(0).accept(this, context),
+                nullSafeEqual.child(1).accept(this, context));
     }
 
     /**
-     * Convert to stale literal.
+     * translate to stale literal.
      */
     @Override
     public Expr visitLiteral(Literal literal, PlanTranslatorContext context) {
@@ -133,7 +146,7 @@ public Expr visitLiteral(Literal literal, PlanTranslatorContext context) {
     public Expr visitBoundFunction(BoundFunction function, PlanTranslatorContext context) {
         List<Expr> paramList = new ArrayList<>();
         for (Expression expr : function.getArguments()) {
-            paramList.add(visit(expr, context));
+            paramList.add(expr.accept(this, context));
         }
         return new FunctionCallExpr(function.getName(), paramList);
     }
@@ -161,16 +174,16 @@ public Expr visitCompoundPredicate(CompoundPredicate compoundPredicate, PlanTran
                 throw new RuntimeException(String.format("Unknown node type: %s", nodeType.name()));
         }
         return new org.apache.doris.analysis.CompoundPredicate(staleOp,
-                visit(compoundPredicate.child(0), context),
-                visit(compoundPredicate.child(1), context));
+                compoundPredicate.child(0).accept(this, context),
+                compoundPredicate.child(1).accept(this, context));
     }
 
     @Override
     public Expr visitArithmetic(Arithmetic arithmetic, PlanTranslatorContext context) {
         Arithmetic.ArithmeticOperator arithmeticOperator = arithmetic.getArithmeticOperator();
         return new ArithmeticExpr(arithmeticOperator.getStaleOp(),
-                visit(arithmetic.child(0), context),
-                arithmeticOperator.isBinary() ? visit(arithmetic.child(1), context) : null);
+                arithmetic.child(0).accept(this, context),
+                arithmeticOperator.isBinary() ? arithmetic.child(1).accept(this, context) : null);
     }
 
 }