diff --git a/fe/fe-core/src/main/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessor.java b/fe/fe-core/src/main/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessor.java index ca8bf8e8a1ef5c..2a2add67bb4c85 100644 --- a/fe/fe-core/src/main/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessor.java +++ b/fe/fe-core/src/main/java/com/starrocks/scheduler/PartitionBasedMvRefreshProcessor.java @@ -26,6 +26,7 @@ import com.starrocks.analysis.IsNullPredicate; import com.starrocks.analysis.LiteralExpr; import com.starrocks.analysis.SlotRef; +import com.starrocks.analysis.StringLiteral; import com.starrocks.catalog.BaseTableInfo; import com.starrocks.catalog.Column; import com.starrocks.catalog.DataProperty; @@ -1200,6 +1201,23 @@ private Expr generatePartitionPredicate(Set tablePartitionNames, QuerySt sourceTablePartitionRange.add(refBaseTableRangePartitionMap.get(partitionName)); } sourceTablePartitionRange = MvUtils.mergeRanges(sourceTablePartitionRange); + // for nested mv, the base table may be another mv, which is partition by str2date(dt, '%Y%m%d') + // here we should convert date into '%Y%m%d' format + Expr partitionExpr = materializedView.getFirstPartitionRefTableExpr(); + Pair partitionTableAndColumn = materializedView.getBaseTableAndPartitionColumn(); + boolean isConvertToDate = PartitionUtil.isConvertToDate(partitionExpr, partitionTableAndColumn.second); + if (isConvertToDate && partitionExpr instanceof FunctionCallExpr + && !sourceTablePartitionRange.isEmpty() && MvUtils.isDateRange(sourceTablePartitionRange.get(0))) { + FunctionCallExpr functionCallExpr = (FunctionCallExpr) partitionExpr; + Preconditions.checkState(functionCallExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.STR2DATE)); + String dateFormat = ((StringLiteral) functionCallExpr.getChild(1)).getStringValue(); + List> converted = Lists.newArrayList(); + for (Range range : sourceTablePartitionRange) { + Range varcharPartitionKey = MvUtils.convertToVarcharRange(range, dateFormat); + converted.add(varcharPartitionKey); + } + sourceTablePartitionRange = converted; + } List partitionPredicates = MvUtils.convertRange(outputPartitionSlot, sourceTablePartitionRange); // range contains the min value could be null value diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvPlanContextBuilder.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvPlanContextBuilder.java index 0159c7737ceddf..f0a776f30e3f0d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvPlanContextBuilder.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvPlanContextBuilder.java @@ -35,6 +35,9 @@ public MvPlanContext getPlanContext(MaterializedView mv) { optimizerConfig.disableRule(RuleType.TF_MATERIALIZED_VIEW); } optimizerConfig.setMVRewritePlan(true); - return mvOptimizer.optimize(mv, new ConnectContext(), optimizerConfig); + ConnectContext connectContext = new ConnectContext(); + connectContext.getSessionVariable().setOptimizerExecuteTimeout( + ConnectContext.get().getSessionVariable().getOptimizerExecuteTimeout()); + return mvOptimizer.optimize(mv, connectContext, optimizerConfig); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/scalar/MvNormalizePredicateRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/scalar/MvNormalizePredicateRule.java index 57828bb80f31b2..b1132b57f49426 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/scalar/MvNormalizePredicateRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/scalar/MvNormalizePredicateRule.java @@ -47,7 +47,7 @@ public int compare(ScalarOperator o1, ScalarOperator o2) { } else if (o2 == null) { return 1; } else { - return o1.toString().compareTo(o2.toString()); + return o1.toString().toLowerCase().compareTo(o2.toString().toLowerCase()); } } }; @@ -65,17 +65,21 @@ public int compare(ScalarOperator o1, ScalarOperator o2) { if (o1.isColumnRef() && o2.isColumnRef()) { ColumnRefOperator c1 = (ColumnRefOperator) o1; ColumnRefOperator c2 = (ColumnRefOperator) o2; - int ret = c1.getName().compareTo(c2.getName()); + int ret = c1.getName().toLowerCase().compareTo(c2.getName().toLowerCase()); if (ret != 0) { return ret; } return Integer.compare(c1.getId(), c2.getId()); } else { - String s1 = o1.toString(); - String s2 = o2.toString(); + String s1 = o1.toString().toLowerCase(); + String s2 = o2.toString().toLowerCase(); String n1 = s1.replaceAll("\\d+: ", ""); String n2 = s2.replaceAll("\\d+: ", ""); - int ret = n1.compareTo(n2); + int ret = Integer.compare(n1.length(), n2.length()); + if (ret != 0) { + return ret; + } + ret = n1.compareTo(n2); return (ret == 0) ? s1.compareTo(s2) : ret; } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/mv/JoinDeriveContext.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/mv/JoinDeriveContext.java index f37fe9fe52a4e3..32139c84e52594 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/mv/JoinDeriveContext.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/mv/JoinDeriveContext.java @@ -25,6 +25,8 @@ public class JoinDeriveContext { private final JoinOperator mvJoinType; // join columns for left and right join tables private final List> joinColumns; + // join columns for left and right join tables + private final List> childOutputColumns; private final List> compensatedEquivalenceColumns; @@ -32,11 +34,13 @@ public JoinDeriveContext( JoinOperator queryJoinType, JoinOperator mvJoinType, List> joinColumns, - List> compensatedEquivalenceColumns) { + List> compensatedEquivalenceColumns, + List> childOutputColumns) { this.queryJoinType = queryJoinType; this.mvJoinType = mvJoinType; this.joinColumns = joinColumns; this.compensatedEquivalenceColumns = compensatedEquivalenceColumns; + this.childOutputColumns = childOutputColumns; } public JoinOperator getQueryJoinType() { @@ -58,4 +62,12 @@ public List getRightJoinColumns() { public List> getCompensatedEquivalenceColumns() { return compensatedEquivalenceColumns; } + + public List getLeftChildOutputColumns() { + return childOutputColumns.get(0); + } + + public List getRightChildOutputColumns() { + return childOutputColumns.get(1); + } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MaterializedViewRewriter.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MaterializedViewRewriter.java index 5e0df17d89bdc6..0bffa6d014cb61 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MaterializedViewRewriter.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MaterializedViewRewriter.java @@ -376,9 +376,11 @@ boolean computeCompatibility(OptExpression queryExpr, OptExpression mvExpr) { ScalarOperator queryOnPredicate = queryJoin.getOnPredicate(); // relationId -> join on predicate used columns Map tableToJoinColumns = Maps.newHashMap(); + // first is from left, second is from right List> joinColumnPairs = Lists.newArrayList(); - boolean isSupported = isSupportedPredicate(queryOnPredicate, - materializationContext.getQueryRefFactory(), tableToJoinColumns, joinColumnPairs); + ColumnRefSet leftColumns = queryExpr.inputAt(0).getOutputColumns(); + boolean isSupported = isSupportedPredicate(queryOnPredicate, materializationContext.getQueryRefFactory(), + leftColumns, tableToJoinColumns, joinColumnPairs); if (!isSupported) { logMVRewrite(mvRewriteContext, "join predicate is not supported {}", queryOnPredicate); return false; @@ -389,19 +391,33 @@ boolean computeCompatibility(OptExpression queryExpr, OptExpression mvExpr) { Table table = materializationContext.getQueryRefFactory().getTableForColumn(entry.getValue().getFirstId()); usedColumnsToTable.put(entry.getValue(), table); } - ColumnRefSet leftColumns = queryExpr.inputAt(0).getOutputColumns(); - ColumnRefSet rightColumns = queryExpr.inputAt(1).getOutputColumns(); + + ColumnRefSet leftJoinColumns = new ColumnRefSet(); + ColumnRefSet rightJoinColumns = new ColumnRefSet(); + for (Pair pair : joinColumnPairs) { + leftJoinColumns.union(pair.first); + rightJoinColumns.union(pair.second); + } List> joinColumnRefs = Lists.newArrayList(Lists.newArrayList(), Lists.newArrayList()); // query based join columns pair List> compensatedEquivalenceColumns = Lists.newArrayList(); boolean isCompatible = isJoinCompatible(usedColumnsToTable, queryJoinType, mvJoinType, - leftColumns, rightColumns, joinColumnPairs, joinColumnRefs, compensatedEquivalenceColumns); + leftJoinColumns, rightJoinColumns, joinColumnPairs, joinColumnRefs, compensatedEquivalenceColumns); if (!isCompatible) { - logMVRewrite(mvRewriteContext, "join columns not compatible {} != {}", leftColumns, rightColumns); + logMVRewrite(mvRewriteContext, "join columns not compatible {} != {}, query join type: {}, mv join type: {}", + leftJoinColumns, rightJoinColumns, queryJoinType, mvJoinType); return false; } - JoinDeriveContext joinDeriveContext = - new JoinDeriveContext(queryJoinType, mvJoinType, joinColumnRefs, compensatedEquivalenceColumns); + // here collect output columns to compensate the derived predicates if necessary. + // use mv output columns because the output is decided by mv. + // if the column is not in mv's output, it can not be used as compensated predicate. + List> childOutputColumns = Lists.newArrayList(); + childOutputColumns.add(mvExpr.getChildOutputColumns(0) + .getColumnRefOperators(materializationContext.getMvColumnRefFactory())); + childOutputColumns.add(mvExpr.getChildOutputColumns(1) + .getColumnRefOperators(materializationContext.getMvColumnRefFactory())); + JoinDeriveContext joinDeriveContext = new JoinDeriveContext( + queryJoinType, mvJoinType, joinColumnRefs, compensatedEquivalenceColumns, childOutputColumns); mvRewriteContext.addJoinDeriveContext(joinDeriveContext); return true; } else if (queryOp instanceof LogicalScanOperator) { @@ -502,87 +518,35 @@ private boolean isJoinCompatible( Preconditions.checkNotNull(leftTable); isCompatible = isUniqueColumns(leftTable, columnNames); } else if (mvJoinType.isLeftOuterJoin() && (queryJoinType.isInnerJoin() || queryJoinType.isLeftAntiJoin())) { - Optional rightColumnRefSet = usedColumnsToTable.keySet() - .stream().filter(columnSet -> rightColumns.containsAll(columnSet)).findFirst(); - if (!rightColumnRefSet.isPresent()) { - return false; - } List rightJoinColumnRefs = - rightColumnRefSet.get().getColumnRefOperators(materializationContext.getQueryRefFactory()); + rightColumns.getColumnRefOperators(materializationContext.getQueryRefFactory()); joinColumnRefs.set(1, rightJoinColumnRefs); if (queryJoinType.isInnerJoin()) { // for inner join, we should add the join columns into equivalence compensatedEquivalenceColumns.addAll(joinColumnPairs); } } else if (mvJoinType.isRightOuterJoin() && (queryJoinType.isInnerJoin() || queryJoinType.isRightAntiJoin())) { - Optional leftColumnRefSet = usedColumnsToTable.keySet() - .stream().filter(columnSet -> leftColumns.containsAll(columnSet)).findFirst(); - if (!leftColumnRefSet.isPresent()) { - return false; - } List leftJoinColumnRefs = - leftColumnRefSet.get().getColumnRefOperators(materializationContext.getQueryRefFactory()); + leftColumns.getColumnRefOperators(materializationContext.getQueryRefFactory()); joinColumnRefs.set(0, leftJoinColumnRefs); if (queryJoinType.isInnerJoin()) { // for inner join, we should add the join columns into equivalence compensatedEquivalenceColumns.addAll(joinColumnPairs); } } else if (mvJoinType.isFullOuterJoin() && queryJoinType.isLeftOuterJoin()) { - Optional leftColumnRefSet = usedColumnsToTable.keySet() - .stream().filter(columnSet -> leftColumns.containsAll(columnSet)).findFirst(); - if (!leftColumnRefSet.isPresent()) { - return false; - } - Table leftTable = usedColumnsToTable.get(leftColumnRefSet.get()); - if (leftTable.getColumns().stream().allMatch(column -> column.isAllowNull())) { - // should have at least one nullable column - return false; - } - List leftJoinColumnRefs = - leftColumnRefSet.get().getColumnRefOperators(materializationContext.getQueryRefFactory()); + leftColumns.getColumnRefOperators(materializationContext.getQueryRefFactory()); joinColumnRefs.set(0, leftJoinColumnRefs); } else if (mvJoinType.isFullOuterJoin() && queryJoinType.isRightOuterJoin()) { - Optional rightColumnRefSet = usedColumnsToTable.keySet() - .stream().filter(columnSet -> rightColumns.containsAll(columnSet)).findFirst(); - if (!rightColumnRefSet.isPresent()) { - return false; - } - Table rightTable = usedColumnsToTable.get(rightColumnRefSet.get()); - if (rightTable.getColumns().stream().allMatch(column -> column.isAllowNull())) { - // should have at least one nullable column - return false; - } List rightJoinColumnRefs = - rightColumnRefSet.get().getColumnRefOperators(materializationContext.getQueryRefFactory()); + rightColumns.getColumnRefOperators(materializationContext.getQueryRefFactory()); joinColumnRefs.set(1, rightJoinColumnRefs); } else if (mvJoinType.isFullOuterJoin() && queryJoinType.isInnerJoin()) { - Optional leftColumnRefSet = usedColumnsToTable.keySet() - .stream().filter(columnSet -> leftColumns.containsAll(columnSet)).findFirst(); - if (!leftColumnRefSet.isPresent()) { - return false; - } - Table leftTable = usedColumnsToTable.get(leftColumnRefSet.get()); - if (leftTable.getColumns().stream().allMatch(column -> column.isAllowNull())) { - // should have at least one nullable column - return false; - } List leftJoinColumnRefs = - leftColumnRefSet.get().getColumnRefOperators(materializationContext.getQueryRefFactory()); + leftColumns.getColumnRefOperators(materializationContext.getQueryRefFactory()); joinColumnRefs.set(0, leftJoinColumnRefs); - - Optional rightColumnRefSet = usedColumnsToTable.keySet() - .stream().filter(columnSet -> rightColumns.containsAll(columnSet)).findFirst(); - if (!rightColumnRefSet.isPresent()) { - return false; - } - Table rightTable = usedColumnsToTable.get(rightColumnRefSet.get()); - if (rightTable.getColumns().stream().allMatch(column -> column.isAllowNull())) { - // should have at least one nullable column - return false; - } List rightJoinColumnRefs = - rightColumnRefSet.get().getColumnRefOperators(materializationContext.getQueryRefFactory()); + rightColumns.getColumnRefOperators(materializationContext.getQueryRefFactory()); joinColumnRefs.set(1, rightJoinColumnRefs); // for inner join, we should add the join columns into equivalence compensatedEquivalenceColumns.addAll(joinColumnPairs); @@ -604,8 +568,8 @@ private boolean isUniqueColumns(Table table, List columnNames) { } private boolean isSupportedPredicate( - ScalarOperator onPredicate, ColumnRefFactory columnRefFactory, Map tableToJoinColumns, - List> joinColumnPairs) { + ScalarOperator onPredicate, ColumnRefFactory columnRefFactory, ColumnRefSet leftColumns, + Map tableToJoinColumns, List> joinColumnPairs) { List conjuncts = Utils.extractConjuncts(onPredicate); List binaryPredicates = conjuncts.stream() .filter(conjunct -> ScalarOperator.isColumnEqualBinaryPredicate(conjunct)).collect(Collectors.toList()); @@ -614,7 +578,11 @@ private boolean isSupportedPredicate( } for (ScalarOperator scalarOperator : binaryPredicates) { - joinColumnPairs.add(Pair.create(scalarOperator.getChild(0).cast(), scalarOperator.getChild(1).cast())); + if (leftColumns.containsAll(scalarOperator.getChild(0).getUsedColumns())) { + joinColumnPairs.add(Pair.create(scalarOperator.getChild(0).cast(), scalarOperator.getChild(1).cast())); + } else { + joinColumnPairs.add(Pair.create(scalarOperator.getChild(1).cast(), scalarOperator.getChild(0).cast())); + } } ColumnRefSet usedColumns = Utils.compoundAnd(binaryPredicates).getUsedColumns(); @@ -622,15 +590,11 @@ private boolean isSupportedPredicate( int relationId = columnRefFactory.getRelationId(columnId); if (relationId == -1) { // not use the column from table scan, unsupported - return false; + continue; } ColumnRefSet refColumns = tableToJoinColumns.computeIfAbsent(relationId, k -> new ColumnRefSet()); refColumns.union(columnId); } - if (tableToJoinColumns.size() != 2) { - // join predicate refs more than two tables, unsupported - return false; - } return true; } @@ -797,7 +761,7 @@ private OptExpression rewriteComplete(List queryTables, final PredicateSplit queryPredicateSplit = mvRewriteContext.getQueryPredicateSplit(); final EquivalenceClasses queryEc = createEquivalenceClasses( - queryPredicateSplit.getEqualPredicates(), queryExpression, null); + queryPredicateSplit.getEqualPredicates(), queryExpression, mvRewriteContext.getQueryColumnRefRewriter(), null); if (queryEc == null) { logMVRewrite(mvRewriteContext, "Cannot construct query equivalence classes"); return null; @@ -838,14 +802,19 @@ private OptExpression rewriteComplete(List
queryTables, // construct query based view EC final EquivalenceClasses queryBasedViewEqualPredicate = - createEquivalenceClasses(mvEqualPredicate, mvExpression, columnRewriter); + createEquivalenceClasses(mvEqualPredicate, mvExpression, mvColumnRefRewriter, columnRewriter); if (queryBasedViewEqualPredicate == null) { logMVRewrite(mvRewriteContext, "Cannot construct query based mv equivalence classes"); return null; } for (JoinDeriveContext deriveContext : mvRewriteContext.getJoinDeriveContexts()) { for (Pair pair : deriveContext.getCompensatedEquivalenceColumns()) { - queryBasedViewEqualPredicate.addEquivalence(pair.first, pair.second); + ScalarOperator first = mvRewriteContext.getQueryColumnRefRewriter().rewrite(pair.first); + ScalarOperator second = mvRewriteContext.getQueryColumnRefRewriter().rewrite(pair.second); + // now only support this pattern ec + if (first.isColumnRef() && second.isColumnRef()) { + queryBasedViewEqualPredicate.addEquivalence(first.cast(), second.cast()); + } } } rewriteContext.setQueryBasedViewEquivalenceClasses(queryBasedViewEqualPredicate); @@ -1283,8 +1252,7 @@ private OptExpression tryRewriteForRelationMapping(RewriteContext rewriteContext final ScalarOperator equalPredicates = MvUtils.canonizePredicate(compensationPredicates.getEqualPredicates()); final ScalarOperator otherPredicates = MvUtils.canonizePredicate(Utils.compoundAnd( compensationPredicates.getRangePredicates(), compensationPredicates.getResidualPredicates())); - - final ScalarOperator compensationPredicate = getMVCompensationPredicate(rewriteContext, + ScalarOperator compensationPredicate = getMVCompensationPredicate(rewriteContext, columnRewriter, mvColumnRefToScalarOp, equalPredicates, otherPredicates); if (compensationPredicate == null) { logMVRewrite(mvRewriteContext, "Success to convert query compensation predicates to MV " + @@ -1371,38 +1339,48 @@ private ScalarOperator addJoinDerivePredicate( continue; } - List leftJoinColumns = joinDeriveContext.getLeftJoinColumns(); - List rightJoinColumns = joinDeriveContext.getRightJoinColumns(); Optional derivedPredicateOpt = Optional.empty(); - if (mvJoinOp.isLeftOuterJoin() && queryJoinOp.isInnerJoin()) { - // mv: left outer join , query: inner join - derivedPredicateOpt = getDerivedPredicate(rewriteContext, - rewriter, mvColumnRefToScalarOp, rightJoinColumns, predicates, true, true, false); - } else if (mvJoinOp.isLeftOuterJoin() && queryJoinOp.isLeftAntiJoin()) { - // mv: left outer join , query: left anti join - derivedPredicateOpt = getDerivedPredicate(rewriteContext, - rewriter, mvColumnRefToScalarOp, rightJoinColumns, predicates, false, true, false); - } else if (mvJoinOp.isRightOuterJoin() && queryJoinOp.isInnerJoin()) { - // mv: right outer join , query: inner join - derivedPredicateOpt = getDerivedPredicate(rewriteContext, - rewriter, mvColumnRefToScalarOp, leftJoinColumns, predicates, true, true, false); - } else if (mvJoinOp.isRightOuterJoin() && queryJoinOp.isRightAntiJoin()) { - // mv: right outer join , query: right anti join - derivedPredicateOpt = getDerivedPredicate(rewriteContext, - rewriter, mvColumnRefToScalarOp, leftJoinColumns, predicates, false, true, false); - } else if (mvJoinOp.isFullOuterJoin() && queryJoinOp.isLeftOuterJoin()) { - // mv: full outer join , query: left outer join - derivedPredicateOpt = getDerivedPredicate(rewriteContext, - rewriter, mvColumnRefToScalarOp, leftJoinColumns, predicates, true, false, true); - } else if (mvJoinOp.isFullOuterJoin() && queryJoinOp.isRightOuterJoin()) { - // mv: full outer join , query: right outer join - derivedPredicateOpt = getDerivedPredicate(rewriteContext, - rewriter, mvColumnRefToScalarOp, rightJoinColumns, predicates, true, false, true); - } else if (mvJoinOp.isFullOuterJoin() && queryJoinOp.isInnerJoin()) { - // mv: full outer join , query: inner join - // add right table's not null constrains - derivedPredicateOpt = getDerivedPredicate(rewriteContext, - rewriter, mvColumnRefToScalarOp, rightJoinColumns, predicates, true, false, true); + if (joinDeriveContext.getMvJoinType().isLeftOuterJoin() && joinDeriveContext.getQueryJoinType().isInnerJoin()) { + List rightJoinColumns = joinDeriveContext.getRightJoinColumns(); + derivedPredicateOpt = getDerivedPredicate(rewriteContext, rewriter, + mvColumnRefToScalarOp, rightJoinColumns, joinDeriveContext.getRightChildOutputColumns(), + predicates, true, true, false); + } else if (joinDeriveContext.getMvJoinType().isLeftOuterJoin() + && joinDeriveContext.getQueryJoinType().isLeftAntiJoin()) { + List rightJoinColumns = joinDeriveContext.getRightJoinColumns(); + derivedPredicateOpt = getDerivedPredicate(rewriteContext, rewriter, + mvColumnRefToScalarOp, rightJoinColumns, joinDeriveContext.getRightChildOutputColumns(), + predicates, false, true, false); + } else if (joinDeriveContext.getMvJoinType().isRightOuterJoin() + && joinDeriveContext.getQueryJoinType().isInnerJoin()) { + List leftJoinColumns = joinDeriveContext.getLeftJoinColumns(); + derivedPredicateOpt = getDerivedPredicate(rewriteContext, rewriter, + mvColumnRefToScalarOp, leftJoinColumns, joinDeriveContext.getLeftChildOutputColumns(), + predicates, true, true, false); + } else if (joinDeriveContext.getMvJoinType().isRightOuterJoin() + && joinDeriveContext.getQueryJoinType().isRightAntiJoin()) { + List leftJoinColumns = joinDeriveContext.getLeftJoinColumns(); + derivedPredicateOpt = getDerivedPredicate(rewriteContext, rewriter, + mvColumnRefToScalarOp, leftJoinColumns, joinDeriveContext.getLeftChildOutputColumns(), + predicates, false, true, false); + } else if (joinDeriveContext.getMvJoinType().isFullOuterJoin() + && joinDeriveContext.getQueryJoinType().isLeftOuterJoin()) { + List leftJoinColumns = joinDeriveContext.getLeftJoinColumns(); + derivedPredicateOpt = getDerivedPredicate(rewriteContext, rewriter, + mvColumnRefToScalarOp, leftJoinColumns, joinDeriveContext.getLeftChildOutputColumns(), + predicates, true, false, true); + } else if (joinDeriveContext.getMvJoinType().isFullOuterJoin() + && joinDeriveContext.getQueryJoinType().isRightOuterJoin()) { + List rightJoinColumns = joinDeriveContext.getRightJoinColumns(); + derivedPredicateOpt = getDerivedPredicate(rewriteContext, rewriter, + mvColumnRefToScalarOp, rightJoinColumns, joinDeriveContext.getRightChildOutputColumns(), + predicates, true, false, true); + } else if (joinDeriveContext.getMvJoinType().isFullOuterJoin() + && joinDeriveContext.getQueryJoinType().isInnerJoin()) { + List rightJoinColumns = joinDeriveContext.getRightJoinColumns(); + derivedPredicateOpt = getDerivedPredicate(rewriteContext, rewriter, + mvColumnRefToScalarOp, rightJoinColumns, joinDeriveContext.getRightChildOutputColumns(), + predicates, true, false, true); if (!derivedPredicateOpt.isPresent()) { // can not get derived predicates logMVRewrite(mvRewriteContext, "derive join's extra predicate failed, mv join type: {}, " + @@ -1413,9 +1391,11 @@ private ScalarOperator addJoinDerivePredicate( if (!derivedPredicate.equals(ConstantOperator.TRUE)) { derivedPredicates.add(derivedPredicate); } - // add left table's not null constraints - derivedPredicateOpt = getDerivedPredicate(rewriteContext, - rewriter, mvColumnRefToScalarOp, leftJoinColumns, predicates, true, false, true); + + List leftJoinColumns = joinDeriveContext.getLeftJoinColumns(); + derivedPredicateOpt = getDerivedPredicate(rewriteContext, rewriter, + mvColumnRefToScalarOp, leftJoinColumns, joinDeriveContext.getLeftChildOutputColumns(), + predicates, true, false, true); } if (!derivedPredicateOpt.isPresent()) { // can not get derived predicates @@ -1436,46 +1416,56 @@ private Optional getDerivedPredicate( ColumnRewriter rewriter, Map mvColumnRefToScalarOp, List joinColumns, + List outputColumns, List compensationPredicates, boolean isNotNull, boolean onlyJoinColumns, boolean onlyNotNullColumns) { - Integer relationId = materializationContext.getQueryRefFactory().getRelationId(joinColumns.get(0).getId()); - Map columnToRelationId = materializationContext.getQueryRefFactory().getColumnToRelationIds(); - - List relationColumns = columnToRelationId.entrySet().stream() - .filter(entry -> entry.getValue().equals(relationId)) - .map(entry -> materializationContext.getQueryRefFactory().getColumnRef(entry.getKey())) - .collect(Collectors.toList()); - - // query join column ref -> rewritten compensate mv column ref - Map compensatedColumnsInMv = Maps.newHashMap(); - for (ColumnRefOperator relationColumnRef : relationColumns) { - if (onlyNotNullColumns && relationColumnRef.isNullable()) { - continue; - } - ScalarOperator rewrittenColumnRef = rewriteMVCompensationExpression(rewriteContext, rewriter, - mvColumnRefToScalarOp, relationColumnRef, false, false); - if (rewrittenColumnRef != null) { - compensatedColumnsInMv.put(relationColumnRef, (ColumnRefOperator) rewrittenColumnRef); + // output column to target mv expr + Map compensatedColumnsInMv = Maps.newHashMap(); + // output column to query based expr + Map outputExprMap = Maps.newHashMap(); + Set relatedColumns = Sets.newHashSet(); + // outputColumns are mv based, should be rewritten by MvColumnRefRewriter first to base tables + // and then rewritten to mv to construct the derived predicates + for (ColumnRefOperator outputColumn : outputColumns) { + ScalarOperator rewrittenColumnRef = rewriteContext.getMvColumnRefRewriter().rewrite(outputColumn); + rewrittenColumnRef = rewriter.rewriteViewToQuery(rewrittenColumnRef); + ScalarOperator targetExpr = rewriteMVCompensationExpression(rewriteContext, rewriter, + mvColumnRefToScalarOp, rewrittenColumnRef, false, false); + if (targetExpr != null) { + relatedColumns.addAll(targetExpr.getColumnRefs()); + compensatedColumnsInMv.put(outputColumn, targetExpr); + outputExprMap.put(outputColumn, rewrittenColumnRef); + } + } + + List relatedPredicates = compensationPredicates.stream().filter( + predicate -> predicate.getUsedColumns().containsAny(relatedColumns)).collect(Collectors.toList()); + if (needExtraNotNullDerive(relatedPredicates, relatedColumns)) { + if (compensatedColumnsInMv.isEmpty()) { + // there is no output of compensation table + return Optional.empty(); } - } - if (compensatedColumnsInMv.isEmpty()) { - // there is no output of compensation table - logMVRewrite(mvRewriteContext, "derive extra predicates from mv failed: compensated columns in mv are empty. " + - "isNotNull: {}, onlyJoinColumns:{}, onlyNotNullColumns:{}", isNotNull, onlyJoinColumns, onlyNotNullColumns); - return Optional.empty(); - } - - Collection relatedColumns = compensatedColumnsInMv.values(); - List relatedPredicates = compensationPredicates.stream() - .filter(predicate -> predicate.getUsedColumns().containsAny(relatedColumns)) - .collect(Collectors.toList()); - if (needExtraNotNullDerive(relatedPredicates, compensatedColumnsInMv)) { final List candidateColumns = Lists.newArrayList(); if (onlyJoinColumns) { - compensatedColumnsInMv.keySet().stream().filter(key -> joinColumns.contains(key)) - .forEach(key -> candidateColumns.add(compensatedColumnsInMv.get(key))); + List joinExprs = Lists.newArrayList(); + for (ColumnRefOperator joinColumn : joinColumns) { + joinExprs.add(rewriteContext.getQueryColumnRefRewriter().rewrite(joinColumn)); + } + for (ColumnRefOperator outputColumn : compensatedColumnsInMv.keySet()) { + if (joinExprs.contains(outputExprMap.get(outputColumn))) { + candidateColumns.addAll(compensatedColumnsInMv.get(outputColumn).getColumnRefs()); + } + } } else { - candidateColumns.addAll(compensatedColumnsInMv.values()); + candidateColumns.addAll(relatedColumns); + } + if (onlyNotNullColumns) { + for (ColumnRefOperator columnRef : compensatedColumnsInMv.keySet()) { + // remove nullable columns + if (columnRef.isNullable()) { + candidateColumns.remove(compensatedColumnsInMv.get(columnRef)); + } + } } if (candidateColumns.isEmpty()) { logMVRewrite(mvRewriteContext, "derive extra predicates from mv failed: candidate columns are empty. " + @@ -1492,13 +1482,12 @@ private Optional getDerivedPredicate( } } - private boolean needExtraNotNullDerive(List relatedPredicates, - Map compensatedColumnsInMv) { + private boolean needExtraNotNullDerive( + List relatedPredicates, Collection relatedColumnRefs) { if (relatedPredicates.isEmpty()) { return true; } - Collection relatedColumnRefs = compensatedColumnsInMv.values(); if (relatedPredicates.stream().allMatch(relatedPredicate -> !Utils.canEliminateNull(Sets.newHashSet(relatedColumnRefs), relatedPredicate))) { return true; @@ -2028,6 +2017,7 @@ private ColumnRefOperator getColumnRef(ScalarOperator operator, int index) { private EquivalenceClasses createEquivalenceClasses(ScalarOperator equalPredicates, OptExpression expression, + ReplaceColumnRefRewriter refRewriter, ColumnRewriter columnRewriter) { EquivalenceClasses ec = new EquivalenceClasses(); if (equalPredicates == null) { @@ -2037,7 +2027,8 @@ private EquivalenceClasses createEquivalenceClasses(ScalarOperator equalPredicat List outerJoinOnPredicate = MvUtils.collectOuterAntiJoinOnPredicate(expression); final PredicateSplit outerJoinPredicateSplit = PredicateSplit.splitPredicate(Utils.compoundAnd(outerJoinOnPredicate)); - ScalarOperator outerJoinEqualPredicates = outerJoinPredicateSplit.getEqualPredicates(); + // should use ReplaceColumnRefRewriter to rewrite outerJoinPredicateSplit to base tables + ScalarOperator outerJoinEqualPredicates = refRewriter.rewrite(outerJoinPredicateSplit.getEqualPredicates()); outerJoinEqualPredicates = MvUtils.canonizePredicateForRewrite(outerJoinEqualPredicates); List outerJoinConjuncts = Utils.extractConjuncts(outerJoinEqualPredicates); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java index 02e1303dbcd086..5aa9662f9f8e14 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java @@ -96,6 +96,7 @@ import org.apache.logging.log4j.Logger; import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -1219,6 +1220,44 @@ public static Range convertToDateRange(Range from) t return Range.all(); } + public static boolean isDateRange(Range range) { + if (range.hasUpperBound()) { + PartitionKey partitionKey = range.upperEndpoint(); + return partitionKey.getKeys().get(0) instanceof DateLiteral; + } else if (range.hasLowerBound()) { + PartitionKey partitionKey = range.lowerEndpoint(); + return partitionKey.getKeys().get(0) instanceof DateLiteral; + } + return false; + } + + // convert date to varchar type + public static Range convertToVarcharRange( + Range from, String dateFormat) throws AnalysisException { + DateTimeFormatter formatter = DateUtils.unixDatetimeFormatter(dateFormat); + if (from.hasLowerBound() && from.hasUpperBound()) { + DateLiteral lowerDate = (DateLiteral) from.lowerEndpoint().getKeys().get(0); + String lowerDateString = lowerDate.toLocalDateTime().toLocalDate().format(formatter); + PartitionKey lowerPartitionKey = PartitionKey.ofString(lowerDateString); + + DateLiteral upperDate = (DateLiteral) from.upperEndpoint().getKeys().get(0); + String upperDateString = upperDate.toLocalDateTime().toLocalDate().format(formatter); + PartitionKey upperPartitionKey = PartitionKey.ofString(upperDateString); + return Range.range(lowerPartitionKey, from.lowerBoundType(), upperPartitionKey, from.upperBoundType()); + } else if (from.hasUpperBound()) { + DateLiteral upperDate = (DateLiteral) from.upperEndpoint().getKeys().get(0); + String upperDateString = upperDate.toLocalDateTime().toLocalDate().format(formatter); + PartitionKey upperPartitionKey = PartitionKey.ofString(upperDateString); + return Range.upTo(upperPartitionKey, from.upperBoundType()); + } else if (from.hasLowerBound()) { + DateLiteral lowerDate = (DateLiteral) from.lowerEndpoint().getKeys().get(0); + String lowerDateString = lowerDate.toLocalDateTime().toLocalDate().format(formatter); + PartitionKey lowerPartitionKey = PartitionKey.ofString(lowerDateString); + return Range.downTo(lowerPartitionKey, from.lowerBoundType()); + } + return Range.all(); + } + /** * Return the updated partition key ranges of the specific table. * diff --git a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTest.java b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTest.java index c44aa5708b179f..95ff21a24d5290 100644 --- a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewTest.java @@ -544,7 +544,7 @@ public void testLeftOuterJoinQueryComplete() { testRewriteOK(mv, "select empid as col2, emps.locationid from " + "emps left join locations on emps.locationid = locations.locationid " + "where emps.locationid > 10"); - // TODO: Query's left outer join will be converted to Inner Join. + // no locations.locationid in mv testRewriteFail(mv, "select empid as col2, locations.locationid from " + "emps left join locations on emps.locationid = locations.locationid " + "where locations.locationid > 10"); @@ -3237,7 +3237,8 @@ public void testJoinDeriveRewrite() { " from lineorder left outer join customer" + " on lo_custkey = c_custkey where c_name = 'name' and c_custkey = 100"; MVRewriteChecker checker = testRewriteOK(mv, query); - checker.contains("TABLE: mv0\n" + + checker.contains("0:OlapScanNode\n" + + " TABLE: mv0\n" + " PREAGGREGATION: ON\n" + " PREDICATES: 30: c_custkey = 100, 31: c_name = 'name'\n" + " partitions=1/1"); @@ -3349,6 +3350,8 @@ public void testJoinDeriveRewrite() { " partitions=1/1"); } + + { String mv = "select lo_orderkey, lo_linenumber, lo_quantity, lo_revenue, c_custkey, lo_custkey, c_name" + " from lineorder right outer join customer" + @@ -4815,4 +4818,315 @@ public void testViewDeltaJoinOnSubquery() throws Exception { starRocksAssert.dropTable("table_e"); starRocksAssert.dropMaterializedView("mv_view_delta_1"); } + + @Test + public void testJoinDeriveRewriteOnNestedMv() throws Exception { + starRocksAssert.withTable("CREATE TABLE `IDX_BASE_DIM_TRACEID` (\n" + + " `trace_id` varchar(65533) DEFAULT NULL,\n" + + " `strategy_id_pdl` varchar(65533) DEFAULT NULL,\n" + + " `dt` date DEFAULT NULL\n" + + ") PARTITION BY range(dt) (\n" + + "PARTITION p1 VALUES [ (\"20230702\"),(\"20230703\")),\n" + + "PARTITION p2 VALUES [ (\"20230703\"),(\"20230704\")),\n" + + "PARTITION p3 VALUES [ (\"20230704\"),(\"20230705\")),\n" + + "PARTITION p4 VALUES [ (\"20230705\"),(\"20230706\"))\n" + + ")\n" + + "DISTRIBUTED BY HASH(trace_id)\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");\n" + + "\n"); + + starRocksAssert.withTable("CREATE TABLE `IDX_BASE_OBJ_STRATEGY` (\n" + + " `strategy_id` varchar(65533) DEFAULT NULL,\n" + + " `strategy_type` varchar(65533) DEFAULT NULL,\n" + + " `cust_group_name` varchar(65533) DEFAULT NULL,\n" + + " `dt` date DEFAULT NULL\n" + + ") PARTITION BY range(dt) (\n" + + "PARTITION p1 VALUES [ (\"20230702\"),(\"20230703\")),\n" + + "PARTITION p2 VALUES [ (\"20230703\"),(\"20230704\")),\n" + + "PARTITION p3 VALUES [ (\"20230704\"),(\"20230705\")),\n" + + "PARTITION p4 VALUES [ (\"20230705\"),(\"20230706\"))\n" + + ")\n" + + "DISTRIBUTED BY HASH(strategy_id)\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");\n" + + "\n"); + + starRocksAssert.withTable("CREATE TABLE `IDX_COMMON_EV_BANKAPP_EXP_CLICK_INFO` (\n" + + " `strategy_id_pdl` varchar(65533) DEFAULT NULL,\n" + + " `becif_no` varchar(65533) DEFAULT NULL,\n" + + " `trace_id` varchar(65533) DEFAULT NULL,\n" + + " `pv` int(11) DEFAULT NULL,\n" + + " `dt` date DEFAULT NULL\n" + + ") PARTITION BY range(dt) (\n" + + "PARTITION p1 VALUES [ (\"20230702\"),(\"20230703\")),\n" + + "PARTITION p2 VALUES [ (\"20230703\"),(\"20230704\")),\n" + + "PARTITION p3 VALUES [ (\"20230704\"),(\"20230705\")),\n" + + "PARTITION p4 VALUES [ (\"20230705\"),(\"20230706\"))\n" + + ")\n" + + "DISTRIBUTED BY HASH(trace_id)\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");\n" + + "\n"); + + starRocksAssert.withTable("CREATE TABLE `IDX_COMMON_EV_BANKAPP_CLICK_INFO` (\n" + + " `strategy_id_pdl` varchar(65533) DEFAULT NULL,\n" + + " `becif_no` varchar(65533) DEFAULT NULL,\n" + + " `trace_id` varchar(65533) DEFAULT NULL,\n" + + " `pv` int(11) DEFAULT NULL,\n" + + " `dt` date DEFAULT NULL\n" + + ") PARTITION BY range(dt) (\n" + + "PARTITION p1 VALUES [ (\"20230702\"),(\"20230703\")),\n" + + "PARTITION p2 VALUES [ (\"20230703\"),(\"20230704\")),\n" + + "PARTITION p3 VALUES [ (\"20230704\"),(\"20230705\")),\n" + + "PARTITION p4 VALUES [ (\"20230705\"),(\"20230706\"))\n" + + ")\n" + + "DISTRIBUTED BY HASH(trace_id)\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");\n" + + "\n"); + + starRocksAssert.withTable("CREATE TABLE `IDX_CUST_COMMON_DIM` (\n" + + " `sales_new_cust` varchar(65533) DEFAULT NULL,\n" + + " `payroll_flag` varchar(65533) DEFAULT NULL,\n" + + " `becif_no` varchar(65533) DEFAULT NULL,\n" + + " `dt` date DEFAULT NULL\n" + + ") PARTITION BY range(dt) (\n" + + "PARTITION p1 VALUES [ (\"20230702\"),(\"20230703\")),\n" + + "PARTITION p2 VALUES [ (\"20230703\"),(\"20230704\")),\n" + + "PARTITION p3 VALUES [ (\"20230704\"),(\"20230705\")),\n" + + "PARTITION p4 VALUES [ (\"20230705\"),(\"20230706\"))\n" + + ")\n" + + "DISTRIBUTED BY HASH(becif_no)\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");\n" + + "\n"); + + starRocksAssert.withTable("CREATE TABLE `D_OPERATION_ORDER_CUST_DETAIL_PDL_ID3` (\n" + + " `strategy_id_pdl` varchar(65533) DEFAULT NULL,\n" + + " `trace_id` varchar(65533) DEFAULT NULL,\n" + + " `page_name_gy` varchar(65533) DEFAULT NULL,\n" + + " `becif_no` varchar(65533) DEFAULT NULL,\n" + + " `dt` date DEFAULT NULL\n" + + ") PARTITION BY range(dt) (\n" + + "PARTITION p1 VALUES [ (\"20230702\"),(\"20230703\")),\n" + + "PARTITION p2 VALUES [ (\"20230703\"),(\"20230704\")),\n" + + "PARTITION p3 VALUES [ (\"20230704\"),(\"20230705\")),\n" + + "PARTITION p4 VALUES [ (\"20230705\"),(\"20230706\"))\n" + + ")\n" + + "DISTRIBUTED BY HASH(trace_id)\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");\n" + + "\n"); + + starRocksAssert.withMaterializedView("CREATE MATERIALIZED VIEW `test_perf_mv_pv1`\n" + + "PARTITION BY dt\n" + + "DISTRIBUTED BY HASH(`TRACE_ID`) BUCKETS 8\n" + + "REFRESH DEFERRED MANUAL\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"force_external_table_query_rewrite\" = \"true\"\n" + + ")\n" + + "AS\n" + + " SELECT\n" + + " f.DT,\n" + + " f.TRACE_ID AS TRACE_ID,\n" + + " f.BECIF_NO AS BECIF_NO,\n" + + " SUM(f.PV) AS pv1\n" + + " FROM\n" + + " IDX_COMMON_EV_BANKAPP_CLICK_INFO f\n" + + " GROUP BY\n" + + " f.DT,\n" + + " f.BECIF_NO,\n" + + " f.TRACE_ID;"); + + starRocksAssert.withMaterializedView("CREATE MATERIALIZED VIEW `test_perf_mv_pv2`\n" + + "PARTITION BY dt\n" + + "DISTRIBUTED BY HASH(`BECIF_NO`) BUCKETS 8\n" + + "REFRESH DEFERRED MANUAL\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"force_external_table_query_rewrite\" = \"true\"\n" + + ")\n" + + "AS\n" + + " SELECT\n" + + " f.DT,\n" + + " dim0.STRATEGY_ID_PDL AS STRATEGY_ID_PDL,\n" + + " f.BECIF_NO AS BECIF_NO,\n" + + " SUM(f.PV) AS pv2\n" + + " FROM\n" + + " IDX_COMMON_EV_BANKAPP_EXP_CLICK_INFO f\n" + + " LEFT JOIN IDX_BASE_DIM_TRACEID dim0 ON\n" + + " f.TRACE_ID = dim0.TRACE_ID\n" + + " AND f.DT = dim0.DT\n" + + " GROUP BY\n" + + " f.DT,\n" + + " f.BECIF_NO,\n" + + " dim0.STRATEGY_ID_PDL;"); + starRocksAssert.withMaterializedView("CREATE MATERIALIZED VIEW `test_perf_mv_pv3`\n" + + "PARTITION BY dt \n" + + "DISTRIBUTED BY HASH(`BECIF_NO`) BUCKETS 8\n" + + "REFRESH DEFERRED MANUAL\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"force_external_table_query_rewrite\" = \"true\"\n" + + ")\n" + + "AS\n" + + " SELECT\n" + + " f.DT,\n" + + " dim0.STRATEGY_ID_PDL AS STRATEGY_ID_PDL,\n" + + " f.BECIF_NO AS BECIF_NO,\n" + + " COUNT(f.PAGE_NAME_GY) AS cnt1\n" + + " FROM\n" + + " D_OPERATION_ORDER_CUST_DETAIL_PDL_ID3 f\n" + + " LEFT JOIN IDX_BASE_DIM_TRACEID dim0 ON\n" + + " f.TRACE_ID = dim0.TRACE_ID\n" + + " AND f.DT = dim0.DT\n" + + " GROUP BY\n" + + " f.DT,\n" + + " f.BECIF_NO,\n" + + " dim0.STRATEGY_ID_PDL;"); + starRocksAssert.withMaterializedView("\n" + + "CREATE MATERIALIZED VIEW `test_perf_mv_pv4`\n" + + "PARTITION BY dt\n" + + "DISTRIBUTED BY HASH(DT, SALES_NEW_CUST, PAYROLL_FLAG, STRATEGY_TYPE, CUST_GROUP_NAME)\n" + + "REFRESH DEFERRED MANUAL\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"force_external_table_query_rewrite\" = \"true\"\n" + + ")\n" + + "AS\n" + + "SELECT q.DT, f1.SALES_NEW_CUST AS SALES_NEW_CUST, f1.PAYROLL_FLAG AS PAYROLL_FLAG," + + " f7.STRATEGY_TYPE AS STRATEGY_TYPE, f7.CUST_GROUP_NAME AS CUST_GROUP_NAME\n" + + " , SUM(q.cnt1) AS cnt1, SUM(q.pv1) AS pv1\n" + + " , SUM(q.pv2) AS pv2\n" + + "FROM (\n" + + " SELECT t0.pv1 AS pv1, t1.pv2 AS pv2, t2.cnt1 AS cnt1\n" + + " , t0.DT AS DT\n" + + " , COALESCE(t0.BECIF_NO, t1.BECIF_NO, t2.BECIF_NO) AS pk1\n" + + " , COALESCE(t0.TRACE_ID, t1.STRATEGY_ID_PDL, t2.STRATEGY_ID_PDL) AS pk7\n" + + " FROM (\n" + + " SELECT f.pv1, f.TRACE_ID, f.BECIF_NO, f.DT\n" + + " FROM test_perf_mv_pv1 f\n" + + " ) t0\n" + + " FULL JOIN (\n" + + " SELECT f.STRATEGY_ID_PDL, f.BECIF_NO, f.pv2, f.DT\n" + + " FROM test_perf_mv_pv2 f\n" + + " ) t1\n" + + " ON t1.DT = t0.DT\n" + + " AND t1.BECIF_NO = t0.BECIF_NO\n" + + " AND t1.STRATEGY_ID_PDL = t0.TRACE_ID\n" + + " FULL JOIN (\n" + + " SELECT f.STRATEGY_ID_PDL, f.cnt1, f.BECIF_NO, f.DT\n" + + " FROM test_perf_mv_pv3 f\n" + + " ) t2\n" + + " ON t2.DT = COALESCE(t0.DT , t1.DT)\n" + + " AND t2.BECIF_NO = COALESCE(t0.BECIF_NO, t1.BECIF_NO)\n" + + " AND t2.STRATEGY_ID_PDL = COALESCE(t0.TRACE_ID, t1.STRATEGY_ID_PDL)\n" + + ") q\n" + + " LEFT JOIN (\n" + + " SELECT f.SALES_NEW_CUST AS SALES_NEW_CUST," + + " f.PAYROLL_FLAG AS PAYROLL_FLAG, f.BECIF_NO AS BECIF_NO, f.DT\n" + + " FROM IDX_CUST_COMMON_DIM f\n" + + " ) f1\n" + + " ON f1.BECIF_NO = q.pk1\n" + + " AND f1.DT = q.DT\n" + + " LEFT JOIN (\n" + + " SELECT f.STRATEGY_ID AS STRATEGY_ID, f.STRATEGY_TYPE AS STRATEGY_TYPE," + + " f.CUST_GROUP_NAME AS CUST_GROUP_NAME, f.DT\n" + + " FROM IDX_BASE_OBJ_STRATEGY f\n" + + " ) f7\n" + + " ON f7.STRATEGY_ID = q.pk7\n" + + " AND f7.DT = q.DT\n" + + "GROUP BY q.DT, f1.PAYROLL_FLAG, f7.CUST_GROUP_NAME, f7.STRATEGY_TYPE, f1.SALES_NEW_CUST;"); + String query = "SELECT SUM(q.pv1) AS pv1,\n" + + " f1.SALES_NEW_CUST AS SALES_NEW_CUST,\n" + + " SUM(q.cnt1) AS cnt1,\n" + + " f1.PAYROLL_FLAG AS PAYROLL_FLAG,\n" + + " f7.STRATEGY_TYPE AS STRATEGY_TYPE,\n" + + " f7.CUST_GROUP_NAME AS CUST_GROUP_NAME,\n" + + " SUM(q.pv2) AS pv2\n" + + "FROM\n" + + " ( SELECT t0.pv1 AS pv1,\n" + + " t1.pv2 AS pv2,\n" + + " t2.cnt1 AS cnt1,\n" + + " t0.DT AS DT ,\n" + + " COALESCE(t0.BECIF_NO , t1.BECIF_NO , t2.BECIF_NO) AS pk1 ,\n" + + " COALESCE(t0.TRACE_ID , t1.STRATEGY_ID_PDL , t2.STRATEGY_ID_PDL) AS pk7\n" + + " FROM\n" + + " ( SELECT SUM(f.PV) AS pv1,\n" + + " f.TRACE_ID AS TRACE_ID,\n" + + " f.BECIF_NO AS BECIF_NO,\n" + + " f.DT\n" + + " FROM IDX_COMMON_EV_BANKAPP_CLICK_INFO f\n" + + " GROUP BY f.DT,\n" + + " f.BECIF_NO,\n" + + " f.TRACE_ID) AS t0\n" + + " FULL JOIN\n" + + " ( SELECT dim0.STRATEGY_ID_PDL AS STRATEGY_ID_PDL,\n" + + " f.BECIF_NO AS BECIF_NO,\n" + + " SUM(f.PV) AS pv2,\n" + + " f.DT\n" + + " FROM IDX_COMMON_EV_BANKAPP_EXP_CLICK_INFO f\n" + + " LEFT JOIN IDX_BASE_DIM_TRACEID dim0 ON f.TRACE_ID = dim0.TRACE_ID\n" + + " AND f.DT = dim0.DT\n" + + " GROUP BY f.DT,\n" + + " f.BECIF_NO,\n" + + " dim0.STRATEGY_ID_PDL) AS t1 \n" + + " ON t1.DT = t0.DT\n" + + " AND t1.BECIF_NO = t0.BECIF_NO\n" + + " AND t1.STRATEGY_ID_PDL = t0.TRACE_ID\n" + + " FULL JOIN\n" + + " ( SELECT dim0.STRATEGY_ID_PDL AS STRATEGY_ID_PDL,\n" + + " COUNT(f.PAGE_NAME_GY) AS cnt1,\n" + + " f.BECIF_NO AS BECIF_NO,\n" + + " f.DT\n" + + " FROM D_OPERATION_ORDER_CUST_DETAIL_PDL_ID3 f\n" + + " LEFT JOIN IDX_BASE_DIM_TRACEID dim0 ON f.TRACE_ID = dim0.TRACE_ID\n" + + " AND f.DT = dim0.DT\n" + + " GROUP BY f.DT,\n" + + " f.BECIF_NO,\n" + + " dim0.STRATEGY_ID_PDL) AS t2\n" + + " ON t2.DT = COALESCE(t0.DT , t1.DT)\n" + + " AND t2.BECIF_NO = COALESCE(t0.BECIF_NO , t1.BECIF_NO)\n" + + " AND t2.STRATEGY_ID_PDL = COALESCE(t0.TRACE_ID , t1.STRATEGY_ID_PDL)) AS q\n" + + "LEFT JOIN\n" + + " ( SELECT f.SALES_NEW_CUST AS SALES_NEW_CUST,\n" + + " f.PAYROLL_FLAG AS PAYROLL_FLAG,\n" + + " f.BECIF_NO AS BECIF_NO,\n" + + " f.DT\n" + + " FROM IDX_CUST_COMMON_DIM f) AS f1 ON f1.BECIF_NO = q.pk1\n" + + "AND f1.DT = q.DT\n" + + "LEFT JOIN\n" + + " ( SELECT f.STRATEGY_ID AS STRATEGY_ID,\n" + + " f.STRATEGY_TYPE AS STRATEGY_TYPE,\n" + + " f.CUST_GROUP_NAME AS CUST_GROUP_NAME,\n" + + " f.DT\n" + + " FROM IDX_BASE_OBJ_STRATEGY f) AS f7 ON f7.STRATEGY_ID = q.pk7\n" + + "AND f7.DT = q.DT\n" + + "WHERE q.DT IN ('20230924')\n" + + " AND f1.PAYROLL_FLAG IN ('Y')\n" + + "GROUP BY f1.PAYROLL_FLAG,\n" + + " f7.CUST_GROUP_NAME,\n" + + " f7.STRATEGY_TYPE,\n" + + " f1.SALES_NEW_CUST"; + String plan = getFragmentPlan(query); + PlanTestBase.assertContains(plan, "test_perf_mv_pv4"); + starRocksAssert.dropMaterializedView("test_perf_mv_pv1"); + starRocksAssert.dropMaterializedView("test_perf_mv_pv2"); + starRocksAssert.dropMaterializedView("test_perf_mv_pv3"); + starRocksAssert.dropMaterializedView("test_perf_mv_pv4"); + starRocksAssert.dropTable("IDX_BASE_DIM_TRACEID"); + starRocksAssert.dropTable("IDX_BASE_OBJ_STRATEGY"); + starRocksAssert.dropTable("IDX_COMMON_EV_BANKAPP_EXP_CLICK_INFO"); + starRocksAssert.dropTable("IDX_COMMON_EV_BANKAPP_CLICK_INFO"); + starRocksAssert.dropTable("IDX_CUST_COMMON_DIM"); + starRocksAssert.dropTable("D_OPERATION_ORDER_CUST_DETAIL_PDL_ID3"); + } }