diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MaterializedViewOptimizer.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MaterializedViewOptimizer.java index a13a31771868b..259302b5b2d57 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MaterializedViewOptimizer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MaterializedViewOptimizer.java @@ -20,13 +20,25 @@ import com.starrocks.common.Pair; import com.starrocks.qe.ConnectContext; import com.starrocks.sql.optimizer.base.ColumnRefFactory; +import com.starrocks.sql.optimizer.rule.RuleSetType; +import com.starrocks.sql.optimizer.rule.RuleType; import com.starrocks.sql.optimizer.rule.transformation.materialization.MvUtils; import com.starrocks.sql.optimizer.transformer.LogicalPlan; public class MaterializedViewOptimizer { public MvPlanContext optimize(MaterializedView mv, - ConnectContext connectContext, - OptimizerConfig optimizerConfig) { + ConnectContext connectContext) { + // optimize the sql by rule and disable rule based materialized view rewrite + OptimizerConfig optimizerConfig = new OptimizerConfig(OptimizerConfig.OptimizerAlgorithm.RULE_BASED); + optimizerConfig.disableRuleSet(RuleSetType.PARTITION_PRUNE); + optimizerConfig.disableRuleSet(RuleSetType.SINGLE_TABLE_MV_REWRITE); + optimizerConfig.disableRule(RuleType.TF_REWRITE_GROUP_BY_COUNT_DISTINCT); + optimizerConfig.disableRule(RuleType.TF_PRUNE_EMPTY_SCAN); + // For sync mv, no rewrite query by original sync mv rule to avoid useless rewrite. + if (mv.getRefreshScheme().isSync()) { + optimizerConfig.disableRule(RuleType.TF_MATERIALIZED_VIEW); + } + ColumnRefFactory columnRefFactory = new ColumnRefFactory(); String mvSql = mv.getViewDefineSql(); Pair plans = diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvPlanContextBuilder.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvPlanContextBuilder.java index 0159c7737cedd..2b6b629a97e32 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvPlanContextBuilder.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvPlanContextBuilder.java @@ -17,24 +17,11 @@ import com.starrocks.catalog.MaterializedView; import com.starrocks.catalog.MvPlanContext; import com.starrocks.qe.ConnectContext; -import com.starrocks.sql.optimizer.rule.RuleSetType; -import com.starrocks.sql.optimizer.rule.RuleType; public class MvPlanContextBuilder { public MvPlanContext getPlanContext(MaterializedView mv) { // build mv query logical plan MaterializedViewOptimizer mvOptimizer = new MaterializedViewOptimizer(); - // optimize the sql by rule and disable rule based materialized view rewrite - OptimizerConfig optimizerConfig = new OptimizerConfig(OptimizerConfig.OptimizerAlgorithm.RULE_BASED); - optimizerConfig.disableRuleSet(RuleSetType.PARTITION_PRUNE); - optimizerConfig.disableRuleSet(RuleSetType.SINGLE_TABLE_MV_REWRITE); - optimizerConfig.disableRule(RuleType.TF_REWRITE_GROUP_BY_COUNT_DISTINCT); - optimizerConfig.disableRule(RuleType.TF_PRUNE_EMPTY_SCAN); - // For sync mv, no rewrite query by original sync mv rule to avoid useless rewrite. - if (mv.getRefreshScheme().isSync()) { - optimizerConfig.disableRule(RuleType.TF_MATERIALIZED_VIEW); - } - optimizerConfig.setMVRewritePlan(true); - return mvOptimizer.optimize(mv, new ConnectContext(), optimizerConfig); + return mvOptimizer.optimize(mv, new ConnectContext()); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java index 83c24a80a5b06..a081e59e6dcdc 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java @@ -56,6 +56,7 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -183,11 +184,14 @@ public void prepareRelatedMVs(Set queryTables, Set rela logMVPrepare(connectContext, "queryExcludingMVNames:{}, queryIncludingMVNames:{}", Strings.nullToEmpty(queryExcludingMVNames), Strings.nullToEmpty(queryIncludingMVNames)); - Set queryExcludingMVNamesSet = Sets.newHashSet(queryExcludingMVNames.split(",")); - Set queryIncludingMVNamesSet = Sets.newHashSet(queryIncludingMVNames.split(",")); + final Set queryExcludingMVNamesSet = Strings.isNullOrEmpty(queryExcludingMVNames) ? Sets.newHashSet() + : Arrays.stream(queryExcludingMVNames.split(",")).map(String::trim).collect(Collectors.toSet()); + + final Set queryIncludingMVNamesSet = Strings.isNullOrEmpty(queryIncludingMVNames) ? Sets.newHashSet() + : Arrays.stream(queryIncludingMVNames.split(",")).map(String::trim).collect(Collectors.toSet()); relatedMvs = relatedMvs.stream() - .filter(mv -> queryIncludingMVNamesSet.contains(mv.getName())) - .filter(mv -> !queryExcludingMVNamesSet.contains(mv.getName())) + .filter(mv -> queryIncludingMVNamesSet.isEmpty() || queryIncludingMVNamesSet.contains(mv.getName())) + .filter(mv -> queryExcludingMVNamesSet.isEmpty() || !queryExcludingMVNamesSet.contains(mv.getName())) .collect(Collectors.toSet()); } if (relatedMvs.isEmpty()) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerConfig.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerConfig.java index 279c11eb4c144..eaff5b9a9a402 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerConfig.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerConfig.java @@ -31,8 +31,6 @@ public enum OptimizerAlgorithm { private BitSet ruleSetSwitches; private BitSet ruleSwitches; - private boolean isMVRewritePlan; - private static final OptimizerConfig DEFAULT_CONFIG = new OptimizerConfig(); public static OptimizerConfig defaultConfig() { @@ -70,12 +68,4 @@ public void disableRule(RuleType ruleType) { public boolean isRuleDisable(RuleType ruleType) { return !ruleSwitches.get(ruleType.ordinal()); } - - public boolean isMVRewritePlan() { - return this.isMVRewritePlan; - } - - public void setMVRewritePlan(boolean isMVRewritePlan) { - this.isMVRewritePlan = isMVRewritePlan; - } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/dump/QueryDumpInfo.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/dump/QueryDumpInfo.java index 585e0315f55c1..3566cbf41d19d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/dump/QueryDumpInfo.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/dump/QueryDumpInfo.java @@ -27,7 +27,6 @@ import com.starrocks.qe.VariableMgr; import com.starrocks.sql.ast.StatementBase; import com.starrocks.sql.optimizer.MaterializedViewOptimizer; -import com.starrocks.sql.optimizer.OptimizerConfig; import com.starrocks.sql.optimizer.statistics.ColumnStatistic; import java.util.ArrayList; @@ -120,11 +119,10 @@ public void addTable(String dbName, Table table) { connectContext.getSessionVariable().setQueryExcludingMVNames(table.getName()); { MaterializedViewOptimizer mvOptimizer = new MaterializedViewOptimizer(); - OptimizerConfig optimizerConfig = new OptimizerConfig(OptimizerConfig.OptimizerAlgorithm.COST_BASED); // NOTE: Since materialized view support unique/foreign constraints, we use `optimize` here to visit // all dependent tables again to add it into `dump info`. // NOTE: The optimizer should not contain self to avoid stack overflow. - mvOptimizer.optimize((MaterializedView) table, connectContext, optimizerConfig); + mvOptimizer.optimize((MaterializedView) table, connectContext); tableMap.put(table.getId(), new Pair<>(dbName, table)); } connectContext.getSessionVariable().setQueryExcludingMVNames(queryExcludingMVNames); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteTest.java index 036fe0195cc43..8f6188ef540a4 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteTest.java @@ -2010,4 +2010,77 @@ public void testMVAggregateTable() throws Exception { starRocksAssert.dropMaterializedView("mv_t1_v0"); starRocksAssert.dropTable("t1_agg"); } + + @Test + public void testQueryIncludingExcludingMVNames() throws Exception { + starRocksAssert.getCtx().getSessionVariable().setOptimizerExecuteTimeout(3000000); + createAndRefreshMv("test", "mv_agg_1", "CREATE MATERIALIZED VIEW mv_agg_1 " + + " distributed by hash(empid) " + + "AS " + + "SELECT empid, sum(salary) as total " + + "FROM emps " + + "GROUP BY empid"); + createAndRefreshMv("test", "mv_agg_2", "CREATE MATERIALIZED VIEW mv_agg_2 " + + " distributed by hash(empid) " + + "AS " + + "SELECT empid, sum(salary) as total " + + "FROM emps " + + "GROUP BY empid"); + { + starRocksAssert.getCtx().getSessionVariable().setQueryIncludingMVNames("mv_agg_1"); + String query = "SELECT empid, sum(salary) as total " + + "FROM emps " + + "GROUP BY empid"; + String plan = getFragmentPlan(query); + PlanTestBase.assertContains(plan, "mv_agg_1"); + starRocksAssert.getCtx().getSessionVariable().setQueryIncludingMVNames(""); + } + { + starRocksAssert.getCtx().getSessionVariable().setQueryIncludingMVNames("mv_agg_2"); + String query = "SELECT empid, sum(salary) as total " + + "FROM emps " + + "GROUP BY empid"; + String plan = getFragmentPlan(query); + PlanTestBase.assertContains(plan, "mv_agg_2"); + starRocksAssert.getCtx().getSessionVariable().setQueryIncludingMVNames(""); + } + { + starRocksAssert.getCtx().getSessionVariable().setQueryIncludingMVNames("mv_agg_1, mv_agg_2"); + String query = "SELECT empid, sum(salary) as total " + + "FROM emps " + + "GROUP BY empid"; + String plan = getFragmentPlan(query); + PlanTestBase.assertContains(plan, "mv_agg_"); + starRocksAssert.getCtx().getSessionVariable().setQueryIncludingMVNames(""); + } + { + starRocksAssert.getCtx().getSessionVariable().setQueryExcludingMVNames("mv_agg_1"); + String query = "SELECT empid, sum(salary) as total " + + "FROM emps " + + "GROUP BY empid"; + String plan = getFragmentPlan(query); + PlanTestBase.assertContains(plan, "mv_agg_2"); + starRocksAssert.getCtx().getSessionVariable().setQueryExcludingMVNames(""); + } + { + starRocksAssert.getCtx().getSessionVariable().setQueryExcludingMVNames("mv_agg_2"); + String query = "SELECT empid, sum(salary) as total " + + "FROM emps " + + "GROUP BY empid"; + String plan = getFragmentPlan(query); + PlanTestBase.assertContains(plan, "mv_agg_1"); + starRocksAssert.getCtx().getSessionVariable().setQueryExcludingMVNames(""); + } + { + starRocksAssert.getCtx().getSessionVariable().setQueryExcludingMVNames("mv_agg_1, mv_agg_2"); + String query = "SELECT empid, sum(salary) as total " + + "FROM emps " + + "GROUP BY empid"; + String plan = getFragmentPlan(query); + PlanTestBase.assertNotContains(plan, "mv_agg_"); + starRocksAssert.getCtx().getSessionVariable().setQueryExcludingMVNames(""); + } + starRocksAssert.dropMaterializedView("mv_agg_1"); + starRocksAssert.dropMaterializedView("mv_agg_2"); + } }