From 6f81e6cfc8d2e91280e1fcd163a0cf26fd0906d5 Mon Sep 17 00:00:00 2001 From: ABingHuang Date: Mon, 30 Jan 2023 21:03:38 +0800 Subject: [PATCH 1/3] fix unstable union mv rewrite ut Signed-off-by: ABingHuang --- .../clone/DynamicPartitionScheduler.java | 6 +++++ .../sql/optimizer/rule/RuleType.java | 1 + .../rule/AggregateAggregateScanRule.java | 2 +- .../MvRewriteOptimizationTest.java | 26 +++++++------------ 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/com/starrocks/clone/DynamicPartitionScheduler.java index 4e63c67a04d219..6694feb78f16aa 100644 --- a/fe/fe-core/src/main/java/com/starrocks/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/com/starrocks/clone/DynamicPartitionScheduler.java @@ -34,6 +34,7 @@ package com.starrocks.clone; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Maps; import com.google.common.collect.Range; import com.google.common.collect.Sets; @@ -535,6 +536,11 @@ private void initDynamicPartitionTable() { initialize = true; } + @VisibleForTesting + public void runOnceForTest() { + runAfterCatalogReady(); + } + @Override protected void runAfterCatalogReady() { if (!initialize) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/RuleType.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/RuleType.java index 911e3e8baddd14..f8b96a0d93dd05 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/RuleType.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/RuleType.java @@ -150,6 +150,7 @@ public enum RuleType { TF_MV_ONLY_JOIN_RULE, TF_MV_FILTER_JOIN_RULE, TF_MV_AGGREGATE_SCAN_RULE, + TF_MV_AGGREGATE_AGGREGATE_SCAN_RULE, TF_MV_AGGREGATE_FILTER_SCAN_RULE, TF_MV_AGGREGATE_JOIN_RULE, TF_MV_AGGREGATE_FILTER_JOIN_RULE, diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/AggregateAggregateScanRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/AggregateAggregateScanRule.java index 1cb785b5944404..6d24df340a30bf 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/AggregateAggregateScanRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/AggregateAggregateScanRule.java @@ -34,7 +34,7 @@ public class AggregateAggregateScanRule extends SingleTableRewriteBaseRule { private static AggregateAggregateScanRule INSTANCE = new AggregateAggregateScanRule(); public AggregateAggregateScanRule() { - super(RuleType.TF_MV_AGGREGATE_SCAN_RULE, Pattern.create(OperatorType.LOGICAL_AGGR) + super(RuleType.TF_MV_AGGREGATE_AGGREGATE_SCAN_RULE, Pattern.create(OperatorType.LOGICAL_AGGR) .addChildren(Pattern.create(OperatorType.LOGICAL_AGGR).addChildren(Pattern.create(OperatorType.PATTERN_SCAN)))); } diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java index d22593be81c7f1..91a79255e32038 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java @@ -44,7 +44,7 @@ public class MvRewriteOptimizationTest { @BeforeClass public static void beforeClass() throws Exception { - Config.dynamic_partition_check_interval_seconds = 1; + Config.dynamic_partition_check_interval_seconds = 10000; Config.bdbje_heartbeat_timeout_second = 60; Config.bdbje_replica_ack_timeout_second = 60; Config.bdbje_lock_timeout_second = 60; @@ -1261,6 +1261,7 @@ public void testUnionRewrite() throws Exception { cluster.runSql("test", "insert into test_base_part values(1000, 1, 2, 3)"); cluster.runSql("test", "insert into test_base_part values(2000, 1, 2, 3)"); cluster.runSql("test", "insert into test_base_part values(2500, 1, 2, 3)"); + createAndRefreshMv("test", "ttl_union_mv_1", "CREATE MATERIALIZED VIEW `ttl_union_mv_1`\n" + "COMMENT \"MATERIALIZED_VIEW\"\n" + "PARTITION BY (`c3`)\n" + @@ -1274,12 +1275,16 @@ public void testUnionRewrite() throws Exception { "AS SELECT `test_base_part`.`c1`, `test_base_part`.`c3`, sum(`test_base_part`.`c2`) AS `c2`\n" + "FROM `test_base_part`\n" + "GROUP BY `test_base_part`.`c1`, `test_base_part`.`c3`;"); + MaterializedView ttlMv1 = getMv("test", "ttl_union_mv_1"); Assert.assertNotNull(ttlMv1); - waitTtl(ttlMv1, 3, 200); + GlobalStateMgr.getCurrentState().getDynamicPartitionScheduler().runOnceForTest(); + Assert.assertEquals(3, ttlMv1.getPartitions().size()); + String query4 = "select c3, sum(c2) from test_base_part group by c3"; String plan4 = getFragmentPlan(query4); PlanTestBase.assertContains(plan4, "ttl_union_mv_1", "UNION", "test_base_part"); + PlanTestBase.assertNotContains(plan4, "ttl_union_mv_1"); dropMv("test", "ttl_union_mv_1"); starRocksAssert.withTable("CREATE TABLE multi_mv_table (\n" + @@ -1377,20 +1382,6 @@ public void testNestedMv() throws Exception { starRocksAssert.dropTable("nest_base_table_1"); } - private void waitTtl(MaterializedView mv, int number, int maxRound) throws InterruptedException, TimeoutException { - int round = 0; - while (true) { - if (mv.getPartitions().size() == number) { - break; - } - if (round >= maxRound) { - throw new TimeoutException("wait ttl timeout"); - } - Thread.sleep(1000); - round++; - } - } - @Test public void testPartialPartition() throws Exception { starRocksAssert.getCtx().getSessionVariable().setEnableMaterializedViewUnionRewrite(true); @@ -1598,7 +1589,8 @@ public void testPartialPartition() throws Exception { " )\n" + " AS SELECT k1, sum(v1) as sum_v1 FROM ttl_base_table group by k1;"); MaterializedView ttlMv2 = getMv("test", "ttl_mv_2"); - waitTtl(ttlMv2, 4, 100); + GlobalStateMgr.getCurrentState().getDynamicPartitionScheduler().runOnceForTest(); + Assert.assertEquals(4, ttlMv2.getPartitions().size()); String query16 = "select k1, sum(v1) FROM ttl_base_table where k1=3 group by k1"; String plan16 = getFragmentPlan(query16); From 61d5f758188c91891baaa65624e90a53af40ca5a Mon Sep 17 00:00:00 2001 From: ABingHuang Date: Mon, 30 Jan 2023 21:07:19 +0800 Subject: [PATCH 2/3] fix compile Signed-off-by: ABingHuang --- .../materialization/MvRewriteOptimizationTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java index 91a79255e32038..5b11351ba3500e 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java @@ -20,7 +20,6 @@ import com.starrocks.catalog.OlapTable; import com.starrocks.catalog.Table; import com.starrocks.common.Config; -import com.starrocks.common.TimeoutException; import com.starrocks.pseudocluster.PseudoCluster; import com.starrocks.qe.ConnectContext; import com.starrocks.scheduler.Task; From 2d0295d25d9c1a72a07702b045edd6a2403af0d6 Mon Sep 17 00:00:00 2001 From: ABingHuang Date: Tue, 31 Jan 2023 10:17:42 +0800 Subject: [PATCH 3/3] fix ut --- .../materialization/MvRewriteOptimizationTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java index 5b11351ba3500e..6d91f3867c7520 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteOptimizationTest.java @@ -1283,7 +1283,6 @@ public void testUnionRewrite() throws Exception { String query4 = "select c3, sum(c2) from test_base_part group by c3"; String plan4 = getFragmentPlan(query4); PlanTestBase.assertContains(plan4, "ttl_union_mv_1", "UNION", "test_base_part"); - PlanTestBase.assertNotContains(plan4, "ttl_union_mv_1"); dropMv("test", "ttl_union_mv_1"); starRocksAssert.withTable("CREATE TABLE multi_mv_table (\n" +