diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java new file mode 100644 index 000000000000000..bb8bc4c1be52b1c --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java @@ -0,0 +1,406 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe; + +import org.apache.doris.analysis.CreateCatalogStmt; +import org.apache.doris.analysis.CreateDbStmt; +import org.apache.doris.analysis.CreateTableStmt; +import org.apache.doris.analysis.StatementBase; +import org.apache.doris.analysis.TupleDescriptor; +import org.apache.doris.analysis.TupleId; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.external.HMSExternalDatabase; +import org.apache.doris.catalog.external.HMSExternalTable; +import org.apache.doris.catalog.external.HMSExternalTable.DLAType; +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.CatalogMgr; +import org.apache.doris.datasource.HMSExternalCatalog; +import org.apache.doris.nereids.datasets.tpch.AnalyzeCheckTestBase; +import org.apache.doris.planner.OlapScanNode; +import org.apache.doris.planner.PlanNodeId; +import org.apache.doris.planner.ScanNode; +import org.apache.doris.planner.external.HiveScanNode; +import org.apache.doris.qe.cache.CacheAnalyzer; +import org.apache.doris.qe.cache.SqlCache; + +import com.google.common.collect.Lists; +import mockit.Expectations; +import mockit.Mocked; +import org.junit.Assert; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.locks.ReentrantReadWriteLock; + + +public class HmsQueryCacheTest extends AnalyzeCheckTestBase { + private static final String HMS_CATALOG = "hms_ctl"; + private static final long NOW = System.currentTimeMillis(); + private Env env; + private CatalogMgr mgr; + private OlapScanNode olapScanNode; + + @Mocked + private HMSExternalTable tbl; + @Mocked + private HMSExternalTable view1; + @Mocked + private HMSExternalTable view2; + @Mocked + private HiveScanNode hiveScanNode1; + @Mocked + private HiveScanNode hiveScanNode2; + @Mocked + private HiveScanNode hiveScanNode3; + + @Override + protected void runBeforeAll() throws Exception { + FeConstants.runningUnitTest = true; + Config.enable_query_hive_views = true; + Config.cache_enable_sql_mode = true; + Config.cache_enable_partition_mode = true; + connectContext.getSessionVariable().setEnableSqlCache(true); + + env = Env.getCurrentEnv(); + connectContext.setEnv(env); + mgr = env.getCatalogMgr(); + + // create hms catalog + CreateCatalogStmt hmsCatalogStmt = (CreateCatalogStmt) parseAndAnalyzeStmt( + "create catalog hms_ctl properties('type' = 'hms', 'hive.metastore.uris' = 'thrift://192.168.0.1:9083');", + connectContext); + mgr.createCatalog(hmsCatalogStmt); + + // create inner db and tbl for test + CreateDbStmt createDbStmt = (CreateDbStmt) parseAndAnalyzeStmt("create database test", connectContext); + mgr.getInternalCatalog().createDb(createDbStmt); + + CreateTableStmt createTableStmt = (CreateTableStmt) parseAndAnalyzeStmt("create table test.tbl1(\n" + + "k1 int comment 'test column k1', " + + "k2 int comment 'test column k2') comment 'test table1' " + + "distributed by hash(k1) buckets 1\n" + + "properties(\"replication_num\" = \"1\");"); + mgr.getInternalCatalog().createTable(createTableStmt); + } + + private void init(HMSExternalCatalog hmsCatalog) { + Deencapsulation.setField(hmsCatalog, "initialized", true); + Deencapsulation.setField(hmsCatalog, "objectCreated", true); + + List schema = Lists.newArrayList(); + schema.add(new Column("k1", PrimitiveType.INT)); + + HMSExternalDatabase db = new HMSExternalDatabase(hmsCatalog, 10000, "hms_db"); + Deencapsulation.setField(db, "initialized", true); + + Deencapsulation.setField(tbl, "objectCreated", true); + Deencapsulation.setField(tbl, "rwLock", new ReentrantReadWriteLock(true)); + new Expectations(tbl) { + { + tbl.getId(); + minTimes = 0; + result = 10001; + + tbl.getName(); + minTimes = 0; + result = "hms_tbl"; + + tbl.getDbName(); + minTimes = 0; + result = "hms_db"; + + tbl.getFullSchema(); + minTimes = 0; + result = schema; + + tbl.isSupportedHmsTable(); + minTimes = 0; + result = true; + + tbl.isView(); + minTimes = 0; + result = false; + + tbl.getType(); + minTimes = 0; + result = TableIf.TableType.HMS_EXTERNAL_TABLE; + + tbl.getDlaType(); + minTimes = 0; + result = DLAType.HIVE; + + tbl.getLastUpdateTime(); + minTimes = 0; + result = NOW; + } + }; + + Deencapsulation.setField(view1, "objectCreated", true); + Deencapsulation.setField(view1, "rwLock", new ReentrantReadWriteLock(true)); + + new Expectations(view1) { + { + view1.getId(); + minTimes = 0; + result = 10002; + + view1.getName(); + minTimes = 0; + result = "hms_view1"; + + view1.getDbName(); + minTimes = 0; + result = "hms_db"; + + view1.isView(); + minTimes = 0; + result = true; + + view1.getCatalog(); + minTimes = 0; + result = hmsCatalog; + + view1.getType(); + minTimes = 0; + result = TableIf.TableType.HMS_EXTERNAL_TABLE; + + view1.getFullSchema(); + minTimes = 0; + result = schema; + + view1.getViewText(); + minTimes = 0; + result = "SELECT * FROM hms_db.hms_tbl"; + + view1.isSupportedHmsTable(); + minTimes = 0; + result = true; + + view1.getDlaType(); + minTimes = 0; + result = DLAType.HIVE; + + view1.getLastUpdateTime(); + minTimes = 0; + result = NOW; + } + }; + + Deencapsulation.setField(view2, "objectCreated", true); + Deencapsulation.setField(view2, "rwLock", new ReentrantReadWriteLock(true)); + new Expectations(view2) { + { + view2.getId(); + minTimes = 0; + result = 10003; + + view2.getName(); + minTimes = 0; + result = "hms_view2"; + + view2.getDbName(); + minTimes = 0; + result = "hms_db"; + + view2.isView(); + minTimes = 0; + result = true; + + view2.getCatalog(); + minTimes = 0; + result = hmsCatalog; + + view2.getType(); + minTimes = 0; + result = TableIf.TableType.HMS_EXTERNAL_TABLE; + + view2.getFullSchema(); + minTimes = 0; + result = schema; + + view2.getViewText(); + minTimes = 0; + result = "SELECT * FROM hms_db.hms_view1"; + + view2.isSupportedHmsTable(); + minTimes = 0; + result = true; + + view2.getDlaType(); + minTimes = 0; + result = DLAType.HIVE; + + view2.getLastUpdateTime(); + minTimes = 0; + result = NOW; + } + }; + + db.addTableForTest(tbl); + db.addTableForTest(view1); + db.addTableForTest(view2); + hmsCatalog.addDatabaseForTest(db); + + new Expectations(hiveScanNode1) { + { + hiveScanNode1.getTargetTable(); + minTimes = 0; + result = tbl; + } + }; + + new Expectations(hiveScanNode2) { + { + hiveScanNode2.getTargetTable(); + minTimes = 0; + result = view1; + } + }; + + new Expectations(hiveScanNode3) { + { + hiveScanNode3.getTargetTable(); + minTimes = 0; + result = view2; + } + }; + + TupleDescriptor desc = new TupleDescriptor(new TupleId(1)); + desc.setTable(mgr.getInternalCatalog().getDbNullable("default_cluster:test").getTableNullable("tbl1")); + olapScanNode = new OlapScanNode(new PlanNodeId(1), desc, "tb1ScanNode"); + } + + @Test + public void testHitSqlCache() throws Exception { + init((HMSExternalCatalog) mgr.getCatalog(HMS_CATALOG)); + StatementBase parseStmt = parseAndAnalyzeStmt("select * from hms_ctl.hms_db.hms_tbl", connectContext); + List scanNodes = Arrays.asList(hiveScanNode1); + CacheAnalyzer ca = new CacheAnalyzer(connectContext, parseStmt, scanNodes); + ca.checkCacheMode(NOW + Config.cache_last_version_interval_second * 1000L * 2); + Assert.assertEquals(ca.getCacheMode(), CacheAnalyzer.CacheMode.Sql); + } + + @Test + public void testHitSqlCacheByNereids() throws Exception { + init((HMSExternalCatalog) mgr.getCatalog(HMS_CATALOG)); + StatementBase parseStmt = analyzeAndGetStmtByNereids("select * from hms_ctl.hms_db.hms_tbl", connectContext); + List scanNodes = Arrays.asList(hiveScanNode1); + CacheAnalyzer ca = new CacheAnalyzer(connectContext, parseStmt, scanNodes); + ca.checkCacheModeForNereids(NOW + Config.cache_last_version_interval_second * 1000L * 2); + Assert.assertEquals(ca.getCacheMode(), CacheAnalyzer.CacheMode.Sql); + } + + @Test + public void testHitSqlCacheWithHiveView() throws Exception { + init((HMSExternalCatalog) mgr.getCatalog(HMS_CATALOG)); + StatementBase parseStmt = parseAndAnalyzeStmt("select * from hms_ctl.hms_db.hms_view1", connectContext); + List scanNodes = Arrays.asList(hiveScanNode2); + CacheAnalyzer ca = new CacheAnalyzer(connectContext, parseStmt, scanNodes); + ca.checkCacheMode(NOW + Config.cache_last_version_interval_second * 1000L * 2); + Assert.assertEquals(ca.getCacheMode(), CacheAnalyzer.CacheMode.Sql); + } + + @Test + public void testHitSqlCacheWithHiveViewByNereids() throws Exception { + init((HMSExternalCatalog) mgr.getCatalog(HMS_CATALOG)); + StatementBase parseStmt = analyzeAndGetStmtByNereids("select * from hms_ctl.hms_db.hms_view1", connectContext); + List scanNodes = Arrays.asList(hiveScanNode2); + CacheAnalyzer ca = new CacheAnalyzer(connectContext, parseStmt, scanNodes); + ca.checkCacheModeForNereids(NOW + Config.cache_last_version_interval_second * 1000L * 2); + Assert.assertEquals(ca.getCacheMode(), CacheAnalyzer.CacheMode.Sql); + } + + @Test + public void testHitSqlCacheWithNestedHiveView() throws Exception { + init((HMSExternalCatalog) mgr.getCatalog(HMS_CATALOG)); + StatementBase parseStmt = parseAndAnalyzeStmt("select * from hms_ctl.hms_db.hms_view2", connectContext); + List scanNodes = Arrays.asList(hiveScanNode3); + CacheAnalyzer ca = new CacheAnalyzer(connectContext, parseStmt, scanNodes); + ca.checkCacheMode(NOW + Config.cache_last_version_interval_second * 1000L * 2); + Assert.assertEquals(ca.getCacheMode(), CacheAnalyzer.CacheMode.Sql); + SqlCache sqlCache = (SqlCache) ca.getCache(); + String cacheKey = sqlCache.getSqlWithViewStmt(); + Assert.assertEquals(cacheKey, "SELECT `hms_ctl`.`default_cluster:hms_db`.`hms_view2`.`k1` AS `k1` " + + "FROM `hms_ctl`.`default_cluster:hms_db`.`hms_view2`" + + "|SELECT * FROM hms_db.hms_tbl|SELECT * FROM hms_db.hms_view1"); + } + + @Test + public void testHitSqlCacheWithNestedHiveViewByNereids() throws Exception { + init((HMSExternalCatalog) mgr.getCatalog(HMS_CATALOG)); + StatementBase parseStmt = analyzeAndGetStmtByNereids("select * from hms_ctl.hms_db.hms_view2", connectContext); + List scanNodes = Arrays.asList(hiveScanNode3); + CacheAnalyzer ca = new CacheAnalyzer(connectContext, parseStmt, scanNodes); + ca.checkCacheModeForNereids(NOW + Config.cache_last_version_interval_second * 1000L * 2); + Assert.assertEquals(ca.getCacheMode(), CacheAnalyzer.CacheMode.Sql); + SqlCache sqlCache = (SqlCache) ca.getCache(); + String cacheKey = sqlCache.getSqlWithViewStmt(); + Assert.assertEquals(cacheKey, "select * from hms_ctl.hms_db.hms_view2" + + "|SELECT * FROM hms_db.hms_tbl|SELECT * FROM hms_db.hms_view1"); + } + + @Test + public void testNotHitSqlCache() throws Exception { + init((HMSExternalCatalog) mgr.getCatalog(HMS_CATALOG)); + StatementBase parseStmt = parseAndAnalyzeStmt("select * from hms_ctl.hms_db.hms_tbl", connectContext); + List scanNodes = Arrays.asList(hiveScanNode1); + CacheAnalyzer ca = new CacheAnalyzer(connectContext, parseStmt, scanNodes); + ca.checkCacheMode(NOW); + Assert.assertEquals(ca.getCacheMode(), CacheAnalyzer.CacheMode.None); + } + + @Test + public void testNotHitSqlCacheByNereids() throws Exception { + init((HMSExternalCatalog) mgr.getCatalog(HMS_CATALOG)); + StatementBase parseStmt = analyzeAndGetStmtByNereids("select * from hms_ctl.hms_db.hms_tbl", connectContext); + List scanNodes = Arrays.asList(hiveScanNode1); + CacheAnalyzer ca = new CacheAnalyzer(connectContext, parseStmt, scanNodes); + ca.checkCacheModeForNereids(NOW); + Assert.assertEquals(ca.getCacheMode(), CacheAnalyzer.CacheMode.None); + } + + @Test + public void testNotHitSqlCacheWithFederatedQuery() throws Exception { + init((HMSExternalCatalog) mgr.getCatalog(HMS_CATALOG)); + // cache mode is None if this query is a federated query + StatementBase parseStmt = parseAndAnalyzeStmt("select * from hms_ctl.hms_db.hms_tbl " + + "inner join internal.test.tbl1", connectContext); + List scanNodes = Arrays.asList(hiveScanNode1, olapScanNode); + CacheAnalyzer ca = new CacheAnalyzer(connectContext, parseStmt, scanNodes); + ca.checkCacheMode(NOW + Config.cache_last_version_interval_second * 1000L * 2); + Assert.assertEquals(ca.getCacheMode(), CacheAnalyzer.CacheMode.None); + } + + @Test + public void testNotHitSqlCacheWithFederatedQueryByNereids() throws Exception { + init((HMSExternalCatalog) mgr.getCatalog(HMS_CATALOG)); + // cache mode is None if this query is a federated query + StatementBase parseStmt = analyzeAndGetStmtByNereids("select * from hms_ctl.hms_db.hms_tbl " + + "inner join internal.test.tbl1", connectContext); + List scanNodes = Arrays.asList(hiveScanNode1, olapScanNode); + CacheAnalyzer ca = new CacheAnalyzer(connectContext, parseStmt, scanNodes); + ca.checkCacheModeForNereids(NOW + Config.cache_last_version_interval_second * 1000L * 2); + Assert.assertEquals(ca.getCacheMode(), CacheAnalyzer.CacheMode.None); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java new file mode 100644 index 000000000000000..c3b13aee9912bf7 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java @@ -0,0 +1,1379 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe; + +import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.analysis.SqlParser; +import org.apache.doris.analysis.SqlScanner; +import org.apache.doris.analysis.StatementBase; +import org.apache.doris.analysis.TupleDescriptor; +import org.apache.doris.analysis.TupleId; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.FunctionSet; +import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexState; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.PartitionInfo; +import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.RandomDistributionInfo; +import org.apache.doris.catalog.RangePartitionInfo; +import org.apache.doris.catalog.RangePartitionItem; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.Type; +import org.apache.doris.catalog.View; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; +import org.apache.doris.common.UserException; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.common.telemetry.Telemetry; +import org.apache.doris.common.util.SqlParserUtils; +import org.apache.doris.common.util.Util; +import org.apache.doris.datasource.CatalogMgr; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.metric.MetricRepo; +import org.apache.doris.mysql.MysqlChannel; +import org.apache.doris.mysql.MysqlSerializer; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.MockedAuth; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.glue.LogicalPlanAdapter; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.planner.OlapScanNode; +import org.apache.doris.planner.PlanNodeId; +import org.apache.doris.planner.ScanNode; +import org.apache.doris.proto.Types; +import org.apache.doris.qe.cache.Cache; +import org.apache.doris.qe.cache.CacheAnalyzer; +import org.apache.doris.qe.cache.CacheAnalyzer.CacheMode; +import org.apache.doris.qe.cache.CacheCoordinator; +import org.apache.doris.qe.cache.CacheProxy; +import org.apache.doris.qe.cache.PartitionCache; +import org.apache.doris.qe.cache.PartitionRange; +import org.apache.doris.qe.cache.RowBatchBuilder; +import org.apache.doris.qe.cache.SqlCache; +import org.apache.doris.service.FrontendOptions; +import org.apache.doris.system.Backend; +import org.apache.doris.system.SystemInfoService; +import org.apache.doris.thrift.TStorageType; +import org.apache.doris.thrift.TUniqueId; + +import com.google.common.collect.Lists; +import com.google.common.collect.Range; +import mockit.Expectations; +import mockit.Mock; +import mockit.MockUp; +import mockit.Mocked; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.StringReader; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.function.Function; + +public class OlapQueryCacheTest { + private static final Logger LOG = LogManager.getLogger(OlapQueryCacheTest.class); + public static String clusterName = "testCluster"; + public static String dbName = "testDb"; + public static String fullDbName = "testCluster:testDb"; + public static String tableName = "testTbl"; + public static String userName = "testUser"; + + private static ConnectContext context; + + private List newRangeList; + private Cache.HitRange hitRange; + private Analyzer analyzer; + private Database db; + + @Mocked + private AccessControllerManager accessManager; + @Mocked + private SystemInfoService service; + @Mocked + private Env env; + @Mocked + private InternalCatalog catalog; + @Mocked + private ConnectContext ctx; + @Mocked + MysqlChannel channel; + @Mocked + ConnectScheduler scheduler; + + @BeforeClass + public static void start() { + MetricRepo.init(); + try { + FrontendOptions.init(); + context = new ConnectContext(); + Config.cache_enable_sql_mode = true; + Config.cache_enable_partition_mode = true; + context.getSessionVariable().setEnableSqlCache(true); + context.getSessionVariable().setEnablePartitionCache(true); + Config.cache_last_version_interval_second = 7200; + } catch (UnknownHostException e) { + e.printStackTrace(); + } + } + + @Before + public void setUp() throws Exception { + MockedAuth.mockedAccess(accessManager); + MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1"); + new MockUp() { + @Mock + public boolean showHiddenColumns() { + return true; + } + }; + new MockUp() { + @Mock + public SystemInfoService getCurrentSystemInfo() { + return service; + } + }; + db = new Database(1L, fullDbName); + + OlapTable tbl1 = createOrderTable(); + OlapTable tbl2 = createProfileTable(); + OlapTable tbl3 = createEventTable(); + db.createTable(tbl1); + db.createTable(tbl2); + db.createTable(tbl3); + + View view1 = createEventView1(); + View view2 = createEventView2(); + View view3 = createEventView3(); + View view4 = createEventNestedView(); + db.createTable(view1); + db.createTable(view2); + db.createTable(view3); + db.createTable(view4); + + new Expectations(catalog) { + { + catalog.getDbNullable(fullDbName); + minTimes = 0; + result = db; + + catalog.getDbNullable(dbName); + minTimes = 0; + result = db; + + catalog.getDbNullable(db.getId()); + minTimes = 0; + result = db; + + catalog.getDbNames(); + minTimes = 0; + result = Lists.newArrayList(fullDbName); + } + }; + + CatalogMgr dsMgr = new CatalogMgr(); + new Expectations(dsMgr) { + { + dsMgr.getCatalog((String) any); + minTimes = 0; + result = catalog; + + dsMgr.getCatalogOrException((String) any, (Function) any); + minTimes = 0; + result = catalog; + + dsMgr.getCatalogOrAnalysisException((String) any); + minTimes = 0; + result = catalog; + } + }; + + new Expectations(env) { + { + env.getAccessManager(); + minTimes = 0; + result = accessManager; + + env.getCurrentCatalog(); + minTimes = 0; + result = catalog; + + env.getInternalCatalog(); + minTimes = 0; + result = catalog; + + env.getCatalogMgr(); + minTimes = 0; + result = dsMgr; + } + }; + FunctionSet fs = new FunctionSet(); + fs.init(); + Deencapsulation.setField(env, "functionSet", fs); + QueryState state = new QueryState(); + channel.reset(); + + SessionVariable sessionVariable = new SessionVariable(); + Deencapsulation.setField(sessionVariable, "beNumberForTest", 1); + + new Expectations(channel) { + { + channel.getSerializer(); + minTimes = 0; + result = MysqlSerializer.newInstance(); + } + }; + + new Expectations(ctx) { + { + ctx.getMysqlChannel(); + minTimes = 0; + result = channel; + + ctx.getClusterName(); + minTimes = 0; + result = clusterName; + + ctx.getEnv(); + minTimes = 0; + result = env; + + ctx.getState(); + minTimes = 0; + result = state; + + ctx.getConnectScheduler(); + minTimes = 0; + result = scheduler; + + ctx.getConnectionId(); + minTimes = 0; + result = 1; + + ctx.getQualifiedUser(); + minTimes = 0; + result = userName; + + ctx.getForwardedStmtId(); + minTimes = 0; + result = 123L; + + ctx.setKilled(); + minTimes = 0; + ctx.updateReturnRows(anyInt); + minTimes = 0; + ctx.setQueryId((TUniqueId) any); + minTimes = 0; + + ctx.queryId(); + minTimes = 0; + result = new TUniqueId(); + + ctx.getStartTime(); + minTimes = 0; + result = 0L; + + ctx.getDatabase(); + minTimes = 0; + result = dbName; + + ctx.getSessionVariable(); + minTimes = 0; + result = sessionVariable; + + ctx.setStmtId(anyLong); + minTimes = 0; + + ctx.getStmtId(); + minTimes = 0; + result = 1L; + + ctx.getTracer(); + minTimes = 0; + result = Telemetry.getNoopTracer(); + + ctx.getCurrentCatalog(); + minTimes = 0; + result = catalog; + + ctx.getCatalog(anyString); + minTimes = 0; + result = catalog; + } + }; + + analyzer = new Analyzer(env, ctx); + newRangeList = Lists.newArrayList(); + } + + private OlapTable createOrderTable() { + Column column1 = new Column("date", ScalarType.INT); + Column column2 = new Column("id", ScalarType.INT); + Column column3 = new Column("value", ScalarType.INT); + List columns = Lists.newArrayList(column1, column2, column3); + + MaterializedIndex baseIndex = new MaterializedIndex(10001, IndexState.NORMAL); + RandomDistributionInfo distInfo = new RandomDistributionInfo(10); + + PartitionInfo partInfo = new RangePartitionInfo(Lists.newArrayList(column1)); + + Partition part12 = new Partition(20200112, "p20200112", baseIndex, distInfo); + part12.setVisibleVersionAndTime(1, 1578762000000L); //2020-01-12 1:00:00 + setPartitionItem(partInfo, part12, column1, "20200112", "20200113"); + Partition part13 = new Partition(20200113, "p20200113", baseIndex, distInfo); + part13.setVisibleVersionAndTime(1, 1578848400000L); //2020-01-13 1:00:00 + setPartitionItem(partInfo, part13, column1, "20200113", "20200114"); + Partition part14 = new Partition(20200114, "p20200114", baseIndex, distInfo); + part14.setVisibleVersionAndTime(1, 1578934800000L); //2020-01-14 1:00:00 + setPartitionItem(partInfo, part14, column1, "20200114", "20200115"); + Partition part15 = new Partition(20200115, "p20200115", baseIndex, distInfo); + part15.setVisibleVersionAndTime(2, 1579053661000L); //2020-01-15 10:01:01 + setPartitionItem(partInfo, part15, column1, "20200115", "20200116"); + + OlapTable table = new OlapTable(10000L, "order", columns, KeysType.DUP_KEYS, partInfo, distInfo); + + short shortKeyColumnCount = 1; + table.setIndexMeta(10001, "group1", columns, 1, 1, shortKeyColumnCount, TStorageType.COLUMN, KeysType.DUP_KEYS); + + List idxColumns = Lists.newArrayList(); + idxColumns.add(column1); + table.setIndexMeta(1L, "test", idxColumns, 1, 1, shortKeyColumnCount, TStorageType.COLUMN, + KeysType.DUP_KEYS); + Deencapsulation.setField(table, "baseIndexId", 1000); + + table.addPartition(part12); + table.addPartition(part13); + table.addPartition(part14); + table.addPartition(part15); + + return table; + } + + private void setPartitionItem(PartitionInfo partInfo, Partition partition, Column column, String rangeKeyLower, + String rangeKeyUpper) { + try { + PartitionKey rangeP1Lower = + PartitionKey.createPartitionKey(Lists.newArrayList(new PartitionValue(rangeKeyLower)), + Lists.newArrayList(column)); + PartitionKey rangeP1Upper = + PartitionKey.createPartitionKey(Lists.newArrayList(new PartitionValue(rangeKeyUpper)), + Lists.newArrayList(column)); + Range rangeP1 = Range.closedOpen(rangeP1Lower, rangeP1Upper); + partInfo.setItem(partition.getId(), false, new RangePartitionItem(rangeP1)); + } catch (AnalysisException e) { + LOG.warn("Part,an_ex={}", e); + Assert.fail(e.getMessage()); + } + } + + private ScanNode createOrderScanNode(Collection selectedPartitionIds) { + OlapTable table = createOrderTable(); + TupleDescriptor desc = new TupleDescriptor(new TupleId(10004)); + desc.setTable(table); + OlapScanNode node = new OlapScanNode(new PlanNodeId(10008), desc, "ordernode"); + node.setSelectedPartitionIds(selectedPartitionIds); + return node; + } + + private OlapTable createProfileTable() { + Column column2 = new Column("eventdate", ScalarType.DATE); + Column column3 = new Column("userid", ScalarType.INT); + Column column4 = new Column("country", ScalarType.INT); + List columns = Lists.newArrayList(column2, column3, column4); + + MaterializedIndex baseIndex = new MaterializedIndex(20001, IndexState.NORMAL); + RandomDistributionInfo distInfo = new RandomDistributionInfo(10); + + PartitionInfo partInfo = new RangePartitionInfo(Lists.newArrayList(column2)); + + Partition part12 = new Partition(20200112, "p20200112", baseIndex, distInfo); + part12.setVisibleVersionAndTime(1, 1578762000000L); //2020-01-12 1:00:00 + Partition part13 = new Partition(20200113, "p20200113", baseIndex, distInfo); + part13.setVisibleVersionAndTime(1, 1578848400000L); //2020-01-13 1:00:00 + Partition part14 = new Partition(20200114, "p20200114", baseIndex, distInfo); + part14.setVisibleVersionAndTime(1, 1578934800000L); //2020-01-14 1:00:00 + Partition part15 = new Partition(20200115, "p20200115", baseIndex, distInfo); + part15.setVisibleVersionAndTime(2, 1579021200000L); //2020-01-15 1:00:00 + + OlapTable table = new OlapTable(20000L, "userprofile", columns, KeysType.AGG_KEYS, partInfo, distInfo); + + short shortKeyColumnCount = 1; + table.setIndexMeta(20001, "group1", columns, 1, 1, shortKeyColumnCount, TStorageType.COLUMN, KeysType.AGG_KEYS); + + List idxColumns = Lists.newArrayList(); + idxColumns.add(column2); + table.setIndexMeta(2L, "test", idxColumns, 1, 1, shortKeyColumnCount, TStorageType.COLUMN, + KeysType.AGG_KEYS); + + Deencapsulation.setField(table, "baseIndexId", 1000); + + table.addPartition(part12); + table.addPartition(part13); + table.addPartition(part14); + table.addPartition(part15); + + return table; + } + + private ScanNode createProfileScanNode(Collection selectedPartitionIds) { + OlapTable table = createProfileTable(); + TupleDescriptor desc = new TupleDescriptor(new TupleId(20004)); + desc.setTable(table); + OlapScanNode node = new OlapScanNode(new PlanNodeId(20008), desc, "userprofilenode"); + node.setSelectedPartitionIds(selectedPartitionIds); + return node; + } + + /** + * table appevent(date(pk), userid, eventid, eventtime, city), stream load every 5 miniutes + */ + private OlapTable createEventTable() { + Column column1 = new Column("eventdate", ScalarType.DATE); + Column column2 = new Column("userid", ScalarType.INT); + Column column3 = new Column("eventid", ScalarType.INT); + Column column4 = new Column("eventtime", ScalarType.DATETIME); + Column column5 = new Column("city", ScalarType.VARCHAR); + List columns = Lists.newArrayList(column1, column2, column3, column4, column5); + PartitionInfo partInfo = new RangePartitionInfo(Lists.newArrayList(column1)); + MaterializedIndex baseIndex = new MaterializedIndex(30001, IndexState.NORMAL); + RandomDistributionInfo distInfo = new RandomDistributionInfo(10); + + Partition part12 = new Partition(20200112, "p20200112", baseIndex, distInfo); + part12.setVisibleVersionAndTime(1, 1578762000000L); //2020-01-12 1:00:00 + setPartitionItem(partInfo, part12, column1, "20200112", "20200113"); + Partition part13 = new Partition(20200113, "p20200113", baseIndex, distInfo); + part13.setVisibleVersionAndTime(1, 1578848400000L); //2020-01-13 1:00:00 + setPartitionItem(partInfo, part13, column1, "20200113", "20200114"); + Partition part14 = new Partition(20200114, "p20200114", baseIndex, distInfo); + part14.setVisibleVersionAndTime(1, 1578934800000L); //2020-01-14 1:00:00 + setPartitionItem(partInfo, part14, column1, "20200114", "20200115"); + Partition part15 = new Partition(20200115, "p20200115", baseIndex, distInfo); + part15.setVisibleVersionAndTime(2, 1579053661000L); //2020-01-15 10:01:01 + setPartitionItem(partInfo, part15, column1, "20200115", "20200116"); + + OlapTable table = new OlapTable(30000L, "appevent", columns, KeysType.DUP_KEYS, partInfo, distInfo); + + short shortKeyColumnCount = 1; + table.setIndexMeta(30001, "group1", columns, 1, 1, shortKeyColumnCount, TStorageType.COLUMN, KeysType.AGG_KEYS); + + List column = Lists.newArrayList(); + column.add(column1); + column.add(column2); + column.add(column3); + column.add(column4); + column.add(column5); + + table.setIndexMeta(new Long(2), "test", column, 1, 1, shortKeyColumnCount, TStorageType.COLUMN, + KeysType.AGG_KEYS); + Deencapsulation.setField(table, "baseIndexId", 2); + + table.addPartition(part12); + table.addPartition(part13); + table.addPartition(part14); + table.addPartition(part15); + + return table; + } + + private View createEventView1() { + String originStmt = "select eventdate, COUNT(userid) FROM appevent WHERE " + + "eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-14\" GROUP BY eventdate"; + View view = new View(30000L, "view1", null); + view.setInlineViewDefWithSqlMode(originStmt, 0L); + return view; + } + + private View createEventView2() { + String originStmt = "select eventdate, userid FROM appevent"; + View view = new View(30001L, "view2", null); + view.setInlineViewDefWithSqlMode(originStmt, 0L); + return view; + } + + private View createEventView3() { + String originStmt = "select eventdate, COUNT(userid) FROM appevent WHERE " + + "eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-15\" GROUP BY eventdate"; + View view = new View(30002L, "view3", null); + view.setInlineViewDefWithSqlMode(originStmt, 0L); + return view; + } + + private View createEventNestedView() { + String originStmt = "select eventdate, COUNT(userid) FROM view2 WHERE " + + "eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-14\" GROUP BY eventdate"; + View view = new View(30003L, "view4", null); + view.setInlineViewDefWithSqlMode(originStmt, 0L); + return view; + } + + private ScanNode createEventScanNode(Collection selectedPartitionIds) { + OlapTable table = createEventTable(); + TupleDescriptor desc = new TupleDescriptor(new TupleId(30002)); + desc.setTable(table); + OlapScanNode node = new OlapScanNode(new PlanNodeId(30004), desc, "appeventnode"); + node.setSelectedPartitionIds(selectedPartitionIds); + return node; + } + + private StatementBase parseSqlByNereids(String sql) { + StatementBase stmt = null; + try { + LogicalPlan plan = new NereidsParser().parseSingle(sql); + OriginStatement originStatement = new OriginStatement(sql, 0); + StatementContext statementContext = new StatementContext(ctx, originStatement); + NereidsPlanner nereidsPlanner = new NereidsPlanner(statementContext); + LogicalPlanAdapter adapter = new LogicalPlanAdapter(plan, statementContext); + nereidsPlanner.plan(adapter); + statementContext.setParsedStatement(adapter); + stmt = adapter; + } catch (Throwable throwable) { + LOG.warn("Part,an_ex={}", throwable); + Assert.fail(throwable.getMessage()); + } + return stmt; + } + + private StatementBase parseSql(String sql) { + SqlParser parser = new SqlParser(new SqlScanner(new StringReader(sql))); + StatementBase parseStmt = null; + try { + parseStmt = SqlParserUtils.getFirstStmt(parser); + parseStmt.analyze(analyzer); + } catch (AnalysisException e) { + LOG.warn("Part,an_ex={}", e); + Assert.fail(e.getMessage()); + } catch (UserException e) { + LOG.warn("Part,ue_ex={}", e); + Assert.fail(e.getMessage()); + } catch (Exception e) { + LOG.warn("Part,cm_ex={}", e); + Assert.fail(e.getMessage()); + } + return parseStmt; + } + + @Test + public void testCacheNode() throws Exception { + Env.getCurrentSystemInfo(); + CacheCoordinator cp = CacheCoordinator.getInstance(); + cp.debugModel = true; + Backend bd1 = new Backend(1, "", 1000); + bd1.updateOnce(0, 0, 0); + Backend bd2 = new Backend(2, "", 2000); + bd2.updateOnce(0, 0, 0); + Backend bd3 = new Backend(3, "", 3000); + bd3.updateOnce(0, 0, 0); + cp.addBackend(bd1); + cp.addBackend(bd2); + cp.addBackend(bd3); + + Types.PUniqueId key1 = Types.PUniqueId.newBuilder().setHi(1L).setLo(1L).build(); + Backend bk = cp.findBackend(key1); + Assert.assertNotNull(bk); + Assert.assertEquals(bk.getId(), 3); + + key1 = key1.toBuilder().setHi(669560558156283345L).build(); + bk = cp.findBackend(key1); + Assert.assertNotNull(bk); + Assert.assertEquals(bk.getId(), 1); + } + + @Test + public void testCacheModeNone() throws Exception { + Env.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql("select @@version_comment limit 1"); + List scanNodes = Lists.newArrayList(); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(0); + Assert.assertEquals(ca.getCacheMode(), CacheMode.NoNeed); + } + + @Test + public void testCacheModeTable() throws Exception { + Env.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql( + "SELECT country, COUNT(userid) FROM userprofile GROUP BY country" + ); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); + List scanNodes = Lists.newArrayList(createProfileScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(0); + Assert.assertEquals(ca.getCacheMode(), CacheMode.Sql); + } + + @Test + public void testWithinMinTime() throws Exception { + Env.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql( + "SELECT country, COUNT(userid) FROM userprofile GROUP BY country" + ); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); + List scanNodes = Lists.newArrayList(createProfileScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(1579024800000L); //2020-1-15 02:00:00 + Assert.assertEquals(ca.getCacheMode(), CacheMode.None); + } + + @Test + public void testPartitionModel() throws Exception { + Env.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql( + "SELECT eventdate, COUNT(DISTINCT userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and " + + "eventdate<=\"2020-01-15\" GROUP BY eventdate" + ); + + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); + List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); + } + + @Test + public void testParseByte() throws Exception { + Env.getCurrentSystemInfo(); + RowBatchBuilder sb = new RowBatchBuilder(CacheMode.Partition); + byte[] buffer = new byte[]{10, 50, 48, 50, 48, 45, 48, 51, 45, 49, 48, 1, 51, 2, 67, 78}; + PartitionRange.PartitionKeyType key1 = sb.getKeyFromRow(buffer, 0, Type.DATE); + LOG.info("real value key1 {}", key1.realValue()); + Assert.assertEquals(key1.realValue(), 20200310); + PartitionRange.PartitionKeyType key2 = sb.getKeyFromRow(buffer, 1, Type.INT); + LOG.info("real value key2 {}", key2.realValue()); + Assert.assertEquals(key2.realValue(), 3); + } + + @Test + public void testPartitionIntTypeSql() throws Exception { + Env.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql( + "SELECT `date`, COUNT(id) FROM `order` WHERE `date`>=20200112 and `date`<=20200115 GROUP BY date" + ); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); + List scanNodes = Lists.newArrayList(createOrderScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first + try { + PartitionCache cache = (PartitionCache) ca.getCache(); + cache.rewriteSelectStmt(null); + Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); + + PartitionRange range = cache.getPartitionRange(); + boolean flag = range.analytics(); + Assert.assertEquals(flag, true); + + int size = range.getPartitionSingleList().size(); + LOG.warn("Rewrite partition range size={}", size); + Assert.assertEquals(size, 4); + + String sql; + range.setCacheFlag(20200112L); //get data from cache + range.setCacheFlag(20200113L); //get data from cache + + hitRange = range.buildDiskPartitionRange(newRangeList); + Assert.assertEquals(hitRange, Cache.HitRange.Left); + Assert.assertEquals(newRangeList.size(), 2); + Assert.assertEquals(newRangeList.get(0).getCacheKey().realValue(), 20200114); + Assert.assertEquals(newRangeList.get(1).getCacheKey().realValue(), 20200115); + + cache.rewriteSelectStmt(newRangeList); + sql = ca.getRewriteStmt().getWhereClause().toSql(); + Assert.assertEquals(sql, "`date` >= 20200114 AND `date` <= 20200115"); + } catch (Exception e) { + LOG.warn("ex={}", e); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testSimpleCacheSql() throws Exception { + Env.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql( + "SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and " + + "eventdate<=\"2020-01-15\" GROUP BY eventdate" + ); + + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); + + List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first + + try { + PartitionCache cache = (PartitionCache) ca.getCache(); + cache.rewriteSelectStmt(null); + Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); + + PartitionRange range = cache.getPartitionRange(); + boolean flag = range.analytics(); + Assert.assertEquals(flag, true); + + int size = range.getPartitionSingleList().size(); + LOG.warn("Rewrite partition range size={}", size); + Assert.assertEquals(size, 4); + + String sql; + range.setCacheFlag(20200112L); //get data from cache + range.setCacheFlag(20200113L); //get data from cache + + hitRange = range.buildDiskPartitionRange(newRangeList); + cache.rewriteSelectStmt(newRangeList); + sql = ca.getRewriteStmt().getWhereClause().toSql(); + Assert.assertEquals(sql, "`eventdate` >= '2020-01-14' AND `eventdate` <= '2020-01-15'"); + } catch (Exception e) { + LOG.warn("ex={}", e); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testHitSqlCache() throws Exception { + Env.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql( + "SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and " + + "eventdate<=\"2020-01-14\" GROUP BY eventdate" + ); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L); + List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + Assert.assertEquals(ca.getCacheMode(), CacheMode.Sql); + } + + @Test + public void testHitPartPartition() throws Exception { + Env.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql( + "SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and " + + "eventdate<=\"2020-01-14\" GROUP BY eventdate" + ); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L); + List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(1578675600000L); // set now to 2020-01-11 1:00:00, for hit partition cache + Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first + + try { + PartitionCache cache = (PartitionCache) ca.getCache(); + + cache.rewriteSelectStmt(null); + Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); + + PartitionRange range = cache.getPartitionRange(); + boolean flag = range.analytics(); + Assert.assertEquals(flag, true); + + int size = range.getPartitionSingleList().size(); + LOG.warn("Rewrite partition range size={}", size); + Assert.assertEquals(size, 3); + + range.setCacheFlag(20200113); + range.setCacheFlag(20200114); + + hitRange = range.buildDiskPartitionRange(newRangeList); + Assert.assertEquals(hitRange, Cache.HitRange.Right); + Assert.assertEquals(newRangeList.size(), 2); + Assert.assertEquals(newRangeList.get(0).getCacheKey().realValue(), 20200112); + Assert.assertEquals(newRangeList.get(1).getCacheKey().realValue(), 20200112); + + List updateRangeList = range.buildUpdatePartitionRange(); + Assert.assertEquals(updateRangeList.size(), 1); + Assert.assertEquals(updateRangeList.get(0).getCacheKey().realValue(), 20200112); + } catch (Exception e) { + LOG.warn("ex={}", e); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testNoUpdatePartition() throws Exception { + Env.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql( + "SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and " + + "eventdate<=\"2020-01-14\" GROUP BY eventdate" + ); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L); + List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(1578675600000L); // set now to 2020-01-11 1:00:00, for hit partition cache + Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first + + try { + PartitionCache cache = (PartitionCache) ca.getCache(); + + cache.rewriteSelectStmt(null); + Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); + + PartitionRange range = cache.getPartitionRange(); + boolean flag = range.analytics(); + Assert.assertEquals(flag, true); + + int size = range.getPartitionSingleList().size(); + LOG.warn("Rewrite partition range size={}", size); + Assert.assertEquals(size, 3); + + range.setCacheFlag(20200112); //get data from cache + range.setCacheFlag(20200113); + range.setCacheFlag(20200114); + + hitRange = range.buildDiskPartitionRange(newRangeList); + Assert.assertEquals(hitRange, Cache.HitRange.Full); + Assert.assertEquals(newRangeList.size(), 0); + } catch (Exception e) { + LOG.warn("ex={}", e); + Assert.fail(e.getMessage()); + } + } + + + @Test + public void testUpdatePartition() throws Exception { + Env.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql( + "SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and " + + "eventdate<=\"2020-01-15\" GROUP BY eventdate" + ); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); + List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first + + try { + PartitionCache cache = (PartitionCache) ca.getCache(); + + cache.rewriteSelectStmt(null); + Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); + + PartitionRange range = cache.getPartitionRange(); + boolean flag = range.analytics(); + Assert.assertEquals(flag, true); + + int size = range.getPartitionSingleList().size(); + LOG.warn("Rewrite partition range size={}", size); + Assert.assertEquals(size, 4); + + String sql; + range.setCacheFlag(20200112L); //get data from cache + range.setTooNewByKey(20200115); + + range.buildDiskPartitionRange(newRangeList); + Assert.assertEquals(newRangeList.size(), 2); + cache.rewriteSelectStmt(newRangeList); + + sql = ca.getRewriteStmt().getWhereClause().toSql(); + Assert.assertEquals(sql, "`eventdate` >= '2020-01-13' AND `eventdate` <= '2020-01-15'"); + + List updateRangeList = range.buildUpdatePartitionRange(); + Assert.assertEquals(updateRangeList.size(), 2); + Assert.assertEquals(updateRangeList.get(0).getCacheKey().realValue(), 20200113); + Assert.assertEquals(updateRangeList.get(1).getCacheKey().realValue(), 20200114); + } catch (Exception e) { + LOG.warn("ex={}", e); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testRewriteMultiPredicate1() throws Exception { + Env.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql( + "SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>\"2020-01-11\" and " + + "eventdate<\"2020-01-16\"" + + " and eventid=1 GROUP BY eventdate" + ); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); + List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first + try { + PartitionCache cache = (PartitionCache) ca.getCache(); + + cache.rewriteSelectStmt(null); + LOG.warn("Nokey multi={}", cache.getNokeyStmt().getWhereClause().toSql()); + Assert.assertEquals(cache.getNokeyStmt().getWhereClause().toSql(), "`eventid` = 1"); + + PartitionRange range = cache.getPartitionRange(); + boolean flag = range.analytics(); + Assert.assertEquals(flag, true); + + int size = range.getPartitionSingleList().size(); + Assert.assertEquals(size, 4); + + String sql; + range.setCacheFlag(20200112L); //get data from cache + range.setCacheFlag(20200113L); //get data from cache + + range.buildDiskPartitionRange(newRangeList); + + cache.rewriteSelectStmt(newRangeList); + sql = ca.getRewriteStmt().getWhereClause().toSql(); + LOG.warn("MultiPredicate={}", sql); + Assert.assertEquals(sql, "`eventdate` > '2020-01-13' AND `eventdate` < '2020-01-16' AND `eventid` = 1"); + } catch (Exception e) { + LOG.warn("multi ex={}", e); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testRewriteJoin() throws Exception { + Env.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql( + "SELECT appevent.eventdate, country, COUNT(appevent.userid) FROM appevent" + + " INNER JOIN userprofile ON appevent.userid = userprofile.userid" + + " WHERE appevent.eventdate>=\"2020-01-12\" and appevent.eventdate<=\"2020-01-15\"" + + " and eventid=1 GROUP BY appevent.eventdate, country" + ); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); + List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first + try { + PartitionCache cache = (PartitionCache) ca.getCache(); + cache.rewriteSelectStmt(null); + LOG.warn("Join nokey={}", cache.getNokeyStmt().getWhereClause().toSql()); + Assert.assertEquals(cache.getNokeyStmt().getWhereClause().toSql(), "`eventid` = 1"); + + PartitionRange range = cache.getPartitionRange(); + boolean flag = range.analytics(); + Assert.assertEquals(flag, true); + + int size = range.getPartitionSingleList().size(); + Assert.assertEquals(size, 4); + + String sql; + range.setCacheFlag(20200112L); //get data from cache + range.setCacheFlag(20200113L); //get data from cache + + range.buildDiskPartitionRange(newRangeList); + + cache.rewriteSelectStmt(newRangeList); + sql = ca.getRewriteStmt().getWhereClause().toSql(); + LOG.warn("Join rewrite={}", sql); + Assert.assertEquals(sql, "`appevent`.`eventdate` >= '2020-01-14'" + + " AND `appevent`.`eventdate` <= '2020-01-15' AND `eventid` = 1"); + } catch (Exception e) { + LOG.warn("Join ex={}", e); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testSubSelect() throws Exception { + Env.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql( + "SELECT eventdate, sum(pv) FROM (SELECT eventdate, COUNT(userid) AS pv FROM appevent WHERE " + + "eventdate>\"2020-01-11\" AND eventdate<\"2020-01-16\"" + + " AND eventid=1 GROUP BY eventdate) tbl GROUP BY eventdate" + ); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); + List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first + try { + PartitionCache cache = (PartitionCache) ca.getCache(); + + cache.rewriteSelectStmt(null); + LOG.warn("Sub nokey={}", cache.getNokeyStmt().toSql()); + Assert.assertEquals(cache.getNokeyStmt().toSql(), + "SELECT `eventdate` AS `eventdate`, sum(`pv`) AS `sum(``pv``)` FROM (" + + "SELECT `eventdate` AS `eventdate`, count(`userid`) AS `pv` FROM " + + "`testCluster:testDb`.`appevent` WHERE `eventid` = 1" + + " GROUP BY `eventdate`) tbl GROUP BY `eventdate`"); + + PartitionRange range = cache.getPartitionRange(); + boolean flag = range.analytics(); + Assert.assertEquals(flag, true); + + int size = range.getPartitionSingleList().size(); + Assert.assertEquals(size, 4); + + String sql; + range.setCacheFlag(20200112L); //get data from cache + range.setCacheFlag(20200113L); //get data from cache + + range.buildDiskPartitionRange(newRangeList); + + cache.rewriteSelectStmt(newRangeList); + sql = ca.getRewriteStmt().toSql(); + LOG.warn("Sub rewrite={}", sql); + Assert.assertEquals(sql, + "SELECT `eventdate` AS `eventdate`, sum(`pv`) AS `sum(``pv``)` FROM (" + + "SELECT `eventdate` AS `eventdate`, count(`userid`) AS `pv` FROM " + + "`testCluster:testDb`.`appevent` WHERE " + + "`eventdate` > '2020-01-13' AND `eventdate` < '2020-01-16' AND `eventid` = 1 GROUP BY " + + "`eventdate`) tbl GROUP BY `eventdate`"); + } catch (Exception e) { + LOG.warn("sub ex={}", e); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testNotHitPartition() throws Exception { + Env.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql( + "SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and " + + "eventdate<=\"2020-01-14\" GROUP BY eventdate" + ); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L); + List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(1578675600000L); // set now to 2020-01-11 1:00:00, for hit partition cache + Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first + try { + PartitionCache cache = (PartitionCache) ca.getCache(); + cache.rewriteSelectStmt(null); + PartitionRange range = cache.getPartitionRange(); + range.analytics(); + hitRange = range.buildDiskPartitionRange(newRangeList); + Assert.assertEquals(hitRange, Cache.HitRange.None); + Assert.assertEquals(newRangeList.size(), 2); + Assert.assertEquals(newRangeList.get(0).getCacheKey().realValue(), 20200112); + Assert.assertEquals(newRangeList.get(1).getCacheKey().realValue(), 20200114); + } catch (Exception e) { + LOG.warn("ex={}", e); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testSqlCacheKey() { + Env.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql( + "SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and " + + "eventdate<=\"2020-01-14\" GROUP BY eventdate" + ); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L); + List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + Assert.assertEquals(ca.getCacheMode(), CacheMode.Sql); + + SqlCache sqlCache = (SqlCache) ca.getCache(); + String cacheKey = sqlCache.getSqlWithViewStmt(); + Assert.assertEquals(cacheKey, "SELECT `eventdate` AS `eventdate`, count(`userid`) AS " + + "`count(``userid``)` FROM `testCluster:testDb`.`appevent` WHERE `eventdate` >= '2020-01-12' AND " + + "`eventdate` <= '2020-01-14' GROUP BY `eventdate`|"); + } + + @Test + public void testSqlCacheKeyWithChineseChar() { + Env.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql( + "SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and " + + "eventdate<=\"2020-01-14\" and city=\"北京\" GROUP BY eventdate" + ); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L); + List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + Assert.assertEquals(ca.getCacheMode(), CacheMode.Sql); + SqlCache sqlCache = (SqlCache) ca.getCache(); + String cacheKey = sqlCache.getSqlWithViewStmt(); + Types.PUniqueId sqlKey2 = CacheProxy.getMd5(cacheKey.replace("北京", "上海")); + Assert.assertNotEquals(CacheProxy.getMd5(sqlCache.getSqlWithViewStmt()), sqlKey2); + } + + @Test + public void testSqlCacheKeyWithView() { + Env.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql("SELECT * from testDb.view1"); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L); + List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + Assert.assertEquals(ca.getCacheMode(), CacheMode.Sql); + + SqlCache sqlCache = (SqlCache) ca.getCache(); + String cacheKey = sqlCache.getSqlWithViewStmt(); + Assert.assertEquals(cacheKey, "SELECT `testCluster:testDb`.`view1`.`eventdate` AS `eventdate`, `testCluster:testDb`.`view1`." + + "`count(`userid`)` AS `count(``userid``)` FROM `testCluster:testDb`.`view1`|select eventdate, COUNT(userid) " + + "FROM appevent WHERE eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-14\" GROUP BY eventdate"); + } + + @Test + public void testSqlCacheKeyWithViewForNereids() { + Env.getCurrentSystemInfo(); + StatementBase parseStmt = parseSqlByNereids("SELECT * from testDb.view1"); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L); + List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheModeForNereids(1579053661000L); //2020-1-15 10:01:01 + Assert.assertEquals(ca.getCacheMode(), CacheMode.Sql); + + SqlCache sqlCache = (SqlCache) ca.getCache(); + String cacheKey = sqlCache.getSqlWithViewStmt(); + Assert.assertEquals(cacheKey, "SELECT * from testDb.view1" + + "|select eventdate, COUNT(userid) FROM appevent " + + "WHERE eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-14\" GROUP BY eventdate"); + } + + @Test + public void testSqlCacheKeyWithSubSelectView() { + Env.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql( + "select origin.eventdate as eventdate, origin.userid as userid\n" + + "from (\n" + + " select view2.eventdate as eventdate, view2.userid as userid \n" + + " from testDb.view2 view2 \n" + + " where view2.eventdate >=\"2020-01-12\" and view2.eventdate <= \"2020-01-14\"\n" + + ") origin" + ); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L); + List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + Assert.assertEquals(ca.getCacheMode(), CacheMode.Sql); + + SqlCache sqlCache = (SqlCache) ca.getCache(); + String cacheKey = sqlCache.getSqlWithViewStmt(); + Assert.assertEquals(cacheKey, "SELECT `origin`.`eventdate` AS `eventdate`, `origin`.`userid` AS " + + "`userid` FROM (SELECT `view2`.`eventdate` AS `eventdate`, `view2`.`userid` AS `userid` FROM " + + "`testCluster:testDb`.`view2` view2 WHERE `view2`.`eventdate` >= '2020-01-12' AND `view2`.`eventdate` " + + "<= '2020-01-14') origin|select eventdate, userid FROM appevent"); + } + + + @Test + public void testSqlCacheKeyWithSubSelectViewForNereids() { + Env.getCurrentSystemInfo(); + StatementBase parseStmt = parseSqlByNereids( + "select origin.eventdate as eventdate, origin.userid as userid\n" + + "from (\n" + + " select view2.eventdate as eventdate, view2.userid as userid \n" + + " from testDb.view2 view2 \n" + + " where view2.eventdate >=\"2020-01-12\" and view2.eventdate <= \"2020-01-14\"\n" + + ") origin" + ); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L); + List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheModeForNereids(1579053661000L); //2020-1-15 10:01:01 + Assert.assertEquals(ca.getCacheMode(), CacheMode.Sql); + + SqlCache sqlCache = (SqlCache) ca.getCache(); + String cacheKey = sqlCache.getSqlWithViewStmt(); + Assert.assertEquals(cacheKey, "select origin.eventdate as eventdate, origin.userid as userid\n" + + "from (\n" + + " select view2.eventdate as eventdate, view2.userid as userid \n" + + " from testDb.view2 view2 \n" + + " where view2.eventdate >=\"2020-01-12\" and view2.eventdate <= \"2020-01-14\"\n" + + ") origin" + "|select eventdate, userid FROM appevent"); + } + + @Test + public void testPartitionCacheKeyWithView() { + Env.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql("SELECT * from testDb.view3"); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); + List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first + + try { + PartitionCache cache = (PartitionCache) ca.getCache(); + + cache.rewriteSelectStmt(null); + Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); + Assert.assertEquals(cache.getSqlWithViewStmt(), "SELECT `testCluster:testDb`.`view3`.`eventdate` AS " + + "`eventdate`, `testCluster:testDb`.`view3`.`count(`userid`)` AS `count(``userid``)` FROM " + + "`testCluster:testDb`.`view3`|select eventdate, COUNT(userid) FROM appevent WHERE eventdate>=" + + "\"2020-01-12\" and eventdate<=\"2020-01-15\" GROUP BY eventdate"); + } catch (Exception e) { + LOG.warn("ex={}", e); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testPartitionCacheKeyWithSubSelectView() { + Env.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql( + "select origin.eventdate as eventdate, origin.cnt as cnt\n" + + "from (\n" + + " SELECT eventdate, COUNT(userid) as cnt \n" + + " FROM view2 \n" + + " WHERE eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-15\" GROUP BY eventdate\n" + + ") origin" + ); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); + List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first + + try { + PartitionCache cache = (PartitionCache) ca.getCache(); + + cache.rewriteSelectStmt(null); + Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); + Assert.assertEquals(cache.getSqlWithViewStmt(), + "SELECT `origin`.`eventdate` AS `eventdate`, `origin`.`cnt` AS `cnt` FROM (SELECT " + + " `eventdate` AS `eventdate`, count(`userid`) AS `cnt` FROM " + + "`testDb`.`view2` GROUP BY `eventdate`) origin|select eventdate, userid FROM appevent"); + } catch (Exception e) { + LOG.warn("ex={}", e); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testSqlCacheKeyWithNestedView() { + Env.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql("SELECT * from testDb.view4"); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L); + List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + Assert.assertEquals(ca.getCacheMode(), CacheMode.Sql); + + SqlCache sqlCache = (SqlCache) ca.getCache(); + String cacheKey = sqlCache.getSqlWithViewStmt(); + Assert.assertEquals(cacheKey, "SELECT `testCluster:testDb`.`view4`.`eventdate` AS `eventdate`, " + + "`testCluster:testDb`.`view4`.`count(`userid`)` AS `count(``userid``)` FROM `testCluster:testDb`.`view4`|select " + + "eventdate, COUNT(userid) FROM view2 WHERE eventdate>=\"2020-01-12\" and " + + "eventdate<=\"2020-01-14\" GROUP BY eventdate|select eventdate, userid FROM appevent"); + } + + @Test + public void testSqlCacheKeyWithNestedViewForNereids() { + Env.getCurrentSystemInfo(); + StatementBase parseStmt = parseSqlByNereids("SELECT * from testDb.view4"); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L); + List scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheModeForNereids(1579053661000L); //2020-1-15 10:01:01 + Assert.assertEquals(ca.getCacheMode(), CacheMode.Sql); + + SqlCache sqlCache = (SqlCache) ca.getCache(); + String cacheKey = sqlCache.getSqlWithViewStmt(); + Assert.assertEquals(cacheKey, "SELECT * from testDb.view4" + + "|select eventdate, COUNT(userid) FROM view2 " + + "WHERE eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-14\" GROUP BY eventdate" + + "|select eventdate, userid FROM appevent"); + } + + @Test + public void testCacheLocalViewMultiOperand() { + Env.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql( + "SELECT COUNT(userid)\n" + + "FROM (\n" + + " (SELECT userid FROM userprofile\n" + + " INTERSECT\n" + + " SELECT userid FROM userprofile)\n" + + " UNION\n" + + " SELECT userid FROM userprofile\n" + + ") as tmp" + ); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); + List scanNodes = Lists.newArrayList(createProfileScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(0); + Assert.assertEquals(ca.getCacheMode(), CacheMode.Sql); + } + + @Test + // test that some partitions do not exist in the table + public void testNotExistPartitionSql() { + Env.getCurrentSystemInfo(); + StatementBase parseStmt = parseSql( + "SELECT `date`, COUNT(id) FROM `order` WHERE `date`>=20200110 and `date`<=20200115 GROUP BY date" + ); + ArrayList selectedPartitionIds + = Lists.newArrayList(20200112L, 20200113L, 20200114L, 20200115L); + List scanNodes = Lists.newArrayList(createOrderScanNode(selectedPartitionIds)); + CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes); + ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01 + Assert.assertEquals(ca.getCacheMode(), CacheMode.Partition); //assert cache model first + try { + PartitionCache cache = (PartitionCache) ca.getCache(); + cache.rewriteSelectStmt(null); + Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); + + PartitionRange range = cache.getPartitionRange(); + boolean flag = range.analytics(); + Assert.assertEquals(flag, true); + + int size = range.getPartitionSingleList().size(); + LOG.warn("Rewrite partition range size={}", size); + Assert.assertEquals(size, 4); + + String sql; + range.setCacheFlag(20200112L); //get data from cache + range.setCacheFlag(20200113L); //get data from cache + + hitRange = range.buildDiskPartitionRange(newRangeList); + Assert.assertEquals(hitRange, Cache.HitRange.Left); + Assert.assertEquals(newRangeList.size(), 2); + Assert.assertEquals(newRangeList.get(0).getCacheKey().realValue(), 20200114); + Assert.assertEquals(newRangeList.get(1).getCacheKey().realValue(), 20200115); + + cache.rewriteSelectStmt(newRangeList); + sql = ca.getRewriteStmt().getWhereClause().toSql(); + Assert.assertEquals(sql, "`date` >= 20200114 AND `date` <= 20200115"); + } catch (Exception e) { + LOG.warn("ex={}", e); + Assert.fail(e.getMessage()); + } + } +}