diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/scalar/OperatorFunctionChecker.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/scalar/OperatorFunctionChecker.java index 284e0ad2481a0..91eff1124ef71 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/scalar/OperatorFunctionChecker.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/scalar/OperatorFunctionChecker.java @@ -42,6 +42,12 @@ public Pair visit(ScalarOperator scalarOperator, Void context) } public Pair visitCall(CallOperator call, Void context) { + for (ScalarOperator child : call.getChildren()) { + Pair result = child.accept(this, null); + if (!result.first) { + return result; + } + } if (predicate.test(call)) { return Pair.create(true, ""); } else { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorEvaluator.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorEvaluator.java index 2839001cca4c1..077f21376f8b7 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorEvaluator.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorEvaluator.java @@ -33,6 +33,7 @@ import com.starrocks.sql.optimizer.function.MetaFunctions; import com.starrocks.sql.optimizer.operator.OperatorType; import com.starrocks.sql.optimizer.operator.scalar.CallOperator; +import com.starrocks.sql.optimizer.operator.scalar.CastOperator; import com.starrocks.sql.optimizer.operator.scalar.ConstantOperator; import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator; import org.apache.commons.collections4.ListUtils; @@ -212,6 +213,9 @@ public ScalarOperator evaluation(CallOperator root, boolean needMonotonic) { } public boolean isMonotonicFunction(CallOperator call) { + if (call instanceof CastOperator) { + return true; + } FunctionSignature signature; if (call.getFunction() != null) { Function fn = call.getFunction(); @@ -247,11 +251,11 @@ private boolean isMonotonicFunc(FunctionInvoker invoker, CallOperator operator) return false; } - if (FunctionSet.DATE_FORMAT.equalsIgnoreCase(invoker.getSignature().getName()) + if ((FunctionSet.DATE_FORMAT.equalsIgnoreCase(invoker.getSignature().getName()) || FunctionSet.STR_TO_DATE.equalsIgnoreCase(invoker.getSignature().getName()) || FunctionSet.STR2DATE.equalsIgnoreCase(invoker.getSignature().getName()) - || (FunctionSet.FROM_UNIXTIME.equalsIgnoreCase(invoker.getSignature().getName()) - && operator.getChildren().size() == 2)) { + || FunctionSet.FROM_UNIXTIME.equalsIgnoreCase(invoker.getSignature().getName())) + && operator.getChildren().size() == 2) { String pattern = operator.getChild(1).toString(); if (pattern.isEmpty()) { return true; diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ListPartitionPruner.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ListPartitionPruner.java index 206a5ac6f0252..76d661df1ecfa 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ListPartitionPruner.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ListPartitionPruner.java @@ -43,11 +43,11 @@ import com.starrocks.sql.optimizer.operator.scalar.ConstantOperator; import com.starrocks.sql.optimizer.operator.scalar.InPredicateOperator; import com.starrocks.sql.optimizer.operator.scalar.IsNullPredicateOperator; +import com.starrocks.sql.optimizer.operator.scalar.OperatorFunctionChecker; import com.starrocks.sql.optimizer.operator.scalar.PredicateOperator; import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator; import com.starrocks.sql.optimizer.operator.scalar.ScalarOperatorVisitor; import com.starrocks.sql.optimizer.rewrite.ReplaceColumnRefRewriter; -import com.starrocks.sql.optimizer.rewrite.ScalarOperatorEvaluator; import com.starrocks.sql.optimizer.rewrite.ScalarOperatorRewriter; import com.starrocks.sql.optimizer.transformer.SqlToScalarOperatorTranslator; import com.starrocks.sql.plan.ScalarOperatorToExpr; @@ -278,7 +278,7 @@ public static List deduceGenerateColumns(LogicalScanOperator scanOperato SqlToScalarOperatorTranslator.translateWithSlotRef(generatedExpr, slotRefResolver); if (call instanceof CallOperator && - ScalarOperatorEvaluator.INSTANCE.isMonotonicFunction((CallOperator) call)) { + OperatorFunctionChecker.onlyContainMonotonicFunctions((CallOperator) call).first) { List columnRefOperatorList = Utils.extractColumnRef(call); for (ColumnRefOperator ref : columnRefOperatorList) { result.add(ref.getName()); @@ -325,7 +325,7 @@ private void deduceExtraConjuncts() { SqlToScalarOperatorTranslator.translateWithSlotRef(generatedExpr, slotRefResolver); if (call instanceof CallOperator && - ScalarOperatorEvaluator.INSTANCE.isMonotonicFunction((CallOperator) call)) { + OperatorFunctionChecker.onlyContainMonotonicFunctions((CallOperator) call).first) { List columnRefOperatorList = Utils.extractColumnRef(call); for (ColumnRefOperator ref : columnRefOperatorList) { diff --git a/test/sql/test_automatic_partition/R/test_partition_prune b/test/sql/test_automatic_partition/R/test_partition_prune index 61e5c9052d3c2..468725c142175 100644 --- a/test/sql/test_automatic_partition/R/test_partition_prune +++ b/test/sql/test_automatic_partition/R/test_partition_prune @@ -12,3 +12,195 @@ explain select * from orders_new where ts=20200101; -- result: [REGEX].*partitions=1/2.* -- !result + +-- name: test_range_prune +CREATE TABLE ts ( + ts INT NOT NULL, + id BIGINT NOT NULL, + city STRING NOT NULL ) +PARTITION BY date_trunc('month', str_to_date(CAST(ts as STRING),'%Y%m%d')); +-- result: +-- !result +insert into ts values('20200201',1,'cd'); +-- result: +-- !result +insert into ts values('20200101',1,'cd'); +-- result: +-- !result +insert into ts values('20200301',1,'cd'); +-- result: +-- !result +insert into ts values('20200401',1,'cd'); +-- result: +-- !result +explain select * from ts where ts>20200201; +-- result: +[REGEX].*partitions=3/4.* +-- !result +CREATE TABLE o ( + ts BIGINT NOT NULL, + id BIGINT NOT NULL, + city STRING NOT NULL +) +PARTITION BY from_unixtime(ts,'%Y%m%d'); +-- result: +-- !result +insert into o values(1727224687,1,'cd'); +-- result: +-- !result +insert into o values(1737234687,1,'cd'); +-- result: +-- !result +insert into o values(1747244687,1,'cd'); +-- result: +-- !result +insert into o values(1757254687,1,'cd'); +-- result: +-- !result +explain select * from o where ts>1737234687; +-- result: +[REGEX].*partitions=3/4.* +-- !result +CREATE TABLE t ( + ts INT NOT NULL, + id BIGINT NOT NULL, + city STRING NOT NULL ) +PARTITION BY id, date_trunc('month', str_to_date(CAST(ts as STRING),'%Y%m%d')); +-- result: +-- !result +insert into t values('20200201',1,'cd'); +-- result: +-- !result +insert into t values('20200101',1,'cd'); +-- result: +-- !result +insert into t values('20200301',1,'cd'); +-- result: +-- !result +insert into t values('20200401',1,'cd'); +-- result: +-- !result +insert into t values('20200201',2,'cd'); +-- result: +-- !result +insert into t values('20200101',2,'cd'); +-- result: +-- !result +insert into t values('20200301',3,'cd'); +-- result: +-- !result +insert into t values('20200401',3,'cd'); +-- result: +-- !result +explain select * from t where ts>20200201; +-- result: +[REGEX].*partitions=6/8.* +-- !result +explain select * from t where id>1; +-- result: +[REGEX].*partitions=4/8.* +-- !result +explain select * from t where id>1 and ts>20200201; +-- result: +[REGEX].*partitions=3/8.* +-- !result + +-- name: test_prune_value_function +CREATE TABLE o ( + ts BIGINT NOT NULL, + id BIGINT NOT NULL, + city STRING NOT NULL +) +PARTITION BY from_unixtime(ts,'%Y%m%d'); +-- result: +-- !result +insert into o values(unix_timestamp(),1,'cd'); +-- result: +-- !result +insert into o values(unix_timestamp()+1000000,1,'cd'); +-- result: +-- !result +insert into o values(unix_timestamp()-1000000,1,'cd'); +-- result: +-- !result +explain select * from o where ts>unix_timestamp(); +-- result: +[REGEX].*partitions=2/3.* +-- !result + +-- name: test_prune_cast +CREATE TABLE o ( + ts STRING NOT NULL, + id BIGINT NOT NULL, + city STRING NOT NULL +) +PARTITION BY from_unixtime(cast(ts as INT) + 3600); +-- result: +-- !result +insert into o values(unix_timestamp(),1,'cd'); +-- result: +-- !result +insert into o values(unix_timestamp()+1000000,1,'cd'); +-- result: +-- !result +insert into o values(unix_timestamp()-1000000,1,'cd'); +-- result: +-- !result +explain select * from o where ts>cast(unix_timestamp() as STRING); +-- result: +[REGEX].*partitions=1/3.* +-- !result +explain select * from o where ts>'1740108713'; +-- result: +[REGEX].*partitions=2/3.* +-- !result + +-- name: test_prune_convert_tz +CREATE TABLE ts ( + ts DATETIME NOT NULL, + id BIGINT NOT NULL, + city STRING NOT NULL ) +PARTITION BY date_trunc('hour', convert_tz(ts, 'Asia/Shanghai', 'America/Los_Angeles')); +-- result: +-- !result +insert into ts values('2020-01-01 00:00:00',1,'cd'); +-- result: +-- !result +insert into ts values('2020-02-01 00:00:00',1,'cd'); +-- result: +-- !result +insert into ts values('2020-03-01 00:00:00',1,'cd'); +-- result: +-- !result +insert into ts values('2020-04-01 00:00:00',1,'cd'); +-- result: +-- !result +explain select * from ts where ts > '2020-03-01 00:00:00'; +-- result: +[REGEX].*partitions=4/4.* +-- !result + +-- name: test_prune_if +CREATE TABLE ts ( + dt STRING NOT NULL, + id BIGINT NOT NULL, + city STRING NOT NULL ) +PARTITION BY date_trunc('month', if(length(dt) = 6, str2date(dt, '%y%m%d'), str2date(dt, '%Y%m%d'))); +-- result: +-- !result +insert into ts values('20200201',1,'cd'); +-- result: +-- !result +insert into ts values('20200101',1,'cd'); +-- result: +-- !result +insert into ts values('200301',1,'cd'); +-- result: +-- !result +insert into ts values('20200401',1,'cd'); +-- result: +-- !result +explain select * from ts where dt>20200201; +-- result: +[REGEX].*partitions=4/4.* +-- !result diff --git a/test/sql/test_automatic_partition/T/test_partition_prune b/test/sql/test_automatic_partition/T/test_partition_prune index 0d22e8550cc6e..2007ea71a0047 100644 --- a/test/sql/test_automatic_partition/T/test_partition_prune +++ b/test/sql/test_automatic_partition/T/test_partition_prune @@ -3,3 +3,93 @@ CREATE TABLE orders_new ( ts INT NOT NULL, id BIGINT NOT NULL, city insert into orders_new values('20200201',1,'cd'); insert into orders_new values('20200101',1,'cd'); explain select * from orders_new where ts=20200101; + +-- name: test_range_prune +CREATE TABLE ts ( + ts INT NOT NULL, + id BIGINT NOT NULL, + city STRING NOT NULL ) +PARTITION BY date_trunc('month', str_to_date(CAST(ts as STRING),'%Y%m%d')); +insert into ts values('20200201',1,'cd'); +insert into ts values('20200101',1,'cd'); +insert into ts values('20200301',1,'cd'); +insert into ts values('20200401',1,'cd'); +explain select * from ts where ts>20200201; + +CREATE TABLE o ( + ts BIGINT NOT NULL, + id BIGINT NOT NULL, + city STRING NOT NULL +) +PARTITION BY from_unixtime(ts,'%Y%m%d'); +insert into o values(1727224687,1,'cd'); +insert into o values(1737234687,1,'cd'); +insert into o values(1747244687,1,'cd'); +insert into o values(1757254687,1,'cd'); +explain select * from o where ts>1737234687; + +CREATE TABLE t ( + ts INT NOT NULL, + id BIGINT NOT NULL, + city STRING NOT NULL ) +PARTITION BY id, date_trunc('month', str_to_date(CAST(ts as STRING),'%Y%m%d')); +insert into t values('20200201',1,'cd'); +insert into t values('20200101',1,'cd'); +insert into t values('20200301',1,'cd'); +insert into t values('20200401',1,'cd'); +insert into t values('20200201',2,'cd'); +insert into t values('20200101',2,'cd'); +insert into t values('20200301',3,'cd'); +insert into t values('20200401',3,'cd'); +explain select * from t where ts>20200201; +explain select * from t where id>1; +explain select * from t where id>1 and ts>20200201; + +-- name: test_prune_value_function +CREATE TABLE o ( + ts BIGINT NOT NULL, + id BIGINT NOT NULL, + city STRING NOT NULL +) +PARTITION BY from_unixtime(ts,'%Y%m%d'); +insert into o values(unix_timestamp(),1,'cd'); +insert into o values(unix_timestamp()+1000000,1,'cd'); +insert into o values(unix_timestamp()-1000000,1,'cd'); +explain select * from o where ts>unix_timestamp(); + +-- name: test_prune_cast +CREATE TABLE o ( + ts STRING NOT NULL, + id BIGINT NOT NULL, + city STRING NOT NULL +) +PARTITION BY from_unixtime(cast(ts as INT) + 3600); +insert into o values(unix_timestamp(),1,'cd'); +insert into o values(unix_timestamp()+1000000,1,'cd'); +insert into o values(unix_timestamp()-1000000,1,'cd'); +explain select * from o where ts>cast(unix_timestamp() as STRING); +explain select * from o where ts>'1740108713'; + +-- name: test_prune_convert_tz +CREATE TABLE ts ( + ts DATETIME NOT NULL, + id BIGINT NOT NULL, + city STRING NOT NULL ) +PARTITION BY date_trunc('hour', convert_tz(ts, 'Asia/Shanghai', 'America/Los_Angeles')); +insert into ts values('2020-01-01 00:00:00',1,'cd'); +insert into ts values('2020-02-01 00:00:00',1,'cd'); +insert into ts values('2020-03-01 00:00:00',1,'cd'); +insert into ts values('2020-04-01 00:00:00',1,'cd'); +explain select * from ts where ts > '2020-03-01 00:00:00'; + +-- name: test_prune_if +CREATE TABLE ts ( + dt STRING NOT NULL, + id BIGINT NOT NULL, + city STRING NOT NULL ) +PARTITION BY date_trunc('month', if(length(dt) = 6, str2date(dt, '%y%m%d'), str2date(dt, '%Y%m%d'))); +insert into ts values('20200201',1,'cd'); +insert into ts values('20200101',1,'cd'); +insert into ts values('200301',1,'cd'); +insert into ts values('20200401',1,'cd'); +explain select * from ts where dt>20200201;