Skip to content

Commit

Permalink
[UT] Add some partition prune test case (#56143)
Browse files Browse the repository at this point in the history 
Signed-off-by: meegoo <meegoo.sr@gmail.com>
  • Loading branch information
@meegoo
meegoo authored Feb 25, 2025
1 parent 5dfdf44 commit f38b850
Show file tree
Hide file tree
Showing 5 changed files with 298 additions and 6 deletions.
6 changes: 6 additions & 0 deletions ...re/src/main/java/com/starrocks/sql/optimizer/operator/scalar/OperatorFunctionChecker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,12 @@ public Pair<Boolean, String> visit(ScalarOperator scalarOperator, Void context)
}

public Pair<Boolean, String> visitCall(CallOperator call, Void context) {
for (ScalarOperator child : call.getChildren()) {
Pair<Boolean, String> result = child.accept(this, null);
if (!result.first) {
return result;
}
}
if (predicate.test(call)) {
return Pair.create(true, "");
} else {
Expand Down
10 changes: 7 additions & 3 deletions fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorEvaluator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import com.starrocks.sql.optimizer.function.MetaFunctions;
import com.starrocks.sql.optimizer.operator.OperatorType;
import com.starrocks.sql.optimizer.operator.scalar.CallOperator;
import com.starrocks.sql.optimizer.operator.scalar.CastOperator;
import com.starrocks.sql.optimizer.operator.scalar.ConstantOperator;
import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator;
import org.apache.commons.collections4.ListUtils;
Expand Down Expand Up @@ -212,6 +213,9 @@ public ScalarOperator evaluation(CallOperator root, boolean needMonotonic) {
}

public boolean isMonotonicFunction(CallOperator call) {
if (call instanceof CastOperator) {
return true;
}
FunctionSignature signature;
if (call.getFunction() != null) {
Function fn = call.getFunction();
Expand Down Expand Up @@ -247,11 +251,11 @@ private boolean isMonotonicFunc(FunctionInvoker invoker, CallOperator operator)
return false;
}

if (FunctionSet.DATE_FORMAT.equalsIgnoreCase(invoker.getSignature().getName())
if ((FunctionSet.DATE_FORMAT.equalsIgnoreCase(invoker.getSignature().getName())
|| FunctionSet.STR_TO_DATE.equalsIgnoreCase(invoker.getSignature().getName())
|| FunctionSet.STR2DATE.equalsIgnoreCase(invoker.getSignature().getName())
|| (FunctionSet.FROM_UNIXTIME.equalsIgnoreCase(invoker.getSignature().getName())
&& operator.getChildren().size() == 2)) {
|| FunctionSet.FROM_UNIXTIME.equalsIgnoreCase(invoker.getSignature().getName()))
&& operator.getChildren().size() == 2) {
String pattern = operator.getChild(1).toString();
if (pattern.isEmpty()) {
return true;
Expand Down
6 changes: 3 additions & 3 deletions ...re/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ListPartitionPruner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,11 @@
import com.starrocks.sql.optimizer.operator.scalar.ConstantOperator;
import com.starrocks.sql.optimizer.operator.scalar.InPredicateOperator;
import com.starrocks.sql.optimizer.operator.scalar.IsNullPredicateOperator;
import com.starrocks.sql.optimizer.operator.scalar.OperatorFunctionChecker;
import com.starrocks.sql.optimizer.operator.scalar.PredicateOperator;
import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator;
import com.starrocks.sql.optimizer.operator.scalar.ScalarOperatorVisitor;
import com.starrocks.sql.optimizer.rewrite.ReplaceColumnRefRewriter;
import com.starrocks.sql.optimizer.rewrite.ScalarOperatorEvaluator;
import com.starrocks.sql.optimizer.rewrite.ScalarOperatorRewriter;
import com.starrocks.sql.optimizer.transformer.SqlToScalarOperatorTranslator;
import com.starrocks.sql.plan.ScalarOperatorToExpr;
Expand Down Expand Up @@ -278,7 +278,7 @@ public static List<String> deduceGenerateColumns(LogicalScanOperator scanOperato
SqlToScalarOperatorTranslator.translateWithSlotRef(generatedExpr, slotRefResolver);

if (call instanceof CallOperator &&
ScalarOperatorEvaluator.INSTANCE.isMonotonicFunction((CallOperator) call)) {
OperatorFunctionChecker.onlyContainMonotonicFunctions((CallOperator) call).first) {
List<ColumnRefOperator> columnRefOperatorList = Utils.extractColumnRef(call);
for (ColumnRefOperator ref : columnRefOperatorList) {
result.add(ref.getName());
Expand Down Expand Up @@ -325,7 +325,7 @@ private void deduceExtraConjuncts() {
SqlToScalarOperatorTranslator.translateWithSlotRef(generatedExpr, slotRefResolver);

if (call instanceof CallOperator &&
ScalarOperatorEvaluator.INSTANCE.isMonotonicFunction((CallOperator) call)) {
OperatorFunctionChecker.onlyContainMonotonicFunctions((CallOperator) call).first) {
List<ColumnRefOperator> columnRefOperatorList = Utils.extractColumnRef(call);

for (ColumnRefOperator ref : columnRefOperatorList) {
Expand Down
192 changes: 192 additions & 0 deletions test/sql/test_automatic_partition/R/test_partition_prune
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,195 @@ explain select * from orders_new where ts=20200101;
-- result:
[REGEX].*partitions=1/2.*
-- !result

-- name: test_range_prune
CREATE TABLE ts (
ts INT NOT NULL,
id BIGINT NOT NULL,
city STRING NOT NULL )
PARTITION BY date_trunc('month', str_to_date(CAST(ts as STRING),'%Y%m%d'));
-- result:
-- !result
insert into ts values('20200201',1,'cd');
-- result:
-- !result
insert into ts values('20200101',1,'cd');
-- result:
-- !result
insert into ts values('20200301',1,'cd');
-- result:
-- !result
insert into ts values('20200401',1,'cd');
-- result:
-- !result
explain select * from ts where ts>20200201;
-- result:
[REGEX].*partitions=3/4.*
-- !result
CREATE TABLE o (
ts BIGINT NOT NULL,
id BIGINT NOT NULL,
city STRING NOT NULL
)
PARTITION BY from_unixtime(ts,'%Y%m%d');
-- result:
-- !result
insert into o values(1727224687,1,'cd');
-- result:
-- !result
insert into o values(1737234687,1,'cd');
-- result:
-- !result
insert into o values(1747244687,1,'cd');
-- result:
-- !result
insert into o values(1757254687,1,'cd');
-- result:
-- !result
explain select * from o where ts>1737234687;
-- result:
[REGEX].*partitions=3/4.*
-- !result
CREATE TABLE t (
ts INT NOT NULL,
id BIGINT NOT NULL,
city STRING NOT NULL )
PARTITION BY id, date_trunc('month', str_to_date(CAST(ts as STRING),'%Y%m%d'));
-- result:
-- !result
insert into t values('20200201',1,'cd');
-- result:
-- !result
insert into t values('20200101',1,'cd');
-- result:
-- !result
insert into t values('20200301',1,'cd');
-- result:
-- !result
insert into t values('20200401',1,'cd');
-- result:
-- !result
insert into t values('20200201',2,'cd');
-- result:
-- !result
insert into t values('20200101',2,'cd');
-- result:
-- !result
insert into t values('20200301',3,'cd');
-- result:
-- !result
insert into t values('20200401',3,'cd');
-- result:
-- !result
explain select * from t where ts>20200201;
-- result:
[REGEX].*partitions=6/8.*
-- !result
explain select * from t where id>1;
-- result:
[REGEX].*partitions=4/8.*
-- !result
explain select * from t where id>1 and ts>20200201;
-- result:
[REGEX].*partitions=3/8.*
-- !result

-- name: test_prune_value_function
CREATE TABLE o (
ts BIGINT NOT NULL,
id BIGINT NOT NULL,
city STRING NOT NULL
)
PARTITION BY from_unixtime(ts,'%Y%m%d');
-- result:
-- !result
insert into o values(unix_timestamp(),1,'cd');
-- result:
-- !result
insert into o values(unix_timestamp()+1000000,1,'cd');
-- result:
-- !result
insert into o values(unix_timestamp()-1000000,1,'cd');
-- result:
-- !result
explain select * from o where ts>unix_timestamp();
-- result:
[REGEX].*partitions=2/3.*
-- !result

-- name: test_prune_cast
CREATE TABLE o (
ts STRING NOT NULL,
id BIGINT NOT NULL,
city STRING NOT NULL
)
PARTITION BY from_unixtime(cast(ts as INT) + 3600);
-- result:
-- !result
insert into o values(unix_timestamp(),1,'cd');
-- result:
-- !result
insert into o values(unix_timestamp()+1000000,1,'cd');
-- result:
-- !result
insert into o values(unix_timestamp()-1000000,1,'cd');
-- result:
-- !result
explain select * from o where ts>cast(unix_timestamp() as STRING);
-- result:
[REGEX].*partitions=1/3.*
-- !result
explain select * from o where ts>'1740108713';
-- result:
[REGEX].*partitions=2/3.*
-- !result

-- name: test_prune_convert_tz
CREATE TABLE ts (
ts DATETIME NOT NULL,
id BIGINT NOT NULL,
city STRING NOT NULL )
PARTITION BY date_trunc('hour', convert_tz(ts, 'Asia/Shanghai', 'America/Los_Angeles'));
-- result:
-- !result
insert into ts values('2020-01-01 00:00:00',1,'cd');
-- result:
-- !result
insert into ts values('2020-02-01 00:00:00',1,'cd');
-- result:
-- !result
insert into ts values('2020-03-01 00:00:00',1,'cd');
-- result:
-- !result
insert into ts values('2020-04-01 00:00:00',1,'cd');
-- result:
-- !result
explain select * from ts where ts > '2020-03-01 00:00:00';
-- result:
[REGEX].*partitions=4/4.*
-- !result

-- name: test_prune_if
CREATE TABLE ts (
dt STRING NOT NULL,
id BIGINT NOT NULL,
city STRING NOT NULL )
PARTITION BY date_trunc('month', if(length(dt) = 6, str2date(dt, '%y%m%d'), str2date(dt, '%Y%m%d')));
-- result:
-- !result
insert into ts values('20200201',1,'cd');
-- result:
-- !result
insert into ts values('20200101',1,'cd');
-- result:
-- !result
insert into ts values('200301',1,'cd');
-- result:
-- !result
insert into ts values('20200401',1,'cd');
-- result:
-- !result
explain select * from ts where dt>20200201;
-- result:
[REGEX].*partitions=4/4.*
-- !result
90 changes: 90 additions & 0 deletions test/sql/test_automatic_partition/T/test_partition_prune
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,93 @@ CREATE TABLE orders_new ( ts INT NOT NULL, id BIGINT NOT NULL, city
insert into orders_new values('20200201',1,'cd');
insert into orders_new values('20200101',1,'cd');
explain select * from orders_new where ts=20200101;

-- name: test_range_prune
CREATE TABLE ts (
ts INT NOT NULL,
id BIGINT NOT NULL,
city STRING NOT NULL )
PARTITION BY date_trunc('month', str_to_date(CAST(ts as STRING),'%Y%m%d'));
insert into ts values('20200201',1,'cd');
insert into ts values('20200101',1,'cd');
insert into ts values('20200301',1,'cd');
insert into ts values('20200401',1,'cd');
explain select * from ts where ts>20200201;

CREATE TABLE o (
ts BIGINT NOT NULL,
id BIGINT NOT NULL,
city STRING NOT NULL
)
PARTITION BY from_unixtime(ts,'%Y%m%d');
insert into o values(1727224687,1,'cd');
insert into o values(1737234687,1,'cd');
insert into o values(1747244687,1,'cd');
insert into o values(1757254687,1,'cd');
explain select * from o where ts>1737234687;

CREATE TABLE t (
ts INT NOT NULL,
id BIGINT NOT NULL,
city STRING NOT NULL )
PARTITION BY id, date_trunc('month', str_to_date(CAST(ts as STRING),'%Y%m%d'));
insert into t values('20200201',1,'cd');
insert into t values('20200101',1,'cd');
insert into t values('20200301',1,'cd');
insert into t values('20200401',1,'cd');
insert into t values('20200201',2,'cd');
insert into t values('20200101',2,'cd');
insert into t values('20200301',3,'cd');
insert into t values('20200401',3,'cd');
explain select * from t where ts>20200201;
explain select * from t where id>1;
explain select * from t where id>1 and ts>20200201;

-- name: test_prune_value_function
CREATE TABLE o (
ts BIGINT NOT NULL,
id BIGINT NOT NULL,
city STRING NOT NULL
)
PARTITION BY from_unixtime(ts,'%Y%m%d');
insert into o values(unix_timestamp(),1,'cd');
insert into o values(unix_timestamp()+1000000,1,'cd');
insert into o values(unix_timestamp()-1000000,1,'cd');
explain select * from o where ts>unix_timestamp();

-- name: test_prune_cast
CREATE TABLE o (
ts STRING NOT NULL,
id BIGINT NOT NULL,
city STRING NOT NULL
)
PARTITION BY from_unixtime(cast(ts as INT) + 3600);
insert into o values(unix_timestamp(),1,'cd');
insert into o values(unix_timestamp()+1000000,1,'cd');
insert into o values(unix_timestamp()-1000000,1,'cd');
explain select * from o where ts>cast(unix_timestamp() as STRING);
explain select * from o where ts>'1740108713';

-- name: test_prune_convert_tz
CREATE TABLE ts (
ts DATETIME NOT NULL,
id BIGINT NOT NULL,
city STRING NOT NULL )
PARTITION BY date_trunc('hour', convert_tz(ts, 'Asia/Shanghai', 'America/Los_Angeles'));
insert into ts values('2020-01-01 00:00:00',1,'cd');
insert into ts values('2020-02-01 00:00:00',1,'cd');
insert into ts values('2020-03-01 00:00:00',1,'cd');
insert into ts values('2020-04-01 00:00:00',1,'cd');
explain select * from ts where ts > '2020-03-01 00:00:00';

-- name: test_prune_if
CREATE TABLE ts (
dt STRING NOT NULL,
id BIGINT NOT NULL,
city STRING NOT NULL )
PARTITION BY date_trunc('month', if(length(dt) = 6, str2date(dt, '%y%m%d'), str2date(dt, '%Y%m%d')));
insert into ts values('20200201',1,'cd');
insert into ts values('20200101',1,'cd');
insert into ts values('200301',1,'cd');
insert into ts values('20200401',1,'cd');
explain select * from ts where dt>20200201;

0 comments on commit f38b850

Please sign in to comment.