Skip to content

Commit

Permalink
[BugFix] Fix complex expression partition prune
Browse files Browse the repository at this point in the history 
Signed-off-by: meegoo <meegoo.sr@gmail.com>
  • Loading branch information
@meegoo
meegoo committed Feb 21, 2025
1 parent 71e6df2 commit da3706d
Show file tree
Hide file tree
Showing 3 changed files with 270 additions and 0 deletions.
14 changes: 14 additions & 0 deletions fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorEvaluator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import com.starrocks.sql.optimizer.function.MetaFunctions;
import com.starrocks.sql.optimizer.operator.OperatorType;
import com.starrocks.sql.optimizer.operator.scalar.CallOperator;
import com.starrocks.sql.optimizer.operator.scalar.CastOperator;
import com.starrocks.sql.optimizer.operator.scalar.ConstantOperator;
import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator;
import org.apache.commons.collections4.ListUtils;
Expand Down Expand Up @@ -212,6 +213,19 @@ public ScalarOperator evaluation(CallOperator root, boolean needMonotonic) {
}

public boolean isMonotonicFunction(CallOperator call) {
for (ScalarOperator child : call.getChildren()) {
if (child.isConstant()) {
continue;
}
if (child instanceof CastOperator) {
continue;
}
if (child instanceof CallOperator) {
if (!isMonotonicFunction((CallOperator) child)) {
return false;
}
}
}
FunctionSignature signature;
if (call.getFunction() != null) {
Function fn = call.getFunction();
Expand Down
178 changes: 178 additions & 0 deletions test/sql/test_automatic_partition/R/test_partition_prune
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,181 @@ explain select * from orders_new where ts=20200101;
-- result:
[REGEX].*partitions=1/2.*
-- !result

-- name: test_range_prune
CREATE TABLE ts (
ts INT NOT NULL,
id BIGINT NOT NULL,
city STRING NOT NULL )
PARTITION BY date_trunc('month', str_to_date(CAST(ts as STRING),'%Y%m%d'));
-- result:
-- !result
insert into ts values('20200201',1,'cd');
-- result:
-- !result
insert into ts values('20200101',1,'cd');
-- result:
-- !result
insert into ts values('20200301',1,'cd');
-- result:
-- !result
insert into ts values('20200401',1,'cd');
-- result:
-- !result
explain select * from ts where ts>20200201;
-- result:
[REGEX].*partitions=3/4.*
-- !result
CREATE TABLE o (
ts BIGINT NOT NULL,
id BIGINT NOT NULL,
city STRING NOT NULL
)
PARTITION BY from_unixtime(ts,'%Y%m%d');
-- result:
-- !result
insert into o values(1727224687,1,'cd');
-- result:
-- !result
insert into o values(1737234687,1,'cd');
-- result:
-- !result
insert into o values(1747244687,1,'cd');
-- result:
-- !result
insert into o values(1757254687,1,'cd');
-- result:
-- !result
explain select * from o where ts>1737234687;
-- result:
[REGEX].*partitions=3/4.*
-- !result
CREATE TABLE t (
ts INT NOT NULL,
id BIGINT NOT NULL,
city STRING NOT NULL )
PARTITION BY id, date_trunc('month', str_to_date(CAST(ts as STRING),'%Y%m%d'));
-- result:
-- !result
insert into t values('20200201',1,'cd');
-- result:
-- !result
insert into t values('20200101',1,'cd');
-- result:
-- !result
insert into t values('20200301',1,'cd');
-- result:
-- !result
insert into t values('20200401',1,'cd');
-- result:
-- !result
insert into t values('20200201',2,'cd');
-- result:
-- !result
insert into t values('20200101',2,'cd');
-- result:
-- !result
insert into t values('20200301',3,'cd');
-- result:
-- !result
insert into t values('20200401',3,'cd');
-- result:
-- !result
explain select * from t where ts>20200201;
-- result:
[REGEX].*partitions=6/8.*
-- !result
explain select * from t where id>1;
-- result:
[REGEX].*partitions=4/8.*
-- !result
explain select * from t where id>1 and ts>20200201;
-- result:
[REGEX].*partitions=3/8.*
-- !result





-- name: test_prune_value_function
CREATE TABLE o (
ts BIGINT NOT NULL,
id BIGINT NOT NULL,
city STRING NOT NULL
)
PARTITION BY from_unixtime(ts,'%Y%m%d');
-- result:
-- !result
insert into o values(unix_timestamp(),1,'cd');
-- result:
-- !result
insert into o values(unix_timestamp()+1000000,1,'cd');
-- result:
-- !result
insert into o values(unix_timestamp()-1000000,1,'cd');
-- result:
-- !result
explain select * from o where ts>unix_timestamp();
-- result:
[REGEX].*partitions=2/3.*
-- !result





-- name: test_prune_cast
CREATE TABLE o (
ts STRING NOT NULL,
id BIGINT NOT NULL,
city STRING NOT NULL
)
PARTITION BY from_unixtime(cast(ts as INT) + 3600);
-- result:
-- !result
insert into o values(unix_timestamp(),1,'cd');
-- result:
-- !result
insert into o values(unix_timestamp()+1000000,1,'cd');
-- result:
-- !result
insert into o values(unix_timestamp()-1000000,1,'cd');
-- result:
-- !result
explain select * from o where ts>cast(unix_timestamp() as STRING);
-- result:
[REGEX].*partitions=1/3.*
-- !result
explain select * from o where ts>'1740108713';
-- result:
[REGEX].*partitions=2/3.*
-- !result




-- name: test_prune_convert_tz
CREATE TABLE ts (
ts DATETIME NOT NULL,
id BIGINT NOT NULL,
city STRING NOT NULL )
PARTITION BY date_trunc('hour', convert_tz(ts, 'Asia/Shanghai', 'America/Los_Angeles'));
-- result:
-- !result
insert into ts values('2020-01-01 00:00:00',1,'cd');
-- result:
-- !result
insert into ts values('2020-02-01 00:00:00',1,'cd');
-- result:
-- !result
insert into ts values('2020-03-01 00:00:00',1,'cd');
-- result:
-- !result
insert into ts values('2020-04-01 00:00:00',1,'cd');
-- result:
-- !result
explain select * from ts where ts > '2020-03-01 00:00:00';
-- result:
[REGEX].*partitions=4/4.*
-- !result
78 changes: 78 additions & 0 deletions test/sql/test_automatic_partition/T/test_partition_prune
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,81 @@ CREATE TABLE orders_new ( ts INT NOT NULL, id BIGINT NOT NULL, city
insert into orders_new values('20200201',1,'cd');
insert into orders_new values('20200101',1,'cd');
explain select * from orders_new where ts=20200101;

-- name: test_range_prune
CREATE TABLE ts (
ts INT NOT NULL,
id BIGINT NOT NULL,
city STRING NOT NULL )
PARTITION BY date_trunc('month', str_to_date(CAST(ts as STRING),'%Y%m%d'));
insert into ts values('20200201',1,'cd');
insert into ts values('20200101',1,'cd');
insert into ts values('20200301',1,'cd');
insert into ts values('20200401',1,'cd');
explain select * from ts where ts>20200201;

CREATE TABLE o (
ts BIGINT NOT NULL,
id BIGINT NOT NULL,
city STRING NOT NULL
)
PARTITION BY from_unixtime(ts,'%Y%m%d');
insert into o values(1727224687,1,'cd');
insert into o values(1737234687,1,'cd');
insert into o values(1747244687,1,'cd');
insert into o values(1757254687,1,'cd');
explain select * from o where ts>1737234687;

CREATE TABLE t (
ts INT NOT NULL,
id BIGINT NOT NULL,
city STRING NOT NULL )
PARTITION BY id, date_trunc('month', str_to_date(CAST(ts as STRING),'%Y%m%d'));
insert into t values('20200201',1,'cd');
insert into t values('20200101',1,'cd');
insert into t values('20200301',1,'cd');
insert into t values('20200401',1,'cd');
insert into t values('20200201',2,'cd');
insert into t values('20200101',2,'cd');
insert into t values('20200301',3,'cd');
insert into t values('20200401',3,'cd');
explain select * from t where ts>20200201;
explain select * from t where id>1;
explain select * from t where id>1 and ts>20200201;

-- name: test_prune_value_function
CREATE TABLE o (
ts BIGINT NOT NULL,
id BIGINT NOT NULL,
city STRING NOT NULL
)
PARTITION BY from_unixtime(ts,'%Y%m%d');
insert into o values(unix_timestamp(),1,'cd');
insert into o values(unix_timestamp()+1000000,1,'cd');
insert into o values(unix_timestamp()-1000000,1,'cd');
explain select * from o where ts>unix_timestamp();

-- name: test_prune_cast
CREATE TABLE o (
ts STRING NOT NULL,
id BIGINT NOT NULL,
city STRING NOT NULL
)
PARTITION BY from_unixtime(cast(ts as INT) + 3600);
insert into o values(unix_timestamp(),1,'cd');
insert into o values(unix_timestamp()+1000000,1,'cd');
insert into o values(unix_timestamp()-1000000,1,'cd');
explain select * from o where ts>cast(unix_timestamp() as STRING);
explain select * from o where ts>'1740108713';

-- name: test_prune_convert_tz
CREATE TABLE ts (
ts DATETIME NOT NULL,
id BIGINT NOT NULL,
city STRING NOT NULL )
PARTITION BY date_trunc('hour', convert_tz(ts, 'Asia/Shanghai', 'America/Los_Angeles'));
insert into ts values('2020-01-01 00:00:00',1,'cd');
insert into ts values('2020-02-01 00:00:00',1,'cd');
insert into ts values('2020-03-01 00:00:00',1,'cd');
insert into ts values('2020-04-01 00:00:00',1,'cd');
explain select * from ts where ts > '2020-03-01 00:00:00';

0 comments on commit da3706d

Please sign in to comment.