Skip to content

Commit ffdad2b

Browse files
jackwenerjackwener
committed
revert modify optimizer rules order.
1 parent 5048893 commit ffdad2b

File tree

7 files changed

+53
-55
lines changed

7 files changed

+53
-55
lines changed

‎benchmarks/expected-plans/q17.txt‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,15 @@
11
Projection: CAST(SUM(lineitem.l_extendedprice) AS Float64) / Float64(7) AS avg_yearly
22
Aggregate: groupBy=[[]], aggr=[[SUM(lineitem.l_extendedprice)]]
33
Projection: lineitem.l_extendedprice
4-
Filter: CAST(lineitem.l_quantity AS Decimal128(30, 15)) < CAST(__scalar_sq_1.__value AS Decimal128(30, 15))
5-
Projection: lineitem.l_quantity, lineitem.l_extendedprice, __scalar_sq_1.__value
6-
Inner Join: part.p_partkey = __scalar_sq_1.l_partkey, lineitem.l_partkey = __scalar_sq_1.l_partkey
7-
Inner Join: lineitem.l_partkey = part.p_partkey
8-
TableScan: lineitem projection=[l_partkey, l_quantity, l_extendedprice]
9-
Projection: part.p_partkey
10-
Filter: part.p_brand = Utf8("Brand#23") AND part.p_container = Utf8("MED BOX")
11-
TableScan: part projection=[p_partkey, p_brand, p_container]
4+
Filter: CAST(lineitem.l_quantity AS Decimal128(30, 15)) < CAST(__scalar_sq_1.__value AS Decimal128(30, 15)) AND __scalar_sq_1.l_partkey = lineitem.l_partkey
5+
Projection: lineitem.l_partkey, lineitem.l_quantity, lineitem.l_extendedprice, __scalar_sq_1.l_partkey, __scalar_sq_1.__value
6+
Inner Join: part.p_partkey = __scalar_sq_1.l_partkey
7+
Filter: part.p_partkey = lineitem.l_partkey AND lineitem.l_partkey = part.p_partkey
8+
Inner Join: lineitem.l_partkey = part.p_partkey
9+
TableScan: lineitem projection=[l_partkey, l_quantity, l_extendedprice]
10+
Projection: part.p_partkey
11+
Filter: part.p_brand = Utf8("Brand#23") AND part.p_container = Utf8("MED BOX")
12+
TableScan: part projection=[p_partkey, p_brand, p_container]
1213
SubqueryAlias: __scalar_sq_1
1314
Projection: lineitem.l_partkey, Float64(0.2) * CAST(AVG(lineitem.l_quantity) AS Float64) AS __value
1415
Aggregate: groupBy=[[lineitem.l_partkey]], aggr=[[AVG(lineitem.l_quantity)]]

‎benchmarks/expected-plans/q2.txt‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,18 @@
11
Sort: supplier.s_acctbal DESC NULLS FIRST, nation.n_name ASC NULLS LAST, supplier.s_name ASC NULLS LAST, part.p_partkey ASC NULLS LAST
22
Projection: supplier.s_acctbal, supplier.s_name, nation.n_name, part.p_partkey, part.p_mfgr, supplier.s_address, supplier.s_phone, supplier.s_comment
3-
Inner Join: part.p_partkey = __scalar_sq_1.ps_partkey, partsupp.ps_supplycost = __scalar_sq_1.__value, partsupp.ps_partkey = __scalar_sq_1.ps_partkey
4-
Projection: part.p_partkey, part.p_mfgr, supplier.s_name, supplier.s_address, supplier.s_phone, supplier.s_acctbal, supplier.s_comment, partsupp.ps_partkey, partsupp.ps_supplycost, nation.n_name
3+
Inner Join: part.p_partkey = __scalar_sq_1.ps_partkey, partsupp.ps_supplycost = __scalar_sq_1.__value
4+
Projection: part.p_partkey, part.p_mfgr, partsupp.ps_supplycost, supplier.s_name, supplier.s_address, supplier.s_phone, supplier.s_acctbal, supplier.s_comment, nation.n_name
55
Inner Join: nation.n_regionkey = region.r_regionkey
6-
Projection: part.p_partkey, part.p_mfgr, supplier.s_name, supplier.s_address, supplier.s_phone, supplier.s_acctbal, supplier.s_comment, partsupp.ps_partkey, partsupp.ps_supplycost, nation.n_name, nation.n_regionkey
6+
Projection: part.p_partkey, part.p_mfgr, partsupp.ps_supplycost, supplier.s_name, supplier.s_address, supplier.s_phone, supplier.s_acctbal, supplier.s_comment, nation.n_name, nation.n_regionkey
77
Inner Join: supplier.s_nationkey = nation.n_nationkey
8-
Projection: part.p_partkey, part.p_mfgr, supplier.s_name, supplier.s_address, supplier.s_nationkey, supplier.s_phone, supplier.s_acctbal, supplier.s_comment, partsupp.ps_partkey, partsupp.ps_supplycost
8+
Projection: part.p_partkey, part.p_mfgr, partsupp.ps_supplycost, supplier.s_name, supplier.s_address, supplier.s_nationkey, supplier.s_phone, supplier.s_acctbal, supplier.s_comment
99
Inner Join: partsupp.ps_suppkey = supplier.s_suppkey
10-
Inner Join: part.p_partkey = partsupp.ps_partkey
11-
Projection: part.p_partkey, part.p_mfgr
12-
Filter: part.p_size = Int32(15) AND part.p_type LIKE Utf8("%BRASS")
13-
TableScan: part projection=[p_partkey, p_mfgr, p_type, p_size]
14-
TableScan: partsupp projection=[ps_partkey, ps_suppkey, ps_supplycost]
10+
Projection: part.p_partkey, part.p_mfgr, partsupp.ps_suppkey, partsupp.ps_supplycost
11+
Inner Join: part.p_partkey = partsupp.ps_partkey
12+
Projection: part.p_partkey, part.p_mfgr
13+
Filter: part.p_size = Int32(15) AND part.p_type LIKE Utf8("%BRASS")
14+
TableScan: part projection=[p_partkey, p_mfgr, p_type, p_size]
15+
TableScan: partsupp projection=[ps_partkey, ps_suppkey, ps_supplycost]
1516
TableScan: supplier projection=[s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment]
1617
TableScan: nation projection=[n_nationkey, n_name, n_regionkey]
1718
Projection: region.r_regionkey

‎benchmarks/expected-plans/q8.txt‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,13 @@ Sort: all_nations.o_year ASC NULLS LAST
66
Inner Join: n1.n_regionkey = region.r_regionkey
77
Projection: lineitem.l_extendedprice, lineitem.l_discount, orders.o_orderdate, n1.n_regionkey, n2.n_name
88
Inner Join: supplier.s_nationkey = n2.n_nationkey
9-
Projection: supplier.s_nationkey, lineitem.l_extendedprice, lineitem.l_discount, orders.o_orderdate, n1.n_regionkey
9+
Projection: lineitem.l_extendedprice, lineitem.l_discount, supplier.s_nationkey, orders.o_orderdate, n1.n_regionkey
1010
Inner Join: customer.c_nationkey = n1.n_nationkey
11-
Projection: supplier.s_nationkey, lineitem.l_extendedprice, lineitem.l_discount, orders.o_orderdate, customer.c_nationkey
11+
Projection: lineitem.l_extendedprice, lineitem.l_discount, supplier.s_nationkey, orders.o_orderdate, customer.c_nationkey
1212
Inner Join: orders.o_custkey = customer.c_custkey
13-
Projection: supplier.s_nationkey, lineitem.l_extendedprice, lineitem.l_discount, orders.o_custkey, orders.o_orderdate
13+
Projection: lineitem.l_extendedprice, lineitem.l_discount, supplier.s_nationkey, orders.o_custkey, orders.o_orderdate
1414
Inner Join: lineitem.l_orderkey = orders.o_orderkey
15-
Projection: supplier.s_nationkey, lineitem.l_orderkey, lineitem.l_extendedprice, lineitem.l_discount
15+
Projection: lineitem.l_orderkey, lineitem.l_extendedprice, lineitem.l_discount, supplier.s_nationkey
1616
Inner Join: lineitem.l_suppkey = supplier.s_suppkey
1717
Projection: lineitem.l_orderkey, lineitem.l_suppkey, lineitem.l_extendedprice, lineitem.l_discount
1818
Inner Join: part.p_partkey = lineitem.l_partkey

‎benchmarks/expected-plans/q9.txt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,11 @@ Sort: profit.nation ASC NULLS LAST, profit.o_year DESC NULLS FIRST
44
SubqueryAlias: profit
55
Projection: nation.n_name AS nation, datepart(Utf8("YEAR"), orders.o_orderdate) AS o_year, CAST(lineitem.l_extendedprice AS Decimal128(38, 4)) * CAST(Decimal128(Some(100),23,2) - CAST(lineitem.l_discount AS Decimal128(23, 2)) AS Decimal128(38, 4)) - CAST(partsupp.ps_supplycost * lineitem.l_quantity AS Decimal128(38, 4)) AS amount
66
Inner Join: supplier.s_nationkey = nation.n_nationkey
7-
Projection: supplier.s_nationkey, lineitem.l_quantity, lineitem.l_extendedprice, lineitem.l_discount, partsupp.ps_supplycost, orders.o_orderdate
7+
Projection: lineitem.l_quantity, lineitem.l_extendedprice, lineitem.l_discount, supplier.s_nationkey, partsupp.ps_supplycost, orders.o_orderdate
88
Inner Join: lineitem.l_orderkey = orders.o_orderkey
9-
Projection: supplier.s_nationkey, lineitem.l_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, lineitem.l_discount, partsupp.ps_supplycost
9+
Projection: lineitem.l_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, lineitem.l_discount, supplier.s_nationkey, partsupp.ps_supplycost
1010
Inner Join: lineitem.l_suppkey = partsupp.ps_suppkey, lineitem.l_partkey = partsupp.ps_partkey
11-
Projection: supplier.s_nationkey, lineitem.l_orderkey, lineitem.l_partkey, lineitem.l_suppkey, lineitem.l_quantity, lineitem.l_extendedprice, lineitem.l_discount
11+
Projection: lineitem.l_orderkey, lineitem.l_partkey, lineitem.l_suppkey, lineitem.l_quantity, lineitem.l_extendedprice, lineitem.l_discount, supplier.s_nationkey
1212
Inner Join: lineitem.l_suppkey = supplier.s_suppkey
1313
Projection: lineitem.l_orderkey, lineitem.l_partkey, lineitem.l_suppkey, lineitem.l_quantity, lineitem.l_extendedprice, lineitem.l_discount
1414
Inner Join: part.p_partkey = lineitem.l_partkey

‎datafusion/core/tests/sql/joins.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1604,7 +1604,7 @@ async fn reduce_left_join_3() -> Result<()> {
16041604
" Filter: t1.t1_id < UInt32(100) [t1_id:UInt32;N, t1_name:Utf8;N, t1_int:UInt32;N]",
16051605
" TableScan: t1 projection=[t1_id, t1_name, t1_int] [t1_id:UInt32;N, t1_name:Utf8;N, t1_int:UInt32;N]",
16061606
" Projection: t2.t2_id [t2_id:UInt32;N]",
1607-
" Filter: t2.t2_int < UInt32(3) [t2_id:UInt32;N, t2_int:UInt32;N]",
1607+
" Filter: t2.t2_int < UInt32(3) AND t2.t2_id < UInt32(100) [t2_id:UInt32;N, t2_int:UInt32;N]",
16081608
" TableScan: t2 projection=[t2_id, t2_int] [t2_id:UInt32;N, t2_int:UInt32;N]",
16091609
" TableScan: t2 projection=[t2_id, t2_name, t2_int] [t2_id:UInt32;N, t2_name:Utf8;N, t2_int:UInt32;N]",
16101610
];

‎datafusion/optimizer/src/optimizer.rs‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -216,24 +216,25 @@ impl Optimizer {
216216
// subqueries to joins
217217
Arc::new(SimplifyExpressions::new()),
218218
Arc::new(RewriteDisjunctivePredicate::new()),
219-
Arc::new(EliminateLimit::new()),
220219
Arc::new(EliminateFilter::new()),
221-
// Filters can't be pushed down past Limits, we should do PushDownFilter after PushDownLimit
222-
Arc::new(PushDownLimit::new()),
223-
Arc::new(PushDownFilter::new()),
224-
Arc::new(EliminateOuterJoin::new()),
225220
Arc::new(EliminateCrossJoin::new()),
226-
Arc::new(PushDownProjection::new()),
227-
Arc::new(PushDownLimit::new()),
228-
Arc::new(PropagateEmptyRelation::new()),
229221
Arc::new(CommonSubexprEliminate::new()),
222+
Arc::new(EliminateLimit::new()),
223+
Arc::new(PropagateEmptyRelation::new()),
230224
Arc::new(FilterNullJoinKeys::default()),
225+
Arc::new(EliminateOuterJoin::new()),
226+
// Filters can't be pushed down past Limits, we should do PushDownFilter after PushDownLimit
227+
Arc::new(PushDownLimit::new()),
228+
Arc::new(PushDownFilter::new()),
231229
Arc::new(SingleDistinctToGroupBy::new()),
232230
// The previous optimizations added expressions and projections,
233231
// that might benefit from the following rules
234232
Arc::new(SimplifyExpressions::new()),
235233
Arc::new(UnwrapCastInComparison::new()),
236234
Arc::new(CommonSubexprEliminate::new()),
235+
Arc::new(PushDownProjection::new()),
236+
// PushDownProjection can pushdown Projections through Limits, do PushDownLimit again.
237+
Arc::new(PushDownLimit::new()),
237238
];
238239

239240
Self::with_rules(rules)

‎datafusion/optimizer/tests/integration-test.rs‎

Lines changed: 17 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -275,15 +275,14 @@ fn join_keys_in_subquery_alias() {
275275
let sql = "SELECT * FROM test AS A, ( SELECT col_int32 as key FROM test ) AS B where A.col_int32 = B.key;";
276276
let plan = test_sql(sql).unwrap();
277277
let expected = "Inner Join: a.col_int32 = b.key\
278-
\n Filter: a.col_int32 IS NOT NULL\
279-
\n SubqueryAlias: a\
278+
\n SubqueryAlias: a\
279+
\n Filter: test.col_int32 IS NOT NULL\
280+
\n TableScan: test projection=[col_int32, col_uint32, col_utf8, col_date32, col_date64, col_ts_nano_none, col_ts_nano_utc]\
281+
\n SubqueryAlias: b\
282+
\n Projection: test.col_int32 AS key\
280283
\n Filter: test.col_int32 IS NOT NULL\
281-
\n TableScan: test projection=[col_int32, col_uint32, col_utf8, col_date32, col_date64, col_ts_nano_none, col_ts_nano_utc]\
282-
\n Filter: b.key IS NOT NULL\
283-
\n SubqueryAlias: b\
284-
\n Projection: test.col_int32 AS key\
285-
\n Filter: test.col_int32 IS NOT NULL\
286-
\n TableScan: test projection=[col_int32]";
284+
\n TableScan: test projection=[col_int32]";
285+
287286
assert_eq!(expected, format!("{plan:?}"));
288287
}
289288

@@ -292,21 +291,17 @@ fn join_keys_in_subquery_alias_1() {
292291
let sql = "SELECT * FROM test AS A, ( SELECT test.col_int32 AS key FROM test JOIN test AS C on test.col_int32 = C.col_int32 ) AS B where A.col_int32 = B.key;";
293292
let plan = test_sql(sql).unwrap();
294293
let expected = "Inner Join: a.col_int32 = b.key\
295-
\n Filter: a.col_int32 IS NOT NULL\
296-
\n SubqueryAlias: a\
297-
\n Filter: test.col_int32 IS NOT NULL\
298-
\n TableScan: test projection=[col_int32, col_uint32, col_utf8, col_date32, col_date64, col_ts_nano_none, col_ts_nano_utc]\
299-
\n Filter: b.key IS NOT NULL\
300-
\n SubqueryAlias: b\
301-
\n Projection: test.col_int32 AS key\
302-
\n Inner Join: test.col_int32 = c.col_int32\
294+
\n SubqueryAlias: a\
295+
\n Filter: test.col_int32 IS NOT NULL\
296+
\n TableScan: test projection=[col_int32, col_uint32, col_utf8, col_date32, col_date64, col_ts_nano_none, col_ts_nano_utc]\
297+
\n SubqueryAlias: b\
298+
\n Projection: test.col_int32 AS key\
299+
\n Inner Join: test.col_int32 = c.col_int32\
300+
\n Filter: test.col_int32 IS NOT NULL\
301+
\n TableScan: test projection=[col_int32]\
302+
\n SubqueryAlias: c\
303303
\n Filter: test.col_int32 IS NOT NULL\
304-
\n Filter: test.col_int32 IS NOT NULL\
305-
\n TableScan: test projection=[col_int32]\
306-
\n Filter: c.col_int32 IS NOT NULL\
307-
\n SubqueryAlias: c\
308-
\n Filter: test.col_int32 IS NOT NULL\
309-
\n TableScan: test projection=[col_int32]";
304+
\n TableScan: test projection=[col_int32]";
310305
assert_eq!(expected, format!("{plan:?}"));
311306
}
312307

0 commit comments

Comments
 (0)