Skip to content

Commit

Permalink
planner, CTE: Fix default inline CTE which contains agg or window fun…
Browse files Browse the repository at this point in the history 
…ction and refactor inline CTE strategy (#48436)

close #47711
  • Loading branch information
@ti-chi-bot
ti-chi-bot authored Nov 10, 2023
1 parent ac6bc7a commit 30498cc
Show file tree
Hide file tree
Showing 13 changed files with 209 additions and 148 deletions.
12 changes: 0 additions & 12 deletions cmd/explaintest/r/cte.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -790,15 +790,3 @@ with cte1 as (select 1), cte2 as (select 2) select * from cte1 union (with cte2
1
1
3
explain with cte1 as (select 1), cte2 as (select 2) select * from cte1 union (with cte2 as (select 3) select * from cte2 union all select * from cte2);
id estRows task access object operator info
HashAgg_24 3.00 root group by:Column#9, funcs:firstrow(Column#9)->Column#9
└─Union_25 3.00 root
├─Projection_26 1.00 root 1->Column#9
│ └─TableDual_27 1.00 root rows:1
└─Union_29 2.00 root
├─CTEFullScan_31 1.00 root CTE:cte2 data:CTE_2
└─CTEFullScan_33 1.00 root CTE:cte2 data:CTE_2
CTE_2 1.00 root Non-Recursive CTE
└─Projection_22(Seed Part) 1.00 root 3->Column#5
└─TableDual_23 1.00 root rows:1
27 changes: 0 additions & 27 deletions cmd/explaintest/r/explain_cte.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,33 +167,6 @@ CTE_0 8001.00 root Recursive CTE, limit(offset:0, count:0)
│ └─TableFullScan_17 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
└─Projection_21(Recursive Part) 10000.00 root cast(plus(test.t1.c1, 1), int(11))->test.t1.c1
└─CTETable_22 10000.00 root Scan on CTE_0
explain with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 1) select * from cte1;
id estRows task access object operator info
CTEFullScan_34 1.00 root CTE:cte1 data:CTE_0
CTE_0 1.00 root Non-Recursive CTE
└─Limit_20(Seed Part) 1.00 root offset:0, count:1
└─HashAgg_21 1.00 root group by:Column#11, funcs:firstrow(Column#11)->Column#11
└─Union_22 20000.00 root
├─TableReader_25 10000.00 root data:TableFullScan_24
│ └─TableFullScan_24 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
└─IndexReader_32 10000.00 root index:IndexFullScan_31
└─IndexFullScan_31 10000.00 cop[tikv] table:t2, index:c1(c1) keep order:false, stats:pseudo
explain with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 100 offset 100) select * from cte1;
id estRows task access object operator info
CTEFullScan_34 100.00 root CTE:cte1 data:CTE_0
CTE_0 100.00 root Non-Recursive CTE
└─Limit_20(Seed Part) 100.00 root offset:100, count:100
└─HashAgg_21 200.00 root group by:Column#11, funcs:firstrow(Column#11)->Column#11
└─Union_22 20000.00 root
├─TableReader_25 10000.00 root data:TableFullScan_24
│ └─TableFullScan_24 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
└─IndexReader_32 10000.00 root index:IndexFullScan_31
└─IndexFullScan_31 10000.00 cop[tikv] table:t2, index:c1(c1) keep order:false, stats:pseudo
explain with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 0 offset 0) select * from cte1;
id estRows task access object operator info
CTEFullScan_18 0.00 root CTE:cte1 data:CTE_0
CTE_0 0.00 root Non-Recursive CTE
└─TableDual_16(Seed Part) 0.00 root rows:0
CREATE TABLE `customer` (
`c_customer_sk` int(11) NOT NULL,
`c_customer_id` char(16) NOT NULL,
Expand Down
1 change: 0 additions & 1 deletion cmd/explaintest/t/cte.test
View file
Original file line number Diff line number Diff line change
Expand Up @@ -336,4 +336,3 @@ INSERT INTO `t_dnmxh` VALUES (104,571000,NULL),(104,572000,44.37),(104,573000,59
WITH cte_0 AS (select distinct ref_0.wkey as c0, ref_0.pkey as c1, ref_0.c_xhsndb as c2 from t_dnmxh as ref_0 where (1 <= ( select ref_1.pkey not in ( select ref_5.wkey as c0 from t_dnmxh as ref_5 where (ref_5.wkey < ( select ref_6.pkey as c0 from t_cqmg3b as ref_6 where 88 between 96 and 76)) ) as c0 from (t_cqmg3b as ref_1 left outer join t_dnmxh as ref_2 on (ref_1.wkey = ref_2.wkey )) where ref_0.c_xhsndb is NULL union select 33 <= 91 as c0 from t_cqmg3b as ref_8 ))), cte_1 AS (select ref_9.wkey as c0, ref_9.pkey as c1, ref_9.c_anpf_c as c2, ref_9.c_b_fp_c as c3, ref_9.c_ndccfb as c4, ref_9.c_8rswc as c5 from t_cqmg3b as ref_9) select count(1) from cte_0 as ref_10 where case when 56 < 50 then case when 100 in ( select distinct ref_11.c4 as c0 from cte_1 as ref_11 where (ref_11.c4 > ( select ref_13.pkey as c0 from t_dnmxh as ref_13 where (ref_13.wkey > ( select distinct ref_11.c1 as c0 from cte_0 as ref_14)) )) or (1 = 1)) then null else null end else '7mxv6' end not like 'ki4%vc';
#case
with cte1 as (select 1), cte2 as (select 2) select * from cte1 union (with cte2 as (select 3) select * from cte2 union all select * from cte2) order by 1;
explain with cte1 as (select 1), cte2 as (select 2) select * from cte1 union (with cte2 as (select 3) select * from cte2 union all select * from cte2);
5 changes: 0 additions & 5 deletions cmd/explaintest/t/explain_cte.test
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,6 @@ explain with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 c1 fro
explain with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 c1 from cte1 limit 1) select * from cte1 dt1 join cte1 dt2 on dt1.c1 = dt2.c1;
explain with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 c1 from cte1 limit 0 offset 0) select * from cte1 dt1 join cte1 dt2 on dt1.c1 = dt2.c1;

# non-recursive limit
explain with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 1) select * from cte1;
explain with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 100 offset 100) select * from cte1;
explain with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 0 offset 0) select * from cte1;

# TPC-DS Q11
CREATE TABLE `customer` (
`c_customer_sk` int(11) NOT NULL,
Expand Down
2 changes: 2 additions & 0 deletions planner/core/casetest/physical_plan_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1432,6 +1432,8 @@ func TestSingleConsumerCTE(t *testing.T) {
tk.MustExec("use test")
tk.MustExec("drop table if exists t;")
tk.MustExec("CREATE TABLE `t` (`a` int(11));")
tk.MustExec("create table t1 (c1 int primary key, c2 int, index c2 (c2));")
tk.MustExec("create table t2 (c1 int unique, c2 int);")
tk.MustExec("insert into t values (1), (5), (10), (15), (20), (30), (50);")

var (
Expand Down
75 changes: 5 additions & 70 deletions planner/core/casetest/testdata/flat_plan_suite_out.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,8 +212,8 @@
{
"Depth": 2,
"Label": 0,
"IsRoot": true,
"StoreType": 2,
"IsRoot": false,
"StoreType": 0,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": "│ │ ",
Expand All @@ -232,80 +232,15 @@
{
"Depth": 2,
"Label": 0,
"IsRoot": true,
"StoreType": 2,
"IsRoot": false,
"StoreType": 0,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": " │ ",
"IsLastChild": true
}
],
"CTEs": [
[
{
"Depth": 0,
"Label": 0,
"IsRoot": true,
"StoreType": 2,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": "",
"IsLastChild": true
},
{
"Depth": 1,
"Label": 3,
"IsRoot": true,
"StoreType": 2,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": "│ ",
"IsLastChild": true
},
{
"Depth": 2,
"Label": 0,
"IsRoot": false,
"StoreType": 0,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": " │ ",
"IsLastChild": true
}
],
[
{
"Depth": 0,
"Label": 0,
"IsRoot": true,
"StoreType": 2,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": "",
"IsLastChild": true
},
{
"Depth": 1,
"Label": 3,
"IsRoot": true,
"StoreType": 2,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": "│ ",
"IsLastChild": true
},
{
"Depth": 2,
"Label": 0,
"IsRoot": false,
"StoreType": 0,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": " │ ",
"IsLastChild": true
}
]
]
"CTEs": null
},
{
"SQL": "WITH RECURSIVE cte (n) AS( SELECT 1 UNION ALL SELECT n + 1 FROM cte WHERE n < 5)SELECT * FROM cte;",
Expand Down
23 changes: 10 additions & 13 deletions planner/core/casetest/testdata/integration_suite_out.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6593,24 +6593,21 @@
{
"SQL": "explain format = 'brief' select /*+ qb_name(qb_v8, v8), merge(@qb_v8) */ * from v8;",
"Plan": [
"HashAgg 16000.00 root group by:Column#21, funcs:firstrow(Column#21)->Column#21",
"HashAgg 16000.00 root group by:Column#41, funcs:firstrow(Column#41)->Column#41",
"└─Union 1000000010000.00 root ",
" ├─HashJoin 1000000000000.00 root CARTESIAN inner join",
" │ ├─TableReader(Build) 10000.00 root data:TableFullScan",
" │ │ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
" │ └─CTEFullScan(Probe) 100000000.00 root CTE:cte2 data:CTE_1",
" │ └─Projection(Probe) 100000000.00 root 1->Column#55",
" │ └─HashJoin 100000000.00 root CARTESIAN inner join",
" │ ├─Projection(Build) 10000.00 root 1->Column#54",
" │ │ └─IndexReader 10000.00 root index:IndexFullScan",
" │ │ └─IndexFullScan 10000.00 cop[tikv] table:t3, index:idx_a(a) keep order:false, stats:pseudo",
" │ └─Projection(Probe) 10000.00 root 1->Column#53",
" │ └─IndexReader 10000.00 root index:IndexFullScan",
" │ └─IndexFullScan 10000.00 cop[tikv] table:t2, index:idx_a(a) keep order:false, stats:pseudo",
" └─TableReader 10000.00 root data:TableFullScan",
" └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
"CTE_1 100000000.00 root Non-Recursive CTE",
"└─HashJoin(Seed Part) 100000000.00 root CARTESIAN inner join",
" ├─CTEFullScan(Build) 10000.00 root CTE:cte4 data:CTE_3",
" └─CTEFullScan(Probe) 10000.00 root CTE:cte3 data:CTE_2",
"CTE_3 10000.00 root Non-Recursive CTE",
"└─IndexReader(Seed Part) 10000.00 root index:IndexFullScan",
" └─IndexFullScan 10000.00 cop[tikv] table:t3, index:idx_a(a) keep order:false, stats:pseudo",
"CTE_2 10000.00 root Non-Recursive CTE",
"└─IndexReader(Seed Part) 10000.00 root index:IndexFullScan",
" └─IndexFullScan 10000.00 cop[tikv] table:t2, index:idx_a(a) keep order:false, stats:pseudo"
" └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo"
],
"Warn": null
},
Expand Down
9 changes: 8 additions & 1 deletion planner/core/casetest/testdata/plan_suite_in.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -796,6 +796,10 @@
{
"name": "TestSingleConsumerCTE",
"cases": [
"with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 1) select * from cte1; -- non-recursive limit, inline cte1",
"with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 100 offset 100) select * from cte1; -- non-recursive limit, inline cte1",
"with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 0 offset 0) select * from cte1; -- non-recursive limit, inline cte1",
"with cte1 as (select 1), cte2 as (select 2) select * from cte1 union (with cte2 as (select 3) select * from cte2 union all select * from cte2) -- inline cte1, not inline cte2",
"with base1 as (WITH RECURSIVE cte(a) AS (with tmp as (select 1 as a) SELECT a from tmp UNION SELECT a+1 FROM cte) SELECT * FROM cte) select * from base1; -- issue #43318",
"with cte as (select 1) select * from cte; -- inline cte",
"with cte1 as (select 1), cte2 as (select 2) select * from cte1 union select * from cte2; -- inline cte1, cte2",
Expand All @@ -810,7 +814,10 @@
"set tidb_opt_force_inline_cte=1; -- enable force inline CTE",
"with cte as (select 1) select * from cte union select * from cte; -- force inline cte while multi-consumer",
"set tidb_opt_force_inline_cte=0; -- disable force inline CTE",
"with cte as (select 1) select /*+ MERGE() */ * from cte union select * from cte; -- firstly inline cte, secondly cannot be inlined"
"with cte as (select 1) select /*+ MERGE() */ * from cte union select * from cte; -- firstly inline cte, secondly cannot be inlined",
"with a as (select 8 as id from dual),maxa as (select max(id) as max_id from a),b as (with recursive temp as (select 1 as lvl from dual union all select lvl+1 from temp, maxa where lvl < max_id)select * from temp) select * from b; -- issue #47711, maxa cannot be inlined because it contains agg and in the recursive part of cte temp",
"with a as (select count(*) from t1), b as (select 2 as bb from a), c as (with recursive tmp as (select 1 as res from t1 union all select res+1 from tmp,b where res+1 < bb) select * from tmp) select * from c; -- inline a, cannot be inline b because b indirectly contains agg and in the recursive part of cte tmp",
"with a as (select count(*) from t1), b as (select 2 as bb from a), c as (with recursive tmp as (select bb as res from b union all select res+1 from tmp where res +1 < 10) select * from tmp) select * from c; -- inline a, b, cannot be inline tmp, c"
]
},
{
Expand Down
Loading

0 comments on commit 30498cc

Please sign in to comment.