Skip to content

Commit

Permalink
planner, CTE: Fix default inline CTE which contains agg or window fun…
Browse files Browse the repository at this point in the history
…ction and refactor inline CTE strategy (#48437)

close #47711
  • Loading branch information
ti-chi-bot authored Nov 9, 2023
1 parent a0d7242 commit cf0a91c
Show file tree
Hide file tree
Showing 13 changed files with 208 additions and 148 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -212,8 +212,8 @@
{
"Depth": 2,
"Label": 0,
"IsRoot": true,
"StoreType": 2,
"IsRoot": false,
"StoreType": 0,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": "│ │ ",
Expand All @@ -232,80 +232,15 @@
{
"Depth": 2,
"Label": 0,
"IsRoot": true,
"StoreType": 2,
"IsRoot": false,
"StoreType": 0,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": "",
"IsLastChild": true
}
],
"CTEs": [
[
{
"Depth": 0,
"Label": 0,
"IsRoot": true,
"StoreType": 2,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": "",
"IsLastChild": true
},
{
"Depth": 1,
"Label": 3,
"IsRoot": true,
"StoreType": 2,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": "",
"IsLastChild": true
},
{
"Depth": 2,
"Label": 0,
"IsRoot": false,
"StoreType": 0,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": "",
"IsLastChild": true
}
],
[
{
"Depth": 0,
"Label": 0,
"IsRoot": true,
"StoreType": 2,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": "",
"IsLastChild": true
},
{
"Depth": 1,
"Label": 3,
"IsRoot": true,
"StoreType": 2,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": "",
"IsLastChild": true
},
{
"Depth": 2,
"Label": 0,
"IsRoot": false,
"StoreType": 0,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": "",
"IsLastChild": true
}
]
]
"CTEs": null
},
{
"SQL": "WITH RECURSIVE cte (n) AS( SELECT 1 UNION ALL SELECT n + 1 FROM cte WHERE n < 5)SELECT * FROM cte;",
Expand Down
23 changes: 10 additions & 13 deletions pkg/planner/core/casetest/hint/testdata/integration_suite_out.json
Original file line number Diff line number Diff line change
Expand Up @@ -798,24 +798,21 @@
{
"SQL": "explain format = 'brief' select /*+ qb_name(qb_v8, v8), merge(@qb_v8) */ * from v8;",
"Plan": [
"HashAgg 16000.00 root group by:Column#21, funcs:firstrow(Column#21)->Column#21",
"HashAgg 16000.00 root group by:Column#41, funcs:firstrow(Column#41)->Column#41",
"└─Union 1000000010000.00 root ",
" ├─HashJoin 1000000000000.00 root CARTESIAN inner join",
" │ ├─TableReader(Build) 10000.00 root data:TableFullScan",
" │ │ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
" │ └─CTEFullScan(Probe) 100000000.00 root CTE:cte2 data:CTE_1",
" │ └─Projection(Probe) 100000000.00 root 1->Column#55",
" │ └─HashJoin 100000000.00 root CARTESIAN inner join",
" │ ├─Projection(Build) 10000.00 root 1->Column#54",
" │ │ └─IndexReader 10000.00 root index:IndexFullScan",
" │ │ └─IndexFullScan 10000.00 cop[tikv] table:t3, index:idx_a(a) keep order:false, stats:pseudo",
" │ └─Projection(Probe) 10000.00 root 1->Column#53",
" │ └─IndexReader 10000.00 root index:IndexFullScan",
" │ └─IndexFullScan 10000.00 cop[tikv] table:t2, index:idx_a(a) keep order:false, stats:pseudo",
" └─TableReader 10000.00 root data:TableFullScan",
" └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
"CTE_1 100000000.00 root Non-Recursive CTE",
"└─HashJoin(Seed Part) 100000000.00 root CARTESIAN inner join",
" ├─CTEFullScan(Build) 10000.00 root CTE:cte4 data:CTE_3",
" └─CTEFullScan(Probe) 10000.00 root CTE:cte3 data:CTE_2",
"CTE_3 10000.00 root Non-Recursive CTE",
"└─IndexReader(Seed Part) 10000.00 root index:IndexFullScan",
" └─IndexFullScan 10000.00 cop[tikv] table:t3, index:idx_a(a) keep order:false, stats:pseudo",
"CTE_2 10000.00 root Non-Recursive CTE",
"└─IndexReader(Seed Part) 10000.00 root index:IndexFullScan",
" └─IndexFullScan 10000.00 cop[tikv] table:t2, index:idx_a(a) keep order:false, stats:pseudo"
" └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo"
],
"Warn": null
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1206,6 +1206,8 @@ func TestSingleConsumerCTE(t *testing.T) {
tk.MustExec("use test")
tk.MustExec("drop table if exists t;")
tk.MustExec("CREATE TABLE `t` (`a` int(11));")
tk.MustExec("create table t1 (c1 int primary key, c2 int, index c2 (c2));")
tk.MustExec("create table t2 (c1 int unique, c2 int);")
tk.MustExec("insert into t values (1), (5), (10), (15), (20), (30), (50);")

var (
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -550,6 +550,10 @@
{
"name": "TestSingleConsumerCTE",
"cases": [
"with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 1) select * from cte1; -- non-recursive limit, inline cte1",
"with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 100 offset 100) select * from cte1; -- non-recursive limit, inline cte1",
"with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 0 offset 0) select * from cte1; -- non-recursive limit, inline cte1",
"with cte1 as (select 1), cte2 as (select 2) select * from cte1 union (with cte2 as (select 3) select * from cte2 union all select * from cte2) -- inline cte1, not inline cte2",
"with base1 as (WITH RECURSIVE cte(a) AS (with tmp as (select 1 as a) SELECT a from tmp UNION SELECT a+1 FROM cte) SELECT * FROM cte) select * from base1; -- issue #43318",
"with cte as (select 1) select * from cte; -- inline cte",
"with cte1 as (select 1), cte2 as (select 2) select * from cte1 union select * from cte2; -- inline cte1, cte2",
Expand All @@ -564,7 +568,10 @@
"set tidb_opt_force_inline_cte=1; -- enable force inline CTE",
"with cte as (select 1) select * from cte union select * from cte; -- force inline cte while multi-consumer",
"set tidb_opt_force_inline_cte=0; -- disable force inline CTE",
"with cte as (select 1) select /*+ MERGE() */ * from cte union select * from cte; -- firstly inline cte, secondly cannot be inlined"
"with cte as (select 1) select /*+ MERGE() */ * from cte union select * from cte; -- firstly inline cte, secondly cannot be inlined",
"with a as (select 8 as id from dual),maxa as (select max(id) as max_id from a),b as (with recursive temp as (select 1 as lvl from dual union all select lvl+1 from temp, maxa where lvl < max_id)select * from temp) select * from b; -- issue #47711, maxa cannot be inlined because it contains agg and in the recursive part of cte temp",
"with a as (select count(*) from t1), b as (select 2 as bb from a), c as (with recursive tmp as (select 1 as res from t1 union all select res+1 from tmp,b where res+1 < bb) select * from tmp) select * from c; -- inline a, cannot be inline b because b indirectly contains agg and in the recursive part of cte tmp",
"with a as (select count(*) from t1), b as (select 2 as bb from a), c as (with recursive tmp as (select bb as res from b union all select res+1 from tmp where res +1 < 10) select * from tmp) select * from c; -- inline a, b, cannot be inline tmp, c"
]
},
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2774,7 +2774,7 @@
" └─CTETable 1.00 root Scan on CTE_0"
],
"Warning": [
"[planner:1815]Recursive CTE can not be inlined."
"[planner:1815]Recursive CTE cte1 can not be inlined by merge() or tidb_opt_force_inline_cte."
]
},
{
Expand Down Expand Up @@ -3023,7 +3023,9 @@
" └─Selection 0.80 root lt(Column#3, 100)",
" └─CTETable 1.00 root Scan on CTE_0"
],
"Warning": null
"Warning": [
"[planner:1815]Recursive CTE cte1 can not be inlined by merge() or tidb_opt_force_inline_cte."
]
},
{
"SQL": "with cte1 as (with cte2 as (select * from t) select * from cte2) select * from cte1; -- non-recursive 'cte2' definition inside another non-recursive 'cte1'",
Expand All @@ -3044,7 +3046,9 @@
" └─Selection 8000.00 root lt(test.t.a, 100)",
" └─CTETable 10000.00 root Scan on CTE_0"
],
"Warning": null
"Warning": [
"[planner:1815]Recursive CTE cte2 can not be inlined by merge() or tidb_opt_force_inline_cte."
]
},
{
"SQL": "with cte1 as (with recursive cte2(c1) as (select 1 union select c1 + 1 c1 from cte2 where c1 < 100) select * from cte2) select * from cte1; -- recursive 'cte2' inside non-recursive 'cte1'",
Expand All @@ -3057,7 +3061,10 @@
" └─Selection 0.80 root lt(Column#10, 100)",
" └─CTETable 1.00 root Scan on CTE_2"
],
"Warning": null
"Warning": [
"[planner:1815]Recursive CTE cte2 can not be inlined by merge() or tidb_opt_force_inline_cte.",
"[planner:1815]Recursive CTE cte2 can not be inlined by merge() or tidb_opt_force_inline_cte."
]
},
{
"SQL": "set tidb_opt_force_inline_cte=0; -- disable force inline CTE",
Expand Down Expand Up @@ -3092,7 +3099,7 @@
" └─CTETable 1.00 root Scan on CTE_0"
],
"Warning": [
"[planner:1815]Recursive CTE can not be inlined."
"[planner:1815]Recursive CTE cte1 can not be inlined by merge() or tidb_opt_force_inline_cte."
]
},
{
Expand Down Expand Up @@ -3134,6 +3141,55 @@
{
"Name": "TestSingleConsumerCTE",
"Cases": [
{
"SQL": "with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 1) select * from cte1; -- non-recursive limit, inline cte1",
"Plan": [
"Limit 1.00 root offset:0, count:1",
"└─HashAgg 1.00 root group by:Column#18, funcs:firstrow(Column#18)->Column#18",
" └─Union 20000.00 root ",
" ├─TableReader 10000.00 root data:TableFullScan",
" │ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
" └─IndexReader 10000.00 root index:IndexFullScan",
" └─IndexFullScan 10000.00 cop[tikv] table:t2, index:c1(c1) keep order:false, stats:pseudo"
],
"Warning": null
},
{
"SQL": "with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 100 offset 100) select * from cte1; -- non-recursive limit, inline cte1",
"Plan": [
"Limit 100.00 root offset:100, count:100",
"└─HashAgg 200.00 root group by:Column#18, funcs:firstrow(Column#18)->Column#18",
" └─Union 20000.00 root ",
" ├─TableReader 10000.00 root data:TableFullScan",
" │ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
" └─IndexReader 10000.00 root index:IndexFullScan",
" └─IndexFullScan 10000.00 cop[tikv] table:t2, index:c1(c1) keep order:false, stats:pseudo"
],
"Warning": null
},
{
"SQL": "with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 0 offset 0) select * from cte1; -- non-recursive limit, inline cte1",
"Plan": [
"TableDual 0.00 root rows:0"
],
"Warning": null
},
{
"SQL": "with cte1 as (select 1), cte2 as (select 2) select * from cte1 union (with cte2 as (select 3) select * from cte2 union all select * from cte2) -- inline cte1, not inline cte2",
"Plan": [
"HashAgg 3.00 root group by:Column#9, funcs:firstrow(Column#9)->Column#9",
"└─Union 3.00 root ",
" ├─Projection 1.00 root 1->Column#9",
" │ └─TableDual 1.00 root rows:1",
" └─Union 2.00 root ",
" ├─CTEFullScan 1.00 root CTE:cte2 data:CTE_2",
" └─CTEFullScan 1.00 root CTE:cte2 data:CTE_2",
"CTE_2 1.00 root Non-Recursive CTE",
"└─Projection(Seed Part) 1.00 root 3->Column#5",
" └─TableDual 1.00 root rows:1"
],
"Warning": null
},
{
"SQL": "with base1 as (WITH RECURSIVE cte(a) AS (with tmp as (select 1 as a) SELECT a from tmp UNION SELECT a+1 FROM cte) SELECT * FROM cte) select * from base1; -- issue #43318",
"Plan": [
Expand Down Expand Up @@ -3287,6 +3343,63 @@
" └─TableDual 1.00 root rows:1"
],
"Warning": null
},
{
"SQL": "with a as (select 8 as id from dual),maxa as (select max(id) as max_id from a),b as (with recursive temp as (select 1 as lvl from dual union all select lvl+1 from temp, maxa where lvl < max_id)select * from temp) select * from b; -- issue #47711, maxa cannot be inlined because it contains agg and in the recursive part of cte temp",
"Plan": [
"CTEFullScan 1.64 root CTE:temp data:CTE_4",
"CTE_4 1.64 root Recursive CTE",
"├─Projection(Seed Part) 1.00 root 1->Column#14",
"│ └─TableDual 1.00 root rows:1",
"└─Projection(Recursive Part) 0.64 root cast(plus(Column#15, 1), bigint(1) BINARY)->Column#18",
" └─HashJoin 0.64 root CARTESIAN inner join, other cond:lt(Column#15, Column#16)",
" ├─Selection(Build) 0.80 root not(isnull(Column#16))",
" │ └─CTEFullScan 1.00 root CTE:maxa data:CTE_1",
" └─Selection(Probe) 0.80 root not(isnull(Column#15))",
" └─CTETable 1.00 root Scan on CTE_4",
"CTE_1 1.00 root Non-Recursive CTE",
"└─StreamAgg(Seed Part) 1.00 root funcs:max(Column#3)->Column#4",
" └─Projection 1.00 root 8->Column#3",
" └─TableDual 1.00 root rows:1"
],
"Warning": null
},
{
"SQL": "with a as (select count(*) from t1), b as (select 2 as bb from a), c as (with recursive tmp as (select 1 as res from t1 union all select res+1 from tmp,b where res+1 < bb) select * from tmp) select * from c; -- inline a, cannot be inline b because b indirectly contains agg and in the recursive part of cte tmp",
"Plan": [
"CTEFullScan 20000.00 root CTE:tmp data:CTE_4",
"CTE_4 20000.00 root Recursive CTE",
"├─Projection(Seed Part) 10000.00 root 1->Column#26",
"│ └─TableReader 10000.00 root data:TableFullScan",
"│ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
"└─Projection(Recursive Part) 10000.00 root cast(plus(Column#27, 1), bigint(1) BINARY)->Column#30",
" └─HashJoin 10000.00 root CARTESIAN inner join, other cond:lt(plus(Column#27, 1), Column#28)",
" ├─CTEFullScan(Build) 1.00 root CTE:b data:CTE_1",
" └─CTETable(Probe) 10000.00 root Scan on CTE_4",
"CTE_1 1.00 root Non-Recursive CTE",
"└─Projection(Seed Part) 1.00 root 2->Column#8",
" └─HashAgg 1.00 root funcs:count(Column#37)->Column#35",
" └─TableReader 1.00 root data:HashAgg",
" └─HashAgg 1.00 cop[tikv] funcs:count(1)->Column#37",
" └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo"
],
"Warning": null
},
{
"SQL": "with a as (select count(*) from t1), b as (select 2 as bb from a), c as (with recursive tmp as (select bb as res from b union all select res+1 from tmp where res +1 < 10) select * from tmp) select * from c; -- inline a, b, cannot be inline tmp, c",
"Plan": [
"CTEFullScan 1.80 root CTE:tmp data:CTE_4",
"CTE_4 1.80 root Recursive CTE",
"├─Projection(Seed Part) 1.00 root 2->Column#37",
"│ └─HashAgg 1.00 root funcs:count(Column#46)->Column#44",
"│ └─TableReader 1.00 root data:HashAgg",
"│ └─HashAgg 1.00 cop[tikv] funcs:count(1)->Column#46",
"│ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
"└─Projection(Recursive Part) 0.80 root cast(plus(Column#38, 1), bigint(1) BINARY)->Column#40",
" └─Selection 0.80 root lt(plus(Column#38, 1), 10)",
" └─CTETable 1.00 root Scan on CTE_4"
],
"Warning": null
}
]
},
Expand Down
8 changes: 4 additions & 4 deletions pkg/planner/core/casetest/planstats/plan_stats_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -166,13 +166,13 @@ func TestPlanStatsLoad(t *testing.T) {
{ // CTE
sql: "with cte(x, y) as (select d + 1, b from t where c > 1) select * from cte where x < 3",
check: func(p plannercore.Plan, tableInfo *model.TableInfo) {
ps, ok := p.(*plannercore.PhysicalSelection)
ps, ok := p.(*plannercore.PhysicalProjection)
require.True(t, ok)
pc, ok := ps.Children()[0].(*plannercore.PhysicalCTE)
pc, ok := ps.Children()[0].(*plannercore.PhysicalTableReader)
require.True(t, ok)
pp, ok := pc.SeedPlan.(*plannercore.PhysicalProjection)
pp, ok := pc.GetTablePlan().(*plannercore.PhysicalSelection)
require.True(t, ok)
reader, ok := pp.Children()[0].(*plannercore.PhysicalTableReader)
reader, ok := pp.Children()[0].(*plannercore.PhysicalTableScan)
require.True(t, ok)
require.Greater(t, countFullStats(reader.StatsInfo().HistColl, tableInfo.Columns[2].ID), 0)
},
Expand Down
Loading

0 comments on commit cf0a91c

Please sign in to comment.