Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

planner, CTE: Fix default inline CTE which contains agg or window function and refactor inline CTE strategy | tidb-test=pr/2240 #48437

Merged
merged 8 commits into from
Nov 9, 2023
Merged

planner, CTE: Fix default inline CTE which contains agg or window function and refactor inline CTE strategy | tidb-test=pr/2240 #48437

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
472b215
planner, cte: Fix default inline cte which contains agg or window fun…
elsa0520 Nov 1, 2023
5ac2763
fix
elsa0520 Nov 7, 2023
e9c2583
the cte indirectly contains agg also cannot be inline in recursive part
elsa0520 Nov 7, 2023
d92b2b1
fix ut
elsa0520 Nov 7, 2023
f587517
fix ut
elsa0520 Nov 7, 2023
db24d66
fix ut
elsa0520 Nov 8, 2023
e1e4117
fix ut
elsa0520 Nov 8, 2023
fcf348e
use compute inline function instead
elsa0520 Nov 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 5 additions & 70 deletions pkg/planner/core/casetest/flatplan/testdata/flat_plan_suite_out.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,8 +212,8 @@
{
"Depth": 2,
"Label": 0,
"IsRoot": true,
"StoreType": 2,
"IsRoot": false,
"StoreType": 0,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": "│ │ ",
Expand All @@ -232,80 +232,15 @@
{
"Depth": 2,
"Label": 0,
"IsRoot": true,
"StoreType": 2,
"IsRoot": false,
"StoreType": 0,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": " │ ",
"IsLastChild": true
}
],
"CTEs": [
[
{
"Depth": 0,
"Label": 0,
"IsRoot": true,
"StoreType": 2,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": "",
"IsLastChild": true
},
{
"Depth": 1,
"Label": 3,
"IsRoot": true,
"StoreType": 2,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": "│ ",
"IsLastChild": true
},
{
"Depth": 2,
"Label": 0,
"IsRoot": false,
"StoreType": 0,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": " │ ",
"IsLastChild": true
}
],
[
{
"Depth": 0,
"Label": 0,
"IsRoot": true,
"StoreType": 2,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": "",
"IsLastChild": true
},
{
"Depth": 1,
"Label": 3,
"IsRoot": true,
"StoreType": 2,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": "│ ",
"IsLastChild": true
},
{
"Depth": 2,
"Label": 0,
"IsRoot": false,
"StoreType": 0,
"ReqType": 0,
"IsPhysicalPlan": true,
"TextTreeIndent": " │ ",
"IsLastChild": true
}
]
]
"CTEs": null
},
{
"SQL": "WITH RECURSIVE cte (n) AS( SELECT 1 UNION ALL SELECT n + 1 FROM cte WHERE n < 5)SELECT * FROM cte;",
Expand Down
23 changes: 10 additions & 13 deletions pkg/planner/core/casetest/hint/testdata/integration_suite_out.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -798,24 +798,21 @@
{
"SQL": "explain format = 'brief' select /*+ qb_name(qb_v8, v8), merge(@qb_v8) */ * from v8;",
"Plan": [
"HashAgg 16000.00 root group by:Column#21, funcs:firstrow(Column#21)->Column#21",
"HashAgg 16000.00 root group by:Column#41, funcs:firstrow(Column#41)->Column#41",
"└─Union 1000000010000.00 root ",
" ├─HashJoin 1000000000000.00 root CARTESIAN inner join",
" │ ├─TableReader(Build) 10000.00 root data:TableFullScan",
" │ │ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
" │ └─CTEFullScan(Probe) 100000000.00 root CTE:cte2 data:CTE_1",
" │ └─Projection(Probe) 100000000.00 root 1->Column#55",
" │ └─HashJoin 100000000.00 root CARTESIAN inner join",
" │ ├─Projection(Build) 10000.00 root 1->Column#54",
" │ │ └─IndexReader 10000.00 root index:IndexFullScan",
" │ │ └─IndexFullScan 10000.00 cop[tikv] table:t3, index:idx_a(a) keep order:false, stats:pseudo",
" │ └─Projection(Probe) 10000.00 root 1->Column#53",
" │ └─IndexReader 10000.00 root index:IndexFullScan",
" │ └─IndexFullScan 10000.00 cop[tikv] table:t2, index:idx_a(a) keep order:false, stats:pseudo",
" └─TableReader 10000.00 root data:TableFullScan",
" └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
"CTE_1 100000000.00 root Non-Recursive CTE",
"└─HashJoin(Seed Part) 100000000.00 root CARTESIAN inner join",
" ├─CTEFullScan(Build) 10000.00 root CTE:cte4 data:CTE_3",
" └─CTEFullScan(Probe) 10000.00 root CTE:cte3 data:CTE_2",
"CTE_3 10000.00 root Non-Recursive CTE",
"└─IndexReader(Seed Part) 10000.00 root index:IndexFullScan",
" └─IndexFullScan 10000.00 cop[tikv] table:t3, index:idx_a(a) keep order:false, stats:pseudo",
"CTE_2 10000.00 root Non-Recursive CTE",
"└─IndexReader(Seed Part) 10000.00 root index:IndexFullScan",
" └─IndexFullScan 10000.00 cop[tikv] table:t2, index:idx_a(a) keep order:false, stats:pseudo"
" └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo"
],
"Warn": null
},
Expand Down
2 changes: 2 additions & 0 deletions pkg/planner/core/casetest/physicalplantest/physical_plan_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1206,6 +1206,8 @@ func TestSingleConsumerCTE(t *testing.T) {
tk.MustExec("use test")
tk.MustExec("drop table if exists t;")
tk.MustExec("CREATE TABLE `t` (`a` int(11));")
tk.MustExec("create table t1 (c1 int primary key, c2 int, index c2 (c2));")
tk.MustExec("create table t2 (c1 int unique, c2 int);")
tk.MustExec("insert into t values (1), (5), (10), (15), (20), (30), (50);")

var (
Expand Down
9 changes: 8 additions & 1 deletion pkg/planner/core/casetest/physicalplantest/testdata/plan_suite_in.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -550,6 +550,10 @@
{
"name": "TestSingleConsumerCTE",
"cases": [
"with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 1) select * from cte1; -- non-recursive limit, inline cte1",
"with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 100 offset 100) select * from cte1; -- non-recursive limit, inline cte1",
"with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 0 offset 0) select * from cte1; -- non-recursive limit, inline cte1",
"with cte1 as (select 1), cte2 as (select 2) select * from cte1 union (with cte2 as (select 3) select * from cte2 union all select * from cte2) -- inline cte1, not inline cte2",
"with base1 as (WITH RECURSIVE cte(a) AS (with tmp as (select 1 as a) SELECT a from tmp UNION SELECT a+1 FROM cte) SELECT * FROM cte) select * from base1; -- issue #43318",
"with cte as (select 1) select * from cte; -- inline cte",
"with cte1 as (select 1), cte2 as (select 2) select * from cte1 union select * from cte2; -- inline cte1, cte2",
Expand All @@ -564,7 +568,10 @@
"set tidb_opt_force_inline_cte=1; -- enable force inline CTE",
"with cte as (select 1) select * from cte union select * from cte; -- force inline cte while multi-consumer",
"set tidb_opt_force_inline_cte=0; -- disable force inline CTE",
"with cte as (select 1) select /*+ MERGE() */ * from cte union select * from cte; -- firstly inline cte, secondly cannot be inlined"
"with cte as (select 1) select /*+ MERGE() */ * from cte union select * from cte; -- firstly inline cte, secondly cannot be inlined",
"with a as (select 8 as id from dual),maxa as (select max(id) as max_id from a),b as (with recursive temp as (select 1 as lvl from dual union all select lvl+1 from temp, maxa where lvl < max_id)select * from temp) select * from b; -- issue #47711, maxa cannot be inlined because it contains agg and in the recursive part of cte temp",
"with a as (select count(*) from t1), b as (select 2 as bb from a), c as (with recursive tmp as (select 1 as res from t1 union all select res+1 from tmp,b where res+1 < bb) select * from tmp) select * from c; -- inline a, cannot be inline b because b indirectly contains agg and in the recursive part of cte tmp",
"with a as (select count(*) from t1), b as (select 2 as bb from a), c as (with recursive tmp as (select bb as res from b union all select res+1 from tmp where res +1 < 10) select * from tmp) select * from c; -- inline a, b, cannot be inline tmp, c"
]
},
{
Expand Down
123 changes: 118 additions & 5 deletions pkg/planner/core/casetest/physicalplantest/testdata/plan_suite_out.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2774,7 +2774,7 @@
" └─CTETable 1.00 root Scan on CTE_0"
],
"Warning": [
"[planner:1815]Recursive CTE can not be inlined."
"[planner:1815]Recursive CTE cte1 can not be inlined by merge() or tidb_opt_force_inline_cte."
]
},
{
Expand Down Expand Up @@ -3023,7 +3023,9 @@
" └─Selection 0.80 root lt(Column#3, 100)",
" └─CTETable 1.00 root Scan on CTE_0"
],
"Warning": null
"Warning": [
"[planner:1815]Recursive CTE cte1 can not be inlined by merge() or tidb_opt_force_inline_cte."
]
},
{
"SQL": "with cte1 as (with cte2 as (select * from t) select * from cte2) select * from cte1; -- non-recursive 'cte2' definition inside another non-recursive 'cte1'",
Expand All @@ -3044,7 +3046,9 @@
" └─Selection 8000.00 root lt(test.t.a, 100)",
" └─CTETable 10000.00 root Scan on CTE_0"
],
"Warning": null
"Warning": [
"[planner:1815]Recursive CTE cte2 can not be inlined by merge() or tidb_opt_force_inline_cte."
]
},
{
"SQL": "with cte1 as (with recursive cte2(c1) as (select 1 union select c1 + 1 c1 from cte2 where c1 < 100) select * from cte2) select * from cte1; -- recursive 'cte2' inside non-recursive 'cte1'",
Expand All @@ -3057,7 +3061,10 @@
" └─Selection 0.80 root lt(Column#10, 100)",
" └─CTETable 1.00 root Scan on CTE_2"
],
"Warning": null
"Warning": [
"[planner:1815]Recursive CTE cte2 can not be inlined by merge() or tidb_opt_force_inline_cte.",
"[planner:1815]Recursive CTE cte2 can not be inlined by merge() or tidb_opt_force_inline_cte."
]
},
{
"SQL": "set tidb_opt_force_inline_cte=0; -- disable force inline CTE",
Expand Down Expand Up @@ -3092,7 +3099,7 @@
" └─CTETable 1.00 root Scan on CTE_0"
],
"Warning": [
"[planner:1815]Recursive CTE can not be inlined."
"[planner:1815]Recursive CTE cte1 can not be inlined by merge() or tidb_opt_force_inline_cte."
]
},
{
Expand Down Expand Up @@ -3134,6 +3141,55 @@
{
"Name": "TestSingleConsumerCTE",
"Cases": [
{
"SQL": "with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 1) select * from cte1; -- non-recursive limit, inline cte1",
"Plan": [
"Limit 1.00 root offset:0, count:1",
"└─HashAgg 1.00 root group by:Column#18, funcs:firstrow(Column#18)->Column#18",
" └─Union 20000.00 root ",
" ├─TableReader 10000.00 root data:TableFullScan",
" │ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
" └─IndexReader 10000.00 root index:IndexFullScan",
" └─IndexFullScan 10000.00 cop[tikv] table:t2, index:c1(c1) keep order:false, stats:pseudo"
],
"Warning": null
},
{
"SQL": "with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 100 offset 100) select * from cte1; -- non-recursive limit, inline cte1",
"Plan": [
"Limit 100.00 root offset:100, count:100",
"└─HashAgg 200.00 root group by:Column#18, funcs:firstrow(Column#18)->Column#18",
" └─Union 20000.00 root ",
" ├─TableReader 10000.00 root data:TableFullScan",
" │ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
" └─IndexReader 10000.00 root index:IndexFullScan",
" └─IndexFullScan 10000.00 cop[tikv] table:t2, index:c1(c1) keep order:false, stats:pseudo"
],
"Warning": null
},
{
"SQL": "with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 0 offset 0) select * from cte1; -- non-recursive limit, inline cte1",
"Plan": [
"TableDual 0.00 root rows:0"
],
"Warning": null
},
{
"SQL": "with cte1 as (select 1), cte2 as (select 2) select * from cte1 union (with cte2 as (select 3) select * from cte2 union all select * from cte2) -- inline cte1, not inline cte2",
"Plan": [
"HashAgg 3.00 root group by:Column#9, funcs:firstrow(Column#9)->Column#9",
"└─Union 3.00 root ",
" ├─Projection 1.00 root 1->Column#9",
" │ └─TableDual 1.00 root rows:1",
" └─Union 2.00 root ",
" ├─CTEFullScan 1.00 root CTE:cte2 data:CTE_2",
" └─CTEFullScan 1.00 root CTE:cte2 data:CTE_2",
"CTE_2 1.00 root Non-Recursive CTE",
"└─Projection(Seed Part) 1.00 root 3->Column#5",
" └─TableDual 1.00 root rows:1"
],
"Warning": null
},
{
"SQL": "with base1 as (WITH RECURSIVE cte(a) AS (with tmp as (select 1 as a) SELECT a from tmp UNION SELECT a+1 FROM cte) SELECT * FROM cte) select * from base1; -- issue #43318",
"Plan": [
Expand Down Expand Up @@ -3287,6 +3343,63 @@
" └─TableDual 1.00 root rows:1"
],
"Warning": null
},
{
"SQL": "with a as (select 8 as id from dual),maxa as (select max(id) as max_id from a),b as (with recursive temp as (select 1 as lvl from dual union all select lvl+1 from temp, maxa where lvl < max_id)select * from temp) select * from b; -- issue #47711, maxa cannot be inlined because it contains agg and in the recursive part of cte temp",
"Plan": [
"CTEFullScan 1.64 root CTE:temp data:CTE_4",
"CTE_4 1.64 root Recursive CTE",
"├─Projection(Seed Part) 1.00 root 1->Column#14",
"│ └─TableDual 1.00 root rows:1",
"└─Projection(Recursive Part) 0.64 root cast(plus(Column#15, 1), bigint(1) BINARY)->Column#18",
" └─HashJoin 0.64 root CARTESIAN inner join, other cond:lt(Column#15, Column#16)",
" ├─Selection(Build) 0.80 root not(isnull(Column#16))",
" │ └─CTEFullScan 1.00 root CTE:maxa data:CTE_1",
" └─Selection(Probe) 0.80 root not(isnull(Column#15))",
" └─CTETable 1.00 root Scan on CTE_4",
"CTE_1 1.00 root Non-Recursive CTE",
"└─StreamAgg(Seed Part) 1.00 root funcs:max(Column#3)->Column#4",
" └─Projection 1.00 root 8->Column#3",
" └─TableDual 1.00 root rows:1"
],
"Warning": null
},
{
"SQL": "with a as (select count(*) from t1), b as (select 2 as bb from a), c as (with recursive tmp as (select 1 as res from t1 union all select res+1 from tmp,b where res+1 < bb) select * from tmp) select * from c; -- inline a, cannot be inline b because b indirectly contains agg and in the recursive part of cte tmp",
"Plan": [
"CTEFullScan 20000.00 root CTE:tmp data:CTE_4",
"CTE_4 20000.00 root Recursive CTE",
"├─Projection(Seed Part) 10000.00 root 1->Column#26",
"│ └─TableReader 10000.00 root data:TableFullScan",
"│ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
"└─Projection(Recursive Part) 10000.00 root cast(plus(Column#27, 1), bigint(1) BINARY)->Column#30",
" └─HashJoin 10000.00 root CARTESIAN inner join, other cond:lt(plus(Column#27, 1), Column#28)",
" ├─CTEFullScan(Build) 1.00 root CTE:b data:CTE_1",
" └─CTETable(Probe) 10000.00 root Scan on CTE_4",
"CTE_1 1.00 root Non-Recursive CTE",
"└─Projection(Seed Part) 1.00 root 2->Column#8",
" └─HashAgg 1.00 root funcs:count(Column#37)->Column#35",
" └─TableReader 1.00 root data:HashAgg",
" └─HashAgg 1.00 cop[tikv] funcs:count(1)->Column#37",
" └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo"
],
"Warning": null
},
{
"SQL": "with a as (select count(*) from t1), b as (select 2 as bb from a), c as (with recursive tmp as (select bb as res from b union all select res+1 from tmp where res +1 < 10) select * from tmp) select * from c; -- inline a, b, cannot be inline tmp, c",
"Plan": [
"CTEFullScan 1.80 root CTE:tmp data:CTE_4",
"CTE_4 1.80 root Recursive CTE",
"├─Projection(Seed Part) 1.00 root 2->Column#37",
"│ └─HashAgg 1.00 root funcs:count(Column#46)->Column#44",
"│ └─TableReader 1.00 root data:HashAgg",
"│ └─HashAgg 1.00 cop[tikv] funcs:count(1)->Column#46",
"│ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
"└─Projection(Recursive Part) 0.80 root cast(plus(Column#38, 1), bigint(1) BINARY)->Column#40",
" └─Selection 0.80 root lt(plus(Column#38, 1), 10)",
" └─CTETable 1.00 root Scan on CTE_4"
],
"Warning": null
}
]
},
Expand Down
8 changes: 4 additions & 4 deletions pkg/planner/core/casetest/planstats/plan_stats_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,13 +166,13 @@ func TestPlanStatsLoad(t *testing.T) {
{ // CTE
sql: "with cte(x, y) as (select d + 1, b from t where c > 1) select * from cte where x < 3",
check: func(p plannercore.Plan, tableInfo *model.TableInfo) {
ps, ok := p.(*plannercore.PhysicalSelection)
ps, ok := p.(*plannercore.PhysicalProjection)
require.True(t, ok)
pc, ok := ps.Children()[0].(*plannercore.PhysicalCTE)
pc, ok := ps.Children()[0].(*plannercore.PhysicalTableReader)
require.True(t, ok)
pp, ok := pc.SeedPlan.(*plannercore.PhysicalProjection)
pp, ok := pc.GetTablePlan().(*plannercore.PhysicalSelection)
require.True(t, ok)
reader, ok := pp.Children()[0].(*plannercore.PhysicalTableReader)
reader, ok := pp.Children()[0].(*plannercore.PhysicalTableScan)
require.True(t, ok)
require.Greater(t, countFullStats(reader.StatsInfo().HistColl, tableInfo.Columns[2].ID), 0)
},
Expand Down
Loading