Skip to content

Commit

Permalink
planner: fix possible inconsistent output cols among union's children (…
Browse files Browse the repository at this point in the history 
…#48775)

close #48755
  • Loading branch information
@winoros
winoros authored Nov 24, 2023
1 parent 1263445 commit 8583ab5
Show file tree
Hide file tree
Showing 6 changed files with 202 additions and 93 deletions.
49 changes: 27 additions & 22 deletions pkg/executor/test/indexmergereadtest/index_merge_reader_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,22 +232,26 @@ func TestPessimisticLockOnPartitionForIndexMerge(t *testing.T) {
" ├─IndexReader(Build) 3.00 root index:IndexFullScan",
" │ └─IndexFullScan 3.00 cop[tikv] table:t2, index:c_datetime(c_datetime) keep order:false",
" └─PartitionUnion(Probe) 5545.21 root ",
" ├─IndexMerge 5542.21 root type: union",
" │ ├─IndexRangeScan(Build) 3323.33 cop[tikv] table:t1, partition:p0, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo",
" │ ├─IndexRangeScan(Build) 3323.33 cop[tikv] table:t1, partition:p0, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo",
" │ └─TableRowIDScan(Probe) 5542.21 cop[tikv] table:t1, partition:p0 keep order:false, stats:pseudo",
" ├─IndexMerge 1.00 root type: union",
" │ ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p1, index:c1(c1) range:[-inf,10), keep order:false",
" │ ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p1, index:c2(c2) range:[-inf,10), keep order:false",
" │ └─TableRowIDScan(Probe) 1.00 cop[tikv] table:t1, partition:p1 keep order:false",
" ├─IndexMerge 1.00 root type: union",
" │ ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p2, index:c1(c1) range:[-inf,10), keep order:false",
" │ ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p2, index:c2(c2) range:[-inf,10), keep order:false",
" │ └─TableRowIDScan(Probe) 1.00 cop[tikv] table:t1, partition:p2 keep order:false",
" └─IndexMerge 1.00 root type: union",
" ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p3, index:c1(c1) range:[-inf,10), keep order:false",
" ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p3, index:c2(c2) range:[-inf,10), keep order:false",
" └─TableRowIDScan(Probe) 1.00 cop[tikv] table:t1, partition:p3 keep order:false",
" ├─Projection 5542.21 root test.t1.c_datetime, test.t1.c1, test.t1._tidb_rowid, test.t1._tidb_tid",
" │ └─IndexMerge 5542.21 root type: union",
" │ ├─IndexRangeScan(Build) 3323.33 cop[tikv] table:t1, partition:p0, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo",
" │ ├─IndexRangeScan(Build) 3323.33 cop[tikv] table:t1, partition:p0, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo",
" │ └─TableRowIDScan(Probe) 5542.21 cop[tikv] table:t1, partition:p0 keep order:false, stats:pseudo",
" ├─Projection 1.00 root test.t1.c_datetime, test.t1.c1, test.t1._tidb_rowid, test.t1._tidb_tid",
" │ └─IndexMerge 1.00 root type: union",
" │ ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p1, index:c1(c1) range:[-inf,10), keep order:false",
" │ ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p1, index:c2(c2) range:[-inf,10), keep order:false",
" │ └─TableRowIDScan(Probe) 1.00 cop[tikv] table:t1, partition:p1 keep order:false",
" ├─Projection 1.00 root test.t1.c_datetime, test.t1.c1, test.t1._tidb_rowid, test.t1._tidb_tid",
" │ └─IndexMerge 1.00 root type: union",
" │ ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p2, index:c1(c1) range:[-inf,10), keep order:false",
" │ ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p2, index:c2(c2) range:[-inf,10), keep order:false",
" │ └─TableRowIDScan(Probe) 1.00 cop[tikv] table:t1, partition:p2 keep order:false",
" └─Projection 1.00 root test.t1.c_datetime, test.t1.c1, test.t1._tidb_rowid, test.t1._tidb_tid",
" └─IndexMerge 1.00 root type: union",
" ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p3, index:c1(c1) range:[-inf,10), keep order:false",
" ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p3, index:c2(c2) range:[-inf,10), keep order:false",
" └─TableRowIDScan(Probe) 1.00 cop[tikv] table:t1, partition:p3 keep order:false",
))
tk.MustQuery(`select /*+ use_index_merge(t1) */ c1 from t1 join t2
on t1.c_datetime >= t2.c_datetime
Expand Down Expand Up @@ -372,20 +376,21 @@ func TestIntersectionWithDifferentConcurrency(t *testing.T) {
for _, concurrency := range execCon {
tk.MustExec(fmt.Sprintf("set tidb_executor_concurrency = %d", concurrency))
for i := 0; i < 2; i++ {
sql := "select /*+ use_index_merge(t1, primary, c2, c3) */ c1 from t1 where c2 < 1024 and c3 > 1024"
if i == 0 {
// Dynamic mode.
tk.MustExec("set tidb_partition_prune_mode = 'dynamic'")
res := tk.MustQuery("explain select /*+ use_index_merge(t1, primary, c2, c3) */ c1 from t1 where c2 < 1024 and c3 > 1024")
require.Contains(t, res.Rows()[1][0], "IndexMerge")
tk.MustHavePlan(sql, "IndexMerge")
tk.MustNotHavePlan(sql, "PartitionUnion")
} else {
tk.MustExec("set tidb_partition_prune_mode = 'static'")
res := tk.MustQuery("explain select /*+ use_index_merge(t1, primary, c2, c3) */ c1 from t1 where c2 < 1024 and c3 > 1024")
if tblIdx == 0 {
// partition table
require.Contains(t, res.Rows()[1][0], "PartitionUnion")
require.Contains(t, res.Rows()[2][0], "IndexMerge")
tk.MustHavePlan(sql, "IndexMerge")
tk.MustHavePlan(sql, "PartitionUnion")
} else {
require.Contains(t, res.Rows()[1][0], "IndexMerge")
tk.MustHavePlan(sql, "IndexMerge")
tk.MustNotHavePlan(sql, "PartitionUnion")
}
}
for i := 0; i < queryCnt; i++ {
Expand Down
19 changes: 10 additions & 9 deletions pkg/planner/core/rule_column_pruning.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,9 @@ func (p *LogicalUnionAll) PruneColumns(parentUsedCols []*expression.Column, opt
if !hasBeenUsed {
parentUsedCols = make([]*expression.Column, len(p.schema.Columns))
copy(parentUsedCols, p.schema.Columns)
for i := range used {
used[i] = true
}
}
for _, child := range p.Children() {
err := child.PruneColumns(parentUsedCols, opt, p)
Expand All @@ -284,16 +287,14 @@ func (p *LogicalUnionAll) PruneColumns(parentUsedCols []*expression.Column, opt
}

prunedColumns := make([]*expression.Column, 0)
if hasBeenUsed {
// keep the schema of LogicalUnionAll same as its children's
used := expression.GetUsedList(p.SCtx(), p.children[0].Schema().Columns, p.schema)
for i := len(used) - 1; i >= 0; i-- {
if !used[i] {
prunedColumns = append(prunedColumns, p.schema.Columns[i])
p.schema.Columns = append(p.schema.Columns[:i], p.schema.Columns[i+1:]...)
}
for i := len(used) - 1; i >= 0; i-- {
if !used[i] {
prunedColumns = append(prunedColumns, p.schema.Columns[i])
p.schema.Columns = append(p.schema.Columns[:i], p.schema.Columns[i+1:]...)
}
appendColumnPruneTraceStep(p, prunedColumns, opt)
}
appendColumnPruneTraceStep(p, prunedColumns, opt)
if hasBeenUsed {
// It's possible that the child operator adds extra columns to the schema.
// Currently, (*LogicalAggregation).PruneColumns() might do this.
// But we don't need such columns, so we add an extra Projection to prune this column when this happened.
Expand Down
38 changes: 21 additions & 17 deletions tests/integrationtest/r/planner/core/casetest/integration.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1420,23 +1420,27 @@ create table t1 (a int, b int);
create table t2_part (a int, b int, key(a)) partition by hash(a) partitions 4;
explain select /*+ TIDB_INLJ(t2_part@sel_2) */ * from t1 where t1.b<10 and not exists (select 1 from t2_part where t1.a=t2_part.a and t2_part.b<20);
id estRows task access object operator info
HashJoin_15 2658.67 root anti semi join, equal:[eq(planner__core__casetest__integration.t1.a, planner__core__casetest__integration.t2_part.a)]
├─PartitionUnion_19(Build) 13293.33 root
│ ├─TableReader_22 3323.33 root data:Selection_21
│ │ └─Selection_21 3323.33 cop[tikv] lt(planner__core__casetest__integration.t2_part.b, 20)
│ │ └─TableFullScan_20 10000.00 cop[tikv] table:t2_part, partition:p0 keep order:false, stats:pseudo
│ ├─TableReader_25 3323.33 root data:Selection_24
│ │ └─Selection_24 3323.33 cop[tikv] lt(planner__core__casetest__integration.t2_part.b, 20)
│ │ └─TableFullScan_23 10000.00 cop[tikv] table:t2_part, partition:p1 keep order:false, stats:pseudo
│ ├─TableReader_28 3323.33 root data:Selection_27
│ │ └─Selection_27 3323.33 cop[tikv] lt(planner__core__casetest__integration.t2_part.b, 20)
│ │ └─TableFullScan_26 10000.00 cop[tikv] table:t2_part, partition:p2 keep order:false, stats:pseudo
│ └─TableReader_31 3323.33 root data:Selection_30
│ └─Selection_30 3323.33 cop[tikv] lt(planner__core__casetest__integration.t2_part.b, 20)
│ └─TableFullScan_29 10000.00 cop[tikv] table:t2_part, partition:p3 keep order:false, stats:pseudo
└─TableReader_18(Probe) 3323.33 root data:Selection_17
└─Selection_17 3323.33 cop[tikv] lt(planner__core__casetest__integration.t1.b, 10)
└─TableFullScan_16 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
HashJoin_19 2658.67 root anti semi join, equal:[eq(planner__core__casetest__integration.t1.a, planner__core__casetest__integration.t2_part.a)]
├─PartitionUnion_23(Build) 13293.33 root
│ ├─Projection_24 3323.33 root planner__core__casetest__integration.t2_part.a
│ │ └─TableReader_27 3323.33 root data:Selection_26
│ │ └─Selection_26 3323.33 cop[tikv] lt(planner__core__casetest__integration.t2_part.b, 20)
│ │ └─TableFullScan_25 10000.00 cop[tikv] table:t2_part, partition:p0 keep order:false, stats:pseudo
│ ├─Projection_28 3323.33 root planner__core__casetest__integration.t2_part.a
│ │ └─TableReader_31 3323.33 root data:Selection_30
│ │ └─Selection_30 3323.33 cop[tikv] lt(planner__core__casetest__integration.t2_part.b, 20)
│ │ └─TableFullScan_29 10000.00 cop[tikv] table:t2_part, partition:p1 keep order:false, stats:pseudo
│ ├─Projection_32 3323.33 root planner__core__casetest__integration.t2_part.a
│ │ └─TableReader_35 3323.33 root data:Selection_34
│ │ └─Selection_34 3323.33 cop[tikv] lt(planner__core__casetest__integration.t2_part.b, 20)
│ │ └─TableFullScan_33 10000.00 cop[tikv] table:t2_part, partition:p2 keep order:false, stats:pseudo
│ └─Projection_36 3323.33 root planner__core__casetest__integration.t2_part.a
│ └─TableReader_39 3323.33 root data:Selection_38
│ └─Selection_38 3323.33 cop[tikv] lt(planner__core__casetest__integration.t2_part.b, 20)
│ └─TableFullScan_37 10000.00 cop[tikv] table:t2_part, partition:p3 keep order:false, stats:pseudo
└─TableReader_22(Probe) 3323.33 root data:Selection_21
└─Selection_21 3323.33 cop[tikv] lt(planner__core__casetest__integration.t1.b, 10)
└─TableFullScan_20 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
Level Code Message
Warning 1105 disable dynamic pruning due to t2_part has no global stats
Warning 1815 Optimizer Hint /*+ INL_JOIN(t2_part) */ or /*+ TIDB_INLJ(t2_part) */ is inapplicable
Expand Down
Loading

0 comments on commit 8583ab5

Please sign in to comment.