Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ranger: split detach process from BuildRange. #4741

Merged
merged 4 commits into from
Oct 12, 2017
Merged

ranger: split detach process from BuildRange. #4741

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
97477eb
ranger: split detach process from BuildRange.
winoros Oct 10, 2017
20a4b5d
fix comment.
winoros Oct 11, 2017
f2e046b
fix typo in comment.
winoros Oct 12, 2017
35aaee2
Merge branch 'master' into yiding/split
coocood Oct 12, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions plan/new_physical_plan_builder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -718,7 +718,8 @@ func (p *DataSource) convertToIndexScan(prop *requiredProp, idx *model.IndexInfo
}
if len(idxCols) > 0 {
var ranges []types.Range
ranges, is.AccessCondition, is.filterCondition, err = ranger.BuildRange(sc, conds, ranger.IndexRangeType, idxCols, colLengths)
is.AccessCondition, is.filterCondition = ranger.DetachIndexConditions(conds, idxCols, colLengths)
ranges, err = ranger.BuildRange(sc, is.AccessCondition, ranger.IndexRangeType, idxCols, colLengths)
if err != nil {
return nil, errors.Trace(err)
}
Expand Down Expand Up @@ -926,7 +927,8 @@ func (p *DataSource) convertToTableScan(prop *requiredProp) (task task, err erro
}
if pkCol != nil {
var ranges []types.Range
ranges, ts.AccessCondition, ts.filterCondition, err = ranger.BuildRange(sc, conds, ranger.IntRangeType, []*expression.Column{pkCol}, nil)
ts.AccessCondition, ts.filterCondition = ranger.DetachColumnConditions(conds, pkCol.ColName)
ranges, err = ranger.BuildRange(sc, ts.AccessCondition, ranger.IntRangeType, []*expression.Column{pkCol}, nil)
ts.Ranges = ranger.Ranges2IntRanges(ranges)
if err != nil {
return nil, errors.Trace(err)
Expand Down
3 changes: 2 additions & 1 deletion statistics/selectivity.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,8 @@ func getMaskAndRanges(ctx context.Context, exprs []expression.Expression, rangeT
for _, expr := range exprs {
exprsClone = append(exprsClone, expr.Clone())
}
ranges, accessConds, _, err := ranger.BuildRange(ctx.GetSessionVars().StmtCtx, exprsClone, rangeType, cols, lengths)
accessConds, _ := ranger.DetachCondsForSelectivity(exprsClone, rangeType, cols, lengths)
ranges, err := ranger.BuildRange(ctx.GetSessionVars().StmtCtx, accessConds, rangeType, cols, lengths)
if err != nil {
return 0, nil, errors.Trace(err)
}
Expand Down
70 changes: 41 additions & 29 deletions util/ranger/new_refiner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,28 +21,34 @@ import (
"github.com/pingcap/tidb/util/types"
)

func buildIndexRange(sc *variable.StatementContext, cols []*expression.Column, lengths []int, inAndEqCount int,
func buildIndexRange(sc *variable.StatementContext, cols []*expression.Column, lengths []int,
accessCondition []expression.Expression) ([]*types.IndexRange, error) {
rb := builder{sc: sc}
var ranges []*types.IndexRange
for i := 0; i < inAndEqCount; i++ {
var (
ranges []*types.IndexRange
eqCount int
)
for eqCount = 0; eqCount < len(accessCondition) && eqCount < len(cols); eqCount++ {
if sf, ok := accessCondition[eqCount].(*expression.ScalarFunction); !ok || sf.FuncName.L != ast.EQ {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The original is inAndEqCount, now only Eq is counted.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

in has be rewritten to or. Currently there won't be any in function any more.

break
}
// Build ranges for equal or in access conditions.
point := rb.build(accessCondition[i])
if i == 0 {
ranges = rb.buildIndexRanges(point, cols[i].RetType)
point := rb.build(accessCondition[eqCount])
if eqCount == 0 {
ranges = rb.buildIndexRanges(point, cols[eqCount].RetType)
} else {
ranges = rb.appendIndexRanges(ranges, point, cols[i].RetType)
ranges = rb.appendIndexRanges(ranges, point, cols[eqCount].RetType)
}
}
rangePoints := fullRange
// Build rangePoints for non-equal access conditions.
for i := inAndEqCount; i < len(accessCondition); i++ {
for i := eqCount; i < len(accessCondition); i++ {
rangePoints = rb.intersection(rangePoints, rb.build(accessCondition[i]))
}
if inAndEqCount == 0 {
if eqCount == 0 {
ranges = rb.buildIndexRanges(rangePoints, cols[0].RetType)
} else if inAndEqCount < len(accessCondition) {
ranges = rb.appendIndexRanges(ranges, rangePoints, cols[inAndEqCount].RetType)
} else if eqCount < len(accessCondition) {
ranges = rb.appendIndexRanges(ranges, rangePoints, cols[eqCount].RetType)
}

// Take prefix index into consideration.
Expand Down Expand Up @@ -126,14 +132,15 @@ func getEQColOffset(expr expression.Expression, cols []*expression.Column) int {
return -1
}

// detachIndexScanConditions will detach the index filters from table filters.
func detachIndexScanConditions(conditions []expression.Expression, cols []*expression.Column, lengths []int) (accessConds []expression.Expression,
filterConds []expression.Expression, accessEqualCount int, accessInAndEqCount int) {
// DetachIndexConditions will detach the index filters from table filters.
func DetachIndexConditions(conditions []expression.Expression, cols []*expression.Column, lengths []int) (accessConds []expression.Expression,
filterConds []expression.Expression) {
accessConds = make([]expression.Expression, len(cols))
// PushDownNot here can convert query 'not (a != 1)' to 'a = 1'.
for i, cond := range conditions {
conditions[i] = expression.PushDownNot(cond, false, nil)
}
var accessEqualCount int
for _, cond := range conditions {
offset := getEQColOffset(cond, cols)
if offset != -1 {
Expand All @@ -153,7 +160,6 @@ func detachIndexScanConditions(conditions []expression.Expression, cols []*expre
accessEqualCount = len(accessConds)
}
}
accessInAndEqCount = accessEqualCount
// We should remove all accessConds, so that they will not be added to filter conditions.
conditions = removeAccessConditions(conditions, accessConds)
var curIndex int
Expand All @@ -171,7 +177,6 @@ func detachIndexScanConditions(conditions []expression.Expression, cols []*expre
accessConds, filterConds = checker.extractAccessAndFilterConds(conditions, accessConds, filterConds)
break
}
accessInAndEqCount++
accessConds = append(accessConds, conditions[accessIdx])
if lengths[curIndex] != types.UnspecifiedLength {
filterConds = append(filterConds, conditions[accessIdx])
Expand All @@ -182,7 +187,7 @@ func detachIndexScanConditions(conditions []expression.Expression, cols []*expre
if curIndex == len(cols) {
filterConds = append(filterConds, conditions...)
}
return accessConds, filterConds, accessEqualCount, accessInAndEqCount
return accessConds, filterConds
}

// buildColumnRange builds the range for sampling histogram to calculate the row count.
Expand All @@ -206,35 +211,42 @@ func buildColumnRange(conds []expression.Expression, sc *variable.StatementConte
return ranges, nil
}

// DetachCondsForSelectivity distinguishes between conditions dor selectivity calculation and other conditions
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

dor? Also, Add a dot at the end.

func DetachCondsForSelectivity(conds []expression.Expression, rangeType int, cols []*expression.Column,
lengths []int) (accessConditions, otherConditions []expression.Expression) {
if rangeType == IntRangeType || rangeType == ColumnRangeType {
return DetachColumnConditions(conds, cols[0].ColName)
} else if rangeType == IndexRangeType {
return DetachIndexConditions(conds, cols, lengths)
}
return nil, conds
}

// BuildRange is a method which can calculate IntColumnRange, ColumnRange, IndexRange.
func BuildRange(sc *variable.StatementContext, conds []expression.Expression, rangeType int, cols []*expression.Column, lengths []int) (retRanges []types.Range,
accessConditions, otherConditions []expression.Expression, _ error) {
func BuildRange(sc *variable.StatementContext, conds []expression.Expression, rangeType int, cols []*expression.Column,
lengths []int) (retRanges []types.Range, _ error) {
if rangeType == IntRangeType {
accessConditions, otherConditions = DetachColumnConditions(conds, cols[0].ColName)
ranges, err := BuildTableRange(accessConditions, sc)
ranges, err := BuildTableRange(conds, sc)
if err != nil {
return nil, nil, nil, errors.Trace(err)
return nil, errors.Trace(err)
}
retRanges = make([]types.Range, 0, len(ranges))
for _, ran := range ranges {
retRanges = append(retRanges, ran)
}
} else if rangeType == ColumnRangeType {
accessConditions, otherConditions = DetachColumnConditions(conds, cols[0].ColName)
ranges, err := buildColumnRange(accessConditions, sc, cols[0].RetType)
ranges, err := buildColumnRange(conds, sc, cols[0].RetType)
if err != nil {
return nil, nil, nil, errors.Trace(err)
return nil, errors.Trace(err)
}
retRanges = make([]types.Range, 0, len(ranges))
for _, ran := range ranges {
retRanges = append(retRanges, ran)
}
} else if rangeType == IndexRangeType {
var eqAndInCount int
accessConditions, otherConditions, _, eqAndInCount = detachIndexScanConditions(conds, cols, lengths)
ranges, err := buildIndexRange(sc, cols, lengths, eqAndInCount, accessConditions)
ranges, err := buildIndexRange(sc, cols, lengths, conds)
if err != nil {
return nil, nil, nil, errors.Trace(err)
return nil, errors.Trace(err)
}
retRanges = make([]types.Range, 0, len(ranges))
for _, ran := range ranges {
Expand Down
9 changes: 6 additions & 3 deletions util/ranger/range_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,8 @@ func (s *testRangerSuite) TestTableRange(c *C) {
tbl := selection.Children()[0].(*plan.DataSource).TableInfo()
col := expression.ColInfo2Col(selection.Schema().Columns, tbl.Columns[0])
c.Assert(col, NotNil)
result, _, _, err := ranger.BuildRange(new(variable.StatementContext), conds, ranger.IntRangeType, []*expression.Column{col}, nil)
conds, _ = ranger.DetachColumnConditions(conds, col.ColName)
result, err := ranger.BuildRange(new(variable.StatementContext), conds, ranger.IntRangeType, []*expression.Column{col}, nil)
c.Assert(err, IsNil)
got := fmt.Sprintf("%v", result)
c.Assert(got, Equals, tt.resultStr, Commentf("different for expr %s", tt.exprStr))
Expand Down Expand Up @@ -304,7 +305,8 @@ func (s *testRangerSuite) TestIndexRange(c *C) {
}
cols, lengths := expression.IndexInfo2Cols(selection.Schema().Columns, tbl.Indices[0])
c.Assert(cols, NotNil)
result, _, _, err := ranger.BuildRange(new(variable.StatementContext), conds, ranger.IndexRangeType, cols, lengths)
conds, _ = ranger.DetachIndexConditions(conds, cols, lengths)
result, err := ranger.BuildRange(new(variable.StatementContext), conds, ranger.IndexRangeType, cols, lengths)
c.Assert(err, IsNil)
got := fmt.Sprintf("%v", result)
c.Assert(got, Equals, tt.resultStr, Commentf("different for expr %s", tt.exprStr))
Expand Down Expand Up @@ -465,7 +467,8 @@ func (s *testRangerSuite) TestColumnRange(c *C) {
}
col := expression.ColInfo2Col(sel.Schema().Columns, ds.TableInfo().Columns[0])
c.Assert(col, NotNil)
result, _, _, err := ranger.BuildRange(new(variable.StatementContext), conds, ranger.ColumnRangeType, []*expression.Column{col}, nil)
conds, _ = ranger.DetachColumnConditions(conds, col.ColName)
result, err := ranger.BuildRange(new(variable.StatementContext), conds, ranger.ColumnRangeType, []*expression.Column{col}, nil)
c.Assert(err, IsNil)
got := fmt.Sprintf("%s", result)
c.Assert(got, Equals, tt.resultStr, Commentf("different for expr %s", tt.exprStr))
Expand Down