Skip to content

Commit

Permalink
planner: fix range partition prune with an unsigned column (#50113) (#…
Browse files Browse the repository at this point in the history 
…50195)

close #50082
  • Loading branch information
@ti-chi-bot
ti-chi-bot authored Feb 26, 2024
1 parent a7ec476 commit a4b0f4c
Show file tree
Hide file tree
Showing 5 changed files with 150 additions and 100 deletions.
39 changes: 3 additions & 36 deletions expression/builtin_compare.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2627,22 +2627,8 @@ func (b *builtinNullEQIntSig) evalInt(row chunk.Row) (val int64, isNull bool, er
res = 1
case isNull0 != isNull1:
return res, false, nil
case isUnsigned0 && isUnsigned1 && types.CompareUint64(uint64(arg0), uint64(arg1)) == 0:
res = 1
case !isUnsigned0 && !isUnsigned1 && types.CompareInt64(arg0, arg1) == 0:
res = 1
case isUnsigned0 && !isUnsigned1:
if arg1 < 0 {
return res, false, nil
}
if types.CompareInt64(arg0, arg1) == 0 {
res = 1
}
case !isUnsigned0 && isUnsigned1:
if arg0 < 0 {
return res, false, nil
}
if types.CompareInt64(arg0, arg1) == 0 {
default:
if types.CompareInt(arg0, isUnsigned0, arg1, isUnsigned1) == 0 {
res = 1
}
}
Expand Down Expand Up @@ -2945,26 +2931,7 @@ func CompareInt(sctx sessionctx.Context, lhsArg, rhsArg Expression, lhsRow, rhsR
}

isUnsigned0, isUnsigned1 := mysql.HasUnsignedFlag(lhsArg.GetType().GetFlag()), mysql.HasUnsignedFlag(rhsArg.GetType().GetFlag())
var res int
switch {
case isUnsigned0 && isUnsigned1:
res = types.CompareUint64(uint64(arg0), uint64(arg1))
case isUnsigned0 && !isUnsigned1:
if arg1 < 0 || uint64(arg0) > math.MaxInt64 {
res = 1
} else {
res = types.CompareInt64(arg0, arg1)
}
case !isUnsigned0 && isUnsigned1:
if arg0 < 0 || uint64(arg1) > math.MaxInt64 {
res = -1
} else {
res = types.CompareInt64(arg0, arg1)
}
case !isUnsigned0 && !isUnsigned1:
res = types.CompareInt64(arg0, arg1)
}
return int64(res), false, nil
return int64(types.CompareInt(arg0, isUnsigned0, arg1, isUnsigned1)), false, nil
}

func genCompareString(collation string) func(sctx sessionctx.Context, lhsArg Expression, rhsArg Expression, lhsRow chunk.Row, rhsRow chunk.Row) (int64, bool, error) {
Expand Down
23 changes: 23 additions & 0 deletions planner/core/partition_pruner_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -517,3 +517,26 @@ func TestIssue43459(t *testing.T) {
"1 200000 2022-12-29",
"2 200000 2023-01-01"))
}

func TestIssue50082(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("drop table if exists t;")
tk.MustExec("create table t(a bigint unsigned)" +
"PARTITION BY RANGE (`a`)" +
"(PARTITION `p0` VALUES LESS THAN (5086706)," +
" PARTITION `p1` VALUES LESS THAN (7268292)," +
" PARTITION `p2` VALUES LESS THAN (16545422)," +
" PARTITION `p3` VALUES LESS THAN (9223372036854775810));")
require.True(t, tk.HasNoPlan("select * from t where a BETWEEN -6895222 AND 3125507;", "TableDual"))

tk.MustExec("drop table if exists t;")
tk.MustExec("create table t(a bigint)" +
"PARTITION BY RANGE (`a`)" +
"(PARTITION `p0` VALUES LESS THAN (5086706)," +
" PARTITION `p1` VALUES LESS THAN (7268292)," +
" PARTITION `p2` VALUES LESS THAN (16545422)," +
" PARTITION `p3` VALUES LESS THAN (9223372036854775807));")
require.True(t, tk.HasNoPlan("select * from t where a BETWEEN -6895222 AND 3125507;", "TableDual"))
}
100 changes: 74 additions & 26 deletions planner/core/partition_pruning_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,39 +81,87 @@ func TestCanBePrune(t *testing.T) {
// need to convert 1564761600 to a timestamp, during that step, an error happen and the result is set to <nil>
}

func TestPruneUseBinarySearch(t *testing.T) {
lessThan := lessThanDataInt{data: []int64{4, 7, 11, 14, 17, 0}, maxvalue: true}
func TestPruneUseBinarySearchSigned(t *testing.T) {
lessThan := lessThanDataInt{data: []int64{-3, 4, 7, 11, 14, 17, 0}, maxvalue: true, unsigned: false}
cases := []struct {
input dataForPrune
result partitionRange
}{
{dataForPrune{ast.EQ, 66, false}, partitionRange{6, 7}},
{dataForPrune{ast.EQ, 14, false}, partitionRange{5, 6}},
{dataForPrune{ast.EQ, 10, false}, partitionRange{3, 4}},
{dataForPrune{ast.EQ, 3, false}, partitionRange{1, 2}},
{dataForPrune{ast.EQ, -4, false}, partitionRange{0, 1}},
{dataForPrune{ast.LT, 66, false}, partitionRange{0, 7}},
{dataForPrune{ast.LT, 14, false}, partitionRange{0, 5}},
{dataForPrune{ast.LT, 10, false}, partitionRange{0, 4}},
{dataForPrune{ast.LT, 3, false}, partitionRange{0, 2}},
{dataForPrune{ast.LT, -4, false}, partitionRange{0, 1}},
{dataForPrune{ast.GE, 66, false}, partitionRange{6, 7}},
{dataForPrune{ast.GE, 14, false}, partitionRange{5, 7}},
{dataForPrune{ast.GE, 10, false}, partitionRange{3, 7}},
{dataForPrune{ast.GE, 3, false}, partitionRange{1, 7}},
{dataForPrune{ast.GE, -4, false}, partitionRange{0, 7}},
{dataForPrune{ast.GT, 66, false}, partitionRange{6, 7}},
{dataForPrune{ast.GT, 14, false}, partitionRange{5, 7}},
{dataForPrune{ast.GT, 10, false}, partitionRange{4, 7}},
{dataForPrune{ast.GT, 3, false}, partitionRange{2, 7}},
{dataForPrune{ast.GT, 2, false}, partitionRange{1, 7}},
{dataForPrune{ast.GT, -4, false}, partitionRange{1, 7}},
{dataForPrune{ast.LE, 66, false}, partitionRange{0, 7}},
{dataForPrune{ast.LE, 14, false}, partitionRange{0, 6}},
{dataForPrune{ast.LE, 10, false}, partitionRange{0, 4}},
{dataForPrune{ast.LE, 3, false}, partitionRange{0, 2}},
{dataForPrune{ast.LE, -4, false}, partitionRange{0, 1}},
{dataForPrune{ast.IsNull, 0, false}, partitionRange{0, 1}},
{dataForPrune{"illegal", 0, false}, partitionRange{0, 7}},
}

for i, ca := range cases {
start, end := pruneUseBinarySearch(lessThan, ca.input)
require.Equalf(t, ca.result.start, start, "fail = %d", i)
require.Equalf(t, ca.result.end, end, "fail = %d", i)
}
}

func TestPruneUseBinarySearchUnSigned(t *testing.T) {
lessThan := lessThanDataInt{data: []int64{4, 7, 11, 14, 17, 0}, maxvalue: true, unsigned: true}
cases := []struct {
input dataForPrune
result partitionRange
}{
{dataForPrune{ast.EQ, 66}, partitionRange{5, 6}},
{dataForPrune{ast.EQ, 14}, partitionRange{4, 5}},
{dataForPrune{ast.EQ, 10}, partitionRange{2, 3}},
{dataForPrune{ast.EQ, 3}, partitionRange{0, 1}},
{dataForPrune{ast.LT, 66}, partitionRange{0, 6}},
{dataForPrune{ast.LT, 14}, partitionRange{0, 4}},
{dataForPrune{ast.LT, 10}, partitionRange{0, 3}},
{dataForPrune{ast.LT, 3}, partitionRange{0, 1}},
{dataForPrune{ast.GE, 66}, partitionRange{5, 6}},
{dataForPrune{ast.GE, 14}, partitionRange{4, 6}},
{dataForPrune{ast.GE, 10}, partitionRange{2, 6}},
{dataForPrune{ast.GE, 3}, partitionRange{0, 6}},
{dataForPrune{ast.GT, 66}, partitionRange{5, 6}},
{dataForPrune{ast.GT, 14}, partitionRange{4, 6}},
{dataForPrune{ast.GT, 10}, partitionRange{3, 6}},
{dataForPrune{ast.GT, 3}, partitionRange{1, 6}},
{dataForPrune{ast.GT, 2}, partitionRange{0, 6}},
{dataForPrune{ast.LE, 66}, partitionRange{0, 6}},
{dataForPrune{ast.LE, 14}, partitionRange{0, 5}},
{dataForPrune{ast.LE, 10}, partitionRange{0, 3}},
{dataForPrune{ast.LE, 3}, partitionRange{0, 1}},
{dataForPrune{ast.IsNull, 0}, partitionRange{0, 1}},
{dataForPrune{"illegal", 0}, partitionRange{0, 6}},
{dataForPrune{ast.EQ, 66, false}, partitionRange{5, 6}},
{dataForPrune{ast.EQ, 14, false}, partitionRange{4, 5}},
{dataForPrune{ast.EQ, 10, false}, partitionRange{2, 3}},
{dataForPrune{ast.EQ, 3, false}, partitionRange{0, 1}},
{dataForPrune{ast.EQ, -3, false}, partitionRange{0, 1}},
{dataForPrune{ast.LT, 66, false}, partitionRange{0, 6}},
{dataForPrune{ast.LT, 14, false}, partitionRange{0, 4}},
{dataForPrune{ast.LT, 10, false}, partitionRange{0, 3}},
{dataForPrune{ast.LT, 3, false}, partitionRange{0, 1}},
{dataForPrune{ast.LT, -3, false}, partitionRange{0, 1}},
{dataForPrune{ast.GE, 66, false}, partitionRange{5, 6}},
{dataForPrune{ast.GE, 14, false}, partitionRange{4, 6}},
{dataForPrune{ast.GE, 10, false}, partitionRange{2, 6}},
{dataForPrune{ast.GE, 3, false}, partitionRange{0, 6}},
{dataForPrune{ast.GE, -3, false}, partitionRange{0, 6}},
{dataForPrune{ast.GT, 66, false}, partitionRange{5, 6}},
{dataForPrune{ast.GT, 14, false}, partitionRange{4, 6}},
{dataForPrune{ast.GT, 10, false}, partitionRange{3, 6}},
{dataForPrune{ast.GT, 3, false}, partitionRange{1, 6}},
{dataForPrune{ast.GT, 2, false}, partitionRange{0, 6}},
{dataForPrune{ast.GT, -3, false}, partitionRange{0, 6}},
{dataForPrune{ast.LE, 66, false}, partitionRange{0, 6}},
{dataForPrune{ast.LE, 14, false}, partitionRange{0, 5}},
{dataForPrune{ast.LE, 10, false}, partitionRange{0, 3}},
{dataForPrune{ast.LE, 3, false}, partitionRange{0, 1}},
{dataForPrune{ast.LE, -3, false}, partitionRange{0, 1}},
{dataForPrune{ast.IsNull, 0, false}, partitionRange{0, 1}},
{dataForPrune{"illegal", 0, false}, partitionRange{0, 6}},
}

for i, ca := range cases {
start, end := pruneUseBinarySearch(lessThan, ca.input, false)
start, end := pruneUseBinarySearch(lessThan, ca.input)
require.Equalf(t, ca.result.start, start, "fail = %d", i)
require.Equalf(t, ca.result.end, end, "fail = %d", i)
}
Expand Down
64 changes: 26 additions & 38 deletions planner/core/rule_partition_processor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import (
"bytes"
"context"
"fmt"
gomath "math"
"math"
"sort"
"strings"

Expand Down Expand Up @@ -221,7 +221,7 @@ func (s *partitionProcessor) getUsedHashPartitions(ctx sessionctx.Context,
// issue:#22619
if col.RetType.GetType() == mysql.TypeBit {
// maximum number of partitions is 8192
if col.RetType.GetFlen() > 0 && col.RetType.GetFlen() < int(gomath.Log2(mysql.PartitionCountLimit)) {
if col.RetType.GetFlen() > 0 && col.RetType.GetFlen() < int(math.Log2(mysql.PartitionCountLimit)) {
// all possible hash values
maxUsedPartitions := 1 << col.RetType.GetFlen()
if maxUsedPartitions < numPartitions {
Expand Down Expand Up @@ -794,39 +794,21 @@ func (*partitionProcessor) name() string {

type lessThanDataInt struct {
data []int64
unsigned bool
maxvalue bool
}

func (lt *lessThanDataInt) length() int {
return len(lt.data)
}

func compareUnsigned(v1, v2 int64) int {
switch {
case uint64(v1) > uint64(v2):
return 1
case uint64(v1) == uint64(v2):
return 0
}
return -1
}

func (lt *lessThanDataInt) compare(ith int, v int64, unsigned bool) int {
if ith == len(lt.data)-1 {
if lt.maxvalue {
return 1
}
}
if unsigned {
return compareUnsigned(lt.data[ith], v)
}
switch {
case lt.data[ith] > v:
// TODO: get an extra partition when `v` bigger than `lt.maxvalue``, but the result still correct.
if ith == lt.length()-1 && lt.maxvalue {
return 1
case lt.data[ith] == v:
return 0
}
return -1

return types.CompareInt(lt.data[ith], lt.unsigned, v, unsigned)
}

// partitionRange represents [start, end)
Expand Down Expand Up @@ -970,6 +952,7 @@ func (s *partitionProcessor) pruneRangePartition(ctx sessionctx.Context, pi *mod
pruner := rangePruner{
lessThan: lessThanDataInt{
data: partExpr.ForRangePruning.LessThan,
unsigned: mysql.HasUnsignedFlag(col.GetType().GetFlag()),
maxvalue: partExpr.ForRangePruning.MaxValue,
},
col: col,
Expand Down Expand Up @@ -1279,8 +1262,7 @@ func (p *rangePruner) partitionRangeForExpr(sctx sessionctx.Context, expr expres
return 0, 0, false
}

unsigned := mysql.HasUnsignedFlag(p.col.RetType.GetFlag())
start, end := pruneUseBinarySearch(p.lessThan, dataForPrune, unsigned)
start, end := pruneUseBinarySearch(p.lessThan, dataForPrune)
return start, end, true
}

Expand Down Expand Up @@ -1341,7 +1323,6 @@ func partitionRangeForInExpr(sctx sessionctx.Context, args []expression.Expressi
}

var result partitionRangeOR
unsigned := mysql.HasUnsignedFlag(col.RetType.GetFlag())
for i := 1; i < len(args); i++ {
constExpr, ok := args[i].(*expression.Constant)
if !ok {
Expand All @@ -1354,18 +1335,21 @@ func partitionRangeForInExpr(sctx sessionctx.Context, args []expression.Expressi

var val int64
var err error
var unsigned bool
if pruner.partFn != nil {
// replace fn(col) to fn(const)
partFnConst := replaceColumnWithConst(pruner.partFn, constExpr)
val, _, err = partFnConst.EvalInt(sctx, chunk.Row{})
unsigned = mysql.HasUnsignedFlag(partFnConst.GetType().GetFlag())
} else {
val, err = constExpr.Value.ToInt64(sctx.GetSessionVars().StmtCtx)
val, _, err = constExpr.EvalInt(sctx, chunk.Row{})
unsigned = mysql.HasUnsignedFlag(constExpr.GetType().GetFlag())
}
if err != nil {
return pruner.fullRange()
}

start, end := pruneUseBinarySearch(pruner.lessThan, dataForPrune{op: ast.EQ, c: val}, unsigned)
start, end := pruneUseBinarySearch(pruner.lessThan, dataForPrune{op: ast.EQ, c: val, unsigned: unsigned})
result = append(result, partitionRange{start, end})
}
return result.simplify()
Expand Down Expand Up @@ -1401,8 +1385,9 @@ func getMonotoneMode(fnName string) monotoneMode {

// f(x) op const, op is > = <
type dataForPrune struct {
op string
c int64
op string
c int64
unsigned bool
}

// extractDataForPrune extracts data from the expression for pruning.
Expand Down Expand Up @@ -1476,6 +1461,7 @@ func (p *rangePruner) extractDataForPrune(sctx sessionctx.Context, expr expressi
c, isNull, err := constExpr.EvalInt(sctx, chunk.Row{})
if err == nil && !isNull {
ret.c = c
ret.unsigned = mysql.HasUnsignedFlag(constExpr.GetType().GetFlag())
return ret, true
}
return ret, false
Expand Down Expand Up @@ -1533,44 +1519,46 @@ func relaxOP(op string) string {

// pruneUseBinarySearch returns the start and end of which partitions will match.
// If no match (i.e. value > last partition) the start partition will be the number of partition, not the first partition!
func pruneUseBinarySearch(lessThan lessThanDataInt, data dataForPrune, unsigned bool) (start int, end int) {
func pruneUseBinarySearch(lessThan lessThanDataInt, data dataForPrune) (start int, end int) {
length := lessThan.length()
switch data.op {
case ast.EQ:
// col = 66, lessThan = [4 7 11 14 17] => [5, 5)
// col = 14, lessThan = [4 7 11 14 17] => [4, 5)
// col = 10, lessThan = [4 7 11 14 17] => [2, 3)
// col = 3, lessThan = [4 7 11 14 17] => [0, 1)
pos := sort.Search(length, func(i int) bool { return lessThan.compare(i, data.c, unsigned) > 0 })
pos := sort.Search(length, func(i int) bool { return lessThan.compare(i, data.c, data.unsigned) > 0 })
start, end = pos, pos+1
case ast.LT:
// col < 66, lessThan = [4 7 11 14 17] => [0, 5)
// col < 14, lessThan = [4 7 11 14 17] => [0, 4)
// col < 10, lessThan = [4 7 11 14 17] => [0, 3)
// col < 3, lessThan = [4 7 11 14 17] => [0, 1)
pos := sort.Search(length, func(i int) bool { return lessThan.compare(i, data.c, unsigned) >= 0 })
pos := sort.Search(length, func(i int) bool { return lessThan.compare(i, data.c, data.unsigned) >= 0 })
start, end = 0, pos+1
case ast.GE:
// col >= 66, lessThan = [4 7 11 14 17] => [5, 5)
// col >= 14, lessThan = [4 7 11 14 17] => [4, 5)
// col >= 10, lessThan = [4 7 11 14 17] => [2, 5)
// col >= 3, lessThan = [4 7 11 14 17] => [0, 5)
pos := sort.Search(length, func(i int) bool { return lessThan.compare(i, data.c, unsigned) > 0 })
pos := sort.Search(length, func(i int) bool { return lessThan.compare(i, data.c, data.unsigned) > 0 })
start, end = pos, length
case ast.GT:
// col > 66, lessThan = [4 7 11 14 17] => [5, 5)
// col > 14, lessThan = [4 7 11 14 17] => [4, 5)
// col > 10, lessThan = [4 7 11 14 17] => [3, 5)
// col > 3, lessThan = [4 7 11 14 17] => [1, 5)
// col > 2, lessThan = [4 7 11 14 17] => [0, 5)
pos := sort.Search(length, func(i int) bool { return lessThan.compare(i, data.c+1, unsigned) > 0 })

// Although `data.c+1` will overflow in sometime, this does not affect the correct results obtained.
pos := sort.Search(length, func(i int) bool { return lessThan.compare(i, data.c+1, data.unsigned) > 0 })
start, end = pos, length
case ast.LE:
// col <= 66, lessThan = [4 7 11 14 17] => [0, 6)
// col <= 14, lessThan = [4 7 11 14 17] => [0, 5)
// col <= 10, lessThan = [4 7 11 14 17] => [0, 3)
// col <= 3, lessThan = [4 7 11 14 17] => [0, 1)
pos := sort.Search(length, func(i int) bool { return lessThan.compare(i, data.c, unsigned) > 0 })
pos := sort.Search(length, func(i int) bool { return lessThan.compare(i, data.c, data.unsigned) > 0 })
start, end = 0, pos+1
case ast.IsNull:
start, end = 0, 1
Expand Down
Loading

0 comments on commit a4b0f4c

Please sign in to comment.