Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

*: split index region with lower upper syntax #10409

Merged
merged 34 commits into from
Jun 6, 2019
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
e67ecf4
*: split index region with min max num syntax
crazycs520 May 9, 2019
cbdb504
Merge branch 'master' into split_index_step
crazycs520 May 9, 2019
bd39a3b
refine code
crazycs520 May 9, 2019
4b7b306
Merge branch 'split_index_step' of https://github.com/crazycs520/tidb…
crazycs520 May 9, 2019
1072642
Merge branch 'master' into split_index_step
crazycs520 May 10, 2019
4fa0832
Merge branch 'master' into split_index_step
crazycs520 May 10, 2019
ab3417e
Merge branch 'master' into split_index_step
crazycs520 May 20, 2019
5b5f871
update go.mod for parsre
crazycs520 May 20, 2019
553c9be
update go.mod for parsre
crazycs520 May 20, 2019
cdb0e6e
refactor code
crazycs520 May 20, 2019
92fe00e
Merge branch 'master' into split_index_step
zyguan May 20, 2019
5033512
Merge branch 'master' of https://github.com/pingcap/tidb into split_i…
crazycs520 May 20, 2019
c5334ff
Merge branch 'split_index_step' of https://github.com/crazycs520/tidb…
crazycs520 May 20, 2019
3668f80
add comment
crazycs520 Jun 4, 2019
2558457
refine code
crazycs520 Jun 4, 2019
078b8f4
address comment
crazycs520 Jun 4, 2019
208a15f
refine code
crazycs520 Jun 4, 2019
ad2af9f
Address comment
crazycs520 Jun 4, 2019
90d5bef
address comment
crazycs520 Jun 4, 2019
f54c7f1
use between and regions syntax
crazycs520 Jun 5, 2019
ccf2e8e
fix region num 0 bug and add test
crazycs520 Jun 5, 2019
6ec298f
use lower and upper instead of min/max
crazycs520 Jun 5, 2019
84d4240
Merge branch 'master' into split_index_step
crazycs520 Jun 5, 2019
ffbe922
refine error msg
crazycs520 Jun 5, 2019
289dbb3
refine code
crazycs520 Jun 5, 2019
ab2c03d
address comment
crazycs520 Jun 5, 2019
ccb183b
update go.mod for parsre
crazycs520 Jun 5, 2019
ac519e7
Merge branch 'master' into split_index_step
crazycs520 Jun 5, 2019
5cdbbf1
fix ci
crazycs520 Jun 5, 2019
97a4610
Merge branch 'split_index_step' of https://github.com/crazycs520/tidb…
crazycs520 Jun 5, 2019
95ec64b
address comment
crazycs520 Jun 5, 2019
a2037dd
add comment and fix test
crazycs520 Jun 5, 2019
1d1a2d7
Merge branch 'master' into split_index_step
crazycs520 Jun 6, 2019
e5ba5b6
address comment
crazycs520 Jun 6, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
refine code
  • Loading branch information
crazycs520 committed Jun 4, 2019
commit 255845711c4c1e2ece4352f747f0ba0c66d0d60c
67 changes: 21 additions & 46 deletions executor/split.go
Original file line number Diff line number Diff line change
Expand Up @@ -140,60 +140,36 @@ func longestCommonPrefixLen(s1, s2 []byte) int {
return i
}

// getDiffBytesValue gets the diff-value from the `startIdx`. Normally, `startIdx` is the longest common prefix byte length.
// eg: min: [10,1, 2,3,4,5]
// max: [10,10,9,8,7,6,5,4]
// startIdx: 1
// the diff bytes is [10,9,8,7,6,5,4]
// - [1, 2,3,4,5]
// = [9, 7,5,3,1,5,4]
// I need 8 diff-bytes to convert(decode) a uint64 value. So append 0xff to complete 8 bytes.
// so the return uint64 value is binary.BigEndian.Uint64([]byte{9,7,5,3,1,5,4,255}).
func getDiffBytesValue(startIdx int, min, max []byte) uint64 {
l := len(min)
if len(max) < len(min) {
l = len(max)
}
if l-startIdx > 8 {
l = startIdx + 8
}
diff := make([]byte, 0, 8)
for i := startIdx; i < l; i++ {
diff = append(diff, max[i]-min[i])
}
if len(max) > l {
for i := l; i < len(max); i++ {
diff = append(diff, max[i])
if len(diff) >= 8 {
break
}
}
}
if len(min) > l {
for i := l; i < len(min); i++ {
diff = append(diff, 0xff-min[i])
if len(diff) >= 8 {
break
}
}
}
// getStepValue gets the step of between the min and max value. step = (max-min)/num.
// convert byte slice to uint64 first.
crazycs520 marked this conversation as resolved.
Show resolved Hide resolved
func getStepValue(min, max []byte, num int) uint64 {
minUint := getUint64FromBytes(min, 0)
maxUint := getUint64FromBytes(max, 0xff)
return (maxUint - minUint) / uint64(num)
}

for i := len(diff); i < 8; i++ {
diff = append(diff, 0xff)
// getUint64FromBytes gets a uint64 from the `bs` byte slice.
// If len(bs) < 8, then padding with `pad`.
func getUint64FromBytes(bs []byte, pad byte) uint64 {
buf := bs
if len(buf) < 8 {
buf = make([]byte, 0, 8)
buf = append(buf, bs...)
for i := len(buf); i < 8; i++ {
buf = append(buf, pad)
}
}
diffValue := binary.BigEndian.Uint64(diff)
return diffValue
return binary.BigEndian.Uint64(buf)
}

// getValuesList use to get `num` values between min and max value.
crazycs520 marked this conversation as resolved.
Show resolved Hide resolved
// To Simplify the explain, suppose min and max value type is int64, and min=0, max=100, num=10,
// then calculate the step=(max-min)/num=10, then the function should return 0+10, 10+10, 20+10... all together 9 (num-1) values.
// then the function will return [10,20,30,40,50,60,70,80,90].
// The difference is the max,min value type is []byte, So I use getDiffBytesValue to calculate the (max-min) value.
// The difference is the max,min value type is []byte, So I use getUint64FromBytes to convert []byte to uint64.
func getValuesList(min, max []byte, num int, valuesList [][]byte) [][]byte {
startIdx := longestCommonPrefixLen(min, max)
crazycs520 marked this conversation as resolved.
Show resolved Hide resolved
diffValue := getDiffBytesValue(startIdx, min, max)
step := diffValue / uint64(num)
step := getStepValue(min[startIdx:], max[startIdx:], num)

startValueTemp := min[startIdx:]
if len(startValueTemp) > 8 {
Expand All @@ -206,9 +182,8 @@ func getValuesList(min, max []byte, num int, valuesList [][]byte) [][]byte {
}
startV := binary.BigEndian.Uint64(startValue)
// To get `num` regions, only need to split `num-1` idx keys.
num--
tmp := make([]byte, 8)
for i := 0; i < num; i++ {
for i := 0; i < num-1; i++ {
value := make([]byte, 0, startIdx+8)
value = append(value, min[:startIdx]...)
startV += step
Expand Down
4 changes: 2 additions & 2 deletions executor/split_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ func (s *testSplitIndex) TestLongestCommonPrefixLen(c *C) {
}
}

func (s *testSplitIndex) TestGetDiffBytesValue(c *C) {
func (s *testSplitIndex) TestgetStepValue(c *C) {
cases := []struct {
min []byte
max []byte
Expand All @@ -79,7 +79,7 @@ func (s *testSplitIndex) TestGetDiffBytesValue(c *C) {
for _, ca := range cases {
l0 := longestCommonPrefixLen(ca.min, ca.max)
crazycs520 marked this conversation as resolved.
Show resolved Hide resolved
c.Assert(l0, Equals, ca.l)
v0 := getDiffBytesValue(l0, ca.min, ca.max)
v0 := getStepValue(ca.min[l0:], ca.max[l0:], 1)
c.Assert(v0, Equals, ca.v)
}
}
Expand Down