Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ddl: support pre-split index regions before creating index #57553

Merged
merged 17 commits into from
Dec 18, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
address comments
  • Loading branch information
tangenta committed Nov 21, 2024
commit fddb904e12bcb576c8cb3b3e4735f5980a92dcef
8 changes: 4 additions & 4 deletions pkg/meta/model/job_args.go
Original file line number Diff line number Diff line change
Expand Up @@ -1328,10 +1328,10 @@ type IndexArg struct {

// IndexArgSplitOpt is a field of IndexArg used by index presplit.
type IndexArgSplitOpt struct {
lance6716 marked this conversation as resolved.
Show resolved Hide resolved
Lower []string `json:"lower"`
Upper []string `json:"upper"`
Num int64 `json:"num"`
ValueLists [][]string `json:"value_lists"`
Lower []string `json:"lower,omitempty"`
Upper []string `json:"upper,omitempty"`
Num int64 `json:"num,omitempty"`
ValueLists [][]string `json:"value_lists,omitempty"`
}

// ModifyIndexArgs is the argument for add/drop/rename index jobs,
Expand Down
29 changes: 5 additions & 24 deletions pkg/parser/ast/ddl.go
Original file line number Diff line number Diff line change
Expand Up @@ -735,7 +735,7 @@ type IndexOption struct {
Visibility IndexVisibility
PrimaryKeyTp model.PrimaryKeyType
Global bool
SplitOpt *SplitOption `json:"-"`
SplitOpt *SplitOption `json:"-"` // SplitOption contains expr nodes, which cannot marshal for DDL job arguments.
}

// IsEmpty is true if only default options are given
Expand Down Expand Up @@ -851,30 +851,11 @@ func (n *IndexOption) Accept(v Visitor) (Node, bool) {
}
n = newNode.(*IndexOption)
if n.SplitOpt != nil {
for i, val := range n.SplitOpt.Lower {
node, ok := val.Accept(v)
if !ok {
return n, false
}
n.SplitOpt.Lower[i] = node.(ExprNode)
}
for i, val := range n.SplitOpt.Upper {
node, ok := val.Accept(v)
if !ok {
return n, false
}
n.SplitOpt.Upper[i] = node.(ExprNode)
}

for i, list := range n.SplitOpt.ValueLists {
for j, val := range list {
node, ok := val.Accept(v)
if !ok {
return n, false
}
n.SplitOpt.ValueLists[i][j] = node.(ExprNode)
}
node, ok := n.SplitOpt.Accept(v)
if !ok {
return n, false
}
n.SplitOpt = node.(*SplitOption)
}
return v.Leave(n)
}
Expand Down
61 changes: 41 additions & 20 deletions pkg/parser/ast/dml.go
Original file line number Diff line number Diff line change
Expand Up @@ -3840,6 +3840,8 @@ type SplitRegionStmt struct {
}

type SplitOption struct {
stmtNode

Lower []ExprNode
Upper []ExprNode
Num int64
Expand Down Expand Up @@ -3899,29 +3901,13 @@ func (n *SplitRegionStmt) Accept(v Visitor) (Node, bool) {
return n, false
}
n.Table = node.(*TableName)
for i, val := range n.SplitOpt.Lower {
node, ok := val.Accept(v)
if !ok {
return n, false
}
n.SplitOpt.Lower[i] = node.(ExprNode)
}
for i, val := range n.SplitOpt.Upper {
node, ok := val.Accept(v)

if n.SplitOpt != nil {
node, ok := n.SplitOpt.Accept(v)
if !ok {
return n, false
}
n.SplitOpt.Upper[i] = node.(ExprNode)
}

for i, list := range n.SplitOpt.ValueLists {
for j, val := range list {
node, ok := val.Accept(v)
if !ok {
return n, false
}
n.SplitOpt.ValueLists[i][j] = node.(ExprNode)
}
n.SplitOpt = node.(*SplitOption)
}
return v.Leave(n)
}
Expand Down Expand Up @@ -3974,6 +3960,41 @@ func (n *SplitOption) Restore(ctx *format.RestoreCtx) error {
return nil
}

// Accept implements Node Accept interface.
func (n *SplitOption) Accept(v Visitor) (Node, bool) {
newNode, skipChildren := v.Enter(n)
if skipChildren {
return v.Leave(newNode)
}
n = newNode.(*SplitOption)

for i, val := range n.Lower {
node, ok := val.Accept(v)
if !ok {
return n, false
}
n.Lower[i] = node.(ExprNode)
}
for i, val := range n.Upper {
node, ok := val.Accept(v)
if !ok {
return n, false
}
n.Upper[i] = node.(ExprNode)
}

for i, list := range n.ValueLists {
for j, val := range list {
node, ok := val.Accept(v)
if !ok {
return n, false
}
n.ValueLists[i][j] = node.(ExprNode)
}
}
return v.Leave(n)
}

type FulltextSearchModifier int

const (
Expand Down
8 changes: 4 additions & 4 deletions pkg/parser/parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -1482,7 +1482,7 @@ var (
58097: 547, // voter (1599x)
57971: 548, // weightString (1599x)
57505: 549, // on (1514x)
40: 550, // '(' (1511x)
40: 550, // '(' (1512x)
57590: 551, // with (1381x)
57353: 552, // stringLit (1351x)
58185: 553, // not2 (1315x)
Expand All @@ -1507,7 +1507,7 @@ var (
57381: 572, // charType (1034x)
58174: 573, // eq (1030x)
57426: 574, // fetch (1028x)
58169: 575, // intLit (1023x)
58169: 575, // intLit (1024x)
57541: 576, // set (1021x)
57477: 577, // limit (1019x)
57431: 578, // forKwd (1015x)
Expand Down Expand Up @@ -1754,7 +1754,7 @@ var (
58535: 819, // NUM (124x)
58897: 820, // logAnd (107x)
58898: 821, // logOr (107x)
58389: 822, // EqOpt (101x)
58389: 822, // EqOpt (102x)
57407: 823, // deleteKwd (87x)
58802: 824, // TableName (82x)
58780: 825, // StringName (56x)
Expand Down Expand Up @@ -10090,7 +10090,7 @@ var (
{2156, 2156, 2156, 2156, 2156, 2156, 2156, 8: 2156, 19: 2156, 2156, 52: 2156, 58: 2156, 61: 2156, 100: 2156, 2156, 2156, 2156, 105: 2156, 107: 2156, 551: 2156, 561: 2156, 565: 2156, 581: 2156},
{2155, 2155, 2155, 2155, 2155, 2155, 2155, 8: 2155, 19: 2155, 2155, 52: 2155, 58: 2155, 61: 2155, 100: 2155, 2155, 2155, 2155, 105: 2155, 107: 2155, 551: 2155, 561: 2155, 565: 2155, 581: 2155},
{2154, 2154, 2154, 2154, 2154, 2154, 2154, 8: 2154, 19: 2154, 2154, 52: 2154, 58: 2154, 61: 2154, 100: 2154, 2154, 2154, 2154, 105: 2154, 107: 2154, 551: 2154, 561: 2154, 565: 2154, 581: 2154},
{573: 5491},
{550: 2374, 573: 4695, 575: 2374, 822: 5491},
{200: 5486, 224: 5488, 241: 5485, 247: 5489, 256: 5487, 1027: 5490},
// 2550
{200: 5486, 224: 5488, 241: 5485, 247: 5489, 256: 5487, 1027: 5484},
Expand Down
4 changes: 2 additions & 2 deletions pkg/parser/parser.y
Original file line number Diff line number Diff line change
Expand Up @@ -6645,13 +6645,13 @@ IndexOption:
Global: false,
}
}
| "PRE_SPLIT_REGIONS" "=" '(' SplitOption ')'
| "PRE_SPLIT_REGIONS" EqOpt '(' SplitOption ')'
{
$$ = &ast.IndexOption{
SplitOpt: $4.(*ast.SplitOption),
}
}
| "PRE_SPLIT_REGIONS" "=" Int64Num
| "PRE_SPLIT_REGIONS" EqOpt Int64Num
{
$$ = &ast.IndexOption{
SplitOpt: &ast.SplitOption{
Expand Down
2 changes: 1 addition & 1 deletion pkg/util/split.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ func GetValuesList(lower, upper []byte, num int, valuesList [][]byte) [][]byte {
startV := getUint64FromBytes(lower[commonPrefixIdx:], 0)
// To get `num` regions, only need to split `num-1` idx keys.
buf := make([]byte, 8)
for i := 0; i < num-1; i++ {
for range num - 1 {
value := make([]byte, 0, commonPrefixIdx+8)
value = append(value, lower[:commonPrefixIdx]...)
startV += step
Expand Down