Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

*: support create global binding #9846

Merged
merged 54 commits into from
Apr 17, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
54 commits
Select commit Hold shift + click to select a range
0cebb75
plan binding:add global binding
iamzhoug37 Mar 21, 2019
4f5df56
fix tidy fmt lint
iamzhoug37 Mar 21, 2019
68ee4d8
remove blan
iamzhoug37 Mar 21, 2019
9747bff
fix fmt
iamzhoug37 Mar 21, 2019
44d4619
fix error-check
iamzhoug37 Mar 21, 2019
0d85387
fix createBindPlan originSql and bindSql unnormolize
iamzhoug37 Mar 22, 2019
150b574
fix import
iamzhoug37 Mar 22, 2019
e2dc348
add index for origin_sql and defaultDB
iamzhoug37 Mar 25, 2019
367c7cc
merge master
iamzhoug37 Mar 25, 2019
5aa25ef
fix comment
iamzhoug37 Mar 26, 2019
6c1bd81
add version update
iamzhoug37 Mar 28, 2019
ae25b95
fix go.sum
iamzhoug37 Mar 28, 2019
ea3940a
replace user session ctx with domain's session ctx
iamzhoug37 Mar 29, 2019
e9e9cb0
fix lint
iamzhoug37 Mar 29, 2019
9cadfff
fix comment
iamzhoug37 Apr 1, 2019
787564e
xx
iamzhoug37 Apr 1, 2019
2d7d6d3
fx comment
iamzhoug37 Apr 1, 2019
3416b81
fix ci
iamzhoug37 Apr 1, 2019
ecd00b4
fix version
iamzhoug37 Apr 1, 2019
1df98f0
replace originalSQL with rowId
iamzhoug37 Apr 2, 2019
91b7494
fix ci
iamzhoug37 Apr 2, 2019
8849d3d
fix comment
iamzhoug37 Apr 2, 2019
7e9d879
add privilage check
iamzhoug37 Apr 3, 2019
45c24f3
fix make lint
iamzhoug37 Apr 3, 2019
e89f6c9
create bind plan
iamzhoug37 Apr 3, 2019
d77a038
fix comment
iamzhoug37 Apr 3, 2019
520ea9e
fix comment
iamzhoug37 Apr 4, 2019
23e3b97
remove unused code
iamzhoug37 Apr 4, 2019
924145e
timestamp bugfix
iamzhoug37 Apr 4, 2019
9e539a4
Merge branch 'master' into plan-management-add-global-binding
alivxxx Apr 4, 2019
fca0e5d
Merge branch 'master' into plan-management-add-global-binding
iamzhoug37 Apr 4, 2019
debf0c3
fix undefined error
iamzhoug37 Apr 4, 2019
bd1ae07
cjrioe
iamzhoug37 Apr 4, 2019
9380e05
merge master
iamzhoug37 Apr 12, 2019
8c6e4ec
add create update cache
iamzhoug37 Apr 15, 2019
682e5af
fix comment
iamzhoug37 Apr 16, 2019
d22f5dc
fix comment
iamzhoug37 Apr 16, 2019
3527cbc
lock
iamzhoug37 Apr 16, 2019
f45010a
aa
iamzhoug37 Apr 16, 2019
fda7535
tiny refine
zz-jason Apr 16, 2019
927346a
tiny refine planner and executor
zz-jason Apr 16, 2019
d5ffa60
release lock
iamzhoug37 Apr 16, 2019
918d8d6
Merge branch 'plan-management-add-global-binding' of github.com:iamzh…
iamzhoug37 Apr 16, 2019
efc4ad3
refine mutex in handle
zz-jason Apr 16, 2019
1b59531
Merge branch 'plan-management-add-global-binding' of github.com:iamzh…
iamzhoug37 Apr 16, 2019
33a2af4
code refactor
iamzhoug37 Apr 17, 2019
8a6b7b0
fix vet
iamzhoug37 Apr 17, 2019
85fa209
remove db
iamzhoug37 Apr 17, 2019
07ccf7b
Merge branch 'master' into plan-management-add-global-binding
zz-jason Apr 17, 2019
2bbb6d4
update cache
iamzhoug37 Apr 17, 2019
3e4539a
Merge branch 'plan-management-add-global-binding' of github.com:iamzh…
iamzhoug37 Apr 17, 2019
4fac6c3
make fmt
iamzhoug37 Apr 17, 2019
603074c
add licensed
iamzhoug37 Apr 17, 2019
2f10bb5
move import
iamzhoug37 Apr 17, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 64 additions & 15 deletions bindinfo/bind_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"fmt"
"os"
"testing"
"time"

. "github.com/pingcap/check"
"github.com/pingcap/parser"
Expand Down Expand Up @@ -115,22 +116,70 @@ func (s *testSuite) TestBindParse(c *C) {
sql := fmt.Sprintf(`INSERT INTO mysql.bind_info(original_sql,bind_sql,default_db,status,create_time,update_time,charset,collation) VALUES ('%s', '%s', '%s', '%s', NOW(), NOW(),'%s', '%s')`,
originSQL, bindSQL, defaultDb, status, charset, collation)
tk.MustExec(sql)
bindHandle := bindinfo.NewHandle()
bindCacheUpdater := bindinfo.NewBindCacheUpdater(tk.Se, bindHandle, s.Parser)
err := bindCacheUpdater.Update(true)
bindHandle := bindinfo.NewBindHandle(tk.Se, s.Parser)
err := bindHandle.Update(true)
c.Check(err, IsNil)
c.Check(len(bindHandle.Get()), Equals, 1)
c.Check(bindHandle.Size(), Equals, 1)

hash := parser.DigestHash("select * from t")
bindData := bindHandle.Get()[hash]
bindData := bindHandle.GetBindRecord("select * from t", "test")
c.Check(bindData, NotNil)
c.Check(len(bindData), Equals, 1)
c.Check(bindData[0].OriginalSQL, Equals, "select * from t")
c.Check(bindData[0].BindSQL, Equals, "select * from t use index(index_t)")
c.Check(bindData[0].Db, Equals, "test")
c.Check(bindData[0].Status, Equals, "using")
c.Check(bindData[0].Charset, Equals, "utf8mb4")
c.Check(bindData[0].Collation, Equals, "utf8mb4_bin")
c.Check(bindData[0].CreateTime, NotNil)
c.Check(bindData[0].UpdateTime, NotNil)
c.Check(bindData.OriginalSQL, Equals, "select * from t")
c.Check(bindData.BindSQL, Equals, "select * from t use index(index_t)")
c.Check(bindData.Db, Equals, "test")
c.Check(bindData.Status, Equals, "using")
c.Check(bindData.Charset, Equals, "utf8mb4")
c.Check(bindData.Collation, Equals, "utf8mb4_bin")
c.Check(bindData.CreateTime, NotNil)
c.Check(bindData.UpdateTime, NotNil)
}

func (s *testSuite) TestGlobalBinding(c *C) {
tk := testkit.NewTestKit(c, s.store)
s.cleanBindingEnv(tk)
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
tk.MustExec("drop table if exists t1")
tk.MustExec("create table t(i int, s varchar(20))")
tk.MustExec("create table t1(i int, s varchar(20))")
tk.MustExec("create index index_t on t(i,s)")

_, err := tk.Exec("create global binding for select * from t where i>100 using select * from t use index(index_t) where i>100")
c.Assert(err, IsNil, Commentf("err %v", err))

time.Sleep(time.Second * 1)
_, err = tk.Exec("create global binding for select * from t where i>99 using select * from t use index(index_t) where i>99")
c.Assert(err, IsNil)

bindData := s.domain.BindHandle().GetBindRecord("select * from t where i > ?", "test")
c.Check(bindData, NotNil)
c.Check(bindData.OriginalSQL, Equals, "select * from t where i > ?")
c.Check(bindData.BindSQL, Equals, "select * from t use index(index_t) where i>99")
c.Check(bindData.Db, Equals, "test")
c.Check(bindData.Status, Equals, "using")
c.Check(bindData.Charset, NotNil)
c.Check(bindData.Collation, NotNil)
c.Check(bindData.CreateTime, NotNil)
c.Check(bindData.UpdateTime, NotNil)

bindHandle := bindinfo.NewBindHandle(tk.Se, s.Parser)
err = bindHandle.Update(true)
c.Check(err, IsNil)
c.Check(bindHandle.Size(), Equals, 1)

bindData = bindHandle.GetBindRecord("select * from t where i > ?", "test")
c.Check(bindData, NotNil)
c.Check(bindData.OriginalSQL, Equals, "select * from t where i > ?")
c.Check(bindData.BindSQL, Equals, "select * from t use index(index_t) where i>99")
c.Check(bindData.Db, Equals, "test")
c.Check(bindData.Status, Equals, "using")
c.Check(bindData.Charset, NotNil)
c.Check(bindData.Collation, NotNil)
c.Check(bindData.CreateTime, NotNil)
c.Check(bindData.UpdateTime, NotNil)

_, err = tk.Exec("delete from mysql.bind_info")
c.Assert(err, IsNil)

_, err = tk.Exec("create global binding for select * from t using select * from t1 use index for join(index_t)")
c.Assert(err, NotNil, Commentf("err %v", err))
}
149 changes: 10 additions & 139 deletions bindinfo/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,145 +14,44 @@
package bindinfo

import (
"context"
"fmt"
"sync/atomic"

"github.com/pingcap/errors"
"github.com/pingcap/parser"
"github.com/pingcap/parser/ast"
"github.com/pingcap/parser/terror"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/chunk"
"github.com/pingcap/tidb/util/sqlexec"
)

const (
using = "using"
// using is the bind info's in use status.
using = "using"
// deleted is the bind info's deleted status.
deleted = "deleted"
)

// bindMeta stores the basic bind info and bindSql astNode.
type bindMeta struct {
*bindRecord
*BindRecord
ast ast.StmtNode //ast will be used to do query sql bind check
}

// cache is a k-v map, key is original sql, value is a slice of bindMeta.
type cache map[string][]*bindMeta

// Handle holds an atomic cache.
type Handle struct {
atomic.Value
}

// BindCacheUpdater is used to update the global cache.
// BindCacheUpdater will update the bind cache per 3 seconds in domain
// gorountine loop. When the tidb server first startup, the updater will load
// all bind info into memory; then load diff bind info per 3 second.
type BindCacheUpdater struct {
ctx sessionctx.Context

parser *parser.Parser
lastUpdateTime types.Time
globalHandle *Handle
}

type bindRecord struct {
// BindRecord represents a sql bind record retrieved from the storage.
type BindRecord struct {
OriginalSQL string
BindSQL string
Db string
// Status represents the status of the binding. It can only be one of the following values:
// 1. deleted: bindRecord is deleted, can not be used anymore.
// 2. using: bindRecord is in the normal active mode.
// 1. deleted: BindRecord is deleted, can not be used anymore.
// 2. using: BindRecord is in the normal active mode.
Status string
CreateTime types.Time
UpdateTime types.Time
Charset string
Collation string
}

// NewBindCacheUpdater creates a new BindCacheUpdater.
func NewBindCacheUpdater(ctx sessionctx.Context, handle *Handle, parser *parser.Parser) *BindCacheUpdater {
return &BindCacheUpdater{
ctx: ctx,
parser: parser,
globalHandle: handle,
}
}

// NewHandle creates a Handle with a cache.
func NewHandle() *Handle {
handle := &Handle{}
return handle
}

// Get gets cache from a Handle.
func (h *Handle) Get() cache {
bc := h.Load()
if bc != nil {
return bc.(map[string][]*bindMeta)
}
return make(map[string][]*bindMeta)
}

// LoadDiff is used to load new bind info to cache bc.
func (bindCacheUpdater *BindCacheUpdater) loadDiff(sql string, bc cache) error {
recordSets, err := bindCacheUpdater.ctx.(sqlexec.SQLExecutor).Execute(context.Background(), sql)
if err != nil {
return errors.Trace(err)
}

rs := recordSets[0]
defer terror.Call(rs.Close)
chkBatch := rs.NewRecordBatch()
for {
err = rs.Next(context.TODO(), chkBatch)
if err != nil || chkBatch.NumRows() == 0 {
return errors.Trace(err)
}

it := chunk.NewIterator4Chunk(chkBatch.Chunk)
for row := it.Begin(); row != it.End(); row = it.Next() {
record := newBindMeta(row)
err = bc.appendNode(record, bindCacheUpdater.parser)
if err != nil {
return err
}
if record.UpdateTime.Compare(bindCacheUpdater.lastUpdateTime) == 1 {
bindCacheUpdater.lastUpdateTime = record.UpdateTime
}
}
}
}

// Update updates the BindCacheUpdater's cache.
// The `fullLoad` is true only when tidb first startup, otherwise it is false.
func (bindCacheUpdater *BindCacheUpdater) Update(fullLoad bool) (err error) {
var sql string
bc := bindCacheUpdater.globalHandle.Get()
newBc := make(map[string][]*bindMeta, len(bc))
for hash, bindDataArr := range bc {
newBc[hash] = append(newBc[hash], bindDataArr...)
}

if fullLoad {
sql = "select original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation from mysql.bind_info"
} else {
sql = fmt.Sprintf("select original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation from mysql.bind_info where update_time > \"%s\"", bindCacheUpdater.lastUpdateTime.String())
}
err = bindCacheUpdater.loadDiff(sql, newBc)
if err != nil {
return errors.Trace(err)
}

bindCacheUpdater.globalHandle.Store(newBc)
return nil
}

func newBindMeta(row chunk.Row) *bindRecord {
return &bindRecord{
func newBindRecord(row chunk.Row) *BindRecord {
return &BindRecord{
OriginalSQL: row.GetString(0),
BindSQL: row.GetString(1),
Db: row.GetString(2),
Expand All @@ -163,31 +62,3 @@ func newBindMeta(row chunk.Row) *bindRecord {
Collation: row.GetString(7),
}
}

func (b cache) appendNode(newBindRecord *bindRecord, sparser *parser.Parser) error {
hash := parser.DigestHash(newBindRecord.OriginalSQL)
if bindArr, ok := b[hash]; ok {
for idx, v := range bindArr {
if v.OriginalSQL == newBindRecord.OriginalSQL && v.Db == newBindRecord.Db {
b[hash] = append(b[hash][:idx], b[hash][idx+1:]...)
if len(b[hash]) == 0 {
delete(b, hash)
}
break
}
}
}
if newBindRecord.Status == deleted {
return nil
}
stmtNodes, _, err := sparser.Parse(newBindRecord.BindSQL, newBindRecord.Charset, newBindRecord.Collation)
if err != nil {
return errors.Trace(err)
}
newNode := &bindMeta{
bindRecord: newBindRecord,
ast: stmtNodes[0],
}
b[hash] = append(b[hash], newNode)
return nil
}
Loading