Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

infoschema,planner: implement the CLUSTER_LOG memory table predicate push down #14018

Merged
merged 18 commits into from
Dec 16, 2019
Merged

infoschema,planner: implement the CLUSTER_LOG memory table predicate push down #14018

Show file tree
Hide file tree
Changes from 16 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
74bd94b
implement the CLUSTER_LOG memory table predicate push down
lonng Dec 10, 2019
c6322db
change multiple pattern protocol
lonng Dec 11, 2019
c602fda
Update planner/core/memtable_predicate_extractor.go
lonng Dec 11, 2019
d508a1b
update sysutil
lonng Dec 11, 2019
98be3ad
Merge remote-tracking branch 'origin/master' into cluster-log-push-down
lonng Dec 11, 2019
02a3d27
fix CI
lonng Dec 11, 2019
a49f2cb
Merge remote-tracking branch 'origin/master' into cluster-log-push-down
lonng Dec 11, 2019
ad8d1b1
address comment
lonng Dec 11, 2019
1395624
add some comments
lonng Dec 11, 2019
aae31ee
fix time fraction
lonng Dec 11, 2019
022211d
Merge remote-tracking branch 'origin/master' into cluster-log-push-down
lonng Dec 11, 2019
f7df50c
Merge branch 'master' into cluster-log-push-down
crazycs520 Dec 12, 2019
00fe80b
address comment
lonng Dec 14, 2019
6321029
Merge branch 'cluster-log-push-down' of github.com:lonng/tidb into cl…
lonng Dec 14, 2019
e993168
fix time range extraction
lonng Dec 14, 2019
f9d1d17
Merge branch 'master' into cluster-log-push-down
lonng Dec 14, 2019
861b205
Merge branch 'master' into cluster-log-push-down
crazycs520 Dec 16, 2019
0047748
Merge branch 'master' into cluster-log-push-down
crazycs520 Dec 16, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions executor/cluster_reader_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,11 +174,9 @@ func (s *testClusterReaderSuite) TestTiDBClusterConfig(c *C) {
rows["pd"][2],
),
},
// FIXME: the OR does not extract in current implementation, it equals IN ('pd', 'tikv')
// and this filter are handled in Selection executor
{
sql: "select * from information_schema.cluster_config where type='pd' or type='tikv'",
reqCount: 9,
reqCount: 6,
rows: flatten(
rows["tikv"][0],
rows["tikv"][1],
Expand Down
8 changes: 4 additions & 4 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,15 @@ require (
github.com/opentracing/basictracer-go v1.0.0
github.com/opentracing/opentracing-go v1.0.2
github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4
github.com/pingcap/errors v0.11.4
github.com/pingcap/failpoint v0.0.0-20190512135322-30cc7431d99c
github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011
github.com/pingcap/failpoint v0.0.0-20191029060244-12f4ac2fd11d
github.com/pingcap/fn v0.0.0-20191016082858-07623b84a47d
github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e
github.com/pingcap/kvproto v0.0.0-20191202044712-32be31591b03
github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107
github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9
github.com/pingcap/parser v0.0.0-20191210060830-bdf23a7ade01
github.com/pingcap/pd v1.1.0-beta.0.20191210055626-676ddd3fbd2d
github.com/pingcap/sysutil v0.0.0-20191126040022-986c5b3ed9a3
github.com/pingcap/sysutil v0.0.0-20191211055351-5140a3fd4e53
github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible
github.com/pingcap/tipb v0.0.0-20191209145133-44f75c9bef33
github.com/prometheus/client_golang v1.0.0
Expand Down
18 changes: 11 additions & 7 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,27 +175,28 @@ github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUr
github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9 h1:KH4f4Si9XK6/IW50HtoaiLIFHGkapOM6w83za47UYik=
github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM=
github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011 h1:58naV4XMEqm0hl9LcYo6cZoGBGiLtefMQMF/vo3XLgQ=
github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pingcap/failpoint v0.0.0-20190512135322-30cc7431d99c h1:hvQd3aOLKLF7xvRV6DzvPkKY4QXzfVbjU1BhW0d9yL8=
github.com/pingcap/failpoint v0.0.0-20190512135322-30cc7431d99c/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI=
github.com/pingcap/failpoint v0.0.0-20191029060244-12f4ac2fd11d h1:F8vp38kTAckN+v8Jlc98uMBvKIzr1a+UhnLyVYn8Q5Q=
github.com/pingcap/failpoint v0.0.0-20191029060244-12f4ac2fd11d/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI=
github.com/pingcap/fn v0.0.0-20191016082858-07623b84a47d h1:rCmRK0lCRrHMUbS99BKFYhK9YxJDNw0xB033cQbYo0s=
github.com/pingcap/fn v0.0.0-20191016082858-07623b84a47d/go.mod h1:fMRU1BA1y+r89AxUoaAar4JjrhUkVDt0o0Np6V8XbDQ=
github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e h1:P73/4dPCL96rGrobssy1nVy2VaVpNCuLpCbr+FEaTA8=
github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw=
github.com/pingcap/kvproto v0.0.0-20191030021250-51b332bcb20b/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w=
github.com/pingcap/kvproto v0.0.0-20191121022655-4c654046831d/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w=
github.com/pingcap/kvproto v0.0.0-20191202044712-32be31591b03 h1:IyJl+qesVPf3UfFFmKtX69y1K5KC8uXlot3U0QgH7V4=
github.com/pingcap/kvproto v0.0.0-20191202044712-32be31591b03/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w=
github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107 h1:IXAs9fKFQZJVHf9cXWfUh8Nq8zPO9ihgPgNuU1j7bIo=
github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w=
github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw=
github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww9rkZ1DnWfl0pXV3BOWlkYIjA=
github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
github.com/pingcap/parser v0.0.0-20191210060830-bdf23a7ade01 h1:q1rGnV/296//bArDP7cDWWaSrhaeEKZY+gIo+Jb0Gyk=
github.com/pingcap/parser v0.0.0-20191210060830-bdf23a7ade01/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA=
github.com/pingcap/pd v1.1.0-beta.0.20191210055626-676ddd3fbd2d h1:0uUwCayeh1U4r1n7kb4gHf2ZbiI9fuVtXPCEpb2piBY=
github.com/pingcap/pd v1.1.0-beta.0.20191210055626-676ddd3fbd2d/go.mod h1:Z/VMtXHpkOP+MnYnk4TL5VHc3ZwO1qHwc89zDuf5n8Q=
github.com/pingcap/sysutil v0.0.0-20191126040022-986c5b3ed9a3 h1:HCNif3lukL83gNC2EBAoh2Qbz36+2p0bm0LjgnNfl1s=
github.com/pingcap/sysutil v0.0.0-20191126040022-986c5b3ed9a3/go.mod h1:Futrrmuw98pEsbEmoPsjw8aKLCmixwHEmT2rF+AsXGw=
github.com/pingcap/sysutil v0.0.0-20191211055351-5140a3fd4e53 h1:GJ3nF0k8mpWM2Q4JqBWl+n+TG3xiCWNccwQLFrv/3DQ=
github.com/pingcap/sysutil v0.0.0-20191211055351-5140a3fd4e53/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=
github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible h1:H1jg0aDWz2SLRh3hNBo2HFtnuHtudIUvBumU7syRkic=
github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
github.com/pingcap/tipb v0.0.0-20191209145133-44f75c9bef33 h1:cTSaVv1hue17BCPqt+sURADTFSMpSD26ZuvKRyYIjJs=
Expand Down Expand Up @@ -231,6 +232,8 @@ github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44 h1:tB9NOR21++IjLyV
github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shirou/gopsutil v2.19.10+incompatible h1:lA4Pi29JEVIQIgATSeftHSY0rMGI9CLrl2ZvDLiahto=
github.com/shirou/gopsutil v2.19.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U=
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371 h1:SWV2fHctRpRrp49VXJ6UZja7gU9QLHwRpIPBN89SKEo=
github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
github.com/shurcooL/vfsgen v0.0.0-20181020040650-a97a25d856ca h1:3fECS8atRjByijiI8yYiuwLwQ2ZxXobW7ua/8GRB3pI=
Expand Down Expand Up @@ -341,6 +344,7 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190909082730-f460065e899a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 h1:gSbV7h1NRL2G1xTg/owz62CST1oJBmxy4QpMMregXVQ=
Expand Down
14 changes: 13 additions & 1 deletion infoschema/tables.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,9 @@ const (
tableTiDBServersInfo = "TIDB_SERVERS_INFO"
tableClusterInfo = "CLUSTER_INFO"
// TableClusterConfig is the string constant of cluster configuration memory table
TableClusterConfig = "CLUSTER_CONFIG"
TableClusterConfig = "CLUSTER_CONFIG"
// TableClusterLog is the string constant of cluster log memory table
TableClusterLog = "CLUSTER_LOG"
tableClusterLoad = "CLUSTER_LOAD"
tableTiFlashReplica = "TIFLASH_REPLICA"
)
Expand Down Expand Up @@ -152,6 +154,7 @@ var tableIDMap = map[string]int64{
tableTiFlashReplica: autoid.InformationSchemaDBID + 45,
clusterTableSlowLog: autoid.InformationSchemaDBID + 46,
clusterTableProcesslist: autoid.InformationSchemaDBID + 47,
TableClusterLog: autoid.InformationSchemaDBID + 48,
}

type columnInfo struct {
Expand Down Expand Up @@ -736,6 +739,14 @@ var tableClusterConfigCols = []columnInfo{
{"VALUE", mysql.TypeVarchar, 128, 0, nil, nil},
}

var tableClusterLogCols = []columnInfo{
{"TIME", mysql.TypeVarchar, 32, 0, nil, nil},
{"TYPE", mysql.TypeVarchar, 64, 0, nil, nil},
{"ADDRESS", mysql.TypeVarchar, 64, 0, nil, nil},
{"LEVEL", mysql.TypeVarchar, 8, 0, nil, nil},
{"MESSAGE", mysql.TypeVarString, 1024, 0, nil, nil},
}

var tableClusterLoadCols = []columnInfo{
{"TYPE", mysql.TypeVarchar, 64, 0, nil, nil},
{"ADDRESS", mysql.TypeVarchar, 64, 0, nil, nil},
Expand Down Expand Up @@ -2292,6 +2303,7 @@ var tableNameToColumns = map[string][]columnInfo{
tableTiDBServersInfo: tableTiDBServersInfoCols,
tableClusterInfo: tableClusterInfoCols,
TableClusterConfig: tableClusterConfigCols,
TableClusterLog: tableClusterLogCols,
tableClusterLoad: tableClusterLoadCols,
tableTiFlashReplica: tableTableTiFlashReplicaCols,
}
Expand Down
2 changes: 1 addition & 1 deletion infoschema/tables_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ func (s *testClusterTableSuite) setUpRPCService(c *C, addr string) *grpc.Server
Host: "127.0.0.1",
Command: mysql.ComQuery,
}
srv := server.NewRPCServer(config.GetGlobalConfig().Security, s.dom, sm)
srv := server.NewRPCServer(config.GetGlobalConfig(), s.dom, sm)
go func() {
err = srv.Serve(lis)
c.Assert(err, IsNil)
Expand Down
2 changes: 2 additions & 0 deletions planner/core/logical_plan_builder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2717,6 +2717,8 @@ func (b *PlanBuilder) buildMemTable(ctx context.Context, dbName model.CIStr, tab
switch tableInfo.Name.L {
case strings.ToLower(infoschema.TableClusterConfig):
p.Extractor = &ClusterConfigTableExtractor{}
case strings.ToLower(infoschema.TableClusterLog):
p.Extractor = &ClusterLogTableExtractor{}
}

return p, nil
Expand Down
Loading