diff --git a/config/config.go b/config/config.go index 606fcfa4b0856..1f7005889551c 100644 --- a/config/config.go +++ b/config/config.go @@ -223,6 +223,12 @@ type TiKVClient struct { // GrpcConnectionCount is the max gRPC connections that will be established // with each tikv-server. GrpcConnectionCount uint `toml:"grpc-connection-count" json:"grpc-connection-count"` + // After a duration of this time in seconds if the client doesn't see any activity it pings + // the server to see if the transport is still alive. + GrpcKeepAliveTime uint `toml:"grpc-keepalive-time" json:"grpc-keepalive-time"` + // After having pinged for keepalive check, the client waits for a duration of Timeout in seconds + // and if no activity is seen even after that the connection is closed. + GrpcKeepAliveTimeout uint `toml:"grpc-keepalive-timeout" json:"grpc-keepalive-timeout"` // CommitTimeout is the max time which command 'commit' will wait. CommitTimeout string `toml:"commit-timeout" json:"commit-timeout"` } @@ -301,8 +307,10 @@ var defaultConf = Config{ Reporter: OpenTracingReporter{}, }, TiKVClient: TiKVClient{ - GrpcConnectionCount: 16, - CommitTimeout: "41s", + GrpcConnectionCount: 16, + GrpcKeepAliveTime: 10, + GrpcKeepAliveTimeout: 3, + CommitTimeout: "41s", }, Binlog: Binlog{ WriteTimeout: "15s", diff --git a/config/config.toml.example b/config/config.toml.example index 845cd4ceb3cc3..cda83ca9233db 100644 --- a/config/config.toml.example +++ b/config/config.toml.example @@ -210,6 +210,14 @@ local-agent-host-port = "" # Max gRPC connections that will be established with each tikv-server. grpc-connection-count = 16 +# After a duration of this time in seconds if the client doesn't see any activity it pings +# the server to see if the transport is still alive. +grpc-keepalive-time = 10 + +# After having pinged for keepalive check, the client waits for a duration of Timeout in seconds +# and if no activity is seen even after that the connection is closed. +grpc-keepalive-timeout = 3 + # max time for commit command, must be twice bigger than raft election timeout. commit-timeout = "41s" diff --git a/store/tikv/client.go b/store/tikv/client.go index a4a1886932271..25818b60b8133 100644 --- a/store/tikv/client.go +++ b/store/tikv/client.go @@ -35,12 +35,22 @@ import ( "golang.org/x/net/context" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/keepalive" ) // MaxConnectionCount is the max gRPC connections that will be established with // each tikv-server. var MaxConnectionCount uint = 16 +// GrpcKeepAliveTime is the duration of time after which if the client doesn't see +// any activity it pings the server to see if the transport is still alive. +var GrpcKeepAliveTime = time.Duration(10) * time.Second + +// GrpcKeepAliveTimeout is the duration of time for which the client waits after having +// pinged for keepalive check and if no activity is seen even after that the connection +// is closed. +var GrpcKeepAliveTimeout = time.Duration(3) * time.Second + // MaxSendMsgSize set max gRPC request message size sent to server. If any request message size is larger than // current value, an error will be reported from gRPC. var MaxSendMsgSize = 1<<31 - 1 @@ -126,6 +136,11 @@ func (a *connArray) Init(addr string, security config.Security) error { grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(MaxCallMsgSize)), grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(MaxSendMsgSize)), grpc.WithBackoffMaxDelay(time.Second*3), + grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: GrpcKeepAliveTime, + Timeout: GrpcKeepAliveTimeout, + PermitWithoutStream: true, + }), ) cancel() if err != nil { diff --git a/tidb-server/main.go b/tidb-server/main.go index 936a615f00f12..c9d957434c364 100644 --- a/tidb-server/main.go +++ b/tidb-server/main.go @@ -392,6 +392,8 @@ func setGlobalVars() { if cfg.TiKVClient.GrpcConnectionCount > 0 { tikv.MaxConnectionCount = cfg.TiKVClient.GrpcConnectionCount } + tikv.GrpcKeepAliveTime = time.Duration(cfg.TiKVClient.GrpcKeepAliveTime) * time.Second + tikv.GrpcKeepAliveTimeout = time.Duration(cfg.TiKVClient.GrpcKeepAliveTimeout) * time.Second // set lower_case_table_names variable.SysVars["lower_case_table_names"].Value = strconv.Itoa(cfg.LowerCaseTableNames)