Merge branch 'master' of https://github.com/pingcap/tidb into trace-w…

…rite-memory

docs/QUICKSTART.md

@@ -23,7 +23,7 @@ Notice: OS X user may use `docker-machine ip` to connect it.
 
 #### __Or run TiDB on TiKV cluster__ 
 
-Read the documents for [Ansible deployment](https://github.com/pingcap/docs/blob/master/op-guide/ansible-deployment.md) or [docker deployment](https://github.com/pingcap/docs/blob/master/op-guide/docker-deployment.md).
+Read the documents for [Ansible deployment](https://pingcap.com/docs/stable/how-to/deploy/orchestrated/ansible/) or [Docker deployment](https://pingcap.com/docs/stable/how-to/deploy/orchestrated/docker/).
 
 #### __Pre-requirement__
 

domain/domain.go

@@ -576,6 +576,7 @@ func (do *Domain) Close() {
 	if do.etcdClient != nil {
 		terror.Log(errors.Trace(do.etcdClient.Close()))
 	}
+
 	do.sysSessionPool.Close()
 	do.slowQuery.Close()
 	do.wg.Wait()
@@ -885,6 +886,7 @@ func (do *Domain) handleEvolvePlanTasksLoop(ctx sessionctx.Context) {
 		for {
 			select {
 			case <-do.exit:
+				owner.Cancel()
 				return
 			case <-time.After(bindinfo.Lease):
 			}
@@ -1023,6 +1025,7 @@ func (do *Domain) updateStatsWorker(ctx sessionctx.Context, owner owner.Manager)
 		select {
 		case <-do.exit:
 			statsHandle.FlushStats()
+			owner.Cancel()
 			return
 			// This channel is sent only by ddl owner.
 		case t := <-statsHandle.DDLEventCh():

executor/distsql.go

@@ -207,8 +207,6 @@ type IndexReaderExecutor struct {
 	table           table.Table
 	index           *model.IndexInfo
 	physicalTableID int64
-	keepOrder       bool
-	desc            bool
 	ranges          []*ranger.Range
 	// kvRanges are only used for union scan.
 	kvRanges []kv.KeyRange
@@ -221,8 +219,12 @@ type IndexReaderExecutor struct {
 	columns []*model.ColumnInfo
 	// outputColumns are only required by union scan.
 	outputColumns []*expression.Column
-	streaming     bool
-	feedback      *statistics.QueryFeedback
+
+	feedback  *statistics.QueryFeedback
+	streaming bool
+
+	keepOrder bool
+	desc      bool
 
 	corColInFilter bool
 	corColInAccess bool
@@ -317,30 +319,22 @@ func (e *IndexReaderExecutor) open(ctx context.Context, kvRanges []kv.KeyRange)
 type IndexLookUpExecutor struct {
 	baseExecutor
 
-	table     table.Table
-	index     *model.IndexInfo
-	keepOrder bool
-	desc      bool
-	ranges    []*ranger.Range
-	dagPB     *tipb.DAGRequest
-	startTS   uint64
+	table   table.Table
+	index   *model.IndexInfo
+	ranges  []*ranger.Range
+	dagPB   *tipb.DAGRequest
+	startTS uint64
 	// handleIdx is the index of handle, which is only used for case of keeping order.
 	handleIdx    int
 	tableRequest *tipb.DAGRequest
 	// columns are only required by union scan.
-	columns        []*model.ColumnInfo
-	indexStreaming bool
-	tableStreaming bool
+	columns []*model.ColumnInfo
 	*dataReaderBuilder
 	// All fields above are immutable.
-
 	idxWorkerWg sync.WaitGroup
 	tblWorkerWg sync.WaitGroup
 	finished    chan struct{}
 
-	kvRanges      []kv.KeyRange
-	workerStarted bool
-
 	resultCh   chan *lookupTableTask
 	resultCurr *lookupTableTask
 	feedback   *statistics.QueryFeedback
@@ -351,11 +345,20 @@ type IndexLookUpExecutor struct {
 	// checkIndexValue is used to check the consistency of the index data.
 	*checkIndexValue
 
+	kvRanges      []kv.KeyRange
+	workerStarted bool
+
+	keepOrder bool
+	desc      bool
+
+	indexStreaming bool
+	tableStreaming bool
+
 	corColInIdxSide bool
-	idxPlans        []plannercore.PhysicalPlan
 	corColInTblSide bool
-	tblPlans        []plannercore.PhysicalPlan
 	corColInAccess  bool
+	idxPlans        []plannercore.PhysicalPlan
+	tblPlans        []plannercore.PhysicalPlan
 	idxCols         []*expression.Column
 	colLens         []int
 	// PushedLimit is used to skip the preceding and tailing handles when Limit is sunk into IndexLookUpReader.

expression/builtin_op_vec.go

@@ -488,13 +488,19 @@ func (b *builtinUnaryMinusIntSig) vecEvalInt(input *chunk.Chunk, result *chunk.C
 	args := result.Int64s()
 	if mysql.HasUnsignedFlag(b.args[0].GetType().Flag) {
 		for i := 0; i < n; i++ {
+			if result.IsNull(i) {
+				continue
+			}
 			if uint64(args[i]) > uint64(-math.MinInt64) {
 				return types.ErrOverflow.GenWithStackByArgs("BIGINT", fmt.Sprintf("-%v", uint64(args[i])))
 			}
 			args[i] = -args[i]
 		}
 	} else {
 		for i := 0; i < n; i++ {
+			if result.IsNull(i) {
+				continue
+			}
 			if args[i] == math.MinInt64 {
 				return types.ErrOverflow.GenWithStackByArgs("BIGINT", fmt.Sprintf("-%v", args[i]))
 			}

expression/builtin_op_vec_test.go

@@ -21,6 +21,8 @@ import (
 	"github.com/pingcap/parser/ast"
 	"github.com/pingcap/parser/mysql"
 	"github.com/pingcap/tidb/types"
+	"github.com/pingcap/tidb/util/chunk"
+	"github.com/pingcap/tidb/util/mock"
 )
 
 var vecBuiltinOpCases = map[string][]vecExprBenchCase{
@@ -152,3 +154,37 @@ func (s *testEvaluatorSuite) TestVectorizedBuiltinOpFunc(c *C) {
 func BenchmarkVectorizedBuiltinOpFunc(b *testing.B) {
 	benchmarkVectorizedBuiltinFunc(b, vecBuiltinOpCases)
 }
+
+func (s *testEvaluatorSuite) TestBuiltinUnaryMinusIntSig(c *C) {
+	ctx := mock.NewContext()
+	ft := eType2FieldType(types.ETInt)
+	col0 := &Column{RetType: ft, Index: 0}
+	f, err := funcs[ast.UnaryMinus].getFunction(ctx, []Expression{col0})
+	c.Assert(err, IsNil)
+	input := chunk.NewChunkWithCapacity([]*types.FieldType{ft}, 1024)
+	result := chunk.NewColumn(ft, 1024)
+
+	c.Assert(mysql.HasUnsignedFlag(col0.GetType().Flag), IsFalse)
+	input.AppendInt64(0, 233333)
+	c.Assert(f.vecEvalInt(input, result), IsNil)
+	c.Assert(result.GetInt64(0), Equals, int64(-233333))
+	input.Reset()
+	input.AppendInt64(0, math.MinInt64)
+	c.Assert(f.vecEvalInt(input, result), NotNil)
+	input.Column(0).SetNull(0, true)
+	c.Assert(f.vecEvalInt(input, result), IsNil)
+	c.Assert(result.IsNull(0), IsTrue)
+
+	col0.GetType().Flag |= mysql.UnsignedFlag
+	c.Assert(mysql.HasUnsignedFlag(col0.GetType().Flag), IsTrue)
+	input.Reset()
+	input.AppendUint64(0, 233333)
+	c.Assert(f.vecEvalInt(input, result), IsNil)
+	c.Assert(result.GetInt64(0), Equals, int64(-233333))
+	input.Reset()
+	input.AppendUint64(0, -(math.MinInt64)+1)
+	c.Assert(f.vecEvalInt(input, result), NotNil)
+	input.Column(0).SetNull(0, true)
+	c.Assert(f.vecEvalInt(input, result), IsNil)
+	c.Assert(result.IsNull(0), IsTrue)
+}

kv/kv.go

@@ -194,10 +194,12 @@ type Transaction interface {
 
 // LockCtx contains information for LockKeys method.
 type LockCtx struct {
-	Killed        *uint32
-	ForUpdateTS   uint64
-	LockWaitTime  int64
-	WaitStartTime time.Time
+	Killed                *uint32
+	ForUpdateTS           uint64
+	LockWaitTime          int64
+	WaitStartTime         time.Time
+	PessimisticLockWaited int32
+	LockTimeWaited        time.Duration
 }
 
 // Client is used to send request to KV layer.

metrics/tikvclient.go

@@ -232,4 +232,12 @@ var (
 			Help:      "Bucketed histogram of the txn_heartbeat request duration.",
 			Buckets:   prometheus.ExponentialBuckets(0.001, 2, 18), // 1ms ~ 292s
 		}, []string{LblType})
+	TiKVPessimisticLockKeysDuration = prometheus.NewHistogram(
+		prometheus.HistogramOpts{
+			Namespace: "tidb",
+			Subsystem: "tikvclient",
+			Name:      "pessimistic_lock_keys_duration",
+			Buckets:   prometheus.ExponentialBuckets(0.001, 2, 24), // 1ms ~ 16777s
+			Help:      "tidb txn pessimistic lock keys duration",
+		})
 )

session/isolation_test.go

@@ -14,55 +14,14 @@
 package session_test
 
 import (
-	"time"
-
 	. "github.com/pingcap/check"
-	"github.com/pingcap/tidb/domain"
-	"github.com/pingcap/tidb/kv"
-	"github.com/pingcap/tidb/session"
-	"github.com/pingcap/tidb/store/mockstore"
-	"github.com/pingcap/tidb/store/mockstore/mocktikv"
 	"github.com/pingcap/tidb/util/testkit"
-	"github.com/pingcap/tidb/util/testleak"
 )
 
 var _ = Suite(&testIsolationSuite{})
 
 type testIsolationSuite struct {
-	cluster   *mocktikv.Cluster
-	mvccStore mocktikv.MVCCStore
-	store     kv.Storage
-	dom       *domain.Domain
-}
-
-func (s *testIsolationSuite) SetUpSuite(c *C) {
-	testleak.BeforeTest()
-	s.cluster = mocktikv.NewCluster()
-	mocktikv.BootstrapWithSingleStore(s.cluster)
-	s.mvccStore = mocktikv.MustNewMVCCStore()
-	store, err := mockstore.NewMockTikvStore(
-		mockstore.WithCluster(s.cluster),
-		mockstore.WithMVCCStore(s.mvccStore),
-	)
-	c.Assert(err, IsNil)
-	s.store = store
-	session.SetSchemaLease(0)
-	session.DisableStats4Test()
-	s.dom, err = session.BootstrapSession(s.store)
-	c.Assert(err, IsNil)
-
-	tk := testkit.NewTestKitWithInit(c, s.store)
-	tk.MustExec("set @@global.tidb_retry_limit = 0")
-	time.Sleep(3 * time.Second)
-}
-
-func (s *testIsolationSuite) TearDownSuite(c *C) {
-	tk := testkit.NewTestKitWithInit(c, s.store)
-	tk.MustExec("set @@global.tidb_retry_limit = 10")
-
-	s.dom.Close()
-	s.store.Close()
-	testleak.AfterTest(c)()
+	testSessionSuiteBase
 }
 
 /*

session/pessimistic_test.go

@@ -24,52 +24,29 @@ import (
 	"github.com/pingcap/failpoint"
 	"github.com/pingcap/parser/mysql"
 	"github.com/pingcap/parser/terror"
-	"github.com/pingcap/tidb/domain"
 	"github.com/pingcap/tidb/kv"
 	"github.com/pingcap/tidb/session"
-	"github.com/pingcap/tidb/store/mockstore"
-	"github.com/pingcap/tidb/store/mockstore/mocktikv"
 	"github.com/pingcap/tidb/store/tikv"
 	"github.com/pingcap/tidb/tablecodec"
 	"github.com/pingcap/tidb/util/codec"
 	"github.com/pingcap/tidb/util/testkit"
-	"github.com/pingcap/tidb/util/testleak"
 )
 
 var _ = SerialSuites(&testPessimisticSuite{})
 
 type testPessimisticSuite struct {
-	cluster   *mocktikv.Cluster
-	mvccStore mocktikv.MVCCStore
-	store     kv.Storage
-	dom       *domain.Domain
+	testSessionSuiteBase
 }
 
 func (s *testPessimisticSuite) SetUpSuite(c *C) {
-	testleak.BeforeTest()
+	s.testSessionSuiteBase.SetUpSuite(c)
 	// Set it to 300ms for testing lock resolve.
 	tikv.ManagedLockTTL = 300
 	tikv.PrewriteMaxBackoff = 500
-	s.cluster = mocktikv.NewCluster()
-	mocktikv.BootstrapWithSingleStore(s.cluster)
-	s.mvccStore = mocktikv.MustNewMVCCStore()
-	store, err := mockstore.NewMockTikvStore(
-		mockstore.WithCluster(s.cluster),
-		mockstore.WithMVCCStore(s.mvccStore),
-	)
-	c.Assert(err, IsNil)
-	s.store = store
-	session.SetSchemaLease(0)
-	session.DisableStats4Test()
-	s.dom, err = session.BootstrapSession(s.store)
-	s.dom.GetGlobalVarsCache().Disable()
-	c.Assert(err, IsNil)
 }
 
 func (s *testPessimisticSuite) TearDownSuite(c *C) {
-	s.dom.Close()
-	s.store.Close()
-	testleak.AfterTest(c)()
+	s.testSessionSuiteBase.TearDownSuite(c)
 	tikv.PrewriteMaxBackoff = 20000
 }
 
@@ -200,6 +177,9 @@ func (s *testPessimisticSuite) TestDeadlock(c *C) {
 }
 
 func (s *testPessimisticSuite) TestSingleStatementRollback(c *C) {
+	if *withTiKV {
+		c.Skip("skip with tikv because cluster manipulate is not available")
+	}
 	tk := testkit.NewTestKitWithInit(c, s.store)
 	tk2 := testkit.NewTestKitWithInit(c, s.store)
 
@@ -567,6 +547,7 @@ func (s *testPessimisticSuite) TestWaitLockKill(c *C) {
 	tk.MustExec("create table test_kill (id int primary key, c int)")
 	tk.MustExec("insert test_kill values (1, 1)")
 	tk.MustExec("begin pessimistic")
+	tk2.MustExec("set innodb_lock_wait_timeout = 50")
 	tk2.MustExec("begin pessimistic")
 	tk.MustQuery("select * from test_kill where id = 1 for update")