update master

Signed-off-by: bufferflies <1045931706@qq.com>
pingcap · Sep 4, 2023 · b82ab6e · b82ab6e
2 parents 1b97e7a + a427113
commit b82ab6e
Show file tree

Hide file tree

Showing 390 changed files with 10,542 additions and 6,746 deletions.
diff --git a/DEPS.bzl b/DEPS.bzl
@@ -6937,13 +6937,13 @@ def go_deps():
         name = "com_github_tikv_client_go_v2",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/tikv/client-go/v2",
-        sha256 = "8c8e0582553c9adc286a6ad54fc4f9bbc140f5b348ebfbb1fc51994ed73ef737",
-        strip_prefix = "github.com/tikv/client-go/v2@v2.0.8-0.20230830100950-5c9fcf3e8541",
+        sha256 = "9cf5877cb0b43d73140e280ad9c80dccd9684e89659a358ee75702469368cf95",
+        strip_prefix = "github.com/tikv/client-go/v2@v2.0.8-0.20230829002846-295094e5b534",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20230830100950-5c9fcf3e8541.zip",
-            "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20230830100950-5c9fcf3e8541.zip",
-            "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20230830100950-5c9fcf3e8541.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20230830100950-5c9fcf3e8541.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20230829002846-295094e5b534.zip",
+            "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20230829002846-295094e5b534.zip",
+            "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20230829002846-295094e5b534.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20230829002846-295094e5b534.zip",
         ],
     )
     go_repository(
@@ -10282,39 +10282,39 @@ def go_deps():
         name = "org_modernc_golex",
         build_file_proto_mode = "disable_global",
         importpath = "modernc.org/golex",
-        sha256 = "81ddbe54f9eca46effe625329bfb961ea5e63a8acc1c793e2ee1b8a61a770bf9",
-        strip_prefix = "modernc.org/golex@v1.0.5",
+        sha256 = "3099b4f7e65cc38d113d6558f2a223ba4ce5288a930b182ac6ef679a96dbcfe5",
+        strip_prefix = "modernc.org/golex@v1.1.0",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/golex/org_modernc_golex-v1.0.5.zip",
-            "http://ats.apps.svc/gomod/modernc.org/golex/org_modernc_golex-v1.0.5.zip",
-            "https://cache.hawkingrei.com/gomod/modernc.org/golex/org_modernc_golex-v1.0.5.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/golex/org_modernc_golex-v1.0.5.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/golex/org_modernc_golex-v1.1.0.zip",
+            "http://ats.apps.svc/gomod/modernc.org/golex/org_modernc_golex-v1.1.0.zip",
+            "https://cache.hawkingrei.com/gomod/modernc.org/golex/org_modernc_golex-v1.1.0.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/golex/org_modernc_golex-v1.1.0.zip",
         ],
     )
     go_repository(
         name = "org_modernc_mathutil",
         build_file_proto_mode = "disable_global",
         importpath = "modernc.org/mathutil",
-        sha256 = "c17a767eaa5eb62d9bb105b8ece7f249186dd52b9b533301bec140b3d5fd260f",
-        strip_prefix = "modernc.org/mathutil@v1.5.0",
+        sha256 = "3a9e2065897e172b4c092c3098e15a2d66bc2700432f88ba6812c1b6b0acf2b2",
+        strip_prefix = "modernc.org/mathutil@v1.6.0",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/mathutil/org_modernc_mathutil-v1.5.0.zip",
-            "http://ats.apps.svc/gomod/modernc.org/mathutil/org_modernc_mathutil-v1.5.0.zip",
-            "https://cache.hawkingrei.com/gomod/modernc.org/mathutil/org_modernc_mathutil-v1.5.0.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/mathutil/org_modernc_mathutil-v1.5.0.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/mathutil/org_modernc_mathutil-v1.6.0.zip",
+            "http://ats.apps.svc/gomod/modernc.org/mathutil/org_modernc_mathutil-v1.6.0.zip",
+            "https://cache.hawkingrei.com/gomod/modernc.org/mathutil/org_modernc_mathutil-v1.6.0.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/mathutil/org_modernc_mathutil-v1.6.0.zip",
         ],
     )
     go_repository(
         name = "org_modernc_parser",
         build_file_proto_mode = "disable_global",
         importpath = "modernc.org/parser",
-        sha256 = "fd46145315aac782cfe7199f58b01a88da814200857b136029aa635c55eff705",
-        strip_prefix = "modernc.org/parser@v1.0.7",
+        sha256 = "e50f98025a0dfff5ffe5fe5dba38a11f85e5402cdcb9ed7ed0a8d3db6d811b67",
+        strip_prefix = "modernc.org/parser@v1.1.0",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/parser/org_modernc_parser-v1.0.7.zip",
-            "http://ats.apps.svc/gomod/modernc.org/parser/org_modernc_parser-v1.0.7.zip",
-            "https://cache.hawkingrei.com/gomod/modernc.org/parser/org_modernc_parser-v1.0.7.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/parser/org_modernc_parser-v1.0.7.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/parser/org_modernc_parser-v1.1.0.zip",
+            "http://ats.apps.svc/gomod/modernc.org/parser/org_modernc_parser-v1.1.0.zip",
+            "https://cache.hawkingrei.com/gomod/modernc.org/parser/org_modernc_parser-v1.1.0.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/parser/org_modernc_parser-v1.1.0.zip",
         ],
     )
     go_repository(
@@ -10334,13 +10334,13 @@ def go_deps():
         name = "org_modernc_strutil",
         build_file_proto_mode = "disable_global",
         importpath = "modernc.org/strutil",
-        sha256 = "2e59915393fa6a75021a97a41c60fac71c662bb9d1dc2d06e2c4ed77ea5da8cc",
-        strip_prefix = "modernc.org/strutil@v1.1.3",
+        sha256 = "1ea20b81cf3fd6efad4bb1b791255cc2fd486111d5e9f2cb0e551e9d39aa3f8f",
+        strip_prefix = "modernc.org/strutil@v1.2.0",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/strutil/org_modernc_strutil-v1.1.3.zip",
-            "http://ats.apps.svc/gomod/modernc.org/strutil/org_modernc_strutil-v1.1.3.zip",
-            "https://cache.hawkingrei.com/gomod/modernc.org/strutil/org_modernc_strutil-v1.1.3.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/strutil/org_modernc_strutil-v1.1.3.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/strutil/org_modernc_strutil-v1.2.0.zip",
+            "http://ats.apps.svc/gomod/modernc.org/strutil/org_modernc_strutil-v1.2.0.zip",
+            "https://cache.hawkingrei.com/gomod/modernc.org/strutil/org_modernc_strutil-v1.2.0.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/strutil/org_modernc_strutil-v1.2.0.zip",
         ],
     )
     go_repository(

diff --git a/README.md b/README.md
@@ -28,6 +28,8 @@
     &nbsp;
     <a href="https://ask.pingcap.com/"><img src="https://img.shields.io/badge/- Forum -red?style=social&logo=discourse" height=25></a>
     &nbsp;
+    <a href="https://discord.gg/KVRZBR2DrG?utm_source=github"><img src="https://img.shields.io/badge/- Discord -red?style=social&logo=discord" height=25></a>
+    &nbsp;
     <a href="https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-tidb"><img src="https://img.shields.io/badge/-Slack-red?style=social&logo=slack" height=25></a>
     &nbsp;
     <a href="https://stackoverflow.com/questions/tagged/tidb"><img src="https://img.shields.io/badge/-Stack Overflow-red?style=social&logo=stackoverflow" height=25></a>
@@ -75,6 +77,7 @@ You can join the following groups or channels to discuss or ask questions about
 
 - Seek help when you use TiDB
   - TiDB Forum: [English](https://ask.pingcap.com/), [Chinese](https://asktug.com)
+  - [Discord](https://discord.gg/KVRZBR2DrG?utm_source=github)
   - Slack channels: [#everyone](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-tidb) (English), [#tidb-japan](https://slack.tidb.io/invite?team=tidb-community&channel=tidb-japan&ref=github-tidb) (Japanese)
   - [Stack Overflow](https://stackoverflow.com/questions/tagged/tidb) (questions tagged with #tidb)
 - Discuss TiDB's implementation and design

diff --git a/autoid_service/autoid.go b/autoid_service/autoid.go
@@ -190,6 +190,7 @@ func (alloc *autoIDValue) rebase4Unsigned(ctx context.Context,
 	}
 
 	var newBase, newEnd uint64
+	var oldValue int64
 	startTime := time.Now()
 	ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnMeta)
 	err := kv.RunInNewTxn(ctx, store, true, func(ctx context.Context, txn kv.Transaction) error {
@@ -198,6 +199,7 @@ func (alloc *autoIDValue) rebase4Unsigned(ctx context.Context,
 		if err1 != nil {
 			return err1
 		}
+		oldValue = currentEnd
 		uCurrentEnd := uint64(currentEnd)
 		newBase = mathutil.Max(uCurrentEnd, requiredBase)
 		newEnd = mathutil.Min(math.MaxUint64-uint64(batch), newBase) + uint64(batch)
@@ -208,6 +210,13 @@ func (alloc *autoIDValue) rebase4Unsigned(ctx context.Context,
 	if err != nil {
 		return err
 	}
+
+	logutil.BgLogger().Info("rebase4Unsigned from",
+		zap.String("category", "autoid service"),
+		zap.Int64("dbID", dbID),
+		zap.Int64("tblID", tblID),
+		zap.Int64("from", oldValue),
+		zap.Uint64("to", newEnd))
 	alloc.base, alloc.end = int64(newBase), int64(newEnd)
 	return nil
 }
@@ -223,22 +232,32 @@ func (alloc *autoIDValue) rebase4Signed(ctx context.Context, store kv.Storage, d
 		return nil
 	}
 
-	var newBase, newEnd int64
+	var oldValue, newBase, newEnd int64
+	startTime := time.Now()
 	ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnMeta)
 	err := kv.RunInNewTxn(ctx, store, true, func(ctx context.Context, txn kv.Transaction) error {
 		idAcc := meta.NewMeta(txn).GetAutoIDAccessors(dbID, tblID).IncrementID(model.TableInfoVersion5)
 		currentEnd, err1 := idAcc.Get()
 		if err1 != nil {
 			return err1
 		}
+		oldValue = currentEnd
 		newBase = mathutil.Max(currentEnd, requiredBase)
 		newEnd = mathutil.Min(math.MaxInt64-batch, newBase) + batch
 		_, err1 = idAcc.Inc(newEnd - currentEnd)
 		return err1
 	})
+	metrics.AutoIDHistogram.WithLabelValues(metrics.TableAutoIDRebase, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds())
 	if err != nil {
 		return err
 	}
+
+	logutil.BgLogger().Info("rebase4Signed from",
+		zap.Int64("dbID", dbID),
+		zap.Int64("tblID", tblID),
+		zap.Int64("from", oldValue),
+		zap.Int64("to", newEnd),
+		zap.String("category", "autoid service"))
 	alloc.base, alloc.end = newBase, newEnd
 	return nil
 }
@@ -271,7 +290,9 @@ func New(selfAddr string, etcdAddr []string, store kv.Storage, tlsConfig *tls.Co
 		},
 		TLS: tlsConfig,
 	})
-	etcd.SetEtcdCliByNamespace(cli, keyspace.MakeKeyspaceEtcdNamespace(store.GetCodec()))
+	if store.GetCodec().GetKeyspace() != nil {
+		etcd.SetEtcdCliByNamespace(cli, keyspace.MakeKeyspaceEtcdNamespaceSlash(store.GetCodec()))
+	}
 	if err != nil {
 		panic(err)
 	}
@@ -280,6 +301,11 @@ func New(selfAddr string, etcdAddr []string, store kv.Storage, tlsConfig *tls.Co
 
 func newWithCli(selfAddr string, cli *clientv3.Client, store kv.Storage) *Service {
 	l := owner.NewOwnerManager(context.Background(), cli, "autoid", selfAddr, autoIDLeaderPath)
+	l.SetBeOwnerHook(func() {
+		logutil.BgLogger().Info("leader change of autoid service, this node become owner",
+			zap.String("addr", selfAddr),
+			zap.String("category", "autoid service"))
+	})
 	err := l.CampaignOwner()
 	if err != nil {
 		panic(err)
@@ -325,7 +351,7 @@ func MockForTest(store kv.Storage) autoid.AutoIDAllocClient {
 
 // Close closes the Service and clean up resource.
 func (s *Service) Close() {
-	if s.leaderShip != nil {
+	if s.leaderShip != nil && s.leaderShip.IsOwner() {
 		for k, v := range s.autoIDMap {
 			if v.base > 0 {
 				err := v.forceRebase(context.Background(), s.store, k.dbID, k.tblID, v.base, v.isUnsigned)
@@ -438,7 +464,7 @@ func (s *Service) allocAutoID(ctx context.Context, req *autoid.AutoIDRequest) (*
 		var currentEnd int64
 		ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnMeta)
 		err := kv.RunInNewTxn(ctx, s.store, true, func(ctx context.Context, txn kv.Transaction) error {
-			idAcc := meta.NewMeta(txn).GetAutoIDAccessors(req.DbID, req.TblID).RowID()
+			idAcc := meta.NewMeta(txn).GetAutoIDAccessors(req.DbID, req.TblID).IncrementID(model.TableInfoVersion5)
 			var err1 error
 			currentEnd, err1 = idAcc.Get()
 			if err1 != nil {
@@ -478,12 +504,14 @@ func (s *Service) allocAutoID(ctx context.Context, req *autoid.AutoIDRequest) (*
 
 func (alloc *autoIDValue) forceRebase(ctx context.Context, store kv.Storage, dbID, tblID, requiredBase int64, isUnsigned bool) error {
 	ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnMeta)
+	var oldValue int64
 	err := kv.RunInNewTxn(ctx, store, true, func(ctx context.Context, txn kv.Transaction) error {
 		idAcc := meta.NewMeta(txn).GetAutoIDAccessors(dbID, tblID).IncrementID(model.TableInfoVersion5)
 		currentEnd, err1 := idAcc.Get()
 		if err1 != nil {
 			return err1
 		}
+		oldValue = currentEnd
 		var step int64
 		if !isUnsigned {
 			step = requiredBase - currentEnd
@@ -497,6 +525,13 @@ func (alloc *autoIDValue) forceRebase(ctx context.Context, store kv.Storage, dbI
 	if err != nil {
 		return err
 	}
+	logutil.BgLogger().Info("forceRebase from",
+		zap.Int64("dbID", dbID),
+		zap.Int64("tblID", tblID),
+		zap.Int64("from", oldValue),
+		zap.Int64("to", requiredBase),
+		zap.Bool("isUnsigned", isUnsigned),
+		zap.String("category", "autoid service"))
 	alloc.base, alloc.end = requiredBase, requiredBase
 	return nil
 }

diff --git a/br/pkg/lightning/backend/external/engine.go b/br/pkg/lightning/backend/external/engine.go
@@ -38,6 +38,7 @@ type Engine struct {
 	storage    storage.ExternalStorage
 	dataFiles  []string
 	statsFiles []string
+	splitKeys  [][]byte
 	bufPool    *membuf.Pool
 
 	iter *MergeKVIter
@@ -48,6 +49,9 @@ type Engine struct {
 	dupDetectOpt       common.DupDetectOpt
 	ts                 uint64
 
+	totalKVSize   int64
+	totalKVLength int64
+
 	importedKVSize  *atomic.Int64
 	importedKVCount *atomic.Int64
 }
@@ -62,6 +66,8 @@ func NewExternalEngine(
 	duplicateDB *pebble.DB,
 	dupDetectOpt common.DupDetectOpt,
 	ts uint64,
+	totalKVSize int64,
+	totakKVLength int64,
 ) common.Engine {
 	return &Engine{
 		storage:            storage,
@@ -73,6 +79,8 @@ func NewExternalEngine(
 		duplicateDB:        duplicateDB,
 		dupDetectOpt:       dupDetectOpt,
 		ts:                 ts,
+		totalKVSize:        totalKVSize,
+		totalKVLength:      totakKVLength,
 		importedKVSize:     atomic.NewInt64(0),
 		importedKVCount:    atomic.NewInt64(0),
 	}
@@ -169,6 +177,42 @@ func (e *Engine) createMergeIter(ctx context.Context, start kv.Key) (*MergeKVIte
 	return iter, nil
 }
 
+// KVStatistics returns the total kv size and total kv length.
+func (e *Engine) KVStatistics() (totalKVSize int64, totalKVLength int64) {
+	return e.totalKVSize, e.totalKVLength
+}
+
+// ImportedStatistics returns the imported kv size and imported kv length.
+func (e *Engine) ImportedStatistics() (importedKVSize int64, importedKVLength int64) {
+	return e.importedKVSize.Load(), e.importedKVCount.Load()
+}
+
+// ID is the identifier of an engine.
+func (e *Engine) ID() string {
+	return "external"
+}
+
+// SplitRanges split the ranges by split keys provided by external engine.
+func (e *Engine) SplitRanges(
+	startKey, endKey []byte,
+	_, _ int64,
+	_ log.Logger,
+) ([]common.Range, error) {
+	splitKeys := e.splitKeys
+	ranges := make([]common.Range, 0, len(splitKeys)+1)
+	ranges = append(ranges, common.Range{Start: startKey})
+	for i := 0; i < len(splitKeys); i++ {
+		ranges[len(ranges)-1].End = splitKeys[i]
+		var endK []byte
+		if i < len(splitKeys)-1 {
+			endK = splitKeys[i+1]
+		}
+		ranges = append(ranges, common.Range{Start: splitKeys[i], End: endK})
+	}
+	ranges[len(ranges)-1].End = endKey
+	return ranges, nil
+}
+
 // Close releases the resources of the engine.
 func (e *Engine) Close() error {
 	if e.iter == nil {

diff --git a/br/pkg/lightning/backend/kv/allocator.go b/br/pkg/lightning/backend/kv/allocator.go
@@ -34,6 +34,7 @@ type panickingAllocator struct {
 // we use this to collect the max id(either _tidb_rowid or auto_increment id or auto_random) used
 // during import, and we will use this info to do ALTER TABLE xxx AUTO_RANDOM_BASE or AUTO_INCREMENT
 // on post-process phase.
+// we share the same base among all allocators, so the AllocatorType doesn't matter here.
 func NewPanickingAllocators(base int64) autoid.Allocators {
 	sharedBase := &base
 	return autoid.NewAllocators(