Merge branch 'master' into master

Author: LemonHX

go.mod

@@ -34,7 +34,7 @@ require (
 	github.com/pingcap/kvproto v0.0.0-20211109071446-a8b4d34474bc
 	github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7
 	github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d
-	github.com/pingcap/tidb-dashboard v0.0.0-20211107164327-80363dfbe884
+	github.com/pingcap/tidb-dashboard v0.0.0-20211206031355-bcc43a01d537
 	github.com/prometheus/client_golang v1.1.0
 	github.com/prometheus/common v0.6.0
 	github.com/sasha-s/go-deadlock v0.2.0

go.sum

@@ -341,8 +341,8 @@ github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuR
 github.com/pingcap/sysutil v0.0.0-20210315073920-cc0985d983a3/go.mod h1:tckvA041UWP+NqYzrJ3fMgC/Hw9wnmQ/tUkp/JaHly8=
 github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d h1:k3/APKZjXOyJrFy8VyYwRlZhMelpD3qBLJNsw3bPl/g=
 github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d/go.mod h1:7j18ezaWTao2LHOyMlsc2Dg1vW+mDY9dEbPzVyOlaeM=
-github.com/pingcap/tidb-dashboard v0.0.0-20211107164327-80363dfbe884 h1:6/yOhY2X0oNidvVK1PdjVoSThfUQ+GK/1UaHeXZfnrM=
-github.com/pingcap/tidb-dashboard v0.0.0-20211107164327-80363dfbe884/go.mod h1:OCXbZTBTIMRcIt0jFsuCakZP+goYRv6IjawKbwLS2TQ=
+github.com/pingcap/tidb-dashboard v0.0.0-20211206031355-bcc43a01d537 h1:qoKNQJY2hQ9/+q+aD6SxrNX5cZ8A9XZyw6rQg7MbwlE=
+github.com/pingcap/tidb-dashboard v0.0.0-20211206031355-bcc43a01d537/go.mod h1:OCXbZTBTIMRcIt0jFsuCakZP+goYRv6IjawKbwLS2TQ=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

server/cluster/cluster.go

@@ -1077,10 +1077,9 @@ func (c *RaftCluster) RemoveStore(storeID uint64, physicallyDestroyed bool) erro
 	return err
 }
 
-// buryStore marks a store as tombstone in cluster.
-// The store should be empty before calling this func
-// State transition: Offline -> Tombstone.
-func (c *RaftCluster) buryStore(storeID uint64) error {
+// BuryStore marks a store as tombstone in cluster.
+// If forceBury is false, the store should be offlined and emptied before calling this func.
+func (c *RaftCluster) BuryStore(storeID uint64, forceBury bool) error {
 	c.Lock()
 	defer c.Unlock()
 
@@ -1095,7 +1094,11 @@ func (c *RaftCluster) buryStore(storeID uint64) error {
 	}
 
 	if store.IsUp() {
-		return errs.ErrStoreIsUp.FastGenByArgs()
+		if !forceBury {
+			return errs.ErrStoreIsUp.FastGenByArgs()
+		} else if !store.IsDisconnected() {
+			return errors.Errorf("The store %v is not offline nor disconnected", storeID)
+		}
 	}
 
 	newStore := store.Clone(core.TombstoneStore())
@@ -1219,7 +1222,7 @@ func (c *RaftCluster) checkStores() {
 		// If the store is empty, it can be buried.
 		regionCount := c.core.GetStoreRegionCount(offlineStore.GetId())
 		if regionCount == 0 {
-			if err := c.buryStore(offlineStore.GetId()); err != nil {
+			if err := c.BuryStore(offlineStore.GetId(), false); err != nil {
 				log.Error("bury store failed",
 					zap.Stringer("store", offlineStore),
 					errs.ZapError(err))
@@ -1248,7 +1251,7 @@ func (c *RaftCluster) RemoveTombStoneRecords() error {
 
 	for _, store := range c.GetStores() {
 		if store.IsTombstone() {
-			if store.GetRegionCount() > 0 {
+			if c.core.GetStoreRegionCount(store.GetID()) > 0 {
 				log.Warn("skip removing tombstone", zap.Stringer("store", store.GetMeta()))
 				continue
 			}

server/cluster/cluster_test.go

@@ -250,13 +250,28 @@ func (s *testClusterInfoSuite) TestSetOfflineStore(c *C) {
 	for storeID := uint64(0); storeID <= 4; storeID++ {
 		store := cluster.GetStore(storeID)
 		if store == nil || store.IsUp() {
-			c.Assert(cluster.buryStore(storeID), NotNil)
+			c.Assert(cluster.BuryStore(storeID, false), NotNil)
 		} else {
-			c.Assert(cluster.buryStore(storeID), IsNil)
+			c.Assert(cluster.BuryStore(storeID, false), IsNil)
 		}
 	}
 }
 
+func (s *testClusterInfoSuite) TestForceBuryStore(c *C) {
+	_, opt, err := newTestScheduleConfig()
+	c.Assert(err, IsNil)
+	cluster := newTestRaftCluster(s.ctx, mockid.NewIDAllocator(), opt, core.NewStorage(kv.NewMemoryKV()), core.NewBasicCluster())
+	// Put 2 stores.
+	stores := newTestStores(2, "5.3.0")
+	stores[1] = stores[1].Clone(core.SetLastHeartbeatTS(time.Now()))
+	for _, store := range stores {
+		c.Assert(cluster.PutStore(store.GetMeta()), IsNil)
+	}
+	c.Assert(cluster.BuryStore(uint64(1), true), IsNil)
+	c.Assert(cluster.BuryStore(uint64(2), true), NotNil)
+	c.Assert(errors.ErrorEqual(cluster.BuryStore(uint64(3), true), errs.ErrStoreNotFound.FastGenByArgs(uint64(3))), IsTrue)
+}
+
 func (s *testClusterInfoSuite) TestReuseAddress(c *C) {
 	_, opt, err := newTestScheduleConfig()
 	c.Assert(err, IsNil)
@@ -272,7 +287,7 @@ func (s *testClusterInfoSuite) TestReuseAddress(c *C) {
 	c.Assert(cluster.RemoveStore(3, true), IsNil)
 	// store 4: tombstone
 	c.Assert(cluster.RemoveStore(4, true), IsNil)
-	c.Assert(cluster.buryStore(4), IsNil)
+	c.Assert(cluster.BuryStore(4, false), IsNil)
 
 	for id := uint64(1); id <= 4; id++ {
 		storeInfo := cluster.GetStore(id)

server/cluster/unsafe_recovery_controller.go

@@ -27,6 +27,7 @@ import (
 	"github.com/pingcap/kvproto/pkg/metapb"
 	"github.com/pingcap/kvproto/pkg/pdpb"
 	"github.com/pingcap/log"
+	"github.com/tikv/pd/pkg/errs"
 	"github.com/tikv/pd/server/core"
 	"go.uber.org/zap"
 )
@@ -79,6 +80,18 @@ func (u *unsafeRecoveryController) RemoveFailedStores(failedStores map[uint64]st
 		return errors.Errorf("Another request is working in progress")
 	}
 	u.reset()
+	for failedStore := range failedStores {
+		store := u.cluster.GetStore(failedStore)
+		if store != nil && store.IsUp() && !store.IsDisconnected() {
+			return errors.Errorf("Store %v is up and connected", failedStore)
+		}
+	}
+	for failedStore := range failedStores {
+		err := u.cluster.BuryStore(failedStore, true)
+		if !errors.ErrorEqual(err, errs.ErrStoreNotFound.FastGenByArgs(failedStore)) {
+			return err
+		}
+	}
 	u.failedStores = failedStores
 	for _, s := range u.cluster.GetStores() {
 		if s.IsTombstone() || s.IsPhysicallyDestroyed() || core.IsStoreContainLabel(s.GetMeta(), core.EngineKey, core.EngineTiFlash) {

server/cluster/unsafe_recovery_controller_test.go

@@ -17,6 +17,7 @@ package cluster
 import (
 	"bytes"
 	"context"
+	"time"
 
 	. "github.com/pingcap/check"
 	"github.com/pingcap/kvproto/pkg/metapb"
@@ -617,3 +618,28 @@ func (s *testUnsafeRecoverSuite) TestPlanExecution(c *C) {
 	c.Assert(recoveryController.numStoresPlanExecuted, Equals, 2)
 	c.Assert(recoveryController.stage, Equals, finished)
 }
+
+func (s *testUnsafeRecoverSuite) TestRemoveFailedStores(c *C) {
+	_, opt, _ := newTestScheduleConfig()
+	cluster := newTestRaftCluster(s.ctx, mockid.NewIDAllocator(), opt, core.NewStorage(kv.NewMemoryKV()), core.NewBasicCluster())
+	stores := newTestStores(2, "5.3.0")
+	stores[1] = stores[1].Clone(core.SetLastHeartbeatTS(time.Now()))
+	for _, store := range stores {
+		c.Assert(cluster.PutStore(store.GetMeta()), IsNil)
+	}
+	recoveryController := newUnsafeRecoveryController(cluster)
+	failedStores := map[uint64]string{
+		1: "",
+		3: "",
+	}
+
+	c.Assert(recoveryController.RemoveFailedStores(failedStores), IsNil)
+	c.Assert(cluster.GetStore(uint64(1)).IsTombstone(), IsTrue)
+
+	// Store 2's last heartbeat is recent, and is not allowed to be removed.
+	failedStores = map[uint64]string{
+		2: "",
+	}
+
+	c.Assert(recoveryController.RemoveFailedStores(failedStores), NotNil)
+}

server/handler.go

@@ -217,6 +217,8 @@ func (h *Handler) AddScheduler(name string, args ...string) error {
 		log.Error("can not add scheduler", zap.String("scheduler-name", s.GetName()), zap.Strings("scheduler-args", args), errs.ZapError(err))
 	} else if err = h.opt.Persist(c.GetStorage()); err != nil {
 		log.Error("can not persist scheduler config", errs.ZapError(err))
+	} else {
+		log.Info("add scheduler successfully", zap.String("scheduler-name", name), zap.Strings("scheduler-args", args))
 	}
 	return err
 }
@@ -229,6 +231,8 @@ func (h *Handler) RemoveScheduler(name string) error {
 	}
 	if err = c.RemoveScheduler(name); err != nil {
 		log.Error("can not remove scheduler", zap.String("scheduler-name", name), errs.ZapError(err))
+	} else {
+		log.Info("remove scheduler successfully", zap.String("scheduler-name", name))
 	}
 	return err
 }
@@ -247,6 +251,12 @@ func (h *Handler) PauseOrResumeScheduler(name string, t int64) error {
 		} else {
 			log.Error("can not pause scheduler", zap.String("scheduler-name", name), errs.ZapError(err))
 		}
+	} else {
+		if t == 0 {
+			log.Info("resume scheduler successfully", zap.String("scheduler-name", name))
+		} else {
+			log.Info("pause scheduler successfully", zap.String("scheduler-name", name), zap.Int64("pause-seconds", t))
+		}
 	}
 	return err
 }