diff --git a/server/cluster/coordinator_test.go b/server/cluster/coordinator_test.go
index 9509e74a53e..5b24633c615 100644
--- a/server/cluster/coordinator_test.go
+++ b/server/cluster/coordinator_test.go
@@ -1033,6 +1033,47 @@ func (s *testOperatorControllerSuite) TestStoreOverloadedWithReplace(c *C) {
 	c.Assert(lb.Schedule(tc), NotNil)
 }
 
+func (s *testOperatorControllerSuite) TestDownStoreLimit(c *C) {
+	tc, co, cleanup := prepare(nil, nil, nil, c)
+	defer cleanup()
+	oc := co.opController
+	rc := co.checkers.GetRuleChecker()
+
+	tc.addRegionStore(1, 100)
+	tc.addRegionStore(2, 100)
+	tc.addRegionStore(3, 100)
+	tc.addLeaderRegion(1, 1, 2, 3)
+
+	region := tc.GetRegion(1)
+	tc.setStoreDown(1)
+	tc.SetStoreLimit(1, storelimit.RemovePeer, 1)
+
+	region = region.Clone(core.WithDownPeers([]*pdpb.PeerStats{
+		{
+			Peer:        region.GetStorePeer(1),
+			DownSeconds: 24 * 60 * 60,
+		},
+	}), core.SetApproximateSize(1))
+	tc.putRegion(region)
+	for i := uint64(1); i < 20; i++ {
+		tc.addRegionStore(i+3, 100)
+		op := rc.Check(region)
+		c.Assert(op, NotNil)
+		c.Assert(oc.AddOperator(op), IsTrue)
+		oc.RemoveOperator(op)
+	}
+
+	region = region.Clone(core.SetApproximateSize(100))
+	tc.putRegion(region)
+	for i := uint64(20); i < 25; i++ {
+		tc.addRegionStore(i+3, 100)
+		op := rc.Check(region)
+		c.Assert(op, NotNil)
+		c.Assert(oc.AddOperator(op), IsTrue)
+		oc.RemoveOperator(op)
+	}
+}
+
 var _ = Suite(&testScheduleControllerSuite{})
 
 type testScheduleControllerSuite struct {
diff --git a/server/schedule/checker_controller.go b/server/schedule/checker_controller.go
index b8d06129009..69bbb9ce8f4 100644
--- a/server/schedule/checker_controller.go
+++ b/server/schedule/checker_controller.go
@@ -108,6 +108,11 @@ func (c *CheckerController) GetMergeChecker() *checker.MergeChecker {
 	return c.mergeChecker
 }
 
+// GetRuleChecker returns the rule checker.
+func (c *CheckerController) GetRuleChecker() *checker.RuleChecker {
+	return c.ruleChecker
+}
+
 // GetWaitingRegions returns the regions in the waiting list.
 func (c *CheckerController) GetWaitingRegions() []*cache.Item {
 	return c.regionWaitingList.Elems()
diff --git a/server/schedule/operator/builder.go b/server/schedule/operator/builder.go
index 5af39eb91fd..e54e6480a1c 100644
--- a/server/schedule/operator/builder.go
+++ b/server/schedule/operator/builder.go
@@ -663,9 +663,15 @@ func (b *Builder) execAddPeer(peer *metapb.Peer) {
 }
 
 func (b *Builder) execRemovePeer(peer *metapb.Peer) {
-	b.steps = append(b.steps, RemovePeer{FromStore: peer.GetStoreId(), PeerID: peer.GetId()})
-	delete(b.currentPeers, peer.GetStoreId())
-	delete(b.toRemove, peer.GetStoreId())
+	removeStoreID := peer.GetStoreId()
+	var isDownStore bool
+	store := b.cluster.GetStore(removeStoreID)
+	if store != nil {
+		isDownStore = store.DownTime() > b.cluster.GetOpts().GetMaxStoreDownTime()
+	}
+	b.steps = append(b.steps, RemovePeer{FromStore: removeStoreID, PeerID: peer.GetId(), IsDownStore: isDownStore})
+	delete(b.currentPeers, removeStoreID)
+	delete(b.toRemove, removeStoreID)
 }
 
 func (b *Builder) execChangePeerV2(needEnter bool, needTransferLeader bool) {
diff --git a/server/schedule/operator/influence.go b/server/schedule/operator/influence.go
index efd1e3d6b87..9f6635b3d20 100644
--- a/server/schedule/operator/influence.go
+++ b/server/schedule/operator/influence.go
@@ -80,7 +80,7 @@ func (s *StoreInfluence) addStepCost(limitType storelimit.Type, cost int64) {
 func (s *StoreInfluence) AdjustStepCost(limitType storelimit.Type, regionSize int64) {
 	if regionSize > storelimit.SmallRegionThreshold {
 		s.addStepCost(limitType, storelimit.RegionInfluence[limitType])
-	} else if regionSize <= storelimit.SmallRegionThreshold && regionSize > core.EmptyRegionApproximateSize {
+	} else if regionSize > core.EmptyRegionApproximateSize {
 		s.addStepCost(limitType, storelimit.SmallRegionInfluence[limitType])
 	}
 }
diff --git a/server/schedule/operator/step.go b/server/schedule/operator/step.go
index 0f253299f1a..fdb25eb1d6b 100644
--- a/server/schedule/operator/step.go
+++ b/server/schedule/operator/step.go
@@ -219,6 +219,7 @@ func (pl PromoteLearner) Influence(opInfluence OpInfluence, region *core.RegionI
 // RemovePeer is an OpStep that removes a region peer.
 type RemovePeer struct {
 	FromStore, PeerID uint64
+	IsDownStore       bool
 }
 
 // ConfVerChanged returns the delta value for version increased by this step.
@@ -259,6 +260,10 @@ func (rp RemovePeer) Influence(opInfluence OpInfluence, region *core.RegionInfo)
 	regionSize := region.GetApproximateSize()
 	from.RegionSize -= regionSize
 	from.RegionCount--
+
+	if rp.IsDownStore && regionSize > storelimit.SmallRegionThreshold {
+		regionSize = storelimit.SmallRegionThreshold
+	}
 	from.AdjustStepCost(storelimit.RemovePeer, regionSize)
 }