Skip to content

Commit

Permalink
checker: add disconnected check when fix orphan peers (tikv#7240)
Browse files Browse the repository at this point in the history
close tikv#7249

Signed-off-by: lhy1024 <admin@liudos.us>

Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com>
  • Loading branch information
lhy1024 and ti-chi-bot[bot] committed Nov 8, 2023
1 parent 595d5b0 commit ae93757
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 1 deletion.
13 changes: 13 additions & 0 deletions pkg/schedule/checker/rule_checker.go
Original file line number Diff line number Diff line change
Expand Up @@ -479,6 +479,13 @@ loopFits:
hasUnhealthyFit = true
break loopFits
}
// avoid to meet down store when fix orpahn peers,
// Isdisconnected is more strictly than IsUnhealthy.
if c.cluster.GetStore(p.GetStoreId()).IsDisconnected() {
hasUnhealthyFit = true
pinDownPeer = p
break loopFits
}
}
}

Expand All @@ -491,6 +498,9 @@ loopFits:
// try to use orphan peers to replace unhealthy down peers.
for _, orphanPeer := range fit.OrphanPeers {
if pinDownPeer != nil {
if pinDownPeer.GetId() == orphanPeer.GetId() {
continue
}
// make sure the orphan peer is healthy.
if isUnhealthyPeer(orphanPeer.GetId()) {
continue
Expand All @@ -514,6 +524,9 @@ loopFits:
return operator.CreatePromoteLearnerOperatorAndRemovePeer("replace-down-peer-with-orphan-peer", c.cluster, region, orphanPeer, pinDownPeer)
case orphanPeerRole == metapb.PeerRole_Voter && destRole == metapb.PeerRole_Learner:
return operator.CreateDemoteLearnerOperatorAndRemovePeer("replace-down-peer-with-orphan-peer", c.cluster, region, orphanPeer, pinDownPeer)
case orphanPeerRole == metapb.PeerRole_Voter && destRole == metapb.PeerRole_Voter &&
c.cluster.GetStore(pinDownPeer.GetStoreId()).IsDisconnected() && !dstStore.IsDisconnected():
return operator.CreateRemovePeerOperator("remove-replaced-orphan-peer", c.cluster, 0, region, pinDownPeer.GetStoreId())
default:
// destRole should not same with orphanPeerRole. if role is same, it fit with orphanPeer should be better than now.
// destRole never be leader, so we not consider it.
Expand Down
2 changes: 1 addition & 1 deletion server/cluster/cluster_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2796,7 +2796,7 @@ func TestReplica(t *testing.T) {
re.NoError(tc.addLeaderRegion(2, 1, 2, 3, 4))
region = tc.GetRegion(2)
re.NoError(dispatchHeartbeat(co, region, stream))
region = waitRemovePeer(re, stream, region, 4)
region = waitRemovePeer(re, stream, region, 3) // store3 is down, we should remove it firstly.
re.NoError(dispatchHeartbeat(co, region, stream))
waitNoResponse(re, stream)

Expand Down

0 comments on commit ae93757

Please sign in to comment.