Skip to content

Commit f332c54

Browse files
authored
Merge pull request #11657 from petermattis/pmattis/test-remove-placeholder-race
storage: fix a bug in placeholder locking
2 parents 352275c + 92c4e26 commit f332c54

File tree

3 files changed

+46
-3
lines changed

3 files changed

+46
-3
lines changed

pkg/storage/client_raft_test.go

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1847,6 +1847,42 @@ func TestRaftRemoveRace(t *testing.T) {
18471847
}
18481848
}
18491849

1850+
// TestRemovePlaceholderRace adds and removes a replica repeatedly (similar to
1851+
// TestRaftRemoveRace) in an attempt to stress the locking around replica
1852+
// placeholders.
1853+
func TestRemovePlaceholderRace(t *testing.T) {
1854+
defer leaktest.AfterTest(t)()
1855+
mtc := startMultiTestContext(t, 3)
1856+
defer mtc.Stop()
1857+
1858+
const rangeID = roachpb.RangeID(1)
1859+
mtc.replicateRange(rangeID, 1, 2)
1860+
1861+
repl, err := mtc.stores[0].GetReplica(rangeID)
1862+
if err != nil {
1863+
t.Fatal(err)
1864+
}
1865+
1866+
ident := mtc.idents[1]
1867+
ctx := repl.AnnotateCtx(context.Background())
1868+
1869+
for i := 0; i < 100; i++ {
1870+
for _, action := range []roachpb.ReplicaChangeType{roachpb.REMOVE_REPLICA, roachpb.ADD_REPLICA} {
1871+
if err := repl.ChangeReplicas(
1872+
ctx,
1873+
action,
1874+
roachpb.ReplicaDescriptor{
1875+
NodeID: ident.NodeID,
1876+
StoreID: ident.StoreID,
1877+
},
1878+
repl.Desc(),
1879+
); err != nil {
1880+
t.Fatal(err)
1881+
}
1882+
}
1883+
}
1884+
}
1885+
18501886
// TestStoreRangeRemoveDead verifies that if a store becomes dead, the
18511887
// ReplicateQueue will notice and remove any replicas on it.
18521888
func TestStoreRangeRemoveDead(t *testing.T) {

pkg/storage/replica.go

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2375,8 +2375,6 @@ func (r *Replica) handleRaftReadyRaftMuLocked(
23752375
return stats, err
23762376
}
23772377

2378-
// handleRaftReady is called under the processRaftMu lock, so it is
2379-
// safe to lock the store here.
23802378
if err := func() error {
23812379
r.store.mu.Lock()
23822380
defer r.store.mu.Unlock()

pkg/storage/store.go

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1914,6 +1914,8 @@ func (s *Store) addReplicaInternalLocked(repl *Replica) error {
19141914
return nil
19151915
}
19161916

1917+
// addPlaceholderLocked adds the specified placeholder. Requires that Store.mu
1918+
// and Replica.raftMu are held.
19171919
func (s *Store) addPlaceholderLocked(placeholder *ReplicaPlaceholder) error {
19181920
rangeID := placeholder.Desc().RangeID
19191921
if exRng := s.mu.replicasByKey.ReplaceOrInsert(placeholder); exRng != nil {
@@ -1928,13 +1930,15 @@ func (s *Store) addPlaceholderLocked(placeholder *ReplicaPlaceholder) error {
19281930

19291931
// removePlaceholder removes a placeholder for the specified range if it
19301932
// exists, returning true if a placeholder was present and removed and false
1931-
// otherwise.
1933+
// otherwise. Requires that Replica.raftMu is held.
19321934
func (s *Store) removePlaceholder(rngID roachpb.RangeID) bool {
19331935
s.mu.Lock()
19341936
defer s.mu.Unlock()
19351937
return s.removePlaceholderLocked(rngID)
19361938
}
19371939

1940+
// addPlaceholderLocked adds the specified placeholder. Requires that Store.mu
1941+
// and Replica.raftMu are held.
19381942
func (s *Store) removePlaceholderLocked(rngID roachpb.RangeID) bool {
19391943
placeholder, ok := s.mu.replicaPlaceholders[rngID]
19401944
if !ok {
@@ -3242,9 +3246,14 @@ func (s *Store) processReady(rangeID roachpb.RangeID) {
32423246
// replicasByKey map. While the replica will usually consume the
32433247
// placeholder itself, that isn't guaranteed and so this invocation
32443248
// here is crucial (i.e. don't remove it).
3249+
//
3250+
// We need to hold raftMu here to prevent removing a placeholder that is
3251+
// actively being used by Store.processRaftRequest.
3252+
r.raftMu.Lock()
32453253
if s.removePlaceholder(r.RangeID) {
32463254
atomic.AddInt32(&s.counts.droppedPlaceholders, 1)
32473255
}
3256+
r.raftMu.Unlock()
32483257
}
32493258
}
32503259
}

0 commit comments

Comments
 (0)