Skip to content

Commit

Permalink
fix lock
Browse files Browse the repository at this point in the history
Signed-off-by: lhy1024 <admin@liudos.us>
  • Loading branch information
lhy1024 committed Nov 21, 2023
1 parent 240728f commit 8a2100f
Show file tree
Hide file tree
Showing 2 changed files with 34 additions and 12 deletions.
16 changes: 9 additions & 7 deletions pkg/schedule/schedulers/evict_slow_store.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ func (conf *evictSlowStoreSchedulerConfig) Clone() *evictSlowStoreSchedulerConfi
}

func (conf *evictSlowStoreSchedulerConfig) PersistLocked() error {
name := conf.getSchedulerName()
name := EvictSlowStoreName
data, err := EncodeConfig(conf)
failpoint.Inject("persistFail", func() {
err = errors.New("fail to persist")
Expand All @@ -86,11 +86,9 @@ func (conf *evictSlowStoreSchedulerConfig) PersistLocked() error {
return conf.storage.SaveSchedulerConfig(name, data)
}

func (conf *evictSlowStoreSchedulerConfig) getSchedulerName() string {
return EvictSlowStoreName
}

func (conf *evictSlowStoreSchedulerConfig) getStores() []uint64 {
conf.RLock()
defer conf.RUnlock()
return conf.EvictedStores
}

Expand All @@ -102,10 +100,10 @@ func (conf *evictSlowStoreSchedulerConfig) getKeyRangesByID(id uint64) []core.Ke
}

func (conf *evictSlowStoreSchedulerConfig) evictStore() uint64 {
if len(conf.EvictedStores) == 0 {
if len(conf.getStores()) == 0 {
return 0
}
return conf.EvictedStores[0]
return conf.getStores()[0]
}

// readyForRecovery checks whether the last cpatured candidate is ready for recovery.
Expand All @@ -120,13 +118,17 @@ func (conf *evictSlowStoreSchedulerConfig) readyForRecovery() bool {
}

func (conf *evictSlowStoreSchedulerConfig) setStoreAndPersist(id uint64) error {
conf.Lock()
defer conf.Unlock()
conf.EvictedStores = []uint64{id}
conf.lastSlowStoreCaptureTS = time.Now()
return conf.PersistLocked()
}

func (conf *evictSlowStoreSchedulerConfig) clearAndPersist() (oldID uint64, err error) {
oldID = conf.evictStore()
conf.Lock()
defer conf.Unlock()
if oldID > 0 {
conf.EvictedStores = []uint64{}
conf.lastSlowStoreCaptureTS = time.Time{}
Expand Down
30 changes: 25 additions & 5 deletions pkg/schedule/schedulers/evict_slow_trend.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ func (conf *evictSlowTrendSchedulerConfig) Clone() *evictSlowTrendSchedulerConfi
}

func (conf *evictSlowTrendSchedulerConfig) PersistLocked() error {
name := conf.getSchedulerName()
name := EvictSlowTrendName
data, err := EncodeConfig(conf)
failpoint.Inject("persistFail", func() {
err = errors.New("fail to persist")
Expand All @@ -97,11 +97,9 @@ func (conf *evictSlowTrendSchedulerConfig) PersistLocked() error {
return conf.storage.SaveSchedulerConfig(name, data)
}

func (conf *evictSlowTrendSchedulerConfig) getSchedulerName() string {
return EvictSlowTrendName
}

func (conf *evictSlowTrendSchedulerConfig) getStores() []uint64 {
conf.RLock()
defer conf.RUnlock()
return conf.EvictedStores
}

Expand All @@ -113,32 +111,44 @@ func (conf *evictSlowTrendSchedulerConfig) getKeyRangesByID(id uint64) []core.Ke
}

func (conf *evictSlowTrendSchedulerConfig) hasEvictedStores() bool {
conf.RLock()
defer conf.RUnlock()
return len(conf.EvictedStores) > 0
}

func (conf *evictSlowTrendSchedulerConfig) evictedStore() uint64 {
if !conf.hasEvictedStores() {
return 0
}
conf.RLock()
defer conf.RUnlock()
// If a candidate passes all checks and proved to be slow, it will be
// recorded in `conf.EvictStores`, and `conf.lastEvictCandidate` will record
// the captured timestamp of this store.
return conf.EvictedStores[0]
}

func (conf *evictSlowTrendSchedulerConfig) candidate() uint64 {
conf.RLock()
defer conf.RUnlock()
return conf.evictCandidate.storeID
}

func (conf *evictSlowTrendSchedulerConfig) captureTS() time.Time {
conf.RLock()
defer conf.RUnlock()
return conf.evictCandidate.captureTS
}

func (conf *evictSlowTrendSchedulerConfig) candidateCapturedSecs() uint64 {
conf.RLock()
defer conf.RUnlock()
return DurationSinceAsSecs(conf.evictCandidate.captureTS)
}

func (conf *evictSlowTrendSchedulerConfig) lastCapturedCandidate() *slowCandidate {
conf.RLock()
defer conf.RUnlock()
return &conf.lastEvictCandidate
}

Expand All @@ -158,6 +168,8 @@ func (conf *evictSlowTrendSchedulerConfig) readyForRecovery() bool {
}

func (conf *evictSlowTrendSchedulerConfig) captureCandidate(id uint64) {
conf.Lock()
defer conf.Unlock()
conf.evictCandidate = slowCandidate{
storeID: id,
captureTS: time.Now(),
Expand All @@ -169,6 +181,8 @@ func (conf *evictSlowTrendSchedulerConfig) captureCandidate(id uint64) {
}

func (conf *evictSlowTrendSchedulerConfig) popCandidate(updLast bool) uint64 {
conf.Lock()
defer conf.Unlock()
id := conf.evictCandidate.storeID
if updLast {
conf.lastEvictCandidate = conf.evictCandidate
Expand All @@ -178,12 +192,16 @@ func (conf *evictSlowTrendSchedulerConfig) popCandidate(updLast bool) uint64 {
}

func (conf *evictSlowTrendSchedulerConfig) markCandidateRecovered() {
conf.Lock()
defer conf.Unlock()
if conf.lastEvictCandidate != (slowCandidate{}) {
conf.lastEvictCandidate.recoverTS = time.Now()
}
}

func (conf *evictSlowTrendSchedulerConfig) setStoreAndPersist(id uint64) error {
conf.Lock()
defer conf.Unlock()
conf.EvictedStores = []uint64{id}
return conf.PersistLocked()
}
Expand All @@ -199,6 +217,8 @@ func (conf *evictSlowTrendSchedulerConfig) clearAndPersist(cluster sche.Schedule
address = store.GetAddress()
}
storeSlowTrendEvictedStatusGauge.WithLabelValues(address, strconv.FormatUint(oldID, 10)).Set(0)
conf.Lock()
defer conf.Unlock()
conf.EvictedStores = []uint64{}
return oldID, conf.PersistLocked()
}
Expand Down

0 comments on commit 8a2100f

Please sign in to comment.