diff --git a/compiler-rt/lib/scudo/standalone/secondary.h b/compiler-rt/lib/scudo/standalone/secondary.h index 1a232b9b9fb2d7..360a69ccac8566 100644 --- a/compiler-rt/lib/scudo/standalone/secondary.h +++ b/compiler-rt/lib/scudo/standalone/secondary.h @@ -318,9 +318,10 @@ class MapAllocatorCache { } CachedBlock PrevEntry = Quarantine[QuarantinePos]; Quarantine[QuarantinePos] = Entry; - if (OldestTime == 0) - OldestTime = Entry.Time; Entry = PrevEntry; + // Set entry time once more to reflect time + // that quarantined memory was placed in the cache + Entry.Time = Time; } // All excess entries are evicted from the cache @@ -331,9 +332,6 @@ class MapAllocatorCache { } insert(Entry); - - if (OldestTime == 0) - OldestTime = Entry.Time; } while (0); for (MemMapT &EvictMemMap : EvictionMemMaps) @@ -532,6 +530,9 @@ class MapAllocatorCache { Entries[LRUHead].Prev = static_cast(FreeIndex); } + if (LastUnreleasedEntry == CachedBlock::InvalidEntry) + LastUnreleasedEntry = static_cast(FreeIndex); + Entries[FreeIndex] = Entry; Entries[FreeIndex].Next = LRUHead; Entries[FreeIndex].Prev = CachedBlock::InvalidEntry; @@ -549,6 +550,9 @@ class MapAllocatorCache { Entries[I].invalidate(); + if (I == LastUnreleasedEntry) + LastUnreleasedEntry = Entries[LastUnreleasedEntry].Prev; + if (I == LRUHead) LRUHead = Entries[I].Next; else @@ -590,35 +594,37 @@ class MapAllocatorCache { } } - void releaseIfOlderThan(CachedBlock &Entry, u64 Time) REQUIRES(Mutex) { - if (!Entry.isValid() || !Entry.Time) - return; - if (Entry.Time > Time) { - if (OldestTime == 0 || Entry.Time < OldestTime) - OldestTime = Entry.Time; - return; - } + inline void release(CachedBlock &Entry) { + DCHECK(Entry.Time != 0); Entry.MemMap.releaseAndZeroPagesToOS(Entry.CommitBase, Entry.CommitSize); Entry.Time = 0; } void releaseOlderThan(u64 Time) EXCLUDES(Mutex) { ScopedLock L(Mutex); - if (!EntriesCount || OldestTime == 0 || OldestTime > Time) + if (!EntriesCount) return; - OldestTime = 0; - for (uptr I = 0; I < Config::getQuarantineSize(); I++) - releaseIfOlderThan(Quarantine[I], Time); - for (uptr I = 0; I < Config::getEntriesArraySize(); I++) - releaseIfOlderThan(Entries[I], Time); - } + for (uptr I = 0; I < Config::getQuarantineSize(); I++) { + CachedBlock &ReleaseEntry = Quarantine[I]; + if (!ReleaseEntry.isValid() || ReleaseEntry.Time > Time || + !ReleaseEntry.Time) + continue; + release(ReleaseEntry); + } + + // Release oldest entries first by releasing from decommit base + while (LastUnreleasedEntry != CachedBlock::InvalidEntry && + Entries[LastUnreleasedEntry].Time <= Time) { + release(Entries[LastUnreleasedEntry]); + LastUnreleasedEntry = Entries[LastUnreleasedEntry].Prev; + } + } HybridMutex Mutex; u32 EntriesCount GUARDED_BY(Mutex) = 0; u32 QuarantinePos GUARDED_BY(Mutex) = 0; atomic_u32 MaxEntriesCount = {}; atomic_uptr MaxEntrySize = {}; - u64 OldestTime GUARDED_BY(Mutex) = 0; atomic_s32 ReleaseToOsIntervalMs = {}; u32 CallsToRetrieve GUARDED_BY(Mutex) = 0; u32 SuccessfulRetrieves GUARDED_BY(Mutex) = 0; @@ -633,6 +639,9 @@ class MapAllocatorCache { u16 LRUTail GUARDED_BY(Mutex) = 0; // The AvailableHead is the top of the stack of available entries u16 AvailableHead GUARDED_BY(Mutex) = 0; + // The LastUnreleasedEntry is the least recently used entry that has not + // been released + u16 LastUnreleasedEntry GUARDED_BY(Mutex) = 0; }; template class MapAllocator {