perf(gc): delta-maintained old-gen bytes, releasable-block evacuation metric + hard-RSS bypass, smaller per-thread floors

crates/perry-runtime/src/arena/block.rs

@@ -197,6 +197,62 @@ impl Arena {
         }
     }
 
+    /// Lazy variant of `new`: starts with a single tombstone block
+    /// (`data = null, size = 0`) instead of an eagerly-mapped 1 MB
+    /// block, so JS-touching threads that never allocate in this
+    /// region (spawn workers, tokio callers) don't pay the block.
+    /// The tombstone shape is exactly the one C4b-δ dealloc leaves
+    /// behind, so every walker/reset/alloc path already handles it:
+    /// the first `alloc` misses the tombstone, and the slow path's
+    /// `install_fresh_block` replaces the tombstone slot in place.
+    /// Only used for the non-Eden regions — Eden must stay eager
+    /// because `js_inline_arena_state` hands its current block to
+    /// codegen's inline bump allocator at thread start.
+    fn new_lazy(generation: HeapGeneration, space: HeapSpace) -> Self {
+        Arena {
+            blocks: vec![ArenaBlock {
+                data: std::ptr::null_mut(),
+                size: 0,
+                offset: 0,
+                dead_cycles: 0,
+            }],
+            current: 0,
+            generation,
+            space,
+        }
+    }
+
+    /// Bump-allocate in `blocks[idx]`, delta-maintaining the cached
+    /// old-gen in-use counter (see `OLD_GEN_IN_USE_BYTES`). Every
+    /// successful offset advance of an old-arena block MUST go through
+    /// here — a bypassed site silently skews the OldReclaim trigger.
+    #[inline]
+    fn try_block_alloc(&mut self, idx: usize, size: usize, align: usize) -> Option<*mut u8> {
+        let before = self.blocks[idx].offset;
+        let ptr = self.blocks[idx].alloc(size, align)?;
+        if self.generation == HeapGeneration::Old {
+            old_gen_in_use_bytes_add(self.blocks[idx].offset - before);
+        }
+        Some(ptr)
+    }
+
+    /// `try_block_alloc` twin for the page-excluding path.
+    #[inline]
+    fn try_block_alloc_excluding_pages(
+        &mut self,
+        idx: usize,
+        size: usize,
+        align: usize,
+        excluded_pages: &crate::fast_hash::PtrHashSet<usize>,
+    ) -> Option<*mut u8> {
+        let before = self.blocks[idx].offset;
+        let ptr = self.blocks[idx].alloc_excluding_pages(size, align, excluded_pages)?;
+        if self.generation == HeapGeneration::Old {
+            old_gen_in_use_bytes_add(self.blocks[idx].offset - before);
+        }
+        Some(ptr)
+    }
+
     #[inline]
     fn resync_inline_to_current(&self) {
         // `INLINE_STATE` mirrors ONLY the general nursery-Eden arena — the
@@ -254,8 +310,7 @@ impl Arena {
 
     fn alloc_fresh_block(&mut self, size: usize, align: usize) -> *mut u8 {
         self.install_fresh_block(size);
-        self.blocks[self.current]
-            .alloc(size, align)
+        self.try_block_alloc(self.current, size, align)
             .expect("Fresh block should have space")
     }
 
@@ -268,7 +323,7 @@ impl Arena {
         loop {
             self.install_fresh_block(size);
             if let Some(ptr) =
-                self.blocks[self.current].alloc_excluding_pages(size, align, excluded_pages)
+                self.try_block_alloc_excluding_pages(self.current, size, align, excluded_pages)
             {
                 return ptr;
             }
@@ -278,7 +333,7 @@ impl Arena {
     #[inline]
     pub(crate) fn alloc(&mut self, size: usize, align: usize) -> *mut u8 {
         // Try current block first
-        if let Some(ptr) = self.blocks[self.current].alloc(size, align) {
+        if let Some(ptr) = self.try_block_alloc(self.current, size, align) {
             return ptr;
         }
 
@@ -296,7 +351,7 @@ impl Arena {
         // Retry the (possibly newly-reset) current block. arena.current
         // may have been changed by arena_reset_empty_blocks to point
         // at the lowest reset block.
-        if let Some(ptr) = self.blocks[self.current].alloc(size, align) {
+        if let Some(ptr) = self.try_block_alloc(self.current, size, align) {
             return ptr;
         }
 
@@ -308,7 +363,7 @@ impl Arena {
             if i == self.current {
                 continue;
             }
-            if let Some(ptr) = self.blocks[i].alloc(size, align) {
+            if let Some(ptr) = self.try_block_alloc(i, size, align) {
                 self.current = i;
                 // Resync inline state to the new current block.
                 self.resync_inline_to_current();
@@ -335,15 +390,16 @@ impl Arena {
             return self.alloc(size, align);
         }
         if let Some(ptr) =
-            self.blocks[self.current].alloc_excluding_pages(size, align, excluded_pages)
+            self.try_block_alloc_excluding_pages(self.current, size, align, excluded_pages)
         {
             return ptr;
         }
         for i in 0..self.blocks.len() {
             if i == self.current {
                 continue;
             }
-            if let Some(ptr) = self.blocks[i].alloc_excluding_pages(size, align, excluded_pages) {
+            if let Some(ptr) = self.try_block_alloc_excluding_pages(i, size, align, excluded_pages)
+            {
                 self.current = i;
                 self.resync_inline_to_current();
                 return ptr;
@@ -366,6 +422,31 @@ thread_local! {
     /// `arena_reset_empty_blocks`).
     pub(crate) static ARENA_TOTAL_BYTES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
 
+    /// Cached running sum of `block.offset` across the old-gen arena —
+    /// the delta-maintained twin of `ARENA_TOTAL_BYTES` above, same
+    /// rationale: `gc_budgeted_due_trigger()` reads the old-gen in-use
+    /// total on every `gc_check_trigger` (i.e. every `gc_malloc` and
+    /// every nursery block fill), and recomputing it walked every
+    /// old-arena block each time — a linear-in-old-gen tax on every
+    /// malloc-class allocation once the old gen holds real data.
+    /// Mutation sites (each MUST maintain the delta):
+    ///   - `Arena::try_block_alloc` / `try_block_alloc_excluding_pages`
+    ///     (the only offset-advance funnel for `Arena::alloc`,
+    ///     `alloc_excluding_pages`, and the fresh-block paths),
+    ///   - `reset_region_to_zero` (generation-aware, reset.rs),
+    ///   - the old-arena reclaim family in reset.rs
+    ///     (`old_arena_reclaim_dead_blocks`,
+    ///     `old_arena_reclaim_selected_dead_blocks`,
+    ///     `OldArenaReclaimDeadBlocksState::process_block`) which zero
+    ///     old block offsets on sweep/defrag.
+    /// Block install/dealloc paths don't touch it: fresh blocks start
+    /// at offset 0 and blocks are only deallocated after their offset
+    /// was already zeroed. `old_gen_in_use_bytes()` (stats.rs)
+    /// debug-asserts this cache against the O(blocks) recompute so a
+    /// missed mutation site fails tests instead of silently skewing
+    /// the OldReclaim trigger.
+    pub(crate) static OLD_GEN_IN_USE_BYTES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
+
     pub(crate) static ARENA: UnsafeCell<Arena> =
         UnsafeCell::new(Arena::new(HeapGeneration::Nursery, HeapSpace::NurseryEden));
 
@@ -387,15 +468,15 @@ thread_local! {
     /// scanners (`scan_parse_roots`, `scan_shape_cache_roots`,
     /// `scan_transition_cache_roots`) keep them marked.
     pub(crate) static LONGLIVED_ARENA: UnsafeCell<Arena> =
-        UnsafeCell::new(Arena::new(HeapGeneration::Longlived, HeapSpace::Longlived));
+        UnsafeCell::new(Arena::new_lazy(HeapGeneration::Longlived, HeapSpace::Longlived));
 
     /// Copying nursery survivor semispaces. At most one is the active
     /// from-space at the start of a copying minor GC; the other is reset
     /// and used as to-space for fresh Eden survivors.
     pub(crate) static SURVIVOR_ARENA_0: UnsafeCell<Arena> =
-        UnsafeCell::new(Arena::new(HeapGeneration::Nursery, HeapSpace::Survivor0));
+        UnsafeCell::new(Arena::new_lazy(HeapGeneration::Nursery, HeapSpace::Survivor0));
     pub(crate) static SURVIVOR_ARENA_1: UnsafeCell<Arena> =
-        UnsafeCell::new(Arena::new(HeapGeneration::Nursery, HeapSpace::Survivor1));
+        UnsafeCell::new(Arena::new_lazy(HeapGeneration::Nursery, HeapSpace::Survivor1));
     pub(crate) static ACTIVE_SURVIVOR: Cell<usize> = const { Cell::new(0) };
 
     /// Generational-GC old-generation arena (gen-GC Phase B per
@@ -413,7 +494,7 @@ thread_local! {
     /// mark-sweep can reclaim completely dead old blocks through the
     /// dedicated old-arena reset/deallocation path.
     pub(crate) static OLD_ARENA: UnsafeCell<Arena> =
-        UnsafeCell::new(Arena::new(HeapGeneration::Old, HeapSpace::Old));
+        UnsafeCell::new(Arena::new_lazy(HeapGeneration::Old, HeapSpace::Old));
 
     /// Inline allocator state — a cache of the current arena block's
     /// `(data, offset, size)` triple, exposed via a stable pointer so
@@ -430,3 +511,21 @@ thread_local! {
         size: 0,
     }) };
 }
+
+/// Delta-maintenance for `OLD_GEN_IN_USE_BYTES` — see the thread-local's
+/// doc comment for the full mutation-site inventory.
+#[inline]
+pub(crate) fn old_gen_in_use_bytes_add(delta: usize) {
+    if delta == 0 {
+        return;
+    }
+    OLD_GEN_IN_USE_BYTES.with(|c| c.set(c.get().saturating_add(delta)));
+}
+
+#[inline]
+pub(crate) fn old_gen_in_use_bytes_sub(delta: usize) {
+    if delta == 0 {
+        return;
+    }
+    OLD_GEN_IN_USE_BYTES.with(|c| c.set(c.get().saturating_sub(delta)));
+}

crates/perry-runtime/src/arena/mod.rs

@@ -27,9 +27,9 @@ pub(crate) use allocators::{
     inactive_survivor_index, with_survivor_arena, with_survivor_arena_mut,
 };
 pub(crate) use block::{
-    Arena, ArenaBlock, ACTIVE_SURVIVOR, ARENA, ARENA_TOTAL_BYTES, BLOCK_SIZE,
-    FRESH_GENERAL_BLOCK_MIN_USED_BYTES, INLINE_STATE, LONGLIVED_ARENA, OLD_ARENA, SURVIVOR_ARENA_0,
-    SURVIVOR_ARENA_1,
+    old_gen_in_use_bytes_sub, Arena, ArenaBlock, ACTIVE_SURVIVOR, ARENA, ARENA_TOTAL_BYTES,
+    BLOCK_SIZE, FRESH_GENERAL_BLOCK_MIN_USED_BYTES, INLINE_STATE, LONGLIVED_ARENA, OLD_ARENA,
+    OLD_GEN_IN_USE_BYTES, SURVIVOR_ARENA_0, SURVIVOR_ARENA_1,
 };
 pub(crate) use page_meta::{
     address_span_overlaps_pages, register_block_space, register_old_object_pages,
@@ -63,8 +63,8 @@ pub use walk::{
 };
 pub(crate) use walk::{
     arena_block_snapshots, arena_telemetry_snapshot, general_block_in_recent_window,
-    ArenaBlockSnapshot, ArenaObjectCursor, ArenaObjectCursorBuilder, ArenaTelemetrySnapshot,
-    ArenaWalkOrder,
+    general_block_sizes, ArenaBlockSnapshot, ArenaObjectCursor, ArenaObjectCursorBuilder,
+    ArenaTelemetrySnapshot, ArenaWalkOrder,
 };
 
 // reset.rs
@@ -83,6 +83,8 @@ pub use stats::{
     js_arena_stats, longlived_in_use_bytes, old_gen_in_use_bytes, pointer_in_nursery,
     pointer_in_old_gen,
 };
+#[cfg(test)]
+pub(crate) use stats::{old_gen_in_use_bytes_recomputed, old_gen_in_use_bytes_resync};
 
 // page_meta.rs (public + pub(crate) classification/page-meta API)
 pub(crate) use page_meta::{

crates/perry-runtime/src/arena/reset.rs

@@ -49,6 +49,13 @@ fn reset_region_to_zero(arena: &mut Arena) -> (usize, usize) {
         block.offset = 0;
         block.dead_cycles = 0;
     }
+    // Delta-maintain the cached old-gen in-use counter. Only Eden and
+    // the survivor semispaces are reset through here today, but this
+    // takes any `&mut Arena` — keep the counter honest if an old-gen
+    // caller ever appears.
+    if arena.generation == HeapGeneration::Old {
+        old_gen_in_use_bytes_sub(reusable_bytes);
+    }
     arena.current = 0;
     (reset_blocks, reusable_bytes)
 }
@@ -1014,6 +1021,7 @@ impl OldArenaReclaimDeadBlocksState {
             }
             block.offset = 0;
             block.dead_cycles = 0;
+            old_gen_in_use_bytes_sub(used);
             self.changed = true;
 
             if local_idx == original_current {
@@ -1103,6 +1111,7 @@ pub(crate) fn old_arena_reclaim_dead_blocks(block_has_live: &[bool]) -> ArenaRes
             }
             block.offset = 0;
             block.dead_cycles = 0;
+            old_gen_in_use_bytes_sub(used);
             changed = true;
 
             // Keep the current old allocation target mapped and reusable.
@@ -1204,6 +1213,7 @@ pub(crate) fn old_arena_reclaim_selected_dead_blocks(
             }
             block.offset = 0;
             block.dead_cycles = 0;
+            old_gen_in_use_bytes_sub(used);
             changed = true;
 
             if i == original_current {

crates/perry-runtime/src/arena/stats.rs

@@ -67,16 +67,43 @@ pub fn longlived_in_use_bytes() -> usize {
     })
 }
 
-/// Bytes currently allocated in the old-gen arena (gen-GC Phase B).
-/// Diagnostic-only — empty in Phase B; populated by Phase C's
-/// nursery→old promotion path.
+/// Bytes currently allocated in the old-gen arena (gen-GC Phase C).
+/// Read by `gc_budgeted_due_trigger()` on every `gc_check_trigger` —
+/// i.e. on every `gc_malloc` and every nursery block fill — so this
+/// returns the delta-maintained cache (`OLD_GEN_IN_USE_BYTES`) instead
+/// of recomputing an O(old-blocks) sum each time. Debug builds
+/// cross-check the cache against the recompute so a missed mutation
+/// site fails tests instead of silently skewing the OldReclaim trigger.
 pub fn old_gen_in_use_bytes() -> usize {
+    let cached = OLD_GEN_IN_USE_BYTES.with(|c| c.get());
+    debug_assert_eq!(
+        cached,
+        old_gen_in_use_bytes_recomputed(),
+        "OLD_GEN_IN_USE_BYTES cache drifted from the per-block recompute — \
+         an old-arena offset mutation site is missing its delta update \
+         (see the mutation-site inventory on OLD_GEN_IN_USE_BYTES in arena/block.rs)"
+    );
+    cached
+}
+
+/// O(blocks) recompute of the old-gen in-use total — the cross-check /
+/// resync source of truth for the delta-maintained cache. Not for hot
+/// paths.
+pub(crate) fn old_gen_in_use_bytes_recomputed() -> usize {
     OLD_ARENA.with(|arena| {
         let arena = unsafe { &*arena.get() };
         arena.blocks.iter().map(|b| b.offset).sum()
     })
 }
 
+/// Test-only: force the cache back in sync after a test hand-mutates
+/// old-arena block offsets without going through the tracked paths.
+#[cfg(test)]
+pub(crate) fn old_gen_in_use_bytes_resync() {
+    let recomputed = old_gen_in_use_bytes_recomputed();
+    OLD_GEN_IN_USE_BYTES.with(|c| c.set(recomputed));
+}
+
 #[inline]
 pub(crate) fn active_survivor_space() -> HeapSpace {
     ACTIVE_SURVIVOR.with(|active| match active.get() {