fix(gc): #5476 — reclaim dead old-gen blocks under reclaim pressure on compute-only workloads

crates/perry-runtime/src/gc/policy.rs

@@ -511,6 +511,27 @@ thread_local! {
         const { Cell::new(DeferredGcRequest::None) };
     pub(super) static GC_OLD_RECLAIM_PENDING: Cell<bool> = const { Cell::new(false) };
     pub(super) static GC_LAST_OLD_RECLAIM_IN_USE_BYTES: Cell<usize> = const { Cell::new(0) };
+    /// Re-entrancy guard for the #5476 direct old-gen reclaim driven from
+    /// `gc_check_trigger`: the full collection must not recursively trigger
+    /// another reclaim if a hook it runs allocates.
+    pub(super) static GC_OLD_RECLAIM_IN_PROGRESS: Cell<bool> = const { Cell::new(false) };
+}
+
+/// RAII guard that marks a #5476 direct old-gen reclaim in progress so a nested
+/// `gc_check_trigger` can't re-enter it. See `GC_OLD_RECLAIM_IN_PROGRESS`.
+struct OldReclaimReentryGuard;
+
+impl OldReclaimReentryGuard {
+    fn enter() -> Self {
+        GC_OLD_RECLAIM_IN_PROGRESS.with(|p| p.set(true));
+        Self
+    }
+}
+
+impl Drop for OldReclaimReentryGuard {
+    fn drop(&mut self) {
+        GC_OLD_RECLAIM_IN_PROGRESS.with(|p| p.set(false));
+    }
 }
 
 pub(super) const GC_OLD_GEN_RECLAIM_THRESHOLD_BYTES: usize = 48 * 1024 * 1024;
@@ -1009,6 +1030,40 @@ pub fn gc_check_trigger() {
     if defer_gc_request(DeferredGcRequest::CheckTrigger) {
         return;
     }
+
+    // #5476: a workload that churns *large* temporaries (>16 KB, born directly
+    // in the old arena) grows the old generation without ever exercising the
+    // nursery. Old-gen reclaim pressure schedules a budgeted full cycle that
+    // *would* return the dead old blocks to the OS — but the budgeted stepper is
+    // blocked whenever synchronous-only root scanners are registered (the common
+    // case in a compiled program), and even when it runs it only advances through
+    // bounded mutator-assist steps that a compute-only loop never drives to
+    // completion (no event-loop safepoint ever runs). Either way no collection
+    // completes and RSS climbs unbounded. When old-gen reclaim pressure is what's
+    // due — a rare event, gated by the ~32 MB growth / 48 MB absolute baseline, so
+    // this never fires on the common nursery-churn path — run a direct full
+    // mark-sweep to completion here, the same non-budgeted collection an explicit
+    // `gc()` performs. The conservative native-stack scan (`force_full_scan`)
+    // keeps it safe: anything still referenced from the stack/registers at this
+    // allocation point (e.g. the temporary currently being built) is retained;
+    // only genuinely unreachable old blocks are returned.
+    if !gc_budgeted_cycle_active()
+        && matches!(
+            gc_budgeted_due_trigger(),
+            Some(BudgetedGcTrigger::OldReclaim)
+        )
+        && !GC_OLD_RECLAIM_IN_PROGRESS.with(Cell::get)
+    {
+        let _reentry = OldReclaimReentryGuard::enter();
+        GC_OLD_RECLAIM_PENDING.with(|pending| pending.set(false));
+        let _scan = super::roots::ManualGcScanGuard::force_full_scan();
+        gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot::capture(
+            GcTriggerKind::OldGenBytes,
+        ))
+        .emit_after_current();
+        return;
+    }
+
     if !gc_budgeted_cycle_active() && gc_budgeted_due_trigger().is_none() {
         return;
     }

crates/perry-runtime/src/gc/tests/budgeted_step_api.rs

@@ -171,3 +171,58 @@ fn microsecond_budget_step_remains_bounded_on_multi_slice_heap() {
     assert_eq!(completed.status, JS_GC_STEP_STATUS_COMPLETED);
     assert_eq!(js_shadow_slot_get(0) & POINTER_MASK, live as u64);
 }
+
+/// Allocate one unreachable old-arena object and return only its size. Kept
+/// `#[inline(never)]` so the raw `dead_old` pointer lives and dies entirely
+/// within this frame — it never lands on the caller's stack where a conservative
+/// scan could pin it. (The GC test guard already pins `Auto` scan mode, which
+/// skips the native-stack scan, but isolating the pointer makes the reclaim
+/// assertion robust regardless of scan mode.)
+#[inline(never)]
+fn allocate_unreachable_old_for_reclaim() -> u64 {
+    let dead_old = crate::arena::arena_alloc_gc_old(32, 8, GC_TYPE_STRING);
+    unsafe { (*header_from_user_ptr(dead_old as *const u8)).size as u64 }
+}
+
+/// #5476: a compute-only workload that churns large temporaries never runs a
+/// host GC step, so the old-gen reclaim cycle must complete from the allocator
+/// hook (`gc_check_trigger`) alone. A single call — what every allocation does —
+/// must drive the full reclaim cycle to completion and return the dead old block,
+/// not leave it stalled mid-flight as bounded mutator-assist stepping does.
+#[test]
+fn check_trigger_drives_old_reclaim_to_completion_without_host_stepping() {
+    let _guard = CopyingNurseryTestGuard::new(2);
+    let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers();
+    reset_old_reclaim_pressure();
+
+    let live = young_leaf();
+    js_shadow_slot_set(0, ptr_bits(live));
+    let dead_old_size = allocate_unreachable_old_for_reclaim();
+    let freed_before = GC_STATS.with(|stats| stats.borrow().total_freed_bytes);
+    let collections_before = gc_collection_count();
+
+    // Old-gen reclaim pressure is due, but the host never steps GC.
+    GC_OLD_RECLAIM_PENDING.with(|pending| pending.set(true));
+    gc_check_trigger();
+
+    assert!(
+        !gc_budgeted_cycle_active(),
+        "old-reclaim cycle must not be left stalled mid-flight"
+    );
+    let mut status = JsGcStepResult::default();
+    assert_eq!(js_gc_step_status(&mut status), JS_GC_STEP_STATUS_IDLE);
+    assert!(
+        gc_collection_count() > collections_before,
+        "a full collection must have completed"
+    );
+    let freed_after = GC_STATS.with(|stats| stats.borrow().total_freed_bytes);
+    assert!(
+        freed_after.saturating_sub(freed_before) >= dead_old_size,
+        "gc_check_trigger must reclaim unreachable old-arena bytes under reclaim pressure"
+    );
+    assert_eq!(
+        js_shadow_slot_get(0) & POINTER_MASK,
+        live as u64,
+        "live root must survive the reclaim"
+    );
+}