perf(gc): debt-proportional assist pacing + budgeted-cycle soundness (#6224)

crates/perry-runtime/src/arena/allocators.rs

@@ -70,7 +70,7 @@ pub fn arena_alloc_gc_longlived(size: usize, align: usize, obj_type: u8) -> *mut
     unsafe {
         let header = raw as *mut GcHeader;
         (*header).obj_type = obj_type;
-        (*header).gc_flags = GC_FLAG_ARENA;
+        (*header).gc_flags = GC_FLAG_ARENA | crate::gc::gc_birth_extra_flags();
         (*header)._reserved = 0;
         (*header).size = total as u32;
     }
@@ -120,7 +120,7 @@ pub fn arena_alloc_gc_old(size: usize, align: usize, obj_type: u8) -> *mut u8 {
     unsafe {
         let header = raw as *mut GcHeader;
         (*header).obj_type = obj_type;
-        (*header).gc_flags = GC_FLAG_ARENA;
+        (*header).gc_flags = GC_FLAG_ARENA | crate::gc::gc_birth_extra_flags();
         (*header)._reserved = 0;
         (*header).size = total as u32;
     }
@@ -144,7 +144,7 @@ pub(crate) fn arena_alloc_gc_old_excluding_pages(
     unsafe {
         let header = raw as *mut GcHeader;
         (*header).obj_type = obj_type;
-        (*header).gc_flags = GC_FLAG_ARENA;
+        (*header).gc_flags = GC_FLAG_ARENA | crate::gc::gc_birth_extra_flags();
         (*header)._reserved = 0;
         (*header).size = total as u32;
     }
@@ -191,7 +191,7 @@ pub(crate) fn arena_alloc_gc_survivor(size: usize, align: usize, obj_type: u8) -
     unsafe {
         let header = raw as *mut GcHeader;
         (*header).obj_type = obj_type;
-        (*header).gc_flags = GC_FLAG_ARENA;
+        (*header).gc_flags = GC_FLAG_ARENA | crate::gc::gc_birth_extra_flags();
         (*header)._reserved = 0;
         (*header).size = total as u32;
     }
@@ -268,7 +268,7 @@ pub fn arena_alloc_gc(size: usize, align: usize, obj_type: u8) -> *mut u8 {
         unsafe {
             let header = user_ptr.sub(GC_HEADER_SIZE) as *mut GcHeader;
             (*header).obj_type = obj_type;
-            (*header).gc_flags = GC_FLAG_ARENA;
+            (*header).gc_flags = GC_FLAG_ARENA | crate::gc::gc_birth_extra_flags();
             (*header)._reserved = 0;
             // size field already set from original allocation
         }
@@ -299,7 +299,7 @@ pub fn arena_alloc_gc(size: usize, align: usize, obj_type: u8) -> *mut u8 {
     unsafe {
         let header = raw as *mut GcHeader;
         (*header).obj_type = obj_type;
-        (*header).gc_flags = GC_FLAG_ARENA;
+        (*header).gc_flags = GC_FLAG_ARENA | crate::gc::gc_birth_extra_flags();
         (*header)._reserved = 0;
         (*header).size = total as u32;
     }

crates/perry-runtime/src/gc/barrier.rs

@@ -689,14 +689,46 @@ pub(super) unsafe fn scan_dirty_object_slots(
 // HashSet behavior.
 
 thread_local! {
-    /// Active full-incremental mark barrier state.
+    /// Active incremental mark barrier state (Full AND budgeted Minor
+    /// cycles — a Minor cycle sliced across mutator turns has exactly the
+    /// same lost-store hazard as a Full one; see the #6224 pacing fix, which
+    /// made budgeted minors actually complete and thereby exposed it).
     ///
     /// The valid pointer set is owned by the current `GcCycleState`. This raw
     /// pointer is installed only after that set has been built and is cleared
     /// before sweep/reclaim or if the cycle is dropped.
     pub(super) static INCREMENTAL_MARK_BARRIER_VALID_PTRS: Cell<*const ValidPointerSet> =
         const { Cell::new(std::ptr::null()) };
 
+    /// Extra GcHeader flags stamped on RUNTIME-path allocations at birth:
+    /// `GC_FLAG_MARKED` while an incremental mark barrier is active, 0
+    /// otherwise (allocate-black). A budgeted cycle's sweep may only collect
+    /// what its own trace could have seen; an object born mid-cycle and
+    /// installed via a runtime-internal RAW store (a grown array's elements
+    /// buffer, a map entry node, a string builder's data — none of which pass
+    /// through the nanboxed value-barrier path) would otherwise sit unmarked
+    /// and be freed live. Measured: 2,890 of 32,000 live graph nodes silently
+    /// lost (checksum mismatch) the moment #6224's pacing made budgeted
+    /// cycles complete; escalates to a swept-live-key SIGSEGV with manual
+    /// `gc()` mixed in. Born-marked objects survive to the NEXT cycle —
+    /// bounded floating garbage, already priced by the debt pacer.
+    ///
+    /// Codegen's inline bump allocator (lower_call.rs IR) does NOT read this
+    /// flag; codegen-born objects are ordinary JS values whose installs all
+    /// go through codegen store barriers → `incremental_mark_barrier_value`.
+    /// The runtime choke points below cover every raw-install allocation.
+    pub(crate) static GC_BIRTH_EXTRA_FLAGS: Cell<u8> = const { Cell::new(0) };
+
+    /// True while the active barrier belongs to a MINOR cycle: the barrier
+    /// must then shade only NURSERY children. Marking an old-gen child during
+    /// a minor would leave a stray mark bit that the minor's sweep never
+    /// clears (minors don't walk the old gen), and the next full cycle would
+    /// read that stale MARKED as "already traced" and skip the object's
+    /// children — unmarking-by-omission, i.e. a live-object sweep one cycle
+    /// later. Old children need no shading in a minor anyway: minors never
+    /// collect live old-gen objects.
+    pub(super) static INCREMENTAL_MARK_BARRIER_MINOR_ONLY: Cell<bool> = const { Cell::new(false) };
+
     /// Dirty old-generation pages that have received a YOUNG-gen
     /// pointer since the last collection. This is Perry's compact
     /// modbuf: barriers log bounded page regions, and minor GC scans
@@ -739,7 +771,8 @@ thread_local! {
 
 pub(super) static GENERATED_WRITE_BARRIERS_EMITTED: AtomicUsize = AtomicUsize::new(0);
 
-pub(super) fn incremental_mark_barrier_enable(valid_ptrs: &ValidPointerSet) {
+pub(super) fn incremental_mark_barrier_enable(valid_ptrs: &ValidPointerSet, minor_only: bool) {
+    INCREMENTAL_MARK_BARRIER_MINOR_ONLY.with(|cell| cell.set(minor_only));
     INCREMENTAL_MARK_BARRIER_VALID_PTRS.with(|cell| {
         cell.set(valid_ptrs as *const ValidPointerSet);
     });
@@ -749,6 +782,14 @@ pub(super) fn incremental_mark_barrier_disable() {
     INCREMENTAL_MARK_BARRIER_VALID_PTRS.with(|cell| {
         cell.set(std::ptr::null());
     });
+    INCREMENTAL_MARK_BARRIER_MINOR_ONLY.with(|cell| cell.set(false));
+}
+
+/// Allocate-black birth flags for runtime-path allocations — see
+/// `GC_BIRTH_EXTRA_FLAGS`.
+#[inline(always)]
+pub fn gc_birth_extra_flags() -> u8 {
+    GC_BIRTH_EXTRA_FLAGS.with(|cell| cell.get())
 }
 
 #[inline]
@@ -847,10 +888,18 @@ fn incremental_mark_barrier_value_with_valid_ptrs(
     value_bits: u64,
     valid_ptrs: &ValidPointerSet,
 ) -> bool {
-    let Some((_addr, header)) = current_heap_header_for_heap_word(value_bits, Some(valid_ptrs))
+    let Some((addr, header)) = current_heap_header_for_heap_word(value_bits, Some(valid_ptrs))
     else {
         return false;
     };
+    // Minor cycles shade only nursery children (see the
+    // INCREMENTAL_MARK_BARRIER_MINOR_ONLY doc: stray old-gen marks survive a
+    // minor's sweep and poison the next full cycle's trace).
+    if INCREMENTAL_MARK_BARRIER_MINOR_ONLY.with(|cell| cell.get())
+        && !crate::arena::pointer_in_nursery(addr)
+    {
+        return false;
+    }
     unsafe {
         let flags = (*header).gc_flags;
         if flags & (GC_FLAG_MARKED | GC_FLAG_PINNED | GC_FLAG_FORWARDED) != 0 {

crates/perry-runtime/src/gc/cycle.rs

@@ -791,13 +791,16 @@ struct AtomicFinalizeCycleState {
 }
 
 impl AtomicFinalizeCycleState {
-    fn new(collection_kind: GcCollectionKind) -> Self {
-        let subphase = match collection_kind {
-            GcCollectionKind::Minor => AtomicFinalizeSubphase::WeakProcessing,
-            GcCollectionKind::Full => AtomicFinalizeSubphase::BarrierSeedDrain,
-        };
+    fn new(_collection_kind: GcCollectionKind) -> Self {
+        // Both kinds start by draining the incremental-mark-barrier seeds:
+        // minors run the barrier too now (see step_build_valid_pointer_set),
+        // and the drain must precede WeakProcessing so weak/finalization
+        // decisions read the final marks. The post-drain order stays
+        // kind-specific: Minor → WeakProcessing → MinorPrelude →
+        // RememberedSetRebuild(→Sweep); Full → RememberedSetRebuild →
+        // WeakProcessing → DisableBarrier(→Sweep).
         Self {
-            subphase,
+            subphase: AtomicFinalizeSubphase::BarrierSeedDrain,
             barrier_drain: None,
             remembered_rebuild: None,
         }
@@ -837,6 +840,14 @@ impl GcCycleState {
         let start = Instant::now();
         crate::arena::old_pages_begin_gc_cycle();
         clear_mark_seeds();
+        // Allocate-black for the WHOLE cycle, from the first build slice on:
+        // the mark barrier only engages at the END of BuildValidPointerSet
+        // (the longest phase), so an object born during a build slice and
+        // installed via a runtime-internal raw store would be swept live
+        // (measured: identical 2,890-node loss with barrier-window-only
+        // birth flags). Cleared in the outcome finalizer / Drop, NOT at
+        // barrier disable — sweeping runs after the barrier is off.
+        super::barrier::GC_BIRTH_EXTRA_FLAGS.with(|cell| cell.set(GC_FLAG_MARKED));
         Self {
             collection_kind: GcCollectionKind::Full,
             trigger_kind,
@@ -878,6 +889,14 @@ impl GcCycleState {
     ) -> Self {
         let malloc_sweep_due = copied_minor_malloc_sweep_due(trigger.kind);
         let trigger_kind = trigger.kind;
+        // Allocate-black for the WHOLE cycle, from the first build slice on:
+        // the mark barrier only engages at the END of BuildValidPointerSet
+        // (the longest phase), so an object born during a build slice and
+        // installed via a runtime-internal raw store would be swept live
+        // (measured: identical 2,890-node loss with barrier-window-only
+        // birth flags). Cleared in the outcome finalizer / Drop, NOT at
+        // barrier disable — sweeping runs after the barrier is off.
+        super::barrier::GC_BIRTH_EXTRA_FLAGS.with(|cell| cell.set(GC_FLAG_MARKED));
         Self {
             collection_kind: GcCollectionKind::Minor,
             trigger_kind,
@@ -1027,9 +1046,17 @@ impl GcCycleState {
             .expect("valid-pointer builder exists");
         self.valid_ptrs = Some(builder.finish());
         trace_phase_record(&mut self.trace, "build_valid_pointer_set", phase_start);
-        if matches!(self.collection_kind, GcCollectionKind::Full) {
+        // Enable the incremental mark barrier for BOTH kinds. A budgeted
+        // MINOR cycle sliced across mutator turns has the same lost-store
+        // hazard as a Full one: a store into an already-traced object after
+        // its slot was scanned would leave the stored child unmarked, and
+        // the minor sweep frees it live (measured as a property-key UAF the
+        // moment #6224's pacing made budgeted minors actually complete).
+        // Minor barriers shade nursery children only — see
+        // INCREMENTAL_MARK_BARRIER_MINOR_ONLY.
+        {
             let valid_ptrs = self.valid_ptrs.as_ref().expect("valid pointer set built");
-            incremental_mark_barrier_enable(valid_ptrs);
+            incremental_mark_barrier_enable(valid_ptrs, self.minor.is_some());
         }
 
         let active_elapsed_us = self.active_elapsed_us();
@@ -1213,6 +1240,7 @@ impl GcCycleState {
                     .subphase = AtomicFinalizeSubphase::RememberedSetRebuild;
             }
             AtomicFinalizeSubphase::BarrierSeedDrain => {
+                let minor_only = self.minor.is_some();
                 let valid_ptrs = self.valid_ptrs.as_ref().expect("valid pointer set built");
                 let done = {
                     let state = self
@@ -1221,7 +1249,7 @@ impl GcCycleState {
                         .expect("atomic finalize state exists");
                     let drain = state
                         .barrier_drain
-                        .get_or_insert_with(|| TraceWorklistCycleState::new(false));
+                        .get_or_insert_with(|| TraceWorklistCycleState::new(minor_only));
                     drain.step(valid_ptrs, budget)
                 };
                 if done {
@@ -1230,7 +1258,12 @@ impl GcCycleState {
                         .as_mut()
                         .expect("atomic finalize state exists");
                     state.barrier_drain = None;
-                    state.subphase = AtomicFinalizeSubphase::RememberedSetRebuild;
+                    // Kind-specific continuation (see AtomicFinalizeCycleState::new).
+                    state.subphase = if minor_only {
+                        AtomicFinalizeSubphase::WeakProcessing
+                    } else {
+                        AtomicFinalizeSubphase::RememberedSetRebuild
+                    };
                 }
             }
             AtomicFinalizeSubphase::RememberedSetRebuild => {
@@ -1246,6 +1279,14 @@ impl GcCycleState {
                 // leave `live_old_to_young_sticky` None; reclaim then restores
                 // only `evacuation_sticky` + the pre-clear dirty snapshot.
                 if self.minor.is_some() {
+                    // Barrier off for the minor: first drain any seeds the
+                    // barrier pushed since BarrierSeedDrain completed (late
+                    // stores mark the child but its children still need the
+                    // trace), then disable. Bounded by late-store volume.
+                    let valid_ptrs = self.valid_ptrs.as_ref().expect("valid pointer set built");
+                    let mut final_drain = TraceWorklistCycleState::new(true);
+                    while !final_drain.step(valid_ptrs, usize::MAX) {}
+                    incremental_mark_barrier_disable();
                     self.atomic_finalize = None;
                     self.phase = GcCyclePhase::Sweep;
                     return;
@@ -1282,6 +1323,15 @@ impl GcCycleState {
                 if budget == 0 {
                     return;
                 }
+                // Same late-seed closure as the minor path: trace anything
+                // the barrier shaded after BarrierSeedDrain completed, so no
+                // marked-but-untraced object reaches Sweep with unmarked
+                // children.
+                {
+                    let valid_ptrs = self.valid_ptrs.as_ref().expect("valid pointer set built");
+                    let mut final_drain = TraceWorklistCycleState::new(false);
+                    while !final_drain.step(valid_ptrs, usize::MAX) {}
+                }
                 incremental_mark_barrier_disable();
                 if let Some(state) = self.atomic_finalize.as_mut() {
                     state.subphase = AtomicFinalizeSubphase::Done;
@@ -1597,6 +1647,7 @@ impl GcCycleState {
             .map(|minor| minor.malloc_sweep_due)
             .unwrap_or(true);
 
+        super::barrier::GC_BIRTH_EXTRA_FLAGS.with(|cell| cell.set(0));
         self.outcome = Some(GcCollectOutcome {
             freed_bytes: self.freed_bytes,
             malloc_swept,
@@ -1607,10 +1658,11 @@ impl GcCycleState {
 
 impl Drop for GcCycleState {
     fn drop(&mut self) {
-        if matches!(self.collection_kind, GcCollectionKind::Full)
-            && self.phase != GcCyclePhase::Complete
-        {
+        // Both kinds enable the barrier now (minor cycles too); never let the
+        // raw valid-ptrs pointer dangle past the cycle that owns the set.
+        if self.phase != GcCyclePhase::Complete {
             incremental_mark_barrier_disable();
+            super::barrier::GC_BIRTH_EXTRA_FLAGS.with(|cell| cell.set(0));
             clear_mark_seeds();
         }
     }

crates/perry-runtime/src/gc/malloc.rs

@@ -247,7 +247,7 @@ pub fn gc_malloc(size: usize, obj_type: u8) -> *mut u8 {
 
         let header = raw as *mut GcHeader;
         (*header).obj_type = obj_type;
-        (*header).gc_flags = 0; // not arena
+        (*header).gc_flags = super::barrier::gc_birth_extra_flags(); // not arena; allocate-black while a budgeted cycle marks
         (*header)._reserved = 0;
         (*header).size = total as u32;
 
@@ -300,7 +300,7 @@ pub fn gc_malloc_batch(sizes: &[usize], obj_type: u8) -> Vec<*mut u8> {
             }
             let header = raw as *mut GcHeader;
             (*header).obj_type = obj_type;
-            (*header).gc_flags = 0;
+            (*header).gc_flags = super::barrier::gc_birth_extra_flags();
             (*header)._reserved = 0;
             (*header).size = total as u32;
 

crates/perry-runtime/src/gc/mod.rs

@@ -38,6 +38,8 @@ pub use types::*;
 mod policy;
 pub(crate) use policy::gc_runtime_safepoint;
 pub use policy::*;
+mod progress;
+pub use progress::*;
 mod heap_budget;
 pub use heap_budget::*;
 mod pressure;
@@ -80,6 +82,7 @@ pub fn gc_collect_minor() -> u64 {
 }
 
 pub(super) fn gc_collect_minor_with_trigger(trigger: GcTriggerSnapshot) -> GcCollectOutcome {
+    gc_drain_active_budgeted_cycle();
     // Barriers-off ⇒ the remembered set is not being maintained, and a
     // minor's black-leafed old parents would hide live children. Route
     // every caller (direct arm, moving-safepoint arm, public FFI) to the
@@ -251,6 +254,7 @@ fn gc_collect_inner_with_trigger(trigger: GcTriggerSnapshot) -> GcCollectOutcome
 }
 
 fn gc_collect_full_mark_sweep_with_trigger(trigger: GcTriggerSnapshot) -> GcCollectOutcome {
+    gc_drain_active_budgeted_cycle();
     GC_TRIGGER_BUMPED.with(|c| c.set(false));
     GcCycleState::new_full(trigger).run_to_completion()
 }