#268 bignum nursery/GC representation: kill the malloc_raw leak + bound within-run RSS

.github/workflows/pyre-ci.yml

@@ -182,7 +182,7 @@ jobs:
     steps: *prepare-charon-llbc-steps
 
   cargo-test-linux:
-    name: cargo test (ubuntu-24.04, ${{ matrix.backend }})
+    name: cargo test (ubuntu-24.04)
     runs-on: ubuntu-24.04
     needs: prepare-charon-llbc-linux
     if: ${{ !cancelled() && needs.prepare-charon-llbc-linux.result == 'success' }}
@@ -191,10 +191,6 @@ jobs:
       # artifact into this workspace path.
       PYRE_SHARED_BUILD: ${{ github.workspace }}/.pyre-build
       CHARON_VERSION: nightly-2026.05.29
-    strategy:
-      fail-fast: false
-      matrix:
-        backend: [dynasm, cranelift]
 
     steps: &cargo-test-steps
     - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -245,41 +241,43 @@ jobs:
           /opt/hostedtoolcache/CodeQL /usr/local/.ghcup /usr/local/share/boost
         sudo docker image prune --all --force || true
         df -h /
-    - name: Run cargo test for ${{ matrix.backend }}
+    - name: Run cargo tests
       # Pin bash: on windows-latest the default shell is PowerShell, where
-      # "$BACKEND" does not expand the env var (it would need $env:BACKEND),
-      # so cargo test would run with an empty feature set.
+      # POSIX loop/status handling would otherwise need a separate spelling.
       shell: bash
-      env:
-        BACKEND: ${{ matrix.backend }}
-      run: cargo test --all --no-default-features --features "$BACKEND"
+      run: |
+        status=0
+        for backend in dynasm cranelift; do
+          echo "::group::cargo test ($backend)"
+          if cargo test --all --no-default-features --features "$backend"; then
+            echo "::notice title=cargo test::$backend passed"
+          else
+            code=$?
+            echo "::error title=cargo test::$backend failed with exit $code"
+            status=1
+          fi
+          echo "::endgroup::"
+        done
+        exit "$status"
 
   cargo-test-macos:
-    name: cargo test (macos-latest, ${{ matrix.backend }})
+    name: cargo test (macos-latest)
     runs-on: macos-latest
     needs: prepare-charon-llbc-macos
     if: ${{ !cancelled() && needs.prepare-charon-llbc-macos.result == 'success' }}
     env:
       PYRE_SHARED_BUILD: ${{ github.workspace }}/.pyre-build
       CHARON_VERSION: nightly-2026.05.29
-    strategy:
-      fail-fast: false
-      matrix:
-        backend: [dynasm, cranelift]
     steps: *cargo-test-steps
 
   cargo-test-windows:
-    name: cargo test (windows-latest, ${{ matrix.backend }})
+    name: cargo test (windows-latest)
     runs-on: windows-latest
     needs: prepare-charon-llbc-windows
     if: ${{ !cancelled() && needs.prepare-charon-llbc-windows.result == 'success' }}
     env:
       PYRE_SHARED_BUILD: ${{ github.workspace }}/.pyre-build
       CHARON_VERSION: nightly-2026.05.29
-    strategy:
-      fail-fast: false
-      matrix:
-        backend: [dynasm, cranelift]
     steps: *cargo-test-steps
 
   pyre-check-linux:

majit/majit-backend-cranelift/src/compiler.rs

@@ -1420,6 +1420,29 @@ fn alloc_nursery_typed_via_active_runtime(type_id: u32, size: usize) -> GcRef {
     with_cranelift_gc(|gc| gc.alloc_nursery_no_collect_typed(type_id, size)).unwrap_or(GcRef(0))
 }
 
+/// `majit_gc::AllocNurseryCollectingTypedFn` installed by `set_gc_allocator`.
+/// Unlike [`alloc_nursery_typed_via_active_runtime`] (no-collect), this runs a
+/// minor when the nursery is full. Only the elidable bigint payload helpers use
+/// it, from a residual call whose jitframe gcmap roots the trace's live set, so
+/// the embedded minor cycle reclaims dead bigints instead of spilling to old-gen.
+fn alloc_nursery_collecting_typed_via_active_runtime(type_id: u32, size: usize) -> GcRef {
+    with_cranelift_gc(|gc| gc.alloc_nursery_typed(type_id, size)).unwrap_or(GcRef(0))
+}
+
+/// `majit_gc::ChargeMemoryPressureFn` installed by `set_gc_allocator`. Charges a
+/// freshly-built bignum's off-heap limb-`Vec` bytes so the active GC's minor
+/// cadence reflects true footprint; may force a minor, safe from the same
+/// gcmap-rooted residual call as [`alloc_nursery_collecting_typed_via_active_runtime`].
+fn charge_memory_pressure_via_active_runtime(bytes: usize) {
+    with_cranelift_gc(|gc| gc.charge_memory_pressure(bytes));
+}
+
+/// Charge an old-gen object's off-heap payload against the major threshold on
+/// the active cranelift runtime's GC, without forcing a minor.
+fn charge_oldgen_external_via_active_runtime(obj_addr: usize, bytes: usize) {
+    with_cranelift_gc(|gc| gc.charge_oldgen_external(obj_addr, bytes));
+}
+
 /// `majit_gc::AllocOldgenTypedFn` installed by `set_gc_allocator`.
 /// Routes host-side allocations that need a stable (non-moving)
 /// pointer through the active cranelift-owned GC's old-gen. Used by
@@ -7736,6 +7759,15 @@ impl CraneliftBackend {
             supports_guard_gc_type,
         });
         majit_gc::set_active_alloc_nursery_typed(Some(alloc_nursery_typed_via_active_runtime));
+        majit_gc::set_active_alloc_nursery_collecting_typed(Some(
+            alloc_nursery_collecting_typed_via_active_runtime,
+        ));
+        majit_gc::set_active_charge_memory_pressure(Some(
+            charge_memory_pressure_via_active_runtime,
+        ));
+        majit_gc::set_active_charge_oldgen_external(Some(
+            charge_oldgen_external_via_active_runtime,
+        ));
         majit_gc::set_active_alloc_oldgen_typed(Some(alloc_oldgen_typed_via_active_runtime));
         majit_gc::set_active_collect_full(Some(collect_full_via_active_runtime));
         majit_gc::set_active_collect_oldgen(Some(collect_oldgen_nonmoving_via_active_runtime));

majit/majit-backend-dynasm/src/runner.rs

@@ -234,6 +234,43 @@ fn dynasm_alloc_nursery_typed(type_id: u32, size: usize) -> GcRef {
     })
 }
 
+/// Host-side *collecting* nursery allocation trampoline. Unlike
+/// [`dynasm_alloc_nursery_typed`], this runs a minor collection when the nursery
+/// is full (instead of spilling to old-gen). Used only by the elidable bigint
+/// payload helpers, which are invoked from a residual `CallR` whose gcmap roots
+/// the trace's live set and which hold no unrooted GC pointer across the
+/// allocation — so the embedded minor cycle is safe and dead bigints are
+/// reclaimed instead of accumulating in old-gen.
+fn dynasm_alloc_nursery_collecting_typed(type_id: u32, size: usize) -> GcRef {
+    DYNASM_ACTIVE_GC.with(|cell| {
+        let mut guard = cell.borrow_mut();
+        match guard.as_deref_mut() {
+            Some(gc) => gc.alloc_nursery_typed(type_id, size),
+            None => GcRef(0),
+        }
+    })
+}
+
+/// Host-side old-gen external-byte trampoline — charges off-heap
+/// (GC-invisible) bytes such as a freshly-built bignum's limb `Vec` on the
+/// active GC when the initialized object landed in old-gen. Never forces a
+/// minor collection.
+fn dynasm_charge_oldgen_external(obj_addr: usize, bytes: usize) {
+    DYNASM_ACTIVE_GC.with(|cell| {
+        if let Some(gc) = cell.borrow_mut().as_deref_mut() {
+            gc.charge_oldgen_external(obj_addr, bytes);
+        }
+    })
+}
+
+fn dynasm_charge_memory_pressure(bytes: usize) {
+    DYNASM_ACTIVE_GC.with(|cell| {
+        if let Some(gc) = cell.borrow_mut().as_deref_mut() {
+            gc.charge_memory_pressure(bytes);
+        }
+    })
+}
+
 /// Host-side old-gen allocation trampoline. Used by
 /// pyre-object allocators (`w_int_new`, `w_float_new`) whose
 /// callers cannot register the returned pointer as a GC root before
@@ -1228,6 +1265,11 @@ impl DynasmBackend {
             supports_guard_gc_type,
         });
         majit_gc::set_active_alloc_nursery_typed(Some(dynasm_alloc_nursery_typed));
+        majit_gc::set_active_alloc_nursery_collecting_typed(Some(
+            dynasm_alloc_nursery_collecting_typed,
+        ));
+        majit_gc::set_active_charge_memory_pressure(Some(dynasm_charge_memory_pressure));
+        majit_gc::set_active_charge_oldgen_external(Some(dynasm_charge_oldgen_external));
         majit_gc::set_active_alloc_oldgen_typed(Some(dynasm_alloc_oldgen_typed));
         majit_gc::set_active_collect_full(Some(dynasm_collect_full));
         majit_gc::set_active_collect_oldgen(Some(dynasm_collect_oldgen_nonmoving));