Heap traversal

Cargo.toml

@@ -55,6 +55,7 @@ sysinfo = "0.30.9"
 [dev-dependencies]
 paste = "1.0.8"
 rand = "0.8.5"
+rand_chacha = "0.3.1"
 criterion = "0.4"
 
 [build-dependencies]

benches/regular_bench/bulk_meta/bscan.rs

@@ -0,0 +1,87 @@
+//! Benchmarks for scanning side metadata for non-zero bits.
+
+use criterion::Criterion;
+use mmtk::util::{
+    constants::LOG_BITS_IN_WORD, test_private::scan_non_zero_bits_in_metadata_bytes, Address,
+};
+use rand::{seq::IteratorRandom, SeedableRng};
+use rand_chacha::ChaCha8Rng;
+
+fn allocate_aligned(size: usize) -> Address {
+    let ptr = unsafe {
+        std::alloc::alloc_zeroed(std::alloc::Layout::from_size_align(size, size).unwrap())
+    };
+    Address::from_mut_ptr(ptr)
+}
+
+const BLOCK_BYTES: usize = 32768usize; // Match an Immix block size.
+
+// Asssume one-bit-per-word metadata (matching VO bits).
+const BLOCK_META_BYTES: usize = BLOCK_BYTES >> LOG_BITS_IN_WORD;
+
+/// Set this many distinct bits in the bitmap.
+const NUM_OBJECTS: usize = 200;
+
+/// Get a deterministic seeded Rng.
+fn get_rng() -> ChaCha8Rng {
+    // Create an Rng from a seed and an explicit Rng type.
+    // Not secure at all, but completely deterministic and reproducible.
+    // The following seed is read from /dev/random
+    const SEED64: u64 = 0x4050cb1b5ab26c70;
+    ChaCha8Rng::seed_from_u64(SEED64)
+}
+
+/// A bitmap, with known location of each bit for assertion.
+struct PreparedBitmap {
+    start: Address,
+    end: Address,
+    set_bits: Vec<(Address, u8)>,
+}
+
+/// Make a bitmap of the desired size and set bits.
+fn make_standard_bitmap() -> PreparedBitmap {
+    let start = allocate_aligned(BLOCK_META_BYTES);
+    let end = start + BLOCK_META_BYTES;
+    let mut rng = get_rng();
+
+    let mut set_bits = (0..(BLOCK_BYTES >> LOG_BITS_IN_WORD))
+        .choose_multiple(&mut rng, NUM_OBJECTS)
+        .iter()
+        .map(|total_bit_offset| {
+            let word_offset = total_bit_offset >> LOG_BITS_IN_WORD;
+            let bit_offset = total_bit_offset & ((1 << LOG_BITS_IN_WORD) - 1);
+            (start + (word_offset << LOG_BITS_IN_WORD), bit_offset as u8)
+        })
+        .collect::<Vec<_>>();
+
+    set_bits.sort();
+
+    for (addr, bit) in set_bits.iter() {
+        let word = unsafe { addr.load::<usize>() };
+        let new_word = word | (1 << bit);
+        unsafe { addr.store::<usize>(new_word) };
+    }
+
+    PreparedBitmap {
+        start,
+        end,
+        set_bits,
+    }
+}
+
+pub fn bench(c: &mut Criterion) {
+    c.bench_function("bscan_block", |b| {
+        let bitmap = make_standard_bitmap();
+        let mut holder: Vec<(Address, u8)> = Vec::with_capacity(NUM_OBJECTS);
+
+        b.iter(|| {
+            holder.clear();
+            scan_non_zero_bits_in_metadata_bytes(bitmap.start, bitmap.end, &mut |addr, shift| {
+                holder.push((addr, shift));
+            });
+        });
+
+        assert_eq!(holder.len(), NUM_OBJECTS);
+        assert_eq!(holder, bitmap.set_bits);
+    });
+}

benches/regular_bench/bulk_meta/mod.rs

@@ -1,7 +1,9 @@
+pub mod bscan;
 pub mod bzero_bset;
 
 pub use criterion::Criterion;
 
 pub fn bench(c: &mut Criterion) {
+    bscan::bench(c);
     bzero_bset::bench(c);
 }

src/mmtk.rs

@@ -6,6 +6,8 @@ use crate::plan::Plan;
 use crate::policy::sft_map::{create_sft_map, SFTMap};
 use crate::scheduler::GCWorkScheduler;
 
+#[cfg(feature = "vo_bit")]
+use crate::util::address::ObjectReference;
 #[cfg(feature = "analysis")]
 use crate::util::analysis::AnalysisManager;
 use crate::util::finalizable_processor::FinalizableProcessor;
@@ -467,4 +469,54 @@ impl<VM: VMBinding> MMTK<VM> {
     pub fn get_options(&self) -> &Options {
         &self.options
     }
+
+    /// Enumerate objects in all spaces in this MMTK instance.
+    ///
+    /// The call-back function `f` is called for every object that has the valid object bit (VO
+    /// bit), i.e. objects that are allocated in the heap of this MMTK instance, but has not been
+    /// reclaimed, yet.
+    ///
+    /// # Notes about object initialization and finalization
+    ///
+    /// When this function visits an object, it only guarantees that its VO bit must have been set.
+    /// It is not guaranteed if the object has been "fully initialized" in the sense of the
+    /// programming language the VM is implementing.  For example, the object header and the type
+    /// information may not have been written.
+    ///
+    /// It will also visit objects that have been "finalized" in the sense of the programming
+    /// langauge the VM is implementing, as long as the object has not been reclaimed by the GC,
+    /// yet.  Be careful.  If the object header is destroyed, it may not be safe to access such
+    /// objects in the high-level language.
+    ///
+    /// # Interaction with allocation and GC
+    ///
+    /// This function does not mutate the heap.  It is safe if multiple threads execute this
+    /// function concurrently during mutator time.
+    ///
+    /// It has *undefined behavior* if allocation or GC happens while this function is being
+    /// executed.  The VM binding must ensure no threads are allocating and GC does not start while
+    /// executing this function.  One way to do this is stopping all mutators before calling this
+    /// function.
+    ///
+    /// Some high-level languages may provide an API that allows the user to allocate objects and
+    /// trigger GC while enumerating objects.  One example is [`ObjectSpace::each_object`][os_eo] in
+    /// Ruby.  The VM binding may use the callback of this function to save all visited object
+    /// references and let the user visit those references after this function returns.  Make sure
+    /// those saved references are in the root set or in an object that will live through GCs before
+    /// the high-level language finishes visiting the saved object references.
+    ///
+    /// [os_eo]: https://docs.ruby-lang.org/en/master/ObjectSpace.html#method-c-each_object
+    #[cfg(feature = "vo_bit")]
+    pub fn enumerate_objects<F>(&self, f: F)
+    where
+        F: FnMut(ObjectReference),
+    {
+        use crate::util::object_enum;
+
+        let mut enumerator = object_enum::ClosureObjectEnumerator::<_, VM>::new(f);
+        let plan = self.get_plan();
+        plan.for_each_space(&mut |space| {
+            space.enumerate_objects(&mut enumerator);
+        })
+    }
 }

src/policy/copyspace.rs

@@ -6,10 +6,11 @@ use crate::policy::sft::SFT;
 use crate::policy::space::{CommonSpace, Space};
 use crate::scheduler::GCWorker;
 use crate::util::alloc::allocator::AllocatorContext;
-use crate::util::copy::*;
 use crate::util::heap::{MonotonePageResource, PageResource};
 use crate::util::metadata::{extract_side_metadata, MetadataSpec};
+use crate::util::object_enum::ObjectEnumerator;
 use crate::util::object_forwarding;
+use crate::util::{copy::*, object_enum};
 use crate::util::{Address, ObjectReference};
 use crate::vm::*;
 use libc::{mprotect, PROT_EXEC, PROT_NONE, PROT_READ, PROT_WRITE};
@@ -133,6 +134,10 @@ impl<VM: VMBinding> Space<VM> for CopySpace<VM> {
     fn set_copy_for_sft_trace(&mut self, semantics: Option<CopySemantics>) {
         self.common.copy = semantics;
     }
+
+    fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator) {
+        object_enum::enumerate_blocks_from_monotonic_page_resource(enumerator, &self.pr);
+    }
 }
 
 impl<VM: VMBinding> crate::policy::gc_work::PolicyTraceObject<VM> for CopySpace<VM> {

src/policy/immix/block.rs

@@ -10,6 +10,7 @@ use crate::util::metadata::side_metadata::{MetadataByteArrayRef, SideMetadataSpe
 use crate::util::metadata::vo_bit;
 #[cfg(feature = "object_pinning")]
 use crate::util::metadata::MetadataSpec;
+use crate::util::object_enum::BlockMayHaveObjects;
 use crate::util::Address;
 use crate::vm::*;
 use std::sync::atomic::Ordering;
@@ -86,6 +87,12 @@ impl Region for Block {
     }
 }
 
+impl BlockMayHaveObjects for Block {
+    fn may_have_objects(&self) -> bool {
+        self.get_state() != BlockState::Unallocated
+    }
+}
+
 impl Block {
     /// Log pages in block
     pub const LOG_PAGES: usize = Self::LOG_BYTES - LOG_BYTES_IN_PAGE as usize;

src/policy/immix/immixspace.rs

@@ -17,8 +17,9 @@ use crate::util::metadata::side_metadata::SideMetadataSpec;
 #[cfg(feature = "vo_bit")]
 use crate::util::metadata::vo_bit;
 use crate::util::metadata::{self, MetadataSpec};
+use crate::util::object_enum::ObjectEnumerator;
 use crate::util::object_forwarding;
-use crate::util::{copy::*, epilogue};
+use crate::util::{copy::*, epilogue, object_enum};
 use crate::util::{Address, ObjectReference};
 use crate::vm::*;
 use crate::{
@@ -189,6 +190,10 @@ impl<VM: VMBinding> Space<VM> for ImmixSpace<VM> {
     fn set_copy_for_sft_trace(&mut self, _semantics: Option<CopySemantics>) {
         panic!("We do not use SFT to trace objects for Immix. set_copy_context() cannot be used.")
     }
+
+    fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator) {
+        object_enum::enumerate_blocks_from_chunk_map::<Block>(enumerator, &self.chunk_map);
+    }
 }
 
 impl<VM: VMBinding> crate::policy::gc_work::PolicyTraceObject<VM> for ImmixSpace<VM> {

src/policy/immortalspace.rs

@@ -6,6 +6,7 @@ use crate::util::address::Address;
 use crate::util::heap::{MonotonePageResource, PageResource};
 use crate::util::metadata::mark_bit::MarkState;
 
+use crate::util::object_enum::{self, ObjectEnumerator};
 use crate::util::{metadata, ObjectReference};
 
 use crate::plan::{ObjectQueue, VectorObjectQueue};
@@ -112,6 +113,10 @@ impl<VM: VMBinding> Space<VM> for ImmortalSpace<VM> {
     fn release_multiple_pages(&mut self, _start: Address) {
         panic!("immortalspace only releases pages enmasse")
     }
+
+    fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator) {
+        object_enum::enumerate_blocks_from_monotonic_page_resource(enumerator, &self.pr);
+    }
 }
 
 use crate::scheduler::GCWorker;

src/policy/largeobjectspace.rs

@@ -8,6 +8,7 @@ use crate::policy::space::{CommonSpace, Space};
 use crate::util::constants::BYTES_IN_PAGE;
 use crate::util::heap::{FreeListPageResource, PageResource};
 use crate::util::metadata;
+use crate::util::object_enum::ObjectEnumerator;
 use crate::util::opaque_pointer::*;
 use crate::util::treadmill::TreadMill;
 use crate::util::{Address, ObjectReference};
@@ -175,6 +176,10 @@ impl<VM: VMBinding> Space<VM> for LargeObjectSpace<VM> {
     fn release_multiple_pages(&mut self, start: Address) {
         self.pr.release_pages(start);
     }
+
+    fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator) {
+        self.treadmill.enumerate_objects(enumerator);
+    }
 }
 
 use crate::scheduler::GCWorker;

src/policy/lockfreeimmortalspace.rs

@@ -16,6 +16,7 @@ use crate::util::heap::VMRequest;
 use crate::util::memory::MmapStrategy;
 use crate::util::metadata::side_metadata::SideMetadataContext;
 use crate::util::metadata::side_metadata::SideMetadataSanity;
+use crate::util::object_enum::ObjectEnumerator;
 use crate::util::opaque_pointer::*;
 use crate::util::ObjectReference;
 use crate::vm::VMBinding;
@@ -166,6 +167,10 @@ impl<VM: VMBinding> Space<VM> for LockFreeImmortalSpace<VM> {
         side_metadata_sanity_checker
             .verify_metadata_context(std::any::type_name::<Self>(), &self.metadata)
     }
+
+    fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator) {
+        enumerator.visit_address_range(self.start, self.start + self.total_bytes);
+    }
 }
 
 use crate::plan::{ObjectQueue, VectorObjectQueue};

src/policy/markcompactspace.rs

@@ -11,6 +11,7 @@ use crate::util::constants::LOG_BYTES_IN_WORD;
 use crate::util::copy::CopySemantics;
 use crate::util::heap::{MonotonePageResource, PageResource};
 use crate::util::metadata::{extract_side_metadata, vo_bit};
+use crate::util::object_enum::{self, ObjectEnumerator};
 use crate::util::{Address, ObjectReference};
 use crate::{vm::*, ObjectQueue};
 use atomic::Ordering;
@@ -131,6 +132,10 @@ impl<VM: VMBinding> Space<VM> for MarkCompactSpace<VM> {
     fn release_multiple_pages(&mut self, _start: Address) {
         panic!("markcompactspace only releases pages enmasse")
     }
+
+    fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator) {
+        object_enum::enumerate_blocks_from_monotonic_page_resource(enumerator, &self.pr);
+    }
 }
 
 impl<VM: VMBinding> crate::policy::gc_work::PolicyTraceObject<VM> for MarkCompactSpace<VM> {

src/policy/marksweepspace/malloc_ms/global.rs

@@ -15,6 +15,7 @@ use crate::util::metadata::side_metadata::{
     SideMetadataContext, SideMetadataSanity, SideMetadataSpec,
 };
 use crate::util::metadata::MetadataSpec;
+use crate::util::object_enum::ObjectEnumerator;
 use crate::util::opaque_pointer::*;
 use crate::util::Address;
 use crate::util::ObjectReference;
@@ -229,6 +230,10 @@ impl<VM: VMBinding> Space<VM> for MallocSpace<VM> {
         side_metadata_sanity_checker
             .verify_metadata_context(std::any::type_name::<Self>(), &self.metadata)
     }
+
+    fn enumerate_objects(&self, _enumerator: &mut dyn ObjectEnumerator) {
+        unimplemented!()
+    }
 }
 
 use crate::scheduler::GCWorker;

src/policy/marksweepspace/native_ms/block.rs

@@ -7,6 +7,7 @@ use super::MarkSweepSpace;
 use crate::util::constants::LOG_BYTES_IN_PAGE;
 use crate::util::heap::chunk_map::*;
 use crate::util::linear_scan::Region;
+use crate::util::object_enum::BlockMayHaveObjects;
 use crate::vm::ObjectModel;
 use crate::{
     util::{
@@ -48,6 +49,12 @@ impl Region for Block {
     }
 }
 
+impl BlockMayHaveObjects for Block {
+    fn may_have_objects(&self) -> bool {
+        self.get_state() != BlockState::Unallocated
+    }
+}
+
 impl Block {
     /// Log pages in block
     pub const LOG_PAGES: usize = Self::LOG_BYTES - LOG_BYTES_IN_PAGE as usize;

src/policy/marksweepspace/native_ms/global.rs

@@ -11,6 +11,7 @@ use crate::{
         epilogue,
         heap::{BlockPageResource, PageResource},
         metadata::{self, side_metadata::SideMetadataSpec, MetadataSpec},
+        object_enum::{self, ObjectEnumerator},
         ObjectReference,
     },
     vm::{ActivePlan, VMBinding},
@@ -247,6 +248,10 @@ impl<VM: VMBinding> Space<VM> for MarkSweepSpace<VM> {
     fn release_multiple_pages(&mut self, _start: crate::util::Address) {
         todo!()
     }
+
+    fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator) {
+        object_enum::enumerate_blocks_from_chunk_map::<Block>(enumerator, &self.chunk_map);
+    }
 }
 
 impl<VM: VMBinding> crate::policy::gc_work::PolicyTraceObject<VM> for MarkSweepSpace<VM> {