Typed access to metadata

Cargo.toml

@@ -34,6 +34,7 @@ enum-map = "=2.1"
 downcast-rs = "1.1.1"
 atomic-traits = "0.2.0"
 atomic = "0.4.6"
+num-traits = "0.2"
 spin = "0.5.2"
 env_logger = "0.8.2"
 pfm = { version = "0.1.0-beta.1", optional = true }
@@ -43,6 +44,7 @@ strum_macros = "0.24"
 
 [dev-dependencies]
 rand = "0.7.3"
+paste = "1.0.8"
 
 [build-dependencies]
 built = { version = "0.5.1", features = ["git2"] }

src/plan/barriers.rs

@@ -4,8 +4,7 @@ use atomic::Ordering;
 
 use crate::scheduler::gc_work::*;
 use crate::scheduler::WorkBucketStage;
-use crate::util::metadata::load_metadata;
-use crate::util::metadata::{compare_exchange_metadata, MetadataSpec};
+use crate::util::metadata::MetadataSpec;
 use crate::util::*;
 use crate::MMTK;
 
@@ -69,20 +68,24 @@ impl<E: ProcessEdgesWork> ObjectRememberingBarrier<E> {
             // Try set the bit from 1 to 0 (log object). This may fail, if
             // 1. the bit is cleared by others, or
             // 2. other bits in the same byte may get modified if we use side metadata
-            if compare_exchange_metadata::<E::VM>(
-                &self.meta,
-                object,
-                1,
-                0,
-                None,
-                Ordering::SeqCst,
-                Ordering::SeqCst,
-            ) {
+            if self
+                .meta
+                .compare_exchange_metadata::<E::VM, u8>(
+                    object,
+                    1,
+                    0,
+                    None,
+                    Ordering::SeqCst,
+                    Ordering::SeqCst,
+                )
+                .is_ok()
+            {
                 // We just logged the object
                 return true;
             } else {
-                let old_value =
-                    load_metadata::<E::VM>(&self.meta, object, None, Some(Ordering::SeqCst));
+                let old_value = self
+                    .meta
+                    .load_atomic::<E::VM, u8>(object, None, Ordering::SeqCst);
                 // If the bit is cleared before, someone else has logged the object. Return false.
                 if old_value == 0 {
                     return false;
@@ -104,7 +107,14 @@ impl<E: ProcessEdgesWork> ObjectRememberingBarrier<E> {
 
     #[inline(always)]
     fn barrier(&mut self, obj: ObjectReference) {
-        if load_metadata::<E::VM>(&self.meta, obj, None, None) == 0 {
+        // Perform a relaxed load for performance.
+        // It is okay if this check fails occasionally and
+        // the execution goes to the slowpath, we can take care of that in the slowpath.
+        if self
+            .meta
+            .load_atomic::<E::VM, u8>(obj, None, Ordering::Relaxed)
+            == 0
+        {
             return;
         }
         self.barrier_slow(obj);

src/policy/copyspace.rs

@@ -13,7 +13,7 @@ use crate::util::heap::HeapMeta;
 use crate::util::heap::VMRequest;
 use crate::util::heap::{MonotonePageResource, PageResource};
 use crate::util::metadata::side_metadata::{SideMetadataContext, SideMetadataSpec};
-use crate::util::metadata::{extract_side_metadata, side_metadata, MetadataSpec};
+use crate::util::metadata::{extract_side_metadata, MetadataSpec};
 use crate::util::object_forwarding;
 use crate::util::{Address, ObjectReference};
 use crate::vm::*;
@@ -182,11 +182,8 @@ impl<VM: VMBinding> CopySpace<VM> {
         if let MetadataSpec::OnSide(side_forwarding_status_table) =
             *<VM::VMObjectModel as ObjectModel<VM>>::LOCAL_FORWARDING_BITS_SPEC
         {
-            side_metadata::bzero_metadata(
-                &side_forwarding_status_table,
-                self.common.start,
-                self.pr.cursor() - self.common.start,
-            );
+            side_forwarding_status_table
+                .bzero_metadata(self.common.start, self.pr.cursor() - self.common.start);
         }
     }
 

src/policy/immix/block.rs

@@ -4,7 +4,7 @@ use super::line::Line;
 use super::ImmixSpace;
 use crate::util::constants::*;
 use crate::util::linear_scan::{Region, RegionIterator};
-use crate::util::metadata::side_metadata::{self, *};
+use crate::util::metadata::side_metadata::{MetadataByteArrayRef, SideMetadataSpec};
 use crate::util::Address;
 use crate::vm::*;
 use spin::{Mutex, MutexGuard};
@@ -125,16 +125,15 @@ impl Block {
     /// Get block mark state.
     #[inline(always)]
     pub fn get_state(&self) -> BlockState {
-        let byte =
-            side_metadata::load_atomic(&Self::MARK_TABLE, self.start(), Ordering::SeqCst) as u8;
+        let byte = Self::MARK_TABLE.load_atomic::<u8>(self.start(), Ordering::SeqCst);
         byte.into()
     }
 
     /// Set block mark state.
     #[inline(always)]
     pub fn set_state(&self, state: BlockState) {
-        let state = u8::from(state) as usize;
-        side_metadata::store_atomic(&Self::MARK_TABLE, self.start(), state, Ordering::SeqCst);
+        let state = u8::from(state);
+        Self::MARK_TABLE.store_atomic::<u8>(self.start(), state, Ordering::SeqCst);
     }
 
     // Defrag byte
@@ -144,9 +143,7 @@ impl Block {
     /// Test if the block is marked for defragmentation.
     #[inline(always)]
     pub fn is_defrag_source(&self) -> bool {
-        let byte =
-            side_metadata::load_atomic(&Self::DEFRAG_STATE_TABLE, self.start(), Ordering::SeqCst)
-                as u8;
+        let byte = Self::DEFRAG_STATE_TABLE.load_atomic::<u8>(self.start(), Ordering::SeqCst);
         debug_assert!(byte == 0 || byte == Self::DEFRAG_SOURCE_STATE);
         byte == Self::DEFRAG_SOURCE_STATE
     }
@@ -155,31 +152,19 @@ impl Block {
     #[inline(always)]
     pub fn set_as_defrag_source(&self, defrag: bool) {
         let byte = if defrag { Self::DEFRAG_SOURCE_STATE } else { 0 };
-        side_metadata::store_atomic(
-            &Self::DEFRAG_STATE_TABLE,
-            self.start(),
-            byte as usize,
-            Ordering::SeqCst,
-        );
+        Self::DEFRAG_STATE_TABLE.store_atomic::<u8>(self.start(), byte, Ordering::SeqCst);
     }
 
     /// Record the number of holes in the block.
     #[inline(always)]
     pub fn set_holes(&self, holes: usize) {
-        side_metadata::store_atomic(
-            &Self::DEFRAG_STATE_TABLE,
-            self.start(),
-            holes,
-            Ordering::SeqCst,
-        );
+        Self::DEFRAG_STATE_TABLE.store_atomic::<u8>(self.start(), holes as u8, Ordering::SeqCst);
     }
 
     /// Get the number of holes.
     #[inline(always)]
     pub fn get_holes(&self) -> usize {
-        let byte =
-            side_metadata::load_atomic(&Self::DEFRAG_STATE_TABLE, self.start(), Ordering::SeqCst)
-                as u8;
+        let byte = Self::DEFRAG_STATE_TABLE.load_atomic::<u8>(self.start(), Ordering::SeqCst);
         debug_assert_ne!(byte, Self::DEFRAG_SOURCE_STATE);
         byte as usize
     }
@@ -192,7 +177,7 @@ impl Block {
         } else {
             BlockState::Unmarked
         });
-        side_metadata::store_atomic(&Self::DEFRAG_STATE_TABLE, self.start(), 0, Ordering::SeqCst);
+        Self::DEFRAG_STATE_TABLE.store_atomic::<u8>(self.start(), 0, Ordering::SeqCst);
     }
 
     /// Deinitalize a block before releasing.

src/policy/immix/chunk.rs

@@ -2,7 +2,7 @@ use super::block::{Block, BlockState};
 use super::defrag::Histogram;
 use super::immixspace::ImmixSpace;
 use crate::util::linear_scan::{Region, RegionIterator};
-use crate::util::metadata::side_metadata::{self, SideMetadataSpec};
+use crate::util::metadata::side_metadata::SideMetadataSpec;
 use crate::{
     scheduler::*,
     util::{heap::layout::vm_layout_constants::LOG_BYTES_IN_CHUNK, Address},
@@ -111,7 +111,7 @@ impl ChunkMap {
             return;
         }
         // Update alloc byte
-        unsafe { side_metadata::store(&Self::ALLOC_TABLE, chunk.start(), state as u8 as _) };
+        unsafe { Self::ALLOC_TABLE.store::<u8>(chunk.start(), state as u8) };
         // If this is a newly allcoated chunk, then expand the chunk range.
         if state == ChunkState::Allocated {
             debug_assert!(!chunk.start().is_zero());
@@ -129,7 +129,7 @@ impl ChunkMap {
 
     /// Get chunk state
     pub fn get(&self, chunk: Chunk) -> ChunkState {
-        let byte = unsafe { side_metadata::load(&Self::ALLOC_TABLE, chunk.start()) as u8 };
+        let byte = unsafe { Self::ALLOC_TABLE.load::<u8>(chunk.start()) };
         match byte {
             0 => ChunkState::Free,
             1 => ChunkState::Allocated,

src/policy/immix/immixspace.rs

@@ -14,10 +14,8 @@ use crate::util::heap::HeapMeta;
 use crate::util::heap::PageResource;
 use crate::util::heap::VMRequest;
 use crate::util::linear_scan::{Region, RegionIterator};
-use crate::util::metadata::side_metadata::{self, *};
-use crate::util::metadata::{
-    self, compare_exchange_metadata, load_metadata, store_metadata, MetadataSpec,
-};
+use crate::util::metadata::side_metadata::{SideMetadataContext, SideMetadataSpec};
+use crate::util::metadata::{self, MetadataSpec};
 use crate::util::object_forwarding as ForwardingWord;
 use crate::util::{Address, ObjectReference};
 use crate::vm::*;
@@ -509,25 +507,26 @@ impl<VM: VMBinding> ImmixSpace<VM> {
     #[inline(always)]
     fn attempt_mark(&self, object: ObjectReference, mark_state: u8) -> bool {
         loop {
-            let old_value = load_metadata::<VM>(
-                &VM::VMObjectModel::LOCAL_MARK_BIT_SPEC,
+            let old_value = VM::VMObjectModel::LOCAL_MARK_BIT_SPEC.load_atomic::<VM, u8>(
                 object,
                 None,
-                Some(Ordering::SeqCst),
-            ) as u8;
+                Ordering::SeqCst,
+            );
             if old_value == mark_state {
                 return false;
             }
 
-            if compare_exchange_metadata::<VM>(
-                &VM::VMObjectModel::LOCAL_MARK_BIT_SPEC,
-                object,
-                old_value as usize,
-                mark_state as usize,
-                None,
-                Ordering::SeqCst,
-                Ordering::SeqCst,
-            ) {
+            if VM::VMObjectModel::LOCAL_MARK_BIT_SPEC
+                .compare_exchange_metadata::<VM, u8>(
+                    object,
+                    old_value,
+                    mark_state,
+                    None,
+                    Ordering::SeqCst,
+                    Ordering::SeqCst,
+                )
+                .is_ok()
+            {
                 break;
             }
         }
@@ -537,12 +536,11 @@ impl<VM: VMBinding> ImmixSpace<VM> {
     /// Check if an object is marked.
     #[inline(always)]
     fn is_marked(&self, object: ObjectReference, mark_state: u8) -> bool {
-        let old_value = load_metadata::<VM>(
-            &VM::VMObjectModel::LOCAL_MARK_BIT_SPEC,
+        let old_value = VM::VMObjectModel::LOCAL_MARK_BIT_SPEC.load_atomic::<VM, u8>(
             object,
             None,
-            Some(Ordering::SeqCst),
-        ) as u8;
+            Ordering::SeqCst,
+        );
         old_value == mark_state
     }
 
@@ -618,7 +616,7 @@ impl<VM: VMBinding> PrepareBlockState<VM> {
     #[inline(always)]
     fn reset_object_mark(chunk: Chunk) {
         if let MetadataSpec::OnSide(side) = *VM::VMObjectModel::LOCAL_MARK_BIT_SPEC {
-            side_metadata::bzero_metadata(&side, chunk.start(), Chunk::BYTES);
+            side.bzero_metadata(chunk.start(), Chunk::BYTES);
         }
     }
 }
@@ -689,12 +687,11 @@ impl<VM: VMBinding> PolicyCopyContext for ImmixCopyContext<VM> {
     #[inline(always)]
     fn post_copy(&mut self, obj: ObjectReference, _bytes: usize) {
         // Mark the object
-        store_metadata::<VM>(
-            &VM::VMObjectModel::LOCAL_MARK_BIT_SPEC,
+        VM::VMObjectModel::LOCAL_MARK_BIT_SPEC.store_atomic::<VM, u8>(
             obj,
-            self.get_space().mark_state as usize,
+            self.get_space().mark_state,
             None,
-            Some(Ordering::SeqCst),
+            Ordering::SeqCst,
         );
         // Mark the line
         if !super::MARK_LINE_AT_SCAN_TIME {

src/policy/immix/line.rs

@@ -1,6 +1,6 @@
 use super::block::Block;
 use crate::util::linear_scan::{Region, RegionIterator};
-use crate::util::metadata::side_metadata::{self, *};
+use crate::util::metadata::side_metadata::SideMetadataSpec;
 use crate::{
     util::{Address, ObjectReference},
     vm::*,
@@ -60,15 +60,15 @@ impl Line {
     pub fn mark(&self, state: u8) {
         debug_assert!(!super::BLOCK_ONLY);
         unsafe {
-            side_metadata::store(&Self::MARK_TABLE, self.start(), state as _);
+            Self::MARK_TABLE.store::<u8>(self.start(), state);
         }
     }
 
     /// Test line mark state.
     #[inline(always)]
     pub fn is_marked(&self, state: u8) -> bool {
         debug_assert!(!super::BLOCK_ONLY);
-        unsafe { side_metadata::load(&Self::MARK_TABLE, self.start()) as u8 == state }
+        unsafe { Self::MARK_TABLE.load::<u8>(self.start()) == state }
     }
 
     /// Mark all lines the object is spanned to.