8305896: Alternative full GC forwarding

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -86,6 +86,7 @@
 #include "gc/shared/oopStorageParState.hpp"
 #include "gc/shared/preservedMarks.inline.hpp"
 #include "gc/shared/referenceProcessor.inline.hpp"
+#include "gc/shared/slidingForwarding.hpp"
 #include "gc/shared/suspendibleThreadSet.hpp"
 #include "gc/shared/taskqueue.inline.hpp"
 #include "gc/shared/taskTerminator.hpp"
@@ -1440,6 +1441,8 @@ jint G1CollectedHeap::initialize() {
 
   G1InitLogger::print();
 
+  SlidingForwarding::initialize(heap_rs.region(), HeapRegion::GrainWords);
+
   return JNI_OK;
 }
 

src/hotspot/share/gc/g1/g1FullCollector.cpp

@@ -40,6 +40,7 @@
 #include "gc/shared/preservedMarks.inline.hpp"
 #include "gc/shared/classUnloadingContext.hpp"
 #include "gc/shared/referenceProcessor.hpp"
+#include "gc/shared/slidingForwarding.hpp"
 #include "gc/shared/verifyOption.hpp"
 #include "gc/shared/weakProcessor.inline.hpp"
 #include "gc/shared/workerPolicy.hpp"
@@ -212,6 +213,8 @@ void G1FullCollector::collect() {
   // Don't add any more derived pointers during later phases
   deactivate_derived_pointers();
 
+  SlidingForwarding::begin();
+
   phase2_prepare_compaction();
 
   if (has_compaction_targets()) {
@@ -224,6 +227,8 @@ void G1FullCollector::collect() {
     log_info(gc, phases) ("No Regions selected for compaction. Skipping Phase 3: Adjust pointers and Phase 4: Compact heap");
   }
 
+  SlidingForwarding::end();
+
   phase5_reset_metadata();
 
   G1CollectedHeap::finish_codecache_marking_cycle();

src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp

@@ -30,6 +30,7 @@
 #include "gc/g1/g1FullGCCompactTask.hpp"
 #include "gc/g1/heapRegion.inline.hpp"
 #include "gc/shared/gcTraceTime.inline.hpp"
+#include "gc/shared/slidingForwarding.inline.hpp"
 #include "logging/log.hpp"
 #include "oops/oop.inline.hpp"
 #include "utilities/ticks.hpp"
@@ -41,7 +42,7 @@ void G1FullGCCompactTask::G1CompactRegionClosure::clear_in_bitmap(oop obj) {
 
 size_t G1FullGCCompactTask::G1CompactRegionClosure::apply(oop obj) {
   size_t size = obj->size();
-  if (obj->is_forwarded()) {
+  if (SlidingForwarding::is_forwarded(obj)) {
     G1FullGCCompactTask::copy_object_to_new_location(obj);
   }
 
@@ -52,13 +53,13 @@ size_t G1FullGCCompactTask::G1CompactRegionClosure::apply(oop obj) {
 }
 
 void G1FullGCCompactTask::copy_object_to_new_location(oop obj) {
-  assert(obj->is_forwarded(), "Sanity!");
-  assert(obj->forwardee() != obj, "Object must have a new location");
+  assert(SlidingForwarding::is_forwarded(obj), "Sanity!");
+  assert(SlidingForwarding::forwardee(obj) != obj, "Object must have a new location");
 
   size_t size = obj->size();
   // Copy object and reinit its mark.
   HeapWord* obj_addr = cast_from_oop<HeapWord*>(obj);
-  HeapWord* destination = cast_from_oop<HeapWord*>(obj->forwardee());
+  HeapWord* destination = cast_from_oop<HeapWord*>(SlidingForwarding::forwardee(obj));
   Copy::aligned_conjoint_words(obj_addr, destination, size);
 
   // There is no need to transform stack chunks - marking already did that.
@@ -121,7 +122,7 @@ void G1FullGCCompactTask::compact_humongous_obj(HeapRegion* src_hr) {
   size_t word_size = obj->size();
 
   uint num_regions = (uint)G1CollectedHeap::humongous_obj_size_in_regions(word_size);
-  HeapWord* destination = cast_from_oop<HeapWord*>(obj->forwardee());
+  HeapWord* destination = cast_from_oop<HeapWord*>(SlidingForwarding::forwardee(obj));
 
   assert(collector()->mark_bitmap()->is_marked(obj), "Should only compact marked objects");
   collector()->mark_bitmap()->clear(obj);

src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp

@@ -27,6 +27,7 @@
 #include "gc/g1/g1FullGCCompactionPoint.hpp"
 #include "gc/g1/heapRegion.hpp"
 #include "gc/shared/preservedMarks.inline.hpp"
+#include "gc/shared/slidingForwarding.inline.hpp"
 #include "oops/oop.inline.hpp"
 #include "utilities/debug.hpp"
 
@@ -106,10 +107,10 @@ void G1FullGCCompactionPoint::forward(oop object, size_t size) {
     if (!object->is_forwarded()) {
       preserved_stack()->push_if_necessary(object, object->mark());
     }
-    object->forward_to(cast_to_oop(_compaction_top));
-    assert(object->is_forwarded(), "must be forwarded");
+    SlidingForwarding::forward_to(object, cast_to_oop(_compaction_top));
+    assert(SlidingForwarding::is_forwarded(object), "must be forwarded");
   } else {
-    assert(!object->is_forwarded(), "must not be forwarded");
+    assert(SlidingForwarding::is_not_forwarded(object), "must not be forwarded");
   }
 
   // Update compaction values.
@@ -172,8 +173,8 @@ void G1FullGCCompactionPoint::forward_humongous(HeapRegion* hr) {
   preserved_stack()->push_if_necessary(obj, obj->mark());
 
   HeapRegion* dest_hr = _compaction_regions->at(range_begin);
-  obj->forward_to(cast_to_oop(dest_hr->bottom()));
-  assert(obj->is_forwarded(), "Object must be forwarded!");
+  SlidingForwarding::forward_to(obj, cast_to_oop(dest_hr->bottom()));
+  assert(SlidingForwarding::is_forwarded(obj), "Object must be forwarded!");
 
   // Add the humongous object regions to the compaction point.
   add_humongous(hr);

src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,6 +32,7 @@
 #include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp"
 #include "gc/g1/g1FullGCMarker.inline.hpp"
 #include "gc/g1/heapRegionRemSet.inline.hpp"
+#include "gc/shared/slidingForwarding.inline.hpp"
 #include "memory/iterator.inline.hpp"
 #include "memory/universe.hpp"
 #include "oops/access.inline.hpp"
@@ -65,8 +66,8 @@ template <class T> inline void G1AdjustClosure::adjust_pointer(T* p) {
     return;
   }
 
-  if (obj->is_forwarded()) {
-    oop forwardee = obj->forwardee();
+  if (SlidingForwarding::is_forwarded(obj)) {
+    oop forwardee = SlidingForwarding::forwardee(obj);
     // Forwarded, just update.
     assert(G1CollectedHeap::heap()->is_in_reserved(forwardee), "should be in object space");
     RawAccess<IS_NOT_NULL>::oop_store(p, forwardee);

src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp

@@ -32,6 +32,7 @@
 #include "gc/g1/g1FullGCCompactionPoint.hpp"
 #include "gc/g1/g1FullGCScope.hpp"
 #include "gc/g1/heapRegion.inline.hpp"
+#include "gc/shared/slidingForwarding.inline.hpp"
 
 void G1DetermineCompactionQueueClosure::free_empty_humongous_region(HeapRegion* hr) {
   _g1h->free_humongous_region(hr, nullptr);
@@ -114,10 +115,10 @@ inline bool G1DetermineCompactionQueueClosure::do_heap_region(HeapRegion* hr) {
 }
 
 inline size_t G1SerialRePrepareClosure::apply(oop obj) {
-  if (obj->is_forwarded()) {
+  if (SlidingForwarding::is_forwarded(obj)) {
     // We skip objects compiled into the first region or
     // into regions not part of the serial compaction point.
-    if (cast_from_oop<HeapWord*>(obj->forwardee()) < _dense_prefix_top) {
+    if (cast_from_oop<HeapWord*>(SlidingForwarding::forwardee(obj)) < _dense_prefix_top) {
       return obj->size();
     }
   }

src/hotspot/share/gc/serial/genMarkSweep.cpp

@@ -48,6 +48,7 @@
 #include "gc/shared/preservedMarks.inline.hpp"
 #include "gc/shared/referencePolicy.hpp"
 #include "gc/shared/referenceProcessorPhaseTimes.hpp"
+#include "gc/shared/slidingForwarding.inline.hpp"
 #include "gc/shared/space.inline.hpp"
 #include "gc/shared/strongRootsScope.hpp"
 #include "gc/shared/weakProcessor.hpp"
@@ -198,7 +199,7 @@ class Compacter {
   static void forward_obj(oop obj, HeapWord* new_addr) {
     prefetch_write_scan(obj);
     if (cast_from_oop<HeapWord*>(obj) != new_addr) {
-      obj->forward_to(cast_to_oop(new_addr));
+      SlidingForwarding::forward_to(obj, cast_to_oop(new_addr));
     } else {
       assert(obj->is_gc_marked(), "inv");
       // This obj will stay in-place. Fix the markword.
@@ -223,7 +224,7 @@ class Compacter {
     prefetch_read_scan(addr);
 
     oop obj = cast_to_oop(addr);
-    oop new_obj = obj->forwardee();
+    oop new_obj = SlidingForwarding::forwardee(obj);
     HeapWord* new_addr = cast_from_oop<HeapWord*>(new_obj);
     assert(addr != new_addr, "inv");
     prefetch_write_copy(new_addr);
@@ -319,13 +320,13 @@ class Compacter {
       HeapWord* top = space->top();
 
       // Check if the first obj inside this space is forwarded.
-      if (!cast_to_oop(cur_addr)->is_forwarded()) {
+      if (SlidingForwarding::is_not_forwarded(cast_to_oop(cur_addr))) {
         // Jump over consecutive (in-place) live-objs-chunk
         cur_addr = get_first_dead(i);
       }
 
       while (cur_addr < top) {
-        if (!cast_to_oop(cur_addr)->is_forwarded()) {
+        if (SlidingForwarding::is_not_forwarded(cast_to_oop(cur_addr))) {
           cur_addr = *(HeapWord**) cur_addr;
           continue;
         }
@@ -448,6 +449,8 @@ void GenMarkSweep::invoke_at_safepoint(bool clear_all_softrefs) {
 
   phase1_mark(clear_all_softrefs);
 
+  SlidingForwarding::begin();
+
   Compacter compacter{gch};
 
   {
@@ -495,6 +498,8 @@ void GenMarkSweep::invoke_at_safepoint(bool clear_all_softrefs) {
   // (Should this be in general part?)
   gch->save_marks();
 
+  SlidingForwarding::end();
+
   deallocate_stacks();
 
   MarkSweep::_string_dedup_requests->flush();

src/hotspot/share/gc/serial/markSweep.inline.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -31,6 +31,7 @@
 #include "classfile/javaClasses.inline.hpp"
 #include "gc/shared/continuationGCSupport.inline.hpp"
 #include "gc/serial/serialStringDedup.hpp"
+#include "gc/shared/slidingForwarding.inline.hpp"
 #include "memory/universe.hpp"
 #include "oops/markWord.hpp"
 #include "oops/access.inline.hpp"
@@ -45,8 +46,8 @@ template <class T> inline void MarkSweep::adjust_pointer(T* p) {
     oop obj = CompressedOops::decode_not_null(heap_oop);
     assert(Universe::heap()->is_in(obj), "should be in heap");
 
-    if (obj->is_forwarded()) {
-      oop new_obj = obj->forwardee();
+    if (SlidingForwarding::is_forwarded(obj)) {
+      oop new_obj = SlidingForwarding::forwardee(obj);
       assert(is_object_aligned(new_obj), "oop must be aligned");
       RawAccess<IS_NOT_NULL>::oop_store(p, new_obj);
     }

src/hotspot/share/gc/serial/serialHeap.cpp

@@ -55,6 +55,7 @@
 #include "gc/shared/oopStorageParState.inline.hpp"
 #include "gc/shared/oopStorageSet.inline.hpp"
 #include "gc/shared/scavengableNMethods.hpp"
+#include "gc/shared/slidingForwarding.hpp"
 #include "gc/shared/space.hpp"
 #include "gc/shared/strongRootsScope.hpp"
 #include "gc/shared/suspendibleThreadSet.hpp"
@@ -203,6 +204,8 @@ jint SerialHeap::initialize() {
 
   GCInitLogger::print();
 
+  SlidingForwarding::initialize(_reserved, SpaceAlignment / HeapWordSize);
+
   return JNI_OK;
 }
 

src/hotspot/share/gc/shared/preservedMarks.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,7 @@
 
 #include "precompiled.hpp"
 #include "gc/shared/preservedMarks.inline.hpp"
+#include "gc/shared/slidingForwarding.inline.hpp"
 #include "gc/shared/workerThread.hpp"
 #include "gc/shared/workerUtils.hpp"
 #include "memory/allocation.inline.hpp"
@@ -43,7 +44,7 @@ void PreservedMarks::restore() {
 void PreservedMarks::adjust_preserved_mark(PreservedMark* elem) {
   oop obj = elem->get_oop();
   if (obj->is_forwarded()) {
-    elem->set_oop(obj->forwardee());
+    elem->set_oop(SlidingForwarding::forwardee(obj));
   }
 }