8253237: [REDO] Improve large object handling during evacuation

8253238: [REDO] Improve object array chunking test in G1's copy_to_survivor_space
8253236: [REDO] Investigate fast-path for scanning only objects with references during gc

Generate multiple partial array tasks for large objArrays.

Reviewed-by: sjohanss, ayang, tschatzl

Author: Kim Barrett

src/hotspot/share/gc/g1/g1ParScanThreadState.cpp

@@ -31,10 +31,12 @@
 #include "gc/g1/g1RootClosures.hpp"
 #include "gc/g1/g1StringDedup.hpp"
 #include "gc/g1/g1Trace.hpp"
+#include "gc/shared/partialArrayTaskStepper.inline.hpp"
 #include "gc/shared/taskqueue.inline.hpp"
 #include "memory/allocation.inline.hpp"
 #include "oops/access.inline.hpp"
 #include "oops/oop.inline.hpp"
+#include "runtime/atomic.hpp"
 #include "runtime/prefetch.inline.hpp"
 #include "utilities/globalDefinitions.hpp"
 #include "utilities/macros.hpp"
@@ -50,6 +52,7 @@
 G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h,
                                            G1RedirtyCardsQueueSet* rdcqs,
                                            uint worker_id,
+                                           uint n_workers,
                                            size_t young_cset_length,
                                            size_t optional_cset_length)
   : _g1h(g1h),
@@ -70,6 +73,8 @@ G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h,
     _surviving_young_words(NULL),
     _surviving_words_length(young_cset_length + 1),
     _old_gen_is_full(false),
+    _partial_objarray_chunk_size(ParGCArrayScanChunk),
+    _partial_array_stepper(n_workers),
     _num_optional_regions(optional_cset_length),
     _numa(g1h->numa()),
     _obj_alloc_stat(NULL)
@@ -212,48 +217,60 @@ void G1ParScanThreadState::do_partial_array(PartialArrayScanTask task) {
 
   assert(_g1h->is_in_reserved(from_obj), "must be in heap.");
   assert(from_obj->is_objArray(), "must be obj array");
-  objArrayOop from_obj_array = objArrayOop(from_obj);
-  // The from-space object contains the real length.
-  int length                 = from_obj_array->length();
-
   assert(from_obj->is_forwarded(), "must be forwarded");
-  oop to_obj                 = from_obj->forwardee();
+
+  oop to_obj = from_obj->forwardee();
   assert(from_obj != to_obj, "should not be chunking self-forwarded objects");
-  objArrayOop to_obj_array   = objArrayOop(to_obj);
-  // We keep track of the next start index in the length field of the
-  // to-space object.
-  int next_index             = to_obj_array->length();
-  assert(0 <= next_index && next_index < length,
-         "invariant, next index: %d, length: %d", next_index, length);
-
-  int start                  = next_index;
-  int end                    = length;
-  int remainder              = end - start;
-  // We'll try not to push a range that's smaller than ParGCArrayScanChunk.
-  if (remainder > 2 * ParGCArrayScanChunk) {
-    end = start + ParGCArrayScanChunk;
-    to_obj_array->set_length(end);
-    // Push the remainder before we process the range in case another
-    // worker has run out of things to do and can steal it.
+  assert(to_obj->is_objArray(), "must be obj array");
+  objArrayOop to_array = objArrayOop(to_obj);
+
+  PartialArrayTaskStepper::Step step
+    = _partial_array_stepper.next(objArrayOop(from_obj),
+                                  to_array,
+                                  _partial_objarray_chunk_size);
+  for (uint i = 0; i < step._ncreate; ++i) {
     push_on_queue(ScannerTask(PartialArrayScanTask(from_obj)));
-  } else {
-    assert(length == end, "sanity");
-    // We'll process the final range for this object. Restore the length
-    // so that the heap remains parsable in case of evacuation failure.
-    to_obj_array->set_length(end);
   }
 
-  HeapRegion* hr = _g1h->heap_region_containing(to_obj);
+  HeapRegion* hr = _g1h->heap_region_containing(to_array);
   G1ScanInYoungSetter x(&_scanner, hr->is_young());
-  // Process indexes [start,end). It will also process the header
-  // along with the first chunk (i.e., the chunk with start == 0).
-  // Note that at this point the length field of to_obj_array is not
-  // correct given that we are using it to keep track of the next
-  // start index. oop_iterate_range() (thankfully!) ignores the length
-  // field and only relies on the start / end parameters.  It does
-  // however return the size of the object which will be incorrect. So
-  // we have to ignore it even if we wanted to use it.
-  to_obj_array->oop_iterate_range(&_scanner, start, end);
+  // Process claimed task.  The length of to_array is not correct, but
+  // fortunately the iteration ignores the length field and just relies
+  // on start/end.
+  to_array->oop_iterate_range(&_scanner,
+                              step._index,
+                              step._index + _partial_objarray_chunk_size);
+}
+
+MAYBE_INLINE_EVACUATION
+void G1ParScanThreadState::start_partial_objarray(G1HeapRegionAttr dest_attr,
+                                                  oop from_obj,
+                                                  oop to_obj) {
+  assert(from_obj->is_objArray(), "precondition");
+  assert(from_obj->is_forwarded(), "precondition");
+  assert(from_obj->forwardee() == to_obj, "precondition");
+  assert(from_obj != to_obj, "should not be scanning self-forwarded objects");
+  assert(to_obj->is_objArray(), "precondition");
+
+  objArrayOop to_array = objArrayOop(to_obj);
+
+  PartialArrayTaskStepper::Step step
+    = _partial_array_stepper.start(objArrayOop(from_obj),
+                                   to_array,
+                                   _partial_objarray_chunk_size);
+
+  // Push any needed partial scan tasks.  Pushed before processing the
+  // intitial chunk to allow other workers to steal while we're processing.
+  for (uint i = 0; i < step._ncreate; ++i) {
+    push_on_queue(ScannerTask(PartialArrayScanTask(from_obj)));
+  }
+
+  G1ScanInYoungSetter x(&_scanner, dest_attr.is_young());
+  // Process the initial chunk.  No need to process the type in the
+  // klass, as it will already be handled by processing the built-in
+  // module. The length of to_array is not correct, but fortunately
+  // the iteration ignores that length field and relies on start/end.
+  to_array->oop_iterate_range(&_scanner, 0, step._index);
 }
 
 MAYBE_INLINE_EVACUATION
@@ -409,7 +426,10 @@ oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const regio
   assert(region_attr.is_in_cset(),
          "Unexpected region attr type: %s", region_attr.get_type_str());
 
-  const size_t word_sz = old->size();
+  // Get the klass once.  We'll need it again later, and this avoids
+  // re-decoding when it's compressed.
+  Klass* klass = old->klass();
+  const size_t word_sz = old->size_given_klass(klass);
 
   uint age = 0;
   G1HeapRegionAttr dest_attr = next_region_attr(region_attr, old_mark, age);
@@ -476,6 +496,20 @@ oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const regio
       obj->set_mark_raw(old_mark);
     }
 
+    // Most objects are not arrays, so do one array check rather than
+    // checking for each array category for each object.
+    if (klass->is_array_klass()) {
+      if (klass->is_objArray_klass()) {
+        start_partial_objarray(dest_attr, old, obj);
+      } else {
+        // Nothing needs to be done for typeArrays.  Body doesn't contain
+        // any oops to scan, and the type in the klass will already be handled
+        // by processing the built-in module.
+        assert(klass->is_typeArray_klass(), "invariant");
+      }
+      return obj;
+    }
+
     if (G1StringDedup::is_enabled()) {
       const bool is_from_young = region_attr.is_young();
       const bool is_to_young = dest_attr.is_young();
@@ -489,17 +523,10 @@ oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const regio
                                              obj);
     }
 
-    if (obj->is_objArray() && arrayOop(obj)->length() >= ParGCArrayScanChunk) {
-      // We keep track of the next start index in the length field of
-      // the to-space object. The actual length can be found in the
-      // length field of the from-space object.
-      arrayOop(obj)->set_length(0);
-      do_partial_array(PartialArrayScanTask(old));
-    } else {
-      G1ScanInYoungSetter x(&_scanner, dest_attr.is_young());
-      obj->oop_iterate_backwards(&_scanner);
-    }
+    G1ScanInYoungSetter x(&_scanner, dest_attr.is_young());
+    obj->oop_iterate_backwards(&_scanner);
     return obj;
+
   } else {
     _plab_allocator->undo_allocation(dest_attr, obj_ptr, word_sz, node_index);
     return forward_ptr;
@@ -518,7 +545,9 @@ G1ParScanThreadState* G1ParScanThreadStateSet::state_for_worker(uint worker_id)
   assert(worker_id < _n_workers, "out of bounds access");
   if (_states[worker_id] == NULL) {
     _states[worker_id] =
-      new G1ParScanThreadState(_g1h, _rdcqs, worker_id, _young_cset_length, _optional_cset_length);
+      new G1ParScanThreadState(_g1h, _rdcqs,
+                               worker_id, _n_workers,
+                               _young_cset_length, _optional_cset_length);
   }
   return _states[worker_id];
 }

src/hotspot/share/gc/g1/g1ParScanThreadState.hpp

@@ -32,6 +32,7 @@
 #include "gc/g1/g1RemSet.hpp"
 #include "gc/g1/heapRegionRemSet.hpp"
 #include "gc/shared/ageTable.hpp"
+#include "gc/shared/partialArrayTaskStepper.hpp"
 #include "gc/shared/taskqueue.hpp"
 #include "memory/allocation.hpp"
 #include "oops/oop.hpp"
@@ -79,6 +80,9 @@ class G1ParScanThreadState : public CHeapObj<mtGC> {
   // Indicates whether in the last generation (old) there is no more space
   // available for allocation.
   bool _old_gen_is_full;
+  // Size (in elements) of a partial objArray task chunk.
+  int _partial_objarray_chunk_size;
+  PartialArrayTaskStepper _partial_array_stepper;
 
   G1RedirtyCardsQueue& redirty_cards_queue()     { return _rdcq; }
   G1CardTable* ct()                              { return _ct; }
@@ -105,6 +109,7 @@ class G1ParScanThreadState : public CHeapObj<mtGC> {
   G1ParScanThreadState(G1CollectedHeap* g1h,
                        G1RedirtyCardsQueueSet* rdcqs,
                        uint worker_id,
+                       uint n_workers,
                        size_t young_cset_length,
                        size_t optional_cset_length);
   virtual ~G1ParScanThreadState();
@@ -157,6 +162,7 @@ class G1ParScanThreadState : public CHeapObj<mtGC> {
 
 private:
   void do_partial_array(PartialArrayScanTask task);
+  void start_partial_objarray(G1HeapRegionAttr dest_dir, oop from, oop to);
 
   HeapWord* allocate_copy_slow(G1HeapRegionAttr* dest_attr,
                                oop old,
@@ -249,9 +255,6 @@ class G1ParScanThreadStateSet : public StackObj {
   G1ParScanThreadState* state_for_worker(uint worker_id);
 
   const size_t* surviving_young_words() const;
-
- private:
-  G1ParScanThreadState* new_par_scan_state(uint worker_id, size_t young_cset_length);
 };
 
 #endif // SHARE_GC_G1_G1PARSCANTHREADSTATE_HPP

src/hotspot/share/gc/shared/partialArrayTaskStepper.cpp

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/partialArrayTaskStepper.hpp"
+#include "oops/arrayOop.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+static uint compute_task_limit(uint n_workers) {
+  // Don't need more than n_workers tasks at a time.  But allowing up to
+  // that maximizes available parallelism.
+  return n_workers;
+}
+
+static uint compute_task_fanout(uint task_limit) {
+  assert(task_limit > 0, "precondition");
+  // There is a tradeoff between providing parallelism more quickly and
+  // number of enqueued tasks.  A constant fanout may be too slow when
+  // parallelism (and so task_limit) is large.  A constant fraction might
+  // be overly eager.  Using log2 attempts to balance between those.
+  uint result = log2_uint(task_limit);
+  // result must be > 0.  result should be > 1 if task_limit > 1, to
+  // provide some potentially parallel tasks.  But don't just +1 to
+  // avoid otherwise increasing rate of task generation.
+  if (result < 2) ++result;
+  return result;
+}
+
+PartialArrayTaskStepper::PartialArrayTaskStepper(uint n_workers) :
+  _task_limit(compute_task_limit(n_workers)),
+  _task_fanout(compute_task_fanout(_task_limit))
+{}

src/hotspot/share/gc/shared/partialArrayTaskStepper.hpp

@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHARED_PARTIALARRAYTASKSTEPPER_HPP
+#define SHARE_GC_SHARED_PARTIALARRAYTASKSTEPPER_HPP
+
+#include "oops/arrayOop.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+// Helper for handling PartialArrayTasks.
+//
+// When an array is large, we want to split it up into chunks that can be
+// processed in parallel.  Each task (implicitly) represents such a chunk.
+// We can enqueue multiple tasks at the same time.  We want to enqueue
+// enough tasks to benefit from the available parallelism, while not so many
+// as to substantially expand the task queues.
+//
+// A task directly refers to the from-space array.  The from-space array's
+// forwarding pointer refers to the associated to-space array, and its
+// length is the actual length. The to-space array's length field is used to
+// indicate processing progress.  It is the starting index of the next chunk
+// to process, or equals the actual length when there are no more chunks to
+// be processed.
+class PartialArrayTaskStepper {
+public:
+  PartialArrayTaskStepper(uint n_workers);
+
+  struct Step {
+    int _index;                 // Array index for the step.
+    uint _ncreate;              // Number of new tasks to create.
+  };
+
+  // Set to's length to the end of the initial chunk, which is the start of
+  // the first partial task if the array is large enough to need splitting.
+  // Returns a Step with _index being that index and _ncreate being the
+  // initial number of partial tasks to enqueue.
+  inline Step start(arrayOop from, arrayOop to, int chunk_size) const;
+
+  // Increment to's length by chunk_size to claim the next chunk.  Returns a
+  // Step with _index being the starting index of the claimed chunk and
+  // _ncreate being the number of additional partial tasks to enqueue.
+  // precondition: chunk_size must be the same as used to start the task sequence.
+  inline Step next(arrayOop from, arrayOop to, int chunk_size) const;
+
+  class TestSupport;            // For unit tests
+
+private:
+  // Limit on the number of partial array tasks to create for a given array.
+  uint _task_limit;
+  // Maximum number of new tasks to create when processing an existing task.
+  uint _task_fanout;
+
+  // Split start/next into public part dealing with oops and private
+  // impl dealing with lengths and pointers to lengths, for unit testing.
+  // length is the actual length obtained from the from-space object.
+  // to_length_addr is the address of the to-space object's length value.
+  inline Step start_impl(int length, int* to_length_addr, int chunk_size) const;
+  inline Step next_impl(int length, int* to_length_addr, int chunk_size) const;
+};
+
+#endif // SHARE_GC_SHARED_PARTIALARRAYTASKSTEPPER_HPP