8332527: ZGC: generalize object cloning logic

Reviewed-by: aboldtch, thartmann

Author: robcasloz

src/hotspot/share/gc/shared/c2/barrierSetC2.cpp

@@ -814,8 +814,58 @@ Node* BarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* mem, Node* toobi
   return old_tlab_top;
 }
 
+static const TypeFunc* clone_type() {
+  // Create input type (domain)
+  int argcnt = NOT_LP64(3) LP64_ONLY(4);
+  const Type** const domain_fields = TypeTuple::fields(argcnt);
+  int argp = TypeFunc::Parms;
+  domain_fields[argp++] = TypeInstPtr::NOTNULL;  // src
+  domain_fields[argp++] = TypeInstPtr::NOTNULL;  // dst
+  domain_fields[argp++] = TypeX_X;               // size lower
+  LP64_ONLY(domain_fields[argp++] = Type::HALF); // size upper
+  assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
+  const TypeTuple* const domain = TypeTuple::make(TypeFunc::Parms + argcnt, domain_fields);
+
+  // Create result type (range)
+  const Type** const range_fields = TypeTuple::fields(0);
+  const TypeTuple* const range = TypeTuple::make(TypeFunc::Parms + 0, range_fields);
+
+  return TypeFunc::make(domain, range);
+}
+
 #define XTOP LP64_ONLY(COMMA phase->top())
 
+void BarrierSetC2::clone_in_runtime(PhaseMacroExpand* phase, ArrayCopyNode* ac,
+                                    address clone_addr, const char* clone_name) const {
+  Node* const ctrl = ac->in(TypeFunc::Control);
+  Node* const mem  = ac->in(TypeFunc::Memory);
+  Node* const src  = ac->in(ArrayCopyNode::Src);
+  Node* const dst  = ac->in(ArrayCopyNode::Dest);
+  Node* const size = ac->in(ArrayCopyNode::Length);
+
+  assert(size->bottom_type()->base() == Type_X,
+         "Should be of object size type (int for 32 bits, long for 64 bits)");
+
+  // The native clone we are calling here expects the object size in words.
+  // Add header/offset size to payload size to get object size.
+  Node* const base_offset = phase->MakeConX(arraycopy_payload_base_offset(ac->is_clone_array()) >> LogBytesPerLong);
+  Node* const full_size = phase->transform_later(new AddXNode(size, base_offset));
+  // HeapAccess<>::clone expects size in heap words.
+  // For 64-bits platforms, this is a no-operation.
+  // For 32-bits platforms, we need to multiply full_size by HeapWordsPerLong (2).
+  Node* const full_size_in_heap_words = phase->transform_later(new LShiftXNode(full_size, phase->intcon(LogHeapWordsPerLong)));
+
+  Node* const call = phase->make_leaf_call(ctrl,
+                                           mem,
+                                           clone_type(),
+                                           clone_addr,
+                                           clone_name,
+                                           TypeRawPtr::BOTTOM,
+                                           src, dst, full_size_in_heap_words XTOP);
+  phase->transform_later(call);
+  phase->igvn().replace_node(ac, call);
+}
+
 void BarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
   Node* ctrl = ac->in(TypeFunc::Control);
   Node* mem = ac->in(TypeFunc::Memory);

src/hotspot/share/gc/shared/c2/barrierSetC2.hpp

@@ -280,6 +280,8 @@ class BarrierSetC2: public CHeapObj<mtGC> {
   virtual Node* atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const;
   virtual Node* atomic_add_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const;
   void pin_atomic_op(C2AtomicParseAccess& access) const;
+  void clone_in_runtime(PhaseMacroExpand* phase, ArrayCopyNode* ac,
+                        address call_addr, const char* call_name) const;
 
 public:
   // This is the entry-point for the backend to perform accesses through the Access API.

src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp

@@ -407,23 +407,6 @@ bool ZBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc,
   return type == T_OBJECT || type == T_ARRAY;
 }
 
-// This TypeFunc assumes a 64bit system
-static const TypeFunc* clone_type() {
-  // Create input type (domain)
-  const Type** const domain_fields = TypeTuple::fields(4);
-  domain_fields[TypeFunc::Parms + 0] = TypeInstPtr::NOTNULL;  // src
-  domain_fields[TypeFunc::Parms + 1] = TypeInstPtr::NOTNULL;  // dst
-  domain_fields[TypeFunc::Parms + 2] = TypeLong::LONG;        // size lower
-  domain_fields[TypeFunc::Parms + 3] = Type::HALF;            // size upper
-  const TypeTuple* const domain = TypeTuple::make(TypeFunc::Parms + 4, domain_fields);
-
-  // Create result type (range)
-  const Type** const range_fields = TypeTuple::fields(0);
-  const TypeTuple* const range = TypeTuple::make(TypeFunc::Parms + 0, range_fields);
-
-  return TypeFunc::make(domain, range);
-}
-
 #define XTOP LP64_ONLY(COMMA phase->top())
 
 void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
@@ -479,31 +462,10 @@ void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* a
     return;
   }
 
-  // Clone instance
-  Node* const ctrl       = ac->in(TypeFunc::Control);
-  Node* const mem        = ac->in(TypeFunc::Memory);
-  Node* const dst        = ac->in(ArrayCopyNode::Dest);
-  Node* const size       = ac->in(ArrayCopyNode::Length);
-
-  assert(size->bottom_type()->is_long(), "Should be long");
-
-  // The native clone we are calling here expects the instance size in words
-  // Add header/offset size to payload size to get instance size.
-  Node* const base_offset = phase->longcon(arraycopy_payload_base_offset(ac->is_clone_array()) >> LogBytesPerLong);
-  Node* const full_size = phase->transform_later(new AddLNode(size, base_offset));
-
-  Node* const call = phase->make_leaf_call(ctrl,
-                                           mem,
-                                           clone_type(),
-                                           ZBarrierSetRuntime::clone_addr(),
-                                           "ZBarrierSetRuntime::clone",
-                                           TypeRawPtr::BOTTOM,
-                                           src,
-                                           dst,
-                                           full_size,
-                                           phase->top());
-  phase->transform_later(call);
-  phase->igvn().replace_node(ac, call);
+  // Clone instance or array where 'src' is only known to be an object (ary_ptr
+  // is null). This can happen in bytecode generated dynamically to implement
+  // reflective array clones.
+  clone_in_runtime(phase, ac, ZBarrierSetRuntime::clone_addr(), "ZBarrierSetRuntime::clone");
 }
 
 #undef XTOP