8253049: Enhance itable_stub for AArch64 and x86_64

Author: kuaiwei

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@@ -1035,6 +1035,88 @@ void MacroAssembler::lookup_interface_method(Register recv_klass,
   }
 }
 
+void MacroAssembler::lookup_interface_method_in_stub(Register recv_klass,
+                                                     Register resolved_klass,
+                                                     Register holder_klass,
+                                                     int itable_index,
+                                                     Register method_result,
+                                                     Register scan_temp,
+                                                     Register count,
+                                                     Label& L_no_such_interface
+) {
+  assert_different_registers(recv_klass, resolved_klass, holder_klass);
+  assert_different_registers(method_result, scan_temp, count);
+
+  // Compute start of first itableOffsetEntry (which is at the end of the vtable)
+  int vtable_base = in_bytes(Klass::vtable_start_offset());
+  int itentry_off = itableMethodEntry::method_offset_in_bytes();
+  int scan_step   = itableOffsetEntry::size() * wordSize;
+  int vte_size    = vtableEntry::size_in_bytes();
+  assert(vte_size == wordSize, "else adjust times_vte_scale");
+
+  ldrw(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
+
+  // %%% Could store the aligned, prescaled offset in the klassoop.
+  lea(scan_temp, Address(recv_klass, scan_temp, Address::lsl(3)));
+  add(scan_temp, scan_temp, vtable_base);
+  mov(count, -2);
+
+  // Adjust recv_klass by scaled itable_index, so we can free itable_index.
+  assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
+  if (itable_index != 0) {
+    lea(recv_klass, Address(recv_klass, itable_index, Address::lsl(3)));
+  }
+  if (itentry_off) {
+    add(recv_klass, recv_klass, itentry_off);
+  }
+
+  Label L_fast_loop, L_slow_loop, L_slow_loop_tail, L_exit;
+
+  cmp(holder_klass, resolved_klass);
+  br(Assembler::NE, L_slow_loop);
+
+  // fast loop, check holder klass only
+  bind(L_fast_loop);
+  if (itableOffsetEntry::interface_offset_in_bytes() == 0 ) {
+    ldr(method_result, Address(post(scan_temp, scan_step)));
+  } else {
+    ldr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset_in_bytes()));
+    add(scan_temp, scan_temp, scan_step);
+  }
+  cbz(method_result, L_no_such_interface);
+  // check holder_klass
+  cmp(holder_klass, method_result);
+  br(Assembler::NE, L_fast_loop);
+  // found, holder_klass can be free for other use
+  ldrw(holder_klass, Address(scan_temp, -scan_step + itableOffsetEntry::offset_offset_in_bytes()));
+  b(L_exit);
+
+  // slow loop, search both resolved_klass and holder_klass
+  bind(L_slow_loop);
+  if (itableOffsetEntry::interface_offset_in_bytes() == 0 ) {
+    ldr(method_result, Address(post(scan_temp, scan_step)));
+  } else {
+    ldr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset_in_bytes()));
+    add(scan_temp, scan_temp, scan_step);
+  }
+  cbz(method_result, L_no_such_interface);
+  // check resolved_klass
+  cmp(resolved_klass, method_result);
+  csinc(count, count, count, Assembler::NE);
+  // check holder_klass
+  cmp(holder_klass, method_result);
+  br(Assembler::NE, L_slow_loop_tail);
+  increment(count);
+  // holder_klass can be free for other use
+  ldrw(holder_klass, Address(scan_temp, -scan_step + itableOffsetEntry::offset_offset_in_bytes()));
+  bind(L_slow_loop_tail);
+  cbnz(count, L_slow_loop);
+
+  // Got a hit.
+  bind(L_exit);
+  ldr(method_result, Address(recv_klass, holder_klass, Address::uxtw(0)));
+}
+
 // virtual method calling
 void MacroAssembler::lookup_virtual_method(Register recv_klass,
                                            RegisterOrConstant vtable_index,

src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp

@@ -909,6 +909,16 @@ class MacroAssembler: public Assembler {
                                Label& no_such_interface,
                    bool return_method = true);
 
+  // for itable_stub
+  void lookup_interface_method_in_stub(Register recv_klass,
+                                       Register resolved_klass,
+                                       Register holder_klass,
+                                       int itable_index,
+                                       Register method_result,
+                                       Register scan_temp,
+                                       Register count,
+                                       Label& no_such_interface);
+
   // virtual method calling
   // n.b. x86 allows RegisterOrConstant for vtable_index
   void lookup_virtual_method(Register recv_klass,

src/hotspot/cpu/aarch64/vtableStubs_aarch64.cpp

@@ -174,9 +174,9 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
   // so all registers except arguments are free at this point.
   const Register recv_klass_reg     = r10;
   const Register holder_klass_reg   = r16; // declaring interface klass (DECC)
-  const Register resolved_klass_reg = rmethod; // resolved interface klass (REFC)
+  const Register resolved_klass_reg = r13; // resolved interface klass (REFC)
   const Register temp_reg           = r11;
-  const Register temp_reg2          = r15;
+  const Register count_reg          = r15;
   const Register icholder_reg       = rscratch2;
 
   Label L_no_such_interface;
@@ -190,29 +190,14 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
   address npe_addr = __ pc();
   __ load_klass(recv_klass_reg, j_rarg0);
 
-  // Receiver subtype check against REFC.
-  __ lookup_interface_method(// inputs: rec. class, interface
-                             recv_klass_reg, resolved_klass_reg, noreg,
-                             // outputs:  scan temp. reg1, scan temp. reg2
-                             temp_reg2, temp_reg,
-                             L_no_such_interface,
-                             /*return_method=*/false);
-
-  const ptrdiff_t  typecheckSize = __ pc() - start_pc;
-  start_pc = __ pc();
-
-  // Get selected method from declaring class and itable index
-  __ lookup_interface_method(// inputs: rec. class, interface, itable index
-                             recv_klass_reg, holder_klass_reg, itable_index,
-                             // outputs: method, scan temp. reg
-                             rmethod, temp_reg,
-                             L_no_such_interface);
-
-  const ptrdiff_t lookupSize = __ pc() - start_pc;
+  __ lookup_interface_method_in_stub(
+    recv_klass_reg, resolved_klass_reg, holder_klass_reg, itable_index,
+    rmethod, temp_reg, count_reg,
+    L_no_such_interface);
 
   // Reduce "estimate" such that "padding" does not drop below 8.
   const ptrdiff_t estimate = 124;
-  const ptrdiff_t codesize = typecheckSize + lookupSize;
+  const ptrdiff_t codesize = __ pc() - start_pc;
   slop_delta  = (int)(estimate - codesize);
   slop_bytes += slop_delta;
   assert(slop_delta >= 0, "itable #%d: Code size estimate (%d) for lookup_interface_method too small, required: %d", itable_index, (int)estimate, (int)codesize);

src/hotspot/cpu/x86/macroAssembler_x86.cpp

@@ -3454,6 +3454,73 @@ void MacroAssembler::lookup_interface_method(Register recv_klass,
   }
 }
 
+void MacroAssembler::lookup_interface_method_in_stub(Register recv_klass,
+                                                     Register holder_klass,
+                                                     Register resolved_klass,
+                                                     int itable_index,
+                                                     Register method_result,
+                                                     Register scan_temp,
+                                                     Register count,
+                                                     Label& L_no_such_interface) {
+  assert_different_registers(recv_klass, holder_klass, resolved_klass);
+  assert_different_registers(method_result, scan_temp, count);
+
+  // Compute start of first itableOffsetEntry (which is at the end of the vtable)
+  int vtable_base = in_bytes(Klass::vtable_start_offset());
+  int itentry_off = itableMethodEntry::method_offset_in_bytes();
+  int scan_step   = itableOffsetEntry::size() * wordSize;
+  int vte_size    = vtableEntry::size_in_bytes();
+  Address::ScaleFactor times_vte_scale = Address::times_ptr;
+  Label L_fast_loop, L_slow_loop, L_next_check, L_slow_loop_tail, L_exit;
+
+  assert(vte_size == wordSize, "else adjust times_vte_scale");
+
+  movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
+  lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base));
+
+  assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
+  lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off));
+  movl(count, 2);
+
+  cmpptr(holder_klass, resolved_klass);
+  jcc(Assembler::notEqual, L_slow_loop);
+
+  // fast loop, check holder_klass only
+  bind(L_fast_loop);
+  movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset_in_bytes()));
+  testptr(method_result, method_result);
+  jcc(Assembler::zero, L_no_such_interface);
+  addptr(scan_temp, scan_step);
+  cmpptr(holder_klass, method_result);
+  jccb(Assembler::notEqual, L_fast_loop);
+  movptr(holder_klass, Address(scan_temp, -scan_step + itableOffsetEntry::offset_offset_in_bytes()));
+  jmp(L_exit);
+
+  // slow loop, search both resolved_klass and holder_klass
+  bind(L_slow_loop);
+  movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset_in_bytes()));
+  testptr(method_result, method_result);
+  jcc(Assembler::zero, L_no_such_interface);
+
+  cmpptr(holder_klass, method_result);
+  jccb(Assembler::notEqual, L_next_check);
+  movptr(holder_klass, Address(scan_temp, itableOffsetEntry::offset_offset_in_bytes()));
+  decrement(count);
+
+  bind(L_next_check);
+  cmpptr(resolved_klass, method_result);
+  jccb(Assembler::notEqual, L_slow_loop_tail);
+  decrement(count);
+
+  bind(L_slow_loop_tail);
+  addptr(scan_temp, scan_step);
+  testl(count, count);
+  jcc(Assembler::notZero, L_slow_loop);
+
+  // Got a hit.
+  bind(L_exit);
+  movptr(method_result, Address(recv_klass, holder_klass, Address::times_1));
+}
 
 // virtual method calling
 void MacroAssembler::lookup_virtual_method(Register recv_klass,

src/hotspot/cpu/x86/macroAssembler_x86.hpp

@@ -536,6 +536,16 @@ class MacroAssembler: public Assembler {
                                Label& no_such_interface,
                                bool return_method = true);
 
+  // for itable_stub
+  void lookup_interface_method_in_stub(Register recv_klass,
+                               Register holder_klass,
+                               Register resolved_klass,
+                               int itable_index,
+                               Register method_result,
+                               Register scan_temp,
+                               Register count,
+                               Label& no_such_interface);
+
   // virtual method calling
   void lookup_virtual_method(Register recv_klass,
                              RegisterOrConstant vtable_index,

src/hotspot/cpu/x86/vtableStubs_x86_64.cpp

@@ -175,10 +175,13 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
   // (various calling sequences use r[cd]x, r[sd]i, r[89]; stay away from them)
   const Register recv_klass_reg     = r10;
   const Register holder_klass_reg   = rax; // declaring interface klass (DECC)
-  const Register resolved_klass_reg = rbx; // resolved interface klass (REFC)
+  const Register resolved_klass_reg = r14; // resolved interface klass (REFC)
   const Register temp_reg           = r11;
 
   const Register icholder_reg = rax;
+  const Register method = rbx;
+  const Register count_reg = r13;
+
   __ movptr(resolved_klass_reg, Address(icholder_reg, CompiledICHolder::holder_klass_offset()));
   __ movptr(holder_klass_reg,   Address(icholder_reg, CompiledICHolder::holder_metadata_offset()));
 
@@ -190,35 +193,15 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
   __ load_klass(recv_klass_reg, j_rarg0, temp_reg);
 
   start_pc = __ pc();
-
-  // Receiver subtype check against REFC.
-  // Destroys recv_klass_reg value.
-  __ lookup_interface_method(// inputs: rec. class, interface
-                             recv_klass_reg, resolved_klass_reg, noreg,
-                             // outputs:  scan temp. reg1, scan temp. reg2
-                             recv_klass_reg, temp_reg,
-                             L_no_such_interface,
-                             /*return_method=*/false);
-
-  const ptrdiff_t  typecheckSize = __ pc() - start_pc;
-  start_pc = __ pc();
-
-  // Get selected method from declaring class and itable index
-  const Register method = rbx;
-  __ load_klass(recv_klass_reg, j_rarg0, temp_reg);   // restore recv_klass_reg
-  __ lookup_interface_method(// inputs: rec. class, interface, itable index
-                             recv_klass_reg, holder_klass_reg, itable_index,
-                             // outputs: method, scan temp. reg
-                             method, temp_reg,
-                             L_no_such_interface);
-
-  const ptrdiff_t  lookupSize = __ pc() - start_pc;
+  __ lookup_interface_method_in_stub(recv_klass_reg, holder_klass_reg, resolved_klass_reg, itable_index,
+                                     method, temp_reg, count_reg,
+                                     L_no_such_interface);
 
   // We expect we need index_dependent_slop extra bytes. Reason:
   // The emitted code in lookup_interface_method changes when itable_index exceeds 15.
   // For linux, a very narrow estimate would be 112, but Solaris requires some more space (130).
   const ptrdiff_t estimate = 136;
-  const ptrdiff_t codesize = typecheckSize + lookupSize + index_dependent_slop;
+  const ptrdiff_t codesize = __ pc() - start_pc + index_dependent_slop;
   slop_delta  = (int)(estimate - codesize);
   slop_bytes += slop_delta;
   assert(slop_delta >= 0, "itable #%d: Code size estimate (%d) for lookup_interface_method too small, required: %d", itable_index, (int)estimate, (int)codesize);