Update (2023.09.22)

32609: LA port of 8308869: C2: use profile data in subtype checks when profile has more than one class
32608: LA port of 8292692: Move MethodCounters inline functions out of method.hpp

Author: loongson-jvm

src/hotspot/cpu/loongarch/c1_LIRAssembler_loongarch_64.cpp

@@ -1289,9 +1289,8 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success,
     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
   }
 
-  Label profile_cast_success, profile_cast_failure;
-  Label *success_target = should_profile ? &profile_cast_success : success;
-  Label *failure_target = should_profile ? &profile_cast_failure : failure;
+  Label* success_target = success;
+  Label* failure_target = failure;
 
   if (obj == k_RInfo) {
     k_RInfo = dst;
@@ -1309,16 +1308,24 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success,
 
   if (should_profile) {
     Label not_null;
-    __ bnez(obj, not_null);
-    // Object is null; update MDO and exit
     Register mdo = klass_RInfo;
     __ mov_metadata(mdo, md->constant_encoding());
+    __ bnez(obj, not_null);
+    // Object is null; update MDO and exit
     Address data_addr = Address(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
     __ ld_bu(SCR2, data_addr);
     __ ori(SCR2, SCR2, BitData::null_seen_byte_constant());
     __ st_b(SCR2, data_addr);
     __ b(*obj_is_null);
     __ bind(not_null);
+
+    Label update_done;
+    Register recv = k_RInfo;
+    __ load_klass(recv, obj);
+    type_profile_helper(mdo, md, data, recv, &update_done);
+    Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
+    __ increment(counter_addr, DataLayout::counter_increment);
+    __ bind(update_done);
   } else {
     __ beqz(obj, *obj_is_null);
   }
@@ -1378,23 +1385,6 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success,
       // successful cast, fall through to profile or jump
     }
   }
-  if (should_profile) {
-    Register mdo = klass_RInfo, recv = k_RInfo;
-    __ bind(profile_cast_success);
-    __ mov_metadata(mdo, md->constant_encoding());
-    __ load_klass(recv, obj);
-    Label update_done;
-    type_profile_helper(mdo, md, data, recv, success);
-    __ b(*success);
-
-    __ bind(profile_cast_failure);
-    __ mov_metadata(mdo, md->constant_encoding());
-    Address counter_addr = Address(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
-    __ ld_d(SCR2, counter_addr);
-    __ addi_d(SCR2, SCR2, -DataLayout::counter_increment);
-    __ st_d(SCR2, counter_addr);
-    __ b(*failure);
-  }
   __ b(*success);
 }
 
@@ -1424,22 +1414,30 @@ void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
       assert(data != nullptr, "need data for type check");
       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
     }
-    Label profile_cast_success, profile_cast_failure, done;
-    Label *success_target = should_profile ? &profile_cast_success : &done;
-    Label *failure_target = should_profile ? &profile_cast_failure : stub->entry();
+    Label done;
+    Label* success_target = &done;
+    Label* failure_target = stub->entry();
 
     if (should_profile) {
       Label not_null;
-      __ bnez(value, not_null);
-      // Object is null; update MDO and exit
       Register mdo = klass_RInfo;
       __ mov_metadata(mdo, md->constant_encoding());
+      __ bnez(value, not_null);
+      // Object is null; update MDO and exit
       Address data_addr = Address(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
       __ ld_bu(SCR2, data_addr);
       __ ori(SCR2, SCR2, BitData::null_seen_byte_constant());
       __ st_b(SCR2, data_addr);
       __ b(done);
       __ bind(not_null);
+
+      Label update_done;
+      Register recv = k_RInfo;
+      __ load_klass(recv, value);
+      type_profile_helper(mdo, md, data, recv, &update_done);
+      Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
+      __ increment(counter_addr, DataLayout::counter_increment);
+      __ bind(update_done);
     } else {
       __ beqz(value, done);
     }
@@ -1464,25 +1462,6 @@ void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
     __ beqz(k_RInfo, *failure_target);
     // fall through to the success case
 
-    if (should_profile) {
-      Register mdo = klass_RInfo, recv = k_RInfo;
-      __ bind(profile_cast_success);
-      __ mov_metadata(mdo, md->constant_encoding());
-      __ load_klass(recv, value);
-      Label update_done;
-      type_profile_helper(mdo, md, data, recv, &done);
-      __ b(done);
-
-      __ bind(profile_cast_failure);
-      __ mov_metadata(mdo, md->constant_encoding());
-      Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
-      __ lea(SCR2, counter_addr);
-      __ ld_d(SCR1, Address(SCR2));
-      __ addi_d(SCR1, SCR1, -DataLayout::counter_increment);
-      __ st_d(SCR1, Address(SCR2));
-      __ b(*stub->entry());
-    }
-
     __ bind(done);
   } else if (code == lir_checkcast) {
     Register obj = op->object()->as_register();

src/hotspot/cpu/loongarch/interp_masm_loongarch.hpp

@@ -225,15 +225,14 @@ class InterpreterMacroAssembler: public MacroAssembler {
                         Label& not_equal_continue);
 
   void record_klass_in_profile(Register receiver, Register mdp,
-                               Register reg2, bool is_virtual_call);
+                               Register reg2);
   void record_klass_in_profile_helper(Register receiver, Register mdp,
                                       Register reg2, int start_row,
-                                      Label& done, bool is_virtual_call);
-
+                                      Label& done);
   void record_item_in_profile_helper(Register item, Register mdp,
                                      Register reg2, int start_row, Label& done, int total_rows,
-                                     OffsetFunction item_offset_fn, OffsetFunction item_count_offset_fn,
-                                     int non_profiled_offset);
+                                     OffsetFunction item_offset_fn, OffsetFunction item_count_offset_fn);
+
   void update_mdp_by_offset(Register mdp_in, int offset_of_offset);
   void update_mdp_by_offset(Register mdp_in, Register reg, int offset_of_disp);
   void update_mdp_by_constant(Register mdp_in, int constant);

src/hotspot/cpu/loongarch/interp_masm_loongarch_64.cpp

@@ -405,10 +405,6 @@ void InterpreterMacroAssembler::gen_subtype_check( Register Rsup_klass, Register
 
   // Do the check.
   check_klass_subtype(Rsub_klass, Rsup_klass, T1, ok_is_subtype); // blows T1
-
-  // Profile the failure of the check.
-  profile_typecheck_failed(T4); // blows T4
-
 }
 
 
@@ -1252,7 +1248,7 @@ void InterpreterMacroAssembler::profile_virtual_call(Register receiver,
     }
 
     // Record the receiver type.
-    record_klass_in_profile(receiver, mdp, reg2, true);
+    record_klass_in_profile(receiver, mdp, reg2);
     bind(skip_receiver_profile);
 
     // The method data pointer needs to be updated to reflect the new target.
@@ -1277,36 +1273,18 @@ void InterpreterMacroAssembler::profile_virtual_call(Register receiver,
 void InterpreterMacroAssembler::record_klass_in_profile_helper(
                                         Register receiver, Register mdp,
                                         Register reg2, int start_row,
-                                        Label& done, bool is_virtual_call) {
+                                        Label& done) {
   if (TypeProfileWidth == 0) {
-    if (is_virtual_call) {
-      increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
-    }
-#if INCLUDE_JVMCI
-    else if (EnableJVMCI) {
-      increment_mdp_data_at(mdp, in_bytes(ReceiverTypeData::nonprofiled_receiver_count_offset()));
-    }
-#endif // INCLUDE_JVMCI
+    increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
   } else {
-    int non_profiled_offset = -1;
-    if (is_virtual_call) {
-      non_profiled_offset = in_bytes(CounterData::count_offset());
-    }
-#if INCLUDE_JVMCI
-    else if (EnableJVMCI) {
-      non_profiled_offset = in_bytes(ReceiverTypeData::nonprofiled_receiver_count_offset());
-    }
-#endif // INCLUDE_JVMCI
-
     record_item_in_profile_helper(receiver, mdp, reg2, 0, done, TypeProfileWidth,
-        &VirtualCallData::receiver_offset, &VirtualCallData::receiver_count_offset, non_profiled_offset);
+        &VirtualCallData::receiver_offset, &VirtualCallData::receiver_count_offset);
   }
 }
 
 void InterpreterMacroAssembler::record_item_in_profile_helper(Register item, Register mdp,
                                         Register reg2, int start_row, Label& done, int total_rows,
-                                        OffsetFunction item_offset_fn, OffsetFunction item_count_offset_fn,
-                                        int non_profiled_offset) {
+                                        OffsetFunction item_offset_fn, OffsetFunction item_count_offset_fn) {
   int last_row = total_rows - 1;
   assert(start_row <= last_row, "must be work left to do");
   // Test this row for both the item and for null.
@@ -1337,24 +1315,20 @@ void InterpreterMacroAssembler::record_item_in_profile_helper(Register item, Reg
       // Failed the equality check on item[n]...  Test for null.
       if (start_row == last_row) {
         // The only thing left to do is handle the null case.
-        if (non_profiled_offset >= 0) {
-          beqz(reg2, found_null);
-          // Item did not match any saved item and there is no empty row for it.
-          // Increment total counter to indicate polymorphic case.
-          increment_mdp_data_at(mdp, non_profiled_offset);
-          b(done);
-          bind(found_null);
-        } else {
-          bnez(reg2, done);
-        }
+        beqz(reg2, found_null);
+        // Item did not match any saved item and there is no empty row for it.
+        // Increment total counter to indicate polymorphic case.
+        increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
+        b(done);
+        bind(found_null);
         break;
       }
       // Since null is rare, make it be the branch-taken case.
       beqz(reg2, found_null);
 
       // Put all the "Case 3" tests here.
       record_item_in_profile_helper(item, mdp, reg2, start_row + 1, done, total_rows,
-        item_offset_fn, item_count_offset_fn, non_profiled_offset);
+        item_offset_fn, item_count_offset_fn);
 
       // Found a null.  Keep searching for a matching item,
       // but remember that this is an empty (unused) slot.
@@ -1401,12 +1375,11 @@ void InterpreterMacroAssembler::record_item_in_profile_helper(Register item, Reg
 //   done:
 
 void InterpreterMacroAssembler::record_klass_in_profile(Register receiver,
-                                                        Register mdp, Register reg2,
-                                                        bool is_virtual_call) {
+                                                        Register mdp, Register reg2) {
   assert(ProfileInterpreter, "must be profiling");
   Label done;
 
-  record_klass_in_profile_helper(receiver, mdp, reg2, 0, done, is_virtual_call);
+  record_klass_in_profile_helper(receiver, mdp, reg2, 0, done);
 
   bind (done);
 }
@@ -1469,26 +1442,6 @@ void InterpreterMacroAssembler::profile_null_seen(Register mdp) {
   }
 }
 
-
-void InterpreterMacroAssembler::profile_typecheck_failed(Register mdp) {
-  if (ProfileInterpreter && TypeProfileCasts) {
-    Label profile_continue;
-
-    // If no method data exists, go to profile_continue.
-    test_method_data_pointer(mdp, profile_continue);
-
-    int count_offset = in_bytes(CounterData::count_offset());
-    // Back up the address, since we have already bumped the mdp.
-    count_offset -= in_bytes(VirtualCallData::virtual_call_data_size());
-
-    // *Decrement* the counter.  We expect to see zero or small negatives.
-    increment_mdp_data_at(mdp, count_offset, true);
-
-    bind (profile_continue);
-  }
-}
-
-
 void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass, Register reg2) {
   if (ProfileInterpreter) {
     Label profile_continue;
@@ -1502,7 +1455,7 @@ void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass,
       mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
 
       // Record the object type.
-      record_klass_in_profile(klass, mdp, reg2, false);
+      record_klass_in_profile(klass, mdp, reg2);
     }
     update_mdp_by_constant(mdp, mdp_delta);
 

src/hotspot/cpu/loongarch/templateInterpreterGenerator_loongarch.cpp

@@ -34,8 +34,9 @@
 #include "interpreter/templateInterpreterGenerator.hpp"
 #include "interpreter/templateTable.hpp"
 #include "oops/arrayOop.hpp"
-#include "oops/methodData.hpp"
 #include "oops/method.hpp"
+#include "oops/methodCounters.hpp"
+#include "oops/methodData.hpp"
 #include "oops/oop.inline.hpp"
 #include "oops/resolvedIndyEntry.hpp"
 #include "prims/jvmtiExport.hpp"

src/hotspot/cpu/loongarch/templateTable_loongarch_64.cpp

@@ -32,6 +32,7 @@
 #include "gc/shared/collectedHeap.hpp"
 #include "memory/universe.hpp"
 #include "oops/klass.inline.hpp"
+#include "oops/methodCounters.hpp"
 #include "oops/methodData.hpp"
 #include "oops/objArrayKlass.hpp"
 #include "oops/oop.inline.hpp"