In CAS loop, update old_value from result of CAS

Author: dean-long

src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp

@@ -119,12 +119,15 @@ class NativeNMethodBarrier {
 
   void set_value(int value, int bit_mask) {
     assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
+    value &= bit_mask;
+    int old_value = Atomic::load(guard_addr());
     while (true) {
-      int old = Atomic::load(guard_addr());
       // Only bits in the mask are changed
-      int new_value = value | (old & ~bit_mask);
-      int v = Atomic::cmpxchg(guard_addr(), old, new_value, memory_order_release);
-      if (v == old) break;
+      int new_value = value | (old_value & ~bit_mask);
+      if (new_value == old_value) break;
+      int v = Atomic::cmpxchg(guard_addr(), old_value, new_value, memory_order_release);
+      if (v == old_value) break;
+      old_value = v;
     }
   }
 

src/hotspot/cpu/arm/gc/shared/barrierSetNMethod_arm.cpp

@@ -53,12 +53,15 @@ class NativeNMethodBarrier: public NativeInstruction {
 
   void set_value(int value, int bit_mask) {
     assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
+    value &= bit_mask;
+    int old_value = Atomic::load(guard_addr());
     while (true) {
-      int old = Atomic::load(guard_addr());
       // Only bits in the mask are changed
-      int new_value = value | (old & ~bit_mask);
-      int v = Atomic::cmpxchg(guard_addr(), old, new_value, memory_order_release);
-      if (v == old) break;
+      int new_value = value | (old_value & ~bit_mask);
+      if (new_value == old_value) break;
+      int v = Atomic::cmpxchg(guard_addr(), old_value, new_value, memory_order_release);
+      if (v == old_value) break;
+      old_value = v;
     }
   }
 

src/hotspot/cpu/ppc/gc/shared/barrierSetNMethod_ppc.cpp

@@ -49,6 +49,7 @@ class NativeNMethodBarrier: public NativeInstruction {
 
   void release_set_guard_value(int value, int bit_mask) {
     assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
+    value &= bit_mask;
     // Patching is not atomic.
     // Stale observations of the "armed" state is okay as invoking the barrier stub in that case has no
     // unwanted side effects. Disarming is thus a non-critical operation.
@@ -58,20 +59,22 @@ class NativeNMethodBarrier: public NativeInstruction {
 
     NativeMovRegMem* mov = get_patchable_instruction_handle();
     uint64_t *instr = (uint64_t*)mov->instruction_address();
-    assert(NativeMovRegMem::instruction_size >= sizeof(instr), "must be");
+    assert(NativeMovRegMem::instruction_size == sizeof(*instr), "must be");
     union {
       u_char buf[NativeMovRegMem::instruction_size];
       uint64_t u64;
     } new_mov_instr, old_mov_instr;
+    new_mov_instr.u64 = old_mov_instr.u64 = Atomic::load(instr);
     while (true) {
-      new_mov_instr.u64 = old_mov_instr.u64 = Atomic::load(instr);
-      int old_value = nativeMovRegMem_at(old_mov_instr.buf)->offset();
       // Only bits in the mask are changed
+      int old_value = nativeMovRegMem_at(old_mov_instr.buf)->offset();
       int new_value = value | (old_value & ~bit_mask);
+      if (new_value == old_value) break;
       nativeMovRegMem_at(new_mov_instr.buf)->set_offset(new_value, false /* no icache flush */);
       // Swap in the new value
       uint64_t v = Atomic::cmpxchg(instr, old_mov_instr.u64, new_mov_instr.u64, memory_order_release);
       if (v == old_mov_instr.u64) break;
+      old_mov_instr.u64 = v;
     }
     ICache::ppc64_flush_icache_bytes(addr_at(0), NativeMovRegMem::instruction_size);
   }

src/hotspot/cpu/riscv/gc/shared/barrierSetNMethod_riscv.cpp

@@ -113,12 +113,15 @@ class NativeNMethodBarrier {
 
   void set_value(int value, int bit_mask) {
     assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
+    value &= bit_mask;
+    int old_value = Atomic::load(guard_addr());
     while (true) {
-      int old = Atomic::load(guard_addr());
       // Only bits in the mask are changed
-      int new_value = value | (old & ~bit_mask);
-      int v = Atomic::cmpxchg(guard_addr(), old, new_value, memory_order_release);
-      if (v == old) break;
+      int new_value = value | (old_value & ~bit_mask);
+      if (new_value == old_value) break;
+      int v = Atomic::cmpxchg(guard_addr(), old_value, new_value, memory_order_release);
+      if (v == old_value) break;
+      old_value = v;
     }
   }
 

src/hotspot/cpu/s390/gc/shared/barrierSetNMethod_s390.cpp

@@ -55,13 +55,16 @@ class NativeMethodBarrier: public NativeInstruction {
 
     void set_guard_value(int value, int bit_mask) {
       assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
+      value &= bit_mask;
       int32_t* data_addr = (int32_t*)get_patchable_data_address();
+      int old_value = Atomic::load(data_addr);
       while (true) {
-        int old = Atomic::load(data_addr);
         // Only bits in the mask are changed
-        int new_value = value | (old & ~bit_mask);
-        int v = Atomic::cmpxchg(data_addr, old, new_value, memory_order_release);
-        if (v == old) break;
+        int new_value = value | (old_value & ~bit_mask);
+        if (new_value == old_value) break;
+        int v = Atomic::cmpxchg(data_addr, old_value, new_value, memory_order_release);
+        if (v == old_value) break;
+        old_value = v;
       }
     }
 

src/hotspot/cpu/x86/gc/shared/barrierSetNMethod_x86.cpp

@@ -55,22 +55,25 @@ class NativeNMethodCmpBarrier: public NativeInstruction {
   jint get_immediate() const { return int_at(imm_offset); }
   void set_immediate(jint imm, int bit_mask) {
     assert((imm & ~bit_mask) == 0, "trying to set bits outside the mask");
+    imm &= bit_mask;
 
     uint64_t *instr = (uint64_t*)instruction_address();
     assert(instruction_size >= sizeof(instr), "must be");
     union {
       unsigned char buf[instruction_size];
       uint64_t u64;
     } new_mov_instr, old_mov_instr;
+    new_mov_instr.u64 = old_mov_instr.u64 = Atomic::load(instr);
     while (true) {
-      new_mov_instr.u64 = old_mov_instr.u64 = Atomic::load(instr);
       int old_value = nativeNMethodCmpBarrier_at(old_mov_instr.buf)->get_immediate();
       // Only bits in the mask are changed
       int new_value = imm | (old_value & ~bit_mask);
+      if (new_value == old_value) break;
       nativeNMethodCmpBarrier_at(new_mov_instr.buf)->set_int_at(imm_offset, new_value);
       // Swap in the new value
       uint64_t v = Atomic::cmpxchg(instr, old_mov_instr.u64, new_mov_instr.u64, memory_order_release);
       if (v == old_mov_instr.u64) break;
+      old_mov_instr.u64 = v;
     }
   }
   bool check_barrier(err_msg& msg) const;