From 00b8b53db2d4fff8dd984d29c222333f52c71150 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Mon, 7 Aug 2023 09:01:45 +0000 Subject: [PATCH] Implement support for Zacas for non-narrow atomic cmpxchg --- src/hotspot/cpu/riscv/assembler_riscv.hpp | 2 + src/hotspot/cpu/riscv/globals_riscv.hpp | 1 + .../cpu/riscv/macroAssembler_riscv.cpp | 155 +++++++++++++++--- .../cpu/riscv/macroAssembler_riscv.hpp | 13 ++ src/hotspot/cpu/riscv/vm_version_riscv.hpp | 1 + .../linux_riscv/vm_version_linux_riscv.cpp | 1 + 6 files changed, 153 insertions(+), 20 deletions(-) diff --git a/src/hotspot/cpu/riscv/assembler_riscv.hpp b/src/hotspot/cpu/riscv/assembler_riscv.hpp index a10877b873e1d..c7b83e394122c 100644 --- a/src/hotspot/cpu/riscv/assembler_riscv.hpp +++ b/src/hotspot/cpu/riscv/assembler_riscv.hpp @@ -757,6 +757,8 @@ enum Aqrl {relaxed = 0b00, rl = 0b01, aq = 0b10, aqrl = 0b11}; INSN(amomax_d , 0b0101111, 0b011, 0b10100); INSN(amominu_d, 0b0101111, 0b011, 0b11000); INSN(amomaxu_d, 0b0101111, 0b011, 0b11100); + INSN(amocas_w, 0b0101111, 0b010, 0b00101); + INSN(amocas_d, 0b0101111, 0b011, 0b00101); #undef INSN enum operand_size { int8, int16, int32, uint32, int64 }; diff --git a/src/hotspot/cpu/riscv/globals_riscv.hpp b/src/hotspot/cpu/riscv/globals_riscv.hpp index dd375c2c0f82f..0799b1f6d1c0f 100644 --- a/src/hotspot/cpu/riscv/globals_riscv.hpp +++ b/src/hotspot/cpu/riscv/globals_riscv.hpp @@ -102,6 +102,7 @@ define_pd_global(intx, InlineSmallCode, 1000); product(bool, UseRVC, false, "Use RVC instructions") \ product(bool, UseRVA22U64, false, EXPERIMENTAL, "Use RVA22U64 profile") \ product(bool, UseRVV, false, EXPERIMENTAL, "Use RVV instructions") \ + product(bool, UseZacas, false, EXPERIMENTAL, "Use Zacas instructions") \ product(bool, UseZba, false, EXPERIMENTAL, "Use Zba instructions") \ product(bool, UseZbb, false, EXPERIMENTAL, "Use Zbb instructions") \ product(bool, UseZbs, false, EXPERIMENTAL, "Use Zbs instructions") \ diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp index 4adf317cfca7d..c111cf6cd6473 100644 --- a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp @@ -2567,20 +2567,26 @@ void MacroAssembler::cmpxchgptr(Register oldv, Register newv, Register addr, Reg // oldv holds comparison value // newv holds value to write in exchange // addr identifies memory word to compare against/update - Label retry_load, nope; - bind(retry_load); - // Load reserved from the memory location - lr_d(tmp, addr, Assembler::aqrl); - // Fail and exit if it is not what we expect - bne(tmp, oldv, nope); - // If the store conditional succeeds, tmp will be zero - sc_d(tmp, newv, addr, Assembler::rl); - beqz(tmp, succeed); - // Retry only when the store conditional failed - j(retry_load); - - bind(nope); - membar(AnyAny); + if (UseZacas) { + mv(tmp, oldv); + atomic_cas(tmp, newv, addr, Assembler::int64, Assembler::aq, Assembler::rl); + beq(tmp, oldv, succeed); + } else { + Label retry_load, nope; + bind(retry_load); + // Load reserved from the memory location + lr_d(tmp, addr, Assembler::aqrl); + // Fail and exit if it is not what we expect + bne(tmp, oldv, nope); + // If the store conditional succeeds, tmp will be zero + sc_d(tmp, newv, addr, Assembler::rl); + beqz(tmp, succeed); + // Retry only when the store conditional failed + j(retry_load); + bind(nope); + membar(AnyAny); + } + mv(oldv, tmp); if (fail != nullptr) { j(*fail); @@ -2754,12 +2760,33 @@ void MacroAssembler::cmpxchg(Register addr, Register expected, assert_different_registers(expected, t0); assert_different_registers(new_val, t0); - Label retry_load, done, ne_done; - bind(retry_load); - load_reserved(addr, size, acquire); - bne(t0, expected, ne_done); - store_conditional(addr, new_val, size, release); - bnez(t0, retry_load); + Label done, ne_done; + + if (UseZacas) { + if (!result_as_bool) { + // shortcut: we can avoid branches in that case + if (result != expected) { + mv(result, expected); + atomic_cas(result, new_val, addr, size, acquire, release); + } else { + mv(t0, expected); + atomic_cas(t0, new_val, addr, size, acquire, release); + mv(result, t0); + } + return; + } else { + mv(t0, expected); + atomic_cas(t0, new_val, addr, size, acquire, release); + bne(t0, expected, ne_done); + } + } else { + Label retry_load; + bind(retry_load); + load_reserved(addr, size, acquire); + bne(t0, expected, ne_done); + store_conditional(addr, new_val, size, release); + bnez(t0, retry_load); + } // equal, succeed if (result_as_bool) { @@ -2785,6 +2812,11 @@ void MacroAssembler::cmpxchg_weak(Register addr, Register expected, enum operand_size size, Assembler::Aqrl acquire, Assembler::Aqrl release, Register result) { + if (UseZacas) { + cmpxchg(addr, expected, new_val, size, acquire, release, result, false); + return; + } + assert_different_registers(addr, t0); assert_different_registers(expected, t0); assert_different_registers(new_val, t0); @@ -2851,6 +2883,89 @@ ATOMIC_XCHGU(xchgalwu, xchgalw) #undef ATOMIC_XCHGU +#define ATOMIC_CAS(OP, AOP, ACQUIRE, RELEASE) \ +void MacroAssembler::atomic_##OP(Register prev, Register newv, Register addr) { \ + assert(UseZacas, "invariant"); \ + prev = prev->is_valid() ? prev : zr; \ + AOP(prev, addr, newv, (Assembler::Aqrl)(ACQUIRE | RELEASE)); \ + return; \ +} + +ATOMIC_CAS(cas, amocas_d, Assembler::relaxed, Assembler::relaxed) +ATOMIC_CAS(casw, amocas_w, Assembler::relaxed, Assembler::relaxed) +ATOMIC_CAS(casl, amocas_d, Assembler::relaxed, Assembler::rl) +ATOMIC_CAS(caslw, amocas_w, Assembler::relaxed, Assembler::rl) +ATOMIC_CAS(casal, amocas_d, Assembler::aq, Assembler::rl) +ATOMIC_CAS(casalw, amocas_w, Assembler::aq, Assembler::rl) + +#undef ATOMIC_CAS + +#define ATOMIC_CASU(OP1, OP2) \ +void MacroAssembler::atomic_##OP1(Register prev, Register newv, Register addr) { \ + atomic_##OP2(prev, newv, addr); \ + zero_extend(prev, prev, 32); \ + return; \ +} + +ATOMIC_CASU(caswu, casw) +ATOMIC_CASU(caslwu, caslw) +ATOMIC_CASU(casalwu, casalw) + +#undef ATOMIC_CASU + +void MacroAssembler::atomic_cas( + Register prev, Register newv, Register addr, enum operand_size size, Assembler::Aqrl acquire, Assembler::Aqrl release) { + switch (size) { + case int64: + switch ((Assembler::Aqrl)(acquire | release)) { + case Assembler::relaxed: + atomic_cas(prev, newv, addr); + break; + case Assembler::rl: + atomic_casl(prev, newv, addr); + break; + case Assembler::aqrl: + atomic_casal(prev, newv, addr); + break; + default: + ShouldNotReachHere(); + } + break; + case int32: + switch ((Assembler::Aqrl)(acquire | release)) { + case Assembler::relaxed: + atomic_casw(prev, newv, addr); + break; + case Assembler::rl: + atomic_caslw(prev, newv, addr); + break; + case Assembler::aqrl: + atomic_casalw(prev, newv, addr); + break; + default: + ShouldNotReachHere(); + } + break; + case uint32: + switch ((Assembler::Aqrl)(acquire | release)) { + case Assembler::relaxed: + atomic_caswu(prev, newv, addr); + break; + case Assembler::rl: + atomic_caslwu(prev, newv, addr); + break; + case Assembler::aqrl: + atomic_casalwu(prev, newv, addr); + break; + default: + ShouldNotReachHere(); + } + break; + default: + ShouldNotReachHere(); + } +} + void MacroAssembler::far_jump(Address entry, Register tmp) { assert(ReservedCodeCacheSize < 4*G, "branch out of range"); assert(CodeCache::find_blob(entry.target()) != nullptr, diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp index ca95f76adc3d0..d032b9275df20 100644 --- a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp +++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp @@ -1042,6 +1042,19 @@ class MacroAssembler: public Assembler { void atomic_xchgwu(Register prev, Register newv, Register addr); void atomic_xchgalwu(Register prev, Register newv, Register addr); + void atomic_cas(Register prev, Register newv, Register addr); + void atomic_casw(Register prev, Register newv, Register addr); + void atomic_casl(Register prev, Register newv, Register addr); + void atomic_caslw(Register prev, Register newv, Register addr); + void atomic_casal(Register prev, Register newv, Register addr); + void atomic_casalw(Register prev, Register newv, Register addr); + void atomic_caswu(Register prev, Register newv, Register addr); + void atomic_caslwu(Register prev, Register newv, Register addr); + void atomic_casalwu(Register prev, Register newv, Register addr); + + void atomic_cas(Register prev, Register newv, Register addr, enum operand_size size, + Assembler::Aqrl acquire = Assembler::relaxed, Assembler::Aqrl release = Assembler::relaxed); + static bool far_branches() { return ReservedCodeCacheSize > branch_range; } diff --git a/src/hotspot/cpu/riscv/vm_version_riscv.hpp b/src/hotspot/cpu/riscv/vm_version_riscv.hpp index 39c4150cea8e9..5b6a348753e93 100644 --- a/src/hotspot/cpu/riscv/vm_version_riscv.hpp +++ b/src/hotspot/cpu/riscv/vm_version_riscv.hpp @@ -135,6 +135,7 @@ class VM_Version : public Abstract_VM_Version { decl(ext_Zifencei , "Zifencei" , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT) \ decl(ext_Zic64b , "Zic64b" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZic64b)) \ decl(ext_Zihintpause , "Zihintpause" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZihintpause)) \ + decl(ext_Zacas , "Zacas" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZacas)) \ decl(mvendorid , "VendorId" , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \ decl(marchid , "ArchId" , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \ decl(mimpid , "ImpId" , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \ diff --git a/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp b/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp index 54c8ae13bfa71..4532cd4133a14 100644 --- a/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp +++ b/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp @@ -245,6 +245,7 @@ void VM_Version::rivos_features() { ext_Zifencei.enable_feature(); ext_Zic64b.enable_feature(); ext_Zihintpause.enable_feature(); + ext_Zacas.enable_feature(); unaligned_access.enable_feature(MISALIGNED_FAST); satp_mode.enable_feature(VM_SV48);