Skip to content

Commit

Permalink
Implement support for Zacas for non-narrow atomic cmpxchg
Browse files Browse the repository at this point in the history
  • Loading branch information
@luhenry
luhenry committed Aug 7, 2023
1 parent c17db03 commit 00b8b53
Show file tree
Hide file tree
Showing 6 changed files with 153 additions and 20 deletions.
2 changes: 2 additions & 0 deletions src/hotspot/cpu/riscv/assembler_riscv.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -757,6 +757,8 @@ enum Aqrl {relaxed = 0b00, rl = 0b01, aq = 0b10, aqrl = 0b11};
INSN(amomax_d , 0b0101111, 0b011, 0b10100);
INSN(amominu_d, 0b0101111, 0b011, 0b11000);
INSN(amomaxu_d, 0b0101111, 0b011, 0b11100);
INSN(amocas_w, 0b0101111, 0b010, 0b00101);
INSN(amocas_d, 0b0101111, 0b011, 0b00101);
#undef INSN

enum operand_size { int8, int16, int32, uint32, int64 };
Expand Down
1 change: 1 addition & 0 deletions src/hotspot/cpu/riscv/globals_riscv.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ define_pd_global(intx, InlineSmallCode, 1000);
product(bool, UseRVC, false, "Use RVC instructions") \
product(bool, UseRVA22U64, false, EXPERIMENTAL, "Use RVA22U64 profile") \
product(bool, UseRVV, false, EXPERIMENTAL, "Use RVV instructions") \
product(bool, UseZacas, false, EXPERIMENTAL, "Use Zacas instructions") \
product(bool, UseZba, false, EXPERIMENTAL, "Use Zba instructions") \
product(bool, UseZbb, false, EXPERIMENTAL, "Use Zbb instructions") \
product(bool, UseZbs, false, EXPERIMENTAL, "Use Zbs instructions") \
Expand Down
155 changes: 135 additions & 20 deletions src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2567,20 +2567,26 @@ void MacroAssembler::cmpxchgptr(Register oldv, Register newv, Register addr, Reg
// oldv holds comparison value
// newv holds value to write in exchange
// addr identifies memory word to compare against/update
Label retry_load, nope;
bind(retry_load);
// Load reserved from the memory location
lr_d(tmp, addr, Assembler::aqrl);
// Fail and exit if it is not what we expect
bne(tmp, oldv, nope);
// If the store conditional succeeds, tmp will be zero
sc_d(tmp, newv, addr, Assembler::rl);
beqz(tmp, succeed);
// Retry only when the store conditional failed
j(retry_load);

bind(nope);
membar(AnyAny);
if (UseZacas) {
mv(tmp, oldv);
atomic_cas(tmp, newv, addr, Assembler::int64, Assembler::aq, Assembler::rl);
beq(tmp, oldv, succeed);
} else {
Label retry_load, nope;
bind(retry_load);
// Load reserved from the memory location
lr_d(tmp, addr, Assembler::aqrl);
// Fail and exit if it is not what we expect
bne(tmp, oldv, nope);
// If the store conditional succeeds, tmp will be zero
sc_d(tmp, newv, addr, Assembler::rl);
beqz(tmp, succeed);
// Retry only when the store conditional failed
j(retry_load);
bind(nope);
membar(AnyAny);
}

mv(oldv, tmp);
if (fail != nullptr) {
j(*fail);
Expand Down Expand Up @@ -2754,12 +2760,33 @@ void MacroAssembler::cmpxchg(Register addr, Register expected,
assert_different_registers(expected, t0);
assert_different_registers(new_val, t0);

Label retry_load, done, ne_done;
bind(retry_load);
load_reserved(addr, size, acquire);
bne(t0, expected, ne_done);
store_conditional(addr, new_val, size, release);
bnez(t0, retry_load);
Label done, ne_done;

if (UseZacas) {
if (!result_as_bool) {
// shortcut: we can avoid branches in that case
if (result != expected) {
mv(result, expected);
atomic_cas(result, new_val, addr, size, acquire, release);
} else {
mv(t0, expected);
atomic_cas(t0, new_val, addr, size, acquire, release);
mv(result, t0);
}
return;
} else {
mv(t0, expected);
atomic_cas(t0, new_val, addr, size, acquire, release);
bne(t0, expected, ne_done);
}
} else {
Label retry_load;
bind(retry_load);
load_reserved(addr, size, acquire);
bne(t0, expected, ne_done);
store_conditional(addr, new_val, size, release);
bnez(t0, retry_load);
}

// equal, succeed
if (result_as_bool) {
Expand All @@ -2785,6 +2812,11 @@ void MacroAssembler::cmpxchg_weak(Register addr, Register expected,
enum operand_size size,
Assembler::Aqrl acquire, Assembler::Aqrl release,
Register result) {
if (UseZacas) {
cmpxchg(addr, expected, new_val, size, acquire, release, result, false);
return;
}

assert_different_registers(addr, t0);
assert_different_registers(expected, t0);
assert_different_registers(new_val, t0);
Expand Down Expand Up @@ -2851,6 +2883,89 @@ ATOMIC_XCHGU(xchgalwu, xchgalw)

#undef ATOMIC_XCHGU

#define ATOMIC_CAS(OP, AOP, ACQUIRE, RELEASE) \
void MacroAssembler::atomic_##OP(Register prev, Register newv, Register addr) { \
assert(UseZacas, "invariant"); \
prev = prev->is_valid() ? prev : zr; \
AOP(prev, addr, newv, (Assembler::Aqrl)(ACQUIRE | RELEASE)); \
return; \
}

ATOMIC_CAS(cas, amocas_d, Assembler::relaxed, Assembler::relaxed)
ATOMIC_CAS(casw, amocas_w, Assembler::relaxed, Assembler::relaxed)
ATOMIC_CAS(casl, amocas_d, Assembler::relaxed, Assembler::rl)
ATOMIC_CAS(caslw, amocas_w, Assembler::relaxed, Assembler::rl)
ATOMIC_CAS(casal, amocas_d, Assembler::aq, Assembler::rl)
ATOMIC_CAS(casalw, amocas_w, Assembler::aq, Assembler::rl)

#undef ATOMIC_CAS

#define ATOMIC_CASU(OP1, OP2) \
void MacroAssembler::atomic_##OP1(Register prev, Register newv, Register addr) { \
atomic_##OP2(prev, newv, addr); \
zero_extend(prev, prev, 32); \
return; \
}

ATOMIC_CASU(caswu, casw)
ATOMIC_CASU(caslwu, caslw)
ATOMIC_CASU(casalwu, casalw)

#undef ATOMIC_CASU

void MacroAssembler::atomic_cas(
Register prev, Register newv, Register addr, enum operand_size size, Assembler::Aqrl acquire, Assembler::Aqrl release) {
switch (size) {
case int64:
switch ((Assembler::Aqrl)(acquire | release)) {
case Assembler::relaxed:
atomic_cas(prev, newv, addr);
break;
case Assembler::rl:
atomic_casl(prev, newv, addr);
break;
case Assembler::aqrl:
atomic_casal(prev, newv, addr);
break;
default:
ShouldNotReachHere();
}
break;
case int32:
switch ((Assembler::Aqrl)(acquire | release)) {
case Assembler::relaxed:
atomic_casw(prev, newv, addr);
break;
case Assembler::rl:
atomic_caslw(prev, newv, addr);
break;
case Assembler::aqrl:
atomic_casalw(prev, newv, addr);
break;
default:
ShouldNotReachHere();
}
break;
case uint32:
switch ((Assembler::Aqrl)(acquire | release)) {
case Assembler::relaxed:
atomic_caswu(prev, newv, addr);
break;
case Assembler::rl:
atomic_caslwu(prev, newv, addr);
break;
case Assembler::aqrl:
atomic_casalwu(prev, newv, addr);
break;
default:
ShouldNotReachHere();
}
break;
default:
ShouldNotReachHere();
}
}

void MacroAssembler::far_jump(Address entry, Register tmp) {
assert(ReservedCodeCacheSize < 4*G, "branch out of range");
assert(CodeCache::find_blob(entry.target()) != nullptr,
Expand Down
13 changes: 13 additions & 0 deletions src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1042,6 +1042,19 @@ class MacroAssembler: public Assembler {
void atomic_xchgwu(Register prev, Register newv, Register addr);
void atomic_xchgalwu(Register prev, Register newv, Register addr);

void atomic_cas(Register prev, Register newv, Register addr);
void atomic_casw(Register prev, Register newv, Register addr);
void atomic_casl(Register prev, Register newv, Register addr);
void atomic_caslw(Register prev, Register newv, Register addr);
void atomic_casal(Register prev, Register newv, Register addr);
void atomic_casalw(Register prev, Register newv, Register addr);
void atomic_caswu(Register prev, Register newv, Register addr);
void atomic_caslwu(Register prev, Register newv, Register addr);
void atomic_casalwu(Register prev, Register newv, Register addr);

void atomic_cas(Register prev, Register newv, Register addr, enum operand_size size,
Assembler::Aqrl acquire = Assembler::relaxed, Assembler::Aqrl release = Assembler::relaxed);

static bool far_branches() {
return ReservedCodeCacheSize > branch_range;
}
Expand Down
1 change: 1 addition & 0 deletions src/hotspot/cpu/riscv/vm_version_riscv.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,7 @@ class VM_Version : public Abstract_VM_Version {
decl(ext_Zifencei , "Zifencei" , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT) \
decl(ext_Zic64b , "Zic64b" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZic64b)) \
decl(ext_Zihintpause , "Zihintpause" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZihintpause)) \
decl(ext_Zacas , "Zacas" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZacas)) \
decl(mvendorid , "VendorId" , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \
decl(marchid , "ArchId" , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \
decl(mimpid , "ImpId" , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \
Expand Down
1 change: 1 addition & 0 deletions src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,7 @@ void VM_Version::rivos_features() {
ext_Zifencei.enable_feature();
ext_Zic64b.enable_feature();
ext_Zihintpause.enable_feature();
ext_Zacas.enable_feature();

unaligned_access.enable_feature(MISALIGNED_FAST);
satp_mode.enable_feature(VM_SV48);
Expand Down

0 comments on commit 00b8b53

Please sign in to comment.