Skip to content

[LoongArch64] Add atomic instruction implementation for LoongArch64 #129683

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lawn123 wants to merge 10 commits into
base: main
Choose a base branch
from
Open

[LoongArch64] Add atomic instruction implementation for LoongArch64 #129683

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
5ea3049
[LoongArch64] Add atomic instruction implementation for LoongArch64
lawn123 Jan 21, 2025
d755abe
Fix jit-format failure issue
lawn123 Jun 22, 2026
f680198
Remove the use of TARGET_64BIT for expanding the intrinsics
lawn123 Jun 22, 2026
35fd1c8
Delete CompareExchange and Exchange smalltype simulation implementati…
lawn123 Jun 30, 2026
1cd04d8
Modify helper function to internal and fix compilation failure and fo…
lawn123 Jul 3, 2026
6990eb9
EnableLoongArch64LAM_CAS and EnableLoongArch64LAM_BH are enabled by d…
lawn123 Jul 3, 2026
41f9cf2
Delete System.Runtime.Intrinsics.cs changes
lawn123 Jul 3, 2026
1e08109
Update the corinfoinstructionset.h file auto-generator produced by ge…
lawn123 Jul 3, 2026
622f190
Merge the main branch and update jiteeversionguid.h
lawn123 Jul 3, 2026
707b672
Revert "Merge the main branch and update jiteeversionguid.h"
lawn123 Jul 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions src/coreclr/System.Private.CoreLib/src/System/Threading/Interlocked.CoreCLR.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ public static long Decrement(ref long location) =>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int Exchange(ref int location1, int value)
{
#if TARGET_X86 || TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64
#if TARGET_X86 || TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 || TARGET_LOONGARCH64
return Exchange(ref location1, value); // Must expand intrinsic
#else
if (Unsafe.IsNullRef(ref location1))
Expand All @@ -73,7 +73,7 @@ public static int Exchange(ref int location1, int value)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static long Exchange(ref long location1, long value)
{
#if TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64
#if TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 || TARGET_LOONGARCH64
return Exchange(ref location1, value); // Must expand intrinsic
#else
if (Unsafe.IsNullRef(ref location1))
Expand Down Expand Up @@ -116,7 +116,7 @@ public static long Exchange(ref long location1, long value)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int CompareExchange(ref int location1, int value, int comparand)
{
#if TARGET_X86 || TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64
#if TARGET_X86 || TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 || TARGET_LOONGARCH64
return CompareExchange(ref location1, value, comparand); // Must expand intrinsic
#else
if (Unsafe.IsNullRef(ref location1))
Expand All @@ -137,7 +137,7 @@ public static int CompareExchange(ref int location1, int value, int comparand)
[Intrinsic]
internal static unsafe int CompareExchange(int* location1, int value, int comparand)
{
#if TARGET_X86 || TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64
#if TARGET_X86 || TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 || TARGET_LOONGARCH64
return CompareExchange(location1, value, comparand); // Must expand intrinsic
#else
Debug.Assert(location1 != null);
Expand All @@ -158,7 +158,7 @@ internal static unsafe int CompareExchange(int* location1, int value, int compar
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static long CompareExchange(ref long location1, long value, long comparand)
{
#if TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64
#if TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 || TARGET_LOONGARCH64
return CompareExchange(ref location1, value, comparand); // Must expand intrinsic
#else
if (Unsafe.IsNullRef(ref location1))
Expand Down Expand Up @@ -212,7 +212,7 @@ public static long Add(ref long location1, long value) =>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int ExchangeAdd(ref int location1, int value)
{
#if TARGET_X86 || TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64
#if TARGET_X86 || TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 || TARGET_LOONGARCH64
return ExchangeAdd(ref location1, value); // Must expand intrinsic
#else
if (Unsafe.IsNullRef(ref location1))
Expand All @@ -228,7 +228,7 @@ private static int ExchangeAdd(ref int location1, int value)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static long ExchangeAdd(ref long location1, long value)
{
#if TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64
#if TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 || TARGET_LOONGARCH64
return ExchangeAdd(ref location1, value); // Must expand intrinsic
#else
if (Unsafe.IsNullRef(ref location1))
Expand Down
3 changes: 3 additions & 0 deletions src/coreclr/inc/clrconfigvalues.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -716,6 +716,9 @@ RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64SveSm4, W("EnableArm64Sv
RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableRiscV64Zba, W("EnableRiscV64Zba"), 1, "Allows RiscV64 Zba hardware intrinsics to be disabled")
RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableRiscV64Zbb, W("EnableRiscV64Zbb"), 1, "Allows RiscV64 Zbb hardware intrinsics to be disabled")
RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableRiscV64Zbs, W("EnableRiscV64Zbs"), 1, "Allows RiscV64 Zbs hardware intrinsics to be disabled")
#elif defined(TARGET_LOONGARCH64)
RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableLoongArch64LAM_BH, W("EnableLoongArch64LAM_BH"), 1, "Allows LoongArch64 LAM_BH+ hardware intrinsics to be disabled")
RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableLoongArch64LAM_CAS, W("EnableLoongArch64LAM_CAS"), 1, "Allows LoongArch64 LAM_CAS+ hardware intrinsics to be disabled")
#endif

///
Expand Down
18 changes: 18 additions & 0 deletions src/coreclr/inc/corinfoinstructionset.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,10 @@ enum CORINFO_InstructionSet
InstructionSet_SveSha3_Arm64=37,
InstructionSet_SveSm4_Arm64=38,
#endif // TARGET_ARM64
#ifdef TARGET_LOONGARCH64
InstructionSet_LAM_BH=1,
InstructionSet_LAM_CAS=2,
#endif // TARGET_LOONGARCH64
#ifdef TARGET_RISCV64
InstructionSet_RiscV64Base=1,
InstructionSet_Zba=2,
Expand Down Expand Up @@ -288,6 +292,8 @@ struct CORINFO_InstructionSetFlags
if (HasInstructionSet(InstructionSet_SveSm4))
AddInstructionSet(InstructionSet_SveSm4_Arm64);
#endif // TARGET_ARM64
#ifdef TARGET_LOONGARCH64
#endif // TARGET_LOONGARCH64
#ifdef TARGET_RISCV64
#endif // TARGET_RISCV64
#ifdef TARGET_WASM
Expand Down Expand Up @@ -448,6 +454,8 @@ inline CORINFO_InstructionSetFlags EnsureInstructionSetFlagsAreValid(CORINFO_Ins
if (resultflags.HasInstructionSet(InstructionSet_SveSm4) && !resultflags.HasInstructionSet(InstructionSet_Sm4))
resultflags.RemoveInstructionSet(InstructionSet_SveSm4);
#endif // TARGET_ARM64
#ifdef TARGET_LOONGARCH64
#endif // TARGET_LOONGARCH64
#ifdef TARGET_RISCV64
if (resultflags.HasInstructionSet(InstructionSet_Zbb) && !resultflags.HasInstructionSet(InstructionSet_RiscV64Base))
resultflags.RemoveInstructionSet(InstructionSet_Zbb);
Expand Down Expand Up @@ -746,6 +754,12 @@ inline const char *InstructionSetToString(CORINFO_InstructionSet instructionSet)
case InstructionSet_SveSm4_Arm64 :
return "SveSm4_Arm64";
#endif // TARGET_ARM64
#ifdef TARGET_LOONGARCH64
case InstructionSet_LAM_BH :
return "LAM_BH";
case InstructionSet_LAM_CAS :
return "LAM_CAS";
#endif // TARGET_LOONGARCH64
#ifdef TARGET_RISCV64
case InstructionSet_RiscV64Base :
return "RiscV64Base";
Expand Down Expand Up @@ -959,6 +973,10 @@ inline CORINFO_InstructionSet InstructionSetFromR2RInstructionSet(ReadyToRunInst
case READYTORUN_INSTRUCTION_SveSha3: return InstructionSet_SveSha3;
case READYTORUN_INSTRUCTION_SveSm4: return InstructionSet_SveSm4;
#endif // TARGET_ARM64
#ifdef TARGET_LOONGARCH64
case READYTORUN_INSTRUCTION_LAM_BH: return InstructionSet_LAM_BH;
case READYTORUN_INSTRUCTION_LAM_CAS: return InstructionSet_LAM_CAS;
#endif // TARGET_LOONGARCH64
#ifdef TARGET_RISCV64
case READYTORUN_INSTRUCTION_RiscV64Base: return InstructionSet_RiscV64Base;
case READYTORUN_INSTRUCTION_Zba: return InstructionSet_Zba;
Expand Down
2 changes: 2 additions & 0 deletions src/coreclr/inc/readytoruninstructionset.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,8 @@ enum ReadyToRunInstructionSet
READYTORUN_INSTRUCTION_SveSm4=90,
READYTORUN_INSTRUCTION_WasmBase=91,
READYTORUN_INSTRUCTION_PackedSimd=92,
READYTORUN_INSTRUCTION_LAM_BH=93,
READYTORUN_INSTRUCTION_LAM_CAS=94,

};

Expand Down
163 changes: 159 additions & 4 deletions src/coreclr/jit/codegenloongarch64.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2167,14 +2167,76 @@ void CodeGen::genNonLocalJmp(GenTreeUnOp* tree)
}

//------------------------------------------------------------------------
// genLockedInstructions: Generate code for a GT_XADD or GT_XCHG node.
// genLockedInstructions: Generate code for a GT_XADD, GT_XAND, GT_XORR or GT_XCHG node.
//
// Arguments:
// treeNode - the GT_XADD/XCHG node
// treeNode - the GT_XADD/XAND/XORR/XCHG node
//
void CodeGen::genLockedInstructions(GenTreeOp* treeNode)
{
NYI("unimplemented on LOONGARCH64 yet");
GenTree* data = treeNode->AsOp()->gtOp2;
GenTree* addr = treeNode->AsOp()->gtOp1;
regNumber dataReg = !data->isContained() ? data->GetRegNum() : REG_R0;
regNumber addrReg = addr->GetRegNum();
regNumber targetReg = treeNode->GetRegNum();

genConsumeAddress(addr);
genConsumeRegs(data);

assert(treeNode->OperIs(GT_XCHG) || !varTypeIsSmall(treeNode->TypeGet()));

emitter* emit = GetEmitter();
emitAttr dataSize = emitActualTypeSize(data);
bool is4 = (dataSize == EA_4BYTE);

instruction ins = INS_none;
if (!varTypeIsSmall(treeNode->TypeGet()))
{
switch (treeNode->gtOper)
{
case GT_XORR:
ins = is4 ? INS_amor_db_w : INS_amor_db_d;
break;
case GT_XAND:
ins = is4 ? INS_amand_db_w : INS_amand_db_d;
break;
case GT_XCHG:
ins = is4 ? INS_amswap_db_w : INS_amswap_db_d;
break;
case GT_XADD:
ins = is4 ? INS_amadd_db_w : INS_amadd_db_d;
break;
default:
noway_assert(!"Unexpected treeNode->gtOper");
}
emit->emitIns_R_R_R(ins, dataSize, targetReg, dataReg, addrReg);
}
else
{
// Smalltypes only support atomic instructions for GT_XCHG
assert(!treeNode->OperIs(GT_XORR, GT_XAND, GT_XADD));
// ISA1.1
if (m_compiler->opts.compSupportsISA.HasInstructionSet(InstructionSet_LAM_BH))
{
if (varTypeIsByte(treeNode->TypeGet()))
{
ins = INS_amswap_db_b;
}
else if (varTypeIsShort(treeNode->TypeGet()))
{
ins = INS_amswap_db_h;
}
emit->emitIns_R_R_R(ins, dataSize, targetReg, dataReg, addrReg);
}
}

if (varTypeIsSmall(treeNode->TypeGet()) && varTypeIsUnsigned(treeNode->TypeGet()))
{
int imm1 = varTypeIsByte(treeNode->TypeGet()) ? 7 : 15;
emit->emitIns_R_R_I_I(INS_bstrpick_d, dataSize, targetReg, targetReg, imm1, 0);
}

genProduceReg(treeNode);
}

//------------------------------------------------------------------------
Expand All @@ -2185,7 +2247,98 @@ void CodeGen::genLockedInstructions(GenTreeOp* treeNode)
//
void CodeGen::genCodeForCmpXchg(GenTreeCmpXchg* treeNode)
{
NYI("unimplemented on LOONGARCH64 yet");
assert(treeNode->OperIs(GT_CMPXCHG));

GenTree* addr = treeNode->Addr(); // arg1
GenTree* data = treeNode->Data(); // arg2
GenTree* comparand = treeNode->Comparand(); // arg3

regNumber targetReg = treeNode->GetRegNum();
regNumber dataReg = data->GetRegNum();
regNumber addrReg = addr->GetRegNum();
regNumber comparandReg = !comparand->isContained() ? comparand->GetRegNum() : REG_R0;

genConsumeAddress(addr);
genConsumeRegs(data);
genConsumeRegs(comparand);

emitter* emit = GetEmitter();
emitAttr dataSize = emitActualTypeSize(data);
bool is4 = (dataSize == EA_4BYTE);
if (m_compiler->opts.compSupportsISA.HasInstructionSet(InstructionSet_LAM_CAS))
{
// amcas_db use the comparand as the target reg
emit->emitIns_R_R(INS_mov, dataSize, targetReg, comparandReg);

// Catch case we destroyed data or address before use
noway_assert((addrReg != targetReg) || (targetReg == comparandReg));
noway_assert((dataReg != targetReg) || (targetReg == comparandReg));

instruction ins = INS_none;
ins = is4 ? INS_amcas_db_w : INS_amcas_db_d;
if (varTypeIsByte(treeNode->TypeGet()))
{
ins = INS_amcas_db_b;
}
else if (varTypeIsShort(treeNode->TypeGet()))
{
ins = INS_amcas_db_h;
}
GetEmitter()->emitIns_R_R_R(ins, dataSize, targetReg, dataReg, addrReg);
}
else
{
assert(!varTypeIsSmall(treeNode->TypeGet()));
regNumber exResultReg = internalRegisters.Extract(treeNode, RBM_ALLINT);
// Register allocator should have extended the lifetimes of all input and internal registers
// They should all be different
noway_assert(addrReg != targetReg);
noway_assert(dataReg != targetReg);
noway_assert(comparandReg != targetReg);
noway_assert(addrReg != dataReg);
noway_assert(targetReg != REG_NA);
noway_assert(exResultReg != REG_NA);
noway_assert(exResultReg != targetReg);

assert(addr->isUsedFromReg());
assert(!comparand->isUsedFromMemory());

// Store exclusive unpredictable cases must be avoided
noway_assert(exResultReg != dataReg);
noway_assert(exResultReg != addrReg);

// NOTE: `genConsumeAddress` marks consumed register as not a GC pointer, assuming the input
// registers die at the first generated instruction. However, here the input registers are reused,
// so mark the location register as a GC pointer until code generation for this node is finished.
gcInfo.gcMarkRegPtrVal(addrReg, addr->TypeGet());

BasicBlock* labelRetry = genCreateTempLabel();
BasicBlock* fail = genCreateTempLabel();

if (is4)
{
// INS_bne is 64 bit comparison, high bits may be contaminated.
emit->emitIns_R_R_I(INS_slli_w, dataSize, comparandReg, comparandReg, 0x0);
}
instGen_MemoryBarrier();
genDefineTempLabel(labelRetry);
emit->emitIns_R_R_I(is4 ? INS_ll_w : INS_ll_d, dataSize, targetReg, addrReg, 0); // load original value
emit->emitIns_J_cond_la(INS_bne, fail, targetReg, comparandReg); // fail if doesn’t match
emit->emitIns_R_R(INS_mov, dataSize, exResultReg, dataReg);
emit->emitIns_R_R_I(is4 ? INS_sc_w : INS_sc_d, dataSize, exResultReg, addrReg, 0); // try to update
emit->emitIns_J_cond_la(INS_beq, labelRetry, exResultReg, REG_R0); // retry if update failed
genDefineTempLabel(fail);
instGen_MemoryBarrier();
gcInfo.gcMarkRegSetNpt(addr->gtGetRegMask());
}

if (varTypeIsSmall(treeNode->TypeGet()) && varTypeIsUnsigned(treeNode->TypeGet()))
{
int imm1 = varTypeIsByte(treeNode->TypeGet()) ? 7 : 15;
GetEmitter()->emitIns_R_R_I_I(INS_bstrpick_d, dataSize, targetReg, targetReg, imm1, 0);
}

genProduceReg(treeNode);
}

static inline bool isImmed(GenTree* treeNode)
Expand Down Expand Up @@ -4194,6 +4347,8 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode)

case GT_XCHG:
case GT_XADD:
case GT_XORR:
case GT_XAND:
genLockedInstructions(treeNode->AsOp());
break;

Expand Down
2 changes: 1 addition & 1 deletion src/coreclr/jit/compiler.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10714,7 +10714,7 @@ class Compiler
// on which the function is executed (except for CoreLib, where there are special rules)
bool compExactlyDependsOn(CORINFO_InstructionSet isa) const
{
#if defined(TARGET_XARCH) || defined(TARGET_ARM64) || defined(TARGET_RISCV64)
#if defined(TARGET_XARCH) || defined(TARGET_ARM64) || defined(TARGET_RISCV64) || defined(TARGET_LOONGARCH64)
if ((opts.compSupportsISAReported.HasInstructionSet(isa)) == false)
{
if (notifyInstructionSetUsage(isa, (opts.compSupportsISA.HasInstructionSet(isa))))
Expand Down
23 changes: 22 additions & 1 deletion src/coreclr/jit/emitloongarch64.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1557,17 +1557,33 @@ void emitter::emitIns_R_R_R(
code |= (reg2 /*& 0x1f*/) << 5; // rj
code |= (reg3 /*& 0x1f*/) << 10; // rk
}
else if ((INS_amswap_w <= ins) && (ins <= INS_ammin_db_du))
else if ((INS_amcas_b <= ins) && (ins <= INS_ammin_db_du))
{
#ifdef DEBUG
switch (ins)
{
case INS_amcas_b:
case INS_amcas_h:
case INS_amcas_w:
case INS_amcas_d:
case INS_amcas_db_b:
case INS_amcas_db_h:
case INS_amcas_db_w:
case INS_amcas_db_d:
case INS_amswap_b:
case INS_amswap_h:
case INS_amswap_w:
case INS_amswap_d:
case INS_amswap_db_b:
case INS_amswap_db_h:
case INS_amswap_db_w:
case INS_amswap_db_d:
case INS_amadd_b:
case INS_amadd_h:
case INS_amadd_w:
case INS_amadd_d:
case INS_amadd_db_b:
case INS_amadd_db_h:
case INS_amadd_db_w:
case INS_amadd_db_d:
case INS_amand_w:
Expand Down Expand Up @@ -4239,6 +4255,11 @@ void emitter::emitDisInsName(code_t code, const BYTE* addr, instrDesc* id)
printf("0x%x, %s, %s\n", regd, RegNames[regj], RegNames[(code >> 10) & 0x1f]);
return;
}
if ((INS_amcas_b <= ins) && (ins <= INS_ammin_db_du))
{
printf("%s, %s, %s\n", RegNames[regd], RegNames[(code >> 10) & 0x1f], RegNames[regj]);
return;
}
printf("%s, %s, %s\n", RegNames[regd], RegNames[regj], RegNames[(code >> 10) & 0x1f]);
return;
}
Expand Down
3 changes: 3 additions & 0 deletions src/coreclr/jit/hwintrinsic.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1012,6 +1012,9 @@ static const HWIntrinsicIsaRange hwintrinsicIsaRangeArray[] = {
{ NI_Illegal, NI_Illegal }, // SveAes_Arm64
{ NI_Illegal, NI_Illegal }, // SveSha3_Arm64
{ NI_Illegal, NI_Illegal }, // SveSm4_Arm64
#elif defined (TARGET_LOONGARCH64)
{ FIRST_NI_LAM_BH, LAST_NI_LAM_BH }, // LAM_BH
{ FIRST_NI_LAM_CAS, LAST_NI_LAM_CAS }, // LAM_CAS
#elif defined(TARGET_WASM)
{ NI_Illegal, NI_Illegal }, // WasmBase
{ FIRST_NI_PackedSimd, LAST_NI_PackedSimd }, // PackedSimd
Expand Down
Loading
Loading