Skip to content

Commit d2bbed8

Browse files
Aman Khalidamanasifkhalid
Aman Khalid
authored and
amanasifkhalid
committed
Enable fake hot/cold splitting on ARM64
This commit contains fixes for various bugs exposed by enabling fake hot/cold splitting on ARM64: - Branches between hot/cold sections are now always long. - The pseudoinstruction for loading a constant from the cold section did not support loading 16-byte data into vector registers, as it temporarily loaded the constant into an 8-byte integer register. Now, 16-byte constants are loaded directly into vector registers via an `ld1` instruction. - Tests involving loading 16-byte constants exposed the data section is not always aligned to its largest constant. Now, the data section is always aligned to `emitConsDsc.alignment` when calling `eeAllocMem`. - Asserts/NYIs blocking hot/cold splitting on ARM64 have been removed. Fake hot/cold splitting requires we fake unwind info by treating each split function as one hot section. A more architecture-agnostic approach for this has been applied.
1 parent 7989a93 commit d2bbed8

File tree

8 files changed

+108
-105
lines changed

8 files changed

+108
-105
lines changed

‎src/coreclr/jit/compiler.h‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8000,10 +8000,6 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
80008000
void unwindReserveFuncHelper(FuncInfoDsc* func, bool isHotCode);
80018001
void unwindEmitFuncHelper(FuncInfoDsc* func, void* pHotCode, void* pColdCode, bool isHotCode);
80028002

8003-
#ifdef DEBUG
8004-
void fakeUnwindEmitFuncHelper(FuncInfoDsc* func, void* pHotCode);
8005-
#endif // DEBUG
8006-
80078003
#endif // TARGET_AMD64 || (TARGET_X86 && FEATURE_EH_FUNCLETS)
80088004

80098005
UNATIVE_OFFSET unwindGetCurrentOffset(FuncInfoDsc* func);

‎src/coreclr/jit/emit.cpp‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6045,13 +6045,8 @@ unsigned emitter::emitEndCodeGen(Compiler* comp,
60456045
// For arm64/LoongArch64, we want to allocate JIT data always adjacent to code similar to what native compiler does.
60466046
// This way allows us to use a single `ldr` to access such data like float constant/jmp table.
60476047

6048-
UNATIVE_OFFSET roDataAlignmentDelta = 0;
6049-
if (emitConsDsc.dsdOffs && (emitConsDsc.alignment == TARGET_POINTER_SIZE))
6050-
{
6051-
UNATIVE_OFFSET roDataAlignment = TARGET_POINTER_SIZE; // 8 Byte align by default.
6052-
roDataAlignmentDelta = (UNATIVE_OFFSET)ALIGN_UP(emitTotalHotCodeSize, roDataAlignment) - emitTotalHotCodeSize;
6053-
assert((roDataAlignmentDelta == 0) || (roDataAlignmentDelta == 4));
6054-
}
6048+
const UNATIVE_OFFSET roDataAlignmentDelta =
6049+
(UNATIVE_OFFSET)ALIGN_UP(emitTotalHotCodeSize, emitConsDsc.alignment) - emitTotalHotCodeSize;
60556050

60566051
args.hotCodeSize = emitTotalHotCodeSize + roDataAlignmentDelta + emitConsDsc.dsdOffs;
60576052
args.coldCodeSize = emitTotalColdCodeSize;

‎src/coreclr/jit/emitarm64.cpp‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8422,7 +8422,8 @@ void emitter::emitIns_J(instruction ins, BasicBlock* dst, int instrCount)
84228422
{
84238423
case INS_bl_local:
84248424
idjShort = true;
8425-
// Fall through.
8425+
fmt = IF_BI_0A;
8426+
break;
84268427
case INS_b:
84278428
// Unconditional jump is a single form.
84288429
// Assume is long in case we cross hot/cold sections.
@@ -9825,6 +9826,9 @@ BYTE* emitter::emitOutputLJ(insGroup* ig, BYTE* dst, instrDesc* i)
98259826
// Special case: emit add + ld1 instructions for loading 16-byte data into vector register.
98269827
if (isVectorRegister(dstReg) && (opSize == EA_16BYTE))
98279828
{
9829+
// Low 4 bits should be 0 -- 16-byte JIT data should be aligned on 16 bytes.
9830+
assert((imm12 & 15) == 0);
9831+
98289832
const emitAttr elemSize = EA_1BYTE;
98299833
const insOpts opt = optMakeArrangement(opSize, elemSize);
98309834

‎src/coreclr/jit/unwind.cpp‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,16 @@ void Compiler::unwindGetFuncLocations(FuncInfoDsc* func,
6969
// The hot section only goes up to the cold section
7070
assert(fgFirstFuncletBB == nullptr);
7171

72-
*ppEndLoc = new (this, CMK_UnwindInfo) emitLocation(ehEmitCookie(fgFirstColdBlock));
72+
#ifdef DEBUG
73+
if (JitConfig.JitFakeProcedureSplitting())
74+
{
75+
*ppEndLoc = nullptr; // If fake-splitting, "trick" VM by pretending entire function is hot.
76+
}
77+
else
78+
#endif // DEBUG
79+
{
80+
*ppEndLoc = new (this, CMK_UnwindInfo) emitLocation(ehEmitCookie(fgFirstColdBlock));
81+
}
7382
}
7483
else
7584
{
@@ -259,6 +268,13 @@ void Compiler::unwindEmitFuncCFI(FuncInfoDsc* func, void* pHotCode, void* pColdC
259268
DWORD unwindCodeBytes = 0;
260269
BYTE* pUnwindBlock = nullptr;
261270

271+
#ifdef DEBUG
272+
if (JitConfig.JitFakeProcedureSplitting())
273+
{
274+
pColdCode = nullptr;
275+
}
276+
#endif // DEBUG
277+
262278
if (func->startLoc == nullptr)
263279
{
264280
startOffset = 0;

‎src/coreclr/jit/unwindamd64.cpp‎

Lines changed: 25 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -656,18 +656,17 @@ void Compiler::unwindReserve()
656656
//
657657
void Compiler::unwindReserveFunc(FuncInfoDsc* func)
658658
{
659-
#ifdef DEBUG
660-
if (JitConfig.JitFakeProcedureSplitting() && (fgFirstColdBlock != nullptr))
659+
unwindReserveFuncHelper(func, true);
660+
661+
if (fgFirstColdBlock != nullptr)
661662
{
662-
assert(func->funKind == FUNC_ROOT); // No fake-splitting of funclets.
663-
unwindReserveFuncHelper(func, true);
664-
}
665-
else
663+
#ifdef DEBUG
664+
if (JitConfig.JitFakeProcedureSplitting())
665+
{
666+
assert(func->funKind == FUNC_ROOT); // No splitting of funclets.
667+
}
668+
else
666669
#endif // DEBUG
667-
{
668-
unwindReserveFuncHelper(func, true);
669-
670-
if (fgFirstColdBlock != nullptr)
671670
{
672671
unwindReserveFuncHelper(func, false);
673672
}
@@ -859,7 +858,17 @@ void Compiler::unwindEmitFuncHelper(FuncInfoDsc* func, void* pHotCode, void* pCo
859858

860859
if (isHotCode)
861860
{
862-
assert(endOffset <= info.compTotalHotCodeSize);
861+
#ifdef DEBUG
862+
if (JitConfig.JitFakeProcedureSplitting() && (fgFirstColdBlock != nullptr))
863+
{
864+
assert(endOffset <= info.compNativeCodeSize);
865+
}
866+
else
867+
#endif // DEBUG
868+
{
869+
assert(endOffset <= info.compTotalHotCodeSize);
870+
}
871+
863872
pColdCode = nullptr;
864873
}
865874
else
@@ -890,43 +899,17 @@ void Compiler::unwindEmitFunc(FuncInfoDsc* func, void* pHotCode, void* pColdCode
890899
static_assert_no_msg(FUNC_HANDLER == (FuncKind)CORJIT_FUNC_HANDLER);
891900
static_assert_no_msg(FUNC_FILTER == (FuncKind)CORJIT_FUNC_FILTER);
892901

893-
#ifdef DEBUG
894-
if (JitConfig.JitFakeProcedureSplitting() && (pColdCode != nullptr))
902+
unwindEmitFuncHelper(func, pHotCode, pColdCode, true);
903+
904+
if (pColdCode != nullptr)
895905
{
896-
fakeUnwindEmitFuncHelper(func, pHotCode);
897-
}
898-
else
906+
#ifdef DEBUG
907+
if (!JitConfig.JitFakeProcedureSplitting())
899908
#endif // DEBUG
900-
{
901-
unwindEmitFuncHelper(func, pHotCode, pColdCode, true);
902-
903-
if (pColdCode != nullptr)
904909
{
905910
unwindEmitFuncHelper(func, pHotCode, pColdCode, false);
906911
}
907912
}
908913
}
909914

910-
#ifdef DEBUG
911-
void Compiler::fakeUnwindEmitFuncHelper(FuncInfoDsc* func, void* pHotCode)
912-
{
913-
assert(fgFirstColdBlock != nullptr);
914-
assert(func->funKind == FUNC_ROOT); // No fake-splitting of funclets.
915-
916-
const UNATIVE_OFFSET startOffset = 0;
917-
const UNATIVE_OFFSET endOffset = info.compNativeCodeSize;
918-
const DWORD unwindCodeBytes = sizeof(func->unwindCodes) - func->unwindCodeSlot;
919-
BYTE* pUnwindBlock = &func->unwindCodes[func->unwindCodeSlot];
920-
921-
if (opts.dspUnwind)
922-
{
923-
DumpUnwindInfo(true, startOffset, endOffset, (const UNWIND_INFO* const)pUnwindBlock);
924-
}
925-
926-
// Pass pColdCode = nullptr; VM allocs unwind info for combined hot/cold section
927-
eeAllocUnwindInfo((BYTE*)pHotCode, nullptr, startOffset, endOffset, unwindCodeBytes, pUnwindBlock,
928-
(CorJitFuncKind)func->funKind);
929-
}
930-
#endif // DEBUG
931-
932915
#endif // TARGET_AMD64

‎src/coreclr/jit/unwindarm.cpp‎

Lines changed: 34 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -563,13 +563,20 @@ void Compiler::unwindReserve()
563563
void Compiler::unwindReserveFunc(FuncInfoDsc* func)
564564
{
565565
BOOL isFunclet = (func->funKind == FUNC_ROOT) ? FALSE : TRUE;
566-
bool funcHasColdSection = false;
566+
bool funcHasColdSection = (fgFirstColdBlock != nullptr);
567+
568+
#ifdef DEBUG
569+
if (JitConfig.JitFakeProcedureSplitting() && funcHasColdSection)
570+
{
571+
funcHasColdSection = false; // "Trick" the VM into thinking we don't have a cold section.
572+
}
573+
#endif // DEBUG
567574

568575
#if defined(FEATURE_CFI_SUPPORT)
569576
if (generateCFIUnwindCodes())
570577
{
571578
DWORD unwindCodeBytes = 0;
572-
if (fgFirstColdBlock != nullptr)
579+
if (funcHasColdSection)
573580
{
574581
eeReserveUnwindInfo(isFunclet, true /*isColdCode*/, unwindCodeBytes);
575582
}
@@ -584,7 +591,7 @@ void Compiler::unwindReserveFunc(FuncInfoDsc* func)
584591
// cold section. This needs to be done before we split into fragments, as each
585592
// of the hot and cold sections can have multiple fragments.
586593

587-
if (fgFirstColdBlock != NULL)
594+
if (funcHasColdSection)
588595
{
589596
assert(!isFunclet); // TODO-CQ: support hot/cold splitting with EH
590597

@@ -595,8 +602,6 @@ void Compiler::unwindReserveFunc(FuncInfoDsc* func)
595602
func->uwiCold = new (this, CMK_UnwindInfo) UnwindInfo();
596603
func->uwiCold->InitUnwindInfo(this, startLoc, endLoc);
597604
func->uwiCold->HotColdSplitCodes(&func->uwi);
598-
599-
funcHasColdSection = true;
600605
}
601606

602607
// First we need to split the function or funclet into fragments that are no larger
@@ -1604,11 +1609,19 @@ void UnwindFragmentInfo::Allocate(
16041609
UNATIVE_OFFSET endOffset;
16051610
UNATIVE_OFFSET codeSize;
16061611

1607-
// We don't support hot/cold splitting with EH, so if there is cold code, this
1608-
// better not be a funclet!
1609-
// TODO-CQ: support funclets in cold code
1610-
1611-
noway_assert(isHotCode || funKind == CORJIT_FUNC_ROOT);
1612+
// We don't support hot/cold splitting with EH, so if there is cold code, this
1613+
// better not be a funclet!
1614+
// TODO-CQ: support funclets in cold code
1615+
#ifdef DEBUG
1616+
if (JitConfig.JitFakeProcedureSplitting() && (pColdCode != NULL))
1617+
{
1618+
noway_assert(isHotCode && (funKind == CORJIT_FUNC_ROOT));
1619+
}
1620+
else
1621+
#endif // DEBUG
1622+
{
1623+
noway_assert(isHotCode || (funKind == CORJIT_FUNC_ROOT));
1624+
}
16121625

16131626
// Compute the final size, and start and end offsets of the fragment
16141627

@@ -1656,7 +1669,17 @@ void UnwindFragmentInfo::Allocate(
16561669

16571670
if (isHotCode)
16581671
{
1659-
assert(endOffset <= uwiComp->info.compTotalHotCodeSize);
1672+
#ifdef DEBUG
1673+
if (JitConfig.JitFakeProcedureSplitting() && (pColdCode != NULL))
1674+
{
1675+
assert(endOffset <= uwiComp->info.compNativeCodeSize);
1676+
}
1677+
else
1678+
#endif // DEBUG
1679+
{
1680+
assert(endOffset <= uwiComp->info.compTotalHotCodeSize);
1681+
}
1682+
16601683
pColdCode = NULL;
16611684
}
16621685
else

‎src/coreclr/jit/unwindx86.cpp‎

Lines changed: 25 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -113,18 +113,17 @@ void Compiler::unwindEmit(void* pHotCode, void* pColdCode)
113113
//
114114
void Compiler::unwindReserveFunc(FuncInfoDsc* func)
115115
{
116-
#ifdef DEBUG
117-
if (JitConfig.JitFakeProcedureSplitting() && (fgFirstColdBlock != nullptr))
116+
unwindReserveFuncHelper(func, true);
117+
118+
if (fgFirstColdBlock != nullptr)
118119
{
119-
assert(func->funKind == FUNC_ROOT); // No fake-splitting of funclets.
120-
unwindReserveFuncHelper(func, true);
121-
}
122-
else
120+
#ifdef DEBUG
121+
if (JitConfig.JitFakeProcedureSplitting())
122+
{
123+
assert(func->funKind == FUNC_ROOT); // No splitting of funclets.
124+
}
125+
else
123126
#endif // DEBUG
124-
{
125-
unwindReserveFuncHelper(func, true);
126-
127-
if (fgFirstColdBlock != nullptr)
128127
{
129128
unwindReserveFuncHelper(func, false);
130129
}
@@ -164,17 +163,13 @@ void Compiler::unwindEmitFunc(FuncInfoDsc* func, void* pHotCode, void* pColdCode
164163
static_assert_no_msg(FUNC_HANDLER == (FuncKind)CORJIT_FUNC_HANDLER);
165164
static_assert_no_msg(FUNC_FILTER == (FuncKind)CORJIT_FUNC_FILTER);
166165

167-
#ifdef DEBUG
168-
if (JitConfig.JitFakeProcedureSplitting() && (pColdCode != nullptr))
166+
unwindEmitFuncHelper(func, pHotCode, pColdCode, true);
167+
168+
if (pColdCode != nullptr)
169169
{
170-
fakeUnwindEmitFuncHelper(func, pHotCode);
171-
}
172-
else
170+
#ifdef DEBUG
171+
if (!JitConfig.JitFakeProcedureSplitting())
173172
#endif // DEBUG
174-
{
175-
unwindEmitFuncHelper(func, pHotCode, pColdCode, true);
176-
177-
if (pColdCode != nullptr)
178173
{
179174
unwindEmitFuncHelper(func, pHotCode, pColdCode, false);
180175
}
@@ -258,7 +253,17 @@ void Compiler::unwindEmitFuncHelper(FuncInfoDsc* func, void* pHotCode, void* pCo
258253

259254
if (isHotCode)
260255
{
261-
assert(endOffset <= info.compTotalHotCodeSize);
256+
#ifdef DEBUG
257+
if (JitConfig.JitFakeProcedureSplitting() && (fgFirstColdBlock != nullptr))
258+
{
259+
assert(endOffset <= info.compNativeCodeSize);
260+
}
261+
else
262+
#endif // DEBUG
263+
{
264+
assert(endOffset <= info.compTotalHotCodeSize);
265+
}
266+
262267
pColdCode = nullptr;
263268
}
264269
else
@@ -276,22 +281,4 @@ void Compiler::unwindEmitFuncHelper(FuncInfoDsc* func, void* pHotCode, void* pCo
276281
(BYTE*)&unwindInfo, (CorJitFuncKind)func->funKind);
277282
}
278283

279-
#ifdef DEBUG
280-
void Compiler::fakeUnwindEmitFuncHelper(FuncInfoDsc* func, void* pHotCode)
281-
{
282-
assert(fgFirstColdBlock != nullptr);
283-
assert(func->funKind == FUNC_ROOT); // No fake-splitting of funclets.
284-
285-
const UNATIVE_OFFSET startOffset = 0;
286-
const UNATIVE_OFFSET endOffset = info.compNativeCodeSize;
287-
288-
UNWIND_INFO unwindInfo;
289-
unwindInfo.FunctionLength = (ULONG)(endOffset);
290-
291-
// Pass pColdCode = nullptr; VM allocs unwind info for combined hot/cold section
292-
eeAllocUnwindInfo((BYTE*)pHotCode, nullptr, startOffset, endOffset, sizeof(UNWIND_INFO), (BYTE*)&unwindInfo,
293-
(CorJitFuncKind)func->funKind);
294-
}
295-
#endif // DEBUG
296-
297284
#endif // FEATURE_EH_FUNCLETS

‎src/tests/Common/testenvironment.proj‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@
3636
COMPlus_EnableSSE42;
3737
COMPlus_EnableSSSE3;
3838
COMPlus_ForceRelocs;
39-
COMPlus_GCgen0size;
4039
COMPlus_GCStress;
4140
COMPlus_GCName;
4241
COMPlus_gcServer;

0 commit comments

Comments
 (0)