Skip to content

Commit 54e55db

Browse files
authored
JIT: Always track the context for late devirt (#112396)
* Track InlinersContext in LateDevirtualizationInfo * Rework fix * Update inline context for statement as well * Use IL offset from call * Always set DebugInfo * Track InlineContext * Cleanup * Always record late devirt info * Always record late devirt info * Copy gtLateDevirtualizationInfo * Refactor to take InlineContext explicitly * Set inlinersContext to nullptr * Late devirt info always available * Oops * Remove the reduandant flag
1 parent bde9ba9 commit 54e55db

File tree

6 files changed

+49
-46
lines changed

6 files changed

+49
-46
lines changed

src/coreclr/jit/compiler.h

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5247,6 +5247,7 @@ class Compiler
52475247
CORINFO_METHOD_HANDLE fncHandle,
52485248
unsigned methAttr,
52495249
CORINFO_CONTEXT_HANDLE exactContextHnd,
5250+
InlineContext* inlinersContext,
52505251
InlineCandidateInfo** ppInlineCandidateInfo,
52515252
InlineResult* inlineResult);
52525253

@@ -5268,13 +5269,15 @@ class Compiler
52685269
void impMarkInlineCandidate(GenTree* call,
52695270
CORINFO_CONTEXT_HANDLE exactContextHnd,
52705271
bool exactContextNeedsRuntimeLookup,
5271-
CORINFO_CALL_INFO* callInfo);
5272+
CORINFO_CALL_INFO* callInfo,
5273+
InlineContext* inlinersContext);
52725274

52735275
void impMarkInlineCandidateHelper(GenTreeCall* call,
52745276
uint8_t candidateIndex,
52755277
CORINFO_CONTEXT_HANDLE exactContextHnd,
52765278
bool exactContextNeedsRuntimeLookup,
52775279
CORINFO_CALL_INFO* callInfo,
5280+
InlineContext* inlinersContext,
52785281
InlineResult* inlineResult);
52795282

52805283
bool impTailCallRetTypeCompatible(bool allowWidening,

src/coreclr/jit/fginline.cpp

Lines changed: 4 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -589,22 +589,13 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitor<Substi
589589
}
590590
#endif // DEBUG
591591

592-
CORINFO_CONTEXT_HANDLE context = nullptr;
592+
CORINFO_CONTEXT_HANDLE context = call->gtLateDevirtualizationInfo->exactContextHnd;
593+
InlineContext* inlinersContext = call->gtLateDevirtualizationInfo->inlinersContext;
593594
CORINFO_METHOD_HANDLE method = call->gtCallMethHnd;
594595
unsigned methodFlags = 0;
595596
const bool isLateDevirtualization = true;
596597
const bool explicitTailCall = call->IsTailPrefixedCall();
597598

598-
if ((call->gtCallMoreFlags & GTF_CALL_M_HAS_LATE_DEVIRT_INFO) != 0)
599-
{
600-
context = call->gtLateDevirtualizationInfo->exactContextHnd;
601-
// Note: we might call this multiple times for the same trees.
602-
// If the devirtualization below succeeds, the call becomes
603-
// non-virtual and we won't get here again. If it does not
604-
// succeed we might get here again so we keep the late devirt
605-
// info.
606-
}
607-
608599
CORINFO_CONTEXT_HANDLE contextInput = context;
609600
context = nullptr;
610601
m_compiler->impDevirtualizeCall(call, nullptr, &method, &methodFlags, &contextInput, &context,
@@ -613,10 +604,11 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitor<Substi
613604
if (!call->IsVirtual())
614605
{
615606
assert(context != nullptr);
607+
assert(inlinersContext != nullptr);
616608
CORINFO_CALL_INFO callInfo = {};
617609
callInfo.hMethod = method;
618610
callInfo.methodFlags = methodFlags;
619-
m_compiler->impMarkInlineCandidate(call, context, false, &callInfo);
611+
m_compiler->impMarkInlineCandidate(call, context, false, &callInfo, inlinersContext);
620612

621613
if (call->IsInlineCandidate())
622614
{
@@ -652,9 +644,6 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitor<Substi
652644
*pTree = retExpr;
653645
}
654646

655-
call->GetSingleInlineCandidateInfo()->exactContextHandle = context;
656-
INDEBUG(call->GetSingleInlineCandidateInfo()->inlinersContext = call->gtInlineContext);
657-
658647
JITDUMP("New inline candidate due to late devirtualization:\n");
659648
DISPTREE(call);
660649
}

src/coreclr/jit/gentree.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10002,6 +10002,8 @@ GenTreeCall* Compiler::gtCloneExprCallHelper(GenTreeCall* tree)
1000210002
copy->gtInlineInfoCount = tree->gtInlineInfoCount;
1000310003
}
1000410004

10005+
copy->gtLateDevirtualizationInfo = tree->gtLateDevirtualizationInfo;
10006+
1000510007
copy->gtCallType = tree->gtCallType;
1000610008
copy->gtReturnType = tree->gtReturnType;
1000710009

src/coreclr/jit/gentree.h

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4225,12 +4225,11 @@ enum GenTreeCallFlags : unsigned int
42254225
GTF_CALL_M_ALLOC_SIDE_EFFECTS = 0x00100000, // this is a call to an allocator with side effects
42264226
GTF_CALL_M_SUPPRESS_GC_TRANSITION = 0x00200000, // suppress the GC transition (i.e. during a pinvoke) but a separate GC safe point is required.
42274227
GTF_CALL_M_EXPANDED_EARLY = 0x00800000, // the Virtual Call target address is expanded and placed in gtControlExpr in Morph rather than in Lower
4228-
GTF_CALL_M_HAS_LATE_DEVIRT_INFO = 0x01000000, // this call has late devirtualzation info
4229-
GTF_CALL_M_LDVIRTFTN_INTERFACE = 0x02000000, // ldvirtftn on an interface type
4230-
GTF_CALL_M_CAST_CAN_BE_EXPANDED = 0x04000000, // this cast (helper call) can be expanded if it's profitable. To be removed.
4231-
GTF_CALL_M_CAST_OBJ_NONNULL = 0x08000000, // if we expand this specific cast we don't need to check the input object for null
4228+
GTF_CALL_M_LDVIRTFTN_INTERFACE = 0x01000000, // ldvirtftn on an interface type
4229+
GTF_CALL_M_CAST_CAN_BE_EXPANDED = 0x02000000, // this cast (helper call) can be expanded if it's profitable. To be removed.
4230+
GTF_CALL_M_CAST_OBJ_NONNULL = 0x04000000, // if we expand this specific cast we don't need to check the input object for null
42324231
// NOTE: if needed, this flag can be removed, and we can introduce new _NONNUL cast helpers
4233-
GTF_CALL_M_STACK_ARRAY = 0x10000000, // this call is a new array helper for a stack allocated array.
4232+
GTF_CALL_M_STACK_ARRAY = 0x08000000, // this call is a new array helper for a stack allocated array.
42344233
};
42354234

42364235
inline constexpr GenTreeCallFlags operator ~(GenTreeCallFlags a)
@@ -5742,11 +5741,13 @@ struct GenTreeCall final : public GenTree
57425741
jitstd::vector<InlineCandidateInfo*>* gtInlineCandidateInfoList;
57435742

57445743
HandleHistogramProfileCandidateInfo* gtHandleHistogramProfileCandidateInfo;
5745-
LateDevirtualizationInfo* gtLateDevirtualizationInfo;
5744+
57465745
CORINFO_GENERIC_HANDLE compileTimeHelperArgumentHandle; // Used to track type handle argument of dynamic helpers
57475746
void* gtDirectCallAddress; // Used to pass direct call address between lower and codegen
57485747
};
57495748

5749+
LateDevirtualizationInfo* gtLateDevirtualizationInfo; // Always available for user virtual calls
5750+
57505751
// expression evaluated after args are placed which determines the control target
57515752
GenTree* gtControlExpr;
57525753

src/coreclr/jit/importercalls.cpp

Lines changed: 31 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1026,7 +1026,8 @@ var_types Compiler::impImportCall(OPCODE opcode,
10261026
INDEBUG(call->AsCall()->gtRawILOffset = rawILOffset);
10271027

10281028
// Is it an inline candidate?
1029-
impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo);
1029+
impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo,
1030+
compInlineContext);
10301031
}
10311032

10321033
// append the call node.
@@ -1236,7 +1237,20 @@ var_types Compiler::impImportCall(OPCODE opcode,
12361237
INDEBUG(call->AsCall()->gtRawILOffset = rawILOffset);
12371238

12381239
// Is it an inline candidate?
1239-
impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo);
1240+
impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, compInlineContext);
1241+
1242+
// If the call is virtual, record the inliner's context for possible use during late devirt inlining.
1243+
// Also record the generics context if there is any.
1244+
//
1245+
if (call->AsCall()->IsVirtual() && (call->AsCall()->gtCallType != CT_INDIRECT))
1246+
{
1247+
JITDUMP("\nSaving generic context %p and inline context %p for call [%06u]\n", dspPtr(exactContextHnd),
1248+
dspPtr(compInlineContext), dspTreeID(call->AsCall()));
1249+
LateDevirtualizationInfo* const info = new (this, CMK_Inlining) LateDevirtualizationInfo;
1250+
info->exactContextHnd = exactContextHnd;
1251+
info->inlinersContext = compInlineContext;
1252+
call->AsCall()->gtLateDevirtualizationInfo = info;
1253+
}
12401254
}
12411255

12421256
// Extra checks for tail calls and tail recursion.
@@ -1427,22 +1441,6 @@ var_types Compiler::impImportCall(OPCODE opcode,
14271441
}
14281442
else
14291443
{
1430-
// If the call is virtual, and has a generics context, and is not going to have a class probe,
1431-
// record the context for possible use during late devirt.
1432-
//
1433-
// If we ever want to devirt at Tier0, and/or see issues where OSR methods under PGO lose
1434-
// important devirtualizations, we'll want to allow both a class probe and a captured context.
1435-
//
1436-
if (origCall->IsVirtual() && (origCall->gtCallType != CT_INDIRECT) && (exactContextHnd != nullptr) &&
1437-
(origCall->gtHandleHistogramProfileCandidateInfo == nullptr))
1438-
{
1439-
JITDUMP("\nSaving context %p for call [%06u]\n", dspPtr(exactContextHnd), dspTreeID(origCall));
1440-
origCall->gtCallMoreFlags |= GTF_CALL_M_HAS_LATE_DEVIRT_INFO;
1441-
LateDevirtualizationInfo* const info = new (this, CMK_Inlining) LateDevirtualizationInfo;
1442-
info->exactContextHnd = exactContextHnd;
1443-
origCall->gtLateDevirtualizationInfo = info;
1444-
}
1445-
14461444
if (isFatPointerCandidate)
14471445
{
14481446
// fatPointer candidates should be in statements of the form call() or var = call().
@@ -7451,6 +7449,7 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call,
74517449
// exactContextHnd -- context handle for inlining
74527450
// exactContextNeedsRuntimeLookup -- true if context required runtime lookup
74537451
// callInfo -- call info from VM
7452+
// inlinersContext -- the inliner's context
74547453
//
74557454
// Notes:
74567455
// Mostly a wrapper for impMarkInlineCandidateHelper that also undoes
@@ -7460,7 +7459,8 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call,
74607459
void Compiler::impMarkInlineCandidate(GenTree* callNode,
74617460
CORINFO_CONTEXT_HANDLE exactContextHnd,
74627461
bool exactContextNeedsRuntimeLookup,
7463-
CORINFO_CALL_INFO* callInfo)
7462+
CORINFO_CALL_INFO* callInfo,
7463+
InlineContext* inlinersContext)
74647464
{
74657465
if (!opts.OptEnabled(CLFLG_INLINING))
74667466
{
@@ -7483,7 +7483,7 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode,
74837483

74847484
// Do the actual evaluation
74857485
impMarkInlineCandidateHelper(call, candidateId, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo,
7486-
&inlineResult);
7486+
inlinersContext, &inlineResult);
74877487
// Ignore non-inlineable candidates
74887488
// TODO: Consider keeping them to just devirtualize without inlining, at least for interface
74897489
// calls on NativeAOT, but that requires more changes elsewhere too.
@@ -7506,7 +7506,8 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode,
75067506
const uint8_t candidatesCount = call->GetInlineCandidatesCount();
75077507
assert(candidatesCount <= 1);
75087508
InlineResult inlineResult(this, call, nullptr, "impMarkInlineCandidate");
7509-
impMarkInlineCandidateHelper(call, 0, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, &inlineResult);
7509+
impMarkInlineCandidateHelper(call, 0, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo,
7510+
inlinersContext, &inlineResult);
75107511
}
75117512

75127513
// If this call is an inline candidate or is not a guarded devirtualization
@@ -7539,6 +7540,7 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode,
75397540
// exactContextHnd -- context handle for inlining
75407541
// exactContextNeedsRuntimeLookup -- true if context required runtime lookup
75417542
// callInfo -- call info from VM
7543+
// inlinersContext -- the inliner's context
75427544
//
75437545
// Notes:
75447546
// If callNode is an inline candidate, this method sets the flag
@@ -7555,6 +7557,7 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call,
75557557
CORINFO_CONTEXT_HANDLE exactContextHnd,
75567558
bool exactContextNeedsRuntimeLookup,
75577559
CORINFO_CALL_INFO* callInfo,
7560+
InlineContext* inlinersContext,
75587561
InlineResult* inlineResult)
75597562
{
75607563
// Let the strategy know there's another call
@@ -7749,7 +7752,8 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call,
77497752
}
77507753

77517754
InlineCandidateInfo* inlineCandidateInfo = nullptr;
7752-
impCheckCanInline(call, candidateIndex, fncHandle, methAttr, exactContextHnd, &inlineCandidateInfo, inlineResult);
7755+
impCheckCanInline(call, candidateIndex, fncHandle, methAttr, exactContextHnd, inlinersContext, &inlineCandidateInfo,
7756+
inlineResult);
77537757

77547758
if (inlineResult->IsFailure())
77557759
{
@@ -8378,7 +8382,6 @@ void Compiler::impDevirtualizeCall(GenTreeCall* call,
83788382
// it's a union field used for other things by virtual
83798383
// stubs)
83808384
call->ClearInlineInfo();
8381-
call->gtCallMoreFlags &= ~GTF_CALL_M_HAS_LATE_DEVIRT_INFO;
83828385

83838386
#if defined(DEBUG)
83848387
if (verbose)
@@ -9057,6 +9060,7 @@ bool Compiler::impTailCallRetTypeCompatible(bool allowWideni
90579060
// fncHandle - method that will be called
90589061
// methAttr - attributes for the method
90599062
// exactContextHnd - exact context for the method
9063+
// inlinersContext - the inliner's context
90609064
// ppInlineCandidateInfo [out] - information needed later for inlining
90619065
// inlineResult - result of ongoing inline evaluation
90629066
//
@@ -9069,6 +9073,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call,
90699073
CORINFO_METHOD_HANDLE fncHandle,
90709074
unsigned methAttr,
90719075
CORINFO_CONTEXT_HANDLE exactContextHnd,
9076+
InlineContext* inlinersContext,
90729077
InlineCandidateInfo** ppInlineCandidateInfo,
90739078
InlineResult* inlineResult)
90749079
{
@@ -9083,6 +9088,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call,
90839088
CORINFO_METHOD_HANDLE fncHandle;
90849089
unsigned methAttr;
90859090
CORINFO_CONTEXT_HANDLE exactContextHnd;
9091+
InlineContext* inlinersContext;
90869092
InlineResult* result;
90879093
InlineCandidateInfo** ppInlineCandidateInfo;
90889094
} param;
@@ -9094,6 +9100,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call,
90949100
param.fncHandle = fncHandle;
90959101
param.methAttr = methAttr;
90969102
param.exactContextHnd = (exactContextHnd != nullptr) ? exactContextHnd : MAKE_METHODCONTEXT(fncHandle);
9103+
param.inlinersContext = inlinersContext;
90979104
param.result = inlineResult;
90989105
param.ppInlineCandidateInfo = ppInlineCandidateInfo;
90999106

@@ -9244,7 +9251,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call,
92449251
pInfo->methAttr = pParam->methAttr;
92459252
pInfo->initClassResult = initClassResult;
92469253
pInfo->exactContextNeedsRuntimeLookup = false;
9247-
pInfo->inlinersContext = pParam->pThis->compInlineContext;
9254+
pInfo->inlinersContext = pParam->inlinersContext;
92489255

92499256
// Note exactContextNeedsRuntimeLookup is reset later on,
92509257
// over in impMarkInlineCandidate.

src/coreclr/jit/inline.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -634,6 +634,7 @@ struct InlineCandidateInfo : public HandleHistogramProfileCandidateInfo
634634
struct LateDevirtualizationInfo
635635
{
636636
CORINFO_CONTEXT_HANDLE exactContextHnd;
637+
InlineContext* inlinersContext;
637638
};
638639

639640
// InlArgInfo describes inline candidate argument properties.

0 commit comments

Comments
 (0)