Skip to content

JIT: Tail-merge process all sets at once #129363

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
BoyBaykiller wants to merge 7 commits into
base: main
Choose a base branch
from
+116 −122
Open

JIT: Tail-merge process all sets at once #129363

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
c77a8f2
* process all sets at once
BoyBaykiller Jun 13, 2026
a8dc6bc
* fix error C4018: '>': signed/unsigned mismatch
BoyBaykiller Jun 13, 2026
5753921
* use custom jitstd::partition
BoyBaykiller Jun 13, 2026
56139e4
* dont use std::distance
BoyBaykiller Jun 13, 2026
91edc35
* try fix x86 tp
BoyBaykiller Jun 13, 2026
534ebff
* fix 'const' position
BoyBaykiller Jun 13, 2026
42a53d7
* use custom partition again
BoyBaykiller Jun 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
203 changes: 81 additions & 122 deletions src/coreclr/jit/fgopt.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@
#pragma hdrstop
#endif

#include "lower.h" // for LowerRange()
#include "lower.h" // for LowerRange()
#include "jitstd/algorithm.h" // for partition()

// Flowgraph Optimization

Expand Down Expand Up @@ -5123,9 +5124,8 @@ PhaseStatus Compiler::fgHeadTailMerge(bool early)
Statement* m_stmt;
};

ArrayStack<PredInfo> predInfo(getAllocator(CMK_ArrayStack));
ArrayStack<PredInfo> matchedPredInfo(getAllocator(CMK_ArrayStack));
ArrayStack<BasicBlock*> retryBlocks(getAllocator(CMK_ArrayStack));
jitstd::vector<PredInfo> predInfo(getAllocator(CMK_ArrayStack));
ArrayStack<BasicBlock*> retryBlocks(getAllocator(CMK_ArrayStack));

auto tryRemoveAndFixFlow = [&](BasicBlock* emptyBlock, BasicBlock* newTarget) -> bool {
assert(emptyBlock->isEmpty());
Expand Down Expand Up @@ -5154,13 +5154,15 @@ PhaseStatus Compiler::fgHeadTailMerge(bool early)
// If return value is true, retry.
// May also add to retryBlocks.
//
auto tailMergePreds = [&](BasicBlock* commSucc) -> bool {
auto tailMergePreds = [&](BasicBlock* commSucc) -> int {
int optimizedCount = 0;

// Are there enough preds to make it interesting?
//
if (predInfo.Height() < 2)
if (predInfo.size() < 2)
{
// Not enough preds to merge
return false;
return optimizedCount;
}

// If there are large numbers of viable preds, forgo trying to merge.
Expand All @@ -5176,67 +5178,61 @@ PhaseStatus Compiler::fgHeadTailMerge(bool early)
// multiple that avoids code-size regressions (measured via SPMI asmdiffs).
//
const int effectiveLimit = (commSucc != nullptr) ? mergeLimit : (4 * mergeLimit);
if (predInfo.Height() > effectiveLimit)
if (predInfo.size() > static_cast<size_t>(effectiveLimit))
{
// Too many preds to consider
return false;
return optimizedCount;
}

// Find a matching set of preds. Potentially O(N^2) tree comparisons.
//
int i = 0;
while (i < (predInfo.Height() - 1))
auto matchesEnd = predInfo.begin();
while (matchesEnd < (predInfo.end() - 1))
{
matchedPredInfo.Reset();
matchedPredInfo.Emplace(predInfo.TopRef(i));
Statement* const baseStmt = predInfo.TopRef(i).m_stmt;
BasicBlock* const baseBlock = predInfo.TopRef(i).m_block;

for (int j = i + 1; j < predInfo.Height(); j++)
{
BasicBlock* const otherBlock = predInfo.TopRef(j).m_block;
// Previous 'end' becomes new 'begin'.
auto matchesBegin = matchesEnd;
PredInfo candidateA = *matchesBegin;

// Find all matching candidates and partition them to
// be continous in memory at [matchesBegin, matchesEnd - 1]
//
matchesEnd = partition(matchesBegin + 1, predInfo.end(), [candidateA](PredInfo candidateB) {
// Consider: bypass this for statements that can't cause exceptions.
//
if (!BasicBlock::sameEHRegion(baseBlock, otherBlock))
if (!BasicBlock::sameEHRegion(candidateA.m_block, candidateB.m_block))
{
continue;
return false;
}

Statement* const otherStmt = predInfo.TopRef(j).m_stmt;
return GenTree::Compare(candidateA.m_stmt->GetRootNode(), candidateB.m_stmt->GetRootNode());
});

// Consider: compute and cache hashes to make this faster
//
if (GenTree::Compare(baseStmt->GetRootNode(), otherStmt->GetRootNode()))
{
matchedPredInfo.Emplace(predInfo.TopRef(j));
}
}

if (matchedPredInfo.Height() < 2)
int matchesCount = static_cast<int>(matchesEnd - matchesBegin);
if (matchesCount < 2)
{
// This pred didn't match any other. Check other preds for matches.
i++;
continue;
}

optimizedCount++;
madeChanges = true;

// We can move the identical last statements to commSucc, if it exists,
// and all preds have matching last statements, and we're not changing EH behavior.
//
bool const hasCommSucc = (commSucc != nullptr);
bool const predsInSameEHRegionAsSucc = hasCommSucc && BasicBlock::sameEHRegion(baseBlock, commSucc);
bool const canMergeAllPreds = hasCommSucc && (matchedPredInfo.Height() == (int)commSucc->countOfInEdges());
bool const hasCommSucc = (commSucc != nullptr);
bool const predsInSameEHRegionAsSucc =
hasCommSucc && BasicBlock::sameEHRegion(candidateA.m_block, commSucc);
bool const canMergeAllPreds = hasCommSucc && (matchesCount == (int)commSucc->countOfInEdges());
bool const canMergeIntoSucc = predsInSameEHRegionAsSucc && canMergeAllPreds;

if (canMergeIntoSucc)
{
JITDUMP("All %d preds of " FMT_BB " end with the same tree, moving\n", matchedPredInfo.Height(),
commSucc->bbNum);
JITDUMPEXEC(gtDispStmt(matchedPredInfo.TopRef(0).m_stmt));
JITDUMP("All %d preds of " FMT_BB " end with the same tree, moving\n", matchesCount, commSucc->bbNum);
JITDUMPEXEC(gtDispStmt(candidateA.m_stmt));

for (int j = 0; j < matchedPredInfo.Height(); j++)
for (auto it = matchesBegin; it < matchesEnd; it++)
{
PredInfo& info = matchedPredInfo.TopRef(j);
const PredInfo& info = *it;
Statement* const stmt = info.m_stmt;
BasicBlock* const predBlock = info.m_block;

Expand All @@ -5246,22 +5242,16 @@ PhaseStatus Compiler::fgHeadTailMerge(bool early)
{
tryRemoveAndFixFlow(predBlock, commSucc);
}

// Add one of the matching stmts to block, and
// update its flags.
//
if (j == 0)
{
fgInsertStmtAtBeg(commSucc, stmt);
commSucc->CopyFlags(predBlock, BBF_COPY_PROPAGATE);
}

madeChanges = true;
}

// Add one of the matching stmts to commSucc, and update its flags.
fgInsertStmtAtBeg(commSucc, candidateA.m_stmt);
commSucc->CopyFlags(candidateA.m_block, BBF_COPY_PROPAGATE);

// It's worth retrying tail merge on this block.
//
return true;
retryBlocks.Push(commSucc);

continue;
}

// All or a subset of preds have matching last stmt, we will cross-jump.
Expand All @@ -5270,27 +5260,27 @@ PhaseStatus Compiler::fgHeadTailMerge(bool early)
//
if (predsInSameEHRegionAsSucc)
{
JITDUMP("A subset of %d preds of " FMT_BB " end with the same tree\n", matchedPredInfo.Height(),
commSucc->bbNum);
JITDUMP("A subset of %d preds of " FMT_BB " end with the same tree\n", matchesCount, commSucc->bbNum);
}
else if (commSucc != nullptr)
{
JITDUMP("%s %d preds of " FMT_BB " end with the same tree but are in a different EH region\n",
canMergeAllPreds ? "All" : "A subset of", matchedPredInfo.Height(), commSucc->bbNum);
canMergeAllPreds ? "All" : "A subset of", matchesCount, commSucc->bbNum);
}
else
{
JITDUMP("A set of %d return blocks end with the same tree\n", matchedPredInfo.Height());
JITDUMP("A set of %d return blocks end with the same tree\n", matchesCount);
}

JITDUMPEXEC(gtDispStmt(matchedPredInfo.TopRef(0).m_stmt));
JITDUMPEXEC(gtDispStmt(candidateA.m_stmt));

BasicBlock* crossJumpVictim = nullptr;
Statement* crossJumpStmt = nullptr;
unsigned bestRank = UINT32_MAX;

for (PredInfo& info : matchedPredInfo.TopDownOrder())
for (auto it = matchesBegin; it < matchesEnd; it++)
{
const PredInfo& info = *it;
Statement* const stmt = info.m_stmt;
BasicBlock* const predBlock = info.m_block;

Expand Down Expand Up @@ -5357,8 +5347,9 @@ PhaseStatus Compiler::fgHeadTailMerge(bool early)

// Do the cross jumping
//
for (PredInfo& info : matchedPredInfo.TopDownOrder())
for (auto it = matchesBegin; it < matchesEnd; it++)
{
const PredInfo& info = *it;
BasicBlock* const predBlock = info.m_block;
Statement* const stmt = info.m_stmt;

Expand All @@ -5367,7 +5358,6 @@ PhaseStatus Compiler::fgHeadTailMerge(bool early)
continue;
}

// remove the statement
fgUnlinkStmt(predBlock, stmt);

// Fix up the flow.
Expand Down Expand Up @@ -5395,24 +5385,14 @@ PhaseStatus Compiler::fgHeadTailMerge(bool early)
}
}

// We changed things
//
madeChanges = true;

// We should try tail merging the cross jump target.
//
retryBlocks.Push(crossJumpTarget);

// Continue trying to merge in the current block.
// This is a bit inefficient, we could remember how
// far we got through the pred list perhaps.
//
return true;
if (hasCommSucc)
{
retryBlocks.Push(crossJumpTarget);
}
}

// We've looked at everything.
//
return false;
return optimizedCount;
};

auto tailMerge = [&](BasicBlock* block) -> bool {
Expand All @@ -5422,25 +5402,17 @@ PhaseStatus Compiler::fgHeadTailMerge(bool early)
return false;
}

predInfo.Reset();

// Find the subset of preds that reach along non-critical edges
// and populate predInfo.
//
predInfo.clear();
for (BasicBlock* const predBlock : block->PredBlocks())
{
if (predBlock->GetUniqueSucc() != block)
{
continue;
}

// If this block was already processed, skip it
//
if (predBlock->isEmpty())
{
continue;
}

Statement* lastStmt = predBlock->lastStmt();

// Block might be empty.
Expand Down Expand Up @@ -5476,41 +5448,32 @@ PhaseStatus Compiler::fgHeadTailMerge(bool early)
// We don't expect to see PHIs but watch for them anyways.
//
assert(!lastStmt->IsPhiDefnStmt());
predInfo.Emplace(predBlock, lastStmt);
}

return tailMergePreds(block);
};

auto iterateTailMerge = [&](BasicBlock* block) -> void {
int numOpts = 0;

while (tailMerge(block))
{
numOpts++;
predInfo.push_back(PredInfo{predBlock, lastStmt});
}

int numOpts = tailMergePreds(block);
if (numOpts > 0)
{
JITDUMP("Did %d tail merges in " FMT_BB "\n", numOpts, block->bbNum);
}
};

ArrayStack<BasicBlock*> retOrThrowBlocks(getAllocator(CMK_ArrayStack));
return numOpts;
};

// Visit each block
// Deduplicate RETURN/THROW blocks.
// This can enable tail-merging so do it first.
//
predInfo.clear();
for (BasicBlock* const block : Blocks())
{
iterateTailMerge(block);
if (block->isEmpty())
{
continue;
}

if (block->KindIs(BBJ_THROW))
{
retOrThrowBlocks.Push(block);
predInfo.push_back(PredInfo{block, block->lastStmt()});
}
else if (block->KindIs(BBJ_RETURN) && (block != genReturnBB))
{
Expand All @@ -5527,31 +5490,27 @@ PhaseStatus Compiler::fgHeadTailMerge(bool early)
}
}

retOrThrowBlocks.Push(block);
predInfo.push_back(PredInfo{block, block->lastStmt()});
}
}

JITDUMP("Trying tail merge of return and throw blocks\n");
do
int numOpts = tailMergePreds(nullptr);
if (numOpts > 0)
{
predInfo.Reset();
for (BasicBlock* const block : retOrThrowBlocks.BottomUpOrder())
{
// If this block was already processed, skip it
//
if (!block->KindIs(BBJ_RETURN, BBJ_THROW) || block->isEmpty())
{
continue;
}
predInfo.Push(PredInfo(block, block->lastStmt()));
}
} while (tailMergePreds(nullptr));
JITDUMP("Deduplicated %d sets of return/throw blocks\n", numOpts);
}

// Work through any retries
// Visit each block
//
while (retryBlocks.Height() > 0)
for (BasicBlock* const block : Blocks())
{
iterateTailMerge(retryBlocks.Pop());
tailMerge(block);

// Work through any retries
//
while (retryBlocks.Height() > 0)
{
tailMerge(retryBlocks.Pop());
}
}

// Visit each block and try to merge first statements of successors.
Expand Down
Loading
Loading