[Analysis] Use CycleInfo for BlockFrequencyInfo - #213488
Merged
aengelke merged 4 commits intoAug 5, 2026
Merged
Conversation
Created using spr 1.3.8-wip
llvmorg-github-actions
Bot
added
backend:ARM
backend:AArch64
backend:AMDGPU
backend:RISC-V
backend:PowerPC
backend:WebAssembly
backend:X86
llvm:codegen
llvm:globalisel
backend:m68k
PGO
|
@llvm/pr-subscribers-pgo @llvm/pr-subscribers-backend-powerpc Author: Alexis Engelke (aengelke) ChangesBranchProbabilityAnalysis uses CycleInfo, but BFI doesn't, causing the This requires a minor change to the BFI implementation to avoid Passing the parent down also fixes the loop nest: &Loops.back() is whichever Co-authored-by: Fangrui Song <i@maskray.me> Patch is 93.87 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/213488.diff 45 Files Affected:
diff --git a/llvm/include/llvm/Analysis/BlockFrequencyInfo.h b/llvm/include/llvm/Analysis/BlockFrequencyInfo.h
index 450810d360680..c5120eca584f5 100644
--- a/llvm/include/llvm/Analysis/BlockFrequencyInfo.h
+++ b/llvm/include/llvm/Analysis/BlockFrequencyInfo.h
@@ -26,7 +26,7 @@ namespace llvm {
class BasicBlock;
class BranchProbabilityInfo;
-class LoopInfo;
+class CycleInfo;
class Module;
class raw_ostream;
template <class BlockT> class BlockFrequencyInfoImpl;
@@ -44,7 +44,7 @@ class BlockFrequencyInfo {
LLVM_ABI BlockFrequencyInfo();
LLVM_ABI BlockFrequencyInfo(const Function &F,
const BranchProbabilityInfo &BPI,
- const LoopInfo &LI);
+ const CycleInfo &CI);
BlockFrequencyInfo(const BlockFrequencyInfo &) = delete;
BlockFrequencyInfo &operator=(const BlockFrequencyInfo &) = delete;
LLVM_ABI BlockFrequencyInfo(BlockFrequencyInfo &&Arg);
@@ -94,7 +94,7 @@ class BlockFrequencyInfo {
/// calculate - compute block frequency info for the given function.
LLVM_ABI void calculate(const Function &F, const BranchProbabilityInfo &BPI,
- const LoopInfo &LI);
+ const CycleInfo &CI);
LLVM_ABI BlockFrequency getEntryFreq() const;
LLVM_ABI void releaseMemory();
diff --git a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
index ced61b26f710f..b88200de216e0 100644
--- a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
+++ b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
@@ -17,6 +17,7 @@
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/GenericCycleInfo.h"
#include "llvm/ADT/GraphTraits.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/SmallPtrSet.h"
@@ -59,14 +60,12 @@ extern LLVM_ABI llvm::cl::opt<unsigned> IterativeBFIMaxIterationsPerBlock;
extern LLVM_ABI llvm::cl::opt<double> IterativeBFIPrecision;
class BranchProbabilityInfo;
+class CycleInfo;
class Function;
-class Loop;
-class LoopInfo;
class MachineBasicBlock;
class MachineBranchProbabilityInfo;
+class MachineCycleInfo;
class MachineFunction;
-class MachineLoop;
-class MachineLoopInfo;
namespace bfi_detail {
@@ -235,6 +234,13 @@ class LLVM_ABI BlockFrequencyInfoImplBase {
LoopData(LoopData *Parent, const BlockNode &Header)
: Parent(Parent), Nodes(1, Header), BackedgeMass(1) {}
+ template <class It>
+ LoopData(LoopData *Parent, It FirstHeader, It LastHeader)
+ : Parent(Parent), Nodes(FirstHeader, LastHeader) {
+ NumHeaders = Nodes.size();
+ BackedgeMass.resize(NumHeaders);
+ }
+
template <class It1, class It2>
LoopData(LoopData *Parent, It1 FirstHeader, It1 LastHeader, It2 FirstOther,
It2 LastOther)
@@ -536,15 +542,13 @@ template <> struct TypeMap<BasicBlock> {
using BlockT = BasicBlock;
using FunctionT = Function;
using BranchProbabilityInfoT = BranchProbabilityInfo;
- using LoopT = Loop;
- using LoopInfoT = LoopInfo;
+ using CycleInfoT = CycleInfo;
};
template <> struct TypeMap<MachineBasicBlock> {
using BlockT = MachineBasicBlock;
using FunctionT = MachineFunction;
using BranchProbabilityInfoT = MachineBranchProbabilityInfo;
- using LoopT = MachineLoop;
- using LoopInfoT = MachineLoopInfo;
+ using CycleInfoT = MachineCycleInfo;
};
/// Get the name of a MachineBasicBlock.
@@ -832,13 +836,12 @@ template <class BT> class BlockFrequencyInfoImpl : BlockFrequencyInfoImplBase {
using FunctionT = typename bfi_detail::TypeMap<BT>::FunctionT;
using BranchProbabilityInfoT =
typename bfi_detail::TypeMap<BT>::BranchProbabilityInfoT;
- using LoopT = typename bfi_detail::TypeMap<BT>::LoopT;
- using LoopInfoT = typename bfi_detail::TypeMap<BT>::LoopInfoT;
+ using CycleInfoT = typename bfi_detail::TypeMap<BT>::CycleInfoT;
using Successor = GraphTraits<const BlockT *>;
using Predecessor = GraphTraits<Inverse<const BlockT *>>;
const BranchProbabilityInfoT *BPI = nullptr;
- const LoopInfoT *LI = nullptr;
+ const CycleInfoT *CI = nullptr;
const FunctionT *F = nullptr;
// All blocks in reverse postorder.
@@ -984,7 +987,7 @@ template <class BT> class BlockFrequencyInfoImpl : BlockFrequencyInfoImplBase {
const FunctionT *getFunction() const { return F; }
void calculate(const FunctionT &F, const BranchProbabilityInfoT &BPI,
- const LoopInfoT &LI);
+ const CycleInfoT &CI);
using BlockFrequencyInfoImplBase::getEntryFreq;
@@ -1035,10 +1038,10 @@ template <class BT> class BlockFrequencyInfoImpl : BlockFrequencyInfoImplBase {
template <class BT>
void BlockFrequencyInfoImpl<BT>::calculate(const FunctionT &F,
const BranchProbabilityInfoT &BPI,
- const LoopInfoT &LI) {
+ const CycleInfoT &CI) {
// Save the parameters.
this->BPI = &BPI;
- this->LI = &LI;
+ this->CI = &CI;
this->F = &F;
// Clean up left-over data structures.
@@ -1121,27 +1124,45 @@ template <class BT> void BlockFrequencyInfoImpl<BT>::initializeRPOT() {
template <class BT> void BlockFrequencyInfoImpl<BT>::initializeLoops() {
LLVM_DEBUG(dbgs() << "loop-detection\n");
- if (LI->empty())
- return;
+
+ LLVM_DEBUG(CI->print(dbgs()));
+
+ // Whether \p C describes a loop for BFI. An entry of a cycle an edge
+ // re-enters heads a loop the forest does not represent, because the cycle
+ // absorbed it; which entry that is depends on the order the search found
+ // them in. Represent none of them, so that equal entries stay equal, and
+ // leave the region to the packaging computeIrreducibleMass does.
+ auto hasLoop = [&](CycleRef C) {
+ if (!CI->isReducible(C))
+ return false;
+ for (CycleRef A = CI->getParentCycle(C); A; A = CI->getParentCycle(A))
+ if (!CI->isReducible(A) && CI->isEntry(A, CI->getHeader(C)))
+ return false;
+ return true;
+ };
// Visit loops top down and assign them an index.
- std::deque<std::pair<const LoopT *, LoopData *>> Q;
- for (const LoopT *L : *LI)
- Q.emplace_back(L, nullptr);
+ std::deque<std::pair<CycleRef, LoopData *>> Q;
+ for (CycleRef C : CI->toplevel_cycles())
+ Q.emplace_back(C, nullptr);
+ if (Q.empty())
+ return; // Early exit if there are no cycles.
while (!Q.empty()) {
- const LoopT *Loop = Q.front().first;
+ CycleRef Cycle = Q.front().first;
LoopData *Parent = Q.front().second;
Q.pop_front();
- BlockNode Header = getNode(Loop->getHeader());
- assert(Header.isValid());
+ if (hasLoop(Cycle)) {
+ BlockNode Header = getNode(CI->getHeader(Cycle));
+ Loops.emplace_back(Parent, Header);
- Loops.emplace_back(Parent, Header);
- Working[Header.Index].Loop = &Loops.back();
- LLVM_DEBUG(dbgs() << " - loop = " << getBlockName(Header) << "\n");
+ Working[Header.Index].Loop = &Loops.back();
+ LLVM_DEBUG(dbgs() << " - loop = " << getBlockName(Header) << "\n");
+ Parent = &Loops.back();
+ }
- for (const LoopT *L : *Loop)
- Q.emplace_back(L, &Loops.back());
+ for (CycleRef C : CI->children(Cycle))
+ Q.emplace_back(C, Parent);
}
// Visit nodes in reverse post-order and add them to their deepest containing
@@ -1155,12 +1176,14 @@ template <class BT> void BlockFrequencyInfoImpl<BT>::initializeLoops() {
continue;
}
- const LoopT *Loop = LI->getLoopFor(RPOT[Index]);
- if (!Loop)
+ CycleRef Cycle = CI->getCycle(RPOT[Index]);
+ while (Cycle && !hasLoop(Cycle))
+ Cycle = CI->getParentCycle(Cycle);
+ if (!Cycle)
continue;
// Add this node to its containing loop's member list.
- BlockNode Header = getNode(Loop->getHeader());
+ BlockNode Header = getNode(CI->getHeader(Cycle));
assert(Header.isValid());
const auto &HeaderData = Working[Header.Index];
assert(HeaderData.isLoopHeader());
diff --git a/llvm/include/llvm/Analysis/LazyBlockFrequencyInfo.h b/llvm/include/llvm/Analysis/LazyBlockFrequencyInfo.h
index 494658178bf1b..e9981890bc804 100644
--- a/llvm/include/llvm/Analysis/LazyBlockFrequencyInfo.h
+++ b/llvm/include/llvm/Analysis/LazyBlockFrequencyInfo.h
@@ -22,32 +22,32 @@
namespace llvm {
class Function;
-class LoopInfo;
+class CycleInfo;
/// Wraps a BFI to allow lazy computation of the block frequencies.
///
/// A pass that only conditionally uses BFI can uncondtionally require the
/// analysis without paying for the overhead if BFI doesn't end up being used.
template <typename FunctionT, typename BranchProbabilityInfoPassT,
- typename LoopInfoT, typename BlockFrequencyInfoT>
+ typename CycleInfoT, typename BlockFrequencyInfoT>
class LazyBlockFrequencyInfo {
public:
LazyBlockFrequencyInfo() = default;
/// Set up the per-function input.
void setAnalysis(const FunctionT *F, BranchProbabilityInfoPassT *BPIPass,
- const LoopInfoT *LI) {
+ const CycleInfoT *CI) {
this->F = F;
this->BPIPass = BPIPass;
- this->LI = LI;
+ this->CI = CI;
}
/// Retrieve the BFI with the block frequencies computed.
BlockFrequencyInfoT &getCalculated() {
if (!Calculated) {
- assert(F && BPIPass && LI && "call setAnalysis");
+ assert(F && BPIPass && CI && "call setAnalysis");
BFI.calculate(
- *F, BPIPassTrait<BranchProbabilityInfoPassT>::getBPI(BPIPass), *LI);
+ *F, BPIPassTrait<BranchProbabilityInfoPassT>::getBPI(BPIPass), *CI);
Calculated = true;
}
return BFI;
@@ -68,7 +68,7 @@ class LazyBlockFrequencyInfo {
bool Calculated = false;
const FunctionT *F = nullptr;
BranchProbabilityInfoPassT *BPIPass = nullptr;
- const LoopInfoT *LI = nullptr;
+ const CycleInfoT *CI = nullptr;
};
/// This is an alternative analysis pass to
@@ -88,15 +88,15 @@ class LazyBlockFrequencyInfo {
/// LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage(AU)
///
/// 3. The computed BFI should be requested with
-/// getAnalysis<LazyBlockFrequencyInfoPass>().getBFI() before either LoopInfo
-/// or BPI could be invalidated for example by changing the CFG.
+/// getAnalysis<LazyBlockFrequencyInfoPass>().getBFI() before either
+/// CycleInfo or BPI could be invalidated for example by changing the CFG.
///
/// Note that it is expected that we wouldn't need this functionality for the
/// new PM since with the new PM, analyses are executed on demand.
class LLVM_ABI LazyBlockFrequencyInfoPass : public FunctionPass {
private:
- LazyBlockFrequencyInfo<Function, LazyBranchProbabilityInfoPass, LoopInfo,
+ LazyBlockFrequencyInfo<Function, LazyBranchProbabilityInfoPass, CycleInfo,
BlockFrequencyInfo>
LBFI;
diff --git a/llvm/include/llvm/CodeGen/LazyMachineBlockFrequencyInfo.h b/llvm/include/llvm/CodeGen/LazyMachineBlockFrequencyInfo.h
index 17679b4b3bab0..cde589a4a8bbe 100644
--- a/llvm/include/llvm/CodeGen/LazyMachineBlockFrequencyInfo.h
+++ b/llvm/include/llvm/CodeGen/LazyMachineBlockFrequencyInfo.h
@@ -17,11 +17,12 @@
#define LLVM_CODEGEN_LAZYMACHINEBLOCKFREQUENCYINFO_H
#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
-#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/CodeGen/MachineLoopInfo.h"
namespace llvm {
+
+class MachineCycleInfo;
+
/// This is an alternative analysis pass to MachineBlockFrequencyInfo.
/// The difference is that with this pass, the block frequencies are not
/// computed when the analysis pass is executed but rather when the BFI result
@@ -40,10 +41,7 @@ class LLVM_ABI LazyMachineBlockFrequencyInfoPass : public MachineFunctionPass {
mutable std::unique_ptr<MachineBlockFrequencyInfo> OwnedMBFI;
/// If generated on the fly this own the instance.
- mutable std::unique_ptr<MachineLoopInfo> OwnedMLI;
-
- /// If generated on the fly this own the instance.
- mutable std::unique_ptr<MachineDominatorTree> OwnedMDT;
+ mutable std::unique_ptr<MachineCycleInfo> OwnedMCI;
/// The function.
MachineFunction *MF = nullptr;
diff --git a/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h b/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
index e9c38d0189b2e..717297f8c0b19 100644
--- a/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
+++ b/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
@@ -27,7 +27,7 @@ template <class BlockT> class BlockFrequencyInfoImpl;
class MachineBasicBlock;
class MachineBranchProbabilityInfo;
class MachineFunction;
-class MachineLoopInfo;
+class MachineCycleInfo;
class raw_ostream;
/// MachineBlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation
@@ -40,7 +40,7 @@ class MachineBlockFrequencyInfo {
LLVM_ABI MachineBlockFrequencyInfo(); // Legacy pass manager only.
LLVM_ABI explicit MachineBlockFrequencyInfo(
const MachineFunction &F, const MachineBranchProbabilityInfo &MBPI,
- const MachineLoopInfo &MLI);
+ const MachineCycleInfo &MCI);
LLVM_ABI MachineBlockFrequencyInfo(MachineBlockFrequencyInfo &&);
LLVM_ABI ~MachineBlockFrequencyInfo();
@@ -51,7 +51,7 @@ class MachineBlockFrequencyInfo {
/// calculate - compute block frequency info for the given function.
LLVM_ABI void calculate(const MachineFunction &F,
const MachineBranchProbabilityInfo &MBPI,
- const MachineLoopInfo &MLI);
+ const MachineCycleInfo &MCI);
LLVM_ABI void print(raw_ostream &OS);
diff --git a/llvm/lib/Analysis/BlockFrequencyInfo.cpp b/llvm/lib/Analysis/BlockFrequencyInfo.cpp
index 9a2bd2c91d11a..5f8f073c1de8c 100644
--- a/llvm/lib/Analysis/BlockFrequencyInfo.cpp
+++ b/llvm/lib/Analysis/BlockFrequencyInfo.cpp
@@ -15,7 +15,7 @@
#include "llvm/ADT/iterator.h"
#include "llvm/Analysis/BlockFrequencyInfoImpl.h"
#include "llvm/Analysis/BranchProbabilityInfo.h"
-#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/CycleAnalysis.h"
#include "llvm/IR/CFG.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/PassManager.h"
@@ -153,8 +153,8 @@ BlockFrequencyInfo::BlockFrequencyInfo() = default;
BlockFrequencyInfo::BlockFrequencyInfo(const Function &F,
const BranchProbabilityInfo &BPI,
- const LoopInfo &LI) {
- calculate(F, BPI, LI);
+ const CycleInfo &CI) {
+ calculate(F, BPI, CI);
}
BlockFrequencyInfo::BlockFrequencyInfo(BlockFrequencyInfo &&Arg)
@@ -183,10 +183,10 @@ bool BlockFrequencyInfo::invalidate(Function &F, const PreservedAnalyses &PA,
void BlockFrequencyInfo::calculate(const Function &F,
const BranchProbabilityInfo &BPI,
- const LoopInfo &LI) {
+ const CycleInfo &CI) {
if (!BFI)
BFI.reset(new ImplType);
- BFI->calculate(F, BPI, LI);
+ BFI->calculate(F, BPI, CI);
if (ViewBlockFreqPropagationDAG != GVDT_None &&
(ViewBlockFreqFuncName.empty() || F.getName() == ViewBlockFreqFuncName)) {
view();
@@ -295,7 +295,7 @@ Printable llvm::printBlockFreq(const BlockFrequencyInfo &BFI,
INITIALIZE_PASS_BEGIN(BlockFrequencyInfoWrapperPass, "block-freq",
"Block Frequency Analysis", true, true)
INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass)
-INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(CycleInfoWrapperPass)
INITIALIZE_PASS_END(BlockFrequencyInfoWrapperPass, "block-freq",
"Block Frequency Analysis", true, true)
@@ -313,7 +313,7 @@ void BlockFrequencyInfoWrapperPass::print(raw_ostream &OS,
void BlockFrequencyInfoWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
AU.addRequired<BranchProbabilityInfoWrapperPass>();
- AU.addRequired<LoopInfoWrapperPass>();
+ AU.addRequired<CycleInfoWrapperPass>();
AU.setPreservesAll();
}
@@ -322,7 +322,7 @@ void BlockFrequencyInfoWrapperPass::releaseMemory() { BFI.releaseMemory(); }
bool BlockFrequencyInfoWrapperPass::runOnFunction(Function &F) {
BranchProbabilityInfo &BPI =
getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
- LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
+ CycleInfo &LI = getAnalysis<CycleInfoWrapperPass>().getResult();
BFI.calculate(F, BPI, LI);
return false;
}
@@ -331,7 +331,7 @@ AnalysisKey BlockFrequencyAnalysis::Key;
BlockFrequencyInfo BlockFrequencyAnalysis::run(Function &F,
FunctionAnalysisManager &AM) {
auto &BP = AM.getResult<BranchProbabilityAnalysis>(F);
- auto &LI = AM.getResult<LoopAnalysis>(F);
+ auto &LI = AM.getResult<CycleAnalysis>(F);
BlockFrequencyInfo BFI;
BFI.calculate(F, BP, LI);
return BFI;
diff --git a/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp b/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp
index 011435db66783..6449c31a8b9a6 100644
--- a/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp
+++ b/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp
@@ -329,6 +329,7 @@ bool BlockFrequencyInfoImplBase::addToDist(Distribution &Dist,
<< " [" << Type << "] weight = " << Weight;
if (!isLoopHeader(Resolved))
dbgs() << ", succ = " << getBlockName(Succ);
+ dbgs() << ", pred = " << getBlockName(Pred);
if (Resolved != Succ)
dbgs() << ", resolved = " << getBlockName(Resolved);
dbgs() << "\n";
@@ -350,12 +351,12 @@ bool BlockFrequencyInfoImplBase::addToDist(Distribution &Dist,
if (Resolved < Pred) {
if (!isLoopHeader(Pred)) {
- // If OuterLoop is an irreducible loop, we can't actually handle this.
- assert((!OuterLoop || !OuterLoop->isIrreducible()) &&
- "unhandled irreducible control flow");
// Irreducible backedge. Abort.
LLVM_DEBUG(debugSuccessor("abort!!!"));
+ // If OuterLoop is an irreducible loop, we can't actually handle this.
+ assert((!OuterLoop || !OuterLoop->isIrreducible()) &&
+ "unhandled irreducible control flow");
return false;
}
diff --git a/llvm/lib/Analysis/LazyBlockFrequencyInfo.cpp b/llvm/lib/Analysis/LazyBlockFrequencyInfo.cpp
index ac1072843fd7f..b470cb165c0ab 100644
--- a/llvm/lib/Analysis/LazyBlockFrequencyInfo.cpp
+++ b/llvm/lib/Analysis/LazyBlockFrequencyInfo.cpp
@@ -14,8 +14,8 @@
//===----------------------------------------------------------------------===//
#include "llvm/Analysis/LazyBlockFrequencyInfo.h"
+#include "llvm/Analysis/CycleAnalysis.h"
#include "llvm/Analysis/LazyBranchProbabilityInfo.h"
-#include "llvm/Analysis/LoopInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/InitializePasses.h"
@@ -26,7 +26,7 @@ using namespace llvm;
INITIALIZE_PASS_BEGIN(LazyBlockFrequencyInfoPass, DEBUG_TYPE,
"Lazy Block Frequency Analysis", true, true)
INITIALIZE_PASS_DEPENDENCY(LazyBPIPass)
-INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(CycleInfoWrapperPass)
INITIALIZE_PASS_END(LazyBlockFrequencyInfoPass, DEBUG_TYPE,
"Lazy Block Frequency Analysis", true, true)
@@ -44,7 +44,7 @@ void LazyBlockFrequencyInfoPass::getAnalysisUsage(AnalysisUsage &AU) const {
// asserts that DT is also present so if we don't make sure that we have DT
// here, that assert will trigger.
AU.addRequiredTransitive<DominatorTreeWrapperPass>();
- AU.addRequiredTransitive<LoopInfoWrapperPass>();
+ AU.addRequiredTransitive<CycleInfoWrapperPass>();
AU.setPreservesAll();
}
@@ -52,7 +52,7 @@ void LazyBlockFrequencyInfoPass::releaseMemory() { LBFI.releaseMemory(); }
bool LazyBlockFrequencyInfoPass::runOnFunction(Function &F) {
auto &BPIPass = getAnalysis<LazyBranchProbabilityInfoPass>();
- LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
+ CycleInfo &LI = getAnalysis<CycleInfoWrapperPass>().getResult();
LBFI.setAnalysis(&F, &BPIPass, &LI);
return false;
}
@@ -60,11 +60,11 @@ bool LazyBlockFrequencyInfoPass::runOnFunction(Function &F) {
void LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage(AnalysisUsage &AU) {
LazyBranchProbabilityInfoPass::getLazyBPIAnalysisUsage(AU);
AU.addRequiredTransitive<LazyBlockFrequencyInfoPass>();
- AU.addRequiredTransitive<LoopInfoWrap...
[truncated]
|
|
@llvm/pr-subscribers-backend-risc-v Author: Alexis Engelke (aengelke) ChangesBranchProbabilityAnalysis uses CycleInfo, but BFI doesn't, causing the This requires a minor change to the BFI implementation to avoid Passing the parent down also fixes the loop nest: &Loops.back() is whichever Co-authored-by: Fangrui Song <i@maskray.me> Patch is 93.87 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/213488.diff 45 Files Affected:
diff --git a/llvm/include/llvm/Analysis/BlockFrequencyInfo.h b/llvm/include/llvm/Analysis/BlockFrequencyInfo.h
index 450810d360680..c5120eca584f5 100644
--- a/llvm/include/llvm/Analysis/BlockFrequencyInfo.h
+++ b/llvm/include/llvm/Analysis/BlockFrequencyInfo.h
@@ -26,7 +26,7 @@ namespace llvm {
class BasicBlock;
class BranchProbabilityInfo;
-class LoopInfo;
+class CycleInfo;
class Module;
class raw_ostream;
template <class BlockT> class BlockFrequencyInfoImpl;
@@ -44,7 +44,7 @@ class BlockFrequencyInfo {
LLVM_ABI BlockFrequencyInfo();
LLVM_ABI BlockFrequencyInfo(const Function &F,
const BranchProbabilityInfo &BPI,
- const LoopInfo &LI);
+ const CycleInfo &CI);
BlockFrequencyInfo(const BlockFrequencyInfo &) = delete;
BlockFrequencyInfo &operator=(const BlockFrequencyInfo &) = delete;
LLVM_ABI BlockFrequencyInfo(BlockFrequencyInfo &&Arg);
@@ -94,7 +94,7 @@ class BlockFrequencyInfo {
/// calculate - compute block frequency info for the given function.
LLVM_ABI void calculate(const Function &F, const BranchProbabilityInfo &BPI,
- const LoopInfo &LI);
+ const CycleInfo &CI);
LLVM_ABI BlockFrequency getEntryFreq() const;
LLVM_ABI void releaseMemory();
diff --git a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
index ced61b26f710f..b88200de216e0 100644
--- a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
+++ b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
@@ -17,6 +17,7 @@
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/GenericCycleInfo.h"
#include "llvm/ADT/GraphTraits.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/SmallPtrSet.h"
@@ -59,14 +60,12 @@ extern LLVM_ABI llvm::cl::opt<unsigned> IterativeBFIMaxIterationsPerBlock;
extern LLVM_ABI llvm::cl::opt<double> IterativeBFIPrecision;
class BranchProbabilityInfo;
+class CycleInfo;
class Function;
-class Loop;
-class LoopInfo;
class MachineBasicBlock;
class MachineBranchProbabilityInfo;
+class MachineCycleInfo;
class MachineFunction;
-class MachineLoop;
-class MachineLoopInfo;
namespace bfi_detail {
@@ -235,6 +234,13 @@ class LLVM_ABI BlockFrequencyInfoImplBase {
LoopData(LoopData *Parent, const BlockNode &Header)
: Parent(Parent), Nodes(1, Header), BackedgeMass(1) {}
+ template <class It>
+ LoopData(LoopData *Parent, It FirstHeader, It LastHeader)
+ : Parent(Parent), Nodes(FirstHeader, LastHeader) {
+ NumHeaders = Nodes.size();
+ BackedgeMass.resize(NumHeaders);
+ }
+
template <class It1, class It2>
LoopData(LoopData *Parent, It1 FirstHeader, It1 LastHeader, It2 FirstOther,
It2 LastOther)
@@ -536,15 +542,13 @@ template <> struct TypeMap<BasicBlock> {
using BlockT = BasicBlock;
using FunctionT = Function;
using BranchProbabilityInfoT = BranchProbabilityInfo;
- using LoopT = Loop;
- using LoopInfoT = LoopInfo;
+ using CycleInfoT = CycleInfo;
};
template <> struct TypeMap<MachineBasicBlock> {
using BlockT = MachineBasicBlock;
using FunctionT = MachineFunction;
using BranchProbabilityInfoT = MachineBranchProbabilityInfo;
- using LoopT = MachineLoop;
- using LoopInfoT = MachineLoopInfo;
+ using CycleInfoT = MachineCycleInfo;
};
/// Get the name of a MachineBasicBlock.
@@ -832,13 +836,12 @@ template <class BT> class BlockFrequencyInfoImpl : BlockFrequencyInfoImplBase {
using FunctionT = typename bfi_detail::TypeMap<BT>::FunctionT;
using BranchProbabilityInfoT =
typename bfi_detail::TypeMap<BT>::BranchProbabilityInfoT;
- using LoopT = typename bfi_detail::TypeMap<BT>::LoopT;
- using LoopInfoT = typename bfi_detail::TypeMap<BT>::LoopInfoT;
+ using CycleInfoT = typename bfi_detail::TypeMap<BT>::CycleInfoT;
using Successor = GraphTraits<const BlockT *>;
using Predecessor = GraphTraits<Inverse<const BlockT *>>;
const BranchProbabilityInfoT *BPI = nullptr;
- const LoopInfoT *LI = nullptr;
+ const CycleInfoT *CI = nullptr;
const FunctionT *F = nullptr;
// All blocks in reverse postorder.
@@ -984,7 +987,7 @@ template <class BT> class BlockFrequencyInfoImpl : BlockFrequencyInfoImplBase {
const FunctionT *getFunction() const { return F; }
void calculate(const FunctionT &F, const BranchProbabilityInfoT &BPI,
- const LoopInfoT &LI);
+ const CycleInfoT &CI);
using BlockFrequencyInfoImplBase::getEntryFreq;
@@ -1035,10 +1038,10 @@ template <class BT> class BlockFrequencyInfoImpl : BlockFrequencyInfoImplBase {
template <class BT>
void BlockFrequencyInfoImpl<BT>::calculate(const FunctionT &F,
const BranchProbabilityInfoT &BPI,
- const LoopInfoT &LI) {
+ const CycleInfoT &CI) {
// Save the parameters.
this->BPI = &BPI;
- this->LI = &LI;
+ this->CI = &CI;
this->F = &F;
// Clean up left-over data structures.
@@ -1121,27 +1124,45 @@ template <class BT> void BlockFrequencyInfoImpl<BT>::initializeRPOT() {
template <class BT> void BlockFrequencyInfoImpl<BT>::initializeLoops() {
LLVM_DEBUG(dbgs() << "loop-detection\n");
- if (LI->empty())
- return;
+
+ LLVM_DEBUG(CI->print(dbgs()));
+
+ // Whether \p C describes a loop for BFI. An entry of a cycle an edge
+ // re-enters heads a loop the forest does not represent, because the cycle
+ // absorbed it; which entry that is depends on the order the search found
+ // them in. Represent none of them, so that equal entries stay equal, and
+ // leave the region to the packaging computeIrreducibleMass does.
+ auto hasLoop = [&](CycleRef C) {
+ if (!CI->isReducible(C))
+ return false;
+ for (CycleRef A = CI->getParentCycle(C); A; A = CI->getParentCycle(A))
+ if (!CI->isReducible(A) && CI->isEntry(A, CI->getHeader(C)))
+ return false;
+ return true;
+ };
// Visit loops top down and assign them an index.
- std::deque<std::pair<const LoopT *, LoopData *>> Q;
- for (const LoopT *L : *LI)
- Q.emplace_back(L, nullptr);
+ std::deque<std::pair<CycleRef, LoopData *>> Q;
+ for (CycleRef C : CI->toplevel_cycles())
+ Q.emplace_back(C, nullptr);
+ if (Q.empty())
+ return; // Early exit if there are no cycles.
while (!Q.empty()) {
- const LoopT *Loop = Q.front().first;
+ CycleRef Cycle = Q.front().first;
LoopData *Parent = Q.front().second;
Q.pop_front();
- BlockNode Header = getNode(Loop->getHeader());
- assert(Header.isValid());
+ if (hasLoop(Cycle)) {
+ BlockNode Header = getNode(CI->getHeader(Cycle));
+ Loops.emplace_back(Parent, Header);
- Loops.emplace_back(Parent, Header);
- Working[Header.Index].Loop = &Loops.back();
- LLVM_DEBUG(dbgs() << " - loop = " << getBlockName(Header) << "\n");
+ Working[Header.Index].Loop = &Loops.back();
+ LLVM_DEBUG(dbgs() << " - loop = " << getBlockName(Header) << "\n");
+ Parent = &Loops.back();
+ }
- for (const LoopT *L : *Loop)
- Q.emplace_back(L, &Loops.back());
+ for (CycleRef C : CI->children(Cycle))
+ Q.emplace_back(C, Parent);
}
// Visit nodes in reverse post-order and add them to their deepest containing
@@ -1155,12 +1176,14 @@ template <class BT> void BlockFrequencyInfoImpl<BT>::initializeLoops() {
continue;
}
- const LoopT *Loop = LI->getLoopFor(RPOT[Index]);
- if (!Loop)
+ CycleRef Cycle = CI->getCycle(RPOT[Index]);
+ while (Cycle && !hasLoop(Cycle))
+ Cycle = CI->getParentCycle(Cycle);
+ if (!Cycle)
continue;
// Add this node to its containing loop's member list.
- BlockNode Header = getNode(Loop->getHeader());
+ BlockNode Header = getNode(CI->getHeader(Cycle));
assert(Header.isValid());
const auto &HeaderData = Working[Header.Index];
assert(HeaderData.isLoopHeader());
diff --git a/llvm/include/llvm/Analysis/LazyBlockFrequencyInfo.h b/llvm/include/llvm/Analysis/LazyBlockFrequencyInfo.h
index 494658178bf1b..e9981890bc804 100644
--- a/llvm/include/llvm/Analysis/LazyBlockFrequencyInfo.h
+++ b/llvm/include/llvm/Analysis/LazyBlockFrequencyInfo.h
@@ -22,32 +22,32 @@
namespace llvm {
class Function;
-class LoopInfo;
+class CycleInfo;
/// Wraps a BFI to allow lazy computation of the block frequencies.
///
/// A pass that only conditionally uses BFI can uncondtionally require the
/// analysis without paying for the overhead if BFI doesn't end up being used.
template <typename FunctionT, typename BranchProbabilityInfoPassT,
- typename LoopInfoT, typename BlockFrequencyInfoT>
+ typename CycleInfoT, typename BlockFrequencyInfoT>
class LazyBlockFrequencyInfo {
public:
LazyBlockFrequencyInfo() = default;
/// Set up the per-function input.
void setAnalysis(const FunctionT *F, BranchProbabilityInfoPassT *BPIPass,
- const LoopInfoT *LI) {
+ const CycleInfoT *CI) {
this->F = F;
this->BPIPass = BPIPass;
- this->LI = LI;
+ this->CI = CI;
}
/// Retrieve the BFI with the block frequencies computed.
BlockFrequencyInfoT &getCalculated() {
if (!Calculated) {
- assert(F && BPIPass && LI && "call setAnalysis");
+ assert(F && BPIPass && CI && "call setAnalysis");
BFI.calculate(
- *F, BPIPassTrait<BranchProbabilityInfoPassT>::getBPI(BPIPass), *LI);
+ *F, BPIPassTrait<BranchProbabilityInfoPassT>::getBPI(BPIPass), *CI);
Calculated = true;
}
return BFI;
@@ -68,7 +68,7 @@ class LazyBlockFrequencyInfo {
bool Calculated = false;
const FunctionT *F = nullptr;
BranchProbabilityInfoPassT *BPIPass = nullptr;
- const LoopInfoT *LI = nullptr;
+ const CycleInfoT *CI = nullptr;
};
/// This is an alternative analysis pass to
@@ -88,15 +88,15 @@ class LazyBlockFrequencyInfo {
/// LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage(AU)
///
/// 3. The computed BFI should be requested with
-/// getAnalysis<LazyBlockFrequencyInfoPass>().getBFI() before either LoopInfo
-/// or BPI could be invalidated for example by changing the CFG.
+/// getAnalysis<LazyBlockFrequencyInfoPass>().getBFI() before either
+/// CycleInfo or BPI could be invalidated for example by changing the CFG.
///
/// Note that it is expected that we wouldn't need this functionality for the
/// new PM since with the new PM, analyses are executed on demand.
class LLVM_ABI LazyBlockFrequencyInfoPass : public FunctionPass {
private:
- LazyBlockFrequencyInfo<Function, LazyBranchProbabilityInfoPass, LoopInfo,
+ LazyBlockFrequencyInfo<Function, LazyBranchProbabilityInfoPass, CycleInfo,
BlockFrequencyInfo>
LBFI;
diff --git a/llvm/include/llvm/CodeGen/LazyMachineBlockFrequencyInfo.h b/llvm/include/llvm/CodeGen/LazyMachineBlockFrequencyInfo.h
index 17679b4b3bab0..cde589a4a8bbe 100644
--- a/llvm/include/llvm/CodeGen/LazyMachineBlockFrequencyInfo.h
+++ b/llvm/include/llvm/CodeGen/LazyMachineBlockFrequencyInfo.h
@@ -17,11 +17,12 @@
#define LLVM_CODEGEN_LAZYMACHINEBLOCKFREQUENCYINFO_H
#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
-#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/CodeGen/MachineLoopInfo.h"
namespace llvm {
+
+class MachineCycleInfo;
+
/// This is an alternative analysis pass to MachineBlockFrequencyInfo.
/// The difference is that with this pass, the block frequencies are not
/// computed when the analysis pass is executed but rather when the BFI result
@@ -40,10 +41,7 @@ class LLVM_ABI LazyMachineBlockFrequencyInfoPass : public MachineFunctionPass {
mutable std::unique_ptr<MachineBlockFrequencyInfo> OwnedMBFI;
/// If generated on the fly this own the instance.
- mutable std::unique_ptr<MachineLoopInfo> OwnedMLI;
-
- /// If generated on the fly this own the instance.
- mutable std::unique_ptr<MachineDominatorTree> OwnedMDT;
+ mutable std::unique_ptr<MachineCycleInfo> OwnedMCI;
/// The function.
MachineFunction *MF = nullptr;
diff --git a/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h b/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
index e9c38d0189b2e..717297f8c0b19 100644
--- a/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
+++ b/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
@@ -27,7 +27,7 @@ template <class BlockT> class BlockFrequencyInfoImpl;
class MachineBasicBlock;
class MachineBranchProbabilityInfo;
class MachineFunction;
-class MachineLoopInfo;
+class MachineCycleInfo;
class raw_ostream;
/// MachineBlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation
@@ -40,7 +40,7 @@ class MachineBlockFrequencyInfo {
LLVM_ABI MachineBlockFrequencyInfo(); // Legacy pass manager only.
LLVM_ABI explicit MachineBlockFrequencyInfo(
const MachineFunction &F, const MachineBranchProbabilityInfo &MBPI,
- const MachineLoopInfo &MLI);
+ const MachineCycleInfo &MCI);
LLVM_ABI MachineBlockFrequencyInfo(MachineBlockFrequencyInfo &&);
LLVM_ABI ~MachineBlockFrequencyInfo();
@@ -51,7 +51,7 @@ class MachineBlockFrequencyInfo {
/// calculate - compute block frequency info for the given function.
LLVM_ABI void calculate(const MachineFunction &F,
const MachineBranchProbabilityInfo &MBPI,
- const MachineLoopInfo &MLI);
+ const MachineCycleInfo &MCI);
LLVM_ABI void print(raw_ostream &OS);
diff --git a/llvm/lib/Analysis/BlockFrequencyInfo.cpp b/llvm/lib/Analysis/BlockFrequencyInfo.cpp
index 9a2bd2c91d11a..5f8f073c1de8c 100644
--- a/llvm/lib/Analysis/BlockFrequencyInfo.cpp
+++ b/llvm/lib/Analysis/BlockFrequencyInfo.cpp
@@ -15,7 +15,7 @@
#include "llvm/ADT/iterator.h"
#include "llvm/Analysis/BlockFrequencyInfoImpl.h"
#include "llvm/Analysis/BranchProbabilityInfo.h"
-#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/CycleAnalysis.h"
#include "llvm/IR/CFG.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/PassManager.h"
@@ -153,8 +153,8 @@ BlockFrequencyInfo::BlockFrequencyInfo() = default;
BlockFrequencyInfo::BlockFrequencyInfo(const Function &F,
const BranchProbabilityInfo &BPI,
- const LoopInfo &LI) {
- calculate(F, BPI, LI);
+ const CycleInfo &CI) {
+ calculate(F, BPI, CI);
}
BlockFrequencyInfo::BlockFrequencyInfo(BlockFrequencyInfo &&Arg)
@@ -183,10 +183,10 @@ bool BlockFrequencyInfo::invalidate(Function &F, const PreservedAnalyses &PA,
void BlockFrequencyInfo::calculate(const Function &F,
const BranchProbabilityInfo &BPI,
- const LoopInfo &LI) {
+ const CycleInfo &CI) {
if (!BFI)
BFI.reset(new ImplType);
- BFI->calculate(F, BPI, LI);
+ BFI->calculate(F, BPI, CI);
if (ViewBlockFreqPropagationDAG != GVDT_None &&
(ViewBlockFreqFuncName.empty() || F.getName() == ViewBlockFreqFuncName)) {
view();
@@ -295,7 +295,7 @@ Printable llvm::printBlockFreq(const BlockFrequencyInfo &BFI,
INITIALIZE_PASS_BEGIN(BlockFrequencyInfoWrapperPass, "block-freq",
"Block Frequency Analysis", true, true)
INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass)
-INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(CycleInfoWrapperPass)
INITIALIZE_PASS_END(BlockFrequencyInfoWrapperPass, "block-freq",
"Block Frequency Analysis", true, true)
@@ -313,7 +313,7 @@ void BlockFrequencyInfoWrapperPass::print(raw_ostream &OS,
void BlockFrequencyInfoWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
AU.addRequired<BranchProbabilityInfoWrapperPass>();
- AU.addRequired<LoopInfoWrapperPass>();
+ AU.addRequired<CycleInfoWrapperPass>();
AU.setPreservesAll();
}
@@ -322,7 +322,7 @@ void BlockFrequencyInfoWrapperPass::releaseMemory() { BFI.releaseMemory(); }
bool BlockFrequencyInfoWrapperPass::runOnFunction(Function &F) {
BranchProbabilityInfo &BPI =
getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
- LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
+ CycleInfo &LI = getAnalysis<CycleInfoWrapperPass>().getResult();
BFI.calculate(F, BPI, LI);
return false;
}
@@ -331,7 +331,7 @@ AnalysisKey BlockFrequencyAnalysis::Key;
BlockFrequencyInfo BlockFrequencyAnalysis::run(Function &F,
FunctionAnalysisManager &AM) {
auto &BP = AM.getResult<BranchProbabilityAnalysis>(F);
- auto &LI = AM.getResult<LoopAnalysis>(F);
+ auto &LI = AM.getResult<CycleAnalysis>(F);
BlockFrequencyInfo BFI;
BFI.calculate(F, BP, LI);
return BFI;
diff --git a/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp b/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp
index 011435db66783..6449c31a8b9a6 100644
--- a/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp
+++ b/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp
@@ -329,6 +329,7 @@ bool BlockFrequencyInfoImplBase::addToDist(Distribution &Dist,
<< " [" << Type << "] weight = " << Weight;
if (!isLoopHeader(Resolved))
dbgs() << ", succ = " << getBlockName(Succ);
+ dbgs() << ", pred = " << getBlockName(Pred);
if (Resolved != Succ)
dbgs() << ", resolved = " << getBlockName(Resolved);
dbgs() << "\n";
@@ -350,12 +351,12 @@ bool BlockFrequencyInfoImplBase::addToDist(Distribution &Dist,
if (Resolved < Pred) {
if (!isLoopHeader(Pred)) {
- // If OuterLoop is an irreducible loop, we can't actually handle this.
- assert((!OuterLoop || !OuterLoop->isIrreducible()) &&
- "unhandled irreducible control flow");
// Irreducible backedge. Abort.
LLVM_DEBUG(debugSuccessor("abort!!!"));
+ // If OuterLoop is an irreducible loop, we can't actually handle this.
+ assert((!OuterLoop || !OuterLoop->isIrreducible()) &&
+ "unhandled irreducible control flow");
return false;
}
diff --git a/llvm/lib/Analysis/LazyBlockFrequencyInfo.cpp b/llvm/lib/Analysis/LazyBlockFrequencyInfo.cpp
index ac1072843fd7f..b470cb165c0ab 100644
--- a/llvm/lib/Analysis/LazyBlockFrequencyInfo.cpp
+++ b/llvm/lib/Analysis/LazyBlockFrequencyInfo.cpp
@@ -14,8 +14,8 @@
//===----------------------------------------------------------------------===//
#include "llvm/Analysis/LazyBlockFrequencyInfo.h"
+#include "llvm/Analysis/CycleAnalysis.h"
#include "llvm/Analysis/LazyBranchProbabilityInfo.h"
-#include "llvm/Analysis/LoopInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/InitializePasses.h"
@@ -26,7 +26,7 @@ using namespace llvm;
INITIALIZE_PASS_BEGIN(LazyBlockFrequencyInfoPass, DEBUG_TYPE,
"Lazy Block Frequency Analysis", true, true)
INITIALIZE_PASS_DEPENDENCY(LazyBPIPass)
-INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(CycleInfoWrapperPass)
INITIALIZE_PASS_END(LazyBlockFrequencyInfoPass, DEBUG_TYPE,
"Lazy Block Frequency Analysis", true, true)
@@ -44,7 +44,7 @@ void LazyBlockFrequencyInfoPass::getAnalysisUsage(AnalysisUsage &AU) const {
// asserts that DT is also present so if we don't make sure that we have DT
// here, that assert will trigger.
AU.addRequiredTransitive<DominatorTreeWrapperPass>();
- AU.addRequiredTransitive<LoopInfoWrapperPass>();
+ AU.addRequiredTransitive<CycleInfoWrapperPass>();
AU.setPreservesAll();
}
@@ -52,7 +52,7 @@ void LazyBlockFrequencyInfoPass::releaseMemory() { LBFI.releaseMemory(); }
bool LazyBlockFrequencyInfoPass::runOnFunction(Function &F) {
auto &BPIPass = getAnalysis<LazyBranchProbabilityInfoPass>();
- LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
+ CycleInfo &LI = getAnalysis<CycleInfoWrapperPass>().getResult();
LBFI.setAnalysis(&F, &BPIPass, &LI);
return false;
}
@@ -60,11 +60,11 @@ bool LazyBlockFrequencyInfoPass::runOnFunction(Function &F) {
void LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage(AnalysisUsage &AU) {
LazyBranchProbabilityInfoPass::getLazyBPIAnalysisUsage(AU);
AU.addRequiredTransitive<LazyBlockFrequencyInfoPass>();
- AU.addRequiredTransitive<LoopInfoWrap...
[truncated]
|
Created using spr 1.3.8-wip
Contributor
Author
|
Edit: doesn't seem necessary. The win from not doing the two extra cycle info computations is marginal. |
Member
|
I've been turning this over in my mind for the past few days. ContextBlockFrequencyInfo is based on the Wu-Larus algorithm, which only handles reducible loops. @dexonsmith added irreducible loop support in 2014 (384d0e8). It remains single-pass and does not solve for the Markov https://reviews.llvm.org/D103289 (2021) added 20 of the 21 touched tests are pass-plumbing.
Against an exact rational solve (Gauss elimination, high time complexity) over the irreducible functions in The main branch baseline is weak. ... Future simplificationsThis PR should enable future simplification. We may be able to remove CycleInfoMaking LoopInfo represent irreducible cycles does not work as a direction: The workable convergence is probably CycleInfo as the base with LoopInfo a view over the reducible cycles. This PR adds a consumer. |
Member
|
New tests to make the behavior change on irreducible loops visible: #213492 |
MaskRay
approved these changes
Aug 2, 2026
MaskRay
added a commit
that referenced
this pull request
Aug 2, 2026
The functions here have irreducible control flow, but none of them pins down how mass is divided among the entries of an irreducible region. Add four cases whose exact frequencies follow from the branch weights: - equalrows: all blocks share one successor distribution; 5:3:2. - selfloops: self edges of differing probability; ignoring them, each block splits evenly between the other two; 8:5:5. - unequalrows: symmetric non-header successors, differing header row; 8:3:3. - nonentry: a member of the region that is not an entry, so its mass is never adjusted; 6:4:3. BFI computes the first two exactly and the last two not. #213488 will show up as a diff.
Created using spr 1.3.8-wip
chudur-budur
pushed a commit
to chudur-budur/llvm-project
that referenced
this pull request
Aug 2, 2026
…213492) The functions here have irreducible control flow, but none of them pins down how mass is divided among the entries of an irreducible region. Add four cases whose exact frequencies follow from the branch weights: - equalrows: all blocks share one successor distribution; 5:3:2. - selfloops: self edges of differing probability; ignoring them, each block splits evenly between the other two; 8:5:5. - unequalrows: symmetric non-header successors, differing header row; 8:3:3. - nonentry: a member of the region that is not an entry, so its mass is never adjusted; 6:4:3. BFI computes the first two exactly and the last two not. llvm#213488 will show up as a diff.
frederik-h
pushed a commit
to frederik-h/llvm-project
that referenced
this pull request
Aug 3, 2026
…213492) The functions here have irreducible control flow, but none of them pins down how mass is divided among the entries of an irreducible region. Add four cases whose exact frequencies follow from the branch weights: - equalrows: all blocks share one successor distribution; 5:3:2. - selfloops: self edges of differing probability; ignoring them, each block splits evenly between the other two; 8:5:5. - unequalrows: symmetric non-header successors, differing header row; 8:3:3. - nonentry: a member of the region that is not an entry, so its mass is never adjusted; 6:4:3. BFI computes the first two exactly and the last two not. llvm#213488 will show up as a diff.
jgreenbaum
pushed a commit
to jgreenbaum/llvm-project
that referenced
this pull request
Aug 3, 2026
…213492) The functions here have irreducible control flow, but none of them pins down how mass is divided among the entries of an irreducible region. Add four cases whose exact frequencies follow from the branch weights: - equalrows: all blocks share one successor distribution; 5:3:2. - selfloops: self edges of differing probability; ignoring them, each block splits evenly between the other two; 8:5:5. - unequalrows: symmetric non-header successors, differing header row; 8:3:3. - nonentry: a member of the region that is not an entry, so its mass is never adjusted; 6:4:3. BFI computes the first two exactly and the last two not. llvm#213488 will show up as a diff.
aengelke
enabled auto-merge (squash)
August 5, 2026 09:40
aengelke
deleted the
users/aengelke/spr/analysis-use-cycleinfo-for-blockfrequencyinfo
branch
August 5, 2026 10:21
jinge90
pushed a commit
to jinge90/llvm-project
that referenced
this pull request
Aug 6, 2026
BranchProbabilityAnalysis uses CycleInfo, but BFI doesn't, causing the somewhat redundant construction of an extra LoopInfo. Avoid this by porting BFI to use CycleInfo. This requires a minor change to the BFI implementation to avoid incorrect results with irreducible loops that show up as nested but don't show up as nested loops -- such cycle entries are skipped now. (Such an entry heads a loop the cycle absorbed, and which entry keeps a nested cycle of its own depends on the order the search found the entries in.) @crossloops reaches c1 and c2 alike, yet only c1 heads a nested cycle, so seeding it gave c1 a loop scale c2 never got and their frequencies came out 0.68571 and 1.1429 where the test derives 1.0 for both. Represent none of those entries and leave the region to computeIrreducibleMass, which decomposes it from the reverse postorder as it does when LoopInfo finds no natural loop there. Passing the parent down also fixes the loop nest: &Loops.back() is whichever loop was created last, not the enclosing one, and every crash on this branch came from that. Co-authored-by: Fangrui Song <i@maskray.me>
tfzee
pushed a commit
to tfzee/llvm-project
that referenced
this pull request
Aug 6, 2026
…213492) The functions here have irreducible control flow, but none of them pins down how mass is divided among the entries of an irreducible region. Add four cases whose exact frequencies follow from the branch weights: - equalrows: all blocks share one successor distribution; 5:3:2. - selfloops: self edges of differing probability; ignoring them, each block splits evenly between the other two; 8:5:5. - unequalrows: symmetric non-header successors, differing header row; 8:3:3. - nonentry: a member of the region that is not an entry, so its mass is never adjusted; 6:4:3. BFI computes the first two exactly and the last two not. llvm#213488 will show up as a diff.
Contributor
|
The following crash starts happening with this patch: |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
3 participants
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
BranchProbabilityAnalysis uses CycleInfo, but BFI doesn't, causing the
somewhat redundant construction of an extra LoopInfo. Avoid this by
porting BFI to use CycleInfo.
This requires a minor change to the BFI implementation to avoid
incorrect results with irreducible loops that show up as nested but
don't show up as nested loops -- such cycle entries are skipped now.
(Such an entry heads a loop the cycle absorbed, and which entry keeps a
nested cycle of its own depends on the order the search found the entries
in.) @crossloops reaches c1 and c2 alike, yet only c1 heads a nested cycle,
so seeding it gave c1 a loop scale c2 never got and their frequencies came
out 0.68571 and 1.1429 where the test derives 1.0 for both. Represent none
of those entries and leave the region to computeIrreducibleMass, which
decomposes it from the reverse postorder as it does when LoopInfo finds no
natural loop there.
Passing the parent down also fixes the loop nest: &Loops.back() is whichever
loop was created last, not the enclosing one, and every crash on this branch
came from that.
Co-authored-by: Fangrui Song i@maskray.me