Skip to content

Commit

Permalink
[PassBuilder] Add ThinOrFullLTOPhase to optimizer pipeline (llvm#11…
Browse files Browse the repository at this point in the history 
…4577)
  • Loading branch information
@shiltian
shiltian authored Nov 4, 2024
1 parent dc45ff1 commit 390300d
Show file tree
Hide file tree
Showing 6 changed files with 57 additions and 37 deletions.
22 changes: 12 additions & 10 deletions clang/lib/CodeGen/BackendUtil.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -674,7 +674,7 @@ static void addKCFIPass(const Triple &TargetTriple, const LangOptions &LangOpts,

// Ensure we lower KCFI operand bundles with -O0.
PB.registerOptimizerLastEPCallback(
[&](ModulePassManager &MPM, OptimizationLevel Level) {
[&](ModulePassManager &MPM, OptimizationLevel Level, ThinOrFullLTOPhase) {
if (Level == OptimizationLevel::O0 &&
LangOpts.Sanitize.has(SanitizerKind::KCFI))
MPM.addPass(createModuleToFunctionPassAdaptor(KCFIPass()));
Expand All @@ -693,8 +693,8 @@ static void addKCFIPass(const Triple &TargetTriple, const LangOptions &LangOpts,
static void addSanitizers(const Triple &TargetTriple,
const CodeGenOptions &CodeGenOpts,
const LangOptions &LangOpts, PassBuilder &PB) {
auto SanitizersCallback = [&](ModulePassManager &MPM,
OptimizationLevel Level) {
auto SanitizersCallback = [&](ModulePassManager &MPM, OptimizationLevel Level,
ThinOrFullLTOPhase) {
if (CodeGenOpts.hasSanitizeCoverage()) {
auto SancovOpts = getSancovOptsFromCGOpts(CodeGenOpts);
MPM.addPass(SanitizerCoveragePass(
Expand Down Expand Up @@ -778,9 +778,10 @@ static void addSanitizers(const Triple &TargetTriple,
};
if (ClSanitizeOnOptimizerEarlyEP) {
PB.registerOptimizerEarlyEPCallback(
[SanitizersCallback](ModulePassManager &MPM, OptimizationLevel Level) {
[SanitizersCallback](ModulePassManager &MPM, OptimizationLevel Level,
ThinOrFullLTOPhase Phase) {
ModulePassManager NewMPM;
SanitizersCallback(NewMPM, Level);
SanitizersCallback(NewMPM, Level, Phase);
if (!NewMPM.isEmpty()) {
// Sanitizers can abandon<GlobalsAA>.
NewMPM.addPass(RequireAnalysisPass<GlobalsAA, llvm::Module>());
Expand Down Expand Up @@ -1058,11 +1059,12 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
// TODO: Consider passing the MemoryProfileOutput to the pass builder via
// the PGOOptions, and set this up there.
if (!CodeGenOpts.MemoryProfileOutput.empty()) {
PB.registerOptimizerLastEPCallback(
[](ModulePassManager &MPM, OptimizationLevel Level) {
MPM.addPass(createModuleToFunctionPassAdaptor(MemProfilerPass()));
MPM.addPass(ModuleMemProfilerPass());
});
PB.registerOptimizerLastEPCallback([](ModulePassManager &MPM,
OptimizationLevel Level,
ThinOrFullLTOPhase) {
MPM.addPass(createModuleToFunctionPassAdaptor(MemProfilerPass()));
MPM.addPass(ModuleMemProfilerPass());
});
}

if (CodeGenOpts.FatLTO) {
Expand Down
20 changes: 14 additions & 6 deletions llvm/include/llvm/Passes/PassBuilder.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -490,7 +490,8 @@ class PassBuilder {
/// This extension point allows adding optimizations before the function
/// optimization pipeline.
void registerOptimizerEarlyEPCallback(
const std::function<void(ModulePassManager &, OptimizationLevel)> &C) {
const std::function<void(ModulePassManager &, OptimizationLevel,
ThinOrFullLTOPhase Phase)> &C) {
OptimizerEarlyEPCallbacks.push_back(C);
}

Expand All @@ -499,7 +500,8 @@ class PassBuilder {
/// This extension point allows adding optimizations at the very end of the
/// function optimization pipeline.
void registerOptimizerLastEPCallback(
const std::function<void(ModulePassManager &, OptimizationLevel)> &C) {
const std::function<void(ModulePassManager &, OptimizationLevel,
ThinOrFullLTOPhase)> &C) {
OptimizerLastEPCallbacks.push_back(C);
}

Expand Down Expand Up @@ -630,9 +632,11 @@ class PassBuilder {
void invokeVectorizerStartEPCallbacks(FunctionPassManager &FPM,
OptimizationLevel Level);
void invokeOptimizerEarlyEPCallbacks(ModulePassManager &MPM,
OptimizationLevel Level);
OptimizationLevel Level,
ThinOrFullLTOPhase Phase);
void invokeOptimizerLastEPCallbacks(ModulePassManager &MPM,
OptimizationLevel Level);
OptimizationLevel Level,
ThinOrFullLTOPhase Phase);
void invokeFullLinkTimeOptimizationEarlyEPCallbacks(ModulePassManager &MPM,
OptimizationLevel Level);
void invokeFullLinkTimeOptimizationLastEPCallbacks(ModulePassManager &MPM,
Expand Down Expand Up @@ -756,9 +760,13 @@ class PassBuilder {
SmallVector<std::function<void(FunctionPassManager &, OptimizationLevel)>, 2>
VectorizerStartEPCallbacks;
// Module callbacks
SmallVector<std::function<void(ModulePassManager &, OptimizationLevel)>, 2>
SmallVector<std::function<void(ModulePassManager &, OptimizationLevel,
ThinOrFullLTOPhase)>,
2>
OptimizerEarlyEPCallbacks;
SmallVector<std::function<void(ModulePassManager &, OptimizationLevel)>, 2>
SmallVector<std::function<void(ModulePassManager &, OptimizationLevel,
ThinOrFullLTOPhase)>,
2>
OptimizerLastEPCallbacks;
SmallVector<std::function<void(ModulePassManager &, OptimizationLevel)>, 2>
FullLinkTimeOptimizationEarlyEPCallbacks;
Expand Down
24 changes: 14 additions & 10 deletions llvm/lib/Passes/PassBuilderPipelines.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -359,14 +359,16 @@ void PassBuilder::invokeVectorizerStartEPCallbacks(FunctionPassManager &FPM,
C(FPM, Level);
}
void PassBuilder::invokeOptimizerEarlyEPCallbacks(ModulePassManager &MPM,
OptimizationLevel Level) {
OptimizationLevel Level,
ThinOrFullLTOPhase Phase) {
for (auto &C : OptimizerEarlyEPCallbacks)
C(MPM, Level);
C(MPM, Level, Phase);
}
void PassBuilder::invokeOptimizerLastEPCallbacks(ModulePassManager &MPM,
OptimizationLevel Level) {
OptimizationLevel Level,
ThinOrFullLTOPhase Phase) {
for (auto &C : OptimizerLastEPCallbacks)
C(MPM, Level);
C(MPM, Level, Phase);
}
void PassBuilder::invokeFullLinkTimeOptimizationEarlyEPCallbacks(
ModulePassManager &MPM, OptimizationLevel Level) {
Expand Down Expand Up @@ -1464,7 +1466,7 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
if (EnableGlobalAnalyses)
MPM.addPass(RecomputeGlobalsAAPass());

invokeOptimizerEarlyEPCallbacks(MPM, Level);
invokeOptimizerEarlyEPCallbacks(MPM, Level, LTOPhase);

FunctionPassManager OptimizePM;
// Scheduling LoopVersioningLICM when inlining is over, because after that
Expand Down Expand Up @@ -1559,7 +1561,7 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM),
PTO.EagerlyInvalidateAnalyses));

invokeOptimizerLastEPCallbacks(MPM, Level);
invokeOptimizerLastEPCallbacks(MPM, Level, LTOPhase);

// Split out cold code. Splitting is done late to avoid hiding context from
// other optimizations and inadvertently regressing performance. The tradeoff
Expand Down Expand Up @@ -1716,8 +1718,10 @@ PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level) {
// Handle Optimizer{Early,Last}EPCallbacks added by clang on PreLink. Actual
// optimization is going to be done in PostLink stage, but clang can't add
// callbacks there in case of in-process ThinLTO called by linker.
invokeOptimizerEarlyEPCallbacks(MPM, Level);
invokeOptimizerLastEPCallbacks(MPM, Level);
invokeOptimizerEarlyEPCallbacks(MPM, Level,
/*Phase=*/ThinOrFullLTOPhase::ThinLTOPreLink);
invokeOptimizerLastEPCallbacks(MPM, Level,
/*Phase=*/ThinOrFullLTOPhase::ThinLTOPreLink);

// Emit annotation remarks.
addAnnotationRemarksPass(MPM);
Expand Down Expand Up @@ -2198,7 +2202,7 @@ PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level,
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
}

invokeOptimizerEarlyEPCallbacks(MPM, Level);
invokeOptimizerEarlyEPCallbacks(MPM, Level, Phase);

if (!VectorizerStartEPCallbacks.empty()) {
FunctionPassManager FPM;
Expand All @@ -2216,7 +2220,7 @@ PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level,
CoroPM.addPass(GlobalDCEPass());
MPM.addPass(CoroConditionalWrapper(std::move(CoroPM)));

invokeOptimizerLastEPCallbacks(MPM, Level);
invokeOptimizerLastEPCallbacks(MPM, Level, Phase);

if (isLTOPreLink(Phase))
addRequiredLTOPreLinkPasses(MPM);
Expand Down
23 changes: 14 additions & 9 deletions llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,11 @@ static WWMRegisterRegAlloc
createGreedyWWMRegisterAllocator);
static WWMRegisterRegAlloc fastRegAllocWWMReg("fast", "fast register allocator",
createFastWWMRegisterAllocator);

static bool isLTOPreLink(ThinOrFullLTOPhase Phase) {
return Phase == ThinOrFullLTOPhase::FullLTOPreLink ||
Phase == ThinOrFullLTOPhase::ThinLTOPreLink;
}
} // anonymous namespace

static cl::opt<bool>
Expand Down Expand Up @@ -755,9 +760,7 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
PM.addPass(AMDGPUUnifyMetadataPass());

// We don't want to run internalization at per-module stage.
bool LTOPreLink = Phase == ThinOrFullLTOPhase::FullLTOPreLink ||
Phase == ThinOrFullLTOPhase::ThinLTOPreLink;
if (InternalizeSymbols && !LTOPreLink) {
if (InternalizeSymbols && !isLTOPreLink(Phase)) {
PM.addPass(InternalizePass(mustPreserveGV));
PM.addPass(GlobalDCEPass());
}
Expand Down Expand Up @@ -809,12 +812,14 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
});

// FIXME: Why is AMDGPUAttributor not in CGSCC?
PB.registerOptimizerLastEPCallback(
[this](ModulePassManager &MPM, OptimizationLevel Level) {
if (Level != OptimizationLevel::O0) {
MPM.addPass(AMDGPUAttributorPass(*this));
}
});
PB.registerOptimizerLastEPCallback([this](ModulePassManager &MPM,
OptimizationLevel Level,
ThinOrFullLTOPhase Phase) {
if (Level != OptimizationLevel::O0) {
if (!isLTOPreLink(Phase))
MPM.addPass(AMDGPUAttributorPass(*this));
}
});

PB.registerFullLinkTimeOptimizationLastEPCallback(
[this](ModulePassManager &PM, OptimizationLevel Level) {
Expand Down
1 change: 1 addition & 0 deletions llvm/test/CodeGen/AMDGPU/print-pipeline-passes.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
; O0-NOT: amdgpu-attributor

; PRE-NOT: internalize
; PRE-NOT: amdgpu-attributor

define amdgpu_kernel void @kernel() {
entry:
Expand Down
4 changes: 2 additions & 2 deletions llvm/tools/opt/NewPMDriver.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -300,13 +300,13 @@ static void registerEPCallbacks(PassBuilder &PB) {
});
if (tryParsePipelineText<ModulePassManager>(PB, OptimizerEarlyEPPipeline))
PB.registerOptimizerEarlyEPCallback(
[&PB](ModulePassManager &PM, OptimizationLevel) {
[&PB](ModulePassManager &PM, OptimizationLevel, ThinOrFullLTOPhase) {
ExitOnError Err("Unable to parse OptimizerEarlyEP pipeline: ");
Err(PB.parsePassPipeline(PM, OptimizerEarlyEPPipeline));
});
if (tryParsePipelineText<ModulePassManager>(PB, OptimizerLastEPPipeline))
PB.registerOptimizerLastEPCallback(
[&PB](ModulePassManager &PM, OptimizationLevel) {
[&PB](ModulePassManager &PM, OptimizationLevel, ThinOrFullLTOPhase) {
ExitOnError Err("Unable to parse OptimizerLastEP pipeline: ");
Err(PB.parsePassPipeline(PM, OptimizerLastEPPipeline));
});
Expand Down

0 comments on commit 390300d

Please sign in to comment.