Skip to content

Commit 81083bd

Browse files
aeubanksaeubanks
committed
[PGO] Add ability to mark cold functions as optsize/minsize/optnone
The performance of cold functions shouldn't matter too much, so if we care about binary sizes, add an option to mark cold functions as optsize/minsize for binary size, or optnone for compile times [1]. Clang patch will be in a future patch Initial version: https://reviews.llvm.org/D149800 [1] https://discourse.llvm.org/t/rfc-new-feature-proposal-de-optimizing-cold-functions-using-pgo-info/56388
1 parent eb4a061 commit 81083bd

File tree

13 files changed

+253
-19
lines changed

13 files changed

+253
-19
lines changed

‎clang/lib/CodeGen/BackendUtil.cpp‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -770,7 +770,8 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
770770
CodeGenOpts.InstrProfileOutput.empty() ? getDefaultProfileGenName()
771771
: CodeGenOpts.InstrProfileOutput,
772772
"", "", CodeGenOpts.MemoryProfileUsePath, nullptr, PGOOptions::IRInstr,
773-
PGOOptions::NoCSAction, CodeGenOpts.DebugInfoForProfiling,
773+
PGOOptions::NoCSAction, PGOOptions::ColdFuncAttr::None,
774+
CodeGenOpts.DebugInfoForProfiling,
774775
/*PseudoProbeForProfiling=*/false, CodeGenOpts.AtomicProfileUpdate);
775776
else if (CodeGenOpts.hasProfileIRUse()) {
776777
// -fprofile-use.
@@ -779,28 +780,32 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
779780
PGOOpt = PGOOptions(
780781
CodeGenOpts.ProfileInstrumentUsePath, "",
781782
CodeGenOpts.ProfileRemappingFile, CodeGenOpts.MemoryProfileUsePath, VFS,
782-
PGOOptions::IRUse, CSAction, CodeGenOpts.DebugInfoForProfiling);
783+
PGOOptions::IRUse, CSAction, PGOOptions::ColdFuncAttr::None,
784+
CodeGenOpts.DebugInfoForProfiling);
783785
} else if (!CodeGenOpts.SampleProfileFile.empty())
784786
// -fprofile-sample-use
785787
PGOOpt = PGOOptions(
786788
CodeGenOpts.SampleProfileFile, "", CodeGenOpts.ProfileRemappingFile,
787789
CodeGenOpts.MemoryProfileUsePath, VFS, PGOOptions::SampleUse,
788-
PGOOptions::NoCSAction, CodeGenOpts.DebugInfoForProfiling,
789-
CodeGenOpts.PseudoProbeForProfiling);
790+
PGOOptions::NoCSAction, PGOOptions::ColdFuncAttr::None,
791+
CodeGenOpts.DebugInfoForProfiling, CodeGenOpts.PseudoProbeForProfiling);
790792
else if (!CodeGenOpts.MemoryProfileUsePath.empty())
791793
// -fmemory-profile-use (without any of the above options)
792794
PGOOpt = PGOOptions("", "", "", CodeGenOpts.MemoryProfileUsePath, VFS,
793795
PGOOptions::NoAction, PGOOptions::NoCSAction,
796+
PGOOptions::ColdFuncAttr::None,
794797
CodeGenOpts.DebugInfoForProfiling);
795798
else if (CodeGenOpts.PseudoProbeForProfiling)
796799
// -fpseudo-probe-for-profiling
797800
PGOOpt = PGOOptions("", "", "", /*MemoryProfile=*/"", nullptr,
798801
PGOOptions::NoAction, PGOOptions::NoCSAction,
802+
PGOOptions::ColdFuncAttr::None,
799803
CodeGenOpts.DebugInfoForProfiling, true);
800804
else if (CodeGenOpts.DebugInfoForProfiling)
801805
// -fdebug-info-for-profiling
802806
PGOOpt = PGOOptions("", "", "", /*MemoryProfile=*/"", nullptr,
803-
PGOOptions::NoAction, PGOOptions::NoCSAction, true);
807+
PGOOptions::NoAction, PGOOptions::NoCSAction,
808+
PGOOptions::ColdFuncAttr::None, true);
804809

805810
// Check to see if we want to generate a CS profile.
806811
if (CodeGenOpts.hasProfileCSIRInstr()) {
@@ -823,7 +828,8 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
823828
? getDefaultProfileGenName()
824829
: CodeGenOpts.InstrProfileOutput,
825830
"", /*MemoryProfile=*/"", nullptr, PGOOptions::NoAction,
826-
PGOOptions::CSIRInstr, CodeGenOpts.DebugInfoForProfiling);
831+
PGOOptions::CSIRInstr, PGOOptions::ColdFuncAttr::None,
832+
CodeGenOpts.DebugInfoForProfiling);
827833
}
828834
if (TM)
829835
TM->setPGOOption(PGOOpt);

‎llvm/include/llvm/Support/PGOOptions.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,12 @@ class FileSystem;
2727
struct PGOOptions {
2828
enum PGOAction { NoAction, IRInstr, IRUse, SampleUse };
2929
enum CSPGOAction { NoCSAction, CSIRInstr, CSIRUse };
30+
enum class ColdFuncAttr { None, OptSize, MinSize, OptNone };
3031
PGOOptions(std::string ProfileFile, std::string CSProfileGenFile,
3132
std::string ProfileRemappingFile, std::string MemoryProfile,
3233
IntrusiveRefCntPtr<vfs::FileSystem> FS,
3334
PGOAction Action = NoAction, CSPGOAction CSAction = NoCSAction,
35+
ColdFuncAttr ColdType = ColdFuncAttr::None,
3436
bool DebugInfoForProfiling = false,
3537
bool PseudoProbeForProfiling = false,
3638
bool AtomicCounterUpdate = false);
@@ -44,6 +46,7 @@ struct PGOOptions {
4446
std::string MemoryProfile;
4547
PGOAction Action;
4648
CSPGOAction CSAction;
49+
ColdFuncAttr ColdType;
4750
bool DebugInfoForProfiling;
4851
bool PseudoProbeForProfiling;
4952
bool AtomicCounterUpdate;

‎llvm/include/llvm/Transforms/Instrumentation/MarkColdFunctions.h‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
//===- MarkColdFunctions.h - ------------------------------------*- C++ -*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#ifndef LLVM_TRANSFORMS_INSTRUMENTATION_MARKCOLDFUNCTIONS_H
10+
#define LLVM_TRANSFORMS_INSTRUMENTATION_MARKCOLDFUNCTIONS_H
11+
12+
#include "llvm/IR/PassManager.h"
13+
#include "llvm/Support/PGOOptions.h"
14+
15+
namespace llvm {
16+
17+
struct MarkColdFunctionsPass : public PassInfoMixin<MarkColdFunctionsPass> {
18+
MarkColdFunctionsPass(PGOOptions::ColdFuncAttr ColdType)
19+
: ColdType(ColdType) {}
20+
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
21+
22+
private:
23+
PGOOptions::ColdFuncAttr ColdType;
24+
};
25+
26+
} // namespace llvm
27+
28+
#endif // LLVM_TRANSFORMS_INSTRUMENTATION_MARKCOLDFUNCTIONS_H

‎llvm/lib/LTO/LTOBackend.cpp‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -243,19 +243,23 @@ static void runNewPMPasses(const Config &Conf, Module &Mod, TargetMachine *TM,
243243
if (!Conf.SampleProfile.empty())
244244
PGOOpt = PGOOptions(Conf.SampleProfile, "", Conf.ProfileRemapping,
245245
/*MemoryProfile=*/"", FS, PGOOptions::SampleUse,
246-
PGOOptions::NoCSAction, true);
246+
PGOOptions::NoCSAction, PGOOptions::ColdFuncAttr::None,
247+
true);
247248
else if (Conf.RunCSIRInstr) {
248249
PGOOpt = PGOOptions("", Conf.CSIRProfile, Conf.ProfileRemapping,
249250
/*MemoryProfile=*/"", FS, PGOOptions::IRUse,
250-
PGOOptions::CSIRInstr, Conf.AddFSDiscriminator);
251+
PGOOptions::CSIRInstr, PGOOptions::ColdFuncAttr::None,
252+
Conf.AddFSDiscriminator);
251253
} else if (!Conf.CSIRProfile.empty()) {
252254
PGOOpt = PGOOptions(Conf.CSIRProfile, "", Conf.ProfileRemapping,
253255
/*MemoryProfile=*/"", FS, PGOOptions::IRUse,
254-
PGOOptions::CSIRUse, Conf.AddFSDiscriminator);
256+
PGOOptions::CSIRUse, PGOOptions::ColdFuncAttr::None,
257+
Conf.AddFSDiscriminator);
255258
NoPGOWarnMismatch = !Conf.PGOWarnMismatch;
256259
} else if (Conf.AddFSDiscriminator) {
257260
PGOOpt = PGOOptions("", "", "", /*MemoryProfile=*/"", nullptr,
258-
PGOOptions::NoAction, PGOOptions::NoCSAction, true);
261+
PGOOptions::NoAction, PGOOptions::NoCSAction,
262+
PGOOptions::ColdFuncAttr::None, true);
259263
}
260264
TM->setPGOOption(PGOOpt);
261265

‎llvm/lib/Passes/PassBuilder.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,7 @@
143143
#include "llvm/Transforms/Instrumentation/InstrOrderFile.h"
144144
#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
145145
#include "llvm/Transforms/Instrumentation/KCFI.h"
146+
#include "llvm/Transforms/Instrumentation/MarkColdFunctions.h"
146147
#include "llvm/Transforms/Instrumentation/MemProfiler.h"
147148
#include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
148149
#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
@@ -234,8 +235,8 @@
234235
#include "llvm/Transforms/Utils/CanonicalizeAliases.h"
235236
#include "llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h"
236237
#include "llvm/Transforms/Utils/CountVisits.h"
237-
#include "llvm/Transforms/Utils/Debugify.h"
238238
#include "llvm/Transforms/Utils/DXILUpgrade.h"
239+
#include "llvm/Transforms/Utils/Debugify.h"
239240
#include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
240241
#include "llvm/Transforms/Utils/FixIrreducible.h"
241242
#include "llvm/Transforms/Utils/HelloWorld.h"

‎llvm/lib/Passes/PassBuilderPipelines.cpp‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@
7373
#include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
7474
#include "llvm/Transforms/Instrumentation/InstrOrderFile.h"
7575
#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
76+
#include "llvm/Transforms/Instrumentation/MarkColdFunctions.h"
7677
#include "llvm/Transforms/Instrumentation/MemProfiler.h"
7778
#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
7879
#include "llvm/Transforms/Scalar/ADCE.h"
@@ -212,6 +213,12 @@ static cl::opt<bool>
212213
cl::desc("Enable DFA jump threading"),
213214
cl::init(false), cl::Hidden);
214215

216+
// TODO: turn on and remove flag
217+
static cl::opt<bool>
218+
EnableMarkColdFunctions("enable-mark-cold-functions",
219+
cl::desc("Enable pass to mark cold functions"),
220+
cl::init(false));
221+
215222
static cl::opt<bool>
216223
EnableHotColdSplit("hot-cold-split",
217224
cl::desc("Enable hot-cold splitting pass"));
@@ -1127,6 +1134,11 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
11271134
if (EnableSyntheticCounts && !PGOOpt)
11281135
MPM.addPass(SyntheticCountsPropagation());
11291136

1137+
if (EnableMarkColdFunctions && PGOOpt &&
1138+
(PGOOpt->Action == PGOOptions::SampleUse ||
1139+
PGOOpt->Action == PGOOptions::IRUse))
1140+
MPM.addPass(MarkColdFunctionsPass(PGOOpt->ColdType));
1141+
11301142
if (EnableModuleInliner)
11311143
MPM.addPass(buildModuleInlinerPipeline(Level, Phase));
11321144
else

‎llvm/lib/Passes/PassRegistry.def‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ MODULE_PASS("print-ir-similarity", IRSimilarityAnalysisPrinterPass(dbgs()))
8585
MODULE_PASS("lower-global-dtors", LowerGlobalDtorsPass())
8686
MODULE_PASS("lower-ifunc", LowerIFuncPass())
8787
MODULE_PASS("lowertypetests", LowerTypeTestsPass())
88+
MODULE_PASS("mark-cold-functions", MarkColdFunctionsPass(PGOOpt ? PGOOpt->ColdType : PGOOptions::ColdFuncAttr::None))
8889
MODULE_PASS("metarenamer", MetaRenamerPass())
8990
MODULE_PASS("mergefunc", MergeFunctionsPass())
9091
MODULE_PASS("name-anon-globals", NameAnonGlobalPass())

‎llvm/lib/Support/PGOOptions.cpp‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,12 @@ PGOOptions::PGOOptions(std::string ProfileFile, std::string CSProfileGenFile,
1515
std::string ProfileRemappingFile,
1616
std::string MemoryProfile,
1717
IntrusiveRefCntPtr<vfs::FileSystem> FS, PGOAction Action,
18-
CSPGOAction CSAction, bool DebugInfoForProfiling,
19-
bool PseudoProbeForProfiling, bool AtomicCounterUpdate)
18+
CSPGOAction CSAction, ColdFuncAttr ColdType,
19+
bool DebugInfoForProfiling, bool PseudoProbeForProfiling,
20+
bool AtomicCounterUpdate)
2021
: ProfileFile(ProfileFile), CSProfileGenFile(CSProfileGenFile),
2122
ProfileRemappingFile(ProfileRemappingFile), MemoryProfile(MemoryProfile),
22-
Action(Action), CSAction(CSAction),
23+
Action(Action), CSAction(CSAction), ColdType(ColdType),
2324
DebugInfoForProfiling(DebugInfoForProfiling ||
2425
(Action == SampleUse && !PseudoProbeForProfiling)),
2526
PseudoProbeForProfiling(PseudoProbeForProfiling),

‎llvm/lib/Transforms/Instrumentation/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ add_llvm_component_library(LLVMInstrumentation
66
DataFlowSanitizer.cpp
77
GCOVProfiling.cpp
88
BlockCoverageInference.cpp
9+
MarkColdFunctions.cpp
910
MemProfiler.cpp
1011
MemorySanitizer.cpp
1112
IndirectCallPromotion.cpp

‎llvm/lib/Transforms/Instrumentation/MarkColdFunctions.cpp‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#include "llvm/Transforms/Instrumentation/MarkColdFunctions.h"
10+
#include "llvm/Analysis/BlockFrequencyInfo.h"
11+
#include "llvm/Analysis/ProfileSummaryInfo.h"
12+
#include "llvm/IR/PassManager.h"
13+
14+
using namespace llvm;
15+
16+
PreservedAnalyses MarkColdFunctionsPass::run(Module &M,
17+
ModuleAnalysisManager &AM) {
18+
if (ColdType == PGOOptions::ColdFuncAttr::None)
19+
return PreservedAnalyses::all();
20+
ProfileSummaryInfo &PSI = AM.getResult<ProfileSummaryAnalysis>(M);
21+
if (!PSI.hasProfileSummary())
22+
return PreservedAnalyses::all();
23+
FunctionAnalysisManager &FAM =
24+
AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
25+
bool MadeChange = false;
26+
for (Function &F : M) {
27+
if (F.isDeclaration())
28+
continue;
29+
BlockFrequencyInfo &BFI = FAM.getResult<BlockFrequencyAnalysis>(F);
30+
if (!PSI.isFunctionColdInCallGraph(&F, BFI))
31+
continue;
32+
// Add optsize/minsize/optnone if requested.
33+
switch (ColdType) {
34+
case PGOOptions::ColdFuncAttr::None:
35+
assert(false);
36+
break;
37+
case PGOOptions::ColdFuncAttr::OptSize:
38+
if (!F.hasFnAttribute(Attribute::OptimizeNone) &&
39+
!F.hasFnAttribute(Attribute::OptimizeForSize) &&
40+
!F.hasFnAttribute(Attribute::MinSize)) {
41+
F.addFnAttr(Attribute::OptimizeForSize);
42+
MadeChange = true;
43+
}
44+
break;
45+
case PGOOptions::ColdFuncAttr::MinSize:
46+
// Change optsize to minsize.
47+
if (!F.hasFnAttribute(Attribute::OptimizeNone) &&
48+
!F.hasFnAttribute(Attribute::MinSize)) {
49+
F.removeFnAttr(Attribute::OptimizeForSize);
50+
F.addFnAttr(Attribute::MinSize);
51+
MadeChange = true;
52+
}
53+
break;
54+
case PGOOptions::ColdFuncAttr::OptNone:
55+
// Strip optsize/minsize.
56+
F.removeFnAttr(Attribute::OptimizeForSize);
57+
F.removeFnAttr(Attribute::MinSize);
58+
F.addFnAttr(Attribute::OptimizeNone);
59+
F.addFnAttr(Attribute::NoInline);
60+
MadeChange = true;
61+
break;
62+
}
63+
}
64+
return MadeChange ? PreservedAnalyses::none() : PreservedAnalyses::all();
65+
}

0 commit comments

Comments
 (0)