Skip to content

Commit 55f1a6b

Browse files
fhahnfhahn
committed
[VPlan] Manage noalias/alias_scope metadata in VPlan.
Use VPIRMetadata added in #135272 to also manage no-alias metadata added by versioning. Note that this means we have to build the no-alias metadata up-front once. If it is not used, it will be discarded automatically.
1 parent dde00f5 commit 55f1a6b

File tree

10 files changed

+109
-103
lines changed

10 files changed

+109
-103
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h

+2-1
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ class LoopVectorizationLegality;
3636
class LoopVectorizationCostModel;
3737
class PredicatedScalarEvolution;
3838
class LoopVectorizeHints;
39+
class LoopVersioning;
3940
class OptimizationRemarkEmitter;
4041
class TargetTransformInfo;
4142
class TargetLibraryInfo;
@@ -515,7 +516,7 @@ class LoopVectorizationPlanner {
515516
/// returned VPlan is valid for. If no VPlan can be built for the input range,
516517
/// set the largest included VF to the maximum VF for which no plan could be
517518
/// built.
518-
VPlanPtr tryToBuildVPlanWithVPRecipes(VFRange &Range);
519+
VPlanPtr tryToBuildVPlanWithVPRecipes(VFRange &Range, LoopVersioning *LVer);
519520

520521
/// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive,
521522
/// according to the information gathered by Legal when it checked if it is

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

+41-29
Original file line numberDiff line numberDiff line change
@@ -2364,7 +2364,7 @@ void InnerLoopVectorizer::scalarizeInstruction(const Instruction *Instr,
23642364
InputLane = VPLane::getFirstLane();
23652365
Cloned->setOperand(I.index(), State.get(Operand, InputLane));
23662366
}
2367-
State.addNewMetadata(Cloned, Instr);
2367+
RepRecipe->applyMetadata(*Cloned);
23682368

23692369
// Place the cloned scalar in the new loop.
23702370
State.Builder.Insert(Cloned);
@@ -7900,24 +7900,6 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
79007900
if (VectorizingEpilogue)
79017901
VPlanTransforms::removeDeadRecipes(BestVPlan);
79027902

7903-
// Only use noalias metadata when using memory checks guaranteeing no overlap
7904-
// across all iterations.
7905-
const LoopAccessInfo *LAI = ILV.Legal->getLAI();
7906-
std::unique_ptr<LoopVersioning> LVer = nullptr;
7907-
if (LAI && !LAI->getRuntimePointerChecking()->getChecks().empty() &&
7908-
!LAI->getRuntimePointerChecking()->getDiffChecks()) {
7909-
7910-
// We currently don't use LoopVersioning for the actual loop cloning but we
7911-
// still use it to add the noalias metadata.
7912-
// TODO: Find a better way to re-use LoopVersioning functionality to add
7913-
// metadata.
7914-
LVer = std::make_unique<LoopVersioning>(
7915-
*LAI, LAI->getRuntimePointerChecking()->getChecks(), OrigLoop, LI, DT,
7916-
PSE.getSE());
7917-
State.LVer = &*LVer;
7918-
State.LVer->prepareNoAliasMetadata();
7919-
}
7920-
79217903
ILV.printDebugTracesAtStart();
79227904

79237905
//===------------------------------------------------===//
@@ -8508,13 +8490,14 @@ VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
85088490
Builder.insert(VectorPtr);
85098491
Ptr = VectorPtr;
85108492
}
8493+
auto Metadata = getMetadataToPropagate(I);
85118494
if (LoadInst *Load = dyn_cast<LoadInst>(I))
85128495
return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse,
8513-
I->getDebugLoc());
8496+
Metadata, I->getDebugLoc());
85148497

85158498
StoreInst *Store = cast<StoreInst>(I);
85168499
return new VPWidenStoreRecipe(*Store, Ptr, Operands[0], Mask, Consecutive,
8517-
Reverse, I->getDebugLoc());
8500+
Reverse, Metadata, I->getDebugLoc());
85188501
}
85198502

85208503
/// Creates a VPWidenIntOrFpInductionRecpipe for \p Phi. If needed, it will also
@@ -8889,8 +8872,9 @@ VPRecipeBuilder::handleReplication(Instruction *I, ArrayRef<VPValue *> Operands,
88898872
assert((Range.Start.isScalar() || !IsUniform || !IsPredicated ||
88908873
(Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
88918874
"Should not predicate a uniform recipe");
8892-
auto *Recipe = new VPReplicateRecipe(
8893-
I, make_range(Operands.begin(), Operands.end()), IsUniform, BlockInMask);
8875+
auto *Recipe =
8876+
new VPReplicateRecipe(I, make_range(Operands.begin(), Operands.end()),
8877+
IsUniform, BlockInMask, getMetadataToPropagate(I));
88948878
return Recipe;
88958879
}
88968880

@@ -9011,6 +8995,20 @@ bool VPRecipeBuilder::getScaledReductions(
90118995
return false;
90128996
}
90138997

8998+
SmallVector<std::pair<unsigned, MDNode *>>
8999+
VPRecipeBuilder::getMetadataToPropagate(Instruction *I) const {
9000+
SmallVector<std::pair<unsigned, MDNode *>> Metadata;
9001+
::getMetadataToPropagate(I, Metadata);
9002+
if (LVer && isa<LoadInst, StoreInst>(I)) {
9003+
const auto &[AliasScopeMD, NoAliasMD] = LVer->getNoAliasMetadataFor(I);
9004+
if (AliasScopeMD)
9005+
Metadata.emplace_back(LLVMContext::MD_alias_scope, AliasScopeMD);
9006+
if (NoAliasMD)
9007+
Metadata.emplace_back(LLVMContext::MD_noalias, NoAliasMD);
9008+
}
9009+
return Metadata;
9010+
}
9011+
90149012
VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(
90159013
Instruction *Instr, ArrayRef<VPValue *> Operands, VFRange &Range) {
90169014
// First, check for specific widening recipes that deal with inductions, Phi
@@ -9138,10 +9136,22 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
91389136
ElementCount MaxVF) {
91399137
assert(OrigLoop->isInnermost() && "Inner loop expected.");
91409138

9139+
// Only use noalias metadata when using memory checks guaranteeing no overlap
9140+
// across all iterations.
9141+
const LoopAccessInfo *LAI = Legal->getLAI();
9142+
std::unique_ptr<LoopVersioning> LVer = nullptr;
9143+
if (LAI && !LAI->getRuntimePointerChecking()->getChecks().empty() &&
9144+
!LAI->getRuntimePointerChecking()->getDiffChecks()) {
9145+
LVer = std::make_unique<LoopVersioning>(
9146+
*LAI, LAI->getRuntimePointerChecking()->getChecks(), OrigLoop, LI, DT,
9147+
PSE.getSE());
9148+
LVer->prepareNoAliasMetadata();
9149+
}
9150+
91419151
auto MaxVFTimes2 = MaxVF * 2;
91429152
for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
91439153
VFRange SubRange = {VF, MaxVFTimes2};
9144-
if (auto Plan = tryToBuildVPlanWithVPRecipes(SubRange)) {
9154+
if (auto Plan = tryToBuildVPlanWithVPRecipes(SubRange, LVer.get())) {
91459155
bool HasScalarVF = Plan->hasScalarVFOnly();
91469156
// Now optimize the initial VPlan.
91479157
if (!HasScalarVF)
@@ -9435,7 +9445,8 @@ static void addExitUsersForFirstOrderRecurrences(
94359445
}
94369446

94379447
VPlanPtr
9438-
LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
9448+
LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range,
9449+
LoopVersioning *LVer) {
94399450

94409451
using namespace llvm::VPlanPatternMatch;
94419452
SmallPtrSet<const InterleaveGroup<Instruction> *, 1> InterleaveGroups;
@@ -9481,7 +9492,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
94819492
addCanonicalIVRecipes(*Plan, Legal->getWidestInductionType(), HasNUW, DL);
94829493

94839494
VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
9484-
Builder);
9495+
Builder, LVer);
94859496

94869497
// ---------------------------------------------------------------------------
94879498
// Pre-construction: record ingredients whose recipes we'll need to further
@@ -9595,8 +9606,9 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
95959606
Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) {
95969607
// Only create recipe for the final invariant store of the reduction.
95979608
if (Legal->isInvariantStoreOfReduction(SI)) {
9598-
auto *Recipe =
9599-
new VPReplicateRecipe(SI, R.operands(), true /* IsUniform */);
9609+
auto *Recipe = new VPReplicateRecipe(
9610+
SI, R.operands(), true /* IsUniform */, /*Mask*/ nullptr,
9611+
RecipeBuilder.getMetadataToPropagate(SI));
96009612
Recipe->insertBefore(*MiddleVPBB, MBIP);
96019613
}
96029614
R.eraseFromParent();
@@ -9782,7 +9794,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
97829794
// Collect mapping of IR header phis to header phi recipes, to be used in
97839795
// addScalarResumePhis.
97849796
VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
9785-
Builder);
9797+
Builder, nullptr);
97869798
for (auto &R : Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
97879799
if (isa<VPCanonicalIVPHIRecipe>(&R))
97889800
continue;

‎llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h

+12-2
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,10 @@ class VPRecipeBuilder {
9090
/// A mapping of partial reduction exit instructions to their scaling factor.
9191
DenseMap<const Instruction *, unsigned> ScaledReductionMap;
9292

93+
/// Loop versioning instance for getting noalias metadata guaranteed by
94+
/// runtime checks.
95+
LoopVersioning *LVer;
96+
9397
/// Check if \p I can be widened at the start of \p Range and possibly
9498
/// decrease the range such that the returned value holds for the entire \p
9599
/// Range. The function should not be called for memory instructions or calls.
@@ -155,9 +159,10 @@ class VPRecipeBuilder {
155159
const TargetTransformInfo *TTI,
156160
LoopVectorizationLegality *Legal,
157161
LoopVectorizationCostModel &CM,
158-
PredicatedScalarEvolution &PSE, VPBuilder &Builder)
162+
PredicatedScalarEvolution &PSE, VPBuilder &Builder,
163+
LoopVersioning *LVer)
159164
: Plan(Plan), OrigLoop(OrigLoop), TLI(TLI), TTI(TTI), Legal(Legal),
160-
CM(CM), PSE(PSE), Builder(Builder) {}
165+
CM(CM), PSE(PSE), Builder(Builder), LVer(LVer) {}
161166

162167
std::optional<unsigned> getScalingForReduction(const Instruction *ExitInst) {
163168
auto It = ScaledReductionMap.find(ExitInst);
@@ -233,6 +238,11 @@ class VPRecipeBuilder {
233238
}
234239
return Plan.getOrAddLiveIn(V);
235240
}
241+
242+
/// Returns the metatadata that can be preserved from the original instruction
243+
/// \p I, including noalias metadata guaranteed by runtime checks.
244+
SmallVector<std::pair<unsigned, MDNode *>>
245+
getMetadataToPropagate(Instruction *I) const;
236246
};
237247
} // end namespace llvm
238248

‎llvm/lib/Transforms/Vectorize/VPlan.cpp

+2-10
Original file line numberDiff line numberDiff line change
@@ -220,8 +220,8 @@ VPTransformState::VPTransformState(const TargetTransformInfo *TTI,
220220
InnerLoopVectorizer *ILV, VPlan *Plan,
221221
Loop *CurrentParentLoop, Type *CanonicalIVTy)
222222
: TTI(TTI), VF(VF), CFG(DT), LI(LI), Builder(Builder), ILV(ILV), Plan(Plan),
223-
CurrentParentLoop(CurrentParentLoop), LVer(nullptr),
224-
TypeAnalysis(CanonicalIVTy), VPDT(*Plan) {}
223+
CurrentParentLoop(CurrentParentLoop), TypeAnalysis(CanonicalIVTy),
224+
VPDT(*Plan) {}
225225

226226
Value *VPTransformState::get(const VPValue *Def, const VPLane &Lane) {
227227
if (Def->isLiveIn())
@@ -355,14 +355,6 @@ BasicBlock *VPTransformState::CFGState::getPreheaderBBFor(VPRecipeBase *R) {
355355
return VPBB2IRBB[LoopRegion->getPreheaderVPBB()];
356356
}
357357

358-
void VPTransformState::addNewMetadata(Instruction *To,
359-
const Instruction *Orig) {
360-
// If the loop was versioned with memchecks, add the corresponding no-alias
361-
// metadata.
362-
if (LVer && isa<LoadInst, StoreInst>(Orig))
363-
LVer->annotateInstWithNoAlias(To, Orig);
364-
}
365-
366358
void VPTransformState::setDebugLocFrom(DebugLoc DL) {
367359
const DILocation *DIL = DL;
368360
// When a FSDiscriminator is enabled, we don't need to add the multiply

‎llvm/lib/Transforms/Vectorize/VPlan.h

+27-16
Original file line numberDiff line numberDiff line change
@@ -1190,6 +1190,8 @@ struct VPIRPhi : public VPIRInstruction {
11901190
#endif
11911191
};
11921192

1193+
using MDArrayRef = ArrayRef<std::pair<unsigned, MDNode *>>;
1194+
11931195
/// Helper to manage IR metadata for recipes. It filters out metadata that
11941196
/// cannot be propagated.
11951197
class VPIRMetadata {
@@ -1198,10 +1200,14 @@ class VPIRMetadata {
11981200
protected:
11991201
VPIRMetadata() {}
12001202
VPIRMetadata(Instruction &I) { getMetadataToPropagate(&I, Metadata); }
1203+
VPIRMetadata(MDArrayRef Metadata) : Metadata(Metadata) {}
12011204

12021205
public:
12031206
/// Add all metadata to \p I.
12041207
void applyMetadata(Instruction &I) const;
1208+
1209+
/// Return the IR metadata.
1210+
MDArrayRef getMetadata() const { return Metadata; }
12051211
};
12061212

12071213
/// VPWidenRecipe is a recipe for producing a widened instruction using the
@@ -2459,7 +2465,7 @@ class VPReductionEVLRecipe : public VPReductionRecipe {
24592465
/// copies of the original scalar type, one per lane, instead of producing a
24602466
/// single copy of widened type for all lanes. If the instruction is known to be
24612467
/// uniform only one copy, per lane zero, will be generated.
2462-
class VPReplicateRecipe : public VPRecipeWithIRFlags {
2468+
class VPReplicateRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
24632469
/// Indicator if only a single replica per lane is needed.
24642470
bool IsUniform;
24652471

@@ -2469,19 +2475,20 @@ class VPReplicateRecipe : public VPRecipeWithIRFlags {
24692475
public:
24702476
template <typename IterT>
24712477
VPReplicateRecipe(Instruction *I, iterator_range<IterT> Operands,
2472-
bool IsUniform, VPValue *Mask = nullptr)
2478+
bool IsUniform, VPValue *Mask = nullptr,
2479+
ArrayRef<std::pair<unsigned, MDNode *>> Metadata = {})
24732480
: VPRecipeWithIRFlags(VPDef::VPReplicateSC, Operands, *I),
2474-
IsUniform(IsUniform), IsPredicated(Mask) {
2481+
VPIRMetadata(Metadata), IsUniform(IsUniform), IsPredicated(Mask) {
24752482
if (Mask)
24762483
addOperand(Mask);
24772484
}
24782485

24792486
~VPReplicateRecipe() override = default;
24802487

24812488
VPReplicateRecipe *clone() override {
2482-
auto *Copy =
2483-
new VPReplicateRecipe(getUnderlyingInstr(), operands(), IsUniform,
2484-
isPredicated() ? getMask() : nullptr);
2489+
auto *Copy = new VPReplicateRecipe(
2490+
getUnderlyingInstr(), operands(), IsUniform,
2491+
isPredicated() ? getMask() : nullptr, getMetadata());
24852492
Copy->transferFlags(*this);
24862493
return Copy;
24872494
}
@@ -2641,8 +2648,9 @@ class VPWidenMemoryRecipe : public VPRecipeBase, public VPIRMetadata {
26412648

26422649
VPWidenMemoryRecipe(const char unsigned SC, Instruction &I,
26432650
std::initializer_list<VPValue *> Operands,
2644-
bool Consecutive, bool Reverse, DebugLoc DL)
2645-
: VPRecipeBase(SC, Operands, DL), VPIRMetadata(I), Ingredient(I),
2651+
bool Consecutive, bool Reverse, MDArrayRef Metadata,
2652+
DebugLoc DL)
2653+
: VPRecipeBase(SC, Operands, DL), VPIRMetadata(Metadata), Ingredient(I),
26462654
Consecutive(Consecutive), Reverse(Reverse) {
26472655
assert((Consecutive || !Reverse) && "Reverse implies consecutive");
26482656
}
@@ -2700,16 +2708,17 @@ class VPWidenMemoryRecipe : public VPRecipeBase, public VPIRMetadata {
27002708
/// optional mask.
27012709
struct VPWidenLoadRecipe final : public VPWidenMemoryRecipe, public VPValue {
27022710
VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask,
2703-
bool Consecutive, bool Reverse, DebugLoc DL)
2711+
bool Consecutive, bool Reverse, MDArrayRef Metadata,
2712+
DebugLoc DL)
27042713
: VPWidenMemoryRecipe(VPDef::VPWidenLoadSC, Load, {Addr}, Consecutive,
2705-
Reverse, DL),
2714+
Reverse, Metadata, DL),
27062715
VPValue(this, &Load) {
27072716
setMask(Mask);
27082717
}
27092718

27102719
VPWidenLoadRecipe *clone() override {
27112720
return new VPWidenLoadRecipe(cast<LoadInst>(Ingredient), getAddr(),
2712-
getMask(), Consecutive, Reverse,
2721+
getMask(), Consecutive, Reverse, getMetadata(),
27132722
getDebugLoc());
27142723
}
27152724

@@ -2741,7 +2750,7 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
27412750
VPWidenLoadEVLRecipe(VPWidenLoadRecipe &L, VPValue &EVL, VPValue *Mask)
27422751
: VPWidenMemoryRecipe(VPDef::VPWidenLoadEVLSC, L.getIngredient(),
27432752
{L.getAddr(), &EVL}, L.isConsecutive(),
2744-
L.isReverse(), L.getDebugLoc()),
2753+
L.isReverse(), L.getMetadata(), L.getDebugLoc()),
27452754
VPValue(this, &getIngredient()) {
27462755
setMask(Mask);
27472756
}
@@ -2778,16 +2787,17 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
27782787
/// to store to and an optional mask.
27792788
struct VPWidenStoreRecipe final : public VPWidenMemoryRecipe {
27802789
VPWidenStoreRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredVal,
2781-
VPValue *Mask, bool Consecutive, bool Reverse, DebugLoc DL)
2790+
VPValue *Mask, bool Consecutive, bool Reverse,
2791+
MDArrayRef Metadata, DebugLoc DL)
27822792
: VPWidenMemoryRecipe(VPDef::VPWidenStoreSC, Store, {Addr, StoredVal},
2783-
Consecutive, Reverse, DL) {
2793+
Consecutive, Reverse, Metadata, DL) {
27842794
setMask(Mask);
27852795
}
27862796

27872797
VPWidenStoreRecipe *clone() override {
27882798
return new VPWidenStoreRecipe(cast<StoreInst>(Ingredient), getAddr(),
27892799
getStoredValue(), getMask(), Consecutive,
2790-
Reverse, getDebugLoc());
2800+
Reverse, getMetadata(), getDebugLoc());
27912801
}
27922802

27932803
VP_CLASSOF_IMPL(VPDef::VPWidenStoreSC);
@@ -2821,7 +2831,8 @@ struct VPWidenStoreEVLRecipe final : public VPWidenMemoryRecipe {
28212831
VPWidenStoreEVLRecipe(VPWidenStoreRecipe &S, VPValue &EVL, VPValue *Mask)
28222832
: VPWidenMemoryRecipe(VPDef::VPWidenStoreEVLSC, S.getIngredient(),
28232833
{S.getAddr(), S.getStoredValue(), &EVL},
2824-
S.isConsecutive(), S.isReverse(), S.getDebugLoc()) {
2834+
S.isConsecutive(), S.isReverse(), S.getMetadata(),
2835+
S.getDebugLoc()) {
28252836
setMask(Mask);
28262837
}
28272838

‎llvm/lib/Transforms/Vectorize/VPlanHelpers.h

-15
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@ class VPBasicBlock;
3838
class VPRegionBlock;
3939
class VPlan;
4040
class Value;
41-
class LoopVersioning;
4241

4342
/// Returns a calculation for the total number of elements for a given \p VF.
4443
/// For fixed width vectors this value is a constant, whereas for scalable
@@ -283,13 +282,6 @@ struct VPTransformState {
283282
Iter->second[CacheIdx] = V;
284283
}
285284

286-
/// Add additional metadata to \p To that was not present on \p Orig.
287-
///
288-
/// Currently this is used to add the noalias annotations based on the
289-
/// inserted memchecks. Use this for instructions that are *cloned* into the
290-
/// vector loop.
291-
void addNewMetadata(Instruction *To, const Instruction *Orig);
292-
293285
/// Set the debug location in the builder using the debug location \p DL.
294286
void setDebugLocFrom(DebugLoc DL);
295287

@@ -341,13 +333,6 @@ struct VPTransformState {
341333
/// The parent loop object for the current scope, or nullptr.
342334
Loop *CurrentParentLoop = nullptr;
343335

344-
/// LoopVersioning. It's only set up (non-null) if memchecks were
345-
/// used.
346-
///
347-
/// This is currently only used to add no-alias metadata based on the
348-
/// memchecks. The actually versioning is performed manually.
349-
LoopVersioning *LVer = nullptr;
350-
351336
/// VPlan-based type analysis.
352337
VPTypeAnalysis TypeAnalysis;
353338

0 commit comments

Comments
 (0)