Skip to content

Commit eefb249

Browse files
author
git apple-llvm automerger
committed
Merge commit '5e1e83ee23fe' from llvm.org/master into apple/master
2 parents 73be380 + 5e1e83e commit eefb249

File tree

3 files changed

+50
-44
lines changed

3 files changed

+50
-44
lines changed

‎llvm/include/llvm/IR/Instructions.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5283,12 +5283,12 @@ inline Value *getPointerOperand(Value *V) {
52835283
}
52845284

52855285
/// A helper function that returns the alignment of load or store instruction.
5286-
inline unsigned getLoadStoreAlignment(Value *I) {
5286+
inline MaybeAlign getLoadStoreAlignment(Value *I) {
52875287
assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
52885288
"Expected Load or Store instruction");
52895289
if (auto *LI = dyn_cast<LoadInst>(I))
5290-
return LI->getAlignment();
5291-
return cast<StoreInst>(I)->getAlignment();
5290+
return MaybeAlign(LI->getAlignment());
5291+
return MaybeAlign(cast<StoreInst>(I)->getAlignment());
52925292
}
52935293

52945294
/// A helper function that returns the address space of the pointer operand of

‎llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -741,9 +741,9 @@ bool LoopVectorizationLegality::canVectorizeInstrs() {
741741
// Arbitrarily try a vector of 2 elements.
742742
Type *VecTy = VectorType::get(T, /*NumElements=*/2);
743743
assert(VecTy && "did not find vectorized version of stored type");
744-
unsigned Alignment = getLoadStoreAlignment(ST);
744+
const MaybeAlign Alignment = getLoadStoreAlignment(ST);
745745
assert(Alignment && "Alignment should be set");
746-
if (!TTI->isLegalNTStore(VecTy, Align(Alignment))) {
746+
if (!TTI->isLegalNTStore(VecTy, *Alignment)) {
747747
reportVectorizationFailure(
748748
"nontemporal store instruction cannot be vectorized",
749749
"nontemporal store instruction cannot be vectorized",
@@ -758,9 +758,9 @@ bool LoopVectorizationLegality::canVectorizeInstrs() {
758758
// supported on the target (arbitrarily try a vector of 2 elements).
759759
Type *VecTy = VectorType::get(I.getType(), /*NumElements=*/2);
760760
assert(VecTy && "did not find vectorized version of load type");
761-
unsigned Alignment = getLoadStoreAlignment(LD);
761+
const MaybeAlign Alignment = getLoadStoreAlignment(LD);
762762
assert(Alignment && "Alignment should be set");
763-
if (!TTI->isLegalNTLoad(VecTy, Align(Alignment))) {
763+
if (!TTI->isLegalNTLoad(VecTy, *Alignment)) {
764764
reportVectorizationFailure(
765765
"nontemporal load instruction cannot be vectorized",
766766
"nontemporal load instruction cannot be vectorized",

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 43 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1190,16 +1190,16 @@ class LoopVectorizationCostModel {
11901190

11911191
/// Returns true if the target machine supports masked store operation
11921192
/// for the given \p DataType and kind of access to \p Ptr.
1193-
bool isLegalMaskedStore(Type *DataType, Value *Ptr, unsigned Alignment) {
1193+
bool isLegalMaskedStore(Type *DataType, Value *Ptr, MaybeAlign Alignment) {
11941194
return Legal->isConsecutivePtr(Ptr) &&
1195-
TTI.isLegalMaskedStore(DataType, MaybeAlign(Alignment));
1195+
TTI.isLegalMaskedStore(DataType, Alignment);
11961196
}
11971197

11981198
/// Returns true if the target machine supports masked load operation
11991199
/// for the given \p DataType and kind of access to \p Ptr.
1200-
bool isLegalMaskedLoad(Type *DataType, Value *Ptr, unsigned Alignment) {
1200+
bool isLegalMaskedLoad(Type *DataType, Value *Ptr, MaybeAlign Alignment) {
12011201
return Legal->isConsecutivePtr(Ptr) &&
1202-
TTI.isLegalMaskedLoad(DataType, MaybeAlign(Alignment));
1202+
TTI.isLegalMaskedLoad(DataType, Alignment);
12031203
}
12041204

12051205
/// Returns true if the target machine supports masked scatter operation
@@ -2359,12 +2359,11 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr,
23592359
Type *ScalarDataTy = getMemInstValueType(Instr);
23602360
Type *DataTy = VectorType::get(ScalarDataTy, VF);
23612361
Value *Ptr = getLoadStorePointerOperand(Instr);
2362-
unsigned Alignment = getLoadStoreAlignment(Instr);
23632362
// An alignment of 0 means target abi alignment. We need to use the scalar's
23642363
// target abi alignment in such a case.
23652364
const DataLayout &DL = Instr->getModule()->getDataLayout();
2366-
if (!Alignment)
2367-
Alignment = DL.getABITypeAlignment(ScalarDataTy);
2365+
const Align Alignment =
2366+
DL.getValueOrABITypeAlignment(getLoadStoreAlignment(Instr), ScalarDataTy);
23682367
unsigned AddressSpace = getLoadStoreAddressSpace(Instr);
23692368

23702369
// Determine if the pointer operand of the access is either consecutive or
@@ -2428,8 +2427,8 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr,
24282427
if (CreateGatherScatter) {
24292428
Value *MaskPart = isMaskRequired ? Mask[Part] : nullptr;
24302429
Value *VectorGep = getOrCreateVectorValue(Ptr, Part);
2431-
NewSI = Builder.CreateMaskedScatter(StoredVal, VectorGep, Alignment,
2432-
MaskPart);
2430+
NewSI = Builder.CreateMaskedScatter(StoredVal, VectorGep,
2431+
Alignment.value(), MaskPart);
24332432
} else {
24342433
if (Reverse) {
24352434
// If we store to reverse consecutive memory locations, then we need
@@ -2440,10 +2439,11 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr,
24402439
}
24412440
auto *VecPtr = CreateVecPtr(Part, Ptr);
24422441
if (isMaskRequired)
2443-
NewSI = Builder.CreateMaskedStore(StoredVal, VecPtr, Alignment,
2444-
Mask[Part]);
2442+
NewSI = Builder.CreateMaskedStore(StoredVal, VecPtr,
2443+
Alignment.value(), Mask[Part]);
24452444
else
2446-
NewSI = Builder.CreateAlignedStore(StoredVal, VecPtr, Alignment);
2445+
NewSI =
2446+
Builder.CreateAlignedStore(StoredVal, VecPtr, Alignment.value());
24472447
}
24482448
addMetadata(NewSI, SI);
24492449
}
@@ -2458,18 +2458,18 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr,
24582458
if (CreateGatherScatter) {
24592459
Value *MaskPart = isMaskRequired ? Mask[Part] : nullptr;
24602460
Value *VectorGep = getOrCreateVectorValue(Ptr, Part);
2461-
NewLI = Builder.CreateMaskedGather(VectorGep, Alignment, MaskPart,
2461+
NewLI = Builder.CreateMaskedGather(VectorGep, Alignment.value(), MaskPart,
24622462
nullptr, "wide.masked.gather");
24632463
addMetadata(NewLI, LI);
24642464
} else {
24652465
auto *VecPtr = CreateVecPtr(Part, Ptr);
24662466
if (isMaskRequired)
2467-
NewLI = Builder.CreateMaskedLoad(VecPtr, Alignment, Mask[Part],
2467+
NewLI = Builder.CreateMaskedLoad(VecPtr, Alignment.value(), Mask[Part],
24682468
UndefValue::get(DataTy),
24692469
"wide.masked.load");
24702470
else
2471-
NewLI =
2472-
Builder.CreateAlignedLoad(DataTy, VecPtr, Alignment, "wide.load");
2471+
NewLI = Builder.CreateAlignedLoad(DataTy, VecPtr, Alignment.value(),
2472+
"wide.load");
24732473

24742474
// Add metadata to the load, but setVectorValue to the reverse shuffle.
24752475
addMetadata(NewLI, LI);
@@ -4553,7 +4553,6 @@ bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I, unsigne
45534553
return false;
45544554
auto *Ptr = getLoadStorePointerOperand(I);
45554555
auto *Ty = getMemInstValueType(I);
4556-
unsigned Alignment = getLoadStoreAlignment(I);
45574556
// We have already decided how to vectorize this instruction, get that
45584557
// result.
45594558
if (VF > 1) {
@@ -4562,6 +4561,7 @@ bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I, unsigne
45624561
"Widening decision should be ready at this moment");
45634562
return WideningDecision == CM_Scalarize;
45644563
}
4564+
const MaybeAlign Alignment = getLoadStoreAlignment(I);
45654565
return isa<LoadInst>(I) ?
45664566
!(isLegalMaskedLoad(Ty, Ptr, Alignment) || isLegalMaskedGather(Ty))
45674567
: !(isLegalMaskedStore(Ty, Ptr, Alignment) || isLegalMaskedScatter(Ty));
@@ -4607,9 +4607,9 @@ bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(Instruction *I,
46074607
"Masked interleave-groups for predicated accesses are not enabled.");
46084608

46094609
auto *Ty = getMemInstValueType(I);
4610-
unsigned Alignment = getLoadStoreAlignment(I);
4611-
return isa<LoadInst>(I) ? TTI.isLegalMaskedLoad(Ty, MaybeAlign(Alignment))
4612-
: TTI.isLegalMaskedStore(Ty, MaybeAlign(Alignment));
4610+
const MaybeAlign Alignment = getLoadStoreAlignment(I);
4611+
return isa<LoadInst>(I) ? TTI.isLegalMaskedLoad(Ty, Alignment)
4612+
: TTI.isLegalMaskedStore(Ty, Alignment);
46134613
}
46144614

46154615
bool LoopVectorizationCostModel::memoryInstructionCanBeWidened(Instruction *I,
@@ -5731,7 +5731,6 @@ unsigned LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,
57315731
Type *ValTy = getMemInstValueType(I);
57325732
auto SE = PSE.getSE();
57335733

5734-
unsigned Alignment = getLoadStoreAlignment(I);
57355734
unsigned AS = getLoadStoreAddressSpace(I);
57365735
Value *Ptr = getLoadStorePointerOperand(I);
57375736
Type *PtrTy = ToVectorTy(Ptr->getType(), VF);
@@ -5745,9 +5744,9 @@ unsigned LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,
57455744

57465745
// Don't pass *I here, since it is scalar but will actually be part of a
57475746
// vectorized loop where the user of it is a vectorized instruction.
5748-
Cost += VF *
5749-
TTI.getMemoryOpCost(I->getOpcode(), ValTy->getScalarType(), Alignment,
5750-
AS);
5747+
const MaybeAlign Alignment = getLoadStoreAlignment(I);
5748+
Cost += VF * TTI.getMemoryOpCost(I->getOpcode(), ValTy->getScalarType(),
5749+
Alignment ? Alignment->value() : 0, AS);
57515750

57525751
// Get the overhead of the extractelement and insertelement instructions
57535752
// we might create due to scalarization.
@@ -5772,18 +5771,20 @@ unsigned LoopVectorizationCostModel::getConsecutiveMemOpCost(Instruction *I,
57725771
unsigned VF) {
57735772
Type *ValTy = getMemInstValueType(I);
57745773
Type *VectorTy = ToVectorTy(ValTy, VF);
5775-
unsigned Alignment = getLoadStoreAlignment(I);
57765774
Value *Ptr = getLoadStorePointerOperand(I);
57775775
unsigned AS = getLoadStoreAddressSpace(I);
57785776
int ConsecutiveStride = Legal->isConsecutivePtr(Ptr);
57795777

57805778
assert((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&
57815779
"Stride should be 1 or -1 for consecutive memory access");
5780+
const MaybeAlign Alignment = getLoadStoreAlignment(I);
57825781
unsigned Cost = 0;
57835782
if (Legal->isMaskRequired(I))
5784-
Cost += TTI.getMaskedMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS);
5783+
Cost += TTI.getMaskedMemoryOpCost(I->getOpcode(), VectorTy,
5784+
Alignment ? Alignment->value() : 0, AS);
57855785
else
5786-
Cost += TTI.getMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS, I);
5786+
Cost += TTI.getMemoryOpCost(I->getOpcode(), VectorTy,
5787+
Alignment ? Alignment->value() : 0, AS, I);
57875788

57885789
bool Reverse = ConsecutiveStride < 0;
57895790
if (Reverse)
@@ -5795,33 +5796,37 @@ unsigned LoopVectorizationCostModel::getUniformMemOpCost(Instruction *I,
57955796
unsigned VF) {
57965797
Type *ValTy = getMemInstValueType(I);
57975798
Type *VectorTy = ToVectorTy(ValTy, VF);
5798-
unsigned Alignment = getLoadStoreAlignment(I);
5799+
const MaybeAlign Alignment = getLoadStoreAlignment(I);
57995800
unsigned AS = getLoadStoreAddressSpace(I);
58005801
if (isa<LoadInst>(I)) {
58015802
return TTI.getAddressComputationCost(ValTy) +
5802-
TTI.getMemoryOpCost(Instruction::Load, ValTy, Alignment, AS) +
5803+
TTI.getMemoryOpCost(Instruction::Load, ValTy,
5804+
Alignment ? Alignment->value() : 0, AS) +
58035805
TTI.getShuffleCost(TargetTransformInfo::SK_Broadcast, VectorTy);
58045806
}
58055807
StoreInst *SI = cast<StoreInst>(I);
58065808

58075809
bool isLoopInvariantStoreValue = Legal->isUniform(SI->getValueOperand());
58085810
return TTI.getAddressComputationCost(ValTy) +
5809-
TTI.getMemoryOpCost(Instruction::Store, ValTy, Alignment, AS) +
5810-
(isLoopInvariantStoreValue ? 0 : TTI.getVectorInstrCost(
5811-
Instruction::ExtractElement,
5812-
VectorTy, VF - 1));
5811+
TTI.getMemoryOpCost(Instruction::Store, ValTy,
5812+
Alignment ? Alignment->value() : 0, AS) +
5813+
(isLoopInvariantStoreValue
5814+
? 0
5815+
: TTI.getVectorInstrCost(Instruction::ExtractElement, VectorTy,
5816+
VF - 1));
58135817
}
58145818

58155819
unsigned LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
58165820
unsigned VF) {
58175821
Type *ValTy = getMemInstValueType(I);
58185822
Type *VectorTy = ToVectorTy(ValTy, VF);
5819-
unsigned Alignment = getLoadStoreAlignment(I);
5823+
const MaybeAlign Alignment = getLoadStoreAlignment(I);
58205824
Value *Ptr = getLoadStorePointerOperand(I);
58215825

58225826
return TTI.getAddressComputationCost(VectorTy) +
58235827
TTI.getGatherScatterOpCost(I->getOpcode(), VectorTy, Ptr,
5824-
Legal->isMaskRequired(I), Alignment);
5828+
Legal->isMaskRequired(I),
5829+
Alignment ? Alignment->value() : 0);
58255830
}
58265831

58275832
unsigned LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
@@ -5868,11 +5873,12 @@ unsigned LoopVectorizationCostModel::getMemoryInstructionCost(Instruction *I,
58685873
// moment.
58695874
if (VF == 1) {
58705875
Type *ValTy = getMemInstValueType(I);
5871-
unsigned Alignment = getLoadStoreAlignment(I);
5876+
const MaybeAlign Alignment = getLoadStoreAlignment(I);
58725877
unsigned AS = getLoadStoreAddressSpace(I);
58735878

58745879
return TTI.getAddressComputationCost(ValTy) +
5875-
TTI.getMemoryOpCost(I->getOpcode(), ValTy, Alignment, AS, I);
5880+
TTI.getMemoryOpCost(I->getOpcode(), ValTy,
5881+
Alignment ? Alignment->value() : 0, AS, I);
58765882
}
58775883
return getWideningCost(I, VF);
58785884
}

0 commit comments

Comments
 (0)