Skip to content

Commit 9e462b7

Browse files
ritter-x2aritter-x2a
authored
[LowerMemIntrinsics][NFC] Use Align in TTI::getMemcpyLoopLoweringType (#100984)
...and also in TTI::getMemcpyLoopResidualLoweringType.
1 parent d1f3a92 commit 9e462b7

File tree

6 files changed

+34
-35
lines changed

6 files changed

+34
-35
lines changed

‎llvm/include/llvm/Analysis/TargetTransformInfo.h‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1587,7 +1587,7 @@ class TargetTransformInfo {
15871587
/// \returns The type to use in a loop expansion of a memcpy call.
15881588
Type *getMemcpyLoopLoweringType(
15891589
LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
1590-
unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign,
1590+
unsigned DestAddrSpace, Align SrcAlign, Align DestAlign,
15911591
std::optional<uint32_t> AtomicElementSize = std::nullopt) const;
15921592

15931593
/// \param[out] OpsOut The operand types to copy RemainingBytes of memory.
@@ -1599,7 +1599,7 @@ class TargetTransformInfo {
15991599
void getMemcpyLoopResidualLoweringType(
16001600
SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
16011601
unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
1602-
unsigned SrcAlign, unsigned DestAlign,
1602+
Align SrcAlign, Align DestAlign,
16031603
std::optional<uint32_t> AtomicCpySize = std::nullopt) const;
16041604

16051605
/// \returns True if the two functions have compatible attributes for inlining
@@ -2133,13 +2133,13 @@ class TargetTransformInfo::Concept {
21332133
Type *ExpectedType) = 0;
21342134
virtual Type *getMemcpyLoopLoweringType(
21352135
LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
2136-
unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign,
2136+
unsigned DestAddrSpace, Align SrcAlign, Align DestAlign,
21372137
std::optional<uint32_t> AtomicElementSize) const = 0;
21382138

21392139
virtual void getMemcpyLoopResidualLoweringType(
21402140
SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
21412141
unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
2142-
unsigned SrcAlign, unsigned DestAlign,
2142+
Align SrcAlign, Align DestAlign,
21432143
std::optional<uint32_t> AtomicCpySize) const = 0;
21442144
virtual bool areInlineCompatible(const Function *Caller,
21452145
const Function *Callee) const = 0;
@@ -2838,7 +2838,7 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
28382838
}
28392839
Type *getMemcpyLoopLoweringType(
28402840
LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
2841-
unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign,
2841+
unsigned DestAddrSpace, Align SrcAlign, Align DestAlign,
28422842
std::optional<uint32_t> AtomicElementSize) const override {
28432843
return Impl.getMemcpyLoopLoweringType(Context, Length, SrcAddrSpace,
28442844
DestAddrSpace, SrcAlign, DestAlign,
@@ -2847,7 +2847,7 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
28472847
void getMemcpyLoopResidualLoweringType(
28482848
SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
28492849
unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
2850-
unsigned SrcAlign, unsigned DestAlign,
2850+
Align SrcAlign, Align DestAlign,
28512851
std::optional<uint32_t> AtomicCpySize) const override {
28522852
Impl.getMemcpyLoopResidualLoweringType(OpsOut, Context, RemainingBytes,
28532853
SrcAddrSpace, DestAddrSpace,

‎llvm/include/llvm/Analysis/TargetTransformInfoImpl.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -839,7 +839,7 @@ class TargetTransformInfoImplBase {
839839
Type *
840840
getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length,
841841
unsigned SrcAddrSpace, unsigned DestAddrSpace,
842-
unsigned SrcAlign, unsigned DestAlign,
842+
Align SrcAlign, Align DestAlign,
843843
std::optional<uint32_t> AtomicElementSize) const {
844844
return AtomicElementSize ? Type::getIntNTy(Context, *AtomicElementSize * 8)
845845
: Type::getInt8Ty(Context);
@@ -848,7 +848,7 @@ class TargetTransformInfoImplBase {
848848
void getMemcpyLoopResidualLoweringType(
849849
SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
850850
unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
851-
unsigned SrcAlign, unsigned DestAlign,
851+
Align SrcAlign, Align DestAlign,
852852
std::optional<uint32_t> AtomicCpySize) const {
853853
unsigned OpSizeInBytes = AtomicCpySize ? *AtomicCpySize : 1;
854854
Type *OpType = Type::getIntNTy(Context, OpSizeInBytes * 8);

‎llvm/lib/Analysis/TargetTransformInfo.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1198,7 +1198,7 @@ Value *TargetTransformInfo::getOrCreateResultFromMemIntrinsic(
11981198

11991199
Type *TargetTransformInfo::getMemcpyLoopLoweringType(
12001200
LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
1201-
unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign,
1201+
unsigned DestAddrSpace, Align SrcAlign, Align DestAlign,
12021202
std::optional<uint32_t> AtomicElementSize) const {
12031203
return TTIImpl->getMemcpyLoopLoweringType(Context, Length, SrcAddrSpace,
12041204
DestAddrSpace, SrcAlign, DestAlign,
@@ -1208,7 +1208,7 @@ Type *TargetTransformInfo::getMemcpyLoopLoweringType(
12081208
void TargetTransformInfo::getMemcpyLoopResidualLoweringType(
12091209
SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
12101210
unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
1211-
unsigned SrcAlign, unsigned DestAlign,
1211+
Align SrcAlign, Align DestAlign,
12121212
std::optional<uint32_t> AtomicCpySize) const {
12131213
TTIImpl->getMemcpyLoopResidualLoweringType(
12141214
OpsOut, Context, RemainingBytes, SrcAddrSpace, DestAddrSpace, SrcAlign,

‎llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -418,19 +418,19 @@ int64_t GCNTTIImpl::getMaxMemIntrinsicInlineSizeThreshold() const {
418418
// FIXME: This could use fine tuning and microbenchmarks.
419419
Type *GCNTTIImpl::getMemcpyLoopLoweringType(
420420
LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
421-
unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign,
421+
unsigned DestAddrSpace, Align SrcAlign, Align DestAlign,
422422
std::optional<uint32_t> AtomicElementSize) const {
423423

424424
if (AtomicElementSize)
425425
return Type::getIntNTy(Context, *AtomicElementSize * 8);
426426

427-
unsigned MinAlign = std::min(SrcAlign, DestAlign);
427+
Align MinAlign = std::min(SrcAlign, DestAlign);
428428

429429
// A (multi-)dword access at an address == 2 (mod 4) will be decomposed by the
430430
// hardware into byte accesses. If you assume all alignments are equally
431431
// probable, it's more efficient on average to use short accesses for this
432432
// case.
433-
if (MinAlign == 2)
433+
if (MinAlign == Align(2))
434434
return Type::getInt16Ty(Context);
435435

436436
// Not all subtargets have 128-bit DS instructions, and we currently don't
@@ -450,7 +450,7 @@ Type *GCNTTIImpl::getMemcpyLoopLoweringType(
450450
void GCNTTIImpl::getMemcpyLoopResidualLoweringType(
451451
SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
452452
unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
453-
unsigned SrcAlign, unsigned DestAlign,
453+
Align SrcAlign, Align DestAlign,
454454
std::optional<uint32_t> AtomicCpySize) const {
455455
assert(RemainingBytes < 16);
456456

@@ -459,9 +459,9 @@ void GCNTTIImpl::getMemcpyLoopResidualLoweringType(
459459
OpsOut, Context, RemainingBytes, SrcAddrSpace, DestAddrSpace, SrcAlign,
460460
DestAlign, AtomicCpySize);
461461

462-
unsigned MinAlign = std::min(SrcAlign, DestAlign);
462+
Align MinAlign = std::min(SrcAlign, DestAlign);
463463

464-
if (MinAlign != 2) {
464+
if (MinAlign != Align(2)) {
465465
Type *I64Ty = Type::getInt64Ty(Context);
466466
while (RemainingBytes >= 8) {
467467
OpsOut.push_back(I64Ty);

‎llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -137,15 +137,16 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
137137
unsigned AddrSpace) const;
138138

139139
int64_t getMaxMemIntrinsicInlineSizeThreshold() const;
140-
Type *getMemcpyLoopLoweringType(
141-
LLVMContext & Context, Value * Length, unsigned SrcAddrSpace,
142-
unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign,
143-
std::optional<uint32_t> AtomicElementSize) const;
140+
Type *
141+
getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length,
142+
unsigned SrcAddrSpace, unsigned DestAddrSpace,
143+
Align SrcAlign, Align DestAlign,
144+
std::optional<uint32_t> AtomicElementSize) const;
144145

145146
void getMemcpyLoopResidualLoweringType(
146147
SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
147148
unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
148-
unsigned SrcAlign, unsigned DestAlign,
149+
Align SrcAlign, Align DestAlign,
149150
std::optional<uint32_t> AtomicCpySize) const;
150151
unsigned getMaxInterleaveFactor(ElementCount VF);
151152

‎llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp‎

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,7 @@ void llvm::createMemCpyLoopKnownSize(
4545

4646
Type *TypeOfCopyLen = CopyLen->getType();
4747
Type *LoopOpType = TTI.getMemcpyLoopLoweringType(
48-
Ctx, CopyLen, SrcAS, DstAS, SrcAlign.value(), DstAlign.value(),
49-
AtomicElementSize);
48+
Ctx, CopyLen, SrcAS, DstAS, SrcAlign, DstAlign, AtomicElementSize);
5049
assert((!AtomicElementSize || !LoopOpType->isVectorTy()) &&
5150
"Atomic memcpy lowering is not supported for vector operand type");
5251

@@ -111,8 +110,8 @@ void llvm::createMemCpyLoopKnownSize(
111110

112111
SmallVector<Type *, 5> RemainingOps;
113112
TTI.getMemcpyLoopResidualLoweringType(RemainingOps, Ctx, RemainingBytes,
114-
SrcAS, DstAS, SrcAlign.value(),
115-
DstAlign.value(), AtomicElementSize);
113+
SrcAS, DstAS, SrcAlign, DstAlign,
114+
AtomicElementSize);
116115

117116
for (auto *OpTy : RemainingOps) {
118117
Align PartSrcAlign(commonAlignment(SrcAlign, BytesCopied));
@@ -197,8 +196,7 @@ void llvm::createMemCpyLoopUnknownSize(
197196
unsigned DstAS = cast<PointerType>(DstAddr->getType())->getAddressSpace();
198197

199198
Type *LoopOpType = TTI.getMemcpyLoopLoweringType(
200-
Ctx, CopyLen, SrcAS, DstAS, SrcAlign.value(), DstAlign.value(),
201-
AtomicElementSize);
199+
Ctx, CopyLen, SrcAS, DstAS, SrcAlign, DstAlign, AtomicElementSize);
202200
assert((!AtomicElementSize || !LoopOpType->isVectorTy()) &&
203201
"Atomic memcpy lowering is not supported for vector operand type");
204202
unsigned LoopOpSize = DL.getTypeStoreSize(LoopOpType);
@@ -411,8 +409,8 @@ static void createMemMoveLoopUnknownSize(Instruction *InsertBefore,
411409
unsigned SrcAS = cast<PointerType>(SrcAddr->getType())->getAddressSpace();
412410
unsigned DstAS = cast<PointerType>(DstAddr->getType())->getAddressSpace();
413411

414-
Type *LoopOpType = TTI.getMemcpyLoopLoweringType(
415-
Ctx, CopyLen, SrcAS, DstAS, SrcAlign.value(), DstAlign.value());
412+
Type *LoopOpType = TTI.getMemcpyLoopLoweringType(Ctx, CopyLen, SrcAS, DstAS,
413+
SrcAlign, DstAlign);
416414
unsigned LoopOpSize = DL.getTypeStoreSize(LoopOpType);
417415
Type *Int8Type = Type::getInt8Ty(Ctx);
418416
bool LoopOpIsInt8 = LoopOpType == Int8Type;
@@ -668,8 +666,8 @@ static void createMemMoveLoopKnownSize(Instruction *InsertBefore,
668666
unsigned SrcAS = cast<PointerType>(SrcAddr->getType())->getAddressSpace();
669667
unsigned DstAS = cast<PointerType>(DstAddr->getType())->getAddressSpace();
670668

671-
Type *LoopOpType = TTI.getMemcpyLoopLoweringType(
672-
Ctx, CopyLen, SrcAS, DstAS, SrcAlign.value(), DstAlign.value());
669+
Type *LoopOpType = TTI.getMemcpyLoopLoweringType(Ctx, CopyLen, SrcAS, DstAS,
670+
SrcAlign, DstAlign);
673671
unsigned LoopOpSize = DL.getTypeStoreSize(LoopOpType);
674672

675673
// Calculate the loop trip count and remaining bytes to copy after the loop.
@@ -737,8 +735,8 @@ static void createMemMoveLoopKnownSize(Instruction *InsertBefore,
737735
IRBuilder<> BwdResBuilder(CopyBackwardsBB->getFirstNonPHI());
738736
SmallVector<Type *, 5> RemainingOps;
739737
TTI.getMemcpyLoopResidualLoweringType(RemainingOps, Ctx, RemainingBytes,
740-
SrcAS, DstAS, PartSrcAlign.value(),
741-
PartDstAlign.value());
738+
SrcAS, DstAS, PartSrcAlign,
739+
PartDstAlign);
742740
for (auto *OpTy : RemainingOps) {
743741
// reverse the order of the emitted operations
744742
BwdResBuilder.SetInsertPoint(CopyBackwardsBB->getFirstNonPHI());
@@ -818,8 +816,8 @@ static void createMemMoveLoopKnownSize(Instruction *InsertBefore,
818816
IRBuilder<> FwdResBuilder(FwdResidualBB->getTerminator());
819817
SmallVector<Type *, 5> RemainingOps;
820818
TTI.getMemcpyLoopResidualLoweringType(RemainingOps, Ctx, RemainingBytes,
821-
SrcAS, DstAS, PartSrcAlign.value(),
822-
PartDstAlign.value());
819+
SrcAS, DstAS, PartSrcAlign,
820+
PartDstAlign);
823821
for (auto *OpTy : RemainingOps)
824822
GenerateResidualLdStPair(OpTy, FwdResBuilder, BytesCopied);
825823
}

0 commit comments

Comments
 (0)