Skip to content

Commit 6426d94

Browse files
jaladreipsigcbot
jaladreips
authored and
igcbot
committed
Refactor MergeAllocas
Refactor MergeAllocas
1 parent 704d392 commit 6426d94

32 files changed

+884
-863
lines changed

‎IGC/AdaptorCommon/LivenessUtils/AllocationLivenessAnalyzer.cpp

Lines changed: 404 additions & 0 deletions
Large diffs are not rendered by default.

‎IGC/AdaptorCommon/LivenessUtils/AllocationLivenessAnalyzer.h

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/*========================== begin_copyright_notice ============================
2+
3+
Copyright (C) 2025 Intel Corporation
4+
5+
SPDX-License-Identifier: MIT
6+
7+
============================= end_copyright_notice ===========================*/
8+
9+
#include "common/LLVMWarningsPush.hpp"
10+
#include <llvm/IR/Instructions.h>
11+
#include <llvm/Pass.h>
12+
#include <llvm/ADT/DenseSet.h>
13+
#include <llvm/ADT/SetVector.h>
14+
#include "common/LLVMWarningsPop.hpp"
15+
16+
namespace llvm {
17+
class BasicBlock;
18+
class DominatorTree;
19+
class Instruction;
20+
class LoopInfo;
21+
} // namespace llvm
22+
23+
namespace IGC
24+
{
25+
class AllocationLivenessAnalyzer : public llvm::FunctionPass
26+
{
27+
public:
28+
struct LivenessData {
29+
llvm::Instruction* lifetimeStart = nullptr;
30+
llvm::SmallVector<llvm::Instruction*> lifetimeEnds;
31+
32+
llvm::DenseSet<llvm::BasicBlock*> bbIn;
33+
llvm::DenseSet<llvm::BasicBlock*> bbOut;
34+
35+
LivenessData(
36+
llvm::Instruction* allocationInstruction,
37+
llvm::SetVector<llvm::Instruction*>&& usersOfAllocation,
38+
const llvm::LoopInfo& LI,
39+
const llvm::DominatorTree& DT,
40+
llvm::BasicBlock* userDominatorBlock = nullptr,
41+
llvm::SetVector<llvm::Instruction*>&& lifetimeLeakingUsers = {}
42+
);
43+
44+
bool OverlapsWith(const LivenessData& LD) const;
45+
bool ContainsInstruction(const llvm::Instruction& I) const;
46+
};
47+
48+
AllocationLivenessAnalyzer(char& pid) : llvm::FunctionPass(pid) {}
49+
50+
protected:
51+
LivenessData ProcessInstruction(llvm::Instruction* I);
52+
53+
void getAnalysisUsage(llvm::AnalysisUsage& AU) const override;
54+
virtual void getAdditionalAnalysisUsage(llvm::AnalysisUsage& AU) const = 0;
55+
};
56+
} // namespace IGC

‎IGC/AdaptorCommon/LivenessUtils/MergeAllocas.cpp

Lines changed: 257 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,257 @@
1+
/*========================== begin_copyright_notice ============================
2+
3+
Copyright (C) 2024 Intel Corporation
4+
5+
SPDX-License-Identifier: MIT
6+
7+
============================= end_copyright_notice ===========================*/
8+
9+
#include "MergeAllocas.h"
10+
#include "Compiler/IGCPassSupport.h"
11+
#include "common/igc_regkeys.hpp"
12+
#include "Probe/Assertion.h"
13+
#include "debug/DebugMacros.hpp"
14+
15+
#include "common/LLVMWarningsPush.hpp"
16+
#include <llvm/ADT/SetOperations.h>
17+
#include <llvm/ADT/SetVector.h>
18+
#include <llvm/ADT/SmallSet.h>
19+
#include <llvm/ADT/SmallVector.h>
20+
#include <llvm/Analysis/LoopInfo.h>
21+
#include <llvm/IR/Constants.h>
22+
#include <llvm/IR/DataLayout.h>
23+
#include <llvm/IR/DerivedTypes.h>
24+
#include <llvm/IR/Dominators.h>
25+
#include <llvm/IR/Function.h>
26+
#include <llvm/IR/IRBuilder.h>
27+
#include <llvm/IR/InstIterator.h>
28+
#include <llvm/IR/Instructions.h>
29+
#include "common/LLVMWarningsPop.hpp"
30+
31+
using namespace llvm;
32+
using namespace IGC;
33+
34+
// Get size of bytes allocated for type including padding.
35+
static size_t GetByteSize(Type *T, const DataLayout *DL) {
36+
if (T->isSized())
37+
return static_cast<size_t>(DL->getTypeAllocSize(T));
38+
return 0;
39+
}
40+
41+
static MergeAllocas::AllocaInfo GetAllocaInfo(AllocaInst *allocaI,
42+
AllocationLivenessAnalyzer::LivenessData *LD,
43+
const DataLayout *DL) {
44+
size_t allocationSize = GetByteSize(allocaI->getAllocatedType(), DL);
45+
return {{},
46+
allocaI,
47+
LD,
48+
allocaI->getAddressSpace(),
49+
allocationSize,
50+
allocationSize,
51+
static_cast<size_t>(
52+
DL->getPrefTypeAlign(allocaI->getAllocatedType()).value()),
53+
0};
54+
}
55+
56+
static size_t GetStartingOffset(size_t startOffset, size_t alignment) {
57+
size_t remainder = startOffset % alignment;
58+
if (remainder == 0) {
59+
return startOffset;
60+
}
61+
return startOffset + (alignment - remainder);
62+
}
63+
64+
static bool AddNonOverlappingAlloca(MergeAllocas::AllocaInfo* MergableAlloca,
65+
MergeAllocas::AllocaInfo* NewAlloca) {
66+
if (MergableAlloca->addressSpace != NewAlloca->addressSpace) {
67+
return false;
68+
}
69+
if (MergableAlloca->allocationSize < NewAlloca->allocationSize) {
70+
return false;
71+
}
72+
if (MergableAlloca->livenessData->OverlapsWith(*NewAlloca->livenessData)) {
73+
return false;
74+
}
75+
76+
if (IGC_IS_FLAG_ENABLED(DisableMergingOfAllocasWithDifferentType))
77+
{
78+
auto* AllocatedType = MergableAlloca->allocaI->getAllocatedType();
79+
auto* AllocatedTypeNew = NewAlloca->allocaI->getAllocatedType();
80+
bool IsArray = AllocatedType->isArrayTy()? true : false;
81+
bool IsArrayNew = AllocatedTypeNew->isArrayTy()? true : false;
82+
bool AreBothArrays = IsArray && IsArrayNew;
83+
if (AreBothArrays && AllocatedType->getArrayElementType() != AllocatedTypeNew->getArrayElementType()) {
84+
return false;
85+
}
86+
if (!AreBothArrays && AllocatedType != AllocatedTypeNew) {
87+
return false;
88+
}
89+
}
90+
91+
// Check if we can merge alloca to one of existing non-overlapping allocas.
92+
for (auto *NonOverlappingAlloca : MergableAlloca->nonOverlapingAllocas) {
93+
bool added = AddNonOverlappingAlloca(NonOverlappingAlloca, NewAlloca);
94+
if (added) {
95+
return true;
96+
}
97+
}
98+
99+
// Check if we have still space in existing alloca to add new alloca
100+
if (MergableAlloca->remainingSize >= NewAlloca->allocationSize) {
101+
size_t currentOffset =
102+
MergableAlloca->allocationSize - MergableAlloca->remainingSize;
103+
size_t newStartingOffset =
104+
GetStartingOffset(currentOffset, NewAlloca->alignment);
105+
size_t sizeWithPadding =
106+
NewAlloca->allocationSize + (newStartingOffset - currentOffset);
107+
// When adding alignment in consideration we can't fit new alloca.
108+
if (sizeWithPadding > MergableAlloca->remainingSize) {
109+
return false;
110+
}
111+
size_t newAllocaOffset = newStartingOffset + MergableAlloca->offset;
112+
if (newAllocaOffset != 0 && IGC_IS_FLAG_ENABLED(DisableMergingOfMultipleAllocasWithOffset)) {
113+
return false;
114+
}
115+
NewAlloca->offset = newAllocaOffset;
116+
MergableAlloca->nonOverlapingAllocas.push_back(NewAlloca);
117+
MergableAlloca->remainingSize -= sizeWithPadding;
118+
return true;
119+
}
120+
121+
return false;
122+
}
123+
124+
static void ReplaceAllocas(const MergeAllocas::AllocaInfo &MergableAlloca, Function &F) {
125+
Instruction *topAlloca = MergableAlloca.allocaI;
126+
topAlloca->moveBefore(F.getEntryBlock().getFirstNonPHI());
127+
topAlloca->setName(VALUE_NAME("MergedAlloca"));
128+
129+
IRBuilder<> Builder(topAlloca->getParent());
130+
Instruction *topAllocaBitcast = nullptr;
131+
132+
SmallVector<MergeAllocas::AllocaInfo *> allocasToReplace;
133+
allocasToReplace.insert(allocasToReplace.end(),
134+
MergableAlloca.nonOverlapingAllocas.begin(),
135+
MergableAlloca.nonOverlapingAllocas.end());
136+
137+
while (!allocasToReplace.empty()) {
138+
auto *subAlloca = allocasToReplace.pop_back_val();
139+
140+
auto *subInst = subAlloca->allocaI;
141+
auto *ReplacementValue = topAlloca;
142+
143+
if (topAlloca->getType() != subInst->getType()) {
144+
auto *InsertionPoint =
145+
(topAllocaBitcast != nullptr) ? topAllocaBitcast : topAlloca;
146+
Builder.SetInsertPoint(InsertionPoint->getNextNode());
147+
148+
Value *ValueToCast = nullptr;
149+
// If we have offset from original alloca we need to create GEP
150+
if (subAlloca->offset != 0) {
151+
// We can re-use same bitcast
152+
if (topAllocaBitcast == nullptr) {
153+
topAllocaBitcast = cast<Instruction>(
154+
Builder.CreateBitCast(topAlloca, Builder.getInt8PtrTy(topAlloca->getType()->getPointerAddressSpace())));
155+
}
156+
auto *Offset = Builder.getInt32(subAlloca->offset);
157+
auto *GEP = Builder.CreateGEP(Builder.getInt8Ty(),
158+
topAllocaBitcast, Offset);
159+
ValueToCast = GEP;
160+
} else {
161+
// If no offset is needed we can directly cast to target type
162+
ValueToCast = Builder.CreateBitCast(topAlloca, subInst->getType());
163+
}
164+
auto *CastedValue = llvm::cast<Instruction>(
165+
Builder.CreateBitCast(ValueToCast, subInst->getType()));
166+
ReplacementValue = CastedValue;
167+
}
168+
subInst->replaceAllUsesWith(ReplacementValue);
169+
subInst->eraseFromParent();
170+
171+
allocasToReplace.insert(allocasToReplace.end(),
172+
subAlloca->nonOverlapingAllocas.begin(),
173+
subAlloca->nonOverlapingAllocas.end());
174+
}
175+
}
176+
177+
bool MergeAllocas::runOnFunction(Function &F) {
178+
if (skipFunction(F)) {
179+
return false;
180+
}
181+
182+
// in the past, the analysis pass used to be responsible for the liveness data objects
183+
// the pass got deleted as a part of refactor (it was leaking memory anyway),
184+
// so to avoid changing the logic, we create a backing storage for the liveness data objects
185+
// we should revisit this at some point to clean it up, but for now
186+
// we use std::list so it doesn't invalidate the references when inserting
187+
std::list<LivenessData> storage;
188+
189+
llvm::SmallVector<std::pair<llvm::Instruction*, LivenessData*>> ABLA;
190+
191+
for (auto& I : make_filter_range(instructions(F), [](auto& I) { return isa<AllocaInst>(&I); }))
192+
{
193+
storage.push_back(ProcessInstruction(&I));
194+
ABLA.push_back(std::make_pair(&I, &storage.back()));
195+
}
196+
197+
const auto *DataLayout = &F.getParent()->getDataLayout();
198+
199+
// We group non-overlapping allocas for replacements.
200+
SmallVector<AllocaInfo *> MergableAllocas;
201+
202+
// First we sort analysis results based on allocation size, from larger to
203+
// smaller.
204+
llvm::sort(ABLA, [&](const auto &a, const auto &b) {
205+
return GetByteSize(cast<AllocaInst>(a.first)->getAllocatedType(),
206+
DataLayout) >
207+
GetByteSize(cast<AllocaInst>(b.first)->getAllocatedType(),
208+
DataLayout);
209+
});
210+
211+
// Reserve space for all alloca infos so we can use pointers to them.
212+
AllAllocasInfos.resize(ABLA.size());
213+
size_t currentIndex = 0;
214+
215+
// We iterate over analysis results collecting non-overlapping allocas.
216+
for (const auto &A : ABLA) {
217+
const auto &[currI, currLD] = A;
218+
219+
if (skipInstruction(F, *currLD))
220+
continue;
221+
222+
AllAllocasInfos[currentIndex] =
223+
GetAllocaInfo(cast<AllocaInst>(currI), currLD, DataLayout);
224+
AllocaInfo &AllocaInfo = AllAllocasInfos[currentIndex];
225+
currentIndex++;
226+
227+
// We check if the current alloca overlaps with any of the previously added.
228+
bool added = false;
229+
for (auto *MergableAlloca : MergableAllocas) {
230+
if (AllocaInfo.livenessData->OverlapsWith(*MergableAlloca->livenessData)) {
231+
continue;
232+
}
233+
added = AddNonOverlappingAlloca(MergableAlloca, &AllocaInfo);
234+
if (added) {
235+
break;
236+
}
237+
}
238+
// Alloca overlaps with all of the current ones so it will be added as new
239+
// element.
240+
if (!added && AllocaInfo.allocationSize != 0) {
241+
MergableAllocas.push_back(&AllocaInfo);
242+
}
243+
}
244+
245+
bool changed = false;
246+
247+
// Replace alloca usages
248+
for (auto *MergableAlloca : MergableAllocas) {
249+
if (MergableAlloca->nonOverlapingAllocas.empty()) {
250+
continue;
251+
}
252+
changed = true;
253+
ReplaceAllocas(*MergableAlloca, F);
254+
}
255+
256+
return changed;
257+
}

‎IGC/AdaptorCommon/LivenessUtils/MergeAllocas.h

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*========================== begin_copyright_notice ============================
2+
3+
Copyright (C) 2025 Intel Corporation
4+
5+
SPDX-License-Identifier: MIT
6+
7+
============================= end_copyright_notice ===========================*/
8+
9+
#include "common/LLVMWarningsPush.hpp"
10+
#include <llvm/IR/Instructions.h>
11+
#include <llvm/Pass.h>
12+
#include <llvm/ADT/DenseSet.h>
13+
#include <llvm/ADT/SetVector.h>
14+
#include "common/LLVMWarningsPop.hpp"
15+
16+
#include "AllocationLivenessAnalyzer.h"
17+
18+
namespace llvm {
19+
class Function;
20+
} // namespace llvm
21+
22+
namespace IGC
23+
{
24+
class MergeAllocas : public AllocationLivenessAnalyzer
25+
{
26+
public:
27+
struct AllocaInfo {
28+
llvm::SmallVector<AllocaInfo*> nonOverlapingAllocas;
29+
llvm::AllocaInst* allocaI;
30+
AllocationLivenessAnalyzer::LivenessData* livenessData;
31+
unsigned int addressSpace;
32+
std::size_t allocationSize;
33+
std::size_t remainingSize;
34+
std::size_t alignment;
35+
// start offset of this alloca in top level alloca (if any)
36+
std::size_t offset;
37+
};
38+
39+
MergeAllocas(char& pid) : AllocationLivenessAnalyzer(pid) {}
40+
41+
bool runOnFunction(llvm::Function& F) override;
42+
virtual bool skipInstruction(llvm::Function& F, AllocationLivenessAnalyzer::LivenessData& LD) = 0;
43+
void getAdditionalAnalysisUsage(llvm::AnalysisUsage& AU) const override {};
44+
45+
private:
46+
std::vector<AllocaInfo> AllAllocasInfos;
47+
};
48+
} // namespace IGC

0 commit comments

Comments
 (0)