Skip to content

Commit

Permalink
Delay 'jl_gc_queue_root' call generation until final GC lowering pass
Browse files Browse the repository at this point in the history
  • Loading branch information
@jonathanvdc
jonathanvdc committed Feb 19, 2019
1 parent 537bc2b commit d1d6132
Show file tree
Hide file tree
Showing 2 changed files with 72 additions and 138 deletions.
120 changes: 71 additions & 49 deletions src/llvm-final-gc-lowering.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@ struct FinalLowerGC: public FunctionPass, private JuliaPassContext {
{ }

private:
Function *queueroot_func;
Function *pool_alloc_func;
Function *big_alloc_func;
Function *queueRootFunc;
Function *poolAllocFunc;
Function *bigAllocFunc;
CallInst *ptlsStates;

bool doInitialization(Module &M) override;
Expand All @@ -55,6 +55,9 @@ struct FinalLowerGC: public FunctionPass, private JuliaPassContext {
// Lowers a `julia.gc_alloc_bytes` intrinsic.
Value *lowerGCAllocBytes(CallInst *target, Function &F);

// Lowers a `julia.queue_gc_root` intrinsic.
Value *lowerQueueGCRoot(CallInst *target, Function &F);

Instruction *getPgcstack(Instruction *ptlsStates);
};

Expand Down Expand Up @@ -154,6 +157,13 @@ Value *FinalLowerGC::lowerGetGCFrameSlot(CallInst *target, Function &F)
return gep;
}

Value *FinalLowerGC::lowerQueueGCRoot(CallInst *target, Function &F)
{
assert(target->getNumArgOperands() == 3);
target->setCalledFunction(queueRootFunc);
return target;
}

Instruction *FinalLowerGC::getPgcstack(Instruction *ptlsStates)
{
Constant *offset = ConstantInt::getSigned(T_int32, offsetof(jl_tls_states_t, pgcstack) / sizeof(void*));
Expand All @@ -177,13 +187,13 @@ Value *FinalLowerGC::lowerGCAllocBytes(CallInst *target, Function &F)
CallInst *newI;
if (offset < 0) {
newI = builder.CreateCall(
big_alloc_func,
bigAllocFunc,
{ ptls, ConstantInt::get(T_size, sz + sizeof(void*)) });
}
else {
auto pool_offs = ConstantInt::get(T_int32, offset);
auto pool_osize = ConstantInt::get(T_int32, osize);
newI = builder.CreateCall(pool_alloc_func, { ptls, pool_offs, pool_osize });
newI = builder.CreateCall(poolAllocFunc, { ptls, pool_offs, pool_osize });
}
newI->setAttributes(newI->getCalledFunction()->getAttributes());
newI->takeName(target);
Expand All @@ -195,69 +205,64 @@ bool FinalLowerGC::doInitialization(Module &M) {
initAll(M);

// Initialize platform-specific references.
if (write_barrier_func) {
if (!(queueroot_func = M.getFunction("jl_gc_queue_root"))) {
queueroot_func = Function::Create(
FunctionType::get(
Type::getVoidTy(M.getContext()),
{ T_prjlvalue },
false),
Function::ExternalLinkage,
"jl_gc_queue_root",
&M);
queueroot_func->addFnAttr(Attribute::InaccessibleMemOrArgMemOnly);
}
}
else {
queueroot_func = nullptr;
if (!(queueRootFunc = M.getFunction("jl_gc_queue_root"))) {
queueRootFunc = Function::Create(
FunctionType::get(
Type::getVoidTy(M.getContext()),
{ T_prjlvalue },
false),
Function::ExternalLinkage,
"jl_gc_queue_root",
&M);
queueRootFunc->addFnAttr(Attribute::InaccessibleMemOrArgMemOnly);
}
pool_alloc_func = nullptr;
big_alloc_func = nullptr;
poolAllocFunc = nullptr;
bigAllocFunc = nullptr;
if (alloc_obj_func) {
if (!(pool_alloc_func = M.getFunction("jl_gc_pool_alloc"))) {
if (!(poolAllocFunc = M.getFunction("jl_gc_pool_alloc"))) {
std::vector<Type*> args(0);
args.push_back(T_pint8);
args.push_back(T_int32);
args.push_back(T_int32);
pool_alloc_func = Function::Create(
poolAllocFunc = Function::Create(
FunctionType::get(T_prjlvalue, args, false),
Function::ExternalLinkage,
"jl_gc_pool_alloc",
&M);
pool_alloc_func->setAttributes(
poolAllocFunc->setAttributes(
AttributeList::get(M.getContext(),
alloc_obj_func->getAttributes().getFnAttributes(),
alloc_obj_func->getAttributes().getRetAttributes(),
None));
}
if (!(big_alloc_func = M.getFunction("jl_gc_big_alloc"))) {
if (!(bigAllocFunc = M.getFunction("jl_gc_big_alloc"))) {
std::vector<Type*> args(0);
args.push_back(T_pint8);
args.push_back(T_size);
big_alloc_func = Function::Create(
bigAllocFunc = Function::Create(
FunctionType::get(T_prjlvalue, args, false),
Function::ExternalLinkage,
"jl_gc_big_alloc",
&M);
big_alloc_func->setAttributes(
bigAllocFunc->setAttributes(
AttributeList::get(M.getContext(),
alloc_obj_func->getAttributes().getFnAttributes(),
alloc_obj_func->getAttributes().getRetAttributes(),
None));
}
}

GlobalValue *function_list[] = {queueroot_func, pool_alloc_func, big_alloc_func};
GlobalValue *functionList[] = {queueRootFunc, poolAllocFunc, bigAllocFunc};
unsigned j = 0;
for (unsigned i = 0; i < sizeof(function_list) / sizeof(void*); i++) {
if (!function_list[i])
for (unsigned i = 0; i < sizeof(functionList) / sizeof(void*); i++) {
if (!functionList[i])
continue;
if (i != j)
function_list[j] = function_list[i];
functionList[j] = functionList[i];
j++;
}
if (j != 0)
appendToCompilerUsed(M, ArrayRef<GlobalValue*>(function_list, j));
appendToCompilerUsed(M, ArrayRef<GlobalValue*>(functionList, j));
return true;
}

Expand All @@ -266,32 +271,48 @@ bool FinalLowerGC::doFinalization(Module &M)
auto used = M.getGlobalVariable("llvm.compiler.used");
if (!used)
return false;
GlobalValue *function_list[] = {queueroot_func, pool_alloc_func, big_alloc_func};
SmallPtrSet<Constant*, 16> InitAsSet(function_list,
function_list + sizeof(function_list) / sizeof(void*));
GlobalValue *functionList[] = {queueRootFunc, poolAllocFunc, bigAllocFunc};
SmallPtrSet<Constant*, 16> InitAsSet(
functionList,
functionList + sizeof(functionList) / sizeof(void*));
bool changed = false;
SmallVector<Constant*, 16> Init;
SmallVector<Constant*, 16> init;
ConstantArray *CA = dyn_cast<ConstantArray>(used->getInitializer());
for (auto &Op : CA->operands()) {
Constant *C = cast_or_null<Constant>(Op);
if (InitAsSet.count(C->stripPointerCasts())) {
changed = true;
continue;
}
Init.push_back(C);
init.push_back(C);
}
if (!changed)
return false;
used->eraseFromParent();
if (Init.empty())
if (init.empty())
return true;
ArrayType *ATy = ArrayType::get(T_pint8, Init.size());
ArrayType *ATy = ArrayType::get(T_pint8, init.size());
used = new GlobalVariable(M, ATy, false, GlobalValue::AppendingLinkage,
ConstantArray::get(ATy, Init), "llvm.compiler.used");
ConstantArray::get(ATy, init), "llvm.compiler.used");
used->setSection("llvm.metadata");
return true;
}

template<typename TIterator>
static void replaceInstruction(
Instruction *oldInstruction,
Value *newInstruction,
TIterator &it)
{
if (newInstruction != oldInstruction) {
oldInstruction->replaceAllUsesWith(newInstruction);
it = oldInstruction->eraseFromParent();
}
else {
++it;
}
}

bool FinalLowerGC::runOnFunction(Function &F)
{
DEBUG(dbgs() << "FINAL GC LOWERING: Processing function " << F.getName() << "\n");
Expand All @@ -310,7 +331,8 @@ bool FinalLowerGC::runOnFunction(Function &F)
auto pushGCFrameFunc = getOrNull(jl_intrinsics::pushGCFrame);
auto popGCFrameFunc = getOrNull(jl_intrinsics::popGCFrame);
auto getGCFrameSlotFunc = getOrNull(jl_intrinsics::getGCFrameSlot);
auto GCAllocBytes = getOrNull(jl_intrinsics::GCAllocBytes);
auto GCAllocBytesFunc = getOrNull(jl_intrinsics::GCAllocBytes);
auto queueGCRootFunc = getOrNull(jl_intrinsics::queueGCRoot);

// Lower all calls to supported intrinsics.
for (BasicBlock &BB : F) {
Expand All @@ -324,8 +346,7 @@ bool FinalLowerGC::runOnFunction(Function &F)
auto callee = CI->getCalledValue();

if (callee == newGCFrameFunc) {
CI->replaceAllUsesWith(lowerNewGCFrame(CI, F));
it = CI->eraseFromParent();
replaceInstruction(CI, lowerNewGCFrame(CI, F), it);
}
else if (callee == pushGCFrameFunc) {
lowerPushGCFrame(CI, F);
Expand All @@ -336,12 +357,13 @@ bool FinalLowerGC::runOnFunction(Function &F)
it = CI->eraseFromParent();
}
else if (callee == getGCFrameSlotFunc) {
CI->replaceAllUsesWith(lowerGetGCFrameSlot(CI, F));
it = CI->eraseFromParent();
replaceInstruction(CI, lowerGetGCFrameSlot(CI, F), it);
}
else if (callee == GCAllocBytes) {
CI->replaceAllUsesWith(lowerGCAllocBytes(CI, F));
it = CI->eraseFromParent();
else if (callee == GCAllocBytesFunc) {
replaceInstruction(CI, lowerGCAllocBytes(CI, F), it);
}
else if (callee == queueGCRootFunc) {
replaceInstruction(CI, lowerQueueGCRoot(CI, F), it);
}
else {
++it;
Expand Down
90 changes: 1 addition & 89 deletions src/llvm-late-gc-lowering.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -314,9 +314,6 @@ struct LateLowerGCFrame: public FunctionPass, private JuliaPassContext {
}

private:
Function *queueroot_func;
Function *pool_alloc_func;
Function *big_alloc_func;
CallInst *ptlsStates;

void MaybeNoteDef(State &S, BBState &BBS, Value *Def, const std::vector<int> &SafepointsSoFar, SmallVector<int, 1> &&RefinedPtr = SmallVector<int, 1>());
Expand All @@ -343,7 +340,6 @@ struct LateLowerGCFrame: public FunctionPass, private JuliaPassContext {
void PlaceGCFrameStores(State &S, unsigned MinColorRoot, const std::vector<int> &Colors, Value *GCFrame);
void PlaceRootsAndUpdateCalls(std::vector<int> &Colors, State &S, std::map<Value *, std::pair<int, int>>);
bool doInitialization(Module &M) override;
bool doFinalization(Module &) override;
bool runOnFunction(Function &F) override;
bool CleanupIR(Function &F, State *S=nullptr);
void NoteUseChain(State &S, BBState &BBS, User *TheUser);
Expand Down Expand Up @@ -1800,7 +1796,7 @@ bool LateLowerGCFrame::CleanupIR(Function &F, State *S) {
auto trigTerm = SplitBlockAndInsertIfThen(chldNotMarked, mayTrigTerm, false,
MDB.createBranchWeights(Weights));
builder.SetInsertPoint(trigTerm);
builder.CreateCall(queueroot_func, parent);
builder.CreateCall(getOrDefine(jl_intrinsics::queueGCRoot), parent);
CI->eraseFromParent();
}
if (maxframeargs == 0 && Frame) {
Expand Down Expand Up @@ -1957,90 +1953,6 @@ void LateLowerGCFrame::PlaceRootsAndUpdateCalls(std::vector<int> &Colors, State
bool LateLowerGCFrame::doInitialization(Module &M) {
// Initialize platform-agnostic references.
initAll(M);

// Initialize platform-specific references.
auto &ctx = M.getContext();
if (write_barrier_func) {
if (!(queueroot_func = M.getFunction("jl_gc_queue_root"))) {
queueroot_func = Function::Create(FunctionType::get(Type::getVoidTy(ctx),
{T_prjlvalue}, false),
Function::ExternalLinkage, "jl_gc_queue_root", &M);
queueroot_func->addFnAttr(Attribute::InaccessibleMemOrArgMemOnly);
}
}
else {
queueroot_func = nullptr;
}
pool_alloc_func = nullptr;
big_alloc_func = nullptr;
if (alloc_obj_func) {
if (!(pool_alloc_func = M.getFunction("jl_gc_pool_alloc"))) {
std::vector<Type*> args(0);
args.push_back(T_pint8);
args.push_back(T_int32);
args.push_back(T_int32);
pool_alloc_func = Function::Create(FunctionType::get(T_prjlvalue, args, false),
Function::ExternalLinkage, "jl_gc_pool_alloc", &M);
pool_alloc_func->setAttributes(AttributeList::get(M.getContext(),
alloc_obj_func->getAttributes().getFnAttributes(),
alloc_obj_func->getAttributes().getRetAttributes(),
None));
}
if (!(big_alloc_func = M.getFunction("jl_gc_big_alloc"))) {
std::vector<Type*> args(0);
args.push_back(T_pint8);
args.push_back(T_size);
big_alloc_func = Function::Create(FunctionType::get(T_prjlvalue, args, false),
Function::ExternalLinkage, "jl_gc_big_alloc", &M);
big_alloc_func->setAttributes(AttributeList::get(M.getContext(),
alloc_obj_func->getAttributes().getFnAttributes(),
alloc_obj_func->getAttributes().getRetAttributes(),
None));
}
}

GlobalValue *function_list[] = {queueroot_func, pool_alloc_func, big_alloc_func};
unsigned j = 0;
for (unsigned i = 0; i < sizeof(function_list) / sizeof(void*); i++) {
if (!function_list[i])
continue;
if (i != j)
function_list[j] = function_list[i];
j++;
}
if (j != 0)
appendToCompilerUsed(M, ArrayRef<GlobalValue*>(function_list, j));
return true;
}

bool LateLowerGCFrame::doFinalization(Module &M)
{
auto used = M.getGlobalVariable("llvm.compiler.used");
if (!used)
return false;
GlobalValue *function_list[] = {queueroot_func, pool_alloc_func, big_alloc_func};
SmallPtrSet<Constant*, 16> InitAsSet(function_list,
function_list + sizeof(function_list) / sizeof(void*));
bool changed = false;
SmallVector<Constant*, 16> Init;
ConstantArray *CA = dyn_cast<ConstantArray>(used->getInitializer());
for (auto &Op : CA->operands()) {
Constant *C = cast_or_null<Constant>(Op);
if (InitAsSet.count(C->stripPointerCasts())) {
changed = true;
continue;
}
Init.push_back(C);
}
if (!changed)
return false;
used->eraseFromParent();
if (Init.empty())
return true;
ArrayType *ATy = ArrayType::get(T_pint8, Init.size());
used = new llvm::GlobalVariable(M, ATy, false, GlobalValue::AppendingLinkage,
ConstantArray::get(ATy, Init), "llvm.compiler.used");
used->setSection("llvm.metadata");
return true;
}

Expand Down

0 comments on commit d1d6132

Please sign in to comment.