Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,7 @@ $(build_shlibdir)/libllvmcalltest.$(SHLIB_EXT): $(SRCDIR)/llvm-codegen-shared.h
$(BUILDDIR)/llvm-alloc-helpers.o $(BUILDDIR)/llvm-alloc-helpers.dbg.obj: $(SRCDIR)/llvm-codegen-shared.h $(SRCDIR)/llvm-pass-helpers.h $(SRCDIR)/llvm-alloc-helpers.h
$(BUILDDIR)/llvm-alloc-opt.o $(BUILDDIR)/llvm-alloc-opt.dbg.obj: $(SRCDIR)/llvm-codegen-shared.h $(SRCDIR)/llvm-pass-helpers.h $(SRCDIR)/llvm-alloc-helpers.h
$(BUILDDIR)/llvm-cpufeatures.o $(BUILDDIR)/llvm-cpufeatures.dbg.obj: $(SRCDIR)/jitlayers.h $(SRCDIR)/debug-registry.h
$(BUILDDIR)/llvm-demote-float16.o $(BUILDDIR)/llvm-demote-float16.dbg.obj: $(SRCDIR)/jitlayers.h $(SRCDIR)/debug-registry.h
$(BUILDDIR)/llvm-final-gc-lowering.o $(BUILDDIR)/llvm-final-gc-lowering.dbg.obj: $(SRCDIR)/llvm-pass-helpers.h $(SRCDIR)/llvm-codegen-shared.h
$(BUILDDIR)/llvm-gc-invariant-verifier.o $(BUILDDIR)/llvm-gc-invariant-verifier.dbg.obj: $(SRCDIR)/llvm-codegen-shared.h
$(BUILDDIR)/llvm-julia-licm.o $(BUILDDIR)/llvm-julia-licm.dbg.obj: $(SRCDIR)/llvm-codegen-shared.h $(SRCDIR)/llvm-alloc-helpers.h $(SRCDIR)/llvm-pass-helpers.h
Expand All @@ -322,7 +323,7 @@ $(BUILDDIR)/signal-handling.o $(BUILDDIR)/signal-handling.dbg.obj: $(addprefix $
$(BUILDDIR)/staticdata.o $(BUILDDIR)/staticdata.dbg.obj: $(SRCDIR)/staticdata_utils.c $(SRCDIR)/precompile_utils.c $(SRCDIR)/processor.h $(SRCDIR)/builtin_proto.h
$(BUILDDIR)/toplevel.o $(BUILDDIR)/toplevel.dbg.obj: $(SRCDIR)/builtin_proto.h
$(BUILDDIR)/ircode.o $(BUILDDIR)/ircode.dbg.obj: $(SRCDIR)/serialize.h
$(BUILDDIR)/pipeline.o $(BUILDDIR)/pipeline.dbg.obj: $(SRCDIR)/passes.h $(SRCDIR)/jitlayers.h
$(BUILDDIR)/pipeline.o $(BUILDDIR)/pipeline.dbg.obj: $(SRCDIR)/passes.h $(SRCDIR)/jitlayers.h $(SRCDIR)/debug-registry.h

$(addprefix $(BUILDDIR)/,threading.o threading.dbg.obj gc.o gc.dbg.obj init.c init.dbg.obj task.o task.dbg.obj): $(addprefix $(SRCDIR)/,threading.h)
$(addprefix $(BUILDDIR)/,APInt-C.o APInt-C.dbg.obj runtime_intrinsics.o runtime_intrinsics.dbg.obj): $(SRCDIR)/APInt-C.h
Expand Down
12 changes: 8 additions & 4 deletions src/aotcompile.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@

#include <llvm/IR/LegacyPassManagers.h>
#include <llvm/Transforms/Utils/Cloning.h>
#include <llvm/Linker/Linker.h>


using namespace llvm;
Expand Down Expand Up @@ -289,7 +290,7 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm
orc::ThreadSafeModule backing;
if (!llvmmod) {
ctx = jl_ExecutionEngine->acquireContext();
backing = jl_create_llvm_module("text", ctx, imaging);
backing = jl_create_ts_module("text", ctx, imaging);
}
orc::ThreadSafeModule &clone = llvmmod ? *unwrap(llvmmod) : backing;
auto ctxt = clone.getContext();
Expand Down Expand Up @@ -336,7 +337,7 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm
if (src && !emitted.count(codeinst)) {
// now add it to our compilation results
JL_GC_PROMISE_ROOTED(codeinst->rettype);
orc::ThreadSafeModule result_m = jl_create_llvm_module(name_from_method_instance(codeinst->def),
orc::ThreadSafeModule result_m = jl_create_ts_module(name_from_method_instance(codeinst->def),
params.tsctx, params.imaging,
clone.getModuleUnlocked()->getDataLayout(),
Triple(clone.getModuleUnlocked()->getTargetTriple()));
Expand Down Expand Up @@ -400,6 +401,7 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm

// clones the contents of the module `m` to the shadow_output collector
// while examining and recording what kind of function pointer we have
Linker L(*clone.getModuleUnlocked());
for (auto &def : emitted) {
jl_merge_module(clone, std::move(std::get<0>(def.second)));
jl_code_instance_t *this_code = def.first;
Expand Down Expand Up @@ -427,7 +429,9 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm
data->jl_fvar_map[this_code] = std::make_tuple(func_id, cfunc_id);
}
if (params._shared_module) {
jl_merge_module(clone, std::move(params._shared_module));
bool error = L.linkInModule(std::move(params._shared_module));
assert(!error && "Error linking in shared module");
(void)error;
}

// now get references to the globals in the merged module
Expand Down Expand Up @@ -1095,7 +1099,7 @@ void jl_get_llvmf_defn_impl(jl_llvmf_dump_t* dump, jl_method_instance_t *mi, siz
// emit this function into a new llvm module
if (src && jl_is_code_info(src)) {
auto ctx = jl_ExecutionEngine->getContext();
orc::ThreadSafeModule m = jl_create_llvm_module(name_from_method_instance(mi), *ctx, imaging_default());
orc::ThreadSafeModule m = jl_create_ts_module(name_from_method_instance(mi), *ctx, imaging_default());
uint64_t compiler_start_time = 0;
uint8_t measure_compile_time_enabled = jl_atomic_load_relaxed(&jl_measure_compile_time_enabled);
if (measure_compile_time_enabled)
Expand Down
9 changes: 3 additions & 6 deletions src/ccall.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,7 @@ GlobalVariable *jl_emit_RTLD_DEFAULT_var(Module *M)
static bool runtime_sym_gvs(jl_codectx_t &ctx, const char *f_lib, const char *f_name,
GlobalVariable *&lib, GlobalVariable *&sym)
{
auto &TSM = ctx.emission_context.shared_module(*jl_Module);
//Safe b/c emission context holds context lock
auto M = TSM.getModuleUnlocked();
auto M = &ctx.emission_context.shared_module(*jl_Module);
bool runtime_lib = false;
GlobalVariable *libptrgv;
jl_codegen_params_t::SymMapGV *symMap;
Expand Down Expand Up @@ -238,8 +236,7 @@ static GlobalVariable *emit_plt_thunk(
bool runtime_lib)
{
++PLTThunks;
auto &TSM = ctx.emission_context.shared_module(*jl_Module);
Module *M = TSM.getModuleUnlocked();
auto M = &ctx.emission_context.shared_module(*jl_Module);
PointerType *funcptype = PointerType::get(functype, 0);
libptrgv = prepare_global_in(M, libptrgv);
llvmgv = prepare_global_in(M, llvmgv);
Expand Down Expand Up @@ -987,7 +984,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar
// save the module to be linked later.
// we cannot do this right now, because linking mutates the destination module,
// which might invalidate LLVM values cached in cgval_t's (specifically constant arrays)
ctx.llvmcall_modules.push_back(orc::ThreadSafeModule(std::move(Mod), ctx.emission_context.tsctx));
ctx.llvmcall_modules.push_back(std::move(Mod));

JL_GC_POP();

Expand Down
2 changes: 1 addition & 1 deletion src/cgutils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,7 @@ static Value *julia_pgv(jl_codectx_t &ctx, const char *cname, void *addr)
}
if (gv == nullptr)
gv = new GlobalVariable(*M, ctx.types().T_pjlvalue,
false, GlobalVariable::PrivateLinkage,
false, GlobalVariable::ExternalLinkage,
NULL, localname);
// LLVM passes sometimes strip metadata when moving load around
// since the load at the new location satisfy the same condition as the original one.
Expand Down
40 changes: 20 additions & 20 deletions src/codegen.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1464,7 +1464,7 @@ class jl_codectx_t {
bool external_linkage = false;
const jl_cgparams_t *params = NULL;

std::vector<orc::ThreadSafeModule> llvmcall_modules;
std::vector<std::unique_ptr<Module>> llvmcall_modules;

jl_codectx_t(LLVMContext &llvmctx, jl_codegen_params_t &params)
: builder(llvmctx),
Expand Down Expand Up @@ -2079,22 +2079,21 @@ static jl_cgval_t convert_julia_type(jl_codectx_t &ctx, const jl_cgval_t &v, jl_
return jl_cgval_t(v, typ, new_tindex);
}

orc::ThreadSafeModule jl_create_llvm_module(StringRef name, orc::ThreadSafeContext context, bool imaging_mode, const DataLayout &DL, const Triple &triple)
std::unique_ptr<Module> jl_create_llvm_module(StringRef name, LLVMContext &context, bool imaging_mode, const DataLayout &DL, const Triple &triple)
{
++ModulesCreated;
auto lock = context.getLock();
Module *m = new Module(name, *context.getContext());
orc::ThreadSafeModule TSM(std::unique_ptr<Module>(m), std::move(context));
auto m = std::make_unique<Module>(name, context);
// Some linkers (*cough* OS X) don't understand DWARF v4, so we use v2 in
// imaging mode. The structure of v4 is slightly nicer for debugging JIT
// code.
if (!m->getModuleFlag("Dwarf Version")) {
int dwarf_version = 4;
#ifdef _OS_DARWIN_
if (imaging_mode)
if (triple.isOSDarwin()) {
if (imaging_mode) {
dwarf_version = 2;
#endif
m->addModuleFlag(llvm::Module::Warning, "Dwarf Version", dwarf_version);
}
}
m->addModuleFlag(llvm::Module::Warning, "Dwarf Version", dwarf_version);
}
if (!m->getModuleFlag("Debug Info Version"))
m->addModuleFlag(llvm::Module::Warning, "Debug Info Version",
Expand All @@ -2111,7 +2110,7 @@ orc::ThreadSafeModule jl_create_llvm_module(StringRef name, orc::ThreadSafeConte
#if defined(JL_DEBUG_BUILD) && JL_LLVM_VERSION >= 130000
m->setStackProtectorGuard("global");
#endif
return TSM;
return m;
}

static void jl_init_function(Function *F)
Expand Down Expand Up @@ -4898,7 +4897,7 @@ static std::pair<Function*, Function*> get_oc_function(jl_codectx_t &ctx, jl_met
ir = jl_uncompress_ir(closure_method, ci, (jl_array_t*)inferred);

// TODO: Emit this inline and outline it late using LLVM's coroutine support.
orc::ThreadSafeModule closure_m = jl_create_llvm_module(
orc::ThreadSafeModule closure_m = jl_create_ts_module(
name_from_method_instance(mi), ctx.emission_context.tsctx,
ctx.emission_context.imaging,
jl_Module->getDataLayout(), Triple(jl_Module->getTargetTriple()));
Expand Down Expand Up @@ -8125,14 +8124,15 @@ static jl_llvm_functions_t

// link the dependent llvmcall modules, but switch their function's linkage to internal
// so that they don't conflict when they show up in the execution engine.
for (auto &TSMod : ctx.llvmcall_modules) {
Linker L(*jl_Module);
for (auto &Mod : ctx.llvmcall_modules) {
SmallVector<std::string, 1> Exports;
TSMod.withModuleDo([&](Module &Mod) {
for (const auto &F: Mod.functions())
if (!F.isDeclaration())
Exports.push_back(F.getName().str());
});
jl_merge_module(TSM, std::move(TSMod));
for (const auto &F: Mod->functions())
if (!F.isDeclaration())
Exports.push_back(F.getName().str());
bool error = L.linkInModule(std::move(Mod));
assert(!error && "linking llvmcall modules failed");
(void)error;
for (auto FN: Exports)
jl_Module->getFunction(FN)->setLinkage(GlobalVariable::InternalLinkage);
}
Expand Down Expand Up @@ -8335,7 +8335,7 @@ void jl_compile_workqueue(
src = jl_type_infer(codeinst->def, jl_atomic_load_acquire(&jl_world_counter), 0);
if (src) {
orc::ThreadSafeModule result_m =
jl_create_llvm_module(name_from_method_instance(codeinst->def),
jl_create_ts_module(name_from_method_instance(codeinst->def),
params.tsctx, params.imaging,
original.getDataLayout(), Triple(original.getTargetTriple()));
result.second = jl_emit_code(result_m, codeinst->def, src, src->rettype, params);
Expand All @@ -8344,7 +8344,7 @@ void jl_compile_workqueue(
}
else {
orc::ThreadSafeModule result_m =
jl_create_llvm_module(name_from_method_instance(codeinst->def),
jl_create_ts_module(name_from_method_instance(codeinst->def),
params.tsctx, params.imaging,
original.getDataLayout(), Triple(original.getTargetTriple()));
result.second = jl_emit_codeinst(result_m, codeinst, NULL, params);
Expand Down
12 changes: 6 additions & 6 deletions src/jitlayers.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -199,17 +199,17 @@ static jl_callptr_t _jl_compile_codeinst(
jl_workqueue_t emitted;
{
orc::ThreadSafeModule result_m =
jl_create_llvm_module(name_from_method_instance(codeinst->def), params.tsctx, params.imaging);
jl_create_ts_module(name_from_method_instance(codeinst->def), params.tsctx, params.imaging);
jl_llvm_functions_t decls = jl_emit_codeinst(result_m, codeinst, src, params);
if (result_m)
emitted[codeinst] = {std::move(result_m), std::move(decls)};
{
auto temp_module = jl_create_llvm_module(name_from_method_instance(codeinst->def), params.tsctx, params.imaging);
jl_compile_workqueue(emitted, *temp_module.getModuleUnlocked(), params, CompilationPolicy::Default);
auto temp_module = jl_create_llvm_module(name_from_method_instance(codeinst->def), params.getContext(), params.imaging);
jl_compile_workqueue(emitted, *temp_module, params, CompilationPolicy::Default);
}

if (params._shared_module)
jl_ExecutionEngine->addModule(std::move(params._shared_module));
jl_ExecutionEngine->addModule(orc::ThreadSafeModule(std::move(params._shared_module), params.tsctx));
StringMap<orc::ThreadSafeModule*> NewExports;
StringMap<void*> NewGlobals;
for (auto &global : params.globals) {
Expand Down Expand Up @@ -316,7 +316,7 @@ int jl_compile_extern_c_impl(LLVMOrcThreadSafeModuleRef llvmmod, void *p, void *
if (!pparams) {
ctx = jl_ExecutionEngine->acquireContext();
}
backing = jl_create_llvm_module("cextern", pparams ? pparams->tsctx : ctx, pparams ? pparams->imaging : imaging_default());
backing = jl_create_ts_module("cextern", pparams ? pparams->tsctx : ctx, pparams ? pparams->imaging : imaging_default());
into = &backing;
}
JL_LOCK(&jl_codegen_lock);
Expand All @@ -334,7 +334,7 @@ int jl_compile_extern_c_impl(LLVMOrcThreadSafeModuleRef llvmmod, void *p, void *
jl_jit_globals(params.globals);
assert(params.workqueue.empty());
if (params._shared_module)
jl_ExecutionEngine->addModule(std::move(params._shared_module));
jl_ExecutionEngine->addModule(orc::ThreadSafeModule(std::move(params._shared_module), params.tsctx));
}
if (success && llvmmod == NULL)
jl_ExecutionEngine->addModule(std::move(*into));
Expand Down
27 changes: 17 additions & 10 deletions src/jitlayers.h
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,6 @@ DEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::ThreadSafeModule, LLVMOrcThreadSafeModul
void addTargetPasses(legacy::PassManagerBase *PM, const Triple &triple, TargetIRAnalysis analysis) JL_NOTSAFEPOINT;
void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level, bool lower_intrinsics=true, bool dump_native=false, bool external_use=false) JL_NOTSAFEPOINT;
void addMachinePasses(legacy::PassManagerBase *PM, int optlevel) JL_NOTSAFEPOINT;
void jl_finalize_module(orc::ThreadSafeModule m) JL_NOTSAFEPOINT;
void jl_merge_module(orc::ThreadSafeModule &dest, orc::ThreadSafeModule src) JL_NOTSAFEPOINT;
GlobalVariable *jl_emit_RTLD_DEFAULT_var(Module *M) JL_NOTSAFEPOINT;
DataLayout jl_create_datalayout(TargetMachine &TM) JL_NOTSAFEPOINT;
Expand Down Expand Up @@ -182,6 +181,10 @@ typedef std::tuple<jl_returninfo_t::CallingConv, unsigned, llvm::Function*, bool
typedef struct _jl_codegen_params_t {
orc::ThreadSafeContext tsctx;
orc::ThreadSafeContext::Lock tsctx_lock;

inline LLVMContext &getContext() {
return *tsctx.getContext();
}
typedef StringMap<GlobalVariable*> SymMapGV;
// outputs
std::vector<std::pair<jl_code_instance_t*, jl_codegen_call_target_t>> workqueue;
Expand All @@ -208,8 +211,8 @@ typedef struct _jl_codegen_params_t {
DenseMap<AttributeList, std::map<
std::tuple<GlobalVariable*, FunctionType*, CallingConv::ID>,
GlobalVariable*>> allPltMap;
orc::ThreadSafeModule _shared_module;
inline orc::ThreadSafeModule &shared_module(Module &from);
std::unique_ptr<Module> _shared_module;
inline Module &shared_module(Module &from);
// inputs
size_t world = 0;
const jl_cgparams_t *params = &jl_default_cgparams;
Expand Down Expand Up @@ -535,18 +538,22 @@ class JuliaOJIT {
OptSelLayerT OptSelLayer;
};
extern JuliaOJIT *jl_ExecutionEngine;
orc::ThreadSafeModule jl_create_llvm_module(StringRef name, orc::ThreadSafeContext ctx, bool imaging_mode, const DataLayout &DL = jl_ExecutionEngine->getDataLayout(), const Triple &triple = jl_ExecutionEngine->getTargetTriple()) JL_NOTSAFEPOINT;
std::unique_ptr<Module> jl_create_llvm_module(StringRef name, LLVMContext &ctx, bool imaging_mode, const DataLayout &DL = jl_ExecutionEngine->getDataLayout(), const Triple &triple = jl_ExecutionEngine->getTargetTriple()) JL_NOTSAFEPOINT;
inline orc::ThreadSafeModule jl_create_ts_module(StringRef name, orc::ThreadSafeContext ctx, bool imaging_mode, const DataLayout &DL = jl_ExecutionEngine->getDataLayout(), const Triple &triple = jl_ExecutionEngine->getTargetTriple()) JL_NOTSAFEPOINT {
auto lock = ctx.getLock();
return orc::ThreadSafeModule(jl_create_llvm_module(name, *ctx.getContext(), imaging_mode, DL, triple), ctx);
}

orc::ThreadSafeModule &jl_codegen_params_t::shared_module(Module &from) JL_NOTSAFEPOINT {
Module &jl_codegen_params_t::shared_module(Module &from) JL_NOTSAFEPOINT {
if (!_shared_module) {
_shared_module = jl_create_llvm_module("globals", tsctx, imaging, from.getDataLayout(), Triple(from.getTargetTriple()));
_shared_module = jl_create_llvm_module("globals", getContext(), imaging, from.getDataLayout(), Triple(from.getTargetTriple()));
assert(&from.getContext() == tsctx.getContext() && "Module context differs from codegen_params context!");
} else {
assert(&from.getContext() == _shared_module.getContext().getContext() && "Module context differs from shared module context!");
assert(from.getDataLayout() == _shared_module.getModuleUnlocked()->getDataLayout() && "Module data layout differs from shared module data layout!");
assert(from.getTargetTriple() == _shared_module.getModuleUnlocked()->getTargetTriple() && "Module target triple differs from shared module target triple!");
assert(&from.getContext() == &getContext() && "Module context differs from shared module context!");
assert(from.getDataLayout() == _shared_module->getDataLayout() && "Module data layout differs from shared module data layout!");
assert(from.getTargetTriple() == _shared_module->getTargetTriple() && "Module target triple differs from shared module target triple!");
}
return _shared_module;
return *_shared_module;
}

Pass *createLowerPTLSPass(bool imaging_mode) JL_NOTSAFEPOINT;
Expand Down
17 changes: 17 additions & 0 deletions test/llvmcall.jl
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,23 @@ module CcallableRetTypeTest
@test do_the_call() === 42.0
end

# Issue #48093 - test that non-external globals are not deduplicated
function kernel()
Base.llvmcall(("""
@shmem = internal global i8 0, align 8
define void @entry() {
store i8 1, i8* @shmem
ret void
}""", "entry"), Cvoid, Tuple{})
Base.llvmcall(("""
@shmem = internal global i8 0, align 8
define i8 @entry() {
%1 = load i8, i8* @shmem
ret i8 %1
}""", "entry"), UInt8, Tuple{})
end
@test kernel() == 0x00

# If this test breaks, you've probably broken Cxx.jl - please check
module LLVMCallFunctionTest
using Base: llvmcall
Expand Down