diff --git a/Makefile b/Makefile index 046f18492bc3e1..bd510fda9c8e8b 100644 --- a/Makefile +++ b/Makefile @@ -84,6 +84,9 @@ julia-base: julia-deps $(build_sysconfdir)/julia/startup.jl $(build_man1dir)/jul julia-libccalltest: julia-deps @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src libccalltest +julia-libccalltestdep: julia-deps julia-libccalltest + @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src libccalltestdep + julia-libllvmcalltest: julia-deps @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src libllvmcalltest @@ -102,7 +105,7 @@ julia-sysimg-bc : julia-stdlib julia-base julia-cli-$(JULIA_BUILD_MODE) julia-sr julia-sysimg-release julia-sysimg-debug : julia-sysimg-% : julia-sysimg-ji julia-src-% @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f sysimage.mk sysimg-$* -julia-debug julia-release : julia-% : julia-sysimg-% julia-src-% julia-symlink julia-libccalltest julia-libllvmcalltest julia-base-cache +julia-debug julia-release : julia-% : julia-sysimg-% julia-src-% julia-symlink julia-libccalltest julia-libccalltestdep julia-libllvmcalltest julia-base-cache stdlibs-cache-release stdlibs-cache-debug : stdlibs-cache-% : julia-% @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f pkgimage.mk all-$* @@ -189,7 +192,7 @@ JL_TARGETS := julia-debug endif # private libraries, that are installed in $(prefix)/lib/julia -JL_PRIVATE_LIBS-0 := libccalltest libllvmcalltest +JL_PRIVATE_LIBS-0 := libccalltest libccalltestdep libllvmcalltest ifeq ($(JULIA_BUILD_MODE),release) JL_PRIVATE_LIBS-0 += libjulia-internal libjulia-codegen else ifeq ($(JULIA_BUILD_MODE),debug) diff --git a/base/libdl.jl b/base/libdl.jl index fdf6103d1800b6..f8f14d6113de87 100644 --- a/base/libdl.jl +++ b/base/libdl.jl @@ -9,7 +9,7 @@ import Base.DL_LOAD_PATH export DL_LOAD_PATH, RTLD_DEEPBIND, RTLD_FIRST, RTLD_GLOBAL, RTLD_LAZY, RTLD_LOCAL, RTLD_NODELETE, RTLD_NOLOAD, RTLD_NOW, dlclose, dlopen, dlopen_e, dlsym, dlsym_e, - dlpath, find_library, dlext, dllist + dlpath, find_library, dlext, dllist, LazyLibrary """ DL_LOAD_PATH @@ -45,6 +45,9 @@ applicable. """ (RTLD_DEEPBIND, RTLD_FIRST, RTLD_GLOBAL, RTLD_LAZY, RTLD_LOCAL, RTLD_NODELETE, RTLD_NOLOAD, RTLD_NOW) +# The default flags for `dlopen()` +const default_rtld_flags = RTLD_LAZY | RTLD_DEEPBIND + """ dlsym(handle, sym; throw_error::Bool = true) @@ -72,8 +75,8 @@ end Look up a symbol from a shared library handle, silently return `C_NULL` on lookup failure. This method is now deprecated in favor of `dlsym(handle, sym; throw_error=false)`. """ -function dlsym_e(hnd::Ptr, s::Union{Symbol,AbstractString}) - return something(dlsym(hnd, s; throw_error=false), C_NULL) +function dlsym_e(args...) + return something(dlsym(args...; throw_error=false), C_NULL) end """ @@ -110,10 +113,10 @@ If the library cannot be found, this method throws an error, unless the keyword """ function dlopen end -dlopen(s::Symbol, flags::Integer = RTLD_LAZY | RTLD_DEEPBIND; kwargs...) = +dlopen(s::Symbol, flags::Integer = default_rtld_flags; kwargs...) = dlopen(string(s), flags; kwargs...) -function dlopen(s::AbstractString, flags::Integer = RTLD_LAZY | RTLD_DEEPBIND; throw_error::Bool = true) +function dlopen(s::AbstractString, flags::Integer = default_rtld_flags; throw_error::Bool = true) ret = ccall(:jl_load_dynamic_library, Ptr{Cvoid}, (Cstring,UInt32,Cint), s, flags, Cint(throw_error)) if ret == C_NULL return nothing @@ -226,23 +229,6 @@ function dlpath(handle::Ptr{Cvoid}) return s end -""" - dlpath(libname::Union{AbstractString, Symbol}) - -Get the full path of the library `libname`. - -# Example -```julia-repl -julia> dlpath("libjulia") -``` -""" -function dlpath(libname::Union{AbstractString, Symbol}) - handle = dlopen(libname) - path = dlpath(handle) - dlclose(handle) - return path -end - if Sys.isapple() const dlext = "dylib" elseif Sys.iswindows() @@ -314,4 +300,101 @@ function dllist() return dynamic_libraries end + +""" + LazyLibrary(name, flags = , + dependencies = LazyLibrary[], on_load_callback = nothing) + +Represents a lazily-loaded library that opens itself and its dependencies on first usage +in a `dlopen()`, `dlsym()`, or `ccall()` usage. While this structure contains the +ability to run arbitrary code on first load via `on_load_callback`, we caution that this +should be used sparingly, as it is not expected that `ccall()` should result in large +amounts of Julia code being run. +""" +mutable struct LazyLibrary + # Name and flags to open with + const name::String + const flags::UInt32 + + # Dependencies that must be loaded before we can load + const dependencies::Vector{LazyLibrary} + + # Function that gets called once upon initial load with the pointer as an argument + const on_load_callback::Union{Nothing,Function} + # Function that gets called once upon final unload with the pointer as an argument + const on_unload_callback::Union{Nothing,Function} + + # Pointer that we eventually fill out upon first `dlopen()` + @atomic handle::Ptr{Cvoid} + @atomic refs::Int + function LazyLibrary(name, flags = default_rtld_flags, dependencies = LazyLibrary[], + on_load_callback = nothing, on_unload_callback = nothing) + return new( + String(name), + UInt32(flags), + collect(dependencies), + on_load_callback, + on_unload_callback, + C_NULL, + 0, + ) + end +end + +function dlopen(ll::LazyLibrary, flags::Integer = ll.flags; kwargs...) + # Only load once + handle = @atomic ll.handle + if handle != C_NULL + @atomic ll.refs += 1 + return handle + end + + # Ensure that all dependencies are loaded + for dep in ll.dependencies + dlopen(dep; kwargs...) + end + + # Load our library + handle = dlopen(ll.name, flags; kwargs...) + @atomic ll.handle = handle + @atomic ll.refs += 1 + + # Invoke our on load callback, if it exists + if ll.on_load_callback !== nothing + ll.on_load_callback(handle) + end + return handle +end +dlsym(ll::LazyLibrary, args...; kwargs...) = dlsym(dlopen(ll), args...; kwargs...) +function dlclose(ll::LazyLibrary) + if @atomic(ll.refs) > 0 && @atomic(ll.handle) != C_NULL + @atomic ll.refs -= 1 + if @atomic(ll.refs) <= 0 + if ll.on_unload_callback !== nothing + ll.on_unload_callback(@atomic(ll.handle)) + end + dlclose(@atomic(ll.handle)) + @atomic ll.handle = C_NULL + end + end +end + +""" + dlpath(libname) + +Get the full path of the library `libname`. `libname` can be a string, a symbol +or a `LazyLibrary` object. + +# Example +```julia-repl +julia> dlpath("libjulia") +``` +""" +function dlpath(libname::Union{AbstractString, Symbol, LazyLibrary}) + handle = dlopen(libname) + path = dlpath(handle) + dlclose(handle) + return path +end + end # module Libdl diff --git a/src/Makefile b/src/Makefile index e561aefcdfe043..7a42e134a64383 100644 --- a/src/Makefile +++ b/src/Makefile @@ -249,6 +249,7 @@ $(build_includedir)/julia/uv/*.h: $(LIBUV_INC)/uv/*.h | $(build_includedir)/juli $(INSTALL_F) $^ $(build_includedir)/julia/uv libccalltest: $(build_shlibdir)/libccalltest.$(SHLIB_EXT) +libccalltestdep: $(build_shlibdir)/libccalltestdep.$(SHLIB_EXT) libllvmcalltest: $(build_shlibdir)/libllvmcalltest.$(SHLIB_EXT) ifeq ($(OS), Linux) @@ -257,7 +258,7 @@ else JULIA_SPLITDEBUG := 0 endif $(build_shlibdir)/libccalltest.$(SHLIB_EXT): $(SRCDIR)/ccalltest.c - @$(call PRINT_CC, $(CC) $(JCFLAGS) $(JL_CFLAGS) $(JCPPFLAGS) $(FLAGS) -O3 $< $(fPIC) -shared -o $@.tmp $(LDFLAGS)) + @$(call PRINT_CC, $(CC) $(JCFLAGS) $(JL_CFLAGS) $(JCPPFLAGS) $(FLAGS) -O3 $< $(fPIC) -shared -o $@.tmp $(LDFLAGS) $(call SONAME_FLAGS,libccalltest.$(SHLIB_EXT))) $(INSTALL_NAME_CMD)libccalltest.$(SHLIB_EXT) $@.tmp ifeq ($(JULIA_SPLITDEBUG),1) @# Create split debug info file for libccalltest stacktraces test @@ -273,6 +274,9 @@ endif mv $@.tmp $@ $(INSTALL_NAME_CMD)libccalltest.$(SHLIB_EXT) $@ +$(build_shlibdir)/libccalltestdep.$(SHLIB_EXT): $(SRCDIR)/ccalltestdep.c $(build_shlibdir)/libccalltest.$(SHLIB_EXT) + @$(call PRINT_CC, $(CC) $(JCFLAGS) $(JL_CFLAGS) $(JCPPFLAGS) $(FLAGS) -O3 $< $(fPIC) -shared -o $@ $(LDFLAGS) $(COMMON_LIBPATHS) $(call SONAME_FLAGS,libccalltestdep.$(SHLIB_EXT)) -lccalltest) + $(build_shlibdir)/libllvmcalltest.$(SHLIB_EXT): $(SRCDIR)/llvmcalltest.cpp $(LLVM_CONFIG_ABSOLUTE) @$(call PRINT_CC, $(CXX) $(LLVM_CXXFLAGS) $(FLAGS) $(CPPFLAGS) $(CXXFLAGS) -O3 $< $(fPIC) -shared -o $@ $(LDFLAGS) $(COMMON_LIBPATHS) $(NO_WHOLE_ARCHIVE) $(CG_LLVMLINK)) -lpthread diff --git a/src/ccall.cpp b/src/ccall.cpp index 90f7417c03524c..f2d51b18825332 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -651,6 +651,9 @@ static void interpret_symbol_arg(jl_codectx_t &ctx, native_sym_arg_t &out, jl_va f_lib = jl_symbol_name((jl_sym_t*)t1); else if (jl_is_string(t1)) f_lib = jl_string_data(t1); + else if (jl_lazy_library_type != NULL && jl_isa(t1, (jl_value_t *)jl_lazy_library_type)) { + out.lib_expr = t1; + } else f_name = NULL; } diff --git a/src/ccalltestdep.c b/src/ccalltestdep.c new file mode 100644 index 00000000000000..7212716abba41a --- /dev/null +++ b/src/ccalltestdep.c @@ -0,0 +1,41 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + +#include +#include +#include +#include +#include + +#include "../src/support/platform.h" +#include "../src/support/dtypes.h" + +// Borrow definition from `support/dtypes.h` +#ifdef _OS_WINDOWS_ +# define DLLEXPORT __declspec(dllexport) +#else +# if defined(_OS_LINUX_) && !defined(_COMPILER_CLANG_) +// Clang and ld disagree about the proper relocation for STV_PROTECTED, causing +// linker errors. +# define DLLEXPORT __attribute__ ((visibility("protected"))) +# else +# define DLLEXPORT __attribute__ ((visibility("default"))) +# endif +#endif + +#ifdef _P64 +#define jint int64_t +#else +#define jint int32_t +#endif + +typedef struct { + jint real; + jint imag; +} complex_t; + +// We expect this to come from `libccalltest` +extern complex_t ctest(complex_t); + +DLLEXPORT complex_t dep_ctest(complex_t a) { + return ctest(a); +} diff --git a/src/init.c b/src/init.c index 02769e03c668e5..a1c36642458f4e 100644 --- a/src/init.c +++ b/src/init.c @@ -382,6 +382,7 @@ JL_DLLEXPORT void jl_postoutput_hook(void) } void post_boot_hooks(void); +void post_image_load_hooks(void); JL_DLLEXPORT void *jl_libjulia_internal_handle; JL_DLLEXPORT void *jl_libjulia_handle; @@ -888,6 +889,8 @@ static NOINLINE void _finish_julia_init(JL_IMAGE_SEARCH rel, jl_ptls_t ptls, jl_ jl_module_run_initializer((jl_module_t*)mod); } JL_GC_POP(); + + post_image_load_hooks(); } if (jl_options.handle_signals == JL_OPTIONS_HANDLE_SIGNALS_ON) diff --git a/src/jl_exported_data.inc b/src/jl_exported_data.inc index 092a48be819307..d056bd1b763404 100644 --- a/src/jl_exported_data.inc +++ b/src/jl_exported_data.inc @@ -126,6 +126,9 @@ XX(jl_voidpointer_type) \ XX(jl_void_type) \ XX(jl_weakref_type) \ + XX(jl_libdl_module) \ + XX(jl_libdl_dlopen_func) \ + XX(jl_lazy_library_type) \ // Data symbols that are defined inside the public libjulia #define JL_EXPORTED_DATA_SYMBOLS(XX) \ diff --git a/src/jltypes.c b/src/jltypes.c index 444923f6005692..53260b0be35ee0 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3439,6 +3439,24 @@ void post_boot_hooks(void) } export_small_typeof(); } + +void post_image_load_hooks(void) { + // Ensure that `Base` has been loaded. + assert(jl_base_module != NULL); + + jl_libdl_module = (jl_module_t *)jl_get_global( + jl_get_global(((jl_module_t *)jl_base_module), jl_symbol("Libc")), + jl_symbol("Libdl") + ); + jl_libdl_dlopen_func = jl_get_global( + jl_libdl_module, + jl_symbol("dlopen") + ); + jl_lazy_library_type = (jl_datatype_t *)jl_get_global( + jl_libdl_module, + jl_symbol("LazyLibrary") + ); +} #undef XX #ifdef __cplusplus diff --git a/src/julia.h b/src/julia.h index 2140b0ad0ab90b..a35f23b0626b1c 100644 --- a/src/julia.h +++ b/src/julia.h @@ -883,6 +883,9 @@ extern JL_DLLIMPORT jl_value_t *jl_false JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_nothing JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_kwcall_func JL_GLOBALLY_ROOTED; +extern JL_DLLIMPORT jl_value_t *jl_libdl_dlopen_func JL_GLOBALLY_ROOTED; +extern JL_DLLIMPORT jl_datatype_t *jl_lazy_library_type JL_GLOBALLY_ROOTED; + // gc ------------------------------------------------------------------------- struct _jl_gcframe_t { @@ -1700,6 +1703,7 @@ extern JL_DLLIMPORT jl_module_t *jl_main_module JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_module_t *jl_core_module JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_module_t *jl_base_module JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_module_t *jl_top_module JL_GLOBALLY_ROOTED; +extern JL_DLLIMPORT jl_module_t *jl_libdl_module JL_GLOBALLY_ROOTED; JL_DLLEXPORT jl_module_t *jl_new_module(jl_sym_t *name, jl_module_t *parent); JL_DLLEXPORT void jl_set_module_nospecialize(jl_module_t *self, int on); JL_DLLEXPORT void jl_set_module_optlevel(jl_module_t *self, int lvl); diff --git a/src/runtime_ccall.cpp b/src/runtime_ccall.cpp index 23793254c205d2..79ab4af7050483 100644 --- a/src/runtime_ccall.cpp +++ b/src/runtime_ccall.cpp @@ -66,16 +66,19 @@ void *jl_load_and_lookup(const char *f_lib, const char *f_name, _Atomic(void*) * extern "C" JL_DLLEXPORT void *jl_lazy_load_and_lookup(jl_value_t *lib_val, const char *f_name) { - char *f_lib; + void *lib_ptr; if (jl_is_symbol(lib_val)) - f_lib = jl_symbol_name((jl_sym_t*)lib_val); + lib_ptr = jl_get_library(jl_symbol_name((jl_sym_t*)lib_val)); else if (jl_is_string(lib_val)) - f_lib = jl_string_data(lib_val); - else + lib_ptr = jl_get_library(jl_string_data(lib_val)); + else if (jl_lazy_library_type != NULL && jl_libdl_dlopen_func != NULL && jl_isa(lib_val, (jl_value_t *)jl_lazy_library_type)) { + // Call `dlopen(lib_val)` if `lib_val` is a `LazyLibrary` object, use the returned handle as `lib_ptr`. + lib_ptr = *((void **)jl_apply_generic(jl_libdl_dlopen_func, &lib_val, 1)); + } else jl_type_error("ccall", (jl_value_t*)jl_symbol_type, lib_val); void *ptr; - jl_dlsym(jl_get_library(f_lib), f_name, &ptr, 1); + jl_dlsym(lib_ptr, f_name, &ptr, 1); return ptr; } diff --git a/stdlib/Libdl/src/Libdl.jl b/stdlib/Libdl/src/Libdl.jl index df3f62c807fede..6bc17b7d690b95 100644 --- a/stdlib/Libdl/src/Libdl.jl +++ b/stdlib/Libdl/src/Libdl.jl @@ -4,10 +4,10 @@ module Libdl # Just re-export Base.Libc.Libdl: export DL_LOAD_PATH, RTLD_DEEPBIND, RTLD_FIRST, RTLD_GLOBAL, RTLD_LAZY, RTLD_LOCAL, RTLD_NODELETE, RTLD_NOLOAD, RTLD_NOW, dlclose, dlopen, dlopen_e, dlsym, dlsym_e, - dlpath, find_library, dlext, dllist + dlpath, find_library, dlext, dllist, LazyLibrary import Base.Libc.Libdl: DL_LOAD_PATH, RTLD_DEEPBIND, RTLD_FIRST, RTLD_GLOBAL, RTLD_LAZY, RTLD_LOCAL, RTLD_NODELETE, RTLD_NOLOAD, RTLD_NOW, dlclose, dlopen, dlopen_e, dlsym, dlsym_e, - dlpath, find_library, dlext, dllist + dlpath, find_library, dlext, dllist, LazyLibrary, default_rtld_flags end # module diff --git a/stdlib/Libdl/test/runtests.jl b/stdlib/Libdl/test/runtests.jl index 6863e28959b5e9..1bd7d3f45a8400 100644 --- a/stdlib/Libdl/test/runtests.jl +++ b/stdlib/Libdl/test/runtests.jl @@ -1,7 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license using Test -import Libdl +using Libdl # these could fail on an embedded installation # but for now, we don't handle that case @@ -27,8 +27,6 @@ end @test_throws ArgumentError Libdl.dlsym(C_NULL, :foo) @test_throws ArgumentError Libdl.dlsym_e(C_NULL, :foo) -cd(@__DIR__) do - # Find the library directory by finding the path of libjulia-internal (or libjulia-internal-debug, # as the case may be) to get the private library directory private_libdir = if Base.DARWIN_FRAMEWORK @@ -267,4 +265,47 @@ mktempdir() do dir end end +## Tests for LazyLibrary +@testset "LazyLibrary" begin + default_rtld_flags = Base.Libc.Libdl.default_rtld_flags + # Ensure that `libccalltest` is not currently loaded + @test !any(contains.(dllist(), "libccalltest")) + + # Create a `LazyLibrary` structure that loads `libccalltestdep` + global libccalltest_loaded = false + global libccalltestdep_loaded = false + + global libccalltest = LazyLibrary(joinpath(private_libdir, "libccalltest.$(Libdl.dlext)"), default_rtld_flags, LazyLibrary[], hdl -> global libccalltest_loaded = true) + global libccalltestdep = LazyLibrary(joinpath(private_libdir, "libccalltestdep.$(Libdl.dlext)"), default_rtld_flags, [libccalltest], hdl -> global libccalltestdep_loaded = true) + + @test !libccalltest_loaded + @test !libccalltestdep_loaded + + # Test that the library gets loaded, and that it works + a = 20 + 51im + @test ccall((:ctest, libccalltestdep), Complex{Int}, (Complex{Int},), a) == a + 1 - 2im + @test libccalltest_loaded + @test libccalltestdep_loaded + + # Test that `dlclose()` works + dlclose(libccalltestdep) + dlclose(libccalltest) + @test !any(contains.(dllist(), "libccalltest")) + + # Test that `@ccall` works: + libccalltest_loaded = false + libccalltestdep_loaded = false + x = @ccall(libccalltestdep.ctest(a::Complex{Int})::Complex{Int}) + @test x == a + 1 - 2im + @test libccalltest_loaded + @test libccalltestdep_loaded + + dlclose(libccalltestdep) + dlclose(libccalltest) + + # Test that `dlpath()` works + libccalltest_loaded = false + @test dlpath(libccalltest) == libccalltest.name + @test libccalltest_loaded + dlclose(libccalltest) end