Split libjulia into runtime and codegen components

Make.inc

@@ -1527,10 +1527,18 @@ $$($(PYTHON) $(call python_cygpath,$(JULIAHOME)/contrib/relative_path.py) $(1) $
 endef
 
 LIBJULIAINTERNAL_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_shlibdir)/libjulia-internal.$(JL_MAJOR_MINOR_SHLIB_EXT))
+LIBJULIARUNTIMEINTERNAL_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_shlibdir)/libjulia-runtime-internal.$(JL_MAJOR_MINOR_SHLIB_EXT))
+LIBJULIASTUBINTERNAL_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_shlibdir)/libjulia-stub-internal.$(JL_MAJOR_MINOR_SHLIB_EXT))
 LIBJULIAINTERNAL_INSTALL_DEPLIB := $(call dep_lib_path,$(libdir),$(private_shlibdir)/libjulia-internal.$(JL_MAJOR_MINOR_SHLIB_EXT))
+LIBJULIARUNTIMEINTERNAL_INSTALL_DEPLIB := $(call dep_lib_path,$(libdir),$(private_shlibdir)/libjulia-runtime-internal.$(JL_MAJOR_MINOR_SHLIB_EXT))
+LIBJULIASTUBINTERNAL_INSTALL_DEPLIB := $(call dep_lib_path,$(libdir),$(private_shlibdir)/libjulia-stub-internal.$(JL_MAJOR_MINOR_SHLIB_EXT))
 
 LIBJULIAINTERNAL_DEBUG_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_shlibdir)/libjulia-internal-debug.$(JL_MAJOR_MINOR_SHLIB_EXT))
+LIBJULIARUNTIMEINTERNAL_DEBUG_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_shlibdir)/libjulia-runtime-internal-debug.$(JL_MAJOR_MINOR_SHLIB_EXT))
+LIBJULIASTUBINTERNAL_DEBUG_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_shlibdir)/libjulia-stub-internal-debug.$(JL_MAJOR_MINOR_SHLIB_EXT))
 LIBJULIAINTERNAL_DEBUG_INSTALL_DEPLIB := $(call dep_lib_path,$(libdir),$(private_shlibdir)/libjulia-internal-debug.$(JL_MAJOR_MINOR_SHLIB_EXT))
+LIBJULIARUNTIMEINTERNAL_DEBUG_INSTALL_DEPLIB := $(call dep_lib_path,$(libdir),$(private_shlibdir)/libjulia-runtime-internal-debug.$(JL_MAJOR_MINOR_SHLIB_EXT))
+LIBJULIASTUBINTERNAL_DEBUG_INSTALL_DEPLIB := $(call dep_lib_path,$(libdir),$(private_shlibdir)/libjulia-stub-internal-debug.$(JL_MAJOR_MINOR_SHLIB_EXT))
 
 ifeq ($(OS),WINNT)
 ifeq ($(BINARY),32)
@@ -1565,16 +1573,16 @@ LIBM_INSTALL_DEPLIB := $(call dep_lib_path,$(libdir),$(private_shlibdir)/$(LIBMN
 # We list:
 #  * libgcc_s, because FreeBSD needs to load ours, not the system one.
 #  * libopenlibm, because Windows has an untrustworthy libm, and we want to use ours more than theirs
-#  * libjulia, which must always come last.
+#  * libjulia-internal, which must always come last.
 #
 # We need these four separate variables because:
 #  * debug builds must link against libjuliadebug, not libjulia
 #  * install time relative paths are not equal to build time relative paths (../lib vs. ../lib/julia)
 # That second point will no longer be true for most deps once they are placed within Artifacts directories.
-LOADER_BUILD_DEP_LIBS = $(LIBGCC_BUILD_DEPLIB):$(LIBM_BUILD_DEPLIB):$(LIBJULIAINTERNAL_BUILD_DEPLIB)
-LOADER_DEBUG_BUILD_DEP_LIBS = $(LIBGCC_BUILD_DEPLIB):$(LIBM_BUILD_DEPLIB):$(LIBJULIAINTERNAL_DEBUG_BUILD_DEPLIB)
-LOADER_INSTALL_DEP_LIBS = $(LIBGCC_INSTALL_DEPLIB):$(LIBM_INSTALL_DEPLIB):$(LIBJULIAINTERNAL_INSTALL_DEPLIB)
-LOADER_DEBUG_INSTALL_DEP_LIBS = $(LIBGCC_INSTALL_DEPLIB):$(LIBM_INSTALL_DEPLIB):$(LIBJULIAINTERNAL_DEBUG_INSTALL_DEPLIB)
+LOADER_BUILD_DEP_LIBS = $(LIBGCC_BUILD_DEPLIB):$(LIBM_BUILD_DEPLIB):$(LIBJULIARUNTIMEINTERNAL_BUILD_DEPLIB):$(LIBJULIAINTERNAL_BUILD_DEPLIB)
+LOADER_DEBUG_BUILD_DEP_LIBS = $(LIBGCC_BUILD_DEPLIB):$(LIBM_BUILD_DEPLIB):$(LIBJULIARUNTIMEINTERNAL_DEBUG_BUILD_DEPLIB):$(LIBJULIAINTERNAL_DEBUG_BUILD_DEPLIB)
+LOADER_INSTALL_DEP_LIBS = $(LIBGCC_INSTALL_DEPLIB):$(LIBM_INSTALL_DEPLIB):$(LIBJULIARUNTIMEINTERNAL_INSTALL_DEPLIB):$(LIBJULIAINTERNAL_INSTALL_DEPLIB)
+LOADER_DEBUG_INSTALL_DEP_LIBS = $(LIBGCC_INSTALL_DEPLIB):$(LIBM_INSTALL_DEPLIB):$(LIBJULIARUNTIMEINTERNAL_DEBUG_INSTALL_DEPLIB):$(LIBJULIAINTERNAL_DEBUG_INSTALL_DEPLIB)
 
 # Colors for make
 ifndef VERBOSE

cli/Makefile

@@ -28,7 +28,9 @@ endif
 
 # Build list of dependent libraries that must be opened
 SHIPFLAGS  += -DDEP_LIBS="\"$(LOADER_BUILD_DEP_LIBS)\""
+SHIPFLAGS  += -DSTUB_LIB="\"$(LIBJULIASTUBINTERNAL_BUILD_DEPLIB)\""
 DEBUGFLAGS += -DDEP_LIBS="\"$(LOADER_DEBUG_BUILD_DEP_LIBS)\""
+DEBUGFLAGS += -DSTUB_LIB="\"$(LIBJULIASTUBINTERNAL_DEBUG_BUILD_DEPLIB)\""
 
 EXE_OBJS := $(BUILDDIR)/loader_exe.o
 EXE_DOBJS := $(BUILDDIR)/loader_exe.dbg.obj
@@ -102,7 +104,7 @@ ABI_UNDERSCORE := _\#\#
 else
 ABI_UNDERSCORE :=
 endif
-EXPORTED_FUNCS := $(shell echo -e "#include \"jl_exported_funcs.inc\"\n#define XX(x) $(ABI_UNDERSCORE)x\nJL_EXPORTED_FUNCS(XX)" | $(CPP) -I$(JULIAHOME)/src - | tail -n 1)
+EXPORTED_FUNCS := $(shell echo -e "#include \"jl_exported_funcs.inc\"\n#define XX(x) $(ABI_UNDERSCORE)x\nJL_RUNTIME_EXPORTED_FUNCS(XX)\nJL_CODEGEN_EXPORTED_FUNCS(XX)" | $(CPP) -I$(JULIAHOME)/src - | tail -n 1)
 STRIP_EXPORTED_FUNCS := $(patsubst %,--strip-symbol=%,$(EXPORTED_FUNCS))
 
 # Note that if the objcopy command starts getting too long, we can use `@file` to read

cli/jl_exports.h

@@ -12,20 +12,29 @@ JL_EXPORTED_DATA(XX)
 
 // Define holder locations for function addresses as `const void * $(name)_addr`
 #define XX(name)    JL_HIDDEN const void * name##_addr;
-JL_EXPORTED_FUNCS(XX)
+JL_RUNTIME_EXPORTED_FUNCS(XX)
+JL_CODEGEN_EXPORTED_FUNCS(XX)
 #undef XX
 
 // Generate lists of function names and addresses
 #define XX(name)    #name,
-static const char *const jl_exported_func_names[] = {
-    JL_EXPORTED_FUNCS(XX)
+static const char *const jl_runtime_exported_func_names[] = {
+    JL_RUNTIME_EXPORTED_FUNCS(XX)
+    NULL
+};
+static const char *const jl_codegen_exported_func_names[] = {
+    JL_CODEGEN_EXPORTED_FUNCS(XX)
     NULL
 };
 #undef XX
 
 #define XX(name)    &name##_addr,
-static const void ** jl_exported_func_addrs[] = {
-    JL_EXPORTED_FUNCS(XX)
+static const void ** jl_runtime_exported_func_addrs[] = {
+    JL_RUNTIME_EXPORTED_FUNCS(XX)
+    NULL
+};
+static const void ** jl_codegen_exported_func_addrs[] = {
+    JL_CODEGEN_EXPORTED_FUNCS(XX)
     NULL
 };
 #undef XX

cli/loader.h

@@ -85,6 +85,10 @@
 #define DEP_LIBS "../lib/example.so:../lib/libjulia.so"
 #endif
 
+#if !defined(STUB_LIB)
+#define STUB_LIB "../lib/libjulia-stub-internal.so"
+#endif
+
 // We need to dlopen() ourselves in order to introspect the libdir.
 #if defined(JL_DEBUG_BUILD)
 #define LIBJULIA_NAME "libjulia-debug"

cli/loader_lib.c

@@ -124,10 +124,17 @@ const char * get_libdir()
 }
 
 void * libjulia_internal = NULL;
+void * libjulia_runtime = NULL;
+int use_stubs = 0;
 __attribute__((constructor)) void jl_load_libjulia_internal(void) {
     // Introspect to find our own path
     const char * lib_dir = get_libdir();
 
+    // Check if we should use codegen stub
+    char *use_stubs_env = getenv("JULIA_STUB_CODEGEN");
+    if (use_stubs_env && strcmp(use_stubs_env, "1"))
+        use_stubs = 1;
+
     // Pre-load libraries that libjulia-internal needs.
     int deps_len = strlen(dep_libs);
     char * curr_dep = &dep_libs[0];
@@ -139,18 +146,33 @@ __attribute__((constructor)) void jl_load_libjulia_internal(void) {
 
         // Chop the string at the colon, load this library.
         *colon = '\0';
-        load_library(curr_dep, lib_dir);
+        void *lib = load_library(curr_dep, lib_dir);
+
+        // Grab libjulia-runtime-internal
+        if (strchr(colon+1, ':') == NULL) {
+            libjulia_runtime = lib;
+        }
 
         // Skip ahead to next dependency
         curr_dep = colon + 1;
     }
 
     // Last dependency is `libjulia-internal`, so load that and we're done with `dep_libs`!
-    libjulia_internal = load_library(curr_dep, lib_dir);
+    if (use_stubs) {
+        // If requested, load `libjulia-stub-internal` instead
+        libjulia_internal = load_library(STUB_LIB, lib_dir);
+    } else {
+        libjulia_internal = load_library(curr_dep, lib_dir);
+    }
 
     // Once we have libjulia-internal loaded, re-export its symbols:
-    for (unsigned int symbol_idx=0; jl_exported_func_names[symbol_idx] != NULL; ++symbol_idx) {
-        (*jl_exported_func_addrs[symbol_idx]) = lookup_symbol(libjulia_internal, jl_exported_func_names[symbol_idx]);
+    for (unsigned int symbol_idx=0; jl_runtime_exported_func_names[symbol_idx] != NULL; ++symbol_idx) {
+        const char *name = jl_runtime_exported_func_names[symbol_idx];
+        (*jl_runtime_exported_func_addrs[symbol_idx]) = lookup_symbol(libjulia_runtime, name);
+    }
+    for (unsigned int symbol_idx=0; jl_codegen_exported_func_names[symbol_idx] != NULL; ++symbol_idx) {
+        const char *name = jl_codegen_exported_func_names[symbol_idx];
+        (*jl_codegen_exported_func_addrs[symbol_idx]) = lookup_symbol(libjulia_internal, name);
     }
 }
 

cli/trampolines/trampolines_aarch64.S

@@ -8,5 +8,6 @@ name##:; \
     ldr x0, [x0, #:lo12:name##_addr]; \
     br x0; \
 .cfi_endproc; \
-JL_EXPORTED_FUNCS(XX)
+JL_RUNTIME_EXPORTED_FUNCS(XX)
+JL_CODEGEN_EXPORTED_FUNCS(XX)
 #undef XX

cli/trampolines/trampolines_arm.S

@@ -13,5 +13,6 @@ name##:; \
     .word name##_addr-(.L##name + 8); \
 .cfi_endproc; \
 
-JL_EXPORTED_FUNCS(XX)
+JL_RUNTIME_EXPORTED_FUNCS(XX)
+JL_CODEGEN_EXPORTED_FUNCS(XX)
 #undef XX

cli/trampolines/trampolines_i686.S

@@ -28,5 +28,6 @@ name##:; \
     ud2; \
 .cfi_endproc; \
 
-JL_EXPORTED_FUNCS(XX)
+JL_RUNTIME_EXPORTED_FUNCS(XX)
+JL_CODEGEN_EXPORTED_FUNCS(XX)
 #undef XX

cli/trampolines/trampolines_ppc64le.S

@@ -10,5 +10,6 @@ name##: ; \
     bctr; \
 .cfi_endproc; \
 
-JL_EXPORTED_FUNCS(XX)
+JL_RUNTIME_EXPORTED_FUNCS(XX)
+JL_CODEGEN_EXPORTED_FUNCS(XX)
 #undef XX

cli/trampolines/trampolines_x86_64.S

@@ -47,5 +47,6 @@ SEH_END(); \
 .cfi_endproc; \
 EXPORT(name); \
 
-JL_EXPORTED_FUNCS(XX)
+JL_RUNTIME_EXPORTED_FUNCS(XX)
+JL_CODEGEN_EXPORTED_FUNCS(XX)
 #undef XX