Load special libraries in-order

The `DEPS_LIBS` RPATH-substitute mechanism contains a list of paths to
load, and some of these paths are "special", in that they require more
involved loading than simply `load_library()`.  These libraries are
thereby denoted by a `@` prefixing them.

Previously, we made note of these libraries, then loaded them at the end
of the loading loop, but with the addition of `libstdc++` it is now
important to have the order of the libraries (including special
libraries) to be obeyed by the loading loop, so I have inlined special
library handling into the loading loop.  In the future, we may wish to
denote special libraries more explicitly than simply relying on there
being exactly three libraries, with the ordering being mapped to
`libstdc++`, `libjulia-internal`, and `libjulia-codegen`.

Author: staticfloat

Make.inc

@@ -1465,7 +1465,7 @@ JULIA_SYSIMG_release := $(build_private_libdir)/sys.$(SHLIB_EXT)
 JULIA_SYSIMG := $(JULIA_SYSIMG_$(JULIA_BUILD_MODE))
 
 define dep_lib_path
-$$($(PYTHON) $(call python_cygpath,$(JULIAHOME)/contrib/relative_path.py) $(1) $(2))
+$(shell $(PYTHON) $(call python_cygpath,$(JULIAHOME)/contrib/relative_path.py) $(1) $(2))
 endef
 
 LIBJULIAINTERNAL_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_shlibdir)/libjulia-internal.$(JL_MAJOR_SHLIB_EXT))
@@ -1538,6 +1538,8 @@ LIBM_INSTALL_DEPLIB := $(call dep_lib_path,$(libdir),$(private_shlibdir)/$(LIBMN
 # We list:
 #  * libgcc_s, because FreeBSD needs to load ours, not the system one.
 #  * libopenlibm, because Windows has an untrustworthy libm, and we want to use ours more than theirs
+#  * libstdc++, because while performing `libstdc++` probing we need to
+#    know the path to the bundled `libstdc++` library.
 #  * libjulia-internal, which must always come second-to-last.
 #  * libjulia-codegen, which must always come last
 #
@@ -1546,11 +1548,45 @@ LIBM_INSTALL_DEPLIB := $(call dep_lib_path,$(libdir),$(private_shlibdir)/$(LIBMN
 #  * install time relative paths are not equal to build time relative paths (../lib vs. ../lib/julia)
 # That second point will no longer be true for most deps once they are placed within Artifacts directories.
 # Note that we prefix `libjulia-codegen` and `libjulia-internal` with `@` to signify to the loader that it
-# should not automatically dlopen() it in its loading loop.
-LOADER_BUILD_DEP_LIBS = $(LIBGCC_BUILD_DEPLIB):$(LIBM_BUILD_DEPLIB):@$(LIBSTDCXX_BUILD_DEPLIB):@$(LIBJULIAINTERNAL_BUILD_DEPLIB):@$(LIBJULIACODEGEN_BUILD_DEPLIB):
-LOADER_DEBUG_BUILD_DEP_LIBS = $(LIBGCC_BUILD_DEPLIB):$(LIBM_BUILD_DEPLIB):@$(LIBSTDCXX_BUILD_DEPLIB):@$(LIBJULIAINTERNAL_DEBUG_BUILD_DEPLIB):@$(LIBJULIACODEGEN_DEBUG_BUILD_DEPLIB):
-LOADER_INSTALL_DEP_LIBS = $(LIBGCC_INSTALL_DEPLIB):$(LIBM_INSTALL_DEPLIB):@$(LIBSTDCXX_INSTALL_DEPLIB):@$(LIBJULIAINTERNAL_INSTALL_DEPLIB):@$(LIBJULIACODEGEN_INSTALL_DEPLIB):
-LOADER_DEBUG_INSTALL_DEP_LIBS = $(LIBGCC_INSTALL_DEPLIB):$(LIBM_INSTALL_DEPLIB):@$(LIBSTDCXX_INSTALL_DEPLIB):@$(LIBJULIAINTERNAL_DEBUG_INSTALL_DEPLIB):@$(LIBJULIACODEGEN_DEBUG_INSTALL_DEPLIB):
+# should not automatically dlopen() it in its loading loop, it is "special" and should happen later.
+# We do the same for `libstdc++`, and explicitly place it _after_ `libgcc_s`, and `libm` since `libstdc++`
+# may depend on those libraries (e.g. when USE_SYSTEM_LIBM=1)
+
+# Helper function to join a list with colons, then place an extra at the end.
+define build_deplibs
+$(subst $(SPACE),:,$(strip $(1))):
+endef
+
+LOADER_BUILD_DEP_LIBS = $(call build_deplibs, \
+    $(LIBGCC_BUILD_DEPLIB) \
+    $(LIBM_BUILD_DEPLIB) \
+    @$(LIBSTDCXX_BUILD_DEPLIB) \
+    @$(LIBJULIAINTERNAL_BUILD_DEPLIB) \
+    @$(LIBJULIACODEGEN_BUILD_DEPLIB) \
+)
+
+LOADER_DEBUG_BUILD_DEP_LIBS = $(call build_deplibs, \
+   $(LIBGCC_BUILD_DEPLIB) \
+   $(LIBM_BUILD_DEPLIB) \
+   @$(LIBSTDCXX_BUILD_DEPLIB) \
+   @$(LIBJULIAINTERNAL_DEBUG_BUILD_DEPLIB) \
+   @$(LIBJULIACODEGEN_DEBUG_BUILD_DEPLIB) \
+)
+
+LOADER_INSTALL_DEP_LIBS = $(call build_deplibs, \
+    $(LIBGCC_INSTALL_DEPLIB) \
+    $(LIBM_INSTALL_DEPLIB) \
+    @$(LIBSTDCXX_INSTALL_DEPLIB) \
+    @$(LIBJULIAINTERNAL_INSTALL_DEPLIB) \
+    @$(LIBJULIACODEGEN_INSTALL_DEPLIB) \
+)
+LOADER_DEBUG_INSTALL_DEP_LIBS = $(call build_deplibs, \
+    $(LIBGCC_INSTALL_DEPLIB) \
+    $(LIBM_INSTALL_DEPLIB) \
+    @$(LIBSTDCXX_INSTALL_DEPLIB) \
+    @$(LIBJULIAINTERNAL_DEBUG_INSTALL_DEPLIB) \
+    @$(LIBJULIACODEGEN_DEBUG_INSTALL_DEPLIB) \
+)
 
 # Colors for make
 ifndef VERBOSE

cli/loader_lib.c

@@ -350,7 +350,8 @@ static char *libstdcxxprobe(void)
 }
 #endif
 
-void * libjulia_internal = NULL;
+void *libjulia_internal = NULL;
+void *libjulia_codegen = NULL;
 __attribute__((constructor)) void jl_load_libjulia_internal(void) {
     // Only initialize this once
     if (libjulia_internal != NULL) {
@@ -364,18 +365,14 @@ __attribute__((constructor)) void jl_load_libjulia_internal(void) {
     int deps_len = strlen(&dep_libs[1]);
     char *curr_dep = &dep_libs[1];
 
-    void *cxx_handle;
-
     // We keep track of "special" libraries names (ones whose name is prefixed with `@`)
     // which are libraries that we want to load in some special, custom way.
     // The current list is:
-    // special_library_names = {
-    //   libstdc++,
-    //   libjulia-internal,
-    //   libjulia-codegen,
-    // }
+    //   libstdc++
+    //   libjulia-internal
+    //   libjulia-codegen
+    const int NUM_SPECIAL_LIBRARIES = 3;
     int special_idx = 0;
-    char * special_library_names[3] = {NULL};
     while (1) {
         // try to find next colon character; if we can't, break out
         char * colon = strchr(curr_dep, ':');
@@ -384,61 +381,30 @@ __attribute__((constructor)) void jl_load_libjulia_internal(void) {
 
         // If this library name starts with `@`, don't open it here (but mark it as special)
         if (curr_dep[0] == '@') {
-            if (special_idx > sizeof(special_library_names)/sizeof(char *)) {
+            special_idx += 1;
+            if (special_idx > NUM_SPECIAL_LIBRARIES) {
                 jl_loader_print_stderr("ERROR: Too many special library names specified, check LOADER_BUILD_DEP_LIBS and friends!\n");
                 exit(1);
             }
-            special_library_names[special_idx] = curr_dep + 1;
-            special_idx += 1;
-
-            // Chop the string at the colon so it's a valid-ending-string
-            *colon = '\0';
         }
 
         // Skip to next dep
         curr_dep = colon + 1;
     }
 
     // Assert that we have exactly the right number of special library names
-    if (special_idx != sizeof(special_library_names)/sizeof(char *)) {
+    if (special_idx != NUM_SPECIAL_LIBRARIES) {
         jl_loader_print_stderr("ERROR: Too few special library names specified, check LOADER_BUILD_DEP_LIBS and friends!\n");
         exit(1);
     }
 
-    // Unpack our special library names.  This is why ordering of library names matters.
-    char * bundled_libstdcxx_path = special_library_names[0];
-    libjulia_internal = load_library(special_library_names[1], lib_dir, 1);
-    void *libjulia_codegen = load_library(special_library_names[2], lib_dir, 0);
-
-#if defined(_OS_LINUX_)
-    int do_probe = 1;
-    int done_probe = 0;
-    char *probevar = getenv("JULIA_PROBE_LIBSTDCXX");
-    if (probevar) {
-        if (strcmp(probevar, "1") == 0 || strcmp(probevar, "yes") == 0)
-            do_probe = 1;
-        else if (strcmp(probevar, "0") == 0 || strcmp(probevar, "no") == 0)
-            do_probe = 0;
-    }
-    if (do_probe) {
-        char *cxxpath = libstdcxxprobe();
-        if (cxxpath) {
-            cxx_handle = dlopen(cxxpath, RTLD_LAZY);
-            char *dlr = dlerror();
-            if (dlr) {
-                jl_loader_print_stderr("ERROR: Unable to dlopen(cxxpath) in parent!\n");
-                jl_loader_print_stderr3("Message: ", dlr, "\n");
-                exit(1);
-            }
-            free(cxxpath);
-            done_probe = 1;
-        }
-    }
-    if (!done_probe) {
-        load_library(bundled_libstdcxx_path, lib_dir, 1);
-    }
-#endif
-
+    // Now that we've asserted that we have the right number of special
+    // libraries, actually run a loop over the deps loading them in-order.
+    // If it's a special library, we do slightly different things, especially
+    // for libstdc++, where we actually probe for a system libstdc++ and
+    // load that if it's newer.
+    special_idx = 0;
+    curr_dep = &dep_libs[1];
     while (1) {
         // try to find next colon character; if we can't, break out
         char * colon = strchr(curr_dep, ':');
@@ -448,8 +414,56 @@ __attribute__((constructor)) void jl_load_libjulia_internal(void) {
         // Chop the string at the colon so it's a valid-ending-string
         *colon = '\0';
 
-        // If this library name starts with `@`, don't open it here
-        if (curr_dep[0] != '@') {
+        // If this library name starts with `@`, it's a special library
+        // and requires special handling:
+        if (curr_dep[0] == '@') {
+            // Skip the `@` for future function calls.
+            curr_dep += 1;
+
+            // First special library to be loaded is `libstdc++`; perform probing here.
+            if (special_idx == 0) {
+#if defined(_OS_LINUX_)
+                int do_probe = 1;
+                int probe_successful = 0;
+
+                // Check to see if the user has disabled libstdc++ probing
+                char *probevar = getenv("JULIA_PROBE_LIBSTDCXX");
+                if (probevar) {
+                    if (strcmp(probevar, "1") == 0 || strcmp(probevar, "yes") == 0)
+                        do_probe = 1;
+                    else if (strcmp(probevar, "0") == 0 || strcmp(probevar, "no") == 0)
+                        do_probe = 0;
+                }
+                if (do_probe) {
+                    char *cxxpath = libstdcxxprobe();
+                    if (cxxpath) {
+                        void *cxx_handle = dlopen(cxxpath, RTLD_LAZY);
+                        const char *dlr = dlerror();
+                        if (dlr) {
+                            jl_loader_print_stderr("ERROR: Unable to dlopen(cxxpath) in parent!\n");
+                            jl_loader_print_stderr3("Message: ", dlr, "\n");
+                            exit(1);
+                        }
+                        free(cxxpath);
+                        probe_successful = 1;
+                    }
+                }
+                // If the probe rejected the system libstdc++ (or didn't find one!)
+                // just load our bundled libstdc++ as identified by curr_dep;
+                if (!probe_successful) {
+                    load_library(curr_dep, lib_dir, 1);
+                }
+#endif
+            } else if (special_idx == 1) {
+                // This special library is `libjulia-internal`
+                libjulia_internal = load_library(curr_dep, lib_dir, 1);
+            } else if (special_idx == 2) {
+                // This special library is `libjulia-codegen`
+                libjulia_codegen = load_library(curr_dep, lib_dir, 0);
+            }
+            special_idx++;
+        } else {
+            // Otherwise, just load it as "normal"
             load_library(curr_dep, lib_dir, 1);
         }