Remove libjulia-internal.so dependency on libstdc++ and libgcc (#60248)

Some testing by @BenChung has revealed how hard it is to load a
JuliaC-built library into a program that has already loaded a very old
version of libstdc++. Even with probing disabled, `libjulia-internal.so`
fails because of missing GLIBCXX symbols.

We use so little of the C++ standard library in `libjulia-internal.so`
that it's worth the tradeoff to link it statically: it barely changes
the size of the resulting library, removes a medium-size library we have
to ship in trimmed bundles, and solves some of our hermeticity issues
when being loaded by other software. `libjulia-codegen.so` uses it more
extensively, and we expect to be able to load it as a plugin for `opt`,
meaning it may have to remain dynamically linked.

This PR contains a series of changes to enable statically linked
libstdc++ by default and mitigate the size impact:
- We enable `--gc-sections` when building with gcc/ld.bfd. This saves us
some code already, since we can trim out a lot of
libLLVMSupport/libLLVMTargetParser. On macOS, we can use `-dead_strip`
to save some space even though the rest of these changes are not
applicable.
- Two flags for `-static-libstdc++` and `-static-libgcc`, called
`USE_RT_STATIC_LIBSTDCXX` and `USE_RT_STATIC_LIBGCC`, are added and
enabled by default.
- Most of the additional code that gets linked into
`libjulia-internal.so` is related to locales for iostreams. Replace
these with standard C IO and LLVM helpers that don't have weird
locale-related behaviour.
- (NOT IMPLEMENTED) `--gc-sections` removes unused executable code, but
leaves us with a lot of irrelevant debug info. I tested the effect of
`llvm-dwarfutil --garbage-collection` and saw pretty good savings, but
that is not included in this PR because BinaryBuilder doesn't have a new
enough version of the LLVM tools.

| Change | Size of `libjulia-internal.so` (KiB) |

|--------------------------------------|--------------------------------------|
| No change | 14524 |
| Linker GC enabled | 13220 |
| -static-libstdc++ and -static-libgcc | 22136 |
| Excise iostreams | 15036 |
| DWARF GC (not in this PR) | 11488 |

This is a comparison of symbols that were added and removed. Even though
15 times more code is removed than added, the resulting
`libjulia-internal.so` has a similar size to the original because of the
additional debug info.

(cherry picked from commit f36882f)

Author: xal-0

Make.inc

@@ -63,6 +63,10 @@ USE_SYSTEM_ZSTD:=0
 USE_SYSTEM_P7ZIP:=0
 USE_SYSTEM_LLD:=0
 
+# Link libjulia-internal with static libgcc and libstdc++
+USE_RT_STATIC_LIBGCC:=1
+USE_RT_STATIC_LIBSTDCXX:=1
+
 # Link to the LLVM shared library
 USE_LLVM_SHLIB := 1
 
@@ -97,6 +101,9 @@ WITH_TRACY_CALLSTACKS := 0
 # Enable Timing Counts support
 WITH_TIMING_COUNTS := 0
 
+# Should --gc-sections/-dead_strip be used to remove unreferenced code?
+USE_LINKER_GC:=1
+
 # Prevent picking up $ARCH from the environment variables
 ARCH:=
 
@@ -932,6 +939,23 @@ JCXXFLAGS += -DUSE_NVTX
 JCFLAGS += -DUSE_NVTX
 endif
 
+ifneq ($(findstring $(OS),WINNT FreeBSD OpenBSD),)
+  USE_LINKER_GC := 0
+  USE_RT_STATIC_LIBGCC := 0
+  USE_RT_STATIC_LIBSTDCXX := 0
+endif
+
+# Linker garbage collection
+ifeq ($(USE_LINKER_GC), 1)
+ifeq ($(OS), Darwin)
+  JLDFLAGS += -Wl,-dead_strip
+else
+  JLDFLAGS += -Wl,--gc-sections
+  JCFLAGS += -ffunction-sections -fdata-sections
+  JCXXFLAGS += -ffunction-sections -fdata-sections
+endif
+endif
+
 # ===========================================================================
 
 # Select the cpu architecture to target, or automatically detects the user's compiler

cli/Makefile

@@ -39,6 +39,11 @@ $(BUILDDIR)/loader_lib.o: export MSYS2_ARG_CONV_EXCL = -DDEP_LIBS=
 $(BUILDDIR)/loader_lib.dbg.obj: export MSYS2_ARG_CONV_EXCL = -DDEP_LIBS=
 endif # MSYS2
 
+ifeq ($(USE_RT_STATIC_LIBSTDCXX),1)
+SHIPFLAGS += -DRT_STATIC_LIBSTDCXX
+DEBUGFLAGS += -DRT_STATIC_LIBSTDCXX
+endif # USE_RT_STATIC_LIBSTDCXX
+
 EXE_OBJS := $(BUILDDIR)/loader_exe.o
 EXE_DOBJS := $(BUILDDIR)/loader_exe.dbg.obj
 LIB_OBJS := $(BUILDDIR)/loader_lib.o

cli/loader_lib.c

@@ -497,7 +497,13 @@ __attribute__((constructor)) void jl_load_libjulia_internal(void) {
                 // If the probe rejected the system libstdc++ (or didn't find one!)
                 // just load our bundled libstdc++ as identified by curr_dep;
                 if (!probe_successful) {
+# ifdef RT_STATIC_LIBSTDCXX
+                    // If we have a statically-linked libstdc++, it is ok for
+                    // this to fail.
+                    load_library(curr_dep, lib_dir, 0);
+# else
                     load_library(curr_dep, lib_dir, 1);
+# endif
                 }
 #endif
             } else if (special_idx == 1) {

src/Makefile

@@ -207,6 +207,15 @@ RT_LIBS := $(WHOLE_ARCHIVE) $(LIBUV) $(WHOLE_ARCHIVE) $(LIBUTF8PROC) $(NO_WHOLE_
 # NB: CG needs uv_mutex_* symbols, but we expect to export them from libjulia-internal
 CG_LIBS := $(LIBUNWIND) $(CG_LLVMLINK) $(OSLIBS) $(LIBTRACYCLIENT) $(LIBITTAPI)
 
+ifeq ($(USEGCC),1)
+ifeq ($(USE_RT_STATIC_LIBGCC),1)
+RT_LIBS += -static-libgcc
+endif
+ifeq ($(USE_RT_STATIC_LIBSTDCXX),1)
+RT_LIBS += -static-libstdc++
+endif
+endif
+
 ifeq (${USE_THIRD_PARTY_GC},mmtk)
 RT_LIBS += $(MMTK_LIB)
 CG_LIBS += $(MMTK_LIB)

src/coverage.cpp

@@ -3,9 +3,6 @@
 #include <cstdint>
 #include <pthread.h>
 #include <string>
-#include <fstream>
-#include <map>
-#include <vector>
 
 #include "llvm-version.h"
 #include <llvm/ADT/StringRef.h>
@@ -137,26 +134,20 @@ static void write_log_data(logdata_t &logData, const char *extension) JL_NOTSAFE
         if (!values.empty()) {
             if (!jl_isabspath(filename.c_str()))
                 filename = base + filename;
-            std::ifstream inf(filename.c_str());
-            if (!inf.is_open())
+            FILE *inf = fopen(filename.c_str(), "r");
+            if (!inf)
                 continue;
             std::string outfile = filename + extension;
-            std::ofstream outf(outfile.c_str(), std::ofstream::trunc | std::ofstream::out | std::ofstream::binary);
-            if (outf.is_open()) {
-                inf.exceptions(std::ifstream::badbit);
-                outf.exceptions(std::ifstream::failbit | std::ifstream::badbit);
+            FILE *outf = fopen(outfile.c_str(), "wb");
+            if (outf) {
                 char line[1024];
                 int l = 1;
                 unsigned block = 0;
-                while (!inf.eof()) {
-                    inf.getline(line, sizeof(line));
-                    if (inf.fail()) {
-                        if (inf.eof())
-                            break; // no content on trailing line
-                        // Read through lines longer than sizeof(line)
-                        inf.clear();
-                        inf.ignore(std::numeric_limits<std::streamsize>::max(), '\n');
-                    }
+                int ret = 0;
+                while (ret != EOF && (ret = fscanf(inf, "%1023[^\n]", line)) != EOF) {
+                    // Skip n non-newline chars and a single trailing newline
+                    if ((ret = fscanf(inf, "%*[^\n]")) != EOF)
+                        ret = fscanf(inf, "%*1[\n]");
                     logdata_block *data = NULL;
                     if (block < values.size()) {
                         data = values[block];
@@ -166,32 +157,32 @@ static void write_log_data(logdata_t &logData, const char *extension) JL_NOTSAFE
                         l = 0;
                         block++;
                     }
-                    outf.width(9);
                     if (value == 0)
-                        outf << '-';
+                        fprintf(outf, "        -");
                     else
-                        outf << (value - 1);
-                    outf.width(0);
-                    outf << " " << line << '\n';
+                        fprintf(outf, "%9" PRIu64, value - 1);
+                    fprintf(outf, " %s\n", line);
+                    line[0] = 0;
                 }
-                outf.close();
+                fclose(outf);
             }
-            inf.close();
+            fclose(inf);
         }
     }
 }
 
 static void write_lcov_data(logdata_t &logData, const std::string &outfile) JL_NOTSAFEPOINT
 {
-    std::ofstream outf(outfile.c_str(), std::ofstream::ate | std::ofstream::out | std::ofstream::binary);
+    FILE *outf = fopen(outfile.c_str(), "ab");
+    if (!outf) return;
     //std::string base = std::string(jl_options.julia_bindir);
     //base = base + "/../share/julia/base/";
     logdata_t::iterator it = logData.begin();
     for (; it != logData.end(); it++) {
         StringRef filename = it->first();
         const SmallVector<logdata_block*, 0> &values = it->second;
         if (!values.empty()) {
-            outf << "SF:" << filename.str() << '\n';
+            fprintf(outf, "SF:%.*s\n", (int)filename.size(), filename.data());
             size_t n_covered = 0;
             size_t n_instrumented = 0;
             size_t lno = 0;
@@ -204,7 +195,7 @@ static void write_lcov_data(logdata_t &logData, const std::string &outfile) JL_N
                             n_instrumented++;
                             if (cov > 1)
                                 n_covered++;
-                            outf << "DA:" << lno << ',' << (cov - 1) << '\n';
+                            fprintf(outf, "DA:%zu,%" PRIu64 "\n", lno, cov - 1);
                         }
                         lno++;
                     }
@@ -213,12 +204,12 @@ static void write_lcov_data(logdata_t &logData, const std::string &outfile) JL_N
                     lno += logdata_blocksize;
                 }
             }
-            outf << "LH:" << n_covered << '\n';
-            outf << "LF:" << n_instrumented << '\n';
-            outf << "end_of_record\n";
+            fprintf(outf, "LH:%zu\n", n_covered);
+            fprintf(outf, "LF:%zu\n", n_instrumented);
+            fprintf(outf, "end_of_record\n");
         }
     }
-    outf.close();
+    fclose(outf);
 }
 
 extern "C" JL_DLLEXPORT void jl_write_coverage_data(const char *output) JL_NOTSAFEPOINT

src/gc-heap-snapshot.cpp

@@ -6,25 +6,19 @@
 #include "julia_internal.h"
 #include "julia_assert.h"
 
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
-#include "llvm/ADT/DenseMap.h"
-
-#include <vector>
-#include <string>
-#include <sstream>
-#include <iostream>
-#include <set>
+#include "llvm/Support/FormatVariadic.h"
 
-using std::string;
-using std::set;
-using std::ostringstream;
-using std::pair;
 using std::make_pair;
 using llvm::SmallVector;
 using llvm::StringMap;
 using llvm::DenseMap;
 using llvm::StringRef;
+using llvm::SmallString;
+using llvm::formatv;
 
 // https://stackoverflow.com/a/33799784/751061
 void print_str_escape_json(ios_t *stream, StringRef s)
@@ -422,18 +416,16 @@ static size_t record_pointer_to_gc_snapshot(void *a, size_t bytes, StringRef nam
     return val.first->second;
 }
 
-static string _fieldpath_for_slot(void *obj, void *slot) JL_NOTSAFEPOINT
+static SmallString<128> _fieldpath_for_slot(void *obj, void *slot) JL_NOTSAFEPOINT
 {
-    string res;
+    SmallString<128> res;
     jl_datatype_t *objtype = (jl_datatype_t*)jl_typeof(obj);
 
     while (1) {
         int i = gc_slot_to_fieldidx(obj, slot, objtype);
 
         if (jl_is_tuple_type(objtype) || jl_is_namedtuple_type(objtype)) {
-            ostringstream ss;
-            ss << "[" << i << "]";
-            res += ss.str();
+            res += formatv("[{0}]", i).sstr<8>();
         }
         else {
             jl_svec_t *field_names = jl_field_names(objtype);
@@ -472,9 +464,8 @@ void _gc_heap_snapshot_record_gc_roots(jl_value_t *root, char *name) JL_NOTSAFEP
 void _gc_heap_snapshot_record_finlist(jl_value_t *obj, size_t index) JL_NOTSAFEPOINT
 {
     auto to_node_idx = record_node_to_gc_snapshot(obj);
-    ostringstream ss;
-    ss << "finlist-" << index;
-    auto edge_label = g_snapshot->names.serialize_if_necessary(g_snapshot->strings, ss.str());
+    SmallString<16> ss = formatv("finlist-{0}", index);
+    auto edge_label = g_snapshot->names.serialize_if_necessary(g_snapshot->strings, ss);
     _record_gc_just_edge("internal", g_snapshot->_gc_finlist_root_idx, to_node_idx, edge_label);
 }
 
@@ -536,7 +527,7 @@ void _gc_heap_snapshot_record_array_edge(jl_value_t *from, jl_value_t *to, size_
 
 void _gc_heap_snapshot_record_object_edge(jl_value_t *from, jl_value_t *to, void *slot) JL_NOTSAFEPOINT
 {
-    string path = _fieldpath_for_slot(from, slot);
+    SmallString<128> path = _fieldpath_for_slot(from, slot);
     _record_gc_edge("property", from, to,
                     g_snapshot->names.serialize_if_necessary(g_snapshot->strings, path));
 }

src/processor.cpp

@@ -16,7 +16,6 @@
 #include "julia.h"
 #include "julia_internal.h"
 
-#include <map>
 #include <algorithm>
 
 #include "julia_assert.h"
@@ -25,8 +24,6 @@
 #include <dlfcn.h>
 #endif
 
-#include <iostream>
-
 // CPU target string is a list of strings separated by `;` each string starts with a CPU
 // or architecture name and followed by an optional list of features separated by `,`.
 // A "generic" or empty CPU name means the basic required feature set of the target ISA

src/runtime_ccall.cpp

@@ -1,7 +1,6 @@
 // This file is a part of Julia. License is MIT: https://julialang.org/license
 
 #include "llvm-version.h"
-#include <map>
 #include <string>
 #include <llvm/ADT/StringMap.h>
 #include <llvm/TargetParser/Host.h>
@@ -25,7 +24,7 @@ using namespace llvm;
 jl_value_t *jl_libdl_dlopen_func JL_GLOBALLY_ROOTED;
 
 // map from user-specified lib names to handles
-static std::map<std::string, void*> libMap;
+static StringMap<void*> libMap;
 static jl_mutex_t libmap_lock;
 extern "C"
 void *jl_get_library_(const char *f_lib, int throw_err)