Build a complete sysroot in the cache directory

ChangeLog.md

@@ -20,6 +20,9 @@ See docs/process.md for more on how version tagging works.
 
 Current Trunk
 -------------
+- Emscripten now builds a complete sysroot inside the EM_CACHE directory.
+  This includes the system headers which get copied into place there rather
+  than adding a sequence of extra include directories.
 
 2.0.12: 01/09/2021
 ------------------

embuilder.py

@@ -171,6 +171,10 @@ def main():
       if force:
         library.erase()
       library.get_path()
+    elif what == 'sysroot':
+      if force:
+        shared.Cache.erase_file('sysroot_install.stamp')
+      system_libs.ensure_sysroot()
     elif what == 'struct_info':
       if force:
         shared.Cache.erase_file('generated_struct_info.json')

emcc.py

@@ -1331,12 +1331,6 @@ def filter_out_duplicate_dynamic_libs(inputs):
                                              '_emscripten_stack_get_end',
                                              '_emscripten_stack_set_limits']
 
-    if not compile_only and not options.post_link:
-      ldflags = shared.emsdk_ldflags(newargs)
-      for f in ldflags:
-        newargs.append(f)
-        add_link_flag(len(newargs), f)
-
     # SSEx is implemented on top of SIMD128 instruction set, but do not pass SSE flags to LLVM
     # so it won't think about generating native x86 SSE code.
     newargs = [x for x in newargs if x not in shared.SIMD_INTEL_FEATURE_TOWER and x not in shared.SIMD_NEON_FLAGS]
@@ -1950,6 +1944,7 @@ def is_link_flag(flag):
 
     compile_args = [a for a in newargs if a and not is_link_flag(a)]
     cflags = calc_cflags(options)
+    system_libs.ensure_sysroot()
     system_libs.add_ports_cflags(cflags, shared.Settings)
 
     def use_cxx(src):
@@ -1976,9 +1971,8 @@ def get_compiler(cxx):
       return CC
 
     def get_clang_command(src_file):
-      cxx = use_cxx(src_file)
-      per_file_cflags = shared.get_cflags(args, cxx)
-      return get_compiler(cxx) + cflags + per_file_cflags + compile_args + [src_file]
+      per_file_cflags = shared.get_cflags(args)
+      return get_compiler(use_cxx(src_file)) + cflags + per_file_cflags + compile_args + [src_file]
 
     def get_clang_command_asm(src_file):
       asflags = shared.get_clang_flags()
@@ -2070,6 +2064,10 @@ def compile_source_file(i, input_file):
   if specified_target and specified_target.startswith('-'):
     exit_with_error('invalid output filename: `%s`' % specified_target)
 
+  ldflags = shared.emsdk_ldflags(newargs)
+  for f in ldflags:
+    add_link_flag(sys.maxsize, f)
+
   using_lld = not (link_to_object and shared.Settings.LTO)
   link_flags = filter_link_flags(link_flags, using_lld)
 

src/struct_info.json

@@ -1454,7 +1454,7 @@
             ]
     },
     {
-        "file": "../lib/libc/musl/src/internal/pthread_impl.h",
+        "file": "system/lib/libc/musl/src/internal/pthread_impl.h",
         "structs": {
             "pthread": [
               "threadStatus",
@@ -1477,7 +1477,7 @@
         "defines": ["__ATTRP_C11_THREAD"]
     },
     {
-        "file": "../lib/libc/musl/src/internal/libc.h",
+        "file": "system/lib/libc/musl/src/internal/libc.h",
         "structs": {
             "libc": [
               "global_locale"

system/lib/fetch/asmfs.cpp

@@ -15,7 +15,7 @@
 #include <emscripten/emscripten.h>
 #include <emscripten/fetch.h>
 #include <emscripten/threading.h>
-#include <libc/fcntl.h>
+#include <fcntl.h>
 #include <math.h>
 #include <string.h>
 #include <sys/stat.h>

tests/test_other.py

@@ -480,10 +480,10 @@ def test_emcc_asm_v_wasm(self):
   def test_emcc_cflags(self):
     output = self.run_process([EMCC, '--cflags'], stdout=PIPE)
     flags = output.stdout.strip()
-    self.assertContained(shared.shlex_join(shared.emsdk_cflags([], False)), flags)
+    self.assertContained(shared.shlex_join(shared.emsdk_cflags([])), flags)
     output = self.run_process([EMXX, '--cflags'], stdout=PIPE)
     flags = output.stdout.strip()
-    self.assertContained(shared.shlex_join(shared.emsdk_cflags([], True)), flags)
+    self.assertContained(shared.shlex_join(shared.emsdk_cflags([])), flags)
     # check they work
     cmd = [CLANG_CXX, path_from_root('tests', 'hello_world.cpp')] + shlex.split(flags.replace('\\', '\\\\')) + ['-c', '-o', 'out.o']
     self.run_process(cmd)

tests/test_sanity.py

@@ -691,10 +691,10 @@ def test_embuilder_wasm_backend(self):
     # the --lto flag makes us build wasm-bc
     self.do([EMCC, '--clear-cache'])
     self.run_process([EMBUILDER, 'build', 'libemmalloc'])
-    self.assertExists(os.path.join(config.CACHE, 'wasm'))
+    self.assertExists(os.path.join(config.CACHE, 'sysroot', 'lib', 'wasm32-emscripten'))
     self.do([EMCC, '--clear-cache'])
     self.run_process([EMBUILDER, 'build', 'libemmalloc', '--lto'])
-    self.assertExists(os.path.join(config.CACHE, 'wasm-lto'))
+    self.assertExists(os.path.join(config.CACHE, 'sysroot', 'lib', 'wasm32-emscripten', 'lto'))
 
   def test_binaryen_version(self):
     restore_and_set_up()

tools/cache.py

@@ -84,21 +84,32 @@ def erase(self):
   def get_path(self, name):
     return os.path.join(self.dirname, name)
 
+  def get_sysroot_dir(self, absolute):
+    if absolute:
+      return os.path.join(self.dirname, 'sysroot')
+    return 'sysroot'
+
   def get_include_dir(self):
-    return os.path.join(self.dirname, 'include')
+    return os.path.join(self.get_sysroot_dir(absolute=True), 'include')
 
-  def get_lib_dir(self):
-    subdir = 'wasm'
+  def get_lib_dir(self, absolute):
+    path = os.path.join(self.get_sysroot_dir(absolute=absolute), 'lib')
+    if shared.Settings.MEMORY64:
+      path = os.path.join(path, 'wasm64-emscripten')
+    else:
+      path = os.path.join(path, 'wasm32-emscripten')
+    # if relevant, use a subdir of the cache
+    subdir = []
     if shared.Settings.LTO:
-      subdir += '-lto'
+      subdir.append('lto')
     if shared.Settings.RELOCATABLE:
-      subdir += '-pic'
-    if shared.Settings.MEMORY64:
-      subdir += '-memory64'
-    return subdir
+      subdir.append('pic')
+    if subdir:
+      path = os.path.join(path, '-'.join(subdir))
+    return path
 
   def get_lib_name(self, name):
-    return os.path.join(self.get_lib_dir(), name)
+    return os.path.join(self.get_lib_dir(absolute=False), name)
 
   def erase_lib(self, name):
     self.erase_file(self.get_lib_name(name))

tools/gen_struct_info.py

@@ -246,6 +246,7 @@ def inspect_code(headers, cpp_opts, structs, defines):
                                     '-O0',
                                     '-Werror',
                                     '-Wno-format',
+                                    '-I', shared.path_from_root(),
                                     '-s', 'BOOTSTRAPPING_STRUCT_INFO=1',
                                     '-s', 'WARN_ON_UNDEFINED_SYMBOLS=0',
                                     '-s', 'STRICT=1',

tools/ports/regal.py

@@ -46,7 +46,7 @@ def create():
 
     # includes
     source_path_include = os.path.join(ports.get_dir(), 'regal', 'regal-' + TAG, 'include', 'GL')
-    ports.install_header_dir(source_path_include)
+    ports.install_headers(source_path_include, target='GL')
 
     # build
     srcs_regal = ['regal/RegalShaderInstance.cpp',

tools/shared.py

@@ -420,9 +420,7 @@ def emsdk_ldflags(user_args):
     return []
 
   library_paths = [
-      path_from_root('system', 'local', 'lib'),
-      path_from_root('system', 'lib'),
-      Cache.get_path(Cache.get_lib_dir())
+      Cache.get_lib_dir(absolute=True)
   ]
   ldflags = ['-L' + l for l in library_paths]
 
@@ -437,12 +435,12 @@ def emsdk_ldflags(user_args):
   return ldflags
 
 
-def emsdk_cflags(user_args, cxx):
+def emsdk_cflags(user_args):
   # Disable system C and C++ include directories, and add our own (using
   # -isystem so they are last, like system dirs, which allows projects to
   # override them)
 
-  c_opts = ['--sysroot=' + path_from_root('system')]
+  c_opts = ['--sysroot=' + Cache.get_sysroot_dir(absolute=True)]
 
   def array_contains_any_of(hay, needles):
     for n in needles:
@@ -477,24 +475,13 @@ def array_contains_any_of(hay, needles):
 
   sysroot_include_paths = []
 
-  if cxx:
-    sysroot_include_paths += [
-      os.path.join('/include', 'libcxx'),
-      os.path.join('/lib', 'libcxxabi', 'include'),
-    ]
-
-  # TODO: Merge the cache into the sysroot.
-  c_opts += ['-Xclang', '-isystem' + Cache.get_path('include')]
-
   sysroot_include_paths += [
     os.path.join('/include', 'compat'),
-    os.path.join('/include', 'libc'),
-    os.path.join('/lib', 'libc', 'musl', 'arch', 'emscripten'),
-    os.path.join('/local', 'include'),
+    # TODO(sbc): Ideally we wouldn't need these, we could just copy them into the sysroot/include
+    # However, clang puts its internal header directory first so it finds its internal versions
+    # first.
     os.path.join('/include', 'SSE'),
     os.path.join('/include', 'neon'),
-    os.path.join('/lib', 'compiler-rt', 'include'),
-    os.path.join('/lib', 'libunwind', 'include'),
   ]
 
   def include_directive(paths):
@@ -511,7 +498,7 @@ def get_clang_flags():
   return ['-target', get_llvm_target()]
 
 
-def get_cflags(user_args, cxx):
+def get_cflags(user_args):
   c_opts = get_clang_flags()
 
   # Set the LIBCPP ABI version to at least 2 so that we get nicely aligned string
@@ -539,7 +526,7 @@ def get_cflags(user_args, cxx):
   if os.environ.get('EMMAKEN_NO_SDK') or '-nostdinc' in user_args:
     return c_opts
 
-  return c_opts + emsdk_cflags(user_args, cxx)
+  return c_opts + emsdk_cflags(user_args)
 
 
 # Settings. A global singleton. Not pretty, but nicer than passing |, settings| everywhere

tools/system_libs.py

@@ -83,6 +83,10 @@ def run_one_command(cmd):
 
 
 def run_build_commands(commands):
+  # Before running a set of build commands make sure the common sysroot
+  # headers are installed.  This prevents each sub-process from attempting
+  # to setup the sysroot itself.
+  ensure_sysroot()
   cores = min(len(commands), building.get_num_cores())
   if cores <= 1:
     for command in commands:
@@ -952,6 +956,9 @@ class libunwind(NoExceptLibrary, MTLibrary):
   name = 'libunwind'
   cflags = ['-Oz', '-D_LIBUNWIND_DISABLE_VISIBILITY_ANNOTATIONS']
   src_dir = ['system', 'lib', 'libunwind', 'src']
+  # Without this we can't build libunwind since it will pickup the unwind.h
+  # that is part of llvm (which is not compatible for some reason).
+  includes = [['system', 'lib', 'libunwind', 'include']]
   src_files = ['Unwind-wasm.cpp']
 
   def __init__(self, **kwargs):
@@ -1883,7 +1890,7 @@ def build_port(port_name, settings):
 def add_ports_cflags(args, settings):
   # Legacy SDL1 port is not actually a port at all but builtin
   if settings.USE_SDL == 1:
-    args += ['-Xclang', '-isystem' + shared.path_from_root('system', 'include', 'SDL')]
+    args += ['-Xclang', '-iwithsysroot/include/SDL']
 
   needed = get_needed_ports(settings)
 
@@ -1900,3 +1907,51 @@ def show_ports():
   print('Available ports:')
   for port in ports.ports:
     print('   ', port.show())
+
+
+# Once we require python 3.8 we can use shutil.copytree with
+# dirs_exist_ok=True and remove this function.
+def copytree_exist_ok(src, dest):
+  with utils.chdir(src):
+    for dirname, dirs, files in os.walk('.'):
+      destdir = os.path.join(dest, dirname)
+      utils.safe_ensure_dirs(destdir)
+      for f in files:
+        shared.safe_copy(os.path.join(src, dirname, f), os.path.join(destdir, f))
+
+
+def install_system_headers():
+  install_dirs = {
+    ('include',): '',
+    ('lib', 'compiler-rt', 'include'): '',
+    ('include', 'compat'): 'compat',
+    ('lib', 'libunwind', 'include'): '',
+    ('lib', 'libc', 'musl', 'arch', 'emscripten'): '',
+    ('lib', 'libcxx'): '',
+    ('include', 'libc'): '',
+    ('include', 'libcxx'): os.path.join('c++', 'v1'),
+    ('lib', 'libcxxabi', 'include'): os.path.join('c++', 'v1'),
+  }
+
+  target_include_dir = shared.Cache.get_include_dir()
+  for src, dest in install_dirs.items():
+    src = shared.path_from_root('system', *src)
+    dest = os.path.join(target_include_dir, dest)
+    copytree_exist_ok(src, dest)
+
+  # TODO(sbc): Move these headers back into thier respecive source trees
+  for dirname in ['libc', 'libcxx']:
+    shutil.rmtree(os.path.join(target_include_dir, dirname))
+
+  # Create a stamp file that signal the the header have been installed
+  # Removing this file, or running `emcc --clear-cache` or running
+  # `./embuilder build sysroot --force` will cause the re-installation of
+  # the system headers.
+  stamp = shared.Cache.get_path('sysroot_install.stamp')
+  with open(stamp, 'w') as f:
+    f.write('x')
+  return stamp
+
+
+def ensure_sysroot():
+  shared.Cache.get('sysroot_install.stamp', install_system_headers, what='system headers')