Experimental support for bulding system libraries via ninja

The primary advantage of doing this is that it avoids having force
a completely rebuild of a system library when you want to rebuilt it.
Instead not have precise dependencies.

Author: sbc100

embuilder.py

@@ -237,7 +237,7 @@ def main():
       if do_clear:
         library.erase()
       if do_build:
-        library.get_path()
+        library.build()
     elif what == 'sysroot':
       if do_clear:
         shared.Cache.erase_file('sysroot_install.stamp')

tools/cache.py

@@ -108,8 +108,8 @@ def get_lib_dir(self, absolute, varies=True):
       path = Path(path, '-'.join(subdir))
     return path
 
-  def get_lib_name(self, name, varies=True):
-    return str(self.get_lib_dir(absolute=False, varies=varies).joinpath(name))
+  def get_lib_name(self, name, varies=True, absolute=False):
+    return str(self.get_lib_dir(absolute=absolute, varies=varies).joinpath(name))
 
   def erase_lib(self, name):
     self.erase_file(self.get_lib_name(name))
@@ -127,7 +127,7 @@ def get_lib(self, libname, *args, **kwargs):
 
   # Request a cached file. If it isn't in the cache, it will be created with
   # the given creator function
-  def get(self, shortname, creator, what=None, force=False):
+  def get(self, shortname, creator, what=None, force=False, quiet=False):
     cachename = Path(self.dirname, shortname)
     # Check for existence before taking the lock in case we can avoid the
     # lock completely.
@@ -147,11 +147,12 @@ def get(self, shortname, creator, what=None, force=False):
           what = 'system library'
         else:
           what = 'system asset'
-      message = f'generating {what}: {shortname}... (this will be cached in "{cachename}" for subsequent builds)'
+      message = f'generating {what}: "{cachename}"'
       logger.info(message)
       utils.safe_ensure_dirs(cachename.parent)
       creator(str(cachename))
       assert cachename.exists()
-      logger.info(' - ok')
+      if not quiet:
+        logger.info(' - ok')
 
     return str(cachename)

tools/gen_struct_info.py

@@ -246,7 +246,7 @@ def inspect_headers(headers, cflags):
   # TODO(sbc): If we can remove EM_EXCLUSIVE_CACHE_ACCESS then this would not longer be needed.
   shared.check_sanity()
 
-  compiler_rt = system_libs.Library.get_usable_variations()['libcompiler_rt'].get_path()
+  compiler_rt = system_libs.Library.get_usable_variations()['libcompiler_rt'].build()
 
   # Close all unneeded FDs.
   os.close(src_file[0])

tools/system_libs.py

@@ -28,6 +28,14 @@
                 'getsockopt.c', 'setsockopt.c', 'freeaddrinfo.c',
                 'in6addr_any.c', 'in6addr_loopback.c', 'accept4.c']
 
+# Experimental: Setting EMCC_USE_NINJA will cause system libraries to get built with ninja rather
+# than simple subprocesses.  The primary benefit here is that we get accurate dependency tracking.
+# This means we can avoid completely rebuilding a library and just rebiuld based on what changed.
+#
+# Setting EMCC_USE_NINJA=2 means that ninja will automatically be run for each library needed at
+# link time.
+USE_NINJA = int(os.environ.get('EMCC_USE_NINJA', '0'))
+
 
 def files_in_path(path, filenames):
   srcdir = utils.path_from_root(path)
@@ -239,23 +247,26 @@ def can_build(self):
     return True
 
   def erase(self):
-    shared.Cache.erase_file(shared.Cache.get_lib_name(self.get_filename()))
+    shared.Cache.erase_file(self.get_path())
+
+  def get_path(self, absolute=False):
+    return shared.Cache.get_lib_name(self.get_filename(), absolute=absolute)
 
-  def get_path(self):
+  def build(self):
     """
     Gets the cached path of this library.
 
     This will trigger a build if this library is not in the cache.
     """
-    return shared.Cache.get_lib(self.get_filename(), self.build)
+    return shared.Cache.get(self.get_path(), self.do_build, force=USE_NINJA == 2, quiet=USE_NINJA)
 
   def get_link_flag(self):
     """
     Gets the link flags needed to use the library.
 
     This will trigger a build if this library is not in the cache.
     """
-    fullpath = self.get_path()
+    fullpath = self.build()
     # For non-libaries (e.g. crt1.o) we pass the entire path to the linker
     if self.get_ext() != '.a':
       return fullpath
@@ -281,6 +292,100 @@ def get_files(self):
 
     raise NotImplementedError()
 
+  def write_ninja_file(self, filename, libname):
+    cflags = self.get_cflags()
+    asflags = get_base_cflags()
+    # TODO(sbc) There is an llvm bug that causes a crash when `-g` is used with
+    # assembly files that define wasm globals.
+    asflags = [arg for arg in asflags if arg != '-g']
+    cflags_asm = [arg for arg in cflags if arg != '-g']
+
+    def join(flags):
+      return ' '.join(flags)
+
+    out = f'''\
+# Automatically generated by tools/system_libs.py.  DO NOT EDIT
+
+ninja_required_version = 1.5
+
+ASFLAGS = {join(asflags)}
+CFLAGS = {join(cflags)}
+CFLAGS_ASM = {join(cflags_asm)}
+EMCC = {shared.EMCC}
+EMXX = {shared.EMXX}
+EMAR = {shared.EMAR}
+
+rule cc
+  depfile = $out.d
+  command = $EMCC -MD -MF $out.d $CFLAGS -c $in -o $out
+  description = CC $in
+
+rule cxx
+  depfile = $out.d
+  command = $EMXX -MD -MF $out.d $CFLAGS -c $in -o $out
+  description = CXX $out
+
+rule asm
+  command = $EMCC $ASFLAGS -c $in -o $out
+  description = ASM $out
+
+rule asm_cpp
+  depfile = $out.d
+  command = $EMCC -MD -MF $out.d $CFLAGS_ASM -c $in -o $out
+  description = ASM $out
+
+rule direct_cc
+  depfile = $with_depfile
+  command = $EMCC -MD -MF $with_depfile $CFLAGS -c $in -o $out
+  description = CC $out
+
+rule archive
+  command = $EMAR cr $out $in
+  description = AR $out
+
+'''
+    suffix = shared.suffix(libname)
+    input_files = self.get_files()
+
+    if suffix == '.o':
+      assert len(input_files) == 1
+      depfile = shared.unsuffixed_basename(input_files[0]) + '.d'
+      out += f'build {libname}: direct_cc {input_files[0]}\n'
+      out += f'  with_depfile = {depfile}\n'
+    else:
+      objects = []
+      for src in input_files:
+        o = shared.unsuffixed_basename(src) + '.o'
+        object_uuid = 0
+        # Find a unique basename
+        while o in objects:
+          object_uuid += 1
+          o = f'{object_basename}__{object_uuid}.o'
+        objects.append(o)
+        ext = shared.suffix(src)
+        if ext == '.s':
+          out += f'build {o}: asm {src}\n'
+          flags = asflags
+        elif ext == '.S':
+          out += f'build {o}: asm_cpp {src}\n'
+          flags = cflags_asm
+        elif ext == '.c':
+          out += f'build {o}: cc {src}\n'
+          flags = cflags
+        else:
+          out += f'build {o}: cxx {src}\n'
+          flags = cflags
+        custom_flags = self.customize_build_flags(src, flags)
+        if custom_flags != flags:
+          out += f'  CFLAGS = {join(custom_flags)}'
+        out += '\n'
+
+      objects = ' '.join(objects)
+      out += f'build {libname}: archive {objects}\n'
+
+    if not os.path.exists(filename) or utils.read_file(filename) != out:
+      utils.write_file(filename, out)
+
   def build_objects(self, build_dir):
     """
     Returns a list of compiled object files for this library.
@@ -307,40 +412,52 @@ def build_objects(self, build_dir):
         o = os.path.join(build_dir, f'{object_basename}__{object_uuid}.o')
       ext = shared.suffix(src)
       if ext in ('.s', '.S', '.c'):
-        cmd = [shared.EMCC]
+        compiler = shared.EMCC
       else:
-        cmd = [shared.EMXX]
+        compiler = shared.EMXX
 
       if ext == '.s':
         # .s files are processed directly by the assembler.  In this case we can't pass
         # pre-processor flags such as `-I` and `-D` but we still want core flags such as
         # `-sMEMORY64`.
-        cmd += get_base_cflags()
+        flags = get_base_cflags()
       else:
-        cmd += cflags
+        flags = cflags
       if ext in ('.s', '.S'):
         # TODO(sbc) There is an llvm bug that causes a crash when `-g` is used with
         # assembly files that define wasm globals.
-        cmd = [arg for arg in cmd if arg != '-g']
-      cmd = self.customize_build_cmd(cmd, src)
-      commands.append(cmd + ['-c', src, '-o', o])
+        flags = [arg for arg in flags if arg != '-g']
+      flags = self.customize_build_flags(src, flags)
+      commands.append([compiler] + flags + ['-c', src, '-o', o])
       objects.append(o)
     run_build_commands(commands)
+    logger.info('compiled %d inputs' % len(objects))
     return objects
 
-  def customize_build_cmd(self, cmd, filename):  # noqa
+  def customize_build_flags(self, filename, flags):  # noqa
     """Allows libraries to customize the build command used on per-file basis.
 
     For example, libc uses this to replace -Oz with -O2 for some subset of files."""
-    return cmd
+    return flags
 
-  def build(self, out_filename):
+  def do_build(self, out_filename):
     """Builds the library and returns the path to the file."""
-    build_dir = shared.Cache.get_path(os.path.join('build', self.get_base_name()))
-    utils.safe_ensure_dirs(build_dir)
-    create_lib(out_filename, self.build_objects(build_dir))
-    if not shared.DEBUG:
-      utils.delete_dir(build_dir)
+    assert out_filename == self.get_path(absolute=True)
+    build_dir = os.path.join(shared.Cache.get_path('build'), self.get_base_name())
+    if USE_NINJA:
+      ensure_sysroot()
+      utils.safe_ensure_dirs(build_dir)
+      ninja_file = os.path.join(build_dir, 'build.ninja')
+      self.write_ninja_file(ninja_file, out_filename)
+      shared.check_call(['ninja', '-C', build_dir], env=clean_env())
+    else:
+      # Use a seperate build directory to the ninja flavor so that building without
+      # EMCC_USE_NINJA doesn't clobber the ninja build tree
+      build_dir += '-tmp'
+      utils.safe_ensure_dirs(build_dir)
+      create_lib(out_filename, self.build_objects(build_dir))
+      if not shared.DEBUG:
+        utils.delete_dir(build_dir)
 
   @classmethod
   def _inherit_list(cls, attr):
@@ -1002,18 +1119,18 @@ def get_files(self):
 
     return libc_files
 
-  def customize_build_cmd(self, cmd, filename):
-    if filename in self.non_lto_files:
+  def customize_build_flags(self, filename, flags):
+    if filename not in self.non_lto_files:
       # These files act more like the part of compiler-rt in that
       # references to them can be generated at compile time.
       # Treat them like compiler-rt in as much as never compile
       # them as LTO and build them with -O2 rather then -Os (which
       # use used for the rest of libc) because this set of files
       # also contains performance sensitive math functions.
-      cmd = [a for a in cmd if not a.startswith('-flto')]
-      cmd = [a for a in cmd if not a.startswith('-O')]
-      cmd += ['-O2']
-    return cmd
+      flags = [a for a in flags if not a.startswith('-flto')]
+      flags = [a for a in flags if not a.startswith('-O')]
+      flags += ['-O2']
+    return flags
 
 
 # Contains the files from libc that are optimized differently in -Oz mode, where
@@ -1029,7 +1146,7 @@ def __init__(self, **kwargs):
     self.non_lto_files = self.get_libcall_files()
 
   def get_libcall_files(self):
-    # see comments in libc.customize_build_cmd
+    # see comments in libc.customize_build_flags
 
     # some files also appear in libc, and a #define affects them
     mem_files = files_in_path(
@@ -1054,13 +1171,13 @@ def get_files(self):
 
     return libcall_files + mem_files
 
-  def customize_build_cmd(self, cmd, filename):
+  def customize_build_flags(self, filename, flags):
     if filename in self.non_lto_files:
-      # see comments in libc.customize_build_cmd
-      cmd = [a for a in cmd if not a.startswith('-flto')]
-      cmd = [a for a in cmd if not a.startswith('-O')]
-      cmd += ['-O2']
-    return cmd
+      # see comments in libc.customize_build_flags
+      flags = [f for f in flags if not f.startswith('-flto')]
+      flags = [f for f in flags if not f.startswith('-O')]
+      flags += ['-O2']
+    return flags
 
   def can_use(self):
     # Because libc_optz overrides parts of libc, it is not compatible with