Experimental support for bulding system libraries via ninja

The primary advantage of doing this is that it avoids having force
a completely rebuild of a system library when you want to rebuilt it.
Instead not have precise dependencies.

Author: sbc100

embuilder.py

@@ -237,7 +237,7 @@ def main():
       if do_clear:
         library.erase()
       if do_build:
-        library.get_path()
+        library.build()
     elif what == 'sysroot':
       if do_clear:
         shared.Cache.erase_file('sysroot_install.stamp')

tools/cache.py

@@ -108,8 +108,8 @@ def get_lib_dir(self, absolute, varies=True):
       path = Path(path, '-'.join(subdir))
     return path
 
-  def get_lib_name(self, name, varies=True):
-    return str(self.get_lib_dir(absolute=False, varies=varies).joinpath(name))
+  def get_lib_name(self, name, varies=True, absolute=False):
+    return str(self.get_lib_dir(absolute=absolute, varies=varies).joinpath(name))
 
   def erase_lib(self, name):
     self.erase_file(self.get_lib_name(name))
@@ -127,7 +127,7 @@ def get_lib(self, libname, *args, **kwargs):
 
   # Request a cached file. If it isn't in the cache, it will be created with
   # the given creator function
-  def get(self, shortname, creator, what=None, force=False):
+  def get(self, shortname, creator, what=None, force=False, quiet=False):
     cachename = Path(self.dirname, shortname)
     # Check for existence before taking the lock in case we can avoid the
     # lock completely.
@@ -147,11 +147,12 @@ def get(self, shortname, creator, what=None, force=False):
           what = 'system library'
         else:
           what = 'system asset'
-      message = f'generating {what}: {shortname}... (this will be cached in "{cachename}" for subsequent builds)'
+      message = f'generating {what}: "{cachename}"'
       logger.info(message)
       utils.safe_ensure_dirs(cachename.parent)
       creator(str(cachename))
       assert cachename.exists()
-      logger.info(' - ok')
+      if not quiet:
+        logger.info(' - ok')
 
     return str(cachename)

tools/gen_struct_info.py

@@ -246,7 +246,7 @@ def inspect_headers(headers, cflags):
   # TODO(sbc): If we can remove EM_EXCLUSIVE_CACHE_ACCESS then this would not longer be needed.
   shared.check_sanity()
 
-  compiler_rt = system_libs.Library.get_usable_variations()['libcompiler_rt'].get_path()
+  compiler_rt = system_libs.Library.get_usable_variations()['libcompiler_rt'].build()
 
   # Close all unneeded FDs.
   os.close(src_file[0])

tools/ports/__init__.py

@@ -129,16 +129,27 @@ def build_port(src_dir, output_path, build_dir, includes=[], flags=[], exclude_f
       if not os.path.exists(build_dir):
         os.makedirs(build_dir)
       build_dir = src_dir
-    commands = []
-    objects = []
-    for src in srcs:
-      relpath = os.path.relpath(src, src_dir)
-      obj = os.path.join(build_dir, relpath) + '.o'
-      commands.append([shared.EMCC, '-c', src, '-o', obj] + cflags)
-      objects.append(obj)
-
-    system_libs.run_build_commands(commands)
-    system_libs.create_lib(output_path, objects)
+
+    if system_libs.USE_NINJA:
+      ninja_file = os.path.join(build_dir, 'build.ninja')
+      system_libs.ensure_sysroot()
+      system_libs.create_ninja_file(srcs, ninja_file, output_path, cflags=cflags)
+      cmd = ['ninja', '-C', build_dir]
+      if shared.PRINT_STAGES:
+        cmd.append('-v')
+      shared.check_call(cmd, env=system_libs.clean_env())
+    else:
+      commands = []
+      objects = []
+      for src in srcs:
+        relpath = os.path.relpath(src, src_dir)
+        obj = os.path.join(build_dir, relpath) + '.o'
+        commands.append([shared.EMCC, '-c', src, '-o', obj] + cflags)
+        objects.append(obj)
+
+      system_libs.run_build_commands(commands)
+      system_libs.create_lib(output_path, objects)
+
     return output_path
 
   @staticmethod
@@ -271,9 +282,16 @@ def clear_project_build(name):
     port = ports_by_name[name]
     port.clear(Ports, settings, shared)
     build_dir = os.path.join(Ports.get_build_dir(), name)
-    utils.delete_dir(build_dir)
+    if not system_libs.USE_NINJA:
+      utils.delete_dir(build_dir)
     return build_dir
 
+  @staticmethod
+  def write_file(filename, contents):
+    if os.path.exists(filename) and utils.read_file(filename) == contents:
+      return
+    utils.write_file(filename, contents)
+
 
 def dependency_order(port_list):
   # Perform topological sort of ports according to the dependency DAG

tools/ports/boost_headers.py

@@ -5,7 +5,6 @@
 
 import logging
 import os
-from pathlib import Path
 
 TAG = '1.75.0'
 HASH = '8c38be1ebef1b8ada358ad6b7c9ec17f5e0a300e8085db3473a13e19712c95eeb3c3defacd3c53482eb96368987c4b022efa8da2aac2431a154e40153d3c3dcd'
@@ -31,7 +30,7 @@ def create(final):
     build_dir = ports.clear_project_build('boost_headers')
     dummy_file = os.path.join(build_dir, 'dummy.cpp')
     shared.safe_ensure_dirs(os.path.dirname(dummy_file))
-    Path(dummy_file).write_text('static void dummy() {}')
+    ports.write_file(dummy_file, 'static void dummy() {}')
 
     ports.build_port(build_dir, final, build_dir)
 

tools/ports/freetype.py

@@ -19,7 +19,7 @@ def get(ports, settings, shared):
 
   def create(final):
     source_path = os.path.join(ports.get_dir(), 'freetype', 'FreeType-' + TAG)
-    Path(source_path, 'include/ftconfig.h').write_text(ftconf_h)
+    ports.write_file(Path(source_path, 'include/ftconfig.h'), ftconf_h)
     ports.install_header_dir(os.path.join(source_path, 'include'),
                              target=os.path.join('freetype2'))
 

tools/ports/libjpeg.py

@@ -25,7 +25,7 @@ def create(final):
     logging.info('building port: libjpeg')
     source_path = os.path.join(ports.get_dir(), 'libjpeg', 'jpeg-9c')
     dest_path = ports.clear_project_build('libjpeg')
-    Path(source_path, 'jconfig.h').write_text(jconfig_h)
+    ports.write_file(Path(source_path, 'jconfig.h'), jconfig_h)
     ports.install_headers(source_path)
     excludes = [
       'ansi2knr.c', 'cjpeg.c', 'ckconfig.c', 'djpeg.c', 'example.c',

tools/ports/libmodplug.py

@@ -25,7 +25,7 @@ def create(final):
     src_dir = os.path.join(source_path, 'src')
     libmodplug_path = os.path.join(src_dir, 'libmodplug')
 
-    Path(source_path, 'config.h').write_text(config_h)
+    ports.write_file(Path(source_path, 'config.h'), config_h)
 
     flags = [
       '-Wno-deprecated-register',

tools/ports/libpng.py

@@ -30,7 +30,7 @@ def create(final):
     logging.info('building port: libpng')
 
     source_path = os.path.join(ports.get_dir(), 'libpng', 'libpng-' + TAG)
-    Path(source_path, 'pnglibconf.h').write_text(pnglibconf_h)
+    ports.write_file(Path(source_path, 'pnglibconf.h'), pnglibconf_h)
     ports.install_headers(source_path)
 
     flags = ['-sUSE_ZLIB=1']

tools/ports/mpg123.py

@@ -27,8 +27,8 @@ def create(final):
     libmpg123_path = os.path.join(src_path, 'libmpg123')
     compat_path = os.path.join(src_path, 'compat')
 
-    Path(src_path, 'config.h').write_text(config_h)
-    Path(libmpg123_path, 'mpg123.h').write_text(mpg123_h)
+    ports.write_file(Path(src_path, 'config.h'), config_h)
+    ports.write_file(Path(libmpg123_path, 'mpg123.h'), mpg123_h)
 
     # copy header to a location so it can be used as 'MPG123/'
     ports.install_headers(libmpg123_path, pattern="*123.h", target='')

tools/ports/ogg.py

@@ -22,7 +22,7 @@ def create(final):
     logging.info('building port: ogg')
 
     source_path = os.path.join(ports.get_dir(), 'ogg', 'Ogg-' + TAG)
-    Path(source_path, 'include', 'ogg', 'config_types.h').write_text(config_types_h)
+    ports.write_file(Path(source_path, 'include', 'ogg', 'config_types.h'), config_types_h)
     ports.install_header_dir(os.path.join(source_path, 'include', 'ogg'), 'ogg')
     dest_path = ports.clear_project_build('ogg')
     ports.build_port(os.path.join(source_path, 'src'), final, dest_path)

tools/ports/zlib.py

@@ -4,7 +4,6 @@
 # found in the LICENSE file.
 
 import os
-from pathlib import Path
 
 VERSION = '1.2.12'
 HASH = 'cc2366fa45d5dfee1f983c8c51515e0cff959b61471e2e8d24350dea22d3f6fcc50723615a911b046ffc95f51ba337d39ae402131a55e6d1541d3b095d6c0a14'
@@ -19,7 +18,7 @@ def get(ports, settings, shared):
 
   def create(final):
     source_path = os.path.join(ports.get_dir(), 'zlib', 'zlib-' + VERSION)
-    Path(source_path, 'zconf.h').write_text(zconf_h)
+    ports.write_file(os.path.join(source_path, 'zconf.h'), zconf_h)
     ports.install_headers(source_path)
 
     # build

tools/system_libs.py

@@ -28,6 +28,14 @@
                 'getsockopt.c', 'setsockopt.c', 'freeaddrinfo.c',
                 'in6addr_any.c', 'in6addr_loopback.c', 'accept4.c']
 
+# Experimental: Setting EMCC_USE_NINJA will cause system libraries to get built with ninja rather
+# than simple subprocesses.  The primary benefit here is that we get accurate dependency tracking.
+# This means we can avoid completely rebuilding a library and just rebuild based on what changed.
+#
+# Setting EMCC_USE_NINJA=2 means that ninja will automatically be run for each library needed at
+# link time.
+USE_NINJA = int(os.environ.get('EMCC_USE_NINJA', '0'))
+
 
 def files_in_path(path, filenames):
   srcdir = utils.path_from_root(path)
@@ -93,6 +101,107 @@ def create_lib(libname, inputs):
     building.emar('cr', libname, inputs)
 
 
+def create_ninja_file(input_files, filename, libname, cflags, asflags=None, customize_build_flags=None):
+  if asflags is None:
+    asflags = []
+  # TODO(sbc) There is an llvm bug that causes a crash when `-g` is used with
+  # assembly files that define wasm globals.
+  asflags = [arg for arg in asflags if arg != '-g']
+  cflags_asm = [arg for arg in cflags if arg != '-g']
+
+  def join(flags):
+    return ' '.join(flags)
+
+  out = f'''\
+# Automatically generated by tools/system_libs.py.  DO NOT EDIT
+
+ninja_required_version = 1.5
+
+ASFLAGS = {join(asflags)}
+CFLAGS = {join(cflags)}
+CFLAGS_ASM = {join(cflags_asm)}
+EMCC = {shared.EMCC}
+EMXX = {shared.EMXX}
+EMAR = {shared.EMAR}
+
+rule cc
+  depfile = $out.d
+  command = $EMCC -MD -MF $out.d $CFLAGS -c $in -o $out
+  description = CC $out
+
+rule cxx
+  depfile = $out.d
+  command = $EMXX -MD -MF $out.d $CFLAGS -c $in -o $out
+  description = CXX $out
+
+rule asm
+  command = $EMCC $ASFLAGS -c $in -o $out
+  description = ASM $out
+
+rule asm_cpp
+  depfile = $out.d
+  command = $EMCC -MD -MF $out.d $CFLAGS_ASM -c $in -o $out
+  description = ASM $out
+
+rule direct_cc
+  depfile = $with_depfile
+  command = $EMCC -MD -MF $with_depfile $CFLAGS -c $in -o $out
+  description = CC $out
+
+rule archive
+  command = $EMAR cr $out $in
+  description = AR $out
+
+'''
+  suffix = shared.suffix(libname)
+
+  case_insensitive = is_case_insensitive(os.path.dirname(filename))
+  if suffix == '.o':
+    assert len(input_files) == 1
+    depfile = shared.unsuffixed_basename(input_files[0]) + '.d'
+    out += f'build {libname}: direct_cc {input_files[0]}\n'
+    out += f'  with_depfile = {depfile}\n'
+  else:
+    objects = []
+    for src in input_files:
+      # Resolve duplicates by appending unique.
+      # This is needed on case insensitve filesystem to handle,
+      # for example, _exit.o and _Exit.o.
+      o = shared.unsuffixed_basename(src) + '.o'
+      object_uuid = 0
+      if case_insensitive:
+        o = o.lower()
+      # Find a unique basename
+      while o in objects:
+        object_uuid += 1
+        o = f'{o}__{object_uuid}.o'
+      objects.append(o)
+      ext = shared.suffix(src)
+      if ext == '.s':
+        out += f'build {o}: asm {src}\n'
+        flags = asflags
+      elif ext == '.S':
+        out += f'build {o}: asm_cpp {src}\n'
+        flags = cflags_asm
+      elif ext == '.c':
+        out += f'build {o}: cc {src}\n'
+        flags = cflags
+      else:
+        out += f'build {o}: cxx {src}\n'
+        flags = cflags
+      if customize_build_flags:
+        custom_flags = customize_build_flags(flags, src)
+        if custom_flags != flags:
+          out += f'  CFLAGS = {join(custom_flags)}'
+      out += '\n'
+
+    objects = sorted(objects, key=lambda x: os.path.basename(x))
+    objects = ' '.join(objects)
+    out += f'build {libname}: archive {objects}\n'
+
+  utils.write_file(filename, out)
+
+
 def is_case_insensitive(path):
   """Returns True if the filesystem at `path` is case insensitive."""
   utils.write_file(os.path.join(path, 'test_file'), '')
@@ -240,23 +349,26 @@ def can_build(self):
     return True
 
   def erase(self):
-    shared.Cache.erase_file(shared.Cache.get_lib_name(self.get_filename()))
+    shared.Cache.erase_file(self.get_path())
 
-  def get_path(self):
+  def get_path(self, absolute=False):
+    return shared.Cache.get_lib_name(self.get_filename(), absolute=absolute)
+
+  def build(self):
     """
     Gets the cached path of this library.
 
     This will trigger a build if this library is not in the cache.
     """
-    return shared.Cache.get_lib(self.get_filename(), self.build)
+    return shared.Cache.get(self.get_path(), self.do_build, force=USE_NINJA == 2, quiet=USE_NINJA)
 
   def get_link_flag(self):
     """
     Gets the link flags needed to use the library.
 
     This will trigger a build if this library is not in the cache.
     """
-    fullpath = self.get_path()
+    fullpath = self.build()
     # For non-libaries (e.g. crt1.o) we pass the entire path to the linker
     if self.get_ext() != '.a':
       return fullpath
@@ -282,6 +394,12 @@ def get_files(self):
 
     raise NotImplementedError()
 
+  def write_ninja_file(self, filename, libname):
+    cflags = self.get_cflags()
+    asflags = get_base_cflags()
+    input_files = self.get_files()
+    create_ninja_file(input_files, filename, libname, cflags, asflags=asflags, customize_build_flags=self.customize_build_cmd)
+
   def build_objects(self, build_dir):
     """
     Returns a list of compiled object files for this library.
@@ -335,13 +453,27 @@ def customize_build_cmd(self, cmd, filename):  # noqa
     For example, libc uses this to replace -Oz with -O2 for some subset of files."""
     return cmd
 
-  def build(self, out_filename):
+  def do_build(self, out_filename):
     """Builds the library and returns the path to the file."""
-    build_dir = shared.Cache.get_path(os.path.join('build', self.get_base_name()))
-    utils.safe_ensure_dirs(build_dir)
-    create_lib(out_filename, self.build_objects(build_dir))
-    if not shared.DEBUG:
-      utils.delete_dir(build_dir)
+    assert out_filename == self.get_path(absolute=True)
+    build_dir = os.path.join(shared.Cache.get_path('build'), self.get_base_name())
+    if USE_NINJA:
+      ensure_sysroot()
+      utils.safe_ensure_dirs(build_dir)
+      ninja_file = os.path.join(build_dir, 'build.ninja')
+      self.write_ninja_file(ninja_file, out_filename)
+      cmd = ['ninja', '-C', build_dir]
+      if shared.PRINT_STAGES:
+        cmd.append('-v')
+      shared.check_call(cmd, env=clean_env())
+    else:
+      # Use a seperate build directory to the ninja flavor so that building without
+      # EMCC_USE_NINJA doesn't clobber the ninja build tree
+      build_dir += '-tmp'
+      utils.safe_ensure_dirs(build_dir)
+      create_lib(out_filename, self.build_objects(build_dir))
+      if not shared.DEBUG:
+        utils.delete_dir(build_dir)
 
   @classmethod
   def _inherit_list(cls, attr):