Experimental support for bulding system libraries via ninja

The primary advantage of doing this is that it avoids having force
a completely rebuild of a system library when you want to rebuilt it.
Instead not have precise dependencies.

Author: sbc100

embuilder.py

@@ -237,7 +237,7 @@ def main():
       if do_clear:
         library.erase()
       if do_build:
-        library.get_path()
+        library.build()
     elif what == 'sysroot':
       if do_clear:
         shared.Cache.erase_file('sysroot_install.stamp')

tools/cache.py

@@ -108,8 +108,8 @@ def get_lib_dir(self, absolute, varies=True):
       path = Path(path, '-'.join(subdir))
     return path
 
-  def get_lib_name(self, name, varies=True):
-    return str(self.get_lib_dir(absolute=False, varies=varies).joinpath(name))
+  def get_lib_name(self, name, varies=True, absolute=False):
+    return str(self.get_lib_dir(absolute=absolute, varies=varies).joinpath(name))
 
   def erase_lib(self, name):
     self.erase_file(self.get_lib_name(name))
@@ -127,7 +127,7 @@ def get_lib(self, libname, *args, **kwargs):
 
   # Request a cached file. If it isn't in the cache, it will be created with
   # the given creator function
-  def get(self, shortname, creator, what=None, force=False):
+  def get(self, shortname, creator, what=None, force=False, quiet=False):
     cachename = Path(self.dirname, shortname)
     # Check for existence before taking the lock in case we can avoid the
     # lock completely.
@@ -152,6 +152,7 @@ def get(self, shortname, creator, what=None, force=False):
       utils.safe_ensure_dirs(cachename.parent)
       creator(str(cachename))
       assert cachename.exists()
-      logger.info(' - ok')
+      if not quiet:
+        logger.info(' - ok')
 
     return str(cachename)

tools/gen_struct_info.py

@@ -246,7 +246,7 @@ def inspect_headers(headers, cflags):
   # TODO(sbc): If we can remove EM_EXCLUSIVE_CACHE_ACCESS then this would not longer be needed.
   shared.check_sanity()
 
-  compiler_rt = system_libs.Library.get_usable_variations()['libcompiler_rt'].get_path()
+  compiler_rt = system_libs.Library.get_usable_variations()['libcompiler_rt'].build()
 
   # Close all unneeded FDs.
   os.close(src_file[0])

tools/ports/__init__.py

@@ -126,19 +126,28 @@ def build_port(src_dir, output_path, port_name, includes=[], flags=[], exclude_f
     for include in includes:
       cflags.append('-I' + include)
 
-    commands = []
-    objects = []
-    for src in srcs:
-      relpath = os.path.relpath(src, src_dir)
-      obj = os.path.join(build_dir, relpath) + '.o'
-      dirname = os.path.dirname(obj)
-      if not os.path.exists(dirname):
-        os.makedirs(dirname)
-      commands.append([shared.EMCC, '-c', src, '-o', obj] + cflags)
-      objects.append(obj)
-
-    system_libs.run_build_commands(commands)
-    system_libs.create_lib(output_path, objects)
+    if system_libs.USE_NINJA:
+      if not os.path.exists(build_dir):
+        os.makedirs(build_dir)
+      ninja_file = os.path.join(build_dir, 'build.ninja')
+      system_libs.ensure_sysroot()
+      system_libs.create_ninja_file(srcs, ninja_file, output_path, cflags=cflags)
+      system_libs.run_ninja(build_dir)
+    else:
+      commands = []
+      objects = []
+      for src in srcs:
+        relpath = os.path.relpath(src, src_dir)
+        obj = os.path.join(build_dir, relpath) + '.o'
+        dirname = os.path.dirname(obj)
+        if not os.path.exists(dirname):
+          os.makedirs(dirname)
+        commands.append([shared.EMCC, '-c', src, '-o', obj] + cflags)
+        objects.append(obj)
+
+      system_libs.run_build_commands(commands)
+      system_libs.create_lib(output_path, objects)
+
     return output_path
 
   @staticmethod
@@ -274,6 +283,12 @@ def clear_project_build(name):
     utils.delete_dir(build_dir)
     return build_dir
 
+  @staticmethod
+  def write_file(filename, contents):
+    if os.path.exists(filename) and utils.read_file(filename) == contents:
+      return
+    utils.write_file(filename, contents)
+
 
 def dependency_order(port_list):
   # Perform topological sort of ports according to the dependency DAG

tools/ports/boost_headers.py

@@ -5,7 +5,6 @@
 
 import logging
 import os
-from pathlib import Path
 
 TAG = '1.75.0'
 HASH = '8c38be1ebef1b8ada358ad6b7c9ec17f5e0a300e8085db3473a13e19712c95eeb3c3defacd3c53482eb96368987c4b022efa8da2aac2431a154e40153d3c3dcd'
@@ -31,7 +30,7 @@ def create(final):
     # this is needed as emscripted ports expect this, even if it is not used
     dummy_file = os.path.join(source_path, 'dummy.cpp')
     shared.safe_ensure_dirs(os.path.dirname(dummy_file))
-    Path(dummy_file).write_text('static void dummy() {}')
+    ports.write_file(dummy_file, 'static void dummy() {}')
 
     ports.build_port(source_path, final, 'boost_headers', srcs=['dummy.cpp'])
 

tools/ports/freetype.py

@@ -4,7 +4,6 @@
 # found in the LICENSE file.
 
 import os
-from pathlib import Path
 
 TAG = 'version_1'
 HASH = '0d0b1280ba0501ad0a23cf1daa1f86821c722218b59432734d3087a89acd22aabd5c3e5e1269700dcd41e87073046e906060f167c032eb91a3ac8c5808a02783'
@@ -19,7 +18,7 @@ def get(ports, settings, shared):
 
   def create(final):
     source_path = os.path.join(ports.get_dir(), 'freetype', 'FreeType-' + TAG)
-    Path(source_path, 'include/ftconfig.h').write_text(ftconf_h)
+    ports.write_file(os.path.join(source_path, 'include/ftconfig.h'), ftconf_h)
     ports.install_header_dir(os.path.join(source_path, 'include'),
                              target=os.path.join('freetype2'))
 

tools/ports/libjpeg.py

@@ -5,7 +5,6 @@
 
 import os
 import logging
-from pathlib import Path
 
 VERSION = '9c'
 HASH = 'b2affe9a1688bd49fc033f4682c4a242d4ee612f1affaef532f5adcb4602efc4433c4a52a4b3d69e7440ff1f6413b1b041b419bc90efd6d697999961a9a6afb7'
@@ -24,7 +23,7 @@ def get(ports, settings, shared):
   def create(final):
     logging.info('building port: libjpeg')
     source_path = os.path.join(ports.get_dir(), 'libjpeg', 'jpeg-9c')
-    Path(source_path, 'jconfig.h').write_text(jconfig_h)
+    ports.write_file(os.path.join(source_path, 'jconfig.h'), jconfig_h)
     ports.install_headers(source_path)
     excludes = [
       'ansi2knr.c', 'cjpeg.c', 'ckconfig.c', 'djpeg.c', 'example.c',

tools/ports/libmodplug.py

@@ -5,7 +5,6 @@
 
 import os
 import logging
-from pathlib import Path
 
 TAG = '11022021'
 HASH = 'f770031ad6c2152cbed8c8eab8edf2be1d27f9e74bc255a9930c17019944ee5fdda5308ea992c66a78af9fe1d8dca090f6c956910ce323f8728247c10e44036b'
@@ -25,7 +24,7 @@ def create(final):
     src_dir = os.path.join(source_path, 'src')
     libmodplug_path = os.path.join(src_dir, 'libmodplug')
 
-    Path(source_path, 'config.h').write_text(config_h)
+    ports.write_file(os.path.join(source_path, 'config.h'), config_h)
 
     flags = [
       '-Wno-deprecated-register',

tools/ports/libpng.py

@@ -5,7 +5,6 @@
 
 import os
 import logging
-from pathlib import Path
 
 TAG = '1.6.37'
 HASH = '2ce2b855af307ca92a6e053f521f5d262c36eb836b4810cb53c809aa3ea2dcc08f834aee0ffd66137768a54397e28e92804534a74abb6fc9f6f3127f14c9c338'
@@ -30,7 +29,7 @@ def create(final):
     logging.info('building port: libpng')
 
     source_path = os.path.join(ports.get_dir(), 'libpng', 'libpng-' + TAG)
-    Path(source_path, 'pnglibconf.h').write_text(pnglibconf_h)
+    ports.write_file(os.path.join(source_path, 'pnglibconf.h'), pnglibconf_h)
     ports.install_headers(source_path)
 
     flags = ['-sUSE_ZLIB=1']

tools/ports/mpg123.py

@@ -5,7 +5,6 @@
 
 import os
 import logging
-from pathlib import Path
 
 TAG = '1.26.2'
 HASH = 'aa63fcb08b243a1e09f7701b3d84a19d7412a87253d54d49f014fdb9e75bbc81d152a41ed750fccde901453929b2a001585a7645351b41845ad205c17a73dcc9'
@@ -27,8 +26,8 @@ def create(final):
     libmpg123_path = os.path.join(src_path, 'libmpg123')
     compat_path = os.path.join(src_path, 'compat')
 
-    Path(src_path, 'config.h').write_text(config_h)
-    Path(libmpg123_path, 'mpg123.h').write_text(mpg123_h)
+    ports.write_file(os.path.join(src_path, 'config.h'), config_h)
+    ports.write_file(os.path.join(libmpg123_path, 'mpg123.h'), mpg123_h)
 
     # copy header to a location so it can be used as 'MPG123/'
     ports.install_headers(libmpg123_path, pattern="*123.h", target='')

tools/ports/ogg.py

@@ -5,7 +5,6 @@
 
 import logging
 import os
-from pathlib import Path
 
 TAG = 'version_1'
 HASH = '929e8d6003c06ae09593021b83323c8f1f54532b67b8ba189f4aedce52c25dc182bac474de5392c46ad5b0dea5a24928e4ede1492d52f4dd5cd58eea9be4dba7'
@@ -22,7 +21,7 @@ def create(final):
     logging.info('building port: ogg')
 
     source_path = os.path.join(ports.get_dir(), 'ogg', 'Ogg-' + TAG)
-    Path(source_path, 'include', 'ogg', 'config_types.h').write_text(config_types_h)
+    ports.write_file(os.path.join(source_path, 'include', 'ogg', 'config_types.h'), config_types_h)
     ports.install_header_dir(os.path.join(source_path, 'include', 'ogg'), 'ogg')
     ports.build_port(os.path.join(source_path, 'src'), final, 'ogg')
 

tools/ports/zlib.py

@@ -4,7 +4,6 @@
 # found in the LICENSE file.
 
 import os
-from pathlib import Path
 
 VERSION = '1.2.12'
 HASH = 'cc2366fa45d5dfee1f983c8c51515e0cff959b61471e2e8d24350dea22d3f6fcc50723615a911b046ffc95f51ba337d39ae402131a55e6d1541d3b095d6c0a14'
@@ -19,7 +18,7 @@ def get(ports, settings, shared):
 
   def create(final):
     source_path = os.path.join(ports.get_dir(), 'zlib', 'zlib-' + VERSION)
-    Path(source_path, 'zconf.h').write_text(zconf_h)
+    ports.write_file(os.path.join(source_path, 'zconf.h'), zconf_h)
     ports.install_headers(source_path)
 
     # build