Convert all ports to use build_port. NFC (#17822)

This will allow more easy transition to alternative build systems
such as ninja (See #17809).

Also, update build_port to allow separate source and build directories.
This avoids copying the entire source code of each port from the
extracts archive into the build directory.

Author: sbc100

test/test_browser.py

@@ -3285,8 +3285,8 @@ def test_sdl2_mixer_music(self, formats, flags, music_name):
   def test_cocos2d_hello(self):
     # cocos2d build contains a bunch of warnings about tiff symbols being missing at link time:
     # e.g. warning: undefined symbol: TIFFClientOpen
-    cocos2d_root = os.path.join(ports.Ports.get_build_dir(), 'cocos2d')
-    preload_file = os.path.join(cocos2d_root, 'samples', 'HelloCpp', 'Resources') + '@'
+    cocos2d_root = os.path.join(ports.Ports.get_dir(), 'cocos2d', 'Cocos2d-version_3_3')
+    preload_file = os.path.join(cocos2d_root, 'samples', 'Cpp', 'HelloCpp', 'Resources') + '@'
     self.btest('cocos2d_hello.cpp', reference='cocos2d_hello.png', reference_slack=1,
                args=['-sUSE_COCOS2D=3', '-sERROR_ON_UNDEFINED_SYMBOLS=0',
                      '-Wno-js-compiler',

tools/ports/__init__.py

@@ -108,45 +108,39 @@ def install_headers(src_dir, pattern='*.h', target=None):
       shutil.copyfile(f, os.path.join(dest, os.path.basename(f)))
 
   @staticmethod
-  def build_port(src_path, output_path, includes=[], flags=[], exclude_files=[], exclude_dirs=[]):  # noqa
-    srcs = []
-    for root, _, files in os.walk(src_path, topdown=False):
-      if any((excluded in root) for excluded in exclude_dirs):
-        continue
-      for f in files:
-        ext = shared.suffix(f)
-        if ext in ('.c', '.cpp') and not any((excluded in f) for excluded in exclude_files):
-          srcs.append(os.path.join(root, f))
-    include_commands = ['-I' + src_path]
+  def build_port(src_dir, output_path, build_dir, includes=[], flags=[], exclude_files=[], exclude_dirs=[], srcs=[]):  # noqa
+    if srcs:
+      srcs = [os.path.join(src_dir, s) for s in srcs]
+    else:
+      srcs = []
+      for root, _, files in os.walk(src_dir, topdown=False):
+        if any((excluded in root) for excluded in exclude_dirs):
+          continue
+        for f in files:
+          ext = shared.suffix(f)
+          if ext in ('.c', '.cpp') and not any((excluded in f) for excluded in exclude_files):
+            srcs.append(os.path.join(root, f))
+
+    cflags = system_libs.get_base_cflags() + ['-O2', '-w', '-I' + src_dir] + flags
     for include in includes:
-      include_commands.append('-I' + include)
+      cflags.append('-I' + include)
 
+    if build_dir:
+      if not os.path.exists(build_dir):
+        os.makedirs(build_dir)
+      build_dir = src_dir
     commands = []
     objects = []
     for src in srcs:
-      obj = src + '.o'
-      commands.append([shared.EMCC, '-c', src, '-O2', '-o', obj, '-w'] + include_commands + flags)
+      relpath = os.path.relpath(src, src_dir)
+      obj = os.path.join(build_dir, relpath) + '.o'
+      commands.append([shared.EMCC, '-c', src, '-o', obj] + cflags)
       objects.append(obj)
 
-    Ports.run_commands(commands)
+    system_libs.run_build_commands(commands)
     system_libs.create_lib(output_path, objects)
     return output_path
 
-  @staticmethod
-  def run_commands(commands):
-    # Runs a sequence of compiler commands, adding importand cflags as defined by get_cflags() so
-    # that the ports are built in the correct configuration.
-    def add_args(cmd):
-      # this must only be called on a standard build command
-      assert cmd[0] in (shared.EMCC, shared.EMXX)
-      # add standard cflags, but also allow the cmd to override them
-      return cmd[:1] + system_libs.get_base_cflags() + cmd[1:]
-    system_libs.run_build_commands([add_args(c) for c in commands])
-
-  @staticmethod
-  def create_lib(libname, inputs): # make easily available for port objects
-    system_libs.create_lib(libname, inputs)
-
   @staticmethod
   def get_dir():
     dirname = config.PORTS

tools/ports/boost_headers.py

@@ -5,6 +5,7 @@
 
 import logging
 import os
+from pathlib import Path
 
 TAG = '1.75.0'
 HASH = '8c38be1ebef1b8ada358ad6b7c9ec17f5e0a300e8085db3473a13e19712c95eeb3c3defacd3c53482eb96368987c4b022efa8da2aac2431a154e40153d3c3dcd'
@@ -20,27 +21,19 @@ def get(ports, settings, shared):
 
   def create(final):
     logging.info('building port: boost_headers')
-    build_dir = ports.clear_project_build('boost_headers')
 
     # includes
     source_path_include = os.path.join(ports.get_dir(), 'boost_headers', 'boost')
     ports.install_header_dir(source_path_include, 'boost')
 
     # write out a dummy cpp file, to create an empty library
     # this is needed as emscripted ports expect this, even if it is not used
+    build_dir = ports.clear_project_build('boost_headers')
     dummy_file = os.path.join(build_dir, 'dummy.cpp')
     shared.safe_ensure_dirs(os.path.dirname(dummy_file))
-    with open(dummy_file, 'w') as f:
-      f.write('static void dummy() {}')
-
-    commands = []
-    o_s = []
-    obj = dummy_file + '.o'
-    command = [shared.EMCC, '-c', dummy_file, '-o', obj]
-    commands.append(command)
-    ports.run_commands(commands)
-    o_s.append(obj)
-    ports.create_lib(final, o_s)
+    Path(dummy_file).write_text('static void dummy() {}')
+
+    ports.build_port(build_dir, final, build_dir)
 
   return [shared.Cache.get_lib('libboost_headers.a', create, what='port')]
 

tools/ports/bullet.py

@@ -23,8 +23,7 @@ def create(final):
 
     source_path = os.path.join(ports.get_dir(), 'bullet', 'Bullet-' + TAG)
     dest_path = ports.clear_project_build('bullet')
-    shutil.copytree(source_path, dest_path)
-    src_path = os.path.join(dest_path, 'bullet', 'src')
+    src_path = os.path.join(source_path, 'bullet', 'src')
 
     dest_include_path = ports.get_include_dir('bullet')
     for base, _, files in os.walk(src_path):
@@ -42,8 +41,7 @@ def create(final):
       for dir in dirs:
         includes.append(os.path.join(base, dir))
 
-    ports.build_port(src_path, final, includes=includes, exclude_dirs=['MiniCL'],
-                     flags=['-std=gnu++14'])
+    ports.build_port(src_path, final, dest_path, includes=includes, exclude_dirs=['MiniCL'], flags=['-std=gnu++14'])
 
   return [shared.Cache.get_lib('libbullet.a', create)]
 

tools/ports/bzip2.py

@@ -4,7 +4,6 @@
 # found in the LICENSE file.
 
 import os
-import shutil
 
 VERSION = '1.0.6'
 HASH = '512cbfde5144067f677496452f3335e9368fd5d7564899cb49e77847b9ae7dca598218276637cbf5ec524523be1e8ace4ad36a148ef7f4badf3f6d5a002a4bb2'
@@ -18,26 +17,16 @@ def get(ports, settings, shared):
   ports.fetch_project('bzip2', 'https://github.com/emscripten-ports/bzip2/archive/' + VERSION + '.zip', 'bzip2-' + VERSION, sha512hash=HASH)
 
   def create(final):
-    dest_path = ports.clear_project_build('bzip2')
     source_path = os.path.join(ports.get_dir(), 'bzip2', 'bzip2-' + VERSION)
-    shutil.copytree(source_path, dest_path)
+    ports.install_headers(source_path)
 
     # build
     srcs = [
       'blocksort.c', 'compress.c', 'decompress.c', 'huffman.c',
       'randtable.c', 'bzlib.c', 'crctable.c',
     ]
-    commands = []
-    o_s = []
-    for src in srcs:
-      o = os.path.join(dest_path, shared.replace_suffix(src, '.o'))
-      shared.safe_ensure_dirs(os.path.dirname(o))
-      commands.append([shared.EMCC, '-c', os.path.join(dest_path, src), '-O2', '-o', o, '-I' + dest_path, '-w', ])
-      o_s.append(o)
-    ports.run_commands(commands)
-
-    ports.create_lib(final, o_s)
-    ports.install_headers(source_path)
+    dest_path = ports.clear_project_build('bzip2')
+    ports.build_port(source_path, final, dest_path, srcs=srcs)
 
   return [shared.Cache.get_lib('libbz2.a', create, what='port')]
 

tools/ports/cocos2d.py

@@ -4,7 +4,6 @@
 # found in the LICENSE file.
 
 import os
-import shutil
 import logging
 import re
 
@@ -28,49 +27,35 @@ def create(final):
     cocos2d_src = os.path.join(ports.get_dir(), 'cocos2d')
     cocos2d_root = os.path.join(cocos2d_src, 'Cocos2d-' + TAG)
     cocos2dx_root = os.path.join(cocos2d_root, 'cocos2dx')
-    cocos2dx_src = make_source_list(cocos2d_root, cocos2dx_root)
-    cocos2dx_includes = make_includes(cocos2d_root)
-
-    cocos2d_build = ports.clear_project_build('cocos2d')
-    shutil.copytree(os.path.join(cocos2d_root, 'samples', 'Cpp'),
-                    os.path.join(cocos2d_build, 'samples'))
-
-    commands = []
-    o_s = []
-    for src in cocos2dx_src:
-      o = os.path.join(cocos2d_build, 'Cocos2d-' + TAG, 'build', src + '.o')
-      shared.safe_ensure_dirs(os.path.dirname(o))
-      command = [shared.EMCC,
-                 '-c', src,
-                 '-Wno-overloaded-virtual',
-                 '-Wno-deprecated-declarations',
-                 '-D__CC_PLATFORM_FILEUTILS_CPP__',
-                 '-DCC_ENABLE_CHIPMUNK_INTEGRATION',
-                 '-DCC_KEYBOARD_SUPPORT',
-                 '-DGL_ES=1',
-                 '-DNDEBUG', # '-DCOCOS2D_DEBUG=1' 1 - error/warn, 2 - verbose
-                 # Cocos2d source code hasn't switched to __EMSCRIPTEN__.
-                 # See https://github.com/emscripten-ports/Cocos2d/pull/3
-                 '-DEMSCRIPTEN',
-                 '-DCP_USE_DOUBLES=0',
-                 '-O2',
-                 '-sUSE_ZLIB=1',
-                 '-sUSE_LIBPNG=1',
-                 '-o', o, '-w']
-
-      for include in cocos2dx_includes:
-        command.append('-I' + include)
-
-      commands.append(command)
-      o_s.append(o)
-    shared.safe_ensure_dirs(os.path.dirname(o_s[0]))
-    ports.run_commands(commands)
-    ports.create_lib(final, o_s)
-
-    for dirname in cocos2dx_includes:
+
+    srcs = make_source_list(cocos2d_root, cocos2dx_root)
+    includes = make_includes(cocos2d_root)
+    flags = [
+      '-Wno-overloaded-virtual',
+      '-Wno-deprecated-declarations',
+      '-D__CC_PLATFORM_FILEUTILS_CPP__',
+      '-DCC_ENABLE_CHIPMUNK_INTEGRATION',
+      '-DCC_KEYBOARD_SUPPORT',
+      '-DGL_ES=1',
+      '-DNDEBUG', # '-DCOCOS2D_DEBUG=1' 1 - error/warn, 2 - verbose
+      # Cocos2d source code hasn't switched to __EMSCRIPTEN__.
+      # See https://github.com/emscripten-ports/Cocos2d/pull/3
+      '-DEMSCRIPTEN',
+      '-DCP_USE_DOUBLES=0',
+      '-sUSE_ZLIB=1',
+      '-sUSE_LIBPNG=1',
+    ]
+    build_dir = ports.clear_project_build('cocos2d')
+
+    for dirname in includes:
       target = os.path.join('cocos2d', os.path.relpath(dirname, cocos2d_root))
       ports.install_header_dir(dirname, target=target)
 
+    ports.build_port(cocos2d_src, final, build_dir,
+                     flags=flags,
+                     includes=includes,
+                     srcs=srcs)
+
   return [shared.Cache.get_lib('libcocos2d.a', create, what='port')]
 
 

tools/ports/freetype.py

@@ -4,7 +4,6 @@
 # found in the LICENSE file.
 
 import os
-import shutil
 from pathlib import Path
 
 TAG = 'version_1'
@@ -19,10 +18,10 @@ def get(ports, settings, shared):
   ports.fetch_project('freetype', 'https://github.com/emscripten-ports/FreeType/archive/' + TAG + '.zip', 'FreeType-' + TAG, sha512hash=HASH)
 
   def create(final):
-    dest_path = ports.clear_project_build('freetype')
     source_path = os.path.join(ports.get_dir(), 'freetype', 'FreeType-' + TAG)
-    shutil.copytree(source_path, dest_path)
-    Path(dest_path, 'include/ftconfig.h').write_text(ftconf_h)
+    Path(source_path, 'include/ftconfig.h').write_text(ftconf_h)
+    ports.install_header_dir(os.path.join(source_path, 'include'),
+                             target=os.path.join('freetype2'))
 
     # build
     srcs = ['src/autofit/autofit.c',
@@ -77,31 +76,23 @@ def create(final):
             'src/type1/type1.c',
             'src/type42/type42.c',
             'src/winfonts/winfnt.c']
-    commands = []
-    o_s = []
-    for src in srcs:
-      o = os.path.join(dest_path, shared.replace_suffix(src, '.o'))
-      shared.safe_ensure_dirs(os.path.dirname(o))
-      commands.append([shared.EMCC, '-c', os.path.join(dest_path, src), '-o', o,
-                       '-DFT2_BUILD_LIBRARY', '-O2',
-                       '-I' + dest_path + '/include',
-                       '-I' + dest_path + '/truetype',
-                       '-I' + dest_path + '/sfnt',
-                       '-I' + dest_path + '/autofit',
-                       '-I' + dest_path + '/smooth',
-                       '-I' + dest_path + '/raster',
-                       '-I' + dest_path + '/psaux',
-                       '-I' + dest_path + '/psnames',
-                       '-I' + dest_path + '/truetype',
-                       '-w',
-                       '-pthread'])
-      o_s.append(o)
-
-    ports.run_commands(commands)
-    ports.create_lib(final, o_s)
-
-    ports.install_header_dir(os.path.join(dest_path, 'include'),
-                             target=os.path.join('freetype2'))
+
+    flags = [
+      '-DFT2_BUILD_LIBRARY',
+      '-I' + source_path + '/include',
+      '-I' + source_path + '/truetype',
+      '-I' + source_path + '/sfnt',
+      '-I' + source_path + '/autofit',
+      '-I' + source_path + '/smooth',
+      '-I' + source_path + '/raster',
+      '-I' + source_path + '/psaux',
+      '-I' + source_path + '/psnames',
+      '-I' + source_path + '/truetype',
+      '-pthread'
+    ]
+
+    dest_path = ports.clear_project_build('freetype')
+    ports.build_port(source_path, final, dest_path, flags=flags, srcs=srcs)
 
   return [shared.Cache.get_lib('libfreetype.a', create, what='port')]
 

tools/ports/giflib.py

@@ -4,7 +4,6 @@
 # found in the LICENSE file.
 
 import os
-import shutil
 import logging
 
 VERSION = '5.2.1'
@@ -20,14 +19,10 @@ def get(ports, settings, shared):
 
   def create(final):
     logging.info('building port: giflib')
-
     source_path = os.path.join(ports.get_dir(), 'giflib', f'giflib-{VERSION}')
     dest_path = ports.clear_project_build('giflib')
-    shutil.copytree(source_path, dest_path)
-
-    ports.install_headers(dest_path)
-
-    ports.build_port(dest_path, final)
+    ports.install_headers(source_path)
+    ports.build_port(source_path, final, dest_path)
 
   return [shared.Cache.get_lib('libgif.a', create, what='port')]
 

tools/ports/harfbuzz.py

@@ -87,7 +87,6 @@ def get(ports, settings, shared):
 
   def create(final):
     logging.info('building port: harfbuzz')
-    build_path = ports.clear_project_build('harfbuzz')
 
     source_path = os.path.join(ports.get_dir(), 'harfbuzz', 'harfbuzz-' + VERSION)
     freetype_include = ports.get_include_dir('freetype2')
@@ -131,15 +130,8 @@ def create(final):
     else:
       cflags.append('-DHB_NO_MT')
 
-    commands = []
-    o_s = []
-    for src in srcs:
-      o = os.path.join(build_path, src + '.o')
-      shared.safe_ensure_dirs(os.path.dirname(o))
-      commands.append([shared.EMCC, '-c', os.path.join(source_path, 'src', src), '-o', o] + cflags)
-      o_s.append(o)
-    ports.run_commands(commands)
-    ports.create_lib(final, o_s)
+    build_dir = ports.clear_project_build('harfbuzz')
+    ports.build_port(os.path.join(source_path, 'src'), final, build_dir, flags=cflags, srcs=srcs)
 
   return [shared.Cache.get_lib(get_lib_name(settings), create, what='port')]