Use single cache rather then reconfiguring/moving it

This was split out from the change to use a single sysroot (#13090).  I
think it cleaner this way: There is single cache, with single lock file
and the root doesn't change, but the libraries live in sub-directories
within the cache.

Author: sbc100

embuilder.py

@@ -129,13 +129,9 @@ def main():
 
   if args.lto:
     shared.Settings.LTO = "full"
-    # Reconfigure the cache dir to reflect the change
-    shared.reconfigure_cache()
 
   if args.pic:
     shared.Settings.RELOCATABLE = 1
-    # Reconfigure the cache dir to reflect the change
-    shared.reconfigure_cache()
 
   if args.force:
     force = True

emcc.py

@@ -1331,10 +1331,6 @@ def filter_out_duplicate_dynamic_libs(inputs):
                                              '_emscripten_stack_get_end',
                                              '_emscripten_stack_set_limits']
 
-    # Reconfigure the cache now that settings have been applied. Some settings
-    # such as LTO and SIDE_MODULE/MAIN_MODULE effect which cache directory we use.
-    shared.reconfigure_cache()
-
     if not compile_only and not options.post_link:
       ldflags = shared.emsdk_ldflags(newargs)
       for f in ldflags:

tests/test_sanity.py

@@ -21,7 +21,7 @@
 from tools.shared import EXPECTED_LLVM_VERSION, Cache
 from tools import shared, system_libs, utils
 
-SANITY_FILE = shared.Cache.get_path('sanity.txt', root=True)
+SANITY_FILE = shared.Cache.get_path('sanity.txt')
 commands = [[EMCC], [PYTHON, path_from_root('tests', 'runner.py'), 'blahblah']]
 
 
@@ -411,31 +411,29 @@ def ensure_cache(self):
     self.do([EMCC, '-O2', path_from_root('tests', 'hello_world.c')])
 
   def test_emcc_caching(self):
-    BUILDING_MESSAGE = 'generating system library: X'
+    BUILDING_MESSAGE = 'generating system library: %s'
 
     restore_and_set_up()
     self.erase_cache()
 
     # Building a file that *does* need something *should* trigger cache
     # generation, but only the first time
-    libname = 'libc++'
+    libname = Cache.get_lib_name('libc++.a')
     for i in range(3):
       print(i)
       self.clear()
       output = self.do([EMCC, '-O' + str(i), path_from_root('tests', 'hello_libcxx.cpp'), '-s', 'DISABLE_EXCEPTION_CATCHING=0'])
       print('\n\n\n', output)
-      self.assertContainedIf(BUILDING_MESSAGE.replace('X', libname), output, i == 0)
+      self.assertContainedIf(BUILDING_MESSAGE % libname, output, i == 0)
       self.assertContained('hello, world!', self.run_js('a.out.js'))
       self.assertExists(Cache.dirname)
-      full_libname = libname + '.bc' if libname != 'libc++' else libname + '.a'
-      self.assertExists(os.path.join(Cache.dirname, full_libname))
+      self.assertExists(os.path.join(Cache.dirname, libname))
 
   def test_cache_clearing_manual(self):
     # Manual cache clearing
     restore_and_set_up()
     self.ensure_cache()
     self.assertTrue(os.path.exists(Cache.dirname))
-    self.assertTrue(os.path.exists(Cache.root_dirname))
     output = self.do([EMCC, '--clear-cache'])
     self.assertIn('clearing cache', output)
     self.assertIn(SANITY_MESSAGE, output)
@@ -459,12 +457,10 @@ def test_FROZEN_CACHE(self):
     self.erase_cache()
     self.ensure_cache()
     self.assertTrue(os.path.exists(Cache.dirname))
-    self.assertTrue(os.path.exists(Cache.root_dirname))
     # changing config file should not clear cache
     add_to_config('FROZEN_CACHE = True')
     self.do([EMCC])
     self.assertTrue(os.path.exists(Cache.dirname))
-    self.assertTrue(os.path.exists(Cache.root_dirname))
     # building libraries is disallowed
     output = self.do([EMBUILDER, 'build', 'libemmalloc'])
     self.assertIn('FROZEN_CACHE disallows building system libs', output)
@@ -483,6 +479,7 @@ def test_emcc_multiprocess_cache_access(self):
       }
       ''')
     cache_dir_name = self.in_dir('test_cache')
+    libname = Cache.get_lib_name('libc.a')
     with env_modify({'EM_CACHE': cache_dir_name}):
       tasks = []
       num_times_libc_was_built = 0
@@ -491,13 +488,13 @@ def test_emcc_multiprocess_cache_access(self):
         tasks += [p]
       for p in tasks:
         print('stdout:\n', p.stdout)
-        if 'generating system library: libc' in p.stdout:
+        if 'generating system library: ' + libname in p.stdout:
           num_times_libc_was_built += 1
 
     # The cache directory must exist after the build
     self.assertTrue(os.path.exists(cache_dir_name))
     # The cache directory must contain a built libc
-    self.assertTrue(os.path.exists(os.path.join(cache_dir_name, 'wasm', 'libc.a')))
+    self.assertTrue(os.path.exists(os.path.join(cache_dir_name, libname)))
     # Exactly one child process should have triggered libc build!
     self.assertEqual(num_times_libc_was_built, 1)
 
@@ -531,12 +528,11 @@ def test_emcc_ports(self):
     restore_and_set_up()
 
     # listing ports
-
     out = self.do([EMCC, '--show-ports'])
-    assert 'Available ports:' in out, out
-    assert 'SDL2' in out, out
-    assert 'SDL2_image' in out, out
-    assert 'SDL2_net' in out, out
+    self.assertContained('Available ports:', out)
+    self.assertContained('SDL2', out)
+    self.assertContained('SDL2_image', out)
+    self.assertContained('SDL2_net', out)
 
     # using ports
     RETRIEVING_MESSAGE = 'retrieving port'
@@ -551,27 +547,27 @@ def test_emcc_ports(self):
         try_delete(PORTS_DIR)
       else:
         self.do([EMCC, '--clear-ports'])
-      assert not os.path.exists(PORTS_DIR)
+      self.assertNotExists(PORTS_DIR)
 
       # Building a file that doesn't need ports should not trigger anything
       output = self.do([EMCC, path_from_root('tests', 'hello_world_sdl.cpp')])
       assert RETRIEVING_MESSAGE not in output, output
       assert BUILDING_MESSAGE not in output
       print('no', output)
-      assert not os.path.exists(PORTS_DIR)
+      self.assertNotExists(PORTS_DIR)
 
       def first_use():
         output = self.do([EMCC, path_from_root('tests', 'hello_world_sdl.cpp'), '-s', 'USE_SDL=2'])
-        assert RETRIEVING_MESSAGE in output, output
-        assert BUILDING_MESSAGE in output, output
+        self.assertContained(RETRIEVING_MESSAGE, output)
+        self.assertContained(BUILDING_MESSAGE, output)
         self.assertExists(PORTS_DIR)
         print('yes', output)
 
       def second_use():
         # Using it again avoids retrieve and build
         output = self.do([EMCC, path_from_root('tests', 'hello_world_sdl.cpp'), '-s', 'USE_SDL=2'])
-        assert RETRIEVING_MESSAGE not in output, output
-        assert BUILDING_MESSAGE not in output, output
+        self.assertNotContained(RETRIEVING_MESSAGE, output)
+        self.assertNotContained(BUILDING_MESSAGE, output)
 
       # Building a file that need a port does trigger stuff
       first_use()

tools/cache.py

@@ -24,22 +24,9 @@ class Cache:
   # acquired.
   EM_EXCLUSIVE_CACHE_ACCESS = int(os.environ.get('EM_EXCLUSIVE_CACHE_ACCESS', '0'))
 
-  def __init__(self, dirname, use_subdir=True):
+  def __init__(self, dirname):
     # figure out the root directory for all caching
     dirname = os.path.normpath(dirname)
-    self.root_dirname = dirname
-
-    # if relevant, use a subdir of the cache
-    if use_subdir:
-      subdir = 'wasm'
-      if shared.Settings.LTO:
-        subdir += '-lto'
-      if shared.Settings.RELOCATABLE:
-        subdir += '-pic'
-      if shared.Settings.MEMORY64:
-        subdir += '-memory64'
-      dirname = os.path.join(dirname, subdir)
-
     self.dirname = dirname
     self.acquired_count = 0
 
@@ -90,28 +77,46 @@ def ensure(self):
 
   def erase(self):
     with self.lock():
-      if os.path.exists(self.root_dirname):
-        for f in os.listdir(self.root_dirname):
-          tempfiles.try_delete(os.path.join(self.root_dirname, f))
+      if os.path.exists(self.dirname):
+        for f in os.listdir(self.dirname):
+          tempfiles.try_delete(os.path.join(self.dirname, f))
+
+  def get_path(self, name):
+    return os.path.join(self.dirname, name)
+
+  def get_include_dir(self):
+    return os.path.join(self.dirname, 'include')
 
-  def get_path(self, shortname, root=False):
-    if root:
-      return os.path.join(self.root_dirname, shortname)
-    return os.path.join(self.dirname, shortname)
+  def get_lib_dir(self):
+    subdir = 'wasm'
+    if shared.Settings.LTO:
+      subdir += '-lto'
+    if shared.Settings.RELOCATABLE:
+      subdir += '-pic'
+    if shared.Settings.MEMORY64:
+      subdir += '-memory64'
+    return subdir
+
+  def get_lib_name(self, name):
+    return os.path.join(self.get_lib_dir(), name)
+
+  def erase_lib(self, name):
+    self.erase_file(self.get_lib_name(name))
 
   def erase_file(self, shortname):
     name = os.path.join(self.dirname, shortname)
     if os.path.exists(name):
       logging.info('Cache: deleting cached file: %s', name)
       tempfiles.try_delete(name)
 
+  def get_lib(self, libname, *args, **kwargs):
+    name = self.get_lib_name(libname)
+    return self.get(name, *args, **kwargs)
+
   # Request a cached file. If it isn't in the cache, it will be created with
   # the given creator function
-  def get(self, shortname, creator, what=None, force=False, root=False):
-    if root:
-      cachename = os.path.join(self.root_dirname, shortname)
-    else:
-      cachename = os.path.join(self.dirname, shortname)
+  def get(self, shortname, creator, what=None, force=False):
+    cachename = os.path.join(self.dirname, shortname)
     cachename = os.path.abspath(cachename)
     # Check for existence before taking the lock in case we can avoid the
     # lock completely.

tools/ports/boost_headers.py

@@ -48,7 +48,7 @@ def create():
     ports.create_lib(final, o_s)
     return final
 
-  return [shared.Cache.get(libname, create, what='port')]
+  return [shared.Cache.get_lib(libname, create, what='port')]
 
 
 def clear(ports, settings, shared):

tools/ports/bullet.py

@@ -50,7 +50,7 @@ def create():
     ports.build_port(src_path, final, includes=includes, exclude_dirs=['MiniCL'])
     return final
 
-  return [shared.Cache.get(libname, create)]
+  return [shared.Cache.get_lib(libname, create)]
 
 
 def clear(ports, settings, shared):

tools/ports/bzip2.py

@@ -47,7 +47,7 @@ def create():
     ports.install_headers(source_path)
     return final
 
-  return [shared.Cache.get('libbz2.a', create, what='port')]
+  return [shared.Cache.get_lib('libbz2.a', create, what='port')]
 
 
 def clear(ports, settings, shared):

tools/ports/cocos2d.py

@@ -74,7 +74,7 @@ def create():
 
     return final
 
-  return [shared.Cache.get(libname, create, what='port')]
+  return [shared.Cache.get_lib(libname, create, what='port')]
 
 
 def clear(ports, settings, shared):

tools/ports/freetype.py

@@ -109,7 +109,7 @@ def create():
                              target=os.path.join('freetype2', 'freetype'))
     return final
 
-  return [shared.Cache.get('libfreetype.a', create, what='port')]
+  return [shared.Cache.get_lib('libfreetype.a', create, what='port')]
 
 
 def clear(ports, settings, shared):

tools/ports/harfbuzz.py

@@ -32,7 +32,7 @@ def create():
     source_path = os.path.join(ports.get_dir(), 'harfbuzz', 'harfbuzz-' + TAG)
     dest_path = os.path.join(ports.get_build_dir(), 'harfbuzz')
 
-    freetype_lib = shared.Cache.get_path('libfreetype.a')
+    freetype_lib = shared.Cache.get_path(shared.Cache.get_lib_name('libfreetype.a'))
     freetype_include = os.path.join(ports.get_include_dir(), 'freetype2', 'freetype')
     freetype_include_dirs = freetype_include + ';' + os.path.join(freetype_include, 'config')
 
@@ -67,7 +67,7 @@ def create():
 
     return os.path.join(dest_path, 'libharfbuzz.a')
 
-  return [shared.Cache.get(get_lib_name(settings), create, what='port')]
+  return [shared.Cache.get_lib(get_lib_name(settings), create, what='port')]
 
 
 def clear(ports, settings, shared):

tools/ports/icu.py

@@ -36,7 +36,7 @@ def create():
     ports.install_header_dir(os.path.join(dest_path, 'source', 'common', 'unicode'))
     return final
 
-  return [shared.Cache.get(libname, create)]
+  return [shared.Cache.get_lib(libname, create)]
 
 
 def clear(ports, settings, shared):

tools/ports/libjpeg.py

@@ -43,7 +43,7 @@ def create():
     )
     return final
 
-  return [shared.Cache.get(libname, create, what='port')]
+  return [shared.Cache.get_lib(libname, create, what='port')]
 
 
 def clear(ports, settings, shared):

tools/ports/libpng.py

@@ -38,7 +38,7 @@ def create():
     ports.build_port(dest_path, final, flags=['-s', 'USE_ZLIB=1'], exclude_files=['pngtest'], exclude_dirs=['scripts', 'contrib'])
     return final
 
-  return [shared.Cache.get(libname, create, what='port')]
+  return [shared.Cache.get_lib(libname, create, what='port')]
 
 
 def clear(ports, settings, shared):

tools/ports/mpg123.py

@@ -93,7 +93,7 @@ def create():
     ports.install_headers(libmpg123_path, pattern="*123.h", target='')
     return output_path
 
-  return [shared.Cache.get(libname, create, what='port')]
+  return [shared.Cache.get_lib(libname, create, what='port')]
 
 
 def clear(ports, settings, shared):

tools/ports/ogg.py

@@ -39,7 +39,7 @@ def create():
     ports.build_port(os.path.join(dest_path, 'src'), final)
     return final
 
-  return [shared.Cache.get(libname, create)]
+  return [shared.Cache.get_lib(libname, create)]
 
 
 def clear(ports, settings, shared):

tools/ports/regal.py

@@ -138,7 +138,7 @@ def create():
     ports.create_lib(final, o_s)
     return final
 
-  return [shared.Cache.get(get_lib_name(settings), create, what='port')]
+  return [shared.Cache.get_lib(get_lib_name(settings), create, what='port')]
 
 
 def clear(ports, settings, shared):

tools/ports/sdl2.py

@@ -86,11 +86,11 @@ def create():
     ports.create_lib(final, o_s)
     return final
 
-  return [shared.Cache.get(libname, create, what='port')]
+  return [shared.Cache.get_lib(libname, create, what='port')]
 
 
 def clear(ports, settings, shared):
-  shared.Cache.erase_file(get_lib_name(settings))
+  shared.Cache.erase_lib(get_lib_name(settings))
 
 
 def process_dependencies(settings):

tools/ports/sdl2_gfx.py

@@ -35,7 +35,7 @@ def create():
     ports.install_headers(source_path, target='SDL2')
     return final
 
-  return [shared.Cache.get(libname, create)]
+  return [shared.Cache.get_lib(libname, create)]
 
 
 def clear(ports, settings, shared):

tools/ports/sdl2_image.py

@@ -55,7 +55,7 @@ def create():
     ports.create_lib(final, o_s)
     return final
 
-  return [shared.Cache.get(libname, create, what='port')]
+  return [shared.Cache.get_lib(libname, create, what='port')]
 
 
 def clear(ports, settings, shared):

tools/ports/sdl2_mixer.py

@@ -76,7 +76,7 @@ def create():
     ports.install_headers(source_path, pattern='SDL_*.h', target='SDL2')
     return final
 
-  return [shared.Cache.get(libname, create, what='port')]
+  return [shared.Cache.get_lib(libname, create, what='port')]
 
 
 def clear(ports, settings, shared):