Avoid writing even sanity.txt if FROZEN_CACHE is enabled

These improvements to the FROZE_CACHE setting allow for a
pre-populated read-only cache.  In thie senario the cache
is cannot ever be locked, and that we don't even write
the sanity file.

Users can still forece a sanity check with `emcc --check`
but no sanity file will ever be written.

Author: sbc100

tests/test_sanity.py

@@ -399,8 +399,8 @@ def test_em_config_env_var(self):
 
     self.assertContained('hello from emcc with no config file', self.run_js('a.out.js'))
 
-  def erase_cache(self):
-    Cache.erase()
+  def clear_cache(self):
+    self.run_process([EMCC, '--clear-cache'])
     self.assertCacheEmpty()
 
   def assertCacheEmpty(self):
@@ -415,7 +415,7 @@ def test_emcc_caching(self):
     BUILDING_MESSAGE = 'generating system library: %s'
 
     restore_and_set_up()
-    self.erase_cache()
+    self.clear_cache()
 
     # Building a file that *does* need something *should* trigger cache
     # generation, but only the first time
@@ -455,7 +455,7 @@ def test_cache_clearing_auto(self):
   # FROZEN_CACHE prevents cache clears, and prevents building
   def test_FROZEN_CACHE(self):
     restore_and_set_up()
-    self.erase_cache()
+    self.clear_cache()
     self.ensure_cache()
     self.assertTrue(os.path.exists(Cache.dirname))
     # changing config file should not clear cache
@@ -634,19 +634,19 @@ def test():
 
     print('normal build')
     with env_modify({'EMCC_FORCE_STDLIBS': None}):
-      Cache.erase()
+      self.clear_cache()
       build()
       test()
 
     print('wacky env vars, these should not mess our bootstrapping')
     with env_modify({'EMCC_FORCE_STDLIBS': '1'}):
-      Cache.erase()
+      self.clear_cache()
       build()
       test()
 
   def test_vanilla(self):
     restore_and_set_up()
-    Cache.erase()
+    self.clear_cache()
 
     def make_fake(report):
       with open(config_file, 'a') as f:
@@ -690,10 +690,10 @@ def test_embuilder_force(self):
   def test_embuilder_wasm_backend(self):
     restore_and_set_up()
     # the --lto flag makes us build wasm-bc
-    self.do([EMCC, '--clear-cache'])
+    self.clear_cache()
     self.run_process([EMBUILDER, 'build', 'libemmalloc'])
     self.assertExists(os.path.join(config.CACHE, 'sysroot', 'lib', 'wasm32-emscripten'))
-    self.do([EMCC, '--clear-cache'])
+    self.clear_cache()
     self.run_process([EMBUILDER, 'build', 'libemmalloc', '--lto'])
     self.assertExists(os.path.join(config.CACHE, 'sysroot', 'lib', 'wasm32-emscripten', 'lto'))
 

tools/cache.py

@@ -37,6 +37,11 @@ def __init__(self, dirname):
     self.filelock = filelock.FileLock(self.filelock_name)
 
   def acquire_cache_lock(self):
+    if config.FROZEN_CACHE:
+      # Raise an exception here rather than exit_with_error since in practice this
+      # should never happen
+      raise Exception('Attempt to lock the cache but FROZEN_CACHE is set')
+
     if not self.EM_EXCLUSIVE_CACHE_ACCESS and self.acquired_count == 0:
       logger.debug('PID %s acquiring multiprocess file lock to Emscripten cache at %s' % (str(os.getpid()), self.dirname))
       try:
@@ -115,10 +120,11 @@ def erase_lib(self, name):
     self.erase_file(self.get_lib_name(name))
 
   def erase_file(self, shortname):
-    name = os.path.join(self.dirname, shortname)
-    if os.path.exists(name):
-      logging.info('Cache: deleting cached file: %s', name)
-      tempfiles.try_delete(name)
+    with self.lock():
+      name = os.path.join(self.dirname, shortname)
+      if os.path.exists(name):
+        logging.info('Cache: deleting cached file: %s', name)
+        tempfiles.try_delete(name)
 
   def get_lib(self, libname, *args, **kwargs):
     name = self.get_lib_name(libname)
@@ -134,14 +140,14 @@ def get(self, shortname, creator, what=None, force=False):
     if os.path.exists(cachename) and not force:
       return cachename
 
+    if config.FROZEN_CACHE:
+      # Raise an exception here rather than exit_with_error since in practice this
+      # should never happen
+      raise Exception('FROZEN_CACHE is set, but cache file is missing: %s' % shortname)
+
     with self.lock():
       if os.path.exists(cachename) and not force:
         return cachename
-      # it doesn't exist yet, create it
-      if config.FROZEN_CACHE:
-        # it's ok to build small .txt marker files like "vanilla"
-        if not shortname.endswith('.txt'):
-          raise Exception('FROZEN_CACHE disallows building system libs: %s' % shortname)
       if what is None:
         if shortname.endswith(('.bc', '.so', '.a')):
           what = 'system library'

tools/config.py

@@ -75,7 +75,7 @@ def normalize_config_settings():
   JS_ENGINES = [listify(engine) for engine in JS_ENGINES]
   WASM_ENGINES = [listify(engine) for engine in WASM_ENGINES]
   if not CACHE:
-    if root_is_writable():
+    if FROZEN_CACHE or root_is_writable():
       CACHE = path_from_root('cache')
     else:
       # Use the legacy method of putting the cache in the user's home directory

tools/shared.py

@@ -239,8 +239,21 @@ def generate_sanity():
 
 
 def perform_sanity_checks():
+  # some warning, mostly not fatal checks - do them even if EM_IGNORE_SANITY is on
+  check_node_version()
+  check_llvm_version()
+
+  llvm_ok = check_llvm()
+
+  if os.environ.get('EM_IGNORE_SANITY'):
+    logger.info('EM_IGNORE_SANITY set, ignoring sanity checks')
+    return
+
   logger.info('(Emscripten: Running sanity checks)')
 
+  if not llvm_ok:
+    exit_with_error('failing sanity checks due to previous llvm failure')
+
   with ToolchainProfiler.profile_block('sanity compiler_engine'):
     try:
       run_process(config.NODE_JS + ['-e', 'console.log("hello")'], stdout=PIPE)
@@ -264,11 +277,21 @@ def check_sanity(force=False):
   """
   if not force and os.environ.get('EMCC_SKIP_SANITY_CHECK') == '1':
     return
+
   # We set EMCC_SKIP_SANITY_CHECK so that any subprocesses that we launch will
   # not re-run the tests.
   os.environ['EMCC_SKIP_SANITY_CHECK'] = '1'
+
+  if config.FROZEN_CACHE:
+    if force:
+      perform_sanity_checks()
+    return
+
+  if os.environ.get('EM_IGNORE_SANITY'):
+    perform_sanity_checks()
+    return
+
   with ToolchainProfiler.profile_block('sanity'):
-    check_llvm_version()
     if not config.config_file:
       return # config stored directly in EM_CONFIG => skip sanity checks
     expected = generate_sanity()
@@ -280,14 +303,11 @@ def check_sanity(force=False):
         if sanity_data != expected:
           logger.info('old sanity: %s' % sanity_data)
           logger.info('new sanity: %s' % expected)
-          if config.FROZEN_CACHE:
-            logger.info('(Emscripten: config changed, cache may need to be cleared, but FROZEN_CACHE is set)')
-          else:
-            logger.info('(Emscripten: config changed, clearing cache)')
-            Cache.erase()
-            # the check actually failed, so definitely write out the sanity file, to
-            # avoid others later seeing failures too
-            force = False
+          logger.info('(Emscripten: config changed, clearing cache)')
+          Cache.erase()
+          # the check actually failed, so definitely write out the sanity file, to
+          # avoid others later seeing failures too
+          force = False
         else:
           if force:
             logger.debug(f'sanity file up-to-date but check forced: {sanity_file}')
@@ -297,18 +317,6 @@ def check_sanity(force=False):
       else:
         logger.debug(f'sanity file not found: {sanity_file}')
 
-      # some warning, mostly not fatal checks - do them even if EM_IGNORE_SANITY is on
-      check_node_version()
-
-      llvm_ok = check_llvm()
-
-      if os.environ.get('EM_IGNORE_SANITY'):
-        logger.info('EM_IGNORE_SANITY set, ignoring sanity checks')
-        return
-
-      if not llvm_ok:
-        exit_with_error('failing sanity checks due to previous llvm failure')
-
       perform_sanity_checks()
 
       if not force: