Avoid writing even sanity.txt if FROZEN_CACHE is enabled (#13316)

These improvements to the `FROZE_CACHE` setting allow for a
pre-populated, read-only cache.  In this scenario the cache
is cannot ever be locked, and that we don't even write
the sanity file.

Users can still force a sanity check with `emcc --check`
but no sanity file will ever be written.

Fixes: #13369

Author: sbc100

tests/test_sanity.py

@@ -390,8 +390,8 @@ def test_em_config_env_var(self):
 
     self.assertContained('hello from emcc with no config file', self.run_js('a.out.js'))
 
-  def erase_cache(self):
-    Cache.erase()
+  def clear_cache(self):
+    self.run_process([EMCC, '--clear-cache'])
     self.assertCacheEmpty()
 
   def assertCacheEmpty(self):
@@ -406,7 +406,7 @@ def test_emcc_caching(self):
     BUILDING_MESSAGE = 'generating system library: %s'
 
     restore_and_set_up()
-    self.erase_cache()
+    self.clear_cache()
 
     # Building a file that *does* need something *should* trigger cache
     # generation, but only the first time
@@ -425,7 +425,7 @@ def test_cache_clearing_manual(self):
     # Manual cache clearing
     restore_and_set_up()
     self.ensure_cache()
-    self.assertTrue(os.path.exists(Cache.dirname))
+    self.assertExists(Cache.dirname)
     output = self.do([EMCC, '--clear-cache'])
     self.assertIn('clearing cache', output)
     self.assertIn(SANITY_MESSAGE, output)
@@ -438,24 +438,24 @@ def test_cache_clearing_auto(self):
     make_fake_clang(self.in_dir('fake', 'bin', 'clang'), EXPECTED_LLVM_VERSION)
     make_fake_llc(self.in_dir('fake', 'bin', 'llc'), 'got wasm32 backend! WebAssembly 32-bit')
     with env_modify({'EM_LLVM_ROOT': self.in_dir('fake', 'bin')}):
-      self.assertTrue(os.path.exists(Cache.dirname))
+      self.assertExists(Cache.dirname)
       output = self.do([EMCC])
       self.assertIn('clearing cache', output)
       self.assertCacheEmpty()
 
   # FROZEN_CACHE prevents cache clears, and prevents building
   def test_FROZEN_CACHE(self):
     restore_and_set_up()
-    self.erase_cache()
+    self.clear_cache()
     self.ensure_cache()
-    self.assertTrue(os.path.exists(Cache.dirname))
+    self.assertExists(Cache.dirname)
     # changing config file should not clear cache
     add_to_config('FROZEN_CACHE = True')
     self.do([EMCC])
-    self.assertTrue(os.path.exists(Cache.dirname))
+    self.assertExists(Cache.dirname)
     # building libraries is disallowed
     output = self.do([EMBUILDER, 'build', 'libemmalloc'])
-    self.assertIn('FROZEN_CACHE disallows building system libs', output)
+    self.assertContained('FROZEN_CACHE is set, but cache file is missing', output)
 
   # Test that if multiple processes attempt to access or build stuff to the
   # cache on demand, that exactly one of the processes will, and the other
@@ -484,9 +484,9 @@ def test_emcc_multiprocess_cache_access(self):
           num_times_libc_was_built += 1
 
     # The cache directory must exist after the build
-    self.assertTrue(os.path.exists(cache_dir_name))
+    self.assertExists(cache_dir_name)
     # The cache directory must contain a built libc
-    self.assertTrue(os.path.exists(os.path.join(cache_dir_name, libname)))
+    self.assertExists(os.path.join(cache_dir_name, libname))
     # Exactly one child process should have triggered libc build!
     self.assertEqual(num_times_libc_was_built, 1)
 
@@ -625,19 +625,19 @@ def test():
 
     print('normal build')
     with env_modify({'EMCC_FORCE_STDLIBS': None}):
-      Cache.erase()
+      self.clear_cache()
       build()
       test()
 
     print('wacky env vars, these should not mess our bootstrapping')
     with env_modify({'EMCC_FORCE_STDLIBS': '1'}):
-      Cache.erase()
+      self.clear_cache()
       build()
       test()
 
   def test_vanilla(self):
     restore_and_set_up()
-    Cache.erase()
+    self.clear_cache()
 
     def make_fake(report):
       with open(config_file, 'a') as f:
@@ -689,10 +689,10 @@ def test_embuilder_force_port(self):
   def test_embuilder_wasm_backend(self):
     restore_and_set_up()
     # the --lto flag makes us build wasm-bc
-    self.do([EMCC, '--clear-cache'])
+    self.clear_cache()
     self.run_process([EMBUILDER, 'build', 'libemmalloc'])
     self.assertExists(os.path.join(config.CACHE, 'sysroot', 'lib', 'wasm32-emscripten'))
-    self.do([EMCC, '--clear-cache'])
+    self.clear_cache()
     self.run_process([EMBUILDER, 'build', 'libemmalloc', '--lto'])
     self.assertExists(os.path.join(config.CACHE, 'sysroot', 'lib', 'wasm32-emscripten', 'lto'))
 

tools/cache.py

@@ -36,6 +36,11 @@ def __init__(self, dirname):
     self.filelock = filelock.FileLock(self.filelock_name)
 
   def acquire_cache_lock(self):
+    if config.FROZEN_CACHE:
+      # Raise an exception here rather than exit_with_error since in practice this
+      # should never happen
+      raise Exception('Attempt to lock the cache but FROZEN_CACHE is set')
+
     if not self.EM_EXCLUSIVE_CACHE_ACCESS and self.acquired_count == 0:
       logger.debug('PID %s acquiring multiprocess file lock to Emscripten cache at %s' % (str(os.getpid()), self.dirname))
       try:
@@ -114,10 +119,11 @@ def erase_lib(self, name):
     self.erase_file(self.get_lib_name(name))
 
   def erase_file(self, shortname):
-    name = os.path.join(self.dirname, shortname)
-    if os.path.exists(name):
-      logger.info('deleting cached file: %s', name)
-      tempfiles.try_delete(name)
+    with self.lock():
+      name = os.path.join(self.dirname, shortname)
+      if os.path.exists(name):
+        logger.info('deleting cached file: %s', name)
+        tempfiles.try_delete(name)
 
   def get_lib(self, libname, *args, **kwargs):
     name = self.get_lib_name(libname)
@@ -133,14 +139,14 @@ def get(self, shortname, creator, what=None, force=False):
     if os.path.exists(cachename) and not force:
       return cachename
 
+    if config.FROZEN_CACHE:
+      # Raise an exception here rather than exit_with_error since in practice this
+      # should never happen
+      raise Exception('FROZEN_CACHE is set, but cache file is missing: %s' % shortname)
+
     with self.lock():
       if os.path.exists(cachename) and not force:
         return cachename
-      # it doesn't exist yet, create it
-      if config.FROZEN_CACHE:
-        # it's ok to build small .txt marker files like "vanilla"
-        if not shortname.endswith('.txt'):
-          raise Exception('FROZEN_CACHE disallows building system libs: %s' % shortname)
       if what is None:
         if shortname.endswith(('.bc', '.so', '.a')):
           what = 'system library'

tools/config.py

@@ -75,7 +75,7 @@ def normalize_config_settings():
   JS_ENGINES = [listify(engine) for engine in JS_ENGINES]
   WASM_ENGINES = [listify(engine) for engine in WASM_ENGINES]
   if not CACHE:
-    if root_is_writable():
+    if FROZEN_CACHE or root_is_writable():
       CACHE = path_from_root('cache')
     else:
       # Use the legacy method of putting the cache in the user's home directory

tools/shared.py

@@ -242,8 +242,21 @@ def generate_sanity():
 
 
 def perform_sanity_checks():
+  # some warning, mostly not fatal checks - do them even if EM_IGNORE_SANITY is on
+  check_node_version()
+  check_llvm_version()
+
+  llvm_ok = check_llvm()
+
+  if os.environ.get('EM_IGNORE_SANITY'):
+    logger.info('EM_IGNORE_SANITY set, ignoring sanity checks')
+    return
+
   logger.info('(Emscripten: Running sanity checks)')
 
+  if not llvm_ok:
+    exit_with_error('failing sanity checks due to previous llvm failure')
+
   with ToolchainProfiler.profile_block('sanity compiler_engine'):
     try:
       run_process(config.NODE_JS + ['-e', 'console.log("hello")'], stdout=PIPE)
@@ -267,11 +280,21 @@ def check_sanity(force=False):
   """
   if not force and os.environ.get('EMCC_SKIP_SANITY_CHECK') == '1':
     return
+
   # We set EMCC_SKIP_SANITY_CHECK so that any subprocesses that we launch will
   # not re-run the tests.
   os.environ['EMCC_SKIP_SANITY_CHECK'] = '1'
+
+  if config.FROZEN_CACHE:
+    if force:
+      perform_sanity_checks()
+    return
+
+  if os.environ.get('EM_IGNORE_SANITY'):
+    perform_sanity_checks()
+    return
+
   with ToolchainProfiler.profile_block('sanity'):
-    check_llvm_version()
     if not config.config_file:
       return # config stored directly in EM_CONFIG => skip sanity checks
     expected = generate_sanity()
@@ -283,14 +306,11 @@ def check_sanity(force=False):
         if sanity_data != expected:
           logger.debug('old sanity: %s' % sanity_data)
           logger.debug('new sanity: %s' % expected)
-          if config.FROZEN_CACHE:
-            logger.info('(Emscripten: config changed, cache may need to be cleared, but FROZEN_CACHE is set)')
-          else:
-            logger.info('(Emscripten: config changed, clearing cache)')
-            Cache.erase()
-            # the check actually failed, so definitely write out the sanity file, to
-            # avoid others later seeing failures too
-            force = False
+          logger.info('(Emscripten: config changed, clearing cache)')
+          Cache.erase()
+          # the check actually failed, so definitely write out the sanity file, to
+          # avoid others later seeing failures too
+          force = False
         else:
           if force:
             logger.debug(f'sanity file up-to-date but check forced: {sanity_file}')
@@ -300,18 +320,6 @@ def check_sanity(force=False):
       else:
         logger.debug(f'sanity file not found: {sanity_file}')
 
-      # some warning, mostly not fatal checks - do them even if EM_IGNORE_SANITY is on
-      check_node_version()
-
-      llvm_ok = check_llvm()
-
-      if os.environ.get('EM_IGNORE_SANITY'):
-        logger.info('EM_IGNORE_SANITY set, ignoring sanity checks')
-        return
-
-      if not llvm_ok:
-        exit_with_error('failing sanity checks due to previous llvm failure')
-
       perform_sanity_checks()
 
       if not force: