Sourcemap improvements

emcc.py

@@ -1196,8 +1196,8 @@ def consume_arg_file():
           else:
             settings.SEPARATE_DWARF = True
           settings.GENERATE_DWARF = 1
-        elif requested_level == 'source-map':
-          settings.GENERATE_SOURCE_MAP = 1
+        elif requested_level in ['source-map', 'source-map=inline']:
+          settings.GENERATE_SOURCE_MAP = 1 if requested_level == 'source-map' else 2
           settings.EMIT_NAME_SECTION = 1
           newargs[i] = '-g'
         else:

site/source/docs/tools_reference/settings_reference.rst

@@ -3130,6 +3130,17 @@ This is enabled automatically when using -gsource-map with sanitizers.
 
 Default value: false
 
+.. _source_map_prefixes:
+
+SOURCE_MAP_PREFIXES
+===================
+
+List of path substitutions to apply in the "sources" field of the source map.
+Corresponds to the ``--prefix`` option used in ``tools/wasm-sourcemap.py``.
+Must be used with ``-gsource-map``.
+
+Default value: []
+
 .. _default_to_cxx:
 
 DEFAULT_TO_CXX

src/settings.js

@@ -2045,6 +2045,12 @@ var USE_OFFSET_CONVERTER = false;
 // This is enabled automatically when using -gsource-map with sanitizers.
 var LOAD_SOURCE_MAP = false;
 
+// List of path substitutions to apply in the "sources" field of the source map.
+// Corresponds to the ``--prefix`` option used in ``tools/wasm-sourcemap.py``.
+// Must be used with ``-gsource-map``.
+// [link]
+var SOURCE_MAP_PREFIXES = [];
+
 // Default to c++ mode even when run as ``emcc`` rather then ``emc++``.
 // When this is disabled ``em++`` is required linking C++ programs. Disabling
 // this will match the behaviour of gcc/g++ and clang/clang++.

src/settings_internal.js

@@ -204,7 +204,10 @@ var USE_READY_PROMISE = true;
 // If true, building against Emscripten's wasm heap memory profiler.
 var MEMORYPROFILER = false;
 
-var GENERATE_SOURCE_MAP = false;
+// Set automatically to :
+// - 1 when using `-gsource-map`
+// - 2 when using `gsource-map=inline` (embed sources content in souce map)
+var GENERATE_SOURCE_MAP = 0;
 
 var GENERATE_DWARF = false;
 

test/test_other.py

@@ -50,6 +50,7 @@
 import line_endings
 from tools import webassembly
 from tools.settings import settings
+from tools.system_libs import DETERMINISTIC_PREFIX
 
 scons_path = shutil.which('scons')
 emmake = shared.bat_suffix(path_from_root('emmake'))
@@ -10350,25 +10351,84 @@ def test_check_sourcemapurl_default(self, *args):
     source_mapping_url_content = webassembly.to_leb(len('sourceMappingURL')) + b'sourceMappingURL' + webassembly.to_leb(len('a.wasm.map')) + b'a.wasm.map'
     self.assertIn(source_mapping_url_content, output)
 
-  def test_wasm_sourcemap(self):
-    # The no_main.c will be read (from relative location) due to speficied "-s"
+  @parameterized({
+    '': ([], [], []),
+    'prefix_wildcard': ([], ['--prefix', '=wasm-src://'], []),
+    'prefix_partial': ([], ['--prefix', '/emscripten/=wasm-src:///emscripten/'], []),
+    'sources': (['--sources'], [], ['--load-prefix', '/emscripten/test/other/wasm_sourcemap=.'])
+  })
+  @parameterized({
+    '': ('/',),
+    'basepath': ('/emscripten/test',)
+  })
+  def test_wasm_sourcemap(self, sources, prefix, load_prefix, basepath):
+    # The no_main.c will be read from relative location if necessary (depends
+    # on --sources and --load-prefix options).
     shutil.copy(test_file('other/wasm_sourcemap/no_main.c'), '.')
+    DW_AT_decl_file = '/emscripten/test/other/wasm_sourcemap/no_main.c'
     wasm_map_cmd = [PYTHON, path_from_root('tools/wasm-sourcemap.py'),
-                    '--sources', '--prefix', '=wasm-src://',
-                    '--load-prefix', '/emscripten/test/other/wasm_sourcemap=.',
+                    *sources, *prefix, *load_prefix,
                     '--dwarfdump-output',
                     test_file('other/wasm_sourcemap/foo.wasm.dump'),
                     '-o', 'a.out.wasm.map',
                     test_file('other/wasm_sourcemap/foo.wasm'),
-                    '--basepath=' + os.getcwd()]
+                    '--basepath=' + basepath]
     self.run_process(wasm_map_cmd)
     output = read_file('a.out.wasm.map')
-    # has "sources" entry with file (includes also `--prefix =wasm-src:///` replacement)
-    self.assertIn('wasm-src:///emscripten/test/other/wasm_sourcemap/no_main.c', output)
-    # has "sourcesContent" entry with source code (included with `-s` option)
-    self.assertIn('int foo()', output)
-    # has some entries
-    self.assertRegex(output, r'"mappings":\s*"[A-Za-z0-9+/]')
+    # "sourcesContent" contains source code iff --sources is specified.
+    self.assertIn('int foo()' if sources else '"sourcesContent":[]', output)
+    if prefix: # "sources" contains URL with prefix path substition if provided
+      sources_url = 'wasm-src:///emscripten/test/other/wasm_sourcemap/no_main.c'
+    else: # otherwise a path relative to the given basepath.
+      sources_url = utils.normalize_path(os.path.relpath(DW_AT_decl_file, basepath))
+    self.assertIn(sources_url, output)
+    # "mappings" contains valid Base64 VLQ segments.
+    self.assertRegex(output, r'"mappings":\s*"(?:[A-Za-z0-9+\/]+[,;]?)+"')
+
+  @parameterized({
+    '': ([], 0),
+    'prefix': ([
+      '<cwd>=file:///path/to/src',
+      DETERMINISTIC_PREFIX + '=file:///path/to/emscripten',
+    ], 0),
+    'sources': ([], 1)
+  })
+  def test_emcc_sourcemap_options(self, prefixes, sources):
+    wasm_sourcemap = importlib.import_module('tools.wasm-sourcemap')
+    cwd = os.getcwd()
+    src_file = shutil.copy(test_file('hello_123.c'), cwd)
+    lib_file = DETERMINISTIC_PREFIX + '/system/lib/libc/musl/src/stdio/fflush.c'
+    if prefixes:
+      prefixes = [p.replace('<cwd>', cwd) for p in prefixes]
+    self.set_setting('SOURCE_MAP_PREFIXES', prefixes)
+    args = ['-gsource-map=inline' if sources else '-gsource-map']
+    self.emcc(src_file, args=args, output_filename='test.js')
+    output = read_file('test.wasm.map')
+    # Check source file resolution
+    p = wasm_sourcemap.Prefixes(prefixes, base_path=cwd)
+    self.assertEqual(len(p.prefixes), len(prefixes))
+    src_file_url = p.resolve(utils.normalize_path(src_file))
+    lib_file_url = p.resolve(utils.normalize_path(lib_file))
+    if prefixes:
+      self.assertEqual(src_file_url, 'file:///path/to/src/hello_123.c')
+      self.assertEqual(lib_file_url, 'file:///path/to/emscripten/system/lib/libc/musl/src/stdio/fflush.c')
+    else:
+      self.assertEqual(src_file_url, 'hello_123.c')
+      self.assertEqual(lib_file_url, '/emsdk/emscripten/system/lib/libc/musl/src/stdio/fflush.c')
+    # "sources" contains resolved filepath.
+    self.assertIn(f'"{src_file_url}"', output)
+    self.assertIn(f'"{lib_file_url}"', output)
+    # "sourcesContent" contains source code iff -gsource-map=inline is specified.
+    if sources:
+      p = wasm_sourcemap.Prefixes(prefixes, preserve_deterministic_prefix=False)
+      for filepath in [src_file, lib_file]:
+        resolved_path = p.resolve(utils.normalize_path(filepath))
+        sources_content = json.dumps(read_file(resolved_path))
+        self.assertIn(sources_content, output)
+    else:
+      self.assertIn('"sourcesContent":[]', output)
+    # "mappings" contains valid Base64 VLQ segments.
+    self.assertRegex(output, r'"mappings":\s*"(?:[A-Za-z0-9+\/]+[,;]?)+"')
 
   def test_wasm_sourcemap_dead(self):
     wasm_map_cmd = [PYTHON, path_from_root('tools/wasm-sourcemap.py'),

tools/building.py

@@ -1131,6 +1131,13 @@ def emit_wasm_source_map(wasm_file, map_file, final_wasm):
                    '--dwarfdump=' + LLVM_DWARFDUMP,
                    '-o',  map_file,
                    '--basepath=' + base_path]
+
+  if settings.SOURCE_MAP_PREFIXES:
+    sourcemap_cmd += ['--prefix', *settings.SOURCE_MAP_PREFIXES]
+
+  if settings.GENERATE_SOURCE_MAP == 2:
+    sourcemap_cmd += ['--sources']
+
   check_call(sourcemap_cmd)
 
 

tools/system_libs.py

@@ -44,7 +44,7 @@
 
 # A (fake) deterministic emscripten path to use in __FILE__ macro and debug info
 # to produce reproducible builds across platforms.
-DETERMINISITIC_PREFIX = '/emsdk/emscripten'
+DETERMINISTIC_PREFIX = '/emsdk/emscripten'
 
 
 def files_in_path(path, filenames):
@@ -588,9 +588,9 @@ def get_cflags(self):
 
     source_dir = utils.path_from_root()
     relative_source_dir = os.path.relpath(source_dir, self.build_dir)
-    cflags += [f'-ffile-prefix-map={source_dir}={DETERMINISITIC_PREFIX}',
-               f'-ffile-prefix-map={relative_source_dir}={DETERMINISITIC_PREFIX}',
-               f'-fdebug-compilation-dir={DETERMINISITIC_PREFIX}']
+    cflags += [f'-ffile-prefix-map={source_dir}={DETERMINISTIC_PREFIX}',
+               f'-ffile-prefix-map={relative_source_dir}={DETERMINISTIC_PREFIX}',
+               f'-fdebug-compilation-dir={DETERMINISTIC_PREFIX}']
     return cflags
 
   def get_base_name_prefix(self):

tools/wasm-sourcemap.py

@@ -25,6 +25,10 @@
 sys.path.insert(0, __rootdir__)
 
 from tools import utils
+from tools.system_libs import DETERMINISTIC_PREFIX
+from tools.shared import path_from_root
+
+EMSCRIPTEN_PREFIX = utils.normalize_path(path_from_root())
 
 logger = logging.getLogger('wasm-sourcemap')
 
@@ -46,42 +50,57 @@ def parse_args():
 
 
 class Prefixes:
-  def __init__(self, args):
+  def __init__(self, args, base_path=None, preserve_deterministic_prefix=True):
     prefixes = []
     for p in args:
       if '=' in p:
         prefix, replacement = p.split('=')
         prefixes.append({'prefix': prefix, 'replacement': replacement})
       else:
-        prefixes.append({'prefix': p, 'replacement': None})
+        prefixes.append({'prefix': p, 'replacement': ''})
+    self.base_path = base_path
+    self.preserve_deterministic_prefix = preserve_deterministic_prefix
     self.prefixes = prefixes
     self.cache = {}
 
   def resolve(self, name):
     if name in self.cache:
       return self.cache[name]
 
+    source = name
+    if not self.preserve_deterministic_prefix and name.startswith(DETERMINISTIC_PREFIX):
+      source = EMSCRIPTEN_PREFIX + utils.removeprefix(name, DETERMINISTIC_PREFIX)
+
+    provided = False
     for p in self.prefixes:
-      if name.startswith(p['prefix']):
-        if p['replacement'] is None:
-          result = utils.removeprefix(name, p['prefix'])
-        else:
-          result = p['replacement'] + utils.removeprefix(name, p['prefix'])
+      if source.startswith(p['prefix']):
+        source = p['replacement'] + utils.removeprefix(source, p['prefix'])
+        provided = True
         break
-    self.cache[name] = result
-    return result
+
+    # If prefixes were provided, we use that; otherwise if base_path is set, we
+    # emit a relative path. For files with deterministic prefix, we never use
+    # a relative path, precisely to preserve determinism, and because it would
+    # still point to the wrong location, so we leave the filepath untouched to
+    # let users map it to the proper location using prefix options.
+    if not (source.startswith(DETERMINISTIC_PREFIX) or provided or self.base_path is None):
+      try:
+        source = os.path.relpath(source, self.base_path)
+      except ValueError:
+        source = os.path.abspath(source)
+      source = utils.normalize_path(source)
+
+    self.cache[name] = source
+    return source
 
 
 # SourceMapPrefixes contains resolver for file names that are:
 #  - "sources" is for names that output to source maps JSON
 #  - "load" is for paths that used to load source text
 class SourceMapPrefixes:
-  def __init__(self, sources, load):
-    self.sources = sources
-    self.load = load
-
-  def provided(self):
-    return bool(self.sources.prefixes or self.load.prefixes)
+  def __init__(self, sources, load, base_path):
+    self.sources = Prefixes(sources, base_path=base_path)
+    self.load = Prefixes(load, preserve_deterministic_prefix=False)
 
 
 def encode_vlq(n):
@@ -259,15 +278,20 @@ def read_dwarf_entries(wasm, options):
   return sorted(entries, key=lambda entry: entry['address'])
 
 
-def build_sourcemap(entries, code_section_offset, prefixes, collect_sources, base_path):
+def build_sourcemap(entries, code_section_offset, options):
+  base_path = options.basepath
+  collect_sources = options.sources
+  prefixes = SourceMapPrefixes(options.prefix, options.load_prefix, base_path)
+
   sources = []
-  sources_content = [] if collect_sources else None
+  sources_content = []
   mappings = []
   sources_map = {}
   last_address = 0
   last_source_id = 0
   last_line = 1
   last_column = 1
+
   for entry in entries:
     line = entry['line']
     column = entry['column']
@@ -277,20 +301,11 @@ def build_sourcemap(entries, code_section_offset, prefixes, collect_sources, bas
     # start at least at column 1
     if column == 0:
       column = 1
+
     address = entry['address'] + code_section_offset
-    file_name = entry['file']
-    file_name = utils.normalize_path(file_name)
-    # if prefixes were provided, we use that; otherwise, we emit a relative
-    # path
-    if prefixes.provided():
-      source_name = prefixes.sources.resolve(file_name)
-    else:
-      try:
-        file_name = os.path.relpath(file_name, base_path)
-      except ValueError:
-        file_name = os.path.abspath(file_name)
-      file_name = utils.normalize_path(file_name)
-      source_name = file_name
+    file_name = utils.normalize_path(entry['file'])
+    source_name = prefixes.sources.resolve(file_name)
+
     if source_name not in sources_map:
       source_id = len(sources)
       sources_map[source_name] = source_id
@@ -316,6 +331,7 @@ def build_sourcemap(entries, code_section_offset, prefixes, collect_sources, bas
     last_source_id = source_id
     last_line = line
     last_column = column
+
   return {'version': 3,
           'sources': sources,
           'sourcesContent': sources_content,
@@ -334,10 +350,8 @@ def main():
 
   code_section_offset = get_code_section_offset(wasm)
 
-  prefixes = SourceMapPrefixes(sources=Prefixes(options.prefix), load=Prefixes(options.load_prefix))
-
   logger.debug('Saving to %s' % options.output)
-  map = build_sourcemap(entries, code_section_offset, prefixes, options.sources, options.basepath)
+  map = build_sourcemap(entries, code_section_offset, options)
   with open(options.output, 'w') as outfile:
     json.dump(map, outfile, separators=(',', ':'))