Share logic for generating compile DB with run_tool.py

BUG=676588
R=thakis@chromium.org

Review-Url: https://codereview.chromium.org/2798233002 .
Cr-Commit-Position: refs/heads/master@{#462582}

docs/clang_tool_refactoring.md

@@ -125,13 +125,6 @@ $gen_targets = $(ninja -C out/gn -t targets all \
 ninja -C out/Debug $gen_targets
 ```
 
-On Windows, generate the compile DB first, and after making any source changes.
-Then omit the `--generate-compdb` in later steps.
-
-```shell
-tools/clang/scripts/generate_win_compdb.py out/Debug
-```
-
 Then run the actual clang tool to generate a list of edits:
 
 ```shell

tools/clang/pylib/clang/compile_db.py

@@ -5,22 +5,93 @@
 
 import json
 import os
+import re
+import shlex
+import sys
 import subprocess
 
 
+_RSP_RE = re.compile(r' (@(.+?\.rsp)) ')
+_debugging = False
+
+
+def _ProcessEntry(entry):
+  """Transforms one entry in the compile database to be clang-tool friendly."""
+  # Escape backslashes to prevent shlex from interpreting them.
+  escaped_command = entry['command'].replace('\\', '\\\\')
+  split_command = shlex.split(escaped_command)
+  # Drop gomacc.exe from the front, if present.
+  if split_command[0].endswith('gomacc.exe'):
+    split_command = split_command[1:]
+  # Insert --driver-mode=cl as the first argument.
+  split_command = split_command[:1] + ['--driver-mode=cl'] + split_command[1:]
+  entry['command'] = ' '.join(split_command)
+
+  # Expand the contents of the response file, if any.
+  # http://llvm.org/bugs/show_bug.cgi?id=21634
+  try:
+    match = _RSP_RE.search(entry['command'])
+    if match:
+      rsp_path = os.path.join(entry['directory'], match.group(2))
+      rsp_contents = file(rsp_path).read()
+      entry['command'] = ''.join([
+          entry['command'][:match.start(1)],
+          rsp_contents,
+          entry['command'][match.end(1):]])
+  except IOError:
+    if _debugging:
+      print 'Couldn\'t read response file for %s' % entry['file']
+
+  return entry
+
+
+def _ProcessCompileDatabaseForWindows(compile_db):
+  """Make the compile db generated by ninja on Windows more clang-tool friendly.
+
+  Args:
+    compile_db: The compile database parsed as a Python dictionary.
+
+  Returns:
+    A postprocessed compile db that clang tooling can use.
+  """
+  if _debugging > 0:
+    print 'Read in %d entries from the compile db' % len(compile_db)
+  compile_db = [_ProcessEntry(e) for e in compile_db]
+  original_length = len(compile_db)
+
+  # Filter out NaCl stuff. The clang tooling chokes on them.
+  # TODO(dcheng): This doesn't appear to do anything anymore, remove?
+  compile_db = [e for e in compile_db if '_nacl.cc.pdb' not in e['command']
+      and '_nacl_win64.cc.pdb' not in e['command']]
+  if _debugging > 0:
+    print 'Filtered out %d entries...' % (original_length - len(compile_db))
+
+  # TODO(dcheng): Also filter out multiple commands for the same file. Not sure
+  # how that happens, but apparently it's an issue on Windows.
+  return compile_db
+
+
 def GenerateWithNinja(path):
   """Generates a compile database using ninja.
 
   Args:
     path: The build directory to generate a compile database for.
   """
-  # TODO(dcheng): Incorporate Windows-specific compile DB munging from
-  # https://codereview.chromium.org/718873004
-  print 'Generating compile database in %s...' % path
-  args = ['ninja', '-C', path, '-t', 'compdb', 'cc', 'cxx', 'objc', 'objcxx']
-  output = subprocess.check_output(args)
-  with file(os.path.join(path, 'compile_commands.json'), 'w') as f:
-    f.write(output)
+  # TODO(dcheng): Ensure that clang is enabled somehow.
+
+  # First, generate the compile database.
+  json_compile_db = subprocess.check_output([
+      'ninja', '-C', path, '-t', 'compdb', 'cc', 'cxx', 'objc', 'objcxx'])
+  compile_db = json.loads(json_compile_db)
+
+  # TODO(dcheng): Ideally this would check target_os... but not sure there's an
+  # easy way to do that, and (for now) cross-compiles don't work without custom
+  # patches anyway.
+  if sys.platform == 'win32':
+    compile_db = _ProcessCompileDatabaseForWindows(compile_db)
+
+  with open(os.path.join(path, 'compile_commands.json'), 'w') as f:
+    f.write(json.dumps(compile_db, indent=2))
 
 
 def Read(path):

tools/clang/scripts/generate_compdb.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+Helper for generating compile DBs for clang tooling. On non-Windows platforms,
+this is pretty straightforward. On Windows, the tool does a bit of extra work to
+integrate the content of response files, force clang tooling to run in clang-cl
+mode, etc.
+"""
+
+import argparse
+import os
+import sys
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+tool_dir = os.path.abspath(os.path.join(script_dir, '../pylib'))
+sys.path.insert(0, tool_dir)
+
+from clang import compile_db
+
+
+def main(argv):
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+      'build_path',
+      nargs='?',
+      help='Path to build directory',
+      default='out/Debug')
+  args = parser.parse_args()
+
+  compile_db.GenerateWithNinja(args.build_path)
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv[1:]))