build: have configure.py create compile_commands.json

compile_commands.json (a.k.a. "compdb",
https://clang.llvm.org/docs/JSONCompilationDatabase.html) is intended
to help stand-alone C-family LSP servers index the codebase as
precisely as possible.

The actively maintained LSP servers with good C++ support are:
- Clangd (https://clangd.llvm.org/)
- CCLS (https://github.com/MaskRay/ccls)

This change causes a successful invocation of configure.py to create a
unified Scylla+Seastar+Abseil compdb for every selected build mode,
and to leave a valid symlink in the source root (if a valid symlink
already exists, it will be left alone).

Signed-off-by: Michael Livshin <michael.livshin@scylladb.com>

Closes scylladb#9558

.gitignore

@@ -28,3 +28,4 @@ test/*/*.reject
 docs/_build
 docs/poetry.lock
 compile_commands.json
+.ccls-cache/

configure.py

@@ -1572,6 +1572,7 @@ def configure_seastar(build_dir, mode, mode_config):
         '-DSeastar_CXX_DIALECT=gnu++20',
         '-DSeastar_API_LEVEL=6',
         '-DSeastar_UNUSED_RESULT_ERROR=ON',
+        '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON',
     ] + distro_extra_cmake_args
 
     if args.stack_guards is not None:
@@ -1638,6 +1639,7 @@ def configure_abseil(build_dir, mode, mode_config):
         '-DCMAKE_C_COMPILER={}'.format(args.cc),
         '-DCMAKE_CXX_COMPILER={}'.format(args.cxx),
         '-DCMAKE_CXX_FLAGS_{}={}'.format(cmake_mode.upper(), abseil_cflags),
+        '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON',
     ] + distro_extra_cmake_args
 
     abseil_cmd = ['cmake', '-G', 'Ninja', real_relpath('abseil', abseil_build_dir)] + abseil_cmake_args
@@ -2188,3 +2190,23 @@ def configure_abseil(build_dir, mode, mode_config):
         ''').format(**globals()))
 
 os.rename(buildfile_tmp, buildfile)
+
+# create compdbs
+compdb = 'compile_commands.json'
+for mode in selected_modes:
+    mode_out = outdir + '/' + mode
+    submodule_compdbs = [mode_out + '/' + sm + '/' + compdb
+                         for sm in ['abseil', 'seastar']]
+    subprocess.run(['/bin/sh', '-c',
+                    ninja + ' -t compdb ' +
+                    '| ./scripts/merge-compdb.py build/' + mode + ' - ' +
+                    ' '.join(submodule_compdbs) +
+                    '> ' + mode_out + '/' + compdb])
+# make sure there is a valid compile_commands.json link in the source root
+if not os.path.exists(compdb):
+    # sort modes by supposed indexing speed
+    for mode in ['dev', 'debug', 'release', 'sanitize']:
+        compdb_target = outdir + '/' + mode + '/' + compdb
+        if os.path.exists(compdb_target):
+            os.symlink(compdb_target, compdb)
+            break

scripts/merge-compdb.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021 ScyllaDB
+#
+
+#
+# This file is part of Scylla.
+#
+# Scylla is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Scylla is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+#
+# Called by configure.py to merge several compdbs into one, so your
+# favourite compatible code indexer can present you with a relevant
+# maximally-complete unified index.
+#
+
+# Usage: merge-compdb.py build/{debug,release,dev,sanitize} [input...]
+#
+# Glue inputs, which must be compdb ("compile_command.json") files,
+# together, filtering out anything that is not indexable or not built
+# under the given prefix, and write the result to stdout.
+#
+# An input is either a file name or "-" for stdin.
+#
+# configure.py calls this with "-" for the ninja-generated scylla
+# compdb and mode-specific seastar & abseil compdbs.
+#
+
+import json
+import logging
+import re
+import sys
+
+prefix = sys.argv[1]
+inputs = sys.argv[2:]
+
+def read_input(fname):
+    try:
+        if fname == '-':
+            f = sys.stdin
+        else:
+            f = open(fname)
+
+        return json.load(f)
+    except Exception as e:
+        logging.error('failed to parse %s: %s', fname, e)
+        return [];
+    finally:
+        if fname != '-':
+            f.close()
+
+def is_indexable(e):
+    return any(e['file'].endswith('.' + suffix) for suffix in ['c', 'C', 'cc', 'cxx'])
+
+# We can only definitely say whether an entry is built under the right
+# prefix when it has an "output" field, so those without are assumed
+# to be OK.  This works for our usage, where only the "-" input (which
+# results from "ninja -t compdb") has 'output' entries _and_ needs to
+# be filtered in the first place.
+def is_relevant(e):
+    return ('output' not in e) or (e['output'].startswith(prefix))
+
+json.dump([e for fname in inputs
+           for e in read_input(fname) if is_indexable(e) and is_relevant(e)],
+          sys.stdout, indent=True)