Add support for generating a clangd compilation database (#823)

This will help with IDE integration of the C++ part of the code.

Ref: https://chromium.googlesource.com/chromium/src/+/refs/heads/main/docs/clangd.md

.clangd

@@ -0,0 +1,2 @@
+CompileFlags:
+  Remove: -imsvc*

.gitignore

@@ -3,4 +3,6 @@
 
 /.vscode/
 /.idea/
+/.cache
 /target/
+/compile_commands.json

README.md

@@ -122,6 +122,28 @@ Arguments can be passed to `gn` by setting the `$GN_ARGS` environmental variable
 Env vars used in when building from source: `SCCACHE`, `CCACHE`, `GN`, `NINJA`,
 `CLANG_BASE_PATH`, `GN_ARGS`
 
+## C++ IDE integration
+
+`rusty_v8` supports IDE integration for the C++ bindings through the use of the
+`clangd` language server, bringing features such as diagnostics, code completion
+and code navigations to your editor. [See the instructions for how to set it up
+with your favorite editor.](https://clangd.llvm.org/installation.html#editor-plugins)
+
+Before you can use `clangd` with `rusty_v8`, you must first generate the
+compilation database:
+
+```sh
+V8_FROM_SOURCE=1 GENERATE_COMPDB= cargo build
+```
+
+This will write the `clang` compilation database as the `compile_commands.json`
+file at the root of the project repository. You can pass a path to the
+`GENERATE_COMPDB` environment variable to change the location where the
+compilation database will be written.
+
+You must pass the `GENERATE_COMPDB` environment variable to regenerate the
+compilation database, it will not be regenerated automatically.
+
 ## FAQ
 
 **Building V8 takes over 30 minutes, this is too slow for me to use this crate.

build.rs

@@ -22,6 +22,7 @@ fn main() {
     "CLANG_BASE_PATH",
     "DENO_TRYBUILD",
     "DOCS_RS",
+    "GENERATE_COMPDB",
     "GN",
     "GN_ARGS",
     "HOST",
@@ -530,6 +531,37 @@ fn ninja(gn_out_dir: &Path, maybe_env: Option<NinjaEnv>) -> Command {
   cmd
 }
 
+fn generate_compdb(
+  gn_out_dir: &Path,
+  target: &str,
+  output_path: Option<&Path>,
+) {
+  let mut cmd = Command::new("python");
+  cmd.arg("tools/generate_compdb.py");
+  cmd.arg("-p");
+  cmd.arg(&gn_out_dir);
+  cmd.arg(target);
+  cmd.arg("-o");
+  cmd.arg(output_path.unwrap_or_else(|| Path::new("compile_commands.json")));
+  cmd.envs(env::vars());
+  cmd.stdout(Stdio::inherit());
+  cmd.stderr(Stdio::inherit());
+
+  if let Ok(ninja_path) = env::var("NINJA") {
+    let ninja_folder = Path::new(&ninja_path).parent().unwrap();
+    // Add `ninja_folder` to the PATH envvar.
+    let original_path = env::var_os("PATH").unwrap();
+    let new_path = env::join_paths(
+      env::split_paths(&original_path)
+        .chain(std::iter::once(ninja_folder.to_owned())),
+    )
+    .unwrap();
+    cmd.env("PATH", new_path);
+  }
+
+  run(&mut cmd, "python");
+}
+
 pub type GnArgs = Vec<String>;
 
 pub fn maybe_gen(manifest_dir: &str, gn_args: GnArgs) -> PathBuf {
@@ -569,6 +601,16 @@ pub fn build(target: &str, maybe_env: Option<NinjaEnv>) {
   cmd.arg(target);
   run(&mut cmd, "ninja");
 
+  if let Some(compdb_env) = std::env::var_os("GENERATE_COMPDB") {
+    // Only use compdb_path if it's not empty.
+    let compdb_path = if !compdb_env.is_empty() {
+      Some(Path::new(&compdb_env))
+    } else {
+      None
+    };
+    generate_compdb(&gn_out_dir, target, compdb_path);
+  }
+
   rerun_if_changed(&gn_out_dir, maybe_env, target);
 
   // TODO This is not sufficent. We need to use "gn desc" to query the target

tools/generate_compdb.py

@@ -0,0 +1,193 @@
+#!/usr/bin/env python
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This file is a combination of tools/clang/scripts/generate_compdb.py and
+# tools/clang/pylib/clang/compile_db.py from the Chromium source code.
+# They are modified to use the ninja executable in PATH, rather than finding
+# the binary in the Chromium directory structure.
+#
+# Delete when tools/clang is updated to include this commit:
+# https://chromium.googlesource.com/chromium/src/tools/clang.git/+/d324a17c34dba948e42565378bcdfdac919e62c2
+
+"""
+Helper for generating compile DBs for clang tooling. On non-Windows platforms,
+this is pretty straightforward. On Windows, the tool does a bit of extra work to
+integrate the content of response files, force clang tooling to run in clang-cl
+mode, etc.
+"""
+
+import argparse
+import json
+import os
+import re
+import sys
+import subprocess
+
+
+_RSP_RE = re.compile(r' (@(.+?\.rsp)) ')
+_CMD_LINE_RE = re.compile(
+    r'^(?P<gomacc>.*gomacc(\.exe)?"?\s+)?(?P<clang>\S*clang\S*)\s+(?P<args>.*)$'
+)
+_debugging = False
+
+
+def _IsTargettingWindows(target_os):
+    if target_os is not None:
+        # Available choices are based on: gn help target_os
+        assert target_os in [
+            'android', 'chromeos', 'ios', 'linux', 'nacl', 'mac', 'win'
+        ]
+        return target_os == 'win'
+    return sys.platform == 'win32'
+
+
+def _ProcessCommand(command, target_os):
+    """Removes gomacc(.exe). On Windows inserts --driver-mode=cl as the first arg.
+
+    Note that we deliberately don't use shlex.split here, because it doesn't work
+    predictably for Windows commands (specifically, it doesn't parse args the same
+    way that Clang does on Windows).
+
+    Instead, we just use a regex, with the simplifying assumption that the path to
+    clang-cl.exe contains no spaces.
+    """
+    # If the driver mode is not already set then define it. Driver mode is
+    # automatically included in the compile db by clang starting with release
+    # 9.0.0.
+    driver_mode = ''
+    # Only specify for Windows. Other platforms do fine without it.
+    if _IsTargettingWindows(target_os) and '--driver-mode' not in command:
+        driver_mode = '--driver-mode=cl'
+
+    match = _CMD_LINE_RE.search(command)
+    if match:
+        match_dict = match.groupdict()
+        command = ' '.join(
+            [match_dict['clang'], driver_mode, match_dict['args']])
+    elif _debugging:
+        print('Compile command didn\'t match expected regex!')
+        print('Command:', command)
+        print('Regex:', _CMD_LINE_RE.pattern)
+
+    # Remove some blocklisted arguments. These are VisualStudio specific arguments
+    # not recognized or used by clangd. They only suppress or activate graphical
+    # output anyway.
+    blocklisted_arguments = ['/nologo', '/showIncludes']
+    command_parts = filter(lambda arg: arg not in blocklisted_arguments,
+                           command.split())
+
+    return " ".join(command_parts)
+
+
+def _ProcessEntry(entry, target_os):
+    """Transforms one entry in a Windows compile db to be clang-tool friendly."""
+    entry['command'] = _ProcessCommand(entry['command'], target_os)
+
+    # Expand the contents of the response file, if any.
+    # http://llvm.org/bugs/show_bug.cgi?id=21634
+    try:
+        match = _RSP_RE.search(entry['command'])
+        if match:
+            rsp_path = os.path.join(entry['directory'], match.group(2))
+            rsp_contents = open(rsp_path).read()
+            entry['command'] = ''.join([
+                entry['command'][:match.start(1)],
+                rsp_contents,
+                entry['command'][match.end(1):]])
+    except IOError:
+        if _debugging:
+            print('Couldn\'t read response file for %s' % entry['file'])
+
+    return entry
+
+
+def ProcessCompileDatabaseIfNeeded(compile_db, target_os=None):
+    """Make the compile db generated by ninja on Windows more clang-tool friendly.
+
+    Args:
+      compile_db: The compile database parsed as a Python dictionary.
+
+    Returns:
+      A postprocessed compile db that clang tooling can use.
+    """
+    compile_db = [_ProcessEntry(e, target_os) for e in compile_db]
+
+    if not _IsTargettingWindows(target_os):
+        return compile_db
+
+    if _debugging:
+        print('Read in %d entries from the compile db' % len(compile_db))
+    original_length = len(compile_db)
+
+    # Filter out NaCl stuff. The clang tooling chokes on them.
+    # TODO(dcheng): This doesn't appear to do anything anymore, remove?
+    compile_db = [e for e in compile_db if '_nacl.cc.pdb' not in e['command']
+                  and '_nacl_win64.cc.pdb' not in e['command']]
+    if _debugging:
+        print('Filtered out %d entries...' %
+              (original_length - len(compile_db)))
+
+    # TODO(dcheng): Also filter out multiple commands for the same file. Not sure
+    # how that happens, but apparently it's an issue on Windows.
+    return compile_db
+
+
+def GetNinjaExecutable():
+    return 'ninja.exe' if sys.platform == 'win32' else 'ninja'
+
+
+# FIXME: This really should be a build target, rather than generated at runtime.
+def GenerateWithNinja(path, targets=[]):
+    """Generates a compile database using ninja.
+
+    Args:
+      path: The build directory to generate a compile database for.
+      targets: Additional targets to pass to ninja.
+
+    Returns:
+      List of the contents of the compile database.
+    """
+    # TODO(dcheng): Ensure that clang is enabled somehow.
+
+    # First, generate the compile database.
+    json_compile_db = subprocess.check_output(
+        [GetNinjaExecutable(), '-C', path] + targets +
+        ['-t', 'compdb', 'cc', 'cxx', 'objc', 'objcxx'])
+    return json.loads(json_compile_db)
+
+
+def main(argv):
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '-p',
+        required=True,
+        help='Path to build directory')
+    parser.add_argument(
+        'targets',
+        nargs='*',
+        help='Additional targets to pass to ninja')
+    parser.add_argument(
+        '--target_os',
+        choices=['android', 'chromeos', 'ios', 'linux', 'nacl', 'mac', 'win'],
+        help='Target OS - see `gn help target_os`. Set to "win" when ' +
+        'cross-compiling Windows from Linux or another host')
+    parser.add_argument(
+        '-o',
+        help='File to write the compilation database to. Defaults to stdout')
+
+    args = parser.parse_args()
+
+    compdb_text = json.dumps(ProcessCompileDatabaseIfNeeded(
+        GenerateWithNinja(args.p, args.targets), args.target_os),
+        indent=2)
+    if args.o is None:
+        print(compdb_text)
+    else:
+        with open(args.o, 'w') as f:
+            f.write(compdb_text)
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))