Add support to auto-index Bazel builds with an aspect

.github/workflows/ci.yml

@@ -53,9 +53,9 @@ jobs:
             git clone https://github.com/$REPO.git .repos/$REPO
 
             docker run -v $PWD/.repos/$REPO:/sources -w /sources sourcegraph/scip-java:latest scip-java index
-            file .repos/$REPO/index.scip || (echo "$REPO SCIP index doesn't exist!"; exit 1) 
+            file .repos/$REPO/index.scip || (echo "$REPO SCIP index doesn't exist!"; exit 1)
           }
-          
+
           sudo apt install parallel
           export -f check_repo
 
@@ -76,6 +76,24 @@ jobs:
       - run: du -h index.scip
         working-directory: examples/bazel-example
 
+  bazel_aspect:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: yarn global add @bazel/bazelisk
+      - run: sbt cli/pack
+      - run: echo "$PWD/scip-java/target/pack/bin" >> $GITHUB_PATH
+      - name: Auto-index scip-java codebase
+        run: |
+          scip-java index --build-tool=bazel --bazel-scip-java-binary=$(which scip-java)
+      - run: du -h index.scip
+      - name: Auto-index example/bazel-workspace
+        run: |
+          scip-java index --build-tool=bazel --bazel-scip-java-binary=$(which scip-java)
+        working-directory: examples/bazel-example
+      - run: du -h index.scip
+        working-directory: examples/bazel-example
+
   check:
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -65,3 +65,4 @@ bazel-lsif-java
 VERSION
 
 semanticdb-gradle-plugin/gradle
+aspects/scip_java.bzl

build.sbt

@@ -163,7 +163,24 @@ lazy val javacPlugin = project
       Seq(
         ShadeRule
           .rename(
+            // Don't rename SemanticdbPlugin since the fully-qualified name is
+            // referenced from META-INF/services/com.sun.source.util.Plugin
+            "com.sourcegraph.semanticdb_javac.SemanticdbPlugin" ->
+              "com.sourcegraph.semanticdb_javac.SemanticdbPlugin",
+            // Don't rename PrintJavaVersion because we load it via FQN to
+            // detect the Java of a JVM installation.
+            "com.sourcegraph.semanticdb_javac.PrintJavaVersion" ->
+              "com.sourcegraph.semanticdb_javac.PrintJavaVersion",
+            // Don't rename InjectSemanticdbOptions because we load it via FQN to
+            // process a list of Java compiler options.
+            "com.sourcegraph.semanticdb_javac.InjectSemanticdbOptions" ->
+              "com.sourcegraph.semanticdb_javac.InjectSemanticdbOptions",
             "com.google.**" -> "com.sourcegraph.shaded.com.google.@1",
+            // Shade everything else in the semanticdb-javac compiler plugin in
+            // order to be able to index the plugin code itself. Without this step,
+            // we can't add the plugin to the classpath while compiling the source
+            // code of the plugin itself because it results in cryptic compile errors.
+            "com.sourcegraph.**" -> "com.sourcegraph.shaded.com.sourcegraph.@1",
             "google.**" -> "com.sourcegraph.shaded.google.@1",
             "org.relaxng.**" -> "com.sourcegraph.shaded.relaxng.@1"
           )
@@ -259,7 +276,12 @@ lazy val cli = project
           )
           addJar((gradlePlugin / Compile / assembly).value, "gradle-plugin.jar")
 
-          IO.copy(outs)
+          IO.copy(
+            outs,
+            overwrite = true,
+            preserveLastModified = false,
+            preserveExecutable = true
+          )
           val props = new Properties()
           val propsFile = out.resolve("scip-java.properties").toFile
           val copiedJars = outs.collect { case (_, out) =>

docs/getting-started.md

@@ -4,8 +4,8 @@ title: Getting started
 ---
 
 By following the instructions on this page, you should be able to generate a
-[SCIP](https://github.com/sourcegraph/scip)
-index of your Java codebase using Gradle or Maven. See
+[SCIP](https://github.com/sourcegraph/scip) index of your Java codebase using
+Gradle, Maven, sbt, or Bazel. See
 [Supported build tools](#supported-build-tools) for an overview of other build
 tools that we're planning to support in the future.
 
@@ -328,13 +328,41 @@ projects, with the following caveats:
 
 ### Bazel
 
-Bazel is supported by scip-java, but it requires custom configuration to work
-correctly. The `scip-java index` command does not automatically index Bazel builds.
+There are two approaches to index Bazel/Java codebases: automatic and manual.
 
-The Bazel integration for scip-java is specifically designed to be compatible
-with the Bazel build cache to enable incremental indexing. To achieve this,
-scip-java must be configured in `WORKSPACE` and `BUILD` files. The scip-java
-repository contains an example for how to configure everything.
+Don't hesitate to open an issue in the
+[scip-java repository](https://github.com/sourcegraph/scip-java) if you have any
+questions about using scip-java with Bazel builds.
+
+#### Automatic - `aspect`
+
+Since scip-java v0.8.24, it's possible to automatically index Bazel/Java
+codebases via `scip-java index`.
+
+```sh
+scip-java index "--bazel-scip-java-binary=$(which scip-java)"
+```
+
+When using this approach, indexing happens mostly inside the Bazel action graph,
+benefitting from parallel compilation and the Bazel build cache.
+
+The `--bazel-scip-java-binary` argument is required due to implementation
+details, scip-java runs an [aspect](https://bazel.build/extending/aspects) that
+requires the absolute path to the `scip-java` binary.
+
+> The current solution for automatic indexing step is not yet 100% hermetic and,
+> therefore, relies on `--spawn_strategy=local` under the hood. Depending on
+> your use-case, this might be OK or not. If there is demand for it, it's should
+> be possible to make the indexing fully hermetic and compatible with Bazel's
+> sandbox with some extra work.
+
+#### Manual - `select`
+
+It's possible to index Bazel codebases by integrating scip-java directly into
+the build configuration. To achieve this, scip-java must be configured in
+`WORKSPACE` and `BUILD` files. The scip-java repository contains an example for
+how to configure everything, including how to build scip-java itself from
+source.
 
 - [WORKSPACE](https://github.com/sourcegraph/scip-java/blob/main/examples/bazel-example/WORKSPACE):
   adds the required dependencies to be able to run scip-java itself.
@@ -343,6 +371,7 @@ repository contains an example for how to configure everything.
   scip-java.
 
 Once configured, build the codebase with the SemanticDB compiler plugin.
+
 ```sh
 bazel build //... --@scip_java//semanticdb-javac:enabled=true
 ```
@@ -356,7 +385,7 @@ bazel run @scip_java//scip-semanticdb:bazel -- --sourceroot $PWD
 # The command below works for the `examples/bazel-example` directory in the sourcegraph/scip-java repository.
 ❯ jar tf bazel-bin/src/main/java/example/libexample.jar | grep semanticdb$
 META-INF/semanticdb/src/main/java/example/Example.java.semanticdb
- ```
+```
 
 Finally, run the following commands to upload the SCIP index to Sourcegraph.
 
@@ -372,8 +401,3 @@ src login # validate the token authenticates correctly
 # 3. Upload SCIP index to Sourcegraph
 src code-intel upload # requires index.scip file to exist
 ```
-
-
-Don't hesitate to open an issue in the
-[scip-java repository](https://github.com/sourcegraph/scip-java) if you have any
-questions about using scip-java with Bazel builds.

examples/bazel-example/.gitignore

@@ -1 +1,2 @@
 bazel-bazel-example
+aspects/scip_java.bzl

examples/bazel-example/WORKSPACE

@@ -1,5 +1,6 @@
 # This is an end-to-end example of how to consume scip-java from an external repository.
 workspace(name = "scip_java_example")
+
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
 ##############
@@ -20,11 +21,11 @@ http_archive(
 ##############
 local_repository(
     name = "scip_java",
-    path = "../.."
+    path = "../..",
 )
 
 # Copy and paste this, not the local_repository:
-# 
+#
 # SCIP_JAVA_VERSION="0.8.20"
 # http_archive(
 #   name = "scip_java",
@@ -44,36 +45,47 @@ http_archive(
         "https://github.com/bazelbuild/rules_proto/archive/refs/tags/4.0.0-3.20.0.tar.gz",
     ],
 )
+
 load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
+
 rules_proto_dependencies()
+
 rules_proto_toolchains()
 
 ##############
 # JVM External
 ##############
 # To update this version, copy-paste instructions from https://github.com/bazelbuild/rules_jvm_external/releases
 RULES_JVM_EXTERNAL_TAG = "4.2"
+
 RULES_JVM_EXTERNAL_SHA = "cd1a77b7b02e8e008439ca76fd34f5b07aecb8c752961f9640dea15e9e5ba1ca"
+
 http_archive(
     name = "rules_jvm_external",
-    strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG,
     sha256 = RULES_JVM_EXTERNAL_SHA,
+    strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG,
     url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG,
 )
+
 load("@rules_jvm_external//:repositories.bzl", "rules_jvm_external_deps")
+
 rules_jvm_external_deps()
+
 load("@rules_jvm_external//:setup.bzl", "rules_jvm_external_setup")
+
 rules_jvm_external_setup()
+
 load("@rules_jvm_external//:defs.bzl", "maven_install")
+
 maven_install(
     artifacts = [
-        "com.google.protobuf:protobuf-java:3.15.6", # Required dependency by scip-java.
-        "com.google.protobuf:protobuf-java-util:3.15.6", # Required dependency by scip-java.
+        "com.google.protobuf:protobuf-java:3.15.6",  # Required dependency by scip-java.
+        "com.google.protobuf:protobuf-java-util:3.15.6",  # Required dependency by scip-java.
         # These dependencies are only required for the tests
-        "com.google.guava:guava:31.0-jre", 
+        "com.google.guava:guava:31.0-jre",
         "com.google.auto.value:auto-value:1.5.3",
     ],
     repositories = [
-         "https://repo1.maven.org/maven2",
+        "https://repo1.maven.org/maven2",
     ],
 )

scip-java/src/main/resources/scip-java/scip_java.bzl

@@ -0,0 +1,132 @@
+"""
+Bazel aspect to run scip-java against a Java Bazel codebase.
+
+You can optionally commit this file into your git repository, gitignore it, or
+just delete it. When you run `scip-java index` in a Bazel codebase, this file
+will get re-created and the command will error if the file already exists but with
+different contents.
+
+This aspect is needed for scip-java to inspect the structure of the Bazel build
+and register actions to index all java_library/java_test/java_binary targets.
+The result of running this aspect is that your bazel-bin/ directory will contain
+many *.scip (https://github.com/sourcegraph/scip) and 
+*.semanticdb (https://scalameta.org/docs/semanticdb/specification.html) files.
+These files encode information about which symbols are referenced from which
+locations in your source code.
+
+This aspect only works on Linux when using the `local` spawn strategy because
+the `run_shell` action writes SemanticDB and SCIP files to the provided
+--targetroot argument. It should be possible to avoid this requirement
+in the future if there's a strong desire to make the aspect work with the
+default (sandboxed) spawn strategy.
+
+Use the command below to merge all of these SCIP files into a single index:
+
+    find bazel-bin/ -type f -name '*.scip' | xargs cat > index.scip
+
+Use `src code-intel upload` to upload the unified SCIP file to Sourcegraph:
+
+    npm install -g @sourcegraph/src
+    export SRC_ENDPOINT=SOURCEGRAPH_URL
+    export SRC_ACCESS_TOKEN=TOKEN_VALUE
+    src login # confirm you are correctly authenticated
+    src code-intel upload -file=index.scip
+
+Example command to run this aspect directly:
+
+    bazel build //... --spawn_strategy=local  --aspects path/to/scip_java.bzl%scip_java_aspect --output_groups=scip --define=sourceroot=$(pwd) --define=scip_java_binary=$(which scip-java) --define=java_home=$JAVA_HOME
+
+To learn more about aspects: https://bazel.build/extending/aspects
+"""
+
+def _scip_java(target, ctx):
+    if JavaInfo not in target or not hasattr(ctx.rule.attr, "srcs"):
+        return None
+
+    javac_action = None
+    for a in target.actions:
+        if a.mnemonic == "Javac":
+            javac_action = a
+            break
+
+    if not javac_action:
+        return None
+
+    info = target[JavaInfo]
+    compilation = info.compilation_info
+    annotations = info.annotation_processing
+
+    source_files = []
+    for src in ctx.rule.files.srcs:
+        source_files.append(src.path)
+    if len(source_files) == 0:
+        return None
+
+    classpath = [j.path for j in compilation.compilation_classpath.to_list()]
+    bootclasspath = [j.path for j in compilation.boot_classpath]
+
+    processorpath = []
+    processors = []
+    if annotations and annotations.enabled:
+        processorpath += [j.path for j in annotations.processor_classpath.to_list()]
+        processors = annotations.processor_classnames
+
+    build_config = struct(**{
+        "javaHome": ctx.var["java_home"],
+        "classpath": classpath,
+        "sourceFiles": source_files,
+        "javacOptions": compilation.javac_options,
+        "processors": processors,
+        "processorpath": processorpath,
+        "bootclasspath": bootclasspath,
+        "reportWarningOnEmptyIndex": False,
+    })
+    build_config_path = ctx.actions.declare_file(ctx.label.name + ".scip.json")
+
+    scip_output = ctx.actions.declare_file(ctx.label.name + ".scip")
+    targetroot = ctx.actions.declare_directory(ctx.label.name + ".semanticdb")
+    ctx.actions.write(
+        output = build_config_path,
+        content = build_config.to_json(),
+    )
+
+    deps = [javac_action.inputs, annotations.processor_classpath]
+    ctx.actions.run_shell(
+        command = "\"{}\" index --no-cleanup --index-semanticdb.allow-empty-index --cwd \"{}\" --targetroot {} --scip-config \"{}\" --output \"{}\"".format(
+            ctx.var["scip_java_binary"],
+            ctx.var["sourceroot"],
+            targetroot.path,
+            build_config_path.path,
+            scip_output.path,
+        ),
+        env = {
+            "JAVA_HOME": ctx.var["java_home"],
+            "NO_PROGRESS_BAR": "true",
+        },
+        inputs = depset([build_config_path], transitive = deps),
+        outputs = [scip_output, targetroot],
+    )
+
+    return scip_output
+
+def _scip_java_aspect(target, ctx):
+    scip = _scip_java(target, ctx)
+    if not scip:
+        return struct()
+    return [OutputGroupInfo(scip = [scip])]
+
+scip_java_aspect = aspect(
+    _scip_java_aspect,
+)
+
+def _scip_java_impl(ctx):
+    output = ctx.attr.compilation[OutputGroupInfo]
+    return [
+        OutputGroupInfo(scip = output.scip),
+        DefaultInfo(files = output.scip),
+    ]
+
+scip_java = rule(
+    implementation = _scip_java_impl,
+    attrs = {"compilation": attr.label(aspects = [scip_java_aspect])},
+)

scip-java/src/main/scala/com/sourcegraph/scip_java/Embedded.scala

@@ -112,6 +112,19 @@ object Embedded {
     }
   }
 
+  /**
+   * Returns the string contents of the scip_java.bzl file on disk.
+   */
+  def bazelAspectFile(tmpDir: Path): String = {
+    // We could in theory load the resource straight into a string but it was
+    // easier to copy it to a file and read it from there.
+    val tmpFile = copyFile(tmpDir, "scip-java/scip_java.bzl")
+    val contents =
+      new String(Files.readAllBytes(tmpFile), StandardCharsets.UTF_8)
+    Files.deleteIfExists(tmpFile)
+    contents
+  }
+
   private def copyFile(tmpDir: Path, filename: String): Path = {
     val in = this.getClass.getResourceAsStream(s"/$filename")
     val out = tmpDir.resolve(filename)