Add benchmark results

The benchmark results indicate that the overhead from the compiler
plugin is ~4x compared to regular compilation, but CPU flamegraphs
indicate that this overhead may be related to a JVM mis-configuration
instead of logic inside the plugin instead.

Until we better understand the performance overhead, let's done down
claims in the readme about "low compile overhead". I still think 4x
overhead will be acceptable for many projects given that this job runs
only in CI for LSIF uploads.

Author: olafurpg

README.md

@@ -63,9 +63,9 @@ using SemanticDB as an intermediary representation for LSIF:
   LSIF because compiler plugins does not have access to a project-wide context,
   which is necessary to produce accurate definitions and hovers in multi-module
   projects with external library dependencies.
-- **Performance**: SemanticDB is fast to write and read. The compiler adds low
-  overhead on compilation and the final conversion from SemanticDB to LSIF can
-  be safely parallelized.
+- **Performance**: SemanticDB is fast to write and read. Each compilation unit
+  can be processed independently to keep memory usage low. The final conversion
+  from SemanticDB to LSIF can be safely parallelized.
 - **Cross-language**: SemanticDB has a
   [spec](https://scalameta.org/docs/semanticdb/specification.html) for Java and
   Scala enabling cross-language navigation in hybrid Java/Scala codebases.
@@ -149,3 +149,7 @@ write tests because:
   code (which is always multiline). Modern versions of Java support multiline
   string literals, but they're not supported in Java 8, which is supported by
   lsif-java.
+
+## Benchmarks
+
+See [docs/benchmarks.md] for benchmark results.

build.sbt

@@ -138,3 +138,13 @@ lazy val snapshots = project
   )
   .dependsOn(minimizedScala, unit)
   .enablePlugins(BuildInfoPlugin)
+
+lazy val bench = project
+  .in(file("tests/benchmarks"))
+  .settings(
+    moduleName := "lsif-java-bench",
+    fork.in(run) := true,
+    skip.in(publish) := true
+  )
+  .dependsOn(unit)
+  .enablePlugins(JmhPlugin)

docs/benchmarks.md

@@ -0,0 +1,25 @@
+# Benchmarks results
+
+```
+sbt:root> bench/jmh:run -i 3 -wi 3 -f1 -t1
+...
+[info] Benchmark                           (lib)  Mode  Cnt      Score       Error  Units Overhead
+[info] CompileBench.compile                guava    ss    3  10456.139 ±  9794.634  ms/op 1x
+[info] CompileBench.compileSemanticdb      guava    ss    3  48218.454 ± 29336.227  ms/op 4.6x
+[info] CompileBench.compile            bytebuddy    ss    3   5821.383 ±  9201.910  ms/op 1x
+[info] CompileBench.compileSemanticdb  bytebuddy    ss    3  21521.375 ±  5582.770  ms/op 3.7x
+```
+
+- Date: Monday 15 Feb 2021.
+- Hardware: MacBook Pro (16-inch, 2019), 2,6 GHz 6-Core Intel Core i7, 32 GB
+  2667 MHz DDR4.
+- Interpretation: enabling the SemanticDB compiler plugin slows down normal
+  compilation by ~4x.
+- Recommendation: do not enable the SemanticDB compiler plugin during local
+  edit-and-test workflows. The compiler plugin is primarily intended to be
+  enabled in custom CI jobs to upload LSIF indexes.
+- Note: the CPU flamegraphs below indicate a significant fraction of the
+  performance overhead comes from `jvmtiError 23`, which hints that the overhead
+  is not related to the logic of the compiler plugin itself.
+
+![](https://i.imgur.com/OPldbmD.png).

project/plugins.sbt

@@ -6,6 +6,7 @@ addSbtPlugin("com.thesamet" % "sbt-protoc" % "1.0.0")
 addSbtPlugin("ch.epfl.scala" % "sbt-bloop" % "1.4.6-21-464e4ec4")
 addSbtPlugin("com.sourcegraph" % "sbt-sourcegraph" % "0.1.8")
 addSbtPlugin("com.lightbend.sbt" % "sbt-java-formatter" % "0.6.0")
+addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.0")
 
 // sbt-jdi-tools appears to fix an error related to this message:
 // [error] (plugin / Compile / compileIncremental) java.lang.NoClassDefFoundError: com/sun/tools/javac/code/Symbol

tests/benchmarks/src/main/scala/benchmarks/CompileBench.scala

@@ -0,0 +1,90 @@
+package benchmarks
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.FileSystems
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.PathMatcher
+import java.util.concurrent.TimeUnit
+
+import scala.meta.inputs.Input
+import scala.meta.internal.io.FileIO
+
+import org.openjdk.jmh.annotations._
+import tests.DeleteVisitor
+import tests.Dependencies
+import tests.TestCompiler
+
+@State(Scope.Benchmark)
+class CompileBench {
+
+  var deps: Dependencies = _
+  var tmp: Path = _
+  var compiler: TestCompiler = _
+  var javaPattern: PathMatcher = _
+  @Param(Array("bytebuddy", "guava"))
+  var lib: String = _
+
+  val libs = Map(
+    "bytebuddy" -> "net.bytebuddy:byte-buddy:1.10.20",
+    "guava" -> "com.google.guava:guava:30.1-jre"
+  )
+
+  @Setup()
+  def setup(): Unit = {
+    javaPattern = FileSystems.getDefault.getPathMatcher("glob:**.java")
+    tmp = Files.createTempDirectory("benchmarks")
+    deps = Dependencies.resolveDependencies(List(libs(lib)), Nil)
+    compiler = new TestCompiler(deps.classpathSyntax, List.empty[String], tmp)
+  }
+
+  @TearDown()
+  def teardown(): Unit = {
+    Files.walkFileTree(tmp, new DeleteVisitor)
+  }
+
+  def foreachSource(fn: Input.VirtualFile => Int): Long = {
+    var sum = 0L
+    deps
+      .sources
+      .foreach { source =>
+        FileIO.withJarFileSystem(source, create = false, close = true) { root =>
+          val files =
+            FileIO
+              .listAllFilesRecursively(root)
+              .iterator
+              .filter(p => javaPattern.matches(p.toNIO))
+              .toArray
+          files.foreach { source =>
+            val text = FileIO.slurp(source, StandardCharsets.UTF_8)
+            val relativePath = source.toString().stripPrefix("/")
+            val input = Input.VirtualFile(relativePath, text)
+            sum += fn(input)
+          }
+        }
+      }
+    sum
+  }
+
+  @Benchmark
+  @BenchmarkMode(Array(Mode.SingleShotTime))
+  @OutputTimeUnit(TimeUnit.MILLISECONDS)
+  def compile(): Long = {
+    foreachSource { input =>
+      compiler.compile(input.path, input.text).textDocument.getOccurrencesCount
+    }
+  }
+
+  @Benchmark
+  @BenchmarkMode(Array(Mode.SingleShotTime))
+  @OutputTimeUnit(TimeUnit.MILLISECONDS)
+  def compileSemanticdb(): Long = {
+    foreachSource { input =>
+      compiler
+        .compileSemanticdb(input.path, input.text)
+        .textDocument
+        .getOccurrencesCount
+    }
+  }
+
+}

tests/snapshots/src/main/scala/tests/LibrarySnapshotGenerator.scala

@@ -6,7 +6,6 @@ import java.nio.file.Files
 import java.util.concurrent.atomic.AtomicInteger
 
 import scala.collection.parallel.CollectionConverters._
-import scala.jdk.CollectionConverters._
 
 import scala.meta.internal.io.FileIO
 import scala.meta.io.AbsolutePath
@@ -35,7 +34,7 @@ class LibrarySnapshotGenerator extends SnapshotGenerator {
       println(s"indexing library '$name'")
       val deps = Dependencies
         .resolveDependencies(name :: provided.toList, repos)
-      val options = List("-Xlint:none").asJava
+      val options = List("-Xlint:none")
       val counter = new AtomicInteger()
       val targetroot = Files.createTempDirectory("semanticdb-javac")
 

tests/unit/src/main/scala/tests/CompileResult.scala

@@ -0,0 +1,10 @@
+package tests
+
+import com.sourcegraph.semanticdb_javac.Semanticdb
+
+case class CompileResult(
+    byteCode: Array[Byte],
+    stdout: String,
+    textDocument: Semanticdb.TextDocument,
+    isSuccess: Boolean
+)

tests/unit/src/main/scala/tests/TestCompiler.java

@@ -0,0 +1,93 @@
+package tests
+
+import java.io.StringWriter
+import java.nio.file.Files
+import java.nio.file.Path
+import java.util.Collections.singletonList
+import javax.tools.ToolProvider
+
+import scala.collection.mutable.ListBuffer
+import scala.jdk.CollectionConverters._
+
+import com.sourcegraph.semanticdb_javac.Semanticdb
+
+object TestCompiler {
+  private val PROCESSOR_PATH = System.getProperty("java.class.path")
+}
+
+class TestCompiler(
+    val classpath: String,
+    val options: List[String],
+    val targetroot: Path
+) {
+
+  val sourceroot = Files.createTempDirectory("semanticdb-javac")
+  val compiler = ToolProvider.getSystemJavaCompiler
+  val fileManager =
+    new SimpleFileManager(compiler.getStandardFileManager(null, null, null))
+
+  def this(targetroot: Path) {
+    this(TestCompiler.PROCESSOR_PATH, Nil, targetroot)
+  }
+
+  def compileSemanticdb(
+      relativePath: String,
+      testSource: String
+  ): CompileResult = {
+    compile(
+      relativePath,
+      testSource,
+      List(
+        String.format(
+          "-Xplugin:semanticdb -verbose -text:on -sourceroot:%s -targetroot:%s",
+          sourceroot,
+          targetroot
+        )
+      )
+    )
+  }
+  def compile(
+      relativePath: String,
+      testSource: String,
+      extraArguments: List[String] = Nil
+  ): CompileResult = {
+    val output = new StringWriter
+    val source = sourceroot.resolve(relativePath)
+    val compilationUnits = singletonList(
+      new SimpleSourceFile(source, testSource)
+    )
+    val arguments = ListBuffer.empty[String]
+    arguments += "-processorpath"
+    arguments += TestCompiler.PROCESSOR_PATH
+    arguments += "-classpath"
+    arguments += classpath
+    arguments ++= extraArguments
+    arguments ++= options
+    val task = compiler.getTask(
+      output,
+      fileManager,
+      null,
+      arguments.asJava,
+      null,
+      compilationUnits
+    )
+    val isSuccess = task.call()
+    var bytecode = new Array[Byte](0)
+    if (!fileManager.compiled.isEmpty)
+      bytecode = fileManager.compiled.iterator.next.getCompiledBinaries
+    var textDocument = Semanticdb.TextDocument.newBuilder.build
+    val outputPath = targetroot
+      .resolve("META-INF")
+      .resolve("semanticdb")
+      .resolve(relativePath + ".semanticdb")
+    if (Files.isRegularFile(outputPath)) {
+      val textDocuments = Semanticdb
+        .TextDocuments
+        .parseFrom(Files.readAllBytes(outputPath))
+      if (textDocuments.getDocumentsCount > 0)
+        textDocument = textDocuments.getDocuments(0)
+    }
+    val stdout = output.toString
+    CompileResult(bytecode, stdout, textDocument, isSuccess)
+  }
+}