Merge pull request #86 from olafurpg/bench

Add benchmark results

Author: olafurpg

README.md

@@ -63,9 +63,9 @@ using SemanticDB as an intermediary representation for LSIF:
   LSIF because compiler plugins does not have access to a project-wide context,
   which is necessary to produce accurate definitions and hovers in multi-module
   projects with external library dependencies.
-- **Performance**: SemanticDB is fast to write and read. The compiler adds low
-  overhead on compilation and the final conversion from SemanticDB to LSIF can
-  be safely parallelized.
+- **Performance**: SemanticDB is fast to write and read. Each compilation unit
+  can be processed independently to keep memory usage low. The final conversion
+  from SemanticDB to LSIF can be safely parallelized.
 - **Cross-language**: SemanticDB has a
   [spec](https://scalameta.org/docs/semanticdb/specification.html) for Java and
   Scala enabling cross-language navigation in hybrid Java/Scala codebases.
@@ -149,3 +149,7 @@ write tests because:
   code (which is always multiline). Modern versions of Java support multiline
   string literals, but they're not supported in Java 8, which is supported by
   lsif-java.
+
+## Benchmarks
+
+See [docs/benchmarks.md] for benchmark results.

build.sbt

@@ -138,3 +138,13 @@ lazy val snapshots = project
   )
   .dependsOn(minimizedScala, unit)
   .enablePlugins(BuildInfoPlugin)
+
+lazy val bench = project
+  .in(file("tests/benchmarks"))
+  .settings(
+    moduleName := "lsif-java-bench",
+    fork.in(run) := true,
+    skip.in(publish) := true
+  )
+  .dependsOn(unit)
+  .enablePlugins(JmhPlugin)

docs/benchmarks.md

@@ -0,0 +1,20 @@
+# Benchmarks results
+
+```
+sbt:root> bench/jmh:run -i 3 -wi 3 -f1 -t1
+...
+[info] Benchmark                           (lib)  Mode  Cnt     Score     Error  Units
+[info] CompileBench.compile                guava    ss   10  2291.036 ± 243.428  ms/op 1x
+[info] CompileBench.compileSemanticdb      guava    ss   10  3444.978 ± 408.569  ms/op 1.5x
+[info] CompileBench.compile            bytebuddy    ss   10  1819.150 ± 191.530  ms/op 1x
+[info] CompileBench.compileSemanticdb  bytebuddy    ss   10  2641.590 ± 203.537  ms/op 1.45x
+```
+
+- Date: Monday 15 Feb 2021.
+- Hardware: MacBook Pro (16-inch, 2019), 2,6 GHz 6-Core Intel Core i7, 32 GB
+  2667 MHz DDR4.
+- Interpretation: enabling the SemanticDB compiler plugin slows down normal
+  compilation by 45-50%.
+- Recommendation: do not enable the SemanticDB compiler plugin during local
+  edit-and-test workflows. The compiler plugin is primarily intended to be
+  enabled in custom CI jobs to upload LSIF indexes.

project/plugins.sbt

@@ -6,6 +6,7 @@ addSbtPlugin("com.thesamet" % "sbt-protoc" % "1.0.0")
 addSbtPlugin("ch.epfl.scala" % "sbt-bloop" % "1.4.6-21-464e4ec4")
 addSbtPlugin("com.sourcegraph" % "sbt-sourcegraph" % "0.1.8")
 addSbtPlugin("com.lightbend.sbt" % "sbt-java-formatter" % "0.6.0")
+addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.0")
 
 // sbt-jdi-tools appears to fix an error related to this message:
 // [error] (plugin / Compile / compileIncremental) java.lang.NoClassDefFoundError: com/sun/tools/javac/code/Symbol

tests/benchmarks/src/main/scala/benchmarks/CompileBench.scala

@@ -0,0 +1,87 @@
+package benchmarks
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.FileSystems
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.PathMatcher
+import java.util.concurrent.TimeUnit
+
+import scala.meta.inputs.Input
+import scala.meta.internal.io.FileIO
+
+import org.openjdk.jmh.annotations._
+import tests.DeleteVisitor
+import tests.Dependencies
+import tests.TestCompiler
+
+@State(Scope.Benchmark)
+class CompileBench {
+
+  var deps: Dependencies = _
+  var tmp: Path = _
+  var compiler: TestCompiler = _
+  var javaPattern: PathMatcher = _
+  @Param(Array("bytebuddy", "guava"))
+  var lib: String = _
+
+  val libs = Map(
+    "bytebuddy" -> "net.bytebuddy:byte-buddy:1.10.20",
+    "guava" -> "com.google.guava:guava:30.1-jre"
+  )
+
+  @Setup()
+  def setup(): Unit = {
+    javaPattern = FileSystems.getDefault.getPathMatcher("glob:**.java")
+    tmp = Files.createTempDirectory("benchmarks")
+    deps = Dependencies.resolveDependencies(List(libs(lib)), Nil)
+    compiler = new TestCompiler(deps.classpathSyntax, List.empty[String], tmp)
+  }
+
+  @TearDown()
+  def teardown(): Unit = {
+    Files.walkFileTree(tmp, new DeleteVisitor)
+  }
+
+  def foreachSource(fn: Seq[Input.VirtualFile] => Int): Long = {
+    var sum = 0L
+    deps
+      .sources
+      .foreach { source =>
+        FileIO.withJarFileSystem(source, create = false, close = true) { root =>
+          val files =
+            FileIO
+              .listAllFilesRecursively(root)
+              .iterator
+              .filter(p => javaPattern.matches(p.toNIO))
+              .toArray
+          val inputs = files.map { source =>
+            val text = FileIO.slurp(source, StandardCharsets.UTF_8)
+            val relativePath = source.toString().stripPrefix("/")
+            Input.VirtualFile(relativePath, text)
+          }
+          sum += fn(inputs)
+        }
+      }
+    sum
+  }
+
+  @Benchmark
+  @BenchmarkMode(Array(Mode.SingleShotTime))
+  @OutputTimeUnit(TimeUnit.MILLISECONDS)
+  def compile(): Long = {
+    foreachSource { inputs =>
+      compiler.compile(inputs).byteCode.length
+    }
+  }
+
+  @Benchmark
+  @BenchmarkMode(Array(Mode.SingleShotTime))
+  @OutputTimeUnit(TimeUnit.MILLISECONDS)
+  def compileSemanticdb(): Long = {
+    foreachSource { inputs =>
+      compiler.compileSemanticdb(inputs).textDocument.getOccurrencesCount
+    }
+  }
+
+}

tests/snapshots/src/main/generated/com/airbnb/epoxy/AsyncEpoxyController.java

@@ -24,7 +24,7 @@
  */
 public abstract class AsyncEpoxyController extends EpoxyController {
 //              ^^^^^^^^^^^^^^^^^^^^ definition com/airbnb/epoxy/AsyncEpoxyController#
-//                                                 ^^^^^^^^^^^^^^^ reference _root_/
+//                                                 ^^^^^^^^^^^^^^^ reference com/airbnb/epoxy/EpoxyController#
 
   /**
    * A new instance that does model building and diffing asynchronously.
@@ -56,6 +56,7 @@ public AsyncEpoxyController(boolean enableAsyncModelBuilding, boolean enableAsyn
 //                                    ^^^^^^^^^^^^^^^^^^^^^^^^ definition local1
 //                                                                      ^^^^^^^^^^^^^^^^^^ definition local2
     super(getHandler(enableAsyncModelBuilding), getHandler(enableAsyncDiffing));
+//  ^^^^^ reference com/airbnb/epoxy/EpoxyController#`<init>`(+1).
 //        ^^^^^^^^^^ reference com/airbnb/epoxy/AsyncEpoxyController#getHandler().
 //                   ^^^^^^^^^^^^^^^^^^^^^^^^ reference local1
 //                                              ^^^^^^^^^^ reference com/airbnb/epoxy/AsyncEpoxyController#getHandler().
@@ -68,7 +69,7 @@ private static Handler getHandler(boolean enableAsync) {
 //                                          ^^^^^^^^^^^ definition local3
     return enableAsync ? getAsyncBackgroundHandler() : MAIN_THREAD_HANDLER;
 //         ^^^^^^^^^^^ reference local3
-//                       ^^^^^^^^^^^^^^^^^^^^^^^^^ reference com/airbnb/epoxy/AsyncEpoxyController#getAsyncBackgroundHandler#
-//                                                     ^^^^^^^^^^^^^^^^^^^ reference _root_/
+//                       ^^^^^^^^^^^^^^^^^^^^^^^^^ reference com/airbnb/epoxy/EpoxyAsyncUtil#getAsyncBackgroundHandler().
+//                                                     ^^^^^^^^^^^^^^^^^^^ reference com/airbnb/epoxy/EpoxyAsyncUtil#MAIN_THREAD_HANDLER.
   }
 }