Add sbt plugin to upload test reports to Google Cloud

This commit adds a new sbt-munit plugin that registers a new test
listener that uploads JSON test reports to Google Cloud Storage (when
provided the correct configuration). The end goal is to have a way for
projects to collect valuable data about their testing infrastructure to
answer questions like:

* which test cases are most flaky?
* which tests are slow/fast?
* which tests never fail?
* which tests fail frequently?

.github/workflows/ci.yml

@@ -14,7 +14,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: olafurpg/setup-scala@v2
-      - run: sbt +testsJVM/test
+      - run: sbt +testsJVM/test plugin/test
   js:
     strategy:
       fail-fast: false

.gitignore

@@ -48,3 +48,4 @@ website/i18n/*
 project/metals.sbt
 out/
 *.hnir
+test-report.json

build.sbt

@@ -148,6 +148,19 @@ lazy val munitJVM = munit.jvm
 lazy val munitJS = munit.js
 lazy val munitNative = munit.native
 
+lazy val plugin = project
+  .in(file("munit-sbt"))
+  .settings(
+    sharedSettings,
+    moduleName := "sbt-munit",
+    sbtPlugin := true,
+    scalaVersion := scala212,
+    crossScalaVersions := List(scala212),
+    libraryDependencies ++= List(
+      "com.google.cloud" % "google-cloud-storage" % "1.103.0"
+    )
+  )
+
 lazy val tests = crossProject(JSPlatform, JVMPlatform, NativePlatform)
   .dependsOn(munit)
   .enablePlugins(BuildInfoPlugin)

munit-sbt/src/main/scala/munit/MUnitGcpListener.scala

@@ -0,0 +1,52 @@
+package munit
+
+import com.google.cloud.storage.StorageOptions
+import com.google.cloud.storage.BucketInfo
+import com.google.gson.Gson
+import java.nio.charset.StandardCharsets
+import com.google.cloud.storage.Bucket.BlobTargetOption
+import com.google.cloud.storage.Storage.PredefinedAcl
+import com.google.cloud.storage.StorageException
+import sbt.util.Logger
+
+class MUnitGcpListener(
+    val bucketName: String = "munit-test-reports",
+    val maxRetries: Int = 100,
+    logger: Logger = sbt.ConsoleLogger()
+) extends MUnitReportListener {
+  private lazy val storage = StorageOptions.getDefaultInstance().getService()
+  private lazy val bucket = Option(storage.get(bucketName)).getOrElse {
+    storage.create(BucketInfo.of(bucketName))
+  }
+  def onReport(report: MUnitTestReport.TestReport): Unit = synchronized {
+    val suffixes = Stream
+      .from(0)
+      .map {
+        case 0 => ".json"
+        case n => s"-$n.json"
+      }
+      .take(maxRetries)
+    val bytes = new Gson().toJson(report).getBytes(StandardCharsets.UTF_8)
+    val success = suffixes.find { suffix =>
+      val name = s"${report.repository}/${report.runId}$suffix"
+      try {
+        val blob = bucket.create(
+          name,
+          bytes,
+          BlobTargetOption.predefinedAcl(PredefinedAcl.PUBLIC_READ),
+          BlobTargetOption.doesNotExist()
+        )
+        logger.info(
+          s"uploaded test report: gs://${blob.getBucket()}/${blob.getBlobId().getName()}"
+        )
+        true
+      } catch {
+        case _: StorageException =>
+          false
+      }
+    }
+    if (success.isEmpty) {
+      logger.error(s"warn: failed to upload report after $maxRetries retries.")
+    }
+  }
+}

munit-sbt/src/main/scala/munit/MUnitLocalListener.scala

@@ -0,0 +1,28 @@
+package munit
+
+import java.nio.file.Path
+import java.nio.file.Paths
+import munit.MUnitTestReport.TestReport
+import sbt.util.Logger
+import com.google.gson.Gson
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.StandardOpenOption
+
+class MUnitLocalListener(
+    path: Path =
+      Paths.get(".").toAbsolutePath().normalize().resolve("test-report.json"),
+    maxRetries: Int = 100,
+    logger: Logger = sbt.ConsoleLogger()
+) extends MUnitReportListener {
+  def onReport(report: TestReport): Unit = {
+    val bytes = new Gson().toJson(report).getBytes(StandardCharsets.UTF_8)
+    Files.write(
+      path,
+      bytes,
+      StandardOpenOption.CREATE,
+      StandardOpenOption.TRUNCATE_EXISTING
+    )
+    logger.info(s"wrote test report: $path")
+  }
+}

munit-sbt/src/main/scala/munit/MUnitPlugin.scala

@@ -0,0 +1,63 @@
+package munit
+
+import sbt._
+import sbt.Keys._
+import sbt.plugins._
+import java.nio.file.Paths
+import java.nio.file.Files
+import java.{util => ju}
+
+object MUnitPlugin extends AutoPlugin {
+  override def trigger = allRequirements
+  override def requires = JvmPlugin
+  object autoImport {
+    val munitRepository: SettingKey[Option[String]] =
+      settingKey[Option[String]](
+        "The repository of this project, for example GitHub URL."
+      )
+    val munitRunId: SettingKey[Option[String]] =
+      settingKey[Option[String]]("Unique identifier for this test run.")
+    val munitReportListener: SettingKey[Option[MUnitReportListener]] =
+      settingKey[Option[MUnitReportListener]](
+        "The listener to handle reports."
+      )
+  }
+  import autoImport._
+
+  override val globalSettings: List[Setting[_ <: Option[Object]]] = List(
+    munitReportListener := {
+      for {
+        credentials <- Option(System.getenv("GOOGLE_APPLICATION_CREDENTIALS"))
+        path = Paths.get(credentials).toAbsolutePath()
+        if Files.exists(path) || {
+          Option(System.getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON")) match {
+            case Some(base64) =>
+              val json = ju.Base64.getDecoder().decode(base64)
+              Files.createDirectories(path.getParent())
+              Files.write(path, json)
+              true
+            case None => false
+          }
+        }
+      } yield new MUnitGcpListener()
+    },
+    munitRepository := Option(System.getenv("GITHUB_REPOSITORY")),
+    munitRunId := Option(System.getenv("GITHUB_ACTION"))
+  )
+
+  override val projectSettings: Seq[Def.Setting[_]] = List(
+    testListeners ++= {
+      for {
+        runId <- munitRunId.value.toList
+        repository <- munitRepository.value.toList
+        listener <- munitReportListener.value.toList
+      } yield new MUnitTestsListener(
+        listener,
+        repository,
+        runId,
+        scalaVersion.value,
+        thisProject.value.id
+      )
+    }
+  )
+}

munit-sbt/src/main/scala/munit/MUnitReportListener.scala

@@ -0,0 +1,7 @@
+package munit
+
+import munit.MUnitTestReport.TestReport
+
+trait MUnitReportListener {
+  def onReport(report: TestReport): Unit
+}

munit-sbt/src/main/scala/munit/MUnitTestReport.scala

@@ -0,0 +1,34 @@
+package munit
+
+object MUnitTestReport {
+  case class TestReport(
+      repository: String,
+      runId: String,
+      timestamp: String,
+      scalaVersion: String,
+      projectName: String,
+      javaVersion: String,
+      osName: String,
+      groups: Array[TestGroup]
+  )
+  case class TestGroup(
+      name: String,
+      result: String,
+      events: Array[TestGroupEvent]
+  )
+  case class TestGroupEvent(
+      status: String,
+      name: String,
+      // NOTE(olafur): this field should be typed as Double to match JSON types
+      // but then all numbers get formatted as `2.0` with a redundant `.0`
+      // suffix.
+      duration: Long,
+      exception: TestException
+  )
+  case class TestException(
+      className: String,
+      message: String,
+      stack: Array[String],
+      cause: TestException
+  )
+}

munit-sbt/src/main/scala/munit/MUnitTestsListener.scala

@@ -0,0 +1,143 @@
+package munit
+
+import sbt._
+import scala.collection.JavaConverters._
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.ConcurrentLinkedQueue
+import sbt.testing.Status
+import sbt.testing.Event
+import munit.MUnitTestReport._
+import java.{util => ju}
+import java.text.SimpleDateFormat
+
+class MUnitTestsListener(
+    listener: MUnitReportListener,
+    repository: String,
+    runId: String,
+    scalaVersion: String,
+    projectName: String
+) extends TestsListener {
+  private val groups =
+    new ConcurrentHashMap[String, ConcurrentLinkedQueue[TestEvent]]
+  @volatile
+  private var currentGroup: String = "unknown"
+
+  def startGroup(name: String): Unit = {
+    currentGroup = name
+  }
+  def testEvent(event: TestEvent): Unit = {
+    val group = groups.computeIfAbsent(
+      currentGroup,
+      (_: String) => new ConcurrentLinkedQueue[TestEvent]()
+    )
+    group.add(event)
+  }
+  def endGroup(name: String, t: Throwable): Unit = {}
+  def endGroup(name: String, result: TestResult): Unit = {}
+  def doInit(): Unit = {
+    groups.clear()
+  }
+  def doComplete(finalResult: TestResult): Unit = {
+    listener.onReport(newReport(finalResult))
+  }
+
+  val ISO_8601 =
+    new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", ju.Locale.US);
+  private def newReport(testResult: TestResult): TestReport = {
+    TestReport(
+      repository,
+      runId,
+      ISO_8601.format(new ju.Date()),
+      scalaVersion,
+      projectName,
+      System.getProperty("java.version"),
+      System.getProperty("os.name"),
+      groups.asScala.iterator.map {
+        case (group, events) =>
+          TestGroup(
+            group,
+            overallResult(events.asScala).toString,
+            events.asScala.iterator
+              .flatMap(_.detail)
+              .map(newTestEvent)
+              .toArray
+          )
+      }.toArray
+    )
+  }
+
+  private def newTestEvent(event: Event): TestGroupEvent = {
+    TestGroupEvent(
+      event.status().toString(),
+      event.fullyQualifiedName(),
+      event.duration(),
+      if (event.throwable().isEmpty()) null
+      else newTestException(event.throwable().get())
+    )
+  }
+
+  private def newTestException(ex: Throwable): TestException = {
+    if (ex == null) null
+    else {
+      val plainMessage = Option(ex.getMessage()).map(filterAnsi).getOrElse("")
+      val plainClassName = ex.getClass().getName()
+      val (className, message) =
+        if (plainClassName == "sbt.ForkMain$ForkError") {
+          // When `fork := true`, the exception has class name
+          // `sbt.ForkMain$ForkError` and the underlying exception class name is
+          // formatted in the message.
+          val colon = plainMessage.indexOf(": ")
+          val space = plainMessage.indexOf(' ')
+          if (colon < 0 && (space < 0 || space > colon)) {
+            (plainClassName, plainMessage)
+          } else {
+            (
+              plainMessage.substring(0, colon),
+              plainMessage.substring(colon + 2)
+            )
+          }
+        } else {
+          (plainClassName, plainMessage)
+        }
+      TestException(
+        className,
+        message,
+        Option(ex.getStackTrace()).getOrElse(Array()).map(_.toString),
+        newTestException(ex.getCause())
+      )
+    }
+  }
+  private def filterAnsi(s: String): String = {
+    if (s == null) {
+      null
+    } else {
+      var r: String = ""
+      val len = s.length
+      var i = 0
+      while (i < len) {
+        val c = s.charAt(i)
+        if (c == '\u001B') {
+          i += 1
+          while (i < len && s.charAt(i) != 'm') i += 1
+        } else {
+          r += c
+        }
+        i += 1
+      }
+      r
+    }
+  }
+  private def overallResult(events: Iterable[TestEvent]): TestResult = {
+    events.iterator
+      .flatMap(_.detail)
+      .foldLeft(TestResult.Passed: TestResult) { (sum, event) =>
+        (sum, event.status) match {
+          case (TestResult.Error, _)  => TestResult.Error
+          case (_, Status.Error)      => TestResult.Error
+          case (TestResult.Failed, _) => TestResult.Failed
+          case (_, Status.Failure)    => TestResult.Failed
+          case _                      => TestResult.Passed
+        }
+      }
+  }
+}

munit/non-jvm/src/main/scala/com/geirsson/junit/JUnitReporter.scala

@@ -42,7 +42,7 @@ final class JUnitReporter(
     if (settings.verbose) {
       log(Info, AnsiColors.c(s"==> s $method skipped", AnsiColors.YELLOW))
     }
-    emitEvent(method, Status.Skipped)
+    emitEvent(method, Status.Skipped, new OptionalThrowable(e))
   }
   def reportTestPassed(method: String, elapsedSeconds: Double): Unit = {
     log(