diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 04c6c2f1..fc4bcc6a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: steps: - uses: actions/checkout@v2 - uses: olafurpg/setup-scala@v2 - - run: sbt +testsJVM/test + - run: sbt +testsJVM/test plugin/test js: strategy: fail-fast: false diff --git a/.gitignore b/.gitignore index 074e7ced..f02b3c51 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,4 @@ website/i18n/* project/metals.sbt out/ *.hnir +test-report.json diff --git a/build.sbt b/build.sbt index cfb7b1c9..8ba42a4a 100644 --- a/build.sbt +++ b/build.sbt @@ -148,6 +148,19 @@ lazy val munitJVM = munit.jvm lazy val munitJS = munit.js lazy val munitNative = munit.native +lazy val plugin = project + .in(file("munit-sbt")) + .settings( + sharedSettings, + moduleName := "sbt-munit", + sbtPlugin := true, + scalaVersion := scala212, + crossScalaVersions := List(scala212), + libraryDependencies ++= List( + "com.google.cloud" % "google-cloud-storage" % "1.103.0" + ) + ) + lazy val tests = crossProject(JSPlatform, JVMPlatform, NativePlatform) .dependsOn(munit) .enablePlugins(BuildInfoPlugin) diff --git a/munit-sbt/src/main/scala/munit/MUnitGcpListener.scala b/munit-sbt/src/main/scala/munit/MUnitGcpListener.scala new file mode 100644 index 00000000..6b711a08 --- /dev/null +++ b/munit-sbt/src/main/scala/munit/MUnitGcpListener.scala @@ -0,0 +1,52 @@ +package munit + +import com.google.cloud.storage.StorageOptions +import com.google.cloud.storage.BucketInfo +import com.google.gson.Gson +import java.nio.charset.StandardCharsets +import com.google.cloud.storage.Bucket.BlobTargetOption +import com.google.cloud.storage.Storage.PredefinedAcl +import com.google.cloud.storage.StorageException +import sbt.util.Logger + +class MUnitGcpListener( + val bucketName: String = "munit-test-reports", + val maxRetries: Int = 100, + logger: Logger = sbt.ConsoleLogger() +) extends MUnitReportListener { + private lazy val storage = StorageOptions.getDefaultInstance().getService() + private lazy val bucket = Option(storage.get(bucketName)).getOrElse { + storage.create(BucketInfo.of(bucketName)) + } + def onReport(report: MUnitTestReport.TestReport): Unit = synchronized { + val suffixes = Stream + .from(0) + .map { + case 0 => ".json" + case n => s"-$n.json" + } + .take(maxRetries) + val bytes = new Gson().toJson(report).getBytes(StandardCharsets.UTF_8) + val success = suffixes.find { suffix => + val name = s"${report.repository}/${report.runId}$suffix" + try { + val blob = bucket.create( + name, + bytes, + BlobTargetOption.predefinedAcl(PredefinedAcl.PUBLIC_READ), + BlobTargetOption.doesNotExist() + ) + logger.info( + s"uploaded test report: gs://${blob.getBucket()}/${blob.getBlobId().getName()}" + ) + true + } catch { + case _: StorageException => + false + } + } + if (success.isEmpty) { + logger.error(s"warn: failed to upload report after $maxRetries retries.") + } + } +} diff --git a/munit-sbt/src/main/scala/munit/MUnitLocalListener.scala b/munit-sbt/src/main/scala/munit/MUnitLocalListener.scala new file mode 100644 index 00000000..0dafe191 --- /dev/null +++ b/munit-sbt/src/main/scala/munit/MUnitLocalListener.scala @@ -0,0 +1,28 @@ +package munit + +import java.nio.file.Path +import java.nio.file.Paths +import munit.MUnitTestReport.TestReport +import sbt.util.Logger +import com.google.gson.Gson +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.StandardOpenOption + +class MUnitLocalListener( + path: Path = + Paths.get(".").toAbsolutePath().normalize().resolve("test-report.json"), + maxRetries: Int = 100, + logger: Logger = sbt.ConsoleLogger() +) extends MUnitReportListener { + def onReport(report: TestReport): Unit = { + val bytes = new Gson().toJson(report).getBytes(StandardCharsets.UTF_8) + Files.write( + path, + bytes, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ) + logger.info(s"wrote test report: $path") + } +} diff --git a/munit-sbt/src/main/scala/munit/MUnitPlugin.scala b/munit-sbt/src/main/scala/munit/MUnitPlugin.scala new file mode 100644 index 00000000..e7ea9d2e --- /dev/null +++ b/munit-sbt/src/main/scala/munit/MUnitPlugin.scala @@ -0,0 +1,63 @@ +package munit + +import sbt._ +import sbt.Keys._ +import sbt.plugins._ +import java.nio.file.Paths +import java.nio.file.Files +import java.{util => ju} + +object MUnitPlugin extends AutoPlugin { + override def trigger = allRequirements + override def requires = JvmPlugin + object autoImport { + val munitRepository: SettingKey[Option[String]] = + settingKey[Option[String]]( + "The repository of this project, for example GitHub URL." + ) + val munitRunId: SettingKey[Option[String]] = + settingKey[Option[String]]("Unique identifier for this test run.") + val munitReportListener: SettingKey[Option[MUnitReportListener]] = + settingKey[Option[MUnitReportListener]]( + "The listener to handle reports." + ) + } + import autoImport._ + + override val globalSettings: List[Setting[_ <: Option[Object]]] = List( + munitReportListener := { + for { + credentials <- Option(System.getenv("GOOGLE_APPLICATION_CREDENTIALS")) + path = Paths.get(credentials).toAbsolutePath() + if Files.exists(path) || { + Option(System.getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON")) match { + case Some(base64) => + val json = ju.Base64.getDecoder().decode(base64) + Files.createDirectories(path.getParent()) + Files.write(path, json) + true + case None => false + } + } + } yield new MUnitGcpListener() + }, + munitRepository := Option(System.getenv("GITHUB_REPOSITORY")), + munitRunId := Option(System.getenv("GITHUB_ACTION")) + ) + + override val projectSettings: Seq[Def.Setting[_]] = List( + testListeners ++= { + for { + runId <- munitRunId.value.toList + repository <- munitRepository.value.toList + listener <- munitReportListener.value.toList + } yield new MUnitTestsListener( + listener, + repository, + runId, + scalaVersion.value, + thisProject.value.id + ) + } + ) +} diff --git a/munit-sbt/src/main/scala/munit/MUnitReportListener.scala b/munit-sbt/src/main/scala/munit/MUnitReportListener.scala new file mode 100644 index 00000000..11805311 --- /dev/null +++ b/munit-sbt/src/main/scala/munit/MUnitReportListener.scala @@ -0,0 +1,7 @@ +package munit + +import munit.MUnitTestReport.TestReport + +trait MUnitReportListener { + def onReport(report: TestReport): Unit +} diff --git a/munit-sbt/src/main/scala/munit/MUnitTestReport.scala b/munit-sbt/src/main/scala/munit/MUnitTestReport.scala new file mode 100644 index 00000000..6a5831e9 --- /dev/null +++ b/munit-sbt/src/main/scala/munit/MUnitTestReport.scala @@ -0,0 +1,34 @@ +package munit + +object MUnitTestReport { + case class TestReport( + repository: String, + runId: String, + timestamp: String, + scalaVersion: String, + projectName: String, + javaVersion: String, + osName: String, + groups: Array[TestGroup] + ) + case class TestGroup( + name: String, + result: String, + events: Array[TestGroupEvent] + ) + case class TestGroupEvent( + status: String, + name: String, + // NOTE(olafur): this field should be typed as Double to match JSON types + // but then all numbers get formatted as `2.0` with a redundant `.0` + // suffix. + duration: Long, + exception: TestException + ) + case class TestException( + className: String, + message: String, + stack: Array[String], + cause: TestException + ) +} diff --git a/munit-sbt/src/main/scala/munit/MUnitTestsListener.scala b/munit-sbt/src/main/scala/munit/MUnitTestsListener.scala new file mode 100644 index 00000000..a427b0d8 --- /dev/null +++ b/munit-sbt/src/main/scala/munit/MUnitTestsListener.scala @@ -0,0 +1,143 @@ +package munit + +import sbt._ +import scala.collection.JavaConverters._ +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ConcurrentLinkedQueue +import sbt.testing.Status +import sbt.testing.Event +import munit.MUnitTestReport._ +import java.{util => ju} +import java.text.SimpleDateFormat + +class MUnitTestsListener( + listener: MUnitReportListener, + repository: String, + runId: String, + scalaVersion: String, + projectName: String +) extends TestsListener { + private val groups = + new ConcurrentHashMap[String, ConcurrentLinkedQueue[TestEvent]] + @volatile + private var currentGroup: String = "unknown" + + def startGroup(name: String): Unit = { + currentGroup = name + } + def testEvent(event: TestEvent): Unit = { + val group = groups.computeIfAbsent( + currentGroup, + (_: String) => new ConcurrentLinkedQueue[TestEvent]() + ) + group.add(event) + } + def endGroup(name: String, t: Throwable): Unit = {} + def endGroup(name: String, result: TestResult): Unit = {} + def doInit(): Unit = { + groups.clear() + } + def doComplete(finalResult: TestResult): Unit = { + listener.onReport(newReport(finalResult)) + } + + val ISO_8601 = + new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", ju.Locale.US); + private def newReport(testResult: TestResult): TestReport = { + TestReport( + repository, + runId, + ISO_8601.format(new ju.Date()), + scalaVersion, + projectName, + System.getProperty("java.version"), + System.getProperty("os.name"), + groups.asScala.iterator.map { + case (group, events) => + TestGroup( + group, + overallResult(events.asScala).toString, + events.asScala.iterator + .flatMap(_.detail) + .map(newTestEvent) + .toArray + ) + }.toArray + ) + } + + private def newTestEvent(event: Event): TestGroupEvent = { + TestGroupEvent( + event.status().toString(), + event.fullyQualifiedName(), + event.duration(), + if (event.throwable().isEmpty()) null + else newTestException(event.throwable().get()) + ) + } + + private def newTestException(ex: Throwable): TestException = { + if (ex == null) null + else { + val plainMessage = Option(ex.getMessage()).map(filterAnsi).getOrElse("") + val plainClassName = ex.getClass().getName() + val (className, message) = + if (plainClassName == "sbt.ForkMain$ForkError") { + // When `fork := true`, the exception has class name + // `sbt.ForkMain$ForkError` and the underlying exception class name is + // formatted in the message. + val colon = plainMessage.indexOf(": ") + val space = plainMessage.indexOf(' ') + if (colon < 0 && (space < 0 || space > colon)) { + (plainClassName, plainMessage) + } else { + ( + plainMessage.substring(0, colon), + plainMessage.substring(colon + 2) + ) + } + } else { + (plainClassName, plainMessage) + } + TestException( + className, + message, + Option(ex.getStackTrace()).getOrElse(Array()).map(_.toString), + newTestException(ex.getCause()) + ) + } + } + private def filterAnsi(s: String): String = { + if (s == null) { + null + } else { + var r: String = "" + val len = s.length + var i = 0 + while (i < len) { + val c = s.charAt(i) + if (c == '\u001B') { + i += 1 + while (i < len && s.charAt(i) != 'm') i += 1 + } else { + r += c + } + i += 1 + } + r + } + } + private def overallResult(events: Iterable[TestEvent]): TestResult = { + events.iterator + .flatMap(_.detail) + .foldLeft(TestResult.Passed: TestResult) { (sum, event) => + (sum, event.status) match { + case (TestResult.Error, _) => TestResult.Error + case (_, Status.Error) => TestResult.Error + case (TestResult.Failed, _) => TestResult.Failed + case (_, Status.Failure) => TestResult.Failed + case _ => TestResult.Passed + } + } + } +} diff --git a/munit/non-jvm/src/main/scala/com/geirsson/junit/JUnitReporter.scala b/munit/non-jvm/src/main/scala/com/geirsson/junit/JUnitReporter.scala index 16a398cf..5b6cf5e6 100644 --- a/munit/non-jvm/src/main/scala/com/geirsson/junit/JUnitReporter.scala +++ b/munit/non-jvm/src/main/scala/com/geirsson/junit/JUnitReporter.scala @@ -42,7 +42,7 @@ final class JUnitReporter( if (settings.verbose) { log(Info, AnsiColors.c(s"==> s $method skipped", AnsiColors.YELLOW)) } - emitEvent(method, Status.Skipped) + emitEvent(method, Status.Skipped, new OptionalThrowable(e)) } def reportTestPassed(method: String, elapsedSeconds: Double): Unit = { log( diff --git a/munit/non-jvm/src/main/scala/com/geirsson/junit/MUnitRunNotifier.scala b/munit/non-jvm/src/main/scala/com/geirsson/junit/MUnitRunNotifier.scala index 0eb57f85..cb2750c4 100644 --- a/munit/non-jvm/src/main/scala/com/geirsson/junit/MUnitRunNotifier.scala +++ b/munit/non-jvm/src/main/scala/com/geirsson/junit/MUnitRunNotifier.scala @@ -17,9 +17,9 @@ class MUnitRunNotifier(reporter: JUnitReporter) extends RunNotifier { startedTimestamp = System.nanoTime() reporter.reportTestStarted(description.getMethodName) } - def elapsedSeconds(): Double = { + def elapsedMillis(): Double = { val elapsedNanos = System.nanoTime() - startedTimestamp - elapsedNanos / 1000000000.0 + elapsedNanos / 1000000.0 } override def fireTestIgnored(description: Description): Unit = { ignored += 1 @@ -30,7 +30,7 @@ class MUnitRunNotifier(reporter: JUnitReporter) extends RunNotifier { ): Unit = { reporter.reportAssumptionViolation( failure.description.getMethodName, - elapsedSeconds(), + elapsedMillis(), failure.ex ) } @@ -40,7 +40,7 @@ class MUnitRunNotifier(reporter: JUnitReporter) extends RunNotifier { reporter.reportTestFailed( methodName, failure.ex, - elapsedSeconds() + elapsedMillis() ) } override def fireTestFinished(description: Description): Unit = { @@ -49,7 +49,7 @@ class MUnitRunNotifier(reporter: JUnitReporter) extends RunNotifier { if (!isFailed(methodName)) { reporter.reportTestPassed( methodName, - elapsedSeconds() + elapsedMillis() ) } } diff --git a/munit/shared/src/main/scala/munit/MUnitRunner.scala b/munit/shared/src/main/scala/munit/MUnitRunner.scala index 6280beec..542fac31 100644 --- a/munit/shared/src/main/scala/munit/MUnitRunner.scala +++ b/munit/shared/src/main/scala/munit/MUnitRunner.scala @@ -185,6 +185,7 @@ class MUnitRunner(val cls: Class[_ <: Suite], newInstance: () => Suite) } result match { case f: TestValues.FlakyFailure => + StackTraces.trimStackTrace(f) notifier.fireTestAssumptionFailed(new Failure(description, f)) case TestValues.Ignore => notifier.fireTestIgnored(description) diff --git a/project/plugins.sbt b/project/plugins.sbt index 0e7cd407..fa93a6ac 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -12,3 +12,9 @@ addSbtPlugin("org.portable-scala" % "sbt-scalajs-crossproject" % "0.6.1") addSbtPlugin("org.portable-scala" % "sbt-scala-native-crossproject" % "0.6.1") addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.4.0-M2") addSbtPlugin("org.scala-js" % "sbt-scalajs" % scalaJSVersion) + +libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "1.103.0" + +unmanagedSourceDirectories.in(Compile) += + baseDirectory.in(ThisBuild).value.getParentFile / + "munit-sbt" / "src" / "main" / "scala" diff --git a/tests/shared/src/test/scala/munit/DemoSuite.scala b/tests/shared/src/test/scala/munit/DemoSuite.scala index 047cb4bc..19d66a38 100644 --- a/tests/shared/src/test/scala/munit/DemoSuite.scala +++ b/tests/shared/src/test/scala/munit/DemoSuite.scala @@ -1,9 +1,6 @@ package munit -import scala.util.Properties - abstract class DemoSuite extends FunSuite { - override def munitIgnore: Boolean = Properties.isWin def someCondition(n: Int): Boolean = n != 2 test("source-locations") { assert(someCondition(1)) @@ -17,7 +14,8 @@ abstract class DemoSuite extends FunSuite { assertEquals(john, susan) } - test("stack-traces") { + override def munitFlakyOK: Boolean = true + test("stack-traces".flaky) { List(List(1, 2, 3).iterator).iterator.flatten.foreach { i => require(i < 2, i) }