diff --git a/community-build/test/java/dotty/communitybuild/FailureSummarizer.java b/community-build/test/java/dotty/communitybuild/FailureSummarizer.java
new file mode 100644
index 000000000000..a28af2bb8d53
--- /dev/null
+++ b/community-build/test/java/dotty/communitybuild/FailureSummarizer.java
@@ -0,0 +1,36 @@
+package dotty.communitybuild;
+
+import java.util.List;
+
+import org.junit.runner.Description;
+import org.junit.runner.Result;
+import org.junit.runner.notification.Failure;
+import org.junit.runner.notification.RunListener;
+
+public class FailureSummarizer extends RunListener {
+  @Override
+  public void testRunFinished(Result result) throws Exception {
+    super.testRunFinished(result);
+    if (result.getFailureCount() > 0) {
+      Thread.sleep(500);  // pause to give sbt log buffers some time to flush
+      summarizeFailures(result.getFailures());
+    }
+  }
+
+  private void summarizeFailures(List<Failure> failures) {
+    err("********************************************************************************");
+    err("Failed projects:");
+    for (Failure f : failures) {
+      err(" - " + getProjectName(f.getDescription()));
+    }
+    err("********************************************************************************");
+  }
+
+  private String getProjectName(Description desc) {
+    return desc.getClassName() + "." + desc.getMethodName();
+  }
+
+  private void err(String msg) {
+    System.err.println(msg);
+  }
+}
diff --git a/project/Build.scala b/project/Build.scala
index fa3a55ff8878..3c797c2473ee 100644
--- a/project/Build.scala
+++ b/project/Build.scala
@@ -1360,6 +1360,7 @@ object Build {
       testOptions in Test += Tests.Argument(
         TestFrameworks.JUnit,
         "--include-categories=dotty.communitybuild.TestCategory",
+        "--run-listener=dotty.communitybuild.FailureSummarizer",
       ),
       Compile/run := (Compile/run).dependsOn(prepareCommunityBuild).evaluated,
       (Test / testOnly) := ((Test / testOnly) dependsOn prepareCommunityBuild).evaluated,