Exit non-zero status when test fail on browser

Sources/CartonCLI/Commands/TestRunners/BrowserTestRunner.swift

@@ -166,8 +166,9 @@ struct BrowserTestRunner: TestRunner {
         disposer = {}
         openInSystemBrowser(url: localURL)
       }
-      try await server.waitUntilStop()
+      let hadError = try await server.waitUntilTestFinished()
       try await disposer()
+      exit(hadError ? EXIT_FAILURE : EXIT_SUCCESS)
     } catch {
       try await disposer()
       throw error

Sources/CartonKit/Server/Server.swift

@@ -24,15 +24,20 @@ private enum Event {
     case kind
     case stackTrace
     case testRunOutput
+    case errorReport
   }
 
   enum Kind: String, Decodable {
     case stackTrace
     case testRunOutput
+    case testPassed
+    case errorReport
   }
 
   case stackTrace(String)
   case testRunOutput(String)
+  case testPassed
+  case errorReport(String)
 }
 
 extension Event: Decodable {
@@ -48,6 +53,11 @@ extension Event: Decodable {
     case .testRunOutput:
       let output = try container.decode(String.self, forKey: .testRunOutput)
       self = .testRunOutput(output)
+    case .testPassed:
+      self = .testPassed
+    case .errorReport:
+      let output = try container.decode(String.self, forKey: .errorReport)
+      self = .errorReport(output)
     }
   }
 }
@@ -85,6 +95,9 @@ public actor Server {
   /// Whether a subsequent build is currently scheduled on top of a currently running build.
   private var isSubsequentBuildScheduled = false
 
+  /// Continuation for waitUntilTestFinished, passing `hadError: Bool`
+  private var onTestFinishedContinuation: CheckedContinuation<Bool, Never>?
+
   public struct Configuration {
     let builder: Builder?
     let mainWasmPath: AbsolutePath
@@ -231,6 +244,18 @@ public actor Server {
     try closeSockets()
   }
 
+  /// Wait and handle the shutdown
+  public func waitUntilTestFinished() async throws -> Bool {
+    defer { self.app.shutdown() }
+    let hadError = await withCheckedContinuation { cont in
+      self.onTestFinishedContinuation = cont
+    }
+    self.onTestFinishedContinuation = nil
+    app.running?.stop()
+    try closeSockets()
+    return hadError
+  }
+
   func closeSockets() throws {
     for conn in connections {
       try conn.close().wait()
@@ -247,6 +272,10 @@ public actor Server {
     terminal.logLookup("The app is currently hosted at ", localURL)
     connections.forEach { $0.send("reload") }
   }
+
+  private func stopTest(hadError: Bool) {
+    self.onTestFinishedContinuation?.resume(returning: hadError)
+  }
 }
 
 extension Server {
@@ -257,9 +286,10 @@ extension Server {
     terminal: InteractiveWriter
   ) -> (WebSocket, String) -> () {
     { [weak self] _, text in
+      guard let self = self else { return }
       guard
         let data = text.data(using: .utf8),
-        let event = try? self?.decoder.decode(Event.self, from: data)
+        let event = try? self.decoder.decode(Event.self, from: data)
       else {
         return
       }
@@ -283,10 +313,14 @@ extension Server {
       case let .testRunOutput(output):
         TestsParser().parse(output, terminal)
 
-        // Test run finished, no need to keep the server running anymore.
-        if configuration.builder == nil {
-          kill(getpid(), SIGINT)
-        }
+      case .testPassed:
+        Task { await self.stopTest(hadError: false) }
+
+      case let .errorReport(output):
+        terminal.write("\nAn error occurred:\n", inColor: .red)
+        terminal.write(output + "\n")
+
+        Task { await self.stopTest(hadError: true) }
       }
     }
   }

Tests/CartonCommandTests/TestCommandTests.swift

@@ -24,6 +24,8 @@ import XCTest
 private enum Constants {
   static let testAppPackageName = "TestApp"
   static let nodeJSKitPackageName = "NodeJSKitTest"
+  static let crashTestPackageName = "CrashTest"
+  static let failTestPackageName = "FailTest"
 }
 
 final class TestCommandTests: XCTestCase {
@@ -82,6 +84,27 @@ final class TestCommandTests: XCTestCase {
     }
   }
 
+  func testHeadlessBrowserWithCrash() throws {
+    try checkCartonTestFail(fixture: Constants.crashTestPackageName)
+  }
+
+  func testHeadlessBrowserWithFail() throws {
+    try checkCartonTestFail(fixture: Constants.failTestPackageName)
+  }
+
+  func checkCartonTestFail(fixture: String) throws {
+    guard Process.findExecutable("safaridriver") != nil else {
+      throw XCTSkip("WebDriver is required")
+    }
+    try withFixture(fixture) { packageDirectory in
+      try ProcessEnv.chdir(packageDirectory)
+      let process = Process(arguments: [cartonPath, "test", "--environment", "defaultBrowser", "--headless"])
+      try process.launch()
+      let result = try process.waitUntilExit()
+      XCTAssertNotEqual(result.exitStatus, .terminated(code: 0))
+    }
+  }
+
   // This test is prone to hanging on Linux.
   #if os(macOS)
   func testEnvironmentDefaultBrowser() throws {

Tests/Fixtures/CrashTest/Package.swift

@@ -0,0 +1,7 @@
+// swift-tools-version:5.5
+import PackageDescription
+
+let package = Package(
+  name: "Test",
+  targets: [.testTarget(name: "CrashTest", path: "Tests")]
+)

Tests/Fixtures/CrashTest/Tests/Tests.swift

@@ -0,0 +1,8 @@
+import XCTest
+
+class Tests: XCTestCase {
+    func testCrash() {
+        // recursive call would cause stack overflow
+        testCrash()
+    }
+}

Tests/Fixtures/FailTest/Package.swift

@@ -0,0 +1,7 @@
+// swift-tools-version:5.5
+import PackageDescription
+
+let package = Package(
+  name: "Test",
+  targets: [.testTarget(name: "FailTest", path: "Tests")]
+)

Tests/Fixtures/FailTest/Tests/Tests.swift

@@ -0,0 +1,7 @@
+import XCTest
+
+class Tests: XCTestCase {
+    func testFail() {
+        XCTFail("Yeah")
+    }
+}

entrypoint/test.js

@@ -60,34 +60,57 @@ const startWasiTask = async () => {
 
   // Instantiate the WebAssembly file
   const wasmBytes = new Uint8Array(responseArrayBuffer).buffer;
-  // Start the WebAssembly WASI instance
-  try {
-    await wasmRunner.run(wasmBytes);
-  } catch (error) {
-    if (!(error instanceof WASIExitError) || error.code != 0) {
-      throw error; // not a successful test run, rethrow
-    }
-  } finally {
-    // pass the output to the server in any case
-    socket.send(
-      JSON.stringify({
-        kind: "testRunOutput",
-        testRunOutput,
-      })
-    );
 
+  // There are 6 cases to exit test
+  // 1. Successfully finished XCTest with `exit(0)` synchronously
+  // 2. Unsuccessfully finished XCTest with `exit(non-zero)` synchronously
+  // 3. Successfully finished XCTest with `exit(0)` asynchronously
+  // 4. Unsuccessfully finished XCTest with `exit(non-zero)` asynchronously
+  // 5. Crash by throwing JS exception synchronously
+  // 6. Crash by throwing JS exception asynchronously
+
+  const handleExitOrError = (error) => {
+    // XCTest always calls `exit` at the end when no crash
+    if (error instanceof WASIExitError) {
+      // pass the output to the server in any case
+      socket.send(JSON.stringify({ kind: "testRunOutput", testRunOutput }));
+      if (error.code === 0) {
+        socket.send(JSON.stringify({ kind: "testPassed" }));
+      } else {
+        handleError(error) // test failed
+      }
+    } else {
+      handleError(error) // something wrong happens during test
+    }
     const divElement = document.createElement("p");
     divElement.innerHTML =
       "Test run finished. Check the output of <code>carton test</code> for details.";
     document.body.appendChild(divElement);
   }
+  // Handle asynchronous exits (case 3, 4, 6)
+  window.addEventListener("unhandledrejection", event => {
+    event.preventDefault();
+    const error = event.reason;
+    handleExitOrError(error);
+  });
+  // Start the WebAssembly WASI instance
+  try {
+    await wasmRunner.run(wasmBytes);
+  } catch (error) {
+    // Handle synchronous exits (case 1, 2, 5)
+    handleExitOrError(error)
+    return
+  }
+  // When JavaScriptEventLoop executor is still running,
+  // reachable here without catch (case 3, 4, 6)
 };
 
 function handleError(e) {
   console.error(e);
   if (e instanceof WebAssembly.RuntimeError) {
     console.log(e.stack);
   }
+  socket.send(JSON.stringify({ kind: "errorReport", errorReport: e.toString() }));
 }
 
 try {