Distinguish when an error is part of a recorded issue in an exit test. (#718)

Hot on the heels of #697…

If an exit test records an issue of kind `.errorCaught()`, we currently
treat it the same as any other issue and translate it to a flat
`.unconditional` issue in the parent process:

```swift
await #expect(exitsWith: .failure) {
  Issue.record(NoSuchWeaselError()) // unconditional issue recorded
}
```

This PR distinguishes the `.errorCaught` issue kind and records an
`.errorCaught` issue in the parent process. Fidelity of errors across a
process boundary is weak, but we encode a minimal representation of the
error that should be good enough for most uses. If you need
`withKnownIssue()`, or `#expect(throws:)`, you can call them _inside_
the exit test.

### Checklist:

- [x] Code and documentation should follow the style of the [Style
Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md).
- [x] If public symbols are renamed or modified, DocC references should
be updated.

Author: grynspan

Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedBacktrace.swift

@@ -15,6 +15,8 @@ extension ABIv0 {
   /// This type is not part of the public interface of the testing library. It
   /// assists in converting values to JSON; clients that consume this JSON are
   /// expected to write their own decoders.
+  ///
+  /// - Warning: Backtraces are not yet part of the JSON schema.
   struct EncodedBacktrace: Sendable {
     /// The frames in the backtrace.
     var symbolicatedAddresses: [Backtrace.SymbolicatedAddress]

Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedError.swift

@@ -0,0 +1,67 @@
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for Swift project authors
+//
+
+extension ABIv0 {
+  /// A type implementing the JSON encoding of ``Error`` for the ABI entry point
+  /// and event stream output.
+  ///
+  /// This type is not part of the public interface of the testing library. It
+  /// assists in converting values to JSON; clients that consume this JSON are
+  /// expected to write their own decoders.
+  ///
+  /// - Warning: Errors are not yet part of the JSON schema.
+  struct EncodedError: Sendable {
+    /// The error's description
+    var description: String
+
+    /// The domain of the error.
+    var domain: String
+
+    /// The code of the error.
+    var code: Int
+
+    // TODO: userInfo (partial) encoding
+
+    init(encoding error: some Error, in eventContext: borrowing Event.Context) {
+      description = String(describingForTest: error)
+      domain = error._domain
+      code = error._code
+    }
+  }
+}
+
+// MARK: - Error
+
+extension ABIv0.EncodedError: Error {
+  var _domain: String {
+    domain
+  }
+
+  var _code: Int {
+    code
+  }
+
+  var _userInfo: AnyObject? {
+    // TODO: userInfo (partial) encoding
+    nil
+  }
+}
+
+// MARK: - Codable
+
+extension ABIv0.EncodedError: Codable {}
+
+// MARK: - CustomTestStringConvertible
+
+extension ABIv0.EncodedError: CustomTestStringConvertible {
+  var testDescription: String {
+    description
+  }
+}

Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift

@@ -23,14 +23,24 @@ extension ABIv0 {
     var sourceLocation: SourceLocation?
 
     /// The backtrace where this issue occurred, if available.
+    ///
+    /// - Warning: Backtraces are not yet part of the JSON schema.
     var _backtrace: EncodedBacktrace?
 
+    /// The error associated with this issue, if applicable.
+    ///
+    /// - Warning: Errors are not yet part of the JSON schema.
+    var _error: EncodedError?
+
     init(encoding issue: borrowing Issue, in eventContext: borrowing Event.Context) {
       isKnown = issue.isKnown
       sourceLocation = issue.sourceLocation
       if let backtrace = issue.sourceContext.backtrace {
         _backtrace = EncodedBacktrace(encoding: backtrace, in: eventContext)
       }
+      if let error = issue.error {
+        _error = EncodedError(encoding: error, in: eventContext)
+      }
     }
   }
 }

Sources/Testing/CMakeLists.txt

@@ -14,6 +14,7 @@ add_library(Testing
   ABI/v0/ABIv0.Record+Streaming.swift
   ABI/v0/ABIv0.swift
   ABI/v0/Encoded/ABIv0.EncodedBacktrace.swift
+  ABI/v0/Encoded/ABIv0.EncodedError.swift
   ABI/v0/Encoded/ABIv0.EncodedEvent.swift
   ABI/v0/Encoded/ABIv0.EncodedInstant.swift
   ABI/v0/Encoded/ABIv0.EncodedIssue.swift

Sources/Testing/ExitTests/ExitTest.swift

@@ -505,12 +505,17 @@ extension ExitTest {
       let comments: [Comment] = event.messages.compactMap { message in
         message.symbol == .details ? Comment(rawValue: message.text) : nil
       }
+      let issueKind: Issue.Kind = if let error = issue._error {
+        .errorCaught(error)
+      } else {
+        // TODO: improve fidelity of issue kind reporting (especially those without associated values)
+        .unconditional
+      }
       let sourceContext = SourceContext(
         backtrace: nil, // `issue._backtrace` will have the wrong address space.
         sourceLocation: issue.sourceLocation
       )
-      // TODO: improve fidelity of issue kind reporting (especially those without associated values)
-      var issueCopy = Issue(kind: .unconditional, comments: comments, sourceContext: sourceContext)
+      var issueCopy = Issue(kind: issueKind, comments: comments, sourceContext: sourceContext)
       issueCopy.isKnown = issue.isKnown
       issueCopy.record()
     }

Tests/TestingTests/ExitTestTests.swift

@@ -32,9 +32,6 @@ private import _TestingInternals
       await Task.yield()
       exit(123)
     }
-    await #expect(exitsWith: .failure) {
-      throw MyError()
-    }
 #if !os(Windows)
     await #expect(exitsWith: .signal(SIGKILL)) {
       _ = kill(getpid(), SIGKILL)
@@ -203,22 +200,30 @@ private import _TestingInternals
 
   @Test("Exit test forwards issues") func forwardsIssues() async {
     await confirmation("Issue recorded") { issueRecorded in
-      var configuration = Configuration()
-      configuration.eventHandler = { event, _ in
-        if case let .issueRecorded(issue) = event.kind,
-           case .unconditional = issue.kind,
-           issue.comments.contains("Something went wrong!") {
-          issueRecorded()
+      await confirmation("Error caught") { errorCaught in
+        var configuration = Configuration()
+        configuration.eventHandler = { event, _ in
+          guard case let .issueRecorded(issue) = event.kind else {
+            return
+          }
+          if case .unconditional = issue.kind, issue.comments.contains("Something went wrong!") {
+            issueRecorded()
+          } else if issue.error != nil {
+            errorCaught()
+          }
         }
-      }
-      configuration.exitTestHandler = ExitTest.handlerForEntryPoint()
+        configuration.exitTestHandler = ExitTest.handlerForEntryPoint()
 
-      await Test {
-        await #expect(exitsWith: .success) {
-          #expect(Bool(false), "Something went wrong!")
-          exit(0)
-        }
-      }.run(configuration: configuration)
+        await Test {
+          await #expect(exitsWith: .success) {
+            #expect(Bool(false), "Something went wrong!")
+            exit(0)
+          }
+          await #expect(exitsWith: .failure) {
+            Issue.record(MyError())
+          }
+        }.run(configuration: configuration)
+      }
     }
   }