Test: ensure swift-[build|test] emit expected number of fatal error on console

Fixtures/Miscellaneous/Errors/FatalErrorInSingleXCTest/TypeLibrary/.gitignore

@@ -0,0 +1,8 @@
+.DS_Store
+/.build
+/Packages
+xcuserdata/
+DerivedData/
+.swiftpm/configuration/registries.json
+.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata
+.netrc

Fixtures/Miscellaneous/Errors/FatalErrorInSingleXCTest/TypeLibrary/Package.swift

@@ -0,0 +1,24 @@
+// swift-tools-version:5.9
+// The swift-tools-version declares the minimum version of Swift required to build this package.
+
+import PackageDescription
+
+let package = Package(
+    name: "TypeLibrary",
+    products: [
+        // Products define the executables and libraries a package produces, making them visible to other packages.
+        .library(
+            name: "TypeLibrary",
+            targets: ["TypeLibrary"]),
+    ],
+    targets: [
+        // Targets are the basic building blocks of a package, defining a module or a test suite.
+        // Targets can depend on other targets in this package and products from dependencies.
+        .target(
+            name: "TypeLibrary"),
+        .testTarget(
+            name: "TypeLibraryTests",
+            dependencies: ["TypeLibrary"]
+        ),
+    ]
+)

Fixtures/Miscellaneous/Errors/FatalErrorInSingleXCTest/TypeLibrary/Sources/TypeLibrary/TypeLibrary.swift

@@ -0,0 +1,2 @@
+// The Swift Programming Language
+// https://docs.swift.org/swift-book

Fixtures/Miscellaneous/Errors/FatalErrorInSingleXCTest/TypeLibrary/Tests/TypeLibraryTests/TypeLibraryTests.swift

@@ -0,0 +1,5 @@
+func testExample() throws {
+    let x = 0
+    let y = 1 / x
+    print(y)
+}

Package.swift

@@ -873,7 +873,12 @@ if ProcessInfo.processInfo.environment["SWIFTCI_DISABLE_SDK_DEPENDENT_TESTS"] ==
                 "Basics",
             ]
         ),
-
+        .testTarget(
+            name: "_InternalTestSupportTests",
+            dependencies: [
+                "_InternalTestSupport"
+            ]
+        ),
         .testTarget(
             name: "CommandsTests",
             dependencies: [

Sources/_InternalTestSupport/misc.swift

@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+import Foundation
 import Basics
 import struct Foundation.URL
 #if os(macOS)
@@ -465,3 +466,8 @@ extension PackageIdentity: @retroactive ExpressibleByStringInterpolation {}
 extension AbsolutePath: @retroactive ExpressibleByStringLiteral {}
 extension AbsolutePath: @retroactive ExpressibleByStringInterpolation {}
 #endif
+
+public func getNumberOfMatches(of match: String, in value: String) -> Int {
+    guard match.count != 0 else { return 0 }
+    return value.ranges(of: match).count
+}

Tests/CommandsTests/BuildCommandTests.swift

@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+import Foundation
 import Basics
 @testable import Commands
 @testable import CoreCommands
@@ -678,4 +679,41 @@ final class BuildCommandTests: CommandsTestCase {
             XCTAssertGreaterThan(codeCovFiles.count, 0)
         }
     }
+
+    func testFatalErrorDisplayedCorrectNumberOfTimesWhenSingleXCTestHasFatalErrorInBuildCompilation() async throws {
+        // Test for GitHub Issue #6605
+        // GIVEN we have a Swift Package that has a fatalError building the tests
+        #if compiler(>=6)
+        let expected = 0
+        #else
+        let expected = 1
+        #endif
+        try await fixture(name: "Miscellaneous/Errors/FatalErrorInSingleXCTest/TypeLibrary") { fixturePath in
+            // WHEN swift-build --build-tests is executed"
+            await XCTAssertAsyncThrowsError(try await self.execute(["--build-tests"], packagePath: fixturePath)) { error in
+                // THEN I expect a failure
+                guard case SwiftPMError.executionFailure(_, let stdout, let stderr) = error else {
+                    XCTFail("Building the package was expected to fail, but it was successful")
+                    return
+                }
+
+                let matchString = "error: fatalError"
+                let stdoutMatches = getNumberOfMatches(of: matchString, in: stdout)
+                let stderrMatches = getNumberOfMatches(of: matchString, in: stderr)
+                let actualNumMatches = stdoutMatches + stderrMatches
+
+                // AND a fatal error message is printed \(expected) times
+                XCTAssertEqual(
+                    actualNumMatches,
+                    expected,
+                    [
+                        "Actual (\(actualNumMatches)) is not as expected (\(expected))",
+                        "stdout: \(stdout.debugDescription)",
+                        "stderr: \(stderr.debugDescription)"
+                    ].joined(separator: "\n")
+                )
+            }
+        }
+    }
+
 }

Tests/CommandsTests/TestCommandTests.swift

@@ -361,4 +361,41 @@ final class TestCommandTests: CommandsTestCase {
             XCTAssertNoMatch(stderr, .contains("No matching test cases were run"))
         }
     }
+
+    func testFatalErrorDisplayedCorrectNumberOfTimesWhenSingleXCTestHasFatalErrorInBuildCompilation() async throws {
+        // Test for GitHub Issue #6605
+        // GIVEN we have a Swift Package that has a fatalError building the tests
+        #if compiler(>=6)
+        let expected = 1
+        #else
+        let expected = 2
+        #endif
+        try await fixture(name: "Miscellaneous/Errors/FatalErrorInSingleXCTest/TypeLibrary") { fixturePath in
+            // WHEN swift-test is executed
+            await XCTAssertAsyncThrowsError(try await self.execute([], packagePath: fixturePath)) { error in
+                // THEN I expect a failure
+                guard case SwiftPMError.executionFailure(_, let stdout, let stderr) = error else {
+                    XCTFail("Building the package was expected to fail, but it was successful")
+                    return
+                }
+
+                let matchString = "error: fatalError"
+                let stdoutMatches = getNumberOfMatches(of: matchString, in: stdout)
+                let stderrMatches = getNumberOfMatches(of: matchString, in: stderr)
+                let actualNumMatches = stdoutMatches + stderrMatches
+
+                // AND a fatal error message is printed \(expected) times
+                XCTAssertEqual(
+                    actualNumMatches,
+                    expected,
+                    [
+                        "Actual (\(actualNumMatches)) is not as expected (\(expected))",
+                        "stdout: \(stdout.debugDescription)",
+                        "stderr: \(stderr.debugDescription)"
+                    ].joined(separator: "\n")
+                )
+            }
+        }
+    }
+
 }

Tests/_InternalTestSupportTests/Misc.swift

@@ -0,0 +1,119 @@
+import _InternalTestSupport
+import XCTest
+
+final class TestGetNumberOfMatches: XCTestCase {
+    func testEmptyStringMatchesOnEmptyStringZeroTimes() {
+        let matchOn = ""
+        let value = ""
+        let expectedNumMatches = 0
+
+        let actual = getNumberOfMatches(of: matchOn, in: value)
+
+        XCTAssertEqual(actual, expectedNumMatches, "Actual is not as expected")
+    }
+
+    func testEmptyStringMatchesOnNonEmptySingleLineStringZeroTimes() {
+        let matchOn = ""
+        let value = "This is a non-empty string"
+        let expectedNumMatches = 0
+
+        let actual = getNumberOfMatches(of: matchOn, in: value)
+
+        XCTAssertEqual(actual, expectedNumMatches, "Actual is not as expected")
+    }
+
+    func testEmptyStringMatchesOnNonEmptyMultilineStringWithNeLineCharacterZeroTimes() {
+        let matchOn = ""
+        let value = "This is a non-empty string\nThis is the second line"
+        let expectedNumMatches = 0
+
+        let actual = getNumberOfMatches(of: matchOn, in: value)
+
+        XCTAssertEqual(actual, expectedNumMatches, "Actual is not as expected")
+    }
+
+    func testEmptyStringMatchesOnNonEmptyMultilineStringUsingTripleDoubleQuotesZeroTimes() {
+        let matchOn = ""
+        let value = """
+        This is a non-empty string
+        This is the second line
+        This is the third line
+        """
+        let expectedNumMatches = 0
+
+        let actual = getNumberOfMatches(of: matchOn, in: value)
+
+        XCTAssertEqual(actual, expectedNumMatches, "Actual is not as expected")
+    }
+
+    func testNonEmptyStringMatchesOnEmptyStringReturnsZero() {
+        let matchOn = """
+        This is a non-empty string
+        This is the second line
+        This is the third line
+        """
+        let value = ""
+        let expectedNumMatches = 0
+
+        let actual = getNumberOfMatches(of: matchOn, in: value)
+
+        XCTAssertEqual(actual, expectedNumMatches, "Actual is not as expected")
+    }
+
+    func testfatalErrorMatchesOnMultiLineWithTwoOccurencesReturnsTwo() {
+        let matchOn = "error: fatalError"
+        let value = """
+        > swift test                                                                                          25/10/24 10:44:14
+        Building for debugging...
+        /Users/arandomuser/Documents/personal/repro-swiftpm-6605/Tests/repro-swiftpm-6605Tests/repro_swiftpm_6605Tests.swift:7:19: error: division by zero
+                let y = 1 / x
+                        ^
+        error: fatalError
+
+        error: fatalError
+        """
+        let expectedNumMatches = 2
+
+        let actual = getNumberOfMatches(of: matchOn, in: value)
+
+        XCTAssertEqual(actual, expectedNumMatches, "Actual is not as expected")
+    }
+
+    func testfatalErrorWithLeadingNewLineMatchesOnMultiLineWithTwoOccurencesReturnsTwo() {
+        let matchOn = "\nerror: fatalError"
+        let value = """
+        > swift test                                                                                          25/10/24 10:44:14
+        Building for debugging...
+        /Users/arandomuser/Documents/personal/repro-swiftpm-6605/Tests/repro-swiftpm-6605Tests/repro_swiftpm_6605Tests.swift:7:19: error: division by zero
+                let y = 1 / x
+                        ^
+        error: fatalError
+
+        error: fatalError
+        """
+        let expectedNumMatches = 2
+
+        let actual = getNumberOfMatches(of: matchOn, in: value)
+
+        XCTAssertEqual(actual, expectedNumMatches, "Actual is not as expected")
+    }
+
+    func testfatalErrorWithLeadingAndTrailingNewLineMatchesOnMultiLineWithOneOccurencesReturnsOne() {
+        let matchOn = "\nerror: fatalError\n"
+        let value = """
+        > swift test                                                                                          25/10/24 10:44:14
+        Building for debugging...
+        /Users/arandomuser/Documents/personal/repro-swiftpm-6605/Tests/repro-swiftpm-6605Tests/repro_swiftpm_6605Tests.swift:7:19: error: division by zero
+                let y = 1 / x
+                        ^
+        error: fatalError
+
+        error: fatalError
+        """
+        let expectedNumMatches = 1
+
+        let actual = getNumberOfMatches(of: matchOn, in: value)
+
+        XCTAssertEqual(actual, expectedNumMatches, "Actual is not as expected")
+    }
+}