Update xUnit to display output on failures

bkhouri · bkhouri · commit 20c2b2d5c4d3 · 2024-11-29T15:32:43.000-05:00
The generated xUnit XML file is not helpful when tests fail as it
contains the message "failure", which is redundant with the test
results.

Update the XML output to dipslay the result output instead of static
"failure" message.
diff --git a/Fixtures/Miscellaneous/TestFailuresSwiftTesting/.gitignore b/Fixtures/Miscellaneous/TestFailuresSwiftTesting/.gitignore
@@ -0,0 +1,9 @@
+.DS_Store
+/.build
+/.index-build
+/Packages
+xcuserdata/
+DerivedData/
+.swiftpm/configuration/registries.json
+.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata
+.netrc
diff --git a/Fixtures/Miscellaneous/TestFailuresSwiftTesting/Package.swift b/Fixtures/Miscellaneous/TestFailuresSwiftTesting/Package.swift
@@ -0,0 +1,24 @@
+// swift-tools-version: 5.9
+// The swift-tools-version declares the minimum version of Swift required to build this package.
+
+import PackageDescription
+
+let package = Package(
+    name: "TestFailuresSwiftTesting",
+    products: [
+        // Products define the executables and libraries a package produces, making them visible to other packages.
+        .library(
+            name: "TestFailuresSwiftTesting",
+            targets: ["TestFailuresSwiftTesting"])
+    ],
+    targets: [
+        // Targets are the basic building blocks of a package, defining a module or a test suite.
+        // Targets can depend on other targets in this package and products from dependencies.
+        .target(
+            name: "TestFailuresSwiftTesting"),
+        .testTarget(
+            name: "TestFailuresSwiftTestingTests",
+            dependencies: ["TestFailuresSwiftTesting"]
+        )
+    ]
+)
diff --git a/Fixtures/Miscellaneous/TestFailuresSwiftTesting/Sources/TestFailuresSwiftTesting/TestFailuresSwiftTesting.swift b/Fixtures/Miscellaneous/TestFailuresSwiftTesting/Sources/TestFailuresSwiftTesting/TestFailuresSwiftTesting.swift
@@ -0,0 +1,2 @@
+// The Swift Programming Language
+// https://docs.swift.org/swift-book
diff --git a/Fixtures/Miscellaneous/TestFailuresSwiftTesting/Tests/TestFailuresSwiftTestingTests/TestFailuresSwiftTestingTests.swift b/Fixtures/Miscellaneous/TestFailuresSwiftTesting/Tests/TestFailuresSwiftTestingTests/TestFailuresSwiftTestingTests.swift
@@ -0,0 +1,6 @@
+import Testing
+@testable import TestFailuresSwiftTesting
+
+@Test func example() async throws {
+    #expect(Bool(false), "Purposely failing & validating XML espace \"'<>")
+}
diff --git a/Fixtures/Miscellaneous/TestFailuresXCTest/.gitignore b/Fixtures/Miscellaneous/TestFailuresXCTest/.gitignore
@@ -0,0 +1,9 @@
+.DS_Store
+/.build
+/.index-build
+/Packages
+xcuserdata/
+DerivedData/
+.swiftpm/configuration/registries.json
+.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata
+.netrc
diff --git a/Fixtures/Miscellaneous/TestFailuresXCTest/Package.swift b/Fixtures/Miscellaneous/TestFailuresXCTest/Package.swift
@@ -0,0 +1,24 @@
+// swift-tools-version: 5.9
+// The swift-tools-version declares the minimum version of Swift required to build this package.
+
+import PackageDescription
+
+let package = Package(
+    name: "TestFailures",
+    products: [
+        // Products define the executables and libraries a package produces, making them visible to other packages.
+        .library(
+            name: "TestFailures",
+            targets: ["TestFailures"])
+    ],
+    targets: [
+        // Targets are the basic building blocks of a package, defining a module or a test suite.
+        // Targets can depend on other targets in this package and products from dependencies.
+        .target(
+            name: "TestFailures"),
+        .testTarget(
+            name: "TestFailuresTests",
+            dependencies: ["TestFailures"]
+        )
+    ]
+)
diff --git a/Fixtures/Miscellaneous/TestFailuresXCTest/Sources/TestFailures/TestFailures.swift b/Fixtures/Miscellaneous/TestFailuresXCTest/Sources/TestFailures/TestFailures.swift
@@ -0,0 +1,2 @@
+// The Swift Programming Language
+// https://docs.swift.org/swift-book
diff --git a/Fixtures/Miscellaneous/TestFailuresXCTest/Tests/TestFailuresTests/TestFailuresTests.swift b/Fixtures/Miscellaneous/TestFailuresXCTest/Tests/TestFailuresTests/TestFailuresTests.swift
@@ -0,0 +1,9 @@
+import XCTest
+@testable import TestFailures
+
+final class TestFailuresTests: XCTestCase {
+    func testExample() throws {
+        XCTAssertFalse(true, "Purposely failing & validating XML espace \"'<>")
+    }
+}
+
diff --git a/Sources/Commands/SwiftTestCommand.swift b/Sources/Commands/SwiftTestCommand.swift
@@ -1399,7 +1399,12 @@ final class XUnitGenerator {
                 """
 
             if !result.success {
-                content += "<failure message=\"failed\"></failure>\n"
+                var failureMessage = result.output
+                failureMessage.replace("&", with: "&amp;")
+                failureMessage.replace("\"", with:"&quot;")
+                failureMessage.replace(">", with: "&gt;")
+                failureMessage.replace("<", with: "&lt;")
+                content += "<failure message=\"\(failureMessage)\"></failure>\n"
             }
 
             content += "</testcase>\n"
diff --git a/Sources/_InternalTestSupport/SwiftPMProduct.swift b/Sources/_InternalTestSupport/SwiftPMProduct.swift
@@ -82,7 +82,8 @@ extension SwiftPM {
     public func execute(
         _ args: [String] = [],
         packagePath: AbsolutePath? = nil,
-        env: Environment? = nil
+        env: Environment? = nil,
+        errorIfCommandUnsuccessful: Bool = true
     ) async throws -> (stdout: String, stderr: String) {
         let result = try await executeProcess(
             args,
@@ -93,8 +94,11 @@ extension SwiftPM {
         let stdout = try result.utf8Output()
         let stderr = try result.utf8stderrOutput()
         
+        let returnValue = (stdout: stdout, stderr: stderr)
+        if (!errorIfCommandUnsuccessful) { return returnValue }
+
         if result.exitStatus == .terminated(code: 0) {
-            return (stdout: stdout, stderr: stderr)
+            return returnValue
         }
         throw SwiftPMError.executionFailure(
             underlying: AsyncProcessResult.Error.nonZeroExit(result),
diff --git a/Tests/CommandsTests/TestCommandTests.swift b/Tests/CommandsTests/TestCommandTests.swift
@@ -17,8 +17,12 @@ import _InternalTestSupport
 import XCTest
 
 final class TestCommandTests: CommandsTestCase {
-    private func execute(_ args: [String], packagePath: AbsolutePath? = nil) async throws -> (stdout: String, stderr: String) {
-        try await SwiftPM.Test.execute(args, packagePath: packagePath)
+    private func execute(
+        _ args: [String],
+        packagePath: AbsolutePath? = nil,
+        errorIfCommandUnsuccessful: Bool = true
+    ) async throws -> (stdout: String, stderr: String) {
+        try await SwiftPM.Test.execute(args, packagePath: packagePath, errorIfCommandUnsuccessful: errorIfCommandUnsuccessful)
     }
 
     func testUsage() async throws {
@@ -157,6 +161,71 @@ final class TestCommandTests: CommandsTestCase {
         }
     }
 
+    enum TestRunner {
+        case XCTest
+        case SwiftTesting
+
+        var fileSuffix: String {
+            switch self {
+                case .XCTest: return ""
+                case .SwiftTesting: return "-swift-testing"
+            }
+        }
+    }
+    func _testSwiftTestXMLOutputFailureMessage(
+        fixtureName: String,
+        testRunner: TestRunner
+    ) async throws {
+        try await fixture(name: fixtureName) { fixturePath in
+            // GIVEN we have a Package with a failing \(testRunner) test cases
+            let xUnitOutput = fixturePath.appending("result.xml")
+            let xUnitUnderTest = fixturePath.appending("result\(testRunner.fileSuffix).xml")
+
+            // WHEN we execute swift-test in parallel while specifying xUnit generation
+            _ = try await execute(
+                [
+                    "--parallel",
+                    "--verbose",
+                    "--enable-swift-testing",
+                    "--enable-xctest",
+                    "--xunit-output",
+                    xUnitOutput.pathString
+                ],
+                packagePath: fixturePath,
+                errorIfCommandUnsuccessful: false
+            )
+
+            // THEN we expect \(xUnitUnderTest) to exists
+            XCTAssertFileExists(xUnitUnderTest)
+            let contents: String = try localFileSystem.readFileContents(xUnitUnderTest)
+            // AND that the xUnit file has the expected contents"
+            XCTAssertMatch(
+                contents,
+                .contains("Purposely failing &amp; validating XML espace &quot;'&lt;&gt;")
+            )
+        }
+    }
+
+    func testSwiftTestXMLOutputVerifyFailureMessageXCTest() async throws {
+        try await self._testSwiftTestXMLOutputFailureMessage(
+            fixtureName: "Miscellaneous/TestFailuresXCTest",
+            testRunner: .XCTest
+        )
+    }
+
+    func testSwiftTestXMLOutputVerifyFailureMessageSwiftTesting() async throws {
+        #if compiler(<6)
+        _ = XCTSkip("SwifT Testing is not available by default in this Swift compiler version")
+
+        #else
+
+        try await self._testSwiftTestXMLOutputFailureMessage(
+            fixtureName: "Miscellaneous/TestFailuresSwiftTesting",
+            testRunner: .SwiftTesting
+        )
+        #endif
+    }
+
     func testSwiftTestFilter() async throws {
         try await fixture(name: "Miscellaneous/SkipTests") { fixturePath in
             let (stdout, _) = try await SwiftPM.Test.execute(["--filter", ".*1"], packagePath: fixturePath)

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+// The Swift Programming Language`
	`2`	`+// https://docs.swift.org/swift-book`