Skip to content

Emit a specific exit code when no tests match inputs to swift test. #536

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jul 15, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,11 @@ private func entryPoint(

let eventHandler = try eventHandlerForStreamingEvents(version: args?.eventStreamVersion, forwardingTo: recordHandler)
let exitCode = await entryPoint(passing: args, eventHandler: eventHandler)

// To maintain compatibility with Xcode 16 Beta 1, suppress custom exit codes.
if exitCode == EXIT_NO_TESTS_FOUND {
return EXIT_SUCCESS
}
return exitCode
}
#endif
19 changes: 18 additions & 1 deletion Sources/Testing/ABI/EntryPoints/EntryPoint.swift
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,13 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha
}
}

// The set of matching tests (or, in the case of `swift test list`, the set
// of all tests.)
let tests: [Test]

if args.listTests ?? false {
let tests = await Test.all
tests = await Array(Test.all)

if args.verbosity > .min {
for testID in listTestsForEntryPoint(tests) {
// Print the test ID to stdout (classical CLI behavior.)
Expand All @@ -95,8 +100,20 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha
} else {
// Run the tests.
let runner = await Runner(configuration: configuration)
tests = runner.tests
await runner.run()
}

// If there were no matching tests, exit with a dedicated exit code so that
// the caller (assumed to be Swift Package Manager) can implement special
// handling.
if tests.isEmpty {
exitCode.withLock { exitCode in
if exitCode == EXIT_SUCCESS {
exitCode = EXIT_NO_TESTS_FOUND
}
}
}
} catch {
#if !SWT_NO_FILE_IO
try? FileHandle.stderr.write(String(describing: error))
Expand Down
24 changes: 24 additions & 0 deletions Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,30 @@
private import _TestingInternals
#endif

/// The exit code returned to Swift Package Manager by Swift Testing when no
/// tests matched the inputs specified by the developer (or, for the case of
/// `swift test list`, when no tests were found.)
///
/// Because Swift Package Manager does not directly link to the testing library,
/// it duplicates the definition of this constant in its own source. Any changes
/// to this constant in either package must be mirrored in the other.
///
/// Tools authors using the ABI entry point function can determine if no tests
/// matched the developer's inputs by counting the number of test records passed
/// to the event handler or written to the event stream output path.
///
/// This constant is not part of the public interface of the testing library.
var EXIT_NO_TESTS_FOUND: CInt {
#if SWT_TARGET_OS_APPLE || os(Linux)
EX_UNAVAILABLE
#elseif os(Windows)
ERROR_NOT_FOUND
#else
#warning("Platform-specific implementation missing: value for EXIT_NO_TESTS_FOUND unavailable")
return 2 // We're assuming that EXIT_SUCCESS = 0 and EXIT_FAILURE = 1.
#endif
}

/// The entry point to the testing library used by Swift Package Manager.
///
/// - Parameters:
Expand Down
4 changes: 4 additions & 0 deletions Sources/_TestingInternals/include/Includes.h
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,10 @@
#include <dlfcn.h>
#endif

#if __has_include(<sysexits.h>)
#include <sysexits.h>
#endif

#if defined(_WIN32)
#define WIN32_LEAN_AND_MEAN
#define NOMINMAX
Expand Down
14 changes: 14 additions & 0 deletions Tests/TestingTests/SwiftPMTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,12 @@ struct SwiftPMTests {
#expect(!CommandLine.arguments.isEmpty)
}

@Test("EXIT_NO_TESTS_FOUND is unique")
func valueOfEXIT_NO_TESTS_FOUND() {
#expect(EXIT_NO_TESTS_FOUND != EXIT_SUCCESS)
#expect(EXIT_NO_TESTS_FOUND != EXIT_FAILURE)
}

@Test("--parallel/--no-parallel argument")
func parallel() throws {
var configuration = try configurationForEntryPoint(withArguments: ["PATH"])
Expand Down Expand Up @@ -88,6 +94,14 @@ struct SwiftPMTests {
}
}

@Test("--filter with no matches")
func filterWithNoMatches() async {
var args = __CommandLineArguments_v0()
args.filter = ["NOTHING_MATCHES_THIS_TEST_NAME_HOPEFULLY"]
let exitCode = await __swiftPMEntryPoint(passing: args) as CInt
#expect(exitCode == EXIT_NO_TESTS_FOUND)
}

@Test("--skip argument")
@available(_regexAPI, *)
func skip() async throws {
Expand Down