Skip to content

Re-enable APIDiff tests #8196

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Jan 9, 2025
Merged

Re-enable APIDiff tests #8196

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
db71a25
Update condition for skipping APIDiff tests.
yyvch Jan 3, 2025
5d12c02
Fix the tests: update asserts to be less brittle.
yyvch Jan 4, 2025
2b8437d
Make APIDiff tests opt-out.
yyvch Jan 4, 2025
8e9cdbf
Use more specific expected messages for asserts.
yyvch Jan 7, 2025
52348a8
Add TODO comment to cleanup the environemnt variable.
yyvch Jan 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 31 additions & 41 deletions Tests/CommandsTests/APIDiffTests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,25 +39,27 @@ final class APIDiffTests: CommandsTestCase {

func skipIfApiDigesterUnsupportedOrUnset() throws {
try skipIfApiDigesterUnsupported()
// The following is added to separate out the integration point testing of the API
// diff digester with SwiftPM from the functionality tests of the digester itself
guard Environment.current["SWIFTPM_TEST_API_DIFF_OUTPUT"] == "1" else {
throw XCTSkip("Env var SWIFTPM_TEST_API_DIFF_OUTPUT must be set to test the output")
}
// Opt out from testing the API diff if necessary.
// TODO: Cleanup after March 2025.
// The opt-in/opt-out mechanism doesn't seem to be used.
// It is kept around for abundance of caution. If "why is it needed"
// not identified by March 2025, then it is OK to remove it.
try XCTSkipIf(
Environment.current["SWIFTPM_TEST_API_DIFF_OUTPUT"] == "0",
"Env var SWIFTPM_TEST_API_DIFF_OUTPUT is set to skip the API diff tests."
)
}

func skipIfApiDigesterUnsupported() throws {
// swift-api-digester is required to run tests.
guard (try? UserToolchain.default.getSwiftAPIDigester()) != nil else {
throw XCTSkip("swift-api-digester unavailable")
}
// SwiftPM's swift-api-digester integration relies on post-5.5 bugfixes and features,
// not all of which can be tested for easily. Fortunately, we can test for the
// `-disable-fail-on-error` option, and any version which supports this flag
// will meet the other requirements.
guard DriverSupport.checkSupportedFrontendFlags(flags: ["disable-fail-on-error"], toolchain: try UserToolchain.default, fileSystem: localFileSystem) else {
throw XCTSkip("swift-api-digester is too old")
}
// The tests rely on swift-api-digester post-5.5 version and are certain
// to work with Swift compiler v6.0 and later.
#if compiler(<6.0)
throw XCTSkip("Skipping because test requires at least Swift compiler v6.0")
#endif
}

func testInvokeAPIDiffDigester() async throws {
Expand Down Expand Up @@ -165,18 +167,13 @@ final class APIDiffTests: CommandsTestCase {
string: "public class Qux<T, U> { private let x = 1 }"
)
await XCTAssertThrowsCommandExecutionError(try await execute(["diagnose-api-breaking-changes", "1.2.3"], packagePath: packageRoot)) { error in
XCTAssertMatch(error.stdout, .contains("1 breaking change detected in Foo"))
XCTAssertMatch(error.stdout, .contains("💔 API breakage: struct Foo has been removed"))
XCTAssertMatch(error.stdout, .contains("2 breaking changes detected in Bar"))
XCTAssertMatch(error.stdout, .contains("💔 API breakage: import Baz has been removed"))
XCTAssertMatch(error.stdout, .contains("💔 API breakage: func bar() has been removed"))
XCTAssertMatch(error.stdout, .contains("1 breaking change detected in Baz"))
XCTAssertMatch(error.stdout, .contains("💔 API breakage: enumelement Baz.b has been added as a new enum case"))
XCTAssertMatch(error.stdout, .contains("💔 API breakage"))
XCTAssertMatch(error.stdout, .regex("\\d+ breaking change(s?) detected in Foo"))
XCTAssertMatch(error.stdout, .regex("\\d+ breaking change(s?) detected in Bar"))
XCTAssertMatch(error.stdout, .regex("\\d+ breaking change(s?) detected in Baz"))

// Qux is not part of a library product, so any API changes should be ignored
XCTAssertNoMatch(error.stdout, .contains("2 breaking changes detected in Qux"))
XCTAssertNoMatch(error.stdout, .contains("💔 API breakage: class Qux has generic signature change from <T> to <T, U>"))
XCTAssertNoMatch(error.stdout, .contains("💔 API breakage: var Qux.x has been removed"))
XCTAssertNoMatch(error.stdout, .contains("Qux"))
}
}
}
Expand Down Expand Up @@ -204,33 +201,26 @@ final class APIDiffTests: CommandsTestCase {
await XCTAssertThrowsCommandExecutionError(
try await execute(["diagnose-api-breaking-changes", "1.2.3", "--products", "One", "--targets", "Bar"], packagePath: packageRoot)
) { error in
XCTAssertMatch(error.stdout, .contains("1 breaking change detected in Foo"))
XCTAssertMatch(error.stdout, .contains("💔 API breakage: struct Foo has been removed"))
XCTAssertMatch(error.stdout, .contains("2 breaking changes detected in Bar"))
XCTAssertMatch(error.stdout, .contains("💔 API breakage: import Baz has been removed"))
XCTAssertMatch(error.stdout, .contains("💔 API breakage: func bar() has been removed"))
XCTAssertMatch(error.stdout, .contains("💔 API breakage"))
XCTAssertMatch(error.stdout, .regex("\\d+ breaking change(s?) detected in Foo"))
XCTAssertMatch(error.stdout, .regex("\\d+ breaking change(s?) detected in Bar"))

XCTAssertNoMatch(error.stdout, .contains("1 breaking change detected in Baz"))
XCTAssertNoMatch(error.stdout, .contains("💔 API breakage: enumelement Baz.b has been added as a new enum case"))
XCTAssertNoMatch(error.stdout, .contains("2 breaking changes detected in Qux"))
XCTAssertNoMatch(error.stdout, .contains("💔 API breakage: class Qux has generic signature change from <T> to <T, U>"))
XCTAssertNoMatch(error.stdout, .contains("💔 API breakage: var Qux.x has been removed"))
// Baz and Qux are not included in the filter, so any API changes should be ignored.
XCTAssertNoMatch(error.stdout, .contains("Baz"))
XCTAssertNoMatch(error.stdout, .contains("Qux"))
}

// Diff a target which didn't have a baseline generated as part of the first invocation
await XCTAssertThrowsCommandExecutionError(
try await execute(["diagnose-api-breaking-changes", "1.2.3", "--targets", "Baz"], packagePath: packageRoot)
) { error in
XCTAssertMatch(error.stdout, .contains("1 breaking change detected in Baz"))
XCTAssertMatch(error.stdout, .contains("💔 API breakage: enumelement Baz.b has been added as a new enum case"))
XCTAssertMatch(error.stdout, .contains("💔 API breakage"))
XCTAssertMatch(error.stdout, .regex("\\d+ breaking change(s?) detected in Baz"))

XCTAssertNoMatch(error.stdout, .contains("1 breaking change detected in Foo"))
XCTAssertNoMatch(error.stdout, .contains("💔 API breakage: struct Foo has been removed"))
XCTAssertNoMatch(error.stdout, .contains("1 breaking change detected in Bar"))
XCTAssertNoMatch(error.stdout, .contains("💔 API breakage: func bar() has been removed"))
XCTAssertNoMatch(error.stdout, .contains("2 breaking changes detected in Qux"))
XCTAssertNoMatch(error.stdout, .contains("💔 API breakage: class Qux has generic signature change from <T> to <T, U>"))
XCTAssertNoMatch(error.stdout, .contains("💔 API breakage: var Qux.x has been removed"))
// Only Baz is included, we should not see any other API changes.
XCTAssertNoMatch(error.stdout, .contains("Foo"))
XCTAssertNoMatch(error.stdout, .contains("Bar"))
XCTAssertNoMatch(error.stdout, .contains("Qux"))
}

// Test diagnostics
Expand Down