Generalize the selectedTests facility to be closure-based, to allow filtering tests via other criteria than ID (#48)

* Generalize the selectedTests facility to be closure-based, to allow filtering tests via other criteria than ID
---------

Co-authored-by: Suzy Ratcliff <suzyr@apple.com>

Author: stmontgomery

Sources/Testing/Running/Configuration.swift

@@ -137,38 +137,38 @@ public struct Configuration: Sendable {
 
   // MARK: - Test selection
 
-  /// The selected tests to run, if any.
+  /// A function that handles filtering tests.
   ///
-  /// This property should be used for testing membership (whether a test ID has
-  /// been selected) since it is more optimized for that use case. It also
-  /// provides the backing storage for ``selectedTestIDs``.
-  ///
-  /// This property is optional and defaults to `nil` because it is possible to
-  /// select specific tests to run but not provide any tests in that list. That
-  /// is a supported use case: it results in zero tests being run and no issues
-  /// recorded.
-  ///
-  /// A practical example of when this situation can happen is when testing is
-  /// configured via an Xcode Test Plan, the "Automatically Include New Tests"
-  /// option is disabled, and zero tests are enabled.
-  var selectedTests: Test.ID.Selection?
-
-  /// The IDs of the selected tests to run, if any.
-  ///
-  /// This property is optional and defaults to `nil` because it is possible to
-  /// select specific tests to run but not provide any tests in that list. That
-  /// is a supported use case: it results in zero tests being run and no issues
-  /// recorded.
+  /// - Parameters:
+  ///   - test: An test that needs to be filtered.
+  ///   
+  /// - Returns: A Boolean value representing if the test satisfied the filter.
+  public typealias TestFilter = @Sendable (Test) -> Bool
+
+  /// The test filter to which tests should be filtered when run.
+  public var testFilter: TestFilter?
+
+  /// The granularity to enforce test filtering.
+  /// 
+  /// By default, all tests are run and no filter is set.
+  /// - Parameters:
+  ///   - selection: An set of test ids to be filtered.
+  public mutating func setTestFilter(toMatch selection: Set<Test.ID>?) {
+      self.setTestFilter(toMatch: selection.map(Test.ID.Selection.init))
+  }
+  
+  /// The granularity to enforce test filtering.
   ///
-  /// A practical example of when this situation can happen is when testing is
-  /// configured via an Xcode Test Plan, the "Automatically Include New Tests"
-  /// option is disabled, and zero tests are enabled.
-  public var selectedTestIDs: Set<Test.ID>? {
-    get {
-      selectedTests?.testIDs
+  /// By default, all tests are run and no filter is set.
+  /// - Parameters:
+  ///   - selection: An selection of test ids to be filtered.
+  mutating func setTestFilter(toMatch selection: Test.ID.Selection?) {
+    guard let selectedTests = selection else {
+        self.testFilter = nil
+        return
     }
-    set {
-      selectedTests = newValue.map { .init(testIDs: $0) }
+    self.testFilter = { test in
+        selectedTests.contains(test)
     }
   }
 }

Sources/Testing/Running/Runner.Plan.swift

@@ -127,17 +127,16 @@ extension Runner.Plan {
   ///
   /// - Parameters:
   ///   - test: The test to query.
-  ///   - selectedTests: The selected test IDs to use in determining whether
-  ///     `test` is selected, if one is configured.
+  ///   - filter: The filter to decide if the test is included.
   ///
   /// - Returns: Whether or not the specified test is selected. If
   ///   `selectedTests` is `nil`, `test` is considered selected if it is not
   ///   hidden.
-  private static func _isTestIncluded(_ test: Test, in selectedTests: Test.ID.Selection?) -> Bool {
-    guard let selectedTests else {
+  private static func _isTestIncluded(_ test: Test, using filter: Configuration.TestFilter?) -> Bool {
+    guard let filter else {
       return !test.isHidden
     }
-    return selectedTests.contains(test)
+    return filter(test)
   }
 
   /// Construct a graph of runner plan steps for the specified tests.
@@ -160,8 +159,7 @@ extension Runner.Plan {
     // them, in which case it will be .recordIssue().
     var testGraph = Graph<String, Test?>()
     var actionGraph = Graph<String, Action>(value: .run)
-    let selectedTests = configuration.selectedTests
-    for test in tests where _isTestIncluded(test, in: selectedTests) {
+    for test in tests where _isTestIncluded(test, using: configuration.testFilter) {
       let idComponents = test.id.keyPathRepresentation
       testGraph.insertValue(test, at: idComponents)
       actionGraph.insertValue(.run, at: idComponents, intermediateValue: .run)

Tests/TestingTests/PlanTests.swift

@@ -7,7 +7,7 @@
 // See https://swift.org/LICENSE.txt for license information
 // See https://swift.org/CONTRIBUTORS.txt for Swift project authors
 //
-
+    
 @testable @_spi(ExperimentalTestRunning) import Testing
 
 @Suite("Runner.Plan Tests")
@@ -26,8 +26,9 @@ struct PlanTests {
       testB,
     ]
 
+    let selection = Test.ID.Selection(testIDs: [innerTestType.id])
     var configuration = Configuration()
-    configuration.selectedTestIDs = [innerTestType.id]
+    configuration.setTestFilter(toMatch: selection)
 
     let plan = await Runner.Plan(tests: tests, configuration: configuration)
     #expect(plan.steps.contains(where: { $0.test == outerTestType }))
@@ -51,7 +52,8 @@ struct PlanTests {
     ]
 
     var configuration = Configuration()
-    configuration.selectedTestIDs = [innerTestType.id, outerTestType.id]
+    let selection = Test.ID.Selection(testIDs: [innerTestType.id, outerTestType.id])
+    configuration.setTestFilter(toMatch: selection)
 
     let plan = await Runner.Plan(tests: tests, configuration: configuration)
     let planTests = plan.steps.map(\.test)
@@ -70,7 +72,10 @@ struct PlanTests {
     let tests = [outerTestType, deeplyNestedTest]
 
     var configuration = Configuration()
-    configuration.selectedTestIDs = [outerTestType.id, deeplyNestedTest.id]
+    let selection = Test.ID.Selection(testIDs: [outerTestType.id, deeplyNestedTest.id])
+    configuration.testFilter = { test in
+      selection.contains(test)
+    }
 
     let plan = await Runner.Plan(tests: tests, configuration: configuration)
 
@@ -88,7 +93,10 @@ struct PlanTests {
     let tests = [testSuiteA, testSuiteB, testSuiteC, testFuncX]
 
     var configuration = Configuration()
-    configuration.selectedTestIDs = [testSuiteA.id]
+    let selection = Test.ID.Selection(testIDs: [testSuiteA.id])
+    configuration.testFilter = { test in
+      selection.contains(test)
+    }
 
     let plan = await Runner.Plan(tests: tests, configuration: configuration)
     let testFuncXWithTraits = try #require(plan.steps.map(\.test).first { $0.name == "x()" })

Tests/TestingTests/RunnerTests.swift

@@ -250,7 +250,10 @@ final class RunnerTests: XCTestCase {
     let testFunc = try #require(await testFunction(named: "duelingConditions()", in: NeverRunTests.self))
 
     var configuration = Configuration()
-    configuration.selectedTests = .init(testIDs: [testSuite.id])
+    let selection = Test.ID.Selection(testIDs: [testSuite.id])
+    configuration.testFilter = { test in
+      selection.contains(test)
+    }
 
     let runner = await Runner(testing: [
       testSuite,
@@ -294,11 +297,16 @@ final class RunnerTests: XCTestCase {
       (SendableTests.self, "disabled()"),
     ]
 
-    var configuration = Configuration()
-    configuration.selectedTestIDs = Set(tests.map {
+    let selectedTestIDs = Set(tests.map {
       Test.ID(type: $0).child(named: $1)
     })
-    XCTAssertEqual(false, configuration.selectedTestIDs?.isEmpty)
+    XCTAssertFalse(selectedTestIDs.isEmpty)
+
+    var configuration = Configuration()
+    let selection = Test.ID.Selection(testIDs: selectedTestIDs)
+    configuration.testFilter = { test in
+      selection.contains(test)
+    }
 
     let runner = await Runner(configuration: configuration)
     let plan = runner.plan

Tests/TestingTests/TestSupport/TestingAdditions.swift

@@ -67,7 +67,10 @@ func runTest(for containingType: Any.Type, configuration: Configuration = .init(
 /// If no test is found representing `containingType`, nothing is run.
 func runTestFunction(named name: String, in containingType: Any.Type, configuration: Configuration = .init()) async {
   var configuration = configuration
-  configuration.selectedTestIDs = [Test.ID(type: containingType).child(named: name)]
+  let testID = Test.ID.Selection(testIDs: [Test.ID(type: containingType).child(named: name)])
+  configuration.testFilter = { test in
+    testID.contains(test)
+  }
 
   let runner = await Runner(configuration: configuration)
   await runner.run()
@@ -90,7 +93,8 @@ extension Runner {
     let moduleName = String(fileID[..<fileID.lastIndex(of: "/")!])
 
     var configuration = configuration
-    configuration.selectedTestIDs = [Test.ID(moduleName: moduleName, nameComponents: [testName], sourceLocation: nil)]
+    let selection = Test.ID.Selection(testIDs: [Test.ID(moduleName: moduleName, nameComponents: [testName], sourceLocation: nil)])
+    configuration.setTestFilter(toMatch: selection)
 
     await self.init(configuration: configuration)
   }
@@ -104,7 +108,10 @@ extension Runner.Plan {
   ///   - configuration: The configuration to use for planning.
   init(selecting containingType: Any.Type, configuration: Configuration = .init()) async {
     var configuration = configuration
-    configuration.selectedTestIDs = [Test.ID(type: containingType)]
+    let selection = Test.ID.Selection(testIDs: [Test.ID(type: containingType)])
+    configuration.testFilter = { test in
+      selection.contains(test)
+    }
 
     await self.init(configuration: configuration)
   }