Allow arguments to tests to be gathered asynchronously.

This change allows a developer to specify an argument to a test that is the result of an asynchronous function/operation. For example:

```swift
@test(arguments: await downloadFilesFromInternet())
func readFile(file: File) {
  ...
}
```

These arguments are resolved at runtime during the test planning stage.

Author: grynspan

Sources/Testing/Test+Macro.swift

@@ -542,7 +542,7 @@ public func __invokeXCTestCaseMethod<T>(
 @_alwaysEmitConformanceMetadata
 public protocol __TestContainer {
   /// The set of tests contained by this type.
-  static var __tests: [Test] { get }
+  static var __tests: [Test] { get async }
 }
 
 extension Test {
@@ -554,38 +554,42 @@ extension Test {
   ///
   /// The order of values in this sequence is unspecified.
   static var all: some Sequence<Test> {
-    // Convert the raw sequence of tests to a dictionary keyed by ID.
-    var result = testsByID(_all)
+    get async {
+      // Convert the raw sequence of tests to a dictionary keyed by ID.
+      var result = await testsByID(_all)
 
-    // Ensure test suite types that don't have the @Suite attribute are still
-    // represented in the result.
-    _synthesizeSuiteTypes(into: &result)
+      // Ensure test suite types that don't have the @Suite attribute are still
+      // represented in the result.
+      _synthesizeSuiteTypes(into: &result)
 
-    return result.values
+      return result.values
+    }
   }
 
   /// All available ``Test`` instances in the process, according to the runtime.
   ///
   /// The order of values in this sequence is unspecified. This sequence may
   /// contain duplicates; callers should use ``all`` instead.
-  private static var _all: some Sequence<Test> {
-    var result = [Self]()
-
-    withUnsafeMutablePointer(to: &result) { result in
-      swt_enumerateTypes({ typeName, _ in
-        // strstr() lets us avoid copying either string before comparing.
-        Self._testContainerTypeNameMagic.withCString { testContainerTypeNameMagic in
-          nil != strstr(typeName, testContainerTypeNameMagic)
-        }
-      }, /*typeEnumerator:*/ { type, context in
-        if let context, let type = unsafeBitCast(type, to: Any.Type.self) as? any __TestContainer.Type {
-          let result = context.assumingMemoryBound(to: Array<Self>.self)
-          result.pointee.append(contentsOf: type.__tests)
-        }
-      }, result)
+  private static var _all: some Sequence<Self> {
+    get async {
+      await withTaskGroup(of: [Self].self) { taskGroup in
+        swt_enumerateTypes({ typeName, _ in
+          // strstr() lets us avoid copying either string before comparing.
+          Self._testContainerTypeNameMagic.withCString { testContainerTypeNameMagic in
+            nil != strstr(typeName, testContainerTypeNameMagic)
+          }
+        }, /*typeEnumerator:*/ { type, context in
+          if let context, let type = unsafeBitCast(type, to: Any.Type.self) as? any __TestContainer.Type {
+            let taskGroup = context.assumingMemoryBound(to: ThrowingTaskGroup<[Self], any Error>.self)
+            taskGroup.pointee.addTask {
+              return await type.__tests
+            }
+          }
+        }, &taskGroup)
+
+        return await taskGroup.reduce(into: [], +=)
+      }
     }
-
-    return result
   }
 
   /// Create a dictionary mapping the IDs of a sequence of tests to those tests.

Sources/TestingMacros/TestDeclarationMacro.swift

@@ -480,16 +480,18 @@ public struct TestDeclarationMacro: PeerMacro, Sendable {
       @available(*, unavailable, message: "This type is an implementation detail of the testing library. It cannot be used directly.")
       @available(*, deprecated)
       @frozen public enum \(enumName): Testing.__TestContainer {
-        public static var __tests: [Testing.Test] {[
-          .__function(
-            named: \(literal: functionDecl.completeName),
-            in: \(typealiasExpr),
-            xcTestCompatibleSelector: \(selectorExpr ?? "nil"),
-            \(raw: attributeInfo.functionArgumentList(in: context)),
-            parameters: \(raw: functionDecl.testFunctionParameterList),
-            testFunction: \(thunkDecl.name)
-          )
-        ]}
+        public static var __tests: [Testing.Test] {
+          get async {[
+            .__function(
+              named: \(literal: functionDecl.completeName),
+              in: \(typealiasExpr),
+              xcTestCompatibleSelector: \(selectorExpr ?? "nil"),
+              \(raw: attributeInfo.functionArgumentList(in: context)),
+              parameters: \(raw: functionDecl.testFunctionParameterList),
+              testFunction: \(thunkDecl.name)
+            )
+          ]}
+        }
       }
       """
     )

Tests/TestingTests/MiscellaneousTests.swift

@@ -172,28 +172,35 @@ struct TestsWithStaticMemberAccessBySelfKeyword {
 @Test(.hidden, arguments: [0]) func A(🙃: Int) {}
 @Test(.hidden, arguments: [0]) func A(🙂: Int) {}
 
+@Suite(.hidden)
+struct TestsWithAsyncArguments {
+  static func asyncCollection() async -> [Int] { [] }
+
+  @Test(.hidden, arguments: await asyncCollection()) func f(i: Int) {}
+}
+
 @Suite("Miscellaneous tests")
 struct MiscellaneousTests {
   @Test("Free function's name")
   func unnamedFreeFunctionTest() async throws {
-    let testFunction = try #require(Test.all.first(where: { $0.name.contains("freeSyncFunction") }))
+    let testFunction = try #require(await Test.all.first(where: { $0.name.contains("freeSyncFunction") }))
     #expect(testFunction.name == "freeSyncFunction()")
   }
 
   @Test("Test suite type's name")
   func unnamedMemberFunctionTest() async throws {
-    let testType = try #require(test(for: SendableTests.self))
+    let testType = try #require(await test(for: SendableTests.self))
     #expect(testType.name == "SendableTests")
   }
 
   @Test("Free function has custom display name")
   func namedFreeFunctionTest() async throws {
-    #expect(Test.all.first { $0.displayName == "Named Free Sync Function" && !$0.isSuite && $0.containingType == nil } != nil)
+    #expect(await Test.all.first { $0.displayName == "Named Free Sync Function" && !$0.isSuite && $0.containingType == nil } != nil)
   }
 
   @Test("Member function has custom display name")
   func namedMemberFunctionTest() async throws {
-    let testType = try #require(test(for: NamedSendableTests.self))
+    let testType = try #require(await test(for: NamedSendableTests.self))
     #expect(testType.displayName == "Named Sendable test type")
   }
 
@@ -301,53 +308,53 @@ struct MiscellaneousTests {
   @Test("Test.underestimatedCaseCount property")
   func underestimatedCaseCount() async throws {
     do {
-      let test = try #require(testFunction(named: "parameterized(i:)", in: NonSendableTests.self))
+      let test = try #require(await testFunction(named: "parameterized(i:)", in: NonSendableTests.self))
       #expect(test.underestimatedCaseCount == FixtureData.zeroUpTo100.count)
     }
     do {
-      let test = try #require(testFunction(named: "parameterized2(i:j:)", in: NonSendableTests.self))
+      let test = try #require(await testFunction(named: "parameterized2(i:j:)", in: NonSendableTests.self))
       #expect(test.underestimatedCaseCount == FixtureData.zeroUpTo100.count * FixtureData.smallStringArray.count)
     }
     do {
-      let test = try #require(testFunction(named: "parameterized(i:)", in: SendableTests.self))
+      let test = try #require(await testFunction(named: "parameterized(i:)", in: SendableTests.self))
       #expect(test.underestimatedCaseCount == FixtureData.zeroUpTo100.count)
     }
 #if !SWT_NO_GLOBAL_ACTORS
     do {
-      let test = try #require(testFunction(named: "parameterized(i:)", in: MainActorIsolatedTests.self))
+      let test = try #require(await testFunction(named: "parameterized(i:)", in: MainActorIsolatedTests.self))
       #expect(test.underestimatedCaseCount == FixtureData.zeroUpTo100.count)
     }
     do {
-      let test = try #require(testFunction(named: "parameterizedNonisolated(i:)", in: MainActorIsolatedTests.self))
+      let test = try #require(await testFunction(named: "parameterizedNonisolated(i:)", in: MainActorIsolatedTests.self))
       #expect(test.underestimatedCaseCount == FixtureData.zeroUpTo100.count)
     }
 #endif
 
     do {
-      let thisTest = try #require(testFunction(named: "succeeds()", in: SendableTests.self))
+      let thisTest = try #require(await testFunction(named: "succeeds()", in: SendableTests.self))
       #expect(thisTest.underestimatedCaseCount == 1)
     }
     do {
-      let thisTest = try #require(test(for: SendableTests.self))
+      let thisTest = try #require(await test(for: SendableTests.self))
       #expect(thisTest.underestimatedCaseCount == nil)
     }
   }
 
   @Test("Test.parameters property")
   func parametersProperty() async throws {
     do {
-      let theTest = try #require(test(for: SendableTests.self))
+      let theTest = try #require(await test(for: SendableTests.self))
       #expect(theTest.parameters == nil)
     }
 
     do {
-      let test = try #require(testFunction(named: "succeeds()", in: SendableTests.self))
+      let test = try #require(await testFunction(named: "succeeds()", in: SendableTests.self))
       let parameters = try #require(test.parameters)
       #expect(parameters.isEmpty)
     } catch {}
 
     do {
-      let test = try #require(testFunction(named: "parameterized(i:)", in: NonSendableTests.self))
+      let test = try #require(await testFunction(named: "parameterized(i:)", in: NonSendableTests.self))
       let parameters = try #require(test.parameters)
       #expect(parameters.count == 1)
       let firstParameter = try #require(parameters.first)
@@ -356,7 +363,7 @@ struct MiscellaneousTests {
     } catch {}
 
     do {
-      let test = try #require(testFunction(named: "parameterized2(i:j:)", in: NonSendableTests.self))
+      let test = try #require(await testFunction(named: "parameterized2(i:j:)", in: NonSendableTests.self))
       let parameters = try #require(test.parameters)
       #expect(parameters.count == 2)
       let firstParameter = try #require(parameters.first)
@@ -476,7 +483,7 @@ struct MiscellaneousTests {
     let line = 12345
     let column = 67890
     let sourceLocation = SourceLocation(fileID: fileID, filePath: filePath, line: line, column: column)
-    let testFunction = Test.__function(named: "myTestFunction()", in: nil, xcTestCompatibleSelector: nil, displayName: nil, traits: [], sourceLocation: sourceLocation) {}
+    let testFunction = await Test.__function(named: "myTestFunction()", in: nil, xcTestCompatibleSelector: nil, displayName: nil, traits: [], sourceLocation: sourceLocation) {}
     #expect(String(describing: testFunction.id) == "Module/myTestFunction()/Y.swift:12345:67890")
   }
 

Tests/TestingTests/PlanTests.swift

@@ -14,10 +14,10 @@
 struct PlanTests {
   @Test("Selected tests")
   func selectedTests() async throws {
-    let outerTestType = try #require(test(for: SendableTests.self))
-    let testA = try #require(testFunction(named: "succeeds()", in: SendableTests.self))
-    let innerTestType = try #require(test(for: SendableTests.NestedSendableTests.self))
-    let testB = try #require(testFunction(named: "succeeds()", in: SendableTests.NestedSendableTests.self))
+    let outerTestType = try #require(await test(for: SendableTests.self))
+    let testA = try #require(await testFunction(named: "succeeds()", in: SendableTests.self))
+    let innerTestType = try #require(await test(for: SendableTests.NestedSendableTests.self))
+    let testB = try #require(await testFunction(named: "succeeds()", in: SendableTests.NestedSendableTests.self))
 
     let tests = [
       outerTestType,
@@ -38,10 +38,10 @@ struct PlanTests {
 
   @Test("Multiple selected tests")
   func multipleSelectedTests() async throws {
-    let outerTestType = try #require(test(for: SendableTests.self))
-    let testA = try #require(testFunction(named: "succeeds()", in: SendableTests.self))
-    let innerTestType = try #require(test(for: SendableTests.NestedSendableTests.self))
-    let testB = try #require(testFunction(named: "succeeds()", in: SendableTests.NestedSendableTests.self))
+    let outerTestType = try #require(await test(for: SendableTests.self))
+    let testA = try #require(await testFunction(named: "succeeds()", in: SendableTests.self))
+    let innerTestType = try #require(await test(for: SendableTests.NestedSendableTests.self))
+    let testB = try #require(await testFunction(named: "succeeds()", in: SendableTests.NestedSendableTests.self))
 
     let tests = [
       outerTestType,
@@ -63,9 +63,9 @@ struct PlanTests {
 
   @Test("Recursive trait application")
   func recursiveTraitApplication() async throws {
-    let outerTestType = try #require(test(for: OuterTest.self))
+    let outerTestType = try #require(await test(for: OuterTest.self))
     // Intentionally omitting intermediate tests here...
-    let deeplyNestedTest = try #require(testFunction(named: "example()", in: OuterTest.IntermediateType.InnerTest.self))
+    let deeplyNestedTest = try #require(await testFunction(named: "example()", in: OuterTest.IntermediateType.InnerTest.self))
 
     let tests = [outerTestType, deeplyNestedTest]
 
@@ -80,10 +80,10 @@ struct PlanTests {
 
   @Test("Relative order of recursively applied traits")
   func recursiveTraitOrder() async throws {
-    let testSuiteA = try #require(test(for: RelativeTraitOrderingTests.A.self))
-    let testSuiteB = try #require(test(for: RelativeTraitOrderingTests.A.B.self))
-    let testSuiteC = try #require(test(for: RelativeTraitOrderingTests.A.B.C.self))
-    let testFuncX = try #require(testFunction(named: "x()", in: RelativeTraitOrderingTests.A.B.C.self))
+    let testSuiteA = try #require(await test(for: RelativeTraitOrderingTests.A.self))
+    let testSuiteB = try #require(await test(for: RelativeTraitOrderingTests.A.B.self))
+    let testSuiteC = try #require(await test(for: RelativeTraitOrderingTests.A.B.C.self))
+    let testFuncX = try #require(await testFunction(named: "x()", in: RelativeTraitOrderingTests.A.B.C.self))
 
     let tests = [testSuiteA, testSuiteB, testSuiteC, testFuncX]
 

Tests/TestingTests/RunnerTests.swift

@@ -246,8 +246,8 @@ final class RunnerTests: XCTestCase {
   }
 
   func testConditionTraitsAreEvaluatedOutermostToInnermost() async throws {
-    let testSuite = try #require(test(for: NeverRunTests.self))
-    let testFunc = try #require(testFunction(named: "duelingConditions()", in: NeverRunTests.self))
+    let testSuite = try #require(await test(for: NeverRunTests.self))
+    let testFunc = try #require(await testFunction(named: "duelingConditions()", in: NeverRunTests.self))
 
     var configuration = Configuration()
     configuration.selectedTests = .init(testIDs: [testSuite.id])
@@ -313,11 +313,11 @@ final class RunnerTests: XCTestCase {
   }
 
   func testHardCodedPlan() async throws {
-    let tests = try [
-      XCTUnwrap(testFunction(named: "succeeds()", in: SendableTests.self)),
-      XCTUnwrap(testFunction(named: "succeedsAsync()", in: SendableTests.self)),
-      XCTUnwrap(testFunction(named: "succeeds()", in: SendableTests.NestedSendableTests.self)),
-    ]
+    let tests = try await [
+      testFunction(named: "succeeds()", in: SendableTests.self),
+      testFunction(named: "succeedsAsync()", in: SendableTests.self),
+      testFunction(named: "succeeds()", in: SendableTests.NestedSendableTests.self),
+    ].map { try XCTUnwrap($0) }
     let steps: [Runner.Plan.Step] = tests
       .map { .init(test: $0, action: .skip()) }
     let plan = Runner.Plan(steps: steps)

Tests/TestingTests/TestSupport/TestingAdditions.swift

@@ -21,8 +21,8 @@ import XCTest
 ///
 /// - Returns: The test instance representing the specified type, or `nil` if
 ///   none is found.
-func test(for containingType: Any.Type) -> Test? {
-  Test.all.first {
+func test(for containingType: Any.Type) async -> Test? {
+  await Test.all.first {
     $0.isSuite && $0.containingType == containingType
   }
 }
@@ -35,8 +35,8 @@ func test(for containingType: Any.Type) -> Test? {
 ///
 /// - Returns: The test instance representing the specified test function, or
 ///   `nil` if none is found.
-func testFunction(named name: String, in containingType: Any.Type) -> Test? {
-  Test.all.first {
+func testFunction(named name: String, in containingType: Any.Type) async -> Test? {
+  await Test.all.first {
     $0.name == name && !$0.isSuite && $0.containingType == containingType
   }
 }