Automatically discover dynamically defined tests

vscode/src/streamingRunner.ts

@@ -15,7 +15,7 @@ type TestEventWithMessage = TestEventId & { message: string };
 
 // All notification types that may be produce by our custom JSON test reporter
 const NOTIFICATION_TYPES = {
-  start: new rpc.NotificationType<TestEventId>("start"),
+  start: new rpc.NotificationType<TestEventId & { line: number }>("start"),
   pass: new rpc.NotificationType<TestEventId>("pass"),
   skip: new rpc.NotificationType<TestEventId>("skip"),
   fail: new rpc.NotificationType<TestEventWithMessage>("fail"),
@@ -37,6 +37,7 @@ export class StreamingRunner implements vscode.Disposable {
   private readonly findTestItem: (
     id: string,
     uri: vscode.Uri,
+    line?: number,
   ) => Promise<vscode.TestItem | undefined>;
 
   private readonly tcpServer: net.Server;
@@ -284,14 +285,16 @@ export class StreamingRunner implements vscode.Disposable {
     this.disposables.push(
       this.connection.onNotification(NOTIFICATION_TYPES.start, (params) => {
         this.promises.push(
-          this.findTestItem(params.id, vscode.Uri.parse(params.uri)).then(
-            (test) => {
-              if (test) {
-                this.run!.started(test);
-                startTimestamps.set(test.id, Date.now());
-              }
-            },
-          ),
+          this.findTestItem(
+            params.id,
+            vscode.Uri.parse(params.uri),
+            params.line,
+          ).then((test) => {
+            if (test) {
+              this.run!.started(test);
+              startTimestamps.set(test.id, Date.now());
+            }
+          }),
         );
       }),
     );

vscode/src/test/suite/testController.test.ts

@@ -866,4 +866,38 @@ suite("TestController", () => {
     assert.ok(runStub.passed.calledWith(testItem));
     assert.ok(runStub.end.calledWithExactly());
   }).timeout(10000);
+
+  test("findTestItem creates items automatically when line is passed", async () => {
+    assert.strictEqual(
+      "ServerTest",
+      (await controller.findTestItem("ServerTest", serverTestUri))!.id,
+    );
+    assert.strictEqual(
+      "ServerTest::NestedTest",
+      (await controller.findTestItem("ServerTest::NestedTest", serverTestUri))!
+        .id,
+    );
+
+    const dynamicallyDefinedTest = await controller.findTestItem(
+      "ServerTest#test_dynamically_defined",
+      serverTestUri,
+      32,
+    );
+
+    assert.strictEqual(
+      "ServerTest#test_dynamically_defined",
+      dynamicallyDefinedTest!.id,
+    );
+
+    const dynamicallyDefinedNestedTest = await controller.findTestItem(
+      "ServerTest::NestedTest#test_nested_dynamically_defined",
+      serverTestUri,
+      32,
+    );
+
+    assert.strictEqual(
+      "ServerTest::NestedTest#test_nested_dynamically_defined",
+      dynamicallyDefinedNestedTest!.id,
+    );
+  });
 });

vscode/src/testController.ts

@@ -526,8 +526,10 @@ export class TestController {
     linkedCancellationSource.dispose();
   }
 
-  // Public for testing purposes. Finds a test item based on its ID and URI
-  async findTestItem(id: string, uri: vscode.Uri) {
+  // Public for testing purposes. Finds a test item based on its ID and URI. If a line number is provided, this method
+  // will automatically create the test item if it cannot be found. This behavior is used to support dynamically defined
+  // tests
+  async findTestItem(id: string, uri: vscode.Uri, line?: number) {
     if (this.testController.items.size === 0) {
       // Discover and test items immediately if the test explorer hasn't been expanded
       await this.resolveHandler(undefined);
@@ -564,7 +566,7 @@ export class TestController {
     }
 
     // If not, the ID might be nested under groups
-    return this.findTestInGroup(id, testFileItem);
+    return this.findTestInGroup(id, testFileItem, line);
   }
 
   // Execute all of the test commands reported by the server in the background using JSON RPC to receive streaming
@@ -651,6 +653,7 @@ export class TestController {
   private findTestInGroup(
     id: string,
     group: vscode.TestItem,
+    line: number | undefined,
   ): vscode.TestItem | undefined {
     let found: vscode.TestItem | undefined;
 
@@ -664,7 +667,39 @@ export class TestController {
       return;
     }
 
-    return found.children.get(id) ?? this.findTestInGroup(id, found);
+    // If we found the exact item, return it
+    const target = found.children.get(id);
+    if (target) {
+      return target;
+    }
+
+    // If the ID we're looking for starts with the found item's ID suffixed by a `::`, it means that there are more
+    // groups nested inside and we can continue searching
+    if (id.startsWith(`${found.id}::`)) {
+      return this.findTestInGroup(id, found, line);
+    }
+
+    if (!line) {
+      return;
+    }
+
+    // If neither of the previous are true, then this test is dynamically defined and we need to create the items for it
+    // automatically
+    const label = id.split("#")[1]!;
+    const testItem = this.testController.createTestItem(id, label, found.uri);
+
+    testItem.range = new vscode.Range(
+      new vscode.Position(line, 0),
+      new vscode.Position(line, 1),
+    );
+
+    const frameworkTag = found.tags.find((tag) =>
+      tag.id.startsWith("framework"),
+    );
+
+    testItem.tags = frameworkTag ? [DEBUG_TAG, frameworkTag] : [DEBUG_TAG];
+    found.children.add(testItem);
+    return testItem;
   }
 
   // Get an existing terminal or create a new one. For multiple workspaces, it's important to create a new terminal for