test: accomodate new test runner

Also update normalization to be friendlier: keep digits, compress
underscores, and strip trailing underscores.

Author: rwoll

tests/report_downstream_test_diffs.py renamed to scripts/report_downstream_test_diffs.py

@@ -3,16 +3,22 @@
 import os
 import re
 import subprocess
-import tempfile
+import typing
 from collections import namedtuple
 
+from playwright.path_utils import get_file_dirname
+
+_dirname = get_file_dirname()
+
 TestCase = namedtuple("TestCase", ["api", "file", "test"])
 
 
-def pytest_test_cases():
+def pytest_test_cases() -> typing.Generator[TestCase, None, None]:
     p = subprocess.run(
         ["pytest", "--browser", "chromium", "--collect-only", "-q"],
+        cwd=_dirname / ".." / "tests",
         stdout=subprocess.PIPE,
+        check=True,
     )
     regex = (
         r"tests/(?P<api>a?sync)/test_(?P<file>.*)\.py::test_(?P<test>.*)\[chromium\]"
@@ -24,32 +30,44 @@ def pytest_test_cases():
         )
 
 
-def jest_test_cases(playwright_js_path):
-    env = os.environ.copy()
-    env["REPORT_ONLY"] = "true"
-    with tempfile.NamedTemporaryFile() as results:
-        subprocess.run(
-            ["npx", "jest", "--json", "--outputFile", results.name],
-            env=env,
-            cwd=playwright_js_path,
-            stderr=subprocess.DEVNULL,
-        )
+def jest_test_cases(playwright_js_path: str) -> typing.Generator[TestCase, None, None]:
+    p = subprocess.run(
+        [
+            "node",
+            os.path.join("test", "runner"),
+            "test",
+            "--trial-run",
+            "--reporter",
+            "json",
+        ],
+        cwd=playwright_js_path,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.DEVNULL,
+        check=True,
+    )
+
+    tests = json.loads(p.stdout.decode())
+    for test in [*tests["pending"], *tests["passes"], *tests["failures"]]:
+        regex = r"(.*/)?(?P<file>[^/]+)\.spec\.[jt]s$"
+
+        match = re.match(regex, test["file"])
+        if not match:
+            continue
+
+        file = match.group("file")
 
-        for test_suite in json.load(results)["testResults"]:
-            regex = r"(.*/)?(?P<file>[^/]+)\.spec\.[jt]s$"
-            file = re.match(regex, test_suite["name"]).group("file")
-            for assertion in test_suite["assertionResults"]:
-                yield TestCase("sync", normalized(file), normalized(assertion["title"]))
-                yield TestCase(
-                    "async", normalized(file), normalized(assertion["title"])
-                )
+        yield TestCase("sync", normalized(file), normalized(test["title"]))
+        yield TestCase("async", normalized(file), normalized(test["title"]))
 
 
-def normalized(original):
-    return re.sub(r"[^a-z_]", "_", original, flags=re.IGNORECASE)
+def normalized(original: str) -> str:
+    cleaned = re.sub(r"[^a-z0-9_]", "_", original, flags=re.IGNORECASE)
+    cleaned = re.sub(r"[_]+", "_", cleaned)
+    cleaned = cleaned.strip("_")
+    return cleaned
 
 
-def main():
+def main() -> None:
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "--playwright-js-path",
@@ -69,12 +87,13 @@ def main():
     javascript_tests = set(jest_test_cases(args.playwright_js_path))
 
     if args.api:
-        javascript_tests = set([x for x in javascript_tests if x[0] == args.api])
+        javascript_tests = set([x for x in javascript_tests if x.api == args.api])
 
     missing = javascript_tests.difference(python_tests)
     found = javascript_tests.intersection(python_tests)
 
     print("MISSING, MISPELLED, OR MISNAMED:")
+    print("=" * 80)
     for (api, file, test) in sorted(missing):
         print(f"{api}/test_{file}.py::test_{test}")