Conformance test suite: Reduce change of merge conflicts (#1686)

- Provide option to skip updating timing data
- Split HTML report into multiple lines

This makes it far less likely that two concurrent PRs to different test cases will have merge conflicts with each other.

conformance/src/main.py

@@ -22,6 +22,7 @@ def run_tests(
     root_dir: Path,
     type_checker: TypeChecker,
     test_cases: Sequence[Path],
+    skip_timing: bool = False,
 ):
     print(f"Running tests for {type_checker.name}")
 
@@ -39,7 +40,7 @@ def run_tests(
             type_checker, results_dir, test_case, tests_output.get(test_case.name, "")
         )
 
-    update_type_checker_info(type_checker, root_dir, test_duration)
+    update_type_checker_info(type_checker, root_dir, test_duration, skip_timing=skip_timing)
 
 
 def get_expected_errors(test_case: Path) -> tuple[
@@ -199,7 +200,7 @@ def update_output_for_test(
 
 
 def update_type_checker_info(
-    type_checker: TypeChecker, root_dir: Path, test_duration: float
+    type_checker: TypeChecker, root_dir: Path, test_duration: float, skip_timing: bool = False
 ):
     # Record the version of the type checker used for the latest run.
     version_file = root_dir / "results" / type_checker.name / "version.toml"
@@ -215,7 +216,8 @@ def update_type_checker_info(
         existing_info = {}
 
     existing_info["version"] = type_checker.get_version()
-    existing_info["test_duration"] = round(test_duration, 1)
+    if not skip_timing:
+        existing_info["test_duration"] = round(test_duration, 1)
 
     version_file.parent.mkdir(parents=True, exist_ok=True)
     with open(version_file, "w") as f:
@@ -246,7 +248,7 @@ def main():
             if not type_checker.install():
                 print(f"Skipping tests for {type_checker.name}")
             else:
-                run_tests(root_dir, type_checker, test_cases)
+                run_tests(root_dir, type_checker, test_cases, skip_timing=options.skip_timing)
 
     # Generate a summary report.
     generate_summary(root_dir)

conformance/src/options.py

@@ -9,6 +9,7 @@
 @dataclass
 class _Options:
     report_only: bool | None
+    skip_timing: bool
 
 
 def parse_options(argv: list[str]) -> _Options:
@@ -17,7 +18,12 @@ def parse_options(argv: list[str]) -> _Options:
     reporting_group.add_argument(
         "--report-only",
         action="store_true",
-        help=("regenerates the test suite report from past results"),
+        help="regenerates the test suite report from past results",
+    )
+    reporting_group.add_argument(
+        "--skip-timing",
+        action="store_true",
+        help="do not update timing information in the output files",
     )
     ret = _Options(**vars(parser.parse_args(argv)))
     return ret

conformance/src/reporting.py

@@ -116,4 +116,4 @@ def generate_summary_html(root_dir: Path) -> str:
 
     summary_html.append("</tbody></table></div>\n")
 
-    return "".join(summary_html)
+    return "\n".join(summary_html)