Add Istanbul coverage checking to jest test runner

CLAUDE.md

@@ -224,7 +224,7 @@ ssh -i infrastructure/nat-instance-ssh-private-key.pem ec2-user@54.176.165.89
 ## Coding Standards
 
 - No DOCSTRINGS: Do not add docstrings unless explicitly told to. Do not delete existing docstrings unless they are outdated.
-- COMMENTS: Do not delete existing comments unless they are outdated. Preserve URL references and API documentation links. Never explain what was removed or why in the code itself. Explanations belong in terminal responses, not in code.
+- COMMENTS: Do not delete existing comments unless they are outdated. Preserve URL references and API documentation links. Never explain what was removed or why in the code itself. Explanations belong in terminal responses, not in code. Do not split comments across multiple lines unnecessarily - keep them on one line when possible to maintain readability.
 - LOGGERS: Do not delete existing logger statements unless they are outdated.
 - API URLS: Always verify API documentation URLs using WebFetch before using them. Never guess API endpoints.
 - NO TYPE HINTS USING ->: Do not add return type hints using -> because it overwrites the inferred return type without actually validating that the implementation returns that type. It's a lie to the type checker that cannot be verified at runtime. Return type hints are PROHIBITED.

services/agents/verify_task_is_complete.py

@@ -5,12 +5,14 @@
     ensure_eslint_relaxed_for_tests,
 )
 from services.eslint.run_eslint_fix import run_eslint_fix
+from services.github.comments.create_comment import create_comment
 from services.github.commits.replace_remote_file import replace_remote_file_content
 from services.github.files.get_eslint_config import get_eslint_config
 from services.github.files.get_raw_content import get_raw_content
 from services.github.pulls.get_pull_request_files import get_pull_request_files
 from services.github.trees.get_file_tree import get_file_tree
 from services.github.types.github_types import BaseArgs
+from services.jest.format_coverage_comment import format_coverage_comment
 from services.jest.run_jest_test import run_jest_test
 from services.node.ensure_jest_uses_tsconfig_for_tests import (
     ensure_jest_uses_tsconfig_for_tests,
@@ -173,13 +175,43 @@ async def verify_task_is_complete(base_args: BaseArgs, **_kwargs):
             )
             create_tsc_issue(base_args=base_args, unrelated_errors=unrelated_tsc_errors)
 
+    # Set by issue_handler for schedule issues so run_jest_test collects coverage using Istanbul instead of V8.
+    impl_file_to_collect_coverage_from = base_args.get(
+        "impl_file_to_collect_coverage_from", ""
+    )
+
     # Run jest/vitest tests on test files (with -u to auto-update stale snapshots)
-    jest_result = await run_jest_test(base_args=base_args, file_paths=non_removed_files)
+    jest_result = await run_jest_test(
+        base_args=base_args,
+        test_file_paths=js_test_files,
+        impl_file_to_collect_coverage_from=impl_file_to_collect_coverage_from,
+    )
     if jest_result.errors:
         for err in jest_result.errors:
             remaining_errors.append(f"- {jest_result.runner_name}: {err}")
         error_files.update(jest_result.error_files)
 
+    # Post coverage results as PR comment and check for incomplete coverage
+    if jest_result.coverage and not jest_result.errors:
+        cov = jest_result.coverage
+        if cov.error:
+            logger.warning("coverage: %s (not fixable by agent, skipping)", cov.error)
+        else:
+            comment_body = format_coverage_comment(
+                cov, impl_file_to_collect_coverage_from
+            )
+            create_comment(body=comment_body, base_args=base_args, target="pr")
+
+            has_incomplete_coverage = (
+                cov.statement_pct < 100
+                or cov.branch_pct < 100
+                or cov.function_pct < 100
+                or cov.line_pct < 100
+            )
+            if has_incomplete_coverage:
+                remaining_errors.append(f"- coverage:\n{comment_body}")
+                error_files.update(js_test_files)
+
     # Commit any snapshot files updated by jest -u
     clone_dir = base_args.get("clone_dir", "")
     for snap_path in jest_result.updated_snapshots:

services/agents/verify_task_is_ready.py

@@ -118,7 +118,11 @@ async def verify_task_is_ready(
 
     # Run jest/vitest tests if requested (for check_suite and review handlers)
     if run_jest:
-        jest_result = await run_jest_test(base_args=base_args, file_paths=file_paths)
+        jest_result = await run_jest_test(
+            base_args=base_args,
+            test_file_paths=js_ts_files,
+            impl_file_to_collect_coverage_from="",
+        )
         if jest_result.errors:
             for err in jest_result.errors:
                 errors.append(f"- {jest_result.runner_name}: {err}")

services/claude/tools/tools.py

@@ -7,7 +7,7 @@
 # Local imports
 from services.agents.verify_task_is_complete import verify_task_is_complete
 from services.env.set_env import SET_ENV, set_env
-from services.github.comments.create_comment import create_comment
+from services.github.comments.create_comment import CREATE_COMMENT, create_comment
 from services.github.commits.apply_diff_to_file import apply_diff_to_file
 from services.github.commits.replace_remote_file import (
     REPLACE_REMOTE_FILE_CONTENT,
@@ -108,23 +108,6 @@
     "strict": True,
 }
 
-# See https://docs.anthropic.com/en/docs/build-with-claude/tool-use#defining-tools
-CREATE_COMMENT: ToolUnionParam = {
-    "name": "create_comment",
-    "description": "Creates a note/notification on the GitHub issue or pull request. The user is not there - they will see it later. After commenting, continue working on what you CAN do. WHEN TO USE: To inform the user about something they need to know (e.g., you are restricted to test files but the fix requires source file changes, or secrets need to be added via GitHub UI). WHEN NOT TO USE: Status updates, progress reports, or asking questions. WHAT TO SAY: State the fact briefly - what you found and what the user needs to do later. Do not ask questions.",
-    "input_schema": {
-        "type": "object",
-        "properties": {
-            "body": {
-                "type": "string",
-                "description": "The comment text to post.",
-            },
-        },
-        "required": ["body"],
-        "additionalProperties": False,
-    },
-    "strict": True,
-}
 
 _TOOLS_BASE: list[ToolUnionParam] = [
     APPLY_DIFF_TO_FILE,

services/github/comments/combine_and_create_comment.py

@@ -79,4 +79,4 @@ def combine_and_create_comment(
             "issue_number": issue_number,
         },
     )
-    create_comment(body=body, base_args=comment_args)
+    create_comment(body=body, base_args=comment_args, target="issue")

services/github/comments/create_comment.py

@@ -1,21 +1,54 @@
+from typing import Literal
+
 import requests
+from anthropic.types import ToolUnionParam
 
 from config import GITHUB_API_URL, TIMEOUT
 from services.github.types.github_types import BaseArgs
 from services.github.utils.create_headers import create_headers
 from utils.error.handle_exceptions import handle_exceptions
 
+# See https://docs.anthropic.com/en/docs/build-with-claude/tool-use#defining-tools
+CREATE_COMMENT: ToolUnionParam = {
+    "name": "create_comment",
+    "description": "Creates a note/notification on the GitHub issue or pull request. The user is not there - they will see it later. After commenting, continue working on what you CAN do. WHEN TO USE: To inform the user about something they need to know (e.g., you are restricted to test files but the fix requires source file changes, or secrets need to be added via GitHub UI). WHEN NOT TO USE: Status updates, progress reports, or asking questions. WHAT TO SAY: State the fact briefly - what you found and what the user needs to do later. Do not ask questions.",
+    "input_schema": {
+        "type": "object",
+        "properties": {
+            "body": {
+                "type": "string",
+                "description": "The comment text to post.",
+            },
+            "target": {
+                "type": "string",
+                "enum": ["issue", "pr"],
+                "description": "Whether to comment on the issue or pull request.",
+            },
+        },
+        "required": ["body", "target"],
+        "additionalProperties": False,
+    },
+    "strict": True,
+}
+
 
 @handle_exceptions(default_return_value=None, raise_on_error=False)
-def create_comment(body: str, base_args: BaseArgs, **_kwargs):
+def create_comment(
+    body: str, base_args: BaseArgs, target: Literal["issue", "pr"], **_kwargs
+):
     # https://docs.github.com/en/rest/issues/comments?apiVersion=2022-11-28#create-an-issue-comment
+    # PRs are issues in GitHub's data model, so this works for both issues and PRs.
     owner = base_args["owner"]
     repo = base_args["repo"]
     token = base_args["token"]
-    issue_number = base_args["issue_number"]
+    number = (
+        base_args.get("pull_number") if target == "pr" else base_args["issue_number"]
+    )
+    if number is None:
+        number = base_args["issue_number"]
 
     response = requests.post(
-        url=f"{GITHUB_API_URL}/repos/{owner}/{repo}/issues/{issue_number}/comments",
+        url=f"{GITHUB_API_URL}/repos/{owner}/{repo}/issues/{number}/comments",
         headers=create_headers(token=token),
         json={"body": body},
         timeout=TIMEOUT,

services/github/comments/test_create_comment.py

@@ -29,7 +29,9 @@ def test_create_comment_success(test_owner, test_repo, test_token):
     ) as mock_create_headers:
         mock_create_headers.return_value = {"Authorization": f"Bearer {test_token}"}
         mock_post.return_value = mock_response
-        result = create_comment(body="Test comment", base_args=comment_args)
+        result = create_comment(
+            body="Test comment", base_args=comment_args, target="issue"
+        )
 
     # Assert
     mock_post.assert_called_once()
@@ -54,7 +56,9 @@ def test_create_comment_request_error(test_owner, test_repo, test_token):
     # Act
     with patch("services.github.comments.create_comment.requests.post") as mock_post:
         mock_post.return_value = mock_response
-        result = create_comment(body="Test comment", base_args=comment_args)
+        result = create_comment(
+            body="Test comment", base_args=comment_args, target="issue"
+        )
 
     # Assert
     assert result is None

services/github/types/github_types.py

@@ -58,6 +58,7 @@ class BaseArgs(TypedDict):
     pr_body: NotRequired[str]
     pr_number: NotRequired[int]
     baseline_tsc_errors: NotRequired[set[str]]
+    impl_file_to_collect_coverage_from: NotRequired[str]
     review_id: NotRequired[int]
     skip_ci: NotRequired[bool]
 

services/jest/coverage_types.py

@@ -0,0 +1,34 @@
+from typing import TypedDict
+
+
+class _Position(TypedDict):
+    line: int
+    column: int
+
+
+class _Location(TypedDict):
+    start: _Position
+
+
+class _StatementEntry(TypedDict):
+    start: _Position
+
+
+class _FnEntry(TypedDict):
+    name: str
+    loc: _Location
+
+
+class _BranchEntry(TypedDict):
+    type: str
+    loc: _Location
+    locations: list[_Location]
+
+
+class IstanbulFileCoverage(TypedDict):
+    s: dict[str, int]
+    f: dict[str, int]
+    b: dict[str, list[int]]
+    statementMap: dict[str, _StatementEntry]
+    fnMap: dict[str, _FnEntry]
+    branchMap: dict[str, _BranchEntry]

services/jest/fixtures/coverage-final.json

@@ -0,0 +1,2 @@
+{"/Users/rwest/Repositories/website/utils/is-test-file.ts": {"path":"/Users/rwest/Repositories/website/utils/is-test-file.ts","statementMap":{"0":{"start":{"line":4,"column":16},"end":{"line":4,"column":27}},"1":{"start":{"line":5,"column":2},"end":{"line":5,"column":null}},"2":{"start":{"line":5,"column":49},"end":{"line":5,"column":null}},"3":{"start":{"line":7,"column":24},"end":{"line":7,"column":null}},"4":{"start":{"line":11,"column":29},"end":{"line":54,"column":null}},"5":{"start":{"line":57,"column":2},"end":{"line":59,"column":null}},"6":{"start":{"line":57,"column":43},"end":{"line":57,"column":73}},"7":{"start":{"line":58,"column":4},"end":{"line":58,"column":null}},"8":{"start":{"line":63,"column":27},"end":{"line":68,"column":null}},"9":{"start":{"line":71,"column":29},"end":{"line":71,"column":null}},"10":{"start":{"line":71,"column":64},"end":{"line":71,"column":null}},"11":{"start":{"line":73,"column":2},"end":{"line":73,"column":null}},"12":{"start":{"line":73,"column":27},"end":{"line":73,"column":null}},"13":{"start":{"line":76,"column":30},"end":{"line":82,"column":null}},"14":{"start":{"line":85,"column":2},"end":{"line":85,"column":null}},"15":{"start":{"line":85,"column":48},"end":{"line":85,"column":null}}},"fnMap":{"0":{"name":"isTestFile","decl":{"start":{"line":4,"column":16},"end":{"line":4,"column":27}},"loc":{"start":{"line":4,"column":43},"end":{"line":86,"column":null}}},"1":{"name":"(anonymous_2)","decl":{"start":{"line":57,"column":30},"end":{"line":57,"column":31}},"loc":{"start":{"line":57,"column":43},"end":{"line":57,"column":73}}},"2":{"name":"(anonymous_3)","decl":{"start":{"line":71,"column":51},"end":{"line":71,"column":52}},"loc":{"start":{"line":71,"column":64},"end":{"line":71,"column":null}}},"3":{"name":"(anonymous_4)","decl":{"start":{"line":85,"column":35},"end":{"line":85,"column":36}},"loc":{"start":{"line":85,"column":48},"end":{"line":85,"column":null}}}},"branchMap":{"0":{"loc":{"start":{"line":5,"column":2},"end":{"line":5,"column":null}},"type":"if","locations":[{"start":{"line":5,"column":2},"end":{"line":5,"column":null}}]},"1":{"loc":{"start":{"line":5,"column":6},"end":{"line":5,"column":49}},"type":"binary-expr","locations":[{"start":{"line":5,"column":6},"end":{"line":5,"column":19}},{"start":{"line":5,"column":19},"end":{"line":5,"column":49}}]},"2":{"loc":{"start":{"line":57,"column":2},"end":{"line":59,"column":null}},"type":"if","locations":[{"start":{"line":57,"column":2},"end":{"line":59,"column":null}}]},"3":{"loc":{"start":{"line":73,"column":2},"end":{"line":73,"column":null}},"type":"if","locations":[{"start":{"line":73,"column":2},"end":{"line":73,"column":null}}]}},"s":{"0":143,"1":143,"2":5,"3":138,"4":138,"5":138,"6":2652,"7":107,"8":31,"9":31,"10":98,"11":31,"12":18,"13":13,"14":13,"15":56},"f":{"0":143,"1":2652,"2":98,"3":56},"b":{"0":[5],"1":[143,141],"2":[107],"3":[18]}}
+}

services/jest/format_coverage_comment.py

@@ -0,0 +1,29 @@
+from services.jest.parse_coverage_json import Coverage
+from utils.error.handle_exceptions import handle_exceptions
+
+
+@handle_exceptions(default_return_value="", raise_on_error=False)
+def format_coverage_comment(coverage: Coverage, impl_file: str):
+    # Column order matches jest/vitest coverage table: Stmts, Branch, Funcs, Lines
+    lines = [
+        f"## Coverage for `{impl_file}`",
+        "",
+        "| Stmts | Branch | Funcs | Lines |",
+        "|-------|--------|-------|-------|",
+        f"| {coverage.statement_pct}% | {coverage.branch_pct}% | {coverage.function_pct}% | {coverage.line_pct}% |",
+    ]
+    uncovered_parts = []
+    if coverage.uncovered_statements:
+        uncovered_parts.append(
+            f"- Uncovered statements: {coverage.uncovered_statements}"
+        )
+    if coverage.uncovered_branches:
+        uncovered_parts.append(f"- Uncovered branches: {coverage.uncovered_branches}")
+    if coverage.uncovered_functions:
+        uncovered_parts.append(f"- Uncovered functions: {coverage.uncovered_functions}")
+    if coverage.uncovered_lines:
+        uncovered_parts.append(f"- Uncovered lines: {coverage.uncovered_lines}")
+    if uncovered_parts:
+        lines.append("")
+        lines.extend(uncovered_parts)
+    return "\n".join(lines)