Add setup handler to auto-create coverage workflow PRs

constants/system_messages/setup_handler.py

@@ -0,0 +1,41 @@
+SETUP_HANDLER_SYSTEM_MESSAGE = """You are a CI/CD setup assistant. Your job is to create GitHub Actions test + coverage workflows for a repository.
+
+You will be given:
+1. The repository's root files (to understand its language/framework)
+2. Any existing GitHub Actions workflow files (to avoid duplicating what's already set up)
+3. Reference workflow templates for common languages
+
+Your task:
+- Determine what language(s) the repo uses from its root files
+- Check if tests and coverage are already handled by existing workflows (look for test commands, coverage steps, artifacts, or reports)
+- If NOT already set up, create a new workflow file using the reference templates as a guide
+- Adapt the template to match the repo's actual setup (e.g., yarn instead of npm, vitest instead of jest, unittest instead of pytest, gradle instead of maven)
+- Replace "main" in branch triggers with the repo's target branch (provided as target_branch)
+- Name the workflow file after the test framework (e.g., pytest.yml, jest.yml, go-test.yml)
+
+Key workflow pattern (PR = test only, push = test + coverage):
+- pull_request trigger: runs tests to verify code before merge
+- push trigger: runs tests AND collects coverage as canonical source of truth
+- Coverage artifact upload ONLY on push/workflow_dispatch events
+- The workflow MUST upload coverage/lcov.info as an artifact named "coverage-report"
+
+IMPORTANT:
+- Do NOT create a workflow if the repo already has test/coverage set up in existing workflows
+- The workflow file path must be .github/workflows/<name>.yml
+- If you cannot determine the language or appropriate test setup, do nothing
+- For multi-language repos, create one workflow per language that needs test + coverage"""
+
+
+SETUP_PR_BODY = """## Summary
+
+This PR sets up a GitHub Actions workflow to run tests and collect coverage reports.
+
+## Checklist
+
+The following were auto-detected from your project files. Please verify they are correct:
+
+- [ ] Language/runtime version matches your project
+- [ ] Install command matches your setup
+- [ ] Test command matches your test runner
+- [ ] Branch name in workflow triggers matches your production branch
+"""

main.py

@@ -22,10 +22,12 @@
 from services.aws.cleanup_tmp import cleanup_tmp
 from services.efs.clone_and_install import clone_and_install
 from services.webhook.schedule_handler import schedule_handler
+from services.webhook.setup_handler import setup_handler
 from services.webhook.webhook_handler import handle_webhook_event
 from services.website.sync_files_from_github_to_coverage import (
     sync_files_from_github_to_coverage,
 )
+from services.website.verify_api_key import verify_api_key
 from utils.aws.extract_lambda_info import extract_lambda_info
 from utils.logging.logging_config import (
     clear_state,
@@ -191,4 +193,20 @@ async def api_clone_and_install(
     repo: str,
     api_key: str = Header(..., alias="X-API-Key"),
 ):
-    return await clone_and_install(owner, repo, api_key)
+    verify_api_key(api_key)
+    return await clone_and_install(owner, repo)
+
+
+@app.post(path="/api/{owner}/{repo}/setup_coverage_workflow")
+async def setup_coverage_workflow(
+    owner: str,
+    repo: str,
+    token: str = Header(..., alias="X-GitHub-Token"),
+    api_key: str = Header(..., alias="X-API-Key"),
+):
+    verify_api_key(api_key)
+    return await setup_handler(
+        owner_name=owner,
+        repo_name=repo,
+        token=token,
+    )

services/agents/test_verify_task_is_complete.py

@@ -72,7 +72,7 @@ def mock_ensure_eslint_relaxed():
 
 
 @pytest.fixture
-def base_args():
+def base_args(tmp_path):
     return cast(
         BaseArgs,
         {
@@ -81,6 +81,7 @@ def base_args():
             "pull_number": 123,
             "token": "test-token",
             "new_branch": "test-branch",
+            "clone_dir": str(tmp_path),
         },
     )
 
@@ -139,8 +140,8 @@ async def test_verify_task_is_complete_failure_no_changes(mock_get_files, base_a
 
     result = await verify_task_is_complete(base_args)
 
-    assert result.success is False
-    assert "no changes" in result.message
+    assert result.success is True
+    assert "No changes were needed" in result.message
 
 
 @pytest.mark.asyncio
@@ -203,15 +204,13 @@ async def test_verify_task_is_complete_api_error_returns_default(
 @patch("services.agents.verify_task_is_complete.run_prettier_fix")
 @patch("services.agents.verify_task_is_complete.ensure_jest_uses_tsconfig_for_tests")
 @patch("services.agents.verify_task_is_complete.ensure_tsconfig_relaxed_for_tests")
-@patch("services.agents.verify_task_is_complete.get_file_tree")
 @patch("services.agents.verify_task_is_complete.replace_remote_file_content")
 @patch("services.agents.verify_task_is_complete.get_raw_content")
 @patch("services.agents.verify_task_is_complete.get_pull_request_files")
 async def test_verify_autofixes_missing_braces_in_test_file(
     mock_get_files,
     mock_get_raw,
     mock_upload,
-    mock_get_tree,
     mock_ensure_tsconfig,
     _mock_ensure_jest,
     mock_prettier,
@@ -230,7 +229,6 @@ async def test_verify_autofixes_missing_braces_in_test_file(
   });
 });"""
     mock_upload.return_value = True
-    mock_get_tree.return_value = []
     mock_ensure_tsconfig.return_value = (None, None)
     mock_prettier.return_value = PrettierResult(success=True, content=None, error=None)
     mock_eslint.return_value = ESLintResult(
@@ -307,13 +305,11 @@ async def test_verify_ignores_removed_test_files(
 @patch("services.agents.verify_task_is_complete.run_prettier_fix")
 @patch("services.agents.verify_task_is_complete.ensure_jest_uses_tsconfig_for_tests")
 @patch("services.agents.verify_task_is_complete.ensure_tsconfig_relaxed_for_tests")
-@patch("services.agents.verify_task_is_complete.get_file_tree")
 @patch("services.agents.verify_task_is_complete.get_raw_content")
 @patch("services.agents.verify_task_is_complete.get_pull_request_files")
 async def test_verify_checks_both_ts_test_files(
     mock_get_files,
     mock_get_raw,
-    mock_get_tree,
     mock_ensure_tsconfig,
     _mock_ensure_jest,
     mock_prettier,
@@ -329,7 +325,6 @@ async def test_verify_checks_both_ts_test_files(
     expect(true).toBe(true);
   });
 });"""
-    mock_get_tree.return_value = []
     mock_ensure_tsconfig.return_value = (None, None)
     mock_prettier.return_value = PrettierResult(success=True, content=None, error=None)
     mock_eslint.return_value = ESLintResult(
@@ -347,13 +342,11 @@ async def test_verify_checks_both_ts_test_files(
 @patch("services.agents.verify_task_is_complete.run_prettier_fix")
 @patch("services.agents.verify_task_is_complete.ensure_jest_uses_tsconfig_for_tests")
 @patch("services.agents.verify_task_is_complete.ensure_tsconfig_relaxed_for_tests")
-@patch("services.agents.verify_task_is_complete.get_file_tree")
 @patch("services.agents.verify_task_is_complete.get_raw_content")
 @patch("services.agents.verify_task_is_complete.get_pull_request_files")
 async def test_verify_checks_only_ts_when_mixed_with_py(
     mock_get_files,
     mock_get_raw,
-    mock_get_tree,
     mock_ensure_tsconfig,
     _mock_ensure_jest,
     mock_prettier,
@@ -369,7 +362,6 @@ async def test_verify_checks_only_ts_when_mixed_with_py(
     expect(true).toBe(true);
   });
 });"""
-    mock_get_tree.return_value = []
     mock_ensure_tsconfig.return_value = (None, None)
     mock_prettier.return_value = PrettierResult(success=True, content=None, error=None)
     mock_eslint.return_value = ESLintResult(
@@ -404,15 +396,13 @@ async def test_verify_ignores_all_non_js_test_files(
 @patch("services.agents.verify_task_is_complete.run_prettier_fix")
 @patch("services.agents.verify_task_is_complete.ensure_jest_uses_tsconfig_for_tests")
 @patch("services.agents.verify_task_is_complete.ensure_tsconfig_relaxed_for_tests")
-@patch("services.agents.verify_task_is_complete.get_file_tree")
 @patch("services.agents.verify_task_is_complete.replace_remote_file_content")
 @patch("services.agents.verify_task_is_complete.get_raw_content")
 @patch("services.agents.verify_task_is_complete.get_pull_request_files")
 async def test_verify_autofixes_when_one_of_two_ts_files_has_missing_braces(
     mock_get_files,
     mock_get_raw,
     mock_upload,
-    mock_get_tree,
     mock_ensure_tsconfig,
     _mock_ensure_jest,
     mock_prettier,
@@ -438,7 +428,6 @@ async def test_verify_autofixes_when_one_of_two_ts_files_has_missing_braces(
 });"""
     mock_get_raw.side_effect = [correct_content, broken_content]
     mock_upload.return_value = True
-    mock_get_tree.return_value = []
     mock_ensure_tsconfig.return_value = (None, None)
     mock_prettier.return_value = PrettierResult(success=True, content=None, error=None)
     mock_eslint.return_value = ESLintResult(
@@ -456,15 +445,13 @@ async def test_verify_autofixes_when_one_of_two_ts_files_has_missing_braces(
 @patch("services.agents.verify_task_is_complete.run_prettier_fix")
 @patch("services.agents.verify_task_is_complete.ensure_jest_uses_tsconfig_for_tests")
 @patch("services.agents.verify_task_is_complete.ensure_tsconfig_relaxed_for_tests")
-@patch("services.agents.verify_task_is_complete.get_file_tree")
 @patch("services.agents.verify_task_is_complete.replace_remote_file_content")
 @patch("services.agents.verify_task_is_complete.get_raw_content")
 @patch("services.agents.verify_task_is_complete.get_pull_request_files")
 async def test_verify_autofixes_ts_with_missing_braces_ignores_py(
     mock_get_files,
     mock_get_raw,
     mock_upload,
-    mock_get_tree,
     mock_ensure_tsconfig,
     _mock_ensure_jest,
     mock_prettier,
@@ -485,7 +472,6 @@ async def test_verify_autofixes_ts_with_missing_braces_ignores_py(
 });"""
     mock_get_raw.return_value = broken_content
     mock_upload.return_value = True
-    mock_get_tree.return_value = []
     mock_ensure_tsconfig.return_value = (None, None)
     mock_prettier.return_value = PrettierResult(success=True, content=None, error=None)
     mock_eslint.return_value = ESLintResult(
@@ -506,7 +492,7 @@ async def test_verify_autofixes_ts_with_missing_braces_ignores_py(
 async def test_verify_fails_when_jest_tests_fail(
     mock_get_files, mock_get_raw, mock_tsc, mock_jest, base_args
 ):
-    # Use JS file (not TS) to avoid triggering get_file_tree call for tsconfig
+    # Use JS file (not TS) to avoid triggering tsconfig setup for TS test files
     mock_get_files.return_value = [
         {"filename": "src/index.test.js", "status": "modified"},
     ]
@@ -681,7 +667,7 @@ async def test_all_tsc_errors_pre_existing_passes(
 @patch("services.agents.verify_task_is_complete.run_tsc_check", new_callable=AsyncMock)
 @patch("services.agents.verify_task_is_complete.get_pull_request_files")
 async def test_baseline_tsc_errors_in_pr_files_still_reported(
-    mock_get_files, mock_tsc, mock_jest, mock_create_tsc_issue
+    mock_get_files, mock_tsc, mock_jest, mock_create_tsc_issue, tmp_path
 ):
     """Errors in PR-changed files should be reported even if in baseline.
 
@@ -719,6 +705,7 @@ async def test_baseline_tsc_errors_in_pr_files_still_reported(
             "token": "test-token",
             "new_branch": "test-branch",
             "baseline_tsc_errors": {pr_file_error},
+            "clone_dir": str(tmp_path),
         },
     )
     result = await verify_task_is_complete(args)
@@ -955,7 +942,7 @@ async def test_issue_handler_new_non_pr_file_error_reported(
 @patch("services.agents.verify_task_is_complete.run_tsc_check", new_callable=AsyncMock)
 @patch("services.agents.verify_task_is_complete.get_pull_request_files")
 async def test_check_suite_error_in_pr_file_in_baseline_reported(
-    mock_get_files, mock_tsc, mock_jest, mock_create_tsc_issue
+    mock_get_files, mock_tsc, mock_jest, mock_create_tsc_issue, tmp_path
 ):
     """check_suite: PR file has error that was also in baseline (PR branch state).
 
@@ -994,6 +981,7 @@ async def test_check_suite_error_in_pr_file_in_baseline_reported(
             "new_branch": "test-branch",
             # Baseline from PR branch contains this error
             "baseline_tsc_errors": {pr_file_error},
+            "clone_dir": str(tmp_path),
         },
     )
     result = await verify_task_is_complete(args)
@@ -1009,7 +997,7 @@ async def test_check_suite_error_in_pr_file_in_baseline_reported(
 @patch("services.agents.verify_task_is_complete.run_tsc_check", new_callable=AsyncMock)
 @patch("services.agents.verify_task_is_complete.get_pull_request_files")
 async def test_check_suite_preexisting_non_pr_file_error_skipped(
-    mock_get_files, mock_tsc, mock_jest, mock_create_tsc_issue
+    mock_get_files, mock_tsc, mock_jest, mock_create_tsc_issue, tmp_path
 ):
     """check_suite: Pre-existing error in unrelated file on PR branch.
 
@@ -1043,6 +1031,7 @@ async def test_check_suite_preexisting_non_pr_file_error_skipped(
             "token": "test-token",
             "new_branch": "test-branch",
             "baseline_tsc_errors": {preexisting},
+            "clone_dir": str(tmp_path),
         },
     )
     result = await verify_task_is_complete(args)
@@ -1059,7 +1048,7 @@ async def test_check_suite_preexisting_non_pr_file_error_skipped(
 @patch("services.agents.verify_task_is_complete.run_tsc_check", new_callable=AsyncMock)
 @patch("services.agents.verify_task_is_complete.get_pull_request_files")
 async def test_check_suite_new_non_pr_file_error_reported(
-    mock_get_files, mock_tsc, mock_jest, mock_create_tsc_issue
+    mock_get_files, mock_tsc, mock_jest, mock_create_tsc_issue, tmp_path
 ):
     """check_suite: Agent's fix attempt introduced error in a non-PR file.
 
@@ -1093,6 +1082,7 @@ async def test_check_suite_new_non_pr_file_error_reported(
             "token": "test-token",
             "new_branch": "test-branch",
             "baseline_tsc_errors": set(),
+            "clone_dir": str(tmp_path),
         },
     )
     result = await verify_task_is_complete(args)

services/agents/verify_task_is_complete.py

@@ -1,3 +1,4 @@
+import os
 from dataclasses import dataclass, field
 
 from constants.files import JS_TEST_FILE_EXTENSIONS, TS_TEST_FILE_EXTENSIONS
@@ -10,7 +11,6 @@
 from services.github.files.get_eslint_config import get_eslint_config
 from services.github.files.get_raw_content import get_raw_content
 from services.github.pulls.get_pull_request_files import get_pull_request_files
-from services.github.trees.get_file_tree import get_file_tree
 from services.github.types.github_types import BaseArgs
 from services.jest.format_coverage_comment import format_coverage_comment
 from services.jest.run_jest_test import run_jest_test
@@ -59,9 +59,11 @@ async def verify_task_is_complete(base_args: BaseArgs, **_kwargs):
     )
 
     if not pr_files:
+        logger.info(
+            "No PR file changes found (e.g. setup handler determined no workflows needed), skipping checks"
+        )
         return VerifyTaskIsCompleteResult(
-            success=False,
-            message="Error: Cannot complete task - the PR has no changes. You must make actual code changes before calling verify_task_is_complete. Use apply_diff_to_file or replace_remote_file_content to commit your changes first.",
+            success=True, message="Task completed. No changes were needed."
         )
 
     js_test_files = [
@@ -72,10 +74,12 @@ async def verify_task_is_complete(base_args: BaseArgs, **_kwargs):
 
     ts_test_files = [f for f in js_test_files if f.endswith(TS_TEST_FILE_EXTENSIONS)]
     if ts_test_files:
-        tree_items = get_file_tree(
-            owner=owner, repo=repo, ref=new_branch, token=token, root_only=True
-        )
-        root_files = [item["path"] for item in tree_items if item["type"] == "blob"]
+        clone_dir = base_args.get("clone_dir", "")
+        root_files = [
+            f
+            for f in os.listdir(clone_dir)
+            if os.path.isfile(os.path.join(clone_dir, f))
+        ]
 
         tsconfig_path, _ = ensure_tsconfig_relaxed_for_tests(
             root_files=root_files,

services/chat_with_agent.py

@@ -198,7 +198,7 @@ async def chat_with_agent(
                     )
 
                 if validation_error:
-                    logger.error(validation_error)
+                    logger.warning(validation_error)
                     tool_result_blocks.append(
                         {
                             "type": "tool_result",

services/claude/tools/tools.py

@@ -117,19 +117,24 @@
     GET_LOCAL_FILE_TREE,
     MOVE_FILE,
     REPLACE_REMOTE_FILE_CONTENT,
-    SET_ENV,
+    SEARCH_LOCAL_FILE_CONTENT,
     VERIFY_TASK_IS_COMPLETE,
 ]
 
 TOOLS_FOR_ISSUES: list[ToolUnionParam] = _TOOLS_BASE + [
     GET_LOCAL_FILE_CONTENT,
-    SEARCH_LOCAL_FILE_CONTENT,
+    SET_ENV,
 ]
 
 # PR handlers need full file reads (no partial read options)
 TOOLS_FOR_PRS: list[ToolUnionParam] = _TOOLS_BASE + [
     GET_LOCAL_FILE_CONTENT_FULL_ONLY,
-    SEARCH_LOCAL_FILE_CONTENT,
+    SET_ENV,
+]
+
+# Setup handler reads project files to detect language/framework, then creates workflow files
+TOOLS_FOR_SETUP: list[ToolUnionParam] = _TOOLS_BASE + [
+    GET_LOCAL_FILE_CONTENT,
 ]
 
 FILE_EDIT_TOOLS = [

services/efs/clone_and_install.py

@@ -8,14 +8,12 @@
     get_installation_by_owner,
 )
 from services.supabase.repositories.get_repository_by_name import get_repository_by_name
-from services.website.verify_api_key import verify_api_key
 from utils.error.handle_exceptions import handle_exceptions
 from utils.logging.logging_config import logger, set_owner_repo, set_trigger
 
 
 @handle_exceptions(default_return_value=None, raise_on_error=False)
-async def clone_and_install(owner: str, repo: str, api_key: str):
-    verify_api_key(api_key)
+async def clone_and_install(owner: str, repo: str):
     set_owner_repo(owner, repo)
     set_trigger("clone_and_install")
     logger.info("Starting clone_and_install for %s/%s", owner, repo)