Merge pull request #660 from ruby/mcp-same-commit-flaky

Mark same-commit failure boundary as flaky

Author: hsbt

app/tools/mcp_tools/find_failure_origin.rb

@@ -128,6 +128,12 @@ def self.analyze_query(rows, match_ids)
     end
 
     def self.config_entry(first_bad, last_good, boundary)
+      same_commit = last_good&.git_sha && last_good.git_sha == first_bad.git_sha
+      note = if same_commit
+        "last_good and first_bad ran the same commit; the failure is likely flaky or environment-dependent rather than caused by a code change."
+      elsif last_good.nil?
+        "No earlier report within the window (window exhausted or scan capped at #{MAX_REPORTS_PER_CONFIG} reports); the failure may have started earlier."
+      end
       {
         server: first_bad.server.name,
         server_id: first_bad.server_id,
@@ -136,9 +142,8 @@ def self.config_entry(first_bad, last_good, boundary)
         occurrences: boundary[:occurrences],
         first_bad: first_bad.as_mcp_json,
         last_good: last_good&.as_mcp_json,
-        compare_url: compare_url(last_good, first_bad),
-        note: last_good ? nil :
-          "No earlier report within the window (window exhausted or scan capped at #{MAX_REPORTS_PER_CONFIG} reports); the failure may have started earlier.",
+        compare_url: (compare_url(last_good, first_bad) unless same_commit),
+        note: note,
       }.compact
     end
   end

test/controllers/mcp_controller_test.rb

@@ -104,6 +104,22 @@ class McpControllerTest < ActionDispatch::IntegrationTest
     assert_equal "mcp-srv", result.dig("earliest", "server")
   end
 
+  test "find_failure_origin flags same-commit boundary as flaky" do
+    base = Time.now.utc.change(usec: 0)
+    good = create_report(@server, base - 10.hours, SHA_HEAD, success: true)
+    good.update!(branch: "4.0")
+    bad = create_report(@server, base - 5.hours, SHA_HEAD, success: false)
+    bad.update!(branch: "4.0")
+    LogExcerpt.create!(report: bad, content: "TestFlaky#test_same_commit failed")
+
+    result = call_tool("find_failure_origin", { branch: "4.0", query: "test_same_commit" })
+    config = result["configs"].first
+    assert_equal bad.id, config.dig("first_bad", "id")
+    assert_equal good.id, config.dig("last_good", "id")
+    assert_nil config["compare_url"]
+    assert_match "flaky", config["note"]
+  end
+
   test "find_failure_origin with unmatched query" do
     result = call_tool("find_failure_origin", { branch: "master", query: "no_such_failure" })
     assert_match "No failure log", result["message"]