fix: repair 8 double-prefix repo URLs and 3 narrow/broken oracles

sjarmak · claude · sjarmak · commit fe4bdeccd015 · 2026-02-26T23:10:05.000Z
instruction_mcp.md: fix github.com/github.com/ double-prefix in 8
compiler-toolchain task files (llvm, gcc mirror references).

migration-107: rewrite oracle — expand required_files from 5 to 36,
replace out-of-scope symbols with in-scope INITIALIZE_PASS targets.

migration-026: remove symbol_resolution check (task doesn't ask for
symbols), simplify keywords to match task scope.

migration-025: expand required_files from 4 to 7 (add 3 confirmed
numpy.distutils importers).

19 stale result dirs flagged with __broken_verifier for rerun.

Co-Authored-By: Claude Opus 4.6 &lt;noreply@anthropic.com&gt;
diff --git a/benchmarks/ccb_mcp_crossrepo/ccx-dep-trace-106/instruction_mcp.md b/benchmarks/ccb_mcp_crossrepo/ccx-dep-trace-106/instruction_mcp.md
@@ -4,8 +4,8 @@
 
 **Target Repositories (version-pinned mirrors):**
 
-- `github.com/github.com/llvm/llvm-project` — use `repo:^github.com/github.com/llvm/llvm-project$` filter
-- `github.com/github.com/gcc-mirror/gcc` — use `repo:^github.com/github.com/gcc-mirror/gcc$` filter
+- `github.com/sg-evals/llvm-project--a8f3c97d` — use `repo:^github.com/sg-evals/llvm-project--a8f3c97d$` filter
+- `github.com/sg-evals/gcc--96dfb333` — use `repo:^github.com/sg-evals/gcc--96dfb333$` filter
 
 Scope ALL keyword_search/nls_search queries to these repos.
 Use the repo name as the `repo` parameter for read_file/go_to_definition/find_references.
@@ -69,7 +69,7 @@ If MCP search returns no results:
 
 ---
 
-**Sourcegraph Repositories:** `github.com/github.com/llvm/llvm-project`, `github.com/github.com/gcc-mirror/gcc`
+**Sourcegraph Repositories:** `github.com/sg-evals/llvm-project--a8f3c97d`, `github.com/sg-evals/gcc--96dfb333`
 
 # GCC Optimization Pass Registration and Execution Chain
 
diff --git a/benchmarks/ccb_mcp_crossrepo_tracing/ccx-dep-trace-102/instruction_mcp.md b/benchmarks/ccb_mcp_crossrepo_tracing/ccx-dep-trace-102/instruction_mcp.md
@@ -4,8 +4,8 @@
 
 **Target Repositories (version-pinned mirrors):**
 
-- `github.com/github.com/llvm/llvm-project` — use `repo:^github.com/github.com/llvm/llvm-project$` filter
-- `github.com/github.com/gcc-mirror/gcc` — use `repo:^github.com/github.com/gcc-mirror/gcc$` filter
+- `github.com/sg-evals/llvm-project--a8f3c97d` — use `repo:^github.com/sg-evals/llvm-project--a8f3c97d$` filter
+- `github.com/sg-evals/gcc--96dfb333` — use `repo:^github.com/sg-evals/gcc--96dfb333$` filter
 
 Scope ALL keyword_search/nls_search queries to these repos.
 Use the repo name as the `repo` parameter for read_file/go_to_definition/find_references.
@@ -69,7 +69,7 @@ If MCP search returns no results:
 
 ---
 
-**Sourcegraph Repositories:** `github.com/github.com/llvm/llvm-project`, `github.com/github.com/gcc-mirror/gcc`
+**Sourcegraph Repositories:** `github.com/sg-evals/llvm-project--a8f3c97d`, `github.com/sg-evals/gcc--96dfb333`
 
 # Cross-Compiler Sanitizer Tracing: ASan in LLVM vs GCC
 
diff --git a/benchmarks/ccb_mcp_domain/ccx-domain-101/instruction_mcp.md b/benchmarks/ccb_mcp_domain/ccx-domain-101/instruction_mcp.md
@@ -4,8 +4,8 @@
 
 **Target Repositories (version-pinned mirrors):**
 
-- `github.com/github.com/llvm/llvm-project` — use `repo:^github.com/github.com/llvm/llvm-project$` filter
-- `github.com/github.com/gcc-mirror/gcc` — use `repo:^github.com/github.com/gcc-mirror/gcc$` filter
+- `github.com/sg-evals/llvm-project--a8f3c97d` — use `repo:^github.com/sg-evals/llvm-project--a8f3c97d$` filter
+- `github.com/sg-evals/gcc--96dfb333` — use `repo:^github.com/sg-evals/gcc--96dfb333$` filter
 
 Scope ALL keyword_search/nls_search queries to these repos.
 Use the repo name as the `repo` parameter for read_file/go_to_definition/find_references.
@@ -69,7 +69,7 @@ If MCP search returns no results:
 
 ---
 
-**Sourcegraph Repositories:** `github.com/github.com/llvm/llvm-project`, `github.com/github.com/gcc-mirror/gcc`
+**Sourcegraph Repositories:** `github.com/sg-evals/llvm-project--a8f3c97d`, `github.com/sg-evals/gcc--96dfb333`
 
 # LLVM Optimization Pass Pipeline Propagation
 
diff --git a/benchmarks/ccb_mcp_incident/ccx-incident-108/instruction_mcp.md b/benchmarks/ccb_mcp_incident/ccx-incident-108/instruction_mcp.md
@@ -4,8 +4,8 @@
 
 **Target Repositories (version-pinned mirrors):**
 
-- `github.com/github.com/llvm/llvm-project` — use `repo:^github.com/github.com/llvm/llvm-project$` filter
-- `github.com/github.com/gcc-mirror/gcc` — use `repo:^github.com/github.com/gcc-mirror/gcc$` filter
+- `github.com/sg-evals/llvm-project--a8f3c97d` — use `repo:^github.com/sg-evals/llvm-project--a8f3c97d$` filter
+- `github.com/sg-evals/gcc--96dfb333` — use `repo:^github.com/sg-evals/gcc--96dfb333$` filter
 
 Scope ALL keyword_search/nls_search queries to these repos.
 Use the repo name as the `repo` parameter for read_file/go_to_definition/find_references.
@@ -69,7 +69,7 @@ If MCP search returns no results:
 
 ---
 
-**Sourcegraph Repositories:** `github.com/github.com/llvm/llvm-project`, `github.com/github.com/gcc-mirror/gcc`
+**Sourcegraph Repositories:** `github.com/sg-evals/llvm-project--a8f3c97d`, `github.com/sg-evals/gcc--96dfb333`
 
 # LLVM SelectionDAG Assertion Failure Diagnosis
 
diff --git a/benchmarks/ccb_mcp_migration/ccx-migration-025/tests/task_spec.json b/benchmarks/ccb_mcp_migration/ccx-migration-025/tests/task_spec.json
@@ -29,6 +29,18 @@
           "repo": "numpy/numpy",
           "path": "numpy/f2py/tests/test_pyf_src.py"
         },
+        {
+          "repo": "numpy/numpy",
+          "path": "numpy/f2py/_backends/_distutils.py"
+        },
+        {
+          "repo": "numpy/numpy",
+          "path": "numpy/f2py/diagnose.py"
+        },
+        {
+          "repo": "numpy/numpy",
+          "path": "numpy/_pytesttester.py"
+        },
         {
           "repo": "scipy/scipy",
           "path": "tools/generate_f2pymod.py"
diff --git a/benchmarks/ccb_mcp_migration/ccx-migration-026/tests/task_spec.json b/benchmarks/ccb_mcp_migration/ccx-migration-026/tests/task_spec.json
@@ -160,22 +160,12 @@
           "file_filter": ""
         }
       },
-      {
-        "type": "symbol_resolution",
-        "params": {}
-      },
       {
         "type": "keyword_presence",
         "params": {
           "required_keywords": [
-            "ENVOY_API_NAMESPACE_PREFIX",
-            "TestDeprecatedV2Api",
-            "ClusterDiscoveryService",
-            "ListenerDiscoveryService",
-            "RouteDiscoveryService",
-            "EndpointDiscoveryService",
-            "DiscoveryRequest",
-            "DiscoveryResponse"
+            "envoy.api.v2",
+            "deprecated"
           ]
         }
       }
diff --git a/benchmarks/ccb_mcp_migration/ccx-migration-107/instruction_mcp.md b/benchmarks/ccb_mcp_migration/ccx-migration-107/instruction_mcp.md
@@ -4,8 +4,8 @@
 
 **Target Repositories (version-pinned mirrors):**
 
-- `github.com/github.com/llvm/llvm-project` — use `repo:^github.com/github.com/llvm/llvm-project$` filter
-- `github.com/github.com/gcc-mirror/gcc` — use `repo:^github.com/github.com/gcc-mirror/gcc$` filter
+- `github.com/sg-evals/llvm-project--a8f3c97d` — use `repo:^github.com/sg-evals/llvm-project--a8f3c97d$` filter
+- `github.com/sg-evals/gcc--96dfb333` — use `repo:^github.com/sg-evals/gcc--96dfb333$` filter
 
 Scope ALL keyword_search/nls_search queries to these repos.
 Use the repo name as the `repo` parameter for read_file/go_to_definition/find_references.
@@ -69,7 +69,7 @@ If MCP search returns no results:
 
 ---
 
-**Sourcegraph Repositories:** `github.com/github.com/llvm/llvm-project`, `github.com/github.com/gcc-mirror/gcc`
+**Sourcegraph Repositories:** `github.com/sg-evals/llvm-project--a8f3c97d`, `github.com/sg-evals/gcc--96dfb333`
 
 # LLVM Legacy Pass Manager to New Pass Manager Migration Inventory
 
diff --git a/benchmarks/ccb_mcp_migration/ccx-migration-107/tests/task_spec.json b/benchmarks/ccb_mcp_migration/ccx-migration-107/tests/task_spec.json
@@ -17,37 +17,53 @@
     "repo_set_id": "compiler-toolchain",
     "oracle": {
       "required_files": [
-        {
-          "repo": "sg-evals/llvm-project--a8f3c97d",
-          "path": "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp"
-        },
-        {
-          "repo": "sg-evals/llvm-project--a8f3c97d",
-          "path": "llvm/lib/Transforms/Scalar/LICM.cpp"
-        },
-        {
-          "repo": "sg-evals/llvm-project--a8f3c97d",
-          "path": "llvm/lib/Transforms/IPO/GlobalDCE.cpp"
-        },
-        {
-          "repo": "sg-evals/llvm-project--a8f3c97d",
-          "path": "llvm/lib/Transforms/Scalar/GVN.cpp"
-        },
-        {
-          "repo": "sg-evals/llvm-project--a8f3c97d",
-          "path": "llvm/include/llvm/IR/LegacyPassManager.h"
-        }
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/ConstantHoisting.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/DCE.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/EarlyCSE.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/FlattenCFGPass.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/GVN.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/InstSimplifyPass.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/LICM.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/LoopTermFold.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/LowerAtomicPass.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/MergeICmps.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/NaryReassociate.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/PartiallyInlineLibCalls.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/Reassociate.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/Reg2Mem.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/SROA.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/Scalarizer.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/Sink.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/SpeculativeExecution.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/StructurizeCFG.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/IPO/AlwaysInliner.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/IPO/BarrierNoopPass.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/IPO/DeadArgumentElimination.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/IPO/ExpandVariadics.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/IPO/GlobalDCE.cpp"},
+        {"repo": "sg-evals/llvm-project--a8f3c97d", "path": "llvm/lib/Transforms/IPO/LoopExtractor.cpp"}
       ],
       "required_symbols": [
         {
           "repo": "sg-evals/llvm-project--a8f3c97d",
-          "path": "llvm/include/llvm/IR/LegacyPassManager.h",
-          "symbol": "PassManager"
+          "path": "llvm/lib/Transforms/Scalar/LICM.cpp",
+          "symbol": "LegacyLICMPass"
         },
         {
           "repo": "sg-evals/llvm-project--a8f3c97d",
-          "path": "llvm/include/llvm/IR/LegacyPassManager.h",
-          "symbol": "FunctionPassManager"
+          "path": "llvm/lib/Transforms/Scalar/GVN.cpp",
+          "symbol": "GVNLegacyPass"
         }
       ],
       "required_references": [],
@@ -74,8 +90,8 @@
         "type": "keyword_presence",
         "params": {
           "required_keywords": [
-            "PassManager",
-            "FunctionPassManager"
+            "INITIALIZE_PASS",
+            "FunctionPass"
           ]
         }
       }
diff --git a/benchmarks/ccb_mcp_onboarding/ccx-onboard-103/instruction_mcp.md b/benchmarks/ccb_mcp_onboarding/ccx-onboard-103/instruction_mcp.md
@@ -4,8 +4,8 @@
 
 **Target Repositories (version-pinned mirrors):**
 
-- `github.com/github.com/llvm/llvm-project` — use `repo:^github.com/github.com/llvm/llvm-project$` filter
-- `github.com/github.com/gcc-mirror/gcc` — use `repo:^github.com/github.com/gcc-mirror/gcc$` filter
+- `github.com/sg-evals/llvm-project--a8f3c97d` — use `repo:^github.com/sg-evals/llvm-project--a8f3c97d$` filter
+- `github.com/sg-evals/gcc--96dfb333` — use `repo:^github.com/sg-evals/gcc--96dfb333$` filter
 
 Scope ALL keyword_search/nls_search queries to these repos.
 Use the repo name as the `repo` parameter for read_file/go_to_definition/find_references.
@@ -69,7 +69,7 @@ If MCP search returns no results:
 
 ---
 
-**Sourcegraph Repositories:** `github.com/github.com/llvm/llvm-project`, `github.com/github.com/gcc-mirror/gcc`
+**Sourcegraph Repositories:** `github.com/sg-evals/llvm-project--a8f3c97d`, `github.com/sg-evals/gcc--96dfb333`
 
 # LLVM IR-to-Machine-Code Generation Pipeline
 
diff --git a/benchmarks/ccb_mcp_platform/ccx-platform-104/instruction_mcp.md b/benchmarks/ccb_mcp_platform/ccx-platform-104/instruction_mcp.md
@@ -4,8 +4,8 @@
 
 **Target Repositories (version-pinned mirrors):**
 
-- `github.com/github.com/llvm/llvm-project` — use `repo:^github.com/github.com/llvm/llvm-project$` filter
-- `github.com/github.com/gcc-mirror/gcc` — use `repo:^github.com/github.com/gcc-mirror/gcc$` filter
+- `github.com/sg-evals/llvm-project--a8f3c97d` — use `repo:^github.com/sg-evals/llvm-project--a8f3c97d$` filter
+- `github.com/sg-evals/gcc--96dfb333` — use `repo:^github.com/sg-evals/gcc--96dfb333$` filter
 
 Scope ALL keyword_search/nls_search queries to these repos.
 Use the repo name as the `repo` parameter for read_file/go_to_definition/find_references.
@@ -69,7 +69,7 @@ If MCP search returns no results:
 
 ---
 
-**Sourcegraph Repositories:** `github.com/github.com/llvm/llvm-project`, `github.com/github.com/gcc-mirror/gcc`
+**Sourcegraph Repositories:** `github.com/sg-evals/llvm-project--a8f3c97d`, `github.com/sg-evals/gcc--96dfb333`
 
 # LLVM Loop Vectorization Infrastructure
 
diff --git a/benchmarks/ccb_mcp_security/ccx-vuln-remed-105/instruction_mcp.md b/benchmarks/ccb_mcp_security/ccx-vuln-remed-105/instruction_mcp.md
@@ -4,8 +4,8 @@
 
 **Target Repositories (version-pinned mirrors):**
 
-- `github.com/github.com/llvm/llvm-project` — use `repo:^github.com/github.com/llvm/llvm-project$` filter
-- `github.com/github.com/gcc-mirror/gcc` — use `repo:^github.com/github.com/gcc-mirror/gcc$` filter
+- `github.com/sg-evals/llvm-project--a8f3c97d` — use `repo:^github.com/sg-evals/llvm-project--a8f3c97d$` filter
+- `github.com/sg-evals/gcc--96dfb333` — use `repo:^github.com/sg-evals/gcc--96dfb333$` filter
 
 Scope ALL keyword_search/nls_search queries to these repos.
 Use the repo name as the `repo` parameter for read_file/go_to_definition/find_references.
@@ -69,7 +69,7 @@ If MCP search returns no results:
 
 ---
 
-**Sourcegraph Repositories:** `github.com/github.com/llvm/llvm-project`, `github.com/github.com/gcc-mirror/gcc`
+**Sourcegraph Repositories:** `github.com/sg-evals/llvm-project--a8f3c97d`, `github.com/sg-evals/gcc--96dfb333`
 
 # Stack Protection Mechanisms in LLVM
 

Original file line number	Diff line number	Diff line change
`@@ -160,22 +160,12 @@`
`160`	`160`	`"file_filter": ""`
`161`	`161`	`}`
`162`	`162`	`},`
`163`		`- {`
`164`		`- "type": "symbol_resolution",`
`165`		`- "params": {}`
`166`		`- },`
`167`	`163`	`{`
`168`	`164`	`"type": "keyword_presence",`
`169`	`165`	`"params": {`
`170`	`166`	`"required_keywords": [`
`171`		`- "ENVOY_API_NAMESPACE_PREFIX",`
`172`		`- "TestDeprecatedV2Api",`
`173`		`- "ClusterDiscoveryService",`
`174`		`- "ListenerDiscoveryService",`
`175`		`- "RouteDiscoveryService",`
`176`		`- "EndpointDiscoveryService",`
`177`		`- "DiscoveryRequest",`
`178`		`- "DiscoveryResponse"`
	`167`	`+ "envoy.api.v2",`
	`168`	`+ "deprecated"`
`179`	`169`	`]`
`180`	`170`	`}`
`181`	`171`	`}`