fix: oracle sg-benchmarks alias resolution + account round-robin bug

sjarmak · claude · sjarmak · commit ba67770cfb05 · 2026-02-21T18:18:48.000Z
Add _SG_MIRROR_ALIASES to all 12 MCP-unique oracle_checks.py so that
agent answers using sg-benchmarks mirror names (e.g. sg-benchmarks/etcd-io-etcd)
are correctly resolved to upstream canonical names (etcd-io/etcd) during scoring.

Fix _next_account in run_selected_tasks.sh: the $() subshell prevented
_ACCOUNT_IDX from advancing in the parent shell, so all tasks used account 0.
Renamed to _pick_next_account which sets _PICKED_HOME directly.

Also adds etcd mirror entry to instance_to_mirror.json and adds etcd + k8s-api
to remirror script.

Co-Authored-By: Claude Opus 4.6 &lt;noreply@anthropic.com&gt;
diff --git a/benchmarks/ccb_mcp_crossorg/ccx-crossorg-061/tests/oracle_checks.py b/benchmarks/ccb_mcp_crossorg/ccx-crossorg-061/tests/oracle_checks.py
@@ -31,16 +31,32 @@
 # Agents sometimes forget to strip these; normalize before comparison.
 _HOSTING_PREFIX_RE = re.compile(r"^(?:github\.com|gitlab\.com|bitbucket\.org)/")
 
+# sg-benchmarks mirrors → upstream canonical names.
+# Allows oracle answers and agent answers to use either name interchangeably.
+_SG_MIRROR_ALIASES = {
+    "sg-benchmarks/kubernetes-client-go": "kubernetes/client-go",
+    "sg-benchmarks/kubernetes-api": "kubernetes/api",
+    "sg-benchmarks/etcd-io-etcd": "etcd-io/etcd",
+    "sg-benchmarks/expressjs-express": "expressjs/express",
+    "sg-benchmarks/grafana-loki": "grafana/loki",
+    "sg-benchmarks/grafana-mimir": "grafana/mimir",
+    "sg-benchmarks/prisma-prisma": "prisma/prisma",
+    "sg-benchmarks/lodash": "lodash/lodash",
+}
+
 
 def _normalize_repo(repo: str) -> str:
-    """Strip hosting-provider prefix from a repo name for fuzzy matching.
+    """Strip hosting-provider prefix and resolve sg-benchmarks aliases.
 
     >>> _normalize_repo("github.com/sg-benchmarks/kubernetes-client-go")
-    'sg-benchmarks/kubernetes-client-go'
+    'kubernetes/client-go'
     >>> _normalize_repo("sg-benchmarks/kubernetes-client-go")
-    'sg-benchmarks/kubernetes-client-go'
+    'kubernetes/client-go'
+    >>> _normalize_repo("etcd-io/etcd")
+    'etcd-io/etcd'
     """
-    return _HOSTING_PREFIX_RE.sub("", repo)
+    repo = _HOSTING_PREFIX_RE.sub("", repo)
+    return _SG_MIRROR_ALIASES.get(repo, repo)
 
 
 def check_file_set_match(
diff --git a/benchmarks/ccb_mcp_crossorg/ccx-crossorg-066/tests/oracle_checks.py b/benchmarks/ccb_mcp_crossorg/ccx-crossorg-066/tests/oracle_checks.py
@@ -31,16 +31,32 @@
 # Agents sometimes forget to strip these; normalize before comparison.
 _HOSTING_PREFIX_RE = re.compile(r"^(?:github\.com|gitlab\.com|bitbucket\.org)/")
 
+# sg-benchmarks mirrors → upstream canonical names.
+# Allows oracle answers and agent answers to use either name interchangeably.
+_SG_MIRROR_ALIASES = {
+    "sg-benchmarks/kubernetes-client-go": "kubernetes/client-go",
+    "sg-benchmarks/kubernetes-api": "kubernetes/api",
+    "sg-benchmarks/etcd-io-etcd": "etcd-io/etcd",
+    "sg-benchmarks/expressjs-express": "expressjs/express",
+    "sg-benchmarks/grafana-loki": "grafana/loki",
+    "sg-benchmarks/grafana-mimir": "grafana/mimir",
+    "sg-benchmarks/prisma-prisma": "prisma/prisma",
+    "sg-benchmarks/lodash": "lodash/lodash",
+}
+
 
 def _normalize_repo(repo: str) -> str:
-    """Strip hosting-provider prefix from a repo name for fuzzy matching.
+    """Strip hosting-provider prefix and resolve sg-benchmarks aliases.
 
     >>> _normalize_repo("github.com/sg-benchmarks/kubernetes-client-go")
-    'sg-benchmarks/kubernetes-client-go'
+    'kubernetes/client-go'
     >>> _normalize_repo("sg-benchmarks/kubernetes-client-go")
-    'sg-benchmarks/kubernetes-client-go'
+    'kubernetes/client-go'
+    >>> _normalize_repo("etcd-io/etcd")
+    'etcd-io/etcd'
     """
-    return _HOSTING_PREFIX_RE.sub("", repo)
+    repo = _HOSTING_PREFIX_RE.sub("", repo)
+    return _SG_MIRROR_ALIASES.get(repo, repo)
 
 
 def check_file_set_match(
diff --git a/benchmarks/ccb_mcp_crossrepo_tracing/ccx-config-trace-010/tests/oracle_checks.py b/benchmarks/ccb_mcp_crossrepo_tracing/ccx-config-trace-010/tests/oracle_checks.py
@@ -31,16 +31,32 @@
 # Agents sometimes forget to strip these; normalize before comparison.
 _HOSTING_PREFIX_RE = re.compile(r"^(?:github\.com|gitlab\.com|bitbucket\.org)/")
 
+# sg-benchmarks mirrors → upstream canonical names.
+# Allows oracle answers and agent answers to use either name interchangeably.
+_SG_MIRROR_ALIASES = {
+    "sg-benchmarks/kubernetes-client-go": "kubernetes/client-go",
+    "sg-benchmarks/kubernetes-api": "kubernetes/api",
+    "sg-benchmarks/etcd-io-etcd": "etcd-io/etcd",
+    "sg-benchmarks/expressjs-express": "expressjs/express",
+    "sg-benchmarks/grafana-loki": "grafana/loki",
+    "sg-benchmarks/grafana-mimir": "grafana/mimir",
+    "sg-benchmarks/prisma-prisma": "prisma/prisma",
+    "sg-benchmarks/lodash": "lodash/lodash",
+}
+
 
 def _normalize_repo(repo: str) -> str:
-    """Strip hosting-provider prefix from a repo name for fuzzy matching.
+    """Strip hosting-provider prefix and resolve sg-benchmarks aliases.
 
     >>> _normalize_repo("github.com/sg-benchmarks/kubernetes-client-go")
-    'sg-benchmarks/kubernetes-client-go'
+    'kubernetes/client-go'
     >>> _normalize_repo("sg-benchmarks/kubernetes-client-go")
-    'sg-benchmarks/kubernetes-client-go'
+    'kubernetes/client-go'
+    >>> _normalize_repo("etcd-io/etcd")
+    'etcd-io/etcd'
     """
-    return _HOSTING_PREFIX_RE.sub("", repo)
+    repo = _HOSTING_PREFIX_RE.sub("", repo)
+    return _SG_MIRROR_ALIASES.get(repo, repo)
 
 
 def check_file_set_match(
diff --git a/benchmarks/ccb_mcp_crossrepo_tracing/ccx-dep-trace-001/tests/oracle_checks.py b/benchmarks/ccb_mcp_crossrepo_tracing/ccx-dep-trace-001/tests/oracle_checks.py
@@ -31,16 +31,32 @@
 # Agents sometimes forget to strip these; normalize before comparison.
 _HOSTING_PREFIX_RE = re.compile(r"^(?:github\.com|gitlab\.com|bitbucket\.org)/")
 
+# sg-benchmarks mirrors → upstream canonical names.
+# Allows oracle answers and agent answers to use either name interchangeably.
+_SG_MIRROR_ALIASES = {
+    "sg-benchmarks/kubernetes-client-go": "kubernetes/client-go",
+    "sg-benchmarks/kubernetes-api": "kubernetes/api",
+    "sg-benchmarks/etcd-io-etcd": "etcd-io/etcd",
+    "sg-benchmarks/expressjs-express": "expressjs/express",
+    "sg-benchmarks/grafana-loki": "grafana/loki",
+    "sg-benchmarks/grafana-mimir": "grafana/mimir",
+    "sg-benchmarks/prisma-prisma": "prisma/prisma",
+    "sg-benchmarks/lodash": "lodash/lodash",
+}
+
 
 def _normalize_repo(repo: str) -> str:
-    """Strip hosting-provider prefix from a repo name for fuzzy matching.
+    """Strip hosting-provider prefix and resolve sg-benchmarks aliases.
 
     >>> _normalize_repo("github.com/sg-benchmarks/kubernetes-client-go")
-    'sg-benchmarks/kubernetes-client-go'
+    'kubernetes/client-go'
     >>> _normalize_repo("sg-benchmarks/kubernetes-client-go")
-    'sg-benchmarks/kubernetes-client-go'
+    'kubernetes/client-go'
+    >>> _normalize_repo("etcd-io/etcd")
+    'etcd-io/etcd'
     """
-    return _HOSTING_PREFIX_RE.sub("", repo)
+    repo = _HOSTING_PREFIX_RE.sub("", repo)
+    return _SG_MIRROR_ALIASES.get(repo, repo)
 
 
 def check_file_set_match(
diff --git a/benchmarks/ccb_mcp_crossrepo_tracing/ccx-dep-trace-004/tests/oracle_checks.py b/benchmarks/ccb_mcp_crossrepo_tracing/ccx-dep-trace-004/tests/oracle_checks.py
@@ -31,16 +31,32 @@
 # Agents sometimes forget to strip these; normalize before comparison.
 _HOSTING_PREFIX_RE = re.compile(r"^(?:github\.com|gitlab\.com|bitbucket\.org)/")
 
+# sg-benchmarks mirrors → upstream canonical names.
+# Allows oracle answers and agent answers to use either name interchangeably.
+_SG_MIRROR_ALIASES = {
+    "sg-benchmarks/kubernetes-client-go": "kubernetes/client-go",
+    "sg-benchmarks/kubernetes-api": "kubernetes/api",
+    "sg-benchmarks/etcd-io-etcd": "etcd-io/etcd",
+    "sg-benchmarks/expressjs-express": "expressjs/express",
+    "sg-benchmarks/grafana-loki": "grafana/loki",
+    "sg-benchmarks/grafana-mimir": "grafana/mimir",
+    "sg-benchmarks/prisma-prisma": "prisma/prisma",
+    "sg-benchmarks/lodash": "lodash/lodash",
+}
+
 
 def _normalize_repo(repo: str) -> str:
-    """Strip hosting-provider prefix from a repo name for fuzzy matching.
+    """Strip hosting-provider prefix and resolve sg-benchmarks aliases.
 
     >>> _normalize_repo("github.com/sg-benchmarks/kubernetes-client-go")
-    'sg-benchmarks/kubernetes-client-go'
+    'kubernetes/client-go'
     >>> _normalize_repo("sg-benchmarks/kubernetes-client-go")
-    'sg-benchmarks/kubernetes-client-go'
+    'kubernetes/client-go'
+    >>> _normalize_repo("etcd-io/etcd")
+    'etcd-io/etcd'
     """
-    return _HOSTING_PREFIX_RE.sub("", repo)
+    repo = _HOSTING_PREFIX_RE.sub("", repo)
+    return _SG_MIRROR_ALIASES.get(repo, repo)
 
 
 def check_file_set_match(
diff --git a/benchmarks/ccb_mcp_incident/ccx-incident-031/tests/oracle_checks.py b/benchmarks/ccb_mcp_incident/ccx-incident-031/tests/oracle_checks.py
@@ -31,16 +31,32 @@
 # Agents sometimes forget to strip these; normalize before comparison.
 _HOSTING_PREFIX_RE = re.compile(r"^(?:github\.com|gitlab\.com|bitbucket\.org)/")
 
+# sg-benchmarks mirrors → upstream canonical names.
+# Allows oracle answers and agent answers to use either name interchangeably.
+_SG_MIRROR_ALIASES = {
+    "sg-benchmarks/kubernetes-client-go": "kubernetes/client-go",
+    "sg-benchmarks/kubernetes-api": "kubernetes/api",
+    "sg-benchmarks/etcd-io-etcd": "etcd-io/etcd",
+    "sg-benchmarks/expressjs-express": "expressjs/express",
+    "sg-benchmarks/grafana-loki": "grafana/loki",
+    "sg-benchmarks/grafana-mimir": "grafana/mimir",
+    "sg-benchmarks/prisma-prisma": "prisma/prisma",
+    "sg-benchmarks/lodash": "lodash/lodash",
+}
+
 
 def _normalize_repo(repo: str) -> str:
-    """Strip hosting-provider prefix from a repo name for fuzzy matching.
+    """Strip hosting-provider prefix and resolve sg-benchmarks aliases.
 
     >>> _normalize_repo("github.com/sg-benchmarks/kubernetes-client-go")
-    'sg-benchmarks/kubernetes-client-go'
+    'kubernetes/client-go'
     >>> _normalize_repo("sg-benchmarks/kubernetes-client-go")
-    'sg-benchmarks/kubernetes-client-go'
+    'kubernetes/client-go'
+    >>> _normalize_repo("etcd-io/etcd")
+    'etcd-io/etcd'
     """
-    return _HOSTING_PREFIX_RE.sub("", repo)
+    repo = _HOSTING_PREFIX_RE.sub("", repo)
+    return _SG_MIRROR_ALIASES.get(repo, repo)
 
 
 def check_file_set_match(
diff --git a/benchmarks/ccb_mcp_onboarding/ccx-explore-042-ds/tests/oracle_checks.py b/benchmarks/ccb_mcp_onboarding/ccx-explore-042-ds/tests/oracle_checks.py
@@ -31,16 +31,32 @@
 # Agents sometimes forget to strip these; normalize before comparison.
 _HOSTING_PREFIX_RE = re.compile(r"^(?:github\.com|gitlab\.com|bitbucket\.org)/")
 
+# sg-benchmarks mirrors → upstream canonical names.
+# Allows oracle answers and agent answers to use either name interchangeably.
+_SG_MIRROR_ALIASES = {
+    "sg-benchmarks/kubernetes-client-go": "kubernetes/client-go",
+    "sg-benchmarks/kubernetes-api": "kubernetes/api",
+    "sg-benchmarks/etcd-io-etcd": "etcd-io/etcd",
+    "sg-benchmarks/expressjs-express": "expressjs/express",
+    "sg-benchmarks/grafana-loki": "grafana/loki",
+    "sg-benchmarks/grafana-mimir": "grafana/mimir",
+    "sg-benchmarks/prisma-prisma": "prisma/prisma",
+    "sg-benchmarks/lodash": "lodash/lodash",
+}
+
 
 def _normalize_repo(repo: str) -> str:
-    """Strip hosting-provider prefix from a repo name for fuzzy matching.
+    """Strip hosting-provider prefix and resolve sg-benchmarks aliases.
 
     >>> _normalize_repo("github.com/sg-benchmarks/kubernetes-client-go")
-    'sg-benchmarks/kubernetes-client-go'
+    'kubernetes/client-go'
     >>> _normalize_repo("sg-benchmarks/kubernetes-client-go")
-    'sg-benchmarks/kubernetes-client-go'
+    'kubernetes/client-go'
+    >>> _normalize_repo("etcd-io/etcd")
+    'etcd-io/etcd'
     """
-    return _HOSTING_PREFIX_RE.sub("", repo)
+    repo = _HOSTING_PREFIX_RE.sub("", repo)
+    return _SG_MIRROR_ALIASES.get(repo, repo)
 
 
 def check_file_set_match(
diff --git a/benchmarks/ccb_mcp_onboarding/ccx-onboard-041/tests/oracle_checks.py b/benchmarks/ccb_mcp_onboarding/ccx-onboard-041/tests/oracle_checks.py
@@ -31,16 +31,32 @@
 # Agents sometimes forget to strip these; normalize before comparison.
 _HOSTING_PREFIX_RE = re.compile(r"^(?:github\.com|gitlab\.com|bitbucket\.org)/")
 
+# sg-benchmarks mirrors → upstream canonical names.
+# Allows oracle answers and agent answers to use either name interchangeably.
+_SG_MIRROR_ALIASES = {
+    "sg-benchmarks/kubernetes-client-go": "kubernetes/client-go",
+    "sg-benchmarks/kubernetes-api": "kubernetes/api",
+    "sg-benchmarks/etcd-io-etcd": "etcd-io/etcd",
+    "sg-benchmarks/expressjs-express": "expressjs/express",
+    "sg-benchmarks/grafana-loki": "grafana/loki",
+    "sg-benchmarks/grafana-mimir": "grafana/mimir",
+    "sg-benchmarks/prisma-prisma": "prisma/prisma",
+    "sg-benchmarks/lodash": "lodash/lodash",
+}
+
 
 def _normalize_repo(repo: str) -> str:
-    """Strip hosting-provider prefix from a repo name for fuzzy matching.
+    """Strip hosting-provider prefix and resolve sg-benchmarks aliases.
 
     >>> _normalize_repo("github.com/sg-benchmarks/kubernetes-client-go")
-    'sg-benchmarks/kubernetes-client-go'
+    'kubernetes/client-go'
     >>> _normalize_repo("sg-benchmarks/kubernetes-client-go")
-    'sg-benchmarks/kubernetes-client-go'
+    'kubernetes/client-go'
+    >>> _normalize_repo("etcd-io/etcd")
+    'etcd-io/etcd'
     """
-    return _HOSTING_PREFIX_RE.sub("", repo)
+    repo = _HOSTING_PREFIX_RE.sub("", repo)
+    return _SG_MIRROR_ALIASES.get(repo, repo)
 
 
 def check_file_set_match(
diff --git a/benchmarks/ccb_mcp_onboarding/ccx-onboard-050-ds/tests/oracle_checks.py b/benchmarks/ccb_mcp_onboarding/ccx-onboard-050-ds/tests/oracle_checks.py
@@ -31,16 +31,32 @@
 # Agents sometimes forget to strip these; normalize before comparison.
 _HOSTING_PREFIX_RE = re.compile(r"^(?:github\.com|gitlab\.com|bitbucket\.org)/")
 
+# sg-benchmarks mirrors → upstream canonical names.
+# Allows oracle answers and agent answers to use either name interchangeably.
+_SG_MIRROR_ALIASES = {
+    "sg-benchmarks/kubernetes-client-go": "kubernetes/client-go",
+    "sg-benchmarks/kubernetes-api": "kubernetes/api",
+    "sg-benchmarks/etcd-io-etcd": "etcd-io/etcd",
+    "sg-benchmarks/expressjs-express": "expressjs/express",
+    "sg-benchmarks/grafana-loki": "grafana/loki",
+    "sg-benchmarks/grafana-mimir": "grafana/mimir",
+    "sg-benchmarks/prisma-prisma": "prisma/prisma",
+    "sg-benchmarks/lodash": "lodash/lodash",
+}
+
 
 def _normalize_repo(repo: str) -> str:
-    """Strip hosting-provider prefix from a repo name for fuzzy matching.
+    """Strip hosting-provider prefix and resolve sg-benchmarks aliases.
 
     >>> _normalize_repo("github.com/sg-benchmarks/kubernetes-client-go")
-    'sg-benchmarks/kubernetes-client-go'
+    'kubernetes/client-go'
     >>> _normalize_repo("sg-benchmarks/kubernetes-client-go")
-    'sg-benchmarks/kubernetes-client-go'
+    'kubernetes/client-go'
+    >>> _normalize_repo("etcd-io/etcd")
+    'etcd-io/etcd'
     """
-    return _HOSTING_PREFIX_RE.sub("", repo)
+    repo = _HOSTING_PREFIX_RE.sub("", repo)
+    return _SG_MIRROR_ALIASES.get(repo, repo)
 
 
 def check_file_set_match(
diff --git a/benchmarks/ccb_mcp_platform/ccx-explore-091-ds/tests/oracle_checks.py b/benchmarks/ccb_mcp_platform/ccx-explore-091-ds/tests/oracle_checks.py
@@ -31,16 +31,32 @@
 # Agents sometimes forget to strip these; normalize before comparison.
 _HOSTING_PREFIX_RE = re.compile(r"^(?:github\.com|gitlab\.com|bitbucket\.org)/")
 
+# sg-benchmarks mirrors → upstream canonical names.
+# Allows oracle answers and agent answers to use either name interchangeably.
+_SG_MIRROR_ALIASES = {
+    "sg-benchmarks/kubernetes-client-go": "kubernetes/client-go",
+    "sg-benchmarks/kubernetes-api": "kubernetes/api",
+    "sg-benchmarks/etcd-io-etcd": "etcd-io/etcd",
+    "sg-benchmarks/expressjs-express": "expressjs/express",
+    "sg-benchmarks/grafana-loki": "grafana/loki",
+    "sg-benchmarks/grafana-mimir": "grafana/mimir",
+    "sg-benchmarks/prisma-prisma": "prisma/prisma",
+    "sg-benchmarks/lodash": "lodash/lodash",
+}
+
 
 def _normalize_repo(repo: str) -> str:
-    """Strip hosting-provider prefix from a repo name for fuzzy matching.
+    """Strip hosting-provider prefix and resolve sg-benchmarks aliases.
 
     >>> _normalize_repo("github.com/sg-benchmarks/kubernetes-client-go")
-    'sg-benchmarks/kubernetes-client-go'
+    'kubernetes/client-go'
     >>> _normalize_repo("sg-benchmarks/kubernetes-client-go")
-    'sg-benchmarks/kubernetes-client-go'
+    'kubernetes/client-go'
+    >>> _normalize_repo("etcd-io/etcd")
+    'etcd-io/etcd'
     """
-    return _HOSTING_PREFIX_RE.sub("", repo)
+    repo = _HOSTING_PREFIX_RE.sub("", repo)
+    return _SG_MIRROR_ALIASES.get(repo, repo)
 
 
 def check_file_set_match(
diff --git a/benchmarks/ccb_mcp_security/ccx-vuln-remed-011/tests/oracle_checks.py b/benchmarks/ccb_mcp_security/ccx-vuln-remed-011/tests/oracle_checks.py
diff --git a/benchmarks/ccb_mcp_security/ccx-vuln-remed-014/tests/oracle_checks.py b/benchmarks/ccb_mcp_security/ccx-vuln-remed-014/tests/oracle_checks.py
diff --git a/configs/instance_to_mirror.json b/configs/instance_to_mirror.json
diff --git a/configs/run_selected_tasks.sh b/configs/run_selected_tasks.sh
diff --git a/scripts/remirror_mcp_unique_repos.sh b/scripts/remirror_mcp_unique_repos.sh
