Skip to content

Commit 87d48f2

Browse files
sjarmakclaude
andcommitted
fix: resolve PRE_FIX_REV dynamically in ccb_fix verifiers
sg-evals mirrors use orphan commits with different SHAs than upstream. Hardcoded upstream SHAs caused `git diff <sha>` to silently return empty, making all diff_similarity verifiers report 0.0 regardless of agent changes. Fix: use `git rev-parse HEAD` to resolve the actual initial commit. Affects 8 tasks: 5 pytorch, 2 envoy, 1 terraform. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 5c1fc7f commit 87d48f2

File tree

8 files changed

+16
-8
lines changed
  • benchmarks/ccb_fix
    • envoy-dfp-host-leak-fix-001/tests
    • envoy-udp-proxy-cds-fix-001/tests
    • pytorch-cudnn-version-fix-001/tests
    • pytorch-dynamo-keyerror-fix-001/tests
    • pytorch-release-210-fix-001/tests
    • pytorch-relu-gelu-fusion-fix-001/tests
    • pytorch-tracer-graph-cleanup-fix-001/tests
    • terraform-plan-null-unknown-fix-001/tests

8 files changed

+16
-8
lines changed

benchmarks/ccb_fix/envoy-dfp-host-leak-fix-001/tests/test.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@ set -eo pipefail
1212
mkdir -p /logs/verifier
1313
cd "${VERIFY_REPO:-/workspace}"
1414
git config --global --add safe.directory /workspace 2>/dev/null || true
15-
PRE_FIX_REV="5160151e14837c6f352b9d0b3a9f55119a9317e8"
15+
# Resolve initial commit — mirrors use orphan commits with different SHAs than upstream
16+
PRE_FIX_REV=$(git rev-parse HEAD 2>/dev/null || echo "HEAD")
1617
python3 /tests/verify_diff.py \
1718
--expected /tests/expected.diff \
1819
--pre-fix-rev "$PRE_FIX_REV" \

benchmarks/ccb_fix/envoy-udp-proxy-cds-fix-001/tests/test.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@ set -eo pipefail
1212
mkdir -p /logs/verifier
1313
cd "${VERIFY_REPO:-/workspace}"
1414
git config --global --add safe.directory /workspace 2>/dev/null || true
15-
PRE_FIX_REV="1ae957c1f92b8e0b0322ab702c67612aa618d214"
15+
# Resolve initial commit — mirrors use orphan commits with different SHAs than upstream
16+
PRE_FIX_REV=$(git rev-parse HEAD 2>/dev/null || echo "HEAD")
1617
python3 /tests/verify_diff.py \
1718
--expected /tests/expected.diff \
1819
--pre-fix-rev "$PRE_FIX_REV" \

benchmarks/ccb_fix/pytorch-cudnn-version-fix-001/tests/test.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@ set -eo pipefail
1212
mkdir -p /logs/verifier
1313
cd "${VERIFY_REPO:-/workspace}"
1414
git config --global --add safe.directory /workspace 2>/dev/null || true
15-
PRE_FIX_REV="5811a8d7da873dd699ff6687092c225caffcf1bb"
15+
# Resolve initial commit — mirrors use orphan commits with different SHAs than upstream
16+
PRE_FIX_REV=$(git rev-parse HEAD 2>/dev/null || echo "HEAD")
1617
python3 /tests/verify_diff.py \
1718
--expected /tests/expected.diff \
1819
--pre-fix-rev "$PRE_FIX_REV" \

benchmarks/ccb_fix/pytorch-dynamo-keyerror-fix-001/tests/test.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@ set -eo pipefail
1212
mkdir -p /logs/verifier
1313
cd "${VERIFY_REPO:-/workspace}"
1414
git config --global --add safe.directory /workspace 2>/dev/null || true
15-
PRE_FIX_REV="cbe1a35dbdfdc9b7490212bdd812f3e2f4ce9e2d"
15+
# Resolve initial commit — mirrors use orphan commits with different SHAs than upstream
16+
PRE_FIX_REV=$(git rev-parse HEAD 2>/dev/null || echo "HEAD")
1617
python3 /tests/verify_diff.py \
1718
--expected /tests/expected.diff \
1819
--pre-fix-rev "$PRE_FIX_REV" \

benchmarks/ccb_fix/pytorch-release-210-fix-001/tests/test.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@ set -eo pipefail
1212
mkdir -p /logs/verifier
1313
cd "${VERIFY_REPO:-/workspace}"
1414
git config --global --add safe.directory /workspace 2>/dev/null || true
15-
PRE_FIX_REV="863edc787f5fa7950e0f3f86d4952641e1ef9c2c"
15+
# Resolve initial commit — mirrors use orphan commits with different SHAs than upstream
16+
PRE_FIX_REV=$(git rev-parse HEAD 2>/dev/null || echo "HEAD")
1617
python3 /tests/verify_diff.py \
1718
--expected /tests/expected.diff \
1819
--pre-fix-rev "$PRE_FIX_REV" \

benchmarks/ccb_fix/pytorch-relu-gelu-fusion-fix-001/tests/test.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@ set -eo pipefail
1212
mkdir -p /logs/verifier
1313
cd "${VERIFY_REPO:-/workspace}"
1414
git config --global --add safe.directory /workspace 2>/dev/null || true
15-
PRE_FIX_REV="ca2466126a00ba8fd877f5a185e40e36ddaceb87"
15+
# Resolve initial commit — mirrors use orphan commits with different SHAs than upstream
16+
PRE_FIX_REV=$(git rev-parse HEAD 2>/dev/null || echo "HEAD")
1617
python3 /tests/verify_diff.py \
1718
--expected /tests/expected.diff \
1819
--pre-fix-rev "$PRE_FIX_REV" \

benchmarks/ccb_fix/pytorch-tracer-graph-cleanup-fix-001/tests/test.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@ set -eo pipefail
1212
mkdir -p /logs/verifier
1313
cd "${VERIFY_REPO:-/workspace}"
1414
git config --global --add safe.directory /workspace 2>/dev/null || true
15-
PRE_FIX_REV="d18007a1d0b1ea9a6b2bcd74e8b128eab4b434a0"
15+
# Resolve initial commit — mirrors use orphan commits with different SHAs than upstream
16+
PRE_FIX_REV=$(git rev-parse HEAD 2>/dev/null || echo "HEAD")
1617
python3 /tests/verify_diff.py \
1718
--expected /tests/expected.diff \
1819
--pre-fix-rev "$PRE_FIX_REV" \

benchmarks/ccb_fix/terraform-plan-null-unknown-fix-001/tests/test.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@ set -eo pipefail
1212
mkdir -p /logs/verifier
1313
cd "${VERIFY_REPO:-/workspace}"
1414
git config --global --add safe.directory /workspace 2>/dev/null || true
15-
PRE_FIX_REV="abd6b9ef1ba5e98ed273f59f667d3b9f2077a87b"
15+
# Resolve initial commit — mirrors use orphan commits with different SHAs than upstream
16+
PRE_FIX_REV=$(git rev-parse HEAD 2>/dev/null || echo "HEAD")
1617
python3 /tests/verify_diff.py \
1718
--expected /tests/expected.diff \
1819
--pre-fix-rev "$PRE_FIX_REV" \

0 commit comments

Comments
 (0)