fix: add || true to oracle_checks.py invocation in 61 eval.sh files

sjarmak · claude · sjarmak · commit a5d9671bdcf5 · 2026-02-24T17:08:21.000Z
oracle_checks.py exits 1 when composite score is 0 (by design, for
SWE-Factory exit-code-first pattern). With set -euo pipefail in eval.sh,
this kills the script before reward.txt gets written, causing Harbor to
report RewardFileNotFoundError for every zero-score task.

Fix: append `|| true` to the SCORE= pipeline so eval.sh can proceed to
write reward.txt regardless of oracle_checks.py exit code. Also fixes
the eval.sh.j2 template to prevent regression in future task generation.

Affected: 61 of 81 MCP-unique eval.sh files (the original 20 already
had the fix from initial generation).

Co-Authored-By: Claude Opus 4.6 &lt;noreply@anthropic.com&gt;
diff --git a/benchmarks/ccb_mcp_compliance/ccx-compliance-052/tests/eval.sh b/benchmarks/ccb_mcp_compliance/ccx-compliance-052/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_compliance/ccx-compliance-053/tests/eval.sh b/benchmarks/ccb_mcp_compliance/ccx-compliance-053/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_compliance/ccx-compliance-115/tests/eval.sh b/benchmarks/ccb_mcp_compliance/ccx-compliance-115/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_compliance/ccx-compliance-118/tests/eval.sh b/benchmarks/ccb_mcp_compliance/ccx-compliance-118/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_compliance/ccx-compliance-124/tests/eval.sh b/benchmarks/ccb_mcp_compliance/ccx-compliance-124/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_crossorg/ccx-crossorg-062/tests/eval.sh b/benchmarks/ccb_mcp_crossorg/ccx-crossorg-062/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_crossorg/ccx-crossorg-121/tests/eval.sh b/benchmarks/ccb_mcp_crossorg/ccx-crossorg-121/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_crossorg/ccx-crossorg-132/tests/eval.sh b/benchmarks/ccb_mcp_crossorg/ccx-crossorg-132/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_crossrepo/ccx-dep-trace-106/tests/eval.sh b/benchmarks/ccb_mcp_crossrepo/ccx-dep-trace-106/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_crossrepo_tracing/ccx-config-trace-003/tests/eval.sh b/benchmarks/ccb_mcp_crossrepo_tracing/ccx-config-trace-003/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_crossrepo_tracing/ccx-dep-trace-002/tests/eval.sh b/benchmarks/ccb_mcp_crossrepo_tracing/ccx-dep-trace-002/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_crossrepo_tracing/ccx-dep-trace-102/tests/eval.sh b/benchmarks/ccb_mcp_crossrepo_tracing/ccx-dep-trace-102/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_crossrepo_tracing/ccx-dep-trace-116/tests/eval.sh b/benchmarks/ccb_mcp_crossrepo_tracing/ccx-dep-trace-116/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_crossrepo_tracing/ccx-dep-trace-123/tests/eval.sh b/benchmarks/ccb_mcp_crossrepo_tracing/ccx-dep-trace-123/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_crossrepo_tracing/ccx-dep-trace-133/tests/eval.sh b/benchmarks/ccb_mcp_crossrepo_tracing/ccx-dep-trace-133/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_domain/ccx-domain-071/tests/eval.sh b/benchmarks/ccb_mcp_domain/ccx-domain-071/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_domain/ccx-domain-072/tests/eval.sh b/benchmarks/ccb_mcp_domain/ccx-domain-072/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_domain/ccx-domain-073/tests/eval.sh b/benchmarks/ccb_mcp_domain/ccx-domain-073/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_domain/ccx-domain-074/tests/eval.sh b/benchmarks/ccb_mcp_domain/ccx-domain-074/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_domain/ccx-domain-101/tests/eval.sh b/benchmarks/ccb_mcp_domain/ccx-domain-101/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_domain/ccx-domain-112/tests/eval.sh b/benchmarks/ccb_mcp_domain/ccx-domain-112/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_domain/ccx-domain-120/tests/eval.sh b/benchmarks/ccb_mcp_domain/ccx-domain-120/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_domain/ccx-domain-129/tests/eval.sh b/benchmarks/ccb_mcp_domain/ccx-domain-129/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_domain/ccx-domain-137/tests/eval.sh b/benchmarks/ccb_mcp_domain/ccx-domain-137/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_domain/ccx-domain-140/tests/eval.sh b/benchmarks/ccb_mcp_domain/ccx-domain-140/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_incident/ccx-incident-032/tests/eval.sh b/benchmarks/ccb_mcp_incident/ccx-incident-032/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_incident/ccx-incident-033/tests/eval.sh b/benchmarks/ccb_mcp_incident/ccx-incident-033/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_incident/ccx-incident-108/tests/eval.sh b/benchmarks/ccb_mcp_incident/ccx-incident-108/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_incident/ccx-incident-110/tests/eval.sh b/benchmarks/ccb_mcp_incident/ccx-incident-110/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_incident/ccx-incident-113/tests/eval.sh b/benchmarks/ccb_mcp_incident/ccx-incident-113/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_incident/ccx-incident-125/tests/eval.sh b/benchmarks/ccb_mcp_incident/ccx-incident-125/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_incident/ccx-incident-131/tests/eval.sh b/benchmarks/ccb_mcp_incident/ccx-incident-131/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_incident/ccx-incident-139/tests/eval.sh b/benchmarks/ccb_mcp_incident/ccx-incident-139/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_migration/ccx-migration-022/tests/eval.sh b/benchmarks/ccb_mcp_migration/ccx-migration-022/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_migration/ccx-migration-026/tests/eval.sh b/benchmarks/ccb_mcp_migration/ccx-migration-026/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_migration/ccx-migration-107/tests/eval.sh b/benchmarks/ccb_mcp_migration/ccx-migration-107/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_migration/ccx-migration-114/tests/eval.sh b/benchmarks/ccb_mcp_migration/ccx-migration-114/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_migration/ccx-migration-117/tests/eval.sh b/benchmarks/ccb_mcp_migration/ccx-migration-117/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_onboarding/ccx-onboard-043/tests/eval.sh b/benchmarks/ccb_mcp_onboarding/ccx-onboard-043/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_onboarding/ccx-onboard-044/tests/eval.sh b/benchmarks/ccb_mcp_onboarding/ccx-onboard-044/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_onboarding/ccx-onboard-103/tests/eval.sh b/benchmarks/ccb_mcp_onboarding/ccx-onboard-103/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_onboarding/ccx-onboard-109/tests/eval.sh b/benchmarks/ccb_mcp_onboarding/ccx-onboard-109/tests/eval.sh
@@ -51,7 +51,7 @@ if [ ! -f "$ORACLE_CHECKS" ]; then
 fi
 
 echo "Running oracle checks..."
-SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1)
+SCORE=$(python3 "$ORACLE_CHECKS" --answer "$ANSWER_PATH" --spec "$TASK_SPEC_PATH" --verbose 2>&1 | tee /dev/stderr | tail -1) || true
 
 # Validate score is a number
 if ! echo "$SCORE" | python3 -c "import sys; float(sys.stdin.read().strip())" 2>/dev/null; then
diff --git a/benchmarks/ccb_mcp_onboarding/ccx-onboard-128/tests/eval.sh b/benchmarks/ccb_mcp_onboarding/ccx-onboard-128/tests/eval.sh
diff --git a/benchmarks/ccb_mcp_onboarding/ccx-onboard-134/tests/eval.sh b/benchmarks/ccb_mcp_onboarding/ccx-onboard-134/tests/eval.sh
diff --git a/benchmarks/ccb_mcp_onboarding/ccx-onboard-136/tests/eval.sh b/benchmarks/ccb_mcp_onboarding/ccx-onboard-136/tests/eval.sh
diff --git a/benchmarks/ccb_mcp_onboarding/ccx-onboard-138/tests/eval.sh b/benchmarks/ccb_mcp_onboarding/ccx-onboard-138/tests/eval.sh
diff --git a/benchmarks/ccb_mcp_org/ccx-agentic-081/tests/eval.sh b/benchmarks/ccb_mcp_org/ccx-agentic-081/tests/eval.sh
diff --git a/benchmarks/ccb_mcp_org/ccx-agentic-082/tests/eval.sh b/benchmarks/ccb_mcp_org/ccx-agentic-082/tests/eval.sh
diff --git a/benchmarks/ccb_mcp_org/ccx-agentic-083/tests/eval.sh b/benchmarks/ccb_mcp_org/ccx-agentic-083/tests/eval.sh
diff --git a/benchmarks/ccb_mcp_org/ccx-agentic-122/tests/eval.sh b/benchmarks/ccb_mcp_org/ccx-agentic-122/tests/eval.sh
diff --git a/benchmarks/ccb_mcp_org/ccx-agentic-127/tests/eval.sh b/benchmarks/ccb_mcp_org/ccx-agentic-127/tests/eval.sh
diff --git a/benchmarks/ccb_mcp_platform/ccx-platform-104/tests/eval.sh b/benchmarks/ccb_mcp_platform/ccx-platform-104/tests/eval.sh
diff --git a/benchmarks/ccb_mcp_platform/ccx-platform-119/tests/eval.sh b/benchmarks/ccb_mcp_platform/ccx-platform-119/tests/eval.sh
diff --git a/benchmarks/ccb_mcp_security/ccx-vuln-remed-012/tests/eval.sh b/benchmarks/ccb_mcp_security/ccx-vuln-remed-012/tests/eval.sh
diff --git a/benchmarks/ccb_mcp_security/ccx-vuln-remed-013/tests/eval.sh b/benchmarks/ccb_mcp_security/ccx-vuln-remed-013/tests/eval.sh
diff --git a/benchmarks/ccb_mcp_security/ccx-vuln-remed-105/tests/eval.sh b/benchmarks/ccb_mcp_security/ccx-vuln-remed-105/tests/eval.sh
diff --git a/benchmarks/ccb_mcp_security/ccx-vuln-remed-111/tests/eval.sh b/benchmarks/ccb_mcp_security/ccx-vuln-remed-111/tests/eval.sh
diff --git a/benchmarks/ccb_mcp_security/ccx-vuln-remed-126/tests/eval.sh b/benchmarks/ccb_mcp_security/ccx-vuln-remed-126/tests/eval.sh
diff --git a/benchmarks/ccb_mcp_security/ccx-vuln-remed-130/tests/eval.sh b/benchmarks/ccb_mcp_security/ccx-vuln-remed-130/tests/eval.sh
diff --git a/benchmarks/ccb_mcp_security/ccx-vuln-remed-135/tests/eval.sh b/benchmarks/ccb_mcp_security/ccx-vuln-remed-135/tests/eval.sh
diff --git a/benchmarks/ccb_mcp_security/ccx-vuln-remed-141/tests/eval.sh b/benchmarks/ccb_mcp_security/ccx-vuln-remed-141/tests/eval.sh
diff --git a/templates/mcp_unique_task/eval.sh.j2 b/templates/mcp_unique_task/eval.sh.j2
