fix: resolve Docker build failures for multi-repo and private sg-benchmarks tasks

sjarmak · claude · sjarmak · commit fd41857aade4 · 2026-02-23T13:57:25.000Z
Docker build containers can't access the host's git credential store, causing
git clone failures for private sg-benchmarks repos. Three fixes applied:

1. Created 8 new base images (flask, requests, etcd, containerd, numpy,
   scikit-learn, pandas, rust) using local clone cache + COPY pattern
2. Rewrote 6 multi-repo task Dockerfiles to use multi-stage COPY --from
   instead of git clone during build
3. Added credential injection to prebuild_images.sh so the standard
   prebuild phase injects GitHub tokens via build-arg for any Dockerfile
   that clones from sg-benchmarks

Also added run_all_sdlc_suites.sh for sequential multi-suite execution
and prebuild_with_credentials.sh for standalone credential-based prebuild.

Co-Authored-By: Claude Opus 4.6 &lt;noreply@anthropic.com&gt;
diff --git a/base_images/Dockerfile.containerd-317286ac b/base_images/Dockerfile.containerd-317286ac
@@ -0,0 +1,14 @@
+FROM golang:1.23-bookworm
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY _clone_cache/containerd-317286ac/ /workspace/
+
+RUN cd /workspace && \
+    git config user.email "agent@example.com" && \
+    git config user.name "Agent"
diff --git a/base_images/Dockerfile.etcd-d89978e8 b/base_images/Dockerfile.etcd-d89978e8
@@ -0,0 +1,14 @@
+FROM golang:1.23-bookworm
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY _clone_cache/etcd-d89978e8/ /workspace/
+
+RUN cd /workspace && \
+    git config user.email "agent@example.com" && \
+    git config user.name "Agent"
diff --git a/base_images/Dockerfile.flask-798e006f b/base_images/Dockerfile.flask-798e006f
@@ -0,0 +1,14 @@
+FROM python:3.11-slim
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY _clone_cache/flask-798e006f/ /workspace/
+
+RUN cd /workspace && \
+    git config user.email "agent@example.com" && \
+    git config user.name "Agent"
diff --git a/base_images/Dockerfile.numpy-a639fbf5 b/base_images/Dockerfile.numpy-a639fbf5
@@ -0,0 +1,14 @@
+FROM python:3.11-slim
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY _clone_cache/numpy-a639fbf5/ /workspace/
+
+RUN cd /workspace && \
+    git config user.email "agent@example.com" && \
+    git config user.name "Agent"
diff --git a/base_images/Dockerfile.pandas-41968da5 b/base_images/Dockerfile.pandas-41968da5
@@ -0,0 +1,14 @@
+FROM python:3.11-slim
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY _clone_cache/pandas-41968da5/ /workspace/
+
+RUN cd /workspace && \
+    git config user.email "agent@example.com" && \
+    git config user.name "Agent"
diff --git a/base_images/Dockerfile.requests-421b8733 b/base_images/Dockerfile.requests-421b8733
@@ -0,0 +1,14 @@
+FROM python:3.11-slim
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY _clone_cache/requests-421b8733/ /workspace/
+
+RUN cd /workspace && \
+    git config user.email "agent@example.com" && \
+    git config user.name "Agent"
diff --git a/base_images/Dockerfile.rust-01f6ddf7 b/base_images/Dockerfile.rust-01f6ddf7
@@ -0,0 +1,14 @@
+FROM rust:1.82-bookworm
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY _clone_cache/rust-01f6ddf7/ /workspace/
+
+RUN cd /workspace && \
+    git config user.email "agent@example.com" && \
+    git config user.name "Agent"
diff --git a/base_images/Dockerfile.scikit-learn-cb7e82dd b/base_images/Dockerfile.scikit-learn-cb7e82dd
@@ -0,0 +1,14 @@
+FROM python:3.11-slim
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY _clone_cache/scikit-learn-cb7e82dd/ /workspace/
+
+RUN cd /workspace && \
+    git config user.email "agent@example.com" && \
+    git config user.name "Agent"
diff --git a/base_images/_clone_cache/.gitignore b/base_images/_clone_cache/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/base_images/build.sh b/base_images/build.sh
@@ -57,6 +57,14 @@ IMAGES=(
     "Dockerfile.curl-09e25b9d      ccb-repo-curl-09e25b9d"
     "Dockerfile.envoy-1d0ba73a     ccb-repo-envoy-1d0ba73a"
     "Dockerfile.envoy-d7809ba2     ccb-repo-envoy-d7809ba2"
+    "Dockerfile.flask-798e006f     ccb-repo-flask-798e006f"
+    "Dockerfile.requests-421b8733  ccb-repo-requests-421b8733"
+    "Dockerfile.etcd-d89978e8      ccb-repo-etcd-d89978e8"
+    "Dockerfile.containerd-317286ac ccb-repo-containerd-317286ac"
+    "Dockerfile.numpy-a639fbf5     ccb-repo-numpy-a639fbf5"
+    "Dockerfile.scikit-learn-cb7e82dd ccb-repo-scikit-learn-cb7e82dd"
+    "Dockerfile.pandas-41968da5    ccb-repo-pandas-41968da5"
+    "Dockerfile.rust-01f6ddf7      ccb-repo-rust-01f6ddf7"
 )
 
 TOTAL_START=$SECONDS
diff --git a/benchmarks/ccb_build/python-http-class-naming-refac-001/environment/Dockerfile b/benchmarks/ccb_build/python-http-class-naming-refac-001/environment/Dockerfile
@@ -1,2 +1,25 @@
-FROM harbor-ccb_crossrepo:base
+# python-http-class-naming-refac-001 — multi-repo baseline
+# Repos cloned into /ccb_crossrepo/src/ to match verifier expectations.
+FROM ccb-repo-django-674eda1c AS django-src
+FROM ccb-repo-flask-798e006f AS flask-src
+FROM ccb-repo-requests-421b8733 AS requests-src
+
+FROM python:3.11-slim
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git \
+    curl \
+    ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /ccb_crossrepo/src
+COPY --from=django-src /workspace /ccb_crossrepo/src/django
+COPY --from=flask-src /workspace /ccb_crossrepo/src/flask
+COPY --from=requests-src /workspace /ccb_crossrepo/src/requests
+
 WORKDIR /workspace
+
+RUN git init && \
+    git config user.email "agent@example.com" && \
+    git config user.name "Agent" && \
+    git config --global safe.directory '*'
diff --git a/benchmarks/ccb_build/rust-subtype-relation-refac-001/environment/Dockerfile b/benchmarks/ccb_build/rust-subtype-relation-refac-001/environment/Dockerfile
@@ -1,3 +1,5 @@
+FROM ccb-repo-rust-01f6ddf7 AS rust-src
+
 FROM rust:1.82-bookworm
 
 WORKDIR /workspace
@@ -10,7 +12,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     python3-pip \
     && rm -rf /var/lib/apt/lists/*
 
-# Clone Rust compiler at pinned commit (1.93.1)
-RUN git clone --depth 1 https://github.com/sg-benchmarks/rust--01f6ddf7.git . && \
-    git config user.email "agent@example.com" && \
+COPY --from=rust-src /workspace /workspace
+
+RUN git config user.email "agent@example.com" && \
     git config user.name "Agent"
diff --git a/benchmarks/ccb_design/etcd-grpc-api-upgrade-001/environment/Dockerfile b/benchmarks/ccb_design/etcd-grpc-api-upgrade-001/environment/Dockerfile
@@ -1,2 +1,26 @@
-FROM harbor-ccb_crossrepo:base
+# etcd-grpc-api-upgrade-001 — multi-repo baseline
+# Repos cloned into /ccb_crossrepo/src/ to match verifier expectations.
+FROM ccb-repo-etcd-d89978e8 AS etcd-src
+FROM ccb-repo-k8s-8c9c67c0 AS k8s-src
+FROM ccb-repo-containerd-317286ac AS containerd-src
+
+FROM golang:1.23-bookworm
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git \
+    curl \
+    python3 \
+    ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /ccb_crossrepo/src
+COPY --from=etcd-src /workspace /ccb_crossrepo/src/etcd
+COPY --from=k8s-src /workspace /ccb_crossrepo/src/kubernetes
+COPY --from=containerd-src /workspace /ccb_crossrepo/src/containerd
+
 WORKDIR /workspace
+
+RUN git init && \
+    git config user.email "agent@example.com" && \
+    git config user.name "Agent" && \
+    git config --global safe.directory '*'
diff --git a/benchmarks/ccb_design/kafka-flink-streaming-arch-001/environment/Dockerfile b/benchmarks/ccb_design/kafka-flink-streaming-arch-001/environment/Dockerfile
@@ -1,22 +1,21 @@
+# kafka-flink-streaming-arch-001 — multi-repo baseline
+# Uses pre-built ccb-repo images to avoid git clone during Docker build.
+FROM ccb-repo-kafka-0753c489 AS kafka-src
+FROM ccb-repo-flink-0cc95fcc AS flink-src
+
 FROM eclipse-temurin:17-jdk
 
 WORKDIR /workspace
 
-# Install dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
     git \
     curl \
     python3 \
     python3-pip \
     && rm -rf /var/lib/apt/lists/*
 
-# Clone Apache Kafka at pinned tag (4.1.1) into /workspace/kafka/
-RUN git clone --depth 1 https://github.com/sg-benchmarks/kafka--0753c489.git /workspace/kafka && \
-    git config user.email "agent@example.com" && \
-    git config user.name "Agent"
-
-# Clone Apache Flink at pinned branch (release-2.2.0) into /workspace/flink/
-RUN git clone --depth 1 https://github.com/sg-benchmarks/flink--0cc95fcc.git /workspace/flink && \
-    git config user.email "agent@example.com" && \
-    git config user.name "Agent"
+COPY --from=kafka-src /workspace /workspace/kafka
+COPY --from=flink-src /workspace /workspace/flink
 
+RUN cd /workspace/kafka && git config user.email "agent@example.com" && git config user.name "Agent" && \
+    cd /workspace/flink && git config user.email "agent@example.com" && git config user.name "Agent"
diff --git a/benchmarks/ccb_understand/k8s-cri-containerd-reason-001/environment/Dockerfile b/benchmarks/ccb_understand/k8s-cri-containerd-reason-001/environment/Dockerfile
@@ -1,2 +1,24 @@
-FROM harbor-ccb_crossrepo:base
+# k8s-cri-containerd-reason-001 — multi-repo baseline
+# Repos cloned into /ccb_crossrepo/src/ to match verifier expectations.
+FROM ccb-repo-k8s-8c9c67c0 AS k8s-src
+FROM ccb-repo-containerd-317286ac AS containerd-src
+
+FROM golang:1.23-bookworm
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git \
+    curl \
+    python3 \
+    ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /ccb_crossrepo/src
+COPY --from=k8s-src /workspace /ccb_crossrepo/src/kubernetes
+COPY --from=containerd-src /workspace /ccb_crossrepo/src/containerd
+
 WORKDIR /workspace
+
+RUN git init && \
+    git config user.email "agent@example.com" && \
+    git config user.name "Agent" && \
+    git config --global safe.directory '*'
diff --git a/benchmarks/ccb_understand/numpy-dtype-localize-001/environment/Dockerfile b/benchmarks/ccb_understand/numpy-dtype-localize-001/environment/Dockerfile
@@ -1,2 +1,25 @@
-FROM harbor-ccb_crossrepo:base
+# numpy-dtype-localize-001 — multi-repo baseline
+# Repos cloned into /ccb_crossrepo/src/ to match verifier expectations.
+FROM ccb-repo-numpy-a639fbf5 AS numpy-src
+FROM ccb-repo-scikit-learn-cb7e82dd AS sklearn-src
+FROM ccb-repo-pandas-41968da5 AS pandas-src
+
+FROM python:3.11-slim
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git \
+    curl \
+    ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /ccb_crossrepo/src
+COPY --from=numpy-src /workspace /ccb_crossrepo/src/numpy
+COPY --from=sklearn-src /workspace /ccb_crossrepo/src/scikit-learn
+COPY --from=pandas-src /workspace /ccb_crossrepo/src/pandas
+
 WORKDIR /workspace
+
+RUN git init && \
+    git config user.email "agent@example.com" && \
+    git config user.name "Agent" && \
+    git config --global safe.directory '*'
diff --git a/configs/run_all_sdlc_suites.sh b/configs/run_all_sdlc_suites.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+# Run all 8 SDLC suites sequentially with haiku model.
+# Each suite runs paired configs (baseline + MCP) with full parallelism.
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+export MODEL=anthropic/claude-haiku-4-5-20251001
+
+SUITES=(build debug document fix secure test understand)
+
+echo "=============================================="
+echo "Running all SDLC suites with haiku"
+echo "Suites: ${SUITES[*]}"
+echo "Model: $MODEL"
+echo "=============================================="
+echo ""
+
+FAILED_SUITES=()
+
+for suite in "${SUITES[@]}"; do
+    echo ""
+    echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
+    echo ">>> Starting suite: ccb_${suite}"
+    echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
+    echo ""
+
+    if bash "${SCRIPT_DIR}/${suite}_2config.sh"; then
+        echo ">>> Suite ccb_${suite} completed successfully"
+    else
+        echo ">>> WARNING: Suite ccb_${suite} had errors (exit code: $?)"
+        FAILED_SUITES+=("$suite")
+    fi
+done
+
+echo ""
+echo "=============================================="
+echo "All SDLC suites complete!"
+echo "=============================================="
+
+if [ ${#FAILED_SUITES[@]} -gt 0 ]; then
+    echo "Suites with errors: ${FAILED_SUITES[*]}"
+else
+    echo "All suites completed successfully"
+fi
diff --git a/scripts/prebuild_images.sh b/scripts/prebuild_images.sh
@@ -185,15 +185,45 @@ build_one() {
 
     local log_file="${LOG_DIR}/${image_tag}.log"
     local start=$SECONDS
+    local build_args=()
+
+    # If the Dockerfile clones from sg-benchmarks, inject GitHub credentials.
+    # Docker build containers can't access the host's git credential store,
+    # so we inject a token via build-arg to authenticate HTTPS git clones.
+    local dockerfile="${context_dir}/Dockerfile"
+    if grep -q "git clone.*github.com/sg-benchmarks/" "$dockerfile" 2>/dev/null; then
+        local gh_token=""
+        if [ -f "$HOME/.git-credentials" ]; then
+            gh_token=$(grep "^https://.*@github.com" "$HOME/.git-credentials" | head -1 | sed 's|https://[^:]*:\([^@]*\)@github.com.*|\1|')
+        fi
+        if [ -n "$gh_token" ]; then
+            # Create a patched Dockerfile with ARG and token-injected URLs
+            local patched_dir
+            patched_dir=$(mktemp -d)
+            cp -a "$context_dir/." "$patched_dir/"
+            # Add ARG after the first FROM line (handles multi-stage builds)
+            sed -i '0,/^FROM /{/^FROM /a\ARG GITHUB_TOKEN
+}' "$patched_dir/Dockerfile"
+            # Replace sg-benchmarks URLs to include token
+            sed -i "s|https://github.com/sg-benchmarks/|https://x-access-token:\${GITHUB_TOKEN}@github.com/sg-benchmarks/|g" "$patched_dir/Dockerfile"
+            build_args=(--build-arg "GITHUB_TOKEN=$gh_token")
+            context_dir="$patched_dir"
+        fi
+    fi
 
-    if docker build --quiet -t "$image_tag" "$context_dir" > "$log_file" 2>&1; then
+    if docker build --quiet "${build_args[@]}" -t "$image_tag" "$context_dir" > "$log_file" 2>&1; then
         echo "OK   $image_tag ($(( SECONDS - start ))s)"
         local rc=0
     else
         echo "ERR  $image_tag ($(( SECONDS - start ))s) — see $log_file"
         local rc=1
     fi
 
+    # Cleanup patched dir
+    if [ -n "${patched_dir:-}" ] && [ -d "${patched_dir:-}" ]; then
+        rm -rf "$patched_dir"
+    fi
+
     # Cleanup sg_only temp dir
     if [ -n "$cleanup_dir" ] && [ -d "$cleanup_dir" ]; then
         rm -rf "$cleanup_dir"
diff --git a/scripts/prebuild_with_credentials.sh b/scripts/prebuild_with_credentials.sh
