[ci] Support running re-execution benchmark with arbitrary version of Firewood

.github/actions/c-chain-reexecution-benchmark/action.yml

@@ -63,6 +63,12 @@ inputs:
   push-post-state:
     description: 'S3 destination to copy the current-state directory after completing re-execution. If empty, this will be skipped.'
     default: ''
+  libevm-ref:
+    description: 'libevm commit/branch/tag (optional). If provided, updates libevm dependency before benchmark.'
+    default: ''
+  firewood-ref:
+    description: 'Firewood commit/branch/tag (optional). If provided, builds custom firewood and skips benchmark comparison (Firewood tracks separately).'
+    default: ''
   # The following inputs need never be provided by the caller. They
   # default to context values that the action's steps are unable to
   # access directly.
@@ -154,11 +160,15 @@ runs:
         if [[ -n "${{ inputs.task }}" ]]; then
           # Task-based approach
           ./scripts/run_task.sh ${{ inputs.task }} \
+            LIBEVM_REF="${{ inputs.libevm-ref }}" \
+            FIREWOOD_REF="${{ inputs.firewood-ref }}" \
             BENCHMARK_OUTPUT_FILE="${{ env.BENCHMARK_OUTPUT_FILE }}" \
             EXECUTION_DATA_DIR="${{ env.EXECUTION_DATA_DIR }}"
         else
           # Granular approach
           ./scripts/run_task.sh reexecute-cchain-range-with-copied-data \
+            LIBEVM_REF="${{ inputs.libevm-ref }}" \
+            FIREWOOD_REF="${{ inputs.firewood-ref }}" \
             CONFIG=${{ inputs.config }} \
             EXECUTION_DATA_DIR=${{ env.EXECUTION_DATA_DIR }} \
             BLOCK_DIR_SRC=${{ inputs.block-dir-src }} \
@@ -180,7 +190,13 @@ runs:
         GH_RUN_NUMBER: ${{ inputs.run_number }}
         GH_RUN_ATTEMPT: ${{ inputs.run_attempt }}
         GH_JOB_ID: ${{ inputs.job }}
+        # Required for Nix builds in CI self-hosted runners (e.g., building Firewood via polyrepo)
+        TMPDIR: /tmp
+        NIX_BUILD_TOP: /tmp
+    # Skip when using custom firewood - Firewood downloads the artifact and
+    # publishes to their own gh-pages for tracking
     - name: Compare Benchmark Results
+      if: inputs.firewood-ref == ''
       uses: benchmark-action/github-action-benchmark@v1
       with:
         tool: 'go'

.github/workflows/c-chain-reexecution-benchmark-container.yml

@@ -25,12 +25,15 @@ on:
       runner:
         description: 'Runner to execute the benchmark. Input to the runs-on field of the job.'
         required: true
-      push-post-state:
-        description: 'S3 location to push post-execution state directory. Skips this step if left unpopulated.'
-        default: ''
       timeout-minutes:
         description: 'Timeout in minutes for the job.'
         default: 30
+      firewood-ref:
+        description: 'Firewood commit/branch/tag (optional). Builds custom firewood and skips benchmark comparison (Firewood tracks separately).'
+        default: ''
+      libevm-ref:
+        description: 'libevm commit/branch/tag (optional). If provided, updates libevm dependency.'
+        default: ''
 
   # Disabled because scheduled trigger is empty. To enable, uncomment and add at least one vector to the schedule
   # entry in the corresponding JSON file.
@@ -82,8 +85,6 @@ jobs:
       id-token: write
       contents: write
     runs-on: ${{ matrix.runner }}
-    container:
-      image: ghcr.io/actions/actions-runner:2.325.0
     steps:
       - uses: actions/checkout@v4
       - name: Install ARC Dependencies
@@ -103,13 +104,25 @@ jobs:
           end-block: ${{ matrix.end-block }}
           block-dir-src: ${{ matrix.block-dir-src }}
           current-state-dir-src: ${{ matrix.current-state-dir-src }}
+          libevm-ref: ${{ inputs.libevm-ref }}
+          firewood-ref: ${{ inputs.firewood-ref }}
           prometheus-url: ${{ secrets.PROMETHEUS_URL || '' }}
           prometheus-push-url: ${{ secrets.PROMETHEUS_PUSH_URL || '' }}
           prometheus-username: ${{ secrets.PROMETHEUS_USERNAME || '' }}
           prometheus-password: ${{ secrets.PROMETHEUS_PASSWORD || '' }}
           push-github-action-benchmark: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.repository == 'ava-labs/avalanchego' && github.ref_name == 'master') }}
-          aws-role: ${{ github.event.inputs.push-post-state != '' && secrets.AWS_S3_RW_ROLE || secrets.AWS_S3_READ_ONLY_ROLE }}
+          aws-role: ${{ matrix.push-post-state != '' && secrets.AWS_S3_RW_ROLE || secrets.AWS_S3_READ_ONLY_ROLE }}
           aws-region: 'us-east-2'
           github-token: ${{ secrets.GITHUB_TOKEN }}
-          push-post-state: ${{ github.event.inputs.push-post-state }}
+          push-post-state: ${{ matrix.push-post-state || '' }}
           runner_name: ${{ matrix.runner }}
+      - name: Upload benchmark results
+        # Only upload when using custom firewood - Firewood downloads the artifact
+        # and publishes to their own gh-pages for tracking
+        if: inputs.firewood-ref != ''
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-output
+          path: ${{ github.workspace }}/benchmark-output.txt
+          retention-days: 30
+          if-no-files-found: error

.github/workflows/c-chain-reexecution-benchmark-gh-native.yml

@@ -25,12 +25,15 @@ on:
       runner:
         description: 'Runner to execute the benchmark. Input to the runs-on field of the job.'
         required: true
-      push-post-state:
-        description: 'S3 location to push post-execution state directory. Skips this step if left unpopulated.'
-        default: ''
       timeout-minutes:
         description: 'Timeout in minutes for the job.'
         default: 30
+      firewood-ref:
+        description: 'Firewood commit/branch/tag (optional). Builds custom firewood and skips benchmark comparison (Firewood tracks separately).'
+        default: ''
+      libevm-ref:
+        description: 'libevm commit/branch/tag (optional). If provided, updates libevm dependency.'
+        default: ''
 
   # Disabled because scheduled trigger is empty. To enable, uncomment and add at least one vector to the schedule
   # entry in the corresponding JSON file.
@@ -93,13 +96,25 @@ jobs:
           end-block: ${{ matrix.end-block }}
           block-dir-src: ${{ matrix.block-dir-src }}
           current-state-dir-src: ${{ matrix.current-state-dir-src }}
+          libevm-ref: ${{ inputs.libevm-ref }}
+          firewood-ref: ${{ inputs.firewood-ref }}
           prometheus-url: ${{ secrets.PROMETHEUS_URL || '' }}
           prometheus-push-url: ${{ secrets.PROMETHEUS_PUSH_URL || '' }}
           prometheus-username: ${{ secrets.PROMETHEUS_USERNAME || '' }}
           prometheus-password: ${{ secrets.PROMETHEUS_PASSWORD || '' }}
           push-github-action-benchmark: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.repository == 'ava-labs/avalanchego' && github.ref_name == 'master') }}
-          aws-role: ${{ github.event.inputs.push-post-state != '' && secrets.AWS_S3_RW_ROLE || secrets.AWS_S3_READ_ONLY_ROLE }}
+          aws-role: ${{ matrix.push-post-state != '' && secrets.AWS_S3_RW_ROLE || secrets.AWS_S3_READ_ONLY_ROLE }}
           aws-region: 'us-east-2'
           github-token: ${{ secrets.GITHUB_TOKEN }}
-          push-post-state: ${{ github.event.inputs.push-post-state }}
+          push-post-state: ${{ matrix.push-post-state || '' }}
           runner_name: ${{ matrix.runner }}
+      - name: Upload benchmark results
+        # Only upload when using custom firewood - Firewood downloads the artifact
+        # and publishes to their own gh-pages for tracking
+        if: inputs.firewood-ref != ''
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-output
+          path: ${{ github.workspace }}/benchmark-output.txt
+          retention-days: 30
+          if-no-files-found: error

.gitignore

@@ -66,5 +66,9 @@ vendor
 # debug files
 __debug_*
 
+# polyrepo
+polyrepo.log
+firewood
+
 # solc downloads from setup-solc GitHub Action
 setup-solc_downloads/

Taskfile.yml

@@ -233,6 +233,19 @@ tasks:
 
   reexecute-cchain-range-with-copied-data:
     desc: Combines import-cchain-reexecute-range and reexecute-cchain-range
+    summary: |
+      Optionally accepts LIBEVM_REF and FIREWOOD_REF to test with custom versions.
+      If not provided, uses versions from go.mod.
+
+      Examples:
+        # Use go.mod versions
+        task c-chain-reexecution-firewood-101-250k
+
+        # Use custom firewood
+        task c-chain-reexecution-firewood-101-250k FIREWOOD_REF=abc123
+
+        # Use custom both
+        task c-chain-reexecution-firewood-101-250k FIREWOOD_REF=abc123 LIBEVM_REF=v1.2.3
     vars:
       TASK_NAME: '{{.TASK_NAME}}'
       BLOCK_DIR_SRC: '{{.BLOCK_DIR_SRC}}'
@@ -243,6 +256,17 @@ tasks:
       TIMESTAMP: '{{.TIMESTAMP | default (now | date "20060102-150405")}}'
       EXECUTION_DATA_DIR: '{{.EXECUTION_DATA_DIR | default (printf "/tmp/%s-%s" .TASK_NAME .TIMESTAMP)}}'
     cmds:
+      - cmd: |
+          if [[ -n "{{.LIBEVM_REF}}" ]]; then
+            echo "Updating libevm to: {{.LIBEVM_REF}}"
+            go get github.com/ava-labs/libevm@{{.LIBEVM_REF}}
+            cd graft/coreth && go get github.com/ava-labs/libevm@{{.LIBEVM_REF}}
+          fi
+      - cmd: |
+          if [[ -n "{{.FIREWOOD_REF}}" && "{{.CONFIG}}" == "firewood" ]]; then
+            echo "Setting up firewood at: {{.FIREWOOD_REF}}"
+            ./scripts/run_polyrepo.sh sync firewood@{{.FIREWOOD_REF}} --force
+          fi
       - task: import-cchain-reexecute-range
         vars:
           BLOCK_DIR_SRC: '{{.S3_BOOTSTRAP_BUCKET}}/{{.BLOCK_DIR_SRC}}/**'

scripts/run_polyrepo.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+POLYREPO_REVISION=6239973c9b
+echo "Running polyrepo@${POLYREPO_REVISION} via go run..."
+go run github.com/ava-labs/avalanchego/tests/fixture/polyrepo@"${POLYREPO_REVISION}" "${@}"

tests/reexecute/c/README.md

@@ -247,6 +247,39 @@ To execute a benchmark with any of these options, you must use a compatible `CUR
 
 The `CONFIG` parameter currently only supports pre-defined configs and not passing a full JSON blob in, so that we can define corresponding names for each config option. The config name is attached as a label to the exported metrics and included in the name of the sub-benchmark (used by GitHub Action Benchmark to separate historical results with different configs).
 
+## Testing with Custom Dependency Versions
+
+The benchmarks support testing with custom versions of `libevm` and `firewood` dependencies. This is useful for:
+- Testing unreleased versions before merging
+- Benchmarking performance changes in dependencies
+- Reproducing issues with specific dependency versions
+
+### Local Usage
+
+**Prerequisite**: You must be in a nix shell (`nix develop`) to run benchmark tasks (`c-chain-reexecution-*`) .
+
+Pass `LIBEVM_REF` and/or `FIREWOOD_REF` to any benchmark task:
+
+```bash
+# Test with custom firewood version
+./scripts/run_task.sh c-chain-reexecution-firewood-101-250k FIREWOOD_REF=abc123def
+
+# Test with custom libevm version  
+./scripts/run_task.sh c-chain-reexecution-firewood-101-250k LIBEVM_REF=v1.2.3
+
+# Test with both custom versions
+./scripts/run_task.sh c-chain-reexecution-firewood-101-250k \
+  FIREWOOD_REF=abc123def \
+  LIBEVM_REF=v1.2.3
+```
+
+**Note**: `FIREWOOD_REF` only takes effect when `CONFIG=firewood`. For hashdb benchmarks, it will be ignored.
+
+### How It Works
+
+- **`LIBEVM_REF`**: Runs `go get github.com/ava-labs/libevm@<ref>` to update the dependency
+- **`FIREWOOD_REF`**: Uses [polyrepo](../../../scripts/run_polyrepo.sh) to clone, build (via Nix), and configure firewood at the specified ref
+
 ## Metrics
 
 The C-Chain benchmarks export VM metrics to the same Grafana instance as AvalancheGo CI: https://grafana-poc.avax-dev.network/.
@@ -320,6 +353,25 @@ gh workflow run "C-Chain Re-Execution Benchmark GH Native" \
   -f timeout-minutes=60
 ```
 
+### Using Custom Dependency Versions
+
+```bash
+# Test with custom firewood commit
+gh workflow run "C-Chain Re-Execution Benchmark GH Native" \
+  -f task=c-chain-reexecution-firewood-101-250k \
+  -f firewood-ref=abc123def \
+  -f runner=blacksmith-4vcpu-ubuntu-2404 \
+  -f timeout-minutes=60
+
+# Test with custom libevm and firewood
+gh workflow run "C-Chain Re-Execution Benchmark GH Native" \
+  -f task=c-chain-reexecution-firewood-101-250k \
+  -f libevm-ref=v1.2.3 \
+  -f firewood-ref=abc123def \
+  -f runner=blacksmith-4vcpu-ubuntu-2404 \
+  -f timeout-minutes=60
+```
+
 ### Using Custom Parameters
 
 ```bash