Merge remote-tracking branch 'origin/unstable' into remove-reprocess-channel

Author: michaelsproul

.github/workflows/local-testnet.yml

@@ -20,7 +20,7 @@ jobs:
 
       - name: Build Docker image
         run: |
-          docker build --build-arg FEATURES=portable -t lighthouse:local .
+          docker build --build-arg FEATURES=portable,spec-minimal -t lighthouse:local .
           docker save lighthouse:local -o lighthouse-docker.tar
 
       - name: Upload Docker image artifact
@@ -52,30 +52,52 @@ jobs:
       - name: Load Docker image
         run: docker load -i lighthouse-docker.tar
 
-      - name: Start local testnet
-        run: ./start_local_testnet.sh -e local -c -b false && sleep 60
+      - name: Start local testnet with Assertoor
+        run: ./start_local_testnet.sh -e local-assertoor -c -a -b false && sleep 60
         working-directory: scripts/local_testnet
 
-      - name: Stop local testnet and dump logs
-        run: ./stop_local_testnet.sh local
-        working-directory: scripts/local_testnet
-
-      - name: Start local testnet with blinded block production
-        run: ./start_local_testnet.sh -e local-blinded -c -p -b false && sleep 60
-        working-directory: scripts/local_testnet
+      - name: Await Assertoor test result
+        id: assertoor_test_result
+        uses: ethpandaops/assertoor-github-action@v1
+        with:
+          kurtosis_enclave_name: local-assertoor
 
       - name: Stop local testnet and dump logs
-        run: ./stop_local_testnet.sh local-blinded
+        run: ./stop_local_testnet.sh local-assertoor
         working-directory: scripts/local_testnet
 
       - name: Upload logs artifact
+        if: always()
         uses: actions/upload-artifact@v4
         with:
           name: logs-local-testnet
           path: |
             scripts/local_testnet/logs
           retention-days: 3
 
+      - name: Return Assertoor test result
+        shell: bash
+        run: |
+          test_result="${{ steps.assertoor_test_result.outputs.result }}"
+          test_status=$(
+          cat <<"EOF"
+          ${{ steps.assertoor_test_result.outputs.test_overview }}
+          EOF
+          )
+          failed_test_status=$(
+          cat <<"EOF"
+          ${{ steps.assertoor_test_result.outputs.failed_test_details }}
+          EOF
+          )
+          
+          echo "Test Result:  $test_result"
+          echo "$test_status"
+          if ! [ "$test_result" == "success" ]; then
+            echo "Failed Test Task Status:"
+            echo "$failed_test_status"
+            exit 1
+          fi
+
   doppelganger-protection-success-test:
     needs: dockerfile-ubuntu
     runs-on: ubuntu-22.04
@@ -104,6 +126,7 @@ jobs:
         working-directory: scripts/tests
 
       - name: Upload logs artifact
+        if: always()
         uses: actions/upload-artifact@v4
         with:
           name: logs-doppelganger-protection-success
@@ -139,13 +162,99 @@ jobs:
         working-directory: scripts/tests
 
       - name: Upload logs artifact
+        if: always()
         uses: actions/upload-artifact@v4
         with:
           name: logs-doppelganger-protection-failure
           path: |
             scripts/local_testnet/logs
           retention-days: 3
 
+  # Tests checkpoint syncing to a live network (current fork) and a running devnet (usually next scheduled fork)
+  checkpoint-sync-test:
+    name: checkpoint-sync-test-${{ matrix.network }}
+    runs-on: ubuntu-latest
+    needs: dockerfile-ubuntu
+    if: contains(github.event.pull_request.labels.*.name, 'syncing')
+    continue-on-error: true
+    strategy:
+      matrix:
+        network: [sepolia, devnet]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Kurtosis
+        run: |
+          echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
+          sudo apt update
+          sudo apt install -y kurtosis-cli
+          kurtosis analytics disable
+
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: lighthouse-docker
+          path: .
+
+      - name: Load Docker image
+        run: docker load -i lighthouse-docker.tar
+
+      - name: Run the checkpoint sync test script
+        run: |
+          ./checkpoint-sync.sh "sync-${{ matrix.network }}" "checkpoint-sync-config-${{ matrix.network }}.yaml"
+        working-directory: scripts/tests
+
+      - name: Upload logs artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: logs-checkpoint-sync-${{ matrix.network }}
+          path: |
+            scripts/local_testnet/logs
+          retention-days: 3
+
+  # Test syncing from genesis on a local testnet. Aims to cover forward syncing both short and long distances.
+  genesis-sync-test:
+    name: genesis-sync-test-${{ matrix.fork }}-${{ matrix.offline_secs }}s
+    runs-on: ubuntu-latest
+    needs: dockerfile-ubuntu
+    if: contains(github.event.pull_request.labels.*.name, 'syncing')
+    strategy:
+      matrix:
+        fork: [electra, fulu]
+        offline_secs: [120, 300]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Kurtosis
+        run: |
+          echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
+          sudo apt update
+          sudo apt install -y kurtosis-cli
+          kurtosis analytics disable
+
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: lighthouse-docker
+          path: .
+
+      - name: Load Docker image
+        run: docker load -i lighthouse-docker.tar
+
+      - name: Run the genesis sync test script
+        run: |
+          ./genesis-sync.sh "sync-${{ matrix.fork }}-${{ matrix.offline_secs }}s" "genesis-sync-config-${{ matrix.fork }}.yaml" "${{ matrix.fork }}" "${{ matrix.offline_secs }}"
+        working-directory: scripts/tests
+
+      - name: Upload logs artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: logs-genesis-sync-${{ matrix.fork }}-${{ matrix.offline_secs }}s
+          path: |
+            scripts/local_testnet/logs
+          retention-days: 3
 
   # This job succeeds ONLY IF all others succeed. It is used by the merge queue to determine whether
   # a PR is safe to merge. New jobs should be added here.
@@ -161,4 +270,6 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Check that success job is dependent on all others
-        run: ./scripts/ci/check-success-job.sh ./.github/workflows/local-testnet.yml local-testnet-success
+        run: |
+          exclude_jobs='checkpoint-sync-test|genesis-sync-test'
+          ./scripts/ci/check-success-job.sh ./.github/workflows/local-testnet.yml local-testnet-success "$exclude_jobs" 

.github/workflows/test-suite.yml

@@ -196,7 +196,7 @@ jobs:
     - name: Run network tests for all known forks
       run: make test-network
       env:
-        TEST_FEATURES: portable,ci_logger
+        TEST_FEATURES: portable
         CI_LOGGER_DIR: ${{ runner.temp }}/network_test_logs
     - name: Upload logs
       uses: actions/upload-artifact@v4