Sync Supported Chains #3113
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sync Supported Chains | |
| on: | |
| workflow_run: | |
| workflows: ["Publish Docker image"] | |
| branches: [master, release/*] | |
| types: | |
| - completed | |
| workflow_dispatch: | |
| inputs: | |
| nethermind_image: | |
| description: "Docker image to be used by action" | |
| default: "" | |
| required: false | |
| env: | |
| DOTNET_SYSTEM_CONSOLE_ALLOW_ANSI_COLOR_REDIRECTION: "1" | |
| TERM: xterm | |
| jobs: | |
| setup-matrix: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| path: nethermind | |
| - name: Set Matrix | |
| id: set-matrix | |
| run: echo "matrix=$(jq -c . nethermind/scripts/workflow_config/sync_testnets_matrix.json)" >> $GITHUB_OUTPUT | |
| create_a_runner: | |
| needs: [setup-matrix] | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 5 | |
| matrix: | |
| config: ${{fromJson(needs.setup-matrix.outputs.matrix)}} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| runner_label: ${{ steps.run-linode-action.outputs.runner_label }} | |
| machine_id: ${{ steps.run-linode-action.outputs.machine_id }} | |
| steps: | |
| - name: Install sshpass | |
| run: sudo apt-get update && sudo apt-get install -y sshpass | |
| - name: Create a Runner | |
| id: run-linode-action | |
| uses: kamilchodola/linode-github-runner/.github/actions/linode-machine-manager@main | |
| with: | |
| linode_token: ${{ secrets.LINODE_TOKEN }} | |
| github_token: "${{ secrets.REPOSITORY_DISPATCH_TOKEN }}" | |
| action: "create" | |
| runner_label: t-${{ github.run_id }}-${{ matrix.config.network }} | |
| root_password: ${{ secrets.LINODE_ROOT_PASSWORD }} | |
| machine_type: "${{ matrix.config.agent }}" | |
| image: "linode/ubuntu24.04" | |
| tags: "core, self-hosted, dynamic" | |
| organization: "NethermindEth" | |
| repo_name: "nethermind" | |
| blocked_ports: "8545,8546,8551,8552" | |
| sync-chain: | |
| needs: [setup-matrix, create_a_runner] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: ${{fromJson(needs.setup-matrix.outputs.matrix)}} | |
| name: "Run sync of ${{ matrix.config.network }} chain" | |
| runs-on: t-${{ github.run_id }}-${{ matrix.config.network }} | |
| timeout-minutes: ${{ matrix.config.timeout }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| clean: true | |
| - name: Installing requirements | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y make build-essential jq screen lshw dmidecode fio | |
| - name: Setup Go environment | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: 'stable' | |
| check-latest: true | |
| cache: true | |
| - name: Install Sedge environment | |
| run: | | |
| echo "Downloading sedge sources..." | |
| git clone https://github.com/NethermindEth/sedge.git sedge --branch core --single-branch | |
| echo "Sources downloaded." | |
| cd sedge | |
| echo "Building sedge..." | |
| make compile | |
| - name: Run Sedge | |
| working-directory: sedge | |
| run: | | |
| docker_image="" | |
| network="${{ matrix.config.network }}" | |
| if [ -z "${{ inputs.nethermind_image }}" ]; then | |
| REF_NAME=${{ github.ref }} | |
| if [[ $REF_NAME == refs/heads/release/* ]]; then | |
| CLEAN_REF=$(echo "${REF_NAME/refs\/heads\//}" | sed 's/[^a-zA-Z0-9._-]/-/g') | |
| docker_image="nethermindeth/nethermind:$CLEAN_REF" | |
| else | |
| docker_image="nethermindeth/nethermind:master" | |
| fi | |
| else | |
| docker_image="${{ inputs.nethermind_image }}" | |
| fi | |
| echo 'Generating sedge docker...' | |
| ./build/sedge deps install | |
| if [[ "$network" == base-* || "$network" == op-* ]]; then | |
| if [[ "$network" == *mainnet* ]]; then | |
| CONSENSUS_URL="${{ secrets.MAINNET_CONSENSUS_URL }}" | |
| EXECUTION_URL="${{ secrets.MAINNET_EXECUTION_URL }}" | |
| elif [[ "$network" == *sepolia* ]]; then | |
| CONSENSUS_URL="${{ secrets.SEPOLIA_CONSENSUS_URL }}" | |
| EXECUTION_URL="${{ secrets.SEPOLIA_EXECUTION_URL }}" | |
| else | |
| echo "Unknown network" | |
| exit 1 | |
| fi | |
| if [[ "$network" == base-* ]]; then | |
| extra_param="--base" | |
| fi | |
| stripped_network="${network#base-}" | |
| stripped_network="${stripped_network#op-}" | |
| ./build/sedge generate \ | |
| --logging none \ | |
| -p $GITHUB_WORKSPACE/sedge \ | |
| op-full-node \ | |
| --op-execution opnethermind:$docker_image \ | |
| --op-image op-node:us-docker.pkg.dev/oplabs-tools-artifacts/images/op-node:latest \ | |
| --map-all \ | |
| --network $stripped_network \ | |
| --consensus-url $CONSENSUS_URL \ | |
| --execution-api-url $EXECUTION_URL \ | |
| $extra_param | |
| else | |
| ./build/sedge generate \ | |
| --logging none \ | |
| -p $GITHUB_WORKSPACE/sedge \ | |
| full-node \ | |
| -c ${{ matrix.config.cl }}:${{ matrix.config.cl_image }} \ | |
| -e nethermind:$docker_image \ | |
| --map-all \ | |
| --no-mev-boost \ | |
| --no-validator \ | |
| --network ${{ matrix.config.network }} \ | |
| --el-extra-flag Sync.NonValidatorNode=true \ | |
| --el-extra-flag Sync.DownloadBodiesInFastSync=false \ | |
| --el-extra-flag Sync.DownloadReceiptsInFastSync=false \ | |
| --el-extra-flag JsonRpc.EnabledModules=[Eth,Subscribe,Trace,TxPool,Web3,Personal,Proof,Net,Parity,Health,Rpc,Debug] \ | |
| --el-extra-flag Sync.SnapSync=true \ | |
| --checkpoint-sync-url=${{ matrix.config.checkpoint-sync-url }} | |
| fi | |
| echo 'Running sedge...' | |
| ./build/sedge run -p $GITHUB_WORKSPACE/sedge | |
| - name: Wait for ${{ matrix.config.network }} to sync | |
| id: wait | |
| run: | | |
| declare -A bad_logs | |
| declare -A good_logs | |
| declare -A required_count | |
| bad_logs["Corrupt"]=1 | |
| bad_logs["Exception"]=1 | |
| good_logs["Processed"]=0 | |
| required_count["Processed"]=20 | |
| network="${{ matrix.config.network }}" | |
| if [[ "$network" != "joc-mainnet" && "$network" != "joc-testnet" ]]; then | |
| good_logs["Synced Chain Head"]=0 | |
| required_count["Synced Chain Head"]=20 | |
| fi | |
| counter=0 | |
| found_bad_log=false | |
| if [[ "$network" == base-* || "$network" == op-* ]]; then | |
| container_name="sedge-execution-op-l2-client" | |
| else | |
| container_name="sedge-execution-client" | |
| fi | |
| docker logs -f "$container_name" | while read -r line; do | |
| echo "$line" | |
| if [[ "$line" == *"All done"* ]]; then | |
| echo "Unexpected termination detected: $line" | |
| exit 1 | |
| fi | |
| if [ "$found_bad_log" = true ]; then | |
| counter=$((counter + 1)) | |
| if [ $counter -ge 100 ]; then | |
| echo "Exiting after capturing extra logs due to error." | |
| exit 1 | |
| else | |
| continue | |
| fi | |
| fi | |
| for bad_log in "${!bad_logs[@]}"; do | |
| if [[ "$line" == *"$bad_log"* ]]; then | |
| echo "Error: $bad_log found in Docker logs." | |
| found_bad_log=true | |
| break | |
| fi | |
| done | |
| for good_log in "${!good_logs[@]}"; do | |
| if [[ "$line" == *"$good_log"* ]]; then | |
| good_logs["$good_log"]=$((good_logs["$good_log"]+1)) | |
| fi | |
| done | |
| # Check if all good logs have reached the required count | |
| all_reached_required_count=true | |
| for good_log in "${!good_logs[@]}"; do | |
| if [[ ${good_logs[$good_log]} -lt ${required_count[$good_log]} ]]; then | |
| all_reached_required_count=false | |
| break | |
| fi | |
| done | |
| if $all_reached_required_count; then | |
| echo "All required logs found." | |
| exit 0 | |
| fi | |
| done | |
| - name: Get Consensus Logs | |
| if: always() && matrix.config.network != 'joc-mainnet' && matrix.config.network != 'joc-testnet' | |
| run: | | |
| network="${{ matrix.config.network }}" | |
| if [[ "$network" == base-* || "$network" == op-* ]]; then | |
| docker logs sedge-consensus-op-l2-client | |
| else | |
| docker logs sedge-consensus-client | |
| fi | |
| - name: Check size of DB | |
| run: | | |
| du -h $GITHUB_WORKSPACE/sedge | |
| - name: Destroy VM | |
| if: always() | |
| id: run-linode-action | |
| uses: kamilchodola/linode-github-runner/.github/actions/linode-machine-manager@main | |
| with: | |
| linode_token: ${{ secrets.LINODE_TOKEN }} | |
| github_token: "${{ secrets.REPOSITORY_DISPATCH_TOKEN }}" | |
| action: "destroy-machine" | |
| runner_label: t-${{ github.run_id }}-${{ matrix.config.network }} | |
| search_phrase: t-${{ github.run_id }}-${{ matrix.config.network }} | |
| root_password: ${{ secrets.LINODE_ROOT_PASSWORD }} | |
| organization: "NethermindEth" | |
| repo_name: "nethermind" | |
| destroy_runner: | |
| needs: [setup-matrix, create_a_runner, sync-chain] | |
| if: always() | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: ${{fromJson(needs.setup-matrix.outputs.matrix)}} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Destroy VM (make sure is removed if by any unexpected reason it did not removed it on ) | |
| continue-on-error: true | |
| uses: kamilchodola/linode-github-runner/.github/actions/linode-machine-manager@main | |
| with: | |
| linode_token: ${{ secrets.LINODE_TOKEN }} | |
| github_token: "${{ secrets.REPOSITORY_DISPATCH_TOKEN }}" | |
| action: "destroy-machine" | |
| runner_label: t-${{ github.run_id }}-${{ matrix.config.network }} | |
| search_phrase: t-${{ github.run_id }}-${{ matrix.config.network }} | |
| root_password: ${{ secrets.LINODE_ROOT_PASSWORD }} | |
| organization: "NethermindEth" | |
| repo_name: "nethermind" | |
| - name: Destroy Runner | |
| uses: kamilchodola/linode-github-runner/.github/actions/linode-machine-manager@main | |
| with: | |
| linode_token: ${{ secrets.LINODE_TOKEN }} | |
| github_token: "${{ secrets.REPOSITORY_DISPATCH_TOKEN }}" | |
| action: "destroy-runner" | |
| runner_label: t-${{ github.run_id }}-${{ matrix.config.network }} | |
| search_phrase: t-${{ github.run_id }}-${{ matrix.config.network }} | |
| root_password: ${{ secrets.LINODE_ROOT_PASSWORD }} | |
| organization: "NethermindEth" | |
| repo_name: "nethermind" | |
| trigger_tests: | |
| needs: [setup-matrix, create_a_runner, sync-chain, destroy_runner] | |
| if: success() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Authenticate App | |
| id: gh-app | |
| uses: actions/create-github-app-token@v1 | |
| with: | |
| app-id: ${{ vars.APP_ID }} | |
| private-key: ${{ secrets.APP_PRIVATE_KEY }} | |
| repositories: "nethermind,post-merge-smoke-tests" | |
| - name: Trigger notification action on test repo | |
| if: github.event.workflow_run.head_branch == 'master' || startsWith(github.event.workflow_run.head_branch, 'release/') | |
| continue-on-error: true | |
| uses: benc-uk/workflow-dispatch@v1 | |
| with: | |
| workflow: receive-push-notification.yml | |
| repo: NethermindEth/post-merge-smoke-tests | |
| ref: "main" | |
| token: "${{ steps.gh-app.outputs.token }}" | |
| inputs: '{ | |
| "nethermind_branch": "${{ github.ref}}" | |
| }' |