1+ #! /usr/bin/env bash
2+ #
3+ # Genesis sync test on a local network.
4+ #
5+ # Start a local testnet, shut down non-validator nodes for a period, then restart them
6+ # and monitor their sync progress from genesis to head.
7+ SCRIPT_DIR=" $( cd -- " $( dirname -- " ${BASH_SOURCE[0]} " ) " & > /dev/null && pwd ) "
8+
9+ ENCLAVE_NAME=${1:- genesis-sync-testnet}
10+ CONFIG=${2:- $SCRIPT_DIR / genesis-sync-config-electra.yaml}
11+ FORK_TYPE=${3:- electra} # electra or fulu
12+ OFFLINE_DURATION_SECS=${4:- 120} # stopped duration of non validating nodes
13+
14+ # Test configuration
15+ # ------------------------------------------------------
16+ # Interval for polling the /lighthouse/syncing endpoint for sync status
17+ # Reduce the polling time so that some progress can be seen
18+ POLL_INTERVAL_SECS=0.5
19+ # Timeout for this test, if the nodes fail to sync, fail the test.
20+ TIMEOUT_MINS=5
21+ TIMEOUT_SECS=$(( TIMEOUT_MINS * 60 ))
22+ # ------------------------------------------------------
23+
24+ echo " Starting genesis sync test with:"
25+ echo " Fork: $FORK_TYPE "
26+ echo " Offline duration: ${OFFLINE_DURATION_SECS} s"
27+
28+ # Polls a node's sync status
29+ poll_node () {
30+ local node_type=$1
31+ local url=${node_urls[$node_type]}
32+
33+ response=$( curl -s " ${url} /lighthouse/syncing" 2> /dev/null)
34+
35+ if [ -z " $response " || [ " $response " = " null" ; then
36+ echo " ${node_type} status: No response or null response"
37+ return
38+ fi
39+
40+ # Print syncing status
41+ sync_state=$( echo " $response " | jq -r ' if (.data | type) == "object" then "object" else "string" end' 2> /dev/null)
42+
43+ if [ " $sync_state " = " object" ; then
44+ status=$( echo " $response " | jq -r ' .data | keys[0] // "Unknown"' )
45+ fields=$( echo " $response " | jq -r " .data.${status} | to_entries | map(\" \(.key): \(.value)\" ) | join(\" , \" )" )
46+ echo " ${node_type} status: ${status} , ${fields} "
47+ else
48+ status=$( echo " $response " | jq -r ' .data' 2> /dev/null)
49+ echo " ${node_type} status: ${status:- Unknown} "
50+
51+ # The test is complete when the node is synced
52+ if [ " $status " = " Synced" ; then
53+ mark_node_complete " $node_type "
54+ fi
55+ fi
56+ }
57+
58+ # Marks a node as complete and record time
59+ mark_node_complete () {
60+ local node_type=$1
61+ if [ " ${node_completed[$node_type]} " = false ]; then
62+ node_completed[$node_type ]=true
63+ node_complete_time[$node_type ]=$( date +%s)
64+ echo " ${node_type} completed sync in $(( node_complete_time[$node_type ] - sync_start_time)) "
65+ fi
66+ }
67+
68+ exit_and_dump_logs () {
69+ local exit_code=$1
70+ echo " Shutting down..."
71+ $SCRIPT_DIR /../local_testnet/stop_local_testnet.sh $ENCLAVE_NAME
72+ echo " Test completed with exit code $exit_code ."
73+ exit $exit_code
74+ }
75+
76+ # Start the nodes
77+ $SCRIPT_DIR /../local_testnet/start_local_testnet.sh -e $ENCLAVE_NAME -b false -n $CONFIG
78+ if [ $? -ne 0 ]; then
79+ echo " Failed to start local testnet"
80+ exit_and_dump_logs 1
81+ fi
82+
83+ # Wait for 10s before stopping non-validating nodes
84+ sleep 10
85+
86+ # These are non validating nodes
87+ supernode=" cl-3-lighthouse-geth"
88+ fullnode=" cl-4-lighthouse-geth"
89+
90+ # Stop the non-validator nodes
91+ kurtosis service stop $ENCLAVE_NAME $supernode
92+ kurtosis service stop $ENCLAVE_NAME $fullnode
93+
94+ echo " Non-validator nodes stopped. Waiting ${OFFLINE_DURATION_SECS} seconds..."
95+
96+ # Display the time every 10s when the nodes are stopped
97+ remaining_time=$OFFLINE_DURATION_SECS
98+ while [ $remaining_time -gt 0 ]; do
99+ sleep 10
100+ remaining_time=$(( remaining_time - 10 ))
101+ echo " Nodes are stopped for $(( OFFLINE_DURATION_SECS - remaining_time)) ${remaining_time} s remains..."
102+ done
103+
104+ echo " Resuming non-validator nodes..."
105+
106+ # Resume the non validating nodes
107+ kurtosis service start $ENCLAVE_NAME $supernode
108+ kurtosis service start $ENCLAVE_NAME $fullnode
109+
110+ # The time at which syncing starts after the node was stopped
111+ sync_start_time=$( date +%s)
112+
113+ # Get beacon API URLs for non validating nodes for query
114+ supernode_url=$( kurtosis port print $ENCLAVE_NAME $supernode http)
115+ fullnode_url=$( kurtosis port print $ENCLAVE_NAME $fullnode http)
116+
117+ # Initialize statuses
118+ declare -A node_completed
119+ declare -A node_complete_time
120+ declare -A node_urls
121+
122+ node_urls[" supernode" " $supernode_url "
123+ node_urls[" fullnode" " $fullnode_url "
124+ node_completed[" supernode"
125+ node_completed[" fullnode"
126+
127+ echo " Polling sync status until nodes are synced or timeout of ${TIMEOUT_MINS} mins"
128+
129+ while [ " ${node_completed[supernode]} " = false ] || [ " ${node_completed[fullnode]} " = false ]; do
130+ current_time=$( date +%s)
131+ elapsed=$(( current_time - sync_start_time))
132+
133+ if [ " $elapsed " -ge " $TIMEOUT_SECS " ; then
134+ echo " ERROR: Nodes timed out syncing after ${TIMEOUT_MINS} minutes. Exiting."
135+ exit_and_dump_logs 1
136+ fi
137+
138+ # Poll each node that hasn't completed yet
139+ for node in " supernode" " fullnode" ; do
140+ if [ " ${node_completed[$node]} " = false ]; then
141+ poll_node " $node "
142+ fi
143+ done
144+
145+ sleep $POLL_INTERVAL_SECS
146+ done
147+
148+ echo " Genesis sync test complete! Both supernode and fullnode have synced successfully."
149+ echo " Supernode time: $(( node_complete_time[supernode] - sync_start_time)) "
150+ echo " Fullnode time: $(( node_complete_time[fullnode] - sync_start_time)) "
151+ exit_and_dump_logs 0
0 commit comments