Restart test without supermajority (solana-labs#17808)

* Restart test without supermajority

* Change check to look at stake rather than transaction confirmation

* Add stake info while waiting for consensus

* replace jq with grep (buildkite doesn't have jq installed)

system-test/automation_utils.sh

@@ -96,6 +96,20 @@ function get_bootstrap_validator_ip_address {
   echo "${validatorIpList[0]}"
 }
 
+function get_active_stake {
+  source "${REPO_ROOT}"/net/common.sh
+  loadConfigFile
+  ssh "${sshOptions[@]}" "${validatorIpList[0]}" \
+    '$HOME/.cargo/bin/solana --url http://127.0.0.1:8899 validators --output=json | grep -o "totalActiveStake\": [0-9]*" | cut -d: -f2'
+}
+
+function get_current_stake {
+  source "${REPO_ROOT}"/net/common.sh
+  loadConfigFile
+  ssh "${sshOptions[@]}" "${validatorIpList[0]}" \
+    '$HOME/.cargo/bin/solana --url http://127.0.0.1:8899 validators --output=json | grep -o "totalCurrentStake\": [0-9]*" | cut -d: -f2'
+}
+
 function collect_performance_statistics {
   execution_step "Collect performance statistics about run"
   declare q_mean_tps='

system-test/restart-testcases/restart_gce.yml

@@ -9,9 +9,11 @@ steps:
       VALIDATOR_NODE_MACHINE_TYPE: "--machine-type n1-standard-16"
       NUMBER_OF_CLIENT_NODES: 1
       ADDITIONAL_FLAGS: "--dedicated"
-      BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 66
       SKIP_PERF_RESULTS: "true"
+      EXTRA_PRIMORDIAL_STAKES: 4
+      WAIT_FOR_EQUAL_STAKE: "true"
       TEST_TYPE: "script"
+      CONSENSUS_TIMEOUT: 60
       CUSTOM_SCRIPT: "system-test/restart-testcases/restart_test_automation.sh"
     agents:
       - "queue=gce-deploy"

system-test/restart-testcases/restart_test_automation.sh

@@ -8,6 +8,10 @@ source "$(dirname "$0")"/../automation_utils.sh
 
 RESULT_FILE="$1"
 
+if [[ -z $CONSENSUS_TIMEOUT ]]; then
+  CONSENSUS_TIMEOUT=180
+fi
+
 startGpuMode="off"
 if [[ -z $ENABLE_GPU ]]; then
   ENABLE_GPU=false
@@ -21,20 +25,27 @@ if [[ "$ASYNC_NODE_INIT" = "true" ]]; then
   maybeAsyncNodeInit="--async-node-init"
 fi
 
-declare maybeExtraPrimordialStakes
-if [[ -n "$EXTRA_PRIMORDIAL_STAKES" ]]; then
-  maybeExtraPrimordialStakes="--extra-primordial-stakes $EXTRA_PRIMORDIAL_STAKES"
-fi
-
 # Restart the network
 "$REPO_ROOT"/net/net.sh stop
 
 sleep 2
 
 # shellcheck disable=SC2086
 "$REPO_ROOT"/net/net.sh start --skip-setup --no-snapshot-fetch --no-deploy \
-  --gpu-mode $startGpuMode $maybeAsyncNodeInit $maybeExtraPrimordialStakes
-
-# TODO add the test here
-
-echo "Restart Test Succeeded" >>"$RESULT_FILE"
+  --gpu-mode $startGpuMode $maybeAsyncNodeInit
+
+# wait until consensus
+start=$SECONDS
+activeStake=$(get_active_stake)
+while [[ $((SECONDS - start)) -lt $CONSENSUS_TIMEOUT ]]; do
+  currentStake=$(get_current_stake)
+  echo "$((SECONDS - start))s: Current stake $currentStake, Active stake $activeStake" >> "$RESULT_FILE"
+  if [[ $activeStake -eq $currentStake ]]; then
+    echo "Restart Test Succeeded" >>"$RESULT_FILE"
+    exit 0
+  fi
+  sleep 5
+done
+
+echo "Could not establish consensus in $CONSENSUS_TIMEOUT seconds" >> "$RESULT_FILE"
+exit 1