Don't stop to find newer cluster-confirmed roots (solana-labs#14557)

* Don't stop to find newer cluster-confirmed roots

* Fix and add new tests

* nits

ledger/src/blockstore_processor.rs

@@ -830,7 +830,7 @@ fn load_frozen_forks(
     let mut last_status_report = Instant::now();
     let mut last_free = Instant::now();
     let mut pending_slots = vec![];
-    let mut last_root_slot = root_bank.slot();
+    let mut last_root = root_bank.slot();
     let mut slots_elapsed = 0;
     let mut txs = 0;
     let blockstore_max_root = blockstore.max_root();
@@ -859,7 +859,7 @@ fn load_frozen_forks(
                 info!(
                     "processing ledger: slot={}, last root slot={} slots={} slots/s={:?} txs/s={}",
                     slot,
-                    last_root_slot,
+                    last_root,
                     slots_elapsed,
                     slots_elapsed as f32 / secs,
                     txs as f32 / secs,
@@ -896,7 +896,7 @@ fn load_frozen_forks(
             // If we've reached the last known root in blockstore, start looking
             // for newer cluster confirmed roots
             let new_root_bank = {
-                if *root == max_root {
+                if *root >= max_root {
                     supermajority_root_from_vote_accounts(
                         bank.slot(),
                         bank.total_epoch_stake(),
@@ -926,7 +926,8 @@ fn load_frozen_forks(
 
             if let Some(new_root_bank) = new_root_bank {
                 *root = new_root_bank.slot();
-                last_root_slot = new_root_bank.slot();
+                last_root = new_root_bank.slot();
+
                 leader_schedule_cache.set_root(&new_root_bank);
                 new_root_bank.squash();
 
@@ -949,7 +950,7 @@ fn load_frozen_forks(
 
             trace!(
                 "Bank for {}slot {} is complete. {} bytes allocated",
-                if last_root_slot == slot { "root " } else { "" },
+                if last_root == slot { "root " } else { "" },
                 slot,
                 allocated.since(initial_allocation)
             );
@@ -3117,17 +3118,21 @@ pub mod tests {
                   ...    minor fork
                   /
             `last_slot`
+                 |
+            `really_last_slot`
         */
         let starting_fork_slot = 5;
         let mut main_fork = tr(starting_fork_slot);
         let mut main_fork_ref = main_fork.root_mut();
 
         // Make enough slots to make a root slot > blockstore_root
         let expected_root_slot = starting_fork_slot + blockstore_root.unwrap_or(0);
+        let really_expected_root_slot = expected_root_slot + 1;
         let last_main_fork_slot = expected_root_slot + MAX_LOCKOUT_HISTORY as u64 + 1;
+        let really_last_main_fork_slot = last_main_fork_slot + 1;
 
         // Make `minor_fork`
-        let last_minor_fork_slot = last_main_fork_slot + 1;
+        let last_minor_fork_slot = really_last_main_fork_slot + 1;
         let minor_fork = tr(last_minor_fork_slot);
 
         // Make 'main_fork`
@@ -3162,6 +3167,7 @@ pub mod tests {
         let (bank_forks, _leader_schedule) =
             process_blockstore(&genesis_config, &blockstore, Vec::new(), opts.clone()).unwrap();
 
+        // prepare to add votes
         let last_vote_bank_hash = bank_forks.get(last_main_fork_slot - 1).unwrap().hash();
         let last_vote_blockhash = bank_forks
             .get(last_main_fork_slot - 1)
@@ -3191,19 +3197,23 @@ pub mod tests {
         );
 
         let (bank_forks, _leader_schedule) =
-            process_blockstore(&genesis_config, &blockstore, Vec::new(), opts).unwrap();
+            process_blockstore(&genesis_config, &blockstore, Vec::new(), opts.clone()).unwrap();
 
         assert_eq!(bank_forks.root(), expected_root_slot);
         assert_eq!(
             bank_forks.frozen_banks().len() as u64,
-            last_minor_fork_slot - expected_root_slot + 1
+            last_minor_fork_slot - really_expected_root_slot + 1
         );
 
         // Minor fork at `last_main_fork_slot + 1` was above the `expected_root_slot`
         // so should not have been purged
         //
         // Fork at slot 2 was purged because it was below the `expected_root_slot`
         for slot in 0..=last_minor_fork_slot {
+            // this slot will be created below
+            if slot == really_last_main_fork_slot {
+                continue;
+            }
             if slot >= expected_root_slot {
                 let bank = bank_forks.get(slot).unwrap();
                 assert_eq!(bank.slot(), slot);
@@ -3212,11 +3222,48 @@ pub mod tests {
                 assert!(bank_forks.get(slot).is_none());
             }
         }
+
+        // really prepare to add votes
+        let last_vote_bank_hash = bank_forks.get(last_main_fork_slot).unwrap().hash();
+        let last_vote_blockhash = bank_forks
+            .get(last_main_fork_slot)
+            .unwrap()
+            .last_blockhash();
+        let slots: Vec<_> = vec![last_main_fork_slot];
+        let vote_tx = vote_transaction::new_vote_transaction(
+            slots,
+            last_vote_bank_hash,
+            last_vote_blockhash,
+            &leader_keypair,
+            &validator_keypairs.vote_keypair,
+            &validator_keypairs.vote_keypair,
+            None,
+        );
+
+        // Add votes to `really_last_slot` so that `root` will be confirmed again
+        make_slot_with_vote_tx(
+            &blockstore,
+            ticks_per_slot,
+            really_last_main_fork_slot,
+            last_main_fork_slot,
+            &last_vote_blockhash,
+            vote_tx,
+            &leader_keypair,
+        );
+
+        let (bank_forks, _leader_schedule) =
+            process_blockstore(&genesis_config, &blockstore, Vec::new(), opts).unwrap();
+
+        assert_eq!(bank_forks.root(), really_expected_root_slot);
     }
 
     #[test]
-    fn test_process_blockstore_with_supermajority_root() {
+    fn test_process_blockstore_with_supermajority_root_without_blockstore_root() {
         run_test_process_blockstore_with_supermajority_root(None);
+    }
+
+    #[test]
+    fn test_process_blockstore_with_supermajority_root_with_blockstore_root() {
         run_test_process_blockstore_with_supermajority_root(Some(1))
     }