AcctIdx: move cached store id to bit (solana-labs#21892)

* AcctIdx: move cached store id to bit

* add comments/rename

runtime/src/account_info.rs

@@ -3,7 +3,7 @@
 //! AccountInfo is purely runtime state.
 //! Note that AccountInfo is saved to disk buckets during runtime, but disk buckets are recreated at startup.
 use crate::{
-    accounts_db::{AppendVecId, CACHE_VIRTUAL_OFFSET, CACHE_VIRTUAL_STORAGE_ID},
+    accounts_db::{AppendVecId, CACHE_VIRTUAL_OFFSET},
     accounts_index::{IsCached, ZeroLamport},
 };
 
@@ -64,22 +64,28 @@ pub struct AccountInfo {
 
     /// needed to track shrink candidacy in bytes. Used to update the number
     /// of alive bytes in an AppendVec as newer slots purge outdated entries
-    /// Note that highest bit is used for ZERO_LAMPORT_BIT
-    stored_size: StoredSize,
+    /// Note that highest bits are used by ALL_FLAGS
+    /// So, 'stored_size' is 'stored_size_mask' with ALL_FLAGS masked out.
+    stored_size_mask: StoredSize,
 }
 
-/// presence of this bit in stored_size indicates this account info references an account with zero lamports
-const ZERO_LAMPORT_BIT: StoredSize = 1 << (StoredSize::BITS - 1);
+/// These flags can be present in stored_size_mask to indicate additional info about the AccountInfo
+
+/// presence of this flag in stored_size_mask indicates this account info references an account with zero lamports
+const IS_ZERO_LAMPORT_FLAG: StoredSize = 1 << (StoredSize::BITS - 1);
+/// presence of this flag in stored_size_mask indicates this account info references an account stored in the cache
+const IS_CACHED_STORE_ID_FLAG: StoredSize = 1 << (StoredSize::BITS - 2);
+const ALL_FLAGS: StoredSize = IS_ZERO_LAMPORT_FLAG | IS_CACHED_STORE_ID_FLAG;
 
 impl ZeroLamport for AccountInfo {
     fn is_zero_lamport(&self) -> bool {
-        self.stored_size & ZERO_LAMPORT_BIT == ZERO_LAMPORT_BIT
+        self.stored_size_mask & IS_ZERO_LAMPORT_FLAG == IS_ZERO_LAMPORT_FLAG
     }
 }
 
 impl IsCached for AccountInfo {
     fn is_cached(&self) -> bool {
-        self.store_id == CACHE_VIRTUAL_STORAGE_ID
+        self.stored_size_mask & IS_CACHED_STORE_ID_FLAG == IS_CACHED_STORE_ID_FLAG
     }
 }
 
@@ -90,27 +96,30 @@ impl IsCached for StorageLocation {
 }
 
 impl AccountInfo {
-    pub fn new(
-        storage_location: StorageLocation,
-        mut stored_size: StoredSize,
-        lamports: u64,
-    ) -> Self {
+    pub fn new(storage_location: StorageLocation, stored_size: StoredSize, lamports: u64) -> Self {
+        assert_eq!(stored_size & ALL_FLAGS, 0);
+        let mut stored_size_mask = stored_size;
         let (store_id, offset) = match storage_location {
             StorageLocation::AppendVec(store_id, offset) => (store_id, offset),
-            StorageLocation::Cached => (CACHE_VIRTUAL_STORAGE_ID, CACHE_VIRTUAL_OFFSET),
+            StorageLocation::Cached => {
+                stored_size_mask |= IS_CACHED_STORE_ID_FLAG;
+                // We have to have SOME value for store_id when we are cached
+                const CACHE_VIRTUAL_STORAGE_ID: AppendVecId = AppendVecId::MAX;
+                (CACHE_VIRTUAL_STORAGE_ID, CACHE_VIRTUAL_OFFSET)
+            }
         };
-        assert!(stored_size < ZERO_LAMPORT_BIT);
         if lamports == 0 {
-            stored_size |= ZERO_LAMPORT_BIT;
+            stored_size_mask |= IS_ZERO_LAMPORT_FLAG;
         }
         Self {
             store_id,
             offset,
-            stored_size,
+            stored_size_mask,
         }
     }
 
     pub fn store_id(&self) -> AppendVecId {
+        assert!(!self.is_cached());
         self.store_id
     }
 
@@ -120,7 +129,7 @@ impl AccountInfo {
 
     pub fn stored_size(&self) -> StoredSize {
         // elminate the special bit that indicates the info references an account with zero lamports
-        self.stored_size & !ZERO_LAMPORT_BIT
+        self.stored_size_mask & !ALL_FLAGS
     }
 
     pub fn storage_location(&self) -> StorageLocation {

runtime/src/accounts_db.rs

@@ -110,12 +110,6 @@ pub const NUM_SCAN_PASSES_DEFAULT: usize = 2;
 // Metrics indicate a sweet spot in the 2.5k-5k range for mnb.
 const MAX_ITEMS_PER_CHUNK: Slot = 2_500;
 
-// A specially reserved storage id just for entries in the cache, so that
-// operations that take a storage entry can maintain a common interface
-// when interacting with cached accounts. This id is "virtual" in that it
-// doesn't actually refer to an actual storage entry.
-pub(crate) const CACHE_VIRTUAL_STORAGE_ID: AppendVecId = AppendVecId::MAX;
-
 // A specially reserved write version (identifier for ordering writes in an AppendVec)
 // for entries in the cache, so that  operations that take a storage entry can maintain
 // a common interface when interacting with cached accounts. This version is "virtual" in
@@ -3877,10 +3871,6 @@ impl AccountsDb {
             Self::page_align(size),
         ));
 
-        assert!(
-            store.append_vec_id() != CACHE_VIRTUAL_STORAGE_ID,
-            "We've run out of storage ids!"
-        );
         debug!(
             "creating store: {} slot: {} len: {} size: {} from: {} path: {:?}",
             store.append_vec_id(),