Skip to content
This repository has been archived by the owner on Jan 22, 2025. It is now read-only.

Commit

Permalink
remove unused historical_roots (#31539)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jeffwashington
jeffwashington authored May 8, 2023
1 parent b20024c commit 8d98589
Show file tree
Hide file tree
Showing 5 changed files with 3 additions and 349 deletions.
65 changes: 0 additions & 65 deletions runtime/src/accounts_db.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1811,7 +1811,6 @@ struct FlushStats {
#[derive(Debug, Default)]
struct LatestAccountsIndexRootsStats {
roots_len: AtomicUsize,
historical_roots_len: AtomicUsize,
uncleaned_roots_len: AtomicUsize,
previous_uncleaned_roots_len: AtomicUsize,
roots_range: AtomicU64,
Expand All @@ -1833,9 +1832,6 @@ impl LatestAccountsIndexRootsStats {
self.previous_uncleaned_roots_len
.store(value, Ordering::Relaxed);
}
if let Some(value) = accounts_index_roots_stats.historical_roots_len {
self.historical_roots_len.store(value, Ordering::Relaxed);
}
if let Some(value) = accounts_index_roots_stats.roots_range {
self.roots_range.store(value, Ordering::Relaxed);
}
Expand Down Expand Up @@ -1865,11 +1861,6 @@ impl LatestAccountsIndexRootsStats {
self.roots_len.load(Ordering::Relaxed) as i64,
i64
),
(
"historical_roots_len",
self.historical_roots_len.load(Ordering::Relaxed) as i64,
i64
),
(
"uncleaned_roots_len",
self.uncleaned_roots_len.load(Ordering::Relaxed) as i64,
Expand Down Expand Up @@ -2931,12 +2922,6 @@ impl AccountsDb {
*accounts_hash_complete_oldest_non_ancient_slot,
one_epoch_old_slot,
);

let accounts_hash_complete_oldest_non_ancient_slot =
*accounts_hash_complete_oldest_non_ancient_slot;

// now that accounts hash calculation is complete, we can remove old historical roots
self.remove_old_historical_roots(accounts_hash_complete_oldest_non_ancient_slot);
}

/// get the oldest slot that is within one epoch of the highest slot that has been used for hash calculation.
Expand Down Expand Up @@ -6888,11 +6873,6 @@ impl AccountsDb {
);
}

/// find slot >= 'slot' which is a root or in 'ancestors'
pub fn find_unskipped_slot(&self, slot: Slot, ancestors: Option<&Ancestors>) -> Option<Slot> {
self.accounts_index.get_next_original_root(slot, ancestors)
}

pub fn checked_iterative_sum_for_capitalization(total_cap: u64, new_cap: u64) -> u64 {
let new_total = total_cap as u128 + new_cap as u128;
AccountsHasher::checked_cast_for_capitalization(new_total)
Expand Down Expand Up @@ -7727,32 +7707,6 @@ impl AccountsDb {
result
}

/// return alive roots to retain, even though they are ancient
fn calc_alive_ancient_historical_roots(&self, min_root: Slot) -> HashSet<Slot> {
let mut ancient_alive_roots = HashSet::default();
{
let all_roots = self.accounts_index.roots_tracker.read().unwrap();

if let Some(min) = all_roots.historical_roots.min() {
for slot in min..min_root {
if all_roots.alive_roots.contains(&slot) {
// there was a storage for this root, so it counts as a root
ancient_alive_roots.insert(slot);
}
}
}
}
ancient_alive_roots
}

/// get rid of historical roots that are older than 'min_root'.
/// These will be older than an epoch from a current root.
fn remove_old_historical_roots(&self, min_root: Slot) {
let alive_roots = self.calc_alive_ancient_historical_roots(min_root);
self.accounts_index
.remove_old_historical_roots(min_root, &alive_roots);
}

/// Verify accounts hash at startup (or tests)
///
/// Calculate accounts hash(es) and compare them to the values set at startup.
Expand Down Expand Up @@ -16339,25 +16293,6 @@ pub mod tests {
}
}

#[test]
fn test_calc_alive_ancient_historical_roots() {
let db = AccountsDb::new_single_for_tests();
let min_root = 0;
let result = db.calc_alive_ancient_historical_roots(min_root);
assert!(result.is_empty());
for extra in 1..3 {
let result = db.calc_alive_ancient_historical_roots(extra);
assert_eq!(result, HashSet::default(), "extra: {extra}");
}

let extra = 3;
let active_root = 2;
db.accounts_index.add_root(active_root);
let result = db.calc_alive_ancient_historical_roots(extra);
let expected_alive_roots = [active_root].into_iter().collect();
assert_eq!(result, expected_alive_roots, "extra: {extra}");
}

impl AccountsDb {
/// useful to adapt tests written prior to introduction of the write cache
/// to use the write cache
Expand Down
214 changes: 0 additions & 214 deletions runtime/src/accounts_index.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -450,12 +450,6 @@ pub struct RootsTracker {
/// Updated every time we add a new root or clean/shrink an append vec into irrelevancy.
/// Range is approximately the last N slots where N is # slots per epoch.
pub(crate) alive_roots: RollingBitField,
/// Set of roots that are roots now or were roots at one point in time.
/// Range is approximately the last N slots where N is # slots per epoch.
/// A root could remain here if all entries in the append vec at that root are cleaned/shrunk and there are no
/// more entries for that slot. 'alive_roots' will no longer contain such roots.
/// This is a superset of 'alive_roots'
pub(crate) historical_roots: RollingBitField,
uncleaned_roots: HashSet<Slot>,
previous_uncleaned_roots: HashSet<Slot>,
}
Expand All @@ -473,7 +467,6 @@ impl RootsTracker {
pub fn new(max_width: u64) -> Self {
Self {
alive_roots: RollingBitField::new(max_width),
historical_roots: RollingBitField::new(max_width),
uncleaned_roots: HashSet::new(),
previous_uncleaned_roots: HashSet::new(),
}
Expand All @@ -490,7 +483,6 @@ pub struct AccountsIndexRootsStats {
pub uncleaned_roots_len: Option<usize>,
pub previous_uncleaned_roots_len: Option<usize>,
pub roots_range: Option<u64>,
pub historical_roots_len: Option<usize>,
pub rooted_cleaned_count: usize,
pub unrooted_cleaned_count: usize,
pub clean_unref_from_storage_us: u64,
Expand Down Expand Up @@ -1876,7 +1868,6 @@ impl<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> AccountsIndex<T, U> {
assert!(slot >= w_roots_tracker.alive_roots.max_inclusive());
// 'slot' is a root, so it is both 'root' and 'original'
w_roots_tracker.alive_roots.insert(slot);
w_roots_tracker.historical_roots.insert(slot);
}

pub fn add_uncleaned_roots<I>(&self, roots: I)
Expand All @@ -1895,52 +1886,6 @@ impl<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> AccountsIndex<T, U> {
.max_inclusive()
}

/// return the lowest original root >= slot, including historical_roots and ancestors
pub fn get_next_original_root(
&self,
slot: Slot,
ancestors: Option<&Ancestors>,
) -> Option<Slot> {
{
let roots_tracker = self.roots_tracker.read().unwrap();
for root in slot..roots_tracker.historical_roots.max_exclusive() {
if roots_tracker.historical_roots.contains(&root) {
return Some(root);
}
}
}
// ancestors are higher than roots, so look for roots first
if let Some(ancestors) = ancestors {
let min = std::cmp::max(slot, ancestors.min_slot());
for root in min..=ancestors.max_slot() {
if ancestors.contains_key(&root) {
return Some(root);
}
}
}
None
}

/// roots are inserted into 'historical_roots' and 'roots' as a new root is made.
/// roots are removed form 'roots' as all entries in the append vec become outdated.
/// This function exists to clean older entries from 'historical_roots'.
/// all roots < 'oldest_slot_to_keep' are removed from 'historical_roots'.
pub fn remove_old_historical_roots(&self, oldest_slot_to_keep: Slot, keep: &HashSet<Slot>) {
let mut roots = self
.roots_tracker
.read()
.unwrap()
.historical_roots
.get_all_less_than(oldest_slot_to_keep);
roots.retain(|root| !keep.contains(root));
if !roots.is_empty() {
let mut w_roots_tracker = self.roots_tracker.write().unwrap();
roots.into_iter().for_each(|root| {
w_roots_tracker.historical_roots.remove(&root);
});
}
}

/// Remove the slot when the storage for the slot is freed
/// Accounts no longer reference this slot.
/// return true if slot was a root
Expand Down Expand Up @@ -1972,7 +1917,6 @@ impl<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> AccountsIndex<T, U> {
stats.previous_uncleaned_roots_len =
Some(w_roots_tracker.previous_uncleaned_roots.len());
stats.roots_range = Some(w_roots_tracker.alive_roots.range_width());
stats.historical_roots_len = Some(w_roots_tracker.historical_roots.len());
drop(w_roots_tracker);
self.roots_removed.fetch_add(1, Ordering::Relaxed);
true
Expand Down Expand Up @@ -2178,164 +2122,6 @@ pub mod tests {
}
}

#[test]
fn test_get_next_original_root() {
let ancestors = None;
let index = AccountsIndex::<bool, bool>::default_for_tests();
for slot in 0..2 {
assert_eq!(index.get_next_original_root(slot, ancestors), None);
}
// roots are now [1]. 0 and 1 both return 1
index.add_root(1);
for slot in 0..2 {
assert_eq!(index.get_next_original_root(slot, ancestors), Some(1));
}
assert_eq!(index.get_next_original_root(2, ancestors), None); // no roots after 1, so asking for root >= 2 is None

// roots are now [1, 3]. 0 and 1 both return 1. 2 and 3 both return 3
index.add_root(3);
for slot in 0..2 {
assert_eq!(index.get_next_original_root(slot, ancestors), Some(1));
}
for slot in 2..4 {
assert_eq!(index.get_next_original_root(slot, ancestors), Some(3));
}
assert_eq!(index.get_next_original_root(4, ancestors), None); // no roots after 3, so asking for root >= 4 is None
}

#[test]
fn test_get_next_original_root_ancestors() {
let orig_ancestors = Ancestors::default();
let ancestors = Some(&orig_ancestors);
let index = AccountsIndex::<bool, bool>::default_for_tests();
for slot in 0..2 {
assert_eq!(index.get_next_original_root(slot, ancestors), None);
}
// ancestors are now [1]. 0 and 1 both return 1
let orig_ancestors = Ancestors::from(vec![1]);
let ancestors = Some(&orig_ancestors);
for slot in 0..2 {
assert_eq!(index.get_next_original_root(slot, ancestors), Some(1));
}
assert_eq!(index.get_next_original_root(2, ancestors), None); // no roots after 1, so asking for root >= 2 is None

// ancestors are now [1, 3]. 0 and 1 both return 1. 2 and 3 both return 3
let orig_ancestors = Ancestors::from(vec![1, 3]);
let ancestors = Some(&orig_ancestors);
for slot in 0..2 {
assert_eq!(index.get_next_original_root(slot, ancestors), Some(1));
}
for slot in 2..4 {
assert_eq!(index.get_next_original_root(slot, ancestors), Some(3));
}
assert_eq!(index.get_next_original_root(4, ancestors), None); // no roots after 3, so asking for root >= 4 is None
}

#[test]
fn test_get_next_original_root_roots_and_ancestors() {
let orig_ancestors = Ancestors::default();
let ancestors = Some(&orig_ancestors);
let index = AccountsIndex::<bool, bool>::default_for_tests();
for slot in 0..2 {
assert_eq!(index.get_next_original_root(slot, ancestors), None);
}
// roots are now [1]. 0 and 1 both return 1
index.add_root(1);
for slot in 0..2 {
assert_eq!(index.get_next_original_root(slot, ancestors), Some(1));
}
assert_eq!(index.get_next_original_root(2, ancestors), None); // no roots after 1, so asking for root >= 2 is None

// roots are now [1] and ancestors are now [3]. 0 and 1 both return 1. 2 and 3 both return 3
let orig_ancestors = Ancestors::from(vec![3]);
let ancestors = Some(&orig_ancestors);
for slot in 0..2 {
assert_eq!(index.get_next_original_root(slot, ancestors), Some(1));
}
for slot in 2..4 {
assert_eq!(index.get_next_original_root(slot, ancestors), Some(3));
}
assert_eq!(index.get_next_original_root(4, ancestors), None); // no roots after 3, so asking for root >= 4 is None
}

#[test]
fn test_remove_old_historical_roots() {
let index = AccountsIndex::<bool, bool>::default_for_tests();
index.add_root(1);
index.add_root(2);
assert_eq!(
index
.roots_tracker
.read()
.unwrap()
.historical_roots
.get_all(),
vec![1, 2]
);
let empty_hash_set = HashSet::default();
index.remove_old_historical_roots(2, &empty_hash_set);
assert_eq!(
index
.roots_tracker
.read()
.unwrap()
.historical_roots
.get_all(),
vec![2]
);
index.remove_old_historical_roots(3, &empty_hash_set);
assert!(
index
.roots_tracker
.read()
.unwrap()
.historical_roots
.is_empty(),
"{:?}",
index
.roots_tracker
.read()
.unwrap()
.historical_roots
.get_all()
);

// now use 'keep'
let index = AccountsIndex::<bool, bool>::default_for_tests();
index.add_root(1);
index.add_root(2);
let hash_set_1 = vec![1].into_iter().collect();
assert_eq!(
index
.roots_tracker
.read()
.unwrap()
.historical_roots
.get_all(),
vec![1, 2]
);
index.remove_old_historical_roots(2, &hash_set_1);
assert_eq!(
index
.roots_tracker
.read()
.unwrap()
.historical_roots
.get_all(),
vec![1, 2]
);
index.remove_old_historical_roots(3, &hash_set_1);
assert_eq!(
index
.roots_tracker
.read()
.unwrap()
.historical_roots
.get_all(),
vec![1]
);
}

const COLLECT_ALL_UNSORTED_FALSE: bool = false;

#[test]
Expand Down
Loading

0 comments on commit 8d98589

Please sign in to comment.