Merge remote-tracking branch 'origin/unstable' into cell_proofs

# Conflicts:
#	beacon_node/beacon_chain/src/fetch_blobs.rs

Author: jimmygchen

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "beacon_node"
-version = "7.0.0-beta.5"
+version = "7.1.0-beta.0"
 authors = [
     "Paul Hauner <paul@paulhauner.com>",
     "Age Manning <Age@AgeManning.com",

beacon_node/Cargo.toml

@@ -34,7 +34,6 @@ use crate::execution_payload::{get_execution_payload, NotifyExecutionLayer, Prep
 use crate::fetch_blobs::EngineGetBlobsOutput;
 use crate::fork_choice_signal::{ForkChoiceSignalRx, ForkChoiceSignalTx, ForkChoiceWaitResult};
 use crate::graffiti_calculator::GraffitiCalculator;
-use crate::head_tracker::{HeadTracker, HeadTrackerReader, SszHeadTracker};
 use crate::kzg_utils::reconstruct_blobs;
 use crate::light_client_finality_update_verification::{
     Error as LightClientFinalityUpdateError, VerifiedLightClientFinalityUpdate,
@@ -58,7 +57,7 @@ use crate::observed_block_producers::ObservedBlockProducers;
 use crate::observed_data_sidecars::ObservedDataSidecars;
 use crate::observed_operations::{ObservationOutcome, ObservedOperations};
 use crate::observed_slashable::ObservedSlashable;
-use crate::persisted_beacon_chain::{PersistedBeaconChain, DUMMY_CANONICAL_HEAD_BLOCK_ROOT};
+use crate::persisted_beacon_chain::PersistedBeaconChain;
 use crate::persisted_fork_choice::PersistedForkChoice;
 use crate::pre_finalization_cache::PreFinalizationBlockCache;
 use crate::shuffling_cache::{BlockShufflingIds, ShufflingCache};
@@ -454,8 +453,6 @@ pub struct BeaconChain<T: BeaconChainTypes> {
     /// A handler for events generated by the beacon chain. This is only initialized when the
     /// HTTP server is enabled.
     pub event_handler: Option<ServerSentEventHandler<T::EthSpec>>,
-    /// Used to track the heads of the beacon chain.
-    pub(crate) head_tracker: Arc<HeadTracker>,
     /// Caches the attester shuffling for a given epoch and shuffling key root.
     pub shuffling_cache: RwLock<ShufflingCache>,
     /// A cache of eth1 deposit data at epoch boundaries for deposit finalization
@@ -607,57 +604,13 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         })
     }
 
-    /// Persists the head tracker and fork choice.
+    /// Return a database operation for writing the `PersistedBeaconChain` to disk.
     ///
-    /// We do it atomically even though no guarantees need to be made about blocks from
-    /// the head tracker also being present in fork choice.
-    pub fn persist_head_and_fork_choice(&self) -> Result<(), Error> {
-        let mut batch = vec![];
-
-        let _head_timer = metrics::start_timer(&metrics::PERSIST_HEAD);
-
-        // Hold a lock to head_tracker until it has been persisted to disk. Otherwise there's a race
-        // condition with the pruning thread which can result in a block present in the head tracker
-        // but absent in the DB. This inconsistency halts pruning and dramastically increases disk
-        // size. Ref: https://github.com/sigp/lighthouse/issues/4773
-        let head_tracker = self.head_tracker.0.read();
-        batch.push(self.persist_head_in_batch(&head_tracker));
-
-        let _fork_choice_timer = metrics::start_timer(&metrics::PERSIST_FORK_CHOICE);
-        batch.push(self.persist_fork_choice_in_batch());
-
-        self.store.hot_db.do_atomically(batch)?;
-        drop(head_tracker);
-
-        Ok(())
-    }
-
-    /// Return a `PersistedBeaconChain` without reference to a `BeaconChain`.
-    pub fn make_persisted_head(
-        genesis_block_root: Hash256,
-        head_tracker_reader: &HeadTrackerReader,
-    ) -> PersistedBeaconChain {
-        PersistedBeaconChain {
-            _canonical_head_block_root: DUMMY_CANONICAL_HEAD_BLOCK_ROOT,
-            genesis_block_root,
-            ssz_head_tracker: SszHeadTracker::from_map(head_tracker_reader),
-        }
-    }
-
-    /// Return a database operation for writing the beacon chain head to disk.
-    pub fn persist_head_in_batch(
-        &self,
-        head_tracker_reader: &HeadTrackerReader,
-    ) -> KeyValueStoreOp {
-        Self::persist_head_in_batch_standalone(self.genesis_block_root, head_tracker_reader)
-    }
-
-    pub fn persist_head_in_batch_standalone(
-        genesis_block_root: Hash256,
-        head_tracker_reader: &HeadTrackerReader,
-    ) -> KeyValueStoreOp {
-        Self::make_persisted_head(genesis_block_root, head_tracker_reader)
-            .as_kv_store_op(BEACON_CHAIN_DB_KEY)
+    /// These days the `PersistedBeaconChain` is only used to store the genesis block root, so it
+    /// should only ever be written once at startup. It used to be written more frequently, but
+    /// this is no longer necessary.
+    pub fn persist_head_in_batch_standalone(genesis_block_root: Hash256) -> KeyValueStoreOp {
+        PersistedBeaconChain { genesis_block_root }.as_kv_store_op(BEACON_CHAIN_DB_KEY)
     }
 
     /// Load fork choice from disk, returning `None` if it isn't found.
@@ -1450,12 +1403,13 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     ///
     /// Returns `(block_root, block_slot)`.
     pub fn heads(&self) -> Vec<(Hash256, Slot)> {
-        self.head_tracker.heads()
-    }
-
-    /// Only used in tests.
-    pub fn knows_head(&self, block_hash: &SignedBeaconBlockHash) -> bool {
-        self.head_tracker.contains_head((*block_hash).into())
+        self.canonical_head
+            .fork_choice_read_lock()
+            .proto_array()
+            .heads_descended_from_finalization::<T::EthSpec>()
+            .iter()
+            .map(|node| (node.root, node.slot))
+            .collect()
     }
 
     /// Returns the `BeaconState` at the given slot.
@@ -1735,8 +1689,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         let notif = ManualFinalizationNotification {
             state_root: state_root.into(),
             checkpoint,
-            head_tracker: self.head_tracker.clone(),
-            genesis_block_root: self.genesis_block_root,
         };
 
         self.store_migrator.process_manual_finalization(notif);
@@ -3763,7 +3715,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             state,
             parent_block,
             parent_eth1_finalization_data,
-            confirmed_state_roots,
             consensus_context,
         } = import_data;
 
@@ -3787,7 +3738,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                         block,
                         block_root,
                         state,
-                        confirmed_state_roots,
                         payload_verification_outcome.payload_verification_status,
                         parent_block,
                         parent_eth1_finalization_data,
@@ -3825,7 +3775,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         signed_block: AvailableBlock<T::EthSpec>,
         block_root: Hash256,
         mut state: BeaconState<T::EthSpec>,
-        confirmed_state_roots: Vec<Hash256>,
         payload_verification_status: PayloadVerificationStatus,
         parent_block: SignedBlindedBeaconBlock<T::EthSpec>,
         parent_eth1_finalization_data: Eth1FinalizationData,
@@ -4013,11 +3962,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
 
         let block = signed_block.message();
         let db_write_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_DB_WRITE);
-        ops.extend(
-            confirmed_state_roots
-                .into_iter()
-                .map(StoreOp::DeleteStateTemporaryFlag),
-        );
         ops.push(StoreOp::PutBlock(block_root, signed_block.clone()));
         ops.push(StoreOp::PutState(block.state_root(), &state));
 
@@ -4044,9 +3988,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         // about it.
         let block_time_imported = timestamp_now();
 
-        let parent_root = block.parent_root();
-        let slot = block.slot();
-
         let current_eth1_finalization_data = Eth1FinalizationData {
             eth1_data: state.eth1_data().clone(),
             eth1_deposit_index: state.eth1_deposit_index(),
@@ -4067,9 +4008,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                 });
         }
 
-        self.head_tracker
-            .register_block(block_root, parent_root, slot);
-
         metrics::stop_timer(db_write_timer);
 
         metrics::inc_counter(&metrics::BLOCK_PROCESSING_SUCCESSES);
@@ -7203,7 +7141,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
 impl<T: BeaconChainTypes> Drop for BeaconChain<T> {
     fn drop(&mut self) {
         let drop = || -> Result<(), Error> {
-            self.persist_head_and_fork_choice()?;
+            self.persist_fork_choice()?;
             self.persist_op_pool()?;
             self.persist_eth1_cache()
         };

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -1460,22 +1460,8 @@ impl<T: BeaconChainTypes> ExecutionPendingBlock<T> {
 
         let catchup_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_CATCHUP_STATE);
 
-        // Stage a batch of operations to be completed atomically if this block is imported
-        // successfully. If there is a skipped slot, we include the state root of the pre-state,
-        // which may be an advanced state that was stored in the DB with a `temporary` flag.
         let mut state = parent.pre_state;
 
-        let mut confirmed_state_roots =
-            if block.slot() > state.slot() && state.slot() > parent.beacon_block.slot() {
-                // Advanced pre-state. Delete its temporary flag.
-                let pre_state_root = state.update_tree_hash_cache()?;
-                vec![pre_state_root]
-            } else {
-                // Pre state is either unadvanced, or should not be stored long-term because there
-                // is no skipped slot between `parent` and `block`.
-                vec![]
-            };
-
         // The block must have a higher slot than its parent.
         if block.slot() <= parent.beacon_block.slot() {
             return Err(BlockError::BlockIsNotLaterThanParent {
@@ -1522,38 +1508,29 @@ impl<T: BeaconChainTypes> ExecutionPendingBlock<T> {
                 // processing, but we get early access to it.
                 let state_root = state.update_tree_hash_cache()?;
 
-                // Store the state immediately, marking it as temporary, and staging the deletion
-                // of its temporary status as part of the larger atomic operation.
+                // Store the state immediately.
                 let txn_lock = chain.store.hot_db.begin_rw_transaction();
                 let state_already_exists =
                     chain.store.load_hot_state_summary(&state_root)?.is_some();
 
                 let state_batch = if state_already_exists {
-                    // If the state exists, it could be temporary or permanent, but in neither case
-                    // should we rewrite it or store a new temporary flag for it. We *will* stage
-                    // the temporary flag for deletion because it's OK to double-delete the flag,
-                    // and we don't mind if another thread gets there first.
+                    // If the state exists, we do not need to re-write it.
                     vec![]
                 } else {
-                    vec![
-                        if state.slot() % T::EthSpec::slots_per_epoch() == 0 {
-                            StoreOp::PutState(state_root, &state)
-                        } else {
-                            StoreOp::PutStateSummary(
-                                state_root,
-                                HotStateSummary::new(&state_root, &state)?,
-                            )
-                        },
-                        StoreOp::PutStateTemporaryFlag(state_root),
-                    ]
+                    vec![if state.slot() % T::EthSpec::slots_per_epoch() == 0 {
+                        StoreOp::PutState(state_root, &state)
+                    } else {
+                        StoreOp::PutStateSummary(
+                            state_root,
+                            HotStateSummary::new(&state_root, &state)?,
+                        )
+                    }]
                 };
                 chain
                     .store
                     .do_atomically_with_block_and_blobs_cache(state_batch)?;
                 drop(txn_lock);
 
-                confirmed_state_roots.push(state_root);
-
                 state_root
             };
 
@@ -1720,7 +1697,6 @@ impl<T: BeaconChainTypes> ExecutionPendingBlock<T> {
                 state,
                 parent_block: parent.beacon_block,
                 parent_eth1_finalization_data,
-                confirmed_state_roots,
                 consensus_context,
             },
             payload_verification_handle,

beacon_node/beacon_chain/src/block_verification.rs

@@ -358,7 +358,6 @@ pub struct BlockImportData<E: EthSpec> {
     pub state: BeaconState<E>,
     pub parent_block: SignedBeaconBlock<E, BlindedPayload<E>>,
     pub parent_eth1_finalization_data: Eth1FinalizationData,
-    pub confirmed_state_roots: Vec<Hash256>,
     pub consensus_context: ConsensusContext<E>,
 }
 
@@ -376,7 +375,6 @@ impl<E: EthSpec> BlockImportData<E> {
                 eth1_data: <_>::default(),
                 eth1_deposit_index: 0,
             },
-            confirmed_state_roots: vec![],
             consensus_context: ConsensusContext::new(Slot::new(0)),
         }
     }