merge queue: embarking main (ecd4e1f) and #7437 together

zebra-chain/src/block.rs

@@ -2,6 +2,8 @@
 
 use std::{collections::HashMap, fmt, ops::Neg, sync::Arc};
 
+use halo2::pasta::pallas;
+
 use crate::{
     amount::NegativeAllowed,
     block::merkle::AuthDataRoot,
@@ -152,16 +154,30 @@ impl Block {
 
     /// Access the [`orchard::Nullifier`]s from all transactions in this block.
     pub fn orchard_nullifiers(&self) -> impl Iterator<Item = &orchard::Nullifier> {
-        // Work around a compiler panic (ICE) with flat_map():
-        // https://github.com/rust-lang/rust/issues/105044
-        #[allow(clippy::needless_collect)]
-        let nullifiers: Vec<_> = self
-            .transactions
+        self.transactions
             .iter()
             .flat_map(|transaction| transaction.orchard_nullifiers())
-            .collect();
+    }
+
+    /// Access the [`sprout::NoteCommitment`]s from all transactions in this block.
+    pub fn sprout_note_commitments(&self) -> impl Iterator<Item = &sprout::NoteCommitment> {
+        self.transactions
+            .iter()
+            .flat_map(|transaction| transaction.sprout_note_commitments())
+    }
 
-        nullifiers.into_iter()
+    /// Access the [sapling note commitments](jubjub::Fq) from all transactions in this block.
+    pub fn sapling_note_commitments(&self) -> impl Iterator<Item = &jubjub::Fq> {
+        self.transactions
+            .iter()
+            .flat_map(|transaction| transaction.sapling_note_commitments())
+    }
+
+    /// Access the [orchard note commitments](pallas::Base) from all transactions in this block.
+    pub fn orchard_note_commitments(&self) -> impl Iterator<Item = &pallas::Base> {
+        self.transactions
+            .iter()
+            .flat_map(|transaction| transaction.orchard_note_commitments())
     }
 
     /// Count how many Sapling transactions exist in a block,

zebra-chain/src/orchard/tree.rs

@@ -32,7 +32,7 @@ use crate::{
     serialization::{
         serde_helpers, ReadZcashExt, SerializationError, ZcashDeserialize, ZcashSerialize,
     },
-    subtree::TRACKED_SUBTREE_HEIGHT,
+    subtree::{NoteCommitmentSubtreeIndex, TRACKED_SUBTREE_HEIGHT},
 };
 
 pub mod legacy;
@@ -392,28 +392,48 @@ impl NoteCommitmentTree {
         }
     }
 
+    /// Returns frontier of non-empty tree, or `None` if the tree is empty.
+    fn frontier(&self) -> Option<&NonEmptyFrontier<Node>> {
+        self.inner.value()
+    }
+
     /// Returns true if the most recently appended leaf completes the subtree
-    pub fn is_complete_subtree(tree: &NonEmptyFrontier<Node>) -> bool {
+    pub fn is_complete_subtree(&self) -> bool {
+        let Some(tree) = self.frontier() else {
+            // An empty tree can't be a complete subtree.
+            return false;
+        };
+
         tree.position()
             .is_complete_subtree(TRACKED_SUBTREE_HEIGHT.into())
     }
 
-    /// Returns subtree address at [`TRACKED_SUBTREE_HEIGHT`]
-    pub fn subtree_address(tree: &NonEmptyFrontier<Node>) -> incrementalmerkletree::Address {
-        incrementalmerkletree::Address::above_position(
+    /// Returns the subtree index at [`TRACKED_SUBTREE_HEIGHT`].
+    /// This is the number of complete or incomplete subtrees that are currently in the tree.
+    /// Returns `None` if the tree is empty.
+    #[allow(clippy::unwrap_in_result)]
+    pub fn subtree_index(&self) -> Option<NoteCommitmentSubtreeIndex> {
+        let tree = self.frontier()?;
+
+        let index = incrementalmerkletree::Address::above_position(
             TRACKED_SUBTREE_HEIGHT.into(),
             tree.position(),
         )
+        .index()
+        .try_into()
+        .expect("fits in u16");
+
+        Some(index)
     }
 
     /// Returns subtree index and root if the most recently appended leaf completes the subtree
-    #[allow(clippy::unwrap_in_result)]
-    pub fn completed_subtree_index_and_root(&self) -> Option<(u16, Node)> {
-        let value = self.inner.value()?;
-        Self::is_complete_subtree(value).then_some(())?;
-        let address = Self::subtree_address(value);
-        let index = address.index().try_into().expect("should fit in u16");
-        let root = value.root(Some(TRACKED_SUBTREE_HEIGHT.into()));
+    pub fn completed_subtree_index_and_root(&self) -> Option<(NoteCommitmentSubtreeIndex, Node)> {
+        if !self.is_complete_subtree() {
+            return None;
+        }
+
+        let index = self.subtree_index()?;
+        let root = self.frontier()?.root(Some(TRACKED_SUBTREE_HEIGHT.into()));
 
         Some((index, root))
     }

zebra-chain/src/parallel/tree.rs

@@ -4,7 +4,11 @@ use std::sync::Arc;
 
 use thiserror::Error;
 
-use crate::{block::Block, orchard, sapling, sprout, subtree::NoteCommitmentSubtree};
+use crate::{
+    block::Block,
+    orchard, sapling, sprout,
+    subtree::{NoteCommitmentSubtree, NoteCommitmentSubtreeIndex},
+};
 
 /// An argument wrapper struct for note commitment trees.
 #[derive(Clone, Debug)]
@@ -65,24 +69,9 @@ impl NoteCommitmentTrees {
             ..
         } = self.clone();
 
-        let sprout_note_commitments: Vec<_> = block
-            .transactions
-            .iter()
-            .flat_map(|tx| tx.sprout_note_commitments())
-            .cloned()
-            .collect();
-        let sapling_note_commitments: Vec<_> = block
-            .transactions
-            .iter()
-            .flat_map(|tx| tx.sapling_note_commitments())
-            .cloned()
-            .collect();
-        let orchard_note_commitments: Vec<_> = block
-            .transactions
-            .iter()
-            .flat_map(|tx| tx.orchard_note_commitments())
-            .cloned()
-            .collect();
+        let sprout_note_commitments: Vec<_> = block.sprout_note_commitments().cloned().collect();
+        let sapling_note_commitments: Vec<_> = block.sapling_note_commitments().cloned().collect();
+        let orchard_note_commitments: Vec<_> = block.orchard_note_commitments().cloned().collect();
 
         let mut sprout_result = None;
         let mut sapling_result = None;
@@ -163,7 +152,7 @@ impl NoteCommitmentTrees {
     ) -> Result<
         (
             Arc<sapling::tree::NoteCommitmentTree>,
-            Option<(u16, sapling::tree::Node)>,
+            Option<(NoteCommitmentSubtreeIndex, sapling::tree::Node)>,
         ),
         NoteCommitmentTreeError,
     > {
@@ -202,7 +191,7 @@ impl NoteCommitmentTrees {
     ) -> Result<
         (
             Arc<orchard::tree::NoteCommitmentTree>,
-            Option<(u16, orchard::tree::Node)>,
+            Option<(NoteCommitmentSubtreeIndex, orchard::tree::Node)>,
         ),
         NoteCommitmentTreeError,
     > {

zebra-chain/src/sapling/tree.rs

@@ -33,7 +33,7 @@ use crate::{
     serialization::{
         serde_helpers, ReadZcashExt, SerializationError, ZcashDeserialize, ZcashSerialize,
     },
-    subtree::TRACKED_SUBTREE_HEIGHT,
+    subtree::{NoteCommitmentSubtreeIndex, TRACKED_SUBTREE_HEIGHT},
 };
 
 pub mod legacy;
@@ -373,28 +373,48 @@ impl NoteCommitmentTree {
         }
     }
 
+    /// Returns frontier of non-empty tree, or None.
+    fn frontier(&self) -> Option<&NonEmptyFrontier<Node>> {
+        self.inner.value()
+    }
+
     /// Returns true if the most recently appended leaf completes the subtree
-    pub fn is_complete_subtree(tree: &NonEmptyFrontier<Node>) -> bool {
+    pub fn is_complete_subtree(&self) -> bool {
+        let Some(tree) = self.frontier() else {
+            // An empty tree can't be a complete subtree.
+            return false;
+        };
+
         tree.position()
             .is_complete_subtree(TRACKED_SUBTREE_HEIGHT.into())
     }
 
-    /// Returns subtree address at [`TRACKED_SUBTREE_HEIGHT`]
-    pub fn subtree_address(tree: &NonEmptyFrontier<Node>) -> incrementalmerkletree::Address {
-        incrementalmerkletree::Address::above_position(
+    /// Returns the subtree index at [`TRACKED_SUBTREE_HEIGHT`].
+    /// This is the number of complete or incomplete subtrees that are currently in the tree.
+    /// Returns `None` if the tree is empty.
+    #[allow(clippy::unwrap_in_result)]
+    pub fn subtree_index(&self) -> Option<NoteCommitmentSubtreeIndex> {
+        let tree = self.frontier()?;
+
+        let index = incrementalmerkletree::Address::above_position(
             TRACKED_SUBTREE_HEIGHT.into(),
             tree.position(),
         )
+        .index()
+        .try_into()
+        .expect("fits in u16");
+
+        Some(index)
     }
 
     /// Returns subtree index and root if the most recently appended leaf completes the subtree
-    #[allow(clippy::unwrap_in_result)]
-    pub fn completed_subtree_index_and_root(&self) -> Option<(u16, Node)> {
-        let value = self.inner.value()?;
-        Self::is_complete_subtree(value).then_some(())?;
-        let address = Self::subtree_address(value);
-        let index = address.index().try_into().expect("should fit in u16");
-        let root = value.root(Some(TRACKED_SUBTREE_HEIGHT.into()));
+    pub fn completed_subtree_index_and_root(&self) -> Option<(NoteCommitmentSubtreeIndex, Node)> {
+        if !self.is_complete_subtree() {
+            return None;
+        }
+
+        let index = self.subtree_index()?;
+        let root = self.frontier()?.root(Some(TRACKED_SUBTREE_HEIGHT.into()));
 
         Some((index, root))
     }

zebra-chain/src/subtree.rs

@@ -1,5 +1,7 @@
 //! Struct representing Sapling/Orchard note commitment subtrees
 
+use std::num::TryFromIntError;
+
 use serde::{Deserialize, Serialize};
 
 use crate::block::Height;
@@ -23,6 +25,22 @@ impl From<u16> for NoteCommitmentSubtreeIndex {
     }
 }
 
+impl TryFrom<u64> for NoteCommitmentSubtreeIndex {
+    type Error = TryFromIntError;
+
+    fn try_from(value: u64) -> Result<Self, Self::Error> {
+        u16::try_from(value).map(Self)
+    }
+}
+
+// If we want to automatically convert NoteCommitmentSubtreeIndex to the generic integer literal
+// type, we can only implement conversion into u64. (Or u16, but not both.)
+impl From<NoteCommitmentSubtreeIndex> for u64 {
+    fn from(value: NoteCommitmentSubtreeIndex) -> Self {
+        value.0.into()
+    }
+}
+
 // TODO:
 // - consider defining sapling::SubtreeRoot and orchard::SubtreeRoot types or type wrappers,
 //   to avoid type confusion between the leaf Node and subtree root types.

zebra-state/src/constants.rs

@@ -48,11 +48,11 @@ pub(crate) const DATABASE_FORMAT_VERSION: u64 = 25;
 /// - adding new column families,
 /// - changing the format of a column family in a compatible way, or
 /// - breaking changes with compatibility code in all supported Zebra versions.
-pub(crate) const DATABASE_FORMAT_MINOR_VERSION: u64 = 1;
+pub(crate) const DATABASE_FORMAT_MINOR_VERSION: u64 = 2;
 
 /// The database format patch version, incremented each time the on-disk database format has a
 /// significant format compatibility fix.
-pub(crate) const DATABASE_FORMAT_PATCH_VERSION: u64 = 1;
+pub(crate) const DATABASE_FORMAT_PATCH_VERSION: u64 = 0;
 
 /// The name of the file containing the minor and patch database versions.
 ///

zebra-state/src/service/finalized_state.rs

@@ -98,7 +98,7 @@ impl FinalizedState {
         network: Network,
         #[cfg(feature = "elasticsearch")] elastic_db: Option<elasticsearch::Elasticsearch>,
     ) -> Self {
-        let db = ZebraDb::new(config, network);
+        let db = ZebraDb::new(config, network, false);
 
         #[cfg(feature = "elasticsearch")]
         let new_state = Self {

zebra-state/src/service/finalized_state/disk_format/upgrade.rs

@@ -25,6 +25,8 @@ use crate::{
     Config,
 };
 
+pub(crate) mod add_subtrees;
+
 /// The kind of database format change we're performing.
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub enum DbFormatChange {
@@ -195,7 +197,7 @@ impl DbFormatChange {
                 network,
                 initial_tip_height,
                 upgrade_db.clone(),
-                cancel_receiver,
+                &cancel_receiver,
             )?,
 
             NewlyCreated { .. } => {
@@ -218,12 +220,14 @@ impl DbFormatChange {
             }
         }
 
-        // This check should pass for all format changes:
-        // - upgrades should de-duplicate trees if needed (and they already do this check)
-        // - an empty state doesn't have any trees, so it can't have duplicate trees
-        // - since this Zebra code knows how to de-duplicate trees, downgrades using this code
-        //   still know how to make sure trees are unique
-        Self::check_for_duplicate_trees(upgrade_db);
+        // These checks should pass for all format changes:
+        // - upgrades should produce a valid format (and they already do that check)
+        // - an empty state should pass all the format checks
+        // - since the running Zebra code knows how to upgrade the database to this format,
+        //   downgrades using this running code still know how to create a valid database
+        //   (unless a future upgrade breaks these format checks)
+        Self::check_for_duplicate_trees(upgrade_db.clone());
+        add_subtrees::check(&upgrade_db);
 
         Ok(())
     }
@@ -245,7 +249,7 @@ impl DbFormatChange {
         network: Network,
         initial_tip_height: Option<Height>,
         db: ZebraDb,
-        cancel_receiver: mpsc::Receiver<CancelFormatChange>,
+        cancel_receiver: &mpsc::Receiver<CancelFormatChange>,
     ) -> Result<(), CancelFormatChange> {
         let Upgrade {
             newer_running_version,
@@ -277,7 +281,7 @@ impl DbFormatChange {
             return Ok(());
         };
 
-        // Start of a database upgrade task.
+        // Note commitment tree de-duplication database upgrade task.
 
         let version_for_pruning_trees =
             Version::parse("25.1.1").expect("Hardcoded version string should be valid.");
@@ -339,20 +343,38 @@ impl DbFormatChange {
             }
 
             // Before marking the state as upgraded, check that the upgrade completed successfully.
-            Self::check_for_duplicate_trees(db);
+            Self::check_for_duplicate_trees(db.clone());
 
             // Mark the database as upgraded. Zebra won't repeat the upgrade anymore once the
             // database is marked, so the upgrade MUST be complete at this point.
             Self::mark_as_upgraded_to(&version_for_pruning_trees, &config, network);
         }
 
+        // Note commitment subtree creation database upgrade task.
+
+        let version_for_adding_subtrees =
+            Version::parse("25.2.0").expect("Hardcoded version string should be valid.");
+
+        // Check if we need to add note commitment subtrees to the database.
+        if older_disk_version < version_for_adding_subtrees {
+            add_subtrees::run(initial_tip_height, &db, cancel_receiver)?;
+
+            // Before marking the state as upgraded, check that the upgrade completed successfully.
+            add_subtrees::check(&db);
+
+            // Mark the database as upgraded. Zebra won't repeat the upgrade anymore once the
+            // database is marked, so the upgrade MUST be complete at this point.
+            Self::mark_as_upgraded_to(&version_for_adding_subtrees, &config, network);
+        }
+
         // # New Upgrades Usually Go Here
         //
         // New code goes above this comment!
         //
         // Run the latest format upgrade code after the other upgrades are complete,
         // then mark the format as upgraded. The code should check `cancel_receiver`
         // every time it runs its inner update loop.
+
         info!(
             ?newer_running_version,
             "Zebra automatically upgraded the database format to:"