build(deps): update ecc dependencies for zcashd 5.6.0, and create legacy state format compatibility layer

deny.toml

@@ -82,6 +82,12 @@ skip-tree = [
     # wait for metrics-exporter-prometheus to upgrade
     { name = "hashbrown", version = "=0.13.2" },
 
+    # wait for zebra-chain to upgrade
+    { name = "secp256k1", version = "=0.21.3" },
+
+    # wait for zebra-chain to upgrade `secp256k1`
+    { name = "secp256k1-sys", version = "=0.4.2" },
+
     # ECC crates
 
     # wait for zcash_primitives to remove duplicated dependencies

zebra-chain/Cargo.toml

@@ -53,11 +53,12 @@ bitflags = "2.3.3"
 bitflags-serde-legacy = "0.1.1"
 blake2b_simd = "1.0.1"
 blake2s_simd = "1.0.1"
+bridgetree = "0.3.0"
 bs58 = { version = "0.5.0", features = ["check"] }
 byteorder = "1.4.3"
 equihash = "0.2.0"
 group = "0.13.0"
-incrementalmerkletree = "0.3.1"
+incrementalmerkletree = "0.4.0"
 jubjub = "0.10.0"
 lazy_static = "1.4.0"
 num-integer = "0.1.45"
@@ -72,11 +73,11 @@ x25519-dalek = { version = "2.0.0-rc.3", features = ["serde"] }
 
 # ECC deps
 halo2 = { package = "halo2_proofs", version = "0.3.0" }
-orchard = "0.4.0"
+orchard = "0.5.0"
 zcash_encoding = "0.2.0"
 zcash_history = "0.3.0"
-zcash_note_encryption = "0.3.0"
-zcash_primitives = { version = "0.11.0", features = ["transparent-inputs"] }
+zcash_note_encryption = "0.4.0"
+zcash_primitives = { version = "0.12.0", features = ["transparent-inputs"] }
 
 # Time
 chrono = { version = "0.4.26", default-features = false, features = ["clock", "std", "serde"] }
@@ -108,7 +109,7 @@ reddsa = "0.5.0"
 serde_json = { version = "1.0.100", optional = true }
 
 # Experimental feature getblocktemplate-rpcs
-zcash_address = { version = "0.2.1", optional = true }
+zcash_address = { version = "0.3.0", optional = true }
 
 # Optional testing dependencies
 proptest = { version = "1.2.0", optional = true }

zebra-chain/src/orchard/tree.rs

@@ -18,18 +18,22 @@ use std::{
 };
 
 use bitvec::prelude::*;
+use bridgetree;
 use halo2::pasta::{group::ff::PrimeField, pallas};
-use incrementalmerkletree::{bridgetree, Frontier};
+use incrementalmerkletree::Hashable;
 use lazy_static::lazy_static;
 use thiserror::Error;
-use zcash_primitives::merkle_tree::{self, CommitmentTree};
+use zcash_primitives::merkle_tree::{write_commitment_tree, HashSer};
 
 use super::sinsemilla::*;
 
 use crate::serialization::{
     serde_helpers, ReadZcashExt, SerializationError, ZcashDeserialize, ZcashSerialize,
 };
 
+pub mod legacy;
+use legacy::LegacyNoteCommitmentTree;
+
 /// The type that is used to update the note commitment tree.
 ///
 /// Unfortunately, this is not the same as `orchard::NoteCommitment`.
@@ -164,18 +168,18 @@ impl ZcashDeserialize for Root {
 
 /// A node of the Orchard Incremental Note Commitment Tree.
 #[derive(Copy, Clone, Debug, Eq, PartialEq)]
-struct Node(pallas::Base);
+pub struct Node(pallas::Base);
 
 /// Required to convert [`NoteCommitmentTree`] into [`SerializedTree`].
 ///
 /// Zebra stores Orchard note commitment trees as [`Frontier`][1]s while the
 /// [`z_gettreestate`][2] RPC requires [`CommitmentTree`][3]s. Implementing
-/// [`merkle_tree::Hashable`] for [`Node`]s allows the conversion.
+/// [`HashSer`] for [`Node`]s allows the conversion.
 ///
 /// [1]: bridgetree::Frontier
 /// [2]: https://zcash.github.io/rpc/z_gettreestate.html
-/// [3]: merkle_tree::CommitmentTree
-impl merkle_tree::Hashable for Node {
+/// [3]: incrementalmerkletree::frontier::CommitmentTree
+impl HashSer for Node {
     fn read<R: io::Read>(mut reader: R) -> io::Result<Self> {
         let mut repr = [0u8; 32];
         reader.read_exact(&mut repr)?;
@@ -192,38 +196,23 @@ impl merkle_tree::Hashable for Node {
     fn write<W: io::Write>(&self, mut writer: W) -> io::Result<()> {
         writer.write_all(&self.0.to_repr())
     }
-
-    fn combine(level: usize, a: &Self, b: &Self) -> Self {
-        let level = u8::try_from(level).expect("level must fit into u8");
-        let layer = MERKLE_DEPTH - 1 - level;
-        Self(merkle_crh_orchard(layer, a.0, b.0))
-    }
-
-    fn blank() -> Self {
-        Self(NoteCommitmentTree::uncommitted())
-    }
-
-    fn empty_root(level: usize) -> Self {
-        let layer_below = usize::from(MERKLE_DEPTH) - level;
-        Self(EMPTY_ROOTS[layer_below])
-    }
 }
 
-impl incrementalmerkletree::Hashable for Node {
+impl Hashable for Node {
     fn empty_leaf() -> Self {
         Self(NoteCommitmentTree::uncommitted())
     }
 
     /// Combine two nodes to generate a new node in the given level.
     /// Level 0 is the layer above the leaves (layer 31).
     /// Level 31 is the root (layer 0).
-    fn combine(level: incrementalmerkletree::Altitude, a: &Self, b: &Self) -> Self {
+    fn combine(level: incrementalmerkletree::Level, a: &Self, b: &Self) -> Self {
         let layer = MERKLE_DEPTH - 1 - u8::from(level);
         Self(merkle_crh_orchard(layer, a.0, b.0))
     }
 
     /// Return the node for the level below the given level. (A quirk of the API)
-    fn empty_root(level: incrementalmerkletree::Altitude) -> Self {
+    fn empty_root(level: incrementalmerkletree::Level) -> Self {
         let layer_below = usize::from(MERKLE_DEPTH) - usize::from(level);
         Self(EMPTY_ROOTS[layer_below])
     }
@@ -265,6 +254,8 @@ pub enum NoteCommitmentTreeError {
 
 /// Orchard Incremental Note Commitment Tree
 #[derive(Debug, Serialize, Deserialize)]
+#[serde(into = "LegacyNoteCommitmentTree")]
+#[serde(from = "LegacyNoteCommitmentTree")]
 pub struct NoteCommitmentTree {
     /// The tree represented as a Frontier.
     ///
@@ -311,7 +302,7 @@ impl NoteCommitmentTree {
     /// Returns an error if the tree is full.
     #[allow(clippy::unwrap_in_result)]
     pub fn append(&mut self, cm_x: NoteCommitmentUpdate) -> Result<(), NoteCommitmentTreeError> {
-        if self.inner.append(&cm_x.into()) {
+        if self.inner.append(cm_x.into()) {
             // Invalidate cached root
             let cached_root = self
                 .cached_root
@@ -385,7 +376,9 @@ impl NoteCommitmentTree {
     ///
     /// For Orchard, the tree is capped at 2^32.
     pub fn count(&self) -> u64 {
-        self.inner.position().map_or(0, |pos| u64::from(pos) + 1)
+        self.inner
+            .value()
+            .map_or(0, |x| u64::from(x.position()) + 1)
     }
 
     /// Checks if the tree roots and inner data structures of `self` and `other` are equal.
@@ -459,7 +452,7 @@ impl From<Vec<pallas::Base>> for NoteCommitmentTree {
 /// A serialized Orchard note commitment tree.
 ///
 /// The format of the serialized data is compatible with
-/// [`CommitmentTree`](merkle_tree::CommitmentTree) from `librustzcash` and not
+/// [`CommitmentTree`](incrementalmerkletree::frontier::CommitmentTree) from `librustzcash` and not
 /// with [`Frontier`](bridgetree::Frontier) from the crate
 /// [`incrementalmerkletree`]. Zebra follows the former format in order to stay
 /// consistent with `zcashd` in RPCs. Note that [`NoteCommitmentTree`] itself is
@@ -468,7 +461,7 @@ impl From<Vec<pallas::Base>> for NoteCommitmentTree {
 /// The formats are semantically equivalent. The primary difference between them
 /// is that in [`Frontier`](bridgetree::Frontier), the vector of parents is
 /// dense (we know where the gaps are from the position of the leaf in the
-/// overall tree); whereas in [`CommitmentTree`](merkle_tree::CommitmentTree),
+/// overall tree); whereas in [`CommitmentTree`](incrementalmerkletree::frontier::CommitmentTree),
 /// the vector of parent hashes is sparse with [`None`] values in the gaps.
 ///
 /// The sparse format, used in this implementation, allows representing invalid
@@ -498,8 +491,9 @@ impl From<&NoteCommitmentTree> for SerializedTree {
         // Convert the note commitment tree from
         // [`Frontier`](bridgetree::Frontier) to
         // [`CommitmentTree`](merkle_tree::CommitmentTree).
-        let tree = CommitmentTree::from_frontier(&tree.inner);
-        tree.write(&mut serialized_tree)
+        let tree = incrementalmerkletree::frontier::CommitmentTree::from_frontier(&tree.inner);
+
+        write_commitment_tree(&tree, &mut serialized_tree)
             .expect("note commitment tree should be serializable");
         Self(serialized_tree)
     }

zebra-chain/src/orchard/tree/legacy.rs

@@ -0,0 +1,122 @@
+//! Orchard serialization legacy code.
+//!
+//! We create a [`LegacyNoteCommitmentTree`] which is a copy of [`NoteCommitmentTree`] but where serialization and
+//! deserialization can be derived.
+//! To do this we create a [`LegacyFrontier`] which is a legacy `Frontier` structure that can be found in [1],
+//! In order to make [`LegacyFrontier`] serializable we also have our own versions of `NonEmptyFrontier` ([`LegacyNonEmptyFrontier`]),
+//! `Leaf`([`LegacyLeaf`]) and `Position`([`LegacyPosition`]) that can be found in [1] or [2].
+//!
+//! Conversions methods to/from [`LegacyNoteCommitmentTree`] to/from [`NoteCommitmentTree`] are defined also in this file.
+//!
+//! [1]: https://github.com/zcash/incrementalmerkletree/blob/incrementalmerkletree-v0.3.1/src/bridgetree.rs
+//! [2]: https://github.com/zcash/incrementalmerkletree/blob/incrementalmerkletree-v0.3.1/src/lib.rs
+
+use incrementalmerkletree::{frontier::Frontier, Position};
+
+use super::{Node, NoteCommitmentTree, Root, MERKLE_DEPTH};
+
+/// A legacy version of [`NoteCommitmentTree`].
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename = "NoteCommitmentTree")]
+#[allow(missing_docs)]
+pub struct LegacyNoteCommitmentTree {
+    pub inner: LegacyFrontier<Node, MERKLE_DEPTH>,
+    cached_root: std::sync::RwLock<Option<Root>>,
+}
+
+impl From<NoteCommitmentTree> for LegacyNoteCommitmentTree {
+    fn from(nct: NoteCommitmentTree) -> Self {
+        LegacyNoteCommitmentTree {
+            inner: nct.inner.into(),
+            cached_root: nct.cached_root,
+        }
+    }
+}
+
+impl From<LegacyNoteCommitmentTree> for NoteCommitmentTree {
+    fn from(legacy_nct: LegacyNoteCommitmentTree) -> Self {
+        NoteCommitmentTree {
+            inner: legacy_nct.inner.into(),
+            cached_root: legacy_nct.cached_root,
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename = "Frontier")]
+#[allow(missing_docs)]
+pub struct LegacyFrontier<H, const DEPTH: u8> {
+    frontier: Option<LegacyNonEmptyFrontier<H>>,
+}
+
+impl From<LegacyFrontier<Node, MERKLE_DEPTH>> for Frontier<Node, MERKLE_DEPTH> {
+    fn from(legacy_frontier: LegacyFrontier<Node, MERKLE_DEPTH>) -> Self {
+        if let Some(legacy_frontier_data) = legacy_frontier.frontier {
+            let mut ommers = legacy_frontier_data.ommers;
+            let position = Position::from(
+                u64::try_from(legacy_frontier_data.position.0)
+                    .expect("old `usize` always fits in `u64`"),
+            );
+            let leaf = match legacy_frontier_data.leaf {
+                LegacyLeaf::Left(a) => a,
+                LegacyLeaf::Right(a, b) => {
+                    ommers.insert(0, a);
+                    b
+                }
+            };
+            Frontier::from_parts(
+                position,
+                leaf,
+                ommers,
+            )
+            .expect("We should be able to construct a frontier from parts given legacy frontier is not empty")
+        } else {
+            Frontier::empty()
+        }
+    }
+}
+
+impl From<Frontier<Node, MERKLE_DEPTH>> for LegacyFrontier<Node, MERKLE_DEPTH> {
+    fn from(frontier: Frontier<Node, MERKLE_DEPTH>) -> Self {
+        if let Some(frontier_data) = frontier.value() {
+            let leaf_from_frontier = *frontier_data.leaf();
+            let mut leaf = LegacyLeaf::Left(leaf_from_frontier);
+            let mut ommers = frontier_data.ommers().to_vec();
+            let position = usize::try_from(u64::from(frontier_data.position()))
+                .expect("new position should fit in a `usize`");
+            if frontier_data.position().is_odd() {
+                let left = ommers.remove(0);
+                leaf = LegacyLeaf::Right(left, leaf_from_frontier);
+            }
+            LegacyFrontier {
+                frontier: Some(LegacyNonEmptyFrontier {
+                    position: LegacyPosition(position),
+                    leaf,
+                    ommers: ommers.to_vec(),
+                }),
+            }
+        } else {
+            LegacyFrontier { frontier: None }
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename = "NonEmptyFrontier")]
+struct LegacyNonEmptyFrontier<H> {
+    position: LegacyPosition,
+    leaf: LegacyLeaf<H>,
+    ommers: Vec<H>,
+}
+
+/// A set of leaves of a Merkle tree.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename = "Leaf")]
+enum LegacyLeaf<A> {
+    Left(A),
+    Right(A, A),
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+#[repr(transparent)]
+struct LegacyPosition(usize);