add(tests): Add snapshot tests for sprout database formats

Cargo.lock

@@ -5881,6 +5881,7 @@ dependencies = [
  "once_cell",
  "proptest",
  "proptest-derive",
+ "rand 0.8.5",
  "rayon",
  "regex",
  "rlimit",

tower-batch-control/Cargo.toml

@@ -39,7 +39,7 @@ color-eyre = "0.6.2"
 tinyvec = { version = "1.6.0", features = ["rustc_1_55"] }
 
 ed25519-zebra = "4.0.0"
-rand = { version = "0.8.5", package = "rand" }
+rand = "0.8.5"
 
 tokio = { version = "1.28.2", features = ["full", "tracing", "test-util"] }
 tokio-test = "0.4.2"

zebra-chain/Cargo.toml

@@ -114,7 +114,7 @@ zcash_address = { version = "0.2.1", optional = true }
 proptest = { version = "1.2.0", optional = true }
 proptest-derive = { version = "0.3.0", optional = true }
 
-rand = { version = "0.8.5", optional = true, package = "rand" }
+rand = { version = "0.8.5", optional = true }
 rand_chacha = { version = "0.3.1", optional = true }
 
 tokio = { version = "1.28.2", features = ["tracing"], optional = true }
@@ -137,7 +137,7 @@ tracing = "0.1.37"
 proptest = "1.2.0"
 proptest-derive = "0.3.0"
 
-rand = { version = "0.8.5", package = "rand" }
+rand = "0.8.5"
 rand_chacha = "0.3.1"
 
 tokio = { version = "1.28.2", features = ["full", "tracing", "test-util"] }

zebra-consensus/Cargo.toml

@@ -40,7 +40,7 @@ bellman = "0.14.0"
 bls12_381 = "0.8.0"
 halo2 = { package = "halo2_proofs", version = "0.3.0" }
 jubjub = "0.10.0"
-rand = { version = "0.8.5", package = "rand" }
+rand = "0.8.5"
 rayon = "1.7.0"
 
 chrono = { version = "0.4.26", default-features = false, features = ["clock", "std"] }

zebra-network/Cargo.toml

@@ -53,7 +53,7 @@ lazy_static = "1.4.0"
 num-integer = "0.1.45"
 ordered-map = "0.4.2"
 pin-project = "1.1.0"
-rand = { version = "0.8.5", package = "rand" }
+rand = "0.8.5"
 rayon = "1.7.0"
 regex = "1.8.4"
 serde = { version = "1.0.164", features = ["serde_derive"] }

zebra-rpc/Cargo.toml

@@ -63,7 +63,7 @@ hex = { version = "0.4.3", features = ["serde"] }
 serde = { version = "1.0.164", features = ["serde_derive"] }
 
 # Experimental feature getblocktemplate-rpcs
-rand = { version = "0.8.5", package = "rand", optional = true }
+rand = { version = "0.8.5", optional = true }
 # ECC deps used by getblocktemplate-rpcs feature
 zcash_address = { version = "0.2.1", optional = true }
 

zebra-state/Cargo.toml

@@ -91,10 +91,11 @@ once_cell = "1.18.0"
 spandoc = "0.2.2"
 
 hex = { version = "0.4.3", features = ["serde"] }
-insta = { version = "1.30.0", features = ["ron"] }
+insta = { version = "1.30.0", features = ["ron", "redactions"] }
 
 proptest = "1.2.0"
 proptest-derive = "0.3.0"
+rand = "0.8.5"
 
 halo2 = { package = "halo2_proofs", version = "0.3.0" }
 jubjub = "0.10.0"

zebra-state/src/service/finalized_state/disk_db.rs

@@ -10,7 +10,14 @@
 //! The [`crate::constants::DATABASE_FORMAT_VERSION`] constant must
 //! be incremented each time the database format (column, serialization, etc) changes.
 
-use std::{cmp::Ordering, fmt::Debug, path::Path, sync::Arc};
+use std::{
+    cmp::Ordering,
+    collections::{BTreeMap, HashMap},
+    fmt::Debug,
+    ops::RangeBounds,
+    path::Path,
+    sync::Arc,
+};
 
 use itertools::Itertools;
 use rlimit::increase_nofile_limit;
@@ -146,6 +153,7 @@ impl WriteDisk for DiskWriteBatch {
 /// defined format
 //
 // TODO: just implement these methods directly on DiskDb
+//       move this trait, its methods, and support methods to another module
 pub trait ReadDisk {
     /// Returns true if a rocksdb column family `cf` does not contain any entries.
     fn zs_is_empty<C>(&self, cf: &C) -> bool
@@ -202,6 +210,26 @@ pub trait ReadDisk {
         C: rocksdb::AsColumnFamilyRef,
         K: IntoDisk + FromDisk,
         V: FromDisk;
+
+    /// Returns the keys and values in `cf` in `range`, in an ordered `BTreeMap`.
+    ///
+    /// Holding this iterator open might delay block commit transactions.
+    fn zs_items_in_range_ordered<C, K, V, R>(&self, cf: &C, range: R) -> BTreeMap<K, V>
+    where
+        C: rocksdb::AsColumnFamilyRef,
+        K: IntoDisk + FromDisk + Ord,
+        V: FromDisk,
+        R: RangeBounds<K>;
+
+    /// Returns the keys and values in `cf` in `range`, in an unordered `HashMap`.
+    ///
+    /// Holding this iterator open might delay block commit transactions.
+    fn zs_items_in_range_unordered<C, K, V, R>(&self, cf: &C, range: R) -> HashMap<K, V>
+    where
+        C: rocksdb::AsColumnFamilyRef,
+        K: IntoDisk + FromDisk + Eq + std::hash::Hash,
+        V: FromDisk,
+        R: RangeBounds<K>;
 }
 
 impl PartialEq for DiskDb {
@@ -342,6 +370,26 @@ impl ReadDisk for DiskDb {
             })
             .expect("unexpected database failure")
     }
+
+    fn zs_items_in_range_ordered<C, K, V, R>(&self, cf: &C, range: R) -> BTreeMap<K, V>
+    where
+        C: rocksdb::AsColumnFamilyRef,
+        K: IntoDisk + FromDisk + Ord,
+        V: FromDisk,
+        R: RangeBounds<K>,
+    {
+        self.zs_range_iter(cf, range).collect()
+    }
+
+    fn zs_items_in_range_unordered<C, K, V, R>(&self, cf: &C, range: R) -> HashMap<K, V>
+    where
+        C: rocksdb::AsColumnFamilyRef,
+        K: IntoDisk + FromDisk + Eq + std::hash::Hash,
+        V: FromDisk,
+        R: RangeBounds<K>,
+    {
+        self.zs_range_iter(cf, range).collect()
+    }
 }
 
 impl DiskWriteBatch {
@@ -366,6 +414,58 @@ impl DiskWriteBatch {
 }
 
 impl DiskDb {
+    /// Returns an iterator over the items in `cf` in `range`.
+    ///
+    /// Holding this iterator open might delay block commit transactions.
+    fn zs_range_iter<C, K, V, R>(&self, cf: &C, range: R) -> impl Iterator<Item = (K, V)> + '_
+    where
+        C: rocksdb::AsColumnFamilyRef,
+        K: IntoDisk + FromDisk,
+        V: FromDisk,
+        R: RangeBounds<K>,
+    {
+        use std::ops::Bound::{self, *};
+
+        // Replace with map() when it stabilises:
+        // https://github.com/rust-lang/rust/issues/86026
+        let map_to_vec = |bound: Bound<&K>| -> Bound<Vec<u8>> {
+            match bound {
+                Unbounded => Unbounded,
+                Included(x) => Included(x.as_bytes().as_ref().to_vec()),
+                Excluded(x) => Excluded(x.as_bytes().as_ref().to_vec()),
+            }
+        };
+
+        let start_bound = map_to_vec(range.start_bound());
+        let end_bound = map_to_vec(range.end_bound());
+        let range = (start_bound.clone(), end_bound);
+
+        let start_bound_vec =
+            if let Included(ref start_bound) | Excluded(ref start_bound) = start_bound {
+                start_bound.clone()
+            } else {
+                // Actually unused
+                Vec::new()
+            };
+
+        let start_mode = if matches!(start_bound, Unbounded) {
+            // Unbounded iterators start at the first item
+            rocksdb::IteratorMode::Start
+        } else {
+            rocksdb::IteratorMode::From(start_bound_vec.as_slice(), rocksdb::Direction::Forward)
+        };
+
+        // Reading multiple items from iterators has caused database hangs,
+        // in previous RocksDB versions
+        self.db
+            .iterator_cf(cf, start_mode)
+            .map(|result| result.expect("unexpected database failure"))
+            .map(|(key, value)| (key.to_vec(), value))
+            // Handle Excluded start and the end bound
+            .filter(move |(key, _value)| range.contains(key))
+            .map(|(key, value)| (K::from_bytes(key), V::from_bytes(value)))
+    }
+
     /// The ideal open file limit for Zebra
     const IDEAL_OPEN_FILE_LIMIT: u64 = 1024;
 

zebra-state/src/service/finalized_state/disk_format/shielded.rs

@@ -44,6 +44,13 @@ impl IntoDisk for sprout::tree::Root {
     }
 }
 
+impl FromDisk for sprout::tree::Root {
+    fn from_bytes(bytes: impl AsRef<[u8]>) -> Self {
+        let array: [u8; 32] = bytes.as_ref().try_into().unwrap();
+        array.into()
+    }
+}
+
 impl IntoDisk for sapling::tree::Root {
     type Bytes = [u8; 32];
 
@@ -52,6 +59,13 @@ impl IntoDisk for sapling::tree::Root {
     }
 }
 
+impl FromDisk for sapling::tree::Root {
+    fn from_bytes(bytes: impl AsRef<[u8]>) -> Self {
+        let array: [u8; 32] = bytes.as_ref().try_into().unwrap();
+        array.try_into().expect("finalized data must be valid")
+    }
+}
+
 impl IntoDisk for orchard::tree::Root {
     type Bytes = [u8; 32];
 
@@ -60,6 +74,13 @@ impl IntoDisk for orchard::tree::Root {
     }
 }
 
+impl FromDisk for orchard::tree::Root {
+    fn from_bytes(bytes: impl AsRef<[u8]>) -> Self {
+        let array: [u8; 32] = bytes.as_ref().try_into().unwrap();
+        array.try_into().expect("finalized data must be valid")
+    }
+}
+
 // The following implementations for the note commitment trees use `serde` and
 // `bincode` because currently the inner Merkle tree frontier (from
 // `incrementalmerkletree`) only supports `serde` for serialization. `bincode`

zebra-state/src/service/finalized_state/disk_format/tests/prop.rs

@@ -279,6 +279,13 @@ fn serialized_sprout_tree_root_equal() {
     );
 }
 
+#[test]
+fn roundtrip_sprout_tree_root() {
+    let _init_guard = zebra_test::init();
+
+    proptest!(|(val in any::<sprout::tree::Root>())| assert_value_properties(val));
+}
+
 // TODO: test note commitment tree round-trip, after implementing proptest::Arbitrary
 
 // Sapling
@@ -347,6 +354,13 @@ fn serialized_sapling_tree_root_equal() {
     );
 }
 
+#[test]
+fn roundtrip_sapling_tree_root() {
+    let _init_guard = zebra_test::init();
+
+    proptest!(|(val in any::<sapling::tree::Root>())| assert_value_properties(val));
+}
+
 // TODO: test note commitment tree round-trip, after implementing proptest::Arbitrary
 
 // Orchard
@@ -415,6 +429,13 @@ fn serialized_orchard_tree_root_equal() {
     );
 }
 
+#[test]
+fn roundtrip_orchard_tree_root() {
+    let _init_guard = zebra_test::init();
+
+    proptest!(|(val in any::<orchard::tree::Root>())| assert_value_properties(val));
+}
+
 // TODO: test note commitment tree round-trip, after implementing proptest::Arbitrary
 
 // Chain