Merge remote-tracking branch 'origin/staging' into feat/tracing-instrument

Author: kaimast

.circleci/config.yml

@@ -684,21 +684,21 @@ jobs:
 
   ledger-with-rocksdb-partition1:
     executor: rust-docker
-    resource_class: << pipeline.parameters.large >>
+    resource_class: << pipeline.parameters.xlarge >>
     steps:
       - run_test:
           workspace_member: snarkvm-ledger
-          flags: --release --features=rocks --partition count:1/2 -- --test-threads=8
+          flags: --release --features=rocks --partition count:1/2 -- --test-threads=10
           no_output_timeout: 20m
           timeout: 30m
 
   ledger-with-rocksdb-partition2:
     executor: rust-docker
-    resource_class: << pipeline.parameters.large >>
+    resource_class: << pipeline.parameters.xlarge >>
     steps:
       - run_test:
           workspace_member: snarkvm-ledger
-          flags: --release --features=rocks --partition count:2/2 -- --test-threads=8
+          flags: --release --features=rocks --partition count:2/2 -- --test-threads=10
           no_output_timeout: 20m
           timeout: 30m
 
@@ -1063,7 +1063,10 @@ jobs:
           name: Check for unused dependencies
           no_output_timeout: 10m
           command: |
-            cargo install cargo-machete@0.7.0
+            set -euo pipefail
+            if ! command -v cargo-machete >/dev/null 2>&1; then
+              cargo install cargo-machete@0.7.0 --locked
+            fi
             cargo machete
       - clear_environment:
           cache_key: v4.2.0-rust-1.88.0-machete-cache
@@ -1088,7 +1091,8 @@ jobs:
     executor: rust-docker
     resource_class: << pipeline.parameters.twoxlarge >>
     steps:
-      - checkout
+      - checkout:
+          method: full
       - setup_environment:
           cache_key: v4.2.0-rust-1.88.0-cargo-semver-checks-cache
       - run:

Cargo.lock

@@ -686,6 +686,11 @@ impl<E: PairingEngine> FromBytes for BatchLCProof<E> {
         CanonicalDeserialize::deserialize_compressed(&mut reader)
             .map_err(|err| into_io_error(anyhow::Error::from(err).context("could not deserialize struct")))
     }
+
+    fn read_le_unchecked<R: Read>(mut reader: R) -> io::Result<Self> {
+        CanonicalDeserialize::deserialize_compressed_unchecked(&mut reader)
+            .map_err(|err| into_io_error(anyhow::Error::from(err).context("could not deserialize struct")))
+    }
 }
 
 impl<E: PairingEngine> ToBytes for BatchLCProof<E> {

algorithms/src/polycommit/sonic_pc/data_structures.rs

@@ -44,4 +44,9 @@ impl<E: PairingEngine> FromBytes for Certificate<E> {
         Self::deserialize_compressed(&mut r)
             .map_err(|err| into_io_error(anyhow::Error::from(err).context("Failed to deserialize certificate")))
     }
+
+    fn read_le_unchecked<R: Read>(mut r: R) -> io::Result<Self> {
+        Self::deserialize_compressed_unchecked(&mut r)
+            .map_err(|err| into_io_error(anyhow::Error::from(err).context("Failed to deserialize certificate")))
+    }
 }

algorithms/src/snark/varuna/data_structures/certificate.rs

@@ -43,6 +43,11 @@ impl<E: PairingEngine> FromBytes for CircuitVerifyingKey<E> {
         Self::deserialize_compressed(r)
             .map_err(|err| into_io_error(anyhow::Error::from(err).context("could not deserialize CircuitVerifyingKey")))
     }
+
+    fn read_le_unchecked<R: Read>(r: R) -> io::Result<Self> {
+        Self::deserialize_compressed_unchecked(r)
+            .map_err(|err| into_io_error(anyhow::Error::from(err).context("could not deserialize CircuitVerifyingKey")))
+    }
 }
 
 impl<E: PairingEngine> ToBytes for CircuitVerifyingKey<E> {

algorithms/src/snark/varuna/data_structures/circuit_verifying_key.rs

@@ -379,6 +379,11 @@ impl<E: PairingEngine> FromBytes for Proof<E> {
         Self::deserialize_compressed(&mut r)
             .map_err(|err| into_io_error(anyhow::Error::from(err).context("could not deserialize Proof")))
     }
+
+    fn read_le_unchecked<R: Read>(mut r: R) -> io::Result<Self> {
+        Self::deserialize_compressed_unchecked(&mut r)
+            .map_err(|err| into_io_error(anyhow::Error::from(err).context("could not deserialize Proof")))
+    }
 }
 
 /// Computes the size in bytes of a Varuna proof as produced by

algorithms/src/snark/varuna/data_structures/proof.rs

@@ -34,7 +34,7 @@ use console::{
         Error,
         Formatter,
         FromBytes,
-        FromBytesDeserializer,
+        FromBytesUncheckedDeserializer,
         FromStr,
         IoResult,
         Read,

ledger/authority/src/lib.rs

@@ -59,7 +59,7 @@ impl<'de, N: Network> Deserialize<'de> for Authority<N> {
                     _ => Err(de::Error::custom(error("Invalid authority type"))),
                 }
             }
-            false => FromBytesDeserializer::<Self>::deserialize_with_size_encoding(deserializer, "authority"),
+            false => FromBytesUncheckedDeserializer::<Self>::deserialize_with_size_encoding(deserializer, "authority"),
         }
     }
 }

ledger/authority/src/serialize.rs

@@ -224,7 +224,7 @@ impl<N: Network> RestQuery<N> {
             .with_context(|| format!("Failed to fetch from {endpoint}"))?;
 
         if response.status().is_success() {
-            response.body_mut().read_json().with_context(|| "Failed to parse JSON response")
+            response.body_mut().read_json().with_context(|| format!("Failed to parse JSON response from {endpoint}"))
         } else {
             let content_type = response
                 .headers()
@@ -236,11 +236,16 @@ impl<N: Network> RestQuery<N> {
             // Convert returned error into an `anyhow::Error`.
             // Depending on the API version, the error is either encoded as a string or as a JSON.
             if content_type.contains("json") {
-                let error: RestError =
-                    response.body_mut().read_json().with_context(|| "Failed to parse JSON error response")?;
+                let error: RestError = response
+                    .body_mut()
+                    .read_json()
+                    .with_context(|| format!("Failed to parse JSON error response from {endpoint}"))?;
                 Err(error.parse().context(format!("Failed to fetch from {endpoint}")))
             } else {
-                let error = response.body_mut().read_to_string().with_context(|| "Failed to read error message")?;
+                let error = response
+                    .body_mut()
+                    .read_to_string()
+                    .with_context(|| format!("Failed to read error message {endpoint}"))?;
                 Err(anyhow!(error).context(format!("Failed to fetch from {endpoint}")))
             }
         }
@@ -256,10 +261,13 @@ impl<N: Network> RestQuery<N> {
         let response = reqwest::get(&endpoint).await.with_context(|| format!("Failed to fetch from {endpoint}"))?;
 
         if response.status().is_success() {
-            response.json().await.with_context(|| "Failed to parse JSON response")
+            response.json().await.with_context(|| format!("Failed to parse JSON response from {endpoint}"))
         } else {
             // Convert returned error into an `anyhow::Error`.
-            let error: RestError = response.json().await.with_context(|| "Failed to parse JSON error response")?;
+            let error: RestError = response
+                .json()
+                .await
+                .with_context(|| format!("Failed to parse JSON error response from {endpoint}"))?;
             Err(error.parse().context(format!("Failed to fetch from {endpoint}")))
         }
     }

ledger/query/src/query/rest.rs

@@ -68,12 +68,31 @@ pub struct TestChainBuilder<N: Network> {
 }
 
 /// Additional options you can pass to the builder when generating a set of blocks.
-#[derive(Clone, Default)]
-pub struct GenerateBlocksOptions {
+#[derive(Clone)]
+pub struct GenerateBlocksOptions<N: Network> {
     /// Do not include votes to the previous leader certificate
     pub skip_votes: bool,
     /// Do not generate certificates for the specific node indices (to simulate a partition).
     pub skip_nodes: Vec<usize>,
+    /// A flag indicating that a number of initial "placeholder blocks" should be baked
+    /// wthout transactions in order to skip to the latest version of consensus.
+    pub skip_to_current_version: bool,
+    /// The number of validators.
+    pub num_validators: usize,
+    /// Preloaded transactions to populate the blocks with.
+    pub transactions: Vec<Transaction<N>>,
+}
+
+impl<N: Network> Default for GenerateBlocksOptions<N> {
+    fn default() -> Self {
+        Self {
+            skip_votes: false,
+            skip_nodes: Default::default(),
+            skip_to_current_version: false,
+            num_validators: 0,
+            transactions: Default::default(),
+        }
+    }
 }
 
 /// Additional options you can pass to the builder when generating a single block.
@@ -178,7 +197,6 @@ impl<N: Network> TestChainBuilder<N> {
         Ok(Self {
             private_keys,
             ledger,
-
             genesis_block,
             last_batch_round: Default::default(),
             last_committed_batch_round: Default::default(),
@@ -190,7 +208,9 @@ impl<N: Network> TestChainBuilder<N> {
 
     /// Create multiple blocks, with fully-connected DAGs.
     pub fn generate_blocks(&mut self, num_blocks: usize, rng: &mut TestRng) -> Result<Vec<Block<N>>> {
-        self.generate_blocks_with_opts(num_blocks, GenerateBlocksOptions::default(), rng)
+        let num_validators = self.private_keys.len();
+
+        self.generate_blocks_with_opts(num_blocks, GenerateBlocksOptions { num_validators, ..Default::default() }, rng)
     }
 
     /// Create multiple blocks, with additional parameters.
@@ -200,20 +220,53 @@ impl<N: Network> TestChainBuilder<N> {
     pub fn generate_blocks_with_opts(
         &mut self,
         num_blocks: usize,
-        options: GenerateBlocksOptions,
+        mut options: GenerateBlocksOptions<N>,
         rng: &mut TestRng,
     ) -> Result<Vec<Block<N>>> {
         assert!(num_blocks > 0, "Need to build at least one block");
 
-        let options = GenerateBlockOptions {
-            skip_votes: options.skip_votes,
-            skip_nodes: options.skip_nodes,
-            ..Default::default()
-        };
-
         let mut result = vec![];
-        for _ in 0..num_blocks {
-            let block = self.generate_block_with_opts(options.clone(), rng)?;
+
+        // If configured, skip enough blocks to reach the current consensus version.
+        if options.skip_to_current_version {
+            let (version, target_height) = TEST_CONSENSUS_VERSION_HEIGHTS.last().unwrap();
+            let mut current_height = self.ledger.latest_height();
+
+            let diff = target_height.saturating_sub(current_height);
+
+            if diff > 0 {
+                println!("Skipping {diff} blocks to reach {version}");
+
+                while current_height < *target_height && result.len() < num_blocks {
+                    let options = GenerateBlockOptions {
+                        skip_votes: options.skip_votes,
+                        skip_nodes: options.skip_nodes.clone(),
+                        ..Default::default()
+                    };
+
+                    let block = self.generate_block_with_opts(options, rng)?;
+                    current_height = block.height();
+                    result.push(block);
+                }
+
+                println!("Advanced to the current consensus version at height {target_height}");
+            } else {
+                debug!("Already at the current consensus version. No blocks to skip.");
+            }
+        }
+
+        while result.len() < num_blocks {
+            let num_txs = (BatchHeader::<N>::MAX_TRANSMISSIONS_PER_BATCH * options.num_validators)
+                .min(options.transactions.len());
+
+            let options = GenerateBlockOptions {
+                skip_votes: options.skip_votes,
+                skip_nodes: options.skip_nodes.clone(),
+                transactions: options.transactions.drain(..num_txs).collect(),
+                ..Default::default()
+            };
+
+            let block = self.generate_block_with_opts(options, rng)?;
             result.push(block);
         }
 
@@ -262,8 +315,6 @@ impl<N: Network> TestChainBuilder<N> {
             transmissions.insert(transmission_id, transmission);
         }
 
-        let transmission_ids: IndexSet<_> = transmissions.keys().copied().collect();
-
         // =======================================
         // Create certificates for the new block.
         // =======================================
@@ -303,6 +354,13 @@ impl<N: Network> TestChainBuilder<N> {
                     continue;
                 }
 
+                let transmission_ids: IndexSet<_> = transmissions
+                    .keys()
+                    .skip(key1_idx * BatchHeader::<N>::MAX_TRANSMISSIONS_PER_BATCH)
+                    .take(BatchHeader::<N>::MAX_TRANSMISSIONS_PER_BATCH)
+                    .copied()
+                    .collect();
+
                 let batch_header = BatchHeader::new(
                     private_key_1,
                     round,
@@ -404,6 +462,7 @@ impl<N: Network> TestChainBuilder<N> {
 
         // Construct the block.
         let subdag = Subdag::from(subdag_map).unwrap();
+
         let block = self.ledger.prepare_advance_to_next_quorum_block(subdag, transmissions, rng)?;
 
         // Skip to increase performance.