From 86e468f3b0df43e67574aec8411a0e05d65c4ae7 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 8 Nov 2023 19:05:51 -0300 Subject: [PATCH] poc(scanner): get started with the blockchain scanner proof of concept (#7758) * get started with the blockchain scanner poc * rustfmt * fix the tests * Reads blocks from db * Adds conversion functions * scans blocks and counts transactions * fix bug * split into 2 tests * add duplicated dependencies to deny.toml * upgrade zebra-scanner version * try removing ecc duplicated dependencies * try fix deny.toml * remove unintended paste from deny.toml * remove duplicated code from the other test * remove strict version of `zcash_primitives` crate * change description * remove feture * remove tokio features * change lib doc * add more documentation * change expect * do not use default in compact block creation * more docs * add more checks to test * remove zebra-consensus dependency * move all deps to dev-deps * change crate version * rename crate to zebra-scan * lock file * ifix cargo.lock * remove internal dev dependencies versions Co-authored-by: teor * fix docs url Co-authored-by: teor * fix expect messages Co-authored-by: teor * remove duplicated in deny.toml Co-authored-by: teor * add a comment about moving code to production --------- Co-authored-by: arya2 Co-authored-by: teor --- Cargo.lock | 64 +++++++ Cargo.toml | 1 + zebra-scan/Cargo.toml | 39 +++++ zebra-scan/src/lib.rs | 8 + zebra-scan/src/tests.rs | 373 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 485 insertions(+) create mode 100644 zebra-scan/Cargo.toml create mode 100644 zebra-scan/src/lib.rs create mode 100644 zebra-scan/src/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 9a71e6f0eb9..d1e05d657d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4018,6 +4018,18 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shardtree" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c19f96dde3a8693874f7e7c53d95616569b4009379a903789efbd448f4ea9cc7" +dependencies = [ + "bitflags 2.4.1", + "either", + "incrementalmerkletree", + "tracing", +] + [[package]] name = "shlex" version = "1.2.0" @@ -5413,6 +5425,39 @@ dependencies = [ "zcash_encoding", ] +[[package]] +name = "zcash_client_backend" +version = "0.10.0-rc.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc33f71747a93d509f7e1c047961e359a271bdf4869cc07f7f65ee1ba7df8c2" +dependencies = [ + "base64 0.21.5", + "bech32", + "bls12_381", + "bs58", + "crossbeam-channel", + "group", + "hex", + "incrementalmerkletree", + "memuse", + "nom", + "orchard", + "percent-encoding", + "prost", + "rayon", + "secrecy", + "shardtree", + "subtle", + "time", + "tonic-build", + "tracing", + "which", + "zcash_address", + "zcash_encoding", + "zcash_note_encryption", + "zcash_primitives", +] + [[package]] name = "zcash_encoding" version = "0.2.0" @@ -5733,6 +5778,25 @@ dependencies = [ "zebra-test", ] +[[package]] +name = "zebra-scan" +version = "0.1.0-alpha.0" +dependencies = [ + "bls12_381", + "color-eyre", + "ff", + "group", + "jubjub", + "rand 0.8.5", + "tokio", + "zcash_client_backend", + "zcash_note_encryption", + "zcash_primitives", + "zebra-chain", + "zebra-state", + "zebra-test", +] + [[package]] name = "zebra-script" version = "1.0.0-beta.31" diff --git a/Cargo.toml b/Cargo.toml index d9589c4353f..e2c5e03372a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "zebra-node-services", "zebra-test", "zebra-utils", + "zebra-scan", "tower-batch-control", "tower-fallback", ] diff --git a/zebra-scan/Cargo.toml b/zebra-scan/Cargo.toml new file mode 100644 index 00000000000..ae8df1d4667 --- /dev/null +++ b/zebra-scan/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "zebra-scan" +version = "0.1.0-alpha.0" +authors = ["Zcash Foundation "] +description = "Shielded transaction scanner for the Zcash blockchain" +license = "MIT OR Apache-2.0" +repository = "https://github.com/ZcashFoundation/zebra" +edition = "2021" + +readme = "../README.md" +homepage = "https://zfnd.org/zebra/" +# crates.io is limited to 5 keywords and categories +keywords = ["zebra", "zcash"] +# Must be one of +categories = ["cryptography::cryptocurrencies"] + +[features] + +# Production features that activate extra dependencies, or extra features in dependencies + +[dependencies] + +[dev-dependencies] + +zcash_client_backend = "0.10.0-rc.1" +zcash_primitives = "0.13.0-rc.1" +zcash_note_encryption = "0.4.0" + +color-eyre = { version = "0.6.2" } +rand = "0.8.5" +bls12_381 = "0.8.0" +jubjub = "0.10.0" +ff = "0.13.0" +group = "0.13.0" +tokio = { version = "1.33.0", features = ["test-util"] } + +zebra-state = { path = "../zebra-state" } +zebra-chain = { path = "../zebra-chain" } +zebra-test = { path = "../zebra-test" } diff --git a/zebra-scan/src/lib.rs b/zebra-scan/src/lib.rs new file mode 100644 index 00000000000..47a569e8ea3 --- /dev/null +++ b/zebra-scan/src/lib.rs @@ -0,0 +1,8 @@ +//! Shielded transaction scanner for the Zcash blockchain. + +#![doc(html_favicon_url = "https://zfnd.org/wp-content/uploads/2022/03/zebra-favicon-128.png")] +#![doc(html_logo_url = "https://zfnd.org/wp-content/uploads/2022/03/zebra-icon.png")] +#![doc(html_root_url = "https://docs.rs/zebra_scan")] + +#[cfg(test)] +mod tests; diff --git a/zebra-scan/src/tests.rs b/zebra-scan/src/tests.rs new file mode 100644 index 00000000000..19069e7316f --- /dev/null +++ b/zebra-scan/src/tests.rs @@ -0,0 +1,373 @@ +//! Test that we can scan the Zebra blockchain using the external `zcash_client_backend` crate +//! scanning functionality. +//! +//! This tests belong to the proof of concept stage of the external wallet support functionality. + +use std::sync::Arc; + +use zcash_client_backend::{ + proto::compact_formats::{ + self as compact, ChainMetadata, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, + CompactTx, + }, + scanning::scan_block, +}; +use zcash_note_encryption::Domain; +use zcash_primitives::{ + block::BlockHash, + consensus::{BlockHeight, Network}, + constants::SPENDING_KEY_GENERATOR, + memo::MemoBytes, + sapling::{ + note_encryption::{sapling_note_encryption, SaplingDomain}, + util::generate_random_rseed, + value::NoteValue, + Note, Nullifier, SaplingIvk, + }, + zip32::{AccountId, DiversifiableFullViewingKey, ExtendedSpendingKey}, +}; + +use color_eyre::Result; + +use rand::{rngs::OsRng, RngCore}; + +use ff::{Field, PrimeField}; +use group::GroupEncoding; +use zebra_chain::{ + block::Block, + chain_tip::ChainTip, + serialization::{ZcashDeserializeInto, ZcashSerialize}, + transaction::Transaction, +}; + +/// Prove that Zebra blocks can be scanned using the `zcash_client_backend::scanning::scan_block` function: +/// - Populates the state with a continuous chain of mainnet blocks from genesis. +/// - Scan the chain from the tip going backwards down to genesis. +/// - Verifies that no relevant transaction is found in the chain when scanning for a fake account's nullifier. +#[tokio::test] +async fn scanning_from_populated_zebra_state() -> Result<()> { + let account = AccountId::from(12); + let vks: Vec<(&AccountId, &SaplingIvk)> = vec![]; + let nf = Nullifier([7; 32]); + + let network = zebra_chain::parameters::Network::default(); + + // Create a continuous chain of mainnet blocks from genesis + let blocks: Vec> = zebra_test::vectors::CONTINUOUS_MAINNET_BLOCKS + .iter() + .map(|(_height, block_bytes)| block_bytes.zcash_deserialize_into().unwrap()) + .collect(); + + // Create a populated state service. + let (_state_service, read_only_state_service, latest_chain_tip, _chain_tip_change) = + zebra_state::populated_state(blocks.clone(), network).await; + + let db = read_only_state_service.db(); + + // use the tip as starting height + let mut height = latest_chain_tip.best_tip_height().unwrap(); + + let mut transactions_found = 0; + let mut transactions_scanned = 0; + let mut blocks_scanned = 0; + // TODO: Accessing the state database directly is ok in the tests, but not in production code. + // Use `Request::Block` if the code is copied to production. + while let Some(block) = db.block(height.into()) { + // We fake the sapling tree size to 1 because we are not in Sapling heights. + let sapling_tree_size = 1; + let orchard_tree_size = db + .orchard_tree_by_hash_or_height(height.into()) + .expect("each state block must have a sapling tree") + .count(); + + let chain_metadata = ChainMetadata { + sapling_commitment_tree_size: sapling_tree_size + .try_into() + .expect("sapling position is limited to u32::MAX"), + orchard_commitment_tree_size: orchard_tree_size + .try_into() + .expect("orchard position is limited to u32::MAX"), + }; + + let compact_block = block_to_compact(block, chain_metadata); + + let res = scan_block( + &zcash_primitives::consensus::MainNetwork, + compact_block.clone(), + &vks[..], + &[(account, nf)], + None, + ) + .unwrap(); + + transactions_found += res.transactions().len(); + transactions_scanned += compact_block.vtx.len(); + blocks_scanned += 1; + + // scan backwards + if height.is_min() { + break; + } + height = height.previous()?; + } + + // make sure all blocks and transactions were scanned + assert_eq!(blocks_scanned, 11); + assert_eq!(transactions_scanned, 11); + + // no relevant transactions should be found + assert_eq!(transactions_found, 0); + + Ok(()) +} + +/// Prove that we can create fake blocks with fake notes and scan them using the +/// `zcash_client_backend::scanning::scan_block` function: +/// - Function `fake_compact_block` will generate 1 block with one pre created fake nullifier in +/// the transaction and one additional random transaction without it. +/// - Verify one relevant transaction is found in the chain when scanning for the pre created fake +/// account's nullifier. +#[tokio::test] +async fn scanning_from_fake_generated_blocks() -> Result<()> { + let account = AccountId::from(12); + let extsk = ExtendedSpendingKey::master(&[]); + let dfvk: DiversifiableFullViewingKey = extsk.to_diversifiable_full_viewing_key(); + let vks: Vec<(&AccountId, &SaplingIvk)> = vec![]; + let nf = Nullifier([7; 32]); + + let cb = fake_compact_block( + 1u32.into(), + BlockHash([0; 32]), + nf, + &dfvk, + 1, + false, + Some(0), + ); + + // The fake block function will have our transaction and a random one. + assert_eq!(cb.vtx.len(), 2); + + let res = scan_block( + &zcash_primitives::consensus::MainNetwork, + cb.clone(), + &vks[..], + &[(account, nf)], + None, + ) + .unwrap(); + + // The response should have one transaction relevant to the key we provided. + assert_eq!(res.transactions().len(), 1); + // The transaction should be the one we provided, second one in the block. + // (random transaction is added before ours in `fake_compact_block` function) + assert_eq!(res.transactions()[0].txid, cb.vtx[1].txid()); + // The block hash of the response should be the same as the one provided. + assert_eq!(res.block_hash(), cb.hash()); + + Ok(()) +} + +/// Convert a zebra block and meta data into a compact block. +fn block_to_compact(block: Arc, chain_metadata: ChainMetadata) -> CompactBlock { + CompactBlock { + height: block + .coinbase_height() + .expect("verified block should have a valid height") + .0 + .into(), + hash: block.hash().bytes_in_display_order().to_vec(), + prev_hash: block + .header + .previous_block_hash + .bytes_in_display_order() + .to_vec(), + time: block + .header + .time + .timestamp() + .try_into() + .expect("unsigned 32-bit times should work until 2105"), + header: block + .header + .zcash_serialize_to_vec() + .expect("verified block should serialize"), + vtx: block + .transactions + .iter() + .cloned() + .enumerate() + .map(transaction_to_compact) + .collect(), + chain_metadata: Some(chain_metadata), + + // The protocol version is used for the gRPC wire format, so it isn't needed here. + proto_version: 0, + } +} + +/// Convert a zebra transaction into a compact transaction. +fn transaction_to_compact((index, tx): (usize, Arc)) -> CompactTx { + CompactTx { + index: index + .try_into() + .expect("tx index in block should fit in u64"), + hash: tx.hash().bytes_in_display_order().to_vec(), + + // `fee` is not checked by the `scan_block` function. It is allowed to be unset. + // + fee: 0, + + spends: tx + .sapling_nullifiers() + .map(|nf| CompactSaplingSpend { + nf: <[u8; 32]>::from(*nf).to_vec(), + }) + .collect(), + + // > output encodes the cmu field, ephemeralKey field, and a 52-byte prefix of the encCiphertext field of a Sapling Output + // + // + outputs: tx + .sapling_outputs() + .map(|output| CompactSaplingOutput { + cmu: output.cm_u.to_bytes().to_vec(), + ephemeral_key: output + .ephemeral_key + .zcash_serialize_to_vec() + .expect("verified output should serialize successfully"), + ciphertext: output + .enc_ciphertext + .zcash_serialize_to_vec() + .expect("verified output should serialize successfully") + .into_iter() + .take(52) + .collect(), + }) + .collect(), + + // `actions` is not checked by the `scan_block` function. + actions: vec![], + } +} + +/// Create a fake compact block with provided fake account data. +// This is a copy of zcash_primitives `fake_compact_block` where the `value` argument was changed to +// be a number for easier conversion: +// https://github.com/zcash/librustzcash/blob/zcash_primitives-0.13.0/zcash_client_backend/src/scanning.rs#L635 +// We need to copy because this is a test private function upstream. +fn fake_compact_block( + height: BlockHeight, + prev_hash: BlockHash, + nf: Nullifier, + dfvk: &DiversifiableFullViewingKey, + value: u64, + tx_after: bool, + initial_sapling_tree_size: Option, +) -> CompactBlock { + let to = dfvk.default_address().1; + + // Create a fake Note for the account + let mut rng = OsRng; + let rseed = generate_random_rseed(&Network::TestNetwork, height, &mut rng); + let note = Note::from_parts(to, NoteValue::from_raw(value), rseed); + let encryptor = sapling_note_encryption::<_, Network>( + Some(dfvk.fvk().ovk), + note.clone(), + MemoBytes::empty(), + &mut rng, + ); + let cmu = note.cmu().to_bytes().to_vec(); + let ephemeral_key = SaplingDomain::::epk_bytes(encryptor.epk()) + .0 + .to_vec(); + let enc_ciphertext = encryptor.encrypt_note_plaintext(); + + // Create a fake CompactBlock containing the note + let mut cb = CompactBlock { + hash: { + let mut hash = vec![0; 32]; + rng.fill_bytes(&mut hash); + hash + }, + prev_hash: prev_hash.0.to_vec(), + height: height.into(), + ..Default::default() + }; + + // Add a random Sapling tx before ours + { + let mut tx = random_compact_tx(&mut rng); + tx.index = cb.vtx.len() as u64; + cb.vtx.push(tx); + } + + let cspend = CompactSaplingSpend { nf: nf.0.to_vec() }; + let cout = CompactSaplingOutput { + cmu, + ephemeral_key, + ciphertext: enc_ciphertext.as_ref()[..52].to_vec(), + }; + let mut ctx = CompactTx::default(); + let mut txid = vec![0; 32]; + rng.fill_bytes(&mut txid); + ctx.hash = txid; + ctx.spends.push(cspend); + ctx.outputs.push(cout); + ctx.index = cb.vtx.len() as u64; + cb.vtx.push(ctx); + + // Optionally add another random Sapling tx after ours + if tx_after { + let mut tx = random_compact_tx(&mut rng); + tx.index = cb.vtx.len() as u64; + cb.vtx.push(tx); + } + + cb.chain_metadata = initial_sapling_tree_size.map(|s| compact::ChainMetadata { + sapling_commitment_tree_size: s + cb + .vtx + .iter() + .map(|tx| tx.outputs.len() as u32) + .sum::(), + ..Default::default() + }); + + cb +} + +/// Create a random compact transaction. +// This is an exact copy of `zcash_client_backend::scanning::random_compact_tx`: +// https://github.com/zcash/librustzcash/blob/zcash_primitives-0.13.0/zcash_client_backend/src/scanning.rs#L597 +// We need to copy because this is a test private function upstream. +fn random_compact_tx(mut rng: impl RngCore) -> CompactTx { + let fake_nf = { + let mut nf = vec![0; 32]; + rng.fill_bytes(&mut nf); + nf + }; + let fake_cmu = { + let fake_cmu = bls12_381::Scalar::random(&mut rng); + fake_cmu.to_repr().as_ref().to_owned() + }; + let fake_epk = { + let mut buffer = [0; 64]; + rng.fill_bytes(&mut buffer); + let fake_esk = jubjub::Fr::from_bytes_wide(&buffer); + let fake_epk = SPENDING_KEY_GENERATOR * fake_esk; + fake_epk.to_bytes().to_vec() + }; + let cspend = CompactSaplingSpend { nf: fake_nf }; + let cout = CompactSaplingOutput { + cmu: fake_cmu, + ephemeral_key: fake_epk, + ciphertext: vec![0; 52], + }; + let mut ctx = CompactTx::default(); + let mut txid = vec![0; 32]; + rng.fill_bytes(&mut txid); + ctx.hash = txid; + ctx.spends.push(cspend); + ctx.outputs.push(cout); + ctx +}