feat: add a transaction mirror binary (#7183)

This adds code that mirrors traffic from a source chain (e.g. mainnet
or testnet) to a test chain with genesis state forked from the source
chain. The goal is to produce traffic that looks like source chain
traffic. So in a mocknet test where we fork mainnet state for example,
we can then actually observe what happens when we subsequently get
traffic equivalent to mainnet traffic after the fork point. For more
info, see the README in this commit.

Cargo.toml

@@ -45,6 +45,7 @@ members = [
     "tools/chainsync-loadtest",
     "tools/delay-detector",
     "tools/indexer/example",
+    "tools/mirror",
     "tools/mock-node",
     "tools/restaked",
     "tools/rpctypegen/core",
@@ -111,6 +112,7 @@ fs2 = "0.4"
 futures = "0.3.5"
 futures-util = "0.3"
 hex = { version = "0.4.2", features = ["serde"] }
+hkdf = "0.12.3"
 hyper = { version = "0.14", features = ["full"] }
 hyper-tls = "0.5.0"
 im = "15"
@@ -148,6 +150,7 @@ protobuf-codegen = "3.0.1"
 quote = "1.0"
 rand = "0.8.5"
 rand_chacha = "0.3.1"
+rand_core = "0.5"
 rand_hc = "0.3.1"
 rand_xorshift = "0.3"
 rayon = "1.5"

core/chain-configs/src/genesis_config.rs

@@ -393,7 +393,7 @@ impl<'de, F: FnMut(StateRecord)> DeserializeSeed<'de> for RecordsProcessor<&'_ m
     }
 }
 
-fn stream_records_from_file(
+pub fn stream_records_from_file(
     reader: impl Read,
     mut callback: impl FnMut(StateRecord),
 ) -> serde_json::Result<()> {

core/chain-configs/src/lib.rs

@@ -7,6 +7,6 @@ pub use client_config::{
     MIN_GC_NUM_EPOCHS_TO_KEEP, TEST_STATE_SYNC_TIMEOUT,
 };
 pub use genesis_config::{
-    get_initial_supply, Genesis, GenesisChangeConfig, GenesisConfig, GenesisRecords,
-    GenesisValidationMode, ProtocolConfig, ProtocolConfigView,
+    get_initial_supply, stream_records_from_file, Genesis, GenesisChangeConfig, GenesisConfig,
+    GenesisRecords, GenesisValidationMode, ProtocolConfig, ProtocolConfigView,
 };

neard/Cargo.toml

@@ -34,6 +34,7 @@ tracing.workspace = true
 nearcore = { path = "../nearcore" }
 near-chain-configs = { path = "../core/chain-configs" }
 near-jsonrpc-primitives = { path = "../chain/jsonrpc-primitives" }
+near-mirror = { path = "../tools/mirror" }
 near-primitives = { path = "../core/primitives" }
 near-performance-metrics = { path = "../utils/near-performance-metrics" }
 near-state-viewer = { path = "../tools/state-viewer", package = "state-viewer" }

neard/src/cli.rs

@@ -3,6 +3,7 @@ use anyhow::Context;
 use clap::{Args, Parser};
 use near_chain_configs::GenesisValidationMode;
 use near_jsonrpc_primitives::types::light_client::RpcLightClientExecutionProofResponse;
+use near_mirror::MirrorCommand;
 use near_o11y::tracing_subscriber::EnvFilter;
 use near_o11y::{
     default_subscriber, default_subscriber_with_opentelemetry, BuildEnvFilterError,
@@ -93,6 +94,9 @@ impl NeardCmd {
             NeardSubCommand::VerifyProof(cmd) => {
                 cmd.run();
             }
+            NeardSubCommand::Mirror(cmd) => {
+                cmd.run()?;
+            }
         };
         Ok(())
     }
@@ -104,6 +108,8 @@ pub(crate) enum RunError {
     EnvFilter(#[source] BuildEnvFilterError),
     #[error("could not install a rayon thread pool")]
     RayonInstall(#[source] rayon::ThreadPoolBuildError),
+    #[error(transparent)]
+    Other(#[from] anyhow::Error),
 }
 
 #[derive(Parser)]
@@ -189,6 +195,10 @@ pub(super) enum NeardSubCommand {
     /// Verify proofs
     #[clap(alias = "verify_proof")]
     VerifyProof(VerifyProofSubCommand),
+
+    /// Mirror transactions from a source chain to a test chain with state forked
+    /// from it, reproducing traffic and state as closely as possible.
+    Mirror(MirrorCommand),
 }
 
 #[derive(Parser)]

pytest/lib/key.py

@@ -17,6 +17,13 @@ def __init__(self, account_id: str, pk: str, sk: str) -> None:
         self.pk = pk
         self.sk = sk
 
+    @classmethod
+    def from_random(cls, account_id: str) -> 'Key':
+        keys = ed25519.create_keypair(entropy=os.urandom)
+        sk = 'ed25519:' + base58.b58encode(keys[0].to_bytes()).decode('ascii')
+        pk = 'ed25519:' + base58.b58encode(keys[1].to_bytes()).decode('ascii')
+        return cls(account_id, pk, sk)
+
     @classmethod
     def implicit_account(cls) -> 'Key':
         keys = ed25519.create_keypair(entropy=os.urandom)