[sui-test-validator] Implementing a persisted state that takes in net…

…work config (MystenLabs#12451)

## Description 

This PR adds the option to run `sui-test-validator` with a config
directory passed in.

This is a rewrite of
MystenLabs@f8477e8
to use builder patterns in the network builders instead so that the code
is cleaner and has less duplication.

In this current implementation I added an extra wallet account that
populates in the network config so a faucet can be spun up from that
config.

1. Generate a config to store db and genesis configs `sui genesis -f
--with-faucet --working-dir=[some-directory]`
2. `sui-test-validator --config-dir [some-directory]`

## Test Plan 

How did you test the new or updated feature?
local run
---
If your changes are not user-facing and not a breaking change, you can
skip the following section. Otherwise, please indicate what changed, and
then add to the Release Notes section as highlighted during the release
process.

### Type of Change (Check all that apply)

- [ ] protocol change
- [ ] user-visible impact
- [ ] breaking change for a client SDKs
- [ ] breaking change for FNs (FN binary must upgrade)
- [ ] breaking change for validators or node operators (must upgrade
binaries)
- [ ] breaking change for on-chain data layout
- [ ] necessitate either a data wipe or data migration

### Release notes

crates/sui-cluster-test/src/cluster.rs

@@ -7,14 +7,15 @@ use clap::*;
 use std::net::SocketAddr;
 use std::path::Path;
 use sui_config::Config;
-use sui_config::SUI_KEYSTORE_FILENAME;
+use sui_config::{PersistedConfig, SUI_KEYSTORE_FILENAME, SUI_NETWORK_CONFIG};
 use sui_indexer::test_utils::start_test_indexer;
 use sui_indexer::IndexerConfig;
 use sui_keys::keystore::{AccountKeystore, FileBasedKeystore, Keystore};
 use sui_sdk::sui_client_config::{SuiClientConfig, SuiEnv};
 use sui_sdk::wallet_context::WalletContext;
 use sui_swarm::memory::Swarm;
 use sui_swarm_config::genesis_config::GenesisConfig;
+use sui_swarm_config::network_config::NetworkConfig;
 use sui_types::base_types::SuiAddress;
 use sui_types::crypto::KeypairTraits;
 use sui_types::crypto::SuiKeyPair;
@@ -164,9 +165,6 @@ impl LocalNewCluster {
 #[async_trait]
 impl Cluster for LocalNewCluster {
     async fn start(options: &ClusterTestOpt) -> Result<Self, anyhow::Error> {
-        // Let the faucet account hold 1000 gas objects on genesis
-        let genesis_config = GenesisConfig::custom_genesis(1, 100);
-
         // TODO: options should contain port instead of address
         let fullnode_port = options.fullnode_address.as_ref().map(|addr| {
             addr.parse::<SocketAddr>()
@@ -179,13 +177,34 @@ impl Cluster for LocalNewCluster {
                 .expect("Unable to parse indexer address")
         });
 
-        let mut cluster_builder = TestClusterBuilder::new()
-            .set_genesis_config(genesis_config)
-            .enable_fullnode_events();
-
-        if let Some(epoch_duration_ms) = options.epoch_duration_ms {
-            cluster_builder = cluster_builder.with_epoch_duration_ms(epoch_duration_ms);
+        let mut cluster_builder = TestClusterBuilder::new().enable_fullnode_events();
+
+        // Check if we already have a config directory that is passed
+        if let Some(config_dir) = options.config_dir.clone() {
+            assert!(options.epoch_duration_ms.is_none());
+            // Load the config of the Sui authority.
+            let network_config_path = config_dir.join(SUI_NETWORK_CONFIG);
+            let network_config: NetworkConfig = PersistedConfig::read(&network_config_path)
+                .map_err(|err| {
+                    err.context(format!(
+                        "Cannot open Sui network config file at {:?}",
+                        network_config_path
+                    ))
+                })?;
+
+            cluster_builder = cluster_builder.set_network_config(network_config);
+            cluster_builder = cluster_builder.with_config_dir(config_dir);
+        } else {
+            // Let the faucet account hold 1000 gas objects on genesis
+            let genesis_config = GenesisConfig::custom_genesis(1, 100);
+            // Custom genesis should be build here where we add the extra accounts
+            cluster_builder = cluster_builder.set_genesis_config(genesis_config);
+
+            if let Some(epoch_duration_ms) = options.epoch_duration_ms {
+                cluster_builder = cluster_builder.with_epoch_duration_ms(epoch_duration_ms);
+            }
         }
+
         if let Some(rpc_port) = fullnode_port {
             cluster_builder = cluster_builder.with_fullnode_rpc_port(rpc_port);
         }

crates/sui-cluster-test/src/config.rs

@@ -1,3 +1,5 @@
+use std::path::PathBuf;
+
 // Copyright (c) Mysten Labs, Inc.
 // SPDX-License-Identifier: Apache-2.0
 use clap::*;
@@ -33,6 +35,8 @@ pub struct ClusterTestOpt {
     /// TODO(gegao): remove this after indexer migration is complete.
     #[clap(long)]
     pub use_indexer_experimental_methods: bool,
+    #[clap(long)]
+    pub config_dir: Option<PathBuf>,
 }
 
 impl ClusterTestOpt {
@@ -45,6 +49,7 @@ impl ClusterTestOpt {
             indexer_address: None,
             pg_address: None,
             use_indexer_experimental_methods: false,
+            config_dir: None,
         }
     }
 }

crates/sui-swarm-config/src/genesis_config.rs

@@ -331,4 +331,12 @@ impl GenesisConfig {
         let mut rng = StdRng::seed_from_u64(Self::BENCHMARKS_RNG_SEED);
         SuiKeyPair::Ed25519(NetworkKeyPair::generate(&mut rng))
     }
+
+    pub fn add_faucet_account(mut self) -> Self {
+        self.accounts.push(AccountConfig {
+            address: None,
+            gas_amounts: vec![DEFAULT_GAS_AMOUNT; DEFAULT_NUMBER_OF_OBJECT_PER_ACCOUNT],
+        });
+        self
+    }
 }

crates/sui-test-validator/README.md

@@ -13,3 +13,9 @@ Refer to [sui-local-network.md](../../doc/src/build/sui-local-network.md)
 2. Make sure the `Posgresdb` starts on your local machine
 3. run `RUST_LOG="consensus=off" ./target/debug/sui-test-validator --with-indexer`
 4. To check your local db, if you use the default db url `postgres://postgres:postgres@localhost:5432/sui_indexer`, you can login to the `postgres` database and run `\dt` to show all tables.
+
+## Run with a persisted state
+You can combine this with indexer runs as well to save a persisted state on local developement.
+
+1. Generate a config to store db and genesis configs `sui genesis -f --with-faucet --working-dir=[some-directory]`
+2. `sui-test-validator --config-dir [some-directory]`

crates/sui-test-validator/src/main.rs

@@ -23,10 +23,12 @@ use tower_http::cors::{Any, CorsLayer};
 #[derive(Parser, Debug)]
 #[clap(author, version, about, long_about = None)]
 struct Args {
-    // TODO: Support a configuration directory for persisted networks:
-    // /// Config directory that will be used to store network configuration
-    // #[clap(short, long, parse(from_os_str), value_hint = ValueHint::DirPath)]
-    // config: Option<std::path::PathBuf>,
+    /// Config directory that will be used to store network config, node db, keystore
+    /// sui genesis -f --with-faucet generates a genesis config that can be used to start this process.
+    /// Example: sui-test-validator --config-dir ~/.sui/sui_config
+    /// We can use any config dir that is generated by the sui genesis.
+    #[clap(short, long, parse(from_os_str))]
+    config_dir: Option<std::path::PathBuf>,
     /// Port to start the Fullnode RPC server on
     #[clap(long, default_value = "9000")]
     fullnode_rpc_port: u16,
@@ -69,6 +71,7 @@ async fn main() -> Result<()> {
 
     let args = Args::parse();
     let Args {
+        config_dir,
         fullnode_rpc_port,
         indexer_rpc_port,
         pg_port,
@@ -79,6 +82,13 @@ async fn main() -> Result<()> {
         use_indexer_experimental_methods,
     } = args;
 
+    // We don't pass epoch duration if we have a genesis config.
+    let epoch_duration_ms = if config_dir.is_some() {
+        None
+    } else {
+        Some(epoch_duration_ms)
+    };
+
     let cluster = LocalNewCluster::start(&ClusterTestOpt {
         env: Env::NewLocal,
         fullnode_address: Some(format!("127.0.0.1:{}", fullnode_rpc_port)),
@@ -87,8 +97,9 @@ async fn main() -> Result<()> {
             "postgres://postgres@{pg_host}:{pg_port}/sui_indexer"
         )),
         faucet_address: None,
-        epoch_duration_ms: Some(epoch_duration_ms),
+        epoch_duration_ms,
         use_indexer_experimental_methods,
+        config_dir,
     })
     .await?;
 

crates/sui/src/sui_commands.rs

@@ -82,6 +82,11 @@ pub enum SuiCommand {
             help = "A list of ip addresses to generate a genesis suitable for benchmarks"
         )]
         benchmark_ips: Option<Vec<String>>,
+        #[clap(
+            long,
+            help = "Creates an extra faucet configuration for sui-test-validator persisted runs."
+        )]
+        with_faucet: bool,
     },
     GenesisCeremony(Ceremony),
     /// Sui keystore tool.
@@ -160,7 +165,7 @@ impl SuiCommand {
             } => {
                 // Auto genesis if path is none and sui directory doesn't exists.
                 if config.is_none() && !sui_config_dir()?.join(SUI_NETWORK_CONFIG).exists() {
-                    genesis(None, None, None, false, None, None).await?;
+                    genesis(None, None, None, false, None, None, false).await?;
                 }
 
                 // Load the config of the Sui authority.
@@ -238,6 +243,7 @@ impl SuiCommand {
                 write_config,
                 epoch_duration_ms,
                 benchmark_ips,
+                with_faucet,
             } => {
                 genesis(
                     from_config,
@@ -246,6 +252,7 @@ impl SuiCommand {
                     force,
                     epoch_duration_ms,
                     benchmark_ips,
+                    with_faucet,
                 )
                 .await
             }
@@ -317,6 +324,7 @@ async fn genesis(
     force: bool,
     epoch_duration_ms: Option<u64>,
     benchmark_ips: Option<Vec<String>>,
+    with_faucet: bool,
 ) -> Result<(), anyhow::Error> {
     let sui_config_dir = &match working_dir {
         // if a directory is specified, it must exist (it
@@ -403,6 +411,12 @@ async fn genesis(
         }
     };
 
+    // Adds an extra faucet account to the genesis
+    if with_faucet {
+        info!("Adding faucet account in genesis config...");
+        genesis_conf = genesis_conf.add_faucet_account();
+    }
+
     if let Some(path) = write_config {
         let persisted = genesis_conf.persisted(&path);
         persisted.save()?;

crates/sui/tests/cli_tests.rs

@@ -69,6 +69,7 @@ async fn test_genesis() -> Result<(), anyhow::Error> {
         from_config: None,
         epoch_duration_ms: None,
         benchmark_ips: None,
+        with_faucet: false,
     }
     .execute()
     .await?;
@@ -107,6 +108,7 @@ async fn test_genesis() -> Result<(), anyhow::Error> {
         from_config: None,
         epoch_duration_ms: None,
         benchmark_ips: None,
+        with_faucet: false,
     }
     .execute()
     .await;
@@ -139,6 +141,7 @@ async fn test_genesis_for_benchmarks() -> Result<(), anyhow::Error> {
         from_config: None,
         epoch_duration_ms: None,
         benchmark_ips: Some(benchmark_ips.clone()),
+        with_faucet: false,
     }
     .execute()
     .await?;

crates/test-cluster/src/lib.rs

@@ -9,6 +9,7 @@ use rand::{distributions::*, rngs::OsRng, seq::SliceRandom};
 use std::collections::{HashMap, HashSet};
 use std::net::SocketAddr;
 use std::num::NonZeroUsize;
+use std::path::PathBuf;
 use std::sync::{Arc, Mutex};
 use std::time::Duration;
 use sui_config::node::DBCheckpointConfig;
@@ -27,6 +28,7 @@ use sui_swarm::memory::{Swarm, SwarmBuilder};
 use sui_swarm_config::genesis_config::{
     AccountConfig, GenesisConfig, ValidatorGenesisConfig, DEFAULT_GAS_AMOUNT,
 };
+use sui_swarm_config::network_config::NetworkConfig;
 use sui_swarm_config::network_config_builder::{
     ProtocolVersionsConfig, SupportedProtocolVersionsCallback,
 };
@@ -48,7 +50,6 @@ use sui_types::transaction::{TransactionData, VerifiedTransaction};
 use tokio::time::{timeout, Instant};
 use tokio::{task::JoinHandle, time::sleep};
 use tracing::info;
-
 const NUM_VALIDATOR: usize = 4;
 
 pub struct FullNodeHandle {
@@ -567,6 +568,7 @@ impl Drop for RandomNodeRestarter {
 
 pub struct TestClusterBuilder {
     genesis_config: Option<GenesisConfig>,
+    network_config: Option<NetworkConfig>,
     additional_objects: Vec<Object>,
     num_validators: Option<usize>,
     fullnode_rpc_port: Option<u16>,
@@ -577,12 +579,14 @@ pub struct TestClusterBuilder {
     db_checkpoint_config_validators: DBCheckpointConfig,
     db_checkpoint_config_fullnodes: DBCheckpointConfig,
     num_unpruned_validators: Option<usize>,
+    config_dir: Option<PathBuf>,
 }
 
 impl TestClusterBuilder {
     pub fn new() -> Self {
         TestClusterBuilder {
             genesis_config: None,
+            network_config: None,
             additional_objects: vec![],
             fullnode_rpc_port: None,
             num_validators: None,
@@ -592,6 +596,7 @@ impl TestClusterBuilder {
             db_checkpoint_config_validators: DBCheckpointConfig::default(),
             db_checkpoint_config_fullnodes: DBCheckpointConfig::default(),
             num_unpruned_validators: None,
+            config_dir: None,
         }
     }
 
@@ -601,11 +606,17 @@ impl TestClusterBuilder {
     }
 
     pub fn set_genesis_config(mut self, genesis_config: GenesisConfig) -> Self {
-        assert!(self.genesis_config.is_none());
+        assert!(self.genesis_config.is_none() && self.network_config.is_none());
         self.genesis_config = Some(genesis_config);
         self
     }
 
+    pub fn set_network_config(mut self, network_config: NetworkConfig) -> Self {
+        assert!(self.genesis_config.is_none() && self.network_config.is_none());
+        self.network_config = Some(network_config);
+        self
+    }
+
     pub fn with_objects<I: IntoIterator<Item = Object>>(mut self, objects: I) -> Self {
         self.additional_objects.extend(objects);
         self
@@ -709,6 +720,11 @@ impl TestClusterBuilder {
         self
     }
 
+    pub fn with_config_dir(mut self, config_dir: PathBuf) -> Self {
+        self.config_dir = Some(config_dir);
+        self
+    }
+
     pub async fn build(mut self) -> TestCluster {
         let swarm = self.start_swarm().await.unwrap();
         let working_dir = swarm.dir();
@@ -765,13 +781,22 @@ impl TestClusterBuilder {
         if let Some(genesis_config) = self.genesis_config.take() {
             builder = builder.with_genesis_config(genesis_config);
         }
+
+        if let Some(network_config) = self.network_config.take() {
+            builder = builder.with_network_config(network_config);
+        }
+
         if let Some(fullnode_rpc_port) = self.fullnode_rpc_port {
             builder = builder.with_fullnode_rpc_port(fullnode_rpc_port);
         }
         if let Some(num_unpruned_validators) = self.num_unpruned_validators {
             builder = builder.with_num_unpruned_validators(num_unpruned_validators);
         }
 
+        if let Some(config_dir) = self.config_dir.take() {
+            builder = builder.dir(config_dir);
+        }
+
         let mut swarm = builder.build();
         swarm.launch().await?;