Convert a bunch of tests to sim_test (MystenLabs#4851)

* Switch tests to sim_test

* Provide an escape hatch from determinism. (mainly for creating temp dirs)

* Convert more tests to sim_test

* Convert more tests to sim_test using SuiNodeHandle API

* Allow time for all batch subscribers to be notified

* More simtests

* fix lints

* Move more tests to sim_test

* Move CLI tests to sim_test

crates/sui-benchmark/src/bin/stress.rs

@@ -32,7 +32,6 @@ use sui_core::epoch::epoch_store::EpochStore;
 use sui_core::gateway_state::GatewayState;
 use sui_core::safe_client::SafeClientMetrics;
 use sui_node::metrics;
-use sui_node::SuiNode;
 use sui_types::base_types::ObjectID;
 use sui_types::base_types::SuiAddress;
 use sui_types::batch::UpdateItem;
@@ -300,7 +299,7 @@ async fn main() -> Result<()> {
                 .unwrap();
             server_runtime.block_on(async move {
                 // Setup the network
-                let nodes: Vec<SuiNode> = spawn_test_authorities(cloned_gas, &cloned_config).await;
+                let nodes: Vec<_> = spawn_test_authorities(cloned_gas, &cloned_config).await;
                 let handles: Vec<_> = nodes.into_iter().map(move |node| node.wait()).collect();
                 cloned_barrier.wait().await;
                 let mut follower_handles = vec![];

crates/sui-core/Cargo.toml

@@ -61,6 +61,9 @@ workspace-hack.workspace = true
 thiserror = "1.0.34"
 eyre = "0.6.8"
 
+sui-simulator = { path = "../sui-simulator" }
+sui-macros = { path = "../sui-macros" }
+
 [dev-dependencies]
 clap = { version = "3.2.17", features = ["derive"] }
 rand = "0.8.5"
@@ -71,6 +74,8 @@ serde_yaml = "0.8.26"
 pretty_assertions = "1.2.1"
 telemetry-subscribers.workspace = true
 
+sui-macros = { path = "../sui-macros" }
+
 test-fuzz = "3.0.4"
 test-utils = { path = "../test-utils" }
 

crates/sui-core/src/authority.rs

@@ -44,6 +44,7 @@ use sui_adapter::adapter;
 use sui_adapter::temporary_store::InnerTemporaryStore;
 use sui_config::genesis::Genesis;
 use sui_json_rpc_types::{SuiEventEnvelope, SuiTransactionEffects};
+use sui_simulator::nondeterministic;
 use sui_storage::{
     event_store::{EventStore, EventStoreType, StoredEvent},
     write_ahead_log::{DBTxGuard, TxGuard, WriteAheadLog},
@@ -1345,7 +1346,7 @@ impl AuthorityState {
             Some(path) => path,
             None => {
                 let dir = std::env::temp_dir();
-                let path = dir.join(format!("DB_{:?}", ObjectID::random()));
+                let path = dir.join(format!("DB_{:?}", nondeterministic!(ObjectID::random())));
                 std::fs::create_dir(&path).unwrap();
                 path
             }

crates/sui-core/src/authority_active/checkpoint_driver/tests.rs

@@ -12,9 +12,11 @@ use crate::authority_active::checkpoint_driver::CheckpointMetrics;
 use std::{collections::BTreeSet, sync::Arc, time::Duration};
 use sui_types::messages::ExecutionStatus;
 
+use sui_macros::*;
+
 use crate::checkpoints::checkpoint_tests::checkpoint_tests_setup;
 
-#[tokio::test(flavor = "current_thread", start_paused = true)]
+#[sim_test]
 async fn checkpoint_active_flow_happy_path() {
     use telemetry_subscribers::init_for_testing;
     init_for_testing();
@@ -85,7 +87,7 @@ async fn checkpoint_active_flow_happy_path() {
     }
 }
 
-#[tokio::test(flavor = "current_thread", start_paused = true)]
+#[sim_test]
 async fn checkpoint_active_flow_crash_client_with_gossip() {
     use telemetry_subscribers::init_for_testing;
     init_for_testing();
@@ -176,7 +178,7 @@ async fn checkpoint_active_flow_crash_client_with_gossip() {
     }
 }
 
-#[tokio::test(flavor = "current_thread", start_paused = true)]
+#[sim_test]
 async fn checkpoint_active_flow_crash_client_no_gossip() {
     use telemetry_subscribers::init_for_testing;
     init_for_testing();
@@ -267,7 +269,7 @@ async fn checkpoint_active_flow_crash_client_no_gossip() {
     }
 }
 
-#[tokio::test(flavor = "current_thread", start_paused = true)]
+#[sim_test]
 async fn test_empty_checkpoint() {
     use telemetry_subscribers::init_for_testing;
     init_for_testing();

crates/sui-core/src/checkpoints/tests/checkpoint_tests.rs

@@ -33,6 +33,9 @@ use crate::authority_aggregator::AuthAggMetrics;
 use crate::epoch::epoch_store::EpochStore;
 use parking_lot::Mutex;
 
+use sui_macros::sim_test;
+use sui_simulator::nondeterministic;
+
 pub struct TestCausalOrderPendCertNoop;
 
 impl CausalOrder for TestCausalOrderPendCertNoop {
@@ -885,7 +888,7 @@ fn checkpoint_integration() {
 
 // Now check the connection between state / bacth and checkpoint mechanism
 
-#[tokio::test]
+#[sim_test]
 async fn test_batch_to_checkpointing() {
     // Create an authority
     // Make a test key pair
@@ -972,11 +975,11 @@ async fn test_batch_to_checkpointing() {
     _join.await.expect("No errors in task").expect("ok");
 }
 
-#[tokio::test(flavor = "current_thread", start_paused = true)]
+#[sim_test]
 async fn test_batch_to_checkpointing_init_crash() {
     // Create a random directory to store the DB
     let dir = env::temp_dir();
-    let path = dir.join(format!("DB_{:?}", ObjectID::random()));
+    let path = dir.join(format!("DB_{:?}", nondeterministic!(ObjectID::random())));
     fs::create_dir(&path).unwrap();
 
     // Make a test key pair
@@ -1007,7 +1010,7 @@ async fn test_batch_to_checkpointing_init_crash() {
                 .await
         });
 
-        tokio::time::advance(Duration::from_millis(10)).await;
+        tokio::time::sleep(Duration::from_millis(10)).await;
         tokio::task::yield_now().await;
 
         // Send transactions out of order

crates/sui-core/src/epoch/epoch_store.rs

@@ -3,6 +3,7 @@
 
 use rocksdb::Options;
 use std::path::PathBuf;
+use sui_simulator::nondeterministic;
 use sui_storage::default_db_options;
 use sui_types::base_types::ObjectID;
 use sui_types::committee::{Committee, EpochId};
@@ -40,7 +41,7 @@ impl EpochStore {
 
     pub fn new_for_testing(genesis_committee: &Committee) -> Self {
         let dir = std::env::temp_dir();
-        let path = dir.join(format!("DB_{:?}", ObjectID::random()));
+        let path = dir.join(format!("DB_{:?}", nondeterministic!(ObjectID::random())));
         std::fs::create_dir(&path).unwrap();
         Self::new(path, genesis_committee, None)
     }

crates/sui-core/src/node_sync/node_follower.rs

@@ -328,6 +328,7 @@ mod test {
         node_sync::SyncStatus, test_utils::test_authority_aggregator,
     };
     use std::sync::{Arc, Mutex};
+    use sui_macros::sim_test;
     use sui_types::{
         base_types::ObjectID,
         crypto::get_account_key_pair,
@@ -426,7 +427,7 @@ mod test {
         Arc::new(NodeSyncStore::open_tables_read_write(db_path, None, None))
     }
 
-    #[tokio::test(flavor = "current_thread", start_paused = true)]
+    #[sim_test]
     async fn test_follower() {
         telemetry_subscribers::init_for_testing();
 
@@ -462,7 +463,7 @@ mod test {
             let sync_store = new_sync_store();
             let test_handler = TestNodeSyncHandler::new();
 
-            let peer = authorities[0].state().name;
+            let peer = authorities[0].with(|node| node.state().name);
             let metrics = GossipMetrics::new_for_tests();
             follow_one_peer(
                 test_handler.clone().break_after(1),

crates/sui-core/src/unit_tests/authority_aggregator_tests.rs

@@ -14,8 +14,8 @@ use sui_types::crypto::{
     AuthorityPublicKeyBytes, NetworkKeyPair, SuiKeyPair,
 };
 use sui_types::crypto::{KeypairTraits, Signature};
-use test_utils::authority::SuiNode;
 
+use sui_macros::sim_test;
 use sui_types::messages::*;
 use sui_types::object::{Object, GAS_VALUE_FOR_TESTING};
 use test_utils::authority::{spawn_test_authorities, test_and_configure_authority_configs};
@@ -35,7 +35,7 @@ pub async fn init_network_authorities(
     genesis_objects: Vec<Object>,
 ) -> AuthorityAggregator<NetworkAuthorityClient> {
     let configs = test_and_configure_authority_configs(committee_size);
-    let _nodes: Vec<SuiNode> = spawn_test_authorities(genesis_objects, &configs).await;
+    let _nodes = spawn_test_authorities(genesis_objects, &configs).await;
     let gateway_config = GatewayConfig {
         epoch: 0,
         validator_set: configs.validator_set().to_vec(),
@@ -433,7 +433,7 @@ async fn execute_transaction_with_fault_configs(
 /// we spawn a tokio task on the server, client timing out and
 /// terminating the connection does not stop server from completing
 /// execution on its side
-#[tokio::test(flavor = "multi_thread", worker_threads = 6)]
+#[sim_test]
 async fn test_quorum_map_and_reduce_timeout() {
     let build_config = BuildConfig::default();
     let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
@@ -475,7 +475,7 @@ async fn test_quorum_map_and_reduce_timeout() {
     }
 }
 
-#[tokio::test]
+#[sim_test]
 async fn test_map_reducer() {
     let (authorities, _, _) = init_local_authorities(4, vec![]).await;
 
@@ -607,7 +607,7 @@ async fn test_map_reducer() {
     assert!(!res.as_ref().unwrap().contains(&bad_auth));
 }
 
-#[tokio::test]
+#[sim_test]
 async fn test_get_all_owned_objects() {
     let (addr1, key1): (_, AccountKeyPair) = get_key_pair();
     let (addr2, _): (_, AccountKeyPair) = get_key_pair();
@@ -693,7 +693,7 @@ async fn test_get_all_owned_objects() {
     assert_eq!(1, owned_object.len());
 }
 
-#[tokio::test]
+#[sim_test]
 async fn test_sync_all_owned_objects() {
     let (addr1, key1): (_, AccountKeyPair) = get_key_pair();
     let (addr2, _): (_, AccountKeyPair) = get_key_pair();
@@ -808,7 +808,7 @@ async fn get_owned_objects(
     owned_objects
 }
 
-#[tokio::test]
+#[sim_test]
 async fn test_process_certificate() {
     let (addr1, key1): (_, AccountKeyPair) = get_key_pair();
     let gas_object1 = Object::with_owner_for_testing(addr1);
@@ -861,7 +861,7 @@ async fn test_process_certificate() {
     assert_eq!(SequenceNumber::from(2), new_object_version);
 }
 
-#[tokio::test]
+#[sim_test]
 async fn test_execute_cert_to_true_effects() {
     let (addr1, key1): (_, AccountKeyPair) = get_key_pair();
     let gas_object1 = Object::with_owner_for_testing(addr1);
@@ -900,7 +900,7 @@ async fn test_execute_cert_to_true_effects() {
     assert!(count >= 2);
 }
 
-#[tokio::test]
+#[sim_test]
 async fn test_process_transaction_fault_success() {
     // This test exercises the 4 different possible fauling case when one authority is faulty.
     // A transaction is sent to all authories, however one of them will error out either before or after processing the transaction.
@@ -928,7 +928,7 @@ async fn test_process_transaction_fault_success() {
     }
 }
 
-#[tokio::test]
+#[sim_test]
 async fn test_process_transaction_fault_fail() {
     // This test exercises the cases when there are 2 authorities faulty,
     // and hence no quorum could be formed. This is tested on both the

crates/sui-node/Cargo.toml

@@ -31,6 +31,9 @@ telemetry-subscribers.workspace = true
 mysten-network.workspace = true
 workspace-hack.workspace = true
 
+[target.'cfg(msim)'.dependencies]
+sui-simulator = { path = "../sui-simulator" }
+
 [target.'cfg(not(target_env = "msvc"))'.dependencies]
 jemallocator = { version = "^0.5", features = ["profiling"] }
 jemalloc-ctl = "^0.5"