Update typed-store and typed-store-derive pointers (MystenLabs#5557)

Cargo.toml

@@ -71,12 +71,12 @@ opt-level = 1
 [workspace.dependencies]
 
 # github.com/MystenLabs/mysten-infra dependencies
-typed-store = "0.1.0"
-typed-store-derive = "0.1.0"
+typed-store = "0.4.0"
+typed-store-derive = "0.3.0"
 telemetry-subscribers = { version = "0.2.0", features = ["jaeger", "tokio-console"] }
 mysten-network = "0.2.0"
 name-variant = "0.1.0"
-store = { version = "0.1.0", package = "typed-store" }
+store = { version = "0.4.0", package = "typed-store" }
 
 # Move dependencies
 move-binary-format = { git = "https://github.com/move-language/move", rev = "1ffd0a3e7bdc4bba7dafb8c814279e750113d030" }

crates/sui-core/src/authority/authority_store_tables.rs

@@ -13,7 +13,7 @@ use sui_storage::default_db_options;
 use sui_types::base_types::{ExecutionDigests, SequenceNumber};
 use sui_types::batch::{SignedBatch, TxSequenceNumber};
 use sui_types::messages::{TrustedCertificate, TrustedTransactionEnvelope};
-use typed_store::rocks::DBMap;
+use typed_store::rocks::{DBMap, DBOptions};
 use typed_store::traits::TypedStoreDebug;
 
 use typed_store_derive::DBMapUtils;
@@ -247,15 +247,15 @@ pub struct ExecutionIndicesWithHash {
 }
 
 // These functions are used to initialize the DB tables
-fn objects_table_default_config() -> Options {
+fn objects_table_default_config() -> DBOptions {
     default_db_options(None, None).1
 }
-fn transactions_table_default_config() -> Options {
+fn transactions_table_default_config() -> DBOptions {
     default_db_options(None, None).1
 }
-fn certificates_table_default_config() -> Options {
+fn certificates_table_default_config() -> DBOptions {
     default_db_options(None, None).1
 }
-fn effects_table_default_config() -> Options {
+fn effects_table_default_config() -> DBOptions {
     default_db_options(None, None).1
 }

crates/sui-core/src/checkpoints/causal_order_effects.rs

@@ -340,9 +340,9 @@ mod tests {
         }
     }
 
-    #[test]
+    #[tokio::test]
     #[allow(clippy::redundant_clone)]
-    fn causal_just_reorder() {
+    async fn causal_just_reorder() {
         let mut rng = StdRng::from_seed([1; 32]);
         let (keys, committee) = make_committee_key(&mut rng);
         let k = keys[0].copy();
@@ -468,9 +468,9 @@ mod tests {
         assert_eq!(x.len(), 4);
     }
 
-    #[test]
+    #[tokio::test]
     // Check that we are summing up the gas costs of txns correctly.
-    fn test_gas_costs() {
+    async fn test_gas_costs() {
         let (_committee, _keys, mut stores) = random_ckpoint_store();
         let (_, mut cps) = stores.pop().unwrap();
         let txn_digest_0 = TransactionDigest::random();

crates/sui-core/src/checkpoints/mod.rs

@@ -31,6 +31,7 @@ use sui_types::{
 use tap::TapFallible;
 use tokio::sync::broadcast;
 use tracing::{debug, error, info};
+use typed_store::rocks::DBOptions;
 use typed_store::traits::TypedStoreDebug;
 
 use typed_store::{
@@ -149,21 +150,21 @@ pub struct CheckpointStoreTables {
 }
 
 // These functions are used to initialize the DB tables
-fn transactions_to_checkpoint_table_default_config() -> Options {
+fn transactions_to_checkpoint_table_default_config() -> DBOptions {
     default_db_options(None, None).1
 }
-fn extra_transactions_table_default_config() -> Options {
+fn extra_transactions_table_default_config() -> DBOptions {
     default_db_options(None, None).1
 }
 
-fn checkpoints_table_default_config() -> Options {
+fn checkpoints_table_default_config() -> DBOptions {
     default_db_options(None, None).1
 }
-fn local_fragments_table_default_config() -> Options {
+fn local_fragments_table_default_config() -> DBOptions {
     default_db_options(None, None).1
 }
 
-fn locals_table_default_config() -> Options {
+fn locals_table_default_config() -> DBOptions {
     default_db_options(None, None).1
 }
 

crates/sui-core/src/checkpoints/tests/checkpoint_tests.rs

@@ -75,8 +75,8 @@ fn random_ckpoint_store_num(
     (committee, keys, stores)
 }
 
-#[test]
-fn crash_recovery() {
+#[tokio::test]
+async fn crash_recovery() {
     let mut rng = StdRng::from_seed(RNG_SEED);
     let (keys, committee) = make_committee_key(&mut rng);
     let k = keys[0].copy();
@@ -135,6 +135,10 @@ fn crash_recovery() {
     // Delete and re-open DB
     drop(cps);
 
+    // TODO: The right fix is to invoke some function on DBMap and release the rocksdb arc references
+    // being held in the background thread but this will suffice for now
+    tokio::time::sleep(Duration::from_secs(1)).await;
+
     let mut cps_new = CheckpointStore::open(
         &path,
         None,
@@ -163,8 +167,8 @@ fn crash_recovery() {
     );
 }
 
-#[test]
-fn make_checkpoint_db() {
+#[tokio::test]
+async fn make_checkpoint_db() {
     let (_committee, _keys, mut stores) = random_ckpoint_store();
     let (_, mut cps) = stores.pop().unwrap();
 
@@ -217,8 +221,8 @@ fn make_checkpoint_db() {
     assert_eq!(cp_seq, 0);
 }
 
-#[test]
-fn make_proposals() {
+#[tokio::test]
+async fn make_proposals() {
     let (committee, _keys, mut stores) = random_ckpoint_store();
     let (_, mut cps1) = stores.pop().unwrap();
     let (_, mut cps2) = stores.pop().unwrap();
@@ -305,8 +309,8 @@ fn make_proposals() {
     );
 }
 
-#[test]
-fn make_diffs() {
+#[tokio::test]
+async fn make_diffs() {
     let (committee, _keys, mut stores) = random_ckpoint_store();
     let (_, mut cps1) = stores.pop().unwrap();
     let (_, mut cps2) = stores.pop().unwrap();
@@ -359,8 +363,8 @@ fn make_diffs() {
     assert_eq!(all_items1, all_items4);
 }
 
-#[test]
-fn latest_proposal() {
+#[tokio::test]
+async fn latest_proposal() {
     let (committee, _keys, mut stores) = random_ckpoint_store();
     let (_, mut cps1) = stores.pop().unwrap();
     let (_, mut cps2) = stores.pop().unwrap();
@@ -580,8 +584,8 @@ fn latest_proposal() {
     }
 }
 
-#[test]
-fn update_processed_transactions_already_in_checkpoint() {
+#[tokio::test]
+async fn update_processed_transactions_already_in_checkpoint() {
     let (_committee, _keys, mut stores) = random_ckpoint_store_num(1);
     let (_, mut cps) = stores.pop().unwrap();
 
@@ -602,8 +606,8 @@ fn update_processed_transactions_already_in_checkpoint() {
     );
 }
 
-#[test]
-fn set_get_checkpoint() {
+#[tokio::test]
+async fn set_get_checkpoint() {
     let (committee, _keys, mut stores) = random_ckpoint_store();
     let (_, mut cps1) = stores.pop().unwrap();
     let (_, mut cps2) = stores.pop().unwrap();
@@ -783,8 +787,8 @@ fn set_get_checkpoint() {
     ));
 }
 
-#[test]
-fn checkpoint_integration() {
+#[tokio::test]
+async fn checkpoint_integration() {
     telemetry_subscribers::init_for_testing();
 
     let mut rng = StdRng::from_seed(RNG_SEED);
@@ -1095,6 +1099,10 @@ async fn test_batch_to_checkpointing_init_crash() {
         _join.await.expect("No errors in task").expect("ok");
     }
 
+    // TODO: The right fix is to invoke some function on DBMap and release the rocksdb arc references
+    // being held in the background thread but this will suffice for now
+    tokio::time::sleep(Duration::from_secs(1)).await;
+
     // Scope to ensure all variables are dropped
     {
         let (tx_reconfigure_consensus, _rx_reconfigure_consensus) = tokio::sync::mpsc::channel(10);
@@ -1124,8 +1132,8 @@ async fn test_batch_to_checkpointing_init_crash() {
     }
 }
 
-#[test]
-fn set_fragment_external() {
+#[tokio::test]
+async fn set_fragment_external() {
     let (committee, keys, mut test_objects) = random_ckpoint_store();
     let (test_tx, _rx) = TestConsensus::new();
 
@@ -1198,8 +1206,8 @@ fn set_fragment_external() {
         .is_err());
 }
 
-#[test]
-fn set_fragment_reconstruct() {
+#[tokio::test]
+async fn set_fragment_reconstruct() {
     telemetry_subscribers::init_for_testing();
     let (committee, _keys, mut test_objects) = random_ckpoint_store();
     let (_, mut cps1) = test_objects.pop().unwrap();
@@ -1245,8 +1253,8 @@ fn set_fragment_reconstruct() {
     assert_eq!(reconstruction.global.authority_waypoints.len(), 4);
 }
 
-#[test]
-fn set_fragment_reconstruct_two_components() {
+#[tokio::test]
+async fn set_fragment_reconstruct_two_components() {
     let (committee, _keys, mut test_objects) = random_ckpoint_store_num(2 * 3 + 1);
 
     let t2 = ExecutionDigests::random();
@@ -1299,8 +1307,8 @@ fn set_fragment_reconstruct_two_components() {
     assert_eq!(reconstruction.global.authority_waypoints.len(), 5);
 }
 
-#[test]
-fn set_fragment_reconstruct_two_mutual() {
+#[tokio::test]
+async fn set_fragment_reconstruct_two_mutual() {
     let (committee, _, mut test_objects) = random_ckpoint_store_num(4);
 
     let t2 = ExecutionDigests::random();
@@ -1355,8 +1363,8 @@ impl TestConsensus {
     }
 }
 
-#[test]
-fn test_fragment_full_flow() {
+#[tokio::test]
+async fn test_fragment_full_flow() {
     let (committee, _keys, mut test_objects) = random_ckpoint_store_num(2 * 3 + 1);
 
     let (test_tx, rx) = TestConsensus::new();
@@ -1503,8 +1511,8 @@ fn test_fragment_full_flow() {
     // But recording of fragments is closed
 }
 
-#[test]
-fn test_slow_fragment() {
+#[tokio::test]
+async fn test_slow_fragment() {
     let (committee, _keys, mut cp_stores) = random_ckpoint_store();
     let proposals: Vec<_> = cp_stores
         .iter_mut()

crates/sui-core/src/epoch/committee_store.rs

@@ -7,7 +7,7 @@ use sui_storage::default_db_options;
 use sui_types::base_types::ObjectID;
 use sui_types::committee::{Committee, EpochId};
 use sui_types::error::{SuiError, SuiResult};
-use typed_store::rocks::DBMap;
+use typed_store::rocks::{DBMap, DBOptions};
 use typed_store::traits::TypedStoreDebug;
 
 use sui_types::fp_ensure;
@@ -26,7 +26,7 @@ pub struct CommitteeStore {
 }
 
 // These functions are used to initialize the DB tables
-fn committee_table_default_config() -> Options {
+fn committee_table_default_config() -> DBOptions {
     default_db_options(None, None).1
 }
 

crates/sui-core/src/unit_tests/authority_tests.rs

@@ -1699,6 +1699,10 @@ async fn test_authority_persist() {
     // Close the authority
     drop(authority);
 
+    // TODO: The right fix is to invoke some function on DBMap and release the rocksdb arc references
+    // being held in the background thread but this will suffice for now
+    tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+
     // Reopen the same authority with the same path
     let seed = [1u8; 32];
     let (committee, _, authority_key) =

crates/sui-core/src/unit_tests/batch_tests.rs

@@ -140,6 +140,11 @@ async fn test_open_manager() {
         drop(store);
         drop(authority_state);
     }
+
+    // TODO: The right fix is to invoke some function on DBMap and release the rocksdb arc references
+    // being held in the background thread but this will suffice for now
+    tokio::time::sleep(Duration::from_secs(1)).await;
+
     // drop all
     let (committee, _, authority_key) =
         init_state_parameters_from_rng(&mut StdRng::from_seed(seed));
@@ -163,6 +168,10 @@ async fn test_open_manager() {
         drop(store);
         drop(authority_state);
     }
+    // TODO: The right fix is to invoke some function on DBMap and release the rocksdb arc references
+    // being held in the background thread but this will suffice for now
+    tokio::time::sleep(Duration::from_secs(1)).await;
+
     // drop all
     let (committee, _, authority_key) =
         init_state_parameters_from_rng(&mut StdRng::from_seed(seed));
@@ -489,6 +498,7 @@ async fn test_batch_store_retrieval() {
 
     // Give a change to the channels to send.
     tokio::task::yield_now().await;
+    tokio::task::yield_now().await;
 
     // TEST 1: Get batches across boundaries
 

crates/sui-core/tests/staged/sui.yaml

@@ -538,6 +538,8 @@ TypedStoreError:
         NEWTYPE: STR
     3:
       CrossDBBatch: UNIT
+    4:
+      MetricsReporting: UNIT
 UpdateItem:
   ENUM:
     0:

crates/sui-macros/Cargo.toml

@@ -16,4 +16,4 @@ syn = "1"
 workspace-hack.workspace = true
 
 [target.'cfg(msim)'.dependencies]
-msim-macros = { git = "https://github.com/MystenLabs/mysten-sim.git", rev = "982cf1a544ebd1e2818330f9b1247a4c3dd26c13", package = "msim-macros" }
+msim-macros = { git = "https://github.com/MystenLabs/mysten-sim.git", rev = "b28d77f88434535b166cc60bd3e5018aa75cd705", package = "msim-macros" }

crates/sui-node/src/lib.rs

@@ -45,6 +45,7 @@ use sui_types::messages::{CertifiedTransaction, CertifiedTransactionEffects};
 use tokio::sync::mpsc::channel;
 use tower::ServiceBuilder;
 use tracing::{error, info, warn};
+use typed_store::DBMetrics;
 
 use crate::metrics::GrpcMetrics;
 use sui_core::authority_client::NetworkAuthorityClientMetrics;
@@ -95,6 +96,9 @@ impl SuiNode {
             "Initializing sui-node listening on {}", config.network_address
         );
 
+        // Initialize metrics to track db usage before creating any stores
+        DBMetrics::init(&prometheus_registry);
+
         let genesis = config.genesis()?;
 
         let secret = Arc::pin(config.protocol_key_pair().copy());

crates/sui-rosetta/Cargo.toml

@@ -36,8 +36,8 @@ sui-keys = { path = "../sui-keys" }
 
 move-core-types.workspace = true
 
-typed-store = "0.1.0"
-typed-store-derive = "0.1.0"
+typed-store = "0.4.0"
+typed-store-derive = "0.3.0"
 telemetry-subscribers = "0.1.0"
 
 workspace-hack = { path = "../workspace-hack" }

crates/sui-simulator/Cargo.toml

@@ -16,4 +16,4 @@ anemo.workspace = true
 fastcrypto = { workspace = true, features = ["copy_key"] }
 
 [target.'cfg(msim)'.dependencies]
-msim = { git = "https://github.com/MystenLabs/mysten-sim.git", rev = "982cf1a544ebd1e2818330f9b1247a4c3dd26c13", package = "msim" }
+msim = { git = "https://github.com/MystenLabs/mysten-sim.git", rev = "b28d77f88434535b166cc60bd3e5018aa75cd705", package = "msim" }