diff --git a/Cargo.lock b/Cargo.lock index 955f988548fed..671cb011950b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -247,12 +247,6 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" -[[package]] -name = "ascii" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbf56136a5198c7b01a49e3afcbef6cf84597273d298f54432926024107b0109" - [[package]] name = "async-lock" version = "2.5.0" @@ -798,12 +792,6 @@ dependencies = [ "phf_codegen", ] -[[package]] -name = "chunked_transfer" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fff857943da45f546682664a79488be82e69e43c1a7a2307679ab9afb3a66d2e" - [[package]] name = "clang-sys" version = "1.3.3" @@ -4795,20 +4783,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "prometheus_exporter" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "019a192344efa197e8edfb2b864a5369ba8a837578d1bee469f21d98a8ed1233" -dependencies = [ - "ascii", - "lazy_static 1.4.0", - "log", - "prometheus", - "thiserror", - "tiny_http", -] - [[package]] name = "prost" version = "0.10.4" @@ -6044,7 +6018,6 @@ dependencies = [ "parking_lot 0.12.1", "pretty_assertions", "prometheus", - "prometheus_exporter", "rand 0.7.3", "rayon", "rocksdb", @@ -6203,7 +6176,6 @@ dependencies = [ "parking_lot 0.12.1", "pretty_assertions", "prometheus", - "prometheus_exporter", "rand 0.7.3", "rocksdb", "schemars", @@ -6320,7 +6292,7 @@ dependencies = [ "move-core-types", "mysten-network 0.1.0 (git+https://github.com/MystenLabs/mysten-infra?rev=ff5c1d69057fe93be658377462ca2875a57a0223)", "once_cell", - "prometheus_exporter", + "prometheus", "schemars", "serde 1.0.137", "serde_json", @@ -6372,7 +6344,6 @@ dependencies = [ "move-core-types", "once_cell", "prometheus", - "prometheus_exporter", "serde 1.0.137", "serde_json", "sui-core", @@ -6850,6 +6821,7 @@ dependencies = [ "jsonrpsee-http-server", "move-core-types", "move-package", + "prometheus", "rand 0.7.3", "rocksdb", "sui", @@ -6957,15 +6929,8 @@ dependencies = [ "itoa 1.0.2", "libc", "num_threads", - "time-macros", ] -[[package]] -name = "time-macros" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" - [[package]] name = "tint" version = "1.0.1" @@ -6975,19 +6940,6 @@ dependencies = [ "lazy_static 0.2.11", ] -[[package]] -name = "tiny_http" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f8734c6d6943ad6df6b588d228a87b4af184998bcffa268ceddf05c2055a8c" -dependencies = [ - "ascii", - "chunked_transfer", - "log", - "time 0.3.9", - "url", -] - [[package]] name = "tinyvec" version = "1.6.0" @@ -8082,7 +8034,6 @@ dependencies = [ "arrayref", "arrayvec 0.5.2", "arrayvec 0.7.2", - "ascii", "async-lock", "async-recursion", "async-stream", @@ -8133,7 +8084,6 @@ dependencies = [ "chrono", "chrono-tz", "chrono-tz-build", - "chunked_transfer", "clang-sys", "clap 2.34.0", "clap 3.1.18", @@ -8442,11 +8392,11 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2", "prometheus", - "prometheus_exporter", "prost", "prost-build", "prost-derive", "prost-types", + "protobuf", "ptree", "quote", "radix_trie", @@ -8566,9 +8516,7 @@ dependencies = [ "thrift", "time 0.1.43", "time 0.3.9", - "time-macros", "tint", - "tiny_http", "tinyvec", "tinyvec_macros", "tokio", diff --git a/crates/sui-core/Cargo.toml b/crates/sui-core/Cargo.toml index 4ac340966f75a..f34e519943965 100644 --- a/crates/sui-core/Cargo.toml +++ b/crates/sui-core/Cargo.toml @@ -35,7 +35,6 @@ schemars = {version = "0.8.10", features = ["either"]} either = "1.6.1" multiaddr = "0.14.0" mysten-network = { git = "https://github.com/MystenLabs/mysten-infra", rev = "ff5c1d69057fe93be658377462ca2875a57a0223" } -prometheus_exporter = "0.8.4" prometheus = "0.13.1" once_cell = "1.11.0" colored = "2.0.0" diff --git a/crates/sui-core/src/authority.rs b/crates/sui-core/src/authority.rs index e455077698e52..f3a2fc0f6f845 100644 --- a/crates/sui-core/src/authority.rs +++ b/crates/sui-core/src/authority.rs @@ -26,7 +26,6 @@ use move_core_types::{ use move_vm_runtime::{move_vm::MoveVM, native_functions::NativeFunctionTable}; use narwhal_executor::ExecutionStateError; use narwhal_executor::{ExecutionIndices, ExecutionState}; -use once_cell::sync::Lazy; use parking_lot::Mutex; use prometheus::{ register_histogram_with_registry, register_int_counter_with_registry, Histogram, IntCounter, diff --git a/crates/sui-core/src/authority_active.rs b/crates/sui-core/src/authority_active.rs index 801f06983981e..87d6b97306fdb 100644 --- a/crates/sui-core/src/authority_active.rs +++ b/crates/sui-core/src/authority_active.rs @@ -44,7 +44,7 @@ use tracing::error; use crate::{ authority::AuthorityState, authority_aggregator::AuthorityAggregator, - authority_client::AuthorityAPI, + authority_client::AuthorityAPI, gateway_state::GatewayMetrics, }; use tokio::time::Instant; @@ -110,6 +110,7 @@ pub struct ActiveAuthority { pub net: ArcSwap>, // Network health pub health: Arc>>, + pub gateway_metrics: GatewayMetrics, } impl ActiveAuthority { @@ -117,6 +118,7 @@ impl ActiveAuthority { authority: Arc, follower_store: Arc, authority_clients: BTreeMap, + gateway_metrics: GatewayMetrics, ) -> SuiResult { let committee = authority.clone_committee(); @@ -133,17 +135,25 @@ impl ActiveAuthority { net: ArcSwap::from(Arc::new(AuthorityAggregator::new( committee, authority_clients, + gateway_metrics.clone(), ))), + gateway_metrics, }) } pub fn new_with_ephemeral_follower_store( authority: Arc, authority_clients: BTreeMap, + gateway_metrics: GatewayMetrics, ) -> SuiResult { let working_dir = tempfile::tempdir().unwrap(); let follower_store = Arc::new(FollowerStore::open(&working_dir).expect("cannot open db")); - Self::new(authority, follower_store, authority_clients) + Self::new( + authority, + follower_store, + authority_clients, + gateway_metrics, + ) } /// Returns the amount of time we should wait to be able to contact at least @@ -199,6 +209,7 @@ impl Clone for ActiveAuthority { follower_store: self.follower_store.clone(), net: ArcSwap::from(self.net.load().clone()), health: self.health.clone(), + gateway_metrics: self.gateway_metrics.clone(), } } } diff --git a/crates/sui-core/src/authority_active/checkpoint_driver/tests.rs b/crates/sui-core/src/authority_active/checkpoint_driver/tests.rs index a3e60acfd528d..be611d8c2cefe 100644 --- a/crates/sui-core/src/authority_active/checkpoint_driver/tests.rs +++ b/crates/sui-core/src/authority_active/checkpoint_driver/tests.rs @@ -5,6 +5,7 @@ use crate::{ authority_active::{checkpoint_driver::CheckpointProcessControl, ActiveAuthority}, authority_client::LocalAuthorityClient, checkpoints::checkpoint_tests::TestSetup, + gateway_state::GatewayMetrics, safe_client::SafeClient, }; @@ -35,6 +36,7 @@ async fn checkpoint_active_flow_happy_path() { ActiveAuthority::new_with_ephemeral_follower_store( inner_state.authority.clone(), clients, + GatewayMetrics::new_for_tests(), ) .unwrap(), ); @@ -112,6 +114,7 @@ async fn checkpoint_active_flow_crash_client_with_gossip() { ActiveAuthority::new_with_ephemeral_follower_store( inner_state.authority.clone(), clients, + GatewayMetrics::new_for_tests(), ) .unwrap(), ); @@ -202,6 +205,7 @@ async fn checkpoint_active_flow_crash_client_no_gossip() { ActiveAuthority::new_with_ephemeral_follower_store( inner_state.authority.clone(), clients, + GatewayMetrics::new_for_tests(), ) .unwrap(), ); diff --git a/crates/sui-core/src/authority_active/execution_driver/tests.rs b/crates/sui-core/src/authority_active/execution_driver/tests.rs index d54a4dc494fde..ad1b09c2fc6a2 100644 --- a/crates/sui-core/src/authority_active/execution_driver/tests.rs +++ b/crates/sui-core/src/authority_active/execution_driver/tests.rs @@ -1,6 +1,7 @@ // Copyright (c) 2022, Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 +use crate::gateway_state::GatewayMetrics; use crate::{authority_active::ActiveAuthority, checkpoints::checkpoint_tests::TestSetup}; use std::sync::Arc; @@ -33,6 +34,7 @@ async fn pending_exec_storage_notify() { ActiveAuthority::new_with_ephemeral_follower_store( inner_state.authority.clone(), clients, + GatewayMetrics::new_for_tests(), ) .unwrap(), ); @@ -113,6 +115,7 @@ async fn pending_exec_full() { ActiveAuthority::new_with_ephemeral_follower_store( inner_state.authority.clone(), clients, + GatewayMetrics::new_for_tests(), ) .unwrap(), ); diff --git a/crates/sui-core/src/authority_active/gossip/tests.rs b/crates/sui-core/src/authority_active/gossip/tests.rs index a8e6b5cd53b23..b3c3da07fefad 100644 --- a/crates/sui-core/src/authority_active/gossip/tests.rs +++ b/crates/sui-core/src/authority_active/gossip/tests.rs @@ -6,6 +6,7 @@ use crate::authority_active::gossip::configurable_batch_action_client::{ init_configurable_authorities, BatchAction, }; use crate::authority_active::MAX_RETRY_DELAY_MS; +use crate::gateway_state::GatewayMetrics; use std::time::Duration; #[tokio::test(flavor = "current_thread", start_paused = true)] @@ -26,8 +27,12 @@ pub async fn test_gossip_plain() { let handle = tokio::task::spawn(async move { let active_state = Arc::new( - ActiveAuthority::new_with_ephemeral_follower_store(inner_state, inner_clients) - .unwrap(), + ActiveAuthority::new_with_ephemeral_follower_store( + inner_state, + inner_clients, + GatewayMetrics::new_for_tests(), + ) + .unwrap(), ); active_state.spawn_gossip_process(3).await; }); @@ -69,8 +74,12 @@ pub async fn test_gossip_error() { let handle = tokio::task::spawn(async move { let active_state = Arc::new( - ActiveAuthority::new_with_ephemeral_follower_store(inner_state, inner_clients) - .unwrap(), + ActiveAuthority::new_with_ephemeral_follower_store( + inner_state, + inner_clients, + GatewayMetrics::new_for_tests(), + ) + .unwrap(), ); active_state.spawn_gossip_process(3).await; }); diff --git a/crates/sui-core/src/authority_aggregator.rs b/crates/sui-core/src/authority_aggregator.rs index c8e900f8bba66..e70e15fd2e154 100644 --- a/crates/sui-core/src/authority_aggregator.rs +++ b/crates/sui-core/src/authority_aggregator.rs @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 use crate::authority_client::AuthorityAPI; -use crate::gateway_state::{GatewayMetrics, METRICS}; +use crate::gateway_state::GatewayMetrics; use crate::safe_client::SafeClient; use async_trait::async_trait; @@ -59,18 +59,23 @@ pub struct AuthorityAggregator { /// How to talk to this committee. pub authority_clients: BTreeMap>, // Metrics - pub metrics: &'static GatewayMetrics, + pub metrics: GatewayMetrics, pub timeouts: TimeoutConfig, } impl AuthorityAggregator { - pub fn new(committee: Committee, authority_clients: BTreeMap) -> Self { - Self::new_with_timeouts(committee, authority_clients, Default::default()) + pub fn new( + committee: Committee, + authority_clients: BTreeMap, + metrics: GatewayMetrics, + ) -> Self { + Self::new_with_timeouts(committee, authority_clients, metrics, Default::default()) } pub fn new_with_timeouts( committee: Committee, authority_clients: BTreeMap, + metrics: GatewayMetrics, timeouts: TimeoutConfig, ) -> Self { Self { @@ -79,7 +84,7 @@ impl AuthorityAggregator { .into_iter() .map(|(name, api)| (name, SafeClient::new(api, committee.clone(), name))) .collect(), - metrics: &METRICS, + metrics, timeouts, } } diff --git a/crates/sui-core/src/checkpoints/tests/checkpoint_tests.rs b/crates/sui-core/src/checkpoints/tests/checkpoint_tests.rs index f6cfcbed578af..e41ff00066bfc 100644 --- a/crates/sui-core/src/checkpoints/tests/checkpoint_tests.rs +++ b/crates/sui-core/src/checkpoints/tests/checkpoint_tests.rs @@ -10,6 +10,7 @@ use crate::{ }, authority_batch::batch_tests::init_state_parameters_from_rng, authority_client::LocalAuthorityClient, + gateway_state::GatewayMetrics, }; use rand::prelude::StdRng; use rand::SeedableRng; @@ -1471,6 +1472,7 @@ pub async fn checkpoint_tests_setup(num_objects: usize, batch_interval: Duration ) }) .collect(), + GatewayMetrics::new_for_tests(), ); TestSetup { diff --git a/crates/sui-core/src/epoch/reconfiguration.rs b/crates/sui-core/src/epoch/reconfiguration.rs index 08a60b0c88e47..bfb4827764146 100644 --- a/crates/sui-core/src/epoch/reconfiguration.rs +++ b/crates/sui-core/src/epoch/reconfiguration.rs @@ -100,6 +100,7 @@ where let new_net = Arc::new(AuthorityAggregator::new( new_committee, self.net.load().clone_inner_clients(), + self.gateway_metrics.clone(), )); self.net.store(new_net.clone()); // TODO: Also reconnect network if changed. diff --git a/crates/sui-core/src/epoch/tests/reconfiguration_tests.rs b/crates/sui-core/src/epoch/tests/reconfiguration_tests.rs index 973e7c816b333..e75396221494f 100644 --- a/crates/sui-core/src/epoch/tests/reconfiguration_tests.rs +++ b/crates/sui-core/src/epoch/tests/reconfiguration_tests.rs @@ -20,13 +20,13 @@ use sui_types::{ SUI_SYSTEM_STATE_OBJECT_ID, }; -use crate::transaction_input_checker::InputObjects; use crate::{ authority::AuthorityTemporaryStore, authority_active::ActiveAuthority, authority_aggregator::authority_aggregator_tests::init_local_authorities, checkpoints::CheckpointLocals, epoch::reconfiguration::CHECKPOINT_COUNT_PER_EPOCH, execution_engine, }; +use crate::{gateway_state::GatewayMetrics, transaction_input_checker::InputObjects}; #[tokio::test] async fn test_start_epoch_change() { @@ -62,6 +62,7 @@ async fn test_start_epoch_change() { let active = ActiveAuthority::new_with_ephemeral_follower_store( state.clone(), net.clone_inner_clients(), + GatewayMetrics::new_for_tests(), ) .unwrap(); // Make the high watermark differ from low watermark. @@ -180,6 +181,7 @@ async fn test_finish_epoch_change() { ActiveAuthority::new_with_ephemeral_follower_store( state.clone(), net.clone_inner_clients(), + GatewayMetrics::new_for_tests(), ) .unwrap() }) diff --git a/crates/sui-core/src/gateway_state.rs b/crates/sui-core/src/gateway_state.rs index d33b0e09882db..4f4d35f629945 100644 --- a/crates/sui-core/src/gateway_state.rs +++ b/crates/sui-core/src/gateway_state.rs @@ -13,9 +13,8 @@ use async_trait::async_trait; use futures::future; use move_bytecode_utils::module_cache::SyncModuleCache; use move_core_types::identifier::Identifier; -use once_cell::sync::Lazy; -use prometheus_exporter::prometheus::{ - register_histogram, register_int_counter, Histogram, IntCounter, +use prometheus::{ + register_histogram_with_registry, register_int_counter_with_registry, Histogram, IntCounter, }; use tracing::{debug, error, Instrument}; @@ -63,6 +62,7 @@ pub type GatewayTxSeqNumber = u64; const MAX_NUM_TX_RETRIES: usize = 5; /// Prometheus metrics which can be displayed in Grafana, queried and alerted on +#[derive(Clone)] pub struct GatewayMetrics { total_tx_processed: IntCounter, total_tx_errored: IntCounter, @@ -85,93 +85,101 @@ const POSITIVE_INT_BUCKETS: &[f64] = &[ ]; impl GatewayMetrics { - pub fn new() -> GatewayMetrics { + pub fn new(registry: &prometheus::Registry) -> Self { Self { - total_tx_processed: register_int_counter!( + total_tx_processed: register_int_counter_with_registry!( "total_tx_processed", - "Total number of transaction certificates processed in Gateway" + "Total number of transaction certificates processed in Gateway", + registry, ) .unwrap(), - total_tx_errored: register_int_counter!( + total_tx_errored: register_int_counter_with_registry!( "total_tx_errored", - "Total number of transactions which errored out" + "Total number of transactions which errored out", + registry, ) .unwrap(), // total_effects == total transactions finished - num_tx_publish: register_int_counter!( + num_tx_publish: register_int_counter_with_registry!( "num_tx_publish", "Number of publish transactions", + registry, ) .unwrap(), - num_tx_movecall: register_int_counter!( + num_tx_movecall: register_int_counter_with_registry!( "num_tx_movecall", "Number of MOVE call transactions", + registry, ) .unwrap(), - num_tx_splitcoin: register_int_counter!( + num_tx_splitcoin: register_int_counter_with_registry!( "num_tx_splitcoin", "Number of split coin transactions", + registry, ) .unwrap(), - num_tx_mergecoin: register_int_counter!( + num_tx_mergecoin: register_int_counter_with_registry!( "num_tx_mergecoin", "Number of merge coin transactions", + registry, ) .unwrap(), - total_tx_certificates: register_int_counter!( + total_tx_certificates: register_int_counter_with_registry!( "total_tx_certificates", "Total number of certificates made from validators", + registry, ) .unwrap(), - total_tx_retries: register_int_counter!( + total_tx_retries: register_int_counter_with_registry!( "total_tx_retries", "Total number of retries for transactions", + registry, ) .unwrap(), - shared_obj_tx: register_int_counter!( + shared_obj_tx: register_int_counter_with_registry!( "gateway_shared_obj_tx", - "Number of transactions involving shared objects" + "Number of transactions involving shared objects", + registry, ) .unwrap(), // It's really important to use the right histogram buckets for accurate histogram collection. // Otherwise values get clipped - num_signatures: register_histogram!( + num_signatures: register_histogram_with_registry!( "num_signatures_per_tx", "Number of signatures collected per transaction", - POSITIVE_INT_BUCKETS.to_vec() + POSITIVE_INT_BUCKETS.to_vec(), + registry, ) .unwrap(), - num_good_stake: register_histogram!( + num_good_stake: register_histogram_with_registry!( "num_good_stake_per_tx", "Amount of good stake collected per transaction", - POSITIVE_INT_BUCKETS.to_vec() + POSITIVE_INT_BUCKETS.to_vec(), + registry, ) .unwrap(), - num_bad_stake: register_histogram!( + num_bad_stake: register_histogram_with_registry!( "num_bad_stake_per_tx", "Amount of bad stake collected per transaction", - POSITIVE_INT_BUCKETS.to_vec() + POSITIVE_INT_BUCKETS.to_vec(), + registry, ) .unwrap(), - transaction_latency: register_histogram!( + transaction_latency: register_histogram_with_registry!( "transaction_latency", - "Latency of execute_transaction_impl" + "Latency of execute_transaction_impl", + registry, ) .unwrap(), } } -} -impl Default for GatewayMetrics { - fn default() -> Self { - Self::new() + pub fn new_for_tests() -> Self { + let registry = prometheus::Registry::new(); + Self::new(®istry) } } -// One cannot register a metric multiple times. We protect initialization with lazy_static -// for cases such as local tests or "sui start" which starts multiple authorities in one process. -pub static METRICS: Lazy = Lazy::new(GatewayMetrics::new); - pub struct GatewayState { authorities: AuthorityAggregator, store: Arc, @@ -181,7 +189,7 @@ pub struct GatewayState { /// It's useful if we need some kind of ordering for transactions /// from a gateway. next_tx_seq_number: AtomicU64, - metrics: &'static GatewayMetrics, + metrics: GatewayMetrics, module_cache: SyncModuleCache>, } @@ -191,13 +199,19 @@ impl GatewayState { path: PathBuf, committee: Committee, authority_clients: BTreeMap, + metrics: GatewayMetrics, ) -> SuiResult { - Self::new_with_authorities(path, AuthorityAggregator::new(committee, authority_clients)) + Self::new_with_authorities( + path, + AuthorityAggregator::new(committee, authority_clients, metrics.clone()), + metrics, + ) } pub fn new_with_authorities( path: PathBuf, authorities: AuthorityAggregator, + metrics: GatewayMetrics, ) -> SuiResult { let store = Arc::new(GatewayStore::open(path, None)); let next_tx_seq_number = AtomicU64::new(store.next_sequence_number()?); @@ -205,7 +219,7 @@ impl GatewayState { store: store.clone(), authorities, next_tx_seq_number, - metrics: &METRICS, + metrics, module_cache: SyncModuleCache::new(ResolverWrapper(store)), }) } diff --git a/crates/sui-core/src/unit_tests/authority_aggregator_tests.rs b/crates/sui-core/src/unit_tests/authority_aggregator_tests.rs index 552e9a7703129..bc880276c2aac 100644 --- a/crates/sui-core/src/unit_tests/authority_aggregator_tests.rs +++ b/crates/sui-core/src/unit_tests/authority_aggregator_tests.rs @@ -78,7 +78,12 @@ pub async fn init_local_authorities_with_genesis( post_quorum_timeout: Duration::from_secs(5), }; ( - AuthorityAggregator::new_with_timeouts(committee, clients, timeouts), + AuthorityAggregator::new_with_timeouts( + committee, + clients, + GatewayMetrics::new_for_tests(), + timeouts, + ), states, ) } diff --git a/crates/sui-core/src/unit_tests/gateway_state_tests.rs b/crates/sui-core/src/unit_tests/gateway_state_tests.rs index 057d0138d184e..9263edf94c25f 100644 --- a/crates/sui-core/src/unit_tests/gateway_state_tests.rs +++ b/crates/sui-core/src/unit_tests/gateway_state_tests.rs @@ -33,7 +33,9 @@ async fn create_gateway_state( .collect(); let (authorities, _) = init_local_authorities(genesis_objects).await; let path = tempfile::tempdir().unwrap().into_path(); - let gateway = GatewayState::new_with_authorities(path, authorities).unwrap(); + let gateway = + GatewayState::new_with_authorities(path, authorities, GatewayMetrics::new_for_tests()) + .unwrap(); for owner in all_owners { gateway.sync_account_state(owner).await.unwrap(); } @@ -658,7 +660,12 @@ async fn test_multiple_gateways() { let gateway1 = create_gateway_state(genesis_objects).await; let path = tempfile::tempdir().unwrap().into_path(); // gateway2 shares the same set of authorities as gateway1. - let gateway2 = GatewayState::new_with_authorities(path, gateway1.authorities.clone()).unwrap(); + let gateway2 = GatewayState::new_with_authorities( + path, + gateway1.authorities.clone(), + GatewayMetrics::new_for_tests(), + ) + .unwrap(); let response = transfer_coin( &gateway1, addr1, diff --git a/crates/sui-gateway/Cargo.toml b/crates/sui-gateway/Cargo.toml index 5206983496b56..df61da2dec5d7 100644 --- a/crates/sui-gateway/Cargo.toml +++ b/crates/sui-gateway/Cargo.toml @@ -21,7 +21,7 @@ tokio = { version = "1.18.2", features = ["full"] } tokio-stream = "0.1.8" futures = "0.3.21" ed25519-dalek = { version = "1.0.1", features = ["batch", "serde"] } -prometheus_exporter = "0.8.4" +prometheus = "0.13.1" strum = "0.24.0" strum_macros = "0.24.0" once_cell = "1.11.0" diff --git a/crates/sui-gateway/src/config.rs b/crates/sui-gateway/src/config.rs index 9188cac7313ff..2400fb24f021f 100644 --- a/crates/sui-gateway/src/config.rs +++ b/crates/sui-gateway/src/config.rs @@ -12,6 +12,7 @@ use std::{ }; use sui_config::Config; use sui_config::ValidatorInfo; +use sui_core::gateway_state::GatewayMetrics; use sui_core::{ authority_client::NetworkAuthorityClient, gateway_state::{GatewayClient, GatewayState}, @@ -66,7 +67,13 @@ impl GatewayType { let path = config.db_folder_path.clone(); let committee = config.make_committee(); let authority_clients = config.make_authority_clients(); - Arc::new(GatewayState::new(path, committee, authority_clients)?) + let metrics = GatewayMetrics::new(&prometheus::Registry::new()); + Arc::new(GatewayState::new( + path, + committee, + authority_clients, + metrics, + )?) } GatewayType::RPC(url) => Arc::new(RpcGatewayClient::new(url.clone())?), }) diff --git a/crates/sui-gateway/src/lib.rs b/crates/sui-gateway/src/lib.rs index b2dac75045740..7dcfff680763d 100644 --- a/crates/sui-gateway/src/lib.rs +++ b/crates/sui-gateway/src/lib.rs @@ -6,12 +6,15 @@ use anyhow::anyhow; use std::path::Path; use std::sync::Arc; use sui_config::PersistedConfig; -use sui_core::gateway_state::{GatewayClient, GatewayState}; +use sui_core::gateway_state::{GatewayClient, GatewayMetrics, GatewayState}; pub mod config; pub mod rpc_gateway_client; -pub fn create_client(config_path: &Path) -> Result { +pub fn create_client( + config_path: &Path, + gateway_metrics: GatewayMetrics, +) -> Result { let config: GatewayConfig = PersistedConfig::read(config_path).map_err(|e| { anyhow!( "Failed to read config file at {:?}: {}. Have you run `sui genesis` first?", @@ -25,5 +28,6 @@ pub fn create_client(config_path: &Path) -> Result config.db_folder_path, committee, authority_clients, + gateway_metrics, )?)) } diff --git a/crates/sui-json-rpc/Cargo.toml b/crates/sui-json-rpc/Cargo.toml index 5ec41fa72412d..baf43e84e1dc8 100644 --- a/crates/sui-json-rpc/Cargo.toml +++ b/crates/sui-json-rpc/Cargo.toml @@ -10,7 +10,6 @@ edition = "2021" jsonrpsee = { version = "0.13.1", features = ["full"] } jsonrpsee-core = "0.13.1" once_cell = "1.12.0" -prometheus_exporter = "0.8.4" prometheus = "0.13.1" anyhow = "1.0.57" sui-open-rpc = { path = "../sui-open-rpc" } diff --git a/crates/sui-node/src/lib.rs b/crates/sui-node/src/lib.rs index 4df3a6416346b..f0163ceea9175 100644 --- a/crates/sui-node/src/lib.rs +++ b/crates/sui-node/src/lib.rs @@ -27,7 +27,7 @@ use sui_json_rpc::read_api::FullNodeApi; use sui_json_rpc::read_api::ReadApi; use sui_json_rpc_api::EventApiServer; -mod metrics; +pub mod metrics; pub struct SuiNode { grpc_server: tokio::task::JoinHandle>, @@ -116,10 +116,13 @@ impl SuiNode { authority_clients.insert(validator.public_key(), client); } + let gateway_metrics = + sui_core::gateway_state::GatewayMetrics::new(&prometheus_registry); let active_authority = Arc::new(ActiveAuthority::new( state.clone(), follower_store, authority_clients, + gateway_metrics, )?); Some(if is_validator { diff --git a/crates/sui/Cargo.toml b/crates/sui/Cargo.toml index a0d25a2066488..f0f98def46f9b 100644 --- a/crates/sui/Cargo.toml +++ b/crates/sui/Cargo.toml @@ -34,7 +34,6 @@ serde-name = "0.2.1" dirs = "4.0.0" clap = { version = "3.1.17", features = ["derive"] } telemetry-subscribers = { git = "https://github.com/MystenLabs/mysten-infra", rev = "ff5c1d69057fe93be658377462ca2875a57a0223" } -prometheus_exporter = "0.8.4" prometheus = "0.13.1" multiaddr = "0.14.0" mysten-network = { git = "https://github.com/MystenLabs/mysten-infra", rev = "ff5c1d69057fe93be658377462ca2875a57a0223" } diff --git a/crates/sui/src/bin/rpc-server.rs b/crates/sui/src/bin/rpc-server.rs index ae74f9313f7ed..c26249e2a7752 100644 --- a/crates/sui/src/bin/rpc-server.rs +++ b/crates/sui/src/bin/rpc-server.rs @@ -8,6 +8,7 @@ use std::{ }; use sui_config::sui_config_dir; use sui_config::SUI_GATEWAY_CONFIG; +use sui_core::gateway_state::GatewayMetrics; use sui_gateway::create_client; use sui_json_rpc::bcs_api::BcsApiImpl; use sui_json_rpc::gateway_api::RpcGatewayImpl; @@ -53,13 +54,14 @@ async fn main() -> anyhow::Result<()> { info!(?config_path, "Gateway config file path"); let prom_binding = PROM_PORT_ADDR.parse().unwrap(); - info!("Starting Prometheus HTTP endpoint at {}", PROM_PORT_ADDR); - prometheus_exporter::start(prom_binding).expect("Failed to start Prometheus exporter"); + info!("Starting Prometheus HTTP endpoint at {}", prom_binding); + let prometheus_registry = sui_node::metrics::start_prometheus_server(prom_binding); - let client = create_client(&config_path)?; + let metrics = GatewayMetrics::new(&prometheus_registry); + let client = create_client(&config_path, metrics)?; let address = SocketAddr::new(IpAddr::V4(options.host), options.port); - let mut server = JsonRpcServerBuilder::new(prometheus::default_registry())?; + let mut server = JsonRpcServerBuilder::new(&prometheus_registry)?; server.register_module(RpcGatewayImpl::new(client.clone()))?; server.register_module(GatewayReadApiImpl::new(client.clone()))?; server.register_module(TransactionBuilderImpl::new(client.clone()))?; diff --git a/crates/sui/tests/checkpoints_tests.rs b/crates/sui/tests/checkpoints_tests.rs index 63fca7287af4a..fcd8b37db6f3f 100644 --- a/crates/sui/tests/checkpoints_tests.rs +++ b/crates/sui/tests/checkpoints_tests.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use sui_core::{ authority::AuthorityState, authority_active::{checkpoint_driver::CheckpointProcessControl, ActiveAuthority}, + gateway_state::GatewayMetrics, }; use sui_types::{ base_types::{ExecutionDigests, TransactionDigest}, @@ -144,7 +145,12 @@ async fn end_to_end() { let clients = aggregator.clone_inner_clients(); let _active_authority_handle = tokio::spawn(async move { let active_state = Arc::new( - ActiveAuthority::new_with_ephemeral_follower_store(state, clients).unwrap(), + ActiveAuthority::new_with_ephemeral_follower_store( + state, + clients, + GatewayMetrics::new_for_tests(), + ) + .unwrap(), ); let checkpoint_process_control = CheckpointProcessControl { long_pause_between_checkpoints: Duration::from_millis(10), @@ -229,7 +235,12 @@ async fn checkpoint_with_shared_objects() { let clients = aggregator.clone_inner_clients(); let _active_authority_handle = tokio::spawn(async move { let active_state = Arc::new( - ActiveAuthority::new_with_ephemeral_follower_store(state, clients).unwrap(), + ActiveAuthority::new_with_ephemeral_follower_store( + state, + clients, + GatewayMetrics::new_for_tests(), + ) + .unwrap(), ); let checkpoint_process_control = CheckpointProcessControl { long_pause_between_checkpoints: Duration::from_millis(10), diff --git a/crates/sui/tests/shared_objects_tests.rs b/crates/sui/tests/shared_objects_tests.rs index 4a7ead668e9c0..bf7f0ea44a863 100644 --- a/crates/sui/tests/shared_objects_tests.rs +++ b/crates/sui/tests/shared_objects_tests.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use sui_core::authority_client::AuthorityAPI; -use sui_core::gateway_state::{GatewayAPI, GatewayState}; +use sui_core::gateway_state::{GatewayAPI, GatewayMetrics, GatewayState}; use sui_types::messages::{ CallArg, ExecutionStatus, ObjectArg, ObjectInfoRequest, ObjectInfoRequestKind, }; @@ -369,7 +369,9 @@ async fn shared_object_on_gateway() { let _handles = spawn_test_authorities(gas_objects.clone(), &configs).await; let clients = test_authority_aggregator(&configs); let path = tempfile::tempdir().unwrap().into_path(); - let gateway = Arc::new(GatewayState::new_with_authorities(path, clients).unwrap()); + let gateway = Arc::new( + GatewayState::new_with_authorities(path, clients, GatewayMetrics::new_for_tests()).unwrap(), + ); // Publish the move package to all authorities and get the new package ref. tokio::task::yield_now().await; diff --git a/crates/test-utils/Cargo.toml b/crates/test-utils/Cargo.toml index 896fe1a706e80..3fb39f367a54d 100644 --- a/crates/test-utils/Cargo.toml +++ b/crates/test-utils/Cargo.toml @@ -20,6 +20,7 @@ tempfile = "3.3.0" bcs = "0.1.3" jsonrpsee-http-server = "0.13.1" jsonrpsee-http-client = "0.13.1" +prometheus = "0.13.1" sui-adapter = { path = "../sui-adapter" } sui-framework = { path = "../sui-framework" } diff --git a/crates/test-utils/src/authority.rs b/crates/test-utils/src/authority.rs index 75e9d8a2bcb2f..12430888fb22e 100644 --- a/crates/test-utils/src/authority.rs +++ b/crates/test-utils/src/authority.rs @@ -81,7 +81,8 @@ pub fn test_authority_aggregator( ) }) .collect(); - AuthorityAggregator::new(committee, clients) + let metrics = sui_core::gateway_state::GatewayMetrics::new(&prometheus::Registry::new()); + AuthorityAggregator::new(committee, clients, metrics) } /// Get a network client to communicate with the consensus. diff --git a/crates/test-utils/src/network.rs b/crates/test-utils/src/network.rs index c6d1b397dd4c8..5719460c82756 100644 --- a/crates/test-utils/src/network.rs +++ b/crates/test-utils/src/network.rs @@ -14,6 +14,7 @@ use sui::{ use sui_config::genesis_config::GenesisConfig; use sui_config::PersistedConfig; use sui_config::{Config, SUI_GATEWAY_CONFIG, SUI_NETWORK_CONFIG, SUI_WALLET_CONFIG}; +use sui_core::gateway_state::GatewayMetrics; use sui_gateway::create_client; use sui_json_rpc::gateway_api::{GatewayReadApiImpl, RpcGatewayImpl, TransactionBuilderImpl}; use sui_json_rpc_api::QuorumDriverApiServer; @@ -107,7 +108,8 @@ async fn start_rpc_gateway( ) -> Result<(SocketAddr, HttpServerHandle), anyhow::Error> { let server = HttpServerBuilder::default().build("127.0.0.1:0").await?; let addr = server.local_addr()?; - let client = create_client(config_path)?; + let metrics = GatewayMetrics::new(&prometheus::Registry::new()); + let client = create_client(config_path, metrics)?; let mut module = RpcModule::new(()); module.merge(RpcGatewayImpl::new(client.clone()).into_rpc())?; module.merge(GatewayReadApiImpl::new(client.clone()).into_rpc())?; diff --git a/crates/workspace-hack/Cargo.toml b/crates/workspace-hack/Cargo.toml index f16da68e8c272..b6da24f0fae4d 100644 --- a/crates/workspace-hack/Cargo.toml +++ b/crates/workspace-hack/Cargo.toml @@ -35,7 +35,6 @@ ark-std = { version = "0.3", features = ["parallel", "rayon", "std"] } arrayref = { version = "0.3", default-features = false } arrayvec-d8f496e17d97b5cb = { package = "arrayvec", version = "0.5", features = ["array-sizes-33-128", "std"] } arrayvec-ca01ad9e24f5d932 = { package = "arrayvec", version = "0.7", features = ["std"] } -ascii = { version = "1", features = ["std"] } async-lock = { version = "2", default-features = false } async-stream = { version = "0.3", default-features = false } atoi = { version = "0.4", default-features = false } @@ -76,7 +75,6 @@ cfg-if-c65f7effa3be6d31 = { package = "cfg-if", version = "0.1", default-feature cfg-if-dff4ba8e3ae991db = { package = "cfg-if", version = "1", default-features = false } chrono = { version = "0.4", features = ["clock", "libc", "oldtime", "std", "time", "winapi"] } chrono-tz = { version = "0.6", features = ["std"] } -chunked_transfer = { version = "1", default-features = false } clap-f595c2ba2a3f28df = { package = "clap", version = "2", features = ["ansi_term", "atty", "color", "strsim", "suggestions", "vec_map"] } clap-7b89eefb6aaa9bf3 = { package = "clap", version = "3", features = ["atty", "clap_derive", "color", "derive", "lazy_static", "std", "strsim", "suggestions", "termcolor"] } clap_lex = { version = "0.2", default-features = false } @@ -340,10 +338,10 @@ pretty = { version = "0.10", default-features = false } pretty_assertions = { version = "1", features = ["std"] } primary = { git = "https://github.com/MystenLabs/narwhal", rev = "2c5e8236c0702a3ff47dd769c2bbc94b029bf4a9", default-features = false, features = ["benchmark"] } proc-macro2 = { version = "1", features = ["proc-macro", "span-locations"] } -prometheus = { version = "0.13", default-features = false } -prometheus_exporter = { version = "0.8", features = ["internal_metrics", "lazy_static", "log", "logging"] } +prometheus = { version = "0.13", features = ["protobuf"] } prost = { version = "0.10", features = ["prost-derive", "std"] } prost-types = { version = "0.10", features = ["std"] } +protobuf = { version = "2", default-features = false } ptree = { version = "0.4", features = ["ansi", "ansi_term", "atty", "conf", "config", "directories", "petgraph", "serde-value", "tint", "value"] } quote = { version = "1", features = ["proc-macro"] } radix_trie = { version = "0.2", default-features = false } @@ -443,9 +441,8 @@ thread_local = { version = "1", default-features = false } threadpool = { version = "1", default-features = false } thrift = { version = "0.15", default-features = false } time-c65f7effa3be6d31 = { package = "time", version = "0.1", default-features = false } -time-468e82937335b1c9 = { package = "time", version = "0.3", features = ["alloc", "formatting", "itoa", "macros", "parsing", "std", "time-macros"] } +time-468e82937335b1c9 = { package = "time", version = "0.3", features = ["alloc", "formatting", "itoa", "std"] } tint = { version = "1", default-features = false } -tiny_http = { version = "0.10", default-features = false } tinyvec = { version = "1", features = ["alloc", "tinyvec_macros"] } tinyvec_macros = { version = "0.1", default-features = false } tokio = { version = "1", features = ["bytes", "fs", "full", "io-std", "io-util", "libc", "macros", "memchr", "mio", "net", "num_cpus", "once_cell", "parking_lot", "process", "rt", "rt-multi-thread", "signal", "signal-hook-registry", "socket2", "sync", "test-util", "time", "tokio-macros", "tracing"] } @@ -542,7 +539,6 @@ ark-std = { version = "0.3", features = ["parallel", "rayon", "std"] } arrayref = { version = "0.3", default-features = false } arrayvec-d8f496e17d97b5cb = { package = "arrayvec", version = "0.5", features = ["array-sizes-33-128", "std"] } arrayvec-ca01ad9e24f5d932 = { package = "arrayvec", version = "0.7", features = ["std"] } -ascii = { version = "1", features = ["std"] } async-lock = { version = "2", default-features = false } async-recursion = { version = "1", default-features = false } async-stream = { version = "0.3", default-features = false } @@ -593,7 +589,6 @@ cfg-if-dff4ba8e3ae991db = { package = "cfg-if", version = "1", default-features chrono = { version = "0.4", features = ["clock", "libc", "oldtime", "std", "time", "winapi"] } chrono-tz = { version = "0.6", features = ["std"] } chrono-tz-build = { version = "0.0.2", default-features = false } -chunked_transfer = { version = "1", default-features = false } clang-sys = { version = "1", default-features = false, features = ["clang_3_5", "clang_3_6", "clang_3_7", "clang_3_8", "clang_3_9", "clang_4_0", "clang_5_0", "clang_6_0", "libloading", "runtime"] } clap-f595c2ba2a3f28df = { package = "clap", version = "2", features = ["ansi_term", "atty", "color", "strsim", "suggestions", "vec_map"] } clap-7b89eefb6aaa9bf3 = { package = "clap", version = "3", features = ["atty", "clap_derive", "color", "derive", "lazy_static", "std", "strsim", "suggestions", "termcolor"] } @@ -901,12 +896,12 @@ proc-macro-crate = { version = "1", default-features = false } proc-macro-error = { version = "1", features = ["syn", "syn-error"] } proc-macro-error-attr = { version = "1", default-features = false } proc-macro2 = { version = "1", features = ["proc-macro", "span-locations"] } -prometheus = { version = "0.13", default-features = false } -prometheus_exporter = { version = "0.8", features = ["internal_metrics", "lazy_static", "log", "logging"] } +prometheus = { version = "0.13", features = ["protobuf"] } prost = { version = "0.10", features = ["prost-derive", "std"] } prost-build = { version = "0.10" } prost-derive = { version = "0.10", default-features = false } prost-types = { version = "0.10", features = ["std"] } +protobuf = { version = "2", default-features = false } ptree = { version = "0.4", features = ["ansi", "ansi_term", "atty", "conf", "config", "directories", "petgraph", "serde-value", "tint", "value"] } quote = { version = "1", features = ["proc-macro"] } radix_trie = { version = "0.2", default-features = false } @@ -1025,10 +1020,8 @@ thread_local = { version = "1", default-features = false } threadpool = { version = "1", default-features = false } thrift = { version = "0.15", default-features = false } time-c65f7effa3be6d31 = { package = "time", version = "0.1", default-features = false } -time-468e82937335b1c9 = { package = "time", version = "0.3", features = ["alloc", "formatting", "itoa", "macros", "parsing", "std", "time-macros"] } -time-macros = { version = "0.2", default-features = false } +time-468e82937335b1c9 = { package = "time", version = "0.3", features = ["alloc", "formatting", "itoa", "std"] } tint = { version = "1", default-features = false } -tiny_http = { version = "0.10", default-features = false } tinyvec = { version = "1", features = ["alloc", "tinyvec_macros"] } tinyvec_macros = { version = "0.1", default-features = false } tokio = { version = "1", features = ["bytes", "fs", "full", "io-std", "io-util", "libc", "macros", "memchr", "mio", "net", "num_cpus", "once_cell", "parking_lot", "process", "rt", "rt-multi-thread", "signal", "signal-hook-registry", "socket2", "sync", "test-util", "time", "tokio-macros", "tracing"] }