Add execution utilization metrics (#9649)

This PR adds few metrics that help to reason about execution utilization

(a) Utilization for execution threads
(b) Number of certificates that are submitted to execution task but not
dispatched
(c) Utilization timer for transaction manager write lock

crates/sui-core/src/authority.rs

@@ -191,6 +191,7 @@ pub struct AuthorityMetrics {
     pub(crate) transaction_manager_num_ready: IntGauge,
 
     pub(crate) execution_driver_executed_transactions: IntCounter,
+    pub(crate) execution_driver_dispatch_queue: IntGauge,
 
     pub(crate) skipped_consensus_txns: IntCounter,
     pub(crate) skipped_consensus_txns_cache_hit: IntCounter,
@@ -368,6 +369,12 @@ impl AuthorityMetrics {
                 registry,
             )
             .unwrap(),
+            execution_driver_dispatch_queue: register_int_gauge_with_registry!(
+                "execution_driver_dispatch_queue",
+                "Number of transaction pending in execution driver dispatch queue",
+                registry,
+            )
+            .unwrap(),
             skipped_consensus_txns: register_int_counter_with_registry!(
                 "skipped_consensus_txns",
                 "Total number of consensus transactions skipped",

crates/sui-core/src/execution_driver.rs

@@ -6,7 +6,7 @@ use std::{
     time::Duration,
 };
 
-use mysten_metrics::spawn_monitored_task;
+use mysten_metrics::{monitored_scope, spawn_monitored_task};
 use sui_types::messages::VerifiedExecutableTransaction;
 use tokio::{
     sync::{mpsc::UnboundedReceiver, oneshot, Semaphore},
@@ -65,6 +65,7 @@ pub async fn execution_process(
             info!("Authority state has shutdown. Exiting ...");
             return;
         };
+        authority.metrics.execution_driver_dispatch_queue.dec();
 
         // TODO: Ideally execution_driver should own a copy of epoch store and recreate each epoch.
         let epoch_store = authority.load_epoch_store_one_call_per_task();
@@ -79,6 +80,7 @@ pub async fn execution_process(
 
         // Certificate execution can take significant time, so run it in a separate task.
         spawn_monitored_task!(async move {
+            let _scope = monitored_scope("ExecutionDriver");
             let _guard = permit;
             if let Ok(true) = authority.is_tx_already_executed(&digest) {
                 return;

crates/sui-core/src/transaction_manager.rs

@@ -6,6 +6,7 @@ use std::{
     sync::Arc,
 };
 
+use mysten_metrics::monitored_scope;
 use parking_lot::RwLock;
 use sui_types::{
     base_types::ObjectID,
@@ -163,6 +164,7 @@ impl TransactionManager {
 
         // Internal lock is held only for updating the internal state.
         let mut inner = self.inner.write();
+        let _scope = monitored_scope("TransactionManager::enqueue::wlock");
 
         for pending_cert in pending {
             // Tx lock is not held here, which makes it possible to send duplicated transactions to
@@ -298,6 +300,7 @@ impl TransactionManager {
         let mut ready_digests = Vec::new();
 
         let inner = &mut self.inner.write();
+        let _scope = monitored_scope("TransactionManager::objects_available::wlock");
         if inner.epoch != epoch_store.epoch() {
             warn!(
                 "Ignoring objects committed from wrong epoch. Expected={} Actual={} \
@@ -356,6 +359,7 @@ impl TransactionManager {
     ) {
         {
             let inner = &mut self.inner.write();
+            let _scope = monitored_scope("TransactionManager::certificate_executed::wlock");
             if inner.epoch != epoch_store.epoch() {
                 warn!("Ignoring committed certificate from wrong epoch. Expected={} Actual={} CertificateDigest={:?}", inner.epoch, epoch_store.epoch(), digest);
                 return;
@@ -372,6 +376,7 @@ impl TransactionManager {
     fn certificate_ready(&self, certificate: VerifiedExecutableTransaction) {
         self.metrics.transaction_manager_num_ready.inc();
         let _ = self.tx_ready_certificates.send(certificate);
+        self.metrics.execution_driver_dispatch_queue.inc();
     }
 
     /// Gets the missing input object keys for the given transaction.