sui-node: start an anemo network on node startup

crates/sui-node/Cargo.toml

@@ -7,6 +7,8 @@ publish = false
 edition = "2021"
 
 [dependencies]
+anemo.workspace = true
+anemo-tower.workspace = true
 axum = "0.5.16"
 anyhow = { version = "1.0.64", features = ["backtrace"] }
 clap = { version = "3.2.17", features = ["derive"] }
@@ -18,6 +20,7 @@ parking_lot = "0.12.1"
 futures = "0.3.23"
 typed-store.workspace = true
 chrono = "0.4.0"
+tower = "0.4.13"
 
 sui-config = { path = "../sui-config" }
 sui-core = { path = "../sui-core" }
@@ -26,6 +29,7 @@ sui-network = { path = "../sui-network" }
 sui-json-rpc = { path = "../sui-json-rpc" }
 sui-telemetry = { path = "../sui-telemetry" }
 sui-types = { path = "../sui-types" }
+narwhal-network = { path = "../../narwhal/network" }
 
 telemetry-subscribers.workspace = true
 mysten-network.workspace = true

crates/sui-node/src/lib.rs

@@ -1,11 +1,15 @@
 // Copyright (c) Mysten Labs, Inc.
 // SPDX-License-Identifier: Apache-2.0
 
+use anemo_tower::callback::CallbackLayer;
+use anemo_tower::trace::TraceLayer;
 use anyhow::anyhow;
 use anyhow::bail;
 use anyhow::Result;
 use futures::TryFutureExt;
 use mysten_network::server::ServerBuilder;
+use narwhal_network::metrics::MetricsMakeCallbackHandler;
+use narwhal_network::metrics::{NetworkConnectionMetrics, NetworkMetrics};
 use parking_lot::Mutex;
 use prometheus::Registry;
 use std::option::Option::None;
@@ -39,6 +43,7 @@ use sui_storage::{
 };
 use sui_types::messages::{CertifiedTransaction, CertifiedTransactionEffects};
 use tokio::sync::mpsc::channel;
+use tower::ServiceBuilder;
 use tracing::{error, info, warn};
 
 use crate::metrics::GrpcMetrics;
@@ -74,6 +79,8 @@ pub struct SuiNode {
     transaction_orchestrator: Option<Arc<TransactiondOrchestrator<NetworkAuthorityClient>>>,
     _prometheus_registry: Registry,
 
+    _p2p_network: anemo::Network,
+
     #[cfg(msim)]
     sim_node: sui_simulator::runtime::NodeHandle,
 }
@@ -294,6 +301,47 @@ impl SuiNode {
             tokio::spawn(server.serve().map_err(Into::into))
         };
 
+        let p2p_network = {
+            let inbound_network_metrics =
+                NetworkMetrics::new("sui", "inbound", &prometheus_registry);
+            let outbound_network_metrics =
+                NetworkMetrics::new("sui", "outbound", &prometheus_registry);
+            let network_connection_metrics =
+                NetworkConnectionMetrics::new("sui", &prometheus_registry);
+
+            let routes = anemo::Router::new();
+
+            let service = ServiceBuilder::new()
+                .layer(TraceLayer::new())
+                .layer(CallbackLayer::new(MetricsMakeCallbackHandler::new(
+                    Arc::new(inbound_network_metrics),
+                )))
+                .service(routes);
+
+            let outbound_layer = ServiceBuilder::new()
+                .layer(TraceLayer::new())
+                .layer(CallbackLayer::new(MetricsMakeCallbackHandler::new(
+                    Arc::new(outbound_network_metrics),
+                )))
+                .into_inner();
+
+            let network = anemo::Network::bind(config.p2p_config.listen_address)
+                .server_name("sui")
+                .private_key(config.network_key_pair.copy().private().0.to_bytes())
+                .config(config.p2p_config.anemo_config.clone().unwrap_or_default())
+                .outbound_request_layer(outbound_layer)
+                .start(service)?;
+            info!("P2p network started on {}", network.local_addr());
+
+            let _connection_monitor_handle =
+                narwhal_network::connectivity::ConnectionMonitor::spawn(
+                    network.downgrade(),
+                    network_connection_metrics,
+                );
+
+            network
+        };
+
         let (json_rpc_service, ws_subscription_service) = build_http_servers(
             state.clone(),
             &transaction_orchestrator.clone(),
@@ -315,6 +363,7 @@ impl SuiNode {
             active: active_authority,
             transaction_orchestrator,
             _prometheus_registry: prometheus_registry,
+            _p2p_network: p2p_network,
 
             #[cfg(msim)]
             sim_node: sui_simulator::runtime::NodeHandle::current(),