Forward transactions to the expected leader instead of your own TPU p…

…ort (solana-labs#12004)

* Use PoHRecorder to send to the right leader

* cleanup

* fmt

* clippy

* Cleanup, fix bug

Co-authored-by: Carl <carl@solana.com>

banks-server/Cargo.toml

@@ -11,9 +11,11 @@ edition = "2018"
 [dependencies]
 bincode = "1.3.1"
 futures = "0.3"
+log = "0.4.8"
 solana-banks-interface = { path = "../banks-interface", version = "1.4.0" }
 solana-runtime = { path = "../runtime", version = "1.4.0" }
 solana-sdk = { path = "../sdk", version = "1.4.0" }
+solana-metrics = { path = "../metrics", version = "1.4.0" }
 tarpc = { version = "0.21.0", features = ["full"] }
 tokio = "0.2"
 tokio-serde = { version = "0.6", features = ["bincode"] }

banks-server/src/banks_server.rs

@@ -1,3 +1,4 @@
+use crate::send_transaction_service::{SendTransactionService, TransactionInfo};
 use bincode::{deserialize, serialize};
 use futures::{
     future,
@@ -8,7 +9,6 @@ use solana_runtime::{
     bank::Bank,
     bank_forks::BankForks,
     commitment::{BlockCommitmentCache, CommitmentSlots},
-    send_transaction_service::{SendTransactionService, TransactionInfo},
 };
 use solana_sdk::{
     account::Account,

banks-server/src/lib.rs

@@ -1,2 +1,6 @@
 pub mod banks_server;
 pub mod rpc_banks_service;
+pub mod send_transaction_service;
+
+#[macro_use]
+extern crate solana_metrics;

runtime/src/send_transaction_service.rs → banks-server/src/send_transaction_service.rs

@@ -1,6 +1,6 @@
-use crate::{bank::Bank, bank_forks::BankForks};
 use log::*;
 use solana_metrics::{datapoint_warn, inc_new_counter_info};
+use solana_runtime::{bank::Bank, bank_forks::BankForks};
 use solana_sdk::{clock::Slot, signature::Signature};
 use std::{
     collections::HashMap,
@@ -205,8 +205,6 @@ mod test {
 
     #[test]
     fn process_transactions() {
-        solana_logger::setup();
-
         let (genesis_config, mint_keypair) = create_genesis_config(4);
         let bank = Bank::new(&genesis_config);
         let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));

core/src/lib.rs

@@ -58,6 +58,7 @@ pub mod rpc_pubsub;
 pub mod rpc_pubsub_service;
 pub mod rpc_service;
 pub mod rpc_subscriptions;
+pub mod send_transaction_service;
 pub mod serve_repair;
 pub mod serve_repair_service;
 pub mod sigverify;

core/src/rpc.rs

@@ -1,9 +1,13 @@
 //! The `rpc` module implements the Solana RPC interface.
 
 use crate::{
-    cluster_info::ClusterInfo, contact_info::ContactInfo,
-    non_circulating_supply::calculate_non_circulating_supply, rpc_error::RpcCustomError,
-    rpc_health::*, validator::ValidatorExit,
+    cluster_info::ClusterInfo,
+    contact_info::ContactInfo,
+    non_circulating_supply::calculate_non_circulating_supply,
+    rpc_error::RpcCustomError,
+    rpc_health::*,
+    send_transaction_service::{SendTransactionService, TransactionInfo},
+    validator::ValidatorExit,
 };
 use bincode::{config::Options, serialize};
 use jsonrpc_core::{types::error, Error, Metadata, Result};
@@ -36,7 +40,6 @@ use solana_runtime::{
     bank::Bank,
     bank_forks::BankForks,
     commitment::{BlockCommitmentArray, BlockCommitmentCache, CommitmentSlots},
-    send_transaction_service::{SendTransactionService, TransactionInfo},
 };
 use solana_sdk::{
     account::Account,
@@ -213,7 +216,7 @@ impl JsonRpcRequestProcessor {
         let cluster_info = Arc::new(ClusterInfo::default());
         let tpu_address = cluster_info.my_contact_info().tpu;
         let (sender, receiver) = channel();
-        SendTransactionService::new(tpu_address, &bank_forks, &exit, receiver);
+        SendTransactionService::new(tpu_address, &bank_forks, None, &exit, receiver);
 
         Self {
             config: JsonRpcConfig::default(),
@@ -2692,7 +2695,7 @@ pub mod tests {
             &runtime::Runtime::new().unwrap(),
             None,
         );
-        SendTransactionService::new(tpu_address, &bank_forks, &exit, receiver);
+        SendTransactionService::new(tpu_address, &bank_forks, None, &exit, receiver);
 
         cluster_info.insert_info(ContactInfo::new_with_pubkey_socketaddr(
             &leader_pubkey,
@@ -4004,7 +4007,7 @@ pub mod tests {
             &runtime::Runtime::new().unwrap(),
             None,
         );
-        SendTransactionService::new(tpu_address, &bank_forks, &exit, receiver);
+        SendTransactionService::new(tpu_address, &bank_forks, None, &exit, receiver);
 
         let req = r#"{"jsonrpc":"2.0","id":1,"method":"sendTransaction","params":["37u9WtQpcm6ULa3Vmu7ySnANv"]}"#;
         let res = io.handle_request_sync(req, meta);
@@ -4045,7 +4048,7 @@ pub mod tests {
             &runtime::Runtime::new().unwrap(),
             None,
         );
-        SendTransactionService::new(tpu_address, &bank_forks, &exit, receiver);
+        SendTransactionService::new(tpu_address, &bank_forks, None, &exit, receiver);
 
         let mut bad_transaction =
             system_transaction::transfer(&mint_keypair, &Pubkey::new_rand(), 42, Hash::default());
@@ -4227,7 +4230,7 @@ pub mod tests {
             &runtime::Runtime::new().unwrap(),
             None,
         );
-        SendTransactionService::new(tpu_address, &bank_forks, &exit, receiver);
+        SendTransactionService::new(tpu_address, &bank_forks, None, &exit, receiver);
         assert_eq!(request_processor.validator_exit(), false);
         assert_eq!(exit.load(Ordering::Relaxed), false);
     }
@@ -4256,7 +4259,7 @@ pub mod tests {
             &runtime::Runtime::new().unwrap(),
             None,
         );
-        SendTransactionService::new(tpu_address, &bank_forks, &exit, receiver);
+        SendTransactionService::new(tpu_address, &bank_forks, None, &exit, receiver);
         assert_eq!(request_processor.validator_exit(), true);
         assert_eq!(exit.load(Ordering::Relaxed), true);
     }
@@ -4342,7 +4345,7 @@ pub mod tests {
             &runtime::Runtime::new().unwrap(),
             None,
         );
-        SendTransactionService::new(tpu_address, &bank_forks, &exit, receiver);
+        SendTransactionService::new(tpu_address, &bank_forks, None, &exit, receiver);
         assert_eq!(
             request_processor.get_block_commitment(0),
             RpcBlockCommitment {

core/src/rpc_service.rs

@@ -1,8 +1,13 @@
 //! The `rpc_service` module implements the Solana JSON RPC service.
 
 use crate::{
-    bigtable_upload_service::BigTableUploadService, cluster_info::ClusterInfo, rpc::*,
-    rpc_health::*, validator::ValidatorExit,
+    bigtable_upload_service::BigTableUploadService,
+    cluster_info::ClusterInfo,
+    poh_recorder::PohRecorder,
+    rpc::*,
+    rpc_health::*,
+    send_transaction_service::{LeaderInfo, SendTransactionService},
+    validator::ValidatorExit,
 };
 use jsonrpc_core::MetaIoHandler;
 use jsonrpc_http_server::{
@@ -14,7 +19,6 @@ use solana_ledger::blockstore::Blockstore;
 use solana_runtime::{
     bank_forks::{BankForks, SnapshotConfig},
     commitment::BlockCommitmentCache,
-    send_transaction_service::SendTransactionService,
     snapshot_utils,
 };
 use solana_sdk::{hash::Hash, native_token::lamports_to_sol, pubkey::Pubkey};
@@ -23,7 +27,7 @@ use std::{
     net::SocketAddr,
     path::{Path, PathBuf},
     sync::atomic::{AtomicBool, Ordering},
-    sync::{mpsc::channel, Arc, RwLock},
+    sync::{mpsc::channel, Arc, Mutex, RwLock},
     thread::{self, Builder, JoinHandle},
 };
 use tokio::runtime;
@@ -239,6 +243,7 @@ impl JsonRpcService {
         block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
         blockstore: Arc<Blockstore>,
         cluster_info: Arc<ClusterInfo>,
+        poh_recorder: Option<Arc<Mutex<PohRecorder>>>,
         genesis_hash: Hash,
         ledger_path: &Path,
         validator_exit: Arc<RwLock<Option<ValidatorExit>>>,
@@ -302,16 +307,19 @@ impl JsonRpcService {
             blockstore,
             validator_exit.clone(),
             health.clone(),
-            cluster_info,
+            cluster_info.clone(),
             genesis_hash,
             &runtime,
             bigtable_ledger_storage,
         );
 
         let exit_send_transaction_service = Arc::new(AtomicBool::new(false));
+        let leader_info =
+            poh_recorder.map(|recorder| LeaderInfo::new(cluster_info.clone(), recorder));
         let _send_transaction_service = Arc::new(SendTransactionService::new(
             tpu_address,
             &bank_forks,
+            leader_info,
             &exit_send_transaction_service,
             receiver,
         ));
@@ -438,6 +446,7 @@ mod tests {
             block_commitment_cache,
             blockstore,
             cluster_info,
+            None,
             Hash::default(),
             &PathBuf::from("farf"),
             validator_exit,