Pull streamer out into its own module. (solana-labs#8917)

automerge

Cargo.toml

@@ -26,6 +26,7 @@ members = [
     "logger",
     "log-analyzer",
     "merkle-tree",
+    "streamer",
     "measure",
     "metrics",
     "net-shaper",

archiver-lib/Cargo.toml

@@ -29,6 +29,7 @@ solana-logger = { path = "../logger", version = "1.1.0" }
 solana-perf = { path = "../perf", version = "1.1.0" }
 solana-sdk = { path = "../sdk", version = "1.1.0" }
 solana-core = { path = "../core", version = "1.1.0" }
+solana-streamer = { path = "../streamer", version = "1.1.0" }
 solana-archiver-utils = { path = "../archiver-utils", version = "1.1.0" }
 solana-metrics = { path = "../metrics", version = "1.1.0" }
 

archiver-lib/src/archiver.rs

@@ -12,21 +12,20 @@ use solana_core::{
     cluster_info::{ClusterInfo, Node, VALIDATOR_PORT_RANGE},
     contact_info::ContactInfo,
     gossip_service::GossipService,
-    packet::{limited_deserialize, PACKET_DATA_SIZE},
     repair_service,
     repair_service::{RepairService, RepairSlotRange, RepairStrategy},
     serve_repair::ServeRepair,
     shred_fetch_stage::ShredFetchStage,
     sigverify_stage::{DisabledSigVerifier, SigVerifyStage},
     storage_stage::NUM_STORAGE_SAMPLES,
-    streamer::{receiver, responder, PacketReceiver},
     window_service::WindowService,
 };
 use solana_ledger::{
     blockstore::Blockstore, leader_schedule_cache::LeaderScheduleCache, shred::Shred,
 };
 use solana_net_utils::bind_in_range;
 use solana_perf::packet::Packets;
+use solana_perf::packet::{limited_deserialize, PACKET_DATA_SIZE};
 use solana_perf::recycler::Recycler;
 use solana_sdk::packet::Packet;
 use solana_sdk::{
@@ -45,6 +44,7 @@ use solana_storage_program::{
     storage_contract::StorageContract,
     storage_instruction::{self, StorageAccountType},
 };
+use solana_streamer::streamer::{receiver, responder, PacketReceiver};
 use std::{
     io::{self, ErrorKind},
     net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket},

banking-bench/Cargo.toml

@@ -11,6 +11,8 @@ homepage = "https://solana.com/"
 log = "0.4.6"
 rayon = "1.3.0"
 solana-core = { path = "../core", version = "1.1.0" }
+solana-streamer = { path = "../streamer", version = "1.1.0" }
+solana-perf = { path = "../perf", version = "1.1.0" }
 solana-ledger = { path = "../ledger", version = "1.1.0" }
 solana-logger = { path = "../logger", version = "1.1.0" }
 solana-runtime = { path = "../runtime", version = "1.1.0" }

banking-bench/src/main.rs

@@ -6,12 +6,12 @@ use solana_core::banking_stage::{create_test_recorder, BankingStage};
 use solana_core::cluster_info::ClusterInfo;
 use solana_core::cluster_info::Node;
 use solana_core::genesis_utils::{create_genesis_config, GenesisConfigInfo};
-use solana_core::packet::to_packets_chunked;
 use solana_core::poh_recorder::PohRecorder;
 use solana_core::poh_recorder::WorkingBankEntry;
 use solana_ledger::bank_forks::BankForks;
 use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
 use solana_measure::measure::Measure;
+use solana_perf::packet::to_packets_chunked;
 use solana_runtime::bank::Bank;
 use solana_sdk::hash::Hash;
 use solana_sdk::pubkey::Pubkey;

bench-streamer/Cargo.toml

@@ -10,6 +10,6 @@ homepage = "https://solana.com/"
 [dependencies]
 clap = "2.33.0"
 solana-clap-utils = { path = "../clap-utils", version = "1.1.0" }
-solana-core = { path = "../core", version = "1.1.0" }
+solana-streamer = { path = "../streamer", version = "1.1.0" }
 solana-logger = { path = "../logger", version = "1.1.0" }
 solana-net-utils = { path = "../net-utils", version = "1.1.0" }

bench-streamer/src/main.rs

@@ -1,6 +1,6 @@
 use clap::{crate_description, crate_name, App, Arg};
-use solana_core::packet::{Packet, Packets, PacketsRecycler, PACKET_DATA_SIZE};
-use solana_core::streamer::{receiver, PacketReceiver};
+use solana_streamer::packet::{Packet, Packets, PacketsRecycler, PACKET_DATA_SIZE};
+use solana_streamer::streamer::{receiver, PacketReceiver};
 use std::cmp::max;
 use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket};
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};

core/Cargo.toml

@@ -31,9 +31,7 @@ jsonrpc-derive = "14.0.5"
 jsonrpc-http-server = "14.0.6"
 jsonrpc-pubsub = "14.0.6"
 jsonrpc-ws-server = "14.0.6"
-libc = "0.2.67"
 log = "0.4.8"
-nix = "0.17.0"
 num_cpus = "1.0.0"
 num-traits = "0.2"
 rand = "0.6.5"
@@ -60,6 +58,7 @@ solana-runtime = { path = "../runtime", version = "1.1.0" }
 solana-sdk = { path = "../sdk", version = "1.1.0" }
 solana-stake-program = { path = "../programs/stake", version = "1.1.0" }
 solana-storage-program = { path = "../programs/storage", version = "1.1.0" }
+solana-streamer = { path = "../streamer", version = "1.1.0" }
 solana-vote-program = { path = "../programs/vote", version = "1.1.0" }
 solana-vote-signer = { path = "../vote-signer", version = "1.1.0" }
 solana-sys-tuner = { path = "../sys-tuner", version = "1.1.0" }

core/benches/banking_stage.rs

@@ -10,11 +10,11 @@ use solana_core::banking_stage::{create_test_recorder, BankingStage};
 use solana_core::cluster_info::ClusterInfo;
 use solana_core::cluster_info::Node;
 use solana_core::genesis_utils::{create_genesis_config, GenesisConfigInfo};
-use solana_core::packet::to_packets_chunked;
 use solana_core::poh_recorder::WorkingBankEntry;
 use solana_ledger::blockstore_processor::process_entries;
 use solana_ledger::entry::{next_hash, Entry};
 use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
+use solana_perf::packet::to_packets_chunked;
 use solana_perf::test_tx::test_tx;
 use solana_runtime::bank::Bank;
 use solana_sdk::genesis_config::GenesisConfig;

core/benches/retransmit_stage.rs

@@ -7,11 +7,11 @@ use log::*;
 use solana_core::cluster_info::{ClusterInfo, Node};
 use solana_core::contact_info::ContactInfo;
 use solana_core::genesis_utils::{create_genesis_config, GenesisConfigInfo};
-use solana_core::packet::to_packets_chunked;
 use solana_core::retransmit_stage::retransmitter;
 use solana_ledger::bank_forks::BankForks;
 use solana_ledger::leader_schedule_cache::LeaderScheduleCache;
 use solana_measure::measure::Measure;
+use solana_perf::packet::to_packets_chunked;
 use solana_perf::test_tx::test_tx;
 use solana_runtime::bank::Bank;
 use solana_sdk::pubkey::Pubkey;

core/benches/sigverify_stage.rs

@@ -6,9 +6,9 @@ extern crate test;
 use crossbeam_channel::unbounded;
 use log::*;
 use rand::{thread_rng, Rng};
-use solana_core::packet::to_packets_chunked;
 use solana_core::sigverify::TransactionSigVerifier;
 use solana_core::sigverify_stage::SigVerifyStage;
+use solana_perf::packet::to_packets_chunked;
 use solana_perf::test_tx::test_tx;
 use solana_sdk::hash::Hash;
 use solana_sdk::signature::{Keypair, Signer};

core/src/banking_stage.rs

@@ -3,7 +3,6 @@
 //! can do its processing in parallel with signature verification on the GPU.
 use crate::{
     cluster_info::ClusterInfo,
-    packet::{limited_deserialize, Packet, Packets, PACKETS_PER_BATCH},
     poh_recorder::{PohRecorder, PohRecorderError, WorkingBankEntry},
     poh_service::PohService,
 };
@@ -17,7 +16,11 @@ use solana_ledger::{
 };
 use solana_measure::{measure::Measure, thread_mem_usage};
 use solana_metrics::{inc_new_counter_debug, inc_new_counter_info, inc_new_counter_warn};
-use solana_perf::{cuda_runtime::PinnedVec, perf_libs};
+use solana_perf::{
+    cuda_runtime::PinnedVec,
+    packet::{limited_deserialize, Packet, Packets, PACKETS_PER_BATCH},
+    perf_libs,
+};
 use solana_runtime::{
     accounts_db::ErrorCounters,
     bank::{Bank, TransactionBalancesSet, TransactionProcessResult},
@@ -1011,7 +1014,6 @@ mod tests {
     use crate::{
         cluster_info::Node,
         genesis_utils::{create_genesis_config, GenesisConfigInfo},
-        packet::to_packets,
         poh_recorder::WorkingBank,
         transaction_status_service::TransactionStatusService,
     };
@@ -1023,6 +1025,7 @@ mod tests {
         entry::{next_entry, Entry, EntrySlice},
         get_tmp_ledger_path,
     };
+    use solana_perf::packet::to_packets;
     use solana_runtime::bank::HashAgeKind;
     use solana_sdk::{
         instruction::InstructionError,

core/src/cluster_info.rs

@@ -12,8 +12,6 @@
 //! * layer 2 - Everyone else, if layer 1 is `2^10`, layer 2 should be able to fit `2^20` number of nodes.
 //!
 //! Bank needs to provide an interface for us to query the stake weight
-use crate::packet::limited_deserialize;
-use crate::streamer::{PacketReceiver, PacketSender};
 use crate::{
     contact_info::ContactInfo,
     crds_gossip::CrdsGossip,
@@ -23,9 +21,7 @@ use crate::{
         self, CrdsData, CrdsValue, CrdsValueLabel, EpochSlotsIndex, LowestSlot, SnapshotHash, Vote,
     },
     epoch_slots::EpochSlots,
-    packet::{Packet, PACKET_DATA_SIZE},
     result::{Error, Result},
-    sendmmsg::{multicast, send_mmsg},
     weighted_shuffle::{weighted_best, weighted_shuffle},
 };
 use bincode::{serialize, serialized_size};
@@ -41,7 +37,10 @@ use solana_net_utils::{
     bind_common, bind_common_in_range, bind_in_range, find_available_port_in_range,
     multi_bind_in_range, PortRange,
 };
-use solana_perf::packet::{to_packets_with_destination, Packets, PacketsRecycler};
+use solana_perf::packet::{
+    limited_deserialize, to_packets_with_destination, Packet, Packets, PacketsRecycler,
+    PACKET_DATA_SIZE,
+};
 use solana_rayon_threadlimit::get_thread_count;
 use solana_sdk::hash::Hash;
 use solana_sdk::timing::duration_as_s;
@@ -52,6 +51,8 @@ use solana_sdk::{
     timing::{duration_as_ms, timestamp},
     transaction::Transaction,
 };
+use solana_streamer::sendmmsg::{multicast, send_mmsg};
+use solana_streamer::streamer::{PacketReceiver, PacketSender};
 use std::{
     borrow::Cow,
     cmp::min,

core/src/cluster_info_vote_listener.rs

@@ -1,7 +1,6 @@
 use crate::{
     cluster_info::{ClusterInfo, GOSSIP_SLEEP_MILLIS},
     crds_value::CrdsValueLabel,
-    packet::{self, Packets},
     poh_recorder::PohRecorder,
     result::{Error, Result},
     sigverify,
@@ -14,6 +13,7 @@ use itertools::izip;
 use log::*;
 use solana_ledger::bank_forks::BankForks;
 use solana_metrics::inc_new_counter_debug;
+use solana_perf::packet::{self, Packets};
 use solana_runtime::bank::Bank;
 use solana_sdk::{
     account::Account,
@@ -668,7 +668,7 @@ impl ClusterInfoVoteListener {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::packet;
+    use solana_perf::packet;
     use solana_runtime::{
         bank::Bank,
         genesis_utils::{self, GenesisConfigInfo, ValidatorVoteKeypairs},

core/src/fetch_stage.rs

@@ -1,14 +1,14 @@
 //! The `fetch_stage` batches input from a UDP socket and sends it to a channel.
 
 use crate::banking_stage::FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET;
-use crate::packet::PacketsRecycler;
 use crate::poh_recorder::PohRecorder;
 use crate::result::{Error, Result};
-use crate::streamer::{self, PacketReceiver, PacketSender};
 use solana_measure::thread_mem_usage;
 use solana_metrics::{inc_new_counter_debug, inc_new_counter_info};
+use solana_perf::packet::PacketsRecycler;
 use solana_perf::recycler::Recycler;
 use solana_sdk::clock::DEFAULT_TICKS_PER_SLOT;
+use solana_streamer::streamer::{self, PacketReceiver, PacketSender};
 use std::net::UdpSocket;
 use std::sync::atomic::AtomicBool;
 use std::sync::mpsc::{channel, RecvTimeoutError};

core/src/gossip_service.rs

@@ -2,13 +2,13 @@
 
 use crate::cluster_info::{ClusterInfo, VALIDATOR_PORT_RANGE};
 use crate::contact_info::ContactInfo;
-use crate::streamer;
 use rand::{thread_rng, Rng};
 use solana_client::thin_client::{create_client, ThinClient};
 use solana_ledger::bank_forks::BankForks;
 use solana_perf::recycler::Recycler;
 use solana_sdk::pubkey::Pubkey;
 use solana_sdk::signature::{Keypair, Signer};
+use solana_streamer::streamer;
 use std::net::{IpAddr, Ipv4Addr, SocketAddr, TcpListener, UdpSocket};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::mpsc::channel;

core/src/lib.rs

@@ -32,10 +32,8 @@ pub mod genesis_utils;
 pub mod gossip_service;
 pub mod ledger_cleanup_service;
 pub mod local_vote_signer_service;
-pub mod packet;
 pub mod poh_recorder;
 pub mod poh_service;
-pub mod recvmmsg;
 pub mod repair_service;
 pub mod replay_stage;
 mod result;
@@ -46,15 +44,13 @@ pub mod rpc_pubsub;
 pub mod rpc_pubsub_service;
 pub mod rpc_service;
 pub mod rpc_subscriptions;
-pub mod sendmmsg;
 pub mod serve_repair;
 pub mod serve_repair_service;
 pub mod sigverify;
 pub mod sigverify_shreds;
 pub mod sigverify_stage;
 pub mod snapshot_packager_service;
 pub mod storage_stage;
-pub mod streamer;
 pub mod tpu;
 pub mod transaction_status_service;
 pub mod tvu;