Network sync refactoring (part 4) (#11412)

* Remove direct dependency of `sc-network` on `sc-network-light`

* Move `WarpSyncProvider` trait and surrounding data structures into `sc-network-common`

* Move `WarpSyncProvider` trait and surrounding data structures into `sc-network-common`

* Create `sync` module in `sc-network-common`, create `ChainSync` trait there (not used yet), move a bunch of associated data structures from `sc-network-sync`

* Switch from concrete implementation to `ChainSync` trait from `sc-network-common`

* Introduce `OpaqueStateRequest`/`OpaqueStateResponse` to remove generics from `StateSync` trait

* Introduce `OpaqueBlockRequest`/`OpaqueBlockResponse`, make `scheme` module of `sc-network-sync` private

* Surface `sc-network-sync` into `sc-service` and make `sc-network` not depend on it anymore

* Remove now unnecessary dependency from `sc-network`

* Replace crate links with just text since dependencies are gone now

* Remove `warp_sync` re-export from `sc-network-common`

* Update copyright in network-related files

* Address review comments about documentation

* Apply review suggestion

* Rename `extra_requests` module to `metrics`

Co-authored-by: Bastian Köcher <info@kchr.de>

client/beefy/src/tests.rs

@@ -28,7 +28,6 @@ use sc_chain_spec::{ChainSpec, GenericChainSpec};
 use sc_client_api::HeaderBackend;
 use sc_consensus::BoxJustificationImport;
 use sc_keystore::LocalKeystore;
-use sc_network::config::ProtocolConfig;
 use sc_network_test::{
 	Block, BlockImportAdapter, FullPeerConfig, PassThroughVerifier, Peer, PeersClient,
 	TestNetFactory,
@@ -111,6 +110,7 @@ pub(crate) struct PeerData {
 	pub(crate) beefy_link_half: Mutex<Option<BeefyLinkHalf>>,
 }
 
+#[derive(Default)]
 pub(crate) struct BeefyTestNet {
 	peers: Vec<BeefyPeer>,
 }
@@ -166,17 +166,7 @@ impl TestNetFactory for BeefyTestNet {
 	type BlockImport = PeersClient;
 	type PeerData = PeerData;
 
-	/// Create new test network with peers and given config.
-	fn from_config(_config: &ProtocolConfig) -> Self {
-		BeefyTestNet { peers: Vec::new() }
-	}
-
-	fn make_verifier(
-		&self,
-		_client: PeersClient,
-		_cfg: &ProtocolConfig,
-		_: &PeerData,
-	) -> Self::Verifier {
+	fn make_verifier(&self, _client: PeersClient, _: &PeerData) -> Self::Verifier {
 		PassThroughVerifier::new(false) // use non-instant finality.
 	}
 

client/consensus/aura/src/lib.rs

@@ -566,7 +566,6 @@ mod tests {
 	use sc_consensus::BoxJustificationImport;
 	use sc_consensus_slots::{BackoffAuthoringOnFinalizedHeadLagging, SimpleSlotWorker};
 	use sc_keystore::LocalKeystore;
-	use sc_network::config::ProtocolConfig;
 	use sc_network_test::{Block as TestBlock, *};
 	use sp_application_crypto::key_types::AURA;
 	use sp_consensus::{
@@ -645,6 +644,7 @@ mod tests {
 	>;
 	type AuraPeer = Peer<(), PeersClient>;
 
+	#[derive(Default)]
 	pub struct AuraTestNet {
 		peers: Vec<AuraPeer>,
 	}
@@ -654,17 +654,7 @@ mod tests {
 		type PeerData = ();
 		type BlockImport = PeersClient;
 
-		/// Create new test network with peers and given config.
-		fn from_config(_config: &ProtocolConfig) -> Self {
-			AuraTestNet { peers: Vec::new() }
-		}
-
-		fn make_verifier(
-			&self,
-			client: PeersClient,
-			_cfg: &ProtocolConfig,
-			_peer_data: &(),
-		) -> Self::Verifier {
+		fn make_verifier(&self, client: PeersClient, _peer_data: &()) -> Self::Verifier {
 			let client = client.as_client();
 			let slot_duration = slot_duration(&*client).expect("slot duration available");
 

client/consensus/babe/src/tests.rs

@@ -29,7 +29,6 @@ use sc_client_api::{backend::TransactionFor, BlockchainEvents, Finalizer};
 use sc_consensus::{BoxBlockImport, BoxJustificationImport};
 use sc_consensus_slots::BackoffAuthoringOnFinalizedHeadLagging;
 use sc_keystore::LocalKeystore;
-use sc_network::config::ProtocolConfig;
 use sc_network_test::{Block as TestBlock, *};
 use sp_application_crypto::key_types::BABE;
 use sp_consensus::{AlwaysCanAuthor, DisableProofRecording, NoNetwork as DummyOracle, Proposal};
@@ -220,6 +219,7 @@ where
 
 type BabePeer = Peer<Option<PeerData>, BabeBlockImport>;
 
+#[derive(Default)]
 pub struct BabeTestNet {
 	peers: Vec<BabePeer>,
 }
@@ -278,12 +278,6 @@ impl TestNetFactory for BabeTestNet {
 	type PeerData = Option<PeerData>;
 	type BlockImport = BabeBlockImport;
 
-	/// Create new test network with peers and given config.
-	fn from_config(_config: &ProtocolConfig) -> Self {
-		debug!(target: "babe", "Creating test network from config");
-		BabeTestNet { peers: Vec::new() }
-	}
-
 	fn make_block_import(
 		&self,
 		client: PeersClient,
@@ -309,12 +303,7 @@ impl TestNetFactory for BabeTestNet {
 		)
 	}
 
-	fn make_verifier(
-		&self,
-		client: PeersClient,
-		_cfg: &ProtocolConfig,
-		maybe_link: &Option<PeerData>,
-	) -> Self::Verifier {
+	fn make_verifier(&self, client: PeersClient, maybe_link: &Option<PeerData>) -> Self::Verifier {
 		use substrate_test_runtime_client::DefaultTestClientBuilderExt;
 
 		let client = client.as_client();

client/finality-grandpa/Cargo.toml

@@ -36,6 +36,7 @@ sc-consensus = { version = "0.10.0-dev", path = "../consensus/common" }
 sc-keystore = { version = "4.0.0-dev", path = "../keystore" }
 sc-network = { version = "0.10.0-dev", path = "../network" }
 sc-network-gossip = { version = "0.10.0-dev", path = "../network-gossip" }
+sc-network-common = { version = "0.10.0-dev", path = "../network/common" }
 sc-telemetry = { version = "4.0.0-dev", path = "../telemetry" }
 sc-utils = { version = "4.0.0-dev", path = "../utils" }
 sp-api = { version = "4.0.0-dev", path = "../../primitives/api" }

client/finality-grandpa/src/tests.rs

@@ -28,7 +28,7 @@ use sc_consensus::{
 	BlockImport, BlockImportParams, BoxJustificationImport, ForkChoiceStrategy, ImportResult,
 	ImportedAux,
 };
-use sc_network::config::{ProtocolConfig, Role};
+use sc_network::config::Role;
 use sc_network_test::{
 	Block, BlockImportAdapter, FullPeerConfig, Hash, PassThroughVerifier, Peer, PeersClient,
 	PeersFullClient, TestClient, TestNetFactory,
@@ -73,6 +73,7 @@ type GrandpaBlockImport = crate::GrandpaBlockImport<
 	LongestChain<substrate_test_runtime_client::Backend, Block>,
 >;
 
+#[derive(Default)]
 struct GrandpaTestNet {
 	peers: Vec<GrandpaPeer>,
 	test_config: TestApi,
@@ -110,16 +111,6 @@ impl TestNetFactory for GrandpaTestNet {
 	type PeerData = PeerData;
 	type BlockImport = GrandpaBlockImport;
 
-	/// Create new test network with peers and given config.
-	fn from_config(_config: &ProtocolConfig) -> Self {
-		GrandpaTestNet { peers: Vec::new(), test_config: Default::default() }
-	}
-
-	fn default_config() -> ProtocolConfig {
-		// This is unused.
-		ProtocolConfig::default()
-	}
-
 	fn add_full_peer(&mut self) {
 		self.add_full_peer_with_config(FullPeerConfig {
 			notifications_protocols: vec![grandpa_protocol_name::NAME.into()],
@@ -128,12 +119,7 @@ impl TestNetFactory for GrandpaTestNet {
 		})
 	}
 
-	fn make_verifier(
-		&self,
-		_client: PeersClient,
-		_cfg: &ProtocolConfig,
-		_: &PeerData,
-	) -> Self::Verifier {
+	fn make_verifier(&self, _client: PeersClient, _: &PeerData) -> Self::Verifier {
 		PassThroughVerifier::new(false) // use non-instant finality.
 	}
 

client/finality-grandpa/src/warp_proof.rs

@@ -23,7 +23,7 @@ use crate::{
 	BlockNumberOps, GrandpaJustification, SharedAuthoritySet,
 };
 use sc_client_api::Backend as ClientBackend;
-use sc_network::warp_request_handler::{EncodedProof, VerificationResult, WarpSyncProvider};
+use sc_network_common::sync::warp::{EncodedProof, VerificationResult, WarpSyncProvider};
 use sp_blockchain::{Backend as BlockchainBackend, HeaderBackend};
 use sp_finality_grandpa::{AuthorityList, SetId, GRANDPA_ENGINE_ID};
 use sp_runtime::{

client/network/Cargo.toml

@@ -51,22 +51,21 @@ sc-block-builder = { version = "0.10.0-dev", path = "../block-builder" }
 sc-client-api = { version = "4.0.0-dev", path = "../api" }
 sc-consensus = { version = "0.10.0-dev", path = "../consensus/common" }
 sc-network-common = { version = "0.10.0-dev", path = "./common" }
-sc-network-light = { version = "0.10.0-dev", path = "./light" }
-sc-network-sync = { version = "0.10.0-dev", path = "./sync" }
 sc-peerset = { version = "4.0.0-dev", path = "../peerset" }
 sc-utils = { version = "4.0.0-dev", path = "../utils" }
 sp-arithmetic = { version = "5.0.0", path = "../../primitives/arithmetic" }
 sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" }
 sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/common" }
 sp-core = { version = "6.0.0", path = "../../primitives/core" }
-sp-finality-grandpa = { version = "4.0.0-dev", path = "../../primitives/finality-grandpa" }
 sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" }
 
 [dev-dependencies]
 assert_matches = "1.3"
 async-std = "1.11.0"
 rand = "0.7.2"
 tempfile = "3.1.0"
+sc-network-light = { version = "0.10.0-dev", path = "./light" }
+sc-network-sync = { version = "0.10.0-dev", path = "./sync" }
 sp-test-primitives = { version = "2.0.0", path = "../../primitives/test-primitives" }
 sp-tracing = { version = "5.0.0", path = "../../primitives/tracing" }
 substrate-test-runtime = { version = "2.0.0", path = "../../test-utils/runtime" }

client/network/common/Cargo.toml

@@ -17,10 +17,15 @@ targets = ["x86_64-unknown-linux-gnu"]
 prost-build = "0.10"
 
 [dependencies]
+bitflags = "1.3.2"
 codec = { package = "parity-scale-codec", version = "3.0.0", features = [
 	"derive",
 ] }
 futures = "0.3.21"
 libp2p = "0.46.1"
 smallvec = "1.8.0"
+sc-consensus = { version = "0.10.0-dev", path = "../../consensus/common" }
 sc-peerset = { version = "4.0.0-dev", path = "../../peerset" }
+sp-consensus = { version = "0.10.0-dev", path = "../../../primitives/consensus/common" }
+sp-finality-grandpa = { version = "4.0.0-dev", path = "../../../primitives/finality-grandpa" }
+sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" }

client/network/common/src/lib.rs

@@ -21,3 +21,4 @@
 pub mod config;
 pub mod message;
 pub mod request_responses;
+pub mod sync;