Companion pr for Substrate #3860

availability-store/Cargo.toml

@@ -14,18 +14,18 @@ log = "0.4.8"
 futures = "0.3.1"
 tokio = { version = "0.2.4", features = ["rt-core"] }
 exit-future = "0.2.0"
-codec = { package = "parity-scale-codec", version = "1.1.0", default-features = false, features = ["derive"] }
+codec = { package = "parity-scale-codec", version = "1.1.0", features = ["derive"] }
 sc-network = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
 consensus_common = { package = "sp-consensus", git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
 client = { package = "sc-client-api", git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
 sc-client = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
-sp-runtime = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-master" }
+sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
 sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
 sp-api = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
 keystore = { package = "sc-keystore", git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
-kvdb = "0.2.0"
-kvdb-memorydb = "0.2.0"
+kvdb = "0.3.1"
+kvdb-memorydb = "0.3.1"
 
 [target.'cfg(not(target_os = "unknown"))'.dependencies]
-kvdb-rocksdb = "0.3"
+kvdb-rocksdb = "0.4.1"

availability-store/src/lib.rs

@@ -32,19 +32,20 @@ use polkadot_primitives::{
 		ParachainHost,
 	},
 };
-use sp_runtime::traits::{BlakeTwo256, Hash as HashT, ProvideRuntimeApi};
+use sp_runtime::traits::{BlakeTwo256, Hash as HashT, HasherFor};
 use sp_blockchain::{Result as ClientResult};
 use client::{
 	BlockchainEvents, BlockBody,
 };
-use sp_api::ApiExt;
-use std::pin::Pin;
+use sp_api::{ApiExt, ProvideRuntimeApi};
+
 use log::warn;
 
 use std::sync::Arc;
 use std::collections::HashSet;
 use std::path::PathBuf;
 use std::io;
+use std::pin::Pin;
 
 mod worker;
 mod store;
@@ -200,9 +201,11 @@ impl Store {
 		keystore: KeyStorePtr,
 	) -> ClientResult<AvailabilityBlockImport<I, P>>
 	where
-		P: ProvideRuntimeApi + BlockchainEvents<Block> + BlockBody<Block> + Send + Sync + 'static,
+		P: ProvideRuntimeApi<Block> + BlockchainEvents<Block> + BlockBody<Block> + Send + Sync + 'static,
 		P::Api: ParachainHost<Block>,
 		P::Api: ApiExt<Block, Error=sp_blockchain::Error>,
+		// Rust bug: https://github.com/rust-lang/rust/issues/24159
+		sp_api::StateBackendFor<P, Block>: sp_api::StateBackend<HasherFor<Block>>,
 	{
 		let to_worker = self.to_worker.clone();
 

availability-store/src/worker.rs

@@ -21,8 +21,8 @@ use std::thread;
 
 use log::{error, info, trace, warn};
 use sp_blockchain::{Result as ClientResult};
-use sp_runtime::traits::{Header as HeaderT, ProvideRuntimeApi, Block as BlockT};
-use sp_api::ApiExt;
+use sp_runtime::traits::{Header as HeaderT, Block as BlockT, HasherFor};
+use sp_api::{ApiExt, ProvideRuntimeApi};
 use client::{
 	BlockchainEvents, BlockBody,
 	blockchain::ProvideCache,
@@ -207,8 +207,10 @@ where
 fn fetch_candidates<P>(client: &P, extrinsics: Vec<<Block as BlockT>::Extrinsic>, parent: &BlockId)
 	-> ClientResult<Option<Vec<CandidateReceipt>>>
 where
-	P: ProvideRuntimeApi,
+	P: ProvideRuntimeApi<Block>,
 	P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
+	// Rust bug: https://github.com/rust-lang/rust/issues/24159
+	sp_api::StateBackendFor<P, Block>: sp_api::StateBackend<HasherFor<Block>>,
 {
 	let api = client.runtime_api();
 
@@ -228,9 +230,11 @@ where
 /// Creates a task to prune entries in availability store upon block finalization.
 async fn prune_unneeded_availability<P, S>(client: Arc<P>, mut sender: S)
 where
-	P: ProvideRuntimeApi + BlockchainEvents<Block> + BlockBody<Block> + Send + Sync + 'static,
+	P: ProvideRuntimeApi<Block> + BlockchainEvents<Block> + BlockBody<Block> + Send + Sync + 'static,
 	P::Api: ParachainHost<Block> + ApiExt<Block, Error=sp_blockchain::Error>,
 	S: Sink<WorkerMsg> + Clone + Send + Sync + Unpin,
+	// Rust bug: https://github.com/rust-lang/rust/issues/24159
+	sp_api::StateBackendFor<P, Block>: sp_api::StateBackend<HasherFor<Block>>,
 {
 	let mut finality_notification_stream = client.finality_notification_stream();
 
@@ -624,16 +628,19 @@ impl<I, P> Drop for AvailabilityBlockImport<I, P> {
 }
 
 impl<I, P> BlockImport<Block> for AvailabilityBlockImport<I, P> where
-	I: BlockImport<Block> + Send + Sync,
+	I: BlockImport<Block, Transaction = sp_api::TransactionFor<P, Block>> + Send + Sync,
 	I::Error: Into<ConsensusError>,
-	P: ProvideRuntimeApi + ProvideCache<Block>,
+	P: ProvideRuntimeApi<Block> + ProvideCache<Block>,
 	P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
+	// Rust bug: https://github.com/rust-lang/rust/issues/24159
+	sp_api::StateBackendFor<P, Block>: sp_api::StateBackend<sp_core::Blake2Hasher>
 {
 	type Error = ConsensusError;
+	type Transaction = sp_api::TransactionFor<P, Block>;
 
 	fn import_block(
 		&mut self,
-		block: BlockImportParams<Block>,
+		block: BlockImportParams<Block, Self::Transaction>,
 		new_cache: HashMap<CacheKeyId, Vec<u8>>,
 	) -> Result<ImportResult, Self::Error> {
 		trace!(
@@ -745,9 +752,11 @@ impl<I, P> AvailabilityBlockImport<I, P> {
 		to_worker: mpsc::UnboundedSender<WorkerMsg>,
 	) -> Self
 	where
-		P: ProvideRuntimeApi + BlockBody<Block> + BlockchainEvents<Block> + Send + Sync + 'static,
+		P: ProvideRuntimeApi<Block> + BlockBody<Block> + BlockchainEvents<Block> + Send + Sync + 'static,
 		P::Api: ParachainHost<Block>,
 		P::Api: ApiExt<Block, Error = sp_blockchain::Error>,
+		// Rust bug: https://github.com/rust-lang/rust/issues/24159
+		sp_api::StateBackendFor<P, Block>: sp_api::StateBackend<HasherFor<Block>>,
 	{
 		let (signal, exit) = exit_future::signal();
 

cli/Cargo.toml

@@ -14,6 +14,13 @@ futures = { version = "0.3.1", features = ["compat"] }
 futures01 = { package = "futures", version = "0.1.29" }
 structopt = "=0.3.7"
 sc-cli = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
+sp-api = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
+sp-core = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
+sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
+sc-client = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
+sc-client-api = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
+sc-client-db = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
+sc-executor = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
 service = { package = "polkadot-service", path = "../service", default-features = false }
 
 tokio = { version = "0.1.22", optional = true }

cli/src/lib.rs

@@ -31,10 +31,11 @@ use futures::{
 use tokio::runtime::Runtime;
 use log::info;
 use structopt::StructOpt;
+use sp_api::ConstructRuntimeApi;
 
 pub use service::{
-	AbstractService, CustomConfiguration, ProvideRuntimeApi, CoreApi, ParachainHost, IsKusama, self,
-	WrappedExecutor
+	AbstractService, CustomConfiguration, ProvideRuntimeApi, CoreApi, ParachainHost, IsKusama,
+	WrappedExecutor, Block, self, RuntimeApiCollection, TFullClient
 };
 
 pub use sc_cli::{VersionInfo, IntoExit, NoCustom, SharedParams};
@@ -101,6 +102,25 @@ pub fn run<E: IntoExit>(exit: E, version: sc_cli::VersionInfo) -> error::Result<
 	}
 }
 
+#[cfg(feature = "cli")]
+use sp_core::Blake2Hasher;
+
+#[cfg(feature = "cli")]
+// We can't simply use `service::TLightClient` due to a
+// Rust bug: https://github.com/rust-lang/rust/issues/43580
+type TLightClient<Runtime, Dispatch> = sc_client::Client<
+	sc_client::light::backend::Backend<sc_client_db::light::LightStorage<Block>, Blake2Hasher>,
+	sc_client::light::call_executor::GenesisCallExecutor<
+		sc_client::light::backend::Backend<sc_client_db::light::LightStorage<Block>, Blake2Hasher>,
+		sc_client::LocalCallExecutor<
+			sc_client::light::backend::Backend<sc_client_db::light::LightStorage<Block>, Blake2Hasher>,
+			sc_executor::NativeExecutor<Dispatch>
+		>
+	>,
+	Block,
+	Runtime
+>;
+
 /// Execute the given `cmd` with the given runtime.
 #[cfg(feature = "cli")]
 fn execute_cmd_with_runtime<R, D, E, X>(
@@ -110,14 +130,23 @@ fn execute_cmd_with_runtime<R, D, E, X>(
 	spec: Option<service::ChainSpec>,
 ) -> error::Result<()>
 where
-	R: service::ConstructRuntimeApi<service::Block, service::TFullClient<service::Block, R, D>>
-		+ service::ConstructRuntimeApi<service::Block, service::TLightClient<service::Block, R, D>>
+	R: ConstructRuntimeApi<Block, service::TFullClient<Block, R, D>>
 		+ Send + Sync + 'static,
-	<R as service::ConstructRuntimeApi<service::Block, service::TFullClient<service::Block, R, D>>>::RuntimeApi: service::RuntimeApiCollection<E>,
-	<R as service::ConstructRuntimeApi<service::Block, service::TLightClient<service::Block, R, D>>>::RuntimeApi: service::RuntimeApiCollection<E>,
+	<R as ConstructRuntimeApi<Block, service::TFullClient<Block, R, D>>>::RuntimeApi:
+		RuntimeApiCollection<E, StateBackend = sc_client_api::StateBackendFor<service::TFullBackend<Block>, Block>>,
+	<R as ConstructRuntimeApi<Block, service::TLightClient<Block, R, D>>>::RuntimeApi:
+		RuntimeApiCollection<E, StateBackend = sc_client_api::StateBackendFor<service::TLightBackend<Block>, Block>>,
 	E: service::Codec + Send + Sync + 'static,
 	D: service::NativeExecutionDispatch + 'static,
 	X: IntoExit,
+	// Rust bug: https://github.com/rust-lang/rust/issues/24159
+	<<R as ConstructRuntimeApi<Block, TFullClient<Block, R, D>>>::RuntimeApi as sp_api::ApiExt<Block>>::StateBackend:
+		sp_api::StateBackend<Blake2Hasher>,
+	// Rust bug: https://github.com/rust-lang/rust/issues/43580
+	R: ConstructRuntimeApi<
+		Block,
+		TLightClient<R, D>
+	>,
 {
 	let is_kusama = spec.as_ref().map_or(false, |s| s.is_kusama());
 	// Use preloaded spec

collator/Cargo.toml

@@ -7,11 +7,13 @@ edition = "2018"
 
 [dependencies]
 futures = "0.3.1"
-client = { package = "sc-client", git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
-client-api = { package = "sc-client-api", git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
-primitives = { package = "sp-core", git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
+sc-client = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
+sc-client-api = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
 sc-network = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
 sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
+sp-api = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
+sp-core = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
+sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
 consensus_common = { package = "sp-consensus", git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
 polkadot-primitives = { path = "../primitives" }
 polkadot-cli = { path = "../cli" }
@@ -21,6 +23,7 @@ polkadot-service = { path = "../service" }
 log = "0.4.8"
 tokio = "0.1.22"
 futures-timer = "1.0"
+codec = { package = "parity-scale-codec", version = "1.1.0" }
 
 [dev-dependencies]
 keyring = { package = "sp-keyring", git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }

collator/src/lib.rs

@@ -52,8 +52,8 @@ use std::pin::Pin;
 
 use futures::{future, Future, Stream, FutureExt, TryFutureExt, StreamExt, task::Spawn};
 use log::warn;
-use client::BlockchainEvents;
-use primitives::{Pair, Blake2Hasher};
+use sc_client::BlockchainEvents;
+use sp_core::{Pair, Blake2Hasher};
 use polkadot_primitives::{
 	BlockId, Hash, Block,
 	parachain::{
@@ -128,23 +128,28 @@ impl<R: fmt::Display> fmt::Display for Error<R> {
 }
 
 /// The Polkadot client type.
-pub type PolkadotClient<B, E, R> = client::Client<B, E, Block, R>;
+pub type PolkadotClient<B, E, R> = sc_client::Client<B, E, Block, R>;
 
 /// Something that can build a `ParachainContext`.
 pub trait BuildParachainContext {
 	/// The parachain context produced by the `build` function.
 	type ParachainContext: self::ParachainContext;
 
 	/// Build the `ParachainContext`.
-	fn build<B, E, R, SP>(
+	fn build<B, E, R, SP, Extrinsic>(
 		self,
 		client: Arc<PolkadotClient<B, E, R>>,
 		spawner: SP,
 		network: Arc<dyn Network>,
 	) -> Result<Self::ParachainContext, ()>
 		where
-			B: client_api::backend::Backend<Block, Blake2Hasher> + 'static,
-			E: client::CallExecutor<Block, Blake2Hasher> + Clone + Send + Sync + 'static,
+			PolkadotClient<B, E, R>: ProvideRuntimeApi<Block>,
+			<PolkadotClient<B, E, R> as ProvideRuntimeApi<Block>>::Api: service::RuntimeApiCollection<Extrinsic>,
+			// Rust bug: https://github.com/rust-lang/rust/issues/24159
+			<<PolkadotClient<B, E, R> as ProvideRuntimeApi<Block>>::Api as sp_api::ApiExt<Block>>::StateBackend:
+				sp_api::StateBackend<Blake2Hasher>,
+			Extrinsic: codec::Codec + Send + Sync + 'static,
+			E: sc_client::CallExecutor<Block> + Clone + Send + Sync + 'static,
 			SP: Spawn + Clone + Send + Sync + 'static;
 }
 
@@ -236,7 +241,7 @@ struct ApiContext<P, E, SP> {
 }
 
 impl<P: 'static, E: 'static, SP: 'static> RelayChainContext for ApiContext<P, E, SP> where
-	P: ProvideRuntimeApi + Send + Sync,
+	P: ProvideRuntimeApi<Block> + Send + Sync,
 	P::Api: ParachainHost<Block>,
 	E: futures::Future<Output=()> + Clone + Send + Sync + 'static,
 	SP: Spawn + Clone + Send + Sync
@@ -266,7 +271,7 @@ impl<P: 'static, E: 'static, SP: 'static> RelayChainContext for ApiContext<P, E,
 }
 
 /// Run the collator node using the given `service`.
-fn run_collator_node<S, E, P>(
+fn run_collator_node<S, E, P, Extrinsic>(
 	service: S,
 	exit: E,
 	para_id: ParaId,
@@ -275,19 +280,27 @@ fn run_collator_node<S, E, P>(
 ) -> polkadot_cli::error::Result<()>
 	where
 		S: AbstractService<Block = service::Block, NetworkSpecialization = service::PolkadotProtocol>,
-		client::Client<S::Backend, S::CallExecutor, service::Block, S::RuntimeApi>: ProvideRuntimeApi,
-		<client::Client<S::Backend, S::CallExecutor, service::Block, S::RuntimeApi> as ProvideRuntimeApi>::Api:
-			ParachainHost<service::Block, Error = sp_blockchain::Error>,
+		sc_client::Client<S::Backend, S::CallExecutor, service::Block, S::RuntimeApi>: ProvideRuntimeApi<Block>,
+		<sc_client::Client<S::Backend, S::CallExecutor, service::Block, S::RuntimeApi> as ProvideRuntimeApi<Block>>::Api:
+			service::RuntimeApiCollection<
+				Extrinsic,
+				Error = sp_blockchain::Error,
+				StateBackend = sc_client_api::StateBackendFor<S::Backend, Block>
+			>,
 		// Rust bug: https://github.com/rust-lang/rust/issues/24159
-		S::Backend: service::Backend<service::Block, service::Blake2Hasher>,
+		S::Backend: service::Backend<service::Block>,
 		// Rust bug: https://github.com/rust-lang/rust/issues/24159
-		S::CallExecutor: service::CallExecutor<service::Block, service::Blake2Hasher>,
+		<S::Backend as service::Backend<service::Block>>::State:
+			sp_api::StateBackend<sp_runtime::traits::HasherFor<Block>>,
+		// Rust bug: https://github.com/rust-lang/rust/issues/24159
+		S::CallExecutor: service::CallExecutor<service::Block>,
 		// Rust bug: https://github.com/rust-lang/rust/issues/24159
 		S::SelectChain: service::SelectChain<service::Block>,
 		E: futures::Future<Output=()> + Clone + Unpin + Send + Sync + 'static,
 		P: BuildParachainContext,
 		P::ParachainContext: Send + 'static,
 		<P::ParachainContext as ParachainContext>::ProduceCandidate: Send,
+		Extrinsic: service::Codec + Send + Sync + 'static,
 {
 	let runtime = tokio::runtime::Runtime::new().map_err(|e| format!("{:?}", e))?;
 	let spawner = WrappedExecutor(service.spawn_task_handle());

network/Cargo.toml

@@ -22,8 +22,9 @@ log = "0.4.8"
 exit-future = "0.2.0"
 sc-client = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
 sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
+sp-api = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
 wasm-timer = "0.2.4"
 
 [dev-dependencies]
 sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
-sp-api = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
+sp-state-machine = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }

network/src/gossip.rs

@@ -49,7 +49,7 @@
 //! Peers who send information which was not allowed under a recent neighbor packet
 //! will be noted as non-beneficial to Substrate's peer-set management utility.
 
-use sp_runtime::{generic::BlockId, traits::{ProvideRuntimeApi, BlakeTwo256, Hash as HashT}};
+use sp_runtime::{generic::BlockId, traits::{BlakeTwo256, Hash as HashT}};
 use sp_blockchain::Error as ClientError;
 use sc_network::{config::Roles, Context, PeerId, ReputationChange};
 use sc_network_gossip::{
@@ -63,6 +63,7 @@ use polkadot_primitives::parachain::{
 };
 use polkadot_erasure_coding::{self as erasure};
 use codec::{Decode, Encode};
+use sp_api::ProvideRuntimeApi;
 
 use std::collections::HashMap;
 use std::sync::Arc;
@@ -273,8 +274,8 @@ pub trait ChainContext: Send + Sync {
 impl<F, P> ChainContext for (F, P) where
 	F: Fn(&Hash) -> Option<Known> + Send + Sync,
 	P: Send + Sync + std::ops::Deref,
-	P::Target: ProvideRuntimeApi,
-	<P::Target as ProvideRuntimeApi>::Api: ParachainHost<Block, Error = ClientError>,
+	P::Target: ProvideRuntimeApi<Block>,
+	<P::Target as ProvideRuntimeApi<Block>>::Api: ParachainHost<Block, Error = ClientError>,
 {
 	fn is_known(&self, block_hash: &Hash) -> Option<Known> {
 		(self.0)(block_hash)