Storage chain: Runtime module (paritytech#8624)

* Transaction storage runtime module * WIP: Tests * Tests, benchmarks and docs * Made check_proof mandatory * Typo * Renamed a crate * Apply suggestions from code review Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> * Added weight for on_finalize * Fixed counter mutations * Reorganized tests * Fixed build * Update for the new inherent API * Reworked for the new inherents API * Apply suggestions from code review Co-authored-by: cheme <emericchevalier.pro@gmail.com> Co-authored-by: Alexander Popiak <alexander.popiak@parity.io> Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com> * Store transactions in a Vec * Added FeeDestination * Get rid of constants * Fixed node runtime build * Fixed benches * Update frame/transaction-storage/src/lib.rs Co-authored-by: cheme <emericchevalier.pro@gmail.com> Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Co-authored-by: cheme <emericchevalier.pro@gmail.com> Co-authored-by: Alexander Popiak <alexander.popiak@parity.io> Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>
nathanwhit · Jun 4, 2021 · a57bc44 · a57bc44
1 parent ea5d357
commit a57bc44
Show file tree

Hide file tree

Showing 30 changed files with 1,534 additions and 29 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -123,6 +123,7 @@ members = [
 	"frame/transaction-payment",
 	"frame/transaction-payment/rpc",
 	"frame/transaction-payment/rpc/runtime-api",
+	"frame/transaction-storage",
 	"frame/treasury",
 	"frame/tips",
 	"frame/uniques",
@@ -180,6 +181,7 @@ members = [
 	"primitives/timestamp",
 	"primitives/tracing",
 	"primitives/transaction-pool",
+	"primitives/transaction-storage-proof",
 	"primitives/trie",
 	"primitives/utils",
 	"primitives/version",

diff --git a/bin/node/cli/src/chain_spec.rs b/bin/node/cli/src/chain_spec.rs
@@ -335,6 +335,7 @@ pub fn testnet_genesis(
 		},
 		pallet_vesting: Default::default(),
 		pallet_gilt: Default::default(),
+		pallet_transaction_storage: Default::default(),
 	}
 }
 

diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml
@@ -85,6 +85,7 @@ pallet-treasury = { version = "3.0.0", default-features = false, path = "../../.
 pallet-utility = { version = "3.0.0", default-features = false, path = "../../../frame/utility" }
 pallet-transaction-payment = { version = "3.0.0", default-features = false, path = "../../../frame/transaction-payment" }
 pallet-transaction-payment-rpc-runtime-api = { version = "3.0.0", default-features = false, path = "../../../frame/transaction-payment/rpc/runtime-api/" }
+pallet-transaction-storage = { version = "3.0.0", default-features = false, path = "../../../frame/transaction-storage" }
 pallet-uniques = { version = "3.0.0", default-features = false, path = "../../../frame/uniques" }
 pallet-vesting = { version = "3.0.0", default-features = false, path = "../../../frame/vesting" }
 
@@ -152,6 +153,7 @@ std = [
 	"pallet-tips/std",
 	"pallet-transaction-payment-rpc-runtime-api/std",
 	"pallet-transaction-payment/std",
+	"pallet-transaction-storage/std",
 	"pallet-treasury/std",
 	"sp-transaction-pool/std",
 	"pallet-utility/std",
@@ -194,6 +196,7 @@ runtime-benchmarks = [
 	"pallet-staking/runtime-benchmarks",
 	"pallet-timestamp/runtime-benchmarks",
 	"pallet-tips/runtime-benchmarks",
+	"pallet-transaction-storage/runtime-benchmarks",
 	"pallet-treasury/runtime-benchmarks",
 	"pallet-utility/runtime-benchmarks",
 	"pallet-uniques/runtime-benchmarks",

diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs
@@ -1114,6 +1114,14 @@ impl pallet_uniques::Config for Runtime {
 	type WeightInfo = pallet_uniques::weights::SubstrateWeight<Runtime>;
 }
 
+impl pallet_transaction_storage::Config for Runtime {
+	type Event = Event;
+	type Currency = Balances;
+	type Call = Call;
+	type FeeDestination = ();
+	type WeightInfo = pallet_transaction_storage::weights::SubstrateWeight<Runtime>;
+}
+
 construct_runtime!(
 	pub enum Runtime where
 		Block = Block,
@@ -1159,6 +1167,7 @@ construct_runtime!(
 		Lottery: pallet_lottery::{Pallet, Call, Storage, Event<T>},
 		Gilt: pallet_gilt::{Pallet, Call, Storage, Event<T>, Config},
 		Uniques: pallet_uniques::{Pallet, Call, Storage, Event<T>},
+		TransactionStorage: pallet_transaction_storage::{Pallet, Call, Storage, Inherent, Config<T>, Event<T>},
 	}
 );
 
@@ -1532,6 +1541,7 @@ impl_runtime_apis! {
 			add_benchmark!(params, batches, frame_system, SystemBench::<Runtime>);
 			add_benchmark!(params, batches, pallet_timestamp, Timestamp);
 			add_benchmark!(params, batches, pallet_tips, Tips);
+			add_benchmark!(params, batches, pallet_transaction_storage, TransactionStorage);
 			add_benchmark!(params, batches, pallet_treasury, Treasury);
 			add_benchmark!(params, batches, pallet_uniques, Uniques);
 			add_benchmark!(params, batches, pallet_utility, Utility);

diff --git a/bin/node/testing/src/genesis.rs b/bin/node/testing/src/genesis.rs
@@ -120,5 +120,6 @@ pub fn config_endowed(
 		},
 		pallet_vesting: Default::default(),
 		pallet_gilt: Default::default(),
+		pallet_transaction_storage: Default::default(),
 	}
 }
diff --git a/client/api/src/client.rs b/client/api/src/client.rs
@@ -84,6 +84,16 @@ pub trait BlockBackend<Block: BlockT> {
 		id: &BlockId<Block>
 	) -> sp_blockchain::Result<Option<Vec<<Block as BlockT>::Extrinsic>>>;
 
+	/// Get all indexed transactions for a block,
+	/// including renewed transactions.
+	///
+	/// Note that this will only fetch transactions
+	/// that are indexed by the runtime with `storage_index_transaction`.
+	fn block_indexed_body(
+		&self,
+		id: &BlockId<Block>,
+	) -> sp_blockchain::Result<Option<Vec<Vec<u8>>>>;
+
 	/// Get full block by id.
 	fn block(&self, id: &BlockId<Block>) -> sp_blockchain::Result<Option<SignedBlock<Block>>>;
 

diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs
@@ -419,6 +419,13 @@ impl<Block: BlockT> blockchain::Backend<Block> for Blockchain<Block> {
 	) -> sp_blockchain::Result<Option<Vec<u8>>> {
 		unimplemented!("Not supported by the in-mem backend.")
 	}
+
+	fn block_indexed_body(
+		&self,
+		_id: BlockId<Block>
+	) -> sp_blockchain::Result<Option<Vec<Vec<u8>>>> {
+		unimplemented!("Not supported by the in-mem backend.")
+	}
 }
 
 impl<Block: BlockT> blockchain::ProvideCache<Block> for Blockchain<Block> {

diff --git a/client/api/src/lib.rs b/client/api/src/lib.rs
@@ -38,6 +38,7 @@ pub use client::*;
 pub use light::*;
 pub use notifications::*;
 pub use proof_provider::*;
+pub use sp_blockchain::HeaderBackend;
 
 pub use sp_state_machine::{StorageProof, ExecutionStrategy};
 

diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs
@@ -67,7 +67,7 @@ use codec::{Decode, Encode};
 use hash_db::Prefix;
 use sp_trie::{MemoryDB, PrefixedMemoryDB, prefixed_key};
 use sp_database::Transaction;
-use sp_core::{Hasher, ChangesTrieConfiguration};
+use sp_core::ChangesTrieConfiguration;
 use sp_core::offchain::OffchainOverlayedChange;
 use sp_core::storage::{well_known_keys, ChildInfo};
 use sp_arithmetic::traits::Saturating;
@@ -591,6 +591,37 @@ impl<Block: BlockT> sc_client_api::blockchain::Backend<Block> for BlockchainDb<B
 	fn has_indexed_transaction(&self, hash: &Block::Hash) -> ClientResult<bool> {
 		Ok(self.db.contains(columns::TRANSACTION, hash.as_ref()))
 	}
+
+	fn block_indexed_body(&self, id: BlockId<Block>) -> ClientResult<Option<Vec<Vec<u8>>>> {
+		match self.transaction_storage {
+			TransactionStorageMode::BlockBody => Ok(None),
+			TransactionStorageMode::StorageChain => {
+				let body = match read_db(&*self.db, columns::KEY_LOOKUP, columns::BODY, id)? {
+					Some(body) => body,
+					None => return Ok(None),
+				};
+				match Vec::<ExtrinsicHeader>::decode(&mut &body[..]) {
+					Ok(index) => {
+						let mut transactions = Vec::new();
+						for ExtrinsicHeader { indexed_hash, .. } in index.into_iter() {
+							if indexed_hash != Default::default() {
+								match self.db.get(columns::TRANSACTION, indexed_hash.as_ref()) {
+									Some(t) => transactions.push(t),
+									None => return Err(sp_blockchain::Error::Backend(
+										format!("Missing indexed transaction {:?}", indexed_hash))
+									)
+								}
+							}
+						}
+						Ok(Some(transactions))
+					}
+					Err(err) => return Err(sp_blockchain::Error::Backend(
+							format!("Error decoding body list: {}", err)
+					)),
+				}
+			}
+		}
+	}
 }
 
 impl<Block: BlockT> sc_client_api::blockchain::ProvideCache<Block> for BlockchainDb<Block> {
@@ -1624,10 +1655,10 @@ fn apply_index_ops<Block: BlockT>(
 	let mut renewed_map = HashMap::new();
 	for op in ops {
 		match op {
-			IndexOperation::Insert { extrinsic, offset } => {
-				index_map.insert(extrinsic, offset);
+			IndexOperation::Insert { extrinsic, hash, size } => {
+				index_map.insert(extrinsic, (hash, size));
 			}
-			IndexOperation::Renew { extrinsic, hash, .. } => {
+			IndexOperation::Renew { extrinsic, hash } => {
 				renewed_map.insert(extrinsic, DbHash::from_slice(hash.as_ref()));
 			}
 		}
@@ -1643,9 +1674,8 @@ fn apply_index_ops<Block: BlockT>(
 			}
 		} else {
 			match index_map.get(&(index as u32)) {
-				Some(offset) if *offset as usize <= extrinsic.len() => {
-					let offset = *offset as usize;
-					let hash = HashFor::<Block>::hash(&extrinsic[offset..]);
+				Some((hash, size)) if *size as usize <= extrinsic.len() => {
+					let offset = extrinsic.len() - *size as usize;
 					transaction.store(
 						columns::TRANSACTION,
 						DbHash::from_slice(hash.as_ref()),
@@ -3024,13 +3054,16 @@ pub(crate) mod tests {
 		for i in 0 .. 10 {
 			let mut index = Vec::new();
 			if i == 0 {
-				index.push(IndexOperation::Insert { extrinsic: 0, offset: 1 });
+				index.push(IndexOperation::Insert {
+					extrinsic: 0,
+					hash: x1_hash.as_ref().to_vec(),
+					size: (x1.len() - 1) as u32,
+				});
 			} else if i < 5 {
 				// keep renewing 1st
 				index.push(IndexOperation::Renew {
 					extrinsic: 0,
 					hash: x1_hash.as_ref().to_vec(),
-					size: (x1.len() - 1) as u32,
 				});
 			} // else stop renewing
 			let hash = insert_block(

diff --git a/client/light/src/blockchain.rs b/client/light/src/blockchain.rs
@@ -135,6 +135,13 @@ impl<S, Block> BlockchainBackend<Block> for Blockchain<S> where Block: BlockT, S
 	) -> ClientResult<Option<Vec<u8>>> {
 		Err(ClientError::NotAvailableOnLightClient)
 	}
+
+	fn block_indexed_body(
+		&self,
+		_id: BlockId<Block>
+	) -> sp_blockchain::Result<Option<Vec<Vec<u8>>>> {
+		Err(ClientError::NotAvailableOnLightClient)
+	}
 }
 
 impl<S: Storage<Block>, Block: BlockT> ProvideCache<Block> for Blockchain<S> {

diff --git a/client/service/Cargo.toml b/client/service/Cargo.toml
@@ -65,6 +65,7 @@ codec = { package = "parity-scale-codec", version = "2.0.0" }
 sc-executor = { version = "0.9.0", path = "../executor" }
 sc-transaction-pool = { version = "3.0.0", path = "../transaction-pool" }
 sp-transaction-pool = { version = "3.0.0", path = "../../primitives/transaction-pool" }
+sp-transaction-storage-proof = { version = "3.0.0", path = "../../primitives/transaction-storage-proof" }
 sc-rpc-server = { version = "3.0.0", path = "../rpc-servers" }
 sc-rpc = { version = "3.0.0", path = "../rpc" }
 sc-block-builder = { version = "0.9.0", path = "../block-builder" }

diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs
@@ -1982,6 +1982,13 @@ impl<B, E, Block, RA> BlockBackend<Block> for Client<B, E, Block, RA>
 	fn has_indexed_transaction(&self, hash: &Block::Hash) -> sp_blockchain::Result<bool> {
 		self.backend.blockchain().has_indexed_transaction(hash)
 	}
+
+	fn block_indexed_body(
+		&self,
+		id: &BlockId<Block>
+	) -> sp_blockchain::Result<Option<Vec<Vec<u8>>>> {
+		self.backend.blockchain().block_indexed_body(*id)
+	}
 }
 
 impl<B, E, Block, RA> backend::AuxStore for Client<B, E, Block, RA>
@@ -2050,3 +2057,26 @@ impl<BE, E, B, RA> sp_consensus::block_validation::Chain<B> for Client<BE, E, B,
 		Client::block_status(self, id).map_err(|e| Box::new(e) as Box<_>)
 	}
 }
+
+impl<BE, E, B, RA> sp_transaction_storage_proof::IndexedBody<B> for Client<BE, E, B, RA>
+where
+	BE: backend::Backend<B>,
+	E: CallExecutor<B>,
+	B: BlockT,
+{
+	fn block_indexed_body(
+		&self,
+		number: NumberFor<B>,
+	) ->Result<Option<Vec<Vec<u8>>>, sp_transaction_storage_proof::Error> {
+		self.backend.blockchain().block_indexed_body(BlockId::number(number))
+			.map_err(|e| sp_transaction_storage_proof::Error::Application(Box::new(e)))
+	}
+
+	fn number(
+		&self,
+		hash: B::Hash,
+	) -> Result<Option<NumberFor<B>>, sp_transaction_storage_proof::Error> {
+		self.backend.blockchain().number(hash)
+			.map_err(|e| sp_transaction_storage_proof::Error::Application(Box::new(e)))
+	}
+}
-Original file line number
+Diff line change
@@ Expand Up / @@ -335,6 +335,7 @@ pub fn testnet_genesis( @@
     		},
     		pallet_vesting: Default::default(),
     		pallet_gilt: Default::default(),
+    		pallet_transaction_storage: Default::default(),
     	}
     }
@@ Expand Down @@