Add new line after writing block to hex file. (#10984)

Author: seunlanlege

ethcore/blockchain/src/blockchain.rs

@@ -592,7 +592,7 @@ impl BlockChain {
 			let best_block_rlp = bc.block(&best_block_hash)
 				.expect("Best block is from a known block hash; qed");
 
-			// and write them
+			// and write them to the cache.
 			let mut best_block = bc.best_block.write();
 			*best_block = BestBlock {
 				total_difficulty: best_block_total_difficulty,
@@ -858,12 +858,31 @@ impl BlockChain {
 		}
 	}
 
-	/// clears all caches for testing purposes
+	/// clears all caches, re-loads best block from disk for testing purposes
 	pub fn clear_cache(&self) {
 		self.block_bodies.write().clear();
 		self.block_details.write().clear();
 		self.block_hashes.write().clear();
 		self.block_headers.write().clear();
+		// Fetch best block details from disk
+		let best_block_hash = self.db.key_value().get(db::COL_EXTRA, b"best")
+			.expect("Low-level database error when fetching 'best' block. Some issue with disk?")
+			.as_ref()
+			.map(|r| H256::from_slice(r))
+			.unwrap();
+		let best_block_total_difficulty = self.block_details(&best_block_hash)
+			.expect("Best block is from a known block hash; a known block hash always comes with a known block detail; qed")
+			.total_difficulty;
+		let best_block_rlp = self.block(&best_block_hash)
+			.expect("Best block is from a known block hash; qed");
+
+		// and write them to the cache
+		let mut best_block = self.best_block.write();
+		*best_block = BestBlock {
+			total_difficulty: best_block_total_difficulty,
+			header: best_block_rlp.decode_header(),
+			block: best_block_rlp,
+		};
 	}
 
 	/// Update the best ancient block to the given hash, after checking that

ethcore/src/client/client.rs

@@ -17,21 +17,23 @@
 use std::cmp;
 use std::collections::{HashSet, BTreeMap, VecDeque};
 use std::str::FromStr;
+use std::str::from_utf8;
 use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering as AtomicOrdering};
 use std::sync::{Arc, Weak};
-use std::time::{Instant, Duration};
+use std::io::{BufReader, BufRead};
+use std::time::{Duration, Instant};
 
 use blockchain::{BlockReceipts, BlockChain, BlockChainDB, BlockProvider, TreeRoute, ImportRoute, TransactionAddress, ExtrasInsert, BlockNumberKey};
 use bytes::Bytes;
-use call_contract::{CallContract, RegistryInfo};
-use ethcore_miner::pool::VerifiedTransaction;
-use ethereum_types::{H256, H264, Address, U256};
-use evm::Schedule;
+use rlp::PayloadInfo;
+use bytes::ToPretty;
+use error::Error;
+use ethereum_types::{Address, H256, H264, U256};
 use hash::keccak;
-use io::IoChannel;
+use call_contract::CallContract;
+use ethcore_miner::pool::VerifiedTransaction;
 use itertools::Itertools;
-use journaldb;
-use kvdb::{DBValue, KeyValueDB, DBTransaction};
+use kvdb::{DBTransaction, DBValue, KeyValueDB};
 use parking_lot::{Mutex, RwLock};
 use rand::OsRng;
 use types::transaction::{self, LocalizedTransaction, UnverifiedTransaction, SignedTransaction, Action};
@@ -43,6 +45,7 @@ use types::log_entry::LocalizedLogEntry;
 use types::receipt::{Receipt, LocalizedReceipt};
 use types::{BlockNumber, header::{Header, ExtendedHeader}};
 use vm::{EnvInfo, LastHashes};
+use types::data_format::DataFormat;
 
 use block::{LockedBlock, Drain, ClosedBlock, OpenBlock, enact_verified, SealedBlock};
 use client::ancient_import::AncientVerifier;
@@ -51,8 +54,9 @@ use client::{
 	ReopenBlock, PrepareOpenBlock, ScheduleInfo, ImportSealedBlock,
 	BroadcastProposalBlock, ImportBlock, StateOrBlock, StateInfo, StateClient, Call,
 	AccountData, BlockChain as BlockChainTrait, BlockProducer, SealedBlockImporter,
-	ClientIoMessage, BlockChainReset
+	ClientIoMessage, BlockChainReset, ImportExportBlocks
 };
+use rustc_hex::FromHex;
 use client::{
 	BlockId, TransactionId, UncleId, TraceId, ClientConfig, BlockChainClient,
 	TraceFilter, CallAnalytics, Mode,
@@ -80,7 +84,9 @@ use verification::queue::kind::blocks::Unverified;
 use verification::{PreverifiedBlock, Verifier, BlockQueue};
 use verification;
 use ansi_term::Colour;
-
+use call_contract::RegistryInfo;
+use io::IoChannel;
+use vm::Schedule;
 // re-export
 pub use types::blockchain_info::BlockChainInfo;
 pub use types::block_status::BlockStatus;
@@ -2568,6 +2574,116 @@ impl Drop for Client {
 	}
 }
 
+impl ImportExportBlocks for Client {
+	fn export_blocks<'a>(
+		&self,
+		mut out: Box<dyn std::io::Write + 'a>,
+		from: BlockId,
+		to: BlockId,
+		format: Option<DataFormat>
+	) -> Result<(), String> {
+		let from = self.block_number(from).ok_or("Starting block could not be found")?;
+		let to = self.block_number(to).ok_or("End block could not be found")?;
+		let format = format.unwrap_or_default();
+
+		for i in from..=to {
+			if i % 10000 == 0 {
+				info!("#{}", i);
+			}
+			let b = self.block(BlockId::Number(i)).ok_or("Error exporting incomplete chain")?.into_inner();
+			match format {
+				DataFormat::Binary => {
+					out.write(&b).map_err(|e| format!("Couldn't write to stream. Cause: {}", e))?;
+				}
+				DataFormat::Hex => {
+					out.write_fmt(format_args!("{}\n", b.pretty())).map_err(|e| format!("Couldn't write to stream. Cause: {}", e))?;
+				}
+			}
+		}
+		Ok(())
+	}
+
+	fn import_blocks<'a>(
+		&self,
+		mut source: Box<dyn std::io::Read + 'a>,
+		format: Option<DataFormat>
+	) -> Result<(), String> {
+		const READAHEAD_BYTES: usize = 8;
+
+		let mut first_bytes: Vec<u8> = vec![0; READAHEAD_BYTES];
+		let mut first_read = 0;
+
+		let format = match format {
+			Some(format) => format,
+			None => {
+				first_read = source.read(&mut first_bytes).map_err(|_| "Error reading from the file/stream.")?;
+				match first_bytes[0] {
+					0xf9 => DataFormat::Binary,
+					_ => DataFormat::Hex,
+				}
+			}
+		};
+
+		let do_import = |bytes: Vec<u8>| {
+			let block = Unverified::from_rlp(bytes).map_err(|_| "Invalid block rlp")?;
+			let number = block.header.number();
+			while self.queue_info().is_full() { std::thread::sleep(Duration::from_secs(1)); }
+			match self.import_block(block) {
+				Err(Error(EthcoreErrorKind::Import(ImportErrorKind::AlreadyInChain), _)) => {
+					trace!("Skipping block #{}: already in chain.", number);
+				}
+				Err(e) => {
+					return Err(format!("Cannot import block #{}: {:?}", number, e));
+				},
+				Ok(_) => {},
+			}
+			Ok(())
+		};
+
+		match format {
+			DataFormat::Binary => {
+				loop {
+					let (mut bytes, n) = if first_read > 0 {
+						(first_bytes.clone(), first_read)
+					} else {
+						let mut bytes = vec![0; READAHEAD_BYTES];
+						let n = source.read(&mut bytes)
+							.map_err(|err| format!("Error reading from the file/stream: {:?}", err))?;
+						(bytes, n)
+					};
+					if n == 0 { break; }
+					first_read = 0;
+					let s = PayloadInfo::from(&bytes)
+						.map_err(|e| format!("Invalid RLP in the file/stream: {:?}", e))?.total();
+					bytes.resize(s, 0);
+					source.read_exact(&mut bytes[n..])
+						.map_err(|err| format!("Error reading from the file/stream: {:?}", err))?;
+					do_import(bytes)?;
+				}
+			}
+			DataFormat::Hex => {
+				for line in BufReader::new(source).lines() {
+					let s = line
+						.map_err(|err| format!("Error reading from the file/stream: {:?}", err))?;
+					let s = if first_read > 0 {
+						from_utf8(&first_bytes)
+							.map_err(|err| format!("Invalid UTF-8: {:?}", err))?
+							.to_owned() + &(s[..])
+					} else {
+						s
+					};
+					first_read = 0;
+					let bytes = s.from_hex()
+						.map_err(|err| format!("Invalid hex in file/stream: {:?}", err))?;
+					do_import(bytes)?;
+				}
+			}
+		};
+		self.flush_queue();
+		Ok(())
+	}
+}
+
 /// Returns `LocalizedReceipt` given `LocalizedTransaction`
 /// and a vector of receipts from given block up to transaction index.
 fn transaction_receipt(

ethcore/src/client/mod.rs

@@ -38,7 +38,7 @@ pub use self::chain_notify::{ChainNotify, NewBlocks, ChainRoute, ChainRouteType,
 pub use self::traits::{
     Nonce, Balance, ChainInfo, BlockInfo, ReopenBlock, PrepareOpenBlock, TransactionInfo, ScheduleInfo, ImportSealedBlock, BroadcastProposalBlock, ImportBlock,
     StateOrBlock, StateClient, Call, EngineInfo, AccountData, BlockChain, BlockProducer, SealedBlockImporter, BadBlocks,
-	BlockChainReset
+	BlockChainReset, ImportExportBlocks
 };
 pub use state::StateInfo;
 pub use self::traits::{BlockChainClient, EngineClient, ProvingBlockChainClient, IoClient};

ethcore/src/client/traits.rs

@@ -38,6 +38,7 @@ use types::ids::*;
 use types::log_entry::LocalizedLogEntry;
 use types::pruning_info::PruningInfo;
 use types::receipt::LocalizedReceipt;
+use types::data_format::DataFormat;
 use types::trace_filter::Filter as TraceFilter;
 use vm::LastHashes;
 
@@ -477,3 +478,30 @@ pub trait BlockChainReset {
 	/// reset to best_block - n
 	fn reset(&self, num: u32) -> Result<(), String>;
 }
+
+/// Provides a method for importing/exporting blocks
+pub trait ImportExportBlocks {
+    /// Export blocks to destination, with the given from, to and format argument.
+    /// destination could be a file or stdout.
+    /// If the format is hex, each block is written on a new line.
+    /// For binary exports, all block data is written to the same line.
+	fn export_blocks<'a>(
+        &self,
+        destination: Box<dyn std::io::Write + 'a>,
+        from: BlockId,
+        to: BlockId,
+        format: Option<DataFormat>
+    ) -> Result<(), String>;
+
+	/// Import blocks from destination, with the given format argument
+	/// Source could be a file or stdout.
+	/// For hex format imports, it attempts to read the blocks on a line by line basis.
+	/// For binary format imports, reads the 8 byte RLP header in order to decode the block
+	/// length to be read.
+	fn import_blocks<'a>(
+		&self,
+		source: Box<dyn std::io::Read + 'a>,
+		format: Option<DataFormat>
+	) -> Result<(), String>;
+}
+

ethcore/src/tests/client.rs

@@ -14,20 +14,29 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity Ethereum.  If not, see <http://www.gnu.org/licenses/>.
 
-use std::str::FromStr;
+use std::str::{FromStr, from_utf8};
 use std::sync::Arc;
 
 use ethereum_types::{U256, Address};
 use ethkey::KeyPair;
 use hash::keccak;
 use io::IoChannel;
 use tempdir::TempDir;
-use types::transaction::{PendingTransaction, Transaction, Action, Condition};
-use types::filter::Filter;
-use types::view;
-use types::views::BlockView;
-
-use client::{BlockChainClient, BlockChainReset, Client, ClientConfig, BlockId, ChainInfo, BlockInfo, PrepareOpenBlock, ImportSealedBlock, ImportBlock};
+use types::{
+	data_format::DataFormat,
+	ids::BlockId,
+	transaction::{PendingTransaction, Transaction, Action, Condition},
+	filter::Filter,
+	view,
+	views::BlockView,
+};
+use verification::queue::kind::blocks::Unverified;
+use client::{Client, ClientConfig, PrepareOpenBlock, ImportSealedBlock};
+use client::traits::{
+	BlockInfo, BlockChainClient, BlockChainReset, ChainInfo,
+	ImportExportBlocks, ImportBlock
+};
+use spec;
 use ethereum;
 use executive::{Executive, TransactOptions};
 use miner::{Miner, PendingOrdering, MinerService};
@@ -38,7 +47,7 @@ use test_helpers::{
 	generate_dummy_client, push_blocks_to_client, get_test_client_with_blocks, get_good_dummy_block_seq,
 	generate_dummy_client_with_data, get_good_dummy_block, get_bad_state_dummy_block
 };
-use verification::queue::kind::blocks::Unverified;
+use rustc_hex::ToHex;
 
 #[test]
 fn imports_from_empty() {
@@ -386,3 +395,79 @@ fn reset_blockchain() {
 
 	assert!(client.block_header(BlockId::Number(15)).is_some());
 }
+
+#[test]
+fn import_export_hex() {
+	let client = get_test_client_with_blocks(get_good_dummy_block_seq(19));
+	let block_rlps = (15..20)
+		.filter_map(|num| client.block(BlockId::Number(num)))
+		.map(|header| {
+			header.raw().to_hex()
+		})
+		.collect::<Vec<_>>();
+
+	let mut out = Vec::new();
+
+	client.export_blocks(
+		Box::new(&mut out),
+		BlockId::Number(15),
+		BlockId::Number(20),
+		Some(DataFormat::Hex)
+	).unwrap();
+
+	let written = from_utf8(&out)
+		.unwrap()
+		.split("\n")
+		// last line is empty, ignore it.
+		.take(5)
+		.collect::<Vec<_>>();
+	assert_eq!(block_rlps, written);
+
+	assert!(client.reset(5).is_ok());
+	client.chain().clear_cache();
+
+	assert!(client.block_header(BlockId::Number(20)).is_none());
+	assert!(client.block_header(BlockId::Number(19)).is_none());
+	assert!(client.block_header(BlockId::Number(18)).is_none());
+	assert!(client.block_header(BlockId::Number(17)).is_none());
+	assert!(client.block_header(BlockId::Number(16)).is_none());
+
+	client.import_blocks(Box::new(&*out), Some(DataFormat::Hex)).unwrap();
+
+	assert!(client.block_header(BlockId::Number(20)).is_some());
+	assert!(client.block_header(BlockId::Number(19)).is_some());
+	assert!(client.block_header(BlockId::Number(18)).is_some());
+	assert!(client.block_header(BlockId::Number(17)).is_some());
+	assert!(client.block_header(BlockId::Number(16)).is_some());
+}
+
+#[test]
+fn import_export_binary() {
+	let client = get_test_client_with_blocks(get_good_dummy_block_seq(19));
+
+	let mut out = Vec::new();
+
+	client.export_blocks(
+		Box::new(&mut out),
+		BlockId::Number(15),
+		BlockId::Number(20),
+		Some(DataFormat::Binary)
+	).unwrap();
+
+	assert!(client.reset(5).is_ok());
+	client.chain().clear_cache();
+
+	assert!(client.block_header(BlockId::Number(20)).is_none());
+	assert!(client.block_header(BlockId::Number(19)).is_none());
+	assert!(client.block_header(BlockId::Number(18)).is_none());
+	assert!(client.block_header(BlockId::Number(17)).is_none());
+	assert!(client.block_header(BlockId::Number(16)).is_none());
+
+	client.import_blocks(Box::new(&*out), Some(DataFormat::Binary)).unwrap();
+
+	assert!(client.block_header(BlockId::Number(19)).is_some());
+	assert!(client.block_header(BlockId::Number(18)).is_some());
+	assert!(client.block_header(BlockId::Number(20)).is_some());
+	assert!(client.block_header(BlockId::Number(17)).is_some());
+	assert!(client.block_header(BlockId::Number(16)).is_some());
+}