Use custom runner for import benchmarks (paritytech#5477)

* custom benchmark runner

* add license preamble

* delete old benchmarks

* update toml

* imporove macro

* tabs

* add size

* size fixes

* add docs for cli

Cargo.toml

@@ -3,6 +3,7 @@ members = [
 	"bin/node-template/node",
 	"bin/node-template/runtime",
 	"bin/node-template/pallets/template",
+	"bin/node/bench",
 	"bin/node/cli",
 	"bin/node/executor",
 	"bin/node/primitives",

bin/node/bench/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "node-bench"
+version = "0.8.0-alpha.5"
+authors = ["Parity Technologies <admin@parity.io>"]
+description = "Substrate node integration benchmarks."
+edition = "2018"
+license = "GPL-3.0"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+log = "0.4.8"
+node-primitives = { version = "2.0.0-alpha.5", path = "../primitives" }
+node-testing = { version = "2.0.0-alpha.5", path = "../testing" }
+sc-cli = { version = "0.8.0-alpha.5", path = "../../../client/cli" }
+sc-client-api = { version = "2.0.0-alpha.5", path = "../../../client/api/" }
+sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" }
+serde = "1.0.101"
+serde_json = "1.0.41"
+structopt = "0.3"

bin/node/bench/src/core.rs

@@ -0,0 +1,140 @@
+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{fmt, borrow::{Cow, ToOwned}};
+use serde::Serialize;
+
+pub struct Path(Vec<String>);
+
+impl Path {
+	pub fn new(initial: &'static [&'static str]) -> Self {
+		Path(initial.iter().map(|x| x.to_string()).collect())
+	}
+}
+
+impl Path {
+	pub fn push(&mut self, item: &str) {
+		self.0.push(item.to_string());
+	}
+
+	pub fn full(&self) -> String {
+		self.0.iter().fold(String::new(), |mut val, next| { val.push_str("::"); val.push_str(next); val })
+	}
+
+	pub fn has(&self, path: &str) -> bool {
+		self.full().contains(path)
+	}
+}
+
+pub trait BenchmarkDescription {
+	fn path(&self) -> Path;
+
+	fn setup(self: Box<Self>) -> Box<dyn Benchmark>;
+
+	fn name(&self) -> Cow<'static, str>;
+}
+
+pub trait Benchmark {
+	fn run(&mut self) -> std::time::Duration;
+}
+
+#[derive(Debug, Clone, Serialize)]
+pub struct BenchmarkOutput {
+	name: String,
+	raw_average: u64,
+	average: u64,
+}
+
+struct NsFormatter(u64);
+
+impl fmt::Display for NsFormatter {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		let v = self.0;
+
+		if v < 100 {
+			return write!(f, "{} ns", v)
+		}
+
+		if self.0 < 10_000 {
+			return write!(f, "{:.1} µs", v as f64 / 1000.0)
+		}
+
+		if self.0 < 1_000_000 {
+			return write!(f, "{:.1} ms", v as f64 / 1_000_000.0)
+		}
+
+		if self.0 < 100_000_000 {
+			return write!(f, "{} ms", v as f64 / 1_000_000.0)
+		}
+
+		write!(f, "{:.2} s", v as f64 / 1_000_000_000.0)
+	}
+}
+
+impl fmt::Display for BenchmarkOutput {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+			f,
+			"({}: avg {}, w_avg {})",
+			self.name,
+			NsFormatter(self.raw_average),
+			NsFormatter(self.average),
+		)
+    }
+}
+
+pub fn run_benchmark(benchmark: Box<dyn BenchmarkDescription>) -> BenchmarkOutput {
+	let name = benchmark.name().to_owned();
+	let mut benchmark = benchmark.setup();
+
+	let mut durations: Vec<u128> = vec![];
+	for _ in 0..50 {
+		let duration = benchmark.run();
+		durations.push(duration.as_nanos());
+	}
+
+	durations.sort();
+
+	let raw_average = (durations.iter().sum::<u128>() / (durations.len() as u128)) as u64;
+	let average = (durations.iter().skip(10).take(30).sum::<u128>() / 30) as u64;
+
+	BenchmarkOutput {
+		name: name.into(),
+		raw_average,
+		average,
+	}
+}
+
+macro_rules! matrix(
+	( $var:ident in $over:expr => $tt:expr,  $( $rest:tt )* ) => {
+		{
+			let mut res = Vec::<Box<dyn crate::core::BenchmarkDescription>>::new();
+			for $var in $over.iter() {
+				res.push(Box::new($tt));
+			}
+			res.extend(matrix!( $($rest)* ));
+			res
+		}
+	};
+	( $var:expr, $( $rest:tt )*) => {
+		{
+			let mut res = vec![Box::new($var) as Box<dyn crate::core::BenchmarkDescription>];
+			res.extend(matrix!( $($rest)* ));
+			res
+		}
+	};
+	() => { vec![] }
+);

bin/node/bench/src/import.rs

@@ -0,0 +1,136 @@
+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Block import benchmark.
+//!
+//! This benchmark is expected to measure block import operation of
+//! some more or less full block.
+//!
+//! As we also want to protect against cold-cache attacks, this
+//! benchmark should not rely on any caching (except those that
+//! DO NOT depend on user input). Thus block generation should be
+//! based on randomized operation.
+//!
+//! This is supposed to be very simple benchmark and is not subject
+//! to much configuring - just block full of randomized transactions.
+//! It is not supposed to measure runtime modules weight correctness
+
+use std::borrow::Cow;
+
+use node_testing::bench::{BenchDb, Profile, BlockType, KeyTypes};
+use node_primitives::Block;
+use sc_client_api::backend::Backend;
+use sp_runtime::generic::BlockId;
+
+use crate::core::{self, Path};
+
+#[derive(Clone, Copy, Debug)]
+pub enum SizeType { Small, Medium, Large }
+
+impl SizeType {
+	fn transactions(&self) -> usize {
+		match self {
+			SizeType::Small => 10,
+			SizeType::Medium => 100,
+			SizeType::Large => 500,
+		}
+	}
+}
+
+pub struct ImportBenchmarkDescription {
+	pub profile: Profile,
+	pub key_types: KeyTypes,
+	pub size: SizeType,
+}
+
+pub struct ImportBenchmark {
+	profile: Profile,
+	database: BenchDb,
+	block: Block,
+}
+
+impl core::BenchmarkDescription for ImportBenchmarkDescription {
+	fn path(&self) -> Path {
+
+		let mut path = Path::new(&["node", "import"]);
+
+		match self.profile {
+			Profile::Wasm => path.push("wasm"),
+			Profile::Native => path.push("native"),
+		}
+
+		match self.key_types {
+			KeyTypes::Sr25519 => path.push("sr25519"),
+			KeyTypes::Ed25519 => path.push("ed25519"),
+		}
+
+		match self.size {
+			SizeType::Small => path.push("small"),
+			SizeType::Medium => path.push("medium"),
+			SizeType::Large => path.push("large"),
+		}
+
+		path
+	}
+
+	fn setup(self: Box<Self>) -> Box<dyn core::Benchmark> {
+		let profile = self.profile;
+		let mut bench_db = BenchDb::with_key_types(self.size.transactions(), self.key_types);
+		let block = bench_db.generate_block(BlockType::RandomTransfers(self.size.transactions()));
+		Box::new(ImportBenchmark {
+			database: bench_db,
+			block,
+			profile,
+		})
+	}
+
+	fn name(&self) -> Cow<'static, str> {
+		match self.profile {
+			Profile::Wasm => "Import benchmark (random transfers, wasm)".into(),
+			Profile::Native => "Import benchmark (random transfers, native)".into(),
+		}
+	}
+}
+
+impl core::Benchmark for ImportBenchmark {
+	fn run(&mut self) -> std::time::Duration {
+		let mut context = self.database.create_context(self.profile);
+
+		let _ = context.client.runtime_version_at(&BlockId::Number(0))
+			.expect("Failed to get runtime version")
+			.spec_version;
+
+		let start = std::time::Instant::now();
+		context.import_block(self.block.clone());
+		let elapsed = start.elapsed();
+
+		log::info!(
+			target: "bench-logistics",
+			"imported block with {} tx, took: {:#?}",
+			self.block.extrinsics.len(),
+			elapsed,
+		);
+
+		log::info!(
+			target: "bench-logistics",
+			"usage info: {}",
+			context.backend.usage_info()
+				.expect("RocksDB backend always provides usage info!"),
+		);
+
+		elapsed
+	}
+}