Integrate benchmark-block

cli/Cargo.toml

@@ -21,6 +21,7 @@ futures = "0.3.21"
 pyro = { package = "pyroscope", version = "0.3.1", optional = true }
 
 service = { package = "polkadot-service", path = "../node/service", default-features = false, optional = true }
+polkadot-client = { path = "../node/client", optional = true }
 polkadot-node-core-pvf = { path = "../node/core/pvf", optional = true }
 polkadot-performance-test = { path = "../node/test/performance-test", optional = true }
 
@@ -50,6 +51,7 @@ cli = [
 	"sc-tracing",
 	"frame-benchmarking-cli",
 	"try-runtime-cli",
+	"polkadot-client",
 	"polkadot-node-core-pvf",
 	"polkadot-performance-test",
 ]

cli/src/cli.rs

@@ -54,6 +54,14 @@ pub enum Subcommand {
 	#[clap(name = "benchmark", about = "Benchmark runtime pallets.")]
 	Benchmark(frame_benchmarking_cli::BenchmarkCmd),
 
+	/// Benchmark the execution time of historic blocks and compare it to their consumed weight.
+	#[clap(
+		name = "benchmark-block",
+		about = "Benchmark the execution time of historic blocks and compare it to their consumed weight."
+	)]
+	BenchmarkBlock(frame_benchmarking_cli::BlockCmd),
+
+	/// Sub command for benchmarking the storage speed.
 	#[clap(name = "benchmark-storage", about = "Benchmark storage speed.")]
 	BenchmarkStorage(frame_benchmarking_cli::StorageCmd),
 

cli/src/command.rs

@@ -475,6 +475,66 @@ pub fn run() -> Result<()> {
 			#[cfg(not(feature = "polkadot-native"))]
 			panic!("No runtime feature (polkadot, kusama, westend, rococo) is enabled")
 		},
+		Some(Subcommand::BenchmarkBlock(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			let chain_spec = &runner.config().chain_spec;
+			set_default_ss58_version(chain_spec);
+
+			#[cfg(feature = "rococo-native")]
+			if chain_spec.is_rococo() || chain_spec.is_wococo() || chain_spec.is_versi() {
+				return Ok(runner.async_run(|mut config| {
+					let (client, _, _, task_manager) = service::new_chain_ops(&mut config, None)?;
+
+					if let polkadot_client::Client::Rococo(pd) = &*client {
+						Ok((cmd.run(pd.clone()).map_err(Error::SubstrateCli), task_manager))
+					} else {
+						unreachable!("Checked above; qed")
+					}
+				})?)
+			}
+
+			#[cfg(feature = "kusama-native")]
+			if chain_spec.is_kusama() {
+				return Ok(runner.async_run(|mut config| {
+					let (client, _, _, task_manager) = service::new_chain_ops(&mut config, None)?;
+
+					if let polkadot_client::Client::Kusama(pd) = &*client {
+						Ok((cmd.run(pd.clone()).map_err(Error::SubstrateCli), task_manager))
+					} else {
+						unreachable!("Checked above; qed")
+					}
+				})?)
+			}
+
+			#[cfg(feature = "westend-native")]
+			if chain_spec.is_westend() {
+				return Ok(runner.async_run(|mut config| {
+					let (client, _, _, task_manager) = service::new_chain_ops(&mut config, None)?;
+
+					if let polkadot_client::Client::Westend(pd) = &*client {
+						Ok((cmd.run(pd.clone()).map_err(Error::SubstrateCli), task_manager))
+					} else {
+						unreachable!("Checked above; qed")
+					}
+				})?)
+			}
+
+			#[cfg(feature = "polkadot-native")]
+			{
+				return Ok(runner.async_run(|mut config| {
+					let (client, _, _, task_manager) = service::new_chain_ops(&mut config, None)?;
+
+					if let polkadot_client::Client::Polkadot(pd) = &*client {
+						Ok((cmd.run(pd.clone()).map_err(Error::SubstrateCli), task_manager))
+					} else {
+						unreachable!("Checked above; qed")
+					}
+				})?)
+			}
+
+			#[cfg(not(feature = "polkadot-native"))]
+			unreachable!("No runtime feature (polkadot, kusama, westend, rococo) is enabled")
+		},
 		Some(Subcommand::BenchmarkStorage(cmd)) => {
 			let runner = cli.create_runner(cmd)?;
 			let chain_spec = &runner.config().chain_spec;

tests/benchmark_block_works.rs

@@ -0,0 +1,91 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+// Unix only since it uses signals.
+#![cfg(unix)]
+
+use assert_cmd::cargo::cargo_bin;
+use nix::{
+	sys::signal::{kill, Signal::SIGINT},
+	unistd::Pid,
+};
+use std::{path::Path, process::Command, result::Result, time::Duration};
+use tempfile::tempdir;
+
+pub mod common;
+
+static RUNTIMES: [&'static str; 6] = ["polkadot", "kusama", "westend", "rococo", "wococo", "versi"];
+
+/// `benchmark-block` works for all dev runtimes using the wasm executor.
+#[tokio::test]
+async fn benchmark_block_works() {
+	for runtime in RUNTIMES {
+		let tmp_dir = tempdir().expect("could not create a temp dir");
+		let base_path = tmp_dir.path();
+		let runtime = format!("{}-dev", runtime);
+
+		// Build a chain with a single block.
+		build_chain(&runtime, base_path).await.unwrap();
+		// Benchmark the that one.
+		benchmark_block(&runtime, base_path, 1).unwrap();
+	}
+}
+
+/// Builds a chain with one block for the given runtime and base path.
+async fn build_chain(runtime: &str, base_path: &Path) -> Result<(), String> {
+	let mut cmd = Command::new(cargo_bin("polkadot"))
+		.args(["--chain", &runtime, "--force-authoring", "--alice"])
+		.arg("-d")
+		.arg(base_path)
+		.arg("--port")
+		.arg("33034")
+		.spawn()
+		.unwrap();
+
+	// Wait for the chain to produce one block.
+	let ok = common::wait_n_finalized_blocks(1, Duration::from_secs(60)).await;
+	// Send SIGINT to node.
+	kill(Pid::from_raw(cmd.id().try_into().unwrap()), SIGINT).unwrap();
+	// Wait for the node to handle it and exit.
+	assert!(common::wait_for(&mut cmd, 30).map(|x| x.success()).unwrap_or_default());
+
+	ok.map_err(|e| format!("Node did not build the chain: {:?}", e))
+}
+
+/// Benchmarks the given block with the wasm executor.
+fn benchmark_block(runtime: &str, base_path: &Path, block: u32) -> Result<(), String> {
+	// Invoke `benchmark-block` with all options to make sure that they are valid.
+	let status = Command::new(cargo_bin("polkadot"))
+		.args(["benchmark-block", "--chain", &runtime])
+		.arg("-d")
+		.arg(base_path)
+		.arg("--weight-path")
+		.arg(base_path)
+		.args(["--pruning", "archive"])
+		.args(["--from", &block.to_string(), "--to", &block.to_string()])
+		.args(["--repeat", "2"])
+		.args(["--execution", "wasm", "--wasm-execution", "compiled"])
+		.status()
+		.map_err(|e| format!("command failed: {:?}", e))?;
+
+	if !status.success() {
+		return Err("Command failed".into())
+	}
+
+	Ok(())
+}