Add steps setting to benchmarking CLI (paritytech#5033)

* Add steps setting to CLI, use max value to hit worst case.

* Bump impl_version.

* Apply review suggestion.

bin/node/runtime/src/lib.rs

@@ -82,7 +82,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion {
 	// implementation changes and behavior does not, then leave spec_version as
 	// is and increment impl_version.
 	spec_version: 225,
-	impl_version: 0,
+	impl_version: 1,
 	apis: RUNTIME_API_VERSIONS,
 };
 
@@ -819,7 +819,7 @@ impl_runtime_apis! {
 		fn dispatch_benchmark(
 			module: Vec<u8>,
 			extrinsic: Vec<u8>,
-			steps: u32,
+			steps: Vec<u32>,
 			repeat: u32,
 		) -> Option<Vec<frame_benchmarking::BenchmarkResults>> {
 			use frame_benchmarking::Benchmarking;

frame/benchmarking/src/lib.rs

@@ -140,7 +140,7 @@ macro_rules! impl_benchmark {
 		$( $name:ident ),*
 	) => {
 		impl<T: Trait> $crate::Benchmarking<$crate::BenchmarkResults> for Module<T> {
-			fn run_benchmark(extrinsic: Vec<u8>, steps: u32, repeat: u32) -> Result<Vec<$crate::BenchmarkResults>, &'static str> {
+			fn run_benchmark(extrinsic: Vec<u8>, steps: Vec<u32>, repeat: u32) -> Result<Vec<$crate::BenchmarkResults>, &'static str> {
 				// Map the input to the selected benchmark.
 				let extrinsic = sp_std::str::from_utf8(extrinsic.as_slice())
 					.map_err(|_| "Could not find extrinsic")?;
@@ -153,23 +153,29 @@ macro_rules! impl_benchmark {
 				$crate::benchmarking::commit_db();
 				$crate::benchmarking::wipe_db();
 
-				// first one is set_identity.
 				let components = <SelectedBenchmark as $crate::BenchmarkingSetup<T, crate::Call<T>, RawOrigin<T::AccountId>>>::components(&selected_benchmark);
-				// results go here
 				let mut results: Vec<$crate::BenchmarkResults> = Vec::new();
+
+				// Default number of steps for a component.
+				let mut prev_steps = &10;
+
 				// Select the component we will be benchmarking. Each component will be benchmarked.
-				for (name, low, high) in components.iter() {
+				for (idx, (name, low, high)) in components.iter().enumerate() {
+					// Get the number of steps for this component.
+					let steps = steps.get(idx).unwrap_or(&prev_steps);
+					prev_steps = steps;
+
 					// Create up to `STEPS` steps for that component between high and low.
 					let step_size = ((high - low) / steps).max(1);
-					let num_of_steps = (high - low) / step_size;
+					let num_of_steps = (high - low) / step_size + 1;
 					for s in 0..num_of_steps {
 						// This is the value we will be testing for component `name`
 						let component_value = low + step_size * s;
 
 						// Select the mid value for all the other components.
 						let c: Vec<($crate::BenchmarkParameter, u32)> = components.iter()
 							.map(|(n, l, h)|
-								(*n, if n == name { component_value } else { (h - l) / 2 + l })
+								(*n, if n == name { component_value } else { *h })
 							).collect();
 
 						// Run the benchmark `repeat` times.

frame/benchmarking/src/utils.rs

@@ -40,7 +40,7 @@ sp_api::decl_runtime_apis! {
 		fn dispatch_benchmark(
 			module: Vec<u8>,
 			extrinsic: Vec<u8>,
-			steps: u32,
+			steps: Vec<u32>,
 			repeat: u32,
 		) -> Option<Vec<BenchmarkResults>>;
 	}
@@ -78,7 +78,7 @@ pub trait Benchmarking<T> {
 	/// - `extrinsic`: The name of extrinsic function you want to benchmark encoded as bytes.
 	/// - `steps`: The number of sample points you want to take across the range of parameters.
 	/// - `repeat`: The number of times you want to repeat a benchmark.
-	fn run_benchmark(extrinsic: Vec<u8>, steps: u32, repeat: u32) -> Result<Vec<T>, &'static str>;
+	fn run_benchmark(extrinsic: Vec<u8>, steps: Vec<u32>, repeat: u32) -> Result<Vec<T>, &'static str>;
 }
 
 /// The required setup for creating a benchmark.

utils/frame/benchmarking-cli/src/lib.rs

@@ -36,8 +36,8 @@ pub struct BenchmarkCmd {
 	pub extrinsic: String,
 
 	/// Select how many samples we should take across the variable components.
-	#[structopt(short, long, default_value = "1")]
-	pub steps: u32,
+	#[structopt(short, long, use_delimiter = true)]
+	pub steps: Vec<u32>,
 
 	/// Select how many repetitions of this benchmark should run.
 	#[structopt(short, long, default_value = "1")]
@@ -97,13 +97,14 @@ impl BenchmarkCmd {
 			wasm_method,
 			None, // heap pages
 		);
+
 		let result = StateMachine::<_, _, NumberFor<BB>, _>::new(
 			&state,
 			None,
 			&mut changes,
 			&executor,
 			"Benchmark_dispatch_benchmark",
-			&(&self.pallet, &self.extrinsic, self.steps, self.repeat).encode(),
+			&(&self.pallet, &self.extrinsic, self.steps.clone(), self.repeat).encode(),
 			Default::default(),
 		)
 		.execute(strategy.into())