feat: build spec runtime (#647)

* feat(cli): allow build spec command to be non-interactive

* feat(cli): allow build spec command to specify the base directory

* fix(cli): lifecycle test

* test(cli): display error in conflictive test

* refactor: specify default locations at core logic

* test(cli): avoid hanging node process

* fix(cli): compilation issues

* ci: exclude integration tests from coverage

* test(cli): run coverage with nextest

* fix(cli): path canonicalization

* refactor(cli): simplify functions

* feat: adapt the build-spec command to use the runtime

* refactor(common): do not add unnecessary log entries

* fix: import errors

* fix: required chainspec parameters

* fix(cli): relay_chain field in raw chain specs

* test(chains): address some chainspec-related errors

* refactor(cli): do not use full path but alias

* feat(cli): use canonical path and display them as info

* fix(cli): use the correct node binary path

* fix: documentation tests

* feat(cli): allow to generate spec with node/runtime features

* refactor: clippy issues

* fix: compilation issues

* fix(cli): do not make features mandatory

* feat(cli): allow to skip the build

* fix: clippy issues

* fix: doc tests

* test(chain): add tests for ChainSpecBuilder

* feat(cli): get to choose the runtime preset to build

* fix(cli): correctly generate the raw chainspec if using runtime

* chore: add extra message for fetching presets

* fix: test for para id replacement

* fix: clippy issuefix: clippy issues

* fix: misssing import

* fix: documentation

* fix: correct fallback version of polkadot-omni-node

* fix: add tag pattern to polkadot-omni-node

* feat: reuse an existing spiner when fetching binaries

* fix: compilation issue

* test: add tests for the new build-spec command

* fix(cli): correctly set the default runtime location

* fix: some tests

* fix: release version ordering

* fix(cli): remove unused dependency

* refactor(chains): update build-spec documentation

Author: moliholy

Cargo.lock

@@ -77,6 +77,7 @@ srtool-lib = { version = "0.13.2", default-features = false }
 zombienet-configuration = { version = "0.3.1", default-features = false }
 zombienet-sdk = { version = "0.3.1", default-features = false }
 git2_credentials = "0.13.0"
+cumulus-client-cli = { version = "0.22.0", default-features = false }
 
 # benchmarking
 cumulus-primitives-proof-size-hostfunction = "0.12.0"

Cargo.toml

@@ -33,6 +33,8 @@ srtool-lib.workspace = true
 symlink.workspace = true
 toml_edit.workspace = true
 walkdir.workspace = true
+cumulus-client-cli.workspace = true
+
 # Zombienet
 zombienet-configuration.workspace = true
 zombienet-sdk.workspace = true

crates/pop-chains/Cargo.toml

@@ -25,34 +25,34 @@ Build a chain:
 
 ```rust,no_run
 use pop_common::Profile;
-use pop_chains::build_chain;
+use pop_chains::ChainSpecBuilder;
 use std::path::Path;
 
 let path = Path::new("./");
-let package = None;  // The optional package to be built.
-let binary_path = build_chain(&path, package, &Profile::Release, None, vec![]).unwrap();
+let builder = ChainSpecBuilder::Node { node_path: path.join("node"), default_bootnode: false, profile: Profile::Release };
+let binary_path = builder.build(&[]).unwrap();
 ```
 
 Build a chain with `runtime-benchmarks` feature:
 
 ```rs
-let binary_path = build_chain(&path, package, &Profile::Release, None, vec!["runtime-benchmarks"]).unwrap();
+let binary_path = builder.build(&["runtime-benchmarks".to_string()]).unwrap();
 ```
 
 Generate a plain chain specification file and customize it with your specific chain values:
 
 ```rust,no_run
 use pop_common::Profile;
-use pop_chains::{build_chain, export_wasm_file, generate_plain_chain_spec, generate_raw_chain_spec, generate_genesis_state_file, ChainSpec};
+use pop_chains::{ChainSpecBuilder, ChainSpec};
 use std::path::Path;
 
 let path = Path::new("./"); // Location of the parachain project.
-let package = None;  // The optional package to be built.
-// The path to the node binary executable.
-let binary_path = build_chain(&path, package, &Profile::Release, None, vec![]).unwrap();;
+let builder = ChainSpecBuilder::Node { node_path: path.join("node"), default_bootnode: false, profile: Profile::Release };
+// Build the node binary first
+builder.build(&[]).unwrap();
 // Generate a plain chain specification file of a parachain
 let plain_chain_spec_path = path.join("plain-parachain-chainspec.json");
-generate_plain_chain_spec(&binary_path, &plain_chain_spec_path, true, "dev");
+builder.generate_plain_chain_spec("dev", &plain_chain_spec_path).unwrap();
 // Customize your chain specification
 let mut chain_spec = ChainSpec::from(&plain_chain_spec_path).unwrap();
 chain_spec.replace_para_id(2002);
@@ -67,22 +67,22 @@ Generate a raw chain specification file and export the WASM and genesis state fi
 
 ```rust,no_run
 use pop_common::Profile;
-use pop_chains::{build_chain, export_wasm_file, generate_plain_chain_spec, generate_raw_chain_spec, generate_genesis_state_file};
+use pop_chains::{ChainSpecBuilder, generate_genesis_state_file_with_node};
 use std::path::Path;
 
 let path = Path::new("./"); // Location of the parachain project.
-let package = None;  // The optional package to be built.
-// The path to the node binary executable.
-let binary_path = build_chain(&path, package, &Profile::Release, None, vec![]).unwrap();;
+let builder = ChainSpecBuilder::Node { node_path: path.join("node"), default_bootnode: false, profile: Profile::Release };
+// Build the node binary first
+let binary_path = builder.build(&[]).unwrap();
 // Generate a plain chain specification file of a parachain
 let plain_chain_spec_path = path.join("plain-parachain-chainspec.json");
-generate_plain_chain_spec(&binary_path, &plain_chain_spec_path, true, "dev");
+builder.generate_plain_chain_spec("dev", &plain_chain_spec_path).unwrap();
 // Generate a raw chain specification file of a parachain
-let chain_spec = generate_raw_chain_spec(&binary_path, &plain_chain_spec_path, "raw-parachain-chainspec.json").unwrap();
+let chain_spec = builder.generate_raw_chain_spec(&plain_chain_spec_path, "raw-parachain-chainspec.json").unwrap();
 // Export the WebAssembly runtime for the parachain.
-let wasm_file = export_wasm_file(&binary_path, &chain_spec, "para-2000-wasm").unwrap();
+let wasm_file = builder.export_wasm_file(&chain_spec, "para-2000-wasm").unwrap();
 // Generate the parachain genesis state.
-let genesis_state_file = generate_genesis_state_file(&binary_path, &chain_spec, "para-2000-genesis-state").unwrap();
+let genesis_state_file = generate_genesis_state_file_with_node(&binary_path, &chain_spec, "para-2000-genesis-state").unwrap();
 ```
 
 Run a chain:

crates/pop-chains/README.md

@@ -1,17 +1,15 @@
 // SPDX-License-Identifier: GPL-3.0
 
-use crate::Error;
+use crate::{Error, utils::helpers::HostFunctions};
 use clap::Parser;
 use duct::cmd;
 use frame_benchmarking_cli::PalletCmd;
 pub use frame_benchmarking_cli::{BlockCmd, MachineCmd, OverheadCmd, StorageCmd};
-use sc_chain_spec::GenesisConfigBuilderRuntimeCaller;
 use serde::{Deserialize, Serialize};
 use sp_runtime::traits::BlakeTwo256;
 use std::{
 	collections::BTreeMap,
 	fmt::Display,
-	fs,
 	io::Read,
 	path::{Path, PathBuf},
 };
@@ -27,11 +25,6 @@ pub mod binary;
 /// (Recommended for testing with a single node, e.g., for benchmarking)
 pub const GENESIS_BUILDER_DEV_PRESET: &str = "development";
 
-type HostFunctions = (
-	sp_statement_store::runtime_api::HostFunctions,
-	cumulus_primitives_proof_size_hostfunction::storage_proof_size::HostFunctions,
-);
-
 /// Type alias for records where the key is the pallet name and the value is an array of its
 /// extrinsics.
 pub type PalletExtrinsicsRegistry = BTreeMap<String, Vec<String>>;
@@ -106,18 +99,6 @@ impl TryFrom<String> for GenesisBuilderPolicy {
 	}
 }
 
-/// Get genesis builder preset names of the runtime.
-///
-/// # Arguments
-/// * `binary_path` - Path to the runtime binary.
-pub fn get_preset_names(binary_path: &PathBuf) -> Result<Vec<String>, Error> {
-	let binary = fs::read(binary_path)?;
-	let genesis_config_builder = GenesisConfigBuilderRuntimeCaller::<HostFunctions>::new(&binary);
-	genesis_config_builder
-		.preset_names()
-		.map_err(|e| Error::GenesisBuilderError(e.to_string()))
-}
-
 /// Get the runtime folder path and throws error if it does not exist.
 ///
 /// # Arguments
@@ -263,6 +244,7 @@ pub fn generate_omni_bencher_benchmarks(
 mod tests {
 	use super::*;
 	use binary::omni_bencher_generator;
+	use std::fs;
 	use tempfile::tempdir;
 
 	#[test]