From fee481f9e970a980c4b567016940cbafc905ea54 Mon Sep 17 00:00:00 2001 From: tmpolaczyk <44604217+tmpolaczyk@users.noreply.github.com> Date: Wed, 24 Jul 2024 11:04:52 +0200 Subject: [PATCH] Use jobserver in wasm-builder to limit concurrency of spawned cargo processes (#4946) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When building multiple runtimes in parallel, each of them will try to use the concurrency set by the parent cargo process. For example, in a system with 8 cpu cores, building 3 runtimes in parallel creates 8 * 3 tasks. This results in the system hanging because of the high cpu and memory usage. This PR allows the substrate_wasm_builder to use the same [jobserver][1] as the parent cargo process, making all invocations of cargo share the same concurrency pool. So in a system with 8 cores, there will never be more than 8 tasks running at the same time. Implementation roughly based on [cargo][2] but with less unsafe. This can be tested by telling cargo to use half the cpu cores, like `cargo build -j4` in an 8 core machine. Before this PR it will use 100% cpu when building 2 runtimes in parallel, after this PR it will always use 50%. [1]: https://doc.rust-lang.org/cargo/reference/build-scripts.html#jobserver [2]: https://github.com/rust-lang/cargo/blob/d1b5f0759eedf5f1126c781c64232856956069ad/src/cargo/util/context/mod.rs#L271 --------- Co-authored-by: Bastian Köcher --- Cargo.lock | 1 + Cargo.toml | 1 + substrate/utils/wasm-builder/Cargo.toml | 1 + substrate/utils/wasm-builder/src/builder.rs | 2 ++ .../utils/wasm-builder/src/wasm_project.rs | 17 +++++++++++++++++ 5 files changed, 22 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 13e6b716e662..90415606bd35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21520,6 +21520,7 @@ dependencies = [ "console", "filetime", "frame-metadata", + "jobserver", "merkleized-metadata", "parity-scale-codec", "parity-wasm", diff --git a/Cargo.toml b/Cargo.toml index 1699bd97a8ee..9b14dfd3cdfb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -803,6 +803,7 @@ is-terminal = { version = "0.4.9" } is_executable = { version = "1.0.1" } isahc = { version = "1.2" } itertools = { version = "0.11" } +jobserver = { version = "0.1.26" } jsonpath_lib = { version = "0.3" } jsonrpsee = { version = "0.23.2" } jsonrpsee-core = { version = "0.23.2" } diff --git a/substrate/utils/wasm-builder/Cargo.toml b/substrate/utils/wasm-builder/Cargo.toml index f084400c12e8..28fdebf209bd 100644 --- a/substrate/utils/wasm-builder/Cargo.toml +++ b/substrate/utils/wasm-builder/Cargo.toml @@ -27,6 +27,7 @@ filetime = { workspace = true } wasm-opt = { workspace = true } parity-wasm = { workspace = true } polkavm-linker = { workspace = true } +jobserver = { workspace = true } # Dependencies required for the `metadata-hash` feature. merkleized-metadata = { optional = true, workspace = true } diff --git a/substrate/utils/wasm-builder/src/builder.rs b/substrate/utils/wasm-builder/src/builder.rs index 37c6c4aa7431..eb761a103d62 100644 --- a/substrate/utils/wasm-builder/src/builder.rs +++ b/substrate/utils/wasm-builder/src/builder.rs @@ -346,6 +346,8 @@ fn build_project( check_for_runtime_version_section: bool, #[cfg(feature = "metadata-hash")] enable_metadata_hash: Option, ) { + // Init jobserver as soon as possible + crate::wasm_project::get_jobserver(); let cargo_cmd = match crate::prerequisites::check(target) { Ok(cmd) => cmd, Err(err_msg) => { diff --git a/substrate/utils/wasm-builder/src/wasm_project.rs b/substrate/utils/wasm-builder/src/wasm_project.rs index 20fa9fc1aa33..63887389fb1e 100644 --- a/substrate/utils/wasm-builder/src/wasm_project.rs +++ b/substrate/utils/wasm-builder/src/wasm_project.rs @@ -31,6 +31,7 @@ use std::{ ops::Deref, path::{Path, PathBuf}, process, + sync::OnceLock, }; use strum::{EnumIter, IntoEnumIterator}; use toml::value::Table; @@ -899,6 +900,12 @@ fn build_bloaty_blob( } } + // Inherit jobserver in child cargo command to ensure we don't try to use more concurrency than + // available + if let Some(c) = get_jobserver() { + c.configure(&mut build_cmd); + } + println!("{}", colorize_info_message("Information that should be included in a bug report.")); println!("{} {:?}", colorize_info_message("Executing build command:"), build_cmd); println!("{} {}", colorize_info_message("Using rustc version:"), cargo_cmd.rustc_version()); @@ -1178,3 +1185,13 @@ fn copy_blob_to_target_directory(cargo_manifest: &Path, blob_binary: &WasmBinary ) .expect("Copies blob binary to `WASM_TARGET_DIRECTORY`."); } + +// Get jobserver from parent cargo command +pub fn get_jobserver() -> &'static Option { + static JOBSERVER: OnceLock> = OnceLock::new(); + + JOBSERVER.get_or_init(|| { + // Unsafe because it deals with raw fds + unsafe { jobserver::Client::from_env() } + }) +}