Skip to content

Add central execution context to bootstrap #141909

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 14 commits into from
Jun 10, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions src/bootstrap/src/core/build_steps/tool.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ use crate::core::builder::{
Builder, Cargo as CargoCommand, RunConfig, ShouldRun, Step, cargo_profile_var,
};
use crate::core::config::{DebuginfoLevel, RustcLto, TargetSelection};
use crate::utils::channel::GitInfo;
use crate::utils::exec::{BootstrapCommand, command};
use crate::utils::helpers::{add_dylib_path, exe, t};
use crate::{Compiler, FileType, Kind, Mode, gha};
Expand Down Expand Up @@ -278,7 +277,7 @@ pub fn prepare_tool_cargo(
cargo.env("CFG_VER_DESCRIPTION", description);
}

let info = GitInfo::new(builder.config.omit_git_hash, &dir);
let info = builder.config.git_info(builder.config.omit_git_hash, &dir);
if let Some(sha) = info.sha() {
cargo.env("CFG_COMMIT_HASH", sha);
}
Expand Down
2 changes: 1 addition & 1 deletion src/bootstrap/src/core/builder/cargo.rs
Original file line number Diff line number Diff line change
Expand Up @@ -551,7 +551,7 @@ impl Builder<'_> {
let libdir = self.rustc_libdir(compiler);

let sysroot_str = sysroot.as_os_str().to_str().expect("sysroot should be UTF-8");
if self.is_verbose() && !matches!(self.config.dry_run, DryRun::SelfCheck) {
if self.is_verbose() && !matches!(self.config.get_dry_run(), DryRun::SelfCheck) {
println!("using sysroot {sysroot_str}");
}

Expand Down
17 changes: 15 additions & 2 deletions src/bootstrap/src/core/builder/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ use crate::core::config::flags::Subcommand;
use crate::core::config::{DryRun, TargetSelection};
use crate::utils::cache::Cache;
use crate::utils::exec::{BootstrapCommand, command};
use crate::utils::execution_context::ExecutionContext;
use crate::utils::helpers::{self, LldThreads, add_dylib_path, exe, libdir, linker_args, t};
use crate::{Build, Crate, trace};

Expand Down Expand Up @@ -442,13 +443,15 @@ impl StepDescription {

fn is_excluded(&self, builder: &Builder<'_>, pathset: &PathSet) -> bool {
if builder.config.skip.iter().any(|e| pathset.has(e, builder.kind)) {
if !matches!(builder.config.dry_run, DryRun::SelfCheck) {
if !matches!(builder.config.get_dry_run(), DryRun::SelfCheck) {
println!("Skipping {pathset:?} because it is excluded");
}
return true;
}

if !builder.config.skip.is_empty() && !matches!(builder.config.dry_run, DryRun::SelfCheck) {
if !builder.config.skip.is_empty()
&& !matches!(builder.config.get_dry_run(), DryRun::SelfCheck)
{
builder.verbose(|| {
println!(
"{:?} not skipped for {:?} -- not in {:?}",
Expand Down Expand Up @@ -1633,4 +1636,14 @@ impl<'a> Builder<'a> {
self.info(&format!("{err}\n"));
}
}

pub fn exec_ctx(&self) -> &ExecutionContext {
&self.config.exec_ctx
}
}

impl<'a> AsRef<ExecutionContext> for Builder<'a> {
fn as_ref(&self) -> &ExecutionContext {
self.exec_ctx()
}
}
2 changes: 1 addition & 1 deletion src/bootstrap/src/core/builder/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ fn configure_with_args(cmd: &[String], host: &[&str], target: &[&str]) -> Config
let mut config = Config::parse(Flags::parse(cmd));
// don't save toolstates
config.save_toolstates = None;
config.dry_run = DryRun::SelfCheck;
config.set_dry_run(DryRun::SelfCheck);

// Ignore most submodules, since we don't need them for a dry run, and the
// tests run much faster without them.
Expand Down
138 changes: 76 additions & 62 deletions src/bootstrap/src/core/config/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ use std::{cmp, env, fs};

use build_helper::ci::CiEnv;
use build_helper::exit;
use build_helper::git::{GitConfig, PathFreshness, check_path_modifications, output_result};
use build_helper::git::{GitConfig, PathFreshness, check_path_modifications};
use serde::Deserialize;
#[cfg(feature = "tracing")]
use tracing::{instrument, span};
Expand All @@ -47,8 +47,10 @@ use crate::core::config::{
};
use crate::core::download::is_download_ci_available;
use crate::utils::channel;
use crate::utils::exec::command;
use crate::utils::execution_context::ExecutionContext;
use crate::utils::helpers::exe;
use crate::{Command, GitInfo, OnceLock, TargetSelection, check_ci_llvm, helpers, output, t};
use crate::{GitInfo, OnceLock, TargetSelection, check_ci_llvm, helpers, t};

/// Each path in this list is considered "allowed" in the `download-rustc="if-unchanged"` logic.
/// This means they can be modified and changes to these paths should never trigger a compiler build
Expand Down Expand Up @@ -129,7 +131,6 @@ pub struct Config {
pub jobs: Option<u32>,
pub cmd: Subcommand,
pub incremental: bool,
pub dry_run: DryRun,
pub dump_bootstrap_shims: bool,
/// Arguments appearing after `--` to be forwarded to tools,
/// e.g. `--fix-broken` or test arguments.
Expand Down Expand Up @@ -304,6 +305,8 @@ pub struct Config {
/// This is mostly for RA as building the stage1 compiler to check the library tree
/// on each code change might be too much for some computers.
pub skip_std_check_if_no_download_rustc: bool,

pub exec_ctx: ExecutionContext,
}

impl Config {
Expand Down Expand Up @@ -360,6 +363,14 @@ impl Config {
}
}

pub fn set_dry_run(&mut self, dry_run: DryRun) {
self.exec_ctx.set_dry_run(dry_run);
}

pub fn get_dry_run(&self) -> &DryRun {
self.exec_ctx.get_dry_run()
}

#[cfg_attr(
feature = "tracing",
instrument(target = "CONFIG_HANDLING", level = "trace", name = "Config::parse", skip_all)
Expand All @@ -382,6 +393,11 @@ impl Config {
get_toml: impl Fn(&Path) -> Result<TomlConfig, toml::de::Error>,
) -> Config {
let mut config = Config::default_opts();
let mut exec_ctx = ExecutionContext::new();
exec_ctx.set_verbose(flags.verbose);
exec_ctx.set_fail_fast(flags.cmd.fail_fast());

config.exec_ctx = exec_ctx;

// Set flags.
config.paths = std::mem::take(&mut flags.paths);
Expand Down Expand Up @@ -410,7 +426,7 @@ impl Config {
config.on_fail = flags.on_fail;
config.cmd = flags.cmd;
config.incremental = flags.incremental;
config.dry_run = if flags.dry_run { DryRun::UserSelected } else { DryRun::Disabled };
config.set_dry_run(if flags.dry_run { DryRun::UserSelected } else { DryRun::Disabled });
config.dump_bootstrap_shims = flags.dump_bootstrap_shims;
config.keep_stage = flags.keep_stage;
config.keep_stage_std = flags.keep_stage_std;
Expand Down Expand Up @@ -440,14 +456,9 @@ impl Config {
// has already been (kinda-cross-)compiled to Windows land, we require a normal Windows path.
cmd.arg("rev-parse").arg("--show-cdup");
// Discard stderr because we expect this to fail when building from a tarball.
let output = cmd
.as_command_mut()
.stderr(std::process::Stdio::null())
.output()
.ok()
.and_then(|output| if output.status.success() { Some(output) } else { None });
if let Some(output) = output {
let git_root_relative = String::from_utf8(output.stdout).unwrap();
let output = cmd.allow_failure().run_capture_stdout(&config);
if output.is_success() {
let git_root_relative = output.stdout();
// We need to canonicalize this path to make sure it uses backslashes instead of forward slashes,
// and to resolve any relative components.
let git_root = env::current_dir()
Expand Down Expand Up @@ -542,7 +553,7 @@ impl Config {
build.cargo = build.cargo.take().or(std::env::var_os("CARGO").map(|p| p.into()));
}

if GitInfo::new(false, &config.src).is_from_tarball() && toml.profile.is_none() {
if config.git_info(false, &config.src).is_from_tarball() && toml.profile.is_none() {
toml.profile = Some("dist".into());
}

Expand Down Expand Up @@ -749,7 +760,12 @@ impl Config {
};

config.initial_sysroot = t!(PathBuf::from_str(
output(Command::new(&config.initial_rustc).args(["--print", "sysroot"])).trim()
command(&config.initial_rustc)
.args(["--print", "sysroot"])
.run_always()
.run_capture_stdout(&config)
.stdout()
.trim()
));

config.initial_cargo_clippy = cargo_clippy;
Expand Down Expand Up @@ -845,19 +861,21 @@ impl Config {
let default = config.channel == "dev";
config.omit_git_hash = toml.rust.as_ref().and_then(|r| r.omit_git_hash).unwrap_or(default);

config.rust_info = GitInfo::new(config.omit_git_hash, &config.src);
config.cargo_info = GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/cargo"));
config.rust_info = config.git_info(config.omit_git_hash, &config.src);
config.cargo_info =
config.git_info(config.omit_git_hash, &config.src.join("src/tools/cargo"));
config.rust_analyzer_info =
GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/rust-analyzer"));
config.git_info(config.omit_git_hash, &config.src.join("src/tools/rust-analyzer"));
config.clippy_info =
GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/clippy"));
config.miri_info = GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/miri"));
config.git_info(config.omit_git_hash, &config.src.join("src/tools/clippy"));
config.miri_info =
config.git_info(config.omit_git_hash, &config.src.join("src/tools/miri"));
config.rustfmt_info =
GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/rustfmt"));
config.git_info(config.omit_git_hash, &config.src.join("src/tools/rustfmt"));
config.enzyme_info =
GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/enzyme"));
config.in_tree_llvm_info = GitInfo::new(false, &config.src.join("src/llvm-project"));
config.in_tree_gcc_info = GitInfo::new(false, &config.src.join("src/gcc"));
config.git_info(config.omit_git_hash, &config.src.join("src/tools/enzyme"));
config.in_tree_llvm_info = config.git_info(false, &config.src.join("src/llvm-project"));
config.in_tree_gcc_info = config.git_info(false, &config.src.join("src/gcc"));

config.vendor = vendor.unwrap_or(
config.rust_info.is_from_tarball()
Expand Down Expand Up @@ -1017,28 +1035,13 @@ impl Config {
}

pub fn dry_run(&self) -> bool {
match self.dry_run {
DryRun::Disabled => false,
DryRun::SelfCheck | DryRun::UserSelected => true,
}
self.exec_ctx.dry_run()
}

pub fn is_explicit_stage(&self) -> bool {
self.explicit_stage_from_cli || self.explicit_stage_from_config
}

/// Runs a command, printing out nice contextual information if it fails.
/// Exits if the command failed to execute at all, otherwise returns its
/// `status.success()`.
#[deprecated = "use `Builder::try_run` instead where possible"]
pub(crate) fn try_run(&self, cmd: &mut Command) -> Result<(), ()> {
if self.dry_run() {
return Ok(());
}
self.verbose(|| println!("running: {cmd:?}"));
build_helper::util::try_run(cmd, self.is_verbose())
}

pub(crate) fn test_args(&self) -> Vec<&str> {
let mut test_args = match self.cmd {
Subcommand::Test { ref test_args, .. }
Expand Down Expand Up @@ -1072,7 +1075,7 @@ impl Config {

let mut git = helpers::git(Some(&self.src));
git.arg("show").arg(format!("{commit}:{}", file.to_str().unwrap()));
output(git.as_command_mut())
git.run_capture_stdout(self).stdout()
}

/// Bootstrap embeds a version number into the name of shared libraries it uploads in CI.
Expand Down Expand Up @@ -1260,9 +1263,7 @@ impl Config {

/// Runs a function if verbosity is greater than 0
pub fn verbose(&self, f: impl Fn()) {
if self.is_verbose() {
f()
}
self.exec_ctx.verbose(f);
}

pub fn any_sanitizers_to_build(&self) -> bool {
Expand Down Expand Up @@ -1324,7 +1325,7 @@ impl Config {

// NOTE: The check for the empty directory is here because when running x.py the first time,
// the submodule won't be checked out. Check it out now so we can build it.
if !GitInfo::new(false, &absolute_path).is_managed_git_subrepository()
if !self.git_info(false, &absolute_path).is_managed_git_subrepository()
&& !helpers::dir_is_empty(&absolute_path)
{
return;
Expand All @@ -1343,16 +1344,16 @@ impl Config {
};

// Determine commit checked out in submodule.
let checked_out_hash = output(submodule_git().args(["rev-parse", "HEAD"]).as_command_mut());
let checked_out_hash =
submodule_git().args(["rev-parse", "HEAD"]).run_capture_stdout(self).stdout();
let checked_out_hash = checked_out_hash.trim_end();
// Determine commit that the submodule *should* have.
let recorded = output(
helpers::git(Some(&self.src))
.run_always()
.args(["ls-tree", "HEAD"])
.arg(relative_path)
.as_command_mut(),
);
let recorded = helpers::git(Some(&self.src))
.run_always()
.args(["ls-tree", "HEAD"])
.arg(relative_path)
.run_capture_stdout(self)
.stdout();

let actual_hash = recorded
.split_whitespace()
Expand All @@ -1376,20 +1377,18 @@ impl Config {
let update = |progress: bool| {
// Git is buggy and will try to fetch submodules from the tracking branch for *this* repository,
// even though that has no relation to the upstream for the submodule.
let current_branch = output_result(
helpers::git(Some(&self.src))
.allow_failure()
.run_always()
.args(["symbolic-ref", "--short", "HEAD"])
.as_command_mut(),
)
.map(|b| b.trim().to_owned());
let current_branch = helpers::git(Some(&self.src))
.allow_failure()
.run_always()
.args(["symbolic-ref", "--short", "HEAD"])
.run_capture(self);
Comment on lines -1379 to +1384
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When running ./x.py -vv build, I see following commands being executed, and then errors similar to those reported in #142350:

env -u GIT_ALTERNATE_OBJECT_DIRECTORIES -u GIT_DIR -u GIT_INDEX_FILE -u GIT_OBJECT_DIRECTORY -u GIT_WORK_TREE "git" "symbolic-ref" "--short" "HEAD" (failure_mode=Ignore) (created at src/bootstrap/src/core/config/config.rs:1393:34, executed at src/bootstrap/src/core/config/config.rs:1397:18)
env -u GIT_ALTERNATE_OBJECT_DIRECTORIES -u GIT_DIR -u GIT_INDEX_FILE -u GIT_OBJECT_DIRECTORY -u GIT_WORK_TREE "git" "-c" "branch.master\n.remote=origin" "submodule" "update" "--init" "--recursive" "--depth=1" "--progress" "library/stdarch" (failure_mode=Ignore)
error: invalid key (newline): branch.master

It looks like the new implementation no longer trims the trailing newline.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, I missed that, sorry. #142374 should fix this.


let mut git = helpers::git(Some(&self.src)).allow_failure();
git.run_always();
if let Ok(branch) = current_branch {
if current_branch.is_success() {
// If there is a tag named after the current branch, git will try to disambiguate by prepending `heads/` to the branch name.
// This syntax isn't accepted by `branch.{branch}`. Strip it.
let branch = current_branch.stdout();
let branch = branch.strip_prefix("heads/").unwrap_or(&branch);
git.arg("-c").arg(format!("branch.{branch}.remote=origin"));
}
Expand Down Expand Up @@ -1435,7 +1434,8 @@ impl Config {
return;
}

let stage0_output = output(Command::new(program_path).arg("--version"));
let stage0_output =
command(program_path).arg("--version").run_capture_stdout(self).stdout();
let mut stage0_output = stage0_output.lines().next().unwrap().split(' ');

let stage0_name = stage0_output.next().unwrap();
Expand Down Expand Up @@ -1741,4 +1741,18 @@ impl Config {
_ => !self.is_system_llvm(target),
}
}

pub fn exec_ctx(&self) -> &ExecutionContext {
&self.exec_ctx
}

pub fn git_info(&self, omit_git_hash: bool, dir: &Path) -> GitInfo {
GitInfo::new(omit_git_hash, dir, self)
}
}

impl AsRef<ExecutionContext> for Config {
fn as_ref(&self) -> &ExecutionContext {
&self.exec_ctx
}
}
3 changes: 2 additions & 1 deletion src/bootstrap/src/core/config/flags.rs
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,8 @@ impl Flags {
HelpVerboseOnly::try_parse_from(normalize_args(args))
{
println!("NOTE: updating submodules before printing available paths");
let config = Config::parse(Self::parse(&[String::from("build")]));
let flags = Self::parse(&[String::from("build")]);
let config = Config::parse(flags);
let build = Build::new(config);
let paths = Builder::get_help(&build, subcommand);
if let Some(s) = paths {
Expand Down
Loading
Loading