From 2b5a9821eb730ab2e294bc2600af8ee580421e27 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Wed, 17 Jul 2024 18:14:54 +0300 Subject: [PATCH 1/2] abstract merge-base commit logic Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/llvm.rs | 33 ++++++++------------ src/bootstrap/src/core/config/config.rs | 36 ++++++++++------------ src/bootstrap/src/utils/helpers.rs | 24 +++++++++++++++ src/tools/build_helper/src/git.rs | 2 +- 4 files changed, 54 insertions(+), 41 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 888290a0479ba..b6b8906746206 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -26,7 +26,6 @@ use crate::{generate_smart_stamp_hash, CLang, GitRepo, Kind}; use crate::utils::exec::command; use build_helper::ci::CiEnv; -use build_helper::git::get_git_merge_base; #[derive(Clone)] pub struct LlvmResult { @@ -154,26 +153,18 @@ pub fn prebuilt_llvm_config(builder: &Builder<'_>, target: TargetSelection) -> L /// This retrieves the LLVM sha we *want* to use, according to git history. pub(crate) fn detect_llvm_sha(config: &Config, is_git: bool) -> String { let llvm_sha = if is_git { - // We proceed in 2 steps. First we get the closest commit that is actually upstream. Then we - // walk back further to the last bors merge commit that actually changed LLVM. The first - // step will fail on CI because only the `auto` branch exists; we just fall back to `HEAD` - // in that case. - let closest_upstream = get_git_merge_base(&config.git_config(), Some(&config.src)) - .unwrap_or_else(|_| "HEAD".into()); - let mut rev_list = helpers::git(Some(&config.src)); - rev_list.args(&[ - PathBuf::from("rev-list"), - format!("--author={}", config.stage0_metadata.config.git_merge_commit_email).into(), - "-n1".into(), - "--first-parent".into(), - closest_upstream.into(), - "--".into(), - config.src.join("src/llvm-project"), - config.src.join("src/bootstrap/download-ci-llvm-stamp"), - // the LLVM shared object file is named `LLVM-12-rust-{version}-nightly` - config.src.join("src/version"), - ]); - output(rev_list.as_command_mut()).trim().to_owned() + helpers::get_closest_merge_base_commit( + Some(&config.src), + &config.git_config(), + &config.stage0_metadata.config.git_merge_commit_email, + &[ + config.src.join("src/llvm-project"), + config.src.join("src/bootstrap/download-ci-llvm-stamp"), + // the LLVM shared object file is named `LLVM-12-rust-{version}-nightly` + config.src.join("src/version"), + ], + ) + .unwrap() } else if let Some(info) = channel::read_commit_info_file(&config.src) { info.sha.trim().to_owned() } else { diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index f96633b059a17..9d5aa795c6c0d 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -20,7 +20,7 @@ use crate::core::build_steps::llvm; use crate::core::config::flags::{Color, Flags, Warnings}; use crate::utils::cache::{Interned, INTERNER}; use crate::utils::channel::{self, GitInfo}; -use crate::utils::helpers::{self, exe, output, t}; +use crate::utils::helpers::{self, exe, get_closest_merge_base_commit, output, t}; use build_helper::exit; use serde::{Deserialize, Deserializer}; use serde_derive::Deserialize; @@ -2471,14 +2471,13 @@ impl Config { // Look for a version to compare to based on the current commit. // Only commits merged by bors will have CI artifacts. - let merge_base = output( - helpers::git(Some(&self.src)) - .arg("rev-list") - .arg(format!("--author={}", self.stage0_metadata.config.git_merge_commit_email)) - .args(["-n1", "--first-parent", "HEAD"]) - .as_command_mut(), - ); - let commit = merge_base.trim_end(); + let commit = get_closest_merge_base_commit( + Some(&self.src), + &self.git_config(), + &self.stage0_metadata.config.git_merge_commit_email, + &[], + ) + .unwrap(); if commit.is_empty() { println!("ERROR: could not find commit hash for downloading rustc"); println!("HELP: maybe your repository history is too shallow?"); @@ -2489,7 +2488,7 @@ impl Config { // Warn if there were changes to the compiler or standard library since the ancestor commit. let has_changes = !t!(helpers::git(Some(&self.src)) - .args(["diff-index", "--quiet", commit]) + .args(["diff-index", "--quiet", &commit]) .arg("--") .args([self.src.join("compiler"), self.src.join("library")]) .as_command_mut() @@ -2565,14 +2564,13 @@ impl Config { ) -> Option { // Look for a version to compare to based on the current commit. // Only commits merged by bors will have CI artifacts. - let merge_base = output( - helpers::git(Some(&self.src)) - .arg("rev-list") - .arg(format!("--author={}", self.stage0_metadata.config.git_merge_commit_email)) - .args(["-n1", "--first-parent", "HEAD"]) - .as_command_mut(), - ); - let commit = merge_base.trim_end(); + let commit = get_closest_merge_base_commit( + Some(&self.src), + &self.git_config(), + &self.stage0_metadata.config.git_merge_commit_email, + &[], + ) + .unwrap(); if commit.is_empty() { println!("error: could not find commit hash for downloading components from CI"); println!("help: maybe your repository history is too shallow?"); @@ -2583,7 +2581,7 @@ impl Config { // Warn if there were changes to the compiler or standard library since the ancestor commit. let mut git = helpers::git(Some(&self.src)); - git.args(["diff-index", "--quiet", commit, "--"]); + git.args(["diff-index", "--quiet", &commit, "--"]); // Handle running from a directory other than the top level let top_level = &self.src; diff --git a/src/bootstrap/src/utils/helpers.rs b/src/bootstrap/src/utils/helpers.rs index 3c82fa189be36..773a873e47cc0 100644 --- a/src/bootstrap/src/utils/helpers.rs +++ b/src/bootstrap/src/utils/helpers.rs @@ -3,6 +3,7 @@ //! Simple things like testing the various filesystem operations here and there, //! not a lot of interesting happenings here unfortunately. +use build_helper::git::{get_git_merge_base, output_result, GitConfig}; use build_helper::util::fail; use std::env; use std::ffi::OsStr; @@ -521,3 +522,26 @@ pub fn git(source_dir: Option<&Path>) -> BootstrapCommand { git } + +/// Returns the closest commit available from upstream for the given `author` and `target_paths`. +/// +/// If it fails to find the commit from upstream using `git merge-base`, fallbacks to HEAD. +pub fn get_closest_merge_base_commit( + source_dir: Option<&Path>, + config: &GitConfig<'_>, + author: &str, + target_paths: &[PathBuf], +) -> Result { + let mut git = git(source_dir).capture_stdout(); + + let merge_base = get_git_merge_base(config, source_dir).unwrap_or_else(|_| "HEAD".into()); + + git.arg(Path::new("rev-list")); + git.args([&format!("--author={author}"), "-n1", "--first-parent", &merge_base]); + + if !target_paths.is_empty() { + git.arg("--").args(target_paths); + } + + Ok(output_result(git.as_command_mut())?.trim().to_owned()) +} diff --git a/src/tools/build_helper/src/git.rs b/src/tools/build_helper/src/git.rs index b4522de6897d4..8be38dc855f7a 100644 --- a/src/tools/build_helper/src/git.rs +++ b/src/tools/build_helper/src/git.rs @@ -7,7 +7,7 @@ pub struct GitConfig<'a> { } /// Runs a command and returns the output -fn output_result(cmd: &mut Command) -> Result { +pub fn output_result(cmd: &mut Command) -> Result { let output = match cmd.stderr(Stdio::inherit()).output() { Ok(status) => status, Err(e) => return Err(format!("failed to run command: {:?}: {}", cmd, e)), From 0636293fb740c96e7a52cf53452b728b1cf9e834 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sun, 7 Jul 2024 19:02:32 +0300 Subject: [PATCH 2/2] use precompiled rustdoc with CI rustc When CI rustc is enabled and rustdoc sources are unchanged, we can use the precompiled rustdoc from the CI rustc's sysroot. This speeds up bootstrapping quite a lot by avoiding unnecessary rustdoc compilation. Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/tool.rs | 59 +++++++++++++++++++--- 1 file changed, 53 insertions(+), 6 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index 7bc410b9e8878..459e59e1b3446 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -9,7 +9,7 @@ use crate::core::builder::{Builder, Cargo as CargoCommand, RunConfig, ShouldRun, use crate::core::config::TargetSelection; use crate::utils::channel::GitInfo; use crate::utils::exec::{command, BootstrapCommand}; -use crate::utils::helpers::{add_dylib_path, exe, t}; +use crate::utils::helpers::{add_dylib_path, exe, get_closest_merge_base_commit, git, t}; use crate::Compiler; use crate::Mode; use crate::{gha, Kind}; @@ -554,6 +554,57 @@ impl Step for Rustdoc { } let target = target_compiler.host; + let bin_rustdoc = || { + let sysroot = builder.sysroot(target_compiler); + let bindir = sysroot.join("bin"); + t!(fs::create_dir_all(&bindir)); + let bin_rustdoc = bindir.join(exe("rustdoc", target_compiler.host)); + let _ = fs::remove_file(&bin_rustdoc); + bin_rustdoc + }; + + // If CI rustc is enabled and we haven't modified the rustdoc sources, + // use the precompiled rustdoc from CI rustc's sysroot to speed up bootstrapping. + if builder.download_rustc() + && target_compiler.stage > 0 + && builder.rust_info().is_managed_git_subrepository() + { + let commit = get_closest_merge_base_commit( + Some(&builder.config.src), + &builder.config.git_config(), + &builder.config.stage0_metadata.config.git_merge_commit_email, + &[], + ) + .unwrap(); + + let librustdoc_src = builder.config.src.join("src/librustdoc"); + let rustdoc_src = builder.config.src.join("src/tools/rustdoc"); + + // FIXME: The change detection logic here is quite similar to `Config::download_ci_rustc_commit`. + // It would be better to unify them. + let has_changes = !git(Some(&builder.config.src)) + .allow_failure() + .run_always() + .args(["diff-index", "--quiet", &commit]) + .arg("--") + .arg(librustdoc_src) + .arg(rustdoc_src) + .run(builder) + .is_success(); + + if !has_changes { + let precompiled_rustdoc = builder + .config + .ci_rustc_dir() + .join("bin") + .join(exe("rustdoc", target_compiler.host)); + + let bin_rustdoc = bin_rustdoc(); + builder.copy_link(&precompiled_rustdoc, &bin_rustdoc); + return bin_rustdoc; + } + } + let build_compiler = if builder.download_rustc() && target_compiler.stage == 1 { // We already have the stage 1 compiler, we don't need to cut the stage. builder.compiler(target_compiler.stage, builder.config.build) @@ -614,11 +665,7 @@ impl Step for Rustdoc { // don't create a stage0-sysroot/bin directory. if target_compiler.stage > 0 { - let sysroot = builder.sysroot(target_compiler); - let bindir = sysroot.join("bin"); - t!(fs::create_dir_all(&bindir)); - let bin_rustdoc = bindir.join(exe("rustdoc", target_compiler.host)); - let _ = fs::remove_file(&bin_rustdoc); + let bin_rustdoc = bin_rustdoc(); builder.copy_link(&tool_rustdoc, &bin_rustdoc); bin_rustdoc } else {