Auto merge of rust-lang#138597 - Kobzol:git-ci-beta, r=<try>

[do not merge] beta test for git change detection (rust-lang#138591)

Opening to test CI/bootstrap changes.

r? `@ghost`

try-job: x86_64-gnu-stable
try-job: x86_64-gnu
try-job: x86_64-gnu-llvm-19-1
try-job: dist-x86_64-linux

Author: bors

.github/workflows/ci.yml

@@ -111,9 +111,6 @@ jobs:
           # which then uses log commands to actually set them.
           EXTRA_VARIABLES: ${{ toJson(matrix.env) }}
 
-      - name: setup upstream remote
-        run: src/ci/scripts/setup-upstream-remote.sh
-
       - name: ensure the channel matches the target branch
         run: src/ci/scripts/verify-channel.sh
 

Cargo.lock

@@ -317,6 +317,7 @@ version = "0.1.0"
 dependencies = [
  "serde",
  "serde_derive",
+ "tempfile",
 ]
 
 [[package]]
@@ -2135,6 +2136,12 @@ version = "0.4.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
 
+[[package]]
+name = "linux-raw-sys"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413"
+
 [[package]]
 name = "litemap"
 version = "0.7.4"
@@ -4832,7 +4839,20 @@ dependencies = [
  "bitflags",
  "errno",
  "libc",
- "linux-raw-sys",
+ "linux-raw-sys 0.4.15",
+ "windows-sys 0.59.0",
+]
+
+[[package]]
+name = "rustix"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7178faa4b75a30e269c71e61c353ce2748cf3d76f0c44c393f4e60abf49b825"
+dependencies = [
+ "bitflags",
+ "errno",
+ "libc",
+ "linux-raw-sys 0.9.3",
  "windows-sys 0.59.0",
 ]
 
@@ -5246,15 +5266,14 @@ dependencies = [
 
 [[package]]
 name = "tempfile"
-version = "3.15.0"
+version = "3.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704"
+checksum = "488960f40a3fd53d72c2a29a58722561dee8afdd175bd88e3db4677d7b2ba600"
 dependencies = [
- "cfg-if",
  "fastrand",
- "getrandom 0.2.15",
+ "getrandom 0.3.1",
  "once_cell",
- "rustix",
+ "rustix 1.0.2",
  "windows-sys 0.59.0",
 ]
 
@@ -6599,8 +6618,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e105d177a3871454f754b33bb0ee637ecaaac997446375fd3e5d43a2ed00c909"
 dependencies = [
  "libc",
- "linux-raw-sys",
- "rustix",
+ "linux-raw-sys 0.4.15",
+ "rustix 0.38.43",
 ]
 
 [[package]]

src/bootstrap/Cargo.lock

@@ -225,12 +225,12 @@ dependencies = [
 
 [[package]]
 name = "errno"
-version = "0.3.9"
+version = "0.3.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba"
+checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d"
 dependencies = [
  "libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -334,9 +334,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
 
 [[package]]
 name = "libc"
-version = "0.2.167"
+version = "0.2.171"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc"
+checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6"
 
 [[package]]
 name = "libredox"

src/bootstrap/src/core/build_steps/compile.rs

@@ -107,7 +107,11 @@ impl Step for Std {
         // the `rust.download-rustc=true` option.
         let force_recompile = builder.rust_info().is_managed_git_subrepository()
             && builder.download_rustc()
-            && builder.config.last_modified_commit(&["library"], "download-rustc", true).is_none();
+            && builder.config.has_changes_from_upstream(&["library"]);
+
+        trace!("is managed git repo: {}", builder.rust_info().is_managed_git_subrepository());
+        trace!("download_rustc: {}", builder.download_rustc());
+        trace!(force_recompile);
 
         run.builder.ensure(Std {
             compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()),

src/bootstrap/src/core/build_steps/format.rs

@@ -94,7 +94,7 @@ fn get_modified_rs_files(build: &Builder<'_>) -> Result<Option<Vec<String>>, Str
         return Ok(None);
     }
 
-    get_git_modified_files(&build.config.git_config(), Some(&build.config.src), &["rs"])
+    get_git_modified_files(&build.config.git_config(), Some(&build.config.src), &["rs"]).map(Some)
 }
 
 #[derive(serde_derive::Deserialize)]

src/bootstrap/src/core/build_steps/gcc.rs

@@ -14,13 +14,72 @@ use std::sync::OnceLock;
 
 use build_helper::ci::CiEnv;
 
-use crate::Kind;
-use crate::core::builder::{Builder, RunConfig, ShouldRun, Step};
+use crate::core::builder::{Builder, Cargo, Kind, RunConfig, ShouldRun, Step};
 use crate::core::config::TargetSelection;
 use crate::utils::build_stamp::{BuildStamp, generate_smart_stamp_hash};
 use crate::utils::exec::command;
 use crate::utils::helpers::{self, t};
 
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
+pub struct Gcc {
+    pub target: TargetSelection,
+}
+
+#[derive(Clone)]
+pub struct GccOutput {
+    pub libgccjit: PathBuf,
+}
+
+impl Step for Gcc {
+    type Output = GccOutput;
+
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/gcc").alias("gcc")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(Gcc { target: run.target });
+    }
+
+    /// Compile GCC (specifically `libgccjit`) for `target`.
+    fn run(self, builder: &Builder<'_>) -> Self::Output {
+        let target = self.target;
+
+        // If GCC has already been built, we avoid building it again.
+        let metadata = match get_gcc_build_status(builder, target) {
+            GccBuildStatus::AlreadyBuilt(path) => return GccOutput { libgccjit: path },
+            GccBuildStatus::ShouldBuild(m) => m,
+        };
+
+        let _guard = builder.msg_unstaged(Kind::Build, "GCC", target);
+        t!(metadata.stamp.remove());
+        let _time = helpers::timeit(builder);
+
+        let libgccjit_path = libgccjit_built_path(&metadata.install_dir);
+        if builder.config.dry_run() {
+            return GccOutput { libgccjit: libgccjit_path };
+        }
+
+        build_gcc(&metadata, builder, target);
+        create_lib_alias(builder, &libgccjit_path);
+
+        t!(metadata.stamp.write());
+
+        GccOutput { libgccjit: libgccjit_path }
+    }
+}
+
+/// Creates a libgccjit.so.0 alias next to libgccjit.so if it does not
+/// already exist
+fn create_lib_alias(builder: &Builder<'_>, libgccjit: &PathBuf) {
+    let lib_alias = libgccjit.parent().unwrap().join("libgccjit.so.0");
+    if !lib_alias.exists() {
+        t!(builder.symlink_file(libgccjit, lib_alias));
+    }
+}
+
 pub struct Meta {
     stamp: BuildStamp,
     out_dir: PathBuf,
@@ -33,7 +92,54 @@ pub enum GccBuildStatus {
     ShouldBuild(Meta),
 }
 
-/// This returns whether we've already previously built GCC.
+/// Tries to download GCC from CI if it is enabled and GCC artifacts
+/// are available for the given target.
+/// Returns a path to the libgccjit.so file.
+#[cfg(not(test))]
+fn try_download_gcc(builder: &Builder<'_>, target: TargetSelection) -> Option<PathBuf> {
+    use build_helper::git::PathFreshness;
+
+    // Try to download GCC from CI if configured and available
+    if !matches!(builder.config.gcc_ci_mode, crate::core::config::GccCiMode::DownloadFromCi) {
+        return None;
+    }
+    if target != "x86_64-unknown-linux-gnu" {
+        eprintln!("GCC CI download is only available for the `x86_64-unknown-linux-gnu` target");
+        return None;
+    }
+    let source = detect_gcc_freshness(
+        &builder.config,
+        builder.config.rust_info.is_managed_git_subrepository(),
+    );
+    match source {
+        PathFreshness::LastModifiedUpstream { upstream } => {
+            // Download from upstream CI
+            let root = ci_gcc_root(&builder.config);
+            let gcc_stamp = BuildStamp::new(&root).with_prefix("gcc").add_stamp(&upstream);
+            if !gcc_stamp.is_up_to_date() && !builder.config.dry_run() {
+                builder.config.download_ci_gcc(&upstream, &root);
+                t!(gcc_stamp.write());
+            }
+
+            let libgccjit = root.join("lib").join("libgccjit.so");
+            create_lib_alias(builder, &libgccjit);
+            Some(libgccjit)
+        }
+        PathFreshness::HasLocalModifications { .. } => {
+            // We have local modifications, rebuild GCC.
+            eprintln!("Found local GCC modifications, GCC will *not* be downloaded");
+            None
+        }
+    }
+}
+
+#[cfg(test)]
+fn try_download_gcc(_builder: &Builder<'_>, _target: TargetSelection) -> Option<PathBuf> {
+    None
+}
+
+/// This returns information about whether GCC should be built or if it's already built.
+/// It transparently handles downloading GCC from CI if needed.
 ///
 /// It's used to avoid busting caches during x.py check -- if we've already built
 /// GCC, it's fine for us to not try to avoid doing so.
@@ -178,6 +284,34 @@ impl Step for Gcc {
 
         t!(stamp.write());
 
-        true
-    }
+/// Detect whether GCC sources have been modified locally or not.
+#[cfg(not(test))]
+fn detect_gcc_freshness(config: &crate::Config, is_git: bool) -> build_helper::git::PathFreshness {
+    use build_helper::git::{PathFreshness, check_path_modifications};
+
+    let freshness = if is_git {
+        Some(
+            check_path_modifications(
+                Some(&config.src),
+                &config.git_config(),
+                &["src/gcc", "src/bootstrap/download-ci-gcc-stamp"],
+                CiEnv::current(),
+            )
+            .unwrap(),
+        )
+    } else if let Some(info) = crate::utils::channel::read_commit_info_file(&config.src) {
+        Some(PathFreshness::LastModifiedUpstream { upstream: info.sha.trim().to_owned() })
+    } else {
+        None
+    };
+
+    let Some(freshness) = freshness else {
+        eprintln!("error: could not find commit hash for downloading GCC");
+        eprintln!("HELP: maybe your repository history is too shallow?");
+        eprintln!("HELP: consider disabling `download-ci-gcc`");
+        eprintln!("HELP: or fetch enough history to include one upstream commit");
+        panic!();
+    };
+
+    freshness
 }

src/bootstrap/src/core/build_steps/llvm.rs

@@ -15,7 +15,9 @@ use std::sync::OnceLock;
 use std::{env, fs};
 
 use build_helper::ci::CiEnv;
-use build_helper::git::get_closest_merge_commit;
+use build_helper::git::{PathFreshness, check_path_modifications};
+#[cfg(feature = "tracing")]
+use tracing::instrument;
 
 use crate::core::builder::{Builder, RunConfig, ShouldRun, Step};
 use crate::core::config::{Config, TargetSelection};
@@ -172,35 +174,38 @@ pub fn prebuilt_llvm_config(
     LlvmBuildStatus::ShouldBuild(Meta { stamp, res, out_dir, root: root.into() })
 }
 
-/// This retrieves the LLVM sha we *want* to use, according to git history.
-pub(crate) fn detect_llvm_sha(config: &Config, is_git: bool) -> String {
-    let llvm_sha = if is_git {
-        get_closest_merge_commit(
-            Some(&config.src),
-            &config.git_config(),
-            &[
-                config.src.join("src/llvm-project"),
-                config.src.join("src/bootstrap/download-ci-llvm-stamp"),
-                // the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
-                config.src.join("src/version"),
-            ],
+/// Detect whether LLVM sources have been modified locally or not.
+pub(crate) fn detect_llvm_freshness(config: &Config, is_git: bool) -> PathFreshness {
+    let freshness = if is_git {
+        Some(
+            check_path_modifications(
+                Some(&config.src),
+                &config.git_config(),
+                &[
+                    "src/llvm-project",
+                    "src/bootstrap/download-ci-llvm-stamp",
+                    // the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
+                    "src/version",
+                ],
+                CiEnv::current(),
+            )
+            .unwrap(),
         )
-        .unwrap()
     } else if let Some(info) = crate::utils::channel::read_commit_info_file(&config.src) {
-        info.sha.trim().to_owned()
+        Some(PathFreshness::LastModifiedUpstream { upstream: info.sha.trim().to_owned() })
     } else {
-        "".to_owned()
+        None
     };
 
-    if llvm_sha.is_empty() {
+    let Some(freshness) = freshness else {
         eprintln!("error: could not find commit hash for downloading LLVM");
         eprintln!("HELP: maybe your repository history is too shallow?");
         eprintln!("HELP: consider disabling `download-ci-llvm`");
         eprintln!("HELP: or fetch enough history to include one upstream commit");
         panic!();
-    }
+    };
 
-    llvm_sha
+    freshness
 }
 
 /// Returns whether the CI-found LLVM is currently usable.
@@ -280,12 +285,7 @@ pub(crate) fn is_ci_llvm_modified(config: &Config) -> bool {
         return false;
     }
 
-    let llvm_sha = detect_llvm_sha(config, true);
-    let head_sha = crate::output(
-        helpers::git(Some(&config.src)).arg("rev-parse").arg("HEAD").as_command_mut(),
-    );
-    let head_sha = head_sha.trim();
-    llvm_sha == head_sha
+    matches!(detect_llvm_freshness(config, true), PathFreshness::HasLocalModifications { .. })
 }
 
 #[derive(Debug, Clone, Hash, PartialEq, Eq)]

src/bootstrap/src/core/build_steps/tool.rs

@@ -604,8 +604,7 @@ impl Step for Rustdoc {
             let files_to_track = &["src/librustdoc", "src/tools/rustdoc"];
 
             // Check if unchanged
-            if builder.config.last_modified_commit(files_to_track, "download-rustc", true).is_some()
-            {
+            if !builder.config.has_changes_from_upstream(files_to_track) {
                 let precompiled_rustdoc = builder
                     .config
                     .ci_rustc_dir()