Auto merge of rust-lang#130739 - jieyouxu:stage0_run_make, r=Kobzol

Fix cargo staging for run-make tests

Follow-up to rust-lang#130642 (comment) to make sure that when

```
$ COMPILETEST_FORCE_STAGE0=1 ./x test run-make --stage 0
```

is used, bootstrap cargo is used in order to avoid building stage 1 rustc. Note that run-make tests are usually not written with `--stage 0` in mind and some tests may rely on stage1 rustc (nightly) behavior, and it is expected that some tests will fail under this invocation.

This PR also fixes `tool::Cargo` staging in compiletest when preparing for `run-make` test mode, by chopping off a stage from the `compiler` passed to `tool::Cargo` such that when the user invokes with stage `N`

```
./x test run-make --stage N
```

the `run-make` test suite will be tested against the cargo built by stage `N` compiler. Let's take `N=1`, i.e. `--stage 1`, without chopping off a stage, previously `./x test run-make --stage 1` will cause stage 1 rustc + std to be built, then stage 2 rustc, and cargo will be produced by the stage 2 rustc, which is clearly not what we want. By chopping off a stage, it means that cargo will be produced by the stage 1 rustc.

cc rust-lang#119946, rust-lang#59864.
See discussions regarding the tool staging at https://rust-lang.zulipchat.com/#narrow/stream/326414-t-infra.2Fbootstrap/topic/.E2.9C.94.20stage1.20run-make.20tests.20now.20need.20stage2.20rustc.20built.20for.20c.2E.2E.2E.

Author: bors

src/bootstrap/src/core/build_steps/test.rs

@@ -1730,8 +1730,23 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the
         let is_rustdoc = suite.ends_with("rustdoc-ui") || suite.ends_with("rustdoc-js");
 
         if mode == "run-make" {
-            let cargo = builder.ensure(tool::Cargo { compiler, target: compiler.host });
-            cmd.arg("--cargo-path").arg(cargo);
+            let cargo_path = if builder.top_stage == 0 {
+                // If we're using `--stage 0`, we should provide the bootstrap cargo.
+                builder.initial_cargo.clone()
+            } else {
+                // We need to properly build cargo using the suitable stage compiler.
+
+                // HACK: currently tool stages are off-by-one compared to compiler stages, i.e. if
+                // you give `tool::Cargo` a stage 1 rustc, it will cause stage 2 rustc to be built
+                // and produce a cargo built with stage 2 rustc. To fix this, we need to chop off
+                // the compiler stage by 1 to align with expected `./x test run-make --stage N`
+                // behavior, i.e. we need to pass `N - 1` compiler stage to cargo. See also Miri
+                // which does a similar hack.
+                let compiler = builder.compiler(builder.top_stage - 1, compiler.host);
+                builder.ensure(tool::Cargo { compiler, target: compiler.host })
+            };
+
+            cmd.arg("--cargo-path").arg(cargo_path);
         }
 
         // Avoid depending on rustdoc when we don't need it.
@@ -2088,8 +2103,6 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the
             cmd.arg("--rustfix-coverage");
         }
 
-        cmd.env("BOOTSTRAP_CARGO", &builder.initial_cargo);
-
         cmd.arg("--channel").arg(&builder.config.channel);
 
         if !builder.config.omit_git_hash {

src/tools/run-make-support/src/external_deps/cargo.rs

@@ -1,7 +1,8 @@
 use crate::command::Command;
 use crate::env_var;
 
-/// Returns a command that can be used to invoke Cargo.
+/// Returns a command that can be used to invoke cargo. The cargo is provided by compiletest
+/// through the `CARGO` env var.
 pub fn cargo() -> Command {
-    Command::new(env_var("BOOTSTRAP_CARGO"))
+    Command::new(env_var("CARGO"))
 }

tests/run-make/compiler-builtins/rmake.rs

@@ -15,46 +15,36 @@
 #![deny(warnings)]
 
 use std::collections::HashSet;
-use std::path::PathBuf;
 
 use run_make_support::object::read::Object;
 use run_make_support::object::read::archive::ArchiveFile;
 use run_make_support::object::{ObjectSection, ObjectSymbol, RelocationTarget};
 use run_make_support::rfs::{read, read_dir};
-use run_make_support::{cmd, env_var, object};
+use run_make_support::{cargo, object, path, target};
 
 fn main() {
-    let target_dir = PathBuf::from("target");
-    let target = env_var("TARGET");
-
-    println!("Testing compiler_builtins for {}", target);
-
-    let manifest_path = PathBuf::from("Cargo.toml");
-
-    let path = env_var("PATH");
-    let rustc = env_var("RUSTC");
-    let cargo = env_var("CARGO");
-    let mut cmd = cmd(cargo);
-    cmd.args(&[
-        "build",
-        "--manifest-path",
-        manifest_path.to_str().unwrap(),
-        "-Zbuild-std=core",
-        "--target",
-        &target,
-    ])
-    .env("PATH", path)
-    .env("RUSTC", rustc)
-    .env("RUSTFLAGS", "-Copt-level=0 -Cdebug-assertions=yes")
-    .env("CARGO_TARGET_DIR", &target_dir)
-    .env("RUSTC_BOOTSTRAP", "1")
-    // Visual Studio 2022 requires that the LIB env var be set so it can
-    // find the Windows SDK.
-    .env("LIB", std::env::var("LIB").unwrap_or_default());
-
-    cmd.run();
-
-    let rlibs_path = target_dir.join(target).join("debug").join("deps");
+    let target_dir = path("target");
+
+    println!("Testing compiler_builtins for {}", target());
+
+    cargo()
+        .args(&[
+            "build",
+            "--manifest-path",
+            "Cargo.toml",
+            "-Zbuild-std=core",
+            "--target",
+            &target(),
+        ])
+        .env("RUSTFLAGS", "-Copt-level=0 -Cdebug-assertions=yes")
+        .env("CARGO_TARGET_DIR", &target_dir)
+        .env("RUSTC_BOOTSTRAP", "1")
+        // Visual Studio 2022 requires that the LIB env var be set so it can
+        // find the Windows SDK.
+        .env("LIB", std::env::var("LIB").unwrap_or_default())
+        .run();
+
+    let rlibs_path = target_dir.join(target()).join("debug").join("deps");
     let compiler_builtins_rlib = read_dir(rlibs_path)
         .find_map(|e| {
             let path = e.unwrap().path();

tests/run-make/thumb-none-cortex-m/rmake.rs

@@ -14,10 +14,7 @@
 
 //@ only-thumb
 
-use std::path::PathBuf;
-
-use run_make_support::rfs::create_dir;
-use run_make_support::{cmd, env_var, target};
+use run_make_support::{cargo, cmd, env, env_var, target};
 
 const CRATE: &str = "cortex-m";
 const CRATE_URL: &str = "https://github.com/rust-embedded/cortex-m";
@@ -28,32 +25,21 @@ fn main() {
     // See below link for git usage:
     // https://stackoverflow.com/questions/3489173#14091182
     cmd("git").args(["clone", CRATE_URL, CRATE]).run();
-    std::env::set_current_dir(CRATE).unwrap();
+    env::set_current_dir(CRATE);
     cmd("git").args(["reset", "--hard", CRATE_SHA1]).run();
 
-    let target_dir = PathBuf::from("target");
-    let manifest_path = PathBuf::from("Cargo.toml");
-
-    let path = env_var("PATH");
-    let rustc = env_var("RUSTC");
-    let cargo = env_var("CARGO");
-    // FIXME: extract cargo invocations to a proper command
-    // https://github.com/rust-lang/rust/issues/128734
-    let mut cmd = cmd(cargo);
-    cmd.args(&[
-        "build",
-        "--manifest-path",
-        manifest_path.to_str().unwrap(),
-        "-Zbuild-std=core",
-        "--target",
-        &target(),
-    ])
-    .env("PATH", path)
-    .env("RUSTC", rustc)
-    .env("CARGO_TARGET_DIR", &target_dir)
-    // Don't make lints fatal, but they need to at least warn
-    // or they break Cargo's target info parsing.
-    .env("RUSTFLAGS", "-Copt-level=0 -Cdebug-assertions=yes --cap-lints=warn");
-
-    cmd.run();
+    cargo()
+        .args(&[
+            "build",
+            "--manifest-path",
+            "Cargo.toml",
+            "-Zbuild-std=core",
+            "--target",
+            &target(),
+        ])
+        .env("CARGO_TARGET_DIR", "target")
+        // Don't make lints fatal, but they need to at least warn
+        // or they break Cargo's target info parsing.
+        .env("RUSTFLAGS", "-Copt-level=0 -Cdebug-assertions=yes --cap-lints=warn")
+        .run();
 }

tests/run-make/thumb-none-qemu/rmake.rs

@@ -14,49 +14,34 @@
 //!
 //! FIXME: https://github.com/rust-lang/rust/issues/128733 this test uses external
 //! dependencies, and needs an active internet connection
-//!
-//! FIXME: https://github.com/rust-lang/rust/issues/128734 extract bootstrap cargo
-//! to a proper command
 
 //@ only-thumb
 
 use std::path::PathBuf;
 
-use run_make_support::{cmd, env_var, path_helpers, target};
+use run_make_support::{cargo, cmd, env_var, path, target};
 
 const CRATE: &str = "example";
 
 fn main() {
     std::env::set_current_dir(CRATE).unwrap();
 
-    let bootstrap_cargo = env_var("BOOTSTRAP_CARGO");
-    let path = env_var("PATH");
-    let rustc = env_var("RUSTC");
-
-    let target_dir = path_helpers::path("target");
-    let manifest_path = path_helpers::path("Cargo.toml");
-
-    let debug = {
-        let mut cmd = cmd(&bootstrap_cargo);
-        cmd.args(&["run", "--target", &target()])
-            .env("RUSTFLAGS", "-C linker=arm-none-eabi-ld -C link-arg=-Tlink.x")
-            .env("CARGO_TARGET_DIR", &target_dir)
-            .env("PATH", &path)
-            .env("RUSTC", &rustc);
-        cmd.run()
-    };
-
-    debug.assert_stdout_contains("x = 42");
-
-    let release = {
-        let mut cmd = cmd(&bootstrap_cargo);
-        cmd.args(&["run", "--release", "--target", &target()])
-            .env("RUSTFLAGS", "-C linker=arm-none-eabi-ld -C link-arg=-Tlink.x")
-            .env("CARGO_TARGET_DIR", &target_dir)
-            .env("PATH", &path)
-            .env("RUSTC", &rustc);
-        cmd.run()
-    };
-
-    release.assert_stdout_contains("x = 42");
+    let target_dir = path("target");
+    let manifest_path = path("Cargo.toml");
+
+    // Debug
+    cargo()
+        .args(&["run", "--target", &target()])
+        .env("RUSTFLAGS", "-C linker=arm-none-eabi-ld -C link-arg=-Tlink.x")
+        .env("CARGO_TARGET_DIR", &target_dir)
+        .run()
+        .assert_stdout_contains("x = 42");
+
+    // Release
+    cargo()
+        .args(&["run", "--release", "--target", &target()])
+        .env("RUSTFLAGS", "-C linker=arm-none-eabi-ld -C link-arg=-Tlink.x")
+        .env("CARGO_TARGET_DIR", &target_dir)
+        .run()
+        .assert_stdout_contains("x = 42");
 }