Seperated build directory from target directory

This commits implements the seperation of the intermidate artifact
directory (called "build directory") from the target directory. (see #14125)

Author: ranger-ross

src/cargo/core/compiler/build_runner/compilation_files.rs

@@ -217,7 +217,7 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
         } else if unit.target.is_custom_build() {
             self.build_script_dir(unit)
         } else if unit.target.is_example() {
-            self.layout(unit.kind).examples().to_path_buf()
+            self.layout(unit.kind).build_examples().to_path_buf()
         } else if unit.artifact.is_true() {
             self.artifact_dir(unit)
         } else {

src/cargo/core/compiler/custom_build.rs

@@ -1124,7 +1124,7 @@ fn prepare_metabuild(
     let path = unit
         .pkg
         .manifest()
-        .metabuild_path(build_runner.bcx.ws.target_dir());
+        .metabuild_path(build_runner.bcx.ws.build_dir());
     paths::create_dir_all(path.parent().unwrap())?;
     paths::write_if_changed(path, &output)?;
     Ok(())

src/cargo/core/compiler/fingerprint/dep_info.rs

@@ -51,9 +51,9 @@ pub struct RustcDepInfo {
 pub enum DepInfoPathType {
     /// src/, e.g. src/lib.rs
     PackageRootRelative,
-    /// target/debug/deps/lib...
+    /// {build-dir}/debug/deps/lib...
     /// or an absolute path /.../sysroot/...
-    TargetRootRelative,
+    BuildRootRelative,
 }
 
 /// Same as [`RustcDepInfo`] except avoids absolute paths as much as possible to
@@ -126,7 +126,7 @@ impl EncodedDepInfo {
         for _ in 0..nfiles {
             let ty = match read_u8(bytes)? {
                 0 => DepInfoPathType::PackageRootRelative,
-                1 => DepInfoPathType::TargetRootRelative,
+                1 => DepInfoPathType::BuildRootRelative,
                 _ => return None,
             };
             let path_bytes = read_bytes(bytes)?;
@@ -210,7 +210,7 @@ impl EncodedDepInfo {
         for (ty, file, checksum_info) in self.files.iter() {
             match ty {
                 DepInfoPathType::PackageRootRelative => dst.push(0),
-                DepInfoPathType::TargetRootRelative => dst.push(1),
+                DepInfoPathType::BuildRootRelative => dst.push(1),
             }
             write_bytes(dst, paths::path2bytes(file)?);
             write_bool(dst, checksum_info.is_some());
@@ -292,14 +292,14 @@ pub fn translate_dep_info(
     cargo_dep_info: &Path,
     rustc_cwd: &Path,
     pkg_root: &Path,
-    target_root: &Path,
+    build_root: &Path,
     rustc_cmd: &ProcessBuilder,
     allow_package: bool,
     env_config: &Arc<HashMap<String, OsString>>,
 ) -> CargoResult<()> {
     let depinfo = parse_rustc_dep_info(rustc_dep_info)?;
 
-    let target_root = crate::util::try_canonicalize(target_root)?;
+    let build_root = crate::util::try_canonicalize(build_root)?;
     let pkg_root = crate::util::try_canonicalize(pkg_root)?;
     let mut on_disk_info = EncodedDepInfo::default();
     on_disk_info.env = depinfo.env;
@@ -351,8 +351,8 @@ pub fn translate_dep_info(
         let canon_file =
             crate::util::try_canonicalize(&abs_file).unwrap_or_else(|_| abs_file.clone());
 
-        let (ty, path) = if let Ok(stripped) = canon_file.strip_prefix(&target_root) {
-            (DepInfoPathType::TargetRootRelative, stripped)
+        let (ty, path) = if let Ok(stripped) = canon_file.strip_prefix(&build_root) {
+            (DepInfoPathType::BuildRootRelative, stripped)
         } else if let Ok(stripped) = canon_file.strip_prefix(&pkg_root) {
             if !allow_package {
                 return None;
@@ -362,7 +362,7 @@ pub fn translate_dep_info(
             // It's definitely not target root relative, but this is an absolute path (since it was
             // joined to rustc_cwd) and as such re-joining it later to the target root will have no
             // effect.
-            (DepInfoPathType::TargetRootRelative, &*abs_file)
+            (DepInfoPathType::BuildRootRelative, &*abs_file)
         };
         Some((ty, path.to_owned()))
     };
@@ -472,7 +472,7 @@ pub fn parse_rustc_dep_info(rustc_dep_info: &Path) -> CargoResult<RustcDepInfo>
 /// indicates that the crate should likely be rebuilt.
 pub fn parse_dep_info(
     pkg_root: &Path,
-    target_root: &Path,
+    build_root: &Path,
     dep_info: &Path,
 ) -> CargoResult<Option<RustcDepInfo>> {
     let Ok(data) = paths::read_bytes(dep_info) else {
@@ -487,7 +487,7 @@ pub fn parse_dep_info(
     ret.files
         .extend(info.files.into_iter().map(|(ty, path, checksum_info)| {
             (
-                make_absolute_path(ty, pkg_root, target_root, path),
+                make_absolute_path(ty, pkg_root, build_root, path),
                 checksum_info.and_then(|(file_len, checksum)| {
                     Checksum::from_str(&checksum).ok().map(|c| (file_len, c))
                 }),
@@ -499,13 +499,13 @@ pub fn parse_dep_info(
 fn make_absolute_path(
     ty: DepInfoPathType,
     pkg_root: &Path,
-    target_root: &Path,
+    build_root: &Path,
     path: PathBuf,
 ) -> PathBuf {
     match ty {
         DepInfoPathType::PackageRootRelative => pkg_root.join(path),
         // N.B. path might be absolute here in which case the join will have no effect
-        DepInfoPathType::TargetRootRelative => target_root.join(path),
+        DepInfoPathType::BuildRootRelative => build_root.join(path),
     }
 }
 
@@ -678,7 +678,7 @@ mod encoded_dep_info {
     fn gen_test(checksum: bool) {
         let checksum = checksum.then_some((768, "c01efc669f09508b55eced32d3c88702578a7c3e".into()));
         let lib_rs = (
-            DepInfoPathType::TargetRootRelative,
+            DepInfoPathType::BuildRootRelative,
             PathBuf::from("src/lib.rs"),
             checksum.clone(),
         );
@@ -691,7 +691,7 @@ mod encoded_dep_info {
         assert_eq!(EncodedDepInfo::parse(&data).unwrap(), depinfo);
 
         let mod_rs = (
-            DepInfoPathType::TargetRootRelative,
+            DepInfoPathType::BuildRootRelative,
             PathBuf::from("src/mod.rs"),
             checksum.clone(),
         );

src/cargo/core/compiler/fingerprint/mod.rs

@@ -839,7 +839,7 @@ impl LocalFingerprint {
         mtime_cache: &mut HashMap<PathBuf, FileTime>,
         checksum_cache: &mut HashMap<PathBuf, Checksum>,
         pkg: &Package,
-        target_root: &Path,
+        build_root: &Path,
         cargo_exe: &Path,
         gctx: &GlobalContext,
     ) -> CargoResult<Option<StaleItem>> {
@@ -852,8 +852,8 @@ impl LocalFingerprint {
             // the `dep_info` file itself whose mtime represents the start of
             // rustc.
             LocalFingerprint::CheckDepInfo { dep_info, checksum } => {
-                let dep_info = target_root.join(dep_info);
-                let Some(info) = parse_dep_info(pkg_root, target_root, &dep_info)? else {
+                let dep_info = build_root.join(dep_info);
+                let Some(info) = parse_dep_info(pkg_root, build_root, &dep_info)? else {
                     return Ok(Some(StaleItem::MissingFile(dep_info)));
                 };
                 for (key, previous) in info.env.iter() {
@@ -910,7 +910,7 @@ impl LocalFingerprint {
             LocalFingerprint::RerunIfChanged { output, paths } => Ok(find_stale_file(
                 mtime_cache,
                 checksum_cache,
-                &target_root.join(output),
+                &build_root.join(output),
                 paths.iter().map(|p| (pkg_root.join(p), None)),
                 false,
             )),
@@ -1153,7 +1153,7 @@ impl Fingerprint {
         mtime_cache: &mut HashMap<PathBuf, FileTime>,
         checksum_cache: &mut HashMap<PathBuf, Checksum>,
         pkg: &Package,
-        target_root: &Path,
+        build_root: &Path,
         cargo_exe: &Path,
         gctx: &GlobalContext,
     ) -> CargoResult<()> {
@@ -1261,7 +1261,7 @@ impl Fingerprint {
                 mtime_cache,
                 checksum_cache,
                 pkg,
-                target_root,
+                build_root,
                 cargo_exe,
                 gctx,
             )? {
@@ -1449,13 +1449,13 @@ fn calculate(build_runner: &mut BuildRunner<'_, '_>, unit: &Unit) -> CargoResult
 
     // After we built the initial `Fingerprint` be sure to update the
     // `fs_status` field of it.
-    let target_root = target_root(build_runner);
+    let build_root = build_root(build_runner);
     let cargo_exe = build_runner.bcx.gctx.cargo_exe()?;
     fingerprint.check_filesystem(
         &mut build_runner.mtime_cache,
         &mut build_runner.checksum_cache,
         &unit.pkg,
-        &target_root,
+        &build_root,
         cargo_exe,
         build_runner.bcx.gctx,
     )?;
@@ -1493,7 +1493,7 @@ fn calculate_normal(
     };
 
     // Afterwards calculate our own fingerprint information.
-    let target_root = target_root(build_runner);
+    let build_root = build_root(build_runner);
     let local = if unit.mode.is_doc() || unit.mode.is_doc_scrape() {
         // rustdoc does not have dep-info files.
         let fingerprint = pkg_fingerprint(build_runner.bcx, &unit.pkg).with_context(|| {
@@ -1505,7 +1505,7 @@ fn calculate_normal(
         vec![LocalFingerprint::Precalculated(fingerprint)]
     } else {
         let dep_info = dep_info_loc(build_runner, unit);
-        let dep_info = dep_info.strip_prefix(&target_root).unwrap().to_path_buf();
+        let dep_info = dep_info.strip_prefix(&build_root).unwrap().to_path_buf();
         vec![LocalFingerprint::CheckDepInfo {
             dep_info,
             checksum: build_runner.bcx.gctx.cli_unstable().checksum_freshness,
@@ -1714,7 +1714,7 @@ fn build_script_local_fingerprints(
     // longstanding bug, in Cargo. Recent refactorings just made it painfully
     // obvious.
     let pkg_root = unit.pkg.root().to_path_buf();
-    let target_dir = target_root(build_runner);
+    let build_dir = build_root(build_runner);
     let env_config = Arc::clone(build_runner.bcx.gctx.env_config()?);
     let calculate =
         move |deps: &BuildDeps, pkg_fingerprint: Option<&dyn Fn() -> CargoResult<String>>| {
@@ -1747,7 +1747,7 @@ fn build_script_local_fingerprints(
             // them all here.
             Ok(Some(local_fingerprints_deps(
                 deps,
-                &target_dir,
+                &build_dir,
                 &pkg_root,
                 &env_config,
             )))
@@ -1783,7 +1783,7 @@ fn build_script_override_fingerprint(
 /// [`RunCustomBuild`]: crate::core::compiler::CompileMode::RunCustomBuild
 fn local_fingerprints_deps(
     deps: &BuildDeps,
-    target_root: &Path,
+    build_root: &Path,
     pkg_root: &Path,
     env_config: &Arc<HashMap<String, OsString>>,
 ) -> Vec<LocalFingerprint> {
@@ -1796,7 +1796,7 @@ fn local_fingerprints_deps(
         // absolute prefixes from them.
         let output = deps
             .build_script_output
-            .strip_prefix(target_root)
+            .strip_prefix(build_root)
             .unwrap()
             .to_path_buf();
         let paths = deps
@@ -1854,10 +1854,10 @@ pub fn dep_info_loc(build_runner: &mut BuildRunner<'_, '_>, unit: &Unit) -> Path
     build_runner.files().fingerprint_file_path(unit, "dep-")
 }
 
-/// Returns an absolute path that target directory.
+/// Returns an absolute path that build directory.
 /// All paths are rewritten to be relative to this.
-fn target_root(build_runner: &BuildRunner<'_, '_>) -> PathBuf {
-    build_runner.bcx.ws.target_dir().into_path_unlocked()
+fn build_root(build_runner: &BuildRunner<'_, '_>) -> PathBuf {
+    build_runner.bcx.ws.build_dir().into_path_unlocked()
 }
 
 /// Reads the value from the old fingerprint hash file and compare.

src/cargo/core/compiler/layout.rs

@@ -128,13 +128,20 @@ pub struct Layout {
     fingerprint: PathBuf,
     /// The directory for examples: `$dest/examples`
     examples: PathBuf,
+    /// The directory for pre-uplifted examples: `$build-dir/debug/examples`
+    build_examples: PathBuf,
     /// The directory for rustdoc output: `$root/doc`
     doc: PathBuf,
     /// The directory for temporary data of integration tests and benches: `$dest/tmp`
     tmp: PathBuf,
     /// The lockfile for a build (`.cargo-lock`). Will be unlocked when this
     /// struct is `drop`ped.
     _lock: FileLock,
+    /// Same as `_lock` but for the build directory.
+    ///
+    /// Will be `None` when the build-dir and target-dir are the same path as we cannot
+    /// lock the same path twice.
+    _build_lock: Option<FileLock>,
 }
 
 impl Layout {
@@ -150,15 +157,22 @@ impl Layout {
         dest: &str,
     ) -> CargoResult<Layout> {
         let mut root = ws.target_dir();
+        let mut build_root = ws.build_dir();
         if let Some(target) = target {
             root.push(target.short_name());
+            build_root.push(target.short_name());
         }
+        let build_dest = build_root.join(dest);
         let dest = root.join(dest);
         // If the root directory doesn't already exist go ahead and create it
         // here. Use this opportunity to exclude it from backups as well if the
         // system supports it since this is a freshly created folder.
         //
         paths::create_dir_all_excluded_from_backups_atomic(root.as_path_unlocked())?;
+        if root != build_root {
+            paths::create_dir_all_excluded_from_backups_atomic(build_root.as_path_unlocked())?;
+        }
+
         // Now that the excluded from backups target root is created we can create the
         // actual destination (sub)subdirectory.
         paths::create_dir_all(dest.as_path_unlocked())?;
@@ -167,23 +181,37 @@ impl Layout {
         // directory, so just lock the entire thing for the duration of this
         // compile.
         let lock = dest.open_rw_exclusive_create(".cargo-lock", ws.gctx(), "build directory")?;
+
+        let build_lock = if root != build_root {
+            Some(build_dest.open_rw_exclusive_create(
+                ".cargo-lock",
+                ws.gctx(),
+                "build directory",
+            )?)
+        } else {
+            None
+        };
         let root = root.into_path_unlocked();
+        let build_root = build_root.into_path_unlocked();
         let dest = dest.into_path_unlocked();
-        let deps = dest.join("deps");
+        let build_dest = build_dest.as_path_unlocked();
+        let deps = build_dest.join("deps");
         let artifact = deps.join("artifact");
 
         Ok(Layout {
             deps,
-            build: dest.join("build"),
+            build: build_dest.join("build"),
             artifact,
-            incremental: dest.join("incremental"),
-            fingerprint: dest.join(".fingerprint"),
+            incremental: build_dest.join("incremental"),
+            fingerprint: build_dest.join(".fingerprint"),
             examples: dest.join("examples"),
+            build_examples: build_dest.join("examples"),
             doc: root.join("doc"),
-            tmp: root.join("tmp"),
+            tmp: build_root.join("tmp"),
             root,
             dest,
             _lock: lock,
+            _build_lock: build_lock,
         })
     }
 
@@ -193,6 +221,7 @@ impl Layout {
         paths::create_dir_all(&self.incremental)?;
         paths::create_dir_all(&self.fingerprint)?;
         paths::create_dir_all(&self.examples)?;
+        paths::create_dir_all(&self.build_examples)?;
         paths::create_dir_all(&self.build)?;
 
         Ok(())
@@ -210,6 +239,10 @@ impl Layout {
     pub fn examples(&self) -> &Path {
         &self.examples
     }
+    /// Fetch the build examples path.
+    pub fn build_examples(&self) -> &Path {
+        &self.build_examples
+    }
     /// Fetch the doc path.
     pub fn doc(&self) -> &Path {
         &self.doc