Implement fine grain locking for the target directory

src/cargo/core/compiler/build_runner/compilation_files.rs

@@ -10,6 +10,7 @@ use lazycell::LazyCell;
 use tracing::debug;
 
 use super::{BuildContext, BuildRunner, CompileKind, FileFlavor, Layout};
+use crate::core::compiler::layout::BuildUnitLockLocation;
 use crate::core::compiler::{CompileMode, CompileTarget, CrateType, FileType, Unit};
 use crate::core::{Target, TargetKind, Workspace};
 use crate::util::{self, CargoResult, StableHasher};
@@ -277,6 +278,13 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
         self.layout(unit.kind).build_dir().fingerprint(&dir)
     }
 
+    /// The path of the partial and full locks for a given build unit
+    /// when fine grain locking is enabled.
+    pub fn build_unit_lock(&self, unit: &Unit) -> BuildUnitLockLocation {
+        let dir = self.pkg_dir(unit);
+        self.layout(unit.kind).build_dir().build_unit_lock(&dir)
+    }
+
     /// Directory where incremental output for the given unit should go.
     pub fn incremental_dir(&self, unit: &Unit) -> &Path {
         self.layout(unit.kind).build_dir().incremental()

src/cargo/core/compiler/build_runner/mod.rs

@@ -6,15 +6,19 @@ use std::sync::{Arc, Mutex};
 
 use crate::core::PackageId;
 use crate::core::compiler::compilation::{self, UnitOutput};
+use crate::core::compiler::locking::LockingMode;
 use crate::core::compiler::{self, Unit, artifact};
 use crate::util::cache_lock::CacheLockMode;
 use crate::util::errors::CargoResult;
+use crate::util::flock::is_on_nfs_mount;
+use crate::util::rlimit;
 use annotate_snippets::{Level, Message};
 use anyhow::{Context as _, bail};
 use cargo_util::paths;
 use filetime::FileTime;
 use itertools::Itertools;
 use jobserver::Client;
+use tracing::{debug, warn};
 
 use super::build_plan::BuildPlan;
 use super::custom_build::{self, BuildDeps, BuildScriptOutputs, BuildScripts};
@@ -89,6 +93,10 @@ pub struct BuildRunner<'a, 'gctx> {
     /// because the target has a type error. This is in an Arc<Mutex<..>>
     /// because it is continuously updated as the job progresses.
     pub failed_scrape_units: Arc<Mutex<HashSet<UnitHash>>>,
+
+    /// The locking mode to use for this build.
+    /// We use fine grain by default, but fallback to coarse grain for some systems.
+    pub locking_mode: LockingMode,
 }
 
 impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
@@ -111,6 +119,12 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
             }
         };
 
+        let locking_mode = if bcx.gctx.cli_unstable().fine_grain_locking {
+            determine_locking_mode(&bcx)?
+        } else {
+            LockingMode::Coarse
+        };
+
         Ok(Self {
             bcx,
             compilation: Compilation::new(bcx)?,
@@ -128,6 +142,7 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
             lto: HashMap::new(),
             metadata_for_doc_units: HashMap::new(),
             failed_scrape_units: Arc::new(Mutex::new(HashSet::new())),
+            locking_mode,
         })
     }
 
@@ -360,11 +375,11 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
     #[tracing::instrument(skip_all)]
     pub fn prepare_units(&mut self) -> CargoResult<()> {
         let dest = self.bcx.profiles.get_dir_name();
-        let host_layout = Layout::new(self.bcx.ws, None, &dest)?;
+        let host_layout = Layout::new(self.bcx.ws, None, &dest, &self.locking_mode)?;
         let mut targets = HashMap::new();
         for kind in self.bcx.all_kinds.iter() {
             if let CompileKind::Target(target) = *kind {
-                let layout = Layout::new(self.bcx.ws, Some(target), &dest)?;
+                let layout = Layout::new(self.bcx.ws, Some(target), &dest, &self.locking_mode)?;
                 targets.insert(target, layout);
             }
         }
@@ -724,3 +739,54 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
         }
     }
 }
+
+/// Determines if we have enough resources to safely use fine grain locking.
+/// This function will raises the number of max number file descriptors the current process
+/// can have open at once to make sure we are able to compile without running out of fds.
+///
+/// If we cannot acquire a safe max number of file descriptors, we fallback to coarse grain
+/// locking.
+pub fn determine_locking_mode(bcx: &BuildContext<'_, '_>) -> CargoResult<LockingMode> {
+    let total_units = bcx.unit_graph.keys().len() as u64;
+
+    if is_on_nfs_mount(bcx.ws.build_dir().as_path_unlocked()) {
+        debug!("NFS detected. Disabling file system locking");
+        return Ok(LockingMode::None);
+    }
+
+    // This is a bit arbitrary but if we do not have at least 10 times the remaining file
+    // descriptors than total build units there is a chance we could hit the limit.
+    // This is a fairly conservative estimate to make sure we don't hit max fd errors.
+    let safe_threshold = total_units * 10;
+
+    let Ok(mut limit) = rlimit::get_max_file_descriptors() else {
+        return Ok(LockingMode::Coarse);
+    };
+
+    if limit.soft_limit >= safe_threshold {
+        // The limit is higher or equal to what we deemed safe, so
+        // there is no need to raise the limit.
+        return Ok(LockingMode::Fine);
+    }
+
+    let display_fd_warning = || -> CargoResult<()> {
+        bcx.gctx.shell().verbose(|shell| shell.warn("ulimit was to low to safely enable fine grain locking, falling back to coarse grain locking"))
+    };
+
+    if limit.hard_limit < safe_threshold {
+        // The max we could raise the limit to is still not enough to safely compile.
+        display_fd_warning()?;
+        return Ok(LockingMode::Coarse);
+    }
+
+    limit.soft_limit = safe_threshold;
+
+    debug!("raising fd limit to {safe_threshold}");
+    if let Err(err) = rlimit::set_max_file_descriptors(limit) {
+        warn!("failed to raise max fds: {err}");
+        display_fd_warning()?;
+        return Ok(LockingMode::Coarse);
+    }
+
+    return Ok(LockingMode::Fine);
+}

src/cargo/core/compiler/layout.rs

@@ -103,6 +103,7 @@
 
 use crate::core::Workspace;
 use crate::core::compiler::CompileTarget;
+use crate::core::compiler::locking::LockingMode;
 use crate::util::{CargoResult, FileLock};
 use cargo_util::paths;
 use std::path::{Path, PathBuf};
@@ -126,6 +127,7 @@ impl Layout {
         ws: &Workspace<'_>,
         target: Option<CompileTarget>,
         dest: &str,
+        locking_mode: &LockingMode,
     ) -> CargoResult<Layout> {
         let is_new_layout = ws.gctx().cli_unstable().build_dir_new_layout;
         let mut root = ws.target_dir();
@@ -152,15 +154,30 @@ impl Layout {
         // For now we don't do any more finer-grained locking on the artifact
         // directory, so just lock the entire thing for the duration of this
         // compile.
-        let artifact_dir_lock =
-            dest.open_rw_exclusive_create(".cargo-lock", ws.gctx(), "build directory")?;
+        let artifact_dir_lock = match locking_mode {
+            LockingMode::None => None,
+            LockingMode::Fine => {
+                Some(dest.open_ro_shared_create(".cargo-lock", ws.gctx(), "build directory")?)
+            }
+            LockingMode::Coarse => {
+                Some(dest.open_rw_exclusive_create(".cargo-lock", ws.gctx(), "build directory")?)
+            }
+        };
 
         let build_dir_lock = if root != build_root {
-            Some(build_dest.open_rw_exclusive_create(
-                ".cargo-lock",
-                ws.gctx(),
-                "build directory",
-            )?)
+            match locking_mode {
+                LockingMode::None => None,
+                LockingMode::Fine => Some(build_dest.open_ro_shared_create(
+                    ".cargo-lock",
+                    ws.gctx(),
+                    "build directory",
+                )?),
+                LockingMode::Coarse => Some(build_dest.open_rw_exclusive_create(
+                    ".cargo-lock",
+                    ws.gctx(),
+                    "build directory",
+                )?),
+            }
         } else {
             None
         };
@@ -222,7 +239,7 @@ pub struct ArtifactDirLayout {
     timings: PathBuf,
     /// The lockfile for a build (`.cargo-lock`). Will be unlocked when this
     /// struct is `drop`ped.
-    _lock: FileLock,
+    _lock: Option<FileLock>,
 }
 
 impl ArtifactDirLayout {
@@ -345,6 +362,14 @@ impl BuildDirLayout {
             self.build().join(pkg_dir)
         }
     }
+    /// Fetch the lock paths for a build unit
+    pub fn build_unit_lock(&self, pkg_dir: &str) -> BuildUnitLockLocation {
+        let dir = self.build_unit(pkg_dir);
+        BuildUnitLockLocation {
+            partial: dir.join("partial.lock"),
+            full: dir.join("full.lock"),
+        }
+    }
     /// Fetch the artifact path.
     pub fn artifact(&self) -> &Path {
         &self.artifact
@@ -359,3 +384,10 @@ impl BuildDirLayout {
         Ok(&self.tmp)
     }
 }
+
+/// See [crate::core::compiler::locking] module docs for details about build system locking
+/// structure.
+pub struct BuildUnitLockLocation {
+    pub partial: PathBuf,
+    pub full: PathBuf,
+}