feat: support for rustdoc mergeable cross-crate info

crates/cargo-test-support/src/compare.rs

@@ -338,6 +338,7 @@ static E2E_LITERAL_REDACTIONS: &[(&str, &str)] = &[
     ("[BLOCKING]", "    Blocking"),
     ("[GENERATED]", "   Generated"),
     ("[OPENING]", "     Opening"),
+    ("[MERGING]", "     Merging"),
 ];
 
 /// Checks that the given string contains the given contiguous lines

src/cargo/core/compiler/build_context/target_info.rs

@@ -74,6 +74,8 @@ pub enum FileFlavor {
     DebugInfo,
     /// SBOM (Software Bill of Materials pre-cursor) file (e.g. cargo-sbon.json).
     Sbom,
+    /// Cross-crate info JSON files generated by rustdoc.
+    DocParts,
 }
 
 /// Type of each file generated by a Unit.
@@ -1191,6 +1193,19 @@ impl RustDocFingerprint {
             })
             .filter(|path| path.exists())
             .try_for_each(|path| clean_doc(path))?;
+
+        // Clean docdeps directory as well for `-Zrustdoc-mergeable-info`.
+        //
+        // This could potentially has a rustdoc version prefix
+        // so we can retain between different toolchain versions.
+        build_runner
+            .bcx
+            .all_kinds
+            .iter()
+            .map(|kind| build_runner.files().layout(*kind).build_dir().docdeps())
+            .filter(|path| path.exists())
+            .try_for_each(std::fs::remove_dir_all)?;
+
         write_fingerprint()?;
         return Ok(());
 

src/cargo/core/compiler/build_runner/compilation_files.rs

@@ -328,6 +328,13 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
             .build_script(&dir)
     }
 
+    /// Returns the directory where mergeable cross crate info for docs is stored.
+    pub fn docdeps_dir(&self, unit: &Unit) -> &Path {
+        assert!(unit.mode.is_doc());
+        assert!(self.metas.contains_key(unit));
+        self.layout(unit.kind).build_dir().docdeps()
+    }
+
     /// Returns the directory for compiled artifacts files.
     /// `/path/to/target/{debug,release}/deps/artifact/KIND/PKG-HASH`
     fn artifact_dir(&self, unit: &Unit) -> PathBuf {
@@ -500,12 +507,26 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
                         .join("index.html")
                 };
 
-                vec![OutputFile {
+                let mut outputs = vec![OutputFile {
                     path,
                     hardlink: None,
                     export_path: None,
                     flavor: FileFlavor::Normal,
-                }]
+                }];
+
+                if bcx.gctx.cli_unstable().rustdoc_mergeable_info {
+                    outputs.push(OutputFile {
+                        path: self
+                            .docdeps_dir(unit)
+                            .join(unit.target.crate_name())
+                            .with_extension("json"),
+                        hardlink: None,
+                        export_path: None,
+                        flavor: FileFlavor::DocParts,
+                    })
+                }
+
+                outputs
             }
             CompileMode::RunCustomBuild => {
                 // At this time, this code path does not handle build script

src/cargo/core/compiler/build_runner/mod.rs

@@ -1,6 +1,7 @@
 //! [`BuildRunner`] is the mutable state used during the build process.
 
 use std::collections::{HashMap, HashSet};
+use std::ffi::OsStr;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Mutex};
 
@@ -230,6 +231,8 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
             }
         }
 
+        self.collect_doc_merge_info()?;
+
         // Collect the result of the build into `self.compilation`.
         for unit in &self.bcx.roots {
             self.collect_tests_and_executables(unit)?;
@@ -335,6 +338,132 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
         Ok(())
     }
 
+    fn collect_doc_merge_info(&mut self) -> CargoResult<()> {
+        if !self.bcx.gctx.cli_unstable().rustdoc_mergeable_info {
+            return Ok(());
+        }
+
+        if !self.bcx.build_config.intent.is_doc() {
+            return Ok(());
+        }
+
+        if self.bcx.build_config.intent.wants_doc_json_output() {
+            // rustdoc JSON output doesn't support merge (yet?)
+            return Ok(());
+        }
+
+        let mut doc_merge_info = HashMap::new();
+
+        let unit_iter = if self.bcx.build_config.intent.wants_deps_docs() {
+            itertools::Either::Left(self.bcx.unit_graph.keys())
+        } else {
+            itertools::Either::Right(self.bcx.roots.iter())
+        };
+
+        for unit in unit_iter {
+            let has_doc_parts = unit.mode.is_doc()
+                && self
+                    .outputs(unit)?
+                    .iter()
+                    .any(|o| matches!(o.flavor, FileFlavor::DocParts));
+            if !has_doc_parts {
+                continue;
+            }
+
+            doc_merge_info.entry(unit.kind).or_insert_with(|| {
+                let out_dir = self
+                    .files()
+                    .layout(unit.kind)
+                    .artifact_dir()
+                    .expect("artifact-dir was not locked")
+                    .doc()
+                    .to_owned();
+                let docdeps_dir = self.files().docdeps_dir(unit);
+
+                let mut requires_merge = false;
+
+                // HACK: get mtime of crates.js to inform outside
+                // whether we need to merge cross-crate info.
+                // The content of `crates.js` looks like
+                //
+                // ```
+                // window.ALL_CRATES = ["cargo","cargo_util","cargo_util_schemas","crates_io"]
+                // ```
+                //
+                // and will be updated when any new crate got documented
+                // even with the legacy `--merge=shared` mode.
+                let crates_js = out_dir.join("crates.js");
+                let crates_js_mtime = paths::mtime(&crates_js);
+
+                let mut num_crates = 0;
+
+                for entry in walkdir::WalkDir::new(docdeps_dir).max_depth(1) {
+                    let Ok(entry) = entry else {
+                        tracing::debug!("failed to read entry at {}", docdeps_dir.display());
+                        continue;
+                    };
+
+                    if !entry.file_type().is_file()
+                        || entry.path().extension() != Some(OsStr::new("json"))
+                    {
+                        continue;
+                    }
+
+                    num_crates += 1;
+
+                    if requires_merge {
+                        continue;
+                    }
+
+                    let crates_js_mtime = match crates_js_mtime {
+                        Ok(mtime) => mtime,
+                        Err(ref err) => {
+                            tracing::debug!(
+                                ?err,
+                                "failed to read mtime of {}",
+                                crates_js.display()
+                            );
+                            requires_merge = true;
+                            continue;
+                        }
+                    };
+
+                    let parts_mtime = match paths::mtime(entry.path()) {
+                        Ok(mtime) => mtime,
+                        Err(err) => {
+                            tracing::debug!(
+                                ?err,
+                                "failed to read mtime of {}",
+                                entry.path().display()
+                            );
+                            requires_merge = true;
+                            continue;
+                        }
+                    };
+
+                    if parts_mtime > crates_js_mtime {
+                        requires_merge = true;
+                        continue;
+                    }
+                }
+
+                if requires_merge {
+                    compilation::DocMergeInfo::Merge {
+                        num_crates,
+                        parts_dir: docdeps_dir.to_owned(),
+                        out_dir,
+                    }
+                } else {
+                    compilation::DocMergeInfo::Fresh
+                }
+            });
+        }
+
+        self.compilation.doc_merge_info = doc_merge_info;
+
+        Ok(())
+    }
+
     /// Returns the executable for the specified unit (if any).
     pub fn get_executable(&mut self, unit: &Unit) -> CargoResult<Option<PathBuf>> {
         let is_binary = unit.target.is_executable();

src/cargo/core/compiler/compilation.rs

@@ -106,6 +106,11 @@ pub struct Compilation<'gctx> {
     /// Libraries to test with rustdoc.
     pub to_doc_test: Vec<Doctest>,
 
+    /// Compilation information for running `rustdoc --merge=finalize`.
+    ///
+    /// See `-Zrustdoc-mergeable-info` for more.
+    pub doc_merge_info: HashMap<CompileKind, DocMergeInfo>,
+
     /// The target host triple.
     pub host: String,
 
@@ -143,6 +148,7 @@ impl<'gctx> Compilation<'gctx> {
             root_crate_names: Vec::new(),
             extra_env: HashMap::new(),
             to_doc_test: Vec::new(),
+            doc_merge_info: Default::default(),
             gctx: bcx.gctx,
             host: bcx.host_triple().to_string(),
             rustc_process,
@@ -383,6 +389,25 @@ impl<'gctx> Compilation<'gctx> {
     }
 }
 
+/// Compilation information for running `rustdoc --merge=finalize`.
+#[derive(Default)]
+pub enum DocMergeInfo {
+    /// Doc merge disabled.
+    #[default]
+    None,
+    /// Nothing is stale.
+    Fresh,
+    /// Doc merge is required
+    Merge {
+        /// Number of crates to merge.
+        num_crates: u64,
+        /// Output directory holding every cross-crate info JSON file.
+        parts_dir: PathBuf,
+        /// Output directory for rustdoc.
+        out_dir: PathBuf,
+    },
+}
+
 /// Prepares a `rustc_tool` process with additional environment variables
 /// that are only relevant in a context that has a unit
 fn fill_rustc_tool_env(mut cmd: ProcessBuilder, unit: &Unit) -> ProcessBuilder {

src/cargo/core/compiler/layout.rs

@@ -192,6 +192,7 @@ impl Layout {
                 incremental: build_dest.join("incremental"),
                 fingerprint: build_dest.join(".fingerprint"),
                 examples: build_dest.join("examples"),
+                docdeps: build_dest.join("docdeps"),
                 tmp: build_root.join("tmp"),
                 _lock: build_dir_lock,
                 is_new_layout,
@@ -273,6 +274,8 @@ pub struct BuildDirLayout {
     fingerprint: PathBuf,
     /// The directory for pre-uplifted examples: `build-dir/debug/examples`
     examples: PathBuf,
+    /// The directory with intermediate artifacts from rustdoc.
+    docdeps: PathBuf,
     /// The directory for temporary data of integration tests and benches
     tmp: PathBuf,
     /// The lockfile for a build (`.cargo-lock`). Will be unlocked when this
@@ -290,6 +293,7 @@ impl BuildDirLayout {
         if !self.is_new_layout {
             paths::create_dir_all(&self.deps)?;
             paths::create_dir_all(&self.fingerprint)?;
+            paths::create_dir_all(&self.docdeps)?;
         }
         paths::create_dir_all(&self.incremental)?;
         paths::create_dir_all(&self.examples)?;
@@ -344,6 +348,12 @@ impl BuildDirLayout {
             self.build().join(pkg_dir)
         }
     }
+    /// Fetch the path storing intermediate artifacts from rustdoc.
+    pub fn docdeps(&self) -> &Path {
+        // This doesn't need to consider new build-dir layout (yet?)
+        // because rustdoc artifacts must be appendable.
+        &self.docdeps
+    }
     /// Fetch the build script execution path.
     pub fn build_script_execution(&self, pkg_dir: &str) -> PathBuf {
         if self.is_new_layout {

src/cargo/core/compiler/mod.rs

@@ -76,6 +76,7 @@ pub use self::build_context::{
     BuildContext, FileFlavor, FileType, RustDocFingerprint, RustcTargetData, TargetInfo,
 };
 pub use self::build_runner::{BuildRunner, Metadata, UnitHash};
+pub use self::compilation::DocMergeInfo;
 pub use self::compilation::{Compilation, Doctest, UnitOutput};
 pub use self::compile_kind::{CompileKind, CompileKindFallback, CompileTarget};
 pub use self::crate_type::CrateType;
@@ -830,8 +831,13 @@ fn prepare_rustdoc(build_runner: &BuildRunner<'_, '_>, unit: &Unit) -> CargoResu
     if build_runner.bcx.gctx.cli_unstable().rustdoc_depinfo {
         // toolchain-shared-resources is required for keeping the shared styling resources
         // invocation-specific is required for keeping the original rustdoc emission
-        let mut arg =
-            OsString::from("--emit=toolchain-shared-resources,invocation-specific,dep-info=");
+        let mut arg = if build_runner.bcx.gctx.cli_unstable().rustdoc_mergeable_info {
+            // toolchain resources are written at the end, at the same time as merging
+            OsString::from("--emit=invocation-specific,dep-info=")
+        } else {
+            // if not using mergeable CCI, everything is written every time
+            OsString::from("--emit=toolchain-shared-resources,invocation-specific,dep-info=")
+        };
         arg.push(rustdoc_dep_info_loc(build_runner, unit));
         rustdoc.arg(arg);
 
@@ -840,6 +846,18 @@ fn prepare_rustdoc(build_runner: &BuildRunner<'_, '_>, unit: &Unit) -> CargoResu
         }
 
         rustdoc.arg("-Zunstable-options");
+    } else if build_runner.bcx.gctx.cli_unstable().rustdoc_mergeable_info {
+        // toolchain resources are written at the end, at the same time as merging
+        rustdoc.arg("--emit=invocation-specific");
+        rustdoc.arg("-Zunstable-options");
+    }
+
+    if build_runner.bcx.gctx.cli_unstable().rustdoc_mergeable_info {
+        // write out mergeable data to be imported
+        rustdoc.arg("--merge=none");
+        let mut arg = OsString::from("--parts-out-dir=");
+        arg.push(build_runner.files().docdeps_dir(unit));
+        rustdoc.arg(arg);
     }
 
     if let Some(trim_paths) = unit.profile.trim_paths.as_ref() {

src/cargo/core/features.rs

@@ -884,6 +884,7 @@ unstable_cli_options!(
     rustc_unicode: bool = ("Enable `rustc`'s unicode error format in Cargo's error messages"),
     rustdoc_depinfo: bool = ("Use dep-info files in rustdoc rebuild detection"),
     rustdoc_map: bool = ("Allow passing external documentation mappings to rustdoc"),
+    rustdoc_mergeable_info: bool = ("Use rustdoc mergeable cross-crate-info files"),
     rustdoc_scrape_examples: bool = ("Allows Rustdoc to scrape code examples from reverse-dependencies"),
     sbom: bool = ("Enable the `sbom` option in build config in .cargo/config.toml file"),
     script: bool = ("Enable support for single-file, `.rs` packages"),
@@ -1415,6 +1416,7 @@ impl CliUnstable {
             "rustc-unicode" => self.rustc_unicode = parse_empty(k, v)?,
             "rustdoc-depinfo" => self.rustdoc_depinfo = parse_empty(k, v)?,
             "rustdoc-map" => self.rustdoc_map = parse_empty(k, v)?,
+            "rustdoc-mergeable-info" => self.rustdoc_mergeable_info = parse_empty(k, v)?,
             "rustdoc-scrape-examples" => self.rustdoc_scrape_examples = parse_empty(k, v)?,
             "sbom" => self.sbom = parse_empty(k, v)?,
             "section-timings" => self.section_timings = parse_empty(k, v)?,