perf: stop loading whole files into memory across upload/identify paths

CHANGELOG.md

@@ -27,6 +27,13 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
   The one new constraint: a directory passed to `--type elf` must not be written to while it is
   scanned (a library truncated under its mapping is undefined behaviour, in practice SIGBUS), so
   run it after the native build task has finished, as the Gradle task graph does.
+- **The rest of the whole-file reads are gone too.** The build artefact PUT (`upload build`, an
+  IPA/APK/AAB) and the build-info PUT stream from disk; ProGuard identification hashes in one
+  streamed pass; dSYM, PDB, Rust and `xcode` identity reads memory-map the file instead of copying
+  it; zip-input extraction streams entries to disk; source-map identification streams its hash.
+  Peak RSS: `upload build` with a 300 MB artefact + 200 MB mapping 661 MB -> 32 MB, a 200 MB
+  ProGuard mapping 241 MB -> 32 MB, an ELF zip of four 100 MB libraries 463 MB -> 36 MB. Wire
+  format and exit codes unchanged.
 
 ## [0.7.13] - 2026-10-07
 

src/cli/debug_files.rs

@@ -1954,7 +1954,7 @@ fn stripped_copy_without_sources(
     // Typed, not a bare `?` into anyhow: an I/O failure on the map must classify the way
     // `sourcemap::identify` classifies one on the SAME file moments earlier — exit 10, which
     // integrators are documented not to fall back on — rather than exit 1 ("unexpected").
-    let bytes = std::fs::read(map_path).map_err(crate::error::Error::Io)?;
+    let bytes = crate::symbols::mapped::map_file(map_path).map_err(crate::error::Error::Io)?;
     let Ok(serde_json::Value::Object(mut map)) = serde_json::from_slice(&bytes) else {
         return Ok(None);
     };

src/cli/dsym.rs

@@ -137,25 +137,12 @@ pub fn extract_slices(path: &Path) -> Vec<DsymSliceView> {
     out
 }
 
-/// Hard cap on the size of a single Mach-O file we will read into
-/// memory. Real production dSYMs (large iOS apps + Swift stdlib +
-/// extensions) typically sit between 200 MB and 1 GB per slice. The
-/// cap is a generous-but-bounded safety net against a corrupt or
-/// adversarial input (a "dSYM" that is actually a 10 GB random file
-/// would otherwise drive the build host into swap). Skip rather than
-/// fail — the empty-fallback contract still applies.
-const MAX_MACHO_FILE_BYTES: u64 = 1024 * 1024 * 1024; // 1 GiB
-
 fn push_slices_from_macho(path: &Path, out: &mut Vec<DsymSliceView>) {
-    // Stat the file first so a runaway-size input is rejected without
-    // allocating gigabytes. `std::fs::read` would otherwise sequentially
-    // grow the Vec until the kernel says no.
-    if let Ok(md) = std::fs::metadata(path) {
-        if md.len() > MAX_MACHO_FILE_BYTES {
-            return;
-        }
-    }
-    let data = match std::fs::read(path) {
+    // No size cap: the file is memory-mapped and only its headers are read, so a
+    // 2 GB DWARF slice costs the same few pages as a small one. (A cap used to guard
+    // `std::fs::read` growing a multi-GB Vec; it also made `dsym uuid` report `[]`,
+    // silently, for a legitimate slice over 1 GiB that `debug-files upload` accepted.)
+    let data = match crate::symbols::mapped::map_file(path) {
         Ok(d) => d,
         Err(_) => return,
     };
@@ -193,6 +180,32 @@ mod tests {
     use super::*;
     use tempfile::TempDir;
 
+    /// A DWARF slice over 1 GiB (large games) is identified like any other: there is no
+    /// size cap, since only the headers of the mapping are read. The file is sparse, so
+    /// this costs no disk and the parse touches a few pages.
+    #[cfg(unix)]
+    #[test]
+    fn extract_uuids_has_no_size_cap() {
+        use std::io::Write;
+        let tmp = TempDir::new().unwrap();
+        let p = tmp.path().join("huge");
+        let mut f = std::fs::File::create(&p).unwrap();
+        f.write_all(&crate::symbols::test_macho::thin_macho(
+            0x0100_000c,
+            0,
+            [4; 16],
+        ))
+        .unwrap();
+        f.set_len(1024 * 1024 * 1024 + 4096).unwrap(); // just over 1 GiB, sparse
+        drop(f);
+        assert_eq!(
+            extract_uuids(&p),
+            [format_uuid(DebugId::from_uuid(uuid::Uuid::from_bytes(
+                [4; 16]
+            )))]
+        );
+    }
+
     #[test]
     fn extract_uuids_returns_empty_for_nonexistent_path() {
         let out = extract_uuids(Path::new("/no/such/file"));

src/cli/xcode_ipa.rs

@@ -232,7 +232,7 @@ pub fn main_executable_uuid(app_path: &Path) -> Option<String> {
         return None;
     }
 
-    let data = std::fs::read(&binary_path).ok()?;
+    let data = crate::symbols::mapped::map_file(&binary_path).ok()?;
     let archive = Archive::parse(&data).ok()?;
     let mut slices: Vec<(String, Uuid)> = Vec::new();
     for obj in archive.objects() {

src/inject/mod.rs

@@ -564,8 +564,13 @@ fn write_map_debug_id(
 /// / legacy `uuid`, in that precedence. Mirrors the worker's
 /// `symbolfiles/sourcemap.py:parse`.
 pub fn read_debug_id(map_path: &Path) -> Result<Option<String>> {
-    let raw = std::fs::read_to_string(map_path)?;
-    let value: serde_json::Value = serde_json::from_str(&raw).map_err(|e| {
+    // Mapped rather than read into a String: the JSON tree below is the only
+    // copy this needs. UTF-8 is checked up front so a non-UTF-8 map still fails
+    // as an I/O `InvalidData` error, exactly as `read_to_string` did.
+    let bytes = crate::symbols::mapped::map_file(map_path)?;
+    let raw = std::str::from_utf8(&bytes)
+        .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
+    let value: serde_json::Value = serde_json::from_str(raw).map_err(|e| {
         Error::InputInvalid(format!(
             "sourcemap is not valid JSON: {e} ({})",
             map_path.display()

src/symbols/dsym.rs

@@ -79,7 +79,7 @@ pub fn identify(dsym_path: &Path) -> Result<DsymIdentity> {
             continue;
         }
         let bin_path = entry.path();
-        let data = std::fs::read(&bin_path)?;
+        let data = super::mapped::map_file(&bin_path)?;
         let archive = Archive::parse(&data).map_err(|e| {
             Error::InputInvalid(format!(
                 "failed to parse Mach-O at {}: {}",

src/symbols/elf.rs

@@ -101,23 +101,29 @@ pub fn extract_libs(
             continue;
         }
 
-        let mut bytes = Vec::new();
-        entry.read_to_end(&mut bytes)?;
-        let ElfIdentity {
-            build_id,
-            arch,
-            has_debug_info,
-            has_symbols,
-        } = parse_elf_identity(&bytes);
-        let richness = (has_debug_info, has_symbols, bytes.len() as u64);
-
         let base = Path::new(&name)
             .file_name()
             .and_then(|n| n.to_str())
             .unwrap_or("lib.so");
         // Entries across ABIs can share a basename — prefix with the index.
         let out_path = out_dir.join(format!("{i}_{base}"));
-        std::fs::write(&out_path, &bytes)?;
+        // Stream the entry to disk, then read its identity from the file: an
+        // inflated unstripped library is 100+ MB and was held whole in memory.
+        {
+            let mut out = std::fs::File::create(&out_path)?;
+            std::io::copy(&mut entry, &mut out)?;
+        }
+        let ElfIdentity {
+            build_id,
+            arch,
+            has_debug_info,
+            has_symbols,
+        } = parse_elf_identity(&super::mapped::map_file(&out_path)?);
+        let richness = (
+            has_debug_info,
+            has_symbols,
+            std::fs::metadata(&out_path)?.len(),
+        );
 
         libs.push(ElfLib {
             name,
@@ -173,29 +179,16 @@ pub fn scan_dir(dir: &Path, extra: &ExtraSuffixes) -> std::io::Result<Vec<ElfLib
         let with_path = |e: std::io::Error| {
             std::io::Error::new(e.kind(), format!("{}: {e}", entry.path().display()))
         };
-        let file = std::fs::File::open(entry.path()).map_err(with_path)?;
-        let len = file.metadata().map_err(with_path)?.len();
-        // Map instead of reading: identity needs only the headers and notes, so
-        // a 100+ MB unstripped library costs a few KB of resident memory, not a
-        // full copy. (An empty file cannot be mapped; it has no build-id anyway.)
-        //
-        // SAFETY: the map is read-only and dropped at the end of this iteration.
-        // The contract (memmap2's): nothing may truncate or rewrite the library while
-        // it is mapped (SIGBUS / undefined behaviour otherwise). It holds for
-        // build-finalized outputs; a directory still being written by a linker is
-        // NOT a supported input (documented on `paths` in `--help`).
-        let map = if len == 0 {
-            None
-        } else {
-            Some(unsafe { memmap2::Mmap::map(&file) }.map_err(with_path)?)
-        };
-        let bytes: &[u8] = map.as_deref().unwrap_or(&[]);
+        // Mapped, not read: identity needs only the headers and notes, so a
+        // 100+ MB unstripped library costs a few KB of resident memory.
+        let bytes = super::mapped::map_file(entry.path()).map_err(with_path)?;
+        let len = bytes.len() as u64;
         let ElfIdentity {
             build_id,
             arch,
             has_debug_info,
             has_symbols,
-        } = parse_elf_identity(bytes);
+        } = parse_elf_identity(&bytes);
         let name = entry
             .path()
             .strip_prefix(dir)

src/symbols/mapped.rs

@@ -0,0 +1,54 @@
+//! Read-only memory-mapped file bytes, for identity parsing.
+//!
+//! Identifying a debug file (ELF build-id, Mach-O UUID, PDB GUID) touches only
+//! its headers, yet the files are routinely 100+ MB and `std::fs::read` copies
+//! every byte onto the heap. Mapping instead keeps the cost to the pages the
+//! parser actually reads. This is the ONLY place the crate memory-maps a file, so
+//! that `unsafe` lives here (the crate has two other, unrelated `unsafe` uses: the
+//! `fork` in `daemon.rs` and `gethostname` in `build_env.rs`).
+
+use std::ops::Deref;
+use std::path::Path;
+
+/// The bytes of a file, mapped read-only (empty for an empty file, which cannot
+/// be mapped). Derefs to `[u8]`.
+pub struct Mapped(Option<memmap2::Mmap>);
+
+impl Deref for Mapped {
+    type Target = [u8];
+    fn deref(&self) -> &[u8] {
+        self.0.as_deref().unwrap_or(&[])
+    }
+}
+
+/// Map `path` read-only.
+///
+/// Keep the result scoped to the parse and drop it before the file is
+/// rewritten or removed (Windows refuses to replace a mapped file).
+pub fn map_file(path: &Path) -> std::io::Result<Mapped> {
+    let file = std::fs::File::open(path)?;
+    if file.metadata()?.len() == 0 {
+        return Ok(Mapped(None));
+    }
+    // SAFETY: the map is read-only. The contract (memmap2's): nothing may truncate
+    // or rewrite the file while it is mapped (SIGBUS / undefined behaviour
+    // otherwise). It holds for finished build outputs and symbol files, which is all
+    // this reads; a file still being written by a linker is not a supported input.
+    Ok(Mapped(Some(unsafe { memmap2::Mmap::map(&file) }?)))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn maps_file_contents_and_handles_an_empty_file() {
+        let dir = tempfile::tempdir().unwrap();
+        let p = dir.path().join("f");
+        std::fs::write(&p, b"abc").unwrap();
+        assert_eq!(&*map_file(&p).unwrap(), b"abc");
+        std::fs::write(&p, b"").unwrap();
+        assert!(map_file(&p).unwrap().is_empty());
+        assert!(map_file(&dir.path().join("missing")).is_err());
+    }
+}

src/symbols/mod.rs

@@ -26,6 +26,7 @@
 pub mod dsym;
 pub mod elf;
 pub mod il2cpp_linemap;
+pub mod mapped;
 pub mod pdb;
 pub mod proguard;
 pub mod rust;

src/symbols/pdb.rs

@@ -81,7 +81,7 @@ pub fn identify(pdb_path: &Path) -> Result<PdbIdentity> {
         )));
     }
 
-    let data = std::fs::read(pdb_path)?;
+    let data = super::mapped::map_file(pdb_path)?;
     let archive = Archive::parse(&data).map_err(|e| {
         Error::InputInvalid(format!(
             "failed to parse PDB at {}: {}",

src/symbols/proguard.rs

@@ -25,15 +25,33 @@ pub fn looks_like_mapping_filename(name: &str) -> bool {
 
 /// Read a file from disk and compute (debug-id, sha1 hex).
 ///
-/// Streams the SHA1 chunk-at-a-time to keep memory flat for large mappings
-/// (multi-megabyte mappings are common for big apps). MD5 still needs the
-/// whole buffer because Java's `nameUUIDFromBytes` operates on contiguous
-/// bytes; mappings rarely exceed 50 MB so this is acceptable.
+/// Both digests are computed in ONE streamed pass (64 KiB buffer), so memory
+/// stays flat however large the mapping is: Java's `nameUUIDFromBytes` is just
+/// an MD5 of the bytes with the version/variant bits set, and MD5 is
+/// incremental like SHA-1.
 pub fn identify(path: &Path) -> std::io::Result<MappingIdentity> {
-    let bytes = std::fs::read(path)?;
-    Ok(identify_bytes(&bytes))
+    use std::io::Read as _;
+    let mut file = std::fs::File::open(path)?;
+    let mut md5 = Md5::new();
+    let mut sha1 = Sha1::new();
+    let mut buf = [0u8; 64 * 1024];
+    loop {
+        let n = file.read(&mut buf)?;
+        if n == 0 {
+            break;
+        }
+        md5.update(&buf[..n]);
+        sha1.update(&buf[..n]);
+    }
+    let md5_bytes: [u8; 16] = md5.finalize().into();
+    let sha1_bytes: [u8; 20] = sha1.finalize().into();
+    Ok(MappingIdentity {
+        debug_id: Builder::from_md5_bytes(md5_bytes).into_uuid(),
+        content_sha1_hex: hex::encode(sha1_bytes),
+    })
 }
 
+#[cfg(test)]
 pub fn identify_bytes(bytes: &[u8]) -> MappingIdentity {
     MappingIdentity {
         debug_id: java_name_uuid_from_bytes(bytes),
@@ -51,6 +69,7 @@ pub struct MappingIdentity {
     pub content_sha1_hex: String,
 }
 
+#[cfg(test)]
 /// Replicates Java `UUID.nameUUIDFromBytes(name)`:
 ///   1. MD5 of input bytes.
 ///   2. Set the version nibble of byte 6 to 3.
@@ -62,6 +81,7 @@ pub fn java_name_uuid_from_bytes(name: &[u8]) -> Uuid {
     Builder::from_md5_bytes(md5_bytes).into_uuid()
 }
 
+#[cfg(test)]
 fn sha1_hex(bytes: &[u8]) -> String {
     let digest: [u8; 20] = Sha1::digest(bytes).into();
     hex::encode(digest)
@@ -88,6 +108,25 @@ mod tests {
         );
     }
 
+    #[test]
+    fn streamed_identify_equals_in_memory_identify_across_buffer_boundaries() {
+        // Bigger than the 64 KiB buffer and not a multiple of it: a chunking bug
+        // would change either digest relative to the one-shot computation.
+        let dir = tempfile::tempdir().unwrap();
+        let p = dir.path().join("mapping.txt");
+        let data: Vec<u8> = (0..300_001u32).map(|i| (i % 251) as u8).collect();
+        std::fs::write(&p, &data).unwrap();
+        let streamed = identify(&p).unwrap();
+        let whole = identify_bytes(&data);
+        assert_eq!(streamed.debug_id, whole.debug_id);
+        assert_eq!(streamed.content_sha1_hex, whole.content_sha1_hex);
+        std::fs::write(&p, b"").unwrap();
+        assert_eq!(
+            identify(&p).unwrap().debug_id.to_string(),
+            "d41d8cd9-8f00-3204-a980-0998ecf8427e"
+        );
+    }
+
     #[test]
     fn sha1_hex_matches_reference() {
         assert_eq!(sha1_hex(b""), "da39a3ee5e6b4b0d3255bfef95601890afd80709");

src/symbols/rust.rs

@@ -153,7 +153,7 @@ fn is_bundle_like(p: &Path, extra: &ExtraSuffixes) -> bool {
 /// on — and to the `code_id` the Rust SDK reports for that module at crash
 /// time. All three agreeing is what makes symbolication resolve.
 fn parse_elf(path: &Path) -> Option<(Option<String>, String, bool)> {
-    let bytes = std::fs::read(path).ok()?;
+    let bytes = super::mapped::map_file(path).ok()?;
     let archive = Archive::parse(&bytes).ok()?;
     let obj = archive.objects().next()?.ok()?;
     Some((