From b354b2511eafd0f58625904d0bb6c3d45d353080 Mon Sep 17 00:00:00 2001
From: Chris Staite <christopher.staite@menlosecurity.com>
Date: Wed, 21 Jun 2023 10:58:14 +0000
Subject: [PATCH] Perform parallel reading of content directory to speed up
 startup.

---
 cas/store/filesystem_store.rs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/cas/store/filesystem_store.rs b/cas/store/filesystem_store.rs
index 44daaf3f3..c382e5cf2 100644
--- a/cas/store/filesystem_store.rs
+++ b/cas/store/filesystem_store.rs
@@ -319,6 +319,10 @@ pub fn digest_from_filename(file_name: &str) -> Result<DigestInfo, Error> {
     DigestInfo::try_new(hash, size)
 }
 
+/// The number of files to read the metadata for at the same time when running
+/// add_files_to_cache.
+const SIMULTANEOUS_METADATA_READS: usize = 200;
+
 async fn add_files_to_cache<Fe: FileEntry>(
     evicting_map: &EvictingMap<Arc<Fe>, SystemTime>,
     anchor_time: &SystemTime,
@@ -359,7 +363,7 @@ async fn add_files_to_cache<Fe: FileEntry>(
 
         let read_dir_stream = ReadDirStream::new(dir_handle);
         read_dir_stream
-            .then(|dir_entry| async move {
+            .map(|dir_entry| async move {
                 let dir_entry = dir_entry.unwrap();
                 let file_name = dir_entry.file_name().into_string().unwrap();
                 let metadata = dir_entry
@@ -381,6 +385,7 @@ async fn add_files_to_cache<Fe: FileEntry>(
                 };
                 Result::<(String, SystemTime, u64), Error>::Ok((file_name, atime, metadata.len()))
             })
+            .buffer_unordered(SIMULTANEOUS_METADATA_READS)
             .try_collect()
             .await?
     };