[Turbopack] use file.read() instead of file.track() in webpack loaders (#69659)

### What?

`track()` invalidates based on watcher events and might have false
invalidation when events occur but files do not actually change. This
also happens when restoring from persistent caching where all files are
considered as changed.

`read()` invalidates only when the file content actually changed.

Since webpack loaders are usually pretty expensive we want to use
`read()` instead of `track()` here.

Author: sokra

turbopack/crates/turbo-tasks-fs/src/lib.rs

@@ -311,45 +311,14 @@ impl DiskFileSystem {
 
         Ok(Self::cell(instance))
     }
-}
-
-impl Debug for DiskFileSystem {
-    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
-        write!(f, "name: {}, root: {}", self.name, self.root)
-    }
-}
 
-#[turbo_tasks::value_impl]
-impl FileSystem for DiskFileSystem {
     #[turbo_tasks::function(fs)]
-    async fn read(&self, fs_path: Vc<FileSystemPath>) -> Result<Vc<FileContent>> {
-        let full_path = self.to_sys_path(fs_path).await?;
-        self.register_invalidator(&full_path)?;
-
-        let _lock = self.lock_path(&full_path).await;
-        let content = match retry_future(|| File::from_path(full_path.clone()))
-            .instrument(tracing::info_span!(
-                "read file",
-                path = display(full_path.display())
-            ))
-            .await
-        {
-            Ok(file) => FileContent::new(file),
-            Err(e) if e.kind() == ErrorKind::NotFound || e.kind() == ErrorKind::InvalidFilename => {
-                FileContent::NotFound
-            }
-            Err(e) => {
-                bail!(anyhow!(e).context(format!("reading file {}", full_path.display())))
-            }
-        };
-        Ok(content.cell())
-    }
-
-    #[turbo_tasks::function(fs)]
-    async fn read_dir(&self, fs_path: Vc<FileSystemPath>) -> Result<Vc<DirectoryContent>> {
+    async fn read_dir_internal(
+        &self,
+        fs_path: Vc<FileSystemPath>,
+    ) -> Result<Vc<InternalDirectoryContent>> {
         let full_path = self.to_sys_path(fs_path).await?;
         self.register_dir_invalidator(&full_path)?;
-        let fs_path = fs_path.await?;
 
         // we use the sync std function here as it's a lot faster (600%) in
         // node-file-trace
@@ -366,7 +335,7 @@ impl FileSystem for DiskFileSystem {
                     || e.kind() == ErrorKind::NotADirectory
                     || e.kind() == ErrorKind::InvalidFilename =>
             {
-                return Ok(DirectoryContent::not_found());
+                return Ok(InternalDirectoryContent::not_found());
             }
             Err(e) => {
                 bail!(anyhow!(e).context(format!("reading dir {}", full_path.display())))
@@ -386,13 +355,13 @@ impl FileSystem for DiskFileSystem {
                 let file_name: RcStr = path.file_name()?.to_str()?.into();
                 let path_to_root = sys_to_unix(path.strip_prefix(&self.root).ok()?.to_str()?);
 
-                let fs_path = FileSystemPath::new_normalized(fs_path.fs, path_to_root.into());
+                let path = path_to_root.into();
 
                 let entry = match e.file_type() {
-                    Ok(t) if t.is_file() => DirectoryEntry::File(fs_path),
-                    Ok(t) if t.is_dir() => DirectoryEntry::Directory(fs_path),
-                    Ok(t) if t.is_symlink() => DirectoryEntry::Symlink(fs_path),
-                    Ok(_) => DirectoryEntry::Other(fs_path),
+                    Ok(t) if t.is_file() => InternalDirectoryEntry::File(path),
+                    Ok(t) if t.is_dir() => InternalDirectoryEntry::Directory(path),
+                    Ok(t) if t.is_symlink() => InternalDirectoryEntry::Symlink(path),
+                    Ok(_) => InternalDirectoryEntry::Other(path),
                     Err(err) => return Some(Err(err.into())),
                 };
 
@@ -401,7 +370,72 @@ impl FileSystem for DiskFileSystem {
             .collect::<Result<_>>()
             .with_context(|| format!("reading directory item in {}", full_path.display()))?;
 
-        Ok(DirectoryContent::new(entries))
+        Ok(InternalDirectoryContent::new(entries))
+    }
+}
+
+impl Debug for DiskFileSystem {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        write!(f, "name: {}, root: {}", self.name, self.root)
+    }
+}
+
+#[turbo_tasks::value_impl]
+impl FileSystem for DiskFileSystem {
+    #[turbo_tasks::function(fs)]
+    async fn read(&self, fs_path: Vc<FileSystemPath>) -> Result<Vc<FileContent>> {
+        let full_path = self.to_sys_path(fs_path).await?;
+        self.register_invalidator(&full_path)?;
+
+        let _lock = self.lock_path(&full_path).await;
+        let content = match retry_future(|| File::from_path(full_path.clone()))
+            .instrument(tracing::info_span!(
+                "read file",
+                path = display(full_path.display())
+            ))
+            .await
+        {
+            Ok(file) => FileContent::new(file),
+            Err(e) if e.kind() == ErrorKind::NotFound || e.kind() == ErrorKind::InvalidFilename => {
+                FileContent::NotFound
+            }
+            Err(e) => {
+                bail!(anyhow!(e).context(format!("reading file {}", full_path.display())))
+            }
+        };
+        Ok(content.cell())
+    }
+
+    #[turbo_tasks::function]
+    async fn read_dir(self: Vc<Self>, fs_path: Vc<FileSystemPath>) -> Result<Vc<DirectoryContent>> {
+        match &*self.read_dir_internal(fs_path).await? {
+            InternalDirectoryContent::NotFound => Ok(DirectoryContent::not_found()),
+            InternalDirectoryContent::Entries(entries) => {
+                let fs = fs_path.await?.fs;
+                let entries = entries
+                    .iter()
+                    .map(|(name, entry)| {
+                        let entry = match entry {
+                            InternalDirectoryEntry::File(path) => DirectoryEntry::File(
+                                FileSystemPath::new_normalized(fs, path.clone()),
+                            ),
+                            InternalDirectoryEntry::Directory(path) => DirectoryEntry::Directory(
+                                FileSystemPath::new_normalized(fs, path.clone()),
+                            ),
+                            InternalDirectoryEntry::Symlink(path) => DirectoryEntry::Symlink(
+                                FileSystemPath::new_normalized(fs, path.clone()),
+                            ),
+                            InternalDirectoryEntry::Other(path) => DirectoryEntry::Other(
+                                FileSystemPath::new_normalized(fs, path.clone()),
+                            ),
+                            InternalDirectoryEntry::Error => DirectoryEntry::Error,
+                        };
+                        (name.clone(), entry)
+                    })
+                    .collect();
+                Ok(DirectoryContent::new(entries))
+            }
+        }
     }
 
     #[turbo_tasks::function(fs)]
@@ -1849,6 +1883,15 @@ pub enum FileLinesContent {
     NotFound,
 }
 
+#[derive(Hash, Clone, Debug, PartialEq, Eq, TraceRawVcs, Serialize, Deserialize)]
+pub enum InternalDirectoryEntry {
+    File(RcStr),
+    Directory(RcStr),
+    Symlink(RcStr),
+    Other(RcStr),
+    Error,
+}
+
 #[derive(Hash, Clone, Copy, Debug, PartialEq, Eq, TraceRawVcs, Serialize, Deserialize)]
 pub enum DirectoryEntry {
     File(Vc<FileSystemPath>),
@@ -1916,6 +1959,23 @@ impl From<&DirectoryEntry> for FileSystemEntryType {
     }
 }
 
+#[turbo_tasks::value]
+#[derive(Debug)]
+pub enum InternalDirectoryContent {
+    Entries(Vec<(RcStr, InternalDirectoryEntry)>),
+    NotFound,
+}
+
+impl InternalDirectoryContent {
+    pub fn new(entries: Vec<(RcStr, InternalDirectoryEntry)>) -> Vc<Self> {
+        Self::cell(InternalDirectoryContent::Entries(entries))
+    }
+
+    pub fn not_found() -> Vc<Self> {
+        Self::cell(InternalDirectoryContent::NotFound)
+    }
+}
+
 #[turbo_tasks::value]
 #[derive(Debug)]
 pub enum DirectoryContent {

turbopack/crates/turbopack-node/src/transforms/webpack.rs

@@ -716,7 +716,7 @@ async fn dir_dependency_shallow(glob: Vc<ReadGlobResult>) -> Result<Vc<Completio
         // Reading all files to add itself as dependency
         match *item {
             DirectoryEntry::File(file) => {
-                file.track().await?;
+                file.read().await?;
             }
             DirectoryEntry::Directory(dir) => {
                 dir_dependency(dir.read_glob(Glob::new("**".into()), false)).await?;