fix: performance of _ls_tree

Author: awgr

src/huggingface_hub/hf_file_system.py

@@ -24,7 +24,7 @@
     REPO_TYPES_URL_PREFIXES,
 )
 from .file_download import hf_hub_url, http_get
-from .hf_api import HfApi, LastCommitInfo, RepoFile
+from .hf_api import HfApi, LastCommitInfo, RepoFile, RepoFolder
 from .utils import (
     EntryNotFoundError,
     HFValidationError,
@@ -375,29 +375,14 @@ def _ls_tree(
                 revision=resolved_path.revision,
                 repo_type=resolved_path.repo_type,
             )
+
             for path_info in tree:
-                if isinstance(path_info, RepoFile):
-                    cache_path_info = {
-                        "name": root_path + "/" + path_info.path,
-                        "size": path_info.size,
-                        "type": "file",
-                        "blob_id": path_info.blob_id,
-                        "lfs": path_info.lfs,
-                        "last_commit": path_info.last_commit,
-                        "security": path_info.security,
-                    }
-                else:
-                    cache_path_info = {
-                        "name": root_path + "/" + path_info.path,
-                        "size": 0,
-                        "type": "directory",
-                        "tree_id": path_info.tree_id,
-                        "last_commit": path_info.last_commit,
-                    }
+                cache_path_info = self._make_cache_path_info(root_path, path_info, should_copy=False)
                 parent_path = self._parent(cache_path_info["name"])
                 self.dircache.setdefault(parent_path, []).append(cache_path_info)
-                out.append(cache_path_info)
-        return copy.deepcopy(out)  # copy to not let users modify the dircache
+                out_cache_path_info = self._make_cache_path_info(root_path, path_info)
+                out.append(out_cache_path_info)
+        return out
 
     def glob(self, path, **kwargs):
         # Set expand_info=False by default to get a x10 speed boost
@@ -540,28 +525,34 @@ def info(self, path: str, refresh: bool = False, revision: Optional[str] = None,
                     path_in_repo="",
                     _raw_revision=resolved_path._raw_revision,
                 ).unresolve()
-                if isinstance(path_info, RepoFile):
-                    out = {
-                        "name": root_path + "/" + path_info.path,
-                        "size": path_info.size,
-                        "type": "file",
-                        "blob_id": path_info.blob_id,
-                        "lfs": path_info.lfs,
-                        "last_commit": path_info.last_commit,
-                        "security": path_info.security,
-                    }
-                else:
-                    out = {
-                        "name": root_path + "/" + path_info.path,
-                        "size": 0,
-                        "type": "directory",
-                        "tree_id": path_info.tree_id,
-                        "last_commit": path_info.last_commit,
-                    }
+                out = self._make_path_info(root_path, path_info)
                 if not expand_info:
                     out = {k: out[k] for k in ["name", "size", "type"]}
         assert out is not None
-        return copy.deepcopy(out)  # copy to not let users modify the dircache
+        return out
+
+    def _make_cache_path_info(
+        self, root_path: str, path_info: RepoFile | RepoFolder, shallow_copy: bool = True
+    ) -> Dict[str, Any]:
+        return (
+            {
+                "name": root_path + "/" + path_info.path,
+                "size": path_info.size,
+                "type": "file",
+                "blob_id": path_info.blob_id,
+                "lfs": copy.copy(path_info.lfs) if shallow_copy else path_info.lfs,
+                "last_commit": copy.copy(path_info.last_commit) if shallow_copy else path_info.last_commit,
+                "security": copy.copy(path_info.security) if shallow_copy else path_info.security,
+            }
+            if isinstance(path_info, RepoFile)
+            else {
+                "name": root_path + "/" + path_info.path,
+                "size": 0,
+                "type": "directory",
+                "tree_id": path_info.tree_id,
+                "last_commit": copy.copy(path_info.last_commit) if shallow_copy else path_info.last_commit,
+            }
+        )
 
     def exists(self, path, **kwargs):
         """Is there a file at the given path"""