Support more than 20,000 files on file listing

cirro/services/dataset.py

@@ -126,27 +126,40 @@ def delete(self, project_id: str, dataset_id: str) -> None:
         """
         delete_dataset.sync_detailed(project_id=project_id, dataset_id=dataset_id, client=self._api_client)
 
-    def get_file_listing(self, project_id: str, dataset_id: str) -> List[File]:
+    def get_file_listing(self, project_id: str, dataset_id: str, file_limit: int = 100000) -> List[File]:
         """
         Gets a listing of files, charts, and other assets available for the dataset
 
         Args:
             project_id (str): ID of the Project
             dataset_id (str): ID of the Dataset
+            file_limit (int): Maximum number of files to get (default 100,000)
         """
-        manifest = get_dataset_manifest.sync(
-            project_id=project_id,
-            dataset_id=dataset_id,
-            client=self._api_client
-        )
+        all_files = []
+        file_offset = 0
+        domain = None
+
+        while len(all_files) < file_limit:
+            manifest = get_dataset_manifest.sync(
+                project_id=project_id,
+                dataset_id=dataset_id,
+                file_offset=file_offset,
+                client=self._api_client
+            )
+            all_files.extend(manifest.files)
+            file_offset += len(manifest.files)
+
+            if len(all_files) >= manifest.total_files or len(manifest.files) == 0:
+                break
+            domain = manifest.domain
 
         files = [
             File.from_file_entry(
                 f,
                 project_id=project_id,
-                domain=manifest.domain
+                domain=domain
             )
-            for f in manifest.files
+            for f in all_files
         ]
         return files
 
@@ -176,11 +189,11 @@ def upload_files(self, project_id: str, dataset_id: str, local_directory: str, f
         )
 
     def download_files(
-        self,
-        project_id: str,
-        dataset_id: str,
-        download_location: str,
-        files: Union[List[File], List[str]] = None
+            self,
+            project_id: str,
+            dataset_id: str,
+            download_location: str,
+            files: Union[List[File], List[str]] = None
     ) -> None:
         """
         Downloads files from a dataset