(feat:file_abstract) process files method

Author: ManishMadan2882

application/storage/base.py

@@ -1,6 +1,6 @@
 """Base storage class for file system abstraction."""
 from abc import ABC, abstractmethod
-from typing import BinaryIO, List
+from typing import BinaryIO, List, Optional, Callable
 
 
 class BaseStorage(ABC):
@@ -33,6 +33,24 @@ def get_file(self, path: str) -> BinaryIO:
         """
         pass
 
+    @abstractmethod
+    def process_file(self, path: str, processor_func: Callable, **kwargs):
+        """
+        Process a file using the provided processor function.
+        
+        This method handles the details of retrieving the file and providing
+        it to the processor function in an appropriate way based on the storage type.
+        
+        Args:
+            path: Path to the file
+            processor_func: Function that processes the file
+            **kwargs: Additional arguments to pass to the processor function
+            
+        Returns:
+            The result of the processor function
+        """
+        pass
+    
     @abstractmethod
     def delete_file(self, path: str) -> bool:
         """

application/storage/local.py

@@ -1,9 +1,8 @@
 """Local file system implementation."""
 import os
 import shutil
-from typing import BinaryIO, List
+from typing import BinaryIO, List, Callable
 
-from application.core.settings import settings
 from application.storage.base import BaseStorage
 
 
@@ -83,3 +82,24 @@ def list_files(self, directory: str) -> List[str]:
                 result.append(rel_path)
 
         return result
+
+    def process_file(self, path: str, processor_func: Callable, **kwargs):
+        """
+        Process a file using the provided processor function.
+        
+        For local storage, we can directly pass the full path to the processor.
+        
+        Args:
+            path: Path to the file
+            processor_func: Function that processes the file
+            **kwargs: Additional arguments to pass to the processor function
+            
+        Returns:
+            The result of the processor function
+        """
+        full_path = self._get_full_path(path)
+        
+        if not os.path.exists(full_path):
+            raise FileNotFoundError(f"File not found: {full_path}")
+        
+        return processor_func(file_path=full_path, **kwargs)

application/storage/s3.py

@@ -1,6 +1,6 @@
 """S3 storage implementation."""
 import io
-from typing import BinaryIO, List
+from typing import BinaryIO, List, Callable
 
 import boto3
 from botocore.exceptions import ClientError
@@ -24,7 +24,6 @@ def __init__(self, bucket_name: str, aws_access_key_id=None,
         """
         self.bucket_name = bucket_name
 
-        # Initialize S3 client
         self.s3 = boto3.client(
             's3',
             aws_access_key_id=aws_access_key_id,
@@ -78,4 +77,38 @@ def list_files(self, directory: str) -> List[str]:
                 for obj in page['Contents']:
                     result.append(obj['Key'])
 
-        return result
+        return result
+
+    def process_file(self, path: str, processor_func: Callable, **kwargs):
+        """
+        Process a file using the provided processor function.
+        
+        For S3 storage, we need to download the file to a temporary location first.
+        
+        Args:
+            path: Path to the file
+            processor_func: Function that processes the file
+            **kwargs: Additional arguments to pass to the processor function
+            
+        Returns:
+            The result of the processor function
+        """
+        import tempfile
+        import os
+        
+        if not self.file_exists(path):
+            raise FileNotFoundError(f"File not found: {path}")
+        
+        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+            self.s3.download_fileobj(self.bucket_name, path, temp_file)
+            temp_path = temp_file.name
+        
+        try:
+            result = processor_func(file_path=temp_path, **kwargs)
+            return result
+        finally:
+            try:
+                os.unlink(temp_path)
+            except Exception as e:
+                import logging
+                logging.warning(f"Failed to delete temporary file: {e}")