feat: Add Google Drive storage backend

photosight/storage/__init__.py

@@ -0,0 +1,11 @@
+"""
+Google Drive Storage Backend for PhotoSight
+
+Provides cloud storage capabilities for photos with chunked uploads,
+retry logic, and batch processing support.
+"""
+
+from .gdrive_manager import GoogleDriveManager
+from .chunked_upload import ChunkedUploader, UploadConfig
+
+__all__ = ['GoogleDriveManager', 'ChunkedUploader', 'UploadConfig']

photosight/storage/chunked_upload.py

@@ -0,0 +1,233 @@
+"""
+Chunked upload implementation for Google Drive.
+
+Handles large RAW files (20-50MB) with resumable uploads and retry logic.
+"""
+
+import os
+import time
+import logging
+from pathlib import Path
+from typing import Optional, Dict, Any
+from dataclasses import dataclass
+import hashlib
+
+from google.oauth2 import service_account
+from googleapiclient.discovery import build
+from googleapiclient.http import MediaFileUpload
+from googleapiclient.errors import HttpError
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class UploadConfig:
+    """Configuration for chunked uploads."""
+    chunk_size: int = 5 * 1024 * 1024  # 5MB chunks for RAW files
+    max_retries: int = 5
+    initial_backoff: float = 1.0
+    backoff_multiplier: float = 2.0
+    max_backoff: float = 60.0
+
+
+class ChunkedUploader:
+    """Handles chunked uploads to Google Drive with retry logic."""
+
+    def __init__(self, service, config: Optional[UploadConfig] = None):
+        """
+        Initialize uploader with Drive service and config.
+
+        Args:
+            service: Authenticated Google Drive service instance
+            config: Upload configuration
+        """
+        self.service = service
+        self.config = config or UploadConfig()
+
+    def upload_with_retry(self, 
+                         file_path: str,
+                         folder_id: str,
+                         mime_type: Optional[str] = None) -> Optional[Dict[str, Any]]:
+        """
+        Upload file with automatic retry and exponential backoff.
+
+        Args:
+            file_path: Path to local file
+            folder_id: Google Drive folder ID
+            mime_type: MIME type (auto-detected if None)
+
+        Returns:
+            File metadata dict or None if failed
+        """
+        file_path = Path(file_path)
+        if not file_path.exists():
+            logger.error(f"File not found: {file_path}")
+            return None
+
+        # Auto-detect MIME type for RAW files
+        if mime_type is None:
+            mime_type = self._get_mime_type(file_path)
+
+        # Prepare metadata
+        file_metadata = {
+            'name': file_path.name,
+            'parents': [folder_id]
+        }
+
+        # Calculate file hash for verification
+        file_hash = self._calculate_hash(file_path)
+        file_metadata['properties'] = {'md5': file_hash}
+
+        # Attempt upload with retries
+        backoff = self.config.initial_backoff
+        last_error = None
+
+        for attempt in range(self.config.max_retries):
+            try:
+                logger.info(f"Upload attempt {attempt + 1}/{self.config.max_retries} for {file_path.name}")
+
+                # Create resumable upload
+                media = MediaFileUpload(
+                    str(file_path),
+                    mimetype=mime_type,
+                    resumable=True,
+                    chunksize=self.config.chunk_size
+                )
+
+                # Initialize request
+                request = self.service.files().create(
+                    body=file_metadata,
+                    media_body=media,
+                    fields='id,name,size,md5Checksum,webViewLink,webContentLink'
+                )
+
+                # Execute with progress tracking
+                response = None
+                while response is None:
+                    status, response = request.next_chunk()
+                    if status:
+                        progress = int(status.progress() * 100)
+                        logger.debug(f"Upload progress: {progress}%")
+
+                # Verify upload
+                if response and self._verify_upload(response, file_hash):
+                    logger.info(f"Successfully uploaded {file_path.name} (ID: {response['id']})")
+                    return response
+
+            except HttpError as e:
+                last_error = e
+                if e.resp.status in [403, 429]:  # Rate limit or quota exceeded
+                    logger.warning(f"Rate limited, backing off {backoff}s: {e}")
+                    time.sleep(backoff)
+                    backoff = min(backoff * self.config.backoff_multiplier, self.config.max_backoff)
+                elif e.resp.status >= 500:  # Server error, retry
+                    logger.warning(f"Server error, retrying after {backoff}s: {e}")
+                    time.sleep(backoff)
+                    backoff = min(backoff * self.config.backoff_multiplier, self.config.max_backoff)
+                else:
+                    logger.error(f"Non-retryable error: {e}")
+                    raise
+
+            except Exception as e:
+                last_error = e
+                logger.error(f"Unexpected error during upload: {e}")
+                if attempt < self.config.max_retries - 1:
+                    time.sleep(backoff)
+                    backoff = min(backoff * self.config.backoff_multiplier, self.config.max_backoff)
+
+        logger.error(f"Failed to upload {file_path.name} after {self.config.max_retries} attempts: {last_error}")
+        return None
+
+    def _get_mime_type(self, file_path: Path) -> str:
+        """Determine MIME type based on file extension."""
+        ext = file_path.suffix.lower()
+        mime_types = {
+            '.arw': 'image/x-sony-arw',
+            '.raw': 'image/x-raw',
+            '.dng': 'image/x-adobe-dng',
+            '.cr2': 'image/x-canon-cr2',
+            '.nef': 'image/x-nikon-nef',
+            '.jpg': 'image/jpeg',
+            '.jpeg': 'image/jpeg',
+            '.png': 'image/png',
+            '.tiff': 'image/tiff',
+            '.tif': 'image/tiff'
+        }
+        return mime_types.get(ext, 'application/octet-stream')
+
+    def _calculate_hash(self, file_path: Path, algorithm: str = 'md5') -> str:
+        """Calculate file hash for verification."""
+        hash_func = hashlib.md5() if algorithm == 'md5' else hashlib.sha256()
+
+        with open(file_path, 'rb') as f:
+            for chunk in iter(lambda: f.read(8192), b''):
+                hash_func.update(chunk)
+
+        return hash_func.hexdigest()
+
+    def _verify_upload(self, response: Dict[str, Any], expected_hash: str) -> bool:
+        """Verify uploaded file integrity."""
+        if 'md5Checksum' in response:
+            actual_hash = response['md5Checksum']
+            if actual_hash.lower() != expected_hash.lower():
+                logger.error(f"Hash mismatch! Expected: {expected_hash}, Got: {actual_hash}")
+                return False
+        return True
+
+
+class BatchUploader:
+    """Handles batch uploads with parallel processing."""
+
+    def __init__(self, service, config: Optional[UploadConfig] = None):
+        """Initialize batch uploader."""
+        self.uploader = ChunkedUploader(service, config)
+
+    def upload_batch(self, 
+                     files: list[Path],
+                     folder_id: str,
+                     max_parallel: int = 3) -> Dict[str, Any]:
+        """
+        Upload multiple files with controlled parallelism.
+
+        Args:
+            files: List of file paths to upload
+            folder_id: Google Drive folder ID
+            max_parallel: Maximum parallel uploads
+
+        Returns:
+            Dict with 'success' and 'failed' lists
+        """
+        from concurrent.futures import ThreadPoolExecutor, as_completed
+
+        results = {'success': [], 'failed': []}
+
+        with ThreadPoolExecutor(max_workers=max_parallel) as executor:
+            # Submit all upload tasks
+            future_to_file = {
+                executor.submit(
+                    self.uploader.upload_with_retry,
+                    str(file_path),
+                    folder_id
+                ): file_path
+                for file_path in files
+            }
+
+            # Process completed uploads
+            for future in as_completed(future_to_file):
+                file_path = future_to_file[future]
+                try:
+                    result = future.result()
+                    if result:
+                        results['success'].append({
+                            'file': str(file_path),
+                            'id': result['id'],
+                            'link': result.get('webViewLink', '')
+                        })
+                    else:
+                        results['failed'].append(str(file_path))
+                except Exception as e:
+                    logger.error(f"Failed to upload {file_path}: {e}")
+                    results['failed'].append(str(file_path))
+
+        logger.info(f"Batch upload complete: {len(results['success'])} success, {len(results['failed'])} failed")
+        return results