Support azure blob storage

CHANGELOG.md

@@ -12,6 +12,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Try to enforce using metric tools rather than downloading assets.
 - Rule to avoid overvalidating.
 
+## Added
+- Support for azure blob storage + azurite locally.
+
 ## [v0.10.0] - 2.10.2025
 
 ### Fixed

README.md

@@ -18,12 +18,23 @@ KEYCLOAK_ISSUER=...
 NEXTAUTH_SECRET=...
 ```
 
-3. Start the services and initialize the database:
+3. Start the services and initialize the database :
 ```bash
 docker compose up
+```
+Both AWS (minio locally) and Azure (azurite locally) are supported. Choose one option and initialise the storage.
+
+- AWS - Minio :
+```bash
 docker exec -it neuroagent-minio-1 mc alias set myminio http://minio:9000 minioadmin minioadmin && docker exec -it neuroagent-minio-1 mc mb myminio/neuroagent
 ```
 
+- Azure - Azurite :
+```bash
+az storage container create --name neuroagent --connection-string "DefaultEndpointsProtocol=http;AccountName=azuriteadmin;AccountKey=YXp1cml0ZQ==;BlobEndpoint=http://127.0.0.1:10000/azuriteadmin;"
+az storage cors add --services b --origins '*' --methods GET PUT POST DELETE OPTIONS --allowed-headers '*' --exposed-headers '*' --connection-string "DefaultEndpointsProtocol=http;AccountName=azuriteadmin;AccountKey=YXp1cml0ZQ==;BlobEndpoint=http://127.0.0.1:10000/azuriteadmin;"
+```
+
 4. Access the application at `http://localhost:3000`
 
 Notes:

backend/pyproject.toml

@@ -15,6 +15,8 @@ dependencies = [
     "alembic",
     "asgi-correlation-id",
     "asyncpg",
+    "azure-core",
+    "azure-storage-blob",
     "bluepysnap",
     "boto3",
     "duckdb",

backend/src/neuroagent/app/config.py

@@ -26,10 +26,17 @@ class SettingsAgent(BaseModel):
 class SettingsStorage(BaseModel):
     """Storage settings."""
 
-    endpoint_url: str | None = None
-    bucket_name: str = "neuroagent"
-    access_key: SecretStr | None = None
-    secret_key: SecretStr | None = None
+    provider: Literal["s3", "azure"] = "azure"
+    container_name: str = "neuroagent"
+    # Minio
+    s3_endpoint_url: str | None = None
+    s3_access_key: SecretStr | None = None
+    s3_secret_key: SecretStr | None = None
+    # Azurite
+    azure_endpoint_url: str | None = None
+    azure_account_name: SecretStr | None = None
+    azure_account_key: SecretStr | None = None
+    # presigned_url expiration
     expires_in: int = 600
 
     model_config = ConfigDict(frozen=True)

backend/src/neuroagent/app/dependencies.py

@@ -8,7 +8,6 @@
 from pathlib import Path
 from typing import Annotated, Any, AsyncIterator
 
-import boto3
 from fastapi import Depends, HTTPException, Request
 from fastapi.security import HTTPBearer
 from httpx import AsyncClient, HTTPStatusError, get
@@ -26,6 +25,9 @@
 from neuroagent.app.schemas import OpenRouterModelResponse, UserInfo
 from neuroagent.mcp import MCPClient, create_dynamic_tool
 from neuroagent.new_types import Agent
+from neuroagent.storage.azure_storage import AzureBlobStorageClient
+from neuroagent.storage.base_storage import StorageClient
+from neuroagent.storage.s3_storage import S3StorageClient
 from neuroagent.tools import (
     AssetDownloadOneTool,
     AssetGetAllTool,
@@ -609,36 +611,61 @@ def get_starting_agent(
     return agent
 
 
-def get_s3_client(
+def get_storage_client(
     settings: Annotated[Settings, Depends(get_settings)],
-) -> Any:
-    """Get the S3 client."""
-    if settings.storage.access_key is None:
-        access_key = None
-    else:
-        access_key = settings.storage.access_key.get_secret_value()
+) -> StorageClient:
+    """Get the storage client."""
+    if settings.storage.provider == "s3":
+        access = (
+            settings.storage.s3_access_key.get_secret_value()
+            if settings.storage.s3_access_key
+            else None
+        )
+        secret = (
+            settings.storage.s3_secret_key.get_secret_value()
+            if settings.storage.s3_secret_key
+            else None
+        )
 
-    if settings.storage.secret_key is None:
-        secret_key = None
-    else:
-        secret_key = settings.storage.secret_key.get_secret_value()
-
-    return boto3.client(
-        "s3",
-        endpoint_url=settings.storage.endpoint_url,
-        aws_access_key_id=access_key,
-        aws_secret_access_key=secret_key,
-        aws_session_token=None,
-        config=boto3.session.Config(signature_version="s3v4"),
-    )
+        return S3StorageClient(
+            endpoint_url=settings.storage.s3_endpoint_url,
+            access_key=access,
+            secret_key=secret,
+        )
+    if settings.storage.provider == "azure":
+        if (
+            not settings.storage.azure_account_name
+            or not settings.storage.azure_account_key
+        ):
+            raise RuntimeError(
+                "Azure storage requires azure_account_name and azure_account_key"
+            )
+
+        if settings.storage.azure_endpoint_url:
+            # Local Azurite: use connection string format
+            return AzureBlobStorageClient(
+                azure_endpoint_url=settings.storage.azure_endpoint_url,
+                account_name=settings.storage.azure_account_name.get_secret_value(),
+                account_key=settings.storage.azure_account_key.get_secret_value(),
+                container=settings.storage.container_name,
+            )
+        else:
+            # Prod azure blob storage.
+            return AzureBlobStorageClient(
+                account_name=settings.storage.azure_account_name.get_secret_value(),
+                account_key=settings.storage.azure_account_key.get_secret_value(),
+                container=settings.storage.container_name,
+            )
+
+    raise ValueError("No storage provider defined.")
 
 
 async def get_context_variables(
     request: Request,
     settings: Annotated[Settings, Depends(get_settings)],
     httpx_client: Annotated[AsyncClient, Depends(get_httpx_client)],
     thread: Annotated[Threads, Depends(get_thread)],
-    s3_client: Annotated[Any, Depends(get_s3_client)],
+    storage_client: Annotated[Any, Depends(get_storage_client)],
     user_info: Annotated[UserInfo, Depends(get_user_info)],
     openai_client: Annotated[AsyncOpenAI, Depends(get_openai_client)],
 ) -> dict[str, Any]:
@@ -651,15 +678,15 @@ async def get_context_variables(
 
     return {
         "bluenaas_url": settings.tools.bluenaas.url,
-        "bucket_name": settings.storage.bucket_name,
+        "container_name": settings.storage.container_name,
         "entitycore_url": settings.tools.entitycore.url,
         "current_frontend_url": current_frontend_url,
         "entity_frontend_url": entity_frontend_url,
         "httpx_client": httpx_client,
         "obi_one_url": settings.tools.obi_one.url,
         "openai_client": openai_client,
         "project_id": thread.project_id,
-        "s3_client": s3_client,
+        "storage_client": storage_client,
         "sanity_url": settings.tools.sanity.url,
         "thread_id": thread.thread_id,
         "thumbnail_generation_url": settings.tools.thumbnail_generation.url,

backend/src/neuroagent/app/routers/storage.py

@@ -1,14 +1,14 @@
 """Storage related operations."""
 
 import logging
-from typing import Annotated, Any
+from typing import Annotated
 
-from botocore.exceptions import ClientError
 from fastapi import APIRouter, Depends, HTTPException
 
 from neuroagent.app.config import Settings
-from neuroagent.app.dependencies import get_s3_client, get_settings, get_user_info
+from neuroagent.app.dependencies import get_settings, get_storage_client, get_user_info
 from neuroagent.app.schemas import UserInfo
+from neuroagent.storage.base_storage import StorageClient
 
 logger = logging.getLogger(__name__)
 
@@ -20,30 +20,42 @@ async def generate_presigned_url(
     file_identifier: str,
     user_info: Annotated[UserInfo, Depends(get_user_info)],
     settings: Annotated[Settings, Depends(get_settings)],
-    s3_client: Annotated[Any, Depends(get_s3_client)],
+    storage_client: Annotated[StorageClient, Depends(get_storage_client)],
 ) -> str:
-    """Generate a presigned URL for file access."""
-    # Construct the key with user-specific path (without bucket name)
+    """
+    Generate a presigned URL for file access using the storage abstraction.
+
+    Supports S3/MinIO and Azure/Azurite storage backends.
+
+    Returns
+    -------
+        str: The presigned URL for accessing the file.
+    """
+    # Build key from the authenticated user's id
     key = f"{user_info.sub}/{file_identifier}"
 
-    # Check if object exists first
-    try:
-        s3_client.head_object(Bucket=settings.storage.bucket_name, Key=key)
-    except ClientError as e:
-        if e.response["Error"]["Code"] == "404":
-            raise HTTPException(
-                status_code=404, detail=f"File {file_identifier} not found"
-            )
-        raise HTTPException(status_code=500, detail="Error accessing the file")
-
-    # Generate presigned URL that's valid for 10 minutes
-    presigned_url = s3_client.generate_presigned_url(
-        "get_object",
-        Params={
-            "Bucket": settings.storage.bucket_name,
-            "Key": key,
-        },
-        ExpiresIn=settings.storage.expires_in,
+    # Check existence with the abstracted method (provider-agnostic)
+    metadata = storage_client.get_metadata(
+        container=settings.storage.container_name, key=key
     )
+    if metadata is None:
+        raise HTTPException(status_code=404, detail=f"File {file_identifier} not found")
 
-    return presigned_url
+    # Ask the provider to build a presigned URL / SAS
+    try:
+        url = storage_client.generate_presigned_url(
+            container=settings.storage.container_name,
+            key=key,
+            expires_in=settings.storage.expires_in,
+        )
+    except NotImplementedError:
+        raise HTTPException(
+            status_code=501,
+            detail="Presigned URL generation not implemented for this provider",
+        )
+    except Exception as exc:
+        raise HTTPException(
+            status_code=500, detail=f"Error generating presigned URL: {exc}"
+        )
+
+    return url

backend/src/neuroagent/app/routers/threads.py

@@ -24,9 +24,9 @@
 from neuroagent.app.dependencies import (
     get_openai_client,
     get_redis_client,
-    get_s3_client,
     get_session,
     get_settings,
+    get_storage_client,
     get_thread,
     get_tool_list,
     get_user_info,
@@ -284,7 +284,7 @@ async def update_thread_title(
 async def delete_thread(
     session: Annotated[AsyncSession, Depends(get_session)],
     thread: Annotated[Threads, Depends(get_thread)],
-    s3_client: Annotated[Any, Depends(get_s3_client)],
+    storage_client: Annotated[Any, Depends(get_storage_client)],
     settings: Annotated[Settings, Depends(get_settings)],
     user_info: Annotated[UserInfo, Depends(get_user_info)],
 ) -> dict[str, str]:
@@ -295,8 +295,8 @@ async def delete_thread(
 
     # Delete associated S3 objects first
     delete_from_storage(
-        s3_client=s3_client,
-        bucket_name=settings.storage.bucket_name,
+        storage_client=storage_client,
+        container_name=settings.storage.container_name,
         user_id=user_info.sub,
         thread_id=thread.thread_id,
     )