Merge branch 'main' into bugfix/providers

Author: DJJones66

.gitignore

@@ -47,6 +47,9 @@ backend/alembic/versions/
 *.log
 backend/logs/
 
+backend/services_runtime/
+backend/persistent/
+
 # Environment variables
 .env
 .env.local

INSTALL.md

@@ -68,53 +68,7 @@ copy .env-dev .env     # Windows
 
 #### Option B: Manual `.env` Setup
 
-Paste the following into `backend/.env`:
-
-```env
-# Application Settings
-APP_NAME="BrainDrive"
-APP_ENV="dev"
-API_V1_PREFIX="/api/v1"
-DEBUG=true
-
-# Server Settings
-HOST="0.0.0.0"
-PORT=8005
-RELOAD=true
-LOG_LEVEL="info"
-
-# Security
-SECRET_KEY="your-secret-key-here"
-ACCESS_TOKEN_EXPIRE_MINUTES=15
-REFRESH_TOKEN_EXPIRE_DAYS=30
-ALGORITHM="HS256"
-
-# Database
-DATABASE_URL="sqlite:///braindrive.db"
-DATABASE_TYPE="sqlite"
-USE_JSON_STORAGE=false
-JSON_DB_PATH="./storage/database.json"
-SQL_LOG_LEVEL="WARNING"
-
-# Redis
-USE_REDIS=false
-REDIS_HOST="localhost"
-REDIS_PORT=6379
-
-# CORS
-CORS_ORIGINS='["http://127.0.0.1:5173", "http://localhost:5173"]'
-CORS_METHODS='["GET", "POST", "PUT", "DELETE", "OPTIONS", "HEAD"]'
-CORS_HEADERS='["Authorization", "Content-Type", "Accept", "Origin", "X-Requested-With"]'
-CORS_EXPOSE_HEADERS='["*"]'
-CORS_MAX_AGE=3600
-CORS_ALLOW_CREDENTIALS=true
-
-# Allowed Hosts
-ALLOWED_HOSTS='["localhost", "127.0.0.1"]'
-
-# AI Providers
-ENABLE_TEST_ROUTES=true
-```
+Look at [/backend/.env-dev](https://github.com/BrainDriveAI/BrainDrive/blob/main/backend/.env-dev) and set as your needs require. 
 
 ---
 

ROADMAP.md

@@ -59,11 +59,12 @@ Our roadmap is broken into clearly defined versions, each building toward a stab
 ## Version 0.6.0 – Open Beta
 > Goal: AI System with core functionality for developers
 
-- [ ] All plugins moved to the Life Cycle Manager
-- [ ] Ollama plugin updated to include server manager
-- [ ] User initializer - Plugin install from remote
-- [ ] User initializer - Restructure navigation 
-- [ ] Prompt Library
+- [x] All plugins moved to the Life Cycle Manager
+- [x] Ollama plugin updated to include server manager
+- [x] User initializer - Plugin install from remote
+- [x] User initializer - Restructure navigation 
+- [ ] Improve Registration
+- [ ] Ollama AI Provider
 - [ ] One-Click Installer - (Windows first)
 
 ---
@@ -73,6 +74,7 @@ Our roadmap is broken into clearly defined versions, each building toward a stab
 
 - [ ] Unified Dynamic Page Renderer - Bounce
 - [ ] Unified Dynamic Page Renderer - Finetune
+- [ ] Prompt Library
 
 ---
 

backend/app/core/security.py

@@ -1,5 +1,5 @@
 from sqlalchemy.ext.asyncio import AsyncSession
-from datetime import datetime, timedelta as datetime_timedelta
+from datetime import datetime, timedelta as datetime_timedelta, UTC
 from typing import Optional
 from jose import jwt, JWTError
 import logging
@@ -41,16 +41,16 @@ def create_access_token(data: dict, expires_delta: Optional[datetime_timedelta]
     """Create a new access token."""
     to_encode = data.copy()
     if expires_delta:
-        expire = datetime.utcnow() + expires_delta
+        expire = datetime.now(UTC) + expires_delta
     else:
-        expire = datetime.utcnow() + datetime_timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES)
+        expire = datetime.now(UTC) + datetime_timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES)
 
     # Convert datetime to Unix timestamp for JWT
     to_encode.update({"exp": expire.timestamp()})
 
     # Let JWT library handle iat automatically if not provided
     if "iat" not in to_encode:
-        to_encode.update({"iat": datetime.utcnow().timestamp()})
+        to_encode.update({"iat": datetime.now(UTC).timestamp()})
 
     encoded_jwt = jwt.encode(to_encode, settings.SECRET_KEY, algorithm=settings.ALGORITHM)
     return encoded_jwt

backend/app/dto/__init__.py

@@ -0,0 +1,65 @@
+from pydantic import BaseModel
+from typing import List, Optional, Union
+from datetime import datetime
+import uuid
+
+# This schema is used for returning data from the repository.
+# It ensures that JSON fields are correctly converted to Python types.
+class PluginServiceRuntimeDTO(BaseModel):
+    """
+    A Pydantic model to represent a PluginServiceRuntime object,
+    with required_env_vars as a list of strings.
+    """
+    id: str
+    plugin_id: str
+    plugin_slug: str
+    name: str
+    source_url: Optional[str] = None
+    type: Optional[str] = None
+    install_command: Optional[str] = None
+    start_command: Optional[str] = None
+    healthcheck_url: Optional[str] = None
+    required_env_vars: List[str] = []
+    status: Optional[str] = None
+    user_id: str
+    created_at: Optional[datetime] = None
+    updated_at: Optional[datetime] = None
+
+    @classmethod
+    def from_github_data(cls, service_dict: dict, plugin_id: str, plugin_slug: str, user_id: str) -> 'PluginServiceRuntimeDTO':
+        """
+        Create a PluginServiceRuntimeDTO from raw GitHub service data (dict).
+        This handles first-time installation where database fields don't exist yet.
+        """
+        return cls(
+            id=str(uuid.uuid4()),  # Generate new UUID for first install
+            plugin_id=plugin_id,
+            plugin_slug=plugin_slug,
+            user_id=user_id,
+            name=service_dict.get('name'),
+            source_url=service_dict.get('source_url'),
+            type=service_dict.get('type', 'python'),
+            install_command=service_dict.get('install_command'),
+            start_command=service_dict.get('start_command'),
+            healthcheck_url=service_dict.get('healthcheck_url'),
+            required_env_vars=service_dict.get('required_env_vars', []),
+            status='installing',
+            created_at=datetime.now(),
+            updated_at=datetime.now()
+        )
+
+    @classmethod
+    def from_dict_or_dto(cls, data: Union[dict, 'PluginServiceRuntimeDTO'], plugin_id: str = None, plugin_slug: str = None, user_id: str = None) -> 'PluginServiceRuntimeDTO':
+        """
+        Flexible factory method that handles both dict (GitHub) and DTO (database) inputs.
+        """
+        if isinstance(data, cls):
+            return data  # Already a DTO, return as-is
+        elif isinstance(data, dict):
+            # Dict from GitHub, convert using factory method
+            if not all([plugin_id, plugin_slug, user_id]):
+                raise ValueError("plugin_id, plugin_slug, and user_id are required when converting from dict")
+            return cls.from_github_data(data, plugin_id, plugin_slug, user_id)
+        else:
+            raise TypeError(f"Expected dict or {cls.__name__}, got {type(data)}")
+

backend/app/dto/plugin.py

@@ -5,6 +5,7 @@
 from app.api.v1.api import api_router
 from app.core.config import settings
 from app.routers.plugins import plugin_manager
+from app.plugins.service_installler.start_stop_plugin_services import start_plugin_services_on_startup, stop_all_plugin_services_on_shutdown
 import logging
 import time
 import structlog
@@ -29,8 +30,21 @@ async def startup_event():
     logger.info("Initializing application settings...")
     from app.init_settings import init_ollama_settings
     await init_ollama_settings()
+    # Start plugin services
+    await start_plugin_services_on_startup()
     logger.info("Settings initialization completed")
 
+
+# Add shutdown event to gracefully stop services
+@app.on_event("shutdown")
+async def shutdown_event():
+    """Gracefully stop all plugin services on application shutdown."""
+    logger.info("Shutting down application and stopping plugin services...")
+    # Stop all plugin services gracefully
+    await stop_all_plugin_services_on_shutdown()
+    logger.info("Application shutdown completed.")
+
+
 # Add middleware to log all requests
 logger = structlog.get_logger()
 
@@ -94,5 +108,3 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
 
 # Include API routers
 app.include_router(api_router)
-
-

backend/app/main.py

@@ -1,10 +1,13 @@
-from sqlalchemy import Column, String, Integer, Boolean, ForeignKey, Text, JSON, UniqueConstraint, TIMESTAMP
+from sqlalchemy import Column, String, Integer, Boolean, ForeignKey, Text, JSON, UniqueConstraint, TIMESTAMP, DateTime
 import sqlalchemy
 from sqlalchemy.orm import relationship
 from sqlalchemy.sql import func
+from datetime import datetime, UTC
+import json
 
 from app.models.base import Base
 
+
 class Plugin(Base):
     """SQLAlchemy model for plugins."""
 
@@ -44,6 +47,7 @@ class Plugin(Base):
     config_fields = Column(Text)  # Stored as JSON string
     messages = Column(Text)       # Stored as JSON string
     dependencies = Column(Text)   # Stored as JSON string
+    required_services_runtime = Column(Text, nullable=True)
 
     # Timestamps
     created_at = Column(String, default=func.now())
@@ -60,6 +64,7 @@ class Plugin(Base):
 
     # Relationships
     modules = relationship("Module", back_populates="plugin", cascade="all, delete-orphan")
+    service_runtimes = relationship("PluginServiceRuntime", back_populates="plugin", cascade="all, delete-orphan")
 
     def to_dict(self):
         """Convert model to dictionary."""
@@ -118,6 +123,11 @@ def to_dict(self):
         else:
             result["permissions"] = []
 
+        if self.required_services_runtime:
+            result["requiredServicesRuntime"] = json.loads(self.required_services_runtime)
+        else:
+            result["requiredServicesRuntime"] = []
+
         return result
 
     @classmethod
@@ -162,10 +172,81 @@ def from_dict(cls, data):
         # Remove modules from data as they are handled separately
         if "modules" in db_data:
             db_data.pop("modules")
+
+        # Handle service runtimes (only store names in plugin table)
+        if "requiredServicesRuntime" in db_data and db_data["requiredServicesRuntime"] is not None:
+            db_data["required_services_runtime"] = json.dumps([
+                r["name"] for r in db_data["requiredServicesRuntime"]
+            ])
+            db_data.pop("requiredServicesRuntime")
 
         return cls(**db_data)
 
 
+class PluginServiceRuntime(Base):
+    """SQLAlchemy model for plugin service runtimes."""
+
+    __tablename__ = "plugin_service_runtime"
+
+    id = Column(String, primary_key=True, index=True)
+    plugin_id = Column(String, ForeignKey("plugin.id"), nullable=False, index=True)
+    plugin_slug = Column(String, nullable=False, index=True)
+
+    name = Column(String, nullable=False)
+    source_url = Column(String)
+    type = Column(String)
+    install_command = Column(Text)
+    start_command = Column(Text)
+    healthcheck_url = Column(String)
+    required_env_vars = Column(Text)  # store as JSON string
+    status = Column(String, default="pending")
+
+    created_at = Column(DateTime, default=datetime.now(UTC))
+    updated_at = Column(DateTime, default=datetime.now(UTC), onupdate=datetime.now(UTC))
+
+    user_id = Column(String(32), ForeignKey("users.id", name="fk_plugin_service_runtime_user_id"), nullable=False)
+    user = relationship("User")
+
+    # Relationship back to plugin
+    plugin = relationship("Plugin", back_populates="service_runtimes")
+
+    def to_dict(self):
+        """
+        Convert the model instance to a dictionary, handling JSON fields.
+        """
+        return {
+            "id": self.id,
+            "plugin_id": self.plugin_id,
+            "plugin_slug": self.plugin_slug,
+            "name": self.name,
+            "source_url": self.source_url,
+            "type": self.type,
+            "install_command": self.install_command,
+            "start_command": self.start_command,
+            "healthcheck_url": self.healthcheck_url,
+            "required_env_vars": json.loads(self.required_env_vars) if self.required_env_vars else [],
+            "status": self.status,
+            "user_id": self.user_id,
+            "created_at": self.created_at.isoformat() if self.created_at else None,
+            "updated_at": self.updated_at.isoformat() if self.updated_at else None,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        """
+        Create a new instance from a dictionary, serializing JSON fields.
+        """
+        db_data = data.copy()
+        
+        if "required_env_vars" in db_data and db_data["required_env_vars"] is not None:
+            db_data["required_env_vars"] = json.dumps(db_data["required_env_vars"])
+            
+        # Handle conversion from camelCase to snake_case if necessary
+        # For simplicity, we are assuming keys in the incoming dict match model attributes
+        
+        return cls(**db_data)
+
+
 class Module(Base):
     """SQLAlchemy model for plugin modules."""