Patch 1

.env.example

@@ -1,87 +1,129 @@
 # HippocampAI Configuration
 # Copy this file to .env and configure your settings
 
-# ============================================
-# Qdrant Vector Database
-# ============================================
+# ==================== API Configuration ====================
+API_PORT=8000
+
+# ==================== Qdrant Vector Database ====================
 QDRANT_URL=http://localhost:6333
+QDRANT_PORT=6333
+QDRANT_GRPC_PORT=6334
 COLLECTION_FACTS=hippocampai_facts
 COLLECTION_PREFS=hippocampai_prefs
 
-# ============================================
-# HNSW Index Tuning (Advanced)
-# ============================================
+# ==================== HNSW Index Tuning (Advanced) ====================
 # Higher M = better recall, more memory
 HNSW_M=48
 # Higher ef_construction = better index quality, slower build
 EF_CONSTRUCTION=256
 # Higher ef_search = better recall, slower search
 EF_SEARCH=128
 
-# ============================================
-# Embedding Model
-# ============================================
+# ==================== Embedding Model ====================
 EMBED_MODEL=BAAI/bge-small-en-v1.5
 EMBED_DIMENSION=384
 EMBED_BATCH_SIZE=32
 EMBED_QUANTIZED=false
 
-# ============================================
-# Reranker (Cross-Encoder)
-# ============================================
+# ==================== Reranker (Cross-Encoder) ====================
 RERANKER_MODEL=cross-encoder/ms-marco-MiniLM-L-6-v2
 RERANK_CACHE_TTL=86400
 
-# ============================================
-# LLM Provider Configuration
-# ============================================
+# ==================== LLM Provider Configuration ====================
 # Options: ollama, openai, anthropic, groq
 LLM_PROVIDER=ollama
+LLM_MODEL=qwen2.5:7b-instruct
 
 # For Ollama (local, self-hosted)
-LLM_MODEL=qwen2.5:7b-instruct
 LLM_BASE_URL=http://localhost:11434
 
 # For OpenAI (cloud)
 # LLM_PROVIDER=openai
 # LLM_MODEL=gpt-4o-mini
 # OPENAI_API_KEY=sk-your-key-here
 
+# For Groq (fast, cost-effective cloud inference)
+# LLM_PROVIDER=groq
+# LLM_MODEL=llama-3.3-70b-versatile
+# GROQ_API_KEY=gsk_your-groq-api-key-here
+
+# For Anthropic (cloud)
+# LLM_PROVIDER=anthropic
+# LLM_MODEL=claude-3-haiku-20240307
+# ANTHROPIC_API_KEY=sk-ant-your-key-here
+
 # Allow cloud providers (set to true if using OpenAI/Anthropic/Groq)
 ALLOW_CLOUD=false
 
-# ============================================
-# Retrieval Parameters
-# ============================================
+# ==================== Retrieval Parameters ====================
 # Number of candidates from vector search
 TOP_K_QDRANT=200
 # Final number of results after reranking
 TOP_K_FINAL=20
 # Reciprocal rank fusion parameter
 RRF_K=60
 
-# ============================================
-# Scoring Weights (must sum to ~1.0)
-# ============================================
+# ==================== Scoring Weights (must sum to ~1.0) ====================
 WEIGHT_SIM=0.55
 WEIGHT_RERANK=0.20
 WEIGHT_RECENCY=0.15
 WEIGHT_IMPORTANCE=0.10
 
-# ============================================
-# Importance Decay Half-Lives (days)
-# ============================================
+# ==================== Importance Decay Half-Lives (days) ====================
 HALF_LIFE_PREFS=90
 HALF_LIFE_FACTS=30
 HALF_LIFE_EVENTS=14
 
-# ============================================
-# Background Jobs (Optional)
-# ============================================
+# ==================== Background Jobs & Scheduling ====================
 ENABLE_SCHEDULER=true
 # Decay importance daily at 2am
 DECAY_CRON=0 2 * * *
 # Consolidate memories weekly on Sunday at 3am
 CONSOLIDATE_CRON=0 3 * * 0
 # Create snapshots hourly
 SNAPSHOT_CRON=0 * * * *
+
+# Background Tasks
+ENABLE_BACKGROUND_TASKS=true
+AUTO_DEDUP_ENABLED=true
+AUTO_CONSOLIDATION_ENABLED=false
+DEDUP_INTERVAL_HOURS=24
+CONSOLIDATION_INTERVAL_HOURS=168
+EXPIRATION_INTERVAL_HOURS=1
+
+# ==================== Redis Cache ====================
+REDIS_PORT=6379
+REDIS_URL=redis://localhost:6379
+REDIS_DB=0
+REDIS_CACHE_TTL=300
+REDIS_MAX_CONNECTIONS=100
+REDIS_MIN_IDLE=10
+
+# ==================== Celery Task Queue ====================
+CELERY_BROKER_URL=redis://localhost:6379/1
+CELERY_RESULT_BACKEND=redis://localhost:6379/2
+CELERY_WORKER_CONCURRENCY=4
+CELERY_WORKER_PREFETCH_MULTIPLIER=4
+CELERY_TASK_ACKS_LATE=true
+CELERY_WORKER_MAX_TASKS_PER_CHILD=1000
+
+# Flower UI (Celery Monitoring)
+FLOWER_PORT=5555
+FLOWER_USER=admin
+FLOWER_PASSWORD=changeme_in_production
+
+# ==================== Monitoring ====================
+PROMETHEUS_ENABLED=true
+PROMETHEUS_PORT=9090
+
+# Grafana
+GRAFANA_PORT=3000
+GRAFANA_ADMIN_USER=admin
+GRAFANA_ADMIN_PASSWORD=changeme_in_production
+GRAFANA_ROOT_URL=http://localhost:3000
+
+# ==================== Security (Recommended for Production) ====================
+# Enable these in production deployments
+# ENABLE_AUTH=true
+# JWT_SECRET=your_jwt_secret_here
+# RATE_LIMIT_PER_MINUTE=60

.github/workflows/ci.yml

@@ -3,6 +3,8 @@ name: CI
 on:
   push:
     branches: ["*"]
+  pull_request:
+    branches: ["main"]
 
 env:
   PYTHONUNBUFFERED: "1"
@@ -16,64 +18,40 @@ jobs:
       matrix:
         python-version: ["3.10", "3.11", "3.12"]
 
-    services:
-      qdrant:
-        image: qdrant/qdrant:latest
-        ports:
-          - 6333:6333
-          - 6334:6334
-
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955   # v4.3.0
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Cache Python dependencies
-        uses: actions/cache@v4
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830       # v4.3.0
         with:
           path: |
             ~/.cache/pip
             ${{ env.pythonLocation }}
           key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml', '**/requirements.txt') }}
           restore-keys: |
             ${{ runner.os }}-python-${{ matrix.python-version }}-
+
+      - name: Ruff lint
+        run: pip install ruff && ruff check . --output-format=github
 
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -e '.[dev,api]'
-
-      - name: Wait for Qdrant to be ready
-        run: |
-          echo "Waiting for Qdrant to be ready..."
-          for i in {1..30}; do
-            if curl -f http://localhost:6333/healthz > /dev/null 2>&1; then
-              echo "Qdrant is ready!"
-              curl -s http://localhost:6333/ | head -20
-              exit 0
-            fi
-            echo "Attempt $i/30: Qdrant not ready yet, waiting..."
-            sleep 2
-          done
-          echo "Qdrant failed to become ready"
-          exit 1
-
-      - name: Lint with Ruff
+      - name: Ruff format (warn only)
         run: |
-          ruff check . --output-format=github
-          ruff format . --check
+          if ! ruff format . --check; then
+            echo "::warning ::Some files would be reformatted by Ruff"
+          fi
 
-      - name: Run Tests
+      - name: Bandit Security Scan
         run: |
-          pytest -v --maxfail=1 --disable-warnings --cov=src --cov-report=term-missing
-        env:
-          QDRANT_HOST: localhost
-          QDRANT_PORT: 6333
-          QDRANT_URL: http://localhost:6333
+          pip install bandit
+          if ! bandit -r src -lll; then
+            echo "::warning ::Bandit found potential security issues (review above)"
+          fi
 
   build:
     name: Build Package
@@ -82,30 +60,31 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955   # v4.3.0
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
           python-version: "3.11"
 
       - name: Cache Python dependencies
-        uses: actions/cache@v4
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830       # v4.3.0
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-build-${{ hashFiles('**/pyproject.toml') }}
           restore-keys: |
             ${{ runner.os }}-pip-build-
 
-      - name: Install build tools
+      - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install build twine
+          pip install -e '.[dev,api]'
+
+      - name: Install build tools
+        run: pip install build twine
 
       - name: Build package
-        run: |
-          python -m build
+        run: python -m build
 
       - name: Validate build with Twine
-        run: |
-          twine check dist/*
+        run: twine check dist/*

.github/workflows/sonar_scan.yml

@@ -2,25 +2,30 @@ name: SonarQube Scan
 
 on:
   push:
-    branches: [ '*']
-
+    branches: ['*']
 
 jobs:
   build:
     name: Build and analyze
     runs-on: ubuntu-latest
-    
+
     steps:
-      - uses: actions/checkout@v4
+      # actions/checkout v4.3.0 (full SHA)
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955
         with:
-          fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
-      - uses: SonarSource/sonarqube-scan-action@v6
+          fetch-depth: 0
+
+      # SonarQube Scan action v6.0.0 (full SHA)
+      - name: SonarQube Scan
+        uses: SonarSource/sonarqube-scan-action@fd88b7d7ccbaefd23d8f36f73b59db7a3d246602
         env:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
           SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
-      # If you wish to fail your job when the Quality Gate is red, uncomment the
-      # following lines. This would typically be used to fail a deployment.
-      # - uses: SonarSource/sonarqube-quality-gate-action@v1
+
+      # If you want to enforce the Quality Gate, un-comment and keep the pin:
+      # - name: SonarQube Quality Gate
+      #   uses: SonarSource/sonarqube-quality-gate-action@cf038b0e0cdecfa9e56c198bbb7d21d751d62c3b
       #   timeout-minutes: 5
       #   env:
-      #     SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+      #     SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+      #     SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}