Merge pull request #11 from memfuse/dev

Dev

Author: savourylie

.env.example

@@ -2,6 +2,10 @@
 OPENAI_API_KEY=sk-your-openai-api-key-here
 OPENAI_BASE_URL=https://api.openai.com/v1
 
+# OpenAI Compatible Model Configuration
+# Model name to use for benchmarks (defaults to gpt-4o-mini if not set)
+OPENAI_COMPATIBLE_MODEL=gpt-4o-mini
+
 # Anthropic API Key
 ANTHROPIC_API_KEY=sk-ant-your-anthropic-api-key-here
 

.gitignore

@@ -174,5 +174,10 @@ cython_debug/
 .pypirc
 
 .DS_Store
-scripts/
-benchmarks/
+CLAUDE.md
+
+.cursor/
+results/
+
+benchmarks/results/
+.claude/

README.md

@@ -55,6 +55,13 @@ When the context window reaches its limit, previous conversations, user preferen
 
 This repository contains the official Python SDK for seamless integration with MemFuse servers. For comprehensive information about the MemFuse server architecture and advanced features, please visit the [MemFuse Server repository](https://github.com/memfuse/memfuse).
 
+## Recent Updates
+
+- **Enhanced Testing:** Comprehensive E2E testing with semantic memory validation
+- **Better Error Handling:** Improved error messages and logging for easier debugging  
+- **Prompt Templates:** Structured prompt management system for consistent LLM interactions
+- **Performance Benchmarks:** MSC dataset accuracy testing with 95% validation threshold
+
 ## Installation
 
 > **Note:** This is the standalone Client SDK repository. If you need to install and run the MemFuse server, which is essential to use the SDK, please visit the [MemFuse Server repository](https://github.com/memfuse/memfuse).
@@ -129,6 +136,22 @@ print(f"Follow-up: {followup_response.choices[0].message.content}")
 
 MemFuse automatically manages the retrieval of relevant information and storage of new memories from conversations within the specified `memory` scope.
 
+## Advanced Features
+
+### Memory Validation & Testing
+The SDK includes comprehensive testing capabilities to validate memory accuracy:
+
+- **E2E Memory Tests:** Automated tests that verify conversational context retention
+- **Semantic Similarity Validation:** Uses RAGAS framework for intelligent response verification
+- **Performance Benchmarks:** MSC (Multi-Session Chat) dataset testing with accuracy metrics
+
+### Error Handling & Debugging
+Enhanced error messages provide clear guidance:
+
+- **Connection Issues:** Helpful instructions for starting the MemFuse server
+- **API Errors:** Detailed error responses with actionable information
+- **Logging:** Comprehensive logging for troubleshooting and monitoring
+
 ## Examples
 
 Explore comprehensive examples in the [examples/](examples/) directory of this repository, featuring:

benchmarks/__init__.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python
+import os
+import sys
+import logging
+from dotenv import load_dotenv
+import google.genai as genai
+from google.genai.errors import ClientError
+from openai import OpenAI
+from openai import OpenAIError
+
+load_dotenv(override=True)
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+def check_gemini_api():
+    """Check if Gemini API is ready and accessible."""
+    
+    api_key = os.getenv("GEMINI_API_KEY")
+    base_url = os.getenv("GEMINI_BASE_URL")
+    
+    if not api_key:
+        logger.error("GEMINI_API_KEY not found in environment variables")
+        return False
+    
+    try:
+        # Configure the client
+        from google.genai import types
+        
+        if base_url:
+            logger.info(f"Using custom base URL: {base_url}")
+            http_options = types.HttpOptions(base_url=base_url)
+            client = genai.Client(api_key=api_key, http_options=http_options)
+        else:
+            client = genai.Client(api_key=api_key)
+        
+        # Test with a simple model list or generation call
+        logger.info("Testing Gemini API connection...")
+        
+        # Try to make a simple generation request
+        response = client.models.generate_content(
+            model="gemini-2.5-flash-lite",
+            contents="Hello, can you respond with just 'OK'?"
+        )
+        
+        if response and response.text:
+            logger.info("SUCCESS: Gemini API is ready and responding correctly")
+            logger.info(f"Response: {response.text.strip()}")
+            return True
+        else:
+            logger.error("ERROR: Gemini API responded but with empty content")
+            return False
+            
+    except ClientError as e:
+        # Check if it's a quota/rate limit error, which means the API is accessible
+        if "RESOURCE_EXHAUSTED" in str(e) or "RATE_LIMIT_EXCEEDED" in str(e) or "429" in str(e):
+            logger.info("SUCCESS: Gemini API is accessible (quota/rate limit reached)")
+            logger.info(f"Rate limit details: {e}")
+            return True
+        else:
+            logger.error(f"ERROR: Gemini API client error: {e}")
+            return False
+    except Exception as e:
+        logger.error(f"ERROR: Unexpected error testing Gemini API: {e}")
+        return False
+
+def check_openai_api():
+    """Check if OpenAI API is ready and accessible."""
+    
+    api_key = os.getenv("OPENAI_API_KEY")
+    base_url = os.getenv("OPENAI_BASE_URL")
+    openai_model = os.getenv("OPENAI_COMPATIBLE_MODEL")
+    
+    if not api_key:
+        logger.error("OPENAI_API_KEY not found in environment variables")
+        return False
+    
+    try:
+        # Configure the client
+        if base_url:
+            logger.info(f"Using custom base URL: {base_url}")
+            client = OpenAI(api_key=api_key, base_url=base_url)
+        else:
+            client = OpenAI(api_key=api_key)
+        
+        # Test with a simple completion request
+        logger.info("Testing OpenAI API connection...")
+        
+        response = client.chat.completions.create(
+            model=openai_model,
+            messages=[{"role": "user", "content": "Hello, can you respond with just 'OK'?"}],
+            max_tokens=10
+        )
+        
+        if response and response.choices and response.choices[0].message.content:
+            logger.info("SUCCESS: OpenAI API is ready and responding correctly")
+            logger.info(f"Response: {response.choices[0].message.content.strip()}")
+            return True
+        else:
+            logger.error("ERROR: OpenAI API responded but with empty content")
+            return False
+            
+    except OpenAIError as e:
+        # Check if it's a quota/rate limit error, which means the API is accessible
+        if "rate_limit_exceeded" in str(e).lower() or "quota" in str(e).lower() or "429" in str(e):
+            logger.info("SUCCESS: OpenAI API is accessible (quota/rate limit reached)")
+            logger.info(f"Rate limit details: {e}")
+            return True
+        else:
+            logger.error(f"ERROR: OpenAI API client error: {e}")
+            return False
+    except Exception as e:
+        logger.error(f"ERROR: Unexpected error testing OpenAI API: {e}")
+        return False
+
+def main():
+    """Main function to check API readiness."""
+    logger.info("Checking API readiness...")
+    
+    gemini_success = check_gemini_api()
+    openai_success = check_openai_api()
+    
+    if gemini_success and openai_success:
+        logger.info("All checks passed - Both APIs are ready!")
+        sys.exit(0)
+    else:
+        if not gemini_success:
+            logger.error("Gemini API check failed")
+        if not openai_success:
+            logger.error("OpenAI API check failed")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

benchmarks/check_api_ready.py

@@ -0,0 +1,24 @@
+# Dataset configurations
+DATASET_CONFIGS = {
+    "msc": {
+        "name": "MSC (Multiple-choice Situation Comprehension)",
+        "dataset_id": "Percena/msc-memfuse-mc10",
+        "data_file": "data/msc_memfuse_mc10.json",
+        "description": "Multiple-choice Situation Comprehension dataset",
+        "question_type": "conversation"
+    },
+    "lme": {
+        "name": "LME (LongMemEval)",
+        "dataset_id": "Percena/lme-mc10",
+        "data_file": "data/lme_s_mc10.json",
+        "description": "LongMemEval dataset",
+        "question_type": "factual"
+    },
+    "locomo": {
+        "name": "LoCoMo (Long Conversation Memory)",
+        "dataset_id": "Percena/locomo-mc10",
+        "data_file": "data/locomo_mc10.json",
+        "description": "Long Conversation Memory dataset",
+        "question_type": "conversation"
+    }
+}