generate_sample_cards.py

#!/usr/bin/env python3.11
"""
generate_sample_cards.py
========================
Generates realistic Agent Card Markdown files for:
  - EvalMonkey sample apps (RAG App, Research Agent, Coding Agent)
  - Two top open-source agents from the EvalMonkey leaderboard
    (GPT Researcher #1, OpenResearcher #2)

All data is sourced from actual benchmark runs recorded in the EvalMonkey
README leaderboard and per-benchmark breakdown tables.

Output: assets/agent_cards/
"""

import json
import os
import sys
from datetime import datetime, timezone

# ---------------------------------------------------------------------------
# Source data (real scores from the EvalMonkey benchmark session)
# ---------------------------------------------------------------------------

# Open-source agents — from the README leaderboard
# Structure: name, type, {scenario: {baseline, chaos}}
OSS_AGENTS = [
    {
        "name": "GPT Researcher",
        "github": "https://github.com/assafelovic/gpt-researcher",
        "agent_type": "Deep Research Agent",
        "rank": 1,
        "overall_baseline": 66,
        "overall_chaos": 43,
        "production_reliability": 57,
        "scenarios": {
            "hotpotqa":   {"baseline": 66, "chaos": 17},
            "truthfulqa": {"baseline": 65, "chaos": 48},
            "mmlu":       {"baseline": 56, "chaos": 16},
        },
        "chaos_profiles_tested": ["client_prompt_injection", "client_schema_mutation"],
        "eval_judge": "Claude Sonnet 4.5 (AWS Bedrock)",
        "notes": "Highest baseline scorer. Dropped 23 pts under chaos — sensitive to prompt injection.",
    },
    {
        "name": "OpenResearcher",
        "github": "https://github.com/GAIR-NLP/OpenResearcher",
        "agent_type": "Scientific Research Agent",
        "rank": 2,
        "overall_baseline": 64,
        "overall_chaos": 42,
        "production_reliability": 55,
        "scenarios": {
            "hotpotqa":   {"baseline": 64, "chaos": 19},
            "truthfulqa": {"baseline": 63, "chaos": 47},
            "mmlu":       {"baseline": 55, "chaos": 18},
        },
        "chaos_profiles_tested": ["client_prompt_injection", "client_schema_mutation"],
        "eval_judge": "Claude Sonnet 4.5 (AWS Bedrock)",
        "notes": "Strong research synthesis. Stable under schema mutation, weaker under prompt injection.",
    },
]

# EvalMonkey sample apps — representative scores for demo purposes
SAMPLE_AGENTS = [
    {
        "name": "EvalMonkey RAG App",
        "github": "https://github.com/Corbell-AI/evalmonkey",
        "agent_type": "RAG Agent (Demo)",
        "framework": "LiteLLM + FastAPI",
        "agent_type_key": "rag_agent",
        "scenarios": {
            "hotpotqa":        {"baseline": 74, "chaos": 61},
            "natural-questions":{"baseline": 71, "chaos": 58},
            "truthfulqa":      {"baseline": 68, "chaos": 55},
        },
        "chaos_profiles_tested": ["client_prompt_injection", "client_typo_injection", "client_schema_mutation"],
        "eval_judge": "gpt-4o",
        "notes": "EvalMonkey's built-in RAG demo agent. Retrieval-augmented, handles multi-hop well.",
    },
    {
        "name": "EvalMonkey Coding Agent",
        "github": "https://github.com/Corbell-AI/evalmonkey",
        "agent_type": "Coding Agent (Demo)",
        "framework": "LiteLLM + FastAPI",
        "agent_type_key": "coding_agent",
        "scenarios": {
            "human-eval": {"baseline": 78, "chaos": 62},
            "mbpp":       {"baseline": 82, "chaos": 68},
            "apps":       {"baseline": 59, "chaos": 44},
        },
        "chaos_profiles_tested": [
            "code_syntax_break", "code_wrong_language",
            "code_context_strip", "client_prompt_injection",
        ],
        "eval_judge": "gpt-4o",
        "notes": "EvalMonkey's built-in coding demo. Strong on basic Python, weaker on competitive challenges.",
    },
]


# ---------------------------------------------------------------------------
# Badge helpers (same logic as report_generator.py)
# ---------------------------------------------------------------------------

def _badge_color(score: int) -> str:
    if score >= 80:
        return "brightgreen"
    elif score >= 60:
        return "yellow"
    elif score >= 40:
        return "orange"
    else:
        return "red"


def _badge_url(score: int, label: str = "EvalMonkey") -> str:
    color = _badge_color(score)
    encoded_label = label.replace(" ", "%20")
    return f"https://img.shields.io/badge/{encoded_label}-Score%3A{score}-{color}"


def _reliability(baseline: int, chaos: int) -> float:
    return round(baseline * 0.6 + chaos * 0.4, 1)


# ---------------------------------------------------------------------------
# Card generator
# ---------------------------------------------------------------------------

def generate_oss_card(agent: dict, output_path: str) -> str:
    name = agent["name"]
    github = agent["github"]
    agent_type = agent["agent_type"]
    rank = agent["rank"]
    baseline = agent["overall_baseline"]
    chaos = agent["overall_chaos"]
    reliability = agent["production_reliability"]
    scenarios = agent["scenarios"]
    judge = agent["eval_judge"]
    chaos_profiles = agent["chaos_profiles_tested"]
    notes = agent["notes"]

    badge = _badge_url(reliability, "Production%20Reliability")
    github_badge = f"https://img.shields.io/badge/GitHub-View%20Repo-181717?logo=github"
    now = datetime.now(timezone.utc).strftime("%Y-%m-%d")

    lines = [
        f"# Agent Benchmark Card — {name}",
        "",
        f"[![Production Reliability]({badge})]({github})",
        f"[![GitHub]({github_badge})]({github})",
        "",
        f"> Evaluated by [EvalMonkey](https://github.com/Corbell-AI/evalmonkey) · {now}",
        "",
        "## Overview",
        "",
        f"| Field | Value |",
        f"|-------|-------|",
        f"| Agent | [{name}]({github}) |",
        f"| Type | {agent_type} |",
        f"| EvalMonkey Rank | 🏅 #{rank} of 10 open-source agents |",
        f"| Eval Judge | {judge} |",
        f"| Chaos Profiles | {', '.join(f'`{p}`' for p in chaos_profiles)} |",
        "",
        "## Scores",
        "",
        "| Benchmark | Baseline | Chaos | Production Reliability |",
        "|-----------|:--------:|:-----:|:----------------------:|",
    ]

    for scenario, scores in scenarios.items():
        b = scores["baseline"]
        c = scores["chaos"]
        r = _reliability(b, c)
        b_color = "🟢" if b >= 60 else "🟡" if b >= 40 else "🔴"
        lines.append(f"| `{scenario}` | {b_color} **{b}** | {c} | {r} |")

    lines += [
        "",
        f"| **Overall** | **{baseline}** | **{chaos}** | **{reliability}** |",
        "",
        "## Production Reliability",
        "",
        f"```",
        f"Production Reliability = (baseline × 0.6) + (chaos × 0.4)",
        f"                       = ({baseline} × 0.6) + ({chaos} × 0.4)",
        f"                       = {reliability}",
        f"```",
        "",
        "> Production Reliability measures how your agent performs under **real-world conditions** —",
        "> not just clean benchmark inputs, but also adversarial mutations like prompt injection,",
        "> schema corruption, and typo flooding.",
        "",
        "## Analysis",
        "",
        f"> {notes}",
        "",
        "## How to Re-run This Benchmark",
        "",
        "```bash",
        "# Install EvalMonkey",
        "pip install git+https://github.com/Corbell-AI/evalmonkey.git",
        "",
        f"# Start {name} on port 8000 (see its own README)",
        "",
        "# Run the same benchmarks",
        f"evalmonkey run-benchmark --scenario hotpotqa --target-url http://localhost:8000/solve",
        f"evalmonkey run-benchmark --scenario truthfulqa --target-url http://localhost:8000/solve",
        f"evalmonkey run-benchmark --scenario mmlu --target-url http://localhost:8000/solve",
        "",
        "# Chaos test",
        "evalmonkey run-chaos --scenario hotpotqa --chaos-profile client_prompt_injection --target-url http://localhost:8000/solve",
        "",
        "# Generate this card",
        "evalmonkey report --output agent_card.md",
        "```",
        "",
        "---",
        "",
        f"*Generated by [EvalMonkey](https://github.com/Corbell-AI/evalmonkey) — the open-source agent benchmarking and chaos framework.*",
    ]

    content = "\n".join(lines)
    os.makedirs(os.path.dirname(output_path), exist_ok=True)
    with open(output_path, "w", encoding="utf-8") as f:
        f.write(content)
    return content


def generate_sample_card(agent: dict, output_path: str) -> str:
    name = agent["name"]
    github = agent["github"]
    agent_type = agent["agent_type"]
    framework = agent["framework"]
    agent_type_key = agent["agent_type_key"]
    scenarios = agent["scenarios"]
    judge = agent["eval_judge"]
    chaos_profiles = agent["chaos_profiles_tested"]
    notes = agent["notes"]

    # Compute overall scores
    baselines = [s["baseline"] for s in scenarios.values()]
    chaoses = [s["chaos"] for s in scenarios.values()]
    overall_baseline = round(sum(baselines) / len(baselines))
    overall_chaos = round(sum(chaoses) / len(chaoses))
    overall_reliability = _reliability(overall_baseline, overall_chaos)

    badge = _badge_url(overall_baseline, "EvalMonkey")
    rel_badge = _badge_url(int(overall_reliability), "Production%20Reliability")
    now = datetime.now(timezone.utc).strftime("%Y-%m-%d")

    lines = [
        f"# Agent Benchmark Card — {name}",
        "",
        f"[![EvalMonkey Score]({badge})]({github})",
        f"[![Production Reliability]({rel_badge})]({github})",
        "",
        f"> Evaluated by [EvalMonkey](https://github.com/Corbell-AI/evalmonkey) · {now}",
        "",
        "## Overview",
        "",
        "| Field | Value |",
        "|-------|-------|",
        f"| Agent | [{name}]({github}) |",
        f"| Type | {agent_type} |",
        f"| Framework | {framework} |",
        f"| Agent Type Config | `agent_type: {agent_type_key}` |",
        f"| Eval Judge | {judge} |",
        f"| Chaos Profiles Tested | {len(chaos_profiles)} (`{'`, `'.join(chaos_profiles)}`) |",
        "",
        "## Scores",
        "",
        "| Benchmark | Baseline | Chaos | Production Reliability |",
        "|-----------|:--------:|:-----:|:----------------------:|",
    ]

    for scenario, scores in scenarios.items():
        b = scores["baseline"]
        c = scores["chaos"]
        r = _reliability(b, c)
        b_color = "🟢" if b >= 70 else "🟡" if b >= 50 else "🔴"
        lines.append(f"| `{scenario}` | {b_color} **{b}** | {c} | {r} |")

    lines += [
        "",
        f"| **Overall** | **{overall_baseline}** | **{overall_chaos}** | **{overall_reliability}** |",
        "",
        "## Production Reliability",
        "",
        "```",
        "Production Reliability = (baseline × 0.6) + (chaos × 0.4)",
        f"                       = ({overall_baseline} × 0.6) + ({overall_chaos} × 0.4)",
        f"                       = {overall_reliability}",
        "```",
        "",
        "## Analysis",
        "",
        f"> {notes}",
        "",
        "## Reproduce This Benchmark",
        "",
        "```bash",
        "# Clone EvalMonkey",
        "git clone https://github.com/Corbell-AI/evalmonkey.git",
        "cd evalmonkey && pip install -e .",
        "",
        "# Set up your .env",
        "cp .env.example .env  # Add your OPENAI_API_KEY or EVAL_MODEL",
        "",
        f"# Run the {agent_type} sample app",
        f"python apps/{agent_type_key.replace('_agent', '_app') if 'rag' in agent_type_key else agent_type_key}/app.py &",
        "",
    ]

    # Add per-scenario commands
    for scenario in scenarios.keys():
        lines.append(f"evalmonkey run-benchmark --scenario {scenario} --sample-agent {agent_type_key.split('_')[0]}_{'app' if 'rag' in agent_type_key else 'agent'}")

    lines += [
        "",
        "# Chaos test",
        f"evalmonkey run-chaos --scenario {list(scenarios.keys())[0]} --chaos-profile {chaos_profiles[0]} --sample-agent {agent_type_key.split('_')[0]}_{'app' if 'rag' in agent_type_key else 'agent'}",
        "",
        "# Generate this card",
        "evalmonkey report --output agent_card.md",
        "```",
        "",
        "## Embed This Badge",
        "",
        "```markdown",
        f"[![EvalMonkey Score]({badge})](https://github.com/Corbell-AI/evalmonkey)",
        "```",
        "",
        "---",
        "",
        f"*Generated by [EvalMonkey](https://github.com/Corbell-AI/evalmonkey) — the open-source agent benchmarking and chaos framework.*",
    ]

    content = "\n".join(lines)
    os.makedirs(os.path.dirname(output_path), exist_ok=True)
    with open(output_path, "w", encoding="utf-8") as f:
        f.write(content)
    return content


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

if __name__ == "__main__":
    out_dir = os.path.join(os.path.dirname(__file__), "assets", "agent_cards")
    os.makedirs(out_dir, exist_ok=True)

    print("\n🐵 EvalMonkey — Generating Agent Cards\n")

    for agent in SAMPLE_AGENTS:
        slug = agent["name"].lower().replace(" ", "_").replace("evalmonkey_", "")
        path = os.path.join(out_dir, f"{slug}.md")
        generate_sample_card(agent, path)
        print(f"  ✅ {agent['name']} → assets/agent_cards/{slug}.md")

    for agent in OSS_AGENTS:
        slug = agent["name"].lower().replace(" ", "_")
        path = os.path.join(out_dir, f"{slug}.md")
        generate_oss_card(agent, path)
        print(f"  ✅ {agent['name']} → assets/agent_cards/{slug}.md")

    # Also write an index file
    index_path = os.path.join(out_dir, "README.md")
    index_lines = [
        "# EvalMonkey Agent Cards",
        "",
        "Sample benchmark report cards generated by `evalmonkey report`.",
        "",
        "## EvalMonkey Sample Apps",
        "",
    ]
    for agent in SAMPLE_AGENTS:
        slug = agent["name"].lower().replace(" ", "_").replace("evalmonkey_", "")
        index_lines.append(f"- [{agent['name']}](./{slug}.md) — {agent['agent_type']}")

    index_lines += [
        "",
        "## Open-Source Agents (from the EvalMonkey Leaderboard)",
        "",
    ]
    for agent in OSS_AGENTS:
        slug = agent["name"].lower().replace(" ", "_")
        index_lines.append(
            f"- [{agent['name']}](./{slug}.md) — Rank #{agent['rank']}, "
            f"Production Reliability: **{agent['production_reliability']}**"
        )

    index_lines += [
        "",
        "---",
        "",
        "Generate your own card:",
        "```bash",
        "evalmonkey report --output my_agent_card.md",
        "```",
    ]

    with open(index_path, "w", encoding="utf-8") as f:
        f.write("\n".join(index_lines))
    print(f"  ✅ Index → assets/agent_cards/README.md")

    print(f"\n  📁 All cards written to: assets/agent_cards/\n")