generate.py

#!/usr/bin/env python3
"""
DevData Practice — Generate realistic development economics datasets.

Usage:
    python generate.py                      # Generate all datasets
    python generate.py household_survey     # Generate a single dataset
    python generate.py --list               # List available datasets
    python generate.py --rows 50000         # Override default size
    python generate.py --seed 99            # Set random seed
    python generate.py --format parquet     # Output as parquet
    python generate.py --output ./mydata    # Custom output directory
"""

import argparse
import importlib
import sys
import time
from pathlib import Path

GENERATORS = {
    "household_survey": {
        "module": "generators.household_survey",
        "size_param": "n_households",
        "default_size": 15000,
        "description": "LSMS-style household survey (demographics, consumption, assets)",
    },
    "rct_experiment": {
        "module": "generators.rct_experiment",
        "size_param": "n_individuals",
        "default_size": 25000,
        "description": "Multi-arm RCT with compliance, attrition, and spillovers",
    },
    "panel_data": {
        "module": "generators.panel_data",
        "size_param": None,
        "default_size": None,
        "description": "Country-level panel (25 countries × 25 years × 20 indicators)",
    },
    "agriculture": {
        "module": "generators.agriculture",
        "size_param": "n_households",
        "default_size": 15000,
        "description": "Plot-level agricultural survey (yields, inputs, weather)",
    },
    "health_nutrition": {
        "module": "generators.health_nutrition",
        "size_param": "n_children",
        "default_size": 35000,
        "description": "DHS-style health survey (anthropometrics, vaccination, maternal)",
    },
    "education": {
        "module": "generators.education",
        "size_param": "n_schools",
        "default_size": 500,
        "description": "School + student-level data (test scores, attendance, resources)",
    },
    "labor_market": {
        "module": "generators.labor_market",
        "size_param": "n_individuals",
        "default_size": 40000,
        "description": "Labor force survey (employment, wages, formality, migration)",
    },
    "microfinance": {
        "module": "generators.microfinance",
        "size_param": "n_loans",
        "default_size": 30000,
        "description": "MFI loan records (group lending, repayment, default)",
    },
    "targeting": {
        "module": "generators.targeting",
        "size_param": "n_households",
        "default_size": 20000,
        "description": "PMT targeting data (true vs. predicted poverty, errors)",
    },
    "trade_markets": {
        "module": "generators.trade_markets",
        "size_param": "n_markets",
        "default_size": 80,
        "description": "Weekly market prices (80 markets × 104 weeks × 8 commodities)",
    },
    # ── Sector-specific generators ──
    "gender_programme": {
        "module": "generators.gender_programme",
        "size_param": "n_individuals",
        "default_size": 25000,
        "description": "Gender programme (empowerment, GBV, decision-making, SRH)",
    },
    "girls_education": {
        "module": "generators.girls_education",
        "size_param": "n_girls",
        "default_size": 20000,
        "description": "Girls' education (enrollment, MHM, safety, barriers, transition)",
    },
    "climate_resilience": {
        "module": "generators.climate_resilience",
        "size_param": "n_households",
        "default_size": 20000,
        "description": "Climate & resilience (shocks, coping, adaptation, RIMA index)",
    },
    "agri_value_chain": {
        "module": "generators.agri_value_chain",
        "size_param": "n_transactions",
        "default_size": 25000,
        "description": "Agriculture value chain (farm→market, margins, quality, PHL)",
    },
    "animal_welfare": {
        "module": "generators.animal_welfare",
        "size_param": "n_records",
        "default_size": 15000,
        "description": "Animal welfare (Five Freedoms, BCS, vet access, working animals)",
    },
    "public_health": {
        "module": "generators.public_health",
        "size_param": "n_individuals",
        "default_size": 20000,
        "description": "Public health (disease surveillance, CHW, insurance, mental health)",
    },
    "livelihoods": {
        "module": "generators.livelihoods",
        "size_param": "n_households",
        "default_size": 20000,
        "description": "Livelihoods (VSLA, enterprise, financial inclusion, FCS, assets)",
    },
    "advocacy_rights": {
        "module": "generators.advocacy_rights",
        "size_param": "n_individuals",
        "default_size": 15000,
        "description": "Advocacy & rights (legal aid, tenure, justice, civic space)",
    },
    "wash": {
        "module": "generators.wash",
        "size_param": "n_households",
        "default_size": 18000,
        "description": "WASH (JMP ladders, water quality, sanitation, hygiene, diarrhea)",
    },
    "humanitarian": {
        "module": "generators.humanitarian",
        "size_param": "n_individuals",
        "default_size": 18000,
        "description": "Humanitarian response (displacement, needs, aid, protection, CwC)",
    },
    "social_protection": {
        "module": "generators.social_protection",
        "size_param": "n_households",
        "default_size": 20000,
        "description": "Social protection (cash transfers, conditionality, graduation)",
    },
    "governance": {
        "module": "generators.governance",
        "size_param": "n_citizens",
        "default_size": 15000,
        "description": "Governance & accountability (service delivery, corruption, trust)",
    },
    # ── Cross-cutting & methodological generators ──
    "behaviour_change": {
        "module": "generators.behaviour_change",
        "size_param": "n_individuals",
        "default_size": 20000,
        "description": "BCC / KAP survey (knowledge, attitude, practice, campaign effects)",
    },
    "cost_effectiveness": {
        "module": "generators.cost_effectiveness",
        "size_param": "n_programmes",
        "default_size": 15000,
        "description": "Programme costing & CEA (costs, outcomes, ICER, DALY, QALY)",
    },
    "decent_work": {
        "module": "generators.decent_work",
        "size_param": "n_workers",
        "default_size": 25000,
        "description": "ILO decent work (formal/informal, earnings, protection, unions)",
    },
    "care_economy": {
        "module": "generators.care_economy",
        "size_param": "n_individuals",
        "default_size": 20000,
        "description": "Unpaid care & time use (diary data, gender gap, time poverty)",
    },
    "intersectionality": {
        "module": "generators.intersectionality",
        "size_param": "n_individuals",
        "default_size": 25000,
        "description": "Intersectional inequality (caste, gender, disability, outcomes)",
    },
    "environmental_justice": {
        "module": "generators.environmental_justice",
        "size_param": "n_households",
        "default_size": 20000,
        "description": "Environmental justice (pollution, health, climate vulnerability)",
    },
    "community_development": {
        "module": "generators.community_development",
        "size_param": "n_individuals",
        "default_size": 18000,
        "description": "Social capital & collective action (trust, CDD, participation)",
    },
    "digital_access": {
        "module": "generators.digital_access",
        "size_param": "n_individuals",
        "default_size": 20000,
        "description": "Digital divide (access, literacy, misinformation, privacy)",
    },
    "social_emotional_learning": {
        "module": "generators.social_emotional_learning",
        "size_param": "n_students",
        "default_size": 15000,
        "description": "Youth SEL (5 CASEL domains, wellbeing, behaviour, academics)",
    },
    "ngo_finance": {
        "module": "generators.ngo_finance",
        "size_param": "n_programmes",
        "default_size": 10000,
        "description": "NGO programme finance (budgets, donors, VfM, compliance)",
    },
    "aid_effectiveness": {
        "module": "generators.aid_effectiveness",
        "size_param": "n_flows",
        "default_size": 12000,
        "description": "ODA & aid effectiveness (Paris Declaration, fragmentation, coordination)",
    },
    "media_development": {
        "module": "generators.media_development",
        "size_param": "n_individuals",
        "default_size": 18000,
        "description": "Media & information ecosystems (access, literacy, misinformation)",
    },
    "irt_assessment": {
        "module": "generators.irt_assessment",
        "size_param": "n_respondents",
        "default_size": 20000,
        "description": "IRT psychometric assessment (3PL model, 30 items, DIF, response time)",
    },
    "field_survey_quality": {
        "module": "generators.field_survey_quality",
        "size_param": "n_surveys",
        "default_size": 20000,
        "description": "Survey paradata & QC (timing, GPS, straightlining, back-checks)",
    },
}


def main():
    parser = argparse.ArgumentParser(
        description="Generate realistic development economics practice datasets.",
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument("dataset", nargs="*", default=[],
                        help="Dataset name(s) to generate. Omit for all.")
    parser.add_argument("--list", action="store_true",
                        help="List available datasets and exit.")
    parser.add_argument("--rows", type=int, default=None,
                        help="Override default dataset size.")
    parser.add_argument("--seed", type=int, default=None,
                        help="Random seed (default varies by dataset).")
    parser.add_argument("--format", choices=["csv", "parquet"], default="csv",
                        help="Output format (default: csv).")
    parser.add_argument("--output", type=str, default="output",
                        help="Output directory (default: ./output).")

    args = parser.parse_args()

    if args.list:
        print("\nAvailable datasets:\n")
        for name, info in GENERATORS.items():
            size_str = f"(default ~{info['default_size']:,} units)" if info["default_size"] else "(fixed size)"
            print(f"  {name:<22} {info['description']}")
            print(f"  {'':22} {size_str}\n")
        return

    datasets = args.dataset if args.dataset else list(GENERATORS.keys())

    # Validate names
    for name in datasets:
        if name not in GENERATORS:
            print(f"Error: Unknown dataset '{name}'.")
            print(f"Available: {', '.join(GENERATORS.keys())}")
            sys.exit(1)

    output_dir = Path(args.output)
    output_dir.mkdir(parents=True, exist_ok=True)

    total_rows = 0
    print(f"\n{'='*60}")
    print(f"  DevData Practice — Dataset Generator")
    print(f"{'='*60}\n")

    for name in datasets:
        info = GENERATORS[name]
        print(f"  Generating: {name}...", end=" ", flush=True)
        t0 = time.time()

        mod = importlib.import_module(info["module"])

        kwargs = {}
        if info["size_param"] and args.rows:
            kwargs[info["size_param"]] = args.rows
        if args.seed is not None:
            kwargs["seed"] = args.seed

        df = mod.generate(**kwargs)

        # Save
        ext = args.format
        out_path = output_dir / f"{name}.{ext}"
        if ext == "csv":
            df.to_csv(out_path, index=False)
        else:
            df.to_parquet(out_path, index=False)

        elapsed = time.time() - t0
        rows, cols = df.shape
        total_rows += rows
        size_mb = out_path.stat().st_size / (1024 * 1024)
        print(f"{rows:>8,} rows × {cols:>3} cols  ({size_mb:.1f} MB)  [{elapsed:.1f}s]")

    print(f"\n  Total: {total_rows:,} rows across {len(datasets)} datasets")
    print(f"  Output: {output_dir.resolve()}/")
    print(f"{'='*60}\n")


if __name__ == "__main__":
    main()