GenerateFullReport.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1c3324aa",
   "metadata": {},
   "source": [
    "# Generate a human readable report of all questions and bot responses from all four bots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5ed8b527",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/thomas/anaconda3/lib/python3.11/site-packages/pandas/core/arrays/masked.py:60: UserWarning: Pandas requires version '1.3.6' or newer of 'bottleneck' (version '1.3.5' currently installed).\n",
      "  from pandas.core import (\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import json, re\n",
    "\n",
    "MODELS = [\"gpt-3.5-turbo\", \"gpt-4\", \"gpt-4o\", \"insolvency_bot_with_gpt-3.5-turbo\", \"insolvency_bot_with_gpt-4\", \"insolvency_bot_with_gpt-4o\"]\n",
    "DATASETS = [\"train\", \"test\"]\n",
    "\n",
    "for DATASET in DATASETS:\n",
    "    df_all_data = pd.DataFrame()\n",
    "    for MODEL in MODELS:    \n",
    "        df = pd.read_csv(f\"scores_{DATASET}_{MODEL}.csv\", encoding=\"utf-8\", sep=\"\\t\")\n",
    "        # Drop the TOTAL row\n",
    "#         df = df[df.question_no != \"TOTAL\"]\n",
    "        \n",
    "        if MODEL == \"gpt-3.5-turbo\":\n",
    "            df_all_data[\"question_no\"] = df['question_no']\n",
    "            df_all_data[\"question_text\"] = df['question_text']            \n",
    "            df_all_data[\"max_points_available\"] = df['max_points_available']\n",
    "            df_all_data[\"mark_scheme\"] = df['mark_scheme']\n",
    "        \n",
    "        \n",
    "        df_all_data[\"response_\" + MODEL] = df['bot_response']\n",
    "        df_all_data[\"score_breakdown_\" + MODEL] = df['bot_score_breakdown']\n",
    "        df_all_data[\"score_\" + MODEL] = df['bot_score']\n",
    "        \n",
    "        \n",
    "    df_all_data = df_all_data[df_all_data.max_points_available > 0]    \n",
    "    \n",
    "    with open(f\"all_responses_{DATASET}.md\", \"w\", encoding=\"utf-8\") as f:\n",
    "        f.write(f\"# All responses for {DATASET} questions\\n\\n\")\n",
    "        for idx in range(len(df_all_data)):\n",
    "            if df_all_data.question_no.iloc[idx] == \"TOTAL\":\n",
    "                continue\n",
    "            f.write(f\"## Question {re.sub(r'Q', '', df_all_data.question_no.iloc[idx])}\\n\\n\")\n",
    "            f.write(f\"*Text of {df_all_data.question_no.iloc[idx]}*: {df_all_data.question_text.iloc[idx]}\\n\\n\")\n",
    "            f.write(f\"*Maximum points available for {df_all_data.question_no.iloc[idx]}*: {df_all_data.max_points_available.iloc[idx]}\\n\\n\")\n",
    "            f.write(f\"*Mark scheme {df_all_data.question_no.iloc[idx]}*\\n\\n\")\n",
    "            ms = json.loads(df_all_data.mark_scheme.iloc[idx])\n",
    "\n",
    "            mst = pd.DataFrame()\n",
    "            mst[\"no\"] = list(range(1, len(ms[0]) + 1))\n",
    "            mst[\"points\"] = ms[1]\n",
    "            mst[\"criterion\"] = ms[0]\n",
    "\n",
    "            f.write(mst.to_markdown(index=False) +\"\\n\\n\")\n",
    "\n",
    "            for MODEL in MODELS:\n",
    "                sc = df_all_data[\"score_\" + MODEL].iloc[idx]\n",
    "                f.write(f\"### {MODEL} response to {df_all_data.question_no.iloc[idx]} (score: {sc})\\n\\n\")\n",
    "\n",
    "                f.write(df_all_data[\"response_\" + MODEL].iloc[idx] + \"\\n\\n\")\n",
    "\n",
    "            mst.drop(columns=[\"criterion\"], axis=1, inplace=True)\n",
    "            mst.rename(columns={\"points\":\"max points\"}, inplace=True)\n",
    "            f.write(f\"### Scores of all four models on {df_all_data.question_no.iloc[idx]}\\n\\n\")\n",
    "            for MODEL in MODELS:\n",
    "                mst[MODEL] = json.loads(df_all_data[\"score_breakdown_\" + MODEL].iloc[idx])\n",
    "\n",
    "            f.write(mst.to_markdown(index=False) +\"\\n\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fcc4196e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_all_data.to_excel(\"score_summary_2.xlsx\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91c3fa24",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}