RunValidation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate the trained model against the validation subset of the training data\n",
    "\n",
    "Calculates:\n",
    "\n",
    "* precision\n",
    "* recall\n",
    "* F1\n",
    "\n",
    "\n",
    "Converts the validation ground truth CONLL file and the validation predictions CONLL file to the same format (similar to the task output format but with token indices instead of character indices).\n",
    "\n",
    "This allows you to diff the files to work out a score."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "DATA_DIR = os.environ['RNN_HOME']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "raw_dir = DATA_DIR + \"raw/\"\n",
    "preprocessed_dir = DATA_DIR + \"preprocessed/\"\n",
    "train_dir = DATA_DIR + \"train/\"\n",
    "validation_dir = DATA_DIR + \"validation/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ground truth file /media/thomas/5849-28A2/rnndir_3/preprocessed/protein-validation-ground-truth-annotations.txt\n",
      "predicted file /media/thomas/5849-28A2/rnndir_3/train/brain_pos/greedy/128-0.08-3600-0.9-0/tagged-tuning-corpus\n"
     ]
    }
   ],
   "source": [
    "raw_text_file = raw_dir + 'protein-test.txt'\n",
    "\n",
    "parser_output_file_ground_truth = preprocessed_dir + 'protein-validation-ground-truth-annotations.txt'\n",
    "parser_output_file_predicted = train_dir + 'brain_pos/greedy/128-0.08-3600-0.9-0/tagged-tuning-corpus'\n",
    "\n",
    "output_file_ground_truth = validation_dir + \"protein-validation-ground-truth.txt\"\n",
    "output_file_predicted = validation_dir + \"protein-validation-predicted.txt\"\n",
    "\n",
    "print \"ground truth file\", parser_output_file_ground_truth\n",
    "print \"predicted file\", parser_output_file_predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def convert_conll_to_entity_format(parser_output_file, output_file):\n",
    "    result = []\n",
    "    current_protein = None\n",
    "    proteins = []\n",
    "\n",
    "    import csv\n",
    "\n",
    "    counter = 0\n",
    "\n",
    "    with open(parser_output_file, \"r\") as f:\n",
    "        tsvin = csv.reader(f, delimiter='\\t')\n",
    "        for cols in tsvin:\n",
    "            '''\n",
    "            if counter > 10:\n",
    "                break\n",
    "            '''\n",
    "            if len(cols) < 4:\n",
    "                continue\n",
    "            counter += 1\n",
    "\n",
    "            token = cols[1]\n",
    "            tag = cols[3]\n",
    "\n",
    "            token_index = counter\n",
    "\n",
    "            if (tag == \"O\" or tag == \"BPROTEIN\") and current_protein is not None:            \n",
    "                proteins.append((current_protein[\"start\"], current_protein[\"end\"], current_protein[\"tokens\"]))\n",
    "                current_protein = None\n",
    "            if tag == \"BPROTEIN\":\n",
    "                current_protein = {\"start\":token_index,\"tokens\":[]}\n",
    "            if current_protein is not None:\n",
    "                current_protein[\"end\"] = token_index\n",
    "                current_protein[\"tokens\"].append(token)\n",
    "\n",
    "    if current_protein is not None:\n",
    "        proteins.append((current_protein[\"start\"], current_protein[\"end\"], current_protein[\"tokens\"]))\n",
    "\n",
    "    with open(output_file, \"w\") as f_test:\n",
    "        output_writer = csv.writer(f_test, delimiter='\\t')\n",
    "        output_writer.writerow([\"ExampleID\", \"Class\", \"Start\", \"End\", \"Entity\"])\n",
    "        for index, protein in enumerate(proteins):\n",
    "            cols = [\"T\" + str(index+1), \"Protein\", str(protein[0]), str(protein[1]),  \" \".join(protein[2])]\n",
    "            output_writer.writerow(cols)\n",
    "            result.append(\" \".join(cols[1:]))\n",
    "    return result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compare the ground truth and predicted files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1722 groundtruth entities present\n",
      "1732 predicted entities present\n"
     ]
    }
   ],
   "source": [
    "results = dict()\n",
    "for key, parser_output_file, output_file in [(\"groundtruth\", parser_output_file_ground_truth, output_file_ground_truth),\n",
    "                                       (\"predicted\", parser_output_file_predicted, output_file_predicted)]:\n",
    "    results[key] = convert_conll_to_entity_format(parser_output_file, output_file)\n",
    "    print len(results[key]), key, \"entities present\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Now work out how many entities are in common between ground truth and predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1503 of the entities are present in groundtruth and predicted\n"
     ]
    }
   ],
   "source": [
    "num_intersection = 0\n",
    "all_predicted = set(results[\"predicted\"])\n",
    "for g in results[\"groundtruth\"]:\n",
    "    if g in all_predicted:\n",
    "        num_intersection += 1\n",
    "print num_intersection, \"of the entities are present in groundtruth and predicted\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A quick overview of the first entities found in ground truth and predicted (shown as start and end token indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Protein 13 14 CD 40',\n",
       " 'Protein 39 40 CD 40',\n",
       " 'Protein 46 47 CD 40',\n",
       " 'Protein 71 73 beta - globin',\n",
       " 'Protein 109 112 B 7 . 1',\n",
       " 'Protein 116 119 B 7 . 1',\n",
       " 'Protein 147 150 B 7 . 1',\n",
       " 'Protein 208 210 GATA - 3',\n",
       " 'Protein 217 219 IL - 5',\n",
       " 'Protein 232 234 GATA - 3']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results[\"groundtruth\"][0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Protein 13 14 CD 40',\n",
       " 'Protein 39 40 CD 40',\n",
       " 'Protein 46 47 CD 40',\n",
       " 'Protein 71 73 beta - globin',\n",
       " 'Protein 208 210 GATA - 3',\n",
       " 'Protein 217 219 IL - 5',\n",
       " 'Protein 232 234 GATA - 3',\n",
       " 'Protein 243 245 IL - 5',\n",
       " 'Protein 256 258 GATA - 3',\n",
       " 'Protein 272 274 IL - 5']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results[\"predicted\"][0:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculate precision, recall and F-score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision 0.867782909931\n",
      "recall 0.872822299652\n",
      "f1 0.870295309786\n"
     ]
    }
   ],
   "source": [
    "precision = 1.0 * num_intersection / (len(results[\"predicted\"]))\n",
    "recall = 1.0 * num_intersection / (len(results[\"groundtruth\"]))\n",
    "\n",
    "f1 = 2.0 * precision * recall / (precision + recall)\n",
    "\n",
    "print \"precision\", precision\n",
    "print \"recall\", recall\n",
    "print \"f1\", f1"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}