notebooks/models/Evaluate flair models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluate Flair models for person names, orgs and locations using the Presidio Evaluator framework\n",
    "\n",
    "Data = `generated_test_November 12 2019`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "from presidio_evaluator.data_generator import read_synth_dataset\n",
    "from presidio_evaluator.evaluation import ModelError, Evaluator\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Select data for evaluation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "synth_samples = read_synth_dataset(\"../../data/synth_dataset.txt\")\n",
    "print(len(synth_samples))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "source": [
    "Map entity types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "presidio_entities_map = {\n",
    "    \"PERSON\": \"PER\",\n",
    "    \"EMAIL_ADDRESS\": \"O\",\n",
    "    \"CREDIT_CARD\": \"O\",\n",
    "    \"FIRST_NAME\": \"PER\",\n",
    "    \"PHONE_NUMBER\": \"O\",\n",
    "    \"BIRTHDAY\": \"O\",\n",
    "    \"DATE_TIME\": \"O\",\n",
    "    \"DOMAIN\": \"O\",\n",
    "    \"CITY\": \"LOC\",\n",
    "    \"ADDRESS\": \"LOC\",\n",
    "    \"NATIONALITY\": \"LOC\",\n",
    "    \"LOCATION\": \"LOC\",\n",
    "    \"IBAN\": \"O\",\n",
    "    \"URL\": \"O\",\n",
    "    \"US_SSN\": \"O\",\n",
    "    \"IP_ADDRESS\": \"O\",\n",
    "    \"ORGANIZATION\": \"ORG\",\n",
    "    \"TITLE\" : \"O\", # skipping evaluation of titles\n",
    "    \"O\": \"O\",\n",
    "}\n",
    "\n",
    "synth_samples = Evaluator.align_entity_types(synth_samples, presidio_entities_map)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "entity_counter = Counter()\n",
    "for sample in synth_samples:\n",
    "    for tag in sample.tags:\n",
    "        entity_counter[tag]+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "entity_counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "#max length sentence\n",
    "max([len(sample.tokens) for sample in synth_samples])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Select models for evaluation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "flair_ner = 'ner'\n",
    "flair_ner_fast = 'ner-fast'\n",
    "flair_ontonotes = 'ner-ontonotes-fast'\n",
    "models = [flair_ner, flair_ner_fast]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "from presidio_evaluator.models import FlairModel\n",
    "\n",
    "for model in models:\n",
    "    print(\"-----------------------------------\")\n",
    "    print(\"Evaluating model {}\".format(model))\n",
    "    flair_model = FlairModel(model_path=model)\n",
    "    evaluator = Evaluator(model=flair_model)\n",
    "    evaluation_results = evaluator.evaluate_all(synth_samples)\n",
    "    scores = evaluator.calculate_score(evaluation_results)\n",
    "    \n",
    "     \n",
    "    print(\"Confusion matrix:\")\n",
    "    print(scores.results)\n",
    "\n",
    "    print(\"Precision and recall\")\n",
    "    scores.print()\n",
    "    errors = scores.model_errors\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Custom evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### False positives"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Most false positive tokens:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "errors = scores.model_errors\n",
    "\n",
    "ModelError.most_common_fp_tokens(errors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "fps_df = ModelError.get_fps_dataframe(errors,entity=['PERSON'])\n",
    "fps_df[['full_text','token','prediction']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. False negative examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "ModelError.most_common_fn_tokens(errors,n=50, entity=['PER'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "More FN analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "fns_df = ModelError.get_fns_dataframe(errors,entity=['PERSON'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "fns_df[['full_text','token','annotation','prediction']]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "presidio-research",
   "language": "python",
   "name": "presidio-research"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}