notebooks/4_Evaluate_Presidio_Analyzer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "847acd88",
   "metadata": {},
   "source": [
    "Evaluate Presidio Analyzer using the Presidio Evaluator framework"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae85cae9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "from copy import deepcopy\n",
    "from pprint import pprint\n",
    "from collections import Counter\n",
    "\n",
    "from presidio_evaluator import InputSample\n",
    "from presidio_evaluator.evaluation import Evaluator, ModelError\n",
    "from presidio_evaluator.models import PresidioAnalyzerWrapper\n",
    "from presidio_evaluator.experiment_tracking import get_experiment_tracker\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "pd.set_option(\"display.max_columns\", None)\n",
    "pd.set_option(\"display.max_rows\", None)\n",
    "pd.set_option(\"display.max_colwidth\", None)\n",
    "\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "736fdd23",
   "metadata": {},
   "source": [
    "Select data for evaluation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4cbd55c",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = \"synth_dataset_v2.json\"\n",
    "dataset = InputSample.read_dataset_json(Path(Path.cwd().parent, \"data\", dataset_name))\n",
    "print(len(dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c164ea07",
   "metadata": {},
   "outputs": [],
   "source": [
    "entity_counter = Counter()\n",
    "for sample in dataset:\n",
    "    for tag in sample.tags:\n",
    "        entity_counter[tag] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77aedae6",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Count per entity:\")\n",
    "pprint(entity_counter.most_common())\n",
    "\n",
    "print(\"\\nExample sentence:\")\n",
    "print(dataset[1])\n",
    "\n",
    "print(\"\\nMin and max number of tokens in dataset:\")\n",
    "print(\n",
    "    f\"Min: {min([len(sample.tokens) for sample in dataset])}, \"\n",
    "    f\"Max: {max([len(sample.tokens) for sample in dataset])}\"\n",
    ")\n",
    "\n",
    "print(\"\\nMin and max sentence length in dataset:\")\n",
    "print(\n",
    "    f\"Min: {min([len(sample.full_text) for sample in dataset])}, \"\n",
    "    f\"Max: {max([len(sample.full_text) for sample in dataset])}\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aae4c379",
   "metadata": {},
   "source": [
    "Run evaluation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf65af8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Evaluating Presidio Analyzer\")\n",
    "\n",
    "experiment = get_experiment_tracker()\n",
    "model_name = \"Presidio Analyzer\"\n",
    "model = PresidioAnalyzerWrapper()\n",
    "\n",
    "evaluator = Evaluator(model=model)\n",
    "dataset = Evaluator.align_entity_types(\n",
    "    deepcopy(dataset), entities_mapping=PresidioAnalyzerWrapper.presidio_entities_map\n",
    ")\n",
    "\n",
    "evaluation_results = evaluator.evaluate_all(dataset)\n",
    "results = evaluator.calculate_score(evaluation_results)\n",
    "\n",
    "# update params tracking\n",
    "params = {\"dataset_name\": dataset_name, \"model_name\": model_name}\n",
    "params.update(model.to_log())\n",
    "experiment.log_parameters(params)\n",
    "experiment.log_dataset_hash(dataset)\n",
    "experiment.log_metrics(results.to_log())\n",
    "entities, confmatrix = results.to_confusion_matrix()\n",
    "experiment.log_confusion_matrix(matrix=confmatrix, labels=entities)\n",
    "\n",
    "print(\"Confusion matrix:\")\n",
    "print(pd.DataFrame(confmatrix, columns=entities, index=entities))\n",
    "\n",
    "print(\"Precision and recall\")\n",
    "print(results)\n",
    "\n",
    "# end experiment\n",
    "experiment.end()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "070f8287",
   "metadata": {},
   "source": [
    "### Results analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd04db3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "sent = \"I am taiwanese but I live in Cambodia.\"\n",
    "# sent = input(\"Enter sentence: \")\n",
    "model.predict(InputSample(full_text=sent))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08ae9bda",
   "metadata": {},
   "outputs": [],
   "source": [
    "errors = results.model_errors"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "819eb905",
   "metadata": {},
   "source": [
    "#### False positives"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98f4802e",
   "metadata": {},
   "source": [
    "1. Most false positive tokens:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "640037af",
   "metadata": {},
   "outputs": [],
   "source": [
    "ModelError.most_common_fp_tokens(errors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "03432506",
   "metadata": {},
   "outputs": [],
   "source": [
    "fps_df = ModelError.get_fps_dataframe(errors, entity=[\"LOCATION\"])\n",
    "fps_df[[\"full_text\", \"token\", \"prediction\"]]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d0852513",
   "metadata": {},
   "source": [
    "2. False negative examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "afae40fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "ModelError.most_common_fn_tokens(errors, n=50, entity=[\"PERSON\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "44ed6416",
   "metadata": {},
   "source": [
    "More FN analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7abfcbe9",
   "metadata": {},
   "outputs": [],
   "source": [
    "fns_df = ModelError.get_fns_dataframe(errors, entity=[\"PHONE_NUMBER\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ae73b2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "fns_df[[\"full_text\", \"token\", \"annotation\", \"prediction\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24c8be1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"All errors:\\n\")\n",
    "[print(error, \"\\n\") for error in errors]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "presidio",
   "language": "python",
   "name": "presidio"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}