notebooks/PII EDA.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fake PII data: Exploratory data analysis\n",
    "\n",
    "This notebook is used to verify the different fake entities before and after the creation of a synthetic dataset / augmented dataset. First part looks at the generation details and stats, second part evaluates the created synthetic dataset after it has been generated."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "from presidio_evaluator.data_generator.extensions import generate_iban, generate_ip_addresses, generate_SSNs, \\\n",
    "    generate_company_names, generate_url, generate_roles, generate_titles, generate_nationality, generate_nation_man, \\\n",
    "    generate_nation_woman, generate_nation_plural, generate_title\n",
    "\n",
    "from presidio_evaluator.data_generator import FakeDataGenerator, read_synth_dataset\n",
    "\n",
    "from collections import Counter\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Evaluate generation logic and the fake PII bank used during generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"../presidio_evaluator/data_generator/raw_data/FakeNameGenerator.com_3000.csv\",encoding=\"utf-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generator = FakeDataGenerator(fake_pii_df=df, \n",
    "                              templates=None, \n",
    "                              dictionary_path=None,\n",
    "                              ignore_types={\"IP_ADDRESS\", 'US_SSN', 'URL','ADDRESS'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pii_df = generator.prep_fake_pii(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for (name, series) in pii_df.iteritems():\n",
    "    print(name)\n",
    "    print(\"Unique values: {}\".format(len(series.unique())))\n",
    "    print(series.value_counts())\n",
    "    print(\"\\n**************\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install wordcloud\n",
    "from wordcloud import WordCloud\n",
    "\n",
    "def series_to_wordcloud(series):\n",
    "    freqs = series.value_counts()\n",
    "    wordcloud = WordCloud(background_color='white',width=800,height=400).generate_from_frequencies(freqs)\n",
    "    fig = plt.figure(figsize=(16, 8))\n",
    "    plt.suptitle(\"{} word cloud\".format(series.name))\n",
    "    plt.imshow(wordcloud, interpolation='bilinear')\n",
    "    plt.axis(\"off\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "series_to_wordcloud(pii_df.FIRST_NAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "series_to_wordcloud(pii_df.LAST_NAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "series_to_wordcloud(pii_df.COUNTRY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "series_to_wordcloud(pii_df.ORGANIZATION)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "series_to_wordcloud(pii_df.CITY)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. Evaluate different entities in the synthetic dataset after creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "synth = read_synth_dataset(\"../data/generated_train_November 12 2019.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences_only = [(sample.full_text,sample.metadata) for sample in synth]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences_only[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Proportions of female vs. male based samples:\")\n",
    "Counter([sentence[1]['Gender'] for sentence in sentences_only])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Proportion of lower case samples:\")\n",
    "Counter([sentence[1]['Lowercase'] for sentence in sentences_only])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Proportion of nameset across samples:\")\n",
    "Counter([sentence[1]['NameSet'] for sentence in sentences_only])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_entity_values_from_sample(sample,entity_types):\n",
    "    name_entities = [span.entity_value for span in sample.spans if span.entity_type in entity_types]\n",
    "    return name_entities\n",
    "    \n",
    "names = [get_entity_values_from_sample(sample,['PERSON','FIRST_NAME','LAST_NAME']) for sample in synth]\n",
    "names = [item for sublist in names for item in sublist]\n",
    "series_to_wordcloud(pd.Series(names,name='PERSON, FIRST_NAME, LAST_NAME'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "countries = [get_entity_values_from_sample(sample,['LOCATION']) for sample in synth]\n",
    "countries = [item for sublist in countries for item in sublist]\n",
    "series_to_wordcloud(pd.Series(countries,name='LOCATION'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "orgs = [get_entity_values_from_sample(sample,['ORGANIZATION']) for sample in synth]\n",
    "orgs = [item for sublist in orgs for item in sublist]\n",
    "series_to_wordcloud(pd.Series(orgs,name='ORGANIZATION'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "presidio-research",
   "language": "python",
   "name": "presidio-research"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}