experiment_analysis/frequency_baseline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import glue_processors\n",
    "from transformers import glue_compute_metrics\n",
    "from collections import Counter\n",
    "from scipy.stats import norm\n",
    "import numpy as np\n",
    "np.random.seed(1337)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "counter_dict = {}\n",
    "for task in [\"CoLA\", \"MNLI\", \"MNLI-mm\", \"MRPC\", \"QNLI\", \"QQP\", \"RTE\", \"SST-2\", \"STS-B\", \"WNLI\"]:\n",
    "    processor = glue_processors[task.lower()]()\n",
    "    data_dir = \"MNLI\" if task.startswith(\"MNLI\") else task\n",
    "    eval_examples = processor.get_dev_examples(f\"../data/glue/{data_dir}\")\n",
    "    if task == \"STS-B\":\n",
    "        items = []\n",
    "        for example in eval_examples:\n",
    "            items.append(float(example.label))\n",
    "        counter_dict[task] = norm.fit(items)\n",
    "    else:\n",
    "        counter = Counter()\n",
    "        for example in eval_examples:\n",
    "            counter[example.label] += 1\n",
    "        counter_dict[task] = counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'CoLA': Counter({'1': 721, '0': 322}),\n",
       " 'MNLI': Counter({'neutral': 3123, 'contradiction': 3213, 'entailment': 3479}),\n",
       " 'MNLI-mm': Counter({'contradiction': 3240,\n",
       "          'entailment': 3463,\n",
       "          'neutral': 3129}),\n",
       " 'MRPC': Counter({'1': 279, '0': 129}),\n",
       " 'QNLI': Counter({'entailment': 2702, 'not_entailment': 2761}),\n",
       " 'QQP': Counter({'0': 25545, '1': 14885}),\n",
       " 'RTE': Counter({'not_entailment': 131, 'entailment': 146}),\n",
       " 'SST-2': Counter({'1': 444, '0': 428}),\n",
       " 'STS-B': (2.3639075555555555, 1.4999854042902065),\n",
       " 'WNLI': Counter({'0': 40, '1': 31})}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counter_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Frequency baseline prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "freq_base_line_prediction = {}\n",
    "for task in counter_dict.keys():\n",
    "    if task != \"STS-B\":\n",
    "        freq_base_line_prediction[task] = counter_dict[task].most_common()[0][0]\n",
    "    else:\n",
    "        freq_base_line_prediction[task] = counter_dict[task]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'CoLA': '1',\n",
       " 'MNLI': 'entailment',\n",
       " 'MNLI-mm': 'entailment',\n",
       " 'MRPC': '1',\n",
       " 'QNLI': 'not_entailment',\n",
       " 'QQP': '0',\n",
       " 'RTE': 'entailment',\n",
       " 'SST-2': '1',\n",
       " 'STS-B': (2.3639075555555555, 1.4999854042902065),\n",
       " 'WNLI': '0'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "freq_base_line_prediction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Frequency baseline prediction Benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "freq_baseline = {}\n",
    "for task in freq_base_line_prediction.keys():\n",
    "    processor = glue_processors[task.lower()]()\n",
    "    data_dir = \"MNLI\" if task.startswith(\"MNLI\") else task\n",
    "    eval_examples = processor.get_dev_examples(f\"../data/glue/{data_dir}\")\n",
    "    label_list = processor.get_labels()\n",
    "    prediction = freq_base_line_prediction[task]\n",
    "    if task == \"STS-B\":\n",
    "        labels = []\n",
    "        predictions = np.random.normal(prediction[0], prediction[1], len(eval_examples))\n",
    "        for example in eval_examples:\n",
    "            labels.append(float(example.label))\n",
    "        labels = np.array(labels)\n",
    "    else:\n",
    "        labels = []\n",
    "        predictions = []\n",
    "        for example in eval_examples:\n",
    "            labels.append(label_list.index(example.label))\n",
    "            predictions.append(label_list.index(prediction))\n",
    "        predictions = np.array(predictions)\n",
    "        labels = np.array(labels)\n",
    "    results = glue_compute_metrics(task.lower(), predictions, labels)\n",
    "    if task.startswith(\"MNLI\"):\n",
    "        if \"MNLI\" not in freq_baseline:\n",
    "            freq_baseline[\"MNLI\"] = {}\n",
    "        for k, v in results.items():\n",
    "            freq_baseline[\"MNLI\"][f\"{task.lower()}_{k}\"] = v\n",
    "    else:\n",
    "        freq_baseline[task] = results\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'CoLA': {'mcc': 0.0},\n",
       " 'MNLI': {'mnli_acc': 0.3544574630667346, 'mnli-mm_acc': 0.3522172497965826},\n",
       " 'MRPC': {'acc': 0.6838235294117647,\n",
       "  'f1': 0.8122270742358079,\n",
       "  'acc_and_f1': 0.7480253018237863},\n",
       " 'QNLI': {'acc': 0.5053999633900788},\n",
       " 'QQP': {'acc': 0.6318327974276527,\n",
       "  'f1': 0.0,\n",
       "  'acc_and_f1': 0.3159163987138264},\n",
       " 'RTE': {'acc': 0.5270758122743683},\n",
       " 'SST-2': {'acc': 0.5091743119266054},\n",
       " 'STS-B': {'pearson': 0.02425364185684005,\n",
       "  'spearmanr': 0.020099238016388975,\n",
       "  'corr': 0.022176439936614514},\n",
       " 'WNLI': {'acc': 0.5633802816901409}}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "freq_baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "def write_results(results, output_file_path):\n",
    "    with open(output_file_path, \"w\") as fp:\n",
    "        json.dump(results, fp, indent=4, sort_keys=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir -p ../experiments/freq_baseline\n",
    "write_results(freq_baseline, \"../experiments/freq_baseline/results.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "file_extension": ".py",
  "kernelspec": {
   "display_name": "Python 3.7.7 64-bit",
   "language": "python",
   "name": "python37764bit04702c7235a6448a9727054baaa32e98"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "mimetype": "text/x-python",
  "name": "python",
  "npconvert_exporter": "python",
  "pygments_lexer": "ipython3",
  "version": 3
 },
 "nbformat": 4,
 "nbformat_minor": 4
}