material/notebooks/Session_1.2_Solution.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Word Embedding Association Tests (WEAT)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we will implement and apply the WEAT to evaluate the indirect bias of our word embeddings. You can either evaluate the standard glove 50 dimensinal embeddings provided by the authors of the glove paper or your own embeddings (e.g. from Session_1.3).  \n",
    "Before you can apply WEAT you need to complete the function `word_attribute_association`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install git+https://github.com/millawell/bias-ml-dh.git#subdirectory=material/notebooks/bias_ml_dh_utils\n",
    "!pip install --upgrade tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "#WEAT Word Embedding Association Tests\n",
    "import torch as tr\n",
    "import numpy as np\n",
    "import spacy\n",
    "from tqdm.notebook import tqdm\n",
    "import itertools\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "tokenizer = nlp.Defaults.create_tokenizer(nlp)\n",
    "import bias_ml_dh_utils as utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_dim = 50\n",
    "embedding_identifier = \"glove.6B.{}d\".format(embedding_dim)\n",
    "\n",
    "embedding_path = utils.download_dataset(embedding_identifier)\n",
    "\n",
    "embedding_matrix, vocab = utils.create_embedding_matrix(embedding_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def word_attribute_association(w, A, B, vocab):\n",
    "    #s(w,A,B) = mean_a cos(w,a) - mean_b cos(w,b)\n",
    "    A_embed = utils.lookup_embeddings(A, vocab, embedding_matrix)\n",
    "    B_embed = utils.lookup_embeddings(B, vocab, embedding_matrix)\n",
    "    w_embed = utils.lookup_embeddings(w,vocab, embedding_matrix)\n",
    "    \n",
    "    wA = np.dot(w_embed/np.linalg.norm(w_embed, axis=1)[:,np.newaxis],\n",
    "                (A_embed/np.linalg.norm(A_embed, axis=1)[:,np.newaxis]).T).sum()\n",
    "    wB = np.dot(w_embed/np.linalg.norm(w_embed, axis=1)[:,np.newaxis],\n",
    "                (B_embed/np.linalg.norm(B_embed, axis=1)[:,np.newaxis]).T).sum()\n",
    "    \n",
    "    return wA/len(A) -  wB/len(B)\n",
    "\n",
    "def test_statistic(A,B,X,Y, vocab):\n",
    "    \n",
    "    wA = 0\n",
    "    \n",
    "    for ix in X:\n",
    "        wA += word_attribute_association([ix], A, B, vocab)\n",
    "        \n",
    "    wB = 0\n",
    "    \n",
    "    for iy in Y:\n",
    "        wB -= word_attribute_association([iy], A, B, vocab)\n",
    "        \n",
    "    return wA+wB\n",
    "\n",
    "def calculate_pvalue(A,B,X,Y,vocab,alpha=0.05):\n",
    "    \n",
    "    #check out-of-vocab words\n",
    "    A = list(set(A).intersection(vocab))\n",
    "    B = list(set(B).intersection(vocab))\n",
    "    X = list(set(X).intersection(vocab))\n",
    "    Y = list(set(Y).intersection(vocab))\n",
    "        \n",
    "    \n",
    "    test_stat_orig = test_statistic(A,B,X,Y,vocab)\n",
    "    \n",
    "    union = set(X+Y)\n",
    "    subset_size = len(union)//2\n",
    "    \n",
    "    larger = 0\n",
    "    total = 0\n",
    "    \n",
    "    for subset in tqdm(set(itertools.combinations(union, subset_size))):\n",
    "        total += 1\n",
    "        Xi = list(set(subset))\n",
    "        Yi = list(union - set(subset))\n",
    "        if test_statistic(A, B, Xi, Yi,vocab) > test_stat_orig:\n",
    "            larger += 1\n",
    "    if larger/float(total)<alpha:\n",
    "        print(\"The difference between the attributes {} and {} \\nfor the given target words is significant.\".format(A,B))\n",
    "    else:\n",
    "        print(\"The difference between the attributes {} and {} \\nfor the given target words is not significant.\".format(A,B))\n",
    "\n",
    "    return larger/float(total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "############################################################################\n",
    "#We use a one-sided test, therefore it is not symmetric. \n",
    "#The target words you assume are more associated with A should be in X \n",
    "#and respectively the target words for B should be in Y\n",
    "############################################################################\n",
    "#choose your attributes in A and B\n",
    "A = ['female', 'woman']\n",
    "B = ['male', 'man']\n",
    "\n",
    "#choose your target words in X and Y\n",
    "X = ['home', 'parents', 'children', 'family', 'cousins', 'marriage', 'wedding', 'relatives']\n",
    "# X = ['nurse','teacher','librarian']\n",
    "Y = ['executive', 'management', 'professional', 'corporation', 'salary', 'office', 'business', 'career']\n",
    "# Y = ['programmer','engineer','scientist']\n",
    "\n",
    "p = calculate_pvalue(A,B,X,Y, vocab)\n",
    "print(p)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}