generate_embedding_measures.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# imports\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from numpy import linalg as LA\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.decomposition import PCA\n",
    "from joblib import Parallel, delayed\n",
    "from glob import glob\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Put together list of identities\n",
    "\n",
    "Note, we focus in the paper only on the identities in our survey (below), but we include identities from a bunch of other papers we consider in case others want to explore those results\n",
    "\n",
    "Also, we use \"policeman\" for \"police_officer\".  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "identities = {'adult', 'american', 'arab', 'asian', 'banker', 'black', 'boss',\n",
    "       'boy', 'brother', 'bully', 'child', 'coach', 'conservative',\n",
    "       'consultant', 'cop', 'criminal', 'daughter', 'democrat', 'dentist',\n",
    "       'doctor', 'executive', 'father', 'girl', 'goon', 'guy', 'hispanic',\n",
    "       'idiot', 'intern', 'jerk', 'judge', 'kid', 'lady', 'lawyer',\n",
    "       'liberal', 'man', 'minor', 'mother', 'nurse', 'patient',\n",
    "       'physician', 'police_officer', 'politician', 'principal', 'punk',\n",
    "       'republican', 'scientist', 'secretary', 'senator', 'sister', 'son',\n",
    "       'surgeon', 'teenager', 'thug', 'toddler', 'white', 'witness',\n",
    "       'woman'}\n",
    "identities.remove(\"police_officer\")\n",
    "identities.add('policeman')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "uga_identities = [x.strip() for x in open(\"./uga_identities.txt\")]\n",
    "identities |= set(uga_identities)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "uga_identities = [x.strip() for x in open(\"./personality_identities.txt\")]\n",
    "identities |= set(uga_identities)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "garg_dat = pd.read_csv(\"mturk_stereotypes.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "identities |= set(garg_dat.occupation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1371"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(identities)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate dimension & word position measurement combinations\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "measures = [{\n",
    "\"names\" : ['bad','good'],\n",
    "\"group\": \"evaluation\",\n",
    "\"sets\": [\n",
    "['bad', 'awful'],\n",
    "['good', 'nice']\n",
    "],\n",
    "\"paper\": \"this_short\",\n",
    "\"is_paired\" : True\n",
    "},\n",
    "\n",
    "{\n",
    "\"group\": \"evaluation\",\n",
    "\"names\" : ['bad','good'],\n",
    "\"sets\": [\n",
    "['bad', 'awful', 'negative', 'terrible', 'worse', 'horrible'],\n",
    "['good', 'nice', 'positive', 'great', 'better', 'awesome']\n",
    "],\n",
    "\"paper\": \"this_long\",\n",
    "\"is_paired\" : True\n",
    "},\n",
    "\n",
    "{\n",
    "\"group\": \"potency\",\n",
    "\"names\" : ['weak','strong'],\n",
    "\"sets\": [\n",
    "['powerless', 'little'],\n",
    "['powerful', 'big']\n",
    "],\n",
    "\"paper\": \"this_short\",\n",
    "\"is_paired\" : True\n",
    "},\n",
    "\n",
    "{\n",
    "\"group\": \"potency\",\n",
    "\"names\" : ['weak','strong'],\n",
    "\"sets\": [\n",
    "['powerless', 'little','weak','impotent','dominant'],\n",
    "['powerful', 'big','strong','potent','feeble']\n",
    "],\n",
    "\"paper\": \"this_long\",\n",
    "\"is_paired\" : True\n",
    "},\n",
    "\n",
    "{\n",
    "\"group\": \"activity\",\n",
    "\"names\" : ['inactive','active'],\n",
    "\"sets\": [\n",
    "['slow', 'quiet', 'inactive'],\n",
    "['fast', 'noisy', 'active']\n",
    "],\n",
    "\"paper\": \"this_short\",\n",
    "\"is_paired\" : True\n",
    "},\n",
    "\n",
    "{\n",
    "\"group\": \"activity\",\n",
    "\"names\" : ['inactive','active'],\n",
    "\"sets\": [\n",
    "['slow', 'quiet', 'inactive', 'dead',  'die', 'stopped'],\n",
    "['fast', 'noisy', 'active', 'alive', 'live', 'moving']\n",
    "],\n",
    "\"paper\": \"this_long\",\n",
    "\"is_paired\" : True\n",
    "},\n",
    "\n",
    "{\n",
    "\"group\": \"age\",\n",
    "\"names\" : ['young','old'],\n",
    "\"sets\": [\n",
    "['young'],\n",
    "['old']\n",
    "],\n",
    "\"paper\": \"this_short\",\n",
    "\"is_paired\" : True\n",
    "},\n",
    "\n",
    "{\n",
    "\"group\": \"age\",\n",
    "\"names\" : ['young','old'],\n",
    "\"sets\": [\n",
    "['young', 'new', 'youthful', 'young'],\n",
    "['old', 'old', 'elderly', 'aged']\n",
    "],\n",
    "\"paper\": \"this_long\",\n",
    "\"is_paired\" : True\n",
    "},\n",
    "\n",
    "{\n",
    "\"group\": \"gender\",\n",
    "\"names\" : ['woman','man'],\n",
    "\"sets\": [\n",
    "['female'],\n",
    "['male']\n",
    "],\n",
    "\"paper\": \"this_short\",\n",
    "\"is_paired\" : True\n",
    "},\n",
    "\n",
    "{\n",
    "\"group\": \"gender\",\n",
    "\"names\" : ['woman','man'],\n",
    "\"sets\": [\n",
    "['she'],\n",
    "['he']\n",
    "],\n",
    "\"paper\": \"gonen\",\n",
    "\"is_paired\" : True\n",
    "},\n",
    "\n",
    "{\n",
    "\"group\": \"gender\",\n",
    "\"names\" : ['woman','man'],\n",
    "\"sets\": [\n",
    "['woman', 'girl', 'she', 'mother', 'daughter', 'gal', 'female', 'her', 'herself', 'Mary'],\n",
    "['man', 'boy', 'he', 'father', 'son', 'guy', 'male', 'his', 'himself', 'John']\n",
    "],\n",
    "\"paper\": \"bolukbasi_words\",\n",
    "\"is_paired\" : True\n",
    "},\n",
    "\n",
    "{\n",
    "\"group\": \"gender\",\n",
    "\"names\" : ['woman','man'],\n",
    "\"sets\": [\n",
    "['amy', 'joan', 'lisa', 'sarah', 'diana', 'kate', 'ann', 'donna'],\n",
    "['john', 'paul', 'mike', 'kevin', 'steve', 'greg', 'jeff', 'bill']\n",
    "],\n",
    "\"paper\": \"bolukbasi_names\",\n",
    "\"is_paired\" : True\n",
    "},\n",
    "\n",
    "\n",
    "{\n",
    "\"group\": \"gender\",\n",
    "\"names\" : ['woman','man'],\n",
    "\"sets\": [\n",
    "['woman', 'women', 'she', 'her', 'her', 'hers', 'girl', 'girls', 'female', 'feminine'],\n",
    "['man', 'men', 'he', 'him', 'his', 'his', 'boy', 'boys', 'male', 'masculine']\n",
    "],\n",
    "\"paper\": \"kozlowski\",\n",
    "\"is_paired\" : True\n",
    "}\n",
    "]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "institutions = ['family','politics','justice','medicine','business','education','religion']\n",
    "race_ethnicities = ['white','latino','asian','middle eastern','black'] \n",
    "\n",
    "measures += [\n",
    "{ \"sets\" : [[x] for x in institutions], \n",
    "  \"names\":institutions, \n",
    "  \"group\" :\"instituiton\",\n",
    "  \"paper\" : \"this_short\",\n",
    "\"is_paired\" : False},\n",
    "\n",
    "{ \"sets\" : [\n",
    "    ['family','kinsperson', 'household', 'kinfolk', 'home', 'kinsfolk', 'kin'],\n",
    "    ['politics', 'government', 'political'],\n",
    "    ['justice', 'law', 'legal', 'illegal'],\n",
    "    ['medicine', 'medical', 'prescription', 'medication'],\n",
    "    ['business', 'job', 'profession', 'career', 'employment', 'occupation', 'clientele'],\n",
    "    ['education', 'school', 'university', 'instruction', 'teaching'],\n",
    "    ['religion', 'faith', 'worship']], \n",
    "  \"names\":institutions, \n",
    "  \"group\" :\"instituiton\",\n",
    "  \"paper\" : \"this_long\",\n",
    "\"is_paired\" : False},\n",
    "\n",
    "{ \"sets\" : [['white','White'],\n",
    "             ['hispanic','Hispanic','latino','Latino'],\n",
    "             ['asian','Asian'],\n",
    "             ['arab','Arab'],\n",
    "             ['black','Black']],\n",
    "  \"names\":race_ethnicities, \n",
    "  \"group\" :\"race_ethnicity\",\n",
    "  \"paper\" : \"this_short\",\n",
    "\"is_paired\" : False},\n",
    "    \n",
    "{ \"sets\" : [ ['white',    'whites',    'White',   'Whites',   'European', 'Caucasian','Anglo'],\n",
    "             ['hispanic', 'hispanics', 'Hispanic','Hispanics','Mexican',  'Latino',   'Hispanic'],\n",
    "             ['asian',    'asians',    'Asian',   'Asians',   'Chinese',  'Japanese', 'Korean'],\n",
    "             ['arab',     'arabs',     'Arab',    'Arabs',    'Muslim',   'Muslim',   'Muslim'],\n",
    "             ['black',    'blacks',    'Black',   'Blacks',   'African',  'African',  'Afro']\n",
    "            ],\n",
    "  \"names\":race_ethnicities, \n",
    " \"group\" :\"race_ethnicity\",\n",
    "  \"paper\" : \"koslowski\",\n",
    "\"is_paired\" : False},   \n",
    "    \n",
    "{ \"sets\" : [ ['white',    'whites',   'european', 'caucasian','anglo'],\n",
    "             ['hispanic', 'hispanics','mexican',  'latino',   'hispanic'],\n",
    "             ['asian',    'asians',   'chinese',  'japanese', 'korean'],\n",
    "             ['arab',     'arabs',    'muslims',   'muslim',   'arab'],\n",
    "             ['black',    'blacks',   'african',  'african',  'afro']\n",
    "            ],\n",
    "  \"names\":race_ethnicities, \n",
    " \"group\" :\"race_ethnicity\",\n",
    "  \"paper\" : \"koslowski_lowercase\",\n",
    "\"is_paired\" : False},   \n",
    "\n",
    "{ \"sets\" : [ ['white',    'whites',    'White',   'Whites',   'European', 'Caucasian','Anglo'],\n",
    "             ['hispanic', 'hispanics', 'Hispanic','Hispanics','Mexican',  'latino','latinos','latina','latinas','Latino','Latinos','Latina','Latinas'],\n",
    "             ['asian',    'asians',    'Asian',   'Asians',   'Chinese',  'Japanese','Korean'],\n",
    "             ['arab',     'arabs',     'Arab',    'Arabs',    'Muslim',   'Muslims','Muslim','Muslims'],\n",
    "             ['black',    'blacks',    'Black',   'Blacks',   'African',  'African',]\n",
    "            ],\n",
    "  \"names\":race_ethnicities, \n",
    " \"group\" :\"race_ethnicity\",\n",
    "  \"paper\" : \"this_long\",\n",
    "\"is_paired\" : False}   \n",
    "]\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# From Personality Paper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "measures += [{\n",
    "\"group\": \"openness\",\n",
    "\"names\" : ['open','closed'],\n",
    "\"sets\": [\n",
    "[\"intelligent\", \"perceptive\", \"analytical\", \"reflective\", \"curious\", \"imaginative\", \"creative\", \"cultured\", \"refined\", \"sophisticated\"],\n",
    "[\"unintelligent\", \"imperceptive\", \"unanalytical\", \"unreflective\", \"uninquisitive\", \"unimaginative\", \"uncreative\", \"uncultured\", \"unrefined\", \"unsophisticated\"]\n",
    "],\n",
    "\"paper\": \"agarwal\",\n",
    "\"is_paired\" : True\n",
    "},\n",
    "    \n",
    " {\n",
    "\"group\": \"conscientiousness\",\n",
    "\"names\" : ['conscientious','unconscientious'],\n",
    "\"sets\": [\n",
    "[\"organized\", \"responsible\", \"reliable\", \"conscientious\", \"practical\", \"thorough\", \"hardworking\", \"thrifty\", \"cautious\", \"serious\"],\n",
    "[\"disorganized\", \"irresponsible\", \"undependable\", \"negligent\", \"impractical\", \"careless\", \"lazy\", \"extravagant\", \"rash\", \"frivolous\"],\n",
    "],\n",
    "\"paper\": \"agarwal\",\n",
    "\"is_paired\" : True\n",
    "},\n",
    "    \n",
    "{\n",
    "\"group\": \"extroversion\",\n",
    "\"names\" : ['extroverted','introverted'],\n",
    "\"sets\": [\n",
    "[\"extroverted\", \"energetic\", \"talkative\", \"enthusiastic\", \"bold\", \"active\", \"spontaneous\", \"assertive\", \"adventurous\", \"sociable\"],\n",
    "[\"introverted\", \"unenergetic\", \"silent\", \"unenthusiastic\", \"timid\", \"inactive\", \"inhibited\", \"unassertive\", \"unadventurous\", \"unsociable\"],\n",
    "],\n",
    "\"paper\": \"agarwal\",\n",
    "\"is_paired\" : True\n",
    "},\n",
    "    \n",
    "{\n",
    "\"group\": \"agreeableness\",\n",
    "\"names\" : ['agreeable','unagreeable'],\n",
    "\"sets\": [\n",
    "[\"warm\", \"kind\", \"cooperative\", \"unselfish\", \"polite\", \"agreeable\", \"trustful\", \"generous\", \"flexible\", \"fair\"],\n",
    "[\"cold\", \"unkind\", \"uncooperative\", \"selfish\", \"rude\", \"disagreeable\", \"distrustful\", \"stingy\", \"inflexible\", \"unfair\"],\n",
    "],\n",
    "\"paper\": \"agarwal\",\n",
    "\"is_paired\" : True\n",
    "},\n",
    "    \n",
    "{\n",
    "\"group\": \"neuroticism\",\n",
    "\"names\" : ['neurotic','calm'],\n",
    "\"sets\": [\n",
    "[\"angry\", \"tense\", \"nervous\", \"envious\", \"unstable\", \"discontented\", \"insecure\", \"emotional\", \"guilt-ridden\", \"moody\"],\n",
    "[\"calm\", \"relaxed\", \"at ease\", \"not envious\", \"stable\", \"contended\", \"secure\", \"unemotional\", \"guilt-free\", \"steady\"],\n",
    "],\n",
    "\"paper\": \"agarwal\",\n",
    "\"is_paired\" : True\n",
    "}]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# From Kozlowski Paper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "measures += [\n",
    "{\n",
    "\"group\": \"cultivation\",\n",
    "\"names\" : ['uncultivated','cultivated'],\n",
    "\"sets\": [\n",
    "['uncultivated', 'uncultured', 'uncivilized', 'discourteous', 'improper', 'rude', 'uncordial', 'informal', 'uncourtly', 'boorish', 'unpolished', 'unrefined', 'incivility', 'uncivil', 'boorishness', 'rudeness', 'loutish', 'unmannerly', 'gruff', 'ungracious', 'unobliging', 'uncultured', 'ungenteel', 'unmannered', 'blunt'],\n",
    "['cultivated', 'cultured', 'civilized', 'courteous', 'proper', 'polite', 'cordial', 'formal', 'courtly', 'urbane', 'polished', 'refined', 'civility', 'civil', 'urbanity', 'politesse', 'edified', 'mannerly', 'polished', 'gracious', 'obliging', 'cultured', 'genteel', 'mannered', 'polite']\n",
    "],\n",
    "\"paper\": \"kozlowski\",\n",
    "\"is_paired\" : True\n",
    "},\n",
    "\n",
    "{\n",
    "\"group\": \"employment\",\n",
    "\"names\" : ['employee','employer'],\n",
    "\"sets\": [\n",
    "['employee', 'employees', 'worker', 'worker', 'laborer', 'laborers', 'employee', 'employees', 'proletarian', 'proletariat', 'staff', 'staff', 'employee', 'employees', 'worker', 'workers', 'laborer', 'laborers', 'staff', 'staff'],\n",
    "['employer', 'employers', 'owner', 'owners', 'industrialist', 'industrialists', 'proprietor', 'proprietors', 'capitalist', 'capitalists', 'manager', 'managers', 'director', 'directors', 'boss', 'bosses', 'foreman', 'foremen', 'supervisor', 'superintendent']\n",
    "],\n",
    "\"paper\": \"kozlowski\",\n",
    "\"is_paired\" : True\n",
    "},\n",
    "\n",
    "{\n",
    "\"group\": \"education\",\n",
    "\"names\" : ['uneducated','educated'],\n",
    "\"sets\": [\n",
    "['uneducated', 'unlearned', 'ignorant', 'untrained', 'untaught', 'illiterate', 'unschooled', 'untutored', 'unlettered'],\n",
    "['educated', 'learned', 'knowledgeable', 'trained', 'taught', 'literate', 'schooled', 'tutored', 'lettered']\n",
    "],\n",
    "\"paper\": \"kozlowski\",\n",
    "\"is_paired\" : True\n",
    "},\n",
    "\n",
    "{\n",
    "\"group\": \"status\",\n",
    "\"names\" : ['low_status','high_status'],\n",
    "\"sets\": [\n",
    "['unprestigious', 'dishonorable', 'lowly', 'uninfluential', 'disreputable', 'commonplace', 'mundane', 'humble', 'prosaic', 'modest', 'commoner', 'unpretentious', 'ordinary', 'lowly', 'common'],\n",
    "['prestigious', 'honorable', 'esteemed', 'influential', 'reputable', 'distinguished', 'eminent', 'illustrious', 'renowned', 'acclaimed', 'dignitary', 'venerable', 'exalted', 'estimable', 'prominent']\n",
    "],\n",
    "\"paper\": \"kozlowski\",\n",
    "\"is_paired\" : True\n",
    "},\n",
    "\n",
    "{\n",
    "\"group\": \"morality\",\n",
    "\"names\" : ['immoral','moral'],\n",
    "\"sets\": [\n",
    "['evil', 'immoral', 'bad', 'dishonest', 'sinful', 'vice', 'wicked', 'transgressive', 'unprincipled', 'questionable', 'nefarious', 'corrupt', 'unscrupulous', 'selfish', 'knavish', 'crooked', 'reprehensible', 'impure', 'undignified', 'unholy', 'fiendish', 'villainous', 'guilty', 'indecent', 'unsavory', 'odious', 'unethical'],\n",
    "['good', 'moral', 'good', 'honest', 'virtuous', 'virtue', 'righteous', 'chaste', 'principled', 'unquestionable', 'noble', 'uncorrupt', 'scrupulous', 'altruistic', 'chivalrous', 'honest', 'commendable', 'pure', 'dignified', 'holy', 'valiant', 'upstanding', 'guiltless', 'decent', 'chaste', 'righteous', 'ethical']\n",
    "],\n",
    "\"paper\": \"kozlowski\",\n",
    "\"is_paired\" : True\n",
    "},\n",
    "{\n",
    "\"group\": \"affluence\",\n",
    "\"names\" : ['poor','rich'],\n",
    "\"sets\": [\n",
    "['poor', 'poorer', 'poorest', 'poverty', 'destitute', 'needy', 'impoverished', 'economical', 'impecunious', 'inexpensive', 'ruined', 'necessitous', 'skint', 'cheap', 'economical', 'penurious', 'threadbare', 'cheap', 'unmonied', 'indigent', 'threadbare', 'penurious', 'cheap', 'worthless', 'underprivileged', 'bankrupt', 'unprosperous', 'underdeveloped', 'insolvency', 'unsuccessful', 'plain', 'basic', 'disadvantaged', 'squalid', 'valueless', 'beggarly', 'ramshackle', 'indigence', 'insolvent', 'moneyless', 'penniless', 'penury', 'plain', 'indigence'],\n",
    "['rich', 'richer', 'richest', 'affluence', 'affluent', 'advantaged', 'wealthy', 'costly', 'exorbitant', 'expensive', 'exquisite', 'extravagant', 'flush', 'invaluable', 'lavish', 'luxuriant', 'luxurious', 'luxury', 'moneyed', 'opulent', 'plush', 'luxuriant', 'precious', 'priceless', 'privileged', 'propertied', 'prosperous', 'developed', 'solvency', 'successful', 'sumptuous', 'swanky', 'thriving', 'upscale', 'valuable', 'classy', 'ritzy', 'opulence', 'solvent', 'moneyed', 'rich', 'affluence', 'posh', 'opulence']\n",
    "],\n",
    "\"paper\": \"kozlowski\",\n",
    "\"is_paired\" : True\n",
    "}\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# From Garg Paper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "measures.append({\n",
    "\"group\": \"gender\",\n",
    "\"names\" : ['woman','man'],\n",
    "\"sets\": [\n",
    "[\"she\", \"daughter\", \"hers\", \"her\", \"mother\", \"woman\", \"girl\", \"herself\", \"female\", \"sister\", \"daughters\", \"mothers\", \"women\",\n",
    "\"girls\", \"femen\", \"sisters\", \"aunt\", \"aunts\", \"niece\", \"nieces\"],\n",
    "[\"he\", \"son\", \"his\", \"him\", \"father\", \"man\", \"boy\", \"himself\", \"male\", \"brother\", \"sons\", \"fathers\", \"men\", \"boys\", \"males\", \"brothers\", \"uncle\",\n",
    "\"uncles\", \"nephew\", \"nephews\"]],\n",
    "\"paper\": \"garg\",\n",
    "\"is_paired\" : True\n",
    "})\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i, m in enumerate(measures):\n",
    "    m['ind'] = i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for m in measures:\n",
    "    assert(len(m['names']) == len(m['sets']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "57928\n"
     ]
    }
   ],
   "source": [
    "all_emb_vocab = None\n",
    "for v in glob(\"data/embeddings/*.vocab\"):\n",
    "    if all_emb_vocab:\n",
    "        all_emb_vocab &= {line.strip() for line in open(v)}\n",
    "    else:\n",
    "        all_emb_vocab = {line.strip() for line in open(v)}\n",
    "print(len(all_emb_vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "488 identities not in vocab\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['', 'Dutch', 'speech language pathologist', 'malingerer', 'North Korean']"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "identities_not_in_vocab = [identity for identity in identities if identity not in all_emb_vocab]\n",
    "print(\"{} identities not in vocab\".format(len(identities_not_in_vocab)))\n",
    "identities_not_in_vocab[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# check all measures have at least one identity in vocab\n",
    "for m in measures:\n",
    "    for s in m['sets']:\n",
    "        k = 0\n",
    "        for w in s:\n",
    "            if w not in all_emb_vocab:\n",
    "                pass\n",
    "            else:\n",
    "                k += 1\n",
    "        assert(k > 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def normalize(wv):\n",
    "    # normalize vectors\n",
    "    norms = np.apply_along_axis(LA.norm, 1, wv)\n",
    "    wv = wv / norms[:, np.newaxis]\n",
    "    return wv\n",
    "\n",
    "def load_embeddings_from_np(filename):\n",
    "    with open(filename + '.vocab', 'r') as f_embed:\n",
    "        vocab = [line.strip() for line in f_embed]\n",
    "        \n",
    "    w2i = {w: i for i, w in enumerate(vocab)}\n",
    "    wv = np.load(filename + '.wv.npy')\n",
    "\n",
    "    return vocab, wv, w2i\n",
    "\n",
    "\n",
    "def load_wo_normalize(space, filename, vocab, wv, w2i):\n",
    "    vocab_muse, wv_muse, w2i_muse = load_embeddings_from_np(filename)\n",
    "    vocab[space] = vocab_muse \n",
    "    wv[space] = wv_muse\n",
    "    w2i[space] = w2i_muse\n",
    "    \n",
    "def my_cosine(x,y):\n",
    "    return np.dot(x,y.T)/(LA.norm(x)*LA.norm(y))\n",
    "\n",
    "def ripa(w,b):\n",
    "    return w.dot(b)/LA.norm(b)\n",
    "\n",
    "def garg(w,b):\n",
    "    return - LA.norm(w-b)\n",
    "\n",
    "def kozlowski_b(setv,  wv, w2i):\n",
    "    return np.mean([wv[w2i[y]] - wv[w2i[x]] for x, y in setv],axis=0)\n",
    "    \n",
    "def bolukbasi_b(setv,  wv, w2i):\n",
    "    if len(setv) == 1:\n",
    "        # don't run PCA if there's only a single direction\n",
    "        return wv[w2i[setv[0][0]]] - wv[w2i[setv[0][1]]]\n",
    "\n",
    "    matrix = []\n",
    "    for x,y in setv:\n",
    "        center = (wv[w2i[x]] + wv[w2i[y]])/2\n",
    "        matrix.append(wv[w2i[x]] - center)\n",
    "        matrix.append(wv[w2i[y]] - center)\n",
    "    matrix = np.array(matrix)\n",
    "    pca = PCA(n_components = min(matrix.shape[0],10))\n",
    "    return pca.fit(matrix).components_[0]\n",
    "\n",
    "def ripa_kozlowski_fun(setv, identities, norm_wv, wv, w2i):\n",
    "    b = kozlowski_b(setv, wv, w2i)\n",
    "    for identity in identities:\n",
    "        yield  identity, ripa(wv[w2i[identity]], b)\n",
    "\n",
    "def ripa_bolukbasi_fun(setv, identities, norm_wv, wv, w2i):\n",
    "    b = bolukbasi_b(setv, wv, w2i)\n",
    "    for identity in identities:\n",
    "        yield identity, ripa(wv[w2i[identity]], b)\n",
    "\n",
    "def kozlowski_fun(setv, identities, norm_wv, wv, w2i):\n",
    "    b = kozlowski_b(setv, norm_wv, w2i)\n",
    "    for identity in identities:\n",
    "        yield identity, my_cosine(norm_wv[w2i[identity]],b)\n",
    "\n",
    "def bolukbasi_fun(setv, identities, norm_wv, wv, w2i):\n",
    "    b = bolukbasi_b(setv, norm_wv, w2i)\n",
    "    for identity in identities:\n",
    "        yield identity, my_cosine(norm_wv[w2i[identity]], b)\n",
    "        \n",
    "def caliksan_fun(setv, identities, norm_wv, wv, w2i):\n",
    "    for identity in identities:\n",
    "        x_vals = [my_cosine(norm_wv[w2i[identity]],norm_wv[w2i[x]]) for x,y in setv]\n",
    "        y_vals = [my_cosine(norm_wv[w2i[identity]],norm_wv[w2i[y]]) for x,y in setv]\n",
    "        yield identity, (np.mean(x_vals)-np.mean(y_vals))/np.std(x_vals+y_vals)\n",
    "                            \n",
    "\n",
    "def group_fun(names, sets, identities, wv, w2i, measure):\n",
    "     # create mean vectors for each group\n",
    "    group_mean_vecs = {}\n",
    "    for i, setv in enumerate(sets):\n",
    "        v = [wv[w2i[x]] for x in setv if x in w2i]\n",
    "        group_mean_vecs[names[i]] = np.mean(v,axis=0)\n",
    "        \n",
    "    for identity in identities:\n",
    "        identity_vec = wv[w2i[identity]]\n",
    "        for i, group_name in enumerate(names):\n",
    "            group_dist = measure(group_mean_vecs[group_name], identity_vec)\n",
    "            other_group_dist = sum([measure(group_mean_vecs[other_group_name], identity_vec) \n",
    "                                    for other_group_name in names if group_name != other_group_name])\n",
    "            other_group_dist /= float(len(names)-1)\n",
    "        \n",
    "            yield (group_name, measure.__name__, identity, group_dist - other_group_dist )\n",
    "\n",
    "\n",
    "def garg_fun(names, sets, identities, norm_wv, wv, w2i):\n",
    "    return group_fun(names, sets, identities, norm_wv, w2i, garg)\n",
    "\n",
    "def ripa_fun(names, sets, identities, norm_wv, wv, w2i):\n",
    "    return group_fun(names, sets, identities, wv,      w2i, ripa)\n",
    "\n",
    "\n",
    "def swinger_fun(names, sets, identities, norm_wv, wv, w2i):\n",
    "    mean_identity_vec = np.mean([norm_wv[w2i[i]] for i in identities], axis=0)\n",
    "    \n",
    "    group_mean_vecs = {}\n",
    "    for i, setv in enumerate(sets):\n",
    "        v = [norm_wv[w2i[x]] for x in setv if x in w2i]\n",
    "        group_mean_vecs[names[i]] = np.mean(v,axis=0)\n",
    "    universal_group_mean = np.mean([x for x in group_mean_vecs.values()], axis=0)\n",
    "    \n",
    "    for group_name, group_vec in group_mean_vecs.items():\n",
    "        for identity in identities:\n",
    "            v = np.dot( (group_vec - universal_group_mean), (norm_wv[w2i[identity]] - mean_identity_vec) )\n",
    "            yield group_name, 'swinger', identity, v\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def run_measures(f, measures, identities):\n",
    "    res = []\n",
    "    vocab = {}\n",
    "    wv = {}\n",
    "    w2i = {}\n",
    "\n",
    "    embedding = os.path.basename(f)[:-7]\n",
    "    print(embedding)\n",
    "    f = f[:-7]\n",
    "\n",
    "    # load unnormalized for ripa\n",
    "    load_wo_normalize(embedding,f,vocab,wv,w2i)\n",
    "    emb_vocab = vocab[embedding]\n",
    "    emb_wv = wv[embedding]\n",
    "    emb_w2i = w2i[embedding]\n",
    "    \n",
    "    if 'gn_glove' in f:\n",
    "        emb_wv = emb_wv[:,:-1]\n",
    "    \n",
    "    # load normalized for other methods\n",
    "    norm_emb_wv = normalize(emb_wv)\n",
    "    \n",
    "    identities = [i for i in identities if i in emb_w2i]\n",
    "    \n",
    "    for measure in measures:\n",
    "        print(measure['ind'])\n",
    "        \n",
    "        tmp_sets = measure['sets']\n",
    "        names = measure['names']\n",
    "        measure_ind = measure['ind']\n",
    "        \n",
    "        # created paired data for the pairwise metrics,\n",
    "        # assuming there is a default identity (dimension)\n",
    "        pair_sets = []\n",
    "        \n",
    "        # For the default, compare to the culturally-defined opposite (ensured by ordering of measure)\n",
    "        dlen = min(len(tmp_sets[0]), len(tmp_sets[1]))\n",
    "        # if paired, we have to make sure that we have all the words in each pair in the vocabulary\n",
    "        s0 = [(tmp_sets[1][i],tmp_sets[0][i]) \n",
    "                  for i in range(dlen) \n",
    "                      if tmp_sets[0][i] in emb_w2i and tmp_sets[1][i] in emb_w2i]\n",
    "        pair_sets.append(s0)\n",
    "        \n",
    "        ## compare the rest to the default\n",
    "        default_set = tmp_sets[0]\n",
    "        for s in tmp_sets[1:]:\n",
    "            dlen = min(len(default_set), len(s))\n",
    "            # if paired, we have to make sure that we have all the words in each pair in the vocabulary\n",
    "            s0 = [(default_set[i],s[i]) \n",
    "                      for i in range(dlen) \n",
    "                          if default_set[i] in emb_w2i and s[i] in emb_w2i]\n",
    "            pair_sets.append(s0)\n",
    "            \n",
    "        \n",
    "        # create for the multi-class metrics\n",
    "        sets = []\n",
    "        for s in tmp_sets:\n",
    "            sets.append([w for w in s if w in emb_w2i])\n",
    "        \n",
    "        \n",
    "        ######### Pairwise metrics setup\n",
    "        for i, pairset in enumerate(pair_sets):\n",
    "            for fun in [ripa_kozlowski_fun, \n",
    "                        ripa_bolukbasi_fun, \n",
    "                        kozlowski_fun, \n",
    "                        bolukbasi_fun, \n",
    "                        caliksan_fun]:\n",
    "                res += [(embedding,measure_ind, names[i], fun.__name__) + x for x in \n",
    "                            fun(pairset, identities,norm_emb_wv, emb_wv, emb_w2i)]\n",
    "\n",
    "        ##### Multiclass\n",
    "        for fun in [garg_fun,\n",
    "                    ripa_fun,\n",
    "                    swinger_fun]:\n",
    "            res += [(embedding,measure_ind) +x for x in \n",
    "                        fun(names,sets, identities, norm_emb_wv, emb_wv, emb_w2i)]\n",
    "\n",
    "    pd.DataFrame(res, columns=['embedding','ind','name',  'fun','identity','value']).to_csv(\"measurement_results/\"+embedding+\".csv\",index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Note ...\n",
    "\n",
    "If you haven't yet pulled down and untarred the embeddings, the next two lines will do that for you!\n",
    "\n",
    "Otherwise, you can skip them!\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "! wget \n",
    "! tar -xzvf embeddings.tgz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_fils = glob('embeddings/*.npy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Note...\n",
    "\n",
    "Running this in parallel, avoid (or use a smaller value for n_jobs) if it doesn't make sense for you to do so!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = Parallel(n_jobs=7)(delayed(run_measures)(f, measures,identities) for f in embedding_fils)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "d = []\n",
    "for m in measures:\n",
    "    meas = {'ind' : m['ind'],\n",
    "            \"name\" : m['group'],\n",
    "            \"len_sets\" : len(m['sets']),\n",
    "           \"paper\" : m['paper'],\n",
    "            \"len_first_set\" : len(m['sets'][0])}\n",
    "    d.append(meas)\n",
    "        \n",
    "pd.DataFrame(d).to_csv(\"measure_info.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "lcss",
   "language": "python",
   "name": "lcss"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}