notebooks/word2vec_simple.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Very simple word2vec example @ [nlintz's tutoral](https://github.com/nlintz/TensorFlow-Tutorials)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Packages loaded.\n"
     ]
    }
   ],
   "source": [
    "import collections\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline  \n",
    "\n",
    "print (\"Packages loaded.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Configuration\n",
    "batch_size     = 20\n",
    "embedding_size = 2     # This is just for visualization\n",
    "num_sampled    = 15    # Number of negative examples to sample."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentences, we will use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'sentences' is <type 'list'> and length is 12.\n"
     ]
    }
   ],
   "source": [
    "# Sample sentences\n",
    "sentences = [\"the quick brown fox jumped over the lazy dog\",\n",
    "            \"I love cats and dogs\",\n",
    "            \"we all love cats and dogs\",\n",
    "            \"cats and dogs are great\",\n",
    "            \"sung likes cats\",\n",
    "            \"she loves dogs\",\n",
    "            \"cats can be very independent\",\n",
    "            \"cats are great companions when they want to be\",\n",
    "            \"cats are playful\",\n",
    "            \"cats are natural hunters\",\n",
    "            \"It's raining cats and dogs\",\n",
    "            \"dogs and cats love sung\"]\n",
    "# 'sentences' is 'list' \n",
    "print (\"'sentences' is %s and length is %d.\" \n",
    "       % (type(sentences), len(sentences)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# sentences to words and count"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# words: list of all words (just concatenation) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'words' is <type 'list'> and length is 62.\n",
      "['the', 'quick', 'brown', 'fox', 'jumped', 'over', 'the', 'lazy', 'dog', 'I', 'love', 'cats', 'and', 'dogs', 'we', 'all', 'love', 'cats', 'and', 'dogs', 'cats', 'and', 'dogs', 'are', 'great', 'sung', 'likes', 'cats', 'she', 'loves', 'dogs', 'cats', 'can', 'be', 'very', 'independent', 'cats', 'are', 'great', 'companions', 'when', 'they', 'want', 'to', 'be', 'cats', 'are', 'playful', 'cats', 'are', 'natural', 'hunters', \"It's\", 'raining', 'cats', 'and', 'dogs', 'dogs', 'and', 'cats', 'love', 'sung']\n"
     ]
    }
   ],
   "source": [
    "words = \" \".join(sentences).split() \n",
    "print (\"'words' is %s and length is %d.\" % (type(words), len(words)))\n",
    "print (words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# count: list of pairs, each pair consists of 'cats', 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'count' is <type 'list'> and length is 35.\n",
      "Word count of top five is [('cats', 10), ('dogs', 6), ('and', 5), ('are', 4), ('love', 3)]\n",
      "[('cats', 10), ('dogs', 6), ('and', 5), ('are', 4), ('love', 3), ('be', 2), ('sung', 2), ('great', 2), ('the', 2), ('raining', 1), ('all', 1), ('when', 1), ('over', 1), ('we', 1), ('playful', 1), ('want', 1), ('to', 1), ('jumped', 1), ('hunters', 1), ('companions', 1), ('fox', 1), ('very', 1), (\"It's\", 1), ('can', 1), ('brown', 1), ('lazy', 1), ('I', 1), ('independent', 1), ('they', 1), ('natural', 1), ('dog', 1), ('she', 1), ('loves', 1), ('quick', 1), ('likes', 1)]\n"
     ]
    }
   ],
   "source": [
    "count = collections.Counter(words).most_common() \n",
    "print (\"'count' is %s and length is %d.\" % (type(count), len(count)))\n",
    "print ((\"Word count of top five is %s\") % (count[:5]))\n",
    "print (count)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# See what's in the 'words' and 'count'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['the', 'quick', 'brown', 'fox', 'jumped']\n",
      "[('cats', 10), ('dogs', 6), ('and', 5)]\n"
     ]
    }
   ],
   "source": [
    "print (words[0:5])\n",
    "print (count[0:3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build dictionaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'rdic' is <type 'list'> and length is 35.\n",
      "'dic' is <type 'dict'> and length is 35.\n"
     ]
    }
   ],
   "source": [
    "rdic = [i[0] for i in count] #reverse dic, idx -> word\n",
    "dic = {w: i for i, w in enumerate(rdic)} #dic, word -> id\n",
    "voc_size = len(dic) # Number of vocabulary \n",
    "print (\"'rdic' is %s and length is %d.\" % (type(rdic), len(rdic)))\n",
    "print (\"'dic' is %s and length is %d.\" % (type(dic), len(dic)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['cats', 'dogs', 'and', 'are', 'love', 'be', 'sung', 'great', 'the', 'raining', 'all', 'when', 'over', 'we', 'playful', 'want', 'to', 'jumped', 'hunters', 'companions', 'fox', 'very', \"It's\", 'can', 'brown', 'lazy', 'I', 'independent', 'they', 'natural', 'dog', 'she', 'loves', 'quick', 'likes']\n"
     ]
    }
   ],
   "source": [
    "print (rdic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'and': 2, 'raining': 9, 'all': 10, 'love': 4, 'brown': 24, 'when': 11, 'over': 12, 'lazy': 25, 'playful': 14, 'are': 3, 'want': 15, 'sung': 6, 'jumped': 17, 'hunters': 18, 'companions': 19, 'fox': 20, 'to': 16, 'cats': 0, \"It's\": 22, 'dogs': 1, 'she': 31, 'be': 5, 'we': 13, 'very': 21, 'independent': 27, 'they': 28, 'natural': 29, 'great': 7, 'I': 26, 'dog': 30, 'can': 23, 'loves': 32, 'quick': 33, 'the': 8, 'likes': 34}\n"
     ]
    }
   ],
   "source": [
    "print (dic)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# See what's in the 'rdic' and 'rdic'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cats\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "print (rdic[0])\n",
    "print (dic['cats'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Make indexed word data (ordered)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'data' is <type 'list'> and length is 62.\n",
      "Sample data: numbers: [8, 33, 24, 20, 17, 12, 8, 25, 30, 26] / words: ['the', 'quick', 'brown', 'fox', 'jumped', 'over', 'the', 'lazy', 'dog', 'I']\n"
     ]
    }
   ],
   "source": [
    "data = [dic[word] for word in words]\n",
    "print (\"'data' is %s and length is %d.\" % (type(data), len(data)))\n",
    "print('Sample data: numbers: %s / words: %s'% (data[:10], [rdic[t] for t in data[:10]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[8, 33, 24, 20, 17, 12, 8, 25, 30, 26, 4, 0, 2, 1, 13, 10, 4, 0, 2, 1, 0, 2, 1, 3, 7, 6, 34, 0, 31, 32, 1, 0, 23, 5, 21, 27, 0, 3, 7, 19, 11, 28, 15, 16, 5, 0, 3, 14, 0, 3, 29, 18, 22, 9, 0, 2, 1, 1, 2, 0, 4, 6]\n"
     ]
    }
   ],
   "source": [
    "# See what's in the data\n",
    "print (data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Let's make a training data for window size 1 for simplicity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Context pairs: [[[8, 24], 33], [[33, 20], 24], [[24, 17], 20], [[20, 12], 17], [[17, 8], 12], [[12, 25], 8], [[8, 30], 25], [[25, 26], 30], [[30, 4], 26], [[26, 0], 4]]\n"
     ]
    }
   ],
   "source": [
    "# ([the, brown], quick), ([quick, fox], brown), ([brown, jumped], fox),\n",
    "cbow_pairs = [];\n",
    "for i in range(1, len(data)-1) :\n",
    "    cbow_pairs.append([[data[i-1], data[i+1]], data[i]]);\n",
    "print('Context pairs: %s' % (cbow_pairs[:10]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# See type and length of 'cbow_pairs'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'cbow_pairs' is <type 'list'> and length is 60.\n"
     ]
    }
   ],
   "source": [
    "print (\"'cbow_pairs' is %s and length is %d.\" \n",
    "       % (type(cbow_pairs), len(cbow_pairs)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Let's make skip-gram pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'skip_gram_pairs' is <type 'list'> and length is 120.\n",
      "('skip-gram pairs', [[33, 8], [33, 24], [24, 33], [24, 20], [20, 24]])\n"
     ]
    }
   ],
   "source": [
    "# (quick, the), (quick, brown), (brown, quick), (brown, fox), ...\n",
    "skip_gram_pairs = [];\n",
    "for c in cbow_pairs:\n",
    "    skip_gram_pairs.append([c[1], c[0][0]])\n",
    "    skip_gram_pairs.append([c[1], c[0][1]])\n",
    "    \n",
    "print (\"'skip_gram_pairs' is %s and length is %d.\"\n",
    "       % (type(skip_gram_pairs), len(skip_gram_pairs)))\n",
    "print('skip-gram pairs', skip_gram_pairs[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Batches (x, y)', ([3, 8, 7], [[7], [25], [3]]))\n"
     ]
    }
   ],
   "source": [
    "def generate_batch(size):\n",
    "    assert size < len(skip_gram_pairs)\n",
    "    x_data=[]\n",
    "    y_data = []\n",
    "    r = np.random.choice(range(len(skip_gram_pairs)), size, replace=False)\n",
    "    for i in r:\n",
    "        x_data.append(skip_gram_pairs[i][0])  # n dim\n",
    "        y_data.append([skip_gram_pairs[i][1]])  # n, 1 dim\n",
    "    return x_data, y_data\n",
    "\n",
    "# generate_batch test\n",
    "print ('Batches (x, y)', generate_batch(3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Construct network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Network ready\n"
     ]
    }
   ],
   "source": [
    "# Input data\n",
    "train_inputs = tf.placeholder(tf.int32, shape=[batch_size])\n",
    "# need to shape [batch_size, 1] for nn.nce_loss\n",
    "train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])\n",
    "# missing GPU implementation?\n",
    "with tf.device('/cpu:0'):\n",
    "    # Look up embeddings for inputs.\n",
    "    embeddings = tf.Variable(\n",
    "        tf.random_uniform([voc_size, embedding_size], -1.0, 1.0))\n",
    "    embed = tf.nn.embedding_lookup(embeddings, train_inputs) # lookup table\n",
    "\n",
    "# Construct the variables for the NCE loss\n",
    "nce_weights = tf.Variable(\n",
    "    tf.random_uniform([voc_size, embedding_size], -1.0, 1.0))\n",
    "nce_biases = tf.Variable(tf.zeros([voc_size]))\n",
    "\n",
    "# Compute the average NCE loss for the batch.\n",
    "loss = tf.reduce_mean(\n",
    "    tf.nn.nce_loss(nce_weights, nce_biases, embed, train_labels,\n",
    "                 num_sampled, voc_size))\n",
    "\n",
    "# Use the adam optimizer\n",
    "train_op = tf.train.AdamOptimizer(0.01).minimize(loss)\n",
    "print (\"Network ready\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at 0: 18.82132\n",
      "Loss at 500: 3.69854\n",
      "Loss at 1000: 3.22232\n",
      "Loss at 1500: 2.94147\n",
      "Loss at 2000: 2.90176\n",
      "Loss at 2500: 2.71721\n"
     ]
    }
   ],
   "source": [
    "# Launch the graph in a session\n",
    "with tf.Session() as sess:\n",
    "    # Initializing all variables\n",
    "    tf.initialize_all_variables().run()\n",
    "\n",
    "    for step in range(3000):\n",
    "        batch_inputs, batch_labels = generate_batch(batch_size)\n",
    "        _, loss_val = sess.run([train_op, loss],\n",
    "                feed_dict={train_inputs: batch_inputs, train_labels: batch_labels})\n",
    "        if step % 500 == 0:\n",
    "            print(\"Loss at %d: %.5f\" % (step, loss_val)) \n",
    "            # Report the loss\n",
    "\n",
    "    # Final embeddings are ready for you to use. \n",
    "    # Need to normalize for practical use\n",
    "    trained_embeddings = embeddings.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(35, 2)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trained_embeddings.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plot results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/lib/pymodules/python2.7/matplotlib/collections.py:548: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  if self._edgecolors == 'face':\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEACAYAAACqOy3+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XlYVGX7wPHvsO+IuCfJkoIomyiKiJBblktpimmWqNlb\nqW1u7ZGtr0lpvmm9mqnlrqXycykTR4MkFQF3cwFftVSUFJFN4Pn9MTihooIsw8D9uS6uzpx5zjn3\nDHj3nGc7IIQQQgghhBBCCCGEEEIIIYQQQgghhBBCCHHP5gPngH2GDkQIIUT5hAIBSAIXQgij5Iok\ncCGEqFYmhg5ACCHEvZEELoQQRsqsqi/g4eGhjh8/XtWXEUKI2uY48MCdClR5Dfz48eMopWrUz7vv\nvmvwGCoal52dXY2LqSZ+TxKT4eMwhrhqYkyAx93ya2Uk8KXAb0Ar4BQwshLOKcpIKcWkSZPw8fHB\n19eXFStWADB06FA2bNigLxcZGckPP/xAUVERkyZNIigoCD8/P/773/8aKvRa4fLly8yZMwcArVZL\nv379DByRqEsqowllaCWcQ5RTfn4+s2bNolmzZqxYsYIHHniApUuX4uPjw/Lly/Hx8SEyMhIXFxfc\n3NxISEjg66+/Zt68edSrV4+dO3eSl5dHly5d6NWrF66urob+SEbp77//Zvbs2Tz++OOGDkXUQXWy\nEzM8PNzQIZSqPHGZmpry66+/EhcXh6WlJVevXqV+/fo0adIEe3t7tmzZgqWlJTt27MDW1hZnZ2cs\nLS35+eefWbRoEQEBAXTq1ImMjAyOHTtWKTFVl5oU02uvvcbx48dZvHgxkydPJisri8GDB9O6dWuG\nDx+uL5eYmEh4eDjt27end+/enD17tspjq0nfU0k1Ma6aGFNNoUTls7W1Ve7u7mrs2LHK29tbvfzy\ny2rHjh2qSZMm6tlnn1UNGjRQTk5Oys3NTTk6OqpevXoppZR6/PHH1c8//2zg6GuPtLQ01bZtW6WU\nUlqtVjk6OqozZ86ooqIiFRwcrOLi4lR+fr4KDg5WFy5cUEoptWzZMjVq1ChDhi2MAKDullyrfBSK\nqBoajQY3Nzdyc3PJy8ujc+fOxMTEkJ6eTmhoKFeuXGH48OHMnTuXxMRE1q9fD8BDDz3E7NmzefDB\nBzEzM+OPP/6gefPm2NjYGPgTGSfdv7N/toOCgmjWrBkA/v7+pKWl4ejoyIEDB+jRowcAhYWF+jJC\nVESdbEKpDTQaDaGhoWzevJnAwEDeeecdPv30UwIDA+nduzfx8fF4eHiwfft2wsPDSU1NBeCZZ57B\n29ubdu3a4ePjw/PPP09BQYGBP03tYWlpqd82NTXVf7dt2rQhKSmJpKQk9u7dy6ZNm244LiQkpFrj\nLI10whofqYEbqczMTGJjY/noo49YsGAB1tbWeHp6MnToUBo0aMCCBQt46qmnaN68OSkpKfzyyy9s\n2LABOzs7Xn/9dT788ENDf4Rawd7enitXrtz2fY1Gg6enJ+np6SQkJNCpUyeuXbvG0aNH8fb21peL\nj4+vjnBFLSM1cCPWrVs38vLysLa2BuDIkSO8/PLLnD17FgsLC3788UdSUlKYMWMGEye+w5Qpf/Di\ni+vw8wsmMzPTwNHXDs7OzoSEhODj48PkyZPRaDS3lDE3N2fVqlVMmTIFf39/AgIC2LFjxw1l7Ozs\n2LZt2w014HHjxrFw4UIAXF1deeONNwgICKB9+/bs2bOHXr168cADD/D1118Duhp0165d6du3L15e\nXjz//PP6Jp6ff/6Zzp07ExgYSEREBFevXgVg06ZNtG7dmsDAQH788ccq+Y6EcTNwV0DdsnTpcmVt\nXV85OnZU1tb11dy585WnZ3sFaxQoBUpZWg5T06ZNM3SodcLp06fVyJHPq549H1czZsxShYWFpZaz\ns7NTWq1W9e3bV79v3LhxauHChUoppVxdXdVXX32llFLqlVdeUT4+PiorK0ulp6erxo0bK6WU2rp1\nq7KyslKpqamqsLBQ9ezZU61atUqlp6errl27quzsbKWUUp988omaOnWqysnJUS4uLurYsWNKKaUi\nIiJUv379quy7EOWDdGLWLX///TejRj1HTs5WcnL8gD948cVgrK1tgX9u1/PyvDl//qLB4qwrMjIy\naNcuhIsXh1JYGEZ8/Gekpp5ixox/39P5+vfvD4CPjw9Xr17F1tYWW1tbLC0t9XdUQUFB+jH9Q4cO\nJS4uDisrKw4ePEjnzp0B3RyCzp07c+TIEdzc3PDw0E34Gz58uEzsMjLShFKLnDp1CjOzpoBf8Z5W\nWFi0on17P6ys3gT+BvZhY/NfevXqbrhA64h169Zx9Wp7Cgs/BoaQnb2O2bP/c8PIlZLMzMwoKirS\nv87Jybnh/esdpCYmJlhYWOj3m5iY6DtLSzbhKKXQaDQopejZs6e+E/XAgQPMnTv3luvfLi5Rc0kC\nr0VatGhBYeFZYGfxnn3k5//B7NkzeOQRSywsXHBw6El09Fv07NnTkKHWCYWFhShlXmKPOUoV3TZR\ntmjRgoMHD5Kfn8+lS5eIjY0lPT0dHx+fG8rdKdHu3LmTtLQ0ioqKWLFiBaGhoXTq1In4+HgmTJgA\nwNWrVzl69CheXl6kpaVx4sQJAJYuXVqxDyyqnSTwWsTR0ZGlSxdgY/MwDg5tsbYOY968L/Hw8GD1\n6u/Iy8vi8uWzPPfcGEOHavTGjBnDoUOH7lgmIyMD2ISJyb+Bn7CxGcTTT4/CxOTWf3YajYbmzZsT\nERFB27ZtGTJkCO3atSv1vBqN5oaadsntDh06MG7cOLy9vXF3d2fAgAH6UUkzZszAz89P33xiaWnJ\nf//7X/r06UNgYCCNGze+pRO25B2BqJsM2Q9QJ126dEklJyerixcvGjoUo1ZUVKSKiooqdI6jR4+q\n/v2Hqvbtu6t33nlfXbt27ZYyFy5cUC1atCj1+NTUVNW6dWs1ZswY1aZNG9WrVy+Vk5OjwsLC1O7d\nu5VSSqWnpytXV1e1detW5efnpwYMGKB69+6tWrZsqSZPnqyUUmrKlCnK1NRU+fv7q+HDhyullPru\nu+9UUFCQ8vf3V//617/0Hay2trZqwoQJys/PT8XFxakpU6Yob29v5evrqyZOnFih70OUHWXoxJQa\neC3k6OiIn58f9evXN3QoRictLQ1PT09GjBiBj48Po0ePpkOHDrRt25aoqCh9ufDwcPbs2QPohgC+\n9dZb+Pv7ExwczPnz5wGIiopi7dq1rF27BFvbAvLysggJCcHT05O4uDhAt9yyq6sreXl5DBw4kE6d\nOpGYmHhDTEePHmXcuHHs37+fevXqsXr16ltq4fBPzTwlJYUVK1awb98+li9fzpkzZ/jkk0+wtrYm\nKSmJefPm0a/fIJ5+OpL9+48xbNjTmJiYsHjxYgCys7Pp1KkTycnJeHl5sWbNGg4cOMDUqVOJiIio\nqq9e3ANJ4ELc5NixY4wdO5b9+/cTHR3Nrl27SElJYdu2bezbp3v0a8nkmZ2dTXBwMMnJyXTt2lXf\nQVgyyWo0GgoLC/n999+ZMWMG7733HgA//vgjw4cP56+//uL9998nMTHxlsTs5uaGr68vAIGBgaSl\npZUad1hYGC+99BLdu3fH3t4eS0tLvL29OXny5A3lJkx4k02bDqFUQ7KzGzNlyuusXbtWP1vX1NRU\nv7qio6MjVlZWjB49mhkzZujby0XNIAlciJu0aNGCoKAgAJYvX05gYCDt2rXjwIEDpbZ7W1hY0KdP\nH+DOCXbgwIEAtGvXTl8mPj6eJ554AtBNt7+eqEsqbXq+mZkZhYWFAOTm5t5S/v3338fLy4uEhATe\neecdoqOjycnJ4ZVXXmHu3NkUFDQGHgYaoVQjwJRnn31Wf42OHTvi7+/PE088wbZt2/D29ua3334j\nMjKSgICAuyZyOzu7O74vKockcCFuYmtrC0BqairR0dHExsaSkpJCnz59bkmWoJtpeV3JIX03u56I\nS66RAvc2fM/V1VXf1LJq1aob3jt37hw//PADe/fupWPHjhw5cgTQ3QXk5eXh5xcMdEP3LJavMTfv\ng6+vLy+99BKgG864c+dOkpOT8fDwYPbs2UyYMIFBgwZhbm5OUlIS7u7u+uv9+eefDB48+IYYSpuR\nKipfZSTw3sBh4CgwpRLOJ0SNkJmZia2tLQ4ODpw7d46NGzeW63j1z6OxbiskJET/FKWDBw/qm2hK\nKq2te+LEicyZM4d27dpx8eLFG5pqzp49y2OPPYaFhQVmZmb6CTxNmzZlw4YNODtbYG0dDVwDvLl2\n7Wu02i36phalFKGhofj6+rJ06VL96JWNGzcSGRkJwKeffsqsWbP027oRNxAbG8uTTz4JUGq/QHp6\nOoMGDSIoKIigoCB+++03QNdfMGrUKB588EE8PDz05xZ3VtGZmKbAf4AewBlgF7AOuPP4KiFqsOvJ\n0M/Pj4CAALy8vHBxcaFLly53LH99u2QyvV1N9Pr+F154gREjRtCmTRu8vLxo06YNjo6O+nKurq7s\n3bsXgEWLFrFo0SI0Gg3Hjx9n3bp1jBw5knXr1uHu7s6pU6cYMWIEX375Jf/3f//Hpk2bOH/+PIGB\ngSxdupSLFy/SvXt31q1bx4YNG+jbty/du3fn9OnT3HfffSxbtgwAa2trLly4QF5eHra2toSFhfHd\nd9/xwAMPcPz4cUJCQjh58iT3338/48ePJy4ujgMHDlBQUMDWrVtJTU0lKyuLZcuWMW/ePDZu3Khf\nlyUhIQFTU1OGDRvG2LFjeeihh+jYsSMbN24kKyuLqVOnEhkZiaenJy+88AKmpqYV/G2KOwkGSq6L\n+VrxT0kGHYojRE0UHx+vnn76Xyoy8jmVkJCglFLq2LFjys3NrdShhvv371etWrXSDw3NyMhQffv2\nVYsWLVJKKTV//nz12GOPKaWU6tOnj3JyclK5ublq2bJlysTERE2cOFGFhYWp1q1bq5SUFJWXl6cA\n9d577ymllHr33XfVsGHDlFJKOTg4KFvbBsrevqcCM1WvnrMqLCxUXl5eqkOHDkoppVJSUpSZmZnK\nzMxUISEhytnZWe3YsUO1bNlSPf7448rS0lIdPnxY3X///er7779XXbt2Ve7u7srZ2Vn5+voqc3Nz\n5e3trRo1aqS6deumoqKi1EcffaQuXbqklFKqdevW6syZM1X4G6j5qIZhhPehe5DxdaeL9wkhbmPr\n1q306PEYixa1YsGCpnTu3IVWrVoxcOBA5syZg5nZrTfGsbGxRERE6IeGOjk5kZCQwLBhwwDdOibX\nhyY2bNiQbt264evry/Tp07GxscHT0xMTExPc3d1JS0vDwsICU1NTfvnlF/z9/VmyZAm//vorAEqZ\nc/XqZa5ciQNsuHz5KitXrsTd3Z0///yTwMBA7OzsUEqxYMECAgMDsbW1JTY2ljNnzjBu3DjMzc3x\n9PSkRYsWnD17lqKiIrp3745Go2HXrl306NGDuXPncuTIEU6ePMmGDRtITU3FwcEBuLWfQJSuok0o\nZep9uXn8rDx/TtRl7733OTk504GnASgqcsTXdw+rVi287THX1zS5WWn7AAYMGMCqVas4dOgQgYGB\ntG/fnmeeeYaRI0fekBi1Wi0mJiacOHGCQYMGAXDlSgbwKxACLASmERcXh7OzMzNnztQPMTQ1NWX6\n9Ol8/PHH/Pzzz3z11VfUq1fvtjFZWlrSq1cvvvjiC32CPnnyJCkpKYwaNYodO3bwzDPP8M0339z1\nO6yNtFotWq22XMdUNIGfAVxKvHZBVwu/QckELkRdl5eXDziU2ONIbm7+HY/p1q0bAwYM4NVXX6V+\n/fpkZGTQuXNnli1bxvDhw1m8eDFdu3bVl//qq6+YPn06mZmZODg44O/vf8s5i4qKWLlyJUOGDGHJ\nkiWEhoYCFA9R/D+U6gB8g1KpfPPNekxMMvRlQJfAz549S7t27TAzM8Pa2prWrVuzePFiNBoNf/zx\nB//73/9o3ry5/pgvvviCsWPH6sfUd+vWjRkzZuDt7c3999/PL7/8ck/faW1wc+X2+lyBqmQGHAdc\nAQsgGWh9UxnDNiQJUcMsXPidsrHxUPCTghhlY9NcxcTElOG4hapt27bKz89PjRw5Up08eVJ169ZN\n+fr6qh49eqhTp04ppZSKjIxUq1evVkrppuL7+Pjoz3H9vccee0yZmJgoZ2dn1axZM9W9e3dlY2Oj\nJkyYoJo0aaJMTc0UmBS3w1or+JfSaHxVu3bB+nPZ29vfco3c3Fw1cuRI5ePjowICApRWq1VKKbVg\nwQI1fvx4/bF9+/ZV27ZtUykpKcrb21s1b95cubq6lul7qCsoYwtHRT0MHAGOAa+X8r6hvwchapxv\nvvlWtW0bovz8uqrly1dU+/UzMjKUnZ2dys7OVm3btlUXL15UGo1GrVy5Uiml6zRt0uQ+BdOLHwTy\nvIKpytXVt1KuX1RUpA4ePKg+++wzZW3dQFlbj1F2dl1VYGBXlZeXVynXMHbVlcDvxtDfgxDiJu++\n+64yMTFRfn5+ql69eiohIUGZmZnpF++aNWuWcnR0VBqNnQI/BZ7KxKSz6t9/aIWvXVBQoB59dKiy\nsblPaTTuCu5XkKqgUNnadlcLFiyo8DVqA2QxKyHEzbRaLVu2bOHq1askJyfj7+9Pbm4uVlZWN4xb\nf+655+jT5xGsrc9hZ1fAAw/kMHfujApff8GCBWzefIrs7GModRx4vvjHhPx8H9LT0yt8jbpCHqkm\nRB2TmZmJk5MTVlZWHDp0iISEhFvKdO/enUcffZS4uDiuXLnCuXPnaNKkCY0aNarw9ffuPUR2dl/A\nqnjPIOBrIAUzs2WEhq6p8DXuZtu2bVhYWBAcHFzl16pKUgMXoo7p3bs3BQUFeHt788Ybb+iTWMna\nd+vWrfnggw946KGHGDhwIC+++CIXLlyolOv7+7fBxmYdkF28ZzFwATu7Hnz11ad07NixUq5zJ1u3\nbtVP4zdm1bHiTHFzjhDCGKSmpjJ16jTS0y8REdGHp556slIXpyoqKmLo0FHExGzCzKw+Tk6Kbds2\n6h/GXJpPP/0UKysrxo8fzyuvvMLevXvZsmULsbGxzJ8/HwcHB3bt2kVOTg6DBg3SD112dXUlMjKS\nmJgYrl27xsqVK7G0tCQ4OBhTU1MaNmzIrFmzbrtMgiEVf+d3/OKlCUUIoXfmzBnatQshM3MMRUXB\nbN36IWfPnmfy5Fcr7RomJiYsW/YtJ06cICsrCy8vrxuWzC1N165diY6OZvz48ezevZtr165RUFDA\nr7/+SlhYGIMGDcLJyYnCwkJ69OjB/v37adu2LRqNhoYNG5KYmMicOXOYPn06c+fO5bnnnsPe3p5X\nX628z2UI0oQihNBbunQp2dl9KSp6D3ia7OyV/PvfFe+4vJlGo8HDwwM/P7+7Jm/QraGemJjIlStX\nsLKyIjg4mN27dxMXF0eXLl1uWbf94MGD+mNLW4cd7m0Z35pGauBCCL3CwkKKiqxK7LGiqKjQYPFc\nZ25ujpubGwsWLKBz5874+voSGxvLsWPHsLa2Jjo6mt27d+Po6MjIkSNvWLf9duuw1wZSAxdC6D3+\n+ONYWS0D5gCbsbEZzpgxIw0dFgChoaFMnz6dsLAwQkND+eqrr2jXrt09rdtub2/PlStXqiHqqiWd\nmEKIGyQlJTFp0lQyMi4xaNAjvPbaBExMqqeud/XqVSIiIjhz5gyFhYW8/fbbTJkyhcTERJKTk+nd\nuzedOnVi+/btNGjQAA8PD2xsbNi9ezc2Njb4+PjoF9Q6cOAAaWlp9OnTh86dOxMeHs6kSZOIjY3l\n6NGjDBo0CBMTE/7zn/8QEhJSLZ+vPMrSiVkdDDmZSQhhRFatWqXGjBmjf3358mXl6uqqXwd9165d\nKjw8XCmlm00aEhKi8vPz1YULF5Szs7MqKChQO3fuVP7+/iovL09duXJFtWzZUkVHRxvk81QEMhNT\nCGFMfH192bx5M6+99hpxcXH69cFLo9Fo6NOnD+bm5jg7O9OoUSPOnj1LfHy8/pFydnZ29OvXj8LC\nQiZPfht39wACAsLKvWxrTSWdmEKIGqNly5YkJSWxfv163nrrLbp164aZmRlFRUUAtzxU2sLCQr99\nvZPy5rXTlVKsXbuepCQN2dn/BVLp02cwO3ZswdfXt1o+V1WRGrgQosb466+/sLKy4sknn2TixIkk\nJSXh5ubG7t27AVi9erW+rCqlb02j0RASEkJMTAx5eXlkZWWxfv169uxJIjt7HtABiCA3dyRr1qyt\npk9VdaQGLoSoMfbt28ekSZMwMTHBwsKCOXPmkJ2dzejRo3FwcCA8PPyuD41u3749/fv3x9fXl8aN\nG+Pj48PZs3+Tk3MR8ADAzOwi1tYNq/OjVQkZhSKEqDVWrFhBTMwvODs78OabU7C1tSUsLIxHHunL\n9Onzyc5+FTOzVOrV+4H9+3fRuHFjQ4d8W2UZhSIJXAhRK3zyyXTef38e2dkvotHMwNT0JO7urowa\nNYopU6awYcMGVq9ej7OzI6+8Mp6mTZsaOuQ7qo4EPhiIArzQNS7tKaWMJHAhRJWzs2vA1asJwAMA\n2Ng8zsyZD/PMM89U2jWu57LKXNzrdsqSwCvaibkPGABsr+B5hBCiQq5dywWc9K+LiuqTl5fHZ599\nho+PDz4+PsycOZPXX3+d2bNn68tFRUURHR0NwNSpU3F1dcXPz0+/omFaWhqenp6MGDECHx8fTp++\n5bntBlPRBH4Y+KMyAhFCiIqIiBiGtfXTwE7gW8zM1uLi4sKCBQvYuXMnCQkJzJ07lyFDhrBixQr9\ncStXruSJJ57g559/5vDhw9jb25OUlERiYiK//vorAMeOHWPs2LHs378fFxcXw3zAUsgoFCFErTBv\n3izq13+H9eufp1GjBsyatZG4uDgGDhyItbU1oFuZcPv27Zw/f56//vqL8+fP4+TkxH333cfnn3/O\n2rVrycnJwc7ODisrKz766CNSU1MxNTUlLS2NoKAgA3/KG5UlgW8GmpSy/w0gpiwXuX4rAhAeHk54\neHhZDhNCiDKztLRk5sx/M3PmP/vi4+NvmdSj0WgYPHgwq1at4uzZszzxxBP699966y2WLFnCvn37\nWL16NV9//TUbN27kkUceYdKkSXTt2pUmTUpLhxWn1WrLPUO0slritwITkE5MIUQNkpSURGRkJAkJ\nCRQVFREYGEhBQQHr1q3jmWee4eLFi2zfvp3GjRuzefNmJk+eTH5+PgcOHGDMmDH4+vrSr18/+vXr\nR0BAAIMHD6Zfv37VEnt1dGLecL1KPJcQQlRYQEAAkZGRBAUF0alTJ4YMGYK1tTXe3t5kZWXRvHlz\n/Vjwnj170r9/f44fP46vry/r168nJycH4IbJQzVJRaMZAHwBNAAuA0nAwzeVkRq4EKJGSEtL4+GH\nHyYwMJA9e/bQpk0bFi1axMGDB5kwYQKXLl3iyJEjHD58mBkzZvDTTz+xcuVKGjVqRIcOHdi5cyeN\nGjWqllhlIo8QQpSQlpaGu7s78fHxBAcHM3r0aLy8vFizZg1r166lQYMGdO7cmd9/34mpaTOKipwo\nKjpAixYuTJs2jcGDB1dbrJLAhRCihLS0NMLCwjh58iQAW7du5cMPP2TXrl24u7sDkJ6ezl9/5VJU\ntBmwAE7TvPnLnDp1qFpjre42cCGEqPFKtmMrpXBwcKBNmzYkJSWRlJTE008/TVGRDTAU6AN8SkbG\neUOFe0eSwIUQdcr//vc/EhISAFiyZAmdOnUiPT1dvy8hYS8QDBwCjgEWNGt2n6HCvSNJ4EKIOkOj\n0eDp6cmXX36Jt7c3ly9f5sUXX2TVqlVMmTIFf39/fvstFnBD13phBgyhZcvWhg38NmQmphCizmjR\nogWHDt3Yll1YWEirVq3Ytm0bAE88MYoff8wmP18BBVhb/0DHjjVrBuZ1UgMXQtRZH330KdbW9tjb\nO9Gly0NkZGTwn/9Mw939N+zsPLGx8SAoqIDXXpto6FBLJaNQhBB10vr164mIeJns7K1AEywsxtOr\n19/ExCzj2rVrHDp0CHNzc7y8vAwygacso1CkCUUIUSdt3x5PdvbTQHMA8vOnEB8fCoC5ublRPPBY\nmlCEEHWSi0szrK13AkXFe36nSZNmhgyp3KQJRQhRJ+Xm5tK5c0+OHi0EXICt/PJLDB07djR0aIDM\nxBRCiDvKz89n06ZNZGZmEhYWVqMe1iAJXAghjJRMpRdCiFpMErgQQi88PJzExMR7OnblypV4e3vT\nvXv3O5ZzdXUlIyPjnq4hblTnEridnZ2hQxCixtJoNPc85vmbb75h3rx5bNmy5a7XEJWjziVw+eMR\nQresqpeXF8OHD8fb25vBgwfrnz5z3QsvvECHDh1o27at/rm2sbGxDBgwQF9m8+bNDBw4kPfff5/4\n+HhGjRrF5MmTWbhwIePHj9eX69u3L9u3b6+Wz1aXVDSBf4puya4U4AfAscIRlcGAAQNo3749bdu2\nZe7cuYCuZv3WW2/h7+9PcHAw58/rln9MTU0lODgYX19f3nrrreoITwij8McffzB27FgOHjyIg4MD\ns2fPvuH96+tkp6SksG3bNvbv30+3bt04fPgwFy9eBODbb79l9OjRvP3227Rv354lS5Ywbdq0W64l\nFaeqUdEE/jPQBvAD/gBer3BEZTB//nx2797Nrl27+OKLL8jIyCA7O5vg4GCSk5Pp2rWrPrG/9NJL\njB07lr1799KsmXEN0heiKrm4uBAcHAzA8OHDiYuLu+H95cuXExgYSLt27Thw4AAHDx4E4KmnnuK7\n777j0qVLJCQk8PDDNz9FUVSXiibwzZScxnR9TmoVmzlzpr6mffr0aY4ePYqFhQV9+vQBIDAwkLS0\nNAB+++03hg4dCuj+SIUQOjc/2KDk69TUVKKjo4mNjSUlJYU+ffrom1hGjhzJ999/z7Jly4iIiMDE\n5NY0YmZmRlFRkf51bm5uFX6Suqsy28BHARsq8Xyl0mq1bNmyhYSEBJKTk/H39yc3Nxdzc3N9GRMT\nEwoKCqo6FCGM2s0PNujSpQugS+aZmZnY2tri4ODAuXPn2Lhxoz7BN23alGbNmvHBBx8wcuTIUs/t\n6upKcnJsMo73AAARt0lEQVQySilOnTrFzp07q+dD1TFlWcxqM9CklP1vADHF228C+cCS0k5wvQME\ndMOUwsPDyxPjDTIzM3FycsLKyopDhw7p/wBvJyQkhGXLlvHkk0+yePHie76uELXN9QcbjBo1ijZt\n2vD8888TExODRqPBz8+PgIAAvLy8cHFx0Sf364YNG8aFCxfw9PQs9dwhISG4ubnh7e1N69atCQwM\nrI6PZNS0Wi1arbZcx1RGz0IkMAboDpR2n1SpMzHz8/N57LHHSEtLw9PTk8uXL/Puu+/Sr18/MjMz\nAVi9ejXr169n/vz5pKWlMWzYMLKysnj00UeZOXOmvpwQdVVaWhr9+vVj375993T8uHHjCAwMvG0N\n/LqioiLy8vKwtra+p+vUZdUxlb43EA2EARduU8ZgU+kTExNJTEykRYsW9OrVS3rChSiWlpZG//79\n2bt3b7mPDQwMxN7ens2bN9/QdHmzWbNmM3HiZAoLC/D378iGDStp1KhRRcKuU6ojgR8FLIDr06p2\nAC/cVMYgCXz27K+ZNGkq0BuN5ncefbQT338/V5K4ENVg27ZtPPLIU2RnawFXzMwmExJyGK32/wwd\nmtGos4tZ5ebm4ujYgPz8FMADyMbW1pdffvmeTp06VWssQtRFH3/8MW+//TeFhdfHhF/E2tqD7OxL\nBo3LmNTZxawuXbqEiYk1uuQNYIOpaWvOnj1ryLCEqDOaNm2KldVuoLB4z+80aCDzMCpbrUzgjRo1\nomFDZzSaWeiGqWspKEiQnnAhqsmTTz6Jn58ZdnbB2NkNxdZ2BAsXfmnosGqdWtmEAnD06FH69BnC\nsWN7qVevMcuWfUuvXr2qPQ4h6qqCggI2bdrE33//TZcuXXBzczN0SEalzraBl3Tt2rU79pQLIURN\nJAlcCCGMVJ3txBRCiLpAErgQQhgpSeBCCGGkJIELIYSRkgQuhBBGShK4EEIYKUngQghhpCSBCyGE\nkZIELoQQRkoSuBBCGKmKJPD3gRQgGdgCuFRKREIIIcqkImuh2ANXirfHA37AM6WUk7VQhBCinKp6\nLZQrJbbtuP0zMYUQQlQBswoe/yHwFJANyLPKhBCiGt2tCWUz0KSU/W8AMSVevwZ4AiNLKStNKEII\nUU5laUK5Ww28ZxmvtQTYcLs3o6Ki9Nvh4eGEh4eX8bRCCFE3aLVatFptuY6pSCdmS+Bo8fZ4IAhd\nc8rNpAYuhBDlVNVP5FmFrtmkEDgOPA+cL6WcJHAhhCgneaSaEEIYKXmkmhBC1GKSwIUQwkhJAhdC\nCCMlCVwIIYyUJHAhhDBSksCFEMJISQIXQggjJQlcCCGMlCRwIYQwUpLAhRDCSEkCF0IIIyUJXAgh\njJQkcCGEMFKSwIUQwkhJAhdCCCMlCVwIIYxUZSTwCUARUL8SziWEEKKMKprAXdA9+PhkJcQihBCi\nHCqawD8DJldGIEIIIcqnIgn8UeA0sLeSYhFCCFEOZnd5fzPQpJT9bwKvA71K7KuOByQLIYQodrcE\n3vM2+9sCbkBK8evmQCIQBJy/uXBUVJR+Ozw8nPDw8HKGKYQQtZtWq0Wr1ZbrmMqqNacCgUBGKe8p\npVQlXUYIIeoGjUYDd8nRlTUOXDK0EEJUs+pot5YauBBClFN11sCFEKJW27ZtGzt27DB0GDeQBC6E\nEGWwdetWfvvtN0OHcQNpQhFC1GmLFi0iOjoajUaDr68vERERfPDBB+Tn5+Ps7MzixYvJzs4mODgY\nU1NTGjZsyKxZs/jrr7+YOnUqpqamODo6sm3btkqNqyxNKJLAhRB11oEDBxg4cCA7duygfv36/P33\n32g0GurVqwfAvHnzOHz4MNOnT+e9997D3t6eV199FQBfX19++uknmjZtSmZmJg4ODgC4urqyZ88e\n6tevj52dHVlZWfcUW1kS+N3GgQshRK0VGxtLREQE9evr1uJzcnJi3759REREcPbsWfLz83F3d9eX\nL1kZDQkJYcSIEURERDBw4ED9/uLEe8t2VZA2cCFEnaXRaLi5hWD8+PG8+OKL7N27l6+//pqcnJxS\nj50zZw75+fm88847NGnShBkzZlRHyDeQBC6EqLO6devGypUrycjQzUHMyMggMzOTZs2aAbBgwQJ9\nWXt7e65cuaJ/ffz4cX788Uf+/PNPfH19+fLLL/XnqS7ShCKEqLO8vb158803CQsLw9TUlICAAKKi\nohg8eDBOTk5069aNkyd1q2X369ePQYMGsW7dOr744gs+//xz4uLiyMzMxM7OjoKCAo4ePVqt8Usn\nphBClFFeXh7Lly8nPT0dOzs7vv/+ezZv3oyVlRUPPvggUVFRREZGkpiYSP369W+ptZeHdGIKIUQl\nyc/Pp3Pnnhw5YsG1a22AhbRt+wBWVlYcOnSIhISEao9JErgQQpTBqlWrOHLElKtXN6OrGD/B3r0P\n4u3tjaenJ8HBwUD1jkKRBC6EEGVw8eJFCgpa80+rhh+gG0teMlGfOHFCv52ZmVmlMckoFCGEKIOw\nsDBMTFYB8UAm5uaTCAnpgUaj4cyZM/To8RiNG3vQpUtvjh8/Xi0xSSemEEKU0Zo1a3j22Ze5dCmd\n0NAerFgxHwcHB7y8Ajl58lEKC5/CxGQdDRvO5vjxfdja2t7ztWQqvRBCVLHDhw/ToUMfsrKOcT2l\nOjgEsWHD54SEhNzzeWU5WSGEqGK2trYUFFwGrhbvyaewUDfMsKpVJIFHoXsqfVLxT+/KCEgIIYyJ\ni4sLgwYNxMamBzANG5vehIa2w9fXt8qvXZEmlHeBK8BndyknTShCiFqtqKiIRYsWsWfPPtq0acXo\n0aMxM6vYIL+qbgN/F8gCou9SThK4EEKUU3W0gY8HUoBvgHoVPJcQQohyuFsdfzPQpJT9bwJzgKnF\nr99HVxMfXdpJoqKi9Nvh4eGEh4eXM0whhKjdtFotWq22XMdU1jBCVyAG8CnlPWlCEUKIcqrqJpSm\nJbYHAPsqcC4hhBDlVJFu0n8D/oACUoF/VUpEQgghykRmYgohRA0kMzGFEKIWkwQuhBBGShK4EEIY\nKUngQghhpCSBCyGEkZIELoQQRkoSuBBCGClJ4EIIYaQkgQshhJGSBC6EEEZKErgQQhgpSeBCCGGk\nJIELIYSRkgQuhBBGShK4EEIYKUngQghhpCrjqfSHgP3ontAjhBCimlTkkWoPAv0BX+Aa0LBSIhJC\nCFEmFamBPw98jC55A6RXPBwhhBBlVZEE3hLoCiQAWqB9ZQQkhBCibO7WhLIZaFLK/jeLj3UCOgEd\ngBWAe2kniYqK0m+Hh4cTHh5e/kiFEKIW02q1aLXach1TkafSbwQ+AbYVvz4GdAQu3lROnkovhBDl\nVNVPpV8DdCvebgVYcGvyFkIIUUUqMgplfvHPPiAfeLpSIhJCCFEmFWlCKStpQhFCiHKq6iYUIYQQ\nBiQJXAghjJQkcCGEMFKSwIUQwkhJAhdCCCMlCVwIIYyUJHAhhFGJiooiOjra0GHUCJLAhRBGpXh8\ntEASuBDCCHz44Yd4enoSGhrKkSNHAEhOTqZTp074+fkxcOBALl26BMCuXbvw9fUlICCASZMm4ePj\nA8CBAwfo2LEjAQEB+Pn5cezYMYN9nsoiCVwIUaMlJiayfPlyUlJS2LBhA7t27QJgxIgRfPrpp6Sk\npODj48N7770HwMiRI5k7dy5JSUmYmZnpa+xfffUVL730EklJSSQmJtK8eXODfabKIglcCFGj/frr\nrwwcOBArKyvs7e3p378/V69e5dKlS4SGhgK6ZL59+3YuX75MVlYWHTt2BGDYsGFcX8qjc+fOfPTR\nR0ybNo20tDSsrKwM9pkqiyRwIUSNptFouNt6Srd7v+T+oUOHEhMTg7W1NY888ghbt26t1DgNQRK4\nEKJG69q1K2vWrCE3N5crV64QExODra0tTk5OxMXFAfDdd98RHh6Oo6Mj9vb27Ny5E4Bly5bpz3Pi\nxAnc3NwYP348jz76KPv27TPI56lMFVlOVgghqlxAQABDhgzBz8+PRo0aERQUhEajYeHChTz33HNk\nZ2fj4eHBt99+C8A333zDmDFjMDExISwsDEdHRwBWrFjB999/j7m5OU2bNuXNN9805MeqFLKcrBCi\nVrl69Sq2trYAfPLJJ5w7d47PP//cwFGVnywnK4Soc9avX09AQAA+Pj7Ex8czYsQIevR4DBeXNvTr\n9wTnzp0zdIiVpiI18GWAZ/F2PeASEFBKOamBCyEMIjs7m5Yt/Th3biSFhX0xN/8OD4+t7N//O6am\npoYO747KUgOvSBv4EyW2p6NL4EIIUWPs2bOHrKx6FBa+AcC1a9M4dcqNEydO0LJlSwNHV3GV0Ymp\nASKAByvhXEIIUWlsbGwoLPwbuAaYAzkUFmZhbW1t4MgqR2W0gYcC54DjlXAuIYSoNP7+/nTo4IW1\ndX/gC2xsetO/f99aMQsT7l4D3ww0KWX/G0BM8fZQYMmdThIVFaXfDg8PJzw8vMwBCiHEvTIxMeGn\nn37gyy9ns3//ETp0eJIxY54xdFil0mq1aLXach1T0WGEZsBpoB3w523KSCemEEKUU3UMI+wBHOL2\nyVsIIUQVqWgCHwIsrYxAhBBClI/MxBRCiBpIZmIKIUQtJglcCCGMlCRwIYQwUpLAhRDCSEkCF0II\nIyUJXAghjJQkcCGEMFKSwIUQwkhJAhdCCCMlCVwIIYyUJHAhhDBSksCFEMJISQIXQggjJQlcCCGM\nlCRwIYQwUhVJ4EHATiAJ2AV0qJSIhBBClElFEvg04G0gAHin+LVRKO+DQ6tLTYxLYiobiansamJc\nNTGmsqhIAv8LcCzergecqXg41aOm/rJqYlwSU9lITGVXE+OqiTGVhVkFjn0NiAOmo/sfQXClRCSE\nEKJM7pbANwNNStn/JvBi8c+PwGBgPtCzUqMTQghxWxV5qHEm4FDiPJf4p0mlpGOARwWuI4QQddFx\n4IGqOvkeIKx4uzu6kShCCCGMQHvgdyAZ2IFuNIoQQgghhBCiJvBDV0vfC6wD7A0bDlAzJyItQxdP\nEpBa/N+aYDxwCNgP/NvAsVwXBZzmn++rt0GjudEEoAiob+hAgPeBFHR3ylsAF8OGA8Cn6P6eUoAf\nKL3vrLoNBg4AhUA7A8cCur/nw8BRYIqBY2EXEFq8PRKYasBYrtMCDxVvPwxsNVwopZoOvGXoIIAH\n0Y1GMi9+3dCAsZT0LvCqoYMohQuwCd3/gGtCAi9ZWRoPzDNUICX05J85KJ8U/xiaF9AKXR4wdAI3\nRTf4wxXdv7tkoHVpBatrLZSWwK/F278Aj1fTde+kJk9E0gARwFJDBwI8D3wMXCt+nW7AWG5WkVFU\nVeUzYLKhgyjhSoltO+CCoQIpYTO6OxTQ9aM1N2As1x0G/jB0EMWC0CXwNHT/7pYBj5ZWsLoS+IES\nAQymZtzGvQZEA/9Dd0v3umHDuUEocA7dMCJDawl0BRLQ3bW0N2g0NxqP7jb8G3T/Eza0R9E16+w1\ndCA3+RDd3/kIakZtt6RRwAZDB1HD3AecKvH6dPG+W1RkJubNbjfp5w10v6Qv0K2dsg7Ir8Tr3ktM\nhpyIdKfvKaZ4eyiwpBpiue5O35MZ4AR0QtdPsAJwrwFxzeGfprj30f3PeLSBY3od6FViX3XdIdzt\nb+rN4p/XgM/RNWMaOiaKY8qn+v7WyxJTTaAMHcCdtEJ322RomSW2NcBlQwVyEzPgLNDM0IEU28g/\n4/1Bd2vnbKBYbscV2GfgGNqiu2tKLf65hu4WuJEBY7rZ/eg6omuCSCAesDJwHDerCW3gndD1o1z3\nOgbuyLze8WUCLEL3yzO0mjoRqTc1q0P1X8B7xdut0N2K1wRNS2y/QvXesZRFTenEbFliezzwnaEC\nKaE3umbVBoYOpBRbgUADx2CGrvnUFbDgDp2Y1eVF4Ejxz0eGDKSEmjoR6VvgWUMHUYI5un/0+4BE\nINyg0fxjEbq25hRgDdDYsOHc4gQ1I4GvQve7SwZWUzPuCI4CJ/lnCOhsw4YDwAB07c456O6ANxo2\nHB5Gly+PUbP654QQQgghhBBCCCGEEEIIIYQQQgghhBBCCCGEEEIIIYSoe/4fiefF9fHNv8MAAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fae01e647d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Show word2vec if dim is 2\n",
    "if trained_embeddings.shape[1] == 2:\n",
    "    labels = rdic[:20] # Show top 20 words\n",
    "    for i, label in enumerate(labels):\n",
    "        x, y = trained_embeddings[i,:]\n",
    "        plt.scatter(x, y)\n",
    "        plt.annotate(label, xy=(x, y), xytext=(5, 2),\n",
    "            textcoords='offset points', ha='right', va='bottom')\n",
    "    plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}