bag_of_words.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train: (870, 34)\n",
      "X_test: (373, 34)\n",
      "Y_train: (870, 1)\n",
      "Y_test: (373, 1)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from clustering_alg import custom_clustering\n",
    "# import gensim.models\n",
    "import nltk\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize, RegexpTokenizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn import neural_network\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import adjusted_rand_score\n",
    "import gensim\n",
    "from itertools import chain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Comment</th>\n",
       "      <th>Topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0x840</td>\n",
       "      <td>A few things. You might have negative- frequen...</td>\n",
       "      <td>Biology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0xbf0</td>\n",
       "      <td>Is it so hard to believe that there exist part...</td>\n",
       "      <td>Physics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0x1dfc</td>\n",
       "      <td>There are bees</td>\n",
       "      <td>Biology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0xc7e</td>\n",
       "      <td>I'm a medication technician. And that's alot o...</td>\n",
       "      <td>Biology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0xbba</td>\n",
       "      <td>Cesium is such a pretty metal.</td>\n",
       "      <td>Chemistry</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Id                                            Comment      Topic\n",
       "0   0x840  A few things. You might have negative- frequen...    Biology\n",
       "1   0xbf0  Is it so hard to believe that there exist part...    Physics\n",
       "2  0x1dfc                                     There are bees    Biology\n",
       "3   0xc7e  I'm a medication technician. And that's alot o...    Biology\n",
       "4   0xbba                     Cesium is such a pretty metal.  Chemistry"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = pd.read_csv('./train.csv')\n",
    "df_test = pd.read_csv('./test.csv')\n",
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8695, 3)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for comment in df_train.Comment:\n",
    "#     df_train.loc[df_train['Comment']==comment,'Comment'] = comment.replace(\"\\n\",\" \")\n",
    "#     df_train.loc[df_train['Comment']==comment,'Comment'] = comment.replace(\"\\\\n\",\" \")\n",
    "# df_train.iloc[0,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Comment</th>\n",
       "      <th>Topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0x840</td>\n",
       "      <td>A few things. You might have negative- frequen...</td>\n",
       "      <td>Biology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0xbf0</td>\n",
       "      <td>Is it so hard to believe that there exist part...</td>\n",
       "      <td>Physics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0x1dfc</td>\n",
       "      <td>There are bees</td>\n",
       "      <td>Biology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0xc7e</td>\n",
       "      <td>I'm a medication technician. And that's alot o...</td>\n",
       "      <td>Biology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0xbba</td>\n",
       "      <td>Cesium is such a pretty metal.</td>\n",
       "      <td>Chemistry</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8690</th>\n",
       "      <td>0x1e02</td>\n",
       "      <td>I make similar observations over the last week...</td>\n",
       "      <td>Biology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8691</th>\n",
       "      <td>0xc8d</td>\n",
       "      <td>You would know.</td>\n",
       "      <td>Biology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8692</th>\n",
       "      <td>0x723</td>\n",
       "      <td>Also use the correct number of sig figs</td>\n",
       "      <td>Chemistry</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8693</th>\n",
       "      <td>0x667</td>\n",
       "      <td>What about the ethical delimmas,  groundbreaki...</td>\n",
       "      <td>Biology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8694</th>\n",
       "      <td>0x1476</td>\n",
       "      <td>I would like to know too.</td>\n",
       "      <td>Biology</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8695 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          Id                                            Comment      Topic\n",
       "0      0x840  A few things. You might have negative- frequen...    Biology\n",
       "1      0xbf0  Is it so hard to believe that there exist part...    Physics\n",
       "2     0x1dfc                                     There are bees    Biology\n",
       "3      0xc7e  I'm a medication technician. And that's alot o...    Biology\n",
       "4      0xbba                     Cesium is such a pretty metal.  Chemistry\n",
       "...      ...                                                ...        ...\n",
       "8690  0x1e02  I make similar observations over the last week...    Biology\n",
       "8691   0xc8d                                    You would know.    Biology\n",
       "8692   0x723            Also use the correct number of sig figs  Chemistry\n",
       "8693   0x667  What about the ethical delimmas,  groundbreaki...    Biology\n",
       "8694  0x1476                          I would like to know too.    Biology\n",
       "\n",
       "[8695 rows x 3 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'A few things. You might have negative- frequency dependent selection going on where the least common phenotype, reflected by genotype, is going to have an advantage in the environment. For instance, if a prey animal such as a vole were to have a light and a dark phenotype, a predator might recognize the more common phenotype as food.  So if the light voles are more common, foxes may be keeping a closer eye out for light phenotypic voles, recognising them as good prey. This would reduce the light causing alleles due to increased predation and the dark genotypes would increase their proportion of the population until this scenario is reversed. This cycle continues perpetually. \\\\n\\\\nHowever, this is unlikely to be strictly yearly as it usually takes more time than a year for an entire populations allele frequencies to change enough to make a large enough difference to alter fitness. \\\\n\\\\nMore likely on a *year to year* basis, the population is experiencing fluctuating selection where alternating conditions in the environment favor one genotype over another. Perhaps a plant species is living in an area that is flooded every other year and the two phenotypes in the population are plants that do much better in the dryer year and one that does better in the wet year. If there is no flooding, the dry-type genotype will have more fitness leading to more offspring and therefore more dry alleles in the population, however, in flooded years the wet-liking phenotype will do better and propagate the wet genes.'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train['Comment'].tolist()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a', 'few', 'things', '.']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comments_list = df_train['Comment'].tolist()\n",
    "w2v_data = list()\n",
    "for comment in comments_list:\n",
    "    for sentence in sent_tokenize(comment):\n",
    "        temp = []\n",
    "        for word in word_tokenize(sentence):\n",
    "            temp.append(word.lower())\n",
    "        w2v_data.append(temp)    \n",
    "w2v_data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Comment</th>\n",
       "      <th>Topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0x840</td>\n",
       "      <td>A few things. You might have negative- frequen...</td>\n",
       "      <td>Biology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0xbf0</td>\n",
       "      <td>Is it so hard to believe that there exist part...</td>\n",
       "      <td>Physics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0x1dfc</td>\n",
       "      <td>There are bees</td>\n",
       "      <td>Biology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0xc7e</td>\n",
       "      <td>I'm a medication technician. And that's alot o...</td>\n",
       "      <td>Biology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0xbba</td>\n",
       "      <td>Cesium is such a pretty metal.</td>\n",
       "      <td>Chemistry</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8690</th>\n",
       "      <td>0x1e02</td>\n",
       "      <td>I make similar observations over the last week...</td>\n",
       "      <td>Biology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8691</th>\n",
       "      <td>0xc8d</td>\n",
       "      <td>You would know.</td>\n",
       "      <td>Biology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8692</th>\n",
       "      <td>0x723</td>\n",
       "      <td>Also use the correct number of sig figs</td>\n",
       "      <td>Chemistry</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8693</th>\n",
       "      <td>0x667</td>\n",
       "      <td>What about the ethical delimmas,  groundbreaki...</td>\n",
       "      <td>Biology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8694</th>\n",
       "      <td>0x1476</td>\n",
       "      <td>I would like to know too.</td>\n",
       "      <td>Biology</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8695 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          Id                                            Comment      Topic\n",
       "0      0x840  A few things. You might have negative- frequen...    Biology\n",
       "1      0xbf0  Is it so hard to believe that there exist part...    Physics\n",
       "2     0x1dfc                                     There are bees    Biology\n",
       "3      0xc7e  I'm a medication technician. And that's alot o...    Biology\n",
       "4      0xbba                     Cesium is such a pretty metal.  Chemistry\n",
       "...      ...                                                ...        ...\n",
       "8690  0x1e02  I make similar observations over the last week...    Biology\n",
       "8691   0xc8d                                    You would know.    Biology\n",
       "8692   0x723            Also use the correct number of sig figs  Chemistry\n",
       "8693   0x667  What about the ethical delimmas,  groundbreaki...    Biology\n",
       "8694  0x1476                          I would like to know too.    Biology\n",
       "\n",
       "[8695 rows x 3 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = gensim.models.Word2Vec(w2v_data, min_count = 1, vector_size = 30, window = 5, hs=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<gensim.models.keyedvectors.KeyedVectors at 0x1c7ce729e40>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pickle\n",
    "\n",
    "model = pickle.load(open(\"w2v.sav\", 'rb'))\n",
    "model.wv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'the': array([-0.8928743 , -1.4065392 ,  2.6949294 ,  0.6247602 , -0.47386536,\n",
       "        -0.8281255 ,  1.7018446 ,  1.4255651 , -1.5988052 ,  0.03083327,\n",
       "         0.74528885,  0.47732657, -1.7000575 , -0.78259325,  0.04934599,\n",
       "        -0.7311268 ,  0.66943   ,  0.45484573,  0.15037026, -0.94900465,\n",
       "         0.1786488 ,  1.8901759 ,  0.31204507,  1.7691251 ,  0.82538706,\n",
       "         0.4895314 , -0.08870589,  0.8328206 ,  0.73379105, -1.2082686 ],\n",
       "       dtype=float32),\n",
       " 'to': array([-3.0631602 ,  0.5675518 ,  3.2337596 , -0.86805797, -0.93946326,\n",
       "         0.21112137,  0.29869556,  1.4037484 , -0.29039633, -0.78236586,\n",
       "         0.3477027 ,  0.53060234, -1.0156095 , -1.0663389 ,  0.919265  ,\n",
       "        -2.9839041 ,  0.37433183,  1.4779383 , -0.5701126 ,  0.31992874,\n",
       "        -1.3170632 , -1.3559316 , -1.8348321 ,  0.22616665,  0.5700524 ,\n",
       "         1.2552073 ,  0.28103167, -0.45927277,  0.36451355, -1.738156  ],\n",
       "       dtype=float32),\n",
       " 'a': array([-1.1015084 , -1.12419   ,  2.5788941 ,  0.21873924, -0.61220497,\n",
       "        -0.7409767 , -0.40277007,  1.4637225 , -1.4750462 ,  0.71336627,\n",
       "         0.6114191 ,  2.2080917 , -0.63819075, -1.3341627 ,  0.8375159 ,\n",
       "        -0.6583576 ,  0.66716796, -1.0464579 , -0.8808398 , -1.0165275 ,\n",
       "         0.19050452,  1.6856227 , -0.9896033 ,  1.8959011 ,  1.3195537 ,\n",
       "         1.1417123 , -0.3935069 ,  0.7067496 , -1.3966857 , -1.7853335 ],\n",
       "       dtype=float32),\n",
       " 'it': array([-0.95158255, -0.74323726,  3.5756376 ,  0.7211971 , -1.8919095 ,\n",
       "         0.7216994 ,  0.32748812,  1.0831764 , -2.279911  ,  1.3384287 ,\n",
       "         0.69379205,  2.5468497 , -0.84516853,  0.6332677 , -0.15382577,\n",
       "        -0.5464685 , -0.48311678,  1.2220925 , -0.8417283 , -2.1055279 ,\n",
       "         1.6350452 , -1.7636468 , -1.2978969 ,  1.9522622 ,  1.878526  ,\n",
       "         1.3461362 ,  0.81239915,  1.26687   , -0.21995157,  0.38969335],\n",
       "       dtype=float32),\n",
       " 'of': array([ 0.0597503 , -1.6291234 ,  2.4425673 ,  0.8242154 ,  2.2580929 ,\n",
       "         0.0525077 ,  0.8795427 ,  1.0614141 , -0.5785525 , -0.14245419,\n",
       "         2.6954608 , -1.0677282 , -2.544181  , -0.99687403, -0.09388059,\n",
       "        -0.711793  ,  1.9792551 ,  0.08274094,  0.5433    ,  1.2885247 ,\n",
       "         0.22283877,  1.0147848 , -0.13807723,  1.4042405 , -0.74512297,\n",
       "         2.3966944 ,  0.85077155, -0.01761653,  0.6880583 , -2.0013492 ],\n",
       "       dtype=float32),\n",
       " 'and': array([-1.1166627 , -0.34918717,  2.4383113 , -0.14192091,  0.19474672,\n",
       "        -0.5092588 ,  1.1453022 ,  0.8409166 , -0.8569095 , -0.79523927,\n",
       "         0.88313586,  0.38903257, -1.1621627 , -0.52096736,  0.35612217,\n",
       "        -1.2958518 ,  0.01134487,  0.5120077 , -0.71239   ,  0.4100643 ,\n",
       "        -0.27078554, -0.24167328, -0.81284016,  1.376228  ,  0.8276323 ,\n",
       "         1.1735095 ,  0.42255896, -0.26107398,  0.4357485 , -1.9622775 ],\n",
       "       dtype=float32),\n",
       " 'i': array([-4.9147816 ,  2.2565553 ,  0.14959605, -1.5576729 , -1.1825714 ,\n",
       "         0.27559465,  0.48331296,  2.3928373 , -1.9236032 ,  2.0548408 ,\n",
       "         2.1995797 ,  1.2093686 , -2.7972124 , -0.17334652, -1.2437781 ,\n",
       "        -1.9658613 , -0.95361066, -1.5909542 , -3.0759811 , -0.09098113,\n",
       "         2.5273266 , -0.46489942, -3.0838726 ,  0.2515588 ,  2.3553069 ,\n",
       "         1.4444622 ,  1.9194763 ,  1.1649308 , -2.452306  ,  1.673573  ],\n",
       "       dtype=float32),\n",
       " 'you': array([-3.3626854 ,  1.7068038 ,  1.410204  , -0.77555096, -0.9052146 ,\n",
       "        -0.59831417,  1.5911678 ,  4.030895  , -1.599383  ,  0.08978802,\n",
       "         0.65101737,  0.14969127, -1.610812  ,  1.5410973 , -0.02695622,\n",
       "        -3.0014293 ,  0.7059931 , -0.96093154, -1.948049  , -1.2152255 ,\n",
       "         0.3773773 , -2.9880633 , -3.6500087 ,  0.98947567, -0.08343642,\n",
       "         2.3925166 , -0.28313324,  0.7394669 ,  1.2741071 ,  1.0416288 ],\n",
       "       dtype=float32),\n",
       " 'is': array([-0.23713447, -1.2479607 ,  2.0480795 , -0.09877115,  0.06506305,\n",
       "         0.8957929 ,  1.7856561 , -1.4002289 , -0.39156228, -0.83891714,\n",
       "         1.5987942 ,  0.14961426, -2.795673  , -0.6359837 , -0.7504267 ,\n",
       "        -1.2318566 ,  1.3986428 , -2.9561205 , -1.8106017 ,  0.709435  ,\n",
       "        -0.1643975 ,  0.46325165,  1.4895204 ,  1.0642395 , -1.3661264 ,\n",
       "         1.0926188 ,  0.92909825,  1.5640416 , -1.7219081 , -1.5369695 ],\n",
       "       dtype=float32),\n",
       " 'that': array([-0.0947404 , -0.01872282,  3.1208305 , -0.39893463, -0.5888263 ,\n",
       "         0.9740287 ,  0.74627614,  1.0316734 , -1.4351419 ,  0.97307265,\n",
       "         1.2881588 ,  1.0268975 , -1.3606938 , -0.19625539,  0.68676716,\n",
       "        -1.2034712 ,  0.39268672,  0.39323112, -0.13000588, -0.90492886,\n",
       "         1.8190813 , -1.012434  , -0.96717715,  2.1277308 ,  1.7670641 ,\n",
       "         0.7843478 ,  0.85365826,  1.0391243 ,  0.5290291 , -0.57735854],\n",
       "       dtype=float32)}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_dict = {}\n",
    "for i in range(10):\n",
    "    word = model.wv.index_to_key[i]\n",
    "    my_dict[word] = model.wv[word]\n",
    "\n",
    "my_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.17089069 0.5621512\n"
     ]
    }
   ],
   "source": [
    "print(model.wv.similarity('cesium','observations'),model.wv.similarity('ride','car'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "words_vecs = pd.DataFrame(model.wv.vectors)\n",
    "# clusters_count = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(200, 100)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reduced_word_vecs = words_vecs.iloc[:200]\n",
    "reduced_word_vecs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "words_labels = custom_clustering(words_vecs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20884"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_labels_list = []\n",
    "for i in range(6961):\n",
    "    sample_labels_list.append(25)\n",
    "for j in range(6961):\n",
    "    sample_labels_list.append(400)\n",
    "for k in range(6962):\n",
    "    sample_labels_list.append(6432)\n",
    "len(sample_labels_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20884"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_words_list = list(chain.from_iterable(w2v_data))\n",
    "all_unique_words = list()\n",
    "seen = set()\n",
    "for word in all_words_list:\n",
    "    if(word not in seen):\n",
    "        all_unique_words.append(word)\n",
    "    seen.add(word)\n",
    "len(all_unique_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "normalized_df = pd.DataFrame(df_train)\n",
    "for i in range(normalized_df.shape[0]):\n",
    "    normalized_df.loc[i,'Comment'] = normalized_df.loc[i,'Comment'].lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Comment</th>\n",
       "      <th>Topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0x840</td>\n",
       "      <td>advantage advantage things. you might hadvanta...</td>\n",
       "      <td>Biology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0xbf0</td>\n",
       "      <td>advantage advantage so hard to believe that th...</td>\n",
       "      <td>Physics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0x1dfc</td>\n",
       "      <td>advantage advantage bees</td>\n",
       "      <td>Biology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0xc7e</td>\n",
       "      <td>advantageadvantage a medadvantagecatadvantageo...</td>\n",
       "      <td>Biology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0xbba</td>\n",
       "      <td>advantage advantage such a pretty metal.</td>\n",
       "      <td>Chemistry</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8690</th>\n",
       "      <td>0x1e02</td>\n",
       "      <td>advantage advantage sadvantagemadvantagelar ob...</td>\n",
       "      <td>Biology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8691</th>\n",
       "      <td>0xc8d</td>\n",
       "      <td>advantage would know.</td>\n",
       "      <td>Biology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8692</th>\n",
       "      <td>0x723</td>\n",
       "      <td>advantage use the correct number of sig figs</td>\n",
       "      <td>Chemistry</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8693</th>\n",
       "      <td>0x667</td>\n",
       "      <td>advantage advantage the ethical delimmas,  gro...</td>\n",
       "      <td>Biology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8694</th>\n",
       "      <td>0x1476</td>\n",
       "      <td>advantage would ladvantageke to know too.</td>\n",
       "      <td>Biology</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8695 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          Id                                            Comment      Topic\n",
       "0      0x840  advantage advantage things. you might hadvanta...    Biology\n",
       "1      0xbf0  advantage advantage so hard to believe that th...    Physics\n",
       "2     0x1dfc                           advantage advantage bees    Biology\n",
       "3      0xc7e  advantageadvantage a medadvantagecatadvantageo...    Biology\n",
       "4      0xbba           advantage advantage such a pretty metal.  Chemistry\n",
       "...      ...                                                ...        ...\n",
       "8690  0x1e02  advantage advantage sadvantagemadvantagelar ob...    Biology\n",
       "8691   0xc8d                              advantage would know.    Biology\n",
       "8692   0x723       advantage use the correct number of sig figs  Chemistry\n",
       "8693   0x667  advantage advantage the ethical delimmas,  gro...    Biology\n",
       "8694  0x1476          advantage would ladvantageke to know too.    Biology\n",
       "\n",
       "[8695 rows x 3 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for comment in normalized_df.Comment:\n",
    "    for word in word_tokenize(comment):\n",
    "        if(word in all_unique_words):\n",
    "            normalized_df.loc[normalized_df['Comment']==comment,'Comment'] = comment.replace(word, all_unique_words[sample_labels_list[all_unique_words.index(word)]])\n",
    "normalized_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#spliting data to train and test sets.\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(df['Comment'], df['ُTopic'], test_size=0.3, random_state=30)\n",
    "\n",
    "X_train = pd.DataFrame(X_train, columns=['Comment'])\n",
    "X_test = pd.DataFrame(X_test, columns=['Comment'])\n",
    "Y_train = pd.DataFrame(Y_train, columns=['Topic'])\n",
    "Y_test = pd.DataFrame(Y_test, columns=['Topic'])\n",
    "print(\"X_train: \" + str(X_train.shape))\n",
    "print(\"X_test: \" + str(X_test.shape))\n",
    "print(\"Y_train: \" + str(Y_train.shape))\n",
    "print(\"Y_test: \" + str(Y_test.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.08860872 0.6494742\n"
     ]
    }
   ],
   "source": [
    "train_docs = X_train['Comment'].tolist()\n",
    "test_docs = X_test['Comment'].tolist()\n",
    "\n",
    "for i in range(len(train_docs)):\n",
    "    train_docs[i] = re.sub('[.,#$@!?\"()]', '', str(train_docs[i]))\n",
    "\n",
    "for i in range(len(test_docs)):\n",
    "    test_docs[i] = re.sub('[.,#$@!?\"()]', '', str(test_docs[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer(max_df=0.99, min_df=0.01)\n",
    "train_Tfidf = vectorizer.fit_transform(train_docs).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_names = vectorizer.get_feature_names()\n",
    "\n",
    "vocab = {}\n",
    "for value, key in enumerate(feature_names):\n",
    "    vocab[key] = value\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer(vocabulary=vocab)\n",
    "test_Tfidf = vectorizer.fit_transform(test_docs).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlp = neural_network.MLPClassifier(hidden_layer_sizes=(128, 128, 32), learning_rate_init=0.001, max_iter=1000, tol=0.00001)\n",
    "mlp.fit(train_Tfidf, np.array(Y_train['Topic'].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted_labels = mlp.predict(test_Tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def purity_score(y_true, y_pred):\n",
    "    contingency_matrix = metrics.cluster.contingency_matrix(y_true, y_pred)\n",
    "    # return purity\n",
    "    return np.sum(np.amax(contingency_matrix, axis=0)) / np.sum(contingency_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "randIndex = adjusted_rand_score(predicted_labels, np.array(Y_test['Topic'].tolist()))\n",
    "print('rand index ', randIndex)\n",
    "purity = purity_score(np.array(Y_test['Topic'].tolist()), predicted_labels)\n",
    "print('purity ', purity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>first</th>\n",
       "      <th>second</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   first  second\n",
       "0      1      12\n",
       "1      2      23\n",
       "2      3      34\n",
       "3      4      45\n",
       "4      5      56"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "lst = [1,2,3,4,5]\n",
    "pd.DataFrame([lst,[12,23,34,45,56]], index=['first','second']).T"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.0 ('.venv': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "76d032ffc56220cd5f5f26e1a8011f0aff343b3dca0b0ed2536b8891b2defaba"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}